2 * IOMMU helpers in MMU context.
4 * Copyright (C) 2015 IBM Corp. <aik@ozlabs.ru>
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
13 #include <linux/sched/signal.h>
14 #include <linux/slab.h>
15 #include <linux/rculist.h>
16 #include <linux/vmalloc.h>
17 #include <linux/mutex.h>
18 #include <linux/migrate.h>
19 #include <linux/hugetlb.h>
20 #include <linux/swap.h>
21 #include <asm/mmu_context.h>
22 #include <asm/pte-walk.h>
24 static DEFINE_MUTEX(mem_list_mutex);
26 struct mm_iommu_table_group_mem_t {
27 struct list_head next;
31 unsigned int pageshift;
32 u64 ua; /* userspace address */
33 u64 entries; /* number of entries in hpas[] */
34 u64 *hpas; /* vmalloc'ed */
37 static long mm_iommu_adjust_locked_vm(struct mm_struct *mm,
38 unsigned long npages, bool incr)
40 long ret = 0, locked, lock_limit;
45 down_write(&mm->mmap_sem);
48 locked = mm->locked_vm + npages;
49 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
50 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
53 mm->locked_vm += npages;
55 if (WARN_ON_ONCE(npages > mm->locked_vm))
56 npages = mm->locked_vm;
57 mm->locked_vm -= npages;
60 pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n",
61 current ? current->pid : 0,
64 mm->locked_vm << PAGE_SHIFT,
65 rlimit(RLIMIT_MEMLOCK));
66 up_write(&mm->mmap_sem);
71 bool mm_iommu_preregistered(struct mm_struct *mm)
73 return !list_empty(&mm->context.iommu_group_mem_list);
75 EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
78 * Taken from alloc_migrate_target with changes to remove CMA allocations
80 struct page *new_iommu_non_cma_page(struct page *page, unsigned long private)
82 gfp_t gfp_mask = GFP_USER;
83 struct page *new_page;
85 if (PageCompound(page))
88 if (PageHighMem(page))
89 gfp_mask |= __GFP_HIGHMEM;
92 * We don't want the allocation to force an OOM if possibe
94 new_page = alloc_page(gfp_mask | __GFP_NORETRY | __GFP_NOWARN);
98 static int mm_iommu_move_page_from_cma(struct page *page)
101 LIST_HEAD(cma_migrate_pages);
103 /* Ignore huge pages for now */
104 if (PageCompound(page))
108 ret = isolate_lru_page(page);
112 list_add(&page->lru, &cma_migrate_pages);
113 put_page(page); /* Drop the gup reference */
115 ret = migrate_pages(&cma_migrate_pages, new_iommu_non_cma_page,
116 NULL, 0, MIGRATE_SYNC, MR_CONTIG_RANGE);
118 if (!list_empty(&cma_migrate_pages))
119 putback_movable_pages(&cma_migrate_pages);
125 long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries,
126 struct mm_iommu_table_group_mem_t **pmem)
128 struct mm_iommu_table_group_mem_t *mem;
129 long i, j, ret = 0, locked_entries = 0;
130 unsigned int pageshift;
132 struct page *page = NULL;
134 mutex_lock(&mem_list_mutex);
136 list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list,
138 if ((mem->ua == ua) && (mem->entries == entries)) {
145 if ((mem->ua < (ua + (entries << PAGE_SHIFT))) &&
147 (mem->entries << PAGE_SHIFT)))) {
154 ret = mm_iommu_adjust_locked_vm(mm, entries, true);
158 locked_entries = entries;
160 mem = kzalloc(sizeof(*mem), GFP_KERNEL);
167 * For a starting point for a maximum page size calculation
168 * we use @ua and @entries natural alignment to allow IOMMU pages
169 * smaller than huge pages but still bigger than PAGE_SIZE.
171 mem->pageshift = __ffs(ua | (entries << PAGE_SHIFT));
172 mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0])));
179 for (i = 0; i < entries; ++i) {
180 if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
181 1/* pages */, 1/* iswrite */, &page)) {
183 for (j = 0; j < i; ++j)
184 put_page(pfn_to_page(mem->hpas[j] >>
191 * If we get a page from the CMA zone, since we are going to
192 * be pinning these entries, we might as well move them out
193 * of the CMA zone if possible. NOTE: faulting in + migration
194 * can be expensive. Batching can be considered later
196 if (is_migrate_cma_page(page)) {
197 if (mm_iommu_move_page_from_cma(page))
199 if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
200 1/* pages */, 1/* iswrite */,
203 for (j = 0; j < i; ++j)
204 put_page(pfn_to_page(mem->hpas[j] >>
212 pageshift = PAGE_SHIFT;
213 if (PageCompound(page)) {
215 struct page *head = compound_head(page);
216 unsigned int compshift = compound_order(head);
218 local_irq_save(flags); /* disables as well */
219 pte = find_linux_pte(mm->pgd, ua, NULL, &pageshift);
220 local_irq_restore(flags);
222 /* Double check it is still the same pinned page */
223 if (pte && pte_page(*pte) == head &&
224 pageshift == compshift)
225 pageshift = max_t(unsigned int, pageshift,
228 mem->pageshift = min(mem->pageshift, pageshift);
229 mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
232 atomic64_set(&mem->mapped, 1);
235 mem->entries = entries;
238 list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list);
241 if (locked_entries && ret)
242 mm_iommu_adjust_locked_vm(mm, locked_entries, false);
244 mutex_unlock(&mem_list_mutex);
248 EXPORT_SYMBOL_GPL(mm_iommu_get);
250 static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem)
253 struct page *page = NULL;
255 for (i = 0; i < mem->entries; ++i) {
259 page = pfn_to_page(mem->hpas[i] >> PAGE_SHIFT);
268 static void mm_iommu_do_free(struct mm_iommu_table_group_mem_t *mem)
276 static void mm_iommu_free(struct rcu_head *head)
278 struct mm_iommu_table_group_mem_t *mem = container_of(head,
279 struct mm_iommu_table_group_mem_t, rcu);
281 mm_iommu_do_free(mem);
284 static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem)
286 list_del_rcu(&mem->next);
287 call_rcu(&mem->rcu, mm_iommu_free);
290 long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
294 mutex_lock(&mem_list_mutex);
296 if (mem->used == 0) {
302 /* There are still users, exit */
306 /* Are there still mappings? */
307 if (atomic_cmpxchg(&mem->mapped, 1, 0) != 1) {
313 /* @mapped became 0 so now mappings are disabled, release the region */
314 mm_iommu_release(mem);
316 mm_iommu_adjust_locked_vm(mm, mem->entries, false);
319 mutex_unlock(&mem_list_mutex);
323 EXPORT_SYMBOL_GPL(mm_iommu_put);
325 struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
326 unsigned long ua, unsigned long size)
328 struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
330 list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
331 if ((mem->ua <= ua) &&
332 (ua + size <= mem->ua +
333 (mem->entries << PAGE_SHIFT))) {
341 EXPORT_SYMBOL_GPL(mm_iommu_lookup);
343 struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(struct mm_struct *mm,
344 unsigned long ua, unsigned long size)
346 struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
348 list_for_each_entry_lockless(mem, &mm->context.iommu_group_mem_list,
350 if ((mem->ua <= ua) &&
351 (ua + size <= mem->ua +
352 (mem->entries << PAGE_SHIFT))) {
360 EXPORT_SYMBOL_GPL(mm_iommu_lookup_rm);
362 struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm,
363 unsigned long ua, unsigned long entries)
365 struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
367 list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
368 if ((mem->ua == ua) && (mem->entries == entries)) {
376 EXPORT_SYMBOL_GPL(mm_iommu_find);
378 long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
379 unsigned long ua, unsigned int pageshift, unsigned long *hpa)
381 const long entry = (ua - mem->ua) >> PAGE_SHIFT;
382 u64 *va = &mem->hpas[entry];
384 if (entry >= mem->entries)
387 if (pageshift > mem->pageshift)
390 *hpa = *va | (ua & ~PAGE_MASK);
394 EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa);
396 long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem,
397 unsigned long ua, unsigned int pageshift, unsigned long *hpa)
399 const long entry = (ua - mem->ua) >> PAGE_SHIFT;
400 void *va = &mem->hpas[entry];
403 if (entry >= mem->entries)
406 if (pageshift > mem->pageshift)
409 pa = (void *) vmalloc_to_phys(va);
413 *hpa = *pa | (ua & ~PAGE_MASK);
417 EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa_rm);
419 long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem)
421 if (atomic64_inc_not_zero(&mem->mapped))
424 /* Last mm_iommu_put() has been called, no more mappings allowed() */
427 EXPORT_SYMBOL_GPL(mm_iommu_mapped_inc);
429 void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem)
431 atomic64_add_unless(&mem->mapped, -1, 1);
433 EXPORT_SYMBOL_GPL(mm_iommu_mapped_dec);
435 void mm_iommu_init(struct mm_struct *mm)
437 INIT_LIST_HEAD_RCU(&mm->context.iommu_group_mem_list);