Commit | Line | Data |
---|---|---|
15b244a8 AK |
1 | /* |
2 | * IOMMU helpers in MMU context. | |
3 | * | |
4 | * Copyright (C) 2015 IBM Corp. <aik@ozlabs.ru> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; either version | |
9 | * 2 of the License, or (at your option) any later version. | |
10 | * | |
11 | */ | |
12 | ||
3f07c014 | 13 | #include <linux/sched/signal.h> |
15b244a8 AK |
14 | #include <linux/slab.h> |
15 | #include <linux/rculist.h> | |
16 | #include <linux/vmalloc.h> | |
17 | #include <linux/mutex.h> | |
2e5bbb54 BS |
18 | #include <linux/migrate.h> |
19 | #include <linux/hugetlb.h> | |
20 | #include <linux/swap.h> | |
15b244a8 AK |
21 | #include <asm/mmu_context.h> |
22 | ||
23 | static DEFINE_MUTEX(mem_list_mutex); | |
24 | ||
25 | struct mm_iommu_table_group_mem_t { | |
26 | struct list_head next; | |
27 | struct rcu_head rcu; | |
28 | unsigned long used; | |
29 | atomic64_t mapped; | |
30 | u64 ua; /* userspace address */ | |
31 | u64 entries; /* number of entries in hpas[] */ | |
32 | u64 *hpas; /* vmalloc'ed */ | |
33 | }; | |
34 | ||
35 | static long mm_iommu_adjust_locked_vm(struct mm_struct *mm, | |
36 | unsigned long npages, bool incr) | |
37 | { | |
38 | long ret = 0, locked, lock_limit; | |
39 | ||
40 | if (!npages) | |
41 | return 0; | |
42 | ||
43 | down_write(&mm->mmap_sem); | |
44 | ||
45 | if (incr) { | |
46 | locked = mm->locked_vm + npages; | |
47 | lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; | |
48 | if (locked > lock_limit && !capable(CAP_IPC_LOCK)) | |
49 | ret = -ENOMEM; | |
50 | else | |
51 | mm->locked_vm += npages; | |
52 | } else { | |
53 | if (WARN_ON_ONCE(npages > mm->locked_vm)) | |
54 | npages = mm->locked_vm; | |
55 | mm->locked_vm -= npages; | |
56 | } | |
57 | ||
58 | pr_debug("[%d] RLIMIT_MEMLOCK HASH64 %c%ld %ld/%ld\n", | |
d7baee69 | 59 | current ? current->pid : 0, |
15b244a8 AK |
60 | incr ? '+' : '-', |
61 | npages << PAGE_SHIFT, | |
62 | mm->locked_vm << PAGE_SHIFT, | |
63 | rlimit(RLIMIT_MEMLOCK)); | |
64 | up_write(&mm->mmap_sem); | |
65 | ||
66 | return ret; | |
67 | } | |
68 | ||
d7baee69 | 69 | bool mm_iommu_preregistered(struct mm_struct *mm) |
15b244a8 | 70 | { |
d7baee69 | 71 | return !list_empty(&mm->context.iommu_group_mem_list); |
15b244a8 AK |
72 | } |
73 | EXPORT_SYMBOL_GPL(mm_iommu_preregistered); | |
74 | ||
2e5bbb54 BS |
75 | /* |
76 | * Taken from alloc_migrate_target with changes to remove CMA allocations | |
77 | */ | |
666feb21 | 78 | struct page *new_iommu_non_cma_page(struct page *page, unsigned long private) |
2e5bbb54 BS |
79 | { |
80 | gfp_t gfp_mask = GFP_USER; | |
81 | struct page *new_page; | |
82 | ||
e889e96e | 83 | if (PageCompound(page)) |
2e5bbb54 BS |
84 | return NULL; |
85 | ||
86 | if (PageHighMem(page)) | |
87 | gfp_mask |= __GFP_HIGHMEM; | |
88 | ||
89 | /* | |
90 | * We don't want the allocation to force an OOM if possibe | |
91 | */ | |
92 | new_page = alloc_page(gfp_mask | __GFP_NORETRY | __GFP_NOWARN); | |
93 | return new_page; | |
94 | } | |
95 | ||
96 | static int mm_iommu_move_page_from_cma(struct page *page) | |
97 | { | |
98 | int ret = 0; | |
99 | LIST_HEAD(cma_migrate_pages); | |
100 | ||
101 | /* Ignore huge pages for now */ | |
e889e96e | 102 | if (PageCompound(page)) |
2e5bbb54 BS |
103 | return -EBUSY; |
104 | ||
105 | lru_add_drain(); | |
106 | ret = isolate_lru_page(page); | |
107 | if (ret) | |
108 | return ret; | |
109 | ||
110 | list_add(&page->lru, &cma_migrate_pages); | |
111 | put_page(page); /* Drop the gup reference */ | |
112 | ||
113 | ret = migrate_pages(&cma_migrate_pages, new_iommu_non_cma_page, | |
31025351 | 114 | NULL, 0, MIGRATE_SYNC, MR_CONTIG_RANGE); |
2e5bbb54 BS |
115 | if (ret) { |
116 | if (!list_empty(&cma_migrate_pages)) | |
117 | putback_movable_pages(&cma_migrate_pages); | |
118 | } | |
119 | ||
120 | return 0; | |
121 | } | |
122 | ||
d7baee69 | 123 | long mm_iommu_get(struct mm_struct *mm, unsigned long ua, unsigned long entries, |
15b244a8 AK |
124 | struct mm_iommu_table_group_mem_t **pmem) |
125 | { | |
126 | struct mm_iommu_table_group_mem_t *mem; | |
127 | long i, j, ret = 0, locked_entries = 0; | |
128 | struct page *page = NULL; | |
129 | ||
15b244a8 AK |
130 | mutex_lock(&mem_list_mutex); |
131 | ||
d7baee69 | 132 | list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, |
15b244a8 AK |
133 | next) { |
134 | if ((mem->ua == ua) && (mem->entries == entries)) { | |
135 | ++mem->used; | |
136 | *pmem = mem; | |
137 | goto unlock_exit; | |
138 | } | |
139 | ||
140 | /* Overlap? */ | |
141 | if ((mem->ua < (ua + (entries << PAGE_SHIFT))) && | |
142 | (ua < (mem->ua + | |
143 | (mem->entries << PAGE_SHIFT)))) { | |
144 | ret = -EINVAL; | |
145 | goto unlock_exit; | |
146 | } | |
147 | ||
148 | } | |
149 | ||
d7baee69 | 150 | ret = mm_iommu_adjust_locked_vm(mm, entries, true); |
15b244a8 AK |
151 | if (ret) |
152 | goto unlock_exit; | |
153 | ||
154 | locked_entries = entries; | |
155 | ||
156 | mem = kzalloc(sizeof(*mem), GFP_KERNEL); | |
157 | if (!mem) { | |
158 | ret = -ENOMEM; | |
159 | goto unlock_exit; | |
160 | } | |
161 | ||
fad953ce | 162 | mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0]))); |
15b244a8 AK |
163 | if (!mem->hpas) { |
164 | kfree(mem); | |
165 | ret = -ENOMEM; | |
166 | goto unlock_exit; | |
167 | } | |
168 | ||
169 | for (i = 0; i < entries; ++i) { | |
170 | if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT), | |
171 | 1/* pages */, 1/* iswrite */, &page)) { | |
2e5bbb54 | 172 | ret = -EFAULT; |
15b244a8 | 173 | for (j = 0; j < i; ++j) |
2e5bbb54 BS |
174 | put_page(pfn_to_page(mem->hpas[j] >> |
175 | PAGE_SHIFT)); | |
15b244a8 AK |
176 | vfree(mem->hpas); |
177 | kfree(mem); | |
15b244a8 AK |
178 | goto unlock_exit; |
179 | } | |
2e5bbb54 BS |
180 | /* |
181 | * If we get a page from the CMA zone, since we are going to | |
182 | * be pinning these entries, we might as well move them out | |
183 | * of the CMA zone if possible. NOTE: faulting in + migration | |
184 | * can be expensive. Batching can be considered later | |
185 | */ | |
a05ef161 | 186 | if (is_migrate_cma_page(page)) { |
2e5bbb54 BS |
187 | if (mm_iommu_move_page_from_cma(page)) |
188 | goto populate; | |
189 | if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT), | |
190 | 1/* pages */, 1/* iswrite */, | |
191 | &page)) { | |
192 | ret = -EFAULT; | |
193 | for (j = 0; j < i; ++j) | |
194 | put_page(pfn_to_page(mem->hpas[j] >> | |
195 | PAGE_SHIFT)); | |
196 | vfree(mem->hpas); | |
197 | kfree(mem); | |
198 | goto unlock_exit; | |
199 | } | |
200 | } | |
201 | populate: | |
15b244a8 AK |
202 | mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT; |
203 | } | |
204 | ||
205 | atomic64_set(&mem->mapped, 1); | |
206 | mem->used = 1; | |
207 | mem->ua = ua; | |
208 | mem->entries = entries; | |
209 | *pmem = mem; | |
210 | ||
d7baee69 | 211 | list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list); |
15b244a8 AK |
212 | |
213 | unlock_exit: | |
214 | if (locked_entries && ret) | |
d7baee69 | 215 | mm_iommu_adjust_locked_vm(mm, locked_entries, false); |
15b244a8 AK |
216 | |
217 | mutex_unlock(&mem_list_mutex); | |
218 | ||
219 | return ret; | |
220 | } | |
221 | EXPORT_SYMBOL_GPL(mm_iommu_get); | |
222 | ||
223 | static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem) | |
224 | { | |
225 | long i; | |
226 | struct page *page = NULL; | |
227 | ||
228 | for (i = 0; i < mem->entries; ++i) { | |
229 | if (!mem->hpas[i]) | |
230 | continue; | |
231 | ||
232 | page = pfn_to_page(mem->hpas[i] >> PAGE_SHIFT); | |
233 | if (!page) | |
234 | continue; | |
235 | ||
236 | put_page(page); | |
237 | mem->hpas[i] = 0; | |
238 | } | |
239 | } | |
240 | ||
241 | static void mm_iommu_do_free(struct mm_iommu_table_group_mem_t *mem) | |
242 | { | |
243 | ||
244 | mm_iommu_unpin(mem); | |
245 | vfree(mem->hpas); | |
246 | kfree(mem); | |
247 | } | |
248 | ||
249 | static void mm_iommu_free(struct rcu_head *head) | |
250 | { | |
251 | struct mm_iommu_table_group_mem_t *mem = container_of(head, | |
252 | struct mm_iommu_table_group_mem_t, rcu); | |
253 | ||
254 | mm_iommu_do_free(mem); | |
255 | } | |
256 | ||
257 | static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem) | |
258 | { | |
259 | list_del_rcu(&mem->next); | |
15b244a8 AK |
260 | call_rcu(&mem->rcu, mm_iommu_free); |
261 | } | |
262 | ||
d7baee69 | 263 | long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem) |
15b244a8 AK |
264 | { |
265 | long ret = 0; | |
266 | ||
15b244a8 AK |
267 | mutex_lock(&mem_list_mutex); |
268 | ||
269 | if (mem->used == 0) { | |
270 | ret = -ENOENT; | |
271 | goto unlock_exit; | |
272 | } | |
273 | ||
274 | --mem->used; | |
275 | /* There are still users, exit */ | |
276 | if (mem->used) | |
277 | goto unlock_exit; | |
278 | ||
279 | /* Are there still mappings? */ | |
280 | if (atomic_cmpxchg(&mem->mapped, 1, 0) != 1) { | |
281 | ++mem->used; | |
282 | ret = -EBUSY; | |
283 | goto unlock_exit; | |
284 | } | |
285 | ||
286 | /* @mapped became 0 so now mappings are disabled, release the region */ | |
287 | mm_iommu_release(mem); | |
288 | ||
d7baee69 AK |
289 | mm_iommu_adjust_locked_vm(mm, mem->entries, false); |
290 | ||
15b244a8 AK |
291 | unlock_exit: |
292 | mutex_unlock(&mem_list_mutex); | |
293 | ||
294 | return ret; | |
295 | } | |
296 | EXPORT_SYMBOL_GPL(mm_iommu_put); | |
297 | ||
d7baee69 AK |
298 | struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm, |
299 | unsigned long ua, unsigned long size) | |
15b244a8 AK |
300 | { |
301 | struct mm_iommu_table_group_mem_t *mem, *ret = NULL; | |
302 | ||
d7baee69 | 303 | list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { |
15b244a8 AK |
304 | if ((mem->ua <= ua) && |
305 | (ua + size <= mem->ua + | |
306 | (mem->entries << PAGE_SHIFT))) { | |
307 | ret = mem; | |
308 | break; | |
309 | } | |
310 | } | |
311 | ||
312 | return ret; | |
313 | } | |
314 | EXPORT_SYMBOL_GPL(mm_iommu_lookup); | |
315 | ||
6b5c19c5 AK |
316 | struct mm_iommu_table_group_mem_t *mm_iommu_lookup_rm(struct mm_struct *mm, |
317 | unsigned long ua, unsigned long size) | |
318 | { | |
319 | struct mm_iommu_table_group_mem_t *mem, *ret = NULL; | |
320 | ||
321 | list_for_each_entry_lockless(mem, &mm->context.iommu_group_mem_list, | |
322 | next) { | |
323 | if ((mem->ua <= ua) && | |
324 | (ua + size <= mem->ua + | |
325 | (mem->entries << PAGE_SHIFT))) { | |
326 | ret = mem; | |
327 | break; | |
328 | } | |
329 | } | |
330 | ||
331 | return ret; | |
332 | } | |
333 | EXPORT_SYMBOL_GPL(mm_iommu_lookup_rm); | |
334 | ||
d7baee69 AK |
335 | struct mm_iommu_table_group_mem_t *mm_iommu_find(struct mm_struct *mm, |
336 | unsigned long ua, unsigned long entries) | |
15b244a8 AK |
337 | { |
338 | struct mm_iommu_table_group_mem_t *mem, *ret = NULL; | |
339 | ||
d7baee69 | 340 | list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { |
15b244a8 AK |
341 | if ((mem->ua == ua) && (mem->entries == entries)) { |
342 | ret = mem; | |
343 | break; | |
344 | } | |
345 | } | |
346 | ||
347 | return ret; | |
348 | } | |
349 | EXPORT_SYMBOL_GPL(mm_iommu_find); | |
350 | ||
351 | long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem, | |
352 | unsigned long ua, unsigned long *hpa) | |
353 | { | |
354 | const long entry = (ua - mem->ua) >> PAGE_SHIFT; | |
355 | u64 *va = &mem->hpas[entry]; | |
356 | ||
357 | if (entry >= mem->entries) | |
358 | return -EFAULT; | |
359 | ||
360 | *hpa = *va | (ua & ~PAGE_MASK); | |
361 | ||
362 | return 0; | |
363 | } | |
364 | EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa); | |
365 | ||
6b5c19c5 AK |
366 | long mm_iommu_ua_to_hpa_rm(struct mm_iommu_table_group_mem_t *mem, |
367 | unsigned long ua, unsigned long *hpa) | |
368 | { | |
369 | const long entry = (ua - mem->ua) >> PAGE_SHIFT; | |
370 | void *va = &mem->hpas[entry]; | |
371 | unsigned long *pa; | |
372 | ||
373 | if (entry >= mem->entries) | |
374 | return -EFAULT; | |
375 | ||
376 | pa = (void *) vmalloc_to_phys(va); | |
377 | if (!pa) | |
378 | return -EFAULT; | |
379 | ||
380 | *hpa = *pa | (ua & ~PAGE_MASK); | |
381 | ||
382 | return 0; | |
383 | } | |
384 | EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa_rm); | |
385 | ||
15b244a8 AK |
386 | long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem) |
387 | { | |
388 | if (atomic64_inc_not_zero(&mem->mapped)) | |
389 | return 0; | |
390 | ||
391 | /* Last mm_iommu_put() has been called, no more mappings allowed() */ | |
392 | return -ENXIO; | |
393 | } | |
394 | EXPORT_SYMBOL_GPL(mm_iommu_mapped_inc); | |
395 | ||
396 | void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem) | |
397 | { | |
398 | atomic64_add_unless(&mem->mapped, -1, 1); | |
399 | } | |
400 | EXPORT_SYMBOL_GPL(mm_iommu_mapped_dec); | |
401 | ||
88f54a35 | 402 | void mm_iommu_init(struct mm_struct *mm) |
15b244a8 | 403 | { |
88f54a35 | 404 | INIT_LIST_HEAD_RCU(&mm->context.iommu_group_mem_list); |
15b244a8 | 405 | } |