Commit | Line | Data |
---|---|---|
c942fddf | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
133ff0ea JG |
2 | /* |
3 | * Copyright 2013 Red Hat Inc. | |
4 | * | |
f813f219 | 5 | * Authors: Jérôme Glisse <jglisse@redhat.com> |
133ff0ea JG |
6 | */ |
7 | /* | |
8 | * Refer to include/linux/hmm.h for information about heterogeneous memory | |
9 | * management or HMM for short. | |
10 | */ | |
a520110e | 11 | #include <linux/pagewalk.h> |
133ff0ea | 12 | #include <linux/hmm.h> |
858b54da | 13 | #include <linux/init.h> |
da4c3c73 JG |
14 | #include <linux/rmap.h> |
15 | #include <linux/swap.h> | |
133ff0ea JG |
16 | #include <linux/slab.h> |
17 | #include <linux/sched.h> | |
4ef589dc JG |
18 | #include <linux/mmzone.h> |
19 | #include <linux/pagemap.h> | |
da4c3c73 JG |
20 | #include <linux/swapops.h> |
21 | #include <linux/hugetlb.h> | |
4ef589dc | 22 | #include <linux/memremap.h> |
c8a53b2d | 23 | #include <linux/sched/mm.h> |
7b2d55d2 | 24 | #include <linux/jump_label.h> |
55c0ece8 | 25 | #include <linux/dma-mapping.h> |
c0b12405 | 26 | #include <linux/mmu_notifier.h> |
4ef589dc JG |
27 | #include <linux/memory_hotplug.h> |
28 | ||
74eee180 JG |
29 | struct hmm_vma_walk { |
30 | struct hmm_range *range; | |
992de9a8 | 31 | struct dev_pagemap *pgmap; |
74eee180 | 32 | unsigned long last; |
9a4903e4 | 33 | unsigned int flags; |
74eee180 JG |
34 | }; |
35 | ||
2aee09d8 JG |
36 | static int hmm_vma_do_fault(struct mm_walk *walk, unsigned long addr, |
37 | bool write_fault, uint64_t *pfn) | |
74eee180 | 38 | { |
9b1ae605 | 39 | unsigned int flags = FAULT_FLAG_REMOTE; |
74eee180 | 40 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
f88a1e90 | 41 | struct hmm_range *range = hmm_vma_walk->range; |
74eee180 | 42 | struct vm_area_struct *vma = walk->vma; |
50a7ca3c | 43 | vm_fault_t ret; |
74eee180 | 44 | |
6c64f2bb RC |
45 | if (!vma) |
46 | goto err; | |
47 | ||
9a4903e4 CH |
48 | if (write_fault) |
49 | flags |= FAULT_FLAG_WRITE; | |
50 | ||
50a7ca3c | 51 | ret = handle_mm_fault(vma, addr, flags); |
6c64f2bb RC |
52 | if (ret & VM_FAULT_ERROR) |
53 | goto err; | |
74eee180 | 54 | |
73231612 | 55 | return -EBUSY; |
6c64f2bb RC |
56 | |
57 | err: | |
58 | *pfn = range->values[HMM_PFN_ERROR]; | |
59 | return -EFAULT; | |
74eee180 JG |
60 | } |
61 | ||
d28c2c9a RC |
62 | static int hmm_pfns_fill(unsigned long addr, unsigned long end, |
63 | struct hmm_range *range, enum hmm_pfn_value_e value) | |
da4c3c73 | 64 | { |
ff05c0c6 | 65 | uint64_t *pfns = range->pfns; |
da4c3c73 JG |
66 | unsigned long i; |
67 | ||
68 | i = (addr - range->start) >> PAGE_SHIFT; | |
69 | for (; addr < end; addr += PAGE_SIZE, i++) | |
d28c2c9a | 70 | pfns[i] = range->values[value]; |
da4c3c73 JG |
71 | |
72 | return 0; | |
73 | } | |
74 | ||
5504ed29 | 75 | /* |
f8c888a3 | 76 | * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s) |
d2e8d551 | 77 | * @addr: range virtual start address (inclusive) |
5504ed29 | 78 | * @end: range virtual end address (exclusive) |
2aee09d8 JG |
79 | * @fault: should we fault or not ? |
80 | * @write_fault: write fault ? | |
5504ed29 | 81 | * @walk: mm_walk structure |
f8c888a3 | 82 | * Return: -EBUSY after page fault, or page fault error |
5504ed29 JG |
83 | * |
84 | * This function will be called whenever pmd_none() or pte_none() returns true, | |
85 | * or whenever there is no page directory covering the virtual address range. | |
86 | */ | |
f8c888a3 | 87 | static int hmm_vma_fault(unsigned long addr, unsigned long end, |
2aee09d8 JG |
88 | bool fault, bool write_fault, |
89 | struct mm_walk *walk) | |
da4c3c73 | 90 | { |
74eee180 JG |
91 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
92 | struct hmm_range *range = hmm_vma_walk->range; | |
ff05c0c6 | 93 | uint64_t *pfns = range->pfns; |
f8c888a3 | 94 | unsigned long i = (addr - range->start) >> PAGE_SHIFT; |
da4c3c73 | 95 | |
f8c888a3 | 96 | WARN_ON_ONCE(!fault && !write_fault); |
74eee180 | 97 | hmm_vma_walk->last = addr; |
63d5066f | 98 | |
c18ce674 RC |
99 | if (write_fault && walk->vma && !(walk->vma->vm_flags & VM_WRITE)) |
100 | return -EPERM; | |
101 | ||
7f08263d | 102 | for (; addr < end; addr += PAGE_SIZE, i++) { |
f8c888a3 | 103 | int ret; |
da4c3c73 | 104 | |
f8c888a3 CH |
105 | ret = hmm_vma_do_fault(walk, addr, write_fault, &pfns[i]); |
106 | if (ret != -EBUSY) | |
107 | return ret; | |
74eee180 JG |
108 | } |
109 | ||
f8c888a3 | 110 | return -EBUSY; |
2aee09d8 JG |
111 | } |
112 | ||
113 | static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk, | |
114 | uint64_t pfns, uint64_t cpu_flags, | |
115 | bool *fault, bool *write_fault) | |
116 | { | |
f88a1e90 JG |
117 | struct hmm_range *range = hmm_vma_walk->range; |
118 | ||
d45d464b | 119 | if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) |
2aee09d8 JG |
120 | return; |
121 | ||
023a019a JG |
122 | /* |
123 | * So we not only consider the individual per page request we also | |
124 | * consider the default flags requested for the range. The API can | |
d2e8d551 RC |
125 | * be used 2 ways. The first one where the HMM user coalesces |
126 | * multiple page faults into one request and sets flags per pfn for | |
127 | * those faults. The second one where the HMM user wants to pre- | |
023a019a JG |
128 | * fault a range with specific flags. For the latter one it is a |
129 | * waste to have the user pre-fill the pfn arrays with a default | |
130 | * flags value. | |
131 | */ | |
132 | pfns = (pfns & range->pfn_flags_mask) | range->default_flags; | |
133 | ||
2aee09d8 | 134 | /* We aren't ask to do anything ... */ |
f88a1e90 | 135 | if (!(pfns & range->flags[HMM_PFN_VALID])) |
2aee09d8 | 136 | return; |
d2e8d551 | 137 | /* If this is device memory then only fault if explicitly requested */ |
f88a1e90 JG |
138 | if ((cpu_flags & range->flags[HMM_PFN_DEVICE_PRIVATE])) { |
139 | /* Do we fault on device memory ? */ | |
140 | if (pfns & range->flags[HMM_PFN_DEVICE_PRIVATE]) { | |
141 | *write_fault = pfns & range->flags[HMM_PFN_WRITE]; | |
142 | *fault = true; | |
143 | } | |
2aee09d8 JG |
144 | return; |
145 | } | |
f88a1e90 JG |
146 | |
147 | /* If CPU page table is not valid then we need to fault */ | |
148 | *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]); | |
149 | /* Need to write fault ? */ | |
150 | if ((pfns & range->flags[HMM_PFN_WRITE]) && | |
151 | !(cpu_flags & range->flags[HMM_PFN_WRITE])) { | |
152 | *write_fault = true; | |
2aee09d8 JG |
153 | *fault = true; |
154 | } | |
155 | } | |
156 | ||
157 | static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk, | |
158 | const uint64_t *pfns, unsigned long npages, | |
159 | uint64_t cpu_flags, bool *fault, | |
160 | bool *write_fault) | |
161 | { | |
162 | unsigned long i; | |
163 | ||
d45d464b | 164 | if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) { |
2aee09d8 JG |
165 | *fault = *write_fault = false; |
166 | return; | |
167 | } | |
168 | ||
a3e0d41c | 169 | *fault = *write_fault = false; |
2aee09d8 JG |
170 | for (i = 0; i < npages; ++i) { |
171 | hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags, | |
172 | fault, write_fault); | |
a3e0d41c | 173 | if ((*write_fault)) |
2aee09d8 JG |
174 | return; |
175 | } | |
176 | } | |
177 | ||
178 | static int hmm_vma_walk_hole(unsigned long addr, unsigned long end, | |
b7a16c7a | 179 | __always_unused int depth, struct mm_walk *walk) |
2aee09d8 JG |
180 | { |
181 | struct hmm_vma_walk *hmm_vma_walk = walk->private; | |
182 | struct hmm_range *range = hmm_vma_walk->range; | |
183 | bool fault, write_fault; | |
184 | unsigned long i, npages; | |
185 | uint64_t *pfns; | |
186 | ||
187 | i = (addr - range->start) >> PAGE_SHIFT; | |
188 | npages = (end - addr) >> PAGE_SHIFT; | |
189 | pfns = &range->pfns[i]; | |
190 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, | |
191 | 0, &fault, &write_fault); | |
f8c888a3 CH |
192 | if (fault || write_fault) |
193 | return hmm_vma_fault(addr, end, fault, write_fault, walk); | |
194 | hmm_vma_walk->last = addr; | |
195 | return hmm_pfns_fill(addr, end, range, HMM_PFN_NONE); | |
2aee09d8 JG |
196 | } |
197 | ||
f88a1e90 | 198 | static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd) |
2aee09d8 JG |
199 | { |
200 | if (pmd_protnone(pmd)) | |
201 | return 0; | |
f88a1e90 JG |
202 | return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] | |
203 | range->flags[HMM_PFN_WRITE] : | |
204 | range->flags[HMM_PFN_VALID]; | |
da4c3c73 JG |
205 | } |
206 | ||
992de9a8 | 207 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
9d3973d6 CH |
208 | static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, |
209 | unsigned long end, uint64_t *pfns, pmd_t pmd) | |
210 | { | |
53f5c3f4 | 211 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
f88a1e90 | 212 | struct hmm_range *range = hmm_vma_walk->range; |
2aee09d8 | 213 | unsigned long pfn, npages, i; |
2aee09d8 | 214 | bool fault, write_fault; |
f88a1e90 | 215 | uint64_t cpu_flags; |
53f5c3f4 | 216 | |
2aee09d8 | 217 | npages = (end - addr) >> PAGE_SHIFT; |
f88a1e90 | 218 | cpu_flags = pmd_to_hmm_pfn_flags(range, pmd); |
2aee09d8 JG |
219 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags, |
220 | &fault, &write_fault); | |
53f5c3f4 | 221 | |
24cee8ab | 222 | if (fault || write_fault) |
f8c888a3 | 223 | return hmm_vma_fault(addr, end, fault, write_fault, walk); |
53f5c3f4 | 224 | |
309f9a4f | 225 | pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); |
992de9a8 JG |
226 | for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) { |
227 | if (pmd_devmap(pmd)) { | |
228 | hmm_vma_walk->pgmap = get_dev_pagemap(pfn, | |
229 | hmm_vma_walk->pgmap); | |
230 | if (unlikely(!hmm_vma_walk->pgmap)) | |
231 | return -EBUSY; | |
232 | } | |
391aab11 | 233 | pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags; |
992de9a8 JG |
234 | } |
235 | if (hmm_vma_walk->pgmap) { | |
236 | put_dev_pagemap(hmm_vma_walk->pgmap); | |
237 | hmm_vma_walk->pgmap = NULL; | |
238 | } | |
53f5c3f4 JG |
239 | hmm_vma_walk->last = end; |
240 | return 0; | |
241 | } | |
9d3973d6 CH |
242 | #else /* CONFIG_TRANSPARENT_HUGEPAGE */ |
243 | /* stub to allow the code below to compile */ | |
244 | int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr, | |
245 | unsigned long end, uint64_t *pfns, pmd_t pmd); | |
246 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ | |
53f5c3f4 | 247 | |
f88a1e90 | 248 | static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte) |
2aee09d8 | 249 | { |
789c2af8 | 250 | if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte)) |
2aee09d8 | 251 | return 0; |
f88a1e90 JG |
252 | return pte_write(pte) ? range->flags[HMM_PFN_VALID] | |
253 | range->flags[HMM_PFN_WRITE] : | |
254 | range->flags[HMM_PFN_VALID]; | |
2aee09d8 JG |
255 | } |
256 | ||
53f5c3f4 JG |
257 | static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr, |
258 | unsigned long end, pmd_t *pmdp, pte_t *ptep, | |
259 | uint64_t *pfn) | |
260 | { | |
261 | struct hmm_vma_walk *hmm_vma_walk = walk->private; | |
f88a1e90 | 262 | struct hmm_range *range = hmm_vma_walk->range; |
2aee09d8 JG |
263 | bool fault, write_fault; |
264 | uint64_t cpu_flags; | |
53f5c3f4 | 265 | pte_t pte = *ptep; |
f88a1e90 | 266 | uint64_t orig_pfn = *pfn; |
53f5c3f4 | 267 | |
f88a1e90 | 268 | *pfn = range->values[HMM_PFN_NONE]; |
73231612 | 269 | fault = write_fault = false; |
53f5c3f4 JG |
270 | |
271 | if (pte_none(pte)) { | |
73231612 JG |
272 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0, |
273 | &fault, &write_fault); | |
2aee09d8 | 274 | if (fault || write_fault) |
53f5c3f4 JG |
275 | goto fault; |
276 | return 0; | |
277 | } | |
278 | ||
279 | if (!pte_present(pte)) { | |
280 | swp_entry_t entry = pte_to_swp_entry(pte); | |
281 | ||
53f5c3f4 JG |
282 | /* |
283 | * This is a special swap entry, ignore migration, use | |
284 | * device and report anything else as error. | |
285 | */ | |
286 | if (is_device_private_entry(entry)) { | |
f88a1e90 JG |
287 | cpu_flags = range->flags[HMM_PFN_VALID] | |
288 | range->flags[HMM_PFN_DEVICE_PRIVATE]; | |
2aee09d8 | 289 | cpu_flags |= is_write_device_private_entry(entry) ? |
f88a1e90 JG |
290 | range->flags[HMM_PFN_WRITE] : 0; |
291 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, | |
292 | &fault, &write_fault); | |
293 | if (fault || write_fault) | |
294 | goto fault; | |
391aab11 JG |
295 | *pfn = hmm_device_entry_from_pfn(range, |
296 | swp_offset(entry)); | |
f88a1e90 | 297 | *pfn |= cpu_flags; |
53f5c3f4 JG |
298 | return 0; |
299 | } | |
300 | ||
76612d6c JG |
301 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0, &fault, |
302 | &write_fault); | |
303 | if (!fault && !write_fault) | |
53f5c3f4 | 304 | return 0; |
76612d6c JG |
305 | |
306 | if (!non_swap_entry(entry)) | |
307 | goto fault; | |
308 | ||
309 | if (is_migration_entry(entry)) { | |
310 | pte_unmap(ptep); | |
311 | hmm_vma_walk->last = addr; | |
312 | migration_entry_wait(walk->mm, pmdp, addr); | |
313 | return -EBUSY; | |
53f5c3f4 JG |
314 | } |
315 | ||
316 | /* Report error for everything else */ | |
dfdc2207 | 317 | pte_unmap(ptep); |
f88a1e90 | 318 | *pfn = range->values[HMM_PFN_ERROR]; |
53f5c3f4 JG |
319 | return -EFAULT; |
320 | } | |
321 | ||
76612d6c JG |
322 | cpu_flags = pte_to_hmm_pfn_flags(range, pte); |
323 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, &fault, | |
324 | &write_fault); | |
2aee09d8 | 325 | if (fault || write_fault) |
53f5c3f4 JG |
326 | goto fault; |
327 | ||
992de9a8 JG |
328 | if (pte_devmap(pte)) { |
329 | hmm_vma_walk->pgmap = get_dev_pagemap(pte_pfn(pte), | |
330 | hmm_vma_walk->pgmap); | |
dfdc2207 JG |
331 | if (unlikely(!hmm_vma_walk->pgmap)) { |
332 | pte_unmap(ptep); | |
992de9a8 | 333 | return -EBUSY; |
dfdc2207 | 334 | } |
40550627 JG |
335 | } |
336 | ||
337 | /* | |
338 | * Since each architecture defines a struct page for the zero page, just | |
339 | * fall through and treat it like a normal page. | |
340 | */ | |
341 | if (pte_special(pte) && !is_zero_pfn(pte_pfn(pte))) { | |
342 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0, &fault, | |
343 | &write_fault); | |
344 | if (fault || write_fault) { | |
dfdc2207 | 345 | pte_unmap(ptep); |
ac541f25 RC |
346 | return -EFAULT; |
347 | } | |
40550627 JG |
348 | *pfn = range->values[HMM_PFN_SPECIAL]; |
349 | return 0; | |
992de9a8 JG |
350 | } |
351 | ||
391aab11 | 352 | *pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags; |
53f5c3f4 JG |
353 | return 0; |
354 | ||
355 | fault: | |
992de9a8 JG |
356 | if (hmm_vma_walk->pgmap) { |
357 | put_dev_pagemap(hmm_vma_walk->pgmap); | |
358 | hmm_vma_walk->pgmap = NULL; | |
359 | } | |
53f5c3f4 JG |
360 | pte_unmap(ptep); |
361 | /* Fault any virtual address we were asked to fault */ | |
f8c888a3 | 362 | return hmm_vma_fault(addr, end, fault, write_fault, walk); |
53f5c3f4 JG |
363 | } |
364 | ||
da4c3c73 JG |
365 | static int hmm_vma_walk_pmd(pmd_t *pmdp, |
366 | unsigned long start, | |
367 | unsigned long end, | |
368 | struct mm_walk *walk) | |
369 | { | |
74eee180 JG |
370 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
371 | struct hmm_range *range = hmm_vma_walk->range; | |
2288a9a6 JG |
372 | uint64_t *pfns = &range->pfns[(start - range->start) >> PAGE_SHIFT]; |
373 | unsigned long npages = (end - start) >> PAGE_SHIFT; | |
374 | unsigned long addr = start; | |
375 | bool fault, write_fault; | |
da4c3c73 | 376 | pte_t *ptep; |
d08faca0 | 377 | pmd_t pmd; |
da4c3c73 | 378 | |
da4c3c73 | 379 | again: |
d08faca0 JG |
380 | pmd = READ_ONCE(*pmdp); |
381 | if (pmd_none(pmd)) | |
b7a16c7a | 382 | return hmm_vma_walk_hole(start, end, -1, walk); |
da4c3c73 | 383 | |
d08faca0 | 384 | if (thp_migration_supported() && is_pmd_migration_entry(pmd)) { |
d08faca0 JG |
385 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, |
386 | 0, &fault, &write_fault); | |
387 | if (fault || write_fault) { | |
388 | hmm_vma_walk->last = addr; | |
d2e8d551 | 389 | pmd_migration_entry_wait(walk->mm, pmdp); |
73231612 | 390 | return -EBUSY; |
d08faca0 | 391 | } |
7d082987 | 392 | return hmm_pfns_fill(start, end, range, HMM_PFN_NONE); |
2288a9a6 JG |
393 | } |
394 | ||
395 | if (!pmd_present(pmd)) { | |
396 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, 0, &fault, | |
397 | &write_fault); | |
398 | if (fault || write_fault) | |
399 | return -EFAULT; | |
d28c2c9a | 400 | return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); |
2288a9a6 | 401 | } |
da4c3c73 | 402 | |
d08faca0 | 403 | if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) { |
da4c3c73 | 404 | /* |
d2e8d551 | 405 | * No need to take pmd_lock here, even if some other thread |
da4c3c73 JG |
406 | * is splitting the huge pmd we will get that event through |
407 | * mmu_notifier callback. | |
408 | * | |
d2e8d551 | 409 | * So just read pmd value and check again it's a transparent |
da4c3c73 JG |
410 | * huge or device mapping one and compute corresponding pfn |
411 | * values. | |
412 | */ | |
413 | pmd = pmd_read_atomic(pmdp); | |
414 | barrier(); | |
415 | if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd)) | |
416 | goto again; | |
74eee180 | 417 | |
2288a9a6 | 418 | return hmm_vma_handle_pmd(walk, addr, end, pfns, pmd); |
da4c3c73 JG |
419 | } |
420 | ||
d08faca0 | 421 | /* |
d2e8d551 | 422 | * We have handled all the valid cases above ie either none, migration, |
d08faca0 JG |
423 | * huge or transparent huge. At this point either it is a valid pmd |
424 | * entry pointing to pte directory or it is a bad pmd that will not | |
425 | * recover. | |
426 | */ | |
2288a9a6 JG |
427 | if (pmd_bad(pmd)) { |
428 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, 0, &fault, | |
429 | &write_fault); | |
430 | if (fault || write_fault) | |
431 | return -EFAULT; | |
d28c2c9a | 432 | return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); |
2288a9a6 | 433 | } |
da4c3c73 JG |
434 | |
435 | ptep = pte_offset_map(pmdp, addr); | |
2288a9a6 | 436 | for (; addr < end; addr += PAGE_SIZE, ptep++, pfns++) { |
53f5c3f4 | 437 | int r; |
74eee180 | 438 | |
2288a9a6 | 439 | r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, pfns); |
53f5c3f4 | 440 | if (r) { |
dfdc2207 | 441 | /* hmm_vma_handle_pte() did pte_unmap() */ |
53f5c3f4 JG |
442 | hmm_vma_walk->last = addr; |
443 | return r; | |
74eee180 | 444 | } |
da4c3c73 | 445 | } |
992de9a8 JG |
446 | if (hmm_vma_walk->pgmap) { |
447 | /* | |
448 | * We do put_dev_pagemap() here and not in hmm_vma_handle_pte() | |
449 | * so that we can leverage get_dev_pagemap() optimization which | |
450 | * will not re-take a reference on a pgmap if we already have | |
451 | * one. | |
452 | */ | |
453 | put_dev_pagemap(hmm_vma_walk->pgmap); | |
454 | hmm_vma_walk->pgmap = NULL; | |
455 | } | |
da4c3c73 JG |
456 | pte_unmap(ptep - 1); |
457 | ||
53f5c3f4 | 458 | hmm_vma_walk->last = addr; |
da4c3c73 JG |
459 | return 0; |
460 | } | |
461 | ||
f0b3c45c CH |
462 | #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \ |
463 | defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) | |
464 | static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud) | |
465 | { | |
466 | if (!pud_present(pud)) | |
467 | return 0; | |
468 | return pud_write(pud) ? range->flags[HMM_PFN_VALID] | | |
469 | range->flags[HMM_PFN_WRITE] : | |
470 | range->flags[HMM_PFN_VALID]; | |
471 | } | |
472 | ||
473 | static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end, | |
474 | struct mm_walk *walk) | |
992de9a8 JG |
475 | { |
476 | struct hmm_vma_walk *hmm_vma_walk = walk->private; | |
477 | struct hmm_range *range = hmm_vma_walk->range; | |
3afc4236 | 478 | unsigned long addr = start; |
992de9a8 | 479 | pud_t pud; |
3afc4236 SP |
480 | int ret = 0; |
481 | spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma); | |
482 | ||
483 | if (!ptl) | |
484 | return 0; | |
485 | ||
486 | /* Normally we don't want to split the huge page */ | |
487 | walk->action = ACTION_CONTINUE; | |
992de9a8 | 488 | |
992de9a8 | 489 | pud = READ_ONCE(*pudp); |
3afc4236 | 490 | if (pud_none(pud)) { |
05fc1df9 JG |
491 | spin_unlock(ptl); |
492 | return hmm_vma_walk_hole(start, end, -1, walk); | |
3afc4236 | 493 | } |
992de9a8 JG |
494 | |
495 | if (pud_huge(pud) && pud_devmap(pud)) { | |
496 | unsigned long i, npages, pfn; | |
497 | uint64_t *pfns, cpu_flags; | |
498 | bool fault, write_fault; | |
499 | ||
3afc4236 | 500 | if (!pud_present(pud)) { |
05fc1df9 JG |
501 | spin_unlock(ptl); |
502 | return hmm_vma_walk_hole(start, end, -1, walk); | |
3afc4236 | 503 | } |
992de9a8 JG |
504 | |
505 | i = (addr - range->start) >> PAGE_SHIFT; | |
506 | npages = (end - addr) >> PAGE_SHIFT; | |
507 | pfns = &range->pfns[i]; | |
508 | ||
509 | cpu_flags = pud_to_hmm_pfn_flags(range, pud); | |
510 | hmm_range_need_fault(hmm_vma_walk, pfns, npages, | |
511 | cpu_flags, &fault, &write_fault); | |
3afc4236 | 512 | if (fault || write_fault) { |
05fc1df9 | 513 | spin_unlock(ptl); |
f8c888a3 | 514 | return hmm_vma_fault(addr, end, fault, write_fault, |
05fc1df9 | 515 | walk); |
3afc4236 | 516 | } |
992de9a8 | 517 | |
992de9a8 JG |
518 | pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); |
519 | for (i = 0; i < npages; ++i, ++pfn) { | |
520 | hmm_vma_walk->pgmap = get_dev_pagemap(pfn, | |
521 | hmm_vma_walk->pgmap); | |
3afc4236 SP |
522 | if (unlikely(!hmm_vma_walk->pgmap)) { |
523 | ret = -EBUSY; | |
524 | goto out_unlock; | |
525 | } | |
391aab11 JG |
526 | pfns[i] = hmm_device_entry_from_pfn(range, pfn) | |
527 | cpu_flags; | |
992de9a8 JG |
528 | } |
529 | if (hmm_vma_walk->pgmap) { | |
530 | put_dev_pagemap(hmm_vma_walk->pgmap); | |
531 | hmm_vma_walk->pgmap = NULL; | |
532 | } | |
533 | hmm_vma_walk->last = end; | |
3afc4236 | 534 | goto out_unlock; |
992de9a8 JG |
535 | } |
536 | ||
3afc4236 SP |
537 | /* Ask for the PUD to be split */ |
538 | walk->action = ACTION_SUBTREE; | |
992de9a8 | 539 | |
3afc4236 SP |
540 | out_unlock: |
541 | spin_unlock(ptl); | |
542 | return ret; | |
992de9a8 | 543 | } |
f0b3c45c CH |
544 | #else |
545 | #define hmm_vma_walk_pud NULL | |
546 | #endif | |
992de9a8 | 547 | |
251bbe59 | 548 | #ifdef CONFIG_HUGETLB_PAGE |
63d5066f JG |
549 | static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask, |
550 | unsigned long start, unsigned long end, | |
551 | struct mm_walk *walk) | |
552 | { | |
05c23af4 | 553 | unsigned long addr = start, i, pfn; |
63d5066f JG |
554 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
555 | struct hmm_range *range = hmm_vma_walk->range; | |
556 | struct vm_area_struct *vma = walk->vma; | |
63d5066f JG |
557 | uint64_t orig_pfn, cpu_flags; |
558 | bool fault, write_fault; | |
559 | spinlock_t *ptl; | |
560 | pte_t entry; | |
63d5066f | 561 | |
d2e8d551 | 562 | ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte); |
63d5066f JG |
563 | entry = huge_ptep_get(pte); |
564 | ||
7f08263d | 565 | i = (start - range->start) >> PAGE_SHIFT; |
63d5066f JG |
566 | orig_pfn = range->pfns[i]; |
567 | range->pfns[i] = range->values[HMM_PFN_NONE]; | |
568 | cpu_flags = pte_to_hmm_pfn_flags(range, entry); | |
569 | fault = write_fault = false; | |
570 | hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, | |
571 | &fault, &write_fault); | |
572 | if (fault || write_fault) { | |
45050692 | 573 | spin_unlock(ptl); |
f8c888a3 | 574 | return hmm_vma_fault(addr, end, fault, write_fault, walk); |
63d5066f JG |
575 | } |
576 | ||
05c23af4 | 577 | pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT); |
7f08263d | 578 | for (; addr < end; addr += PAGE_SIZE, i++, pfn++) |
391aab11 JG |
579 | range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) | |
580 | cpu_flags; | |
63d5066f | 581 | hmm_vma_walk->last = end; |
63d5066f | 582 | spin_unlock(ptl); |
45050692 | 583 | return 0; |
63d5066f | 584 | } |
251bbe59 CH |
585 | #else |
586 | #define hmm_vma_walk_hugetlb_entry NULL | |
587 | #endif /* CONFIG_HUGETLB_PAGE */ | |
63d5066f | 588 | |
d28c2c9a RC |
589 | static int hmm_vma_walk_test(unsigned long start, unsigned long end, |
590 | struct mm_walk *walk) | |
33cd47dc | 591 | { |
d28c2c9a RC |
592 | struct hmm_vma_walk *hmm_vma_walk = walk->private; |
593 | struct hmm_range *range = hmm_vma_walk->range; | |
594 | struct vm_area_struct *vma = walk->vma; | |
595 | ||
596 | /* | |
c2579c9c JG |
597 | * Skip vma ranges that don't have struct page backing them or map I/O |
598 | * devices directly. | |
599 | * | |
d28c2c9a | 600 | * If the vma does not allow read access, then assume that it does not |
c2579c9c JG |
601 | * allow write access either. HMM does not support architectures that |
602 | * allow write without read. | |
d28c2c9a | 603 | */ |
c2579c9c JG |
604 | if ((vma->vm_flags & (VM_IO | VM_PFNMAP | VM_MIXEDMAP)) || |
605 | !(vma->vm_flags & VM_READ)) { | |
d28c2c9a RC |
606 | bool fault, write_fault; |
607 | ||
608 | /* | |
609 | * Check to see if a fault is requested for any page in the | |
610 | * range. | |
611 | */ | |
612 | hmm_range_need_fault(hmm_vma_walk, range->pfns + | |
613 | ((start - range->start) >> PAGE_SHIFT), | |
614 | (end - start) >> PAGE_SHIFT, | |
615 | 0, &fault, &write_fault); | |
616 | if (fault || write_fault) | |
617 | return -EFAULT; | |
618 | ||
c2579c9c | 619 | hmm_pfns_fill(start, end, range, HMM_PFN_ERROR); |
d28c2c9a RC |
620 | hmm_vma_walk->last = end; |
621 | ||
622 | /* Skip this vma and continue processing the next vma. */ | |
623 | return 1; | |
624 | } | |
625 | ||
626 | return 0; | |
33cd47dc JG |
627 | } |
628 | ||
7b86ac33 CH |
629 | static const struct mm_walk_ops hmm_walk_ops = { |
630 | .pud_entry = hmm_vma_walk_pud, | |
631 | .pmd_entry = hmm_vma_walk_pmd, | |
632 | .pte_hole = hmm_vma_walk_hole, | |
633 | .hugetlb_entry = hmm_vma_walk_hugetlb_entry, | |
d28c2c9a | 634 | .test_walk = hmm_vma_walk_test, |
7b86ac33 CH |
635 | }; |
636 | ||
9a4903e4 CH |
637 | /** |
638 | * hmm_range_fault - try to fault some address in a virtual address range | |
639 | * @range: range being faulted | |
640 | * @flags: HMM_FAULT_* flags | |
641 | * | |
642 | * Return: the number of valid pages in range->pfns[] (from range start | |
643 | * address), which may be zero. On error one of the following status codes | |
644 | * can be returned: | |
73231612 | 645 | * |
9a4903e4 CH |
646 | * -EINVAL: Invalid arguments or mm or virtual address is in an invalid vma |
647 | * (e.g., device file vma). | |
648 | * -ENOMEM: Out of memory. | |
649 | * -EPERM: Invalid permission (e.g., asking for write and range is read | |
650 | * only). | |
9a4903e4 CH |
651 | * -EBUSY: The range has been invalidated and the caller needs to wait for |
652 | * the invalidation to finish. | |
653 | * -EFAULT: Invalid (i.e., either no valid vma or it is illegal to access | |
654 | * that range) number of valid pages in range->pfns[] (from | |
655 | * range start address). | |
74eee180 JG |
656 | * |
657 | * This is similar to a regular CPU page fault except that it will not trigger | |
73231612 JG |
658 | * any memory migration if the memory being faulted is not accessible by CPUs |
659 | * and caller does not ask for migration. | |
74eee180 | 660 | * |
ff05c0c6 JG |
661 | * On error, for one virtual address in the range, the function will mark the |
662 | * corresponding HMM pfn entry with an error flag. | |
74eee180 | 663 | */ |
9a4903e4 | 664 | long hmm_range_fault(struct hmm_range *range, unsigned int flags) |
74eee180 | 665 | { |
d28c2c9a RC |
666 | struct hmm_vma_walk hmm_vma_walk = { |
667 | .range = range, | |
668 | .last = range->start, | |
669 | .flags = flags, | |
670 | }; | |
a22dd506 | 671 | struct mm_struct *mm = range->notifier->mm; |
74eee180 JG |
672 | int ret; |
673 | ||
04ec32fb | 674 | lockdep_assert_held(&mm->mmap_sem); |
704f3f2c | 675 | |
a3e0d41c JG |
676 | do { |
677 | /* If range is no longer valid force retry. */ | |
a22dd506 JG |
678 | if (mmu_interval_check_retry(range->notifier, |
679 | range->notifier_seq)) | |
2bcbeaef | 680 | return -EBUSY; |
d28c2c9a RC |
681 | ret = walk_page_range(mm, hmm_vma_walk.last, range->end, |
682 | &hmm_walk_ops, &hmm_vma_walk); | |
683 | } while (ret == -EBUSY); | |
74eee180 | 684 | |
d28c2c9a RC |
685 | if (ret) |
686 | return ret; | |
73231612 | 687 | return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT; |
74eee180 | 688 | } |
73231612 | 689 | EXPORT_SYMBOL(hmm_range_fault); |