Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e6473092 MM |
2 | #include <linux/mm.h> |
3 | #include <linux/highmem.h> | |
4 | #include <linux/sched.h> | |
d33b9f45 | 5 | #include <linux/hugetlb.h> |
e6473092 MM |
6 | |
7 | static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, | |
2165009b | 8 | struct mm_walk *walk) |
e6473092 MM |
9 | { |
10 | pte_t *pte; | |
11 | int err = 0; | |
12 | ||
13 | pte = pte_offset_map(pmd, addr); | |
556637cd | 14 | for (;;) { |
2165009b | 15 | err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk); |
e6473092 MM |
16 | if (err) |
17 | break; | |
556637cd JW |
18 | addr += PAGE_SIZE; |
19 | if (addr == end) | |
20 | break; | |
21 | pte++; | |
22 | } | |
e6473092 MM |
23 | |
24 | pte_unmap(pte); | |
25 | return err; | |
26 | } | |
27 | ||
28 | static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, | |
2165009b | 29 | struct mm_walk *walk) |
e6473092 MM |
30 | { |
31 | pmd_t *pmd; | |
32 | unsigned long next; | |
33 | int err = 0; | |
34 | ||
35 | pmd = pmd_offset(pud, addr); | |
36 | do { | |
03319327 | 37 | again: |
e6473092 | 38 | next = pmd_addr_end(addr, end); |
48684a65 | 39 | if (pmd_none(*pmd) || !walk->vma) { |
e6473092 | 40 | if (walk->pte_hole) |
2165009b | 41 | err = walk->pte_hole(addr, next, walk); |
e6473092 MM |
42 | if (err) |
43 | break; | |
44 | continue; | |
45 | } | |
03319327 DH |
46 | /* |
47 | * This implies that each ->pmd_entry() handler | |
48 | * needs to know about pmd_trans_huge() pmds | |
49 | */ | |
e6473092 | 50 | if (walk->pmd_entry) |
2165009b | 51 | err = walk->pmd_entry(pmd, addr, next, walk); |
03319327 DH |
52 | if (err) |
53 | break; | |
54 | ||
55 | /* | |
56 | * Check this here so we only break down trans_huge | |
57 | * pages when we _need_ to | |
58 | */ | |
59 | if (!walk->pte_entry) | |
60 | continue; | |
61 | ||
78ddc534 | 62 | split_huge_pmd(walk->vma, pmd, addr); |
fafaa426 | 63 | if (pmd_trans_unstable(pmd)) |
03319327 DH |
64 | goto again; |
65 | err = walk_pte_range(pmd, addr, next, walk); | |
e6473092 MM |
66 | if (err) |
67 | break; | |
68 | } while (pmd++, addr = next, addr != end); | |
69 | ||
70 | return err; | |
71 | } | |
72 | ||
c2febafc | 73 | static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, |
2165009b | 74 | struct mm_walk *walk) |
e6473092 MM |
75 | { |
76 | pud_t *pud; | |
77 | unsigned long next; | |
78 | int err = 0; | |
79 | ||
c2febafc | 80 | pud = pud_offset(p4d, addr); |
e6473092 | 81 | do { |
a00cc7d9 | 82 | again: |
e6473092 | 83 | next = pud_addr_end(addr, end); |
a00cc7d9 | 84 | if (pud_none(*pud) || !walk->vma) { |
e6473092 | 85 | if (walk->pte_hole) |
2165009b | 86 | err = walk->pte_hole(addr, next, walk); |
e6473092 MM |
87 | if (err) |
88 | break; | |
89 | continue; | |
90 | } | |
a00cc7d9 MW |
91 | |
92 | if (walk->pud_entry) { | |
93 | spinlock_t *ptl = pud_trans_huge_lock(pud, walk->vma); | |
94 | ||
95 | if (ptl) { | |
96 | err = walk->pud_entry(pud, addr, next, walk); | |
97 | spin_unlock(ptl); | |
98 | if (err) | |
99 | break; | |
100 | continue; | |
101 | } | |
102 | } | |
103 | ||
104 | split_huge_pud(walk->vma, pud, addr); | |
105 | if (pud_none(*pud)) | |
106 | goto again; | |
107 | ||
0b1fbfe5 | 108 | if (walk->pmd_entry || walk->pte_entry) |
2165009b | 109 | err = walk_pmd_range(pud, addr, next, walk); |
e6473092 MM |
110 | if (err) |
111 | break; | |
112 | } while (pud++, addr = next, addr != end); | |
113 | ||
114 | return err; | |
115 | } | |
116 | ||
c2febafc KS |
117 | static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, |
118 | struct mm_walk *walk) | |
119 | { | |
120 | p4d_t *p4d; | |
121 | unsigned long next; | |
122 | int err = 0; | |
123 | ||
124 | p4d = p4d_offset(pgd, addr); | |
125 | do { | |
126 | next = p4d_addr_end(addr, end); | |
127 | if (p4d_none_or_clear_bad(p4d)) { | |
128 | if (walk->pte_hole) | |
129 | err = walk->pte_hole(addr, next, walk); | |
130 | if (err) | |
131 | break; | |
132 | continue; | |
133 | } | |
134 | if (walk->pmd_entry || walk->pte_entry) | |
135 | err = walk_pud_range(p4d, addr, next, walk); | |
136 | if (err) | |
137 | break; | |
138 | } while (p4d++, addr = next, addr != end); | |
139 | ||
140 | return err; | |
141 | } | |
142 | ||
fafaa426 NH |
143 | static int walk_pgd_range(unsigned long addr, unsigned long end, |
144 | struct mm_walk *walk) | |
145 | { | |
146 | pgd_t *pgd; | |
147 | unsigned long next; | |
148 | int err = 0; | |
149 | ||
150 | pgd = pgd_offset(walk->mm, addr); | |
151 | do { | |
152 | next = pgd_addr_end(addr, end); | |
153 | if (pgd_none_or_clear_bad(pgd)) { | |
154 | if (walk->pte_hole) | |
155 | err = walk->pte_hole(addr, next, walk); | |
156 | if (err) | |
157 | break; | |
158 | continue; | |
159 | } | |
160 | if (walk->pmd_entry || walk->pte_entry) | |
c2febafc | 161 | err = walk_p4d_range(pgd, addr, next, walk); |
fafaa426 NH |
162 | if (err) |
163 | break; | |
164 | } while (pgd++, addr = next, addr != end); | |
165 | ||
166 | return err; | |
167 | } | |
168 | ||
116354d1 NH |
169 | #ifdef CONFIG_HUGETLB_PAGE |
170 | static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr, | |
171 | unsigned long end) | |
172 | { | |
173 | unsigned long boundary = (addr & huge_page_mask(h)) + huge_page_size(h); | |
174 | return boundary < end ? boundary : end; | |
175 | } | |
176 | ||
fafaa426 | 177 | static int walk_hugetlb_range(unsigned long addr, unsigned long end, |
116354d1 NH |
178 | struct mm_walk *walk) |
179 | { | |
fafaa426 | 180 | struct vm_area_struct *vma = walk->vma; |
116354d1 NH |
181 | struct hstate *h = hstate_vma(vma); |
182 | unsigned long next; | |
183 | unsigned long hmask = huge_page_mask(h); | |
7868a208 | 184 | unsigned long sz = huge_page_size(h); |
116354d1 NH |
185 | pte_t *pte; |
186 | int err = 0; | |
187 | ||
188 | do { | |
189 | next = hugetlb_entry_end(h, addr, end); | |
7868a208 | 190 | pte = huge_pte_offset(walk->mm, addr & hmask, sz); |
116354d1 NH |
191 | if (pte && walk->hugetlb_entry) |
192 | err = walk->hugetlb_entry(pte, hmask, addr, next, walk); | |
193 | if (err) | |
fafaa426 | 194 | break; |
116354d1 NH |
195 | } while (addr = next, addr != end); |
196 | ||
fafaa426 | 197 | return err; |
116354d1 | 198 | } |
6c6d5280 | 199 | |
6c6d5280 | 200 | #else /* CONFIG_HUGETLB_PAGE */ |
fafaa426 | 201 | static int walk_hugetlb_range(unsigned long addr, unsigned long end, |
6c6d5280 KM |
202 | struct mm_walk *walk) |
203 | { | |
204 | return 0; | |
205 | } | |
206 | ||
207 | #endif /* CONFIG_HUGETLB_PAGE */ | |
208 | ||
fafaa426 NH |
209 | /* |
210 | * Decide whether we really walk over the current vma on [@start, @end) | |
211 | * or skip it via the returned value. Return 0 if we do walk over the | |
212 | * current vma, and return 1 if we skip the vma. Negative values means | |
213 | * error, where we abort the current walk. | |
fafaa426 NH |
214 | */ |
215 | static int walk_page_test(unsigned long start, unsigned long end, | |
216 | struct mm_walk *walk) | |
217 | { | |
218 | struct vm_area_struct *vma = walk->vma; | |
6c6d5280 | 219 | |
fafaa426 NH |
220 | if (walk->test_walk) |
221 | return walk->test_walk(start, end, walk); | |
222 | ||
223 | /* | |
48684a65 NH |
224 | * vma(VM_PFNMAP) doesn't have any valid struct pages behind VM_PFNMAP |
225 | * range, so we don't walk over it as we do for normal vmas. However, | |
226 | * Some callers are interested in handling hole range and they don't | |
227 | * want to just ignore any single address range. Such users certainly | |
228 | * define their ->pte_hole() callbacks, so let's delegate them to handle | |
229 | * vma(VM_PFNMAP). | |
fafaa426 | 230 | */ |
48684a65 NH |
231 | if (vma->vm_flags & VM_PFNMAP) { |
232 | int err = 1; | |
233 | if (walk->pte_hole) | |
234 | err = walk->pte_hole(start, end, walk); | |
235 | return err ? err : 1; | |
236 | } | |
fafaa426 NH |
237 | return 0; |
238 | } | |
239 | ||
240 | static int __walk_page_range(unsigned long start, unsigned long end, | |
241 | struct mm_walk *walk) | |
242 | { | |
243 | int err = 0; | |
244 | struct vm_area_struct *vma = walk->vma; | |
245 | ||
246 | if (vma && is_vm_hugetlb_page(vma)) { | |
247 | if (walk->hugetlb_entry) | |
248 | err = walk_hugetlb_range(start, end, walk); | |
249 | } else | |
250 | err = walk_pgd_range(start, end, walk); | |
251 | ||
252 | return err; | |
253 | } | |
116354d1 | 254 | |
e6473092 | 255 | /** |
fafaa426 | 256 | * walk_page_range - walk page table with caller specific callbacks |
e6473092 | 257 | * |
fafaa426 NH |
258 | * Recursively walk the page table tree of the process represented by @walk->mm |
259 | * within the virtual address range [@start, @end). During walking, we can do | |
260 | * some caller-specific works for each entry, by setting up pmd_entry(), | |
261 | * pte_entry(), and/or hugetlb_entry(). If you don't set up for some of these | |
262 | * callbacks, the associated entries/pages are just ignored. | |
263 | * The return values of these callbacks are commonly defined like below: | |
264 | * - 0 : succeeded to handle the current entry, and if you don't reach the | |
265 | * end address yet, continue to walk. | |
266 | * - >0 : succeeded to handle the current entry, and return to the caller | |
267 | * with caller specific value. | |
268 | * - <0 : failed to handle the current entry, and return to the caller | |
269 | * with error code. | |
e6473092 | 270 | * |
fafaa426 NH |
271 | * Before starting to walk page table, some callers want to check whether |
272 | * they really want to walk over the current vma, typically by checking | |
273 | * its vm_flags. walk_page_test() and @walk->test_walk() are used for this | |
274 | * purpose. | |
e6473092 | 275 | * |
fafaa426 NH |
276 | * struct mm_walk keeps current values of some common data like vma and pmd, |
277 | * which are useful for the access from callbacks. If you want to pass some | |
278 | * caller-specific data to callbacks, @walk->private should be helpful. | |
c27fe4c8 | 279 | * |
fafaa426 NH |
280 | * Locking: |
281 | * Callers of walk_page_range() and walk_page_vma() should hold | |
282 | * @walk->mm->mmap_sem, because these function traverse vma list and/or | |
283 | * access to vma's data. | |
e6473092 | 284 | */ |
fafaa426 | 285 | int walk_page_range(unsigned long start, unsigned long end, |
2165009b | 286 | struct mm_walk *walk) |
e6473092 | 287 | { |
e6473092 | 288 | int err = 0; |
fafaa426 NH |
289 | unsigned long next; |
290 | struct vm_area_struct *vma; | |
e6473092 | 291 | |
fafaa426 NH |
292 | if (start >= end) |
293 | return -EINVAL; | |
e6473092 | 294 | |
2165009b DH |
295 | if (!walk->mm) |
296 | return -EINVAL; | |
297 | ||
96dad67f | 298 | VM_BUG_ON_MM(!rwsem_is_locked(&walk->mm->mmap_sem), walk->mm); |
a9ff785e | 299 | |
fafaa426 | 300 | vma = find_vma(walk->mm, start); |
e6473092 | 301 | do { |
fafaa426 NH |
302 | if (!vma) { /* after the last vma */ |
303 | walk->vma = NULL; | |
304 | next = end; | |
305 | } else if (start < vma->vm_start) { /* outside vma */ | |
306 | walk->vma = NULL; | |
307 | next = min(end, vma->vm_start); | |
308 | } else { /* inside vma */ | |
309 | walk->vma = vma; | |
310 | next = min(end, vma->vm_end); | |
311 | vma = vma->vm_next; | |
5f0af70a | 312 | |
fafaa426 | 313 | err = walk_page_test(start, next, walk); |
f6837395 NH |
314 | if (err > 0) { |
315 | /* | |
316 | * positive return values are purely for | |
317 | * controlling the pagewalk, so should never | |
318 | * be passed to the callers. | |
319 | */ | |
320 | err = 0; | |
a9ff785e | 321 | continue; |
f6837395 | 322 | } |
fafaa426 | 323 | if (err < 0) |
e6473092 | 324 | break; |
e6473092 | 325 | } |
fafaa426 NH |
326 | if (walk->vma || walk->pte_hole) |
327 | err = __walk_page_range(start, next, walk); | |
e6473092 MM |
328 | if (err) |
329 | break; | |
fafaa426 | 330 | } while (start = next, start < end); |
e6473092 MM |
331 | return err; |
332 | } | |
900fc5f1 NH |
333 | |
334 | int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk) | |
335 | { | |
336 | int err; | |
337 | ||
338 | if (!walk->mm) | |
339 | return -EINVAL; | |
340 | ||
341 | VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_sem)); | |
342 | VM_BUG_ON(!vma); | |
343 | walk->vma = vma; | |
344 | err = walk_page_test(vma->vm_start, vma->vm_end, walk); | |
345 | if (err > 0) | |
346 | return 0; | |
347 | if (err < 0) | |
348 | return err; | |
349 | return __walk_page_range(vma->vm_start, vma->vm_end, walk); | |
350 | } |