Merge suspend-to-idle rework material for v5.4.
[linux-2.6-block.git] / mm / gup.c
CommitLineData
457c8996 1// SPDX-License-Identifier: GPL-2.0-only
4bbd4c77
KS
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/err.h>
5#include <linux/spinlock.h>
6
4bbd4c77 7#include <linux/mm.h>
3565fce3 8#include <linux/memremap.h>
4bbd4c77
KS
9#include <linux/pagemap.h>
10#include <linux/rmap.h>
11#include <linux/swap.h>
12#include <linux/swapops.h>
13
174cd4b1 14#include <linux/sched/signal.h>
2667f50e 15#include <linux/rwsem.h>
f30c59e9 16#include <linux/hugetlb.h>
9a4e9f3b
AK
17#include <linux/migrate.h>
18#include <linux/mm_inline.h>
19#include <linux/sched/mm.h>
1027e443 20
33a709b2 21#include <asm/mmu_context.h>
2667f50e 22#include <asm/pgtable.h>
1027e443 23#include <asm/tlbflush.h>
2667f50e 24
4bbd4c77
KS
25#include "internal.h"
26
df06b37f
KB
27struct follow_page_context {
28 struct dev_pagemap *pgmap;
29 unsigned int page_mask;
30};
31
fc1d8e7c
JH
32typedef int (*set_dirty_func_t)(struct page *page);
33
34static void __put_user_pages_dirty(struct page **pages,
35 unsigned long npages,
36 set_dirty_func_t sdf)
37{
38 unsigned long index;
39
40 for (index = 0; index < npages; index++) {
41 struct page *page = compound_head(pages[index]);
42
43 /*
44 * Checking PageDirty at this point may race with
45 * clear_page_dirty_for_io(), but that's OK. Two key cases:
46 *
47 * 1) This code sees the page as already dirty, so it skips
48 * the call to sdf(). That could happen because
49 * clear_page_dirty_for_io() called page_mkclean(),
50 * followed by set_page_dirty(). However, now the page is
51 * going to get written back, which meets the original
52 * intention of setting it dirty, so all is well:
53 * clear_page_dirty_for_io() goes on to call
54 * TestClearPageDirty(), and write the page back.
55 *
56 * 2) This code sees the page as clean, so it calls sdf().
57 * The page stays dirty, despite being written back, so it
58 * gets written back again in the next writeback cycle.
59 * This is harmless.
60 */
61 if (!PageDirty(page))
62 sdf(page);
63
64 put_user_page(page);
65 }
66}
67
68/**
69 * put_user_pages_dirty() - release and dirty an array of gup-pinned pages
70 * @pages: array of pages to be marked dirty and released.
71 * @npages: number of pages in the @pages array.
72 *
73 * "gup-pinned page" refers to a page that has had one of the get_user_pages()
74 * variants called on that page.
75 *
76 * For each page in the @pages array, make that page (or its head page, if a
77 * compound page) dirty, if it was previously listed as clean. Then, release
78 * the page using put_user_page().
79 *
80 * Please see the put_user_page() documentation for details.
81 *
82 * set_page_dirty(), which does not lock the page, is used here.
83 * Therefore, it is the caller's responsibility to ensure that this is
84 * safe. If not, then put_user_pages_dirty_lock() should be called instead.
85 *
86 */
87void put_user_pages_dirty(struct page **pages, unsigned long npages)
88{
89 __put_user_pages_dirty(pages, npages, set_page_dirty);
90}
91EXPORT_SYMBOL(put_user_pages_dirty);
92
93/**
94 * put_user_pages_dirty_lock() - release and dirty an array of gup-pinned pages
95 * @pages: array of pages to be marked dirty and released.
96 * @npages: number of pages in the @pages array.
97 *
98 * For each page in the @pages array, make that page (or its head page, if a
99 * compound page) dirty, if it was previously listed as clean. Then, release
100 * the page using put_user_page().
101 *
102 * Please see the put_user_page() documentation for details.
103 *
104 * This is just like put_user_pages_dirty(), except that it invokes
105 * set_page_dirty_lock(), instead of set_page_dirty().
106 *
107 */
108void put_user_pages_dirty_lock(struct page **pages, unsigned long npages)
109{
110 __put_user_pages_dirty(pages, npages, set_page_dirty_lock);
111}
112EXPORT_SYMBOL(put_user_pages_dirty_lock);
113
114/**
115 * put_user_pages() - release an array of gup-pinned pages.
116 * @pages: array of pages to be marked dirty and released.
117 * @npages: number of pages in the @pages array.
118 *
119 * For each page in the @pages array, release the page using put_user_page().
120 *
121 * Please see the put_user_page() documentation for details.
122 */
123void put_user_pages(struct page **pages, unsigned long npages)
124{
125 unsigned long index;
126
127 /*
128 * TODO: this can be optimized for huge pages: if a series of pages is
129 * physically contiguous and part of the same compound page, then a
130 * single operation to the head page should suffice.
131 */
132 for (index = 0; index < npages; index++)
133 put_user_page(pages[index]);
134}
135EXPORT_SYMBOL(put_user_pages);
136
050a9adc 137#ifdef CONFIG_MMU
69e68b4f
KS
138static struct page *no_page_table(struct vm_area_struct *vma,
139 unsigned int flags)
4bbd4c77 140{
69e68b4f
KS
141 /*
142 * When core dumping an enormous anonymous area that nobody
143 * has touched so far, we don't want to allocate unnecessary pages or
144 * page tables. Return error instead of NULL to skip handle_mm_fault,
145 * then get_dump_page() will return NULL to leave a hole in the dump.
146 * But we can only make this optimization where a hole would surely
147 * be zero-filled if handle_mm_fault() actually did handle it.
148 */
149 if ((flags & FOLL_DUMP) && (!vma->vm_ops || !vma->vm_ops->fault))
150 return ERR_PTR(-EFAULT);
151 return NULL;
152}
4bbd4c77 153
1027e443
KS
154static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
155 pte_t *pte, unsigned int flags)
156{
157 /* No page to get reference */
158 if (flags & FOLL_GET)
159 return -EFAULT;
160
161 if (flags & FOLL_TOUCH) {
162 pte_t entry = *pte;
163
164 if (flags & FOLL_WRITE)
165 entry = pte_mkdirty(entry);
166 entry = pte_mkyoung(entry);
167
168 if (!pte_same(*pte, entry)) {
169 set_pte_at(vma->vm_mm, address, pte, entry);
170 update_mmu_cache(vma, address, pte);
171 }
172 }
173
174 /* Proper page table entry exists, but no corresponding struct page */
175 return -EEXIST;
176}
177
19be0eaf
LT
178/*
179 * FOLL_FORCE can write to even unwritable pte's, but only
180 * after we've gone through a COW cycle and they are dirty.
181 */
182static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
183{
f6f37321 184 return pte_write(pte) ||
19be0eaf
LT
185 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
186}
187
69e68b4f 188static struct page *follow_page_pte(struct vm_area_struct *vma,
df06b37f
KB
189 unsigned long address, pmd_t *pmd, unsigned int flags,
190 struct dev_pagemap **pgmap)
69e68b4f
KS
191{
192 struct mm_struct *mm = vma->vm_mm;
193 struct page *page;
194 spinlock_t *ptl;
195 pte_t *ptep, pte;
4bbd4c77 196
69e68b4f 197retry:
4bbd4c77 198 if (unlikely(pmd_bad(*pmd)))
69e68b4f 199 return no_page_table(vma, flags);
4bbd4c77
KS
200
201 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
4bbd4c77
KS
202 pte = *ptep;
203 if (!pte_present(pte)) {
204 swp_entry_t entry;
205 /*
206 * KSM's break_ksm() relies upon recognizing a ksm page
207 * even while it is being migrated, so for that case we
208 * need migration_entry_wait().
209 */
210 if (likely(!(flags & FOLL_MIGRATION)))
211 goto no_page;
0661a336 212 if (pte_none(pte))
4bbd4c77
KS
213 goto no_page;
214 entry = pte_to_swp_entry(pte);
215 if (!is_migration_entry(entry))
216 goto no_page;
217 pte_unmap_unlock(ptep, ptl);
218 migration_entry_wait(mm, pmd, address);
69e68b4f 219 goto retry;
4bbd4c77 220 }
8a0516ed 221 if ((flags & FOLL_NUMA) && pte_protnone(pte))
4bbd4c77 222 goto no_page;
19be0eaf 223 if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
69e68b4f
KS
224 pte_unmap_unlock(ptep, ptl);
225 return NULL;
226 }
4bbd4c77
KS
227
228 page = vm_normal_page(vma, address, pte);
3565fce3
DW
229 if (!page && pte_devmap(pte) && (flags & FOLL_GET)) {
230 /*
231 * Only return device mapping pages in the FOLL_GET case since
232 * they are only valid while holding the pgmap reference.
233 */
df06b37f
KB
234 *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
235 if (*pgmap)
3565fce3
DW
236 page = pte_page(pte);
237 else
238 goto no_page;
239 } else if (unlikely(!page)) {
1027e443
KS
240 if (flags & FOLL_DUMP) {
241 /* Avoid special (like zero) pages in core dumps */
242 page = ERR_PTR(-EFAULT);
243 goto out;
244 }
245
246 if (is_zero_pfn(pte_pfn(pte))) {
247 page = pte_page(pte);
248 } else {
249 int ret;
250
251 ret = follow_pfn_pte(vma, address, ptep, flags);
252 page = ERR_PTR(ret);
253 goto out;
254 }
4bbd4c77
KS
255 }
256
6742d293
KS
257 if (flags & FOLL_SPLIT && PageTransCompound(page)) {
258 int ret;
259 get_page(page);
260 pte_unmap_unlock(ptep, ptl);
261 lock_page(page);
262 ret = split_huge_page(page);
263 unlock_page(page);
264 put_page(page);
265 if (ret)
266 return ERR_PTR(ret);
267 goto retry;
268 }
269
8fde12ca
LT
270 if (flags & FOLL_GET) {
271 if (unlikely(!try_get_page(page))) {
272 page = ERR_PTR(-ENOMEM);
273 goto out;
274 }
275 }
4bbd4c77
KS
276 if (flags & FOLL_TOUCH) {
277 if ((flags & FOLL_WRITE) &&
278 !pte_dirty(pte) && !PageDirty(page))
279 set_page_dirty(page);
280 /*
281 * pte_mkyoung() would be more correct here, but atomic care
282 * is needed to avoid losing the dirty bit: it is easier to use
283 * mark_page_accessed().
284 */
285 mark_page_accessed(page);
286 }
de60f5f1 287 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
e90309c9
KS
288 /* Do not mlock pte-mapped THP */
289 if (PageTransCompound(page))
290 goto out;
291
4bbd4c77
KS
292 /*
293 * The preliminary mapping check is mainly to avoid the
294 * pointless overhead of lock_page on the ZERO_PAGE
295 * which might bounce very badly if there is contention.
296 *
297 * If the page is already locked, we don't need to
298 * handle it now - vmscan will handle it later if and
299 * when it attempts to reclaim the page.
300 */
301 if (page->mapping && trylock_page(page)) {
302 lru_add_drain(); /* push cached pages to LRU */
303 /*
304 * Because we lock page here, and migration is
305 * blocked by the pte's page reference, and we
306 * know the page is still mapped, we don't even
307 * need to check for file-cache page truncation.
308 */
309 mlock_vma_page(page);
310 unlock_page(page);
311 }
312 }
1027e443 313out:
4bbd4c77 314 pte_unmap_unlock(ptep, ptl);
4bbd4c77 315 return page;
4bbd4c77
KS
316no_page:
317 pte_unmap_unlock(ptep, ptl);
318 if (!pte_none(pte))
69e68b4f
KS
319 return NULL;
320 return no_page_table(vma, flags);
321}
322
080dbb61
AK
323static struct page *follow_pmd_mask(struct vm_area_struct *vma,
324 unsigned long address, pud_t *pudp,
df06b37f
KB
325 unsigned int flags,
326 struct follow_page_context *ctx)
69e68b4f 327{
68827280 328 pmd_t *pmd, pmdval;
69e68b4f
KS
329 spinlock_t *ptl;
330 struct page *page;
331 struct mm_struct *mm = vma->vm_mm;
332
080dbb61 333 pmd = pmd_offset(pudp, address);
68827280
HY
334 /*
335 * The READ_ONCE() will stabilize the pmdval in a register or
336 * on the stack so that it will stop changing under the code.
337 */
338 pmdval = READ_ONCE(*pmd);
339 if (pmd_none(pmdval))
69e68b4f 340 return no_page_table(vma, flags);
68827280 341 if (pmd_huge(pmdval) && vma->vm_flags & VM_HUGETLB) {
e66f17ff
NH
342 page = follow_huge_pmd(mm, address, pmd, flags);
343 if (page)
344 return page;
345 return no_page_table(vma, flags);
69e68b4f 346 }
68827280 347 if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
4dc71451 348 page = follow_huge_pd(vma, address,
68827280 349 __hugepd(pmd_val(pmdval)), flags,
4dc71451
AK
350 PMD_SHIFT);
351 if (page)
352 return page;
353 return no_page_table(vma, flags);
354 }
84c3fc4e 355retry:
68827280 356 if (!pmd_present(pmdval)) {
84c3fc4e
ZY
357 if (likely(!(flags & FOLL_MIGRATION)))
358 return no_page_table(vma, flags);
359 VM_BUG_ON(thp_migration_supported() &&
68827280
HY
360 !is_pmd_migration_entry(pmdval));
361 if (is_pmd_migration_entry(pmdval))
84c3fc4e 362 pmd_migration_entry_wait(mm, pmd);
68827280
HY
363 pmdval = READ_ONCE(*pmd);
364 /*
365 * MADV_DONTNEED may convert the pmd to null because
366 * mmap_sem is held in read mode
367 */
368 if (pmd_none(pmdval))
369 return no_page_table(vma, flags);
84c3fc4e
ZY
370 goto retry;
371 }
68827280 372 if (pmd_devmap(pmdval)) {
3565fce3 373 ptl = pmd_lock(mm, pmd);
df06b37f 374 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
3565fce3
DW
375 spin_unlock(ptl);
376 if (page)
377 return page;
378 }
68827280 379 if (likely(!pmd_trans_huge(pmdval)))
df06b37f 380 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
6742d293 381
68827280 382 if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
db08f203
AK
383 return no_page_table(vma, flags);
384
84c3fc4e 385retry_locked:
6742d293 386 ptl = pmd_lock(mm, pmd);
68827280
HY
387 if (unlikely(pmd_none(*pmd))) {
388 spin_unlock(ptl);
389 return no_page_table(vma, flags);
390 }
84c3fc4e
ZY
391 if (unlikely(!pmd_present(*pmd))) {
392 spin_unlock(ptl);
393 if (likely(!(flags & FOLL_MIGRATION)))
394 return no_page_table(vma, flags);
395 pmd_migration_entry_wait(mm, pmd);
396 goto retry_locked;
397 }
6742d293
KS
398 if (unlikely(!pmd_trans_huge(*pmd))) {
399 spin_unlock(ptl);
df06b37f 400 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
6742d293 401 }
6742d293
KS
402 if (flags & FOLL_SPLIT) {
403 int ret;
404 page = pmd_page(*pmd);
405 if (is_huge_zero_page(page)) {
406 spin_unlock(ptl);
407 ret = 0;
78ddc534 408 split_huge_pmd(vma, pmd, address);
337d9abf
NH
409 if (pmd_trans_unstable(pmd))
410 ret = -EBUSY;
6742d293 411 } else {
8fde12ca
LT
412 if (unlikely(!try_get_page(page))) {
413 spin_unlock(ptl);
414 return ERR_PTR(-ENOMEM);
415 }
69e68b4f 416 spin_unlock(ptl);
6742d293
KS
417 lock_page(page);
418 ret = split_huge_page(page);
419 unlock_page(page);
420 put_page(page);
baa355fd
KS
421 if (pmd_none(*pmd))
422 return no_page_table(vma, flags);
6742d293
KS
423 }
424
425 return ret ? ERR_PTR(ret) :
df06b37f 426 follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
69e68b4f 427 }
6742d293
KS
428 page = follow_trans_huge_pmd(vma, address, pmd, flags);
429 spin_unlock(ptl);
df06b37f 430 ctx->page_mask = HPAGE_PMD_NR - 1;
6742d293 431 return page;
4bbd4c77
KS
432}
433
080dbb61
AK
434static struct page *follow_pud_mask(struct vm_area_struct *vma,
435 unsigned long address, p4d_t *p4dp,
df06b37f
KB
436 unsigned int flags,
437 struct follow_page_context *ctx)
080dbb61
AK
438{
439 pud_t *pud;
440 spinlock_t *ptl;
441 struct page *page;
442 struct mm_struct *mm = vma->vm_mm;
443
444 pud = pud_offset(p4dp, address);
445 if (pud_none(*pud))
446 return no_page_table(vma, flags);
447 if (pud_huge(*pud) && vma->vm_flags & VM_HUGETLB) {
448 page = follow_huge_pud(mm, address, pud, flags);
449 if (page)
450 return page;
451 return no_page_table(vma, flags);
452 }
4dc71451
AK
453 if (is_hugepd(__hugepd(pud_val(*pud)))) {
454 page = follow_huge_pd(vma, address,
455 __hugepd(pud_val(*pud)), flags,
456 PUD_SHIFT);
457 if (page)
458 return page;
459 return no_page_table(vma, flags);
460 }
080dbb61
AK
461 if (pud_devmap(*pud)) {
462 ptl = pud_lock(mm, pud);
df06b37f 463 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
080dbb61
AK
464 spin_unlock(ptl);
465 if (page)
466 return page;
467 }
468 if (unlikely(pud_bad(*pud)))
469 return no_page_table(vma, flags);
470
df06b37f 471 return follow_pmd_mask(vma, address, pud, flags, ctx);
080dbb61
AK
472}
473
080dbb61
AK
474static struct page *follow_p4d_mask(struct vm_area_struct *vma,
475 unsigned long address, pgd_t *pgdp,
df06b37f
KB
476 unsigned int flags,
477 struct follow_page_context *ctx)
080dbb61
AK
478{
479 p4d_t *p4d;
4dc71451 480 struct page *page;
080dbb61
AK
481
482 p4d = p4d_offset(pgdp, address);
483 if (p4d_none(*p4d))
484 return no_page_table(vma, flags);
485 BUILD_BUG_ON(p4d_huge(*p4d));
486 if (unlikely(p4d_bad(*p4d)))
487 return no_page_table(vma, flags);
488
4dc71451
AK
489 if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
490 page = follow_huge_pd(vma, address,
491 __hugepd(p4d_val(*p4d)), flags,
492 P4D_SHIFT);
493 if (page)
494 return page;
495 return no_page_table(vma, flags);
496 }
df06b37f 497 return follow_pud_mask(vma, address, p4d, flags, ctx);
080dbb61
AK
498}
499
500/**
501 * follow_page_mask - look up a page descriptor from a user-virtual address
502 * @vma: vm_area_struct mapping @address
503 * @address: virtual address to look up
504 * @flags: flags modifying lookup behaviour
78179556
MR
505 * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
506 * pointer to output page_mask
080dbb61
AK
507 *
508 * @flags can have FOLL_ flags set, defined in <linux/mm.h>
509 *
78179556
MR
510 * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
511 * the device's dev_pagemap metadata to avoid repeating expensive lookups.
512 *
513 * On output, the @ctx->page_mask is set according to the size of the page.
514 *
515 * Return: the mapped (struct page *), %NULL if no mapping exists, or
080dbb61
AK
516 * an error pointer if there is a mapping to something not represented
517 * by a page descriptor (see also vm_normal_page()).
518 */
a7030aea 519static struct page *follow_page_mask(struct vm_area_struct *vma,
080dbb61 520 unsigned long address, unsigned int flags,
df06b37f 521 struct follow_page_context *ctx)
080dbb61
AK
522{
523 pgd_t *pgd;
524 struct page *page;
525 struct mm_struct *mm = vma->vm_mm;
526
df06b37f 527 ctx->page_mask = 0;
080dbb61
AK
528
529 /* make this handle hugepd */
530 page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
531 if (!IS_ERR(page)) {
532 BUG_ON(flags & FOLL_GET);
533 return page;
534 }
535
536 pgd = pgd_offset(mm, address);
537
538 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
539 return no_page_table(vma, flags);
540
faaa5b62
AK
541 if (pgd_huge(*pgd)) {
542 page = follow_huge_pgd(mm, address, pgd, flags);
543 if (page)
544 return page;
545 return no_page_table(vma, flags);
546 }
4dc71451
AK
547 if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
548 page = follow_huge_pd(vma, address,
549 __hugepd(pgd_val(*pgd)), flags,
550 PGDIR_SHIFT);
551 if (page)
552 return page;
553 return no_page_table(vma, flags);
554 }
faaa5b62 555
df06b37f
KB
556 return follow_p4d_mask(vma, address, pgd, flags, ctx);
557}
558
559struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
560 unsigned int foll_flags)
561{
562 struct follow_page_context ctx = { NULL };
563 struct page *page;
564
565 page = follow_page_mask(vma, address, foll_flags, &ctx);
566 if (ctx.pgmap)
567 put_dev_pagemap(ctx.pgmap);
568 return page;
080dbb61
AK
569}
570
f2b495ca
KS
571static int get_gate_page(struct mm_struct *mm, unsigned long address,
572 unsigned int gup_flags, struct vm_area_struct **vma,
573 struct page **page)
574{
575 pgd_t *pgd;
c2febafc 576 p4d_t *p4d;
f2b495ca
KS
577 pud_t *pud;
578 pmd_t *pmd;
579 pte_t *pte;
580 int ret = -EFAULT;
581
582 /* user gate pages are read-only */
583 if (gup_flags & FOLL_WRITE)
584 return -EFAULT;
585 if (address > TASK_SIZE)
586 pgd = pgd_offset_k(address);
587 else
588 pgd = pgd_offset_gate(mm, address);
b5d1c39f
AL
589 if (pgd_none(*pgd))
590 return -EFAULT;
c2febafc 591 p4d = p4d_offset(pgd, address);
b5d1c39f
AL
592 if (p4d_none(*p4d))
593 return -EFAULT;
c2febafc 594 pud = pud_offset(p4d, address);
b5d1c39f
AL
595 if (pud_none(*pud))
596 return -EFAULT;
f2b495ca 597 pmd = pmd_offset(pud, address);
84c3fc4e 598 if (!pmd_present(*pmd))
f2b495ca
KS
599 return -EFAULT;
600 VM_BUG_ON(pmd_trans_huge(*pmd));
601 pte = pte_offset_map(pmd, address);
602 if (pte_none(*pte))
603 goto unmap;
604 *vma = get_gate_vma(mm);
605 if (!page)
606 goto out;
607 *page = vm_normal_page(*vma, address, *pte);
608 if (!*page) {
609 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
610 goto unmap;
611 *page = pte_page(*pte);
612 }
8fde12ca
LT
613 if (unlikely(!try_get_page(*page))) {
614 ret = -ENOMEM;
615 goto unmap;
616 }
f2b495ca
KS
617out:
618 ret = 0;
619unmap:
620 pte_unmap(pte);
621 return ret;
622}
623
9a95f3cf
PC
624/*
625 * mmap_sem must be held on entry. If @nonblocking != NULL and
626 * *@flags does not include FOLL_NOWAIT, the mmap_sem may be released.
627 * If it is, *@nonblocking will be set to 0 and -EBUSY returned.
628 */
16744483
KS
629static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
630 unsigned long address, unsigned int *flags, int *nonblocking)
631{
16744483 632 unsigned int fault_flags = 0;
2b740303 633 vm_fault_t ret;
16744483 634
de60f5f1
EM
635 /* mlock all present pages, but do not fault in new pages */
636 if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
637 return -ENOENT;
16744483
KS
638 if (*flags & FOLL_WRITE)
639 fault_flags |= FAULT_FLAG_WRITE;
1b2ee126
DH
640 if (*flags & FOLL_REMOTE)
641 fault_flags |= FAULT_FLAG_REMOTE;
16744483
KS
642 if (nonblocking)
643 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
644 if (*flags & FOLL_NOWAIT)
645 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
234b239b
ALC
646 if (*flags & FOLL_TRIED) {
647 VM_WARN_ON_ONCE(fault_flags & FAULT_FLAG_ALLOW_RETRY);
648 fault_flags |= FAULT_FLAG_TRIED;
649 }
16744483 650
dcddffd4 651 ret = handle_mm_fault(vma, address, fault_flags);
16744483 652 if (ret & VM_FAULT_ERROR) {
9a291a7c
JM
653 int err = vm_fault_to_errno(ret, *flags);
654
655 if (err)
656 return err;
16744483
KS
657 BUG();
658 }
659
660 if (tsk) {
661 if (ret & VM_FAULT_MAJOR)
662 tsk->maj_flt++;
663 else
664 tsk->min_flt++;
665 }
666
667 if (ret & VM_FAULT_RETRY) {
96312e61 668 if (nonblocking && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
16744483
KS
669 *nonblocking = 0;
670 return -EBUSY;
671 }
672
673 /*
674 * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when
675 * necessary, even if maybe_mkwrite decided not to set pte_write. We
676 * can thus safely do subsequent page lookups as if they were reads.
677 * But only do so when looping for pte_write is futile: in some cases
678 * userspace may also be wanting to write to the gotten user page,
679 * which a read fault here might prevent (a readonly page might get
680 * reCOWed by userspace write).
681 */
682 if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
2923117b 683 *flags |= FOLL_COW;
16744483
KS
684 return 0;
685}
686
fa5bb209
KS
687static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
688{
689 vm_flags_t vm_flags = vma->vm_flags;
1b2ee126
DH
690 int write = (gup_flags & FOLL_WRITE);
691 int foreign = (gup_flags & FOLL_REMOTE);
fa5bb209
KS
692
693 if (vm_flags & (VM_IO | VM_PFNMAP))
694 return -EFAULT;
695
7f7ccc2c
WT
696 if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
697 return -EFAULT;
698
1b2ee126 699 if (write) {
fa5bb209
KS
700 if (!(vm_flags & VM_WRITE)) {
701 if (!(gup_flags & FOLL_FORCE))
702 return -EFAULT;
703 /*
704 * We used to let the write,force case do COW in a
705 * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could
706 * set a breakpoint in a read-only mapping of an
707 * executable, without corrupting the file (yet only
708 * when that file had been opened for writing!).
709 * Anon pages in shared mappings are surprising: now
710 * just reject it.
711 */
46435364 712 if (!is_cow_mapping(vm_flags))
fa5bb209 713 return -EFAULT;
fa5bb209
KS
714 }
715 } else if (!(vm_flags & VM_READ)) {
716 if (!(gup_flags & FOLL_FORCE))
717 return -EFAULT;
718 /*
719 * Is there actually any vma we can reach here which does not
720 * have VM_MAYREAD set?
721 */
722 if (!(vm_flags & VM_MAYREAD))
723 return -EFAULT;
724 }
d61172b4
DH
725 /*
726 * gups are always data accesses, not instruction
727 * fetches, so execute=false here
728 */
729 if (!arch_vma_access_permitted(vma, write, false, foreign))
33a709b2 730 return -EFAULT;
fa5bb209
KS
731 return 0;
732}
733
4bbd4c77
KS
734/**
735 * __get_user_pages() - pin user pages in memory
736 * @tsk: task_struct of target task
737 * @mm: mm_struct of target mm
738 * @start: starting user address
739 * @nr_pages: number of pages from start to pin
740 * @gup_flags: flags modifying pin behaviour
741 * @pages: array that receives pointers to the pages pinned.
742 * Should be at least nr_pages long. Or NULL, if caller
743 * only intends to ensure the pages are faulted in.
744 * @vmas: array of pointers to vmas corresponding to each page.
745 * Or NULL if the caller does not require them.
746 * @nonblocking: whether waiting for disk IO or mmap_sem contention
747 *
748 * Returns number of pages pinned. This may be fewer than the number
749 * requested. If nr_pages is 0 or negative, returns 0. If no pages
750 * were pinned, returns -errno. Each page returned must be released
751 * with a put_page() call when it is finished with. vmas will only
752 * remain valid while mmap_sem is held.
753 *
9a95f3cf 754 * Must be called with mmap_sem held. It may be released. See below.
4bbd4c77
KS
755 *
756 * __get_user_pages walks a process's page tables and takes a reference to
757 * each struct page that each user address corresponds to at a given
758 * instant. That is, it takes the page that would be accessed if a user
759 * thread accesses the given user virtual address at that instant.
760 *
761 * This does not guarantee that the page exists in the user mappings when
762 * __get_user_pages returns, and there may even be a completely different
763 * page there in some cases (eg. if mmapped pagecache has been invalidated
764 * and subsequently re faulted). However it does guarantee that the page
765 * won't be freed completely. And mostly callers simply care that the page
766 * contains data that was valid *at some point in time*. Typically, an IO
767 * or similar operation cannot guarantee anything stronger anyway because
768 * locks can't be held over the syscall boundary.
769 *
770 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
771 * the page is written to, set_page_dirty (or set_page_dirty_lock, as
772 * appropriate) must be called after the page is finished with, and
773 * before put_page is called.
774 *
775 * If @nonblocking != NULL, __get_user_pages will not wait for disk IO
776 * or mmap_sem contention, and if waiting is needed to pin all pages,
9a95f3cf
PC
777 * *@nonblocking will be set to 0. Further, if @gup_flags does not
778 * include FOLL_NOWAIT, the mmap_sem will be released via up_read() in
779 * this case.
780 *
781 * A caller using such a combination of @nonblocking and @gup_flags
782 * must therefore hold the mmap_sem for reading only, and recognize
783 * when it's been released. Otherwise, it must be held for either
784 * reading or writing and will not be released.
4bbd4c77
KS
785 *
786 * In most cases, get_user_pages or get_user_pages_fast should be used
787 * instead of __get_user_pages. __get_user_pages should be used only if
788 * you need some special @gup_flags.
789 */
0d731759 790static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
4bbd4c77
KS
791 unsigned long start, unsigned long nr_pages,
792 unsigned int gup_flags, struct page **pages,
793 struct vm_area_struct **vmas, int *nonblocking)
794{
df06b37f 795 long ret = 0, i = 0;
fa5bb209 796 struct vm_area_struct *vma = NULL;
df06b37f 797 struct follow_page_context ctx = { NULL };
4bbd4c77
KS
798
799 if (!nr_pages)
800 return 0;
801
802 VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
803
804 /*
805 * If FOLL_FORCE is set then do not force a full fault as the hinting
806 * fault information is unrelated to the reference behaviour of a task
807 * using the address space
808 */
809 if (!(gup_flags & FOLL_FORCE))
810 gup_flags |= FOLL_NUMA;
811
4bbd4c77 812 do {
fa5bb209
KS
813 struct page *page;
814 unsigned int foll_flags = gup_flags;
815 unsigned int page_increm;
816
817 /* first iteration or cross vma bound */
818 if (!vma || start >= vma->vm_end) {
819 vma = find_extend_vma(mm, start);
820 if (!vma && in_gate_area(mm, start)) {
fa5bb209
KS
821 ret = get_gate_page(mm, start & PAGE_MASK,
822 gup_flags, &vma,
823 pages ? &pages[i] : NULL);
824 if (ret)
08be37b7 825 goto out;
df06b37f 826 ctx.page_mask = 0;
fa5bb209
KS
827 goto next_page;
828 }
4bbd4c77 829
df06b37f
KB
830 if (!vma || check_vma_flags(vma, gup_flags)) {
831 ret = -EFAULT;
832 goto out;
833 }
fa5bb209
KS
834 if (is_vm_hugetlb_page(vma)) {
835 i = follow_hugetlb_page(mm, vma, pages, vmas,
836 &start, &nr_pages, i,
87ffc118 837 gup_flags, nonblocking);
fa5bb209 838 continue;
4bbd4c77 839 }
fa5bb209
KS
840 }
841retry:
842 /*
843 * If we have a pending SIGKILL, don't keep faulting pages and
844 * potentially allocating memory.
845 */
fa45f116 846 if (fatal_signal_pending(current)) {
df06b37f
KB
847 ret = -ERESTARTSYS;
848 goto out;
849 }
fa5bb209 850 cond_resched();
df06b37f
KB
851
852 page = follow_page_mask(vma, start, foll_flags, &ctx);
fa5bb209 853 if (!page) {
fa5bb209
KS
854 ret = faultin_page(tsk, vma, start, &foll_flags,
855 nonblocking);
856 switch (ret) {
857 case 0:
858 goto retry;
df06b37f
KB
859 case -EBUSY:
860 ret = 0;
861 /* FALLTHRU */
fa5bb209
KS
862 case -EFAULT:
863 case -ENOMEM:
864 case -EHWPOISON:
df06b37f 865 goto out;
fa5bb209
KS
866 case -ENOENT:
867 goto next_page;
4bbd4c77 868 }
fa5bb209 869 BUG();
1027e443
KS
870 } else if (PTR_ERR(page) == -EEXIST) {
871 /*
872 * Proper page table entry exists, but no corresponding
873 * struct page.
874 */
875 goto next_page;
876 } else if (IS_ERR(page)) {
df06b37f
KB
877 ret = PTR_ERR(page);
878 goto out;
1027e443 879 }
fa5bb209
KS
880 if (pages) {
881 pages[i] = page;
882 flush_anon_page(vma, page, start);
883 flush_dcache_page(page);
df06b37f 884 ctx.page_mask = 0;
4bbd4c77 885 }
4bbd4c77 886next_page:
fa5bb209
KS
887 if (vmas) {
888 vmas[i] = vma;
df06b37f 889 ctx.page_mask = 0;
fa5bb209 890 }
df06b37f 891 page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
fa5bb209
KS
892 if (page_increm > nr_pages)
893 page_increm = nr_pages;
894 i += page_increm;
895 start += page_increm * PAGE_SIZE;
896 nr_pages -= page_increm;
4bbd4c77 897 } while (nr_pages);
df06b37f
KB
898out:
899 if (ctx.pgmap)
900 put_dev_pagemap(ctx.pgmap);
901 return i ? i : ret;
4bbd4c77 902}
4bbd4c77 903
771ab430
TK
904static bool vma_permits_fault(struct vm_area_struct *vma,
905 unsigned int fault_flags)
d4925e00 906{
1b2ee126
DH
907 bool write = !!(fault_flags & FAULT_FLAG_WRITE);
908 bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
33a709b2 909 vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
d4925e00
DH
910
911 if (!(vm_flags & vma->vm_flags))
912 return false;
913
33a709b2
DH
914 /*
915 * The architecture might have a hardware protection
1b2ee126 916 * mechanism other than read/write that can deny access.
d61172b4
DH
917 *
918 * gup always represents data access, not instruction
919 * fetches, so execute=false here:
33a709b2 920 */
d61172b4 921 if (!arch_vma_access_permitted(vma, write, false, foreign))
33a709b2
DH
922 return false;
923
d4925e00
DH
924 return true;
925}
926
4bbd4c77
KS
927/*
928 * fixup_user_fault() - manually resolve a user page fault
929 * @tsk: the task_struct to use for page fault accounting, or
930 * NULL if faults are not to be recorded.
931 * @mm: mm_struct of target mm
932 * @address: user address
933 * @fault_flags:flags to pass down to handle_mm_fault()
4a9e1cda
DD
934 * @unlocked: did we unlock the mmap_sem while retrying, maybe NULL if caller
935 * does not allow retry
4bbd4c77
KS
936 *
937 * This is meant to be called in the specific scenario where for locking reasons
938 * we try to access user memory in atomic context (within a pagefault_disable()
939 * section), this returns -EFAULT, and we want to resolve the user fault before
940 * trying again.
941 *
942 * Typically this is meant to be used by the futex code.
943 *
944 * The main difference with get_user_pages() is that this function will
945 * unconditionally call handle_mm_fault() which will in turn perform all the
946 * necessary SW fixup of the dirty and young bits in the PTE, while
4a9e1cda 947 * get_user_pages() only guarantees to update these in the struct page.
4bbd4c77
KS
948 *
949 * This is important for some architectures where those bits also gate the
950 * access permission to the page because they are maintained in software. On
951 * such architectures, gup() will not be enough to make a subsequent access
952 * succeed.
953 *
4a9e1cda
DD
954 * This function will not return with an unlocked mmap_sem. So it has not the
955 * same semantics wrt the @mm->mmap_sem as does filemap_fault().
4bbd4c77
KS
956 */
957int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
4a9e1cda
DD
958 unsigned long address, unsigned int fault_flags,
959 bool *unlocked)
4bbd4c77
KS
960{
961 struct vm_area_struct *vma;
2b740303 962 vm_fault_t ret, major = 0;
4a9e1cda
DD
963
964 if (unlocked)
965 fault_flags |= FAULT_FLAG_ALLOW_RETRY;
4bbd4c77 966
4a9e1cda 967retry:
4bbd4c77
KS
968 vma = find_extend_vma(mm, address);
969 if (!vma || address < vma->vm_start)
970 return -EFAULT;
971
d4925e00 972 if (!vma_permits_fault(vma, fault_flags))
4bbd4c77
KS
973 return -EFAULT;
974
dcddffd4 975 ret = handle_mm_fault(vma, address, fault_flags);
4a9e1cda 976 major |= ret & VM_FAULT_MAJOR;
4bbd4c77 977 if (ret & VM_FAULT_ERROR) {
9a291a7c
JM
978 int err = vm_fault_to_errno(ret, 0);
979
980 if (err)
981 return err;
4bbd4c77
KS
982 BUG();
983 }
4a9e1cda
DD
984
985 if (ret & VM_FAULT_RETRY) {
986 down_read(&mm->mmap_sem);
987 if (!(fault_flags & FAULT_FLAG_TRIED)) {
988 *unlocked = true;
989 fault_flags &= ~FAULT_FLAG_ALLOW_RETRY;
990 fault_flags |= FAULT_FLAG_TRIED;
991 goto retry;
992 }
993 }
994
4bbd4c77 995 if (tsk) {
4a9e1cda 996 if (major)
4bbd4c77
KS
997 tsk->maj_flt++;
998 else
999 tsk->min_flt++;
1000 }
1001 return 0;
1002}
add6a0cd 1003EXPORT_SYMBOL_GPL(fixup_user_fault);
4bbd4c77 1004
f0818f47
AA
1005static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
1006 struct mm_struct *mm,
1007 unsigned long start,
1008 unsigned long nr_pages,
f0818f47
AA
1009 struct page **pages,
1010 struct vm_area_struct **vmas,
e716712f 1011 int *locked,
0fd71a56 1012 unsigned int flags)
f0818f47 1013{
f0818f47
AA
1014 long ret, pages_done;
1015 bool lock_dropped;
1016
1017 if (locked) {
1018 /* if VM_FAULT_RETRY can be returned, vmas become invalid */
1019 BUG_ON(vmas);
1020 /* check caller initialized locked */
1021 BUG_ON(*locked != 1);
1022 }
1023
1024 if (pages)
1025 flags |= FOLL_GET;
f0818f47
AA
1026
1027 pages_done = 0;
1028 lock_dropped = false;
1029 for (;;) {
1030 ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages,
1031 vmas, locked);
1032 if (!locked)
1033 /* VM_FAULT_RETRY couldn't trigger, bypass */
1034 return ret;
1035
1036 /* VM_FAULT_RETRY cannot return errors */
1037 if (!*locked) {
1038 BUG_ON(ret < 0);
1039 BUG_ON(ret >= nr_pages);
1040 }
1041
f0818f47
AA
1042 if (ret > 0) {
1043 nr_pages -= ret;
1044 pages_done += ret;
1045 if (!nr_pages)
1046 break;
1047 }
1048 if (*locked) {
96312e61
AA
1049 /*
1050 * VM_FAULT_RETRY didn't trigger or it was a
1051 * FOLL_NOWAIT.
1052 */
f0818f47
AA
1053 if (!pages_done)
1054 pages_done = ret;
1055 break;
1056 }
df17277b
MR
1057 /*
1058 * VM_FAULT_RETRY triggered, so seek to the faulting offset.
1059 * For the prefault case (!pages) we only update counts.
1060 */
1061 if (likely(pages))
1062 pages += ret;
f0818f47
AA
1063 start += ret << PAGE_SHIFT;
1064
1065 /*
1066 * Repeat on the address that fired VM_FAULT_RETRY
1067 * without FAULT_FLAG_ALLOW_RETRY but with
1068 * FAULT_FLAG_TRIED.
1069 */
1070 *locked = 1;
1071 lock_dropped = true;
1072 down_read(&mm->mmap_sem);
1073 ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED,
1074 pages, NULL, NULL);
1075 if (ret != 1) {
1076 BUG_ON(ret > 1);
1077 if (!pages_done)
1078 pages_done = ret;
1079 break;
1080 }
1081 nr_pages--;
1082 pages_done++;
1083 if (!nr_pages)
1084 break;
df17277b
MR
1085 if (likely(pages))
1086 pages++;
f0818f47
AA
1087 start += PAGE_SIZE;
1088 }
e716712f 1089 if (lock_dropped && *locked) {
f0818f47
AA
1090 /*
1091 * We must let the caller know we temporarily dropped the lock
1092 * and so the critical section protected by it was lost.
1093 */
1094 up_read(&mm->mmap_sem);
1095 *locked = 0;
1096 }
1097 return pages_done;
1098}
1099
4bbd4c77 1100/*
1e987790 1101 * get_user_pages_remote() - pin user pages in memory
4bbd4c77
KS
1102 * @tsk: the task_struct to use for page fault accounting, or
1103 * NULL if faults are not to be recorded.
1104 * @mm: mm_struct of target mm
1105 * @start: starting user address
1106 * @nr_pages: number of pages from start to pin
9beae1ea 1107 * @gup_flags: flags modifying lookup behaviour
4bbd4c77
KS
1108 * @pages: array that receives pointers to the pages pinned.
1109 * Should be at least nr_pages long. Or NULL, if caller
1110 * only intends to ensure the pages are faulted in.
1111 * @vmas: array of pointers to vmas corresponding to each page.
1112 * Or NULL if the caller does not require them.
5b56d49f
LS
1113 * @locked: pointer to lock flag indicating whether lock is held and
1114 * subsequently whether VM_FAULT_RETRY functionality can be
1115 * utilised. Lock must initially be held.
4bbd4c77
KS
1116 *
1117 * Returns number of pages pinned. This may be fewer than the number
1118 * requested. If nr_pages is 0 or negative, returns 0. If no pages
1119 * were pinned, returns -errno. Each page returned must be released
1120 * with a put_page() call when it is finished with. vmas will only
1121 * remain valid while mmap_sem is held.
1122 *
1123 * Must be called with mmap_sem held for read or write.
1124 *
1125 * get_user_pages walks a process's page tables and takes a reference to
1126 * each struct page that each user address corresponds to at a given
1127 * instant. That is, it takes the page that would be accessed if a user
1128 * thread accesses the given user virtual address at that instant.
1129 *
1130 * This does not guarantee that the page exists in the user mappings when
1131 * get_user_pages returns, and there may even be a completely different
1132 * page there in some cases (eg. if mmapped pagecache has been invalidated
1133 * and subsequently re faulted). However it does guarantee that the page
1134 * won't be freed completely. And mostly callers simply care that the page
1135 * contains data that was valid *at some point in time*. Typically, an IO
1136 * or similar operation cannot guarantee anything stronger anyway because
1137 * locks can't be held over the syscall boundary.
1138 *
9beae1ea
LS
1139 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
1140 * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
1141 * be called after the page is finished with, and before put_page is called.
4bbd4c77
KS
1142 *
1143 * get_user_pages is typically used for fewer-copy IO operations, to get a
1144 * handle on the memory by some means other than accesses via the user virtual
1145 * addresses. The pages may be submitted for DMA to devices or accessed via
1146 * their kernel linear mapping (via the kmap APIs). Care should be taken to
1147 * use the correct cache flushing APIs.
1148 *
1149 * See also get_user_pages_fast, for performance critical applications.
f0818f47
AA
1150 *
1151 * get_user_pages should be phased out in favor of
1152 * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
1153 * should use get_user_pages because it cannot pass
1154 * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
4bbd4c77 1155 */
1e987790
DH
1156long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
1157 unsigned long start, unsigned long nr_pages,
9beae1ea 1158 unsigned int gup_flags, struct page **pages,
5b56d49f 1159 struct vm_area_struct **vmas, int *locked)
4bbd4c77 1160{
932f4a63
IW
1161 /*
1162 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1163 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1164 * vmas. As there are no users of this flag in this call we simply
1165 * disallow this option for now.
1166 */
1167 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1168 return -EINVAL;
1169
859110d7 1170 return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
e716712f 1171 locked,
9beae1ea 1172 gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1e987790
DH
1173}
1174EXPORT_SYMBOL(get_user_pages_remote);
1175
d3649f68
CH
1176/**
1177 * populate_vma_page_range() - populate a range of pages in the vma.
1178 * @vma: target vma
1179 * @start: start address
1180 * @end: end address
1181 * @nonblocking:
1182 *
1183 * This takes care of mlocking the pages too if VM_LOCKED is set.
1184 *
1185 * return 0 on success, negative error code on error.
1186 *
1187 * vma->vm_mm->mmap_sem must be held.
1188 *
1189 * If @nonblocking is NULL, it may be held for read or write and will
1190 * be unperturbed.
1191 *
1192 * If @nonblocking is non-NULL, it must held for read only and may be
1193 * released. If it's released, *@nonblocking will be set to 0.
1194 */
1195long populate_vma_page_range(struct vm_area_struct *vma,
1196 unsigned long start, unsigned long end, int *nonblocking)
1197{
1198 struct mm_struct *mm = vma->vm_mm;
1199 unsigned long nr_pages = (end - start) / PAGE_SIZE;
1200 int gup_flags;
1201
1202 VM_BUG_ON(start & ~PAGE_MASK);
1203 VM_BUG_ON(end & ~PAGE_MASK);
1204 VM_BUG_ON_VMA(start < vma->vm_start, vma);
1205 VM_BUG_ON_VMA(end > vma->vm_end, vma);
1206 VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_sem), mm);
1207
1208 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1209 if (vma->vm_flags & VM_LOCKONFAULT)
1210 gup_flags &= ~FOLL_POPULATE;
1211 /*
1212 * We want to touch writable mappings with a write fault in order
1213 * to break COW, except for shared mappings because these don't COW
1214 * and we would not want to dirty them for nothing.
1215 */
1216 if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
1217 gup_flags |= FOLL_WRITE;
1218
1219 /*
1220 * We want mlock to succeed for regions that have any permissions
1221 * other than PROT_NONE.
1222 */
1223 if (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))
1224 gup_flags |= FOLL_FORCE;
1225
1226 /*
1227 * We made sure addr is within a VMA, so the following will
1228 * not result in a stack expansion that recurses back here.
1229 */
1230 return __get_user_pages(current, mm, start, nr_pages, gup_flags,
1231 NULL, NULL, nonblocking);
1232}
1233
1234/*
1235 * __mm_populate - populate and/or mlock pages within a range of address space.
1236 *
1237 * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
1238 * flags. VMAs must be already marked with the desired vm_flags, and
1239 * mmap_sem must not be held.
1240 */
1241int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
1242{
1243 struct mm_struct *mm = current->mm;
1244 unsigned long end, nstart, nend;
1245 struct vm_area_struct *vma = NULL;
1246 int locked = 0;
1247 long ret = 0;
1248
1249 end = start + len;
1250
1251 for (nstart = start; nstart < end; nstart = nend) {
1252 /*
1253 * We want to fault in pages for [nstart; end) address range.
1254 * Find first corresponding VMA.
1255 */
1256 if (!locked) {
1257 locked = 1;
1258 down_read(&mm->mmap_sem);
1259 vma = find_vma(mm, nstart);
1260 } else if (nstart >= vma->vm_end)
1261 vma = vma->vm_next;
1262 if (!vma || vma->vm_start >= end)
1263 break;
1264 /*
1265 * Set [nstart; nend) to intersection of desired address
1266 * range with the first VMA. Also, skip undesirable VMA types.
1267 */
1268 nend = min(end, vma->vm_end);
1269 if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1270 continue;
1271 if (nstart < vma->vm_start)
1272 nstart = vma->vm_start;
1273 /*
1274 * Now fault in a range of pages. populate_vma_page_range()
1275 * double checks the vma flags, so that it won't mlock pages
1276 * if the vma was already munlocked.
1277 */
1278 ret = populate_vma_page_range(vma, nstart, nend, &locked);
1279 if (ret < 0) {
1280 if (ignore_errors) {
1281 ret = 0;
1282 continue; /* continue at next VMA */
1283 }
1284 break;
1285 }
1286 nend = nstart + ret * PAGE_SIZE;
1287 ret = 0;
1288 }
1289 if (locked)
1290 up_read(&mm->mmap_sem);
1291 return ret; /* 0 or negative error code */
1292}
1293
1294/**
1295 * get_dump_page() - pin user page in memory while writing it to core dump
1296 * @addr: user address
1297 *
1298 * Returns struct page pointer of user page pinned for dump,
1299 * to be freed afterwards by put_page().
1300 *
1301 * Returns NULL on any kind of failure - a hole must then be inserted into
1302 * the corefile, to preserve alignment with its headers; and also returns
1303 * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
1304 * allowing a hole to be left in the corefile to save diskspace.
1305 *
1306 * Called without mmap_sem, but after all other threads have been killed.
1307 */
1308#ifdef CONFIG_ELF_CORE
1309struct page *get_dump_page(unsigned long addr)
1310{
1311 struct vm_area_struct *vma;
1312 struct page *page;
1313
1314 if (__get_user_pages(current, current->mm, addr, 1,
1315 FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma,
1316 NULL) < 1)
1317 return NULL;
1318 flush_cache_page(vma, addr, page_to_pfn(page));
1319 return page;
1320}
1321#endif /* CONFIG_ELF_CORE */
050a9adc
CH
1322#else /* CONFIG_MMU */
1323static long __get_user_pages_locked(struct task_struct *tsk,
1324 struct mm_struct *mm, unsigned long start,
1325 unsigned long nr_pages, struct page **pages,
1326 struct vm_area_struct **vmas, int *locked,
1327 unsigned int foll_flags)
1328{
1329 struct vm_area_struct *vma;
1330 unsigned long vm_flags;
1331 int i;
1332
1333 /* calculate required read or write permissions.
1334 * If FOLL_FORCE is set, we only require the "MAY" flags.
1335 */
1336 vm_flags = (foll_flags & FOLL_WRITE) ?
1337 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
1338 vm_flags &= (foll_flags & FOLL_FORCE) ?
1339 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1340
1341 for (i = 0; i < nr_pages; i++) {
1342 vma = find_vma(mm, start);
1343 if (!vma)
1344 goto finish_or_fault;
1345
1346 /* protect what we can, including chardevs */
1347 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
1348 !(vm_flags & vma->vm_flags))
1349 goto finish_or_fault;
1350
1351 if (pages) {
1352 pages[i] = virt_to_page(start);
1353 if (pages[i])
1354 get_page(pages[i]);
1355 }
1356 if (vmas)
1357 vmas[i] = vma;
1358 start = (start + PAGE_SIZE) & PAGE_MASK;
1359 }
1360
1361 return i;
1362
1363finish_or_fault:
1364 return i ? : -EFAULT;
1365}
1366#endif /* !CONFIG_MMU */
d3649f68 1367
9a4e9f3b 1368#if defined(CONFIG_FS_DAX) || defined (CONFIG_CMA)
9a4e9f3b
AK
1369static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
1370{
1371 long i;
1372 struct vm_area_struct *vma_prev = NULL;
1373
1374 for (i = 0; i < nr_pages; i++) {
1375 struct vm_area_struct *vma = vmas[i];
1376
1377 if (vma == vma_prev)
1378 continue;
1379
1380 vma_prev = vma;
1381
1382 if (vma_is_fsdax(vma))
1383 return true;
1384 }
1385 return false;
1386}
9a4e9f3b
AK
1387
1388#ifdef CONFIG_CMA
1389static struct page *new_non_cma_page(struct page *page, unsigned long private)
1390{
1391 /*
1392 * We want to make sure we allocate the new page from the same node
1393 * as the source page.
1394 */
1395 int nid = page_to_nid(page);
1396 /*
1397 * Trying to allocate a page for migration. Ignore allocation
1398 * failure warnings. We don't force __GFP_THISNODE here because
1399 * this node here is the node where we have CMA reservation and
1400 * in some case these nodes will have really less non movable
1401 * allocation memory.
1402 */
1403 gfp_t gfp_mask = GFP_USER | __GFP_NOWARN;
1404
1405 if (PageHighMem(page))
1406 gfp_mask |= __GFP_HIGHMEM;
1407
1408#ifdef CONFIG_HUGETLB_PAGE
1409 if (PageHuge(page)) {
1410 struct hstate *h = page_hstate(page);
1411 /*
1412 * We don't want to dequeue from the pool because pool pages will
1413 * mostly be from the CMA region.
1414 */
1415 return alloc_migrate_huge_page(h, gfp_mask, nid, NULL);
1416 }
1417#endif
1418 if (PageTransHuge(page)) {
1419 struct page *thp;
1420 /*
1421 * ignore allocation failure warnings
1422 */
1423 gfp_t thp_gfpmask = GFP_TRANSHUGE | __GFP_NOWARN;
1424
1425 /*
1426 * Remove the movable mask so that we don't allocate from
1427 * CMA area again.
1428 */
1429 thp_gfpmask &= ~__GFP_MOVABLE;
1430 thp = __alloc_pages_node(nid, thp_gfpmask, HPAGE_PMD_ORDER);
1431 if (!thp)
1432 return NULL;
1433 prep_transhuge_page(thp);
1434 return thp;
1435 }
1436
1437 return __alloc_pages_node(nid, gfp_mask, 0);
1438}
1439
932f4a63
IW
1440static long check_and_migrate_cma_pages(struct task_struct *tsk,
1441 struct mm_struct *mm,
1442 unsigned long start,
1443 unsigned long nr_pages,
9a4e9f3b 1444 struct page **pages,
932f4a63
IW
1445 struct vm_area_struct **vmas,
1446 unsigned int gup_flags)
9a4e9f3b 1447{
aa712399
PL
1448 unsigned long i;
1449 unsigned long step;
9a4e9f3b
AK
1450 bool drain_allow = true;
1451 bool migrate_allow = true;
1452 LIST_HEAD(cma_page_list);
1453
1454check_again:
aa712399
PL
1455 for (i = 0; i < nr_pages;) {
1456
1457 struct page *head = compound_head(pages[i]);
1458
1459 /*
1460 * gup may start from a tail page. Advance step by the left
1461 * part.
1462 */
1463 step = (1 << compound_order(head)) - (pages[i] - head);
9a4e9f3b
AK
1464 /*
1465 * If we get a page from the CMA zone, since we are going to
1466 * be pinning these entries, we might as well move them out
1467 * of the CMA zone if possible.
1468 */
aa712399
PL
1469 if (is_migrate_cma_page(head)) {
1470 if (PageHuge(head))
9a4e9f3b 1471 isolate_huge_page(head, &cma_page_list);
aa712399 1472 else {
9a4e9f3b
AK
1473 if (!PageLRU(head) && drain_allow) {
1474 lru_add_drain_all();
1475 drain_allow = false;
1476 }
1477
1478 if (!isolate_lru_page(head)) {
1479 list_add_tail(&head->lru, &cma_page_list);
1480 mod_node_page_state(page_pgdat(head),
1481 NR_ISOLATED_ANON +
1482 page_is_file_cache(head),
1483 hpage_nr_pages(head));
1484 }
1485 }
1486 }
aa712399
PL
1487
1488 i += step;
9a4e9f3b
AK
1489 }
1490
1491 if (!list_empty(&cma_page_list)) {
1492 /*
1493 * drop the above get_user_pages reference.
1494 */
1495 for (i = 0; i < nr_pages; i++)
1496 put_page(pages[i]);
1497
1498 if (migrate_pages(&cma_page_list, new_non_cma_page,
1499 NULL, 0, MIGRATE_SYNC, MR_CONTIG_RANGE)) {
1500 /*
1501 * some of the pages failed migration. Do get_user_pages
1502 * without migration.
1503 */
1504 migrate_allow = false;
1505
1506 if (!list_empty(&cma_page_list))
1507 putback_movable_pages(&cma_page_list);
1508 }
1509 /*
932f4a63
IW
1510 * We did migrate all the pages, Try to get the page references
1511 * again migrating any new CMA pages which we failed to isolate
1512 * earlier.
9a4e9f3b 1513 */
932f4a63
IW
1514 nr_pages = __get_user_pages_locked(tsk, mm, start, nr_pages,
1515 pages, vmas, NULL,
1516 gup_flags);
1517
9a4e9f3b
AK
1518 if ((nr_pages > 0) && migrate_allow) {
1519 drain_allow = true;
1520 goto check_again;
1521 }
1522 }
1523
1524 return nr_pages;
1525}
1526#else
932f4a63
IW
1527static long check_and_migrate_cma_pages(struct task_struct *tsk,
1528 struct mm_struct *mm,
1529 unsigned long start,
1530 unsigned long nr_pages,
1531 struct page **pages,
1532 struct vm_area_struct **vmas,
1533 unsigned int gup_flags)
9a4e9f3b
AK
1534{
1535 return nr_pages;
1536}
050a9adc 1537#endif /* CONFIG_CMA */
9a4e9f3b 1538
2bb6d283 1539/*
932f4a63
IW
1540 * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
1541 * allows us to process the FOLL_LONGTERM flag.
2bb6d283 1542 */
932f4a63
IW
1543static long __gup_longterm_locked(struct task_struct *tsk,
1544 struct mm_struct *mm,
1545 unsigned long start,
1546 unsigned long nr_pages,
1547 struct page **pages,
1548 struct vm_area_struct **vmas,
1549 unsigned int gup_flags)
2bb6d283 1550{
932f4a63
IW
1551 struct vm_area_struct **vmas_tmp = vmas;
1552 unsigned long flags = 0;
2bb6d283
DW
1553 long rc, i;
1554
932f4a63
IW
1555 if (gup_flags & FOLL_LONGTERM) {
1556 if (!pages)
1557 return -EINVAL;
1558
1559 if (!vmas_tmp) {
1560 vmas_tmp = kcalloc(nr_pages,
1561 sizeof(struct vm_area_struct *),
1562 GFP_KERNEL);
1563 if (!vmas_tmp)
1564 return -ENOMEM;
1565 }
1566 flags = memalloc_nocma_save();
2bb6d283
DW
1567 }
1568
932f4a63
IW
1569 rc = __get_user_pages_locked(tsk, mm, start, nr_pages, pages,
1570 vmas_tmp, NULL, gup_flags);
2bb6d283 1571
932f4a63
IW
1572 if (gup_flags & FOLL_LONGTERM) {
1573 memalloc_nocma_restore(flags);
1574 if (rc < 0)
1575 goto out;
1576
1577 if (check_dax_vmas(vmas_tmp, rc)) {
1578 for (i = 0; i < rc; i++)
1579 put_page(pages[i]);
1580 rc = -EOPNOTSUPP;
1581 goto out;
1582 }
1583
1584 rc = check_and_migrate_cma_pages(tsk, mm, start, rc, pages,
1585 vmas_tmp, gup_flags);
9a4e9f3b 1586 }
2bb6d283 1587
2bb6d283 1588out:
932f4a63
IW
1589 if (vmas_tmp != vmas)
1590 kfree(vmas_tmp);
2bb6d283
DW
1591 return rc;
1592}
932f4a63
IW
1593#else /* !CONFIG_FS_DAX && !CONFIG_CMA */
1594static __always_inline long __gup_longterm_locked(struct task_struct *tsk,
1595 struct mm_struct *mm,
1596 unsigned long start,
1597 unsigned long nr_pages,
1598 struct page **pages,
1599 struct vm_area_struct **vmas,
1600 unsigned int flags)
1601{
1602 return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
1603 NULL, flags);
1604}
1605#endif /* CONFIG_FS_DAX || CONFIG_CMA */
1606
1607/*
1608 * This is the same as get_user_pages_remote(), just with a
1609 * less-flexible calling convention where we assume that the task
1610 * and mm being operated on are the current task's and don't allow
1611 * passing of a locked parameter. We also obviously don't pass
1612 * FOLL_REMOTE in here.
1613 */
1614long get_user_pages(unsigned long start, unsigned long nr_pages,
1615 unsigned int gup_flags, struct page **pages,
1616 struct vm_area_struct **vmas)
1617{
1618 return __gup_longterm_locked(current, current->mm, start, nr_pages,
1619 pages, vmas, gup_flags | FOLL_TOUCH);
1620}
1621EXPORT_SYMBOL(get_user_pages);
2bb6d283 1622
d3649f68
CH
1623/*
1624 * We can leverage the VM_FAULT_RETRY functionality in the page fault
1625 * paths better by using either get_user_pages_locked() or
1626 * get_user_pages_unlocked().
acc3c8d1 1627 *
d3649f68 1628 * get_user_pages_locked() is suitable to replace the form:
acc3c8d1 1629 *
d3649f68
CH
1630 * down_read(&mm->mmap_sem);
1631 * do_something()
1632 * get_user_pages(tsk, mm, ..., pages, NULL);
1633 * up_read(&mm->mmap_sem);
acc3c8d1 1634 *
d3649f68 1635 * to:
acc3c8d1 1636 *
d3649f68
CH
1637 * int locked = 1;
1638 * down_read(&mm->mmap_sem);
1639 * do_something()
1640 * get_user_pages_locked(tsk, mm, ..., pages, &locked);
1641 * if (locked)
1642 * up_read(&mm->mmap_sem);
acc3c8d1 1643 */
d3649f68
CH
1644long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
1645 unsigned int gup_flags, struct page **pages,
1646 int *locked)
acc3c8d1 1647{
acc3c8d1 1648 /*
d3649f68
CH
1649 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1650 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1651 * vmas. As there are no users of this flag in this call we simply
1652 * disallow this option for now.
acc3c8d1 1653 */
d3649f68
CH
1654 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1655 return -EINVAL;
acc3c8d1 1656
d3649f68
CH
1657 return __get_user_pages_locked(current, current->mm, start, nr_pages,
1658 pages, NULL, locked,
1659 gup_flags | FOLL_TOUCH);
acc3c8d1 1660}
d3649f68 1661EXPORT_SYMBOL(get_user_pages_locked);
acc3c8d1
KS
1662
1663/*
d3649f68 1664 * get_user_pages_unlocked() is suitable to replace the form:
acc3c8d1 1665 *
d3649f68
CH
1666 * down_read(&mm->mmap_sem);
1667 * get_user_pages(tsk, mm, ..., pages, NULL);
1668 * up_read(&mm->mmap_sem);
1669 *
1670 * with:
1671 *
1672 * get_user_pages_unlocked(tsk, mm, ..., pages);
1673 *
1674 * It is functionally equivalent to get_user_pages_fast so
1675 * get_user_pages_fast should be used instead if specific gup_flags
1676 * (e.g. FOLL_FORCE) are not required.
acc3c8d1 1677 */
d3649f68
CH
1678long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
1679 struct page **pages, unsigned int gup_flags)
acc3c8d1
KS
1680{
1681 struct mm_struct *mm = current->mm;
d3649f68
CH
1682 int locked = 1;
1683 long ret;
acc3c8d1 1684
d3649f68
CH
1685 /*
1686 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
1687 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
1688 * vmas. As there are no users of this flag in this call we simply
1689 * disallow this option for now.
1690 */
1691 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1692 return -EINVAL;
acc3c8d1 1693
d3649f68
CH
1694 down_read(&mm->mmap_sem);
1695 ret = __get_user_pages_locked(current, mm, start, nr_pages, pages, NULL,
1696 &locked, gup_flags | FOLL_TOUCH);
acc3c8d1
KS
1697 if (locked)
1698 up_read(&mm->mmap_sem);
d3649f68 1699 return ret;
4bbd4c77 1700}
d3649f68 1701EXPORT_SYMBOL(get_user_pages_unlocked);
2667f50e
SC
1702
1703/*
67a929e0 1704 * Fast GUP
2667f50e
SC
1705 *
1706 * get_user_pages_fast attempts to pin user pages by walking the page
1707 * tables directly and avoids taking locks. Thus the walker needs to be
1708 * protected from page table pages being freed from under it, and should
1709 * block any THP splits.
1710 *
1711 * One way to achieve this is to have the walker disable interrupts, and
1712 * rely on IPIs from the TLB flushing code blocking before the page table
1713 * pages are freed. This is unsuitable for architectures that do not need
1714 * to broadcast an IPI when invalidating TLBs.
1715 *
1716 * Another way to achieve this is to batch up page table containing pages
1717 * belonging to more than one mm_user, then rcu_sched a callback to free those
1718 * pages. Disabling interrupts will allow the fast_gup walker to both block
1719 * the rcu_sched callback, and an IPI that we broadcast for splitting THPs
1720 * (which is a relatively rare event). The code below adopts this strategy.
1721 *
1722 * Before activating this code, please be aware that the following assumptions
1723 * are currently made:
1724 *
e585513b
KS
1725 * *) Either HAVE_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
1726 * free pages containing page tables or TLB flushing requires IPI broadcast.
2667f50e 1727 *
2667f50e
SC
1728 * *) ptes can be read atomically by the architecture.
1729 *
1730 * *) access_ok is sufficient to validate userspace address ranges.
1731 *
1732 * The last two assumptions can be relaxed by the addition of helper functions.
1733 *
1734 * This code is based heavily on the PowerPC implementation by Nick Piggin.
1735 */
67a929e0 1736#ifdef CONFIG_HAVE_FAST_GUP
39656e83
CH
1737#ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
1738/*
1739 * WARNING: only to be used in the get_user_pages_fast() implementation.
1740 *
1741 * With get_user_pages_fast(), we walk down the pagetables without taking any
1742 * locks. For this we would like to load the pointers atomically, but sometimes
1743 * that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE). What
1744 * we do have is the guarantee that a PTE will only either go from not present
1745 * to present, or present to not present or both -- it will not switch to a
1746 * completely different present page without a TLB flush in between; something
1747 * that we are blocking by holding interrupts off.
1748 *
1749 * Setting ptes from not present to present goes:
1750 *
1751 * ptep->pte_high = h;
1752 * smp_wmb();
1753 * ptep->pte_low = l;
1754 *
1755 * And present to not present goes:
1756 *
1757 * ptep->pte_low = 0;
1758 * smp_wmb();
1759 * ptep->pte_high = 0;
1760 *
1761 * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
1762 * We load pte_high *after* loading pte_low, which ensures we don't see an older
1763 * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't
1764 * picked up a changed pte high. We might have gotten rubbish values from
1765 * pte_low and pte_high, but we are guaranteed that pte_low will not have the
1766 * present bit set *unless* it is 'l'. Because get_user_pages_fast() only
1767 * operates on present ptes we're safe.
1768 */
1769static inline pte_t gup_get_pte(pte_t *ptep)
1770{
1771 pte_t pte;
2667f50e 1772
39656e83
CH
1773 do {
1774 pte.pte_low = ptep->pte_low;
1775 smp_rmb();
1776 pte.pte_high = ptep->pte_high;
1777 smp_rmb();
1778 } while (unlikely(pte.pte_low != ptep->pte_low));
1779
1780 return pte;
1781}
1782#else /* CONFIG_GUP_GET_PTE_LOW_HIGH */
0005d20b 1783/*
39656e83 1784 * We require that the PTE can be read atomically.
0005d20b
KS
1785 */
1786static inline pte_t gup_get_pte(pte_t *ptep)
1787{
1788 return READ_ONCE(*ptep);
1789}
39656e83 1790#endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
0005d20b 1791
790c7369
GR
1792static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
1793 struct page **pages)
b59f65fa
KS
1794{
1795 while ((*nr) - nr_start) {
1796 struct page *page = pages[--(*nr)];
1797
1798 ClearPageReferenced(page);
1799 put_page(page);
1800 }
1801}
1802
8fde12ca
LT
1803/*
1804 * Return the compund head page with ref appropriately incremented,
1805 * or NULL if that failed.
1806 */
1807static inline struct page *try_get_compound_head(struct page *page, int refs)
1808{
1809 struct page *head = compound_head(page);
1810 if (WARN_ON_ONCE(page_ref_count(head) < 0))
1811 return NULL;
1812 if (unlikely(!page_cache_add_speculative(head, refs)))
1813 return NULL;
1814 return head;
1815}
1816
3010a5ea 1817#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
2667f50e 1818static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
b798bec4 1819 unsigned int flags, struct page **pages, int *nr)
2667f50e 1820{
b59f65fa
KS
1821 struct dev_pagemap *pgmap = NULL;
1822 int nr_start = *nr, ret = 0;
2667f50e 1823 pte_t *ptep, *ptem;
2667f50e
SC
1824
1825 ptem = ptep = pte_offset_map(&pmd, addr);
1826 do {
0005d20b 1827 pte_t pte = gup_get_pte(ptep);
7aef4172 1828 struct page *head, *page;
2667f50e
SC
1829
1830 /*
1831 * Similar to the PMD case below, NUMA hinting must take slow
8a0516ed 1832 * path using the pte_protnone check.
2667f50e 1833 */
e7884f8e
KS
1834 if (pte_protnone(pte))
1835 goto pte_unmap;
1836
b798bec4 1837 if (!pte_access_permitted(pte, flags & FOLL_WRITE))
e7884f8e
KS
1838 goto pte_unmap;
1839
b59f65fa 1840 if (pte_devmap(pte)) {
7af75561
IW
1841 if (unlikely(flags & FOLL_LONGTERM))
1842 goto pte_unmap;
1843
b59f65fa
KS
1844 pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
1845 if (unlikely(!pgmap)) {
1846 undo_dev_pagemap(nr, nr_start, pages);
1847 goto pte_unmap;
1848 }
1849 } else if (pte_special(pte))
2667f50e
SC
1850 goto pte_unmap;
1851
1852 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
1853 page = pte_page(pte);
1854
8fde12ca
LT
1855 head = try_get_compound_head(page, 1);
1856 if (!head)
2667f50e
SC
1857 goto pte_unmap;
1858
1859 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
7aef4172 1860 put_page(head);
2667f50e
SC
1861 goto pte_unmap;
1862 }
1863
7aef4172 1864 VM_BUG_ON_PAGE(compound_head(page) != head, page);
e9348053
KS
1865
1866 SetPageReferenced(page);
2667f50e
SC
1867 pages[*nr] = page;
1868 (*nr)++;
1869
1870 } while (ptep++, addr += PAGE_SIZE, addr != end);
1871
1872 ret = 1;
1873
1874pte_unmap:
832d7aa0
CH
1875 if (pgmap)
1876 put_dev_pagemap(pgmap);
2667f50e
SC
1877 pte_unmap(ptem);
1878 return ret;
1879}
1880#else
1881
1882/*
1883 * If we can't determine whether or not a pte is special, then fail immediately
1884 * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
1885 * to be special.
1886 *
1887 * For a futex to be placed on a THP tail page, get_futex_key requires a
1888 * __get_user_pages_fast implementation that can pin pages. Thus it's still
1889 * useful to have gup_huge_pmd even if we can't operate on ptes.
1890 */
1891static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
b798bec4 1892 unsigned int flags, struct page **pages, int *nr)
2667f50e
SC
1893{
1894 return 0;
1895}
3010a5ea 1896#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
2667f50e 1897
17596731 1898#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
b59f65fa
KS
1899static int __gup_device_huge(unsigned long pfn, unsigned long addr,
1900 unsigned long end, struct page **pages, int *nr)
1901{
1902 int nr_start = *nr;
1903 struct dev_pagemap *pgmap = NULL;
1904
1905 do {
1906 struct page *page = pfn_to_page(pfn);
1907
1908 pgmap = get_dev_pagemap(pfn, pgmap);
1909 if (unlikely(!pgmap)) {
1910 undo_dev_pagemap(nr, nr_start, pages);
1911 return 0;
1912 }
1913 SetPageReferenced(page);
1914 pages[*nr] = page;
1915 get_page(page);
b59f65fa
KS
1916 (*nr)++;
1917 pfn++;
1918 } while (addr += PAGE_SIZE, addr != end);
832d7aa0
CH
1919
1920 if (pgmap)
1921 put_dev_pagemap(pgmap);
b59f65fa
KS
1922 return 1;
1923}
1924
a9b6de77 1925static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
b59f65fa
KS
1926 unsigned long end, struct page **pages, int *nr)
1927{
1928 unsigned long fault_pfn;
a9b6de77
DW
1929 int nr_start = *nr;
1930
1931 fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1932 if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))
1933 return 0;
b59f65fa 1934
a9b6de77
DW
1935 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
1936 undo_dev_pagemap(nr, nr_start, pages);
1937 return 0;
1938 }
1939 return 1;
b59f65fa
KS
1940}
1941
a9b6de77 1942static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
b59f65fa
KS
1943 unsigned long end, struct page **pages, int *nr)
1944{
1945 unsigned long fault_pfn;
a9b6de77
DW
1946 int nr_start = *nr;
1947
1948 fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
1949 if (!__gup_device_huge(fault_pfn, addr, end, pages, nr))
1950 return 0;
b59f65fa 1951
a9b6de77
DW
1952 if (unlikely(pud_val(orig) != pud_val(*pudp))) {
1953 undo_dev_pagemap(nr, nr_start, pages);
1954 return 0;
1955 }
1956 return 1;
b59f65fa
KS
1957}
1958#else
a9b6de77 1959static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
b59f65fa
KS
1960 unsigned long end, struct page **pages, int *nr)
1961{
1962 BUILD_BUG();
1963 return 0;
1964}
1965
a9b6de77 1966static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
b59f65fa
KS
1967 unsigned long end, struct page **pages, int *nr)
1968{
1969 BUILD_BUG();
1970 return 0;
1971}
1972#endif
1973
cbd34da7
CH
1974#ifdef CONFIG_ARCH_HAS_HUGEPD
1975static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
1976 unsigned long sz)
1977{
1978 unsigned long __boundary = (addr + sz) & ~(sz-1);
1979 return (__boundary - 1 < end - 1) ? __boundary : end;
1980}
1981
1982static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
1983 unsigned long end, int write, struct page **pages, int *nr)
1984{
1985 unsigned long pte_end;
1986 struct page *head, *page;
1987 pte_t pte;
1988 int refs;
1989
1990 pte_end = (addr + sz) & ~(sz-1);
1991 if (pte_end < end)
1992 end = pte_end;
1993
1994 pte = READ_ONCE(*ptep);
1995
1996 if (!pte_access_permitted(pte, write))
1997 return 0;
1998
1999 /* hugepages are never "special" */
2000 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2001
2002 refs = 0;
2003 head = pte_page(pte);
2004
2005 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2006 do {
2007 VM_BUG_ON(compound_head(page) != head);
2008 pages[*nr] = page;
2009 (*nr)++;
2010 page++;
2011 refs++;
2012 } while (addr += PAGE_SIZE, addr != end);
2013
01a36916
CH
2014 head = try_get_compound_head(head, refs);
2015 if (!head) {
cbd34da7
CH
2016 *nr -= refs;
2017 return 0;
2018 }
2019
2020 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2021 /* Could be optimized better */
2022 *nr -= refs;
2023 while (refs--)
2024 put_page(head);
2025 return 0;
2026 }
2027
520b4a44 2028 SetPageReferenced(head);
cbd34da7
CH
2029 return 1;
2030}
2031
2032static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2033 unsigned int pdshift, unsigned long end, int write,
2034 struct page **pages, int *nr)
2035{
2036 pte_t *ptep;
2037 unsigned long sz = 1UL << hugepd_shift(hugepd);
2038 unsigned long next;
2039
2040 ptep = hugepte_offset(hugepd, addr, pdshift);
2041 do {
2042 next = hugepte_addr_end(addr, end, sz);
2043 if (!gup_hugepte(ptep, sz, addr, end, write, pages, nr))
2044 return 0;
2045 } while (ptep++, addr = next, addr != end);
2046
2047 return 1;
2048}
2049#else
2050static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2051 unsigned pdshift, unsigned long end, int write,
2052 struct page **pages, int *nr)
2053{
2054 return 0;
2055}
2056#endif /* CONFIG_ARCH_HAS_HUGEPD */
2057
2667f50e 2058static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
b798bec4 2059 unsigned long end, unsigned int flags, struct page **pages, int *nr)
2667f50e 2060{
ddc58f27 2061 struct page *head, *page;
2667f50e
SC
2062 int refs;
2063
b798bec4 2064 if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
2667f50e
SC
2065 return 0;
2066
7af75561
IW
2067 if (pmd_devmap(orig)) {
2068 if (unlikely(flags & FOLL_LONGTERM))
2069 return 0;
a9b6de77 2070 return __gup_device_huge_pmd(orig, pmdp, addr, end, pages, nr);
7af75561 2071 }
b59f65fa 2072
2667f50e 2073 refs = 0;
d63206ee 2074 page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2667f50e 2075 do {
2667f50e
SC
2076 pages[*nr] = page;
2077 (*nr)++;
2078 page++;
2079 refs++;
2080 } while (addr += PAGE_SIZE, addr != end);
2081
8fde12ca
LT
2082 head = try_get_compound_head(pmd_page(orig), refs);
2083 if (!head) {
2667f50e
SC
2084 *nr -= refs;
2085 return 0;
2086 }
2087
2088 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2089 *nr -= refs;
2090 while (refs--)
2091 put_page(head);
2092 return 0;
2093 }
2094
e9348053 2095 SetPageReferenced(head);
2667f50e
SC
2096 return 1;
2097}
2098
2099static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
b798bec4 2100 unsigned long end, unsigned int flags, struct page **pages, int *nr)
2667f50e 2101{
ddc58f27 2102 struct page *head, *page;
2667f50e
SC
2103 int refs;
2104
b798bec4 2105 if (!pud_access_permitted(orig, flags & FOLL_WRITE))
2667f50e
SC
2106 return 0;
2107
7af75561
IW
2108 if (pud_devmap(orig)) {
2109 if (unlikely(flags & FOLL_LONGTERM))
2110 return 0;
a9b6de77 2111 return __gup_device_huge_pud(orig, pudp, addr, end, pages, nr);
7af75561 2112 }
b59f65fa 2113
2667f50e 2114 refs = 0;
d63206ee 2115 page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2667f50e 2116 do {
2667f50e
SC
2117 pages[*nr] = page;
2118 (*nr)++;
2119 page++;
2120 refs++;
2121 } while (addr += PAGE_SIZE, addr != end);
2122
8fde12ca
LT
2123 head = try_get_compound_head(pud_page(orig), refs);
2124 if (!head) {
2667f50e
SC
2125 *nr -= refs;
2126 return 0;
2127 }
2128
2129 if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2130 *nr -= refs;
2131 while (refs--)
2132 put_page(head);
2133 return 0;
2134 }
2135
e9348053 2136 SetPageReferenced(head);
2667f50e
SC
2137 return 1;
2138}
2139
f30c59e9 2140static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
b798bec4 2141 unsigned long end, unsigned int flags,
f30c59e9
AK
2142 struct page **pages, int *nr)
2143{
2144 int refs;
ddc58f27 2145 struct page *head, *page;
f30c59e9 2146
b798bec4 2147 if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
f30c59e9
AK
2148 return 0;
2149
b59f65fa 2150 BUILD_BUG_ON(pgd_devmap(orig));
f30c59e9 2151 refs = 0;
d63206ee 2152 page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
f30c59e9 2153 do {
f30c59e9
AK
2154 pages[*nr] = page;
2155 (*nr)++;
2156 page++;
2157 refs++;
2158 } while (addr += PAGE_SIZE, addr != end);
2159
8fde12ca
LT
2160 head = try_get_compound_head(pgd_page(orig), refs);
2161 if (!head) {
f30c59e9
AK
2162 *nr -= refs;
2163 return 0;
2164 }
2165
2166 if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
2167 *nr -= refs;
2168 while (refs--)
2169 put_page(head);
2170 return 0;
2171 }
2172
e9348053 2173 SetPageReferenced(head);
f30c59e9
AK
2174 return 1;
2175}
2176
2667f50e 2177static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
b798bec4 2178 unsigned int flags, struct page **pages, int *nr)
2667f50e
SC
2179{
2180 unsigned long next;
2181 pmd_t *pmdp;
2182
2183 pmdp = pmd_offset(&pud, addr);
2184 do {
38c5ce93 2185 pmd_t pmd = READ_ONCE(*pmdp);
2667f50e
SC
2186
2187 next = pmd_addr_end(addr, end);
84c3fc4e 2188 if (!pmd_present(pmd))
2667f50e
SC
2189 return 0;
2190
414fd080
YZ
2191 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
2192 pmd_devmap(pmd))) {
2667f50e
SC
2193 /*
2194 * NUMA hinting faults need to be handled in the GUP
2195 * slowpath for accounting purposes and so that they
2196 * can be serialised against THP migration.
2197 */
8a0516ed 2198 if (pmd_protnone(pmd))
2667f50e
SC
2199 return 0;
2200
b798bec4 2201 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
2667f50e
SC
2202 pages, nr))
2203 return 0;
2204
f30c59e9
AK
2205 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
2206 /*
2207 * architecture have different format for hugetlbfs
2208 * pmd format and THP pmd format
2209 */
2210 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
b798bec4 2211 PMD_SHIFT, next, flags, pages, nr))
f30c59e9 2212 return 0;
b798bec4 2213 } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
2923117b 2214 return 0;
2667f50e
SC
2215 } while (pmdp++, addr = next, addr != end);
2216
2217 return 1;
2218}
2219
c2febafc 2220static int gup_pud_range(p4d_t p4d, unsigned long addr, unsigned long end,
b798bec4 2221 unsigned int flags, struct page **pages, int *nr)
2667f50e
SC
2222{
2223 unsigned long next;
2224 pud_t *pudp;
2225
c2febafc 2226 pudp = pud_offset(&p4d, addr);
2667f50e 2227 do {
e37c6982 2228 pud_t pud = READ_ONCE(*pudp);
2667f50e
SC
2229
2230 next = pud_addr_end(addr, end);
2231 if (pud_none(pud))
2232 return 0;
f30c59e9 2233 if (unlikely(pud_huge(pud))) {
b798bec4 2234 if (!gup_huge_pud(pud, pudp, addr, next, flags,
f30c59e9
AK
2235 pages, nr))
2236 return 0;
2237 } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
2238 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
b798bec4 2239 PUD_SHIFT, next, flags, pages, nr))
2667f50e 2240 return 0;
b798bec4 2241 } else if (!gup_pmd_range(pud, addr, next, flags, pages, nr))
2667f50e
SC
2242 return 0;
2243 } while (pudp++, addr = next, addr != end);
2244
2245 return 1;
2246}
2247
c2febafc 2248static int gup_p4d_range(pgd_t pgd, unsigned long addr, unsigned long end,
b798bec4 2249 unsigned int flags, struct page **pages, int *nr)
c2febafc
KS
2250{
2251 unsigned long next;
2252 p4d_t *p4dp;
2253
2254 p4dp = p4d_offset(&pgd, addr);
2255 do {
2256 p4d_t p4d = READ_ONCE(*p4dp);
2257
2258 next = p4d_addr_end(addr, end);
2259 if (p4d_none(p4d))
2260 return 0;
2261 BUILD_BUG_ON(p4d_huge(p4d));
2262 if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
2263 if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
b798bec4 2264 P4D_SHIFT, next, flags, pages, nr))
c2febafc 2265 return 0;
b798bec4 2266 } else if (!gup_pud_range(p4d, addr, next, flags, pages, nr))
c2febafc
KS
2267 return 0;
2268 } while (p4dp++, addr = next, addr != end);
2269
2270 return 1;
2271}
2272
5b65c467 2273static void gup_pgd_range(unsigned long addr, unsigned long end,
b798bec4 2274 unsigned int flags, struct page **pages, int *nr)
5b65c467
KS
2275{
2276 unsigned long next;
2277 pgd_t *pgdp;
2278
2279 pgdp = pgd_offset(current->mm, addr);
2280 do {
2281 pgd_t pgd = READ_ONCE(*pgdp);
2282
2283 next = pgd_addr_end(addr, end);
2284 if (pgd_none(pgd))
2285 return;
2286 if (unlikely(pgd_huge(pgd))) {
b798bec4 2287 if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
5b65c467
KS
2288 pages, nr))
2289 return;
2290 } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
2291 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
b798bec4 2292 PGDIR_SHIFT, next, flags, pages, nr))
5b65c467 2293 return;
b798bec4 2294 } else if (!gup_p4d_range(pgd, addr, next, flags, pages, nr))
5b65c467
KS
2295 return;
2296 } while (pgdp++, addr = next, addr != end);
2297}
050a9adc
CH
2298#else
2299static inline void gup_pgd_range(unsigned long addr, unsigned long end,
2300 unsigned int flags, struct page **pages, int *nr)
2301{
2302}
2303#endif /* CONFIG_HAVE_FAST_GUP */
5b65c467
KS
2304
2305#ifndef gup_fast_permitted
2306/*
2307 * Check if it's allowed to use __get_user_pages_fast() for the range, or
2308 * we need to fall back to the slow version:
2309 */
26f4c328 2310static bool gup_fast_permitted(unsigned long start, unsigned long end)
5b65c467 2311{
26f4c328 2312 return true;
5b65c467
KS
2313}
2314#endif
2315
2667f50e
SC
2316/*
2317 * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
d0811078
MT
2318 * the regular GUP.
2319 * Note a difference with get_user_pages_fast: this always returns the
2320 * number of pages pinned, 0 if no pages were pinned.
050a9adc
CH
2321 *
2322 * If the architecture does not support this function, simply return with no
2323 * pages pinned.
2667f50e
SC
2324 */
2325int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
2326 struct page **pages)
2327{
d4faa402 2328 unsigned long len, end;
5b65c467 2329 unsigned long flags;
2667f50e
SC
2330 int nr = 0;
2331
f455c854 2332 start = untagged_addr(start) & PAGE_MASK;
2667f50e
SC
2333 len = (unsigned long) nr_pages << PAGE_SHIFT;
2334 end = start + len;
2335
26f4c328
CH
2336 if (end <= start)
2337 return 0;
96d4f267 2338 if (unlikely(!access_ok((void __user *)start, len)))
2667f50e
SC
2339 return 0;
2340
2341 /*
2342 * Disable interrupts. We use the nested form as we can already have
2343 * interrupts disabled by get_futex_key.
2344 *
2345 * With interrupts disabled, we block page table pages from being
2ebe8228
FW
2346 * freed from under us. See struct mmu_table_batch comments in
2347 * include/asm-generic/tlb.h for more details.
2667f50e
SC
2348 *
2349 * We do not adopt an rcu_read_lock(.) here as we also want to
2350 * block IPIs that come from THPs splitting.
2351 */
2352
050a9adc
CH
2353 if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&
2354 gup_fast_permitted(start, end)) {
5b65c467 2355 local_irq_save(flags);
b798bec4 2356 gup_pgd_range(start, end, write ? FOLL_WRITE : 0, pages, &nr);
5b65c467
KS
2357 local_irq_restore(flags);
2358 }
2667f50e
SC
2359
2360 return nr;
2361}
050a9adc 2362EXPORT_SYMBOL_GPL(__get_user_pages_fast);
2667f50e 2363
7af75561
IW
2364static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
2365 unsigned int gup_flags, struct page **pages)
2366{
2367 int ret;
2368
2369 /*
2370 * FIXME: FOLL_LONGTERM does not work with
2371 * get_user_pages_unlocked() (see comments in that function)
2372 */
2373 if (gup_flags & FOLL_LONGTERM) {
2374 down_read(&current->mm->mmap_sem);
2375 ret = __gup_longterm_locked(current, current->mm,
2376 start, nr_pages,
2377 pages, NULL, gup_flags);
2378 up_read(&current->mm->mmap_sem);
2379 } else {
2380 ret = get_user_pages_unlocked(start, nr_pages,
2381 pages, gup_flags);
2382 }
2383
2384 return ret;
2385}
2386
2667f50e
SC
2387/**
2388 * get_user_pages_fast() - pin user pages in memory
2389 * @start: starting user address
2390 * @nr_pages: number of pages from start to pin
73b0140b 2391 * @gup_flags: flags modifying pin behaviour
2667f50e
SC
2392 * @pages: array that receives pointers to the pages pinned.
2393 * Should be at least nr_pages long.
2394 *
2395 * Attempt to pin user pages in memory without taking mm->mmap_sem.
2396 * If not successful, it will fall back to taking the lock and
2397 * calling get_user_pages().
2398 *
2399 * Returns number of pages pinned. This may be fewer than the number
2400 * requested. If nr_pages is 0 or negative, returns 0. If no pages
2401 * were pinned, returns -errno.
2402 */
73b0140b
IW
2403int get_user_pages_fast(unsigned long start, int nr_pages,
2404 unsigned int gup_flags, struct page **pages)
2667f50e 2405{
5b65c467 2406 unsigned long addr, len, end;
73e10a61 2407 int nr = 0, ret = 0;
2667f50e 2408
817be129
CH
2409 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM)))
2410 return -EINVAL;
2411
f455c854 2412 start = untagged_addr(start) & PAGE_MASK;
5b65c467
KS
2413 addr = start;
2414 len = (unsigned long) nr_pages << PAGE_SHIFT;
2415 end = start + len;
2416
26f4c328 2417 if (end <= start)
c61611f7 2418 return 0;
96d4f267 2419 if (unlikely(!access_ok((void __user *)start, len)))
c61611f7 2420 return -EFAULT;
73e10a61 2421
050a9adc
CH
2422 if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&
2423 gup_fast_permitted(start, end)) {
5b65c467 2424 local_irq_disable();
73b0140b 2425 gup_pgd_range(addr, end, gup_flags, pages, &nr);
5b65c467 2426 local_irq_enable();
73e10a61
KS
2427 ret = nr;
2428 }
2667f50e
SC
2429
2430 if (nr < nr_pages) {
2431 /* Try to get the remaining pages with get_user_pages */
2432 start += nr << PAGE_SHIFT;
2433 pages += nr;
2434
7af75561
IW
2435 ret = __gup_longterm_unlocked(start, nr_pages - nr,
2436 gup_flags, pages);
2667f50e
SC
2437
2438 /* Have to be a bit careful with return values */
2439 if (nr > 0) {
2440 if (ret < 0)
2441 ret = nr;
2442 else
2443 ret += nr;
2444 }
2445 }
2446
2447 return ret;
2448}
050a9adc 2449EXPORT_SYMBOL_GPL(get_user_pages_fast);