mm/hmm.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright 2013 Red Hat Inc.
   4  *
   5  * Authors: Jérôme Glisse <jglisse@redhat.com>
   6  */
   7 /*
   8  * Refer to include/linux/hmm.h for information about heterogeneous memory
   9  * management or HMM for short.
  10  */
  11 #include <linux/pagewalk.h>
  12 #include <linux/hmm.h>
  13 #include <linux/init.h>
  14 #include <linux/rmap.h>
  15 #include <linux/swap.h>
  16 #include <linux/slab.h>
  17 #include <linux/sched.h>
  18 #include <linux/mmzone.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/swapops.h>
  21 #include <linux/hugetlb.h>
  22 #include <linux/memremap.h>
  23 #include <linux/sched/mm.h>
  24 #include <linux/jump_label.h>
  25 #include <linux/dma-mapping.h>
  26 #include <linux/mmu_notifier.h>
  27 #include <linux/memory_hotplug.h>
  28
  29 struct hmm_vma_walk {
  30         struct hmm_range        *range;
  31         struct dev_pagemap      *pgmap;
  32         unsigned long           last;
  33         unsigned int            flags;
  34 };
  35
  36 static int hmm_pfns_fill(unsigned long addr, unsigned long end,
  37                 struct hmm_range *range, enum hmm_pfn_value_e value)
  38 {
  39         uint64_t *pfns = range->pfns;
  40         unsigned long i;
  41
  42         i = (addr - range->start) >> PAGE_SHIFT;
  43         for (; addr < end; addr += PAGE_SIZE, i++)
  44                 pfns[i] = range->values[value];
  45
  46         return 0;
  47 }
  48
  49 /*
  50  * hmm_vma_fault() - fault in a range lacking valid pmd or pte(s)
  51  * @addr: range virtual start address (inclusive)
  52  * @end: range virtual end address (exclusive)
  53  * @fault: should we fault or not ?
  54  * @write_fault: write fault ?
  55  * @walk: mm_walk structure
  56  * Return: -EBUSY after page fault, or page fault error
  57  *
  58  * This function will be called whenever pmd_none() or pte_none() returns true,
  59  * or whenever there is no page directory covering the virtual address range.
  60  */
  61 static int hmm_vma_fault(unsigned long addr, unsigned long end,
  62                               bool fault, bool write_fault,
  63                               struct mm_walk *walk)
  64 {
  65         struct hmm_vma_walk *hmm_vma_walk = walk->private;
  66         struct hmm_range *range = hmm_vma_walk->range;
  67         struct vm_area_struct *vma = walk->vma;
  68         uint64_t *pfns = range->pfns;
  69         unsigned long i = (addr - range->start) >> PAGE_SHIFT;
  70         unsigned int fault_flags = FAULT_FLAG_REMOTE;
  71
  72         WARN_ON_ONCE(!fault && !write_fault);
  73         hmm_vma_walk->last = addr;
  74
  75         if (!vma)
  76                 goto out_error;
  77
  78         if (write_fault) {
  79                 if (!(vma->vm_flags & VM_WRITE))
  80                         return -EPERM;
  81                 fault_flags |= FAULT_FLAG_WRITE;
  82         }
  83
  84         for (; addr < end; addr += PAGE_SIZE, i++)
  85                 if (handle_mm_fault(vma, addr, fault_flags) & VM_FAULT_ERROR)
  86                         goto out_error;
  87
  88         return -EBUSY;
  89
  90 out_error:
  91         pfns[i] = range->values[HMM_PFN_ERROR];
  92         return -EFAULT;
  93 }
  94
  95 static inline void hmm_pte_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
  96                                       uint64_t pfns, uint64_t cpu_flags,
  97                                       bool *fault, bool *write_fault)
  98 {
  99         struct hmm_range *range = hmm_vma_walk->range;
 100
 101         if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT)
 102                 return;
 103
 104         /*
 105          * So we not only consider the individual per page request we also
 106          * consider the default flags requested for the range. The API can
 107          * be used 2 ways. The first one where the HMM user coalesces
 108          * multiple page faults into one request and sets flags per pfn for
 109          * those faults. The second one where the HMM user wants to pre-
 110          * fault a range with specific flags. For the latter one it is a
 111          * waste to have the user pre-fill the pfn arrays with a default
 112          * flags value.
 113          */
 114         pfns = (pfns & range->pfn_flags_mask) | range->default_flags;
 115
 116         /* We aren't ask to do anything ... */
 117         if (!(pfns & range->flags[HMM_PFN_VALID]))
 118                 return;
 119
 120         /* If CPU page table is not valid then we need to fault */
 121         *fault = !(cpu_flags & range->flags[HMM_PFN_VALID]);
 122         /* Need to write fault ? */
 123         if ((pfns & range->flags[HMM_PFN_WRITE]) &&
 124             !(cpu_flags & range->flags[HMM_PFN_WRITE])) {
 125                 *write_fault = true;
 126                 *fault = true;
 127         }
 128 }
 129
 130 static void hmm_range_need_fault(const struct hmm_vma_walk *hmm_vma_walk,
 131                                  const uint64_t *pfns, unsigned long npages,
 132                                  uint64_t cpu_flags, bool *fault,
 133                                  bool *write_fault)
 134 {
 135         unsigned long i;
 136
 137         if (hmm_vma_walk->flags & HMM_FAULT_SNAPSHOT) {
 138                 *fault = *write_fault = false;
 139                 return;
 140         }
 141
 142         *fault = *write_fault = false;
 143         for (i = 0; i < npages; ++i) {
 144                 hmm_pte_need_fault(hmm_vma_walk, pfns[i], cpu_flags,
 145                                    fault, write_fault);
 146                 if ((*write_fault))
 147                         return;
 148         }
 149 }
 150
 151 static int hmm_vma_walk_hole(unsigned long addr, unsigned long end,
 152                              __always_unused int depth, struct mm_walk *walk)
 153 {
 154         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 155         struct hmm_range *range = hmm_vma_walk->range;
 156         bool fault, write_fault;
 157         unsigned long i, npages;
 158         uint64_t *pfns;
 159
 160         i = (addr - range->start) >> PAGE_SHIFT;
 161         npages = (end - addr) >> PAGE_SHIFT;
 162         pfns = &range->pfns[i];
 163         hmm_range_need_fault(hmm_vma_walk, pfns, npages,
 164                              0, &fault, &write_fault);
 165         if (fault || write_fault)
 166                 return hmm_vma_fault(addr, end, fault, write_fault, walk);
 167         hmm_vma_walk->last = addr;
 168         return hmm_pfns_fill(addr, end, range, HMM_PFN_NONE);
 169 }
 170
 171 static inline uint64_t pmd_to_hmm_pfn_flags(struct hmm_range *range, pmd_t pmd)
 172 {
 173         if (pmd_protnone(pmd))
 174                 return 0;
 175         return pmd_write(pmd) ? range->flags[HMM_PFN_VALID] |
 176                                 range->flags[HMM_PFN_WRITE] :
 177                                 range->flags[HMM_PFN_VALID];
 178 }
 179
 180 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 181 static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
 182                 unsigned long end, uint64_t *pfns, pmd_t pmd)
 183 {
 184         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 185         struct hmm_range *range = hmm_vma_walk->range;
 186         unsigned long pfn, npages, i;
 187         bool fault, write_fault;
 188         uint64_t cpu_flags;
 189
 190         npages = (end - addr) >> PAGE_SHIFT;
 191         cpu_flags = pmd_to_hmm_pfn_flags(range, pmd);
 192         hmm_range_need_fault(hmm_vma_walk, pfns, npages, cpu_flags,
 193                              &fault, &write_fault);
 194
 195         if (fault || write_fault)
 196                 return hmm_vma_fault(addr, end, fault, write_fault, walk);
 197
 198         pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
 199         for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
 200                 if (pmd_devmap(pmd)) {
 201                         hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
 202                                               hmm_vma_walk->pgmap);
 203                         if (unlikely(!hmm_vma_walk->pgmap))
 204                                 return -EBUSY;
 205                 }
 206                 pfns[i] = hmm_device_entry_from_pfn(range, pfn) | cpu_flags;
 207         }
 208         if (hmm_vma_walk->pgmap) {
 209                 put_dev_pagemap(hmm_vma_walk->pgmap);
 210                 hmm_vma_walk->pgmap = NULL;
 211         }
 212         hmm_vma_walk->last = end;
 213         return 0;
 214 }
 215 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
 216 /* stub to allow the code below to compile */
 217 int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
 218                 unsigned long end, uint64_t *pfns, pmd_t pmd);
 219 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 220
 221 static inline bool hmm_is_device_private_entry(struct hmm_range *range,
 222                 swp_entry_t entry)
 223 {
 224         return is_device_private_entry(entry) &&
 225                 device_private_entry_to_page(entry)->pgmap->owner ==
 226                 range->dev_private_owner;
 227 }
 228
 229 static inline uint64_t pte_to_hmm_pfn_flags(struct hmm_range *range, pte_t pte)
 230 {
 231         if (pte_none(pte) || !pte_present(pte) || pte_protnone(pte))
 232                 return 0;
 233         return pte_write(pte) ? range->flags[HMM_PFN_VALID] |
 234                                 range->flags[HMM_PFN_WRITE] :
 235                                 range->flags[HMM_PFN_VALID];
 236 }
 237
 238 static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
 239                               unsigned long end, pmd_t *pmdp, pte_t *ptep,
 240                               uint64_t *pfn)
 241 {
 242         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 243         struct hmm_range *range = hmm_vma_walk->range;
 244         bool fault, write_fault;
 245         uint64_t cpu_flags;
 246         pte_t pte = *ptep;
 247         uint64_t orig_pfn = *pfn;
 248
 249         *pfn = range->values[HMM_PFN_NONE];
 250         fault = write_fault = false;
 251
 252         if (pte_none(pte)) {
 253                 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0,
 254                                    &fault, &write_fault);
 255                 if (fault || write_fault)
 256                         goto fault;
 257                 return 0;
 258         }
 259
 260         if (!pte_present(pte)) {
 261                 swp_entry_t entry = pte_to_swp_entry(pte);
 262
 263                 /*
 264                  * Never fault in device private pages pages, but just report
 265                  * the PFN even if not present.
 266                  */
 267                 if (hmm_is_device_private_entry(range, entry)) {
 268                         *pfn = hmm_device_entry_from_pfn(range,
 269                                             swp_offset(entry));
 270                         *pfn |= range->flags[HMM_PFN_VALID];
 271                         if (is_write_device_private_entry(entry))
 272                                 *pfn |= range->flags[HMM_PFN_WRITE];
 273                         return 0;
 274                 }
 275
 276                 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0, &fault,
 277                                    &write_fault);
 278                 if (!fault && !write_fault)
 279                         return 0;
 280
 281                 if (!non_swap_entry(entry))
 282                         goto fault;
 283
 284                 if (is_migration_entry(entry)) {
 285                         pte_unmap(ptep);
 286                         hmm_vma_walk->last = addr;
 287                         migration_entry_wait(walk->mm, pmdp, addr);
 288                         return -EBUSY;
 289                 }
 290
 291                 /* Report error for everything else */
 292                 pte_unmap(ptep);
 293                 *pfn = range->values[HMM_PFN_ERROR];
 294                 return -EFAULT;
 295         }
 296
 297         cpu_flags = pte_to_hmm_pfn_flags(range, pte);
 298         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags, &fault,
 299                            &write_fault);
 300         if (fault || write_fault)
 301                 goto fault;
 302
 303         if (pte_devmap(pte)) {
 304                 hmm_vma_walk->pgmap = get_dev_pagemap(pte_pfn(pte),
 305                                               hmm_vma_walk->pgmap);
 306                 if (unlikely(!hmm_vma_walk->pgmap)) {
 307                         pte_unmap(ptep);
 308                         return -EBUSY;
 309                 }
 310         }
 311
 312         /*
 313          * Since each architecture defines a struct page for the zero page, just
 314          * fall through and treat it like a normal page.
 315          */
 316         if (pte_special(pte) && !is_zero_pfn(pte_pfn(pte))) {
 317                 hmm_pte_need_fault(hmm_vma_walk, orig_pfn, 0, &fault,
 318                                    &write_fault);
 319                 if (fault || write_fault) {
 320                         pte_unmap(ptep);
 321                         return -EFAULT;
 322                 }
 323                 *pfn = range->values[HMM_PFN_SPECIAL];
 324                 return 0;
 325         }
 326
 327         *pfn = hmm_device_entry_from_pfn(range, pte_pfn(pte)) | cpu_flags;
 328         return 0;
 329
 330 fault:
 331         if (hmm_vma_walk->pgmap) {
 332                 put_dev_pagemap(hmm_vma_walk->pgmap);
 333                 hmm_vma_walk->pgmap = NULL;
 334         }
 335         pte_unmap(ptep);
 336         /* Fault any virtual address we were asked to fault */
 337         return hmm_vma_fault(addr, end, fault, write_fault, walk);
 338 }
 339
 340 static int hmm_vma_walk_pmd(pmd_t *pmdp,
 341                             unsigned long start,
 342                             unsigned long end,
 343                             struct mm_walk *walk)
 344 {
 345         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 346         struct hmm_range *range = hmm_vma_walk->range;
 347         uint64_t *pfns = &range->pfns[(start - range->start) >> PAGE_SHIFT];
 348         unsigned long npages = (end - start) >> PAGE_SHIFT;
 349         unsigned long addr = start;
 350         bool fault, write_fault;
 351         pte_t *ptep;
 352         pmd_t pmd;
 353
 354 again:
 355         pmd = READ_ONCE(*pmdp);
 356         if (pmd_none(pmd))
 357                 return hmm_vma_walk_hole(start, end, -1, walk);
 358
 359         if (thp_migration_supported() && is_pmd_migration_entry(pmd)) {
 360                 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
 361                                      0, &fault, &write_fault);
 362                 if (fault || write_fault) {
 363                         hmm_vma_walk->last = addr;
 364                         pmd_migration_entry_wait(walk->mm, pmdp);
 365                         return -EBUSY;
 366                 }
 367                 return hmm_pfns_fill(start, end, range, HMM_PFN_NONE);
 368         }
 369
 370         if (!pmd_present(pmd)) {
 371                 hmm_range_need_fault(hmm_vma_walk, pfns, npages, 0, &fault,
 372                                      &write_fault);
 373                 if (fault || write_fault)
 374                         return -EFAULT;
 375                 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
 376         }
 377
 378         if (pmd_devmap(pmd) || pmd_trans_huge(pmd)) {
 379                 /*
 380                  * No need to take pmd_lock here, even if some other thread
 381                  * is splitting the huge pmd we will get that event through
 382                  * mmu_notifier callback.
 383                  *
 384                  * So just read pmd value and check again it's a transparent
 385                  * huge or device mapping one and compute corresponding pfn
 386                  * values.
 387                  */
 388                 pmd = pmd_read_atomic(pmdp);
 389                 barrier();
 390                 if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
 391                         goto again;
 392
 393                 return hmm_vma_handle_pmd(walk, addr, end, pfns, pmd);
 394         }
 395
 396         /*
 397          * We have handled all the valid cases above ie either none, migration,
 398          * huge or transparent huge. At this point either it is a valid pmd
 399          * entry pointing to pte directory or it is a bad pmd that will not
 400          * recover.
 401          */
 402         if (pmd_bad(pmd)) {
 403                 hmm_range_need_fault(hmm_vma_walk, pfns, npages, 0, &fault,
 404                                      &write_fault);
 405                 if (fault || write_fault)
 406                         return -EFAULT;
 407                 return hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
 408         }
 409
 410         ptep = pte_offset_map(pmdp, addr);
 411         for (; addr < end; addr += PAGE_SIZE, ptep++, pfns++) {
 412                 int r;
 413
 414                 r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, pfns);
 415                 if (r) {
 416                         /* hmm_vma_handle_pte() did pte_unmap() */
 417                         hmm_vma_walk->last = addr;
 418                         return r;
 419                 }
 420         }
 421         if (hmm_vma_walk->pgmap) {
 422                 /*
 423                  * We do put_dev_pagemap() here and not in hmm_vma_handle_pte()
 424                  * so that we can leverage get_dev_pagemap() optimization which
 425                  * will not re-take a reference on a pgmap if we already have
 426                  * one.
 427                  */
 428                 put_dev_pagemap(hmm_vma_walk->pgmap);
 429                 hmm_vma_walk->pgmap = NULL;
 430         }
 431         pte_unmap(ptep - 1);
 432
 433         hmm_vma_walk->last = addr;
 434         return 0;
 435 }
 436
 437 #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && \
 438     defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
 439 static inline uint64_t pud_to_hmm_pfn_flags(struct hmm_range *range, pud_t pud)
 440 {
 441         if (!pud_present(pud))
 442                 return 0;
 443         return pud_write(pud) ? range->flags[HMM_PFN_VALID] |
 444                                 range->flags[HMM_PFN_WRITE] :
 445                                 range->flags[HMM_PFN_VALID];
 446 }
 447
 448 static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
 449                 struct mm_walk *walk)
 450 {
 451         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 452         struct hmm_range *range = hmm_vma_walk->range;
 453         unsigned long addr = start;
 454         pud_t pud;
 455         int ret = 0;
 456         spinlock_t *ptl = pud_trans_huge_lock(pudp, walk->vma);
 457
 458         if (!ptl)
 459                 return 0;
 460
 461         /* Normally we don't want to split the huge page */
 462         walk->action = ACTION_CONTINUE;
 463
 464         pud = READ_ONCE(*pudp);
 465         if (pud_none(pud)) {
 466                 spin_unlock(ptl);
 467                 return hmm_vma_walk_hole(start, end, -1, walk);
 468         }
 469
 470         if (pud_huge(pud) && pud_devmap(pud)) {
 471                 unsigned long i, npages, pfn;
 472                 uint64_t *pfns, cpu_flags;
 473                 bool fault, write_fault;
 474
 475                 if (!pud_present(pud)) {
 476                         spin_unlock(ptl);
 477                         return hmm_vma_walk_hole(start, end, -1, walk);
 478                 }
 479
 480                 i = (addr - range->start) >> PAGE_SHIFT;
 481                 npages = (end - addr) >> PAGE_SHIFT;
 482                 pfns = &range->pfns[i];
 483
 484                 cpu_flags = pud_to_hmm_pfn_flags(range, pud);
 485                 hmm_range_need_fault(hmm_vma_walk, pfns, npages,
 486                                      cpu_flags, &fault, &write_fault);
 487                 if (fault || write_fault) {
 488                         spin_unlock(ptl);
 489                         return hmm_vma_fault(addr, end, fault, write_fault,
 490                                                   walk);
 491                 }
 492
 493                 pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
 494                 for (i = 0; i < npages; ++i, ++pfn) {
 495                         hmm_vma_walk->pgmap = get_dev_pagemap(pfn,
 496                                               hmm_vma_walk->pgmap);
 497                         if (unlikely(!hmm_vma_walk->pgmap)) {
 498                                 ret = -EBUSY;
 499                                 goto out_unlock;
 500                         }
 501                         pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
 502                                   cpu_flags;
 503                 }
 504                 if (hmm_vma_walk->pgmap) {
 505                         put_dev_pagemap(hmm_vma_walk->pgmap);
 506                         hmm_vma_walk->pgmap = NULL;
 507                 }
 508                 hmm_vma_walk->last = end;
 509                 goto out_unlock;
 510         }
 511
 512         /* Ask for the PUD to be split */
 513         walk->action = ACTION_SUBTREE;
 514
 515 out_unlock:
 516         spin_unlock(ptl);
 517         return ret;
 518 }
 519 #else
 520 #define hmm_vma_walk_pud        NULL
 521 #endif
 522
 523 #ifdef CONFIG_HUGETLB_PAGE
 524 static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
 525                                       unsigned long start, unsigned long end,
 526                                       struct mm_walk *walk)
 527 {
 528         unsigned long addr = start, i, pfn;
 529         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 530         struct hmm_range *range = hmm_vma_walk->range;
 531         struct vm_area_struct *vma = walk->vma;
 532         uint64_t orig_pfn, cpu_flags;
 533         bool fault, write_fault;
 534         spinlock_t *ptl;
 535         pte_t entry;
 536
 537         ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte);
 538         entry = huge_ptep_get(pte);
 539
 540         i = (start - range->start) >> PAGE_SHIFT;
 541         orig_pfn = range->pfns[i];
 542         range->pfns[i] = range->values[HMM_PFN_NONE];
 543         cpu_flags = pte_to_hmm_pfn_flags(range, entry);
 544         fault = write_fault = false;
 545         hmm_pte_need_fault(hmm_vma_walk, orig_pfn, cpu_flags,
 546                            &fault, &write_fault);
 547         if (fault || write_fault) {
 548                 spin_unlock(ptl);
 549                 return hmm_vma_fault(addr, end, fault, write_fault, walk);
 550         }
 551
 552         pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
 553         for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
 554                 range->pfns[i] = hmm_device_entry_from_pfn(range, pfn) |
 555                                  cpu_flags;
 556         hmm_vma_walk->last = end;
 557         spin_unlock(ptl);
 558         return 0;
 559 }
 560 #else
 561 #define hmm_vma_walk_hugetlb_entry NULL
 562 #endif /* CONFIG_HUGETLB_PAGE */
 563
 564 static int hmm_vma_walk_test(unsigned long start, unsigned long end,
 565                              struct mm_walk *walk)
 566 {
 567         struct hmm_vma_walk *hmm_vma_walk = walk->private;
 568         struct hmm_range *range = hmm_vma_walk->range;
 569         struct vm_area_struct *vma = walk->vma;
 570
 571         /*
 572          * Skip vma ranges that don't have struct page backing them or map I/O
 573          * devices directly.
 574          *
 575          * If the vma does not allow read access, then assume that it does not
 576          * allow write access either. HMM does not support architectures that
 577          * allow write without read.
 578          */
 579         if ((vma->vm_flags & (VM_IO | VM_PFNMAP | VM_MIXEDMAP)) ||
 580             !(vma->vm_flags & VM_READ)) {
 581                 bool fault, write_fault;
 582
 583                 /*
 584                  * Check to see if a fault is requested for any page in the
 585                  * range.
 586                  */
 587                 hmm_range_need_fault(hmm_vma_walk, range->pfns +
 588                                         ((start - range->start) >> PAGE_SHIFT),
 589                                         (end - start) >> PAGE_SHIFT,
 590                                         0, &fault, &write_fault);
 591                 if (fault || write_fault)
 592                         return -EFAULT;
 593
 594                 hmm_pfns_fill(start, end, range, HMM_PFN_ERROR);
 595                 hmm_vma_walk->last = end;
 596
 597                 /* Skip this vma and continue processing the next vma. */
 598                 return 1;
 599         }
 600
 601         return 0;
 602 }
 603
 604 static const struct mm_walk_ops hmm_walk_ops = {
 605         .pud_entry      = hmm_vma_walk_pud,
 606         .pmd_entry      = hmm_vma_walk_pmd,
 607         .pte_hole       = hmm_vma_walk_hole,
 608         .hugetlb_entry  = hmm_vma_walk_hugetlb_entry,
 609         .test_walk      = hmm_vma_walk_test,
 610 };
 611
 612 /**
 613  * hmm_range_fault - try to fault some address in a virtual address range
 614  * @range:      range being faulted
 615  * @flags:      HMM_FAULT_* flags
 616  *
 617  * Return: the number of valid pages in range->pfns[] (from range start
 618  * address), which may be zero.  On error one of the following status codes
 619  * can be returned:
 620  *
 621  * -EINVAL:     Invalid arguments or mm or virtual address is in an invalid vma
 622  *              (e.g., device file vma).
 623  * -ENOMEM:     Out of memory.
 624  * -EPERM:      Invalid permission (e.g., asking for write and range is read
 625  *              only).
 626  * -EBUSY:      The range has been invalidated and the caller needs to wait for
 627  *              the invalidation to finish.
 628  * -EFAULT:     Invalid (i.e., either no valid vma or it is illegal to access
 629  *              that range) number of valid pages in range->pfns[] (from
 630  *              range start address).
 631  *
 632  * This is similar to a regular CPU page fault except that it will not trigger
 633  * any memory migration if the memory being faulted is not accessible by CPUs
 634  * and caller does not ask for migration.
 635  *
 636  * On error, for one virtual address in the range, the function will mark the
 637  * corresponding HMM pfn entry with an error flag.
 638  */
 639 long hmm_range_fault(struct hmm_range *range, unsigned int flags)
 640 {
 641         struct hmm_vma_walk hmm_vma_walk = {
 642                 .range = range,
 643                 .last = range->start,
 644                 .flags = flags,
 645         };
 646         struct mm_struct *mm = range->notifier->mm;
 647         int ret;
 648
 649         lockdep_assert_held(&mm->mmap_sem);
 650
 651         do {
 652                 /* If range is no longer valid force retry. */
 653                 if (mmu_interval_check_retry(range->notifier,
 654                                              range->notifier_seq))
 655                         return -EBUSY;
 656                 ret = walk_page_range(mm, hmm_vma_walk.last, range->end,
 657                                       &hmm_walk_ops, &hmm_vma_walk);
 658         } while (ret == -EBUSY);
 659
 660         if (ret)
 661                 return ret;
 662         return (hmm_vma_walk.last - range->start) >> PAGE_SHIFT;
 663 }
 664 EXPORT_SYMBOL(hmm_range_fault);