Merge tag 'mm-stable-2022-10-13' of git://git.kernel.org/pub/scm/linux/kernel/git...

[linux-block.git] / drivers / gpu / drm / amd / amdkfd / kfd_migrate.c

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 97a684568ae01e9e2f8301a1a1ca2a5313f42975..2797029bd50015f421a1ff74f03ddf37564ccc8f 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -322,12 +322,13 @@ svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
        for (i = j = 0; i < npages; i++) {
                struct page *spage;
 
+               dst[i] = cursor.start + (j << PAGE_SHIFT);
+               migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+               svm_migrate_get_vram_page(prange, migrate->dst[i]);
+               migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
                spage = migrate_pfn_to_page(migrate->src[i]);
                if (spage && !is_zone_device_page(spage)) {
-                       dst[i] = cursor.start + (j << PAGE_SHIFT);
-                       migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-                       svm_migrate_get_vram_page(prange, migrate->dst[i]);
-                       migrate->dst[i] = migrate_pfn(migrate->dst[i]);
                        src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
                                              DMA_TO_DEVICE);
                        r = dma_mapping_error(dev, src[i]);
@@ -522,9 +523,6 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
        pr_debug("svms 0x%p [0x%lx 0x%lx] to gpu 0x%x\n", prange->svms,
                 prange->start, prange->last, best_loc);
 
-       /* FIXME: workaround for page locking bug with invalid pages */
-       svm_range_prefault(prange, mm, SVM_ADEV_PGMAP_OWNER(adev));
-
        start = prange->start << PAGE_SHIFT;
        end = (prange->last + 1) << PAGE_SHIFT;
 
@@ -888,7 +886,7 @@ svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
 static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 {
        unsigned long addr = vmf->address;
-       struct vm_area_struct *vma;
+       struct svm_range_bo *svm_bo;
        enum svm_work_list_ops op;
        struct svm_range *parent;
        struct svm_range *prange;
@@ -896,29 +894,42 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
        struct mm_struct *mm;
        int r = 0;
 
-       vma = vmf->vma;
-       mm = vma->vm_mm;
+       svm_bo = vmf->page->zone_device_data;
+       if (!svm_bo) {
+               pr_debug("failed get device page at addr 0x%lx\n", addr);
+               return VM_FAULT_SIGBUS;
+       }
+       if (!mmget_not_zero(svm_bo->eviction_fence->mm)) {
+               pr_debug("addr 0x%lx of process mm is destroyed\n", addr);
+               return VM_FAULT_SIGBUS;
+       }
+
+       mm = svm_bo->eviction_fence->mm;
+       if (mm != vmf->vma->vm_mm)
+               pr_debug("addr 0x%lx is COW mapping in child process\n", addr);
 
-       p = kfd_lookup_process_by_mm(vma->vm_mm);
+       p = kfd_lookup_process_by_mm(mm);
        if (!p) {
                pr_debug("failed find process at fault address 0x%lx\n", addr);
-               return VM_FAULT_SIGBUS;
+               r = VM_FAULT_SIGBUS;
+               goto out_mmput;
        }
        if (READ_ONCE(p->svms.faulting_task) == current) {
                pr_debug("skipping ram migration\n");
-               kfd_unref_process(p);
-               return 0;
+               r = 0;
+               goto out_unref_process;
        }
-       addr >>= PAGE_SHIFT;
+
        pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr);
+       addr >>= PAGE_SHIFT;
 
        mutex_lock(&p->svms.lock);
 
        prange = svm_range_from_addr(&p->svms, addr, &parent);
        if (!prange) {
-               pr_debug("cannot find svm range at 0x%lx\n", addr);
+               pr_debug("failed get range svms 0x%p addr 0x%lx\n", &p->svms, addr);
                r = -EFAULT;
-               goto out;
+               goto out_unlock_svms;
        }
 
        mutex_lock(&parent->migrate_mutex);
@@ -940,11 +951,12 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
                goto out_unlock_prange;
        }
 
-       r = svm_migrate_vram_to_ram(prange, mm, KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU,
-                               vmf->page);
+       r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm,
+                                   KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU,
+                                   vmf->page);
        if (r)
-               pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r,
-                        prange, prange->start, prange->last);
+               pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",
+                        r, prange->svms, prange, prange->start, prange->last);
 
        /* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */
        if (p->xnack_enabled && parent == prange)
@@ -958,9 +970,12 @@ out_unlock_prange:
        if (prange != parent)
                mutex_unlock(&prange->migrate_mutex);
        mutex_unlock(&parent->migrate_mutex);
-out:
+out_unlock_svms:
        mutex_unlock(&p->svms.lock);
+out_unref_process:
        kfd_unref_process(p);
+out_mmput:
+       mmput(mm);
 
        pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr);