Merge tag 'drm-next-2023-11-07' of git://anongit.freedesktop.org/drm/drm

[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_amdkfd_gpuvm.c

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 1eccad4ce2434b17d60daf7a284a92a74356a876..41fbc4fd0fac303176d7ecb386f81d8af8cac2c6 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -425,6 +425,32 @@ validate_fail:
        return ret;
 }
 
+static int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo,
+                                              uint32_t domain,
+                                              struct dma_fence *fence)
+{
+       int ret = amdgpu_bo_reserve(bo, false);
+
+       if (ret)
+               return ret;
+
+       ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
+       if (ret)
+               goto unreserve_out;
+
+       ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1);
+       if (ret)
+               goto unreserve_out;
+
+       dma_resv_add_fence(bo->tbo.base.resv, fence,
+                          DMA_RESV_USAGE_BOOKKEEP);
+
+unreserve_out:
+       amdgpu_bo_unreserve(bo);
+
+       return ret;
+}
+
 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo)
 {
        return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);
@@ -1784,6 +1810,15 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
                }
                bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
                bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
+       } else {
+               mutex_lock(&avm->process_info->lock);
+               if (avm->process_info->eviction_fence &&
+                   !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))
+                       ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain,
+                               &avm->process_info->eviction_fence->base);
+               mutex_unlock(&avm->process_info->lock);
+               if (ret)
+                       goto err_validate_bo;
        }
 
        if (offset)
@@ -1793,6 +1828,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 
 allocate_init_user_pages_failed:
 err_pin_bo:
+err_validate_bo:
        remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
        drm_vma_node_revoke(&gobj->vma_node, drm_priv);
 err_node_allow:
@@ -1866,10 +1902,6 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
        if (unlikely(ret))
                return ret;
 
-       /* The eviction fence should be removed by the last unmap.
-        * TODO: Log an error condition if the bo still has the eviction fence
-        * attached
-        */
        amdgpu_amdkfd_remove_eviction_fence(mem->bo,
                                        process_info->eviction_fence);
        pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
@@ -1998,19 +2030,6 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
        if (unlikely(ret))
                goto out_unreserve;
 
-       if (mem->mapped_to_gpu_memory == 0 &&
-           !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
-               /* Validate BO only once. The eviction fence gets added to BO
-                * the first time it is mapped. Validate will wait for all
-                * background evictions to complete.
-                */
-               ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
-               if (ret) {
-                       pr_debug("Validate failed\n");
-                       goto out_unreserve;
-               }
-       }
-
        list_for_each_entry(entry, &mem->attachments, list) {
                if (entry->bo_va->base.vm != avm || entry->is_mapped)
                        continue;
@@ -2037,10 +2056,6 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
                         mem->mapped_to_gpu_memory);
        }
 
-       if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count)
-               dma_resv_add_fence(bo->tbo.base.resv,
-                                  &avm->process_info->eviction_fence->base,
-                                  DMA_RESV_USAGE_BOOKKEEP);
        ret = unreserve_bo_and_vms(&ctx, false, false);
 
        goto out;
@@ -2074,7 +2089,6 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
                struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv)
 {
        struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
-       struct amdkfd_process_info *process_info = avm->process_info;
        unsigned long bo_size = mem->bo->tbo.base.size;
        struct kfd_mem_attachment *entry;
        struct bo_vm_reservation_context ctx;
@@ -2115,15 +2129,6 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
                         mem->mapped_to_gpu_memory);
        }
 
-       /* If BO is unmapped from all VMs, unfence it. It can be evicted if
-        * required.
-        */
-       if (mem->mapped_to_gpu_memory == 0 &&
-           !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) &&
-           !mem->bo->tbo.pin_count)
-               amdgpu_amdkfd_remove_eviction_fence(mem->bo,
-                                               process_info->eviction_fence);
-
 unreserve_out:
        unreserve_bo_and_vms(&ctx, false, false);
 out:
@@ -2351,8 +2356,20 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev,
        amdgpu_sync_create(&(*mem)->sync);
        (*mem)->is_imported = true;
 
+       mutex_lock(&avm->process_info->lock);
+       if (avm->process_info->eviction_fence &&
+           !dma_fence_is_signaled(&avm->process_info->eviction_fence->base))
+               ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain,
+                               &avm->process_info->eviction_fence->base);
+       mutex_unlock(&avm->process_info->lock);
+       if (ret)
+               goto err_remove_mem;
+
        return 0;
 
+err_remove_mem:
+       remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+       drm_vma_node_revoke(&obj->vma_node, drm_priv);
 err_free_mem:
        kfree(*mem);
 err_put_obj: