drm/amdkfd: Add quiesce_mm and resume_mm to kgd2kfd_calls

These interfaces allow KGD to stop and resume all GPU user mode queue
access to a process address space. This is needed for handling MMU
notifiers of userptrs mapped for GPU access in KFD VMs.

Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 0434f659eeafb7364b3185eb22531bb99db9a990..7b5799530c0f7bf1c0ac2f422edaba68a224efd7 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -541,6 +541,44 @@ void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
        spin_unlock(&kfd->interrupt_lock);
 }
 
+int kgd2kfd_quiesce_mm(struct mm_struct *mm)
+{
+       struct kfd_process *p;
+       int r;
+
+       /* Because we are called from arbitrary context (workqueue) as opposed
+        * to process context, kfd_process could attempt to exit while we are
+        * running so the lookup function increments the process ref count.
+        */
+       p = kfd_lookup_process_by_mm(mm);
+       if (!p)
+               return -ESRCH;
+
+       r = kfd_process_evict_queues(p);
+
+       kfd_unref_process(p);
+       return r;
+}
+
+int kgd2kfd_resume_mm(struct mm_struct *mm)
+{
+       struct kfd_process *p;
+       int r;
+
+       /* Because we are called from arbitrary context (workqueue) as opposed
+        * to process context, kfd_process could attempt to exit while we are
+        * running so the lookup function increments the process ref count.
+        */
+       p = kfd_lookup_process_by_mm(mm);
+       if (!p)
+               return -ESRCH;
+
+       r = kfd_process_restore_queues(p);
+
+       kfd_unref_process(p);
+       return r;
+}
+
 /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will
  *   prepare for safe eviction of KFD BOs that belong to the specified
  *   process.

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
index e0c07d24d25170b2df4bb6354c85d32ccf7496a4..45bc458f7348702083074bc9459e41e345004572 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c
@@ -43,6 +43,8 @@ static const struct kgd2kfd_calls kgd2kfd = {
        .interrupt      = kgd2kfd_interrupt,
        .suspend        = kgd2kfd_suspend,
        .resume         = kgd2kfd_resume,
+       .quiesce_mm     = kgd2kfd_quiesce_mm,
+       .resume_mm      = kgd2kfd_resume_mm,
        .schedule_evict_and_restore_process =
                          kgd2kfd_schedule_evict_and_restore_process,
 };

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 96a9cc0f02c97307f4084ce5445e344adbcf371e..4d5c49ef2dc5b7ca03935d8e5e062dac7e71171e 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -512,6 +512,8 @@ struct qcm_process_device {
 /* Approx. time before evicting the process again */
 #define PROCESS_ACTIVE_TIME_MS 10
 
+int kgd2kfd_quiesce_mm(struct mm_struct *mm);
+int kgd2kfd_resume_mm(struct mm_struct *mm);
 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
                                               struct dma_fence *fence);
 
@@ -681,6 +683,8 @@ struct kfd_process *kfd_get_process(const struct task_struct *);
 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
 void kfd_unref_process(struct kfd_process *p);
+int kfd_process_evict_queues(struct kfd_process *p);
+int kfd_process_restore_queues(struct kfd_process *p);
 void kfd_suspend_all_processes(void);
 int kfd_resume_all_processes(void);
 

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 1711ad0642f74f0050eb1e2ebeb083c9ca054d63..2791e72c2058e7a8194f528c0f174035ab12c5ee 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -808,7 +808,7 @@ struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
  * Eviction is reference-counted per process-device. This means multiple
  * evictions from different sources can be nested safely.
  */
-static int process_evict_queues(struct kfd_process *p)
+int kfd_process_evict_queues(struct kfd_process *p)
 {
        struct kfd_process_device *pdd;
        int r = 0;
@@ -844,7 +844,7 @@ fail:
 }
 
 /* process_restore_queues - Restore all user queues of a process */
-static  int process_restore_queues(struct kfd_process *p)
+int kfd_process_restore_queues(struct kfd_process *p)
 {
        struct kfd_process_device *pdd;
        int r, ret = 0;
@@ -886,7 +886,7 @@ static void evict_process_worker(struct work_struct *work)
        flush_delayed_work(&p->restore_work);
 
        pr_debug("Started evicting pasid %d\n", p->pasid);
-       ret = process_evict_queues(p);
+       ret = kfd_process_evict_queues(p);
        if (!ret) {
                dma_fence_signal(p->ef);
                dma_fence_put(p->ef);
@@ -946,7 +946,7 @@ static void restore_process_worker(struct work_struct *work)
                return;
        }
 
-       ret = process_restore_queues(p);
+       ret = kfd_process_restore_queues(p);
        if (!ret)
                pr_debug("Finished restoring pasid %d\n", p->pasid);
        else
@@ -963,7 +963,7 @@ void kfd_suspend_all_processes(void)
                cancel_delayed_work_sync(&p->eviction_work);
                cancel_delayed_work_sync(&p->restore_work);
 
-               if (process_evict_queues(p))
+               if (kfd_process_evict_queues(p))
                        pr_err("Failed to suspend process %d\n", p->pasid);
                dma_fence_signal(p->ef);
                dma_fence_put(p->ef);

diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 237289a72bb7b4b697d3d266f8bc83a51484498c..286cfe7068c1e6b65f91b1e3f56e2d683a8300a4 100644 (file)
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -382,6 +382,10 @@ struct kfd2kgd_calls {
  *
  * @resume: Notifies amdkfd about a resume action done to a kgd device
  *
+ * @quiesce_mm: Quiesce all user queue access to specified MM address space
+ *
+ * @resume_mm: Resume user queue access to specified MM address space
+ *
  * @schedule_evict_and_restore_process: Schedules work queue that will prepare
  * for safe eviction of KFD BOs that belong to the specified process.
  *
@@ -399,6 +403,8 @@ struct kgd2kfd_calls {
        void (*interrupt)(struct kfd_dev *kfd, const void *ih_ring_entry);
        void (*suspend)(struct kfd_dev *kfd);
        int (*resume)(struct kfd_dev *kfd);
+       int (*quiesce_mm)(struct mm_struct *mm);
+       int (*resume_mm)(struct mm_struct *mm);
        int (*schedule_evict_and_restore_process)(struct mm_struct *mm,
                        struct dma_fence *fence);
 };