drm/xe: Use GuC to do GGTT invalidations for the GuC firmware

Only the GuC should be issuing TLB invalidations if it is enabled. Part
of this patch is sanitize the device on driver unload to ensure we do
not send GuC based TLB invalidations during driver unload.

Signed-off-by: Matthew Brost <matthew.brost@intel.com>
Signed-off-by: Rodrigo Vivi <rodrigo.vivi@intel.com>
Reviewed-by: Niranjana Vishwanathapura <niranjana.vishwanathapura@intel.com>

diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index 98f08cd9d4b015ab254471c4bf9e5505f8db7065..8fe0324ccef35fb4c04dd4dd86d567580a2f68d3 100644 (file)
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -215,6 +215,16 @@ err_put:
        return ERR_PTR(err);
 }
 
+static void xe_device_sanitize(struct drm_device *drm, void *arg)
+{
+       struct xe_device *xe = arg;
+       struct xe_gt *gt;
+       u8 id;
+
+       for_each_gt(gt, xe, id)
+               xe_gt_sanitize(gt);
+}
+
 int xe_device_probe(struct xe_device *xe)
 {
        struct xe_gt *gt;
@@ -274,6 +284,10 @@ int xe_device_probe(struct xe_device *xe)
 
        xe_debugfs_register(xe);
 
+       err = drmm_add_action_or_reset(&xe->drm, xe_device_sanitize, xe);
+       if (err)
+               return err;
+
        return 0;
 
 err_irq_shutdown:

diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index baa080cd11337baf45c76d89c2838a93ec925856..20450ed8400b1d0e29a0c1c0975e77569a0dfc3c 100644 (file)
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -13,6 +13,7 @@
 #include "xe_device.h"
 #include "xe_bo.h"
 #include "xe_gt.h"
+#include "xe_gt_tlb_invalidation.h"
 #include "xe_map.h"
 #include "xe_mmio.h"
 #include "xe_wopcm.h"
@@ -200,10 +201,17 @@ void xe_ggtt_invalidate(struct xe_gt *gt)
         * therefore flushing WC buffers.  Is that really true here?
         */
        xe_mmio_write32(gt, GFX_FLSH_CNTL_GEN6.reg, GFX_FLSH_CNTL_EN);
-       if (xe_device_guc_submission_enabled(gt_to_xe(gt))) {
+
+       if (gt->uc.guc.submission_state.enabled) {
+               int seqno;
+
+               seqno = xe_gt_tlb_invalidation_guc(gt);
+               XE_WARN_ON(seqno <= 0);
+               if (seqno > 0)
+                       xe_gt_tlb_invalidation_wait(gt, seqno);
+       } else if (xe_device_guc_submission_enabled(gt_to_xe(gt))) {
                struct xe_device *xe = gt_to_xe(gt);
 
-               /* TODO: also use vfunc here */
                if (xe->info.platform == XE_PVC) {
                        xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1.reg,
                                        PVC_GUC_TLB_INV_DESC1_INVALIDATE);

diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index 28bbb3159531f9c472192ccc55d5373c8ec48a76..0e0d5cadb3e7d42989e5603ef61f871ccbbc7209 100644 (file)
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -196,6 +196,15 @@ static int gt_ttm_mgr_init(struct xe_gt *gt)
        return 0;
 }
 
+void xe_gt_sanitize(struct xe_gt *gt)
+{
+       /*
+        * FIXME: if xe_uc_sanitize is called here, on TGL driver will not
+        * reload
+        */
+       gt->uc.guc.submission_state.enabled = false;
+}
+
 static void gt_fini(struct drm_device *drm, void *arg)
 {
        struct xe_gt *gt = arg;
@@ -662,6 +671,8 @@ static int gt_reset(struct xe_gt *gt)
 
        drm_info(&xe->drm, "GT reset started\n");
 
+       xe_gt_sanitize(gt);
+
        xe_device_mem_access_get(gt_to_xe(gt));
        err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
        if (err)
@@ -742,6 +753,8 @@ int xe_gt_suspend(struct xe_gt *gt)
        if (!xe_device_guc_submission_enabled(gt_to_xe(gt)))
                return -ENODEV;
 
+       xe_gt_sanitize(gt);
+
        xe_device_mem_access_get(gt_to_xe(gt));
        err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
        if (err)

diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h
index 5dc08a993cfeb865b97457a64b3316ffec77cef2..5635f280317043af96c37cc04a87bf0e5db8bba5 100644 (file)
--- a/drivers/gpu/drm/xe/xe_gt.h
+++ b/drivers/gpu/drm/xe/xe_gt.h
@@ -26,6 +26,7 @@ int xe_gt_suspend(struct xe_gt *gt);
 int xe_gt_resume(struct xe_gt *gt);
 void xe_gt_reset_async(struct xe_gt *gt);
 void xe_gt_migrate_wait(struct xe_gt *gt);
+void xe_gt_sanitize(struct xe_gt *gt);
 
 struct xe_gt *xe_find_full_gt(struct xe_gt *gt);
 

diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c
index e1a5a3a70c9227b96b847d9e235c6a3dfd5dc3a5..ce79eb48feb84bdc8a6c20f31bc403707baab1fb 100644 (file)
--- a/drivers/gpu/drm/xe/xe_gt_pagefault.c
+++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c
@@ -240,7 +240,7 @@ unlock_vm:
                goto retry_userptr;
 
        if (!ret) {
-               ret = xe_gt_tlb_invalidation(gt, NULL, vma);
+               ret = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
                if (ret >= 0)
                        ret = 0;
        }

diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
index 0b37cd09a59a2df28ff17acd12acb2d57b2bb698..f6a2dd26cad4f6282675bccb47fabf68dd0b9f10 100644 (file)
--- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
+++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c
@@ -135,8 +135,34 @@ static int send_tlb_invalidation(struct xe_guc *guc,
        return ret;
 }
 
+#define MAKE_INVAL_OP(type)    ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
+               XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
+               XE_GUC_TLB_INVAL_FLUSH_CACHE)
+
 /**
- * xe_gt_tlb_invalidation - Issue a TLB invalidation on this GT
+ * xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC
+ * @gt: graphics tile
+ *
+ * Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and
+ * caller can use seqno + xe_gt_tlb_invalidation_wait to wait for completion.
+ *
+ * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
+ * negative error code on error.
+ */
+int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
+{
+       u32 action[] = {
+               XE_GUC_ACTION_TLB_INVALIDATION,
+               0,  /* seqno, replaced in send_tlb_invalidation */
+               MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
+       };
+
+       return send_tlb_invalidation(&gt->uc.guc, NULL, action,
+                                    ARRAY_SIZE(action));
+}
+
+/**
+ * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
  * @gt: graphics tile
  * @fence: invalidation fence which will be signal on TLB invalidation
  * completion, can be NULL
@@ -150,9 +176,9 @@ static int send_tlb_invalidation(struct xe_guc *guc,
  * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
  * negative error code on error.
  */
-int xe_gt_tlb_invalidation(struct xe_gt *gt,
-                          struct xe_gt_tlb_invalidation_fence *fence,
-                          struct xe_vma *vma)
+int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
+                              struct xe_gt_tlb_invalidation_fence *fence,
+                              struct xe_vma *vma)
 {
        struct xe_device *xe = gt_to_xe(gt);
 #define MAX_TLB_INVALIDATION_LEN       7
@@ -161,12 +187,9 @@ int xe_gt_tlb_invalidation(struct xe_gt *gt,
 
        XE_BUG_ON(!vma);
 
+       action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
+       action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */
        if (!xe->info.has_range_tlb_invalidation) {
-               action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
-               action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */
-#define MAKE_INVAL_OP(type)    ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
-               XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
-               XE_GUC_TLB_INVAL_FLUSH_CACHE)
                action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
        } else {
                u64 start = vma->start;
@@ -205,8 +228,6 @@ int xe_gt_tlb_invalidation(struct xe_gt *gt,
                XE_BUG_ON(length & GENMASK(ilog2(SZ_16M) - 1, ilog2(SZ_2M) + 1));
                XE_BUG_ON(!IS_ALIGNED(start, length));
 
-               action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
-               action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */
                action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
                action[len++] = vma->vm->usm.asid;
                action[len++] = lower_32_bits(start);

diff --git a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
index b4c4f717bc8ab48184564ba27ec55a8351333804..b333c1709397b7187bff59ba45e088de4b24d44d 100644 (file)
--- a/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
+++ b/drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h
@@ -16,9 +16,10 @@ struct xe_vma;
 
 int xe_gt_tlb_invalidation_init(struct xe_gt *gt);
 void xe_gt_tlb_invalidation_reset(struct xe_gt *gt);
-int xe_gt_tlb_invalidation(struct xe_gt *gt,
-                          struct xe_gt_tlb_invalidation_fence *fence,
-                          struct xe_vma *vma);
+int xe_gt_tlb_invalidation_guc(struct xe_gt *gt);
+int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
+                              struct xe_gt_tlb_invalidation_fence *fence,
+                              struct xe_vma *vma);
 int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno);
 int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len);
 

diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index 88a3a96da0845c78b79793905ed75582abbb9006..5cdfdfd0de40f3a45f8c67cf50961e334bc5dca5 100644 (file)
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -309,6 +309,7 @@ int xe_guc_init_post_hwconfig(struct xe_guc *guc)
 int xe_guc_post_load_init(struct xe_guc *guc)
 {
        xe_guc_ads_populate_post_load(&guc->ads);
+       guc->submission_state.enabled = true;
 
        return 0;
 }
@@ -795,6 +796,7 @@ void xe_guc_sanitize(struct xe_guc *guc)
 {
        xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE);
        xe_guc_ct_disable(&guc->ct);
+       guc->submission_state.enabled = false;
 }
 
 int xe_guc_reset_prepare(struct xe_guc *guc)

diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h
index c2a484282ef2ce47e00cf1a40a6df49c1706e102..ac7eec28934d0e172ee34f5a3e499de675b667da 100644 (file)
--- a/drivers/gpu/drm/xe/xe_guc_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_types.h
@@ -60,6 +60,8 @@ struct xe_guc {
                        /** @patch: patch version of GuC submission */
                        u32 patch;
                } version;
+               /** @enabled: submission is enabled */
+               bool enabled;
        } submission_state;
        /** @hwconfig: Hardware config state */
        struct {

diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index cde75708d8439f5d92304e3f2936b34b529f47db..3333b413686ebbf8974cbdf96ffddee609b5a96d 100644 (file)
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -1506,7 +1506,7 @@ static void invalidation_fence_work_func(struct work_struct *w)
                container_of(w, struct invalidation_fence, work);
 
        trace_xe_gt_tlb_invalidation_fence_work_func(&ifence->base);
-       xe_gt_tlb_invalidation(ifence->gt, &ifence->base, ifence->vma);
+       xe_gt_tlb_invalidation_vma(ifence->gt, &ifence->base, ifence->vma);
 }
 
 static int invalidation_fence_init(struct xe_gt *gt,

diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c
index 938d146980030ab903d9ac2419508c11d20a7b93..7886c8b853972ced4e7f63c19eb516a87c228547 100644 (file)
--- a/drivers/gpu/drm/xe/xe_uc.c
+++ b/drivers/gpu/drm/xe/xe_uc.c
@@ -88,10 +88,15 @@ static int uc_reset(struct xe_uc *uc)
        return 0;
 }
 
-static int uc_sanitize(struct xe_uc *uc)
+void xe_uc_sanitize(struct xe_uc *uc)
 {
        xe_huc_sanitize(&uc->huc);
        xe_guc_sanitize(&uc->guc);
+}
+
+static int xe_uc_sanitize_reset(struct xe_uc *uc)
+{
+       xe_uc_sanitize(uc);
 
        return uc_reset(uc);
 }
@@ -129,7 +134,7 @@ int xe_uc_init_hw(struct xe_uc *uc)
        if (!xe_device_guc_submission_enabled(uc_to_xe(uc)))
                return 0;
 
-       ret = uc_sanitize(uc);
+       ret = xe_uc_sanitize_reset(uc);
        if (ret)
                return ret;
 

diff --git a/drivers/gpu/drm/xe/xe_uc.h b/drivers/gpu/drm/xe/xe_uc.h
index 380e722f95fcec1866694b3518c3912a92df3868..d6efc9ef00d3d71077e0c236b2f3ab126ef4500c 100644 (file)
--- a/drivers/gpu/drm/xe/xe_uc.h
+++ b/drivers/gpu/drm/xe/xe_uc.h
@@ -17,5 +17,6 @@ void xe_uc_stop_prepare(struct xe_uc *uc);
 int xe_uc_stop(struct xe_uc *uc);
 int xe_uc_start(struct xe_uc *uc);
 int xe_uc_suspend(struct xe_uc *uc);
+void xe_uc_sanitize(struct xe_uc *uc);
 
 #endif

diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index 8ba548e49add4f9cfd3cb8ac49b9121a22aa3a83..4bbb0d0b092867b12515af3ddd47beec72d611da 100644 (file)
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -3356,7 +3356,7 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)
                if (xe_pt_zap_ptes(gt, vma)) {
                        gt_needs_invalidate |= BIT(id);
                        xe_device_wmb(xe);
-                       seqno[id] = xe_gt_tlb_invalidation(gt, NULL, vma);
+                       seqno[id] = xe_gt_tlb_invalidation_vma(gt, NULL, vma);
                        if (seqno[id] < 0)
                                return seqno[id];
                }