Merge drm/drm-next into drm-intel-next

[linux-block.git] / drivers / gpu / drm / i915 / gt / intel_gt.c

diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index 68c2b0d8f18761d4866aa665316b71f631bf863f..e4bac2431e41607de83512f4e6620acaa4c498d4 100644 (file)
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -11,7 +11,9 @@
 #include "pxp/intel_pxp.h"
 
 #include "i915_drv.h"
+#include "i915_perf_oa_regs.h"
 #include "intel_context.h"
+#include "intel_engine_pm.h"
 #include "intel_engine_regs.h"
 #include "intel_ggtt_gmch.h"
 #include "intel_gt.h"
@@ -24,6 +26,7 @@
 #include "intel_gt_requests.h"
 #include "intel_migrate.h"
 #include "intel_mocs.h"
+#include "intel_pci_config.h"
 #include "intel_pm.h"
 #include "intel_rc6.h"
 #include "intel_renderstate.h"
@@ -36,8 +39,6 @@ static void __intel_gt_init_early(struct intel_gt *gt)
 {
        spin_lock_init(&gt->irq_lock);
 
-       mutex_init(&gt->tlb_invalidate_lock);
-
        INIT_LIST_HEAD(&gt->closed_vma);
        spin_lock_init(&gt->closed_lock);
 
@@ -48,6 +49,8 @@ static void __intel_gt_init_early(struct intel_gt *gt)
        intel_gt_init_reset(gt);
        intel_gt_init_requests(gt);
        intel_gt_init_timelines(gt);
+       mutex_init(&gt->tlb.invalidate_lock);
+       seqcount_mutex_init(&gt->tlb.seqno, &gt->tlb.invalidate_lock);
        intel_gt_pm_init_early(gt);
 
        intel_uc_init_early(&gt->uc);
@@ -768,6 +771,7 @@ void intel_gt_driver_late_release_all(struct drm_i915_private *i915)
                intel_gt_fini_requests(gt);
                intel_gt_fini_reset(gt);
                intel_gt_fini_timelines(gt);
+               mutex_destroy(&gt->tlb.invalidate_lock);
                intel_engines_free(gt);
        }
 }
@@ -827,7 +831,7 @@ int intel_gt_probe_all(struct drm_i915_private *i915)
        unsigned int mmio_bar;
        int ret;
 
-       mmio_bar = GRAPHICS_VER(i915) == 2 ? 1 : 0;
+       mmio_bar = GRAPHICS_VER(i915) == 2 ? GEN2_GTTMMADR_BAR : GTTMMADR_BAR;
        phys_addr = pci_resource_start(pdev, mmio_bar);
 
        /*
@@ -906,7 +910,7 @@ get_reg_and_bit(const struct intel_engine_cs *engine, const bool gen8,
        return rb;
 }
 
-void intel_gt_invalidate_tlbs(struct intel_gt *gt)
+static void mmio_invalidate_full(struct intel_gt *gt)
 {
        static const i915_reg_t gen8_regs[] = {
                [RENDER_CLASS]                  = GEN8_RTCR,
@@ -924,13 +928,11 @@ void intel_gt_invalidate_tlbs(struct intel_gt *gt)
        struct drm_i915_private *i915 = gt->i915;
        struct intel_uncore *uncore = gt->uncore;
        struct intel_engine_cs *engine;
+       intel_engine_mask_t awake, tmp;
        enum intel_engine_id id;
        const i915_reg_t *regs;
        unsigned int num = 0;
 
-       if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
-               return;
-
        if (GRAPHICS_VER(i915) == 12) {
                regs = gen12_regs;
                num = ARRAY_SIZE(gen12_regs);
@@ -945,28 +947,41 @@ void intel_gt_invalidate_tlbs(struct intel_gt *gt)
                          "Platform does not implement TLB invalidation!"))
                return;
 
-       GEM_TRACE("\n");
-
-       assert_rpm_wakelock_held(&i915->runtime_pm);
-
-       mutex_lock(&gt->tlb_invalidate_lock);
        intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
 
        spin_lock_irq(&uncore->lock); /* serialise invalidate with GT reset */
 
+       awake = 0;
        for_each_engine(engine, gt, id) {
                struct reg_and_bit rb;
 
+               if (!intel_engine_pm_is_awake(engine))
+                       continue;
+
                rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
                if (!i915_mmio_reg_offset(rb.reg))
                        continue;
 
                intel_uncore_write_fw(uncore, rb.reg, rb.bit);
+               awake |= engine->mask;
        }
 
+       GT_TRACE(gt, "invalidated engines %08x\n", awake);
+
+       /* Wa_2207587034:tgl,dg1,rkl,adl-s,adl-p */
+       if (awake &&
+           (IS_TIGERLAKE(i915) ||
+            IS_DG1(i915) ||
+            IS_ROCKETLAKE(i915) ||
+            IS_ALDERLAKE_S(i915) ||
+            IS_ALDERLAKE_P(i915)))
+               intel_uncore_write_fw(uncore, GEN12_OA_TLB_INV_CR, 1);
+
        spin_unlock_irq(&uncore->lock);
 
-       for_each_engine(engine, gt, id) {
+       for_each_engine_masked(engine, gt, awake, tmp) {
+               struct reg_and_bit rb;
+
                /*
                 * HW architecture suggest typical invalidation time at 40us,
                 * with pessimistic cases up to 100us and a recommendation to
@@ -974,12 +989,8 @@ void intel_gt_invalidate_tlbs(struct intel_gt *gt)
                 */
                const unsigned int timeout_us = 100;
                const unsigned int timeout_ms = 4;
-               struct reg_and_bit rb;
 
                rb = get_reg_and_bit(engine, regs == gen8_regs, regs, num);
-               if (!i915_mmio_reg_offset(rb.reg))
-                       continue;
-
                if (__intel_wait_for_register_fw(uncore,
                                                 rb.reg, rb.bit, 0,
                                                 timeout_us, timeout_ms,
@@ -996,5 +1007,38 @@ void intel_gt_invalidate_tlbs(struct intel_gt *gt)
         * transitions.
         */
        intel_uncore_forcewake_put_delayed(uncore, FORCEWAKE_ALL);
-       mutex_unlock(&gt->tlb_invalidate_lock);
+}
+
+static bool tlb_seqno_passed(const struct intel_gt *gt, u32 seqno)
+{
+       u32 cur = intel_gt_tlb_seqno(gt);
+
+       /* Only skip if a *full* TLB invalidate barrier has passed */
+       return (s32)(cur - ALIGN(seqno, 2)) > 0;
+}
+
+void intel_gt_invalidate_tlb(struct intel_gt *gt, u32 seqno)
+{
+       intel_wakeref_t wakeref;
+
+       if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
+               return;
+
+       if (intel_gt_is_wedged(gt))
+               return;
+
+       if (tlb_seqno_passed(gt, seqno))
+               return;
+
+       with_intel_gt_pm_if_awake(gt, wakeref) {
+               mutex_lock(&gt->tlb.invalidate_lock);
+               if (tlb_seqno_passed(gt, seqno))
+                       goto unlock;
+
+               mmio_invalidate_full(gt);
+
+               write_seqcount_invalidate(&gt->tlb.seqno);
+unlock:
+               mutex_unlock(&gt->tlb.invalidate_lock);
+       }
 }