drivers/gpu/drm/i915/gt/intel_gt_pm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7 #include <linux/suspend.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "i915_params.h"
  12 #include "intel_context.h"
  13 #include "intel_engine_pm.h"
  14 #include "intel_gt.h"
  15 #include "intel_gt_clock_utils.h"
  16 #include "intel_gt_mcr.h"
  17 #include "intel_gt_pm.h"
  18 #include "intel_gt_print.h"
  19 #include "intel_gt_requests.h"
  20 #include "intel_llc.h"
  21 #include "intel_rc6.h"
  22 #include "intel_rps.h"
  23 #include "intel_wakeref.h"
  24 #include "pxp/intel_pxp_pm.h"
  25
  26 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
  27
  28 static void user_forcewake(struct intel_gt *gt, bool suspend)
  29 {
  30         int count = atomic_read(&gt->user_wakeref);
  31         intel_wakeref_t wakeref;
  32
  33         /* Inside suspend/resume so single threaded, no races to worry about. */
  34         if (likely(!count))
  35                 return;
  36
  37         wakeref = intel_gt_pm_get(gt);
  38         if (suspend) {
  39                 GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
  40                 atomic_sub(count, &gt->wakeref.count);
  41         } else {
  42                 atomic_add(count, &gt->wakeref.count);
  43         }
  44         intel_gt_pm_put(gt, wakeref);
  45 }
  46
  47 static void runtime_begin(struct intel_gt *gt)
  48 {
  49         local_irq_disable();
  50         write_seqcount_begin(&gt->stats.lock);
  51         gt->stats.start = ktime_get();
  52         gt->stats.active = true;
  53         write_seqcount_end(&gt->stats.lock);
  54         local_irq_enable();
  55 }
  56
  57 static void runtime_end(struct intel_gt *gt)
  58 {
  59         local_irq_disable();
  60         write_seqcount_begin(&gt->stats.lock);
  61         gt->stats.active = false;
  62         gt->stats.total =
  63                 ktime_add(gt->stats.total,
  64                           ktime_sub(ktime_get(), gt->stats.start));
  65         write_seqcount_end(&gt->stats.lock);
  66         local_irq_enable();
  67 }
  68
  69 static int __gt_unpark(struct intel_wakeref *wf)
  70 {
  71         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
  72         struct drm_i915_private *i915 = gt->i915;
  73
  74         GT_TRACE(gt, "\n");
  75
  76         /*
  77          * It seems that the DMC likes to transition between the DC states a lot
  78          * when there are no connected displays (no active power domains) during
  79          * command submission.
  80          *
  81          * This activity has negative impact on the performance of the chip with
  82          * huge latencies observed in the interrupt handler and elsewhere.
  83          *
  84          * Work around it by grabbing a GT IRQ power domain whilst there is any
  85          * GT activity, preventing any DC state transitions.
  86          */
  87         gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
  88         GEM_BUG_ON(!gt->awake);
  89
  90         intel_rc6_unpark(&gt->rc6);
  91         intel_rps_unpark(&gt->rps);
  92         i915_pmu_gt_unparked(gt);
  93         intel_guc_busyness_unpark(gt);
  94
  95         intel_gt_unpark_requests(gt);
  96         runtime_begin(gt);
  97
  98         return 0;
  99 }
 100
 101 static int __gt_park(struct intel_wakeref *wf)
 102 {
 103         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
 104         intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
 105         struct drm_i915_private *i915 = gt->i915;
 106
 107         GT_TRACE(gt, "\n");
 108
 109         runtime_end(gt);
 110         intel_gt_park_requests(gt);
 111
 112         intel_guc_busyness_park(gt);
 113         i915_vma_parked(gt);
 114         i915_pmu_gt_parked(gt);
 115         intel_rps_park(&gt->rps);
 116         intel_rc6_park(&gt->rc6);
 117
 118         /* Everything switched off, flush any residual interrupt just in case */
 119         intel_synchronize_irq(i915);
 120
 121         /* Defer dropping the display power well for 100ms, it's slow! */
 122         GEM_BUG_ON(!wakeref);
 123         intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 124
 125         return 0;
 126 }
 127
 128 static const struct intel_wakeref_ops wf_ops = {
 129         .get = __gt_unpark,
 130         .put = __gt_park,
 131 };
 132
 133 void intel_gt_pm_init_early(struct intel_gt *gt)
 134 {
 135         /*
 136          * We access the runtime_pm structure via gt->i915 here rather than
 137          * gt->uncore as we do elsewhere in the file because gt->uncore is not
 138          * yet initialized for all tiles at this point in the driver startup.
 139          * runtime_pm is per-device rather than per-tile, so this is still the
 140          * correct structure.
 141          */
 142         intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops, "GT");
 143         seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
 144 }
 145
 146 void intel_gt_pm_init(struct intel_gt *gt)
 147 {
 148         /*
 149          * Enabling power-management should be "self-healing". If we cannot
 150          * enable a feature, simply leave it disabled with a notice to the
 151          * user.
 152          */
 153         intel_rc6_init(&gt->rc6);
 154         intel_rps_init(&gt->rps);
 155 }
 156
 157 static bool reset_engines(struct intel_gt *gt)
 158 {
 159         if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
 160                 return false;
 161
 162         return intel_gt_reset_all_engines(gt) == 0;
 163 }
 164
 165 static void gt_sanitize(struct intel_gt *gt, bool force)
 166 {
 167         struct intel_engine_cs *engine;
 168         enum intel_engine_id id;
 169         intel_wakeref_t wakeref;
 170
 171         GT_TRACE(gt, "force:%s\n", str_yes_no(force));
 172
 173         /* Use a raw wakeref to avoid calling intel_display_power_get early */
 174         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
 175         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 176
 177         intel_gt_check_clock_frequency(gt);
 178
 179         /*
 180          * As we have just resumed the machine and woken the device up from
 181          * deep PCI sleep (presumably D3_cold), assume the HW has been reset
 182          * back to defaults, recovering from whatever wedged state we left it
 183          * in and so worth trying to use the device once more.
 184          */
 185         if (intel_gt_is_wedged(gt))
 186                 intel_gt_unset_wedged(gt);
 187
 188         /* For GuC mode, ensure submission is disabled before stopping ring */
 189         intel_uc_reset_prepare(&gt->uc);
 190
 191         for_each_engine(engine, gt, id) {
 192                 if (engine->reset.prepare)
 193                         engine->reset.prepare(engine);
 194
 195                 if (engine->sanitize)
 196                         engine->sanitize(engine);
 197         }
 198
 199         if (reset_engines(gt) || force) {
 200                 for_each_engine(engine, gt, id)
 201                         __intel_engine_reset(engine, false);
 202         }
 203
 204         intel_uc_reset(&gt->uc, false);
 205
 206         for_each_engine(engine, gt, id)
 207                 if (engine->reset.finish)
 208                         engine->reset.finish(engine);
 209
 210         intel_rps_sanitize(&gt->rps);
 211
 212         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 213         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
 214 }
 215
 216 void intel_gt_pm_fini(struct intel_gt *gt)
 217 {
 218         intel_rc6_fini(&gt->rc6);
 219 }
 220
 221 void intel_gt_resume_early(struct intel_gt *gt)
 222 {
 223         /*
 224          * Sanitize steer semaphores during driver resume. This is necessary
 225          * to address observed cases of steer semaphores being
 226          * held after a suspend operation. Confirmation from the hardware team
 227          * assures the safety of this operation, as no lock acquisitions
 228          * by other agents occur during driver load/resume process.
 229          */
 230         intel_gt_mcr_lock_sanitize(gt);
 231
 232         intel_uncore_resume_early(gt->uncore);
 233         intel_gt_check_and_clear_faults(gt);
 234 }
 235
 236 int intel_gt_resume(struct intel_gt *gt)
 237 {
 238         struct intel_engine_cs *engine;
 239         enum intel_engine_id id;
 240         intel_wakeref_t wakeref;
 241         int err;
 242
 243         err = intel_gt_has_unrecoverable_error(gt);
 244         if (err)
 245                 return err;
 246
 247         GT_TRACE(gt, "\n");
 248
 249         /*
 250          * After resume, we may need to poke into the pinned kernel
 251          * contexts to paper over any damage caused by the sudden suspend.
 252          * Only the kernel contexts should remain pinned over suspend,
 253          * allowing us to fixup the user contexts on their first pin.
 254          */
 255         gt_sanitize(gt, true);
 256
 257         wakeref = intel_gt_pm_get(gt);
 258
 259         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 260         intel_rc6_sanitize(&gt->rc6);
 261         if (intel_gt_is_wedged(gt)) {
 262                 err = -EIO;
 263                 goto out_fw;
 264         }
 265
 266         /* Only when the HW is re-initialised, can we replay the requests */
 267         err = intel_gt_init_hw(gt);
 268         if (err) {
 269                 gt_probe_error(gt, "Failed to initialize GPU, declaring it wedged!\n");
 270                 goto err_wedged;
 271         }
 272
 273         intel_uc_reset_finish(&gt->uc);
 274
 275         intel_rps_enable(&gt->rps);
 276         intel_llc_enable(&gt->llc);
 277
 278         for_each_engine(engine, gt, id) {
 279                 intel_engine_pm_get(engine);
 280
 281                 engine->serial++; /* kernel context lost */
 282                 err = intel_engine_resume(engine);
 283
 284                 intel_engine_pm_put(engine);
 285                 if (err) {
 286                         gt_err(gt, "Failed to restart %s (%d)\n",
 287                                engine->name, err);
 288                         goto err_wedged;
 289                 }
 290         }
 291
 292         intel_rc6_enable(&gt->rc6);
 293
 294         intel_uc_resume(&gt->uc);
 295
 296         user_forcewake(gt, false);
 297
 298 out_fw:
 299         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 300         intel_gt_pm_put(gt, wakeref);
 301         intel_gt_bind_context_set_ready(gt);
 302         return err;
 303
 304 err_wedged:
 305         intel_gt_set_wedged(gt);
 306         goto out_fw;
 307 }
 308
 309 static void wait_for_suspend(struct intel_gt *gt)
 310 {
 311         if (!intel_gt_pm_is_awake(gt))
 312                 return;
 313
 314         if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
 315                 /*
 316                  * Forcibly cancel outstanding work and leave
 317                  * the gpu quiet.
 318                  */
 319                 intel_gt_set_wedged(gt);
 320                 intel_gt_retire_requests(gt);
 321         }
 322
 323         intel_gt_pm_wait_for_idle(gt);
 324 }
 325
 326 void intel_gt_suspend_prepare(struct intel_gt *gt)
 327 {
 328         intel_gt_bind_context_set_unready(gt);
 329         user_forcewake(gt, true);
 330         wait_for_suspend(gt);
 331 }
 332
 333 static suspend_state_t pm_suspend_target(void)
 334 {
 335 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
 336         return pm_suspend_target_state;
 337 #else
 338         return PM_SUSPEND_TO_IDLE;
 339 #endif
 340 }
 341
 342 void intel_gt_suspend_late(struct intel_gt *gt)
 343 {
 344         intel_wakeref_t wakeref;
 345
 346         /* We expect to be idle already; but also want to be independent */
 347         wait_for_suspend(gt);
 348
 349         if (is_mock_gt(gt))
 350                 return;
 351
 352         GEM_BUG_ON(gt->awake);
 353
 354         intel_uc_suspend(&gt->uc);
 355
 356         /*
 357          * On disabling the device, we want to turn off HW access to memory
 358          * that we no longer own.
 359          *
 360          * However, not all suspend-states disable the device. S0 (s2idle)
 361          * is effectively runtime-suspend, the device is left powered on
 362          * but needs to be put into a low power state. We need to keep
 363          * powermanagement enabled, but we also retain system state and so
 364          * it remains safe to keep on using our allocated memory.
 365          */
 366         if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
 367                 return;
 368
 369         with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 370                 intel_rps_disable(&gt->rps);
 371                 intel_rc6_disable(&gt->rc6);
 372                 intel_llc_disable(&gt->llc);
 373         }
 374
 375         gt_sanitize(gt, false);
 376
 377         GT_TRACE(gt, "\n");
 378 }
 379
 380 void intel_gt_runtime_suspend(struct intel_gt *gt)
 381 {
 382         intel_gt_bind_context_set_unready(gt);
 383         intel_uc_runtime_suspend(&gt->uc);
 384
 385         GT_TRACE(gt, "\n");
 386 }
 387
 388 int intel_gt_runtime_resume(struct intel_gt *gt)
 389 {
 390         int ret;
 391
 392         GT_TRACE(gt, "\n");
 393         intel_gt_init_swizzling(gt);
 394         intel_ggtt_restore_fences(gt->ggtt);
 395
 396         ret = intel_uc_runtime_resume(&gt->uc);
 397         if (ret)
 398                 return ret;
 399
 400         intel_gt_bind_context_set_ready(gt);
 401         return 0;
 402 }
 403
 404 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
 405 {
 406         ktime_t total = gt->stats.total;
 407
 408         if (gt->stats.active)
 409                 total = ktime_add(total,
 410                                   ktime_sub(ktime_get(), gt->stats.start));
 411
 412         return total;
 413 }
 414
 415 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
 416 {
 417         unsigned int seq;
 418         ktime_t total;
 419
 420         do {
 421                 seq = read_seqcount_begin(&gt->stats.lock);
 422                 total = __intel_gt_get_awake_time(gt);
 423         } while (read_seqcount_retry(&gt->stats.lock, seq));
 424
 425         return total;
 426 }
 427
 428 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 429 #include "selftest_gt_pm.c"
 430 #endif