drivers/gpu/drm/i915/gt/intel_gt_pm.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7 #include <linux/suspend.h>
   8
   9 #include "i915_drv.h"
  10 #include "i915_irq.h"
  11 #include "i915_params.h"
  12 #include "intel_context.h"
  13 #include "intel_engine_pm.h"
  14 #include "intel_gt.h"
  15 #include "intel_gt_clock_utils.h"
  16 #include "intel_gt_pm.h"
  17 #include "intel_gt_requests.h"
  18 #include "intel_llc.h"
  19 #include "intel_pm.h"
  20 #include "intel_rc6.h"
  21 #include "intel_rps.h"
  22 #include "intel_wakeref.h"
  23 #include "pxp/intel_pxp_pm.h"
  24
  25 #define I915_GT_SUSPEND_IDLE_TIMEOUT (HZ / 2)
  26
  27 static void user_forcewake(struct intel_gt *gt, bool suspend)
  28 {
  29         int count = atomic_read(&gt->user_wakeref);
  30
  31         /* Inside suspend/resume so single threaded, no races to worry about. */
  32         if (likely(!count))
  33                 return;
  34
  35         intel_gt_pm_get(gt);
  36         if (suspend) {
  37                 GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));
  38                 atomic_sub(count, &gt->wakeref.count);
  39         } else {
  40                 atomic_add(count, &gt->wakeref.count);
  41         }
  42         intel_gt_pm_put(gt);
  43 }
  44
  45 static void runtime_begin(struct intel_gt *gt)
  46 {
  47         local_irq_disable();
  48         write_seqcount_begin(&gt->stats.lock);
  49         gt->stats.start = ktime_get();
  50         gt->stats.active = true;
  51         write_seqcount_end(&gt->stats.lock);
  52         local_irq_enable();
  53 }
  54
  55 static void runtime_end(struct intel_gt *gt)
  56 {
  57         local_irq_disable();
  58         write_seqcount_begin(&gt->stats.lock);
  59         gt->stats.active = false;
  60         gt->stats.total =
  61                 ktime_add(gt->stats.total,
  62                           ktime_sub(ktime_get(), gt->stats.start));
  63         write_seqcount_end(&gt->stats.lock);
  64         local_irq_enable();
  65 }
  66
  67 static int __gt_unpark(struct intel_wakeref *wf)
  68 {
  69         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
  70         struct drm_i915_private *i915 = gt->i915;
  71
  72         GT_TRACE(gt, "\n");
  73
  74         /*
  75          * It seems that the DMC likes to transition between the DC states a lot
  76          * when there are no connected displays (no active power domains) during
  77          * command submission.
  78          *
  79          * This activity has negative impact on the performance of the chip with
  80          * huge latencies observed in the interrupt handler and elsewhere.
  81          *
  82          * Work around it by grabbing a GT IRQ power domain whilst there is any
  83          * GT activity, preventing any DC state transitions.
  84          */
  85         gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
  86         GEM_BUG_ON(!gt->awake);
  87
  88         intel_rc6_unpark(&gt->rc6);
  89         intel_rps_unpark(&gt->rps);
  90         i915_pmu_gt_unparked(i915);
  91         intel_guc_busyness_unpark(gt);
  92
  93         intel_gt_unpark_requests(gt);
  94         runtime_begin(gt);
  95
  96         return 0;
  97 }
  98
  99 static int __gt_park(struct intel_wakeref *wf)
 100 {
 101         struct intel_gt *gt = container_of(wf, typeof(*gt), wakeref);
 102         intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);
 103         struct drm_i915_private *i915 = gt->i915;
 104
 105         GT_TRACE(gt, "\n");
 106
 107         runtime_end(gt);
 108         intel_gt_park_requests(gt);
 109
 110         intel_guc_busyness_park(gt);
 111         i915_vma_parked(gt);
 112         i915_pmu_gt_parked(i915);
 113         intel_rps_park(&gt->rps);
 114         intel_rc6_park(&gt->rc6);
 115
 116         /* Everything switched off, flush any residual interrupt just in case */
 117         intel_synchronize_irq(i915);
 118
 119         /* Defer dropping the display power well for 100ms, it's slow! */
 120         GEM_BUG_ON(!wakeref);
 121         intel_display_power_put_async(i915, POWER_DOMAIN_GT_IRQ, wakeref);
 122
 123         return 0;
 124 }
 125
 126 static const struct intel_wakeref_ops wf_ops = {
 127         .get = __gt_unpark,
 128         .put = __gt_park,
 129 };
 130
 131 void intel_gt_pm_init_early(struct intel_gt *gt)
 132 {
 133         /*
 134          * We access the runtime_pm structure via gt->i915 here rather than
 135          * gt->uncore as we do elsewhere in the file because gt->uncore is not
 136          * yet initialized for all tiles at this point in the driver startup.
 137          * runtime_pm is per-device rather than per-tile, so this is still the
 138          * correct structure.
 139          */
 140         intel_wakeref_init(&gt->wakeref, &gt->i915->runtime_pm, &wf_ops);
 141         seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);
 142 }
 143
 144 void intel_gt_pm_init(struct intel_gt *gt)
 145 {
 146         /*
 147          * Enabling power-management should be "self-healing". If we cannot
 148          * enable a feature, simply leave it disabled with a notice to the
 149          * user.
 150          */
 151         intel_rc6_init(&gt->rc6);
 152         intel_rps_init(&gt->rps);
 153 }
 154
 155 static bool reset_engines(struct intel_gt *gt)
 156 {
 157         if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
 158                 return false;
 159
 160         return __intel_gt_reset(gt, ALL_ENGINES) == 0;
 161 }
 162
 163 static void gt_sanitize(struct intel_gt *gt, bool force)
 164 {
 165         struct intel_engine_cs *engine;
 166         enum intel_engine_id id;
 167         intel_wakeref_t wakeref;
 168
 169         GT_TRACE(gt, "force:%s", str_yes_no(force));
 170
 171         /* Use a raw wakeref to avoid calling intel_display_power_get early */
 172         wakeref = intel_runtime_pm_get(gt->uncore->rpm);
 173         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 174
 175         intel_gt_check_clock_frequency(gt);
 176
 177         /*
 178          * As we have just resumed the machine and woken the device up from
 179          * deep PCI sleep (presumably D3_cold), assume the HW has been reset
 180          * back to defaults, recovering from whatever wedged state we left it
 181          * in and so worth trying to use the device once more.
 182          */
 183         if (intel_gt_is_wedged(gt))
 184                 intel_gt_unset_wedged(gt);
 185
 186         /* For GuC mode, ensure submission is disabled before stopping ring */
 187         intel_uc_reset_prepare(&gt->uc);
 188
 189         for_each_engine(engine, gt, id) {
 190                 if (engine->reset.prepare)
 191                         engine->reset.prepare(engine);
 192
 193                 if (engine->sanitize)
 194                         engine->sanitize(engine);
 195         }
 196
 197         if (reset_engines(gt) || force) {
 198                 for_each_engine(engine, gt, id)
 199                         __intel_engine_reset(engine, false);
 200         }
 201
 202         intel_uc_reset(&gt->uc, false);
 203
 204         for_each_engine(engine, gt, id)
 205                 if (engine->reset.finish)
 206                         engine->reset.finish(engine);
 207
 208         intel_rps_sanitize(&gt->rps);
 209
 210         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 211         intel_runtime_pm_put(gt->uncore->rpm, wakeref);
 212 }
 213
 214 void intel_gt_pm_fini(struct intel_gt *gt)
 215 {
 216         intel_rc6_fini(&gt->rc6);
 217 }
 218
 219 int intel_gt_resume(struct intel_gt *gt)
 220 {
 221         struct intel_engine_cs *engine;
 222         enum intel_engine_id id;
 223         int err;
 224
 225         err = intel_gt_has_unrecoverable_error(gt);
 226         if (err)
 227                 return err;
 228
 229         GT_TRACE(gt, "\n");
 230
 231         /*
 232          * After resume, we may need to poke into the pinned kernel
 233          * contexts to paper over any damage caused by the sudden suspend.
 234          * Only the kernel contexts should remain pinned over suspend,
 235          * allowing us to fixup the user contexts on their first pin.
 236          */
 237         gt_sanitize(gt, true);
 238
 239         intel_gt_pm_get(gt);
 240
 241         intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
 242         intel_rc6_sanitize(&gt->rc6);
 243         if (intel_gt_is_wedged(gt)) {
 244                 err = -EIO;
 245                 goto out_fw;
 246         }
 247
 248         /* Only when the HW is re-initialised, can we replay the requests */
 249         err = intel_gt_init_hw(gt);
 250         if (err) {
 251                 i915_probe_error(gt->i915,
 252                                  "Failed to initialize GPU, declaring it wedged!\n");
 253                 goto err_wedged;
 254         }
 255
 256         intel_uc_reset_finish(&gt->uc);
 257
 258         intel_rps_enable(&gt->rps);
 259         intel_llc_enable(&gt->llc);
 260
 261         for_each_engine(engine, gt, id) {
 262                 intel_engine_pm_get(engine);
 263
 264                 engine->serial++; /* kernel context lost */
 265                 err = intel_engine_resume(engine);
 266
 267                 intel_engine_pm_put(engine);
 268                 if (err) {
 269                         drm_err(&gt->i915->drm,
 270                                 "Failed to restart %s (%d)\n",
 271                                 engine->name, err);
 272                         goto err_wedged;
 273                 }
 274         }
 275
 276         intel_rc6_enable(&gt->rc6);
 277
 278         intel_uc_resume(&gt->uc);
 279
 280         intel_pxp_resume(&gt->pxp);
 281
 282         user_forcewake(gt, false);
 283
 284 out_fw:
 285         intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
 286         intel_gt_pm_put(gt);
 287         return err;
 288
 289 err_wedged:
 290         intel_gt_set_wedged(gt);
 291         goto out_fw;
 292 }
 293
 294 static void wait_for_suspend(struct intel_gt *gt)
 295 {
 296         if (!intel_gt_pm_is_awake(gt))
 297                 return;
 298
 299         if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {
 300                 /*
 301                  * Forcibly cancel outstanding work and leave
 302                  * the gpu quiet.
 303                  */
 304                 intel_gt_set_wedged(gt);
 305                 intel_gt_retire_requests(gt);
 306         }
 307
 308         intel_gt_pm_wait_for_idle(gt);
 309 }
 310
 311 void intel_gt_suspend_prepare(struct intel_gt *gt)
 312 {
 313         user_forcewake(gt, true);
 314         wait_for_suspend(gt);
 315
 316         intel_pxp_suspend_prepare(&gt->pxp);
 317 }
 318
 319 static suspend_state_t pm_suspend_target(void)
 320 {
 321 #if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
 322         return pm_suspend_target_state;
 323 #else
 324         return PM_SUSPEND_TO_IDLE;
 325 #endif
 326 }
 327
 328 void intel_gt_suspend_late(struct intel_gt *gt)
 329 {
 330         intel_wakeref_t wakeref;
 331
 332         /* We expect to be idle already; but also want to be independent */
 333         wait_for_suspend(gt);
 334
 335         if (is_mock_gt(gt))
 336                 return;
 337
 338         GEM_BUG_ON(gt->awake);
 339
 340         intel_uc_suspend(&gt->uc);
 341         intel_pxp_suspend(&gt->pxp);
 342
 343         /*
 344          * On disabling the device, we want to turn off HW access to memory
 345          * that we no longer own.
 346          *
 347          * However, not all suspend-states disable the device. S0 (s2idle)
 348          * is effectively runtime-suspend, the device is left powered on
 349          * but needs to be put into a low power state. We need to keep
 350          * powermanagement enabled, but we also retain system state and so
 351          * it remains safe to keep on using our allocated memory.
 352          */
 353         if (pm_suspend_target() == PM_SUSPEND_TO_IDLE)
 354                 return;
 355
 356         with_intel_runtime_pm(gt->uncore->rpm, wakeref) {
 357                 intel_rps_disable(&gt->rps);
 358                 intel_rc6_disable(&gt->rc6);
 359                 intel_llc_disable(&gt->llc);
 360         }
 361
 362         gt_sanitize(gt, false);
 363
 364         GT_TRACE(gt, "\n");
 365 }
 366
 367 void intel_gt_runtime_suspend(struct intel_gt *gt)
 368 {
 369         intel_pxp_runtime_suspend(&gt->pxp);
 370         intel_uc_runtime_suspend(&gt->uc);
 371
 372         GT_TRACE(gt, "\n");
 373 }
 374
 375 int intel_gt_runtime_resume(struct intel_gt *gt)
 376 {
 377         int ret;
 378
 379         GT_TRACE(gt, "\n");
 380         intel_gt_init_swizzling(gt);
 381         intel_ggtt_restore_fences(gt->ggtt);
 382
 383         ret = intel_uc_runtime_resume(&gt->uc);
 384         if (ret)
 385                 return ret;
 386
 387         intel_pxp_runtime_resume(&gt->pxp);
 388
 389         return 0;
 390 }
 391
 392 static ktime_t __intel_gt_get_awake_time(const struct intel_gt *gt)
 393 {
 394         ktime_t total = gt->stats.total;
 395
 396         if (gt->stats.active)
 397                 total = ktime_add(total,
 398                                   ktime_sub(ktime_get(), gt->stats.start));
 399
 400         return total;
 401 }
 402
 403 ktime_t intel_gt_get_awake_time(const struct intel_gt *gt)
 404 {
 405         unsigned int seq;
 406         ktime_t total;
 407
 408         do {
 409                 seq = read_seqcount_begin(&gt->stats.lock);
 410                 total = __intel_gt_get_awake_time(gt);
 411         } while (read_seqcount_retry(&gt->stats.lock, seq));
 412
 413         return total;
 414 }
 415
 416 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 417 #include "selftest_gt_pm.c"
 418 #endif