drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26 #include "i915_vgpu.h"
  27
  28 #include <asm/iosf_mbi.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #define FORCEWAKE_ACK_TIMEOUT_MS 50
  32 #define GT_FIFO_TIMEOUT_MS       10
  33
  34 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
  35
  36 static const char * const forcewake_domain_names[] = {
  37         "render",
  38         "blitter",
  39         "media",
  40 };
  41
  42 const char *
  43 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
  44 {
  45         BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
  46
  47         if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
  48                 return forcewake_domain_names[id];
  49
  50         WARN_ON(id);
  51
  52         return "unknown";
  53 }
  54
  55 static inline void
  56 fw_domain_reset(struct drm_i915_private *i915,
  57                 const struct intel_uncore_forcewake_domain *d)
  58 {
  59         __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset);
  60 }
  61
  62 static inline void
  63 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
  64 {
  65         d->wake_count++;
  66         hrtimer_start_range_ns(&d->timer,
  67                                NSEC_PER_MSEC,
  68                                NSEC_PER_MSEC,
  69                                HRTIMER_MODE_REL);
  70 }
  71
  72 static inline int
  73 __wait_for_ack(const struct drm_i915_private *i915,
  74                const struct intel_uncore_forcewake_domain *d,
  75                const u32 ack,
  76                const u32 value)
  77 {
  78         return wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) & ack) == value,
  79                                FORCEWAKE_ACK_TIMEOUT_MS);
  80 }
  81
  82 static inline int
  83 wait_ack_clear(const struct drm_i915_private *i915,
  84                const struct intel_uncore_forcewake_domain *d,
  85                const u32 ack)
  86 {
  87         return __wait_for_ack(i915, d, ack, 0);
  88 }
  89
  90 static inline int
  91 wait_ack_set(const struct drm_i915_private *i915,
  92              const struct intel_uncore_forcewake_domain *d,
  93              const u32 ack)
  94 {
  95         return __wait_for_ack(i915, d, ack, ack);
  96 }
  97
  98 static inline void
  99 fw_domain_wait_ack_clear(const struct drm_i915_private *i915,
 100                          const struct intel_uncore_forcewake_domain *d)
 101 {
 102         if (wait_ack_clear(i915, d, FORCEWAKE_KERNEL))
 103                 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
 104                           intel_uncore_forcewake_domain_to_str(d->id));
 105 }
 106
 107 enum ack_type {
 108         ACK_CLEAR = 0,
 109         ACK_SET
 110 };
 111
 112 static int
 113 fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915,
 114                                  const struct intel_uncore_forcewake_domain *d,
 115                                  const enum ack_type type)
 116 {
 117         const u32 ack_bit = FORCEWAKE_KERNEL;
 118         const u32 value = type == ACK_SET ? ack_bit : 0;
 119         unsigned int pass;
 120         bool ack_detected;
 121
 122         /*
 123          * There is a possibility of driver's wake request colliding
 124          * with hardware's own wake requests and that can cause
 125          * hardware to not deliver the driver's ack message.
 126          *
 127          * Use a fallback bit toggle to kick the gpu state machine
 128          * in the hope that the original ack will be delivered along with
 129          * the fallback ack.
 130          *
 131          * This workaround is described in HSDES #1604254524
 132          */
 133
 134         pass = 1;
 135         do {
 136                 wait_ack_clear(i915, d, FORCEWAKE_KERNEL_FALLBACK);
 137
 138                 __raw_i915_write32(i915, d->reg_set,
 139                                    _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK));
 140                 /* Give gt some time to relax before the polling frenzy */
 141                 udelay(10 * pass);
 142                 wait_ack_set(i915, d, FORCEWAKE_KERNEL_FALLBACK);
 143
 144                 ack_detected = (__raw_i915_read32(i915, d->reg_ack) & ack_bit) == value;
 145
 146                 __raw_i915_write32(i915, d->reg_set,
 147                                    _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK));
 148         } while (!ack_detected && pass++ < 10);
 149
 150         DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",
 151                          intel_uncore_forcewake_domain_to_str(d->id),
 152                          type == ACK_SET ? "set" : "clear",
 153                          __raw_i915_read32(i915, d->reg_ack),
 154                          pass);
 155
 156         return ack_detected ? 0 : -ETIMEDOUT;
 157 }
 158
 159 static inline void
 160 fw_domain_wait_ack_clear_fallback(const struct drm_i915_private *i915,
 161                                   const struct intel_uncore_forcewake_domain *d)
 162 {
 163         if (likely(!wait_ack_clear(i915, d, FORCEWAKE_KERNEL)))
 164                 return;
 165
 166         if (fw_domain_wait_ack_with_fallback(i915, d, ACK_CLEAR))
 167                 fw_domain_wait_ack_clear(i915, d);
 168 }
 169
 170 static inline void
 171 fw_domain_get(struct drm_i915_private *i915,
 172               const struct intel_uncore_forcewake_domain *d)
 173 {
 174         __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_set);
 175 }
 176
 177 static inline void
 178 fw_domain_wait_ack_set(const struct drm_i915_private *i915,
 179                        const struct intel_uncore_forcewake_domain *d)
 180 {
 181         if (wait_ack_set(i915, d, FORCEWAKE_KERNEL))
 182                 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
 183                           intel_uncore_forcewake_domain_to_str(d->id));
 184 }
 185
 186 static inline void
 187 fw_domain_wait_ack_set_fallback(const struct drm_i915_private *i915,
 188                                 const struct intel_uncore_forcewake_domain *d)
 189 {
 190         if (likely(!wait_ack_set(i915, d, FORCEWAKE_KERNEL)))
 191                 return;
 192
 193         if (fw_domain_wait_ack_with_fallback(i915, d, ACK_SET))
 194                 fw_domain_wait_ack_set(i915, d);
 195 }
 196
 197 static inline void
 198 fw_domain_put(const struct drm_i915_private *i915,
 199               const struct intel_uncore_forcewake_domain *d)
 200 {
 201         __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_clear);
 202 }
 203
 204 static void
 205 fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
 206 {
 207         struct intel_uncore_forcewake_domain *d;
 208         unsigned int tmp;
 209
 210         GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
 211
 212         for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
 213                 fw_domain_wait_ack_clear(i915, d);
 214                 fw_domain_get(i915, d);
 215         }
 216
 217         for_each_fw_domain_masked(d, fw_domains, i915, tmp)
 218                 fw_domain_wait_ack_set(i915, d);
 219
 220         i915->uncore.fw_domains_active |= fw_domains;
 221 }
 222
 223 static void
 224 fw_domains_get_with_fallback(struct drm_i915_private *i915,
 225                              enum forcewake_domains fw_domains)
 226 {
 227         struct intel_uncore_forcewake_domain *d;
 228         unsigned int tmp;
 229
 230         GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
 231
 232         for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
 233                 fw_domain_wait_ack_clear_fallback(i915, d);
 234                 fw_domain_get(i915, d);
 235         }
 236
 237         for_each_fw_domain_masked(d, fw_domains, i915, tmp)
 238                 fw_domain_wait_ack_set_fallback(i915, d);
 239
 240         i915->uncore.fw_domains_active |= fw_domains;
 241 }
 242
 243 static void
 244 fw_domains_put(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
 245 {
 246         struct intel_uncore_forcewake_domain *d;
 247         unsigned int tmp;
 248
 249         GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
 250
 251         for_each_fw_domain_masked(d, fw_domains, i915, tmp)
 252                 fw_domain_put(i915, d);
 253
 254         i915->uncore.fw_domains_active &= ~fw_domains;
 255 }
 256
 257 static void
 258 fw_domains_reset(struct drm_i915_private *i915,
 259                  enum forcewake_domains fw_domains)
 260 {
 261         struct intel_uncore_forcewake_domain *d;
 262         unsigned int tmp;
 263
 264         if (!fw_domains)
 265                 return;
 266
 267         GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
 268
 269         for_each_fw_domain_masked(d, fw_domains, i915, tmp)
 270                 fw_domain_reset(i915, d);
 271 }
 272
 273 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
 274 {
 275         /* w/a for a sporadic read returning 0 by waiting for the GT
 276          * thread to wake up.
 277          */
 278         if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) &
 279                                 GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500))
 280                 DRM_ERROR("GT thread status wait timed out\n");
 281 }
 282
 283 static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
 284                                               enum forcewake_domains fw_domains)
 285 {
 286         fw_domains_get(dev_priv, fw_domains);
 287
 288         /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
 289         __gen6_gt_wait_for_thread_c0(dev_priv);
 290 }
 291
 292 static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
 293 {
 294         u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);
 295
 296         return count & GT_FIFO_FREE_ENTRIES_MASK;
 297 }
 298
 299 static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 300 {
 301         u32 n;
 302
 303         /* On VLV, FIFO will be shared by both SW and HW.
 304          * So, we need to read the FREE_ENTRIES everytime */
 305         if (IS_VALLEYVIEW(dev_priv))
 306                 n = fifo_free_entries(dev_priv);
 307         else
 308                 n = dev_priv->uncore.fifo_count;
 309
 310         if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
 311                 if (wait_for_atomic((n = fifo_free_entries(dev_priv)) >
 312                                     GT_FIFO_NUM_RESERVED_ENTRIES,
 313                                     GT_FIFO_TIMEOUT_MS)) {
 314                         DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n);
 315                         return;
 316                 }
 317         }
 318
 319         dev_priv->uncore.fifo_count = n - 1;
 320 }
 321
 322 static enum hrtimer_restart
 323 intel_uncore_fw_release_timer(struct hrtimer *timer)
 324 {
 325         struct intel_uncore_forcewake_domain *domain =
 326                container_of(timer, struct intel_uncore_forcewake_domain, timer);
 327         struct drm_i915_private *dev_priv =
 328                 container_of(domain, struct drm_i915_private, uncore.fw_domain[domain->id]);
 329         unsigned long irqflags;
 330
 331         assert_rpm_device_not_suspended(dev_priv);
 332
 333         if (xchg(&domain->active, false))
 334                 return HRTIMER_RESTART;
 335
 336         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 337         if (WARN_ON(domain->wake_count == 0))
 338                 domain->wake_count++;
 339
 340         if (--domain->wake_count == 0)
 341                 dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
 342
 343         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 344
 345         return HRTIMER_NORESTART;
 346 }
 347
 348 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 349 static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
 350                                          bool restore)
 351 {
 352         unsigned long irqflags;
 353         struct intel_uncore_forcewake_domain *domain;
 354         int retry_count = 100;
 355         enum forcewake_domains fw, active_domains;
 356
 357         iosf_mbi_assert_punit_acquired();
 358
 359         /* Hold uncore.lock across reset to prevent any register access
 360          * with forcewake not set correctly. Wait until all pending
 361          * timers are run before holding.
 362          */
 363         while (1) {
 364                 unsigned int tmp;
 365
 366                 active_domains = 0;
 367
 368                 for_each_fw_domain(domain, dev_priv, tmp) {
 369                         smp_store_mb(domain->active, false);
 370                         if (hrtimer_cancel(&domain->timer) == 0)
 371                                 continue;
 372
 373                         intel_uncore_fw_release_timer(&domain->timer);
 374                 }
 375
 376                 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 377
 378                 for_each_fw_domain(domain, dev_priv, tmp) {
 379                         if (hrtimer_active(&domain->timer))
 380                                 active_domains |= domain->mask;
 381                 }
 382
 383                 if (active_domains == 0)
 384                         break;
 385
 386                 if (--retry_count == 0) {
 387                         DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
 388                         break;
 389                 }
 390
 391                 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 392                 cond_resched();
 393         }
 394
 395         WARN_ON(active_domains);
 396
 397         fw = dev_priv->uncore.fw_domains_active;
 398         if (fw)
 399                 dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
 400
 401         fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains);
 402
 403         if (restore) { /* If reset with a user forcewake, try to restore */
 404                 if (fw)
 405                         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 406
 407                 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
 408                         dev_priv->uncore.fifo_count =
 409                                 fifo_free_entries(dev_priv);
 410         }
 411
 412         if (!restore)
 413                 assert_forcewakes_inactive(dev_priv);
 414
 415         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 416 }
 417
 418 static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
 419 {
 420         const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
 421         const unsigned int sets[4] = { 1, 1, 2, 2 };
 422         const u32 cap = dev_priv->edram_cap;
 423
 424         return EDRAM_NUM_BANKS(cap) *
 425                 ways[EDRAM_WAYS_IDX(cap)] *
 426                 sets[EDRAM_SETS_IDX(cap)] *
 427                 1024 * 1024;
 428 }
 429
 430 u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
 431 {
 432         if (!HAS_EDRAM(dev_priv))
 433                 return 0;
 434
 435         /* The needed capability bits for size calculation
 436          * are not there with pre gen9 so return 128MB always.
 437          */
 438         if (INTEL_GEN(dev_priv) < 9)
 439                 return 128 * 1024 * 1024;
 440
 441         return gen9_edram_size(dev_priv);
 442 }
 443
 444 static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
 445 {
 446         if (IS_HASWELL(dev_priv) ||
 447             IS_BROADWELL(dev_priv) ||
 448             INTEL_GEN(dev_priv) >= 9) {
 449                 dev_priv->edram_cap = __raw_i915_read32(dev_priv,
 450                                                         HSW_EDRAM_CAP);
 451
 452                 /* NB: We can't write IDICR yet because we do not have gt funcs
 453                  * set up */
 454         } else {
 455                 dev_priv->edram_cap = 0;
 456         }
 457
 458         if (HAS_EDRAM(dev_priv))
 459                 DRM_INFO("Found %lluMB of eDRAM\n",
 460                          intel_uncore_edram_size(dev_priv) / (1024 * 1024));
 461 }
 462
 463 static bool
 464 fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 465 {
 466         u32 dbg;
 467
 468         dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
 469         if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
 470                 return false;
 471
 472         __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 473
 474         return true;
 475 }
 476
 477 static bool
 478 vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 479 {
 480         u32 cer;
 481
 482         cer = __raw_i915_read32(dev_priv, CLAIM_ER);
 483         if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
 484                 return false;
 485
 486         __raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);
 487
 488         return true;
 489 }
 490
 491 static bool
 492 gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv)
 493 {
 494         u32 fifodbg;
 495
 496         fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 497
 498         if (unlikely(fifodbg)) {
 499                 DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg);
 500                 __raw_i915_write32(dev_priv, GTFIFODBG, fifodbg);
 501         }
 502
 503         return fifodbg;
 504 }
 505
 506 static bool
 507 check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 508 {
 509         bool ret = false;
 510
 511         if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
 512                 ret |= fpga_check_for_unclaimed_mmio(dev_priv);
 513
 514         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 515                 ret |= vlv_check_for_unclaimed_mmio(dev_priv);
 516
 517         if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
 518                 ret |= gen6_check_for_fifo_debug(dev_priv);
 519
 520         return ret;
 521 }
 522
 523 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 524                                           bool restore_forcewake)
 525 {
 526         /* clear out unclaimed reg detection bit */
 527         if (check_for_unclaimed_mmio(dev_priv))
 528                 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
 529
 530         /* WaDisableShadowRegForCpd:chv */
 531         if (IS_CHERRYVIEW(dev_priv)) {
 532                 __raw_i915_write32(dev_priv, GTFIFOCTL,
 533                                    __raw_i915_read32(dev_priv, GTFIFOCTL) |
 534                                    GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
 535                                    GT_FIFO_CTL_RC6_POLICY_STALL);
 536         }
 537
 538         iosf_mbi_punit_acquire();
 539         intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
 540         iosf_mbi_punit_release();
 541 }
 542
 543 void intel_uncore_suspend(struct drm_i915_private *dev_priv)
 544 {
 545         iosf_mbi_punit_acquire();
 546         iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 547                 &dev_priv->uncore.pmic_bus_access_nb);
 548         intel_uncore_forcewake_reset(dev_priv, false);
 549         iosf_mbi_punit_release();
 550 }
 551
 552 void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
 553 {
 554         __intel_uncore_early_sanitize(dev_priv, true);
 555         iosf_mbi_register_pmic_bus_access_notifier(
 556                 &dev_priv->uncore.pmic_bus_access_nb);
 557         i915_check_and_clear_faults(dev_priv);
 558 }
 559
 560 void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv)
 561 {
 562         iosf_mbi_register_pmic_bus_access_notifier(
 563                 &dev_priv->uncore.pmic_bus_access_nb);
 564 }
 565
 566 void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
 567 {
 568         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 569         intel_sanitize_gt_powersave(dev_priv);
 570 }
 571
 572 static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 573                                          enum forcewake_domains fw_domains)
 574 {
 575         struct intel_uncore_forcewake_domain *domain;
 576         unsigned int tmp;
 577
 578         fw_domains &= dev_priv->uncore.fw_domains;
 579
 580         for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
 581                 if (domain->wake_count++) {
 582                         fw_domains &= ~domain->mask;
 583                         domain->active = true;
 584                 }
 585         }
 586
 587         if (fw_domains)
 588                 dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
 589 }
 590
 591 /**
 592  * intel_uncore_forcewake_get - grab forcewake domain references
 593  * @dev_priv: i915 device instance
 594  * @fw_domains: forcewake domains to get reference on
 595  *
 596  * This function can be used get GT's forcewake domain references.
 597  * Normal register access will handle the forcewake domains automatically.
 598  * However if some sequence requires the GT to not power down a particular
 599  * forcewake domains this function should be called at the beginning of the
 600  * sequence. And subsequently the reference should be dropped by symmetric
 601  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 602  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
 603  */
 604 void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
 605                                 enum forcewake_domains fw_domains)
 606 {
 607         unsigned long irqflags;
 608
 609         if (!dev_priv->uncore.funcs.force_wake_get)
 610                 return;
 611
 612         assert_rpm_wakelock_held(dev_priv);
 613
 614         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 615         __intel_uncore_forcewake_get(dev_priv, fw_domains);
 616         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 617 }
 618
 619 /**
 620  * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
 621  * @dev_priv: i915 device instance
 622  *
 623  * This function is a wrapper around intel_uncore_forcewake_get() to acquire
 624  * the GT powerwell and in the process disable our debugging for the
 625  * duration of userspace's bypass.
 626  */
 627 void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv)
 628 {
 629         spin_lock_irq(&dev_priv->uncore.lock);
 630         if (!dev_priv->uncore.user_forcewake.count++) {
 631                 intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL);
 632
 633                 /* Save and disable mmio debugging for the user bypass */
 634                 dev_priv->uncore.user_forcewake.saved_mmio_check =
 635                         dev_priv->uncore.unclaimed_mmio_check;
 636                 dev_priv->uncore.user_forcewake.saved_mmio_debug =
 637                         i915_modparams.mmio_debug;
 638
 639                 dev_priv->uncore.unclaimed_mmio_check = 0;
 640                 i915_modparams.mmio_debug = 0;
 641         }
 642         spin_unlock_irq(&dev_priv->uncore.lock);
 643 }
 644
 645 /**
 646  * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
 647  * @dev_priv: i915 device instance
 648  *
 649  * This function complements intel_uncore_forcewake_user_get() and releases
 650  * the GT powerwell taken on behalf of the userspace bypass.
 651  */
 652 void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv)
 653 {
 654         spin_lock_irq(&dev_priv->uncore.lock);
 655         if (!--dev_priv->uncore.user_forcewake.count) {
 656                 if (intel_uncore_unclaimed_mmio(dev_priv))
 657                         dev_info(dev_priv->drm.dev,
 658                                  "Invalid mmio detected during user access\n");
 659
 660                 dev_priv->uncore.unclaimed_mmio_check =
 661                         dev_priv->uncore.user_forcewake.saved_mmio_check;
 662                 i915_modparams.mmio_debug =
 663                         dev_priv->uncore.user_forcewake.saved_mmio_debug;
 664
 665                 intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
 666         }
 667         spin_unlock_irq(&dev_priv->uncore.lock);
 668 }
 669
 670 /**
 671  * intel_uncore_forcewake_get__locked - grab forcewake domain references
 672  * @dev_priv: i915 device instance
 673  * @fw_domains: forcewake domains to get reference on
 674  *
 675  * See intel_uncore_forcewake_get(). This variant places the onus
 676  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 677  */
 678 void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
 679                                         enum forcewake_domains fw_domains)
 680 {
 681         lockdep_assert_held(&dev_priv->uncore.lock);
 682
 683         if (!dev_priv->uncore.funcs.force_wake_get)
 684                 return;
 685
 686         __intel_uncore_forcewake_get(dev_priv, fw_domains);
 687 }
 688
 689 static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 690                                          enum forcewake_domains fw_domains)
 691 {
 692         struct intel_uncore_forcewake_domain *domain;
 693         unsigned int tmp;
 694
 695         fw_domains &= dev_priv->uncore.fw_domains;
 696
 697         for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
 698                 if (WARN_ON(domain->wake_count == 0))
 699                         continue;
 700
 701                 if (--domain->wake_count) {
 702                         domain->active = true;
 703                         continue;
 704                 }
 705
 706                 fw_domain_arm_timer(domain);
 707         }
 708 }
 709
 710 /**
 711  * intel_uncore_forcewake_put - release a forcewake domain reference
 712  * @dev_priv: i915 device instance
 713  * @fw_domains: forcewake domains to put references
 714  *
 715  * This function drops the device-level forcewakes for specified
 716  * domains obtained by intel_uncore_forcewake_get().
 717  */
 718 void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
 719                                 enum forcewake_domains fw_domains)
 720 {
 721         unsigned long irqflags;
 722
 723         if (!dev_priv->uncore.funcs.force_wake_put)
 724                 return;
 725
 726         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 727         __intel_uncore_forcewake_put(dev_priv, fw_domains);
 728         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 729 }
 730
 731 /**
 732  * intel_uncore_forcewake_put__locked - grab forcewake domain references
 733  * @dev_priv: i915 device instance
 734  * @fw_domains: forcewake domains to get reference on
 735  *
 736  * See intel_uncore_forcewake_put(). This variant places the onus
 737  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 738  */
 739 void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
 740                                         enum forcewake_domains fw_domains)
 741 {
 742         lockdep_assert_held(&dev_priv->uncore.lock);
 743
 744         if (!dev_priv->uncore.funcs.force_wake_put)
 745                 return;
 746
 747         __intel_uncore_forcewake_put(dev_priv, fw_domains);
 748 }
 749
 750 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
 751 {
 752         if (!dev_priv->uncore.funcs.force_wake_get)
 753                 return;
 754
 755         WARN(dev_priv->uncore.fw_domains_active,
 756              "Expected all fw_domains to be inactive, but %08x are still on\n",
 757              dev_priv->uncore.fw_domains_active);
 758 }
 759
 760 void assert_forcewakes_active(struct drm_i915_private *dev_priv,
 761                               enum forcewake_domains fw_domains)
 762 {
 763         if (!dev_priv->uncore.funcs.force_wake_get)
 764                 return;
 765
 766         assert_rpm_wakelock_held(dev_priv);
 767
 768         fw_domains &= dev_priv->uncore.fw_domains;
 769         WARN(fw_domains & ~dev_priv->uncore.fw_domains_active,
 770              "Expected %08x fw_domains to be active, but %08x are off\n",
 771              fw_domains, fw_domains & ~dev_priv->uncore.fw_domains_active);
 772 }
 773
 774 /* We give fast paths for the really cool registers */
 775 #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
 776
 777 #define __gen6_reg_read_fw_domains(offset) \
 778 ({ \
 779         enum forcewake_domains __fwd; \
 780         if (NEEDS_FORCE_WAKE(offset)) \
 781                 __fwd = FORCEWAKE_RENDER; \
 782         else \
 783                 __fwd = 0; \
 784         __fwd; \
 785 })
 786
 787 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
 788 {
 789         if (offset < entry->start)
 790                 return -1;
 791         else if (offset > entry->end)
 792                 return 1;
 793         else
 794                 return 0;
 795 }
 796
 797 /* Copied and "macroized" from lib/bsearch.c */
 798 #define BSEARCH(key, base, num, cmp) ({                                 \
 799         unsigned int start__ = 0, end__ = (num);                        \
 800         typeof(base) result__ = NULL;                                   \
 801         while (start__ < end__) {                                       \
 802                 unsigned int mid__ = start__ + (end__ - start__) / 2;   \
 803                 int ret__ = (cmp)((key), (base) + mid__);               \
 804                 if (ret__ < 0) {                                        \
 805                         end__ = mid__;                                  \
 806                 } else if (ret__ > 0) {                                 \
 807                         start__ = mid__ + 1;                            \
 808                 } else {                                                \
 809                         result__ = (base) + mid__;                      \
 810                         break;                                          \
 811                 }                                                       \
 812         }                                                               \
 813         result__;                                                       \
 814 })
 815
 816 static enum forcewake_domains
 817 find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
 818 {
 819         const struct intel_forcewake_range *entry;
 820
 821         entry = BSEARCH(offset,
 822                         dev_priv->uncore.fw_domains_table,
 823                         dev_priv->uncore.fw_domains_table_entries,
 824                         fw_range_cmp);
 825
 826         if (!entry)
 827                 return 0;
 828
 829         WARN(entry->domains & ~dev_priv->uncore.fw_domains,
 830              "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
 831              entry->domains & ~dev_priv->uncore.fw_domains, offset);
 832
 833         return entry->domains;
 834 }
 835
 836 #define GEN_FW_RANGE(s, e, d) \
 837         { .start = (s), .end = (e), .domains = (d) }
 838
 839 #define HAS_FWTABLE(dev_priv) \
 840         (INTEL_GEN(dev_priv) >= 9 || \
 841          IS_CHERRYVIEW(dev_priv) || \
 842          IS_VALLEYVIEW(dev_priv))
 843
 844 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 845 static const struct intel_forcewake_range __vlv_fw_ranges[] = {
 846         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
 847         GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
 848         GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
 849         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 850         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
 851         GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
 852         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 853 };
 854
 855 #define __fwtable_reg_read_fw_domains(offset) \
 856 ({ \
 857         enum forcewake_domains __fwd = 0; \
 858         if (NEEDS_FORCE_WAKE((offset))) \
 859                 __fwd = find_fw_domain(dev_priv, offset); \
 860         __fwd; \
 861 })
 862
 863 /* *Must* be sorted by offset! See intel_shadow_table_check(). */
 864 static const i915_reg_t gen8_shadowed_regs[] = {
 865         RING_TAIL(RENDER_RING_BASE),    /* 0x2000 (base) */
 866         GEN6_RPNSWREQ,                  /* 0xA008 */
 867         GEN6_RC_VIDEO_FREQ,             /* 0xA00C */
 868         RING_TAIL(GEN6_BSD_RING_BASE),  /* 0x12000 (base) */
 869         RING_TAIL(VEBOX_RING_BASE),     /* 0x1a000 (base) */
 870         RING_TAIL(BLT_RING_BASE),       /* 0x22000 (base) */
 871         /* TODO: Other registers are not yet used */
 872 };
 873
 874 static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
 875 {
 876         u32 offset = i915_mmio_reg_offset(*reg);
 877
 878         if (key < offset)
 879                 return -1;
 880         else if (key > offset)
 881                 return 1;
 882         else
 883                 return 0;
 884 }
 885
 886 static bool is_gen8_shadowed(u32 offset)
 887 {
 888         const i915_reg_t *regs = gen8_shadowed_regs;
 889
 890         return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
 891                        mmio_reg_cmp);
 892 }
 893
 894 #define __gen8_reg_write_fw_domains(offset) \
 895 ({ \
 896         enum forcewake_domains __fwd; \
 897         if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
 898                 __fwd = FORCEWAKE_RENDER; \
 899         else \
 900                 __fwd = 0; \
 901         __fwd; \
 902 })
 903
 904 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 905 static const struct intel_forcewake_range __chv_fw_ranges[] = {
 906         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
 907         GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 908         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
 909         GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 910         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
 911         GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 912         GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
 913         GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 914         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
 915         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
 916         GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
 917         GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 918         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 919         GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
 920         GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
 921         GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
 922 };
 923
 924 #define __fwtable_reg_write_fw_domains(offset) \
 925 ({ \
 926         enum forcewake_domains __fwd = 0; \
 927         if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
 928                 __fwd = find_fw_domain(dev_priv, offset); \
 929         __fwd; \
 930 })
 931
 932 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 933 static const struct intel_forcewake_range __gen9_fw_ranges[] = {
 934         GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
 935         GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
 936         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
 937         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
 938         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
 939         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
 940         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
 941         GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
 942         GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
 943         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
 944         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
 945         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
 946         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
 947         GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
 948         GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
 949         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
 950         GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
 951         GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 952         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
 953         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
 954         GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
 955         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
 956         GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
 957         GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
 958         GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
 959         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 960         GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
 961         GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
 962         GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
 963         GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
 964         GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
 965         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 966 };
 967
 968 static void
 969 ilk_dummy_write(struct drm_i915_private *dev_priv)
 970 {
 971         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
 972          * the chip from rc6 before touching it for real. MI_MODE is masked,
 973          * hence harmless to write 0 into. */
 974         __raw_i915_write32(dev_priv, MI_MODE, 0);
 975 }
 976
 977 static void
 978 __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 979                       const i915_reg_t reg,
 980                       const bool read,
 981                       const bool before)
 982 {
 983         if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
 984                  "Unclaimed %s register 0x%x\n",
 985                  read ? "read from" : "write to",
 986                  i915_mmio_reg_offset(reg)))
 987                 /* Only report the first N failures */
 988                 i915_modparams.mmio_debug--;
 989 }
 990
 991 static inline void
 992 unclaimed_reg_debug(struct drm_i915_private *dev_priv,
 993                     const i915_reg_t reg,
 994                     const bool read,
 995                     const bool before)
 996 {
 997         if (likely(!i915_modparams.mmio_debug))
 998                 return;
 999
1000         __unclaimed_reg_debug(dev_priv, reg, read, before);
1001 }
1002
1003 #define GEN2_READ_HEADER(x) \
1004         u##x val = 0; \
1005         assert_rpm_wakelock_held(dev_priv);
1006
1007 #define GEN2_READ_FOOTER \
1008         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1009         return val
1010
1011 #define __gen2_read(x) \
1012 static u##x \
1013 gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1014         GEN2_READ_HEADER(x); \
1015         val = __raw_i915_read##x(dev_priv, reg); \
1016         GEN2_READ_FOOTER; \
1017 }
1018
1019 #define __gen5_read(x) \
1020 static u##x \
1021 gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1022         GEN2_READ_HEADER(x); \
1023         ilk_dummy_write(dev_priv); \
1024         val = __raw_i915_read##x(dev_priv, reg); \
1025         GEN2_READ_FOOTER; \
1026 }
1027
1028 __gen5_read(8)
1029 __gen5_read(16)
1030 __gen5_read(32)
1031 __gen5_read(64)
1032 __gen2_read(8)
1033 __gen2_read(16)
1034 __gen2_read(32)
1035 __gen2_read(64)
1036
1037 #undef __gen5_read
1038 #undef __gen2_read
1039
1040 #undef GEN2_READ_FOOTER
1041 #undef GEN2_READ_HEADER
1042
1043 #define GEN6_READ_HEADER(x) \
1044         u32 offset = i915_mmio_reg_offset(reg); \
1045         unsigned long irqflags; \
1046         u##x val = 0; \
1047         assert_rpm_wakelock_held(dev_priv); \
1048         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1049         unclaimed_reg_debug(dev_priv, reg, true, true)
1050
1051 #define GEN6_READ_FOOTER \
1052         unclaimed_reg_debug(dev_priv, reg, true, false); \
1053         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
1054         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1055         return val
1056
1057 static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
1058                                         enum forcewake_domains fw_domains)
1059 {
1060         struct intel_uncore_forcewake_domain *domain;
1061         unsigned int tmp;
1062
1063         GEM_BUG_ON(fw_domains & ~dev_priv->uncore.fw_domains);
1064
1065         for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp)
1066                 fw_domain_arm_timer(domain);
1067
1068         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
1069 }
1070
1071 static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
1072                                      enum forcewake_domains fw_domains)
1073 {
1074         if (WARN_ON(!fw_domains))
1075                 return;
1076
1077         /* Turn on all requested but inactive supported forcewake domains. */
1078         fw_domains &= dev_priv->uncore.fw_domains;
1079         fw_domains &= ~dev_priv->uncore.fw_domains_active;
1080
1081         if (fw_domains)
1082                 ___force_wake_auto(dev_priv, fw_domains);
1083 }
1084
1085 #define __gen_read(func, x) \
1086 static u##x \
1087 func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1088         enum forcewake_domains fw_engine; \
1089         GEN6_READ_HEADER(x); \
1090         fw_engine = __##func##_reg_read_fw_domains(offset); \
1091         if (fw_engine) \
1092                 __force_wake_auto(dev_priv, fw_engine); \
1093         val = __raw_i915_read##x(dev_priv, reg); \
1094         GEN6_READ_FOOTER; \
1095 }
1096 #define __gen6_read(x) __gen_read(gen6, x)
1097 #define __fwtable_read(x) __gen_read(fwtable, x)
1098
1099 __fwtable_read(8)
1100 __fwtable_read(16)
1101 __fwtable_read(32)
1102 __fwtable_read(64)
1103 __gen6_read(8)
1104 __gen6_read(16)
1105 __gen6_read(32)
1106 __gen6_read(64)
1107
1108 #undef __fwtable_read
1109 #undef __gen6_read
1110 #undef GEN6_READ_FOOTER
1111 #undef GEN6_READ_HEADER
1112
1113 #define GEN2_WRITE_HEADER \
1114         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1115         assert_rpm_wakelock_held(dev_priv); \
1116
1117 #define GEN2_WRITE_FOOTER
1118
1119 #define __gen2_write(x) \
1120 static void \
1121 gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1122         GEN2_WRITE_HEADER; \
1123         __raw_i915_write##x(dev_priv, reg, val); \
1124         GEN2_WRITE_FOOTER; \
1125 }
1126
1127 #define __gen5_write(x) \
1128 static void \
1129 gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1130         GEN2_WRITE_HEADER; \
1131         ilk_dummy_write(dev_priv); \
1132         __raw_i915_write##x(dev_priv, reg, val); \
1133         GEN2_WRITE_FOOTER; \
1134 }
1135
1136 __gen5_write(8)
1137 __gen5_write(16)
1138 __gen5_write(32)
1139 __gen2_write(8)
1140 __gen2_write(16)
1141 __gen2_write(32)
1142
1143 #undef __gen5_write
1144 #undef __gen2_write
1145
1146 #undef GEN2_WRITE_FOOTER
1147 #undef GEN2_WRITE_HEADER
1148
1149 #define GEN6_WRITE_HEADER \
1150         u32 offset = i915_mmio_reg_offset(reg); \
1151         unsigned long irqflags; \
1152         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1153         assert_rpm_wakelock_held(dev_priv); \
1154         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1155         unclaimed_reg_debug(dev_priv, reg, false, true)
1156
1157 #define GEN6_WRITE_FOOTER \
1158         unclaimed_reg_debug(dev_priv, reg, false, false); \
1159         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
1160
1161 #define __gen6_write(x) \
1162 static void \
1163 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1164         GEN6_WRITE_HEADER; \
1165         if (NEEDS_FORCE_WAKE(offset)) \
1166                 __gen6_gt_wait_for_fifo(dev_priv); \
1167         __raw_i915_write##x(dev_priv, reg, val); \
1168         GEN6_WRITE_FOOTER; \
1169 }
1170
1171 #define __gen_write(func, x) \
1172 static void \
1173 func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1174         enum forcewake_domains fw_engine; \
1175         GEN6_WRITE_HEADER; \
1176         fw_engine = __##func##_reg_write_fw_domains(offset); \
1177         if (fw_engine) \
1178                 __force_wake_auto(dev_priv, fw_engine); \
1179         __raw_i915_write##x(dev_priv, reg, val); \
1180         GEN6_WRITE_FOOTER; \
1181 }
1182 #define __gen8_write(x) __gen_write(gen8, x)
1183 #define __fwtable_write(x) __gen_write(fwtable, x)
1184
1185 __fwtable_write(8)
1186 __fwtable_write(16)
1187 __fwtable_write(32)
1188 __gen8_write(8)
1189 __gen8_write(16)
1190 __gen8_write(32)
1191 __gen6_write(8)
1192 __gen6_write(16)
1193 __gen6_write(32)
1194
1195 #undef __fwtable_write
1196 #undef __gen8_write
1197 #undef __gen6_write
1198 #undef GEN6_WRITE_FOOTER
1199 #undef GEN6_WRITE_HEADER
1200
1201 #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \
1202 do { \
1203         (i915)->uncore.funcs.mmio_writeb = x##_write8; \
1204         (i915)->uncore.funcs.mmio_writew = x##_write16; \
1205         (i915)->uncore.funcs.mmio_writel = x##_write32; \
1206 } while (0)
1207
1208 #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \
1209 do { \
1210         (i915)->uncore.funcs.mmio_readb = x##_read8; \
1211         (i915)->uncore.funcs.mmio_readw = x##_read16; \
1212         (i915)->uncore.funcs.mmio_readl = x##_read32; \
1213         (i915)->uncore.funcs.mmio_readq = x##_read64; \
1214 } while (0)
1215
1216
1217 static void fw_domain_init(struct drm_i915_private *dev_priv,
1218                            enum forcewake_domain_id domain_id,
1219                            i915_reg_t reg_set,
1220                            i915_reg_t reg_ack)
1221 {
1222         struct intel_uncore_forcewake_domain *d;
1223
1224         if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
1225                 return;
1226
1227         d = &dev_priv->uncore.fw_domain[domain_id];
1228
1229         WARN_ON(d->wake_count);
1230
1231         WARN_ON(!i915_mmio_reg_valid(reg_set));
1232         WARN_ON(!i915_mmio_reg_valid(reg_ack));
1233
1234         d->wake_count = 0;
1235         d->reg_set = reg_set;
1236         d->reg_ack = reg_ack;
1237
1238         d->id = domain_id;
1239
1240         BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
1241         BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
1242         BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
1243
1244         d->mask = BIT(domain_id);
1245
1246         hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1247         d->timer.function = intel_uncore_fw_release_timer;
1248
1249         dev_priv->uncore.fw_domains |= BIT(domain_id);
1250
1251         fw_domain_reset(dev_priv, d);
1252 }
1253
1254 static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
1255 {
1256         if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv))
1257                 return;
1258
1259         if (IS_GEN6(dev_priv)) {
1260                 dev_priv->uncore.fw_reset = 0;
1261                 dev_priv->uncore.fw_set = FORCEWAKE_KERNEL;
1262                 dev_priv->uncore.fw_clear = 0;
1263         } else {
1264                 /* WaRsClearFWBitsAtReset:bdw,skl */
1265                 dev_priv->uncore.fw_reset = _MASKED_BIT_DISABLE(0xffff);
1266                 dev_priv->uncore.fw_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL);
1267                 dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
1268         }
1269
1270         if (INTEL_GEN(dev_priv) >= 9) {
1271                 dev_priv->uncore.funcs.force_wake_get =
1272                         fw_domains_get_with_fallback;
1273                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1274                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1275                                FORCEWAKE_RENDER_GEN9,
1276                                FORCEWAKE_ACK_RENDER_GEN9);
1277                 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
1278                                FORCEWAKE_BLITTER_GEN9,
1279                                FORCEWAKE_ACK_BLITTER_GEN9);
1280                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
1281                                FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
1282         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1283                 dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
1284                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1285                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1286                                FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
1287                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
1288                                FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
1289         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1290                 dev_priv->uncore.funcs.force_wake_get =
1291                         fw_domains_get_with_thread_status;
1292                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1293                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1294                                FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
1295         } else if (IS_IVYBRIDGE(dev_priv)) {
1296                 u32 ecobus;
1297
1298                 /* IVB configs may use multi-threaded forcewake */
1299
1300                 /* A small trick here - if the bios hasn't configured
1301                  * MT forcewake, and if the device is in RC6, then
1302                  * force_wake_mt_get will not wake the device and the
1303                  * ECOBUS read will return zero. Which will be
1304                  * (correctly) interpreted by the test below as MT
1305                  * forcewake being disabled.
1306                  */
1307                 dev_priv->uncore.funcs.force_wake_get =
1308                         fw_domains_get_with_thread_status;
1309                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1310
1311                 /* We need to init first for ECOBUS access and then
1312                  * determine later if we want to reinit, in case of MT access is
1313                  * not working. In this stage we don't know which flavour this
1314                  * ivb is, so it is better to reset also the gen6 fw registers
1315                  * before the ecobus check.
1316                  */
1317
1318                 __raw_i915_write32(dev_priv, FORCEWAKE, 0);
1319                 __raw_posting_read(dev_priv, ECOBUS);
1320
1321                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1322                                FORCEWAKE_MT, FORCEWAKE_MT_ACK);
1323
1324                 spin_lock_irq(&dev_priv->uncore.lock);
1325                 fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER);
1326                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1327                 fw_domains_put(dev_priv, FORCEWAKE_RENDER);
1328                 spin_unlock_irq(&dev_priv->uncore.lock);
1329
1330                 if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
1331                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
1332                         DRM_INFO("when using vblank-synced partial screen updates.\n");
1333                         fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1334                                        FORCEWAKE, FORCEWAKE_ACK);
1335                 }
1336         } else if (IS_GEN6(dev_priv)) {
1337                 dev_priv->uncore.funcs.force_wake_get =
1338                         fw_domains_get_with_thread_status;
1339                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1340                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1341                                FORCEWAKE, FORCEWAKE_ACK);
1342         }
1343
1344         /* All future platforms are expected to require complex power gating */
1345         WARN_ON(dev_priv->uncore.fw_domains == 0);
1346 }
1347
1348 #define ASSIGN_FW_DOMAINS_TABLE(d) \
1349 { \
1350         dev_priv->uncore.fw_domains_table = \
1351                         (struct intel_forcewake_range *)(d); \
1352         dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
1353 }
1354
1355 static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
1356                                          unsigned long action, void *data)
1357 {
1358         struct drm_i915_private *dev_priv = container_of(nb,
1359                         struct drm_i915_private, uncore.pmic_bus_access_nb);
1360
1361         switch (action) {
1362         case MBI_PMIC_BUS_ACCESS_BEGIN:
1363                 /*
1364                  * forcewake all now to make sure that we don't need to do a
1365                  * forcewake later which on systems where this notifier gets
1366                  * called requires the punit to access to the shared pmic i2c
1367                  * bus, which will be busy after this notification, leading to:
1368                  * "render: timed out waiting for forcewake ack request."
1369                  * errors.
1370                  *
1371                  * The notifier is unregistered during intel_runtime_suspend(),
1372                  * so it's ok to access the HW here without holding a RPM
1373                  * wake reference -> disable wakeref asserts for the time of
1374                  * the access.
1375                  */
1376                 disable_rpm_wakeref_asserts(dev_priv);
1377                 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1378                 enable_rpm_wakeref_asserts(dev_priv);
1379                 break;
1380         case MBI_PMIC_BUS_ACCESS_END:
1381                 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1382                 break;
1383         }
1384
1385         return NOTIFY_OK;
1386 }
1387
1388 void intel_uncore_init(struct drm_i915_private *dev_priv)
1389 {
1390         i915_check_vgpu(dev_priv);
1391
1392         intel_uncore_edram_detect(dev_priv);
1393         intel_uncore_fw_domains_init(dev_priv);
1394         __intel_uncore_early_sanitize(dev_priv, false);
1395
1396         dev_priv->uncore.unclaimed_mmio_check = 1;
1397         dev_priv->uncore.pmic_bus_access_nb.notifier_call =
1398                 i915_pmic_bus_access_notifier;
1399
1400         if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) {
1401                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2);
1402                 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2);
1403         } else if (IS_GEN5(dev_priv)) {
1404                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5);
1405                 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5);
1406         } else if (IS_GEN(dev_priv, 6, 7)) {
1407                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6);
1408
1409                 if (IS_VALLEYVIEW(dev_priv)) {
1410                         ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
1411                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1412                 } else {
1413                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1414                 }
1415         } else if (IS_GEN8(dev_priv)) {
1416                 if (IS_CHERRYVIEW(dev_priv)) {
1417                         ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
1418                         ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1419                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1420
1421                 } else {
1422                         ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8);
1423                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1424                 }
1425         } else {
1426                 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1427                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1428                 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1429         }
1430
1431         iosf_mbi_register_pmic_bus_access_notifier(
1432                 &dev_priv->uncore.pmic_bus_access_nb);
1433 }
1434
1435 void intel_uncore_fini(struct drm_i915_private *dev_priv)
1436 {
1437         /* Paranoia: make sure we have disabled everything before we exit. */
1438         intel_uncore_sanitize(dev_priv);
1439
1440         iosf_mbi_punit_acquire();
1441         iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
1442                 &dev_priv->uncore.pmic_bus_access_nb);
1443         intel_uncore_forcewake_reset(dev_priv, false);
1444         iosf_mbi_punit_release();
1445 }
1446
1447 static const struct reg_whitelist {
1448         i915_reg_t offset_ldw;
1449         i915_reg_t offset_udw;
1450         u16 gen_mask;
1451         u8 size;
1452 } reg_read_whitelist[] = { {
1453         .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
1454         .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
1455         .gen_mask = INTEL_GEN_MASK(4, 11),
1456         .size = 8
1457 } };
1458
1459 int i915_reg_read_ioctl(struct drm_device *dev,
1460                         void *data, struct drm_file *file)
1461 {
1462         struct drm_i915_private *dev_priv = to_i915(dev);
1463         struct drm_i915_reg_read *reg = data;
1464         struct reg_whitelist const *entry;
1465         unsigned int flags;
1466         int remain;
1467         int ret = 0;
1468
1469         entry = reg_read_whitelist;
1470         remain = ARRAY_SIZE(reg_read_whitelist);
1471         while (remain) {
1472                 u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw);
1473
1474                 GEM_BUG_ON(!is_power_of_2(entry->size));
1475                 GEM_BUG_ON(entry->size > 8);
1476                 GEM_BUG_ON(entry_offset & (entry->size - 1));
1477
1478                 if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask &&
1479                     entry_offset == (reg->offset & -entry->size))
1480                         break;
1481                 entry++;
1482                 remain--;
1483         }
1484
1485         if (!remain)
1486                 return -EINVAL;
1487
1488         flags = reg->offset & (entry->size - 1);
1489
1490         intel_runtime_pm_get(dev_priv);
1491         if (entry->size == 8 && flags == I915_REG_READ_8B_WA)
1492                 reg->val = I915_READ64_2x32(entry->offset_ldw,
1493                                             entry->offset_udw);
1494         else if (entry->size == 8 && flags == 0)
1495                 reg->val = I915_READ64(entry->offset_ldw);
1496         else if (entry->size == 4 && flags == 0)
1497                 reg->val = I915_READ(entry->offset_ldw);
1498         else if (entry->size == 2 && flags == 0)
1499                 reg->val = I915_READ16(entry->offset_ldw);
1500         else if (entry->size == 1 && flags == 0)
1501                 reg->val = I915_READ8(entry->offset_ldw);
1502         else
1503                 ret = -EINVAL;
1504         intel_runtime_pm_put(dev_priv);
1505
1506         return ret;
1507 }
1508
1509 static void gen3_stop_engine(struct intel_engine_cs *engine)
1510 {
1511         struct drm_i915_private *dev_priv = engine->i915;
1512         const u32 base = engine->mmio_base;
1513         const i915_reg_t mode = RING_MI_MODE(base);
1514
1515         I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
1516         if (intel_wait_for_register_fw(dev_priv,
1517                                        mode,
1518                                        MODE_IDLE,
1519                                        MODE_IDLE,
1520                                        500))
1521                 DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
1522                                  engine->name);
1523
1524         I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base)));
1525         POSTING_READ_FW(RING_HEAD(base)); /* paranoia */
1526
1527         I915_WRITE_FW(RING_HEAD(base), 0);
1528         I915_WRITE_FW(RING_TAIL(base), 0);
1529         POSTING_READ_FW(RING_TAIL(base));
1530
1531         /* The ring must be empty before it is disabled */
1532         I915_WRITE_FW(RING_CTL(base), 0);
1533
1534         /* Check acts as a post */
1535         if (I915_READ_FW(RING_HEAD(base)) != 0)
1536                 DRM_DEBUG_DRIVER("%s: ring head not parked\n",
1537                                  engine->name);
1538 }
1539
1540 static void i915_stop_engines(struct drm_i915_private *dev_priv,
1541                               unsigned engine_mask)
1542 {
1543         struct intel_engine_cs *engine;
1544         enum intel_engine_id id;
1545
1546         if (INTEL_GEN(dev_priv) < 3)
1547                 return;
1548
1549         for_each_engine_masked(engine, dev_priv, engine_mask, id)
1550                 gen3_stop_engine(engine);
1551 }
1552
1553 static bool i915_in_reset(struct pci_dev *pdev)
1554 {
1555         u8 gdrst;
1556
1557         pci_read_config_byte(pdev, I915_GDRST, &gdrst);
1558         return gdrst & GRDOM_RESET_STATUS;
1559 }
1560
1561 static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1562 {
1563         struct pci_dev *pdev = dev_priv->drm.pdev;
1564         int err;
1565
1566         /* Assert reset for at least 20 usec, and wait for acknowledgement. */
1567         pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
1568         usleep_range(50, 200);
1569         err = wait_for(i915_in_reset(pdev), 500);
1570
1571         /* Clear the reset request. */
1572         pci_write_config_byte(pdev, I915_GDRST, 0);
1573         usleep_range(50, 200);
1574         if (!err)
1575                 err = wait_for(!i915_in_reset(pdev), 500);
1576
1577         return err;
1578 }
1579
1580 static bool g4x_reset_complete(struct pci_dev *pdev)
1581 {
1582         u8 gdrst;
1583
1584         pci_read_config_byte(pdev, I915_GDRST, &gdrst);
1585         return (gdrst & GRDOM_RESET_ENABLE) == 0;
1586 }
1587
1588 static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1589 {
1590         struct pci_dev *pdev = dev_priv->drm.pdev;
1591
1592         pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
1593         return wait_for(g4x_reset_complete(pdev), 500);
1594 }
1595
1596 static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1597 {
1598         struct pci_dev *pdev = dev_priv->drm.pdev;
1599         int ret;
1600
1601         /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1602         I915_WRITE(VDECCLK_GATE_D,
1603                    I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
1604         POSTING_READ(VDECCLK_GATE_D);
1605
1606         pci_write_config_byte(pdev, I915_GDRST,
1607                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1608         ret =  wait_for(g4x_reset_complete(pdev), 500);
1609         if (ret) {
1610                 DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1611                 goto out;
1612         }
1613
1614         pci_write_config_byte(pdev, I915_GDRST,
1615                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
1616         ret =  wait_for(g4x_reset_complete(pdev), 500);
1617         if (ret) {
1618                 DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1619                 goto out;
1620         }
1621
1622 out:
1623         pci_write_config_byte(pdev, I915_GDRST, 0);
1624
1625         I915_WRITE(VDECCLK_GATE_D,
1626                    I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
1627         POSTING_READ(VDECCLK_GATE_D);
1628
1629         return ret;
1630 }
1631
1632 static int ironlake_do_reset(struct drm_i915_private *dev_priv,
1633                              unsigned engine_mask)
1634 {
1635         int ret;
1636
1637         I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
1638         ret = intel_wait_for_register(dev_priv,
1639                                       ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
1640                                       500);
1641         if (ret) {
1642                 DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1643                 goto out;
1644         }
1645
1646         I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
1647         ret = intel_wait_for_register(dev_priv,
1648                                       ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
1649                                       500);
1650         if (ret) {
1651                 DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1652                 goto out;
1653         }
1654
1655 out:
1656         I915_WRITE(ILK_GDSR, 0);
1657         POSTING_READ(ILK_GDSR);
1658         return ret;
1659 }
1660
1661 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
1662 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
1663                                 u32 hw_domain_mask)
1664 {
1665         int err;
1666
1667         /* GEN6_GDRST is not in the gt power well, no need to check
1668          * for fifo space for the write or forcewake the chip for
1669          * the read
1670          */
1671         __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
1672
1673         /* Wait for the device to ack the reset requests */
1674         err = intel_wait_for_register_fw(dev_priv,
1675                                           GEN6_GDRST, hw_domain_mask, 0,
1676                                           500);
1677         if (err)
1678                 DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
1679                                  hw_domain_mask);
1680
1681         return err;
1682 }
1683
1684 /**
1685  * gen6_reset_engines - reset individual engines
1686  * @dev_priv: i915 device
1687  * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
1688  *
1689  * This function will reset the individual engines that are set in engine_mask.
1690  * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
1691  *
1692  * Note: It is responsibility of the caller to handle the difference between
1693  * asking full domain reset versus reset for all available individual engines.
1694  *
1695  * Returns 0 on success, nonzero on error.
1696  */
1697 static int gen6_reset_engines(struct drm_i915_private *dev_priv,
1698                               unsigned engine_mask)
1699 {
1700         struct intel_engine_cs *engine;
1701         const u32 hw_engine_mask[I915_NUM_ENGINES] = {
1702                 [RCS] = GEN6_GRDOM_RENDER,
1703                 [BCS] = GEN6_GRDOM_BLT,
1704                 [VCS] = GEN6_GRDOM_MEDIA,
1705                 [VCS2] = GEN8_GRDOM_MEDIA2,
1706                 [VECS] = GEN6_GRDOM_VECS,
1707         };
1708         u32 hw_mask;
1709
1710         if (engine_mask == ALL_ENGINES) {
1711                 hw_mask = GEN6_GRDOM_FULL;
1712         } else {
1713                 unsigned int tmp;
1714
1715                 hw_mask = 0;
1716                 for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1717                         hw_mask |= hw_engine_mask[engine->id];
1718         }
1719
1720         return gen6_hw_domain_reset(dev_priv, hw_mask);
1721 }
1722
1723 /**
1724  * __intel_wait_for_register_fw - wait until register matches expected state
1725  * @dev_priv: the i915 device
1726  * @reg: the register to read
1727  * @mask: mask to apply to register value
1728  * @value: expected value
1729  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
1730  * @slow_timeout_ms: slow timeout in millisecond
1731  * @out_value: optional placeholder to hold registry value
1732  *
1733  * This routine waits until the target register @reg contains the expected
1734  * @value after applying the @mask, i.e. it waits until ::
1735  *
1736  *     (I915_READ_FW(reg) & mask) == value
1737  *
1738  * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
1739  * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
1740  * must be not larger than 20,0000 microseconds.
1741  *
1742  * Note that this routine assumes the caller holds forcewake asserted, it is
1743  * not suitable for very long waits. See intel_wait_for_register() if you
1744  * wish to wait without holding forcewake for the duration (i.e. you expect
1745  * the wait to be slow).
1746  *
1747  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
1748  */
1749 int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
1750                                  i915_reg_t reg,
1751                                  u32 mask,
1752                                  u32 value,
1753                                  unsigned int fast_timeout_us,
1754                                  unsigned int slow_timeout_ms,
1755                                  u32 *out_value)
1756 {
1757         u32 uninitialized_var(reg_value);
1758 #define done (((reg_value = I915_READ_FW(reg)) & mask) == value)
1759         int ret;
1760
1761         /* Catch any overuse of this function */
1762         might_sleep_if(slow_timeout_ms);
1763         GEM_BUG_ON(fast_timeout_us > 20000);
1764
1765         ret = -ETIMEDOUT;
1766         if (fast_timeout_us && fast_timeout_us <= 20000)
1767                 ret = _wait_for_atomic(done, fast_timeout_us, 0);
1768         if (ret && slow_timeout_ms)
1769                 ret = wait_for(done, slow_timeout_ms);
1770
1771         if (out_value)
1772                 *out_value = reg_value;
1773
1774         return ret;
1775 #undef done
1776 }
1777
1778 /**
1779  * __intel_wait_for_register - wait until register matches expected state
1780  * @dev_priv: the i915 device
1781  * @reg: the register to read
1782  * @mask: mask to apply to register value
1783  * @value: expected value
1784  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
1785  * @slow_timeout_ms: slow timeout in millisecond
1786  * @out_value: optional placeholder to hold registry value
1787  *
1788  * This routine waits until the target register @reg contains the expected
1789  * @value after applying the @mask, i.e. it waits until ::
1790  *
1791  *     (I915_READ(reg) & mask) == value
1792  *
1793  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
1794  *
1795  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
1796  */
1797 int __intel_wait_for_register(struct drm_i915_private *dev_priv,
1798                             i915_reg_t reg,
1799                             u32 mask,
1800                             u32 value,
1801                             unsigned int fast_timeout_us,
1802                             unsigned int slow_timeout_ms,
1803                             u32 *out_value)
1804 {
1805         unsigned fw =
1806                 intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
1807         u32 reg_value;
1808         int ret;
1809
1810         might_sleep();
1811
1812         spin_lock_irq(&dev_priv->uncore.lock);
1813         intel_uncore_forcewake_get__locked(dev_priv, fw);
1814
1815         ret = __intel_wait_for_register_fw(dev_priv,
1816                                            reg, mask, value,
1817                                            fast_timeout_us, 0, &reg_value);
1818
1819         intel_uncore_forcewake_put__locked(dev_priv, fw);
1820         spin_unlock_irq(&dev_priv->uncore.lock);
1821
1822         if (ret)
1823                 ret = __wait_for(reg_value = I915_READ_NOTRACE(reg),
1824                                  (reg_value & mask) == value,
1825                                  slow_timeout_ms * 1000, 10, 1000);
1826
1827         if (out_value)
1828                 *out_value = reg_value;
1829
1830         return ret;
1831 }
1832
1833 static int gen8_reset_engine_start(struct intel_engine_cs *engine)
1834 {
1835         struct drm_i915_private *dev_priv = engine->i915;
1836         int ret;
1837
1838         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
1839                       _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
1840
1841         ret = intel_wait_for_register_fw(dev_priv,
1842                                          RING_RESET_CTL(engine->mmio_base),
1843                                          RESET_CTL_READY_TO_RESET,
1844                                          RESET_CTL_READY_TO_RESET,
1845                                          700);
1846         if (ret)
1847                 DRM_ERROR("%s: reset request timeout\n", engine->name);
1848
1849         return ret;
1850 }
1851
1852 static void gen8_reset_engine_cancel(struct intel_engine_cs *engine)
1853 {
1854         struct drm_i915_private *dev_priv = engine->i915;
1855
1856         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
1857                       _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
1858 }
1859
1860 static int gen8_reset_engines(struct drm_i915_private *dev_priv,
1861                               unsigned engine_mask)
1862 {
1863         struct intel_engine_cs *engine;
1864         unsigned int tmp;
1865
1866         for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1867                 if (gen8_reset_engine_start(engine))
1868                         goto not_ready;
1869
1870         return gen6_reset_engines(dev_priv, engine_mask);
1871
1872 not_ready:
1873         for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1874                 gen8_reset_engine_cancel(engine);
1875
1876         return -EIO;
1877 }
1878
1879 typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask);
1880
1881 static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
1882 {
1883         if (!i915_modparams.reset)
1884                 return NULL;
1885
1886         if (INTEL_GEN(dev_priv) >= 8)
1887                 return gen8_reset_engines;
1888         else if (INTEL_GEN(dev_priv) >= 6)
1889                 return gen6_reset_engines;
1890         else if (IS_GEN5(dev_priv))
1891                 return ironlake_do_reset;
1892         else if (IS_G4X(dev_priv))
1893                 return g4x_do_reset;
1894         else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
1895                 return g33_do_reset;
1896         else if (INTEL_GEN(dev_priv) >= 3)
1897                 return i915_do_reset;
1898         else
1899                 return NULL;
1900 }
1901
1902 int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1903 {
1904         reset_func reset = intel_get_gpu_reset(dev_priv);
1905         int retry;
1906         int ret;
1907
1908         might_sleep();
1909
1910         /* If the power well sleeps during the reset, the reset
1911          * request may be dropped and never completes (causing -EIO).
1912          */
1913         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1914         for (retry = 0; retry < 3; retry++) {
1915
1916                 /* We stop engines, otherwise we might get failed reset and a
1917                  * dead gpu (on elk). Also as modern gpu as kbl can suffer
1918                  * from system hang if batchbuffer is progressing when
1919                  * the reset is issued, regardless of READY_TO_RESET ack.
1920                  * Thus assume it is best to stop engines on all gens
1921                  * where we have a gpu reset.
1922                  *
1923                  * WaMediaResetMainRingCleanup:ctg,elk (presumably)
1924                  *
1925                  * FIXME: Wa for more modern gens needs to be validated
1926                  */
1927                 i915_stop_engines(dev_priv, engine_mask);
1928
1929                 ret = -ENODEV;
1930                 if (reset)
1931                         ret = reset(dev_priv, engine_mask);
1932                 if (ret != -ETIMEDOUT)
1933                         break;
1934
1935                 cond_resched();
1936         }
1937         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1938
1939         return ret;
1940 }
1941
1942 bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
1943 {
1944         return intel_get_gpu_reset(dev_priv) != NULL;
1945 }
1946
1947 bool intel_has_reset_engine(struct drm_i915_private *dev_priv)
1948 {
1949         return (dev_priv->info.has_reset_engine &&
1950                 i915_modparams.reset >= 2);
1951 }
1952
1953 int intel_reset_guc(struct drm_i915_private *dev_priv)
1954 {
1955         int ret;
1956
1957         GEM_BUG_ON(!HAS_GUC(dev_priv));
1958
1959         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1960         ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);
1961         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1962
1963         return ret;
1964 }
1965
1966 bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
1967 {
1968         return check_for_unclaimed_mmio(dev_priv);
1969 }
1970
1971 bool
1972 intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
1973 {
1974         if (unlikely(i915_modparams.mmio_debug ||
1975                      dev_priv->uncore.unclaimed_mmio_check <= 0))
1976                 return false;
1977
1978         if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
1979                 DRM_DEBUG("Unclaimed register detected, "
1980                           "enabling oneshot unclaimed register reporting. "
1981                           "Please use i915.mmio_debug=N for more information.\n");
1982                 i915_modparams.mmio_debug++;
1983                 dev_priv->uncore.unclaimed_mmio_check--;
1984                 return true;
1985         }
1986
1987         return false;
1988 }
1989
1990 static enum forcewake_domains
1991 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
1992                                 i915_reg_t reg)
1993 {
1994         u32 offset = i915_mmio_reg_offset(reg);
1995         enum forcewake_domains fw_domains;
1996
1997         if (HAS_FWTABLE(dev_priv)) {
1998                 fw_domains = __fwtable_reg_read_fw_domains(offset);
1999         } else if (INTEL_GEN(dev_priv) >= 6) {
2000                 fw_domains = __gen6_reg_read_fw_domains(offset);
2001         } else {
2002                 WARN_ON(!IS_GEN(dev_priv, 2, 5));
2003                 fw_domains = 0;
2004         }
2005
2006         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
2007
2008         return fw_domains;
2009 }
2010
2011 static enum forcewake_domains
2012 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
2013                                  i915_reg_t reg)
2014 {
2015         u32 offset = i915_mmio_reg_offset(reg);
2016         enum forcewake_domains fw_domains;
2017
2018         if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
2019                 fw_domains = __fwtable_reg_write_fw_domains(offset);
2020         } else if (IS_GEN8(dev_priv)) {
2021                 fw_domains = __gen8_reg_write_fw_domains(offset);
2022         } else if (IS_GEN(dev_priv, 6, 7)) {
2023                 fw_domains = FORCEWAKE_RENDER;
2024         } else {
2025                 WARN_ON(!IS_GEN(dev_priv, 2, 5));
2026                 fw_domains = 0;
2027         }
2028
2029         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
2030
2031         return fw_domains;
2032 }
2033
2034 /**
2035  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
2036  *                                  a register
2037  * @dev_priv: pointer to struct drm_i915_private
2038  * @reg: register in question
2039  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
2040  *
2041  * Returns a set of forcewake domains required to be taken with for example
2042  * intel_uncore_forcewake_get for the specified register to be accessible in the
2043  * specified mode (read, write or read/write) with raw mmio accessors.
2044  *
2045  * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
2046  * callers to do FIFO management on their own or risk losing writes.
2047  */
2048 enum forcewake_domains
2049 intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
2050                                i915_reg_t reg, unsigned int op)
2051 {
2052         enum forcewake_domains fw_domains = 0;
2053
2054         WARN_ON(!op);
2055
2056         if (intel_vgpu_active(dev_priv))
2057                 return 0;
2058
2059         if (op & FW_REG_READ)
2060                 fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
2061
2062         if (op & FW_REG_WRITE)
2063                 fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);
2064
2065         return fw_domains;
2066 }
2067
2068 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2069 #include "selftests/mock_uncore.c"
2070 #include "selftests/intel_uncore.c"
2071 #endif