drivers/gpu/drm/i915/intel_uncore.c

   1 /*
   2  * Copyright © 2013 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #include "i915_drv.h"
  25 #include "intel_drv.h"
  26 #include "i915_vgpu.h"
  27
  28 #include <asm/iosf_mbi.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #define FORCEWAKE_ACK_TIMEOUT_MS 50
  32 #define GT_FIFO_TIMEOUT_MS       10
  33
  34 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
  35
  36 static const char * const forcewake_domain_names[] = {
  37         "render",
  38         "blitter",
  39         "media",
  40         "vdbox0",
  41         "vdbox1",
  42         "vdbox2",
  43         "vdbox3",
  44         "vebox0",
  45         "vebox1",
  46 };
  47
  48 const char *
  49 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
  50 {
  51         BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
  52
  53         if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
  54                 return forcewake_domain_names[id];
  55
  56         WARN_ON(id);
  57
  58         return "unknown";
  59 }
  60
  61 #define fw_ack(d) readl((d)->reg_ack)
  62 #define fw_set(d, val) writel(_MASKED_BIT_ENABLE((val)), (d)->reg_set)
  63 #define fw_clear(d, val) writel(_MASKED_BIT_DISABLE((val)), (d)->reg_set)
  64
  65 static inline void
  66 fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
  67 {
  68         /*
  69          * We don't really know if the powerwell for the forcewake domain we are
  70          * trying to reset here does exist at this point (engines could be fused
  71          * off in ICL+), so no waiting for acks
  72          */
  73         /* WaRsClearFWBitsAtReset:bdw,skl */
  74         fw_clear(d, 0xffff);
  75 }
  76
  77 static inline void
  78 fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
  79 {
  80         d->wake_count++;
  81         hrtimer_start_range_ns(&d->timer,
  82                                NSEC_PER_MSEC,
  83                                NSEC_PER_MSEC,
  84                                HRTIMER_MODE_REL);
  85 }
  86
  87 static inline int
  88 __wait_for_ack(const struct intel_uncore_forcewake_domain *d,
  89                const u32 ack,
  90                const u32 value)
  91 {
  92         return wait_for_atomic((fw_ack(d) & ack) == value,
  93                                FORCEWAKE_ACK_TIMEOUT_MS);
  94 }
  95
  96 static inline int
  97 wait_ack_clear(const struct intel_uncore_forcewake_domain *d,
  98                const u32 ack)
  99 {
 100         return __wait_for_ack(d, ack, 0);
 101 }
 102
 103 static inline int
 104 wait_ack_set(const struct intel_uncore_forcewake_domain *d,
 105              const u32 ack)
 106 {
 107         return __wait_for_ack(d, ack, ack);
 108 }
 109
 110 static inline void
 111 fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
 112 {
 113         if (wait_ack_clear(d, FORCEWAKE_KERNEL))
 114                 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
 115                           intel_uncore_forcewake_domain_to_str(d->id));
 116 }
 117
 118 enum ack_type {
 119         ACK_CLEAR = 0,
 120         ACK_SET
 121 };
 122
 123 static int
 124 fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d,
 125                                  const enum ack_type type)
 126 {
 127         const u32 ack_bit = FORCEWAKE_KERNEL;
 128         const u32 value = type == ACK_SET ? ack_bit : 0;
 129         unsigned int pass;
 130         bool ack_detected;
 131
 132         /*
 133          * There is a possibility of driver's wake request colliding
 134          * with hardware's own wake requests and that can cause
 135          * hardware to not deliver the driver's ack message.
 136          *
 137          * Use a fallback bit toggle to kick the gpu state machine
 138          * in the hope that the original ack will be delivered along with
 139          * the fallback ack.
 140          *
 141          * This workaround is described in HSDES #1604254524 and it's known as:
 142          * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
 143          * although the name is a bit misleading.
 144          */
 145
 146         pass = 1;
 147         do {
 148                 wait_ack_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 149
 150                 fw_set(d, FORCEWAKE_KERNEL_FALLBACK);
 151                 /* Give gt some time to relax before the polling frenzy */
 152                 udelay(10 * pass);
 153                 wait_ack_set(d, FORCEWAKE_KERNEL_FALLBACK);
 154
 155                 ack_detected = (fw_ack(d) & ack_bit) == value;
 156
 157                 fw_clear(d, FORCEWAKE_KERNEL_FALLBACK);
 158         } while (!ack_detected && pass++ < 10);
 159
 160         DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",
 161                          intel_uncore_forcewake_domain_to_str(d->id),
 162                          type == ACK_SET ? "set" : "clear",
 163                          fw_ack(d),
 164                          pass);
 165
 166         return ack_detected ? 0 : -ETIMEDOUT;
 167 }
 168
 169 static inline void
 170 fw_domain_wait_ack_clear_fallback(const struct intel_uncore_forcewake_domain *d)
 171 {
 172         if (likely(!wait_ack_clear(d, FORCEWAKE_KERNEL)))
 173                 return;
 174
 175         if (fw_domain_wait_ack_with_fallback(d, ACK_CLEAR))
 176                 fw_domain_wait_ack_clear(d);
 177 }
 178
 179 static inline void
 180 fw_domain_get(const struct intel_uncore_forcewake_domain *d)
 181 {
 182         fw_set(d, FORCEWAKE_KERNEL);
 183 }
 184
 185 static inline void
 186 fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d)
 187 {
 188         if (wait_ack_set(d, FORCEWAKE_KERNEL))
 189                 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
 190                           intel_uncore_forcewake_domain_to_str(d->id));
 191 }
 192
 193 static inline void
 194 fw_domain_wait_ack_set_fallback(const struct intel_uncore_forcewake_domain *d)
 195 {
 196         if (likely(!wait_ack_set(d, FORCEWAKE_KERNEL)))
 197                 return;
 198
 199         if (fw_domain_wait_ack_with_fallback(d, ACK_SET))
 200                 fw_domain_wait_ack_set(d);
 201 }
 202
 203 static inline void
 204 fw_domain_put(const struct intel_uncore_forcewake_domain *d)
 205 {
 206         fw_clear(d, FORCEWAKE_KERNEL);
 207 }
 208
 209 static void
 210 fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 211 {
 212         struct intel_uncore_forcewake_domain *d;
 213         unsigned int tmp;
 214
 215         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 216
 217         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 218                 fw_domain_wait_ack_clear(d);
 219                 fw_domain_get(d);
 220         }
 221
 222         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 223                 fw_domain_wait_ack_set(d);
 224
 225         uncore->fw_domains_active |= fw_domains;
 226 }
 227
 228 static void
 229 fw_domains_get_with_fallback(struct intel_uncore *uncore,
 230                              enum forcewake_domains fw_domains)
 231 {
 232         struct intel_uncore_forcewake_domain *d;
 233         unsigned int tmp;
 234
 235         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 236
 237         for_each_fw_domain_masked(d, fw_domains, uncore, tmp) {
 238                 fw_domain_wait_ack_clear_fallback(d);
 239                 fw_domain_get(d);
 240         }
 241
 242         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 243                 fw_domain_wait_ack_set_fallback(d);
 244
 245         uncore->fw_domains_active |= fw_domains;
 246 }
 247
 248 static void
 249 fw_domains_put(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 250 {
 251         struct intel_uncore_forcewake_domain *d;
 252         unsigned int tmp;
 253
 254         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 255
 256         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 257                 fw_domain_put(d);
 258
 259         uncore->fw_domains_active &= ~fw_domains;
 260 }
 261
 262 static void
 263 fw_domains_reset(struct intel_uncore *uncore,
 264                  enum forcewake_domains fw_domains)
 265 {
 266         struct intel_uncore_forcewake_domain *d;
 267         unsigned int tmp;
 268
 269         if (!fw_domains)
 270                 return;
 271
 272         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
 273
 274         for_each_fw_domain_masked(d, fw_domains, uncore, tmp)
 275                 fw_domain_reset(d);
 276 }
 277
 278 static inline u32 gt_thread_status(struct drm_i915_private *dev_priv)
 279 {
 280         u32 val;
 281
 282         val = __raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG);
 283         val &= GEN6_GT_THREAD_STATUS_CORE_MASK;
 284
 285         return val;
 286 }
 287
 288 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
 289 {
 290         /*
 291          * w/a for a sporadic read returning 0 by waiting for the GT
 292          * thread to wake up.
 293          */
 294         WARN_ONCE(wait_for_atomic_us(gt_thread_status(dev_priv) == 0, 5000),
 295                   "GT thread status wait timed out\n");
 296 }
 297
 298 static void fw_domains_get_with_thread_status(struct intel_uncore *uncore,
 299                                               enum forcewake_domains fw_domains)
 300 {
 301         fw_domains_get(uncore, fw_domains);
 302
 303         /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
 304         __gen6_gt_wait_for_thread_c0(uncore_to_i915(uncore));
 305 }
 306
 307 static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
 308 {
 309         u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);
 310
 311         return count & GT_FIFO_FREE_ENTRIES_MASK;
 312 }
 313
 314 static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 315 {
 316         u32 n;
 317
 318         /* On VLV, FIFO will be shared by both SW and HW.
 319          * So, we need to read the FREE_ENTRIES everytime */
 320         if (IS_VALLEYVIEW(dev_priv))
 321                 n = fifo_free_entries(dev_priv);
 322         else
 323                 n = dev_priv->uncore.fifo_count;
 324
 325         if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
 326                 if (wait_for_atomic((n = fifo_free_entries(dev_priv)) >
 327                                     GT_FIFO_NUM_RESERVED_ENTRIES,
 328                                     GT_FIFO_TIMEOUT_MS)) {
 329                         DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n);
 330                         return;
 331                 }
 332         }
 333
 334         dev_priv->uncore.fifo_count = n - 1;
 335 }
 336
 337 static enum hrtimer_restart
 338 intel_uncore_fw_release_timer(struct hrtimer *timer)
 339 {
 340         struct intel_uncore_forcewake_domain *domain =
 341                container_of(timer, struct intel_uncore_forcewake_domain, timer);
 342         struct intel_uncore *uncore = forcewake_domain_to_uncore(domain);
 343         unsigned long irqflags;
 344
 345         assert_rpm_device_not_suspended(uncore_to_i915(uncore));
 346
 347         if (xchg(&domain->active, false))
 348                 return HRTIMER_RESTART;
 349
 350         spin_lock_irqsave(&uncore->lock, irqflags);
 351         if (WARN_ON(domain->wake_count == 0))
 352                 domain->wake_count++;
 353
 354         if (--domain->wake_count == 0)
 355                 uncore->funcs.force_wake_put(uncore, domain->mask);
 356
 357         spin_unlock_irqrestore(&uncore->lock, irqflags);
 358
 359         return HRTIMER_NORESTART;
 360 }
 361
 362 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 363 static unsigned int
 364 intel_uncore_forcewake_reset(struct intel_uncore *uncore)
 365 {
 366         unsigned long irqflags;
 367         struct intel_uncore_forcewake_domain *domain;
 368         int retry_count = 100;
 369         enum forcewake_domains fw, active_domains;
 370
 371         iosf_mbi_assert_punit_acquired();
 372
 373         /* Hold uncore.lock across reset to prevent any register access
 374          * with forcewake not set correctly. Wait until all pending
 375          * timers are run before holding.
 376          */
 377         while (1) {
 378                 unsigned int tmp;
 379
 380                 active_domains = 0;
 381
 382                 for_each_fw_domain(domain, uncore, tmp) {
 383                         smp_store_mb(domain->active, false);
 384                         if (hrtimer_cancel(&domain->timer) == 0)
 385                                 continue;
 386
 387                         intel_uncore_fw_release_timer(&domain->timer);
 388                 }
 389
 390                 spin_lock_irqsave(&uncore->lock, irqflags);
 391
 392                 for_each_fw_domain(domain, uncore, tmp) {
 393                         if (hrtimer_active(&domain->timer))
 394                                 active_domains |= domain->mask;
 395                 }
 396
 397                 if (active_domains == 0)
 398                         break;
 399
 400                 if (--retry_count == 0) {
 401                         DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
 402                         break;
 403                 }
 404
 405                 spin_unlock_irqrestore(&uncore->lock, irqflags);
 406                 cond_resched();
 407         }
 408
 409         WARN_ON(active_domains);
 410
 411         fw = uncore->fw_domains_active;
 412         if (fw)
 413                 uncore->funcs.force_wake_put(uncore, fw);
 414
 415         fw_domains_reset(uncore, uncore->fw_domains);
 416         assert_forcewakes_inactive(uncore);
 417
 418         spin_unlock_irqrestore(&uncore->lock, irqflags);
 419
 420         return fw; /* track the lost user forcewake domains */
 421 }
 422
 423 static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
 424 {
 425         const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
 426         const unsigned int sets[4] = { 1, 1, 2, 2 };
 427         const u32 cap = dev_priv->edram_cap;
 428
 429         return EDRAM_NUM_BANKS(cap) *
 430                 ways[EDRAM_WAYS_IDX(cap)] *
 431                 sets[EDRAM_SETS_IDX(cap)] *
 432                 1024 * 1024;
 433 }
 434
 435 u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
 436 {
 437         if (!HAS_EDRAM(dev_priv))
 438                 return 0;
 439
 440         /* The needed capability bits for size calculation
 441          * are not there with pre gen9 so return 128MB always.
 442          */
 443         if (INTEL_GEN(dev_priv) < 9)
 444                 return 128 * 1024 * 1024;
 445
 446         return gen9_edram_size(dev_priv);
 447 }
 448
 449 static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
 450 {
 451         if (IS_HASWELL(dev_priv) ||
 452             IS_BROADWELL(dev_priv) ||
 453             INTEL_GEN(dev_priv) >= 9) {
 454                 dev_priv->edram_cap = __raw_i915_read32(dev_priv,
 455                                                         HSW_EDRAM_CAP);
 456
 457                 /* NB: We can't write IDICR yet because we do not have gt funcs
 458                  * set up */
 459         } else {
 460                 dev_priv->edram_cap = 0;
 461         }
 462
 463         if (HAS_EDRAM(dev_priv))
 464                 DRM_INFO("Found %lluMB of eDRAM\n",
 465                          intel_uncore_edram_size(dev_priv) / (1024 * 1024));
 466 }
 467
 468 static bool
 469 fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 470 {
 471         u32 dbg;
 472
 473         dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
 474         if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
 475                 return false;
 476
 477         __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
 478
 479         return true;
 480 }
 481
 482 static bool
 483 vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 484 {
 485         u32 cer;
 486
 487         cer = __raw_i915_read32(dev_priv, CLAIM_ER);
 488         if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
 489                 return false;
 490
 491         __raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);
 492
 493         return true;
 494 }
 495
 496 static bool
 497 gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv)
 498 {
 499         u32 fifodbg;
 500
 501         fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
 502
 503         if (unlikely(fifodbg)) {
 504                 DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg);
 505                 __raw_i915_write32(dev_priv, GTFIFODBG, fifodbg);
 506         }
 507
 508         return fifodbg;
 509 }
 510
 511 static bool
 512 check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
 513 {
 514         bool ret = false;
 515
 516         if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
 517                 ret |= fpga_check_for_unclaimed_mmio(dev_priv);
 518
 519         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 520                 ret |= vlv_check_for_unclaimed_mmio(dev_priv);
 521
 522         if (IS_GEN_RANGE(dev_priv, 6, 7))
 523                 ret |= gen6_check_for_fifo_debug(dev_priv);
 524
 525         return ret;
 526 }
 527
 528 static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
 529                                           unsigned int restore_forcewake)
 530 {
 531         /* clear out unclaimed reg detection bit */
 532         if (check_for_unclaimed_mmio(dev_priv))
 533                 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
 534
 535         /* WaDisableShadowRegForCpd:chv */
 536         if (IS_CHERRYVIEW(dev_priv)) {
 537                 __raw_i915_write32(dev_priv, GTFIFOCTL,
 538                                    __raw_i915_read32(dev_priv, GTFIFOCTL) |
 539                                    GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
 540                                    GT_FIFO_CTL_RC6_POLICY_STALL);
 541         }
 542
 543         iosf_mbi_punit_acquire();
 544         intel_uncore_forcewake_reset(&dev_priv->uncore);
 545         if (restore_forcewake) {
 546                 spin_lock_irq(&dev_priv->uncore.lock);
 547                 dev_priv->uncore.funcs.force_wake_get(&dev_priv->uncore,
 548                                                       restore_forcewake);
 549
 550                 if (IS_GEN_RANGE(dev_priv, 6, 7))
 551                         dev_priv->uncore.fifo_count =
 552                                 fifo_free_entries(dev_priv);
 553                 spin_unlock_irq(&dev_priv->uncore.lock);
 554         }
 555         iosf_mbi_punit_release();
 556 }
 557
 558 void intel_uncore_suspend(struct drm_i915_private *dev_priv)
 559 {
 560         iosf_mbi_punit_acquire();
 561         iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
 562                 &dev_priv->uncore.pmic_bus_access_nb);
 563         dev_priv->uncore.fw_domains_saved =
 564                 intel_uncore_forcewake_reset(&dev_priv->uncore);
 565         iosf_mbi_punit_release();
 566 }
 567
 568 void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
 569 {
 570         unsigned int restore_forcewake;
 571
 572         restore_forcewake = fetch_and_zero(&dev_priv->uncore.fw_domains_saved);
 573         __intel_uncore_early_sanitize(dev_priv, restore_forcewake);
 574
 575         iosf_mbi_register_pmic_bus_access_notifier(
 576                 &dev_priv->uncore.pmic_bus_access_nb);
 577         i915_check_and_clear_faults(dev_priv);
 578 }
 579
 580 void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv)
 581 {
 582         iosf_mbi_register_pmic_bus_access_notifier(
 583                 &dev_priv->uncore.pmic_bus_access_nb);
 584 }
 585
 586 void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
 587 {
 588         /* BIOS often leaves RC6 enabled, but disable it for hw init */
 589         intel_sanitize_gt_powersave(dev_priv);
 590 }
 591
 592 static void __intel_uncore_forcewake_get(struct intel_uncore *uncore,
 593                                          enum forcewake_domains fw_domains)
 594 {
 595         struct intel_uncore_forcewake_domain *domain;
 596         unsigned int tmp;
 597
 598         fw_domains &= uncore->fw_domains;
 599
 600         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 601                 if (domain->wake_count++) {
 602                         fw_domains &= ~domain->mask;
 603                         domain->active = true;
 604                 }
 605         }
 606
 607         if (fw_domains)
 608                 uncore->funcs.force_wake_get(uncore, fw_domains);
 609 }
 610
 611 /**
 612  * intel_uncore_forcewake_get - grab forcewake domain references
 613  * @uncore: the intel_uncore structure
 614  * @fw_domains: forcewake domains to get reference on
 615  *
 616  * This function can be used get GT's forcewake domain references.
 617  * Normal register access will handle the forcewake domains automatically.
 618  * However if some sequence requires the GT to not power down a particular
 619  * forcewake domains this function should be called at the beginning of the
 620  * sequence. And subsequently the reference should be dropped by symmetric
 621  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 622  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
 623  */
 624 void intel_uncore_forcewake_get(struct intel_uncore *uncore,
 625                                 enum forcewake_domains fw_domains)
 626 {
 627         unsigned long irqflags;
 628
 629         if (!uncore->funcs.force_wake_get)
 630                 return;
 631
 632         assert_rpm_wakelock_held(uncore_to_i915(uncore));
 633
 634         spin_lock_irqsave(&uncore->lock, irqflags);
 635         __intel_uncore_forcewake_get(uncore, fw_domains);
 636         spin_unlock_irqrestore(&uncore->lock, irqflags);
 637 }
 638
 639 /**
 640  * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
 641  * @uncore: the intel_uncore structure
 642  *
 643  * This function is a wrapper around intel_uncore_forcewake_get() to acquire
 644  * the GT powerwell and in the process disable our debugging for the
 645  * duration of userspace's bypass.
 646  */
 647 void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
 648 {
 649         spin_lock_irq(&uncore->lock);
 650         if (!uncore->user_forcewake.count++) {
 651                 intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL);
 652
 653                 /* Save and disable mmio debugging for the user bypass */
 654                 uncore->user_forcewake.saved_mmio_check =
 655                         uncore->unclaimed_mmio_check;
 656                 uncore->user_forcewake.saved_mmio_debug =
 657                         i915_modparams.mmio_debug;
 658
 659                 uncore->unclaimed_mmio_check = 0;
 660                 i915_modparams.mmio_debug = 0;
 661         }
 662         spin_unlock_irq(&uncore->lock);
 663 }
 664
 665 /**
 666  * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
 667  * @uncore: the intel_uncore structure
 668  *
 669  * This function complements intel_uncore_forcewake_user_get() and releases
 670  * the GT powerwell taken on behalf of the userspace bypass.
 671  */
 672 void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
 673 {
 674         struct drm_i915_private *i915 = uncore_to_i915(uncore);
 675
 676         spin_lock_irq(&uncore->lock);
 677         if (!--uncore->user_forcewake.count) {
 678                 if (intel_uncore_unclaimed_mmio(i915))
 679                         dev_info(i915->drm.dev,
 680                                  "Invalid mmio detected during user access\n");
 681
 682                 uncore->unclaimed_mmio_check =
 683                         uncore->user_forcewake.saved_mmio_check;
 684                 i915_modparams.mmio_debug =
 685                         uncore->user_forcewake.saved_mmio_debug;
 686
 687                 intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL);
 688         }
 689         spin_unlock_irq(&uncore->lock);
 690 }
 691
 692 /**
 693  * intel_uncore_forcewake_get__locked - grab forcewake domain references
 694  * @uncore: the intel_uncore structure
 695  * @fw_domains: forcewake domains to get reference on
 696  *
 697  * See intel_uncore_forcewake_get(). This variant places the onus
 698  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 699  */
 700 void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore,
 701                                         enum forcewake_domains fw_domains)
 702 {
 703         lockdep_assert_held(&uncore->lock);
 704
 705         if (!uncore->funcs.force_wake_get)
 706                 return;
 707
 708         __intel_uncore_forcewake_get(uncore, fw_domains);
 709 }
 710
 711 static void __intel_uncore_forcewake_put(struct intel_uncore *uncore,
 712                                          enum forcewake_domains fw_domains)
 713 {
 714         struct intel_uncore_forcewake_domain *domain;
 715         unsigned int tmp;
 716
 717         fw_domains &= uncore->fw_domains;
 718
 719         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) {
 720                 if (WARN_ON(domain->wake_count == 0))
 721                         continue;
 722
 723                 if (--domain->wake_count) {
 724                         domain->active = true;
 725                         continue;
 726                 }
 727
 728                 fw_domain_arm_timer(domain);
 729         }
 730 }
 731
 732 /**
 733  * intel_uncore_forcewake_put - release a forcewake domain reference
 734  * @uncore: the intel_uncore structure
 735  * @fw_domains: forcewake domains to put references
 736  *
 737  * This function drops the device-level forcewakes for specified
 738  * domains obtained by intel_uncore_forcewake_get().
 739  */
 740 void intel_uncore_forcewake_put(struct intel_uncore *uncore,
 741                                 enum forcewake_domains fw_domains)
 742 {
 743         unsigned long irqflags;
 744
 745         if (!uncore->funcs.force_wake_put)
 746                 return;
 747
 748         spin_lock_irqsave(&uncore->lock, irqflags);
 749         __intel_uncore_forcewake_put(uncore, fw_domains);
 750         spin_unlock_irqrestore(&uncore->lock, irqflags);
 751 }
 752
 753 /**
 754  * intel_uncore_forcewake_put__locked - grab forcewake domain references
 755  * @uncore: the intel_uncore structure
 756  * @fw_domains: forcewake domains to get reference on
 757  *
 758  * See intel_uncore_forcewake_put(). This variant places the onus
 759  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 760  */
 761 void intel_uncore_forcewake_put__locked(struct intel_uncore *uncore,
 762                                         enum forcewake_domains fw_domains)
 763 {
 764         lockdep_assert_held(&uncore->lock);
 765
 766         if (!uncore->funcs.force_wake_put)
 767                 return;
 768
 769         __intel_uncore_forcewake_put(uncore, fw_domains);
 770 }
 771
 772 void assert_forcewakes_inactive(struct intel_uncore *uncore)
 773 {
 774         if (!uncore->funcs.force_wake_get)
 775                 return;
 776
 777         WARN(uncore->fw_domains_active,
 778              "Expected all fw_domains to be inactive, but %08x are still on\n",
 779              uncore->fw_domains_active);
 780 }
 781
 782 void assert_forcewakes_active(struct intel_uncore *uncore,
 783                               enum forcewake_domains fw_domains)
 784 {
 785         if (!uncore->funcs.force_wake_get)
 786                 return;
 787
 788         assert_rpm_wakelock_held(uncore_to_i915(uncore));
 789
 790         fw_domains &= uncore->fw_domains;
 791         WARN(fw_domains & ~uncore->fw_domains_active,
 792              "Expected %08x fw_domains to be active, but %08x are off\n",
 793              fw_domains, fw_domains & ~uncore->fw_domains_active);
 794 }
 795
 796 /* We give fast paths for the really cool registers */
 797 #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
 798
 799 #define GEN11_NEEDS_FORCE_WAKE(reg) \
 800         ((reg) < 0x40000 || ((reg) >= 0x1c0000 && (reg) < 0x1dc000))
 801
 802 #define __gen6_reg_read_fw_domains(offset) \
 803 ({ \
 804         enum forcewake_domains __fwd; \
 805         if (NEEDS_FORCE_WAKE(offset)) \
 806                 __fwd = FORCEWAKE_RENDER; \
 807         else \
 808                 __fwd = 0; \
 809         __fwd; \
 810 })
 811
 812 static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
 813 {
 814         if (offset < entry->start)
 815                 return -1;
 816         else if (offset > entry->end)
 817                 return 1;
 818         else
 819                 return 0;
 820 }
 821
 822 /* Copied and "macroized" from lib/bsearch.c */
 823 #define BSEARCH(key, base, num, cmp) ({                                 \
 824         unsigned int start__ = 0, end__ = (num);                        \
 825         typeof(base) result__ = NULL;                                   \
 826         while (start__ < end__) {                                       \
 827                 unsigned int mid__ = start__ + (end__ - start__) / 2;   \
 828                 int ret__ = (cmp)((key), (base) + mid__);               \
 829                 if (ret__ < 0) {                                        \
 830                         end__ = mid__;                                  \
 831                 } else if (ret__ > 0) {                                 \
 832                         start__ = mid__ + 1;                            \
 833                 } else {                                                \
 834                         result__ = (base) + mid__;                      \
 835                         break;                                          \
 836                 }                                                       \
 837         }                                                               \
 838         result__;                                                       \
 839 })
 840
 841 static enum forcewake_domains
 842 find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
 843 {
 844         const struct intel_forcewake_range *entry;
 845
 846         entry = BSEARCH(offset,
 847                         dev_priv->uncore.fw_domains_table,
 848                         dev_priv->uncore.fw_domains_table_entries,
 849                         fw_range_cmp);
 850
 851         if (!entry)
 852                 return 0;
 853
 854         /*
 855          * The list of FW domains depends on the SKU in gen11+ so we
 856          * can't determine it statically. We use FORCEWAKE_ALL and
 857          * translate it here to the list of available domains.
 858          */
 859         if (entry->domains == FORCEWAKE_ALL)
 860                 return dev_priv->uncore.fw_domains;
 861
 862         WARN(entry->domains & ~dev_priv->uncore.fw_domains,
 863              "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
 864              entry->domains & ~dev_priv->uncore.fw_domains, offset);
 865
 866         return entry->domains;
 867 }
 868
 869 #define GEN_FW_RANGE(s, e, d) \
 870         { .start = (s), .end = (e), .domains = (d) }
 871
 872 #define HAS_FWTABLE(dev_priv) \
 873         (INTEL_GEN(dev_priv) >= 9 || \
 874          IS_CHERRYVIEW(dev_priv) || \
 875          IS_VALLEYVIEW(dev_priv))
 876
 877 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 878 static const struct intel_forcewake_range __vlv_fw_ranges[] = {
 879         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
 880         GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
 881         GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
 882         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 883         GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
 884         GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
 885         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
 886 };
 887
 888 #define __fwtable_reg_read_fw_domains(offset) \
 889 ({ \
 890         enum forcewake_domains __fwd = 0; \
 891         if (NEEDS_FORCE_WAKE((offset))) \
 892                 __fwd = find_fw_domain(dev_priv, offset); \
 893         __fwd; \
 894 })
 895
 896 #define __gen11_fwtable_reg_read_fw_domains(offset) \
 897 ({ \
 898         enum forcewake_domains __fwd = 0; \
 899         if (GEN11_NEEDS_FORCE_WAKE((offset))) \
 900                 __fwd = find_fw_domain(dev_priv, offset); \
 901         __fwd; \
 902 })
 903
 904 /* *Must* be sorted by offset! See intel_shadow_table_check(). */
 905 static const i915_reg_t gen8_shadowed_regs[] = {
 906         RING_TAIL(RENDER_RING_BASE),    /* 0x2000 (base) */
 907         GEN6_RPNSWREQ,                  /* 0xA008 */
 908         GEN6_RC_VIDEO_FREQ,             /* 0xA00C */
 909         RING_TAIL(GEN6_BSD_RING_BASE),  /* 0x12000 (base) */
 910         RING_TAIL(VEBOX_RING_BASE),     /* 0x1a000 (base) */
 911         RING_TAIL(BLT_RING_BASE),       /* 0x22000 (base) */
 912         /* TODO: Other registers are not yet used */
 913 };
 914
 915 static const i915_reg_t gen11_shadowed_regs[] = {
 916         RING_TAIL(RENDER_RING_BASE),            /* 0x2000 (base) */
 917         GEN6_RPNSWREQ,                          /* 0xA008 */
 918         GEN6_RC_VIDEO_FREQ,                     /* 0xA00C */
 919         RING_TAIL(BLT_RING_BASE),               /* 0x22000 (base) */
 920         RING_TAIL(GEN11_BSD_RING_BASE),         /* 0x1C0000 (base) */
 921         RING_TAIL(GEN11_BSD2_RING_BASE),        /* 0x1C4000 (base) */
 922         RING_TAIL(GEN11_VEBOX_RING_BASE),       /* 0x1C8000 (base) */
 923         RING_TAIL(GEN11_BSD3_RING_BASE),        /* 0x1D0000 (base) */
 924         RING_TAIL(GEN11_BSD4_RING_BASE),        /* 0x1D4000 (base) */
 925         RING_TAIL(GEN11_VEBOX2_RING_BASE),      /* 0x1D8000 (base) */
 926         /* TODO: Other registers are not yet used */
 927 };
 928
 929 static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
 930 {
 931         u32 offset = i915_mmio_reg_offset(*reg);
 932
 933         if (key < offset)
 934                 return -1;
 935         else if (key > offset)
 936                 return 1;
 937         else
 938                 return 0;
 939 }
 940
 941 #define __is_genX_shadowed(x) \
 942 static bool is_gen##x##_shadowed(u32 offset) \
 943 { \
 944         const i915_reg_t *regs = gen##x##_shadowed_regs; \
 945         return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \
 946                        mmio_reg_cmp); \
 947 }
 948
 949 __is_genX_shadowed(8)
 950 __is_genX_shadowed(11)
 951
 952 #define __gen8_reg_write_fw_domains(offset) \
 953 ({ \
 954         enum forcewake_domains __fwd; \
 955         if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
 956                 __fwd = FORCEWAKE_RENDER; \
 957         else \
 958                 __fwd = 0; \
 959         __fwd; \
 960 })
 961
 962 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 963 static const struct intel_forcewake_range __chv_fw_ranges[] = {
 964         GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
 965         GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 966         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
 967         GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 968         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
 969         GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 970         GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
 971         GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 972         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
 973         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
 974         GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
 975         GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
 976         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
 977         GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
 978         GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
 979         GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
 980 };
 981
 982 #define __fwtable_reg_write_fw_domains(offset) \
 983 ({ \
 984         enum forcewake_domains __fwd = 0; \
 985         if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
 986                 __fwd = find_fw_domain(dev_priv, offset); \
 987         __fwd; \
 988 })
 989
 990 #define __gen11_fwtable_reg_write_fw_domains(offset) \
 991 ({ \
 992         enum forcewake_domains __fwd = 0; \
 993         if (GEN11_NEEDS_FORCE_WAKE((offset)) && !is_gen11_shadowed(offset)) \
 994                 __fwd = find_fw_domain(dev_priv, offset); \
 995         __fwd; \
 996 })
 997
 998 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
 999 static const struct intel_forcewake_range __gen9_fw_ranges[] = {
1000         GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
1001         GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1002         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1003         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
1004         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1005         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
1006         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1007         GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
1008         GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
1009         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1010         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
1011         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1012         GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
1013         GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
1014         GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
1015         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1016         GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
1017         GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
1018         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
1019         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1020         GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
1021         GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
1022         GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
1023         GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1024         GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
1025         GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
1026         GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
1027         GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
1028         GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
1029         GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1030         GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
1031         GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
1032 };
1033
1034 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
1035 static const struct intel_forcewake_range __gen11_fw_ranges[] = {
1036         GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
1037         GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1038         GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
1039         GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
1040         GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
1041         GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
1042         GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
1043         GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER),
1044         GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
1045         GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
1046         GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
1047         GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER),
1048         GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
1049         GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
1050         GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL),
1051         GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
1052         GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
1053         GEN_FW_RANGE(0xb480, 0xdfff, FORCEWAKE_BLITTER),
1054         GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
1055         GEN_FW_RANGE(0xe900, 0x243ff, FORCEWAKE_BLITTER),
1056         GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
1057         GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER),
1058         GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1059         GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0),
1060         GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1),
1061         GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0),
1062         GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER),
1063         GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2),
1064         GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3),
1065         GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1)
1066 };
1067
1068 static void
1069 ilk_dummy_write(struct drm_i915_private *dev_priv)
1070 {
1071         /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
1072          * the chip from rc6 before touching it for real. MI_MODE is masked,
1073          * hence harmless to write 0 into. */
1074         __raw_i915_write32(dev_priv, MI_MODE, 0);
1075 }
1076
1077 static void
1078 __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
1079                       const i915_reg_t reg,
1080                       const bool read,
1081                       const bool before)
1082 {
1083         if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
1084                  "Unclaimed %s register 0x%x\n",
1085                  read ? "read from" : "write to",
1086                  i915_mmio_reg_offset(reg)))
1087                 /* Only report the first N failures */
1088                 i915_modparams.mmio_debug--;
1089 }
1090
1091 static inline void
1092 unclaimed_reg_debug(struct drm_i915_private *dev_priv,
1093                     const i915_reg_t reg,
1094                     const bool read,
1095                     const bool before)
1096 {
1097         if (likely(!i915_modparams.mmio_debug))
1098                 return;
1099
1100         __unclaimed_reg_debug(dev_priv, reg, read, before);
1101 }
1102
1103 #define GEN2_READ_HEADER(x) \
1104         u##x val = 0; \
1105         assert_rpm_wakelock_held(dev_priv);
1106
1107 #define GEN2_READ_FOOTER \
1108         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1109         return val
1110
1111 #define __gen2_read(x) \
1112 static u##x \
1113 gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1114         GEN2_READ_HEADER(x); \
1115         val = __raw_i915_read##x(dev_priv, reg); \
1116         GEN2_READ_FOOTER; \
1117 }
1118
1119 #define __gen5_read(x) \
1120 static u##x \
1121 gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1122         GEN2_READ_HEADER(x); \
1123         ilk_dummy_write(dev_priv); \
1124         val = __raw_i915_read##x(dev_priv, reg); \
1125         GEN2_READ_FOOTER; \
1126 }
1127
1128 __gen5_read(8)
1129 __gen5_read(16)
1130 __gen5_read(32)
1131 __gen5_read(64)
1132 __gen2_read(8)
1133 __gen2_read(16)
1134 __gen2_read(32)
1135 __gen2_read(64)
1136
1137 #undef __gen5_read
1138 #undef __gen2_read
1139
1140 #undef GEN2_READ_FOOTER
1141 #undef GEN2_READ_HEADER
1142
1143 #define GEN6_READ_HEADER(x) \
1144         u32 offset = i915_mmio_reg_offset(reg); \
1145         unsigned long irqflags; \
1146         u##x val = 0; \
1147         assert_rpm_wakelock_held(dev_priv); \
1148         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1149         unclaimed_reg_debug(dev_priv, reg, true, true)
1150
1151 #define GEN6_READ_FOOTER \
1152         unclaimed_reg_debug(dev_priv, reg, true, false); \
1153         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
1154         trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1155         return val
1156
1157 static noinline void ___force_wake_auto(struct intel_uncore *uncore,
1158                                         enum forcewake_domains fw_domains)
1159 {
1160         struct intel_uncore_forcewake_domain *domain;
1161         unsigned int tmp;
1162
1163         GEM_BUG_ON(fw_domains & ~uncore->fw_domains);
1164
1165         for_each_fw_domain_masked(domain, fw_domains, uncore, tmp)
1166                 fw_domain_arm_timer(domain);
1167
1168         uncore->funcs.force_wake_get(uncore, fw_domains);
1169 }
1170
1171 static inline void __force_wake_auto(struct intel_uncore *uncore,
1172                                      enum forcewake_domains fw_domains)
1173 {
1174         if (WARN_ON(!fw_domains))
1175                 return;
1176
1177         /* Turn on all requested but inactive supported forcewake domains. */
1178         fw_domains &= uncore->fw_domains;
1179         fw_domains &= ~uncore->fw_domains_active;
1180
1181         if (fw_domains)
1182                 ___force_wake_auto(uncore, fw_domains);
1183 }
1184
1185 #define __gen_read(func, x) \
1186 static u##x \
1187 func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1188         enum forcewake_domains fw_engine; \
1189         GEN6_READ_HEADER(x); \
1190         fw_engine = __##func##_reg_read_fw_domains(offset); \
1191         if (fw_engine) \
1192                 __force_wake_auto(&dev_priv->uncore, fw_engine); \
1193         val = __raw_i915_read##x(dev_priv, reg); \
1194         GEN6_READ_FOOTER; \
1195 }
1196 #define __gen6_read(x) __gen_read(gen6, x)
1197 #define __fwtable_read(x) __gen_read(fwtable, x)
1198 #define __gen11_fwtable_read(x) __gen_read(gen11_fwtable, x)
1199
1200 __gen11_fwtable_read(8)
1201 __gen11_fwtable_read(16)
1202 __gen11_fwtable_read(32)
1203 __gen11_fwtable_read(64)
1204 __fwtable_read(8)
1205 __fwtable_read(16)
1206 __fwtable_read(32)
1207 __fwtable_read(64)
1208 __gen6_read(8)
1209 __gen6_read(16)
1210 __gen6_read(32)
1211 __gen6_read(64)
1212
1213 #undef __gen11_fwtable_read
1214 #undef __fwtable_read
1215 #undef __gen6_read
1216 #undef GEN6_READ_FOOTER
1217 #undef GEN6_READ_HEADER
1218
1219 #define GEN2_WRITE_HEADER \
1220         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1221         assert_rpm_wakelock_held(dev_priv); \
1222
1223 #define GEN2_WRITE_FOOTER
1224
1225 #define __gen2_write(x) \
1226 static void \
1227 gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1228         GEN2_WRITE_HEADER; \
1229         __raw_i915_write##x(dev_priv, reg, val); \
1230         GEN2_WRITE_FOOTER; \
1231 }
1232
1233 #define __gen5_write(x) \
1234 static void \
1235 gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1236         GEN2_WRITE_HEADER; \
1237         ilk_dummy_write(dev_priv); \
1238         __raw_i915_write##x(dev_priv, reg, val); \
1239         GEN2_WRITE_FOOTER; \
1240 }
1241
1242 __gen5_write(8)
1243 __gen5_write(16)
1244 __gen5_write(32)
1245 __gen2_write(8)
1246 __gen2_write(16)
1247 __gen2_write(32)
1248
1249 #undef __gen5_write
1250 #undef __gen2_write
1251
1252 #undef GEN2_WRITE_FOOTER
1253 #undef GEN2_WRITE_HEADER
1254
1255 #define GEN6_WRITE_HEADER \
1256         u32 offset = i915_mmio_reg_offset(reg); \
1257         unsigned long irqflags; \
1258         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1259         assert_rpm_wakelock_held(dev_priv); \
1260         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1261         unclaimed_reg_debug(dev_priv, reg, false, true)
1262
1263 #define GEN6_WRITE_FOOTER \
1264         unclaimed_reg_debug(dev_priv, reg, false, false); \
1265         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
1266
1267 #define __gen6_write(x) \
1268 static void \
1269 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1270         GEN6_WRITE_HEADER; \
1271         if (NEEDS_FORCE_WAKE(offset)) \
1272                 __gen6_gt_wait_for_fifo(dev_priv); \
1273         __raw_i915_write##x(dev_priv, reg, val); \
1274         GEN6_WRITE_FOOTER; \
1275 }
1276
1277 #define __gen_write(func, x) \
1278 static void \
1279 func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1280         enum forcewake_domains fw_engine; \
1281         GEN6_WRITE_HEADER; \
1282         fw_engine = __##func##_reg_write_fw_domains(offset); \
1283         if (fw_engine) \
1284                 __force_wake_auto(&dev_priv->uncore, fw_engine); \
1285         __raw_i915_write##x(dev_priv, reg, val); \
1286         GEN6_WRITE_FOOTER; \
1287 }
1288 #define __gen8_write(x) __gen_write(gen8, x)
1289 #define __fwtable_write(x) __gen_write(fwtable, x)
1290 #define __gen11_fwtable_write(x) __gen_write(gen11_fwtable, x)
1291
1292 __gen11_fwtable_write(8)
1293 __gen11_fwtable_write(16)
1294 __gen11_fwtable_write(32)
1295 __fwtable_write(8)
1296 __fwtable_write(16)
1297 __fwtable_write(32)
1298 __gen8_write(8)
1299 __gen8_write(16)
1300 __gen8_write(32)
1301 __gen6_write(8)
1302 __gen6_write(16)
1303 __gen6_write(32)
1304
1305 #undef __gen11_fwtable_write
1306 #undef __fwtable_write
1307 #undef __gen8_write
1308 #undef __gen6_write
1309 #undef GEN6_WRITE_FOOTER
1310 #undef GEN6_WRITE_HEADER
1311
1312 #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \
1313 do { \
1314         (i915)->uncore.funcs.mmio_writeb = x##_write8; \
1315         (i915)->uncore.funcs.mmio_writew = x##_write16; \
1316         (i915)->uncore.funcs.mmio_writel = x##_write32; \
1317 } while (0)
1318
1319 #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \
1320 do { \
1321         (i915)->uncore.funcs.mmio_readb = x##_read8; \
1322         (i915)->uncore.funcs.mmio_readw = x##_read16; \
1323         (i915)->uncore.funcs.mmio_readl = x##_read32; \
1324         (i915)->uncore.funcs.mmio_readq = x##_read64; \
1325 } while (0)
1326
1327
1328 static void fw_domain_init(struct drm_i915_private *dev_priv,
1329                            enum forcewake_domain_id domain_id,
1330                            i915_reg_t reg_set,
1331                            i915_reg_t reg_ack)
1332 {
1333         struct intel_uncore *uncore = &dev_priv->uncore;
1334         struct intel_uncore_forcewake_domain *d;
1335
1336         if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
1337                 return;
1338
1339         d = &uncore->fw_domain[domain_id];
1340
1341         WARN_ON(d->wake_count);
1342
1343         WARN_ON(!i915_mmio_reg_valid(reg_set));
1344         WARN_ON(!i915_mmio_reg_valid(reg_ack));
1345
1346         d->wake_count = 0;
1347         d->reg_set = dev_priv->regs + i915_mmio_reg_offset(reg_set);
1348         d->reg_ack = dev_priv->regs + i915_mmio_reg_offset(reg_ack);
1349
1350         d->id = domain_id;
1351
1352         BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
1353         BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
1354         BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
1355         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX0));
1356         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX1));
1357         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX2));
1358         BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3 != (1 << FW_DOMAIN_ID_MEDIA_VDBOX3));
1359         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX0));
1360         BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1 != (1 << FW_DOMAIN_ID_MEDIA_VEBOX1));
1361
1362
1363         d->mask = BIT(domain_id);
1364
1365         hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1366         d->timer.function = intel_uncore_fw_release_timer;
1367
1368         uncore->fw_domains |= BIT(domain_id);
1369
1370         fw_domain_reset(d);
1371 }
1372
1373 static void fw_domain_fini(struct drm_i915_private *dev_priv,
1374                            enum forcewake_domain_id domain_id)
1375 {
1376         struct intel_uncore_forcewake_domain *d;
1377
1378         if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
1379                 return;
1380
1381         d = &dev_priv->uncore.fw_domain[domain_id];
1382
1383         WARN_ON(d->wake_count);
1384         WARN_ON(hrtimer_cancel(&d->timer));
1385         memset(d, 0, sizeof(*d));
1386
1387         dev_priv->uncore.fw_domains &= ~BIT(domain_id);
1388 }
1389
1390 static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
1391 {
1392         if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv))
1393                 return;
1394
1395         if (INTEL_GEN(dev_priv) >= 11) {
1396                 int i;
1397
1398                 dev_priv->uncore.funcs.force_wake_get =
1399                         fw_domains_get_with_fallback;
1400                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1401                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1402                                FORCEWAKE_RENDER_GEN9,
1403                                FORCEWAKE_ACK_RENDER_GEN9);
1404                 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
1405                                FORCEWAKE_BLITTER_GEN9,
1406                                FORCEWAKE_ACK_BLITTER_GEN9);
1407                 for (i = 0; i < I915_MAX_VCS; i++) {
1408                         if (!HAS_ENGINE(dev_priv, _VCS(i)))
1409                                 continue;
1410
1411                         fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VDBOX0 + i,
1412                                        FORCEWAKE_MEDIA_VDBOX_GEN11(i),
1413                                        FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i));
1414                 }
1415                 for (i = 0; i < I915_MAX_VECS; i++) {
1416                         if (!HAS_ENGINE(dev_priv, _VECS(i)))
1417                                 continue;
1418
1419                         fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA_VEBOX0 + i,
1420                                        FORCEWAKE_MEDIA_VEBOX_GEN11(i),
1421                                        FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i));
1422                 }
1423         } else if (IS_GEN_RANGE(dev_priv, 9, 10)) {
1424                 dev_priv->uncore.funcs.force_wake_get =
1425                         fw_domains_get_with_fallback;
1426                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1427                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1428                                FORCEWAKE_RENDER_GEN9,
1429                                FORCEWAKE_ACK_RENDER_GEN9);
1430                 fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
1431                                FORCEWAKE_BLITTER_GEN9,
1432                                FORCEWAKE_ACK_BLITTER_GEN9);
1433                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
1434                                FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
1435         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1436                 dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
1437                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1438                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1439                                FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
1440                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
1441                                FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
1442         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1443                 dev_priv->uncore.funcs.force_wake_get =
1444                         fw_domains_get_with_thread_status;
1445                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1446                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1447                                FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
1448         } else if (IS_IVYBRIDGE(dev_priv)) {
1449                 u32 ecobus;
1450
1451                 /* IVB configs may use multi-threaded forcewake */
1452
1453                 /* A small trick here - if the bios hasn't configured
1454                  * MT forcewake, and if the device is in RC6, then
1455                  * force_wake_mt_get will not wake the device and the
1456                  * ECOBUS read will return zero. Which will be
1457                  * (correctly) interpreted by the test below as MT
1458                  * forcewake being disabled.
1459                  */
1460                 dev_priv->uncore.funcs.force_wake_get =
1461                         fw_domains_get_with_thread_status;
1462                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1463
1464                 /* We need to init first for ECOBUS access and then
1465                  * determine later if we want to reinit, in case of MT access is
1466                  * not working. In this stage we don't know which flavour this
1467                  * ivb is, so it is better to reset also the gen6 fw registers
1468                  * before the ecobus check.
1469                  */
1470
1471                 __raw_i915_write32(dev_priv, FORCEWAKE, 0);
1472                 __raw_posting_read(dev_priv, ECOBUS);
1473
1474                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1475                                FORCEWAKE_MT, FORCEWAKE_MT_ACK);
1476
1477                 spin_lock_irq(&dev_priv->uncore.lock);
1478                 fw_domains_get_with_thread_status(&dev_priv->uncore, FORCEWAKE_RENDER);
1479                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1480                 fw_domains_put(&dev_priv->uncore, FORCEWAKE_RENDER);
1481                 spin_unlock_irq(&dev_priv->uncore.lock);
1482
1483                 if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
1484                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
1485                         DRM_INFO("when using vblank-synced partial screen updates.\n");
1486                         fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1487                                        FORCEWAKE, FORCEWAKE_ACK);
1488                 }
1489         } else if (IS_GEN(dev_priv, 6)) {
1490                 dev_priv->uncore.funcs.force_wake_get =
1491                         fw_domains_get_with_thread_status;
1492                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1493                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
1494                                FORCEWAKE, FORCEWAKE_ACK);
1495         }
1496
1497         /* All future platforms are expected to require complex power gating */
1498         WARN_ON(dev_priv->uncore.fw_domains == 0);
1499 }
1500
1501 #define ASSIGN_FW_DOMAINS_TABLE(d) \
1502 { \
1503         dev_priv->uncore.fw_domains_table = \
1504                         (struct intel_forcewake_range *)(d); \
1505         dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
1506 }
1507
1508 static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
1509                                          unsigned long action, void *data)
1510 {
1511         struct drm_i915_private *dev_priv = container_of(nb,
1512                         struct drm_i915_private, uncore.pmic_bus_access_nb);
1513
1514         switch (action) {
1515         case MBI_PMIC_BUS_ACCESS_BEGIN:
1516                 /*
1517                  * forcewake all now to make sure that we don't need to do a
1518                  * forcewake later which on systems where this notifier gets
1519                  * called requires the punit to access to the shared pmic i2c
1520                  * bus, which will be busy after this notification, leading to:
1521                  * "render: timed out waiting for forcewake ack request."
1522                  * errors.
1523                  *
1524                  * The notifier is unregistered during intel_runtime_suspend(),
1525                  * so it's ok to access the HW here without holding a RPM
1526                  * wake reference -> disable wakeref asserts for the time of
1527                  * the access.
1528                  */
1529                 disable_rpm_wakeref_asserts(dev_priv);
1530                 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
1531                 enable_rpm_wakeref_asserts(dev_priv);
1532                 break;
1533         case MBI_PMIC_BUS_ACCESS_END:
1534                 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1535                 break;
1536         }
1537
1538         return NOTIFY_OK;
1539 }
1540
1541 void intel_uncore_init(struct drm_i915_private *dev_priv)
1542 {
1543         i915_check_vgpu(dev_priv);
1544
1545         intel_uncore_edram_detect(dev_priv);
1546         intel_uncore_fw_domains_init(dev_priv);
1547         __intel_uncore_early_sanitize(dev_priv, 0);
1548
1549         dev_priv->uncore.unclaimed_mmio_check = 1;
1550         dev_priv->uncore.pmic_bus_access_nb.notifier_call =
1551                 i915_pmic_bus_access_notifier;
1552
1553         if (IS_GEN_RANGE(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) {
1554                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2);
1555                 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2);
1556         } else if (IS_GEN(dev_priv, 5)) {
1557                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5);
1558                 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5);
1559         } else if (IS_GEN_RANGE(dev_priv, 6, 7)) {
1560                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6);
1561
1562                 if (IS_VALLEYVIEW(dev_priv)) {
1563                         ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
1564                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1565                 } else {
1566                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1567                 }
1568         } else if (IS_GEN(dev_priv, 8)) {
1569                 if (IS_CHERRYVIEW(dev_priv)) {
1570                         ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
1571                         ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1572                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1573
1574                 } else {
1575                         ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8);
1576                         ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1577                 }
1578         } else if (IS_GEN_RANGE(dev_priv, 9, 10)) {
1579                 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1580                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1581                 ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1582         } else {
1583                 ASSIGN_FW_DOMAINS_TABLE(__gen11_fw_ranges);
1584                 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen11_fwtable);
1585                 ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen11_fwtable);
1586         }
1587
1588         iosf_mbi_register_pmic_bus_access_notifier(
1589                 &dev_priv->uncore.pmic_bus_access_nb);
1590 }
1591
1592 /*
1593  * We might have detected that some engines are fused off after we initialized
1594  * the forcewake domains. Prune them, to make sure they only reference existing
1595  * engines.
1596  */
1597 void intel_uncore_prune(struct drm_i915_private *dev_priv)
1598 {
1599         if (INTEL_GEN(dev_priv) >= 11) {
1600                 enum forcewake_domains fw_domains = dev_priv->uncore.fw_domains;
1601                 enum forcewake_domain_id domain_id;
1602                 int i;
1603
1604                 for (i = 0; i < I915_MAX_VCS; i++) {
1605                         domain_id = FW_DOMAIN_ID_MEDIA_VDBOX0 + i;
1606
1607                         if (HAS_ENGINE(dev_priv, _VCS(i)))
1608                                 continue;
1609
1610                         if (fw_domains & BIT(domain_id))
1611                                 fw_domain_fini(dev_priv, domain_id);
1612                 }
1613
1614                 for (i = 0; i < I915_MAX_VECS; i++) {
1615                         domain_id = FW_DOMAIN_ID_MEDIA_VEBOX0 + i;
1616
1617                         if (HAS_ENGINE(dev_priv, _VECS(i)))
1618                                 continue;
1619
1620                         if (fw_domains & BIT(domain_id))
1621                                 fw_domain_fini(dev_priv, domain_id);
1622                 }
1623         }
1624 }
1625
1626 void intel_uncore_fini(struct drm_i915_private *dev_priv)
1627 {
1628         /* Paranoia: make sure we have disabled everything before we exit. */
1629         intel_uncore_sanitize(dev_priv);
1630
1631         iosf_mbi_punit_acquire();
1632         iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
1633                 &dev_priv->uncore.pmic_bus_access_nb);
1634         intel_uncore_forcewake_reset(&dev_priv->uncore);
1635         iosf_mbi_punit_release();
1636 }
1637
1638 static const struct reg_whitelist {
1639         i915_reg_t offset_ldw;
1640         i915_reg_t offset_udw;
1641         u16 gen_mask;
1642         u8 size;
1643 } reg_read_whitelist[] = { {
1644         .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
1645         .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
1646         .gen_mask = INTEL_GEN_MASK(4, 11),
1647         .size = 8
1648 } };
1649
1650 int i915_reg_read_ioctl(struct drm_device *dev,
1651                         void *data, struct drm_file *file)
1652 {
1653         struct drm_i915_private *dev_priv = to_i915(dev);
1654         struct drm_i915_reg_read *reg = data;
1655         struct reg_whitelist const *entry;
1656         intel_wakeref_t wakeref;
1657         unsigned int flags;
1658         int remain;
1659         int ret = 0;
1660
1661         entry = reg_read_whitelist;
1662         remain = ARRAY_SIZE(reg_read_whitelist);
1663         while (remain) {
1664                 u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw);
1665
1666                 GEM_BUG_ON(!is_power_of_2(entry->size));
1667                 GEM_BUG_ON(entry->size > 8);
1668                 GEM_BUG_ON(entry_offset & (entry->size - 1));
1669
1670                 if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask &&
1671                     entry_offset == (reg->offset & -entry->size))
1672                         break;
1673                 entry++;
1674                 remain--;
1675         }
1676
1677         if (!remain)
1678                 return -EINVAL;
1679
1680         flags = reg->offset & (entry->size - 1);
1681
1682         with_intel_runtime_pm(dev_priv, wakeref) {
1683                 if (entry->size == 8 && flags == I915_REG_READ_8B_WA)
1684                         reg->val = I915_READ64_2x32(entry->offset_ldw,
1685                                                     entry->offset_udw);
1686                 else if (entry->size == 8 && flags == 0)
1687                         reg->val = I915_READ64(entry->offset_ldw);
1688                 else if (entry->size == 4 && flags == 0)
1689                         reg->val = I915_READ(entry->offset_ldw);
1690                 else if (entry->size == 2 && flags == 0)
1691                         reg->val = I915_READ16(entry->offset_ldw);
1692                 else if (entry->size == 1 && flags == 0)
1693                         reg->val = I915_READ8(entry->offset_ldw);
1694                 else
1695                         ret = -EINVAL;
1696         }
1697
1698         return ret;
1699 }
1700
1701 /**
1702  * __intel_wait_for_register_fw - wait until register matches expected state
1703  * @dev_priv: the i915 device
1704  * @reg: the register to read
1705  * @mask: mask to apply to register value
1706  * @value: expected value
1707  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
1708  * @slow_timeout_ms: slow timeout in millisecond
1709  * @out_value: optional placeholder to hold registry value
1710  *
1711  * This routine waits until the target register @reg contains the expected
1712  * @value after applying the @mask, i.e. it waits until ::
1713  *
1714  *     (I915_READ_FW(reg) & mask) == value
1715  *
1716  * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
1717  * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
1718  * must be not larger than 20,0000 microseconds.
1719  *
1720  * Note that this routine assumes the caller holds forcewake asserted, it is
1721  * not suitable for very long waits. See intel_wait_for_register() if you
1722  * wish to wait without holding forcewake for the duration (i.e. you expect
1723  * the wait to be slow).
1724  *
1725  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
1726  */
1727 int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
1728                                  i915_reg_t reg,
1729                                  u32 mask,
1730                                  u32 value,
1731                                  unsigned int fast_timeout_us,
1732                                  unsigned int slow_timeout_ms,
1733                                  u32 *out_value)
1734 {
1735         u32 uninitialized_var(reg_value);
1736 #define done (((reg_value = I915_READ_FW(reg)) & mask) == value)
1737         int ret;
1738
1739         /* Catch any overuse of this function */
1740         might_sleep_if(slow_timeout_ms);
1741         GEM_BUG_ON(fast_timeout_us > 20000);
1742
1743         ret = -ETIMEDOUT;
1744         if (fast_timeout_us && fast_timeout_us <= 20000)
1745                 ret = _wait_for_atomic(done, fast_timeout_us, 0);
1746         if (ret && slow_timeout_ms)
1747                 ret = wait_for(done, slow_timeout_ms);
1748
1749         if (out_value)
1750                 *out_value = reg_value;
1751
1752         return ret;
1753 #undef done
1754 }
1755
1756 /**
1757  * __intel_wait_for_register - wait until register matches expected state
1758  * @dev_priv: the i915 device
1759  * @reg: the register to read
1760  * @mask: mask to apply to register value
1761  * @value: expected value
1762  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
1763  * @slow_timeout_ms: slow timeout in millisecond
1764  * @out_value: optional placeholder to hold registry value
1765  *
1766  * This routine waits until the target register @reg contains the expected
1767  * @value after applying the @mask, i.e. it waits until ::
1768  *
1769  *     (I915_READ(reg) & mask) == value
1770  *
1771  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
1772  *
1773  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
1774  */
1775 int __intel_wait_for_register(struct drm_i915_private *dev_priv,
1776                             i915_reg_t reg,
1777                             u32 mask,
1778                             u32 value,
1779                             unsigned int fast_timeout_us,
1780                             unsigned int slow_timeout_ms,
1781                             u32 *out_value)
1782 {
1783         unsigned fw =
1784                 intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
1785         u32 reg_value;
1786         int ret;
1787
1788         might_sleep_if(slow_timeout_ms);
1789
1790         spin_lock_irq(&dev_priv->uncore.lock);
1791         intel_uncore_forcewake_get__locked(&dev_priv->uncore, fw);
1792
1793         ret = __intel_wait_for_register_fw(dev_priv,
1794                                            reg, mask, value,
1795                                            fast_timeout_us, 0, &reg_value);
1796
1797         intel_uncore_forcewake_put__locked(&dev_priv->uncore, fw);
1798         spin_unlock_irq(&dev_priv->uncore.lock);
1799
1800         if (ret && slow_timeout_ms)
1801                 ret = __wait_for(reg_value = I915_READ_NOTRACE(reg),
1802                                  (reg_value & mask) == value,
1803                                  slow_timeout_ms * 1000, 10, 1000);
1804
1805         /* just trace the final value */
1806         trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true);
1807
1808         if (out_value)
1809                 *out_value = reg_value;
1810
1811         return ret;
1812 }
1813
1814 bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
1815 {
1816         return check_for_unclaimed_mmio(dev_priv);
1817 }
1818
1819 bool
1820 intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
1821 {
1822         bool ret = false;
1823
1824         spin_lock_irq(&dev_priv->uncore.lock);
1825
1826         if (unlikely(dev_priv->uncore.unclaimed_mmio_check <= 0))
1827                 goto out;
1828
1829         if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
1830                 if (!i915_modparams.mmio_debug) {
1831                         DRM_DEBUG("Unclaimed register detected, "
1832                                   "enabling oneshot unclaimed register reporting. "
1833                                   "Please use i915.mmio_debug=N for more information.\n");
1834                         i915_modparams.mmio_debug++;
1835                 }
1836                 dev_priv->uncore.unclaimed_mmio_check--;
1837                 ret = true;
1838         }
1839
1840 out:
1841         spin_unlock_irq(&dev_priv->uncore.lock);
1842
1843         return ret;
1844 }
1845
1846 static enum forcewake_domains
1847 intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
1848                                 i915_reg_t reg)
1849 {
1850         u32 offset = i915_mmio_reg_offset(reg);
1851         enum forcewake_domains fw_domains;
1852
1853         if (INTEL_GEN(dev_priv) >= 11) {
1854                 fw_domains = __gen11_fwtable_reg_read_fw_domains(offset);
1855         } else if (HAS_FWTABLE(dev_priv)) {
1856                 fw_domains = __fwtable_reg_read_fw_domains(offset);
1857         } else if (INTEL_GEN(dev_priv) >= 6) {
1858                 fw_domains = __gen6_reg_read_fw_domains(offset);
1859         } else {
1860                 WARN_ON(!IS_GEN_RANGE(dev_priv, 2, 5));
1861                 fw_domains = 0;
1862         }
1863
1864         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
1865
1866         return fw_domains;
1867 }
1868
1869 static enum forcewake_domains
1870 intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
1871                                  i915_reg_t reg)
1872 {
1873         u32 offset = i915_mmio_reg_offset(reg);
1874         enum forcewake_domains fw_domains;
1875
1876         if (INTEL_GEN(dev_priv) >= 11) {
1877                 fw_domains = __gen11_fwtable_reg_write_fw_domains(offset);
1878         } else if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
1879                 fw_domains = __fwtable_reg_write_fw_domains(offset);
1880         } else if (IS_GEN(dev_priv, 8)) {
1881                 fw_domains = __gen8_reg_write_fw_domains(offset);
1882         } else if (IS_GEN_RANGE(dev_priv, 6, 7)) {
1883                 fw_domains = FORCEWAKE_RENDER;
1884         } else {
1885                 WARN_ON(!IS_GEN_RANGE(dev_priv, 2, 5));
1886                 fw_domains = 0;
1887         }
1888
1889         WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
1890
1891         return fw_domains;
1892 }
1893
1894 /**
1895  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
1896  *                                  a register
1897  * @dev_priv: pointer to struct drm_i915_private
1898  * @reg: register in question
1899  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
1900  *
1901  * Returns a set of forcewake domains required to be taken with for example
1902  * intel_uncore_forcewake_get for the specified register to be accessible in the
1903  * specified mode (read, write or read/write) with raw mmio accessors.
1904  *
1905  * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
1906  * callers to do FIFO management on their own or risk losing writes.
1907  */
1908 enum forcewake_domains
1909 intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
1910                                i915_reg_t reg, unsigned int op)
1911 {
1912         enum forcewake_domains fw_domains = 0;
1913
1914         WARN_ON(!op);
1915
1916         if (intel_vgpu_active(dev_priv))
1917                 return 0;
1918
1919         if (op & FW_REG_READ)
1920                 fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
1921
1922         if (op & FW_REG_WRITE)
1923                 fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);
1924
1925         return fw_domains;
1926 }
1927
1928 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1929 #include "selftests/mock_uncore.c"
1930 #include "selftests/intel_uncore.c"
1931 #endif