drivers/gpu/drm/msm/adreno/a6xx_gmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
   3
   4 #include <linux/bitfield.h>
   5 #include <linux/clk.h>
   6 #include <linux/interconnect.h>
   7 #include <linux/of_platform.h>
   8 #include <linux/platform_device.h>
   9 #include <linux/pm_domain.h>
  10 #include <linux/pm_opp.h>
  11 #include <soc/qcom/cmd-db.h>
  12 #include <drm/drm_gem.h>
  13
  14 #include "a6xx_gpu.h"
  15 #include "a6xx_gmu.xml.h"
  16 #include "msm_gem.h"
  17 #include "msm_gpu_trace.h"
  18 #include "msm_mmu.h"
  19
  20 static void a6xx_gmu_fault(struct a6xx_gmu *gmu)
  21 {
  22         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
  23         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
  24         struct msm_gpu *gpu = &adreno_gpu->base;
  25
  26         /* FIXME: add a banner here */
  27         gmu->hung = true;
  28
  29         /* Turn off the hangcheck timer while we are resetting */
  30         del_timer(&gpu->hangcheck_timer);
  31
  32         /* Queue the GPU handler because we need to treat this as a recovery */
  33         kthread_queue_work(gpu->worker, &gpu->recover_work);
  34 }
  35
  36 static irqreturn_t a6xx_gmu_irq(int irq, void *data)
  37 {
  38         struct a6xx_gmu *gmu = data;
  39         u32 status;
  40
  41         status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS);
  42         gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status);
  43
  44         if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) {
  45                 dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n");
  46
  47                 a6xx_gmu_fault(gmu);
  48         }
  49
  50         if (status &  A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR)
  51                 dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n");
  52
  53         if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
  54                 dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n",
  55                         gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS));
  56
  57         return IRQ_HANDLED;
  58 }
  59
  60 static irqreturn_t a6xx_hfi_irq(int irq, void *data)
  61 {
  62         struct a6xx_gmu *gmu = data;
  63         u32 status;
  64
  65         status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO);
  66         gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status);
  67
  68         if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) {
  69                 dev_err_ratelimited(gmu->dev, "GMU firmware fault\n");
  70
  71                 a6xx_gmu_fault(gmu);
  72         }
  73
  74         return IRQ_HANDLED;
  75 }
  76
  77 bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu)
  78 {
  79         u32 val;
  80
  81         /* This can be called from gpu state code so make sure GMU is valid */
  82         if (!gmu->initialized)
  83                 return false;
  84
  85         val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
  86
  87         return !(val &
  88                 (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF |
  89                 A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF));
  90 }
  91
  92 /* Check to see if the GX rail is still powered */
  93 bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu)
  94 {
  95         u32 val;
  96
  97         /* This can be called from gpu state code so make sure GMU is valid */
  98         if (!gmu->initialized)
  99                 return false;
 100
 101         val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
 102
 103         return !(val &
 104                 (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF |
 105                 A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
 106 }
 107
 108 void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp,
 109                        bool suspended)
 110 {
 111         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 112         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 113         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
 114         u32 perf_index;
 115         unsigned long gpu_freq;
 116         int ret = 0;
 117
 118         gpu_freq = dev_pm_opp_get_freq(opp);
 119
 120         if (gpu_freq == gmu->freq)
 121                 return;
 122
 123         for (perf_index = 0; perf_index < gmu->nr_gpu_freqs - 1; perf_index++)
 124                 if (gpu_freq == gmu->gpu_freqs[perf_index])
 125                         break;
 126
 127         gmu->current_perf_index = perf_index;
 128         gmu->freq = gmu->gpu_freqs[perf_index];
 129
 130         trace_msm_gmu_freq_change(gmu->freq, perf_index);
 131
 132         /*
 133          * This can get called from devfreq while the hardware is idle. Don't
 134          * bring up the power if it isn't already active. All we're doing here
 135          * is updating the frequency so that when we come back online we're at
 136          * the right rate.
 137          */
 138         if (suspended)
 139                 return;
 140
 141         if (!gmu->legacy) {
 142                 a6xx_hfi_set_freq(gmu, perf_index);
 143                 dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
 144                 return;
 145         }
 146
 147         gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0);
 148
 149         gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING,
 150                         ((3 & 0xf) << 28) | perf_index);
 151
 152         /*
 153          * Send an invalid index as a vote for the bus bandwidth and let the
 154          * firmware decide on the right vote
 155          */
 156         gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff);
 157
 158         /* Set and clear the OOB for DCVS to trigger the GMU */
 159         a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET);
 160         a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET);
 161
 162         ret = gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN);
 163         if (ret)
 164                 dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
 165
 166         dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
 167 }
 168
 169 unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
 170 {
 171         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 172         struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
 173         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
 174
 175         return  gmu->freq;
 176 }
 177
 178 static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu)
 179 {
 180         u32 val;
 181         int local = gmu->idle_level;
 182
 183         /* SPTP and IFPC both report as IFPC */
 184         if (gmu->idle_level == GMU_IDLE_STATE_SPTP)
 185                 local = GMU_IDLE_STATE_IFPC;
 186
 187         val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
 188
 189         if (val == local) {
 190                 if (gmu->idle_level != GMU_IDLE_STATE_IFPC ||
 191                         !a6xx_gmu_gx_is_on(gmu))
 192                         return true;
 193         }
 194
 195         return false;
 196 }
 197
 198 /* Wait for the GMU to get to its most idle state */
 199 int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu)
 200 {
 201         return spin_until(a6xx_gmu_check_idle_level(gmu));
 202 }
 203
 204 static int a6xx_gmu_start(struct a6xx_gmu *gmu)
 205 {
 206         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 207         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 208         u32 mask, reset_val, val;
 209         int ret;
 210
 211         val = gmu_read(gmu, REG_A6XX_GMU_CM3_DTCM_START + 0xff8);
 212         if (val <= 0x20010004) {
 213                 mask = 0xffffffff;
 214                 reset_val = 0xbabeface;
 215         } else {
 216                 mask = 0x1ff;
 217                 reset_val = 0x100;
 218         }
 219
 220         gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
 221
 222         /* Set the log wptr index
 223          * note: downstream saves the value in poweroff and restores it here
 224          */
 225         if (adreno_is_a7xx(adreno_gpu))
 226                 gmu_write(gmu, REG_A6XX_GMU_GENERAL_9, 0);
 227         else
 228                 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0);
 229
 230
 231         gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0);
 232
 233         ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val,
 234                 (val & mask) == reset_val, 100, 10000);
 235
 236         if (ret)
 237                 DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n");
 238
 239         return ret;
 240 }
 241
 242 static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu)
 243 {
 244         u32 val;
 245         int ret;
 246
 247         gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1);
 248
 249         ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val,
 250                 val & 1, 100, 10000);
 251         if (ret)
 252                 DRM_DEV_ERROR(gmu->dev, "Unable to start the HFI queues\n");
 253
 254         return ret;
 255 }
 256
 257 struct a6xx_gmu_oob_bits {
 258         int set, ack, set_new, ack_new, clear, clear_new;
 259         const char *name;
 260 };
 261
 262 /* These are the interrupt / ack bits for each OOB request that are set
 263  * in a6xx_gmu_set_oob and a6xx_clear_oob
 264  */
 265 static const struct a6xx_gmu_oob_bits a6xx_gmu_oob_bits[] = {
 266         [GMU_OOB_GPU_SET] = {
 267                 .name = "GPU_SET",
 268                 .set = 16,
 269                 .ack = 24,
 270                 .set_new = 30,
 271                 .ack_new = 31,
 272                 .clear = 24,
 273                 .clear_new = 31,
 274         },
 275
 276         [GMU_OOB_PERFCOUNTER_SET] = {
 277                 .name = "PERFCOUNTER",
 278                 .set = 17,
 279                 .ack = 25,
 280                 .set_new = 28,
 281                 .ack_new = 30,
 282                 .clear = 25,
 283                 .clear_new = 29,
 284         },
 285
 286         [GMU_OOB_BOOT_SLUMBER] = {
 287                 .name = "BOOT_SLUMBER",
 288                 .set = 22,
 289                 .ack = 30,
 290                 .clear = 30,
 291         },
 292
 293         [GMU_OOB_DCVS_SET] = {
 294                 .name = "GPU_DCVS",
 295                 .set = 23,
 296                 .ack = 31,
 297                 .clear = 31,
 298         },
 299 };
 300
 301 /* Trigger a OOB (out of band) request to the GMU */
 302 int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
 303 {
 304         int ret;
 305         u32 val;
 306         int request, ack;
 307
 308         WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
 309
 310         if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
 311                 return -EINVAL;
 312
 313         if (gmu->legacy) {
 314                 request = a6xx_gmu_oob_bits[state].set;
 315                 ack = a6xx_gmu_oob_bits[state].ack;
 316         } else {
 317                 request = a6xx_gmu_oob_bits[state].set_new;
 318                 ack = a6xx_gmu_oob_bits[state].ack_new;
 319                 if (!request || !ack) {
 320                         DRM_DEV_ERROR(gmu->dev,
 321                                       "Invalid non-legacy GMU request %s\n",
 322                                       a6xx_gmu_oob_bits[state].name);
 323                         return -EINVAL;
 324                 }
 325         }
 326
 327         /* Trigger the equested OOB operation */
 328         gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request);
 329
 330         /* Wait for the acknowledge interrupt */
 331         ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
 332                 val & (1 << ack), 100, 10000);
 333
 334         if (ret)
 335                 DRM_DEV_ERROR(gmu->dev,
 336                         "Timeout waiting for GMU OOB set %s: 0x%x\n",
 337                                 a6xx_gmu_oob_bits[state].name,
 338                                 gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO));
 339
 340         /* Clear the acknowledge interrupt */
 341         gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack);
 342
 343         return ret;
 344 }
 345
 346 /* Clear a pending OOB state in the GMU */
 347 void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
 348 {
 349         int bit;
 350
 351         WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
 352
 353         if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
 354                 return;
 355
 356         if (gmu->legacy)
 357                 bit = a6xx_gmu_oob_bits[state].clear;
 358         else
 359                 bit = a6xx_gmu_oob_bits[state].clear_new;
 360
 361         gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << bit);
 362 }
 363
 364 /* Enable CPU control of SPTP power power collapse */
 365 int a6xx_sptprac_enable(struct a6xx_gmu *gmu)
 366 {
 367         int ret;
 368         u32 val;
 369
 370         if (!gmu->legacy)
 371                 return 0;
 372
 373         gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000);
 374
 375         ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
 376                 (val & 0x38) == 0x28, 1, 100);
 377
 378         if (ret) {
 379                 DRM_DEV_ERROR(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n",
 380                         gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
 381         }
 382
 383         return 0;
 384 }
 385
 386 /* Disable CPU control of SPTP power power collapse */
 387 void a6xx_sptprac_disable(struct a6xx_gmu *gmu)
 388 {
 389         u32 val;
 390         int ret;
 391
 392         if (!gmu->legacy)
 393                 return;
 394
 395         /* Make sure retention is on */
 396         gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11));
 397
 398         gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001);
 399
 400         ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
 401                 (val & 0x04), 100, 10000);
 402
 403         if (ret)
 404                 DRM_DEV_ERROR(gmu->dev, "failed to power off SPTPRAC: 0x%x\n",
 405                         gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
 406 }
 407
 408 /* Let the GMU know we are starting a boot sequence */
 409 static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu)
 410 {
 411         u32 vote;
 412
 413         /* Let the GMU know we are getting ready for boot */
 414         gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0);
 415
 416         /* Choose the "default" power level as the highest available */
 417         vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1];
 418
 419         gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff);
 420         gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff);
 421
 422         /* Let the GMU know the boot sequence has started */
 423         return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
 424 }
 425
 426 /* Let the GMU know that we are about to go into slumber */
 427 static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu)
 428 {
 429         int ret;
 430
 431         /* Disable the power counter so the GMU isn't busy */
 432         gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0);
 433
 434         /* Disable SPTP_PC if the CPU is responsible for it */
 435         if (gmu->idle_level < GMU_IDLE_STATE_SPTP)
 436                 a6xx_sptprac_disable(gmu);
 437
 438         if (!gmu->legacy) {
 439                 ret = a6xx_hfi_send_prep_slumber(gmu);
 440                 goto out;
 441         }
 442
 443         /* Tell the GMU to get ready to slumber */
 444         gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1);
 445
 446         ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER);
 447         a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
 448
 449         if (!ret) {
 450                 /* Check to see if the GMU really did slumber */
 451                 if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE)
 452                         != 0x0f) {
 453                         DRM_DEV_ERROR(gmu->dev, "The GMU did not go into slumber\n");
 454                         ret = -ETIMEDOUT;
 455                 }
 456         }
 457
 458 out:
 459         /* Put fence into allow mode */
 460         gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
 461         return ret;
 462 }
 463
 464 static int a6xx_rpmh_start(struct a6xx_gmu *gmu)
 465 {
 466         int ret;
 467         u32 val;
 468
 469         gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1 << 1);
 470         /* Wait for the register to finish posting */
 471         wmb();
 472
 473         ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val,
 474                 val & (1 << 1), 100, 10000);
 475         if (ret) {
 476                 DRM_DEV_ERROR(gmu->dev, "Unable to power on the GPU RSC\n");
 477                 return ret;
 478         }
 479
 480         ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
 481                 !val, 100, 10000);
 482
 483         if (ret) {
 484                 DRM_DEV_ERROR(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n");
 485                 return ret;
 486         }
 487
 488         gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
 489
 490         return 0;
 491 }
 492
 493 static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
 494 {
 495         int ret;
 496         u32 val;
 497
 498         gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1);
 499
 500         ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
 501                 val, val & (1 << 16), 100, 10000);
 502         if (ret)
 503                 DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n");
 504
 505         gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
 506 }
 507
 508 static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value)
 509 {
 510         msm_writel(value, ptr + (offset << 2));
 511 }
 512
 513 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev,
 514                 const char *name);
 515
 516 static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
 517 {
 518         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 519         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 520         struct platform_device *pdev = to_platform_device(gmu->dev);
 521         void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
 522         u32 seqmem0_drv0_reg = REG_A6XX_RSCC_SEQ_MEM_0_DRV0;
 523         void __iomem *seqptr = NULL;
 524         uint32_t pdc_address_offset;
 525         bool pdc_in_aop = false;
 526
 527         if (IS_ERR(pdcptr))
 528                 goto err;
 529
 530         if (adreno_is_a650(adreno_gpu) ||
 531             adreno_is_a660_family(adreno_gpu) ||
 532             adreno_is_a7xx(adreno_gpu))
 533                 pdc_in_aop = true;
 534         else if (adreno_is_a618(adreno_gpu) || adreno_is_a640_family(adreno_gpu))
 535                 pdc_address_offset = 0x30090;
 536         else if (adreno_is_a619(adreno_gpu))
 537                 pdc_address_offset = 0x300a0;
 538         else
 539                 pdc_address_offset = 0x30080;
 540
 541         if (!pdc_in_aop) {
 542                 seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq");
 543                 if (IS_ERR(seqptr))
 544                         goto err;
 545         }
 546
 547         /* Disable SDE clock gating */
 548         gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
 549
 550         /* Setup RSC PDC handshake for sleep and wakeup */
 551         gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
 552         gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
 553         gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
 554         gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
 555         gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
 556         gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4,
 557                        adreno_is_a740_family(adreno_gpu) ? 0x80000021 : 0x80000000);
 558         gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
 559         gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
 560         gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
 561         gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
 562         gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
 563
 564         /* The second spin of A7xx GPUs messed with some register offsets.. */
 565         if (adreno_is_a740_family(adreno_gpu))
 566                 seqmem0_drv0_reg = REG_A7XX_RSCC_SEQ_MEM_0_DRV0_A740;
 567
 568         /* Load RSC sequencer uCode for sleep and wakeup */
 569         if (adreno_is_a650_family(adreno_gpu) ||
 570             adreno_is_a7xx(adreno_gpu)) {
 571                 gmu_write_rscc(gmu, seqmem0_drv0_reg, 0xeaaae5a0);
 572                 gmu_write_rscc(gmu, seqmem0_drv0_reg + 1, 0xe1a1ebab);
 573                 gmu_write_rscc(gmu, seqmem0_drv0_reg + 2, 0xa2e0a581);
 574                 gmu_write_rscc(gmu, seqmem0_drv0_reg + 3, 0xecac82e2);
 575                 gmu_write_rscc(gmu, seqmem0_drv0_reg + 4, 0x0020edad);
 576         } else {
 577                 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
 578                 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
 579                 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
 580                 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
 581                 gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
 582         }
 583
 584         if (pdc_in_aop)
 585                 goto setup_pdc;
 586
 587         /* Load PDC sequencer uCode for power up and power down sequence */
 588         pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
 589         pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2);
 590         pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0);
 591         pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284);
 592         pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc);
 593
 594         /* Set TCS commands used by PDC sequence for low power modes */
 595         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
 596         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
 597         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0);
 598         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
 599         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010);
 600         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1);
 601         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108);
 602         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000);
 603         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
 604
 605         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
 606         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, pdc_address_offset);
 607         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
 608
 609         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
 610         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
 611         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0);
 612         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
 613         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010);
 614         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2);
 615
 616         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
 617         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
 618         if (adreno_is_a618(adreno_gpu) || adreno_is_a619(adreno_gpu) ||
 619                         adreno_is_a650_family(adreno_gpu))
 620                 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x2);
 621         else
 622                 pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
 623         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
 624         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, pdc_address_offset);
 625         pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
 626
 627         /* Setup GPU PDC */
 628 setup_pdc:
 629         pdc_write(pdcptr, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0);
 630         pdc_write(pdcptr, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001);
 631
 632         /* ensure no writes happen before the uCode is fully written */
 633         wmb();
 634
 635         a6xx_rpmh_stop(gmu);
 636
 637 err:
 638         if (!IS_ERR_OR_NULL(pdcptr))
 639                 iounmap(pdcptr);
 640         if (!IS_ERR_OR_NULL(seqptr))
 641                 iounmap(seqptr);
 642 }
 643
 644 /*
 645  * The lowest 16 bits of this value are the number of XO clock cycles for main
 646  * hysteresis which is set at 0x1680 cycles (300 us).  The higher 16 bits are
 647  * for the shorter hysteresis that happens after main - this is 0xa (.5 us)
 648  */
 649
 650 #define GMU_PWR_COL_HYST 0x000a1680
 651
 652 /* Set up the idle state for the GMU */
 653 static void a6xx_gmu_power_config(struct a6xx_gmu *gmu)
 654 {
 655         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 656         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 657
 658         /* Disable GMU WB/RB buffer */
 659         gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1);
 660         gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1);
 661         gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1);
 662
 663         /* A7xx knows better by default! */
 664         if (adreno_is_a7xx(adreno_gpu))
 665                 return;
 666
 667         gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400);
 668
 669         switch (gmu->idle_level) {
 670         case GMU_IDLE_STATE_IFPC:
 671                 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST,
 672                         GMU_PWR_COL_HYST);
 673                 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
 674                         A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
 675                         A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE);
 676                 fallthrough;
 677         case GMU_IDLE_STATE_SPTP:
 678                 gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST,
 679                         GMU_PWR_COL_HYST);
 680                 gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
 681                         A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE |
 682                         A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE);
 683         }
 684
 685         /* Enable RPMh GPU client */
 686         gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0,
 687                 A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE |
 688                 A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE |
 689                 A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE |
 690                 A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE |
 691                 A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE |
 692                 A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE);
 693 }
 694
 695 struct block_header {
 696         u32 addr;
 697         u32 size;
 698         u32 type;
 699         u32 value;
 700         u32 data[];
 701 };
 702
 703 static bool fw_block_mem(struct a6xx_gmu_bo *bo, const struct block_header *blk)
 704 {
 705         if (!in_range(blk->addr, bo->iova, bo->size))
 706                 return false;
 707
 708         memcpy(bo->virt + blk->addr - bo->iova, blk->data, blk->size);
 709         return true;
 710 }
 711
 712 static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu)
 713 {
 714         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 715         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 716         const struct firmware *fw_image = adreno_gpu->fw[ADRENO_FW_GMU];
 717         const struct block_header *blk;
 718         u32 reg_offset;
 719
 720         u32 itcm_base = 0x00000000;
 721         u32 dtcm_base = 0x00040000;
 722
 723         if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu))
 724                 dtcm_base = 0x10004000;
 725
 726         if (gmu->legacy) {
 727                 /* Sanity check the size of the firmware that was loaded */
 728                 if (fw_image->size > 0x8000) {
 729                         DRM_DEV_ERROR(gmu->dev,
 730                                 "GMU firmware is bigger than the available region\n");
 731                         return -EINVAL;
 732                 }
 733
 734                 gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START,
 735                                (u32*) fw_image->data, fw_image->size);
 736                 return 0;
 737         }
 738
 739
 740         for (blk = (const struct block_header *) fw_image->data;
 741              (const u8*) blk < fw_image->data + fw_image->size;
 742              blk = (const struct block_header *) &blk->data[blk->size >> 2]) {
 743                 if (blk->size == 0)
 744                         continue;
 745
 746                 if (in_range(blk->addr, itcm_base, SZ_16K)) {
 747                         reg_offset = (blk->addr - itcm_base) >> 2;
 748                         gmu_write_bulk(gmu,
 749                                 REG_A6XX_GMU_CM3_ITCM_START + reg_offset,
 750                                 blk->data, blk->size);
 751                 } else if (in_range(blk->addr, dtcm_base, SZ_16K)) {
 752                         reg_offset = (blk->addr - dtcm_base) >> 2;
 753                         gmu_write_bulk(gmu,
 754                                 REG_A6XX_GMU_CM3_DTCM_START + reg_offset,
 755                                 blk->data, blk->size);
 756                 } else if (!fw_block_mem(&gmu->icache, blk) &&
 757                            !fw_block_mem(&gmu->dcache, blk) &&
 758                            !fw_block_mem(&gmu->dummy, blk)) {
 759                         DRM_DEV_ERROR(gmu->dev,
 760                                 "failed to match fw block (addr=%.8x size=%d data[0]=%.8x)\n",
 761                                 blk->addr, blk->size, blk->data[0]);
 762                 }
 763         }
 764
 765         return 0;
 766 }
 767
 768 static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
 769 {
 770         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 771         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 772         u32 fence_range_lower, fence_range_upper;
 773         u32 chipid, chipid_min = 0;
 774         int ret;
 775
 776         /* Vote veto for FAL10 */
 777         if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) {
 778                 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FALNEXT_INTF, 1);
 779                 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1);
 780         }
 781
 782         /* Turn on TCM (Tightly Coupled Memory) retention */
 783         if (adreno_is_a7xx(adreno_gpu))
 784                 a6xx_llc_write(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL, 1);
 785         else
 786                 gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
 787
 788         if (state == GMU_WARM_BOOT) {
 789                 ret = a6xx_rpmh_start(gmu);
 790                 if (ret)
 791                         return ret;
 792         } else {
 793                 if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU],
 794                         "GMU firmware is not loaded\n"))
 795                         return -ENOENT;
 796
 797                 ret = a6xx_rpmh_start(gmu);
 798                 if (ret)
 799                         return ret;
 800
 801                 ret = a6xx_gmu_fw_load(gmu);
 802                 if (ret)
 803                         return ret;
 804         }
 805
 806         /* Clear init result to make sure we are getting a fresh value */
 807         gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0);
 808         gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02);
 809
 810         /* Write the iova of the HFI table */
 811         gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova);
 812         gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1);
 813
 814         if (adreno_is_a7xx(adreno_gpu)) {
 815                 fence_range_upper = 0x32;
 816                 fence_range_lower = 0x8a0;
 817         } else {
 818                 fence_range_upper = 0xa;
 819                 fence_range_lower = 0xa0;
 820         }
 821
 822         gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0,
 823                   BIT(31) |
 824                   FIELD_PREP(GENMASK(30, 18), fence_range_upper) |
 825                   FIELD_PREP(GENMASK(17, 0), fence_range_lower));
 826
 827         /*
 828          * Snapshots toggle the NMI bit which will result in a jump to the NMI
 829          * handler instead of __main. Set the M3 config value to avoid that.
 830          */
 831         gmu_write(gmu, REG_A6XX_GMU_CM3_CFG, 0x4052);
 832
 833         /* NOTE: A730 may also fall in this if-condition with a future GMU fw update. */
 834         if (adreno_is_a7xx(adreno_gpu) && !adreno_is_a730(adreno_gpu)) {
 835                 /* A7xx GPUs have obfuscated chip IDs. Use constant maj = 7 */
 836                 chipid = FIELD_PREP(GENMASK(31, 24), 0x7);
 837
 838                 /*
 839                  * The min part has a 1-1 mapping for each GPU SKU.
 840                  * This chipid that the GMU expects corresponds to the "GENX_Y_Z" naming,
 841                  * where X = major, Y = minor, Z = patchlevel, e.g. GEN7_2_1 for prod A740.
 842                  */
 843                 if (adreno_is_a740(adreno_gpu))
 844                         chipid_min = 2;
 845                 else
 846                         return -EINVAL;
 847
 848                 chipid |= FIELD_PREP(GENMASK(23, 16), chipid_min);
 849
 850                 /* Get the patchid (which may vary) from the device tree */
 851                 chipid |= FIELD_PREP(GENMASK(15, 8), adreno_patchid(adreno_gpu));
 852         } else {
 853                 /*
 854                  * Note that the GMU has a slightly different layout for
 855                  * chip_id, for whatever reason, so a bit of massaging
 856                  * is needed.  The upper 16b are the same, but minor and
 857                  * patchid are packed in four bits each with the lower
 858                  * 8b unused:
 859                  */
 860                 chipid  = adreno_gpu->chip_id & 0xffff0000;
 861                 chipid |= (adreno_gpu->chip_id << 4) & 0xf000; /* minor */
 862                 chipid |= (adreno_gpu->chip_id << 8) & 0x0f00; /* patchid */
 863         }
 864
 865         if (adreno_is_a7xx(adreno_gpu)) {
 866                 gmu_write(gmu, REG_A6XX_GMU_GENERAL_10, chipid);
 867                 gmu_write(gmu, REG_A6XX_GMU_GENERAL_8,
 868                           (gmu->log.iova & GENMASK(31, 12)) |
 869                           ((gmu->log.size / SZ_4K - 1) & GENMASK(7, 0)));
 870         } else {
 871                 gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid);
 872
 873                 gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG,
 874                           gmu->log.iova | (gmu->log.size / SZ_4K - 1));
 875         }
 876
 877         /* Set up the lowest idle level on the GMU */
 878         a6xx_gmu_power_config(gmu);
 879
 880         ret = a6xx_gmu_start(gmu);
 881         if (ret)
 882                 return ret;
 883
 884         if (gmu->legacy) {
 885                 ret = a6xx_gmu_gfx_rail_on(gmu);
 886                 if (ret)
 887                         return ret;
 888         }
 889
 890         /* Enable SPTP_PC if the CPU is responsible for it */
 891         if (gmu->idle_level < GMU_IDLE_STATE_SPTP) {
 892                 ret = a6xx_sptprac_enable(gmu);
 893                 if (ret)
 894                         return ret;
 895         }
 896
 897         ret = a6xx_gmu_hfi_start(gmu);
 898         if (ret)
 899                 return ret;
 900
 901         /* FIXME: Do we need this wmb() here? */
 902         wmb();
 903
 904         return 0;
 905 }
 906
 907 #define A6XX_HFI_IRQ_MASK \
 908         (A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT)
 909
 910 #define A6XX_GMU_IRQ_MASK \
 911         (A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE | \
 912          A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \
 913          A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
 914
 915 static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
 916 {
 917         disable_irq(gmu->gmu_irq);
 918         disable_irq(gmu->hfi_irq);
 919
 920         gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0);
 921         gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0);
 922 }
 923
 924 static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu)
 925 {
 926         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 927         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 928         u32 val, seqmem_off = 0;
 929
 930         /* The second spin of A7xx GPUs messed with some register offsets.. */
 931         if (adreno_is_a740_family(adreno_gpu))
 932                 seqmem_off = 4;
 933
 934         /* Make sure there are no outstanding RPMh votes */
 935         gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS + seqmem_off,
 936                 val, (val & 1), 100, 10000);
 937         gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS + seqmem_off,
 938                 val, (val & 1), 100, 10000);
 939         gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS + seqmem_off,
 940                 val, (val & 1), 100, 10000);
 941         gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS + seqmem_off,
 942                 val, (val & 1), 100, 1000);
 943 }
 944
 945 /* Force the GMU off in case it isn't responsive */
 946 static void a6xx_gmu_force_off(struct a6xx_gmu *gmu)
 947 {
 948         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
 949         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
 950         struct msm_gpu *gpu = &adreno_gpu->base;
 951
 952         /*
 953          * Turn off keep alive that might have been enabled by the hang
 954          * interrupt
 955          */
 956         gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
 957
 958         /* Flush all the queues */
 959         a6xx_hfi_stop(gmu);
 960
 961         /* Stop the interrupts */
 962         a6xx_gmu_irq_disable(gmu);
 963
 964         /* Force off SPTP in case the GMU is managing it */
 965         a6xx_sptprac_disable(gmu);
 966
 967         /* Make sure there are no outstanding RPMh votes */
 968         a6xx_gmu_rpmh_off(gmu);
 969
 970         /* Clear the WRITEDROPPED fields and put fence into allow mode */
 971         gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS_CLR, 0x7);
 972         gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
 973
 974         /* Make sure the above writes go through */
 975         wmb();
 976
 977         /* Halt the gmu cm3 core */
 978         gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
 979
 980         a6xx_bus_clear_pending_transactions(adreno_gpu, true);
 981
 982         /* Reset GPU core blocks */
 983         a6xx_gpu_sw_reset(gpu, true);
 984 }
 985
 986 static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu)
 987 {
 988         struct dev_pm_opp *gpu_opp;
 989         unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
 990
 991         gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
 992         if (IS_ERR(gpu_opp))
 993                 return;
 994
 995         gmu->freq = 0; /* so a6xx_gmu_set_freq() doesn't exit early */
 996         a6xx_gmu_set_freq(gpu, gpu_opp, false);
 997         dev_pm_opp_put(gpu_opp);
 998 }
 999
1000 static void a6xx_gmu_set_initial_bw(struct msm_gpu *gpu, struct a6xx_gmu *gmu)
1001 {
1002         struct dev_pm_opp *gpu_opp;
1003         unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
1004
1005         gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
1006         if (IS_ERR(gpu_opp))
1007                 return;
1008
1009         dev_pm_opp_set_opp(&gpu->pdev->dev, gpu_opp);
1010         dev_pm_opp_put(gpu_opp);
1011 }
1012
1013 int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
1014 {
1015         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1016         struct msm_gpu *gpu = &adreno_gpu->base;
1017         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1018         int status, ret;
1019
1020         if (WARN(!gmu->initialized, "The GMU is not set up yet\n"))
1021                 return -EINVAL;
1022
1023         gmu->hung = false;
1024
1025         /* Notify AOSS about the ACD state (unimplemented for now => disable it) */
1026         if (!IS_ERR(gmu->qmp)) {
1027                 ret = qmp_send(gmu->qmp, "{class: gpu, res: acd, val: %d}",
1028                                0 /* Hardcode ACD to be disabled for now */);
1029                 if (ret)
1030                         dev_err(gmu->dev, "failed to send GPU ACD state\n");
1031         }
1032
1033         /* Turn on the resources */
1034         pm_runtime_get_sync(gmu->dev);
1035
1036         /*
1037          * "enable" the GX power domain which won't actually do anything but it
1038          * will make sure that the refcounting is correct in case we need to
1039          * bring down the GX after a GMU failure
1040          */
1041         if (!IS_ERR_OR_NULL(gmu->gxpd))
1042                 pm_runtime_get_sync(gmu->gxpd);
1043
1044         /* Use a known rate to bring up the GMU */
1045         clk_set_rate(gmu->core_clk, 200000000);
1046         clk_set_rate(gmu->hub_clk, adreno_is_a740_family(adreno_gpu) ?
1047                      200000000 : 150000000);
1048         ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
1049         if (ret) {
1050                 pm_runtime_put(gmu->gxpd);
1051                 pm_runtime_put(gmu->dev);
1052                 return ret;
1053         }
1054
1055         /* Set the bus quota to a reasonable value for boot */
1056         a6xx_gmu_set_initial_bw(gpu, gmu);
1057
1058         /* Enable the GMU interrupt */
1059         gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
1060         gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~A6XX_GMU_IRQ_MASK);
1061         enable_irq(gmu->gmu_irq);
1062
1063         /* Check to see if we are doing a cold or warm boot */
1064         if (adreno_is_a7xx(adreno_gpu)) {
1065                 status = a6xx_llc_read(a6xx_gpu, REG_A7XX_CX_MISC_TCM_RET_CNTL) == 1 ?
1066                         GMU_WARM_BOOT : GMU_COLD_BOOT;
1067         } else if (gmu->legacy) {
1068                 status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
1069                         GMU_WARM_BOOT : GMU_COLD_BOOT;
1070         } else {
1071                 /*
1072                  * Warm boot path does not work on newer A6xx GPUs
1073                  * Presumably this is because icache/dcache regions must be restored
1074                  */
1075                 status = GMU_COLD_BOOT;
1076         }
1077
1078         ret = a6xx_gmu_fw_start(gmu, status);
1079         if (ret)
1080                 goto out;
1081
1082         ret = a6xx_hfi_start(gmu, status);
1083         if (ret)
1084                 goto out;
1085
1086         /*
1087          * Turn on the GMU firmware fault interrupt after we know the boot
1088          * sequence is successful
1089          */
1090         gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
1091         gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~A6XX_HFI_IRQ_MASK);
1092         enable_irq(gmu->hfi_irq);
1093
1094         /* Set the GPU to the current freq */
1095         a6xx_gmu_set_initial_freq(gpu, gmu);
1096
1097 out:
1098         /* On failure, shut down the GMU to leave it in a good state */
1099         if (ret) {
1100                 disable_irq(gmu->gmu_irq);
1101                 a6xx_rpmh_stop(gmu);
1102                 pm_runtime_put(gmu->gxpd);
1103                 pm_runtime_put(gmu->dev);
1104         }
1105
1106         return ret;
1107 }
1108
1109 bool a6xx_gmu_isidle(struct a6xx_gmu *gmu)
1110 {
1111         u32 reg;
1112
1113         if (!gmu->initialized)
1114                 return true;
1115
1116         reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS);
1117
1118         if (reg &  A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB)
1119                 return false;
1120
1121         return true;
1122 }
1123
1124 /* Gracefully try to shut down the GMU and by extension the GPU */
1125 static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
1126 {
1127         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1128         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1129         u32 val;
1130
1131         /*
1132          * The GMU may still be in slumber unless the GPU started so check and
1133          * skip putting it back into slumber if so
1134          */
1135         val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
1136
1137         if (val != 0xf) {
1138                 int ret = a6xx_gmu_wait_for_idle(gmu);
1139
1140                 /* If the GMU isn't responding assume it is hung */
1141                 if (ret) {
1142                         a6xx_gmu_force_off(gmu);
1143                         return;
1144                 }
1145
1146                 a6xx_bus_clear_pending_transactions(adreno_gpu, a6xx_gpu->hung);
1147
1148                 /* tell the GMU we want to slumber */
1149                 ret = a6xx_gmu_notify_slumber(gmu);
1150                 if (ret) {
1151                         a6xx_gmu_force_off(gmu);
1152                         return;
1153                 }
1154
1155                 ret = gmu_poll_timeout(gmu,
1156                         REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val,
1157                         !(val & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB),
1158                         100, 10000);
1159
1160                 /*
1161                  * Let the user know we failed to slumber but don't worry too
1162                  * much because we are powering down anyway
1163                  */
1164
1165                 if (ret)
1166                         DRM_DEV_ERROR(gmu->dev,
1167                                 "Unable to slumber GMU: status = 0%x/0%x\n",
1168                                 gmu_read(gmu,
1169                                         REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS),
1170                                 gmu_read(gmu,
1171                                         REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2));
1172         }
1173
1174         /* Turn off HFI */
1175         a6xx_hfi_stop(gmu);
1176
1177         /* Stop the interrupts and mask the hardware */
1178         a6xx_gmu_irq_disable(gmu);
1179
1180         /* Tell RPMh to power off the GPU */
1181         a6xx_rpmh_stop(gmu);
1182 }
1183
1184
1185 int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
1186 {
1187         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1188         struct msm_gpu *gpu = &a6xx_gpu->base.base;
1189
1190         if (!pm_runtime_active(gmu->dev))
1191                 return 0;
1192
1193         /*
1194          * Force the GMU off if we detected a hang, otherwise try to shut it
1195          * down gracefully
1196          */
1197         if (gmu->hung)
1198                 a6xx_gmu_force_off(gmu);
1199         else
1200                 a6xx_gmu_shutdown(gmu);
1201
1202         /* Remove the bus vote */
1203         dev_pm_opp_set_opp(&gpu->pdev->dev, NULL);
1204
1205         /*
1206          * Make sure the GX domain is off before turning off the GMU (CX)
1207          * domain. Usually the GMU does this but only if the shutdown sequence
1208          * was successful
1209          */
1210         if (!IS_ERR_OR_NULL(gmu->gxpd))
1211                 pm_runtime_put_sync(gmu->gxpd);
1212
1213         clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
1214
1215         pm_runtime_put_sync(gmu->dev);
1216
1217         return 0;
1218 }
1219
1220 static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu)
1221 {
1222         msm_gem_kernel_put(gmu->hfi.obj, gmu->aspace);
1223         msm_gem_kernel_put(gmu->debug.obj, gmu->aspace);
1224         msm_gem_kernel_put(gmu->icache.obj, gmu->aspace);
1225         msm_gem_kernel_put(gmu->dcache.obj, gmu->aspace);
1226         msm_gem_kernel_put(gmu->dummy.obj, gmu->aspace);
1227         msm_gem_kernel_put(gmu->log.obj, gmu->aspace);
1228
1229         gmu->aspace->mmu->funcs->detach(gmu->aspace->mmu);
1230         msm_gem_address_space_put(gmu->aspace);
1231 }
1232
1233 static int a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo,
1234                 size_t size, u64 iova, const char *name)
1235 {
1236         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1237         struct drm_device *dev = a6xx_gpu->base.base.dev;
1238         uint32_t flags = MSM_BO_WC;
1239         u64 range_start, range_end;
1240         int ret;
1241
1242         size = PAGE_ALIGN(size);
1243         if (!iova) {
1244                 /* no fixed address - use GMU's uncached range */
1245                 range_start = 0x60000000 + PAGE_SIZE; /* skip dummy page */
1246                 range_end = 0x80000000;
1247         } else {
1248                 /* range for fixed address */
1249                 range_start = iova;
1250                 range_end = iova + size;
1251                 /* use IOMMU_PRIV for icache/dcache */
1252                 flags |= MSM_BO_MAP_PRIV;
1253         }
1254
1255         bo->obj = msm_gem_new(dev, size, flags);
1256         if (IS_ERR(bo->obj))
1257                 return PTR_ERR(bo->obj);
1258
1259         ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->aspace, &bo->iova,
1260                                              range_start, range_end);
1261         if (ret) {
1262                 drm_gem_object_put(bo->obj);
1263                 return ret;
1264         }
1265
1266         bo->virt = msm_gem_get_vaddr(bo->obj);
1267         bo->size = size;
1268
1269         msm_gem_object_set_name(bo->obj, name);
1270
1271         return 0;
1272 }
1273
1274 static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu)
1275 {
1276         struct msm_mmu *mmu;
1277
1278         mmu = msm_iommu_new(gmu->dev, 0);
1279         if (!mmu)
1280                 return -ENODEV;
1281         if (IS_ERR(mmu))
1282                 return PTR_ERR(mmu);
1283
1284         gmu->aspace = msm_gem_address_space_create(mmu, "gmu", 0x0, 0x80000000);
1285         if (IS_ERR(gmu->aspace))
1286                 return PTR_ERR(gmu->aspace);
1287
1288         return 0;
1289 }
1290
1291 /* Return the 'arc-level' for the given frequency */
1292 static unsigned int a6xx_gmu_get_arc_level(struct device *dev,
1293                                            unsigned long freq)
1294 {
1295         struct dev_pm_opp *opp;
1296         unsigned int val;
1297
1298         if (!freq)
1299                 return 0;
1300
1301         opp = dev_pm_opp_find_freq_exact(dev, freq, true);
1302         if (IS_ERR(opp))
1303                 return 0;
1304
1305         val = dev_pm_opp_get_level(opp);
1306
1307         dev_pm_opp_put(opp);
1308
1309         return val;
1310 }
1311
1312 static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes,
1313                 unsigned long *freqs, int freqs_count, const char *id)
1314 {
1315         int i, j;
1316         const u16 *pri, *sec;
1317         size_t pri_count, sec_count;
1318
1319         pri = cmd_db_read_aux_data(id, &pri_count);
1320         if (IS_ERR(pri))
1321                 return PTR_ERR(pri);
1322         /*
1323          * The data comes back as an array of unsigned shorts so adjust the
1324          * count accordingly
1325          */
1326         pri_count >>= 1;
1327         if (!pri_count)
1328                 return -EINVAL;
1329
1330         sec = cmd_db_read_aux_data("mx.lvl", &sec_count);
1331         if (IS_ERR(sec))
1332                 return PTR_ERR(sec);
1333
1334         sec_count >>= 1;
1335         if (!sec_count)
1336                 return -EINVAL;
1337
1338         /* Construct a vote for each frequency */
1339         for (i = 0; i < freqs_count; i++) {
1340                 u8 pindex = 0, sindex = 0;
1341                 unsigned int level = a6xx_gmu_get_arc_level(dev, freqs[i]);
1342
1343                 /* Get the primary index that matches the arc level */
1344                 for (j = 0; j < pri_count; j++) {
1345                         if (pri[j] >= level) {
1346                                 pindex = j;
1347                                 break;
1348                         }
1349                 }
1350
1351                 if (j == pri_count) {
1352                         DRM_DEV_ERROR(dev,
1353                                       "Level %u not found in the RPMh list\n",
1354                                       level);
1355                         DRM_DEV_ERROR(dev, "Available levels:\n");
1356                         for (j = 0; j < pri_count; j++)
1357                                 DRM_DEV_ERROR(dev, "  %u\n", pri[j]);
1358
1359                         return -EINVAL;
1360                 }
1361
1362                 /*
1363                  * Look for a level in in the secondary list that matches. If
1364                  * nothing fits, use the maximum non zero vote
1365                  */
1366
1367                 for (j = 0; j < sec_count; j++) {
1368                         if (sec[j] >= level) {
1369                                 sindex = j;
1370                                 break;
1371                         } else if (sec[j]) {
1372                                 sindex = j;
1373                         }
1374                 }
1375
1376                 /* Construct the vote */
1377                 votes[i] = ((pri[pindex] & 0xffff) << 16) |
1378                         (sindex << 8) | pindex;
1379         }
1380
1381         return 0;
1382 }
1383
1384 /*
1385  * The GMU votes with the RPMh for itself and on behalf of the GPU but we need
1386  * to construct the list of votes on the CPU and send it over. Query the RPMh
1387  * voltage levels and build the votes
1388  */
1389
1390 static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu)
1391 {
1392         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1393         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1394         struct msm_gpu *gpu = &adreno_gpu->base;
1395         int ret;
1396
1397         /* Build the GX votes */
1398         ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes,
1399                 gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl");
1400
1401         /* Build the CX votes */
1402         ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes,
1403                 gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl");
1404
1405         return ret;
1406 }
1407
1408 static int a6xx_gmu_build_freq_table(struct device *dev, unsigned long *freqs,
1409                 u32 size)
1410 {
1411         int count = dev_pm_opp_get_opp_count(dev);
1412         struct dev_pm_opp *opp;
1413         int i, index = 0;
1414         unsigned long freq = 1;
1415
1416         /*
1417          * The OPP table doesn't contain the "off" frequency level so we need to
1418          * add 1 to the table size to account for it
1419          */
1420
1421         if (WARN(count + 1 > size,
1422                 "The GMU frequency table is being truncated\n"))
1423                 count = size - 1;
1424
1425         /* Set the "off" frequency */
1426         freqs[index++] = 0;
1427
1428         for (i = 0; i < count; i++) {
1429                 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
1430                 if (IS_ERR(opp))
1431                         break;
1432
1433                 dev_pm_opp_put(opp);
1434                 freqs[index++] = freq++;
1435         }
1436
1437         return index;
1438 }
1439
1440 static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu)
1441 {
1442         struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
1443         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1444         struct msm_gpu *gpu = &adreno_gpu->base;
1445
1446         int ret = 0;
1447
1448         /*
1449          * The GMU handles its own frequency switching so build a list of
1450          * available frequencies to send during initialization
1451          */
1452         ret = devm_pm_opp_of_add_table(gmu->dev);
1453         if (ret) {
1454                 DRM_DEV_ERROR(gmu->dev, "Unable to set the OPP table for the GMU\n");
1455                 return ret;
1456         }
1457
1458         gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev,
1459                 gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs));
1460
1461         /*
1462          * The GMU also handles GPU frequency switching so build a list
1463          * from the GPU OPP table
1464          */
1465         gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev,
1466                 gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs));
1467
1468         gmu->current_perf_index = gmu->nr_gpu_freqs - 1;
1469
1470         /* Build the list of RPMh votes that we'll send to the GMU */
1471         return a6xx_gmu_rpmh_votes_init(gmu);
1472 }
1473
1474 static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu)
1475 {
1476         int ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks);
1477
1478         if (ret < 1)
1479                 return ret;
1480
1481         gmu->nr_clocks = ret;
1482
1483         gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks,
1484                 gmu->nr_clocks, "gmu");
1485
1486         gmu->hub_clk = msm_clk_bulk_get_clock(gmu->clocks,
1487                 gmu->nr_clocks, "hub");
1488
1489         return 0;
1490 }
1491
1492 static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev,
1493                 const char *name)
1494 {
1495         void __iomem *ret;
1496         struct resource *res = platform_get_resource_byname(pdev,
1497                         IORESOURCE_MEM, name);
1498
1499         if (!res) {
1500                 DRM_DEV_ERROR(&pdev->dev, "Unable to find the %s registers\n", name);
1501                 return ERR_PTR(-EINVAL);
1502         }
1503
1504         ret = ioremap(res->start, resource_size(res));
1505         if (!ret) {
1506                 DRM_DEV_ERROR(&pdev->dev, "Unable to map the %s registers\n", name);
1507                 return ERR_PTR(-EINVAL);
1508         }
1509
1510         return ret;
1511 }
1512
1513 static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev,
1514                 const char *name, irq_handler_t handler)
1515 {
1516         int irq, ret;
1517
1518         irq = platform_get_irq_byname(pdev, name);
1519
1520         ret = request_irq(irq, handler, IRQF_TRIGGER_HIGH, name, gmu);
1521         if (ret) {
1522                 DRM_DEV_ERROR(&pdev->dev, "Unable to get interrupt %s %d\n",
1523                               name, ret);
1524                 return ret;
1525         }
1526
1527         disable_irq(irq);
1528
1529         return irq;
1530 }
1531
1532 void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
1533 {
1534         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1535         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1536         struct platform_device *pdev = to_platform_device(gmu->dev);
1537
1538         mutex_lock(&gmu->lock);
1539         if (!gmu->initialized) {
1540                 mutex_unlock(&gmu->lock);
1541                 return;
1542         }
1543
1544         gmu->initialized = false;
1545
1546         mutex_unlock(&gmu->lock);
1547
1548         pm_runtime_force_suspend(gmu->dev);
1549
1550         /*
1551          * Since cxpd is a virt device, the devlink with gmu-dev will be removed
1552          * automatically when we do detach
1553          */
1554         dev_pm_domain_detach(gmu->cxpd, false);
1555
1556         if (!IS_ERR_OR_NULL(gmu->gxpd)) {
1557                 pm_runtime_disable(gmu->gxpd);
1558                 dev_pm_domain_detach(gmu->gxpd, false);
1559         }
1560
1561         iounmap(gmu->mmio);
1562         if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
1563                 iounmap(gmu->rscc);
1564         gmu->mmio = NULL;
1565         gmu->rscc = NULL;
1566
1567         if (!adreno_has_gmu_wrapper(adreno_gpu)) {
1568                 a6xx_gmu_memory_free(gmu);
1569
1570                 free_irq(gmu->gmu_irq, gmu);
1571                 free_irq(gmu->hfi_irq, gmu);
1572         }
1573
1574         /* Drop reference taken in of_find_device_by_node */
1575         put_device(gmu->dev);
1576 }
1577
1578 static int cxpd_notifier_cb(struct notifier_block *nb,
1579                         unsigned long action, void *data)
1580 {
1581         struct a6xx_gmu *gmu = container_of(nb, struct a6xx_gmu, pd_nb);
1582
1583         if (action == GENPD_NOTIFY_OFF)
1584                 complete_all(&gmu->pd_gate);
1585
1586         return 0;
1587 }
1588
1589 int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
1590 {
1591         struct platform_device *pdev = of_find_device_by_node(node);
1592         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1593         int ret;
1594
1595         if (!pdev)
1596                 return -ENODEV;
1597
1598         gmu->dev = &pdev->dev;
1599
1600         of_dma_configure(gmu->dev, node, true);
1601
1602         pm_runtime_enable(gmu->dev);
1603
1604         /* Mark legacy for manual SPTPRAC control */
1605         gmu->legacy = true;
1606
1607         /* Map the GMU registers */
1608         gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
1609         if (IS_ERR(gmu->mmio)) {
1610                 ret = PTR_ERR(gmu->mmio);
1611                 goto err_mmio;
1612         }
1613
1614         gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx");
1615         if (IS_ERR(gmu->cxpd)) {
1616                 ret = PTR_ERR(gmu->cxpd);
1617                 goto err_mmio;
1618         }
1619
1620         if (!device_link_add(gmu->dev, gmu->cxpd, DL_FLAG_PM_RUNTIME)) {
1621                 ret = -ENODEV;
1622                 goto detach_cxpd;
1623         }
1624
1625         init_completion(&gmu->pd_gate);
1626         complete_all(&gmu->pd_gate);
1627         gmu->pd_nb.notifier_call = cxpd_notifier_cb;
1628
1629         /* Get a link to the GX power domain to reset the GPU */
1630         gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
1631         if (IS_ERR(gmu->gxpd)) {
1632                 ret = PTR_ERR(gmu->gxpd);
1633                 goto err_mmio;
1634         }
1635
1636         gmu->initialized = true;
1637
1638         return 0;
1639
1640 detach_cxpd:
1641         dev_pm_domain_detach(gmu->cxpd, false);
1642
1643 err_mmio:
1644         iounmap(gmu->mmio);
1645
1646         /* Drop reference taken in of_find_device_by_node */
1647         put_device(gmu->dev);
1648
1649         return ret;
1650 }
1651
1652 int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
1653 {
1654         struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1655         struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1656         struct platform_device *pdev = of_find_device_by_node(node);
1657         int ret;
1658
1659         if (!pdev)
1660                 return -ENODEV;
1661
1662         gmu->dev = &pdev->dev;
1663
1664         of_dma_configure(gmu->dev, node, true);
1665
1666         /* Fow now, don't do anything fancy until we get our feet under us */
1667         gmu->idle_level = GMU_IDLE_STATE_ACTIVE;
1668
1669         pm_runtime_enable(gmu->dev);
1670
1671         /* Get the list of clocks */
1672         ret = a6xx_gmu_clocks_probe(gmu);
1673         if (ret)
1674                 goto err_put_device;
1675
1676         ret = a6xx_gmu_memory_probe(gmu);
1677         if (ret)
1678                 goto err_put_device;
1679
1680
1681         /* A660 now requires handling "prealloc requests" in GMU firmware
1682          * For now just hardcode allocations based on the known firmware.
1683          * note: there is no indication that these correspond to "dummy" or
1684          * "debug" regions, but this "guess" allows reusing these BOs which
1685          * are otherwise unused by a660.
1686          */
1687         gmu->dummy.size = SZ_4K;
1688         if (adreno_is_a660_family(adreno_gpu) ||
1689             adreno_is_a7xx(adreno_gpu)) {
1690                 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_4K * 7,
1691                                             0x60400000, "debug");
1692                 if (ret)
1693                         goto err_memory;
1694
1695                 gmu->dummy.size = SZ_8K;
1696         }
1697
1698         /* Allocate memory for the GMU dummy page */
1699         ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, gmu->dummy.size,
1700                                     0x60000000, "dummy");
1701         if (ret)
1702                 goto err_memory;
1703
1704         /* Note that a650 family also includes a660 family: */
1705         if (adreno_is_a650_family(adreno_gpu) ||
1706             adreno_is_a7xx(adreno_gpu)) {
1707                 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
1708                         SZ_16M - SZ_16K, 0x04000, "icache");
1709                 if (ret)
1710                         goto err_memory;
1711         /*
1712          * NOTE: when porting legacy ("pre-650-family") GPUs you may be tempted to add a condition
1713          * to allocate icache/dcache here, as per downstream code flow, but it may not actually be
1714          * necessary. If you omit this step and you don't get random pagefaults, you are likely
1715          * good to go without this!
1716          */
1717         } else if (adreno_is_a640_family(adreno_gpu)) {
1718                 ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
1719                         SZ_256K - SZ_16K, 0x04000, "icache");
1720                 if (ret)
1721                         goto err_memory;
1722
1723                 ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache,
1724                         SZ_256K - SZ_16K, 0x44000, "dcache");
1725                 if (ret)
1726                         goto err_memory;
1727         } else if (adreno_is_a630_family(adreno_gpu)) {
1728                 /* HFI v1, has sptprac */
1729                 gmu->legacy = true;
1730
1731                 /* Allocate memory for the GMU debug region */
1732                 ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0, "debug");
1733                 if (ret)
1734                         goto err_memory;
1735         }
1736
1737         /* Allocate memory for the GMU log region */
1738         ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_16K, 0, "log");
1739         if (ret)
1740                 goto err_memory;
1741
1742         /* Allocate memory for for the HFI queues */
1743         ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0, "hfi");
1744         if (ret)
1745                 goto err_memory;
1746
1747         /* Map the GMU registers */
1748         gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
1749         if (IS_ERR(gmu->mmio)) {
1750                 ret = PTR_ERR(gmu->mmio);
1751                 goto err_memory;
1752         }
1753
1754         if (adreno_is_a650_family(adreno_gpu) ||
1755             adreno_is_a7xx(adreno_gpu)) {
1756                 gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc");
1757                 if (IS_ERR(gmu->rscc)) {
1758                         ret = -ENODEV;
1759                         goto err_mmio;
1760                 }
1761         } else {
1762                 gmu->rscc = gmu->mmio + 0x23000;
1763         }
1764
1765         /* Get the HFI and GMU interrupts */
1766         gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
1767         gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq);
1768
1769         if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0) {
1770                 ret = -ENODEV;
1771                 goto err_mmio;
1772         }
1773
1774         gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx");
1775         if (IS_ERR(gmu->cxpd)) {
1776                 ret = PTR_ERR(gmu->cxpd);
1777                 goto err_mmio;
1778         }
1779
1780         if (!device_link_add(gmu->dev, gmu->cxpd,
1781                                         DL_FLAG_PM_RUNTIME)) {
1782                 ret = -ENODEV;
1783                 goto detach_cxpd;
1784         }
1785
1786         gmu->qmp = qmp_get(gmu->dev);
1787         if (IS_ERR(gmu->qmp) && adreno_is_a7xx(adreno_gpu))
1788                 return PTR_ERR(gmu->qmp);
1789
1790         init_completion(&gmu->pd_gate);
1791         complete_all(&gmu->pd_gate);
1792         gmu->pd_nb.notifier_call = cxpd_notifier_cb;
1793
1794         /*
1795          * Get a link to the GX power domain to reset the GPU in case of GMU
1796          * crash
1797          */
1798         gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
1799
1800         /* Get the power levels for the GMU and GPU */
1801         a6xx_gmu_pwrlevels_probe(gmu);
1802
1803         /* Set up the HFI queues */
1804         a6xx_hfi_init(gmu);
1805
1806         /* Initialize RPMh */
1807         a6xx_gmu_rpmh_init(gmu);
1808
1809         gmu->initialized = true;
1810
1811         return 0;
1812
1813         if (!IS_ERR_OR_NULL(gmu->qmp))
1814                 qmp_put(gmu->qmp);
1815
1816 detach_cxpd:
1817         dev_pm_domain_detach(gmu->cxpd, false);
1818
1819 err_mmio:
1820         iounmap(gmu->mmio);
1821         if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
1822                 iounmap(gmu->rscc);
1823         free_irq(gmu->gmu_irq, gmu);
1824         free_irq(gmu->hfi_irq, gmu);
1825
1826 err_memory:
1827         a6xx_gmu_memory_free(gmu);
1828 err_put_device:
1829         /* Drop reference taken in of_find_device_by_node */
1830         put_device(gmu->dev);
1831
1832         return ret;
1833 }