drivers/accel/ivpu/ivpu_mmu.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2023 Intel Corporation
   4  */
   5
   6 #include <linux/circ_buf.h>
   7 #include <linux/highmem.h>
   8
   9 #include "ivpu_drv.h"
  10 #include "ivpu_hw_mtl_reg.h"
  11 #include "ivpu_hw_reg_io.h"
  12 #include "ivpu_mmu.h"
  13 #include "ivpu_mmu_context.h"
  14 #include "ivpu_pm.h"
  15
  16 #define IVPU_MMU_IDR0_REF               0x080f3e0f
  17 #define IVPU_MMU_IDR0_REF_SIMICS        0x080f3e1f
  18 #define IVPU_MMU_IDR1_REF               0x0e739d18
  19 #define IVPU_MMU_IDR3_REF               0x0000003c
  20 #define IVPU_MMU_IDR5_REF               0x00040070
  21 #define IVPU_MMU_IDR5_REF_SIMICS        0x00000075
  22 #define IVPU_MMU_IDR5_REF_FPGA          0x00800075
  23
  24 #define IVPU_MMU_CDTAB_ENT_SIZE         64
  25 #define IVPU_MMU_CDTAB_ENT_COUNT_LOG2   8 /* 256 entries */
  26 #define IVPU_MMU_CDTAB_ENT_COUNT        ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2)
  27
  28 #define IVPU_MMU_STREAM_ID0             0
  29 #define IVPU_MMU_STREAM_ID3             3
  30
  31 #define IVPU_MMU_STRTAB_ENT_SIZE        64
  32 #define IVPU_MMU_STRTAB_ENT_COUNT       4
  33 #define IVPU_MMU_STRTAB_CFG_LOG2SIZE    2
  34 #define IVPU_MMU_STRTAB_CFG             IVPU_MMU_STRTAB_CFG_LOG2SIZE
  35
  36 #define IVPU_MMU_Q_COUNT_LOG2           4 /* 16 entries */
  37 #define IVPU_MMU_Q_COUNT                ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)
  38 #define IVPU_MMU_Q_WRAP_BIT             (IVPU_MMU_Q_COUNT << 1)
  39 #define IVPU_MMU_Q_WRAP_MASK            (IVPU_MMU_Q_WRAP_BIT - 1)
  40 #define IVPU_MMU_Q_IDX_MASK             (IVPU_MMU_Q_COUNT - 1)
  41 #define IVPU_MMU_Q_IDX(val)             ((val) & IVPU_MMU_Q_IDX_MASK)
  42
  43 #define IVPU_MMU_CMDQ_CMD_SIZE          16
  44 #define IVPU_MMU_CMDQ_SIZE              (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)
  45
  46 #define IVPU_MMU_EVTQ_CMD_SIZE          32
  47 #define IVPU_MMU_EVTQ_SIZE              (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)
  48
  49 #define IVPU_MMU_CMD_OPCODE             GENMASK(7, 0)
  50
  51 #define IVPU_MMU_CMD_SYNC_0_CS          GENMASK(13, 12)
  52 #define IVPU_MMU_CMD_SYNC_0_MSH         GENMASK(23, 22)
  53 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR    GENMASK(27, 24)
  54 #define IVPU_MMU_CMD_SYNC_0_MSI_ATTR    GENMASK(27, 24)
  55 #define IVPU_MMU_CMD_SYNC_0_MSI_DATA    GENMASK(63, 32)
  56
  57 #define IVPU_MMU_CMD_CFGI_0_SSEC        BIT(10)
  58 #define IVPU_MMU_CMD_CFGI_0_SSV         BIT(11)
  59 #define IVPU_MMU_CMD_CFGI_0_SSID        GENMASK(31, 12)
  60 #define IVPU_MMU_CMD_CFGI_0_SID         GENMASK(63, 32)
  61 #define IVPU_MMU_CMD_CFGI_1_RANGE       GENMASK(4, 0)
  62
  63 #define IVPU_MMU_CMD_TLBI_0_ASID        GENMASK(63, 48)
  64 #define IVPU_MMU_CMD_TLBI_0_VMID        GENMASK(47, 32)
  65
  66 #define CMD_PREFETCH_CFG                0x1
  67 #define CMD_CFGI_STE                    0x3
  68 #define CMD_CFGI_ALL                    0x4
  69 #define CMD_CFGI_CD                     0x5
  70 #define CMD_CFGI_CD_ALL                 0x6
  71 #define CMD_TLBI_NH_ASID                0x11
  72 #define CMD_TLBI_EL2_ALL                0x20
  73 #define CMD_TLBI_NSNH_ALL               0x30
  74 #define CMD_SYNC                        0x46
  75
  76 #define IVPU_MMU_EVT_F_UUT              0x01
  77 #define IVPU_MMU_EVT_C_BAD_STREAMID     0x02
  78 #define IVPU_MMU_EVT_F_STE_FETCH        0x03
  79 #define IVPU_MMU_EVT_C_BAD_STE          0x04
  80 #define IVPU_MMU_EVT_F_BAD_ATS_TREQ     0x05
  81 #define IVPU_MMU_EVT_F_STREAM_DISABLED  0x06
  82 #define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07
  83 #define IVPU_MMU_EVT_C_BAD_SUBSTREAMID  0x08
  84 #define IVPU_MMU_EVT_F_CD_FETCH         0x09
  85 #define IVPU_MMU_EVT_C_BAD_CD           0x0a
  86 #define IVPU_MMU_EVT_F_WALK_EABT        0x0b
  87 #define IVPU_MMU_EVT_F_TRANSLATION      0x10
  88 #define IVPU_MMU_EVT_F_ADDR_SIZE        0x11
  89 #define IVPU_MMU_EVT_F_ACCESS           0x12
  90 #define IVPU_MMU_EVT_F_PERMISSION       0x13
  91 #define IVPU_MMU_EVT_F_TLB_CONFLICT     0x20
  92 #define IVPU_MMU_EVT_F_CFG_CONFLICT     0x21
  93 #define IVPU_MMU_EVT_E_PAGE_REQUEST     0x24
  94 #define IVPU_MMU_EVT_F_VMS_FETCH        0x25
  95
  96 #define IVPU_MMU_EVT_OP_MASK            GENMASK_ULL(7, 0)
  97 #define IVPU_MMU_EVT_SSID_MASK          GENMASK_ULL(31, 12)
  98
  99 #define IVPU_MMU_Q_BASE_RWA             BIT(62)
 100 #define IVPU_MMU_Q_BASE_ADDR_MASK       GENMASK_ULL(51, 5)
 101 #define IVPU_MMU_STRTAB_BASE_RA         BIT(62)
 102 #define IVPU_MMU_STRTAB_BASE_ADDR_MASK  GENMASK_ULL(51, 6)
 103
 104 #define IVPU_MMU_IRQ_EVTQ_EN            BIT(2)
 105 #define IVPU_MMU_IRQ_GERROR_EN          BIT(0)
 106
 107 #define IVPU_MMU_CR0_ATSCHK             BIT(4)
 108 #define IVPU_MMU_CR0_CMDQEN             BIT(3)
 109 #define IVPU_MMU_CR0_EVTQEN             BIT(2)
 110 #define IVPU_MMU_CR0_PRIQEN             BIT(1)
 111 #define IVPU_MMU_CR0_SMMUEN             BIT(0)
 112
 113 #define IVPU_MMU_CR1_TABLE_SH           GENMASK(11, 10)
 114 #define IVPU_MMU_CR1_TABLE_OC           GENMASK(9, 8)
 115 #define IVPU_MMU_CR1_TABLE_IC           GENMASK(7, 6)
 116 #define IVPU_MMU_CR1_QUEUE_SH           GENMASK(5, 4)
 117 #define IVPU_MMU_CR1_QUEUE_OC           GENMASK(3, 2)
 118 #define IVPU_MMU_CR1_QUEUE_IC           GENMASK(1, 0)
 119 #define IVPU_MMU_CACHE_NC               0
 120 #define IVPU_MMU_CACHE_WB               1
 121 #define IVPU_MMU_CACHE_WT               2
 122 #define IVPU_MMU_SH_NSH                 0
 123 #define IVPU_MMU_SH_OSH                 2
 124 #define IVPU_MMU_SH_ISH                 3
 125
 126 #define IVPU_MMU_CMDQ_OP                GENMASK_ULL(7, 0)
 127
 128 #define IVPU_MMU_CD_0_TCR_T0SZ          GENMASK_ULL(5, 0)
 129 #define IVPU_MMU_CD_0_TCR_TG0           GENMASK_ULL(7, 6)
 130 #define IVPU_MMU_CD_0_TCR_IRGN0         GENMASK_ULL(9, 8)
 131 #define IVPU_MMU_CD_0_TCR_ORGN0         GENMASK_ULL(11, 10)
 132 #define IVPU_MMU_CD_0_TCR_SH0           GENMASK_ULL(13, 12)
 133 #define IVPU_MMU_CD_0_TCR_EPD0          BIT_ULL(14)
 134 #define IVPU_MMU_CD_0_TCR_EPD1          BIT_ULL(30)
 135 #define IVPU_MMU_CD_0_ENDI              BIT(15)
 136 #define IVPU_MMU_CD_0_V                 BIT(31)
 137 #define IVPU_MMU_CD_0_TCR_IPS           GENMASK_ULL(34, 32)
 138 #define IVPU_MMU_CD_0_TCR_TBI0          BIT_ULL(38)
 139 #define IVPU_MMU_CD_0_AA64              BIT(41)
 140 #define IVPU_MMU_CD_0_S                 BIT(44)
 141 #define IVPU_MMU_CD_0_R                 BIT(45)
 142 #define IVPU_MMU_CD_0_A                 BIT(46)
 143 #define IVPU_MMU_CD_0_ASET              BIT(47)
 144 #define IVPU_MMU_CD_0_ASID              GENMASK_ULL(63, 48)
 145
 146 #define IVPU_MMU_CD_1_TTB0_MASK         GENMASK_ULL(51, 4)
 147
 148 #define IVPU_MMU_STE_0_S1CDMAX          GENMASK_ULL(63, 59)
 149 #define IVPU_MMU_STE_0_S1FMT            GENMASK_ULL(5, 4)
 150 #define IVPU_MMU_STE_0_S1FMT_LINEAR     0
 151 #define IVPU_MMU_STE_DWORDS             8
 152 #define IVPU_MMU_STE_0_CFG_S1_TRANS     5
 153 #define IVPU_MMU_STE_0_CFG              GENMASK_ULL(3, 1)
 154 #define IVPU_MMU_STE_0_S1CTXPTR_MASK    GENMASK_ULL(51, 6)
 155 #define IVPU_MMU_STE_0_V                        BIT(0)
 156
 157 #define IVPU_MMU_STE_1_STRW_NSEL1       0ul
 158 #define IVPU_MMU_STE_1_CONT             GENMASK_ULL(16, 13)
 159 #define IVPU_MMU_STE_1_STRW             GENMASK_ULL(31, 30)
 160 #define IVPU_MMU_STE_1_PRIVCFG          GENMASK_ULL(49, 48)
 161 #define IVPU_MMU_STE_1_PRIVCFG_UNPRIV   2ul
 162 #define IVPU_MMU_STE_1_INSTCFG          GENMASK_ULL(51, 50)
 163 #define IVPU_MMU_STE_1_INSTCFG_DATA     2ul
 164 #define IVPU_MMU_STE_1_MEV              BIT(19)
 165 #define IVPU_MMU_STE_1_S1STALLD         BIT(27)
 166 #define IVPU_MMU_STE_1_S1C_CACHE_NC     0ul
 167 #define IVPU_MMU_STE_1_S1C_CACHE_WBRA   1ul
 168 #define IVPU_MMU_STE_1_S1C_CACHE_WT     2ul
 169 #define IVPU_MMU_STE_1_S1C_CACHE_WB     3ul
 170 #define IVPU_MMU_STE_1_S1CIR            GENMASK_ULL(3, 2)
 171 #define IVPU_MMU_STE_1_S1COR            GENMASK_ULL(5, 4)
 172 #define IVPU_MMU_STE_1_S1CSH            GENMASK_ULL(7, 6)
 173 #define IVPU_MMU_STE_1_S1DSS            GENMASK_ULL(1, 0)
 174 #define IVPU_MMU_STE_1_S1DSS_TERMINATE  0x0
 175
 176 #define IVPU_MMU_REG_TIMEOUT_US         (10 * USEC_PER_MSEC)
 177 #define IVPU_MMU_QUEUE_TIMEOUT_US       (100 * USEC_PER_MSEC)
 178
 179 #define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ)) | \
 180                                   (REG_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT)) | \
 181                                   (REG_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT)) | \
 182                                   (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT)) | \
 183                                   (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT)) | \
 184                                   (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT)) | \
 185                                   (REG_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT)))
 186
 187 static char *ivpu_mmu_event_to_str(u32 cmd)
 188 {
 189         switch (cmd) {
 190         case IVPU_MMU_EVT_F_UUT:
 191                 return "Unsupported Upstream Transaction";
 192         case IVPU_MMU_EVT_C_BAD_STREAMID:
 193                 return "Transaction StreamID out of range";
 194         case IVPU_MMU_EVT_F_STE_FETCH:
 195                 return "Fetch of STE caused external abort";
 196         case IVPU_MMU_EVT_C_BAD_STE:
 197                 return "Used STE invalid";
 198         case IVPU_MMU_EVT_F_BAD_ATS_TREQ:
 199                 return "Address Request disallowed for a StreamID";
 200         case IVPU_MMU_EVT_F_STREAM_DISABLED:
 201                 return "Transaction marks non-substream disabled";
 202         case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN:
 203                 return "MMU bypass is disallowed for this StreamID";
 204         case IVPU_MMU_EVT_C_BAD_SUBSTREAMID:
 205                 return "Invalid StreamID";
 206         case IVPU_MMU_EVT_F_CD_FETCH:
 207                 return "Fetch of CD caused external abort";
 208         case IVPU_MMU_EVT_C_BAD_CD:
 209                 return "Fetched CD invalid";
 210         case IVPU_MMU_EVT_F_WALK_EABT:
 211                 return " An external abort occurred fetching a TLB";
 212         case IVPU_MMU_EVT_F_TRANSLATION:
 213                 return "Translation fault";
 214         case IVPU_MMU_EVT_F_ADDR_SIZE:
 215                 return " Output address caused address size fault";
 216         case IVPU_MMU_EVT_F_ACCESS:
 217                 return "Access flag fault";
 218         case IVPU_MMU_EVT_F_PERMISSION:
 219                 return "Permission fault occurred on page access";
 220         case IVPU_MMU_EVT_F_TLB_CONFLICT:
 221                 return "A TLB conflict";
 222         case IVPU_MMU_EVT_F_CFG_CONFLICT:
 223                 return "A configuration cache conflict";
 224         case IVPU_MMU_EVT_E_PAGE_REQUEST:
 225                 return "Page request hint from a client device";
 226         case IVPU_MMU_EVT_F_VMS_FETCH:
 227                 return "Fetch of VMS caused external abort";
 228         default:
 229                 return "Unknown CMDQ command";
 230         }
 231 }
 232
 233 static void ivpu_mmu_config_check(struct ivpu_device *vdev)
 234 {
 235         u32 val_ref;
 236         u32 val;
 237
 238         if (ivpu_is_simics(vdev))
 239                 val_ref = IVPU_MMU_IDR0_REF_SIMICS;
 240         else
 241                 val_ref = IVPU_MMU_IDR0_REF;
 242
 243         val = REGV_RD32(MTL_VPU_HOST_MMU_IDR0);
 244         if (val != val_ref)
 245                 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);
 246
 247         val = REGV_RD32(MTL_VPU_HOST_MMU_IDR1);
 248         if (val != IVPU_MMU_IDR1_REF)
 249                 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);
 250
 251         val = REGV_RD32(MTL_VPU_HOST_MMU_IDR3);
 252         if (val != IVPU_MMU_IDR3_REF)
 253                 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);
 254
 255         if (ivpu_is_simics(vdev))
 256                 val_ref = IVPU_MMU_IDR5_REF_SIMICS;
 257         else if (ivpu_is_fpga(vdev))
 258                 val_ref = IVPU_MMU_IDR5_REF_FPGA;
 259         else
 260                 val_ref = IVPU_MMU_IDR5_REF;
 261
 262         val = REGV_RD32(MTL_VPU_HOST_MMU_IDR5);
 263         if (val != val_ref)
 264                 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);
 265 }
 266
 267 static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)
 268 {
 269         struct ivpu_mmu_info *mmu = vdev->mmu;
 270         struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
 271         size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE;
 272
 273         cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);
 274         if (!cdtab->base)
 275                 return -ENOMEM;
 276
 277         ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);
 278
 279         return 0;
 280 }
 281
 282 static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)
 283 {
 284         struct ivpu_mmu_info *mmu = vdev->mmu;
 285         struct ivpu_mmu_strtab *strtab = &mmu->strtab;
 286         size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE;
 287
 288         strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);
 289         if (!strtab->base)
 290                 return -ENOMEM;
 291
 292         strtab->base_cfg = IVPU_MMU_STRTAB_CFG;
 293         strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA;
 294         strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK;
 295
 296         ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",
 297                  &strtab->dma, &strtab->dma_q, size);
 298
 299         return 0;
 300 }
 301
 302 static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)
 303 {
 304         struct ivpu_mmu_info *mmu = vdev->mmu;
 305         struct ivpu_mmu_queue *q = &mmu->cmdq;
 306
 307         q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);
 308         if (!q->base)
 309                 return -ENOMEM;
 310
 311         q->dma_q = IVPU_MMU_Q_BASE_RWA;
 312         q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
 313         q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
 314
 315         ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",
 316                  &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE);
 317
 318         return 0;
 319 }
 320
 321 static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)
 322 {
 323         struct ivpu_mmu_info *mmu = vdev->mmu;
 324         struct ivpu_mmu_queue *q = &mmu->evtq;
 325
 326         q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);
 327         if (!q->base)
 328                 return -ENOMEM;
 329
 330         q->dma_q = IVPU_MMU_Q_BASE_RWA;
 331         q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;
 332         q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;
 333
 334         ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",
 335                  &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE);
 336
 337         return 0;
 338 }
 339
 340 static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)
 341 {
 342         int ret;
 343
 344         ret = ivpu_mmu_cdtab_alloc(vdev);
 345         if (ret) {
 346                 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);
 347                 return ret;
 348         }
 349
 350         ret = ivpu_mmu_strtab_alloc(vdev);
 351         if (ret) {
 352                 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);
 353                 return ret;
 354         }
 355
 356         ret = ivpu_mmu_cmdq_alloc(vdev);
 357         if (ret) {
 358                 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);
 359                 return ret;
 360         }
 361
 362         ret = ivpu_mmu_evtq_alloc(vdev);
 363         if (ret)
 364                 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);
 365
 366         return ret;
 367 }
 368
 369 static int ivpu_mmu_reg_write(struct ivpu_device *vdev, u32 reg, u32 val)
 370 {
 371         u32 reg_ack = reg + 4; /* ACK register is 4B after base register */
 372         u32 val_ack;
 373         int ret;
 374
 375         REGV_WR32(reg, val);
 376
 377         ret = REGV_POLL(reg_ack, val_ack, (val == val_ack), IVPU_MMU_REG_TIMEOUT_US);
 378         if (ret)
 379                 ivpu_err(vdev, "Failed to write register 0x%x\n", reg);
 380
 381         return ret;
 382 }
 383
 384 static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)
 385 {
 386         u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN;
 387         int ret;
 388
 389         ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, 0);
 390         if (ret)
 391                 return ret;
 392
 393         return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_IRQ_CTRL, irq_ctrl);
 394 }
 395
 396 static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)
 397 {
 398         struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;
 399
 400         return REGV_POLL(MTL_VPU_HOST_MMU_CMDQ_CONS, cmdq->cons, (cmdq->prod == cmdq->cons),
 401                          IVPU_MMU_QUEUE_TIMEOUT_US);
 402 }
 403
 404 static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)
 405 {
 406         struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
 407         u64 *queue_buffer = q->base;
 408         int idx = IVPU_MMU_Q_IDX(q->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));
 409
 410         if (!CIRC_SPACE(IVPU_MMU_Q_IDX(q->prod), IVPU_MMU_Q_IDX(q->cons), IVPU_MMU_Q_COUNT)) {
 411                 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);
 412                 return -EBUSY;
 413         }
 414
 415         queue_buffer[idx] = data0;
 416         queue_buffer[idx + 1] = data1;
 417         q->prod = (q->prod + 1) & IVPU_MMU_Q_WRAP_MASK;
 418
 419         ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);
 420
 421         return 0;
 422 }
 423
 424 static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)
 425 {
 426         struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;
 427         u64 val;
 428         int ret;
 429
 430         val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC) |
 431               FIELD_PREP(IVPU_MMU_CMD_SYNC_0_CS, 0x2) |
 432               FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSH, 0x3) |
 433               FIELD_PREP(IVPU_MMU_CMD_SYNC_0_MSI_ATTR, 0xf);
 434
 435         ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);
 436         if (ret)
 437                 return ret;
 438
 439         clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE);
 440         REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, q->prod);
 441
 442         ret = ivpu_mmu_cmdq_wait_for_cons(vdev);
 443         if (ret)
 444                 ivpu_err(vdev, "Timed out waiting for consumer: %d\n", ret);
 445
 446         return ret;
 447 }
 448
 449 static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)
 450 {
 451         u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL);
 452         u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f);
 453
 454         return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);
 455 }
 456
 457 static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)
 458 {
 459         u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) |
 460                   FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid);
 461
 462         return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);
 463 }
 464
 465 static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)
 466 {
 467         u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL);
 468
 469         return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);
 470 }
 471
 472 static int ivpu_mmu_reset(struct ivpu_device *vdev)
 473 {
 474         struct ivpu_mmu_info *mmu = vdev->mmu;
 475         u32 val;
 476         int ret;
 477
 478         memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);
 479         clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);
 480         mmu->cmdq.prod = 0;
 481         mmu->cmdq.cons = 0;
 482
 483         memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);
 484         clflush_cache_range(mmu->evtq.base, IVPU_MMU_EVTQ_SIZE);
 485         mmu->evtq.prod = 0;
 486         mmu->evtq.cons = 0;
 487
 488         ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, 0);
 489         if (ret)
 490                 return ret;
 491
 492         val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) |
 493               FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) |
 494               FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) |
 495               FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) |
 496               FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) |
 497               FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB);
 498         REGV_WR32(MTL_VPU_HOST_MMU_CR1, val);
 499
 500         REGV_WR64(MTL_VPU_HOST_MMU_STRTAB_BASE, mmu->strtab.dma_q);
 501         REGV_WR32(MTL_VPU_HOST_MMU_STRTAB_BASE_CFG, mmu->strtab.base_cfg);
 502
 503         REGV_WR64(MTL_VPU_HOST_MMU_CMDQ_BASE, mmu->cmdq.dma_q);
 504         REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_PROD, 0);
 505         REGV_WR32(MTL_VPU_HOST_MMU_CMDQ_CONS, 0);
 506
 507         val = IVPU_MMU_CR0_CMDQEN;
 508         ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
 509         if (ret)
 510                 return ret;
 511
 512         ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
 513         if (ret)
 514                 return ret;
 515
 516         ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
 517         if (ret)
 518                 return ret;
 519
 520         ret = ivpu_mmu_cmdq_sync(vdev);
 521         if (ret)
 522                 return ret;
 523
 524         REGV_WR64(MTL_VPU_HOST_MMU_EVTQ_BASE, mmu->evtq.dma_q);
 525         REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC, 0);
 526         REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, 0);
 527
 528         val |= IVPU_MMU_CR0_EVTQEN;
 529         ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
 530         if (ret)
 531                 return ret;
 532
 533         val |= IVPU_MMU_CR0_ATSCHK;
 534         ret = ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
 535         if (ret)
 536                 return ret;
 537
 538         ret = ivpu_mmu_irqs_setup(vdev);
 539         if (ret)
 540                 return ret;
 541
 542         val |= IVPU_MMU_CR0_SMMUEN;
 543         return ivpu_mmu_reg_write(vdev, MTL_VPU_HOST_MMU_CR0, val);
 544 }
 545
 546 static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)
 547 {
 548         struct ivpu_mmu_info *mmu = vdev->mmu;
 549         struct ivpu_mmu_strtab *strtab = &mmu->strtab;
 550         struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
 551         u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE);
 552         u64 str[2];
 553
 554         str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) |
 555                  FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) |
 556                  FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) |
 557                  IVPU_MMU_STE_0_V |
 558                  (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK);
 559
 560         str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) |
 561                  FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
 562                  FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) |
 563                  FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) |
 564                  FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) |
 565                  FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) |
 566                  FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) |
 567                  FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) |
 568                  IVPU_MMU_STE_1_MEV |
 569                  IVPU_MMU_STE_1_S1STALLD;
 570
 571         WRITE_ONCE(entry[1], str[1]);
 572         WRITE_ONCE(entry[0], str[0]);
 573
 574         clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE);
 575
 576         ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);
 577 }
 578
 579 static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)
 580 {
 581         ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);
 582         ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);
 583
 584         return 0;
 585 }
 586
 587 int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
 588 {
 589         struct ivpu_mmu_info *mmu = vdev->mmu;
 590         int ret = 0;
 591
 592         mutex_lock(&mmu->lock);
 593         if (!mmu->on)
 594                 goto unlock;
 595
 596         ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);
 597         if (ret)
 598                 goto unlock;
 599
 600         ret = ivpu_mmu_cmdq_sync(vdev);
 601 unlock:
 602         mutex_unlock(&mmu->lock);
 603         return ret;
 604 }
 605
 606 static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)
 607 {
 608         struct ivpu_mmu_info *mmu = vdev->mmu;
 609         struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
 610         u64 *entry;
 611         u64 cd[4];
 612         int ret = 0;
 613
 614         if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
 615                 return -EINVAL;
 616
 617         entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
 618
 619         if (cd_dma != 0) {
 620                 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, 26) |
 621                         FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) |
 622                         FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) |
 623                         FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) |
 624                         FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) |
 625                         FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, 3) |
 626                         FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) |
 627                         IVPU_MMU_CD_0_TCR_EPD1 |
 628                         IVPU_MMU_CD_0_AA64 |
 629                         IVPU_MMU_CD_0_R |
 630                         IVPU_MMU_CD_0_ASET |
 631                         IVPU_MMU_CD_0_V;
 632                 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;
 633                 cd[2] = 0;
 634                 cd[3] = 0x0000000000007444;
 635
 636                 /* For global context generate memory fault on VPU */
 637                 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID)
 638                         cd[0] |= IVPU_MMU_CD_0_A;
 639         } else {
 640                 memset(cd, 0, sizeof(cd));
 641         }
 642
 643         WRITE_ONCE(entry[1], cd[1]);
 644         WRITE_ONCE(entry[2], cd[2]);
 645         WRITE_ONCE(entry[3], cd[3]);
 646         WRITE_ONCE(entry[0], cd[0]);
 647
 648         clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
 649
 650         ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
 651                  cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);
 652
 653         mutex_lock(&mmu->lock);
 654         if (!mmu->on)
 655                 goto unlock;
 656
 657         ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
 658         if (ret)
 659                 goto unlock;
 660
 661         ret = ivpu_mmu_cmdq_sync(vdev);
 662 unlock:
 663         mutex_unlock(&mmu->lock);
 664         return ret;
 665 }
 666
 667 static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev)
 668 {
 669         int ret;
 670
 671         ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma);
 672         if (ret)
 673                 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret);
 674
 675         return ret;
 676 }
 677
 678 static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma)
 679 {
 680         int ret;
 681
 682         if (ssid == 0) {
 683                 ivpu_err(vdev, "Invalid SSID: %u\n", ssid);
 684                 return -EINVAL;
 685         }
 686
 687         ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma);
 688         if (ret)
 689                 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret);
 690
 691         return ret;
 692 }
 693
 694 int ivpu_mmu_init(struct ivpu_device *vdev)
 695 {
 696         struct ivpu_mmu_info *mmu = vdev->mmu;
 697         int ret;
 698
 699         ivpu_dbg(vdev, MMU, "Init..\n");
 700
 701         drmm_mutex_init(&vdev->drm, &mmu->lock);
 702         ivpu_mmu_config_check(vdev);
 703
 704         ret = ivpu_mmu_structs_alloc(vdev);
 705         if (ret)
 706                 return ret;
 707
 708         ret = ivpu_mmu_strtab_init(vdev);
 709         if (ret) {
 710                 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
 711                 return ret;
 712         }
 713
 714         ret = ivpu_mmu_cd_add_gbl(vdev);
 715         if (ret) {
 716                 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);
 717                 return ret;
 718         }
 719
 720         ret = ivpu_mmu_enable(vdev);
 721         if (ret) {
 722                 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
 723                 return ret;
 724         }
 725
 726         ivpu_dbg(vdev, MMU, "Init done\n");
 727
 728         return 0;
 729 }
 730
 731 int ivpu_mmu_enable(struct ivpu_device *vdev)
 732 {
 733         struct ivpu_mmu_info *mmu = vdev->mmu;
 734         int ret;
 735
 736         mutex_lock(&mmu->lock);
 737
 738         mmu->on = true;
 739
 740         ret = ivpu_mmu_reset(vdev);
 741         if (ret) {
 742                 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);
 743                 goto err;
 744         }
 745
 746         ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
 747         if (ret)
 748                 goto err;
 749
 750         ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);
 751         if (ret)
 752                 goto err;
 753
 754         ret = ivpu_mmu_cmdq_sync(vdev);
 755         if (ret)
 756                 goto err;
 757
 758         mutex_unlock(&mmu->lock);
 759
 760         return 0;
 761 err:
 762         mmu->on = false;
 763         mutex_unlock(&mmu->lock);
 764         return ret;
 765 }
 766
 767 void ivpu_mmu_disable(struct ivpu_device *vdev)
 768 {
 769         struct ivpu_mmu_info *mmu = vdev->mmu;
 770
 771         mutex_lock(&mmu->lock);
 772         mmu->on = false;
 773         mutex_unlock(&mmu->lock);
 774 }
 775
 776 static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)
 777 {
 778         u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
 779         u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]);
 780         u64 fetch_addr = ((u64)event[7]) << 32 | event[6];
 781         u64 in_addr = ((u64)event[5]) << 32 | event[4];
 782         u32 sid = event[1];
 783
 784         ivpu_err(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n",
 785                  op, ivpu_mmu_event_to_str(op), ssid, sid, event[2], event[3], in_addr, fetch_addr);
 786 }
 787
 788 static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)
 789 {
 790         struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;
 791         u32 idx = IVPU_MMU_Q_IDX(evtq->cons);
 792         u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE);
 793
 794         evtq->prod = REGV_RD32(MTL_VPU_HOST_MMU_EVTQ_PROD_SEC);
 795         if (!CIRC_CNT(IVPU_MMU_Q_IDX(evtq->prod), IVPU_MMU_Q_IDX(evtq->cons), IVPU_MMU_Q_COUNT))
 796                 return NULL;
 797
 798         clflush_cache_range(evt, IVPU_MMU_EVTQ_CMD_SIZE);
 799
 800         evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;
 801         REGV_WR32(MTL_VPU_HOST_MMU_EVTQ_CONS_SEC, evtq->cons);
 802
 803         return evt;
 804 }
 805
 806 void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)
 807 {
 808         bool schedule_recovery = false;
 809         u32 *event;
 810         u32 ssid;
 811
 812         ivpu_dbg(vdev, IRQ, "MMU event queue\n");
 813
 814         while ((event = ivpu_mmu_get_event(vdev)) != NULL) {
 815                 ivpu_mmu_dump_event(vdev, event);
 816
 817                 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);
 818                 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID)
 819                         schedule_recovery = true;
 820                 else
 821                         ivpu_mmu_user_context_mark_invalid(vdev, ssid);
 822         }
 823
 824         if (schedule_recovery)
 825                 ivpu_pm_schedule_recovery(vdev);
 826 }
 827
 828 void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)
 829 {
 830         u32 gerror_val, gerrorn_val, active;
 831
 832         ivpu_dbg(vdev, IRQ, "MMU error\n");
 833
 834         gerror_val = REGV_RD32(MTL_VPU_HOST_MMU_GERROR);
 835         gerrorn_val = REGV_RD32(MTL_VPU_HOST_MMU_GERRORN);
 836
 837         active = gerror_val ^ gerrorn_val;
 838         if (!(active & IVPU_MMU_GERROR_ERR_MASK))
 839                 return;
 840
 841         if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_ABT, active))
 842                 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");
 843
 844         if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_PRIQ_ABT, active))
 845                 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");
 846
 847         if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_EVTQ_ABT, active))
 848                 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");
 849
 850         if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, MSI_CMDQ_ABT, active))
 851                 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");
 852
 853         if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, PRIQ_ABT, active))
 854                 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");
 855
 856         if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, EVTQ_ABT, active))
 857                 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");
 858
 859         if (REG_TEST_FLD(MTL_VPU_HOST_MMU_GERROR, CMDQ, active))
 860                 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");
 861
 862         REGV_WR32(MTL_VPU_HOST_MMU_GERRORN, gerror_val);
 863 }
 864
 865 int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)
 866 {
 867         return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma);
 868 }
 869
 870 void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid)
 871 {
 872         ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */
 873 }