drivers/media/pci/intel/ipu6/ipu6.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013--2024 Intel Corporation
   4  */
   5
   6 #include <linux/bitfield.h>
   7 #include <linux/bits.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/err.h>
  10 #include <linux/firmware.h>
  11 #include <linux/kernel.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/io.h>
  14 #include <linux/list.h>
  15 #include <linux/module.h>
  16 #include <linux/pci-ats.h>
  17 #include <linux/pm_runtime.h>
  18 #include <linux/property.h>
  19 #include <linux/scatterlist.h>
  20 #include <linux/slab.h>
  21 #include <linux/types.h>
  22
  23 #include <media/ipu-bridge.h>
  24 #include <media/ipu6-pci-table.h>
  25
  26 #include "ipu6.h"
  27 #include "ipu6-bus.h"
  28 #include "ipu6-buttress.h"
  29 #include "ipu6-cpd.h"
  30 #include "ipu6-isys.h"
  31 #include "ipu6-mmu.h"
  32 #include "ipu6-platform-buttress-regs.h"
  33 #include "ipu6-platform-isys-csi2-reg.h"
  34 #include "ipu6-platform-regs.h"
  35
  36 #define IPU6_PCI_BAR            0
  37
  38 struct ipu6_cell_program {
  39         u32 magic_number;
  40
  41         u32 blob_offset;
  42         u32 blob_size;
  43
  44         u32 start[3];
  45
  46         u32 icache_source;
  47         u32 icache_target;
  48         u32 icache_size;
  49
  50         u32 pmem_source;
  51         u32 pmem_target;
  52         u32 pmem_size;
  53
  54         u32 data_source;
  55         u32 data_target;
  56         u32 data_size;
  57
  58         u32 bss_target;
  59         u32 bss_size;
  60
  61         u32 cell_id;
  62         u32 regs_addr;
  63
  64         u32 cell_pmem_data_bus_address;
  65         u32 cell_dmem_data_bus_address;
  66         u32 cell_pmem_control_bus_address;
  67         u32 cell_dmem_control_bus_address;
  68
  69         u32 next;
  70         u32 dummy[2];
  71 };
  72
  73 static struct ipu6_isys_internal_pdata isys_ipdata = {
  74         .hw_variant = {
  75                 .offset = IPU6_UNIFIED_OFFSET,
  76                 .nr_mmus = 3,
  77                 .mmu_hw = {
  78                         {
  79                                 .offset = IPU6_ISYS_IOMMU0_OFFSET,
  80                                 .info_bits = IPU6_INFO_REQUEST_DESTINATION_IOSF,
  81                                 .nr_l1streams = 16,
  82                                 .l1_block_sz = {
  83                                         3, 8, 2, 2, 2, 2, 2, 2, 1, 1,
  84                                         1, 1, 1, 1, 1, 1
  85                                 },
  86                                 .nr_l2streams = 16,
  87                                 .l2_block_sz = {
  88                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  89                                         2, 2, 2, 2, 2, 2
  90                                 },
  91                                 .insert_read_before_invalidate = false,
  92                                 .l1_stream_id_reg_offset =
  93                                 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
  94                                 .l2_stream_id_reg_offset =
  95                                 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
  96                         },
  97                         {
  98                                 .offset = IPU6_ISYS_IOMMU1_OFFSET,
  99                                 .info_bits = 0,
 100                                 .nr_l1streams = 16,
 101                                 .l1_block_sz = {
 102                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 103                                         2, 2, 2, 1, 1, 4
 104                                 },
 105                                 .nr_l2streams = 16,
 106                                 .l2_block_sz = {
 107                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 108                                         2, 2, 2, 2, 2, 2
 109                                 },
 110                                 .insert_read_before_invalidate = false,
 111                                 .l1_stream_id_reg_offset =
 112                                 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
 113                                 .l2_stream_id_reg_offset =
 114                                 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
 115                         },
 116                         {
 117                                 .offset = IPU6_ISYS_IOMMUI_OFFSET,
 118                                 .info_bits = 0,
 119                                 .nr_l1streams = 0,
 120                                 .nr_l2streams = 0,
 121                                 .insert_read_before_invalidate = false,
 122                         },
 123                 },
 124                 .cdc_fifos = 3,
 125                 .cdc_fifo_threshold = {6, 8, 2},
 126                 .dmem_offset = IPU6_ISYS_DMEM_OFFSET,
 127                 .spc_offset = IPU6_ISYS_SPC_OFFSET,
 128         },
 129         .isys_dma_overshoot = IPU6_ISYS_OVERALLOC_MIN,
 130 };
 131
 132 static struct ipu6_psys_internal_pdata psys_ipdata = {
 133         .hw_variant = {
 134                 .offset = IPU6_UNIFIED_OFFSET,
 135                 .nr_mmus = 4,
 136                 .mmu_hw = {
 137                         {
 138                                 .offset = IPU6_PSYS_IOMMU0_OFFSET,
 139                                 .info_bits =
 140                                 IPU6_INFO_REQUEST_DESTINATION_IOSF,
 141                                 .nr_l1streams = 16,
 142                                 .l1_block_sz = {
 143                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 144                                         2, 2, 2, 2, 2, 2
 145                                 },
 146                                 .nr_l2streams = 16,
 147                                 .l2_block_sz = {
 148                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 149                                         2, 2, 2, 2, 2, 2
 150                                 },
 151                                 .insert_read_before_invalidate = false,
 152                                 .l1_stream_id_reg_offset =
 153                                 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
 154                                 .l2_stream_id_reg_offset =
 155                                 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
 156                         },
 157                         {
 158                                 .offset = IPU6_PSYS_IOMMU1_OFFSET,
 159                                 .info_bits = 0,
 160                                 .nr_l1streams = 32,
 161                                 .l1_block_sz = {
 162                                         1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 163                                         2, 2, 2, 2, 2, 10,
 164                                         5, 4, 14, 6, 4, 14, 6, 4, 8,
 165                                         4, 2, 1, 1, 1, 1, 14
 166                                 },
 167                                 .nr_l2streams = 32,
 168                                 .l2_block_sz = {
 169                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 170                                         2, 2, 2, 2, 2, 2,
 171                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 172                                         2, 2, 2, 2, 2, 2
 173                                 },
 174                                 .insert_read_before_invalidate = false,
 175                                 .l1_stream_id_reg_offset =
 176                                 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
 177                                 .l2_stream_id_reg_offset =
 178                                 IPU6_PSYS_MMU1W_L2_STREAM_ID_REG_OFFSET,
 179                         },
 180                         {
 181                                 .offset = IPU6_PSYS_IOMMU1R_OFFSET,
 182                                 .info_bits = 0,
 183                                 .nr_l1streams = 16,
 184                                 .l1_block_sz = {
 185                                         1, 4, 4, 4, 4, 16, 8, 4, 32,
 186                                         16, 16, 2, 2, 2, 1, 12
 187                                 },
 188                                 .nr_l2streams = 16,
 189                                 .l2_block_sz = {
 190                                         2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
 191                                         2, 2, 2, 2, 2, 2
 192                                 },
 193                                 .insert_read_before_invalidate = false,
 194                                 .l1_stream_id_reg_offset =
 195                                 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
 196                                 .l2_stream_id_reg_offset =
 197                                 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
 198                         },
 199                         {
 200                                 .offset = IPU6_PSYS_IOMMUI_OFFSET,
 201                                 .info_bits = 0,
 202                                 .nr_l1streams = 0,
 203                                 .nr_l2streams = 0,
 204                                 .insert_read_before_invalidate = false,
 205                         },
 206                 },
 207                 .dmem_offset = IPU6_PSYS_DMEM_OFFSET,
 208         },
 209 };
 210
 211 static const struct ipu6_buttress_ctrl isys_buttress_ctrl = {
 212         .ratio = IPU6_IS_FREQ_CTL_DEFAULT_RATIO,
 213         .qos_floor = IPU6_IS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO,
 214         .freq_ctl = IPU6_BUTTRESS_REG_IS_FREQ_CTL,
 215         .pwr_sts_shift = IPU6_BUTTRESS_PWR_STATE_IS_PWR_SHIFT,
 216         .pwr_sts_mask = IPU6_BUTTRESS_PWR_STATE_IS_PWR_MASK,
 217         .pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE,
 218         .pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
 219 };
 220
 221 static const struct ipu6_buttress_ctrl psys_buttress_ctrl = {
 222         .ratio = IPU6_PS_FREQ_CTL_DEFAULT_RATIO,
 223         .qos_floor = IPU6_PS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO,
 224         .freq_ctl = IPU6_BUTTRESS_REG_PS_FREQ_CTL,
 225         .pwr_sts_shift = IPU6_BUTTRESS_PWR_STATE_PS_PWR_SHIFT,
 226         .pwr_sts_mask = IPU6_BUTTRESS_PWR_STATE_PS_PWR_MASK,
 227         .pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE,
 228         .pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
 229 };
 230
 231 static void
 232 ipu6_pkg_dir_configure_spc(struct ipu6_device *isp,
 233                            const struct ipu6_hw_variants *hw_variant,
 234                            int pkg_dir_idx, void __iomem *base,
 235                            u64 *pkg_dir, dma_addr_t pkg_dir_vied_address)
 236 {
 237         struct ipu6_cell_program *prog;
 238         void __iomem *spc_base;
 239         u32 server_fw_addr;
 240         dma_addr_t dma_addr;
 241         u32 pg_offset;
 242
 243         server_fw_addr = lower_32_bits(*(pkg_dir + (pkg_dir_idx + 1) * 2));
 244         if (pkg_dir_idx == IPU6_CPD_PKG_DIR_ISYS_SERVER_IDX)
 245                 dma_addr = sg_dma_address(isp->isys->fw_sgt.sgl);
 246         else
 247                 dma_addr = sg_dma_address(isp->psys->fw_sgt.sgl);
 248
 249         pg_offset = server_fw_addr - dma_addr;
 250         prog = (struct ipu6_cell_program *)((u64)isp->cpd_fw->data + pg_offset);
 251         spc_base = base + prog->regs_addr;
 252         if (spc_base != (base + hw_variant->spc_offset))
 253                 dev_warn(&isp->pdev->dev,
 254                          "SPC reg addr %p not matching value from CPD %p\n",
 255                          base + hw_variant->spc_offset, spc_base);
 256         writel(server_fw_addr + prog->blob_offset +
 257                prog->icache_source, spc_base + IPU6_PSYS_REG_SPC_ICACHE_BASE);
 258         writel(IPU6_INFO_REQUEST_DESTINATION_IOSF,
 259                spc_base + IPU6_REG_PSYS_INFO_SEG_0_CONFIG_ICACHE_MASTER);
 260         writel(prog->start[1], spc_base + IPU6_PSYS_REG_SPC_START_PC);
 261         writel(pkg_dir_vied_address, base + hw_variant->dmem_offset);
 262 }
 263
 264 void ipu6_configure_spc(struct ipu6_device *isp,
 265                         const struct ipu6_hw_variants *hw_variant,
 266                         int pkg_dir_idx, void __iomem *base, u64 *pkg_dir,
 267                         dma_addr_t pkg_dir_dma_addr)
 268 {
 269         void __iomem *dmem_base = base + hw_variant->dmem_offset;
 270         void __iomem *spc_regs_base = base + hw_variant->spc_offset;
 271         u32 val;
 272
 273         val = readl(spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL);
 274         val |= IPU6_PSYS_SPC_STATUS_CTRL_ICACHE_INVALIDATE;
 275         writel(val, spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL);
 276
 277         if (isp->secure_mode)
 278                 writel(IPU6_PKG_DIR_IMR_OFFSET, dmem_base);
 279         else
 280                 ipu6_pkg_dir_configure_spc(isp, hw_variant, pkg_dir_idx, base,
 281                                            pkg_dir, pkg_dir_dma_addr);
 282 }
 283 EXPORT_SYMBOL_NS_GPL(ipu6_configure_spc, INTEL_IPU6);
 284
 285 #define IPU6_ISYS_CSI2_NPORTS           4
 286 #define IPU6SE_ISYS_CSI2_NPORTS         4
 287 #define IPU6_TGL_ISYS_CSI2_NPORTS       8
 288 #define IPU6EP_MTL_ISYS_CSI2_NPORTS     4
 289
 290 static void ipu6_internal_pdata_init(struct ipu6_device *isp)
 291 {
 292         u8 hw_ver = isp->hw_ver;
 293
 294         isys_ipdata.num_parallel_streams = IPU6_ISYS_NUM_STREAMS;
 295         isys_ipdata.sram_gran_shift = IPU6_SRAM_GRANULARITY_SHIFT;
 296         isys_ipdata.sram_gran_size = IPU6_SRAM_GRANULARITY_SIZE;
 297         isys_ipdata.max_sram_size = IPU6_MAX_SRAM_SIZE;
 298         isys_ipdata.sensor_type_start = IPU6_FW_ISYS_SENSOR_TYPE_START;
 299         isys_ipdata.sensor_type_end = IPU6_FW_ISYS_SENSOR_TYPE_END;
 300         isys_ipdata.max_streams = IPU6_ISYS_NUM_STREAMS;
 301         isys_ipdata.max_send_queues = IPU6_N_MAX_SEND_QUEUES;
 302         isys_ipdata.max_sram_blocks = IPU6_NOF_SRAM_BLOCKS_MAX;
 303         isys_ipdata.max_devq_size = IPU6_DEV_SEND_QUEUE_SIZE;
 304         isys_ipdata.csi2.nports = IPU6_ISYS_CSI2_NPORTS;
 305         isys_ipdata.csi2.irq_mask = IPU6_CSI_RX_ERROR_IRQ_MASK;
 306         isys_ipdata.csi2.ctrl0_irq_edge = IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_EDGE;
 307         isys_ipdata.csi2.ctrl0_irq_clear =
 308                 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_CLEAR;
 309         isys_ipdata.csi2.ctrl0_irq_mask = IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_MASK;
 310         isys_ipdata.csi2.ctrl0_irq_enable =
 311                 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_ENABLE;
 312         isys_ipdata.csi2.ctrl0_irq_status =
 313                 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_STATUS;
 314         isys_ipdata.csi2.ctrl0_irq_lnp =
 315                 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_LEVEL_NOT_PULSE;
 316         isys_ipdata.enhanced_iwake = is_ipu6ep_mtl(hw_ver) || is_ipu6ep(hw_ver);
 317         psys_ipdata.hw_variant.spc_offset = IPU6_PSYS_SPC_OFFSET;
 318         isys_ipdata.csi2.fw_access_port_ofs = CSI_REG_HUB_FW_ACCESS_PORT_OFS;
 319
 320         if (is_ipu6ep(hw_ver)) {
 321                 isys_ipdata.ltr = IPU6EP_LTR_VALUE;
 322                 isys_ipdata.memopen_threshold = IPU6EP_MIN_MEMOPEN_TH;
 323         }
 324
 325         if (is_ipu6_tgl(hw_ver))
 326                 isys_ipdata.csi2.nports = IPU6_TGL_ISYS_CSI2_NPORTS;
 327
 328         if (is_ipu6ep_mtl(hw_ver)) {
 329                 isys_ipdata.csi2.nports = IPU6EP_MTL_ISYS_CSI2_NPORTS;
 330
 331                 isys_ipdata.csi2.ctrl0_irq_edge =
 332                         IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_EDGE;
 333                 isys_ipdata.csi2.ctrl0_irq_clear =
 334                         IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_CLEAR;
 335                 isys_ipdata.csi2.ctrl0_irq_mask =
 336                         IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_MASK;
 337                 isys_ipdata.csi2.ctrl0_irq_enable =
 338                         IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_ENABLE;
 339                 isys_ipdata.csi2.ctrl0_irq_lnp =
 340                         IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_LEVEL_NOT_PULSE;
 341                 isys_ipdata.csi2.ctrl0_irq_status =
 342                         IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_STATUS;
 343                 isys_ipdata.csi2.fw_access_port_ofs =
 344                         CSI_REG_HUB_FW_ACCESS_PORT_V6OFS;
 345                 isys_ipdata.ltr = IPU6EP_MTL_LTR_VALUE;
 346                 isys_ipdata.memopen_threshold = IPU6EP_MTL_MIN_MEMOPEN_TH;
 347         }
 348
 349         if (is_ipu6se(hw_ver)) {
 350                 isys_ipdata.csi2.nports = IPU6SE_ISYS_CSI2_NPORTS;
 351                 isys_ipdata.csi2.irq_mask = IPU6SE_CSI_RX_ERROR_IRQ_MASK;
 352                 isys_ipdata.num_parallel_streams = IPU6SE_ISYS_NUM_STREAMS;
 353                 isys_ipdata.sram_gran_shift = IPU6SE_SRAM_GRANULARITY_SHIFT;
 354                 isys_ipdata.sram_gran_size = IPU6SE_SRAM_GRANULARITY_SIZE;
 355                 isys_ipdata.max_sram_size = IPU6SE_MAX_SRAM_SIZE;
 356                 isys_ipdata.sensor_type_start =
 357                         IPU6SE_FW_ISYS_SENSOR_TYPE_START;
 358                 isys_ipdata.sensor_type_end = IPU6SE_FW_ISYS_SENSOR_TYPE_END;
 359                 isys_ipdata.max_streams = IPU6SE_ISYS_NUM_STREAMS;
 360                 isys_ipdata.max_send_queues = IPU6SE_N_MAX_SEND_QUEUES;
 361                 isys_ipdata.max_sram_blocks = IPU6SE_NOF_SRAM_BLOCKS_MAX;
 362                 isys_ipdata.max_devq_size = IPU6SE_DEV_SEND_QUEUE_SIZE;
 363                 psys_ipdata.hw_variant.spc_offset = IPU6SE_PSYS_SPC_OFFSET;
 364         }
 365 }
 366
 367 static struct ipu6_bus_device *
 368 ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
 369                struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
 370                const struct ipu6_isys_internal_pdata *ipdata)
 371 {
 372         struct device *dev = &pdev->dev;
 373         struct ipu6_bus_device *isys_adev;
 374         struct ipu6_isys_pdata *pdata;
 375         int ret;
 376
 377         ret = ipu_bridge_init(dev, ipu_bridge_parse_ssdb);
 378         if (ret) {
 379                 dev_err_probe(dev, ret, "IPU6 bridge init failed\n");
 380                 return ERR_PTR(ret);
 381         }
 382
 383         pdata = kzalloc(sizeof(*pdata), GFP_KERNEL);
 384         if (!pdata)
 385                 return ERR_PTR(-ENOMEM);
 386
 387         pdata->base = base;
 388         pdata->ipdata = ipdata;
 389
 390         isys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl,
 391                                                IPU6_ISYS_NAME);
 392         if (IS_ERR(isys_adev)) {
 393                 dev_err_probe(dev, PTR_ERR(isys_adev),
 394                               "ipu6_bus_initialize_device isys failed\n");
 395                 kfree(pdata);
 396                 return ERR_CAST(isys_adev);
 397         }
 398
 399         isys_adev->mmu = ipu6_mmu_init(dev, base, ISYS_MMID,
 400                                        &ipdata->hw_variant);
 401         if (IS_ERR(isys_adev->mmu)) {
 402                 dev_err_probe(dev, PTR_ERR(isys_adev->mmu),
 403                               "ipu6_mmu_init(isys_adev->mmu) failed\n");
 404                 put_device(&isys_adev->auxdev.dev);
 405                 kfree(pdata);
 406                 return ERR_CAST(isys_adev->mmu);
 407         }
 408
 409         isys_adev->mmu->dev = &isys_adev->auxdev.dev;
 410
 411         ret = ipu6_bus_add_device(isys_adev);
 412         if (ret) {
 413                 kfree(pdata);
 414                 return ERR_PTR(ret);
 415         }
 416
 417         return isys_adev;
 418 }
 419
 420 static struct ipu6_bus_device *
 421 ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
 422                struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
 423                const struct ipu6_psys_internal_pdata *ipdata)
 424 {
 425         struct ipu6_bus_device *psys_adev;
 426         struct ipu6_psys_pdata *pdata;
 427         int ret;
 428
 429         pdata = kzalloc(sizeof(*pdata), GFP_KERNEL);
 430         if (!pdata)
 431                 return ERR_PTR(-ENOMEM);
 432
 433         pdata->base = base;
 434         pdata->ipdata = ipdata;
 435
 436         psys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl,
 437                                                IPU6_PSYS_NAME);
 438         if (IS_ERR(psys_adev)) {
 439                 dev_err_probe(&pdev->dev, PTR_ERR(psys_adev),
 440                               "ipu6_bus_initialize_device psys failed\n");
 441                 kfree(pdata);
 442                 return ERR_CAST(psys_adev);
 443         }
 444
 445         psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, PSYS_MMID,
 446                                        &ipdata->hw_variant);
 447         if (IS_ERR(psys_adev->mmu)) {
 448                 dev_err_probe(&pdev->dev, PTR_ERR(psys_adev->mmu),
 449                               "ipu6_mmu_init(psys_adev->mmu) failed\n");
 450                 put_device(&psys_adev->auxdev.dev);
 451                 kfree(pdata);
 452                 return ERR_CAST(psys_adev->mmu);
 453         }
 454
 455         psys_adev->mmu->dev = &psys_adev->auxdev.dev;
 456
 457         ret = ipu6_bus_add_device(psys_adev);
 458         if (ret) {
 459                 kfree(pdata);
 460                 return ERR_PTR(ret);
 461         }
 462
 463         return psys_adev;
 464 }
 465
 466 static int ipu6_pci_config_setup(struct pci_dev *dev, u8 hw_ver)
 467 {
 468         int ret;
 469
 470         /* disable IPU6 PCI ATS on mtl ES2 */
 471         if (is_ipu6ep_mtl(hw_ver) && boot_cpu_data.x86_stepping == 0x2 &&
 472             pci_ats_supported(dev))
 473                 pci_disable_ats(dev);
 474
 475         /* No PCI msi capability for IPU6EP */
 476         if (is_ipu6ep(hw_ver) || is_ipu6ep_mtl(hw_ver)) {
 477                 /* likely do nothing as msi not enabled by default */
 478                 pci_disable_msi(dev);
 479                 return 0;
 480         }
 481
 482         ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_MSI);
 483         if (ret < 0)
 484                 return dev_err_probe(&dev->dev, ret, "Request msi failed");
 485
 486         return 0;
 487 }
 488
 489 static void ipu6_configure_vc_mechanism(struct ipu6_device *isp)
 490 {
 491         u32 val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL);
 492
 493         if (IPU6_BTRS_ARB_STALL_MODE_VC0 == IPU6_BTRS_ARB_MODE_TYPE_STALL)
 494                 val |= BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC0;
 495         else
 496                 val &= ~BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC0;
 497
 498         if (IPU6_BTRS_ARB_STALL_MODE_VC1 == IPU6_BTRS_ARB_MODE_TYPE_STALL)
 499                 val |= BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC1;
 500         else
 501                 val &= ~BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC1;
 502
 503         writel(val, isp->base + BUTTRESS_REG_BTRS_CTRL);
 504 }
 505
 506 static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 507 {
 508         struct ipu6_buttress_ctrl *isys_ctrl = NULL, *psys_ctrl = NULL;
 509         struct device *dev = &pdev->dev;
 510         void __iomem *isys_base = NULL;
 511         void __iomem *psys_base = NULL;
 512         struct ipu6_device *isp;
 513         phys_addr_t phys;
 514         u32 val, version, sku_id;
 515         int ret;
 516
 517         isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL);
 518         if (!isp)
 519                 return -ENOMEM;
 520
 521         isp->pdev = pdev;
 522         INIT_LIST_HEAD(&isp->devices);
 523
 524         ret = pcim_enable_device(pdev);
 525         if (ret)
 526                 return dev_err_probe(dev, ret, "Enable PCI device failed\n");
 527
 528         phys = pci_resource_start(pdev, IPU6_PCI_BAR);
 529         dev_dbg(dev, "IPU6 PCI bar[%u] = %pa\n", IPU6_PCI_BAR, &phys);
 530
 531         ret = pcim_iomap_regions(pdev, 1 << IPU6_PCI_BAR, pci_name(pdev));
 532         if (ret)
 533                 return dev_err_probe(dev, ret, "Failed to I/O mem remapping\n");
 534
 535         isp->base = pcim_iomap_table(pdev)[IPU6_PCI_BAR];
 536         pci_set_drvdata(pdev, isp);
 537         pci_set_master(pdev);
 538
 539         isp->cpd_metadata_cmpnt_size = sizeof(struct ipu6_cpd_metadata_cmpnt);
 540         switch (id->device) {
 541         case PCI_DEVICE_ID_INTEL_IPU6:
 542                 isp->hw_ver = IPU6_VER_6;
 543                 isp->cpd_fw_name = IPU6_FIRMWARE_NAME;
 544                 break;
 545         case PCI_DEVICE_ID_INTEL_IPU6SE:
 546                 isp->hw_ver = IPU6_VER_6SE;
 547                 isp->cpd_fw_name = IPU6SE_FIRMWARE_NAME;
 548                 isp->cpd_metadata_cmpnt_size =
 549                         sizeof(struct ipu6se_cpd_metadata_cmpnt);
 550                 break;
 551         case PCI_DEVICE_ID_INTEL_IPU6EP_ADLP:
 552         case PCI_DEVICE_ID_INTEL_IPU6EP_RPLP:
 553                 isp->hw_ver = IPU6_VER_6EP;
 554                 isp->cpd_fw_name = IPU6EP_FIRMWARE_NAME;
 555                 break;
 556         case PCI_DEVICE_ID_INTEL_IPU6EP_ADLN:
 557                 isp->hw_ver = IPU6_VER_6EP;
 558                 isp->cpd_fw_name = IPU6EPADLN_FIRMWARE_NAME;
 559                 break;
 560         case PCI_DEVICE_ID_INTEL_IPU6EP_MTL:
 561                 isp->hw_ver = IPU6_VER_6EP_MTL;
 562                 isp->cpd_fw_name = IPU6EPMTL_FIRMWARE_NAME;
 563                 break;
 564         default:
 565                 return dev_err_probe(dev, -ENODEV,
 566                                      "Unsupported IPU6 device %x\n",
 567                                      id->device);
 568         }
 569
 570         ipu6_internal_pdata_init(isp);
 571
 572         isys_base = isp->base + isys_ipdata.hw_variant.offset;
 573         psys_base = isp->base + psys_ipdata.hw_variant.offset;
 574
 575         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(39));
 576         if (ret)
 577                 return dev_err_probe(dev, ret, "Failed to set DMA mask\n");
 578
 579         ret = dma_set_max_seg_size(dev, UINT_MAX);
 580         if (ret)
 581                 return dev_err_probe(dev, ret, "Failed to set max_seg_size\n");
 582
 583         ret = ipu6_pci_config_setup(pdev, isp->hw_ver);
 584         if (ret)
 585                 return ret;
 586
 587         ret = ipu6_buttress_init(isp);
 588         if (ret)
 589                 return ret;
 590
 591         ret = request_firmware(&isp->cpd_fw, isp->cpd_fw_name, dev);
 592         if (ret) {
 593                 dev_err_probe(&isp->pdev->dev, ret,
 594                               "Requesting signed firmware %s failed\n",
 595                               isp->cpd_fw_name);
 596                 goto buttress_exit;
 597         }
 598
 599         ret = ipu6_cpd_validate_cpd_file(isp, isp->cpd_fw->data,
 600                                          isp->cpd_fw->size);
 601         if (ret) {
 602                 dev_err_probe(&isp->pdev->dev, ret,
 603                               "Failed to validate cpd\n");
 604                 goto out_ipu6_bus_del_devices;
 605         }
 606
 607         isys_ctrl = devm_kmemdup(dev, &isys_buttress_ctrl,
 608                                  sizeof(isys_buttress_ctrl), GFP_KERNEL);
 609         if (!isys_ctrl) {
 610                 ret = -ENOMEM;
 611                 goto out_ipu6_bus_del_devices;
 612         }
 613
 614         isp->isys = ipu6_isys_init(pdev, dev, isys_ctrl, isys_base,
 615                                    &isys_ipdata);
 616         if (IS_ERR(isp->isys)) {
 617                 ret = PTR_ERR(isp->isys);
 618                 goto out_ipu6_bus_del_devices;
 619         }
 620
 621         psys_ctrl = devm_kmemdup(dev, &psys_buttress_ctrl,
 622                                  sizeof(psys_buttress_ctrl), GFP_KERNEL);
 623         if (!psys_ctrl) {
 624                 ret = -ENOMEM;
 625                 goto out_ipu6_bus_del_devices;
 626         }
 627
 628         isp->psys = ipu6_psys_init(pdev, &isp->isys->auxdev.dev, psys_ctrl,
 629                                    psys_base, &psys_ipdata);
 630         if (IS_ERR(isp->psys)) {
 631                 ret = PTR_ERR(isp->psys);
 632                 goto out_ipu6_bus_del_devices;
 633         }
 634
 635         ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);
 636         if (ret < 0)
 637                 goto out_ipu6_bus_del_devices;
 638
 639         ret = ipu6_mmu_hw_init(isp->psys->mmu);
 640         if (ret) {
 641                 dev_err_probe(&isp->pdev->dev, ret,
 642                               "Failed to set MMU hardware\n");
 643                 goto out_ipu6_bus_del_devices;
 644         }
 645
 646         ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw,
 647                                          &isp->psys->fw_sgt);
 648         if (ret) {
 649                 dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n");
 650                 goto out_ipu6_bus_del_devices;
 651         }
 652
 653         ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data);
 654         if (ret) {
 655                 dev_err_probe(&isp->pdev->dev, ret,
 656                               "failed to create pkg dir\n");
 657                 goto out_ipu6_bus_del_devices;
 658         }
 659
 660         ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr,
 661                                         ipu6_buttress_isr_threaded,
 662                                         IRQF_SHARED, IPU6_NAME, isp);
 663         if (ret) {
 664                 dev_err_probe(dev, ret, "Requesting irq failed\n");
 665                 goto out_ipu6_bus_del_devices;
 666         }
 667
 668         ret = ipu6_buttress_authenticate(isp);
 669         if (ret) {
 670                 dev_err_probe(&isp->pdev->dev, ret,
 671                               "FW authentication failed\n");
 672                 goto out_free_irq;
 673         }
 674
 675         ipu6_mmu_hw_cleanup(isp->psys->mmu);
 676         pm_runtime_put(&isp->psys->auxdev.dev);
 677
 678         /* Configure the arbitration mechanisms for VC requests */
 679         ipu6_configure_vc_mechanism(isp);
 680
 681         val = readl(isp->base + BUTTRESS_REG_SKU);
 682         sku_id = FIELD_GET(GENMASK(6, 4), val);
 683         version = FIELD_GET(GENMASK(3, 0), val);
 684         dev_info(dev, "IPU%u-v%u[%x] hardware version %d\n", version, sku_id,
 685                  pdev->device, isp->hw_ver);
 686
 687         pm_runtime_put_noidle(dev);
 688         pm_runtime_allow(dev);
 689
 690         isp->bus_ready_to_probe = true;
 691
 692         return 0;
 693
 694 out_free_irq:
 695         devm_free_irq(dev, pdev->irq, isp);
 696 out_ipu6_bus_del_devices:
 697         if (isp->psys) {
 698                 ipu6_cpd_free_pkg_dir(isp->psys);
 699                 ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);
 700         }
 701         if (!IS_ERR_OR_NULL(isp->psys) && !IS_ERR_OR_NULL(isp->psys->mmu))
 702                 ipu6_mmu_cleanup(isp->psys->mmu);
 703         if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu))
 704                 ipu6_mmu_cleanup(isp->isys->mmu);
 705         ipu6_bus_del_devices(pdev);
 706         release_firmware(isp->cpd_fw);
 707 buttress_exit:
 708         ipu6_buttress_exit(isp);
 709
 710         return ret;
 711 }
 712
 713 static void ipu6_pci_remove(struct pci_dev *pdev)
 714 {
 715         struct ipu6_device *isp = pci_get_drvdata(pdev);
 716         struct ipu6_mmu *isys_mmu = isp->isys->mmu;
 717         struct ipu6_mmu *psys_mmu = isp->psys->mmu;
 718
 719         devm_free_irq(&pdev->dev, pdev->irq, isp);
 720         ipu6_cpd_free_pkg_dir(isp->psys);
 721
 722         ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);
 723         ipu6_buttress_exit(isp);
 724
 725         ipu6_bus_del_devices(pdev);
 726
 727         pm_runtime_forbid(&pdev->dev);
 728         pm_runtime_get_noresume(&pdev->dev);
 729
 730         release_firmware(isp->cpd_fw);
 731
 732         ipu6_mmu_cleanup(psys_mmu);
 733         ipu6_mmu_cleanup(isys_mmu);
 734 }
 735
 736 static void ipu6_pci_reset_prepare(struct pci_dev *pdev)
 737 {
 738         struct ipu6_device *isp = pci_get_drvdata(pdev);
 739
 740         pm_runtime_forbid(&isp->pdev->dev);
 741 }
 742
 743 static void ipu6_pci_reset_done(struct pci_dev *pdev)
 744 {
 745         struct ipu6_device *isp = pci_get_drvdata(pdev);
 746
 747         ipu6_buttress_restore(isp);
 748         if (isp->secure_mode)
 749                 ipu6_buttress_reset_authentication(isp);
 750
 751         isp->need_ipc_reset = true;
 752         pm_runtime_allow(&isp->pdev->dev);
 753 }
 754
 755 /*
 756  * PCI base driver code requires driver to provide these to enable
 757  * PCI device level PM state transitions (D0<->D3)
 758  */
 759 static int ipu6_suspend(struct device *dev)
 760 {
 761         return 0;
 762 }
 763
 764 static int ipu6_resume(struct device *dev)
 765 {
 766         struct pci_dev *pdev = to_pci_dev(dev);
 767         struct ipu6_device *isp = pci_get_drvdata(pdev);
 768         struct ipu6_buttress *b = &isp->buttress;
 769         int ret;
 770
 771         /* Configure the arbitration mechanisms for VC requests */
 772         ipu6_configure_vc_mechanism(isp);
 773
 774         isp->secure_mode = ipu6_buttress_get_secure_mode(isp);
 775         dev_info(dev, "IPU6 in %s mode\n",
 776                  isp->secure_mode ? "secure" : "non-secure");
 777
 778         ipu6_buttress_restore(isp);
 779
 780         ret = ipu6_buttress_ipc_reset(isp, &b->cse);
 781         if (ret)
 782                 dev_err(&isp->pdev->dev, "IPC reset protocol failed!\n");
 783
 784         ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);
 785         if (ret < 0) {
 786                 dev_err(&isp->psys->auxdev.dev, "Failed to get runtime PM\n");
 787                 return 0;
 788         }
 789
 790         ret = ipu6_buttress_authenticate(isp);
 791         if (ret)
 792                 dev_err(&isp->pdev->dev, "FW authentication failed(%d)\n", ret);
 793
 794         pm_runtime_put(&isp->psys->auxdev.dev);
 795
 796         return 0;
 797 }
 798
 799 static int ipu6_runtime_resume(struct device *dev)
 800 {
 801         struct pci_dev *pdev = to_pci_dev(dev);
 802         struct ipu6_device *isp = pci_get_drvdata(pdev);
 803         int ret;
 804
 805         ipu6_configure_vc_mechanism(isp);
 806         ipu6_buttress_restore(isp);
 807
 808         if (isp->need_ipc_reset) {
 809                 struct ipu6_buttress *b = &isp->buttress;
 810
 811                 isp->need_ipc_reset = false;
 812                 ret = ipu6_buttress_ipc_reset(isp, &b->cse);
 813                 if (ret)
 814                         dev_err(&isp->pdev->dev, "IPC reset protocol failed\n");
 815         }
 816
 817         return 0;
 818 }
 819
 820 static const struct dev_pm_ops ipu6_pm_ops = {
 821         SYSTEM_SLEEP_PM_OPS(&ipu6_suspend, &ipu6_resume)
 822         RUNTIME_PM_OPS(&ipu6_suspend, &ipu6_runtime_resume, NULL)
 823 };
 824
 825 MODULE_DEVICE_TABLE(pci, ipu6_pci_tbl);
 826
 827 static const struct pci_error_handlers pci_err_handlers = {
 828         .reset_prepare = ipu6_pci_reset_prepare,
 829         .reset_done = ipu6_pci_reset_done,
 830 };
 831
 832 static struct pci_driver ipu6_pci_driver = {
 833         .name = IPU6_NAME,
 834         .id_table = ipu6_pci_tbl,
 835         .probe = ipu6_pci_probe,
 836         .remove = ipu6_pci_remove,
 837         .driver = {
 838                 .pm = pm_ptr(&ipu6_pm_ops),
 839         },
 840         .err_handler = &pci_err_handlers,
 841 };
 842
 843 module_pci_driver(ipu6_pci_driver);
 844
 845 MODULE_IMPORT_NS(INTEL_IPU_BRIDGE);
 846 MODULE_AUTHOR("Sakari Ailus <sakari.ailus@linux.intel.com>");
 847 MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>");
 848 MODULE_AUTHOR("Bingbu Cao <bingbu.cao@intel.com>");
 849 MODULE_AUTHOR("Qingwu Zhang <qingwu.zhang@intel.com>");
 850 MODULE_AUTHOR("Yunliang Ding <yunliang.ding@intel.com>");
 851 MODULE_AUTHOR("Hongju Wang <hongju.wang@intel.com>");
 852 MODULE_LICENSE("GPL");
 853 MODULE_DESCRIPTION("Intel IPU6 PCI driver");