projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
media: intel/ipu6: update the maximum supported csi2 port number to 6
[linux-2.6-block.git] / drivers / media / pci / intel / ipu6 / ipu6.c
CommitLineData
25fedc02
BC		 1// SPDX-License-Identifier: GPL-2.0-only
2/*
f50c4ca0 3 * Copyright (C) 2013--2024 Intel Corporation
25fedc02
BC		 4 */
5
6#include <linux/bitfield.h>
7#include <linux/bits.h>
8#include <linux/dma-mapping.h>
9#include <linux/err.h>
10#include <linux/firmware.h>
11#include <linux/kernel.h>
12#include <linux/interrupt.h>
13#include <linux/io.h>
14#include <linux/list.h>
15#include <linux/module.h>
16#include <linux/pci-ats.h>
17#include <linux/pm_runtime.h>
18#include <linux/property.h>
19#include <linux/scatterlist.h>
20#include <linux/slab.h>
21#include <linux/types.h>
22
23#include <media/ipu-bridge.h>
24#include <media/ipu6-pci-table.h>
25
26#include "ipu6.h"
27#include "ipu6-bus.h"
28#include "ipu6-buttress.h"
29#include "ipu6-cpd.h"
30#include "ipu6-isys.h"
31#include "ipu6-mmu.h"
32#include "ipu6-platform-buttress-regs.h"
33#include "ipu6-platform-isys-csi2-reg.h"
34#include "ipu6-platform-regs.h"
35
36#define IPU6_PCI_BAR 0
37
38struct ipu6_cell_program {
39 u32 magic_number;
40
41 u32 blob_offset;
42 u32 blob_size;
43
44 u32 start[3];
45
46 u32 icache_source;
47 u32 icache_target;
48 u32 icache_size;
49
50 u32 pmem_source;
51 u32 pmem_target;
52 u32 pmem_size;
53
54 u32 data_source;
55 u32 data_target;
56 u32 data_size;
57
58 u32 bss_target;
59 u32 bss_size;
60
61 u32 cell_id;
62 u32 regs_addr;
63
64 u32 cell_pmem_data_bus_address;
65 u32 cell_dmem_data_bus_address;
66 u32 cell_pmem_control_bus_address;
67 u32 cell_dmem_control_bus_address;
68
69 u32 next;
70 u32 dummy[2];
71};
72
73static struct ipu6_isys_internal_pdata isys_ipdata = {
74 .hw_variant = {
75 .offset = IPU6_UNIFIED_OFFSET,
76 .nr_mmus = 3,
77 .mmu_hw = {
78 {
79 .offset = IPU6_ISYS_IOMMU0_OFFSET,
80 .info_bits = IPU6_INFO_REQUEST_DESTINATION_IOSF,
81 .nr_l1streams = 16,
82 .l1_block_sz = {
83 3, 8, 2, 2, 2, 2, 2, 2, 1, 1,
84 1, 1, 1, 1, 1, 1
85 },
86 .nr_l2streams = 16,
87 .l2_block_sz = {
88 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
89 2, 2, 2, 2, 2, 2
90 },
91 .insert_read_before_invalidate = false,
92 .l1_stream_id_reg_offset =
93 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
94 .l2_stream_id_reg_offset =
95 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
96 },
97 {
98 .offset = IPU6_ISYS_IOMMU1_OFFSET,
99 .info_bits = 0,
100 .nr_l1streams = 16,
101 .l1_block_sz = {
102 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
103 2, 2, 2, 1, 1, 4
104 },
105 .nr_l2streams = 16,
106 .l2_block_sz = {
107 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
108 2, 2, 2, 2, 2, 2
109 },
110 .insert_read_before_invalidate = false,
111 .l1_stream_id_reg_offset =
112 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
113 .l2_stream_id_reg_offset =
114 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
115 },
116 {
117 .offset = IPU6_ISYS_IOMMUI_OFFSET,
118 .info_bits = 0,
119 .nr_l1streams = 0,
120 .nr_l2streams = 0,
121 .insert_read_before_invalidate = false,
122 },
123 },
124 .cdc_fifos = 3,
125 .cdc_fifo_threshold = {6, 8, 2},
126 .dmem_offset = IPU6_ISYS_DMEM_OFFSET,
127 .spc_offset = IPU6_ISYS_SPC_OFFSET,
128 },
129 .isys_dma_overshoot = IPU6_ISYS_OVERALLOC_MIN,
130};
131
132static struct ipu6_psys_internal_pdata psys_ipdata = {
133 .hw_variant = {
134 .offset = IPU6_UNIFIED_OFFSET,
135 .nr_mmus = 4,
136 .mmu_hw = {
137 {
138 .offset = IPU6_PSYS_IOMMU0_OFFSET,
139 .info_bits =
140 IPU6_INFO_REQUEST_DESTINATION_IOSF,
141 .nr_l1streams = 16,
142 .l1_block_sz = {
143 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
144 2, 2, 2, 2, 2, 2
145 },
146 .nr_l2streams = 16,
147 .l2_block_sz = {
148 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
149 2, 2, 2, 2, 2, 2
150 },
151 .insert_read_before_invalidate = false,
152 .l1_stream_id_reg_offset =
153 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
154 .l2_stream_id_reg_offset =
155 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
156 },
157 {
158 .offset = IPU6_PSYS_IOMMU1_OFFSET,
159 .info_bits = 0,
160 .nr_l1streams = 32,
161 .l1_block_sz = {
162 1, 2, 2, 2, 2, 2, 2, 2, 2, 2,
163 2, 2, 2, 2, 2, 10,
164 5, 4, 14, 6, 4, 14, 6, 4, 8,
165 4, 2, 1, 1, 1, 1, 14
166 },
167 .nr_l2streams = 32,
168 .l2_block_sz = {
169 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
170 2, 2, 2, 2, 2, 2,
171 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
172 2, 2, 2, 2, 2, 2
173 },
174 .insert_read_before_invalidate = false,
175 .l1_stream_id_reg_offset =
176 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
177 .l2_stream_id_reg_offset =
178 IPU6_PSYS_MMU1W_L2_STREAM_ID_REG_OFFSET,
179 },
180 {
181 .offset = IPU6_PSYS_IOMMU1R_OFFSET,
182 .info_bits = 0,
183 .nr_l1streams = 16,
184 .l1_block_sz = {
185 1, 4, 4, 4, 4, 16, 8, 4, 32,
186 16, 16, 2, 2, 2, 1, 12
187 },
188 .nr_l2streams = 16,
189 .l2_block_sz = {
190 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
191 2, 2, 2, 2, 2, 2
192 },
193 .insert_read_before_invalidate = false,
194 .l1_stream_id_reg_offset =
195 IPU6_MMU_L1_STREAM_ID_REG_OFFSET,
196 .l2_stream_id_reg_offset =
197 IPU6_MMU_L2_STREAM_ID_REG_OFFSET,
198 },
199 {
200 .offset = IPU6_PSYS_IOMMUI_OFFSET,
201 .info_bits = 0,
202 .nr_l1streams = 0,
203 .nr_l2streams = 0,
204 .insert_read_before_invalidate = false,
205 },
206 },
207 .dmem_offset = IPU6_PSYS_DMEM_OFFSET,
208 },
209};
210
211static const struct ipu6_buttress_ctrl isys_buttress_ctrl = {
212 .ratio = IPU6_IS_FREQ_CTL_DEFAULT_RATIO,
213 .qos_floor = IPU6_IS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO,
214 .freq_ctl = IPU6_BUTTRESS_REG_IS_FREQ_CTL,
215 .pwr_sts_shift = IPU6_BUTTRESS_PWR_STATE_IS_PWR_SHIFT,
216 .pwr_sts_mask = IPU6_BUTTRESS_PWR_STATE_IS_PWR_MASK,
217 .pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE,
218 .pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
219};
220
221static const struct ipu6_buttress_ctrl psys_buttress_ctrl = {
222 .ratio = IPU6_PS_FREQ_CTL_DEFAULT_RATIO,
223 .qos_floor = IPU6_PS_FREQ_CTL_DEFAULT_QOS_FLOOR_RATIO,
224 .freq_ctl = IPU6_BUTTRESS_REG_PS_FREQ_CTL,
225 .pwr_sts_shift = IPU6_BUTTRESS_PWR_STATE_PS_PWR_SHIFT,
226 .pwr_sts_mask = IPU6_BUTTRESS_PWR_STATE_PS_PWR_MASK,
227 .pwr_sts_on = IPU6_BUTTRESS_PWR_STATE_UP_DONE,
228 .pwr_sts_off = IPU6_BUTTRESS_PWR_STATE_DN_DONE,
229};
230
231static void
232ipu6_pkg_dir_configure_spc(struct ipu6_device *isp,
233 const struct ipu6_hw_variants *hw_variant,
234 int pkg_dir_idx, void __iomem *base,
235 u64 *pkg_dir, dma_addr_t pkg_dir_vied_address)
236{
237 struct ipu6_cell_program *prog;
238 void __iomem *spc_base;
239 u32 server_fw_addr;
240 dma_addr_t dma_addr;
241 u32 pg_offset;
242
243 server_fw_addr = lower_32_bits(*(pkg_dir + (pkg_dir_idx + 1) * 2));
244 if (pkg_dir_idx == IPU6_CPD_PKG_DIR_ISYS_SERVER_IDX)
245 dma_addr = sg_dma_address(isp->isys->fw_sgt.sgl);
246 else
247 dma_addr = sg_dma_address(isp->psys->fw_sgt.sgl);
248
249 pg_offset = server_fw_addr - dma_addr;
250 prog = (struct ipu6_cell_program *)((u64)isp->cpd_fw->data + pg_offset);
251 spc_base = base + prog->regs_addr;
252 if (spc_base != (base + hw_variant->spc_offset))
253 dev_warn(&isp->pdev->dev,
254 "SPC reg addr %p not matching value from CPD %p\n",
255 base + hw_variant->spc_offset, spc_base);
256 writel(server_fw_addr + prog->blob_offset +
257 prog->icache_source, spc_base + IPU6_PSYS_REG_SPC_ICACHE_BASE);
258 writel(IPU6_INFO_REQUEST_DESTINATION_IOSF,
259 spc_base + IPU6_REG_PSYS_INFO_SEG_0_CONFIG_ICACHE_MASTER);
260 writel(prog->start[1], spc_base + IPU6_PSYS_REG_SPC_START_PC);
261 writel(pkg_dir_vied_address, base + hw_variant->dmem_offset);
262}
263
264void ipu6_configure_spc(struct ipu6_device *isp,
265 const struct ipu6_hw_variants *hw_variant,
266 int pkg_dir_idx, void __iomem *base, u64 *pkg_dir,
267 dma_addr_t pkg_dir_dma_addr)
268{
269 void __iomem *dmem_base = base + hw_variant->dmem_offset;
270 void __iomem *spc_regs_base = base + hw_variant->spc_offset;
271 u32 val;
272
273 val = readl(spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL);
274 val |= IPU6_PSYS_SPC_STATUS_CTRL_ICACHE_INVALIDATE;
275 writel(val, spc_regs_base + IPU6_PSYS_REG_SPC_STATUS_CTRL);
276
277 if (isp->secure_mode)
278 writel(IPU6_PKG_DIR_IMR_OFFSET, dmem_base);
279 else
280 ipu6_pkg_dir_configure_spc(isp, hw_variant, pkg_dir_idx, base,
281 pkg_dir, pkg_dir_dma_addr);
282}
283EXPORT_SYMBOL_NS_GPL(ipu6_configure_spc, INTEL_IPU6);
284
285#define IPU6_ISYS_CSI2_NPORTS 4
286#define IPU6SE_ISYS_CSI2_NPORTS 4
287#define IPU6_TGL_ISYS_CSI2_NPORTS 8
54880795 288#define IPU6EP_MTL_ISYS_CSI2_NPORTS 6
25fedc02
BC		 289
290static void ipu6_internal_pdata_init(struct ipu6_device *isp)
291{
292 u8 hw_ver = isp->hw_ver;
293
294 isys_ipdata.num_parallel_streams = IPU6_ISYS_NUM_STREAMS;
295 isys_ipdata.sram_gran_shift = IPU6_SRAM_GRANULARITY_SHIFT;
296 isys_ipdata.sram_gran_size = IPU6_SRAM_GRANULARITY_SIZE;
297 isys_ipdata.max_sram_size = IPU6_MAX_SRAM_SIZE;
298 isys_ipdata.sensor_type_start = IPU6_FW_ISYS_SENSOR_TYPE_START;
299 isys_ipdata.sensor_type_end = IPU6_FW_ISYS_SENSOR_TYPE_END;
300 isys_ipdata.max_streams = IPU6_ISYS_NUM_STREAMS;
301 isys_ipdata.max_send_queues = IPU6_N_MAX_SEND_QUEUES;
302 isys_ipdata.max_sram_blocks = IPU6_NOF_SRAM_BLOCKS_MAX;
303 isys_ipdata.max_devq_size = IPU6_DEV_SEND_QUEUE_SIZE;
304 isys_ipdata.csi2.nports = IPU6_ISYS_CSI2_NPORTS;
305 isys_ipdata.csi2.irq_mask = IPU6_CSI_RX_ERROR_IRQ_MASK;
306 isys_ipdata.csi2.ctrl0_irq_edge = IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_EDGE;
307 isys_ipdata.csi2.ctrl0_irq_clear =
308 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_CLEAR;
309 isys_ipdata.csi2.ctrl0_irq_mask = IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_MASK;
310 isys_ipdata.csi2.ctrl0_irq_enable =
311 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_ENABLE;
312 isys_ipdata.csi2.ctrl0_irq_status =
313 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_STATUS;
314 isys_ipdata.csi2.ctrl0_irq_lnp =
315 IPU6_REG_ISYS_CSI_TOP_CTRL0_IRQ_LEVEL_NOT_PULSE;
316 isys_ipdata.enhanced_iwake = is_ipu6ep_mtl(hw_ver) || is_ipu6ep(hw_ver);
317 psys_ipdata.hw_variant.spc_offset = IPU6_PSYS_SPC_OFFSET;
318 isys_ipdata.csi2.fw_access_port_ofs = CSI_REG_HUB_FW_ACCESS_PORT_OFS;
319
320 if (is_ipu6ep(hw_ver)) {
321 isys_ipdata.ltr = IPU6EP_LTR_VALUE;
322 isys_ipdata.memopen_threshold = IPU6EP_MIN_MEMOPEN_TH;
323 }
324
325 if (is_ipu6_tgl(hw_ver))
326 isys_ipdata.csi2.nports = IPU6_TGL_ISYS_CSI2_NPORTS;
327
328 if (is_ipu6ep_mtl(hw_ver)) {
329 isys_ipdata.csi2.nports = IPU6EP_MTL_ISYS_CSI2_NPORTS;
330
331 isys_ipdata.csi2.ctrl0_irq_edge =
332 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_EDGE;
333 isys_ipdata.csi2.ctrl0_irq_clear =
334 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_CLEAR;
335 isys_ipdata.csi2.ctrl0_irq_mask =
336 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_MASK;
337 isys_ipdata.csi2.ctrl0_irq_enable =
338 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_ENABLE;
339 isys_ipdata.csi2.ctrl0_irq_lnp =
340 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_LEVEL_NOT_PULSE;
341 isys_ipdata.csi2.ctrl0_irq_status =
342 IPU6V6_REG_ISYS_CSI_TOP_CTRL0_IRQ_STATUS;
343 isys_ipdata.csi2.fw_access_port_ofs =
344 CSI_REG_HUB_FW_ACCESS_PORT_V6OFS;
345 isys_ipdata.ltr = IPU6EP_MTL_LTR_VALUE;
346 isys_ipdata.memopen_threshold = IPU6EP_MTL_MIN_MEMOPEN_TH;
347 }
348
349 if (is_ipu6se(hw_ver)) {
350 isys_ipdata.csi2.nports = IPU6SE_ISYS_CSI2_NPORTS;
351 isys_ipdata.csi2.irq_mask = IPU6SE_CSI_RX_ERROR_IRQ_MASK;
352 isys_ipdata.num_parallel_streams = IPU6SE_ISYS_NUM_STREAMS;
353 isys_ipdata.sram_gran_shift = IPU6SE_SRAM_GRANULARITY_SHIFT;
354 isys_ipdata.sram_gran_size = IPU6SE_SRAM_GRANULARITY_SIZE;
355 isys_ipdata.max_sram_size = IPU6SE_MAX_SRAM_SIZE;
356 isys_ipdata.sensor_type_start =
357 IPU6SE_FW_ISYS_SENSOR_TYPE_START;
358 isys_ipdata.sensor_type_end = IPU6SE_FW_ISYS_SENSOR_TYPE_END;
359 isys_ipdata.max_streams = IPU6SE_ISYS_NUM_STREAMS;
360 isys_ipdata.max_send_queues = IPU6SE_N_MAX_SEND_QUEUES;
361 isys_ipdata.max_sram_blocks = IPU6SE_NOF_SRAM_BLOCKS_MAX;
362 isys_ipdata.max_devq_size = IPU6SE_DEV_SEND_QUEUE_SIZE;
363 psys_ipdata.hw_variant.spc_offset = IPU6SE_PSYS_SPC_OFFSET;
364 }
365}
366
25fedc02
BC		 367static struct ipu6_bus_device *
368ipu6_isys_init(struct pci_dev *pdev, struct device *parent,
369 struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
370 const struct ipu6_isys_internal_pdata *ipdata)
371{
372 struct device *dev = &pdev->dev;
25fedc02
BC		 373 struct ipu6_bus_device *isys_adev;
374 struct ipu6_isys_pdata *pdata;
375 int ret;
376
f50c4ca0 377 ret = ipu_bridge_init(dev, ipu_bridge_parse_ssdb);
25fedc02 378 if (ret) {
f50c4ca0
BC		 379 dev_err_probe(dev, ret, "IPU6 bridge init failed\n");
380 return ERR_PTR(ret);
25fedc02
BC		 381 }
382
383 pdata = kzalloc(sizeof(*pdata), GFP_KERNEL);
384 if (!pdata)
385 return ERR_PTR(-ENOMEM);
386
387 pdata->base = base;
388 pdata->ipdata = ipdata;
389
390 isys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl,
391 IPU6_ISYS_NAME);
392 if (IS_ERR(isys_adev)) {
393 dev_err_probe(dev, PTR_ERR(isys_adev),
394 "ipu6_bus_initialize_device isys failed\n");
395 kfree(pdata);
396 return ERR_CAST(isys_adev);
397 }
398
399 isys_adev->mmu = ipu6_mmu_init(dev, base, ISYS_MMID,
400 &ipdata->hw_variant);
401 if (IS_ERR(isys_adev->mmu)) {
402 dev_err_probe(dev, PTR_ERR(isys_adev->mmu),
403 "ipu6_mmu_init(isys_adev->mmu) failed\n");
404 put_device(&isys_adev->auxdev.dev);
405 kfree(pdata);
406 return ERR_CAST(isys_adev->mmu);
407 }
408
409 isys_adev->mmu->dev = &isys_adev->auxdev.dev;
410
411 ret = ipu6_bus_add_device(isys_adev);
412 if (ret) {
413 kfree(pdata);
414 return ERR_PTR(ret);
415 }
416
417 return isys_adev;
418}
419
420static struct ipu6_bus_device *
421ipu6_psys_init(struct pci_dev *pdev, struct device *parent,
422 struct ipu6_buttress_ctrl *ctrl, void __iomem *base,
423 const struct ipu6_psys_internal_pdata *ipdata)
424{
425 struct ipu6_bus_device *psys_adev;
426 struct ipu6_psys_pdata *pdata;
427 int ret;
428
429 pdata = kzalloc(sizeof(*pdata), GFP_KERNEL);
430 if (!pdata)
431 return ERR_PTR(-ENOMEM);
432
433 pdata->base = base;
434 pdata->ipdata = ipdata;
435
436 psys_adev = ipu6_bus_initialize_device(pdev, parent, pdata, ctrl,
437 IPU6_PSYS_NAME);
438 if (IS_ERR(psys_adev)) {
439 dev_err_probe(&pdev->dev, PTR_ERR(psys_adev),
440 "ipu6_bus_initialize_device psys failed\n");
441 kfree(pdata);
442 return ERR_CAST(psys_adev);
443 }
444
445 psys_adev->mmu = ipu6_mmu_init(&pdev->dev, base, PSYS_MMID,
446 &ipdata->hw_variant);
447 if (IS_ERR(psys_adev->mmu)) {
448 dev_err_probe(&pdev->dev, PTR_ERR(psys_adev->mmu),
449 "ipu6_mmu_init(psys_adev->mmu) failed\n");
450 put_device(&psys_adev->auxdev.dev);
451 kfree(pdata);
452 return ERR_CAST(psys_adev->mmu);
453 }
454
455 psys_adev->mmu->dev = &psys_adev->auxdev.dev;
456
457 ret = ipu6_bus_add_device(psys_adev);
458 if (ret) {
459 kfree(pdata);
460 return ERR_PTR(ret);
461 }
462
463 return psys_adev;
464}
465
466static int ipu6_pci_config_setup(struct pci_dev *dev, u8 hw_ver)
467{
468 int ret;
469
470 /* disable IPU6 PCI ATS on mtl ES2 */
471 if (is_ipu6ep_mtl(hw_ver) && boot_cpu_data.x86_stepping == 0x2 &&
472 pci_ats_supported(dev))
473 pci_disable_ats(dev);
474
475 /* No PCI msi capability for IPU6EP */
476 if (is_ipu6ep(hw_ver) || is_ipu6ep_mtl(hw_ver)) {
477 /* likely do nothing as msi not enabled by default */
478 pci_disable_msi(dev);
479 return 0;
480 }
481
482 ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_MSI);
483 if (ret < 0)
484 return dev_err_probe(&dev->dev, ret, "Request msi failed");
485
486 return 0;
487}
488
489static void ipu6_configure_vc_mechanism(struct ipu6_device *isp)
490{
491 u32 val = readl(isp->base + BUTTRESS_REG_BTRS_CTRL);
492
493 if (IPU6_BTRS_ARB_STALL_MODE_VC0 == IPU6_BTRS_ARB_MODE_TYPE_STALL)
494 val |= BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC0;
495 else
496 val &= ~BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC0;
497
498 if (IPU6_BTRS_ARB_STALL_MODE_VC1 == IPU6_BTRS_ARB_MODE_TYPE_STALL)
499 val |= BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC1;
500 else
501 val &= ~BUTTRESS_REG_BTRS_CTRL_STALL_MODE_VC1;
502
503 writel(val, isp->base + BUTTRESS_REG_BTRS_CTRL);
504}
505
25fedc02
BC		 506static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
507{
508 struct ipu6_buttress_ctrl *isys_ctrl = NULL, *psys_ctrl = NULL;
509 struct device *dev = &pdev->dev;
510 void __iomem *isys_base = NULL;
511 void __iomem *psys_base = NULL;
512 struct ipu6_device *isp;
513 phys_addr_t phys;
514 u32 val, version, sku_id;
515 int ret;
516
517 isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL);
518 if (!isp)
519 return -ENOMEM;
520
521 isp->pdev = pdev;
522 INIT_LIST_HEAD(&isp->devices);
523
524 ret = pcim_enable_device(pdev);
525 if (ret)
526 return dev_err_probe(dev, ret, "Enable PCI device failed\n");
527
528 phys = pci_resource_start(pdev, IPU6_PCI_BAR);
529 dev_dbg(dev, "IPU6 PCI bar[%u] = %pa\n", IPU6_PCI_BAR, &phys);
530
531 ret = pcim_iomap_regions(pdev, 1 << IPU6_PCI_BAR, pci_name(pdev));
532 if (ret)
48259b90 533 return dev_err_probe(dev, ret, "Failed to I/O mem remapping\n");
25fedc02
BC		 534
535 isp->base = pcim_iomap_table(pdev)[IPU6_PCI_BAR];
536 pci_set_drvdata(pdev, isp);
537 pci_set_master(pdev);
538
539 isp->cpd_metadata_cmpnt_size = sizeof(struct ipu6_cpd_metadata_cmpnt);
540 switch (id->device) {
541 case PCI_DEVICE_ID_INTEL_IPU6:
542 isp->hw_ver = IPU6_VER_6;
543 isp->cpd_fw_name = IPU6_FIRMWARE_NAME;
544 break;
545 case PCI_DEVICE_ID_INTEL_IPU6SE:
546 isp->hw_ver = IPU6_VER_6SE;
547 isp->cpd_fw_name = IPU6SE_FIRMWARE_NAME;
548 isp->cpd_metadata_cmpnt_size =
549 sizeof(struct ipu6se_cpd_metadata_cmpnt);
550 break;
551 case PCI_DEVICE_ID_INTEL_IPU6EP_ADLP:
552 case PCI_DEVICE_ID_INTEL_IPU6EP_RPLP:
553 isp->hw_ver = IPU6_VER_6EP;
554 isp->cpd_fw_name = IPU6EP_FIRMWARE_NAME;
555 break;
556 case PCI_DEVICE_ID_INTEL_IPU6EP_ADLN:
557 isp->hw_ver = IPU6_VER_6EP;
558 isp->cpd_fw_name = IPU6EPADLN_FIRMWARE_NAME;
559 break;
560 case PCI_DEVICE_ID_INTEL_IPU6EP_MTL:
561 isp->hw_ver = IPU6_VER_6EP_MTL;
562 isp->cpd_fw_name = IPU6EPMTL_FIRMWARE_NAME;
563 break;
564 default:
565 return dev_err_probe(dev, -ENODEV,
566 "Unsupported IPU6 device %x\n",
567 id->device);
568 }
569
570 ipu6_internal_pdata_init(isp);
571
572 isys_base = isp->base + isys_ipdata.hw_variant.offset;
573 psys_base = isp->base + psys_ipdata.hw_variant.offset;
574
575 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(39));
576 if (ret)
577 return dev_err_probe(dev, ret, "Failed to set DMA mask\n");
578
579 ret = dma_set_max_seg_size(dev, UINT_MAX);
580 if (ret)
581 return dev_err_probe(dev, ret, "Failed to set max_seg_size\n");
582
583 ret = ipu6_pci_config_setup(pdev, isp->hw_ver);
584 if (ret)
585 return ret;
586
587 ret = ipu6_buttress_init(isp);
588 if (ret)
589 return ret;
590
8a09bb1b 591 ret = request_firmware(&isp->cpd_fw, isp->cpd_fw_name, dev);
25fedc02
BC		 592 if (ret) {
593 dev_err_probe(&isp->pdev->dev, ret,
594 "Requesting signed firmware %s failed\n",
595 isp->cpd_fw_name);
596 goto buttress_exit;
597 }
598
599 ret = ipu6_cpd_validate_cpd_file(isp, isp->cpd_fw->data,
600 isp->cpd_fw->size);
601 if (ret) {
602 dev_err_probe(&isp->pdev->dev, ret,
603 "Failed to validate cpd\n");
604 goto out_ipu6_bus_del_devices;
605 }
606
607 isys_ctrl = devm_kmemdup(dev, &isys_buttress_ctrl,
608 sizeof(isys_buttress_ctrl), GFP_KERNEL);
609 if (!isys_ctrl) {
610 ret = -ENOMEM;
611 goto out_ipu6_bus_del_devices;
612 }
613
614 isp->isys = ipu6_isys_init(pdev, dev, isys_ctrl, isys_base,
615 &isys_ipdata);
616 if (IS_ERR(isp->isys)) {
617 ret = PTR_ERR(isp->isys);
618 goto out_ipu6_bus_del_devices;
619 }
620
621 psys_ctrl = devm_kmemdup(dev, &psys_buttress_ctrl,
622 sizeof(psys_buttress_ctrl), GFP_KERNEL);
623 if (!psys_ctrl) {
624 ret = -ENOMEM;
625 goto out_ipu6_bus_del_devices;
626 }
627
628 isp->psys = ipu6_psys_init(pdev, &isp->isys->auxdev.dev, psys_ctrl,
629 psys_base, &psys_ipdata);
630 if (IS_ERR(isp->psys)) {
631 ret = PTR_ERR(isp->psys);
632 goto out_ipu6_bus_del_devices;
633 }
634
635 ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);
636 if (ret < 0)
637 goto out_ipu6_bus_del_devices;
638
639 ret = ipu6_mmu_hw_init(isp->psys->mmu);
640 if (ret) {
641 dev_err_probe(&isp->pdev->dev, ret,
642 "Failed to set MMU hardware\n");
643 goto out_ipu6_bus_del_devices;
644 }
645
646 ret = ipu6_buttress_map_fw_image(isp->psys, isp->cpd_fw,
647 &isp->psys->fw_sgt);
648 if (ret) {
649 dev_err_probe(&isp->pdev->dev, ret, "failed to map fw image\n");
650 goto out_ipu6_bus_del_devices;
651 }
652
653 ret = ipu6_cpd_create_pkg_dir(isp->psys, isp->cpd_fw->data);
654 if (ret) {
655 dev_err_probe(&isp->pdev->dev, ret,
656 "failed to create pkg dir\n");
657 goto out_ipu6_bus_del_devices;
658 }
659
660 ret = devm_request_threaded_irq(dev, pdev->irq, ipu6_buttress_isr,
661 ipu6_buttress_isr_threaded,
662 IRQF_SHARED, IPU6_NAME, isp);
663 if (ret) {
664 dev_err_probe(dev, ret, "Requesting irq failed\n");
665 goto out_ipu6_bus_del_devices;
666 }
667
668 ret = ipu6_buttress_authenticate(isp);
669 if (ret) {
670 dev_err_probe(&isp->pdev->dev, ret,
671 "FW authentication failed\n");
672 goto out_free_irq;
673 }
674
675 ipu6_mmu_hw_cleanup(isp->psys->mmu);
676 pm_runtime_put(&isp->psys->auxdev.dev);
677
678 /* Configure the arbitration mechanisms for VC requests */
679 ipu6_configure_vc_mechanism(isp);
680
681 val = readl(isp->base + BUTTRESS_REG_SKU);
682 sku_id = FIELD_GET(GENMASK(6, 4), val);
683 version = FIELD_GET(GENMASK(3, 0), val);
684 dev_info(dev, "IPU%u-v%u[%x] hardware version %d\n", version, sku_id,
685 pdev->device, isp->hw_ver);
686
687 pm_runtime_put_noidle(dev);
688 pm_runtime_allow(dev);
689
690 isp->bus_ready_to_probe = true;
691
692 return 0;
693
694out_free_irq:
695 devm_free_irq(dev, pdev->irq, isp);
696out_ipu6_bus_del_devices:
697 if (isp->psys) {
698 ipu6_cpd_free_pkg_dir(isp->psys);
699 ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);
700 }
701 if (!IS_ERR_OR_NULL(isp->psys) && !IS_ERR_OR_NULL(isp->psys->mmu))
702 ipu6_mmu_cleanup(isp->psys->mmu);
703 if (!IS_ERR_OR_NULL(isp->isys) && !IS_ERR_OR_NULL(isp->isys->mmu))
704 ipu6_mmu_cleanup(isp->isys->mmu);
705 ipu6_bus_del_devices(pdev);
706 release_firmware(isp->cpd_fw);
707buttress_exit:
708 ipu6_buttress_exit(isp);
709
710 return ret;
711}
712
713static void ipu6_pci_remove(struct pci_dev *pdev)
714{
715 struct ipu6_device *isp = pci_get_drvdata(pdev);
716 struct ipu6_mmu *isys_mmu = isp->isys->mmu;
717 struct ipu6_mmu *psys_mmu = isp->psys->mmu;
718
719 devm_free_irq(&pdev->dev, pdev->irq, isp);
720 ipu6_cpd_free_pkg_dir(isp->psys);
721
722 ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt);
723 ipu6_buttress_exit(isp);
724
725 ipu6_bus_del_devices(pdev);
726
727 pm_runtime_forbid(&pdev->dev);
728 pm_runtime_get_noresume(&pdev->dev);
729
25fedc02
BC		 730 release_firmware(isp->cpd_fw);
731
732 ipu6_mmu_cleanup(psys_mmu);
733 ipu6_mmu_cleanup(isys_mmu);
734}
735
736static void ipu6_pci_reset_prepare(struct pci_dev *pdev)
737{
738 struct ipu6_device *isp = pci_get_drvdata(pdev);
739
740 pm_runtime_forbid(&isp->pdev->dev);
741}
742
743static void ipu6_pci_reset_done(struct pci_dev *pdev)
744{
745 struct ipu6_device *isp = pci_get_drvdata(pdev);
746
747 ipu6_buttress_restore(isp);
748 if (isp->secure_mode)
749 ipu6_buttress_reset_authentication(isp);
750
751 isp->need_ipc_reset = true;
752 pm_runtime_allow(&isp->pdev->dev);
753}
754
755/*
756 * PCI base driver code requires driver to provide these to enable
757 * PCI device level PM state transitions (D0<->D3)
758 */
759static int ipu6_suspend(struct device *dev)
760{
761 return 0;
762}
763
764static int ipu6_resume(struct device *dev)
765{
766 struct pci_dev *pdev = to_pci_dev(dev);
767 struct ipu6_device *isp = pci_get_drvdata(pdev);
768 struct ipu6_buttress *b = &isp->buttress;
769 int ret;
770
771 /* Configure the arbitration mechanisms for VC requests */
772 ipu6_configure_vc_mechanism(isp);
773
774 isp->secure_mode = ipu6_buttress_get_secure_mode(isp);
775 dev_info(dev, "IPU6 in %s mode\n",
776 isp->secure_mode ? "secure" : "non-secure");
777
778 ipu6_buttress_restore(isp);
779
780 ret = ipu6_buttress_ipc_reset(isp, &b->cse);
781 if (ret)
782 dev_err(&isp->pdev->dev, "IPC reset protocol failed!\n");
783
784 ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev);
785 if (ret < 0) {
786 dev_err(&isp->psys->auxdev.dev, "Failed to get runtime PM\n");
787 return 0;
788 }
789
790 ret = ipu6_buttress_authenticate(isp);
791 if (ret)
792 dev_err(&isp->pdev->dev, "FW authentication failed(%d)\n", ret);
793
794 pm_runtime_put(&isp->psys->auxdev.dev);
795
796 return 0;
797}
798
799static int ipu6_runtime_resume(struct device *dev)
800{
801 struct pci_dev *pdev = to_pci_dev(dev);
802 struct ipu6_device *isp = pci_get_drvdata(pdev);
803 int ret;
804
805 ipu6_configure_vc_mechanism(isp);
806 ipu6_buttress_restore(isp);
807
808 if (isp->need_ipc_reset) {
809 struct ipu6_buttress *b = &isp->buttress;
810
811 isp->need_ipc_reset = false;
812 ret = ipu6_buttress_ipc_reset(isp, &b->cse);
813 if (ret)
814 dev_err(&isp->pdev->dev, "IPC reset protocol failed\n");
815 }
816
817 return 0;
818}
819
820static const struct dev_pm_ops ipu6_pm_ops = {
01708813
RR		 821 SYSTEM_SLEEP_PM_OPS(&ipu6_suspend, &ipu6_resume)
822 RUNTIME_PM_OPS(&ipu6_suspend, &ipu6_runtime_resume, NULL)
25fedc02
BC		 823};
824
825MODULE_DEVICE_TABLE(pci, ipu6_pci_tbl);
826
827static const struct pci_error_handlers pci_err_handlers = {
828 .reset_prepare = ipu6_pci_reset_prepare,
829 .reset_done = ipu6_pci_reset_done,
830};
831
832static struct pci_driver ipu6_pci_driver = {
833 .name = IPU6_NAME,
834 .id_table = ipu6_pci_tbl,
835 .probe = ipu6_pci_probe,
836 .remove = ipu6_pci_remove,
837 .driver = {
838 .pm = pm_ptr(&ipu6_pm_ops),
839 },
840 .err_handler = &pci_err_handlers,
841};
842
843module_pci_driver(ipu6_pci_driver);
844
845MODULE_IMPORT_NS(INTEL_IPU_BRIDGE);
846MODULE_AUTHOR("Sakari Ailus <sakari.ailus@linux.intel.com>");
847MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>");
848MODULE_AUTHOR("Bingbu Cao <bingbu.cao@intel.com>");
849MODULE_AUTHOR("Qingwu Zhang <qingwu.zhang@intel.com>");
850MODULE_AUTHOR("Yunliang Ding <yunliang.ding@intel.com>");
851MODULE_AUTHOR("Hongju Wang <hongju.wang@intel.com>");
852MODULE_LICENSE("GPL");
853MODULE_DESCRIPTION("Intel IPU6 PCI driver");
Linux 6.x block layer and io_uring tree(s)