projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dmaengine: usb-dmac: Avoid format-overflow warning
[linux-block.git] / drivers / dma / hisi_dma.c
CommitLineData
e9f08b65 1// SPDX-License-Identifier: GPL-2.0-only
fd5273fa
JH		 2/* Copyright(c) 2019-2022 HiSilicon Limited. */
3
e9f08b65
ZW		 4#include <linux/bitfield.h>
5#include <linux/dmaengine.h>
6#include <linux/init.h>
7#include <linux/iopoll.h>
8#include <linux/module.h>
9#include <linux/pci.h>
10#include <linux/spinlock.h>
11#include "virt-dma.h"
12
fd5273fa
JH		 13/* HiSilicon DMA register common field define */
14#define HISI_DMA_Q_SQ_BASE_L 0x0
15#define HISI_DMA_Q_SQ_BASE_H 0x4
16#define HISI_DMA_Q_SQ_DEPTH 0x8
17#define HISI_DMA_Q_SQ_TAIL_PTR 0xc
18#define HISI_DMA_Q_CQ_BASE_L 0x10
19#define HISI_DMA_Q_CQ_BASE_H 0x14
20#define HISI_DMA_Q_CQ_DEPTH 0x18
21#define HISI_DMA_Q_CQ_HEAD_PTR 0x1c
22#define HISI_DMA_Q_CTRL0 0x20
23#define HISI_DMA_Q_CTRL0_QUEUE_EN BIT(0)
24#define HISI_DMA_Q_CTRL0_QUEUE_PAUSE BIT(4)
25#define HISI_DMA_Q_CTRL1 0x24
26#define HISI_DMA_Q_CTRL1_QUEUE_RESET BIT(0)
27#define HISI_DMA_Q_FSM_STS 0x30
28#define HISI_DMA_Q_FSM_STS_MASK GENMASK(3, 0)
29#define HISI_DMA_Q_ERR_INT_NUM0 0x84
30#define HISI_DMA_Q_ERR_INT_NUM1 0x88
31#define HISI_DMA_Q_ERR_INT_NUM2 0x8c
32
33/* HiSilicon IP08 DMA register and field define */
34#define HISI_DMA_HIP08_MODE 0x217C
35#define HISI_DMA_HIP08_Q_BASE 0x0
36#define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN BIT(2)
37#define HISI_DMA_HIP08_Q_INT_STS 0x40
38#define HISI_DMA_HIP08_Q_INT_MSK 0x44
39#define HISI_DMA_HIP08_Q_INT_STS_MASK GENMASK(14, 0)
40#define HISI_DMA_HIP08_Q_ERR_INT_NUM3 0x90
41#define HISI_DMA_HIP08_Q_ERR_INT_NUM4 0x94
42#define HISI_DMA_HIP08_Q_ERR_INT_NUM5 0x98
43#define HISI_DMA_HIP08_Q_ERR_INT_NUM6 0x48
44#define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT BIT(24)
45
46/* HiSilicon IP09 DMA register and field define */
47#define HISI_DMA_HIP09_DMA_FLR_DISABLE 0xA00
48#define HISI_DMA_HIP09_DMA_FLR_DISABLE_B BIT(0)
49#define HISI_DMA_HIP09_Q_BASE 0x2000
50#define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN GENMASK(31, 28)
51#define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT BIT(26)
52#define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT BIT(27)
53#define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE BIT(2)
54#define HISI_DMA_HIP09_Q_INT_STS 0x40
55#define HISI_DMA_HIP09_Q_INT_MSK 0x44
56#define HISI_DMA_HIP09_Q_INT_STS_MASK 0x1
57#define HISI_DMA_HIP09_Q_ERR_INT_STS 0x48
58#define HISI_DMA_HIP09_Q_ERR_INT_MSK 0x4C
59#define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK GENMASK(18, 1)
60#define HISI_DMA_HIP09_PORT_CFG_REG(port_id) (0x800 + \
61 (port_id) * 0x20)
62#define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B BIT(16)
63
64#define HISI_DMA_HIP09_MAX_PORT_NUM 16
65
66#define HISI_DMA_HIP08_MSI_NUM 32
67#define HISI_DMA_HIP08_CHAN_NUM 30
68#define HISI_DMA_HIP09_MSI_NUM 4
69#define HISI_DMA_HIP09_CHAN_NUM 4
70#define HISI_DMA_REVISION_HIP08B 0x21
71#define HISI_DMA_REVISION_HIP09A 0x30
72
73#define HISI_DMA_Q_OFFSET 0x100
74#define HISI_DMA_Q_DEPTH_VAL 1024
75
76#define PCI_BAR_2 2
77
78#define HISI_DMA_POLL_Q_STS_DELAY_US 10
79#define HISI_DMA_POLL_Q_STS_TIME_OUT_US 1000
80
5dda7a62
JH		 81#define HISI_DMA_MAX_DIR_NAME_LEN 128
82
fd5273fa
JH		 83/*
84 * The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they
85 * have the same pci device id but different pci revision.
86 * Unfortunately, they have different register layouts, so two layout
87 * enumerations are defined.
88 */
89enum hisi_dma_reg_layout {
90 HISI_DMA_REG_LAYOUT_INVALID = 0,
91 HISI_DMA_REG_LAYOUT_HIP08,
92 HISI_DMA_REG_LAYOUT_HIP09
93};
4aa69cf7 94
e9f08b65
ZW		 95enum hisi_dma_mode {
96 EP = 0,
97 RC,
98};
99
100enum hisi_dma_chan_status {
101 DISABLE = -1,
102 IDLE = 0,
103 RUN,
104 CPL,
105 PAUSE,
106 HALT,
107 ABORT,
108 WAIT,
109 BUFFCLR,
110};
111
112struct hisi_dma_sqe {
113 __le32 dw0;
114#define OPCODE_MASK GENMASK(3, 0)
115#define OPCODE_SMALL_PACKAGE 0x1
116#define OPCODE_M2M 0x4
117#define LOCAL_IRQ_EN BIT(8)
118#define ATTR_SRC_MASK GENMASK(14, 12)
119 __le32 dw1;
120 __le32 dw2;
121#define ATTR_DST_MASK GENMASK(26, 24)
122 __le32 length;
123 __le64 src_addr;
124 __le64 dst_addr;
125};
126
127struct hisi_dma_cqe {
128 __le32 rsv0;
129 __le32 rsv1;
130 __le16 sq_head;
131 __le16 rsv2;
132 __le16 rsv3;
133 __le16 w0;
134#define STATUS_MASK GENMASK(15, 1)
135#define STATUS_SUCC 0x0
136#define VALID_BIT BIT(0)
137};
138
139struct hisi_dma_desc {
140 struct virt_dma_desc vd;
141 struct hisi_dma_sqe sqe;
142};
143
144struct hisi_dma_chan {
145 struct virt_dma_chan vc;
146 struct hisi_dma_dev *hdma_dev;
147 struct hisi_dma_sqe *sq;
148 struct hisi_dma_cqe *cq;
149 dma_addr_t sq_dma;
150 dma_addr_t cq_dma;
151 u32 sq_tail;
152 u32 cq_head;
153 u32 qp_num;
154 enum hisi_dma_chan_status status;
155 struct hisi_dma_desc *desc;
156};
157
158struct hisi_dma_dev {
159 struct pci_dev *pdev;
160 void __iomem *base;
161 struct dma_device dma_dev;
162 u32 chan_num;
163 u32 chan_depth;
fd5273fa
JH		 164 enum hisi_dma_reg_layout reg_layout;
165 void __iomem *queue_base; /* queue region start of register */
7d4b8218 166 struct hisi_dma_chan chan[] __counted_by(chan_num);
e9f08b65
ZW		 167};
168
5dda7a62
JH		 169#ifdef CONFIG_DEBUG_FS
170
171static const struct debugfs_reg32 hisi_dma_comm_chan_regs[] = {
172 {"DMA_QUEUE_SQ_DEPTH ", 0x0008ull},
173 {"DMA_QUEUE_SQ_TAIL_PTR ", 0x000Cull},
174 {"DMA_QUEUE_CQ_DEPTH ", 0x0018ull},
175 {"DMA_QUEUE_CQ_HEAD_PTR ", 0x001Cull},
176 {"DMA_QUEUE_CTRL0 ", 0x0020ull},
177 {"DMA_QUEUE_CTRL1 ", 0x0024ull},
178 {"DMA_QUEUE_FSM_STS ", 0x0030ull},
179 {"DMA_QUEUE_SQ_STS ", 0x0034ull},
180 {"DMA_QUEUE_CQ_TAIL_PTR ", 0x003Cull},
181 {"DMA_QUEUE_INT_STS ", 0x0040ull},
182 {"DMA_QUEUE_INT_MSK ", 0x0044ull},
183 {"DMA_QUEUE_INT_RO ", 0x006Cull},
184};
185
186static const struct debugfs_reg32 hisi_dma_hip08_chan_regs[] = {
187 {"DMA_QUEUE_BYTE_CNT ", 0x0038ull},
188 {"DMA_ERR_INT_NUM6 ", 0x0048ull},
189 {"DMA_QUEUE_DESP0 ", 0x0050ull},
190 {"DMA_QUEUE_DESP1 ", 0x0054ull},
191 {"DMA_QUEUE_DESP2 ", 0x0058ull},
192 {"DMA_QUEUE_DESP3 ", 0x005Cull},
193 {"DMA_QUEUE_DESP4 ", 0x0074ull},
194 {"DMA_QUEUE_DESP5 ", 0x0078ull},
195 {"DMA_QUEUE_DESP6 ", 0x007Cull},
196 {"DMA_QUEUE_DESP7 ", 0x0080ull},
197 {"DMA_ERR_INT_NUM0 ", 0x0084ull},
198 {"DMA_ERR_INT_NUM1 ", 0x0088ull},
199 {"DMA_ERR_INT_NUM2 ", 0x008Cull},
200 {"DMA_ERR_INT_NUM3 ", 0x0090ull},
201 {"DMA_ERR_INT_NUM4 ", 0x0094ull},
202 {"DMA_ERR_INT_NUM5 ", 0x0098ull},
203 {"DMA_QUEUE_SQ_STS2 ", 0x00A4ull},
204};
205
206static const struct debugfs_reg32 hisi_dma_hip09_chan_regs[] = {
207 {"DMA_QUEUE_ERR_INT_STS ", 0x0048ull},
208 {"DMA_QUEUE_ERR_INT_MSK ", 0x004Cull},
209 {"DFX_SQ_READ_ERR_PTR ", 0x0068ull},
210 {"DFX_DMA_ERR_INT_NUM0 ", 0x0084ull},
211 {"DFX_DMA_ERR_INT_NUM1 ", 0x0088ull},
212 {"DFX_DMA_ERR_INT_NUM2 ", 0x008Cull},
213 {"DFX_DMA_QUEUE_SQ_STS2 ", 0x00A4ull},
214};
215
216static const struct debugfs_reg32 hisi_dma_hip08_comm_regs[] = {
217 {"DMA_ECC_ERR_ADDR ", 0x2004ull},
218 {"DMA_ECC_ECC_CNT ", 0x2014ull},
219 {"COMMON_AND_CH_ERR_STS ", 0x2030ull},
220 {"LOCAL_CPL_ID_STS_0 ", 0x20E0ull},
221 {"LOCAL_CPL_ID_STS_1 ", 0x20E4ull},
222 {"LOCAL_CPL_ID_STS_2 ", 0x20E8ull},
223 {"LOCAL_CPL_ID_STS_3 ", 0x20ECull},
224 {"LOCAL_TLP_NUM ", 0x2158ull},
225 {"SQCQ_TLP_NUM ", 0x2164ull},
226 {"CPL_NUM ", 0x2168ull},
227 {"INF_BACK_PRESS_STS ", 0x2170ull},
228 {"DMA_CH_RAS_LEVEL ", 0x2184ull},
229 {"DMA_CM_RAS_LEVEL ", 0x2188ull},
230 {"DMA_CH_ERR_STS ", 0x2190ull},
231 {"DMA_CH_DONE_STS ", 0x2194ull},
232 {"DMA_SQ_TAG_STS_0 ", 0x21A0ull},
233 {"DMA_SQ_TAG_STS_1 ", 0x21A4ull},
234 {"DMA_SQ_TAG_STS_2 ", 0x21A8ull},
235 {"DMA_SQ_TAG_STS_3 ", 0x21ACull},
236 {"LOCAL_P_ID_STS_0 ", 0x21B0ull},
237 {"LOCAL_P_ID_STS_1 ", 0x21B4ull},
238 {"LOCAL_P_ID_STS_2 ", 0x21B8ull},
239 {"LOCAL_P_ID_STS_3 ", 0x21BCull},
240 {"DMA_PREBUFF_INFO_0 ", 0x2200ull},
241 {"DMA_CM_TABLE_INFO_0 ", 0x2220ull},
242 {"DMA_CM_CE_RO ", 0x2244ull},
243 {"DMA_CM_NFE_RO ", 0x2248ull},
244 {"DMA_CM_FE_RO ", 0x224Cull},
245};
246
247static const struct debugfs_reg32 hisi_dma_hip09_comm_regs[] = {
248 {"COMMON_AND_CH_ERR_STS ", 0x0030ull},
249 {"DMA_PORT_IDLE_STS ", 0x0150ull},
250 {"DMA_CH_RAS_LEVEL ", 0x0184ull},
251 {"DMA_CM_RAS_LEVEL ", 0x0188ull},
252 {"DMA_CM_CE_RO ", 0x0244ull},
253 {"DMA_CM_NFE_RO ", 0x0248ull},
254 {"DMA_CM_FE_RO ", 0x024Cull},
255 {"DFX_INF_BACK_PRESS_STS0 ", 0x1A40ull},
256 {"DFX_INF_BACK_PRESS_STS1 ", 0x1A44ull},
257 {"DFX_INF_BACK_PRESS_STS2 ", 0x1A48ull},
258 {"DFX_DMA_WRR_DISABLE ", 0x1A4Cull},
259 {"DFX_PA_REQ_TLP_NUM ", 0x1C00ull},
260 {"DFX_PA_BACK_TLP_NUM ", 0x1C04ull},
261 {"DFX_PA_RETRY_TLP_NUM ", 0x1C08ull},
262 {"DFX_LOCAL_NP_TLP_NUM ", 0x1C0Cull},
263 {"DFX_LOCAL_CPL_HEAD_TLP_NUM ", 0x1C10ull},
264 {"DFX_LOCAL_CPL_DATA_TLP_NUM ", 0x1C14ull},
265 {"DFX_LOCAL_CPL_EXT_DATA_TLP_NUM ", 0x1C18ull},
266 {"DFX_LOCAL_P_HEAD_TLP_NUM ", 0x1C1Cull},
267 {"DFX_LOCAL_P_ACK_TLP_NUM ", 0x1C20ull},
268 {"DFX_BUF_ALOC_PORT_REQ_NUM ", 0x1C24ull},
269 {"DFX_BUF_ALOC_PORT_RESULT_NUM ", 0x1C28ull},
270 {"DFX_BUF_FAIL_SIZE_NUM ", 0x1C2Cull},
271 {"DFX_BUF_ALOC_SIZE_NUM ", 0x1C30ull},
272 {"DFX_BUF_NP_RELEASE_SIZE_NUM ", 0x1C34ull},
273 {"DFX_BUF_P_RELEASE_SIZE_NUM ", 0x1C38ull},
274 {"DFX_BUF_PORT_RELEASE_SIZE_NUM ", 0x1C3Cull},
275 {"DFX_DMA_PREBUF_MEM0_ECC_ERR_ADDR ", 0x1CA8ull},
276 {"DFX_DMA_PREBUF_MEM0_ECC_CNT ", 0x1CACull},
277 {"DFX_DMA_LOC_NP_OSTB_ECC_ERR_ADDR ", 0x1CB0ull},
278 {"DFX_DMA_LOC_NP_OSTB_ECC_CNT ", 0x1CB4ull},
279 {"DFX_DMA_PREBUF_MEM1_ECC_ERR_ADDR ", 0x1CC0ull},
280 {"DFX_DMA_PREBUF_MEM1_ECC_CNT ", 0x1CC4ull},
281 {"DMA_CH_DONE_STS ", 0x02E0ull},
282 {"DMA_CH_ERR_STS ", 0x0320ull},
283};
284#endif /* CONFIG_DEBUG_FS*/
285
fd5273fa
JH		 286static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev)
287{
288 if (pdev->revision == HISI_DMA_REVISION_HIP08B)
289 return HISI_DMA_REG_LAYOUT_HIP08;
290 else if (pdev->revision >= HISI_DMA_REVISION_HIP09A)
291 return HISI_DMA_REG_LAYOUT_HIP09;
292
293 return HISI_DMA_REG_LAYOUT_INVALID;
294}
295
296static u32 hisi_dma_get_chan_num(struct pci_dev *pdev)
297{
298 if (pdev->revision == HISI_DMA_REVISION_HIP08B)
299 return HISI_DMA_HIP08_CHAN_NUM;
300
301 return HISI_DMA_HIP09_CHAN_NUM;
302}
303
304static u32 hisi_dma_get_msi_num(struct pci_dev *pdev)
305{
306 if (pdev->revision == HISI_DMA_REVISION_HIP08B)
307 return HISI_DMA_HIP08_MSI_NUM;
308
309 return HISI_DMA_HIP09_MSI_NUM;
310}
311
312static u32 hisi_dma_get_queue_base(struct pci_dev *pdev)
313{
314 if (pdev->revision == HISI_DMA_REVISION_HIP08B)
315 return HISI_DMA_HIP08_Q_BASE;
316
317 return HISI_DMA_HIP09_Q_BASE;
318}
319
e9f08b65
ZW		 320static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
321{
322 return container_of(c, struct hisi_dma_chan, vc.chan);
323}
324
325static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
326{
327 return container_of(vd, struct hisi_dma_desc, vd);
328}
329
330static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
331 u32 val)
332{
fd5273fa 333 writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET);
e9f08b65
ZW		 334}
335
336static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
337{
338 u32 tmp;
339
340 tmp = readl_relaxed(addr);
fd5273fa 341 tmp = val ? tmp | pos : tmp & ~pos;
e9f08b65
ZW		 342 writel_relaxed(tmp, addr);
343}
344
e9f08b65
ZW		 345static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
346 bool pause)
347{
fd5273fa 348 void __iomem *addr;
e9f08b65 349
fd5273fa
JH		 350 addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
351 index * HISI_DMA_Q_OFFSET;
352 hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause);
e9f08b65
ZW		 353}
354
355static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
356 bool enable)
357{
fd5273fa 358 void __iomem *addr;
e9f08b65 359
fd5273fa
JH		 360 addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
361 index * HISI_DMA_Q_OFFSET;
362 hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable);
e9f08b65
ZW		 363}
364
365static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
366{
fd5273fa
JH		 367 void __iomem *q_base = hdma_dev->queue_base;
368
369 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
370 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
371 qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
372 else {
373 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
374 qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
375 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
376 qp_index,
377 HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
378 }
e9f08b65
ZW		 379}
380
381static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
382{
fd5273fa
JH		 383 void __iomem *q_base = hdma_dev->queue_base;
384
385 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
386 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS,
387 qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
388 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
389 qp_index, 0);
390 } else {
391 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS,
392 qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
393 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS,
394 qp_index,
395 HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
396 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
397 qp_index, 0);
398 hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
399 qp_index, 0);
400 }
e9f08b65
ZW		 401}
402
403static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
404{
fd5273fa 405 void __iomem *addr;
e9f08b65 406
fd5273fa
JH		 407 addr = hdma_dev->queue_base +
408 HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
409 hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1);
e9f08b65
ZW		 410}
411
412static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
413{
fd5273fa
JH		 414 void __iomem *q_base = hdma_dev->queue_base;
415
416 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
417 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
e9f08b65
ZW		 418}
419
e3bdaa04
JH		 420static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
421 bool disable)
e9f08b65
ZW		 422{
423 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
424 u32 index = chan->qp_num, tmp;
4aa69cf7 425 void __iomem *addr;
e9f08b65
ZW		 426 int ret;
427
428 hisi_dma_pause_dma(hdma_dev, index, true);
429 hisi_dma_enable_dma(hdma_dev, index, false);
430 hisi_dma_mask_irq(hdma_dev, index);
431
fd5273fa
JH		 432 addr = hdma_dev->queue_base +
433 HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET;
4aa69cf7
JH		 434
435 ret = readl_relaxed_poll_timeout(addr, tmp,
fd5273fa 436 FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN,
4aa69cf7 437 HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
e9f08b65
ZW		 438 if (ret) {
439 dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
440 WARN_ON(1);
441 }
442
443 hisi_dma_do_reset(hdma_dev, index);
444 hisi_dma_reset_qp_point(hdma_dev, index);
445 hisi_dma_pause_dma(hdma_dev, index, false);
e3bdaa04
JH		 446
447 if (!disable) {
448 hisi_dma_enable_dma(hdma_dev, index, true);
449 hisi_dma_unmask_irq(hdma_dev, index);
450 }
e9f08b65 451
4aa69cf7 452 ret = readl_relaxed_poll_timeout(addr, tmp,
fd5273fa 453 FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE,
4aa69cf7 454 HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
e9f08b65
ZW		 455 if (ret) {
456 dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
457 WARN_ON(1);
458 }
459}
460
461static void hisi_dma_free_chan_resources(struct dma_chan *c)
462{
463 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
464 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
465
e3bdaa04 466 hisi_dma_reset_or_disable_hw_chan(chan, false);
e9f08b65
ZW		 467 vchan_free_chan_resources(&chan->vc);
468
469 memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth);
470 memset(chan->cq, 0, sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth);
471 chan->sq_tail = 0;
472 chan->cq_head = 0;
473 chan->status = DISABLE;
474}
475
476static void hisi_dma_desc_free(struct virt_dma_desc *vd)
477{
478 kfree(to_hisi_dma_desc(vd));
479}
480
481static struct dma_async_tx_descriptor *
482hisi_dma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dst, dma_addr_t src,
483 size_t len, unsigned long flags)
484{
485 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
486 struct hisi_dma_desc *desc;
487
488 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
489 if (!desc)
490 return NULL;
491
492 desc->sqe.length = cpu_to_le32(len);
493 desc->sqe.src_addr = cpu_to_le64(src);
494 desc->sqe.dst_addr = cpu_to_le64(dst);
495
496 return vchan_tx_prep(&chan->vc, &desc->vd, flags);
497}
498
499static enum dma_status
500hisi_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie,
501 struct dma_tx_state *txstate)
502{
503 return dma_cookie_status(c, cookie, txstate);
504}
505
506static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
507{
508 struct hisi_dma_sqe *sqe = chan->sq + chan->sq_tail;
509 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
510 struct hisi_dma_desc *desc;
511 struct virt_dma_desc *vd;
512
513 vd = vchan_next_desc(&chan->vc);
514 if (!vd) {
e9f08b65
ZW		 515 chan->desc = NULL;
516 return;
517 }
518 list_del(&vd->node);
519 desc = to_hisi_dma_desc(vd);
520 chan->desc = desc;
521
522 memcpy(sqe, &desc->sqe, sizeof(struct hisi_dma_sqe));
523
524 /* update other field in sqe */
525 sqe->dw0 = cpu_to_le32(FIELD_PREP(OPCODE_MASK, OPCODE_M2M));
526 sqe->dw0 |= cpu_to_le32(LOCAL_IRQ_EN);
527
528 /* make sure data has been updated in sqe */
529 wmb();
530
531 /* update sq tail, point to new sqe position */
532 chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
533
534 /* update sq_tail to trigger a new task */
fd5273fa
JH		 535 hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR,
536 chan->qp_num, chan->sq_tail);
e9f08b65
ZW		 537}
538
539static void hisi_dma_issue_pending(struct dma_chan *c)
540{
541 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
542 unsigned long flags;
543
544 spin_lock_irqsave(&chan->vc.lock, flags);
545
2cbb9588 546 if (vchan_issue_pending(&chan->vc) && !chan->desc)
e9f08b65
ZW		 547 hisi_dma_start_transfer(chan);
548
549 spin_unlock_irqrestore(&chan->vc.lock, flags);
550}
551
552static int hisi_dma_terminate_all(struct dma_chan *c)
553{
554 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
555 unsigned long flags;
556 LIST_HEAD(head);
557
558 spin_lock_irqsave(&chan->vc.lock, flags);
559
560 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, true);
561 if (chan->desc) {
562 vchan_terminate_vdesc(&chan->desc->vd);
563 chan->desc = NULL;
564 }
565
566 vchan_get_all_descriptors(&chan->vc, &head);
567
568 spin_unlock_irqrestore(&chan->vc.lock, flags);
569
570 vchan_dma_desc_free_list(&chan->vc, &head);
571 hisi_dma_pause_dma(chan->hdma_dev, chan->qp_num, false);
572
573 return 0;
574}
575
576static void hisi_dma_synchronize(struct dma_chan *c)
577{
578 struct hisi_dma_chan *chan = to_hisi_dma_chan(c);
579
580 vchan_synchronize(&chan->vc);
581}
582
583static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
584{
585 size_t sq_size = sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth;
586 size_t cq_size = sizeof(struct hisi_dma_cqe) * hdma_dev->chan_depth;
587 struct device *dev = &hdma_dev->pdev->dev;
588 struct hisi_dma_chan *chan;
589 int i;
590
591 for (i = 0; i < hdma_dev->chan_num; i++) {
592 chan = &hdma_dev->chan[i];
593 chan->sq = dmam_alloc_coherent(dev, sq_size, &chan->sq_dma,
594 GFP_KERNEL);
595 if (!chan->sq)
596 return -ENOMEM;
597
598 chan->cq = dmam_alloc_coherent(dev, cq_size, &chan->cq_dma,
599 GFP_KERNEL);
600 if (!chan->cq)
601 return -ENOMEM;
602 }
603
604 return 0;
605}
606
607static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
608{
609 struct hisi_dma_chan *chan = &hdma_dev->chan[index];
fd5273fa 610 void __iomem *q_base = hdma_dev->queue_base;
e9f08b65 611 u32 hw_depth = hdma_dev->chan_depth - 1;
fd5273fa
JH		 612 void __iomem *addr;
613 u32 tmp;
e9f08b65
ZW		 614
615 /* set sq, cq base */
fd5273fa 616 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index,
e9f08b65 617 lower_32_bits(chan->sq_dma));
fd5273fa 618 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index,
e9f08b65 619 upper_32_bits(chan->sq_dma));
fd5273fa 620 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index,
e9f08b65 621 lower_32_bits(chan->cq_dma));
fd5273fa 622 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index,
e9f08b65
ZW		 623 upper_32_bits(chan->cq_dma));
624
625 /* set sq, cq depth */
fd5273fa
JH		 626 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth);
627 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth);
e9f08b65
ZW		 628
629 /* init sq tail and cq head */
fd5273fa
JH		 630 hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
631 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
632
633 /* init error interrupt stats */
634 hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0);
635 hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0);
636 hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0);
637
638 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
639 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3,
640 index, 0);
641 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4,
642 index, 0);
643 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5,
644 index, 0);
645 hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6,
646 index, 0);
647 /*
648 * init SQ/CQ direction selecting register.
649 * "0" is to local side and "1" is to remote side.
650 */
651 addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
652 hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0);
653
654 /*
655 * 0 - Continue to next descriptor if error occurs.
656 * 1 - Abort the DMA queue if error occurs.
657 */
658 hisi_dma_update_bit(addr,
659 HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0);
660 } else {
661 addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
662
663 /*
664 * init SQ/CQ direction selecting register.
665 * "0" is to local side and "1" is to remote side.
666 */
667 hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0);
668 hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0);
669
670 /*
671 * 0 - Continue to next descriptor if error occurs.
672 * 1 - Abort the DMA queue if error occurs.
673 */
674
675 tmp = readl_relaxed(addr);
676 tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN;
677 writel_relaxed(tmp, addr);
678
679 /*
680 * 0 - dma should process FLR whith CPU.
681 * 1 - dma not process FLR, only cpu process FLR.
682 */
683 addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE +
684 index * HISI_DMA_Q_OFFSET;
685 hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0);
686
687 addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
688 hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1);
689 }
e9f08b65
ZW		 690}
691
692static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
693{
694 hisi_dma_init_hw_qp(hdma_dev, qp_index);
695 hisi_dma_unmask_irq(hdma_dev, qp_index);
696 hisi_dma_enable_dma(hdma_dev, qp_index, true);
697}
698
699static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
700{
e3bdaa04 701 hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true);
e9f08b65
ZW		 702}
703
704static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev)
705{
706 int i;
707
708 for (i = 0; i < hdma_dev->chan_num; i++) {
709 hdma_dev->chan[i].qp_num = i;
710 hdma_dev->chan[i].hdma_dev = hdma_dev;
711 hdma_dev->chan[i].vc.desc_free = hisi_dma_desc_free;
712 vchan_init(&hdma_dev->chan[i].vc, &hdma_dev->dma_dev);
713 hisi_dma_enable_qp(hdma_dev, i);
714 }
715}
716
717static void hisi_dma_disable_qps(struct hisi_dma_dev *hdma_dev)
718{
719 int i;
720
721 for (i = 0; i < hdma_dev->chan_num; i++) {
722 hisi_dma_disable_qp(hdma_dev, i);
723 tasklet_kill(&hdma_dev->chan[i].vc.task);
724 }
725}
726
727static irqreturn_t hisi_dma_irq(int irq, void *data)
728{
729 struct hisi_dma_chan *chan = data;
730 struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
731 struct hisi_dma_desc *desc;
732 struct hisi_dma_cqe *cqe;
fd5273fa 733 void __iomem *q_base;
e9f08b65 734
d9c8d4b2 735 spin_lock(&chan->vc.lock);
e9f08b65
ZW		 736
737 desc = chan->desc;
738 cqe = chan->cq + chan->cq_head;
fd5273fa 739 q_base = hdma_dev->queue_base;
e9f08b65 740 if (desc) {
94477a79 741 chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
fd5273fa 742 hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR,
94477a79 743 chan->qp_num, chan->cq_head);
e9f08b65 744 if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
e9f08b65 745 vchan_cookie_complete(&desc->vd);
2cbb9588 746 hisi_dma_start_transfer(chan);
e9f08b65
ZW		 747 } else {
748 dev_err(&hdma_dev->pdev->dev, "task error!\n");
749 }
e9f08b65
ZW		 750 }
751
d9c8d4b2 752 spin_unlock(&chan->vc.lock);
e9f08b65
ZW		 753
754 return IRQ_HANDLED;
755}
756
757static int hisi_dma_request_qps_irq(struct hisi_dma_dev *hdma_dev)
758{
759 struct pci_dev *pdev = hdma_dev->pdev;
760 int i, ret;
761
762 for (i = 0; i < hdma_dev->chan_num; i++) {
763 ret = devm_request_irq(&pdev->dev, pci_irq_vector(pdev, i),
764 hisi_dma_irq, IRQF_SHARED, "hisi_dma",
765 &hdma_dev->chan[i]);
766 if (ret)
767 return ret;
768 }
769
770 return 0;
771}
772
773/* This function enables all hw channels in a device */
774static int hisi_dma_enable_hw_channels(struct hisi_dma_dev *hdma_dev)
775{
776 int ret;
777
778 ret = hisi_dma_alloc_qps_mem(hdma_dev);
779 if (ret) {
780 dev_err(&hdma_dev->pdev->dev, "fail to allocate qp memory!\n");
781 return ret;
782 }
783
784 ret = hisi_dma_request_qps_irq(hdma_dev);
785 if (ret) {
786 dev_err(&hdma_dev->pdev->dev, "fail to request qp irq!\n");
787 return ret;
788 }
789
790 hisi_dma_enable_qps(hdma_dev);
791
792 return 0;
793}
794
795static void hisi_dma_disable_hw_channels(void *data)
796{
797 hisi_dma_disable_qps(data);
798}
799
800static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
801 enum hisi_dma_mode mode)
802{
fd5273fa
JH		 803 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
804 writel_relaxed(mode == RC ? 1 : 0,
805 hdma_dev->base + HISI_DMA_HIP08_MODE);
806}
807
808static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev)
809{
810 void __iomem *addr;
811 int i;
812
813 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {
814 for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) {
815 addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i);
816 hisi_dma_update_bit(addr,
817 HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1);
818 }
819 }
820}
821
822static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev)
823{
824 struct dma_device *dma_dev;
825
826 dma_dev = &hdma_dev->dma_dev;
827 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
828 dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
829 dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
830 dma_dev->device_tx_status = hisi_dma_tx_status;
831 dma_dev->device_issue_pending = hisi_dma_issue_pending;
832 dma_dev->device_terminate_all = hisi_dma_terminate_all;
833 dma_dev->device_synchronize = hisi_dma_synchronize;
834 dma_dev->directions = BIT(DMA_MEM_TO_MEM);
835 dma_dev->dev = &hdma_dev->pdev->dev;
836 INIT_LIST_HEAD(&dma_dev->channels);
e9f08b65
ZW		 837}
838
5dda7a62
JH		 839/* --- debugfs implementation --- */
840#ifdef CONFIG_DEBUG_FS
841#include <linux/debugfs.h>
842static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev,
843 u32 *regs_sz)
844{
845 struct device *dev = &hdma_dev->pdev->dev;
846 struct debugfs_reg32 *regs;
847 u32 regs_sz_comm;
848
849 regs_sz_comm = ARRAY_SIZE(hisi_dma_comm_chan_regs);
850
851 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
852 *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip08_chan_regs);
853 else
854 *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip09_chan_regs);
855
856 regs = devm_kcalloc(dev, *regs_sz, sizeof(struct debugfs_reg32),
857 GFP_KERNEL);
858 if (!regs)
859 return NULL;
860 memcpy(regs, hisi_dma_comm_chan_regs, sizeof(hisi_dma_comm_chan_regs));
861
862 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
863 memcpy(regs + regs_sz_comm, hisi_dma_hip08_chan_regs,
864 sizeof(hisi_dma_hip08_chan_regs));
865 else
866 memcpy(regs + regs_sz_comm, hisi_dma_hip09_chan_regs,
867 sizeof(hisi_dma_hip09_chan_regs));
868
869 return regs;
870}
871
872static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev)
873{
874 char dir_name[HISI_DMA_MAX_DIR_NAME_LEN];
875 struct debugfs_regset32 *regsets;
876 struct debugfs_reg32 *regs;
877 struct dentry *chan_dir;
878 struct device *dev;
879 u32 regs_sz;
880 int ret;
881 int i;
882
883 dev = &hdma_dev->pdev->dev;
884
885 regsets = devm_kcalloc(dev, hdma_dev->chan_num,
886 sizeof(*regsets), GFP_KERNEL);
887 if (!regsets)
888 return -ENOMEM;
889
890 regs = hisi_dma_get_ch_regs(hdma_dev, &regs_sz);
891 if (!regs)
892 return -ENOMEM;
893
894 for (i = 0; i < hdma_dev->chan_num; i++) {
895 regsets[i].regs = regs;
896 regsets[i].nregs = regs_sz;
897 regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET;
898 regsets[i].dev = dev;
899
900 memset(dir_name, 0, HISI_DMA_MAX_DIR_NAME_LEN);
901 ret = sprintf(dir_name, "channel%d", i);
902 if (ret < 0)
903 return ret;
904
905 chan_dir = debugfs_create_dir(dir_name,
906 hdma_dev->dma_dev.dbg_dev_root);
907 debugfs_create_regset32("regs", 0444, chan_dir, &regsets[i]);
908 }
909
910 return 0;
911}
912
913static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev)
914{
915 struct debugfs_regset32 *regset;
916 struct device *dev;
917 int ret;
918
919 dev = &hdma_dev->pdev->dev;
920
921 if (hdma_dev->dma_dev.dbg_dev_root == NULL)
922 return;
923
924 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
925 if (!regset)
926 return;
927
928 if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
929 regset->regs = hisi_dma_hip08_comm_regs;
930 regset->nregs = ARRAY_SIZE(hisi_dma_hip08_comm_regs);
931 } else {
932 regset->regs = hisi_dma_hip09_comm_regs;
933 regset->nregs = ARRAY_SIZE(hisi_dma_hip09_comm_regs);
934 }
935 regset->base = hdma_dev->base;
936 regset->dev = dev;
937
938 debugfs_create_regset32("regs", 0444,
939 hdma_dev->dma_dev.dbg_dev_root, regset);
940
941 ret = hisi_dma_create_chan_dir(hdma_dev);
942 if (ret < 0)
943 dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n");
944}
945#else
946static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { }
947#endif /* CONFIG_DEBUG_FS*/
948/* --- debugfs implementation --- */
949
e9f08b65
ZW		 950static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
951{
fd5273fa 952 enum hisi_dma_reg_layout reg_layout;
e9f08b65
ZW		 953 struct device *dev = &pdev->dev;
954 struct hisi_dma_dev *hdma_dev;
955 struct dma_device *dma_dev;
fd5273fa
JH		 956 u32 chan_num;
957 u32 msi_num;
e9f08b65
ZW		 958 int ret;
959
fd5273fa
JH		 960 reg_layout = hisi_dma_get_reg_layout(pdev);
961 if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) {
962 dev_err(dev, "unsupported device!\n");
963 return -EINVAL;
964 }
965
e9f08b65
ZW		 966 ret = pcim_enable_device(pdev);
967 if (ret) {
968 dev_err(dev, "failed to enable device mem!\n");
969 return ret;
970 }
971
972 ret = pcim_iomap_regions(pdev, 1 << PCI_BAR_2, pci_name(pdev));
973 if (ret) {
974 dev_err(dev, "failed to remap I/O region!\n");
975 return ret;
976 }
977
d77143dd 978 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
e9f08b65
ZW		 979 if (ret)
980 return ret;
981
fd5273fa
JH		 982 chan_num = hisi_dma_get_chan_num(pdev);
983 hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
984 GFP_KERNEL);
e9f08b65
ZW		 985 if (!hdma_dev)
986 return -EINVAL;
987
988 hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
989 hdma_dev->pdev = pdev;
e9f08b65 990 hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
fd5273fa
JH		 991 hdma_dev->chan_num = chan_num;
992 hdma_dev->reg_layout = reg_layout;
993 hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev);
e9f08b65
ZW		 994
995 pci_set_drvdata(pdev, hdma_dev);
996 pci_set_master(pdev);
997
fd5273fa
JH		 998 msi_num = hisi_dma_get_msi_num(pdev);
999
26f1ca91 1000 /* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */
fd5273fa 1001 ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI);
e9f08b65
ZW		 1002 if (ret < 0) {
1003 dev_err(dev, "Failed to allocate MSI vectors!\n");
1004 return ret;
1005 }
1006
fd5273fa 1007 hisi_dma_init_dma_dev(hdma_dev);
e9f08b65
ZW		 1008
1009 hisi_dma_set_mode(hdma_dev, RC);
1010
fd5273fa
JH		 1011 hisi_dma_init_hw(hdma_dev);
1012
e9f08b65
ZW		 1013 ret = hisi_dma_enable_hw_channels(hdma_dev);
1014 if (ret < 0) {
1015 dev_err(dev, "failed to enable hw channel!\n");
1016 return ret;
1017 }
1018
1019 ret = devm_add_action_or_reset(dev, hisi_dma_disable_hw_channels,
1020 hdma_dev);
1021 if (ret)
1022 return ret;
1023
fd5273fa 1024 dma_dev = &hdma_dev->dma_dev;
e9f08b65 1025 ret = dmaenginem_async_device_register(dma_dev);
5dda7a62 1026 if (ret < 0) {
e9f08b65 1027 dev_err(dev, "failed to register device!\n");
5dda7a62
JH		 1028 return ret;
1029 }
1030
1031 hisi_dma_create_debugfs(hdma_dev);
e9f08b65 1032
5dda7a62 1033 return 0;
e9f08b65
ZW		 1034}
1035
1036static const struct pci_device_id hisi_dma_pci_tbl[] = {
1037 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
1038 { 0, }
1039};
1040
1041static struct pci_driver hisi_dma_pci_driver = {
1042 .name = "hisi_dma",
1043 .id_table = hisi_dma_pci_tbl,
1044 .probe = hisi_dma_probe,
1045};
1046
1047module_pci_driver(hisi_dma_pci_driver);
1048
1049MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
1050MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
1051MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
1052MODULE_LICENSE("GPL v2");
1053MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
Linux 6.x block layer and io_uring tree(s)