projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'vfs-6.7.misc' of gitolite.kernel.org:pub/scm/linux/kernel/git/vfs/vfs
[linux-block.git] / drivers / dma / ti / k3-udma-glue.c
CommitLineData
d7024191
GS		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * K3 NAVSS DMA glue interface
4 *
5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
6 *
7 */
8
56b0a668 9#include <linux/module.h>
d7024191
GS		 10#include <linux/atomic.h>
11#include <linux/delay.h>
12#include <linux/dma-mapping.h>
13#include <linux/io.h>
14#include <linux/init.h>
15#include <linux/of.h>
16#include <linux/platform_device.h>
17#include <linux/soc/ti/k3-ringacc.h>
18#include <linux/dma/ti-cppi5.h>
19#include <linux/dma/k3-udma-glue.h>
20
21#include "k3-udma.h"
22#include "k3-psil-priv.h"
23
24struct k3_udma_glue_common {
25 struct device *dev;
5b65781d 26 struct device chan_dev;
d7024191
GS		 27 struct udma_dev *udmax;
28 const struct udma_tisci_rm *tisci_rm;
29 struct k3_ringacc *ringacc;
30 u32 src_thread;
31 u32 dst_thread;
32
33 u32 hdesc_size;
34 bool epib;
35 u32 psdata_size;
36 u32 swdata_size;
5b65781d
VR		 37 u32 atype_asel;
38 struct psil_endpoint_config *ep_config;
d7024191
GS		 39};
40
41struct k3_udma_glue_tx_channel {
42 struct k3_udma_glue_common common;
43
44 struct udma_tchan *udma_tchanx;
45 int udma_tchan_id;
46
47 struct k3_ring *ringtx;
48 struct k3_ring *ringtxcq;
49
50 bool psil_paired;
51
52 int virq;
53
54 atomic_t free_pkts;
55 bool tx_pause_on_err;
56 bool tx_filt_einfo;
57 bool tx_filt_pswords;
58 bool tx_supr_tdpkt;
5b65781d
VR		 59
60 int udma_tflow_id;
d7024191
GS		 61};
62
63struct k3_udma_glue_rx_flow {
64 struct udma_rflow *udma_rflow;
65 int udma_rflow_id;
66 struct k3_ring *ringrx;
67 struct k3_ring *ringrxfdq;
68
69 int virq;
70};
71
72struct k3_udma_glue_rx_channel {
73 struct k3_udma_glue_common common;
74
75 struct udma_rchan *udma_rchanx;
76 int udma_rchan_id;
77 bool remote;
78
79 bool psil_paired;
80
81 u32 swdata_size;
82 int flow_id_base;
83
84 struct k3_udma_glue_rx_flow *flows;
85 u32 flow_num;
86 u32 flows_ready;
87};
88
5b65781d
VR		 89static void k3_udma_chan_dev_release(struct device *dev)
90{
91 /* The struct containing the device is devm managed */
92}
93
94static struct class k3_udma_glue_devclass = {
95 .name = "k3_udma_glue_chan",
96 .dev_release = k3_udma_chan_dev_release,
97};
98
d7024191
GS		 99#define K3_UDMAX_TDOWN_TIMEOUT_US 1000
100
101static int of_k3_udma_glue_parse(struct device_node *udmax_np,
102 struct k3_udma_glue_common *common)
103{
d7024191
GS		 104 common->udmax = of_xudma_dev_get(udmax_np, NULL);
105 if (IS_ERR(common->udmax))
106 return PTR_ERR(common->udmax);
107
aa8a4c4e 108 common->ringacc = xudma_get_ringacc(common->udmax);
d7024191
GS		 109 common->tisci_rm = xudma_dev_get_tisci_rm(common->udmax);
110
111 return 0;
112}
113
114static int of_k3_udma_glue_parse_chn(struct device_node *chn_np,
115 const char *name, struct k3_udma_glue_common *common,
116 bool tx_chn)
117{
d7024191
GS		 118 struct of_phandle_args dma_spec;
119 u32 thread_id;
120 int ret = 0;
121 int index;
122
123 if (unlikely(!name))
124 return -EINVAL;
125
126 index = of_property_match_string(chn_np, "dma-names", name);
127 if (index < 0)
128 return index;
129
130 if (of_parse_phandle_with_args(chn_np, "dmas", "#dma-cells", index,
131 &dma_spec))
132 return -ENOENT;
133
5b65781d
VR		 134 ret = of_k3_udma_glue_parse(dma_spec.np, common);
135 if (ret)
136 goto out_put_spec;
137
d7024191 138 thread_id = dma_spec.args[0];
0ebcf1a2 139 if (dma_spec.args_count == 2) {
5b65781d 140 if (dma_spec.args[1] > 2 && !xudma_is_pktdma(common->udmax)) {
0ebcf1a2
PU		 141 dev_err(common->dev, "Invalid channel atype: %u\n",
142 dma_spec.args[1]);
143 ret = -EINVAL;
144 goto out_put_spec;
145 }
5b65781d
VR		 146 if (dma_spec.args[1] > 15 && xudma_is_pktdma(common->udmax)) {
147 dev_err(common->dev, "Invalid channel asel: %u\n",
148 dma_spec.args[1]);
149 ret = -EINVAL;
150 goto out_put_spec;
151 }
152
153 common->atype_asel = dma_spec.args[1];
0ebcf1a2 154 }
d7024191
GS		 155
156 if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
157 ret = -EINVAL;
158 goto out_put_spec;
159 }
160
161 if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) {
162 ret = -EINVAL;
163 goto out_put_spec;
164 }
165
166 /* get psil endpoint config */
5b65781d
VR		 167 common->ep_config = psil_get_ep_config(thread_id);
168 if (IS_ERR(common->ep_config)) {
d7024191
GS		 169 dev_err(common->dev,
170 "No configuration for psi-l thread 0x%04x\n",
171 thread_id);
5b65781d 172 ret = PTR_ERR(common->ep_config);
d7024191
GS		 173 goto out_put_spec;
174 }
175
5b65781d
VR		 176 common->epib = common->ep_config->needs_epib;
177 common->psdata_size = common->ep_config->psd_size;
d7024191
GS		 178
179 if (tx_chn)
180 common->dst_thread = thread_id;
181 else
182 common->src_thread = thread_id;
183
d7024191
GS		 184out_put_spec:
185 of_node_put(dma_spec.np);
186 return ret;
187};
188
189static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
190{
191 struct device *dev = tx_chn->common.dev;
192
193 dev_dbg(dev, "dump_tx_chn:\n"
194 "udma_tchan_id: %d\n"
195 "src_thread: %08x\n"
196 "dst_thread: %08x\n",
197 tx_chn->udma_tchan_id,
198 tx_chn->common.src_thread,
199 tx_chn->common.dst_thread);
200}
201
202static void k3_udma_glue_dump_tx_rt_chn(struct k3_udma_glue_tx_channel *chn,
203 char *mark)
204{
205 struct device *dev = chn->common.dev;
206
207 dev_dbg(dev, "=== dump ===> %s\n", mark);
bc7e5523
PU		 208 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
209 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG));
210 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
d7024191 211 xudma_tchanrt_read(chn->udma_tchanx,
bc7e5523
PU		 212 UDMA_CHAN_RT_PEER_RT_EN_REG));
213 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
214 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_PCNT_REG));
215 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
216 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_BCNT_REG));
217 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
218 xudma_tchanrt_read(chn->udma_tchanx, UDMA_CHAN_RT_SBCNT_REG));
d7024191
GS		 219}
220
221static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
222{
223 const struct udma_tisci_rm *tisci_rm = tx_chn->common.tisci_rm;
224 struct ti_sci_msg_rm_udmap_tx_ch_cfg req;
225
226 memset(&req, 0, sizeof(req));
227
228 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_PAUSE_ON_ERR_VALID |
229 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_EINFO_VALID |
230 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_FILT_PSWORDS_VALID |
231 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
232 TI_SCI_MSG_VALUE_RM_UDMAP_CH_TX_SUPR_TDPKT_VALID |
233 TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
0ebcf1a2
PU		 234 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
235 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
d7024191
GS		 236 req.nav_id = tisci_rm->tisci_dev_id;
237 req.index = tx_chn->udma_tchan_id;
238 if (tx_chn->tx_pause_on_err)
239 req.tx_pause_on_err = 1;
240 if (tx_chn->tx_filt_einfo)
241 req.tx_filt_einfo = 1;
242 if (tx_chn->tx_filt_pswords)
243 req.tx_filt_pswords = 1;
244 req.tx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
245 if (tx_chn->tx_supr_tdpkt)
246 req.tx_supr_tdpkt = 1;
247 req.tx_fetch_size = tx_chn->common.hdesc_size >> 2;
248 req.txcq_qnum = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
5b65781d 249 req.tx_atype = tx_chn->common.atype_asel;
d7024191
GS		 250
251 return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req);
252}
253
254struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev,
255 const char *name, struct k3_udma_glue_tx_channel_cfg *cfg)
256{
257 struct k3_udma_glue_tx_channel *tx_chn;
258 int ret;
259
260 tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL);
261 if (!tx_chn)
262 return ERR_PTR(-ENOMEM);
263
264 tx_chn->common.dev = dev;
265 tx_chn->common.swdata_size = cfg->swdata_size;
266 tx_chn->tx_pause_on_err = cfg->tx_pause_on_err;
267 tx_chn->tx_filt_einfo = cfg->tx_filt_einfo;
268 tx_chn->tx_filt_pswords = cfg->tx_filt_pswords;
269 tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt;
270
271 /* parse of udmap channel */
272 ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
273 &tx_chn->common, true);
274 if (ret)
275 goto err;
276
277 tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib,
278 tx_chn->common.psdata_size,
279 tx_chn->common.swdata_size);
280
5b65781d
VR		 281 if (xudma_is_pktdma(tx_chn->common.udmax))
282 tx_chn->udma_tchan_id = tx_chn->common.ep_config->mapped_channel_id;
283 else
284 tx_chn->udma_tchan_id = -1;
285
d7024191 286 /* request and cfg UDMAP TX channel */
5b65781d
VR		 287 tx_chn->udma_tchanx = xudma_tchan_get(tx_chn->common.udmax,
288 tx_chn->udma_tchan_id);
d7024191
GS		 289 if (IS_ERR(tx_chn->udma_tchanx)) {
290 ret = PTR_ERR(tx_chn->udma_tchanx);
291 dev_err(dev, "UDMAX tchanx get err %d\n", ret);
292 goto err;
293 }
294 tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx);
295
5b65781d
VR		 296 tx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
297 tx_chn->common.chan_dev.parent = xudma_get_device(tx_chn->common.udmax);
298 dev_set_name(&tx_chn->common.chan_dev, "tchan%d-0x%04x",
299 tx_chn->udma_tchan_id, tx_chn->common.dst_thread);
300 ret = device_register(&tx_chn->common.chan_dev);
301 if (ret) {
302 dev_err(dev, "Channel Device registration failed %d\n", ret);
ac2b9f34 303 put_device(&tx_chn->common.chan_dev);
5b65781d
VR		 304 tx_chn->common.chan_dev.parent = NULL;
305 goto err;
306 }
307
308 if (xudma_is_pktdma(tx_chn->common.udmax)) {
309 /* prepare the channel device as coherent */
310 tx_chn->common.chan_dev.dma_coherent = true;
311 dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
312 DMA_BIT_MASK(48));
313 }
314
d7024191
GS		 315 atomic_set(&tx_chn->free_pkts, cfg->txcq_cfg.size);
316
5b65781d
VR		 317 if (xudma_is_pktdma(tx_chn->common.udmax))
318 tx_chn->udma_tflow_id = tx_chn->common.ep_config->default_flow_id;
319 else
320 tx_chn->udma_tflow_id = tx_chn->udma_tchan_id;
321
d7024191 322 /* request and cfg rings */
4927b1ab 323 ret = k3_ringacc_request_rings_pair(tx_chn->common.ringacc,
5b65781d 324 tx_chn->udma_tflow_id, -1,
4927b1ab
PU		 325 &tx_chn->ringtx,
326 &tx_chn->ringtxcq);
327 if (ret) {
328 dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret);
d7024191
GS		 329 goto err;
330 }
331
d553e2ab
PU		 332 /* Set the dma_dev for the rings to be configured */
333 cfg->tx_cfg.dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn);
334 cfg->txcq_cfg.dma_dev = cfg->tx_cfg.dma_dev;
335
5b65781d
VR		 336 /* Set the ASEL value for DMA rings of PKTDMA */
337 if (xudma_is_pktdma(tx_chn->common.udmax)) {
338 cfg->tx_cfg.asel = tx_chn->common.atype_asel;
339 cfg->txcq_cfg.asel = tx_chn->common.atype_asel;
340 }
341
d7024191
GS		 342 ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg);
343 if (ret) {
344 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
345 goto err;
346 }
347
348 ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg);
349 if (ret) {
350 dev_err(dev, "Failed to cfg ringtx %d\n", ret);
351 goto err;
352 }
353
354 /* request and cfg psi-l */
355 tx_chn->common.src_thread =
356 xudma_dev_get_psil_base(tx_chn->common.udmax) +
357 tx_chn->udma_tchan_id;
358
359 ret = k3_udma_glue_cfg_tx_chn(tx_chn);
360 if (ret) {
361 dev_err(dev, "Failed to cfg tchan %d\n", ret);
362 goto err;
363 }
364
d7024191
GS		 365 k3_udma_glue_dump_tx_chn(tx_chn);
366
367 return tx_chn;
368
369err:
370 k3_udma_glue_release_tx_chn(tx_chn);
371 return ERR_PTR(ret);
372}
373EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn);
374
375void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
376{
377 if (tx_chn->psil_paired) {
378 xudma_navss_psil_unpair(tx_chn->common.udmax,
379 tx_chn->common.src_thread,
380 tx_chn->common.dst_thread);
381 tx_chn->psil_paired = false;
382 }
383
384 if (!IS_ERR_OR_NULL(tx_chn->udma_tchanx))
385 xudma_tchan_put(tx_chn->common.udmax,
386 tx_chn->udma_tchanx);
387
388 if (tx_chn->ringtxcq)
389 k3_ringacc_ring_free(tx_chn->ringtxcq);
390
391 if (tx_chn->ringtx)
392 k3_ringacc_ring_free(tx_chn->ringtx);
5b65781d
VR		 393
394 if (tx_chn->common.chan_dev.parent) {
395 device_unregister(&tx_chn->common.chan_dev);
396 tx_chn->common.chan_dev.parent = NULL;
397 }
d7024191
GS		 398}
399EXPORT_SYMBOL_GPL(k3_udma_glue_release_tx_chn);
400
401int k3_udma_glue_push_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
402 struct cppi5_host_desc_t *desc_tx,
403 dma_addr_t desc_dma)
404{
405 u32 ringtxcq_id;
406
407 if (!atomic_add_unless(&tx_chn->free_pkts, -1, 0))
408 return -ENOMEM;
409
410 ringtxcq_id = k3_ringacc_get_ring_id(tx_chn->ringtxcq);
411 cppi5_desc_set_retpolicy(&desc_tx->hdr, 0, ringtxcq_id);
412
413 return k3_ringacc_ring_push(tx_chn->ringtx, &desc_dma);
414}
415EXPORT_SYMBOL_GPL(k3_udma_glue_push_tx_chn);
416
417int k3_udma_glue_pop_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
418 dma_addr_t *desc_dma)
419{
420 int ret;
421
422 ret = k3_ringacc_ring_pop(tx_chn->ringtxcq, desc_dma);
423 if (!ret)
424 atomic_inc(&tx_chn->free_pkts);
425
426 return ret;
427}
428EXPORT_SYMBOL_GPL(k3_udma_glue_pop_tx_chn);
429
430int k3_udma_glue_enable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
431{
69973b48
GS		 432 int ret;
433
434 ret = xudma_navss_psil_pair(tx_chn->common.udmax,
435 tx_chn->common.src_thread,
436 tx_chn->common.dst_thread);
437 if (ret) {
438 dev_err(tx_chn->common.dev, "PSI-L request err %d\n", ret);
439 return ret;
440 }
441
442 tx_chn->psil_paired = true;
443
bc7e5523 444 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
52c74d3d 445 UDMA_PEER_RT_EN_ENABLE);
d7024191 446
bc7e5523 447 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
52c74d3d 448 UDMA_CHAN_RT_CTL_EN);
d7024191
GS		 449
450 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn en");
451 return 0;
452}
453EXPORT_SYMBOL_GPL(k3_udma_glue_enable_tx_chn);
454
455void k3_udma_glue_disable_tx_chn(struct k3_udma_glue_tx_channel *tx_chn)
456{
457 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis1");
458
bc7e5523 459 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG, 0);
d7024191
GS		 460
461 xudma_tchanrt_write(tx_chn->udma_tchanx,
bc7e5523 462 UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
d7024191 463 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn dis2");
69973b48
GS		 464
465 if (tx_chn->psil_paired) {
466 xudma_navss_psil_unpair(tx_chn->common.udmax,
467 tx_chn->common.src_thread,
468 tx_chn->common.dst_thread);
469 tx_chn->psil_paired = false;
470 }
d7024191
GS		 471}
472EXPORT_SYMBOL_GPL(k3_udma_glue_disable_tx_chn);
473
474void k3_udma_glue_tdown_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
475 bool sync)
476{
477 int i = 0;
478 u32 val;
479
480 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown1");
481
bc7e5523 482 xudma_tchanrt_write(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG,
d7024191
GS		 483 UDMA_CHAN_RT_CTL_EN | UDMA_CHAN_RT_CTL_TDOWN);
484
bc7e5523 485 val = xudma_tchanrt_read(tx_chn->udma_tchanx, UDMA_CHAN_RT_CTL_REG);
d7024191
GS		 486
487 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
488 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
bc7e5523 489 UDMA_CHAN_RT_CTL_REG);
d7024191
GS		 490 udelay(1);
491 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
492 dev_err(tx_chn->common.dev, "TX tdown timeout\n");
493 break;
494 }
495 i++;
496 }
497
498 val = xudma_tchanrt_read(tx_chn->udma_tchanx,
bc7e5523 499 UDMA_CHAN_RT_PEER_RT_EN_REG);
d7024191
GS		 500 if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
501 dev_err(tx_chn->common.dev, "TX tdown peer not stopped\n");
502 k3_udma_glue_dump_tx_rt_chn(tx_chn, "txchn tdown2");
503}
504EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_tx_chn);
505
506void k3_udma_glue_reset_tx_chn(struct k3_udma_glue_tx_channel *tx_chn,
507 void *data,
508 void (*cleanup)(void *data, dma_addr_t desc_dma))
509{
5b65781d 510 struct device *dev = tx_chn->common.dev;
d7024191
GS		 511 dma_addr_t desc_dma;
512 int occ_tx, i, ret;
513
d7024191
GS		 514 /*
515 * TXQ reset need to be special way as it is input for udma and its
516 * state cached by udma, so:
517 * 1) save TXQ occ
518 * 2) clean up TXQ and call callback .cleanup() for each desc
519 * 3) reset TXQ in a special way
520 */
521 occ_tx = k3_ringacc_ring_get_occ(tx_chn->ringtx);
5b65781d 522 dev_dbg(dev, "TX reset occ_tx %u\n", occ_tx);
d7024191
GS		 523
524 for (i = 0; i < occ_tx; i++) {
525 ret = k3_ringacc_ring_pop(tx_chn->ringtx, &desc_dma);
526 if (ret) {
5b65781d
VR		 527 if (ret != -ENODATA)
528 dev_err(dev, "TX reset pop %d\n", ret);
d7024191
GS		 529 break;
530 }
531 cleanup(data, desc_dma);
532 }
533
5b65781d
VR		 534 /* reset TXCQ as it is not input for udma - expected to be empty */
535 k3_ringacc_ring_reset(tx_chn->ringtxcq);
d7024191
GS		 536 k3_ringacc_ring_reset_dma(tx_chn->ringtx, occ_tx);
537}
538EXPORT_SYMBOL_GPL(k3_udma_glue_reset_tx_chn);
539
540u32 k3_udma_glue_tx_get_hdesc_size(struct k3_udma_glue_tx_channel *tx_chn)
541{
542 return tx_chn->common.hdesc_size;
543}
544EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_hdesc_size);
545
546u32 k3_udma_glue_tx_get_txcq_id(struct k3_udma_glue_tx_channel *tx_chn)
547{
548 return k3_ringacc_get_ring_id(tx_chn->ringtxcq);
549}
550EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_txcq_id);
551
552int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn)
553{
5b65781d
VR		 554 if (xudma_is_pktdma(tx_chn->common.udmax)) {
555 tx_chn->virq = xudma_pktdma_tflow_get_irq(tx_chn->common.udmax,
556 tx_chn->udma_tflow_id);
557 } else {
558 tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq);
559 }
d7024191 560
f9a1d321
DC		 561 if (!tx_chn->virq)
562 return -ENXIO;
563
d7024191
GS		 564 return tx_chn->virq;
565}
566EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq);
567
426506a7
PU		 568struct device *
569 k3_udma_glue_tx_get_dma_device(struct k3_udma_glue_tx_channel *tx_chn)
570{
5b65781d
VR		 571 if (xudma_is_pktdma(tx_chn->common.udmax) &&
572 (tx_chn->common.atype_asel == 14 || tx_chn->common.atype_asel == 15))
573 return &tx_chn->common.chan_dev;
574
426506a7
PU		 575 return xudma_get_device(tx_chn->common.udmax);
576}
577EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_dma_device);
578
5b65781d
VR		 579void k3_udma_glue_tx_dma_to_cppi5_addr(struct k3_udma_glue_tx_channel *tx_chn,
580 dma_addr_t *addr)
581{
582 if (!xudma_is_pktdma(tx_chn->common.udmax) ||
583 !tx_chn->common.atype_asel)
584 return;
585
586 *addr |= (u64)tx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
587}
588EXPORT_SYMBOL_GPL(k3_udma_glue_tx_dma_to_cppi5_addr);
589
590void k3_udma_glue_tx_cppi5_to_dma_addr(struct k3_udma_glue_tx_channel *tx_chn,
591 dma_addr_t *addr)
592{
593 if (!xudma_is_pktdma(tx_chn->common.udmax) ||
594 !tx_chn->common.atype_asel)
595 return;
596
597 *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
598}
599EXPORT_SYMBOL_GPL(k3_udma_glue_tx_cppi5_to_dma_addr);
600
d7024191
GS		 601static int k3_udma_glue_cfg_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
602{
603 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
604 struct ti_sci_msg_rm_udmap_rx_ch_cfg req;
605 int ret;
606
607 memset(&req, 0, sizeof(req));
608
609 req.valid_params = TI_SCI_MSG_VALUE_RM_UDMAP_CH_FETCH_SIZE_VALID |
610 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CQ_QNUM_VALID |
611 TI_SCI_MSG_VALUE_RM_UDMAP_CH_CHAN_TYPE_VALID |
0ebcf1a2 612 TI_SCI_MSG_VALUE_RM_UDMAP_CH_ATYPE_VALID;
d7024191
GS		 613
614 req.nav_id = tisci_rm->tisci_dev_id;
615 req.index = rx_chn->udma_rchan_id;
616 req.rx_fetch_size = rx_chn->common.hdesc_size >> 2;
617 /*
618 * TODO: we can't support rxcq_qnum/RCHAN[a]_RCQ cfg with current sysfw
619 * and udmax impl, so just configure it to invalid value.
620 * req.rxcq_qnum = k3_ringacc_get_ring_id(rx_chn->flows[0].ringrx);
621 */
622 req.rxcq_qnum = 0xFFFF;
5b65781d
VR		 623 if (!xudma_is_pktdma(rx_chn->common.udmax) && rx_chn->flow_num &&
624 rx_chn->flow_id_base != rx_chn->udma_rchan_id) {
d7024191 625 /* Default flow + extra ones */
5b65781d
VR		 626 req.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_START_VALID |
627 TI_SCI_MSG_VALUE_RM_UDMAP_CH_RX_FLOWID_CNT_VALID;
d7024191
GS		 628 req.flowid_start = rx_chn->flow_id_base;
629 req.flowid_cnt = rx_chn->flow_num;
630 }
631 req.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR;
5b65781d 632 req.rx_atype = rx_chn->common.atype_asel;
d7024191
GS		 633
634 ret = tisci_rm->tisci_udmap_ops->rx_ch_cfg(tisci_rm->tisci, &req);
635 if (ret)
636 dev_err(rx_chn->common.dev, "rchan%d cfg failed %d\n",
637 rx_chn->udma_rchan_id, ret);
638
639 return ret;
640}
641
642static void k3_udma_glue_release_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
643 u32 flow_num)
644{
645 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
646
647 if (IS_ERR_OR_NULL(flow->udma_rflow))
648 return;
649
650 if (flow->ringrxfdq)
651 k3_ringacc_ring_free(flow->ringrxfdq);
652
653 if (flow->ringrx)
654 k3_ringacc_ring_free(flow->ringrx);
655
656 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
657 flow->udma_rflow = NULL;
658 rx_chn->flows_ready--;
659}
660
661static int k3_udma_glue_cfg_rx_flow(struct k3_udma_glue_rx_channel *rx_chn,
662 u32 flow_idx,
663 struct k3_udma_glue_rx_flow_cfg *flow_cfg)
664{
665 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
666 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
667 struct device *dev = rx_chn->common.dev;
668 struct ti_sci_msg_rm_udmap_flow_cfg req;
669 int rx_ring_id;
670 int rx_ringfdq_id;
671 int ret = 0;
672
673 flow->udma_rflow = xudma_rflow_get(rx_chn->common.udmax,
674 flow->udma_rflow_id);
675 if (IS_ERR(flow->udma_rflow)) {
676 ret = PTR_ERR(flow->udma_rflow);
677 dev_err(dev, "UDMAX rflow get err %d\n", ret);
018af9be 678 return ret;
d7024191
GS		 679 }
680
681 if (flow->udma_rflow_id != xudma_rflow_get_id(flow->udma_rflow)) {
018af9be
CJ		 682 ret = -ENODEV;
683 goto err_rflow_put;
d7024191
GS		 684 }
685
5b65781d
VR		 686 if (xudma_is_pktdma(rx_chn->common.udmax)) {
687 rx_ringfdq_id = flow->udma_rflow_id +
688 xudma_get_rflow_ring_offset(rx_chn->common.udmax);
689 rx_ring_id = 0;
690 } else {
691 rx_ring_id = flow_cfg->ring_rxq_id;
692 rx_ringfdq_id = flow_cfg->ring_rxfdq0_id;
693 }
694
d7024191 695 /* request and cfg rings */
4927b1ab 696 ret = k3_ringacc_request_rings_pair(rx_chn->common.ringacc,
5b65781d 697 rx_ringfdq_id, rx_ring_id,
4927b1ab
PU		 698 &flow->ringrxfdq,
699 &flow->ringrx);
700 if (ret) {
701 dev_err(dev, "Failed to get RX/RXFDQ rings %d\n", ret);
018af9be 702 goto err_rflow_put;
d7024191
GS		 703 }
704
d553e2ab
PU		 705 /* Set the dma_dev for the rings to be configured */
706 flow_cfg->rx_cfg.dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn);
707 flow_cfg->rxfdq_cfg.dma_dev = flow_cfg->rx_cfg.dma_dev;
708
5b65781d
VR		 709 /* Set the ASEL value for DMA rings of PKTDMA */
710 if (xudma_is_pktdma(rx_chn->common.udmax)) {
711 flow_cfg->rx_cfg.asel = rx_chn->common.atype_asel;
712 flow_cfg->rxfdq_cfg.asel = rx_chn->common.atype_asel;
713 }
714
d7024191
GS		 715 ret = k3_ringacc_ring_cfg(flow->ringrx, &flow_cfg->rx_cfg);
716 if (ret) {
717 dev_err(dev, "Failed to cfg ringrx %d\n", ret);
018af9be 718 goto err_ringrxfdq_free;
d7024191
GS		 719 }
720
721 ret = k3_ringacc_ring_cfg(flow->ringrxfdq, &flow_cfg->rxfdq_cfg);
722 if (ret) {
723 dev_err(dev, "Failed to cfg ringrxfdq %d\n", ret);
018af9be 724 goto err_ringrxfdq_free;
d7024191
GS		 725 }
726
727 if (rx_chn->remote) {
728 rx_ring_id = TI_SCI_RESOURCE_NULL;
729 rx_ringfdq_id = TI_SCI_RESOURCE_NULL;
730 } else {
731 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
732 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
733 }
734
735 memset(&req, 0, sizeof(req));
736
737 req.valid_params =
738 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_EINFO_PRESENT_VALID |
739 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_PSINFO_PRESENT_VALID |
740 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_ERROR_HANDLING_VALID |
741 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DESC_TYPE_VALID |
742 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
743 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_HI_SEL_VALID |
744 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_SRC_TAG_LO_SEL_VALID |
745 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_HI_SEL_VALID |
746 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_TAG_LO_SEL_VALID |
747 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
748 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
749 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
750 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
751 req.nav_id = tisci_rm->tisci_dev_id;
752 req.flow_index = flow->udma_rflow_id;
753 if (rx_chn->common.epib)
754 req.rx_einfo_present = 1;
755 if (rx_chn->common.psdata_size)
756 req.rx_psinfo_present = 1;
757 if (flow_cfg->rx_error_handling)
758 req.rx_error_handling = 1;
759 req.rx_desc_type = 0;
760 req.rx_dest_qnum = rx_ring_id;
761 req.rx_src_tag_hi_sel = 0;
762 req.rx_src_tag_lo_sel = flow_cfg->src_tag_lo_sel;
763 req.rx_dest_tag_hi_sel = 0;
764 req.rx_dest_tag_lo_sel = 0;
765 req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
766 req.rx_fdq1_qnum = rx_ringfdq_id;
767 req.rx_fdq2_qnum = rx_ringfdq_id;
768 req.rx_fdq3_qnum = rx_ringfdq_id;
769
770 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
771 if (ret) {
772 dev_err(dev, "flow%d config failed: %d\n", flow->udma_rflow_id,
773 ret);
018af9be 774 goto err_ringrxfdq_free;
d7024191
GS		 775 }
776
777 rx_chn->flows_ready++;
778 dev_dbg(dev, "flow%d config done. ready:%d\n",
779 flow->udma_rflow_id, rx_chn->flows_ready);
780
781 return 0;
018af9be
CJ		 782
783err_ringrxfdq_free:
784 k3_ringacc_ring_free(flow->ringrxfdq);
018af9be
CJ		 785 k3_ringacc_ring_free(flow->ringrx);
786
787err_rflow_put:
788 xudma_rflow_put(rx_chn->common.udmax, flow->udma_rflow);
789 flow->udma_rflow = NULL;
790
d7024191
GS		 791 return ret;
792}
793
794static void k3_udma_glue_dump_rx_chn(struct k3_udma_glue_rx_channel *chn)
795{
796 struct device *dev = chn->common.dev;
797
798 dev_dbg(dev, "dump_rx_chn:\n"
799 "udma_rchan_id: %d\n"
800 "src_thread: %08x\n"
801 "dst_thread: %08x\n"
802 "epib: %d\n"
803 "hdesc_size: %u\n"
804 "psdata_size: %u\n"
805 "swdata_size: %u\n"
806 "flow_id_base: %d\n"
807 "flow_num: %d\n",
808 chn->udma_rchan_id,
809 chn->common.src_thread,
810 chn->common.dst_thread,
811 chn->common.epib,
812 chn->common.hdesc_size,
813 chn->common.psdata_size,
814 chn->common.swdata_size,
815 chn->flow_id_base,
816 chn->flow_num);
817}
818
819static void k3_udma_glue_dump_rx_rt_chn(struct k3_udma_glue_rx_channel *chn,
820 char *mark)
821{
822 struct device *dev = chn->common.dev;
823
824 dev_dbg(dev, "=== dump ===> %s\n", mark);
825
bc7e5523
PU		 826 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_CTL_REG,
827 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG));
828 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PEER_RT_EN_REG,
d7024191 829 xudma_rchanrt_read(chn->udma_rchanx,
bc7e5523
PU		 830 UDMA_CHAN_RT_PEER_RT_EN_REG));
831 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_PCNT_REG,
832 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_PCNT_REG));
833 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_BCNT_REG,
834 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_BCNT_REG));
835 dev_dbg(dev, "0x%08X: %08X\n", UDMA_CHAN_RT_SBCNT_REG,
836 xudma_rchanrt_read(chn->udma_rchanx, UDMA_CHAN_RT_SBCNT_REG));
d7024191
GS		 837}
838
839static int
840k3_udma_glue_allocate_rx_flows(struct k3_udma_glue_rx_channel *rx_chn,
841 struct k3_udma_glue_rx_channel_cfg *cfg)
842{
843 int ret;
844
845 /* default rflow */
846 if (cfg->flow_id_use_rxchan_id)
847 return 0;
848
849 /* not a GP rflows */
850 if (rx_chn->flow_id_base != -1 &&
851 !xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
852 return 0;
853
854 /* Allocate range of GP rflows */
855 ret = xudma_alloc_gp_rflow_range(rx_chn->common.udmax,
856 rx_chn->flow_id_base,
857 rx_chn->flow_num);
858 if (ret < 0) {
859 dev_err(rx_chn->common.dev, "UDMAX reserve_rflow %d cnt:%d err: %d\n",
860 rx_chn->flow_id_base, rx_chn->flow_num, ret);
861 return ret;
862 }
863 rx_chn->flow_id_base = ret;
864
865 return 0;
866}
867
868static struct k3_udma_glue_rx_channel *
869k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
870 struct k3_udma_glue_rx_channel_cfg *cfg)
871{
872 struct k3_udma_glue_rx_channel *rx_chn;
5b65781d 873 struct psil_endpoint_config *ep_cfg;
d7024191
GS		 874 int ret, i;
875
876 if (cfg->flow_id_num <= 0)
877 return ERR_PTR(-EINVAL);
878
879 if (cfg->flow_id_num != 1 &&
880 (cfg->def_flow_cfg || cfg->flow_id_use_rxchan_id))
881 return ERR_PTR(-EINVAL);
882
883 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
884 if (!rx_chn)
885 return ERR_PTR(-ENOMEM);
886
887 rx_chn->common.dev = dev;
888 rx_chn->common.swdata_size = cfg->swdata_size;
889 rx_chn->remote = false;
890
891 /* parse of udmap channel */
892 ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
893 &rx_chn->common, false);
894 if (ret)
895 goto err;
896
897 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
898 rx_chn->common.psdata_size,
899 rx_chn->common.swdata_size);
900
5b65781d
VR		 901 ep_cfg = rx_chn->common.ep_config;
902
903 if (xudma_is_pktdma(rx_chn->common.udmax))
904 rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id;
905 else
906 rx_chn->udma_rchan_id = -1;
907
d7024191 908 /* request and cfg UDMAP RX channel */
5b65781d
VR		 909 rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax,
910 rx_chn->udma_rchan_id);
d7024191
GS		 911 if (IS_ERR(rx_chn->udma_rchanx)) {
912 ret = PTR_ERR(rx_chn->udma_rchanx);
913 dev_err(dev, "UDMAX rchanx get err %d\n", ret);
914 goto err;
915 }
916 rx_chn->udma_rchan_id = xudma_rchan_get_id(rx_chn->udma_rchanx);
917
5b65781d
VR		 918 rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
919 rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
920 dev_set_name(&rx_chn->common.chan_dev, "rchan%d-0x%04x",
921 rx_chn->udma_rchan_id, rx_chn->common.src_thread);
922 ret = device_register(&rx_chn->common.chan_dev);
923 if (ret) {
924 dev_err(dev, "Channel Device registration failed %d\n", ret);
ac2b9f34 925 put_device(&rx_chn->common.chan_dev);
5b65781d
VR		 926 rx_chn->common.chan_dev.parent = NULL;
927 goto err;
928 }
d7024191 929
5b65781d
VR		 930 if (xudma_is_pktdma(rx_chn->common.udmax)) {
931 /* prepare the channel device as coherent */
932 rx_chn->common.chan_dev.dma_coherent = true;
933 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
934 DMA_BIT_MASK(48));
935 }
936
937 if (xudma_is_pktdma(rx_chn->common.udmax)) {
938 int flow_start = cfg->flow_id_base;
939 int flow_end;
940
941 if (flow_start == -1)
942 flow_start = ep_cfg->flow_start;
943
944 flow_end = flow_start + cfg->flow_id_num - 1;
945 if (flow_start < ep_cfg->flow_start ||
946 flow_end > (ep_cfg->flow_start + ep_cfg->flow_num - 1)) {
947 dev_err(dev, "Invalid flow range requested\n");
948 ret = -EINVAL;
949 goto err;
950 }
951 rx_chn->flow_id_base = flow_start;
952 } else {
953 rx_chn->flow_id_base = cfg->flow_id_base;
954
955 /* Use RX channel id as flow id: target dev can't generate flow_id */
956 if (cfg->flow_id_use_rxchan_id)
957 rx_chn->flow_id_base = rx_chn->udma_rchan_id;
958 }
959
960 rx_chn->flow_num = cfg->flow_id_num;
d7024191
GS		 961
962 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
963 sizeof(*rx_chn->flows), GFP_KERNEL);
964 if (!rx_chn->flows) {
965 ret = -ENOMEM;
966 goto err;
967 }
968
969 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
970 if (ret)
971 goto err;
972
973 for (i = 0; i < rx_chn->flow_num; i++)
974 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
975
976 /* request and cfg psi-l */
977 rx_chn->common.dst_thread =
978 xudma_dev_get_psil_base(rx_chn->common.udmax) +
979 rx_chn->udma_rchan_id;
980
981 ret = k3_udma_glue_cfg_rx_chn(rx_chn);
982 if (ret) {
983 dev_err(dev, "Failed to cfg rchan %d\n", ret);
984 goto err;
985 }
986
987 /* init default RX flow only if flow_num = 1 */
988 if (cfg->def_flow_cfg) {
989 ret = k3_udma_glue_cfg_rx_flow(rx_chn, 0, cfg->def_flow_cfg);
990 if (ret)
991 goto err;
992 }
993
d7024191
GS		 994 k3_udma_glue_dump_rx_chn(rx_chn);
995
996 return rx_chn;
997
998err:
999 k3_udma_glue_release_rx_chn(rx_chn);
1000 return ERR_PTR(ret);
1001}
1002
1003static struct k3_udma_glue_rx_channel *
1004k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name,
1005 struct k3_udma_glue_rx_channel_cfg *cfg)
1006{
1007 struct k3_udma_glue_rx_channel *rx_chn;
1008 int ret, i;
1009
1010 if (cfg->flow_id_num <= 0 ||
1011 cfg->flow_id_use_rxchan_id ||
1012 cfg->def_flow_cfg ||
1013 cfg->flow_id_base < 0)
1014 return ERR_PTR(-EINVAL);
1015
1016 /*
1017 * Remote RX channel is under control of Remote CPU core, so
1018 * Linux can only request and manipulate by dedicated RX flows
1019 */
1020
1021 rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL);
1022 if (!rx_chn)
1023 return ERR_PTR(-ENOMEM);
1024
1025 rx_chn->common.dev = dev;
1026 rx_chn->common.swdata_size = cfg->swdata_size;
1027 rx_chn->remote = true;
1028 rx_chn->udma_rchan_id = -1;
1029 rx_chn->flow_num = cfg->flow_id_num;
1030 rx_chn->flow_id_base = cfg->flow_id_base;
1031 rx_chn->psil_paired = false;
1032
1033 /* parse of udmap channel */
1034 ret = of_k3_udma_glue_parse_chn(dev->of_node, name,
1035 &rx_chn->common, false);
1036 if (ret)
1037 goto err;
1038
1039 rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib,
1040 rx_chn->common.psdata_size,
1041 rx_chn->common.swdata_size);
1042
1043 rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num,
1044 sizeof(*rx_chn->flows), GFP_KERNEL);
1045 if (!rx_chn->flows) {
1046 ret = -ENOMEM;
1047 goto err;
1048 }
1049
5b65781d
VR		 1050 rx_chn->common.chan_dev.class = &k3_udma_glue_devclass;
1051 rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax);
1052 dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x",
1053 rx_chn->common.src_thread);
1054 ret = device_register(&rx_chn->common.chan_dev);
1055 if (ret) {
1056 dev_err(dev, "Channel Device registration failed %d\n", ret);
ac2b9f34 1057 put_device(&rx_chn->common.chan_dev);
5b65781d
VR		 1058 rx_chn->common.chan_dev.parent = NULL;
1059 goto err;
1060 }
1061
1062 if (xudma_is_pktdma(rx_chn->common.udmax)) {
1063 /* prepare the channel device as coherent */
1064 rx_chn->common.chan_dev.dma_coherent = true;
1065 dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
1066 DMA_BIT_MASK(48));
1067 }
1068
d7024191
GS		 1069 ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg);
1070 if (ret)
1071 goto err;
1072
1073 for (i = 0; i < rx_chn->flow_num; i++)
1074 rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i;
1075
1076 k3_udma_glue_dump_rx_chn(rx_chn);
1077
1078 return rx_chn;
1079
1080err:
1081 k3_udma_glue_release_rx_chn(rx_chn);
1082 return ERR_PTR(ret);
1083}
1084
1085struct k3_udma_glue_rx_channel *
1086k3_udma_glue_request_rx_chn(struct device *dev, const char *name,
1087 struct k3_udma_glue_rx_channel_cfg *cfg)
1088{
1089 if (cfg->remote)
1090 return k3_udma_glue_request_remote_rx_chn(dev, name, cfg);
1091 else
1092 return k3_udma_glue_request_rx_chn_priv(dev, name, cfg);
1093}
1094EXPORT_SYMBOL_GPL(k3_udma_glue_request_rx_chn);
1095
1096void k3_udma_glue_release_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1097{
1098 int i;
1099
1100 if (IS_ERR_OR_NULL(rx_chn->common.udmax))
1101 return;
1102
1103 if (rx_chn->psil_paired) {
1104 xudma_navss_psil_unpair(rx_chn->common.udmax,
1105 rx_chn->common.src_thread,
1106 rx_chn->common.dst_thread);
1107 rx_chn->psil_paired = false;
1108 }
1109
1110 for (i = 0; i < rx_chn->flow_num; i++)
1111 k3_udma_glue_release_rx_flow(rx_chn, i);
1112
1113 if (xudma_rflow_is_gp(rx_chn->common.udmax, rx_chn->flow_id_base))
1114 xudma_free_gp_rflow_range(rx_chn->common.udmax,
1115 rx_chn->flow_id_base,
1116 rx_chn->flow_num);
1117
1118 if (!IS_ERR_OR_NULL(rx_chn->udma_rchanx))
1119 xudma_rchan_put(rx_chn->common.udmax,
1120 rx_chn->udma_rchanx);
5b65781d
VR		 1121
1122 if (rx_chn->common.chan_dev.parent) {
1123 device_unregister(&rx_chn->common.chan_dev);
1124 rx_chn->common.chan_dev.parent = NULL;
1125 }
d7024191
GS		 1126}
1127EXPORT_SYMBOL_GPL(k3_udma_glue_release_rx_chn);
1128
1129int k3_udma_glue_rx_flow_init(struct k3_udma_glue_rx_channel *rx_chn,
1130 u32 flow_idx,
1131 struct k3_udma_glue_rx_flow_cfg *flow_cfg)
1132{
1133 if (flow_idx >= rx_chn->flow_num)
1134 return -EINVAL;
1135
1136 return k3_udma_glue_cfg_rx_flow(rx_chn, flow_idx, flow_cfg);
1137}
1138EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_init);
1139
1140u32 k3_udma_glue_rx_flow_get_fdq_id(struct k3_udma_glue_rx_channel *rx_chn,
1141 u32 flow_idx)
1142{
1143 struct k3_udma_glue_rx_flow *flow;
1144
1145 if (flow_idx >= rx_chn->flow_num)
1146 return -EINVAL;
1147
1148 flow = &rx_chn->flows[flow_idx];
1149
1150 return k3_ringacc_get_ring_id(flow->ringrxfdq);
1151}
1152EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_get_fdq_id);
1153
1154u32 k3_udma_glue_rx_get_flow_id_base(struct k3_udma_glue_rx_channel *rx_chn)
1155{
1156 return rx_chn->flow_id_base;
1157}
1158EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_flow_id_base);
1159
1160int k3_udma_glue_rx_flow_enable(struct k3_udma_glue_rx_channel *rx_chn,
1161 u32 flow_idx)
1162{
1163 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1164 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1165 struct device *dev = rx_chn->common.dev;
1166 struct ti_sci_msg_rm_udmap_flow_cfg req;
1167 int rx_ring_id;
1168 int rx_ringfdq_id;
1169 int ret = 0;
1170
1171 if (!rx_chn->remote)
1172 return -EINVAL;
1173
1174 rx_ring_id = k3_ringacc_get_ring_id(flow->ringrx);
1175 rx_ringfdq_id = k3_ringacc_get_ring_id(flow->ringrxfdq);
1176
1177 memset(&req, 0, sizeof(req));
1178
1179 req.valid_params =
1180 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1181 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1182 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1183 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1184 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1185 req.nav_id = tisci_rm->tisci_dev_id;
1186 req.flow_index = flow->udma_rflow_id;
1187 req.rx_dest_qnum = rx_ring_id;
1188 req.rx_fdq0_sz0_qnum = rx_ringfdq_id;
1189 req.rx_fdq1_qnum = rx_ringfdq_id;
1190 req.rx_fdq2_qnum = rx_ringfdq_id;
1191 req.rx_fdq3_qnum = rx_ringfdq_id;
1192
1193 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1194 if (ret) {
1195 dev_err(dev, "flow%d enable failed: %d\n", flow->udma_rflow_id,
1196 ret);
1197 }
1198
1199 return ret;
1200}
1201EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_enable);
1202
1203int k3_udma_glue_rx_flow_disable(struct k3_udma_glue_rx_channel *rx_chn,
1204 u32 flow_idx)
1205{
1206 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_idx];
1207 const struct udma_tisci_rm *tisci_rm = rx_chn->common.tisci_rm;
1208 struct device *dev = rx_chn->common.dev;
1209 struct ti_sci_msg_rm_udmap_flow_cfg req;
1210 int ret = 0;
1211
1212 if (!rx_chn->remote)
1213 return -EINVAL;
1214
1215 memset(&req, 0, sizeof(req));
1216 req.valid_params =
1217 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_DEST_QNUM_VALID |
1218 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ0_SZ0_QNUM_VALID |
1219 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ1_QNUM_VALID |
1220 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ2_QNUM_VALID |
1221 TI_SCI_MSG_VALUE_RM_UDMAP_FLOW_FDQ3_QNUM_VALID;
1222 req.nav_id = tisci_rm->tisci_dev_id;
1223 req.flow_index = flow->udma_rflow_id;
1224 req.rx_dest_qnum = TI_SCI_RESOURCE_NULL;
1225 req.rx_fdq0_sz0_qnum = TI_SCI_RESOURCE_NULL;
1226 req.rx_fdq1_qnum = TI_SCI_RESOURCE_NULL;
1227 req.rx_fdq2_qnum = TI_SCI_RESOURCE_NULL;
1228 req.rx_fdq3_qnum = TI_SCI_RESOURCE_NULL;
1229
1230 ret = tisci_rm->tisci_udmap_ops->rx_flow_cfg(tisci_rm->tisci, &req);
1231 if (ret) {
1232 dev_err(dev, "flow%d disable failed: %d\n", flow->udma_rflow_id,
1233 ret);
1234 }
1235
1236 return ret;
1237}
1238EXPORT_SYMBOL_GPL(k3_udma_glue_rx_flow_disable);
1239
1240int k3_udma_glue_enable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1241{
69973b48
GS		 1242 int ret;
1243
d7024191
GS		 1244 if (rx_chn->remote)
1245 return -EINVAL;
1246
1247 if (rx_chn->flows_ready < rx_chn->flow_num)
1248 return -EINVAL;
1249
69973b48
GS		 1250 ret = xudma_navss_psil_pair(rx_chn->common.udmax,
1251 rx_chn->common.src_thread,
1252 rx_chn->common.dst_thread);
1253 if (ret) {
1254 dev_err(rx_chn->common.dev, "PSI-L request err %d\n", ret);
1255 return ret;
1256 }
1257
1258 rx_chn->psil_paired = true;
1259
bc7e5523 1260 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG,
52c74d3d 1261 UDMA_CHAN_RT_CTL_EN);
d7024191 1262
bc7e5523 1263 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
d7024191
GS		 1264 UDMA_PEER_RT_EN_ENABLE);
1265
1266 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt en");
1267 return 0;
1268}
1269EXPORT_SYMBOL_GPL(k3_udma_glue_enable_rx_chn);
1270
1271void k3_udma_glue_disable_rx_chn(struct k3_udma_glue_rx_channel *rx_chn)
1272{
1273 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis1");
1274
1275 xudma_rchanrt_write(rx_chn->udma_rchanx,
bc7e5523
PU		 1276 UDMA_CHAN_RT_PEER_RT_EN_REG, 0);
1277 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG, 0);
d7024191
GS		 1278
1279 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt dis2");
69973b48
GS		 1280
1281 if (rx_chn->psil_paired) {
1282 xudma_navss_psil_unpair(rx_chn->common.udmax,
1283 rx_chn->common.src_thread,
1284 rx_chn->common.dst_thread);
1285 rx_chn->psil_paired = false;
1286 }
d7024191
GS		 1287}
1288EXPORT_SYMBOL_GPL(k3_udma_glue_disable_rx_chn);
1289
1290void k3_udma_glue_tdown_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1291 bool sync)
1292{
1293 int i = 0;
1294 u32 val;
1295
1296 if (rx_chn->remote)
1297 return;
1298
1299 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown1");
1300
bc7e5523 1301 xudma_rchanrt_write(rx_chn->udma_rchanx, UDMA_CHAN_RT_PEER_RT_EN_REG,
d7024191
GS		 1302 UDMA_PEER_RT_EN_ENABLE | UDMA_PEER_RT_EN_TEARDOWN);
1303
bc7e5523 1304 val = xudma_rchanrt_read(rx_chn->udma_rchanx, UDMA_CHAN_RT_CTL_REG);
d7024191
GS		 1305
1306 while (sync && (val & UDMA_CHAN_RT_CTL_EN)) {
1307 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
bc7e5523 1308 UDMA_CHAN_RT_CTL_REG);
d7024191
GS		 1309 udelay(1);
1310 if (i > K3_UDMAX_TDOWN_TIMEOUT_US) {
1311 dev_err(rx_chn->common.dev, "RX tdown timeout\n");
1312 break;
1313 }
1314 i++;
1315 }
1316
1317 val = xudma_rchanrt_read(rx_chn->udma_rchanx,
bc7e5523 1318 UDMA_CHAN_RT_PEER_RT_EN_REG);
d7024191
GS		 1319 if (sync && (val & UDMA_PEER_RT_EN_ENABLE))
1320 dev_err(rx_chn->common.dev, "TX tdown peer not stopped\n");
1321 k3_udma_glue_dump_rx_rt_chn(rx_chn, "rxrt tdown2");
1322}
1323EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn);
1324
1325void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1326 u32 flow_num, void *data,
1327 void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq)
1328{
1329 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1330 struct device *dev = rx_chn->common.dev;
1331 dma_addr_t desc_dma;
1332 int occ_rx, i, ret;
1333
1334 /* reset RXCQ as it is not input for udma - expected to be empty */
1335 occ_rx = k3_ringacc_ring_get_occ(flow->ringrx);
1336 dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx);
d7024191
GS		 1337
1338 /* Skip RX FDQ in case one FDQ is used for the set of flows */
1339 if (skip_fdq)
5b65781d 1340 goto do_reset;
d7024191
GS		 1341
1342 /*
1343 * RX FDQ reset need to be special way as it is input for udma and its
1344 * state cached by udma, so:
1345 * 1) save RX FDQ occ
1346 * 2) clean up RX FDQ and call callback .cleanup() for each desc
1347 * 3) reset RX FDQ in a special way
1348 */
1349 occ_rx = k3_ringacc_ring_get_occ(flow->ringrxfdq);
1350 dev_dbg(dev, "RX reset flow %u occ_rx_fdq %u\n", flow_num, occ_rx);
1351
1352 for (i = 0; i < occ_rx; i++) {
1353 ret = k3_ringacc_ring_pop(flow->ringrxfdq, &desc_dma);
1354 if (ret) {
5b65781d
VR		 1355 if (ret != -ENODATA)
1356 dev_err(dev, "RX reset pop %d\n", ret);
d7024191
GS		 1357 break;
1358 }
1359 cleanup(data, desc_dma);
1360 }
1361
1362 k3_ringacc_ring_reset_dma(flow->ringrxfdq, occ_rx);
5b65781d
VR		 1363
1364do_reset:
1365 k3_ringacc_ring_reset(flow->ringrx);
d7024191
GS		 1366}
1367EXPORT_SYMBOL_GPL(k3_udma_glue_reset_rx_chn);
1368
1369int k3_udma_glue_push_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1370 u32 flow_num, struct cppi5_host_desc_t *desc_rx,
1371 dma_addr_t desc_dma)
1372{
1373 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1374
1375 return k3_ringacc_ring_push(flow->ringrxfdq, &desc_dma);
1376}
1377EXPORT_SYMBOL_GPL(k3_udma_glue_push_rx_chn);
1378
1379int k3_udma_glue_pop_rx_chn(struct k3_udma_glue_rx_channel *rx_chn,
1380 u32 flow_num, dma_addr_t *desc_dma)
1381{
1382 struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num];
1383
1384 return k3_ringacc_ring_pop(flow->ringrx, desc_dma);
1385}
1386EXPORT_SYMBOL_GPL(k3_udma_glue_pop_rx_chn);
1387
1388int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn,
1389 u32 flow_num)
1390{
1391 struct k3_udma_glue_rx_flow *flow;
1392
1393 flow = &rx_chn->flows[flow_num];
1394
5b65781d
VR		 1395 if (xudma_is_pktdma(rx_chn->common.udmax)) {
1396 flow->virq = xudma_pktdma_rflow_get_irq(rx_chn->common.udmax,
1397 flow->udma_rflow_id);
1398 } else {
1399 flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx);
1400 }
d7024191
GS		 1401
1402 return flow->virq;
1403}
1404EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq);
426506a7
PU		 1405
1406struct device *
1407 k3_udma_glue_rx_get_dma_device(struct k3_udma_glue_rx_channel *rx_chn)
1408{
5b65781d
VR		 1409 if (xudma_is_pktdma(rx_chn->common.udmax) &&
1410 (rx_chn->common.atype_asel == 14 || rx_chn->common.atype_asel == 15))
1411 return &rx_chn->common.chan_dev;
1412
426506a7
PU		 1413 return xudma_get_device(rx_chn->common.udmax);
1414}
1415EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_dma_device);
5b65781d
VR		 1416
1417void k3_udma_glue_rx_dma_to_cppi5_addr(struct k3_udma_glue_rx_channel *rx_chn,
1418 dma_addr_t *addr)
1419{
1420 if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1421 !rx_chn->common.atype_asel)
1422 return;
1423
1424 *addr |= (u64)rx_chn->common.atype_asel << K3_ADDRESS_ASEL_SHIFT;
1425}
1426EXPORT_SYMBOL_GPL(k3_udma_glue_rx_dma_to_cppi5_addr);
1427
1428void k3_udma_glue_rx_cppi5_to_dma_addr(struct k3_udma_glue_rx_channel *rx_chn,
1429 dma_addr_t *addr)
1430{
1431 if (!xudma_is_pktdma(rx_chn->common.udmax) ||
1432 !rx_chn->common.atype_asel)
1433 return;
1434
1435 *addr &= (u64)GENMASK(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1436}
1437EXPORT_SYMBOL_GPL(k3_udma_glue_rx_cppi5_to_dma_addr);
1438
1439static int __init k3_udma_glue_class_init(void)
1440{
1441 return class_register(&k3_udma_glue_devclass);
1442}
56b0a668
KH		 1443
1444module_init(k3_udma_glue_class_init);
1445MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)