projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'pinctrl-v6.9-2' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw...
[linux-block.git] / drivers / net / ethernet / ti / icssg / icssg_prueth.c
CommitLineData
128d5874
RQ		 1// SPDX-License-Identifier: GPL-2.0
2
3/* Texas Instruments ICSSG Ethernet Driver
4 *
5 * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/
6 *
7 */
8
9#include <linux/bitops.h>
10#include <linux/clk.h>
c1e10d5d 11#include <linux/delay.h>
128d5874
RQ		 12#include <linux/dma-mapping.h>
13#include <linux/dma/ti-cppi5.h>
14#include <linux/etherdevice.h>
15#include <linux/genalloc.h>
16#include <linux/if_vlan.h>
17#include <linux/interrupt.h>
18#include <linux/kernel.h>
19#include <linux/mfd/syscon.h>
20#include <linux/module.h>
21#include <linux/of.h>
128d5874
RQ		 22#include <linux/of_mdio.h>
23#include <linux/of_net.h>
b0377116 24#include <linux/platform_device.h>
128d5874 25#include <linux/phy.h>
b0377116 26#include <linux/property.h>
128d5874
RQ		 27#include <linux/remoteproc/pruss.h>
28#include <linux/regmap.h>
29#include <linux/remoteproc.h>
30
31#include "icssg_prueth.h"
32#include "icssg_mii_rt.h"
33#include "../k3-cppi-desc-pool.h"
34
35#define PRUETH_MODULE_DESCRIPTION "PRUSS ICSSG Ethernet driver"
36
37/* Netif debug messages possible */
38#define PRUETH_EMAC_DEBUG (NETIF_MSG_DRV | \
39 NETIF_MSG_PROBE | \
40 NETIF_MSG_LINK | \
41 NETIF_MSG_TIMER | \
42 NETIF_MSG_IFDOWN | \
43 NETIF_MSG_IFUP | \
44 NETIF_MSG_RX_ERR | \
45 NETIF_MSG_TX_ERR | \
46 NETIF_MSG_TX_QUEUED | \
47 NETIF_MSG_INTR | \
48 NETIF_MSG_TX_DONE | \
49 NETIF_MSG_RX_STATUS | \
50 NETIF_MSG_PKTDATA | \
51 NETIF_MSG_HW | \
52 NETIF_MSG_WOL)
53
54#define prueth_napi_to_emac(napi) container_of(napi, struct prueth_emac, napi_rx)
55
56/* CTRLMMR_ICSSG_RGMII_CTRL register bits */
57#define ICSSG_CTRL_RGMII_ID_MODE BIT(24)
58
186734c1
RQ		 59#define IEP_DEFAULT_CYCLE_TIME_NS 1000000 /* 1 ms */
60
128d5874
RQ		 61static void prueth_cleanup_rx_chns(struct prueth_emac *emac,
62 struct prueth_rx_chn *rx_chn,
63 int max_rflows)
64{
65 if (rx_chn->desc_pool)
66 k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
67
68 if (rx_chn->rx_chn)
69 k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
70}
71
72static void prueth_cleanup_tx_chns(struct prueth_emac *emac)
73{
74 int i;
75
76 for (i = 0; i < emac->tx_ch_num; i++) {
77 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
78
79 if (tx_chn->desc_pool)
80 k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
81
82 if (tx_chn->tx_chn)
83 k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
84
85 /* Assume prueth_cleanup_tx_chns() is called at the
86 * end after all channel resources are freed
87 */
88 memset(tx_chn, 0, sizeof(*tx_chn));
89 }
90}
91
92static void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num)
93{
94 int i;
95
96 for (i = 0; i < num; i++) {
97 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
98
99 if (tx_chn->irq)
100 free_irq(tx_chn->irq, tx_chn);
101 netif_napi_del(&tx_chn->napi_tx);
102 }
103}
104
105static void prueth_xmit_free(struct prueth_tx_chn *tx_chn,
106 struct cppi5_host_desc_t *desc)
107{
108 struct cppi5_host_desc_t *first_desc, *next_desc;
109 dma_addr_t buf_dma, next_desc_dma;
110 u32 buf_dma_len;
111
112 first_desc = desc;
113 next_desc = first_desc;
114
115 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
116 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
117
118 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len,
119 DMA_TO_DEVICE);
120
121 next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
122 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
123 while (next_desc_dma) {
124 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
125 next_desc_dma);
126 cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
127 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
128
129 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
130 DMA_TO_DEVICE);
131
132 next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
133 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
134
135 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
136 }
137
138 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
139}
140
141static int emac_tx_complete_packets(struct prueth_emac *emac, int chn,
142 int budget)
143{
144 struct net_device *ndev = emac->ndev;
145 struct cppi5_host_desc_t *desc_tx;
146 struct netdev_queue *netif_txq;
147 struct prueth_tx_chn *tx_chn;
148 unsigned int total_bytes = 0;
149 struct sk_buff *skb;
150 dma_addr_t desc_dma;
151 int res, num_tx = 0;
152 void **swdata;
153
154 tx_chn = &emac->tx_chns[chn];
155
156 while (true) {
157 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
158 if (res == -ENODATA)
159 break;
160
161 /* teardown completion */
162 if (cppi5_desc_is_tdcm(desc_dma)) {
163 if (atomic_dec_and_test(&emac->tdown_cnt))
164 complete(&emac->tdown_complete);
165 break;
166 }
167
168 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
169 desc_dma);
170 swdata = cppi5_hdesc_get_swdata(desc_tx);
171
172 skb = *(swdata);
173 prueth_xmit_free(tx_chn, desc_tx);
174
175 ndev = skb->dev;
176 ndev->stats.tx_packets++;
177 ndev->stats.tx_bytes += skb->len;
178 total_bytes += skb->len;
179 napi_consume_skb(skb, budget);
180 num_tx++;
181 }
182
183 if (!num_tx)
184 return 0;
185
186 netif_txq = netdev_get_tx_queue(ndev, chn);
187 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
188
189 if (netif_tx_queue_stopped(netif_txq)) {
190 /* If the TX queue was stopped, wake it now
191 * if we have enough room.
192 */
193 __netif_tx_lock(netif_txq, smp_processor_id());
194 if (netif_running(ndev) &&
195 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
196 MAX_SKB_FRAGS))
197 netif_tx_wake_queue(netif_txq);
198 __netif_tx_unlock(netif_txq);
199 }
200
201 return num_tx;
202}
203
204static int emac_napi_tx_poll(struct napi_struct *napi_tx, int budget)
205{
206 struct prueth_tx_chn *tx_chn = prueth_napi_to_tx_chn(napi_tx);
207 struct prueth_emac *emac = tx_chn->emac;
208 int num_tx_packets;
209
210 num_tx_packets = emac_tx_complete_packets(emac, tx_chn->id, budget);
211
212 if (num_tx_packets >= budget)
213 return budget;
214
215 if (napi_complete_done(napi_tx, num_tx_packets))
216 enable_irq(tx_chn->irq);
217
218 return num_tx_packets;
219}
220
221static irqreturn_t prueth_tx_irq(int irq, void *dev_id)
222{
223 struct prueth_tx_chn *tx_chn = dev_id;
224
225 disable_irq_nosync(irq);
226 napi_schedule(&tx_chn->napi_tx);
227
228 return IRQ_HANDLED;
229}
230
231static int prueth_ndev_add_tx_napi(struct prueth_emac *emac)
232{
233 struct prueth *prueth = emac->prueth;
234 int i, ret;
235
236 for (i = 0; i < emac->tx_ch_num; i++) {
237 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
238
239 netif_napi_add_tx(emac->ndev, &tx_chn->napi_tx, emac_napi_tx_poll);
240 ret = request_irq(tx_chn->irq, prueth_tx_irq,
241 IRQF_TRIGGER_HIGH, tx_chn->name,
242 tx_chn);
243 if (ret) {
244 netif_napi_del(&tx_chn->napi_tx);
245 dev_err(prueth->dev, "unable to request TX IRQ %d\n",
246 tx_chn->irq);
247 goto fail;
248 }
249 }
250
251 return 0;
252fail:
253 prueth_ndev_del_tx_napi(emac, i);
254 return ret;
255}
256
257static int prueth_init_tx_chns(struct prueth_emac *emac)
258{
259 static const struct k3_ring_cfg ring_cfg = {
260 .elm_size = K3_RINGACC_RING_ELSIZE_8,
261 .mode = K3_RINGACC_RING_MODE_RING,
262 .flags = 0,
263 .size = PRUETH_MAX_TX_DESC,
264 };
265 struct k3_udma_glue_tx_channel_cfg tx_cfg;
266 struct device *dev = emac->prueth->dev;
267 struct net_device *ndev = emac->ndev;
268 int ret, slice, i;
269 u32 hdesc_size;
270
271 slice = prueth_emac_slice(emac);
272 if (slice < 0)
273 return slice;
274
275 init_completion(&emac->tdown_complete);
276
277 hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE,
278 PRUETH_NAV_SW_DATA_SIZE);
279 memset(&tx_cfg, 0, sizeof(tx_cfg));
280 tx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE;
281 tx_cfg.tx_cfg = ring_cfg;
282 tx_cfg.txcq_cfg = ring_cfg;
283
284 for (i = 0; i < emac->tx_ch_num; i++) {
285 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
286
287 /* To differentiate channels for SLICE0 vs SLICE1 */
288 snprintf(tx_chn->name, sizeof(tx_chn->name),
289 "tx%d-%d", slice, i);
290
291 tx_chn->emac = emac;
292 tx_chn->id = i;
293 tx_chn->descs_num = PRUETH_MAX_TX_DESC;
294
295 tx_chn->tx_chn =
296 k3_udma_glue_request_tx_chn(dev, tx_chn->name,
297 &tx_cfg);
298 if (IS_ERR(tx_chn->tx_chn)) {
299 ret = PTR_ERR(tx_chn->tx_chn);
300 tx_chn->tx_chn = NULL;
301 netdev_err(ndev,
302 "Failed to request tx dma ch: %d\n", ret);
303 goto fail;
304 }
305
306 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);
307 tx_chn->desc_pool =
308 k3_cppi_desc_pool_create_name(tx_chn->dma_dev,
309 tx_chn->descs_num,
310 hdesc_size,
311 tx_chn->name);
312 if (IS_ERR(tx_chn->desc_pool)) {
313 ret = PTR_ERR(tx_chn->desc_pool);
314 tx_chn->desc_pool = NULL;
315 netdev_err(ndev, "Failed to create tx pool: %d\n", ret);
316 goto fail;
317 }
318
a325f174 319 ret = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
f9a1d321 320 if (ret < 0) {
128d5874
RQ		 321 netdev_err(ndev, "failed to get tx irq\n");
322 goto fail;
323 }
a325f174 324 tx_chn->irq = ret;
128d5874
RQ		 325
326 snprintf(tx_chn->name, sizeof(tx_chn->name), "%s-tx%d",
327 dev_name(dev), tx_chn->id);
328 }
329
330 return 0;
331
332fail:
333 prueth_cleanup_tx_chns(emac);
334 return ret;
335}
336
337static int prueth_init_rx_chns(struct prueth_emac *emac,
338 struct prueth_rx_chn *rx_chn,
339 char *name, u32 max_rflows,
340 u32 max_desc_num)
341{
342 struct k3_udma_glue_rx_channel_cfg rx_cfg;
343 struct device *dev = emac->prueth->dev;
344 struct net_device *ndev = emac->ndev;
345 u32 fdqring_id, hdesc_size;
346 int i, ret = 0, slice;
347
348 slice = prueth_emac_slice(emac);
349 if (slice < 0)
350 return slice;
351
352 /* To differentiate channels for SLICE0 vs SLICE1 */
353 snprintf(rx_chn->name, sizeof(rx_chn->name), "%s%d", name, slice);
354
355 hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE,
356 PRUETH_NAV_SW_DATA_SIZE);
357 memset(&rx_cfg, 0, sizeof(rx_cfg));
358 rx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE;
359 rx_cfg.flow_id_num = max_rflows;
360 rx_cfg.flow_id_base = -1; /* udmax will auto select flow id base */
361
362 /* init all flows */
363 rx_chn->dev = dev;
364 rx_chn->descs_num = max_desc_num;
365
366 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, rx_chn->name,
367 &rx_cfg);
368 if (IS_ERR(rx_chn->rx_chn)) {
369 ret = PTR_ERR(rx_chn->rx_chn);
370 rx_chn->rx_chn = NULL;
371 netdev_err(ndev, "Failed to request rx dma ch: %d\n", ret);
372 goto fail;
373 }
374
375 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);
376 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,
377 rx_chn->descs_num,
378 hdesc_size,
379 rx_chn->name);
380 if (IS_ERR(rx_chn->desc_pool)) {
381 ret = PTR_ERR(rx_chn->desc_pool);
382 rx_chn->desc_pool = NULL;
383 netdev_err(ndev, "Failed to create rx pool: %d\n", ret);
384 goto fail;
385 }
386
387 emac->rx_flow_id_base = k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
388 netdev_dbg(ndev, "flow id base = %d\n", emac->rx_flow_id_base);
389
390 fdqring_id = K3_RINGACC_RING_ID_ANY;
391 for (i = 0; i < rx_cfg.flow_id_num; i++) {
392 struct k3_ring_cfg rxring_cfg = {
393 .elm_size = K3_RINGACC_RING_ELSIZE_8,
394 .mode = K3_RINGACC_RING_MODE_RING,
395 .flags = 0,
396 };
397 struct k3_ring_cfg fdqring_cfg = {
398 .elm_size = K3_RINGACC_RING_ELSIZE_8,
399 .flags = K3_RINGACC_RING_SHARED,
400 };
401 struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
402 .rx_cfg = rxring_cfg,
403 .rxfdq_cfg = fdqring_cfg,
404 .ring_rxq_id = K3_RINGACC_RING_ID_ANY,
405 .src_tag_lo_sel =
406 K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
407 };
408
409 rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
410 rx_flow_cfg.rx_cfg.size = max_desc_num;
411 rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
412 rx_flow_cfg.rxfdq_cfg.mode = emac->prueth->pdata.fdqring_mode;
413
414 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,
415 i, &rx_flow_cfg);
416 if (ret) {
417 netdev_err(ndev, "Failed to init rx flow%d %d\n",
418 i, ret);
419 goto fail;
420 }
421 if (!i)
422 fdqring_id = k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,
423 i);
4dcd0e83
DC		 424 ret = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);
425 if (ret <= 0) {
426 if (!ret)
427 ret = -ENXIO;
128d5874
RQ		 428 netdev_err(ndev, "Failed to get rx dma irq");
429 goto fail;
430 }
4dcd0e83 431 rx_chn->irq[i] = ret;
128d5874
RQ		 432 }
433
434 return 0;
435
436fail:
437 prueth_cleanup_rx_chns(emac, rx_chn, max_rflows);
438 return ret;
439}
440
441static int prueth_dma_rx_push(struct prueth_emac *emac,
442 struct sk_buff *skb,
443 struct prueth_rx_chn *rx_chn)
444{
445 struct net_device *ndev = emac->ndev;
446 struct cppi5_host_desc_t *desc_rx;
447 u32 pkt_len = skb_tailroom(skb);
448 dma_addr_t desc_dma;
449 dma_addr_t buf_dma;
450 void **swdata;
451
452 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
453 if (!desc_rx) {
454 netdev_err(ndev, "rx push: failed to allocate descriptor\n");
455 return -ENOMEM;
456 }
457 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
458
459 buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len, DMA_FROM_DEVICE);
460 if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
461 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
462 netdev_err(ndev, "rx push: failed to map rx pkt buffer\n");
463 return -EINVAL;
464 }
465
466 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
467 PRUETH_NAV_PS_DATA_SIZE);
468 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
469 cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb));
470
471 swdata = cppi5_hdesc_get_swdata(desc_rx);
472 *swdata = skb;
473
474 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0,
475 desc_rx, desc_dma);
476}
477
186734c1
RQ		 478static u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns)
479{
480 u32 iepcount_lo, iepcount_hi, hi_rollover_count;
481 u64 ns;
482
483 iepcount_lo = lo & GENMASK(19, 0);
484 iepcount_hi = (hi & GENMASK(11, 0)) << 12 | lo >> 20;
485 hi_rollover_count = hi >> 11;
486
487 ns = ((u64)hi_rollover_count) << 23 | (iepcount_hi + hi_sw);
488 ns = ns * cycle_time_ns + iepcount_lo;
489
490 return ns;
491}
492
493static void emac_rx_timestamp(struct prueth_emac *emac,
494 struct sk_buff *skb, u32 *psdata)
495{
496 struct skb_shared_hwtstamps *ssh;
497 u64 ns;
498
499 u32 hi_sw = readl(emac->prueth->shram.va +
500 TIMESYNC_FW_WC_COUNT_HI_SW_OFFSET_OFFSET);
501 ns = icssg_ts_to_ns(hi_sw, psdata[1], psdata[0],
502 IEP_DEFAULT_CYCLE_TIME_NS);
503
504 ssh = skb_hwtstamps(skb);
505 memset(ssh, 0, sizeof(*ssh));
506 ssh->hwtstamp = ns_to_ktime(ns);
507}
508
128d5874
RQ		 509static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id)
510{
511 struct prueth_rx_chn *rx_chn = &emac->rx_chns;
512 u32 buf_dma_len, pkt_len, port_id = 0;
513 struct net_device *ndev = emac->ndev;
514 struct cppi5_host_desc_t *desc_rx;
515 struct sk_buff *skb, *new_skb;
516 dma_addr_t desc_dma, buf_dma;
517 void **swdata;
186734c1 518 u32 *psdata;
128d5874
RQ		 519 int ret;
520
521 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma);
522 if (ret) {
523 if (ret != -ENODATA)
524 netdev_err(ndev, "rx pop: failed: %d\n", ret);
525 return ret;
526 }
527
528 if (cppi5_desc_is_tdcm(desc_dma)) /* Teardown ? */
529 return 0;
530
531 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
532
533 swdata = cppi5_hdesc_get_swdata(desc_rx);
534 skb = *swdata;
535
186734c1
RQ		 536 psdata = cppi5_hdesc_get_psdata(desc_rx);
537 /* RX HW timestamp */
538 if (emac->rx_ts_enabled)
539 emac_rx_timestamp(emac, skb, psdata);
540
128d5874
RQ		 541 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
542 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
543 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
544 /* firmware adds 4 CRC bytes, strip them */
545 pkt_len -= 4;
546 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
547
548 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
549 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
550
551 skb->dev = ndev;
552 new_skb = netdev_alloc_skb_ip_align(ndev, PRUETH_MAX_PKT_SIZE);
553 /* if allocation fails we drop the packet but push the
554 * descriptor back to the ring with old skb to prevent a stall
555 */
556 if (!new_skb) {
557 ndev->stats.rx_dropped++;
558 new_skb = skb;
559 } else {
560 /* send the filled skb up the n/w stack */
561 skb_put(skb, pkt_len);
562 skb->protocol = eth_type_trans(skb, ndev);
563 napi_gro_receive(&emac->napi_rx, skb);
564 ndev->stats.rx_bytes += pkt_len;
565 ndev->stats.rx_packets++;
566 }
567
568 /* queue another RX DMA */
569 ret = prueth_dma_rx_push(emac, new_skb, &emac->rx_chns);
570 if (WARN_ON(ret < 0)) {
571 dev_kfree_skb_any(new_skb);
572 ndev->stats.rx_errors++;
573 ndev->stats.rx_dropped++;
574 }
575
576 return ret;
577}
578
579static void prueth_rx_cleanup(void *data, dma_addr_t desc_dma)
580{
581 struct prueth_rx_chn *rx_chn = data;
582 struct cppi5_host_desc_t *desc_rx;
583 struct sk_buff *skb;
584 dma_addr_t buf_dma;
585 u32 buf_dma_len;
586 void **swdata;
587
588 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
589 swdata = cppi5_hdesc_get_swdata(desc_rx);
590 skb = *swdata;
591 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
592 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
593
594 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len,
595 DMA_FROM_DEVICE);
596 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
597
598 dev_kfree_skb_any(skb);
599}
600
186734c1
RQ		 601static int emac_get_tx_ts(struct prueth_emac *emac,
602 struct emac_tx_ts_response *rsp)
603{
604 struct prueth *prueth = emac->prueth;
605 int slice = prueth_emac_slice(emac);
606 int addr;
607
608 addr = icssg_queue_pop(prueth, slice == 0 ?
609 ICSSG_TS_POP_SLICE0 : ICSSG_TS_POP_SLICE1);
610 if (addr < 0)
611 return addr;
612
613 memcpy_fromio(rsp, prueth->shram.va + addr, sizeof(*rsp));
614 /* return buffer back for to pool */
615 icssg_queue_push(prueth, slice == 0 ?
616 ICSSG_TS_PUSH_SLICE0 : ICSSG_TS_PUSH_SLICE1, addr);
617
618 return 0;
619}
620
621static void tx_ts_work(struct prueth_emac *emac)
622{
623 struct skb_shared_hwtstamps ssh;
624 struct emac_tx_ts_response tsr;
625 struct sk_buff *skb;
626 int ret = 0;
627 u32 hi_sw;
628 u64 ns;
629
630 /* There may be more than one pending requests */
631 while (1) {
632 ret = emac_get_tx_ts(emac, &tsr);
633 if (ret) /* nothing more */
634 break;
635
636 if (tsr.cookie >= PRUETH_MAX_TX_TS_REQUESTS ||
637 !emac->tx_ts_skb[tsr.cookie]) {
638 netdev_err(emac->ndev, "Invalid TX TS cookie 0x%x\n",
639 tsr.cookie);
640 break;
641 }
642
643 skb = emac->tx_ts_skb[tsr.cookie];
644 emac->tx_ts_skb[tsr.cookie] = NULL; /* free slot */
645 if (!skb) {
646 netdev_err(emac->ndev, "Driver Bug! got NULL skb\n");
647 break;
648 }
649
650 hi_sw = readl(emac->prueth->shram.va +
651 TIMESYNC_FW_WC_COUNT_HI_SW_OFFSET_OFFSET);
652 ns = icssg_ts_to_ns(hi_sw, tsr.hi_ts, tsr.lo_ts,
653 IEP_DEFAULT_CYCLE_TIME_NS);
654
655 memset(&ssh, 0, sizeof(ssh));
656 ssh.hwtstamp = ns_to_ktime(ns);
657
658 skb_tstamp_tx(skb, &ssh);
659 dev_consume_skb_any(skb);
660
661 if (atomic_dec_and_test(&emac->tx_ts_pending)) /* no more? */
662 break;
663 }
664}
665
666static int prueth_tx_ts_cookie_get(struct prueth_emac *emac)
667{
668 int i;
669
670 /* search and get the next free slot */
671 for (i = 0; i < PRUETH_MAX_TX_TS_REQUESTS; i++) {
672 if (!emac->tx_ts_skb[i]) {
673 emac->tx_ts_skb[i] = ERR_PTR(-EBUSY); /* reserve slot */
674 return i;
675 }
676 }
677
678 return -EBUSY;
679}
680
128d5874
RQ		 681/**
682 * emac_ndo_start_xmit - EMAC Transmit function
683 * @skb: SKB pointer
684 * @ndev: EMAC network adapter
685 *
686 * Called by the system to transmit a packet - we queue the packet in
687 * EMAC hardware transmit queue
688 * Doesn't wait for completion we'll check for TX completion in
689 * emac_tx_complete_packets().
690 *
691 * Return: enum netdev_tx
692 */
693static enum netdev_tx emac_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev)
694{
695 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
696 struct prueth_emac *emac = netdev_priv(ndev);
697 struct netdev_queue *netif_txq;
698 struct prueth_tx_chn *tx_chn;
699 dma_addr_t desc_dma, buf_dma;
700 int i, ret = 0, q_idx;
186734c1
RQ		 701 bool in_tx_ts = 0;
702 int tx_ts_cookie;
128d5874
RQ		 703 void **swdata;
704 u32 pkt_len;
705 u32 *epib;
706
707 pkt_len = skb_headlen(skb);
708 q_idx = skb_get_queue_mapping(skb);
709
710 tx_chn = &emac->tx_chns[q_idx];
711 netif_txq = netdev_get_tx_queue(ndev, q_idx);
712
713 /* Map the linear buffer */
714 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len, DMA_TO_DEVICE);
715 if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) {
716 netdev_err(ndev, "tx: failed to map skb buffer\n");
717 ret = NETDEV_TX_OK;
718 goto drop_free_skb;
719 }
720
721 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
722 if (!first_desc) {
723 netdev_dbg(ndev, "tx: failed to allocate descriptor\n");
724 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len, DMA_TO_DEVICE);
725 goto drop_stop_q_busy;
726 }
727
728 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
729 PRUETH_NAV_PS_DATA_SIZE);
730 cppi5_hdesc_set_pkttype(first_desc, 0);
731 epib = first_desc->epib;
732 epib[0] = 0;
733 epib[1] = 0;
186734c1
RQ		 734 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
735 emac->tx_ts_enabled) {
736 tx_ts_cookie = prueth_tx_ts_cookie_get(emac);
737 if (tx_ts_cookie >= 0) {
738 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
739 /* Request TX timestamp */
740 epib[0] = (u32)tx_ts_cookie;
741 epib[1] = 0x80000000; /* TX TS request */
742 emac->tx_ts_skb[tx_ts_cookie] = skb_get(skb);
743 in_tx_ts = 1;
744 }
745 }
128d5874
RQ		 746
747 /* set dst tag to indicate internal qid at the firmware which is at
748 * bit8..bit15. bit0..bit7 indicates port num for directed
749 * packets in case of switch mode operation
750 */
751 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, (emac->port_id | (q_idx << 8)));
752 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
753 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
754 swdata = cppi5_hdesc_get_swdata(first_desc);
755 *swdata = skb;
756
757 /* Handle the case where skb is fragmented in pages */
758 cur_desc = first_desc;
759 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
760 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
761 u32 frag_size = skb_frag_size(frag);
762
763 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
764 if (!next_desc) {
765 netdev_err(ndev,
766 "tx: failed to allocate frag. descriptor\n");
186734c1 767 goto free_desc_stop_q_busy_cleanup_tx_ts;
128d5874
RQ		 768 }
769
770 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
771 DMA_TO_DEVICE);
772 if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) {
773 netdev_err(ndev, "tx: Failed to map skb page\n");
774 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
775 ret = NETDEV_TX_OK;
186734c1 776 goto cleanup_tx_ts;
128d5874
RQ		 777 }
778
779 cppi5_hdesc_reset_hbdesc(next_desc);
780 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
781 cppi5_hdesc_attach_buf(next_desc,
782 buf_dma, frag_size, buf_dma, frag_size);
783
784 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
785 next_desc);
786 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);
787 cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
788
789 pkt_len += frag_size;
790 cur_desc = next_desc;
791 }
792 WARN_ON_ONCE(pkt_len != skb->len);
793
794 /* report bql before sending packet */
795 netdev_tx_sent_queue(netif_txq, pkt_len);
796
797 cppi5_hdesc_set_pktlen(first_desc, pkt_len);
798 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
799 /* cppi5_desc_dump(first_desc, 64); */
800
801 skb_tx_timestamp(skb); /* SW timestamp if SKBTX_IN_PROGRESS not set */
802 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
803 if (ret) {
804 netdev_err(ndev, "tx: push failed: %d\n", ret);
805 goto drop_free_descs;
806 }
807
186734c1
RQ		 808 if (in_tx_ts)
809 atomic_inc(&emac->tx_ts_pending);
810
128d5874
RQ		 811 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
812 netif_tx_stop_queue(netif_txq);
813 /* Barrier, so that stop_queue visible to other cpus */
814 smp_mb__after_atomic();
815
816 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
817 MAX_SKB_FRAGS)
818 netif_tx_wake_queue(netif_txq);
819 }
820
821 return NETDEV_TX_OK;
822
186734c1
RQ		 823cleanup_tx_ts:
824 if (in_tx_ts) {
825 dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]);
826 emac->tx_ts_skb[tx_ts_cookie] = NULL;
827 }
828
128d5874
RQ		 829drop_free_descs:
830 prueth_xmit_free(tx_chn, first_desc);
831
832drop_free_skb:
833 dev_kfree_skb_any(skb);
834
835 /* error */
836 ndev->stats.tx_dropped++;
837 netdev_err(ndev, "tx: error: %d\n", ret);
838
839 return ret;
840
186734c1
RQ		 841free_desc_stop_q_busy_cleanup_tx_ts:
842 if (in_tx_ts) {
843 dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]);
844 emac->tx_ts_skb[tx_ts_cookie] = NULL;
845 }
128d5874
RQ		 846 prueth_xmit_free(tx_chn, first_desc);
847
848drop_stop_q_busy:
849 netif_tx_stop_queue(netif_txq);
850 return NETDEV_TX_BUSY;
851}
852
853static void prueth_tx_cleanup(void *data, dma_addr_t desc_dma)
854{
855 struct prueth_tx_chn *tx_chn = data;
856 struct cppi5_host_desc_t *desc_tx;
857 struct sk_buff *skb;
858 void **swdata;
859
860 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
861 swdata = cppi5_hdesc_get_swdata(desc_tx);
862 skb = *(swdata);
863 prueth_xmit_free(tx_chn, desc_tx);
864
865 dev_kfree_skb_any(skb);
866}
867
186734c1
RQ		 868static irqreturn_t prueth_tx_ts_irq(int irq, void *dev_id)
869{
870 struct prueth_emac *emac = dev_id;
871
872 /* currently only TX timestamp is being returned */
873 tx_ts_work(emac);
874
875 return IRQ_HANDLED;
876}
877
128d5874
RQ		 878static irqreturn_t prueth_rx_irq(int irq, void *dev_id)
879{
880 struct prueth_emac *emac = dev_id;
881
882 disable_irq_nosync(irq);
883 napi_schedule(&emac->napi_rx);
884
885 return IRQ_HANDLED;
886}
887
888struct icssg_firmwares {
889 char *pru;
890 char *rtu;
891 char *txpru;
892};
893
894static struct icssg_firmwares icssg_emac_firmwares[] = {
895 {
896 .pru = "ti-pruss/am65x-sr2-pru0-prueth-fw.elf",
897 .rtu = "ti-pruss/am65x-sr2-rtu0-prueth-fw.elf",
898 .txpru = "ti-pruss/am65x-sr2-txpru0-prueth-fw.elf",
899 },
900 {
901 .pru = "ti-pruss/am65x-sr2-pru1-prueth-fw.elf",
902 .rtu = "ti-pruss/am65x-sr2-rtu1-prueth-fw.elf",
903 .txpru = "ti-pruss/am65x-sr2-txpru1-prueth-fw.elf",
904 }
905};
906
907static int prueth_emac_start(struct prueth *prueth, struct prueth_emac *emac)
908{
909 struct icssg_firmwares *firmwares;
910 struct device *dev = prueth->dev;
911 int slice, ret;
912
913 firmwares = icssg_emac_firmwares;
914
915 slice = prueth_emac_slice(emac);
916 if (slice < 0) {
917 netdev_err(emac->ndev, "invalid port\n");
918 return -EINVAL;
919 }
920
921 ret = icssg_config(prueth, emac, slice);
922 if (ret)
923 return ret;
924
925 ret = rproc_set_firmware(prueth->pru[slice], firmwares[slice].pru);
926 ret = rproc_boot(prueth->pru[slice]);
927 if (ret) {
928 dev_err(dev, "failed to boot PRU%d: %d\n", slice, ret);
929 return -EINVAL;
930 }
931
932 ret = rproc_set_firmware(prueth->rtu[slice], firmwares[slice].rtu);
933 ret = rproc_boot(prueth->rtu[slice]);
934 if (ret) {
935 dev_err(dev, "failed to boot RTU%d: %d\n", slice, ret);
936 goto halt_pru;
937 }
938
939 ret = rproc_set_firmware(prueth->txpru[slice], firmwares[slice].txpru);
940 ret = rproc_boot(prueth->txpru[slice]);
941 if (ret) {
942 dev_err(dev, "failed to boot TX_PRU%d: %d\n", slice, ret);
943 goto halt_rtu;
944 }
945
946 emac->fw_running = 1;
947 return 0;
948
949halt_rtu:
950 rproc_shutdown(prueth->rtu[slice]);
951
952halt_pru:
953 rproc_shutdown(prueth->pru[slice]);
954
955 return ret;
956}
957
958static void prueth_emac_stop(struct prueth_emac *emac)
959{
960 struct prueth *prueth = emac->prueth;
961 int slice;
962
963 switch (emac->port_id) {
964 case PRUETH_PORT_MII0:
965 slice = ICSS_SLICE0;
966 break;
967 case PRUETH_PORT_MII1:
968 slice = ICSS_SLICE1;
969 break;
970 default:
971 netdev_err(emac->ndev, "invalid port\n");
972 return;
973 }
974
975 emac->fw_running = 0;
976 rproc_shutdown(prueth->txpru[slice]);
977 rproc_shutdown(prueth->rtu[slice]);
978 rproc_shutdown(prueth->pru[slice]);
979}
980
186734c1
RQ		 981static void prueth_cleanup_tx_ts(struct prueth_emac *emac)
982{
983 int i;
984
985 for (i = 0; i < PRUETH_MAX_TX_TS_REQUESTS; i++) {
986 if (emac->tx_ts_skb[i]) {
987 dev_kfree_skb_any(emac->tx_ts_skb[i]);
988 emac->tx_ts_skb[i] = NULL;
989 }
990 }
991}
992
128d5874
RQ		 993/* called back by PHY layer if there is change in link state of hw port*/
994static void emac_adjust_link(struct net_device *ndev)
995{
996 struct prueth_emac *emac = netdev_priv(ndev);
997 struct phy_device *phydev = ndev->phydev;
998 struct prueth *prueth = emac->prueth;
999 bool new_state = false;
1000 unsigned long flags;
1001
1002 if (phydev->link) {
1003 /* check the mode of operation - full/half duplex */
1004 if (phydev->duplex != emac->duplex) {
1005 new_state = true;
1006 emac->duplex = phydev->duplex;
1007 }
1008 if (phydev->speed != emac->speed) {
1009 new_state = true;
1010 emac->speed = phydev->speed;
1011 }
1012 if (!emac->link) {
1013 new_state = true;
1014 emac->link = 1;
1015 }
1016 } else if (emac->link) {
1017 new_state = true;
1018 emac->link = 0;
1019
1020 /* f/w should support 100 & 1000 */
1021 emac->speed = SPEED_1000;
1022
1023 /* half duplex may not be supported by f/w */
1024 emac->duplex = DUPLEX_FULL;
1025 }
1026
1027 if (new_state) {
1028 phy_print_status(phydev);
1029
1030 /* update RGMII and MII configuration based on PHY negotiated
1031 * values
1032 */
1033 if (emac->link) {
0a205f0f
MDA		 1034 if (emac->duplex == DUPLEX_HALF)
1035 icssg_config_half_duplex(emac);
128d5874
RQ		 1036 /* Set the RGMII cfg for gig en and full duplex */
1037 icssg_update_rgmii_cfg(prueth->miig_rt, emac);
1038
1039 /* update the Tx IPG based on 100M/1G speed */
1040 spin_lock_irqsave(&emac->lock, flags);
1041 icssg_config_ipg(emac);
1042 spin_unlock_irqrestore(&emac->lock, flags);
1043 icssg_config_set_speed(emac);
1044 emac_set_port_state(emac, ICSSG_EMAC_PORT_FORWARD);
1045
1046 } else {
1047 emac_set_port_state(emac, ICSSG_EMAC_PORT_DISABLE);
1048 }
1049 }
1050
1051 if (emac->link) {
1052 /* reactivate the transmit queue */
1053 netif_tx_wake_all_queues(ndev);
1054 } else {
1055 netif_tx_stop_all_queues(ndev);
186734c1 1056 prueth_cleanup_tx_ts(emac);
128d5874
RQ		 1057 }
1058}
1059
1060static int emac_napi_rx_poll(struct napi_struct *napi_rx, int budget)
1061{
1062 struct prueth_emac *emac = prueth_napi_to_emac(napi_rx);
1063 int rx_flow = PRUETH_RX_FLOW_DATA;
1064 int flow = PRUETH_MAX_RX_FLOWS;
1065 int num_rx = 0;
1066 int cur_budget;
1067 int ret;
1068
1069 while (flow--) {
1070 cur_budget = budget - num_rx;
1071
1072 while (cur_budget--) {
1073 ret = emac_rx_packet(emac, flow);
1074 if (ret)
1075 break;
1076 num_rx++;
1077 }
1078
1079 if (num_rx >= budget)
1080 break;
1081 }
1082
1083 if (num_rx < budget && napi_complete_done(napi_rx, num_rx))
1084 enable_irq(emac->rx_chns.irq[rx_flow]);
1085
1086 return num_rx;
1087}
1088
1089static int prueth_prepare_rx_chan(struct prueth_emac *emac,
1090 struct prueth_rx_chn *chn,
1091 int buf_size)
1092{
1093 struct sk_buff *skb;
1094 int i, ret;
1095
1096 for (i = 0; i < chn->descs_num; i++) {
1097 skb = __netdev_alloc_skb_ip_align(NULL, buf_size, GFP_KERNEL);
1098 if (!skb)
1099 return -ENOMEM;
1100
1101 ret = prueth_dma_rx_push(emac, skb, chn);
1102 if (ret < 0) {
1103 netdev_err(emac->ndev,
1104 "cannot submit skb for rx chan %s ret %d\n",
1105 chn->name, ret);
1106 kfree_skb(skb);
1107 return ret;
1108 }
1109 }
1110
1111 return 0;
1112}
1113
1114static void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num,
1115 bool free_skb)
1116{
1117 int i;
1118
1119 for (i = 0; i < ch_num; i++) {
1120 if (free_skb)
1121 k3_udma_glue_reset_tx_chn(emac->tx_chns[i].tx_chn,
1122 &emac->tx_chns[i],
1123 prueth_tx_cleanup);
1124 k3_udma_glue_disable_tx_chn(emac->tx_chns[i].tx_chn);
1125 }
1126}
1127
1128static void prueth_reset_rx_chan(struct prueth_rx_chn *chn,
1129 int num_flows, bool disable)
1130{
1131 int i;
1132
1133 for (i = 0; i < num_flows; i++)
1134 k3_udma_glue_reset_rx_chn(chn->rx_chn, i, chn,
1135 prueth_rx_cleanup, !!i);
1136 if (disable)
1137 k3_udma_glue_disable_rx_chn(chn->rx_chn);
1138}
1139
1140static int emac_phy_connect(struct prueth_emac *emac)
1141{
1142 struct prueth *prueth = emac->prueth;
1143 struct net_device *ndev = emac->ndev;
1144 /* connect PHY */
1145 ndev->phydev = of_phy_connect(emac->ndev, emac->phy_node,
1146 &emac_adjust_link, 0,
1147 emac->phy_if);
1148 if (!ndev->phydev) {
1149 dev_err(prueth->dev, "couldn't connect to phy %s\n",
1150 emac->phy_node->full_name);
1151 return -ENODEV;
1152 }
1153
0a205f0f
MDA		 1154 if (!emac->half_duplex) {
1155 dev_dbg(prueth->dev, "half duplex mode is not supported\n");
1156 phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
1157 phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);
1158 }
1159
128d5874 1160 /* remove unsupported modes */
128d5874
RQ		 1161 phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
1162 phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_Pause_BIT);
1163 phy_remove_link_mode(ndev->phydev, ETHTOOL_LINK_MODE_Asym_Pause_BIT);
1164
1165 if (emac->phy_if == PHY_INTERFACE_MODE_MII)
1166 phy_set_max_speed(ndev->phydev, SPEED_100);
1167
1168 return 0;
1169}
1170
186734c1
RQ		 1171static u64 prueth_iep_gettime(void *clockops_data, struct ptp_system_timestamp *sts)
1172{
1173 u32 hi_rollover_count, hi_rollover_count_r;
1174 struct prueth_emac *emac = clockops_data;
1175 struct prueth *prueth = emac->prueth;
1176 void __iomem *fw_hi_r_count_addr;
1177 void __iomem *fw_count_hi_addr;
1178 u32 iepcount_hi, iepcount_hi_r;
1179 unsigned long flags;
1180 u32 iepcount_lo;
1181 u64 ts = 0;
1182
1183 fw_count_hi_addr = prueth->shram.va + TIMESYNC_FW_WC_COUNT_HI_SW_OFFSET_OFFSET;
1184 fw_hi_r_count_addr = prueth->shram.va + TIMESYNC_FW_WC_HI_ROLLOVER_COUNT_OFFSET;
1185
1186 local_irq_save(flags);
1187 do {
1188 iepcount_hi = icss_iep_get_count_hi(emac->iep);
1189 iepcount_hi += readl(fw_count_hi_addr);
1190 hi_rollover_count = readl(fw_hi_r_count_addr);
1191 ptp_read_system_prets(sts);
1192 iepcount_lo = icss_iep_get_count_low(emac->iep);
1193 ptp_read_system_postts(sts);
1194
1195 iepcount_hi_r = icss_iep_get_count_hi(emac->iep);
1196 iepcount_hi_r += readl(fw_count_hi_addr);
1197 hi_rollover_count_r = readl(fw_hi_r_count_addr);
1198 } while ((iepcount_hi_r != iepcount_hi) ||
1199 (hi_rollover_count != hi_rollover_count_r));
1200 local_irq_restore(flags);
1201
1202 ts = ((u64)hi_rollover_count) << 23 | iepcount_hi;
1203 ts = ts * (u64)IEP_DEFAULT_CYCLE_TIME_NS + iepcount_lo;
1204
1205 return ts;
1206}
1207
1208static void prueth_iep_settime(void *clockops_data, u64 ns)
1209{
1210 struct icssg_setclock_desc __iomem *sc_descp;
1211 struct prueth_emac *emac = clockops_data;
1212 struct icssg_setclock_desc sc_desc;
1213 u64 cyclecount;
1214 u32 cycletime;
1215 int timeout;
1216
1217 if (!emac->fw_running)
1218 return;
1219
1220 sc_descp = emac->prueth->shram.va + TIMESYNC_FW_WC_SETCLOCK_DESC_OFFSET;
1221
1222 cycletime = IEP_DEFAULT_CYCLE_TIME_NS;
1223 cyclecount = ns / cycletime;
1224
1225 memset(&sc_desc, 0, sizeof(sc_desc));
1226 sc_desc.margin = cycletime - 1000;
1227 sc_desc.cyclecounter0_set = cyclecount & GENMASK(31, 0);
1228 sc_desc.cyclecounter1_set = (cyclecount & GENMASK(63, 32)) >> 32;
1229 sc_desc.iepcount_set = ns % cycletime;
1230 sc_desc.CMP0_current = cycletime - 4; //Count from 0 to (cycle time)-4
1231
1232 memcpy_toio(sc_descp, &sc_desc, sizeof(sc_desc));
1233
1234 writeb(1, &sc_descp->request);
1235
1236 timeout = 5; /* fw should take 2-3 ms */
1237 while (timeout--) {
1238 if (readb(&sc_descp->acknowledgment))
1239 return;
1240
1241 usleep_range(500, 1000);
1242 }
1243
1244 dev_err(emac->prueth->dev, "settime timeout\n");
1245}
1246
1247static int prueth_perout_enable(void *clockops_data,
1248 struct ptp_perout_request *req, int on,
1249 u64 *cmp)
1250{
1251 struct prueth_emac *emac = clockops_data;
1252 u32 reduction_factor = 0, offset = 0;
1253 struct timespec64 ts;
1254 u64 ns_period;
1255
1256 if (!on)
1257 return 0;
1258
1259 /* Any firmware specific stuff for PPS/PEROUT handling */
1260 ts.tv_sec = req->period.sec;
1261 ts.tv_nsec = req->period.nsec;
1262 ns_period = timespec64_to_ns(&ts);
1263
1264 /* f/w doesn't support period less than cycle time */
1265 if (ns_period < IEP_DEFAULT_CYCLE_TIME_NS)
1266 return -ENXIO;
1267
1268 reduction_factor = ns_period / IEP_DEFAULT_CYCLE_TIME_NS;
1269 offset = ns_period % IEP_DEFAULT_CYCLE_TIME_NS;
1270
1271 /* f/w requires at least 1uS within a cycle so CMP
1272 * can trigger after SYNC is enabled
1273 */
1274 if (offset < 5 * NSEC_PER_USEC)
1275 offset = 5 * NSEC_PER_USEC;
1276
1277 /* if offset is close to cycle time then we will miss
1278 * the CMP event for last tick when IEP rolls over.
1279 * In normal mode, IEP tick is 4ns.
1280 * In slow compensation it could be 0ns or 8ns at
1281 * every slow compensation cycle.
1282 */
1283 if (offset > IEP_DEFAULT_CYCLE_TIME_NS - 8)
1284 offset = IEP_DEFAULT_CYCLE_TIME_NS - 8;
1285
1286 /* we're in shadow mode so need to set upper 32-bits */
1287 *cmp = (u64)offset << 32;
1288
1289 writel(reduction_factor, emac->prueth->shram.va +
1290 TIMESYNC_FW_WC_SYNCOUT_REDUCTION_FACTOR_OFFSET);
1291
1292 writel(0, emac->prueth->shram.va +
1293 TIMESYNC_FW_WC_SYNCOUT_START_TIME_CYCLECOUNT_OFFSET);
1294
1295 return 0;
1296}
1297
1298const struct icss_iep_clockops prueth_iep_clockops = {
1299 .settime = prueth_iep_settime,
1300 .gettime = prueth_iep_gettime,
1301 .perout_enable = prueth_perout_enable,
1302};
1303
128d5874
RQ		 1304/**
1305 * emac_ndo_open - EMAC device open
1306 * @ndev: network adapter device
1307 *
1308 * Called when system wants to start the interface.
1309 *
1310 * Return: 0 for a successful open, or appropriate error code
1311 */
1312static int emac_ndo_open(struct net_device *ndev)
1313{
1314 struct prueth_emac *emac = netdev_priv(ndev);
1315 int ret, i, num_data_chn = emac->tx_ch_num;
1316 struct prueth *prueth = emac->prueth;
1317 int slice = prueth_emac_slice(emac);
1318 struct device *dev = prueth->dev;
1319 int max_rx_flows;
1320 int rx_flow;
1321
1322 /* clear SMEM and MSMC settings for all slices */
1323 if (!prueth->emacs_initialized) {
1324 memset_io(prueth->msmcram.va, 0, prueth->msmcram.size);
1325 memset_io(prueth->shram.va, 0, ICSSG_CONFIG_OFFSET_SLICE1 * PRUETH_NUM_MACS);
1326 }
1327
1328 /* set h/w MAC as user might have re-configured */
1329 ether_addr_copy(emac->mac_addr, ndev->dev_addr);
1330
1331 icssg_class_set_mac_addr(prueth->miig_rt, slice, emac->mac_addr);
1332 icssg_ft1_set_mac_addr(prueth->miig_rt, slice, emac->mac_addr);
1333
1334 icssg_class_default(prueth->miig_rt, slice, 0);
1335
1336 /* Notify the stack of the actual queue counts. */
1337 ret = netif_set_real_num_tx_queues(ndev, num_data_chn);
1338 if (ret) {
1339 dev_err(dev, "cannot set real number of tx queues\n");
1340 return ret;
1341 }
1342
1343 init_completion(&emac->cmd_complete);
1344 ret = prueth_init_tx_chns(emac);
1345 if (ret) {
1346 dev_err(dev, "failed to init tx channel: %d\n", ret);
1347 return ret;
1348 }
1349
1350 max_rx_flows = PRUETH_MAX_RX_FLOWS;
1351 ret = prueth_init_rx_chns(emac, &emac->rx_chns, "rx",
1352 max_rx_flows, PRUETH_MAX_RX_DESC);
1353 if (ret) {
1354 dev_err(dev, "failed to init rx channel: %d\n", ret);
1355 goto cleanup_tx;
1356 }
1357
1358 ret = prueth_ndev_add_tx_napi(emac);
1359 if (ret)
1360 goto cleanup_rx;
1361
1362 /* we use only the highest priority flow for now i.e. @irq[3] */
1363 rx_flow = PRUETH_RX_FLOW_DATA;
1364 ret = request_irq(emac->rx_chns.irq[rx_flow], prueth_rx_irq,
1365 IRQF_TRIGGER_HIGH, dev_name(dev), emac);
1366 if (ret) {
1367 dev_err(dev, "unable to request RX IRQ\n");
1368 goto cleanup_napi;
1369 }
1370
1371 /* reset and start PRU firmware */
1372 ret = prueth_emac_start(prueth, emac);
1373 if (ret)
1374 goto free_rx_irq;
1375
1376 icssg_mii_update_mtu(prueth->mii_rt, slice, ndev->max_mtu);
1377
186734c1
RQ		 1378 if (!prueth->emacs_initialized) {
1379 ret = icss_iep_init(emac->iep, &prueth_iep_clockops,
1380 emac, IEP_DEFAULT_CYCLE_TIME_NS);
1381 }
1382
1383 ret = request_threaded_irq(emac->tx_ts_irq, NULL, prueth_tx_ts_irq,
1384 IRQF_ONESHOT, dev_name(dev), emac);
1385 if (ret)
1386 goto stop;
1387
128d5874
RQ		 1388 /* Prepare RX */
1389 ret = prueth_prepare_rx_chan(emac, &emac->rx_chns, PRUETH_MAX_PKT_SIZE);
1390 if (ret)
186734c1 1391 goto free_tx_ts_irq;
128d5874
RQ		 1392
1393 ret = k3_udma_glue_enable_rx_chn(emac->rx_chns.rx_chn);
1394 if (ret)
1395 goto reset_rx_chn;
1396
1397 for (i = 0; i < emac->tx_ch_num; i++) {
1398 ret = k3_udma_glue_enable_tx_chn(emac->tx_chns[i].tx_chn);
1399 if (ret)
1400 goto reset_tx_chan;
1401 }
1402
1403 /* Enable NAPI in Tx and Rx direction */
1404 for (i = 0; i < emac->tx_ch_num; i++)
1405 napi_enable(&emac->tx_chns[i].napi_tx);
1406 napi_enable(&emac->napi_rx);
1407
1408 /* start PHY */
1409 phy_start(ndev->phydev);
1410
1411 prueth->emacs_initialized++;
1412
c1e10d5d
MDA		 1413 queue_work(system_long_wq, &emac->stats_work.work);
1414
128d5874
RQ		 1415 return 0;
1416
1417reset_tx_chan:
1418 /* Since interface is not yet up, there is wouldn't be
1419 * any SKB for completion. So set false to free_skb
1420 */
1421 prueth_reset_tx_chan(emac, i, false);
1422reset_rx_chn:
1423 prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, false);
186734c1
RQ		 1424free_tx_ts_irq:
1425 free_irq(emac->tx_ts_irq, emac);
128d5874
RQ		 1426stop:
1427 prueth_emac_stop(emac);
1428free_rx_irq:
1429 free_irq(emac->rx_chns.irq[rx_flow], emac);
1430cleanup_napi:
1431 prueth_ndev_del_tx_napi(emac, emac->tx_ch_num);
1432cleanup_rx:
1433 prueth_cleanup_rx_chns(emac, &emac->rx_chns, max_rx_flows);
1434cleanup_tx:
1435 prueth_cleanup_tx_chns(emac);
1436
1437 return ret;
1438}
1439
1440/**
1441 * emac_ndo_stop - EMAC device stop
1442 * @ndev: network adapter device
1443 *
1444 * Called when system wants to stop or down the interface.
1445 *
1446 * Return: Always 0 (Success)
1447 */
1448static int emac_ndo_stop(struct net_device *ndev)
1449{
1450 struct prueth_emac *emac = netdev_priv(ndev);
1451 struct prueth *prueth = emac->prueth;
1452 int rx_flow = PRUETH_RX_FLOW_DATA;
1453 int max_rx_flows;
1454 int ret, i;
1455
1456 /* inform the upper layers. */
1457 netif_tx_stop_all_queues(ndev);
1458
1459 /* block packets from wire */
1460 if (ndev->phydev)
1461 phy_stop(ndev->phydev);
1462
1463 icssg_class_disable(prueth->miig_rt, prueth_emac_slice(emac));
1464
1465 atomic_set(&emac->tdown_cnt, emac->tx_ch_num);
1466 /* ensure new tdown_cnt value is visible */
1467 smp_mb__after_atomic();
1468 /* tear down and disable UDMA channels */
1469 reinit_completion(&emac->tdown_complete);
1470 for (i = 0; i < emac->tx_ch_num; i++)
1471 k3_udma_glue_tdown_tx_chn(emac->tx_chns[i].tx_chn, false);
1472
1473 ret = wait_for_completion_timeout(&emac->tdown_complete,
1474 msecs_to_jiffies(1000));
1475 if (!ret)
1476 netdev_err(ndev, "tx teardown timeout\n");
1477
1478 prueth_reset_tx_chan(emac, emac->tx_ch_num, true);
1479 for (i = 0; i < emac->tx_ch_num; i++)
1480 napi_disable(&emac->tx_chns[i].napi_tx);
1481
1482 max_rx_flows = PRUETH_MAX_RX_FLOWS;
1483 k3_udma_glue_tdown_rx_chn(emac->rx_chns.rx_chn, true);
1484
1485 prueth_reset_rx_chan(&emac->rx_chns, max_rx_flows, true);
1486
1487 napi_disable(&emac->napi_rx);
1488
1489 cancel_work_sync(&emac->rx_mode_work);
1490
c1e10d5d
MDA		 1491 /* Destroying the queued work in ndo_stop() */
1492 cancel_delayed_work_sync(&emac->stats_work);
1493
186734c1
RQ		 1494 if (prueth->emacs_initialized == 1)
1495 icss_iep_exit(emac->iep);
1496
1497 /* stop PRUs */
1498 prueth_emac_stop(emac);
1499
1500 free_irq(emac->tx_ts_irq, emac);
1501
128d5874
RQ		 1502 free_irq(emac->rx_chns.irq[rx_flow], emac);
1503 prueth_ndev_del_tx_napi(emac, emac->tx_ch_num);
128d5874
RQ		 1504
1505 prueth_cleanup_rx_chns(emac, &emac->rx_chns, max_rx_flows);
1506 prueth_cleanup_tx_chns(emac);
1507
1508 prueth->emacs_initialized--;
1509
1510 return 0;
1511}
1512
1513static void emac_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue)
1514{
1515 ndev->stats.tx_errors++;
1516}
1517
1518static void emac_ndo_set_rx_mode_work(struct work_struct *work)
1519{
1520 struct prueth_emac *emac = container_of(work, struct prueth_emac, rx_mode_work);
1521 struct net_device *ndev = emac->ndev;
1522 bool promisc, allmulti;
1523
1524 if (!netif_running(ndev))
1525 return;
1526
1527 promisc = ndev->flags & IFF_PROMISC;
1528 allmulti = ndev->flags & IFF_ALLMULTI;
1529 emac_set_port_state(emac, ICSSG_EMAC_PORT_UC_FLOODING_DISABLE);
1530 emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_DISABLE);
1531
1532 if (promisc) {
1533 emac_set_port_state(emac, ICSSG_EMAC_PORT_UC_FLOODING_ENABLE);
1534 emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_ENABLE);
1535 return;
1536 }
1537
1538 if (allmulti) {
1539 emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_ENABLE);
1540 return;
1541 }
1542
1543 if (!netdev_mc_empty(ndev)) {
1544 emac_set_port_state(emac, ICSSG_EMAC_PORT_MC_FLOODING_ENABLE);
1545 return;
1546 }
1547}
1548
1549/**
1550 * emac_ndo_set_rx_mode - EMAC set receive mode function
1551 * @ndev: The EMAC network adapter
1552 *
1553 * Called when system wants to set the receive mode of the device.
1554 *
1555 */
1556static void emac_ndo_set_rx_mode(struct net_device *ndev)
1557{
1558 struct prueth_emac *emac = netdev_priv(ndev);
1559
1560 queue_work(emac->cmd_wq, &emac->rx_mode_work);
1561}
1562
186734c1
RQ		 1563static int emac_set_ts_config(struct net_device *ndev, struct ifreq *ifr)
1564{
1565 struct prueth_emac *emac = netdev_priv(ndev);
1566 struct hwtstamp_config config;
1567
1568 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1569 return -EFAULT;
1570
1571 switch (config.tx_type) {
1572 case HWTSTAMP_TX_OFF:
1573 emac->tx_ts_enabled = 0;
1574 break;
1575 case HWTSTAMP_TX_ON:
1576 emac->tx_ts_enabled = 1;
1577 break;
1578 default:
1579 return -ERANGE;
1580 }
1581
1582 switch (config.rx_filter) {
1583 case HWTSTAMP_FILTER_NONE:
1584 emac->rx_ts_enabled = 0;
1585 break;
1586 case HWTSTAMP_FILTER_ALL:
1587 case HWTSTAMP_FILTER_SOME:
1588 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1589 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1590 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1591 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1592 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1593 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1594 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1595 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1596 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1597 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1598 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1599 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1600 case HWTSTAMP_FILTER_NTP_ALL:
1601 emac->rx_ts_enabled = 1;
1602 config.rx_filter = HWTSTAMP_FILTER_ALL;
1603 break;
1604 default:
1605 return -ERANGE;
1606 }
1607
1608 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1609 -EFAULT : 0;
1610}
1611
1612static int emac_get_ts_config(struct net_device *ndev, struct ifreq *ifr)
1613{
1614 struct prueth_emac *emac = netdev_priv(ndev);
1615 struct hwtstamp_config config;
1616
1617 config.flags = 0;
1618 config.tx_type = emac->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1619 config.rx_filter = emac->rx_ts_enabled ? HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1620
1621 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1622 -EFAULT : 0;
1623}
1624
128d5874
RQ		 1625static int emac_ndo_ioctl(struct net_device *ndev, struct ifreq *ifr, int cmd)
1626{
186734c1
RQ		 1627 switch (cmd) {
1628 case SIOCGHWTSTAMP:
1629 return emac_get_ts_config(ndev, ifr);
1630 case SIOCSHWTSTAMP:
1631 return emac_set_ts_config(ndev, ifr);
1632 default:
1633 break;
1634 }
1635
128d5874
RQ		 1636 return phy_do_ioctl(ndev, ifr, cmd);
1637}
1638
c2f67d19
MDA		 1639static void emac_ndo_get_stats64(struct net_device *ndev,
1640 struct rtnl_link_stats64 *stats)
1641{
1642 struct prueth_emac *emac = netdev_priv(ndev);
1643
1644 emac_update_hardware_stats(emac);
1645
1646 stats->rx_packets = emac_get_stat_by_name(emac, "rx_packets");
1647 stats->rx_bytes = emac_get_stat_by_name(emac, "rx_bytes");
1648 stats->tx_packets = emac_get_stat_by_name(emac, "tx_packets");
1649 stats->tx_bytes = emac_get_stat_by_name(emac, "tx_bytes");
1650 stats->rx_crc_errors = emac_get_stat_by_name(emac, "rx_crc_errors");
1651 stats->rx_over_errors = emac_get_stat_by_name(emac, "rx_over_errors");
1652 stats->multicast = emac_get_stat_by_name(emac, "rx_multicast_frames");
1653
1654 stats->rx_errors = ndev->stats.rx_errors;
1655 stats->rx_dropped = ndev->stats.rx_dropped;
1656 stats->tx_errors = ndev->stats.tx_errors;
1657 stats->tx_dropped = ndev->stats.tx_dropped;
1658}
1659
f6e12277
JK		 1660static int emac_ndo_get_phys_port_name(struct net_device *ndev, char *name,
1661 size_t len)
1662{
1663 struct prueth_emac *emac = netdev_priv(ndev);
1664 int ret;
1665
1666 ret = snprintf(name, len, "p%d", emac->port_id);
1667 if (ret >= len)
1668 return -EINVAL;
1669
1670 return 0;
1671}
1672
128d5874
RQ		 1673static const struct net_device_ops emac_netdev_ops = {
1674 .ndo_open = emac_ndo_open,
1675 .ndo_stop = emac_ndo_stop,
1676 .ndo_start_xmit = emac_ndo_start_xmit,
1677 .ndo_set_mac_address = eth_mac_addr,
1678 .ndo_validate_addr = eth_validate_addr,
1679 .ndo_tx_timeout = emac_ndo_tx_timeout,
1680 .ndo_set_rx_mode = emac_ndo_set_rx_mode,
1681 .ndo_eth_ioctl = emac_ndo_ioctl,
c2f67d19 1682 .ndo_get_stats64 = emac_ndo_get_stats64,
f6e12277 1683 .ndo_get_phys_port_name = emac_ndo_get_phys_port_name,
128d5874
RQ		 1684};
1685
1686/* get emac_port corresponding to eth_node name */
1687static int prueth_node_port(struct device_node *eth_node)
1688{
1689 u32 port_id;
1690 int ret;
1691
1692 ret = of_property_read_u32(eth_node, "reg", &port_id);
1693 if (ret)
1694 return ret;
1695
1696 if (port_id == 0)
1697 return PRUETH_PORT_MII0;
1698 else if (port_id == 1)
1699 return PRUETH_PORT_MII1;
1700 else
1701 return PRUETH_PORT_INVALID;
1702}
1703
1704/* get MAC instance corresponding to eth_node name */
1705static int prueth_node_mac(struct device_node *eth_node)
1706{
1707 u32 port_id;
1708 int ret;
1709
1710 ret = of_property_read_u32(eth_node, "reg", &port_id);
1711 if (ret)
1712 return ret;
1713
1714 if (port_id == 0)
1715 return PRUETH_MAC0;
1716 else if (port_id == 1)
1717 return PRUETH_MAC1;
1718 else
1719 return PRUETH_MAC_INVALID;
1720}
1721
1722static int prueth_netdev_init(struct prueth *prueth,
1723 struct device_node *eth_node)
1724{
1725 int ret, num_tx_chn = PRUETH_MAX_TX_QUEUES;
1726 struct prueth_emac *emac;
1727 struct net_device *ndev;
1728 enum prueth_port port;
186734c1 1729 const char *irq_name;
128d5874
RQ		 1730 enum prueth_mac mac;
1731
1732 port = prueth_node_port(eth_node);
1733 if (port == PRUETH_PORT_INVALID)
1734 return -EINVAL;
1735
1736 mac = prueth_node_mac(eth_node);
1737 if (mac == PRUETH_MAC_INVALID)
1738 return -EINVAL;
1739
1740 ndev = alloc_etherdev_mq(sizeof(*emac), num_tx_chn);
1741 if (!ndev)
1742 return -ENOMEM;
1743
1744 emac = netdev_priv(ndev);
1745 emac->prueth = prueth;
1746 emac->ndev = ndev;
1747 emac->port_id = port;
1748 emac->cmd_wq = create_singlethread_workqueue("icssg_cmd_wq");
1749 if (!emac->cmd_wq) {
1750 ret = -ENOMEM;
1751 goto free_ndev;
1752 }
1753 INIT_WORK(&emac->rx_mode_work, emac_ndo_set_rx_mode_work);
1754
c1e10d5d
MDA		 1755 INIT_DELAYED_WORK(&emac->stats_work, emac_stats_work_handler);
1756
128d5874
RQ		 1757 ret = pruss_request_mem_region(prueth->pruss,
1758 port == PRUETH_PORT_MII0 ?
1759 PRUSS_MEM_DRAM0 : PRUSS_MEM_DRAM1,
1760 &emac->dram);
1761 if (ret) {
1762 dev_err(prueth->dev, "unable to get DRAM: %d\n", ret);
1763 ret = -ENOMEM;
1764 goto free_wq;
1765 }
1766
1767 emac->tx_ch_num = 1;
1768
186734c1
RQ		 1769 irq_name = "tx_ts0";
1770 if (emac->port_id == PRUETH_PORT_MII1)
1771 irq_name = "tx_ts1";
1772 emac->tx_ts_irq = platform_get_irq_byname_optional(prueth->pdev, irq_name);
1773 if (emac->tx_ts_irq < 0) {
1774 ret = dev_err_probe(prueth->dev, emac->tx_ts_irq, "could not get tx_ts_irq\n");
1775 goto free;
1776 }
1777
128d5874
RQ		 1778 SET_NETDEV_DEV(ndev, prueth->dev);
1779 spin_lock_init(&emac->lock);
1780 mutex_init(&emac->cmd_lock);
1781
1782 emac->phy_node = of_parse_phandle(eth_node, "phy-handle", 0);
1783 if (!emac->phy_node && !of_phy_is_fixed_link(eth_node)) {
1784 dev_err(prueth->dev, "couldn't find phy-handle\n");
1785 ret = -ENODEV;
1786 goto free;
1787 } else if (of_phy_is_fixed_link(eth_node)) {
1788 ret = of_phy_register_fixed_link(eth_node);
1789 if (ret) {
1790 ret = dev_err_probe(prueth->dev, ret,
1791 "failed to register fixed-link phy\n");
1792 goto free;
1793 }
1794
1795 emac->phy_node = eth_node;
1796 }
1797
1798 ret = of_get_phy_mode(eth_node, &emac->phy_if);
1799 if (ret) {
1800 dev_err(prueth->dev, "could not get phy-mode property\n");
1801 goto free;
1802 }
1803
1804 if (emac->phy_if != PHY_INTERFACE_MODE_MII &&
1805 !phy_interface_mode_is_rgmii(emac->phy_if)) {
1806 dev_err(prueth->dev, "PHY mode unsupported %s\n", phy_modes(emac->phy_if));
1807 ret = -EINVAL;
1808 goto free;
1809 }
1810
1811 /* AM65 SR2.0 has TX Internal delay always enabled by hardware
1812 * and it is not possible to disable TX Internal delay. The below
1813 * switch case block describes how we handle different phy modes
1814 * based on hardware restriction.
1815 */
1816 switch (emac->phy_if) {
1817 case PHY_INTERFACE_MODE_RGMII_ID:
1818 emac->phy_if = PHY_INTERFACE_MODE_RGMII_RXID;
1819 break;
1820 case PHY_INTERFACE_MODE_RGMII_TXID:
1821 emac->phy_if = PHY_INTERFACE_MODE_RGMII;
1822 break;
1823 case PHY_INTERFACE_MODE_RGMII:
1824 case PHY_INTERFACE_MODE_RGMII_RXID:
1825 dev_err(prueth->dev, "RGMII mode without TX delay is not supported");
1826 ret = -EINVAL;
1827 goto free;
1828 default:
1829 break;
1830 }
1831
1832 /* get mac address from DT and set private and netdev addr */
1833 ret = of_get_ethdev_address(eth_node, ndev);
1834 if (!is_valid_ether_addr(ndev->dev_addr)) {
1835 eth_hw_addr_random(ndev);
1836 dev_warn(prueth->dev, "port %d: using random MAC addr: %pM\n",
1837 port, ndev->dev_addr);
1838 }
1839 ether_addr_copy(emac->mac_addr, ndev->dev_addr);
1840
1841 ndev->min_mtu = PRUETH_MIN_PKT_SIZE;
1842 ndev->max_mtu = PRUETH_MAX_MTU;
1843 ndev->netdev_ops = &emac_netdev_ops;
8fb86b0d 1844 ndev->ethtool_ops = &icssg_ethtool_ops;
128d5874
RQ		 1845 ndev->hw_features = NETIF_F_SG;
1846 ndev->features = ndev->hw_features;
1847
1848 netif_napi_add(ndev, &emac->napi_rx, emac_napi_rx_poll);
1849 prueth->emac[mac] = emac;
1850
1851 return 0;
1852
1853free:
1854 pruss_release_mem_region(prueth->pruss, &emac->dram);
1855free_wq:
1856 destroy_workqueue(emac->cmd_wq);
1857free_ndev:
1858 emac->ndev = NULL;
1859 prueth->emac[mac] = NULL;
1860 free_netdev(ndev);
1861
1862 return ret;
1863}
1864
1865static void prueth_netdev_exit(struct prueth *prueth,
1866 struct device_node *eth_node)
1867{
1868 struct prueth_emac *emac;
1869 enum prueth_mac mac;
1870
1871 mac = prueth_node_mac(eth_node);
1872 if (mac == PRUETH_MAC_INVALID)
1873 return;
1874
1875 emac = prueth->emac[mac];
1876 if (!emac)
1877 return;
1878
1879 if (of_phy_is_fixed_link(emac->phy_node))
1880 of_phy_deregister_fixed_link(emac->phy_node);
1881
1882 netif_napi_del(&emac->napi_rx);
1883
1884 pruss_release_mem_region(prueth->pruss, &emac->dram);
1885 destroy_workqueue(emac->cmd_wq);
1886 free_netdev(emac->ndev);
1887 prueth->emac[mac] = NULL;
1888}
1889
1890static int prueth_get_cores(struct prueth *prueth, int slice)
1891{
1892 struct device *dev = prueth->dev;
1893 enum pruss_pru_id pruss_id;
1894 struct device_node *np;
1895 int idx = -1, ret;
1896
1897 np = dev->of_node;
1898
1899 switch (slice) {
1900 case ICSS_SLICE0:
1901 idx = 0;
1902 break;
1903 case ICSS_SLICE1:
1904 idx = 3;
1905 break;
1906 default:
1907 return -EINVAL;
1908 }
1909
1910 prueth->pru[slice] = pru_rproc_get(np, idx, &pruss_id);
1911 if (IS_ERR(prueth->pru[slice])) {
1912 ret = PTR_ERR(prueth->pru[slice]);
1913 prueth->pru[slice] = NULL;
1914 return dev_err_probe(dev, ret, "unable to get PRU%d\n", slice);
1915 }
1916 prueth->pru_id[slice] = pruss_id;
1917
1918 idx++;
1919 prueth->rtu[slice] = pru_rproc_get(np, idx, NULL);
1920 if (IS_ERR(prueth->rtu[slice])) {
1921 ret = PTR_ERR(prueth->rtu[slice]);
1922 prueth->rtu[slice] = NULL;
1923 return dev_err_probe(dev, ret, "unable to get RTU%d\n", slice);
1924 }
1925
1926 idx++;
1927 prueth->txpru[slice] = pru_rproc_get(np, idx, NULL);
1928 if (IS_ERR(prueth->txpru[slice])) {
1929 ret = PTR_ERR(prueth->txpru[slice]);
1930 prueth->txpru[slice] = NULL;
1931 return dev_err_probe(dev, ret, "unable to get TX_PRU%d\n", slice);
1932 }
1933
1934 return 0;
1935}
1936
1937static void prueth_put_cores(struct prueth *prueth, int slice)
1938{
1939 if (prueth->txpru[slice])
1940 pru_rproc_put(prueth->txpru[slice]);
1941
1942 if (prueth->rtu[slice])
1943 pru_rproc_put(prueth->rtu[slice]);
1944
1945 if (prueth->pru[slice])
1946 pru_rproc_put(prueth->pru[slice]);
1947}
1948
128d5874
RQ		 1949static int prueth_probe(struct platform_device *pdev)
1950{
1951 struct device_node *eth_node, *eth_ports_node;
1952 struct device_node *eth0_node = NULL;
1953 struct device_node *eth1_node = NULL;
1954 struct genpool_data_align gp_data = {
1955 .align = SZ_64K,
1956 };
128d5874
RQ		 1957 struct device *dev = &pdev->dev;
1958 struct device_node *np;
1959 struct prueth *prueth;
1960 struct pruss *pruss;
1961 u32 msmc_ram_size;
1962 int i, ret;
1963
1964 np = dev->of_node;
1965
128d5874
RQ		 1966 prueth = devm_kzalloc(dev, sizeof(*prueth), GFP_KERNEL);
1967 if (!prueth)
1968 return -ENOMEM;
1969
1970 dev_set_drvdata(dev, prueth);
1971 prueth->pdev = pdev;
b0377116 1972 prueth->pdata = *(const struct prueth_pdata *)device_get_match_data(dev);
128d5874
RQ		 1973
1974 prueth->dev = dev;
1975 eth_ports_node = of_get_child_by_name(np, "ethernet-ports");
1976 if (!eth_ports_node)
1977 return -ENOENT;
1978
1979 for_each_child_of_node(eth_ports_node, eth_node) {
1980 u32 reg;
1981
1982 if (strcmp(eth_node->name, "port"))
1983 continue;
1984 ret = of_property_read_u32(eth_node, "reg", &reg);
1985 if (ret < 0) {
1986 dev_err(dev, "%pOF error reading port_id %d\n",
1987 eth_node, ret);
1988 }
1989
1990 of_node_get(eth_node);
1991
1992 if (reg == 0) {
1993 eth0_node = eth_node;
1994 if (!of_device_is_available(eth0_node)) {
1995 of_node_put(eth0_node);
1996 eth0_node = NULL;
1997 }
1998 } else if (reg == 1) {
1999 eth1_node = eth_node;
2000 if (!of_device_is_available(eth1_node)) {
2001 of_node_put(eth1_node);
2002 eth1_node = NULL;
2003 }
2004 } else {
2005 dev_err(dev, "port reg should be 0 or 1\n");
2006 }
2007 }
2008
2009 of_node_put(eth_ports_node);
2010
2011 /* At least one node must be present and available else we fail */
2012 if (!eth0_node && !eth1_node) {
2013 dev_err(dev, "neither port0 nor port1 node available\n");
2014 return -ENODEV;
2015 }
2016
2017 if (eth0_node == eth1_node) {
2018 dev_err(dev, "port0 and port1 can't have same reg\n");
2019 of_node_put(eth0_node);
2020 return -ENODEV;
2021 }
2022
2023 prueth->eth_node[PRUETH_MAC0] = eth0_node;
2024 prueth->eth_node[PRUETH_MAC1] = eth1_node;
2025
2026 prueth->miig_rt = syscon_regmap_lookup_by_phandle(np, "ti,mii-g-rt");
2027 if (IS_ERR(prueth->miig_rt)) {
2028 dev_err(dev, "couldn't get ti,mii-g-rt syscon regmap\n");
2029 return -ENODEV;
2030 }
2031
2032 prueth->mii_rt = syscon_regmap_lookup_by_phandle(np, "ti,mii-rt");
2033 if (IS_ERR(prueth->mii_rt)) {
2034 dev_err(dev, "couldn't get ti,mii-rt syscon regmap\n");
2035 return -ENODEV;
2036 }
2037
2038 if (eth0_node) {
2039 ret = prueth_get_cores(prueth, ICSS_SLICE0);
2040 if (ret)
2041 goto put_cores;
2042 }
2043
2044 if (eth1_node) {
2045 ret = prueth_get_cores(prueth, ICSS_SLICE1);
2046 if (ret)
2047 goto put_cores;
2048 }
2049
2050 pruss = pruss_get(eth0_node ?
2051 prueth->pru[ICSS_SLICE0] : prueth->pru[ICSS_SLICE1]);
2052 if (IS_ERR(pruss)) {
2053 ret = PTR_ERR(pruss);
2054 dev_err(dev, "unable to get pruss handle\n");
2055 goto put_cores;
2056 }
2057
2058 prueth->pruss = pruss;
2059
2060 ret = pruss_request_mem_region(pruss, PRUSS_MEM_SHRD_RAM2,
2061 &prueth->shram);
2062 if (ret) {
2063 dev_err(dev, "unable to get PRUSS SHRD RAM2: %d\n", ret);
2bd5b559 2064 goto put_pruss;
128d5874
RQ		 2065 }
2066
2067 prueth->sram_pool = of_gen_pool_get(np, "sram", 0);
2068 if (!prueth->sram_pool) {
2069 dev_err(dev, "unable to get SRAM pool\n");
2070 ret = -ENODEV;
2071
2072 goto put_mem;
2073 }
2074
2075 msmc_ram_size = MSMC_RAM_SIZE;
2076
2077 /* NOTE: FW bug needs buffer base to be 64KB aligned */
2078 prueth->msmcram.va =
2079 (void __iomem *)gen_pool_alloc_algo(prueth->sram_pool,
2080 msmc_ram_size,
2081 gen_pool_first_fit_align,
2082 &gp_data);
2083
2084 if (!prueth->msmcram.va) {
2085 ret = -ENOMEM;
2086 dev_err(dev, "unable to allocate MSMC resource\n");
2087 goto put_mem;
2088 }
2089 prueth->msmcram.pa = gen_pool_virt_to_phys(prueth->sram_pool,
2090 (unsigned long)prueth->msmcram.va);
2091 prueth->msmcram.size = msmc_ram_size;
2092 memset_io(prueth->msmcram.va, 0, msmc_ram_size);
2093 dev_dbg(dev, "sram: pa %llx va %p size %zx\n", prueth->msmcram.pa,
2094 prueth->msmcram.va, prueth->msmcram.size);
2095
186734c1
RQ		 2096 prueth->iep0 = icss_iep_get_idx(np, 0);
2097 if (IS_ERR(prueth->iep0)) {
2098 ret = dev_err_probe(dev, PTR_ERR(prueth->iep0), "iep0 get failed\n");
2099 prueth->iep0 = NULL;
2100 goto free_pool;
2101 }
2102
443a2367
GS		 2103 prueth->iep1 = icss_iep_get_idx(np, 1);
2104 if (IS_ERR(prueth->iep1)) {
2105 ret = dev_err_probe(dev, PTR_ERR(prueth->iep1), "iep1 get failed\n");
e409d734 2106 goto put_iep0;
443a2367
GS		 2107 }
2108
2109 if (prueth->pdata.quirk_10m_link_issue) {
2110 /* Enable IEP1 for FW in 64bit mode as W/A for 10M FD link detect issue under TX
2111 * traffic.
2112 */
2113 icss_iep_init_fw(prueth->iep1);
2114 }
2115
128d5874
RQ		 2116 /* setup netdev interfaces */
2117 if (eth0_node) {
2118 ret = prueth_netdev_init(prueth, eth0_node);
2119 if (ret) {
2120 dev_err_probe(dev, ret, "netdev init %s failed\n",
2121 eth0_node->name);
443a2367 2122 goto exit_iep;
128d5874 2123 }
0a205f0f
MDA		 2124
2125 if (of_find_property(eth0_node, "ti,half-duplex-capable", NULL))
2126 prueth->emac[PRUETH_MAC0]->half_duplex = 1;
2127
186734c1 2128 prueth->emac[PRUETH_MAC0]->iep = prueth->iep0;
128d5874
RQ		 2129 }
2130
2131 if (eth1_node) {
2132 ret = prueth_netdev_init(prueth, eth1_node);
2133 if (ret) {
2134 dev_err_probe(dev, ret, "netdev init %s failed\n",
2135 eth1_node->name);
2136 goto netdev_exit;
2137 }
186734c1 2138
0a205f0f
MDA		 2139 if (of_find_property(eth1_node, "ti,half-duplex-capable", NULL))
2140 prueth->emac[PRUETH_MAC1]->half_duplex = 1;
2141
186734c1 2142 prueth->emac[PRUETH_MAC1]->iep = prueth->iep0;
128d5874
RQ		 2143 }
2144
2145 /* register the network devices */
2146 if (eth0_node) {
2147 ret = register_netdev(prueth->emac[PRUETH_MAC0]->ndev);
2148 if (ret) {
2149 dev_err(dev, "can't register netdev for port MII0");
2150 goto netdev_exit;
2151 }
2152
2153 prueth->registered_netdevs[PRUETH_MAC0] = prueth->emac[PRUETH_MAC0]->ndev;
2154
2155 emac_phy_connect(prueth->emac[PRUETH_MAC0]);
2156 phy_attached_info(prueth->emac[PRUETH_MAC0]->ndev->phydev);
2157 }
2158
2159 if (eth1_node) {
2160 ret = register_netdev(prueth->emac[PRUETH_MAC1]->ndev);
2161 if (ret) {
2162 dev_err(dev, "can't register netdev for port MII1");
2163 goto netdev_unregister;
2164 }
2165
2166 prueth->registered_netdevs[PRUETH_MAC1] = prueth->emac[PRUETH_MAC1]->ndev;
2167 emac_phy_connect(prueth->emac[PRUETH_MAC1]);
2168 phy_attached_info(prueth->emac[PRUETH_MAC1]->ndev->phydev);
2169 }
2170
2171 dev_info(dev, "TI PRU ethernet driver initialized: %s EMAC mode\n",
2172 (!eth0_node || !eth1_node) ? "single" : "dual");
2173
2174 if (eth1_node)
2175 of_node_put(eth1_node);
2176 if (eth0_node)
2177 of_node_put(eth0_node);
2178 return 0;
2179
2180netdev_unregister:
2181 for (i = 0; i < PRUETH_NUM_MACS; i++) {
2182 if (!prueth->registered_netdevs[i])
2183 continue;
2184 if (prueth->emac[i]->ndev->phydev) {
2185 phy_disconnect(prueth->emac[i]->ndev->phydev);
2186 prueth->emac[i]->ndev->phydev = NULL;
2187 }
2188 unregister_netdev(prueth->registered_netdevs[i]);
2189 }
2190
2191netdev_exit:
2192 for (i = 0; i < PRUETH_NUM_MACS; i++) {
2193 eth_node = prueth->eth_node[i];
2194 if (!eth_node)
2195 continue;
2196
2197 prueth_netdev_exit(prueth, eth_node);
2198 }
2199
443a2367
GS		 2200exit_iep:
2201 if (prueth->pdata.quirk_10m_link_issue)
2202 icss_iep_exit_fw(prueth->iep1);
e409d734
JK		 2203 icss_iep_put(prueth->iep1);
2204
2205put_iep0:
2206 icss_iep_put(prueth->iep0);
2207 prueth->iep0 = NULL;
2208 prueth->iep1 = NULL;
443a2367 2209
186734c1 2210free_pool:
128d5874
RQ		 2211 gen_pool_free(prueth->sram_pool,
2212 (unsigned long)prueth->msmcram.va, msmc_ram_size);
2213
2214put_mem:
2215 pruss_release_mem_region(prueth->pruss, &prueth->shram);
2bd5b559
JK		 2216
2217put_pruss:
128d5874
RQ		 2218 pruss_put(prueth->pruss);
2219
2220put_cores:
2221 if (eth1_node) {
2222 prueth_put_cores(prueth, ICSS_SLICE1);
2223 of_node_put(eth1_node);
2224 }
2225
2226 if (eth0_node) {
2227 prueth_put_cores(prueth, ICSS_SLICE0);
2228 of_node_put(eth0_node);
2229 }
2230
2231 return ret;
2232}
2233
2234static void prueth_remove(struct platform_device *pdev)
2235{
2236 struct prueth *prueth = platform_get_drvdata(pdev);
2237 struct device_node *eth_node;
2238 int i;
2239
2240 for (i = 0; i < PRUETH_NUM_MACS; i++) {
2241 if (!prueth->registered_netdevs[i])
2242 continue;
2243 phy_stop(prueth->emac[i]->ndev->phydev);
2244 phy_disconnect(prueth->emac[i]->ndev->phydev);
2245 prueth->emac[i]->ndev->phydev = NULL;
2246 unregister_netdev(prueth->registered_netdevs[i]);
2247 }
2248
2249 for (i = 0; i < PRUETH_NUM_MACS; i++) {
2250 eth_node = prueth->eth_node[i];
2251 if (!eth_node)
2252 continue;
2253
2254 prueth_netdev_exit(prueth, eth_node);
2255 }
2256
443a2367
GS		 2257 if (prueth->pdata.quirk_10m_link_issue)
2258 icss_iep_exit_fw(prueth->iep1);
2259
2260 icss_iep_put(prueth->iep1);
186734c1
RQ		 2261 icss_iep_put(prueth->iep0);
2262
128d5874
RQ		 2263 gen_pool_free(prueth->sram_pool,
2264 (unsigned long)prueth->msmcram.va,
2265 MSMC_RAM_SIZE);
2266
2267 pruss_release_mem_region(prueth->pruss, &prueth->shram);
2268
2269 pruss_put(prueth->pruss);
2270
2271 if (prueth->eth_node[PRUETH_MAC1])
2272 prueth_put_cores(prueth, ICSS_SLICE1);
2273
2274 if (prueth->eth_node[PRUETH_MAC0])
2275 prueth_put_cores(prueth, ICSS_SLICE0);
2276}
2277
a46750a1
MDA		 2278#ifdef CONFIG_PM_SLEEP
2279static int prueth_suspend(struct device *dev)
2280{
2281 struct prueth *prueth = dev_get_drvdata(dev);
2282 struct net_device *ndev;
2283 int i, ret;
2284
2285 for (i = 0; i < PRUETH_NUM_MACS; i++) {
2286 ndev = prueth->registered_netdevs[i];
2287
2288 if (!ndev)
2289 continue;
2290
2291 if (netif_running(ndev)) {
2292 netif_device_detach(ndev);
2293 ret = emac_ndo_stop(ndev);
2294 if (ret < 0) {
2295 netdev_err(ndev, "failed to stop: %d", ret);
2296 return ret;
2297 }
2298 }
2299 }
2300
2301 return 0;
2302}
2303
2304static int prueth_resume(struct device *dev)
2305{
2306 struct prueth *prueth = dev_get_drvdata(dev);
2307 struct net_device *ndev;
2308 int i, ret;
2309
2310 for (i = 0; i < PRUETH_NUM_MACS; i++) {
2311 ndev = prueth->registered_netdevs[i];
2312
2313 if (!ndev)
2314 continue;
2315
2316 if (netif_running(ndev)) {
2317 ret = emac_ndo_open(ndev);
2318 if (ret < 0) {
2319 netdev_err(ndev, "failed to start: %d", ret);
2320 return ret;
2321 }
2322 netif_device_attach(ndev);
2323 }
2324 }
2325
2326 return 0;
2327}
2328#endif /* CONFIG_PM_SLEEP */
2329
2330static const struct dev_pm_ops prueth_dev_pm_ops = {
2331 SET_SYSTEM_SLEEP_PM_OPS(prueth_suspend, prueth_resume)
2332};
2333
128d5874
RQ		 2334static const struct prueth_pdata am654_icssg_pdata = {
2335 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2336 .quirk_10m_link_issue = 1,
2337};
2338
b256e133
MDA		 2339static const struct prueth_pdata am64x_icssg_pdata = {
2340 .fdqring_mode = K3_RINGACC_RING_MODE_RING,
2341};
2342
128d5874
RQ		 2343static const struct of_device_id prueth_dt_match[] = {
2344 { .compatible = "ti,am654-icssg-prueth", .data = &am654_icssg_pdata },
b256e133 2345 { .compatible = "ti,am642-icssg-prueth", .data = &am64x_icssg_pdata },
128d5874
RQ		 2346 { /* sentinel */ }
2347};
2348MODULE_DEVICE_TABLE(of, prueth_dt_match);
2349
2350static struct platform_driver prueth_driver = {
2351 .probe = prueth_probe,
2352 .remove_new = prueth_remove,
2353 .driver = {
2354 .name = "icssg-prueth",
2355 .of_match_table = prueth_dt_match,
a46750a1 2356 .pm = &prueth_dev_pm_ops,
128d5874
RQ		 2357 },
2358};
2359module_platform_driver(prueth_driver);
2360
2361MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>");
2362MODULE_AUTHOR("Md Danish Anwar <danishanwar@ti.com>");
2363MODULE_DESCRIPTION("PRUSS ICSSG Ethernet Driver");
2364MODULE_LICENSE("GPL");
Linux 6.x block layer and io_uring tree(s)