projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
dev_ioctl: split out ndo_eth_ioctl
[linux-2.6-block.git] / drivers / net / ethernet / marvell / octeontx2 / nic / otx2_pf.c
CommitLineData
16547577 1// SPDX-License-Identifier: GPL-2.0
46eb3c10 2/* Marvell OcteonTx2 RVU Physical Function ethernet driver
16547577
SG		 3 *
4 * Copyright (C) 2020 Marvell International Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/module.h>
12#include <linux/interrupt.h>
13#include <linux/pci.h>
14#include <linux/etherdevice.h>
15#include <linux/of.h>
16#include <linux/if_vlan.h>
17#include <linux/iommu.h>
18#include <net/ip.h>
19
caa2da34 20#include "otx2_reg.h"
16547577 21#include "otx2_common.h"
caa2da34
SG		 22#include "otx2_txrx.h"
23#include "otx2_struct.h"
c9c12d33 24#include "otx2_ptp.h"
4c236d5d 25#include "cn10k.h"
31a97460 26#include <rvu_trace.h>
16547577 27
facede82
SS		 28#define DRV_NAME "rvu_nicpf"
29#define DRV_STRING "Marvell RVU NIC Physical Function Driver"
16547577
SG		 30
31/* Supported devices */
32static const struct pci_device_id otx2_pf_id_table[] = {
33 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_PF) },
34 { 0, } /* end of table */
35};
36
fc992e33 37MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
16547577
SG		 38MODULE_DESCRIPTION(DRV_STRING);
39MODULE_LICENSE("GPL v2");
16547577
SG		 40MODULE_DEVICE_TABLE(pci, otx2_pf_id_table);
41
b1dc2040
HK		 42static void otx2_vf_link_event_task(struct work_struct *work);
43
5a6d7c9d
SG		 44enum {
45 TYPE_PFAF,
46 TYPE_PFVF,
47};
48
c9c12d33
AM		 49static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable);
50static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable);
51
34bfe0eb
SG		 52static int otx2_change_mtu(struct net_device *netdev, int new_mtu)
53{
54 bool if_up = netif_running(netdev);
55 int err = 0;
56
57 if (if_up)
58 otx2_stop(netdev);
59
60 netdev_info(netdev, "Changing MTU from %d to %d\n",
61 netdev->mtu, new_mtu);
62 netdev->mtu = new_mtu;
63
64 if (if_up)
65 err = otx2_open(netdev);
66
67 return err;
68}
69
547d20f1
G		 70static void otx2_disable_flr_me_intr(struct otx2_nic *pf)
71{
72 int irq, vfs = pf->total_vfs;
73
74 /* Disable VFs ME interrupts */
75 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
76 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0);
77 free_irq(irq, pf);
78
79 /* Disable VFs FLR interrupts */
80 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
81 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0);
82 free_irq(irq, pf);
83
84 if (vfs <= 64)
85 return;
86
87 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
88 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME1);
89 free_irq(irq, pf);
90
91 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
92 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR1);
93 free_irq(irq, pf);
94}
95
96static void otx2_flr_wq_destroy(struct otx2_nic *pf)
97{
98 if (!pf->flr_wq)
99 return;
100 destroy_workqueue(pf->flr_wq);
101 pf->flr_wq = NULL;
102 devm_kfree(pf->dev, pf->flr_wrk);
103}
104
105static void otx2_flr_handler(struct work_struct *work)
106{
107 struct flr_work *flrwork = container_of(work, struct flr_work, work);
108 struct otx2_nic *pf = flrwork->pf;
4c3212f5 109 struct mbox *mbox = &pf->mbox;
547d20f1
G		 110 struct msg_req *req;
111 int vf, reg = 0;
112
113 vf = flrwork - pf->flr_wrk;
114
4c3212f5
SG		 115 mutex_lock(&mbox->lock);
116 req = otx2_mbox_alloc_msg_vf_flr(mbox);
547d20f1 117 if (!req) {
4c3212f5 118 mutex_unlock(&mbox->lock);
547d20f1
G		 119 return;
120 }
121 req->hdr.pcifunc &= RVU_PFVF_FUNC_MASK;
122 req->hdr.pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK;
123
124 if (!otx2_sync_mbox_msg(&pf->mbox)) {
125 if (vf >= 64) {
126 reg = 1;
127 vf = vf - 64;
128 }
129 /* clear transcation pending bit */
130 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
131 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
132 }
133
4c3212f5 134 mutex_unlock(&mbox->lock);
547d20f1
G		 135}
136
137static irqreturn_t otx2_pf_flr_intr_handler(int irq, void *pf_irq)
138{
139 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
140 int reg, dev, vf, start_vf, num_reg = 1;
141 u64 intr;
142
143 if (pf->total_vfs > 64)
144 num_reg = 2;
145
146 for (reg = 0; reg < num_reg; reg++) {
147 intr = otx2_read64(pf, RVU_PF_VFFLR_INTX(reg));
148 if (!intr)
149 continue;
150 start_vf = 64 * reg;
151 for (vf = 0; vf < 64; vf++) {
152 if (!(intr & BIT_ULL(vf)))
153 continue;
154 dev = vf + start_vf;
155 queue_work(pf->flr_wq, &pf->flr_wrk[dev].work);
156 /* Clear interrupt */
157 otx2_write64(pf, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
158 /* Disable the interrupt */
159 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(reg),
160 BIT_ULL(vf));
161 }
162 }
163 return IRQ_HANDLED;
164}
165
166static irqreturn_t otx2_pf_me_intr_handler(int irq, void *pf_irq)
167{
168 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
169 int vf, reg, num_reg = 1;
170 u64 intr;
171
172 if (pf->total_vfs > 64)
173 num_reg = 2;
174
175 for (reg = 0; reg < num_reg; reg++) {
176 intr = otx2_read64(pf, RVU_PF_VFME_INTX(reg));
177 if (!intr)
178 continue;
179 for (vf = 0; vf < 64; vf++) {
180 if (!(intr & BIT_ULL(vf)))
181 continue;
182 /* clear trpend bit */
183 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
184 /* clear interrupt */
185 otx2_write64(pf, RVU_PF_VFME_INTX(reg), BIT_ULL(vf));
186 }
187 }
188 return IRQ_HANDLED;
189}
190
191static int otx2_register_flr_me_intr(struct otx2_nic *pf, int numvfs)
192{
193 struct otx2_hw *hw = &pf->hw;
194 char *irq_name;
195 int ret;
196
197 /* Register ME interrupt handler*/
198 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME0 * NAME_SIZE];
199 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME0", rvu_get_pf(pf->pcifunc));
200 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0),
201 otx2_pf_me_intr_handler, 0, irq_name, pf);
202 if (ret) {
203 dev_err(pf->dev,
204 "RVUPF: IRQ registration failed for ME0\n");
205 }
206
207 /* Register FLR interrupt handler */
208 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR0 * NAME_SIZE];
209 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR0", rvu_get_pf(pf->pcifunc));
210 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0),
211 otx2_pf_flr_intr_handler, 0, irq_name, pf);
212 if (ret) {
213 dev_err(pf->dev,
214 "RVUPF: IRQ registration failed for FLR0\n");
215 return ret;
216 }
217
218 if (numvfs > 64) {
219 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME1 * NAME_SIZE];
220 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME1",
221 rvu_get_pf(pf->pcifunc));
222 ret = request_irq(pci_irq_vector
223 (pf->pdev, RVU_PF_INT_VEC_VFME1),
224 otx2_pf_me_intr_handler, 0, irq_name, pf);
225 if (ret) {
226 dev_err(pf->dev,
227 "RVUPF: IRQ registration failed for ME1\n");
228 }
229 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR1 * NAME_SIZE];
230 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR1",
231 rvu_get_pf(pf->pcifunc));
232 ret = request_irq(pci_irq_vector
233 (pf->pdev, RVU_PF_INT_VEC_VFFLR1),
234 otx2_pf_flr_intr_handler, 0, irq_name, pf);
235 if (ret) {
236 dev_err(pf->dev,
237 "RVUPF: IRQ registration failed for FLR1\n");
238 return ret;
239 }
240 }
241
242 /* Enable ME interrupt for all VFs*/
243 otx2_write64(pf, RVU_PF_VFME_INTX(0), INTR_MASK(numvfs));
244 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(numvfs));
245
246 /* Enable FLR interrupt for all VFs*/
247 otx2_write64(pf, RVU_PF_VFFLR_INTX(0), INTR_MASK(numvfs));
248 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(numvfs));
249
250 if (numvfs > 64) {
251 numvfs -= 64;
252
253 otx2_write64(pf, RVU_PF_VFME_INTX(1), INTR_MASK(numvfs));
254 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(1),
255 INTR_MASK(numvfs));
256
257 otx2_write64(pf, RVU_PF_VFFLR_INTX(1), INTR_MASK(numvfs));
258 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(1),
259 INTR_MASK(numvfs));
260 }
261 return 0;
262}
263
264static int otx2_pf_flr_init(struct otx2_nic *pf, int num_vfs)
265{
266 int vf;
267
268 pf->flr_wq = alloc_workqueue("otx2_pf_flr_wq",
269 WQ_UNBOUND | WQ_HIGHPRI, 1);
270 if (!pf->flr_wq)
271 return -ENOMEM;
272
273 pf->flr_wrk = devm_kcalloc(pf->dev, num_vfs,
274 sizeof(struct flr_work), GFP_KERNEL);
275 if (!pf->flr_wrk) {
276 destroy_workqueue(pf->flr_wq);
277 return -ENOMEM;
278 }
279
280 for (vf = 0; vf < num_vfs; vf++) {
281 pf->flr_wrk[vf].pf = pf;
282 INIT_WORK(&pf->flr_wrk[vf].work, otx2_flr_handler);
283 }
284
285 return 0;
286}
287
5a6d7c9d
SG		 288static void otx2_queue_work(struct mbox *mw, struct workqueue_struct *mbox_wq,
289 int first, int mdevs, u64 intr, int type)
290{
291 struct otx2_mbox_dev *mdev;
292 struct otx2_mbox *mbox;
293 struct mbox_hdr *hdr;
294 int i;
295
296 for (i = first; i < mdevs; i++) {
297 /* start from 0 */
298 if (!(intr & BIT_ULL(i - first)))
299 continue;
300
301 mbox = &mw->mbox;
302 mdev = &mbox->dev[i];
303 if (type == TYPE_PFAF)
304 otx2_sync_mbox_bbuf(mbox, i);
305 hdr = mdev->mbase + mbox->rx_start;
306 /* The hdr->num_msgs is set to zero immediately in the interrupt
307 * handler to ensure that it holds a correct value next time
308 * when the interrupt handler is called.
309 * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler
310 * pf>mbox.up_num_msgs holds the data for use in
311 * pfaf_mbox_up_handler.
312 */
313 if (hdr->num_msgs) {
314 mw[i].num_msgs = hdr->num_msgs;
315 hdr->num_msgs = 0;
316 if (type == TYPE_PFAF)
317 memset(mbox->hwbase + mbox->rx_start, 0,
318 ALIGN(sizeof(struct mbox_hdr),
319 sizeof(u64)));
320
321 queue_work(mbox_wq, &mw[i].mbox_wrk);
322 }
323
324 mbox = &mw->mbox_up;
325 mdev = &mbox->dev[i];
326 if (type == TYPE_PFAF)
327 otx2_sync_mbox_bbuf(mbox, i);
328 hdr = mdev->mbase + mbox->rx_start;
329 if (hdr->num_msgs) {
330 mw[i].up_num_msgs = hdr->num_msgs;
331 hdr->num_msgs = 0;
332 if (type == TYPE_PFAF)
333 memset(mbox->hwbase + mbox->rx_start, 0,
334 ALIGN(sizeof(struct mbox_hdr),
335 sizeof(u64)));
336
337 queue_work(mbox_wq, &mw[i].mbox_up_wrk);
338 }
339 }
340}
341
d424b6c0
SG		 342static void otx2_forward_msg_pfvf(struct otx2_mbox_dev *mdev,
343 struct otx2_mbox *pfvf_mbox, void *bbuf_base,
344 int devid)
345{
346 struct otx2_mbox_dev *src_mdev = mdev;
347 int offset;
348
349 /* Msgs are already copied, trigger VF's mbox irq */
350 smp_wmb();
351
352 offset = pfvf_mbox->trigger | (devid << pfvf_mbox->tr_shift);
353 writeq(1, (void __iomem *)pfvf_mbox->reg_base + offset);
354
355 /* Restore VF's mbox bounce buffer region address */
356 src_mdev->mbase = bbuf_base;
357}
358
359static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf,
360 struct otx2_mbox *src_mbox,
361 int dir, int vf, int num_msgs)
362{
363 struct otx2_mbox_dev *src_mdev, *dst_mdev;
364 struct mbox_hdr *mbox_hdr;
365 struct mbox_hdr *req_hdr;
366 struct mbox *dst_mbox;
367 int dst_size, err;
368
369 if (dir == MBOX_DIR_PFAF) {
370 /* Set VF's mailbox memory as PF's bounce buffer memory, so
371 * that explicit copying of VF's msgs to PF=>AF mbox region
372 * and AF=>PF responses to VF's mbox region can be avoided.
373 */
374 src_mdev = &src_mbox->dev[vf];
375 mbox_hdr = src_mbox->hwbase +
376 src_mbox->rx_start + (vf * MBOX_SIZE);
377
378 dst_mbox = &pf->mbox;
379 dst_size = dst_mbox->mbox.tx_size -
380 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
66a5209b
HK		 381 /* Check if msgs fit into destination area and has valid size */
382 if (mbox_hdr->msg_size > dst_size || !mbox_hdr->msg_size)
d424b6c0
SG		 383 return -EINVAL;
384
385 dst_mdev = &dst_mbox->mbox.dev[0];
386
4c3212f5 387 mutex_lock(&pf->mbox.lock);
d424b6c0
SG		 388 dst_mdev->mbase = src_mdev->mbase;
389 dst_mdev->msg_size = mbox_hdr->msg_size;
390 dst_mdev->num_msgs = num_msgs;
391 err = otx2_sync_mbox_msg(dst_mbox);
392 if (err) {
393 dev_warn(pf->dev,
394 "AF not responding to VF%d messages\n", vf);
395 /* restore PF mbase and exit */
396 dst_mdev->mbase = pf->mbox.bbuf_base;
4c3212f5 397 mutex_unlock(&pf->mbox.lock);
d424b6c0
SG		 398 return err;
399 }
400 /* At this point, all the VF messages sent to AF are acked
401 * with proper responses and responses are copied to VF
402 * mailbox hence raise interrupt to VF.
403 */
404 req_hdr = (struct mbox_hdr *)(dst_mdev->mbase +
405 dst_mbox->mbox.rx_start);
406 req_hdr->num_msgs = num_msgs;
407
408 otx2_forward_msg_pfvf(dst_mdev, &pf->mbox_pfvf[0].mbox,
409 pf->mbox.bbuf_base, vf);
4c3212f5 410 mutex_unlock(&pf->mbox.lock);
d424b6c0
SG		 411 } else if (dir == MBOX_DIR_PFVF_UP) {
412 src_mdev = &src_mbox->dev[0];
413 mbox_hdr = src_mbox->hwbase + src_mbox->rx_start;
414 req_hdr = (struct mbox_hdr *)(src_mdev->mbase +
415 src_mbox->rx_start);
416 req_hdr->num_msgs = num_msgs;
417
418 dst_mbox = &pf->mbox_pfvf[0];
419 dst_size = dst_mbox->mbox_up.tx_size -
420 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
421 /* Check if msgs fit into destination area */
422 if (mbox_hdr->msg_size > dst_size)
423 return -EINVAL;
424
425 dst_mdev = &dst_mbox->mbox_up.dev[vf];
426 dst_mdev->mbase = src_mdev->mbase;
427 dst_mdev->msg_size = mbox_hdr->msg_size;
428 dst_mdev->num_msgs = mbox_hdr->num_msgs;
429 err = otx2_sync_mbox_up_msg(dst_mbox, vf);
430 if (err) {
431 dev_warn(pf->dev,
432 "VF%d is not responding to mailbox\n", vf);
433 return err;
434 }
435 } else if (dir == MBOX_DIR_VFPF_UP) {
436 req_hdr = (struct mbox_hdr *)(src_mbox->dev[0].mbase +
437 src_mbox->rx_start);
438 req_hdr->num_msgs = num_msgs;
439 otx2_forward_msg_pfvf(&pf->mbox_pfvf->mbox_up.dev[vf],
440 &pf->mbox.mbox_up,
441 pf->mbox_pfvf[vf].bbuf_base,
442 0);
443 }
444
445 return 0;
446}
447
448static void otx2_pfvf_mbox_handler(struct work_struct *work)
449{
450 struct mbox_msghdr *msg = NULL;
451 int offset, vf_idx, id, err;
452 struct otx2_mbox_dev *mdev;
453 struct mbox_hdr *req_hdr;
454 struct otx2_mbox *mbox;
455 struct mbox *vf_mbox;
456 struct otx2_nic *pf;
457
458 vf_mbox = container_of(work, struct mbox, mbox_wrk);
459 pf = vf_mbox->pfvf;
460 vf_idx = vf_mbox - pf->mbox_pfvf;
461
462 mbox = &pf->mbox_pfvf[0].mbox;
463 mdev = &mbox->dev[vf_idx];
464 req_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
465
466 offset = ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
467
468 for (id = 0; id < vf_mbox->num_msgs; id++) {
469 msg = (struct mbox_msghdr *)(mdev->mbase + mbox->rx_start +
470 offset);
471
472 if (msg->sig != OTX2_MBOX_REQ_SIG)
473 goto inval_msg;
474
475 /* Set VF's number in each of the msg */
476 msg->pcifunc &= RVU_PFVF_FUNC_MASK;
477 msg->pcifunc |= (vf_idx + 1) & RVU_PFVF_FUNC_MASK;
478 offset = msg->next_msgoff;
479 }
480 err = otx2_forward_vf_mbox_msgs(pf, mbox, MBOX_DIR_PFAF, vf_idx,
481 vf_mbox->num_msgs);
482 if (err)
483 goto inval_msg;
484 return;
485
486inval_msg:
487 otx2_reply_invalid_msg(mbox, vf_idx, 0, msg->id);
488 otx2_mbox_msg_send(mbox, vf_idx);
489}
490
491static void otx2_pfvf_mbox_up_handler(struct work_struct *work)
492{
493 struct mbox *vf_mbox = container_of(work, struct mbox, mbox_up_wrk);
494 struct otx2_nic *pf = vf_mbox->pfvf;
495 struct otx2_mbox_dev *mdev;
496 int offset, id, vf_idx = 0;
497 struct mbox_hdr *rsp_hdr;
498 struct mbox_msghdr *msg;
499 struct otx2_mbox *mbox;
500
501 vf_idx = vf_mbox - pf->mbox_pfvf;
502 mbox = &pf->mbox_pfvf[0].mbox_up;
503 mdev = &mbox->dev[vf_idx];
504
505 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
506 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
507
508 for (id = 0; id < vf_mbox->up_num_msgs; id++) {
509 msg = mdev->mbase + offset;
510
511 if (msg->id >= MBOX_MSG_MAX) {
512 dev_err(pf->dev,
513 "Mbox msg with unknown ID 0x%x\n", msg->id);
514 goto end;
515 }
516
517 if (msg->sig != OTX2_MBOX_RSP_SIG) {
518 dev_err(pf->dev,
519 "Mbox msg with wrong signature %x, ID 0x%x\n",
520 msg->sig, msg->id);
521 goto end;
522 }
523
524 switch (msg->id) {
525 case MBOX_MSG_CGX_LINK_EVENT:
526 break;
527 default:
528 if (msg->rc)
529 dev_err(pf->dev,
530 "Mbox msg response has err %d, ID 0x%x\n",
531 msg->rc, msg->id);
532 break;
533 }
534
535end:
536 offset = mbox->rx_start + msg->next_msgoff;
66a5209b
HK		 537 if (mdev->msgs_acked == (vf_mbox->up_num_msgs - 1))
538 __otx2_mbox_reset(mbox, 0);
d424b6c0
SG		 539 mdev->msgs_acked++;
540 }
d424b6c0
SG		 541}
542
543static irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq)
544{
545 struct otx2_nic *pf = (struct otx2_nic *)(pf_irq);
546 int vfs = pf->total_vfs;
547 struct mbox *mbox;
548 u64 intr;
549
550 mbox = pf->mbox_pfvf;
551 /* Handle VF interrupts */
552 if (vfs > 64) {
553 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(1));
554 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), intr);
555 otx2_queue_work(mbox, pf->mbox_pfvf_wq, 64, vfs, intr,
556 TYPE_PFVF);
557 vfs -= 64;
558 }
559
560 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(0));
561 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), intr);
562
563 otx2_queue_work(mbox, pf->mbox_pfvf_wq, 0, vfs, intr, TYPE_PFVF);
564
31a97460
SS		 565 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
566
d424b6c0
SG		 567 return IRQ_HANDLED;
568}
569
570static int otx2_pfvf_mbox_init(struct otx2_nic *pf, int numvfs)
571{
572 void __iomem *hwbase;
573 struct mbox *mbox;
574 int err, vf;
575 u64 base;
576
577 if (!numvfs)
578 return -EINVAL;
579
580 pf->mbox_pfvf = devm_kcalloc(&pf->pdev->dev, numvfs,
581 sizeof(struct mbox), GFP_KERNEL);
582 if (!pf->mbox_pfvf)
583 return -ENOMEM;
584
585 pf->mbox_pfvf_wq = alloc_workqueue("otx2_pfvf_mailbox",
586 WQ_UNBOUND | WQ_HIGHPRI |
587 WQ_MEM_RECLAIM, 1);
588 if (!pf->mbox_pfvf_wq)
589 return -ENOMEM;
590
facede82
SS		 591 /* On CN10K platform, PF <-> VF mailbox region follows after
592 * PF <-> AF mailbox region.
593 */
594 if (test_bit(CN10K_MBOX, &pf->hw.cap_flag))
595 base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) +
596 MBOX_SIZE;
597 else
598 base = readq((void __iomem *)((u64)pf->reg_base +
599 RVU_PF_VF_BAR4_ADDR));
d424b6c0 600
facede82 601 hwbase = ioremap_wc(base, MBOX_SIZE * pf->total_vfs);
d424b6c0
SG		 602 if (!hwbase) {
603 err = -ENOMEM;
604 goto free_wq;
605 }
606
607 mbox = &pf->mbox_pfvf[0];
608 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
609 MBOX_DIR_PFVF, numvfs);
610 if (err)
611 goto free_iomem;
612
613 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
614 MBOX_DIR_PFVF_UP, numvfs);
615 if (err)
616 goto free_iomem;
617
618 for (vf = 0; vf < numvfs; vf++) {
619 mbox->pfvf = pf;
620 INIT_WORK(&mbox->mbox_wrk, otx2_pfvf_mbox_handler);
621 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfvf_mbox_up_handler);
622 mbox++;
623 }
624
625 return 0;
626
627free_iomem:
628 if (hwbase)
629 iounmap(hwbase);
630free_wq:
631 destroy_workqueue(pf->mbox_pfvf_wq);
632 return err;
633}
634
635static void otx2_pfvf_mbox_destroy(struct otx2_nic *pf)
636{
637 struct mbox *mbox = &pf->mbox_pfvf[0];
638
639 if (!mbox)
640 return;
641
642 if (pf->mbox_pfvf_wq) {
d424b6c0
SG		 643 destroy_workqueue(pf->mbox_pfvf_wq);
644 pf->mbox_pfvf_wq = NULL;
645 }
646
647 if (mbox->mbox.hwbase)
648 iounmap(mbox->mbox.hwbase);
649
650 otx2_mbox_destroy(&mbox->mbox);
651}
652
653static void otx2_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
654{
655 /* Clear PF <=> VF mailbox IRQ */
656 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
657 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
658
659 /* Enable PF <=> VF mailbox IRQ */
660 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(numvfs));
661 if (numvfs > 64) {
662 numvfs -= 64;
663 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
664 INTR_MASK(numvfs));
665 }
666}
667
668static void otx2_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
669{
670 int vector;
671
672 /* Disable PF <=> VF mailbox IRQ */
673 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ull);
674 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ull);
675
676 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
677 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
678 free_irq(vector, pf);
679
680 if (numvfs > 64) {
681 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
682 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
683 free_irq(vector, pf);
684 }
685}
686
687static int otx2_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
688{
689 struct otx2_hw *hw = &pf->hw;
690 char *irq_name;
691 int err;
692
693 /* Register MBOX0 interrupt handler */
694 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX0 * NAME_SIZE];
695 if (pf->pcifunc)
696 snprintf(irq_name, NAME_SIZE,
697 "RVUPF%d_VF Mbox0", rvu_get_pf(pf->pcifunc));
698 else
699 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox0");
700 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0),
701 otx2_pfvf_mbox_intr_handler, 0, irq_name, pf);
702 if (err) {
703 dev_err(pf->dev,
704 "RVUPF: IRQ registration failed for PFVF mbox0 irq\n");
705 return err;
706 }
707
708 if (numvfs > 64) {
709 /* Register MBOX1 interrupt handler */
710 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX1 * NAME_SIZE];
711 if (pf->pcifunc)
712 snprintf(irq_name, NAME_SIZE,
713 "RVUPF%d_VF Mbox1", rvu_get_pf(pf->pcifunc));
714 else
715 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox1");
716 err = request_irq(pci_irq_vector(pf->pdev,
717 RVU_PF_INT_VEC_VFPF_MBOX1),
718 otx2_pfvf_mbox_intr_handler,
719 0, irq_name, pf);
720 if (err) {
721 dev_err(pf->dev,
722 "RVUPF: IRQ registration failed for PFVF mbox1 irq\n");
723 return err;
724 }
725 }
726
727 otx2_enable_pfvf_mbox_intr(pf, numvfs);
728
729 return 0;
730}
731
5a6d7c9d
SG		 732static void otx2_process_pfaf_mbox_msg(struct otx2_nic *pf,
733 struct mbox_msghdr *msg)
734{
ad513ed9
TD		 735 int devid;
736
5a6d7c9d
SG		 737 if (msg->id >= MBOX_MSG_MAX) {
738 dev_err(pf->dev,
739 "Mbox msg with unknown ID 0x%x\n", msg->id);
740 return;
741 }
742
743 if (msg->sig != OTX2_MBOX_RSP_SIG) {
744 dev_err(pf->dev,
745 "Mbox msg with wrong signature %x, ID 0x%x\n",
746 msg->sig, msg->id);
747 return;
748 }
749
ad513ed9
TD		 750 /* message response heading VF */
751 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
752 if (devid) {
753 struct otx2_vf_config *config = &pf->vf_configs[devid - 1];
754 struct delayed_work *dwork;
755
756 switch (msg->id) {
757 case MBOX_MSG_NIX_LF_START_RX:
758 config->intf_down = false;
759 dwork = &config->link_event_work;
760 schedule_delayed_work(dwork, msecs_to_jiffies(100));
761 break;
762 case MBOX_MSG_NIX_LF_STOP_RX:
763 config->intf_down = true;
764 break;
765 }
766
767 return;
768 }
769
5a6d7c9d
SG		 770 switch (msg->id) {
771 case MBOX_MSG_READY:
772 pf->pcifunc = msg->pcifunc;
773 break;
05fcc9e0
SG		 774 case MBOX_MSG_MSIX_OFFSET:
775 mbox_handler_msix_offset(pf, (struct msix_offset_rsp *)msg);
776 break;
777 case MBOX_MSG_NPA_LF_ALLOC:
778 mbox_handler_npa_lf_alloc(pf, (struct npa_lf_alloc_rsp *)msg);
779 break;
780 case MBOX_MSG_NIX_LF_ALLOC:
781 mbox_handler_nix_lf_alloc(pf, (struct nix_lf_alloc_rsp *)msg);
782 break;
caa2da34
SG		 783 case MBOX_MSG_NIX_TXSCH_ALLOC:
784 mbox_handler_nix_txsch_alloc(pf,
785 (struct nix_txsch_alloc_rsp *)msg);
786 break;
75f36270
G		 787 case MBOX_MSG_NIX_BP_ENABLE:
788 mbox_handler_nix_bp_enable(pf, (struct nix_bp_cfg_rsp *)msg);
789 break;
d45d8979
CJ		 790 case MBOX_MSG_CGX_STATS:
791 mbox_handler_cgx_stats(pf, (struct cgx_stats_rsp *)msg);
792 break;
d0cf9503
CJ		 793 case MBOX_MSG_CGX_FEC_STATS:
794 mbox_handler_cgx_fec_stats(pf, (struct cgx_fec_stats_rsp *)msg);
795 break;
5a6d7c9d
SG		 796 default:
797 if (msg->rc)
798 dev_err(pf->dev,
799 "Mbox msg response has err %d, ID 0x%x\n",
800 msg->rc, msg->id);
801 break;
802 }
803}
804
805static void otx2_pfaf_mbox_handler(struct work_struct *work)
806{
807 struct otx2_mbox_dev *mdev;
808 struct mbox_hdr *rsp_hdr;
809 struct mbox_msghdr *msg;
810 struct otx2_mbox *mbox;
811 struct mbox *af_mbox;
812 struct otx2_nic *pf;
813 int offset, id;
814
815 af_mbox = container_of(work, struct mbox, mbox_wrk);
816 mbox = &af_mbox->mbox;
817 mdev = &mbox->dev[0];
818 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
819
820 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
821 pf = af_mbox->pfvf;
822
823 for (id = 0; id < af_mbox->num_msgs; id++) {
824 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
825 otx2_process_pfaf_mbox_msg(pf, msg);
826 offset = mbox->rx_start + msg->next_msgoff;
66a5209b
HK		 827 if (mdev->msgs_acked == (af_mbox->num_msgs - 1))
828 __otx2_mbox_reset(mbox, 0);
5a6d7c9d
SG		 829 mdev->msgs_acked++;
830 }
831
5a6d7c9d
SG		 832}
833
50fe6c02
LC		 834static void otx2_handle_link_event(struct otx2_nic *pf)
835{
836 struct cgx_link_user_info *linfo = &pf->linfo;
837 struct net_device *netdev = pf->netdev;
838
839 pr_info("%s NIC Link is %s %d Mbps %s duplex\n", netdev->name,
840 linfo->link_up ? "UP" : "DOWN", linfo->speed,
841 linfo->full_duplex ? "Full" : "Half");
842 if (linfo->link_up) {
843 netif_carrier_on(netdev);
844 netif_tx_start_all_queues(netdev);
845 } else {
846 netif_tx_stop_all_queues(netdev);
847 netif_carrier_off(netdev);
848 }
849}
850
851int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf,
852 struct cgx_link_info_msg *msg,
853 struct msg_rsp *rsp)
854{
ad513ed9
TD		 855 int i;
856
50fe6c02
LC		 857 /* Copy the link info sent by AF */
858 pf->linfo = msg->link_info;
859
ad513ed9
TD		 860 /* notify VFs about link event */
861 for (i = 0; i < pci_num_vf(pf->pdev); i++) {
862 struct otx2_vf_config *config = &pf->vf_configs[i];
863 struct delayed_work *dwork = &config->link_event_work;
864
865 if (config->intf_down)
866 continue;
867
868 schedule_delayed_work(dwork, msecs_to_jiffies(100));
869 }
870
50fe6c02
LC		 871 /* interface has not been fully configured yet */
872 if (pf->flags & OTX2_FLAG_INTF_DOWN)
873 return 0;
874
875 otx2_handle_link_event(pf);
876 return 0;
877}
878
5a6d7c9d
SG		 879static int otx2_process_mbox_msg_up(struct otx2_nic *pf,
880 struct mbox_msghdr *req)
881{
882 /* Check if valid, if not reply with a invalid msg */
883 if (req->sig != OTX2_MBOX_REQ_SIG) {
884 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
885 return -ENODEV;
886 }
887
888 switch (req->id) {
889#define M(_name, _id, _fn_name, _req_type, _rsp_type) \
890 case _id: { \
891 struct _rsp_type *rsp; \
892 int err; \
893 \
894 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
895 &pf->mbox.mbox_up, 0, \
896 sizeof(struct _rsp_type)); \
897 if (!rsp) \
898 return -ENOMEM; \
899 \
900 rsp->hdr.id = _id; \
901 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
902 rsp->hdr.pcifunc = 0; \
903 rsp->hdr.rc = 0; \
904 \
905 err = otx2_mbox_up_handler_ ## _fn_name( \
906 pf, (struct _req_type *)req, rsp); \
907 return err; \
908 }
909MBOX_UP_CGX_MESSAGES
910#undef M
911 break;
912 default:
913 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
914 return -ENODEV;
915 }
916 return 0;
917}
918
919static void otx2_pfaf_mbox_up_handler(struct work_struct *work)
920{
921 struct mbox *af_mbox = container_of(work, struct mbox, mbox_up_wrk);
922 struct otx2_mbox *mbox = &af_mbox->mbox_up;
923 struct otx2_mbox_dev *mdev = &mbox->dev[0];
924 struct otx2_nic *pf = af_mbox->pfvf;
925 int offset, id, devid = 0;
926 struct mbox_hdr *rsp_hdr;
927 struct mbox_msghdr *msg;
928
929 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
930
931 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
932
933 for (id = 0; id < af_mbox->up_num_msgs; id++) {
934 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
935
936 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
937 /* Skip processing VF's messages */
938 if (!devid)
939 otx2_process_mbox_msg_up(pf, msg);
940 offset = mbox->rx_start + msg->next_msgoff;
941 }
d424b6c0
SG		 942 if (devid) {
943 otx2_forward_vf_mbox_msgs(pf, &pf->mbox.mbox_up,
944 MBOX_DIR_PFVF_UP, devid - 1,
945 af_mbox->up_num_msgs);
946 return;
947 }
5a6d7c9d
SG		 948
949 otx2_mbox_msg_send(mbox, 0);
950}
951
952static irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq)
953{
954 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
955 struct mbox *mbox;
956
957 /* Clear the IRQ */
958 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
959
960 mbox = &pf->mbox;
31a97460
SS		 961
962 trace_otx2_msg_interrupt(mbox->mbox.pdev, "AF to PF", BIT_ULL(0));
963
5a6d7c9d
SG		 964 otx2_queue_work(mbox, pf->mbox_wq, 0, 1, 1, TYPE_PFAF);
965
966 return IRQ_HANDLED;
967}
968
969static void otx2_disable_mbox_intr(struct otx2_nic *pf)
970{
971 int vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX);
972
973 /* Disable AF => PF mailbox IRQ */
974 otx2_write64(pf, RVU_PF_INT_ENA_W1C, BIT_ULL(0));
975 free_irq(vector, pf);
976}
977
978static int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
979{
980 struct otx2_hw *hw = &pf->hw;
981 struct msg_req *req;
982 char *irq_name;
983 int err;
984
985 /* Register mailbox interrupt handler */
986 irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE];
987 snprintf(irq_name, NAME_SIZE, "RVUPFAF Mbox");
988 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX),
989 otx2_pfaf_mbox_intr_handler, 0, irq_name, pf);
990 if (err) {
991 dev_err(pf->dev,
992 "RVUPF: IRQ registration failed for PFAF mbox irq\n");
993 return err;
994 }
995
996 /* Enable mailbox interrupt for msgs coming from AF.
997 * First clear to avoid spurious interrupts, if any.
998 */
999 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
1000 otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0));
1001
1002 if (!probe_af)
1003 return 0;
1004
1005 /* Check mailbox communication with AF */
1006 req = otx2_mbox_alloc_msg_ready(&pf->mbox);
1007 if (!req) {
1008 otx2_disable_mbox_intr(pf);
1009 return -ENOMEM;
1010 }
1011 err = otx2_sync_mbox_msg(&pf->mbox);
1012 if (err) {
1013 dev_warn(pf->dev,
1014 "AF not responding to mailbox, deferring probe\n");
1015 otx2_disable_mbox_intr(pf);
1016 return -EPROBE_DEFER;
1017 }
1018
1019 return 0;
1020}
1021
1022static void otx2_pfaf_mbox_destroy(struct otx2_nic *pf)
1023{
1024 struct mbox *mbox = &pf->mbox;
1025
1026 if (pf->mbox_wq) {
5a6d7c9d
SG		 1027 destroy_workqueue(pf->mbox_wq);
1028 pf->mbox_wq = NULL;
1029 }
1030
1031 if (mbox->mbox.hwbase)
1032 iounmap((void __iomem *)mbox->mbox.hwbase);
1033
1034 otx2_mbox_destroy(&mbox->mbox);
1035 otx2_mbox_destroy(&mbox->mbox_up);
1036}
1037
1038static int otx2_pfaf_mbox_init(struct otx2_nic *pf)
1039{
1040 struct mbox *mbox = &pf->mbox;
1041 void __iomem *hwbase;
1042 int err;
1043
1044 mbox->pfvf = pf;
1045 pf->mbox_wq = alloc_workqueue("otx2_pfaf_mailbox",
1046 WQ_UNBOUND | WQ_HIGHPRI |
1047 WQ_MEM_RECLAIM, 1);
1048 if (!pf->mbox_wq)
1049 return -ENOMEM;
1050
1051 /* Mailbox is a reserved memory (in RAM) region shared between
1052 * admin function (i.e AF) and this PF, shouldn't be mapped as
1053 * device memory to allow unaligned accesses.
1054 */
1055 hwbase = ioremap_wc(pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM),
facede82 1056 MBOX_SIZE);
5a6d7c9d
SG		 1057 if (!hwbase) {
1058 dev_err(pf->dev, "Unable to map PFAF mailbox region\n");
1059 err = -ENOMEM;
1060 goto exit;
1061 }
1062
1063 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
1064 MBOX_DIR_PFAF, 1);
1065 if (err)
1066 goto exit;
1067
1068 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
1069 MBOX_DIR_PFAF_UP, 1);
1070 if (err)
1071 goto exit;
1072
1073 err = otx2_mbox_bbuf_init(mbox, pf->pdev);
1074 if (err)
1075 goto exit;
1076
1077 INIT_WORK(&mbox->mbox_wrk, otx2_pfaf_mbox_handler);
1078 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfaf_mbox_up_handler);
4c3212f5 1079 mutex_init(&mbox->lock);
5a6d7c9d
SG		 1080
1081 return 0;
1082exit:
1083 otx2_pfaf_mbox_destroy(pf);
1084 return err;
1085}
1086
50fe6c02
LC		 1087static int otx2_cgx_config_linkevents(struct otx2_nic *pf, bool enable)
1088{
1089 struct msg_req *msg;
1090 int err;
1091
4c3212f5 1092 mutex_lock(&pf->mbox.lock);
50fe6c02
LC		 1093 if (enable)
1094 msg = otx2_mbox_alloc_msg_cgx_start_linkevents(&pf->mbox);
1095 else
1096 msg = otx2_mbox_alloc_msg_cgx_stop_linkevents(&pf->mbox);
1097
1098 if (!msg) {
4c3212f5 1099 mutex_unlock(&pf->mbox.lock);
50fe6c02
LC		 1100 return -ENOMEM;
1101 }
1102
1103 err = otx2_sync_mbox_msg(&pf->mbox);
4c3212f5 1104 mutex_unlock(&pf->mbox.lock);
50fe6c02
LC		 1105 return err;
1106}
1107
34bfe0eb
SG		 1108static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable)
1109{
1110 struct msg_req *msg;
1111 int err;
1112
79d2be38
HK		 1113 if (enable && bitmap_weight(&pf->flow_cfg->dmacflt_bmap,
1114 pf->flow_cfg->dmacflt_max_flows))
1115 netdev_warn(pf->netdev,
1116 "CGX/RPM internal loopback might not work as DMAC filters are active\n");
1117
4c3212f5 1118 mutex_lock(&pf->mbox.lock);
34bfe0eb
SG		 1119 if (enable)
1120 msg = otx2_mbox_alloc_msg_cgx_intlbk_enable(&pf->mbox);
1121 else
1122 msg = otx2_mbox_alloc_msg_cgx_intlbk_disable(&pf->mbox);
1123
1124 if (!msg) {
4c3212f5 1125 mutex_unlock(&pf->mbox.lock);
34bfe0eb
SG		 1126 return -ENOMEM;
1127 }
1128
1129 err = otx2_sync_mbox_msg(&pf->mbox);
4c3212f5 1130 mutex_unlock(&pf->mbox.lock);
34bfe0eb
SG		 1131 return err;
1132}
1133
d45d8979
CJ		 1134int otx2_set_real_num_queues(struct net_device *netdev,
1135 int tx_queues, int rx_queues)
16547577
SG		 1136{
1137 int err;
1138
1139 err = netif_set_real_num_tx_queues(netdev, tx_queues);
1140 if (err) {
1141 netdev_err(netdev,
1142 "Failed to set no of Tx queues: %d\n", tx_queues);
1143 return err;
1144 }
1145
1146 err = netif_set_real_num_rx_queues(netdev, rx_queues);
1147 if (err)
1148 netdev_err(netdev,
1149 "Failed to set no of Rx queues: %d\n", rx_queues);
1150 return err;
1151}
3184fb5b 1152EXPORT_SYMBOL(otx2_set_real_num_queues);
16547577 1153
4ff7d148
G		 1154static irqreturn_t otx2_q_intr_handler(int irq, void *data)
1155{
1156 struct otx2_nic *pf = data;
1157 u64 val, *ptr;
1158 u64 qidx = 0;
1159
1160 /* CQ */
1161 for (qidx = 0; qidx < pf->qset.cq_cnt; qidx++) {
1162 ptr = otx2_get_regaddr(pf, NIX_LF_CQ_OP_INT);
1163 val = otx2_atomic64_add((qidx << 44), ptr);
1164
1165 otx2_write64(pf, NIX_LF_CQ_OP_INT, (qidx << 44) |
1166 (val & NIX_CQERRINT_BITS));
1167 if (!(val & (NIX_CQERRINT_BITS | BIT_ULL(42))))
1168 continue;
1169
1170 if (val & BIT_ULL(42)) {
1171 netdev_err(pf->netdev, "CQ%lld: error reading NIX_LF_CQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1172 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1173 } else {
1174 if (val & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
1175 netdev_err(pf->netdev, "CQ%lld: Doorbell error",
1176 qidx);
1177 if (val & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
1178 netdev_err(pf->netdev, "CQ%lld: Memory fault on CQE write to LLC/DRAM",
1179 qidx);
1180 }
1181
1182 schedule_work(&pf->reset_task);
1183 }
1184
1185 /* SQ */
1186 for (qidx = 0; qidx < pf->hw.tx_queues; qidx++) {
1187 ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT);
1188 val = otx2_atomic64_add((qidx << 44), ptr);
1189 otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) |
1190 (val & NIX_SQINT_BITS));
1191
1192 if (!(val & (NIX_SQINT_BITS | BIT_ULL(42))))
1193 continue;
1194
1195 if (val & BIT_ULL(42)) {
1196 netdev_err(pf->netdev, "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1197 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1198 } else {
1199 if (val & BIT_ULL(NIX_SQINT_LMT_ERR)) {
1200 netdev_err(pf->netdev, "SQ%lld: LMT store error NIX_LF_SQ_OP_ERR_DBG:0x%llx",
1201 qidx,
1202 otx2_read64(pf,
1203 NIX_LF_SQ_OP_ERR_DBG));
1204 otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG,
1205 BIT_ULL(44));
1206 }
1207 if (val & BIT_ULL(NIX_SQINT_MNQ_ERR)) {
1208 netdev_err(pf->netdev, "SQ%lld: Meta-descriptor enqueue error NIX_LF_MNQ_ERR_DGB:0x%llx\n",
1209 qidx,
1210 otx2_read64(pf, NIX_LF_MNQ_ERR_DBG));
1211 otx2_write64(pf, NIX_LF_MNQ_ERR_DBG,
1212 BIT_ULL(44));
1213 }
1214 if (val & BIT_ULL(NIX_SQINT_SEND_ERR)) {
1215 netdev_err(pf->netdev, "SQ%lld: Send error, NIX_LF_SEND_ERR_DBG 0x%llx",
1216 qidx,
1217 otx2_read64(pf,
1218 NIX_LF_SEND_ERR_DBG));
1219 otx2_write64(pf, NIX_LF_SEND_ERR_DBG,
1220 BIT_ULL(44));
1221 }
1222 if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL))
1223 netdev_err(pf->netdev, "SQ%lld: SQB allocation failed",
1224 qidx);
1225 }
1226
1227 schedule_work(&pf->reset_task);
1228 }
1229
1230 return IRQ_HANDLED;
1231}
1232
04a21ef3
SG		 1233static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
1234{
1235 struct otx2_cq_poll *cq_poll = (struct otx2_cq_poll *)cq_irq;
1236 struct otx2_nic *pf = (struct otx2_nic *)cq_poll->dev;
1237 int qidx = cq_poll->cint_idx;
1238
1239 /* Disable interrupts.
1240 *
1241 * Completion interrupts behave in a level-triggered interrupt
1242 * fashion, and hence have to be cleared only after it is serviced.
1243 */
1244 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
1245
1246 /* Schedule NAPI */
1247 napi_schedule_irqoff(&cq_poll->napi);
1248
1249 return IRQ_HANDLED;
1250}
1251
1252static void otx2_disable_napi(struct otx2_nic *pf)
1253{
1254 struct otx2_qset *qset = &pf->qset;
1255 struct otx2_cq_poll *cq_poll;
1256 int qidx;
1257
1258 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1259 cq_poll = &qset->napi[qidx];
1260 napi_disable(&cq_poll->napi);
1261 netif_napi_del(&cq_poll->napi);
1262 }
1263}
1264
caa2da34
SG		 1265static void otx2_free_cq_res(struct otx2_nic *pf)
1266{
1267 struct otx2_qset *qset = &pf->qset;
1268 struct otx2_cq_queue *cq;
1269 int qidx;
1270
1271 /* Disable CQs */
1272 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_CQ, false);
1273 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1274 cq = &qset->cq[qidx];
1275 qmem_free(pf->dev, cq->cqe);
1276 }
1277}
1278
1279static void otx2_free_sq_res(struct otx2_nic *pf)
1280{
1281 struct otx2_qset *qset = &pf->qset;
1282 struct otx2_snd_queue *sq;
1283 int qidx;
1284
1285 /* Disable SQs */
1286 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_SQ, false);
1287 /* Free SQB pointers */
1288 otx2_sq_free_sqbs(pf);
1289 for (qidx = 0; qidx < pf->hw.tx_queues; qidx++) {
1290 sq = &qset->sq[qidx];
1291 qmem_free(pf->dev, sq->sqe);
86d74760 1292 qmem_free(pf->dev, sq->tso_hdrs);
3ca6c4c8 1293 kfree(sq->sg);
caa2da34
SG		 1294 kfree(sq->sqb_ptrs);
1295 }
1296}
1297
ab58a416
HK		 1298static int otx2_get_rbuf_size(struct otx2_nic *pf, int mtu)
1299{
1300 int frame_size;
1301 int total_size;
1302 int rbuf_size;
1303
1304 /* The data transferred by NIX to memory consists of actual packet
1305 * plus additional data which has timestamp and/or EDSA/HIGIG2
1306 * headers if interface is configured in corresponding modes.
1307 * NIX transfers entire data using 6 segments/buffers and writes
1308 * a CQE_RX descriptor with those segment addresses. First segment
1309 * has additional data prepended to packet. Also software omits a
1310 * headroom of 128 bytes and sizeof(struct skb_shared_info) in
1311 * each segment. Hence the total size of memory needed
1312 * to receive a packet with 'mtu' is:
1313 * frame size = mtu + additional data;
1314 * memory = frame_size + (headroom + struct skb_shared_info size) * 6;
1315 * each receive buffer size = memory / 6;
1316 */
1317 frame_size = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN;
1318 total_size = frame_size + (OTX2_HEAD_ROOM +
1319 OTX2_DATA_ALIGN(sizeof(struct skb_shared_info))) * 6;
1320 rbuf_size = total_size / 6;
1321
1322 return ALIGN(rbuf_size, 2048);
1323}
1324
caa2da34
SG		 1325static int otx2_init_hw_resources(struct otx2_nic *pf)
1326{
55307fcb 1327 struct nix_lf_free_req *free_req;
caa2da34
SG		 1328 struct mbox *mbox = &pf->mbox;
1329 struct otx2_hw *hw = &pf->hw;
1330 struct msg_req *req;
1331 int err = 0, lvl;
1332
1333 /* Set required NPA LF's pool counts
1334 * Auras and Pools are used in a 1:1 mapping,
1335 * so, aura count = pool count.
1336 */
1337 hw->rqpool_cnt = hw->rx_queues;
1338 hw->sqpool_cnt = hw->tx_queues;
1339 hw->pool_cnt = hw->rqpool_cnt + hw->sqpool_cnt;
1340
ab58a416
HK		 1341 pf->max_frs = pf->netdev->mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN;
1342
1343 pf->rbsize = otx2_get_rbuf_size(pf, pf->netdev->mtu);
caa2da34 1344
4c3212f5 1345 mutex_lock(&mbox->lock);
caa2da34
SG		 1346 /* NPA init */
1347 err = otx2_config_npa(pf);
1348 if (err)
1349 goto exit;
1350
1351 /* NIX init */
1352 err = otx2_config_nix(pf);
1353 if (err)
1354 goto err_free_npa_lf;
1355
75f36270
G		 1356 /* Enable backpressure */
1357 otx2_nix_config_bp(pf, true);
1358
caa2da34
SG		 1359 /* Init Auras and pools used by NIX RQ, for free buffer ptrs */
1360 err = otx2_rq_aura_pool_init(pf);
1361 if (err) {
4c3212f5 1362 mutex_unlock(&mbox->lock);
caa2da34
SG		 1363 goto err_free_nix_lf;
1364 }
1365 /* Init Auras and pools used by NIX SQ, for queueing SQEs */
1366 err = otx2_sq_aura_pool_init(pf);
1367 if (err) {
4c3212f5 1368 mutex_unlock(&mbox->lock);
caa2da34
SG		 1369 goto err_free_rq_ptrs;
1370 }
1371
1372 err = otx2_txsch_alloc(pf);
1373 if (err) {
4c3212f5 1374 mutex_unlock(&mbox->lock);
caa2da34
SG		 1375 goto err_free_sq_ptrs;
1376 }
1377
1378 err = otx2_config_nix_queues(pf);
1379 if (err) {
4c3212f5 1380 mutex_unlock(&mbox->lock);
caa2da34
SG		 1381 goto err_free_txsch;
1382 }
1383 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1384 err = otx2_txschq_config(pf, lvl);
1385 if (err) {
4c3212f5 1386 mutex_unlock(&mbox->lock);
caa2da34
SG		 1387 goto err_free_nix_queues;
1388 }
1389 }
4c3212f5 1390 mutex_unlock(&mbox->lock);
caa2da34
SG		 1391 return err;
1392
1393err_free_nix_queues:
1394 otx2_free_sq_res(pf);
1395 otx2_free_cq_res(pf);
1396 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1397err_free_txsch:
1398 if (otx2_txschq_stop(pf))
1399 dev_err(pf->dev, "%s failed to stop TX schedulers\n", __func__);
1400err_free_sq_ptrs:
1401 otx2_sq_free_sqbs(pf);
1402err_free_rq_ptrs:
1403 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1404 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1405 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1406 otx2_aura_pool_free(pf);
1407err_free_nix_lf:
4c3212f5 1408 mutex_lock(&mbox->lock);
55307fcb
SS		 1409 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1410 if (free_req) {
1411 free_req->flags = NIX_LF_DISABLE_FLOWS;
caa2da34
SG		 1412 if (otx2_sync_mbox_msg(mbox))
1413 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1414 }
1415err_free_npa_lf:
1416 /* Reset NPA LF */
1417 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1418 if (req) {
1419 if (otx2_sync_mbox_msg(mbox))
1420 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1421 }
1422exit:
4c3212f5 1423 mutex_unlock(&mbox->lock);
caa2da34
SG		 1424 return err;
1425}
1426
1427static void otx2_free_hw_resources(struct otx2_nic *pf)
1428{
abe02543 1429 struct otx2_qset *qset = &pf->qset;
55307fcb 1430 struct nix_lf_free_req *free_req;
caa2da34 1431 struct mbox *mbox = &pf->mbox;
abe02543 1432 struct otx2_cq_queue *cq;
caa2da34 1433 struct msg_req *req;
abe02543 1434 int qidx, err;
caa2da34
SG		 1435
1436 /* Ensure all SQE are processed */
1437 otx2_sqb_flush(pf);
1438
1439 /* Stop transmission */
1440 err = otx2_txschq_stop(pf);
1441 if (err)
1442 dev_err(pf->dev, "RVUPF: Failed to stop/free TX schedulers\n");
1443
4c3212f5 1444 mutex_lock(&mbox->lock);
75f36270
G		 1445 /* Disable backpressure */
1446 if (!(pf->pcifunc & RVU_PFVF_FUNC_MASK))
1447 otx2_nix_config_bp(pf, false);
4c3212f5 1448 mutex_unlock(&mbox->lock);
75f36270 1449
caa2da34
SG		 1450 /* Disable RQs */
1451 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1452
abe02543
SG		 1453 /*Dequeue all CQEs */
1454 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1455 cq = &qset->cq[qidx];
1456 if (cq->cq_type == CQ_RX)
1457 otx2_cleanup_rx_cqes(pf, cq);
3ca6c4c8
SG		 1458 else
1459 otx2_cleanup_tx_cqes(pf, cq);
abe02543
SG		 1460 }
1461
caa2da34
SG		 1462 otx2_free_sq_res(pf);
1463
1464 /* Free RQ buffer pointers*/
1465 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1466
1467 otx2_free_cq_res(pf);
1468
2ca89a2c
SG		 1469 /* Free all ingress bandwidth profiles allocated */
1470 cn10k_free_all_ipolicers(pf);
1471
4c3212f5 1472 mutex_lock(&mbox->lock);
caa2da34 1473 /* Reset NIX LF */
55307fcb
SS		 1474 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1475 if (free_req) {
1476 free_req->flags = NIX_LF_DISABLE_FLOWS;
f0c2982a
NM		 1477 if (!(pf->flags & OTX2_FLAG_PF_SHUTDOWN))
1478 free_req->flags |= NIX_LF_DONT_FREE_TX_VTAG;
caa2da34
SG		 1479 if (otx2_sync_mbox_msg(mbox))
1480 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1481 }
4c3212f5 1482 mutex_unlock(&mbox->lock);
caa2da34
SG		 1483
1484 /* Disable NPA Pool and Aura hw context */
1485 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1486 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1487 otx2_aura_pool_free(pf);
1488
4c3212f5 1489 mutex_lock(&mbox->lock);
caa2da34
SG		 1490 /* Reset NPA LF */
1491 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1492 if (req) {
1493 if (otx2_sync_mbox_msg(mbox))
1494 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1495 }
4c3212f5 1496 mutex_unlock(&mbox->lock);
caa2da34
SG		 1497}
1498
34bfe0eb 1499int otx2_open(struct net_device *netdev)
16547577 1500{
05fcc9e0 1501 struct otx2_nic *pf = netdev_priv(netdev);
04a21ef3 1502 struct otx2_cq_poll *cq_poll = NULL;
caa2da34 1503 struct otx2_qset *qset = &pf->qset;
04a21ef3
SG		 1504 int err = 0, qidx, vec;
1505 char *irq_name;
05fcc9e0 1506
16547577
SG		 1507 netif_carrier_off(netdev);
1508
05fcc9e0 1509 pf->qset.cq_cnt = pf->hw.rx_queues + pf->hw.tx_queues;
04a21ef3
SG		 1510 /* RQ and SQs are mapped to different CQs,
1511 * so find out max CQ IRQs (i.e CINTs) needed.
1512 */
1513 pf->hw.cint_cnt = max(pf->hw.rx_queues, pf->hw.tx_queues);
1514 qset->napi = kcalloc(pf->hw.cint_cnt, sizeof(*cq_poll), GFP_KERNEL);
1515 if (!qset->napi)
1516 return -ENOMEM;
05fcc9e0 1517
caa2da34
SG		 1518 /* CQ size of RQ */
1519 qset->rqe_cnt = qset->rqe_cnt ? qset->rqe_cnt : Q_COUNT(Q_SIZE_256);
1520 /* CQ size of SQ */
1521 qset->sqe_cnt = qset->sqe_cnt ? qset->sqe_cnt : Q_COUNT(Q_SIZE_4K);
1522
1523 err = -ENOMEM;
1524 qset->cq = kcalloc(pf->qset.cq_cnt,
1525 sizeof(struct otx2_cq_queue), GFP_KERNEL);
1526 if (!qset->cq)
1527 goto err_free_mem;
1528
1529 qset->sq = kcalloc(pf->hw.tx_queues,
1530 sizeof(struct otx2_snd_queue), GFP_KERNEL);
1531 if (!qset->sq)
1532 goto err_free_mem;
1533
d45d8979
CJ		 1534 qset->rq = kcalloc(pf->hw.rx_queues,
1535 sizeof(struct otx2_rcv_queue), GFP_KERNEL);
1536 if (!qset->rq)
1537 goto err_free_mem;
1538
6e8ad438
G		 1539 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag)) {
1540 /* Reserve LMT lines for NPA AURA batch free */
5c051207 1541 pf->hw.npa_lmt_base = pf->hw.lmt_base;
6e8ad438 1542 /* Reserve LMT lines for NIX TX */
5c051207
G		 1543 pf->hw.nix_lmt_base = (u64 *)((u64)pf->hw.npa_lmt_base +
1544 (pf->npa_lmt_lines * LMT_LINE_SIZE));
6e8ad438
G		 1545 }
1546
caa2da34 1547 err = otx2_init_hw_resources(pf);
05fcc9e0 1548 if (err)
caa2da34 1549 goto err_free_mem;
05fcc9e0 1550
04a21ef3
SG		 1551 /* Register NAPI handler */
1552 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1553 cq_poll = &qset->napi[qidx];
1554 cq_poll->cint_idx = qidx;
1555 /* RQ0 & SQ0 are mapped to CINT0 and so on..
1556 * 'cq_ids[0]' points to RQ's CQ and
1557 * 'cq_ids[1]' points to SQ's CQ and
1558 */
1559 cq_poll->cq_ids[CQ_RX] =
1560 (qidx < pf->hw.rx_queues) ? qidx : CINT_INVALID_CQ;
1561 cq_poll->cq_ids[CQ_TX] = (qidx < pf->hw.tx_queues) ?
1562 qidx + pf->hw.rx_queues : CINT_INVALID_CQ;
1563 cq_poll->dev = (void *)pf;
1564 netif_napi_add(netdev, &cq_poll->napi,
1565 otx2_napi_handler, NAPI_POLL_WEIGHT);
1566 napi_enable(&cq_poll->napi);
1567 }
1568
34bfe0eb
SG		 1569 /* Set maximum frame size allowed in HW */
1570 err = otx2_hw_set_mtu(pf, netdev->mtu);
1571 if (err)
1572 goto err_disable_napi;
1573
dc1a9bf2
SG		 1574 /* Setup segmentation algorithms, if failed, clear offload capability */
1575 otx2_setup_segmentation(pf);
1576
85069e95
SG		 1577 /* Initialize RSS */
1578 err = otx2_rss_init(pf);
1579 if (err)
1580 goto err_disable_napi;
1581
4ff7d148
G		 1582 /* Register Queue IRQ handlers */
1583 vec = pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START;
1584 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
1585
1586 snprintf(irq_name, NAME_SIZE, "%s-qerr", pf->netdev->name);
1587
1588 err = request_irq(pci_irq_vector(pf->pdev, vec),
1589 otx2_q_intr_handler, 0, irq_name, pf);
1590 if (err) {
1591 dev_err(pf->dev,
1592 "RVUPF%d: IRQ registration failed for QERR\n",
1593 rvu_get_pf(pf->pcifunc));
1594 goto err_disable_napi;
1595 }
1596
1597 /* Enable QINT IRQ */
1598 otx2_write64(pf, NIX_LF_QINTX_ENA_W1S(0), BIT_ULL(0));
1599
04a21ef3
SG		 1600 /* Register CQ IRQ handlers */
1601 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
1602 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1603 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
1604
1605 snprintf(irq_name, NAME_SIZE, "%s-rxtx-%d", pf->netdev->name,
1606 qidx);
1607
1608 err = request_irq(pci_irq_vector(pf->pdev, vec),
1609 otx2_cq_intr_handler, 0, irq_name,
1610 &qset->napi[qidx]);
1611 if (err) {
1612 dev_err(pf->dev,
1613 "RVUPF%d: IRQ registration failed for CQ%d\n",
1614 rvu_get_pf(pf->pcifunc), qidx);
1615 goto err_free_cints;
1616 }
1617 vec++;
1618
1619 otx2_config_irq_coalescing(pf, qidx);
1620
1621 /* Enable CQ IRQ */
1622 otx2_write64(pf, NIX_LF_CINTX_INT(qidx), BIT_ULL(0));
1623 otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(qidx), BIT_ULL(0));
1624 }
1625
1626 otx2_set_cints_affinity(pf);
1627
fd9d7859
HK		 1628 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
1629 otx2_enable_rxvlan(pf, true);
1630
c9c12d33
AM		 1631 /* When reinitializing enable time stamping if it is enabled before */
1632 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED) {
1633 pf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
1634 otx2_config_hw_tx_tstamp(pf, true);
1635 }
1636 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) {
1637 pf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
1638 otx2_config_hw_rx_tstamp(pf, true);
1639 }
1640
50fe6c02
LC		 1641 pf->flags &= ~OTX2_FLAG_INTF_DOWN;
1642 /* 'intf_down' may be checked on any cpu */
1643 smp_wmb();
1644
1645 /* we have already received link status notification */
1646 if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK))
1647 otx2_handle_link_event(pf);
1648
3184fb5b
TD		 1649 /* Restore pause frame settings */
1650 otx2_config_pause_frm(pf);
1651
79d2be38
HK		 1652 /* Install DMAC Filters */
1653 if (pf->flags & OTX2_FLAG_DMACFLTR_SUPPORT)
1654 otx2_dmacflt_reinstall_flows(pf);
1655
50fe6c02
LC		 1656 err = otx2_rxtx_enable(pf, true);
1657 if (err)
1ea0166d 1658 goto err_tx_stop_queues;
50fe6c02 1659
caa2da34 1660 return 0;
04a21ef3 1661
1ea0166d
HK		 1662err_tx_stop_queues:
1663 netif_tx_stop_all_queues(netdev);
1664 netif_carrier_off(netdev);
04a21ef3
SG		 1665err_free_cints:
1666 otx2_free_cints(pf, qidx);
4ff7d148
G		 1667 vec = pci_irq_vector(pf->pdev,
1668 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
1669 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
1670 synchronize_irq(vec);
1671 free_irq(vec, pf);
34bfe0eb 1672err_disable_napi:
04a21ef3
SG		 1673 otx2_disable_napi(pf);
1674 otx2_free_hw_resources(pf);
caa2da34
SG		 1675err_free_mem:
1676 kfree(qset->sq);
1677 kfree(qset->cq);
d45d8979 1678 kfree(qset->rq);
04a21ef3 1679 kfree(qset->napi);
caa2da34 1680 return err;
16547577 1681}
3184fb5b 1682EXPORT_SYMBOL(otx2_open);
16547577 1683
34bfe0eb 1684int otx2_stop(struct net_device *netdev)
16547577 1685{
caa2da34 1686 struct otx2_nic *pf = netdev_priv(netdev);
04a21ef3 1687 struct otx2_cq_poll *cq_poll = NULL;
caa2da34 1688 struct otx2_qset *qset = &pf->qset;
f12098ce 1689 struct otx2_rss_info *rss;
4ff7d148 1690 int qidx, vec, wrk;
04a21ef3
SG		 1691
1692 netif_carrier_off(netdev);
1693 netif_tx_stop_all_queues(netdev);
1694
50fe6c02
LC		 1695 pf->flags |= OTX2_FLAG_INTF_DOWN;
1696 /* 'intf_down' may be checked on any cpu */
1697 smp_wmb();
1698
1699 /* First stop packet Rx/Tx */
1700 otx2_rxtx_enable(pf, false);
1701
f12098ce
G		 1702 /* Clear RSS enable flag */
1703 rss = &pf->hw.rss_info;
1704 rss->enable = false;
1705
4ff7d148
G		 1706 /* Cleanup Queue IRQ */
1707 vec = pci_irq_vector(pf->pdev,
1708 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
1709 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
1710 synchronize_irq(vec);
1711 free_irq(vec, pf);
1712
04a21ef3
SG		 1713 /* Cleanup CQ NAPI and IRQ */
1714 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
1715 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1716 /* Disable interrupt */
1717 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
1718
1719 synchronize_irq(pci_irq_vector(pf->pdev, vec));
1720
1721 cq_poll = &qset->napi[qidx];
1722 napi_synchronize(&cq_poll->napi);
1723 vec++;
1724 }
1725
1726 netif_tx_disable(netdev);
caa2da34
SG		 1727
1728 otx2_free_hw_resources(pf);
04a21ef3
SG		 1729 otx2_free_cints(pf, pf->hw.cint_cnt);
1730 otx2_disable_napi(pf);
1731
1732 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
1733 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
caa2da34 1734
4ff7d148
G		 1735 for (wrk = 0; wrk < pf->qset.cq_cnt; wrk++)
1736 cancel_delayed_work_sync(&pf->refill_wrk[wrk].pool_refill_work);
1737 devm_kfree(pf->dev, pf->refill_wrk);
1738
caa2da34
SG		 1739 kfree(qset->sq);
1740 kfree(qset->cq);
d45d8979 1741 kfree(qset->rq);
04a21ef3 1742 kfree(qset->napi);
caa2da34
SG		 1743 /* Do not clear RQ/SQ ringsize settings */
1744 memset((void *)qset + offsetof(struct otx2_qset, sqe_cnt), 0,
1745 sizeof(*qset) - offsetof(struct otx2_qset, sqe_cnt));
16547577
SG		 1746 return 0;
1747}
3184fb5b 1748EXPORT_SYMBOL(otx2_stop);
16547577 1749
3ca6c4c8
SG		 1750static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev)
1751{
1752 struct otx2_nic *pf = netdev_priv(netdev);
1753 int qidx = skb_get_queue_mapping(skb);
1754 struct otx2_snd_queue *sq;
1755 struct netdev_queue *txq;
1756
86d74760
SG		 1757 /* Check for minimum and maximum packet length */
1758 if (skb->len <= ETH_HLEN ||
1759 (!skb_shinfo(skb)->gso_size && skb->len > pf->max_frs)) {
3ca6c4c8
SG		 1760 dev_kfree_skb(skb);
1761 return NETDEV_TX_OK;
1762 }
1763
1764 sq = &pf->qset.sq[qidx];
1765 txq = netdev_get_tx_queue(netdev, qidx);
1766
1767 if (!otx2_sq_append_skb(netdev, sq, skb, qidx)) {
1768 netif_tx_stop_queue(txq);
1769
1770 /* Check again, incase SQBs got freed up */
1771 smp_mb();
1772 if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb)
1773 > sq->sqe_thresh)
1774 netif_tx_wake_queue(txq);
1775
1776 return NETDEV_TX_BUSY;
1777 }
1778
1779 return NETDEV_TX_OK;
1780}
1781
1d4d9e42
NM		 1782static netdev_features_t otx2_fix_features(struct net_device *dev,
1783 netdev_features_t features)
1784{
1785 /* check if n-tuple filters are ON */
1786 if ((features & NETIF_F_HW_TC) && (dev->features & NETIF_F_NTUPLE)) {
1787 netdev_info(dev, "Disabling n-tuple filters\n");
1788 features &= ~NETIF_F_NTUPLE;
1789 }
1790
1791 /* check if tc hw offload is ON */
1792 if ((features & NETIF_F_NTUPLE) && (dev->features & NETIF_F_HW_TC)) {
1793 netdev_info(dev, "Disabling TC hardware offload\n");
1794 features &= ~NETIF_F_HW_TC;
1795 }
1796
1797 return features;
1798}
1799
34bfe0eb
SG		 1800static void otx2_set_rx_mode(struct net_device *netdev)
1801{
1802 struct otx2_nic *pf = netdev_priv(netdev);
e99b7c84
SG		 1803
1804 queue_work(pf->otx2_wq, &pf->rx_mode_work);
1805}
1806
1807static void otx2_do_set_rx_mode(struct work_struct *work)
1808{
1809 struct otx2_nic *pf = container_of(work, struct otx2_nic, rx_mode_work);
1810 struct net_device *netdev = pf->netdev;
34bfe0eb 1811 struct nix_rx_mode *req;
63ee5157 1812 bool promisc = false;
34bfe0eb
SG		 1813
1814 if (!(netdev->flags & IFF_UP))
1815 return;
1816
63ee5157
HK		 1817 if ((netdev->flags & IFF_PROMISC) ||
1818 (netdev_uc_count(netdev) > OTX2_MAX_UNICAST_FLOWS)) {
1819 promisc = true;
1820 }
1821
1822 /* Write unicast address to mcam entries or del from mcam */
1823 if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT)
1824 __dev_uc_sync(netdev, otx2_add_macfilter, otx2_del_macfilter);
1825
4c3212f5 1826 mutex_lock(&pf->mbox.lock);
34bfe0eb
SG		 1827 req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox);
1828 if (!req) {
4c3212f5 1829 mutex_unlock(&pf->mbox.lock);
34bfe0eb
SG		 1830 return;
1831 }
1832
1833 req->mode = NIX_RX_MODE_UCAST;
1834
63ee5157 1835 if (promisc)
34bfe0eb 1836 req->mode |= NIX_RX_MODE_PROMISC;
967db352 1837 if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST))
34bfe0eb
SG		 1838 req->mode |= NIX_RX_MODE_ALLMULTI;
1839
967db352
NM		 1840 req->mode |= NIX_RX_MODE_USE_MCE;
1841
34bfe0eb 1842 otx2_sync_mbox_msg(&pf->mbox);
4c3212f5 1843 mutex_unlock(&pf->mbox.lock);
34bfe0eb
SG		 1844}
1845
1846static int otx2_set_features(struct net_device *netdev,
1847 netdev_features_t features)
1848{
1849 netdev_features_t changed = features ^ netdev->features;
f0a1913f 1850 bool ntuple = !!(features & NETIF_F_NTUPLE);
34bfe0eb
SG		 1851 struct otx2_nic *pf = netdev_priv(netdev);
1852
1853 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
1854 return otx2_cgx_config_loopback(pf,
1855 features & NETIF_F_LOOPBACK);
f0a1913f 1856
fd9d7859
HK		 1857 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(netdev))
1858 return otx2_enable_rxvlan(pf,
1859 features & NETIF_F_HW_VLAN_CTAG_RX);
1860
f0a1913f
SS		 1861 if ((changed & NETIF_F_NTUPLE) && !ntuple)
1862 otx2_destroy_ntuple_flows(pf);
1863
1d4d9e42
NM		 1864 if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) &&
1865 pf->tc_info.num_entries) {
1866 netdev_err(netdev, "Can't disable TC hardware offload while flows are active\n");
1867 return -EBUSY;
1868 }
1869
34bfe0eb
SG		 1870 return 0;
1871}
1872
4ff7d148
G		 1873static void otx2_reset_task(struct work_struct *work)
1874{
1875 struct otx2_nic *pf = container_of(work, struct otx2_nic, reset_task);
1876
1877 if (!netif_running(pf->netdev))
1878 return;
1879
948a6633 1880 rtnl_lock();
4ff7d148
G		 1881 otx2_stop(pf->netdev);
1882 pf->reset_count++;
1883 otx2_open(pf->netdev);
1884 netif_trans_update(pf->netdev);
948a6633 1885 rtnl_unlock();
4ff7d148
G		 1886}
1887
c9c12d33
AM		 1888static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable)
1889{
1890 struct msg_req *req;
1891 int err;
1892
1893 if (pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED && enable)
1894 return 0;
1895
1896 mutex_lock(&pfvf->mbox.lock);
1897 if (enable)
1898 req = otx2_mbox_alloc_msg_cgx_ptp_rx_enable(&pfvf->mbox);
1899 else
1900 req = otx2_mbox_alloc_msg_cgx_ptp_rx_disable(&pfvf->mbox);
1901 if (!req) {
1902 mutex_unlock(&pfvf->mbox.lock);
1903 return -ENOMEM;
1904 }
1905
1906 err = otx2_sync_mbox_msg(&pfvf->mbox);
1907 if (err) {
1908 mutex_unlock(&pfvf->mbox.lock);
1909 return err;
1910 }
1911
1912 mutex_unlock(&pfvf->mbox.lock);
1913 if (enable)
1914 pfvf->flags |= OTX2_FLAG_RX_TSTAMP_ENABLED;
1915 else
1916 pfvf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
1917 return 0;
1918}
1919
1920static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable)
1921{
1922 struct msg_req *req;
1923 int err;
1924
1925 if (pfvf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED && enable)
1926 return 0;
1927
1928 mutex_lock(&pfvf->mbox.lock);
1929 if (enable)
1930 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_enable(&pfvf->mbox);
1931 else
1932 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_disable(&pfvf->mbox);
1933 if (!req) {
1934 mutex_unlock(&pfvf->mbox.lock);
1935 return -ENOMEM;
1936 }
1937
1938 err = otx2_sync_mbox_msg(&pfvf->mbox);
1939 if (err) {
1940 mutex_unlock(&pfvf->mbox.lock);
1941 return err;
1942 }
1943
1944 mutex_unlock(&pfvf->mbox.lock);
1945 if (enable)
1946 pfvf->flags |= OTX2_FLAG_TX_TSTAMP_ENABLED;
1947 else
1948 pfvf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
1949 return 0;
1950}
1951
1952static int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr)
1953{
1954 struct otx2_nic *pfvf = netdev_priv(netdev);
1955 struct hwtstamp_config config;
1956
1957 if (!pfvf->ptp)
1958 return -ENODEV;
1959
1960 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1961 return -EFAULT;
1962
1963 /* reserved for future extensions */
1964 if (config.flags)
1965 return -EINVAL;
1966
1967 switch (config.tx_type) {
1968 case HWTSTAMP_TX_OFF:
1969 otx2_config_hw_tx_tstamp(pfvf, false);
1970 break;
1971 case HWTSTAMP_TX_ON:
1972 otx2_config_hw_tx_tstamp(pfvf, true);
1973 break;
1974 default:
1975 return -ERANGE;
1976 }
1977
1978 switch (config.rx_filter) {
1979 case HWTSTAMP_FILTER_NONE:
1980 otx2_config_hw_rx_tstamp(pfvf, false);
1981 break;
1982 case HWTSTAMP_FILTER_ALL:
1983 case HWTSTAMP_FILTER_SOME:
1984 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1985 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1986 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1987 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1988 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1989 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1990 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1991 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1992 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1993 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1994 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1995 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1996 otx2_config_hw_rx_tstamp(pfvf, true);
1997 config.rx_filter = HWTSTAMP_FILTER_ALL;
1998 break;
1999 default:
2000 return -ERANGE;
2001 }
2002
2003 memcpy(&pfvf->tstamp, &config, sizeof(config));
2004
2005 return copy_to_user(ifr->ifr_data, &config,
2006 sizeof(config)) ? -EFAULT : 0;
2007}
2008
2009static int otx2_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
2010{
2011 struct otx2_nic *pfvf = netdev_priv(netdev);
2012 struct hwtstamp_config *cfg = &pfvf->tstamp;
2013
2014 switch (cmd) {
2015 case SIOCSHWTSTAMP:
2016 return otx2_config_hwtstamp(netdev, req);
2017 case SIOCGHWTSTAMP:
2018 return copy_to_user(req->ifr_data, cfg,
2019 sizeof(*cfg)) ? -EFAULT : 0;
2020 default:
2021 return -EOPNOTSUPP;
2022 }
2023}
2024
f0c2982a
NM		 2025static int otx2_do_set_vf_mac(struct otx2_nic *pf, int vf, const u8 *mac)
2026{
2027 struct npc_install_flow_req *req;
2028 int err;
2029
2030 mutex_lock(&pf->mbox.lock);
2031 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2032 if (!req) {
2033 err = -ENOMEM;
2034 goto out;
2035 }
2036
2037 ether_addr_copy(req->packet.dmac, mac);
2038 eth_broadcast_addr((u8 *)&req->mask.dmac);
2039 req->features = BIT_ULL(NPC_DMAC);
2040 req->channel = pf->hw.rx_chan_base;
2041 req->intf = NIX_INTF_RX;
2042 req->default_rule = 1;
2043 req->append = 1;
2044 req->vf = vf + 1;
2045 req->op = NIX_RX_ACTION_DEFAULT;
2046
2047 err = otx2_sync_mbox_msg(&pf->mbox);
2048out:
2049 mutex_unlock(&pf->mbox.lock);
2050 return err;
2051}
2052
2053static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2054{
2055 struct otx2_nic *pf = netdev_priv(netdev);
2056 struct pci_dev *pdev = pf->pdev;
2057 struct otx2_vf_config *config;
2058 int ret;
2059
2060 if (!netif_running(netdev))
2061 return -EAGAIN;
2062
b1dc2040 2063 if (vf >= pf->total_vfs)
f0c2982a
NM		 2064 return -EINVAL;
2065
2066 if (!is_valid_ether_addr(mac))
2067 return -EINVAL;
2068
2069 config = &pf->vf_configs[vf];
2070 ether_addr_copy(config->mac, mac);
2071
2072 ret = otx2_do_set_vf_mac(pf, vf, mac);
2073 if (ret == 0)
b1dc2040
HK		 2074 dev_info(&pdev->dev,
2075 "Load/Reload VF driver\n");
f0c2982a
NM		 2076
2077 return ret;
2078}
2079
2080static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos,
2081 __be16 proto)
2082{
2083 struct otx2_flow_config *flow_cfg = pf->flow_cfg;
2084 struct nix_vtag_config_rsp *vtag_rsp;
2085 struct npc_delete_flow_req *del_req;
2086 struct nix_vtag_config *vtag_req;
2087 struct npc_install_flow_req *req;
2088 struct otx2_vf_config *config;
2089 int err = 0;
2090 u32 idx;
2091
2092 config = &pf->vf_configs[vf];
2093
2094 if (!vlan && !config->vlan)
2095 goto out;
2096
2097 mutex_lock(&pf->mbox.lock);
2098
2099 /* free old tx vtag entry */
2100 if (config->vlan) {
2101 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2102 if (!vtag_req) {
2103 err = -ENOMEM;
2104 goto out;
2105 }
2106 vtag_req->cfg_type = 0;
2107 vtag_req->tx.free_vtag0 = 1;
2108 vtag_req->tx.vtag0_idx = config->tx_vtag_idx;
2109
2110 err = otx2_sync_mbox_msg(&pf->mbox);
2111 if (err)
2112 goto out;
2113 }
2114
2115 if (!vlan && config->vlan) {
2116 /* rx */
2117 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2118 if (!del_req) {
2119 err = -ENOMEM;
2120 goto out;
2121 }
2122 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2123 del_req->entry =
9917060f 2124 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
f0c2982a
NM		 2125 err = otx2_sync_mbox_msg(&pf->mbox);
2126 if (err)
2127 goto out;
2128
2129 /* tx */
2130 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2131 if (!del_req) {
2132 err = -ENOMEM;
2133 goto out;
2134 }
2135 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2136 del_req->entry =
9917060f 2137 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
f0c2982a
NM		 2138 err = otx2_sync_mbox_msg(&pf->mbox);
2139
2140 goto out;
2141 }
2142
2143 /* rx */
2144 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2145 if (!req) {
2146 err = -ENOMEM;
2147 goto out;
2148 }
2149
2150 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
9917060f 2151 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
f0c2982a
NM		 2152 req->packet.vlan_tci = htons(vlan);
2153 req->mask.vlan_tci = htons(VLAN_VID_MASK);
2154 /* af fills the destination mac addr */
2155 eth_broadcast_addr((u8 *)&req->mask.dmac);
2156 req->features = BIT_ULL(NPC_OUTER_VID) | BIT_ULL(NPC_DMAC);
2157 req->channel = pf->hw.rx_chan_base;
2158 req->intf = NIX_INTF_RX;
2159 req->vf = vf + 1;
2160 req->op = NIX_RX_ACTION_DEFAULT;
2161 req->vtag0_valid = true;
2162 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
2163 req->set_cntr = 1;
2164
2165 err = otx2_sync_mbox_msg(&pf->mbox);
2166 if (err)
2167 goto out;
2168
2169 /* tx */
2170 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2171 if (!vtag_req) {
2172 err = -ENOMEM;
2173 goto out;
2174 }
2175
2176 /* configure tx vtag params */
2177 vtag_req->vtag_size = VTAGSIZE_T4;
2178 vtag_req->cfg_type = 0; /* tx vlan cfg */
2179 vtag_req->tx.cfg_vtag0 = 1;
583b273d 2180 vtag_req->tx.vtag0 = ((u64)ntohs(proto) << 16) | vlan;
f0c2982a
NM		 2181
2182 err = otx2_sync_mbox_msg(&pf->mbox);
2183 if (err)
2184 goto out;
2185
2186 vtag_rsp = (struct nix_vtag_config_rsp *)otx2_mbox_get_rsp
2187 (&pf->mbox.mbox, 0, &vtag_req->hdr);
2188 if (IS_ERR(vtag_rsp)) {
2189 err = PTR_ERR(vtag_rsp);
2190 goto out;
2191 }
2192 config->tx_vtag_idx = vtag_rsp->vtag0_idx;
2193
2194 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2195 if (!req) {
2196 err = -ENOMEM;
2197 goto out;
2198 }
2199
2200 eth_zero_addr((u8 *)&req->mask.dmac);
2201 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
9917060f 2202 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
f0c2982a
NM		 2203 req->features = BIT_ULL(NPC_DMAC);
2204 req->channel = pf->hw.tx_chan_base;
2205 req->intf = NIX_INTF_TX;
2206 req->vf = vf + 1;
2207 req->op = NIX_TX_ACTIONOP_UCAST_DEFAULT;
2208 req->vtag0_def = vtag_rsp->vtag0_idx;
2209 req->vtag0_op = VTAG_INSERT;
2210 req->set_cntr = 1;
2211
2212 err = otx2_sync_mbox_msg(&pf->mbox);
2213out:
2214 config->vlan = vlan;
2215 mutex_unlock(&pf->mbox.lock);
2216 return err;
2217}
2218
2219static int otx2_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
2220 __be16 proto)
2221{
2222 struct otx2_nic *pf = netdev_priv(netdev);
2223 struct pci_dev *pdev = pf->pdev;
2224
2225 if (!netif_running(netdev))
2226 return -EAGAIN;
2227
2228 if (vf >= pci_num_vf(pdev))
2229 return -EINVAL;
2230
2231 /* qos is currently unsupported */
2232 if (vlan >= VLAN_N_VID || qos)
2233 return -EINVAL;
2234
2235 if (proto != htons(ETH_P_8021Q))
2236 return -EPROTONOSUPPORT;
2237
2238 if (!(pf->flags & OTX2_FLAG_VF_VLAN_SUPPORT))
2239 return -EOPNOTSUPP;
2240
2241 return otx2_do_set_vf_vlan(pf, vf, vlan, qos, proto);
2242}
2243
2244static int otx2_get_vf_config(struct net_device *netdev, int vf,
2245 struct ifla_vf_info *ivi)
2246{
2247 struct otx2_nic *pf = netdev_priv(netdev);
2248 struct pci_dev *pdev = pf->pdev;
2249 struct otx2_vf_config *config;
2250
2251 if (!netif_running(netdev))
2252 return -EAGAIN;
2253
2254 if (vf >= pci_num_vf(pdev))
2255 return -EINVAL;
2256
2257 config = &pf->vf_configs[vf];
2258 ivi->vf = vf;
2259 ether_addr_copy(ivi->mac, config->mac);
2260 ivi->vlan = config->vlan;
b1dc2040 2261 ivi->trusted = config->trusted;
f0c2982a
NM		 2262
2263 return 0;
2264}
2265
b1dc2040
HK		 2266static int otx2_set_vf_permissions(struct otx2_nic *pf, int vf,
2267 int req_perm)
2268{
2269 struct set_vf_perm *req;
2270 int rc;
2271
2272 mutex_lock(&pf->mbox.lock);
2273 req = otx2_mbox_alloc_msg_set_vf_perm(&pf->mbox);
2274 if (!req) {
2275 rc = -ENOMEM;
2276 goto out;
2277 }
2278
2279 /* Let AF reset VF permissions as sriov is disabled */
2280 if (req_perm == OTX2_RESET_VF_PERM) {
2281 req->flags |= RESET_VF_PERM;
2282 } else if (req_perm == OTX2_TRUSTED_VF) {
2283 if (pf->vf_configs[vf].trusted)
2284 req->flags |= VF_TRUSTED;
2285 }
2286
2287 req->vf = vf;
2288 rc = otx2_sync_mbox_msg(&pf->mbox);
2289out:
2290 mutex_unlock(&pf->mbox.lock);
2291 return rc;
2292}
2293
2294static int otx2_ndo_set_vf_trust(struct net_device *netdev, int vf,
2295 bool enable)
2296{
2297 struct otx2_nic *pf = netdev_priv(netdev);
2298 struct pci_dev *pdev = pf->pdev;
2299 int rc;
2300
2301 if (vf >= pci_num_vf(pdev))
2302 return -EINVAL;
2303
2304 if (pf->vf_configs[vf].trusted == enable)
2305 return 0;
2306
2307 pf->vf_configs[vf].trusted = enable;
2308 rc = otx2_set_vf_permissions(pf, vf, OTX2_TRUSTED_VF);
2309
2310 if (rc)
2311 pf->vf_configs[vf].trusted = !enable;
2312 else
2313 netdev_info(pf->netdev, "VF %d is %strusted\n",
2314 vf, enable ? "" : "not ");
2315 return rc;
2316}
2317
16547577
SG		 2318static const struct net_device_ops otx2_netdev_ops = {
2319 .ndo_open = otx2_open,
2320 .ndo_stop = otx2_stop,
3ca6c4c8 2321 .ndo_start_xmit = otx2_xmit,
1d4d9e42 2322 .ndo_fix_features = otx2_fix_features,
34bfe0eb
SG		 2323 .ndo_set_mac_address = otx2_set_mac_address,
2324 .ndo_change_mtu = otx2_change_mtu,
2325 .ndo_set_rx_mode = otx2_set_rx_mode,
2326 .ndo_set_features = otx2_set_features,
4ff7d148 2327 .ndo_tx_timeout = otx2_tx_timeout,
e239d0c7 2328 .ndo_get_stats64 = otx2_get_stats64,
a7605370 2329 .ndo_eth_ioctl = otx2_ioctl,
f0c2982a
NM		 2330 .ndo_set_vf_mac = otx2_set_vf_mac,
2331 .ndo_set_vf_vlan = otx2_set_vf_vlan,
2332 .ndo_get_vf_config = otx2_get_vf_config,
1d4d9e42 2333 .ndo_setup_tc = otx2_setup_tc,
b1dc2040 2334 .ndo_set_vf_trust = otx2_ndo_set_vf_trust,
16547577
SG		 2335};
2336
e99b7c84
SG		 2337static int otx2_wq_init(struct otx2_nic *pf)
2338{
2339 pf->otx2_wq = create_singlethread_workqueue("otx2_wq");
2340 if (!pf->otx2_wq)
2341 return -ENOMEM;
2342
2343 INIT_WORK(&pf->rx_mode_work, otx2_do_set_rx_mode);
2344 INIT_WORK(&pf->reset_task, otx2_reset_task);
2345 return 0;
2346}
2347
16547577
SG		 2348static int otx2_check_pf_usable(struct otx2_nic *nic)
2349{
2350 u64 rev;
2351
2352 rev = otx2_read64(nic, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM));
2353 rev = (rev >> 12) & 0xFF;
2354 /* Check if AF has setup revision for RVUM block,
2355 * otherwise this driver probe should be deferred
2356 * until AF driver comes up.
2357 */
2358 if (!rev) {
2359 dev_warn(nic->dev,
2360 "AF is not initialized, deferring probe\n");
2361 return -EPROBE_DEFER;
2362 }
2363 return 0;
2364}
2365
05fcc9e0
SG		 2366static int otx2_realloc_msix_vectors(struct otx2_nic *pf)
2367{
2368 struct otx2_hw *hw = &pf->hw;
2369 int num_vec, err;
2370
2371 /* NPA interrupts are inot registered, so alloc only
2372 * upto NIX vector offset.
2373 */
2374 num_vec = hw->nix_msixoff;
05fcc9e0
SG		 2375 num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
2376
2377 otx2_disable_mbox_intr(pf);
2378 pci_free_irq_vectors(hw->pdev);
05fcc9e0
SG		 2379 err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX);
2380 if (err < 0) {
2381 dev_err(pf->dev, "%s: Failed to realloc %d IRQ vectors\n",
2382 __func__, num_vec);
2383 return err;
2384 }
2385
2386 return otx2_register_mbox_intr(pf, false);
2387}
2388
b1dc2040
HK		 2389static int otx2_sriov_vfcfg_init(struct otx2_nic *pf)
2390{
2391 int i;
2392
2393 pf->vf_configs = devm_kcalloc(pf->dev, pf->total_vfs,
2394 sizeof(struct otx2_vf_config),
2395 GFP_KERNEL);
2396 if (!pf->vf_configs)
2397 return -ENOMEM;
2398
2399 for (i = 0; i < pf->total_vfs; i++) {
2400 pf->vf_configs[i].pf = pf;
2401 pf->vf_configs[i].intf_down = true;
2402 pf->vf_configs[i].trusted = false;
2403 INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work,
2404 otx2_vf_link_event_task);
2405 }
2406
2407 return 0;
2408}
2409
2410static void otx2_sriov_vfcfg_cleanup(struct otx2_nic *pf)
2411{
2412 int i;
2413
2414 if (!pf->vf_configs)
2415 return;
2416
2417 for (i = 0; i < pf->total_vfs; i++) {
2418 cancel_delayed_work_sync(&pf->vf_configs[i].link_event_work);
2419 otx2_set_vf_permissions(pf, i, OTX2_RESET_VF_PERM);
2420 }
2421}
2422
16547577
SG		 2423static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2424{
2425 struct device *dev = &pdev->dev;
2426 struct net_device *netdev;
2427 struct otx2_nic *pf;
2428 struct otx2_hw *hw;
2429 int err, qcount;
5a6d7c9d 2430 int num_vec;
16547577
SG		 2431
2432 err = pcim_enable_device(pdev);
2433 if (err) {
2434 dev_err(dev, "Failed to enable PCI device\n");
2435 return err;
2436 }
2437
2438 err = pci_request_regions(pdev, DRV_NAME);
2439 if (err) {
2440 dev_err(dev, "PCI request regions failed 0x%x\n", err);
2441 return err;
2442 }
2443
2444 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
2445 if (err) {
2446 dev_err(dev, "DMA mask config failed, abort\n");
2447 goto err_release_regions;
2448 }
2449
2450 pci_set_master(pdev);
2451
2452 /* Set number of queues */
05fcc9e0 2453 qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT);
16547577
SG		 2454
2455 netdev = alloc_etherdev_mqs(sizeof(*pf), qcount, qcount);
2456 if (!netdev) {
2457 err = -ENOMEM;
2458 goto err_release_regions;
2459 }
2460
2461 pci_set_drvdata(pdev, netdev);
2462 SET_NETDEV_DEV(netdev, &pdev->dev);
2463 pf = netdev_priv(netdev);
2464 pf->netdev = netdev;
2465 pf->pdev = pdev;
2466 pf->dev = dev;
d424b6c0 2467 pf->total_vfs = pci_sriov_get_totalvfs(pdev);
50fe6c02 2468 pf->flags |= OTX2_FLAG_INTF_DOWN;
16547577
SG		 2469
2470 hw = &pf->hw;
2471 hw->pdev = pdev;
2472 hw->rx_queues = qcount;
2473 hw->tx_queues = qcount;
2474 hw->max_queues = qcount;
2475
5a6d7c9d
SG		 2476 num_vec = pci_msix_vec_count(pdev);
2477 hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE,
2478 GFP_KERNEL);
654cad8b
WY		 2479 if (!hw->irq_name) {
2480 err = -ENOMEM;
5a6d7c9d 2481 goto err_free_netdev;
654cad8b 2482 }
5a6d7c9d
SG		 2483
2484 hw->affinity_mask = devm_kcalloc(&hw->pdev->dev, num_vec,
2485 sizeof(cpumask_var_t), GFP_KERNEL);
654cad8b
WY		 2486 if (!hw->affinity_mask) {
2487 err = -ENOMEM;
5a6d7c9d 2488 goto err_free_netdev;
654cad8b 2489 }
5a6d7c9d 2490
16547577
SG		 2491 /* Map CSRs */
2492 pf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
2493 if (!pf->reg_base) {
2494 dev_err(dev, "Unable to map physical function CSRs, aborting\n");
2495 err = -ENOMEM;
2496 goto err_free_netdev;
2497 }
2498
2499 err = otx2_check_pf_usable(pf);
2500 if (err)
2501 goto err_free_netdev;
2502
5a6d7c9d
SG		 2503 err = pci_alloc_irq_vectors(hw->pdev, RVU_PF_INT_VEC_CNT,
2504 RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX);
2505 if (err < 0) {
2506 dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n",
2507 __func__, num_vec);
2508 goto err_free_netdev;
2509 }
2510
6e8ad438
G		 2511 otx2_setup_dev_hw_settings(pf);
2512
5a6d7c9d
SG		 2513 /* Init PF <=> AF mailbox stuff */
2514 err = otx2_pfaf_mbox_init(pf);
2515 if (err)
2516 goto err_free_irq_vectors;
2517
2518 /* Register mailbox interrupt */
2519 err = otx2_register_mbox_intr(pf, true);
2520 if (err)
2521 goto err_mbox_destroy;
2522
05fcc9e0
SG		 2523 /* Request AF to attach NPA and NIX LFs to this PF.
2524 * NIX and NPA LFs are needed for this PF to function as a NIC.
2525 */
2526 err = otx2_attach_npa_nix(pf);
16547577 2527 if (err)
5a6d7c9d 2528 goto err_disable_mbox_intr;
16547577 2529
05fcc9e0
SG		 2530 err = otx2_realloc_msix_vectors(pf);
2531 if (err)
2532 goto err_detach_rsrc;
2533
2534 err = otx2_set_real_num_queues(netdev, hw->tx_queues, hw->rx_queues);
2535 if (err)
2536 goto err_detach_rsrc;
2537
5c051207 2538 err = cn10k_lmtst_init(pf);
6e8ad438
G		 2539 if (err)
2540 goto err_detach_rsrc;
04a21ef3 2541
34bfe0eb
SG		 2542 /* Assign default mac address */
2543 otx2_get_mac_from_af(netdev);
2544
c9c12d33
AM		 2545 /* Don't check for error. Proceed without ptp */
2546 otx2_ptp_init(pf);
2547
caa2da34
SG		 2548 /* NPA's pool is a stack to which SW frees buffer pointers via Aura.
2549 * HW allocates buffer pointer from stack and uses it for DMA'ing
2550 * ingress packet. In some scenarios HW can free back allocated buffer
2551 * pointers to pool. This makes it impossible for SW to maintain a
2552 * parallel list where physical addresses of buffer pointers (IOVAs)
2553 * given to HW can be saved for later reference.
2554 *
2555 * So the only way to convert Rx packet's buffer address is to use
2556 * IOMMU's iova_to_phys() handler which translates the address by
2557 * walking through the translation tables.
2558 */
2559 pf->iommu_domain = iommu_get_domain_for_dev(dev);
2560
3ca6c4c8 2561 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
85069e95 2562 NETIF_F_IPV6_CSUM | NETIF_F_RXHASH |
dc1a9bf2
SG		 2563 NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
2564 NETIF_F_GSO_UDP_L4);
abe02543
SG		 2565 netdev->features |= netdev->hw_features;
2566
34bfe0eb 2567 netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL;
abe02543 2568
f0a1913f
SS		 2569 err = otx2_mcam_flow_init(pf);
2570 if (err)
2571 goto err_ptp_destroy;
2572
2573 if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT)
2574 netdev->hw_features |= NETIF_F_NTUPLE;
2575
63ee5157
HK		 2576 if (pf->flags & OTX2_FLAG_UCAST_FLTR_SUPPORT)
2577 netdev->priv_flags |= IFF_UNICAST_FLT;
2578
fd9d7859
HK		 2579 /* Support TSO on tag interface */
2580 netdev->vlan_features |= netdev->features;
2581 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
2582 NETIF_F_HW_VLAN_STAG_TX;
2583 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
2584 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX |
2585 NETIF_F_HW_VLAN_STAG_RX;
2586 netdev->features |= netdev->hw_features;
2587
1d4d9e42
NM		 2588 /* HW supports tc offload but mutually exclusive with n-tuple filters */
2589 if (pf->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)
2590 netdev->hw_features |= NETIF_F_HW_TC;
2591
86d74760 2592 netdev->gso_max_segs = OTX2_MAX_GSO_SEGS;
4ff7d148
G		 2593 netdev->watchdog_timeo = OTX2_TX_TIMEOUT;
2594
16547577
SG		 2595 netdev->netdev_ops = &otx2_netdev_ops;
2596
34bfe0eb
SG		 2597 /* MTU range: 64 - 9190 */
2598 netdev->min_mtu = OTX2_MIN_MTU;
ab58a416 2599 netdev->max_mtu = otx2_get_max_mtu(pf);
34bfe0eb 2600
16547577
SG		 2601 err = register_netdev(netdev);
2602 if (err) {
2603 dev_err(dev, "Failed to register netdevice\n");
f0a1913f 2604 goto err_del_mcam_entries;
16547577
SG		 2605 }
2606
e99b7c84
SG		 2607 err = otx2_wq_init(pf);
2608 if (err)
2609 goto err_unreg_netdev;
2610
d45d8979
CJ		 2611 otx2_set_ethtool_ops(netdev);
2612
1d4d9e42
NM		 2613 err = otx2_init_tc(pf);
2614 if (err)
2615 goto err_mcam_flow_del;
2616
b1dc2040
HK		 2617 /* Initialize SR-IOV resources */
2618 err = otx2_sriov_vfcfg_init(pf);
2619 if (err)
2620 goto err_pf_sriov_init;
2621
50fe6c02
LC		 2622 /* Enable link notifications */
2623 otx2_cgx_config_linkevents(pf, true);
2624
3184fb5b
TD		 2625 /* Enable pause frames by default */
2626 pf->flags |= OTX2_FLAG_RX_PAUSE_ENABLED;
2627 pf->flags |= OTX2_FLAG_TX_PAUSE_ENABLED;
2628
16547577
SG		 2629 return 0;
2630
b1dc2040
HK		 2631err_pf_sriov_init:
2632 otx2_shutdown_tc(pf);
1d4d9e42
NM		 2633err_mcam_flow_del:
2634 otx2_mcam_flow_del(pf);
e99b7c84
SG		 2635err_unreg_netdev:
2636 unregister_netdev(netdev);
f0a1913f
SS		 2637err_del_mcam_entries:
2638 otx2_mcam_flow_del(pf);
c9c12d33
AM		 2639err_ptp_destroy:
2640 otx2_ptp_destroy(pf);
05fcc9e0 2641err_detach_rsrc:
5c051207
G		 2642 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
2643 qmem_free(pf->dev, pf->dync_lmt);
05fcc9e0 2644 otx2_detach_resources(&pf->mbox);
5a6d7c9d
SG		 2645err_disable_mbox_intr:
2646 otx2_disable_mbox_intr(pf);
2647err_mbox_destroy:
2648 otx2_pfaf_mbox_destroy(pf);
2649err_free_irq_vectors:
2650 pci_free_irq_vectors(hw->pdev);
16547577
SG		 2651err_free_netdev:
2652 pci_set_drvdata(pdev, NULL);
2653 free_netdev(netdev);
2654err_release_regions:
2655 pci_release_regions(pdev);
2656 return err;
2657}
2658
ad513ed9
TD		 2659static void otx2_vf_link_event_task(struct work_struct *work)
2660{
2661 struct otx2_vf_config *config;
2662 struct cgx_link_info_msg *req;
2663 struct mbox_msghdr *msghdr;
2664 struct otx2_nic *pf;
2665 int vf_idx;
2666
2667 config = container_of(work, struct otx2_vf_config,
2668 link_event_work.work);
2669 vf_idx = config - config->pf->vf_configs;
2670 pf = config->pf;
2671
2672 msghdr = otx2_mbox_alloc_msg_rsp(&pf->mbox_pfvf[0].mbox_up, vf_idx,
2673 sizeof(*req), sizeof(struct msg_rsp));
2674 if (!msghdr) {
2675 dev_err(pf->dev, "Failed to create VF%d link event\n", vf_idx);
2676 return;
2677 }
2678
2679 req = (struct cgx_link_info_msg *)msghdr;
2680 req->hdr.id = MBOX_MSG_CGX_LINK_EVENT;
2681 req->hdr.sig = OTX2_MBOX_REQ_SIG;
2682 memcpy(&req->link_info, &pf->linfo, sizeof(req->link_info));
2683
2684 otx2_sync_mbox_up_msg(&pf->mbox_pfvf[0], vf_idx);
2685}
2686
d424b6c0
SG		 2687static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs)
2688{
2689 struct net_device *netdev = pci_get_drvdata(pdev);
2690 struct otx2_nic *pf = netdev_priv(netdev);
b1dc2040 2691 int ret;
d424b6c0
SG		 2692
2693 /* Init PF <=> VF mailbox stuff */
2694 ret = otx2_pfvf_mbox_init(pf, numvfs);
2695 if (ret)
2696 return ret;
2697
2698 ret = otx2_register_pfvf_mbox_intr(pf, numvfs);
2699 if (ret)
2700 goto free_mbox;
2701
547d20f1 2702 ret = otx2_pf_flr_init(pf, numvfs);
d424b6c0 2703 if (ret)
b1dc2040 2704 goto free_intr;
d424b6c0 2705
547d20f1
G		 2706 ret = otx2_register_flr_me_intr(pf, numvfs);
2707 if (ret)
2708 goto free_flr;
2709
2710 ret = pci_enable_sriov(pdev, numvfs);
2711 if (ret)
2712 goto free_flr_intr;
2713
d424b6c0 2714 return numvfs;
547d20f1
G		 2715free_flr_intr:
2716 otx2_disable_flr_me_intr(pf);
2717free_flr:
2718 otx2_flr_wq_destroy(pf);
d424b6c0
SG		 2719free_intr:
2720 otx2_disable_pfvf_mbox_intr(pf, numvfs);
2721free_mbox:
2722 otx2_pfvf_mbox_destroy(pf);
2723 return ret;
2724}
2725
2726static int otx2_sriov_disable(struct pci_dev *pdev)
2727{
2728 struct net_device *netdev = pci_get_drvdata(pdev);
2729 struct otx2_nic *pf = netdev_priv(netdev);
2730 int numvfs = pci_num_vf(pdev);
2731
2732 if (!numvfs)
2733 return 0;
2734
2735 pci_disable_sriov(pdev);
2736
547d20f1
G		 2737 otx2_disable_flr_me_intr(pf);
2738 otx2_flr_wq_destroy(pf);
d424b6c0
SG		 2739 otx2_disable_pfvf_mbox_intr(pf, numvfs);
2740 otx2_pfvf_mbox_destroy(pf);
2741
2742 return 0;
2743}
2744
2745static int otx2_sriov_configure(struct pci_dev *pdev, int numvfs)
2746{
2747 if (numvfs == 0)
2748 return otx2_sriov_disable(pdev);
2749 else
2750 return otx2_sriov_enable(pdev, numvfs);
2751}
2752
16547577
SG		 2753static void otx2_remove(struct pci_dev *pdev)
2754{
2755 struct net_device *netdev = pci_get_drvdata(pdev);
2756 struct otx2_nic *pf;
2757
2758 if (!netdev)
2759 return;
2760
2761 pf = netdev_priv(netdev);
2762
f0c2982a
NM		 2763 pf->flags |= OTX2_FLAG_PF_SHUTDOWN;
2764
c9c12d33
AM		 2765 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED)
2766 otx2_config_hw_tx_tstamp(pf, false);
2767 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED)
2768 otx2_config_hw_rx_tstamp(pf, false);
2769
948a6633 2770 cancel_work_sync(&pf->reset_task);
50fe6c02
LC		 2771 /* Disable link notifications */
2772 otx2_cgx_config_linkevents(pf, false);
2773
16547577 2774 unregister_netdev(netdev);
d424b6c0 2775 otx2_sriov_disable(pf->pdev);
b1dc2040 2776 otx2_sriov_vfcfg_cleanup(pf);
e99b7c84
SG		 2777 if (pf->otx2_wq)
2778 destroy_workqueue(pf->otx2_wq);
d424b6c0 2779
c9c12d33 2780 otx2_ptp_destroy(pf);
f0a1913f 2781 otx2_mcam_flow_del(pf);
1d4d9e42 2782 otx2_shutdown_tc(pf);
05fcc9e0 2783 otx2_detach_resources(&pf->mbox);
5c051207
G		 2784 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
2785 qmem_free(pf->dev, pf->dync_lmt);
5a6d7c9d
SG		 2786 otx2_disable_mbox_intr(pf);
2787 otx2_pfaf_mbox_destroy(pf);
16547577
SG		 2788 pci_free_irq_vectors(pf->pdev);
2789 pci_set_drvdata(pdev, NULL);
2790 free_netdev(netdev);
5a6d7c9d 2791
16547577
SG		 2792 pci_release_regions(pdev);
2793}
2794
2795static struct pci_driver otx2_pf_driver = {
2796 .name = DRV_NAME,
2797 .id_table = otx2_pf_id_table,
2798 .probe = otx2_probe,
2799 .shutdown = otx2_remove,
2800 .remove = otx2_remove,
d424b6c0 2801 .sriov_configure = otx2_sriov_configure
16547577
SG		 2802};
2803
2804static int __init otx2_rvupf_init_module(void)
2805{
2806 pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
2807
2808 return pci_register_driver(&otx2_pf_driver);
2809}
2810
2811static void __exit otx2_rvupf_cleanup_module(void)
2812{
2813 pci_unregister_driver(&otx2_pf_driver);
2814}
2815
2816module_init(otx2_rvupf_init_module);
2817module_exit(otx2_rvupf_cleanup_module);
Linux 6.x block layer and io_uring tree(s)