Commit | Line | Data |
---|---|---|
25763b3c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
4863dea3 SG |
2 | /* |
3 | * Copyright (C) 2015 Cavium, Inc. | |
4863dea3 SG |
4 | */ |
5 | ||
6 | #include <linux/module.h> | |
7 | #include <linux/interrupt.h> | |
8 | #include <linux/pci.h> | |
9 | #include <linux/netdevice.h> | |
aa2e259b | 10 | #include <linux/if_vlan.h> |
4863dea3 SG |
11 | #include <linux/etherdevice.h> |
12 | #include <linux/ethtool.h> | |
13 | #include <linux/log2.h> | |
14 | #include <linux/prefetch.h> | |
15 | #include <linux/irq.h> | |
83abb7d7 | 16 | #include <linux/iommu.h> |
05c773f5 | 17 | #include <linux/bpf.h> |
c56d91ce | 18 | #include <linux/bpf_trace.h> |
05c773f5 | 19 | #include <linux/filter.h> |
4a875509 | 20 | #include <linux/net_tstamp.h> |
37c3347e | 21 | #include <linux/workqueue.h> |
4863dea3 SG |
22 | |
23 | #include "nic_reg.h" | |
24 | #include "nic.h" | |
25 | #include "nicvf_queues.h" | |
26 | #include "thunder_bgx.h" | |
4a875509 | 27 | #include "../common/cavium_ptp.h" |
4863dea3 | 28 | |
6b9e6547 | 29 | #define DRV_NAME "nicvf" |
4863dea3 SG |
30 | #define DRV_VERSION "1.0" |
31 | ||
5ee15c10 MC |
32 | /* NOTE: Packets bigger than 1530 are split across multiple pages and XDP needs |
33 | * the buffer to be contiguous. Allow XDP to be set up only if we don't exceed | |
34 | * this value, keeping headroom for the 14 byte Ethernet header and two | |
35 | * VLAN tags (for QinQ) | |
36 | */ | |
37 | #define MAX_XDP_MTU (1530 - ETH_HLEN - VLAN_HLEN * 2) | |
38 | ||
4863dea3 SG |
39 | /* Supported devices */ |
40 | static const struct pci_device_id nicvf_id_table[] = { | |
41 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, | |
42 | PCI_DEVICE_ID_THUNDER_NIC_VF, | |
f7ff0ae8 SG |
43 | PCI_VENDOR_ID_CAVIUM, |
44 | PCI_SUBSYS_DEVID_88XX_NIC_VF) }, | |
4863dea3 SG |
45 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, |
46 | PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF, | |
f7ff0ae8 SG |
47 | PCI_VENDOR_ID_CAVIUM, |
48 | PCI_SUBSYS_DEVID_88XX_PASS1_NIC_VF) }, | |
49 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, | |
50 | PCI_DEVICE_ID_THUNDER_NIC_VF, | |
51 | PCI_VENDOR_ID_CAVIUM, | |
52 | PCI_SUBSYS_DEVID_81XX_NIC_VF) }, | |
53 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM, | |
54 | PCI_DEVICE_ID_THUNDER_NIC_VF, | |
55 | PCI_VENDOR_ID_CAVIUM, | |
56 | PCI_SUBSYS_DEVID_83XX_NIC_VF) }, | |
4863dea3 SG |
57 | { 0, } /* end of table */ |
58 | }; | |
59 | ||
60 | MODULE_AUTHOR("Sunil Goutham"); | |
61 | MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver"); | |
62 | MODULE_LICENSE("GPL v2"); | |
63 | MODULE_VERSION(DRV_VERSION); | |
64 | MODULE_DEVICE_TABLE(pci, nicvf_id_table); | |
65 | ||
66 | static int debug = 0x00; | |
67 | module_param(debug, int, 0644); | |
68 | MODULE_PARM_DESC(debug, "Debug message level bitmap"); | |
69 | ||
70 | static int cpi_alg = CPI_ALG_NONE; | |
d3757ba4 | 71 | module_param(cpi_alg, int, 0444); |
4863dea3 SG |
72 | MODULE_PARM_DESC(cpi_alg, |
73 | "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)"); | |
74 | ||
92dc8769 SG |
75 | static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx) |
76 | { | |
77 | if (nic->sqs_mode) | |
78 | return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS); | |
79 | else | |
80 | return qidx; | |
81 | } | |
82 | ||
4863dea3 SG |
83 | /* The Cavium ThunderX network controller can *only* be found in SoCs |
84 | * containing the ThunderX ARM64 CPU implementation. All accesses to the device | |
85 | * registers on this platform are implicitly strongly ordered with respect | |
86 | * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use | |
87 | * with no memory barriers in this driver. The readq()/writeq() functions add | |
88 | * explicit ordering operation which in this case are redundant, and only | |
89 | * add overhead. | |
90 | */ | |
91 | ||
92 | /* Register read/write APIs */ | |
93 | void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val) | |
94 | { | |
95 | writeq_relaxed(val, nic->reg_base + offset); | |
96 | } | |
97 | ||
98 | u64 nicvf_reg_read(struct nicvf *nic, u64 offset) | |
99 | { | |
100 | return readq_relaxed(nic->reg_base + offset); | |
101 | } | |
102 | ||
103 | void nicvf_queue_reg_write(struct nicvf *nic, u64 offset, | |
104 | u64 qidx, u64 val) | |
105 | { | |
106 | void __iomem *addr = nic->reg_base + offset; | |
107 | ||
108 | writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT)); | |
109 | } | |
110 | ||
111 | u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx) | |
112 | { | |
113 | void __iomem *addr = nic->reg_base + offset; | |
114 | ||
115 | return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT)); | |
116 | } | |
117 | ||
118 | /* VF -> PF mailbox communication */ | |
2cd2a196 AM |
119 | static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx) |
120 | { | |
121 | u64 *msg = (u64 *)mbx; | |
122 | ||
123 | nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]); | |
124 | nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]); | |
125 | } | |
126 | ||
4863dea3 SG |
127 | int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx) |
128 | { | |
605a9bbc | 129 | unsigned long timeout; |
609ea65c VL |
130 | int ret = 0; |
131 | ||
132 | mutex_lock(&nic->rx_mode_mtx); | |
4863dea3 SG |
133 | |
134 | nic->pf_acked = false; | |
135 | nic->pf_nacked = false; | |
136 | ||
2cd2a196 | 137 | nicvf_write_to_mbx(nic, mbx); |
4863dea3 | 138 | |
605a9bbc | 139 | timeout = jiffies + msecs_to_jiffies(NIC_MBOX_MSG_TIMEOUT); |
4863dea3 SG |
140 | /* Wait for previous message to be acked, timeout 2sec */ |
141 | while (!nic->pf_acked) { | |
ecae29cb RB |
142 | if (nic->pf_nacked) { |
143 | netdev_err(nic->netdev, | |
144 | "PF NACK to mbox msg 0x%02x from VF%d\n", | |
145 | (mbx->msg.msg & 0xFF), nic->vf_id); | |
609ea65c VL |
146 | ret = -EINVAL; |
147 | break; | |
ecae29cb | 148 | } |
605a9bbc | 149 | usleep_range(8000, 10000); |
4863dea3 SG |
150 | if (nic->pf_acked) |
151 | break; | |
605a9bbc | 152 | if (time_after(jiffies, timeout)) { |
4863dea3 | 153 | netdev_err(nic->netdev, |
ecae29cb | 154 | "PF didn't ACK to mbox msg 0x%02x from VF%d\n", |
4863dea3 | 155 | (mbx->msg.msg & 0xFF), nic->vf_id); |
609ea65c VL |
156 | ret = -EBUSY; |
157 | break; | |
4863dea3 SG |
158 | } |
159 | } | |
609ea65c VL |
160 | mutex_unlock(&nic->rx_mode_mtx); |
161 | return ret; | |
4863dea3 SG |
162 | } |
163 | ||
164 | /* Checks if VF is able to comminicate with PF | |
165 | * and also gets the VNIC number this VF is associated to. | |
166 | */ | |
167 | static int nicvf_check_pf_ready(struct nicvf *nic) | |
168 | { | |
2cd2a196 AM |
169 | union nic_mbx mbx = {}; |
170 | ||
171 | mbx.msg.msg = NIC_MBOX_MSG_READY; | |
6051cba7 SG |
172 | if (nicvf_send_msg_to_pf(nic, &mbx)) { |
173 | netdev_err(nic->netdev, | |
174 | "PF didn't respond to READY msg\n"); | |
175 | return 0; | |
4863dea3 | 176 | } |
6051cba7 | 177 | |
4863dea3 SG |
178 | return 1; |
179 | } | |
180 | ||
0dd563b9 VL |
181 | static void nicvf_send_cfg_done(struct nicvf *nic) |
182 | { | |
183 | union nic_mbx mbx = {}; | |
184 | ||
185 | mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE; | |
186 | if (nicvf_send_msg_to_pf(nic, &mbx)) { | |
187 | netdev_err(nic->netdev, | |
188 | "PF didn't respond to CFG DONE msg\n"); | |
189 | } | |
190 | } | |
191 | ||
fd7ec062 AM |
192 | static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx) |
193 | { | |
194 | if (bgx->rx) | |
195 | nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats; | |
196 | else | |
197 | nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats; | |
198 | } | |
199 | ||
4863dea3 SG |
200 | static void nicvf_handle_mbx_intr(struct nicvf *nic) |
201 | { | |
202 | union nic_mbx mbx = {}; | |
203 | u64 *mbx_data; | |
204 | u64 mbx_addr; | |
205 | int i; | |
206 | ||
207 | mbx_addr = NIC_VF_PF_MAILBOX_0_1; | |
208 | mbx_data = (u64 *)&mbx; | |
209 | ||
210 | for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) { | |
211 | *mbx_data = nicvf_reg_read(nic, mbx_addr); | |
212 | mbx_data++; | |
213 | mbx_addr += sizeof(u64); | |
214 | } | |
215 | ||
216 | netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg); | |
217 | switch (mbx.msg.msg) { | |
218 | case NIC_MBOX_MSG_READY: | |
6051cba7 | 219 | nic->pf_acked = true; |
4863dea3 SG |
220 | nic->vf_id = mbx.nic_cfg.vf_id & 0x7F; |
221 | nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F; | |
222 | nic->node = mbx.nic_cfg.node_id; | |
bd049a90 | 223 | if (!nic->set_mac_pending) |
f3956ebb | 224 | eth_hw_addr_set(nic->netdev, mbx.nic_cfg.mac_addr); |
92dc8769 | 225 | nic->sqs_mode = mbx.nic_cfg.sqs_mode; |
d77a2384 | 226 | nic->loopback_supported = mbx.nic_cfg.loopback_supported; |
4863dea3 SG |
227 | nic->link_up = false; |
228 | nic->duplex = 0; | |
229 | nic->speed = 0; | |
230 | break; | |
231 | case NIC_MBOX_MSG_ACK: | |
232 | nic->pf_acked = true; | |
233 | break; | |
234 | case NIC_MBOX_MSG_NACK: | |
235 | nic->pf_nacked = true; | |
236 | break; | |
237 | case NIC_MBOX_MSG_RSS_SIZE: | |
238 | nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size; | |
239 | nic->pf_acked = true; | |
240 | break; | |
241 | case NIC_MBOX_MSG_BGX_STATS: | |
242 | nicvf_read_bgx_stats(nic, &mbx.bgx_stats); | |
243 | nic->pf_acked = true; | |
4863dea3 SG |
244 | break; |
245 | case NIC_MBOX_MSG_BGX_LINK_CHANGE: | |
246 | nic->pf_acked = true; | |
2c632ad8 VL |
247 | if (nic->link_up != mbx.link_status.link_up) { |
248 | nic->link_up = mbx.link_status.link_up; | |
249 | nic->duplex = mbx.link_status.duplex; | |
250 | nic->speed = mbx.link_status.speed; | |
251 | nic->mac_type = mbx.link_status.mac_type; | |
252 | if (nic->link_up) { | |
253 | netdev_info(nic->netdev, | |
254 | "Link is Up %d Mbps %s duplex\n", | |
255 | nic->speed, | |
256 | nic->duplex == DUPLEX_FULL ? | |
257 | "Full" : "Half"); | |
258 | netif_carrier_on(nic->netdev); | |
259 | netif_tx_start_all_queues(nic->netdev); | |
260 | } else { | |
261 | netdev_info(nic->netdev, "Link is Down\n"); | |
262 | netif_carrier_off(nic->netdev); | |
263 | netif_tx_stop_all_queues(nic->netdev); | |
264 | } | |
4863dea3 SG |
265 | } |
266 | break; | |
92dc8769 SG |
267 | case NIC_MBOX_MSG_ALLOC_SQS: |
268 | nic->sqs_count = mbx.sqs_alloc.qs_count; | |
269 | nic->pf_acked = true; | |
270 | break; | |
271 | case NIC_MBOX_MSG_SNICVF_PTR: | |
272 | /* Primary VF: make note of secondary VF's pointer | |
273 | * to be used while packet transmission. | |
274 | */ | |
275 | nic->snicvf[mbx.nicvf.sqs_id] = | |
276 | (struct nicvf *)mbx.nicvf.nicvf; | |
277 | nic->pf_acked = true; | |
278 | break; | |
279 | case NIC_MBOX_MSG_PNICVF_PTR: | |
280 | /* Secondary VF/Qset: make note of primary VF's pointer | |
281 | * to be used while packet reception, to handover packet | |
282 | * to primary VF's netdev. | |
283 | */ | |
284 | nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf; | |
285 | nic->pf_acked = true; | |
286 | break; | |
430da208 SG |
287 | case NIC_MBOX_MSG_PFC: |
288 | nic->pfc.autoneg = mbx.pfc.autoneg; | |
289 | nic->pfc.fc_rx = mbx.pfc.fc_rx; | |
290 | nic->pfc.fc_tx = mbx.pfc.fc_tx; | |
291 | nic->pf_acked = true; | |
292 | break; | |
4863dea3 SG |
293 | default: |
294 | netdev_err(nic->netdev, | |
295 | "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); | |
296 | break; | |
297 | } | |
298 | nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0); | |
299 | } | |
300 | ||
301 | static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev) | |
302 | { | |
303 | union nic_mbx mbx = {}; | |
4863dea3 SG |
304 | |
305 | mbx.mac.msg = NIC_MBOX_MSG_SET_MAC; | |
306 | mbx.mac.vf_id = nic->vf_id; | |
e610cb32 | 307 | ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr); |
4863dea3 SG |
308 | |
309 | return nicvf_send_msg_to_pf(nic, &mbx); | |
310 | } | |
311 | ||
fd7ec062 | 312 | static void nicvf_config_cpi(struct nicvf *nic) |
4863dea3 SG |
313 | { |
314 | union nic_mbx mbx = {}; | |
315 | ||
316 | mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG; | |
317 | mbx.cpi_cfg.vf_id = nic->vf_id; | |
318 | mbx.cpi_cfg.cpi_alg = nic->cpi_alg; | |
319 | mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt; | |
320 | ||
321 | nicvf_send_msg_to_pf(nic, &mbx); | |
322 | } | |
323 | ||
fd7ec062 | 324 | static void nicvf_get_rss_size(struct nicvf *nic) |
4863dea3 SG |
325 | { |
326 | union nic_mbx mbx = {}; | |
327 | ||
328 | mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE; | |
329 | mbx.rss_size.vf_id = nic->vf_id; | |
330 | nicvf_send_msg_to_pf(nic, &mbx); | |
331 | } | |
332 | ||
333 | void nicvf_config_rss(struct nicvf *nic) | |
334 | { | |
335 | union nic_mbx mbx = {}; | |
336 | struct nicvf_rss_info *rss = &nic->rss_info; | |
337 | int ind_tbl_len = rss->rss_size; | |
338 | int i, nextq = 0; | |
339 | ||
340 | mbx.rss_cfg.vf_id = nic->vf_id; | |
341 | mbx.rss_cfg.hash_bits = rss->hash_bits; | |
342 | while (ind_tbl_len) { | |
343 | mbx.rss_cfg.tbl_offset = nextq; | |
344 | mbx.rss_cfg.tbl_len = min(ind_tbl_len, | |
345 | RSS_IND_TBL_LEN_PER_MBX_MSG); | |
346 | mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ? | |
347 | NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG; | |
348 | ||
349 | for (i = 0; i < mbx.rss_cfg.tbl_len; i++) | |
350 | mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++]; | |
351 | ||
352 | nicvf_send_msg_to_pf(nic, &mbx); | |
353 | ||
354 | ind_tbl_len -= mbx.rss_cfg.tbl_len; | |
355 | } | |
356 | } | |
357 | ||
358 | void nicvf_set_rss_key(struct nicvf *nic) | |
359 | { | |
360 | struct nicvf_rss_info *rss = &nic->rss_info; | |
361 | u64 key_addr = NIC_VNIC_RSS_KEY_0_4; | |
362 | int idx; | |
363 | ||
364 | for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) { | |
365 | nicvf_reg_write(nic, key_addr, rss->key[idx]); | |
366 | key_addr += sizeof(u64); | |
367 | } | |
368 | } | |
369 | ||
370 | static int nicvf_rss_init(struct nicvf *nic) | |
371 | { | |
372 | struct nicvf_rss_info *rss = &nic->rss_info; | |
373 | int idx; | |
374 | ||
375 | nicvf_get_rss_size(nic); | |
376 | ||
38bb5d4f | 377 | if (cpi_alg != CPI_ALG_NONE) { |
4863dea3 SG |
378 | rss->enable = false; |
379 | rss->hash_bits = 0; | |
380 | return 0; | |
381 | } | |
382 | ||
383 | rss->enable = true; | |
384 | ||
0052c92f | 385 | netdev_rss_key_fill(rss->key, RSS_HASH_KEY_SIZE * sizeof(u64)); |
4863dea3 SG |
386 | nicvf_set_rss_key(nic); |
387 | ||
388 | rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA; | |
389 | nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg); | |
390 | ||
391 | rss->hash_bits = ilog2(rounddown_pow_of_two(rss->rss_size)); | |
392 | ||
393 | for (idx = 0; idx < rss->rss_size; idx++) | |
394 | rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx, | |
92dc8769 | 395 | nic->rx_queues); |
4863dea3 SG |
396 | nicvf_config_rss(nic); |
397 | return 1; | |
398 | } | |
399 | ||
92dc8769 SG |
400 | /* Request PF to allocate additional Qsets */ |
401 | static void nicvf_request_sqs(struct nicvf *nic) | |
402 | { | |
403 | union nic_mbx mbx = {}; | |
404 | int sqs; | |
405 | int sqs_count = nic->sqs_count; | |
406 | int rx_queues = 0, tx_queues = 0; | |
407 | ||
408 | /* Only primary VF should request */ | |
409 | if (nic->sqs_mode || !nic->sqs_count) | |
410 | return; | |
411 | ||
412 | mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS; | |
413 | mbx.sqs_alloc.vf_id = nic->vf_id; | |
414 | mbx.sqs_alloc.qs_count = nic->sqs_count; | |
415 | if (nicvf_send_msg_to_pf(nic, &mbx)) { | |
416 | /* No response from PF */ | |
417 | nic->sqs_count = 0; | |
418 | return; | |
419 | } | |
420 | ||
421 | /* Return if no Secondary Qsets available */ | |
422 | if (!nic->sqs_count) | |
423 | return; | |
424 | ||
425 | if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS) | |
426 | rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS; | |
05c773f5 SG |
427 | |
428 | tx_queues = nic->tx_queues + nic->xdp_tx_queues; | |
429 | if (tx_queues > MAX_SND_QUEUES_PER_QS) | |
430 | tx_queues = tx_queues - MAX_SND_QUEUES_PER_QS; | |
92dc8769 SG |
431 | |
432 | /* Set no of Rx/Tx queues in each of the SQsets */ | |
433 | for (sqs = 0; sqs < nic->sqs_count; sqs++) { | |
434 | mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR; | |
435 | mbx.nicvf.vf_id = nic->vf_id; | |
436 | mbx.nicvf.sqs_id = sqs; | |
437 | nicvf_send_msg_to_pf(nic, &mbx); | |
438 | ||
439 | nic->snicvf[sqs]->sqs_id = sqs; | |
440 | if (rx_queues > MAX_RCV_QUEUES_PER_QS) { | |
441 | nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS; | |
442 | rx_queues -= MAX_RCV_QUEUES_PER_QS; | |
443 | } else { | |
444 | nic->snicvf[sqs]->qs->rq_cnt = rx_queues; | |
445 | rx_queues = 0; | |
446 | } | |
447 | ||
448 | if (tx_queues > MAX_SND_QUEUES_PER_QS) { | |
449 | nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS; | |
450 | tx_queues -= MAX_SND_QUEUES_PER_QS; | |
451 | } else { | |
452 | nic->snicvf[sqs]->qs->sq_cnt = tx_queues; | |
453 | tx_queues = 0; | |
454 | } | |
455 | ||
456 | nic->snicvf[sqs]->qs->cq_cnt = | |
457 | max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt); | |
458 | ||
459 | /* Initialize secondary Qset's queues and its interrupts */ | |
460 | nicvf_open(nic->snicvf[sqs]->netdev); | |
461 | } | |
462 | ||
463 | /* Update stack with actual Rx/Tx queue count allocated */ | |
464 | if (sqs_count != nic->sqs_count) | |
465 | nicvf_set_real_num_queues(nic->netdev, | |
466 | nic->tx_queues, nic->rx_queues); | |
467 | } | |
468 | ||
469 | /* Send this Qset's nicvf pointer to PF. | |
470 | * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs | |
471 | * so that packets received by these Qsets can use primary VF's netdev | |
472 | */ | |
473 | static void nicvf_send_vf_struct(struct nicvf *nic) | |
474 | { | |
475 | union nic_mbx mbx = {}; | |
476 | ||
477 | mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR; | |
478 | mbx.nicvf.sqs_mode = nic->sqs_mode; | |
479 | mbx.nicvf.nicvf = (u64)nic; | |
480 | nicvf_send_msg_to_pf(nic, &mbx); | |
481 | } | |
482 | ||
483 | static void nicvf_get_primary_vf_struct(struct nicvf *nic) | |
484 | { | |
485 | union nic_mbx mbx = {}; | |
486 | ||
487 | mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR; | |
488 | nicvf_send_msg_to_pf(nic, &mbx); | |
489 | } | |
490 | ||
4863dea3 SG |
491 | int nicvf_set_real_num_queues(struct net_device *netdev, |
492 | int tx_queues, int rx_queues) | |
493 | { | |
494 | int err = 0; | |
495 | ||
496 | err = netif_set_real_num_tx_queues(netdev, tx_queues); | |
497 | if (err) { | |
498 | netdev_err(netdev, | |
499 | "Failed to set no of Tx queues: %d\n", tx_queues); | |
500 | return err; | |
501 | } | |
502 | ||
503 | err = netif_set_real_num_rx_queues(netdev, rx_queues); | |
504 | if (err) | |
505 | netdev_err(netdev, | |
506 | "Failed to set no of Rx queues: %d\n", rx_queues); | |
507 | return err; | |
508 | } | |
509 | ||
510 | static int nicvf_init_resources(struct nicvf *nic) | |
511 | { | |
512 | int err; | |
4863dea3 SG |
513 | |
514 | /* Enable Qset */ | |
515 | nicvf_qset_config(nic, true); | |
516 | ||
517 | /* Initialize queues and HW for data transfer */ | |
518 | err = nicvf_config_data_transfer(nic, true); | |
519 | if (err) { | |
520 | netdev_err(nic->netdev, | |
521 | "Failed to alloc/config VF's QSet resources\n"); | |
522 | return err; | |
523 | } | |
524 | ||
4863dea3 SG |
525 | return 0; |
526 | } | |
527 | ||
16f2bccd | 528 | static inline bool nicvf_xdp_rx(struct nicvf *nic, struct bpf_prog *prog, |
e3d06ff9 | 529 | struct cqe_rx_t *cqe_rx, struct snd_queue *sq, |
27e95e36 | 530 | struct rcv_queue *rq, struct sk_buff **skb) |
05c773f5 | 531 | { |
be9df4af | 532 | unsigned char *hard_start, *data; |
05c773f5 | 533 | struct xdp_buff xdp; |
c56d91ce | 534 | struct page *page; |
05c773f5 | 535 | u32 action; |
e6dbe939 | 536 | u16 len, offset = 0; |
05c773f5 | 537 | u64 dma_addr, cpu_addr; |
e3d06ff9 | 538 | void *orig_data; |
05c773f5 SG |
539 | |
540 | /* Retrieve packet buffer's DMA address and length */ | |
541 | len = *((u16 *)((void *)cqe_rx + (3 * sizeof(u64)))); | |
542 | dma_addr = *((u64 *)((void *)cqe_rx + (7 * sizeof(u64)))); | |
543 | ||
544 | cpu_addr = nicvf_iova_to_phys(nic, dma_addr); | |
545 | if (!cpu_addr) | |
546 | return false; | |
e3d06ff9 SG |
547 | cpu_addr = (u64)phys_to_virt(cpu_addr); |
548 | page = virt_to_page((void *)cpu_addr); | |
05c773f5 | 549 | |
43b5169d LB |
550 | xdp_init_buff(&xdp, RCV_FRAG_LEN + XDP_PACKET_HEADROOM, |
551 | &rq->xdp_rxq); | |
be9df4af LB |
552 | hard_start = page_address(page); |
553 | data = (unsigned char *)cpu_addr; | |
554 | xdp_prepare_buff(&xdp, hard_start, data - hard_start, len, false); | |
e3d06ff9 | 555 | orig_data = xdp.data; |
05c773f5 | 556 | |
05c773f5 | 557 | action = bpf_prog_run_xdp(prog, &xdp); |
05c773f5 | 558 | |
a48ce00f | 559 | len = xdp.data_end - xdp.data; |
e3d06ff9 SG |
560 | /* Check if XDP program has changed headers */ |
561 | if (orig_data != xdp.data) { | |
e3d06ff9 SG |
562 | offset = orig_data - xdp.data; |
563 | dma_addr -= offset; | |
564 | } | |
565 | ||
05c773f5 SG |
566 | switch (action) { |
567 | case XDP_PASS: | |
e6dbe939 JDB |
568 | /* Check if it's a recycled page, if not |
569 | * unmap the DMA mapping. | |
570 | * | |
571 | * Recycled page holds an extra reference. | |
572 | */ | |
573 | if (page_ref_count(page) == 1) { | |
574 | dma_addr &= PAGE_MASK; | |
575 | dma_unmap_page_attrs(&nic->pdev->dev, dma_addr, | |
576 | RCV_FRAG_LEN + XDP_PACKET_HEADROOM, | |
577 | DMA_FROM_DEVICE, | |
578 | DMA_ATTR_SKIP_CPU_SYNC); | |
579 | } | |
e3d06ff9 SG |
580 | |
581 | /* Build SKB and pass on packet to network stack */ | |
582 | *skb = build_skb(xdp.data, | |
583 | RCV_FRAG_LEN - cqe_rx->align_pad + offset); | |
584 | if (!*skb) | |
585 | put_page(page); | |
586 | else | |
587 | skb_put(*skb, len); | |
05c773f5 | 588 | return false; |
16f2bccd SG |
589 | case XDP_TX: |
590 | nicvf_xdp_sq_append_pkt(nic, sq, (u64)xdp.data, dma_addr, len); | |
591 | return true; | |
05c773f5 | 592 | default: |
c8064e5b | 593 | bpf_warn_invalid_xdp_action(nic->netdev, prog, action); |
df561f66 | 594 | fallthrough; |
c56d91ce SG |
595 | case XDP_ABORTED: |
596 | trace_xdp_exception(nic->netdev, prog, action); | |
df561f66 | 597 | fallthrough; |
c56d91ce | 598 | case XDP_DROP: |
e6dbe939 JDB |
599 | /* Check if it's a recycled page, if not |
600 | * unmap the DMA mapping. | |
601 | * | |
602 | * Recycled page holds an extra reference. | |
603 | */ | |
604 | if (page_ref_count(page) == 1) { | |
605 | dma_addr &= PAGE_MASK; | |
606 | dma_unmap_page_attrs(&nic->pdev->dev, dma_addr, | |
607 | RCV_FRAG_LEN + XDP_PACKET_HEADROOM, | |
608 | DMA_FROM_DEVICE, | |
609 | DMA_ATTR_SKIP_CPU_SYNC); | |
610 | } | |
c56d91ce SG |
611 | put_page(page); |
612 | return true; | |
05c773f5 SG |
613 | } |
614 | return false; | |
615 | } | |
616 | ||
4a875509 SG |
617 | static void nicvf_snd_ptp_handler(struct net_device *netdev, |
618 | struct cqe_send_t *cqe_tx) | |
619 | { | |
620 | struct nicvf *nic = netdev_priv(netdev); | |
621 | struct skb_shared_hwtstamps ts; | |
622 | u64 ns; | |
623 | ||
624 | nic = nic->pnicvf; | |
625 | ||
626 | /* Sync for 'ptp_skb' */ | |
627 | smp_rmb(); | |
628 | ||
629 | /* New timestamp request can be queued now */ | |
630 | atomic_set(&nic->tx_ptp_skbs, 0); | |
631 | ||
632 | /* Check for timestamp requested skb */ | |
633 | if (!nic->ptp_skb) | |
634 | return; | |
635 | ||
636 | /* Check if timestamping is timedout, which is set to 10us */ | |
637 | if (cqe_tx->send_status == CQ_TX_ERROP_TSTMP_TIMEOUT || | |
638 | cqe_tx->send_status == CQ_TX_ERROP_TSTMP_CONFLICT) | |
639 | goto no_tstamp; | |
640 | ||
641 | /* Get the timestamp */ | |
642 | memset(&ts, 0, sizeof(ts)); | |
643 | ns = cavium_ptp_tstamp2time(nic->ptp_clock, cqe_tx->ptp_timestamp); | |
644 | ts.hwtstamp = ns_to_ktime(ns); | |
645 | skb_tstamp_tx(nic->ptp_skb, &ts); | |
646 | ||
647 | no_tstamp: | |
648 | /* Free the original skb */ | |
649 | dev_kfree_skb_any(nic->ptp_skb); | |
650 | nic->ptp_skb = NULL; | |
651 | /* Sync 'ptp_skb' */ | |
652 | smp_wmb(); | |
653 | } | |
654 | ||
4863dea3 | 655 | static void nicvf_snd_pkt_handler(struct net_device *netdev, |
c43548d2 | 656 | struct cqe_send_t *cqe_tx, |
0dada88b | 657 | int budget, int *subdesc_cnt, |
2c204c2b | 658 | unsigned int *tx_pkts, unsigned int *tx_bytes) |
4863dea3 SG |
659 | { |
660 | struct sk_buff *skb = NULL; | |
16f2bccd | 661 | struct page *page; |
4863dea3 SG |
662 | struct nicvf *nic = netdev_priv(netdev); |
663 | struct snd_queue *sq; | |
664 | struct sq_hdr_subdesc *hdr; | |
7ceb8a13 | 665 | struct sq_hdr_subdesc *tso_sqe; |
4863dea3 SG |
666 | |
667 | sq = &nic->qs->sq[cqe_tx->sq_idx]; | |
668 | ||
669 | hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr); | |
670 | if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) | |
671 | return; | |
672 | ||
0dada88b SG |
673 | /* Check for errors */ |
674 | if (cqe_tx->send_status) | |
675 | nicvf_check_cqe_tx_errs(nic->pnicvf, cqe_tx); | |
4863dea3 | 676 | |
16f2bccd SG |
677 | /* Is this a XDP designated Tx queue */ |
678 | if (sq->is_xdp) { | |
679 | page = (struct page *)sq->xdp_page[cqe_tx->sqe_ptr]; | |
680 | /* Check if it's recycled page or else unmap DMA mapping */ | |
681 | if (page && (page_ref_count(page) == 1)) | |
682 | nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr, | |
683 | hdr->subdesc_cnt); | |
684 | ||
685 | /* Release page reference for recycling */ | |
686 | if (page) | |
687 | put_page(page); | |
688 | sq->xdp_page[cqe_tx->sqe_ptr] = (u64)NULL; | |
689 | *subdesc_cnt += hdr->subdesc_cnt + 1; | |
690 | return; | |
691 | } | |
692 | ||
4863dea3 | 693 | skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr]; |
4863dea3 | 694 | if (skb) { |
7ceb8a13 SG |
695 | /* Check for dummy descriptor used for HW TSO offload on 88xx */ |
696 | if (hdr->dont_send) { | |
697 | /* Get actual TSO descriptors and free them */ | |
698 | tso_sqe = | |
699 | (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2); | |
83abb7d7 SG |
700 | nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2, |
701 | tso_sqe->subdesc_cnt); | |
0dada88b | 702 | *subdesc_cnt += tso_sqe->subdesc_cnt + 1; |
83abb7d7 SG |
703 | } else { |
704 | nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr, | |
705 | hdr->subdesc_cnt); | |
7ceb8a13 | 706 | } |
0dada88b | 707 | *subdesc_cnt += hdr->subdesc_cnt + 1; |
4863dea3 | 708 | prefetch(skb); |
2c204c2b SG |
709 | (*tx_pkts)++; |
710 | *tx_bytes += skb->len; | |
4a875509 SG |
711 | /* If timestamp is requested for this skb, don't free it */ |
712 | if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS && | |
713 | !nic->pnicvf->ptp_skb) | |
714 | nic->pnicvf->ptp_skb = skb; | |
715 | else | |
716 | napi_consume_skb(skb, budget); | |
143ceb0b | 717 | sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL; |
40fb5f8a | 718 | } else { |
7ceb8a13 SG |
719 | /* In case of SW TSO on 88xx, only last segment will have |
720 | * a SKB attached, so just free SQEs here. | |
40fb5f8a SG |
721 | */ |
722 | if (!nic->hw_tso) | |
0dada88b | 723 | *subdesc_cnt += hdr->subdesc_cnt + 1; |
4863dea3 SG |
724 | } |
725 | } | |
726 | ||
38bb5d4f SG |
727 | static inline void nicvf_set_rxhash(struct net_device *netdev, |
728 | struct cqe_rx_t *cqe_rx, | |
729 | struct sk_buff *skb) | |
730 | { | |
731 | u8 hash_type; | |
732 | u32 hash; | |
733 | ||
734 | if (!(netdev->features & NETIF_F_RXHASH)) | |
735 | return; | |
736 | ||
737 | switch (cqe_rx->rss_alg) { | |
738 | case RSS_ALG_TCP_IP: | |
739 | case RSS_ALG_UDP_IP: | |
740 | hash_type = PKT_HASH_TYPE_L4; | |
741 | hash = cqe_rx->rss_tag; | |
742 | break; | |
743 | case RSS_ALG_IP: | |
744 | hash_type = PKT_HASH_TYPE_L3; | |
745 | hash = cqe_rx->rss_tag; | |
746 | break; | |
747 | default: | |
748 | hash_type = PKT_HASH_TYPE_NONE; | |
749 | hash = 0; | |
750 | } | |
751 | ||
752 | skb_set_hash(skb, hash, hash_type); | |
753 | } | |
754 | ||
4a875509 SG |
755 | static inline void nicvf_set_rxtstamp(struct nicvf *nic, struct sk_buff *skb) |
756 | { | |
757 | u64 ns; | |
758 | ||
759 | if (!nic->ptp_clock || !nic->hw_rx_tstamp) | |
760 | return; | |
761 | ||
762 | /* The first 8 bytes is the timestamp */ | |
763 | ns = cavium_ptp_tstamp2time(nic->ptp_clock, | |
764 | be64_to_cpu(*(__be64 *)skb->data)); | |
765 | skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns); | |
766 | ||
767 | __skb_pull(skb, 8); | |
768 | } | |
769 | ||
4863dea3 SG |
770 | static void nicvf_rcv_pkt_handler(struct net_device *netdev, |
771 | struct napi_struct *napi, | |
27e95e36 JDB |
772 | struct cqe_rx_t *cqe_rx, |
773 | struct snd_queue *sq, struct rcv_queue *rq) | |
4863dea3 | 774 | { |
e3d06ff9 | 775 | struct sk_buff *skb = NULL; |
4863dea3 | 776 | struct nicvf *nic = netdev_priv(netdev); |
83abb7d7 | 777 | struct nicvf *snic = nic; |
4863dea3 | 778 | int err = 0; |
92dc8769 SG |
779 | int rq_idx; |
780 | ||
781 | rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx); | |
782 | ||
783 | if (nic->sqs_mode) { | |
784 | /* Use primary VF's 'nicvf' struct */ | |
785 | nic = nic->pnicvf; | |
786 | netdev = nic->netdev; | |
787 | } | |
4863dea3 SG |
788 | |
789 | /* Check for errors */ | |
0dada88b SG |
790 | if (cqe_rx->err_level || cqe_rx->err_opcode) { |
791 | err = nicvf_check_cqe_rx_errs(nic, cqe_rx); | |
792 | if (err && !cqe_rx->rb_cnt) | |
793 | return; | |
794 | } | |
4863dea3 | 795 | |
05c773f5 | 796 | /* For XDP, ignore pkts spanning multiple pages */ |
e3d06ff9 SG |
797 | if (nic->xdp_prog && (cqe_rx->rb_cnt == 1)) { |
798 | /* Packet consumed by XDP */ | |
27e95e36 | 799 | if (nicvf_xdp_rx(snic, nic->xdp_prog, cqe_rx, sq, rq, &skb)) |
05c773f5 | 800 | return; |
e3d06ff9 SG |
801 | } else { |
802 | skb = nicvf_get_rcv_skb(snic, cqe_rx, | |
803 | nic->xdp_prog ? true : false); | |
804 | } | |
05c773f5 | 805 | |
e3d06ff9 | 806 | if (!skb) |
4863dea3 | 807 | return; |
4863dea3 SG |
808 | |
809 | if (netif_msg_pktdata(nic)) { | |
bf24e136 | 810 | netdev_info(nic->netdev, "skb 0x%p, len=%d\n", skb, skb->len); |
4863dea3 SG |
811 | print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1, |
812 | skb->data, skb->len, true); | |
813 | } | |
814 | ||
a2dc5ded SG |
815 | /* If error packet, drop it here */ |
816 | if (err) { | |
817 | dev_kfree_skb_any(skb); | |
818 | return; | |
819 | } | |
820 | ||
4a875509 | 821 | nicvf_set_rxtstamp(nic, skb); |
38bb5d4f SG |
822 | nicvf_set_rxhash(netdev, cqe_rx, skb); |
823 | ||
92dc8769 | 824 | skb_record_rx_queue(skb, rq_idx); |
4863dea3 SG |
825 | if (netdev->hw_features & NETIF_F_RXCSUM) { |
826 | /* HW by default verifies TCP/UDP/SCTP checksums */ | |
827 | skb->ip_summed = CHECKSUM_UNNECESSARY; | |
828 | } else { | |
829 | skb_checksum_none_assert(skb); | |
830 | } | |
831 | ||
832 | skb->protocol = eth_type_trans(skb, netdev); | |
833 | ||
aa2e259b SG |
834 | /* Check for stripped VLAN */ |
835 | if (cqe_rx->vlan_found && cqe_rx->vlan_stripped) | |
836 | __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), | |
837 | ntohs((__force __be16)cqe_rx->vlan_tci)); | |
838 | ||
4863dea3 SG |
839 | if (napi && (netdev->features & NETIF_F_GRO)) |
840 | napi_gro_receive(napi, skb); | |
841 | else | |
842 | netif_receive_skb(skb); | |
843 | } | |
844 | ||
845 | static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx, | |
846 | struct napi_struct *napi, int budget) | |
847 | { | |
74840b83 | 848 | int processed_cqe, work_done = 0, tx_done = 0; |
4863dea3 | 849 | int cqe_count, cqe_head; |
0dada88b | 850 | int subdesc_cnt = 0; |
4863dea3 SG |
851 | struct nicvf *nic = netdev_priv(netdev); |
852 | struct queue_set *qs = nic->qs; | |
853 | struct cmp_queue *cq = &qs->cq[cq_idx]; | |
854 | struct cqe_rx_t *cq_desc; | |
74840b83 | 855 | struct netdev_queue *txq; |
16f2bccd | 856 | struct snd_queue *sq = &qs->sq[cq_idx]; |
27e95e36 | 857 | struct rcv_queue *rq = &qs->rq[cq_idx]; |
16f2bccd | 858 | unsigned int tx_pkts = 0, tx_bytes = 0, txq_idx; |
4863dea3 SG |
859 | |
860 | spin_lock_bh(&cq->lock); | |
861 | loop: | |
862 | processed_cqe = 0; | |
863 | /* Get no of valid CQ entries to process */ | |
864 | cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx); | |
865 | cqe_count &= CQ_CQE_COUNT; | |
866 | if (!cqe_count) | |
867 | goto done; | |
868 | ||
869 | /* Get head of the valid CQ entries */ | |
870 | cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9; | |
871 | cqe_head &= 0xFFFF; | |
872 | ||
4863dea3 SG |
873 | while (processed_cqe < cqe_count) { |
874 | /* Get the CQ descriptor */ | |
875 | cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head); | |
876 | cqe_head++; | |
877 | cqe_head &= (cq->dmem.q_len - 1); | |
878 | /* Initiate prefetch for next descriptor */ | |
879 | prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head)); | |
880 | ||
881 | if ((work_done >= budget) && napi && | |
882 | (cq_desc->cqe_type != CQE_TYPE_SEND)) { | |
883 | break; | |
884 | } | |
885 | ||
4863dea3 SG |
886 | switch (cq_desc->cqe_type) { |
887 | case CQE_TYPE_RX: | |
27e95e36 | 888 | nicvf_rcv_pkt_handler(netdev, napi, cq_desc, sq, rq); |
4863dea3 SG |
889 | work_done++; |
890 | break; | |
891 | case CQE_TYPE_SEND: | |
0dada88b SG |
892 | nicvf_snd_pkt_handler(netdev, (void *)cq_desc, |
893 | budget, &subdesc_cnt, | |
894 | &tx_pkts, &tx_bytes); | |
74840b83 | 895 | tx_done++; |
4863dea3 | 896 | break; |
4a875509 SG |
897 | case CQE_TYPE_SEND_PTP: |
898 | nicvf_snd_ptp_handler(netdev, (void *)cq_desc); | |
899 | break; | |
4863dea3 SG |
900 | case CQE_TYPE_INVALID: |
901 | case CQE_TYPE_RX_SPLIT: | |
902 | case CQE_TYPE_RX_TCP: | |
4863dea3 SG |
903 | /* Ignore for now */ |
904 | break; | |
905 | } | |
906 | processed_cqe++; | |
907 | } | |
4863dea3 SG |
908 | |
909 | /* Ring doorbell to inform H/W to reuse processed CQEs */ | |
910 | nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, | |
911 | cq_idx, processed_cqe); | |
912 | ||
913 | if ((work_done < budget) && napi) | |
914 | goto loop; | |
915 | ||
916 | done: | |
0dada88b SG |
917 | /* Update SQ's descriptor free count */ |
918 | if (subdesc_cnt) | |
919 | nicvf_put_sq_desc(sq, subdesc_cnt); | |
920 | ||
16f2bccd SG |
921 | txq_idx = nicvf_netdev_qidx(nic, cq_idx); |
922 | /* Handle XDP TX queues */ | |
923 | if (nic->pnicvf->xdp_prog) { | |
924 | if (txq_idx < nic->pnicvf->xdp_tx_queues) { | |
925 | nicvf_xdp_sq_doorbell(nic, sq, cq_idx); | |
926 | goto out; | |
927 | } | |
928 | nic = nic->pnicvf; | |
929 | txq_idx -= nic->pnicvf->xdp_tx_queues; | |
930 | } | |
931 | ||
0dada88b | 932 | /* Wakeup TXQ if its stopped earlier due to SQ full */ |
bd3ad7d3 SG |
933 | if (tx_done || |
934 | (atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) { | |
92dc8769 | 935 | netdev = nic->pnicvf->netdev; |
16f2bccd | 936 | txq = netdev_get_tx_queue(netdev, txq_idx); |
2c204c2b SG |
937 | if (tx_pkts) |
938 | netdev_tx_completed_queue(txq, tx_pkts, tx_bytes); | |
939 | ||
bd3ad7d3 SG |
940 | /* To read updated queue and carrier status */ |
941 | smp_mb(); | |
92dc8769 | 942 | if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) { |
bd3ad7d3 SG |
943 | netif_tx_wake_queue(txq); |
944 | nic = nic->pnicvf; | |
964cb69b | 945 | this_cpu_inc(nic->drv_stats->txq_wake); |
bf24e136 JP |
946 | netif_warn(nic, tx_err, netdev, |
947 | "Transmit queue wakeup SQ%d\n", txq_idx); | |
74840b83 SG |
948 | } |
949 | } | |
950 | ||
16f2bccd | 951 | out: |
4863dea3 SG |
952 | spin_unlock_bh(&cq->lock); |
953 | return work_done; | |
954 | } | |
955 | ||
956 | static int nicvf_poll(struct napi_struct *napi, int budget) | |
957 | { | |
958 | u64 cq_head; | |
959 | int work_done = 0; | |
960 | struct net_device *netdev = napi->dev; | |
961 | struct nicvf *nic = netdev_priv(netdev); | |
962 | struct nicvf_cq_poll *cq; | |
4863dea3 SG |
963 | |
964 | cq = container_of(napi, struct nicvf_cq_poll, napi); | |
965 | work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget); | |
966 | ||
4863dea3 SG |
967 | if (work_done < budget) { |
968 | /* Slow packet rate, exit polling */ | |
6ad20165 | 969 | napi_complete_done(napi, work_done); |
4863dea3 SG |
970 | /* Re-enable interrupts */ |
971 | cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, | |
972 | cq->cq_idx); | |
973 | nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx); | |
974 | nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD, | |
975 | cq->cq_idx, cq_head); | |
976 | nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx); | |
977 | } | |
978 | return work_done; | |
979 | } | |
980 | ||
981 | /* Qset error interrupt handler | |
982 | * | |
983 | * As of now only CQ errors are handled | |
984 | */ | |
dfe4e612 | 985 | static void nicvf_handle_qs_err(struct tasklet_struct *t) |
4863dea3 | 986 | { |
dfe4e612 | 987 | struct nicvf *nic = from_tasklet(nic, t, qs_err_task); |
4863dea3 SG |
988 | struct queue_set *qs = nic->qs; |
989 | int qidx; | |
990 | u64 status; | |
991 | ||
992 | netif_tx_disable(nic->netdev); | |
993 | ||
994 | /* Check if it is CQ err */ | |
995 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) { | |
996 | status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, | |
997 | qidx); | |
998 | if (!(status & CQ_ERR_MASK)) | |
999 | continue; | |
1000 | /* Process already queued CQEs and reconfig CQ */ | |
1001 | nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); | |
1002 | nicvf_sq_disable(nic, qidx); | |
1003 | nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0); | |
1004 | nicvf_cmp_queue_config(nic, qs, qidx, true); | |
1005 | nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx); | |
1006 | nicvf_sq_enable(nic, &qs->sq[qidx], qidx); | |
1007 | ||
1008 | nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); | |
1009 | } | |
1010 | ||
1011 | netif_tx_start_all_queues(nic->netdev); | |
1012 | /* Re-enable Qset error interrupt */ | |
1013 | nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1014 | } | |
1015 | ||
39ad6eea SG |
1016 | static void nicvf_dump_intr_status(struct nicvf *nic) |
1017 | { | |
bf24e136 JP |
1018 | netif_info(nic, intr, nic->netdev, "interrupt status 0x%llx\n", |
1019 | nicvf_reg_read(nic, NIC_VF_INT)); | |
39ad6eea SG |
1020 | } |
1021 | ||
4863dea3 SG |
1022 | static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq) |
1023 | { | |
1024 | struct nicvf *nic = (struct nicvf *)nicvf_irq; | |
1025 | u64 intr; | |
1026 | ||
39ad6eea SG |
1027 | nicvf_dump_intr_status(nic); |
1028 | ||
4863dea3 SG |
1029 | intr = nicvf_reg_read(nic, NIC_VF_INT); |
1030 | /* Check for spurious interrupt */ | |
1031 | if (!(intr & NICVF_INTR_MBOX_MASK)) | |
1032 | return IRQ_HANDLED; | |
1033 | ||
1034 | nicvf_handle_mbx_intr(nic); | |
1035 | ||
1036 | return IRQ_HANDLED; | |
1037 | } | |
1038 | ||
39ad6eea SG |
1039 | static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq) |
1040 | { | |
1041 | struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq; | |
1042 | struct nicvf *nic = cq_poll->nicvf; | |
1043 | int qidx = cq_poll->cq_idx; | |
1044 | ||
1045 | nicvf_dump_intr_status(nic); | |
1046 | ||
1047 | /* Disable interrupts */ | |
1048 | nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); | |
1049 | ||
1050 | /* Schedule NAPI */ | |
ef0a4d86 | 1051 | napi_schedule_irqoff(&cq_poll->napi); |
39ad6eea SG |
1052 | |
1053 | /* Clear interrupt */ | |
1054 | nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); | |
1055 | ||
1056 | return IRQ_HANDLED; | |
1057 | } | |
1058 | ||
1059 | static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq) | |
4863dea3 | 1060 | { |
4863dea3 | 1061 | struct nicvf *nic = (struct nicvf *)nicvf_irq; |
39ad6eea | 1062 | u8 qidx; |
4863dea3 | 1063 | |
4863dea3 | 1064 | |
39ad6eea SG |
1065 | nicvf_dump_intr_status(nic); |
1066 | ||
1067 | /* Disable RBDR interrupt and schedule softirq */ | |
1068 | for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) { | |
1069 | if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx)) | |
4863dea3 | 1070 | continue; |
39ad6eea SG |
1071 | nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); |
1072 | tasklet_hi_schedule(&nic->rbdr_task); | |
1073 | /* Clear interrupt */ | |
1074 | nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); | |
1075 | } | |
4863dea3 | 1076 | |
39ad6eea SG |
1077 | return IRQ_HANDLED; |
1078 | } | |
4863dea3 | 1079 | |
39ad6eea SG |
1080 | static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq) |
1081 | { | |
1082 | struct nicvf *nic = (struct nicvf *)nicvf_irq; | |
4863dea3 | 1083 | |
39ad6eea SG |
1084 | nicvf_dump_intr_status(nic); |
1085 | ||
1086 | /* Disable Qset err interrupt and schedule softirq */ | |
1087 | nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1088 | tasklet_hi_schedule(&nic->qs_err_task); | |
1089 | nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); | |
4863dea3 | 1090 | |
4863dea3 SG |
1091 | return IRQ_HANDLED; |
1092 | } | |
1093 | ||
fb4b7d98 SG |
1094 | static void nicvf_set_irq_affinity(struct nicvf *nic) |
1095 | { | |
1096 | int vec, cpu; | |
fb4b7d98 SG |
1097 | |
1098 | for (vec = 0; vec < nic->num_vec; vec++) { | |
1099 | if (!nic->irq_allocated[vec]) | |
1100 | continue; | |
1101 | ||
1102 | if (!zalloc_cpumask_var(&nic->affinity_mask[vec], GFP_KERNEL)) | |
1103 | return; | |
1104 | /* CQ interrupts */ | |
1105 | if (vec < NICVF_INTR_ID_SQ) | |
1106 | /* Leave CPU0 for RBDR and other interrupts */ | |
1107 | cpu = nicvf_netdev_qidx(nic, vec) + 1; | |
1108 | else | |
1109 | cpu = 0; | |
1110 | ||
1111 | cpumask_set_cpu(cpumask_local_spread(cpu, nic->node), | |
1112 | nic->affinity_mask[vec]); | |
ce211b17 TS |
1113 | irq_set_affinity_hint(pci_irq_vector(nic->pdev, vec), |
1114 | nic->affinity_mask[vec]); | |
fb4b7d98 SG |
1115 | } |
1116 | } | |
1117 | ||
4863dea3 SG |
1118 | static int nicvf_register_interrupts(struct nicvf *nic) |
1119 | { | |
39ad6eea | 1120 | int irq, ret = 0; |
4863dea3 SG |
1121 | |
1122 | for_each_cq_irq(irq) | |
e4126213 SG |
1123 | sprintf(nic->irq_name[irq], "%s-rxtx-%d", |
1124 | nic->pnicvf->netdev->name, | |
1125 | nicvf_netdev_qidx(nic, irq)); | |
4863dea3 SG |
1126 | |
1127 | for_each_sq_irq(irq) | |
e4126213 SG |
1128 | sprintf(nic->irq_name[irq], "%s-sq-%d", |
1129 | nic->pnicvf->netdev->name, | |
1130 | nicvf_netdev_qidx(nic, irq - NICVF_INTR_ID_SQ)); | |
4863dea3 SG |
1131 | |
1132 | for_each_rbdr_irq(irq) | |
e4126213 SG |
1133 | sprintf(nic->irq_name[irq], "%s-rbdr-%d", |
1134 | nic->pnicvf->netdev->name, | |
1135 | nic->sqs_mode ? (nic->sqs_id + 1) : 0); | |
4863dea3 | 1136 | |
39ad6eea SG |
1137 | /* Register CQ interrupts */ |
1138 | for (irq = 0; irq < nic->qs->cq_cnt; irq++) { | |
ce211b17 TS |
1139 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
1140 | nicvf_intr_handler, | |
39ad6eea | 1141 | 0, nic->irq_name[irq], nic->napi[irq]); |
4863dea3 | 1142 | if (ret) |
39ad6eea | 1143 | goto err; |
4863dea3 SG |
1144 | nic->irq_allocated[irq] = true; |
1145 | } | |
1146 | ||
39ad6eea SG |
1147 | /* Register RBDR interrupt */ |
1148 | for (irq = NICVF_INTR_ID_RBDR; | |
1149 | irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) { | |
ce211b17 TS |
1150 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
1151 | nicvf_rbdr_intr_handler, | |
4863dea3 SG |
1152 | 0, nic->irq_name[irq], nic); |
1153 | if (ret) | |
39ad6eea | 1154 | goto err; |
4863dea3 SG |
1155 | nic->irq_allocated[irq] = true; |
1156 | } | |
1157 | ||
39ad6eea | 1158 | /* Register QS error interrupt */ |
e4126213 SG |
1159 | sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR], "%s-qset-err-%d", |
1160 | nic->pnicvf->netdev->name, | |
1161 | nic->sqs_mode ? (nic->sqs_id + 1) : 0); | |
39ad6eea | 1162 | irq = NICVF_INTR_ID_QS_ERR; |
ce211b17 | 1163 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
39ad6eea SG |
1164 | nicvf_qs_err_intr_handler, |
1165 | 0, nic->irq_name[irq], nic); | |
fb4b7d98 SG |
1166 | if (ret) |
1167 | goto err; | |
1168 | ||
1169 | nic->irq_allocated[irq] = true; | |
1170 | ||
1171 | /* Set IRQ affinities */ | |
1172 | nicvf_set_irq_affinity(nic); | |
4863dea3 | 1173 | |
39ad6eea SG |
1174 | err: |
1175 | if (ret) | |
1176 | netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq); | |
4863dea3 | 1177 | |
39ad6eea | 1178 | return ret; |
4863dea3 SG |
1179 | } |
1180 | ||
1181 | static void nicvf_unregister_interrupts(struct nicvf *nic) | |
1182 | { | |
ce211b17 | 1183 | struct pci_dev *pdev = nic->pdev; |
4863dea3 SG |
1184 | int irq; |
1185 | ||
1186 | /* Free registered interrupts */ | |
1187 | for (irq = 0; irq < nic->num_vec; irq++) { | |
39ad6eea SG |
1188 | if (!nic->irq_allocated[irq]) |
1189 | continue; | |
1190 | ||
ce211b17 | 1191 | irq_set_affinity_hint(pci_irq_vector(pdev, irq), NULL); |
fb4b7d98 SG |
1192 | free_cpumask_var(nic->affinity_mask[irq]); |
1193 | ||
39ad6eea | 1194 | if (irq < NICVF_INTR_ID_SQ) |
ce211b17 | 1195 | free_irq(pci_irq_vector(pdev, irq), nic->napi[irq]); |
39ad6eea | 1196 | else |
ce211b17 | 1197 | free_irq(pci_irq_vector(pdev, irq), nic); |
39ad6eea | 1198 | |
4863dea3 SG |
1199 | nic->irq_allocated[irq] = false; |
1200 | } | |
1201 | ||
1202 | /* Disable MSI-X */ | |
ce211b17 TS |
1203 | pci_free_irq_vectors(pdev); |
1204 | nic->num_vec = 0; | |
4863dea3 SG |
1205 | } |
1206 | ||
1207 | /* Initialize MSIX vectors and register MISC interrupt. | |
1208 | * Send READY message to PF to check if its alive | |
1209 | */ | |
1210 | static int nicvf_register_misc_interrupt(struct nicvf *nic) | |
1211 | { | |
1212 | int ret = 0; | |
1213 | int irq = NICVF_INTR_ID_MISC; | |
1214 | ||
1215 | /* Return if mailbox interrupt is already registered */ | |
ce211b17 | 1216 | if (nic->pdev->msix_enabled) |
4863dea3 SG |
1217 | return 0; |
1218 | ||
1219 | /* Enable MSI-X */ | |
ce211b17 TS |
1220 | nic->num_vec = pci_msix_vec_count(nic->pdev); |
1221 | ret = pci_alloc_irq_vectors(nic->pdev, nic->num_vec, nic->num_vec, | |
1222 | PCI_IRQ_MSIX); | |
1223 | if (ret < 0) { | |
1224 | netdev_err(nic->netdev, | |
1225 | "Req for #%d msix vectors failed\n", nic->num_vec); | |
c69b2f46 | 1226 | return ret; |
ce211b17 | 1227 | } |
4863dea3 SG |
1228 | |
1229 | sprintf(nic->irq_name[irq], "%s Mbox", "NICVF"); | |
1230 | /* Register Misc interrupt */ | |
ce211b17 | 1231 | ret = request_irq(pci_irq_vector(nic->pdev, irq), |
4863dea3 SG |
1232 | nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic); |
1233 | ||
1234 | if (ret) | |
1235 | return ret; | |
1236 | nic->irq_allocated[irq] = true; | |
1237 | ||
1238 | /* Enable mailbox interrupt */ | |
1239 | nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0); | |
1240 | ||
1241 | /* Check if VF is able to communicate with PF */ | |
1242 | if (!nicvf_check_pf_ready(nic)) { | |
1243 | nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); | |
1244 | nicvf_unregister_interrupts(nic); | |
c69b2f46 | 1245 | return -EIO; |
4863dea3 SG |
1246 | } |
1247 | ||
1248 | return 0; | |
1249 | } | |
1250 | ||
1251 | static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev) | |
1252 | { | |
1253 | struct nicvf *nic = netdev_priv(netdev); | |
1254 | int qid = skb_get_queue_mapping(skb); | |
1255 | struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid); | |
bd3ad7d3 SG |
1256 | struct nicvf *snic; |
1257 | struct snd_queue *sq; | |
1258 | int tmp; | |
4863dea3 SG |
1259 | |
1260 | /* Check for minimum packet length */ | |
1261 | if (skb->len <= ETH_HLEN) { | |
1262 | dev_kfree_skb(skb); | |
1263 | return NETDEV_TX_OK; | |
1264 | } | |
1265 | ||
16f2bccd SG |
1266 | /* In XDP case, initial HW tx queues are used for XDP, |
1267 | * but stack's queue mapping starts at '0', so skip the | |
1268 | * Tx queues attached to Rx queues for XDP. | |
1269 | */ | |
1270 | if (nic->xdp_prog) | |
1271 | qid += nic->xdp_tx_queues; | |
1272 | ||
bd3ad7d3 SG |
1273 | snic = nic; |
1274 | /* Get secondary Qset's SQ structure */ | |
1275 | if (qid >= MAX_SND_QUEUES_PER_QS) { | |
1276 | tmp = qid / MAX_SND_QUEUES_PER_QS; | |
1277 | snic = (struct nicvf *)nic->snicvf[tmp - 1]; | |
1278 | if (!snic) { | |
1279 | netdev_warn(nic->netdev, | |
1280 | "Secondary Qset#%d's ptr not initialized\n", | |
1281 | tmp - 1); | |
1282 | dev_kfree_skb(skb); | |
1283 | return NETDEV_TX_OK; | |
1284 | } | |
1285 | qid = qid % MAX_SND_QUEUES_PER_QS; | |
1286 | } | |
1287 | ||
1288 | sq = &snic->qs->sq[qid]; | |
1289 | if (!netif_tx_queue_stopped(txq) && | |
1290 | !nicvf_sq_append_skb(snic, sq, skb, qid)) { | |
4863dea3 | 1291 | netif_tx_stop_queue(txq); |
bd3ad7d3 SG |
1292 | |
1293 | /* Barrier, so that stop_queue visible to other cpus */ | |
1294 | smp_mb(); | |
1295 | ||
1296 | /* Check again, incase another cpu freed descriptors */ | |
1297 | if (atomic_read(&sq->free_cnt) > MIN_SQ_DESC_PER_PKT_XMIT) { | |
1298 | netif_tx_wake_queue(txq); | |
1299 | } else { | |
1300 | this_cpu_inc(nic->drv_stats->txq_stop); | |
bf24e136 JP |
1301 | netif_warn(nic, tx_err, netdev, |
1302 | "Transmit ring full, stopping SQ%d\n", qid); | |
bd3ad7d3 | 1303 | } |
4863dea3 SG |
1304 | return NETDEV_TX_BUSY; |
1305 | } | |
1306 | ||
1307 | return NETDEV_TX_OK; | |
1308 | } | |
1309 | ||
39ad6eea SG |
1310 | static inline void nicvf_free_cq_poll(struct nicvf *nic) |
1311 | { | |
1312 | struct nicvf_cq_poll *cq_poll; | |
1313 | int qidx; | |
1314 | ||
1315 | for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) { | |
1316 | cq_poll = nic->napi[qidx]; | |
1317 | if (!cq_poll) | |
1318 | continue; | |
1319 | nic->napi[qidx] = NULL; | |
1320 | kfree(cq_poll); | |
1321 | } | |
1322 | } | |
1323 | ||
4863dea3 SG |
1324 | int nicvf_stop(struct net_device *netdev) |
1325 | { | |
1326 | int irq, qidx; | |
1327 | struct nicvf *nic = netdev_priv(netdev); | |
1328 | struct queue_set *qs = nic->qs; | |
1329 | struct nicvf_cq_poll *cq_poll = NULL; | |
1330 | union nic_mbx mbx = {}; | |
1331 | ||
2ecbe4f4 | 1332 | /* wait till all queued set_rx_mode tasks completes */ |
2ec1ed2a LB |
1333 | if (nic->nicvf_rx_mode_wq) { |
1334 | cancel_delayed_work_sync(&nic->link_change_work); | |
1335 | drain_workqueue(nic->nicvf_rx_mode_wq); | |
1336 | } | |
2ecbe4f4 | 1337 | |
4863dea3 SG |
1338 | mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN; |
1339 | nicvf_send_msg_to_pf(nic, &mbx); | |
1340 | ||
1341 | netif_carrier_off(netdev); | |
92dc8769 | 1342 | netif_tx_stop_all_queues(nic->netdev); |
0b72a9a1 | 1343 | nic->link_up = false; |
92dc8769 SG |
1344 | |
1345 | /* Teardown secondary qsets first */ | |
1346 | if (!nic->sqs_mode) { | |
1347 | for (qidx = 0; qidx < nic->sqs_count; qidx++) { | |
1348 | if (!nic->snicvf[qidx]) | |
1349 | continue; | |
1350 | nicvf_stop(nic->snicvf[qidx]->netdev); | |
1351 | nic->snicvf[qidx] = NULL; | |
1352 | } | |
1353 | } | |
4863dea3 SG |
1354 | |
1355 | /* Disable RBDR & QS error interrupts */ | |
1356 | for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { | |
1357 | nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx); | |
1358 | nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx); | |
1359 | } | |
1360 | nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1361 | nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1362 | ||
1363 | /* Wait for pending IRQ handlers to finish */ | |
1364 | for (irq = 0; irq < nic->num_vec; irq++) | |
ce211b17 | 1365 | synchronize_irq(pci_irq_vector(nic->pdev, irq)); |
4863dea3 SG |
1366 | |
1367 | tasklet_kill(&nic->rbdr_task); | |
1368 | tasklet_kill(&nic->qs_err_task); | |
1369 | if (nic->rb_work_scheduled) | |
1370 | cancel_delayed_work_sync(&nic->rbdr_work); | |
1371 | ||
1372 | for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) { | |
1373 | cq_poll = nic->napi[qidx]; | |
1374 | if (!cq_poll) | |
1375 | continue; | |
4863dea3 SG |
1376 | napi_synchronize(&cq_poll->napi); |
1377 | /* CQ intr is enabled while napi_complete, | |
1378 | * so disable it now | |
1379 | */ | |
1380 | nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); | |
1381 | nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx); | |
1382 | napi_disable(&cq_poll->napi); | |
1383 | netif_napi_del(&cq_poll->napi); | |
4863dea3 SG |
1384 | } |
1385 | ||
b49087dd SG |
1386 | netif_tx_disable(netdev); |
1387 | ||
2c204c2b SG |
1388 | for (qidx = 0; qidx < netdev->num_tx_queues; qidx++) |
1389 | netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx)); | |
1390 | ||
4863dea3 SG |
1391 | /* Free resources */ |
1392 | nicvf_config_data_transfer(nic, false); | |
1393 | ||
1394 | /* Disable HW Qset */ | |
1395 | nicvf_qset_config(nic, false); | |
1396 | ||
1397 | /* disable mailbox interrupt */ | |
1398 | nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); | |
1399 | ||
1400 | nicvf_unregister_interrupts(nic); | |
1401 | ||
39ad6eea SG |
1402 | nicvf_free_cq_poll(nic); |
1403 | ||
4a875509 SG |
1404 | /* Free any pending SKB saved to receive timestamp */ |
1405 | if (nic->ptp_skb) { | |
1406 | dev_kfree_skb_any(nic->ptp_skb); | |
1407 | nic->ptp_skb = NULL; | |
1408 | } | |
1409 | ||
92dc8769 SG |
1410 | /* Clear multiqset info */ |
1411 | nic->pnicvf = nic; | |
92dc8769 | 1412 | |
4863dea3 SG |
1413 | return 0; |
1414 | } | |
1415 | ||
4a875509 SG |
1416 | static int nicvf_config_hw_rx_tstamp(struct nicvf *nic, bool enable) |
1417 | { | |
1418 | union nic_mbx mbx = {}; | |
1419 | ||
1420 | mbx.ptp.msg = NIC_MBOX_MSG_PTP_CFG; | |
1421 | mbx.ptp.enable = enable; | |
1422 | ||
1423 | return nicvf_send_msg_to_pf(nic, &mbx); | |
1424 | } | |
1425 | ||
712c3185 SG |
1426 | static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu) |
1427 | { | |
1428 | union nic_mbx mbx = {}; | |
1429 | ||
1430 | mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS; | |
1431 | mbx.frs.max_frs = mtu; | |
1432 | mbx.frs.vf_id = nic->vf_id; | |
1433 | ||
1434 | return nicvf_send_msg_to_pf(nic, &mbx); | |
1435 | } | |
1436 | ||
2c632ad8 VL |
1437 | static void nicvf_link_status_check_task(struct work_struct *work_arg) |
1438 | { | |
1439 | struct nicvf *nic = container_of(work_arg, | |
1440 | struct nicvf, | |
1441 | link_change_work.work); | |
1442 | union nic_mbx mbx = {}; | |
1443 | mbx.msg.msg = NIC_MBOX_MSG_BGX_LINK_CHANGE; | |
1444 | nicvf_send_msg_to_pf(nic, &mbx); | |
1445 | queue_delayed_work(nic->nicvf_rx_mode_wq, | |
1446 | &nic->link_change_work, 2 * HZ); | |
1447 | } | |
1448 | ||
4863dea3 SG |
1449 | int nicvf_open(struct net_device *netdev) |
1450 | { | |
964cb69b | 1451 | int cpu, err, qidx; |
4863dea3 SG |
1452 | struct nicvf *nic = netdev_priv(netdev); |
1453 | struct queue_set *qs = nic->qs; | |
1454 | struct nicvf_cq_poll *cq_poll = NULL; | |
4863dea3 | 1455 | |
2ecbe4f4 | 1456 | /* wait till all queued set_rx_mode tasks completes if any */ |
2ec1ed2a LB |
1457 | if (nic->nicvf_rx_mode_wq) |
1458 | drain_workqueue(nic->nicvf_rx_mode_wq); | |
2ecbe4f4 | 1459 | |
4863dea3 SG |
1460 | netif_carrier_off(netdev); |
1461 | ||
1462 | err = nicvf_register_misc_interrupt(nic); | |
1463 | if (err) | |
1464 | return err; | |
1465 | ||
1466 | /* Register NAPI handler for processing CQEs */ | |
1467 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) { | |
1468 | cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL); | |
1469 | if (!cq_poll) { | |
1470 | err = -ENOMEM; | |
1471 | goto napi_del; | |
1472 | } | |
1473 | cq_poll->cq_idx = qidx; | |
39ad6eea | 1474 | cq_poll->nicvf = nic; |
b48b89f9 | 1475 | netif_napi_add(netdev, &cq_poll->napi, nicvf_poll); |
4863dea3 SG |
1476 | napi_enable(&cq_poll->napi); |
1477 | nic->napi[qidx] = cq_poll; | |
1478 | } | |
1479 | ||
1480 | /* Check if we got MAC address from PF or else generate a radom MAC */ | |
a3a8ce4c | 1481 | if (!nic->sqs_mode && is_zero_ether_addr(netdev->dev_addr)) { |
4863dea3 SG |
1482 | eth_hw_addr_random(netdev); |
1483 | nicvf_hw_set_mac_addr(nic, netdev); | |
1484 | } | |
1485 | ||
bd049a90 PF |
1486 | if (nic->set_mac_pending) { |
1487 | nic->set_mac_pending = false; | |
1488 | nicvf_hw_set_mac_addr(nic, netdev); | |
1489 | } | |
1490 | ||
4863dea3 | 1491 | /* Init tasklet for handling Qset err interrupt */ |
dfe4e612 | 1492 | tasklet_setup(&nic->qs_err_task, nicvf_handle_qs_err); |
4863dea3 SG |
1493 | |
1494 | /* Init RBDR tasklet which will refill RBDR */ | |
dfe4e612 | 1495 | tasklet_setup(&nic->rbdr_task, nicvf_rbdr_task); |
4863dea3 SG |
1496 | INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work); |
1497 | ||
1498 | /* Configure CPI alorithm */ | |
1499 | nic->cpi_alg = cpi_alg; | |
92dc8769 SG |
1500 | if (!nic->sqs_mode) |
1501 | nicvf_config_cpi(nic); | |
1502 | ||
1503 | nicvf_request_sqs(nic); | |
1504 | if (nic->sqs_mode) | |
1505 | nicvf_get_primary_vf_struct(nic); | |
4863dea3 | 1506 | |
4a875509 SG |
1507 | /* Configure PTP timestamp */ |
1508 | if (nic->ptp_clock) | |
1509 | nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp); | |
1510 | atomic_set(&nic->tx_ptp_skbs, 0); | |
1511 | nic->ptp_skb = NULL; | |
1512 | ||
712c3185 SG |
1513 | /* Configure receive side scaling and MTU */ |
1514 | if (!nic->sqs_mode) { | |
92dc8769 | 1515 | nicvf_rss_init(nic); |
60dce04b WY |
1516 | err = nicvf_update_hw_max_frs(nic, netdev->mtu); |
1517 | if (err) | |
712c3185 | 1518 | goto cleanup; |
964cb69b SG |
1519 | |
1520 | /* Clear percpu stats */ | |
1521 | for_each_possible_cpu(cpu) | |
1522 | memset(per_cpu_ptr(nic->drv_stats, cpu), 0, | |
1523 | sizeof(struct nicvf_drv_stats)); | |
712c3185 | 1524 | } |
4863dea3 SG |
1525 | |
1526 | err = nicvf_register_interrupts(nic); | |
1527 | if (err) | |
1528 | goto cleanup; | |
1529 | ||
1530 | /* Initialize the queues */ | |
1531 | err = nicvf_init_resources(nic); | |
1532 | if (err) | |
1533 | goto cleanup; | |
1534 | ||
1535 | /* Make sure queue initialization is written */ | |
1536 | wmb(); | |
1537 | ||
1538 | nicvf_reg_write(nic, NIC_VF_INT, -1); | |
1539 | /* Enable Qset err interrupt */ | |
1540 | nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); | |
1541 | ||
1542 | /* Enable completion queue interrupt */ | |
1543 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) | |
1544 | nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); | |
1545 | ||
1546 | /* Enable RBDR threshold interrupt */ | |
1547 | for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) | |
1548 | nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx); | |
1549 | ||
c94acf80 | 1550 | /* Send VF config done msg to PF */ |
0dd563b9 | 1551 | nicvf_send_cfg_done(nic); |
74840b83 | 1552 | |
2ec1ed2a LB |
1553 | if (nic->nicvf_rx_mode_wq) { |
1554 | INIT_DELAYED_WORK(&nic->link_change_work, | |
1555 | nicvf_link_status_check_task); | |
1556 | queue_delayed_work(nic->nicvf_rx_mode_wq, | |
1557 | &nic->link_change_work, 0); | |
1558 | } | |
2c632ad8 | 1559 | |
4863dea3 SG |
1560 | return 0; |
1561 | cleanup: | |
1562 | nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0); | |
1563 | nicvf_unregister_interrupts(nic); | |
39ad6eea SG |
1564 | tasklet_kill(&nic->qs_err_task); |
1565 | tasklet_kill(&nic->rbdr_task); | |
4863dea3 SG |
1566 | napi_del: |
1567 | for (qidx = 0; qidx < qs->cq_cnt; qidx++) { | |
1568 | cq_poll = nic->napi[qidx]; | |
1569 | if (!cq_poll) | |
1570 | continue; | |
1571 | napi_disable(&cq_poll->napi); | |
1572 | netif_napi_del(&cq_poll->napi); | |
4863dea3 | 1573 | } |
39ad6eea | 1574 | nicvf_free_cq_poll(nic); |
4863dea3 SG |
1575 | return err; |
1576 | } | |
1577 | ||
4863dea3 SG |
1578 | static int nicvf_change_mtu(struct net_device *netdev, int new_mtu) |
1579 | { | |
1580 | struct nicvf *nic = netdev_priv(netdev); | |
f9aa9dc7 | 1581 | int orig_mtu = netdev->mtu; |
4863dea3 | 1582 | |
1f227d16 MC |
1583 | /* For now just support only the usual MTU sized frames, |
1584 | * plus some headroom for VLAN, QinQ. | |
1585 | */ | |
1586 | if (nic->xdp_prog && new_mtu > MAX_XDP_MTU) { | |
1587 | netdev_warn(netdev, "Jumbo frames not yet supported with XDP, current MTU %d.\n", | |
1588 | netdev->mtu); | |
1589 | return -EINVAL; | |
1590 | } | |
1591 | ||
1eb2cded | 1592 | WRITE_ONCE(netdev->mtu, new_mtu); |
712c3185 SG |
1593 | |
1594 | if (!netif_running(netdev)) | |
1595 | return 0; | |
1596 | ||
f9aa9dc7 DM |
1597 | if (nicvf_update_hw_max_frs(nic, new_mtu)) { |
1598 | netdev->mtu = orig_mtu; | |
4863dea3 | 1599 | return -EINVAL; |
f9aa9dc7 | 1600 | } |
4863dea3 SG |
1601 | |
1602 | return 0; | |
1603 | } | |
1604 | ||
1605 | static int nicvf_set_mac_address(struct net_device *netdev, void *p) | |
1606 | { | |
1607 | struct sockaddr *addr = p; | |
1608 | struct nicvf *nic = netdev_priv(netdev); | |
1609 | ||
1610 | if (!is_valid_ether_addr(addr->sa_data)) | |
1611 | return -EADDRNOTAVAIL; | |
1612 | ||
a05e4c0a | 1613 | eth_hw_addr_set(netdev, addr->sa_data); |
4863dea3 | 1614 | |
ce211b17 | 1615 | if (nic->pdev->msix_enabled) { |
4863dea3 SG |
1616 | if (nicvf_hw_set_mac_addr(nic, netdev)) |
1617 | return -EBUSY; | |
bd049a90 PF |
1618 | } else { |
1619 | nic->set_mac_pending = true; | |
1620 | } | |
4863dea3 SG |
1621 | |
1622 | return 0; | |
1623 | } | |
1624 | ||
4863dea3 SG |
1625 | void nicvf_update_lmac_stats(struct nicvf *nic) |
1626 | { | |
1627 | int stat = 0; | |
1628 | union nic_mbx mbx = {}; | |
4863dea3 SG |
1629 | |
1630 | if (!netif_running(nic->netdev)) | |
1631 | return; | |
1632 | ||
1633 | mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS; | |
1634 | mbx.bgx_stats.vf_id = nic->vf_id; | |
1635 | /* Rx stats */ | |
1636 | mbx.bgx_stats.rx = 1; | |
1637 | while (stat < BGX_RX_STATS_COUNT) { | |
4863dea3 | 1638 | mbx.bgx_stats.idx = stat; |
6051cba7 SG |
1639 | if (nicvf_send_msg_to_pf(nic, &mbx)) |
1640 | return; | |
4863dea3 SG |
1641 | stat++; |
1642 | } | |
1643 | ||
1644 | stat = 0; | |
1645 | ||
1646 | /* Tx stats */ | |
1647 | mbx.bgx_stats.rx = 0; | |
1648 | while (stat < BGX_TX_STATS_COUNT) { | |
4863dea3 | 1649 | mbx.bgx_stats.idx = stat; |
6051cba7 SG |
1650 | if (nicvf_send_msg_to_pf(nic, &mbx)) |
1651 | return; | |
4863dea3 SG |
1652 | stat++; |
1653 | } | |
1654 | } | |
1655 | ||
1656 | void nicvf_update_stats(struct nicvf *nic) | |
1657 | { | |
964cb69b SG |
1658 | int qidx, cpu; |
1659 | u64 tmp_stats = 0; | |
a2dc5ded | 1660 | struct nicvf_hw_stats *stats = &nic->hw_stats; |
964cb69b | 1661 | struct nicvf_drv_stats *drv_stats; |
4863dea3 SG |
1662 | struct queue_set *qs = nic->qs; |
1663 | ||
1664 | #define GET_RX_STATS(reg) \ | |
1665 | nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3)) | |
1666 | #define GET_TX_STATS(reg) \ | |
1667 | nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3)) | |
1668 | ||
a2dc5ded SG |
1669 | stats->rx_bytes = GET_RX_STATS(RX_OCTS); |
1670 | stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST); | |
1671 | stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST); | |
1672 | stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST); | |
4863dea3 SG |
1673 | stats->rx_fcs_errors = GET_RX_STATS(RX_FCS); |
1674 | stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR); | |
1675 | stats->rx_drop_red = GET_RX_STATS(RX_RED); | |
a2dc5ded | 1676 | stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS); |
4863dea3 | 1677 | stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN); |
a2dc5ded | 1678 | stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS); |
4863dea3 SG |
1679 | stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST); |
1680 | stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST); | |
1681 | stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST); | |
1682 | stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST); | |
1683 | ||
964cb69b SG |
1684 | stats->tx_bytes = GET_TX_STATS(TX_OCTS); |
1685 | stats->tx_ucast_frames = GET_TX_STATS(TX_UCAST); | |
1686 | stats->tx_bcast_frames = GET_TX_STATS(TX_BCAST); | |
1687 | stats->tx_mcast_frames = GET_TX_STATS(TX_MCAST); | |
4863dea3 SG |
1688 | stats->tx_drops = GET_TX_STATS(TX_DROP); |
1689 | ||
964cb69b SG |
1690 | /* On T88 pass 2.0, the dummy SQE added for TSO notification |
1691 | * via CQE has 'dont_send' set. Hence HW drops the pkt pointed | |
1692 | * pointed by dummy SQE and results in tx_drops counter being | |
1693 | * incremented. Subtracting it from tx_tso counter will give | |
1694 | * exact tx_drops counter. | |
1695 | */ | |
1696 | if (nic->t88 && nic->hw_tso) { | |
1697 | for_each_possible_cpu(cpu) { | |
1698 | drv_stats = per_cpu_ptr(nic->drv_stats, cpu); | |
1699 | tmp_stats += drv_stats->tx_tso; | |
1700 | } | |
1701 | stats->tx_drops = tmp_stats - stats->tx_drops; | |
1702 | } | |
1703 | stats->tx_frames = stats->tx_ucast_frames + | |
1704 | stats->tx_bcast_frames + | |
1705 | stats->tx_mcast_frames; | |
1706 | stats->rx_frames = stats->rx_ucast_frames + | |
1707 | stats->rx_bcast_frames + | |
1708 | stats->rx_mcast_frames; | |
1709 | stats->rx_drops = stats->rx_drop_red + | |
1710 | stats->rx_drop_overrun; | |
4863dea3 SG |
1711 | |
1712 | /* Update RQ and SQ stats */ | |
1713 | for (qidx = 0; qidx < qs->rq_cnt; qidx++) | |
1714 | nicvf_update_rq_stats(nic, qidx); | |
1715 | for (qidx = 0; qidx < qs->sq_cnt; qidx++) | |
1716 | nicvf_update_sq_stats(nic, qidx); | |
1717 | } | |
1718 | ||
bc1f4470 | 1719 | static void nicvf_get_stats64(struct net_device *netdev, |
1720 | struct rtnl_link_stats64 *stats) | |
4863dea3 SG |
1721 | { |
1722 | struct nicvf *nic = netdev_priv(netdev); | |
a2dc5ded | 1723 | struct nicvf_hw_stats *hw_stats = &nic->hw_stats; |
4863dea3 SG |
1724 | |
1725 | nicvf_update_stats(nic); | |
1726 | ||
a2dc5ded | 1727 | stats->rx_bytes = hw_stats->rx_bytes; |
964cb69b SG |
1728 | stats->rx_packets = hw_stats->rx_frames; |
1729 | stats->rx_dropped = hw_stats->rx_drops; | |
a2dc5ded | 1730 | stats->multicast = hw_stats->rx_mcast_frames; |
4863dea3 | 1731 | |
964cb69b SG |
1732 | stats->tx_bytes = hw_stats->tx_bytes; |
1733 | stats->tx_packets = hw_stats->tx_frames; | |
1734 | stats->tx_dropped = hw_stats->tx_drops; | |
4863dea3 | 1735 | |
4863dea3 SG |
1736 | } |
1737 | ||
0290bd29 | 1738 | static void nicvf_tx_timeout(struct net_device *dev, unsigned int txqueue) |
4863dea3 SG |
1739 | { |
1740 | struct nicvf *nic = netdev_priv(dev); | |
1741 | ||
bf24e136 | 1742 | netif_warn(nic, tx_err, dev, "Transmit timed out, resetting\n"); |
4863dea3 | 1743 | |
964cb69b | 1744 | this_cpu_inc(nic->drv_stats->tx_timeout); |
4863dea3 SG |
1745 | schedule_work(&nic->reset_task); |
1746 | } | |
1747 | ||
1748 | static void nicvf_reset_task(struct work_struct *work) | |
1749 | { | |
1750 | struct nicvf *nic; | |
1751 | ||
1752 | nic = container_of(work, struct nicvf, reset_task); | |
1753 | ||
1754 | if (!netif_running(nic->netdev)) | |
1755 | return; | |
1756 | ||
1757 | nicvf_stop(nic->netdev); | |
1758 | nicvf_open(nic->netdev); | |
860e9538 | 1759 | netif_trans_update(nic->netdev); |
4863dea3 SG |
1760 | } |
1761 | ||
d77a2384 SG |
1762 | static int nicvf_config_loopback(struct nicvf *nic, |
1763 | netdev_features_t features) | |
1764 | { | |
1765 | union nic_mbx mbx = {}; | |
1766 | ||
1767 | mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK; | |
1768 | mbx.lbk.vf_id = nic->vf_id; | |
1769 | mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0; | |
1770 | ||
1771 | return nicvf_send_msg_to_pf(nic, &mbx); | |
1772 | } | |
1773 | ||
1774 | static netdev_features_t nicvf_fix_features(struct net_device *netdev, | |
1775 | netdev_features_t features) | |
1776 | { | |
1777 | struct nicvf *nic = netdev_priv(netdev); | |
1778 | ||
1779 | if ((features & NETIF_F_LOOPBACK) && | |
1780 | netif_running(netdev) && !nic->loopback_supported) | |
1781 | features &= ~NETIF_F_LOOPBACK; | |
1782 | ||
1783 | return features; | |
1784 | } | |
1785 | ||
aa2e259b SG |
1786 | static int nicvf_set_features(struct net_device *netdev, |
1787 | netdev_features_t features) | |
1788 | { | |
1789 | struct nicvf *nic = netdev_priv(netdev); | |
1790 | netdev_features_t changed = features ^ netdev->features; | |
1791 | ||
1792 | if (changed & NETIF_F_HW_VLAN_CTAG_RX) | |
1793 | nicvf_config_vlan_stripping(nic, features); | |
1794 | ||
d77a2384 SG |
1795 | if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev)) |
1796 | return nicvf_config_loopback(nic, features); | |
1797 | ||
aa2e259b SG |
1798 | return 0; |
1799 | } | |
1800 | ||
05c773f5 SG |
1801 | static void nicvf_set_xdp_queues(struct nicvf *nic, bool bpf_attached) |
1802 | { | |
1803 | u8 cq_count, txq_count; | |
1804 | ||
1805 | /* Set XDP Tx queue count same as Rx queue count */ | |
1806 | if (!bpf_attached) | |
1807 | nic->xdp_tx_queues = 0; | |
1808 | else | |
1809 | nic->xdp_tx_queues = nic->rx_queues; | |
1810 | ||
1811 | /* If queue count > MAX_CMP_QUEUES_PER_QS, then additional qsets | |
1812 | * needs to be allocated, check how many. | |
1813 | */ | |
1814 | txq_count = nic->xdp_tx_queues + nic->tx_queues; | |
1815 | cq_count = max(nic->rx_queues, txq_count); | |
1816 | if (cq_count > MAX_CMP_QUEUES_PER_QS) { | |
1817 | nic->sqs_count = roundup(cq_count, MAX_CMP_QUEUES_PER_QS); | |
1818 | nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1; | |
1819 | } else { | |
1820 | nic->sqs_count = 0; | |
1821 | } | |
1822 | ||
1823 | /* Set primary Qset's resources */ | |
1824 | nic->qs->rq_cnt = min_t(u8, nic->rx_queues, MAX_RCV_QUEUES_PER_QS); | |
1825 | nic->qs->sq_cnt = min_t(u8, txq_count, MAX_SND_QUEUES_PER_QS); | |
1826 | nic->qs->cq_cnt = max_t(u8, nic->qs->rq_cnt, nic->qs->sq_cnt); | |
1827 | ||
1828 | /* Update stack */ | |
1829 | nicvf_set_real_num_queues(nic->netdev, nic->tx_queues, nic->rx_queues); | |
1830 | } | |
1831 | ||
1832 | static int nicvf_xdp_setup(struct nicvf *nic, struct bpf_prog *prog) | |
1833 | { | |
1834 | struct net_device *dev = nic->netdev; | |
1835 | bool if_up = netif_running(nic->netdev); | |
1836 | struct bpf_prog *old_prog; | |
1837 | bool bpf_attached = false; | |
6d0f60b0 | 1838 | int ret = 0; |
05c773f5 | 1839 | |
5ee15c10 MC |
1840 | /* For now just support only the usual MTU sized frames, |
1841 | * plus some headroom for VLAN, QinQ. | |
1842 | */ | |
1843 | if (prog && dev->mtu > MAX_XDP_MTU) { | |
05c773f5 SG |
1844 | netdev_warn(dev, "Jumbo frames not yet supported with XDP, current MTU %d.\n", |
1845 | dev->mtu); | |
1846 | return -EOPNOTSUPP; | |
1847 | } | |
1848 | ||
05c773f5 SG |
1849 | /* ALL SQs attached to CQs i.e same as RQs, are treated as |
1850 | * XDP Tx queues and more Tx queues are allocated for | |
1851 | * network stack to send pkts out. | |
1852 | * | |
1853 | * No of Tx queues are either same as Rx queues or whatever | |
1854 | * is left in max no of queues possible. | |
1855 | */ | |
1856 | if ((nic->rx_queues + nic->tx_queues) > nic->max_queues) { | |
1857 | netdev_warn(dev, | |
1858 | "Failed to attach BPF prog, RXQs + TXQs > Max %d\n", | |
1859 | nic->max_queues); | |
1860 | return -ENOMEM; | |
1861 | } | |
1862 | ||
1863 | if (if_up) | |
1864 | nicvf_stop(nic->netdev); | |
1865 | ||
1866 | old_prog = xchg(&nic->xdp_prog, prog); | |
1867 | /* Detach old prog, if any */ | |
1868 | if (old_prog) | |
1869 | bpf_prog_put(old_prog); | |
1870 | ||
1871 | if (nic->xdp_prog) { | |
1872 | /* Attach BPF program */ | |
85192dbf AN |
1873 | bpf_prog_add(nic->xdp_prog, nic->rx_queues - 1); |
1874 | bpf_attached = true; | |
05c773f5 SG |
1875 | } |
1876 | ||
1877 | /* Calculate Tx queues needed for XDP and network stack */ | |
1878 | nicvf_set_xdp_queues(nic, bpf_attached); | |
1879 | ||
1880 | if (if_up) { | |
1881 | /* Reinitialize interface, clean slate */ | |
1882 | nicvf_open(nic->netdev); | |
1883 | netif_trans_update(nic->netdev); | |
1884 | } | |
1885 | ||
6d0f60b0 | 1886 | return ret; |
05c773f5 SG |
1887 | } |
1888 | ||
f4e63525 | 1889 | static int nicvf_xdp(struct net_device *netdev, struct netdev_bpf *xdp) |
05c773f5 SG |
1890 | { |
1891 | struct nicvf *nic = netdev_priv(netdev); | |
1892 | ||
1893 | /* To avoid checks while retrieving buffer address from CQE_RX, | |
1894 | * do not support XDP for T88 pass1.x silicons which are anyway | |
1895 | * not in use widely. | |
1896 | */ | |
1897 | if (pass1_silicon(nic->pdev)) | |
1898 | return -EOPNOTSUPP; | |
1899 | ||
1900 | switch (xdp->command) { | |
1901 | case XDP_SETUP_PROG: | |
1902 | return nicvf_xdp_setup(nic, xdp->prog); | |
05c773f5 SG |
1903 | default: |
1904 | return -EINVAL; | |
1905 | } | |
1906 | } | |
1907 | ||
4a875509 SG |
1908 | static int nicvf_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr) |
1909 | { | |
1910 | struct hwtstamp_config config; | |
1911 | struct nicvf *nic = netdev_priv(netdev); | |
1912 | ||
1913 | if (!nic->ptp_clock) | |
1914 | return -ENODEV; | |
1915 | ||
1916 | if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) | |
1917 | return -EFAULT; | |
1918 | ||
4a875509 SG |
1919 | switch (config.tx_type) { |
1920 | case HWTSTAMP_TX_OFF: | |
1921 | case HWTSTAMP_TX_ON: | |
1922 | break; | |
1923 | default: | |
1924 | return -ERANGE; | |
1925 | } | |
1926 | ||
1927 | switch (config.rx_filter) { | |
1928 | case HWTSTAMP_FILTER_NONE: | |
1929 | nic->hw_rx_tstamp = false; | |
1930 | break; | |
1931 | case HWTSTAMP_FILTER_ALL: | |
1932 | case HWTSTAMP_FILTER_SOME: | |
1933 | case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: | |
1934 | case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: | |
1935 | case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: | |
1936 | case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: | |
1937 | case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: | |
1938 | case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: | |
1939 | case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: | |
1940 | case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: | |
1941 | case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: | |
1942 | case HWTSTAMP_FILTER_PTP_V2_EVENT: | |
1943 | case HWTSTAMP_FILTER_PTP_V2_SYNC: | |
1944 | case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: | |
1945 | nic->hw_rx_tstamp = true; | |
1946 | config.rx_filter = HWTSTAMP_FILTER_ALL; | |
1947 | break; | |
1948 | default: | |
1949 | return -ERANGE; | |
1950 | } | |
1951 | ||
1952 | if (netif_running(netdev)) | |
1953 | nicvf_config_hw_rx_tstamp(nic, nic->hw_rx_tstamp); | |
1954 | ||
1955 | if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) | |
1956 | return -EFAULT; | |
1957 | ||
1958 | return 0; | |
1959 | } | |
1960 | ||
1961 | static int nicvf_ioctl(struct net_device *netdev, struct ifreq *req, int cmd) | |
1962 | { | |
1963 | switch (cmd) { | |
1964 | case SIOCSHWTSTAMP: | |
1965 | return nicvf_config_hwtstamp(netdev, req); | |
1966 | default: | |
1967 | return -EOPNOTSUPP; | |
1968 | } | |
1969 | } | |
1970 | ||
469998c8 VL |
1971 | static void __nicvf_set_rx_mode_task(u8 mode, struct xcast_addr_list *mc_addrs, |
1972 | struct nicvf *nic) | |
37c3347e | 1973 | { |
37c3347e | 1974 | union nic_mbx mbx = {}; |
9b5c4dfb | 1975 | int idx; |
37c3347e | 1976 | |
37c3347e VL |
1977 | /* From the inside of VM code flow we have only 128 bits memory |
1978 | * available to send message to host's PF, so send all mc addrs | |
1979 | * one by one, starting from flush command in case if kernel | |
1980 | * requests to configure specific MAC filtering | |
1981 | */ | |
1982 | ||
1983 | /* flush DMAC filters and reset RX mode */ | |
1984 | mbx.xcast.msg = NIC_MBOX_MSG_RESET_XCAST; | |
7db730d9 VL |
1985 | if (nicvf_send_msg_to_pf(nic, &mbx) < 0) |
1986 | goto free_mc; | |
37c3347e | 1987 | |
469998c8 | 1988 | if (mode & BGX_XCAST_MCAST_FILTER) { |
37c3347e VL |
1989 | /* once enabling filtering, we need to signal to PF to add |
1990 | * its' own LMAC to the filter to accept packets for it. | |
1991 | */ | |
1992 | mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST; | |
53544396 | 1993 | mbx.xcast.mac = 0; |
7db730d9 VL |
1994 | if (nicvf_send_msg_to_pf(nic, &mbx) < 0) |
1995 | goto free_mc; | |
37c3347e VL |
1996 | } |
1997 | ||
1998 | /* check if we have any specific MACs to be added to PF DMAC filter */ | |
469998c8 | 1999 | if (mc_addrs) { |
37c3347e | 2000 | /* now go through kernel list of MACs and add them one by one */ |
469998c8 | 2001 | for (idx = 0; idx < mc_addrs->count; idx++) { |
37c3347e | 2002 | mbx.xcast.msg = NIC_MBOX_MSG_ADD_MCAST; |
53544396 | 2003 | mbx.xcast.mac = mc_addrs->mc[idx]; |
7db730d9 VL |
2004 | if (nicvf_send_msg_to_pf(nic, &mbx) < 0) |
2005 | goto free_mc; | |
37c3347e | 2006 | } |
37c3347e VL |
2007 | } |
2008 | ||
2009 | /* and finally set rx mode for PF accordingly */ | |
2010 | mbx.xcast.msg = NIC_MBOX_MSG_SET_XCAST; | |
53544396 | 2011 | mbx.xcast.mode = mode; |
37c3347e VL |
2012 | |
2013 | nicvf_send_msg_to_pf(nic, &mbx); | |
7db730d9 VL |
2014 | free_mc: |
2015 | kfree(mc_addrs); | |
37c3347e VL |
2016 | } |
2017 | ||
469998c8 VL |
2018 | static void nicvf_set_rx_mode_task(struct work_struct *work_arg) |
2019 | { | |
2020 | struct nicvf_work *vf_work = container_of(work_arg, struct nicvf_work, | |
2ecbe4f4 | 2021 | work); |
469998c8 VL |
2022 | struct nicvf *nic = container_of(vf_work, struct nicvf, rx_mode_work); |
2023 | u8 mode; | |
2024 | struct xcast_addr_list *mc; | |
2025 | ||
469998c8 VL |
2026 | /* Save message data locally to prevent them from |
2027 | * being overwritten by next ndo_set_rx_mode call(). | |
2028 | */ | |
bab9693a | 2029 | spin_lock_bh(&nic->rx_mode_wq_lock); |
469998c8 VL |
2030 | mode = vf_work->mode; |
2031 | mc = vf_work->mc; | |
2032 | vf_work->mc = NULL; | |
bab9693a | 2033 | spin_unlock_bh(&nic->rx_mode_wq_lock); |
469998c8 VL |
2034 | |
2035 | __nicvf_set_rx_mode_task(mode, mc, nic); | |
2036 | } | |
2037 | ||
37c3347e VL |
2038 | static void nicvf_set_rx_mode(struct net_device *netdev) |
2039 | { | |
2040 | struct nicvf *nic = netdev_priv(netdev); | |
2041 | struct netdev_hw_addr *ha; | |
2042 | struct xcast_addr_list *mc_list = NULL; | |
2043 | u8 mode = 0; | |
2044 | ||
2045 | if (netdev->flags & IFF_PROMISC) { | |
2046 | mode = BGX_XCAST_BCAST_ACCEPT | BGX_XCAST_MCAST_ACCEPT; | |
2047 | } else { | |
2048 | if (netdev->flags & IFF_BROADCAST) | |
2049 | mode |= BGX_XCAST_BCAST_ACCEPT; | |
2050 | ||
2051 | if (netdev->flags & IFF_ALLMULTI) { | |
2052 | mode |= BGX_XCAST_MCAST_ACCEPT; | |
2053 | } else if (netdev->flags & IFF_MULTICAST) { | |
2054 | mode |= BGX_XCAST_MCAST_FILTER; | |
2055 | /* here we need to copy mc addrs */ | |
2056 | if (netdev_mc_count(netdev)) { | |
f6e5ee6a GS |
2057 | mc_list = kmalloc(struct_size(mc_list, mc, |
2058 | netdev_mc_count(netdev)), | |
9b5c4dfb VL |
2059 | GFP_ATOMIC); |
2060 | if (unlikely(!mc_list)) | |
2061 | return; | |
2062 | mc_list->count = 0; | |
37c3347e | 2063 | netdev_hw_addr_list_for_each(ha, &netdev->mc) { |
9b5c4dfb | 2064 | mc_list->mc[mc_list->count] = |
37c3347e | 2065 | ether_addr_to_u64(ha->addr); |
37c3347e VL |
2066 | mc_list->count++; |
2067 | } | |
2068 | } | |
2069 | } | |
2070 | } | |
469998c8 VL |
2071 | spin_lock(&nic->rx_mode_wq_lock); |
2072 | kfree(nic->rx_mode_work.mc); | |
37c3347e VL |
2073 | nic->rx_mode_work.mc = mc_list; |
2074 | nic->rx_mode_work.mode = mode; | |
2ecbe4f4 | 2075 | queue_work(nic->nicvf_rx_mode_wq, &nic->rx_mode_work.work); |
469998c8 | 2076 | spin_unlock(&nic->rx_mode_wq_lock); |
37c3347e VL |
2077 | } |
2078 | ||
4863dea3 SG |
2079 | static const struct net_device_ops nicvf_netdev_ops = { |
2080 | .ndo_open = nicvf_open, | |
2081 | .ndo_stop = nicvf_stop, | |
2082 | .ndo_start_xmit = nicvf_xmit, | |
2083 | .ndo_change_mtu = nicvf_change_mtu, | |
2084 | .ndo_set_mac_address = nicvf_set_mac_address, | |
2085 | .ndo_get_stats64 = nicvf_get_stats64, | |
2086 | .ndo_tx_timeout = nicvf_tx_timeout, | |
d77a2384 | 2087 | .ndo_fix_features = nicvf_fix_features, |
aa2e259b | 2088 | .ndo_set_features = nicvf_set_features, |
f4e63525 | 2089 | .ndo_bpf = nicvf_xdp, |
a7605370 | 2090 | .ndo_eth_ioctl = nicvf_ioctl, |
37c3347e | 2091 | .ndo_set_rx_mode = nicvf_set_rx_mode, |
4863dea3 SG |
2092 | }; |
2093 | ||
2094 | static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) | |
2095 | { | |
2096 | struct device *dev = &pdev->dev; | |
2097 | struct net_device *netdev; | |
2098 | struct nicvf *nic; | |
92dc8769 | 2099 | int err, qcount; |
7ceb8a13 | 2100 | u16 sdevid; |
4a875509 SG |
2101 | struct cavium_ptp *ptp_clock; |
2102 | ||
2103 | ptp_clock = cavium_ptp_get(); | |
2104 | if (IS_ERR(ptp_clock)) { | |
2105 | if (PTR_ERR(ptp_clock) == -ENODEV) | |
2106 | /* In virtualized environment we proceed without ptp */ | |
2107 | ptp_clock = NULL; | |
2108 | else | |
2109 | return PTR_ERR(ptp_clock); | |
2110 | } | |
4863dea3 SG |
2111 | |
2112 | err = pci_enable_device(pdev); | |
52583c8d CH |
2113 | if (err) |
2114 | return dev_err_probe(dev, err, "Failed to enable PCI device\n"); | |
4863dea3 SG |
2115 | |
2116 | err = pci_request_regions(pdev, DRV_NAME); | |
2117 | if (err) { | |
2118 | dev_err(dev, "PCI request regions failed 0x%x\n", err); | |
2119 | goto err_disable_device; | |
2120 | } | |
2121 | ||
1e0dd56e | 2122 | err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); |
4863dea3 SG |
2123 | if (err) { |
2124 | dev_err(dev, "Unable to get usable DMA configuration\n"); | |
2125 | goto err_release_regions; | |
2126 | } | |
2127 | ||
3a397ebe | 2128 | qcount = netif_get_num_default_rss_queues(); |
92dc8769 SG |
2129 | |
2130 | /* Restrict multiqset support only for host bound VFs */ | |
2131 | if (pdev->is_virtfn) { | |
2132 | /* Set max number of queues per VF */ | |
3a397ebe SG |
2133 | qcount = min_t(int, num_online_cpus(), |
2134 | (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS); | |
92dc8769 SG |
2135 | } |
2136 | ||
2137 | netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount); | |
4863dea3 SG |
2138 | if (!netdev) { |
2139 | err = -ENOMEM; | |
2140 | goto err_release_regions; | |
2141 | } | |
2142 | ||
2143 | pci_set_drvdata(pdev, netdev); | |
2144 | ||
2145 | SET_NETDEV_DEV(netdev, &pdev->dev); | |
2146 | ||
2147 | nic = netdev_priv(netdev); | |
2148 | nic->netdev = netdev; | |
2149 | nic->pdev = pdev; | |
92dc8769 SG |
2150 | nic->pnicvf = nic; |
2151 | nic->max_queues = qcount; | |
87de0838 SG |
2152 | /* If no of CPUs are too low, there won't be any queues left |
2153 | * for XDP_TX, hence double it. | |
2154 | */ | |
2155 | if (!nic->t88) | |
2156 | nic->max_queues *= 2; | |
4a875509 | 2157 | nic->ptp_clock = ptp_clock; |
4863dea3 | 2158 | |
c1055b76 DN |
2159 | /* Initialize mutex that serializes usage of VF's mailbox */ |
2160 | mutex_init(&nic->rx_mode_mtx); | |
2161 | ||
4863dea3 SG |
2162 | /* MAP VF's configuration registers */ |
2163 | nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); | |
2164 | if (!nic->reg_base) { | |
2165 | dev_err(dev, "Cannot map config register space, aborting\n"); | |
2166 | err = -ENOMEM; | |
2167 | goto err_free_netdev; | |
2168 | } | |
2169 | ||
964cb69b SG |
2170 | nic->drv_stats = netdev_alloc_pcpu_stats(struct nicvf_drv_stats); |
2171 | if (!nic->drv_stats) { | |
2172 | err = -ENOMEM; | |
2173 | goto err_free_netdev; | |
2174 | } | |
2175 | ||
4863dea3 SG |
2176 | err = nicvf_set_qset_resources(nic); |
2177 | if (err) | |
2178 | goto err_free_netdev; | |
2179 | ||
4863dea3 SG |
2180 | /* Check if PF is alive and get MAC address for this VF */ |
2181 | err = nicvf_register_misc_interrupt(nic); | |
2182 | if (err) | |
2183 | goto err_free_netdev; | |
2184 | ||
92dc8769 SG |
2185 | nicvf_send_vf_struct(nic); |
2186 | ||
8d210d54 SG |
2187 | if (!pass1_silicon(nic->pdev)) |
2188 | nic->hw_tso = true; | |
2189 | ||
83abb7d7 SG |
2190 | /* Get iommu domain for iova to physical addr conversion */ |
2191 | nic->iommu_domain = iommu_get_domain_for_dev(dev); | |
2192 | ||
7ceb8a13 SG |
2193 | pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid); |
2194 | if (sdevid == 0xA134) | |
2195 | nic->t88 = true; | |
2196 | ||
92dc8769 SG |
2197 | /* Check if this VF is in QS only mode */ |
2198 | if (nic->sqs_mode) | |
2199 | return 0; | |
2200 | ||
2201 | err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues); | |
2202 | if (err) | |
2203 | goto err_unregister_interrupts; | |
2204 | ||
3a9024f5 TS |
2205 | netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_SG | |
2206 | NETIF_F_TSO | NETIF_F_GRO | NETIF_F_TSO6 | | |
2207 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | | |
92dc8769 SG |
2208 | NETIF_F_HW_VLAN_CTAG_RX); |
2209 | ||
2210 | netdev->hw_features |= NETIF_F_RXHASH; | |
aa2e259b SG |
2211 | |
2212 | netdev->features |= netdev->hw_features; | |
d77a2384 | 2213 | netdev->hw_features |= NETIF_F_LOOPBACK; |
38bb5d4f | 2214 | |
3a9024f5 TS |
2215 | netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | |
2216 | NETIF_F_IPV6_CSUM | NETIF_F_TSO | NETIF_F_TSO6; | |
4863dea3 SG |
2217 | |
2218 | netdev->netdev_ops = &nicvf_netdev_ops; | |
3d7a8aaa | 2219 | netdev->watchdog_timeo = NICVF_TX_TIMEOUT; |
4863dea3 | 2220 | |
3c249fe4 LB |
2221 | if (!pass1_silicon(nic->pdev) && |
2222 | nic->rx_queues + nic->tx_queues <= nic->max_queues) | |
2223 | netdev->xdp_features = NETDEV_XDP_ACT_BASIC; | |
66c0e13a | 2224 | |
109cc165 JW |
2225 | /* MTU range: 64 - 9200 */ |
2226 | netdev->min_mtu = NIC_HW_MIN_FRS; | |
2227 | netdev->max_mtu = NIC_HW_MAX_FRS; | |
2228 | ||
4863dea3 SG |
2229 | INIT_WORK(&nic->reset_task, nicvf_reset_task); |
2230 | ||
2ecbe4f4 VL |
2231 | nic->nicvf_rx_mode_wq = alloc_ordered_workqueue("nicvf_rx_mode_wq_VF%d", |
2232 | WQ_MEM_RECLAIM, | |
2233 | nic->vf_id); | |
0b31d98d KL |
2234 | if (!nic->nicvf_rx_mode_wq) { |
2235 | err = -ENOMEM; | |
2236 | dev_err(dev, "Failed to allocate work queue\n"); | |
2237 | goto err_unregister_interrupts; | |
2238 | } | |
2239 | ||
2ecbe4f4 | 2240 | INIT_WORK(&nic->rx_mode_work.work, nicvf_set_rx_mode_task); |
469998c8 | 2241 | spin_lock_init(&nic->rx_mode_wq_lock); |
37c3347e | 2242 | |
4863dea3 SG |
2243 | err = register_netdev(netdev); |
2244 | if (err) { | |
2245 | dev_err(dev, "Failed to register netdevice\n"); | |
42330a32 | 2246 | goto err_destroy_workqueue; |
4863dea3 SG |
2247 | } |
2248 | ||
2249 | nic->msg_enable = debug; | |
2250 | ||
2251 | nicvf_set_ethtool_ops(netdev); | |
2252 | ||
2253 | return 0; | |
2254 | ||
42330a32 YL |
2255 | err_destroy_workqueue: |
2256 | destroy_workqueue(nic->nicvf_rx_mode_wq); | |
4863dea3 SG |
2257 | err_unregister_interrupts: |
2258 | nicvf_unregister_interrupts(nic); | |
2259 | err_free_netdev: | |
2260 | pci_set_drvdata(pdev, NULL); | |
964cb69b SG |
2261 | if (nic->drv_stats) |
2262 | free_percpu(nic->drv_stats); | |
4863dea3 SG |
2263 | free_netdev(netdev); |
2264 | err_release_regions: | |
2265 | pci_release_regions(pdev); | |
2266 | err_disable_device: | |
2267 | pci_disable_device(pdev); | |
2268 | return err; | |
2269 | } | |
2270 | ||
2271 | static void nicvf_remove(struct pci_dev *pdev) | |
2272 | { | |
2273 | struct net_device *netdev = pci_get_drvdata(pdev); | |
7750130d PF |
2274 | struct nicvf *nic; |
2275 | struct net_device *pnetdev; | |
2276 | ||
2277 | if (!netdev) | |
2278 | return; | |
2279 | ||
2280 | nic = netdev_priv(netdev); | |
2281 | pnetdev = nic->pnicvf->netdev; | |
4863dea3 | 2282 | |
92dc8769 SG |
2283 | /* Check if this Qset is assigned to different VF. |
2284 | * If yes, clean primary and all secondary Qsets. | |
2285 | */ | |
2286 | if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED)) | |
2287 | unregister_netdev(pnetdev); | |
2ecbe4f4 VL |
2288 | if (nic->nicvf_rx_mode_wq) { |
2289 | destroy_workqueue(nic->nicvf_rx_mode_wq); | |
2290 | nic->nicvf_rx_mode_wq = NULL; | |
2291 | } | |
4863dea3 SG |
2292 | nicvf_unregister_interrupts(nic); |
2293 | pci_set_drvdata(pdev, NULL); | |
964cb69b SG |
2294 | if (nic->drv_stats) |
2295 | free_percpu(nic->drv_stats); | |
4a875509 | 2296 | cavium_ptp_put(nic->ptp_clock); |
4863dea3 SG |
2297 | free_netdev(netdev); |
2298 | pci_release_regions(pdev); | |
2299 | pci_disable_device(pdev); | |
2300 | } | |
2301 | ||
4adf4351 SG |
2302 | static void nicvf_shutdown(struct pci_dev *pdev) |
2303 | { | |
2304 | nicvf_remove(pdev); | |
2305 | } | |
2306 | ||
4863dea3 SG |
2307 | static struct pci_driver nicvf_driver = { |
2308 | .name = DRV_NAME, | |
2309 | .id_table = nicvf_id_table, | |
2310 | .probe = nicvf_probe, | |
2311 | .remove = nicvf_remove, | |
4adf4351 | 2312 | .shutdown = nicvf_shutdown, |
4863dea3 SG |
2313 | }; |
2314 | ||
2315 | static int __init nicvf_init_module(void) | |
2316 | { | |
2317 | pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION); | |
4863dea3 SG |
2318 | return pci_register_driver(&nicvf_driver); |
2319 | } | |
2320 | ||
2321 | static void __exit nicvf_cleanup_module(void) | |
2322 | { | |
2323 | pci_unregister_driver(&nicvf_driver); | |
2324 | } | |
2325 | ||
2326 | module_init(nicvf_init_module); | |
2327 | module_exit(nicvf_cleanup_module); |