projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: Enable HW GRO for Rev B(=0x21) HNS3 hardware
[linux-block.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
d71d8381
JS		 1// SPDX-License-Identifier: GPL-2.0+
2// Copyright (c) 2016-2017 Hisilicon Limited.
76ad4f0e
S		 3
4#include <linux/dma-mapping.h>
5#include <linux/etherdevice.h>
6#include <linux/interrupt.h>
7#include <linux/if_vlan.h>
8#include <linux/ip.h>
9#include <linux/ipv6.h>
10#include <linux/module.h>
11#include <linux/pci.h>
6ae4e733 12#include <linux/aer.h>
76ad4f0e
S		 13#include <linux/skbuff.h>
14#include <linux/sctp.h>
15#include <linux/vermagic.h>
16#include <net/gre.h>
30d240df 17#include <net/pkt_cls.h>
76ad4f0e
S		 18#include <net/vxlan.h>
19
20#include "hnae3.h"
21#include "hns3_enet.h"
22
7b763f3f
FL		 23static void hns3_clear_all_ring(struct hnae3_handle *h);
24static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
f05e2109 25static void hns3_remove_hw_addr(struct net_device *netdev);
7b763f3f 26
1db9b1bf 27static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 28const char hns3_driver_version[] = VERMAGIC_STRING;
29static const char hns3_driver_string[] =
30 "Hisilicon Ethernet Network Driver for Hip08 Family";
31static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
32static struct hnae3_client client;
33
34/* hns3_pci_tbl - PCI Device ID Table
35 *
36 * Last entry must be all 0s
37 *
38 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
39 * Class, Class Mask, private data (not used) }
40 */
41static const struct pci_device_id hns3_pci_tbl[] = {
42 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 45 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 46 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 47 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 48 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 49 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 50 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 51 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 52 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 53 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
424eb834 54 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
07acf909
JS		 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
56 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
76ad4f0e
S		 57 /* required last entry */
58 {0, }
59};
60MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
ef0c5009 62static irqreturn_t hns3_irq_handle(int irq, void *vector)
76ad4f0e 63{
ef0c5009 64 struct hns3_enet_tqp_vector *tqp_vector = vector;
76ad4f0e
S		 65
66 napi_schedule(&tqp_vector->napi);
67
68 return IRQ_HANDLED;
69}
70
874bff0b
PL		 71/* This callback function is used to set affinity changes to the irq affinity
72 * masks when the irq_set_affinity_notifier function is used.
73 */
74static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
75 const cpumask_t *mask)
76{
77 struct hns3_enet_tqp_vector *tqp_vectors =
78 container_of(notify, struct hns3_enet_tqp_vector,
79 affinity_notify);
80
81 tqp_vectors->affinity_mask = *mask;
82}
83
84static void hns3_nic_irq_affinity_release(struct kref *ref)
85{
86}
87
76ad4f0e
S		 88static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
89{
90 struct hns3_enet_tqp_vector *tqp_vectors;
91 unsigned int i;
92
93 for (i = 0; i < priv->vector_num; i++) {
94 tqp_vectors = &priv->tqp_vector[i];
95
96 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
97 continue;
98
874bff0b
PL		 99 /* clear the affinity notifier and affinity mask */
100 irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
101 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
102
76ad4f0e
S		 103 /* release the irq resource */
104 free_irq(tqp_vectors->vector_irq, tqp_vectors);
105 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
106 }
107}
108
109static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
110{
111 struct hns3_enet_tqp_vector *tqp_vectors;
112 int txrx_int_idx = 0;
113 int rx_int_idx = 0;
114 int tx_int_idx = 0;
115 unsigned int i;
116 int ret;
117
118 for (i = 0; i < priv->vector_num; i++) {
119 tqp_vectors = &priv->tqp_vector[i];
120
121 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
122 continue;
123
124 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
125 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
126 "%s-%s-%d", priv->netdev->name, "TxRx",
127 txrx_int_idx++);
128 txrx_int_idx++;
129 } else if (tqp_vectors->rx_group.ring) {
130 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
131 "%s-%s-%d", priv->netdev->name, "Rx",
132 rx_int_idx++);
133 } else if (tqp_vectors->tx_group.ring) {
134 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
135 "%s-%s-%d", priv->netdev->name, "Tx",
136 tx_int_idx++);
137 } else {
138 /* Skip this unused q_vector */
139 continue;
140 }
141
142 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
143
144 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
145 tqp_vectors->name,
146 tqp_vectors);
147 if (ret) {
148 netdev_err(priv->netdev, "request irq(%d) fail\n",
149 tqp_vectors->vector_irq);
150 return ret;
151 }
152
874bff0b
PL		 153 tqp_vectors->affinity_notify.notify =
154 hns3_nic_irq_affinity_notify;
155 tqp_vectors->affinity_notify.release =
156 hns3_nic_irq_affinity_release;
157 irq_set_affinity_notifier(tqp_vectors->vector_irq,
158 &tqp_vectors->affinity_notify);
159 irq_set_affinity_hint(tqp_vectors->vector_irq,
160 &tqp_vectors->affinity_mask);
161
76ad4f0e
S		 162 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
163 }
164
165 return 0;
166}
167
168static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
169 u32 mask_en)
170{
171 writel(mask_en, tqp_vector->mask_addr);
172}
173
174static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
175{
176 napi_enable(&tqp_vector->napi);
177
178 /* enable vector */
179 hns3_mask_vector_irq(tqp_vector, 1);
180}
181
182static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
183{
184 /* disable vector */
185 hns3_mask_vector_irq(tqp_vector, 0);
186
187 disable_irq(tqp_vector->vector_irq);
188 napi_disable(&tqp_vector->napi);
189}
190
434776a5
FL		 191void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
192 u32 rl_value)
76ad4f0e 193{
434776a5
FL		 194 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
195
76ad4f0e
S		 196 /* this defines the configuration for RL (Interrupt Rate Limiter).
197 * Rl defines rate of interrupts i.e. number of interrupts-per-second
198 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
199 */
434776a5 200
9bc727a9
YL		 201 if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
202 !tqp_vector->rx_group.coal.gl_adapt_enable)
434776a5
FL		 203 /* According to the hardware, the range of rl_reg is
204 * 0-59 and the unit is 4.
205 */
206 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
207
208 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
209}
210
211void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
212 u32 gl_value)
213{
214 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
215
216 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
217}
218
219void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
220 u32 gl_value)
221{
222 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
223
224 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 225}
226
5fd4789a
FL		 227static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
228 struct hns3_nic_priv *priv)
76ad4f0e
S		 229{
230 /* initialize the configuration for interrupt coalescing.
231 * 1. GL (Interrupt Gap Limiter)
232 * 2. RL (Interrupt Rate Limiter)
233 */
234
5fd4789a 235 /* Default: enable interrupt coalescing self-adaptive and GL */
9bc727a9
YL		 236 tqp_vector->tx_group.coal.gl_adapt_enable = 1;
237 tqp_vector->rx_group.coal.gl_adapt_enable = 1;
5fd4789a 238
9bc727a9
YL		 239 tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
240 tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
5fd4789a 241
cd9d187b 242 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
9bc727a9
YL		 243 tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
244 tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 245}
246
dd38c726
YL		 247static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
248 struct hns3_nic_priv *priv)
249{
250 struct hnae3_handle *h = priv->ae_handle;
251
252 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 253 tqp_vector->tx_group.coal.int_gl);
dd38c726 254 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 255 tqp_vector->rx_group.coal.int_gl);
dd38c726
YL		 256 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
257}
258
9df8f79a
YL		 259static int hns3_nic_set_real_num_queue(struct net_device *netdev)
260{
9780cb97 261 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 262 struct hnae3_knic_private_info *kinfo = &h->kinfo;
263 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
a75a8efa
YL		 264 int i, ret;
265
266 if (kinfo->num_tc <= 1) {
267 netdev_reset_tc(netdev);
268 } else {
269 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
270 if (ret) {
271 netdev_err(netdev,
272 "netdev_set_num_tc fail, ret=%d!\n", ret);
273 return ret;
274 }
275
276 for (i = 0; i < HNAE3_MAX_TC; i++) {
277 if (!kinfo->tc_info[i].enable)
278 continue;
279
280 netdev_set_tc_queue(netdev,
281 kinfo->tc_info[i].tc,
282 kinfo->tc_info[i].tqp_count,
283 kinfo->tc_info[i].tqp_offset);
284 }
285 }
9df8f79a
YL		 286
287 ret = netif_set_real_num_tx_queues(netdev, queue_size);
288 if (ret) {
289 netdev_err(netdev,
290 "netif_set_real_num_tx_queues fail, ret=%d!\n",
291 ret);
292 return ret;
293 }
294
295 ret = netif_set_real_num_rx_queues(netdev, queue_size);
296 if (ret) {
297 netdev_err(netdev,
298 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
299 return ret;
300 }
301
302 return 0;
303}
304
678335a1
PL		 305static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
306{
0d43bf45 307 u16 alloc_tqps, max_rss_size, rss_size;
678335a1 308
0d43bf45
HT		 309 h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
310 rss_size = alloc_tqps / h->kinfo.num_tc;
678335a1 311
0d43bf45 312 return min_t(u16, rss_size, max_rss_size);
678335a1
PL		 313}
314
8df0fa91
HT		 315static void hns3_tqp_enable(struct hnae3_queue *tqp)
316{
317 u32 rcb_reg;
318
319 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
320 rcb_reg |= BIT(HNS3_RING_EN_B);
321 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
322}
323
324static void hns3_tqp_disable(struct hnae3_queue *tqp)
325{
326 u32 rcb_reg;
327
328 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
329 rcb_reg &= ~BIT(HNS3_RING_EN_B);
330 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
331}
332
76ad4f0e
S		 333static int hns3_nic_net_up(struct net_device *netdev)
334{
335 struct hns3_nic_priv *priv = netdev_priv(netdev);
336 struct hnae3_handle *h = priv->ae_handle;
337 int i, j;
338 int ret;
339
7b763f3f
FL		 340 ret = hns3_nic_reset_all_ring(h);
341 if (ret)
342 return ret;
343
76ad4f0e
S		 344 /* get irq resource for all vectors */
345 ret = hns3_nic_init_irq(priv);
346 if (ret) {
347 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
348 return ret;
349 }
350
351 /* enable the vectors */
352 for (i = 0; i < priv->vector_num; i++)
353 hns3_vector_enable(&priv->tqp_vector[i]);
354
8df0fa91
HT		 355 /* enable rcb */
356 for (j = 0; j < h->kinfo.num_tqps; j++)
357 hns3_tqp_enable(h->kinfo.tqp[j]);
358
76ad4f0e
S		 359 /* start the ae_dev */
360 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
361 if (ret)
362 goto out_start_err;
363
b875cc37
JS		 364 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
365
76ad4f0e
S		 366 return 0;
367
368out_start_err:
8df0fa91
HT		 369 while (j--)
370 hns3_tqp_disable(h->kinfo.tqp[j]);
371
76ad4f0e
S		 372 for (j = i - 1; j >= 0; j--)
373 hns3_vector_disable(&priv->tqp_vector[j]);
374
375 hns3_nic_uninit_irq(priv);
376
377 return ret;
378}
379
380static int hns3_nic_net_open(struct net_device *netdev)
381{
a75a8efa
YL		 382 struct hnae3_handle *h = hns3_get_handle(netdev);
383 struct hnae3_knic_private_info *kinfo;
384 int i, ret;
76ad4f0e 385
257e4f29
HT		 386 if (hns3_nic_resetting(netdev))
387 return -EBUSY;
388
76ad4f0e
S		 389 netif_carrier_off(netdev);
390
9df8f79a
YL		 391 ret = hns3_nic_set_real_num_queue(netdev);
392 if (ret)
76ad4f0e 393 return ret;
76ad4f0e
S		 394
395 ret = hns3_nic_net_up(netdev);
396 if (ret) {
397 netdev_err(netdev,
398 "hns net up fail, ret=%d!\n", ret);
399 return ret;
400 }
401
a75a8efa
YL		 402 kinfo = &h->kinfo;
403 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
404 netdev_set_prio_tc_map(netdev, i,
405 kinfo->prio_tc[i]);
406 }
407
76ad4f0e
S		 408 return 0;
409}
410
411static void hns3_nic_net_down(struct net_device *netdev)
412{
413 struct hns3_nic_priv *priv = netdev_priv(netdev);
8df0fa91 414 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 415 const struct hnae3_ae_ops *ops;
416 int i;
417
7b763f3f
FL		 418 /* disable vectors */
419 for (i = 0; i < priv->vector_num; i++)
420 hns3_vector_disable(&priv->tqp_vector[i]);
8df0fa91
HT		 421
422 /* disable rcb */
423 for (i = 0; i < h->kinfo.num_tqps; i++)
424 hns3_tqp_disable(h->kinfo.tqp[i]);
7b763f3f 425
76ad4f0e
S		 426 /* stop ae_dev */
427 ops = priv->ae_handle->ae_algo->ops;
428 if (ops->stop)
429 ops->stop(priv->ae_handle);
430
76ad4f0e
S		 431 /* free irq resources */
432 hns3_nic_uninit_irq(priv);
7b763f3f
FL		 433
434 hns3_clear_all_ring(priv->ae_handle);
76ad4f0e
S		 435}
436
437static int hns3_nic_net_stop(struct net_device *netdev)
438{
ff0699e0
HT		 439 struct hns3_nic_priv *priv = netdev_priv(netdev);
440
441 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
442 return 0;
443
76ad4f0e
S		 444 netif_tx_stop_all_queues(netdev);
445 netif_carrier_off(netdev);
446
447 hns3_nic_net_down(netdev);
448
449 return 0;
450}
451
76ad4f0e
S		 452static int hns3_nic_uc_sync(struct net_device *netdev,
453 const unsigned char *addr)
454{
9780cb97 455 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 456
457 if (h->ae_algo->ops->add_uc_addr)
458 return h->ae_algo->ops->add_uc_addr(h, addr);
459
460 return 0;
461}
462
463static int hns3_nic_uc_unsync(struct net_device *netdev,
464 const unsigned char *addr)
465{
9780cb97 466 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 467
468 if (h->ae_algo->ops->rm_uc_addr)
469 return h->ae_algo->ops->rm_uc_addr(h, addr);
470
471 return 0;
472}
473
474static int hns3_nic_mc_sync(struct net_device *netdev,
475 const unsigned char *addr)
476{
9780cb97 477 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 478
720a8478 479 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 480 return h->ae_algo->ops->add_mc_addr(h, addr);
481
482 return 0;
483}
484
485static int hns3_nic_mc_unsync(struct net_device *netdev,
486 const unsigned char *addr)
487{
9780cb97 488 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 489
720a8478 490 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 491 return h->ae_algo->ops->rm_mc_addr(h, addr);
492
493 return 0;
494}
495
c60edc17
JS		 496static u8 hns3_get_netdev_flags(struct net_device *netdev)
497{
498 u8 flags = 0;
499
500 if (netdev->flags & IFF_PROMISC) {
501 flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
502 } else {
503 flags |= HNAE3_VLAN_FLTR;
504 if (netdev->flags & IFF_ALLMULTI)
505 flags |= HNAE3_USER_MPE;
506 }
507
508 return flags;
509}
510
1db9b1bf 511static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 512{
9780cb97 513 struct hnae3_handle *h = hns3_get_handle(netdev);
c60edc17
JS		 514 u8 new_flags;
515 int ret;
76ad4f0e 516
c60edc17
JS		 517 new_flags = hns3_get_netdev_flags(netdev);
518
519 ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
520 if (ret) {
76ad4f0e 521 netdev_err(netdev, "sync uc address fail\n");
c60edc17
JS		 522 if (ret == -ENOSPC)
523 new_flags |= HNAE3_OVERFLOW_UPE;
524 }
525
40cca1c5 526 if (netdev->flags & IFF_MULTICAST) {
c60edc17
JS		 527 ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
528 hns3_nic_mc_unsync);
529 if (ret) {
76ad4f0e 530 netdev_err(netdev, "sync mc address fail\n");
c60edc17
JS		 531 if (ret == -ENOSPC)
532 new_flags |= HNAE3_OVERFLOW_MPE;
533 }
534 }
535
536 hns3_update_promisc_mode(netdev, new_flags);
537 /* User mode Promisc mode enable and vlan filtering is disabled to
538 * let all packets in. MAC-VLAN Table overflow Promisc enabled and
539 * vlan fitering is enabled
540 */
541 hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
542 h->netdev_flags = new_flags;
543}
544
7fa6be4f 545int hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
c60edc17
JS		 546{
547 struct hns3_nic_priv *priv = netdev_priv(netdev);
548 struct hnae3_handle *h = priv->ae_handle;
549
550 if (h->ae_algo->ops->set_promisc_mode) {
7fa6be4f
HT		 551 return h->ae_algo->ops->set_promisc_mode(h,
552 promisc_flags & HNAE3_UPE,
553 promisc_flags & HNAE3_MPE);
c60edc17 554 }
7fa6be4f
HT		 555
556 return 0;
c60edc17
JS		 557}
558
559void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
560{
561 struct hns3_nic_priv *priv = netdev_priv(netdev);
562 struct hnae3_handle *h = priv->ae_handle;
563 bool last_state;
564
565 if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
566 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
567 if (enable != last_state) {
568 netdev_info(netdev,
569 "%s vlan filter\n",
570 enable ? "enable" : "disable");
571 h->ae_algo->ops->enable_vlan_filter(h, enable);
572 }
40cca1c5 573 }
76ad4f0e
S		 574}
575
576static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
577 u16 *mss, u32 *type_cs_vlan_tso)
578{
579 u32 l4_offset, hdr_len;
580 union l3_hdr_info l3;
581 union l4_hdr_info l4;
582 u32 l4_paylen;
583 int ret;
584
585 if (!skb_is_gso(skb))
586 return 0;
587
588 ret = skb_cow_head(skb, 0);
589 if (ret)
590 return ret;
591
592 l3.hdr = skb_network_header(skb);
593 l4.hdr = skb_transport_header(skb);
594
595 /* Software should clear the IPv4's checksum field when tso is
596 * needed.
597 */
598 if (l3.v4->version == 4)
599 l3.v4->check = 0;
600
601 /* tunnel packet.*/
602 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
603 SKB_GSO_GRE_CSUM |
604 SKB_GSO_UDP_TUNNEL |
605 SKB_GSO_UDP_TUNNEL_CSUM)) {
606 if ((!(skb_shinfo(skb)->gso_type &
607 SKB_GSO_PARTIAL)) &&
608 (skb_shinfo(skb)->gso_type &
609 SKB_GSO_UDP_TUNNEL_CSUM)) {
610 /* Software should clear the udp's checksum
611 * field when tso is needed.
612 */
613 l4.udp->check = 0;
614 }
615 /* reset l3&l4 pointers from outer to inner headers */
616 l3.hdr = skb_inner_network_header(skb);
617 l4.hdr = skb_inner_transport_header(skb);
618
619 /* Software should clear the IPv4's checksum field when
620 * tso is needed.
621 */
622 if (l3.v4->version == 4)
623 l3.v4->check = 0;
624 }
625
626 /* normal or tunnel packet*/
627 l4_offset = l4.hdr - skb->data;
628 hdr_len = (l4.tcp->doff * 4) + l4_offset;
629
630 /* remove payload length from inner pseudo checksum when tso*/
631 l4_paylen = skb->len - l4_offset;
632 csum_replace_by_diff(&l4.tcp->check,
633 (__force __wsum)htonl(l4_paylen));
634
635 /* find the txbd field values */
636 *paylen = skb->len - hdr_len;
e4e87715
PL		 637 hnae3_set_bit(*type_cs_vlan_tso,
638 HNS3_TXD_TSO_B, 1);
76ad4f0e
S		 639
640 /* get MSS for TSO */
641 *mss = skb_shinfo(skb)->gso_size;
642
643 return 0;
644}
645
1898d4e4
S		 646static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
647 u8 *il4_proto)
76ad4f0e
S		 648{
649 union {
650 struct iphdr *v4;
651 struct ipv6hdr *v6;
652 unsigned char *hdr;
653 } l3;
654 unsigned char *l4_hdr;
655 unsigned char *exthdr;
656 u8 l4_proto_tmp;
657 __be16 frag_off;
658
659 /* find outer header point */
660 l3.hdr = skb_network_header(skb);
35f58fd7 661 l4_hdr = skb_transport_header(skb);
76ad4f0e
S		 662
663 if (skb->protocol == htons(ETH_P_IPV6)) {
664 exthdr = l3.hdr + sizeof(*l3.v6);
665 l4_proto_tmp = l3.v6->nexthdr;
666 if (l4_hdr != exthdr)
667 ipv6_skip_exthdr(skb, exthdr - skb->data,
668 &l4_proto_tmp, &frag_off);
669 } else if (skb->protocol == htons(ETH_P_IP)) {
670 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 671 } else {
672 return -EINVAL;
76ad4f0e
S		 673 }
674
675 *ol4_proto = l4_proto_tmp;
676
677 /* tunnel packet */
678 if (!skb->encapsulation) {
679 *il4_proto = 0;
1898d4e4 680 return 0;
76ad4f0e
S		 681 }
682
683 /* find inner header point */
684 l3.hdr = skb_inner_network_header(skb);
685 l4_hdr = skb_inner_transport_header(skb);
686
687 if (l3.v6->version == 6) {
688 exthdr = l3.hdr + sizeof(*l3.v6);
689 l4_proto_tmp = l3.v6->nexthdr;
690 if (l4_hdr != exthdr)
691 ipv6_skip_exthdr(skb, exthdr - skb->data,
692 &l4_proto_tmp, &frag_off);
693 } else if (l3.v4->version == 4) {
694 l4_proto_tmp = l3.v4->protocol;
695 }
696
697 *il4_proto = l4_proto_tmp;
1898d4e4
S		 698
699 return 0;
76ad4f0e
S		 700}
701
702static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
703 u8 il4_proto, u32 *type_cs_vlan_tso,
704 u32 *ol_type_vlan_len_msec)
705{
706 union {
707 struct iphdr *v4;
708 struct ipv6hdr *v6;
709 unsigned char *hdr;
710 } l3;
711 union {
712 struct tcphdr *tcp;
713 struct udphdr *udp;
714 struct gre_base_hdr *gre;
715 unsigned char *hdr;
716 } l4;
717 unsigned char *l2_hdr;
718 u8 l4_proto = ol4_proto;
719 u32 ol2_len;
720 u32 ol3_len;
721 u32 ol4_len;
722 u32 l2_len;
723 u32 l3_len;
724
725 l3.hdr = skb_network_header(skb);
726 l4.hdr = skb_transport_header(skb);
727
728 /* compute L2 header size for normal packet, defined in 2 Bytes */
729 l2_len = l3.hdr - skb->data;
e4e87715
PL		 730 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
731 HNS3_TXD_L2LEN_S, l2_len >> 1);
76ad4f0e
S		 732
733 /* tunnel packet*/
734 if (skb->encapsulation) {
735 /* compute OL2 header size, defined in 2 Bytes */
736 ol2_len = l2_len;
e4e87715
PL		 737 hnae3_set_field(*ol_type_vlan_len_msec,
738 HNS3_TXD_L2LEN_M,
739 HNS3_TXD_L2LEN_S, ol2_len >> 1);
76ad4f0e
S		 740
741 /* compute OL3 header size, defined in 4 Bytes */
742 ol3_len = l4.hdr - l3.hdr;
e4e87715
PL		 743 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
744 HNS3_TXD_L3LEN_S, ol3_len >> 2);
76ad4f0e
S		 745
746 /* MAC in UDP, MAC in GRE (0x6558)*/
747 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
748 /* switch MAC header ptr from outer to inner header.*/
749 l2_hdr = skb_inner_mac_header(skb);
750
751 /* compute OL4 header size, defined in 4 Bytes. */
752 ol4_len = l2_hdr - l4.hdr;
e4e87715
PL		 753 hnae3_set_field(*ol_type_vlan_len_msec,
754 HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
755 ol4_len >> 2);
76ad4f0e
S		 756
757 /* switch IP header ptr from outer to inner header */
758 l3.hdr = skb_inner_network_header(skb);
759
760 /* compute inner l2 header size, defined in 2 Bytes. */
761 l2_len = l3.hdr - l2_hdr;
e4e87715
PL		 762 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
763 HNS3_TXD_L2LEN_S, l2_len >> 1);
76ad4f0e
S		 764 } else {
765 /* skb packet types not supported by hardware,
766 * txbd len fild doesn't be filled.
767 */
768 return;
769 }
770
771 /* switch L4 header pointer from outer to inner */
772 l4.hdr = skb_inner_transport_header(skb);
773
774 l4_proto = il4_proto;
775 }
776
777 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
778 l3_len = l4.hdr - l3.hdr;
e4e87715
PL		 779 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
780 HNS3_TXD_L3LEN_S, l3_len >> 2);
76ad4f0e
S		 781
782 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
783 switch (l4_proto) {
784 case IPPROTO_TCP:
e4e87715
PL		 785 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
786 HNS3_TXD_L4LEN_S, l4.tcp->doff);
76ad4f0e
S		 787 break;
788 case IPPROTO_SCTP:
e4e87715
PL		 789 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
790 HNS3_TXD_L4LEN_S,
791 (sizeof(struct sctphdr) >> 2));
76ad4f0e
S		 792 break;
793 case IPPROTO_UDP:
e4e87715
PL		 794 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
795 HNS3_TXD_L4LEN_S,
796 (sizeof(struct udphdr) >> 2));
76ad4f0e
S		 797 break;
798 default:
799 /* skb packet types not supported by hardware,
800 * txbd len fild doesn't be filled.
801 */
802 return;
803 }
804}
805
3db084d2
YL		 806/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
807 * and it is udp packet, which has a dest port as the IANA assigned.
808 * the hardware is expected to do the checksum offload, but the
809 * hardware will not do the checksum offload when udp dest port is
810 * 4789.
811 */
812static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
813{
814#define IANA_VXLAN_PORT 4789
815 union {
816 struct tcphdr *tcp;
817 struct udphdr *udp;
818 struct gre_base_hdr *gre;
819 unsigned char *hdr;
820 } l4;
821
822 l4.hdr = skb_transport_header(skb);
823
824 if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
825 return false;
826
827 skb_checksum_help(skb);
828
829 return true;
830}
831
76ad4f0e
S		 832static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
833 u8 il4_proto, u32 *type_cs_vlan_tso,
834 u32 *ol_type_vlan_len_msec)
835{
836 union {
837 struct iphdr *v4;
838 struct ipv6hdr *v6;
839 unsigned char *hdr;
840 } l3;
841 u32 l4_proto = ol4_proto;
842
843 l3.hdr = skb_network_header(skb);
844
845 /* define OL3 type and tunnel type(OL4).*/
846 if (skb->encapsulation) {
847 /* define outer network header type.*/
848 if (skb->protocol == htons(ETH_P_IP)) {
849 if (skb_is_gso(skb))
e4e87715
PL		 850 hnae3_set_field(*ol_type_vlan_len_msec,
851 HNS3_TXD_OL3T_M,
852 HNS3_TXD_OL3T_S,
853 HNS3_OL3T_IPV4_CSUM);
76ad4f0e 854 else
e4e87715
PL		 855 hnae3_set_field(*ol_type_vlan_len_msec,
856 HNS3_TXD_OL3T_M,
857 HNS3_TXD_OL3T_S,
858 HNS3_OL3T_IPV4_NO_CSUM);
76ad4f0e
S		 859
860 } else if (skb->protocol == htons(ETH_P_IPV6)) {
e4e87715
PL		 861 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
862 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
76ad4f0e
S		 863 }
864
865 /* define tunnel type(OL4).*/
866 switch (l4_proto) {
867 case IPPROTO_UDP:
e4e87715
PL		 868 hnae3_set_field(*ol_type_vlan_len_msec,
869 HNS3_TXD_TUNTYPE_M,
870 HNS3_TXD_TUNTYPE_S,
871 HNS3_TUN_MAC_IN_UDP);
76ad4f0e
S		 872 break;
873 case IPPROTO_GRE:
e4e87715
PL		 874 hnae3_set_field(*ol_type_vlan_len_msec,
875 HNS3_TXD_TUNTYPE_M,
876 HNS3_TXD_TUNTYPE_S,
877 HNS3_TUN_NVGRE);
76ad4f0e
S		 878 break;
879 default:
880 /* drop the skb tunnel packet if hardware don't support,
881 * because hardware can't calculate csum when TSO.
882 */
883 if (skb_is_gso(skb))
884 return -EDOM;
885
886 /* the stack computes the IP header already,
887 * driver calculate l4 checksum when not TSO.
888 */
889 skb_checksum_help(skb);
890 return 0;
891 }
892
893 l3.hdr = skb_inner_network_header(skb);
894 l4_proto = il4_proto;
895 }
896
897 if (l3.v4->version == 4) {
e4e87715
PL		 898 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
899 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
76ad4f0e
S		 900
901 /* the stack computes the IP header already, the only time we
902 * need the hardware to recompute it is in the case of TSO.
903 */
904 if (skb_is_gso(skb))
e4e87715 905 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
76ad4f0e 906 } else if (l3.v6->version == 6) {
e4e87715
PL		 907 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
908 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
76ad4f0e
S		 909 }
910
911 switch (l4_proto) {
912 case IPPROTO_TCP:
5c897197 913 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
e4e87715
PL		 914 hnae3_set_field(*type_cs_vlan_tso,
915 HNS3_TXD_L4T_M,
916 HNS3_TXD_L4T_S,
917 HNS3_L4T_TCP);
76ad4f0e
S		 918 break;
919 case IPPROTO_UDP:
3db084d2
YL		 920 if (hns3_tunnel_csum_bug(skb))
921 break;
922
5c897197 923 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
e4e87715
PL		 924 hnae3_set_field(*type_cs_vlan_tso,
925 HNS3_TXD_L4T_M,
926 HNS3_TXD_L4T_S,
927 HNS3_L4T_UDP);
76ad4f0e
S		 928 break;
929 case IPPROTO_SCTP:
5c897197 930 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
e4e87715
PL		 931 hnae3_set_field(*type_cs_vlan_tso,
932 HNS3_TXD_L4T_M,
933 HNS3_TXD_L4T_S,
934 HNS3_L4T_SCTP);
76ad4f0e
S		 935 break;
936 default:
937 /* drop the skb tunnel packet if hardware don't support,
938 * because hardware can't calculate csum when TSO.
939 */
940 if (skb_is_gso(skb))
941 return -EDOM;
942
943 /* the stack computes the IP header already,
944 * driver calculate l4 checksum when not TSO.
945 */
946 skb_checksum_help(skb);
947 return 0;
948 }
949
950 return 0;
951}
952
953static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
954{
955 /* Config bd buffer end */
e4e87715
PL		 956 hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
957 HNS3_TXD_BDTYPE_S, 0);
958 hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
959 hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
960 hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 961}
962
9699cffe
PL		 963static int hns3_fill_desc_vtags(struct sk_buff *skb,
964 struct hns3_enet_ring *tx_ring,
965 u32 *inner_vlan_flag,
966 u32 *out_vlan_flag,
967 u16 *inner_vtag,
968 u16 *out_vtag)
969{
970#define HNS3_TX_VLAN_PRIO_SHIFT 13
971
972 if (skb->protocol == htons(ETH_P_8021Q) &&
973 !(tx_ring->tqp->handle->kinfo.netdev->features &
974 NETIF_F_HW_VLAN_CTAG_TX)) {
975 /* When HW VLAN acceleration is turned off, and the stack
976 * sets the protocol to 802.1q, the driver just need to
977 * set the protocol to the encapsulated ethertype.
978 */
979 skb->protocol = vlan_get_protocol(skb);
980 return 0;
981 }
982
983 if (skb_vlan_tag_present(skb)) {
984 u16 vlan_tag;
985
986 vlan_tag = skb_vlan_tag_get(skb);
987 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
988
989 /* Based on hw strategy, use out_vtag in two layer tag case,
990 * and use inner_vtag in one tag case.
991 */
992 if (skb->protocol == htons(ETH_P_8021Q)) {
e4e87715 993 hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
9699cffe
PL		 994 *out_vtag = vlan_tag;
995 } else {
e4e87715 996 hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
9699cffe
PL		 997 *inner_vtag = vlan_tag;
998 }
999 } else if (skb->protocol == htons(ETH_P_8021Q)) {
1000 struct vlan_ethhdr *vhdr;
1001 int rc;
1002
1003 rc = skb_cow_head(skb, 0);
1004 if (rc < 0)
1005 return rc;
1006 vhdr = (struct vlan_ethhdr *)skb->data;
1007 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
1008 << HNS3_TX_VLAN_PRIO_SHIFT);
1009 }
1010
1011 skb->protocol = vlan_get_protocol(skb);
1012 return 0;
1013}
1014
76ad4f0e 1015static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
5188f218 1016 int size, int frag_end, enum hns_desc_type type)
76ad4f0e
S		 1017{
1018 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1019 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
5188f218 1020 struct device *dev = ring_to_dev(ring);
76ad4f0e
S		 1021 u32 ol_type_vlan_len_msec = 0;
1022 u16 bdtp_fe_sc_vld_ra_ri = 0;
5188f218 1023 struct skb_frag_struct *frag;
1e8a7977 1024 unsigned int frag_buf_num;
76ad4f0e
S		 1025 u32 type_cs_vlan_tso = 0;
1026 struct sk_buff *skb;
9699cffe
PL		 1027 u16 inner_vtag = 0;
1028 u16 out_vtag = 0;
1e8a7977
FL		 1029 unsigned int k;
1030 int sizeoflast;
76ad4f0e 1031 u32 paylen = 0;
5188f218 1032 dma_addr_t dma;
76ad4f0e 1033 u16 mss = 0;
76ad4f0e
S		 1034 u8 ol4_proto;
1035 u8 il4_proto;
1036 int ret;
1037
76ad4f0e
S		 1038 if (type == DESC_TYPE_SKB) {
1039 skb = (struct sk_buff *)priv;
a90bb9a5 1040 paylen = skb->len;
76ad4f0e 1041
9699cffe
PL		 1042 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1043 &ol_type_vlan_len_msec,
1044 &inner_vtag, &out_vtag);
1045 if (unlikely(ret))
1046 return ret;
1047
76ad4f0e
S		 1048 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1049 skb_reset_mac_len(skb);
76ad4f0e 1050
1898d4e4
S		 1051 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1052 if (ret)
1053 return ret;
76ad4f0e
S		 1054 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1055 &type_cs_vlan_tso,
1056 &ol_type_vlan_len_msec);
1057 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1058 &type_cs_vlan_tso,
1059 &ol_type_vlan_len_msec);
1060 if (ret)
1061 return ret;
1062
1063 ret = hns3_set_tso(skb, &paylen, &mss,
1064 &type_cs_vlan_tso);
1065 if (ret)
1066 return ret;
1067 }
1068
1069 /* Set txbd */
1070 desc->tx.ol_type_vlan_len_msec =
1071 cpu_to_le32(ol_type_vlan_len_msec);
1072 desc->tx.type_cs_vlan_tso_len =
1073 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 1074 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e 1075 desc->tx.mss = cpu_to_le16(mss);
9699cffe
PL		 1076 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1077 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
5188f218
PL		 1078
1079 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1080 } else {
1081 frag = (struct skb_frag_struct *)priv;
1082 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1083 }
1084
1085 if (dma_mapping_error(ring->dev, dma)) {
1086 ring->stats.sw_err_cnt++;
1087 return -ENOMEM;
76ad4f0e
S		 1088 }
1089
bcdb12b7
FL		 1090 desc_cb->length = size;
1091
1e8a7977
FL		 1092 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1093 sizeoflast = size % HNS3_MAX_BD_SIZE;
1094 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1095
1096 /* When frag size is bigger than hardware limit, split this frag */
1097 for (k = 0; k < frag_buf_num; k++) {
1098 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1099 desc_cb->priv = priv;
1e8a7977
FL		 1100 desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1101 desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1102 DESC_TYPE_SKB : DESC_TYPE_PAGE;
1103
1104 /* now, fill the descriptor */
1105 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
bcdb12b7
FL		 1106 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1107 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1e8a7977
FL		 1108 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1109 frag_end && (k == frag_buf_num - 1) ?
1110 1 : 0);
1111 desc->tx.bdtp_fe_sc_vld_ra_ri =
1112 cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1113
1114 /* move ring pointer to next.*/
1115 ring_ptr_move_fw(ring, next_to_use);
1116
1117 desc_cb = &ring->desc_cb[ring->next_to_use];
1118 desc = &ring->desc[ring->next_to_use];
1119 }
76ad4f0e
S		 1120
1121 return 0;
1122}
1123
76ad4f0e
S		 1124static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1125 struct hns3_enet_ring *ring)
1126{
1127 struct sk_buff *skb = *out_skb;
1128 struct skb_frag_struct *frag;
1129 int bdnum_for_frag;
1130 int frag_num;
1131 int buf_num;
1132 int size;
1133 int i;
1134
1135 size = skb_headlen(skb);
1136 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1137
1138 frag_num = skb_shinfo(skb)->nr_frags;
1139 for (i = 0; i < frag_num; i++) {
1140 frag = &skb_shinfo(skb)->frags[i];
1141 size = skb_frag_size(frag);
1142 bdnum_for_frag =
1143 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1144 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1145 return -ENOMEM;
1146
1147 buf_num += bdnum_for_frag;
1148 }
1149
1150 if (buf_num > ring_space(ring))
1151 return -EBUSY;
1152
1153 *bnum = buf_num;
1154 return 0;
1155}
1156
1157static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1158 struct hns3_enet_ring *ring)
1159{
1160 struct sk_buff *skb = *out_skb;
1161 int buf_num;
1162
1163 /* No. of segments (plus a header) */
1164 buf_num = skb_shinfo(skb)->nr_frags + 1;
1165
932d1252 1166 if (unlikely(ring_space(ring) < buf_num))
76ad4f0e
S		 1167 return -EBUSY;
1168
1169 *bnum = buf_num;
1170
1171 return 0;
1172}
1173
ba3f808f 1174static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
76ad4f0e
S		 1175{
1176 struct device *dev = ring_to_dev(ring);
1177 unsigned int i;
1178
1179 for (i = 0; i < ring->desc_num; i++) {
1180 /* check if this is where we started */
1181 if (ring->next_to_use == next_to_use_orig)
1182 break;
1183
1184 /* unmap the descriptor dma address */
1185 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1186 dma_unmap_single(dev,
1187 ring->desc_cb[ring->next_to_use].dma,
1188 ring->desc_cb[ring->next_to_use].length,
1189 DMA_TO_DEVICE);
bcdb12b7 1190 else if (ring->desc_cb[ring->next_to_use].length)
76ad4f0e
S		 1191 dma_unmap_page(dev,
1192 ring->desc_cb[ring->next_to_use].dma,
1193 ring->desc_cb[ring->next_to_use].length,
1194 DMA_TO_DEVICE);
1195
bcdb12b7
FL		 1196 ring->desc_cb[ring->next_to_use].length = 0;
1197
76ad4f0e
S		 1198 /* rollback one */
1199 ring_ptr_move_bw(ring, next_to_use);
1200 }
1201}
1202
d43e5aca 1203netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 1204{
1205 struct hns3_nic_priv *priv = netdev_priv(netdev);
1206 struct hns3_nic_ring_data *ring_data =
1207 &tx_ring_data(priv, skb->queue_mapping);
1208 struct hns3_enet_ring *ring = ring_data->ring;
76ad4f0e
S		 1209 struct netdev_queue *dev_queue;
1210 struct skb_frag_struct *frag;
1211 int next_to_use_head;
1212 int next_to_use_frag;
76ad4f0e
S		 1213 int buf_num;
1214 int seg_num;
1215 int size;
1216 int ret;
1217 int i;
1218
1219 /* Prefetch the data used later */
1220 prefetch(skb->data);
1221
1222 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1223 case -EBUSY:
1224 u64_stats_update_begin(&ring->syncp);
1225 ring->stats.tx_busy++;
1226 u64_stats_update_end(&ring->syncp);
1227
1228 goto out_net_tx_busy;
1229 case -ENOMEM:
1230 u64_stats_update_begin(&ring->syncp);
1231 ring->stats.sw_err_cnt++;
1232 u64_stats_update_end(&ring->syncp);
1233 netdev_err(netdev, "no memory to xmit!\n");
1234
1235 goto out_err_tx_ok;
1236 default:
1237 break;
1238 }
1239
1240 /* No. of segments (plus a header) */
1241 seg_num = skb_shinfo(skb)->nr_frags + 1;
1242 /* Fill the first part */
1243 size = skb_headlen(skb);
1244
1245 next_to_use_head = ring->next_to_use;
1246
5188f218
PL		 1247 ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1248 DESC_TYPE_SKB);
76ad4f0e 1249 if (ret)
ba3f808f 1250 goto head_fill_err;
76ad4f0e
S		 1251
1252 next_to_use_frag = ring->next_to_use;
1253 /* Fill the fragments */
1254 for (i = 1; i < seg_num; i++) {
1255 frag = &skb_shinfo(skb)->frags[i - 1];
1256 size = skb_frag_size(frag);
5188f218
PL		 1257
1258 ret = priv->ops.fill_desc(ring, frag, size,
1259 seg_num - 1 == i ? 1 : 0,
1260 DESC_TYPE_PAGE);
76ad4f0e
S		 1261
1262 if (ret)
ba3f808f 1263 goto frag_fill_err;
76ad4f0e
S		 1264 }
1265
1266 /* Complete translate all packets */
1267 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1268 netdev_tx_sent_queue(dev_queue, skb->len);
1269
1270 wmb(); /* Commit all data before submit */
1271
e4e87715 1272 hnae3_queue_xmit(ring->tqp, buf_num);
76ad4f0e
S		 1273
1274 return NETDEV_TX_OK;
1275
ba3f808f
FL		 1276frag_fill_err:
1277 hns3_clear_desc(ring, next_to_use_frag);
76ad4f0e 1278
ba3f808f
FL		 1279head_fill_err:
1280 hns3_clear_desc(ring, next_to_use_head);
76ad4f0e
S		 1281
1282out_err_tx_ok:
1283 dev_kfree_skb_any(skb);
1284 return NETDEV_TX_OK;
1285
1286out_net_tx_busy:
1287 netif_stop_subqueue(netdev, ring_data->queue_index);
1288 smp_mb(); /* Commit all data before submit */
1289
1290 return NETDEV_TX_BUSY;
1291}
1292
1293static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1294{
9780cb97 1295 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1296 struct sockaddr *mac_addr = p;
1297 int ret;
1298
1299 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1300 return -EADDRNOTAVAIL;
1301
5ec2a51e
JS		 1302 if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1303 netdev_info(netdev, "already using mac address %pM\n",
1304 mac_addr->sa_data);
1305 return 0;
1306 }
1307
59098055 1308 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
76ad4f0e
S		 1309 if (ret) {
1310 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1311 return ret;
1312 }
1313
1314 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1315
1316 return 0;
1317}
1318
26483246
XW		 1319static int hns3_nic_do_ioctl(struct net_device *netdev,
1320 struct ifreq *ifr, int cmd)
1321{
1322 struct hnae3_handle *h = hns3_get_handle(netdev);
1323
1324 if (!netif_running(netdev))
1325 return -EINVAL;
1326
1327 if (!h->ae_algo->ops->do_ioctl)
1328 return -EOPNOTSUPP;
1329
1330 return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1331}
1332
76ad4f0e
S		 1333static int hns3_nic_set_features(struct net_device *netdev,
1334 netdev_features_t features)
1335{
181d454b 1336 netdev_features_t changed = netdev->features ^ features;
76ad4f0e 1337 struct hns3_nic_priv *priv = netdev_priv(netdev);
052ece6d 1338 struct hnae3_handle *h = priv->ae_handle;
052ece6d 1339 int ret;
76ad4f0e 1340
181d454b 1341 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
0bbbf15d 1342 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
181d454b 1343 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
0bbbf15d 1344 else
181d454b 1345 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
76ad4f0e
S		 1346 }
1347
bd368416
JS		 1348 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1349 h->ae_algo->ops->enable_vlan_filter) {
181d454b
JS		 1350 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1351 h->ae_algo->ops->enable_vlan_filter(h, true);
1352 else
1353 h->ae_algo->ops->enable_vlan_filter(h, false);
1354 }
391b5e93 1355
bd368416
JS		 1356 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1357 h->ae_algo->ops->enable_hw_strip_rxvtag) {
052ece6d
PL		 1358 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1359 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1360 else
1361 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1362
1363 if (ret)
1364 return ret;
1365 }
1366
c17852a8
JS		 1367 if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1368 if (features & NETIF_F_NTUPLE)
1369 h->ae_algo->ops->enable_fd(h, true);
1370 else
1371 h->ae_algo->ops->enable_fd(h, false);
1372 }
1373
76ad4f0e
S		 1374 netdev->features = features;
1375 return 0;
1376}
1377
6c88d9d7
PL		 1378static void hns3_nic_get_stats64(struct net_device *netdev,
1379 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 1380{
1381 struct hns3_nic_priv *priv = netdev_priv(netdev);
1382 int queue_num = priv->ae_handle->kinfo.num_tqps;
c5f65480 1383 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e
S		 1384 struct hns3_enet_ring *ring;
1385 unsigned int start;
1386 unsigned int idx;
1387 u64 tx_bytes = 0;
1388 u64 rx_bytes = 0;
1389 u64 tx_pkts = 0;
1390 u64 rx_pkts = 0;
d2a5dca8
JS		 1391 u64 tx_drop = 0;
1392 u64 rx_drop = 0;
76ad4f0e 1393
b875cc37
JS		 1394 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1395 return;
1396
c5f65480
JS		 1397 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1398
76ad4f0e
S		 1399 for (idx = 0; idx < queue_num; idx++) {
1400 /* fetch the tx stats */
1401 ring = priv->ring_data[idx].ring;
1402 do {
d36d36ce 1403 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1404 tx_bytes += ring->stats.tx_bytes;
1405 tx_pkts += ring->stats.tx_pkts;
d2a5dca8
JS		 1406 tx_drop += ring->stats.tx_busy;
1407 tx_drop += ring->stats.sw_err_cnt;
76ad4f0e
S		 1408 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1409
1410 /* fetch the rx stats */
1411 ring = priv->ring_data[idx + queue_num].ring;
1412 do {
d36d36ce 1413 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1414 rx_bytes += ring->stats.rx_bytes;
1415 rx_pkts += ring->stats.rx_pkts;
d2a5dca8
JS		 1416 rx_drop += ring->stats.non_vld_descs;
1417 rx_drop += ring->stats.err_pkt_len;
1418 rx_drop += ring->stats.l2_err;
76ad4f0e
S		 1419 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1420 }
1421
1422 stats->tx_bytes = tx_bytes;
1423 stats->tx_packets = tx_pkts;
1424 stats->rx_bytes = rx_bytes;
1425 stats->rx_packets = rx_pkts;
1426
1427 stats->rx_errors = netdev->stats.rx_errors;
1428 stats->multicast = netdev->stats.multicast;
1429 stats->rx_length_errors = netdev->stats.rx_length_errors;
1430 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1431 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1432
1433 stats->tx_errors = netdev->stats.tx_errors;
d2a5dca8
JS		 1434 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1435 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
76ad4f0e
S		 1436 stats->collisions = netdev->stats.collisions;
1437 stats->rx_over_errors = netdev->stats.rx_over_errors;
1438 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1439 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1440 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1441 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1442 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1443 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1444 stats->tx_window_errors = netdev->stats.tx_window_errors;
1445 stats->rx_compressed = netdev->stats.rx_compressed;
1446 stats->tx_compressed = netdev->stats.tx_compressed;
1447}
1448
30d240df 1449static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1450{
30d240df 1451 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1452 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1453 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1454 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1455 u8 tc = mqprio_qopt->qopt.num_tc;
1456 u16 mode = mqprio_qopt->mode;
1457 u8 hw = mqprio_qopt->qopt.hw;
1458 bool if_running;
76ad4f0e
S		 1459 int ret;
1460
30d240df
YL		 1461 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1462 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1463 return -EOPNOTSUPP;
1464
76ad4f0e
S		 1465 if (tc > HNAE3_MAX_TC)
1466 return -EINVAL;
1467
76ad4f0e
S		 1468 if (!netdev)
1469 return -EINVAL;
1470
30d240df
YL		 1471 if_running = netif_running(netdev);
1472 if (if_running) {
1473 hns3_nic_net_stop(netdev);
1474 msleep(100);
76ad4f0e
S		 1475 }
1476
30d240df
YL		 1477 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1478 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1479 if (ret)
30d240df
YL		 1480 goto out;
1481
30d240df
YL		 1482 ret = hns3_nic_set_real_num_queue(netdev);
1483
1484out:
1485 if (if_running)
1486 hns3_nic_net_open(netdev);
1487
1488 return ret;
76ad4f0e
S		 1489}
1490
2572ac53 1491static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1492 void *type_data)
76ad4f0e 1493{
575ed7d3 1494 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1495 return -EOPNOTSUPP;
76ad4f0e 1496
30d240df 1497 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1498}
1499
1500static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1501 __be16 proto, u16 vid)
1502{
9780cb97 1503 struct hnae3_handle *h = hns3_get_handle(netdev);
681ec399 1504 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1505 int ret = -EIO;
1506
1507 if (h->ae_algo->ops->set_vlan_filter)
1508 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1509
681ec399
YL		 1510 if (!ret)
1511 set_bit(vid, priv->active_vlans);
1512
76ad4f0e
S		 1513 return ret;
1514}
1515
1516static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1517 __be16 proto, u16 vid)
1518{
9780cb97 1519 struct hnae3_handle *h = hns3_get_handle(netdev);
681ec399 1520 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1521 int ret = -EIO;
1522
1523 if (h->ae_algo->ops->set_vlan_filter)
1524 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1525
681ec399
YL		 1526 if (!ret)
1527 clear_bit(vid, priv->active_vlans);
1528
76ad4f0e
S		 1529 return ret;
1530}
1531
7fa6be4f 1532static int hns3_restore_vlan(struct net_device *netdev)
681ec399
YL		 1533{
1534 struct hns3_nic_priv *priv = netdev_priv(netdev);
7fa6be4f 1535 int ret = 0;
681ec399 1536 u16 vid;
681ec399
YL		 1537
1538 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1539 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
7fa6be4f
HT		 1540 if (ret) {
1541 netdev_err(netdev, "Restore vlan: %d filter, ret:%d\n",
1542 vid, ret);
1543 return ret;
1544 }
681ec399 1545 }
7fa6be4f
HT		 1546
1547 return ret;
681ec399
YL		 1548}
1549
76ad4f0e
S		 1550static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1551 u8 qos, __be16 vlan_proto)
1552{
9780cb97 1553 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1554 int ret = -EIO;
1555
1556 if (h->ae_algo->ops->set_vf_vlan_filter)
1557 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1558 qos, vlan_proto);
1559
1560 return ret;
1561}
1562
a8e8b7ff
S		 1563static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1564{
9780cb97 1565 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1566 bool if_running = netif_running(netdev);
1567 int ret;
1568
1569 if (!h->ae_algo->ops->set_mtu)
1570 return -EOPNOTSUPP;
1571
1572 /* if this was called with netdev up then bring netdevice down */
1573 if (if_running) {
1574 (void)hns3_nic_net_stop(netdev);
1575 msleep(100);
1576 }
1577
1578 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
93d8daf4 1579 if (ret)
a8e8b7ff
S		 1580 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1581 ret);
93d8daf4
YL		 1582 else
1583 netdev->mtu = new_mtu;
5bad95a1 1584
a8e8b7ff
S		 1585 /* if the netdev was running earlier, bring it up again */
1586 if (if_running && hns3_nic_net_open(netdev))
1587 ret = -EINVAL;
1588
1589 return ret;
1590}
1591
f8fa222c
L		 1592static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1593{
1594 struct hns3_nic_priv *priv = netdev_priv(ndev);
1595 struct hns3_enet_ring *tx_ring = NULL;
1596 int timeout_queue = 0;
1597 int hw_head, hw_tail;
1598 int i;
1599
1600 /* Find the stopped queue the same way the stack does */
1601 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1602 struct netdev_queue *q;
1603 unsigned long trans_start;
1604
1605 q = netdev_get_tx_queue(ndev, i);
1606 trans_start = q->trans_start;
1607 if (netif_xmit_stopped(q) &&
1608 time_after(jiffies,
1609 (trans_start + ndev->watchdog_timeo))) {
1610 timeout_queue = i;
1611 break;
1612 }
1613 }
1614
1615 if (i == ndev->num_tx_queues) {
1616 netdev_info(ndev,
1617 "no netdev TX timeout queue found, timeout count: %llu\n",
1618 priv->tx_timeout_count);
1619 return false;
1620 }
1621
1622 tx_ring = priv->ring_data[timeout_queue].ring;
1623
1624 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1625 HNS3_RING_TX_RING_HEAD_REG);
1626 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1627 HNS3_RING_TX_RING_TAIL_REG);
1628 netdev_info(ndev,
1629 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1630 priv->tx_timeout_count,
1631 timeout_queue,
1632 tx_ring->next_to_use,
1633 tx_ring->next_to_clean,
1634 hw_head,
1635 hw_tail,
1636 readl(tx_ring->tqp_vector->mask_addr));
1637
1638 return true;
1639}
1640
1641static void hns3_nic_net_timeout(struct net_device *ndev)
1642{
1643 struct hns3_nic_priv *priv = netdev_priv(ndev);
f8fa222c
L		 1644 struct hnae3_handle *h = priv->ae_handle;
1645
1646 if (!hns3_get_tx_timeo_queue_info(ndev))
1647 return;
1648
1649 priv->tx_timeout_count++;
1650
0742ed7c
HT		 1651 /* request the reset, and let the hclge to determine
1652 * which reset level should be done
1653 */
f8fa222c 1654 if (h->ae_algo->ops->reset_event)
6ae4e733 1655 h->ae_algo->ops->reset_event(h->pdev, h);
f8fa222c
L		 1656}
1657
76ad4f0e
S		 1658static const struct net_device_ops hns3_nic_netdev_ops = {
1659 .ndo_open = hns3_nic_net_open,
1660 .ndo_stop = hns3_nic_net_stop,
1661 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1662 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1663 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
26483246 1664 .ndo_do_ioctl = hns3_nic_do_ioctl,
a8e8b7ff 1665 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1666 .ndo_set_features = hns3_nic_set_features,
1667 .ndo_get_stats64 = hns3_nic_get_stats64,
1668 .ndo_setup_tc = hns3_nic_setup_tc,
1669 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
76ad4f0e
S		 1670 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1671 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1672 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1673};
1674
2312e050
FL		 1675static bool hns3_is_phys_func(struct pci_dev *pdev)
1676{
1677 u32 dev_id = pdev->device;
1678
1679 switch (dev_id) {
1680 case HNAE3_DEV_ID_GE:
1681 case HNAE3_DEV_ID_25GE:
1682 case HNAE3_DEV_ID_25GE_RDMA:
1683 case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1684 case HNAE3_DEV_ID_50GE_RDMA:
1685 case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1686 case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1687 return true;
1688 case HNAE3_DEV_ID_100G_VF:
1689 case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1690 return false;
1691 default:
1692 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1693 dev_id);
1694 }
1695
1696 return false;
1697}
1698
2312e050
FL		 1699static void hns3_disable_sriov(struct pci_dev *pdev)
1700{
1701 /* If our VFs are assigned we cannot shut down SR-IOV
1702 * without causing issues, so just leave the hardware
1703 * available but disabled
1704 */
1705 if (pci_vfs_assigned(pdev)) {
1706 dev_warn(&pdev->dev,
1707 "disabling driver while VFs are assigned\n");
1708 return;
1709 }
1710
1711 pci_disable_sriov(pdev);
1712}
1713
d695964d
JS		 1714static void hns3_get_dev_capability(struct pci_dev *pdev,
1715 struct hnae3_ae_dev *ae_dev)
1716{
b26a6fea 1717 if (pdev->revision >= 0x21) {
d695964d 1718 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
b26a6fea
PL		 1719 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_GRO_B, 1);
1720 }
d695964d
JS		 1721}
1722
76ad4f0e
S		 1723/* hns3_probe - Device initialization routine
1724 * @pdev: PCI device information struct
1725 * @ent: entry in hns3_pci_tbl
1726 *
1727 * hns3_probe initializes a PF identified by a pci_dev structure.
1728 * The OS initialization, configuring of the PF private structure,
1729 * and a hardware reset occur.
1730 *
1731 * Returns 0 on success, negative on failure
1732 */
1733static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1734{
1735 struct hnae3_ae_dev *ae_dev;
1736 int ret;
1737
1738 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1739 GFP_KERNEL);
1740 if (!ae_dev) {
1741 ret = -ENOMEM;
1742 return ret;
1743 }
1744
1745 ae_dev->pdev = pdev;
e92a0843 1746 ae_dev->flag = ent->driver_data;
76ad4f0e 1747 ae_dev->dev_type = HNAE3_DEV_KNIC;
6871af29 1748 ae_dev->reset_type = HNAE3_NONE_RESET;
d695964d 1749 hns3_get_dev_capability(pdev, ae_dev);
76ad4f0e
S		 1750 pci_set_drvdata(pdev, ae_dev);
1751
50fbc237 1752 hnae3_register_ae_dev(ae_dev);
2312e050 1753
2312e050 1754 return 0;
76ad4f0e
S		 1755}
1756
1757/* hns3_remove - Device removal routine
1758 * @pdev: PCI device information struct
1759 */
1760static void hns3_remove(struct pci_dev *pdev)
1761{
1762 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1763
2312e050
FL		 1764 if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1765 hns3_disable_sriov(pdev);
1766
76ad4f0e 1767 hnae3_unregister_ae_dev(ae_dev);
76ad4f0e
S		 1768}
1769
fa8d82e8
PL		 1770/**
1771 * hns3_pci_sriov_configure
1772 * @pdev: pointer to a pci_dev structure
1773 * @num_vfs: number of VFs to allocate
1774 *
1775 * Enable or change the number of VFs. Called when the user updates the number
1776 * of VFs in sysfs.
1777 **/
743e1a84 1778static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
fa8d82e8
PL		 1779{
1780 int ret;
1781
1782 if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1783 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1784 return -EINVAL;
1785 }
1786
1787 if (num_vfs) {
1788 ret = pci_enable_sriov(pdev, num_vfs);
1789 if (ret)
1790 dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
743e1a84
SM		 1791 else
1792 return num_vfs;
fa8d82e8
PL		 1793 } else if (!pci_vfs_assigned(pdev)) {
1794 pci_disable_sriov(pdev);
1795 } else {
1796 dev_warn(&pdev->dev,
1797 "Unable to free VFs because some are assigned to VMs.\n");
1798 }
1799
1800 return 0;
1801}
1802
ce2c1d2e
YL		 1803static void hns3_shutdown(struct pci_dev *pdev)
1804{
1805 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1806
1807 hnae3_unregister_ae_dev(ae_dev);
1808 devm_kfree(&pdev->dev, ae_dev);
1809 pci_set_drvdata(pdev, NULL);
1810
1811 if (system_state == SYSTEM_POWER_OFF)
1812 pci_set_power_state(pdev, PCI_D3hot);
1813}
1814
5a9f0eac
SJ		 1815static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1816 pci_channel_state_t state)
1817{
1818 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1819 pci_ers_result_t ret;
1820
1821 dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1822
1823 if (state == pci_channel_io_perm_failure)
1824 return PCI_ERS_RESULT_DISCONNECT;
1825
1826 if (!ae_dev) {
1827 dev_err(&pdev->dev,
1828 "Can't recover - error happened during device init\n");
1829 return PCI_ERS_RESULT_NONE;
1830 }
1831
1832 if (ae_dev->ops->process_hw_error)
1833 ret = ae_dev->ops->process_hw_error(ae_dev);
1834 else
1835 return PCI_ERS_RESULT_NONE;
1836
1837 return ret;
1838}
1839
6ae4e733
SJ		 1840static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
1841{
1842 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1843 struct device *dev = &pdev->dev;
1844
1845 dev_info(dev, "requesting reset due to PCI error\n");
1846
1847 /* request the reset */
1848 if (ae_dev->ops->reset_event) {
1849 ae_dev->ops->reset_event(pdev, NULL);
1850 return PCI_ERS_RESULT_RECOVERED;
1851 }
1852
1853 return PCI_ERS_RESULT_DISCONNECT;
1854}
1855
6b9a97ee
HT		 1856static void hns3_reset_prepare(struct pci_dev *pdev)
1857{
1858 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1859
1860 dev_info(&pdev->dev, "hns3 flr prepare\n");
1861 if (ae_dev && ae_dev->ops && ae_dev->ops->flr_prepare)
1862 ae_dev->ops->flr_prepare(ae_dev);
1863}
1864
1865static void hns3_reset_done(struct pci_dev *pdev)
1866{
1867 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1868
1869 dev_info(&pdev->dev, "hns3 flr done\n");
1870 if (ae_dev && ae_dev->ops && ae_dev->ops->flr_done)
1871 ae_dev->ops->flr_done(ae_dev);
1872}
1873
5a9f0eac
SJ		 1874static const struct pci_error_handlers hns3_err_handler = {
1875 .error_detected = hns3_error_detected,
6ae4e733 1876 .slot_reset = hns3_slot_reset,
6b9a97ee
HT		 1877 .reset_prepare = hns3_reset_prepare,
1878 .reset_done = hns3_reset_done,
5a9f0eac
SJ		 1879};
1880
76ad4f0e
S		 1881static struct pci_driver hns3_driver = {
1882 .name = hns3_driver_name,
1883 .id_table = hns3_pci_tbl,
1884 .probe = hns3_probe,
1885 .remove = hns3_remove,
ce2c1d2e 1886 .shutdown = hns3_shutdown,
fa8d82e8 1887 .sriov_configure = hns3_pci_sriov_configure,
5a9f0eac 1888 .err_handler = &hns3_err_handler,
76ad4f0e
S		 1889};
1890
1891/* set default feature to hns3 */
1892static void hns3_set_default_feature(struct net_device *netdev)
1893{
3e85af6a
PL		 1894 struct hnae3_handle *h = hns3_get_handle(netdev);
1895 struct pci_dev *pdev = h->pdev;
1896
76ad4f0e
S		 1897 netdev->priv_flags |= IFF_UNICAST_FLT;
1898
1899 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1900 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1901 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1902 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1903 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1904
1905 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1906
1907 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1908
1909 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1910 NETIF_F_HW_VLAN_CTAG_FILTER |
052ece6d 1911 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1912 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1913 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1914 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1915 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1916
1917 netdev->vlan_features |=
1918 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1919 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1920 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1921 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1922 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1923
1924 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
b2641e2a 1925 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1926 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1927 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1928 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1929 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
3e85af6a 1930
c17852a8 1931 if (pdev->revision >= 0x21) {
3e85af6a 1932 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
c17852a8
JS		 1933
1934 if (!(h->flags & HNAE3_SUPPORT_VF)) {
1935 netdev->hw_features |= NETIF_F_NTUPLE;
1936 netdev->features |= NETIF_F_NTUPLE;
1937 }
1938 }
76ad4f0e
S		 1939}
1940
1941static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1942 struct hns3_desc_cb *cb)
1943{
e4e87715 1944 unsigned int order = hnae3_page_order(ring);
76ad4f0e
S		 1945 struct page *p;
1946
1947 p = dev_alloc_pages(order);
1948 if (!p)
1949 return -ENOMEM;
1950
1951 cb->priv = p;
1952 cb->page_offset = 0;
1953 cb->reuse_flag = 0;
1954 cb->buf = page_address(p);
e4e87715 1955 cb->length = hnae3_page_size(ring);
76ad4f0e
S		 1956 cb->type = DESC_TYPE_PAGE;
1957
76ad4f0e
S		 1958 return 0;
1959}
1960
1961static void hns3_free_buffer(struct hns3_enet_ring *ring,
1962 struct hns3_desc_cb *cb)
1963{
1964 if (cb->type == DESC_TYPE_SKB)
1965 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1966 else if (!HNAE3_IS_TX_RING(ring))
1967 put_page((struct page *)cb->priv);
1968 memset(cb, 0, sizeof(*cb));
1969}
1970
1971static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1972{
1973 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1974 cb->length, ring_to_dma_dir(ring));
1975
2211f4e1 1976 if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
76ad4f0e
S		 1977 return -EIO;
1978
1979 return 0;
1980}
1981
1982static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1983 struct hns3_desc_cb *cb)
1984{
1985 if (cb->type == DESC_TYPE_SKB)
1986 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1987 ring_to_dma_dir(ring));
bcdb12b7 1988 else if (cb->length)
76ad4f0e
S		 1989 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1990 ring_to_dma_dir(ring));
1991}
1992
1993static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1994{
1995 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1996 ring->desc[i].addr = 0;
1997}
1998
1999static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
2000{
2001 struct hns3_desc_cb *cb = &ring->desc_cb[i];
2002
2003 if (!ring->desc_cb[i].dma)
2004 return;
2005
2006 hns3_buffer_detach(ring, i);
2007 hns3_free_buffer(ring, cb);
2008}
2009
2010static void hns3_free_buffers(struct hns3_enet_ring *ring)
2011{
2012 int i;
2013
2014 for (i = 0; i < ring->desc_num; i++)
2015 hns3_free_buffer_detach(ring, i);
2016}
2017
2018/* free desc along with its attached buffer */
2019static void hns3_free_desc(struct hns3_enet_ring *ring)
2020{
024cc792
HT		 2021 int size = ring->desc_num * sizeof(ring->desc[0]);
2022
76ad4f0e
S		 2023 hns3_free_buffers(ring);
2024
024cc792
HT		 2025 if (ring->desc) {
2026 dma_free_coherent(ring_to_dev(ring), size,
2027 ring->desc, ring->desc_dma_addr);
2028 ring->desc = NULL;
2029 }
76ad4f0e
S		 2030}
2031
2032static int hns3_alloc_desc(struct hns3_enet_ring *ring)
2033{
2034 int size = ring->desc_num * sizeof(ring->desc[0]);
2035
024cc792
HT		 2036 ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
2037 &ring->desc_dma_addr,
2038 GFP_KERNEL);
76ad4f0e
S		 2039 if (!ring->desc)
2040 return -ENOMEM;
2041
76ad4f0e
S		 2042 return 0;
2043}
2044
2045static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
2046 struct hns3_desc_cb *cb)
2047{
2048 int ret;
2049
2050 ret = hns3_alloc_buffer(ring, cb);
2051 if (ret)
2052 goto out;
2053
2054 ret = hns3_map_buffer(ring, cb);
2055 if (ret)
2056 goto out_with_buf;
2057
2058 return 0;
2059
2060out_with_buf:
564883bb 2061 hns3_free_buffer(ring, cb);
76ad4f0e
S		 2062out:
2063 return ret;
2064}
2065
2066static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
2067{
2068 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
2069
2070 if (ret)
2071 return ret;
2072
2073 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2074
2075 return 0;
2076}
2077
2078/* Allocate memory for raw pkg, and map with dma */
2079static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2080{
2081 int i, j, ret;
2082
2083 for (i = 0; i < ring->desc_num; i++) {
2084 ret = hns3_alloc_buffer_attach(ring, i);
2085 if (ret)
2086 goto out_buffer_fail;
2087 }
2088
2089 return 0;
2090
2091out_buffer_fail:
2092 for (j = i - 1; j >= 0; j--)
2093 hns3_free_buffer_detach(ring, j);
2094 return ret;
2095}
2096
2097/* detach a in-used buffer and replace with a reserved one */
2098static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2099 struct hns3_desc_cb *res_cb)
2100{
b9077428 2101 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 2102 ring->desc_cb[i] = *res_cb;
2103 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
7d0b130c 2104 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 2105}
2106
2107static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2108{
2109 ring->desc_cb[i].reuse_flag = 0;
2110 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2111 + ring->desc_cb[i].page_offset);
7d0b130c 2112 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 2113}
2114
2115static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2116 int *pkts)
2117{
2118 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2119
2120 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2121 (*bytes) += desc_cb->length;
e4e87715 2122 /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
76ad4f0e
S		 2123 hns3_free_buffer_detach(ring, ring->next_to_clean);
2124
2125 ring_ptr_move_fw(ring, next_to_clean);
2126}
2127
2128static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2129{
2130 int u = ring->next_to_use;
2131 int c = ring->next_to_clean;
2132
2133 if (unlikely(h > ring->desc_num))
2134 return 0;
2135
2136 return u > c ? (h > c && h <= u) : (h > c || h <= u);
2137}
2138
799997a3 2139void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
76ad4f0e
S		 2140{
2141 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
7a810110 2142 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 2143 struct netdev_queue *dev_queue;
2144 int bytes, pkts;
2145 int head;
2146
2147 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2148 rmb(); /* Make sure head is ready before touch any data */
2149
2150 if (is_ring_empty(ring) || head == ring->next_to_clean)
799997a3 2151 return; /* no data to poll */
76ad4f0e 2152
0e6084aa 2153 if (unlikely(!is_valid_clean_head(ring, head))) {
76ad4f0e
S		 2154 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2155 ring->next_to_use, ring->next_to_clean);
2156
2157 u64_stats_update_begin(&ring->syncp);
2158 ring->stats.io_err_cnt++;
2159 u64_stats_update_end(&ring->syncp);
799997a3 2160 return;
76ad4f0e
S		 2161 }
2162
2163 bytes = 0;
2164 pkts = 0;
799997a3 2165 while (head != ring->next_to_clean) {
76ad4f0e
S		 2166 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2167 /* Issue prefetch for next Tx descriptor */
2168 prefetch(&ring->desc_cb[ring->next_to_clean]);
76ad4f0e
S		 2169 }
2170
2171 ring->tqp_vector->tx_group.total_bytes += bytes;
2172 ring->tqp_vector->tx_group.total_packets += pkts;
2173
2174 u64_stats_update_begin(&ring->syncp);
2175 ring->stats.tx_bytes += bytes;
2176 ring->stats.tx_pkts += pkts;
2177 u64_stats_update_end(&ring->syncp);
2178
2179 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2180 netdev_tx_completed_queue(dev_queue, pkts, bytes);
2181
2182 if (unlikely(pkts && netif_carrier_ok(netdev) &&
2183 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2184 /* Make sure that anybody stopping the queue after this
2185 * sees the new next_to_clean.
2186 */
2187 smp_mb();
7a810110
JS		 2188 if (netif_tx_queue_stopped(dev_queue) &&
2189 !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
76ad4f0e
S		 2190 netif_tx_wake_queue(dev_queue);
2191 ring->stats.restart_queue++;
2192 }
2193 }
76ad4f0e
S		 2194}
2195
2196static int hns3_desc_unused(struct hns3_enet_ring *ring)
2197{
2198 int ntc = ring->next_to_clean;
2199 int ntu = ring->next_to_use;
2200
2201 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2202}
2203
2204static void
2205hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2206{
2207 struct hns3_desc_cb *desc_cb;
2208 struct hns3_desc_cb res_cbs;
2209 int i, ret;
2210
2211 for (i = 0; i < cleand_count; i++) {
2212 desc_cb = &ring->desc_cb[ring->next_to_use];
2213 if (desc_cb->reuse_flag) {
2214 u64_stats_update_begin(&ring->syncp);
2215 ring->stats.reuse_pg_cnt++;
2216 u64_stats_update_end(&ring->syncp);
2217
2218 hns3_reuse_buffer(ring, ring->next_to_use);
2219 } else {
2220 ret = hns3_reserve_buffer_map(ring, &res_cbs);
2221 if (ret) {
2222 u64_stats_update_begin(&ring->syncp);
2223 ring->stats.sw_err_cnt++;
2224 u64_stats_update_end(&ring->syncp);
2225
2226 netdev_err(ring->tqp->handle->kinfo.netdev,
2227 "hnae reserve buffer map failed.\n");
2228 break;
2229 }
2230 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2231 }
2232
2233 ring_ptr_move_fw(ring, next_to_use);
2234 }
2235
2236 wmb(); /* Make all data has been write before submit */
2237 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2238}
2239
76ad4f0e
S		 2240static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2241 struct hns3_enet_ring *ring, int pull_len,
2242 struct hns3_desc_cb *desc_cb)
2243{
2244 struct hns3_desc *desc;
583e7281
HT		 2245 u32 truesize;
2246 int size;
76ad4f0e
S		 2247 int last_offset;
2248 bool twobufs;
2249
2250 twobufs = ((PAGE_SIZE < 8192) &&
e4e87715 2251 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
76ad4f0e
S		 2252
2253 desc = &ring->desc[ring->next_to_clean];
2254 size = le16_to_cpu(desc->rx.size);
2255
e4e87715 2256 truesize = hnae3_buf_size(ring);
f8d291f0
PL		 2257
2258 if (!twobufs)
e4e87715 2259 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
76ad4f0e
S		 2260
2261 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
f8d291f0 2262 size - pull_len, truesize);
76ad4f0e
S		 2263
2264 /* Avoid re-using remote pages,flag default unreuse */
2265 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2266 return;
2267
2268 if (twobufs) {
2269 /* If we are only owner of page we can reuse it */
2270 if (likely(page_count(desc_cb->priv) == 1)) {
2271 /* Flip page offset to other buffer */
2272 desc_cb->page_offset ^= truesize;
2273
2274 desc_cb->reuse_flag = 1;
2275 /* bump ref count on page before it is given*/
2276 get_page(desc_cb->priv);
2277 }
2278 return;
2279 }
2280
2281 /* Move offset up to the next cache line */
2282 desc_cb->page_offset += truesize;
2283
2284 if (desc_cb->page_offset <= last_offset) {
2285 desc_cb->reuse_flag = 1;
2286 /* Bump ref count on page before it is given*/
2287 get_page(desc_cb->priv);
2288 }
2289}
2290
2291static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2292 struct hns3_desc *desc)
2293{
2294 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2295 int l3_type, l4_type;
2296 u32 bd_base_info;
2297 int ol4_type;
2298 u32 l234info;
2299
2300 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2301 l234info = le32_to_cpu(desc->rx.l234_info);
2302
2303 skb->ip_summed = CHECKSUM_NONE;
2304
2305 skb_checksum_none_assert(skb);
2306
2307 if (!(netdev->features & NETIF_F_RXCSUM))
2308 return;
2309
2310 /* check if hardware has done checksum */
e4e87715 2311 if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
76ad4f0e
S		 2312 return;
2313
e4e87715
PL		 2314 if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2315 hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2316 hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2317 hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
76ad4f0e
S		 2318 u64_stats_update_begin(&ring->syncp);
2319 ring->stats.l3l4_csum_err++;
2320 u64_stats_update_end(&ring->syncp);
2321
2322 return;
2323 }
2324
e4e87715
PL		 2325 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2326 HNS3_RXD_L3ID_S);
2327 l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2328 HNS3_RXD_L4ID_S);
76ad4f0e 2329
e4e87715
PL		 2330 ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2331 HNS3_RXD_OL4ID_S);
76ad4f0e
S		 2332 switch (ol4_type) {
2333 case HNS3_OL4_TYPE_MAC_IN_UDP:
2334 case HNS3_OL4_TYPE_NVGRE:
2335 skb->csum_level = 1;
be44b3af 2336 /* fall through */
76ad4f0e
S		 2337 case HNS3_OL4_TYPE_NO_TUN:
2338 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
94c5e532
PL		 2339 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2340 l3_type == HNS3_L3_TYPE_IPV6) &&
2341 (l4_type == HNS3_L4_TYPE_UDP ||
2342 l4_type == HNS3_L4_TYPE_TCP ||
2343 l4_type == HNS3_L4_TYPE_SCTP))
76ad4f0e
S		 2344 skb->ip_summed = CHECKSUM_UNNECESSARY;
2345 break;
fa7a4bd5
JS		 2346 default:
2347 break;
76ad4f0e
S		 2348 }
2349}
2350
d43e5aca
YL		 2351static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2352{
2353 napi_gro_receive(&ring->tqp_vector->napi, skb);
2354}
2355
701a6d6a
JS		 2356static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2357 struct hns3_desc *desc, u32 l234info,
2358 u16 *vlan_tag)
5b5455a9
PL		 2359{
2360 struct pci_dev *pdev = ring->tqp->handle->pdev;
5b5455a9
PL		 2361
2362 if (pdev->revision == 0x20) {
701a6d6a
JS		 2363 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2364 if (!(*vlan_tag & VLAN_VID_MASK))
2365 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
5b5455a9 2366
701a6d6a 2367 return (*vlan_tag != 0);
5b5455a9
PL		 2368 }
2369
2370#define HNS3_STRP_OUTER_VLAN 0x1
2371#define HNS3_STRP_INNER_VLAN 0x2
2372
e4e87715
PL		 2373 switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2374 HNS3_RXD_STRP_TAGP_S)) {
5b5455a9 2375 case HNS3_STRP_OUTER_VLAN:
701a6d6a
JS		 2376 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2377 return true;
5b5455a9 2378 case HNS3_STRP_INNER_VLAN:
701a6d6a
JS		 2379 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2380 return true;
5b5455a9 2381 default:
701a6d6a 2382 return false;
5b5455a9 2383 }
5b5455a9
PL		 2384}
2385
232fc64b
PL		 2386static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2387 struct sk_buff *skb)
2388{
2389 struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2390 struct hnae3_handle *handle = ring->tqp->handle;
2391 enum pkt_hash_types rss_type;
2392
2393 if (le32_to_cpu(desc->rx.rss_hash))
2394 rss_type = handle->kinfo.rss_type;
2395 else
2396 rss_type = PKT_HASH_TYPE_NONE;
2397
2398 skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2399}
2400
76ad4f0e
S		 2401static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2402 struct sk_buff **out_skb, int *out_bnum)
2403{
2404 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2405 struct hns3_desc_cb *desc_cb;
2406 struct hns3_desc *desc;
2407 struct sk_buff *skb;
2408 unsigned char *va;
2409 u32 bd_base_info;
2410 int pull_len;
2411 u32 l234info;
2412 int length;
2413 int bnum;
2414
2415 desc = &ring->desc[ring->next_to_clean];
2416 desc_cb = &ring->desc_cb[ring->next_to_clean];
2417
2418 prefetch(desc);
2419
846fcc83 2420 length = le16_to_cpu(desc->rx.size);
76ad4f0e 2421 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
76ad4f0e
S		 2422
2423 /* Check valid BD */
e4e87715 2424 if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
76ad4f0e
S		 2425 return -EFAULT;
2426
2427 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2428
2429 /* Prefetch first cache line of first page
2430 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2431 * line size is 64B so need to prefetch twice to make it 128B. But in
2432 * actual we can have greater size of caches with 128B Level 1 cache
2433 * lines. In such a case, single fetch would suffice to cache in the
2434 * relevant part of the header.
2435 */
2436 prefetch(va);
2437#if L1_CACHE_BYTES < 128
2438 prefetch(va + L1_CACHE_BYTES);
2439#endif
2440
2441 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2442 HNS3_RX_HEAD_SIZE);
2443 if (unlikely(!skb)) {
2444 netdev_err(netdev, "alloc rx skb fail\n");
2445
2446 u64_stats_update_begin(&ring->syncp);
2447 ring->stats.sw_err_cnt++;
2448 u64_stats_update_end(&ring->syncp);
2449
2450 return -ENOMEM;
2451 }
2452
2453 prefetchw(skb->data);
2454
2455 bnum = 1;
2456 if (length <= HNS3_RX_HEAD_SIZE) {
2457 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2458
2459 /* We can reuse buffer as-is, just make sure it is local */
2460 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2461 desc_cb->reuse_flag = 1;
2462 else /* This page cannot be reused so discard it */
2463 put_page(desc_cb->priv);
2464
2465 ring_ptr_move_fw(ring, next_to_clean);
2466 } else {
2467 u64_stats_update_begin(&ring->syncp);
2468 ring->stats.seg_pkt_cnt++;
2469 u64_stats_update_end(&ring->syncp);
2470
e63cd65f
PL		 2471 pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2472
76ad4f0e
S		 2473 memcpy(__skb_put(skb, pull_len), va,
2474 ALIGN(pull_len, sizeof(long)));
2475
2476 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2477 ring_ptr_move_fw(ring, next_to_clean);
2478
e4e87715 2479 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
76ad4f0e
S		 2480 desc = &ring->desc[ring->next_to_clean];
2481 desc_cb = &ring->desc_cb[ring->next_to_clean];
2482 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2483 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2484 ring_ptr_move_fw(ring, next_to_clean);
2485 bnum++;
2486 }
2487 }
2488
2489 *out_bnum = bnum;
5b5455a9
PL		 2490
2491 l234info = le32_to_cpu(desc->rx.l234_info);
2492
846fcc83
PL		 2493 /* Based on hw strategy, the tag offloaded will be stored at
2494 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2495 * in one layer tag case.
2496 */
2497 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2498 u16 vlan_tag;
2499
701a6d6a 2500 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
846fcc83
PL		 2501 __vlan_hwaccel_put_tag(skb,
2502 htons(ETH_P_8021Q),
2503 vlan_tag);
2504 }
2505
e4e87715 2506 if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
76ad4f0e
S		 2507 u64_stats_update_begin(&ring->syncp);
2508 ring->stats.non_vld_descs++;
2509 u64_stats_update_end(&ring->syncp);
2510
2511 dev_kfree_skb_any(skb);
2512 return -EINVAL;
2513 }
2514
2515 if (unlikely((!desc->rx.pkt_len) ||
e4e87715 2516 hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
76ad4f0e
S		 2517 u64_stats_update_begin(&ring->syncp);
2518 ring->stats.err_pkt_len++;
2519 u64_stats_update_end(&ring->syncp);
2520
2521 dev_kfree_skb_any(skb);
2522 return -EFAULT;
2523 }
2524
e4e87715 2525 if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
76ad4f0e
S		 2526 u64_stats_update_begin(&ring->syncp);
2527 ring->stats.l2_err++;
2528 u64_stats_update_end(&ring->syncp);
2529
2530 dev_kfree_skb_any(skb);
2531 return -EFAULT;
2532 }
2533
2534 u64_stats_update_begin(&ring->syncp);
2535 ring->stats.rx_pkts++;
2536 ring->stats.rx_bytes += skb->len;
2537 u64_stats_update_end(&ring->syncp);
2538
2539 ring->tqp_vector->rx_group.total_bytes += skb->len;
2540
2541 hns3_rx_checksum(ring, skb, desc);
232fc64b
PL		 2542 hns3_set_rx_skb_rss_type(ring, skb);
2543
76ad4f0e
S		 2544 return 0;
2545}
2546
d43e5aca
YL		 2547int hns3_clean_rx_ring(
2548 struct hns3_enet_ring *ring, int budget,
2549 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2550{
2551#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2552 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2553 int recv_pkts, recv_bds, clean_count, err;
2554 int unused_count = hns3_desc_unused(ring);
2555 struct sk_buff *skb = NULL;
2556 int num, bnum = 0;
2557
2558 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2559 rmb(); /* Make sure num taken effect before the other data is touched */
2560
2561 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2562 num -= unused_count;
2563
2564 while (recv_pkts < budget && recv_bds < num) {
2565 /* Reuse or realloc buffers */
2566 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2567 hns3_nic_alloc_rx_buffers(ring,
2568 clean_count + unused_count);
2569 clean_count = 0;
2570 unused_count = hns3_desc_unused(ring);
2571 }
2572
2573 /* Poll one pkt */
2574 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2575 if (unlikely(!skb)) /* This fault cannot be repaired */
2576 goto out;
2577
2578 recv_bds += bnum;
2579 clean_count += bnum;
2580 if (unlikely(err)) { /* Do jump the err */
2581 recv_pkts++;
2582 continue;
2583 }
2584
2585 /* Do update ip stack process */
2586 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2587 rx_fn(ring, skb);
76ad4f0e
S		 2588
2589 recv_pkts++;
2590 }
2591
2592out:
2593 /* Make all data has been write before submit */
2594 if (clean_count + unused_count > 0)
2595 hns3_nic_alloc_rx_buffers(ring,
2596 clean_count + unused_count);
2597
2598 return recv_pkts;
2599}
2600
2601static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2602{
a95e1f86
FL		 2603 struct hns3_enet_tqp_vector *tqp_vector =
2604 ring_group->ring->tqp_vector;
76ad4f0e 2605 enum hns3_flow_level_range new_flow_level;
a95e1f86
FL		 2606 int packets_per_msecs;
2607 int bytes_per_msecs;
2608 u32 time_passed_ms;
76ad4f0e 2609 u16 new_int_gl;
76ad4f0e 2610
a95e1f86 2611 if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
76ad4f0e
S		 2612 return false;
2613
2614 if (ring_group->total_packets == 0) {
9bc727a9
YL		 2615 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2616 ring_group->coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 2617 return true;
2618 }
2619
2620 /* Simple throttlerate management
2621 * 0-10MB/s lower (50000 ints/s)
2622 * 10-20MB/s middle (20000 ints/s)
2623 * 20-1249MB/s high (18000 ints/s)
2624 * > 40000pps ultra (8000 ints/s)
2625 */
9bc727a9
YL		 2626 new_flow_level = ring_group->coal.flow_level;
2627 new_int_gl = ring_group->coal.int_gl;
a95e1f86
FL		 2628 time_passed_ms =
2629 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2630
2631 if (!time_passed_ms)
2632 return false;
2633
2634 do_div(ring_group->total_packets, time_passed_ms);
2635 packets_per_msecs = ring_group->total_packets;
2636
2637 do_div(ring_group->total_bytes, time_passed_ms);
2638 bytes_per_msecs = ring_group->total_bytes;
2639
2640#define HNS3_RX_LOW_BYTE_RATE 10000
2641#define HNS3_RX_MID_BYTE_RATE 20000
76ad4f0e
S		 2642
2643 switch (new_flow_level) {
2644 case HNS3_FLOW_LOW:
a95e1f86 2645 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2646 new_flow_level = HNS3_FLOW_MID;
2647 break;
2648 case HNS3_FLOW_MID:
a95e1f86 2649 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
76ad4f0e 2650 new_flow_level = HNS3_FLOW_HIGH;
a95e1f86 2651 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2652 new_flow_level = HNS3_FLOW_LOW;
2653 break;
2654 case HNS3_FLOW_HIGH:
2655 case HNS3_FLOW_ULTRA:
2656 default:
a95e1f86 2657 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
76ad4f0e
S		 2658 new_flow_level = HNS3_FLOW_MID;
2659 break;
2660 }
2661
a95e1f86
FL		 2662#define HNS3_RX_ULTRA_PACKET_RATE 40
2663
2664 if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2665 &tqp_vector->rx_group == ring_group)
76ad4f0e
S		 2666 new_flow_level = HNS3_FLOW_ULTRA;
2667
2668 switch (new_flow_level) {
2669 case HNS3_FLOW_LOW:
2670 new_int_gl = HNS3_INT_GL_50K;
2671 break;
2672 case HNS3_FLOW_MID:
2673 new_int_gl = HNS3_INT_GL_20K;
2674 break;
2675 case HNS3_FLOW_HIGH:
2676 new_int_gl = HNS3_INT_GL_18K;
2677 break;
2678 case HNS3_FLOW_ULTRA:
2679 new_int_gl = HNS3_INT_GL_8K;
2680 break;
2681 default:
2682 break;
2683 }
2684
2685 ring_group->total_bytes = 0;
2686 ring_group->total_packets = 0;
9bc727a9
YL		 2687 ring_group->coal.flow_level = new_flow_level;
2688 if (new_int_gl != ring_group->coal.int_gl) {
2689 ring_group->coal.int_gl = new_int_gl;
76ad4f0e
S		 2690 return true;
2691 }
2692 return false;
2693}
2694
2695static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2696{
8b1ff1ea
FL		 2697 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2698 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2699 bool rx_update, tx_update;
2700
cd9d187b
FL		 2701 if (tqp_vector->int_adapt_down > 0) {
2702 tqp_vector->int_adapt_down--;
2703 return;
2704 }
2705
9bc727a9 2706 if (rx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2707 rx_update = hns3_get_new_int_gl(rx_group);
2708 if (rx_update)
2709 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 2710 rx_group->coal.int_gl);
8b1ff1ea
FL		 2711 }
2712
9bc727a9 2713 if (tx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2714 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2715 if (tx_update)
2716 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 2717 tx_group->coal.int_gl);
76ad4f0e 2718 }
cd9d187b 2719
a95e1f86 2720 tqp_vector->last_jiffies = jiffies;
cd9d187b 2721 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
76ad4f0e
S		 2722}
2723
2724static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2725{
ff0699e0 2726 struct hns3_nic_priv *priv = netdev_priv(napi->dev);
76ad4f0e
S		 2727 struct hns3_enet_ring *ring;
2728 int rx_pkt_total = 0;
2729
2730 struct hns3_enet_tqp_vector *tqp_vector =
2731 container_of(napi, struct hns3_enet_tqp_vector, napi);
2732 bool clean_complete = true;
2733 int rx_budget;
2734
ff0699e0
HT		 2735 if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
2736 napi_complete(napi);
2737 return 0;
2738 }
2739
76ad4f0e
S		 2740 /* Since the actual Tx work is minimal, we can give the Tx a larger
2741 * budget and be more aggressive about cleaning up the Tx descriptors.
2742 */
799997a3
PL		 2743 hns3_for_each_ring(ring, tqp_vector->tx_group)
2744 hns3_clean_tx_ring(ring);
76ad4f0e
S		 2745
2746 /* make sure rx ring budget not smaller than 1 */
2747 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2748
2749 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2750 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2751 hns3_rx_skb);
76ad4f0e
S		 2752
2753 if (rx_cleaned >= rx_budget)
2754 clean_complete = false;
2755
2756 rx_pkt_total += rx_cleaned;
2757 }
2758
2759 tqp_vector->rx_group.total_packets += rx_pkt_total;
2760
2761 if (!clean_complete)
2762 return budget;
2763
ff0699e0
HT		 2764 if (likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) &&
2765 napi_complete(napi)) {
2766 hns3_update_new_int_gl(tqp_vector);
2767 hns3_mask_vector_irq(tqp_vector, 1);
2768 }
76ad4f0e
S		 2769
2770 return rx_pkt_total;
2771}
2772
2773static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2774 struct hnae3_ring_chain_node *head)
2775{
2776 struct pci_dev *pdev = tqp_vector->handle->pdev;
2777 struct hnae3_ring_chain_node *cur_chain = head;
2778 struct hnae3_ring_chain_node *chain;
2779 struct hns3_enet_ring *tx_ring;
2780 struct hns3_enet_ring *rx_ring;
2781
2782 tx_ring = tqp_vector->tx_group.ring;
2783 if (tx_ring) {
2784 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
e4e87715
PL		 2785 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2786 HNAE3_RING_TYPE_TX);
2787 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2788 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
76ad4f0e
S		 2789
2790 cur_chain->next = NULL;
2791
2792 while (tx_ring->next) {
2793 tx_ring = tx_ring->next;
2794
2795 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2796 GFP_KERNEL);
2797 if (!chain)
73b907a0 2798 goto err_free_chain;
76ad4f0e
S		 2799
2800 cur_chain->next = chain;
2801 chain->tqp_index = tx_ring->tqp->tqp_index;
e4e87715
PL		 2802 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2803 HNAE3_RING_TYPE_TX);
2804 hnae3_set_field(chain->int_gl_idx,
2805 HNAE3_RING_GL_IDX_M,
2806 HNAE3_RING_GL_IDX_S,
2807 HNAE3_RING_GL_TX);
76ad4f0e
S		 2808
2809 cur_chain = chain;
2810 }
2811 }
2812
2813 rx_ring = tqp_vector->rx_group.ring;
2814 if (!tx_ring && rx_ring) {
2815 cur_chain->next = NULL;
2816 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
e4e87715
PL		 2817 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2818 HNAE3_RING_TYPE_RX);
2819 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2820 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
76ad4f0e
S		 2821
2822 rx_ring = rx_ring->next;
2823 }
2824
2825 while (rx_ring) {
2826 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2827 if (!chain)
73b907a0 2828 goto err_free_chain;
76ad4f0e
S		 2829
2830 cur_chain->next = chain;
2831 chain->tqp_index = rx_ring->tqp->tqp_index;
e4e87715
PL		 2832 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2833 HNAE3_RING_TYPE_RX);
2834 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2835 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
11af96a4 2836
76ad4f0e
S		 2837 cur_chain = chain;
2838
2839 rx_ring = rx_ring->next;
2840 }
2841
2842 return 0;
73b907a0
HT		 2843
2844err_free_chain:
2845 cur_chain = head->next;
2846 while (cur_chain) {
2847 chain = cur_chain->next;
2848 devm_kfree(&pdev->dev, chain);
2849 cur_chain = chain;
2850 }
2851
2852 return -ENOMEM;
76ad4f0e
S		 2853}
2854
2855static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2856 struct hnae3_ring_chain_node *head)
2857{
2858 struct pci_dev *pdev = tqp_vector->handle->pdev;
2859 struct hnae3_ring_chain_node *chain_tmp, *chain;
2860
2861 chain = head->next;
2862
2863 while (chain) {
2864 chain_tmp = chain->next;
2865 devm_kfree(&pdev->dev, chain);
2866 chain = chain_tmp;
2867 }
2868}
2869
2870static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2871 struct hns3_enet_ring *ring)
2872{
2873 ring->next = group->ring;
2874 group->ring = ring;
2875
2876 group->count++;
2877}
2878
874bff0b
PL		 2879static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
2880{
2881 struct pci_dev *pdev = priv->ae_handle->pdev;
2882 struct hns3_enet_tqp_vector *tqp_vector;
2883 int num_vectors = priv->vector_num;
2884 int numa_node;
2885 int vector_i;
2886
2887 numa_node = dev_to_node(&pdev->dev);
2888
2889 for (vector_i = 0; vector_i < num_vectors; vector_i++) {
2890 tqp_vector = &priv->tqp_vector[vector_i];
2891 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
2892 &tqp_vector->affinity_mask);
2893 }
2894}
2895
76ad4f0e
S		 2896static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2897{
2898 struct hnae3_ring_chain_node vector_ring_chain;
2899 struct hnae3_handle *h = priv->ae_handle;
2900 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e 2901 int ret = 0;
ece4bf46 2902 int i;
76ad4f0e 2903
874bff0b
PL		 2904 hns3_nic_set_cpumask(priv);
2905
dd38c726
YL		 2906 for (i = 0; i < priv->vector_num; i++) {
2907 tqp_vector = &priv->tqp_vector[i];
2908 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2909 tqp_vector->num_tqps = 0;
2910 }
76ad4f0e 2911
dd38c726
YL		 2912 for (i = 0; i < h->kinfo.num_tqps; i++) {
2913 u16 vector_i = i % priv->vector_num;
2914 u16 tqp_num = h->kinfo.num_tqps;
76ad4f0e
S		 2915
2916 tqp_vector = &priv->tqp_vector[vector_i];
2917
2918 hns3_add_ring_to_group(&tqp_vector->tx_group,
2919 priv->ring_data[i].ring);
2920
2921 hns3_add_ring_to_group(&tqp_vector->rx_group,
2922 priv->ring_data[i + tqp_num].ring);
2923
76ad4f0e
S		 2924 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2925 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
dd38c726 2926 tqp_vector->num_tqps++;
76ad4f0e
S		 2927 }
2928
dd38c726 2929 for (i = 0; i < priv->vector_num; i++) {
76ad4f0e
S		 2930 tqp_vector = &priv->tqp_vector[i];
2931
2932 tqp_vector->rx_group.total_bytes = 0;
2933 tqp_vector->rx_group.total_packets = 0;
2934 tqp_vector->tx_group.total_bytes = 0;
2935 tqp_vector->tx_group.total_packets = 0;
76ad4f0e
S		 2936 tqp_vector->handle = h;
2937
2938 ret = hns3_get_vector_ring_chain(tqp_vector,
2939 &vector_ring_chain);
2940 if (ret)
dd38c726 2941 return ret;
76ad4f0e
S		 2942
2943 ret = h->ae_algo->ops->map_ring_to_vector(h,
2944 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 2945
2946 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2947
dd38c726 2948 if (ret)
ece4bf46 2949 goto map_ring_fail;
dd38c726 2950
76ad4f0e
S		 2951 netif_napi_add(priv->netdev, &tqp_vector->napi,
2952 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2953 }
2954
dd38c726 2955 return 0;
ece4bf46
HT		 2956
2957map_ring_fail:
2958 while (i--)
2959 netif_napi_del(&priv->tqp_vector[i].napi);
2960
2961 return ret;
dd38c726
YL		 2962}
2963
2964static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
2965{
2966 struct hnae3_handle *h = priv->ae_handle;
2967 struct hns3_enet_tqp_vector *tqp_vector;
2968 struct hnae3_vector_info *vector;
2969 struct pci_dev *pdev = h->pdev;
2970 u16 tqp_num = h->kinfo.num_tqps;
2971 u16 vector_num;
2972 int ret = 0;
2973 u16 i;
2974
2975 /* RSS size, cpu online and vector_num should be the same */
2976 /* Should consider 2p/4p later */
2977 vector_num = min_t(u16, num_online_cpus(), tqp_num);
2978 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2979 GFP_KERNEL);
2980 if (!vector)
2981 return -ENOMEM;
2982
2983 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2984
2985 priv->vector_num = vector_num;
2986 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2987 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2988 GFP_KERNEL);
2989 if (!priv->tqp_vector) {
2990 ret = -ENOMEM;
2991 goto out;
2992 }
2993
2994 for (i = 0; i < priv->vector_num; i++) {
2995 tqp_vector = &priv->tqp_vector[i];
2996 tqp_vector->idx = i;
2997 tqp_vector->mask_addr = vector[i].io_addr;
2998 tqp_vector->vector_irq = vector[i].vector;
2999 hns3_vector_gl_rl_init(tqp_vector, priv);
3000 }
3001
76ad4f0e
S		 3002out:
3003 devm_kfree(&pdev->dev, vector);
3004 return ret;
3005}
3006
dd38c726
YL		 3007static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
3008{
3009 group->ring = NULL;
3010 group->count = 0;
3011}
3012
76ad4f0e
S		 3013static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
3014{
3015 struct hnae3_ring_chain_node vector_ring_chain;
3016 struct hnae3_handle *h = priv->ae_handle;
3017 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 3018 int i, ret;
3019
3020 for (i = 0; i < priv->vector_num; i++) {
3021 tqp_vector = &priv->tqp_vector[i];
3022
3023 ret = hns3_get_vector_ring_chain(tqp_vector,
3024 &vector_ring_chain);
3025 if (ret)
3026 return ret;
3027
3028 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
3029 tqp_vector->vector_irq, &vector_ring_chain);
3030 if (ret)
3031 return ret;
3032
3033 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3034
3035 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
3036 (void)irq_set_affinity_hint(
3037 priv->tqp_vector[i].vector_irq,
3038 NULL);
ae064e61 3039 free_irq(priv->tqp_vector[i].vector_irq,
3040 &priv->tqp_vector[i]);
76ad4f0e
S		 3041 }
3042
3043 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
dd38c726
YL		 3044 hns3_clear_ring_group(&tqp_vector->rx_group);
3045 hns3_clear_ring_group(&tqp_vector->tx_group);
76ad4f0e
S		 3046 netif_napi_del(&priv->tqp_vector[i].napi);
3047 }
3048
dd38c726
YL		 3049 return 0;
3050}
3051
3052static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
3053{
3054 struct hnae3_handle *h = priv->ae_handle;
3055 struct pci_dev *pdev = h->pdev;
3056 int i, ret;
3057
3058 for (i = 0; i < priv->vector_num; i++) {
3059 struct hns3_enet_tqp_vector *tqp_vector;
3060
3061 tqp_vector = &priv->tqp_vector[i];
3062 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
3063 if (ret)
3064 return ret;
3065 }
76ad4f0e 3066
dd38c726 3067 devm_kfree(&pdev->dev, priv->tqp_vector);
76ad4f0e
S		 3068 return 0;
3069}
3070
3071static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
3072 int ring_type)
3073{
3074 struct hns3_nic_ring_data *ring_data = priv->ring_data;
3075 int queue_num = priv->ae_handle->kinfo.num_tqps;
3076 struct pci_dev *pdev = priv->ae_handle->pdev;
3077 struct hns3_enet_ring *ring;
3078
3079 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
3080 if (!ring)
3081 return -ENOMEM;
3082
3083 if (ring_type == HNAE3_RING_TYPE_TX) {
3084 ring_data[q->tqp_index].ring = ring;
66b44730 3085 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 3086 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
3087 } else {
3088 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 3089 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 3090 ring->io_base = q->io_base;
3091 }
3092
e4e87715 3093 hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
76ad4f0e 3094
76ad4f0e
S		 3095 ring->tqp = q;
3096 ring->desc = NULL;
3097 ring->desc_cb = NULL;
3098 ring->dev = priv->dev;
3099 ring->desc_dma_addr = 0;
3100 ring->buf_size = q->buf_size;
3101 ring->desc_num = q->desc_num;
3102 ring->next_to_use = 0;
3103 ring->next_to_clean = 0;
3104
3105 return 0;
3106}
3107
3108static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3109 struct hns3_nic_priv *priv)
3110{
3111 int ret;
3112
3113 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3114 if (ret)
3115 return ret;
3116
3117 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
73b907a0
HT		 3118 if (ret) {
3119 devm_kfree(priv->dev, priv->ring_data[tqp->tqp_index].ring);
76ad4f0e 3120 return ret;
73b907a0 3121 }
76ad4f0e
S		 3122
3123 return 0;
3124}
3125
3126static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3127{
3128 struct hnae3_handle *h = priv->ae_handle;
3129 struct pci_dev *pdev = h->pdev;
3130 int i, ret;
3131
a86854d0
KC		 3132 priv->ring_data = devm_kzalloc(&pdev->dev,
3133 array3_size(h->kinfo.num_tqps,
3134 sizeof(*priv->ring_data),
3135 2),
76ad4f0e
S		 3136 GFP_KERNEL);
3137 if (!priv->ring_data)
3138 return -ENOMEM;
3139
3140 for (i = 0; i < h->kinfo.num_tqps; i++) {
3141 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3142 if (ret)
3143 goto err;
3144 }
3145
3146 return 0;
3147err:
73b907a0
HT		 3148 while (i--) {
3149 devm_kfree(priv->dev, priv->ring_data[i].ring);
3150 devm_kfree(priv->dev,
3151 priv->ring_data[i + h->kinfo.num_tqps].ring);
3152 }
3153
76ad4f0e
S		 3154 devm_kfree(&pdev->dev, priv->ring_data);
3155 return ret;
3156}
3157
09f2af64
PL		 3158static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3159{
3160 struct hnae3_handle *h = priv->ae_handle;
3161 int i;
3162
3163 for (i = 0; i < h->kinfo.num_tqps; i++) {
3164 devm_kfree(priv->dev, priv->ring_data[i].ring);
3165 devm_kfree(priv->dev,
3166 priv->ring_data[i + h->kinfo.num_tqps].ring);
3167 }
3168 devm_kfree(priv->dev, priv->ring_data);
3169}
3170
76ad4f0e
S		 3171static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3172{
3173 int ret;
3174
3175 if (ring->desc_num <= 0 || ring->buf_size <= 0)
3176 return -EINVAL;
3177
3178 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3179 GFP_KERNEL);
3180 if (!ring->desc_cb) {
3181 ret = -ENOMEM;
3182 goto out;
3183 }
3184
3185 ret = hns3_alloc_desc(ring);
3186 if (ret)
3187 goto out_with_desc_cb;
3188
3189 if (!HNAE3_IS_TX_RING(ring)) {
3190 ret = hns3_alloc_ring_buffers(ring);
3191 if (ret)
3192 goto out_with_desc;
3193 }
3194
3195 return 0;
3196
3197out_with_desc:
3198 hns3_free_desc(ring);
3199out_with_desc_cb:
3200 kfree(ring->desc_cb);
3201 ring->desc_cb = NULL;
3202out:
3203 return ret;
3204}
3205
3206static void hns3_fini_ring(struct hns3_enet_ring *ring)
3207{
3208 hns3_free_desc(ring);
3209 kfree(ring->desc_cb);
3210 ring->desc_cb = NULL;
3211 ring->next_to_clean = 0;
3212 ring->next_to_use = 0;
3213}
3214
1db9b1bf 3215static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 3216{
3217 int bd_size_type;
3218
3219 switch (buf_size) {
3220 case 512:
3221 bd_size_type = HNS3_BD_SIZE_512_TYPE;
3222 break;
3223 case 1024:
3224 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3225 break;
3226 case 2048:
3227 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3228 break;
3229 case 4096:
3230 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3231 break;
3232 default:
3233 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3234 }
3235
3236 return bd_size_type;
3237}
3238
3239static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3240{
3241 dma_addr_t dma = ring->desc_dma_addr;
3242 struct hnae3_queue *q = ring->tqp;
3243
3244 if (!HNAE3_IS_TX_RING(ring)) {
3245 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
3246 (u32)dma);
3247 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
3248 (u32)((dma >> 31) >> 1));
3249
3250 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
3251 hns3_buf_size2type(ring->buf_size));
3252 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3253 ring->desc_num / 8 - 1);
3254
3255 } else {
3256 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3257 (u32)dma);
3258 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3259 (u32)((dma >> 31) >> 1));
3260
76ad4f0e
S		 3261 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3262 ring->desc_num / 8 - 1);
3263 }
3264}
3265
1c772154
YL		 3266static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3267{
3268 struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3269 int i;
3270
3271 for (i = 0; i < HNAE3_MAX_TC; i++) {
3272 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3273 int j;
3274
3275 if (!tc_info->enable)
3276 continue;
3277
3278 for (j = 0; j < tc_info->tqp_count; j++) {
3279 struct hnae3_queue *q;
3280
3281 q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp;
3282 hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3283 tc_info->tc);
3284 }
3285 }
3286}
3287
5668abda 3288int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3289{
3290 struct hnae3_handle *h = priv->ae_handle;
3291 int ring_num = h->kinfo.num_tqps * 2;
3292 int i, j;
3293 int ret;
3294
3295 for (i = 0; i < ring_num; i++) {
3296 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
3297 if (ret) {
3298 dev_err(priv->dev,
3299 "Alloc ring memory fail! ret=%d\n", ret);
3300 goto out_when_alloc_ring_memory;
3301 }
3302
76ad4f0e
S		 3303 u64_stats_init(&priv->ring_data[i].ring->syncp);
3304 }
3305
3306 return 0;
3307
3308out_when_alloc_ring_memory:
3309 for (j = i - 1; j >= 0; j--)
ee83f776 3310 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 3311
3312 return -ENOMEM;
3313}
3314
5668abda 3315int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3316{
3317 struct hnae3_handle *h = priv->ae_handle;
3318 int i;
3319
3320 for (i = 0; i < h->kinfo.num_tqps; i++) {
76ad4f0e
S		 3321 hns3_fini_ring(priv->ring_data[i].ring);
3322 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3323 }
76ad4f0e
S		 3324 return 0;
3325}
3326
3327/* Set mac addr if it is configured. or leave it to the AE driver */
7fa6be4f 3328static int hns3_init_mac_addr(struct net_device *netdev, bool init)
76ad4f0e
S		 3329{
3330 struct hns3_nic_priv *priv = netdev_priv(netdev);
3331 struct hnae3_handle *h = priv->ae_handle;
3332 u8 mac_addr_temp[ETH_ALEN];
7fa6be4f 3333 int ret = 0;
76ad4f0e 3334
f09555ff 3335 if (h->ae_algo->ops->get_mac_addr && init) {
76ad4f0e
S		 3336 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3337 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3338 }
3339
3340 /* Check if the MAC address is valid, if not get a random one */
3341 if (!is_valid_ether_addr(netdev->dev_addr)) {
3342 eth_hw_addr_random(netdev);
3343 dev_warn(priv->dev, "using random MAC address %pM\n",
3344 netdev->dev_addr);
76ad4f0e 3345 }
139e8792
L		 3346
3347 if (h->ae_algo->ops->set_mac_addr)
7fa6be4f 3348 ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
139e8792 3349
7fa6be4f 3350 return ret;
76ad4f0e
S		 3351}
3352
6871af29
JS		 3353static int hns3_restore_fd_rules(struct net_device *netdev)
3354{
3355 struct hnae3_handle *h = hns3_get_handle(netdev);
3356 int ret = 0;
3357
3358 if (h->ae_algo->ops->restore_fd_rules)
3359 ret = h->ae_algo->ops->restore_fd_rules(h);
3360
3361 return ret;
3362}
3363
3364static void hns3_del_all_fd_rules(struct net_device *netdev, bool clear_list)
3365{
3366 struct hnae3_handle *h = hns3_get_handle(netdev);
3367
3368 if (h->ae_algo->ops->del_all_fd_entries)
3369 h->ae_algo->ops->del_all_fd_entries(h, clear_list);
3370}
3371
76ad4f0e
S		 3372static void hns3_nic_set_priv_ops(struct net_device *netdev)
3373{
3374 struct hns3_nic_priv *priv = netdev_priv(netdev);
3375
0bbbf15d 3376 priv->ops.fill_desc = hns3_fill_desc;
76ad4f0e 3377 if ((netdev->features & NETIF_F_TSO) ||
0bbbf15d 3378 (netdev->features & NETIF_F_TSO6))
76ad4f0e 3379 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
0bbbf15d 3380 else
76ad4f0e 3381 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
76ad4f0e
S		 3382}
3383
3384static int hns3_client_init(struct hnae3_handle *handle)
3385{
3386 struct pci_dev *pdev = handle->pdev;
0d43bf45 3387 u16 alloc_tqps, max_rss_size;
76ad4f0e
S		 3388 struct hns3_nic_priv *priv;
3389 struct net_device *netdev;
3390 int ret;
3391
0d43bf45
HT		 3392 handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
3393 &max_rss_size);
3394 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
76ad4f0e
S		 3395 if (!netdev)
3396 return -ENOMEM;
3397
3398 priv = netdev_priv(netdev);
3399 priv->dev = &pdev->dev;
3400 priv->netdev = netdev;
3401 priv->ae_handle = handle;
f8fa222c 3402 priv->tx_timeout_count = 0;
76ad4f0e
S		 3403
3404 handle->kinfo.netdev = netdev;
3405 handle->priv = (void *)priv;
3406
f09555ff 3407 hns3_init_mac_addr(netdev, true);
76ad4f0e
S		 3408
3409 hns3_set_default_feature(netdev);
3410
3411 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3412 netdev->priv_flags |= IFF_UNICAST_FLT;
3413 netdev->netdev_ops = &hns3_nic_netdev_ops;
3414 SET_NETDEV_DEV(netdev, &pdev->dev);
3415 hns3_ethtool_set_ops(netdev);
3416 hns3_nic_set_priv_ops(netdev);
3417
3418 /* Carrier off reporting is important to ethtool even BEFORE open */
3419 netif_carrier_off(netdev);
3420
3421 ret = hns3_get_ring_config(priv);
3422 if (ret) {
3423 ret = -ENOMEM;
3424 goto out_get_ring_cfg;
3425 }
3426
dd38c726
YL		 3427 ret = hns3_nic_alloc_vector_data(priv);
3428 if (ret) {
3429 ret = -ENOMEM;
3430 goto out_alloc_vector_data;
3431 }
3432
76ad4f0e
S		 3433 ret = hns3_nic_init_vector_data(priv);
3434 if (ret) {
3435 ret = -ENOMEM;
3436 goto out_init_vector_data;
3437 }
3438
3439 ret = hns3_init_all_ring(priv);
3440 if (ret) {
3441 ret = -ENOMEM;
3442 goto out_init_ring_data;
3443 }
3444
3445 ret = register_netdev(netdev);
3446 if (ret) {
3447 dev_err(priv->dev, "probe register netdev fail!\n");
3448 goto out_reg_netdev_fail;
3449 }
3450
986743db
YL		 3451 hns3_dcbnl_setup(handle);
3452
a8e8b7ff
S		 3453 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3454 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3455
814da63c
HT		 3456 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
3457
76ad4f0e
S		 3458 return ret;
3459
3460out_reg_netdev_fail:
3461out_init_ring_data:
3462 (void)hns3_nic_uninit_vector_data(priv);
76ad4f0e 3463out_init_vector_data:
dd38c726
YL		 3464 hns3_nic_dealloc_vector_data(priv);
3465out_alloc_vector_data:
3466 priv->ring_data = NULL;
76ad4f0e
S		 3467out_get_ring_cfg:
3468 priv->ae_handle = NULL;
3469 free_netdev(netdev);
3470 return ret;
3471}
3472
3473static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3474{
3475 struct net_device *netdev = handle->kinfo.netdev;
3476 struct hns3_nic_priv *priv = netdev_priv(netdev);
3477 int ret;
3478
f05e2109
JS		 3479 hns3_remove_hw_addr(netdev);
3480
76ad4f0e
S		 3481 if (netdev->reg_state != NETREG_UNINITIALIZED)
3482 unregister_netdev(netdev);
3483
814da63c
HT		 3484 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
3485 netdev_warn(netdev, "already uninitialized\n");
3486 goto out_netdev_free;
3487 }
3488
dc5e6064
JS		 3489 hns3_del_all_fd_rules(netdev, true);
3490
7b763f3f
FL		 3491 hns3_force_clear_all_rx_ring(handle);
3492
76ad4f0e
S		 3493 ret = hns3_nic_uninit_vector_data(priv);
3494 if (ret)
3495 netdev_err(netdev, "uninit vector error\n");
3496
dd38c726
YL		 3497 ret = hns3_nic_dealloc_vector_data(priv);
3498 if (ret)
3499 netdev_err(netdev, "dealloc vector error\n");
3500
76ad4f0e
S		 3501 ret = hns3_uninit_all_ring(priv);
3502 if (ret)
3503 netdev_err(netdev, "uninit ring error\n");
3504
ec777890
YL		 3505 hns3_put_ring_config(priv);
3506
76ad4f0e
S		 3507 priv->ring_data = NULL;
3508
814da63c 3509out_netdev_free:
76ad4f0e
S		 3510 free_netdev(netdev);
3511}
3512
3513static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3514{
3515 struct net_device *netdev = handle->kinfo.netdev;
3516
3517 if (!netdev)
3518 return;
3519
3520 if (linkup) {
3521 netif_carrier_on(netdev);
3522 netif_tx_wake_all_queues(netdev);
3523 netdev_info(netdev, "link up\n");
3524 } else {
3525 netif_carrier_off(netdev);
3526 netif_tx_stop_all_queues(netdev);
3527 netdev_info(netdev, "link down\n");
3528 }
3529}
3530
9df8f79a
YL		 3531static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3532{
3533 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3534 struct net_device *ndev = kinfo->netdev;
075cfdd6 3535 bool if_running;
9df8f79a 3536 int ret;
9df8f79a
YL		 3537
3538 if (tc > HNAE3_MAX_TC)
3539 return -EINVAL;
3540
3541 if (!ndev)
3542 return -ENODEV;
3543
075cfdd6
CIK		 3544 if_running = netif_running(ndev);
3545
9df8f79a
YL		 3546 if (if_running) {
3547 (void)hns3_nic_net_stop(ndev);
3548 msleep(100);
3549 }
3550
3551 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3552 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3553 if (ret)
3554 goto err_out;
3555
9df8f79a
YL		 3556 ret = hns3_nic_set_real_num_queue(ndev);
3557
3558err_out:
3559 if (if_running)
3560 (void)hns3_nic_net_open(ndev);
3561
3562 return ret;
3563}
3564
7fa6be4f 3565static int hns3_recover_hw_addr(struct net_device *ndev)
bb6b94a8
L		 3566{
3567 struct netdev_hw_addr_list *list;
3568 struct netdev_hw_addr *ha, *tmp;
7fa6be4f 3569 int ret = 0;
bb6b94a8
L		 3570
3571 /* go through and sync uc_addr entries to the device */
3572 list = &ndev->uc;
7fa6be4f
HT		 3573 list_for_each_entry_safe(ha, tmp, &list->list, list) {
3574 ret = hns3_nic_uc_sync(ndev, ha->addr);
3575 if (ret)
3576 return ret;
3577 }
bb6b94a8
L		 3578
3579 /* go through and sync mc_addr entries to the device */
3580 list = &ndev->mc;
7fa6be4f
HT		 3581 list_for_each_entry_safe(ha, tmp, &list->list, list) {
3582 ret = hns3_nic_mc_sync(ndev, ha->addr);
3583 if (ret)
3584 return ret;
3585 }
3586
3587 return ret;
bb6b94a8
L		 3588}
3589
f05e2109
JS		 3590static void hns3_remove_hw_addr(struct net_device *netdev)
3591{
3592 struct netdev_hw_addr_list *list;
3593 struct netdev_hw_addr *ha, *tmp;
3594
3595 hns3_nic_uc_unsync(netdev, netdev->dev_addr);
3596
3597 /* go through and unsync uc_addr entries to the device */
3598 list = &netdev->uc;
3599 list_for_each_entry_safe(ha, tmp, &list->list, list)
3600 hns3_nic_uc_unsync(netdev, ha->addr);
3601
3602 /* go through and unsync mc_addr entries to the device */
3603 list = &netdev->mc;
3604 list_for_each_entry_safe(ha, tmp, &list->list, list)
3605 if (ha->refcount > 1)
3606 hns3_nic_mc_unsync(netdev, ha->addr);
3607}
3608
beebca3a 3609static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
bb6b94a8 3610{
beebca3a 3611 while (ring->next_to_clean != ring->next_to_use) {
7b763f3f 3612 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
beebca3a
YL		 3613 hns3_free_buffer_detach(ring, ring->next_to_clean);
3614 ring_ptr_move_fw(ring, next_to_clean);
3615 }
3616}
3617
7b763f3f
FL		 3618static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3619{
3620 struct hns3_desc_cb res_cbs;
3621 int ret;
3622
3623 while (ring->next_to_use != ring->next_to_clean) {
3624 /* When a buffer is not reused, it's memory has been
3625 * freed in hns3_handle_rx_bd or will be freed by
3626 * stack, so we need to replace the buffer here.
3627 */
3628 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3629 ret = hns3_reserve_buffer_map(ring, &res_cbs);
3630 if (ret) {
3631 u64_stats_update_begin(&ring->syncp);
3632 ring->stats.sw_err_cnt++;
3633 u64_stats_update_end(&ring->syncp);
3634 /* if alloc new buffer fail, exit directly
3635 * and reclear in up flow.
3636 */
3637 netdev_warn(ring->tqp->handle->kinfo.netdev,
3638 "reserve buffer map failed, ret = %d\n",
3639 ret);
3640 return ret;
3641 }
3642 hns3_replace_buffer(ring, ring->next_to_use,
3643 &res_cbs);
3644 }
3645 ring_ptr_move_fw(ring, next_to_use);
3646 }
3647
3648 return 0;
3649}
3650
3651static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
beebca3a 3652{
beebca3a
YL		 3653 while (ring->next_to_use != ring->next_to_clean) {
3654 /* When a buffer is not reused, it's memory has been
3655 * freed in hns3_handle_rx_bd or will be freed by
3656 * stack, so only need to unmap the buffer here.
3657 */
3658 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3659 hns3_unmap_buffer(ring,
3660 &ring->desc_cb[ring->next_to_use]);
3661 ring->desc_cb[ring->next_to_use].dma = 0;
3662 }
3663
3664 ring_ptr_move_fw(ring, next_to_use);
3665 }
bb6b94a8
L		 3666}
3667
7b763f3f
FL		 3668static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3669{
3670 struct net_device *ndev = h->kinfo.netdev;
3671 struct hns3_nic_priv *priv = netdev_priv(ndev);
3672 struct hns3_enet_ring *ring;
3673 u32 i;
3674
3675 for (i = 0; i < h->kinfo.num_tqps; i++) {
3676 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3677 hns3_force_clear_rx_ring(ring);
3678 }
3679}
3680
bb6b94a8
L		 3681static void hns3_clear_all_ring(struct hnae3_handle *h)
3682{
3683 struct net_device *ndev = h->kinfo.netdev;
3684 struct hns3_nic_priv *priv = netdev_priv(ndev);
3685 u32 i;
3686
3687 for (i = 0; i < h->kinfo.num_tqps; i++) {
3688 struct netdev_queue *dev_queue;
3689 struct hns3_enet_ring *ring;
3690
3691 ring = priv->ring_data[i].ring;
beebca3a 3692 hns3_clear_tx_ring(ring);
bb6b94a8
L		 3693 dev_queue = netdev_get_tx_queue(ndev,
3694 priv->ring_data[i].queue_index);
3695 netdev_tx_reset_queue(dev_queue);
3696
3697 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
7b763f3f
FL		 3698 /* Continue to clear other rings even if clearing some
3699 * rings failed.
3700 */
beebca3a 3701 hns3_clear_rx_ring(ring);
bb6b94a8
L		 3702 }
3703}
3704
7b763f3f
FL		 3705int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3706{
3707 struct net_device *ndev = h->kinfo.netdev;
3708 struct hns3_nic_priv *priv = netdev_priv(ndev);
3709 struct hns3_enet_ring *rx_ring;
3710 int i, j;
3711 int ret;
3712
3713 for (i = 0; i < h->kinfo.num_tqps; i++) {
7fa6be4f
HT		 3714 ret = h->ae_algo->ops->reset_queue(h, i);
3715 if (ret)
3716 return ret;
3717
7b763f3f
FL		 3718 hns3_init_ring_hw(priv->ring_data[i].ring);
3719
3720 /* We need to clear tx ring here because self test will
3721 * use the ring and will not run down before up
3722 */
3723 hns3_clear_tx_ring(priv->ring_data[i].ring);
3724 priv->ring_data[i].ring->next_to_clean = 0;
3725 priv->ring_data[i].ring->next_to_use = 0;
3726
3727 rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3728 hns3_init_ring_hw(rx_ring);
3729 ret = hns3_clear_rx_ring(rx_ring);
3730 if (ret)
3731 return ret;
3732
3733 /* We can not know the hardware head and tail when this
3734 * function is called in reset flow, so we reuse all desc.
3735 */
3736 for (j = 0; j < rx_ring->desc_num; j++)
3737 hns3_reuse_buffer(rx_ring, j);
3738
3739 rx_ring->next_to_clean = 0;
3740 rx_ring->next_to_use = 0;
3741 }
3742
1c772154
YL		 3743 hns3_init_tx_ring_tc(priv);
3744
7b763f3f
FL		 3745 return 0;
3746}
3747
e4fd7502
HT		 3748static void hns3_store_coal(struct hns3_nic_priv *priv)
3749{
3750 /* ethtool only support setting and querying one coal
3751 * configuation for now, so save the vector 0' coal
3752 * configuation here in order to restore it.
3753 */
3754 memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
3755 sizeof(struct hns3_enet_coalesce));
3756 memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
3757 sizeof(struct hns3_enet_coalesce));
3758}
3759
3760static void hns3_restore_coal(struct hns3_nic_priv *priv)
3761{
3762 u16 vector_num = priv->vector_num;
3763 int i;
3764
3765 for (i = 0; i < vector_num; i++) {
3766 memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
3767 sizeof(struct hns3_enet_coalesce));
3768 memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
3769 sizeof(struct hns3_enet_coalesce));
3770 }
3771}
3772
bb6b94a8
L		 3773static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3774{
7edff533 3775 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);
bb6b94a8
L		 3776 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3777 struct net_device *ndev = kinfo->netdev;
257e4f29
HT		 3778 struct hns3_nic_priv *priv = netdev_priv(ndev);
3779
3780 if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))
3781 return 0;
bb6b94a8 3782
7edff533
HT		 3783 /* it is cumbersome for hardware to pick-and-choose entries for deletion
3784 * from table space. Hence, for function reset software intervention is
3785 * required to delete the entries
3786 */
3787 if (hns3_dev_ongoing_func_reset(ae_dev)) {
3788 hns3_remove_hw_addr(ndev);
3789 hns3_del_all_fd_rules(ndev, false);
3790 }
3791
bb6b94a8 3792 if (!netif_running(ndev))
6b1385cc 3793 return 0;
bb6b94a8
L		 3794
3795 return hns3_nic_net_stop(ndev);
3796}
3797
3798static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3799{
3800 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
257e4f29 3801 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
bb6b94a8
L		 3802 int ret = 0;
3803
3804 if (netif_running(kinfo->netdev)) {
3805 ret = hns3_nic_net_up(kinfo->netdev);
3806 if (ret) {
3807 netdev_err(kinfo->netdev,
3808 "hns net up fail, ret=%d!\n", ret);
3809 return ret;
3810 }
bb6b94a8
L		 3811 }
3812
257e4f29
HT		 3813 clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
3814
bb6b94a8
L		 3815 return ret;
3816}
3817
3818static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3819{
3820 struct net_device *netdev = handle->kinfo.netdev;
3821 struct hns3_nic_priv *priv = netdev_priv(netdev);
7325523a 3822 bool vlan_filter_enable;
bb6b94a8
L		 3823 int ret;
3824
7fa6be4f
HT		 3825 ret = hns3_init_mac_addr(netdev, false);
3826 if (ret)
3827 return ret;
3828
3829 ret = hns3_recover_hw_addr(netdev);
3830 if (ret)
3831 return ret;
3832
3833 ret = hns3_update_promisc_mode(netdev, handle->netdev_flags);
3834 if (ret)
3835 return ret;
3836
7325523a
JS		 3837 vlan_filter_enable = netdev->flags & IFF_PROMISC ? false : true;
3838 hns3_enable_vlan_filter(netdev, vlan_filter_enable);
3839
681ec399 3840 /* Hardware table is only clear when pf resets */
7fa6be4f
HT		 3841 if (!(handle->flags & HNAE3_SUPPORT_VF)) {
3842 ret = hns3_restore_vlan(netdev);
e12c2252
HT		 3843 if (ret)
3844 return ret;
7fa6be4f 3845 }
681ec399 3846
7fa6be4f
HT		 3847 ret = hns3_restore_fd_rules(netdev);
3848 if (ret)
3849 return ret;
6871af29 3850
bb6b94a8
L		 3851 /* Carrier off reporting is important to ethtool even BEFORE open */
3852 netif_carrier_off(netdev);
3853
862d969a
HT		 3854 ret = hns3_nic_alloc_vector_data(priv);
3855 if (ret)
3856 return ret;
3857
e4fd7502
HT		 3858 hns3_restore_coal(priv);
3859
bb6b94a8
L		 3860 ret = hns3_nic_init_vector_data(priv);
3861 if (ret)
862d969a 3862 goto err_dealloc_vector;
bb6b94a8
L		 3863
3864 ret = hns3_init_all_ring(priv);
862d969a
HT		 3865 if (ret)
3866 goto err_uninit_vector;
bb6b94a8 3867
814da63c
HT		 3868 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
3869
862d969a
HT		 3870 return ret;
3871
3872err_uninit_vector:
3873 hns3_nic_uninit_vector_data(priv);
3874 priv->ring_data = NULL;
3875err_dealloc_vector:
3876 hns3_nic_dealloc_vector_data(priv);
3877
bb6b94a8
L		 3878 return ret;
3879}
3880
3881static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3882{
3883 struct net_device *netdev = handle->kinfo.netdev;
3884 struct hns3_nic_priv *priv = netdev_priv(netdev);
3885 int ret;
3886
814da63c
HT		 3887 if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
3888 netdev_warn(netdev, "already uninitialized\n");
3889 return 0;
3890 }
3891
7b763f3f 3892 hns3_force_clear_all_rx_ring(handle);
bb6b94a8
L		 3893
3894 ret = hns3_nic_uninit_vector_data(priv);
3895 if (ret) {
3896 netdev_err(netdev, "uninit vector error\n");
3897 return ret;
3898 }
3899
e4fd7502
HT		 3900 hns3_store_coal(priv);
3901
862d969a
HT		 3902 ret = hns3_nic_dealloc_vector_data(priv);
3903 if (ret)
3904 netdev_err(netdev, "dealloc vector error\n");
3905
bb6b94a8
L		 3906 ret = hns3_uninit_all_ring(priv);
3907 if (ret)
3908 netdev_err(netdev, "uninit ring error\n");
3909
814da63c
HT		 3910 clear_bit(HNS3_NIC_STATE_INITED, &priv->state);
3911
bb6b94a8
L		 3912 return ret;
3913}
3914
3915static int hns3_reset_notify(struct hnae3_handle *handle,
3916 enum hnae3_reset_notify_type type)
3917{
3918 int ret = 0;
3919
3920 switch (type) {
3921 case HNAE3_UP_CLIENT:
e1586241
SM		 3922 ret = hns3_reset_notify_up_enet(handle);
3923 break;
bb6b94a8
L		 3924 case HNAE3_DOWN_CLIENT:
3925 ret = hns3_reset_notify_down_enet(handle);
3926 break;
3927 case HNAE3_INIT_CLIENT:
3928 ret = hns3_reset_notify_init_enet(handle);
3929 break;
3930 case HNAE3_UNINIT_CLIENT:
3931 ret = hns3_reset_notify_uninit_enet(handle);
3932 break;
3933 default:
3934 break;
3935 }
3936
3937 return ret;
3938}
3939
e4fd7502 3940static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
09f2af64
PL		 3941{
3942 struct hns3_nic_priv *priv = netdev_priv(netdev);
3943 struct hnae3_handle *h = hns3_get_handle(netdev);
3944 int ret;
3945
3946 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3947 if (ret)
3948 return ret;
3949
3950 ret = hns3_get_ring_config(priv);
3951 if (ret)
3952 return ret;
3953
dd38c726
YL		 3954 ret = hns3_nic_alloc_vector_data(priv);
3955 if (ret)
3956 goto err_alloc_vector;
3957
e4fd7502 3958 hns3_restore_coal(priv);
7a242b23 3959
09f2af64
PL		 3960 ret = hns3_nic_init_vector_data(priv);
3961 if (ret)
3962 goto err_uninit_vector;
3963
3964 ret = hns3_init_all_ring(priv);
3965 if (ret)
3966 goto err_put_ring;
3967
3968 return 0;
3969
3970err_put_ring:
3971 hns3_put_ring_config(priv);
3972err_uninit_vector:
3973 hns3_nic_uninit_vector_data(priv);
dd38c726
YL		 3974err_alloc_vector:
3975 hns3_nic_dealloc_vector_data(priv);
09f2af64
PL		 3976 return ret;
3977}
3978
3979static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3980{
3981 return (new_tqp_num / num_tc) * num_tc;
3982}
3983
3984int hns3_set_channels(struct net_device *netdev,
3985 struct ethtool_channels *ch)
3986{
3987 struct hns3_nic_priv *priv = netdev_priv(netdev);
3988 struct hnae3_handle *h = hns3_get_handle(netdev);
3989 struct hnae3_knic_private_info *kinfo = &h->kinfo;
3990 bool if_running = netif_running(netdev);
3991 u32 new_tqp_num = ch->combined_count;
3992 u16 org_tqp_num;
3993 int ret;
3994
3995 if (ch->rx_count || ch->tx_count)
3996 return -EINVAL;
3997
678335a1 3998 if (new_tqp_num > hns3_get_max_available_channels(h) ||
09f2af64
PL		 3999 new_tqp_num < kinfo->num_tc) {
4000 dev_err(&netdev->dev,
4001 "Change tqps fail, the tqp range is from %d to %d",
4002 kinfo->num_tc,
678335a1 4003 hns3_get_max_available_channels(h));
09f2af64
PL		 4004 return -EINVAL;
4005 }
4006
4007 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
4008 if (kinfo->num_tqps == new_tqp_num)
4009 return 0;
4010
4011 if (if_running)
20e4bf98 4012 hns3_nic_net_stop(netdev);
09f2af64 4013
09f2af64
PL		 4014 ret = hns3_nic_uninit_vector_data(priv);
4015 if (ret) {
4016 dev_err(&netdev->dev,
4017 "Unbind vector with tqp fail, nothing is changed");
4018 goto open_netdev;
4019 }
4020
e4fd7502 4021 hns3_store_coal(priv);
7a242b23 4022
dd38c726
YL		 4023 hns3_nic_dealloc_vector_data(priv);
4024
09f2af64 4025 hns3_uninit_all_ring(priv);
ec777890 4026 hns3_put_ring_config(priv);
09f2af64
PL		 4027
4028 org_tqp_num = h->kinfo.num_tqps;
e4fd7502 4029 ret = hns3_modify_tqp_num(netdev, new_tqp_num);
09f2af64 4030 if (ret) {
e4fd7502 4031 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
09f2af64
PL		 4032 if (ret) {
4033 /* If revert to old tqp failed, fatal error occurred */
4034 dev_err(&netdev->dev,
4035 "Revert to old tqp num fail, ret=%d", ret);
4036 return ret;
4037 }
4038 dev_info(&netdev->dev,
4039 "Change tqp num fail, Revert to old tqp num");
4040 }
4041
4042open_netdev:
4043 if (if_running)
20e4bf98 4044 hns3_nic_net_open(netdev);
09f2af64
PL		 4045
4046 return ret;
4047}
4048
1db9b1bf 4049static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 4050 .init_instance = hns3_client_init,
4051 .uninit_instance = hns3_client_uninit,
4052 .link_status_change = hns3_link_status_change,
9df8f79a 4053 .setup_tc = hns3_client_setup_tc,
bb6b94a8 4054 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 4055};
4056
4057/* hns3_init_module - Driver registration routine
4058 * hns3_init_module is the first routine called when the driver is
4059 * loaded. All it does is register with the PCI subsystem.
4060 */
4061static int __init hns3_init_module(void)
4062{
4063 int ret;
4064
4065 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
4066 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
4067
4068 client.type = HNAE3_CLIENT_KNIC;
4069 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
4070 hns3_driver_name);
4071
4072 client.ops = &client_ops;
4073
13562d1f
XW		 4074 INIT_LIST_HEAD(&client.node);
4075
76ad4f0e
S		 4076 ret = hnae3_register_client(&client);
4077 if (ret)
4078 return ret;
4079
4080 ret = pci_register_driver(&hns3_driver);
4081 if (ret)
4082 hnae3_unregister_client(&client);
4083
4084 return ret;
4085}
4086module_init(hns3_init_module);
4087
4088/* hns3_exit_module - Driver exit cleanup routine
4089 * hns3_exit_module is called just before the driver is removed
4090 * from memory.
4091 */
4092static void __exit hns3_exit_module(void)
4093{
4094 pci_unregister_driver(&hns3_driver);
4095 hnae3_unregister_client(&client);
4096}
4097module_exit(hns3_exit_module);
4098
4099MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
4100MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
4101MODULE_LICENSE("GPL");
4102MODULE_ALIAS("pci:hns-nic");
3c7624d8 4103MODULE_VERSION(HNS3_MOD_VERSION);
Linux 6.x block layer and io_uring tree(s)