projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: Add support for ethtool -K to enable/disable HW GRO
[linux-block.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
d71d8381
JS		 1// SPDX-License-Identifier: GPL-2.0+
2// Copyright (c) 2016-2017 Hisilicon Limited.
76ad4f0e
S		 3
4#include <linux/dma-mapping.h>
5#include <linux/etherdevice.h>
6#include <linux/interrupt.h>
7#include <linux/if_vlan.h>
8#include <linux/ip.h>
9#include <linux/ipv6.h>
10#include <linux/module.h>
11#include <linux/pci.h>
6ae4e733 12#include <linux/aer.h>
76ad4f0e
S		 13#include <linux/skbuff.h>
14#include <linux/sctp.h>
15#include <linux/vermagic.h>
16#include <net/gre.h>
30d240df 17#include <net/pkt_cls.h>
76ad4f0e
S		 18#include <net/vxlan.h>
19
20#include "hnae3.h"
21#include "hns3_enet.h"
22
7b763f3f
FL		 23static void hns3_clear_all_ring(struct hnae3_handle *h);
24static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h);
f05e2109 25static void hns3_remove_hw_addr(struct net_device *netdev);
7b763f3f 26
1db9b1bf 27static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 28const char hns3_driver_version[] = VERMAGIC_STRING;
29static const char hns3_driver_string[] =
30 "Hisilicon Ethernet Network Driver for Hip08 Family";
31static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
32static struct hnae3_client client;
33
34/* hns3_pci_tbl - PCI Device ID Table
35 *
36 * Last entry must be all 0s
37 *
38 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
39 * Class, Class Mask, private data (not used) }
40 */
41static const struct pci_device_id hns3_pci_tbl[] = {
42 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 45 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 46 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 47 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 48 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 49 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 50 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 51 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 52 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 53 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
424eb834 54 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
07acf909
JS		 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF),
56 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
76ad4f0e
S		 57 /* required last entry */
58 {0, }
59};
60MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
ef0c5009 62static irqreturn_t hns3_irq_handle(int irq, void *vector)
76ad4f0e 63{
ef0c5009 64 struct hns3_enet_tqp_vector *tqp_vector = vector;
76ad4f0e
S		 65
66 napi_schedule(&tqp_vector->napi);
67
68 return IRQ_HANDLED;
69}
70
874bff0b
PL		 71/* This callback function is used to set affinity changes to the irq affinity
72 * masks when the irq_set_affinity_notifier function is used.
73 */
74static void hns3_nic_irq_affinity_notify(struct irq_affinity_notify *notify,
75 const cpumask_t *mask)
76{
77 struct hns3_enet_tqp_vector *tqp_vectors =
78 container_of(notify, struct hns3_enet_tqp_vector,
79 affinity_notify);
80
81 tqp_vectors->affinity_mask = *mask;
82}
83
84static void hns3_nic_irq_affinity_release(struct kref *ref)
85{
86}
87
76ad4f0e
S		 88static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
89{
90 struct hns3_enet_tqp_vector *tqp_vectors;
91 unsigned int i;
92
93 for (i = 0; i < priv->vector_num; i++) {
94 tqp_vectors = &priv->tqp_vector[i];
95
96 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
97 continue;
98
874bff0b
PL		 99 /* clear the affinity notifier and affinity mask */
100 irq_set_affinity_notifier(tqp_vectors->vector_irq, NULL);
101 irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);
102
76ad4f0e
S		 103 /* release the irq resource */
104 free_irq(tqp_vectors->vector_irq, tqp_vectors);
105 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
106 }
107}
108
109static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
110{
111 struct hns3_enet_tqp_vector *tqp_vectors;
112 int txrx_int_idx = 0;
113 int rx_int_idx = 0;
114 int tx_int_idx = 0;
115 unsigned int i;
116 int ret;
117
118 for (i = 0; i < priv->vector_num; i++) {
119 tqp_vectors = &priv->tqp_vector[i];
120
121 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
122 continue;
123
124 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
125 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
126 "%s-%s-%d", priv->netdev->name, "TxRx",
127 txrx_int_idx++);
128 txrx_int_idx++;
129 } else if (tqp_vectors->rx_group.ring) {
130 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
131 "%s-%s-%d", priv->netdev->name, "Rx",
132 rx_int_idx++);
133 } else if (tqp_vectors->tx_group.ring) {
134 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
135 "%s-%s-%d", priv->netdev->name, "Tx",
136 tx_int_idx++);
137 } else {
138 /* Skip this unused q_vector */
139 continue;
140 }
141
142 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
143
144 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
145 tqp_vectors->name,
146 tqp_vectors);
147 if (ret) {
148 netdev_err(priv->netdev, "request irq(%d) fail\n",
149 tqp_vectors->vector_irq);
150 return ret;
151 }
152
874bff0b
PL		 153 tqp_vectors->affinity_notify.notify =
154 hns3_nic_irq_affinity_notify;
155 tqp_vectors->affinity_notify.release =
156 hns3_nic_irq_affinity_release;
157 irq_set_affinity_notifier(tqp_vectors->vector_irq,
158 &tqp_vectors->affinity_notify);
159 irq_set_affinity_hint(tqp_vectors->vector_irq,
160 &tqp_vectors->affinity_mask);
161
76ad4f0e
S		 162 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
163 }
164
165 return 0;
166}
167
168static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
169 u32 mask_en)
170{
171 writel(mask_en, tqp_vector->mask_addr);
172}
173
174static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
175{
176 napi_enable(&tqp_vector->napi);
177
178 /* enable vector */
179 hns3_mask_vector_irq(tqp_vector, 1);
180}
181
182static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
183{
184 /* disable vector */
185 hns3_mask_vector_irq(tqp_vector, 0);
186
187 disable_irq(tqp_vector->vector_irq);
188 napi_disable(&tqp_vector->napi);
189}
190
434776a5
FL		 191void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,
192 u32 rl_value)
76ad4f0e 193{
434776a5
FL		 194 u32 rl_reg = hns3_rl_usec_to_reg(rl_value);
195
76ad4f0e
S		 196 /* this defines the configuration for RL (Interrupt Rate Limiter).
197 * Rl defines rate of interrupts i.e. number of interrupts-per-second
198 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
199 */
434776a5 200
9bc727a9
YL		 201 if (rl_reg > 0 && !tqp_vector->tx_group.coal.gl_adapt_enable &&
202 !tqp_vector->rx_group.coal.gl_adapt_enable)
434776a5
FL		 203 /* According to the hardware, the range of rl_reg is
204 * 0-59 and the unit is 4.
205 */
206 rl_reg |= HNS3_INT_RL_ENABLE_MASK;
207
208 writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
209}
210
211void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,
212 u32 gl_value)
213{
214 u32 rx_gl_reg = hns3_gl_usec_to_reg(gl_value);
215
216 writel(rx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
217}
218
219void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,
220 u32 gl_value)
221{
222 u32 tx_gl_reg = hns3_gl_usec_to_reg(gl_value);
223
224 writel(tx_gl_reg, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
76ad4f0e
S		 225}
226
5fd4789a
FL		 227static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector,
228 struct hns3_nic_priv *priv)
76ad4f0e
S		 229{
230 /* initialize the configuration for interrupt coalescing.
231 * 1. GL (Interrupt Gap Limiter)
232 * 2. RL (Interrupt Rate Limiter)
233 */
234
5fd4789a 235 /* Default: enable interrupt coalescing self-adaptive and GL */
9bc727a9
YL		 236 tqp_vector->tx_group.coal.gl_adapt_enable = 1;
237 tqp_vector->rx_group.coal.gl_adapt_enable = 1;
5fd4789a 238
9bc727a9
YL		 239 tqp_vector->tx_group.coal.int_gl = HNS3_INT_GL_50K;
240 tqp_vector->rx_group.coal.int_gl = HNS3_INT_GL_50K;
5fd4789a 241
cd9d187b 242 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
9bc727a9
YL		 243 tqp_vector->rx_group.coal.flow_level = HNS3_FLOW_LOW;
244 tqp_vector->tx_group.coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 245}
246
dd38c726
YL		 247static void hns3_vector_gl_rl_init_hw(struct hns3_enet_tqp_vector *tqp_vector,
248 struct hns3_nic_priv *priv)
249{
250 struct hnae3_handle *h = priv->ae_handle;
251
252 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 253 tqp_vector->tx_group.coal.int_gl);
dd38c726 254 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 255 tqp_vector->rx_group.coal.int_gl);
dd38c726
YL		 256 hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);
257}
258
9df8f79a
YL		 259static int hns3_nic_set_real_num_queue(struct net_device *netdev)
260{
9780cb97 261 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 262 struct hnae3_knic_private_info *kinfo = &h->kinfo;
263 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
a75a8efa
YL		 264 int i, ret;
265
266 if (kinfo->num_tc <= 1) {
267 netdev_reset_tc(netdev);
268 } else {
269 ret = netdev_set_num_tc(netdev, kinfo->num_tc);
270 if (ret) {
271 netdev_err(netdev,
272 "netdev_set_num_tc fail, ret=%d!\n", ret);
273 return ret;
274 }
275
276 for (i = 0; i < HNAE3_MAX_TC; i++) {
277 if (!kinfo->tc_info[i].enable)
278 continue;
279
280 netdev_set_tc_queue(netdev,
281 kinfo->tc_info[i].tc,
282 kinfo->tc_info[i].tqp_count,
283 kinfo->tc_info[i].tqp_offset);
284 }
285 }
9df8f79a
YL		 286
287 ret = netif_set_real_num_tx_queues(netdev, queue_size);
288 if (ret) {
289 netdev_err(netdev,
290 "netif_set_real_num_tx_queues fail, ret=%d!\n",
291 ret);
292 return ret;
293 }
294
295 ret = netif_set_real_num_rx_queues(netdev, queue_size);
296 if (ret) {
297 netdev_err(netdev,
298 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
299 return ret;
300 }
301
302 return 0;
303}
304
678335a1
PL		 305static u16 hns3_get_max_available_channels(struct hnae3_handle *h)
306{
0d43bf45 307 u16 alloc_tqps, max_rss_size, rss_size;
678335a1 308
0d43bf45
HT		 309 h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);
310 rss_size = alloc_tqps / h->kinfo.num_tc;
678335a1 311
0d43bf45 312 return min_t(u16, rss_size, max_rss_size);
678335a1
PL		 313}
314
8df0fa91
HT		 315static void hns3_tqp_enable(struct hnae3_queue *tqp)
316{
317 u32 rcb_reg;
318
319 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
320 rcb_reg |= BIT(HNS3_RING_EN_B);
321 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
322}
323
324static void hns3_tqp_disable(struct hnae3_queue *tqp)
325{
326 u32 rcb_reg;
327
328 rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);
329 rcb_reg &= ~BIT(HNS3_RING_EN_B);
330 hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);
331}
332
76ad4f0e
S		 333static int hns3_nic_net_up(struct net_device *netdev)
334{
335 struct hns3_nic_priv *priv = netdev_priv(netdev);
336 struct hnae3_handle *h = priv->ae_handle;
337 int i, j;
338 int ret;
339
7b763f3f
FL		 340 ret = hns3_nic_reset_all_ring(h);
341 if (ret)
342 return ret;
343
76ad4f0e
S		 344 /* get irq resource for all vectors */
345 ret = hns3_nic_init_irq(priv);
346 if (ret) {
347 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
348 return ret;
349 }
350
351 /* enable the vectors */
352 for (i = 0; i < priv->vector_num; i++)
353 hns3_vector_enable(&priv->tqp_vector[i]);
354
8df0fa91
HT		 355 /* enable rcb */
356 for (j = 0; j < h->kinfo.num_tqps; j++)
357 hns3_tqp_enable(h->kinfo.tqp[j]);
358
76ad4f0e
S		 359 /* start the ae_dev */
360 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
361 if (ret)
362 goto out_start_err;
363
b875cc37
JS		 364 clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);
365
76ad4f0e
S		 366 return 0;
367
368out_start_err:
8df0fa91
HT		 369 while (j--)
370 hns3_tqp_disable(h->kinfo.tqp[j]);
371
76ad4f0e
S		 372 for (j = i - 1; j >= 0; j--)
373 hns3_vector_disable(&priv->tqp_vector[j]);
374
375 hns3_nic_uninit_irq(priv);
376
377 return ret;
378}
379
380static int hns3_nic_net_open(struct net_device *netdev)
381{
a75a8efa
YL		 382 struct hnae3_handle *h = hns3_get_handle(netdev);
383 struct hnae3_knic_private_info *kinfo;
384 int i, ret;
76ad4f0e 385
257e4f29
HT		 386 if (hns3_nic_resetting(netdev))
387 return -EBUSY;
388
76ad4f0e
S		 389 netif_carrier_off(netdev);
390
9df8f79a
YL		 391 ret = hns3_nic_set_real_num_queue(netdev);
392 if (ret)
76ad4f0e 393 return ret;
76ad4f0e
S		 394
395 ret = hns3_nic_net_up(netdev);
396 if (ret) {
397 netdev_err(netdev,
398 "hns net up fail, ret=%d!\n", ret);
399 return ret;
400 }
401
a75a8efa
YL		 402 kinfo = &h->kinfo;
403 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
404 netdev_set_prio_tc_map(netdev, i,
405 kinfo->prio_tc[i]);
406 }
407
76ad4f0e
S		 408 return 0;
409}
410
411static void hns3_nic_net_down(struct net_device *netdev)
412{
413 struct hns3_nic_priv *priv = netdev_priv(netdev);
8df0fa91 414 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 415 const struct hnae3_ae_ops *ops;
416 int i;
417
7b763f3f
FL		 418 /* disable vectors */
419 for (i = 0; i < priv->vector_num; i++)
420 hns3_vector_disable(&priv->tqp_vector[i]);
8df0fa91
HT		 421
422 /* disable rcb */
423 for (i = 0; i < h->kinfo.num_tqps; i++)
424 hns3_tqp_disable(h->kinfo.tqp[i]);
7b763f3f 425
76ad4f0e
S		 426 /* stop ae_dev */
427 ops = priv->ae_handle->ae_algo->ops;
428 if (ops->stop)
429 ops->stop(priv->ae_handle);
430
76ad4f0e
S		 431 /* free irq resources */
432 hns3_nic_uninit_irq(priv);
7b763f3f
FL		 433
434 hns3_clear_all_ring(priv->ae_handle);
76ad4f0e
S		 435}
436
437static int hns3_nic_net_stop(struct net_device *netdev)
438{
ff0699e0
HT		 439 struct hns3_nic_priv *priv = netdev_priv(netdev);
440
441 if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))
442 return 0;
443
76ad4f0e
S		 444 netif_tx_stop_all_queues(netdev);
445 netif_carrier_off(netdev);
446
447 hns3_nic_net_down(netdev);
448
449 return 0;
450}
451
76ad4f0e
S		 452static int hns3_nic_uc_sync(struct net_device *netdev,
453 const unsigned char *addr)
454{
9780cb97 455 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 456
457 if (h->ae_algo->ops->add_uc_addr)
458 return h->ae_algo->ops->add_uc_addr(h, addr);
459
460 return 0;
461}
462
463static int hns3_nic_uc_unsync(struct net_device *netdev,
464 const unsigned char *addr)
465{
9780cb97 466 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 467
468 if (h->ae_algo->ops->rm_uc_addr)
469 return h->ae_algo->ops->rm_uc_addr(h, addr);
470
471 return 0;
472}
473
474static int hns3_nic_mc_sync(struct net_device *netdev,
475 const unsigned char *addr)
476{
9780cb97 477 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 478
720a8478 479 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 480 return h->ae_algo->ops->add_mc_addr(h, addr);
481
482 return 0;
483}
484
485static int hns3_nic_mc_unsync(struct net_device *netdev,
486 const unsigned char *addr)
487{
9780cb97 488 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 489
720a8478 490 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 491 return h->ae_algo->ops->rm_mc_addr(h, addr);
492
493 return 0;
494}
495
c60edc17
JS		 496static u8 hns3_get_netdev_flags(struct net_device *netdev)
497{
498 u8 flags = 0;
499
500 if (netdev->flags & IFF_PROMISC) {
501 flags = HNAE3_USER_UPE | HNAE3_USER_MPE;
502 } else {
503 flags |= HNAE3_VLAN_FLTR;
504 if (netdev->flags & IFF_ALLMULTI)
505 flags |= HNAE3_USER_MPE;
506 }
507
508 return flags;
509}
510
1db9b1bf 511static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 512{
9780cb97 513 struct hnae3_handle *h = hns3_get_handle(netdev);
c60edc17
JS		 514 u8 new_flags;
515 int ret;
76ad4f0e 516
c60edc17
JS		 517 new_flags = hns3_get_netdev_flags(netdev);
518
519 ret = __dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);
520 if (ret) {
76ad4f0e 521 netdev_err(netdev, "sync uc address fail\n");
c60edc17
JS		 522 if (ret == -ENOSPC)
523 new_flags |= HNAE3_OVERFLOW_UPE;
524 }
525
40cca1c5 526 if (netdev->flags & IFF_MULTICAST) {
c60edc17
JS		 527 ret = __dev_mc_sync(netdev, hns3_nic_mc_sync,
528 hns3_nic_mc_unsync);
529 if (ret) {
76ad4f0e 530 netdev_err(netdev, "sync mc address fail\n");
c60edc17
JS		 531 if (ret == -ENOSPC)
532 new_flags |= HNAE3_OVERFLOW_MPE;
533 }
534 }
535
536 hns3_update_promisc_mode(netdev, new_flags);
537 /* User mode Promisc mode enable and vlan filtering is disabled to
538 * let all packets in. MAC-VLAN Table overflow Promisc enabled and
539 * vlan fitering is enabled
540 */
541 hns3_enable_vlan_filter(netdev, new_flags & HNAE3_VLAN_FLTR);
542 h->netdev_flags = new_flags;
543}
544
7fa6be4f 545int hns3_update_promisc_mode(struct net_device *netdev, u8 promisc_flags)
c60edc17
JS		 546{
547 struct hns3_nic_priv *priv = netdev_priv(netdev);
548 struct hnae3_handle *h = priv->ae_handle;
549
550 if (h->ae_algo->ops->set_promisc_mode) {
7fa6be4f
HT		 551 return h->ae_algo->ops->set_promisc_mode(h,
552 promisc_flags & HNAE3_UPE,
553 promisc_flags & HNAE3_MPE);
c60edc17 554 }
7fa6be4f
HT		 555
556 return 0;
c60edc17
JS		 557}
558
559void hns3_enable_vlan_filter(struct net_device *netdev, bool enable)
560{
561 struct hns3_nic_priv *priv = netdev_priv(netdev);
562 struct hnae3_handle *h = priv->ae_handle;
563 bool last_state;
564
565 if (h->pdev->revision >= 0x21 && h->ae_algo->ops->enable_vlan_filter) {
566 last_state = h->netdev_flags & HNAE3_VLAN_FLTR ? true : false;
567 if (enable != last_state) {
568 netdev_info(netdev,
569 "%s vlan filter\n",
570 enable ? "enable" : "disable");
571 h->ae_algo->ops->enable_vlan_filter(h, enable);
572 }
40cca1c5 573 }
76ad4f0e
S		 574}
575
576static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
577 u16 *mss, u32 *type_cs_vlan_tso)
578{
579 u32 l4_offset, hdr_len;
580 union l3_hdr_info l3;
581 union l4_hdr_info l4;
582 u32 l4_paylen;
583 int ret;
584
585 if (!skb_is_gso(skb))
586 return 0;
587
588 ret = skb_cow_head(skb, 0);
589 if (ret)
590 return ret;
591
592 l3.hdr = skb_network_header(skb);
593 l4.hdr = skb_transport_header(skb);
594
595 /* Software should clear the IPv4's checksum field when tso is
596 * needed.
597 */
598 if (l3.v4->version == 4)
599 l3.v4->check = 0;
600
601 /* tunnel packet.*/
602 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
603 SKB_GSO_GRE_CSUM |
604 SKB_GSO_UDP_TUNNEL |
605 SKB_GSO_UDP_TUNNEL_CSUM)) {
606 if ((!(skb_shinfo(skb)->gso_type &
607 SKB_GSO_PARTIAL)) &&
608 (skb_shinfo(skb)->gso_type &
609 SKB_GSO_UDP_TUNNEL_CSUM)) {
610 /* Software should clear the udp's checksum
611 * field when tso is needed.
612 */
613 l4.udp->check = 0;
614 }
615 /* reset l3&l4 pointers from outer to inner headers */
616 l3.hdr = skb_inner_network_header(skb);
617 l4.hdr = skb_inner_transport_header(skb);
618
619 /* Software should clear the IPv4's checksum field when
620 * tso is needed.
621 */
622 if (l3.v4->version == 4)
623 l3.v4->check = 0;
624 }
625
626 /* normal or tunnel packet*/
627 l4_offset = l4.hdr - skb->data;
628 hdr_len = (l4.tcp->doff * 4) + l4_offset;
629
630 /* remove payload length from inner pseudo checksum when tso*/
631 l4_paylen = skb->len - l4_offset;
632 csum_replace_by_diff(&l4.tcp->check,
633 (__force __wsum)htonl(l4_paylen));
634
635 /* find the txbd field values */
636 *paylen = skb->len - hdr_len;
e4e87715
PL		 637 hnae3_set_bit(*type_cs_vlan_tso,
638 HNS3_TXD_TSO_B, 1);
76ad4f0e
S		 639
640 /* get MSS for TSO */
641 *mss = skb_shinfo(skb)->gso_size;
642
643 return 0;
644}
645
1898d4e4
S		 646static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
647 u8 *il4_proto)
76ad4f0e
S		 648{
649 union {
650 struct iphdr *v4;
651 struct ipv6hdr *v6;
652 unsigned char *hdr;
653 } l3;
654 unsigned char *l4_hdr;
655 unsigned char *exthdr;
656 u8 l4_proto_tmp;
657 __be16 frag_off;
658
659 /* find outer header point */
660 l3.hdr = skb_network_header(skb);
35f58fd7 661 l4_hdr = skb_transport_header(skb);
76ad4f0e
S		 662
663 if (skb->protocol == htons(ETH_P_IPV6)) {
664 exthdr = l3.hdr + sizeof(*l3.v6);
665 l4_proto_tmp = l3.v6->nexthdr;
666 if (l4_hdr != exthdr)
667 ipv6_skip_exthdr(skb, exthdr - skb->data,
668 &l4_proto_tmp, &frag_off);
669 } else if (skb->protocol == htons(ETH_P_IP)) {
670 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 671 } else {
672 return -EINVAL;
76ad4f0e
S		 673 }
674
675 *ol4_proto = l4_proto_tmp;
676
677 /* tunnel packet */
678 if (!skb->encapsulation) {
679 *il4_proto = 0;
1898d4e4 680 return 0;
76ad4f0e
S		 681 }
682
683 /* find inner header point */
684 l3.hdr = skb_inner_network_header(skb);
685 l4_hdr = skb_inner_transport_header(skb);
686
687 if (l3.v6->version == 6) {
688 exthdr = l3.hdr + sizeof(*l3.v6);
689 l4_proto_tmp = l3.v6->nexthdr;
690 if (l4_hdr != exthdr)
691 ipv6_skip_exthdr(skb, exthdr - skb->data,
692 &l4_proto_tmp, &frag_off);
693 } else if (l3.v4->version == 4) {
694 l4_proto_tmp = l3.v4->protocol;
695 }
696
697 *il4_proto = l4_proto_tmp;
1898d4e4
S		 698
699 return 0;
76ad4f0e
S		 700}
701
702static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
703 u8 il4_proto, u32 *type_cs_vlan_tso,
704 u32 *ol_type_vlan_len_msec)
705{
706 union {
707 struct iphdr *v4;
708 struct ipv6hdr *v6;
709 unsigned char *hdr;
710 } l3;
711 union {
712 struct tcphdr *tcp;
713 struct udphdr *udp;
714 struct gre_base_hdr *gre;
715 unsigned char *hdr;
716 } l4;
717 unsigned char *l2_hdr;
718 u8 l4_proto = ol4_proto;
719 u32 ol2_len;
720 u32 ol3_len;
721 u32 ol4_len;
722 u32 l2_len;
723 u32 l3_len;
724
725 l3.hdr = skb_network_header(skb);
726 l4.hdr = skb_transport_header(skb);
727
728 /* compute L2 header size for normal packet, defined in 2 Bytes */
729 l2_len = l3.hdr - skb->data;
e4e87715
PL		 730 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
731 HNS3_TXD_L2LEN_S, l2_len >> 1);
76ad4f0e
S		 732
733 /* tunnel packet*/
734 if (skb->encapsulation) {
735 /* compute OL2 header size, defined in 2 Bytes */
736 ol2_len = l2_len;
e4e87715
PL		 737 hnae3_set_field(*ol_type_vlan_len_msec,
738 HNS3_TXD_L2LEN_M,
739 HNS3_TXD_L2LEN_S, ol2_len >> 1);
76ad4f0e
S		 740
741 /* compute OL3 header size, defined in 4 Bytes */
742 ol3_len = l4.hdr - l3.hdr;
e4e87715
PL		 743 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
744 HNS3_TXD_L3LEN_S, ol3_len >> 2);
76ad4f0e
S		 745
746 /* MAC in UDP, MAC in GRE (0x6558)*/
747 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
748 /* switch MAC header ptr from outer to inner header.*/
749 l2_hdr = skb_inner_mac_header(skb);
750
751 /* compute OL4 header size, defined in 4 Bytes. */
752 ol4_len = l2_hdr - l4.hdr;
e4e87715
PL		 753 hnae3_set_field(*ol_type_vlan_len_msec,
754 HNS3_TXD_L4LEN_M, HNS3_TXD_L4LEN_S,
755 ol4_len >> 2);
76ad4f0e
S		 756
757 /* switch IP header ptr from outer to inner header */
758 l3.hdr = skb_inner_network_header(skb);
759
760 /* compute inner l2 header size, defined in 2 Bytes. */
761 l2_len = l3.hdr - l2_hdr;
e4e87715
PL		 762 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
763 HNS3_TXD_L2LEN_S, l2_len >> 1);
76ad4f0e
S		 764 } else {
765 /* skb packet types not supported by hardware,
766 * txbd len fild doesn't be filled.
767 */
768 return;
769 }
770
771 /* switch L4 header pointer from outer to inner */
772 l4.hdr = skb_inner_transport_header(skb);
773
774 l4_proto = il4_proto;
775 }
776
777 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
778 l3_len = l4.hdr - l3.hdr;
e4e87715
PL		 779 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
780 HNS3_TXD_L3LEN_S, l3_len >> 2);
76ad4f0e
S		 781
782 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
783 switch (l4_proto) {
784 case IPPROTO_TCP:
e4e87715
PL		 785 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
786 HNS3_TXD_L4LEN_S, l4.tcp->doff);
76ad4f0e
S		 787 break;
788 case IPPROTO_SCTP:
e4e87715
PL		 789 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
790 HNS3_TXD_L4LEN_S,
791 (sizeof(struct sctphdr) >> 2));
76ad4f0e
S		 792 break;
793 case IPPROTO_UDP:
e4e87715
PL		 794 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
795 HNS3_TXD_L4LEN_S,
796 (sizeof(struct udphdr) >> 2));
76ad4f0e
S		 797 break;
798 default:
799 /* skb packet types not supported by hardware,
800 * txbd len fild doesn't be filled.
801 */
802 return;
803 }
804}
805
3db084d2
YL		 806/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL
807 * and it is udp packet, which has a dest port as the IANA assigned.
808 * the hardware is expected to do the checksum offload, but the
809 * hardware will not do the checksum offload when udp dest port is
810 * 4789.
811 */
812static bool hns3_tunnel_csum_bug(struct sk_buff *skb)
813{
814#define IANA_VXLAN_PORT 4789
815 union {
816 struct tcphdr *tcp;
817 struct udphdr *udp;
818 struct gre_base_hdr *gre;
819 unsigned char *hdr;
820 } l4;
821
822 l4.hdr = skb_transport_header(skb);
823
824 if (!(!skb->encapsulation && l4.udp->dest == htons(IANA_VXLAN_PORT)))
825 return false;
826
827 skb_checksum_help(skb);
828
829 return true;
830}
831
76ad4f0e
S		 832static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
833 u8 il4_proto, u32 *type_cs_vlan_tso,
834 u32 *ol_type_vlan_len_msec)
835{
836 union {
837 struct iphdr *v4;
838 struct ipv6hdr *v6;
839 unsigned char *hdr;
840 } l3;
841 u32 l4_proto = ol4_proto;
842
843 l3.hdr = skb_network_header(skb);
844
845 /* define OL3 type and tunnel type(OL4).*/
846 if (skb->encapsulation) {
847 /* define outer network header type.*/
848 if (skb->protocol == htons(ETH_P_IP)) {
849 if (skb_is_gso(skb))
e4e87715
PL		 850 hnae3_set_field(*ol_type_vlan_len_msec,
851 HNS3_TXD_OL3T_M,
852 HNS3_TXD_OL3T_S,
853 HNS3_OL3T_IPV4_CSUM);
76ad4f0e 854 else
e4e87715
PL		 855 hnae3_set_field(*ol_type_vlan_len_msec,
856 HNS3_TXD_OL3T_M,
857 HNS3_TXD_OL3T_S,
858 HNS3_OL3T_IPV4_NO_CSUM);
76ad4f0e
S		 859
860 } else if (skb->protocol == htons(ETH_P_IPV6)) {
e4e87715
PL		 861 hnae3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
862 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
76ad4f0e
S		 863 }
864
865 /* define tunnel type(OL4).*/
866 switch (l4_proto) {
867 case IPPROTO_UDP:
e4e87715
PL		 868 hnae3_set_field(*ol_type_vlan_len_msec,
869 HNS3_TXD_TUNTYPE_M,
870 HNS3_TXD_TUNTYPE_S,
871 HNS3_TUN_MAC_IN_UDP);
76ad4f0e
S		 872 break;
873 case IPPROTO_GRE:
e4e87715
PL		 874 hnae3_set_field(*ol_type_vlan_len_msec,
875 HNS3_TXD_TUNTYPE_M,
876 HNS3_TXD_TUNTYPE_S,
877 HNS3_TUN_NVGRE);
76ad4f0e
S		 878 break;
879 default:
880 /* drop the skb tunnel packet if hardware don't support,
881 * because hardware can't calculate csum when TSO.
882 */
883 if (skb_is_gso(skb))
884 return -EDOM;
885
886 /* the stack computes the IP header already,
887 * driver calculate l4 checksum when not TSO.
888 */
889 skb_checksum_help(skb);
890 return 0;
891 }
892
893 l3.hdr = skb_inner_network_header(skb);
894 l4_proto = il4_proto;
895 }
896
897 if (l3.v4->version == 4) {
e4e87715
PL		 898 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
899 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
76ad4f0e
S		 900
901 /* the stack computes the IP header already, the only time we
902 * need the hardware to recompute it is in the case of TSO.
903 */
904 if (skb_is_gso(skb))
e4e87715 905 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
76ad4f0e 906 } else if (l3.v6->version == 6) {
e4e87715
PL		 907 hnae3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
908 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
76ad4f0e
S		 909 }
910
911 switch (l4_proto) {
912 case IPPROTO_TCP:
5c897197 913 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
e4e87715
PL		 914 hnae3_set_field(*type_cs_vlan_tso,
915 HNS3_TXD_L4T_M,
916 HNS3_TXD_L4T_S,
917 HNS3_L4T_TCP);
76ad4f0e
S		 918 break;
919 case IPPROTO_UDP:
3db084d2
YL		 920 if (hns3_tunnel_csum_bug(skb))
921 break;
922
5c897197 923 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
e4e87715
PL		 924 hnae3_set_field(*type_cs_vlan_tso,
925 HNS3_TXD_L4T_M,
926 HNS3_TXD_L4T_S,
927 HNS3_L4T_UDP);
76ad4f0e
S		 928 break;
929 case IPPROTO_SCTP:
5c897197 930 hnae3_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
e4e87715
PL		 931 hnae3_set_field(*type_cs_vlan_tso,
932 HNS3_TXD_L4T_M,
933 HNS3_TXD_L4T_S,
934 HNS3_L4T_SCTP);
76ad4f0e
S		 935 break;
936 default:
937 /* drop the skb tunnel packet if hardware don't support,
938 * because hardware can't calculate csum when TSO.
939 */
940 if (skb_is_gso(skb))
941 return -EDOM;
942
943 /* the stack computes the IP header already,
944 * driver calculate l4 checksum when not TSO.
945 */
946 skb_checksum_help(skb);
947 return 0;
948 }
949
950 return 0;
951}
952
953static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
954{
955 /* Config bd buffer end */
e4e87715
PL		 956 hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
957 HNS3_TXD_BDTYPE_S, 0);
958 hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
959 hnae3_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
960 hnae3_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 961}
962
9699cffe
PL		 963static int hns3_fill_desc_vtags(struct sk_buff *skb,
964 struct hns3_enet_ring *tx_ring,
965 u32 *inner_vlan_flag,
966 u32 *out_vlan_flag,
967 u16 *inner_vtag,
968 u16 *out_vtag)
969{
970#define HNS3_TX_VLAN_PRIO_SHIFT 13
971
972 if (skb->protocol == htons(ETH_P_8021Q) &&
973 !(tx_ring->tqp->handle->kinfo.netdev->features &
974 NETIF_F_HW_VLAN_CTAG_TX)) {
975 /* When HW VLAN acceleration is turned off, and the stack
976 * sets the protocol to 802.1q, the driver just need to
977 * set the protocol to the encapsulated ethertype.
978 */
979 skb->protocol = vlan_get_protocol(skb);
980 return 0;
981 }
982
983 if (skb_vlan_tag_present(skb)) {
984 u16 vlan_tag;
985
986 vlan_tag = skb_vlan_tag_get(skb);
987 vlan_tag |= (skb->priority & 0x7) << HNS3_TX_VLAN_PRIO_SHIFT;
988
989 /* Based on hw strategy, use out_vtag in two layer tag case,
990 * and use inner_vtag in one tag case.
991 */
992 if (skb->protocol == htons(ETH_P_8021Q)) {
e4e87715 993 hnae3_set_bit(*out_vlan_flag, HNS3_TXD_OVLAN_B, 1);
9699cffe
PL		 994 *out_vtag = vlan_tag;
995 } else {
e4e87715 996 hnae3_set_bit(*inner_vlan_flag, HNS3_TXD_VLAN_B, 1);
9699cffe
PL		 997 *inner_vtag = vlan_tag;
998 }
999 } else if (skb->protocol == htons(ETH_P_8021Q)) {
1000 struct vlan_ethhdr *vhdr;
1001 int rc;
1002
1003 rc = skb_cow_head(skb, 0);
1004 if (rc < 0)
1005 return rc;
1006 vhdr = (struct vlan_ethhdr *)skb->data;
1007 vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority & 0x7)
1008 << HNS3_TX_VLAN_PRIO_SHIFT);
1009 }
1010
1011 skb->protocol = vlan_get_protocol(skb);
1012 return 0;
1013}
1014
76ad4f0e 1015static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
5188f218 1016 int size, int frag_end, enum hns_desc_type type)
76ad4f0e
S		 1017{
1018 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
1019 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
5188f218 1020 struct device *dev = ring_to_dev(ring);
76ad4f0e
S		 1021 u32 ol_type_vlan_len_msec = 0;
1022 u16 bdtp_fe_sc_vld_ra_ri = 0;
5188f218 1023 struct skb_frag_struct *frag;
1e8a7977 1024 unsigned int frag_buf_num;
76ad4f0e
S		 1025 u32 type_cs_vlan_tso = 0;
1026 struct sk_buff *skb;
9699cffe
PL		 1027 u16 inner_vtag = 0;
1028 u16 out_vtag = 0;
1e8a7977
FL		 1029 unsigned int k;
1030 int sizeoflast;
76ad4f0e 1031 u32 paylen = 0;
5188f218 1032 dma_addr_t dma;
76ad4f0e 1033 u16 mss = 0;
76ad4f0e
S		 1034 u8 ol4_proto;
1035 u8 il4_proto;
1036 int ret;
1037
76ad4f0e
S		 1038 if (type == DESC_TYPE_SKB) {
1039 skb = (struct sk_buff *)priv;
a90bb9a5 1040 paylen = skb->len;
76ad4f0e 1041
9699cffe
PL		 1042 ret = hns3_fill_desc_vtags(skb, ring, &type_cs_vlan_tso,
1043 &ol_type_vlan_len_msec,
1044 &inner_vtag, &out_vtag);
1045 if (unlikely(ret))
1046 return ret;
1047
76ad4f0e
S		 1048 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1049 skb_reset_mac_len(skb);
76ad4f0e 1050
1898d4e4
S		 1051 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
1052 if (ret)
1053 return ret;
76ad4f0e
S		 1054 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
1055 &type_cs_vlan_tso,
1056 &ol_type_vlan_len_msec);
1057 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
1058 &type_cs_vlan_tso,
1059 &ol_type_vlan_len_msec);
1060 if (ret)
1061 return ret;
1062
1063 ret = hns3_set_tso(skb, &paylen, &mss,
1064 &type_cs_vlan_tso);
1065 if (ret)
1066 return ret;
1067 }
1068
1069 /* Set txbd */
1070 desc->tx.ol_type_vlan_len_msec =
1071 cpu_to_le32(ol_type_vlan_len_msec);
1072 desc->tx.type_cs_vlan_tso_len =
1073 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 1074 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e 1075 desc->tx.mss = cpu_to_le16(mss);
9699cffe
PL		 1076 desc->tx.vlan_tag = cpu_to_le16(inner_vtag);
1077 desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag);
5188f218
PL		 1078
1079 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
1080 } else {
1081 frag = (struct skb_frag_struct *)priv;
1082 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
1083 }
1084
1085 if (dma_mapping_error(ring->dev, dma)) {
1086 ring->stats.sw_err_cnt++;
1087 return -ENOMEM;
76ad4f0e
S		 1088 }
1089
bcdb12b7
FL		 1090 desc_cb->length = size;
1091
1e8a7977
FL		 1092 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1093 sizeoflast = size % HNS3_MAX_BD_SIZE;
1094 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
1095
1096 /* When frag size is bigger than hardware limit, split this frag */
1097 for (k = 0; k < frag_buf_num; k++) {
1098 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
1099 desc_cb->priv = priv;
1e8a7977
FL		 1100 desc_cb->dma = dma + HNS3_MAX_BD_SIZE * k;
1101 desc_cb->type = (type == DESC_TYPE_SKB && !k) ?
1102 DESC_TYPE_SKB : DESC_TYPE_PAGE;
1103
1104 /* now, fill the descriptor */
1105 desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);
bcdb12b7
FL		 1106 desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?
1107 (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);
1e8a7977
FL		 1108 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri,
1109 frag_end && (k == frag_buf_num - 1) ?
1110 1 : 0);
1111 desc->tx.bdtp_fe_sc_vld_ra_ri =
1112 cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
1113
1114 /* move ring pointer to next.*/
1115 ring_ptr_move_fw(ring, next_to_use);
1116
1117 desc_cb = &ring->desc_cb[ring->next_to_use];
1118 desc = &ring->desc[ring->next_to_use];
1119 }
76ad4f0e
S		 1120
1121 return 0;
1122}
1123
76ad4f0e
S		 1124static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
1125 struct hns3_enet_ring *ring)
1126{
1127 struct sk_buff *skb = *out_skb;
1128 struct skb_frag_struct *frag;
1129 int bdnum_for_frag;
1130 int frag_num;
1131 int buf_num;
1132 int size;
1133 int i;
1134
1135 size = skb_headlen(skb);
1136 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1137
1138 frag_num = skb_shinfo(skb)->nr_frags;
1139 for (i = 0; i < frag_num; i++) {
1140 frag = &skb_shinfo(skb)->frags[i];
1141 size = skb_frag_size(frag);
1142 bdnum_for_frag =
1143 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
1144 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
1145 return -ENOMEM;
1146
1147 buf_num += bdnum_for_frag;
1148 }
1149
1150 if (buf_num > ring_space(ring))
1151 return -EBUSY;
1152
1153 *bnum = buf_num;
1154 return 0;
1155}
1156
1157static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
1158 struct hns3_enet_ring *ring)
1159{
1160 struct sk_buff *skb = *out_skb;
1161 int buf_num;
1162
1163 /* No. of segments (plus a header) */
1164 buf_num = skb_shinfo(skb)->nr_frags + 1;
1165
932d1252 1166 if (unlikely(ring_space(ring) < buf_num))
76ad4f0e
S		 1167 return -EBUSY;
1168
1169 *bnum = buf_num;
1170
1171 return 0;
1172}
1173
ba3f808f 1174static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)
76ad4f0e
S		 1175{
1176 struct device *dev = ring_to_dev(ring);
1177 unsigned int i;
1178
1179 for (i = 0; i < ring->desc_num; i++) {
1180 /* check if this is where we started */
1181 if (ring->next_to_use == next_to_use_orig)
1182 break;
1183
1184 /* unmap the descriptor dma address */
1185 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
1186 dma_unmap_single(dev,
1187 ring->desc_cb[ring->next_to_use].dma,
1188 ring->desc_cb[ring->next_to_use].length,
1189 DMA_TO_DEVICE);
bcdb12b7 1190 else if (ring->desc_cb[ring->next_to_use].length)
76ad4f0e
S		 1191 dma_unmap_page(dev,
1192 ring->desc_cb[ring->next_to_use].dma,
1193 ring->desc_cb[ring->next_to_use].length,
1194 DMA_TO_DEVICE);
1195
bcdb12b7
FL		 1196 ring->desc_cb[ring->next_to_use].length = 0;
1197
76ad4f0e
S		 1198 /* rollback one */
1199 ring_ptr_move_bw(ring, next_to_use);
1200 }
1201}
1202
d43e5aca 1203netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 1204{
1205 struct hns3_nic_priv *priv = netdev_priv(netdev);
1206 struct hns3_nic_ring_data *ring_data =
1207 &tx_ring_data(priv, skb->queue_mapping);
1208 struct hns3_enet_ring *ring = ring_data->ring;
76ad4f0e
S		 1209 struct netdev_queue *dev_queue;
1210 struct skb_frag_struct *frag;
1211 int next_to_use_head;
1212 int next_to_use_frag;
76ad4f0e
S		 1213 int buf_num;
1214 int seg_num;
1215 int size;
1216 int ret;
1217 int i;
1218
1219 /* Prefetch the data used later */
1220 prefetch(skb->data);
1221
1222 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
1223 case -EBUSY:
1224 u64_stats_update_begin(&ring->syncp);
1225 ring->stats.tx_busy++;
1226 u64_stats_update_end(&ring->syncp);
1227
1228 goto out_net_tx_busy;
1229 case -ENOMEM:
1230 u64_stats_update_begin(&ring->syncp);
1231 ring->stats.sw_err_cnt++;
1232 u64_stats_update_end(&ring->syncp);
1233 netdev_err(netdev, "no memory to xmit!\n");
1234
1235 goto out_err_tx_ok;
1236 default:
1237 break;
1238 }
1239
1240 /* No. of segments (plus a header) */
1241 seg_num = skb_shinfo(skb)->nr_frags + 1;
1242 /* Fill the first part */
1243 size = skb_headlen(skb);
1244
1245 next_to_use_head = ring->next_to_use;
1246
5188f218
PL		 1247 ret = priv->ops.fill_desc(ring, skb, size, seg_num == 1 ? 1 : 0,
1248 DESC_TYPE_SKB);
76ad4f0e 1249 if (ret)
ba3f808f 1250 goto head_fill_err;
76ad4f0e
S		 1251
1252 next_to_use_frag = ring->next_to_use;
1253 /* Fill the fragments */
1254 for (i = 1; i < seg_num; i++) {
1255 frag = &skb_shinfo(skb)->frags[i - 1];
1256 size = skb_frag_size(frag);
5188f218
PL		 1257
1258 ret = priv->ops.fill_desc(ring, frag, size,
1259 seg_num - 1 == i ? 1 : 0,
1260 DESC_TYPE_PAGE);
76ad4f0e
S		 1261
1262 if (ret)
ba3f808f 1263 goto frag_fill_err;
76ad4f0e
S		 1264 }
1265
1266 /* Complete translate all packets */
1267 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
1268 netdev_tx_sent_queue(dev_queue, skb->len);
1269
1270 wmb(); /* Commit all data before submit */
1271
e4e87715 1272 hnae3_queue_xmit(ring->tqp, buf_num);
76ad4f0e
S		 1273
1274 return NETDEV_TX_OK;
1275
ba3f808f
FL		 1276frag_fill_err:
1277 hns3_clear_desc(ring, next_to_use_frag);
76ad4f0e 1278
ba3f808f
FL		 1279head_fill_err:
1280 hns3_clear_desc(ring, next_to_use_head);
76ad4f0e
S		 1281
1282out_err_tx_ok:
1283 dev_kfree_skb_any(skb);
1284 return NETDEV_TX_OK;
1285
1286out_net_tx_busy:
1287 netif_stop_subqueue(netdev, ring_data->queue_index);
1288 smp_mb(); /* Commit all data before submit */
1289
1290 return NETDEV_TX_BUSY;
1291}
1292
1293static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1294{
9780cb97 1295 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1296 struct sockaddr *mac_addr = p;
1297 int ret;
1298
1299 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1300 return -EADDRNOTAVAIL;
1301
5ec2a51e
JS		 1302 if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {
1303 netdev_info(netdev, "already using mac address %pM\n",
1304 mac_addr->sa_data);
1305 return 0;
1306 }
1307
59098055 1308 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);
76ad4f0e
S		 1309 if (ret) {
1310 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1311 return ret;
1312 }
1313
1314 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1315
1316 return 0;
1317}
1318
26483246
XW		 1319static int hns3_nic_do_ioctl(struct net_device *netdev,
1320 struct ifreq *ifr, int cmd)
1321{
1322 struct hnae3_handle *h = hns3_get_handle(netdev);
1323
1324 if (!netif_running(netdev))
1325 return -EINVAL;
1326
1327 if (!h->ae_algo->ops->do_ioctl)
1328 return -EOPNOTSUPP;
1329
1330 return h->ae_algo->ops->do_ioctl(h, ifr, cmd);
1331}
1332
76ad4f0e
S		 1333static int hns3_nic_set_features(struct net_device *netdev,
1334 netdev_features_t features)
1335{
181d454b 1336 netdev_features_t changed = netdev->features ^ features;
76ad4f0e 1337 struct hns3_nic_priv *priv = netdev_priv(netdev);
052ece6d 1338 struct hnae3_handle *h = priv->ae_handle;
052ece6d 1339 int ret;
76ad4f0e 1340
181d454b 1341 if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
0bbbf15d 1342 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
181d454b 1343 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
0bbbf15d 1344 else
181d454b 1345 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
76ad4f0e
S		 1346 }
1347
5c9f6b39
PL		 1348 if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {
1349 if (features & NETIF_F_GRO_HW)
1350 ret = h->ae_algo->ops->set_gro_en(h, true);
1351 else
1352 ret = h->ae_algo->ops->set_gro_en(h, false);
1353 if (ret)
1354 return ret;
1355 }
1356
bd368416
JS		 1357 if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&
1358 h->ae_algo->ops->enable_vlan_filter) {
181d454b
JS		 1359 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1360 h->ae_algo->ops->enable_vlan_filter(h, true);
1361 else
1362 h->ae_algo->ops->enable_vlan_filter(h, false);
1363 }
391b5e93 1364
bd368416
JS		 1365 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&
1366 h->ae_algo->ops->enable_hw_strip_rxvtag) {
052ece6d
PL		 1367 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1368 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, true);
1369 else
1370 ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, false);
1371
1372 if (ret)
1373 return ret;
1374 }
1375
c17852a8
JS		 1376 if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {
1377 if (features & NETIF_F_NTUPLE)
1378 h->ae_algo->ops->enable_fd(h, true);
1379 else
1380 h->ae_algo->ops->enable_fd(h, false);
1381 }
1382
76ad4f0e
S		 1383 netdev->features = features;
1384 return 0;
1385}
1386
6c88d9d7
PL		 1387static void hns3_nic_get_stats64(struct net_device *netdev,
1388 struct rtnl_link_stats64 *stats)
76ad4f0e
S		 1389{
1390 struct hns3_nic_priv *priv = netdev_priv(netdev);
1391 int queue_num = priv->ae_handle->kinfo.num_tqps;
c5f65480 1392 struct hnae3_handle *handle = priv->ae_handle;
76ad4f0e
S		 1393 struct hns3_enet_ring *ring;
1394 unsigned int start;
1395 unsigned int idx;
1396 u64 tx_bytes = 0;
1397 u64 rx_bytes = 0;
1398 u64 tx_pkts = 0;
1399 u64 rx_pkts = 0;
d2a5dca8
JS		 1400 u64 tx_drop = 0;
1401 u64 rx_drop = 0;
76ad4f0e 1402
b875cc37
JS		 1403 if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))
1404 return;
1405
c5f65480
JS		 1406 handle->ae_algo->ops->update_stats(handle, &netdev->stats);
1407
76ad4f0e
S		 1408 for (idx = 0; idx < queue_num; idx++) {
1409 /* fetch the tx stats */
1410 ring = priv->ring_data[idx].ring;
1411 do {
d36d36ce 1412 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1413 tx_bytes += ring->stats.tx_bytes;
1414 tx_pkts += ring->stats.tx_pkts;
d2a5dca8
JS		 1415 tx_drop += ring->stats.tx_busy;
1416 tx_drop += ring->stats.sw_err_cnt;
76ad4f0e
S		 1417 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1418
1419 /* fetch the rx stats */
1420 ring = priv->ring_data[idx + queue_num].ring;
1421 do {
d36d36ce 1422 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1423 rx_bytes += ring->stats.rx_bytes;
1424 rx_pkts += ring->stats.rx_pkts;
d2a5dca8
JS		 1425 rx_drop += ring->stats.non_vld_descs;
1426 rx_drop += ring->stats.err_pkt_len;
1427 rx_drop += ring->stats.l2_err;
76ad4f0e
S		 1428 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1429 }
1430
1431 stats->tx_bytes = tx_bytes;
1432 stats->tx_packets = tx_pkts;
1433 stats->rx_bytes = rx_bytes;
1434 stats->rx_packets = rx_pkts;
1435
1436 stats->rx_errors = netdev->stats.rx_errors;
1437 stats->multicast = netdev->stats.multicast;
1438 stats->rx_length_errors = netdev->stats.rx_length_errors;
1439 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1440 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1441
1442 stats->tx_errors = netdev->stats.tx_errors;
d2a5dca8
JS		 1443 stats->rx_dropped = rx_drop + netdev->stats.rx_dropped;
1444 stats->tx_dropped = tx_drop + netdev->stats.tx_dropped;
76ad4f0e
S		 1445 stats->collisions = netdev->stats.collisions;
1446 stats->rx_over_errors = netdev->stats.rx_over_errors;
1447 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1448 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1449 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1450 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1451 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1452 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1453 stats->tx_window_errors = netdev->stats.tx_window_errors;
1454 stats->rx_compressed = netdev->stats.rx_compressed;
1455 stats->tx_compressed = netdev->stats.tx_compressed;
1456}
1457
30d240df 1458static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1459{
30d240df 1460 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1461 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1462 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1463 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1464 u8 tc = mqprio_qopt->qopt.num_tc;
1465 u16 mode = mqprio_qopt->mode;
1466 u8 hw = mqprio_qopt->qopt.hw;
1467 bool if_running;
76ad4f0e
S		 1468 int ret;
1469
30d240df
YL		 1470 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1471 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1472 return -EOPNOTSUPP;
1473
76ad4f0e
S		 1474 if (tc > HNAE3_MAX_TC)
1475 return -EINVAL;
1476
76ad4f0e
S		 1477 if (!netdev)
1478 return -EINVAL;
1479
30d240df
YL		 1480 if_running = netif_running(netdev);
1481 if (if_running) {
1482 hns3_nic_net_stop(netdev);
1483 msleep(100);
76ad4f0e
S		 1484 }
1485
30d240df
YL		 1486 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1487 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1488 if (ret)
30d240df
YL		 1489 goto out;
1490
30d240df
YL		 1491 ret = hns3_nic_set_real_num_queue(netdev);
1492
1493out:
1494 if (if_running)
1495 hns3_nic_net_open(netdev);
1496
1497 return ret;
76ad4f0e
S		 1498}
1499
2572ac53 1500static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1501 void *type_data)
76ad4f0e 1502{
575ed7d3 1503 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1504 return -EOPNOTSUPP;
76ad4f0e 1505
30d240df 1506 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1507}
1508
1509static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1510 __be16 proto, u16 vid)
1511{
9780cb97 1512 struct hnae3_handle *h = hns3_get_handle(netdev);
681ec399 1513 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1514 int ret = -EIO;
1515
1516 if (h->ae_algo->ops->set_vlan_filter)
1517 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1518
681ec399
YL		 1519 if (!ret)
1520 set_bit(vid, priv->active_vlans);
1521
76ad4f0e
S		 1522 return ret;
1523}
1524
1525static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1526 __be16 proto, u16 vid)
1527{
9780cb97 1528 struct hnae3_handle *h = hns3_get_handle(netdev);
681ec399 1529 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 1530 int ret = -EIO;
1531
1532 if (h->ae_algo->ops->set_vlan_filter)
1533 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1534
681ec399
YL		 1535 if (!ret)
1536 clear_bit(vid, priv->active_vlans);
1537
76ad4f0e
S		 1538 return ret;
1539}
1540
7fa6be4f 1541static int hns3_restore_vlan(struct net_device *netdev)
681ec399
YL		 1542{
1543 struct hns3_nic_priv *priv = netdev_priv(netdev);
7fa6be4f 1544 int ret = 0;
681ec399 1545 u16 vid;
681ec399
YL		 1546
1547 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
1548 ret = hns3_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
7fa6be4f
HT		 1549 if (ret) {
1550 netdev_err(netdev, "Restore vlan: %d filter, ret:%d\n",
1551 vid, ret);
1552 return ret;
1553 }
681ec399 1554 }
7fa6be4f
HT		 1555
1556 return ret;
681ec399
YL		 1557}
1558
76ad4f0e
S		 1559static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1560 u8 qos, __be16 vlan_proto)
1561{
9780cb97 1562 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1563 int ret = -EIO;
1564
1565 if (h->ae_algo->ops->set_vf_vlan_filter)
1566 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1567 qos, vlan_proto);
1568
1569 return ret;
1570}
1571
a8e8b7ff
S		 1572static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1573{
9780cb97 1574 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1575 bool if_running = netif_running(netdev);
1576 int ret;
1577
1578 if (!h->ae_algo->ops->set_mtu)
1579 return -EOPNOTSUPP;
1580
1581 /* if this was called with netdev up then bring netdevice down */
1582 if (if_running) {
1583 (void)hns3_nic_net_stop(netdev);
1584 msleep(100);
1585 }
1586
1587 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
93d8daf4 1588 if (ret)
a8e8b7ff
S		 1589 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1590 ret);
93d8daf4
YL		 1591 else
1592 netdev->mtu = new_mtu;
5bad95a1 1593
a8e8b7ff
S		 1594 /* if the netdev was running earlier, bring it up again */
1595 if (if_running && hns3_nic_net_open(netdev))
1596 ret = -EINVAL;
1597
1598 return ret;
1599}
1600
f8fa222c
L		 1601static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1602{
1603 struct hns3_nic_priv *priv = netdev_priv(ndev);
1604 struct hns3_enet_ring *tx_ring = NULL;
1605 int timeout_queue = 0;
1606 int hw_head, hw_tail;
1607 int i;
1608
1609 /* Find the stopped queue the same way the stack does */
1610 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1611 struct netdev_queue *q;
1612 unsigned long trans_start;
1613
1614 q = netdev_get_tx_queue(ndev, i);
1615 trans_start = q->trans_start;
1616 if (netif_xmit_stopped(q) &&
1617 time_after(jiffies,
1618 (trans_start + ndev->watchdog_timeo))) {
1619 timeout_queue = i;
1620 break;
1621 }
1622 }
1623
1624 if (i == ndev->num_tx_queues) {
1625 netdev_info(ndev,
1626 "no netdev TX timeout queue found, timeout count: %llu\n",
1627 priv->tx_timeout_count);
1628 return false;
1629 }
1630
1631 tx_ring = priv->ring_data[timeout_queue].ring;
1632
1633 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1634 HNS3_RING_TX_RING_HEAD_REG);
1635 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1636 HNS3_RING_TX_RING_TAIL_REG);
1637 netdev_info(ndev,
1638 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1639 priv->tx_timeout_count,
1640 timeout_queue,
1641 tx_ring->next_to_use,
1642 tx_ring->next_to_clean,
1643 hw_head,
1644 hw_tail,
1645 readl(tx_ring->tqp_vector->mask_addr));
1646
1647 return true;
1648}
1649
1650static void hns3_nic_net_timeout(struct net_device *ndev)
1651{
1652 struct hns3_nic_priv *priv = netdev_priv(ndev);
f8fa222c
L		 1653 struct hnae3_handle *h = priv->ae_handle;
1654
1655 if (!hns3_get_tx_timeo_queue_info(ndev))
1656 return;
1657
1658 priv->tx_timeout_count++;
1659
0742ed7c
HT		 1660 /* request the reset, and let the hclge to determine
1661 * which reset level should be done
1662 */
f8fa222c 1663 if (h->ae_algo->ops->reset_event)
6ae4e733 1664 h->ae_algo->ops->reset_event(h->pdev, h);
f8fa222c
L		 1665}
1666
76ad4f0e
S		 1667static const struct net_device_ops hns3_nic_netdev_ops = {
1668 .ndo_open = hns3_nic_net_open,
1669 .ndo_stop = hns3_nic_net_stop,
1670 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1671 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1672 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
26483246 1673 .ndo_do_ioctl = hns3_nic_do_ioctl,
a8e8b7ff 1674 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1675 .ndo_set_features = hns3_nic_set_features,
1676 .ndo_get_stats64 = hns3_nic_get_stats64,
1677 .ndo_setup_tc = hns3_nic_setup_tc,
1678 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
76ad4f0e
S		 1679 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1680 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1681 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1682};
1683
2312e050
FL		 1684static bool hns3_is_phys_func(struct pci_dev *pdev)
1685{
1686 u32 dev_id = pdev->device;
1687
1688 switch (dev_id) {
1689 case HNAE3_DEV_ID_GE:
1690 case HNAE3_DEV_ID_25GE:
1691 case HNAE3_DEV_ID_25GE_RDMA:
1692 case HNAE3_DEV_ID_25GE_RDMA_MACSEC:
1693 case HNAE3_DEV_ID_50GE_RDMA:
1694 case HNAE3_DEV_ID_50GE_RDMA_MACSEC:
1695 case HNAE3_DEV_ID_100G_RDMA_MACSEC:
1696 return true;
1697 case HNAE3_DEV_ID_100G_VF:
1698 case HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF:
1699 return false;
1700 default:
1701 dev_warn(&pdev->dev, "un-recognized pci device-id %d",
1702 dev_id);
1703 }
1704
1705 return false;
1706}
1707
2312e050
FL		 1708static void hns3_disable_sriov(struct pci_dev *pdev)
1709{
1710 /* If our VFs are assigned we cannot shut down SR-IOV
1711 * without causing issues, so just leave the hardware
1712 * available but disabled
1713 */
1714 if (pci_vfs_assigned(pdev)) {
1715 dev_warn(&pdev->dev,
1716 "disabling driver while VFs are assigned\n");
1717 return;
1718 }
1719
1720 pci_disable_sriov(pdev);
1721}
1722
d695964d
JS		 1723static void hns3_get_dev_capability(struct pci_dev *pdev,
1724 struct hnae3_ae_dev *ae_dev)
1725{
b26a6fea 1726 if (pdev->revision >= 0x21) {
d695964d 1727 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_FD_B, 1);
b26a6fea
PL		 1728 hnae3_set_bit(ae_dev->flag, HNAE3_DEV_SUPPORT_GRO_B, 1);
1729 }
d695964d
JS		 1730}
1731
76ad4f0e
S		 1732/* hns3_probe - Device initialization routine
1733 * @pdev: PCI device information struct
1734 * @ent: entry in hns3_pci_tbl
1735 *
1736 * hns3_probe initializes a PF identified by a pci_dev structure.
1737 * The OS initialization, configuring of the PF private structure,
1738 * and a hardware reset occur.
1739 *
1740 * Returns 0 on success, negative on failure
1741 */
1742static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1743{
1744 struct hnae3_ae_dev *ae_dev;
1745 int ret;
1746
1747 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1748 GFP_KERNEL);
1749 if (!ae_dev) {
1750 ret = -ENOMEM;
1751 return ret;
1752 }
1753
1754 ae_dev->pdev = pdev;
e92a0843 1755 ae_dev->flag = ent->driver_data;
76ad4f0e 1756 ae_dev->dev_type = HNAE3_DEV_KNIC;
6871af29 1757 ae_dev->reset_type = HNAE3_NONE_RESET;
d695964d 1758 hns3_get_dev_capability(pdev, ae_dev);
76ad4f0e
S		 1759 pci_set_drvdata(pdev, ae_dev);
1760
50fbc237 1761 hnae3_register_ae_dev(ae_dev);
2312e050 1762
2312e050 1763 return 0;
76ad4f0e
S		 1764}
1765
1766/* hns3_remove - Device removal routine
1767 * @pdev: PCI device information struct
1768 */
1769static void hns3_remove(struct pci_dev *pdev)
1770{
1771 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1772
2312e050
FL		 1773 if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))
1774 hns3_disable_sriov(pdev);
1775
76ad4f0e 1776 hnae3_unregister_ae_dev(ae_dev);
76ad4f0e
S		 1777}
1778
fa8d82e8
PL		 1779/**
1780 * hns3_pci_sriov_configure
1781 * @pdev: pointer to a pci_dev structure
1782 * @num_vfs: number of VFs to allocate
1783 *
1784 * Enable or change the number of VFs. Called when the user updates the number
1785 * of VFs in sysfs.
1786 **/
743e1a84 1787static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
fa8d82e8
PL		 1788{
1789 int ret;
1790
1791 if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {
1792 dev_warn(&pdev->dev, "Can not config SRIOV\n");
1793 return -EINVAL;
1794 }
1795
1796 if (num_vfs) {
1797 ret = pci_enable_sriov(pdev, num_vfs);
1798 if (ret)
1799 dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);
743e1a84
SM		 1800 else
1801 return num_vfs;
fa8d82e8
PL		 1802 } else if (!pci_vfs_assigned(pdev)) {
1803 pci_disable_sriov(pdev);
1804 } else {
1805 dev_warn(&pdev->dev,
1806 "Unable to free VFs because some are assigned to VMs.\n");
1807 }
1808
1809 return 0;
1810}
1811
ce2c1d2e
YL		 1812static void hns3_shutdown(struct pci_dev *pdev)
1813{
1814 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1815
1816 hnae3_unregister_ae_dev(ae_dev);
1817 devm_kfree(&pdev->dev, ae_dev);
1818 pci_set_drvdata(pdev, NULL);
1819
1820 if (system_state == SYSTEM_POWER_OFF)
1821 pci_set_power_state(pdev, PCI_D3hot);
1822}
1823
5a9f0eac
SJ		 1824static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,
1825 pci_channel_state_t state)
1826{
1827 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1828 pci_ers_result_t ret;
1829
1830 dev_info(&pdev->dev, "PCI error detected, state(=%d)!!\n", state);
1831
1832 if (state == pci_channel_io_perm_failure)
1833 return PCI_ERS_RESULT_DISCONNECT;
1834
1835 if (!ae_dev) {
1836 dev_err(&pdev->dev,
1837 "Can't recover - error happened during device init\n");
1838 return PCI_ERS_RESULT_NONE;
1839 }
1840
1841 if (ae_dev->ops->process_hw_error)
1842 ret = ae_dev->ops->process_hw_error(ae_dev);
1843 else
1844 return PCI_ERS_RESULT_NONE;
1845
1846 return ret;
1847}
1848
6ae4e733
SJ		 1849static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)
1850{
1851 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1852 struct device *dev = &pdev->dev;
1853
1854 dev_info(dev, "requesting reset due to PCI error\n");
1855
1856 /* request the reset */
1857 if (ae_dev->ops->reset_event) {
1858 ae_dev->ops->reset_event(pdev, NULL);
1859 return PCI_ERS_RESULT_RECOVERED;
1860 }
1861
1862 return PCI_ERS_RESULT_DISCONNECT;
1863}
1864
6b9a97ee
HT		 1865static void hns3_reset_prepare(struct pci_dev *pdev)
1866{
1867 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1868
1869 dev_info(&pdev->dev, "hns3 flr prepare\n");
1870 if (ae_dev && ae_dev->ops && ae_dev->ops->flr_prepare)
1871 ae_dev->ops->flr_prepare(ae_dev);
1872}
1873
1874static void hns3_reset_done(struct pci_dev *pdev)
1875{
1876 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1877
1878 dev_info(&pdev->dev, "hns3 flr done\n");
1879 if (ae_dev && ae_dev->ops && ae_dev->ops->flr_done)
1880 ae_dev->ops->flr_done(ae_dev);
1881}
1882
5a9f0eac
SJ		 1883static const struct pci_error_handlers hns3_err_handler = {
1884 .error_detected = hns3_error_detected,
6ae4e733 1885 .slot_reset = hns3_slot_reset,
6b9a97ee
HT		 1886 .reset_prepare = hns3_reset_prepare,
1887 .reset_done = hns3_reset_done,
5a9f0eac
SJ		 1888};
1889
76ad4f0e
S		 1890static struct pci_driver hns3_driver = {
1891 .name = hns3_driver_name,
1892 .id_table = hns3_pci_tbl,
1893 .probe = hns3_probe,
1894 .remove = hns3_remove,
ce2c1d2e 1895 .shutdown = hns3_shutdown,
fa8d82e8 1896 .sriov_configure = hns3_pci_sriov_configure,
5a9f0eac 1897 .err_handler = &hns3_err_handler,
76ad4f0e
S		 1898};
1899
1900/* set default feature to hns3 */
1901static void hns3_set_default_feature(struct net_device *netdev)
1902{
3e85af6a
PL		 1903 struct hnae3_handle *h = hns3_get_handle(netdev);
1904 struct pci_dev *pdev = h->pdev;
1905
76ad4f0e
S		 1906 netdev->priv_flags |= IFF_UNICAST_FLT;
1907
1908 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1909 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1910 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1911 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1912 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1913
1914 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1915
1916 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1917
1918 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1919 NETIF_F_HW_VLAN_CTAG_FILTER |
052ece6d 1920 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1921 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1922 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1923 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1924 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1925
1926 netdev->vlan_features |=
1927 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1928 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1929 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1930 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1931 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
76ad4f0e
S		 1932
1933 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
b2641e2a 1934 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
76ad4f0e
S		 1935 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1936 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1937 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
5b71ac3c 1938 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_SCTP_CRC;
3e85af6a 1939
c17852a8 1940 if (pdev->revision >= 0x21) {
5c9f6b39
PL		 1941 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER |
1942 NETIF_F_GRO_HW;
1943 netdev->features |= NETIF_F_GRO_HW;
c17852a8
JS		 1944
1945 if (!(h->flags & HNAE3_SUPPORT_VF)) {
1946 netdev->hw_features |= NETIF_F_NTUPLE;
1947 netdev->features |= NETIF_F_NTUPLE;
1948 }
1949 }
76ad4f0e
S		 1950}
1951
1952static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1953 struct hns3_desc_cb *cb)
1954{
e4e87715 1955 unsigned int order = hnae3_page_order(ring);
76ad4f0e
S		 1956 struct page *p;
1957
1958 p = dev_alloc_pages(order);
1959 if (!p)
1960 return -ENOMEM;
1961
1962 cb->priv = p;
1963 cb->page_offset = 0;
1964 cb->reuse_flag = 0;
1965 cb->buf = page_address(p);
e4e87715 1966 cb->length = hnae3_page_size(ring);
76ad4f0e
S		 1967 cb->type = DESC_TYPE_PAGE;
1968
76ad4f0e
S		 1969 return 0;
1970}
1971
1972static void hns3_free_buffer(struct hns3_enet_ring *ring,
1973 struct hns3_desc_cb *cb)
1974{
1975 if (cb->type == DESC_TYPE_SKB)
1976 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1977 else if (!HNAE3_IS_TX_RING(ring))
1978 put_page((struct page *)cb->priv);
1979 memset(cb, 0, sizeof(*cb));
1980}
1981
1982static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1983{
1984 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1985 cb->length, ring_to_dma_dir(ring));
1986
2211f4e1 1987 if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))
76ad4f0e
S		 1988 return -EIO;
1989
1990 return 0;
1991}
1992
1993static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1994 struct hns3_desc_cb *cb)
1995{
1996 if (cb->type == DESC_TYPE_SKB)
1997 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1998 ring_to_dma_dir(ring));
bcdb12b7 1999 else if (cb->length)
76ad4f0e
S		 2000 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
2001 ring_to_dma_dir(ring));
2002}
2003
2004static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
2005{
2006 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
2007 ring->desc[i].addr = 0;
2008}
2009
2010static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
2011{
2012 struct hns3_desc_cb *cb = &ring->desc_cb[i];
2013
2014 if (!ring->desc_cb[i].dma)
2015 return;
2016
2017 hns3_buffer_detach(ring, i);
2018 hns3_free_buffer(ring, cb);
2019}
2020
2021static void hns3_free_buffers(struct hns3_enet_ring *ring)
2022{
2023 int i;
2024
2025 for (i = 0; i < ring->desc_num; i++)
2026 hns3_free_buffer_detach(ring, i);
2027}
2028
2029/* free desc along with its attached buffer */
2030static void hns3_free_desc(struct hns3_enet_ring *ring)
2031{
024cc792
HT		 2032 int size = ring->desc_num * sizeof(ring->desc[0]);
2033
76ad4f0e
S		 2034 hns3_free_buffers(ring);
2035
024cc792
HT		 2036 if (ring->desc) {
2037 dma_free_coherent(ring_to_dev(ring), size,
2038 ring->desc, ring->desc_dma_addr);
2039 ring->desc = NULL;
2040 }
76ad4f0e
S		 2041}
2042
2043static int hns3_alloc_desc(struct hns3_enet_ring *ring)
2044{
2045 int size = ring->desc_num * sizeof(ring->desc[0]);
2046
024cc792
HT		 2047 ring->desc = dma_zalloc_coherent(ring_to_dev(ring), size,
2048 &ring->desc_dma_addr,
2049 GFP_KERNEL);
76ad4f0e
S		 2050 if (!ring->desc)
2051 return -ENOMEM;
2052
76ad4f0e
S		 2053 return 0;
2054}
2055
2056static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
2057 struct hns3_desc_cb *cb)
2058{
2059 int ret;
2060
2061 ret = hns3_alloc_buffer(ring, cb);
2062 if (ret)
2063 goto out;
2064
2065 ret = hns3_map_buffer(ring, cb);
2066 if (ret)
2067 goto out_with_buf;
2068
2069 return 0;
2070
2071out_with_buf:
564883bb 2072 hns3_free_buffer(ring, cb);
76ad4f0e
S		 2073out:
2074 return ret;
2075}
2076
2077static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
2078{
2079 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
2080
2081 if (ret)
2082 return ret;
2083
2084 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
2085
2086 return 0;
2087}
2088
2089/* Allocate memory for raw pkg, and map with dma */
2090static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
2091{
2092 int i, j, ret;
2093
2094 for (i = 0; i < ring->desc_num; i++) {
2095 ret = hns3_alloc_buffer_attach(ring, i);
2096 if (ret)
2097 goto out_buffer_fail;
2098 }
2099
2100 return 0;
2101
2102out_buffer_fail:
2103 for (j = i - 1; j >= 0; j--)
2104 hns3_free_buffer_detach(ring, j);
2105 return ret;
2106}
2107
2108/* detach a in-used buffer and replace with a reserved one */
2109static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
2110 struct hns3_desc_cb *res_cb)
2111{
b9077428 2112 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 2113 ring->desc_cb[i] = *res_cb;
2114 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
7d0b130c 2115 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 2116}
2117
2118static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
2119{
2120 ring->desc_cb[i].reuse_flag = 0;
2121 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
2122 + ring->desc_cb[i].page_offset);
7d0b130c 2123 ring->desc[i].rx.bd_base_info = 0;
76ad4f0e
S		 2124}
2125
2126static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
2127 int *pkts)
2128{
2129 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2130
2131 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
2132 (*bytes) += desc_cb->length;
e4e87715 2133 /* desc_cb will be cleaned, after hnae3_free_buffer_detach*/
76ad4f0e
S		 2134 hns3_free_buffer_detach(ring, ring->next_to_clean);
2135
2136 ring_ptr_move_fw(ring, next_to_clean);
2137}
2138
2139static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
2140{
2141 int u = ring->next_to_use;
2142 int c = ring->next_to_clean;
2143
2144 if (unlikely(h > ring->desc_num))
2145 return 0;
2146
2147 return u > c ? (h > c && h <= u) : (h > c || h <= u);
2148}
2149
799997a3 2150void hns3_clean_tx_ring(struct hns3_enet_ring *ring)
76ad4f0e
S		 2151{
2152 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
7a810110 2153 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 2154 struct netdev_queue *dev_queue;
2155 int bytes, pkts;
2156 int head;
2157
2158 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
2159 rmb(); /* Make sure head is ready before touch any data */
2160
2161 if (is_ring_empty(ring) || head == ring->next_to_clean)
799997a3 2162 return; /* no data to poll */
76ad4f0e 2163
0e6084aa 2164 if (unlikely(!is_valid_clean_head(ring, head))) {
76ad4f0e
S		 2165 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
2166 ring->next_to_use, ring->next_to_clean);
2167
2168 u64_stats_update_begin(&ring->syncp);
2169 ring->stats.io_err_cnt++;
2170 u64_stats_update_end(&ring->syncp);
799997a3 2171 return;
76ad4f0e
S		 2172 }
2173
2174 bytes = 0;
2175 pkts = 0;
799997a3 2176 while (head != ring->next_to_clean) {
76ad4f0e
S		 2177 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
2178 /* Issue prefetch for next Tx descriptor */
2179 prefetch(&ring->desc_cb[ring->next_to_clean]);
76ad4f0e
S		 2180 }
2181
2182 ring->tqp_vector->tx_group.total_bytes += bytes;
2183 ring->tqp_vector->tx_group.total_packets += pkts;
2184
2185 u64_stats_update_begin(&ring->syncp);
2186 ring->stats.tx_bytes += bytes;
2187 ring->stats.tx_pkts += pkts;
2188 u64_stats_update_end(&ring->syncp);
2189
2190 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
2191 netdev_tx_completed_queue(dev_queue, pkts, bytes);
2192
2193 if (unlikely(pkts && netif_carrier_ok(netdev) &&
2194 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
2195 /* Make sure that anybody stopping the queue after this
2196 * sees the new next_to_clean.
2197 */
2198 smp_mb();
7a810110
JS		 2199 if (netif_tx_queue_stopped(dev_queue) &&
2200 !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {
76ad4f0e
S		 2201 netif_tx_wake_queue(dev_queue);
2202 ring->stats.restart_queue++;
2203 }
2204 }
76ad4f0e
S		 2205}
2206
2207static int hns3_desc_unused(struct hns3_enet_ring *ring)
2208{
2209 int ntc = ring->next_to_clean;
2210 int ntu = ring->next_to_use;
2211
2212 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
2213}
2214
2215static void
2216hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
2217{
2218 struct hns3_desc_cb *desc_cb;
2219 struct hns3_desc_cb res_cbs;
2220 int i, ret;
2221
2222 for (i = 0; i < cleand_count; i++) {
2223 desc_cb = &ring->desc_cb[ring->next_to_use];
2224 if (desc_cb->reuse_flag) {
2225 u64_stats_update_begin(&ring->syncp);
2226 ring->stats.reuse_pg_cnt++;
2227 u64_stats_update_end(&ring->syncp);
2228
2229 hns3_reuse_buffer(ring, ring->next_to_use);
2230 } else {
2231 ret = hns3_reserve_buffer_map(ring, &res_cbs);
2232 if (ret) {
2233 u64_stats_update_begin(&ring->syncp);
2234 ring->stats.sw_err_cnt++;
2235 u64_stats_update_end(&ring->syncp);
2236
2237 netdev_err(ring->tqp->handle->kinfo.netdev,
2238 "hnae reserve buffer map failed.\n");
2239 break;
2240 }
2241 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
2242 }
2243
2244 ring_ptr_move_fw(ring, next_to_use);
2245 }
2246
2247 wmb(); /* Make all data has been write before submit */
2248 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
2249}
2250
76ad4f0e
S		 2251static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
2252 struct hns3_enet_ring *ring, int pull_len,
2253 struct hns3_desc_cb *desc_cb)
2254{
2255 struct hns3_desc *desc;
583e7281
HT		 2256 u32 truesize;
2257 int size;
76ad4f0e
S		 2258 int last_offset;
2259 bool twobufs;
2260
2261 twobufs = ((PAGE_SIZE < 8192) &&
e4e87715 2262 hnae3_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
76ad4f0e
S		 2263
2264 desc = &ring->desc[ring->next_to_clean];
2265 size = le16_to_cpu(desc->rx.size);
2266
e4e87715 2267 truesize = hnae3_buf_size(ring);
f8d291f0
PL		 2268
2269 if (!twobufs)
e4e87715 2270 last_offset = hnae3_page_size(ring) - hnae3_buf_size(ring);
76ad4f0e
S		 2271
2272 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
f8d291f0 2273 size - pull_len, truesize);
76ad4f0e
S		 2274
2275 /* Avoid re-using remote pages,flag default unreuse */
2276 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
2277 return;
2278
2279 if (twobufs) {
2280 /* If we are only owner of page we can reuse it */
2281 if (likely(page_count(desc_cb->priv) == 1)) {
2282 /* Flip page offset to other buffer */
2283 desc_cb->page_offset ^= truesize;
2284
2285 desc_cb->reuse_flag = 1;
2286 /* bump ref count on page before it is given*/
2287 get_page(desc_cb->priv);
2288 }
2289 return;
2290 }
2291
2292 /* Move offset up to the next cache line */
2293 desc_cb->page_offset += truesize;
2294
2295 if (desc_cb->page_offset <= last_offset) {
2296 desc_cb->reuse_flag = 1;
2297 /* Bump ref count on page before it is given*/
2298 get_page(desc_cb->priv);
2299 }
2300}
2301
2302static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
2303 struct hns3_desc *desc)
2304{
2305 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2306 int l3_type, l4_type;
2307 u32 bd_base_info;
2308 int ol4_type;
2309 u32 l234info;
2310
2311 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2312 l234info = le32_to_cpu(desc->rx.l234_info);
2313
2314 skb->ip_summed = CHECKSUM_NONE;
2315
2316 skb_checksum_none_assert(skb);
2317
2318 if (!(netdev->features & NETIF_F_RXCSUM))
2319 return;
2320
2321 /* check if hardware has done checksum */
e4e87715 2322 if (!hnae3_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
76ad4f0e
S		 2323 return;
2324
e4e87715
PL		 2325 if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L3E_B) ||
2326 hnae3_get_bit(l234info, HNS3_RXD_L4E_B) ||
2327 hnae3_get_bit(l234info, HNS3_RXD_OL3E_B) ||
2328 hnae3_get_bit(l234info, HNS3_RXD_OL4E_B))) {
76ad4f0e
S		 2329 u64_stats_update_begin(&ring->syncp);
2330 ring->stats.l3l4_csum_err++;
2331 u64_stats_update_end(&ring->syncp);
2332
2333 return;
2334 }
2335
e4e87715
PL		 2336 l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,
2337 HNS3_RXD_L3ID_S);
2338 l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,
2339 HNS3_RXD_L4ID_S);
76ad4f0e 2340
e4e87715
PL		 2341 ol4_type = hnae3_get_field(l234info, HNS3_RXD_OL4ID_M,
2342 HNS3_RXD_OL4ID_S);
76ad4f0e
S		 2343 switch (ol4_type) {
2344 case HNS3_OL4_TYPE_MAC_IN_UDP:
2345 case HNS3_OL4_TYPE_NVGRE:
2346 skb->csum_level = 1;
be44b3af 2347 /* fall through */
76ad4f0e
S		 2348 case HNS3_OL4_TYPE_NO_TUN:
2349 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
94c5e532
PL		 2350 if ((l3_type == HNS3_L3_TYPE_IPV4 ||
2351 l3_type == HNS3_L3_TYPE_IPV6) &&
2352 (l4_type == HNS3_L4_TYPE_UDP ||
2353 l4_type == HNS3_L4_TYPE_TCP ||
2354 l4_type == HNS3_L4_TYPE_SCTP))
76ad4f0e
S		 2355 skb->ip_summed = CHECKSUM_UNNECESSARY;
2356 break;
fa7a4bd5
JS		 2357 default:
2358 break;
76ad4f0e
S		 2359 }
2360}
2361
d43e5aca
YL		 2362static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2363{
2364 napi_gro_receive(&ring->tqp_vector->napi, skb);
2365}
2366
701a6d6a
JS		 2367static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,
2368 struct hns3_desc *desc, u32 l234info,
2369 u16 *vlan_tag)
5b5455a9
PL		 2370{
2371 struct pci_dev *pdev = ring->tqp->handle->pdev;
5b5455a9
PL		 2372
2373 if (pdev->revision == 0x20) {
701a6d6a
JS		 2374 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2375 if (!(*vlan_tag & VLAN_VID_MASK))
2376 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
5b5455a9 2377
701a6d6a 2378 return (*vlan_tag != 0);
5b5455a9
PL		 2379 }
2380
2381#define HNS3_STRP_OUTER_VLAN 0x1
2382#define HNS3_STRP_INNER_VLAN 0x2
2383
e4e87715
PL		 2384 switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,
2385 HNS3_RXD_STRP_TAGP_S)) {
5b5455a9 2386 case HNS3_STRP_OUTER_VLAN:
701a6d6a
JS		 2387 *vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);
2388 return true;
5b5455a9 2389 case HNS3_STRP_INNER_VLAN:
701a6d6a
JS		 2390 *vlan_tag = le16_to_cpu(desc->rx.vlan_tag);
2391 return true;
5b5455a9 2392 default:
701a6d6a 2393 return false;
5b5455a9 2394 }
5b5455a9
PL		 2395}
2396
e5597095
PL		 2397static int hns3_alloc_skb(struct hns3_enet_ring *ring, int length,
2398 unsigned char *va)
2399{
2400#define HNS3_NEED_ADD_FRAG 1
2401 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
2402 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2403 struct sk_buff *skb;
2404
2405 ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);
2406 skb = ring->skb;
2407 if (unlikely(!skb)) {
2408 netdev_err(netdev, "alloc rx skb fail\n");
2409
2410 u64_stats_update_begin(&ring->syncp);
2411 ring->stats.sw_err_cnt++;
2412 u64_stats_update_end(&ring->syncp);
2413
2414 return -ENOMEM;
2415 }
2416
2417 prefetchw(skb->data);
2418
2419 ring->pending_buf = 1;
2420 if (length <= HNS3_RX_HEAD_SIZE) {
2421 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2422
2423 /* We can reuse buffer as-is, just make sure it is local */
2424 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2425 desc_cb->reuse_flag = 1;
2426 else /* This page cannot be reused so discard it */
2427 put_page(desc_cb->priv);
2428
2429 ring_ptr_move_fw(ring, next_to_clean);
2430 return 0;
2431 }
2432 u64_stats_update_begin(&ring->syncp);
2433 ring->stats.seg_pkt_cnt++;
2434 u64_stats_update_end(&ring->syncp);
2435
2436 ring->pull_len = eth_get_headlen(va, HNS3_RX_HEAD_SIZE);
2437 __skb_put(skb, ring->pull_len);
2438 hns3_nic_reuse_page(skb, 0, ring, ring->pull_len,
2439 desc_cb);
2440 ring_ptr_move_fw(ring, next_to_clean);
2441
2442 return HNS3_NEED_ADD_FRAG;
2443}
2444
2445static int hns3_add_frag(struct hns3_enet_ring *ring, struct hns3_desc *desc,
2446 struct sk_buff **out_skb, bool pending)
2447{
2448 struct sk_buff *skb = *out_skb;
2449 struct hns3_desc_cb *desc_cb;
2450 struct hns3_desc *pre_desc;
2451 u32 bd_base_info;
2452 int pre_bd;
2453
2454 /* if there is pending bd, the SW param next_to_clean has moved
2455 * to next and the next is NULL
2456 */
2457 if (pending) {
2458 pre_bd = (ring->next_to_clean - 1 + ring->desc_num) %
2459 ring->desc_num;
2460 pre_desc = &ring->desc[pre_bd];
2461 bd_base_info = le32_to_cpu(pre_desc->rx.bd_base_info);
2462 } else {
2463 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2464 }
2465
2466 while (!hnae3_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2467 desc = &ring->desc[ring->next_to_clean];
2468 desc_cb = &ring->desc_cb[ring->next_to_clean];
2469 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2470 if (!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2471 return -ENXIO;
2472
2473 hns3_nic_reuse_page(skb, ring->pending_buf, ring, 0, desc_cb);
2474 ring_ptr_move_fw(ring, next_to_clean);
2475 ring->pending_buf++;
2476 }
2477
2478 return 0;
2479}
2480
232fc64b
PL		 2481static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,
2482 struct sk_buff *skb)
2483{
2484 struct hns3_desc *desc = &ring->desc[ring->next_to_clean];
2485 struct hnae3_handle *handle = ring->tqp->handle;
2486 enum pkt_hash_types rss_type;
2487
2488 if (le32_to_cpu(desc->rx.rss_hash))
2489 rss_type = handle->kinfo.rss_type;
2490 else
2491 rss_type = PKT_HASH_TYPE_NONE;
2492
2493 skb_set_hash(skb, le32_to_cpu(desc->rx.rss_hash), rss_type);
2494}
2495
76ad4f0e 2496static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
e5597095 2497 struct sk_buff **out_skb)
76ad4f0e
S		 2498{
2499 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
e5597095 2500 struct sk_buff *skb = ring->skb;
76ad4f0e
S		 2501 struct hns3_desc_cb *desc_cb;
2502 struct hns3_desc *desc;
76ad4f0e 2503 u32 bd_base_info;
76ad4f0e
S		 2504 u32 l234info;
2505 int length;
e5597095 2506 int ret;
76ad4f0e
S		 2507
2508 desc = &ring->desc[ring->next_to_clean];
2509 desc_cb = &ring->desc_cb[ring->next_to_clean];
2510
2511 prefetch(desc);
2512
846fcc83 2513 length = le16_to_cpu(desc->rx.size);
76ad4f0e 2514 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
76ad4f0e
S		 2515
2516 /* Check valid BD */
e4e87715 2517 if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B)))
e5597095 2518 return -ENXIO;
76ad4f0e 2519
e5597095
PL		 2520 if (!skb)
2521 ring->va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
76ad4f0e
S		 2522
2523 /* Prefetch first cache line of first page
2524 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2525 * line size is 64B so need to prefetch twice to make it 128B. But in
2526 * actual we can have greater size of caches with 128B Level 1 cache
2527 * lines. In such a case, single fetch would suffice to cache in the
2528 * relevant part of the header.
2529 */
e5597095 2530 prefetch(ring->va);
76ad4f0e 2531#if L1_CACHE_BYTES < 128
e5597095 2532 prefetch(ring->va + L1_CACHE_BYTES);
76ad4f0e
S		 2533#endif
2534
e5597095
PL		 2535 if (!skb) {
2536 ret = hns3_alloc_skb(ring, length, ring->va);
2537 *out_skb = skb = ring->skb;
76ad4f0e 2538
e5597095
PL		 2539 if (ret < 0) /* alloc buffer fail */
2540 return ret;
2541 if (ret > 0) { /* need add frag */
2542 ret = hns3_add_frag(ring, desc, &skb, false);
2543 if (ret)
2544 return ret;
76ad4f0e 2545
e5597095
PL		 2546 /* As the head data may be changed when GRO enable, copy
2547 * the head data in after other data rx completed
2548 */
2549 memcpy(skb->data, ring->va,
2550 ALIGN(ring->pull_len, sizeof(long)));
2551 }
76ad4f0e 2552 } else {
e5597095
PL		 2553 ret = hns3_add_frag(ring, desc, &skb, true);
2554 if (ret)
2555 return ret;
76ad4f0e 2556
e5597095
PL		 2557 /* As the head data may be changed when GRO enable, copy
2558 * the head data in after other data rx completed
2559 */
2560 memcpy(skb->data, ring->va,
2561 ALIGN(ring->pull_len, sizeof(long)));
76ad4f0e
S		 2562 }
2563
5b5455a9 2564 l234info = le32_to_cpu(desc->rx.l234_info);
e5597095 2565 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
5b5455a9 2566
846fcc83
PL		 2567 /* Based on hw strategy, the tag offloaded will be stored at
2568 * ot_vlan_tag in two layer tag case, and stored at vlan_tag
2569 * in one layer tag case.
2570 */
2571 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
2572 u16 vlan_tag;
2573
701a6d6a 2574 if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))
846fcc83
PL		 2575 __vlan_hwaccel_put_tag(skb,
2576 htons(ETH_P_8021Q),
2577 vlan_tag);
2578 }
2579
e4e87715 2580 if (unlikely(!hnae3_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
76ad4f0e
S		 2581 u64_stats_update_begin(&ring->syncp);
2582 ring->stats.non_vld_descs++;
2583 u64_stats_update_end(&ring->syncp);
2584
2585 dev_kfree_skb_any(skb);
2586 return -EINVAL;
2587 }
2588
2589 if (unlikely((!desc->rx.pkt_len) ||
e4e87715 2590 hnae3_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
76ad4f0e
S		 2591 u64_stats_update_begin(&ring->syncp);
2592 ring->stats.err_pkt_len++;
2593 u64_stats_update_end(&ring->syncp);
2594
2595 dev_kfree_skb_any(skb);
2596 return -EFAULT;
2597 }
2598
e4e87715 2599 if (unlikely(hnae3_get_bit(l234info, HNS3_RXD_L2E_B))) {
76ad4f0e
S		 2600 u64_stats_update_begin(&ring->syncp);
2601 ring->stats.l2_err++;
2602 u64_stats_update_end(&ring->syncp);
2603
2604 dev_kfree_skb_any(skb);
2605 return -EFAULT;
2606 }
2607
2608 u64_stats_update_begin(&ring->syncp);
2609 ring->stats.rx_pkts++;
2610 ring->stats.rx_bytes += skb->len;
2611 u64_stats_update_end(&ring->syncp);
2612
2613 ring->tqp_vector->rx_group.total_bytes += skb->len;
2614
2615 hns3_rx_checksum(ring, skb, desc);
e5597095 2616 *out_skb = skb;
232fc64b
PL		 2617 hns3_set_rx_skb_rss_type(ring, skb);
2618
76ad4f0e
S		 2619 return 0;
2620}
2621
d43e5aca
YL		 2622int hns3_clean_rx_ring(
2623 struct hns3_enet_ring *ring, int budget,
2624 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2625{
2626#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2627 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2628 int recv_pkts, recv_bds, clean_count, err;
e5597095
PL		 2629 int unused_count = hns3_desc_unused(ring) - ring->pending_buf;
2630 struct sk_buff *skb = ring->skb;
2631 int num;
76ad4f0e
S		 2632
2633 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2634 rmb(); /* Make sure num taken effect before the other data is touched */
2635
2636 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2637 num -= unused_count;
2638
2639 while (recv_pkts < budget && recv_bds < num) {
2640 /* Reuse or realloc buffers */
2641 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2642 hns3_nic_alloc_rx_buffers(ring,
2643 clean_count + unused_count);
2644 clean_count = 0;
e5597095
PL		 2645 unused_count = hns3_desc_unused(ring) -
2646 ring->pending_buf;
76ad4f0e
S		 2647 }
2648
2649 /* Poll one pkt */
e5597095 2650 err = hns3_handle_rx_bd(ring, &skb);
76ad4f0e
S		 2651 if (unlikely(!skb)) /* This fault cannot be repaired */
2652 goto out;
2653
e5597095
PL		 2654 if (err == -ENXIO) { /* Do not get FE for the packet */
2655 goto out;
2656 } else if (unlikely(err)) { /* Do jump the err */
2657 recv_bds += ring->pending_buf;
2658 clean_count += ring->pending_buf;
2659 ring->skb = NULL;
2660 ring->pending_buf = 0;
76ad4f0e
S		 2661 continue;
2662 }
2663
2664 /* Do update ip stack process */
2665 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2666 rx_fn(ring, skb);
e5597095
PL		 2667 recv_bds += ring->pending_buf;
2668 clean_count += ring->pending_buf;
2669 ring->skb = NULL;
2670 ring->pending_buf = 0;
76ad4f0e
S		 2671
2672 recv_pkts++;
2673 }
2674
2675out:
2676 /* Make all data has been write before submit */
2677 if (clean_count + unused_count > 0)
2678 hns3_nic_alloc_rx_buffers(ring,
2679 clean_count + unused_count);
2680
2681 return recv_pkts;
2682}
2683
2684static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2685{
a95e1f86
FL		 2686 struct hns3_enet_tqp_vector *tqp_vector =
2687 ring_group->ring->tqp_vector;
76ad4f0e 2688 enum hns3_flow_level_range new_flow_level;
a95e1f86
FL		 2689 int packets_per_msecs;
2690 int bytes_per_msecs;
2691 u32 time_passed_ms;
76ad4f0e 2692 u16 new_int_gl;
76ad4f0e 2693
a95e1f86 2694 if (!ring_group->coal.int_gl || !tqp_vector->last_jiffies)
76ad4f0e
S		 2695 return false;
2696
2697 if (ring_group->total_packets == 0) {
9bc727a9
YL		 2698 ring_group->coal.int_gl = HNS3_INT_GL_50K;
2699 ring_group->coal.flow_level = HNS3_FLOW_LOW;
76ad4f0e
S		 2700 return true;
2701 }
2702
2703 /* Simple throttlerate management
2704 * 0-10MB/s lower (50000 ints/s)
2705 * 10-20MB/s middle (20000 ints/s)
2706 * 20-1249MB/s high (18000 ints/s)
2707 * > 40000pps ultra (8000 ints/s)
2708 */
9bc727a9
YL		 2709 new_flow_level = ring_group->coal.flow_level;
2710 new_int_gl = ring_group->coal.int_gl;
a95e1f86
FL		 2711 time_passed_ms =
2712 jiffies_to_msecs(jiffies - tqp_vector->last_jiffies);
2713
2714 if (!time_passed_ms)
2715 return false;
2716
2717 do_div(ring_group->total_packets, time_passed_ms);
2718 packets_per_msecs = ring_group->total_packets;
2719
2720 do_div(ring_group->total_bytes, time_passed_ms);
2721 bytes_per_msecs = ring_group->total_bytes;
2722
2723#define HNS3_RX_LOW_BYTE_RATE 10000
2724#define HNS3_RX_MID_BYTE_RATE 20000
76ad4f0e
S		 2725
2726 switch (new_flow_level) {
2727 case HNS3_FLOW_LOW:
a95e1f86 2728 if (bytes_per_msecs > HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2729 new_flow_level = HNS3_FLOW_MID;
2730 break;
2731 case HNS3_FLOW_MID:
a95e1f86 2732 if (bytes_per_msecs > HNS3_RX_MID_BYTE_RATE)
76ad4f0e 2733 new_flow_level = HNS3_FLOW_HIGH;
a95e1f86 2734 else if (bytes_per_msecs <= HNS3_RX_LOW_BYTE_RATE)
76ad4f0e
S		 2735 new_flow_level = HNS3_FLOW_LOW;
2736 break;
2737 case HNS3_FLOW_HIGH:
2738 case HNS3_FLOW_ULTRA:
2739 default:
a95e1f86 2740 if (bytes_per_msecs <= HNS3_RX_MID_BYTE_RATE)
76ad4f0e
S		 2741 new_flow_level = HNS3_FLOW_MID;
2742 break;
2743 }
2744
a95e1f86
FL		 2745#define HNS3_RX_ULTRA_PACKET_RATE 40
2746
2747 if (packets_per_msecs > HNS3_RX_ULTRA_PACKET_RATE &&
2748 &tqp_vector->rx_group == ring_group)
76ad4f0e
S		 2749 new_flow_level = HNS3_FLOW_ULTRA;
2750
2751 switch (new_flow_level) {
2752 case HNS3_FLOW_LOW:
2753 new_int_gl = HNS3_INT_GL_50K;
2754 break;
2755 case HNS3_FLOW_MID:
2756 new_int_gl = HNS3_INT_GL_20K;
2757 break;
2758 case HNS3_FLOW_HIGH:
2759 new_int_gl = HNS3_INT_GL_18K;
2760 break;
2761 case HNS3_FLOW_ULTRA:
2762 new_int_gl = HNS3_INT_GL_8K;
2763 break;
2764 default:
2765 break;
2766 }
2767
2768 ring_group->total_bytes = 0;
2769 ring_group->total_packets = 0;
9bc727a9
YL		 2770 ring_group->coal.flow_level = new_flow_level;
2771 if (new_int_gl != ring_group->coal.int_gl) {
2772 ring_group->coal.int_gl = new_int_gl;
76ad4f0e
S		 2773 return true;
2774 }
2775 return false;
2776}
2777
2778static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2779{
8b1ff1ea
FL		 2780 struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;
2781 struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;
2782 bool rx_update, tx_update;
2783
cd9d187b
FL		 2784 if (tqp_vector->int_adapt_down > 0) {
2785 tqp_vector->int_adapt_down--;
2786 return;
2787 }
2788
9bc727a9 2789 if (rx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2790 rx_update = hns3_get_new_int_gl(rx_group);
2791 if (rx_update)
2792 hns3_set_vector_coalesce_rx_gl(tqp_vector,
9bc727a9 2793 rx_group->coal.int_gl);
8b1ff1ea
FL		 2794 }
2795
9bc727a9 2796 if (tx_group->coal.gl_adapt_enable) {
8b1ff1ea
FL		 2797 tx_update = hns3_get_new_int_gl(&tqp_vector->tx_group);
2798 if (tx_update)
2799 hns3_set_vector_coalesce_tx_gl(tqp_vector,
9bc727a9 2800 tx_group->coal.int_gl);
76ad4f0e 2801 }
cd9d187b 2802
a95e1f86 2803 tqp_vector->last_jiffies = jiffies;
cd9d187b 2804 tqp_vector->int_adapt_down = HNS3_INT_ADAPT_DOWN_START;
76ad4f0e
S		 2805}
2806
2807static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2808{
ff0699e0 2809 struct hns3_nic_priv *priv = netdev_priv(napi->dev);
76ad4f0e
S		 2810 struct hns3_enet_ring *ring;
2811 int rx_pkt_total = 0;
2812
2813 struct hns3_enet_tqp_vector *tqp_vector =
2814 container_of(napi, struct hns3_enet_tqp_vector, napi);
2815 bool clean_complete = true;
2816 int rx_budget;
2817
ff0699e0
HT		 2818 if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {
2819 napi_complete(napi);
2820 return 0;
2821 }
2822
76ad4f0e
S		 2823 /* Since the actual Tx work is minimal, we can give the Tx a larger
2824 * budget and be more aggressive about cleaning up the Tx descriptors.
2825 */
799997a3
PL		 2826 hns3_for_each_ring(ring, tqp_vector->tx_group)
2827 hns3_clean_tx_ring(ring);
76ad4f0e
S		 2828
2829 /* make sure rx ring budget not smaller than 1 */
2830 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2831
2832 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2833 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2834 hns3_rx_skb);
76ad4f0e
S		 2835
2836 if (rx_cleaned >= rx_budget)
2837 clean_complete = false;
2838
2839 rx_pkt_total += rx_cleaned;
2840 }
2841
2842 tqp_vector->rx_group.total_packets += rx_pkt_total;
2843
2844 if (!clean_complete)
2845 return budget;
2846
ff0699e0
HT		 2847 if (likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) &&
2848 napi_complete(napi)) {
2849 hns3_update_new_int_gl(tqp_vector);
2850 hns3_mask_vector_irq(tqp_vector, 1);
2851 }
76ad4f0e
S		 2852
2853 return rx_pkt_total;
2854}
2855
2856static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2857 struct hnae3_ring_chain_node *head)
2858{
2859 struct pci_dev *pdev = tqp_vector->handle->pdev;
2860 struct hnae3_ring_chain_node *cur_chain = head;
2861 struct hnae3_ring_chain_node *chain;
2862 struct hns3_enet_ring *tx_ring;
2863 struct hns3_enet_ring *rx_ring;
2864
2865 tx_ring = tqp_vector->tx_group.ring;
2866 if (tx_ring) {
2867 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
e4e87715
PL		 2868 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2869 HNAE3_RING_TYPE_TX);
2870 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2871 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX);
76ad4f0e
S		 2872
2873 cur_chain->next = NULL;
2874
2875 while (tx_ring->next) {
2876 tx_ring = tx_ring->next;
2877
2878 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2879 GFP_KERNEL);
2880 if (!chain)
73b907a0 2881 goto err_free_chain;
76ad4f0e
S		 2882
2883 cur_chain->next = chain;
2884 chain->tqp_index = tx_ring->tqp->tqp_index;
e4e87715
PL		 2885 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2886 HNAE3_RING_TYPE_TX);
2887 hnae3_set_field(chain->int_gl_idx,
2888 HNAE3_RING_GL_IDX_M,
2889 HNAE3_RING_GL_IDX_S,
2890 HNAE3_RING_GL_TX);
76ad4f0e
S		 2891
2892 cur_chain = chain;
2893 }
2894 }
2895
2896 rx_ring = tqp_vector->rx_group.ring;
2897 if (!tx_ring && rx_ring) {
2898 cur_chain->next = NULL;
2899 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
e4e87715
PL		 2900 hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2901 HNAE3_RING_TYPE_RX);
2902 hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2903 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
76ad4f0e
S		 2904
2905 rx_ring = rx_ring->next;
2906 }
2907
2908 while (rx_ring) {
2909 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2910 if (!chain)
73b907a0 2911 goto err_free_chain;
76ad4f0e
S		 2912
2913 cur_chain->next = chain;
2914 chain->tqp_index = rx_ring->tqp->tqp_index;
e4e87715
PL		 2915 hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2916 HNAE3_RING_TYPE_RX);
2917 hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M,
2918 HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX);
11af96a4 2919
76ad4f0e
S		 2920 cur_chain = chain;
2921
2922 rx_ring = rx_ring->next;
2923 }
2924
2925 return 0;
73b907a0
HT		 2926
2927err_free_chain:
2928 cur_chain = head->next;
2929 while (cur_chain) {
2930 chain = cur_chain->next;
2931 devm_kfree(&pdev->dev, chain);
2932 cur_chain = chain;
2933 }
2934
2935 return -ENOMEM;
76ad4f0e
S		 2936}
2937
2938static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2939 struct hnae3_ring_chain_node *head)
2940{
2941 struct pci_dev *pdev = tqp_vector->handle->pdev;
2942 struct hnae3_ring_chain_node *chain_tmp, *chain;
2943
2944 chain = head->next;
2945
2946 while (chain) {
2947 chain_tmp = chain->next;
2948 devm_kfree(&pdev->dev, chain);
2949 chain = chain_tmp;
2950 }
2951}
2952
2953static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2954 struct hns3_enet_ring *ring)
2955{
2956 ring->next = group->ring;
2957 group->ring = ring;
2958
2959 group->count++;
2960}
2961
874bff0b
PL		 2962static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)
2963{
2964 struct pci_dev *pdev = priv->ae_handle->pdev;
2965 struct hns3_enet_tqp_vector *tqp_vector;
2966 int num_vectors = priv->vector_num;
2967 int numa_node;
2968 int vector_i;
2969
2970 numa_node = dev_to_node(&pdev->dev);
2971
2972 for (vector_i = 0; vector_i < num_vectors; vector_i++) {
2973 tqp_vector = &priv->tqp_vector[vector_i];
2974 cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),
2975 &tqp_vector->affinity_mask);
2976 }
2977}
2978
76ad4f0e
S		 2979static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2980{
2981 struct hnae3_ring_chain_node vector_ring_chain;
2982 struct hnae3_handle *h = priv->ae_handle;
2983 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e 2984 int ret = 0;
ece4bf46 2985 int i;
76ad4f0e 2986
874bff0b
PL		 2987 hns3_nic_set_cpumask(priv);
2988
dd38c726
YL		 2989 for (i = 0; i < priv->vector_num; i++) {
2990 tqp_vector = &priv->tqp_vector[i];
2991 hns3_vector_gl_rl_init_hw(tqp_vector, priv);
2992 tqp_vector->num_tqps = 0;
2993 }
76ad4f0e 2994
dd38c726
YL		 2995 for (i = 0; i < h->kinfo.num_tqps; i++) {
2996 u16 vector_i = i % priv->vector_num;
2997 u16 tqp_num = h->kinfo.num_tqps;
76ad4f0e
S		 2998
2999 tqp_vector = &priv->tqp_vector[vector_i];
3000
3001 hns3_add_ring_to_group(&tqp_vector->tx_group,
3002 priv->ring_data[i].ring);
3003
3004 hns3_add_ring_to_group(&tqp_vector->rx_group,
3005 priv->ring_data[i + tqp_num].ring);
3006
76ad4f0e
S		 3007 priv->ring_data[i].ring->tqp_vector = tqp_vector;
3008 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
dd38c726 3009 tqp_vector->num_tqps++;
76ad4f0e
S		 3010 }
3011
dd38c726 3012 for (i = 0; i < priv->vector_num; i++) {
76ad4f0e
S		 3013 tqp_vector = &priv->tqp_vector[i];
3014
3015 tqp_vector->rx_group.total_bytes = 0;
3016 tqp_vector->rx_group.total_packets = 0;
3017 tqp_vector->tx_group.total_bytes = 0;
3018 tqp_vector->tx_group.total_packets = 0;
76ad4f0e
S		 3019 tqp_vector->handle = h;
3020
3021 ret = hns3_get_vector_ring_chain(tqp_vector,
3022 &vector_ring_chain);
3023 if (ret)
dd38c726 3024 return ret;
76ad4f0e
S		 3025
3026 ret = h->ae_algo->ops->map_ring_to_vector(h,
3027 tqp_vector->vector_irq, &vector_ring_chain);
76ad4f0e
S		 3028
3029 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3030
dd38c726 3031 if (ret)
ece4bf46 3032 goto map_ring_fail;
dd38c726 3033
76ad4f0e
S		 3034 netif_napi_add(priv->netdev, &tqp_vector->napi,
3035 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
3036 }
3037
dd38c726 3038 return 0;
ece4bf46
HT		 3039
3040map_ring_fail:
3041 while (i--)
3042 netif_napi_del(&priv->tqp_vector[i].napi);
3043
3044 return ret;
dd38c726
YL		 3045}
3046
3047static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)
3048{
3049 struct hnae3_handle *h = priv->ae_handle;
3050 struct hns3_enet_tqp_vector *tqp_vector;
3051 struct hnae3_vector_info *vector;
3052 struct pci_dev *pdev = h->pdev;
3053 u16 tqp_num = h->kinfo.num_tqps;
3054 u16 vector_num;
3055 int ret = 0;
3056 u16 i;
3057
3058 /* RSS size, cpu online and vector_num should be the same */
3059 /* Should consider 2p/4p later */
3060 vector_num = min_t(u16, num_online_cpus(), tqp_num);
3061 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
3062 GFP_KERNEL);
3063 if (!vector)
3064 return -ENOMEM;
3065
3066 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
3067
3068 priv->vector_num = vector_num;
3069 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
3070 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
3071 GFP_KERNEL);
3072 if (!priv->tqp_vector) {
3073 ret = -ENOMEM;
3074 goto out;
3075 }
3076
3077 for (i = 0; i < priv->vector_num; i++) {
3078 tqp_vector = &priv->tqp_vector[i];
3079 tqp_vector->idx = i;
3080 tqp_vector->mask_addr = vector[i].io_addr;
3081 tqp_vector->vector_irq = vector[i].vector;
3082 hns3_vector_gl_rl_init(tqp_vector, priv);
3083 }
3084
76ad4f0e
S		 3085out:
3086 devm_kfree(&pdev->dev, vector);
3087 return ret;
3088}
3089
dd38c726
YL		 3090static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)
3091{
3092 group->ring = NULL;
3093 group->count = 0;
3094}
3095
76ad4f0e
S		 3096static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
3097{
3098 struct hnae3_ring_chain_node vector_ring_chain;
3099 struct hnae3_handle *h = priv->ae_handle;
3100 struct hns3_enet_tqp_vector *tqp_vector;
76ad4f0e
S		 3101 int i, ret;
3102
3103 for (i = 0; i < priv->vector_num; i++) {
3104 tqp_vector = &priv->tqp_vector[i];
3105
3106 ret = hns3_get_vector_ring_chain(tqp_vector,
3107 &vector_ring_chain);
3108 if (ret)
3109 return ret;
3110
3111 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
3112 tqp_vector->vector_irq, &vector_ring_chain);
3113 if (ret)
3114 return ret;
3115
3116 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
3117
3118 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
3119 (void)irq_set_affinity_hint(
3120 priv->tqp_vector[i].vector_irq,
3121 NULL);
ae064e61 3122 free_irq(priv->tqp_vector[i].vector_irq,
3123 &priv->tqp_vector[i]);
76ad4f0e
S		 3124 }
3125
3126 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
dd38c726
YL		 3127 hns3_clear_ring_group(&tqp_vector->rx_group);
3128 hns3_clear_ring_group(&tqp_vector->tx_group);
76ad4f0e
S		 3129 netif_napi_del(&priv->tqp_vector[i].napi);
3130 }
3131
dd38c726
YL		 3132 return 0;
3133}
3134
3135static int hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)
3136{
3137 struct hnae3_handle *h = priv->ae_handle;
3138 struct pci_dev *pdev = h->pdev;
3139 int i, ret;
3140
3141 for (i = 0; i < priv->vector_num; i++) {
3142 struct hns3_enet_tqp_vector *tqp_vector;
3143
3144 tqp_vector = &priv->tqp_vector[i];
3145 ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);
3146 if (ret)
3147 return ret;
3148 }
76ad4f0e 3149
dd38c726 3150 devm_kfree(&pdev->dev, priv->tqp_vector);
76ad4f0e
S		 3151 return 0;
3152}
3153
3154static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
3155 int ring_type)
3156{
3157 struct hns3_nic_ring_data *ring_data = priv->ring_data;
3158 int queue_num = priv->ae_handle->kinfo.num_tqps;
3159 struct pci_dev *pdev = priv->ae_handle->pdev;
3160 struct hns3_enet_ring *ring;
3161
3162 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
3163 if (!ring)
3164 return -ENOMEM;
3165
3166 if (ring_type == HNAE3_RING_TYPE_TX) {
3167 ring_data[q->tqp_index].ring = ring;
66b44730 3168 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 3169 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
3170 } else {
3171 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 3172 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 3173 ring->io_base = q->io_base;
3174 }
3175
e4e87715 3176 hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
76ad4f0e 3177
76ad4f0e
S		 3178 ring->tqp = q;
3179 ring->desc = NULL;
3180 ring->desc_cb = NULL;
3181 ring->dev = priv->dev;
3182 ring->desc_dma_addr = 0;
3183 ring->buf_size = q->buf_size;
3184 ring->desc_num = q->desc_num;
3185 ring->next_to_use = 0;
3186 ring->next_to_clean = 0;
3187
3188 return 0;
3189}
3190
3191static int hns3_queue_to_ring(struct hnae3_queue *tqp,
3192 struct hns3_nic_priv *priv)
3193{
3194 int ret;
3195
3196 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
3197 if (ret)
3198 return ret;
3199
3200 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
73b907a0
HT		 3201 if (ret) {
3202 devm_kfree(priv->dev, priv->ring_data[tqp->tqp_index].ring);
76ad4f0e 3203 return ret;
73b907a0 3204 }
76ad4f0e
S		 3205
3206 return 0;
3207}
3208
3209static int hns3_get_ring_config(struct hns3_nic_priv *priv)
3210{
3211 struct hnae3_handle *h = priv->ae_handle;
3212 struct pci_dev *pdev = h->pdev;
3213 int i, ret;
3214
a86854d0
KC		 3215 priv->ring_data = devm_kzalloc(&pdev->dev,
3216 array3_size(h->kinfo.num_tqps,
3217 sizeof(*priv->ring_data),
3218 2),
76ad4f0e
S		 3219 GFP_KERNEL);
3220 if (!priv->ring_data)
3221 return -ENOMEM;
3222
3223 for (i = 0; i < h->kinfo.num_tqps; i++) {
3224 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
3225 if (ret)
3226 goto err;
3227 }
3228
3229 return 0;
3230err:
73b907a0
HT		 3231 while (i--) {
3232 devm_kfree(priv->dev, priv->ring_data[i].ring);
3233 devm_kfree(priv->dev,
3234 priv->ring_data[i + h->kinfo.num_tqps].ring);
3235 }
3236
76ad4f0e
S		 3237 devm_kfree(&pdev->dev, priv->ring_data);
3238 return ret;
3239}
3240
09f2af64
PL		 3241static void hns3_put_ring_config(struct hns3_nic_priv *priv)
3242{
3243 struct hnae3_handle *h = priv->ae_handle;
3244 int i;
3245
3246 for (i = 0; i < h->kinfo.num_tqps; i++) {
3247 devm_kfree(priv->dev, priv->ring_data[i].ring);
3248 devm_kfree(priv->dev,
3249 priv->ring_data[i + h->kinfo.num_tqps].ring);
3250 }
3251 devm_kfree(priv->dev, priv->ring_data);
3252}
3253
76ad4f0e
S		 3254static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
3255{
3256 int ret;
3257
3258 if (ring->desc_num <= 0 || ring->buf_size <= 0)
3259 return -EINVAL;
3260
3261 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
3262 GFP_KERNEL);
3263 if (!ring->desc_cb) {
3264 ret = -ENOMEM;
3265 goto out;
3266 }
3267
3268 ret = hns3_alloc_desc(ring);
3269 if (ret)
3270 goto out_with_desc_cb;
3271
3272 if (!HNAE3_IS_TX_RING(ring)) {
3273 ret = hns3_alloc_ring_buffers(ring);
3274 if (ret)
3275 goto out_with_desc;
3276 }
3277
3278 return 0;
3279
3280out_with_desc:
3281 hns3_free_desc(ring);
3282out_with_desc_cb:
3283 kfree(ring->desc_cb);
3284 ring->desc_cb = NULL;
3285out:
3286 return ret;
3287}
3288
3289static void hns3_fini_ring(struct hns3_enet_ring *ring)
3290{
3291 hns3_free_desc(ring);
3292 kfree(ring->desc_cb);
3293 ring->desc_cb = NULL;
3294 ring->next_to_clean = 0;
3295 ring->next_to_use = 0;
3296}
3297
1db9b1bf 3298static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 3299{
3300 int bd_size_type;
3301
3302 switch (buf_size) {
3303 case 512:
3304 bd_size_type = HNS3_BD_SIZE_512_TYPE;
3305 break;
3306 case 1024:
3307 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
3308 break;
3309 case 2048:
3310 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3311 break;
3312 case 4096:
3313 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
3314 break;
3315 default:
3316 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
3317 }
3318
3319 return bd_size_type;
3320}
3321
3322static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
3323{
3324 dma_addr_t dma = ring->desc_dma_addr;
3325 struct hnae3_queue *q = ring->tqp;
3326
3327 if (!HNAE3_IS_TX_RING(ring)) {
3328 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
3329 (u32)dma);
3330 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
3331 (u32)((dma >> 31) >> 1));
3332
3333 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
3334 hns3_buf_size2type(ring->buf_size));
3335 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
3336 ring->desc_num / 8 - 1);
3337
3338 } else {
3339 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
3340 (u32)dma);
3341 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
3342 (u32)((dma >> 31) >> 1));
3343
76ad4f0e
S		 3344 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
3345 ring->desc_num / 8 - 1);
3346 }
3347}
3348
1c772154
YL		 3349static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)
3350{
3351 struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;
3352 int i;
3353
3354 for (i = 0; i < HNAE3_MAX_TC; i++) {
3355 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3356 int j;
3357
3358 if (!tc_info->enable)
3359 continue;
3360
3361 for (j = 0; j < tc_info->tqp_count; j++) {
3362 struct hnae3_queue *q;
3363
3364 q = priv->ring_data[tc_info->tqp_offset + j].ring->tqp;
3365 hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG,
3366 tc_info->tc);
3367 }
3368 }
3369}
3370
5668abda 3371int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3372{
3373 struct hnae3_handle *h = priv->ae_handle;
3374 int ring_num = h->kinfo.num_tqps * 2;
3375 int i, j;
3376 int ret;
3377
3378 for (i = 0; i < ring_num; i++) {
3379 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
3380 if (ret) {
3381 dev_err(priv->dev,
3382 "Alloc ring memory fail! ret=%d\n", ret);
3383 goto out_when_alloc_ring_memory;
3384 }
3385
76ad4f0e
S		 3386 u64_stats_init(&priv->ring_data[i].ring->syncp);
3387 }
3388
3389 return 0;
3390
3391out_when_alloc_ring_memory:
3392 for (j = i - 1; j >= 0; j--)
ee83f776 3393 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 3394
3395 return -ENOMEM;
3396}
3397
5668abda 3398int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 3399{
3400 struct hnae3_handle *h = priv->ae_handle;
3401 int i;
3402
3403 for (i = 0; i < h->kinfo.num_tqps; i++) {
76ad4f0e
S		 3404 hns3_fini_ring(priv->ring_data[i].ring);
3405 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
3406 }
76ad4f0e
S		 3407 return 0;
3408}
3409
3410/* Set mac addr if it is configured. or leave it to the AE driver */
7fa6be4f 3411static int hns3_init_mac_addr(struct net_device *netdev, bool init)
76ad4f0e
S		 3412{
3413 struct hns3_nic_priv *priv = netdev_priv(netdev);
3414 struct hnae3_handle *h = priv->ae_handle;
3415 u8 mac_addr_temp[ETH_ALEN];
7fa6be4f 3416 int ret = 0;
76ad4f0e 3417
f09555ff 3418 if (h->ae_algo->ops->get_mac_addr && init) {
76ad4f0e
S		 3419 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
3420 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
3421 }
3422
3423 /* Check if the MAC address is valid, if not get a random one */
3424 if (!is_valid_ether_addr(netdev->dev_addr)) {
3425 eth_hw_addr_random(netdev);
3426 dev_warn(priv->dev, "using random MAC address %pM\n",
3427 netdev->dev_addr);
76ad4f0e 3428 }
139e8792
L		 3429
3430 if (h->ae_algo->ops->set_mac_addr)
7fa6be4f 3431 ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);
139e8792 3432
7fa6be4f 3433 return ret;
76ad4f0e
S		 3434}
3435
6871af29
JS		 3436static int hns3_restore_fd_rules(struct net_device *netdev)
3437{
3438 struct hnae3_handle *h = hns3_get_handle(netdev);
3439 int ret = 0;
3440
3441 if (h->ae_algo->ops->restore_fd_rules)
3442 ret = h->ae_algo->ops->restore_fd_rules(h);
3443
3444 return ret;
3445}
3446
3447static void hns3_del_all_fd_rules(struct net_device *netdev, bool clear_list)
3448{
3449 struct hnae3_handle *h = hns3_get_handle(netdev);
3450
3451 if (h->ae_algo->ops->del_all_fd_entries)
3452 h->ae_algo->ops->del_all_fd_entries(h, clear_list);
3453}
3454
76ad4f0e
S		 3455static void hns3_nic_set_priv_ops(struct net_device *netdev)
3456{
3457 struct hns3_nic_priv *priv = netdev_priv(netdev);
3458
0bbbf15d 3459 priv->ops.fill_desc = hns3_fill_desc;
76ad4f0e 3460 if ((netdev->features & NETIF_F_TSO) ||
0bbbf15d 3461 (netdev->features & NETIF_F_TSO6))
76ad4f0e 3462 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
0bbbf15d 3463 else
76ad4f0e 3464 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
76ad4f0e
S		 3465}
3466
3467static int hns3_client_init(struct hnae3_handle *handle)
3468{
3469 struct pci_dev *pdev = handle->pdev;
0d43bf45 3470 u16 alloc_tqps, max_rss_size;
76ad4f0e
S		 3471 struct hns3_nic_priv *priv;
3472 struct net_device *netdev;
3473 int ret;
3474
0d43bf45
HT		 3475 handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,
3476 &max_rss_size);
3477 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);
76ad4f0e
S		 3478 if (!netdev)
3479 return -ENOMEM;
3480
3481 priv = netdev_priv(netdev);
3482 priv->dev = &pdev->dev;
3483 priv->netdev = netdev;
3484 priv->ae_handle = handle;
f8fa222c 3485 priv->tx_timeout_count = 0;
76ad4f0e
S		 3486
3487 handle->kinfo.netdev = netdev;
3488 handle->priv = (void *)priv;
3489
f09555ff 3490 hns3_init_mac_addr(netdev, true);
76ad4f0e
S		 3491
3492 hns3_set_default_feature(netdev);
3493
3494 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
3495 netdev->priv_flags |= IFF_UNICAST_FLT;
3496 netdev->netdev_ops = &hns3_nic_netdev_ops;
3497 SET_NETDEV_DEV(netdev, &pdev->dev);
3498 hns3_ethtool_set_ops(netdev);
3499 hns3_nic_set_priv_ops(netdev);
3500
3501 /* Carrier off reporting is important to ethtool even BEFORE open */
3502 netif_carrier_off(netdev);
3503
3504 ret = hns3_get_ring_config(priv);
3505 if (ret) {
3506 ret = -ENOMEM;
3507 goto out_get_ring_cfg;
3508 }
3509
dd38c726
YL		 3510 ret = hns3_nic_alloc_vector_data(priv);
3511 if (ret) {
3512 ret = -ENOMEM;
3513 goto out_alloc_vector_data;
3514 }
3515
76ad4f0e
S		 3516 ret = hns3_nic_init_vector_data(priv);
3517 if (ret) {
3518 ret = -ENOMEM;
3519 goto out_init_vector_data;
3520 }
3521
3522 ret = hns3_init_all_ring(priv);
3523 if (ret) {
3524 ret = -ENOMEM;
3525 goto out_init_ring_data;
3526 }
3527
3528 ret = register_netdev(netdev);
3529 if (ret) {
3530 dev_err(priv->dev, "probe register netdev fail!\n");
3531 goto out_reg_netdev_fail;
3532 }
3533
986743db
YL		 3534 hns3_dcbnl_setup(handle);
3535
a8e8b7ff
S		 3536 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
3537 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
3538
814da63c
HT		 3539 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
3540
76ad4f0e
S		 3541 return ret;
3542
3543out_reg_netdev_fail:
3544out_init_ring_data:
3545 (void)hns3_nic_uninit_vector_data(priv);
76ad4f0e 3546out_init_vector_data:
dd38c726
YL		 3547 hns3_nic_dealloc_vector_data(priv);
3548out_alloc_vector_data:
3549 priv->ring_data = NULL;
76ad4f0e
S		 3550out_get_ring_cfg:
3551 priv->ae_handle = NULL;
3552 free_netdev(netdev);
3553 return ret;
3554}
3555
3556static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
3557{
3558 struct net_device *netdev = handle->kinfo.netdev;
3559 struct hns3_nic_priv *priv = netdev_priv(netdev);
3560 int ret;
3561
f05e2109
JS		 3562 hns3_remove_hw_addr(netdev);
3563
76ad4f0e
S		 3564 if (netdev->reg_state != NETREG_UNINITIALIZED)
3565 unregister_netdev(netdev);
3566
814da63c
HT		 3567 if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
3568 netdev_warn(netdev, "already uninitialized\n");
3569 goto out_netdev_free;
3570 }
3571
dc5e6064
JS		 3572 hns3_del_all_fd_rules(netdev, true);
3573
7b763f3f
FL		 3574 hns3_force_clear_all_rx_ring(handle);
3575
76ad4f0e
S		 3576 ret = hns3_nic_uninit_vector_data(priv);
3577 if (ret)
3578 netdev_err(netdev, "uninit vector error\n");
3579
dd38c726
YL		 3580 ret = hns3_nic_dealloc_vector_data(priv);
3581 if (ret)
3582 netdev_err(netdev, "dealloc vector error\n");
3583
76ad4f0e
S		 3584 ret = hns3_uninit_all_ring(priv);
3585 if (ret)
3586 netdev_err(netdev, "uninit ring error\n");
3587
ec777890
YL		 3588 hns3_put_ring_config(priv);
3589
76ad4f0e
S		 3590 priv->ring_data = NULL;
3591
814da63c 3592out_netdev_free:
76ad4f0e
S		 3593 free_netdev(netdev);
3594}
3595
3596static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
3597{
3598 struct net_device *netdev = handle->kinfo.netdev;
3599
3600 if (!netdev)
3601 return;
3602
3603 if (linkup) {
3604 netif_carrier_on(netdev);
3605 netif_tx_wake_all_queues(netdev);
3606 netdev_info(netdev, "link up\n");
3607 } else {
3608 netif_carrier_off(netdev);
3609 netif_tx_stop_all_queues(netdev);
3610 netdev_info(netdev, "link down\n");
3611 }
3612}
3613
9df8f79a
YL		 3614static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
3615{
3616 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3617 struct net_device *ndev = kinfo->netdev;
075cfdd6 3618 bool if_running;
9df8f79a 3619 int ret;
9df8f79a
YL		 3620
3621 if (tc > HNAE3_MAX_TC)
3622 return -EINVAL;
3623
3624 if (!ndev)
3625 return -ENODEV;
3626
075cfdd6
CIK		 3627 if_running = netif_running(ndev);
3628
9df8f79a
YL		 3629 if (if_running) {
3630 (void)hns3_nic_net_stop(ndev);
3631 msleep(100);
3632 }
3633
3634 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
3635 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
3636 if (ret)
3637 goto err_out;
3638
9df8f79a
YL		 3639 ret = hns3_nic_set_real_num_queue(ndev);
3640
3641err_out:
3642 if (if_running)
3643 (void)hns3_nic_net_open(ndev);
3644
3645 return ret;
3646}
3647
7fa6be4f 3648static int hns3_recover_hw_addr(struct net_device *ndev)
bb6b94a8
L		 3649{
3650 struct netdev_hw_addr_list *list;
3651 struct netdev_hw_addr *ha, *tmp;
7fa6be4f 3652 int ret = 0;
bb6b94a8
L		 3653
3654 /* go through and sync uc_addr entries to the device */
3655 list = &ndev->uc;
7fa6be4f
HT		 3656 list_for_each_entry_safe(ha, tmp, &list->list, list) {
3657 ret = hns3_nic_uc_sync(ndev, ha->addr);
3658 if (ret)
3659 return ret;
3660 }
bb6b94a8
L		 3661
3662 /* go through and sync mc_addr entries to the device */
3663 list = &ndev->mc;
7fa6be4f
HT		 3664 list_for_each_entry_safe(ha, tmp, &list->list, list) {
3665 ret = hns3_nic_mc_sync(ndev, ha->addr);
3666 if (ret)
3667 return ret;
3668 }
3669
3670 return ret;
bb6b94a8
L		 3671}
3672
f05e2109
JS		 3673static void hns3_remove_hw_addr(struct net_device *netdev)
3674{
3675 struct netdev_hw_addr_list *list;
3676 struct netdev_hw_addr *ha, *tmp;
3677
3678 hns3_nic_uc_unsync(netdev, netdev->dev_addr);
3679
3680 /* go through and unsync uc_addr entries to the device */
3681 list = &netdev->uc;
3682 list_for_each_entry_safe(ha, tmp, &list->list, list)
3683 hns3_nic_uc_unsync(netdev, ha->addr);
3684
3685 /* go through and unsync mc_addr entries to the device */
3686 list = &netdev->mc;
3687 list_for_each_entry_safe(ha, tmp, &list->list, list)
3688 if (ha->refcount > 1)
3689 hns3_nic_mc_unsync(netdev, ha->addr);
3690}
3691
beebca3a 3692static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)
bb6b94a8 3693{
beebca3a 3694 while (ring->next_to_clean != ring->next_to_use) {
7b763f3f 3695 ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;
beebca3a
YL		 3696 hns3_free_buffer_detach(ring, ring->next_to_clean);
3697 ring_ptr_move_fw(ring, next_to_clean);
3698 }
3699}
3700
7b763f3f
FL		 3701static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)
3702{
3703 struct hns3_desc_cb res_cbs;
3704 int ret;
3705
3706 while (ring->next_to_use != ring->next_to_clean) {
3707 /* When a buffer is not reused, it's memory has been
3708 * freed in hns3_handle_rx_bd or will be freed by
3709 * stack, so we need to replace the buffer here.
3710 */
3711 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3712 ret = hns3_reserve_buffer_map(ring, &res_cbs);
3713 if (ret) {
3714 u64_stats_update_begin(&ring->syncp);
3715 ring->stats.sw_err_cnt++;
3716 u64_stats_update_end(&ring->syncp);
3717 /* if alloc new buffer fail, exit directly
3718 * and reclear in up flow.
3719 */
3720 netdev_warn(ring->tqp->handle->kinfo.netdev,
3721 "reserve buffer map failed, ret = %d\n",
3722 ret);
3723 return ret;
3724 }
3725 hns3_replace_buffer(ring, ring->next_to_use,
3726 &res_cbs);
3727 }
3728 ring_ptr_move_fw(ring, next_to_use);
3729 }
3730
3731 return 0;
3732}
3733
3734static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)
beebca3a 3735{
beebca3a
YL		 3736 while (ring->next_to_use != ring->next_to_clean) {
3737 /* When a buffer is not reused, it's memory has been
3738 * freed in hns3_handle_rx_bd or will be freed by
3739 * stack, so only need to unmap the buffer here.
3740 */
3741 if (!ring->desc_cb[ring->next_to_use].reuse_flag) {
3742 hns3_unmap_buffer(ring,
3743 &ring->desc_cb[ring->next_to_use]);
3744 ring->desc_cb[ring->next_to_use].dma = 0;
3745 }
3746
3747 ring_ptr_move_fw(ring, next_to_use);
3748 }
bb6b94a8
L		 3749}
3750
7b763f3f
FL		 3751static void hns3_force_clear_all_rx_ring(struct hnae3_handle *h)
3752{
3753 struct net_device *ndev = h->kinfo.netdev;
3754 struct hns3_nic_priv *priv = netdev_priv(ndev);
3755 struct hns3_enet_ring *ring;
3756 u32 i;
3757
3758 for (i = 0; i < h->kinfo.num_tqps; i++) {
3759 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3760 hns3_force_clear_rx_ring(ring);
3761 }
3762}
3763
bb6b94a8
L		 3764static void hns3_clear_all_ring(struct hnae3_handle *h)
3765{
3766 struct net_device *ndev = h->kinfo.netdev;
3767 struct hns3_nic_priv *priv = netdev_priv(ndev);
3768 u32 i;
3769
3770 for (i = 0; i < h->kinfo.num_tqps; i++) {
3771 struct netdev_queue *dev_queue;
3772 struct hns3_enet_ring *ring;
3773
3774 ring = priv->ring_data[i].ring;
beebca3a 3775 hns3_clear_tx_ring(ring);
bb6b94a8
L		 3776 dev_queue = netdev_get_tx_queue(ndev,
3777 priv->ring_data[i].queue_index);
3778 netdev_tx_reset_queue(dev_queue);
3779
3780 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
7b763f3f
FL		 3781 /* Continue to clear other rings even if clearing some
3782 * rings failed.
3783 */
beebca3a 3784 hns3_clear_rx_ring(ring);
bb6b94a8
L		 3785 }
3786}
3787
7b763f3f
FL		 3788int hns3_nic_reset_all_ring(struct hnae3_handle *h)
3789{
3790 struct net_device *ndev = h->kinfo.netdev;
3791 struct hns3_nic_priv *priv = netdev_priv(ndev);
3792 struct hns3_enet_ring *rx_ring;
3793 int i, j;
3794 int ret;
3795
3796 for (i = 0; i < h->kinfo.num_tqps; i++) {
7fa6be4f
HT		 3797 ret = h->ae_algo->ops->reset_queue(h, i);
3798 if (ret)
3799 return ret;
3800
7b763f3f
FL		 3801 hns3_init_ring_hw(priv->ring_data[i].ring);
3802
3803 /* We need to clear tx ring here because self test will
3804 * use the ring and will not run down before up
3805 */
3806 hns3_clear_tx_ring(priv->ring_data[i].ring);
3807 priv->ring_data[i].ring->next_to_clean = 0;
3808 priv->ring_data[i].ring->next_to_use = 0;
3809
3810 rx_ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3811 hns3_init_ring_hw(rx_ring);
3812 ret = hns3_clear_rx_ring(rx_ring);
3813 if (ret)
3814 return ret;
3815
3816 /* We can not know the hardware head and tail when this
3817 * function is called in reset flow, so we reuse all desc.
3818 */
3819 for (j = 0; j < rx_ring->desc_num; j++)
3820 hns3_reuse_buffer(rx_ring, j);
3821
3822 rx_ring->next_to_clean = 0;
3823 rx_ring->next_to_use = 0;
3824 }
3825
1c772154
YL		 3826 hns3_init_tx_ring_tc(priv);
3827
7b763f3f
FL		 3828 return 0;
3829}
3830
e4fd7502
HT		 3831static void hns3_store_coal(struct hns3_nic_priv *priv)
3832{
3833 /* ethtool only support setting and querying one coal
3834 * configuation for now, so save the vector 0' coal
3835 * configuation here in order to restore it.
3836 */
3837 memcpy(&priv->tx_coal, &priv->tqp_vector[0].tx_group.coal,
3838 sizeof(struct hns3_enet_coalesce));
3839 memcpy(&priv->rx_coal, &priv->tqp_vector[0].rx_group.coal,
3840 sizeof(struct hns3_enet_coalesce));
3841}
3842
3843static void hns3_restore_coal(struct hns3_nic_priv *priv)
3844{
3845 u16 vector_num = priv->vector_num;
3846 int i;
3847
3848 for (i = 0; i < vector_num; i++) {
3849 memcpy(&priv->tqp_vector[i].tx_group.coal, &priv->tx_coal,
3850 sizeof(struct hns3_enet_coalesce));
3851 memcpy(&priv->tqp_vector[i].rx_group.coal, &priv->rx_coal,
3852 sizeof(struct hns3_enet_coalesce));
3853 }
3854}
3855
bb6b94a8
L		 3856static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3857{
7edff533 3858 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);
bb6b94a8
L		 3859 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3860 struct net_device *ndev = kinfo->netdev;
257e4f29
HT		 3861 struct hns3_nic_priv *priv = netdev_priv(ndev);
3862
3863 if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))
3864 return 0;
bb6b94a8 3865
7edff533
HT		 3866 /* it is cumbersome for hardware to pick-and-choose entries for deletion
3867 * from table space. Hence, for function reset software intervention is
3868 * required to delete the entries
3869 */
3870 if (hns3_dev_ongoing_func_reset(ae_dev)) {
3871 hns3_remove_hw_addr(ndev);
3872 hns3_del_all_fd_rules(ndev, false);
3873 }
3874
bb6b94a8 3875 if (!netif_running(ndev))
6b1385cc 3876 return 0;
bb6b94a8
L		 3877
3878 return hns3_nic_net_stop(ndev);
3879}
3880
3881static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3882{
3883 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
257e4f29 3884 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
bb6b94a8
L		 3885 int ret = 0;
3886
3887 if (netif_running(kinfo->netdev)) {
3888 ret = hns3_nic_net_up(kinfo->netdev);
3889 if (ret) {
3890 netdev_err(kinfo->netdev,
3891 "hns net up fail, ret=%d!\n", ret);
3892 return ret;
3893 }
bb6b94a8
L		 3894 }
3895
257e4f29
HT		 3896 clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);
3897
bb6b94a8
L		 3898 return ret;
3899}
3900
3901static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3902{
3903 struct net_device *netdev = handle->kinfo.netdev;
3904 struct hns3_nic_priv *priv = netdev_priv(netdev);
7325523a 3905 bool vlan_filter_enable;
bb6b94a8
L		 3906 int ret;
3907
7fa6be4f
HT		 3908 ret = hns3_init_mac_addr(netdev, false);
3909 if (ret)
3910 return ret;
3911
3912 ret = hns3_recover_hw_addr(netdev);
3913 if (ret)
3914 return ret;
3915
3916 ret = hns3_update_promisc_mode(netdev, handle->netdev_flags);
3917 if (ret)
3918 return ret;
3919
7325523a
JS		 3920 vlan_filter_enable = netdev->flags & IFF_PROMISC ? false : true;
3921 hns3_enable_vlan_filter(netdev, vlan_filter_enable);
3922
681ec399 3923 /* Hardware table is only clear when pf resets */
7fa6be4f
HT		 3924 if (!(handle->flags & HNAE3_SUPPORT_VF)) {
3925 ret = hns3_restore_vlan(netdev);
e12c2252
HT		 3926 if (ret)
3927 return ret;
7fa6be4f 3928 }
681ec399 3929
7fa6be4f
HT		 3930 ret = hns3_restore_fd_rules(netdev);
3931 if (ret)
3932 return ret;
6871af29 3933
bb6b94a8
L		 3934 /* Carrier off reporting is important to ethtool even BEFORE open */
3935 netif_carrier_off(netdev);
3936
862d969a
HT		 3937 ret = hns3_nic_alloc_vector_data(priv);
3938 if (ret)
3939 return ret;
3940
e4fd7502
HT		 3941 hns3_restore_coal(priv);
3942
bb6b94a8
L		 3943 ret = hns3_nic_init_vector_data(priv);
3944 if (ret)
862d969a 3945 goto err_dealloc_vector;
bb6b94a8
L		 3946
3947 ret = hns3_init_all_ring(priv);
862d969a
HT		 3948 if (ret)
3949 goto err_uninit_vector;
bb6b94a8 3950
814da63c
HT		 3951 set_bit(HNS3_NIC_STATE_INITED, &priv->state);
3952
862d969a
HT		 3953 return ret;
3954
3955err_uninit_vector:
3956 hns3_nic_uninit_vector_data(priv);
3957 priv->ring_data = NULL;
3958err_dealloc_vector:
3959 hns3_nic_dealloc_vector_data(priv);
3960
bb6b94a8
L		 3961 return ret;
3962}
3963
3964static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3965{
3966 struct net_device *netdev = handle->kinfo.netdev;
3967 struct hns3_nic_priv *priv = netdev_priv(netdev);
3968 int ret;
3969
814da63c
HT		 3970 if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) {
3971 netdev_warn(netdev, "already uninitialized\n");
3972 return 0;
3973 }
3974
7b763f3f 3975 hns3_force_clear_all_rx_ring(handle);
bb6b94a8
L		 3976
3977 ret = hns3_nic_uninit_vector_data(priv);
3978 if (ret) {
3979 netdev_err(netdev, "uninit vector error\n");
3980 return ret;
3981 }
3982
e4fd7502
HT		 3983 hns3_store_coal(priv);
3984
862d969a
HT		 3985 ret = hns3_nic_dealloc_vector_data(priv);
3986 if (ret)
3987 netdev_err(netdev, "dealloc vector error\n");
3988
bb6b94a8
L		 3989 ret = hns3_uninit_all_ring(priv);
3990 if (ret)
3991 netdev_err(netdev, "uninit ring error\n");
3992
814da63c
HT		 3993 clear_bit(HNS3_NIC_STATE_INITED, &priv->state);
3994
bb6b94a8
L		 3995 return ret;
3996}
3997
3998static int hns3_reset_notify(struct hnae3_handle *handle,
3999 enum hnae3_reset_notify_type type)
4000{
4001 int ret = 0;
4002
4003 switch (type) {
4004 case HNAE3_UP_CLIENT:
e1586241
SM		 4005 ret = hns3_reset_notify_up_enet(handle);
4006 break;
bb6b94a8
L		 4007 case HNAE3_DOWN_CLIENT:
4008 ret = hns3_reset_notify_down_enet(handle);
4009 break;
4010 case HNAE3_INIT_CLIENT:
4011 ret = hns3_reset_notify_init_enet(handle);
4012 break;
4013 case HNAE3_UNINIT_CLIENT:
4014 ret = hns3_reset_notify_uninit_enet(handle);
4015 break;
4016 default:
4017 break;
4018 }
4019
4020 return ret;
4021}
4022
e4fd7502 4023static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
09f2af64
PL		 4024{
4025 struct hns3_nic_priv *priv = netdev_priv(netdev);
4026 struct hnae3_handle *h = hns3_get_handle(netdev);
4027 int ret;
4028
4029 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
4030 if (ret)
4031 return ret;
4032
4033 ret = hns3_get_ring_config(priv);
4034 if (ret)
4035 return ret;
4036
dd38c726
YL		 4037 ret = hns3_nic_alloc_vector_data(priv);
4038 if (ret)
4039 goto err_alloc_vector;
4040
e4fd7502 4041 hns3_restore_coal(priv);
7a242b23 4042
09f2af64
PL		 4043 ret = hns3_nic_init_vector_data(priv);
4044 if (ret)
4045 goto err_uninit_vector;
4046
4047 ret = hns3_init_all_ring(priv);
4048 if (ret)
4049 goto err_put_ring;
4050
4051 return 0;
4052
4053err_put_ring:
4054 hns3_put_ring_config(priv);
4055err_uninit_vector:
4056 hns3_nic_uninit_vector_data(priv);
dd38c726
YL		 4057err_alloc_vector:
4058 hns3_nic_dealloc_vector_data(priv);
09f2af64
PL		 4059 return ret;
4060}
4061
4062static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
4063{
4064 return (new_tqp_num / num_tc) * num_tc;
4065}
4066
4067int hns3_set_channels(struct net_device *netdev,
4068 struct ethtool_channels *ch)
4069{
4070 struct hns3_nic_priv *priv = netdev_priv(netdev);
4071 struct hnae3_handle *h = hns3_get_handle(netdev);
4072 struct hnae3_knic_private_info *kinfo = &h->kinfo;
4073 bool if_running = netif_running(netdev);
4074 u32 new_tqp_num = ch->combined_count;
4075 u16 org_tqp_num;
4076 int ret;
4077
4078 if (ch->rx_count || ch->tx_count)
4079 return -EINVAL;
4080
678335a1 4081 if (new_tqp_num > hns3_get_max_available_channels(h) ||
09f2af64
PL		 4082 new_tqp_num < kinfo->num_tc) {
4083 dev_err(&netdev->dev,
4084 "Change tqps fail, the tqp range is from %d to %d",
4085 kinfo->num_tc,
678335a1 4086 hns3_get_max_available_channels(h));
09f2af64
PL		 4087 return -EINVAL;
4088 }
4089
4090 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
4091 if (kinfo->num_tqps == new_tqp_num)
4092 return 0;
4093
4094 if (if_running)
20e4bf98 4095 hns3_nic_net_stop(netdev);
09f2af64 4096
09f2af64
PL		 4097 ret = hns3_nic_uninit_vector_data(priv);
4098 if (ret) {
4099 dev_err(&netdev->dev,
4100 "Unbind vector with tqp fail, nothing is changed");
4101 goto open_netdev;
4102 }
4103
e4fd7502 4104 hns3_store_coal(priv);
7a242b23 4105
dd38c726
YL		 4106 hns3_nic_dealloc_vector_data(priv);
4107
09f2af64 4108 hns3_uninit_all_ring(priv);
ec777890 4109 hns3_put_ring_config(priv);
09f2af64
PL		 4110
4111 org_tqp_num = h->kinfo.num_tqps;
e4fd7502 4112 ret = hns3_modify_tqp_num(netdev, new_tqp_num);
09f2af64 4113 if (ret) {
e4fd7502 4114 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
09f2af64
PL		 4115 if (ret) {
4116 /* If revert to old tqp failed, fatal error occurred */
4117 dev_err(&netdev->dev,
4118 "Revert to old tqp num fail, ret=%d", ret);
4119 return ret;
4120 }
4121 dev_info(&netdev->dev,
4122 "Change tqp num fail, Revert to old tqp num");
4123 }
4124
4125open_netdev:
4126 if (if_running)
20e4bf98 4127 hns3_nic_net_open(netdev);
09f2af64
PL		 4128
4129 return ret;
4130}
4131
1db9b1bf 4132static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 4133 .init_instance = hns3_client_init,
4134 .uninit_instance = hns3_client_uninit,
4135 .link_status_change = hns3_link_status_change,
9df8f79a 4136 .setup_tc = hns3_client_setup_tc,
bb6b94a8 4137 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 4138};
4139
4140/* hns3_init_module - Driver registration routine
4141 * hns3_init_module is the first routine called when the driver is
4142 * loaded. All it does is register with the PCI subsystem.
4143 */
4144static int __init hns3_init_module(void)
4145{
4146 int ret;
4147
4148 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
4149 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
4150
4151 client.type = HNAE3_CLIENT_KNIC;
4152 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
4153 hns3_driver_name);
4154
4155 client.ops = &client_ops;
4156
13562d1f
XW		 4157 INIT_LIST_HEAD(&client.node);
4158
76ad4f0e
S		 4159 ret = hnae3_register_client(&client);
4160 if (ret)
4161 return ret;
4162
4163 ret = pci_register_driver(&hns3_driver);
4164 if (ret)
4165 hnae3_unregister_client(&client);
4166
4167 return ret;
4168}
4169module_init(hns3_init_module);
4170
4171/* hns3_exit_module - Driver exit cleanup routine
4172 * hns3_exit_module is called just before the driver is removed
4173 * from memory.
4174 */
4175static void __exit hns3_exit_module(void)
4176{
4177 pci_unregister_driver(&hns3_driver);
4178 hnae3_unregister_client(&client);
4179}
4180module_exit(hns3_exit_module);
4181
4182MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
4183MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
4184MODULE_LICENSE("GPL");
4185MODULE_ALIAS("pci:hns-nic");
3c7624d8 4186MODULE_VERSION(HNS3_MOD_VERSION);
Linux 6.x block layer and io_uring tree(s)