projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: hns3: add vlan offload config command
[linux-block.git] / drivers / net / ethernet / hisilicon / hns3 / hns3_enet.c
CommitLineData
76ad4f0e
S		 1/*
2 * Copyright (c) 2016~2017 Hisilicon Limited.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 */
9
10#include <linux/dma-mapping.h>
11#include <linux/etherdevice.h>
12#include <linux/interrupt.h>
13#include <linux/if_vlan.h>
14#include <linux/ip.h>
15#include <linux/ipv6.h>
16#include <linux/module.h>
17#include <linux/pci.h>
18#include <linux/skbuff.h>
19#include <linux/sctp.h>
20#include <linux/vermagic.h>
21#include <net/gre.h>
30d240df 22#include <net/pkt_cls.h>
76ad4f0e
S		 23#include <net/vxlan.h>
24
25#include "hnae3.h"
26#include "hns3_enet.h"
27
1db9b1bf 28static const char hns3_driver_name[] = "hns3";
76ad4f0e
S		 29const char hns3_driver_version[] = VERMAGIC_STRING;
30static const char hns3_driver_string[] =
31 "Hisilicon Ethernet Network Driver for Hip08 Family";
32static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";
33static struct hnae3_client client;
34
35/* hns3_pci_tbl - PCI Device ID Table
36 *
37 * Last entry must be all 0s
38 *
39 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
40 * Class, Class Mask, private data (not used) }
41 */
42static const struct pci_device_id hns3_pci_tbl[] = {
43 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
44 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
e92a0843 45 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
2daf4a65 46 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 47 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
2daf4a65 48 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 49 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
2daf4a65 50 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 51 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
2daf4a65 52 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
e92a0843 53 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
2daf4a65 54 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
424eb834
SM		 55 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_VF), 0},
56 {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_DCB_PFC_VF), 0},
76ad4f0e
S		 57 /* required last entry */
58 {0, }
59};
60MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
61
62static irqreturn_t hns3_irq_handle(int irq, void *dev)
63{
64 struct hns3_enet_tqp_vector *tqp_vector = dev;
65
66 napi_schedule(&tqp_vector->napi);
67
68 return IRQ_HANDLED;
69}
70
71static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)
72{
73 struct hns3_enet_tqp_vector *tqp_vectors;
74 unsigned int i;
75
76 for (i = 0; i < priv->vector_num; i++) {
77 tqp_vectors = &priv->tqp_vector[i];
78
79 if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)
80 continue;
81
82 /* release the irq resource */
83 free_irq(tqp_vectors->vector_irq, tqp_vectors);
84 tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;
85 }
86}
87
88static int hns3_nic_init_irq(struct hns3_nic_priv *priv)
89{
90 struct hns3_enet_tqp_vector *tqp_vectors;
91 int txrx_int_idx = 0;
92 int rx_int_idx = 0;
93 int tx_int_idx = 0;
94 unsigned int i;
95 int ret;
96
97 for (i = 0; i < priv->vector_num; i++) {
98 tqp_vectors = &priv->tqp_vector[i];
99
100 if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)
101 continue;
102
103 if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {
104 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
105 "%s-%s-%d", priv->netdev->name, "TxRx",
106 txrx_int_idx++);
107 txrx_int_idx++;
108 } else if (tqp_vectors->rx_group.ring) {
109 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
110 "%s-%s-%d", priv->netdev->name, "Rx",
111 rx_int_idx++);
112 } else if (tqp_vectors->tx_group.ring) {
113 snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN - 1,
114 "%s-%s-%d", priv->netdev->name, "Tx",
115 tx_int_idx++);
116 } else {
117 /* Skip this unused q_vector */
118 continue;
119 }
120
121 tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';
122
123 ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,
124 tqp_vectors->name,
125 tqp_vectors);
126 if (ret) {
127 netdev_err(priv->netdev, "request irq(%d) fail\n",
128 tqp_vectors->vector_irq);
129 return ret;
130 }
131
132 tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;
133 }
134
135 return 0;
136}
137
138static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,
139 u32 mask_en)
140{
141 writel(mask_en, tqp_vector->mask_addr);
142}
143
144static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)
145{
146 napi_enable(&tqp_vector->napi);
147
148 /* enable vector */
149 hns3_mask_vector_irq(tqp_vector, 1);
150}
151
152static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)
153{
154 /* disable vector */
155 hns3_mask_vector_irq(tqp_vector, 0);
156
157 disable_irq(tqp_vector->vector_irq);
158 napi_disable(&tqp_vector->napi);
159}
160
161static void hns3_set_vector_coalesc_gl(struct hns3_enet_tqp_vector *tqp_vector,
162 u32 gl_value)
163{
164 /* this defines the configuration for GL (Interrupt Gap Limiter)
165 * GL defines inter interrupt gap.
166 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
167 */
168 writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);
169 writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);
170 writel(gl_value, tqp_vector->mask_addr + HNS3_VECTOR_GL2_OFFSET);
171}
172
173static void hns3_set_vector_coalesc_rl(struct hns3_enet_tqp_vector *tqp_vector,
174 u32 rl_value)
175{
176 /* this defines the configuration for RL (Interrupt Rate Limiter).
177 * Rl defines rate of interrupts i.e. number of interrupts-per-second
178 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing
179 */
180 writel(rl_value, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);
181}
182
183static void hns3_vector_gl_rl_init(struct hns3_enet_tqp_vector *tqp_vector)
184{
185 /* initialize the configuration for interrupt coalescing.
186 * 1. GL (Interrupt Gap Limiter)
187 * 2. RL (Interrupt Rate Limiter)
188 */
189
190 /* Default :enable interrupt coalesce */
191 tqp_vector->rx_group.int_gl = HNS3_INT_GL_50K;
192 tqp_vector->tx_group.int_gl = HNS3_INT_GL_50K;
193 hns3_set_vector_coalesc_gl(tqp_vector, HNS3_INT_GL_50K);
194 /* for now we are disabling Interrupt RL - we
195 * will re-enable later
196 */
197 hns3_set_vector_coalesc_rl(tqp_vector, 0);
198 tqp_vector->rx_group.flow_level = HNS3_FLOW_LOW;
199 tqp_vector->tx_group.flow_level = HNS3_FLOW_LOW;
200}
201
9df8f79a
YL		 202static int hns3_nic_set_real_num_queue(struct net_device *netdev)
203{
9780cb97 204 struct hnae3_handle *h = hns3_get_handle(netdev);
9df8f79a
YL		 205 struct hnae3_knic_private_info *kinfo = &h->kinfo;
206 unsigned int queue_size = kinfo->rss_size * kinfo->num_tc;
207 int ret;
208
209 ret = netif_set_real_num_tx_queues(netdev, queue_size);
210 if (ret) {
211 netdev_err(netdev,
212 "netif_set_real_num_tx_queues fail, ret=%d!\n",
213 ret);
214 return ret;
215 }
216
217 ret = netif_set_real_num_rx_queues(netdev, queue_size);
218 if (ret) {
219 netdev_err(netdev,
220 "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
221 return ret;
222 }
223
224 return 0;
225}
226
76ad4f0e
S		 227static int hns3_nic_net_up(struct net_device *netdev)
228{
229 struct hns3_nic_priv *priv = netdev_priv(netdev);
230 struct hnae3_handle *h = priv->ae_handle;
231 int i, j;
232 int ret;
233
234 /* get irq resource for all vectors */
235 ret = hns3_nic_init_irq(priv);
236 if (ret) {
237 netdev_err(netdev, "hns init irq failed! ret=%d\n", ret);
238 return ret;
239 }
240
241 /* enable the vectors */
242 for (i = 0; i < priv->vector_num; i++)
243 hns3_vector_enable(&priv->tqp_vector[i]);
244
245 /* start the ae_dev */
246 ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;
247 if (ret)
248 goto out_start_err;
249
250 return 0;
251
252out_start_err:
253 for (j = i - 1; j >= 0; j--)
254 hns3_vector_disable(&priv->tqp_vector[j]);
255
256 hns3_nic_uninit_irq(priv);
257
258 return ret;
259}
260
261static int hns3_nic_net_open(struct net_device *netdev)
262{
f8fa222c 263 struct hns3_nic_priv *priv = netdev_priv(netdev);
76ad4f0e
S		 264 int ret;
265
266 netif_carrier_off(netdev);
267
9df8f79a
YL		 268 ret = hns3_nic_set_real_num_queue(netdev);
269 if (ret)
76ad4f0e 270 return ret;
76ad4f0e
S		 271
272 ret = hns3_nic_net_up(netdev);
273 if (ret) {
274 netdev_err(netdev,
275 "hns net up fail, ret=%d!\n", ret);
276 return ret;
277 }
278
f8fa222c 279 priv->last_reset_time = jiffies;
76ad4f0e
S		 280 return 0;
281}
282
283static void hns3_nic_net_down(struct net_device *netdev)
284{
285 struct hns3_nic_priv *priv = netdev_priv(netdev);
286 const struct hnae3_ae_ops *ops;
287 int i;
288
289 /* stop ae_dev */
290 ops = priv->ae_handle->ae_algo->ops;
291 if (ops->stop)
292 ops->stop(priv->ae_handle);
293
294 /* disable vectors */
295 for (i = 0; i < priv->vector_num; i++)
296 hns3_vector_disable(&priv->tqp_vector[i]);
297
298 /* free irq resources */
299 hns3_nic_uninit_irq(priv);
300}
301
302static int hns3_nic_net_stop(struct net_device *netdev)
303{
304 netif_tx_stop_all_queues(netdev);
305 netif_carrier_off(netdev);
306
307 hns3_nic_net_down(netdev);
308
309 return 0;
310}
311
76ad4f0e
S		 312static int hns3_nic_uc_sync(struct net_device *netdev,
313 const unsigned char *addr)
314{
9780cb97 315 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 316
317 if (h->ae_algo->ops->add_uc_addr)
318 return h->ae_algo->ops->add_uc_addr(h, addr);
319
320 return 0;
321}
322
323static int hns3_nic_uc_unsync(struct net_device *netdev,
324 const unsigned char *addr)
325{
9780cb97 326 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 327
328 if (h->ae_algo->ops->rm_uc_addr)
329 return h->ae_algo->ops->rm_uc_addr(h, addr);
330
331 return 0;
332}
333
334static int hns3_nic_mc_sync(struct net_device *netdev,
335 const unsigned char *addr)
336{
9780cb97 337 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 338
720a8478 339 if (h->ae_algo->ops->add_mc_addr)
76ad4f0e
S		 340 return h->ae_algo->ops->add_mc_addr(h, addr);
341
342 return 0;
343}
344
345static int hns3_nic_mc_unsync(struct net_device *netdev,
346 const unsigned char *addr)
347{
9780cb97 348 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 349
720a8478 350 if (h->ae_algo->ops->rm_mc_addr)
76ad4f0e
S		 351 return h->ae_algo->ops->rm_mc_addr(h, addr);
352
353 return 0;
354}
355
1db9b1bf 356static void hns3_nic_set_rx_mode(struct net_device *netdev)
76ad4f0e 357{
9780cb97 358 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 359
360 if (h->ae_algo->ops->set_promisc_mode) {
361 if (netdev->flags & IFF_PROMISC)
362 h->ae_algo->ops->set_promisc_mode(h, 1);
363 else
364 h->ae_algo->ops->set_promisc_mode(h, 0);
365 }
366 if (__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync))
367 netdev_err(netdev, "sync uc address fail\n");
368 if (netdev->flags & IFF_MULTICAST)
369 if (__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync))
370 netdev_err(netdev, "sync mc address fail\n");
371}
372
373static int hns3_set_tso(struct sk_buff *skb, u32 *paylen,
374 u16 *mss, u32 *type_cs_vlan_tso)
375{
376 u32 l4_offset, hdr_len;
377 union l3_hdr_info l3;
378 union l4_hdr_info l4;
379 u32 l4_paylen;
380 int ret;
381
382 if (!skb_is_gso(skb))
383 return 0;
384
385 ret = skb_cow_head(skb, 0);
386 if (ret)
387 return ret;
388
389 l3.hdr = skb_network_header(skb);
390 l4.hdr = skb_transport_header(skb);
391
392 /* Software should clear the IPv4's checksum field when tso is
393 * needed.
394 */
395 if (l3.v4->version == 4)
396 l3.v4->check = 0;
397
398 /* tunnel packet.*/
399 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
400 SKB_GSO_GRE_CSUM |
401 SKB_GSO_UDP_TUNNEL |
402 SKB_GSO_UDP_TUNNEL_CSUM)) {
403 if ((!(skb_shinfo(skb)->gso_type &
404 SKB_GSO_PARTIAL)) &&
405 (skb_shinfo(skb)->gso_type &
406 SKB_GSO_UDP_TUNNEL_CSUM)) {
407 /* Software should clear the udp's checksum
408 * field when tso is needed.
409 */
410 l4.udp->check = 0;
411 }
412 /* reset l3&l4 pointers from outer to inner headers */
413 l3.hdr = skb_inner_network_header(skb);
414 l4.hdr = skb_inner_transport_header(skb);
415
416 /* Software should clear the IPv4's checksum field when
417 * tso is needed.
418 */
419 if (l3.v4->version == 4)
420 l3.v4->check = 0;
421 }
422
423 /* normal or tunnel packet*/
424 l4_offset = l4.hdr - skb->data;
425 hdr_len = (l4.tcp->doff * 4) + l4_offset;
426
427 /* remove payload length from inner pseudo checksum when tso*/
428 l4_paylen = skb->len - l4_offset;
429 csum_replace_by_diff(&l4.tcp->check,
430 (__force __wsum)htonl(l4_paylen));
431
432 /* find the txbd field values */
433 *paylen = skb->len - hdr_len;
434 hnae_set_bit(*type_cs_vlan_tso,
435 HNS3_TXD_TSO_B, 1);
436
437 /* get MSS for TSO */
438 *mss = skb_shinfo(skb)->gso_size;
439
440 return 0;
441}
442
1898d4e4
S		 443static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,
444 u8 *il4_proto)
76ad4f0e
S		 445{
446 union {
447 struct iphdr *v4;
448 struct ipv6hdr *v6;
449 unsigned char *hdr;
450 } l3;
451 unsigned char *l4_hdr;
452 unsigned char *exthdr;
453 u8 l4_proto_tmp;
454 __be16 frag_off;
455
456 /* find outer header point */
457 l3.hdr = skb_network_header(skb);
458 l4_hdr = skb_inner_transport_header(skb);
459
460 if (skb->protocol == htons(ETH_P_IPV6)) {
461 exthdr = l3.hdr + sizeof(*l3.v6);
462 l4_proto_tmp = l3.v6->nexthdr;
463 if (l4_hdr != exthdr)
464 ipv6_skip_exthdr(skb, exthdr - skb->data,
465 &l4_proto_tmp, &frag_off);
466 } else if (skb->protocol == htons(ETH_P_IP)) {
467 l4_proto_tmp = l3.v4->protocol;
1898d4e4
S		 468 } else {
469 return -EINVAL;
76ad4f0e
S		 470 }
471
472 *ol4_proto = l4_proto_tmp;
473
474 /* tunnel packet */
475 if (!skb->encapsulation) {
476 *il4_proto = 0;
1898d4e4 477 return 0;
76ad4f0e
S		 478 }
479
480 /* find inner header point */
481 l3.hdr = skb_inner_network_header(skb);
482 l4_hdr = skb_inner_transport_header(skb);
483
484 if (l3.v6->version == 6) {
485 exthdr = l3.hdr + sizeof(*l3.v6);
486 l4_proto_tmp = l3.v6->nexthdr;
487 if (l4_hdr != exthdr)
488 ipv6_skip_exthdr(skb, exthdr - skb->data,
489 &l4_proto_tmp, &frag_off);
490 } else if (l3.v4->version == 4) {
491 l4_proto_tmp = l3.v4->protocol;
492 }
493
494 *il4_proto = l4_proto_tmp;
1898d4e4
S		 495
496 return 0;
76ad4f0e
S		 497}
498
499static void hns3_set_l2l3l4_len(struct sk_buff *skb, u8 ol4_proto,
500 u8 il4_proto, u32 *type_cs_vlan_tso,
501 u32 *ol_type_vlan_len_msec)
502{
503 union {
504 struct iphdr *v4;
505 struct ipv6hdr *v6;
506 unsigned char *hdr;
507 } l3;
508 union {
509 struct tcphdr *tcp;
510 struct udphdr *udp;
511 struct gre_base_hdr *gre;
512 unsigned char *hdr;
513 } l4;
514 unsigned char *l2_hdr;
515 u8 l4_proto = ol4_proto;
516 u32 ol2_len;
517 u32 ol3_len;
518 u32 ol4_len;
519 u32 l2_len;
520 u32 l3_len;
521
522 l3.hdr = skb_network_header(skb);
523 l4.hdr = skb_transport_header(skb);
524
525 /* compute L2 header size for normal packet, defined in 2 Bytes */
526 l2_len = l3.hdr - skb->data;
527 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
528 HNS3_TXD_L2LEN_S, l2_len >> 1);
529
530 /* tunnel packet*/
531 if (skb->encapsulation) {
532 /* compute OL2 header size, defined in 2 Bytes */
533 ol2_len = l2_len;
534 hnae_set_field(*ol_type_vlan_len_msec,
535 HNS3_TXD_L2LEN_M,
536 HNS3_TXD_L2LEN_S, ol2_len >> 1);
537
538 /* compute OL3 header size, defined in 4 Bytes */
539 ol3_len = l4.hdr - l3.hdr;
540 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_M,
541 HNS3_TXD_L3LEN_S, ol3_len >> 2);
542
543 /* MAC in UDP, MAC in GRE (0x6558)*/
544 if ((ol4_proto == IPPROTO_UDP) || (ol4_proto == IPPROTO_GRE)) {
545 /* switch MAC header ptr from outer to inner header.*/
546 l2_hdr = skb_inner_mac_header(skb);
547
548 /* compute OL4 header size, defined in 4 Bytes. */
549 ol4_len = l2_hdr - l4.hdr;
550 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_M,
551 HNS3_TXD_L4LEN_S, ol4_len >> 2);
552
553 /* switch IP header ptr from outer to inner header */
554 l3.hdr = skb_inner_network_header(skb);
555
556 /* compute inner l2 header size, defined in 2 Bytes. */
557 l2_len = l3.hdr - l2_hdr;
558 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_M,
559 HNS3_TXD_L2LEN_S, l2_len >> 1);
560 } else {
561 /* skb packet types not supported by hardware,
562 * txbd len fild doesn't be filled.
563 */
564 return;
565 }
566
567 /* switch L4 header pointer from outer to inner */
568 l4.hdr = skb_inner_transport_header(skb);
569
570 l4_proto = il4_proto;
571 }
572
573 /* compute inner(/normal) L3 header size, defined in 4 Bytes */
574 l3_len = l4.hdr - l3.hdr;
575 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_M,
576 HNS3_TXD_L3LEN_S, l3_len >> 2);
577
578 /* compute inner(/normal) L4 header size, defined in 4 Bytes */
579 switch (l4_proto) {
580 case IPPROTO_TCP:
581 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
582 HNS3_TXD_L4LEN_S, l4.tcp->doff);
583 break;
584 case IPPROTO_SCTP:
585 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
586 HNS3_TXD_L4LEN_S, (sizeof(struct sctphdr) >> 2));
587 break;
588 case IPPROTO_UDP:
589 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_M,
590 HNS3_TXD_L4LEN_S, (sizeof(struct udphdr) >> 2));
591 break;
592 default:
593 /* skb packet types not supported by hardware,
594 * txbd len fild doesn't be filled.
595 */
596 return;
597 }
598}
599
600static int hns3_set_l3l4_type_csum(struct sk_buff *skb, u8 ol4_proto,
601 u8 il4_proto, u32 *type_cs_vlan_tso,
602 u32 *ol_type_vlan_len_msec)
603{
604 union {
605 struct iphdr *v4;
606 struct ipv6hdr *v6;
607 unsigned char *hdr;
608 } l3;
609 u32 l4_proto = ol4_proto;
610
611 l3.hdr = skb_network_header(skb);
612
613 /* define OL3 type and tunnel type(OL4).*/
614 if (skb->encapsulation) {
615 /* define outer network header type.*/
616 if (skb->protocol == htons(ETH_P_IP)) {
617 if (skb_is_gso(skb))
618 hnae_set_field(*ol_type_vlan_len_msec,
619 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
620 HNS3_OL3T_IPV4_CSUM);
621 else
622 hnae_set_field(*ol_type_vlan_len_msec,
623 HNS3_TXD_OL3T_M, HNS3_TXD_OL3T_S,
624 HNS3_OL3T_IPV4_NO_CSUM);
625
626 } else if (skb->protocol == htons(ETH_P_IPV6)) {
627 hnae_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_M,
628 HNS3_TXD_OL3T_S, HNS3_OL3T_IPV6);
629 }
630
631 /* define tunnel type(OL4).*/
632 switch (l4_proto) {
633 case IPPROTO_UDP:
634 hnae_set_field(*ol_type_vlan_len_msec,
635 HNS3_TXD_TUNTYPE_M,
636 HNS3_TXD_TUNTYPE_S,
637 HNS3_TUN_MAC_IN_UDP);
638 break;
639 case IPPROTO_GRE:
640 hnae_set_field(*ol_type_vlan_len_msec,
641 HNS3_TXD_TUNTYPE_M,
642 HNS3_TXD_TUNTYPE_S,
643 HNS3_TUN_NVGRE);
644 break;
645 default:
646 /* drop the skb tunnel packet if hardware don't support,
647 * because hardware can't calculate csum when TSO.
648 */
649 if (skb_is_gso(skb))
650 return -EDOM;
651
652 /* the stack computes the IP header already,
653 * driver calculate l4 checksum when not TSO.
654 */
655 skb_checksum_help(skb);
656 return 0;
657 }
658
659 l3.hdr = skb_inner_network_header(skb);
660 l4_proto = il4_proto;
661 }
662
663 if (l3.v4->version == 4) {
664 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
665 HNS3_TXD_L3T_S, HNS3_L3T_IPV4);
666
667 /* the stack computes the IP header already, the only time we
668 * need the hardware to recompute it is in the case of TSO.
669 */
670 if (skb_is_gso(skb))
671 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);
672
673 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
674 } else if (l3.v6->version == 6) {
675 hnae_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_M,
676 HNS3_TXD_L3T_S, HNS3_L3T_IPV6);
677 hnae_set_bit(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);
678 }
679
680 switch (l4_proto) {
681 case IPPROTO_TCP:
682 hnae_set_field(*type_cs_vlan_tso,
683 HNS3_TXD_L4T_M,
684 HNS3_TXD_L4T_S,
685 HNS3_L4T_TCP);
686 break;
687 case IPPROTO_UDP:
688 hnae_set_field(*type_cs_vlan_tso,
689 HNS3_TXD_L4T_M,
690 HNS3_TXD_L4T_S,
691 HNS3_L4T_UDP);
692 break;
693 case IPPROTO_SCTP:
694 hnae_set_field(*type_cs_vlan_tso,
695 HNS3_TXD_L4T_M,
696 HNS3_TXD_L4T_S,
697 HNS3_L4T_SCTP);
698 break;
699 default:
700 /* drop the skb tunnel packet if hardware don't support,
701 * because hardware can't calculate csum when TSO.
702 */
703 if (skb_is_gso(skb))
704 return -EDOM;
705
706 /* the stack computes the IP header already,
707 * driver calculate l4 checksum when not TSO.
708 */
709 skb_checksum_help(skb);
710 return 0;
711 }
712
713 return 0;
714}
715
716static void hns3_set_txbd_baseinfo(u16 *bdtp_fe_sc_vld_ra_ri, int frag_end)
717{
718 /* Config bd buffer end */
719 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_BDTYPE_M,
720 HNS3_TXD_BDTYPE_M, 0);
721 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_FE_B, !!frag_end);
722 hnae_set_bit(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_VLD_B, 1);
7036d26f 723 hnae_set_field(*bdtp_fe_sc_vld_ra_ri, HNS3_TXD_SC_M, HNS3_TXD_SC_S, 0);
76ad4f0e
S		 724}
725
726static int hns3_fill_desc(struct hns3_enet_ring *ring, void *priv,
727 int size, dma_addr_t dma, int frag_end,
728 enum hns_desc_type type)
729{
730 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
731 struct hns3_desc *desc = &ring->desc[ring->next_to_use];
732 u32 ol_type_vlan_len_msec = 0;
733 u16 bdtp_fe_sc_vld_ra_ri = 0;
734 u32 type_cs_vlan_tso = 0;
735 struct sk_buff *skb;
736 u32 paylen = 0;
737 u16 mss = 0;
738 __be16 protocol;
739 u8 ol4_proto;
740 u8 il4_proto;
741 int ret;
742
743 /* The txbd's baseinfo of DESC_TYPE_PAGE & DESC_TYPE_SKB */
744 desc_cb->priv = priv;
745 desc_cb->length = size;
746 desc_cb->dma = dma;
747 desc_cb->type = type;
748
749 /* now, fill the descriptor */
750 desc->addr = cpu_to_le64(dma);
751 desc->tx.send_size = cpu_to_le16((u16)size);
752 hns3_set_txbd_baseinfo(&bdtp_fe_sc_vld_ra_ri, frag_end);
753 desc->tx.bdtp_fe_sc_vld_ra_ri = cpu_to_le16(bdtp_fe_sc_vld_ra_ri);
754
755 if (type == DESC_TYPE_SKB) {
756 skb = (struct sk_buff *)priv;
a90bb9a5 757 paylen = skb->len;
76ad4f0e
S		 758
759 if (skb->ip_summed == CHECKSUM_PARTIAL) {
760 skb_reset_mac_len(skb);
761 protocol = skb->protocol;
762
763 /* vlan packet*/
764 if (protocol == htons(ETH_P_8021Q)) {
765 protocol = vlan_get_protocol(skb);
766 skb->protocol = protocol;
767 }
1898d4e4
S		 768 ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);
769 if (ret)
770 return ret;
76ad4f0e
S		 771 hns3_set_l2l3l4_len(skb, ol4_proto, il4_proto,
772 &type_cs_vlan_tso,
773 &ol_type_vlan_len_msec);
774 ret = hns3_set_l3l4_type_csum(skb, ol4_proto, il4_proto,
775 &type_cs_vlan_tso,
776 &ol_type_vlan_len_msec);
777 if (ret)
778 return ret;
779
780 ret = hns3_set_tso(skb, &paylen, &mss,
781 &type_cs_vlan_tso);
782 if (ret)
783 return ret;
784 }
785
786 /* Set txbd */
787 desc->tx.ol_type_vlan_len_msec =
788 cpu_to_le32(ol_type_vlan_len_msec);
789 desc->tx.type_cs_vlan_tso_len =
790 cpu_to_le32(type_cs_vlan_tso);
a90bb9a5 791 desc->tx.paylen = cpu_to_le32(paylen);
76ad4f0e
S		 792 desc->tx.mss = cpu_to_le16(mss);
793 }
794
795 /* move ring pointer to next.*/
796 ring_ptr_move_fw(ring, next_to_use);
797
798 return 0;
799}
800
801static int hns3_fill_desc_tso(struct hns3_enet_ring *ring, void *priv,
802 int size, dma_addr_t dma, int frag_end,
803 enum hns_desc_type type)
804{
805 unsigned int frag_buf_num;
806 unsigned int k;
807 int sizeoflast;
808 int ret;
809
810 frag_buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
811 sizeoflast = size % HNS3_MAX_BD_SIZE;
812 sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;
813
814 /* When the frag size is bigger than hardware, split this frag */
815 for (k = 0; k < frag_buf_num; k++) {
816 ret = hns3_fill_desc(ring, priv,
817 (k == frag_buf_num - 1) ?
818 sizeoflast : HNS3_MAX_BD_SIZE,
819 dma + HNS3_MAX_BD_SIZE * k,
820 frag_end && (k == frag_buf_num - 1) ? 1 : 0,
821 (type == DESC_TYPE_SKB && !k) ?
822 DESC_TYPE_SKB : DESC_TYPE_PAGE);
823 if (ret)
824 return ret;
825 }
826
827 return 0;
828}
829
830static int hns3_nic_maybe_stop_tso(struct sk_buff **out_skb, int *bnum,
831 struct hns3_enet_ring *ring)
832{
833 struct sk_buff *skb = *out_skb;
834 struct skb_frag_struct *frag;
835 int bdnum_for_frag;
836 int frag_num;
837 int buf_num;
838 int size;
839 int i;
840
841 size = skb_headlen(skb);
842 buf_num = (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
843
844 frag_num = skb_shinfo(skb)->nr_frags;
845 for (i = 0; i < frag_num; i++) {
846 frag = &skb_shinfo(skb)->frags[i];
847 size = skb_frag_size(frag);
848 bdnum_for_frag =
849 (size + HNS3_MAX_BD_SIZE - 1) / HNS3_MAX_BD_SIZE;
850 if (bdnum_for_frag > HNS3_MAX_BD_PER_FRAG)
851 return -ENOMEM;
852
853 buf_num += bdnum_for_frag;
854 }
855
856 if (buf_num > ring_space(ring))
857 return -EBUSY;
858
859 *bnum = buf_num;
860 return 0;
861}
862
863static int hns3_nic_maybe_stop_tx(struct sk_buff **out_skb, int *bnum,
864 struct hns3_enet_ring *ring)
865{
866 struct sk_buff *skb = *out_skb;
867 int buf_num;
868
869 /* No. of segments (plus a header) */
870 buf_num = skb_shinfo(skb)->nr_frags + 1;
871
872 if (buf_num > ring_space(ring))
873 return -EBUSY;
874
875 *bnum = buf_num;
876
877 return 0;
878}
879
880static void hns_nic_dma_unmap(struct hns3_enet_ring *ring, int next_to_use_orig)
881{
882 struct device *dev = ring_to_dev(ring);
883 unsigned int i;
884
885 for (i = 0; i < ring->desc_num; i++) {
886 /* check if this is where we started */
887 if (ring->next_to_use == next_to_use_orig)
888 break;
889
890 /* unmap the descriptor dma address */
891 if (ring->desc_cb[ring->next_to_use].type == DESC_TYPE_SKB)
892 dma_unmap_single(dev,
893 ring->desc_cb[ring->next_to_use].dma,
894 ring->desc_cb[ring->next_to_use].length,
895 DMA_TO_DEVICE);
896 else
897 dma_unmap_page(dev,
898 ring->desc_cb[ring->next_to_use].dma,
899 ring->desc_cb[ring->next_to_use].length,
900 DMA_TO_DEVICE);
901
902 /* rollback one */
903 ring_ptr_move_bw(ring, next_to_use);
904 }
905}
906
d43e5aca 907netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)
76ad4f0e
S		 908{
909 struct hns3_nic_priv *priv = netdev_priv(netdev);
910 struct hns3_nic_ring_data *ring_data =
911 &tx_ring_data(priv, skb->queue_mapping);
912 struct hns3_enet_ring *ring = ring_data->ring;
913 struct device *dev = priv->dev;
914 struct netdev_queue *dev_queue;
915 struct skb_frag_struct *frag;
916 int next_to_use_head;
917 int next_to_use_frag;
918 dma_addr_t dma;
919 int buf_num;
920 int seg_num;
921 int size;
922 int ret;
923 int i;
924
925 /* Prefetch the data used later */
926 prefetch(skb->data);
927
928 switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
929 case -EBUSY:
930 u64_stats_update_begin(&ring->syncp);
931 ring->stats.tx_busy++;
932 u64_stats_update_end(&ring->syncp);
933
934 goto out_net_tx_busy;
935 case -ENOMEM:
936 u64_stats_update_begin(&ring->syncp);
937 ring->stats.sw_err_cnt++;
938 u64_stats_update_end(&ring->syncp);
939 netdev_err(netdev, "no memory to xmit!\n");
940
941 goto out_err_tx_ok;
942 default:
943 break;
944 }
945
946 /* No. of segments (plus a header) */
947 seg_num = skb_shinfo(skb)->nr_frags + 1;
948 /* Fill the first part */
949 size = skb_headlen(skb);
950
951 next_to_use_head = ring->next_to_use;
952
953 dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
954 if (dma_mapping_error(dev, dma)) {
955 netdev_err(netdev, "TX head DMA map failed\n");
956 ring->stats.sw_err_cnt++;
957 goto out_err_tx_ok;
958 }
959
960 ret = priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
961 DESC_TYPE_SKB);
962 if (ret)
963 goto head_dma_map_err;
964
965 next_to_use_frag = ring->next_to_use;
966 /* Fill the fragments */
967 for (i = 1; i < seg_num; i++) {
968 frag = &skb_shinfo(skb)->frags[i - 1];
969 size = skb_frag_size(frag);
970 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
971 if (dma_mapping_error(dev, dma)) {
972 netdev_err(netdev, "TX frag(%d) DMA map failed\n", i);
973 ring->stats.sw_err_cnt++;
974 goto frag_dma_map_err;
975 }
976 ret = priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
977 seg_num - 1 == i ? 1 : 0,
978 DESC_TYPE_PAGE);
979
980 if (ret)
981 goto frag_dma_map_err;
982 }
983
984 /* Complete translate all packets */
985 dev_queue = netdev_get_tx_queue(netdev, ring_data->queue_index);
986 netdev_tx_sent_queue(dev_queue, skb->len);
987
988 wmb(); /* Commit all data before submit */
989
990 hnae_queue_xmit(ring->tqp, buf_num);
991
992 return NETDEV_TX_OK;
993
994frag_dma_map_err:
995 hns_nic_dma_unmap(ring, next_to_use_frag);
996
997head_dma_map_err:
998 hns_nic_dma_unmap(ring, next_to_use_head);
999
1000out_err_tx_ok:
1001 dev_kfree_skb_any(skb);
1002 return NETDEV_TX_OK;
1003
1004out_net_tx_busy:
1005 netif_stop_subqueue(netdev, ring_data->queue_index);
1006 smp_mb(); /* Commit all data before submit */
1007
1008 return NETDEV_TX_BUSY;
1009}
1010
1011static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)
1012{
9780cb97 1013 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1014 struct sockaddr *mac_addr = p;
1015 int ret;
1016
1017 if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1018 return -EADDRNOTAVAIL;
1019
1020 ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data);
1021 if (ret) {
1022 netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);
1023 return ret;
1024 }
1025
1026 ether_addr_copy(netdev->dev_addr, mac_addr->sa_data);
1027
1028 return 0;
1029}
1030
1031static int hns3_nic_set_features(struct net_device *netdev,
1032 netdev_features_t features)
1033{
1034 struct hns3_nic_priv *priv = netdev_priv(netdev);
1035
1036 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1037 priv->ops.fill_desc = hns3_fill_desc_tso;
1038 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
1039 } else {
1040 priv->ops.fill_desc = hns3_fill_desc;
1041 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
1042 }
1043
1044 netdev->features = features;
1045 return 0;
1046}
1047
1048static void
1049hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
1050{
1051 struct hns3_nic_priv *priv = netdev_priv(netdev);
1052 int queue_num = priv->ae_handle->kinfo.num_tqps;
1053 struct hns3_enet_ring *ring;
1054 unsigned int start;
1055 unsigned int idx;
1056 u64 tx_bytes = 0;
1057 u64 rx_bytes = 0;
1058 u64 tx_pkts = 0;
1059 u64 rx_pkts = 0;
1060
1061 for (idx = 0; idx < queue_num; idx++) {
1062 /* fetch the tx stats */
1063 ring = priv->ring_data[idx].ring;
1064 do {
d36d36ce 1065 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1066 tx_bytes += ring->stats.tx_bytes;
1067 tx_pkts += ring->stats.tx_pkts;
1068 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1069
1070 /* fetch the rx stats */
1071 ring = priv->ring_data[idx + queue_num].ring;
1072 do {
d36d36ce 1073 start = u64_stats_fetch_begin_irq(&ring->syncp);
76ad4f0e
S		 1074 rx_bytes += ring->stats.rx_bytes;
1075 rx_pkts += ring->stats.rx_pkts;
1076 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
1077 }
1078
1079 stats->tx_bytes = tx_bytes;
1080 stats->tx_packets = tx_pkts;
1081 stats->rx_bytes = rx_bytes;
1082 stats->rx_packets = rx_pkts;
1083
1084 stats->rx_errors = netdev->stats.rx_errors;
1085 stats->multicast = netdev->stats.multicast;
1086 stats->rx_length_errors = netdev->stats.rx_length_errors;
1087 stats->rx_crc_errors = netdev->stats.rx_crc_errors;
1088 stats->rx_missed_errors = netdev->stats.rx_missed_errors;
1089
1090 stats->tx_errors = netdev->stats.tx_errors;
1091 stats->rx_dropped = netdev->stats.rx_dropped;
1092 stats->tx_dropped = netdev->stats.tx_dropped;
1093 stats->collisions = netdev->stats.collisions;
1094 stats->rx_over_errors = netdev->stats.rx_over_errors;
1095 stats->rx_frame_errors = netdev->stats.rx_frame_errors;
1096 stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;
1097 stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;
1098 stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;
1099 stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;
1100 stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;
1101 stats->tx_window_errors = netdev->stats.tx_window_errors;
1102 stats->rx_compressed = netdev->stats.rx_compressed;
1103 stats->tx_compressed = netdev->stats.tx_compressed;
1104}
1105
1106static void hns3_add_tunnel_port(struct net_device *netdev, u16 port,
1107 enum hns3_udp_tnl_type type)
1108{
1109 struct hns3_nic_priv *priv = netdev_priv(netdev);
1110 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1111 struct hnae3_handle *h = priv->ae_handle;
1112
1113 if (udp_tnl->used && udp_tnl->dst_port == port) {
1114 udp_tnl->used++;
1115 return;
1116 }
1117
1118 if (udp_tnl->used) {
1119 netdev_warn(netdev,
1120 "UDP tunnel [%d], port [%d] offload\n", type, port);
1121 return;
1122 }
1123
1124 udp_tnl->dst_port = port;
1125 udp_tnl->used = 1;
1126 /* TBD send command to hardware to add port */
1127 if (h->ae_algo->ops->add_tunnel_udp)
1128 h->ae_algo->ops->add_tunnel_udp(h, port);
1129}
1130
1131static void hns3_del_tunnel_port(struct net_device *netdev, u16 port,
1132 enum hns3_udp_tnl_type type)
1133{
1134 struct hns3_nic_priv *priv = netdev_priv(netdev);
1135 struct hns3_udp_tunnel *udp_tnl = &priv->udp_tnl[type];
1136 struct hnae3_handle *h = priv->ae_handle;
1137
1138 if (!udp_tnl->used || udp_tnl->dst_port != port) {
1139 netdev_warn(netdev,
1140 "Invalid UDP tunnel port %d\n", port);
1141 return;
1142 }
1143
1144 udp_tnl->used--;
1145 if (udp_tnl->used)
1146 return;
1147
1148 udp_tnl->dst_port = 0;
1149 /* TBD send command to hardware to del port */
1150 if (h->ae_algo->ops->del_tunnel_udp)
9537e7cb 1151 h->ae_algo->ops->del_tunnel_udp(h, port);
76ad4f0e
S		 1152}
1153
1154/* hns3_nic_udp_tunnel_add - Get notifiacetion about UDP tunnel ports
1155 * @netdev: This physical ports's netdev
1156 * @ti: Tunnel information
1157 */
1158static void hns3_nic_udp_tunnel_add(struct net_device *netdev,
1159 struct udp_tunnel_info *ti)
1160{
1161 u16 port_n = ntohs(ti->port);
1162
1163 switch (ti->type) {
1164 case UDP_TUNNEL_TYPE_VXLAN:
1165 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1166 break;
1167 case UDP_TUNNEL_TYPE_GENEVE:
1168 hns3_add_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1169 break;
1170 default:
1171 netdev_err(netdev, "unsupported tunnel type %d\n", ti->type);
1172 break;
1173 }
1174}
1175
1176static void hns3_nic_udp_tunnel_del(struct net_device *netdev,
1177 struct udp_tunnel_info *ti)
1178{
1179 u16 port_n = ntohs(ti->port);
1180
1181 switch (ti->type) {
1182 case UDP_TUNNEL_TYPE_VXLAN:
1183 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_VXLAN);
1184 break;
1185 case UDP_TUNNEL_TYPE_GENEVE:
1186 hns3_del_tunnel_port(netdev, port_n, HNS3_UDP_TNL_GENEVE);
1187 break;
1188 default:
1189 break;
1190 }
1191}
1192
30d240df 1193static int hns3_setup_tc(struct net_device *netdev, void *type_data)
76ad4f0e 1194{
30d240df 1195 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
9780cb97 1196 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e 1197 struct hnae3_knic_private_info *kinfo = &h->kinfo;
30d240df
YL		 1198 u8 *prio_tc = mqprio_qopt->qopt.prio_tc_map;
1199 u8 tc = mqprio_qopt->qopt.num_tc;
1200 u16 mode = mqprio_qopt->mode;
1201 u8 hw = mqprio_qopt->qopt.hw;
1202 bool if_running;
76ad4f0e
S		 1203 unsigned int i;
1204 int ret;
1205
30d240df
YL		 1206 if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&
1207 mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))
1208 return -EOPNOTSUPP;
1209
76ad4f0e
S		 1210 if (tc > HNAE3_MAX_TC)
1211 return -EINVAL;
1212
76ad4f0e
S		 1213 if (!netdev)
1214 return -EINVAL;
1215
30d240df
YL		 1216 if_running = netif_running(netdev);
1217 if (if_running) {
1218 hns3_nic_net_stop(netdev);
1219 msleep(100);
76ad4f0e
S		 1220 }
1221
30d240df
YL		 1222 ret = (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?
1223 kinfo->dcb_ops->setup_tc(h, tc, prio_tc) : -EOPNOTSUPP;
76ad4f0e 1224 if (ret)
30d240df
YL		 1225 goto out;
1226
1227 if (tc <= 1) {
1228 netdev_reset_tc(netdev);
1229 } else {
1230 ret = netdev_set_num_tc(netdev, tc);
1231 if (ret)
1232 goto out;
1233
1234 for (i = 0; i < HNAE3_MAX_TC; i++) {
1235 if (!kinfo->tc_info[i].enable)
1236 continue;
76ad4f0e 1237
76ad4f0e
S		 1238 netdev_set_tc_queue(netdev,
1239 kinfo->tc_info[i].tc,
1240 kinfo->tc_info[i].tqp_count,
1241 kinfo->tc_info[i].tqp_offset);
30d240df 1242 }
76ad4f0e
S		 1243 }
1244
30d240df
YL		 1245 ret = hns3_nic_set_real_num_queue(netdev);
1246
1247out:
1248 if (if_running)
1249 hns3_nic_net_open(netdev);
1250
1251 return ret;
76ad4f0e
S		 1252}
1253
2572ac53 1254static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,
de4784ca 1255 void *type_data)
76ad4f0e 1256{
575ed7d3 1257 if (type != TC_SETUP_QDISC_MQPRIO)
38cf0426 1258 return -EOPNOTSUPP;
76ad4f0e 1259
30d240df 1260 return hns3_setup_tc(dev, type_data);
76ad4f0e
S		 1261}
1262
1263static int hns3_vlan_rx_add_vid(struct net_device *netdev,
1264 __be16 proto, u16 vid)
1265{
9780cb97 1266 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1267 int ret = -EIO;
1268
1269 if (h->ae_algo->ops->set_vlan_filter)
1270 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);
1271
1272 return ret;
1273}
1274
1275static int hns3_vlan_rx_kill_vid(struct net_device *netdev,
1276 __be16 proto, u16 vid)
1277{
9780cb97 1278 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1279 int ret = -EIO;
1280
1281 if (h->ae_algo->ops->set_vlan_filter)
1282 ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);
1283
1284 return ret;
1285}
1286
1287static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
1288 u8 qos, __be16 vlan_proto)
1289{
9780cb97 1290 struct hnae3_handle *h = hns3_get_handle(netdev);
76ad4f0e
S		 1291 int ret = -EIO;
1292
1293 if (h->ae_algo->ops->set_vf_vlan_filter)
1294 ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,
1295 qos, vlan_proto);
1296
1297 return ret;
1298}
1299
a8e8b7ff
S		 1300static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)
1301{
9780cb97 1302 struct hnae3_handle *h = hns3_get_handle(netdev);
a8e8b7ff
S		 1303 bool if_running = netif_running(netdev);
1304 int ret;
1305
1306 if (!h->ae_algo->ops->set_mtu)
1307 return -EOPNOTSUPP;
1308
1309 /* if this was called with netdev up then bring netdevice down */
1310 if (if_running) {
1311 (void)hns3_nic_net_stop(netdev);
1312 msleep(100);
1313 }
1314
1315 ret = h->ae_algo->ops->set_mtu(h, new_mtu);
1316 if (ret) {
1317 netdev_err(netdev, "failed to change MTU in hardware %d\n",
1318 ret);
1319 return ret;
1320 }
1321
1322 /* if the netdev was running earlier, bring it up again */
1323 if (if_running && hns3_nic_net_open(netdev))
1324 ret = -EINVAL;
1325
1326 return ret;
1327}
1328
f8fa222c
L		 1329static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)
1330{
1331 struct hns3_nic_priv *priv = netdev_priv(ndev);
1332 struct hns3_enet_ring *tx_ring = NULL;
1333 int timeout_queue = 0;
1334 int hw_head, hw_tail;
1335 int i;
1336
1337 /* Find the stopped queue the same way the stack does */
1338 for (i = 0; i < ndev->real_num_tx_queues; i++) {
1339 struct netdev_queue *q;
1340 unsigned long trans_start;
1341
1342 q = netdev_get_tx_queue(ndev, i);
1343 trans_start = q->trans_start;
1344 if (netif_xmit_stopped(q) &&
1345 time_after(jiffies,
1346 (trans_start + ndev->watchdog_timeo))) {
1347 timeout_queue = i;
1348 break;
1349 }
1350 }
1351
1352 if (i == ndev->num_tx_queues) {
1353 netdev_info(ndev,
1354 "no netdev TX timeout queue found, timeout count: %llu\n",
1355 priv->tx_timeout_count);
1356 return false;
1357 }
1358
1359 tx_ring = priv->ring_data[timeout_queue].ring;
1360
1361 hw_head = readl_relaxed(tx_ring->tqp->io_base +
1362 HNS3_RING_TX_RING_HEAD_REG);
1363 hw_tail = readl_relaxed(tx_ring->tqp->io_base +
1364 HNS3_RING_TX_RING_TAIL_REG);
1365 netdev_info(ndev,
1366 "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, HW_HEAD: 0x%x, HW_TAIL: 0x%x, INT: 0x%x\n",
1367 priv->tx_timeout_count,
1368 timeout_queue,
1369 tx_ring->next_to_use,
1370 tx_ring->next_to_clean,
1371 hw_head,
1372 hw_tail,
1373 readl(tx_ring->tqp_vector->mask_addr));
1374
1375 return true;
1376}
1377
1378static void hns3_nic_net_timeout(struct net_device *ndev)
1379{
1380 struct hns3_nic_priv *priv = netdev_priv(ndev);
1381 unsigned long last_reset_time = priv->last_reset_time;
1382 struct hnae3_handle *h = priv->ae_handle;
1383
1384 if (!hns3_get_tx_timeo_queue_info(ndev))
1385 return;
1386
1387 priv->tx_timeout_count++;
1388
1389 /* This timeout is far away enough from last timeout,
1390 * if timeout again,set the reset type to PF reset
1391 */
1392 if (time_after(jiffies, (last_reset_time + 20 * HZ)))
1393 priv->reset_level = HNAE3_FUNC_RESET;
1394
1395 /* Don't do any new action before the next timeout */
1396 else if (time_before(jiffies, (last_reset_time + ndev->watchdog_timeo)))
1397 return;
1398
1399 priv->last_reset_time = jiffies;
1400
1401 if (h->ae_algo->ops->reset_event)
1402 h->ae_algo->ops->reset_event(h, priv->reset_level);
1403
1404 priv->reset_level++;
1405 if (priv->reset_level > HNAE3_GLOBAL_RESET)
1406 priv->reset_level = HNAE3_GLOBAL_RESET;
1407}
1408
76ad4f0e
S		 1409static const struct net_device_ops hns3_nic_netdev_ops = {
1410 .ndo_open = hns3_nic_net_open,
1411 .ndo_stop = hns3_nic_net_stop,
1412 .ndo_start_xmit = hns3_nic_net_xmit,
f8fa222c 1413 .ndo_tx_timeout = hns3_nic_net_timeout,
76ad4f0e 1414 .ndo_set_mac_address = hns3_nic_net_set_mac_address,
a8e8b7ff 1415 .ndo_change_mtu = hns3_nic_change_mtu,
76ad4f0e
S		 1416 .ndo_set_features = hns3_nic_set_features,
1417 .ndo_get_stats64 = hns3_nic_get_stats64,
1418 .ndo_setup_tc = hns3_nic_setup_tc,
1419 .ndo_set_rx_mode = hns3_nic_set_rx_mode,
1420 .ndo_udp_tunnel_add = hns3_nic_udp_tunnel_add,
1421 .ndo_udp_tunnel_del = hns3_nic_udp_tunnel_del,
1422 .ndo_vlan_rx_add_vid = hns3_vlan_rx_add_vid,
1423 .ndo_vlan_rx_kill_vid = hns3_vlan_rx_kill_vid,
1424 .ndo_set_vf_vlan = hns3_ndo_set_vf_vlan,
1425};
1426
1427/* hns3_probe - Device initialization routine
1428 * @pdev: PCI device information struct
1429 * @ent: entry in hns3_pci_tbl
1430 *
1431 * hns3_probe initializes a PF identified by a pci_dev structure.
1432 * The OS initialization, configuring of the PF private structure,
1433 * and a hardware reset occur.
1434 *
1435 * Returns 0 on success, negative on failure
1436 */
1437static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1438{
1439 struct hnae3_ae_dev *ae_dev;
1440 int ret;
1441
1442 ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev),
1443 GFP_KERNEL);
1444 if (!ae_dev) {
1445 ret = -ENOMEM;
1446 return ret;
1447 }
1448
1449 ae_dev->pdev = pdev;
e92a0843 1450 ae_dev->flag = ent->driver_data;
76ad4f0e
S		 1451 ae_dev->dev_type = HNAE3_DEV_KNIC;
1452 pci_set_drvdata(pdev, ae_dev);
1453
1454 return hnae3_register_ae_dev(ae_dev);
1455}
1456
1457/* hns3_remove - Device removal routine
1458 * @pdev: PCI device information struct
1459 */
1460static void hns3_remove(struct pci_dev *pdev)
1461{
1462 struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);
1463
1464 hnae3_unregister_ae_dev(ae_dev);
1465
1466 devm_kfree(&pdev->dev, ae_dev);
1467
1468 pci_set_drvdata(pdev, NULL);
1469}
1470
1471static struct pci_driver hns3_driver = {
1472 .name = hns3_driver_name,
1473 .id_table = hns3_pci_tbl,
1474 .probe = hns3_probe,
1475 .remove = hns3_remove,
1476};
1477
1478/* set default feature to hns3 */
1479static void hns3_set_default_feature(struct net_device *netdev)
1480{
1481 netdev->priv_flags |= IFF_UNICAST_FLT;
1482
1483 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1484 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1485 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1486 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1487 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1488
1489 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
1490
1491 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
1492
1493 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1494 NETIF_F_HW_VLAN_CTAG_FILTER |
1495 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1496 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1497 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1498 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1499
1500 netdev->vlan_features |=
1501 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
1502 NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO |
1503 NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1504 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1505 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1506
1507 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1508 NETIF_F_HW_VLAN_CTAG_FILTER |
1509 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1510 NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |
1511 NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
1512 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1513}
1514
1515static int hns3_alloc_buffer(struct hns3_enet_ring *ring,
1516 struct hns3_desc_cb *cb)
1517{
1518 unsigned int order = hnae_page_order(ring);
1519 struct page *p;
1520
1521 p = dev_alloc_pages(order);
1522 if (!p)
1523 return -ENOMEM;
1524
1525 cb->priv = p;
1526 cb->page_offset = 0;
1527 cb->reuse_flag = 0;
1528 cb->buf = page_address(p);
1529 cb->length = hnae_page_size(ring);
1530 cb->type = DESC_TYPE_PAGE;
1531
76ad4f0e
S		 1532 return 0;
1533}
1534
1535static void hns3_free_buffer(struct hns3_enet_ring *ring,
1536 struct hns3_desc_cb *cb)
1537{
1538 if (cb->type == DESC_TYPE_SKB)
1539 dev_kfree_skb_any((struct sk_buff *)cb->priv);
1540 else if (!HNAE3_IS_TX_RING(ring))
1541 put_page((struct page *)cb->priv);
1542 memset(cb, 0, sizeof(*cb));
1543}
1544
1545static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)
1546{
1547 cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,
1548 cb->length, ring_to_dma_dir(ring));
1549
1550 if (dma_mapping_error(ring_to_dev(ring), cb->dma))
1551 return -EIO;
1552
1553 return 0;
1554}
1555
1556static void hns3_unmap_buffer(struct hns3_enet_ring *ring,
1557 struct hns3_desc_cb *cb)
1558{
1559 if (cb->type == DESC_TYPE_SKB)
1560 dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,
1561 ring_to_dma_dir(ring));
1562 else
1563 dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,
1564 ring_to_dma_dir(ring));
1565}
1566
1567static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)
1568{
1569 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
1570 ring->desc[i].addr = 0;
1571}
1572
1573static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i)
1574{
1575 struct hns3_desc_cb *cb = &ring->desc_cb[i];
1576
1577 if (!ring->desc_cb[i].dma)
1578 return;
1579
1580 hns3_buffer_detach(ring, i);
1581 hns3_free_buffer(ring, cb);
1582}
1583
1584static void hns3_free_buffers(struct hns3_enet_ring *ring)
1585{
1586 int i;
1587
1588 for (i = 0; i < ring->desc_num; i++)
1589 hns3_free_buffer_detach(ring, i);
1590}
1591
1592/* free desc along with its attached buffer */
1593static void hns3_free_desc(struct hns3_enet_ring *ring)
1594{
1595 hns3_free_buffers(ring);
1596
1597 dma_unmap_single(ring_to_dev(ring), ring->desc_dma_addr,
1598 ring->desc_num * sizeof(ring->desc[0]),
1599 DMA_BIDIRECTIONAL);
1600 ring->desc_dma_addr = 0;
1601 kfree(ring->desc);
1602 ring->desc = NULL;
1603}
1604
1605static int hns3_alloc_desc(struct hns3_enet_ring *ring)
1606{
1607 int size = ring->desc_num * sizeof(ring->desc[0]);
1608
1609 ring->desc = kzalloc(size, GFP_KERNEL);
1610 if (!ring->desc)
1611 return -ENOMEM;
1612
1613 ring->desc_dma_addr = dma_map_single(ring_to_dev(ring), ring->desc,
1614 size, DMA_BIDIRECTIONAL);
1615 if (dma_mapping_error(ring_to_dev(ring), ring->desc_dma_addr)) {
1616 ring->desc_dma_addr = 0;
1617 kfree(ring->desc);
1618 ring->desc = NULL;
1619 return -ENOMEM;
1620 }
1621
1622 return 0;
1623}
1624
1625static int hns3_reserve_buffer_map(struct hns3_enet_ring *ring,
1626 struct hns3_desc_cb *cb)
1627{
1628 int ret;
1629
1630 ret = hns3_alloc_buffer(ring, cb);
1631 if (ret)
1632 goto out;
1633
1634 ret = hns3_map_buffer(ring, cb);
1635 if (ret)
1636 goto out_with_buf;
1637
1638 return 0;
1639
1640out_with_buf:
564883bb 1641 hns3_free_buffer(ring, cb);
76ad4f0e
S		 1642out:
1643 return ret;
1644}
1645
1646static int hns3_alloc_buffer_attach(struct hns3_enet_ring *ring, int i)
1647{
1648 int ret = hns3_reserve_buffer_map(ring, &ring->desc_cb[i]);
1649
1650 if (ret)
1651 return ret;
1652
1653 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1654
1655 return 0;
1656}
1657
1658/* Allocate memory for raw pkg, and map with dma */
1659static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)
1660{
1661 int i, j, ret;
1662
1663 for (i = 0; i < ring->desc_num; i++) {
1664 ret = hns3_alloc_buffer_attach(ring, i);
1665 if (ret)
1666 goto out_buffer_fail;
1667 }
1668
1669 return 0;
1670
1671out_buffer_fail:
1672 for (j = i - 1; j >= 0; j--)
1673 hns3_free_buffer_detach(ring, j);
1674 return ret;
1675}
1676
1677/* detach a in-used buffer and replace with a reserved one */
1678static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,
1679 struct hns3_desc_cb *res_cb)
1680{
b9077428 1681 hns3_unmap_buffer(ring, &ring->desc_cb[i]);
76ad4f0e
S		 1682 ring->desc_cb[i] = *res_cb;
1683 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma);
1684}
1685
1686static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)
1687{
1688 ring->desc_cb[i].reuse_flag = 0;
1689 ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma
1690 + ring->desc_cb[i].page_offset);
1691}
1692
1693static void hns3_nic_reclaim_one_desc(struct hns3_enet_ring *ring, int *bytes,
1694 int *pkts)
1695{
1696 struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
1697
1698 (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
1699 (*bytes) += desc_cb->length;
1700 /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
1701 hns3_free_buffer_detach(ring, ring->next_to_clean);
1702
1703 ring_ptr_move_fw(ring, next_to_clean);
1704}
1705
1706static int is_valid_clean_head(struct hns3_enet_ring *ring, int h)
1707{
1708 int u = ring->next_to_use;
1709 int c = ring->next_to_clean;
1710
1711 if (unlikely(h > ring->desc_num))
1712 return 0;
1713
1714 return u > c ? (h > c && h <= u) : (h > c || h <= u);
1715}
1716
24e750c4 1717bool hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)
76ad4f0e
S		 1718{
1719 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1720 struct netdev_queue *dev_queue;
1721 int bytes, pkts;
1722 int head;
1723
1724 head = readl_relaxed(ring->tqp->io_base + HNS3_RING_TX_RING_HEAD_REG);
1725 rmb(); /* Make sure head is ready before touch any data */
1726
1727 if (is_ring_empty(ring) || head == ring->next_to_clean)
24e750c4 1728 return true; /* no data to poll */
76ad4f0e
S		 1729
1730 if (!is_valid_clean_head(ring, head)) {
1731 netdev_err(netdev, "wrong head (%d, %d-%d)\n", head,
1732 ring->next_to_use, ring->next_to_clean);
1733
1734 u64_stats_update_begin(&ring->syncp);
1735 ring->stats.io_err_cnt++;
1736 u64_stats_update_end(&ring->syncp);
24e750c4 1737 return true;
76ad4f0e
S		 1738 }
1739
1740 bytes = 0;
1741 pkts = 0;
1742 while (head != ring->next_to_clean && budget) {
1743 hns3_nic_reclaim_one_desc(ring, &bytes, &pkts);
1744 /* Issue prefetch for next Tx descriptor */
1745 prefetch(&ring->desc_cb[ring->next_to_clean]);
1746 budget--;
1747 }
1748
1749 ring->tqp_vector->tx_group.total_bytes += bytes;
1750 ring->tqp_vector->tx_group.total_packets += pkts;
1751
1752 u64_stats_update_begin(&ring->syncp);
1753 ring->stats.tx_bytes += bytes;
1754 ring->stats.tx_pkts += pkts;
1755 u64_stats_update_end(&ring->syncp);
1756
1757 dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);
1758 netdev_tx_completed_queue(dev_queue, pkts, bytes);
1759
1760 if (unlikely(pkts && netif_carrier_ok(netdev) &&
1761 (ring_space(ring) > HNS3_MAX_BD_PER_PKT))) {
1762 /* Make sure that anybody stopping the queue after this
1763 * sees the new next_to_clean.
1764 */
1765 smp_mb();
1766 if (netif_tx_queue_stopped(dev_queue)) {
1767 netif_tx_wake_queue(dev_queue);
1768 ring->stats.restart_queue++;
1769 }
1770 }
1771
1772 return !!budget;
1773}
1774
1775static int hns3_desc_unused(struct hns3_enet_ring *ring)
1776{
1777 int ntc = ring->next_to_clean;
1778 int ntu = ring->next_to_use;
1779
1780 return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;
1781}
1782
1783static void
1784hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, int cleand_count)
1785{
1786 struct hns3_desc_cb *desc_cb;
1787 struct hns3_desc_cb res_cbs;
1788 int i, ret;
1789
1790 for (i = 0; i < cleand_count; i++) {
1791 desc_cb = &ring->desc_cb[ring->next_to_use];
1792 if (desc_cb->reuse_flag) {
1793 u64_stats_update_begin(&ring->syncp);
1794 ring->stats.reuse_pg_cnt++;
1795 u64_stats_update_end(&ring->syncp);
1796
1797 hns3_reuse_buffer(ring, ring->next_to_use);
1798 } else {
1799 ret = hns3_reserve_buffer_map(ring, &res_cbs);
1800 if (ret) {
1801 u64_stats_update_begin(&ring->syncp);
1802 ring->stats.sw_err_cnt++;
1803 u64_stats_update_end(&ring->syncp);
1804
1805 netdev_err(ring->tqp->handle->kinfo.netdev,
1806 "hnae reserve buffer map failed.\n");
1807 break;
1808 }
1809 hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);
1810 }
1811
1812 ring_ptr_move_fw(ring, next_to_use);
1813 }
1814
1815 wmb(); /* Make all data has been write before submit */
1816 writel_relaxed(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);
1817}
1818
1819/* hns3_nic_get_headlen - determine size of header for LRO/GRO
1820 * @data: pointer to the start of the headers
1821 * @max: total length of section to find headers in
1822 *
1823 * This function is meant to determine the length of headers that will
1824 * be recognized by hardware for LRO, GRO, and RSC offloads. The main
1825 * motivation of doing this is to only perform one pull for IPv4 TCP
1826 * packets so that we can do basic things like calculating the gso_size
1827 * based on the average data per packet.
1828 */
1829static unsigned int hns3_nic_get_headlen(unsigned char *data, u32 flag,
1830 unsigned int max_size)
1831{
1832 unsigned char *network;
1833 u8 hlen;
1834
1835 /* This should never happen, but better safe than sorry */
1836 if (max_size < ETH_HLEN)
1837 return max_size;
1838
1839 /* Initialize network frame pointer */
1840 network = data;
1841
1842 /* Set first protocol and move network header forward */
1843 network += ETH_HLEN;
1844
1845 /* Handle any vlan tag if present */
1846 if (hnae_get_field(flag, HNS3_RXD_VLAN_M, HNS3_RXD_VLAN_S)
1847 == HNS3_RX_FLAG_VLAN_PRESENT) {
1848 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
1849 return max_size;
1850
1851 network += VLAN_HLEN;
1852 }
1853
1854 /* Handle L3 protocols */
1855 if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1856 == HNS3_RX_FLAG_L3ID_IPV4) {
1857 if ((typeof(max_size))(network - data) >
1858 (max_size - sizeof(struct iphdr)))
1859 return max_size;
1860
1861 /* Access ihl as a u8 to avoid unaligned access on ia64 */
1862 hlen = (network[0] & 0x0F) << 2;
1863
1864 /* Verify hlen meets minimum size requirements */
1865 if (hlen < sizeof(struct iphdr))
1866 return network - data;
1867
1868 /* Record next protocol if header is present */
1869 } else if (hnae_get_field(flag, HNS3_RXD_L3ID_M, HNS3_RXD_L3ID_S)
1870 == HNS3_RX_FLAG_L3ID_IPV6) {
1871 if ((typeof(max_size))(network - data) >
1872 (max_size - sizeof(struct ipv6hdr)))
1873 return max_size;
1874
1875 /* Record next protocol */
1876 hlen = sizeof(struct ipv6hdr);
1877 } else {
1878 return network - data;
1879 }
1880
1881 /* Relocate pointer to start of L4 header */
1882 network += hlen;
1883
1884 /* Finally sort out TCP/UDP */
1885 if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1886 == HNS3_RX_FLAG_L4ID_TCP) {
1887 if ((typeof(max_size))(network - data) >
1888 (max_size - sizeof(struct tcphdr)))
1889 return max_size;
1890
1891 /* Access doff as a u8 to avoid unaligned access on ia64 */
1892 hlen = (network[12] & 0xF0) >> 2;
1893
1894 /* Verify hlen meets minimum size requirements */
1895 if (hlen < sizeof(struct tcphdr))
1896 return network - data;
1897
1898 network += hlen;
1899 } else if (hnae_get_field(flag, HNS3_RXD_L4ID_M, HNS3_RXD_L4ID_S)
1900 == HNS3_RX_FLAG_L4ID_UDP) {
1901 if ((typeof(max_size))(network - data) >
1902 (max_size - sizeof(struct udphdr)))
1903 return max_size;
1904
1905 network += sizeof(struct udphdr);
1906 }
1907
1908 /* If everything has gone correctly network should be the
1909 * data section of the packet and will be the end of the header.
1910 * If not then it probably represents the end of the last recognized
1911 * header.
1912 */
1913 if ((typeof(max_size))(network - data) < max_size)
1914 return network - data;
1915 else
1916 return max_size;
1917}
1918
1919static void hns3_nic_reuse_page(struct sk_buff *skb, int i,
1920 struct hns3_enet_ring *ring, int pull_len,
1921 struct hns3_desc_cb *desc_cb)
1922{
1923 struct hns3_desc *desc;
1924 int truesize, size;
1925 int last_offset;
1926 bool twobufs;
1927
1928 twobufs = ((PAGE_SIZE < 8192) &&
1929 hnae_buf_size(ring) == HNS3_BUFFER_SIZE_2048);
1930
1931 desc = &ring->desc[ring->next_to_clean];
1932 size = le16_to_cpu(desc->rx.size);
1933
1934 if (twobufs) {
1935 truesize = hnae_buf_size(ring);
1936 } else {
1937 truesize = ALIGN(size, L1_CACHE_BYTES);
1938 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
1939 }
1940
1941 skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
1942 size - pull_len, truesize - pull_len);
1943
1944 /* Avoid re-using remote pages,flag default unreuse */
1945 if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
1946 return;
1947
1948 if (twobufs) {
1949 /* If we are only owner of page we can reuse it */
1950 if (likely(page_count(desc_cb->priv) == 1)) {
1951 /* Flip page offset to other buffer */
1952 desc_cb->page_offset ^= truesize;
1953
1954 desc_cb->reuse_flag = 1;
1955 /* bump ref count on page before it is given*/
1956 get_page(desc_cb->priv);
1957 }
1958 return;
1959 }
1960
1961 /* Move offset up to the next cache line */
1962 desc_cb->page_offset += truesize;
1963
1964 if (desc_cb->page_offset <= last_offset) {
1965 desc_cb->reuse_flag = 1;
1966 /* Bump ref count on page before it is given*/
1967 get_page(desc_cb->priv);
1968 }
1969}
1970
1971static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,
1972 struct hns3_desc *desc)
1973{
1974 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
1975 int l3_type, l4_type;
1976 u32 bd_base_info;
1977 int ol4_type;
1978 u32 l234info;
1979
1980 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
1981 l234info = le32_to_cpu(desc->rx.l234_info);
1982
1983 skb->ip_summed = CHECKSUM_NONE;
1984
1985 skb_checksum_none_assert(skb);
1986
1987 if (!(netdev->features & NETIF_F_RXCSUM))
1988 return;
1989
1990 /* check if hardware has done checksum */
1991 if (!hnae_get_bit(bd_base_info, HNS3_RXD_L3L4P_B))
1992 return;
1993
1994 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L3E_B) ||
1995 hnae_get_bit(l234info, HNS3_RXD_L4E_B) ||
1996 hnae_get_bit(l234info, HNS3_RXD_OL3E_B) ||
1997 hnae_get_bit(l234info, HNS3_RXD_OL4E_B))) {
1998 netdev_err(netdev, "L3/L4 error pkt\n");
1999 u64_stats_update_begin(&ring->syncp);
2000 ring->stats.l3l4_csum_err++;
2001 u64_stats_update_end(&ring->syncp);
2002
2003 return;
2004 }
2005
2006 l3_type = hnae_get_field(l234info, HNS3_RXD_L3ID_M,
2007 HNS3_RXD_L3ID_S);
2008 l4_type = hnae_get_field(l234info, HNS3_RXD_L4ID_M,
2009 HNS3_RXD_L4ID_S);
2010
2011 ol4_type = hnae_get_field(l234info, HNS3_RXD_OL4ID_M, HNS3_RXD_OL4ID_S);
2012 switch (ol4_type) {
2013 case HNS3_OL4_TYPE_MAC_IN_UDP:
2014 case HNS3_OL4_TYPE_NVGRE:
2015 skb->csum_level = 1;
2016 case HNS3_OL4_TYPE_NO_TUN:
2017 /* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */
2018 if (l3_type == HNS3_L3_TYPE_IPV4 ||
2019 (l3_type == HNS3_L3_TYPE_IPV6 &&
2020 (l4_type == HNS3_L4_TYPE_UDP ||
2021 l4_type == HNS3_L4_TYPE_TCP ||
2022 l4_type == HNS3_L4_TYPE_SCTP)))
2023 skb->ip_summed = CHECKSUM_UNNECESSARY;
2024 break;
2025 }
2026}
2027
d43e5aca
YL		 2028static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)
2029{
2030 napi_gro_receive(&ring->tqp_vector->napi, skb);
2031}
2032
76ad4f0e
S		 2033static int hns3_handle_rx_bd(struct hns3_enet_ring *ring,
2034 struct sk_buff **out_skb, int *out_bnum)
2035{
2036 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2037 struct hns3_desc_cb *desc_cb;
2038 struct hns3_desc *desc;
2039 struct sk_buff *skb;
2040 unsigned char *va;
2041 u32 bd_base_info;
2042 int pull_len;
2043 u32 l234info;
2044 int length;
2045 int bnum;
2046
2047 desc = &ring->desc[ring->next_to_clean];
2048 desc_cb = &ring->desc_cb[ring->next_to_clean];
2049
2050 prefetch(desc);
2051
2052 length = le16_to_cpu(desc->rx.pkt_len);
2053 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2054 l234info = le32_to_cpu(desc->rx.l234_info);
2055
2056 /* Check valid BD */
2057 if (!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))
2058 return -EFAULT;
2059
2060 va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
2061
2062 /* Prefetch first cache line of first page
2063 * Idea is to cache few bytes of the header of the packet. Our L1 Cache
2064 * line size is 64B so need to prefetch twice to make it 128B. But in
2065 * actual we can have greater size of caches with 128B Level 1 cache
2066 * lines. In such a case, single fetch would suffice to cache in the
2067 * relevant part of the header.
2068 */
2069 prefetch(va);
2070#if L1_CACHE_BYTES < 128
2071 prefetch(va + L1_CACHE_BYTES);
2072#endif
2073
2074 skb = *out_skb = napi_alloc_skb(&ring->tqp_vector->napi,
2075 HNS3_RX_HEAD_SIZE);
2076 if (unlikely(!skb)) {
2077 netdev_err(netdev, "alloc rx skb fail\n");
2078
2079 u64_stats_update_begin(&ring->syncp);
2080 ring->stats.sw_err_cnt++;
2081 u64_stats_update_end(&ring->syncp);
2082
2083 return -ENOMEM;
2084 }
2085
2086 prefetchw(skb->data);
2087
2088 bnum = 1;
2089 if (length <= HNS3_RX_HEAD_SIZE) {
2090 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
2091
2092 /* We can reuse buffer as-is, just make sure it is local */
2093 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
2094 desc_cb->reuse_flag = 1;
2095 else /* This page cannot be reused so discard it */
2096 put_page(desc_cb->priv);
2097
2098 ring_ptr_move_fw(ring, next_to_clean);
2099 } else {
2100 u64_stats_update_begin(&ring->syncp);
2101 ring->stats.seg_pkt_cnt++;
2102 u64_stats_update_end(&ring->syncp);
2103
2104 pull_len = hns3_nic_get_headlen(va, l234info,
2105 HNS3_RX_HEAD_SIZE);
2106 memcpy(__skb_put(skb, pull_len), va,
2107 ALIGN(pull_len, sizeof(long)));
2108
2109 hns3_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
2110 ring_ptr_move_fw(ring, next_to_clean);
2111
2112 while (!hnae_get_bit(bd_base_info, HNS3_RXD_FE_B)) {
2113 desc = &ring->desc[ring->next_to_clean];
2114 desc_cb = &ring->desc_cb[ring->next_to_clean];
2115 bd_base_info = le32_to_cpu(desc->rx.bd_base_info);
2116 hns3_nic_reuse_page(skb, bnum, ring, 0, desc_cb);
2117 ring_ptr_move_fw(ring, next_to_clean);
2118 bnum++;
2119 }
2120 }
2121
2122 *out_bnum = bnum;
2123
2124 if (unlikely(!hnae_get_bit(bd_base_info, HNS3_RXD_VLD_B))) {
2125 netdev_err(netdev, "no valid bd,%016llx,%016llx\n",
2126 ((u64 *)desc)[0], ((u64 *)desc)[1]);
2127 u64_stats_update_begin(&ring->syncp);
2128 ring->stats.non_vld_descs++;
2129 u64_stats_update_end(&ring->syncp);
2130
2131 dev_kfree_skb_any(skb);
2132 return -EINVAL;
2133 }
2134
2135 if (unlikely((!desc->rx.pkt_len) ||
2136 hnae_get_bit(l234info, HNS3_RXD_TRUNCAT_B))) {
2137 netdev_err(netdev, "truncated pkt\n");
2138 u64_stats_update_begin(&ring->syncp);
2139 ring->stats.err_pkt_len++;
2140 u64_stats_update_end(&ring->syncp);
2141
2142 dev_kfree_skb_any(skb);
2143 return -EFAULT;
2144 }
2145
2146 if (unlikely(hnae_get_bit(l234info, HNS3_RXD_L2E_B))) {
2147 netdev_err(netdev, "L2 error pkt\n");
2148 u64_stats_update_begin(&ring->syncp);
2149 ring->stats.l2_err++;
2150 u64_stats_update_end(&ring->syncp);
2151
2152 dev_kfree_skb_any(skb);
2153 return -EFAULT;
2154 }
2155
2156 u64_stats_update_begin(&ring->syncp);
2157 ring->stats.rx_pkts++;
2158 ring->stats.rx_bytes += skb->len;
2159 u64_stats_update_end(&ring->syncp);
2160
2161 ring->tqp_vector->rx_group.total_bytes += skb->len;
2162
2163 hns3_rx_checksum(ring, skb, desc);
2164 return 0;
2165}
2166
d43e5aca
YL		 2167int hns3_clean_rx_ring(
2168 struct hns3_enet_ring *ring, int budget,
2169 void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))
76ad4f0e
S		 2170{
2171#define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
2172 struct net_device *netdev = ring->tqp->handle->kinfo.netdev;
2173 int recv_pkts, recv_bds, clean_count, err;
2174 int unused_count = hns3_desc_unused(ring);
2175 struct sk_buff *skb = NULL;
2176 int num, bnum = 0;
2177
2178 num = readl_relaxed(ring->tqp->io_base + HNS3_RING_RX_RING_FBDNUM_REG);
2179 rmb(); /* Make sure num taken effect before the other data is touched */
2180
2181 recv_pkts = 0, recv_bds = 0, clean_count = 0;
2182 num -= unused_count;
2183
2184 while (recv_pkts < budget && recv_bds < num) {
2185 /* Reuse or realloc buffers */
2186 if (clean_count + unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
2187 hns3_nic_alloc_rx_buffers(ring,
2188 clean_count + unused_count);
2189 clean_count = 0;
2190 unused_count = hns3_desc_unused(ring);
2191 }
2192
2193 /* Poll one pkt */
2194 err = hns3_handle_rx_bd(ring, &skb, &bnum);
2195 if (unlikely(!skb)) /* This fault cannot be repaired */
2196 goto out;
2197
2198 recv_bds += bnum;
2199 clean_count += bnum;
2200 if (unlikely(err)) { /* Do jump the err */
2201 recv_pkts++;
2202 continue;
2203 }
2204
2205 /* Do update ip stack process */
2206 skb->protocol = eth_type_trans(skb, netdev);
d43e5aca 2207 rx_fn(ring, skb);
76ad4f0e
S		 2208
2209 recv_pkts++;
2210 }
2211
2212out:
2213 /* Make all data has been write before submit */
2214 if (clean_count + unused_count > 0)
2215 hns3_nic_alloc_rx_buffers(ring,
2216 clean_count + unused_count);
2217
2218 return recv_pkts;
2219}
2220
2221static bool hns3_get_new_int_gl(struct hns3_enet_ring_group *ring_group)
2222{
2223#define HNS3_RX_ULTRA_PACKET_RATE 40000
2224 enum hns3_flow_level_range new_flow_level;
2225 struct hns3_enet_tqp_vector *tqp_vector;
2226 int packets_per_secs;
2227 int bytes_per_usecs;
2228 u16 new_int_gl;
2229 int usecs;
2230
2231 if (!ring_group->int_gl)
2232 return false;
2233
2234 if (ring_group->total_packets == 0) {
2235 ring_group->int_gl = HNS3_INT_GL_50K;
2236 ring_group->flow_level = HNS3_FLOW_LOW;
2237 return true;
2238 }
2239
2240 /* Simple throttlerate management
2241 * 0-10MB/s lower (50000 ints/s)
2242 * 10-20MB/s middle (20000 ints/s)
2243 * 20-1249MB/s high (18000 ints/s)
2244 * > 40000pps ultra (8000 ints/s)
2245 */
2246 new_flow_level = ring_group->flow_level;
2247 new_int_gl = ring_group->int_gl;
2248 tqp_vector = ring_group->ring->tqp_vector;
2249 usecs = (ring_group->int_gl << 1);
2250 bytes_per_usecs = ring_group->total_bytes / usecs;
2251 /* 1000000 microseconds */
2252 packets_per_secs = ring_group->total_packets * 1000000 / usecs;
2253
2254 switch (new_flow_level) {
2255 case HNS3_FLOW_LOW:
2256 if (bytes_per_usecs > 10)
2257 new_flow_level = HNS3_FLOW_MID;
2258 break;
2259 case HNS3_FLOW_MID:
2260 if (bytes_per_usecs > 20)
2261 new_flow_level = HNS3_FLOW_HIGH;
2262 else if (bytes_per_usecs <= 10)
2263 new_flow_level = HNS3_FLOW_LOW;
2264 break;
2265 case HNS3_FLOW_HIGH:
2266 case HNS3_FLOW_ULTRA:
2267 default:
2268 if (bytes_per_usecs <= 20)
2269 new_flow_level = HNS3_FLOW_MID;
2270 break;
2271 }
2272
2273 if ((packets_per_secs > HNS3_RX_ULTRA_PACKET_RATE) &&
2274 (&tqp_vector->rx_group == ring_group))
2275 new_flow_level = HNS3_FLOW_ULTRA;
2276
2277 switch (new_flow_level) {
2278 case HNS3_FLOW_LOW:
2279 new_int_gl = HNS3_INT_GL_50K;
2280 break;
2281 case HNS3_FLOW_MID:
2282 new_int_gl = HNS3_INT_GL_20K;
2283 break;
2284 case HNS3_FLOW_HIGH:
2285 new_int_gl = HNS3_INT_GL_18K;
2286 break;
2287 case HNS3_FLOW_ULTRA:
2288 new_int_gl = HNS3_INT_GL_8K;
2289 break;
2290 default:
2291 break;
2292 }
2293
2294 ring_group->total_bytes = 0;
2295 ring_group->total_packets = 0;
2296 ring_group->flow_level = new_flow_level;
2297 if (new_int_gl != ring_group->int_gl) {
2298 ring_group->int_gl = new_int_gl;
2299 return true;
2300 }
2301 return false;
2302}
2303
2304static void hns3_update_new_int_gl(struct hns3_enet_tqp_vector *tqp_vector)
2305{
2306 u16 rx_int_gl, tx_int_gl;
2307 bool rx, tx;
2308
2309 rx = hns3_get_new_int_gl(&tqp_vector->rx_group);
2310 tx = hns3_get_new_int_gl(&tqp_vector->tx_group);
2311 rx_int_gl = tqp_vector->rx_group.int_gl;
2312 tx_int_gl = tqp_vector->tx_group.int_gl;
2313 if (rx && tx) {
2314 if (rx_int_gl > tx_int_gl) {
2315 tqp_vector->tx_group.int_gl = rx_int_gl;
2316 tqp_vector->tx_group.flow_level =
2317 tqp_vector->rx_group.flow_level;
2318 hns3_set_vector_coalesc_gl(tqp_vector, rx_int_gl);
2319 } else {
2320 tqp_vector->rx_group.int_gl = tx_int_gl;
2321 tqp_vector->rx_group.flow_level =
2322 tqp_vector->tx_group.flow_level;
2323 hns3_set_vector_coalesc_gl(tqp_vector, tx_int_gl);
2324 }
2325 }
2326}
2327
2328static int hns3_nic_common_poll(struct napi_struct *napi, int budget)
2329{
2330 struct hns3_enet_ring *ring;
2331 int rx_pkt_total = 0;
2332
2333 struct hns3_enet_tqp_vector *tqp_vector =
2334 container_of(napi, struct hns3_enet_tqp_vector, napi);
2335 bool clean_complete = true;
2336 int rx_budget;
2337
2338 /* Since the actual Tx work is minimal, we can give the Tx a larger
2339 * budget and be more aggressive about cleaning up the Tx descriptors.
2340 */
2341 hns3_for_each_ring(ring, tqp_vector->tx_group) {
2342 if (!hns3_clean_tx_ring(ring, budget))
2343 clean_complete = false;
2344 }
2345
2346 /* make sure rx ring budget not smaller than 1 */
2347 rx_budget = max(budget / tqp_vector->num_tqps, 1);
2348
2349 hns3_for_each_ring(ring, tqp_vector->rx_group) {
d43e5aca
YL		 2350 int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,
2351 hns3_rx_skb);
76ad4f0e
S		 2352
2353 if (rx_cleaned >= rx_budget)
2354 clean_complete = false;
2355
2356 rx_pkt_total += rx_cleaned;
2357 }
2358
2359 tqp_vector->rx_group.total_packets += rx_pkt_total;
2360
2361 if (!clean_complete)
2362 return budget;
2363
2364 napi_complete(napi);
2365 hns3_update_new_int_gl(tqp_vector);
2366 hns3_mask_vector_irq(tqp_vector, 1);
2367
2368 return rx_pkt_total;
2369}
2370
2371static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2372 struct hnae3_ring_chain_node *head)
2373{
2374 struct pci_dev *pdev = tqp_vector->handle->pdev;
2375 struct hnae3_ring_chain_node *cur_chain = head;
2376 struct hnae3_ring_chain_node *chain;
2377 struct hns3_enet_ring *tx_ring;
2378 struct hns3_enet_ring *rx_ring;
2379
2380 tx_ring = tqp_vector->tx_group.ring;
2381 if (tx_ring) {
2382 cur_chain->tqp_index = tx_ring->tqp->tqp_index;
2383 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2384 HNAE3_RING_TYPE_TX);
2385
2386 cur_chain->next = NULL;
2387
2388 while (tx_ring->next) {
2389 tx_ring = tx_ring->next;
2390
2391 chain = devm_kzalloc(&pdev->dev, sizeof(*chain),
2392 GFP_KERNEL);
2393 if (!chain)
2394 return -ENOMEM;
2395
2396 cur_chain->next = chain;
2397 chain->tqp_index = tx_ring->tqp->tqp_index;
2398 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2399 HNAE3_RING_TYPE_TX);
2400
2401 cur_chain = chain;
2402 }
2403 }
2404
2405 rx_ring = tqp_vector->rx_group.ring;
2406 if (!tx_ring && rx_ring) {
2407 cur_chain->next = NULL;
2408 cur_chain->tqp_index = rx_ring->tqp->tqp_index;
2409 hnae_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B,
2410 HNAE3_RING_TYPE_RX);
2411
2412 rx_ring = rx_ring->next;
2413 }
2414
2415 while (rx_ring) {
2416 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);
2417 if (!chain)
2418 return -ENOMEM;
2419
2420 cur_chain->next = chain;
2421 chain->tqp_index = rx_ring->tqp->tqp_index;
2422 hnae_set_bit(chain->flag, HNAE3_RING_TYPE_B,
2423 HNAE3_RING_TYPE_RX);
2424 cur_chain = chain;
2425
2426 rx_ring = rx_ring->next;
2427 }
2428
2429 return 0;
2430}
2431
2432static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,
2433 struct hnae3_ring_chain_node *head)
2434{
2435 struct pci_dev *pdev = tqp_vector->handle->pdev;
2436 struct hnae3_ring_chain_node *chain_tmp, *chain;
2437
2438 chain = head->next;
2439
2440 while (chain) {
2441 chain_tmp = chain->next;
2442 devm_kfree(&pdev->dev, chain);
2443 chain = chain_tmp;
2444 }
2445}
2446
2447static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,
2448 struct hns3_enet_ring *ring)
2449{
2450 ring->next = group->ring;
2451 group->ring = ring;
2452
2453 group->count++;
2454}
2455
2456static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
2457{
2458 struct hnae3_ring_chain_node vector_ring_chain;
2459 struct hnae3_handle *h = priv->ae_handle;
2460 struct hns3_enet_tqp_vector *tqp_vector;
2461 struct hnae3_vector_info *vector;
2462 struct pci_dev *pdev = h->pdev;
2463 u16 tqp_num = h->kinfo.num_tqps;
2464 u16 vector_num;
2465 int ret = 0;
2466 u16 i;
2467
2468 /* RSS size, cpu online and vector_num should be the same */
2469 /* Should consider 2p/4p later */
2470 vector_num = min_t(u16, num_online_cpus(), tqp_num);
2471 vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),
2472 GFP_KERNEL);
2473 if (!vector)
2474 return -ENOMEM;
2475
2476 vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);
2477
2478 priv->vector_num = vector_num;
2479 priv->tqp_vector = (struct hns3_enet_tqp_vector *)
2480 devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),
2481 GFP_KERNEL);
2482 if (!priv->tqp_vector)
2483 return -ENOMEM;
2484
2485 for (i = 0; i < tqp_num; i++) {
2486 u16 vector_i = i % vector_num;
2487
2488 tqp_vector = &priv->tqp_vector[vector_i];
2489
2490 hns3_add_ring_to_group(&tqp_vector->tx_group,
2491 priv->ring_data[i].ring);
2492
2493 hns3_add_ring_to_group(&tqp_vector->rx_group,
2494 priv->ring_data[i + tqp_num].ring);
2495
2496 tqp_vector->idx = vector_i;
2497 tqp_vector->mask_addr = vector[vector_i].io_addr;
2498 tqp_vector->vector_irq = vector[vector_i].vector;
2499 tqp_vector->num_tqps++;
2500
2501 priv->ring_data[i].ring->tqp_vector = tqp_vector;
2502 priv->ring_data[i + tqp_num].ring->tqp_vector = tqp_vector;
2503 }
2504
2505 for (i = 0; i < vector_num; i++) {
2506 tqp_vector = &priv->tqp_vector[i];
2507
2508 tqp_vector->rx_group.total_bytes = 0;
2509 tqp_vector->rx_group.total_packets = 0;
2510 tqp_vector->tx_group.total_bytes = 0;
2511 tqp_vector->tx_group.total_packets = 0;
2512 hns3_vector_gl_rl_init(tqp_vector);
2513 tqp_vector->handle = h;
2514
2515 ret = hns3_get_vector_ring_chain(tqp_vector,
2516 &vector_ring_chain);
2517 if (ret)
2518 goto out;
2519
2520 ret = h->ae_algo->ops->map_ring_to_vector(h,
2521 tqp_vector->vector_irq, &vector_ring_chain);
2522 if (ret)
2523 goto out;
2524
2525 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2526
2527 netif_napi_add(priv->netdev, &tqp_vector->napi,
2528 hns3_nic_common_poll, NAPI_POLL_WEIGHT);
2529 }
2530
2531out:
2532 devm_kfree(&pdev->dev, vector);
2533 return ret;
2534}
2535
2536static int hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)
2537{
2538 struct hnae3_ring_chain_node vector_ring_chain;
2539 struct hnae3_handle *h = priv->ae_handle;
2540 struct hns3_enet_tqp_vector *tqp_vector;
2541 struct pci_dev *pdev = h->pdev;
2542 int i, ret;
2543
2544 for (i = 0; i < priv->vector_num; i++) {
2545 tqp_vector = &priv->tqp_vector[i];
2546
2547 ret = hns3_get_vector_ring_chain(tqp_vector,
2548 &vector_ring_chain);
2549 if (ret)
2550 return ret;
2551
2552 ret = h->ae_algo->ops->unmap_ring_from_vector(h,
2553 tqp_vector->vector_irq, &vector_ring_chain);
2554 if (ret)
2555 return ret;
2556
2557 hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain);
2558
2559 if (priv->tqp_vector[i].irq_init_flag == HNS3_VECTOR_INITED) {
2560 (void)irq_set_affinity_hint(
2561 priv->tqp_vector[i].vector_irq,
2562 NULL);
ae064e61 2563 free_irq(priv->tqp_vector[i].vector_irq,
2564 &priv->tqp_vector[i]);
76ad4f0e
S		 2565 }
2566
2567 priv->ring_data[i].ring->irq_init_flag = HNS3_VECTOR_NOT_INITED;
2568
2569 netif_napi_del(&priv->tqp_vector[i].napi);
2570 }
2571
2572 devm_kfree(&pdev->dev, priv->tqp_vector);
2573
2574 return 0;
2575}
2576
2577static int hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,
2578 int ring_type)
2579{
2580 struct hns3_nic_ring_data *ring_data = priv->ring_data;
2581 int queue_num = priv->ae_handle->kinfo.num_tqps;
2582 struct pci_dev *pdev = priv->ae_handle->pdev;
2583 struct hns3_enet_ring *ring;
2584
2585 ring = devm_kzalloc(&pdev->dev, sizeof(*ring), GFP_KERNEL);
2586 if (!ring)
2587 return -ENOMEM;
2588
2589 if (ring_type == HNAE3_RING_TYPE_TX) {
2590 ring_data[q->tqp_index].ring = ring;
66b44730 2591 ring_data[q->tqp_index].queue_index = q->tqp_index;
76ad4f0e
S		 2592 ring->io_base = (u8 __iomem *)q->io_base + HNS3_TX_REG_OFFSET;
2593 } else {
2594 ring_data[q->tqp_index + queue_num].ring = ring;
66b44730 2595 ring_data[q->tqp_index + queue_num].queue_index = q->tqp_index;
76ad4f0e
S		 2596 ring->io_base = q->io_base;
2597 }
2598
2599 hnae_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);
2600
76ad4f0e
S		 2601 ring->tqp = q;
2602 ring->desc = NULL;
2603 ring->desc_cb = NULL;
2604 ring->dev = priv->dev;
2605 ring->desc_dma_addr = 0;
2606 ring->buf_size = q->buf_size;
2607 ring->desc_num = q->desc_num;
2608 ring->next_to_use = 0;
2609 ring->next_to_clean = 0;
2610
2611 return 0;
2612}
2613
2614static int hns3_queue_to_ring(struct hnae3_queue *tqp,
2615 struct hns3_nic_priv *priv)
2616{
2617 int ret;
2618
2619 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);
2620 if (ret)
2621 return ret;
2622
2623 ret = hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);
2624 if (ret)
2625 return ret;
2626
2627 return 0;
2628}
2629
2630static int hns3_get_ring_config(struct hns3_nic_priv *priv)
2631{
2632 struct hnae3_handle *h = priv->ae_handle;
2633 struct pci_dev *pdev = h->pdev;
2634 int i, ret;
2635
2636 priv->ring_data = devm_kzalloc(&pdev->dev, h->kinfo.num_tqps *
2637 sizeof(*priv->ring_data) * 2,
2638 GFP_KERNEL);
2639 if (!priv->ring_data)
2640 return -ENOMEM;
2641
2642 for (i = 0; i < h->kinfo.num_tqps; i++) {
2643 ret = hns3_queue_to_ring(h->kinfo.tqp[i], priv);
2644 if (ret)
2645 goto err;
2646 }
2647
2648 return 0;
2649err:
2650 devm_kfree(&pdev->dev, priv->ring_data);
2651 return ret;
2652}
2653
09f2af64
PL		 2654static void hns3_put_ring_config(struct hns3_nic_priv *priv)
2655{
2656 struct hnae3_handle *h = priv->ae_handle;
2657 int i;
2658
2659 for (i = 0; i < h->kinfo.num_tqps; i++) {
2660 devm_kfree(priv->dev, priv->ring_data[i].ring);
2661 devm_kfree(priv->dev,
2662 priv->ring_data[i + h->kinfo.num_tqps].ring);
2663 }
2664 devm_kfree(priv->dev, priv->ring_data);
2665}
2666
76ad4f0e
S		 2667static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)
2668{
2669 int ret;
2670
2671 if (ring->desc_num <= 0 || ring->buf_size <= 0)
2672 return -EINVAL;
2673
2674 ring->desc_cb = kcalloc(ring->desc_num, sizeof(ring->desc_cb[0]),
2675 GFP_KERNEL);
2676 if (!ring->desc_cb) {
2677 ret = -ENOMEM;
2678 goto out;
2679 }
2680
2681 ret = hns3_alloc_desc(ring);
2682 if (ret)
2683 goto out_with_desc_cb;
2684
2685 if (!HNAE3_IS_TX_RING(ring)) {
2686 ret = hns3_alloc_ring_buffers(ring);
2687 if (ret)
2688 goto out_with_desc;
2689 }
2690
2691 return 0;
2692
2693out_with_desc:
2694 hns3_free_desc(ring);
2695out_with_desc_cb:
2696 kfree(ring->desc_cb);
2697 ring->desc_cb = NULL;
2698out:
2699 return ret;
2700}
2701
2702static void hns3_fini_ring(struct hns3_enet_ring *ring)
2703{
2704 hns3_free_desc(ring);
2705 kfree(ring->desc_cb);
2706 ring->desc_cb = NULL;
2707 ring->next_to_clean = 0;
2708 ring->next_to_use = 0;
2709}
2710
1db9b1bf 2711static int hns3_buf_size2type(u32 buf_size)
76ad4f0e
S		 2712{
2713 int bd_size_type;
2714
2715 switch (buf_size) {
2716 case 512:
2717 bd_size_type = HNS3_BD_SIZE_512_TYPE;
2718 break;
2719 case 1024:
2720 bd_size_type = HNS3_BD_SIZE_1024_TYPE;
2721 break;
2722 case 2048:
2723 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2724 break;
2725 case 4096:
2726 bd_size_type = HNS3_BD_SIZE_4096_TYPE;
2727 break;
2728 default:
2729 bd_size_type = HNS3_BD_SIZE_2048_TYPE;
2730 }
2731
2732 return bd_size_type;
2733}
2734
2735static void hns3_init_ring_hw(struct hns3_enet_ring *ring)
2736{
2737 dma_addr_t dma = ring->desc_dma_addr;
2738 struct hnae3_queue *q = ring->tqp;
2739
2740 if (!HNAE3_IS_TX_RING(ring)) {
2741 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG,
2742 (u32)dma);
2743 hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,
2744 (u32)((dma >> 31) >> 1));
2745
2746 hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,
2747 hns3_buf_size2type(ring->buf_size));
2748 hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,
2749 ring->desc_num / 8 - 1);
2750
2751 } else {
2752 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,
2753 (u32)dma);
2754 hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,
2755 (u32)((dma >> 31) >> 1));
2756
2757 hns3_write_dev(q, HNS3_RING_TX_RING_BD_LEN_REG,
2758 hns3_buf_size2type(ring->buf_size));
2759 hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,
2760 ring->desc_num / 8 - 1);
2761 }
2762}
2763
5668abda 2764int hns3_init_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2765{
2766 struct hnae3_handle *h = priv->ae_handle;
2767 int ring_num = h->kinfo.num_tqps * 2;
2768 int i, j;
2769 int ret;
2770
2771 for (i = 0; i < ring_num; i++) {
2772 ret = hns3_alloc_ring_memory(priv->ring_data[i].ring);
2773 if (ret) {
2774 dev_err(priv->dev,
2775 "Alloc ring memory fail! ret=%d\n", ret);
2776 goto out_when_alloc_ring_memory;
2777 }
2778
2779 hns3_init_ring_hw(priv->ring_data[i].ring);
2780
2781 u64_stats_init(&priv->ring_data[i].ring->syncp);
2782 }
2783
2784 return 0;
2785
2786out_when_alloc_ring_memory:
2787 for (j = i - 1; j >= 0; j--)
ee83f776 2788 hns3_fini_ring(priv->ring_data[j].ring);
76ad4f0e
S		 2789
2790 return -ENOMEM;
2791}
2792
5668abda 2793int hns3_uninit_all_ring(struct hns3_nic_priv *priv)
76ad4f0e
S		 2794{
2795 struct hnae3_handle *h = priv->ae_handle;
2796 int i;
2797
2798 for (i = 0; i < h->kinfo.num_tqps; i++) {
2799 if (h->ae_algo->ops->reset_queue)
2800 h->ae_algo->ops->reset_queue(h, i);
2801
2802 hns3_fini_ring(priv->ring_data[i].ring);
99fdf6b1 2803 devm_kfree(priv->dev, priv->ring_data[i].ring);
76ad4f0e 2804 hns3_fini_ring(priv->ring_data[i + h->kinfo.num_tqps].ring);
99fdf6b1
PL		 2805 devm_kfree(priv->dev,
2806 priv->ring_data[i + h->kinfo.num_tqps].ring);
76ad4f0e 2807 }
99fdf6b1 2808 devm_kfree(priv->dev, priv->ring_data);
76ad4f0e
S		 2809
2810 return 0;
2811}
2812
2813/* Set mac addr if it is configured. or leave it to the AE driver */
2814static void hns3_init_mac_addr(struct net_device *netdev)
2815{
2816 struct hns3_nic_priv *priv = netdev_priv(netdev);
2817 struct hnae3_handle *h = priv->ae_handle;
2818 u8 mac_addr_temp[ETH_ALEN];
2819
2820 if (h->ae_algo->ops->get_mac_addr) {
2821 h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);
2822 ether_addr_copy(netdev->dev_addr, mac_addr_temp);
2823 }
2824
2825 /* Check if the MAC address is valid, if not get a random one */
2826 if (!is_valid_ether_addr(netdev->dev_addr)) {
2827 eth_hw_addr_random(netdev);
2828 dev_warn(priv->dev, "using random MAC address %pM\n",
2829 netdev->dev_addr);
76ad4f0e 2830 }
139e8792
L		 2831
2832 if (h->ae_algo->ops->set_mac_addr)
2833 h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr);
2834
76ad4f0e
S		 2835}
2836
2837static void hns3_nic_set_priv_ops(struct net_device *netdev)
2838{
2839 struct hns3_nic_priv *priv = netdev_priv(netdev);
2840
2841 if ((netdev->features & NETIF_F_TSO) ||
2842 (netdev->features & NETIF_F_TSO6)) {
2843 priv->ops.fill_desc = hns3_fill_desc_tso;
2844 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tso;
2845 } else {
2846 priv->ops.fill_desc = hns3_fill_desc;
2847 priv->ops.maybe_stop_tx = hns3_nic_maybe_stop_tx;
2848 }
2849}
2850
2851static int hns3_client_init(struct hnae3_handle *handle)
2852{
2853 struct pci_dev *pdev = handle->pdev;
2854 struct hns3_nic_priv *priv;
2855 struct net_device *netdev;
2856 int ret;
2857
2858 netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv),
2859 handle->kinfo.num_tqps);
2860 if (!netdev)
2861 return -ENOMEM;
2862
2863 priv = netdev_priv(netdev);
2864 priv->dev = &pdev->dev;
2865 priv->netdev = netdev;
2866 priv->ae_handle = handle;
f8fa222c
L		 2867 priv->last_reset_time = jiffies;
2868 priv->reset_level = HNAE3_FUNC_RESET;
2869 priv->tx_timeout_count = 0;
76ad4f0e
S		 2870
2871 handle->kinfo.netdev = netdev;
2872 handle->priv = (void *)priv;
2873
2874 hns3_init_mac_addr(netdev);
2875
2876 hns3_set_default_feature(netdev);
2877
2878 netdev->watchdog_timeo = HNS3_TX_TIMEOUT;
2879 netdev->priv_flags |= IFF_UNICAST_FLT;
2880 netdev->netdev_ops = &hns3_nic_netdev_ops;
2881 SET_NETDEV_DEV(netdev, &pdev->dev);
2882 hns3_ethtool_set_ops(netdev);
2883 hns3_nic_set_priv_ops(netdev);
2884
2885 /* Carrier off reporting is important to ethtool even BEFORE open */
2886 netif_carrier_off(netdev);
2887
2888 ret = hns3_get_ring_config(priv);
2889 if (ret) {
2890 ret = -ENOMEM;
2891 goto out_get_ring_cfg;
2892 }
2893
2894 ret = hns3_nic_init_vector_data(priv);
2895 if (ret) {
2896 ret = -ENOMEM;
2897 goto out_init_vector_data;
2898 }
2899
2900 ret = hns3_init_all_ring(priv);
2901 if (ret) {
2902 ret = -ENOMEM;
2903 goto out_init_ring_data;
2904 }
2905
2906 ret = register_netdev(netdev);
2907 if (ret) {
2908 dev_err(priv->dev, "probe register netdev fail!\n");
2909 goto out_reg_netdev_fail;
2910 }
2911
986743db
YL		 2912 hns3_dcbnl_setup(handle);
2913
a8e8b7ff
S		 2914 /* MTU range: (ETH_MIN_MTU(kernel default) - 9706) */
2915 netdev->max_mtu = HNS3_MAX_MTU - (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
2916
76ad4f0e
S		 2917 return ret;
2918
2919out_reg_netdev_fail:
2920out_init_ring_data:
2921 (void)hns3_nic_uninit_vector_data(priv);
2922 priv->ring_data = NULL;
2923out_init_vector_data:
2924out_get_ring_cfg:
2925 priv->ae_handle = NULL;
2926 free_netdev(netdev);
2927 return ret;
2928}
2929
2930static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)
2931{
2932 struct net_device *netdev = handle->kinfo.netdev;
2933 struct hns3_nic_priv *priv = netdev_priv(netdev);
2934 int ret;
2935
2936 if (netdev->reg_state != NETREG_UNINITIALIZED)
2937 unregister_netdev(netdev);
2938
2939 ret = hns3_nic_uninit_vector_data(priv);
2940 if (ret)
2941 netdev_err(netdev, "uninit vector error\n");
2942
2943 ret = hns3_uninit_all_ring(priv);
2944 if (ret)
2945 netdev_err(netdev, "uninit ring error\n");
2946
2947 priv->ring_data = NULL;
2948
2949 free_netdev(netdev);
2950}
2951
2952static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)
2953{
2954 struct net_device *netdev = handle->kinfo.netdev;
2955
2956 if (!netdev)
2957 return;
2958
2959 if (linkup) {
2960 netif_carrier_on(netdev);
2961 netif_tx_wake_all_queues(netdev);
2962 netdev_info(netdev, "link up\n");
2963 } else {
2964 netif_carrier_off(netdev);
2965 netif_tx_stop_all_queues(netdev);
2966 netdev_info(netdev, "link down\n");
2967 }
2968}
2969
9df8f79a
YL		 2970static int hns3_client_setup_tc(struct hnae3_handle *handle, u8 tc)
2971{
2972 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
2973 struct net_device *ndev = kinfo->netdev;
075cfdd6 2974 bool if_running;
9df8f79a
YL		 2975 int ret;
2976 u8 i;
2977
2978 if (tc > HNAE3_MAX_TC)
2979 return -EINVAL;
2980
2981 if (!ndev)
2982 return -ENODEV;
2983
075cfdd6
CIK		 2984 if_running = netif_running(ndev);
2985
9df8f79a
YL		 2986 ret = netdev_set_num_tc(ndev, tc);
2987 if (ret)
2988 return ret;
2989
2990 if (if_running) {
2991 (void)hns3_nic_net_stop(ndev);
2992 msleep(100);
2993 }
2994
2995 ret = (kinfo->dcb_ops && kinfo->dcb_ops->map_update) ?
2996 kinfo->dcb_ops->map_update(handle) : -EOPNOTSUPP;
2997 if (ret)
2998 goto err_out;
2999
3000 if (tc <= 1) {
3001 netdev_reset_tc(ndev);
3002 goto out;
3003 }
3004
3005 for (i = 0; i < HNAE3_MAX_TC; i++) {
3006 struct hnae3_tc_info *tc_info = &kinfo->tc_info[i];
3007
3008 if (tc_info->enable)
3009 netdev_set_tc_queue(ndev,
3010 tc_info->tc,
3011 tc_info->tqp_count,
3012 tc_info->tqp_offset);
3013 }
3014
3015 for (i = 0; i < HNAE3_MAX_USER_PRIO; i++) {
3016 netdev_set_prio_tc_map(ndev, i,
3017 kinfo->prio_tc[i]);
3018 }
3019
3020out:
3021 ret = hns3_nic_set_real_num_queue(ndev);
3022
3023err_out:
3024 if (if_running)
3025 (void)hns3_nic_net_open(ndev);
3026
3027 return ret;
3028}
3029
bb6b94a8
L		 3030static void hns3_recover_hw_addr(struct net_device *ndev)
3031{
3032 struct netdev_hw_addr_list *list;
3033 struct netdev_hw_addr *ha, *tmp;
3034
3035 /* go through and sync uc_addr entries to the device */
3036 list = &ndev->uc;
3037 list_for_each_entry_safe(ha, tmp, &list->list, list)
3038 hns3_nic_uc_sync(ndev, ha->addr);
3039
3040 /* go through and sync mc_addr entries to the device */
3041 list = &ndev->mc;
3042 list_for_each_entry_safe(ha, tmp, &list->list, list)
3043 hns3_nic_mc_sync(ndev, ha->addr);
3044}
3045
3046static void hns3_drop_skb_data(struct hns3_enet_ring *ring, struct sk_buff *skb)
3047{
3048 dev_kfree_skb_any(skb);
3049}
3050
3051static void hns3_clear_all_ring(struct hnae3_handle *h)
3052{
3053 struct net_device *ndev = h->kinfo.netdev;
3054 struct hns3_nic_priv *priv = netdev_priv(ndev);
3055 u32 i;
3056
3057 for (i = 0; i < h->kinfo.num_tqps; i++) {
3058 struct netdev_queue *dev_queue;
3059 struct hns3_enet_ring *ring;
3060
3061 ring = priv->ring_data[i].ring;
3062 hns3_clean_tx_ring(ring, ring->desc_num);
3063 dev_queue = netdev_get_tx_queue(ndev,
3064 priv->ring_data[i].queue_index);
3065 netdev_tx_reset_queue(dev_queue);
3066
3067 ring = priv->ring_data[i + h->kinfo.num_tqps].ring;
3068 hns3_clean_rx_ring(ring, ring->desc_num, hns3_drop_skb_data);
3069 }
3070}
3071
3072static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)
3073{
3074 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3075 struct net_device *ndev = kinfo->netdev;
3076
3077 if (!netif_running(ndev))
3078 return -EIO;
3079
3080 return hns3_nic_net_stop(ndev);
3081}
3082
3083static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)
3084{
3085 struct hnae3_knic_private_info *kinfo = &handle->kinfo;
3086 struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);
3087 int ret = 0;
3088
3089 if (netif_running(kinfo->netdev)) {
3090 ret = hns3_nic_net_up(kinfo->netdev);
3091 if (ret) {
3092 netdev_err(kinfo->netdev,
3093 "hns net up fail, ret=%d!\n", ret);
3094 return ret;
3095 }
3096
3097 priv->last_reset_time = jiffies;
3098 }
3099
3100 return ret;
3101}
3102
3103static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)
3104{
3105 struct net_device *netdev = handle->kinfo.netdev;
3106 struct hns3_nic_priv *priv = netdev_priv(netdev);
3107 int ret;
3108
3109 priv->reset_level = 1;
3110 hns3_init_mac_addr(netdev);
3111 hns3_nic_set_rx_mode(netdev);
3112 hns3_recover_hw_addr(netdev);
3113
3114 /* Carrier off reporting is important to ethtool even BEFORE open */
3115 netif_carrier_off(netdev);
3116
3117 ret = hns3_get_ring_config(priv);
3118 if (ret)
3119 return ret;
3120
3121 ret = hns3_nic_init_vector_data(priv);
3122 if (ret)
3123 return ret;
3124
3125 ret = hns3_init_all_ring(priv);
3126 if (ret) {
3127 hns3_nic_uninit_vector_data(priv);
3128 priv->ring_data = NULL;
3129 }
3130
3131 return ret;
3132}
3133
3134static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)
3135{
3136 struct net_device *netdev = handle->kinfo.netdev;
3137 struct hns3_nic_priv *priv = netdev_priv(netdev);
3138 int ret;
3139
3140 hns3_clear_all_ring(handle);
3141
3142 ret = hns3_nic_uninit_vector_data(priv);
3143 if (ret) {
3144 netdev_err(netdev, "uninit vector error\n");
3145 return ret;
3146 }
3147
3148 ret = hns3_uninit_all_ring(priv);
3149 if (ret)
3150 netdev_err(netdev, "uninit ring error\n");
3151
3152 priv->ring_data = NULL;
3153
3154 return ret;
3155}
3156
3157static int hns3_reset_notify(struct hnae3_handle *handle,
3158 enum hnae3_reset_notify_type type)
3159{
3160 int ret = 0;
3161
3162 switch (type) {
3163 case HNAE3_UP_CLIENT:
3164 ret = hns3_reset_notify_up_enet(handle);
3165 break;
3166 case HNAE3_DOWN_CLIENT:
3167 ret = hns3_reset_notify_down_enet(handle);
3168 break;
3169 case HNAE3_INIT_CLIENT:
3170 ret = hns3_reset_notify_init_enet(handle);
3171 break;
3172 case HNAE3_UNINIT_CLIENT:
3173 ret = hns3_reset_notify_uninit_enet(handle);
3174 break;
3175 default:
3176 break;
3177 }
3178
3179 return ret;
3180}
3181
09f2af64
PL		 3182static u16 hns3_get_max_available_channels(struct net_device *netdev)
3183{
3184 struct hnae3_handle *h = hns3_get_handle(netdev);
3185 u16 free_tqps, max_rss_size, max_tqps;
3186
3187 h->ae_algo->ops->get_tqps_and_rss_info(h, &free_tqps, &max_rss_size);
3188 max_tqps = h->kinfo.num_tc * max_rss_size;
3189
3190 return min_t(u16, max_tqps, (free_tqps + h->kinfo.num_tqps));
3191}
3192
3193static int hns3_modify_tqp_num(struct net_device *netdev, u16 new_tqp_num)
3194{
3195 struct hns3_nic_priv *priv = netdev_priv(netdev);
3196 struct hnae3_handle *h = hns3_get_handle(netdev);
3197 int ret;
3198
3199 ret = h->ae_algo->ops->set_channels(h, new_tqp_num);
3200 if (ret)
3201 return ret;
3202
3203 ret = hns3_get_ring_config(priv);
3204 if (ret)
3205 return ret;
3206
3207 ret = hns3_nic_init_vector_data(priv);
3208 if (ret)
3209 goto err_uninit_vector;
3210
3211 ret = hns3_init_all_ring(priv);
3212 if (ret)
3213 goto err_put_ring;
3214
3215 return 0;
3216
3217err_put_ring:
3218 hns3_put_ring_config(priv);
3219err_uninit_vector:
3220 hns3_nic_uninit_vector_data(priv);
3221 return ret;
3222}
3223
3224static int hns3_adjust_tqps_num(u8 num_tc, u32 new_tqp_num)
3225{
3226 return (new_tqp_num / num_tc) * num_tc;
3227}
3228
3229int hns3_set_channels(struct net_device *netdev,
3230 struct ethtool_channels *ch)
3231{
3232 struct hns3_nic_priv *priv = netdev_priv(netdev);
3233 struct hnae3_handle *h = hns3_get_handle(netdev);
3234 struct hnae3_knic_private_info *kinfo = &h->kinfo;
3235 bool if_running = netif_running(netdev);
3236 u32 new_tqp_num = ch->combined_count;
3237 u16 org_tqp_num;
3238 int ret;
3239
3240 if (ch->rx_count || ch->tx_count)
3241 return -EINVAL;
3242
3243 if (new_tqp_num > hns3_get_max_available_channels(netdev) ||
3244 new_tqp_num < kinfo->num_tc) {
3245 dev_err(&netdev->dev,
3246 "Change tqps fail, the tqp range is from %d to %d",
3247 kinfo->num_tc,
3248 hns3_get_max_available_channels(netdev));
3249 return -EINVAL;
3250 }
3251
3252 new_tqp_num = hns3_adjust_tqps_num(kinfo->num_tc, new_tqp_num);
3253 if (kinfo->num_tqps == new_tqp_num)
3254 return 0;
3255
3256 if (if_running)
3257 dev_close(netdev);
3258
3259 hns3_clear_all_ring(h);
3260
3261 ret = hns3_nic_uninit_vector_data(priv);
3262 if (ret) {
3263 dev_err(&netdev->dev,
3264 "Unbind vector with tqp fail, nothing is changed");
3265 goto open_netdev;
3266 }
3267
3268 hns3_uninit_all_ring(priv);
3269
3270 org_tqp_num = h->kinfo.num_tqps;
3271 ret = hns3_modify_tqp_num(netdev, new_tqp_num);
3272 if (ret) {
3273 ret = hns3_modify_tqp_num(netdev, org_tqp_num);
3274 if (ret) {
3275 /* If revert to old tqp failed, fatal error occurred */
3276 dev_err(&netdev->dev,
3277 "Revert to old tqp num fail, ret=%d", ret);
3278 return ret;
3279 }
3280 dev_info(&netdev->dev,
3281 "Change tqp num fail, Revert to old tqp num");
3282 }
3283
3284open_netdev:
3285 if (if_running)
3286 dev_open(netdev);
3287
3288 return ret;
3289}
3290
1db9b1bf 3291static const struct hnae3_client_ops client_ops = {
76ad4f0e
S		 3292 .init_instance = hns3_client_init,
3293 .uninit_instance = hns3_client_uninit,
3294 .link_status_change = hns3_link_status_change,
9df8f79a 3295 .setup_tc = hns3_client_setup_tc,
bb6b94a8 3296 .reset_notify = hns3_reset_notify,
76ad4f0e
S		 3297};
3298
3299/* hns3_init_module - Driver registration routine
3300 * hns3_init_module is the first routine called when the driver is
3301 * loaded. All it does is register with the PCI subsystem.
3302 */
3303static int __init hns3_init_module(void)
3304{
3305 int ret;
3306
3307 pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);
3308 pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);
3309
3310 client.type = HNAE3_CLIENT_KNIC;
3311 snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH - 1, "%s",
3312 hns3_driver_name);
3313
3314 client.ops = &client_ops;
3315
3316 ret = hnae3_register_client(&client);
3317 if (ret)
3318 return ret;
3319
3320 ret = pci_register_driver(&hns3_driver);
3321 if (ret)
3322 hnae3_unregister_client(&client);
3323
3324 return ret;
3325}
3326module_init(hns3_init_module);
3327
3328/* hns3_exit_module - Driver exit cleanup routine
3329 * hns3_exit_module is called just before the driver is removed
3330 * from memory.
3331 */
3332static void __exit hns3_exit_module(void)
3333{
3334 pci_unregister_driver(&hns3_driver);
3335 hnae3_unregister_client(&client);
3336}
3337module_exit(hns3_exit_module);
3338
3339MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");
3340MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
3341MODULE_LICENSE("GPL");
3342MODULE_ALIAS("pci:hns-nic");
Linux 6.x block layer and io_uring tree(s)