drivers/net/ethernet/hisilicon/hns/hns_enet.c

   1 /*
   2  * Copyright (c) 2014-2015 Hisilicon Limited.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  */
   9
  10 #include <linux/clk.h>
  11 #include <linux/cpumask.h>
  12 #include <linux/etherdevice.h>
  13 #include <linux/if_vlan.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/io.h>
  16 #include <linux/ip.h>
  17 #include <linux/ipv6.h>
  18 #include <linux/module.h>
  19 #include <linux/phy.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/skbuff.h>
  22
  23 #include "hnae.h"
  24 #include "hns_enet.h"
  25
  26 #define NIC_MAX_Q_PER_VF 16
  27 #define HNS_NIC_TX_TIMEOUT (5 * HZ)
  28
  29 #define SERVICE_TIMER_HZ (1 * HZ)
  30
  31 #define NIC_TX_CLEAN_MAX_NUM 256
  32 #define NIC_RX_CLEAN_MAX_NUM 64
  33
  34 #define RCB_IRQ_NOT_INITED 0
  35 #define RCB_IRQ_INITED 1
  36 #define HNS_BUFFER_SIZE_2048 2048
  37
  38 #define BD_MAX_SEND_SIZE 8191
  39 #define SKB_TMP_LEN(SKB) \
  40         (((SKB)->transport_header - (SKB)->mac_header) + tcp_hdrlen(SKB))
  41
  42 static void fill_v2_desc(struct hnae_ring *ring, void *priv,
  43                          int size, dma_addr_t dma, int frag_end,
  44                          int buf_num, enum hns_desc_type type, int mtu)
  45 {
  46         struct hnae_desc *desc = &ring->desc[ring->next_to_use];
  47         struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
  48         struct iphdr *iphdr;
  49         struct ipv6hdr *ipv6hdr;
  50         struct sk_buff *skb;
  51         int skb_tmp_len;
  52         __be16 protocol;
  53         u8 bn_pid = 0;
  54         u8 rrcfv = 0;
  55         u8 ip_offset = 0;
  56         u8 tvsvsn = 0;
  57         u16 mss = 0;
  58         u8 l4_len = 0;
  59         u16 paylen = 0;
  60
  61         desc_cb->priv = priv;
  62         desc_cb->length = size;
  63         desc_cb->dma = dma;
  64         desc_cb->type = type;
  65
  66         desc->addr = cpu_to_le64(dma);
  67         desc->tx.send_size = cpu_to_le16((u16)size);
  68
  69         /* config bd buffer end */
  70         hnae_set_bit(rrcfv, HNSV2_TXD_VLD_B, 1);
  71         hnae_set_field(bn_pid, HNSV2_TXD_BUFNUM_M, 0, buf_num - 1);
  72
  73         /* fill port_id in the tx bd for sending management pkts */
  74         hnae_set_field(bn_pid, HNSV2_TXD_PORTID_M,
  75                        HNSV2_TXD_PORTID_S, ring->q->handle->dport_id);
  76
  77         if (type == DESC_TYPE_SKB) {
  78                 skb = (struct sk_buff *)priv;
  79
  80                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
  81                         skb_reset_mac_len(skb);
  82                         protocol = skb->protocol;
  83                         ip_offset = ETH_HLEN;
  84
  85                         if (protocol == htons(ETH_P_8021Q)) {
  86                                 ip_offset += VLAN_HLEN;
  87                                 protocol = vlan_get_protocol(skb);
  88                                 skb->protocol = protocol;
  89                         }
  90
  91                         if (skb->protocol == htons(ETH_P_IP)) {
  92                                 iphdr = ip_hdr(skb);
  93                                 hnae_set_bit(rrcfv, HNSV2_TXD_L3CS_B, 1);
  94                                 hnae_set_bit(rrcfv, HNSV2_TXD_L4CS_B, 1);
  95
  96                                 /* check for tcp/udp header */
  97                                 if (iphdr->protocol == IPPROTO_TCP) {
  98                                         hnae_set_bit(tvsvsn,
  99                                                      HNSV2_TXD_TSE_B, 1);
 100                                         skb_tmp_len = SKB_TMP_LEN(skb);
 101                                         l4_len = tcp_hdrlen(skb);
 102                                         mss = mtu - skb_tmp_len - ETH_FCS_LEN;
 103                                         paylen = skb->len - skb_tmp_len;
 104                                 }
 105                         } else if (skb->protocol == htons(ETH_P_IPV6)) {
 106                                 hnae_set_bit(tvsvsn, HNSV2_TXD_IPV6_B, 1);
 107                                 ipv6hdr = ipv6_hdr(skb);
 108                                 hnae_set_bit(rrcfv, HNSV2_TXD_L4CS_B, 1);
 109
 110                                 /* check for tcp/udp header */
 111                                 if (ipv6hdr->nexthdr == IPPROTO_TCP) {
 112                                         hnae_set_bit(tvsvsn,
 113                                                      HNSV2_TXD_TSE_B, 1);
 114                                         skb_tmp_len = SKB_TMP_LEN(skb);
 115                                         l4_len = tcp_hdrlen(skb);
 116                                         mss = mtu - skb_tmp_len - ETH_FCS_LEN;
 117                                         paylen = skb->len - skb_tmp_len;
 118                                 }
 119                         }
 120                         desc->tx.ip_offset = ip_offset;
 121                         desc->tx.tse_vlan_snap_v6_sctp_nth = tvsvsn;
 122                         desc->tx.mss = cpu_to_le16(mss);
 123                         desc->tx.l4_len = l4_len;
 124                         desc->tx.paylen = cpu_to_le16(paylen);
 125                 }
 126         }
 127
 128         hnae_set_bit(rrcfv, HNSV2_TXD_FE_B, frag_end);
 129
 130         desc->tx.bn_pid = bn_pid;
 131         desc->tx.ra_ri_cs_fe_vld = rrcfv;
 132
 133         ring_ptr_move_fw(ring, next_to_use);
 134 }
 135
 136 static void fill_desc(struct hnae_ring *ring, void *priv,
 137                       int size, dma_addr_t dma, int frag_end,
 138                       int buf_num, enum hns_desc_type type, int mtu)
 139 {
 140         struct hnae_desc *desc = &ring->desc[ring->next_to_use];
 141         struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];
 142         struct sk_buff *skb;
 143         __be16 protocol;
 144         u32 ip_offset;
 145         u32 asid_bufnum_pid = 0;
 146         u32 flag_ipoffset = 0;
 147
 148         desc_cb->priv = priv;
 149         desc_cb->length = size;
 150         desc_cb->dma = dma;
 151         desc_cb->type = type;
 152
 153         desc->addr = cpu_to_le64(dma);
 154         desc->tx.send_size = cpu_to_le16((u16)size);
 155
 156         /*config bd buffer end */
 157         flag_ipoffset |= 1 << HNS_TXD_VLD_B;
 158
 159         asid_bufnum_pid |= buf_num << HNS_TXD_BUFNUM_S;
 160
 161         if (type == DESC_TYPE_SKB) {
 162                 skb = (struct sk_buff *)priv;
 163
 164                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 165                         protocol = skb->protocol;
 166                         ip_offset = ETH_HLEN;
 167
 168                         /*if it is a SW VLAN check the next protocol*/
 169                         if (protocol == htons(ETH_P_8021Q)) {
 170                                 ip_offset += VLAN_HLEN;
 171                                 protocol = vlan_get_protocol(skb);
 172                                 skb->protocol = protocol;
 173                         }
 174
 175                         if (skb->protocol == htons(ETH_P_IP)) {
 176                                 flag_ipoffset |= 1 << HNS_TXD_L3CS_B;
 177                                 /* check for tcp/udp header */
 178                                 flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
 179
 180                         } else if (skb->protocol == htons(ETH_P_IPV6)) {
 181                                 /* ipv6 has not l3 cs, check for L4 header */
 182                                 flag_ipoffset |= 1 << HNS_TXD_L4CS_B;
 183                         }
 184
 185                         flag_ipoffset |= ip_offset << HNS_TXD_IPOFFSET_S;
 186                 }
 187         }
 188
 189         flag_ipoffset |= frag_end << HNS_TXD_FE_B;
 190
 191         desc->tx.asid_bufnum_pid = cpu_to_le16(asid_bufnum_pid);
 192         desc->tx.flag_ipoffset = cpu_to_le32(flag_ipoffset);
 193
 194         ring_ptr_move_fw(ring, next_to_use);
 195 }
 196
 197 static void unfill_desc(struct hnae_ring *ring)
 198 {
 199         ring_ptr_move_bw(ring, next_to_use);
 200 }
 201
 202 static int hns_nic_maybe_stop_tx(
 203         struct sk_buff **out_skb, int *bnum, struct hnae_ring *ring)
 204 {
 205         struct sk_buff *skb = *out_skb;
 206         struct sk_buff *new_skb = NULL;
 207         int buf_num;
 208
 209         /* no. of segments (plus a header) */
 210         buf_num = skb_shinfo(skb)->nr_frags + 1;
 211
 212         if (unlikely(buf_num > ring->max_desc_num_per_pkt)) {
 213                 if (ring_space(ring) < 1)
 214                         return -EBUSY;
 215
 216                 new_skb = skb_copy(skb, GFP_ATOMIC);
 217                 if (!new_skb)
 218                         return -ENOMEM;
 219
 220                 dev_kfree_skb_any(skb);
 221                 *out_skb = new_skb;
 222                 buf_num = 1;
 223         } else if (buf_num > ring_space(ring)) {
 224                 return -EBUSY;
 225         }
 226
 227         *bnum = buf_num;
 228         return 0;
 229 }
 230
 231 static int hns_nic_maybe_stop_tso(
 232         struct sk_buff **out_skb, int *bnum, struct hnae_ring *ring)
 233 {
 234         int i;
 235         int size;
 236         int buf_num;
 237         int frag_num;
 238         struct sk_buff *skb = *out_skb;
 239         struct sk_buff *new_skb = NULL;
 240         struct skb_frag_struct *frag;
 241
 242         size = skb_headlen(skb);
 243         buf_num = (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 244
 245         frag_num = skb_shinfo(skb)->nr_frags;
 246         for (i = 0; i < frag_num; i++) {
 247                 frag = &skb_shinfo(skb)->frags[i];
 248                 size = skb_frag_size(frag);
 249                 buf_num += (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 250         }
 251
 252         if (unlikely(buf_num > ring->max_desc_num_per_pkt)) {
 253                 buf_num = (skb->len + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 254                 if (ring_space(ring) < buf_num)
 255                         return -EBUSY;
 256                 /* manual split the send packet */
 257                 new_skb = skb_copy(skb, GFP_ATOMIC);
 258                 if (!new_skb)
 259                         return -ENOMEM;
 260                 dev_kfree_skb_any(skb);
 261                 *out_skb = new_skb;
 262
 263         } else if (ring_space(ring) < buf_num) {
 264                 return -EBUSY;
 265         }
 266
 267         *bnum = buf_num;
 268         return 0;
 269 }
 270
 271 static void fill_tso_desc(struct hnae_ring *ring, void *priv,
 272                           int size, dma_addr_t dma, int frag_end,
 273                           int buf_num, enum hns_desc_type type, int mtu)
 274 {
 275         int frag_buf_num;
 276         int sizeoflast;
 277         int k;
 278
 279         frag_buf_num = (size + BD_MAX_SEND_SIZE - 1) / BD_MAX_SEND_SIZE;
 280         sizeoflast = size % BD_MAX_SEND_SIZE;
 281         sizeoflast = sizeoflast ? sizeoflast : BD_MAX_SEND_SIZE;
 282
 283         /* when the frag size is bigger than hardware, split this frag */
 284         for (k = 0; k < frag_buf_num; k++)
 285                 fill_v2_desc(ring, priv,
 286                              (k == frag_buf_num - 1) ?
 287                                         sizeoflast : BD_MAX_SEND_SIZE,
 288                              dma + BD_MAX_SEND_SIZE * k,
 289                              frag_end && (k == frag_buf_num - 1) ? 1 : 0,
 290                              buf_num,
 291                              (type == DESC_TYPE_SKB && !k) ?
 292                                         DESC_TYPE_SKB : DESC_TYPE_PAGE,
 293                              mtu);
 294 }
 295
 296 int hns_nic_net_xmit_hw(struct net_device *ndev,
 297                         struct sk_buff *skb,
 298                         struct hns_nic_ring_data *ring_data)
 299 {
 300         struct hns_nic_priv *priv = netdev_priv(ndev);
 301         struct device *dev = priv->dev;
 302         struct hnae_ring *ring = ring_data->ring;
 303         struct netdev_queue *dev_queue;
 304         struct skb_frag_struct *frag;
 305         int buf_num;
 306         int seg_num;
 307         dma_addr_t dma;
 308         int size, next_to_use;
 309         int i;
 310
 311         switch (priv->ops.maybe_stop_tx(&skb, &buf_num, ring)) {
 312         case -EBUSY:
 313                 ring->stats.tx_busy++;
 314                 goto out_net_tx_busy;
 315         case -ENOMEM:
 316                 ring->stats.sw_err_cnt++;
 317                 netdev_err(ndev, "no memory to xmit!\n");
 318                 goto out_err_tx_ok;
 319         default:
 320                 break;
 321         }
 322
 323         /* no. of segments (plus a header) */
 324         seg_num = skb_shinfo(skb)->nr_frags + 1;
 325         next_to_use = ring->next_to_use;
 326
 327         /* fill the first part */
 328         size = skb_headlen(skb);
 329         dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);
 330         if (dma_mapping_error(dev, dma)) {
 331                 netdev_err(ndev, "TX head DMA map failed\n");
 332                 ring->stats.sw_err_cnt++;
 333                 goto out_err_tx_ok;
 334         }
 335         priv->ops.fill_desc(ring, skb, size, dma, seg_num == 1 ? 1 : 0,
 336                             buf_num, DESC_TYPE_SKB, ndev->mtu);
 337
 338         /* fill the fragments */
 339         for (i = 1; i < seg_num; i++) {
 340                 frag = &skb_shinfo(skb)->frags[i - 1];
 341                 size = skb_frag_size(frag);
 342                 dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);
 343                 if (dma_mapping_error(dev, dma)) {
 344                         netdev_err(ndev, "TX frag(%d) DMA map failed\n", i);
 345                         ring->stats.sw_err_cnt++;
 346                         goto out_map_frag_fail;
 347                 }
 348                 priv->ops.fill_desc(ring, skb_frag_page(frag), size, dma,
 349                                     seg_num - 1 == i ? 1 : 0, buf_num,
 350                                     DESC_TYPE_PAGE, ndev->mtu);
 351         }
 352
 353         /*complete translate all packets*/
 354         dev_queue = netdev_get_tx_queue(ndev, skb->queue_mapping);
 355         netdev_tx_sent_queue(dev_queue, skb->len);
 356
 357         wmb(); /* commit all data before submit */
 358         assert(skb->queue_mapping < priv->ae_handle->q_num);
 359         hnae_queue_xmit(priv->ae_handle->qs[skb->queue_mapping], buf_num);
 360         ring->stats.tx_pkts++;
 361         ring->stats.tx_bytes += skb->len;
 362
 363         return NETDEV_TX_OK;
 364
 365 out_map_frag_fail:
 366
 367         while (ring->next_to_use != next_to_use) {
 368                 unfill_desc(ring);
 369                 if (ring->next_to_use != next_to_use)
 370                         dma_unmap_page(dev,
 371                                        ring->desc_cb[ring->next_to_use].dma,
 372                                        ring->desc_cb[ring->next_to_use].length,
 373                                        DMA_TO_DEVICE);
 374                 else
 375                         dma_unmap_single(dev,
 376                                          ring->desc_cb[next_to_use].dma,
 377                                          ring->desc_cb[next_to_use].length,
 378                                          DMA_TO_DEVICE);
 379         }
 380
 381 out_err_tx_ok:
 382
 383         dev_kfree_skb_any(skb);
 384         return NETDEV_TX_OK;
 385
 386 out_net_tx_busy:
 387
 388         netif_stop_subqueue(ndev, skb->queue_mapping);
 389
 390         /* Herbert's original patch had:
 391          *  smp_mb__after_netif_stop_queue();
 392          * but since that doesn't exist yet, just open code it.
 393          */
 394         smp_mb();
 395         return NETDEV_TX_BUSY;
 396 }
 397
 398 /**
 399  * hns_nic_get_headlen - determine size of header for RSC/LRO/GRO/FCOE
 400  * @data: pointer to the start of the headers
 401  * @max: total length of section to find headers in
 402  *
 403  * This function is meant to determine the length of headers that will
 404  * be recognized by hardware for LRO, GRO, and RSC offloads.  The main
 405  * motivation of doing this is to only perform one pull for IPv4 TCP
 406  * packets so that we can do basic things like calculating the gso_size
 407  * based on the average data per packet.
 408  **/
 409 static unsigned int hns_nic_get_headlen(unsigned char *data, u32 flag,
 410                                         unsigned int max_size)
 411 {
 412         unsigned char *network;
 413         u8 hlen;
 414
 415         /* this should never happen, but better safe than sorry */
 416         if (max_size < ETH_HLEN)
 417                 return max_size;
 418
 419         /* initialize network frame pointer */
 420         network = data;
 421
 422         /* set first protocol and move network header forward */
 423         network += ETH_HLEN;
 424
 425         /* handle any vlan tag if present */
 426         if (hnae_get_field(flag, HNS_RXD_VLAN_M, HNS_RXD_VLAN_S)
 427                 == HNS_RX_FLAG_VLAN_PRESENT) {
 428                 if ((typeof(max_size))(network - data) > (max_size - VLAN_HLEN))
 429                         return max_size;
 430
 431                 network += VLAN_HLEN;
 432         }
 433
 434         /* handle L3 protocols */
 435         if (hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S)
 436                 == HNS_RX_FLAG_L3ID_IPV4) {
 437                 if ((typeof(max_size))(network - data) >
 438                     (max_size - sizeof(struct iphdr)))
 439                         return max_size;
 440
 441                 /* access ihl as a u8 to avoid unaligned access on ia64 */
 442                 hlen = (network[0] & 0x0F) << 2;
 443
 444                 /* verify hlen meets minimum size requirements */
 445                 if (hlen < sizeof(struct iphdr))
 446                         return network - data;
 447
 448                 /* record next protocol if header is present */
 449         } else if (hnae_get_field(flag, HNS_RXD_L3ID_M, HNS_RXD_L3ID_S)
 450                 == HNS_RX_FLAG_L3ID_IPV6) {
 451                 if ((typeof(max_size))(network - data) >
 452                     (max_size - sizeof(struct ipv6hdr)))
 453                         return max_size;
 454
 455                 /* record next protocol */
 456                 hlen = sizeof(struct ipv6hdr);
 457         } else {
 458                 return network - data;
 459         }
 460
 461         /* relocate pointer to start of L4 header */
 462         network += hlen;
 463
 464         /* finally sort out TCP/UDP */
 465         if (hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S)
 466                 == HNS_RX_FLAG_L4ID_TCP) {
 467                 if ((typeof(max_size))(network - data) >
 468                     (max_size - sizeof(struct tcphdr)))
 469                         return max_size;
 470
 471                 /* access doff as a u8 to avoid unaligned access on ia64 */
 472                 hlen = (network[12] & 0xF0) >> 2;
 473
 474                 /* verify hlen meets minimum size requirements */
 475                 if (hlen < sizeof(struct tcphdr))
 476                         return network - data;
 477
 478                 network += hlen;
 479         } else if (hnae_get_field(flag, HNS_RXD_L4ID_M, HNS_RXD_L4ID_S)
 480                 == HNS_RX_FLAG_L4ID_UDP) {
 481                 if ((typeof(max_size))(network - data) >
 482                     (max_size - sizeof(struct udphdr)))
 483                         return max_size;
 484
 485                 network += sizeof(struct udphdr);
 486         }
 487
 488         /* If everything has gone correctly network should be the
 489          * data section of the packet and will be the end of the header.
 490          * If not then it probably represents the end of the last recognized
 491          * header.
 492          */
 493         if ((typeof(max_size))(network - data) < max_size)
 494                 return network - data;
 495         else
 496                 return max_size;
 497 }
 498
 499 static void hns_nic_reuse_page(struct sk_buff *skb, int i,
 500                                struct hnae_ring *ring, int pull_len,
 501                                struct hnae_desc_cb *desc_cb)
 502 {
 503         struct hnae_desc *desc;
 504         int truesize, size;
 505         int last_offset;
 506         bool twobufs;
 507
 508         twobufs = ((PAGE_SIZE < 8192) && hnae_buf_size(ring) == HNS_BUFFER_SIZE_2048);
 509
 510         desc = &ring->desc[ring->next_to_clean];
 511         size = le16_to_cpu(desc->rx.size);
 512
 513         if (twobufs) {
 514                 truesize = hnae_buf_size(ring);
 515         } else {
 516                 truesize = ALIGN(size, L1_CACHE_BYTES);
 517                 last_offset = hnae_page_size(ring) - hnae_buf_size(ring);
 518         }
 519
 520         skb_add_rx_frag(skb, i, desc_cb->priv, desc_cb->page_offset + pull_len,
 521                         size - pull_len, truesize - pull_len);
 522
 523          /* avoid re-using remote pages,flag default unreuse */
 524         if (unlikely(page_to_nid(desc_cb->priv) != numa_node_id()))
 525                 return;
 526
 527         if (twobufs) {
 528                 /* if we are only owner of page we can reuse it */
 529                 if (likely(page_count(desc_cb->priv) == 1)) {
 530                         /* flip page offset to other buffer */
 531                         desc_cb->page_offset ^= truesize;
 532
 533                         desc_cb->reuse_flag = 1;
 534                         /* bump ref count on page before it is given*/
 535                         get_page(desc_cb->priv);
 536                 }
 537                 return;
 538         }
 539
 540         /* move offset up to the next cache line */
 541         desc_cb->page_offset += truesize;
 542
 543         if (desc_cb->page_offset <= last_offset) {
 544                 desc_cb->reuse_flag = 1;
 545                 /* bump ref count on page before it is given*/
 546                 get_page(desc_cb->priv);
 547         }
 548 }
 549
 550 static void get_v2rx_desc_bnum(u32 bnum_flag, int *out_bnum)
 551 {
 552         *out_bnum = hnae_get_field(bnum_flag,
 553                                    HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S) + 1;
 554 }
 555
 556 static void get_rx_desc_bnum(u32 bnum_flag, int *out_bnum)
 557 {
 558         *out_bnum = hnae_get_field(bnum_flag,
 559                                    HNS_RXD_BUFNUM_M, HNS_RXD_BUFNUM_S);
 560 }
 561
 562 static int hns_nic_poll_rx_skb(struct hns_nic_ring_data *ring_data,
 563                                struct sk_buff **out_skb, int *out_bnum)
 564 {
 565         struct hnae_ring *ring = ring_data->ring;
 566         struct net_device *ndev = ring_data->napi.dev;
 567         struct hns_nic_priv *priv = netdev_priv(ndev);
 568         struct sk_buff *skb;
 569         struct hnae_desc *desc;
 570         struct hnae_desc_cb *desc_cb;
 571         struct ethhdr *eh;
 572         unsigned char *va;
 573         int bnum, length, i;
 574         int pull_len;
 575         u32 bnum_flag;
 576
 577         desc = &ring->desc[ring->next_to_clean];
 578         desc_cb = &ring->desc_cb[ring->next_to_clean];
 579
 580         prefetch(desc);
 581
 582         va = (unsigned char *)desc_cb->buf + desc_cb->page_offset;
 583
 584         /* prefetch first cache line of first page */
 585         prefetch(va);
 586 #if L1_CACHE_BYTES < 128
 587         prefetch(va + L1_CACHE_BYTES);
 588 #endif
 589
 590         skb = *out_skb = napi_alloc_skb(&ring_data->napi,
 591                                         HNS_RX_HEAD_SIZE);
 592         if (unlikely(!skb)) {
 593                 netdev_err(ndev, "alloc rx skb fail\n");
 594                 ring->stats.sw_err_cnt++;
 595                 return -ENOMEM;
 596         }
 597
 598         prefetchw(skb->data);
 599         length = le16_to_cpu(desc->rx.pkt_len);
 600         bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
 601         priv->ops.get_rxd_bnum(bnum_flag, &bnum);
 602         *out_bnum = bnum;
 603
 604         if (length <= HNS_RX_HEAD_SIZE) {
 605                 memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));
 606
 607                 /* we can reuse buffer as-is, just make sure it is local */
 608                 if (likely(page_to_nid(desc_cb->priv) == numa_node_id()))
 609                         desc_cb->reuse_flag = 1;
 610                 else /* this page cannot be reused so discard it */
 611                         put_page(desc_cb->priv);
 612
 613                 ring_ptr_move_fw(ring, next_to_clean);
 614
 615                 if (unlikely(bnum != 1)) { /* check err*/
 616                         *out_bnum = 1;
 617                         goto out_bnum_err;
 618                 }
 619         } else {
 620                 ring->stats.seg_pkt_cnt++;
 621
 622                 pull_len = hns_nic_get_headlen(va, bnum_flag, HNS_RX_HEAD_SIZE);
 623                 memcpy(__skb_put(skb, pull_len), va,
 624                        ALIGN(pull_len, sizeof(long)));
 625
 626                 hns_nic_reuse_page(skb, 0, ring, pull_len, desc_cb);
 627                 ring_ptr_move_fw(ring, next_to_clean);
 628
 629                 if (unlikely(bnum >= (int)MAX_SKB_FRAGS)) { /* check err*/
 630                         *out_bnum = 1;
 631                         goto out_bnum_err;
 632                 }
 633                 for (i = 1; i < bnum; i++) {
 634                         desc = &ring->desc[ring->next_to_clean];
 635                         desc_cb = &ring->desc_cb[ring->next_to_clean];
 636
 637                         hns_nic_reuse_page(skb, i, ring, 0, desc_cb);
 638                         ring_ptr_move_fw(ring, next_to_clean);
 639                 }
 640         }
 641
 642         /* check except process, free skb and jump the desc */
 643         if (unlikely((!bnum) || (bnum > ring->max_desc_num_per_pkt))) {
 644 out_bnum_err:
 645                 *out_bnum = *out_bnum ? *out_bnum : 1; /* ntc moved,cannot 0*/
 646                 netdev_err(ndev, "invalid bnum(%d,%d,%d,%d),%016llx,%016llx\n",
 647                            bnum, ring->max_desc_num_per_pkt,
 648                            length, (int)MAX_SKB_FRAGS,
 649                            ((u64 *)desc)[0], ((u64 *)desc)[1]);
 650                 ring->stats.err_bd_num++;
 651                 dev_kfree_skb_any(skb);
 652                 return -EDOM;
 653         }
 654
 655         bnum_flag = le32_to_cpu(desc->rx.ipoff_bnum_pid_flag);
 656
 657         if (unlikely(!hnae_get_bit(bnum_flag, HNS_RXD_VLD_B))) {
 658                 netdev_err(ndev, "no valid bd,%016llx,%016llx\n",
 659                            ((u64 *)desc)[0], ((u64 *)desc)[1]);
 660                 ring->stats.non_vld_descs++;
 661                 dev_kfree_skb_any(skb);
 662                 return -EINVAL;
 663         }
 664
 665         if (unlikely((!desc->rx.pkt_len) ||
 666                      hnae_get_bit(bnum_flag, HNS_RXD_DROP_B))) {
 667                 ring->stats.err_pkt_len++;
 668                 dev_kfree_skb_any(skb);
 669                 return -EFAULT;
 670         }
 671
 672         if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L2E_B))) {
 673                 ring->stats.l2_err++;
 674                 dev_kfree_skb_any(skb);
 675                 return -EFAULT;
 676         }
 677
 678         /* filter out multicast pkt with the same src mac as this port */
 679         eh = eth_hdr(skb);
 680         if (unlikely(is_multicast_ether_addr(eh->h_dest) &&
 681                      ether_addr_equal(ndev->dev_addr, eh->h_source))) {
 682                 dev_kfree_skb_any(skb);
 683                 return -EFAULT;
 684         }
 685
 686         ring->stats.rx_pkts++;
 687         ring->stats.rx_bytes += skb->len;
 688
 689         if (unlikely(hnae_get_bit(bnum_flag, HNS_RXD_L3E_B) ||
 690                      hnae_get_bit(bnum_flag, HNS_RXD_L4E_B))) {
 691                 ring->stats.l3l4_csum_err++;
 692                 return 0;
 693         }
 694
 695         skb->ip_summed = CHECKSUM_UNNECESSARY;
 696
 697         return 0;
 698 }
 699
 700 static void
 701 hns_nic_alloc_rx_buffers(struct hns_nic_ring_data *ring_data, int cleand_count)
 702 {
 703         int i, ret;
 704         struct hnae_desc_cb res_cbs;
 705         struct hnae_desc_cb *desc_cb;
 706         struct hnae_ring *ring = ring_data->ring;
 707         struct net_device *ndev = ring_data->napi.dev;
 708
 709         for (i = 0; i < cleand_count; i++) {
 710                 desc_cb = &ring->desc_cb[ring->next_to_use];
 711                 if (desc_cb->reuse_flag) {
 712                         ring->stats.reuse_pg_cnt++;
 713                         hnae_reuse_buffer(ring, ring->next_to_use);
 714                 } else {
 715                         ret = hnae_reserve_buffer_map(ring, &res_cbs);
 716                         if (ret) {
 717                                 ring->stats.sw_err_cnt++;
 718                                 netdev_err(ndev, "hnae reserve buffer map failed.\n");
 719                                 break;
 720                         }
 721                         hnae_replace_buffer(ring, ring->next_to_use, &res_cbs);
 722                 }
 723
 724                 ring_ptr_move_fw(ring, next_to_use);
 725         }
 726
 727         wmb(); /* make all data has been write before submit */
 728         writel_relaxed(i, ring->io_base + RCB_REG_HEAD);
 729 }
 730
 731 /* return error number for error or number of desc left to take
 732  */
 733 static void hns_nic_rx_up_pro(struct hns_nic_ring_data *ring_data,
 734                               struct sk_buff *skb)
 735 {
 736         struct net_device *ndev = ring_data->napi.dev;
 737
 738         skb->protocol = eth_type_trans(skb, ndev);
 739         (void)napi_gro_receive(&ring_data->napi, skb);
 740         ndev->last_rx = jiffies;
 741 }
 742
 743 static int hns_nic_rx_poll_one(struct hns_nic_ring_data *ring_data,
 744                                int budget, void *v)
 745 {
 746         struct hnae_ring *ring = ring_data->ring;
 747         struct sk_buff *skb;
 748         int num, bnum, ex_num;
 749 #define RCB_NOF_ALLOC_RX_BUFF_ONCE 16
 750         int recv_pkts, recv_bds, clean_count, err;
 751
 752         num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 753         rmb(); /* make sure num taken effect before the other data is touched */
 754
 755         recv_pkts = 0, recv_bds = 0, clean_count = 0;
 756 recv:
 757         while (recv_pkts < budget && recv_bds < num) {
 758                 /* reuse or realloc buffers*/
 759                 if (clean_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {
 760                         hns_nic_alloc_rx_buffers(ring_data, clean_count);
 761                         clean_count = 0;
 762                 }
 763
 764                 /* poll one pkg*/
 765                 err = hns_nic_poll_rx_skb(ring_data, &skb, &bnum);
 766                 if (unlikely(!skb)) /* this fault cannot be repaired */
 767                         break;
 768
 769                 recv_bds += bnum;
 770                 clean_count += bnum;
 771                 if (unlikely(err)) {  /* do jump the err */
 772                         recv_pkts++;
 773                         continue;
 774                 }
 775
 776                 /* do update ip stack process*/
 777                 ((void (*)(struct hns_nic_ring_data *, struct sk_buff *))v)(
 778                                                         ring_data, skb);
 779                 recv_pkts++;
 780         }
 781
 782         /* make all data has been write before submit */
 783         if (recv_pkts < budget) {
 784                 ex_num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 785
 786                 if (ex_num > clean_count) {
 787                         num += ex_num - clean_count;
 788                         rmb(); /*complete read rx ring bd number*/
 789                         goto recv;
 790                 }
 791         }
 792
 793         /* make all data has been write before submit */
 794         if (clean_count > 0)
 795                 hns_nic_alloc_rx_buffers(ring_data, clean_count);
 796
 797         return recv_pkts;
 798 }
 799
 800 static void hns_nic_rx_fini_pro(struct hns_nic_ring_data *ring_data)
 801 {
 802         struct hnae_ring *ring = ring_data->ring;
 803         int num = 0;
 804
 805         /* for hardware bug fixed */
 806         num = readl_relaxed(ring->io_base + RCB_REG_FBDNUM);
 807
 808         if (num > 0) {
 809                 ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 810                         ring_data->ring, 1);
 811
 812                 napi_schedule(&ring_data->napi);
 813         }
 814 }
 815
 816 static inline void hns_nic_reclaim_one_desc(struct hnae_ring *ring,
 817                                             int *bytes, int *pkts)
 818 {
 819         struct hnae_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];
 820
 821         (*pkts) += (desc_cb->type == DESC_TYPE_SKB);
 822         (*bytes) += desc_cb->length;
 823         /* desc_cb will be cleaned, after hnae_free_buffer_detach*/
 824         hnae_free_buffer_detach(ring, ring->next_to_clean);
 825
 826         ring_ptr_move_fw(ring, next_to_clean);
 827 }
 828
 829 static int is_valid_clean_head(struct hnae_ring *ring, int h)
 830 {
 831         int u = ring->next_to_use;
 832         int c = ring->next_to_clean;
 833
 834         if (unlikely(h > ring->desc_num))
 835                 return 0;
 836
 837         assert(u > 0 && u < ring->desc_num);
 838         assert(c > 0 && c < ring->desc_num);
 839         assert(u != c && h != c); /* must be checked before call this func */
 840
 841         return u > c ? (h > c && h <= u) : (h > c || h <= u);
 842 }
 843
 844 /* netif_tx_lock will turn down the performance, set only when necessary */
 845 #ifdef CONFIG_NET_POLL_CONTROLLER
 846 #define NETIF_TX_LOCK(ndev) netif_tx_lock(ndev)
 847 #define NETIF_TX_UNLOCK(ndev) netif_tx_unlock(ndev)
 848 #else
 849 #define NETIF_TX_LOCK(ndev)
 850 #define NETIF_TX_UNLOCK(ndev)
 851 #endif
 852 /* reclaim all desc in one budget
 853  * return error or number of desc left
 854  */
 855 static int hns_nic_tx_poll_one(struct hns_nic_ring_data *ring_data,
 856                                int budget, void *v)
 857 {
 858         struct hnae_ring *ring = ring_data->ring;
 859         struct net_device *ndev = ring_data->napi.dev;
 860         struct netdev_queue *dev_queue;
 861         struct hns_nic_priv *priv = netdev_priv(ndev);
 862         int head;
 863         int bytes, pkts;
 864
 865         NETIF_TX_LOCK(ndev);
 866
 867         head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
 868         rmb(); /* make sure head is ready before touch any data */
 869
 870         if (is_ring_empty(ring) || head == ring->next_to_clean) {
 871                 NETIF_TX_UNLOCK(ndev);
 872                 return 0; /* no data to poll */
 873         }
 874
 875         if (!is_valid_clean_head(ring, head)) {
 876                 netdev_err(ndev, "wrong head (%d, %d-%d)\n", head,
 877                            ring->next_to_use, ring->next_to_clean);
 878                 ring->stats.io_err_cnt++;
 879                 NETIF_TX_UNLOCK(ndev);
 880                 return -EIO;
 881         }
 882
 883         bytes = 0;
 884         pkts = 0;
 885         while (head != ring->next_to_clean) {
 886                 hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
 887                 /* issue prefetch for next Tx descriptor */
 888                 prefetch(&ring->desc_cb[ring->next_to_clean]);
 889         }
 890
 891         NETIF_TX_UNLOCK(ndev);
 892
 893         dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
 894         netdev_tx_completed_queue(dev_queue, pkts, bytes);
 895
 896         if (unlikely(priv->link && !netif_carrier_ok(ndev)))
 897                 netif_carrier_on(ndev);
 898
 899         if (unlikely(pkts && netif_carrier_ok(ndev) &&
 900                      (ring_space(ring) >= ring->max_desc_num_per_pkt * 2))) {
 901                 /* Make sure that anybody stopping the queue after this
 902                  * sees the new next_to_clean.
 903                  */
 904                 smp_mb();
 905                 if (netif_tx_queue_stopped(dev_queue) &&
 906                     !test_bit(NIC_STATE_DOWN, &priv->state)) {
 907                         netif_tx_wake_queue(dev_queue);
 908                         ring->stats.restart_queue++;
 909                 }
 910         }
 911         return 0;
 912 }
 913
 914 static void hns_nic_tx_fini_pro(struct hns_nic_ring_data *ring_data)
 915 {
 916         struct hnae_ring *ring = ring_data->ring;
 917         int head = ring->next_to_clean;
 918
 919         /* for hardware bug fixed */
 920         head = readl_relaxed(ring->io_base + RCB_REG_HEAD);
 921
 922         if (head != ring->next_to_clean) {
 923                 ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 924                         ring_data->ring, 1);
 925
 926                 napi_schedule(&ring_data->napi);
 927         }
 928 }
 929
 930 static void hns_nic_tx_clr_all_bufs(struct hns_nic_ring_data *ring_data)
 931 {
 932         struct hnae_ring *ring = ring_data->ring;
 933         struct net_device *ndev = ring_data->napi.dev;
 934         struct netdev_queue *dev_queue;
 935         int head;
 936         int bytes, pkts;
 937
 938         NETIF_TX_LOCK(ndev);
 939
 940         head = ring->next_to_use; /* ntu :soft setted ring position*/
 941         bytes = 0;
 942         pkts = 0;
 943         while (head != ring->next_to_clean)
 944                 hns_nic_reclaim_one_desc(ring, &bytes, &pkts);
 945
 946         NETIF_TX_UNLOCK(ndev);
 947
 948         dev_queue = netdev_get_tx_queue(ndev, ring_data->queue_index);
 949         netdev_tx_reset_queue(dev_queue);
 950 }
 951
 952 static int hns_nic_common_poll(struct napi_struct *napi, int budget)
 953 {
 954         struct hns_nic_ring_data *ring_data =
 955                 container_of(napi, struct hns_nic_ring_data, napi);
 956         int clean_complete = ring_data->poll_one(
 957                                 ring_data, budget, ring_data->ex_process);
 958
 959         if (clean_complete >= 0 && clean_complete < budget) {
 960                 napi_complete(napi);
 961                 ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 962                         ring_data->ring, 0);
 963
 964                 ring_data->fini_process(ring_data);
 965                 return 0;
 966         }
 967
 968         return clean_complete;
 969 }
 970
 971 static irqreturn_t hns_irq_handle(int irq, void *dev)
 972 {
 973         struct hns_nic_ring_data *ring_data = (struct hns_nic_ring_data *)dev;
 974
 975         ring_data->ring->q->handle->dev->ops->toggle_ring_irq(
 976                 ring_data->ring, 1);
 977         napi_schedule(&ring_data->napi);
 978
 979         return IRQ_HANDLED;
 980 }
 981
 982 /**
 983  *hns_nic_adjust_link - adjust net work mode by the phy stat or new param
 984  *@ndev: net device
 985  */
 986 static void hns_nic_adjust_link(struct net_device *ndev)
 987 {
 988         struct hns_nic_priv *priv = netdev_priv(ndev);
 989         struct hnae_handle *h = priv->ae_handle;
 990
 991         h->dev->ops->adjust_link(h, ndev->phydev->speed, ndev->phydev->duplex);
 992 }
 993
 994 /**
 995  *hns_nic_init_phy - init phy
 996  *@ndev: net device
 997  *@h: ae handle
 998  * Return 0 on success, negative on failure
 999  */
1000 int hns_nic_init_phy(struct net_device *ndev, struct hnae_handle *h)
1001 {
1002         struct hns_nic_priv *priv = netdev_priv(ndev);
1003         struct phy_device *phy_dev = NULL;
1004
1005         if (!h->phy_node)
1006                 return 0;
1007
1008         if (h->phy_if != PHY_INTERFACE_MODE_XGMII)
1009                 phy_dev = of_phy_connect(ndev, h->phy_node,
1010                                          hns_nic_adjust_link, 0, h->phy_if);
1011         else
1012                 phy_dev = of_phy_attach(ndev, h->phy_node, 0, h->phy_if);
1013
1014         if (unlikely(!phy_dev) || IS_ERR(phy_dev))
1015                 return !phy_dev ? -ENODEV : PTR_ERR(phy_dev);
1016
1017         phy_dev->supported &= h->if_support;
1018         phy_dev->advertising = phy_dev->supported;
1019
1020         if (h->phy_if == PHY_INTERFACE_MODE_XGMII)
1021                 phy_dev->autoneg = false;
1022
1023         priv->phy = phy_dev;
1024
1025         return 0;
1026 }
1027
1028 static int hns_nic_ring_open(struct net_device *netdev, int idx)
1029 {
1030         struct hns_nic_priv *priv = netdev_priv(netdev);
1031         struct hnae_handle *h = priv->ae_handle;
1032
1033         napi_enable(&priv->ring_data[idx].napi);
1034
1035         enable_irq(priv->ring_data[idx].ring->irq);
1036         h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 0);
1037
1038         return 0;
1039 }
1040
1041 static int hns_nic_net_set_mac_address(struct net_device *ndev, void *p)
1042 {
1043         struct hns_nic_priv *priv = netdev_priv(ndev);
1044         struct hnae_handle *h = priv->ae_handle;
1045         struct sockaddr *mac_addr = p;
1046         int ret;
1047
1048         if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))
1049                 return -EADDRNOTAVAIL;
1050
1051         ret = h->dev->ops->set_mac_addr(h, mac_addr->sa_data);
1052         if (ret) {
1053                 netdev_err(ndev, "set_mac_address fail, ret=%d!\n", ret);
1054                 return ret;
1055         }
1056
1057         memcpy(ndev->dev_addr, mac_addr->sa_data, ndev->addr_len);
1058
1059         return 0;
1060 }
1061
1062 void hns_nic_update_stats(struct net_device *netdev)
1063 {
1064         struct hns_nic_priv *priv = netdev_priv(netdev);
1065         struct hnae_handle *h = priv->ae_handle;
1066
1067         h->dev->ops->update_stats(h, &netdev->stats);
1068 }
1069
1070 /* set mac addr if it is configed. or leave it to the AE driver */
1071 static void hns_init_mac_addr(struct net_device *ndev)
1072 {
1073         struct hns_nic_priv *priv = netdev_priv(ndev);
1074         struct device_node *node = priv->dev->of_node;
1075         const void *mac_addr_temp;
1076
1077         mac_addr_temp = of_get_mac_address(node);
1078         if (mac_addr_temp && is_valid_ether_addr(mac_addr_temp)) {
1079                 memcpy(ndev->dev_addr, mac_addr_temp, ndev->addr_len);
1080         } else {
1081                 eth_hw_addr_random(ndev);
1082                 dev_warn(priv->dev, "No valid mac, use random mac %pM",
1083                          ndev->dev_addr);
1084         }
1085 }
1086
1087 static void hns_nic_ring_close(struct net_device *netdev, int idx)
1088 {
1089         struct hns_nic_priv *priv = netdev_priv(netdev);
1090         struct hnae_handle *h = priv->ae_handle;
1091
1092         h->dev->ops->toggle_ring_irq(priv->ring_data[idx].ring, 1);
1093         disable_irq(priv->ring_data[idx].ring->irq);
1094
1095         napi_disable(&priv->ring_data[idx].napi);
1096 }
1097
1098 static void hns_set_irq_affinity(struct hns_nic_priv *priv)
1099 {
1100         struct hnae_handle *h = priv->ae_handle;
1101         struct hns_nic_ring_data *rd;
1102         int i;
1103         int cpu;
1104         cpumask_t mask;
1105
1106         /*diffrent irq banlance for 16core and 32core*/
1107         if (h->q_num == num_possible_cpus()) {
1108                 for (i = 0; i < h->q_num * 2; i++) {
1109                         rd = &priv->ring_data[i];
1110                         if (cpu_online(rd->queue_index)) {
1111                                 cpumask_clear(&mask);
1112                                 cpu = rd->queue_index;
1113                                 cpumask_set_cpu(cpu, &mask);
1114                                 (void)irq_set_affinity_hint(rd->ring->irq,
1115                                                             &mask);
1116                         }
1117                 }
1118         } else {
1119                 for (i = 0; i < h->q_num; i++) {
1120                         rd = &priv->ring_data[i];
1121                         if (cpu_online(rd->queue_index * 2)) {
1122                                 cpumask_clear(&mask);
1123                                 cpu = rd->queue_index * 2;
1124                                 cpumask_set_cpu(cpu, &mask);
1125                                 (void)irq_set_affinity_hint(rd->ring->irq,
1126                                                             &mask);
1127                         }
1128                 }
1129
1130                 for (i = h->q_num; i < h->q_num * 2; i++) {
1131                         rd = &priv->ring_data[i];
1132                         if (cpu_online(rd->queue_index * 2 + 1)) {
1133                                 cpumask_clear(&mask);
1134                                 cpu = rd->queue_index * 2 + 1;
1135                                 cpumask_set_cpu(cpu, &mask);
1136                                 (void)irq_set_affinity_hint(rd->ring->irq,
1137                                                             &mask);
1138                         }
1139                 }
1140         }
1141 }
1142
1143 static int hns_nic_init_irq(struct hns_nic_priv *priv)
1144 {
1145         struct hnae_handle *h = priv->ae_handle;
1146         struct hns_nic_ring_data *rd;
1147         int i;
1148         int ret;
1149
1150         for (i = 0; i < h->q_num * 2; i++) {
1151                 rd = &priv->ring_data[i];
1152
1153                 if (rd->ring->irq_init_flag == RCB_IRQ_INITED)
1154                         break;
1155
1156                 snprintf(rd->ring->ring_name, RCB_RING_NAME_LEN,
1157                          "%s-%s%d", priv->netdev->name,
1158                          (i < h->q_num ? "tx" : "rx"), rd->queue_index);
1159
1160                 rd->ring->ring_name[RCB_RING_NAME_LEN - 1] = '\0';
1161
1162                 ret = request_irq(rd->ring->irq,
1163                                   hns_irq_handle, 0, rd->ring->ring_name, rd);
1164                 if (ret) {
1165                         netdev_err(priv->netdev, "request irq(%d) fail\n",
1166                                    rd->ring->irq);
1167                         return ret;
1168                 }
1169                 disable_irq(rd->ring->irq);
1170                 rd->ring->irq_init_flag = RCB_IRQ_INITED;
1171         }
1172
1173         /*set cpu affinity*/
1174         hns_set_irq_affinity(priv);
1175
1176         return 0;
1177 }
1178
1179 static int hns_nic_net_up(struct net_device *ndev)
1180 {
1181         struct hns_nic_priv *priv = netdev_priv(ndev);
1182         struct hnae_handle *h = priv->ae_handle;
1183         int i, j, k;
1184         int ret;
1185
1186         ret = hns_nic_init_irq(priv);
1187         if (ret != 0) {
1188                 netdev_err(ndev, "hns init irq failed! ret=%d\n", ret);
1189                 return ret;
1190         }
1191
1192         for (i = 0; i < h->q_num * 2; i++) {
1193                 ret = hns_nic_ring_open(ndev, i);
1194                 if (ret)
1195                         goto out_has_some_queues;
1196         }
1197
1198         for (k = 0; k < h->q_num; k++)
1199                 h->dev->ops->toggle_queue_status(h->qs[k], 1);
1200
1201         ret = h->dev->ops->set_mac_addr(h, ndev->dev_addr);
1202         if (ret)
1203                 goto out_set_mac_addr_err;
1204
1205         ret = h->dev->ops->start ? h->dev->ops->start(h) : 0;
1206         if (ret)
1207                 goto out_start_err;
1208
1209         if (priv->phy)
1210                 phy_start(priv->phy);
1211
1212         clear_bit(NIC_STATE_DOWN, &priv->state);
1213         (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
1214
1215         return 0;
1216
1217 out_start_err:
1218         netif_stop_queue(ndev);
1219 out_set_mac_addr_err:
1220         for (k = 0; k < h->q_num; k++)
1221                 h->dev->ops->toggle_queue_status(h->qs[k], 0);
1222 out_has_some_queues:
1223         for (j = i - 1; j >= 0; j--)
1224                 hns_nic_ring_close(ndev, j);
1225
1226         set_bit(NIC_STATE_DOWN, &priv->state);
1227
1228         return ret;
1229 }
1230
1231 static void hns_nic_net_down(struct net_device *ndev)
1232 {
1233         int i;
1234         struct hnae_ae_ops *ops;
1235         struct hns_nic_priv *priv = netdev_priv(ndev);
1236
1237         if (test_and_set_bit(NIC_STATE_DOWN, &priv->state))
1238                 return;
1239
1240         (void)del_timer_sync(&priv->service_timer);
1241         netif_tx_stop_all_queues(ndev);
1242         netif_carrier_off(ndev);
1243         netif_tx_disable(ndev);
1244         priv->link = 0;
1245
1246         if (priv->phy)
1247                 phy_stop(priv->phy);
1248
1249         ops = priv->ae_handle->dev->ops;
1250
1251         if (ops->stop)
1252                 ops->stop(priv->ae_handle);
1253
1254         netif_tx_stop_all_queues(ndev);
1255
1256         for (i = priv->ae_handle->q_num - 1; i >= 0; i--) {
1257                 hns_nic_ring_close(ndev, i);
1258                 hns_nic_ring_close(ndev, i + priv->ae_handle->q_num);
1259
1260                 /* clean tx buffers*/
1261                 hns_nic_tx_clr_all_bufs(priv->ring_data + i);
1262         }
1263 }
1264
1265 void hns_nic_net_reset(struct net_device *ndev)
1266 {
1267         struct hns_nic_priv *priv = netdev_priv(ndev);
1268         struct hnae_handle *handle = priv->ae_handle;
1269
1270         while (test_and_set_bit(NIC_STATE_RESETTING, &priv->state))
1271                 usleep_range(1000, 2000);
1272
1273         (void)hnae_reinit_handle(handle);
1274
1275         clear_bit(NIC_STATE_RESETTING, &priv->state);
1276 }
1277
1278 void hns_nic_net_reinit(struct net_device *netdev)
1279 {
1280         struct hns_nic_priv *priv = netdev_priv(netdev);
1281
1282         priv->netdev->trans_start = jiffies;
1283         while (test_and_set_bit(NIC_STATE_REINITING, &priv->state))
1284                 usleep_range(1000, 2000);
1285
1286         hns_nic_net_down(netdev);
1287         hns_nic_net_reset(netdev);
1288         (void)hns_nic_net_up(netdev);
1289         clear_bit(NIC_STATE_REINITING, &priv->state);
1290 }
1291
1292 static int hns_nic_net_open(struct net_device *ndev)
1293 {
1294         struct hns_nic_priv *priv = netdev_priv(ndev);
1295         struct hnae_handle *h = priv->ae_handle;
1296         int ret;
1297
1298         if (test_bit(NIC_STATE_TESTING, &priv->state))
1299                 return -EBUSY;
1300
1301         priv->link = 0;
1302         netif_carrier_off(ndev);
1303
1304         ret = netif_set_real_num_tx_queues(ndev, h->q_num);
1305         if (ret < 0) {
1306                 netdev_err(ndev, "netif_set_real_num_tx_queues fail, ret=%d!\n",
1307                            ret);
1308                 return ret;
1309         }
1310
1311         ret = netif_set_real_num_rx_queues(ndev, h->q_num);
1312         if (ret < 0) {
1313                 netdev_err(ndev,
1314                            "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);
1315                 return ret;
1316         }
1317
1318         ret = hns_nic_net_up(ndev);
1319         if (ret) {
1320                 netdev_err(ndev,
1321                            "hns net up fail, ret=%d!\n", ret);
1322                 return ret;
1323         }
1324
1325         return 0;
1326 }
1327
1328 static int hns_nic_net_stop(struct net_device *ndev)
1329 {
1330         hns_nic_net_down(ndev);
1331
1332         return 0;
1333 }
1334
1335 static void hns_tx_timeout_reset(struct hns_nic_priv *priv);
1336 static void hns_nic_net_timeout(struct net_device *ndev)
1337 {
1338         struct hns_nic_priv *priv = netdev_priv(ndev);
1339
1340         hns_tx_timeout_reset(priv);
1341 }
1342
1343 static int hns_nic_do_ioctl(struct net_device *netdev, struct ifreq *ifr,
1344                             int cmd)
1345 {
1346         struct hns_nic_priv *priv = netdev_priv(netdev);
1347         struct phy_device *phy_dev = priv->phy;
1348
1349         if (!netif_running(netdev))
1350                 return -EINVAL;
1351
1352         if (!phy_dev)
1353                 return -ENOTSUPP;
1354
1355         return phy_mii_ioctl(phy_dev, ifr, cmd);
1356 }
1357
1358 /* use only for netconsole to poll with the device without interrupt */
1359 #ifdef CONFIG_NET_POLL_CONTROLLER
1360 void hns_nic_poll_controller(struct net_device *ndev)
1361 {
1362         struct hns_nic_priv *priv = netdev_priv(ndev);
1363         unsigned long flags;
1364         int i;
1365
1366         local_irq_save(flags);
1367         for (i = 0; i < priv->ae_handle->q_num * 2; i++)
1368                 napi_schedule(&priv->ring_data[i].napi);
1369         local_irq_restore(flags);
1370 }
1371 #endif
1372
1373 static netdev_tx_t hns_nic_net_xmit(struct sk_buff *skb,
1374                                     struct net_device *ndev)
1375 {
1376         struct hns_nic_priv *priv = netdev_priv(ndev);
1377         int ret;
1378
1379         assert(skb->queue_mapping < ndev->ae_handle->q_num);
1380         ret = hns_nic_net_xmit_hw(ndev, skb,
1381                                   &tx_ring_data(priv, skb->queue_mapping));
1382         if (ret == NETDEV_TX_OK) {
1383                 ndev->trans_start = jiffies;
1384                 ndev->stats.tx_bytes += skb->len;
1385                 ndev->stats.tx_packets++;
1386         }
1387         return (netdev_tx_t)ret;
1388 }
1389
1390 static int hns_nic_change_mtu(struct net_device *ndev, int new_mtu)
1391 {
1392         struct hns_nic_priv *priv = netdev_priv(ndev);
1393         struct hnae_handle *h = priv->ae_handle;
1394         int ret;
1395
1396         /* MTU < 68 is an error and causes problems on some kernels */
1397         if (new_mtu < 68)
1398                 return -EINVAL;
1399
1400         if (!h->dev->ops->set_mtu)
1401                 return -ENOTSUPP;
1402
1403         if (netif_running(ndev)) {
1404                 (void)hns_nic_net_stop(ndev);
1405                 msleep(100);
1406
1407                 ret = h->dev->ops->set_mtu(h, new_mtu);
1408                 if (ret)
1409                         netdev_err(ndev, "set mtu fail, return value %d\n",
1410                                    ret);
1411
1412                 if (hns_nic_net_open(ndev))
1413                         netdev_err(ndev, "hns net open fail\n");
1414         } else {
1415                 ret = h->dev->ops->set_mtu(h, new_mtu);
1416         }
1417
1418         if (!ret)
1419                 ndev->mtu = new_mtu;
1420
1421         return ret;
1422 }
1423
1424 static int hns_nic_set_features(struct net_device *netdev,
1425                                 netdev_features_t features)
1426 {
1427         struct hns_nic_priv *priv = netdev_priv(netdev);
1428         struct hnae_handle *h = priv->ae_handle;
1429
1430         switch (priv->enet_ver) {
1431         case AE_VERSION_1:
1432                 if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1433                         netdev_info(netdev, "enet v1 do not support tso!\n");
1434                 break;
1435         default:
1436                 if (features & (NETIF_F_TSO | NETIF_F_TSO6)) {
1437                         priv->ops.fill_desc = fill_tso_desc;
1438                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
1439                         /* The chip only support 7*4096 */
1440                         netif_set_gso_max_size(netdev, 7 * 4096);
1441                         h->dev->ops->set_tso_stats(h, 1);
1442                 } else {
1443                         priv->ops.fill_desc = fill_v2_desc;
1444                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
1445                         h->dev->ops->set_tso_stats(h, 0);
1446                 }
1447                 break;
1448         }
1449         netdev->features = features;
1450         return 0;
1451 }
1452
1453 static netdev_features_t hns_nic_fix_features(
1454                 struct net_device *netdev, netdev_features_t features)
1455 {
1456         struct hns_nic_priv *priv = netdev_priv(netdev);
1457
1458         switch (priv->enet_ver) {
1459         case AE_VERSION_1:
1460                 features &= ~(NETIF_F_TSO | NETIF_F_TSO6 |
1461                                 NETIF_F_HW_VLAN_CTAG_FILTER);
1462                 break;
1463         default:
1464                 break;
1465         }
1466         return features;
1467 }
1468
1469 /**
1470  * nic_set_multicast_list - set mutl mac address
1471  * @netdev: net device
1472  * @p: mac address
1473  *
1474  * return void
1475  */
1476 void hns_set_multicast_list(struct net_device *ndev)
1477 {
1478         struct hns_nic_priv *priv = netdev_priv(ndev);
1479         struct hnae_handle *h = priv->ae_handle;
1480         struct netdev_hw_addr *ha = NULL;
1481
1482         if (!h) {
1483                 netdev_err(ndev, "hnae handle is null\n");
1484                 return;
1485         }
1486
1487         if (h->dev->ops->set_mc_addr) {
1488                 netdev_for_each_mc_addr(ha, ndev)
1489                         if (h->dev->ops->set_mc_addr(h, ha->addr))
1490                                 netdev_err(ndev, "set multicast fail\n");
1491         }
1492 }
1493
1494 void hns_nic_set_rx_mode(struct net_device *ndev)
1495 {
1496         struct hns_nic_priv *priv = netdev_priv(ndev);
1497         struct hnae_handle *h = priv->ae_handle;
1498
1499         if (h->dev->ops->set_promisc_mode) {
1500                 if (ndev->flags & IFF_PROMISC)
1501                         h->dev->ops->set_promisc_mode(h, 1);
1502                 else
1503                         h->dev->ops->set_promisc_mode(h, 0);
1504         }
1505
1506         hns_set_multicast_list(ndev);
1507 }
1508
1509 struct rtnl_link_stats64 *hns_nic_get_stats64(struct net_device *ndev,
1510                                               struct rtnl_link_stats64 *stats)
1511 {
1512         int idx = 0;
1513         u64 tx_bytes = 0;
1514         u64 rx_bytes = 0;
1515         u64 tx_pkts = 0;
1516         u64 rx_pkts = 0;
1517         struct hns_nic_priv *priv = netdev_priv(ndev);
1518         struct hnae_handle *h = priv->ae_handle;
1519
1520         for (idx = 0; idx < h->q_num; idx++) {
1521                 tx_bytes += h->qs[idx]->tx_ring.stats.tx_bytes;
1522                 tx_pkts += h->qs[idx]->tx_ring.stats.tx_pkts;
1523                 rx_bytes += h->qs[idx]->rx_ring.stats.rx_bytes;
1524                 rx_pkts += h->qs[idx]->rx_ring.stats.rx_pkts;
1525         }
1526
1527         stats->tx_bytes = tx_bytes;
1528         stats->tx_packets = tx_pkts;
1529         stats->rx_bytes = rx_bytes;
1530         stats->rx_packets = rx_pkts;
1531
1532         stats->rx_errors = ndev->stats.rx_errors;
1533         stats->multicast = ndev->stats.multicast;
1534         stats->rx_length_errors = ndev->stats.rx_length_errors;
1535         stats->rx_crc_errors = ndev->stats.rx_crc_errors;
1536         stats->rx_missed_errors = ndev->stats.rx_missed_errors;
1537
1538         stats->tx_errors = ndev->stats.tx_errors;
1539         stats->rx_dropped = ndev->stats.rx_dropped;
1540         stats->tx_dropped = ndev->stats.tx_dropped;
1541         stats->collisions = ndev->stats.collisions;
1542         stats->rx_over_errors = ndev->stats.rx_over_errors;
1543         stats->rx_frame_errors = ndev->stats.rx_frame_errors;
1544         stats->rx_fifo_errors = ndev->stats.rx_fifo_errors;
1545         stats->tx_aborted_errors = ndev->stats.tx_aborted_errors;
1546         stats->tx_carrier_errors = ndev->stats.tx_carrier_errors;
1547         stats->tx_fifo_errors = ndev->stats.tx_fifo_errors;
1548         stats->tx_heartbeat_errors = ndev->stats.tx_heartbeat_errors;
1549         stats->tx_window_errors = ndev->stats.tx_window_errors;
1550         stats->rx_compressed = ndev->stats.rx_compressed;
1551         stats->tx_compressed = ndev->stats.tx_compressed;
1552
1553         return stats;
1554 }
1555
1556 static const struct net_device_ops hns_nic_netdev_ops = {
1557         .ndo_open = hns_nic_net_open,
1558         .ndo_stop = hns_nic_net_stop,
1559         .ndo_start_xmit = hns_nic_net_xmit,
1560         .ndo_tx_timeout = hns_nic_net_timeout,
1561         .ndo_set_mac_address = hns_nic_net_set_mac_address,
1562         .ndo_change_mtu = hns_nic_change_mtu,
1563         .ndo_do_ioctl = hns_nic_do_ioctl,
1564         .ndo_set_features = hns_nic_set_features,
1565         .ndo_fix_features = hns_nic_fix_features,
1566         .ndo_get_stats64 = hns_nic_get_stats64,
1567 #ifdef CONFIG_NET_POLL_CONTROLLER
1568         .ndo_poll_controller = hns_nic_poll_controller,
1569 #endif
1570         .ndo_set_rx_mode = hns_nic_set_rx_mode,
1571 };
1572
1573 static void hns_nic_update_link_status(struct net_device *netdev)
1574 {
1575         struct hns_nic_priv *priv = netdev_priv(netdev);
1576
1577         struct hnae_handle *h = priv->ae_handle;
1578         int state = 1;
1579
1580         if (priv->phy) {
1581                 if (!genphy_update_link(priv->phy))
1582                         state = priv->phy->link;
1583                 else
1584                         state = 0;
1585         }
1586         state = state && h->dev->ops->get_status(h);
1587
1588         if (state != priv->link) {
1589                 if (state) {
1590                         netif_carrier_on(netdev);
1591                         netif_tx_wake_all_queues(netdev);
1592                         netdev_info(netdev, "link up\n");
1593                 } else {
1594                         netif_carrier_off(netdev);
1595                         netdev_info(netdev, "link down\n");
1596                 }
1597                 priv->link = state;
1598         }
1599 }
1600
1601 /* for dumping key regs*/
1602 static void hns_nic_dump(struct hns_nic_priv *priv)
1603 {
1604         struct hnae_handle *h = priv->ae_handle;
1605         struct hnae_ae_ops *ops = h->dev->ops;
1606         u32 *data, reg_num, i;
1607
1608         if (ops->get_regs_len && ops->get_regs) {
1609                 reg_num = ops->get_regs_len(priv->ae_handle);
1610                 reg_num = (reg_num + 3ul) & ~3ul;
1611                 data = kcalloc(reg_num, sizeof(u32), GFP_KERNEL);
1612                 if (data) {
1613                         ops->get_regs(priv->ae_handle, data);
1614                         for (i = 0; i < reg_num; i += 4)
1615                                 pr_info("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1616                                         i, data[i], data[i + 1],
1617                                         data[i + 2], data[i + 3]);
1618                         kfree(data);
1619                 }
1620         }
1621
1622         for (i = 0; i < h->q_num; i++) {
1623                 pr_info("tx_queue%d_next_to_clean:%d\n",
1624                         i, h->qs[i]->tx_ring.next_to_clean);
1625                 pr_info("tx_queue%d_next_to_use:%d\n",
1626                         i, h->qs[i]->tx_ring.next_to_use);
1627                 pr_info("rx_queue%d_next_to_clean:%d\n",
1628                         i, h->qs[i]->rx_ring.next_to_clean);
1629                 pr_info("rx_queue%d_next_to_use:%d\n",
1630                         i, h->qs[i]->rx_ring.next_to_use);
1631         }
1632 }
1633
1634 /* for resetting suntask*/
1635 static void hns_nic_reset_subtask(struct hns_nic_priv *priv)
1636 {
1637         enum hnae_port_type type = priv->ae_handle->port_type;
1638
1639         if (!test_bit(NIC_STATE2_RESET_REQUESTED, &priv->state))
1640                 return;
1641         clear_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
1642
1643         /* If we're already down, removing or resetting, just bail */
1644         if (test_bit(NIC_STATE_DOWN, &priv->state) ||
1645             test_bit(NIC_STATE_REMOVING, &priv->state) ||
1646             test_bit(NIC_STATE_RESETTING, &priv->state))
1647                 return;
1648
1649         hns_nic_dump(priv);
1650         netdev_info(priv->netdev, "try to reset %s port!\n",
1651                     (type == HNAE_PORT_DEBUG ? "debug" : "service"));
1652
1653         rtnl_lock();
1654         /* put off any impending NetWatchDogTimeout */
1655         priv->netdev->trans_start = jiffies;
1656
1657         if (type == HNAE_PORT_DEBUG) {
1658                 hns_nic_net_reinit(priv->netdev);
1659         } else {
1660                 netif_carrier_off(priv->netdev);
1661                 netif_tx_disable(priv->netdev);
1662         }
1663         rtnl_unlock();
1664 }
1665
1666 /* for doing service complete*/
1667 static void hns_nic_service_event_complete(struct hns_nic_priv *priv)
1668 {
1669         WARN_ON(!test_bit(NIC_STATE_SERVICE_SCHED, &priv->state));
1670
1671         smp_mb__before_atomic();
1672         clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
1673 }
1674
1675 static void hns_nic_service_task(struct work_struct *work)
1676 {
1677         struct hns_nic_priv *priv
1678                 = container_of(work, struct hns_nic_priv, service_task);
1679         struct hnae_handle *h = priv->ae_handle;
1680
1681         hns_nic_update_link_status(priv->netdev);
1682         h->dev->ops->update_led_status(h);
1683         hns_nic_update_stats(priv->netdev);
1684
1685         hns_nic_reset_subtask(priv);
1686         hns_nic_service_event_complete(priv);
1687 }
1688
1689 static void hns_nic_task_schedule(struct hns_nic_priv *priv)
1690 {
1691         if (!test_bit(NIC_STATE_DOWN, &priv->state) &&
1692             !test_bit(NIC_STATE_REMOVING, &priv->state) &&
1693             !test_and_set_bit(NIC_STATE_SERVICE_SCHED, &priv->state))
1694                 (void)schedule_work(&priv->service_task);
1695 }
1696
1697 static void hns_nic_service_timer(unsigned long data)
1698 {
1699         struct hns_nic_priv *priv = (struct hns_nic_priv *)data;
1700
1701         (void)mod_timer(&priv->service_timer, jiffies + SERVICE_TIMER_HZ);
1702
1703         hns_nic_task_schedule(priv);
1704 }
1705
1706 /**
1707  * hns_tx_timeout_reset - initiate reset due to Tx timeout
1708  * @priv: driver private struct
1709  **/
1710 static void hns_tx_timeout_reset(struct hns_nic_priv *priv)
1711 {
1712         /* Do the reset outside of interrupt context */
1713         if (!test_bit(NIC_STATE_DOWN, &priv->state)) {
1714                 set_bit(NIC_STATE2_RESET_REQUESTED, &priv->state);
1715                 netdev_warn(priv->netdev,
1716                             "initiating reset due to tx timeout(%llu,0x%lx)\n",
1717                             priv->tx_timeout_count, priv->state);
1718                 priv->tx_timeout_count++;
1719                 hns_nic_task_schedule(priv);
1720         }
1721 }
1722
1723 static int hns_nic_init_ring_data(struct hns_nic_priv *priv)
1724 {
1725         struct hnae_handle *h = priv->ae_handle;
1726         struct hns_nic_ring_data *rd;
1727         int i;
1728
1729         if (h->q_num > NIC_MAX_Q_PER_VF) {
1730                 netdev_err(priv->netdev, "too much queue (%d)\n", h->q_num);
1731                 return -EINVAL;
1732         }
1733
1734         priv->ring_data = kzalloc(h->q_num * sizeof(*priv->ring_data) * 2,
1735                                   GFP_KERNEL);
1736         if (!priv->ring_data)
1737                 return -ENOMEM;
1738
1739         for (i = 0; i < h->q_num; i++) {
1740                 rd = &priv->ring_data[i];
1741                 rd->queue_index = i;
1742                 rd->ring = &h->qs[i]->tx_ring;
1743                 rd->poll_one = hns_nic_tx_poll_one;
1744                 rd->fini_process = hns_nic_tx_fini_pro;
1745
1746                 netif_napi_add(priv->netdev, &rd->napi,
1747                                hns_nic_common_poll, NIC_TX_CLEAN_MAX_NUM);
1748                 rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
1749         }
1750         for (i = h->q_num; i < h->q_num * 2; i++) {
1751                 rd = &priv->ring_data[i];
1752                 rd->queue_index = i - h->q_num;
1753                 rd->ring = &h->qs[i - h->q_num]->rx_ring;
1754                 rd->poll_one = hns_nic_rx_poll_one;
1755                 rd->ex_process = hns_nic_rx_up_pro;
1756                 rd->fini_process = hns_nic_rx_fini_pro;
1757
1758                 netif_napi_add(priv->netdev, &rd->napi,
1759                                hns_nic_common_poll, NIC_RX_CLEAN_MAX_NUM);
1760                 rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
1761         }
1762
1763         return 0;
1764 }
1765
1766 static void hns_nic_uninit_ring_data(struct hns_nic_priv *priv)
1767 {
1768         struct hnae_handle *h = priv->ae_handle;
1769         int i;
1770
1771         for (i = 0; i < h->q_num * 2; i++) {
1772                 netif_napi_del(&priv->ring_data[i].napi);
1773                 if (priv->ring_data[i].ring->irq_init_flag == RCB_IRQ_INITED) {
1774                         (void)irq_set_affinity_hint(
1775                                 priv->ring_data[i].ring->irq,
1776                                 NULL);
1777                         free_irq(priv->ring_data[i].ring->irq,
1778                                  &priv->ring_data[i]);
1779                 }
1780
1781                 priv->ring_data[i].ring->irq_init_flag = RCB_IRQ_NOT_INITED;
1782         }
1783         kfree(priv->ring_data);
1784 }
1785
1786 static void hns_nic_set_priv_ops(struct net_device *netdev)
1787 {
1788         struct hns_nic_priv *priv = netdev_priv(netdev);
1789         struct hnae_handle *h = priv->ae_handle;
1790
1791         if (AE_IS_VER1(priv->enet_ver)) {
1792                 priv->ops.fill_desc = fill_desc;
1793                 priv->ops.get_rxd_bnum = get_rx_desc_bnum;
1794                 priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
1795         } else {
1796                 priv->ops.get_rxd_bnum = get_v2rx_desc_bnum;
1797                 if ((netdev->features & NETIF_F_TSO) ||
1798                     (netdev->features & NETIF_F_TSO6)) {
1799                         priv->ops.fill_desc = fill_tso_desc;
1800                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tso;
1801                         /* This chip only support 7*4096 */
1802                         netif_set_gso_max_size(netdev, 7 * 4096);
1803                         h->dev->ops->set_tso_stats(h, 1);
1804                 } else {
1805                         priv->ops.fill_desc = fill_v2_desc;
1806                         priv->ops.maybe_stop_tx = hns_nic_maybe_stop_tx;
1807                 }
1808         }
1809 }
1810
1811 static int hns_nic_try_get_ae(struct net_device *ndev)
1812 {
1813         struct hns_nic_priv *priv = netdev_priv(ndev);
1814         struct hnae_handle *h;
1815         int ret;
1816
1817         h = hnae_get_handle(&priv->netdev->dev,
1818                             priv->ae_node, priv->port_id, NULL);
1819         if (IS_ERR_OR_NULL(h)) {
1820                 ret = PTR_ERR(h);
1821                 dev_dbg(priv->dev, "has not handle, register notifier!\n");
1822                 goto out;
1823         }
1824         priv->ae_handle = h;
1825
1826         ret = hns_nic_init_phy(ndev, h);
1827         if (ret) {
1828                 dev_err(priv->dev, "probe phy device fail!\n");
1829                 goto out_init_phy;
1830         }
1831
1832         ret = hns_nic_init_ring_data(priv);
1833         if (ret) {
1834                 ret = -ENOMEM;
1835                 goto out_init_ring_data;
1836         }
1837
1838         hns_nic_set_priv_ops(ndev);
1839
1840         ret = register_netdev(ndev);
1841         if (ret) {
1842                 dev_err(priv->dev, "probe register netdev fail!\n");
1843                 goto out_reg_ndev_fail;
1844         }
1845         return 0;
1846
1847 out_reg_ndev_fail:
1848         hns_nic_uninit_ring_data(priv);
1849         priv->ring_data = NULL;
1850 out_init_phy:
1851 out_init_ring_data:
1852         hnae_put_handle(priv->ae_handle);
1853         priv->ae_handle = NULL;
1854 out:
1855         return ret;
1856 }
1857
1858 static int hns_nic_notifier_action(struct notifier_block *nb,
1859                                    unsigned long action, void *data)
1860 {
1861         struct hns_nic_priv *priv =
1862                 container_of(nb, struct hns_nic_priv, notifier_block);
1863
1864         assert(action == HNAE_AE_REGISTER);
1865
1866         if (!hns_nic_try_get_ae(priv->netdev)) {
1867                 hnae_unregister_notifier(&priv->notifier_block);
1868                 priv->notifier_block.notifier_call = NULL;
1869         }
1870         return 0;
1871 }
1872
1873 static int hns_nic_dev_probe(struct platform_device *pdev)
1874 {
1875         struct device *dev = &pdev->dev;
1876         struct net_device *ndev;
1877         struct hns_nic_priv *priv;
1878         struct device_node *node = dev->of_node;
1879         int ret;
1880
1881         ndev = alloc_etherdev_mq(sizeof(struct hns_nic_priv), NIC_MAX_Q_PER_VF);
1882         if (!ndev)
1883                 return -ENOMEM;
1884
1885         platform_set_drvdata(pdev, ndev);
1886
1887         priv = netdev_priv(ndev);
1888         priv->dev = dev;
1889         priv->netdev = ndev;
1890
1891         if (of_device_is_compatible(node, "hisilicon,hns-nic-v1"))
1892                 priv->enet_ver = AE_VERSION_1;
1893         else
1894                 priv->enet_ver = AE_VERSION_2;
1895
1896         priv->ae_node = (void *)of_parse_phandle(node, "ae-handle", 0);
1897         if (IS_ERR_OR_NULL(priv->ae_node)) {
1898                 ret = PTR_ERR(priv->ae_node);
1899                 dev_err(dev, "not find ae-handle\n");
1900                 goto out_read_prop_fail;
1901         }
1902
1903         ret = of_property_read_u32(node, "port-id", &priv->port_id);
1904         if (ret)
1905                 goto out_read_prop_fail;
1906
1907         hns_init_mac_addr(ndev);
1908
1909         ndev->watchdog_timeo = HNS_NIC_TX_TIMEOUT;
1910         ndev->priv_flags |= IFF_UNICAST_FLT;
1911         ndev->netdev_ops = &hns_nic_netdev_ops;
1912         hns_ethtool_set_ops(ndev);
1913
1914         ndev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1915                 NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1916                 NETIF_F_GRO;
1917         ndev->vlan_features |=
1918                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM;
1919         ndev->vlan_features |= NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;
1920
1921         switch (priv->enet_ver) {
1922         case AE_VERSION_2:
1923                 ndev->features |= NETIF_F_TSO | NETIF_F_TSO6;
1924                 ndev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1925                         NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |
1926                         NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6;
1927                 break;
1928         default:
1929                 break;
1930         }
1931
1932         SET_NETDEV_DEV(ndev, dev);
1933
1934         if (!dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)))
1935                 dev_dbg(dev, "set mask to 64bit\n");
1936         else
1937                 dev_err(dev, "set mask to 32bit fail!\n");
1938
1939         /* carrier off reporting is important to ethtool even BEFORE open */
1940         netif_carrier_off(ndev);
1941
1942         setup_timer(&priv->service_timer, hns_nic_service_timer,
1943                     (unsigned long)priv);
1944         INIT_WORK(&priv->service_task, hns_nic_service_task);
1945
1946         set_bit(NIC_STATE_SERVICE_INITED, &priv->state);
1947         clear_bit(NIC_STATE_SERVICE_SCHED, &priv->state);
1948         set_bit(NIC_STATE_DOWN, &priv->state);
1949
1950         if (hns_nic_try_get_ae(priv->netdev)) {
1951                 priv->notifier_block.notifier_call = hns_nic_notifier_action;
1952                 ret = hnae_register_notifier(&priv->notifier_block);
1953                 if (ret) {
1954                         dev_err(dev, "register notifier fail!\n");
1955                         goto out_notify_fail;
1956                 }
1957                 dev_dbg(dev, "has not handle, register notifier!\n");
1958         }
1959
1960         return 0;
1961
1962 out_notify_fail:
1963         (void)cancel_work_sync(&priv->service_task);
1964 out_read_prop_fail:
1965         free_netdev(ndev);
1966         return ret;
1967 }
1968
1969 static int hns_nic_dev_remove(struct platform_device *pdev)
1970 {
1971         struct net_device *ndev = platform_get_drvdata(pdev);
1972         struct hns_nic_priv *priv = netdev_priv(ndev);
1973
1974         if (ndev->reg_state != NETREG_UNINITIALIZED)
1975                 unregister_netdev(ndev);
1976
1977         if (priv->ring_data)
1978                 hns_nic_uninit_ring_data(priv);
1979         priv->ring_data = NULL;
1980
1981         if (priv->phy)
1982                 phy_disconnect(priv->phy);
1983         priv->phy = NULL;
1984
1985         if (!IS_ERR_OR_NULL(priv->ae_handle))
1986                 hnae_put_handle(priv->ae_handle);
1987         priv->ae_handle = NULL;
1988         if (priv->notifier_block.notifier_call)
1989                 hnae_unregister_notifier(&priv->notifier_block);
1990         priv->notifier_block.notifier_call = NULL;
1991
1992         set_bit(NIC_STATE_REMOVING, &priv->state);
1993         (void)cancel_work_sync(&priv->service_task);
1994
1995         free_netdev(ndev);
1996         return 0;
1997 }
1998
1999 static const struct of_device_id hns_enet_of_match[] = {
2000         {.compatible = "hisilicon,hns-nic-v1",},
2001         {.compatible = "hisilicon,hns-nic-v2",},
2002         {},
2003 };
2004
2005 MODULE_DEVICE_TABLE(of, hns_enet_of_match);
2006
2007 static struct platform_driver hns_nic_dev_driver = {
2008         .driver = {
2009                 .name = "hns-nic",
2010                 .of_match_table = hns_enet_of_match,
2011         },
2012         .probe = hns_nic_dev_probe,
2013         .remove = hns_nic_dev_remove,
2014 };
2015
2016 module_platform_driver(hns_nic_dev_driver);
2017
2018 MODULE_DESCRIPTION("HISILICON HNS Ethernet driver");
2019 MODULE_AUTHOR("Hisilicon, Inc.");
2020 MODULE_LICENSE("GPL");
2021 MODULE_ALIAS("platform:hns-nic");