drivers/net/ethernet/mellanox/mlx4/en_netdev.c

   1 /*
   2  * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/etherdevice.h>
  35 #include <linux/tcp.h>
  36 #include <linux/if_vlan.h>
  37 #include <linux/delay.h>
  38 #include <linux/slab.h>
  39 #include <linux/hash.h>
  40 #include <net/ip.h>
  41 #include <net/busy_poll.h>
  42 #include <net/vxlan.h>
  43
  44 #include <linux/mlx4/driver.h>
  45 #include <linux/mlx4/device.h>
  46 #include <linux/mlx4/cmd.h>
  47 #include <linux/mlx4/cq.h>
  48
  49 #include "mlx4_en.h"
  50 #include "en_port.h"
  51
  52 int mlx4_en_setup_tc(struct net_device *dev, u8 up)
  53 {
  54         struct mlx4_en_priv *priv = netdev_priv(dev);
  55         int i;
  56         unsigned int offset = 0;
  57
  58         if (up && up != MLX4_EN_NUM_UP)
  59                 return -EINVAL;
  60
  61         netdev_set_num_tc(dev, up);
  62
  63         /* Partition Tx queues evenly amongst UP's */
  64         for (i = 0; i < up; i++) {
  65                 netdev_set_tc_queue(dev, i, priv->num_tx_rings_p_up, offset);
  66                 offset += priv->num_tx_rings_p_up;
  67         }
  68
  69         return 0;
  70 }
  71
  72 static int __mlx4_en_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
  73                               struct tc_to_netdev *tc)
  74 {
  75         if (handle != TC_H_ROOT || tc->type != TC_SETUP_MQPRIO)
  76                 return -EINVAL;
  77
  78         return mlx4_en_setup_tc(dev, tc->tc);
  79 }
  80
  81 #ifdef CONFIG_RFS_ACCEL
  82
  83 struct mlx4_en_filter {
  84         struct list_head next;
  85         struct work_struct work;
  86
  87         u8     ip_proto;
  88         __be32 src_ip;
  89         __be32 dst_ip;
  90         __be16 src_port;
  91         __be16 dst_port;
  92
  93         int rxq_index;
  94         struct mlx4_en_priv *priv;
  95         u32 flow_id;                    /* RFS infrastructure id */
  96         int id;                         /* mlx4_en driver id */
  97         u64 reg_id;                     /* Flow steering API id */
  98         u8 activated;                   /* Used to prevent expiry before filter
  99                                          * is attached
 100                                          */
 101         struct hlist_node filter_chain;
 102 };
 103
 104 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv);
 105
 106 static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto)
 107 {
 108         switch (ip_proto) {
 109         case IPPROTO_UDP:
 110                 return MLX4_NET_TRANS_RULE_ID_UDP;
 111         case IPPROTO_TCP:
 112                 return MLX4_NET_TRANS_RULE_ID_TCP;
 113         default:
 114                 return MLX4_NET_TRANS_RULE_NUM;
 115         }
 116 };
 117
 118 static void mlx4_en_filter_work(struct work_struct *work)
 119 {
 120         struct mlx4_en_filter *filter = container_of(work,
 121                                                      struct mlx4_en_filter,
 122                                                      work);
 123         struct mlx4_en_priv *priv = filter->priv;
 124         struct mlx4_spec_list spec_tcp_udp = {
 125                 .id = mlx4_ip_proto_to_trans_rule_id(filter->ip_proto),
 126                 {
 127                         .tcp_udp = {
 128                                 .dst_port = filter->dst_port,
 129                                 .dst_port_msk = (__force __be16)-1,
 130                                 .src_port = filter->src_port,
 131                                 .src_port_msk = (__force __be16)-1,
 132                         },
 133                 },
 134         };
 135         struct mlx4_spec_list spec_ip = {
 136                 .id = MLX4_NET_TRANS_RULE_ID_IPV4,
 137                 {
 138                         .ipv4 = {
 139                                 .dst_ip = filter->dst_ip,
 140                                 .dst_ip_msk = (__force __be32)-1,
 141                                 .src_ip = filter->src_ip,
 142                                 .src_ip_msk = (__force __be32)-1,
 143                         },
 144                 },
 145         };
 146         struct mlx4_spec_list spec_eth = {
 147                 .id = MLX4_NET_TRANS_RULE_ID_ETH,
 148         };
 149         struct mlx4_net_trans_rule rule = {
 150                 .list = LIST_HEAD_INIT(rule.list),
 151                 .queue_mode = MLX4_NET_TRANS_Q_LIFO,
 152                 .exclusive = 1,
 153                 .allow_loopback = 1,
 154                 .promisc_mode = MLX4_FS_REGULAR,
 155                 .port = priv->port,
 156                 .priority = MLX4_DOMAIN_RFS,
 157         };
 158         int rc;
 159         __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
 160
 161         if (spec_tcp_udp.id >= MLX4_NET_TRANS_RULE_NUM) {
 162                 en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n",
 163                         filter->ip_proto);
 164                 goto ignore;
 165         }
 166         list_add_tail(&spec_eth.list, &rule.list);
 167         list_add_tail(&spec_ip.list, &rule.list);
 168         list_add_tail(&spec_tcp_udp.list, &rule.list);
 169
 170         rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn;
 171         memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN);
 172         memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
 173
 174         filter->activated = 0;
 175
 176         if (filter->reg_id) {
 177                 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id);
 178                 if (rc && rc != -ENOENT)
 179                         en_err(priv, "Error detaching flow. rc = %d\n", rc);
 180         }
 181
 182         rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id);
 183         if (rc)
 184                 en_err(priv, "Error attaching flow. err = %d\n", rc);
 185
 186 ignore:
 187         mlx4_en_filter_rfs_expire(priv);
 188
 189         filter->activated = 1;
 190 }
 191
 192 static inline struct hlist_head *
 193 filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip,
 194                    __be16 src_port, __be16 dst_port)
 195 {
 196         unsigned long l;
 197         int bucket_idx;
 198
 199         l = (__force unsigned long)src_port |
 200             ((__force unsigned long)dst_port << 2);
 201         l ^= (__force unsigned long)(src_ip ^ dst_ip);
 202
 203         bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT);
 204
 205         return &priv->filter_hash[bucket_idx];
 206 }
 207
 208 static struct mlx4_en_filter *
 209 mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip,
 210                      __be32 dst_ip, u8 ip_proto, __be16 src_port,
 211                      __be16 dst_port, u32 flow_id)
 212 {
 213         struct mlx4_en_filter *filter = NULL;
 214
 215         filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC);
 216         if (!filter)
 217                 return NULL;
 218
 219         filter->priv = priv;
 220         filter->rxq_index = rxq_index;
 221         INIT_WORK(&filter->work, mlx4_en_filter_work);
 222
 223         filter->src_ip = src_ip;
 224         filter->dst_ip = dst_ip;
 225         filter->ip_proto = ip_proto;
 226         filter->src_port = src_port;
 227         filter->dst_port = dst_port;
 228
 229         filter->flow_id = flow_id;
 230
 231         filter->id = priv->last_filter_id++ % RPS_NO_FILTER;
 232
 233         list_add_tail(&filter->next, &priv->filters);
 234         hlist_add_head(&filter->filter_chain,
 235                        filter_hash_bucket(priv, src_ip, dst_ip, src_port,
 236                                           dst_port));
 237
 238         return filter;
 239 }
 240
 241 static void mlx4_en_filter_free(struct mlx4_en_filter *filter)
 242 {
 243         struct mlx4_en_priv *priv = filter->priv;
 244         int rc;
 245
 246         list_del(&filter->next);
 247
 248         rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id);
 249         if (rc && rc != -ENOENT)
 250                 en_err(priv, "Error detaching flow. rc = %d\n", rc);
 251
 252         kfree(filter);
 253 }
 254
 255 static inline struct mlx4_en_filter *
 256 mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip,
 257                     u8 ip_proto, __be16 src_port, __be16 dst_port)
 258 {
 259         struct mlx4_en_filter *filter;
 260         struct mlx4_en_filter *ret = NULL;
 261
 262         hlist_for_each_entry(filter,
 263                              filter_hash_bucket(priv, src_ip, dst_ip,
 264                                                 src_port, dst_port),
 265                              filter_chain) {
 266                 if (filter->src_ip == src_ip &&
 267                     filter->dst_ip == dst_ip &&
 268                     filter->ip_proto == ip_proto &&
 269                     filter->src_port == src_port &&
 270                     filter->dst_port == dst_port) {
 271                         ret = filter;
 272                         break;
 273                 }
 274         }
 275
 276         return ret;
 277 }
 278
 279 static int
 280 mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
 281                    u16 rxq_index, u32 flow_id)
 282 {
 283         struct mlx4_en_priv *priv = netdev_priv(net_dev);
 284         struct mlx4_en_filter *filter;
 285         const struct iphdr *ip;
 286         const __be16 *ports;
 287         u8 ip_proto;
 288         __be32 src_ip;
 289         __be32 dst_ip;
 290         __be16 src_port;
 291         __be16 dst_port;
 292         int nhoff = skb_network_offset(skb);
 293         int ret = 0;
 294
 295         if (skb->protocol != htons(ETH_P_IP))
 296                 return -EPROTONOSUPPORT;
 297
 298         ip = (const struct iphdr *)(skb->data + nhoff);
 299         if (ip_is_fragment(ip))
 300                 return -EPROTONOSUPPORT;
 301
 302         if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP))
 303                 return -EPROTONOSUPPORT;
 304         ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
 305
 306         ip_proto = ip->protocol;
 307         src_ip = ip->saddr;
 308         dst_ip = ip->daddr;
 309         src_port = ports[0];
 310         dst_port = ports[1];
 311
 312         spin_lock_bh(&priv->filters_lock);
 313         filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto,
 314                                      src_port, dst_port);
 315         if (filter) {
 316                 if (filter->rxq_index == rxq_index)
 317                         goto out;
 318
 319                 filter->rxq_index = rxq_index;
 320         } else {
 321                 filter = mlx4_en_filter_alloc(priv, rxq_index,
 322                                               src_ip, dst_ip, ip_proto,
 323                                               src_port, dst_port, flow_id);
 324                 if (!filter) {
 325                         ret = -ENOMEM;
 326                         goto err;
 327                 }
 328         }
 329
 330         queue_work(priv->mdev->workqueue, &filter->work);
 331
 332 out:
 333         ret = filter->id;
 334 err:
 335         spin_unlock_bh(&priv->filters_lock);
 336
 337         return ret;
 338 }
 339
 340 void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv)
 341 {
 342         struct mlx4_en_filter *filter, *tmp;
 343         LIST_HEAD(del_list);
 344
 345         spin_lock_bh(&priv->filters_lock);
 346         list_for_each_entry_safe(filter, tmp, &priv->filters, next) {
 347                 list_move(&filter->next, &del_list);
 348                 hlist_del(&filter->filter_chain);
 349         }
 350         spin_unlock_bh(&priv->filters_lock);
 351
 352         list_for_each_entry_safe(filter, tmp, &del_list, next) {
 353                 cancel_work_sync(&filter->work);
 354                 mlx4_en_filter_free(filter);
 355         }
 356 }
 357
 358 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv)
 359 {
 360         struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL;
 361         LIST_HEAD(del_list);
 362         int i = 0;
 363
 364         spin_lock_bh(&priv->filters_lock);
 365         list_for_each_entry_safe(filter, tmp, &priv->filters, next) {
 366                 if (i > MLX4_EN_FILTER_EXPIRY_QUOTA)
 367                         break;
 368
 369                 if (filter->activated &&
 370                     !work_pending(&filter->work) &&
 371                     rps_may_expire_flow(priv->dev,
 372                                         filter->rxq_index, filter->flow_id,
 373                                         filter->id)) {
 374                         list_move(&filter->next, &del_list);
 375                         hlist_del(&filter->filter_chain);
 376                 } else
 377                         last_filter = filter;
 378
 379                 i++;
 380         }
 381
 382         if (last_filter && (&last_filter->next != priv->filters.next))
 383                 list_move(&priv->filters, &last_filter->next);
 384
 385         spin_unlock_bh(&priv->filters_lock);
 386
 387         list_for_each_entry_safe(filter, tmp, &del_list, next)
 388                 mlx4_en_filter_free(filter);
 389 }
 390 #endif
 391
 392 static int mlx4_en_vlan_rx_add_vid(struct net_device *dev,
 393                                    __be16 proto, u16 vid)
 394 {
 395         struct mlx4_en_priv *priv = netdev_priv(dev);
 396         struct mlx4_en_dev *mdev = priv->mdev;
 397         int err;
 398         int idx;
 399
 400         en_dbg(HW, priv, "adding VLAN:%d\n", vid);
 401
 402         set_bit(vid, priv->active_vlans);
 403
 404         /* Add VID to port VLAN filter */
 405         mutex_lock(&mdev->state_lock);
 406         if (mdev->device_up && priv->port_up) {
 407                 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv);
 408                 if (err)
 409                         en_err(priv, "Failed configuring VLAN filter\n");
 410         }
 411         if (mlx4_register_vlan(mdev->dev, priv->port, vid, &idx))
 412                 en_dbg(HW, priv, "failed adding vlan %d\n", vid);
 413         mutex_unlock(&mdev->state_lock);
 414
 415         return 0;
 416 }
 417
 418 static int mlx4_en_vlan_rx_kill_vid(struct net_device *dev,
 419                                     __be16 proto, u16 vid)
 420 {
 421         struct mlx4_en_priv *priv = netdev_priv(dev);
 422         struct mlx4_en_dev *mdev = priv->mdev;
 423         int err;
 424
 425         en_dbg(HW, priv, "Killing VID:%d\n", vid);
 426
 427         clear_bit(vid, priv->active_vlans);
 428
 429         /* Remove VID from port VLAN filter */
 430         mutex_lock(&mdev->state_lock);
 431         mlx4_unregister_vlan(mdev->dev, priv->port, vid);
 432
 433         if (mdev->device_up && priv->port_up) {
 434                 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv);
 435                 if (err)
 436                         en_err(priv, "Failed configuring VLAN filter\n");
 437         }
 438         mutex_unlock(&mdev->state_lock);
 439
 440         return 0;
 441 }
 442
 443 static void mlx4_en_u64_to_mac(unsigned char dst_mac[ETH_ALEN + 2], u64 src_mac)
 444 {
 445         int i;
 446         for (i = ETH_ALEN - 1; i >= 0; --i) {
 447                 dst_mac[i] = src_mac & 0xff;
 448                 src_mac >>= 8;
 449         }
 450         memset(&dst_mac[ETH_ALEN], 0, 2);
 451 }
 452
 453
 454 static int mlx4_en_tunnel_steer_add(struct mlx4_en_priv *priv, unsigned char *addr,
 455                                     int qpn, u64 *reg_id)
 456 {
 457         int err;
 458
 459         if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN ||
 460             priv->mdev->dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC)
 461                 return 0; /* do nothing */
 462
 463         err = mlx4_tunnel_steer_add(priv->mdev->dev, addr, priv->port, qpn,
 464                                     MLX4_DOMAIN_NIC, reg_id);
 465         if (err) {
 466                 en_err(priv, "failed to add vxlan steering rule, err %d\n", err);
 467                 return err;
 468         }
 469         en_dbg(DRV, priv, "added vxlan steering rule, mac %pM reg_id %llx\n", addr, *reg_id);
 470         return 0;
 471 }
 472
 473
 474 static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv,
 475                                 unsigned char *mac, int *qpn, u64 *reg_id)
 476 {
 477         struct mlx4_en_dev *mdev = priv->mdev;
 478         struct mlx4_dev *dev = mdev->dev;
 479         int err;
 480
 481         switch (dev->caps.steering_mode) {
 482         case MLX4_STEERING_MODE_B0: {
 483                 struct mlx4_qp qp;
 484                 u8 gid[16] = {0};
 485
 486                 qp.qpn = *qpn;
 487                 memcpy(&gid[10], mac, ETH_ALEN);
 488                 gid[5] = priv->port;
 489
 490                 err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH);
 491                 break;
 492         }
 493         case MLX4_STEERING_MODE_DEVICE_MANAGED: {
 494                 struct mlx4_spec_list spec_eth = { {NULL} };
 495                 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16);
 496
 497                 struct mlx4_net_trans_rule rule = {
 498                         .queue_mode = MLX4_NET_TRANS_Q_FIFO,
 499                         .exclusive = 0,
 500                         .allow_loopback = 1,
 501                         .promisc_mode = MLX4_FS_REGULAR,
 502                         .priority = MLX4_DOMAIN_NIC,
 503                 };
 504
 505                 rule.port = priv->port;
 506                 rule.qpn = *qpn;
 507                 INIT_LIST_HEAD(&rule.list);
 508
 509                 spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH;
 510                 memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN);
 511                 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN);
 512                 list_add_tail(&spec_eth.list, &rule.list);
 513
 514                 err = mlx4_flow_attach(dev, &rule, reg_id);
 515                 break;
 516         }
 517         default:
 518                 return -EINVAL;
 519         }
 520         if (err)
 521                 en_warn(priv, "Failed Attaching Unicast\n");
 522
 523         return err;
 524 }
 525
 526 static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv,
 527                                      unsigned char *mac, int qpn, u64 reg_id)
 528 {
 529         struct mlx4_en_dev *mdev = priv->mdev;
 530         struct mlx4_dev *dev = mdev->dev;
 531
 532         switch (dev->caps.steering_mode) {
 533         case MLX4_STEERING_MODE_B0: {
 534                 struct mlx4_qp qp;
 535                 u8 gid[16] = {0};
 536
 537                 qp.qpn = qpn;
 538                 memcpy(&gid[10], mac, ETH_ALEN);
 539                 gid[5] = priv->port;
 540
 541                 mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH);
 542                 break;
 543         }
 544         case MLX4_STEERING_MODE_DEVICE_MANAGED: {
 545                 mlx4_flow_detach(dev, reg_id);
 546                 break;
 547         }
 548         default:
 549                 en_err(priv, "Invalid steering mode.\n");
 550         }
 551 }
 552
 553 static int mlx4_en_get_qp(struct mlx4_en_priv *priv)
 554 {
 555         struct mlx4_en_dev *mdev = priv->mdev;
 556         struct mlx4_dev *dev = mdev->dev;
 557         int index = 0;
 558         int err = 0;
 559         int *qpn = &priv->base_qpn;
 560         u64 mac = mlx4_mac_to_u64(priv->dev->dev_addr);
 561
 562         en_dbg(DRV, priv, "Registering MAC: %pM for adding\n",
 563                priv->dev->dev_addr);
 564         index = mlx4_register_mac(dev, priv->port, mac);
 565         if (index < 0) {
 566                 err = index;
 567                 en_err(priv, "Failed adding MAC: %pM\n",
 568                        priv->dev->dev_addr);
 569                 return err;
 570         }
 571
 572         if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) {
 573                 int base_qpn = mlx4_get_base_qpn(dev, priv->port);
 574                 *qpn = base_qpn + index;
 575                 return 0;
 576         }
 577
 578         err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP);
 579         en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);
 580         if (err) {
 581                 en_err(priv, "Failed to reserve qp for mac registration\n");
 582                 mlx4_unregister_mac(dev, priv->port, mac);
 583                 return err;
 584         }
 585
 586         return 0;
 587 }
 588
 589 static void mlx4_en_put_qp(struct mlx4_en_priv *priv)
 590 {
 591         struct mlx4_en_dev *mdev = priv->mdev;
 592         struct mlx4_dev *dev = mdev->dev;
 593         int qpn = priv->base_qpn;
 594
 595         if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) {
 596                 u64 mac = mlx4_mac_to_u64(priv->dev->dev_addr);
 597                 en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n",
 598                        priv->dev->dev_addr);
 599                 mlx4_unregister_mac(dev, priv->port, mac);
 600         } else {
 601                 en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n",
 602                        priv->port, qpn);
 603                 mlx4_qp_release_range(dev, qpn, 1);
 604                 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC;
 605         }
 606 }
 607
 608 static int mlx4_en_replace_mac(struct mlx4_en_priv *priv, int qpn,
 609                                unsigned char *new_mac, unsigned char *prev_mac)
 610 {
 611         struct mlx4_en_dev *mdev = priv->mdev;
 612         struct mlx4_dev *dev = mdev->dev;
 613         int err = 0;
 614         u64 new_mac_u64 = mlx4_mac_to_u64(new_mac);
 615
 616         if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) {
 617                 struct hlist_head *bucket;
 618                 unsigned int mac_hash;
 619                 struct mlx4_mac_entry *entry;
 620                 struct hlist_node *tmp;
 621                 u64 prev_mac_u64 = mlx4_mac_to_u64(prev_mac);
 622
 623                 bucket = &priv->mac_hash[prev_mac[MLX4_EN_MAC_HASH_IDX]];
 624                 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) {
 625                         if (ether_addr_equal_64bits(entry->mac, prev_mac)) {
 626                                 mlx4_en_uc_steer_release(priv, entry->mac,
 627                                                          qpn, entry->reg_id);
 628                                 mlx4_unregister_mac(dev, priv->port,
 629                                                     prev_mac_u64);
 630                                 hlist_del_rcu(&entry->hlist);
 631                                 synchronize_rcu();
 632                                 memcpy(entry->mac, new_mac, ETH_ALEN);
 633                                 entry->reg_id = 0;
 634                                 mac_hash = new_mac[MLX4_EN_MAC_HASH_IDX];
 635                                 hlist_add_head_rcu(&entry->hlist,
 636                                                    &priv->mac_hash[mac_hash]);
 637                                 mlx4_register_mac(dev, priv->port, new_mac_u64);
 638                                 err = mlx4_en_uc_steer_add(priv, new_mac,
 639                                                            &qpn,
 640                                                            &entry->reg_id);
 641                                 if (err)
 642                                         return err;
 643                                 if (priv->tunnel_reg_id) {
 644                                         mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id);
 645                                         priv->tunnel_reg_id = 0;
 646                                 }
 647                                 err = mlx4_en_tunnel_steer_add(priv, new_mac, qpn,
 648                                                                &priv->tunnel_reg_id);
 649                                 return err;
 650                         }
 651                 }
 652                 return -EINVAL;
 653         }
 654
 655         return __mlx4_replace_mac(dev, priv->port, qpn, new_mac_u64);
 656 }
 657
 658 static int mlx4_en_do_set_mac(struct mlx4_en_priv *priv,
 659                               unsigned char new_mac[ETH_ALEN + 2])
 660 {
 661         int err = 0;
 662
 663         if (priv->port_up) {
 664                 /* Remove old MAC and insert the new one */
 665                 err = mlx4_en_replace_mac(priv, priv->base_qpn,
 666                                           new_mac, priv->current_mac);
 667                 if (err)
 668                         en_err(priv, "Failed changing HW MAC address\n");
 669         } else
 670                 en_dbg(HW, priv, "Port is down while registering mac, exiting...\n");
 671
 672         if (!err)
 673                 memcpy(priv->current_mac, new_mac, sizeof(priv->current_mac));
 674
 675         return err;
 676 }
 677
 678 static int mlx4_en_set_mac(struct net_device *dev, void *addr)
 679 {
 680         struct mlx4_en_priv *priv = netdev_priv(dev);
 681         struct mlx4_en_dev *mdev = priv->mdev;
 682         struct sockaddr *saddr = addr;
 683         unsigned char new_mac[ETH_ALEN + 2];
 684         int err;
 685
 686         if (!is_valid_ether_addr(saddr->sa_data))
 687                 return -EADDRNOTAVAIL;
 688
 689         mutex_lock(&mdev->state_lock);
 690         memcpy(new_mac, saddr->sa_data, ETH_ALEN);
 691         err = mlx4_en_do_set_mac(priv, new_mac);
 692         if (!err)
 693                 memcpy(dev->dev_addr, saddr->sa_data, ETH_ALEN);
 694         mutex_unlock(&mdev->state_lock);
 695
 696         return err;
 697 }
 698
 699 static void mlx4_en_clear_list(struct net_device *dev)
 700 {
 701         struct mlx4_en_priv *priv = netdev_priv(dev);
 702         struct mlx4_en_mc_list *tmp, *mc_to_del;
 703
 704         list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) {
 705                 list_del(&mc_to_del->list);
 706                 kfree(mc_to_del);
 707         }
 708 }
 709
 710 static void mlx4_en_cache_mclist(struct net_device *dev)
 711 {
 712         struct mlx4_en_priv *priv = netdev_priv(dev);
 713         struct netdev_hw_addr *ha;
 714         struct mlx4_en_mc_list *tmp;
 715
 716         mlx4_en_clear_list(dev);
 717         netdev_for_each_mc_addr(ha, dev) {
 718                 tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC);
 719                 if (!tmp) {
 720                         mlx4_en_clear_list(dev);
 721                         return;
 722                 }
 723                 memcpy(tmp->addr, ha->addr, ETH_ALEN);
 724                 list_add_tail(&tmp->list, &priv->mc_list);
 725         }
 726 }
 727
 728 static void update_mclist_flags(struct mlx4_en_priv *priv,
 729                                 struct list_head *dst,
 730                                 struct list_head *src)
 731 {
 732         struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc;
 733         bool found;
 734
 735         /* Find all the entries that should be removed from dst,
 736          * These are the entries that are not found in src
 737          */
 738         list_for_each_entry(dst_tmp, dst, list) {
 739                 found = false;
 740                 list_for_each_entry(src_tmp, src, list) {
 741                         if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) {
 742                                 found = true;
 743                                 break;
 744                         }
 745                 }
 746                 if (!found)
 747                         dst_tmp->action = MCLIST_REM;
 748         }
 749
 750         /* Add entries that exist in src but not in dst
 751          * mark them as need to add
 752          */
 753         list_for_each_entry(src_tmp, src, list) {
 754                 found = false;
 755                 list_for_each_entry(dst_tmp, dst, list) {
 756                         if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) {
 757                                 dst_tmp->action = MCLIST_NONE;
 758                                 found = true;
 759                                 break;
 760                         }
 761                 }
 762                 if (!found) {
 763                         new_mc = kmemdup(src_tmp,
 764                                          sizeof(struct mlx4_en_mc_list),
 765                                          GFP_KERNEL);
 766                         if (!new_mc)
 767                                 return;
 768
 769                         new_mc->action = MCLIST_ADD;
 770                         list_add_tail(&new_mc->list, dst);
 771                 }
 772         }
 773 }
 774
 775 static void mlx4_en_set_rx_mode(struct net_device *dev)
 776 {
 777         struct mlx4_en_priv *priv = netdev_priv(dev);
 778
 779         if (!priv->port_up)
 780                 return;
 781
 782         queue_work(priv->mdev->workqueue, &priv->rx_mode_task);
 783 }
 784
 785 static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv,
 786                                      struct mlx4_en_dev *mdev)
 787 {
 788         int err = 0;
 789
 790         if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) {
 791                 if (netif_msg_rx_status(priv))
 792                         en_warn(priv, "Entering promiscuous mode\n");
 793                 priv->flags |= MLX4_EN_FLAG_PROMISC;
 794
 795                 /* Enable promiscouos mode */
 796                 switch (mdev->dev->caps.steering_mode) {
 797                 case MLX4_STEERING_MODE_DEVICE_MANAGED:
 798                         err = mlx4_flow_steer_promisc_add(mdev->dev,
 799                                                           priv->port,
 800                                                           priv->base_qpn,
 801                                                           MLX4_FS_ALL_DEFAULT);
 802                         if (err)
 803                                 en_err(priv, "Failed enabling promiscuous mode\n");
 804                         priv->flags |= MLX4_EN_FLAG_MC_PROMISC;
 805                         break;
 806
 807                 case MLX4_STEERING_MODE_B0:
 808                         err = mlx4_unicast_promisc_add(mdev->dev,
 809                                                        priv->base_qpn,
 810                                                        priv->port);
 811                         if (err)
 812                                 en_err(priv, "Failed enabling unicast promiscuous mode\n");
 813
 814                         /* Add the default qp number as multicast
 815                          * promisc
 816                          */
 817                         if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) {
 818                                 err = mlx4_multicast_promisc_add(mdev->dev,
 819                                                                  priv->base_qpn,
 820                                                                  priv->port);
 821                                 if (err)
 822                                         en_err(priv, "Failed enabling multicast promiscuous mode\n");
 823                                 priv->flags |= MLX4_EN_FLAG_MC_PROMISC;
 824                         }
 825                         break;
 826
 827                 case MLX4_STEERING_MODE_A0:
 828                         err = mlx4_SET_PORT_qpn_calc(mdev->dev,
 829                                                      priv->port,
 830                                                      priv->base_qpn,
 831                                                      1);
 832                         if (err)
 833                                 en_err(priv, "Failed enabling promiscuous mode\n");
 834                         break;
 835                 }
 836
 837                 /* Disable port multicast filter (unconditionally) */
 838                 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
 839                                           0, MLX4_MCAST_DISABLE);
 840                 if (err)
 841                         en_err(priv, "Failed disabling multicast filter\n");
 842         }
 843 }
 844
 845 static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv,
 846                                        struct mlx4_en_dev *mdev)
 847 {
 848         int err = 0;
 849
 850         if (netif_msg_rx_status(priv))
 851                 en_warn(priv, "Leaving promiscuous mode\n");
 852         priv->flags &= ~MLX4_EN_FLAG_PROMISC;
 853
 854         /* Disable promiscouos mode */
 855         switch (mdev->dev->caps.steering_mode) {
 856         case MLX4_STEERING_MODE_DEVICE_MANAGED:
 857                 err = mlx4_flow_steer_promisc_remove(mdev->dev,
 858                                                      priv->port,
 859                                                      MLX4_FS_ALL_DEFAULT);
 860                 if (err)
 861                         en_err(priv, "Failed disabling promiscuous mode\n");
 862                 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
 863                 break;
 864
 865         case MLX4_STEERING_MODE_B0:
 866                 err = mlx4_unicast_promisc_remove(mdev->dev,
 867                                                   priv->base_qpn,
 868                                                   priv->port);
 869                 if (err)
 870                         en_err(priv, "Failed disabling unicast promiscuous mode\n");
 871                 /* Disable Multicast promisc */
 872                 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) {
 873                         err = mlx4_multicast_promisc_remove(mdev->dev,
 874                                                             priv->base_qpn,
 875                                                             priv->port);
 876                         if (err)
 877                                 en_err(priv, "Failed disabling multicast promiscuous mode\n");
 878                         priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
 879                 }
 880                 break;
 881
 882         case MLX4_STEERING_MODE_A0:
 883                 err = mlx4_SET_PORT_qpn_calc(mdev->dev,
 884                                              priv->port,
 885                                              priv->base_qpn, 0);
 886                 if (err)
 887                         en_err(priv, "Failed disabling promiscuous mode\n");
 888                 break;
 889         }
 890 }
 891
 892 static void mlx4_en_do_multicast(struct mlx4_en_priv *priv,
 893                                  struct net_device *dev,
 894                                  struct mlx4_en_dev *mdev)
 895 {
 896         struct mlx4_en_mc_list *mclist, *tmp;
 897         u64 mcast_addr = 0;
 898         u8 mc_list[16] = {0};
 899         int err = 0;
 900
 901         /* Enable/disable the multicast filter according to IFF_ALLMULTI */
 902         if (dev->flags & IFF_ALLMULTI) {
 903                 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
 904                                           0, MLX4_MCAST_DISABLE);
 905                 if (err)
 906                         en_err(priv, "Failed disabling multicast filter\n");
 907
 908                 /* Add the default qp number as multicast promisc */
 909                 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) {
 910                         switch (mdev->dev->caps.steering_mode) {
 911                         case MLX4_STEERING_MODE_DEVICE_MANAGED:
 912                                 err = mlx4_flow_steer_promisc_add(mdev->dev,
 913                                                                   priv->port,
 914                                                                   priv->base_qpn,
 915                                                                   MLX4_FS_MC_DEFAULT);
 916                                 break;
 917
 918                         case MLX4_STEERING_MODE_B0:
 919                                 err = mlx4_multicast_promisc_add(mdev->dev,
 920                                                                  priv->base_qpn,
 921                                                                  priv->port);
 922                                 break;
 923
 924                         case MLX4_STEERING_MODE_A0:
 925                                 break;
 926                         }
 927                         if (err)
 928                                 en_err(priv, "Failed entering multicast promisc mode\n");
 929                         priv->flags |= MLX4_EN_FLAG_MC_PROMISC;
 930                 }
 931         } else {
 932                 /* Disable Multicast promisc */
 933                 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) {
 934                         switch (mdev->dev->caps.steering_mode) {
 935                         case MLX4_STEERING_MODE_DEVICE_MANAGED:
 936                                 err = mlx4_flow_steer_promisc_remove(mdev->dev,
 937                                                                      priv->port,
 938                                                                      MLX4_FS_MC_DEFAULT);
 939                                 break;
 940
 941                         case MLX4_STEERING_MODE_B0:
 942                                 err = mlx4_multicast_promisc_remove(mdev->dev,
 943                                                                     priv->base_qpn,
 944                                                                     priv->port);
 945                                 break;
 946
 947                         case MLX4_STEERING_MODE_A0:
 948                                 break;
 949                         }
 950                         if (err)
 951                                 en_err(priv, "Failed disabling multicast promiscuous mode\n");
 952                         priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
 953                 }
 954
 955                 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
 956                                           0, MLX4_MCAST_DISABLE);
 957                 if (err)
 958                         en_err(priv, "Failed disabling multicast filter\n");
 959
 960                 /* Flush mcast filter and init it with broadcast address */
 961                 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST,
 962                                     1, MLX4_MCAST_CONFIG);
 963
 964                 /* Update multicast list - we cache all addresses so they won't
 965                  * change while HW is updated holding the command semaphor */
 966                 netif_addr_lock_bh(dev);
 967                 mlx4_en_cache_mclist(dev);
 968                 netif_addr_unlock_bh(dev);
 969                 list_for_each_entry(mclist, &priv->mc_list, list) {
 970                         mcast_addr = mlx4_mac_to_u64(mclist->addr);
 971                         mlx4_SET_MCAST_FLTR(mdev->dev, priv->port,
 972                                             mcast_addr, 0, MLX4_MCAST_CONFIG);
 973                 }
 974                 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0,
 975                                           0, MLX4_MCAST_ENABLE);
 976                 if (err)
 977                         en_err(priv, "Failed enabling multicast filter\n");
 978
 979                 update_mclist_flags(priv, &priv->curr_list, &priv->mc_list);
 980                 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) {
 981                         if (mclist->action == MCLIST_REM) {
 982                                 /* detach this address and delete from list */
 983                                 memcpy(&mc_list[10], mclist->addr, ETH_ALEN);
 984                                 mc_list[5] = priv->port;
 985                                 err = mlx4_multicast_detach(mdev->dev,
 986                                                             &priv->rss_map.indir_qp,
 987                                                             mc_list,
 988                                                             MLX4_PROT_ETH,
 989                                                             mclist->reg_id);
 990                                 if (err)
 991                                         en_err(priv, "Fail to detach multicast address\n");
 992
 993                                 if (mclist->tunnel_reg_id) {
 994                                         err = mlx4_flow_detach(priv->mdev->dev, mclist->tunnel_reg_id);
 995                                         if (err)
 996                                                 en_err(priv, "Failed to detach multicast address\n");
 997                                 }
 998
 999                                 /* remove from list */
1000                                 list_del(&mclist->list);
1001                                 kfree(mclist);
1002                         } else if (mclist->action == MCLIST_ADD) {
1003                                 /* attach the address */
1004                                 memcpy(&mc_list[10], mclist->addr, ETH_ALEN);
1005                                 /* needed for B0 steering support */
1006                                 mc_list[5] = priv->port;
1007                                 err = mlx4_multicast_attach(mdev->dev,
1008                                                             &priv->rss_map.indir_qp,
1009                                                             mc_list,
1010                                                             priv->port, 0,
1011                                                             MLX4_PROT_ETH,
1012                                                             &mclist->reg_id);
1013                                 if (err)
1014                                         en_err(priv, "Fail to attach multicast address\n");
1015
1016                                 err = mlx4_en_tunnel_steer_add(priv, &mc_list[10], priv->base_qpn,
1017                                                                &mclist->tunnel_reg_id);
1018                                 if (err)
1019                                         en_err(priv, "Failed to attach multicast address\n");
1020                         }
1021                 }
1022         }
1023 }
1024
1025 static void mlx4_en_do_uc_filter(struct mlx4_en_priv *priv,
1026                                  struct net_device *dev,
1027                                  struct mlx4_en_dev *mdev)
1028 {
1029         struct netdev_hw_addr *ha;
1030         struct mlx4_mac_entry *entry;
1031         struct hlist_node *tmp;
1032         bool found;
1033         u64 mac;
1034         int err = 0;
1035         struct hlist_head *bucket;
1036         unsigned int i;
1037         int removed = 0;
1038         u32 prev_flags;
1039
1040         /* Note that we do not need to protect our mac_hash traversal with rcu,
1041          * since all modification code is protected by mdev->state_lock
1042          */
1043
1044         /* find what to remove */
1045         for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) {
1046                 bucket = &priv->mac_hash[i];
1047                 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) {
1048                         found = false;
1049                         netdev_for_each_uc_addr(ha, dev) {
1050                                 if (ether_addr_equal_64bits(entry->mac,
1051                                                             ha->addr)) {
1052                                         found = true;
1053                                         break;
1054                                 }
1055                         }
1056
1057                         /* MAC address of the port is not in uc list */
1058                         if (ether_addr_equal_64bits(entry->mac,
1059                                                     priv->current_mac))
1060                                 found = true;
1061
1062                         if (!found) {
1063                                 mac = mlx4_mac_to_u64(entry->mac);
1064                                 mlx4_en_uc_steer_release(priv, entry->mac,
1065                                                          priv->base_qpn,
1066                                                          entry->reg_id);
1067                                 mlx4_unregister_mac(mdev->dev, priv->port, mac);
1068
1069                                 hlist_del_rcu(&entry->hlist);
1070                                 kfree_rcu(entry, rcu);
1071                                 en_dbg(DRV, priv, "Removed MAC %pM on port:%d\n",
1072                                        entry->mac, priv->port);
1073                                 ++removed;
1074                         }
1075                 }
1076         }
1077
1078         /* if we didn't remove anything, there is no use in trying to add
1079          * again once we are in a forced promisc mode state
1080          */
1081         if ((priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) && 0 == removed)
1082                 return;
1083
1084         prev_flags = priv->flags;
1085         priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC;
1086
1087         /* find what to add */
1088         netdev_for_each_uc_addr(ha, dev) {
1089                 found = false;
1090                 bucket = &priv->mac_hash[ha->addr[MLX4_EN_MAC_HASH_IDX]];
1091                 hlist_for_each_entry(entry, bucket, hlist) {
1092                         if (ether_addr_equal_64bits(entry->mac, ha->addr)) {
1093                                 found = true;
1094                                 break;
1095                         }
1096                 }
1097
1098                 if (!found) {
1099                         entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1100                         if (!entry) {
1101                                 en_err(priv, "Failed adding MAC %pM on port:%d (out of memory)\n",
1102                                        ha->addr, priv->port);
1103                                 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC;
1104                                 break;
1105                         }
1106                         mac = mlx4_mac_to_u64(ha->addr);
1107                         memcpy(entry->mac, ha->addr, ETH_ALEN);
1108                         err = mlx4_register_mac(mdev->dev, priv->port, mac);
1109                         if (err < 0) {
1110                                 en_err(priv, "Failed registering MAC %pM on port %d: %d\n",
1111                                        ha->addr, priv->port, err);
1112                                 kfree(entry);
1113                                 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC;
1114                                 break;
1115                         }
1116                         err = mlx4_en_uc_steer_add(priv, ha->addr,
1117                                                    &priv->base_qpn,
1118                                                    &entry->reg_id);
1119                         if (err) {
1120                                 en_err(priv, "Failed adding MAC %pM on port %d: %d\n",
1121                                        ha->addr, priv->port, err);
1122                                 mlx4_unregister_mac(mdev->dev, priv->port, mac);
1123                                 kfree(entry);
1124                                 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC;
1125                                 break;
1126                         } else {
1127                                 unsigned int mac_hash;
1128                                 en_dbg(DRV, priv, "Added MAC %pM on port:%d\n",
1129                                        ha->addr, priv->port);
1130                                 mac_hash = ha->addr[MLX4_EN_MAC_HASH_IDX];
1131                                 bucket = &priv->mac_hash[mac_hash];
1132                                 hlist_add_head_rcu(&entry->hlist, bucket);
1133                         }
1134                 }
1135         }
1136
1137         if (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) {
1138                 en_warn(priv, "Forcing promiscuous mode on port:%d\n",
1139                         priv->port);
1140         } else if (prev_flags & MLX4_EN_FLAG_FORCE_PROMISC) {
1141                 en_warn(priv, "Stop forcing promiscuous mode on port:%d\n",
1142                         priv->port);
1143         }
1144 }
1145
1146 static void mlx4_en_do_set_rx_mode(struct work_struct *work)
1147 {
1148         struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
1149                                                  rx_mode_task);
1150         struct mlx4_en_dev *mdev = priv->mdev;
1151         struct net_device *dev = priv->dev;
1152
1153         mutex_lock(&mdev->state_lock);
1154         if (!mdev->device_up) {
1155                 en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n");
1156                 goto out;
1157         }
1158         if (!priv->port_up) {
1159                 en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n");
1160                 goto out;
1161         }
1162
1163         if (!netif_carrier_ok(dev)) {
1164                 if (!mlx4_en_QUERY_PORT(mdev, priv->port)) {
1165                         if (priv->port_state.link_state) {
1166                                 priv->last_link_state = MLX4_DEV_EVENT_PORT_UP;
1167                                 netif_carrier_on(dev);
1168                                 en_dbg(LINK, priv, "Link Up\n");
1169                         }
1170                 }
1171         }
1172
1173         if (dev->priv_flags & IFF_UNICAST_FLT)
1174                 mlx4_en_do_uc_filter(priv, dev, mdev);
1175
1176         /* Promsicuous mode: disable all filters */
1177         if ((dev->flags & IFF_PROMISC) ||
1178             (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) {
1179                 mlx4_en_set_promisc_mode(priv, mdev);
1180                 goto out;
1181         }
1182
1183         /* Not in promiscuous mode */
1184         if (priv->flags & MLX4_EN_FLAG_PROMISC)
1185                 mlx4_en_clear_promisc_mode(priv, mdev);
1186
1187         mlx4_en_do_multicast(priv, dev, mdev);
1188 out:
1189         mutex_unlock(&mdev->state_lock);
1190 }
1191
1192 #ifdef CONFIG_NET_POLL_CONTROLLER
1193 static void mlx4_en_netpoll(struct net_device *dev)
1194 {
1195         struct mlx4_en_priv *priv = netdev_priv(dev);
1196         struct mlx4_en_cq *cq;
1197         int i;
1198
1199         for (i = 0; i < priv->rx_ring_num; i++) {
1200                 cq = priv->rx_cq[i];
1201                 napi_schedule(&cq->napi);
1202         }
1203 }
1204 #endif
1205
1206 static int mlx4_en_set_rss_steer_rules(struct mlx4_en_priv *priv)
1207 {
1208         u64 reg_id;
1209         int err = 0;
1210         int *qpn = &priv->base_qpn;
1211         struct mlx4_mac_entry *entry;
1212
1213         err = mlx4_en_uc_steer_add(priv, priv->dev->dev_addr, qpn, &reg_id);
1214         if (err)
1215                 return err;
1216
1217         err = mlx4_en_tunnel_steer_add(priv, priv->dev->dev_addr, *qpn,
1218                                        &priv->tunnel_reg_id);
1219         if (err)
1220                 goto tunnel_err;
1221
1222         entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1223         if (!entry) {
1224                 err = -ENOMEM;
1225                 goto alloc_err;
1226         }
1227
1228         memcpy(entry->mac, priv->dev->dev_addr, sizeof(entry->mac));
1229         memcpy(priv->current_mac, entry->mac, sizeof(priv->current_mac));
1230         entry->reg_id = reg_id;
1231         hlist_add_head_rcu(&entry->hlist,
1232                            &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]);
1233
1234         return 0;
1235
1236 alloc_err:
1237         if (priv->tunnel_reg_id)
1238                 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id);
1239
1240 tunnel_err:
1241         mlx4_en_uc_steer_release(priv, priv->dev->dev_addr, *qpn, reg_id);
1242         return err;
1243 }
1244
1245 static void mlx4_en_delete_rss_steer_rules(struct mlx4_en_priv *priv)
1246 {
1247         u64 mac;
1248         unsigned int i;
1249         int qpn = priv->base_qpn;
1250         struct hlist_head *bucket;
1251         struct hlist_node *tmp;
1252         struct mlx4_mac_entry *entry;
1253
1254         for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) {
1255                 bucket = &priv->mac_hash[i];
1256                 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) {
1257                         mac = mlx4_mac_to_u64(entry->mac);
1258                         en_dbg(DRV, priv, "Registering MAC:%pM for deleting\n",
1259                                entry->mac);
1260                         mlx4_en_uc_steer_release(priv, entry->mac,
1261                                                  qpn, entry->reg_id);
1262
1263                         mlx4_unregister_mac(priv->mdev->dev, priv->port, mac);
1264                         hlist_del_rcu(&entry->hlist);
1265                         kfree_rcu(entry, rcu);
1266                 }
1267         }
1268
1269         if (priv->tunnel_reg_id) {
1270                 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id);
1271                 priv->tunnel_reg_id = 0;
1272         }
1273 }
1274
1275 static void mlx4_en_tx_timeout(struct net_device *dev)
1276 {
1277         struct mlx4_en_priv *priv = netdev_priv(dev);
1278         struct mlx4_en_dev *mdev = priv->mdev;
1279         int i;
1280
1281         if (netif_msg_timer(priv))
1282                 en_warn(priv, "Tx timeout called on port:%d\n", priv->port);
1283
1284         for (i = 0; i < priv->tx_ring_num; i++) {
1285                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, i)))
1286                         continue;
1287                 en_warn(priv, "TX timeout on queue: %d, QP: 0x%x, CQ: 0x%x, Cons: 0x%x, Prod: 0x%x\n",
1288                         i, priv->tx_ring[i]->qpn, priv->tx_ring[i]->cqn,
1289                         priv->tx_ring[i]->cons, priv->tx_ring[i]->prod);
1290         }
1291
1292         priv->port_stats.tx_timeout++;
1293         en_dbg(DRV, priv, "Scheduling watchdog\n");
1294         queue_work(mdev->workqueue, &priv->watchdog_task);
1295 }
1296
1297
1298 static struct net_device_stats *mlx4_en_get_stats(struct net_device *dev)
1299 {
1300         struct mlx4_en_priv *priv = netdev_priv(dev);
1301
1302         spin_lock_bh(&priv->stats_lock);
1303         memcpy(&priv->ret_stats, &priv->stats, sizeof(priv->stats));
1304         spin_unlock_bh(&priv->stats_lock);
1305
1306         return &priv->ret_stats;
1307 }
1308
1309 static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv)
1310 {
1311         struct mlx4_en_cq *cq;
1312         int i;
1313
1314         /* If we haven't received a specific coalescing setting
1315          * (module param), we set the moderation parameters as follows:
1316          * - moder_cnt is set to the number of mtu sized packets to
1317          *   satisfy our coalescing target.
1318          * - moder_time is set to a fixed value.
1319          */
1320         priv->rx_frames = MLX4_EN_RX_COAL_TARGET;
1321         priv->rx_usecs = MLX4_EN_RX_COAL_TIME;
1322         priv->tx_frames = MLX4_EN_TX_COAL_PKTS;
1323         priv->tx_usecs = MLX4_EN_TX_COAL_TIME;
1324         en_dbg(INTR, priv, "Default coalesing params for mtu:%d - rx_frames:%d rx_usecs:%d\n",
1325                priv->dev->mtu, priv->rx_frames, priv->rx_usecs);
1326
1327         /* Setup cq moderation params */
1328         for (i = 0; i < priv->rx_ring_num; i++) {
1329                 cq = priv->rx_cq[i];
1330                 cq->moder_cnt = priv->rx_frames;
1331                 cq->moder_time = priv->rx_usecs;
1332                 priv->last_moder_time[i] = MLX4_EN_AUTO_CONF;
1333                 priv->last_moder_packets[i] = 0;
1334                 priv->last_moder_bytes[i] = 0;
1335         }
1336
1337         for (i = 0; i < priv->tx_ring_num; i++) {
1338                 cq = priv->tx_cq[i];
1339                 cq->moder_cnt = priv->tx_frames;
1340                 cq->moder_time = priv->tx_usecs;
1341         }
1342
1343         /* Reset auto-moderation params */
1344         priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW;
1345         priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW;
1346         priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH;
1347         priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH;
1348         priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL;
1349         priv->adaptive_rx_coal = 1;
1350         priv->last_moder_jiffies = 0;
1351         priv->last_moder_tx_packets = 0;
1352 }
1353
1354 static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv)
1355 {
1356         unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies);
1357         struct mlx4_en_cq *cq;
1358         unsigned long packets;
1359         unsigned long rate;
1360         unsigned long avg_pkt_size;
1361         unsigned long rx_packets;
1362         unsigned long rx_bytes;
1363         unsigned long rx_pkt_diff;
1364         int moder_time;
1365         int ring, err;
1366
1367         if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ)
1368                 return;
1369
1370         for (ring = 0; ring < priv->rx_ring_num; ring++) {
1371                 spin_lock_bh(&priv->stats_lock);
1372                 rx_packets = priv->rx_ring[ring]->packets;
1373                 rx_bytes = priv->rx_ring[ring]->bytes;
1374                 spin_unlock_bh(&priv->stats_lock);
1375
1376                 rx_pkt_diff = ((unsigned long) (rx_packets -
1377                                 priv->last_moder_packets[ring]));
1378                 packets = rx_pkt_diff;
1379                 rate = packets * HZ / period;
1380                 avg_pkt_size = packets ? ((unsigned long) (rx_bytes -
1381                                 priv->last_moder_bytes[ring])) / packets : 0;
1382
1383                 /* Apply auto-moderation only when packet rate
1384                  * exceeds a rate that it matters */
1385                 if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) &&
1386                     avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) {
1387                         if (rate < priv->pkt_rate_low)
1388                                 moder_time = priv->rx_usecs_low;
1389                         else if (rate > priv->pkt_rate_high)
1390                                 moder_time = priv->rx_usecs_high;
1391                         else
1392                                 moder_time = (rate - priv->pkt_rate_low) *
1393                                         (priv->rx_usecs_high - priv->rx_usecs_low) /
1394                                         (priv->pkt_rate_high - priv->pkt_rate_low) +
1395                                         priv->rx_usecs_low;
1396                 } else {
1397                         moder_time = priv->rx_usecs_low;
1398                 }
1399
1400                 if (moder_time != priv->last_moder_time[ring]) {
1401                         priv->last_moder_time[ring] = moder_time;
1402                         cq = priv->rx_cq[ring];
1403                         cq->moder_time = moder_time;
1404                         cq->moder_cnt = priv->rx_frames;
1405                         err = mlx4_en_set_cq_moder(priv, cq);
1406                         if (err)
1407                                 en_err(priv, "Failed modifying moderation for cq:%d\n",
1408                                        ring);
1409                 }
1410                 priv->last_moder_packets[ring] = rx_packets;
1411                 priv->last_moder_bytes[ring] = rx_bytes;
1412         }
1413
1414         priv->last_moder_jiffies = jiffies;
1415 }
1416
1417 static void mlx4_en_do_get_stats(struct work_struct *work)
1418 {
1419         struct delayed_work *delay = to_delayed_work(work);
1420         struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv,
1421                                                  stats_task);
1422         struct mlx4_en_dev *mdev = priv->mdev;
1423         int err;
1424
1425         mutex_lock(&mdev->state_lock);
1426         if (mdev->device_up) {
1427                 if (priv->port_up) {
1428                         err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0);
1429                         if (err)
1430                                 en_dbg(HW, priv, "Could not update stats\n");
1431
1432                         mlx4_en_auto_moderation(priv);
1433                 }
1434
1435                 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY);
1436         }
1437         if (mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port]) {
1438                 mlx4_en_do_set_mac(priv, priv->current_mac);
1439                 mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port] = 0;
1440         }
1441         mutex_unlock(&mdev->state_lock);
1442 }
1443
1444 /* mlx4_en_service_task - Run service task for tasks that needed to be done
1445  * periodically
1446  */
1447 static void mlx4_en_service_task(struct work_struct *work)
1448 {
1449         struct delayed_work *delay = to_delayed_work(work);
1450         struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv,
1451                                                  service_task);
1452         struct mlx4_en_dev *mdev = priv->mdev;
1453
1454         mutex_lock(&mdev->state_lock);
1455         if (mdev->device_up) {
1456                 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)
1457                         mlx4_en_ptp_overflow_check(mdev);
1458
1459                 mlx4_en_recover_from_oom(priv);
1460                 queue_delayed_work(mdev->workqueue, &priv->service_task,
1461                                    SERVICE_TASK_DELAY);
1462         }
1463         mutex_unlock(&mdev->state_lock);
1464 }
1465
1466 static void mlx4_en_linkstate(struct work_struct *work)
1467 {
1468         struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
1469                                                  linkstate_task);
1470         struct mlx4_en_dev *mdev = priv->mdev;
1471         int linkstate = priv->link_state;
1472
1473         mutex_lock(&mdev->state_lock);
1474         /* If observable port state changed set carrier state and
1475          * report to system log */
1476         if (priv->last_link_state != linkstate) {
1477                 if (linkstate == MLX4_DEV_EVENT_PORT_DOWN) {
1478                         en_info(priv, "Link Down\n");
1479                         netif_carrier_off(priv->dev);
1480                 } else {
1481                         en_info(priv, "Link Up\n");
1482                         netif_carrier_on(priv->dev);
1483                 }
1484         }
1485         priv->last_link_state = linkstate;
1486         mutex_unlock(&mdev->state_lock);
1487 }
1488
1489 static int mlx4_en_init_affinity_hint(struct mlx4_en_priv *priv, int ring_idx)
1490 {
1491         struct mlx4_en_rx_ring *ring = priv->rx_ring[ring_idx];
1492         int numa_node = priv->mdev->dev->numa_node;
1493
1494         if (!zalloc_cpumask_var(&ring->affinity_mask, GFP_KERNEL))
1495                 return -ENOMEM;
1496
1497         cpumask_set_cpu(cpumask_local_spread(ring_idx, numa_node),
1498                         ring->affinity_mask);
1499         return 0;
1500 }
1501
1502 static void mlx4_en_free_affinity_hint(struct mlx4_en_priv *priv, int ring_idx)
1503 {
1504         free_cpumask_var(priv->rx_ring[ring_idx]->affinity_mask);
1505 }
1506
1507 int mlx4_en_start_port(struct net_device *dev)
1508 {
1509         struct mlx4_en_priv *priv = netdev_priv(dev);
1510         struct mlx4_en_dev *mdev = priv->mdev;
1511         struct mlx4_en_cq *cq;
1512         struct mlx4_en_tx_ring *tx_ring;
1513         int rx_index = 0;
1514         int tx_index = 0;
1515         int err = 0;
1516         int i;
1517         int j;
1518         u8 mc_list[16] = {0};
1519
1520         if (priv->port_up) {
1521                 en_dbg(DRV, priv, "start port called while port already up\n");
1522                 return 0;
1523         }
1524
1525         INIT_LIST_HEAD(&priv->mc_list);
1526         INIT_LIST_HEAD(&priv->curr_list);
1527         INIT_LIST_HEAD(&priv->ethtool_list);
1528         memset(&priv->ethtool_rules[0], 0,
1529                sizeof(struct ethtool_flow_id) * MAX_NUM_OF_FS_RULES);
1530
1531         /* Calculate Rx buf size */
1532         dev->mtu = min(dev->mtu, priv->max_mtu);
1533         mlx4_en_calc_rx_buf(dev);
1534         en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_skb_size);
1535
1536         /* Configure rx cq's and rings */
1537         err = mlx4_en_activate_rx_rings(priv);
1538         if (err) {
1539                 en_err(priv, "Failed to activate RX rings\n");
1540                 return err;
1541         }
1542         for (i = 0; i < priv->rx_ring_num; i++) {
1543                 cq = priv->rx_cq[i];
1544
1545                 err = mlx4_en_init_affinity_hint(priv, i);
1546                 if (err) {
1547                         en_err(priv, "Failed preparing IRQ affinity hint\n");
1548                         goto cq_err;
1549                 }
1550
1551                 err = mlx4_en_activate_cq(priv, cq, i);
1552                 if (err) {
1553                         en_err(priv, "Failed activating Rx CQ\n");
1554                         mlx4_en_free_affinity_hint(priv, i);
1555                         goto cq_err;
1556                 }
1557
1558                 for (j = 0; j < cq->size; j++) {
1559                         struct mlx4_cqe *cqe = NULL;
1560
1561                         cqe = mlx4_en_get_cqe(cq->buf, j, priv->cqe_size) +
1562                               priv->cqe_factor;
1563                         cqe->owner_sr_opcode = MLX4_CQE_OWNER_MASK;
1564                 }
1565
1566                 err = mlx4_en_set_cq_moder(priv, cq);
1567                 if (err) {
1568                         en_err(priv, "Failed setting cq moderation parameters\n");
1569                         mlx4_en_deactivate_cq(priv, cq);
1570                         mlx4_en_free_affinity_hint(priv, i);
1571                         goto cq_err;
1572                 }
1573                 mlx4_en_arm_cq(priv, cq);
1574                 priv->rx_ring[i]->cqn = cq->mcq.cqn;
1575                 ++rx_index;
1576         }
1577
1578         /* Set qp number */
1579         en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port);
1580         err = mlx4_en_get_qp(priv);
1581         if (err) {
1582                 en_err(priv, "Failed getting eth qp\n");
1583                 goto cq_err;
1584         }
1585         mdev->mac_removed[priv->port] = 0;
1586
1587         priv->counter_index =
1588                         mlx4_get_default_counter_index(mdev->dev, priv->port);
1589
1590         err = mlx4_en_config_rss_steer(priv);
1591         if (err) {
1592                 en_err(priv, "Failed configuring rss steering\n");
1593                 goto mac_err;
1594         }
1595
1596         err = mlx4_en_create_drop_qp(priv);
1597         if (err)
1598                 goto rss_err;
1599
1600         /* Configure tx cq's and rings */
1601         for (i = 0; i < priv->tx_ring_num; i++) {
1602                 /* Configure cq */
1603                 cq = priv->tx_cq[i];
1604                 err = mlx4_en_activate_cq(priv, cq, i);
1605                 if (err) {
1606                         en_err(priv, "Failed allocating Tx CQ\n");
1607                         goto tx_err;
1608                 }
1609                 err = mlx4_en_set_cq_moder(priv, cq);
1610                 if (err) {
1611                         en_err(priv, "Failed setting cq moderation parameters\n");
1612                         mlx4_en_deactivate_cq(priv, cq);
1613                         goto tx_err;
1614                 }
1615                 en_dbg(DRV, priv, "Resetting index of collapsed CQ:%d to -1\n", i);
1616                 cq->buf->wqe_index = cpu_to_be16(0xffff);
1617
1618                 /* Configure ring */
1619                 tx_ring = priv->tx_ring[i];
1620                 err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn,
1621                         i / priv->num_tx_rings_p_up);
1622                 if (err) {
1623                         en_err(priv, "Failed allocating Tx ring\n");
1624                         mlx4_en_deactivate_cq(priv, cq);
1625                         goto tx_err;
1626                 }
1627                 tx_ring->tx_queue = netdev_get_tx_queue(dev, i);
1628
1629                 /* Arm CQ for TX completions */
1630                 mlx4_en_arm_cq(priv, cq);
1631
1632                 /* Set initial ownership of all Tx TXBBs to SW (1) */
1633                 for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE)
1634                         *((u32 *) (tx_ring->buf + j)) = 0xffffffff;
1635                 ++tx_index;
1636         }
1637
1638         /* Configure port */
1639         err = mlx4_SET_PORT_general(mdev->dev, priv->port,
1640                                     priv->rx_skb_size + ETH_FCS_LEN,
1641                                     priv->prof->tx_pause,
1642                                     priv->prof->tx_ppp,
1643                                     priv->prof->rx_pause,
1644                                     priv->prof->rx_ppp);
1645         if (err) {
1646                 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n",
1647                        priv->port, err);
1648                 goto tx_err;
1649         }
1650         /* Set default qp number */
1651         err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0);
1652         if (err) {
1653                 en_err(priv, "Failed setting default qp numbers\n");
1654                 goto tx_err;
1655         }
1656
1657         if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
1658                 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1);
1659                 if (err) {
1660                         en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n",
1661                                err);
1662                         goto tx_err;
1663                 }
1664         }
1665
1666         /* Init port */
1667         en_dbg(HW, priv, "Initializing port\n");
1668         err = mlx4_INIT_PORT(mdev->dev, priv->port);
1669         if (err) {
1670                 en_err(priv, "Failed Initializing port\n");
1671                 goto tx_err;
1672         }
1673
1674         /* Set Unicast and VXLAN steering rules */
1675         if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0 &&
1676             mlx4_en_set_rss_steer_rules(priv))
1677                 mlx4_warn(mdev, "Failed setting steering rules\n");
1678
1679         /* Attach rx QP to bradcast address */
1680         eth_broadcast_addr(&mc_list[10]);
1681         mc_list[5] = priv->port; /* needed for B0 steering support */
1682         if (mlx4_multicast_attach(mdev->dev, &priv->rss_map.indir_qp, mc_list,
1683                                   priv->port, 0, MLX4_PROT_ETH,
1684                                   &priv->broadcast_id))
1685                 mlx4_warn(mdev, "Failed Attaching Broadcast\n");
1686
1687         /* Must redo promiscuous mode setup. */
1688         priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC);
1689
1690         /* Schedule multicast task to populate multicast list */
1691         queue_work(mdev->workqueue, &priv->rx_mode_task);
1692
1693 #ifdef CONFIG_MLX4_EN_VXLAN
1694         if (priv->mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
1695                 vxlan_get_rx_port(dev);
1696 #endif
1697         priv->port_up = true;
1698         netif_tx_start_all_queues(dev);
1699         netif_device_attach(dev);
1700
1701         return 0;
1702
1703 tx_err:
1704         while (tx_index--) {
1705                 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[tx_index]);
1706                 mlx4_en_deactivate_cq(priv, priv->tx_cq[tx_index]);
1707         }
1708         mlx4_en_destroy_drop_qp(priv);
1709 rss_err:
1710         mlx4_en_release_rss_steer(priv);
1711 mac_err:
1712         mlx4_en_put_qp(priv);
1713 cq_err:
1714         while (rx_index--) {
1715                 mlx4_en_deactivate_cq(priv, priv->rx_cq[rx_index]);
1716                 mlx4_en_free_affinity_hint(priv, rx_index);
1717         }
1718         for (i = 0; i < priv->rx_ring_num; i++)
1719                 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]);
1720
1721         return err; /* need to close devices */
1722 }
1723
1724
1725 void mlx4_en_stop_port(struct net_device *dev, int detach)
1726 {
1727         struct mlx4_en_priv *priv = netdev_priv(dev);
1728         struct mlx4_en_dev *mdev = priv->mdev;
1729         struct mlx4_en_mc_list *mclist, *tmp;
1730         struct ethtool_flow_id *flow, *tmp_flow;
1731         int i;
1732         u8 mc_list[16] = {0};
1733
1734         if (!priv->port_up) {
1735                 en_dbg(DRV, priv, "stop port called while port already down\n");
1736                 return;
1737         }
1738
1739         /* close port*/
1740         mlx4_CLOSE_PORT(mdev->dev, priv->port);
1741
1742         /* Synchronize with tx routine */
1743         netif_tx_lock_bh(dev);
1744         if (detach)
1745                 netif_device_detach(dev);
1746         netif_tx_stop_all_queues(dev);
1747         netif_tx_unlock_bh(dev);
1748
1749         netif_tx_disable(dev);
1750
1751         /* Set port as not active */
1752         priv->port_up = false;
1753         priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev);
1754
1755         /* Promsicuous mode */
1756         if (mdev->dev->caps.steering_mode ==
1757             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1758                 priv->flags &= ~(MLX4_EN_FLAG_PROMISC |
1759                                  MLX4_EN_FLAG_MC_PROMISC);
1760                 mlx4_flow_steer_promisc_remove(mdev->dev,
1761                                                priv->port,
1762                                                MLX4_FS_ALL_DEFAULT);
1763                 mlx4_flow_steer_promisc_remove(mdev->dev,
1764                                                priv->port,
1765                                                MLX4_FS_MC_DEFAULT);
1766         } else if (priv->flags & MLX4_EN_FLAG_PROMISC) {
1767                 priv->flags &= ~MLX4_EN_FLAG_PROMISC;
1768
1769                 /* Disable promiscouos mode */
1770                 mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn,
1771                                             priv->port);
1772
1773                 /* Disable Multicast promisc */
1774                 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) {
1775                         mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn,
1776                                                       priv->port);
1777                         priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC;
1778                 }
1779         }
1780
1781         /* Detach All multicasts */
1782         eth_broadcast_addr(&mc_list[10]);
1783         mc_list[5] = priv->port; /* needed for B0 steering support */
1784         mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, mc_list,
1785                               MLX4_PROT_ETH, priv->broadcast_id);
1786         list_for_each_entry(mclist, &priv->curr_list, list) {
1787                 memcpy(&mc_list[10], mclist->addr, ETH_ALEN);
1788                 mc_list[5] = priv->port;
1789                 mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp,
1790                                       mc_list, MLX4_PROT_ETH, mclist->reg_id);
1791                 if (mclist->tunnel_reg_id)
1792                         mlx4_flow_detach(mdev->dev, mclist->tunnel_reg_id);
1793         }
1794         mlx4_en_clear_list(dev);
1795         list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) {
1796                 list_del(&mclist->list);
1797                 kfree(mclist);
1798         }
1799
1800         /* Flush multicast filter */
1801         mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG);
1802
1803         /* Remove flow steering rules for the port*/
1804         if (mdev->dev->caps.steering_mode ==
1805             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1806                 ASSERT_RTNL();
1807                 list_for_each_entry_safe(flow, tmp_flow,
1808                                          &priv->ethtool_list, list) {
1809                         mlx4_flow_detach(mdev->dev, flow->id);
1810                         list_del(&flow->list);
1811                 }
1812         }
1813
1814         mlx4_en_destroy_drop_qp(priv);
1815
1816         /* Free TX Rings */
1817         for (i = 0; i < priv->tx_ring_num; i++) {
1818                 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[i]);
1819                 mlx4_en_deactivate_cq(priv, priv->tx_cq[i]);
1820         }
1821         msleep(10);
1822
1823         for (i = 0; i < priv->tx_ring_num; i++)
1824                 mlx4_en_free_tx_buf(dev, priv->tx_ring[i]);
1825
1826         if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0)
1827                 mlx4_en_delete_rss_steer_rules(priv);
1828
1829         /* Free RSS qps */
1830         mlx4_en_release_rss_steer(priv);
1831
1832         /* Unregister Mac address for the port */
1833         mlx4_en_put_qp(priv);
1834         if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN))
1835                 mdev->mac_removed[priv->port] = 1;
1836
1837         /* Free RX Rings */
1838         for (i = 0; i < priv->rx_ring_num; i++) {
1839                 struct mlx4_en_cq *cq = priv->rx_cq[i];
1840
1841                 napi_synchronize(&cq->napi);
1842                 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]);
1843                 mlx4_en_deactivate_cq(priv, cq);
1844
1845                 mlx4_en_free_affinity_hint(priv, i);
1846         }
1847 }
1848
1849 static void mlx4_en_restart(struct work_struct *work)
1850 {
1851         struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
1852                                                  watchdog_task);
1853         struct mlx4_en_dev *mdev = priv->mdev;
1854         struct net_device *dev = priv->dev;
1855
1856         en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port);
1857
1858         mutex_lock(&mdev->state_lock);
1859         if (priv->port_up) {
1860                 mlx4_en_stop_port(dev, 1);
1861                 if (mlx4_en_start_port(dev))
1862                         en_err(priv, "Failed restarting port %d\n", priv->port);
1863         }
1864         mutex_unlock(&mdev->state_lock);
1865 }
1866
1867 static void mlx4_en_clear_stats(struct net_device *dev)
1868 {
1869         struct mlx4_en_priv *priv = netdev_priv(dev);
1870         struct mlx4_en_dev *mdev = priv->mdev;
1871         int i;
1872
1873         if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1))
1874                 en_dbg(HW, priv, "Failed dumping statistics\n");
1875
1876         memset(&priv->stats, 0, sizeof(priv->stats));
1877         memset(&priv->pstats, 0, sizeof(priv->pstats));
1878         memset(&priv->pkstats, 0, sizeof(priv->pkstats));
1879         memset(&priv->port_stats, 0, sizeof(priv->port_stats));
1880         memset(&priv->rx_flowstats, 0, sizeof(priv->rx_flowstats));
1881         memset(&priv->tx_flowstats, 0, sizeof(priv->tx_flowstats));
1882         memset(&priv->rx_priority_flowstats, 0,
1883                sizeof(priv->rx_priority_flowstats));
1884         memset(&priv->tx_priority_flowstats, 0,
1885                sizeof(priv->tx_priority_flowstats));
1886         memset(&priv->pf_stats, 0, sizeof(priv->pf_stats));
1887
1888         for (i = 0; i < priv->tx_ring_num; i++) {
1889                 priv->tx_ring[i]->bytes = 0;
1890                 priv->tx_ring[i]->packets = 0;
1891                 priv->tx_ring[i]->tx_csum = 0;
1892         }
1893         for (i = 0; i < priv->rx_ring_num; i++) {
1894                 priv->rx_ring[i]->bytes = 0;
1895                 priv->rx_ring[i]->packets = 0;
1896                 priv->rx_ring[i]->csum_ok = 0;
1897                 priv->rx_ring[i]->csum_none = 0;
1898                 priv->rx_ring[i]->csum_complete = 0;
1899         }
1900 }
1901
1902 static int mlx4_en_open(struct net_device *dev)
1903 {
1904         struct mlx4_en_priv *priv = netdev_priv(dev);
1905         struct mlx4_en_dev *mdev = priv->mdev;
1906         int err = 0;
1907
1908         mutex_lock(&mdev->state_lock);
1909
1910         if (!mdev->device_up) {
1911                 en_err(priv, "Cannot open - device down/disabled\n");
1912                 err = -EBUSY;
1913                 goto out;
1914         }
1915
1916         /* Reset HW statistics and SW counters */
1917         mlx4_en_clear_stats(dev);
1918
1919         err = mlx4_en_start_port(dev);
1920         if (err)
1921                 en_err(priv, "Failed starting port:%d\n", priv->port);
1922
1923 out:
1924         mutex_unlock(&mdev->state_lock);
1925         return err;
1926 }
1927
1928
1929 static int mlx4_en_close(struct net_device *dev)
1930 {
1931         struct mlx4_en_priv *priv = netdev_priv(dev);
1932         struct mlx4_en_dev *mdev = priv->mdev;
1933
1934         en_dbg(IFDOWN, priv, "Close port called\n");
1935
1936         mutex_lock(&mdev->state_lock);
1937
1938         mlx4_en_stop_port(dev, 0);
1939         netif_carrier_off(dev);
1940
1941         mutex_unlock(&mdev->state_lock);
1942         return 0;
1943 }
1944
1945 void mlx4_en_free_resources(struct mlx4_en_priv *priv)
1946 {
1947         int i;
1948
1949 #ifdef CONFIG_RFS_ACCEL
1950         priv->dev->rx_cpu_rmap = NULL;
1951 #endif
1952
1953         for (i = 0; i < priv->tx_ring_num; i++) {
1954                 if (priv->tx_ring && priv->tx_ring[i])
1955                         mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]);
1956                 if (priv->tx_cq && priv->tx_cq[i])
1957                         mlx4_en_destroy_cq(priv, &priv->tx_cq[i]);
1958         }
1959
1960         for (i = 0; i < priv->rx_ring_num; i++) {
1961                 if (priv->rx_ring[i])
1962                         mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i],
1963                                 priv->prof->rx_ring_size, priv->stride);
1964                 if (priv->rx_cq[i])
1965                         mlx4_en_destroy_cq(priv, &priv->rx_cq[i]);
1966         }
1967
1968 }
1969
1970 int mlx4_en_alloc_resources(struct mlx4_en_priv *priv)
1971 {
1972         struct mlx4_en_port_profile *prof = priv->prof;
1973         int i;
1974         int node;
1975
1976         /* Create tx Rings */
1977         for (i = 0; i < priv->tx_ring_num; i++) {
1978                 node = cpu_to_node(i % num_online_cpus());
1979                 if (mlx4_en_create_cq(priv, &priv->tx_cq[i],
1980                                       prof->tx_ring_size, i, TX, node))
1981                         goto err;
1982
1983                 if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i],
1984                                            prof->tx_ring_size, TXBB_SIZE,
1985                                            node, i))
1986                         goto err;
1987         }
1988
1989         /* Create rx Rings */
1990         for (i = 0; i < priv->rx_ring_num; i++) {
1991                 node = cpu_to_node(i % num_online_cpus());
1992                 if (mlx4_en_create_cq(priv, &priv->rx_cq[i],
1993                                       prof->rx_ring_size, i, RX, node))
1994                         goto err;
1995
1996                 if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i],
1997                                            prof->rx_ring_size, priv->stride,
1998                                            node))
1999                         goto err;
2000         }
2001
2002 #ifdef CONFIG_RFS_ACCEL
2003         priv->dev->rx_cpu_rmap = mlx4_get_cpu_rmap(priv->mdev->dev, priv->port);
2004 #endif
2005
2006         return 0;
2007
2008 err:
2009         en_err(priv, "Failed to allocate NIC resources\n");
2010         for (i = 0; i < priv->rx_ring_num; i++) {
2011                 if (priv->rx_ring[i])
2012                         mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i],
2013                                                 prof->rx_ring_size,
2014                                                 priv->stride);
2015                 if (priv->rx_cq[i])
2016                         mlx4_en_destroy_cq(priv, &priv->rx_cq[i]);
2017         }
2018         for (i = 0; i < priv->tx_ring_num; i++) {
2019                 if (priv->tx_ring[i])
2020                         mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]);
2021                 if (priv->tx_cq[i])
2022                         mlx4_en_destroy_cq(priv, &priv->tx_cq[i]);
2023         }
2024         return -ENOMEM;
2025 }
2026
2027
2028 void mlx4_en_destroy_netdev(struct net_device *dev)
2029 {
2030         struct mlx4_en_priv *priv = netdev_priv(dev);
2031         struct mlx4_en_dev *mdev = priv->mdev;
2032
2033         en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port);
2034
2035         /* Unregister device - this will close the port if it was up */
2036         if (priv->registered)
2037                 unregister_netdev(dev);
2038
2039         if (priv->allocated)
2040                 mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE);
2041
2042         cancel_delayed_work(&priv->stats_task);
2043         cancel_delayed_work(&priv->service_task);
2044         /* flush any pending task for this netdev */
2045         flush_workqueue(mdev->workqueue);
2046
2047         if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)
2048                 mlx4_en_remove_timestamp(mdev);
2049
2050         /* Detach the netdev so tasks would not attempt to access it */
2051         mutex_lock(&mdev->state_lock);
2052         mdev->pndev[priv->port] = NULL;
2053         mdev->upper[priv->port] = NULL;
2054         mutex_unlock(&mdev->state_lock);
2055
2056         mlx4_en_free_resources(priv);
2057
2058         kfree(priv->tx_ring);
2059         kfree(priv->tx_cq);
2060
2061         free_netdev(dev);
2062 }
2063
2064 static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu)
2065 {
2066         struct mlx4_en_priv *priv = netdev_priv(dev);
2067         struct mlx4_en_dev *mdev = priv->mdev;
2068         int err = 0;
2069
2070         en_dbg(DRV, priv, "Change MTU called - current:%d new:%d\n",
2071                  dev->mtu, new_mtu);
2072
2073         if ((new_mtu < MLX4_EN_MIN_MTU) || (new_mtu > priv->max_mtu)) {
2074                 en_err(priv, "Bad MTU size:%d.\n", new_mtu);
2075                 return -EPERM;
2076         }
2077         dev->mtu = new_mtu;
2078
2079         if (netif_running(dev)) {
2080                 mutex_lock(&mdev->state_lock);
2081                 if (!mdev->device_up) {
2082                         /* NIC is probably restarting - let watchdog task reset
2083                          * the port */
2084                         en_dbg(DRV, priv, "Change MTU called with card down!?\n");
2085                 } else {
2086                         mlx4_en_stop_port(dev, 1);
2087                         err = mlx4_en_start_port(dev);
2088                         if (err) {
2089                                 en_err(priv, "Failed restarting port:%d\n",
2090                                          priv->port);
2091                                 queue_work(mdev->workqueue, &priv->watchdog_task);
2092                         }
2093                 }
2094                 mutex_unlock(&mdev->state_lock);
2095         }
2096         return 0;
2097 }
2098
2099 static int mlx4_en_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
2100 {
2101         struct mlx4_en_priv *priv = netdev_priv(dev);
2102         struct mlx4_en_dev *mdev = priv->mdev;
2103         struct hwtstamp_config config;
2104
2105         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2106                 return -EFAULT;
2107
2108         /* reserved for future extensions */
2109         if (config.flags)
2110                 return -EINVAL;
2111
2112         /* device doesn't support time stamping */
2113         if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS))
2114                 return -EINVAL;
2115
2116         /* TX HW timestamp */
2117         switch (config.tx_type) {
2118         case HWTSTAMP_TX_OFF:
2119         case HWTSTAMP_TX_ON:
2120                 break;
2121         default:
2122                 return -ERANGE;
2123         }
2124
2125         /* RX HW timestamp */
2126         switch (config.rx_filter) {
2127         case HWTSTAMP_FILTER_NONE:
2128                 break;
2129         case HWTSTAMP_FILTER_ALL:
2130         case HWTSTAMP_FILTER_SOME:
2131         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2132         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2133         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2134         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2135         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2136         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2137         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2138         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2139         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2140         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2141         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2142         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2143                 config.rx_filter = HWTSTAMP_FILTER_ALL;
2144                 break;
2145         default:
2146                 return -ERANGE;
2147         }
2148
2149         if (mlx4_en_reset_config(dev, config, dev->features)) {
2150                 config.tx_type = HWTSTAMP_TX_OFF;
2151                 config.rx_filter = HWTSTAMP_FILTER_NONE;
2152         }
2153
2154         return copy_to_user(ifr->ifr_data, &config,
2155                             sizeof(config)) ? -EFAULT : 0;
2156 }
2157
2158 static int mlx4_en_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
2159 {
2160         struct mlx4_en_priv *priv = netdev_priv(dev);
2161
2162         return copy_to_user(ifr->ifr_data, &priv->hwtstamp_config,
2163                             sizeof(priv->hwtstamp_config)) ? -EFAULT : 0;
2164 }
2165
2166 static int mlx4_en_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
2167 {
2168         switch (cmd) {
2169         case SIOCSHWTSTAMP:
2170                 return mlx4_en_hwtstamp_set(dev, ifr);
2171         case SIOCGHWTSTAMP:
2172                 return mlx4_en_hwtstamp_get(dev, ifr);
2173         default:
2174                 return -EOPNOTSUPP;
2175         }
2176 }
2177
2178 static netdev_features_t mlx4_en_fix_features(struct net_device *netdev,
2179                                               netdev_features_t features)
2180 {
2181         struct mlx4_en_priv *en_priv = netdev_priv(netdev);
2182         struct mlx4_en_dev *mdev = en_priv->mdev;
2183
2184         /* Since there is no support for separate RX C-TAG/S-TAG vlan accel
2185          * enable/disable make sure S-TAG flag is always in same state as
2186          * C-TAG.
2187          */
2188         if (features & NETIF_F_HW_VLAN_CTAG_RX &&
2189             !(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN))
2190                 features |= NETIF_F_HW_VLAN_STAG_RX;
2191         else
2192                 features &= ~NETIF_F_HW_VLAN_STAG_RX;
2193
2194         return features;
2195 }
2196
2197 static int mlx4_en_set_features(struct net_device *netdev,
2198                 netdev_features_t features)
2199 {
2200         struct mlx4_en_priv *priv = netdev_priv(netdev);
2201         bool reset = false;
2202         int ret = 0;
2203
2204         if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXFCS)) {
2205                 en_info(priv, "Turn %s RX-FCS\n",
2206                         (features & NETIF_F_RXFCS) ? "ON" : "OFF");
2207                 reset = true;
2208         }
2209
2210         if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXALL)) {
2211                 u8 ignore_fcs_value = (features & NETIF_F_RXALL) ? 1 : 0;
2212
2213                 en_info(priv, "Turn %s RX-ALL\n",
2214                         ignore_fcs_value ? "ON" : "OFF");
2215                 ret = mlx4_SET_PORT_fcs_check(priv->mdev->dev,
2216                                               priv->port, ignore_fcs_value);
2217                 if (ret)
2218                         return ret;
2219         }
2220
2221         if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_RX)) {
2222                 en_info(priv, "Turn %s RX vlan strip offload\n",
2223                         (features & NETIF_F_HW_VLAN_CTAG_RX) ? "ON" : "OFF");
2224                 reset = true;
2225         }
2226
2227         if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_TX))
2228                 en_info(priv, "Turn %s TX vlan strip offload\n",
2229                         (features & NETIF_F_HW_VLAN_CTAG_TX) ? "ON" : "OFF");
2230
2231         if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_STAG_TX))
2232                 en_info(priv, "Turn %s TX S-VLAN strip offload\n",
2233                         (features & NETIF_F_HW_VLAN_STAG_TX) ? "ON" : "OFF");
2234
2235         if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_LOOPBACK)) {
2236                 en_info(priv, "Turn %s loopback\n",
2237                         (features & NETIF_F_LOOPBACK) ? "ON" : "OFF");
2238                 mlx4_en_update_loopback_state(netdev, features);
2239         }
2240
2241         if (reset) {
2242                 ret = mlx4_en_reset_config(netdev, priv->hwtstamp_config,
2243                                            features);
2244                 if (ret)
2245                         return ret;
2246         }
2247
2248         return 0;
2249 }
2250
2251 static int mlx4_en_set_vf_mac(struct net_device *dev, int queue, u8 *mac)
2252 {
2253         struct mlx4_en_priv *en_priv = netdev_priv(dev);
2254         struct mlx4_en_dev *mdev = en_priv->mdev;
2255         u64 mac_u64 = mlx4_mac_to_u64(mac);
2256
2257         if (!is_valid_ether_addr(mac))
2258                 return -EINVAL;
2259
2260         return mlx4_set_vf_mac(mdev->dev, en_priv->port, queue, mac_u64);
2261 }
2262
2263 static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos)
2264 {
2265         struct mlx4_en_priv *en_priv = netdev_priv(dev);
2266         struct mlx4_en_dev *mdev = en_priv->mdev;
2267
2268         return mlx4_set_vf_vlan(mdev->dev, en_priv->port, vf, vlan, qos);
2269 }
2270
2271 static int mlx4_en_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
2272                                int max_tx_rate)
2273 {
2274         struct mlx4_en_priv *en_priv = netdev_priv(dev);
2275         struct mlx4_en_dev *mdev = en_priv->mdev;
2276
2277         return mlx4_set_vf_rate(mdev->dev, en_priv->port, vf, min_tx_rate,
2278                                 max_tx_rate);
2279 }
2280
2281 static int mlx4_en_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
2282 {
2283         struct mlx4_en_priv *en_priv = netdev_priv(dev);
2284         struct mlx4_en_dev *mdev = en_priv->mdev;
2285
2286         return mlx4_set_vf_spoofchk(mdev->dev, en_priv->port, vf, setting);
2287 }
2288
2289 static int mlx4_en_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivf)
2290 {
2291         struct mlx4_en_priv *en_priv = netdev_priv(dev);
2292         struct mlx4_en_dev *mdev = en_priv->mdev;
2293
2294         return mlx4_get_vf_config(mdev->dev, en_priv->port, vf, ivf);
2295 }
2296
2297 static int mlx4_en_set_vf_link_state(struct net_device *dev, int vf, int link_state)
2298 {
2299         struct mlx4_en_priv *en_priv = netdev_priv(dev);
2300         struct mlx4_en_dev *mdev = en_priv->mdev;
2301
2302         return mlx4_set_vf_link_state(mdev->dev, en_priv->port, vf, link_state);
2303 }
2304
2305 static int mlx4_en_get_vf_stats(struct net_device *dev, int vf,
2306                                 struct ifla_vf_stats *vf_stats)
2307 {
2308         struct mlx4_en_priv *en_priv = netdev_priv(dev);
2309         struct mlx4_en_dev *mdev = en_priv->mdev;
2310
2311         return mlx4_get_vf_stats(mdev->dev, en_priv->port, vf, vf_stats);
2312 }
2313
2314 #define PORT_ID_BYTE_LEN 8
2315 static int mlx4_en_get_phys_port_id(struct net_device *dev,
2316                                     struct netdev_phys_item_id *ppid)
2317 {
2318         struct mlx4_en_priv *priv = netdev_priv(dev);
2319         struct mlx4_dev *mdev = priv->mdev->dev;
2320         int i;
2321         u64 phys_port_id = mdev->caps.phys_port_id[priv->port];
2322
2323         if (!phys_port_id)
2324                 return -EOPNOTSUPP;
2325
2326         ppid->id_len = sizeof(phys_port_id);
2327         for (i = PORT_ID_BYTE_LEN - 1; i >= 0; --i) {
2328                 ppid->id[i] =  phys_port_id & 0xff;
2329                 phys_port_id >>= 8;
2330         }
2331         return 0;
2332 }
2333
2334 #ifdef CONFIG_MLX4_EN_VXLAN
2335 static void mlx4_en_add_vxlan_offloads(struct work_struct *work)
2336 {
2337         int ret;
2338         struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
2339                                                  vxlan_add_task);
2340
2341         ret = mlx4_config_vxlan_port(priv->mdev->dev, priv->vxlan_port);
2342         if (ret)
2343                 goto out;
2344
2345         ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port,
2346                                   VXLAN_STEER_BY_OUTER_MAC, 1);
2347 out:
2348         if (ret) {
2349                 en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret);
2350                 return;
2351         }
2352
2353         /* set offloads */
2354         priv->dev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
2355                                       NETIF_F_TSO | NETIF_F_GSO_UDP_TUNNEL;
2356 }
2357
2358 static void mlx4_en_del_vxlan_offloads(struct work_struct *work)
2359 {
2360         int ret;
2361         struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv,
2362                                                  vxlan_del_task);
2363         /* unset offloads */
2364         priv->dev->hw_enc_features &= ~(NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
2365                                       NETIF_F_TSO | NETIF_F_GSO_UDP_TUNNEL);
2366
2367         ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port,
2368                                   VXLAN_STEER_BY_OUTER_MAC, 0);
2369         if (ret)
2370                 en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret);
2371
2372         priv->vxlan_port = 0;
2373 }
2374
2375 static void mlx4_en_add_vxlan_port(struct  net_device *dev,
2376                                    sa_family_t sa_family, __be16 port)
2377 {
2378         struct mlx4_en_priv *priv = netdev_priv(dev);
2379         __be16 current_port;
2380
2381         if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
2382                 return;
2383
2384         if (sa_family == AF_INET6)
2385                 return;
2386
2387         current_port = priv->vxlan_port;
2388         if (current_port && current_port != port) {
2389                 en_warn(priv, "vxlan port %d configured, can't add port %d\n",
2390                         ntohs(current_port), ntohs(port));
2391                 return;
2392         }
2393
2394         priv->vxlan_port = port;
2395         queue_work(priv->mdev->workqueue, &priv->vxlan_add_task);
2396 }
2397
2398 static void mlx4_en_del_vxlan_port(struct  net_device *dev,
2399                                    sa_family_t sa_family, __be16 port)
2400 {
2401         struct mlx4_en_priv *priv = netdev_priv(dev);
2402         __be16 current_port;
2403
2404         if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN)
2405                 return;
2406
2407         if (sa_family == AF_INET6)
2408                 return;
2409
2410         current_port = priv->vxlan_port;
2411         if (current_port != port) {
2412                 en_dbg(DRV, priv, "vxlan port %d isn't configured, ignoring\n", ntohs(port));
2413                 return;
2414         }
2415
2416         queue_work(priv->mdev->workqueue, &priv->vxlan_del_task);
2417 }
2418
2419 static netdev_features_t mlx4_en_features_check(struct sk_buff *skb,
2420                                                 struct net_device *dev,
2421                                                 netdev_features_t features)
2422 {
2423         features = vlan_features_check(skb, features);
2424         return vxlan_features_check(skb, features);
2425 }
2426 #endif
2427
2428 static int mlx4_en_set_tx_maxrate(struct net_device *dev, int queue_index, u32 maxrate)
2429 {
2430         struct mlx4_en_priv *priv = netdev_priv(dev);
2431         struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[queue_index];
2432         struct mlx4_update_qp_params params;
2433         int err;
2434
2435         if (!(priv->mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT))
2436                 return -EOPNOTSUPP;
2437
2438         /* rate provided to us in Mbs, check if it fits into 12 bits, if not use Gbs */
2439         if (maxrate >> 12) {
2440                 params.rate_unit = MLX4_QP_RATE_LIMIT_GBS;
2441                 params.rate_val  = maxrate / 1000;
2442         } else if (maxrate) {
2443                 params.rate_unit = MLX4_QP_RATE_LIMIT_MBS;
2444                 params.rate_val  = maxrate;
2445         } else { /* zero serves to revoke the QP rate-limitation */
2446                 params.rate_unit = 0;
2447                 params.rate_val  = 0;
2448         }
2449
2450         err = mlx4_update_qp(priv->mdev->dev, tx_ring->qpn, MLX4_UPDATE_QP_RATE_LIMIT,
2451                              &params);
2452         return err;
2453 }
2454
2455 static const struct net_device_ops mlx4_netdev_ops = {
2456         .ndo_open               = mlx4_en_open,
2457         .ndo_stop               = mlx4_en_close,
2458         .ndo_start_xmit         = mlx4_en_xmit,
2459         .ndo_select_queue       = mlx4_en_select_queue,
2460         .ndo_get_stats          = mlx4_en_get_stats,
2461         .ndo_set_rx_mode        = mlx4_en_set_rx_mode,
2462         .ndo_set_mac_address    = mlx4_en_set_mac,
2463         .ndo_validate_addr      = eth_validate_addr,
2464         .ndo_change_mtu         = mlx4_en_change_mtu,
2465         .ndo_do_ioctl           = mlx4_en_ioctl,
2466         .ndo_tx_timeout         = mlx4_en_tx_timeout,
2467         .ndo_vlan_rx_add_vid    = mlx4_en_vlan_rx_add_vid,
2468         .ndo_vlan_rx_kill_vid   = mlx4_en_vlan_rx_kill_vid,
2469 #ifdef CONFIG_NET_POLL_CONTROLLER
2470         .ndo_poll_controller    = mlx4_en_netpoll,
2471 #endif
2472         .ndo_set_features       = mlx4_en_set_features,
2473         .ndo_fix_features       = mlx4_en_fix_features,
2474         .ndo_setup_tc           = __mlx4_en_setup_tc,
2475 #ifdef CONFIG_RFS_ACCEL
2476         .ndo_rx_flow_steer      = mlx4_en_filter_rfs,
2477 #endif
2478         .ndo_get_phys_port_id   = mlx4_en_get_phys_port_id,
2479 #ifdef CONFIG_MLX4_EN_VXLAN
2480         .ndo_add_vxlan_port     = mlx4_en_add_vxlan_port,
2481         .ndo_del_vxlan_port     = mlx4_en_del_vxlan_port,
2482         .ndo_features_check     = mlx4_en_features_check,
2483 #endif
2484         .ndo_set_tx_maxrate     = mlx4_en_set_tx_maxrate,
2485 };
2486
2487 static const struct net_device_ops mlx4_netdev_ops_master = {
2488         .ndo_open               = mlx4_en_open,
2489         .ndo_stop               = mlx4_en_close,
2490         .ndo_start_xmit         = mlx4_en_xmit,
2491         .ndo_select_queue       = mlx4_en_select_queue,
2492         .ndo_get_stats          = mlx4_en_get_stats,
2493         .ndo_set_rx_mode        = mlx4_en_set_rx_mode,
2494         .ndo_set_mac_address    = mlx4_en_set_mac,
2495         .ndo_validate_addr      = eth_validate_addr,
2496         .ndo_change_mtu         = mlx4_en_change_mtu,
2497         .ndo_tx_timeout         = mlx4_en_tx_timeout,
2498         .ndo_vlan_rx_add_vid    = mlx4_en_vlan_rx_add_vid,
2499         .ndo_vlan_rx_kill_vid   = mlx4_en_vlan_rx_kill_vid,
2500         .ndo_set_vf_mac         = mlx4_en_set_vf_mac,
2501         .ndo_set_vf_vlan        = mlx4_en_set_vf_vlan,
2502         .ndo_set_vf_rate        = mlx4_en_set_vf_rate,
2503         .ndo_set_vf_spoofchk    = mlx4_en_set_vf_spoofchk,
2504         .ndo_set_vf_link_state  = mlx4_en_set_vf_link_state,
2505         .ndo_get_vf_stats       = mlx4_en_get_vf_stats,
2506         .ndo_get_vf_config      = mlx4_en_get_vf_config,
2507 #ifdef CONFIG_NET_POLL_CONTROLLER
2508         .ndo_poll_controller    = mlx4_en_netpoll,
2509 #endif
2510         .ndo_set_features       = mlx4_en_set_features,
2511         .ndo_fix_features       = mlx4_en_fix_features,
2512         .ndo_setup_tc           = __mlx4_en_setup_tc,
2513 #ifdef CONFIG_RFS_ACCEL
2514         .ndo_rx_flow_steer      = mlx4_en_filter_rfs,
2515 #endif
2516         .ndo_get_phys_port_id   = mlx4_en_get_phys_port_id,
2517 #ifdef CONFIG_MLX4_EN_VXLAN
2518         .ndo_add_vxlan_port     = mlx4_en_add_vxlan_port,
2519         .ndo_del_vxlan_port     = mlx4_en_del_vxlan_port,
2520         .ndo_features_check     = mlx4_en_features_check,
2521 #endif
2522         .ndo_set_tx_maxrate     = mlx4_en_set_tx_maxrate,
2523 };
2524
2525 struct mlx4_en_bond {
2526         struct work_struct work;
2527         struct mlx4_en_priv *priv;
2528         int is_bonded;
2529         struct mlx4_port_map port_map;
2530 };
2531
2532 static void mlx4_en_bond_work(struct work_struct *work)
2533 {
2534         struct mlx4_en_bond *bond = container_of(work,
2535                                                      struct mlx4_en_bond,
2536                                                      work);
2537         int err = 0;
2538         struct mlx4_dev *dev = bond->priv->mdev->dev;
2539
2540         if (bond->is_bonded) {
2541                 if (!mlx4_is_bonded(dev)) {
2542                         err = mlx4_bond(dev);
2543                         if (err)
2544                                 en_err(bond->priv, "Fail to bond device\n");
2545                 }
2546                 if (!err) {
2547                         err = mlx4_port_map_set(dev, &bond->port_map);
2548                         if (err)
2549                                 en_err(bond->priv, "Fail to set port map [%d][%d]: %d\n",
2550                                        bond->port_map.port1,
2551                                        bond->port_map.port2,
2552                                        err);
2553                 }
2554         } else if (mlx4_is_bonded(dev)) {
2555                 err = mlx4_unbond(dev);
2556                 if (err)
2557                         en_err(bond->priv, "Fail to unbond device\n");
2558         }
2559         dev_put(bond->priv->dev);
2560         kfree(bond);
2561 }
2562
2563 static int mlx4_en_queue_bond_work(struct mlx4_en_priv *priv, int is_bonded,
2564                                    u8 v2p_p1, u8 v2p_p2)
2565 {
2566         struct mlx4_en_bond *bond = NULL;
2567
2568         bond = kzalloc(sizeof(*bond), GFP_ATOMIC);
2569         if (!bond)
2570                 return -ENOMEM;
2571
2572         INIT_WORK(&bond->work, mlx4_en_bond_work);
2573         bond->priv = priv;
2574         bond->is_bonded = is_bonded;
2575         bond->port_map.port1 = v2p_p1;
2576         bond->port_map.port2 = v2p_p2;
2577         dev_hold(priv->dev);
2578         queue_work(priv->mdev->workqueue, &bond->work);
2579         return 0;
2580 }
2581
2582 int mlx4_en_netdev_event(struct notifier_block *this,
2583                          unsigned long event, void *ptr)
2584 {
2585         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
2586         u8 port = 0;
2587         struct mlx4_en_dev *mdev;
2588         struct mlx4_dev *dev;
2589         int i, num_eth_ports = 0;
2590         bool do_bond = true;
2591         struct mlx4_en_priv *priv;
2592         u8 v2p_port1 = 0;
2593         u8 v2p_port2 = 0;
2594
2595         if (!net_eq(dev_net(ndev), &init_net))
2596                 return NOTIFY_DONE;
2597
2598         mdev = container_of(this, struct mlx4_en_dev, nb);
2599         dev = mdev->dev;
2600
2601         /* Go into this mode only when two network devices set on two ports
2602          * of the same mlx4 device are slaves of the same bonding master
2603          */
2604         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) {
2605                 ++num_eth_ports;
2606                 if (!port && (mdev->pndev[i] == ndev))
2607                         port = i;
2608                 mdev->upper[i] = mdev->pndev[i] ?
2609                         netdev_master_upper_dev_get(mdev->pndev[i]) : NULL;
2610                 /* condition not met: network device is a slave */
2611                 if (!mdev->upper[i])
2612                         do_bond = false;
2613                 if (num_eth_ports < 2)
2614                         continue;
2615                 /* condition not met: same master */
2616                 if (mdev->upper[i] != mdev->upper[i-1])
2617                         do_bond = false;
2618         }
2619         /* condition not met: 2 salves */
2620         do_bond = (num_eth_ports ==  2) ? do_bond : false;
2621
2622         /* handle only events that come with enough info */
2623         if ((do_bond && (event != NETDEV_BONDING_INFO)) || !port)
2624                 return NOTIFY_DONE;
2625
2626         priv = netdev_priv(ndev);
2627         if (do_bond) {
2628                 struct netdev_notifier_bonding_info *notifier_info = ptr;
2629                 struct netdev_bonding_info *bonding_info =
2630                         &notifier_info->bonding_info;
2631
2632                 /* required mode 1, 2 or 4 */
2633                 if ((bonding_info->master.bond_mode != BOND_MODE_ACTIVEBACKUP) &&
2634                     (bonding_info->master.bond_mode != BOND_MODE_XOR) &&
2635                     (bonding_info->master.bond_mode != BOND_MODE_8023AD))
2636                         do_bond = false;
2637
2638                 /* require exactly 2 slaves */
2639                 if (bonding_info->master.num_slaves != 2)
2640                         do_bond = false;
2641
2642                 /* calc v2p */
2643                 if (do_bond) {
2644                         if (bonding_info->master.bond_mode ==
2645                             BOND_MODE_ACTIVEBACKUP) {
2646                                 /* in active-backup mode virtual ports are
2647                                  * mapped to the physical port of the active
2648                                  * slave */
2649                                 if (bonding_info->slave.state ==
2650                                     BOND_STATE_BACKUP) {
2651                                         if (port == 1) {
2652                                                 v2p_port1 = 2;
2653                                                 v2p_port2 = 2;
2654                                         } else {
2655                                                 v2p_port1 = 1;
2656                                                 v2p_port2 = 1;
2657                                         }
2658                                 } else { /* BOND_STATE_ACTIVE */
2659                                         if (port == 1) {
2660                                                 v2p_port1 = 1;
2661                                                 v2p_port2 = 1;
2662                                         } else {
2663                                                 v2p_port1 = 2;
2664                                                 v2p_port2 = 2;
2665                                         }
2666                                 }
2667                         } else { /* Active-Active */
2668                                 /* in active-active mode a virtual port is
2669                                  * mapped to the native physical port if and only
2670                                  * if the physical port is up */
2671                                 __s8 link = bonding_info->slave.link;
2672
2673                                 if (port == 1)
2674                                         v2p_port2 = 2;
2675                                 else
2676                                         v2p_port1 = 1;
2677                                 if ((link == BOND_LINK_UP) ||
2678                                     (link == BOND_LINK_FAIL)) {
2679                                         if (port == 1)
2680                                                 v2p_port1 = 1;
2681                                         else
2682                                                 v2p_port2 = 2;
2683                                 } else { /* BOND_LINK_DOWN || BOND_LINK_BACK */
2684                                         if (port == 1)
2685                                                 v2p_port1 = 2;
2686                                         else
2687                                                 v2p_port2 = 1;
2688                                 }
2689                         }
2690                 }
2691         }
2692
2693         mlx4_en_queue_bond_work(priv, do_bond,
2694                                 v2p_port1, v2p_port2);
2695
2696         return NOTIFY_DONE;
2697 }
2698
2699 void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev,
2700                                      struct mlx4_en_stats_bitmap *stats_bitmap,
2701                                      u8 rx_ppp, u8 rx_pause,
2702                                      u8 tx_ppp, u8 tx_pause)
2703 {
2704         int last_i = NUM_MAIN_STATS + NUM_PORT_STATS + NUM_PF_STATS;
2705
2706         if (!mlx4_is_slave(dev) &&
2707             (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN)) {
2708                 mutex_lock(&stats_bitmap->mutex);
2709                 bitmap_clear(stats_bitmap->bitmap, last_i, NUM_FLOW_STATS);
2710
2711                 if (rx_ppp)
2712                         bitmap_set(stats_bitmap->bitmap, last_i,
2713                                    NUM_FLOW_PRIORITY_STATS_RX);
2714                 last_i += NUM_FLOW_PRIORITY_STATS_RX;
2715
2716                 if (rx_pause && !(rx_ppp))
2717                         bitmap_set(stats_bitmap->bitmap, last_i,
2718                                    NUM_FLOW_STATS_RX);
2719                 last_i += NUM_FLOW_STATS_RX;
2720
2721                 if (tx_ppp)
2722                         bitmap_set(stats_bitmap->bitmap, last_i,
2723                                    NUM_FLOW_PRIORITY_STATS_TX);
2724                 last_i += NUM_FLOW_PRIORITY_STATS_TX;
2725
2726                 if (tx_pause && !(tx_ppp))
2727                         bitmap_set(stats_bitmap->bitmap, last_i,
2728                                    NUM_FLOW_STATS_TX);
2729                 last_i += NUM_FLOW_STATS_TX;
2730
2731                 mutex_unlock(&stats_bitmap->mutex);
2732         }
2733 }
2734
2735 void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev,
2736                               struct mlx4_en_stats_bitmap *stats_bitmap,
2737                               u8 rx_ppp, u8 rx_pause,
2738                               u8 tx_ppp, u8 tx_pause)
2739 {
2740         int last_i = 0;
2741
2742         mutex_init(&stats_bitmap->mutex);
2743         bitmap_zero(stats_bitmap->bitmap, NUM_ALL_STATS);
2744
2745         if (mlx4_is_slave(dev)) {
2746                 bitmap_set(stats_bitmap->bitmap, last_i +
2747                                          MLX4_FIND_NETDEV_STAT(rx_packets), 1);
2748                 bitmap_set(stats_bitmap->bitmap, last_i +
2749                                          MLX4_FIND_NETDEV_STAT(tx_packets), 1);
2750                 bitmap_set(stats_bitmap->bitmap, last_i +
2751                                          MLX4_FIND_NETDEV_STAT(rx_bytes), 1);
2752                 bitmap_set(stats_bitmap->bitmap, last_i +
2753                                          MLX4_FIND_NETDEV_STAT(tx_bytes), 1);
2754                 bitmap_set(stats_bitmap->bitmap, last_i +
2755                                          MLX4_FIND_NETDEV_STAT(rx_dropped), 1);
2756                 bitmap_set(stats_bitmap->bitmap, last_i +
2757                                          MLX4_FIND_NETDEV_STAT(tx_dropped), 1);
2758         } else {
2759                 bitmap_set(stats_bitmap->bitmap, last_i, NUM_MAIN_STATS);
2760         }
2761         last_i += NUM_MAIN_STATS;
2762
2763         bitmap_set(stats_bitmap->bitmap, last_i, NUM_PORT_STATS);
2764         last_i += NUM_PORT_STATS;
2765
2766         if (mlx4_is_master(dev))
2767                 bitmap_set(stats_bitmap->bitmap, last_i,
2768                            NUM_PF_STATS);
2769         last_i += NUM_PF_STATS;
2770
2771         mlx4_en_update_pfc_stats_bitmap(dev, stats_bitmap,
2772                                         rx_ppp, rx_pause,
2773                                         tx_ppp, tx_pause);
2774         last_i += NUM_FLOW_STATS;
2775
2776         if (!mlx4_is_slave(dev))
2777                 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PKT_STATS);
2778 }
2779
2780 int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port,
2781                         struct mlx4_en_port_profile *prof)
2782 {
2783         struct net_device *dev;
2784         struct mlx4_en_priv *priv;
2785         int i;
2786         int err;
2787
2788         dev = alloc_etherdev_mqs(sizeof(struct mlx4_en_priv),
2789                                  MAX_TX_RINGS, MAX_RX_RINGS);
2790         if (dev == NULL)
2791                 return -ENOMEM;
2792
2793         netif_set_real_num_tx_queues(dev, prof->tx_ring_num);
2794         netif_set_real_num_rx_queues(dev, prof->rx_ring_num);
2795
2796         SET_NETDEV_DEV(dev, &mdev->dev->persist->pdev->dev);
2797         dev->dev_port = port - 1;
2798
2799         /*
2800          * Initialize driver private data
2801          */
2802
2803         priv = netdev_priv(dev);
2804         memset(priv, 0, sizeof(struct mlx4_en_priv));
2805         priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev);
2806         spin_lock_init(&priv->stats_lock);
2807         INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode);
2808         INIT_WORK(&priv->watchdog_task, mlx4_en_restart);
2809         INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate);
2810         INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats);
2811         INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task);
2812 #ifdef CONFIG_MLX4_EN_VXLAN
2813         INIT_WORK(&priv->vxlan_add_task, mlx4_en_add_vxlan_offloads);
2814         INIT_WORK(&priv->vxlan_del_task, mlx4_en_del_vxlan_offloads);
2815 #endif
2816 #ifdef CONFIG_RFS_ACCEL
2817         INIT_LIST_HEAD(&priv->filters);
2818         spin_lock_init(&priv->filters_lock);
2819 #endif
2820
2821         priv->dev = dev;
2822         priv->mdev = mdev;
2823         priv->ddev = &mdev->pdev->dev;
2824         priv->prof = prof;
2825         priv->port = port;
2826         priv->port_up = false;
2827         priv->flags = prof->flags;
2828         priv->pflags = MLX4_EN_PRIV_FLAGS_BLUEFLAME;
2829         priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE |
2830                         MLX4_WQE_CTRL_SOLICITED);
2831         priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up;
2832         priv->tx_ring_num = prof->tx_ring_num;
2833         priv->tx_work_limit = MLX4_EN_DEFAULT_TX_WORK;
2834         netdev_rss_key_fill(priv->rss_key, sizeof(priv->rss_key));
2835
2836         priv->tx_ring = kzalloc(sizeof(struct mlx4_en_tx_ring *) * MAX_TX_RINGS,
2837                                 GFP_KERNEL);
2838         if (!priv->tx_ring) {
2839                 err = -ENOMEM;
2840                 goto out;
2841         }
2842         priv->tx_cq = kzalloc(sizeof(struct mlx4_en_cq *) * MAX_TX_RINGS,
2843                               GFP_KERNEL);
2844         if (!priv->tx_cq) {
2845                 err = -ENOMEM;
2846                 goto out;
2847         }
2848         priv->rx_ring_num = prof->rx_ring_num;
2849         priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0;
2850         priv->cqe_size = mdev->dev->caps.cqe_size;
2851         priv->mac_index = -1;
2852         priv->msg_enable = MLX4_EN_MSG_LEVEL;
2853 #ifdef CONFIG_MLX4_EN_DCB
2854         if (!mlx4_is_slave(priv->mdev->dev)) {
2855                 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) {
2856                         dev->dcbnl_ops = &mlx4_en_dcbnl_ops;
2857                 } else {
2858                         en_info(priv, "enabling only PFC DCB ops\n");
2859                         dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops;
2860                 }
2861         }
2862 #endif
2863
2864         for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i)
2865                 INIT_HLIST_HEAD(&priv->mac_hash[i]);
2866
2867         /* Query for default mac and max mtu */
2868         priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port];
2869
2870         if (mdev->dev->caps.rx_checksum_flags_port[priv->port] &
2871             MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP)
2872                 priv->flags |= MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP;
2873
2874         /* Set default MAC */
2875         dev->addr_len = ETH_ALEN;
2876         mlx4_en_u64_to_mac(dev->dev_addr, mdev->dev->caps.def_mac[priv->port]);
2877         if (!is_valid_ether_addr(dev->dev_addr)) {
2878                 en_err(priv, "Port: %d, invalid mac burned: %pM, quiting\n",
2879                        priv->port, dev->dev_addr);
2880                 err = -EINVAL;
2881                 goto out;
2882         } else if (mlx4_is_slave(priv->mdev->dev) &&
2883                    (priv->mdev->dev->port_random_macs & 1 << priv->port)) {
2884                 /* Random MAC was assigned in mlx4_slave_cap
2885                  * in mlx4_core module
2886                  */
2887                 dev->addr_assign_type |= NET_ADDR_RANDOM;
2888                 en_warn(priv, "Assigned random MAC address %pM\n", dev->dev_addr);
2889         }
2890
2891         memcpy(priv->current_mac, dev->dev_addr, sizeof(priv->current_mac));
2892
2893         priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
2894                                           DS_SIZE * MLX4_EN_MAX_RX_FRAGS);
2895         err = mlx4_en_alloc_resources(priv);
2896         if (err)
2897                 goto out;
2898
2899         /* Initialize time stamping config */
2900         priv->hwtstamp_config.flags = 0;
2901         priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF;
2902         priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
2903
2904         /* Allocate page for receive rings */
2905         err = mlx4_alloc_hwq_res(mdev->dev, &priv->res,
2906                                 MLX4_EN_PAGE_SIZE, MLX4_EN_PAGE_SIZE);
2907         if (err) {
2908                 en_err(priv, "Failed to allocate page for rx qps\n");
2909                 goto out;
2910         }
2911         priv->allocated = 1;
2912
2913         /*
2914          * Initialize netdev entry points
2915          */
2916         if (mlx4_is_master(priv->mdev->dev))
2917                 dev->netdev_ops = &mlx4_netdev_ops_master;
2918         else
2919                 dev->netdev_ops = &mlx4_netdev_ops;
2920         dev->watchdog_timeo = MLX4_EN_WATCHDOG_TIMEOUT;
2921         netif_set_real_num_tx_queues(dev, priv->tx_ring_num);
2922         netif_set_real_num_rx_queues(dev, priv->rx_ring_num);
2923
2924         dev->ethtool_ops = &mlx4_en_ethtool_ops;
2925
2926         /*
2927          * Set driver features
2928          */
2929         dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2930         if (mdev->LSO_support)
2931                 dev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
2932
2933         dev->vlan_features = dev->hw_features;
2934
2935         dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH;
2936         dev->features = dev->hw_features | NETIF_F_HIGHDMA |
2937                         NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
2938                         NETIF_F_HW_VLAN_CTAG_FILTER;
2939         dev->hw_features |= NETIF_F_LOOPBACK |
2940                         NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
2941
2942         if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) {
2943                 dev->features |= NETIF_F_HW_VLAN_STAG_RX |
2944                         NETIF_F_HW_VLAN_STAG_FILTER;
2945                 dev->hw_features |= NETIF_F_HW_VLAN_STAG_RX;
2946         }
2947
2948         if (mlx4_is_slave(mdev->dev)) {
2949                 int phv;
2950
2951                 err = get_phv_bit(mdev->dev, port, &phv);
2952                 if (!err && phv) {
2953                         dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX;
2954                         priv->pflags |= MLX4_EN_PRIV_FLAGS_PHV;
2955                 }
2956         } else {
2957                 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN &&
2958                     !(mdev->dev->caps.flags2 &
2959                       MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN))
2960                         dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX;
2961         }
2962
2963         if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP)
2964                 dev->hw_features |= NETIF_F_RXFCS;
2965
2966         if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS)
2967                 dev->hw_features |= NETIF_F_RXALL;
2968
2969         if (mdev->dev->caps.steering_mode ==
2970             MLX4_STEERING_MODE_DEVICE_MANAGED &&
2971             mdev->dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_STATIC)
2972                 dev->hw_features |= NETIF_F_NTUPLE;
2973
2974         if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0)
2975                 dev->priv_flags |= IFF_UNICAST_FLT;
2976
2977         /* Setting a default hash function value */
2978         if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) {
2979                 priv->rss_hash_fn = ETH_RSS_HASH_TOP;
2980         } else if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_XOR) {
2981                 priv->rss_hash_fn = ETH_RSS_HASH_XOR;
2982         } else {
2983                 en_warn(priv,
2984                         "No RSS hash capabilities exposed, using Toeplitz\n");
2985                 priv->rss_hash_fn = ETH_RSS_HASH_TOP;
2986         }
2987
2988         if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
2989                 dev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
2990                 dev->features    |= NETIF_F_GSO_UDP_TUNNEL;
2991         }
2992
2993         mdev->pndev[port] = dev;
2994         mdev->upper[port] = NULL;
2995
2996         netif_carrier_off(dev);
2997         mlx4_en_set_default_moderation(priv);
2998
2999         en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num);
3000         en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num);
3001
3002         mlx4_en_update_loopback_state(priv->dev, priv->dev->features);
3003
3004         /* Configure port */
3005         mlx4_en_calc_rx_buf(dev);
3006         err = mlx4_SET_PORT_general(mdev->dev, priv->port,
3007                                     priv->rx_skb_size + ETH_FCS_LEN,
3008                                     prof->tx_pause, prof->tx_ppp,
3009                                     prof->rx_pause, prof->rx_ppp);
3010         if (err) {
3011                 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n",
3012                        priv->port, err);
3013                 goto out;
3014         }
3015
3016         if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
3017                 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1);
3018                 if (err) {
3019                         en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n",
3020                                err);
3021                         goto out;
3022                 }
3023         }
3024
3025         /* Init port */
3026         en_warn(priv, "Initializing port\n");
3027         err = mlx4_INIT_PORT(mdev->dev, priv->port);
3028         if (err) {
3029                 en_err(priv, "Failed Initializing port\n");
3030                 goto out;
3031         }
3032         queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY);
3033
3034         /* Initialize time stamp mechanism */
3035         if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)
3036                 mlx4_en_init_timestamp(mdev);
3037
3038         queue_delayed_work(mdev->workqueue, &priv->service_task,
3039                            SERVICE_TASK_DELAY);
3040
3041         mlx4_en_set_stats_bitmap(mdev->dev, &priv->stats_bitmap,
3042                                  mdev->profile.prof[priv->port].rx_ppp,
3043                                  mdev->profile.prof[priv->port].rx_pause,
3044                                  mdev->profile.prof[priv->port].tx_ppp,
3045                                  mdev->profile.prof[priv->port].tx_pause);
3046
3047         err = register_netdev(dev);
3048         if (err) {
3049                 en_err(priv, "Netdev registration failed for port %d\n", port);
3050                 goto out;
3051         }
3052
3053         priv->registered = 1;
3054
3055         return 0;
3056
3057 out:
3058         mlx4_en_destroy_netdev(dev);
3059         return err;
3060 }
3061
3062 int mlx4_en_reset_config(struct net_device *dev,
3063                          struct hwtstamp_config ts_config,
3064                          netdev_features_t features)
3065 {
3066         struct mlx4_en_priv *priv = netdev_priv(dev);
3067         struct mlx4_en_dev *mdev = priv->mdev;
3068         int port_up = 0;
3069         int err = 0;
3070
3071         if (priv->hwtstamp_config.tx_type == ts_config.tx_type &&
3072             priv->hwtstamp_config.rx_filter == ts_config.rx_filter &&
3073             !DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) &&
3074             !DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS))
3075                 return 0; /* Nothing to change */
3076
3077         if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) &&
3078             (features & NETIF_F_HW_VLAN_CTAG_RX) &&
3079             (priv->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE)) {
3080                 en_warn(priv, "Can't turn ON rx vlan offload while time-stamping rx filter is ON\n");
3081                 return -EINVAL;
3082         }
3083
3084         mutex_lock(&mdev->state_lock);
3085         if (priv->port_up) {
3086                 port_up = 1;
3087                 mlx4_en_stop_port(dev, 1);
3088         }
3089
3090         mlx4_en_free_resources(priv);
3091
3092         en_warn(priv, "Changing device configuration rx filter(%x) rx vlan(%x)\n",
3093                 ts_config.rx_filter, !!(features & NETIF_F_HW_VLAN_CTAG_RX));
3094
3095         priv->hwtstamp_config.tx_type = ts_config.tx_type;
3096         priv->hwtstamp_config.rx_filter = ts_config.rx_filter;
3097
3098         if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX)) {
3099                 if (features & NETIF_F_HW_VLAN_CTAG_RX)
3100                         dev->features |= NETIF_F_HW_VLAN_CTAG_RX;
3101                 else
3102                         dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX;
3103         } else if (ts_config.rx_filter == HWTSTAMP_FILTER_NONE) {
3104                 /* RX time-stamping is OFF, update the RX vlan offload
3105                  * to the latest wanted state
3106                  */
3107                 if (dev->wanted_features & NETIF_F_HW_VLAN_CTAG_RX)
3108                         dev->features |= NETIF_F_HW_VLAN_CTAG_RX;
3109                 else
3110                         dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX;
3111         }
3112
3113         if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) {
3114                 if (features & NETIF_F_RXFCS)
3115                         dev->features |= NETIF_F_RXFCS;
3116                 else
3117                         dev->features &= ~NETIF_F_RXFCS;
3118         }
3119
3120         /* RX vlan offload and RX time-stamping can't co-exist !
3121          * Regardless of the caller's choice,
3122          * Turn Off RX vlan offload in case of time-stamping is ON
3123          */
3124         if (ts_config.rx_filter != HWTSTAMP_FILTER_NONE) {
3125                 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
3126                         en_warn(priv, "Turning off RX vlan offload since RX time-stamping is ON\n");
3127                 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX;
3128         }
3129
3130         err = mlx4_en_alloc_resources(priv);
3131         if (err) {
3132                 en_err(priv, "Failed reallocating port resources\n");
3133                 goto out;
3134         }
3135         if (port_up) {
3136                 err = mlx4_en_start_port(dev);
3137                 if (err)
3138                         en_err(priv, "Failed starting port\n");
3139         }
3140
3141 out:
3142         mutex_unlock(&mdev->state_lock);
3143         netdev_features_change(dev);
3144         return err;
3145 }