drivers/net/ethernet/pensando/ionic/ionic_lif.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/ethtool.h>
   5 #include <linux/printk.h>
   6 #include <linux/dynamic_debug.h>
   7 #include <linux/netdevice.h>
   8 #include <linux/etherdevice.h>
   9 #include <linux/if_vlan.h>
  10 #include <linux/rtnetlink.h>
  11 #include <linux/interrupt.h>
  12 #include <linux/pci.h>
  13 #include <linux/cpumask.h>
  14 #include <linux/crash_dump.h>
  15 #include <linux/vmalloc.h>
  16
  17 #include "ionic.h"
  18 #include "ionic_bus.h"
  19 #include "ionic_dev.h"
  20 #include "ionic_lif.h"
  21 #include "ionic_txrx.h"
  22 #include "ionic_ethtool.h"
  23 #include "ionic_debugfs.h"
  24
  25 /* queuetype support level */
  26 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
  27         [IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
  28         [IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
  29         [IONIC_QTYPE_RXQ]     = 2,   /* 0 = Base version with CQ+SG support
  30                                       * 2 =       ... with CMB rings
  31                                       */
  32         [IONIC_QTYPE_TXQ]     = 3,   /* 0 = Base version with CQ+SG support
  33                                       * 1 =       ... with Tx SG version 1
  34                                       * 3 =       ... with CMB rings
  35                                       */
  36 };
  37
  38 static void ionic_link_status_check(struct ionic_lif *lif);
  39 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
  40 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
  41 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
  42
  43 static void ionic_txrx_deinit(struct ionic_lif *lif);
  44 static int ionic_txrx_init(struct ionic_lif *lif);
  45 static int ionic_start_queues(struct ionic_lif *lif);
  46 static void ionic_stop_queues(struct ionic_lif *lif);
  47 static void ionic_lif_queue_identify(struct ionic_lif *lif);
  48
  49 static void ionic_dim_work(struct work_struct *work)
  50 {
  51         struct dim *dim = container_of(work, struct dim, work);
  52         struct dim_cq_moder cur_moder;
  53         struct ionic_qcq *qcq;
  54         u32 new_coal;
  55
  56         cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
  57         qcq = container_of(dim, struct ionic_qcq, dim);
  58         new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
  59         new_coal = new_coal ? new_coal : 1;
  60
  61         if (qcq->intr.dim_coal_hw != new_coal) {
  62                 unsigned int qi = qcq->cq.bound_q->index;
  63                 struct ionic_lif *lif = qcq->q.lif;
  64
  65                 qcq->intr.dim_coal_hw = new_coal;
  66
  67                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
  68                                      lif->rxqcqs[qi]->intr.index,
  69                                      qcq->intr.dim_coal_hw);
  70         }
  71
  72         dim->state = DIM_START_MEASURE;
  73 }
  74
  75 static void ionic_lif_deferred_work(struct work_struct *work)
  76 {
  77         struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
  78         struct ionic_deferred *def = &lif->deferred;
  79         struct ionic_deferred_work *w = NULL;
  80
  81         do {
  82                 spin_lock_bh(&def->lock);
  83                 if (!list_empty(&def->list)) {
  84                         w = list_first_entry(&def->list,
  85                                              struct ionic_deferred_work, list);
  86                         list_del(&w->list);
  87                 }
  88                 spin_unlock_bh(&def->lock);
  89
  90                 if (!w)
  91                         break;
  92
  93                 switch (w->type) {
  94                 case IONIC_DW_TYPE_RX_MODE:
  95                         ionic_lif_rx_mode(lif);
  96                         break;
  97                 case IONIC_DW_TYPE_LINK_STATUS:
  98                         ionic_link_status_check(lif);
  99                         break;
 100                 case IONIC_DW_TYPE_LIF_RESET:
 101                         if (w->fw_status) {
 102                                 ionic_lif_handle_fw_up(lif);
 103                         } else {
 104                                 ionic_lif_handle_fw_down(lif);
 105
 106                                 /* Fire off another watchdog to see
 107                                  * if the FW is already back rather than
 108                                  * waiting another whole cycle
 109                                  */
 110                                 mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
 111                         }
 112                         break;
 113                 default:
 114                         break;
 115                 }
 116                 kfree(w);
 117                 w = NULL;
 118         } while (true);
 119 }
 120
 121 void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
 122                                 struct ionic_deferred_work *work)
 123 {
 124         spin_lock_bh(&def->lock);
 125         list_add_tail(&work->list, &def->list);
 126         spin_unlock_bh(&def->lock);
 127         schedule_work(&def->work);
 128 }
 129
 130 static void ionic_link_status_check(struct ionic_lif *lif)
 131 {
 132         struct net_device *netdev = lif->netdev;
 133         u16 link_status;
 134         bool link_up;
 135
 136         if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 137                 return;
 138
 139         /* Don't put carrier back up if we're in a broken state */
 140         if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
 141                 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 142                 return;
 143         }
 144
 145         link_status = le16_to_cpu(lif->info->status.link_status);
 146         link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 147
 148         if (link_up) {
 149                 int err = 0;
 150
 151                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 152                         mutex_lock(&lif->queue_lock);
 153                         err = ionic_start_queues(lif);
 154                         if (err && err != -EBUSY) {
 155                                 netdev_err(netdev,
 156                                            "Failed to start queues: %d\n", err);
 157                                 set_bit(IONIC_LIF_F_BROKEN, lif->state);
 158                                 netif_carrier_off(lif->netdev);
 159                         }
 160                         mutex_unlock(&lif->queue_lock);
 161                 }
 162
 163                 if (!err && !netif_carrier_ok(netdev)) {
 164                         ionic_port_identify(lif->ionic);
 165                         netdev_info(netdev, "Link up - %d Gbps\n",
 166                                     le32_to_cpu(lif->info->status.link_speed) / 1000);
 167                         netif_carrier_on(netdev);
 168                 }
 169         } else {
 170                 if (netif_carrier_ok(netdev)) {
 171                         lif->link_down_count++;
 172                         netdev_info(netdev, "Link down\n");
 173                         netif_carrier_off(netdev);
 174                 }
 175
 176                 if (netdev->flags & IFF_UP && netif_running(netdev)) {
 177                         mutex_lock(&lif->queue_lock);
 178                         ionic_stop_queues(lif);
 179                         mutex_unlock(&lif->queue_lock);
 180                 }
 181         }
 182
 183         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 184 }
 185
 186 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
 187 {
 188         struct ionic_deferred_work *work;
 189
 190         /* we only need one request outstanding at a time */
 191         if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 192                 return;
 193
 194         if (!can_sleep) {
 195                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
 196                 if (!work) {
 197                         clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 198                         return;
 199                 }
 200
 201                 work->type = IONIC_DW_TYPE_LINK_STATUS;
 202                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 203         } else {
 204                 ionic_link_status_check(lif);
 205         }
 206 }
 207
 208 static void ionic_napi_deadline(struct timer_list *timer)
 209 {
 210         struct ionic_qcq *qcq = container_of(timer, struct ionic_qcq, napi_deadline);
 211
 212         napi_schedule(&qcq->napi);
 213 }
 214
 215 static irqreturn_t ionic_isr(int irq, void *data)
 216 {
 217         struct napi_struct *napi = data;
 218
 219         napi_schedule_irqoff(napi);
 220
 221         return IRQ_HANDLED;
 222 }
 223
 224 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
 225 {
 226         struct ionic_intr_info *intr = &qcq->intr;
 227         struct device *dev = lif->ionic->dev;
 228         struct ionic_queue *q = &qcq->q;
 229         const char *name;
 230
 231         if (lif->registered)
 232                 name = lif->netdev->name;
 233         else
 234                 name = dev_name(dev);
 235
 236         snprintf(intr->name, sizeof(intr->name),
 237                  "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
 238
 239         return devm_request_irq(dev, intr->vector, ionic_isr,
 240                                 0, intr->name, &qcq->napi);
 241 }
 242
 243 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 244 {
 245         struct ionic *ionic = lif->ionic;
 246         int index;
 247
 248         index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
 249         if (index == ionic->nintrs) {
 250                 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
 251                             __func__, index, ionic->nintrs);
 252                 return -ENOSPC;
 253         }
 254
 255         set_bit(index, ionic->intrs);
 256         ionic_intr_init(&ionic->idev, intr, index);
 257
 258         return 0;
 259 }
 260
 261 static void ionic_intr_free(struct ionic *ionic, int index)
 262 {
 263         if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
 264                 clear_bit(index, ionic->intrs);
 265 }
 266
 267 static int ionic_qcq_enable(struct ionic_qcq *qcq)
 268 {
 269         struct ionic_queue *q = &qcq->q;
 270         struct ionic_lif *lif = q->lif;
 271         struct ionic_dev *idev;
 272         struct device *dev;
 273
 274         struct ionic_admin_ctx ctx = {
 275                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 276                 .cmd.q_control = {
 277                         .opcode = IONIC_CMD_Q_CONTROL,
 278                         .lif_index = cpu_to_le16(lif->index),
 279                         .type = q->type,
 280                         .index = cpu_to_le32(q->index),
 281                         .oper = IONIC_Q_ENABLE,
 282                 },
 283         };
 284         int ret;
 285
 286         idev = &lif->ionic->idev;
 287         dev = lif->ionic->dev;
 288
 289         dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
 290                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 291
 292         if (qcq->flags & IONIC_QCQ_F_INTR)
 293                 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
 294
 295         ret = ionic_adminq_post_wait(lif, &ctx);
 296         if (ret)
 297                 return ret;
 298
 299         if (qcq->napi.poll)
 300                 napi_enable(&qcq->napi);
 301
 302         if (qcq->flags & IONIC_QCQ_F_INTR) {
 303                 irq_set_affinity_hint(qcq->intr.vector,
 304                                       &qcq->intr.affinity_mask);
 305                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 306                                 IONIC_INTR_MASK_CLEAR);
 307         }
 308
 309         return 0;
 310 }
 311
 312 static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
 313 {
 314         struct ionic_queue *q;
 315
 316         struct ionic_admin_ctx ctx = {
 317                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 318                 .cmd.q_control = {
 319                         .opcode = IONIC_CMD_Q_CONTROL,
 320                         .oper = IONIC_Q_DISABLE,
 321                 },
 322         };
 323
 324         if (!qcq) {
 325                 netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
 326                 return -ENXIO;
 327         }
 328
 329         q = &qcq->q;
 330
 331         if (qcq->flags & IONIC_QCQ_F_INTR) {
 332                 struct ionic_dev *idev = &lif->ionic->idev;
 333
 334                 cancel_work_sync(&qcq->dim.work);
 335                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 336                                 IONIC_INTR_MASK_SET);
 337                 synchronize_irq(qcq->intr.vector);
 338                 irq_set_affinity_hint(qcq->intr.vector, NULL);
 339                 napi_disable(&qcq->napi);
 340                 del_timer_sync(&qcq->napi_deadline);
 341         }
 342
 343         /* If there was a previous fw communcation error, don't bother with
 344          * sending the adminq command and just return the same error value.
 345          */
 346         if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
 347                 return fw_err;
 348
 349         ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
 350         ctx.cmd.q_control.type = q->type;
 351         ctx.cmd.q_control.index = cpu_to_le32(q->index);
 352         dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
 353                 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 354
 355         return ionic_adminq_post_wait(lif, &ctx);
 356 }
 357
 358 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 359 {
 360         struct ionic_dev *idev = &lif->ionic->idev;
 361
 362         if (!qcq)
 363                 return;
 364
 365         if (!(qcq->flags & IONIC_QCQ_F_INITED))
 366                 return;
 367
 368         if (qcq->flags & IONIC_QCQ_F_INTR) {
 369                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 370                                 IONIC_INTR_MASK_SET);
 371                 netif_napi_del(&qcq->napi);
 372         }
 373
 374         qcq->flags &= ~IONIC_QCQ_F_INITED;
 375 }
 376
 377 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 378 {
 379         if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
 380                 return;
 381
 382         irq_set_affinity_hint(qcq->intr.vector, NULL);
 383         devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
 384         qcq->intr.vector = 0;
 385         ionic_intr_free(lif->ionic, qcq->intr.index);
 386         qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 387 }
 388
 389 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 390 {
 391         struct device *dev = lif->ionic->dev;
 392
 393         if (!qcq)
 394                 return;
 395
 396         ionic_debugfs_del_qcq(qcq);
 397
 398         if (qcq->q_base) {
 399                 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
 400                 qcq->q_base = NULL;
 401                 qcq->q_base_pa = 0;
 402         }
 403
 404         if (qcq->cmb_q_base) {
 405                 iounmap(qcq->cmb_q_base);
 406                 ionic_put_cmb(lif, qcq->cmb_pgid, qcq->cmb_order);
 407                 qcq->cmb_pgid = 0;
 408                 qcq->cmb_order = 0;
 409                 qcq->cmb_q_base = NULL;
 410                 qcq->cmb_q_base_pa = 0;
 411         }
 412
 413         if (qcq->cq_base) {
 414                 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
 415                 qcq->cq_base = NULL;
 416                 qcq->cq_base_pa = 0;
 417         }
 418
 419         if (qcq->sg_base) {
 420                 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
 421                 qcq->sg_base = NULL;
 422                 qcq->sg_base_pa = 0;
 423         }
 424
 425         ionic_qcq_intr_free(lif, qcq);
 426
 427         if (qcq->cq.info) {
 428                 vfree(qcq->cq.info);
 429                 qcq->cq.info = NULL;
 430         }
 431         if (qcq->q.info) {
 432                 vfree(qcq->q.info);
 433                 qcq->q.info = NULL;
 434         }
 435 }
 436
 437 static void ionic_qcqs_free(struct ionic_lif *lif)
 438 {
 439         struct device *dev = lif->ionic->dev;
 440         struct ionic_qcq *adminqcq;
 441         unsigned long irqflags;
 442
 443         if (lif->notifyqcq) {
 444                 ionic_qcq_free(lif, lif->notifyqcq);
 445                 devm_kfree(dev, lif->notifyqcq);
 446                 lif->notifyqcq = NULL;
 447         }
 448
 449         if (lif->adminqcq) {
 450                 spin_lock_irqsave(&lif->adminq_lock, irqflags);
 451                 adminqcq = READ_ONCE(lif->adminqcq);
 452                 lif->adminqcq = NULL;
 453                 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
 454                 if (adminqcq) {
 455                         ionic_qcq_free(lif, adminqcq);
 456                         devm_kfree(dev, adminqcq);
 457                 }
 458         }
 459
 460         if (lif->rxqcqs) {
 461                 devm_kfree(dev, lif->rxqstats);
 462                 lif->rxqstats = NULL;
 463                 devm_kfree(dev, lif->rxqcqs);
 464                 lif->rxqcqs = NULL;
 465         }
 466
 467         if (lif->txqcqs) {
 468                 devm_kfree(dev, lif->txqstats);
 469                 lif->txqstats = NULL;
 470                 devm_kfree(dev, lif->txqcqs);
 471                 lif->txqcqs = NULL;
 472         }
 473 }
 474
 475 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
 476                                       struct ionic_qcq *n_qcq)
 477 {
 478         if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
 479                 ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
 480                 n_qcq->flags &= ~IONIC_QCQ_F_INTR;
 481         }
 482
 483         n_qcq->intr.vector = src_qcq->intr.vector;
 484         n_qcq->intr.index = src_qcq->intr.index;
 485         n_qcq->napi_qcq = src_qcq->napi_qcq;
 486 }
 487
 488 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
 489 {
 490         int err;
 491
 492         if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
 493                 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 494                 return 0;
 495         }
 496
 497         err = ionic_intr_alloc(lif, &qcq->intr);
 498         if (err) {
 499                 netdev_warn(lif->netdev, "no intr for %s: %d\n",
 500                             qcq->q.name, err);
 501                 goto err_out;
 502         }
 503
 504         err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
 505         if (err < 0) {
 506                 netdev_warn(lif->netdev, "no vector for %s: %d\n",
 507                             qcq->q.name, err);
 508                 goto err_out_free_intr;
 509         }
 510         qcq->intr.vector = err;
 511         ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
 512                                IONIC_INTR_MASK_SET);
 513
 514         err = ionic_request_irq(lif, qcq);
 515         if (err) {
 516                 netdev_warn(lif->netdev, "irq request failed %d\n", err);
 517                 goto err_out_free_intr;
 518         }
 519
 520         /* try to get the irq on the local numa node first */
 521         qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
 522                                              dev_to_node(lif->ionic->dev));
 523         if (qcq->intr.cpu != -1)
 524                 cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
 525
 526         netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
 527         return 0;
 528
 529 err_out_free_intr:
 530         ionic_intr_free(lif->ionic, qcq->intr.index);
 531 err_out:
 532         return err;
 533 }
 534
 535 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
 536                            unsigned int index,
 537                            const char *name, unsigned int flags,
 538                            unsigned int num_descs, unsigned int desc_size,
 539                            unsigned int cq_desc_size,
 540                            unsigned int sg_desc_size,
 541                            unsigned int pid, struct ionic_qcq **qcq)
 542 {
 543         struct ionic_dev *idev = &lif->ionic->idev;
 544         struct device *dev = lif->ionic->dev;
 545         void *q_base, *cq_base, *sg_base;
 546         dma_addr_t cq_base_pa = 0;
 547         dma_addr_t sg_base_pa = 0;
 548         dma_addr_t q_base_pa = 0;
 549         struct ionic_qcq *new;
 550         int err;
 551
 552         *qcq = NULL;
 553
 554         new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
 555         if (!new) {
 556                 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
 557                 err = -ENOMEM;
 558                 goto err_out;
 559         }
 560
 561         new->q.dev = dev;
 562         new->flags = flags;
 563
 564         new->q.info = vcalloc(num_descs, sizeof(*new->q.info));
 565         if (!new->q.info) {
 566                 netdev_err(lif->netdev, "Cannot allocate queue info\n");
 567                 err = -ENOMEM;
 568                 goto err_out_free_qcq;
 569         }
 570
 571         new->q.type = type;
 572         new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
 573
 574         err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
 575                            desc_size, sg_desc_size, pid);
 576         if (err) {
 577                 netdev_err(lif->netdev, "Cannot initialize queue\n");
 578                 goto err_out_free_q_info;
 579         }
 580
 581         err = ionic_alloc_qcq_interrupt(lif, new);
 582         if (err)
 583                 goto err_out;
 584
 585         new->cq.info = vcalloc(num_descs, sizeof(*new->cq.info));
 586         if (!new->cq.info) {
 587                 netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
 588                 err = -ENOMEM;
 589                 goto err_out_free_irq;
 590         }
 591
 592         err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
 593         if (err) {
 594                 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
 595                 goto err_out_free_cq_info;
 596         }
 597
 598         if (flags & IONIC_QCQ_F_NOTIFYQ) {
 599                 int q_size;
 600
 601                 /* q & cq need to be contiguous in NotifyQ, so alloc it all in q
 602                  * and don't alloc qc.  We leave new->qc_size and new->qc_base
 603                  * as 0 to be sure we don't try to free it later.
 604                  */
 605                 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
 606                 new->q_size = PAGE_SIZE + q_size +
 607                               ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
 608                 new->q_base = dma_alloc_coherent(dev, new->q_size,
 609                                                  &new->q_base_pa, GFP_KERNEL);
 610                 if (!new->q_base) {
 611                         netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
 612                         err = -ENOMEM;
 613                         goto err_out_free_cq_info;
 614                 }
 615                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 616                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 617                 ionic_q_map(&new->q, q_base, q_base_pa);
 618
 619                 cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
 620                 cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
 621                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 622                 ionic_cq_bind(&new->cq, &new->q);
 623         } else {
 624                 /* regular DMA q descriptors */
 625                 new->q_size = PAGE_SIZE + (num_descs * desc_size);
 626                 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
 627                                                  GFP_KERNEL);
 628                 if (!new->q_base) {
 629                         netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
 630                         err = -ENOMEM;
 631                         goto err_out_free_cq_info;
 632                 }
 633                 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 634                 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 635                 ionic_q_map(&new->q, q_base, q_base_pa);
 636
 637                 if (flags & IONIC_QCQ_F_CMB_RINGS) {
 638                         /* on-chip CMB q descriptors */
 639                         new->cmb_q_size = num_descs * desc_size;
 640                         new->cmb_order = order_base_2(new->cmb_q_size / PAGE_SIZE);
 641
 642                         err = ionic_get_cmb(lif, &new->cmb_pgid, &new->cmb_q_base_pa,
 643                                             new->cmb_order);
 644                         if (err) {
 645                                 netdev_err(lif->netdev,
 646                                            "Cannot allocate queue order %d from cmb: err %d\n",
 647                                            new->cmb_order, err);
 648                                 goto err_out_free_q;
 649                         }
 650
 651                         new->cmb_q_base = ioremap_wc(new->cmb_q_base_pa, new->cmb_q_size);
 652                         if (!new->cmb_q_base) {
 653                                 netdev_err(lif->netdev, "Cannot map queue from cmb\n");
 654                                 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
 655                                 err = -ENOMEM;
 656                                 goto err_out_free_q;
 657                         }
 658
 659                         new->cmb_q_base_pa -= idev->phy_cmb_pages;
 660                         ionic_q_cmb_map(&new->q, new->cmb_q_base, new->cmb_q_base_pa);
 661                 }
 662
 663                 /* cq DMA descriptors */
 664                 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
 665                 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
 666                                                   GFP_KERNEL);
 667                 if (!new->cq_base) {
 668                         netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
 669                         err = -ENOMEM;
 670                         goto err_out_free_q;
 671                 }
 672                 cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
 673                 cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
 674                 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 675                 ionic_cq_bind(&new->cq, &new->q);
 676         }
 677
 678         if (flags & IONIC_QCQ_F_SG) {
 679                 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
 680                 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
 681                                                   GFP_KERNEL);
 682                 if (!new->sg_base) {
 683                         netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
 684                         err = -ENOMEM;
 685                         goto err_out_free_cq;
 686                 }
 687                 sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
 688                 sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
 689                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 690         }
 691
 692         INIT_WORK(&new->dim.work, ionic_dim_work);
 693         new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 694
 695         *qcq = new;
 696
 697         return 0;
 698
 699 err_out_free_cq:
 700         dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
 701 err_out_free_q:
 702         if (new->cmb_q_base) {
 703                 iounmap(new->cmb_q_base);
 704                 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
 705         }
 706         dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
 707 err_out_free_cq_info:
 708         vfree(new->cq.info);
 709 err_out_free_irq:
 710         if (flags & IONIC_QCQ_F_INTR) {
 711                 devm_free_irq(dev, new->intr.vector, &new->napi);
 712                 ionic_intr_free(lif->ionic, new->intr.index);
 713         }
 714 err_out_free_q_info:
 715         vfree(new->q.info);
 716 err_out_free_qcq:
 717         devm_kfree(dev, new);
 718 err_out:
 719         dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
 720         return err;
 721 }
 722
 723 static int ionic_qcqs_alloc(struct ionic_lif *lif)
 724 {
 725         struct device *dev = lif->ionic->dev;
 726         unsigned int flags;
 727         int err;
 728
 729         flags = IONIC_QCQ_F_INTR;
 730         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 731                               IONIC_ADMINQ_LENGTH,
 732                               sizeof(struct ionic_admin_cmd),
 733                               sizeof(struct ionic_admin_comp),
 734                               0, lif->kern_pid, &lif->adminqcq);
 735         if (err)
 736                 return err;
 737         ionic_debugfs_add_qcq(lif, lif->adminqcq);
 738
 739         if (lif->ionic->nnqs_per_lif) {
 740                 flags = IONIC_QCQ_F_NOTIFYQ;
 741                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
 742                                       flags, IONIC_NOTIFYQ_LENGTH,
 743                                       sizeof(struct ionic_notifyq_cmd),
 744                                       sizeof(union ionic_notifyq_comp),
 745                                       0, lif->kern_pid, &lif->notifyqcq);
 746                 if (err)
 747                         goto err_out;
 748                 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
 749
 750                 /* Let the notifyq ride on the adminq interrupt */
 751                 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
 752         }
 753
 754         err = -ENOMEM;
 755         lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
 756                                    sizeof(*lif->txqcqs), GFP_KERNEL);
 757         if (!lif->txqcqs)
 758                 goto err_out;
 759         lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
 760                                    sizeof(*lif->rxqcqs), GFP_KERNEL);
 761         if (!lif->rxqcqs)
 762                 goto err_out;
 763
 764         lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
 765                                      sizeof(*lif->txqstats), GFP_KERNEL);
 766         if (!lif->txqstats)
 767                 goto err_out;
 768         lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
 769                                      sizeof(*lif->rxqstats), GFP_KERNEL);
 770         if (!lif->rxqstats)
 771                 goto err_out;
 772
 773         return 0;
 774
 775 err_out:
 776         ionic_qcqs_free(lif);
 777         return err;
 778 }
 779
 780 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
 781 {
 782         qcq->q.tail_idx = 0;
 783         qcq->q.head_idx = 0;
 784         qcq->cq.tail_idx = 0;
 785         qcq->cq.done_color = 1;
 786         memset(qcq->q_base, 0, qcq->q_size);
 787         if (qcq->cmb_q_base)
 788                 memset_io(qcq->cmb_q_base, 0, qcq->cmb_q_size);
 789         memset(qcq->cq_base, 0, qcq->cq_size);
 790         memset(qcq->sg_base, 0, qcq->sg_size);
 791 }
 792
 793 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 794 {
 795         struct device *dev = lif->ionic->dev;
 796         struct ionic_queue *q = &qcq->q;
 797         struct ionic_cq *cq = &qcq->cq;
 798         struct ionic_admin_ctx ctx = {
 799                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 800                 .cmd.q_init = {
 801                         .opcode = IONIC_CMD_Q_INIT,
 802                         .lif_index = cpu_to_le16(lif->index),
 803                         .type = q->type,
 804                         .ver = lif->qtype_info[q->type].version,
 805                         .index = cpu_to_le32(q->index),
 806                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 807                                              IONIC_QINIT_F_SG),
 808                         .intr_index = cpu_to_le16(qcq->intr.index),
 809                         .pid = cpu_to_le16(q->pid),
 810                         .ring_size = ilog2(q->num_descs),
 811                         .ring_base = cpu_to_le64(q->base_pa),
 812                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 813                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 814                         .features = cpu_to_le64(q->features),
 815                 },
 816         };
 817         int err;
 818
 819         if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
 820                 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
 821                 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
 822         }
 823
 824         dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
 825         dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
 826         dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 827         dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 828         dev_dbg(dev, "txq_init.cq_ring_base 0x%llx\n", ctx.cmd.q_init.cq_ring_base);
 829         dev_dbg(dev, "txq_init.sg_ring_base 0x%llx\n", ctx.cmd.q_init.sg_ring_base);
 830         dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 831         dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
 832         dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 833
 834         ionic_qcq_sanitize(qcq);
 835
 836         err = ionic_adminq_post_wait(lif, &ctx);
 837         if (err)
 838                 return err;
 839
 840         q->hw_type = ctx.comp.q_init.hw_type;
 841         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 842         q->dbval = IONIC_DBELL_QID(q->hw_index);
 843
 844         dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
 845         dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 846
 847         q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
 848         q->dbell_jiffies = jiffies;
 849
 850         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
 851                 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
 852                 qcq->napi_qcq = qcq;
 853                 timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
 854         }
 855
 856         qcq->flags |= IONIC_QCQ_F_INITED;
 857
 858         return 0;
 859 }
 860
 861 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 862 {
 863         struct device *dev = lif->ionic->dev;
 864         struct ionic_queue *q = &qcq->q;
 865         struct ionic_cq *cq = &qcq->cq;
 866         struct ionic_admin_ctx ctx = {
 867                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 868                 .cmd.q_init = {
 869                         .opcode = IONIC_CMD_Q_INIT,
 870                         .lif_index = cpu_to_le16(lif->index),
 871                         .type = q->type,
 872                         .ver = lif->qtype_info[q->type].version,
 873                         .index = cpu_to_le32(q->index),
 874                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 875                                              IONIC_QINIT_F_SG),
 876                         .intr_index = cpu_to_le16(cq->bound_intr->index),
 877                         .pid = cpu_to_le16(q->pid),
 878                         .ring_size = ilog2(q->num_descs),
 879                         .ring_base = cpu_to_le64(q->base_pa),
 880                         .cq_ring_base = cpu_to_le64(cq->base_pa),
 881                         .sg_ring_base = cpu_to_le64(q->sg_base_pa),
 882                         .features = cpu_to_le64(q->features),
 883                 },
 884         };
 885         int err;
 886
 887         if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
 888                 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
 889                 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
 890         }
 891
 892         dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
 893         dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
 894         dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 895         dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 896         dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 897         dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
 898         dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 899
 900         ionic_qcq_sanitize(qcq);
 901
 902         err = ionic_adminq_post_wait(lif, &ctx);
 903         if (err)
 904                 return err;
 905
 906         q->hw_type = ctx.comp.q_init.hw_type;
 907         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 908         q->dbval = IONIC_DBELL_QID(q->hw_index);
 909
 910         dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
 911         dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 912
 913         q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
 914         q->dbell_jiffies = jiffies;
 915
 916         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 917                 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
 918         else
 919                 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
 920
 921         qcq->napi_qcq = qcq;
 922         timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
 923
 924         qcq->flags |= IONIC_QCQ_F_INITED;
 925
 926         return 0;
 927 }
 928
 929 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
 930 {
 931         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 932         unsigned int txq_i, flags;
 933         struct ionic_qcq *txq;
 934         u64 features;
 935         int err;
 936
 937         if (lif->hwstamp_txq)
 938                 return 0;
 939
 940         features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
 941
 942         num_desc = IONIC_MIN_TXRX_DESC;
 943         desc_sz = sizeof(struct ionic_txq_desc);
 944         comp_sz = 2 * sizeof(struct ionic_txq_comp);
 945
 946         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
 947             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
 948                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
 949         else
 950                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
 951
 952         txq_i = lif->ionic->ntxqs_per_lif;
 953         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
 954
 955         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
 956                               num_desc, desc_sz, comp_sz, sg_desc_sz,
 957                               lif->kern_pid, &txq);
 958         if (err)
 959                 goto err_qcq_alloc;
 960
 961         txq->q.features = features;
 962
 963         ionic_link_qcq_interrupts(lif->adminqcq, txq);
 964         ionic_debugfs_add_qcq(lif, txq);
 965
 966         lif->hwstamp_txq = txq;
 967
 968         if (netif_running(lif->netdev)) {
 969                 err = ionic_lif_txq_init(lif, txq);
 970                 if (err)
 971                         goto err_qcq_init;
 972
 973                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 974                         err = ionic_qcq_enable(txq);
 975                         if (err)
 976                                 goto err_qcq_enable;
 977                 }
 978         }
 979
 980         return 0;
 981
 982 err_qcq_enable:
 983         ionic_lif_qcq_deinit(lif, txq);
 984 err_qcq_init:
 985         lif->hwstamp_txq = NULL;
 986         ionic_debugfs_del_qcq(txq);
 987         ionic_qcq_free(lif, txq);
 988         devm_kfree(lif->ionic->dev, txq);
 989 err_qcq_alloc:
 990         return err;
 991 }
 992
 993 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
 994 {
 995         unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 996         unsigned int rxq_i, flags;
 997         struct ionic_qcq *rxq;
 998         u64 features;
 999         int err;
1000
1001         if (lif->hwstamp_rxq)
1002                 return 0;
1003
1004         features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1005
1006         num_desc = IONIC_MIN_TXRX_DESC;
1007         desc_sz = sizeof(struct ionic_rxq_desc);
1008         comp_sz = 2 * sizeof(struct ionic_rxq_comp);
1009         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
1010
1011         rxq_i = lif->ionic->nrxqs_per_lif;
1012         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
1013
1014         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
1015                               num_desc, desc_sz, comp_sz, sg_desc_sz,
1016                               lif->kern_pid, &rxq);
1017         if (err)
1018                 goto err_qcq_alloc;
1019
1020         rxq->q.features = features;
1021
1022         ionic_link_qcq_interrupts(lif->adminqcq, rxq);
1023         ionic_debugfs_add_qcq(lif, rxq);
1024
1025         lif->hwstamp_rxq = rxq;
1026
1027         if (netif_running(lif->netdev)) {
1028                 err = ionic_lif_rxq_init(lif, rxq);
1029                 if (err)
1030                         goto err_qcq_init;
1031
1032                 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1033                         ionic_rx_fill(&rxq->q);
1034                         err = ionic_qcq_enable(rxq);
1035                         if (err)
1036                                 goto err_qcq_enable;
1037                 }
1038         }
1039
1040         return 0;
1041
1042 err_qcq_enable:
1043         ionic_lif_qcq_deinit(lif, rxq);
1044 err_qcq_init:
1045         lif->hwstamp_rxq = NULL;
1046         ionic_debugfs_del_qcq(rxq);
1047         ionic_qcq_free(lif, rxq);
1048         devm_kfree(lif->ionic->dev, rxq);
1049 err_qcq_alloc:
1050         return err;
1051 }
1052
1053 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
1054 {
1055         struct ionic_queue_params qparam;
1056
1057         ionic_init_queue_params(lif, &qparam);
1058
1059         if (rx_all)
1060                 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1061         else
1062                 qparam.rxq_features = 0;
1063
1064         /* if we're not running, just set the values and return */
1065         if (!netif_running(lif->netdev)) {
1066                 lif->rxq_features = qparam.rxq_features;
1067                 return 0;
1068         }
1069
1070         return ionic_reconfigure_queues(lif, &qparam);
1071 }
1072
1073 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1074 {
1075         struct ionic_admin_ctx ctx = {
1076                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1077                 .cmd.lif_setattr = {
1078                         .opcode = IONIC_CMD_LIF_SETATTR,
1079                         .index = cpu_to_le16(lif->index),
1080                         .attr = IONIC_LIF_ATTR_TXSTAMP,
1081                         .txstamp_mode = cpu_to_le16(txstamp_mode),
1082                 },
1083         };
1084
1085         return ionic_adminq_post_wait(lif, &ctx);
1086 }
1087
1088 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1089 {
1090         struct ionic_admin_ctx ctx = {
1091                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1092                 .cmd.rx_filter_del = {
1093                         .opcode = IONIC_CMD_RX_FILTER_DEL,
1094                         .lif_index = cpu_to_le16(lif->index),
1095                 },
1096         };
1097         struct ionic_rx_filter *f;
1098         u32 filter_id;
1099         int err;
1100
1101         spin_lock_bh(&lif->rx_filters.lock);
1102
1103         f = ionic_rx_filter_rxsteer(lif);
1104         if (!f) {
1105                 spin_unlock_bh(&lif->rx_filters.lock);
1106                 return;
1107         }
1108
1109         filter_id = f->filter_id;
1110         ionic_rx_filter_free(lif, f);
1111
1112         spin_unlock_bh(&lif->rx_filters.lock);
1113
1114         netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1115
1116         ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1117
1118         err = ionic_adminq_post_wait(lif, &ctx);
1119         if (err && err != -EEXIST)
1120                 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1121 }
1122
1123 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1124 {
1125         struct ionic_admin_ctx ctx = {
1126                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1127                 .cmd.rx_filter_add = {
1128                         .opcode = IONIC_CMD_RX_FILTER_ADD,
1129                         .lif_index = cpu_to_le16(lif->index),
1130                         .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1131                         .pkt_class = cpu_to_le64(pkt_class),
1132                 },
1133         };
1134         u8 qtype;
1135         u32 qid;
1136         int err;
1137
1138         if (!lif->hwstamp_rxq)
1139                 return -EINVAL;
1140
1141         qtype = lif->hwstamp_rxq->q.type;
1142         ctx.cmd.rx_filter_add.qtype = qtype;
1143
1144         qid = lif->hwstamp_rxq->q.index;
1145         ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1146
1147         netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1148         err = ionic_adminq_post_wait(lif, &ctx);
1149         if (err && err != -EEXIST)
1150                 return err;
1151
1152         spin_lock_bh(&lif->rx_filters.lock);
1153         err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
1154         spin_unlock_bh(&lif->rx_filters.lock);
1155
1156         return err;
1157 }
1158
1159 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1160 {
1161         ionic_lif_del_hwstamp_rxfilt(lif);
1162
1163         if (!pkt_class)
1164                 return 0;
1165
1166         return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1167 }
1168
1169 static bool ionic_notifyq_service(struct ionic_cq *cq,
1170                                   struct ionic_cq_info *cq_info)
1171 {
1172         union ionic_notifyq_comp *comp = cq_info->cq_desc;
1173         struct ionic_deferred_work *work;
1174         struct net_device *netdev;
1175         struct ionic_queue *q;
1176         struct ionic_lif *lif;
1177         u64 eid;
1178
1179         q = cq->bound_q;
1180         lif = q->info[0].cb_arg;
1181         netdev = lif->netdev;
1182         eid = le64_to_cpu(comp->event.eid);
1183
1184         /* Have we run out of new completions to process? */
1185         if ((s64)(eid - lif->last_eid) <= 0)
1186                 return false;
1187
1188         lif->last_eid = eid;
1189
1190         dev_dbg(lif->ionic->dev, "notifyq event:\n");
1191         dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1192                          comp, sizeof(*comp), true);
1193
1194         switch (le16_to_cpu(comp->event.ecode)) {
1195         case IONIC_EVENT_LINK_CHANGE:
1196                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1197                 break;
1198         case IONIC_EVENT_RESET:
1199                 if (lif->ionic->idev.fw_status_ready &&
1200                     !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
1201                     !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
1202                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1203                         if (!work) {
1204                                 netdev_err(lif->netdev, "Reset event dropped\n");
1205                                 clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
1206                         } else {
1207                                 work->type = IONIC_DW_TYPE_LIF_RESET;
1208                                 ionic_lif_deferred_enqueue(&lif->deferred, work);
1209                         }
1210                 }
1211                 break;
1212         default:
1213                 netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1214                             comp->event.ecode, eid);
1215                 break;
1216         }
1217
1218         return true;
1219 }
1220
1221 static bool ionic_adminq_service(struct ionic_cq *cq,
1222                                  struct ionic_cq_info *cq_info)
1223 {
1224         struct ionic_admin_comp *comp = cq_info->cq_desc;
1225
1226         if (!color_match(comp->color, cq->done_color))
1227                 return false;
1228
1229         ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1230
1231         return true;
1232 }
1233
1234 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1235 {
1236         struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1237         struct ionic_lif *lif = napi_to_cq(napi)->lif;
1238         struct ionic_dev *idev = &lif->ionic->idev;
1239         unsigned long irqflags;
1240         unsigned int flags = 0;
1241         bool resched = false;
1242         int rx_work = 0;
1243         int tx_work = 0;
1244         int n_work = 0;
1245         int a_work = 0;
1246         int work_done;
1247         int credits;
1248
1249         if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1250                 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1251                                           ionic_notifyq_service, NULL, NULL);
1252
1253         spin_lock_irqsave(&lif->adminq_lock, irqflags);
1254         if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1255                 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1256                                           ionic_adminq_service, NULL, NULL);
1257         spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1258
1259         if (lif->hwstamp_rxq)
1260                 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1261                                            ionic_rx_service, NULL, NULL);
1262
1263         if (lif->hwstamp_txq)
1264                 tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1265                                            ionic_tx_service, NULL, NULL);
1266
1267         work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1268         if (work_done < budget && napi_complete_done(napi, work_done)) {
1269                 flags |= IONIC_INTR_CRED_UNMASK;
1270                 intr->rearm_count++;
1271         }
1272
1273         if (work_done || flags) {
1274                 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1275                 credits = n_work + a_work + rx_work + tx_work;
1276                 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1277         }
1278
1279         if (!a_work && ionic_adminq_poke_doorbell(&lif->adminqcq->q))
1280                 resched = true;
1281         if (lif->hwstamp_rxq && !rx_work && ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q))
1282                 resched = true;
1283         if (lif->hwstamp_txq && !tx_work && ionic_txq_poke_doorbell(&lif->hwstamp_txq->q))
1284                 resched = true;
1285         if (resched)
1286                 mod_timer(&lif->adminqcq->napi_deadline,
1287                           jiffies + IONIC_NAPI_DEADLINE);
1288
1289         return work_done;
1290 }
1291
1292 void ionic_get_stats64(struct net_device *netdev,
1293                        struct rtnl_link_stats64 *ns)
1294 {
1295         struct ionic_lif *lif = netdev_priv(netdev);
1296         struct ionic_lif_stats *ls;
1297
1298         memset(ns, 0, sizeof(*ns));
1299         ls = &lif->info->stats;
1300
1301         ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1302                          le64_to_cpu(ls->rx_mcast_packets) +
1303                          le64_to_cpu(ls->rx_bcast_packets);
1304
1305         ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1306                          le64_to_cpu(ls->tx_mcast_packets) +
1307                          le64_to_cpu(ls->tx_bcast_packets);
1308
1309         ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1310                        le64_to_cpu(ls->rx_mcast_bytes) +
1311                        le64_to_cpu(ls->rx_bcast_bytes);
1312
1313         ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1314                        le64_to_cpu(ls->tx_mcast_bytes) +
1315                        le64_to_cpu(ls->tx_bcast_bytes);
1316
1317         ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1318                          le64_to_cpu(ls->rx_mcast_drop_packets) +
1319                          le64_to_cpu(ls->rx_bcast_drop_packets);
1320
1321         ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1322                          le64_to_cpu(ls->tx_mcast_drop_packets) +
1323                          le64_to_cpu(ls->tx_bcast_drop_packets);
1324
1325         ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1326
1327         ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1328
1329         ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1330                                le64_to_cpu(ls->rx_queue_disabled) +
1331                                le64_to_cpu(ls->rx_desc_fetch_error) +
1332                                le64_to_cpu(ls->rx_desc_data_error);
1333
1334         ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1335                                 le64_to_cpu(ls->tx_queue_disabled) +
1336                                 le64_to_cpu(ls->tx_desc_fetch_error) +
1337                                 le64_to_cpu(ls->tx_desc_data_error);
1338
1339         ns->rx_errors = ns->rx_over_errors +
1340                         ns->rx_missed_errors;
1341
1342         ns->tx_errors = ns->tx_aborted_errors;
1343 }
1344
1345 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1346 {
1347         return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
1348 }
1349
1350 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1351 {
1352         /* Don't delete our own address from the uc list */
1353         if (ether_addr_equal(addr, netdev->dev_addr))
1354                 return 0;
1355
1356         return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
1357 }
1358
1359 void ionic_lif_rx_mode(struct ionic_lif *lif)
1360 {
1361         struct net_device *netdev = lif->netdev;
1362         unsigned int nfilters;
1363         unsigned int nd_flags;
1364         char buf[128];
1365         u16 rx_mode;
1366         int i;
1367 #define REMAIN(__x) (sizeof(buf) - (__x))
1368
1369         mutex_lock(&lif->config_lock);
1370
1371         /* grab the flags once for local use */
1372         nd_flags = netdev->flags;
1373
1374         rx_mode = IONIC_RX_MODE_F_UNICAST;
1375         rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1376         rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1377         rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1378         rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1379
1380         /* sync the filters */
1381         ionic_rx_filter_sync(lif);
1382
1383         /* check for overflow state
1384          *    if so, we track that we overflowed and enable NIC PROMISC
1385          *    else if the overflow is set and not needed
1386          *       we remove our overflow flag and check the netdev flags
1387          *       to see if we can disable NIC PROMISC
1388          */
1389         nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1390
1391         if (((lif->nucast + lif->nmcast) >= nfilters) ||
1392             (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
1393                 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1394                 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1395         } else {
1396                 if (!(nd_flags & IFF_PROMISC))
1397                         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1398                 if (!(nd_flags & IFF_ALLMULTI))
1399                         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1400         }
1401
1402         i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1403                       lif->rx_mode, rx_mode);
1404         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1405                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1406         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1407                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1408         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1409                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1410         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1411                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1412         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1413                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1414         if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1415                 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1416         netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1417
1418         if (lif->rx_mode != rx_mode) {
1419                 struct ionic_admin_ctx ctx = {
1420                         .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1421                         .cmd.rx_mode_set = {
1422                                 .opcode = IONIC_CMD_RX_MODE_SET,
1423                                 .lif_index = cpu_to_le16(lif->index),
1424                         },
1425                 };
1426                 int err;
1427
1428                 ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1429                 err = ionic_adminq_post_wait(lif, &ctx);
1430                 if (err)
1431                         netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1432                                     rx_mode, err);
1433                 else
1434                         lif->rx_mode = rx_mode;
1435         }
1436
1437         mutex_unlock(&lif->config_lock);
1438 }
1439
1440 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1441 {
1442         struct ionic_lif *lif = netdev_priv(netdev);
1443         struct ionic_deferred_work *work;
1444
1445         /* Sync the kernel filter list with the driver filter list */
1446         __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1447         __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1448
1449         /* Shove off the rest of the rxmode work to the work task
1450          * which will include syncing the filters to the firmware.
1451          */
1452         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1453         if (!work) {
1454                 netdev_err(lif->netdev, "rxmode change dropped\n");
1455                 return;
1456         }
1457         work->type = IONIC_DW_TYPE_RX_MODE;
1458         netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1459         ionic_lif_deferred_enqueue(&lif->deferred, work);
1460 }
1461
1462 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1463 {
1464         u64 wanted = 0;
1465
1466         if (features & NETIF_F_HW_VLAN_CTAG_TX)
1467                 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1468         if (features & NETIF_F_HW_VLAN_CTAG_RX)
1469                 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1470         if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1471                 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1472         if (features & NETIF_F_RXHASH)
1473                 wanted |= IONIC_ETH_HW_RX_HASH;
1474         if (features & NETIF_F_RXCSUM)
1475                 wanted |= IONIC_ETH_HW_RX_CSUM;
1476         if (features & NETIF_F_SG)
1477                 wanted |= IONIC_ETH_HW_TX_SG;
1478         if (features & NETIF_F_HW_CSUM)
1479                 wanted |= IONIC_ETH_HW_TX_CSUM;
1480         if (features & NETIF_F_TSO)
1481                 wanted |= IONIC_ETH_HW_TSO;
1482         if (features & NETIF_F_TSO6)
1483                 wanted |= IONIC_ETH_HW_TSO_IPV6;
1484         if (features & NETIF_F_TSO_ECN)
1485                 wanted |= IONIC_ETH_HW_TSO_ECN;
1486         if (features & NETIF_F_GSO_GRE)
1487                 wanted |= IONIC_ETH_HW_TSO_GRE;
1488         if (features & NETIF_F_GSO_GRE_CSUM)
1489                 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1490         if (features & NETIF_F_GSO_IPXIP4)
1491                 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1492         if (features & NETIF_F_GSO_IPXIP6)
1493                 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1494         if (features & NETIF_F_GSO_UDP_TUNNEL)
1495                 wanted |= IONIC_ETH_HW_TSO_UDP;
1496         if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1497                 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1498
1499         return cpu_to_le64(wanted);
1500 }
1501
1502 static int ionic_set_nic_features(struct ionic_lif *lif,
1503                                   netdev_features_t features)
1504 {
1505         struct device *dev = lif->ionic->dev;
1506         struct ionic_admin_ctx ctx = {
1507                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1508                 .cmd.lif_setattr = {
1509                         .opcode = IONIC_CMD_LIF_SETATTR,
1510                         .index = cpu_to_le16(lif->index),
1511                         .attr = IONIC_LIF_ATTR_FEATURES,
1512                 },
1513         };
1514         u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1515                          IONIC_ETH_HW_VLAN_RX_STRIP |
1516                          IONIC_ETH_HW_VLAN_RX_FILTER;
1517         u64 old_hw_features;
1518         int err;
1519
1520         ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1521
1522         if (lif->phc)
1523                 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1524
1525         err = ionic_adminq_post_wait(lif, &ctx);
1526         if (err)
1527                 return err;
1528
1529         old_hw_features = lif->hw_features;
1530         lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1531                                        ctx.comp.lif_setattr.features);
1532
1533         if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1534                 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1535
1536         if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&
1537             !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1538                 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1539
1540         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1541                 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1542         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1543                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1544         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1545                 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1546         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1547                 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1548         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1549                 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1550         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1551                 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1552         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1553                 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1554         if (lif->hw_features & IONIC_ETH_HW_TSO)
1555                 dev_dbg(dev, "feature ETH_HW_TSO\n");
1556         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1557                 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1558         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1559                 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1560         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1561                 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1562         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1563                 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1564         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1565                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1566         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1567                 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1568         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1569                 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1570         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1571                 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1572         if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1573                 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1574
1575         return 0;
1576 }
1577
1578 static int ionic_init_nic_features(struct ionic_lif *lif)
1579 {
1580         struct net_device *netdev = lif->netdev;
1581         netdev_features_t features;
1582         int err;
1583
1584         /* set up what we expect to support by default */
1585         features = NETIF_F_HW_VLAN_CTAG_TX |
1586                    NETIF_F_HW_VLAN_CTAG_RX |
1587                    NETIF_F_HW_VLAN_CTAG_FILTER |
1588                    NETIF_F_SG |
1589                    NETIF_F_HW_CSUM |
1590                    NETIF_F_RXCSUM |
1591                    NETIF_F_TSO |
1592                    NETIF_F_TSO6 |
1593                    NETIF_F_TSO_ECN |
1594                    NETIF_F_GSO_GRE |
1595                    NETIF_F_GSO_GRE_CSUM |
1596                    NETIF_F_GSO_IPXIP4 |
1597                    NETIF_F_GSO_IPXIP6 |
1598                    NETIF_F_GSO_UDP_TUNNEL |
1599                    NETIF_F_GSO_UDP_TUNNEL_CSUM;
1600
1601         if (lif->nxqs > 1)
1602                 features |= NETIF_F_RXHASH;
1603
1604         err = ionic_set_nic_features(lif, features);
1605         if (err)
1606                 return err;
1607
1608         /* tell the netdev what we actually can support */
1609         netdev->features |= NETIF_F_HIGHDMA;
1610
1611         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1612                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1613         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1614                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1615         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1616                 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1617         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1618                 netdev->hw_features |= NETIF_F_RXHASH;
1619         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1620                 netdev->hw_features |= NETIF_F_SG;
1621
1622         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1623                 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1624         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1625                 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1626         if (lif->hw_features & IONIC_ETH_HW_TSO)
1627                 netdev->hw_enc_features |= NETIF_F_TSO;
1628         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1629                 netdev->hw_enc_features |= NETIF_F_TSO6;
1630         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1631                 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1632         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1633                 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1634         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1635                 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1636         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1637                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1638         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1639                 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1640         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1641                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1642         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1643                 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1644
1645         netdev->hw_features |= netdev->hw_enc_features;
1646         netdev->features |= netdev->hw_features;
1647         netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1648
1649         netdev->priv_flags |= IFF_UNICAST_FLT |
1650                               IFF_LIVE_ADDR_CHANGE;
1651
1652         return 0;
1653 }
1654
1655 static int ionic_set_features(struct net_device *netdev,
1656                               netdev_features_t features)
1657 {
1658         struct ionic_lif *lif = netdev_priv(netdev);
1659         int err;
1660
1661         netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1662                    __func__, (u64)lif->netdev->features, (u64)features);
1663
1664         err = ionic_set_nic_features(lif, features);
1665
1666         return err;
1667 }
1668
1669 static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)
1670 {
1671         struct ionic_admin_ctx ctx = {
1672                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1673                 .cmd.lif_setattr = {
1674                         .opcode = IONIC_CMD_LIF_SETATTR,
1675                         .index = cpu_to_le16(lif->index),
1676                         .attr = IONIC_LIF_ATTR_MAC,
1677                 },
1678         };
1679
1680         ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);
1681         return ionic_adminq_post_wait(lif, &ctx);
1682 }
1683
1684 static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)
1685 {
1686         struct ionic_admin_ctx ctx = {
1687                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1688                 .cmd.lif_getattr = {
1689                         .opcode = IONIC_CMD_LIF_GETATTR,
1690                         .index = cpu_to_le16(lif->index),
1691                         .attr = IONIC_LIF_ATTR_MAC,
1692                 },
1693         };
1694         int err;
1695
1696         err = ionic_adminq_post_wait(lif, &ctx);
1697         if (err)
1698                 return err;
1699
1700         ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);
1701         return 0;
1702 }
1703
1704 static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)
1705 {
1706         u8  get_mac[ETH_ALEN];
1707         int err;
1708
1709         err = ionic_set_attr_mac(lif, mac);
1710         if (err)
1711                 return err;
1712
1713         err = ionic_get_attr_mac(lif, get_mac);
1714         if (err)
1715                 return err;
1716
1717         /* To deal with older firmware that silently ignores the set attr mac:
1718          * doesn't actually change the mac and doesn't return an error, so we
1719          * do the get attr to verify whether or not the set actually happened
1720          */
1721         if (!ether_addr_equal(get_mac, mac))
1722                 return 1;
1723
1724         return 0;
1725 }
1726
1727 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1728 {
1729         struct ionic_lif *lif = netdev_priv(netdev);
1730         struct sockaddr *addr = sa;
1731         u8 *mac;
1732         int err;
1733
1734         mac = (u8 *)addr->sa_data;
1735         if (ether_addr_equal(netdev->dev_addr, mac))
1736                 return 0;
1737
1738         err = ionic_program_mac(lif, mac);
1739         if (err < 0)
1740                 return err;
1741
1742         if (err > 0)
1743                 netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
1744                            __func__);
1745
1746         err = eth_prepare_mac_addr_change(netdev, addr);
1747         if (err)
1748                 return err;
1749
1750         if (!is_zero_ether_addr(netdev->dev_addr)) {
1751                 netdev_info(netdev, "deleting mac addr %pM\n",
1752                             netdev->dev_addr);
1753                 ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
1754         }
1755
1756         eth_commit_mac_addr_change(netdev, addr);
1757         netdev_info(netdev, "updating mac addr %pM\n", mac);
1758
1759         return ionic_lif_addr_add(netdev_priv(netdev), mac);
1760 }
1761
1762 static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1763 {
1764         /* Stop and clean the queues before reconfiguration */
1765         netif_device_detach(lif->netdev);
1766         ionic_stop_queues(lif);
1767         ionic_txrx_deinit(lif);
1768 }
1769
1770 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1771 {
1772         int err;
1773
1774         /* Re-init the queues after reconfiguration */
1775
1776         /* The only way txrx_init can fail here is if communication
1777          * with FW is suddenly broken.  There's not much we can do
1778          * at this point - error messages have already been printed,
1779          * so we can continue on and the user can eventually do a
1780          * DOWN and UP to try to reset and clear the issue.
1781          */
1782         err = ionic_txrx_init(lif);
1783         ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1784         netif_device_attach(lif->netdev);
1785
1786         return err;
1787 }
1788
1789 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1790 {
1791         struct ionic_lif *lif = netdev_priv(netdev);
1792         struct ionic_admin_ctx ctx = {
1793                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1794                 .cmd.lif_setattr = {
1795                         .opcode = IONIC_CMD_LIF_SETATTR,
1796                         .index = cpu_to_le16(lif->index),
1797                         .attr = IONIC_LIF_ATTR_MTU,
1798                         .mtu = cpu_to_le32(new_mtu),
1799                 },
1800         };
1801         int err;
1802
1803         err = ionic_adminq_post_wait(lif, &ctx);
1804         if (err)
1805                 return err;
1806
1807         /* if we're not running, nothing more to do */
1808         if (!netif_running(netdev)) {
1809                 netdev->mtu = new_mtu;
1810                 return 0;
1811         }
1812
1813         mutex_lock(&lif->queue_lock);
1814         ionic_stop_queues_reconfig(lif);
1815         netdev->mtu = new_mtu;
1816         err = ionic_start_queues_reconfig(lif);
1817         mutex_unlock(&lif->queue_lock);
1818
1819         return err;
1820 }
1821
1822 static void ionic_tx_timeout_work(struct work_struct *ws)
1823 {
1824         struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1825
1826         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1827                 return;
1828
1829         /* if we were stopped before this scheduled job was launched,
1830          * don't bother the queues as they are already stopped.
1831          */
1832         if (!netif_running(lif->netdev))
1833                 return;
1834
1835         mutex_lock(&lif->queue_lock);
1836         ionic_stop_queues_reconfig(lif);
1837         ionic_start_queues_reconfig(lif);
1838         mutex_unlock(&lif->queue_lock);
1839 }
1840
1841 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1842 {
1843         struct ionic_lif *lif = netdev_priv(netdev);
1844
1845         netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1846         schedule_work(&lif->tx_timeout_work);
1847 }
1848
1849 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1850                                  u16 vid)
1851 {
1852         struct ionic_lif *lif = netdev_priv(netdev);
1853         int err;
1854
1855         err = ionic_lif_vlan_add(lif, vid);
1856         if (err)
1857                 return err;
1858
1859         ionic_lif_rx_mode(lif);
1860
1861         return 0;
1862 }
1863
1864 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1865                                   u16 vid)
1866 {
1867         struct ionic_lif *lif = netdev_priv(netdev);
1868         int err;
1869
1870         err = ionic_lif_vlan_del(lif, vid);
1871         if (err)
1872                 return err;
1873
1874         ionic_lif_rx_mode(lif);
1875
1876         return 0;
1877 }
1878
1879 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1880                          const u8 *key, const u32 *indir)
1881 {
1882         struct ionic_admin_ctx ctx = {
1883                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1884                 .cmd.lif_setattr = {
1885                         .opcode = IONIC_CMD_LIF_SETATTR,
1886                         .attr = IONIC_LIF_ATTR_RSS,
1887                         .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1888                 },
1889         };
1890         unsigned int i, tbl_sz;
1891
1892         if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1893                 lif->rss_types = types;
1894                 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1895         }
1896
1897         if (key)
1898                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1899
1900         if (indir) {
1901                 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1902                 for (i = 0; i < tbl_sz; i++)
1903                         lif->rss_ind_tbl[i] = indir[i];
1904         }
1905
1906         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1907                IONIC_RSS_HASH_KEY_SIZE);
1908
1909         return ionic_adminq_post_wait(lif, &ctx);
1910 }
1911
1912 static int ionic_lif_rss_init(struct ionic_lif *lif)
1913 {
1914         unsigned int tbl_sz;
1915         unsigned int i;
1916
1917         lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1918                          IONIC_RSS_TYPE_IPV4_TCP |
1919                          IONIC_RSS_TYPE_IPV4_UDP |
1920                          IONIC_RSS_TYPE_IPV6     |
1921                          IONIC_RSS_TYPE_IPV6_TCP |
1922                          IONIC_RSS_TYPE_IPV6_UDP;
1923
1924         /* Fill indirection table with 'default' values */
1925         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1926         for (i = 0; i < tbl_sz; i++)
1927                 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1928
1929         return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1930 }
1931
1932 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1933 {
1934         int tbl_sz;
1935
1936         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1937         memset(lif->rss_ind_tbl, 0, tbl_sz);
1938         memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1939
1940         ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1941 }
1942
1943 static void ionic_lif_quiesce(struct ionic_lif *lif)
1944 {
1945         struct ionic_admin_ctx ctx = {
1946                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1947                 .cmd.lif_setattr = {
1948                         .opcode = IONIC_CMD_LIF_SETATTR,
1949                         .index = cpu_to_le16(lif->index),
1950                         .attr = IONIC_LIF_ATTR_STATE,
1951                         .state = IONIC_LIF_QUIESCE,
1952                 },
1953         };
1954         int err;
1955
1956         err = ionic_adminq_post_wait(lif, &ctx);
1957         if (err)
1958                 netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
1959 }
1960
1961 static void ionic_txrx_disable(struct ionic_lif *lif)
1962 {
1963         unsigned int i;
1964         int err = 0;
1965
1966         if (lif->txqcqs) {
1967                 for (i = 0; i < lif->nxqs; i++)
1968                         err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
1969         }
1970
1971         if (lif->hwstamp_txq)
1972                 err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
1973
1974         if (lif->rxqcqs) {
1975                 for (i = 0; i < lif->nxqs; i++)
1976                         err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
1977         }
1978
1979         if (lif->hwstamp_rxq)
1980                 err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
1981
1982         ionic_lif_quiesce(lif);
1983 }
1984
1985 static void ionic_txrx_deinit(struct ionic_lif *lif)
1986 {
1987         unsigned int i;
1988
1989         if (lif->txqcqs) {
1990                 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1991                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1992                         ionic_tx_flush(&lif->txqcqs[i]->cq);
1993                         ionic_tx_empty(&lif->txqcqs[i]->q);
1994                 }
1995         }
1996
1997         if (lif->rxqcqs) {
1998                 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1999                         ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2000                         ionic_rx_empty(&lif->rxqcqs[i]->q);
2001                 }
2002         }
2003         lif->rx_mode = 0;
2004
2005         if (lif->hwstamp_txq) {
2006                 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
2007                 ionic_tx_flush(&lif->hwstamp_txq->cq);
2008                 ionic_tx_empty(&lif->hwstamp_txq->q);
2009         }
2010
2011         if (lif->hwstamp_rxq) {
2012                 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
2013                 ionic_rx_empty(&lif->hwstamp_rxq->q);
2014         }
2015 }
2016
2017 static void ionic_txrx_free(struct ionic_lif *lif)
2018 {
2019         unsigned int i;
2020
2021         if (lif->txqcqs) {
2022                 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
2023                         ionic_qcq_free(lif, lif->txqcqs[i]);
2024                         devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
2025                         lif->txqcqs[i] = NULL;
2026                 }
2027         }
2028
2029         if (lif->rxqcqs) {
2030                 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2031                         ionic_qcq_free(lif, lif->rxqcqs[i]);
2032                         devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
2033                         lif->rxqcqs[i] = NULL;
2034                 }
2035         }
2036
2037         if (lif->hwstamp_txq) {
2038                 ionic_qcq_free(lif, lif->hwstamp_txq);
2039                 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2040                 lif->hwstamp_txq = NULL;
2041         }
2042
2043         if (lif->hwstamp_rxq) {
2044                 ionic_qcq_free(lif, lif->hwstamp_rxq);
2045                 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2046                 lif->hwstamp_rxq = NULL;
2047         }
2048 }
2049
2050 static int ionic_txrx_alloc(struct ionic_lif *lif)
2051 {
2052         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2053         unsigned int flags, i;
2054         int err = 0;
2055
2056         num_desc = lif->ntxq_descs;
2057         desc_sz = sizeof(struct ionic_txq_desc);
2058         comp_sz = sizeof(struct ionic_txq_comp);
2059
2060         if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2061             lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2062                                           sizeof(struct ionic_txq_sg_desc_v1))
2063                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2064         else
2065                 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2066
2067         flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2068
2069         if (test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state))
2070                 flags |= IONIC_QCQ_F_CMB_RINGS;
2071
2072         if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2073                 flags |= IONIC_QCQ_F_INTR;
2074
2075         for (i = 0; i < lif->nxqs; i++) {
2076                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2077                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2078                                       lif->kern_pid, &lif->txqcqs[i]);
2079                 if (err)
2080                         goto err_out;
2081
2082                 if (flags & IONIC_QCQ_F_INTR) {
2083                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2084                                              lif->txqcqs[i]->intr.index,
2085                                              lif->tx_coalesce_hw);
2086                         if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2087                                 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2088                 }
2089
2090                 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2091         }
2092
2093         flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2094
2095         if (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state))
2096                 flags |= IONIC_QCQ_F_CMB_RINGS;
2097
2098         num_desc = lif->nrxq_descs;
2099         desc_sz = sizeof(struct ionic_rxq_desc);
2100         comp_sz = sizeof(struct ionic_rxq_comp);
2101         sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2102
2103         if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2104                 comp_sz *= 2;
2105
2106         for (i = 0; i < lif->nxqs; i++) {
2107                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2108                                       num_desc, desc_sz, comp_sz, sg_desc_sz,
2109                                       lif->kern_pid, &lif->rxqcqs[i]);
2110                 if (err)
2111                         goto err_out;
2112
2113                 lif->rxqcqs[i]->q.features = lif->rxq_features;
2114
2115                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2116                                      lif->rxqcqs[i]->intr.index,
2117                                      lif->rx_coalesce_hw);
2118                 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2119                         lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2120
2121                 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2122                         ionic_link_qcq_interrupts(lif->rxqcqs[i],
2123                                                   lif->txqcqs[i]);
2124
2125                 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2126         }
2127
2128         return 0;
2129
2130 err_out:
2131         ionic_txrx_free(lif);
2132
2133         return err;
2134 }
2135
2136 static int ionic_txrx_init(struct ionic_lif *lif)
2137 {
2138         unsigned int i;
2139         int err;
2140
2141         for (i = 0; i < lif->nxqs; i++) {
2142                 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2143                 if (err)
2144                         goto err_out;
2145
2146                 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2147                 if (err) {
2148                         ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2149                         goto err_out;
2150                 }
2151         }
2152
2153         if (lif->netdev->features & NETIF_F_RXHASH)
2154                 ionic_lif_rss_init(lif);
2155
2156         ionic_lif_rx_mode(lif);
2157
2158         return 0;
2159
2160 err_out:
2161         while (i--) {
2162                 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2163                 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2164         }
2165
2166         return err;
2167 }
2168
2169 static int ionic_txrx_enable(struct ionic_lif *lif)
2170 {
2171         int derr = 0;
2172         int i, err;
2173
2174         for (i = 0; i < lif->nxqs; i++) {
2175                 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2176                         dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2177                         err = -ENXIO;
2178                         goto err_out;
2179                 }
2180
2181                 ionic_rx_fill(&lif->rxqcqs[i]->q);
2182                 err = ionic_qcq_enable(lif->rxqcqs[i]);
2183                 if (err)
2184                         goto err_out;
2185
2186                 err = ionic_qcq_enable(lif->txqcqs[i]);
2187                 if (err) {
2188                         derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
2189                         goto err_out;
2190                 }
2191         }
2192
2193         if (lif->hwstamp_rxq) {
2194                 ionic_rx_fill(&lif->hwstamp_rxq->q);
2195                 err = ionic_qcq_enable(lif->hwstamp_rxq);
2196                 if (err)
2197                         goto err_out_hwstamp_rx;
2198         }
2199
2200         if (lif->hwstamp_txq) {
2201                 err = ionic_qcq_enable(lif->hwstamp_txq);
2202                 if (err)
2203                         goto err_out_hwstamp_tx;
2204         }
2205
2206         return 0;
2207
2208 err_out_hwstamp_tx:
2209         if (lif->hwstamp_rxq)
2210                 derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
2211 err_out_hwstamp_rx:
2212         i = lif->nxqs;
2213 err_out:
2214         while (i--) {
2215                 derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
2216                 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
2217         }
2218
2219         return err;
2220 }
2221
2222 static int ionic_start_queues(struct ionic_lif *lif)
2223 {
2224         int err;
2225
2226         if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2227                 return -EIO;
2228
2229         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2230                 return -EBUSY;
2231
2232         if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2233                 return 0;
2234
2235         err = ionic_txrx_enable(lif);
2236         if (err) {
2237                 clear_bit(IONIC_LIF_F_UP, lif->state);
2238                 return err;
2239         }
2240         netif_tx_wake_all_queues(lif->netdev);
2241
2242         return 0;
2243 }
2244
2245 static int ionic_open(struct net_device *netdev)
2246 {
2247         struct ionic_lif *lif = netdev_priv(netdev);
2248         int err;
2249
2250         /* If recovering from a broken state, clear the bit and we'll try again */
2251         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2252                 netdev_info(netdev, "clearing broken state\n");
2253
2254         mutex_lock(&lif->queue_lock);
2255
2256         err = ionic_txrx_alloc(lif);
2257         if (err)
2258                 goto err_unlock;
2259
2260         err = ionic_txrx_init(lif);
2261         if (err)
2262                 goto err_txrx_free;
2263
2264         err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2265         if (err)
2266                 goto err_txrx_deinit;
2267
2268         err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2269         if (err)
2270                 goto err_txrx_deinit;
2271
2272         /* don't start the queues until we have link */
2273         if (netif_carrier_ok(netdev)) {
2274                 err = ionic_start_queues(lif);
2275                 if (err)
2276                         goto err_txrx_deinit;
2277         }
2278
2279         /* If hardware timestamping is enabled, but the queues were freed by
2280          * ionic_stop, those need to be reallocated and initialized, too.
2281          */
2282         ionic_lif_hwstamp_recreate_queues(lif);
2283
2284         mutex_unlock(&lif->queue_lock);
2285
2286         return 0;
2287
2288 err_txrx_deinit:
2289         ionic_txrx_deinit(lif);
2290 err_txrx_free:
2291         ionic_txrx_free(lif);
2292 err_unlock:
2293         mutex_unlock(&lif->queue_lock);
2294         return err;
2295 }
2296
2297 static void ionic_stop_queues(struct ionic_lif *lif)
2298 {
2299         if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2300                 return;
2301
2302         netif_tx_disable(lif->netdev);
2303         ionic_txrx_disable(lif);
2304 }
2305
2306 static int ionic_stop(struct net_device *netdev)
2307 {
2308         struct ionic_lif *lif = netdev_priv(netdev);
2309
2310         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2311                 return 0;
2312
2313         mutex_lock(&lif->queue_lock);
2314         ionic_stop_queues(lif);
2315         ionic_txrx_deinit(lif);
2316         ionic_txrx_free(lif);
2317         mutex_unlock(&lif->queue_lock);
2318
2319         return 0;
2320 }
2321
2322 static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2323 {
2324         struct ionic_lif *lif = netdev_priv(netdev);
2325
2326         switch (cmd) {
2327         case SIOCSHWTSTAMP:
2328                 return ionic_lif_hwstamp_set(lif, ifr);
2329         case SIOCGHWTSTAMP:
2330                 return ionic_lif_hwstamp_get(lif, ifr);
2331         default:
2332                 return -EOPNOTSUPP;
2333         }
2334 }
2335
2336 static int ionic_get_fw_vf_config(struct ionic *ionic, int vf, struct ionic_vf *vfdata)
2337 {
2338         struct ionic_vf_getattr_comp comp = { 0 };
2339         int err;
2340         u8 attr;
2341
2342         attr = IONIC_VF_ATTR_VLAN;
2343         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2344         if (err && comp.status != IONIC_RC_ENOSUPP)
2345                 goto err_out;
2346         if (!err)
2347                 vfdata->vlanid = comp.vlanid;
2348
2349         attr = IONIC_VF_ATTR_SPOOFCHK;
2350         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2351         if (err && comp.status != IONIC_RC_ENOSUPP)
2352                 goto err_out;
2353         if (!err)
2354                 vfdata->spoofchk = comp.spoofchk;
2355
2356         attr = IONIC_VF_ATTR_LINKSTATE;
2357         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2358         if (err && comp.status != IONIC_RC_ENOSUPP)
2359                 goto err_out;
2360         if (!err) {
2361                 switch (comp.linkstate) {
2362                 case IONIC_VF_LINK_STATUS_UP:
2363                         vfdata->linkstate = IFLA_VF_LINK_STATE_ENABLE;
2364                         break;
2365                 case IONIC_VF_LINK_STATUS_DOWN:
2366                         vfdata->linkstate = IFLA_VF_LINK_STATE_DISABLE;
2367                         break;
2368                 case IONIC_VF_LINK_STATUS_AUTO:
2369                         vfdata->linkstate = IFLA_VF_LINK_STATE_AUTO;
2370                         break;
2371                 default:
2372                         dev_warn(ionic->dev, "Unexpected link state %u\n", comp.linkstate);
2373                         break;
2374                 }
2375         }
2376
2377         attr = IONIC_VF_ATTR_RATE;
2378         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2379         if (err && comp.status != IONIC_RC_ENOSUPP)
2380                 goto err_out;
2381         if (!err)
2382                 vfdata->maxrate = comp.maxrate;
2383
2384         attr = IONIC_VF_ATTR_TRUST;
2385         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2386         if (err && comp.status != IONIC_RC_ENOSUPP)
2387                 goto err_out;
2388         if (!err)
2389                 vfdata->trusted = comp.trust;
2390
2391         attr = IONIC_VF_ATTR_MAC;
2392         err = ionic_dev_cmd_vf_getattr(ionic, vf, attr, &comp);
2393         if (err && comp.status != IONIC_RC_ENOSUPP)
2394                 goto err_out;
2395         if (!err)
2396                 ether_addr_copy(vfdata->macaddr, comp.macaddr);
2397
2398 err_out:
2399         if (err)
2400                 dev_err(ionic->dev, "Failed to get %s for VF %d\n",
2401                         ionic_vf_attr_to_str(attr), vf);
2402
2403         return err;
2404 }
2405
2406 static int ionic_get_vf_config(struct net_device *netdev,
2407                                int vf, struct ifla_vf_info *ivf)
2408 {
2409         struct ionic_lif *lif = netdev_priv(netdev);
2410         struct ionic *ionic = lif->ionic;
2411         struct ionic_vf vfdata = { 0 };
2412         int ret = 0;
2413
2414         if (!netif_device_present(netdev))
2415                 return -EBUSY;
2416
2417         down_read(&ionic->vf_op_lock);
2418
2419         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2420                 ret = -EINVAL;
2421         } else {
2422                 ivf->vf = vf;
2423                 ivf->qos = 0;
2424
2425                 ret = ionic_get_fw_vf_config(ionic, vf, &vfdata);
2426                 if (!ret) {
2427                         ivf->vlan         = le16_to_cpu(vfdata.vlanid);
2428                         ivf->spoofchk     = vfdata.spoofchk;
2429                         ivf->linkstate    = vfdata.linkstate;
2430                         ivf->max_tx_rate  = le32_to_cpu(vfdata.maxrate);
2431                         ivf->trusted      = vfdata.trusted;
2432                         ether_addr_copy(ivf->mac, vfdata.macaddr);
2433                 }
2434         }
2435
2436         up_read(&ionic->vf_op_lock);
2437         return ret;
2438 }
2439
2440 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2441                               struct ifla_vf_stats *vf_stats)
2442 {
2443         struct ionic_lif *lif = netdev_priv(netdev);
2444         struct ionic *ionic = lif->ionic;
2445         struct ionic_lif_stats *vs;
2446         int ret = 0;
2447
2448         if (!netif_device_present(netdev))
2449                 return -EBUSY;
2450
2451         down_read(&ionic->vf_op_lock);
2452
2453         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2454                 ret = -EINVAL;
2455         } else {
2456                 memset(vf_stats, 0, sizeof(*vf_stats));
2457                 vs = &ionic->vfs[vf].stats;
2458
2459                 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2460                 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2461                 vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2462                 vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2463                 vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2464                 vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2465                 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2466                                        le64_to_cpu(vs->rx_mcast_drop_packets) +
2467                                        le64_to_cpu(vs->rx_bcast_drop_packets);
2468                 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2469                                        le64_to_cpu(vs->tx_mcast_drop_packets) +
2470                                        le64_to_cpu(vs->tx_bcast_drop_packets);
2471         }
2472
2473         up_read(&ionic->vf_op_lock);
2474         return ret;
2475 }
2476
2477 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2478 {
2479         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
2480         struct ionic_lif *lif = netdev_priv(netdev);
2481         struct ionic *ionic = lif->ionic;
2482         int ret;
2483
2484         if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2485                 return -EINVAL;
2486
2487         if (!netif_device_present(netdev))
2488                 return -EBUSY;
2489
2490         down_write(&ionic->vf_op_lock);
2491
2492         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2493                 ret = -EINVAL;
2494         } else {
2495                 ether_addr_copy(vfc.macaddr, mac);
2496                 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
2497                         __func__, vf, vfc.macaddr);
2498
2499                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2500                 if (!ret)
2501                         ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2502         }
2503
2504         up_write(&ionic->vf_op_lock);
2505         return ret;
2506 }
2507
2508 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2509                              u8 qos, __be16 proto)
2510 {
2511         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
2512         struct ionic_lif *lif = netdev_priv(netdev);
2513         struct ionic *ionic = lif->ionic;
2514         int ret;
2515
2516         /* until someday when we support qos */
2517         if (qos)
2518                 return -EINVAL;
2519
2520         if (vlan > 4095)
2521                 return -EINVAL;
2522
2523         if (proto != htons(ETH_P_8021Q))
2524                 return -EPROTONOSUPPORT;
2525
2526         if (!netif_device_present(netdev))
2527                 return -EBUSY;
2528
2529         down_write(&ionic->vf_op_lock);
2530
2531         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2532                 ret = -EINVAL;
2533         } else {
2534                 vfc.vlanid = cpu_to_le16(vlan);
2535                 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
2536                         __func__, vf, le16_to_cpu(vfc.vlanid));
2537
2538                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2539                 if (!ret)
2540                         ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2541         }
2542
2543         up_write(&ionic->vf_op_lock);
2544         return ret;
2545 }
2546
2547 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2548                              int tx_min, int tx_max)
2549 {
2550         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
2551         struct ionic_lif *lif = netdev_priv(netdev);
2552         struct ionic *ionic = lif->ionic;
2553         int ret;
2554
2555         /* setting the min just seems silly */
2556         if (tx_min)
2557                 return -EINVAL;
2558
2559         if (!netif_device_present(netdev))
2560                 return -EBUSY;
2561
2562         down_write(&ionic->vf_op_lock);
2563
2564         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2565                 ret = -EINVAL;
2566         } else {
2567                 vfc.maxrate = cpu_to_le32(tx_max);
2568                 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
2569                         __func__, vf, le32_to_cpu(vfc.maxrate));
2570
2571                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2572                 if (!ret)
2573                         ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2574         }
2575
2576         up_write(&ionic->vf_op_lock);
2577         return ret;
2578 }
2579
2580 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2581 {
2582         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
2583         struct ionic_lif *lif = netdev_priv(netdev);
2584         struct ionic *ionic = lif->ionic;
2585         int ret;
2586
2587         if (!netif_device_present(netdev))
2588                 return -EBUSY;
2589
2590         down_write(&ionic->vf_op_lock);
2591
2592         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2593                 ret = -EINVAL;
2594         } else {
2595                 vfc.spoofchk = set;
2596                 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
2597                         __func__, vf, vfc.spoofchk);
2598
2599                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2600                 if (!ret)
2601                         ionic->vfs[vf].spoofchk = set;
2602         }
2603
2604         up_write(&ionic->vf_op_lock);
2605         return ret;
2606 }
2607
2608 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2609 {
2610         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
2611         struct ionic_lif *lif = netdev_priv(netdev);
2612         struct ionic *ionic = lif->ionic;
2613         int ret;
2614
2615         if (!netif_device_present(netdev))
2616                 return -EBUSY;
2617
2618         down_write(&ionic->vf_op_lock);
2619
2620         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2621                 ret = -EINVAL;
2622         } else {
2623                 vfc.trust = set;
2624                 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
2625                         __func__, vf, vfc.trust);
2626
2627                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2628                 if (!ret)
2629                         ionic->vfs[vf].trusted = set;
2630         }
2631
2632         up_write(&ionic->vf_op_lock);
2633         return ret;
2634 }
2635
2636 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2637 {
2638         struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
2639         struct ionic_lif *lif = netdev_priv(netdev);
2640         struct ionic *ionic = lif->ionic;
2641         u8 vfls;
2642         int ret;
2643
2644         switch (set) {
2645         case IFLA_VF_LINK_STATE_ENABLE:
2646                 vfls = IONIC_VF_LINK_STATUS_UP;
2647                 break;
2648         case IFLA_VF_LINK_STATE_DISABLE:
2649                 vfls = IONIC_VF_LINK_STATUS_DOWN;
2650                 break;
2651         case IFLA_VF_LINK_STATE_AUTO:
2652                 vfls = IONIC_VF_LINK_STATUS_AUTO;
2653                 break;
2654         default:
2655                 return -EINVAL;
2656         }
2657
2658         if (!netif_device_present(netdev))
2659                 return -EBUSY;
2660
2661         down_write(&ionic->vf_op_lock);
2662
2663         if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2664                 ret = -EINVAL;
2665         } else {
2666                 vfc.linkstate = vfls;
2667                 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
2668                         __func__, vf, vfc.linkstate);
2669
2670                 ret = ionic_set_vf_config(ionic, vf, &vfc);
2671                 if (!ret)
2672                         ionic->vfs[vf].linkstate = set;
2673         }
2674
2675         up_write(&ionic->vf_op_lock);
2676         return ret;
2677 }
2678
2679 static void ionic_vf_attr_replay(struct ionic_lif *lif)
2680 {
2681         struct ionic_vf_setattr_cmd vfc = { };
2682         struct ionic *ionic = lif->ionic;
2683         struct ionic_vf *v;
2684         int i;
2685
2686         if (!ionic->vfs)
2687                 return;
2688
2689         down_read(&ionic->vf_op_lock);
2690
2691         for (i = 0; i < ionic->num_vfs; i++) {
2692                 v = &ionic->vfs[i];
2693
2694                 if (v->stats_pa) {
2695                         vfc.attr = IONIC_VF_ATTR_STATSADDR;
2696                         vfc.stats_pa = cpu_to_le64(v->stats_pa);
2697                         ionic_set_vf_config(ionic, i, &vfc);
2698                         vfc.stats_pa = 0;
2699                 }
2700
2701                 if (!is_zero_ether_addr(v->macaddr)) {
2702                         vfc.attr = IONIC_VF_ATTR_MAC;
2703                         ether_addr_copy(vfc.macaddr, v->macaddr);
2704                         ionic_set_vf_config(ionic, i, &vfc);
2705                         eth_zero_addr(vfc.macaddr);
2706                 }
2707
2708                 if (v->vlanid) {
2709                         vfc.attr = IONIC_VF_ATTR_VLAN;
2710                         vfc.vlanid = v->vlanid;
2711                         ionic_set_vf_config(ionic, i, &vfc);
2712                         vfc.vlanid = 0;
2713                 }
2714
2715                 if (v->maxrate) {
2716                         vfc.attr = IONIC_VF_ATTR_RATE;
2717                         vfc.maxrate = v->maxrate;
2718                         ionic_set_vf_config(ionic, i, &vfc);
2719                         vfc.maxrate = 0;
2720                 }
2721
2722                 if (v->spoofchk) {
2723                         vfc.attr = IONIC_VF_ATTR_SPOOFCHK;
2724                         vfc.spoofchk = v->spoofchk;
2725                         ionic_set_vf_config(ionic, i, &vfc);
2726                         vfc.spoofchk = 0;
2727                 }
2728
2729                 if (v->trusted) {
2730                         vfc.attr = IONIC_VF_ATTR_TRUST;
2731                         vfc.trust = v->trusted;
2732                         ionic_set_vf_config(ionic, i, &vfc);
2733                         vfc.trust = 0;
2734                 }
2735
2736                 if (v->linkstate) {
2737                         vfc.attr = IONIC_VF_ATTR_LINKSTATE;
2738                         vfc.linkstate = v->linkstate;
2739                         ionic_set_vf_config(ionic, i, &vfc);
2740                         vfc.linkstate = 0;
2741                 }
2742         }
2743
2744         up_read(&ionic->vf_op_lock);
2745
2746         ionic_vf_start(ionic);
2747 }
2748
2749 static const struct net_device_ops ionic_netdev_ops = {
2750         .ndo_open               = ionic_open,
2751         .ndo_stop               = ionic_stop,
2752         .ndo_eth_ioctl          = ionic_eth_ioctl,
2753         .ndo_start_xmit         = ionic_start_xmit,
2754         .ndo_get_stats64        = ionic_get_stats64,
2755         .ndo_set_rx_mode        = ionic_ndo_set_rx_mode,
2756         .ndo_set_features       = ionic_set_features,
2757         .ndo_set_mac_address    = ionic_set_mac_address,
2758         .ndo_validate_addr      = eth_validate_addr,
2759         .ndo_tx_timeout         = ionic_tx_timeout,
2760         .ndo_change_mtu         = ionic_change_mtu,
2761         .ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2762         .ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2763         .ndo_set_vf_vlan        = ionic_set_vf_vlan,
2764         .ndo_set_vf_trust       = ionic_set_vf_trust,
2765         .ndo_set_vf_mac         = ionic_set_vf_mac,
2766         .ndo_set_vf_rate        = ionic_set_vf_rate,
2767         .ndo_set_vf_spoofchk    = ionic_set_vf_spoofchk,
2768         .ndo_get_vf_config      = ionic_get_vf_config,
2769         .ndo_set_vf_link_state  = ionic_set_vf_link_state,
2770         .ndo_get_vf_stats       = ionic_get_vf_stats,
2771 };
2772
2773 static int ionic_cmb_reconfig(struct ionic_lif *lif,
2774                               struct ionic_queue_params *qparam)
2775 {
2776         struct ionic_queue_params start_qparams;
2777         int err = 0;
2778
2779         /* When changing CMB queue parameters, we're using limited
2780          * on-device memory and don't have extra memory to use for
2781          * duplicate allocations, so we free it all first then
2782          * re-allocate with the new parameters.
2783          */
2784
2785         /* Checkpoint for possible unwind */
2786         ionic_init_queue_params(lif, &start_qparams);
2787
2788         /* Stop and free the queues */
2789         ionic_stop_queues_reconfig(lif);
2790         ionic_txrx_free(lif);
2791
2792         /* Set up new qparams */
2793         ionic_set_queue_params(lif, qparam);
2794
2795         if (netif_running(lif->netdev)) {
2796                 /* Alloc and start the new configuration */
2797                 err = ionic_txrx_alloc(lif);
2798                 if (err) {
2799                         dev_warn(lif->ionic->dev,
2800                                  "CMB reconfig failed, restoring values: %d\n", err);
2801
2802                         /* Back out the changes */
2803                         ionic_set_queue_params(lif, &start_qparams);
2804                         err = ionic_txrx_alloc(lif);
2805                         if (err) {
2806                                 dev_err(lif->ionic->dev,
2807                                         "CMB restore failed: %d\n", err);
2808                                 goto errout;
2809                         }
2810                 }
2811
2812                 ionic_start_queues_reconfig(lif);
2813         } else {
2814                 /* This was detached in ionic_stop_queues_reconfig() */
2815                 netif_device_attach(lif->netdev);
2816         }
2817
2818 errout:
2819         return err;
2820 }
2821
2822 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2823 {
2824         /* only swapping the queues, not the napi, flags, or other stuff */
2825         swap(a->q.features,   b->q.features);
2826         swap(a->q.num_descs,  b->q.num_descs);
2827         swap(a->q.desc_size,  b->q.desc_size);
2828         swap(a->q.base,       b->q.base);
2829         swap(a->q.base_pa,    b->q.base_pa);
2830         swap(a->q.info,       b->q.info);
2831         swap(a->q_base,       b->q_base);
2832         swap(a->q_base_pa,    b->q_base_pa);
2833         swap(a->q_size,       b->q_size);
2834
2835         swap(a->q.sg_desc_size, b->q.sg_desc_size);
2836         swap(a->q.sg_base,    b->q.sg_base);
2837         swap(a->q.sg_base_pa, b->q.sg_base_pa);
2838         swap(a->sg_base,      b->sg_base);
2839         swap(a->sg_base_pa,   b->sg_base_pa);
2840         swap(a->sg_size,      b->sg_size);
2841
2842         swap(a->cq.num_descs, b->cq.num_descs);
2843         swap(a->cq.desc_size, b->cq.desc_size);
2844         swap(a->cq.base,      b->cq.base);
2845         swap(a->cq.base_pa,   b->cq.base_pa);
2846         swap(a->cq.info,      b->cq.info);
2847         swap(a->cq_base,      b->cq_base);
2848         swap(a->cq_base_pa,   b->cq_base_pa);
2849         swap(a->cq_size,      b->cq_size);
2850
2851         ionic_debugfs_del_qcq(a);
2852         ionic_debugfs_add_qcq(a->q.lif, a);
2853 }
2854
2855 int ionic_reconfigure_queues(struct ionic_lif *lif,
2856                              struct ionic_queue_params *qparam)
2857 {
2858         unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2859         struct ionic_qcq **tx_qcqs = NULL;
2860         struct ionic_qcq **rx_qcqs = NULL;
2861         unsigned int flags, i;
2862         int err = 0;
2863
2864         /* Are we changing q params while CMB is on */
2865         if ((test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state) && qparam->cmb_tx) ||
2866             (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state) && qparam->cmb_rx))
2867                 return ionic_cmb_reconfig(lif, qparam);
2868
2869         /* allocate temporary qcq arrays to hold new queue structs */
2870         if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2871                 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2872                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2873                 if (!tx_qcqs) {
2874                         err = -ENOMEM;
2875                         goto err_out;
2876                 }
2877         }
2878         if (qparam->nxqs != lif->nxqs ||
2879             qparam->nrxq_descs != lif->nrxq_descs ||
2880             qparam->rxq_features != lif->rxq_features) {
2881                 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2882                                        sizeof(struct ionic_qcq *), GFP_KERNEL);
2883                 if (!rx_qcqs) {
2884                         err = -ENOMEM;
2885                         goto err_out;
2886                 }
2887         }
2888
2889         /* allocate new desc_info and rings, but leave the interrupt setup
2890          * until later so as to not mess with the still-running queues
2891          */
2892         if (tx_qcqs) {
2893                 num_desc = qparam->ntxq_descs;
2894                 desc_sz = sizeof(struct ionic_txq_desc);
2895                 comp_sz = sizeof(struct ionic_txq_comp);
2896
2897                 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2898                     lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2899                     sizeof(struct ionic_txq_sg_desc_v1))
2900                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2901                 else
2902                         sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2903
2904                 for (i = 0; i < qparam->nxqs; i++) {
2905                         /* If missing, short placeholder qcq needed for swap */
2906                         if (!lif->txqcqs[i]) {
2907                                 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2908                                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2909                                                       4, desc_sz, comp_sz, sg_desc_sz,
2910                                                       lif->kern_pid, &lif->txqcqs[i]);
2911                                 if (err)
2912                                         goto err_out;
2913                         }
2914
2915                         flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2916                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2917                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2918                                               lif->kern_pid, &tx_qcqs[i]);
2919                         if (err)
2920                                 goto err_out;
2921                 }
2922         }
2923
2924         if (rx_qcqs) {
2925                 num_desc = qparam->nrxq_descs;
2926                 desc_sz = sizeof(struct ionic_rxq_desc);
2927                 comp_sz = sizeof(struct ionic_rxq_comp);
2928                 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2929
2930                 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2931                         comp_sz *= 2;
2932
2933                 for (i = 0; i < qparam->nxqs; i++) {
2934                         /* If missing, short placeholder qcq needed for swap */
2935                         if (!lif->rxqcqs[i]) {
2936                                 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
2937                                 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2938                                                       4, desc_sz, comp_sz, sg_desc_sz,
2939                                                       lif->kern_pid, &lif->rxqcqs[i]);
2940                                 if (err)
2941                                         goto err_out;
2942                         }
2943
2944                         flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2945                         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2946                                               num_desc, desc_sz, comp_sz, sg_desc_sz,
2947                                               lif->kern_pid, &rx_qcqs[i]);
2948                         if (err)
2949                                 goto err_out;
2950
2951                         rx_qcqs[i]->q.features = qparam->rxq_features;
2952                 }
2953         }
2954
2955         /* stop and clean the queues */
2956         ionic_stop_queues_reconfig(lif);
2957
2958         if (qparam->nxqs != lif->nxqs) {
2959                 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2960                 if (err)
2961                         goto err_out_reinit_unlock;
2962                 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2963                 if (err) {
2964                         netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2965                         goto err_out_reinit_unlock;
2966                 }
2967         }
2968
2969         /* swap new desc_info and rings, keeping existing interrupt config */
2970         if (tx_qcqs) {
2971                 lif->ntxq_descs = qparam->ntxq_descs;
2972                 for (i = 0; i < qparam->nxqs; i++)
2973                         ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2974         }
2975
2976         if (rx_qcqs) {
2977                 lif->nrxq_descs = qparam->nrxq_descs;
2978                 for (i = 0; i < qparam->nxqs; i++)
2979                         ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2980         }
2981
2982         /* if we need to change the interrupt layout, this is the time */
2983         if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2984             qparam->nxqs != lif->nxqs) {
2985                 if (qparam->intr_split) {
2986                         set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2987                 } else {
2988                         clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2989                         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2990                         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2991                 }
2992
2993                 /* Clear existing interrupt assignments.  We check for NULL here
2994                  * because we're checking the whole array for potential qcqs, not
2995                  * just those qcqs that have just been set up.
2996                  */
2997                 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2998                         if (lif->txqcqs[i])
2999                                 ionic_qcq_intr_free(lif, lif->txqcqs[i]);
3000                         if (lif->rxqcqs[i])
3001                                 ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
3002                 }
3003
3004                 /* re-assign the interrupts */
3005                 for (i = 0; i < qparam->nxqs; i++) {
3006                         lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3007                         err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
3008                         ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3009                                              lif->rxqcqs[i]->intr.index,
3010                                              lif->rx_coalesce_hw);
3011
3012                         if (qparam->intr_split) {
3013                                 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3014                                 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
3015                                 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3016                                                      lif->txqcqs[i]->intr.index,
3017                                                      lif->tx_coalesce_hw);
3018                                 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
3019                                         lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
3020                         } else {
3021                                 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3022                                 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
3023                         }
3024                 }
3025         }
3026
3027         /* now we can rework the debugfs mappings */
3028         if (tx_qcqs) {
3029                 for (i = 0; i < qparam->nxqs; i++) {
3030                         ionic_debugfs_del_qcq(lif->txqcqs[i]);
3031                         ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
3032                 }
3033         }
3034
3035         if (rx_qcqs) {
3036                 for (i = 0; i < qparam->nxqs; i++) {
3037                         ionic_debugfs_del_qcq(lif->rxqcqs[i]);
3038                         ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
3039                 }
3040         }
3041
3042         swap(lif->nxqs, qparam->nxqs);
3043         swap(lif->rxq_features, qparam->rxq_features);
3044
3045 err_out_reinit_unlock:
3046         /* re-init the queues, but don't lose an error code */
3047         if (err)
3048                 ionic_start_queues_reconfig(lif);
3049         else
3050                 err = ionic_start_queues_reconfig(lif);
3051
3052 err_out:
3053         /* free old allocs without cleaning intr */
3054         for (i = 0; i < qparam->nxqs; i++) {
3055                 if (tx_qcqs && tx_qcqs[i]) {
3056                         tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3057                         ionic_qcq_free(lif, tx_qcqs[i]);
3058                         devm_kfree(lif->ionic->dev, tx_qcqs[i]);
3059                         tx_qcqs[i] = NULL;
3060                 }
3061                 if (rx_qcqs && rx_qcqs[i]) {
3062                         rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3063                         ionic_qcq_free(lif, rx_qcqs[i]);
3064                         devm_kfree(lif->ionic->dev, rx_qcqs[i]);
3065                         rx_qcqs[i] = NULL;
3066                 }
3067         }
3068
3069         /* free q array */
3070         if (rx_qcqs) {
3071                 devm_kfree(lif->ionic->dev, rx_qcqs);
3072                 rx_qcqs = NULL;
3073         }
3074         if (tx_qcqs) {
3075                 devm_kfree(lif->ionic->dev, tx_qcqs);
3076                 tx_qcqs = NULL;
3077         }
3078
3079         /* clean the unused dma and info allocations when new set is smaller
3080          * than the full array, but leave the qcq shells in place
3081          */
3082         for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
3083                 if (lif->txqcqs && lif->txqcqs[i]) {
3084                         lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3085                         ionic_qcq_free(lif, lif->txqcqs[i]);
3086                 }
3087
3088                 if (lif->rxqcqs && lif->rxqcqs[i]) {
3089                         lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3090                         ionic_qcq_free(lif, lif->rxqcqs[i]);
3091                 }
3092         }
3093
3094         if (err)
3095                 netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
3096
3097         return err;
3098 }
3099
3100 int ionic_lif_alloc(struct ionic *ionic)
3101 {
3102         struct device *dev = ionic->dev;
3103         union ionic_lif_identity *lid;
3104         struct net_device *netdev;
3105         struct ionic_lif *lif;
3106         int tbl_sz;
3107         int err;
3108
3109         lid = kzalloc(sizeof(*lid), GFP_KERNEL);
3110         if (!lid)
3111                 return -ENOMEM;
3112
3113         netdev = alloc_etherdev_mqs(sizeof(*lif),
3114                                     ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
3115         if (!netdev) {
3116                 dev_err(dev, "Cannot allocate netdev, aborting\n");
3117                 err = -ENOMEM;
3118                 goto err_out_free_lid;
3119         }
3120
3121         SET_NETDEV_DEV(netdev, dev);
3122
3123         lif = netdev_priv(netdev);
3124         lif->netdev = netdev;
3125         ionic->lif = lif;
3126         netdev->netdev_ops = &ionic_netdev_ops;
3127         ionic_ethtool_set_ops(netdev);
3128
3129         netdev->watchdog_timeo = 2 * HZ;
3130         netif_carrier_off(netdev);
3131
3132         lif->identity = lid;
3133         lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
3134         err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
3135         if (err) {
3136                 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
3137                         lif->lif_type, err);
3138                 goto err_out_free_netdev;
3139         }
3140         lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
3141                                      le32_to_cpu(lif->identity->eth.min_frame_size));
3142         lif->netdev->max_mtu =
3143                 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
3144
3145         lif->neqs = ionic->neqs_per_lif;
3146         lif->nxqs = ionic->ntxqs_per_lif;
3147
3148         lif->ionic = ionic;
3149         lif->index = 0;
3150
3151         if (is_kdump_kernel()) {
3152                 lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
3153                 lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
3154         } else {
3155                 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
3156                 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
3157         }
3158
3159         /* Convert the default coalesce value to actual hw resolution */
3160         lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
3161         lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
3162                                                     lif->rx_coalesce_usecs);
3163         lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3164         lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3165         set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
3166         set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
3167
3168         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
3169
3170         mutex_init(&lif->queue_lock);
3171         mutex_init(&lif->config_lock);
3172
3173         spin_lock_init(&lif->adminq_lock);
3174
3175         spin_lock_init(&lif->deferred.lock);
3176         INIT_LIST_HEAD(&lif->deferred.list);
3177         INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
3178
3179         /* allocate lif info */
3180         lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
3181         lif->info = dma_alloc_coherent(dev, lif->info_sz,
3182                                        &lif->info_pa, GFP_KERNEL);
3183         if (!lif->info) {
3184                 dev_err(dev, "Failed to allocate lif info, aborting\n");
3185                 err = -ENOMEM;
3186                 goto err_out_free_mutex;
3187         }
3188
3189         ionic_debugfs_add_lif(lif);
3190
3191         /* allocate control queues and txrx queue arrays */
3192         ionic_lif_queue_identify(lif);
3193         err = ionic_qcqs_alloc(lif);
3194         if (err)
3195                 goto err_out_free_lif_info;
3196
3197         /* allocate rss indirection table */
3198         tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
3199         lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
3200         lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
3201                                               &lif->rss_ind_tbl_pa,
3202                                               GFP_KERNEL);
3203
3204         if (!lif->rss_ind_tbl) {
3205                 err = -ENOMEM;
3206                 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
3207                 goto err_out_free_qcqs;
3208         }
3209         netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
3210
3211         ionic_lif_alloc_phc(lif);
3212
3213         return 0;
3214
3215 err_out_free_qcqs:
3216         ionic_qcqs_free(lif);
3217 err_out_free_lif_info:
3218         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3219         lif->info = NULL;
3220         lif->info_pa = 0;
3221 err_out_free_mutex:
3222         mutex_destroy(&lif->config_lock);
3223         mutex_destroy(&lif->queue_lock);
3224 err_out_free_netdev:
3225         free_netdev(lif->netdev);
3226         lif = NULL;
3227 err_out_free_lid:
3228         kfree(lid);
3229
3230         return err;
3231 }
3232
3233 static void ionic_lif_reset(struct ionic_lif *lif)
3234 {
3235         struct ionic_dev *idev = &lif->ionic->idev;
3236
3237         mutex_lock(&lif->ionic->dev_cmd_lock);
3238         ionic_dev_cmd_lif_reset(idev, lif->index);
3239         ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3240         mutex_unlock(&lif->ionic->dev_cmd_lock);
3241 }
3242
3243 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
3244 {
3245         struct ionic *ionic = lif->ionic;
3246
3247         if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
3248                 return;
3249
3250         dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
3251
3252         netif_device_detach(lif->netdev);
3253
3254         mutex_lock(&lif->queue_lock);
3255         if (test_bit(IONIC_LIF_F_UP, lif->state)) {
3256                 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
3257                 ionic_stop_queues(lif);
3258         }
3259
3260         if (netif_running(lif->netdev)) {
3261                 ionic_txrx_deinit(lif);
3262                 ionic_txrx_free(lif);
3263         }
3264         ionic_lif_deinit(lif);
3265         ionic_reset(ionic);
3266         ionic_qcqs_free(lif);
3267
3268         mutex_unlock(&lif->queue_lock);
3269
3270         clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
3271         dev_info(ionic->dev, "FW Down: LIFs stopped\n");
3272 }
3273
3274 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
3275 {
3276         struct ionic *ionic = lif->ionic;
3277         int err;
3278
3279         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3280                 return;
3281
3282         dev_info(ionic->dev, "FW Up: restarting LIFs\n");
3283
3284         ionic_init_devinfo(ionic);
3285         err = ionic_identify(ionic);
3286         if (err)
3287                 goto err_out;
3288         err = ionic_port_identify(ionic);
3289         if (err)
3290                 goto err_out;
3291         err = ionic_port_init(ionic);
3292         if (err)
3293                 goto err_out;
3294
3295         mutex_lock(&lif->queue_lock);
3296
3297         if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
3298                 dev_info(ionic->dev, "FW Up: clearing broken state\n");
3299
3300         err = ionic_qcqs_alloc(lif);
3301         if (err)
3302                 goto err_unlock;
3303
3304         err = ionic_lif_init(lif);
3305         if (err)
3306                 goto err_qcqs_free;
3307
3308         ionic_vf_attr_replay(lif);
3309
3310         if (lif->registered)
3311                 ionic_lif_set_netdev_info(lif);
3312
3313         ionic_rx_filter_replay(lif);
3314
3315         if (netif_running(lif->netdev)) {
3316                 err = ionic_txrx_alloc(lif);
3317                 if (err)
3318                         goto err_lifs_deinit;
3319
3320                 err = ionic_txrx_init(lif);
3321                 if (err)
3322                         goto err_txrx_free;
3323         }
3324
3325         mutex_unlock(&lif->queue_lock);
3326
3327         clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3328         ionic_link_status_check_request(lif, CAN_SLEEP);
3329         netif_device_attach(lif->netdev);
3330         dev_info(ionic->dev, "FW Up: LIFs restarted\n");
3331
3332         /* restore the hardware timestamping queues */
3333         ionic_lif_hwstamp_replay(lif);
3334
3335         return;
3336
3337 err_txrx_free:
3338         ionic_txrx_free(lif);
3339 err_lifs_deinit:
3340         ionic_lif_deinit(lif);
3341 err_qcqs_free:
3342         ionic_qcqs_free(lif);
3343 err_unlock:
3344         mutex_unlock(&lif->queue_lock);
3345 err_out:
3346         dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3347 }
3348
3349 void ionic_lif_free(struct ionic_lif *lif)
3350 {
3351         struct device *dev = lif->ionic->dev;
3352
3353         ionic_lif_free_phc(lif);
3354
3355         /* free rss indirection table */
3356         dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3357                           lif->rss_ind_tbl_pa);
3358         lif->rss_ind_tbl = NULL;
3359         lif->rss_ind_tbl_pa = 0;
3360
3361         /* free queues */
3362         ionic_qcqs_free(lif);
3363         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3364                 ionic_lif_reset(lif);
3365
3366         /* free lif info */
3367         kfree(lif->identity);
3368         dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3369         lif->info = NULL;
3370         lif->info_pa = 0;
3371
3372         /* unmap doorbell page */
3373         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3374         lif->kern_dbpage = NULL;
3375
3376         mutex_destroy(&lif->config_lock);
3377         mutex_destroy(&lif->queue_lock);
3378
3379         /* free netdev & lif */
3380         ionic_debugfs_del_lif(lif);
3381         free_netdev(lif->netdev);
3382 }
3383
3384 void ionic_lif_deinit(struct ionic_lif *lif)
3385 {
3386         if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3387                 return;
3388
3389         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3390                 cancel_work_sync(&lif->deferred.work);
3391                 cancel_work_sync(&lif->tx_timeout_work);
3392                 ionic_rx_filters_deinit(lif);
3393                 if (lif->netdev->features & NETIF_F_RXHASH)
3394                         ionic_lif_rss_deinit(lif);
3395         }
3396
3397         napi_disable(&lif->adminqcq->napi);
3398         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3399         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3400
3401         ionic_lif_reset(lif);
3402 }
3403
3404 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3405 {
3406         struct device *dev = lif->ionic->dev;
3407         struct ionic_q_init_comp comp;
3408         struct ionic_dev *idev;
3409         struct ionic_qcq *qcq;
3410         struct ionic_queue *q;
3411         int err;
3412
3413         idev = &lif->ionic->idev;
3414         qcq = lif->adminqcq;
3415         q = &qcq->q;
3416
3417         mutex_lock(&lif->ionic->dev_cmd_lock);
3418         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3419         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3420         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3421         mutex_unlock(&lif->ionic->dev_cmd_lock);
3422         if (err) {
3423                 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3424                 return err;
3425         }
3426
3427         q->hw_type = comp.hw_type;
3428         q->hw_index = le32_to_cpu(comp.hw_index);
3429         q->dbval = IONIC_DBELL_QID(q->hw_index);
3430
3431         dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3432         dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3433
3434         q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;
3435         q->dbell_jiffies = jiffies;
3436
3437         netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
3438
3439         qcq->napi_qcq = qcq;
3440         timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
3441
3442         napi_enable(&qcq->napi);
3443
3444         if (qcq->flags & IONIC_QCQ_F_INTR)
3445                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3446                                 IONIC_INTR_MASK_CLEAR);
3447
3448         qcq->flags |= IONIC_QCQ_F_INITED;
3449
3450         return 0;
3451 }
3452
3453 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3454 {
3455         struct ionic_qcq *qcq = lif->notifyqcq;
3456         struct device *dev = lif->ionic->dev;
3457         struct ionic_queue *q = &qcq->q;
3458         int err;
3459
3460         struct ionic_admin_ctx ctx = {
3461                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3462                 .cmd.q_init = {
3463                         .opcode = IONIC_CMD_Q_INIT,
3464                         .lif_index = cpu_to_le16(lif->index),
3465                         .type = q->type,
3466                         .ver = lif->qtype_info[q->type].version,
3467                         .index = cpu_to_le32(q->index),
3468                         .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3469                                              IONIC_QINIT_F_ENA),
3470                         .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3471                         .pid = cpu_to_le16(q->pid),
3472                         .ring_size = ilog2(q->num_descs),
3473                         .ring_base = cpu_to_le64(q->base_pa),
3474                 }
3475         };
3476
3477         dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3478         dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3479         dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3480         dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3481
3482         err = ionic_adminq_post_wait(lif, &ctx);
3483         if (err)
3484                 return err;
3485
3486         lif->last_eid = 0;
3487         q->hw_type = ctx.comp.q_init.hw_type;
3488         q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3489         q->dbval = IONIC_DBELL_QID(q->hw_index);
3490
3491         dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3492         dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3493
3494         /* preset the callback info */
3495         q->info[0].cb_arg = lif;
3496
3497         qcq->flags |= IONIC_QCQ_F_INITED;
3498
3499         return 0;
3500 }
3501
3502 static int ionic_station_set(struct ionic_lif *lif)
3503 {
3504         struct net_device *netdev = lif->netdev;
3505         struct ionic_admin_ctx ctx = {
3506                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3507                 .cmd.lif_getattr = {
3508                         .opcode = IONIC_CMD_LIF_GETATTR,
3509                         .index = cpu_to_le16(lif->index),
3510                         .attr = IONIC_LIF_ATTR_MAC,
3511                 },
3512         };
3513         u8 mac_address[ETH_ALEN];
3514         struct sockaddr addr;
3515         int err;
3516
3517         err = ionic_adminq_post_wait(lif, &ctx);
3518         if (err)
3519                 return err;
3520         netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3521                    ctx.comp.lif_getattr.mac);
3522         ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);
3523
3524         if (is_zero_ether_addr(mac_address)) {
3525                 eth_hw_addr_random(netdev);
3526                 netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);
3527                 ether_addr_copy(mac_address, netdev->dev_addr);
3528
3529                 err = ionic_program_mac(lif, mac_address);
3530                 if (err < 0)
3531                         return err;
3532
3533                 if (err > 0) {
3534                         netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",
3535                                    __func__);
3536                         return 0;
3537                 }
3538         }
3539
3540         if (!is_zero_ether_addr(netdev->dev_addr)) {
3541                 /* If the netdev mac is non-zero and doesn't match the default
3542                  * device address, it was set by something earlier and we're
3543                  * likely here again after a fw-upgrade reset.  We need to be
3544                  * sure the netdev mac is in our filter list.
3545                  */
3546                 if (!ether_addr_equal(mac_address, netdev->dev_addr))
3547                         ionic_lif_addr_add(lif, netdev->dev_addr);
3548         } else {
3549                 /* Update the netdev mac with the device's mac */
3550                 ether_addr_copy(addr.sa_data, mac_address);
3551                 addr.sa_family = AF_INET;
3552                 err = eth_prepare_mac_addr_change(netdev, &addr);
3553                 if (err) {
3554                         netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3555                                     addr.sa_data, err);
3556                         return 0;
3557                 }
3558
3559                 eth_commit_mac_addr_change(netdev, &addr);
3560         }
3561
3562         netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3563                    netdev->dev_addr);
3564         ionic_lif_addr_add(lif, netdev->dev_addr);
3565
3566         return 0;
3567 }
3568
3569 int ionic_lif_init(struct ionic_lif *lif)
3570 {
3571         struct ionic_dev *idev = &lif->ionic->idev;
3572         struct device *dev = lif->ionic->dev;
3573         struct ionic_lif_init_comp comp;
3574         int dbpage_num;
3575         int err;
3576
3577         mutex_lock(&lif->ionic->dev_cmd_lock);
3578         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3579         err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3580         ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3581         mutex_unlock(&lif->ionic->dev_cmd_lock);
3582         if (err)
3583                 return err;
3584
3585         lif->hw_index = le16_to_cpu(comp.hw_index);
3586
3587         /* now that we have the hw_index we can figure out our doorbell page */
3588         lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3589         if (!lif->dbid_count) {
3590                 dev_err(dev, "No doorbell pages, aborting\n");
3591                 return -EINVAL;
3592         }
3593
3594         lif->kern_pid = 0;
3595         dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3596         lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3597         if (!lif->kern_dbpage) {
3598                 dev_err(dev, "Cannot map dbpage, aborting\n");
3599                 return -ENOMEM;
3600         }
3601
3602         err = ionic_lif_adminq_init(lif);
3603         if (err)
3604                 goto err_out_adminq_deinit;
3605
3606         if (lif->ionic->nnqs_per_lif) {
3607                 err = ionic_lif_notifyq_init(lif);
3608                 if (err)
3609                         goto err_out_notifyq_deinit;
3610         }
3611
3612         err = ionic_init_nic_features(lif);
3613         if (err)
3614                 goto err_out_notifyq_deinit;
3615
3616         if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3617                 err = ionic_rx_filters_init(lif);
3618                 if (err)
3619                         goto err_out_notifyq_deinit;
3620         }
3621
3622         err = ionic_station_set(lif);
3623         if (err)
3624                 goto err_out_notifyq_deinit;
3625
3626         lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3627
3628         set_bit(IONIC_LIF_F_INITED, lif->state);
3629
3630         INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3631
3632         return 0;
3633
3634 err_out_notifyq_deinit:
3635         napi_disable(&lif->adminqcq->napi);
3636         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3637 err_out_adminq_deinit:
3638         ionic_lif_qcq_deinit(lif, lif->adminqcq);
3639         ionic_lif_reset(lif);
3640         ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3641         lif->kern_dbpage = NULL;
3642
3643         return err;
3644 }
3645
3646 static void ionic_lif_notify_work(struct work_struct *ws)
3647 {
3648 }
3649
3650 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3651 {
3652         struct ionic_admin_ctx ctx = {
3653                 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3654                 .cmd.lif_setattr = {
3655                         .opcode = IONIC_CMD_LIF_SETATTR,
3656                         .index = cpu_to_le16(lif->index),
3657                         .attr = IONIC_LIF_ATTR_NAME,
3658                 },
3659         };
3660
3661         strscpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3662                 sizeof(ctx.cmd.lif_setattr.name));
3663
3664         ionic_adminq_post_wait(lif, &ctx);
3665 }
3666
3667 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3668 {
3669         if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3670                 return NULL;
3671
3672         return netdev_priv(netdev);
3673 }
3674
3675 static int ionic_lif_notify(struct notifier_block *nb,
3676                             unsigned long event, void *info)
3677 {
3678         struct net_device *ndev = netdev_notifier_info_to_dev(info);
3679         struct ionic *ionic = container_of(nb, struct ionic, nb);
3680         struct ionic_lif *lif = ionic_netdev_lif(ndev);
3681
3682         if (!lif || lif->ionic != ionic)
3683                 return NOTIFY_DONE;
3684
3685         switch (event) {
3686         case NETDEV_CHANGENAME:
3687                 ionic_lif_set_netdev_info(lif);
3688                 break;
3689         }
3690
3691         return NOTIFY_DONE;
3692 }
3693
3694 int ionic_lif_register(struct ionic_lif *lif)
3695 {
3696         int err;
3697
3698         ionic_lif_register_phc(lif);
3699
3700         INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3701
3702         lif->ionic->nb.notifier_call = ionic_lif_notify;
3703
3704         err = register_netdevice_notifier(&lif->ionic->nb);
3705         if (err)
3706                 lif->ionic->nb.notifier_call = NULL;
3707
3708         /* only register LIF0 for now */
3709         err = register_netdev(lif->netdev);
3710         if (err) {
3711                 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3712                 ionic_lif_unregister_phc(lif);
3713                 return err;
3714         }
3715
3716         ionic_link_status_check_request(lif, CAN_SLEEP);
3717         lif->registered = true;
3718         ionic_lif_set_netdev_info(lif);
3719
3720         return 0;
3721 }
3722
3723 void ionic_lif_unregister(struct ionic_lif *lif)
3724 {
3725         if (lif->ionic->nb.notifier_call) {
3726                 unregister_netdevice_notifier(&lif->ionic->nb);
3727                 cancel_work_sync(&lif->ionic->nb_work);
3728                 lif->ionic->nb.notifier_call = NULL;
3729         }
3730
3731         if (lif->netdev->reg_state == NETREG_REGISTERED)
3732                 unregister_netdev(lif->netdev);
3733
3734         ionic_lif_unregister_phc(lif);
3735
3736         lif->registered = false;
3737 }
3738
3739 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3740 {
3741         union ionic_q_identity __iomem *q_ident;
3742         struct ionic *ionic = lif->ionic;
3743         struct ionic_dev *idev;
3744         int qtype;
3745         int err;
3746
3747         idev = &lif->ionic->idev;
3748         q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3749
3750         for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3751                 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3752
3753                 /* filter out the ones we know about */
3754                 switch (qtype) {
3755                 case IONIC_QTYPE_ADMINQ:
3756                 case IONIC_QTYPE_NOTIFYQ:
3757                 case IONIC_QTYPE_RXQ:
3758                 case IONIC_QTYPE_TXQ:
3759                         break;
3760                 default:
3761                         continue;
3762                 }
3763
3764                 memset(qti, 0, sizeof(*qti));
3765
3766                 mutex_lock(&ionic->dev_cmd_lock);
3767                 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3768                                              ionic_qtype_versions[qtype]);
3769                 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3770                 if (!err) {
3771                         qti->version   = readb(&q_ident->version);
3772                         qti->supported = readb(&q_ident->supported);
3773                         qti->features  = readq(&q_ident->features);
3774                         qti->desc_sz   = readw(&q_ident->desc_sz);
3775                         qti->comp_sz   = readw(&q_ident->comp_sz);
3776                         qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3777                         qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3778                         qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3779                 }
3780                 mutex_unlock(&ionic->dev_cmd_lock);
3781
3782                 if (err == -EINVAL) {
3783                         dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3784                         continue;
3785                 } else if (err == -EIO) {
3786                         dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3787                         return;
3788                 } else if (err) {
3789                         dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3790                                 qtype, err);
3791                         return;
3792                 }
3793
3794                 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3795                         qtype, qti->version);
3796                 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3797                         qtype, qti->supported);
3798                 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3799                         qtype, qti->features);
3800                 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3801                         qtype, qti->desc_sz);
3802                 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3803                         qtype, qti->comp_sz);
3804                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3805                         qtype, qti->sg_desc_sz);
3806                 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3807                         qtype, qti->max_sg_elems);
3808                 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3809                         qtype, qti->sg_desc_stride);
3810         }
3811 }
3812
3813 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3814                        union ionic_lif_identity *lid)
3815 {
3816         struct ionic_dev *idev = &ionic->idev;
3817         size_t sz;
3818         int err;
3819
3820         sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3821
3822         mutex_lock(&ionic->dev_cmd_lock);
3823         ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3824         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3825         memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3826         mutex_unlock(&ionic->dev_cmd_lock);
3827         if (err)
3828                 return (err);
3829
3830         dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3831                 le64_to_cpu(lid->capabilities));
3832
3833         dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3834                 le32_to_cpu(lid->eth.max_ucast_filters));
3835         dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3836                 le32_to_cpu(lid->eth.max_mcast_filters));
3837         dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3838                 le64_to_cpu(lid->eth.config.features));
3839         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3840                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3841         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3842                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3843         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3844                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3845         dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3846                 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3847         dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3848         dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3849         dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3850                 le32_to_cpu(lid->eth.config.mtu));
3851
3852         return 0;
3853 }
3854
3855 int ionic_lif_size(struct ionic *ionic)
3856 {
3857         struct ionic_identity *ident = &ionic->ident;
3858         unsigned int nintrs, dev_nintrs;
3859         union ionic_lif_config *lc;
3860         unsigned int ntxqs_per_lif;
3861         unsigned int nrxqs_per_lif;
3862         unsigned int neqs_per_lif;
3863         unsigned int nnqs_per_lif;
3864         unsigned int nxqs, neqs;
3865         unsigned int min_intrs;
3866         int err;
3867
3868         /* retrieve basic values from FW */
3869         lc = &ident->lif.eth.config;
3870         dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3871         neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3872         nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3873         ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3874         nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3875
3876         /* limit values to play nice with kdump */
3877         if (is_kdump_kernel()) {
3878                 dev_nintrs = 2;
3879                 neqs_per_lif = 0;
3880                 nnqs_per_lif = 0;
3881                 ntxqs_per_lif = 1;
3882                 nrxqs_per_lif = 1;
3883         }
3884
3885         /* reserve last queue id for hardware timestamping */
3886         if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3887                 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3888                         lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3889                 } else {
3890                         ntxqs_per_lif -= 1;
3891                         nrxqs_per_lif -= 1;
3892                 }
3893         }
3894
3895         nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3896         nxqs = min(nxqs, num_online_cpus());
3897         neqs = min(neqs_per_lif, num_online_cpus());
3898
3899 try_again:
3900         /* interrupt usage:
3901          *    1 for master lif adminq/notifyq
3902          *    1 for each CPU for master lif TxRx queue pairs
3903          *    whatever's left is for RDMA queues
3904          */
3905         nintrs = 1 + nxqs + neqs;
3906         min_intrs = 2;  /* adminq + 1 TxRx queue pair */
3907
3908         if (nintrs > dev_nintrs)
3909                 goto try_fewer;
3910
3911         err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3912         if (err < 0 && err != -ENOSPC) {
3913                 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3914                 return err;
3915         }
3916         if (err == -ENOSPC)
3917                 goto try_fewer;
3918
3919         if (err != nintrs) {
3920                 ionic_bus_free_irq_vectors(ionic);
3921                 goto try_fewer;
3922         }
3923
3924         ionic->nnqs_per_lif = nnqs_per_lif;
3925         ionic->neqs_per_lif = neqs;
3926         ionic->ntxqs_per_lif = nxqs;
3927         ionic->nrxqs_per_lif = nxqs;
3928         ionic->nintrs = nintrs;
3929
3930         ionic_debugfs_add_sizes(ionic);
3931
3932         return 0;
3933
3934 try_fewer:
3935         if (nnqs_per_lif > 1) {
3936                 nnqs_per_lif >>= 1;
3937                 goto try_again;
3938         }
3939         if (neqs > 1) {
3940                 neqs >>= 1;
3941                 goto try_again;
3942         }
3943         if (nxqs > 1) {
3944                 nxqs >>= 1;
3945                 goto try_again;
3946         }
3947         dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3948         return -ENOSPC;
3949 }