drivers/net/ethernet/pensando/ionic/ionic_dev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/errno.h>
   7 #include <linux/io.h>
   8 #include <linux/slab.h>
   9 #include <linux/etherdevice.h>
  10 #include "ionic.h"
  11 #include "ionic_dev.h"
  12 #include "ionic_lif.h"
  13
  14 static void ionic_watchdog_cb(struct timer_list *t)
  15 {
  16         struct ionic *ionic = from_timer(ionic, t, watchdog_timer);
  17         struct ionic_lif *lif = ionic->lif;
  18         struct ionic_deferred_work *work;
  19         int hb;
  20
  21         mod_timer(&ionic->watchdog_timer,
  22                   round_jiffies(jiffies + ionic->watchdog_period));
  23
  24         if (!lif)
  25                 return;
  26
  27         hb = ionic_heartbeat_check(ionic);
  28         dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n",
  29                 __func__, hb, netif_running(lif->netdev),
  30                 test_bit(IONIC_LIF_F_UP, lif->state));
  31
  32         if (hb >= 0 &&
  33             !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
  34                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
  35
  36         if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) &&
  37             !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
  38                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
  39                 if (!work) {
  40                         netdev_err(lif->netdev, "rxmode change dropped\n");
  41                         return;
  42                 }
  43
  44                 work->type = IONIC_DW_TYPE_RX_MODE;
  45                 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
  46                 ionic_lif_deferred_enqueue(&lif->deferred, work);
  47         }
  48 }
  49
  50 static void ionic_watchdog_init(struct ionic *ionic)
  51 {
  52         struct ionic_dev *idev = &ionic->idev;
  53
  54         timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
  55         ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;
  56
  57         /* set times to ensure the first check will proceed */
  58         atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
  59         idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
  60         /* init as ready, so no transition if the first check succeeds */
  61         idev->last_fw_hb = 0;
  62         idev->fw_hb_ready = true;
  63         idev->fw_status_ready = true;
  64         idev->fw_generation = IONIC_FW_STS_F_GENERATION &
  65                               ioread8(&idev->dev_info_regs->fw_status);
  66 }
  67
  68 void ionic_init_devinfo(struct ionic *ionic)
  69 {
  70         struct ionic_dev *idev = &ionic->idev;
  71
  72         idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
  73         idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);
  74
  75         memcpy_fromio(idev->dev_info.fw_version,
  76                       idev->dev_info_regs->fw_version,
  77                       IONIC_DEVINFO_FWVERS_BUFLEN);
  78
  79         memcpy_fromio(idev->dev_info.serial_num,
  80                       idev->dev_info_regs->serial_num,
  81                       IONIC_DEVINFO_SERIAL_BUFLEN);
  82
  83         idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
  84         idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;
  85
  86         dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
  87 }
  88
  89 int ionic_dev_setup(struct ionic *ionic)
  90 {
  91         struct ionic_dev_bar *bar = ionic->bars;
  92         unsigned int num_bars = ionic->num_bars;
  93         struct ionic_dev *idev = &ionic->idev;
  94         struct device *dev = ionic->dev;
  95         int size;
  96         u32 sig;
  97
  98         /* BAR0: dev_cmd and interrupts */
  99         if (num_bars < 1) {
 100                 dev_err(dev, "No bars found, aborting\n");
 101                 return -EFAULT;
 102         }
 103
 104         if (bar->len < IONIC_BAR0_SIZE) {
 105                 dev_err(dev, "Resource bar size %lu too small, aborting\n",
 106                         bar->len);
 107                 return -EFAULT;
 108         }
 109
 110         idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
 111         idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
 112         idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
 113         idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;
 114
 115         idev->hwstamp_regs = &idev->dev_info_regs->hwstamp;
 116
 117         sig = ioread32(&idev->dev_info_regs->signature);
 118         if (sig != IONIC_DEV_INFO_SIGNATURE) {
 119                 dev_err(dev, "Incompatible firmware signature %x", sig);
 120                 return -EFAULT;
 121         }
 122
 123         ionic_init_devinfo(ionic);
 124
 125         /* BAR1: doorbells */
 126         bar++;
 127         if (num_bars < 2) {
 128                 dev_err(dev, "Doorbell bar missing, aborting\n");
 129                 return -EFAULT;
 130         }
 131
 132         ionic_watchdog_init(ionic);
 133
 134         idev->db_pages = bar->vaddr;
 135         idev->phy_db_pages = bar->bus_addr;
 136
 137         /* BAR2: optional controller memory mapping */
 138         bar++;
 139         mutex_init(&idev->cmb_inuse_lock);
 140         if (num_bars < 3 || !ionic->bars[IONIC_PCI_BAR_CMB].len) {
 141                 idev->cmb_inuse = NULL;
 142                 return 0;
 143         }
 144
 145         idev->phy_cmb_pages = bar->bus_addr;
 146         idev->cmb_npages = bar->len / PAGE_SIZE;
 147         size = BITS_TO_LONGS(idev->cmb_npages) * sizeof(long);
 148         idev->cmb_inuse = kzalloc(size, GFP_KERNEL);
 149         if (!idev->cmb_inuse)
 150                 dev_warn(dev, "No memory for CMB, disabling\n");
 151
 152         return 0;
 153 }
 154
 155 void ionic_dev_teardown(struct ionic *ionic)
 156 {
 157         struct ionic_dev *idev = &ionic->idev;
 158
 159         kfree(idev->cmb_inuse);
 160         idev->cmb_inuse = NULL;
 161         idev->phy_cmb_pages = 0;
 162         idev->cmb_npages = 0;
 163
 164         mutex_destroy(&idev->cmb_inuse_lock);
 165 }
 166
 167 /* Devcmd Interface */
 168 static bool __ionic_is_fw_running(struct ionic_dev *idev, u8 *status_ptr)
 169 {
 170         u8 fw_status;
 171
 172         if (!idev->dev_info_regs) {
 173                 if (status_ptr)
 174                         *status_ptr = 0xff;
 175                 return false;
 176         }
 177
 178         fw_status = ioread8(&idev->dev_info_regs->fw_status);
 179         if (status_ptr)
 180                 *status_ptr = fw_status;
 181
 182         /* firmware is useful only if the running bit is set and
 183          * fw_status != 0xff (bad PCI read)
 184          */
 185         return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
 186 }
 187
 188 bool ionic_is_fw_running(struct ionic_dev *idev)
 189 {
 190         return __ionic_is_fw_running(idev, NULL);
 191 }
 192
 193 int ionic_heartbeat_check(struct ionic *ionic)
 194 {
 195         unsigned long check_time, last_check_time;
 196         struct ionic_dev *idev = &ionic->idev;
 197         struct ionic_lif *lif = ionic->lif;
 198         bool fw_status_ready = true;
 199         bool fw_hb_ready;
 200         u8 fw_generation;
 201         u8 fw_status;
 202         u32 fw_hb;
 203
 204         /* wait a least one second before testing again */
 205         check_time = jiffies;
 206         last_check_time = atomic_long_read(&idev->last_check_time);
 207 do_check_time:
 208         if (time_before(check_time, last_check_time + HZ))
 209                 return 0;
 210         if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time,
 211                                              &last_check_time, check_time)) {
 212                 /* if called concurrently, only the first should proceed. */
 213                 dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__);
 214                 goto do_check_time;
 215         }
 216
 217         /* If fw_status is not ready don't bother with the generation */
 218         if (!__ionic_is_fw_running(idev, &fw_status)) {
 219                 fw_status_ready = false;
 220         } else {
 221                 fw_generation = fw_status & IONIC_FW_STS_F_GENERATION;
 222                 if (idev->fw_generation != fw_generation) {
 223                         dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n",
 224                                  idev->fw_generation, fw_generation);
 225
 226                         idev->fw_generation = fw_generation;
 227
 228                         /* If the generation changed, the fw status is not
 229                          * ready so we need to trigger a fw-down cycle.  After
 230                          * the down, the next watchdog will see the fw is up
 231                          * and the generation value stable, so will trigger
 232                          * the fw-up activity.
 233                          *
 234                          * If we had already moved to FW_RESET from a RESET event,
 235                          * it is possible that we never saw the fw_status go to 0,
 236                          * so we fake the current idev->fw_status_ready here to
 237                          * force the transition and get FW up again.
 238                          */
 239                         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
 240                                 idev->fw_status_ready = false;  /* go to running */
 241                         else
 242                                 fw_status_ready = false;        /* go to down */
 243                 }
 244         }
 245
 246         dev_dbg(ionic->dev, "fw_status 0x%02x ready %d idev->ready %d last_hb 0x%x state 0x%02lx\n",
 247                 fw_status, fw_status_ready, idev->fw_status_ready,
 248                 idev->last_fw_hb, lif->state[0]);
 249
 250         /* is this a transition? */
 251         if (fw_status_ready != idev->fw_status_ready &&
 252             !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
 253                 bool trigger = false;
 254
 255                 idev->fw_status_ready = fw_status_ready;
 256
 257                 if (!fw_status_ready &&
 258                     !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
 259                     !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
 260                         dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status);
 261                         trigger = true;
 262
 263                 } else if (fw_status_ready &&
 264                            test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
 265                         dev_info(ionic->dev, "FW running 0x%02x\n", fw_status);
 266                         trigger = true;
 267                 }
 268
 269                 if (trigger) {
 270                         struct ionic_deferred_work *work;
 271
 272                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
 273                         if (work) {
 274                                 work->type = IONIC_DW_TYPE_LIF_RESET;
 275                                 work->fw_status = fw_status_ready;
 276                                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 277                         }
 278                 }
 279         }
 280
 281         if (!idev->fw_status_ready)
 282                 return -ENXIO;
 283
 284         /* Because of some variability in the actual FW heartbeat, we
 285          * wait longer than the DEVCMD_TIMEOUT before checking again.
 286          */
 287         last_check_time = idev->last_hb_time;
 288         if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ))
 289                 return 0;
 290
 291         fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
 292         fw_hb_ready = fw_hb != idev->last_fw_hb;
 293
 294         /* early FW version had no heartbeat, so fake it */
 295         if (!fw_hb_ready && !fw_hb)
 296                 fw_hb_ready = true;
 297
 298         dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n",
 299                 __func__, fw_hb, idev->last_fw_hb, fw_hb_ready);
 300
 301         idev->last_fw_hb = fw_hb;
 302
 303         /* log a transition */
 304         if (fw_hb_ready != idev->fw_hb_ready) {
 305                 idev->fw_hb_ready = fw_hb_ready;
 306                 if (!fw_hb_ready)
 307                         dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb);
 308                 else
 309                         dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb);
 310         }
 311
 312         if (!fw_hb_ready)
 313                 return -ENXIO;
 314
 315         idev->last_hb_time = check_time;
 316
 317         return 0;
 318 }
 319
 320 u8 ionic_dev_cmd_status(struct ionic_dev *idev)
 321 {
 322         if (!idev->dev_cmd_regs)
 323                 return (u8)PCI_ERROR_RESPONSE;
 324         return ioread8(&idev->dev_cmd_regs->comp.comp.status);
 325 }
 326
 327 bool ionic_dev_cmd_done(struct ionic_dev *idev)
 328 {
 329         if (!idev->dev_cmd_regs)
 330                 return false;
 331         return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
 332 }
 333
 334 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
 335 {
 336         if (!idev->dev_cmd_regs)
 337                 return;
 338         memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
 339 }
 340
 341 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
 342 {
 343         idev->opcode = cmd->cmd.opcode;
 344
 345         if (!idev->dev_cmd_regs)
 346                 return;
 347
 348         memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
 349         iowrite32(0, &idev->dev_cmd_regs->done);
 350         iowrite32(1, &idev->dev_cmd_regs->doorbell);
 351 }
 352
 353 /* Device commands */
 354 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
 355 {
 356         union ionic_dev_cmd cmd = {
 357                 .identify.opcode = IONIC_CMD_IDENTIFY,
 358                 .identify.ver = ver,
 359         };
 360
 361         ionic_dev_cmd_go(idev, &cmd);
 362 }
 363
 364 void ionic_dev_cmd_init(struct ionic_dev *idev)
 365 {
 366         union ionic_dev_cmd cmd = {
 367                 .init.opcode = IONIC_CMD_INIT,
 368                 .init.type = 0,
 369         };
 370
 371         ionic_dev_cmd_go(idev, &cmd);
 372 }
 373
 374 void ionic_dev_cmd_reset(struct ionic_dev *idev)
 375 {
 376         union ionic_dev_cmd cmd = {
 377                 .reset.opcode = IONIC_CMD_RESET,
 378         };
 379
 380         ionic_dev_cmd_go(idev, &cmd);
 381 }
 382
 383 /* Port commands */
 384 void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
 385 {
 386         union ionic_dev_cmd cmd = {
 387                 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
 388                 .port_init.index = 0,
 389         };
 390
 391         ionic_dev_cmd_go(idev, &cmd);
 392 }
 393
 394 void ionic_dev_cmd_port_init(struct ionic_dev *idev)
 395 {
 396         union ionic_dev_cmd cmd = {
 397                 .port_init.opcode = IONIC_CMD_PORT_INIT,
 398                 .port_init.index = 0,
 399                 .port_init.info_pa = cpu_to_le64(idev->port_info_pa),
 400         };
 401
 402         ionic_dev_cmd_go(idev, &cmd);
 403 }
 404
 405 void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
 406 {
 407         union ionic_dev_cmd cmd = {
 408                 .port_reset.opcode = IONIC_CMD_PORT_RESET,
 409                 .port_reset.index = 0,
 410         };
 411
 412         ionic_dev_cmd_go(idev, &cmd);
 413 }
 414
 415 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
 416 {
 417         union ionic_dev_cmd cmd = {
 418                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 419                 .port_setattr.index = 0,
 420                 .port_setattr.attr = IONIC_PORT_ATTR_STATE,
 421                 .port_setattr.state = state,
 422         };
 423
 424         ionic_dev_cmd_go(idev, &cmd);
 425 }
 426
 427 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
 428 {
 429         union ionic_dev_cmd cmd = {
 430                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 431                 .port_setattr.index = 0,
 432                 .port_setattr.attr = IONIC_PORT_ATTR_SPEED,
 433                 .port_setattr.speed = cpu_to_le32(speed),
 434         };
 435
 436         ionic_dev_cmd_go(idev, &cmd);
 437 }
 438
 439 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
 440 {
 441         union ionic_dev_cmd cmd = {
 442                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 443                 .port_setattr.index = 0,
 444                 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
 445                 .port_setattr.an_enable = an_enable,
 446         };
 447
 448         ionic_dev_cmd_go(idev, &cmd);
 449 }
 450
 451 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
 452 {
 453         union ionic_dev_cmd cmd = {
 454                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 455                 .port_setattr.index = 0,
 456                 .port_setattr.attr = IONIC_PORT_ATTR_FEC,
 457                 .port_setattr.fec_type = fec_type,
 458         };
 459
 460         ionic_dev_cmd_go(idev, &cmd);
 461 }
 462
 463 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
 464 {
 465         union ionic_dev_cmd cmd = {
 466                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 467                 .port_setattr.index = 0,
 468                 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
 469                 .port_setattr.pause_type = pause_type,
 470         };
 471
 472         ionic_dev_cmd_go(idev, &cmd);
 473 }
 474
 475 /* VF commands */
 476 int ionic_set_vf_config(struct ionic *ionic, int vf,
 477                         struct ionic_vf_setattr_cmd *vfc)
 478 {
 479         union ionic_dev_cmd cmd = {
 480                 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
 481                 .vf_setattr.attr = vfc->attr,
 482                 .vf_setattr.vf_index = cpu_to_le16(vf),
 483         };
 484         int err;
 485
 486         memcpy(cmd.vf_setattr.pad, vfc->pad, sizeof(vfc->pad));
 487
 488         mutex_lock(&ionic->dev_cmd_lock);
 489         ionic_dev_cmd_go(&ionic->idev, &cmd);
 490         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
 491         mutex_unlock(&ionic->dev_cmd_lock);
 492
 493         return err;
 494 }
 495
 496 void ionic_vf_start(struct ionic *ionic)
 497 {
 498         union ionic_dev_cmd cmd = {
 499                 .vf_ctrl.opcode = IONIC_CMD_VF_CTRL,
 500                 .vf_ctrl.ctrl_opcode = IONIC_VF_CTRL_START_ALL,
 501         };
 502
 503         if (!(ionic->ident.dev.capabilities & cpu_to_le64(IONIC_DEV_CAP_VF_CTRL)))
 504                 return;
 505
 506         ionic_dev_cmd_go(&ionic->idev, &cmd);
 507         ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
 508 }
 509
 510 /* LIF commands */
 511 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
 512                                   u16 lif_type, u8 qtype, u8 qver)
 513 {
 514         union ionic_dev_cmd cmd = {
 515                 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
 516                 .q_identify.lif_type = cpu_to_le16(lif_type),
 517                 .q_identify.type = qtype,
 518                 .q_identify.ver = qver,
 519         };
 520
 521         ionic_dev_cmd_go(idev, &cmd);
 522 }
 523
 524 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
 525 {
 526         union ionic_dev_cmd cmd = {
 527                 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
 528                 .lif_identify.type = type,
 529                 .lif_identify.ver = ver,
 530         };
 531
 532         ionic_dev_cmd_go(idev, &cmd);
 533 }
 534
 535 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
 536                             dma_addr_t info_pa)
 537 {
 538         union ionic_dev_cmd cmd = {
 539                 .lif_init.opcode = IONIC_CMD_LIF_INIT,
 540                 .lif_init.index = cpu_to_le16(lif_index),
 541                 .lif_init.info_pa = cpu_to_le64(info_pa),
 542         };
 543
 544         ionic_dev_cmd_go(idev, &cmd);
 545 }
 546
 547 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
 548 {
 549         union ionic_dev_cmd cmd = {
 550                 .lif_init.opcode = IONIC_CMD_LIF_RESET,
 551                 .lif_init.index = cpu_to_le16(lif_index),
 552         };
 553
 554         ionic_dev_cmd_go(idev, &cmd);
 555 }
 556
 557 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
 558                                u16 lif_index, u16 intr_index)
 559 {
 560         struct ionic_queue *q = &qcq->q;
 561         struct ionic_cq *cq = &qcq->cq;
 562
 563         union ionic_dev_cmd cmd = {
 564                 .q_init.opcode = IONIC_CMD_Q_INIT,
 565                 .q_init.lif_index = cpu_to_le16(lif_index),
 566                 .q_init.type = q->type,
 567                 .q_init.ver = qcq->q.lif->qtype_info[q->type].version,
 568                 .q_init.index = cpu_to_le32(q->index),
 569                 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 570                                             IONIC_QINIT_F_ENA),
 571                 .q_init.pid = cpu_to_le16(q->pid),
 572                 .q_init.intr_index = cpu_to_le16(intr_index),
 573                 .q_init.ring_size = ilog2(q->num_descs),
 574                 .q_init.ring_base = cpu_to_le64(q->base_pa),
 575                 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
 576         };
 577
 578         ionic_dev_cmd_go(idev, &cmd);
 579 }
 580
 581 int ionic_db_page_num(struct ionic_lif *lif, int pid)
 582 {
 583         return (lif->hw_index * lif->dbid_count) + pid;
 584 }
 585
 586 int ionic_get_cmb(struct ionic_lif *lif, u32 *pgid, phys_addr_t *pgaddr, int order)
 587 {
 588         struct ionic_dev *idev = &lif->ionic->idev;
 589         int ret;
 590
 591         mutex_lock(&idev->cmb_inuse_lock);
 592         ret = bitmap_find_free_region(idev->cmb_inuse, idev->cmb_npages, order);
 593         mutex_unlock(&idev->cmb_inuse_lock);
 594
 595         if (ret < 0)
 596                 return ret;
 597
 598         *pgid = ret;
 599         *pgaddr = idev->phy_cmb_pages + ret * PAGE_SIZE;
 600
 601         return 0;
 602 }
 603
 604 void ionic_put_cmb(struct ionic_lif *lif, u32 pgid, int order)
 605 {
 606         struct ionic_dev *idev = &lif->ionic->idev;
 607
 608         mutex_lock(&idev->cmb_inuse_lock);
 609         bitmap_release_region(idev->cmb_inuse, pgid, order);
 610         mutex_unlock(&idev->cmb_inuse_lock);
 611 }
 612
 613 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
 614                   struct ionic_intr_info *intr,
 615                   unsigned int num_descs, size_t desc_size)
 616 {
 617         unsigned int ring_size;
 618
 619         if (desc_size == 0 || !is_power_of_2(num_descs))
 620                 return -EINVAL;
 621
 622         ring_size = ilog2(num_descs);
 623         if (ring_size < 2 || ring_size > 16)
 624                 return -EINVAL;
 625
 626         cq->lif = lif;
 627         cq->bound_intr = intr;
 628         cq->num_descs = num_descs;
 629         cq->desc_size = desc_size;
 630         cq->tail_idx = 0;
 631         cq->done_color = 1;
 632
 633         return 0;
 634 }
 635
 636 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
 637 {
 638         struct ionic_cq_info *cur;
 639         unsigned int i;
 640
 641         cq->base = base;
 642         cq->base_pa = base_pa;
 643
 644         for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
 645                 cur->cq_desc = base + (i * cq->desc_size);
 646 }
 647
 648 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
 649 {
 650         cq->bound_q = q;
 651 }
 652
 653 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
 654                               ionic_cq_cb cb, ionic_cq_done_cb done_cb,
 655                               void *done_arg)
 656 {
 657         struct ionic_cq_info *cq_info;
 658         unsigned int work_done = 0;
 659
 660         if (work_to_do == 0)
 661                 return 0;
 662
 663         cq_info = &cq->info[cq->tail_idx];
 664         while (cb(cq, cq_info)) {
 665                 if (cq->tail_idx == cq->num_descs - 1)
 666                         cq->done_color = !cq->done_color;
 667                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
 668                 cq_info = &cq->info[cq->tail_idx];
 669
 670                 if (++work_done >= work_to_do)
 671                         break;
 672         }
 673
 674         if (work_done && done_cb)
 675                 done_cb(done_arg);
 676
 677         return work_done;
 678 }
 679
 680 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
 681                  struct ionic_queue *q, unsigned int index, const char *name,
 682                  unsigned int num_descs, size_t desc_size,
 683                  size_t sg_desc_size, unsigned int pid)
 684 {
 685         unsigned int ring_size;
 686
 687         if (desc_size == 0 || !is_power_of_2(num_descs))
 688                 return -EINVAL;
 689
 690         ring_size = ilog2(num_descs);
 691         if (ring_size < 2 || ring_size > 16)
 692                 return -EINVAL;
 693
 694         q->lif = lif;
 695         q->idev = idev;
 696         q->index = index;
 697         q->num_descs = num_descs;
 698         q->desc_size = desc_size;
 699         q->sg_desc_size = sg_desc_size;
 700         q->tail_idx = 0;
 701         q->head_idx = 0;
 702         q->pid = pid;
 703
 704         snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);
 705
 706         return 0;
 707 }
 708
 709 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 710 {
 711         struct ionic_desc_info *cur;
 712         unsigned int i;
 713
 714         q->base = base;
 715         q->base_pa = base_pa;
 716
 717         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 718                 cur->desc = base + (i * q->desc_size);
 719 }
 720
 721 void ionic_q_cmb_map(struct ionic_queue *q, void __iomem *base, dma_addr_t base_pa)
 722 {
 723         struct ionic_desc_info *cur;
 724         unsigned int i;
 725
 726         q->cmb_base = base;
 727         q->cmb_base_pa = base_pa;
 728
 729         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 730                 cur->cmb_desc = base + (i * q->desc_size);
 731 }
 732
 733 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 734 {
 735         struct ionic_desc_info *cur;
 736         unsigned int i;
 737
 738         q->sg_base = base;
 739         q->sg_base_pa = base_pa;
 740
 741         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 742                 cur->sg_desc = base + (i * q->sg_desc_size);
 743 }
 744
 745 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
 746                   void *cb_arg)
 747 {
 748         struct ionic_desc_info *desc_info;
 749         struct ionic_lif *lif = q->lif;
 750         struct device *dev = q->dev;
 751
 752         desc_info = &q->info[q->head_idx];
 753         desc_info->cb = cb;
 754         desc_info->cb_arg = cb_arg;
 755
 756         q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 757
 758         dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
 759                 q->lif->index, q->name, q->hw_type, q->hw_index,
 760                 q->head_idx, ring_doorbell);
 761
 762         if (ring_doorbell) {
 763                 ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
 764                                  q->dbval | q->head_idx);
 765
 766                 q->dbell_jiffies = jiffies;
 767
 768                 if (q_to_qcq(q)->napi_qcq)
 769                         mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
 770                                   jiffies + IONIC_NAPI_DEADLINE);
 771         }
 772 }
 773
 774 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
 775 {
 776         unsigned int mask, tail, head;
 777
 778         mask = q->num_descs - 1;
 779         tail = q->tail_idx;
 780         head = q->head_idx;
 781
 782         return ((pos - tail) & mask) < ((head - tail) & mask);
 783 }
 784
 785 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
 786                      unsigned int stop_index)
 787 {
 788         struct ionic_desc_info *desc_info;
 789         ionic_desc_cb cb;
 790         void *cb_arg;
 791         u16 index;
 792
 793         /* check for empty queue */
 794         if (q->tail_idx == q->head_idx)
 795                 return;
 796
 797         /* stop index must be for a descriptor that is not yet completed */
 798         if (unlikely(!ionic_q_is_posted(q, stop_index)))
 799                 dev_err(q->dev,
 800                         "ionic stop is not posted %s stop %u tail %u head %u\n",
 801                         q->name, stop_index, q->tail_idx, q->head_idx);
 802
 803         do {
 804                 desc_info = &q->info[q->tail_idx];
 805                 index = q->tail_idx;
 806                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 807
 808                 cb = desc_info->cb;
 809                 cb_arg = desc_info->cb_arg;
 810
 811                 desc_info->cb = NULL;
 812                 desc_info->cb_arg = NULL;
 813
 814                 if (cb)
 815                         cb(q, desc_info, cq_info, cb_arg);
 816         } while (index != stop_index);
 817 }