drivers/net/ethernet/pensando/ionic/ionic_dev.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/errno.h>
   7 #include <linux/io.h>
   8 #include <linux/slab.h>
   9 #include <linux/etherdevice.h>
  10 #include "ionic.h"
  11 #include "ionic_dev.h"
  12 #include "ionic_lif.h"
  13
  14 static void ionic_watchdog_cb(struct timer_list *t)
  15 {
  16         struct ionic *ionic = from_timer(ionic, t, watchdog_timer);
  17         struct ionic_lif *lif = ionic->lif;
  18         struct ionic_deferred_work *work;
  19         int hb;
  20
  21         mod_timer(&ionic->watchdog_timer,
  22                   round_jiffies(jiffies + ionic->watchdog_period));
  23
  24         if (!lif)
  25                 return;
  26
  27         hb = ionic_heartbeat_check(ionic);
  28         dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n",
  29                 __func__, hb, netif_running(lif->netdev),
  30                 test_bit(IONIC_LIF_F_UP, lif->state));
  31
  32         if (hb >= 0 &&
  33             !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
  34                 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
  35
  36         if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) &&
  37             !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
  38                 work = kzalloc(sizeof(*work), GFP_ATOMIC);
  39                 if (!work) {
  40                         netdev_err(lif->netdev, "rxmode change dropped\n");
  41                         return;
  42                 }
  43
  44                 work->type = IONIC_DW_TYPE_RX_MODE;
  45                 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
  46                 ionic_lif_deferred_enqueue(&lif->deferred, work);
  47         }
  48 }
  49
  50 static void ionic_watchdog_init(struct ionic *ionic)
  51 {
  52         struct ionic_dev *idev = &ionic->idev;
  53
  54         timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
  55         ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;
  56
  57         /* set times to ensure the first check will proceed */
  58         atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
  59         idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
  60         /* init as ready, so no transition if the first check succeeds */
  61         idev->last_fw_hb = 0;
  62         idev->fw_hb_ready = true;
  63         idev->fw_status_ready = true;
  64         idev->fw_generation = IONIC_FW_STS_F_GENERATION &
  65                               ioread8(&idev->dev_info_regs->fw_status);
  66 }
  67
  68 void ionic_init_devinfo(struct ionic *ionic)
  69 {
  70         struct ionic_dev *idev = &ionic->idev;
  71
  72         idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type);
  73         idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev);
  74
  75         memcpy_fromio(idev->dev_info.fw_version,
  76                       idev->dev_info_regs->fw_version,
  77                       IONIC_DEVINFO_FWVERS_BUFLEN);
  78
  79         memcpy_fromio(idev->dev_info.serial_num,
  80                       idev->dev_info_regs->serial_num,
  81                       IONIC_DEVINFO_SERIAL_BUFLEN);
  82
  83         idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0;
  84         idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0;
  85
  86         dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version);
  87 }
  88
  89 int ionic_dev_setup(struct ionic *ionic)
  90 {
  91         struct ionic_dev_bar *bar = ionic->bars;
  92         unsigned int num_bars = ionic->num_bars;
  93         struct ionic_dev *idev = &ionic->idev;
  94         struct device *dev = ionic->dev;
  95         int size;
  96         u32 sig;
  97
  98         /* BAR0: dev_cmd and interrupts */
  99         if (num_bars < 1) {
 100                 dev_err(dev, "No bars found, aborting\n");
 101                 return -EFAULT;
 102         }
 103
 104         if (bar->len < IONIC_BAR0_SIZE) {
 105                 dev_err(dev, "Resource bar size %lu too small, aborting\n",
 106                         bar->len);
 107                 return -EFAULT;
 108         }
 109
 110         idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET;
 111         idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET;
 112         idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET;
 113         idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET;
 114
 115         idev->hwstamp_regs = &idev->dev_info_regs->hwstamp;
 116
 117         sig = ioread32(&idev->dev_info_regs->signature);
 118         if (sig != IONIC_DEV_INFO_SIGNATURE) {
 119                 dev_err(dev, "Incompatible firmware signature %x", sig);
 120                 return -EFAULT;
 121         }
 122
 123         ionic_init_devinfo(ionic);
 124
 125         /* BAR1: doorbells */
 126         bar++;
 127         if (num_bars < 2) {
 128                 dev_err(dev, "Doorbell bar missing, aborting\n");
 129                 return -EFAULT;
 130         }
 131
 132         ionic_watchdog_init(ionic);
 133
 134         idev->db_pages = bar->vaddr;
 135         idev->phy_db_pages = bar->bus_addr;
 136
 137         /* BAR2: optional controller memory mapping */
 138         bar++;
 139         mutex_init(&idev->cmb_inuse_lock);
 140         if (num_bars < 3 || !ionic->bars[IONIC_PCI_BAR_CMB].len) {
 141                 idev->cmb_inuse = NULL;
 142                 return 0;
 143         }
 144
 145         idev->phy_cmb_pages = bar->bus_addr;
 146         idev->cmb_npages = bar->len / PAGE_SIZE;
 147         size = BITS_TO_LONGS(idev->cmb_npages) * sizeof(long);
 148         idev->cmb_inuse = kzalloc(size, GFP_KERNEL);
 149         if (!idev->cmb_inuse)
 150                 dev_warn(dev, "No memory for CMB, disabling\n");
 151
 152         return 0;
 153 }
 154
 155 void ionic_dev_teardown(struct ionic *ionic)
 156 {
 157         struct ionic_dev *idev = &ionic->idev;
 158
 159         kfree(idev->cmb_inuse);
 160         idev->cmb_inuse = NULL;
 161         idev->phy_cmb_pages = 0;
 162         idev->cmb_npages = 0;
 163
 164         mutex_destroy(&idev->cmb_inuse_lock);
 165 }
 166
 167 /* Devcmd Interface */
 168 static bool __ionic_is_fw_running(struct ionic_dev *idev, u8 *status_ptr)
 169 {
 170         u8 fw_status;
 171
 172         if (!idev->dev_info_regs) {
 173                 if (status_ptr)
 174                         *status_ptr = 0xff;
 175                 return false;
 176         }
 177
 178         fw_status = ioread8(&idev->dev_info_regs->fw_status);
 179         if (status_ptr)
 180                 *status_ptr = fw_status;
 181
 182         /* firmware is useful only if the running bit is set and
 183          * fw_status != 0xff (bad PCI read)
 184          */
 185         return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
 186 }
 187
 188 bool ionic_is_fw_running(struct ionic_dev *idev)
 189 {
 190         return __ionic_is_fw_running(idev, NULL);
 191 }
 192
 193 int ionic_heartbeat_check(struct ionic *ionic)
 194 {
 195         unsigned long check_time, last_check_time;
 196         struct ionic_dev *idev = &ionic->idev;
 197         struct ionic_lif *lif = ionic->lif;
 198         bool fw_status_ready = true;
 199         bool fw_hb_ready;
 200         u8 fw_generation;
 201         u8 fw_status;
 202         u32 fw_hb;
 203
 204         /* wait a least one second before testing again */
 205         check_time = jiffies;
 206         last_check_time = atomic_long_read(&idev->last_check_time);
 207 do_check_time:
 208         if (time_before(check_time, last_check_time + HZ))
 209                 return 0;
 210         if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time,
 211                                              &last_check_time, check_time)) {
 212                 /* if called concurrently, only the first should proceed. */
 213                 dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__);
 214                 goto do_check_time;
 215         }
 216
 217         /* If fw_status is not ready don't bother with the generation */
 218         if (!__ionic_is_fw_running(idev, &fw_status)) {
 219                 fw_status_ready = false;
 220         } else {
 221                 fw_generation = fw_status & IONIC_FW_STS_F_GENERATION;
 222                 if (idev->fw_generation != fw_generation) {
 223                         dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n",
 224                                  idev->fw_generation, fw_generation);
 225
 226                         idev->fw_generation = fw_generation;
 227
 228                         /* If the generation changed, the fw status is not
 229                          * ready so we need to trigger a fw-down cycle.  After
 230                          * the down, the next watchdog will see the fw is up
 231                          * and the generation value stable, so will trigger
 232                          * the fw-up activity.
 233                          *
 234                          * If we had already moved to FW_RESET from a RESET event,
 235                          * it is possible that we never saw the fw_status go to 0,
 236                          * so we fake the current idev->fw_status_ready here to
 237                          * force the transition and get FW up again.
 238                          */
 239                         if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
 240                                 idev->fw_status_ready = false;  /* go to running */
 241                         else
 242                                 fw_status_ready = false;        /* go to down */
 243                 }
 244         }
 245
 246         dev_dbg(ionic->dev, "fw_status 0x%02x ready %d idev->ready %d last_hb 0x%x state 0x%02lx\n",
 247                 fw_status, fw_status_ready, idev->fw_status_ready,
 248                 idev->last_fw_hb, lif->state[0]);
 249
 250         /* is this a transition? */
 251         if (fw_status_ready != idev->fw_status_ready &&
 252             !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
 253                 bool trigger = false;
 254
 255                 idev->fw_status_ready = fw_status_ready;
 256
 257                 if (!fw_status_ready &&
 258                     !test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
 259                     !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
 260                         dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status);
 261                         trigger = true;
 262
 263                 } else if (fw_status_ready &&
 264                            test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
 265                         dev_info(ionic->dev, "FW running 0x%02x\n", fw_status);
 266                         trigger = true;
 267                 }
 268
 269                 if (trigger) {
 270                         struct ionic_deferred_work *work;
 271
 272                         work = kzalloc(sizeof(*work), GFP_ATOMIC);
 273                         if (work) {
 274                                 work->type = IONIC_DW_TYPE_LIF_RESET;
 275                                 work->fw_status = fw_status_ready;
 276                                 ionic_lif_deferred_enqueue(&lif->deferred, work);
 277                         }
 278                 }
 279         }
 280
 281         if (!idev->fw_status_ready)
 282                 return -ENXIO;
 283
 284         /* Because of some variability in the actual FW heartbeat, we
 285          * wait longer than the DEVCMD_TIMEOUT before checking again.
 286          */
 287         last_check_time = idev->last_hb_time;
 288         if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ))
 289                 return 0;
 290
 291         fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
 292         fw_hb_ready = fw_hb != idev->last_fw_hb;
 293
 294         /* early FW version had no heartbeat, so fake it */
 295         if (!fw_hb_ready && !fw_hb)
 296                 fw_hb_ready = true;
 297
 298         dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n",
 299                 __func__, fw_hb, idev->last_fw_hb, fw_hb_ready);
 300
 301         idev->last_fw_hb = fw_hb;
 302
 303         /* log a transition */
 304         if (fw_hb_ready != idev->fw_hb_ready) {
 305                 idev->fw_hb_ready = fw_hb_ready;
 306                 if (!fw_hb_ready)
 307                         dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb);
 308                 else
 309                         dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb);
 310         }
 311
 312         if (!fw_hb_ready)
 313                 return -ENXIO;
 314
 315         idev->last_hb_time = check_time;
 316
 317         return 0;
 318 }
 319
 320 u8 ionic_dev_cmd_status(struct ionic_dev *idev)
 321 {
 322         return ioread8(&idev->dev_cmd_regs->comp.comp.status);
 323 }
 324
 325 bool ionic_dev_cmd_done(struct ionic_dev *idev)
 326 {
 327         return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE;
 328 }
 329
 330 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp)
 331 {
 332         memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp));
 333 }
 334
 335 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd)
 336 {
 337         idev->opcode = cmd->cmd.opcode;
 338         memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd));
 339         iowrite32(0, &idev->dev_cmd_regs->done);
 340         iowrite32(1, &idev->dev_cmd_regs->doorbell);
 341 }
 342
 343 /* Device commands */
 344 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver)
 345 {
 346         union ionic_dev_cmd cmd = {
 347                 .identify.opcode = IONIC_CMD_IDENTIFY,
 348                 .identify.ver = ver,
 349         };
 350
 351         ionic_dev_cmd_go(idev, &cmd);
 352 }
 353
 354 void ionic_dev_cmd_init(struct ionic_dev *idev)
 355 {
 356         union ionic_dev_cmd cmd = {
 357                 .init.opcode = IONIC_CMD_INIT,
 358                 .init.type = 0,
 359         };
 360
 361         ionic_dev_cmd_go(idev, &cmd);
 362 }
 363
 364 void ionic_dev_cmd_reset(struct ionic_dev *idev)
 365 {
 366         union ionic_dev_cmd cmd = {
 367                 .reset.opcode = IONIC_CMD_RESET,
 368         };
 369
 370         ionic_dev_cmd_go(idev, &cmd);
 371 }
 372
 373 /* Port commands */
 374 void ionic_dev_cmd_port_identify(struct ionic_dev *idev)
 375 {
 376         union ionic_dev_cmd cmd = {
 377                 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY,
 378                 .port_init.index = 0,
 379         };
 380
 381         ionic_dev_cmd_go(idev, &cmd);
 382 }
 383
 384 void ionic_dev_cmd_port_init(struct ionic_dev *idev)
 385 {
 386         union ionic_dev_cmd cmd = {
 387                 .port_init.opcode = IONIC_CMD_PORT_INIT,
 388                 .port_init.index = 0,
 389                 .port_init.info_pa = cpu_to_le64(idev->port_info_pa),
 390         };
 391
 392         ionic_dev_cmd_go(idev, &cmd);
 393 }
 394
 395 void ionic_dev_cmd_port_reset(struct ionic_dev *idev)
 396 {
 397         union ionic_dev_cmd cmd = {
 398                 .port_reset.opcode = IONIC_CMD_PORT_RESET,
 399                 .port_reset.index = 0,
 400         };
 401
 402         ionic_dev_cmd_go(idev, &cmd);
 403 }
 404
 405 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state)
 406 {
 407         union ionic_dev_cmd cmd = {
 408                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 409                 .port_setattr.index = 0,
 410                 .port_setattr.attr = IONIC_PORT_ATTR_STATE,
 411                 .port_setattr.state = state,
 412         };
 413
 414         ionic_dev_cmd_go(idev, &cmd);
 415 }
 416
 417 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed)
 418 {
 419         union ionic_dev_cmd cmd = {
 420                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 421                 .port_setattr.index = 0,
 422                 .port_setattr.attr = IONIC_PORT_ATTR_SPEED,
 423                 .port_setattr.speed = cpu_to_le32(speed),
 424         };
 425
 426         ionic_dev_cmd_go(idev, &cmd);
 427 }
 428
 429 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable)
 430 {
 431         union ionic_dev_cmd cmd = {
 432                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 433                 .port_setattr.index = 0,
 434                 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG,
 435                 .port_setattr.an_enable = an_enable,
 436         };
 437
 438         ionic_dev_cmd_go(idev, &cmd);
 439 }
 440
 441 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type)
 442 {
 443         union ionic_dev_cmd cmd = {
 444                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 445                 .port_setattr.index = 0,
 446                 .port_setattr.attr = IONIC_PORT_ATTR_FEC,
 447                 .port_setattr.fec_type = fec_type,
 448         };
 449
 450         ionic_dev_cmd_go(idev, &cmd);
 451 }
 452
 453 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
 454 {
 455         union ionic_dev_cmd cmd = {
 456                 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR,
 457                 .port_setattr.index = 0,
 458                 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE,
 459                 .port_setattr.pause_type = pause_type,
 460         };
 461
 462         ionic_dev_cmd_go(idev, &cmd);
 463 }
 464
 465 /* VF commands */
 466 int ionic_set_vf_config(struct ionic *ionic, int vf,
 467                         struct ionic_vf_setattr_cmd *vfc)
 468 {
 469         union ionic_dev_cmd cmd = {
 470                 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
 471                 .vf_setattr.attr = vfc->attr,
 472                 .vf_setattr.vf_index = cpu_to_le16(vf),
 473         };
 474         int err;
 475
 476         memcpy(cmd.vf_setattr.pad, vfc->pad, sizeof(vfc->pad));
 477
 478         mutex_lock(&ionic->dev_cmd_lock);
 479         ionic_dev_cmd_go(&ionic->idev, &cmd);
 480         err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
 481         mutex_unlock(&ionic->dev_cmd_lock);
 482
 483         return err;
 484 }
 485
 486 void ionic_vf_start(struct ionic *ionic)
 487 {
 488         union ionic_dev_cmd cmd = {
 489                 .vf_ctrl.opcode = IONIC_CMD_VF_CTRL,
 490                 .vf_ctrl.ctrl_opcode = IONIC_VF_CTRL_START_ALL,
 491         };
 492
 493         if (!(ionic->ident.dev.capabilities & cpu_to_le64(IONIC_DEV_CAP_VF_CTRL)))
 494                 return;
 495
 496         ionic_dev_cmd_go(&ionic->idev, &cmd);
 497         ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
 498 }
 499
 500 /* LIF commands */
 501 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
 502                                   u16 lif_type, u8 qtype, u8 qver)
 503 {
 504         union ionic_dev_cmd cmd = {
 505                 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY,
 506                 .q_identify.lif_type = cpu_to_le16(lif_type),
 507                 .q_identify.type = qtype,
 508                 .q_identify.ver = qver,
 509         };
 510
 511         ionic_dev_cmd_go(idev, &cmd);
 512 }
 513
 514 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver)
 515 {
 516         union ionic_dev_cmd cmd = {
 517                 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY,
 518                 .lif_identify.type = type,
 519                 .lif_identify.ver = ver,
 520         };
 521
 522         ionic_dev_cmd_go(idev, &cmd);
 523 }
 524
 525 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index,
 526                             dma_addr_t info_pa)
 527 {
 528         union ionic_dev_cmd cmd = {
 529                 .lif_init.opcode = IONIC_CMD_LIF_INIT,
 530                 .lif_init.index = cpu_to_le16(lif_index),
 531                 .lif_init.info_pa = cpu_to_le64(info_pa),
 532         };
 533
 534         ionic_dev_cmd_go(idev, &cmd);
 535 }
 536
 537 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index)
 538 {
 539         union ionic_dev_cmd cmd = {
 540                 .lif_init.opcode = IONIC_CMD_LIF_RESET,
 541                 .lif_init.index = cpu_to_le16(lif_index),
 542         };
 543
 544         ionic_dev_cmd_go(idev, &cmd);
 545 }
 546
 547 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq,
 548                                u16 lif_index, u16 intr_index)
 549 {
 550         struct ionic_queue *q = &qcq->q;
 551         struct ionic_cq *cq = &qcq->cq;
 552
 553         union ionic_dev_cmd cmd = {
 554                 .q_init.opcode = IONIC_CMD_Q_INIT,
 555                 .q_init.lif_index = cpu_to_le16(lif_index),
 556                 .q_init.type = q->type,
 557                 .q_init.ver = qcq->q.lif->qtype_info[q->type].version,
 558                 .q_init.index = cpu_to_le32(q->index),
 559                 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 560                                             IONIC_QINIT_F_ENA),
 561                 .q_init.pid = cpu_to_le16(q->pid),
 562                 .q_init.intr_index = cpu_to_le16(intr_index),
 563                 .q_init.ring_size = ilog2(q->num_descs),
 564                 .q_init.ring_base = cpu_to_le64(q->base_pa),
 565                 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa),
 566         };
 567
 568         ionic_dev_cmd_go(idev, &cmd);
 569 }
 570
 571 int ionic_db_page_num(struct ionic_lif *lif, int pid)
 572 {
 573         return (lif->hw_index * lif->dbid_count) + pid;
 574 }
 575
 576 int ionic_get_cmb(struct ionic_lif *lif, u32 *pgid, phys_addr_t *pgaddr, int order)
 577 {
 578         struct ionic_dev *idev = &lif->ionic->idev;
 579         int ret;
 580
 581         mutex_lock(&idev->cmb_inuse_lock);
 582         ret = bitmap_find_free_region(idev->cmb_inuse, idev->cmb_npages, order);
 583         mutex_unlock(&idev->cmb_inuse_lock);
 584
 585         if (ret < 0)
 586                 return ret;
 587
 588         *pgid = ret;
 589         *pgaddr = idev->phy_cmb_pages + ret * PAGE_SIZE;
 590
 591         return 0;
 592 }
 593
 594 void ionic_put_cmb(struct ionic_lif *lif, u32 pgid, int order)
 595 {
 596         struct ionic_dev *idev = &lif->ionic->idev;
 597
 598         mutex_lock(&idev->cmb_inuse_lock);
 599         bitmap_release_region(idev->cmb_inuse, pgid, order);
 600         mutex_unlock(&idev->cmb_inuse_lock);
 601 }
 602
 603 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq,
 604                   struct ionic_intr_info *intr,
 605                   unsigned int num_descs, size_t desc_size)
 606 {
 607         unsigned int ring_size;
 608
 609         if (desc_size == 0 || !is_power_of_2(num_descs))
 610                 return -EINVAL;
 611
 612         ring_size = ilog2(num_descs);
 613         if (ring_size < 2 || ring_size > 16)
 614                 return -EINVAL;
 615
 616         cq->lif = lif;
 617         cq->bound_intr = intr;
 618         cq->num_descs = num_descs;
 619         cq->desc_size = desc_size;
 620         cq->tail_idx = 0;
 621         cq->done_color = 1;
 622
 623         return 0;
 624 }
 625
 626 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa)
 627 {
 628         struct ionic_cq_info *cur;
 629         unsigned int i;
 630
 631         cq->base = base;
 632         cq->base_pa = base_pa;
 633
 634         for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++)
 635                 cur->cq_desc = base + (i * cq->desc_size);
 636 }
 637
 638 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q)
 639 {
 640         cq->bound_q = q;
 641 }
 642
 643 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
 644                               ionic_cq_cb cb, ionic_cq_done_cb done_cb,
 645                               void *done_arg)
 646 {
 647         struct ionic_cq_info *cq_info;
 648         unsigned int work_done = 0;
 649
 650         if (work_to_do == 0)
 651                 return 0;
 652
 653         cq_info = &cq->info[cq->tail_idx];
 654         while (cb(cq, cq_info)) {
 655                 if (cq->tail_idx == cq->num_descs - 1)
 656                         cq->done_color = !cq->done_color;
 657                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
 658                 cq_info = &cq->info[cq->tail_idx];
 659
 660                 if (++work_done >= work_to_do)
 661                         break;
 662         }
 663
 664         if (work_done && done_cb)
 665                 done_cb(done_arg);
 666
 667         return work_done;
 668 }
 669
 670 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
 671                  struct ionic_queue *q, unsigned int index, const char *name,
 672                  unsigned int num_descs, size_t desc_size,
 673                  size_t sg_desc_size, unsigned int pid)
 674 {
 675         unsigned int ring_size;
 676
 677         if (desc_size == 0 || !is_power_of_2(num_descs))
 678                 return -EINVAL;
 679
 680         ring_size = ilog2(num_descs);
 681         if (ring_size < 2 || ring_size > 16)
 682                 return -EINVAL;
 683
 684         q->lif = lif;
 685         q->idev = idev;
 686         q->index = index;
 687         q->num_descs = num_descs;
 688         q->desc_size = desc_size;
 689         q->sg_desc_size = sg_desc_size;
 690         q->tail_idx = 0;
 691         q->head_idx = 0;
 692         q->pid = pid;
 693
 694         snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index);
 695
 696         return 0;
 697 }
 698
 699 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 700 {
 701         struct ionic_desc_info *cur;
 702         unsigned int i;
 703
 704         q->base = base;
 705         q->base_pa = base_pa;
 706
 707         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 708                 cur->desc = base + (i * q->desc_size);
 709 }
 710
 711 void ionic_q_cmb_map(struct ionic_queue *q, void __iomem *base, dma_addr_t base_pa)
 712 {
 713         struct ionic_desc_info *cur;
 714         unsigned int i;
 715
 716         q->cmb_base = base;
 717         q->cmb_base_pa = base_pa;
 718
 719         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 720                 cur->cmb_desc = base + (i * q->desc_size);
 721 }
 722
 723 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa)
 724 {
 725         struct ionic_desc_info *cur;
 726         unsigned int i;
 727
 728         q->sg_base = base;
 729         q->sg_base_pa = base_pa;
 730
 731         for (i = 0, cur = q->info; i < q->num_descs; i++, cur++)
 732                 cur->sg_desc = base + (i * q->sg_desc_size);
 733 }
 734
 735 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
 736                   void *cb_arg)
 737 {
 738         struct ionic_desc_info *desc_info;
 739         struct ionic_lif *lif = q->lif;
 740         struct device *dev = q->dev;
 741
 742         desc_info = &q->info[q->head_idx];
 743         desc_info->cb = cb;
 744         desc_info->cb_arg = cb_arg;
 745
 746         q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 747
 748         dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n",
 749                 q->lif->index, q->name, q->hw_type, q->hw_index,
 750                 q->head_idx, ring_doorbell);
 751
 752         if (ring_doorbell) {
 753                 ionic_dbell_ring(lif->kern_dbpage, q->hw_type,
 754                                  q->dbval | q->head_idx);
 755
 756                 q->dbell_jiffies = jiffies;
 757
 758                 if (q_to_qcq(q)->napi_qcq)
 759                         mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
 760                                   jiffies + IONIC_NAPI_DEADLINE);
 761         }
 762 }
 763
 764 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos)
 765 {
 766         unsigned int mask, tail, head;
 767
 768         mask = q->num_descs - 1;
 769         tail = q->tail_idx;
 770         head = q->head_idx;
 771
 772         return ((pos - tail) & mask) < ((head - tail) & mask);
 773 }
 774
 775 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
 776                      unsigned int stop_index)
 777 {
 778         struct ionic_desc_info *desc_info;
 779         ionic_desc_cb cb;
 780         void *cb_arg;
 781         u16 index;
 782
 783         /* check for empty queue */
 784         if (q->tail_idx == q->head_idx)
 785                 return;
 786
 787         /* stop index must be for a descriptor that is not yet completed */
 788         if (unlikely(!ionic_q_is_posted(q, stop_index)))
 789                 dev_err(q->dev,
 790                         "ionic stop is not posted %s stop %u tail %u head %u\n",
 791                         q->name, stop_index, q->tail_idx, q->head_idx);
 792
 793         do {
 794                 desc_info = &q->info[q->tail_idx];
 795                 index = q->tail_idx;
 796                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 797
 798                 cb = desc_info->cb;
 799                 cb_arg = desc_info->cb_arg;
 800
 801                 desc_info->cb = NULL;
 802                 desc_info->cb_arg = NULL;
 803
 804                 if (cb)
 805                         cb(q, desc_info, cq_info, cb_arg);
 806         } while (index != stop_index);
 807 }