drivers/scsi/be2iscsi/be_main.c

   1 /*
   2  * Copyright 2017 Broadcom. All Rights Reserved.
   3  * The term "Broadcom" refers to Broadcom Limited and/or its subsidiaries.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation. The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@broadcom.com
  12  *
  13  */
  14
  15 #include <linux/reboot.h>
  16 #include <linux/delay.h>
  17 #include <linux/slab.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/blkdev.h>
  20 #include <linux/pci.h>
  21 #include <linux/string.h>
  22 #include <linux/kernel.h>
  23 #include <linux/semaphore.h>
  24 #include <linux/iscsi_boot_sysfs.h>
  25 #include <linux/module.h>
  26 #include <linux/bsg-lib.h>
  27 #include <linux/irq_poll.h>
  28
  29 #include <scsi/libiscsi.h>
  30 #include <scsi/scsi_bsg_iscsi.h>
  31 #include <scsi/scsi_netlink.h>
  32 #include <scsi/scsi_transport_iscsi.h>
  33 #include <scsi/scsi_transport.h>
  34 #include <scsi/scsi_cmnd.h>
  35 #include <scsi/scsi_device.h>
  36 #include <scsi/scsi_host.h>
  37 #include <scsi/scsi.h>
  38 #include "be_main.h"
  39 #include "be_iscsi.h"
  40 #include "be_mgmt.h"
  41 #include "be_cmds.h"
  42
  43 static unsigned int be_iopoll_budget = 10;
  44 static unsigned int be_max_phys_size = 64;
  45 static unsigned int enable_msix = 1;
  46
  47 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  48 MODULE_VERSION(BUILD_STR);
  49 MODULE_AUTHOR("Emulex Corporation");
  50 MODULE_LICENSE("GPL");
  51 module_param(be_iopoll_budget, int, 0);
  52 module_param(enable_msix, int, 0);
  53 module_param(be_max_phys_size, uint, S_IRUGO);
  54 MODULE_PARM_DESC(be_max_phys_size,
  55                 "Maximum Size (In Kilobytes) of physically contiguous "
  56                 "memory that can be allocated. Range is 16 - 128");
  57
  58 #define beiscsi_disp_param(_name)\
  59 static ssize_t  \
  60 beiscsi_##_name##_disp(struct device *dev,\
  61                         struct device_attribute *attrib, char *buf)     \
  62 {       \
  63         struct Scsi_Host *shost = class_to_shost(dev);\
  64         struct beiscsi_hba *phba = iscsi_host_priv(shost); \
  65         return snprintf(buf, PAGE_SIZE, "%d\n",\
  66                         phba->attr_##_name);\
  67 }
  68
  69 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
  70 static int \
  71 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
  72 {\
  73         if (val >= _minval && val <= _maxval) {\
  74                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
  75                             "BA_%d : beiscsi_"#_name" updated "\
  76                             "from 0x%x ==> 0x%x\n",\
  77                             phba->attr_##_name, val); \
  78                 phba->attr_##_name = val;\
  79                 return 0;\
  80         } \
  81         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
  82                     "BA_%d beiscsi_"#_name" attribute "\
  83                     "cannot be updated to 0x%x, "\
  84                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
  85                 return -EINVAL;\
  86 }
  87
  88 #define beiscsi_store_param(_name)  \
  89 static ssize_t \
  90 beiscsi_##_name##_store(struct device *dev,\
  91                          struct device_attribute *attr, const char *buf,\
  92                          size_t count) \
  93 { \
  94         struct Scsi_Host  *shost = class_to_shost(dev);\
  95         struct beiscsi_hba *phba = iscsi_host_priv(shost);\
  96         uint32_t param_val = 0;\
  97         if (!isdigit(buf[0]))\
  98                 return -EINVAL;\
  99         if (sscanf(buf, "%i", &param_val) != 1)\
 100                 return -EINVAL;\
 101         if (beiscsi_##_name##_change(phba, param_val) == 0) \
 102                 return strlen(buf);\
 103         else \
 104                 return -EINVAL;\
 105 }
 106
 107 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
 108 static int \
 109 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
 110 { \
 111         if (val >= _minval && val <= _maxval) {\
 112                 phba->attr_##_name = val;\
 113                 return 0;\
 114         } \
 115         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
 116                     "BA_%d beiscsi_"#_name" attribute " \
 117                     "cannot be updated to 0x%x, "\
 118                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
 119         phba->attr_##_name = _defval;\
 120         return -EINVAL;\
 121 }
 122
 123 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
 124 static uint beiscsi_##_name = _defval;\
 125 module_param(beiscsi_##_name, uint, S_IRUGO);\
 126 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
 127 beiscsi_disp_param(_name)\
 128 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 129 beiscsi_store_param(_name)\
 130 beiscsi_init_param(_name, _minval, _maxval, _defval)\
 131 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 132               beiscsi_##_name##_disp, beiscsi_##_name##_store)
 133
 134 /*
 135  * When new log level added update the
 136  * the MAX allowed value for log_enable
 137  */
 138 BEISCSI_RW_ATTR(log_enable, 0x00,
 139                 0xFF, 0x00, "Enable logging Bit Mask\n"
 140                 "\t\t\t\tInitialization Events  : 0x01\n"
 141                 "\t\t\t\tMailbox Events         : 0x02\n"
 142                 "\t\t\t\tMiscellaneous Events   : 0x04\n"
 143                 "\t\t\t\tError Handling         : 0x08\n"
 144                 "\t\t\t\tIO Path Events         : 0x10\n"
 145                 "\t\t\t\tConfiguration Path     : 0x20\n"
 146                 "\t\t\t\tiSCSI Protocol         : 0x40\n");
 147
 148 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
 149 DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
 150 DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL);
 151 DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL);
 152 DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO,
 153              beiscsi_active_session_disp, NULL);
 154 DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO,
 155              beiscsi_free_session_disp, NULL);
 156 struct device_attribute *beiscsi_attrs[] = {
 157         &dev_attr_beiscsi_log_enable,
 158         &dev_attr_beiscsi_drvr_ver,
 159         &dev_attr_beiscsi_adapter_family,
 160         &dev_attr_beiscsi_fw_ver,
 161         &dev_attr_beiscsi_active_session_count,
 162         &dev_attr_beiscsi_free_session_count,
 163         &dev_attr_beiscsi_phys_port,
 164         NULL,
 165 };
 166
 167 static char const *cqe_desc[] = {
 168         "RESERVED_DESC",
 169         "SOL_CMD_COMPLETE",
 170         "SOL_CMD_KILLED_DATA_DIGEST_ERR",
 171         "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
 172         "CXN_KILLED_BURST_LEN_MISMATCH",
 173         "CXN_KILLED_AHS_RCVD",
 174         "CXN_KILLED_HDR_DIGEST_ERR",
 175         "CXN_KILLED_UNKNOWN_HDR",
 176         "CXN_KILLED_STALE_ITT_TTT_RCVD",
 177         "CXN_KILLED_INVALID_ITT_TTT_RCVD",
 178         "CXN_KILLED_RST_RCVD",
 179         "CXN_KILLED_TIMED_OUT",
 180         "CXN_KILLED_RST_SENT",
 181         "CXN_KILLED_FIN_RCVD",
 182         "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
 183         "CXN_KILLED_BAD_WRB_INDEX_ERROR",
 184         "CXN_KILLED_OVER_RUN_RESIDUAL",
 185         "CXN_KILLED_UNDER_RUN_RESIDUAL",
 186         "CMD_KILLED_INVALID_STATSN_RCVD",
 187         "CMD_KILLED_INVALID_R2T_RCVD",
 188         "CMD_CXN_KILLED_LUN_INVALID",
 189         "CMD_CXN_KILLED_ICD_INVALID",
 190         "CMD_CXN_KILLED_ITT_INVALID",
 191         "CMD_CXN_KILLED_SEQ_OUTOFORDER",
 192         "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
 193         "CXN_INVALIDATE_NOTIFY",
 194         "CXN_INVALIDATE_INDEX_NOTIFY",
 195         "CMD_INVALIDATED_NOTIFY",
 196         "UNSOL_HDR_NOTIFY",
 197         "UNSOL_DATA_NOTIFY",
 198         "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
 199         "DRIVERMSG_NOTIFY",
 200         "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
 201         "SOL_CMD_KILLED_DIF_ERR",
 202         "CXN_KILLED_SYN_RCVD",
 203         "CXN_KILLED_IMM_DATA_RCVD"
 204 };
 205
 206 static int beiscsi_slave_configure(struct scsi_device *sdev)
 207 {
 208         blk_queue_max_segment_size(sdev->request_queue, 65536);
 209         return 0;
 210 }
 211
 212 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 213 {
 214         struct iscsi_task *abrt_task = (struct iscsi_task *)sc->SCp.ptr;
 215         struct iscsi_cls_session *cls_session;
 216         struct beiscsi_io_task *abrt_io_task;
 217         struct beiscsi_conn *beiscsi_conn;
 218         struct iscsi_session *session;
 219         struct invldt_cmd_tbl inv_tbl;
 220         struct beiscsi_hba *phba;
 221         struct iscsi_conn *conn;
 222         int rc;
 223
 224         cls_session = starget_to_session(scsi_target(sc->device));
 225         session = cls_session->dd_data;
 226
 227         /* check if we raced, task just got cleaned up under us */
 228         spin_lock_bh(&session->back_lock);
 229         if (!abrt_task || !abrt_task->sc) {
 230                 spin_unlock_bh(&session->back_lock);
 231                 return SUCCESS;
 232         }
 233         /* get a task ref till FW processes the req for the ICD used */
 234         __iscsi_get_task(abrt_task);
 235         abrt_io_task = abrt_task->dd_data;
 236         conn = abrt_task->conn;
 237         beiscsi_conn = conn->dd_data;
 238         phba = beiscsi_conn->phba;
 239         /* mark WRB invalid which have been not processed by FW yet */
 240         if (is_chip_be2_be3r(phba)) {
 241                 AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
 242                               abrt_io_task->pwrb_handle->pwrb, 1);
 243         } else {
 244                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, invld,
 245                               abrt_io_task->pwrb_handle->pwrb, 1);
 246         }
 247         inv_tbl.cid = beiscsi_conn->beiscsi_conn_cid;
 248         inv_tbl.icd = abrt_io_task->psgl_handle->sgl_index;
 249         spin_unlock_bh(&session->back_lock);
 250
 251         rc = beiscsi_mgmt_invalidate_icds(phba, &inv_tbl, 1);
 252         iscsi_put_task(abrt_task);
 253         if (rc) {
 254                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 255                             "BM_%d : sc %p invalidation failed %d\n",
 256                             sc, rc);
 257                 return FAILED;
 258         }
 259
 260         return iscsi_eh_abort(sc);
 261 }
 262
 263 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 264 {
 265         struct beiscsi_invldt_cmd_tbl {
 266                 struct invldt_cmd_tbl tbl[BE_INVLDT_CMD_TBL_SZ];
 267                 struct iscsi_task *task[BE_INVLDT_CMD_TBL_SZ];
 268         } *inv_tbl;
 269         struct iscsi_cls_session *cls_session;
 270         struct beiscsi_conn *beiscsi_conn;
 271         struct beiscsi_io_task *io_task;
 272         struct iscsi_session *session;
 273         struct beiscsi_hba *phba;
 274         struct iscsi_conn *conn;
 275         struct iscsi_task *task;
 276         unsigned int i, nents;
 277         int rc, more = 0;
 278
 279         cls_session = starget_to_session(scsi_target(sc->device));
 280         session = cls_session->dd_data;
 281
 282         spin_lock_bh(&session->frwd_lock);
 283         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 284                 spin_unlock_bh(&session->frwd_lock);
 285                 return FAILED;
 286         }
 287
 288         conn = session->leadconn;
 289         beiscsi_conn = conn->dd_data;
 290         phba = beiscsi_conn->phba;
 291
 292         inv_tbl = kzalloc(sizeof(*inv_tbl), GFP_ATOMIC);
 293         if (!inv_tbl) {
 294                 spin_unlock_bh(&session->frwd_lock);
 295                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 296                             "BM_%d : invldt_cmd_tbl alloc failed\n");
 297                 return FAILED;
 298         }
 299         nents = 0;
 300         /* take back_lock to prevent task from getting cleaned up under us */
 301         spin_lock(&session->back_lock);
 302         for (i = 0; i < conn->session->cmds_max; i++) {
 303                 task = conn->session->cmds[i];
 304                 if (!task->sc)
 305                         continue;
 306
 307                 if (sc->device->lun != task->sc->device->lun)
 308                         continue;
 309                 /**
 310                  * Can't fit in more cmds? Normally this won't happen b'coz
 311                  * BEISCSI_CMD_PER_LUN is same as BE_INVLDT_CMD_TBL_SZ.
 312                  */
 313                 if (nents == BE_INVLDT_CMD_TBL_SZ) {
 314                         more = 1;
 315                         break;
 316                 }
 317
 318                 /* get a task ref till FW processes the req for the ICD used */
 319                 __iscsi_get_task(task);
 320                 io_task = task->dd_data;
 321                 /* mark WRB invalid which have been not processed by FW yet */
 322                 if (is_chip_be2_be3r(phba)) {
 323                         AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
 324                                       io_task->pwrb_handle->pwrb, 1);
 325                 } else {
 326                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, invld,
 327                                       io_task->pwrb_handle->pwrb, 1);
 328                 }
 329
 330                 inv_tbl->tbl[nents].cid = beiscsi_conn->beiscsi_conn_cid;
 331                 inv_tbl->tbl[nents].icd = io_task->psgl_handle->sgl_index;
 332                 inv_tbl->task[nents] = task;
 333                 nents++;
 334         }
 335         spin_unlock(&session->back_lock);
 336         spin_unlock_bh(&session->frwd_lock);
 337
 338         rc = SUCCESS;
 339         if (!nents)
 340                 goto end_reset;
 341
 342         if (more) {
 343                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 344                             "BM_%d : number of cmds exceeds size of invalidation table\n");
 345                 rc = FAILED;
 346                 goto end_reset;
 347         }
 348
 349         if (beiscsi_mgmt_invalidate_icds(phba, &inv_tbl->tbl[0], nents)) {
 350                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 351                             "BM_%d : cid %u scmds invalidation failed\n",
 352                             beiscsi_conn->beiscsi_conn_cid);
 353                 rc = FAILED;
 354         }
 355
 356 end_reset:
 357         for (i = 0; i < nents; i++)
 358                 iscsi_put_task(inv_tbl->task[i]);
 359         kfree(inv_tbl);
 360
 361         if (rc == SUCCESS)
 362                 rc = iscsi_eh_device_reset(sc);
 363         return rc;
 364 }
 365
 366 /*------------------- PCI Driver operations and data ----------------- */
 367 static const struct pci_device_id beiscsi_pci_id_table[] = {
 368         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 369         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 370         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 371         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 372         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 373         { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
 374         { 0 }
 375 };
 376 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 377
 378
 379 static struct scsi_host_template beiscsi_sht = {
 380         .module = THIS_MODULE,
 381         .name = "Emulex 10Gbe open-iscsi Initiator Driver",
 382         .proc_name = DRV_NAME,
 383         .queuecommand = iscsi_queuecommand,
 384         .change_queue_depth = scsi_change_queue_depth,
 385         .slave_configure = beiscsi_slave_configure,
 386         .target_alloc = iscsi_target_alloc,
 387         .eh_timed_out = iscsi_eh_cmd_timed_out,
 388         .eh_abort_handler = beiscsi_eh_abort,
 389         .eh_device_reset_handler = beiscsi_eh_device_reset,
 390         .eh_target_reset_handler = iscsi_eh_session_reset,
 391         .shost_attrs = beiscsi_attrs,
 392         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 393         .can_queue = BE2_IO_DEPTH,
 394         .this_id = -1,
 395         .max_sectors = BEISCSI_MAX_SECTORS,
 396         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 397         .use_clustering = ENABLE_CLUSTERING,
 398         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 399         .track_queue_depth = 1,
 400 };
 401
 402 static struct scsi_transport_template *beiscsi_scsi_transport;
 403
 404 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 405 {
 406         struct beiscsi_hba *phba;
 407         struct Scsi_Host *shost;
 408
 409         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 410         if (!shost) {
 411                 dev_err(&pcidev->dev,
 412                         "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 413                 return NULL;
 414         }
 415         shost->max_id = BE2_MAX_SESSIONS;
 416         shost->max_channel = 0;
 417         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 418         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 419         shost->transportt = beiscsi_scsi_transport;
 420         phba = iscsi_host_priv(shost);
 421         memset(phba, 0, sizeof(*phba));
 422         phba->shost = shost;
 423         phba->pcidev = pci_dev_get(pcidev);
 424         pci_set_drvdata(pcidev, phba);
 425         phba->interface_handle = 0xFFFFFFFF;
 426
 427         return phba;
 428 }
 429
 430 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 431 {
 432         if (phba->csr_va) {
 433                 iounmap(phba->csr_va);
 434                 phba->csr_va = NULL;
 435         }
 436         if (phba->db_va) {
 437                 iounmap(phba->db_va);
 438                 phba->db_va = NULL;
 439         }
 440         if (phba->pci_va) {
 441                 iounmap(phba->pci_va);
 442                 phba->pci_va = NULL;
 443         }
 444 }
 445
 446 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 447                                 struct pci_dev *pcidev)
 448 {
 449         u8 __iomem *addr;
 450         int pcicfg_reg;
 451
 452         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 453                                pci_resource_len(pcidev, 2));
 454         if (addr == NULL)
 455                 return -ENOMEM;
 456         phba->ctrl.csr = addr;
 457         phba->csr_va = addr;
 458
 459         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 460         if (addr == NULL)
 461                 goto pci_map_err;
 462         phba->ctrl.db = addr;
 463         phba->db_va = addr;
 464
 465         if (phba->generation == BE_GEN2)
 466                 pcicfg_reg = 1;
 467         else
 468                 pcicfg_reg = 0;
 469
 470         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 471                                pci_resource_len(pcidev, pcicfg_reg));
 472
 473         if (addr == NULL)
 474                 goto pci_map_err;
 475         phba->ctrl.pcicfg = addr;
 476         phba->pci_va = addr;
 477         return 0;
 478
 479 pci_map_err:
 480         beiscsi_unmap_pci_function(phba);
 481         return -ENOMEM;
 482 }
 483
 484 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 485 {
 486         int ret;
 487
 488         ret = pci_enable_device(pcidev);
 489         if (ret) {
 490                 dev_err(&pcidev->dev,
 491                         "beiscsi_enable_pci - enable device failed\n");
 492                 return ret;
 493         }
 494
 495         ret = pci_request_regions(pcidev, DRV_NAME);
 496         if (ret) {
 497                 dev_err(&pcidev->dev,
 498                                 "beiscsi_enable_pci - request region failed\n");
 499                 goto pci_dev_disable;
 500         }
 501
 502         pci_set_master(pcidev);
 503         ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64));
 504         if (ret) {
 505                 ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32));
 506                 if (ret) {
 507                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 508                         goto pci_region_release;
 509                 } else {
 510                         ret = pci_set_consistent_dma_mask(pcidev,
 511                                                           DMA_BIT_MASK(32));
 512                 }
 513         } else {
 514                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64));
 515                 if (ret) {
 516                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 517                         goto pci_region_release;
 518                 }
 519         }
 520         return 0;
 521
 522 pci_region_release:
 523         pci_release_regions(pcidev);
 524 pci_dev_disable:
 525         pci_disable_device(pcidev);
 526
 527         return ret;
 528 }
 529
 530 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 531 {
 532         struct be_ctrl_info *ctrl = &phba->ctrl;
 533         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 534         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 535         int status = 0;
 536
 537         ctrl->pdev = pdev;
 538         status = beiscsi_map_pci_bars(phba, pdev);
 539         if (status)
 540                 return status;
 541         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 542         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 543                                                   mbox_mem_alloc->size,
 544                                                   &mbox_mem_alloc->dma);
 545         if (!mbox_mem_alloc->va) {
 546                 beiscsi_unmap_pci_function(phba);
 547                 return -ENOMEM;
 548         }
 549
 550         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 551         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 552         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 553         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 554         mutex_init(&ctrl->mbox_lock);
 555         spin_lock_init(&phba->ctrl.mcc_lock);
 556
 557         return status;
 558 }
 559
 560 /**
 561  * beiscsi_get_params()- Set the config paramters
 562  * @phba: ptr  device priv structure
 563  **/
 564 static void beiscsi_get_params(struct beiscsi_hba *phba)
 565 {
 566         uint32_t total_cid_count = 0;
 567         uint32_t total_icd_count = 0;
 568         uint8_t ulp_num = 0;
 569
 570         total_cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) +
 571                           BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1);
 572
 573         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
 574                 uint32_t align_mask = 0;
 575                 uint32_t icd_post_per_page = 0;
 576                 uint32_t icd_count_unavailable = 0;
 577                 uint32_t icd_start = 0, icd_count = 0;
 578                 uint32_t icd_start_align = 0, icd_count_align = 0;
 579
 580                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
 581                         icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
 582                         icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
 583
 584                         /* Get ICD count that can be posted on each page */
 585                         icd_post_per_page = (PAGE_SIZE / (BE2_SGE *
 586                                              sizeof(struct iscsi_sge)));
 587                         align_mask = (icd_post_per_page - 1);
 588
 589                         /* Check if icd_start is aligned ICD per page posting */
 590                         if (icd_start % icd_post_per_page) {
 591                                 icd_start_align = ((icd_start +
 592                                                     icd_post_per_page) &
 593                                                     ~(align_mask));
 594                                 phba->fw_config.
 595                                         iscsi_icd_start[ulp_num] =
 596                                         icd_start_align;
 597                         }
 598
 599                         icd_count_align = (icd_count & ~align_mask);
 600
 601                         /* ICD discarded in the process of alignment */
 602                         if (icd_start_align)
 603                                 icd_count_unavailable = ((icd_start_align -
 604                                                           icd_start) +
 605                                                          (icd_count -
 606                                                           icd_count_align));
 607
 608                         /* Updated ICD count available */
 609                         phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count -
 610                                         icd_count_unavailable);
 611
 612                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 613                                         "BM_%d : Aligned ICD values\n"
 614                                         "\t ICD Start : %d\n"
 615                                         "\t ICD Count : %d\n"
 616                                         "\t ICD Discarded : %d\n",
 617                                         phba->fw_config.
 618                                         iscsi_icd_start[ulp_num],
 619                                         phba->fw_config.
 620                                         iscsi_icd_count[ulp_num],
 621                                         icd_count_unavailable);
 622                         break;
 623                 }
 624         }
 625
 626         total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
 627         phba->params.ios_per_ctrl = (total_icd_count -
 628                                     (total_cid_count +
 629                                      BE2_TMFS + BE2_NOPOUT_REQ));
 630         phba->params.cxns_per_ctrl = total_cid_count;
 631         phba->params.icds_per_ctrl = total_icd_count;
 632         phba->params.num_sge_per_io = BE2_SGE;
 633         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 634         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 635         phba->params.num_eq_entries = 1024;
 636         phba->params.num_cq_entries = 1024;
 637         phba->params.wrbs_per_cxn = 256;
 638 }
 639
 640 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 641                            unsigned int id, unsigned int clr_interrupt,
 642                            unsigned int num_processed,
 643                            unsigned char rearm, unsigned char event)
 644 {
 645         u32 val = 0;
 646
 647         if (rearm)
 648                 val |= 1 << DB_EQ_REARM_SHIFT;
 649         if (clr_interrupt)
 650                 val |= 1 << DB_EQ_CLR_SHIFT;
 651         if (event)
 652                 val |= 1 << DB_EQ_EVNT_SHIFT;
 653
 654         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 655         /* Setting lower order EQ_ID Bits */
 656         val |= (id & DB_EQ_RING_ID_LOW_MASK);
 657
 658         /* Setting Higher order EQ_ID Bits */
 659         val |= (((id >> DB_EQ_HIGH_FEILD_SHIFT) &
 660                   DB_EQ_RING_ID_HIGH_MASK)
 661                   << DB_EQ_HIGH_SET_SHIFT);
 662
 663         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 664 }
 665
 666 /**
 667  * be_isr_mcc - The isr routine of the driver.
 668  * @irq: Not used
 669  * @dev_id: Pointer to host adapter structure
 670  */
 671 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 672 {
 673         struct beiscsi_hba *phba;
 674         struct be_eq_entry *eqe;
 675         struct be_queue_info *eq;
 676         struct be_queue_info *mcc;
 677         unsigned int mcc_events;
 678         struct be_eq_obj *pbe_eq;
 679
 680         pbe_eq = dev_id;
 681         eq = &pbe_eq->q;
 682         phba =  pbe_eq->phba;
 683         mcc = &phba->ctrl.mcc_obj.cq;
 684         eqe = queue_tail_node(eq);
 685
 686         mcc_events = 0;
 687         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 688                                 & EQE_VALID_MASK) {
 689                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 690                      resource_id) / 32] &
 691                      EQE_RESID_MASK) >> 16) == mcc->id) {
 692                         mcc_events++;
 693                 }
 694                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 695                 queue_tail_inc(eq);
 696                 eqe = queue_tail_node(eq);
 697         }
 698
 699         if (mcc_events) {
 700                 queue_work(phba->wq, &pbe_eq->mcc_work);
 701                 hwi_ring_eq_db(phba, eq->id, 1, mcc_events, 1, 1);
 702         }
 703         return IRQ_HANDLED;
 704 }
 705
 706 /**
 707  * be_isr_msix - The isr routine of the driver.
 708  * @irq: Not used
 709  * @dev_id: Pointer to host adapter structure
 710  */
 711 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 712 {
 713         struct beiscsi_hba *phba;
 714         struct be_queue_info *eq;
 715         struct be_eq_obj *pbe_eq;
 716
 717         pbe_eq = dev_id;
 718         eq = &pbe_eq->q;
 719
 720         phba = pbe_eq->phba;
 721         /* disable interrupt till iopoll completes */
 722         hwi_ring_eq_db(phba, eq->id, 1, 0, 0, 1);
 723         irq_poll_sched(&pbe_eq->iopoll);
 724
 725         return IRQ_HANDLED;
 726 }
 727
 728 /**
 729  * be_isr - The isr routine of the driver.
 730  * @irq: Not used
 731  * @dev_id: Pointer to host adapter structure
 732  */
 733 static irqreturn_t be_isr(int irq, void *dev_id)
 734 {
 735         struct beiscsi_hba *phba;
 736         struct hwi_controller *phwi_ctrlr;
 737         struct hwi_context_memory *phwi_context;
 738         struct be_eq_entry *eqe;
 739         struct be_queue_info *eq;
 740         struct be_queue_info *mcc;
 741         unsigned int mcc_events, io_events;
 742         struct be_ctrl_info *ctrl;
 743         struct be_eq_obj *pbe_eq;
 744         int isr, rearm;
 745
 746         phba = dev_id;
 747         ctrl = &phba->ctrl;
 748         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 749                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 750         if (!isr)
 751                 return IRQ_NONE;
 752
 753         phwi_ctrlr = phba->phwi_ctrlr;
 754         phwi_context = phwi_ctrlr->phwi_ctxt;
 755         pbe_eq = &phwi_context->be_eq[0];
 756
 757         eq = &phwi_context->be_eq[0].q;
 758         mcc = &phba->ctrl.mcc_obj.cq;
 759         eqe = queue_tail_node(eq);
 760
 761         io_events = 0;
 762         mcc_events = 0;
 763         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 764                                 & EQE_VALID_MASK) {
 765                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 766                       resource_id) / 32] & EQE_RESID_MASK) >> 16) == mcc->id)
 767                         mcc_events++;
 768                 else
 769                         io_events++;
 770                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 771                 queue_tail_inc(eq);
 772                 eqe = queue_tail_node(eq);
 773         }
 774         if (!io_events && !mcc_events)
 775                 return IRQ_NONE;
 776
 777         /* no need to rearm if interrupt is only for IOs */
 778         rearm = 0;
 779         if (mcc_events) {
 780                 queue_work(phba->wq, &pbe_eq->mcc_work);
 781                 /* rearm for MCCQ */
 782                 rearm = 1;
 783         }
 784         if (io_events)
 785                 irq_poll_sched(&pbe_eq->iopoll);
 786         hwi_ring_eq_db(phba, eq->id, 0, (io_events + mcc_events), rearm, 1);
 787         return IRQ_HANDLED;
 788 }
 789
 790 static void beiscsi_free_irqs(struct beiscsi_hba *phba)
 791 {
 792         struct hwi_context_memory *phwi_context;
 793         int i;
 794
 795         if (!phba->pcidev->msix_enabled) {
 796                 if (phba->pcidev->irq)
 797                         free_irq(phba->pcidev->irq, phba);
 798                 return;
 799         }
 800
 801         phwi_context = phba->phwi_ctrlr->phwi_ctxt;
 802         for (i = 0; i <= phba->num_cpus; i++) {
 803                 free_irq(pci_irq_vector(phba->pcidev, i),
 804                          &phwi_context->be_eq[i]);
 805                 kfree(phba->msi_name[i]);
 806         }
 807 }
 808
 809 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 810 {
 811         struct pci_dev *pcidev = phba->pcidev;
 812         struct hwi_controller *phwi_ctrlr;
 813         struct hwi_context_memory *phwi_context;
 814         int ret, i, j;
 815
 816         phwi_ctrlr = phba->phwi_ctrlr;
 817         phwi_context = phwi_ctrlr->phwi_ctxt;
 818
 819         if (pcidev->msix_enabled) {
 820                 for (i = 0; i < phba->num_cpus; i++) {
 821                         phba->msi_name[i] = kasprintf(GFP_KERNEL,
 822                                                       "beiscsi_%02x_%02x",
 823                                                       phba->shost->host_no, i);
 824                         if (!phba->msi_name[i]) {
 825                                 ret = -ENOMEM;
 826                                 goto free_msix_irqs;
 827                         }
 828
 829                         ret = request_irq(pci_irq_vector(pcidev, i),
 830                                           be_isr_msix, 0, phba->msi_name[i],
 831                                           &phwi_context->be_eq[i]);
 832                         if (ret) {
 833                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 834                                             "BM_%d : beiscsi_init_irqs-Failed to"
 835                                             "register msix for i = %d\n",
 836                                             i);
 837                                 kfree(phba->msi_name[i]);
 838                                 goto free_msix_irqs;
 839                         }
 840                 }
 841                 phba->msi_name[i] = kasprintf(GFP_KERNEL, "beiscsi_mcc_%02x",
 842                                               phba->shost->host_no);
 843                 if (!phba->msi_name[i]) {
 844                         ret = -ENOMEM;
 845                         goto free_msix_irqs;
 846                 }
 847                 ret = request_irq(pci_irq_vector(pcidev, i), be_isr_mcc, 0,
 848                                   phba->msi_name[i], &phwi_context->be_eq[i]);
 849                 if (ret) {
 850                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
 851                                     "BM_%d : beiscsi_init_irqs-"
 852                                     "Failed to register beiscsi_msix_mcc\n");
 853                         kfree(phba->msi_name[i]);
 854                         goto free_msix_irqs;
 855                 }
 856
 857         } else {
 858                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
 859                                   "beiscsi", phba);
 860                 if (ret) {
 861                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
 862                                     "BM_%d : beiscsi_init_irqs-"
 863                                     "Failed to register irq\\n");
 864                         return ret;
 865                 }
 866         }
 867         return 0;
 868 free_msix_irqs:
 869         for (j = i - 1; j >= 0; j--) {
 870                 free_irq(pci_irq_vector(pcidev, i), &phwi_context->be_eq[j]);
 871                 kfree(phba->msi_name[j]);
 872         }
 873         return ret;
 874 }
 875
 876 void hwi_ring_cq_db(struct beiscsi_hba *phba,
 877                            unsigned int id, unsigned int num_processed,
 878                            unsigned char rearm)
 879 {
 880         u32 val = 0;
 881
 882         if (rearm)
 883                 val |= 1 << DB_CQ_REARM_SHIFT;
 884
 885         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
 886
 887         /* Setting lower order CQ_ID Bits */
 888         val |= (id & DB_CQ_RING_ID_LOW_MASK);
 889
 890         /* Setting Higher order CQ_ID Bits */
 891         val |= (((id >> DB_CQ_HIGH_FEILD_SHIFT) &
 892                   DB_CQ_RING_ID_HIGH_MASK)
 893                   << DB_CQ_HIGH_SET_SHIFT);
 894
 895         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
 896 }
 897
 898 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
 899 {
 900         struct sgl_handle *psgl_handle;
 901         unsigned long flags;
 902
 903         spin_lock_irqsave(&phba->io_sgl_lock, flags);
 904         if (phba->io_sgl_hndl_avbl) {
 905                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
 906                             "BM_%d : In alloc_io_sgl_handle,"
 907                             " io_sgl_alloc_index=%d\n",
 908                             phba->io_sgl_alloc_index);
 909
 910                 psgl_handle = phba->io_sgl_hndl_base[phba->
 911                                                 io_sgl_alloc_index];
 912                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
 913                 phba->io_sgl_hndl_avbl--;
 914                 if (phba->io_sgl_alloc_index == (phba->params.
 915                                                  ios_per_ctrl - 1))
 916                         phba->io_sgl_alloc_index = 0;
 917                 else
 918                         phba->io_sgl_alloc_index++;
 919         } else
 920                 psgl_handle = NULL;
 921         spin_unlock_irqrestore(&phba->io_sgl_lock, flags);
 922         return psgl_handle;
 923 }
 924
 925 static void
 926 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
 927 {
 928         unsigned long flags;
 929
 930         spin_lock_irqsave(&phba->io_sgl_lock, flags);
 931         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
 932                     "BM_%d : In free_,io_sgl_free_index=%d\n",
 933                     phba->io_sgl_free_index);
 934
 935         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
 936                 /*
 937                  * this can happen if clean_task is called on a task that
 938                  * failed in xmit_task or alloc_pdu.
 939                  */
 940                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
 941                             "BM_%d : Double Free in IO SGL io_sgl_free_index=%d, value there=%p\n",
 942                             phba->io_sgl_free_index,
 943                             phba->io_sgl_hndl_base[phba->io_sgl_free_index]);
 944                 spin_unlock_irqrestore(&phba->io_sgl_lock, flags);
 945                 return;
 946         }
 947         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
 948         phba->io_sgl_hndl_avbl++;
 949         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
 950                 phba->io_sgl_free_index = 0;
 951         else
 952                 phba->io_sgl_free_index++;
 953         spin_unlock_irqrestore(&phba->io_sgl_lock, flags);
 954 }
 955
 956 static inline struct wrb_handle *
 957 beiscsi_get_wrb_handle(struct hwi_wrb_context *pwrb_context,
 958                        unsigned int wrbs_per_cxn)
 959 {
 960         struct wrb_handle *pwrb_handle;
 961         unsigned long flags;
 962
 963         spin_lock_irqsave(&pwrb_context->wrb_lock, flags);
 964         if (!pwrb_context->wrb_handles_available) {
 965                 spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);
 966                 return NULL;
 967         }
 968         pwrb_handle = pwrb_context->pwrb_handle_base[pwrb_context->alloc_index];
 969         pwrb_context->wrb_handles_available--;
 970         if (pwrb_context->alloc_index == (wrbs_per_cxn - 1))
 971                 pwrb_context->alloc_index = 0;
 972         else
 973                 pwrb_context->alloc_index++;
 974         spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);
 975
 976         if (pwrb_handle)
 977                 memset(pwrb_handle->pwrb, 0, sizeof(*pwrb_handle->pwrb));
 978
 979         return pwrb_handle;
 980 }
 981
 982 /**
 983  * alloc_wrb_handle - To allocate a wrb handle
 984  * @phba: The hba pointer
 985  * @cid: The cid to use for allocation
 986  * @pwrb_context: ptr to ptr to wrb context
 987  *
 988  * This happens under session_lock until submission to chip
 989  */
 990 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid,
 991                                     struct hwi_wrb_context **pcontext)
 992 {
 993         struct hwi_wrb_context *pwrb_context;
 994         struct hwi_controller *phwi_ctrlr;
 995         uint16_t cri_index = BE_GET_CRI_FROM_CID(cid);
 996
 997         phwi_ctrlr = phba->phwi_ctrlr;
 998         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
 999         /* return the context address */
1000         *pcontext = pwrb_context;
1001         return beiscsi_get_wrb_handle(pwrb_context, phba->params.wrbs_per_cxn);
1002 }
1003
1004 static inline void
1005 beiscsi_put_wrb_handle(struct hwi_wrb_context *pwrb_context,
1006                        struct wrb_handle *pwrb_handle,
1007                        unsigned int wrbs_per_cxn)
1008 {
1009         unsigned long flags;
1010
1011         spin_lock_irqsave(&pwrb_context->wrb_lock, flags);
1012         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1013         pwrb_context->wrb_handles_available++;
1014         if (pwrb_context->free_index == (wrbs_per_cxn - 1))
1015                 pwrb_context->free_index = 0;
1016         else
1017                 pwrb_context->free_index++;
1018         pwrb_handle->pio_handle = NULL;
1019         spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags);
1020 }
1021
1022 /**
1023  * free_wrb_handle - To free the wrb handle back to pool
1024  * @phba: The hba pointer
1025  * @pwrb_context: The context to free from
1026  * @pwrb_handle: The wrb_handle to free
1027  *
1028  * This happens under session_lock until submission to chip
1029  */
1030 static void
1031 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1032                 struct wrb_handle *pwrb_handle)
1033 {
1034         beiscsi_put_wrb_handle(pwrb_context,
1035                                pwrb_handle,
1036                                phba->params.wrbs_per_cxn);
1037         beiscsi_log(phba, KERN_INFO,
1038                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1039                     "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1040                     "wrb_handles_available=%d\n",
1041                     pwrb_handle, pwrb_context->free_index,
1042                     pwrb_context->wrb_handles_available);
1043 }
1044
1045 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1046 {
1047         struct sgl_handle *psgl_handle;
1048         unsigned long flags;
1049
1050         spin_lock_irqsave(&phba->mgmt_sgl_lock, flags);
1051         if (phba->eh_sgl_hndl_avbl) {
1052                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1053                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1054                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1055                             "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1056                             phba->eh_sgl_alloc_index,
1057                             phba->eh_sgl_alloc_index);
1058
1059                 phba->eh_sgl_hndl_avbl--;
1060                 if (phba->eh_sgl_alloc_index ==
1061                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1062                      1))
1063                         phba->eh_sgl_alloc_index = 0;
1064                 else
1065                         phba->eh_sgl_alloc_index++;
1066         } else
1067                 psgl_handle = NULL;
1068         spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);
1069         return psgl_handle;
1070 }
1071
1072 void
1073 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1074 {
1075         unsigned long flags;
1076
1077         spin_lock_irqsave(&phba->mgmt_sgl_lock, flags);
1078         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1079                     "BM_%d : In  free_mgmt_sgl_handle,"
1080                     "eh_sgl_free_index=%d\n",
1081                     phba->eh_sgl_free_index);
1082
1083         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1084                 /*
1085                  * this can happen if clean_task is called on a task that
1086                  * failed in xmit_task or alloc_pdu.
1087                  */
1088                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1089                             "BM_%d : Double Free in eh SGL ,"
1090                             "eh_sgl_free_index=%d\n",
1091                             phba->eh_sgl_free_index);
1092                 spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);
1093                 return;
1094         }
1095         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1096         phba->eh_sgl_hndl_avbl++;
1097         if (phba->eh_sgl_free_index ==
1098             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1099                 phba->eh_sgl_free_index = 0;
1100         else
1101                 phba->eh_sgl_free_index++;
1102         spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags);
1103 }
1104
1105 static void
1106 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1107                 struct iscsi_task *task,
1108                 struct common_sol_cqe *csol_cqe)
1109 {
1110         struct beiscsi_io_task *io_task = task->dd_data;
1111         struct be_status_bhs *sts_bhs =
1112                                 (struct be_status_bhs *)io_task->cmd_bhs;
1113         struct iscsi_conn *conn = beiscsi_conn->conn;
1114         unsigned char *sense;
1115         u32 resid = 0, exp_cmdsn, max_cmdsn;
1116         u8 rsp, status, flags;
1117
1118         exp_cmdsn = csol_cqe->exp_cmdsn;
1119         max_cmdsn = (csol_cqe->exp_cmdsn +
1120                      csol_cqe->cmd_wnd - 1);
1121         rsp = csol_cqe->i_resp;
1122         status = csol_cqe->i_sts;
1123         flags = csol_cqe->i_flags;
1124         resid = csol_cqe->res_cnt;
1125
1126         if (!task->sc) {
1127                 if (io_task->scsi_cmnd) {
1128                         scsi_dma_unmap(io_task->scsi_cmnd);
1129                         io_task->scsi_cmnd = NULL;
1130                 }
1131
1132                 return;
1133         }
1134         task->sc->result = (DID_OK << 16) | status;
1135         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1136                 task->sc->result = DID_ERROR << 16;
1137                 goto unmap;
1138         }
1139
1140         /* bidi not initially supported */
1141         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1142                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1143                         task->sc->result = DID_ERROR << 16;
1144
1145                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1146                         scsi_set_resid(task->sc, resid);
1147                         if (!status && (scsi_bufflen(task->sc) - resid <
1148                             task->sc->underflow))
1149                                 task->sc->result = DID_ERROR << 16;
1150                 }
1151         }
1152
1153         if (status == SAM_STAT_CHECK_CONDITION) {
1154                 u16 sense_len;
1155                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1156
1157                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1158                 sense_len = be16_to_cpu(*slen);
1159                 memcpy(task->sc->sense_buffer, sense,
1160                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1161         }
1162
1163         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1164                 conn->rxdata_octets += resid;
1165 unmap:
1166         if (io_task->scsi_cmnd) {
1167                 scsi_dma_unmap(io_task->scsi_cmnd);
1168                 io_task->scsi_cmnd = NULL;
1169         }
1170         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1171 }
1172
1173 static void
1174 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1175                     struct iscsi_task *task,
1176                     struct common_sol_cqe *csol_cqe)
1177 {
1178         struct iscsi_logout_rsp *hdr;
1179         struct beiscsi_io_task *io_task = task->dd_data;
1180         struct iscsi_conn *conn = beiscsi_conn->conn;
1181
1182         hdr = (struct iscsi_logout_rsp *)task->hdr;
1183         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1184         hdr->t2wait = 5;
1185         hdr->t2retain = 0;
1186         hdr->flags = csol_cqe->i_flags;
1187         hdr->response = csol_cqe->i_resp;
1188         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1189         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1190                                      csol_cqe->cmd_wnd - 1);
1191
1192         hdr->dlength[0] = 0;
1193         hdr->dlength[1] = 0;
1194         hdr->dlength[2] = 0;
1195         hdr->hlength = 0;
1196         hdr->itt = io_task->libiscsi_itt;
1197         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1198 }
1199
1200 static void
1201 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1202                  struct iscsi_task *task,
1203                  struct common_sol_cqe *csol_cqe)
1204 {
1205         struct iscsi_tm_rsp *hdr;
1206         struct iscsi_conn *conn = beiscsi_conn->conn;
1207         struct beiscsi_io_task *io_task = task->dd_data;
1208
1209         hdr = (struct iscsi_tm_rsp *)task->hdr;
1210         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1211         hdr->flags = csol_cqe->i_flags;
1212         hdr->response = csol_cqe->i_resp;
1213         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1214         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1215                                      csol_cqe->cmd_wnd - 1);
1216
1217         hdr->itt = io_task->libiscsi_itt;
1218         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1219 }
1220
1221 static void
1222 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1223                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1224 {
1225         struct hwi_wrb_context *pwrb_context;
1226         uint16_t wrb_index, cid, cri_index;
1227         struct hwi_controller *phwi_ctrlr;
1228         struct wrb_handle *pwrb_handle;
1229         struct iscsi_session *session;
1230         struct iscsi_task *task;
1231
1232         phwi_ctrlr = phba->phwi_ctrlr;
1233         if (is_chip_be2_be3r(phba)) {
1234                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1235                                           wrb_idx, psol);
1236                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1237                                     cid, psol);
1238         } else {
1239                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1240                                           wrb_idx, psol);
1241                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1242                                     cid, psol);
1243         }
1244
1245         cri_index = BE_GET_CRI_FROM_CID(cid);
1246         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1247         pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
1248         session = beiscsi_conn->conn->session;
1249         spin_lock_bh(&session->back_lock);
1250         task = pwrb_handle->pio_handle;
1251         if (task)
1252                 __iscsi_put_task(task);
1253         spin_unlock_bh(&session->back_lock);
1254 }
1255
1256 static void
1257 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1258                         struct iscsi_task *task,
1259                         struct common_sol_cqe *csol_cqe)
1260 {
1261         struct iscsi_nopin *hdr;
1262         struct iscsi_conn *conn = beiscsi_conn->conn;
1263         struct beiscsi_io_task *io_task = task->dd_data;
1264
1265         hdr = (struct iscsi_nopin *)task->hdr;
1266         hdr->flags = csol_cqe->i_flags;
1267         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1268         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1269                                      csol_cqe->cmd_wnd - 1);
1270
1271         hdr->opcode = ISCSI_OP_NOOP_IN;
1272         hdr->itt = io_task->libiscsi_itt;
1273         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1274 }
1275
1276 static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1277                 struct sol_cqe *psol,
1278                 struct common_sol_cqe *csol_cqe)
1279 {
1280         if (is_chip_be2_be3r(phba)) {
1281                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1282                                                     i_exp_cmd_sn, psol);
1283                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1284                                                   i_res_cnt, psol);
1285                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1286                                                   i_cmd_wnd, psol);
1287                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1288                                                     wrb_index, psol);
1289                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1290                                               cid, psol);
1291                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1292                                                  hw_sts, psol);
1293                 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1294                                                  i_resp, psol);
1295                 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1296                                                 i_sts, psol);
1297                 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1298                                                   i_flags, psol);
1299         } else {
1300                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1301                                                     i_exp_cmd_sn, psol);
1302                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1303                                                   i_res_cnt, psol);
1304                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1305                                                     wrb_index, psol);
1306                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1307                                               cid, psol);
1308                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1309                                                  hw_sts, psol);
1310                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1311                                                   i_cmd_wnd, psol);
1312                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1313                                   cmd_cmpl, psol))
1314                         csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1315                                                         i_sts, psol);
1316                 else
1317                         csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1318                                                          i_sts, psol);
1319                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1320                                   u, psol))
1321                         csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1322
1323                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1324                                   o, psol))
1325                         csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
1326         }
1327 }
1328
1329
1330 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1331                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1332 {
1333         struct iscsi_conn *conn = beiscsi_conn->conn;
1334         struct iscsi_session *session = conn->session;
1335         struct common_sol_cqe csol_cqe = {0};
1336         struct hwi_wrb_context *pwrb_context;
1337         struct hwi_controller *phwi_ctrlr;
1338         struct wrb_handle *pwrb_handle;
1339         struct iscsi_task *task;
1340         uint16_t cri_index = 0;
1341         uint8_t type;
1342
1343         phwi_ctrlr = phba->phwi_ctrlr;
1344
1345         /* Copy the elements to a common structure */
1346         adapter_get_sol_cqe(phba, psol, &csol_cqe);
1347
1348         cri_index = BE_GET_CRI_FROM_CID(csol_cqe.cid);
1349         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1350
1351         pwrb_handle = pwrb_context->pwrb_handle_basestd[
1352                       csol_cqe.wrb_index];
1353
1354         spin_lock_bh(&session->back_lock);
1355         task = pwrb_handle->pio_handle;
1356         if (!task) {
1357                 spin_unlock_bh(&session->back_lock);
1358                 return;
1359         }
1360         type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
1361
1362         switch (type) {
1363         case HWH_TYPE_IO:
1364         case HWH_TYPE_IO_RD:
1365                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1366                      ISCSI_OP_NOOP_OUT)
1367                         be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1368                 else
1369                         be_complete_io(beiscsi_conn, task, &csol_cqe);
1370                 break;
1371
1372         case HWH_TYPE_LOGOUT:
1373                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1374                         be_complete_logout(beiscsi_conn, task, &csol_cqe);
1375                 else
1376                         be_complete_tmf(beiscsi_conn, task, &csol_cqe);
1377                 break;
1378
1379         case HWH_TYPE_LOGIN:
1380                 beiscsi_log(phba, KERN_ERR,
1381                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1382                             "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1383                             " hwi_complete_cmd- Solicited path\n");
1384                 break;
1385
1386         case HWH_TYPE_NOP:
1387                 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1388                 break;
1389
1390         default:
1391                 beiscsi_log(phba, KERN_WARNING,
1392                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1393                             "BM_%d : In hwi_complete_cmd, unknown type = %d"
1394                             "wrb_index 0x%x CID 0x%x\n", type,
1395                             csol_cqe.wrb_index,
1396                             csol_cqe.cid);
1397                 break;
1398         }
1399
1400         spin_unlock_bh(&session->back_lock);
1401 }
1402
1403 /**
1404  * ASYNC PDUs include
1405  * a. Unsolicited NOP-In (target initiated NOP-In)
1406  * b. ASYNC Messages
1407  * c. Reject PDU
1408  * d. Login response
1409  * These headers arrive unprocessed by the EP firmware.
1410  * iSCSI layer processes them.
1411  */
1412 static unsigned int
1413 beiscsi_complete_pdu(struct beiscsi_conn *beiscsi_conn,
1414                 struct pdu_base *phdr, void *pdata, unsigned int dlen)
1415 {
1416         struct beiscsi_hba *phba = beiscsi_conn->phba;
1417         struct iscsi_conn *conn = beiscsi_conn->conn;
1418         struct beiscsi_io_task *io_task;
1419         struct iscsi_hdr *login_hdr;
1420         struct iscsi_task *task;
1421         u8 code;
1422
1423         code = AMAP_GET_BITS(struct amap_pdu_base, opcode, phdr);
1424         switch (code) {
1425         case ISCSI_OP_NOOP_IN:
1426                 pdata = NULL;
1427                 dlen = 0;
1428                 break;
1429         case ISCSI_OP_ASYNC_EVENT:
1430                 break;
1431         case ISCSI_OP_REJECT:
1432                 WARN_ON(!pdata);
1433                 WARN_ON(!(dlen == 48));
1434                 beiscsi_log(phba, KERN_ERR,
1435                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1436                             "BM_%d : In ISCSI_OP_REJECT\n");
1437                 break;
1438         case ISCSI_OP_LOGIN_RSP:
1439         case ISCSI_OP_TEXT_RSP:
1440                 task = conn->login_task;
1441                 io_task = task->dd_data;
1442                 login_hdr = (struct iscsi_hdr *)phdr;
1443                 login_hdr->itt = io_task->libiscsi_itt;
1444                 break;
1445         default:
1446                 beiscsi_log(phba, KERN_WARNING,
1447                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1448                             "BM_%d : unrecognized async PDU opcode 0x%x\n",
1449                             code);
1450                 return 1;
1451         }
1452         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)phdr, pdata, dlen);
1453         return 0;
1454 }
1455
1456 static inline void
1457 beiscsi_hdl_put_handle(struct hd_async_context *pasync_ctx,
1458                          struct hd_async_handle *pasync_handle)
1459 {
1460         pasync_handle->is_final = 0;
1461         pasync_handle->buffer_len = 0;
1462         pasync_handle->in_use = 0;
1463         list_del_init(&pasync_handle->link);
1464 }
1465
1466 static void
1467 beiscsi_hdl_purge_handles(struct beiscsi_hba *phba,
1468                           struct hd_async_context *pasync_ctx,
1469                           u16 cri)
1470 {
1471         struct hd_async_handle *pasync_handle, *tmp_handle;
1472         struct list_head *plist;
1473
1474         plist  = &pasync_ctx->async_entry[cri].wq.list;
1475         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link)
1476                 beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
1477
1478         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wq.list);
1479         pasync_ctx->async_entry[cri].wq.hdr_len = 0;
1480         pasync_ctx->async_entry[cri].wq.bytes_received = 0;
1481         pasync_ctx->async_entry[cri].wq.bytes_needed = 0;
1482 }
1483
1484 static struct hd_async_handle *
1485 beiscsi_hdl_get_handle(struct beiscsi_conn *beiscsi_conn,
1486                        struct hd_async_context *pasync_ctx,
1487                        struct i_t_dpdu_cqe *pdpdu_cqe,
1488                        u8 *header)
1489 {
1490         struct beiscsi_hba *phba = beiscsi_conn->phba;
1491         struct hd_async_handle *pasync_handle;
1492         struct be_bus_address phys_addr;
1493         u16 cid, code, ci, cri;
1494         u8 final, error = 0;
1495         u32 dpl;
1496
1497         cid = beiscsi_conn->beiscsi_conn_cid;
1498         cri = BE_GET_ASYNC_CRI_FROM_CID(cid);
1499         /**
1500          * This function is invoked to get the right async_handle structure
1501          * from a given DEF PDU CQ entry.
1502          *
1503          * - index in CQ entry gives the vertical index
1504          * - address in CQ entry is the offset where the DMA last ended
1505          * - final - no more notifications for this PDU
1506          */
1507         if (is_chip_be2_be3r(phba)) {
1508                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1509                                     dpl, pdpdu_cqe);
1510                 ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1511                                       index, pdpdu_cqe);
1512                 final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1513                                       final, pdpdu_cqe);
1514         } else {
1515                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1516                                     dpl, pdpdu_cqe);
1517                 ci = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1518                                       index, pdpdu_cqe);
1519                 final = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1520                                       final, pdpdu_cqe);
1521         }
1522
1523         /**
1524          * DB addr Hi/Lo is same for BE and SKH.
1525          * Subtract the dataplacementlength to get to the base.
1526          */
1527         phys_addr.u.a32.address_lo = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1528                                                    db_addr_lo, pdpdu_cqe);
1529         phys_addr.u.a32.address_lo -= dpl;
1530         phys_addr.u.a32.address_hi = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1531                                                    db_addr_hi, pdpdu_cqe);
1532
1533         code = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe, code, pdpdu_cqe);
1534         switch (code) {
1535         case UNSOL_HDR_NOTIFY:
1536                 pasync_handle = pasync_ctx->async_entry[ci].header;
1537                 *header = 1;
1538                 break;
1539         case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1540                 error = 1;
1541         case UNSOL_DATA_NOTIFY:
1542                 pasync_handle = pasync_ctx->async_entry[ci].data;
1543                 break;
1544         /* called only for above codes */
1545         default:
1546                 return NULL;
1547         }
1548
1549         if (pasync_handle->pa.u.a64.address != phys_addr.u.a64.address ||
1550             pasync_handle->index != ci) {
1551                 /* driver bug - if ci does not match async handle index */
1552                 error = 1;
1553                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1554                             "BM_%d : cid %u async PDU handle mismatch - addr in %cQE %llx at %u:addr in CQE %llx ci %u\n",
1555                             cid, pasync_handle->is_header ? 'H' : 'D',
1556                             pasync_handle->pa.u.a64.address,
1557                             pasync_handle->index,
1558                             phys_addr.u.a64.address, ci);
1559                 /* FW has stale address - attempt continuing by dropping */
1560         }
1561
1562         /**
1563          * DEF PDU header and data buffers with errors should be simply
1564          * dropped as there are no consumers for it.
1565          */
1566         if (error) {
1567                 beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
1568                 return NULL;
1569         }
1570
1571         if (pasync_handle->in_use || !list_empty(&pasync_handle->link)) {
1572                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1573                             "BM_%d : cid %d async PDU handle in use - code %d ci %d addr %llx\n",
1574                             cid, code, ci, phys_addr.u.a64.address);
1575                 beiscsi_hdl_purge_handles(phba, pasync_ctx, cri);
1576         }
1577
1578         list_del_init(&pasync_handle->link);
1579         /**
1580          * Each CID is associated with unique CRI.
1581          * ASYNC_CRI_FROM_CID mapping and CRI_FROM_CID are totaly different.
1582          **/
1583         pasync_handle->cri = cri;
1584         pasync_handle->is_final = final;
1585         pasync_handle->buffer_len = dpl;
1586         pasync_handle->in_use = 1;
1587
1588         return pasync_handle;
1589 }
1590
1591 static unsigned int
1592 beiscsi_hdl_fwd_pdu(struct beiscsi_conn *beiscsi_conn,
1593                     struct hd_async_context *pasync_ctx,
1594                     u16 cri)
1595 {
1596         struct iscsi_session *session = beiscsi_conn->conn->session;
1597         struct hd_async_handle *pasync_handle, *plast_handle;
1598         struct beiscsi_hba *phba = beiscsi_conn->phba;
1599         void *phdr = NULL, *pdata = NULL;
1600         u32 dlen = 0, status = 0;
1601         struct list_head *plist;
1602
1603         plist = &pasync_ctx->async_entry[cri].wq.list;
1604         plast_handle = NULL;
1605         list_for_each_entry(pasync_handle, plist, link) {
1606                 plast_handle = pasync_handle;
1607                 /* get the header, the first entry */
1608                 if (!phdr) {
1609                         phdr = pasync_handle->pbuffer;
1610                         continue;
1611                 }
1612                 /* use first buffer to collect all the data */
1613                 if (!pdata) {
1614                         pdata = pasync_handle->pbuffer;
1615                         dlen = pasync_handle->buffer_len;
1616                         continue;
1617                 }
1618                 if (!pasync_handle->buffer_len ||
1619                     (dlen + pasync_handle->buffer_len) >
1620                     pasync_ctx->async_data.buffer_size)
1621                         break;
1622                 memcpy(pdata + dlen, pasync_handle->pbuffer,
1623                        pasync_handle->buffer_len);
1624                 dlen += pasync_handle->buffer_len;
1625         }
1626
1627         if (!plast_handle->is_final) {
1628                 /* last handle should have final PDU notification from FW */
1629                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1630                             "BM_%d : cid %u %p fwd async PDU opcode %x with last handle missing - HL%u:DN%u:DR%u\n",
1631                             beiscsi_conn->beiscsi_conn_cid, plast_handle,
1632                             AMAP_GET_BITS(struct amap_pdu_base, opcode, phdr),
1633                             pasync_ctx->async_entry[cri].wq.hdr_len,
1634                             pasync_ctx->async_entry[cri].wq.bytes_needed,
1635                             pasync_ctx->async_entry[cri].wq.bytes_received);
1636         }
1637         spin_lock_bh(&session->back_lock);
1638         status = beiscsi_complete_pdu(beiscsi_conn, phdr, pdata, dlen);
1639         spin_unlock_bh(&session->back_lock);
1640         beiscsi_hdl_purge_handles(phba, pasync_ctx, cri);
1641         return status;
1642 }
1643
1644 static unsigned int
1645 beiscsi_hdl_gather_pdu(struct beiscsi_conn *beiscsi_conn,
1646                        struct hd_async_context *pasync_ctx,
1647                        struct hd_async_handle *pasync_handle)
1648 {
1649         unsigned int bytes_needed = 0, status = 0;
1650         u16 cri = pasync_handle->cri;
1651         struct cri_wait_queue *wq;
1652         struct beiscsi_hba *phba;
1653         struct pdu_base *ppdu;
1654         char *err = "";
1655
1656         phba = beiscsi_conn->phba;
1657         wq = &pasync_ctx->async_entry[cri].wq;
1658         if (pasync_handle->is_header) {
1659                 /* check if PDU hdr is rcv'd when old hdr not completed */
1660                 if (wq->hdr_len) {
1661                         err = "incomplete";
1662                         goto drop_pdu;
1663                 }
1664                 ppdu = pasync_handle->pbuffer;
1665                 bytes_needed = AMAP_GET_BITS(struct amap_pdu_base,
1666                                              data_len_hi, ppdu);
1667                 bytes_needed <<= 16;
1668                 bytes_needed |= be16_to_cpu(AMAP_GET_BITS(struct amap_pdu_base,
1669                                                           data_len_lo, ppdu));
1670                 wq->hdr_len = pasync_handle->buffer_len;
1671                 wq->bytes_received = 0;
1672                 wq->bytes_needed = bytes_needed;
1673                 list_add_tail(&pasync_handle->link, &wq->list);
1674                 if (!bytes_needed)
1675                         status = beiscsi_hdl_fwd_pdu(beiscsi_conn,
1676                                                      pasync_ctx, cri);
1677         } else {
1678                 /* check if data received has header and is needed */
1679                 if (!wq->hdr_len || !wq->bytes_needed) {
1680                         err = "header less";
1681                         goto drop_pdu;
1682                 }
1683                 wq->bytes_received += pasync_handle->buffer_len;
1684                 /* Something got overwritten? Better catch it here. */
1685                 if (wq->bytes_received > wq->bytes_needed) {
1686                         err = "overflow";
1687                         goto drop_pdu;
1688                 }
1689                 list_add_tail(&pasync_handle->link, &wq->list);
1690                 if (wq->bytes_received == wq->bytes_needed)
1691                         status = beiscsi_hdl_fwd_pdu(beiscsi_conn,
1692                                                      pasync_ctx, cri);
1693         }
1694         return status;
1695
1696 drop_pdu:
1697         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_ISCSI,
1698                     "BM_%d : cid %u async PDU %s - def-%c:HL%u:DN%u:DR%u\n",
1699                     beiscsi_conn->beiscsi_conn_cid, err,
1700                     pasync_handle->is_header ? 'H' : 'D',
1701                     wq->hdr_len, wq->bytes_needed,
1702                     pasync_handle->buffer_len);
1703         /* discard this handle */
1704         beiscsi_hdl_put_handle(pasync_ctx, pasync_handle);
1705         /* free all the other handles in cri_wait_queue */
1706         beiscsi_hdl_purge_handles(phba, pasync_ctx, cri);
1707         /* try continuing */
1708         return status;
1709 }
1710
1711 static void
1712 beiscsi_hdq_post_handles(struct beiscsi_hba *phba,
1713                          u8 header, u8 ulp_num, u16 nbuf)
1714 {
1715         struct hd_async_handle *pasync_handle;
1716         struct hd_async_context *pasync_ctx;
1717         struct hwi_controller *phwi_ctrlr;
1718         struct phys_addr *pasync_sge;
1719         u32 ring_id, doorbell = 0;
1720         u32 doorbell_offset;
1721         u16 prod, pi;
1722
1723         phwi_ctrlr = phba->phwi_ctrlr;
1724         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
1725         if (header) {
1726                 pasync_sge = pasync_ctx->async_header.ring_base;
1727                 pi = pasync_ctx->async_header.pi;
1728                 ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id;
1729                 doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num].
1730                                         doorbell_offset;
1731         } else {
1732                 pasync_sge = pasync_ctx->async_data.ring_base;
1733                 pi = pasync_ctx->async_data.pi;
1734                 ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id;
1735                 doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num].
1736                                         doorbell_offset;
1737         }
1738
1739         for (prod = 0; prod < nbuf; prod++) {
1740                 if (header)
1741                         pasync_handle = pasync_ctx->async_entry[pi].header;
1742                 else
1743                         pasync_handle = pasync_ctx->async_entry[pi].data;
1744                 WARN_ON(pasync_handle->is_header != header);
1745                 WARN_ON(pasync_handle->index != pi);
1746                 /* setup the ring only once */
1747                 if (nbuf == pasync_ctx->num_entries) {
1748                         /* note hi is lo */
1749                         pasync_sge[pi].hi = pasync_handle->pa.u.a32.address_lo;
1750                         pasync_sge[pi].lo = pasync_handle->pa.u.a32.address_hi;
1751                 }
1752                 if (++pi == pasync_ctx->num_entries)
1753                         pi = 0;
1754         }
1755
1756         if (header)
1757                 pasync_ctx->async_header.pi = pi;
1758         else
1759                 pasync_ctx->async_data.pi = pi;
1760
1761         doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1762         doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1763         doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1764         doorbell |= (prod & DB_DEF_PDU_CQPROC_MASK) << DB_DEF_PDU_CQPROC_SHIFT;
1765         iowrite32(doorbell, phba->db_va + doorbell_offset);
1766 }
1767
1768 static void
1769 beiscsi_hdq_process_compl(struct beiscsi_conn *beiscsi_conn,
1770                           struct i_t_dpdu_cqe *pdpdu_cqe)
1771 {
1772         struct beiscsi_hba *phba = beiscsi_conn->phba;
1773         struct hd_async_handle *pasync_handle = NULL;
1774         struct hd_async_context *pasync_ctx;
1775         struct hwi_controller *phwi_ctrlr;
1776         u8 ulp_num, consumed, header = 0;
1777         u16 cid_cri;
1778
1779         phwi_ctrlr = phba->phwi_ctrlr;
1780         cid_cri = BE_GET_CRI_FROM_CID(beiscsi_conn->beiscsi_conn_cid);
1781         ulp_num = BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr, cid_cri);
1782         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
1783         pasync_handle = beiscsi_hdl_get_handle(beiscsi_conn, pasync_ctx,
1784                                                pdpdu_cqe, &header);
1785         if (is_chip_be2_be3r(phba))
1786                 consumed = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1787                                          num_cons, pdpdu_cqe);
1788         else
1789                 consumed = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1790                                          num_cons, pdpdu_cqe);
1791         if (pasync_handle)
1792                 beiscsi_hdl_gather_pdu(beiscsi_conn, pasync_ctx, pasync_handle);
1793         /* num_cons indicates number of 8 RQEs consumed */
1794         if (consumed)
1795                 beiscsi_hdq_post_handles(phba, header, ulp_num, 8 * consumed);
1796 }
1797
1798 void beiscsi_process_mcc_cq(struct beiscsi_hba *phba)
1799 {
1800         struct be_queue_info *mcc_cq;
1801         struct  be_mcc_compl *mcc_compl;
1802         unsigned int num_processed = 0;
1803
1804         mcc_cq = &phba->ctrl.mcc_obj.cq;
1805         mcc_compl = queue_tail_node(mcc_cq);
1806         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1807         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
1808                 if (beiscsi_hba_in_error(phba))
1809                         return;
1810
1811                 if (num_processed >= 32) {
1812                         hwi_ring_cq_db(phba, mcc_cq->id,
1813                                         num_processed, 0);
1814                         num_processed = 0;
1815                 }
1816                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
1817                         beiscsi_process_async_event(phba, mcc_compl);
1818                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
1819                         beiscsi_process_mcc_compl(&phba->ctrl, mcc_compl);
1820                 }
1821
1822                 mcc_compl->flags = 0;
1823                 queue_tail_inc(mcc_cq);
1824                 mcc_compl = queue_tail_node(mcc_cq);
1825                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
1826                 num_processed++;
1827         }
1828
1829         if (num_processed > 0)
1830                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1);
1831 }
1832
1833 static void beiscsi_mcc_work(struct work_struct *work)
1834 {
1835         struct be_eq_obj *pbe_eq;
1836         struct beiscsi_hba *phba;
1837
1838         pbe_eq = container_of(work, struct be_eq_obj, mcc_work);
1839         phba = pbe_eq->phba;
1840         beiscsi_process_mcc_cq(phba);
1841         /* rearm EQ for further interrupts */
1842         if (!beiscsi_hba_in_error(phba))
1843                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
1844 }
1845
1846 /**
1847  * beiscsi_process_cq()- Process the Completion Queue
1848  * @pbe_eq: Event Q on which the Completion has come
1849  * @budget: Max number of events to processed
1850  *
1851  * return
1852  *     Number of Completion Entries processed.
1853  **/
1854 unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq, int budget)
1855 {
1856         struct be_queue_info *cq;
1857         struct sol_cqe *sol;
1858         struct dmsg_cqe *dmsg;
1859         unsigned int total = 0;
1860         unsigned int num_processed = 0;
1861         unsigned short code = 0, cid = 0;
1862         uint16_t cri_index = 0;
1863         struct beiscsi_conn *beiscsi_conn;
1864         struct beiscsi_endpoint *beiscsi_ep;
1865         struct iscsi_endpoint *ep;
1866         struct beiscsi_hba *phba;
1867
1868         cq = pbe_eq->cq;
1869         sol = queue_tail_node(cq);
1870         phba = pbe_eq->phba;
1871
1872         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
1873                CQE_VALID_MASK) {
1874                 if (beiscsi_hba_in_error(phba))
1875                         return 0;
1876
1877                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
1878
1879                 code = (sol->dw[offsetof(struct amap_sol_cqe, code) / 32] &
1880                                 CQE_CODE_MASK);
1881
1882                  /* Get the CID */
1883                 if (is_chip_be2_be3r(phba)) {
1884                         cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
1885                 } else {
1886                         if ((code == DRIVERMSG_NOTIFY) ||
1887                             (code == UNSOL_HDR_NOTIFY) ||
1888                             (code == UNSOL_DATA_NOTIFY))
1889                                 cid = AMAP_GET_BITS(
1890                                                     struct amap_i_t_dpdu_cqe_v2,
1891                                                     cid, sol);
1892                          else
1893                                  cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1894                                                      cid, sol);
1895                 }
1896
1897                 cri_index = BE_GET_CRI_FROM_CID(cid);
1898                 ep = phba->ep_array[cri_index];
1899
1900                 if (ep == NULL) {
1901                         /* connection has already been freed
1902                          * just move on to next one
1903                          */
1904                         beiscsi_log(phba, KERN_WARNING,
1905                                     BEISCSI_LOG_INIT,
1906                                     "BM_%d : proc cqe of disconn ep: cid %d\n",
1907                                     cid);
1908                         goto proc_next_cqe;
1909                 }
1910
1911                 beiscsi_ep = ep->dd_data;
1912                 beiscsi_conn = beiscsi_ep->conn;
1913
1914                 /* replenish cq */
1915                 if (num_processed == 32) {
1916                         hwi_ring_cq_db(phba, cq->id, 32, 0);
1917                         num_processed = 0;
1918                 }
1919                 total++;
1920
1921                 switch (code) {
1922                 case SOL_CMD_COMPLETE:
1923                         hwi_complete_cmd(beiscsi_conn, phba, sol);
1924                         break;
1925                 case DRIVERMSG_NOTIFY:
1926                         beiscsi_log(phba, KERN_INFO,
1927                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1928                                     "BM_%d : Received %s[%d] on CID : %d\n",
1929                                     cqe_desc[code], code, cid);
1930
1931                         dmsg = (struct dmsg_cqe *)sol;
1932                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
1933                         break;
1934                 case UNSOL_HDR_NOTIFY:
1935                         beiscsi_log(phba, KERN_INFO,
1936                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1937                                     "BM_%d : Received %s[%d] on CID : %d\n",
1938                                     cqe_desc[code], code, cid);
1939
1940                         spin_lock_bh(&phba->async_pdu_lock);
1941                         beiscsi_hdq_process_compl(beiscsi_conn,
1942                                                   (struct i_t_dpdu_cqe *)sol);
1943                         spin_unlock_bh(&phba->async_pdu_lock);
1944                         break;
1945                 case UNSOL_DATA_NOTIFY:
1946                         beiscsi_log(phba, KERN_INFO,
1947                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1948                                     "BM_%d : Received %s[%d] on CID : %d\n",
1949                                     cqe_desc[code], code, cid);
1950
1951                         spin_lock_bh(&phba->async_pdu_lock);
1952                         beiscsi_hdq_process_compl(beiscsi_conn,
1953                                                   (struct i_t_dpdu_cqe *)sol);
1954                         spin_unlock_bh(&phba->async_pdu_lock);
1955                         break;
1956                 case CXN_INVALIDATE_INDEX_NOTIFY:
1957                 case CMD_INVALIDATED_NOTIFY:
1958                 case CXN_INVALIDATE_NOTIFY:
1959                         beiscsi_log(phba, KERN_ERR,
1960                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1961                                     "BM_%d : Ignoring %s[%d] on CID : %d\n",
1962                                     cqe_desc[code], code, cid);
1963                         break;
1964                 case CXN_KILLED_HDR_DIGEST_ERR:
1965                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
1966                         beiscsi_log(phba, KERN_ERR,
1967                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1968                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
1969                                     cqe_desc[code], code,  cid);
1970                         break;
1971                 case CMD_KILLED_INVALID_STATSN_RCVD:
1972                 case CMD_KILLED_INVALID_R2T_RCVD:
1973                 case CMD_CXN_KILLED_LUN_INVALID:
1974                 case CMD_CXN_KILLED_ICD_INVALID:
1975                 case CMD_CXN_KILLED_ITT_INVALID:
1976                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
1977                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
1978                         beiscsi_log(phba, KERN_ERR,
1979                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1980                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
1981                                     cqe_desc[code], code,  cid);
1982                         break;
1983                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
1984                         beiscsi_log(phba, KERN_ERR,
1985                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1986                                     "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
1987                                     cqe_desc[code], code, cid);
1988                         spin_lock_bh(&phba->async_pdu_lock);
1989                         /* driver consumes the entry and drops the contents */
1990                         beiscsi_hdq_process_compl(beiscsi_conn,
1991                                                   (struct i_t_dpdu_cqe *)sol);
1992                         spin_unlock_bh(&phba->async_pdu_lock);
1993                         break;
1994                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
1995                 case CXN_KILLED_BURST_LEN_MISMATCH:
1996                 case CXN_KILLED_AHS_RCVD:
1997                 case CXN_KILLED_UNKNOWN_HDR:
1998                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
1999                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2000                 case CXN_KILLED_TIMED_OUT:
2001                 case CXN_KILLED_FIN_RCVD:
2002                 case CXN_KILLED_RST_SENT:
2003                 case CXN_KILLED_RST_RCVD:
2004                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2005                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2006                 case CXN_KILLED_OVER_RUN_RESIDUAL:
2007                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2008                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
2009                         beiscsi_log(phba, KERN_ERR,
2010                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2011                                     "BM_%d : Event %s[%d] received on CID : %d\n",
2012                                     cqe_desc[code], code, cid);
2013                         if (beiscsi_conn)
2014                                 iscsi_conn_failure(beiscsi_conn->conn,
2015                                                    ISCSI_ERR_CONN_FAILED);
2016                         break;
2017                 default:
2018                         beiscsi_log(phba, KERN_ERR,
2019                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2020                                     "BM_%d : Invalid CQE Event Received Code : %d"
2021                                     "CID 0x%x...\n",
2022                                     code, cid);
2023                         break;
2024                 }
2025
2026 proc_next_cqe:
2027                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2028                 queue_tail_inc(cq);
2029                 sol = queue_tail_node(cq);
2030                 num_processed++;
2031                 if (total == budget)
2032                         break;
2033         }
2034
2035         hwi_ring_cq_db(phba, cq->id, num_processed, 1);
2036         return total;
2037 }
2038
2039 static int be_iopoll(struct irq_poll *iop, int budget)
2040 {
2041         unsigned int ret, io_events;
2042         struct beiscsi_hba *phba;
2043         struct be_eq_obj *pbe_eq;
2044         struct be_eq_entry *eqe = NULL;
2045         struct be_queue_info *eq;
2046
2047         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2048         phba = pbe_eq->phba;
2049         if (beiscsi_hba_in_error(phba)) {
2050                 irq_poll_complete(iop);
2051                 return 0;
2052         }
2053
2054         io_events = 0;
2055         eq = &pbe_eq->q;
2056         eqe = queue_tail_node(eq);
2057         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] &
2058                         EQE_VALID_MASK) {
2059                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
2060                 queue_tail_inc(eq);
2061                 eqe = queue_tail_node(eq);
2062                 io_events++;
2063         }
2064         hwi_ring_eq_db(phba, eq->id, 1, io_events, 0, 1);
2065
2066         ret = beiscsi_process_cq(pbe_eq, budget);
2067         pbe_eq->cq_count += ret;
2068         if (ret < budget) {
2069                 irq_poll_complete(iop);
2070                 beiscsi_log(phba, KERN_INFO,
2071                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2072                             "BM_%d : rearm pbe_eq->q.id =%d ret %d\n",
2073                             pbe_eq->q.id, ret);
2074                 if (!beiscsi_hba_in_error(phba))
2075                         hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2076         }
2077         return ret;
2078 }
2079
2080 static void
2081 hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2082                   unsigned int num_sg, struct beiscsi_io_task *io_task)
2083 {
2084         struct iscsi_sge *psgl;
2085         unsigned int sg_len, index;
2086         unsigned int sge_len = 0;
2087         unsigned long long addr;
2088         struct scatterlist *l_sg;
2089         unsigned int offset;
2090
2091         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2092                       io_task->bhs_pa.u.a32.address_lo);
2093         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2094                       io_task->bhs_pa.u.a32.address_hi);
2095
2096         l_sg = sg;
2097         for (index = 0; (index < num_sg) && (index < 2); index++,
2098                         sg = sg_next(sg)) {
2099                 if (index == 0) {
2100                         sg_len = sg_dma_len(sg);
2101                         addr = (u64) sg_dma_address(sg);
2102                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2103                                       sge0_addr_lo, pwrb,
2104                                       lower_32_bits(addr));
2105                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2106                                       sge0_addr_hi, pwrb,
2107                                       upper_32_bits(addr));
2108                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2109                                       sge0_len, pwrb,
2110                                       sg_len);
2111                         sge_len = sg_len;
2112                 } else {
2113                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2114                                       pwrb, sge_len);
2115                         sg_len = sg_dma_len(sg);
2116                         addr = (u64) sg_dma_address(sg);
2117                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2118                                       sge1_addr_lo, pwrb,
2119                                       lower_32_bits(addr));
2120                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2121                                       sge1_addr_hi, pwrb,
2122                                       upper_32_bits(addr));
2123                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2124                                       sge1_len, pwrb,
2125                                       sg_len);
2126                 }
2127         }
2128         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2129         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2130
2131         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2132
2133         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2134                       io_task->bhs_pa.u.a32.address_hi);
2135         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2136                       io_task->bhs_pa.u.a32.address_lo);
2137
2138         if (num_sg == 1) {
2139                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2140                               1);
2141                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2142                               0);
2143         } else if (num_sg == 2) {
2144                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2145                               0);
2146                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2147                               1);
2148         } else {
2149                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2150                               0);
2151                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2152                               0);
2153         }
2154
2155         sg = l_sg;
2156         psgl++;
2157         psgl++;
2158         offset = 0;
2159         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2160                 sg_len = sg_dma_len(sg);
2161                 addr = (u64) sg_dma_address(sg);
2162                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2163                               lower_32_bits(addr));
2164                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2165                               upper_32_bits(addr));
2166                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2167                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2168                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2169                 offset += sg_len;
2170         }
2171         psgl--;
2172         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2173 }
2174
2175 static void
2176 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2177               unsigned int num_sg, struct beiscsi_io_task *io_task)
2178 {
2179         struct iscsi_sge *psgl;
2180         unsigned int sg_len, index;
2181         unsigned int sge_len = 0;
2182         unsigned long long addr;
2183         struct scatterlist *l_sg;
2184         unsigned int offset;
2185
2186         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2187                                       io_task->bhs_pa.u.a32.address_lo);
2188         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2189                                       io_task->bhs_pa.u.a32.address_hi);
2190
2191         l_sg = sg;
2192         for (index = 0; (index < num_sg) && (index < 2); index++,
2193                                                          sg = sg_next(sg)) {
2194                 if (index == 0) {
2195                         sg_len = sg_dma_len(sg);
2196                         addr = (u64) sg_dma_address(sg);
2197                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2198                                                 ((u32)(addr & 0xFFFFFFFF)));
2199                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2200                                                         ((u32)(addr >> 32)));
2201                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2202                                                         sg_len);
2203                         sge_len = sg_len;
2204                 } else {
2205                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2206                                                         pwrb, sge_len);
2207                         sg_len = sg_dma_len(sg);
2208                         addr = (u64) sg_dma_address(sg);
2209                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2210                                                 ((u32)(addr & 0xFFFFFFFF)));
2211                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2212                                                         ((u32)(addr >> 32)));
2213                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2214                                                         sg_len);
2215                 }
2216         }
2217         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2218         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2219
2220         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2221
2222         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2223                         io_task->bhs_pa.u.a32.address_hi);
2224         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2225                         io_task->bhs_pa.u.a32.address_lo);
2226
2227         if (num_sg == 1) {
2228                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2229                                                                 1);
2230                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2231                                                                 0);
2232         } else if (num_sg == 2) {
2233                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2234                                                                 0);
2235                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2236                                                                 1);
2237         } else {
2238                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2239                                                                 0);
2240                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2241                                                                 0);
2242         }
2243         sg = l_sg;
2244         psgl++;
2245         psgl++;
2246         offset = 0;
2247         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2248                 sg_len = sg_dma_len(sg);
2249                 addr = (u64) sg_dma_address(sg);
2250                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2251                                                 (addr & 0xFFFFFFFF));
2252                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2253                                                 (addr >> 32));
2254                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2255                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2256                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2257                 offset += sg_len;
2258         }
2259         psgl--;
2260         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2261 }
2262
2263 /**
2264  * hwi_write_buffer()- Populate the WRB with task info
2265  * @pwrb: ptr to the WRB entry
2266  * @task: iscsi task which is to be executed
2267  **/
2268 static int hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2269 {
2270         struct iscsi_sge *psgl;
2271         struct beiscsi_io_task *io_task = task->dd_data;
2272         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2273         struct beiscsi_hba *phba = beiscsi_conn->phba;
2274         uint8_t dsp_value = 0;
2275
2276         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2277         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2278                                 io_task->bhs_pa.u.a32.address_lo);
2279         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2280                                 io_task->bhs_pa.u.a32.address_hi);
2281
2282         if (task->data) {
2283
2284                 /* Check for the data_count */
2285                 dsp_value = (task->data_count) ? 1 : 0;
2286
2287                 if (is_chip_be2_be3r(phba))
2288                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
2289                                       pwrb, dsp_value);
2290                 else
2291                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
2292                                       pwrb, dsp_value);
2293
2294                 /* Map addr only if there is data_count */
2295                 if (dsp_value) {
2296                         io_task->mtask_addr = pci_map_single(phba->pcidev,
2297                                                              task->data,
2298                                                              task->data_count,
2299                                                              PCI_DMA_TODEVICE);
2300                         if (pci_dma_mapping_error(phba->pcidev,
2301                                                   io_task->mtask_addr))
2302                                 return -ENOMEM;
2303                         io_task->mtask_data_count = task->data_count;
2304                 } else
2305                         io_task->mtask_addr = 0;
2306
2307                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2308                               lower_32_bits(io_task->mtask_addr));
2309                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2310                               upper_32_bits(io_task->mtask_addr));
2311                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2312                                                 task->data_count);
2313
2314                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2315         } else {
2316                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2317                 io_task->mtask_addr = 0;
2318         }
2319
2320         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2321
2322         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2323
2324         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2325                       io_task->bhs_pa.u.a32.address_hi);
2326         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2327                       io_task->bhs_pa.u.a32.address_lo);
2328         if (task->data) {
2329                 psgl++;
2330                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2331                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2332                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2333                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2334                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2335                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2336
2337                 psgl++;
2338                 if (task->data) {
2339                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2340                                       lower_32_bits(io_task->mtask_addr));
2341                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2342                                       upper_32_bits(io_task->mtask_addr));
2343                 }
2344                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2345         }
2346         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2347         return 0;
2348 }
2349
2350 /**
2351  * beiscsi_find_mem_req()- Find mem needed
2352  * @phba: ptr to HBA struct
2353  **/
2354 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2355 {
2356         uint8_t mem_descr_index, ulp_num;
2357         unsigned int num_async_pdu_buf_pages;
2358         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2359         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2360
2361         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2362
2363         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2364                                                  BE_ISCSI_PDU_HEADER_SIZE;
2365         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2366                                             sizeof(struct hwi_context_memory);
2367
2368
2369         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2370             * (phba->params.wrbs_per_cxn)
2371             * phba->params.cxns_per_ctrl;
2372         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2373                                  (phba->params.wrbs_per_cxn);
2374         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2375                                 phba->params.cxns_per_ctrl);
2376
2377         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2378                 phba->params.icds_per_ctrl;
2379         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2380                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2381         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2382                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
2383
2384                         num_async_pdu_buf_sgl_pages =
2385                                 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2386                                                phba, ulp_num) *
2387                                                sizeof(struct phys_addr));
2388
2389                         num_async_pdu_buf_pages =
2390                                 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2391                                                phba, ulp_num) *
2392                                                phba->params.defpdu_hdr_sz);
2393
2394                         num_async_pdu_data_pages =
2395                                 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2396                                                phba, ulp_num) *
2397                                                phba->params.defpdu_data_sz);
2398
2399                         num_async_pdu_data_sgl_pages =
2400                                 PAGES_REQUIRED(BEISCSI_ASYNC_HDQ_SIZE(
2401                                                phba, ulp_num) *
2402                                                sizeof(struct phys_addr));
2403
2404                         mem_descr_index = (HWI_MEM_TEMPLATE_HDR_ULP0 +
2405                                           (ulp_num * MEM_DESCR_OFFSET));
2406                         phba->mem_req[mem_descr_index] =
2407                                         BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2408                                         BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE;
2409
2410                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2411                                           (ulp_num * MEM_DESCR_OFFSET));
2412                         phba->mem_req[mem_descr_index] =
2413                                           num_async_pdu_buf_pages *
2414                                           PAGE_SIZE;
2415
2416                         mem_descr_index = (HWI_MEM_ASYNC_DATA_BUF_ULP0 +
2417                                           (ulp_num * MEM_DESCR_OFFSET));
2418                         phba->mem_req[mem_descr_index] =
2419                                           num_async_pdu_data_pages *
2420                                           PAGE_SIZE;
2421
2422                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2423                                           (ulp_num * MEM_DESCR_OFFSET));
2424                         phba->mem_req[mem_descr_index] =
2425                                           num_async_pdu_buf_sgl_pages *
2426                                           PAGE_SIZE;
2427
2428                         mem_descr_index = (HWI_MEM_ASYNC_DATA_RING_ULP0 +
2429                                           (ulp_num * MEM_DESCR_OFFSET));
2430                         phba->mem_req[mem_descr_index] =
2431                                           num_async_pdu_data_sgl_pages *
2432                                           PAGE_SIZE;
2433
2434                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2435                                           (ulp_num * MEM_DESCR_OFFSET));
2436                         phba->mem_req[mem_descr_index] =
2437                                 BEISCSI_ASYNC_HDQ_SIZE(phba, ulp_num) *
2438                                 sizeof(struct hd_async_handle);
2439
2440                         mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
2441                                           (ulp_num * MEM_DESCR_OFFSET));
2442                         phba->mem_req[mem_descr_index] =
2443                                 BEISCSI_ASYNC_HDQ_SIZE(phba, ulp_num) *
2444                                 sizeof(struct hd_async_handle);
2445
2446                         mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2447                                           (ulp_num * MEM_DESCR_OFFSET));
2448                         phba->mem_req[mem_descr_index] =
2449                                 sizeof(struct hd_async_context) +
2450                                 (BEISCSI_ASYNC_HDQ_SIZE(phba, ulp_num) *
2451                                  sizeof(struct hd_async_entry));
2452                 }
2453         }
2454 }
2455
2456 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2457 {
2458         dma_addr_t bus_add;
2459         struct hwi_controller *phwi_ctrlr;
2460         struct be_mem_descriptor *mem_descr;
2461         struct mem_array *mem_arr, *mem_arr_orig;
2462         unsigned int i, j, alloc_size, curr_alloc_size;
2463
2464         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2465         if (!phba->phwi_ctrlr)
2466                 return -ENOMEM;
2467
2468         /* Allocate memory for wrb_context */
2469         phwi_ctrlr = phba->phwi_ctrlr;
2470         phwi_ctrlr->wrb_context = kzalloc(sizeof(struct hwi_wrb_context) *
2471                                           phba->params.cxns_per_ctrl,
2472                                           GFP_KERNEL);
2473         if (!phwi_ctrlr->wrb_context) {
2474                 kfree(phba->phwi_ctrlr);
2475                 return -ENOMEM;
2476         }
2477
2478         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2479                                  GFP_KERNEL);
2480         if (!phba->init_mem) {
2481                 kfree(phwi_ctrlr->wrb_context);
2482                 kfree(phba->phwi_ctrlr);
2483                 return -ENOMEM;
2484         }
2485
2486         mem_arr_orig = kmalloc_array(BEISCSI_MAX_FRAGS_INIT,
2487                                      sizeof(*mem_arr_orig),
2488                                      GFP_KERNEL);
2489         if (!mem_arr_orig) {
2490                 kfree(phba->init_mem);
2491                 kfree(phwi_ctrlr->wrb_context);
2492                 kfree(phba->phwi_ctrlr);
2493                 return -ENOMEM;
2494         }
2495
2496         mem_descr = phba->init_mem;
2497         for (i = 0; i < SE_MEM_MAX; i++) {
2498                 if (!phba->mem_req[i]) {
2499                         mem_descr->mem_array = NULL;
2500                         mem_descr++;
2501                         continue;
2502                 }
2503
2504                 j = 0;
2505                 mem_arr = mem_arr_orig;
2506                 alloc_size = phba->mem_req[i];
2507                 memset(mem_arr, 0, sizeof(struct mem_array) *
2508                        BEISCSI_MAX_FRAGS_INIT);
2509                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2510                 do {
2511                         mem_arr->virtual_address = pci_alloc_consistent(
2512                                                         phba->pcidev,
2513                                                         curr_alloc_size,
2514                                                         &bus_add);
2515                         if (!mem_arr->virtual_address) {
2516                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2517                                         goto free_mem;
2518                                 if (curr_alloc_size -
2519                                         rounddown_pow_of_two(curr_alloc_size))
2520                                         curr_alloc_size = rounddown_pow_of_two
2521                                                              (curr_alloc_size);
2522                                 else
2523                                         curr_alloc_size = curr_alloc_size / 2;
2524                         } else {
2525                                 mem_arr->bus_address.u.
2526                                     a64.address = (__u64) bus_add;
2527                                 mem_arr->size = curr_alloc_size;
2528                                 alloc_size -= curr_alloc_size;
2529                                 curr_alloc_size = min(be_max_phys_size *
2530                                                       1024, alloc_size);
2531                                 j++;
2532                                 mem_arr++;
2533                         }
2534                 } while (alloc_size);
2535                 mem_descr->num_elements = j;
2536                 mem_descr->size_in_bytes = phba->mem_req[i];
2537                 mem_descr->mem_array = kmalloc_array(j, sizeof(*mem_arr),
2538                                                      GFP_KERNEL);
2539                 if (!mem_descr->mem_array)
2540                         goto free_mem;
2541
2542                 memcpy(mem_descr->mem_array, mem_arr_orig,
2543                        sizeof(struct mem_array) * j);
2544                 mem_descr++;
2545         }
2546         kfree(mem_arr_orig);
2547         return 0;
2548 free_mem:
2549         mem_descr->num_elements = j;
2550         while ((i) || (j)) {
2551                 for (j = mem_descr->num_elements; j > 0; j--) {
2552                         pci_free_consistent(phba->pcidev,
2553                                             mem_descr->mem_array[j - 1].size,
2554                                             mem_descr->mem_array[j - 1].
2555                                             virtual_address,
2556                                             (unsigned long)mem_descr->
2557                                             mem_array[j - 1].
2558                                             bus_address.u.a64.address);
2559                 }
2560                 if (i) {
2561                         i--;
2562                         kfree(mem_descr->mem_array);
2563                         mem_descr--;
2564                 }
2565         }
2566         kfree(mem_arr_orig);
2567         kfree(phba->init_mem);
2568         kfree(phba->phwi_ctrlr->wrb_context);
2569         kfree(phba->phwi_ctrlr);
2570         return -ENOMEM;
2571 }
2572
2573 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2574 {
2575         beiscsi_find_mem_req(phba);
2576         return beiscsi_alloc_mem(phba);
2577 }
2578
2579 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2580 {
2581         struct pdu_data_out *pdata_out;
2582         struct pdu_nop_out *pnop_out;
2583         struct be_mem_descriptor *mem_descr;
2584
2585         mem_descr = phba->init_mem;
2586         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2587         pdata_out =
2588             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2589         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2590
2591         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2592                       IIOC_SCSI_DATA);
2593
2594         pnop_out =
2595             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2596                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2597
2598         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2599         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2600         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2601         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2602 }
2603
2604 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2605 {
2606         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2607         struct hwi_context_memory *phwi_ctxt;
2608         struct wrb_handle *pwrb_handle = NULL;
2609         struct hwi_controller *phwi_ctrlr;
2610         struct hwi_wrb_context *pwrb_context;
2611         struct iscsi_wrb *pwrb = NULL;
2612         unsigned int num_cxn_wrbh = 0;
2613         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2614
2615         mem_descr_wrbh = phba->init_mem;
2616         mem_descr_wrbh += HWI_MEM_WRBH;
2617
2618         mem_descr_wrb = phba->init_mem;
2619         mem_descr_wrb += HWI_MEM_WRB;
2620         phwi_ctrlr = phba->phwi_ctrlr;
2621
2622         /* Allocate memory for WRBQ */
2623         phwi_ctxt = phwi_ctrlr->phwi_ctxt;
2624         phwi_ctxt->be_wrbq = kzalloc(sizeof(struct be_queue_info) *
2625                                      phba->params.cxns_per_ctrl,
2626                                      GFP_KERNEL);
2627         if (!phwi_ctxt->be_wrbq) {
2628                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2629                             "BM_%d : WRBQ Mem Alloc Failed\n");
2630                 return -ENOMEM;
2631         }
2632
2633         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
2634                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2635                 pwrb_context->pwrb_handle_base =
2636                                 kzalloc(sizeof(struct wrb_handle *) *
2637                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2638                 if (!pwrb_context->pwrb_handle_base) {
2639                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2640                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2641                         goto init_wrb_hndl_failed;
2642                 }
2643                 pwrb_context->pwrb_handle_basestd =
2644                                 kzalloc(sizeof(struct wrb_handle *) *
2645                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2646                 if (!pwrb_context->pwrb_handle_basestd) {
2647                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2648                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2649                         goto init_wrb_hndl_failed;
2650                 }
2651                 if (!num_cxn_wrbh) {
2652                         pwrb_handle =
2653                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2654                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2655                                         ((sizeof(struct wrb_handle)) *
2656                                          phba->params.wrbs_per_cxn));
2657                         idx++;
2658                 }
2659                 pwrb_context->alloc_index = 0;
2660                 pwrb_context->wrb_handles_available = 0;
2661                 pwrb_context->free_index = 0;
2662
2663                 if (num_cxn_wrbh) {
2664                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2665                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2666                                 pwrb_context->pwrb_handle_basestd[j] =
2667                                                                 pwrb_handle;
2668                                 pwrb_context->wrb_handles_available++;
2669                                 pwrb_handle->wrb_index = j;
2670                                 pwrb_handle++;
2671                         }
2672                         num_cxn_wrbh--;
2673                 }
2674                 spin_lock_init(&pwrb_context->wrb_lock);
2675         }
2676         idx = 0;
2677         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
2678                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2679                 if (!num_cxn_wrb) {
2680                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2681                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2682                                 ((sizeof(struct iscsi_wrb) *
2683                                   phba->params.wrbs_per_cxn));
2684                         idx++;
2685                 }
2686
2687                 if (num_cxn_wrb) {
2688                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2689                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2690                                 pwrb_handle->pwrb = pwrb;
2691                                 pwrb++;
2692                         }
2693                         num_cxn_wrb--;
2694                 }
2695         }
2696         return 0;
2697 init_wrb_hndl_failed:
2698         for (j = index; j > 0; j--) {
2699                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2700                 kfree(pwrb_context->pwrb_handle_base);
2701                 kfree(pwrb_context->pwrb_handle_basestd);
2702         }
2703         return -ENOMEM;
2704 }
2705
2706 static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2707 {
2708         uint8_t ulp_num;
2709         struct hwi_controller *phwi_ctrlr;
2710         struct hba_parameters *p = &phba->params;
2711         struct hd_async_context *pasync_ctx;
2712         struct hd_async_handle *pasync_header_h, *pasync_data_h;
2713         unsigned int index, idx, num_per_mem, num_async_data;
2714         struct be_mem_descriptor *mem_descr;
2715
2716         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2717                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
2718                         /* get async_ctx for each ULP */
2719                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2720                         mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2721                                      (ulp_num * MEM_DESCR_OFFSET));
2722
2723                         phwi_ctrlr = phba->phwi_ctrlr;
2724                         phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] =
2725                                 (struct hd_async_context *)
2726                                  mem_descr->mem_array[0].virtual_address;
2727
2728                         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
2729                         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2730
2731                         pasync_ctx->async_entry =
2732                                         (struct hd_async_entry *)
2733                                         ((long unsigned int)pasync_ctx +
2734                                         sizeof(struct hd_async_context));
2735
2736                         pasync_ctx->num_entries = BEISCSI_ASYNC_HDQ_SIZE(phba,
2737                                                   ulp_num);
2738                         /* setup header buffers */
2739                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2740                         mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2741                                 (ulp_num * MEM_DESCR_OFFSET);
2742                         if (mem_descr->mem_array[0].virtual_address) {
2743                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2744                                             "BM_%d : hwi_init_async_pdu_ctx"
2745                                             " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n",
2746                                             ulp_num,
2747                                             mem_descr->mem_array[0].
2748                                             virtual_address);
2749                         } else
2750                                 beiscsi_log(phba, KERN_WARNING,
2751                                             BEISCSI_LOG_INIT,
2752                                             "BM_%d : No Virtual address for ULP : %d\n",
2753                                             ulp_num);
2754
2755                         pasync_ctx->async_header.pi = 0;
2756                         pasync_ctx->async_header.buffer_size = p->defpdu_hdr_sz;
2757                         pasync_ctx->async_header.va_base =
2758                                 mem_descr->mem_array[0].virtual_address;
2759
2760                         pasync_ctx->async_header.pa_base.u.a64.address =
2761                                 mem_descr->mem_array[0].
2762                                 bus_address.u.a64.address;
2763
2764                         /* setup header buffer sgls */
2765                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2766                         mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2767                                      (ulp_num * MEM_DESCR_OFFSET);
2768                         if (mem_descr->mem_array[0].virtual_address) {
2769                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2770                                             "BM_%d : hwi_init_async_pdu_ctx"
2771                                             " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n",
2772                                             ulp_num,
2773                                             mem_descr->mem_array[0].
2774                                             virtual_address);
2775                         } else
2776                                 beiscsi_log(phba, KERN_WARNING,
2777                                             BEISCSI_LOG_INIT,
2778                                             "BM_%d : No Virtual address for ULP : %d\n",
2779                                             ulp_num);
2780
2781                         pasync_ctx->async_header.ring_base =
2782                                 mem_descr->mem_array[0].virtual_address;
2783
2784                         /* setup header buffer handles */
2785                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2786                         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2787                                      (ulp_num * MEM_DESCR_OFFSET);
2788                         if (mem_descr->mem_array[0].virtual_address) {
2789                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2790                                             "BM_%d : hwi_init_async_pdu_ctx"
2791                                             " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n",
2792                                             ulp_num,
2793                                             mem_descr->mem_array[0].
2794                                             virtual_address);
2795                         } else
2796                                 beiscsi_log(phba, KERN_WARNING,
2797                                             BEISCSI_LOG_INIT,
2798                                             "BM_%d : No Virtual address for ULP : %d\n",
2799                                             ulp_num);
2800
2801                         pasync_ctx->async_header.handle_base =
2802                                 mem_descr->mem_array[0].virtual_address;
2803
2804                         /* setup data buffer sgls */
2805                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2806                         mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
2807                                      (ulp_num * MEM_DESCR_OFFSET);
2808                         if (mem_descr->mem_array[0].virtual_address) {
2809                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2810                                             "BM_%d : hwi_init_async_pdu_ctx"
2811                                             " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n",
2812                                             ulp_num,
2813                                             mem_descr->mem_array[0].
2814                                             virtual_address);
2815                         } else
2816                                 beiscsi_log(phba, KERN_WARNING,
2817                                             BEISCSI_LOG_INIT,
2818                                             "BM_%d : No Virtual address for ULP : %d\n",
2819                                             ulp_num);
2820
2821                         pasync_ctx->async_data.ring_base =
2822                                 mem_descr->mem_array[0].virtual_address;
2823
2824                         /* setup data buffer handles */
2825                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2826                         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
2827                                      (ulp_num * MEM_DESCR_OFFSET);
2828                         if (!mem_descr->mem_array[0].virtual_address)
2829                                 beiscsi_log(phba, KERN_WARNING,
2830                                             BEISCSI_LOG_INIT,
2831                                             "BM_%d : No Virtual address for ULP : %d\n",
2832                                             ulp_num);
2833
2834                         pasync_ctx->async_data.handle_base =
2835                                 mem_descr->mem_array[0].virtual_address;
2836
2837                         pasync_header_h =
2838                                 (struct hd_async_handle *)
2839                                 pasync_ctx->async_header.handle_base;
2840                         pasync_data_h =
2841                                 (struct hd_async_handle *)
2842                                 pasync_ctx->async_data.handle_base;
2843
2844                         /* setup data buffers */
2845                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2846                         mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 +
2847                                      (ulp_num * MEM_DESCR_OFFSET);
2848                         if (mem_descr->mem_array[0].virtual_address) {
2849                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2850                                             "BM_%d : hwi_init_async_pdu_ctx"
2851                                             " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n",
2852                                             ulp_num,
2853                                             mem_descr->mem_array[0].
2854                                             virtual_address);
2855                         } else
2856                                 beiscsi_log(phba, KERN_WARNING,
2857                                             BEISCSI_LOG_INIT,
2858                                             "BM_%d : No Virtual address for ULP : %d\n",
2859                                             ulp_num);
2860
2861                         idx = 0;
2862                         pasync_ctx->async_data.pi = 0;
2863                         pasync_ctx->async_data.buffer_size = p->defpdu_data_sz;
2864                         pasync_ctx->async_data.va_base =
2865                                 mem_descr->mem_array[idx].virtual_address;
2866                         pasync_ctx->async_data.pa_base.u.a64.address =
2867                                 mem_descr->mem_array[idx].
2868                                 bus_address.u.a64.address;
2869
2870                         num_async_data = ((mem_descr->mem_array[idx].size) /
2871                                         phba->params.defpdu_data_sz);
2872                         num_per_mem = 0;
2873
2874                         for (index = 0; index < BEISCSI_ASYNC_HDQ_SIZE
2875                                         (phba, ulp_num); index++) {
2876                                 pasync_header_h->cri = -1;
2877                                 pasync_header_h->is_header = 1;
2878                                 pasync_header_h->index = index;
2879                                 INIT_LIST_HEAD(&pasync_header_h->link);
2880                                 pasync_header_h->pbuffer =
2881                                         (void *)((unsigned long)
2882                                                  (pasync_ctx->
2883                                                   async_header.va_base) +
2884                                                  (p->defpdu_hdr_sz * index));
2885
2886                                 pasync_header_h->pa.u.a64.address =
2887                                         pasync_ctx->async_header.pa_base.u.a64.
2888                                         address + (p->defpdu_hdr_sz * index);
2889
2890                                 pasync_ctx->async_entry[index].header =
2891                                         pasync_header_h;
2892                                 pasync_header_h++;
2893                                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
2894                                                 wq.list);
2895
2896                                 pasync_data_h->cri = -1;
2897                                 pasync_data_h->is_header = 0;
2898                                 pasync_data_h->index = index;
2899                                 INIT_LIST_HEAD(&pasync_data_h->link);
2900
2901                                 if (!num_async_data) {
2902                                         num_per_mem = 0;
2903                                         idx++;
2904                                         pasync_ctx->async_data.va_base =
2905                                                 mem_descr->mem_array[idx].
2906                                                 virtual_address;
2907                                         pasync_ctx->async_data.pa_base.u.
2908                                                 a64.address =
2909                                                 mem_descr->mem_array[idx].
2910                                                 bus_address.u.a64.address;
2911                                         num_async_data =
2912                                                 ((mem_descr->mem_array[idx].
2913                                                   size) /
2914                                                  phba->params.defpdu_data_sz);
2915                                 }
2916                                 pasync_data_h->pbuffer =
2917                                         (void *)((unsigned long)
2918                                         (pasync_ctx->async_data.va_base) +
2919                                         (p->defpdu_data_sz * num_per_mem));
2920
2921                                 pasync_data_h->pa.u.a64.address =
2922                                         pasync_ctx->async_data.pa_base.u.a64.
2923                                         address + (p->defpdu_data_sz *
2924                                         num_per_mem);
2925                                 num_per_mem++;
2926                                 num_async_data--;
2927
2928                                 pasync_ctx->async_entry[index].data =
2929                                         pasync_data_h;
2930                                 pasync_data_h++;
2931                         }
2932                 }
2933         }
2934
2935         return 0;
2936 }
2937
2938 static int
2939 be_sgl_create_contiguous(void *virtual_address,
2940                          u64 physical_address, u32 length,
2941                          struct be_dma_mem *sgl)
2942 {
2943         WARN_ON(!virtual_address);
2944         WARN_ON(!physical_address);
2945         WARN_ON(!length);
2946         WARN_ON(!sgl);
2947
2948         sgl->va = virtual_address;
2949         sgl->dma = (unsigned long)physical_address;
2950         sgl->size = length;
2951
2952         return 0;
2953 }
2954
2955 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
2956 {
2957         memset(sgl, 0, sizeof(*sgl));
2958 }
2959
2960 static void
2961 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
2962                      struct mem_array *pmem, struct be_dma_mem *sgl)
2963 {
2964         if (sgl->va)
2965                 be_sgl_destroy_contiguous(sgl);
2966
2967         be_sgl_create_contiguous(pmem->virtual_address,
2968                                  pmem->bus_address.u.a64.address,
2969                                  pmem->size, sgl);
2970 }
2971
2972 static void
2973 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
2974                            struct mem_array *pmem, struct be_dma_mem *sgl)
2975 {
2976         if (sgl->va)
2977                 be_sgl_destroy_contiguous(sgl);
2978
2979         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
2980                                  pmem->bus_address.u.a64.address,
2981                                  pmem->size, sgl);
2982 }
2983
2984 static int be_fill_queue(struct be_queue_info *q,
2985                 u16 len, u16 entry_size, void *vaddress)
2986 {
2987         struct be_dma_mem *mem = &q->dma_mem;
2988
2989         memset(q, 0, sizeof(*q));
2990         q->len = len;
2991         q->entry_size = entry_size;
2992         mem->size = len * entry_size;
2993         mem->va = vaddress;
2994         if (!mem->va)
2995                 return -ENOMEM;
2996         memset(mem->va, 0, mem->size);
2997         return 0;
2998 }
2999
3000 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
3001                              struct hwi_context_memory *phwi_context)
3002 {
3003         int ret = -ENOMEM, eq_for_mcc;
3004         unsigned int i, num_eq_pages;
3005         struct be_queue_info *eq;
3006         struct be_dma_mem *mem;
3007         void *eq_vaddress;
3008         dma_addr_t paddr;
3009
3010         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3011                                       sizeof(struct be_eq_entry));
3012
3013         if (phba->pcidev->msix_enabled)
3014                 eq_for_mcc = 1;
3015         else
3016                 eq_for_mcc = 0;
3017         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3018                 eq = &phwi_context->be_eq[i].q;
3019                 mem = &eq->dma_mem;
3020                 phwi_context->be_eq[i].phba = phba;
3021                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3022                                                    num_eq_pages * PAGE_SIZE,
3023                                                    &paddr);
3024                 if (!eq_vaddress) {
3025                         ret = -ENOMEM;
3026                         goto create_eq_error;
3027                 }
3028
3029                 mem->va = eq_vaddress;
3030                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3031                                     sizeof(struct be_eq_entry), eq_vaddress);
3032                 if (ret) {
3033                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3034                                     "BM_%d : be_fill_queue Failed for EQ\n");
3035                         goto create_eq_error;
3036                 }
3037
3038                 mem->dma = paddr;
3039                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3040                                             BEISCSI_EQ_DELAY_DEF);
3041                 if (ret) {
3042                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3043                                     "BM_%d : beiscsi_cmd_eq_create"
3044                                     "Failed for EQ\n");
3045                         goto create_eq_error;
3046                 }
3047
3048                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3049                             "BM_%d : eqid = %d\n",
3050                             phwi_context->be_eq[i].q.id);
3051         }
3052         return 0;
3053
3054 create_eq_error:
3055         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3056                 eq = &phwi_context->be_eq[i].q;
3057                 mem = &eq->dma_mem;
3058                 if (mem->va)
3059                         pci_free_consistent(phba->pcidev, num_eq_pages
3060                                             * PAGE_SIZE,
3061                                             mem->va, mem->dma);
3062         }
3063         return ret;
3064 }
3065
3066 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
3067                              struct hwi_context_memory *phwi_context)
3068 {
3069         unsigned int i, num_cq_pages;
3070         struct be_queue_info *cq, *eq;
3071         struct be_dma_mem *mem;
3072         struct be_eq_obj *pbe_eq;
3073         void *cq_vaddress;
3074         int ret = -ENOMEM;
3075         dma_addr_t paddr;
3076
3077         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3078                                       sizeof(struct sol_cqe));
3079
3080         for (i = 0; i < phba->num_cpus; i++) {
3081                 cq = &phwi_context->be_cq[i];
3082                 eq = &phwi_context->be_eq[i].q;
3083                 pbe_eq = &phwi_context->be_eq[i];
3084                 pbe_eq->cq = cq;
3085                 pbe_eq->phba = phba;
3086                 mem = &cq->dma_mem;
3087                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3088                                                    num_cq_pages * PAGE_SIZE,
3089                                                    &paddr);
3090                 if (!cq_vaddress) {
3091                         ret = -ENOMEM;
3092                         goto create_cq_error;
3093                 }
3094
3095                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
3096                                     sizeof(struct sol_cqe), cq_vaddress);
3097                 if (ret) {
3098                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3099                                     "BM_%d : be_fill_queue Failed "
3100                                     "for ISCSI CQ\n");
3101                         goto create_cq_error;
3102                 }
3103
3104                 mem->dma = paddr;
3105                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3106                                             false, 0);
3107                 if (ret) {
3108                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3109                                     "BM_%d : beiscsi_cmd_eq_create"
3110                                     "Failed for ISCSI CQ\n");
3111                         goto create_cq_error;
3112                 }
3113                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3114                             "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3115                             "iSCSI CQ CREATED\n", cq->id, eq->id);
3116         }
3117         return 0;
3118
3119 create_cq_error:
3120         for (i = 0; i < phba->num_cpus; i++) {
3121                 cq = &phwi_context->be_cq[i];
3122                 mem = &cq->dma_mem;
3123                 if (mem->va)
3124                         pci_free_consistent(phba->pcidev, num_cq_pages
3125                                             * PAGE_SIZE,
3126                                             mem->va, mem->dma);
3127         }
3128         return ret;
3129 }
3130
3131 static int
3132 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3133                        struct hwi_context_memory *phwi_context,
3134                        struct hwi_controller *phwi_ctrlr,
3135                        unsigned int def_pdu_ring_sz, uint8_t ulp_num)
3136 {
3137         unsigned int idx;
3138         int ret;
3139         struct be_queue_info *dq, *cq;
3140         struct be_dma_mem *mem;
3141         struct be_mem_descriptor *mem_descr;
3142         void *dq_vaddress;
3143
3144         idx = 0;
3145         dq = &phwi_context->be_def_hdrq[ulp_num];
3146         cq = &phwi_context->be_cq[0];
3147         mem = &dq->dma_mem;
3148         mem_descr = phba->init_mem;
3149         mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
3150                     (ulp_num * MEM_DESCR_OFFSET);
3151         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3152         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3153                             sizeof(struct phys_addr),
3154                             sizeof(struct phys_addr), dq_vaddress);
3155         if (ret) {
3156                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3157                             "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n",
3158                             ulp_num);
3159
3160                 return ret;
3161         }
3162         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3163                                   bus_address.u.a64.address;
3164         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3165                                               def_pdu_ring_sz,
3166                                               phba->params.defpdu_hdr_sz,
3167                                               BEISCSI_DEFQ_HDR, ulp_num);
3168         if (ret) {
3169                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3170                             "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n",
3171                             ulp_num);
3172
3173                 return ret;
3174         }
3175
3176         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3177                     "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n",
3178                     ulp_num,
3179                     phwi_context->be_def_hdrq[ulp_num].id);
3180         return 0;
3181 }
3182
3183 static int
3184 beiscsi_create_def_data(struct beiscsi_hba *phba,
3185                         struct hwi_context_memory *phwi_context,
3186                         struct hwi_controller *phwi_ctrlr,
3187                         unsigned int def_pdu_ring_sz, uint8_t ulp_num)
3188 {
3189         unsigned int idx;
3190         int ret;
3191         struct be_queue_info *dataq, *cq;
3192         struct be_dma_mem *mem;
3193         struct be_mem_descriptor *mem_descr;
3194         void *dq_vaddress;
3195
3196         idx = 0;
3197         dataq = &phwi_context->be_def_dataq[ulp_num];
3198         cq = &phwi_context->be_cq[0];
3199         mem = &dataq->dma_mem;
3200         mem_descr = phba->init_mem;
3201         mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3202                     (ulp_num * MEM_DESCR_OFFSET);
3203         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3204         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3205                             sizeof(struct phys_addr),
3206                             sizeof(struct phys_addr), dq_vaddress);
3207         if (ret) {
3208                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3209                             "BM_%d : be_fill_queue Failed for DEF PDU "
3210                             "DATA on ULP : %d\n",
3211                             ulp_num);
3212
3213                 return ret;
3214         }
3215         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3216                                   bus_address.u.a64.address;
3217         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3218                                               def_pdu_ring_sz,
3219                                               phba->params.defpdu_data_sz,
3220                                               BEISCSI_DEFQ_DATA, ulp_num);
3221         if (ret) {
3222                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3223                             "BM_%d be_cmd_create_default_pdu_queue"
3224                             " Failed for DEF PDU DATA on ULP : %d\n",
3225                             ulp_num);
3226                 return ret;
3227         }
3228
3229         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3230                     "BM_%d : iscsi def data id on ULP : %d is  %d\n",
3231                     ulp_num,
3232                     phwi_context->be_def_dataq[ulp_num].id);
3233
3234         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3235                     "BM_%d : DEFAULT PDU DATA RING CREATED"
3236                     "on ULP : %d\n", ulp_num);
3237         return 0;
3238 }
3239
3240
3241 static int
3242 beiscsi_post_template_hdr(struct beiscsi_hba *phba)
3243 {
3244         struct be_mem_descriptor *mem_descr;
3245         struct mem_array *pm_arr;
3246         struct be_dma_mem sgl;
3247         int status, ulp_num;
3248
3249         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3250                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3251                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3252                         mem_descr += HWI_MEM_TEMPLATE_HDR_ULP0 +
3253                                     (ulp_num * MEM_DESCR_OFFSET);
3254                         pm_arr = mem_descr->mem_array;
3255
3256                         hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3257                         status = be_cmd_iscsi_post_template_hdr(
3258                                  &phba->ctrl, &sgl);
3259
3260                         if (status != 0) {
3261                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3262                                             "BM_%d : Post Template HDR Failed for"
3263                                             "ULP_%d\n", ulp_num);
3264                                 return status;
3265                         }
3266
3267                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3268                                     "BM_%d : Template HDR Pages Posted for"
3269                                     "ULP_%d\n", ulp_num);
3270                 }
3271         }
3272         return 0;
3273 }
3274
3275 static int
3276 beiscsi_post_pages(struct beiscsi_hba *phba)
3277 {
3278         struct be_mem_descriptor *mem_descr;
3279         struct mem_array *pm_arr;
3280         unsigned int page_offset, i;
3281         struct be_dma_mem sgl;
3282         int status, ulp_num = 0;
3283
3284         mem_descr = phba->init_mem;
3285         mem_descr += HWI_MEM_SGE;
3286         pm_arr = mem_descr->mem_array;
3287
3288         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3289                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3290                         break;
3291
3292         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
3293                         phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE;
3294         for (i = 0; i < mem_descr->num_elements; i++) {
3295                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3296                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3297                                                 page_offset,
3298                                                 (pm_arr->size / PAGE_SIZE));
3299                 page_offset += pm_arr->size / PAGE_SIZE;
3300                 if (status != 0) {
3301                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3302                                     "BM_%d : post sgl failed.\n");
3303                         return status;
3304                 }
3305                 pm_arr++;
3306         }
3307         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3308                     "BM_%d : POSTED PAGES\n");
3309         return 0;
3310 }
3311
3312 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3313 {
3314         struct be_dma_mem *mem = &q->dma_mem;
3315         if (mem->va) {
3316                 pci_free_consistent(phba->pcidev, mem->size,
3317                         mem->va, mem->dma);
3318                 mem->va = NULL;
3319         }
3320 }
3321
3322 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3323                 u16 len, u16 entry_size)
3324 {
3325         struct be_dma_mem *mem = &q->dma_mem;
3326
3327         memset(q, 0, sizeof(*q));
3328         q->len = len;
3329         q->entry_size = entry_size;
3330         mem->size = len * entry_size;
3331         mem->va = pci_zalloc_consistent(phba->pcidev, mem->size, &mem->dma);
3332         if (!mem->va)
3333                 return -ENOMEM;
3334         return 0;
3335 }
3336
3337 static int
3338 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3339                          struct hwi_context_memory *phwi_context,
3340                          struct hwi_controller *phwi_ctrlr)
3341 {
3342         unsigned int num_wrb_rings;
3343         u64 pa_addr_lo;
3344         unsigned int idx, num, i, ulp_num;
3345         struct mem_array *pwrb_arr;
3346         void *wrb_vaddr;
3347         struct be_dma_mem sgl;
3348         struct be_mem_descriptor *mem_descr;
3349         struct hwi_wrb_context *pwrb_context;
3350         int status;
3351         uint8_t ulp_count = 0, ulp_base_num = 0;
3352         uint16_t cid_count_ulp[BEISCSI_ULP_COUNT] = { 0 };
3353
3354         idx = 0;
3355         mem_descr = phba->init_mem;
3356         mem_descr += HWI_MEM_WRB;
3357         pwrb_arr = kmalloc_array(phba->params.cxns_per_ctrl,
3358                                  sizeof(*pwrb_arr),
3359                                  GFP_KERNEL);
3360         if (!pwrb_arr) {
3361                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3362                             "BM_%d : Memory alloc failed in create wrb ring.\n");
3363                 return -ENOMEM;
3364         }
3365         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3366         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3367         num_wrb_rings = mem_descr->mem_array[idx].size /
3368                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3369
3370         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3371                 if (num_wrb_rings) {
3372                         pwrb_arr[num].virtual_address = wrb_vaddr;
3373                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3374                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3375                                             sizeof(struct iscsi_wrb);
3376                         wrb_vaddr += pwrb_arr[num].size;
3377                         pa_addr_lo += pwrb_arr[num].size;
3378                         num_wrb_rings--;
3379                 } else {
3380                         idx++;
3381                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3382                         pa_addr_lo = mem_descr->mem_array[idx].\
3383                                         bus_address.u.a64.address;
3384                         num_wrb_rings = mem_descr->mem_array[idx].size /
3385                                         (phba->params.wrbs_per_cxn *
3386                                         sizeof(struct iscsi_wrb));
3387                         pwrb_arr[num].virtual_address = wrb_vaddr;
3388                         pwrb_arr[num].bus_address.u.a64.address\
3389                                                 = pa_addr_lo;
3390                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3391                                                  sizeof(struct iscsi_wrb);
3392                         wrb_vaddr += pwrb_arr[num].size;
3393                         pa_addr_lo   += pwrb_arr[num].size;
3394                         num_wrb_rings--;
3395                 }
3396         }
3397
3398         /* Get the ULP Count */
3399         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3400                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3401                         ulp_count++;
3402                         ulp_base_num = ulp_num;
3403                         cid_count_ulp[ulp_num] =
3404                                 BEISCSI_GET_CID_COUNT(phba, ulp_num);
3405                 }
3406
3407         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3408                 if (ulp_count > 1) {
3409                         ulp_base_num = (ulp_base_num + 1) % BEISCSI_ULP_COUNT;
3410
3411                         if (!cid_count_ulp[ulp_base_num])
3412                                 ulp_base_num = (ulp_base_num + 1) %
3413                                                 BEISCSI_ULP_COUNT;
3414
3415                         cid_count_ulp[ulp_base_num]--;
3416                 }
3417
3418
3419                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3420                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3421                                             &phwi_context->be_wrbq[i],
3422                                             &phwi_ctrlr->wrb_context[i],
3423                                             ulp_base_num);
3424                 if (status != 0) {
3425                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3426                                     "BM_%d : wrbq create failed.");
3427                         kfree(pwrb_arr);
3428                         return status;
3429                 }
3430                 pwrb_context = &phwi_ctrlr->wrb_context[i];
3431                 BE_SET_CID_TO_CRI(i, pwrb_context->cid);
3432         }
3433         kfree(pwrb_arr);
3434         return 0;
3435 }
3436
3437 static void free_wrb_handles(struct beiscsi_hba *phba)
3438 {
3439         unsigned int index;
3440         struct hwi_controller *phwi_ctrlr;
3441         struct hwi_wrb_context *pwrb_context;
3442
3443         phwi_ctrlr = phba->phwi_ctrlr;
3444         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
3445                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3446                 kfree(pwrb_context->pwrb_handle_base);
3447                 kfree(pwrb_context->pwrb_handle_basestd);
3448         }
3449 }
3450
3451 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3452 {
3453         struct be_ctrl_info *ctrl = &phba->ctrl;
3454         struct be_dma_mem *ptag_mem;
3455         struct be_queue_info *q;
3456         int i, tag;
3457
3458         q = &phba->ctrl.mcc_obj.q;
3459         for (i = 0; i < MAX_MCC_CMD; i++) {
3460                 tag = i + 1;
3461                 if (!test_bit(MCC_TAG_STATE_RUNNING,
3462                               &ctrl->ptag_state[tag].tag_state))
3463                         continue;
3464
3465                 if (test_bit(MCC_TAG_STATE_TIMEOUT,
3466                              &ctrl->ptag_state[tag].tag_state)) {
3467                         ptag_mem = &ctrl->ptag_state[tag].tag_mem_state;
3468                         if (ptag_mem->size) {
3469                                 pci_free_consistent(ctrl->pdev,
3470                                                     ptag_mem->size,
3471                                                     ptag_mem->va,
3472                                                     ptag_mem->dma);
3473                                 ptag_mem->size = 0;
3474                         }
3475                         continue;
3476                 }
3477                 /**
3478                  * If MCC is still active and waiting then wake up the process.
3479                  * We are here only because port is going offline. The process
3480                  * sees that (BEISCSI_HBA_ONLINE is cleared) and EIO error is
3481                  * returned for the operation and allocated memory cleaned up.
3482                  */
3483                 if (waitqueue_active(&ctrl->mcc_wait[tag])) {
3484                         ctrl->mcc_tag_status[tag] = MCC_STATUS_FAILED;
3485                         ctrl->mcc_tag_status[tag] |= CQE_VALID_MASK;
3486                         wake_up_interruptible(&ctrl->mcc_wait[tag]);
3487                         /*
3488                          * Control tag info gets reinitialized in enable
3489                          * so wait for the process to clear running state.
3490                          */
3491                         while (test_bit(MCC_TAG_STATE_RUNNING,
3492                                         &ctrl->ptag_state[tag].tag_state))
3493                                 schedule_timeout_uninterruptible(HZ);
3494                 }
3495                 /**
3496                  * For MCC with tag_states MCC_TAG_STATE_ASYNC and
3497                  * MCC_TAG_STATE_IGNORE nothing needs to done.
3498                  */
3499         }
3500         if (q->created) {
3501                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3502                 be_queue_free(phba, q);
3503         }
3504
3505         q = &phba->ctrl.mcc_obj.cq;
3506         if (q->created) {
3507                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3508                 be_queue_free(phba, q);
3509         }
3510 }
3511
3512 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3513                                 struct hwi_context_memory *phwi_context)
3514 {
3515         struct be_queue_info *q, *cq;
3516         struct be_ctrl_info *ctrl = &phba->ctrl;
3517
3518         /* Alloc MCC compl queue */
3519         cq = &phba->ctrl.mcc_obj.cq;
3520         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3521                         sizeof(struct be_mcc_compl)))
3522                 goto err;
3523         /* Ask BE to create MCC compl queue; */
3524         if (phba->pcidev->msix_enabled) {
3525                 if (beiscsi_cmd_cq_create(ctrl, cq,
3526                                         &phwi_context->be_eq[phba->num_cpus].q,
3527                                         false, true, 0))
3528                         goto mcc_cq_free;
3529         } else {
3530                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3531                                           false, true, 0))
3532                         goto mcc_cq_free;
3533         }
3534
3535         /* Alloc MCC queue */
3536         q = &phba->ctrl.mcc_obj.q;
3537         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3538                 goto mcc_cq_destroy;
3539
3540         /* Ask BE to create MCC queue */
3541         if (beiscsi_cmd_mccq_create(phba, q, cq))
3542                 goto mcc_q_free;
3543
3544         return 0;
3545
3546 mcc_q_free:
3547         be_queue_free(phba, q);
3548 mcc_cq_destroy:
3549         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3550 mcc_cq_free:
3551         be_queue_free(phba, cq);
3552 err:
3553         return -ENOMEM;
3554 }
3555
3556 static void be2iscsi_enable_msix(struct beiscsi_hba *phba)
3557 {
3558         int nvec = 1;
3559
3560         switch (phba->generation) {
3561         case BE_GEN2:
3562         case BE_GEN3:
3563                 nvec = BEISCSI_MAX_NUM_CPUS + 1;
3564                 break;
3565         case BE_GEN4:
3566                 nvec = phba->fw_config.eqid_count;
3567                 break;
3568         default:
3569                 nvec = 2;
3570                 break;
3571         }
3572
3573         /* if eqid_count == 1 fall back to INTX */
3574         if (enable_msix && nvec > 1) {
3575                 const struct irq_affinity desc = { .post_vectors = 1 };
3576
3577                 if (pci_alloc_irq_vectors_affinity(phba->pcidev, 2, nvec,
3578                                 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, &desc) < 0) {
3579                         phba->num_cpus = nvec - 1;
3580                         return;
3581                 }
3582         }
3583
3584         phba->num_cpus = 1;
3585 }
3586
3587 static void hwi_purge_eq(struct beiscsi_hba *phba)
3588 {
3589         struct hwi_controller *phwi_ctrlr;
3590         struct hwi_context_memory *phwi_context;
3591         struct be_queue_info *eq;
3592         struct be_eq_entry *eqe = NULL;
3593         int i, eq_msix;
3594         unsigned int num_processed;
3595
3596         if (beiscsi_hba_in_error(phba))
3597                 return;
3598
3599         phwi_ctrlr = phba->phwi_ctrlr;
3600         phwi_context = phwi_ctrlr->phwi_ctxt;
3601         if (phba->pcidev->msix_enabled)
3602                 eq_msix = 1;
3603         else
3604                 eq_msix = 0;
3605
3606         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
3607                 eq = &phwi_context->be_eq[i].q;
3608                 eqe = queue_tail_node(eq);
3609                 num_processed = 0;
3610                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
3611                                         & EQE_VALID_MASK) {
3612                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
3613                         queue_tail_inc(eq);
3614                         eqe = queue_tail_node(eq);
3615                         num_processed++;
3616                 }
3617
3618                 if (num_processed)
3619                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
3620         }
3621 }
3622
3623 static void hwi_cleanup_port(struct beiscsi_hba *phba)
3624 {
3625         struct be_queue_info *q;
3626         struct be_ctrl_info *ctrl = &phba->ctrl;
3627         struct hwi_controller *phwi_ctrlr;
3628         struct hwi_context_memory *phwi_context;
3629         int i, eq_for_mcc, ulp_num;
3630
3631         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3632                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3633                         beiscsi_cmd_iscsi_cleanup(phba, ulp_num);
3634
3635         /**
3636          * Purge all EQ entries that may have been left out. This is to
3637          * workaround a problem we've seen occasionally where driver gets an
3638          * interrupt with EQ entry bit set after stopping the controller.
3639          */
3640         hwi_purge_eq(phba);
3641
3642         phwi_ctrlr = phba->phwi_ctrlr;
3643         phwi_context = phwi_ctrlr->phwi_ctxt;
3644
3645         be_cmd_iscsi_remove_template_hdr(ctrl);
3646
3647         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3648                 q = &phwi_context->be_wrbq[i];
3649                 if (q->created)
3650                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3651         }
3652         kfree(phwi_context->be_wrbq);
3653         free_wrb_handles(phba);
3654
3655         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3656                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3657
3658                         q = &phwi_context->be_def_hdrq[ulp_num];
3659                         if (q->created)
3660                                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3661
3662                         q = &phwi_context->be_def_dataq[ulp_num];
3663                         if (q->created)
3664                                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3665                 }
3666         }
3667
3668         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3669
3670         for (i = 0; i < (phba->num_cpus); i++) {
3671                 q = &phwi_context->be_cq[i];
3672                 if (q->created) {
3673                         be_queue_free(phba, q);
3674                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3675                 }
3676         }
3677
3678         be_mcc_queues_destroy(phba);
3679         if (phba->pcidev->msix_enabled)
3680                 eq_for_mcc = 1;
3681         else
3682                 eq_for_mcc = 0;
3683         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3684                 q = &phwi_context->be_eq[i].q;
3685                 if (q->created) {
3686                         be_queue_free(phba, q);
3687                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3688                 }
3689         }
3690         /* this ensures complete FW cleanup */
3691         beiscsi_cmd_function_reset(phba);
3692         /* last communication, indicate driver is unloading */
3693         beiscsi_cmd_special_wrb(&phba->ctrl, 0);
3694 }
3695
3696 static int hwi_init_port(struct beiscsi_hba *phba)
3697 {
3698         struct hwi_controller *phwi_ctrlr;
3699         struct hwi_context_memory *phwi_context;
3700         unsigned int def_pdu_ring_sz;
3701         struct be_ctrl_info *ctrl = &phba->ctrl;
3702         int status, ulp_num;
3703         u16 nbufs;
3704
3705         phwi_ctrlr = phba->phwi_ctrlr;
3706         phwi_context = phwi_ctrlr->phwi_ctxt;
3707         /* set port optic state to unknown */
3708         phba->optic_state = 0xff;
3709
3710         status = beiscsi_create_eqs(phba, phwi_context);
3711         if (status != 0) {
3712                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3713                             "BM_%d : EQ not created\n");
3714                 goto error;
3715         }
3716
3717         status = be_mcc_queues_create(phba, phwi_context);
3718         if (status != 0)
3719                 goto error;
3720
3721         status = beiscsi_check_supported_fw(ctrl, phba);
3722         if (status != 0) {
3723                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3724                             "BM_%d : Unsupported fw version\n");
3725                 goto error;
3726         }
3727
3728         status = beiscsi_create_cqs(phba, phwi_context);
3729         if (status != 0) {
3730                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3731                             "BM_%d : CQ not created\n");
3732                 goto error;
3733         }
3734
3735         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3736                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3737                         nbufs = phwi_context->pasync_ctx[ulp_num]->num_entries;
3738                         def_pdu_ring_sz = nbufs * sizeof(struct phys_addr);
3739
3740                         status = beiscsi_create_def_hdr(phba, phwi_context,
3741                                                         phwi_ctrlr,
3742                                                         def_pdu_ring_sz,
3743                                                         ulp_num);
3744                         if (status != 0) {
3745                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3746                                             "BM_%d : Default Header not created for ULP : %d\n",
3747                                             ulp_num);
3748                                 goto error;
3749                         }
3750
3751                         status = beiscsi_create_def_data(phba, phwi_context,
3752                                                          phwi_ctrlr,
3753                                                          def_pdu_ring_sz,
3754                                                          ulp_num);
3755                         if (status != 0) {
3756                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3757                                             "BM_%d : Default Data not created for ULP : %d\n",
3758                                             ulp_num);
3759                                 goto error;
3760                         }
3761                         /**
3762                          * Now that the default PDU rings have been created,
3763                          * let EP know about it.
3764                          */
3765                         beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_HDR,
3766                                                  ulp_num, nbufs);
3767                         beiscsi_hdq_post_handles(phba, BEISCSI_DEFQ_DATA,
3768                                                  ulp_num, nbufs);
3769                 }
3770         }
3771
3772         status = beiscsi_post_pages(phba);
3773         if (status != 0) {
3774                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3775                             "BM_%d : Post SGL Pages Failed\n");
3776                 goto error;
3777         }
3778
3779         status = beiscsi_post_template_hdr(phba);
3780         if (status != 0) {
3781                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3782                             "BM_%d : Template HDR Posting for CXN Failed\n");
3783         }
3784
3785         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3786         if (status != 0) {
3787                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3788                             "BM_%d : WRB Rings not created\n");
3789                 goto error;
3790         }
3791
3792         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3793                 uint16_t async_arr_idx = 0;
3794
3795                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3796                         uint16_t cri = 0;
3797                         struct hd_async_context *pasync_ctx;
3798
3799                         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(
3800                                      phwi_ctrlr, ulp_num);
3801                         for (cri = 0; cri <
3802                              phba->params.cxns_per_ctrl; cri++) {
3803                                 if (ulp_num == BEISCSI_GET_ULP_FROM_CRI
3804                                                (phwi_ctrlr, cri))
3805                                         pasync_ctx->cid_to_async_cri_map[
3806                                         phwi_ctrlr->wrb_context[cri].cid] =
3807                                         async_arr_idx++;
3808                         }
3809                 }
3810         }
3811
3812         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3813                     "BM_%d : hwi_init_port success\n");
3814         return 0;
3815
3816 error:
3817         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3818                     "BM_%d : hwi_init_port failed");
3819         hwi_cleanup_port(phba);
3820         return status;
3821 }
3822
3823 static int hwi_init_controller(struct beiscsi_hba *phba)
3824 {
3825         struct hwi_controller *phwi_ctrlr;
3826
3827         phwi_ctrlr = phba->phwi_ctrlr;
3828         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3829                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3830                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3831                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3832                             "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
3833                             phwi_ctrlr->phwi_ctxt);
3834         } else {
3835                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3836                             "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3837                             "than one element.Failing to load\n");
3838                 return -ENOMEM;
3839         }
3840
3841         iscsi_init_global_templates(phba);
3842         if (beiscsi_init_wrb_handle(phba))
3843                 return -ENOMEM;
3844
3845         if (hwi_init_async_pdu_ctx(phba)) {
3846                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3847                             "BM_%d : hwi_init_async_pdu_ctx failed\n");
3848                 return -ENOMEM;
3849         }
3850
3851         if (hwi_init_port(phba) != 0) {
3852                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3853                             "BM_%d : hwi_init_controller failed\n");
3854
3855                 return -ENOMEM;
3856         }
3857         return 0;
3858 }
3859
3860 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3861 {
3862         struct be_mem_descriptor *mem_descr;
3863         int i, j;
3864
3865         mem_descr = phba->init_mem;
3866         i = 0;
3867         j = 0;
3868         for (i = 0; i < SE_MEM_MAX; i++) {
3869                 for (j = mem_descr->num_elements; j > 0; j--) {
3870                         pci_free_consistent(phba->pcidev,
3871                           mem_descr->mem_array[j - 1].size,
3872                           mem_descr->mem_array[j - 1].virtual_address,
3873                           (unsigned long)mem_descr->mem_array[j - 1].
3874                           bus_address.u.a64.address);
3875                 }
3876
3877                 kfree(mem_descr->mem_array);
3878                 mem_descr++;
3879         }
3880         kfree(phba->init_mem);
3881         kfree(phba->phwi_ctrlr->wrb_context);
3882         kfree(phba->phwi_ctrlr);
3883 }
3884
3885 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
3886 {
3887         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
3888         struct sgl_handle *psgl_handle;
3889         struct iscsi_sge *pfrag;
3890         unsigned int arr_index, i, idx;
3891         unsigned int ulp_icd_start, ulp_num = 0;
3892
3893         phba->io_sgl_hndl_avbl = 0;
3894         phba->eh_sgl_hndl_avbl = 0;
3895
3896         mem_descr_sglh = phba->init_mem;
3897         mem_descr_sglh += HWI_MEM_SGLH;
3898         if (1 == mem_descr_sglh->num_elements) {
3899                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3900                                                  phba->params.ios_per_ctrl,
3901                                                  GFP_KERNEL);
3902                 if (!phba->io_sgl_hndl_base) {
3903                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3904                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3905                         return -ENOMEM;
3906                 }
3907                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
3908                                                  (phba->params.icds_per_ctrl -
3909                                                  phba->params.ios_per_ctrl),
3910                                                  GFP_KERNEL);
3911                 if (!phba->eh_sgl_hndl_base) {
3912                         kfree(phba->io_sgl_hndl_base);
3913                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3914                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
3915                         return -ENOMEM;
3916                 }
3917         } else {
3918                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3919                             "BM_%d : HWI_MEM_SGLH is more than one element."
3920                             "Failing to load\n");
3921                 return -ENOMEM;
3922         }
3923
3924         arr_index = 0;
3925         idx = 0;
3926         while (idx < mem_descr_sglh->num_elements) {
3927                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
3928
3929                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
3930                       sizeof(struct sgl_handle)); i++) {
3931                         if (arr_index < phba->params.ios_per_ctrl) {
3932                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
3933                                 phba->io_sgl_hndl_avbl++;
3934                                 arr_index++;
3935                         } else {
3936                                 phba->eh_sgl_hndl_base[arr_index -
3937                                         phba->params.ios_per_ctrl] =
3938                                                                 psgl_handle;
3939                                 arr_index++;
3940                                 phba->eh_sgl_hndl_avbl++;
3941                         }
3942                         psgl_handle++;
3943                 }
3944                 idx++;
3945         }
3946         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3947                     "BM_%d : phba->io_sgl_hndl_avbl=%d"
3948                     "phba->eh_sgl_hndl_avbl=%d\n",
3949                     phba->io_sgl_hndl_avbl,
3950                     phba->eh_sgl_hndl_avbl);
3951
3952         mem_descr_sg = phba->init_mem;
3953         mem_descr_sg += HWI_MEM_SGE;
3954         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3955                     "\n BM_%d : mem_descr_sg->num_elements=%d\n",
3956                     mem_descr_sg->num_elements);
3957
3958         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3959                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3960                         break;
3961
3962         ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
3963
3964         arr_index = 0;
3965         idx = 0;
3966         while (idx < mem_descr_sg->num_elements) {
3967                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
3968
3969                 for (i = 0;
3970                      i < (mem_descr_sg->mem_array[idx].size) /
3971                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
3972                      i++) {
3973                         if (arr_index < phba->params.ios_per_ctrl)
3974                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
3975                         else
3976                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
3977                                                 phba->params.ios_per_ctrl];
3978                         psgl_handle->pfrag = pfrag;
3979                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
3980                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
3981                         pfrag += phba->params.num_sge_per_io;
3982                         psgl_handle->sgl_index = ulp_icd_start + arr_index++;
3983                 }
3984                 idx++;
3985         }
3986         phba->io_sgl_free_index = 0;
3987         phba->io_sgl_alloc_index = 0;
3988         phba->eh_sgl_free_index = 0;
3989         phba->eh_sgl_alloc_index = 0;
3990         return 0;
3991 }
3992
3993 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
3994 {
3995         int ret;
3996         uint16_t i, ulp_num;
3997         struct ulp_cid_info *ptr_cid_info = NULL;
3998
3999         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4000                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4001                         ptr_cid_info = kzalloc(sizeof(struct ulp_cid_info),
4002                                                GFP_KERNEL);
4003
4004                         if (!ptr_cid_info) {
4005                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4006                                             "BM_%d : Failed to allocate memory"
4007                                             "for ULP_CID_INFO for ULP : %d\n",
4008                                             ulp_num);
4009                                 ret = -ENOMEM;
4010                                 goto free_memory;
4011
4012                         }
4013
4014                         /* Allocate memory for CID array */
4015                         ptr_cid_info->cid_array =
4016                                 kcalloc(BEISCSI_GET_CID_COUNT(phba, ulp_num),
4017                                         sizeof(*ptr_cid_info->cid_array),
4018                                         GFP_KERNEL);
4019                         if (!ptr_cid_info->cid_array) {
4020                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4021                                             "BM_%d : Failed to allocate memory"
4022                                             "for CID_ARRAY for ULP : %d\n",
4023                                             ulp_num);
4024                                 kfree(ptr_cid_info);
4025                                 ptr_cid_info = NULL;
4026                                 ret = -ENOMEM;
4027
4028                                 goto free_memory;
4029                         }
4030                         ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT(
4031                                                    phba, ulp_num);
4032
4033                         /* Save the cid_info_array ptr */
4034                         phba->cid_array_info[ulp_num] = ptr_cid_info;
4035                 }
4036         }
4037         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
4038                                  phba->params.cxns_per_ctrl, GFP_KERNEL);
4039         if (!phba->ep_array) {
4040                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4041                             "BM_%d : Failed to allocate memory in "
4042                             "hba_setup_cid_tbls\n");
4043                 ret = -ENOMEM;
4044
4045                 goto free_memory;
4046         }
4047
4048         phba->conn_table = kzalloc(sizeof(struct beiscsi_conn *) *
4049                                    phba->params.cxns_per_ctrl, GFP_KERNEL);
4050         if (!phba->conn_table) {
4051                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4052                             "BM_%d : Failed to allocate memory in"
4053                             "hba_setup_cid_tbls\n");
4054
4055                 kfree(phba->ep_array);
4056                 phba->ep_array = NULL;
4057                 ret = -ENOMEM;
4058
4059                 goto free_memory;
4060         }
4061
4062         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
4063                 ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num;
4064
4065                 ptr_cid_info = phba->cid_array_info[ulp_num];
4066                 ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] =
4067                         phba->phwi_ctrlr->wrb_context[i].cid;
4068
4069         }
4070
4071         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4072                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4073                         ptr_cid_info = phba->cid_array_info[ulp_num];
4074
4075                         ptr_cid_info->cid_alloc = 0;
4076                         ptr_cid_info->cid_free = 0;
4077                 }
4078         }
4079         return 0;
4080
4081 free_memory:
4082         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4083                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4084                         ptr_cid_info = phba->cid_array_info[ulp_num];
4085
4086                         if (ptr_cid_info) {
4087                                 kfree(ptr_cid_info->cid_array);
4088                                 kfree(ptr_cid_info);
4089                                 phba->cid_array_info[ulp_num] = NULL;
4090                         }
4091                 }
4092         }
4093
4094         return ret;
4095 }
4096
4097 static void hwi_enable_intr(struct beiscsi_hba *phba)
4098 {
4099         struct be_ctrl_info *ctrl = &phba->ctrl;
4100         struct hwi_controller *phwi_ctrlr;
4101         struct hwi_context_memory *phwi_context;
4102         struct be_queue_info *eq;
4103         u8 __iomem *addr;
4104         u32 reg, i;
4105         u32 enabled;
4106
4107         phwi_ctrlr = phba->phwi_ctrlr;
4108         phwi_context = phwi_ctrlr->phwi_ctxt;
4109
4110         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
4111                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
4112         reg = ioread32(addr);
4113
4114         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4115         if (!enabled) {
4116                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4117                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4118                             "BM_%d : reg =x%08x addr=%p\n", reg, addr);
4119                 iowrite32(reg, addr);
4120         }
4121
4122         if (!phba->pcidev->msix_enabled) {
4123                 eq = &phwi_context->be_eq[0].q;
4124                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4125                             "BM_%d : eq->id=%d\n", eq->id);
4126
4127                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4128         } else {
4129                 for (i = 0; i <= phba->num_cpus; i++) {
4130                         eq = &phwi_context->be_eq[i].q;
4131                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4132                                     "BM_%d : eq->id=%d\n", eq->id);
4133                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4134                 }
4135         }
4136 }
4137
4138 static void hwi_disable_intr(struct beiscsi_hba *phba)
4139 {
4140         struct be_ctrl_info *ctrl = &phba->ctrl;
4141
4142         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
4143         u32 reg = ioread32(addr);
4144
4145         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4146         if (enabled) {
4147                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4148                 iowrite32(reg, addr);
4149         } else
4150                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4151                             "BM_%d : In hwi_disable_intr, Already Disabled\n");
4152 }
4153
4154 static int beiscsi_init_port(struct beiscsi_hba *phba)
4155 {
4156         int ret;
4157
4158         ret = hwi_init_controller(phba);
4159         if (ret < 0) {
4160                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4161                             "BM_%d : init controller failed\n");
4162                 return ret;
4163         }
4164         ret = beiscsi_init_sgl_handle(phba);
4165         if (ret < 0) {
4166                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4167                             "BM_%d : init sgl handles failed\n");
4168                 goto cleanup_port;
4169         }
4170
4171         ret = hba_setup_cid_tbls(phba);
4172         if (ret < 0) {
4173                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4174                             "BM_%d : setup CID table failed\n");
4175                 kfree(phba->io_sgl_hndl_base);
4176                 kfree(phba->eh_sgl_hndl_base);
4177                 goto cleanup_port;
4178         }
4179         return ret;
4180
4181 cleanup_port:
4182         hwi_cleanup_port(phba);
4183         return ret;
4184 }
4185
4186 static void beiscsi_cleanup_port(struct beiscsi_hba *phba)
4187 {
4188         struct ulp_cid_info *ptr_cid_info = NULL;
4189         int ulp_num;
4190
4191         kfree(phba->io_sgl_hndl_base);
4192         kfree(phba->eh_sgl_hndl_base);
4193         kfree(phba->ep_array);
4194         kfree(phba->conn_table);
4195
4196         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4197                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4198                         ptr_cid_info = phba->cid_array_info[ulp_num];
4199
4200                         if (ptr_cid_info) {
4201                                 kfree(ptr_cid_info->cid_array);
4202                                 kfree(ptr_cid_info);
4203                                 phba->cid_array_info[ulp_num] = NULL;
4204                         }
4205                 }
4206         }
4207 }
4208
4209 /**
4210  * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4211  * @beiscsi_conn: ptr to the conn to be cleaned up
4212  * @task: ptr to iscsi_task resource to be freed.
4213  *
4214  * Free driver mgmt resources binded to CXN.
4215  **/
4216 void
4217 beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn,
4218                                 struct iscsi_task *task)
4219 {
4220         struct beiscsi_io_task *io_task;
4221         struct beiscsi_hba *phba = beiscsi_conn->phba;
4222         struct hwi_wrb_context *pwrb_context;
4223         struct hwi_controller *phwi_ctrlr;
4224         uint16_t cri_index = BE_GET_CRI_FROM_CID(
4225                                 beiscsi_conn->beiscsi_conn_cid);
4226
4227         phwi_ctrlr = phba->phwi_ctrlr;
4228         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4229
4230         io_task = task->dd_data;
4231
4232         if (io_task->pwrb_handle) {
4233                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4234                 io_task->pwrb_handle = NULL;
4235         }
4236
4237         if (io_task->psgl_handle) {
4238                 free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4239                 io_task->psgl_handle = NULL;
4240         }
4241
4242         if (io_task->mtask_addr) {
4243                 pci_unmap_single(phba->pcidev,
4244                                  io_task->mtask_addr,
4245                                  io_task->mtask_data_count,
4246                                  PCI_DMA_TODEVICE);
4247                 io_task->mtask_addr = 0;
4248         }
4249 }
4250
4251 /**
4252  * beiscsi_cleanup_task()- Free driver resources of the task
4253  * @task: ptr to the iscsi task
4254  *
4255  **/
4256 static void beiscsi_cleanup_task(struct iscsi_task *task)
4257 {
4258         struct beiscsi_io_task *io_task = task->dd_data;
4259         struct iscsi_conn *conn = task->conn;
4260         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4261         struct beiscsi_hba *phba = beiscsi_conn->phba;
4262         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4263         struct hwi_wrb_context *pwrb_context;
4264         struct hwi_controller *phwi_ctrlr;
4265         uint16_t cri_index = BE_GET_CRI_FROM_CID(
4266                              beiscsi_conn->beiscsi_conn_cid);
4267
4268         phwi_ctrlr = phba->phwi_ctrlr;
4269         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4270
4271         if (io_task->cmd_bhs) {
4272                 dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4273                               io_task->bhs_pa.u.a64.address);
4274                 io_task->cmd_bhs = NULL;
4275                 task->hdr = NULL;
4276         }
4277
4278         if (task->sc) {
4279                 if (io_task->pwrb_handle) {
4280                         free_wrb_handle(phba, pwrb_context,
4281                                         io_task->pwrb_handle);
4282                         io_task->pwrb_handle = NULL;
4283                 }
4284
4285                 if (io_task->psgl_handle) {
4286                         free_io_sgl_handle(phba, io_task->psgl_handle);
4287                         io_task->psgl_handle = NULL;
4288                 }
4289
4290                 if (io_task->scsi_cmnd) {
4291                         if (io_task->num_sg)
4292                                 scsi_dma_unmap(io_task->scsi_cmnd);
4293                         io_task->scsi_cmnd = NULL;
4294                 }
4295         } else {
4296                 if (!beiscsi_conn->login_in_progress)
4297                         beiscsi_free_mgmt_task_handles(beiscsi_conn, task);
4298         }
4299 }
4300
4301 void
4302 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4303                            struct beiscsi_offload_params *params)
4304 {
4305         struct wrb_handle *pwrb_handle;
4306         struct hwi_wrb_context *pwrb_context = NULL;
4307         struct beiscsi_hba *phba = beiscsi_conn->phba;
4308         struct iscsi_task *task = beiscsi_conn->task;
4309         struct iscsi_session *session = task->conn->session;
4310         u32 doorbell = 0;
4311
4312         /*
4313          * We can always use 0 here because it is reserved by libiscsi for
4314          * login/startup related tasks.
4315          */
4316         beiscsi_conn->login_in_progress = 0;
4317         spin_lock_bh(&session->back_lock);
4318         beiscsi_cleanup_task(task);
4319         spin_unlock_bh(&session->back_lock);
4320
4321         pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid,
4322                                        &pwrb_context);
4323
4324         /* Check for the adapter family */
4325         if (is_chip_be2_be3r(phba))
4326                 beiscsi_offload_cxn_v0(params, pwrb_handle,
4327                                        phba->init_mem,
4328                                        pwrb_context);
4329         else
4330                 beiscsi_offload_cxn_v2(params, pwrb_handle,
4331                                        pwrb_context);
4332
4333         be_dws_le_to_cpu(pwrb_handle->pwrb,
4334                          sizeof(struct iscsi_target_context_update_wrb));
4335
4336         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4337         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
4338                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
4339         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4340         iowrite32(doorbell, phba->db_va +
4341                   beiscsi_conn->doorbell_offset);
4342
4343         /*
4344          * There is no completion for CONTEXT_UPDATE. The completion of next
4345          * WRB posted guarantees FW's processing and DMA'ing of it.
4346          * Use beiscsi_put_wrb_handle to put it back in the pool which makes
4347          * sure zero'ing or reuse of the WRB only after wrbs_per_cxn.
4348          */
4349         beiscsi_put_wrb_handle(pwrb_context, pwrb_handle,
4350                                phba->params.wrbs_per_cxn);
4351         beiscsi_log(phba, KERN_INFO,
4352                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4353                     "BM_%d : put CONTEXT_UPDATE pwrb_handle=%p free_index=0x%x wrb_handles_available=%d\n",
4354                     pwrb_handle, pwrb_context->free_index,
4355                     pwrb_context->wrb_handles_available);
4356 }
4357
4358 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4359                               int *index, int *age)
4360 {
4361         *index = (int)itt;
4362         if (age)
4363                 *age = conn->session->age;
4364 }
4365
4366 /**
4367  * beiscsi_alloc_pdu - allocates pdu and related resources
4368  * @task: libiscsi task
4369  * @opcode: opcode of pdu for task
4370  *
4371  * This is called with the session lock held. It will allocate
4372  * the wrb and sgl if needed for the command. And it will prep
4373  * the pdu's itt. beiscsi_parse_pdu will later translate
4374  * the pdu itt to the libiscsi task itt.
4375  */
4376 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4377 {
4378         struct beiscsi_io_task *io_task = task->dd_data;
4379         struct iscsi_conn *conn = task->conn;
4380         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4381         struct beiscsi_hba *phba = beiscsi_conn->phba;
4382         struct hwi_wrb_context *pwrb_context;
4383         struct hwi_controller *phwi_ctrlr;
4384         itt_t itt;
4385         uint16_t cri_index = 0;
4386         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4387         dma_addr_t paddr;
4388
4389         io_task->cmd_bhs = dma_pool_alloc(beiscsi_sess->bhs_pool,
4390                                           GFP_ATOMIC, &paddr);
4391         if (!io_task->cmd_bhs)
4392                 return -ENOMEM;
4393         io_task->bhs_pa.u.a64.address = paddr;
4394         io_task->libiscsi_itt = (itt_t)task->itt;
4395         io_task->conn = beiscsi_conn;
4396
4397         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4398         task->hdr_max = sizeof(struct be_cmd_bhs);
4399         io_task->psgl_handle = NULL;
4400         io_task->pwrb_handle = NULL;
4401
4402         if (task->sc) {
4403                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4404                 if (!io_task->psgl_handle) {
4405                         beiscsi_log(phba, KERN_ERR,
4406                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4407                                     "BM_%d : Alloc of IO_SGL_ICD Failed"
4408                                     "for the CID : %d\n",
4409                                     beiscsi_conn->beiscsi_conn_cid);
4410                         goto free_hndls;
4411                 }
4412                 io_task->pwrb_handle = alloc_wrb_handle(phba,
4413                                         beiscsi_conn->beiscsi_conn_cid,
4414                                         &io_task->pwrb_context);
4415                 if (!io_task->pwrb_handle) {
4416                         beiscsi_log(phba, KERN_ERR,
4417                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4418                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4419                                     "for the CID : %d\n",
4420                                     beiscsi_conn->beiscsi_conn_cid);
4421                         goto free_io_hndls;
4422                 }
4423         } else {
4424                 io_task->scsi_cmnd = NULL;
4425                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
4426                         beiscsi_conn->task = task;
4427                         if (!beiscsi_conn->login_in_progress) {
4428                                 io_task->psgl_handle = (struct sgl_handle *)
4429                                                 alloc_mgmt_sgl_handle(phba);
4430                                 if (!io_task->psgl_handle) {
4431                                         beiscsi_log(phba, KERN_ERR,
4432                                                     BEISCSI_LOG_IO |
4433                                                     BEISCSI_LOG_CONFIG,
4434                                                     "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4435                                                     "for the CID : %d\n",
4436                                                     beiscsi_conn->
4437                                                     beiscsi_conn_cid);
4438                                         goto free_hndls;
4439                                 }
4440
4441                                 beiscsi_conn->login_in_progress = 1;
4442                                 beiscsi_conn->plogin_sgl_handle =
4443                                                         io_task->psgl_handle;
4444                                 io_task->pwrb_handle =
4445                                         alloc_wrb_handle(phba,
4446                                         beiscsi_conn->beiscsi_conn_cid,
4447                                         &io_task->pwrb_context);
4448                                 if (!io_task->pwrb_handle) {
4449                                         beiscsi_log(phba, KERN_ERR,
4450                                                     BEISCSI_LOG_IO |
4451                                                     BEISCSI_LOG_CONFIG,
4452                                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4453                                                     "for the CID : %d\n",
4454                                                     beiscsi_conn->
4455                                                     beiscsi_conn_cid);
4456                                         goto free_mgmt_hndls;
4457                                 }
4458                                 beiscsi_conn->plogin_wrb_handle =
4459                                                         io_task->pwrb_handle;
4460
4461                         } else {
4462                                 io_task->psgl_handle =
4463                                                 beiscsi_conn->plogin_sgl_handle;
4464                                 io_task->pwrb_handle =
4465                                                 beiscsi_conn->plogin_wrb_handle;
4466                         }
4467                 } else {
4468                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4469                         if (!io_task->psgl_handle) {
4470                                 beiscsi_log(phba, KERN_ERR,
4471                                             BEISCSI_LOG_IO |
4472                                             BEISCSI_LOG_CONFIG,
4473                                             "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4474                                             "for the CID : %d\n",
4475                                             beiscsi_conn->
4476                                             beiscsi_conn_cid);
4477                                 goto free_hndls;
4478                         }
4479                         io_task->pwrb_handle =
4480                                         alloc_wrb_handle(phba,
4481                                         beiscsi_conn->beiscsi_conn_cid,
4482                                         &io_task->pwrb_context);
4483                         if (!io_task->pwrb_handle) {
4484                                 beiscsi_log(phba, KERN_ERR,
4485                                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4486                                             "BM_%d : Alloc of WRB_HANDLE Failed"
4487                                             "for the CID : %d\n",
4488                                             beiscsi_conn->beiscsi_conn_cid);
4489                                 goto free_mgmt_hndls;
4490                         }
4491
4492                 }
4493         }
4494         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4495                                  wrb_index << 16) | (unsigned int)
4496                                 (io_task->psgl_handle->sgl_index));
4497         io_task->pwrb_handle->pio_handle = task;
4498
4499         io_task->cmd_bhs->iscsi_hdr.itt = itt;
4500         return 0;
4501
4502 free_io_hndls:
4503         free_io_sgl_handle(phba, io_task->psgl_handle);
4504         goto free_hndls;
4505 free_mgmt_hndls:
4506         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4507         io_task->psgl_handle = NULL;
4508 free_hndls:
4509         phwi_ctrlr = phba->phwi_ctrlr;
4510         cri_index = BE_GET_CRI_FROM_CID(
4511         beiscsi_conn->beiscsi_conn_cid);
4512         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4513         if (io_task->pwrb_handle)
4514                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4515         io_task->pwrb_handle = NULL;
4516         dma_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4517                       io_task->bhs_pa.u.a64.address);
4518         io_task->cmd_bhs = NULL;
4519         return -ENOMEM;
4520 }
4521 static int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4522                        unsigned int num_sg, unsigned int xferlen,
4523                        unsigned int writedir)
4524 {
4525
4526         struct beiscsi_io_task *io_task = task->dd_data;
4527         struct iscsi_conn *conn = task->conn;
4528         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4529         struct beiscsi_hba *phba = beiscsi_conn->phba;
4530         struct iscsi_wrb *pwrb = NULL;
4531         unsigned int doorbell = 0;
4532
4533         pwrb = io_task->pwrb_handle->pwrb;
4534
4535         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4536
4537         if (writedir) {
4538                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4539                               INI_WR_CMD);
4540                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4541         } else {
4542                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4543                               INI_RD_CMD);
4544                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4545         }
4546
4547         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4548                                           type, pwrb);
4549
4550         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4551                       cpu_to_be16(*(unsigned short *)
4552                       &io_task->cmd_bhs->iscsi_hdr.lun));
4553         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4554         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4555                       io_task->pwrb_handle->wrb_index);
4556         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4557                       be32_to_cpu(task->cmdsn));
4558         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4559                       io_task->psgl_handle->sgl_index);
4560
4561         hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4562         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4563                       io_task->pwrb_handle->wrb_index);
4564         if (io_task->pwrb_context->plast_wrb)
4565                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb,
4566                               io_task->pwrb_context->plast_wrb,
4567                               io_task->pwrb_handle->wrb_index);
4568         io_task->pwrb_context->plast_wrb = pwrb;
4569
4570         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4571
4572         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4573         doorbell |= (io_task->pwrb_handle->wrb_index &
4574                      DB_DEF_PDU_WRB_INDEX_MASK) <<
4575                      DB_DEF_PDU_WRB_INDEX_SHIFT;
4576         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4577         iowrite32(doorbell, phba->db_va +
4578                   beiscsi_conn->doorbell_offset);
4579         return 0;
4580 }
4581
4582 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4583                           unsigned int num_sg, unsigned int xferlen,
4584                           unsigned int writedir)
4585 {
4586
4587         struct beiscsi_io_task *io_task = task->dd_data;
4588         struct iscsi_conn *conn = task->conn;
4589         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4590         struct beiscsi_hba *phba = beiscsi_conn->phba;
4591         struct iscsi_wrb *pwrb = NULL;
4592         unsigned int doorbell = 0;
4593
4594         pwrb = io_task->pwrb_handle->pwrb;
4595         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4596
4597         if (writedir) {
4598                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4599                               INI_WR_CMD);
4600                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
4601         } else {
4602                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4603                               INI_RD_CMD);
4604                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4605         }
4606
4607         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4608                                           type, pwrb);
4609
4610         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4611                       cpu_to_be16(*(unsigned short *)
4612                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4613         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4614         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4615                       io_task->pwrb_handle->wrb_index);
4616         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4617                       be32_to_cpu(task->cmdsn));
4618         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4619                       io_task->psgl_handle->sgl_index);
4620
4621         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4622
4623         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4624                       io_task->pwrb_handle->wrb_index);
4625         if (io_task->pwrb_context->plast_wrb)
4626                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb,
4627                               io_task->pwrb_context->plast_wrb,
4628                               io_task->pwrb_handle->wrb_index);
4629         io_task->pwrb_context->plast_wrb = pwrb;
4630
4631         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4632
4633         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4634         doorbell |= (io_task->pwrb_handle->wrb_index &
4635                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4636         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4637
4638         iowrite32(doorbell, phba->db_va +
4639                   beiscsi_conn->doorbell_offset);
4640         return 0;
4641 }
4642
4643 static int beiscsi_mtask(struct iscsi_task *task)
4644 {
4645         struct beiscsi_io_task *io_task = task->dd_data;
4646         struct iscsi_conn *conn = task->conn;
4647         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4648         struct beiscsi_hba *phba = beiscsi_conn->phba;
4649         struct iscsi_wrb *pwrb = NULL;
4650         unsigned int doorbell = 0;
4651         unsigned int cid;
4652         unsigned int pwrb_typeoffset = 0;
4653         int ret = 0;
4654
4655         cid = beiscsi_conn->beiscsi_conn_cid;
4656         pwrb = io_task->pwrb_handle->pwrb;
4657
4658         if (is_chip_be2_be3r(phba)) {
4659                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4660                               be32_to_cpu(task->cmdsn));
4661                 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4662                               io_task->pwrb_handle->wrb_index);
4663                 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4664                               io_task->psgl_handle->sgl_index);
4665                 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4666                               task->data_count);
4667                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4668                               io_task->pwrb_handle->wrb_index);
4669                 if (io_task->pwrb_context->plast_wrb)
4670                         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb,
4671                                       io_task->pwrb_context->plast_wrb,
4672                                       io_task->pwrb_handle->wrb_index);
4673                 io_task->pwrb_context->plast_wrb = pwrb;
4674
4675                 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
4676         } else {
4677                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4678                               be32_to_cpu(task->cmdsn));
4679                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4680                               io_task->pwrb_handle->wrb_index);
4681                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4682                               io_task->psgl_handle->sgl_index);
4683                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
4684                               task->data_count);
4685                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4686                               io_task->pwrb_handle->wrb_index);
4687                 if (io_task->pwrb_context->plast_wrb)
4688                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb,
4689                                       io_task->pwrb_context->plast_wrb,
4690                                       io_task->pwrb_handle->wrb_index);
4691                 io_task->pwrb_context->plast_wrb = pwrb;
4692
4693                 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
4694         }
4695
4696
4697         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
4698         case ISCSI_OP_LOGIN:
4699                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
4700                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4701                 ret = hwi_write_buffer(pwrb, task);
4702                 break;
4703         case ISCSI_OP_NOOP_OUT:
4704                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
4705                         ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4706                         if (is_chip_be2_be3r(phba))
4707                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4708                                               dmsg, pwrb, 1);
4709                         else
4710                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4711                                               dmsg, pwrb, 1);
4712                 } else {
4713                         ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
4714                         if (is_chip_be2_be3r(phba))
4715                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
4716                                               dmsg, pwrb, 0);
4717                         else
4718                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
4719                                               dmsg, pwrb, 0);
4720                 }
4721                 ret = hwi_write_buffer(pwrb, task);
4722                 break;
4723         case ISCSI_OP_TEXT:
4724                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
4725                 ret = hwi_write_buffer(pwrb, task);
4726                 break;
4727         case ISCSI_OP_SCSI_TMFUNC:
4728                 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
4729                 ret = hwi_write_buffer(pwrb, task);
4730                 break;
4731         case ISCSI_OP_LOGOUT:
4732                 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
4733                 ret = hwi_write_buffer(pwrb, task);
4734                 break;
4735
4736         default:
4737                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4738                             "BM_%d : opcode =%d Not supported\n",
4739                             task->hdr->opcode & ISCSI_OPCODE_MASK);
4740
4741                 return -EINVAL;
4742         }
4743
4744         if (ret)
4745                 return ret;
4746
4747         /* Set the task type */
4748         io_task->wrb_type = (is_chip_be2_be3r(phba)) ?
4749                 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb) :
4750                 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb);
4751
4752         doorbell |= cid & DB_WRB_POST_CID_MASK;
4753         doorbell |= (io_task->pwrb_handle->wrb_index &
4754                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4755         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4756         iowrite32(doorbell, phba->db_va +
4757                   beiscsi_conn->doorbell_offset);
4758         return 0;
4759 }
4760
4761 static int beiscsi_task_xmit(struct iscsi_task *task)
4762 {
4763         struct beiscsi_io_task *io_task = task->dd_data;
4764         struct scsi_cmnd *sc = task->sc;
4765         struct beiscsi_hba *phba;
4766         struct scatterlist *sg;
4767         int num_sg;
4768         unsigned int  writedir = 0, xferlen = 0;
4769
4770         phba = io_task->conn->phba;
4771         /**
4772          * HBA in error includes BEISCSI_HBA_FW_TIMEOUT. IO path might be
4773          * operational if FW still gets heartbeat from EP FW. Is management
4774          * path really needed to continue further?
4775          */
4776         if (!beiscsi_hba_is_online(phba))
4777                 return -EIO;
4778
4779         if (!io_task->conn->login_in_progress)
4780                 task->hdr->exp_statsn = 0;
4781
4782         if (!sc)
4783                 return beiscsi_mtask(task);
4784
4785         io_task->scsi_cmnd = sc;
4786         io_task->num_sg = 0;
4787         num_sg = scsi_dma_map(sc);
4788         if (num_sg < 0) {
4789                 beiscsi_log(phba, KERN_ERR,
4790                             BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI,
4791                             "BM_%d : scsi_dma_map Failed "
4792                             "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n",
4793                             be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt),
4794                             io_task->libiscsi_itt, scsi_bufflen(sc));
4795
4796                 return num_sg;
4797         }
4798         /**
4799          * For scsi cmd task, check num_sg before unmapping in cleanup_task.
4800          * For management task, cleanup_task checks mtask_addr before unmapping.
4801          */
4802         io_task->num_sg = num_sg;
4803         xferlen = scsi_bufflen(sc);
4804         sg = scsi_sglist(sc);
4805         if (sc->sc_data_direction == DMA_TO_DEVICE)
4806                 writedir = 1;
4807         else
4808                 writedir = 0;
4809
4810         return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
4811 }
4812
4813 /**
4814  * beiscsi_bsg_request - handle bsg request from ISCSI transport
4815  * @job: job to handle
4816  */
4817 static int beiscsi_bsg_request(struct bsg_job *job)
4818 {
4819         struct Scsi_Host *shost;
4820         struct beiscsi_hba *phba;
4821         struct iscsi_bsg_request *bsg_req = job->request;
4822         int rc = -EINVAL;
4823         unsigned int tag;
4824         struct be_dma_mem nonemb_cmd;
4825         struct be_cmd_resp_hdr *resp;
4826         struct iscsi_bsg_reply *bsg_reply = job->reply;
4827         unsigned short status, extd_status;
4828
4829         shost = iscsi_job_to_shost(job);
4830         phba = iscsi_host_priv(shost);
4831
4832         if (!beiscsi_hba_is_online(phba)) {
4833                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
4834                             "BM_%d : HBA in error 0x%lx\n", phba->state);
4835                 return -ENXIO;
4836         }
4837
4838         switch (bsg_req->msgcode) {
4839         case ISCSI_BSG_HST_VENDOR:
4840                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
4841                                         job->request_payload.payload_len,
4842                                         &nonemb_cmd.dma);
4843                 if (nonemb_cmd.va == NULL) {
4844                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4845                                     "BM_%d : Failed to allocate memory for "
4846                                     "beiscsi_bsg_request\n");
4847                         return -ENOMEM;
4848                 }
4849                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
4850                                                   &nonemb_cmd);
4851                 if (!tag) {
4852                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4853                                     "BM_%d : MBX Tag Allocation Failed\n");
4854
4855                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4856                                             nonemb_cmd.va, nonemb_cmd.dma);
4857                         return -EAGAIN;
4858                 }
4859
4860                 rc = wait_event_interruptible_timeout(
4861                                         phba->ctrl.mcc_wait[tag],
4862                                         phba->ctrl.mcc_tag_status[tag],
4863                                         msecs_to_jiffies(
4864                                         BEISCSI_HOST_MBX_TIMEOUT));
4865
4866                 if (!test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {
4867                         clear_bit(MCC_TAG_STATE_RUNNING,
4868                                   &phba->ctrl.ptag_state[tag].tag_state);
4869                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4870                                             nonemb_cmd.va, nonemb_cmd.dma);
4871                         return -EIO;
4872                 }
4873                 extd_status = (phba->ctrl.mcc_tag_status[tag] &
4874                                CQE_STATUS_ADDL_MASK) >> CQE_STATUS_ADDL_SHIFT;
4875                 status = phba->ctrl.mcc_tag_status[tag] & CQE_STATUS_MASK;
4876                 free_mcc_wrb(&phba->ctrl, tag);
4877                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
4878                 sg_copy_from_buffer(job->reply_payload.sg_list,
4879                                     job->reply_payload.sg_cnt,
4880                                     nonemb_cmd.va, (resp->response_length
4881                                     + sizeof(*resp)));
4882                 bsg_reply->reply_payload_rcv_len = resp->response_length;
4883                 bsg_reply->result = status;
4884                 bsg_job_done(job, bsg_reply->result,
4885                              bsg_reply->reply_payload_rcv_len);
4886                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4887                                     nonemb_cmd.va, nonemb_cmd.dma);
4888                 if (status || extd_status) {
4889                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4890                                     "BM_%d : MBX Cmd Failed"
4891                                     " status = %d extd_status = %d\n",
4892                                     status, extd_status);
4893
4894                         return -EIO;
4895                 } else {
4896                         rc = 0;
4897                 }
4898                 break;
4899
4900         default:
4901                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
4902                                 "BM_%d : Unsupported bsg command: 0x%x\n",
4903                                 bsg_req->msgcode);
4904                 break;
4905         }
4906
4907         return rc;
4908 }
4909
4910 static void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
4911 {
4912         /* Set the logging parameter */
4913         beiscsi_log_enable_init(phba, beiscsi_log_enable);
4914 }
4915
4916 void beiscsi_start_boot_work(struct beiscsi_hba *phba, unsigned int s_handle)
4917 {
4918         if (phba->boot_struct.boot_kset)
4919                 return;
4920
4921         /* skip if boot work is already in progress */
4922         if (test_and_set_bit(BEISCSI_HBA_BOOT_WORK, &phba->state))
4923                 return;
4924
4925         phba->boot_struct.retry = 3;
4926         phba->boot_struct.tag = 0;
4927         phba->boot_struct.s_handle = s_handle;
4928         phba->boot_struct.action = BEISCSI_BOOT_GET_SHANDLE;
4929         schedule_work(&phba->boot_work);
4930 }
4931
4932 /**
4933  * Boot flag info for iscsi-utilities
4934  * Bit 0 Block valid flag
4935  * Bit 1 Firmware booting selected
4936  */
4937 #define BEISCSI_SYSFS_ISCSI_BOOT_FLAGS  3
4938
4939 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
4940 {
4941         struct beiscsi_hba *phba = data;
4942         struct mgmt_session_info *boot_sess = &phba->boot_struct.boot_sess;
4943         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
4944         char *str = buf;
4945         int rc = -EPERM;
4946
4947         switch (type) {
4948         case ISCSI_BOOT_TGT_NAME:
4949                 rc = sprintf(buf, "%.*s\n",
4950                             (int)strlen(boot_sess->target_name),
4951                             (char *)&boot_sess->target_name);
4952                 break;
4953         case ISCSI_BOOT_TGT_IP_ADDR:
4954                 if (boot_conn->dest_ipaddr.ip_type == BEISCSI_IP_TYPE_V4)
4955                         rc = sprintf(buf, "%pI4\n",
4956                                 (char *)&boot_conn->dest_ipaddr.addr);
4957                 else
4958                         rc = sprintf(str, "%pI6\n",
4959                                 (char *)&boot_conn->dest_ipaddr.addr);
4960                 break;
4961         case ISCSI_BOOT_TGT_PORT:
4962                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
4963                 break;
4964
4965         case ISCSI_BOOT_TGT_CHAP_NAME:
4966                 rc = sprintf(str,  "%.*s\n",
4967                              boot_conn->negotiated_login_options.auth_data.chap.
4968                              target_chap_name_length,
4969                              (char *)&boot_conn->negotiated_login_options.
4970                              auth_data.chap.target_chap_name);
4971                 break;
4972         case ISCSI_BOOT_TGT_CHAP_SECRET:
4973                 rc = sprintf(str,  "%.*s\n",
4974                              boot_conn->negotiated_login_options.auth_data.chap.
4975                              target_secret_length,
4976                              (char *)&boot_conn->negotiated_login_options.
4977                              auth_data.chap.target_secret);
4978                 break;
4979         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
4980                 rc = sprintf(str,  "%.*s\n",
4981                              boot_conn->negotiated_login_options.auth_data.chap.
4982                              intr_chap_name_length,
4983                              (char *)&boot_conn->negotiated_login_options.
4984                              auth_data.chap.intr_chap_name);
4985                 break;
4986         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
4987                 rc = sprintf(str,  "%.*s\n",
4988                              boot_conn->negotiated_login_options.auth_data.chap.
4989                              intr_secret_length,
4990                              (char *)&boot_conn->negotiated_login_options.
4991                              auth_data.chap.intr_secret);
4992                 break;
4993         case ISCSI_BOOT_TGT_FLAGS:
4994                 rc = sprintf(str, "%d\n", BEISCSI_SYSFS_ISCSI_BOOT_FLAGS);
4995                 break;
4996         case ISCSI_BOOT_TGT_NIC_ASSOC:
4997                 rc = sprintf(str, "0\n");
4998                 break;
4999         }
5000         return rc;
5001 }
5002
5003 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
5004 {
5005         struct beiscsi_hba *phba = data;
5006         char *str = buf;
5007         int rc = -EPERM;
5008
5009         switch (type) {
5010         case ISCSI_BOOT_INI_INITIATOR_NAME:
5011                 rc = sprintf(str, "%s\n",
5012                              phba->boot_struct.boot_sess.initiator_iscsiname);
5013                 break;
5014         }
5015         return rc;
5016 }
5017
5018 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
5019 {
5020         struct beiscsi_hba *phba = data;
5021         char *str = buf;
5022         int rc = -EPERM;
5023
5024         switch (type) {
5025         case ISCSI_BOOT_ETH_FLAGS:
5026                 rc = sprintf(str, "%d\n", BEISCSI_SYSFS_ISCSI_BOOT_FLAGS);
5027                 break;
5028         case ISCSI_BOOT_ETH_INDEX:
5029                 rc = sprintf(str, "0\n");
5030                 break;
5031         case ISCSI_BOOT_ETH_MAC:
5032                 rc  = beiscsi_get_macaddr(str, phba);
5033                 break;
5034         }
5035         return rc;
5036 }
5037
5038 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
5039 {
5040         umode_t rc = 0;
5041
5042         switch (type) {
5043         case ISCSI_BOOT_TGT_NAME:
5044         case ISCSI_BOOT_TGT_IP_ADDR:
5045         case ISCSI_BOOT_TGT_PORT:
5046         case ISCSI_BOOT_TGT_CHAP_NAME:
5047         case ISCSI_BOOT_TGT_CHAP_SECRET:
5048         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
5049         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
5050         case ISCSI_BOOT_TGT_NIC_ASSOC:
5051         case ISCSI_BOOT_TGT_FLAGS:
5052                 rc = S_IRUGO;
5053                 break;
5054         }
5055         return rc;
5056 }
5057
5058 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
5059 {
5060         umode_t rc = 0;
5061
5062         switch (type) {
5063         case ISCSI_BOOT_INI_INITIATOR_NAME:
5064                 rc = S_IRUGO;
5065                 break;
5066         }
5067         return rc;
5068 }
5069
5070 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
5071 {
5072         umode_t rc = 0;
5073
5074         switch (type) {
5075         case ISCSI_BOOT_ETH_FLAGS:
5076         case ISCSI_BOOT_ETH_MAC:
5077         case ISCSI_BOOT_ETH_INDEX:
5078                 rc = S_IRUGO;
5079                 break;
5080         }
5081         return rc;
5082 }
5083
5084 static void beiscsi_boot_kobj_release(void *data)
5085 {
5086         struct beiscsi_hba *phba = data;
5087
5088         scsi_host_put(phba->shost);
5089 }
5090
5091 static int beiscsi_boot_create_kset(struct beiscsi_hba *phba)
5092 {
5093         struct boot_struct *bs = &phba->boot_struct;
5094         struct iscsi_boot_kobj *boot_kobj;
5095
5096         if (bs->boot_kset) {
5097                 __beiscsi_log(phba, KERN_ERR,
5098                               "BM_%d: boot_kset already created\n");
5099                 return 0;
5100         }
5101
5102         bs->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
5103         if (!bs->boot_kset) {
5104                 __beiscsi_log(phba, KERN_ERR,
5105                               "BM_%d: boot_kset alloc failed\n");
5106                 return -ENOMEM;
5107         }
5108
5109         /* get shost ref because the show function will refer phba */
5110         if (!scsi_host_get(phba->shost))
5111                 goto free_kset;
5112
5113         boot_kobj = iscsi_boot_create_target(bs->boot_kset, 0, phba,
5114                                              beiscsi_show_boot_tgt_info,
5115                                              beiscsi_tgt_get_attr_visibility,
5116                                              beiscsi_boot_kobj_release);
5117         if (!boot_kobj)
5118                 goto put_shost;
5119
5120         if (!scsi_host_get(phba->shost))
5121                 goto free_kset;
5122
5123         boot_kobj = iscsi_boot_create_initiator(bs->boot_kset, 0, phba,
5124                                                 beiscsi_show_boot_ini_info,
5125                                                 beiscsi_ini_get_attr_visibility,
5126                                                 beiscsi_boot_kobj_release);
5127         if (!boot_kobj)
5128                 goto put_shost;
5129
5130         if (!scsi_host_get(phba->shost))
5131                 goto free_kset;
5132
5133         boot_kobj = iscsi_boot_create_ethernet(bs->boot_kset, 0, phba,
5134                                                beiscsi_show_boot_eth_info,
5135                                                beiscsi_eth_get_attr_visibility,
5136                                                beiscsi_boot_kobj_release);
5137         if (!boot_kobj)
5138                 goto put_shost;
5139
5140         return 0;
5141
5142 put_shost:
5143         scsi_host_put(phba->shost);
5144 free_kset:
5145         iscsi_boot_destroy_kset(bs->boot_kset);
5146         bs->boot_kset = NULL;
5147         return -ENOMEM;
5148 }
5149
5150 static void beiscsi_boot_work(struct work_struct *work)
5151 {
5152         struct beiscsi_hba *phba =
5153                 container_of(work, struct beiscsi_hba, boot_work);
5154         struct boot_struct *bs = &phba->boot_struct;
5155         unsigned int tag = 0;
5156
5157         if (!beiscsi_hba_is_online(phba))
5158                 return;
5159
5160         beiscsi_log(phba, KERN_INFO,
5161                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_MBOX,
5162                     "BM_%d : %s action %d\n",
5163                     __func__, phba->boot_struct.action);
5164
5165         switch (phba->boot_struct.action) {
5166         case BEISCSI_BOOT_REOPEN_SESS:
5167                 tag = beiscsi_boot_reopen_sess(phba);
5168                 break;
5169         case BEISCSI_BOOT_GET_SHANDLE:
5170                 tag = __beiscsi_boot_get_shandle(phba, 1);
5171                 break;
5172         case BEISCSI_BOOT_GET_SINFO:
5173                 tag = beiscsi_boot_get_sinfo(phba);
5174                 break;
5175         case BEISCSI_BOOT_LOGOUT_SESS:
5176                 tag = beiscsi_boot_logout_sess(phba);
5177                 break;
5178         case BEISCSI_BOOT_CREATE_KSET:
5179                 beiscsi_boot_create_kset(phba);
5180                 /**
5181                  * updated boot_kset is made visible to all before
5182                  * ending the boot work.
5183                  */
5184                 mb();
5185                 clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);
5186                 return;
5187         }
5188         if (!tag) {
5189                 if (bs->retry--)
5190                         schedule_work(&phba->boot_work);
5191                 else
5192                         clear_bit(BEISCSI_HBA_BOOT_WORK, &phba->state);
5193         }
5194 }
5195
5196 static void beiscsi_eqd_update_work(struct work_struct *work)
5197 {
5198         struct hwi_context_memory *phwi_context;
5199         struct be_set_eqd set_eqd[MAX_CPUS];
5200         struct hwi_controller *phwi_ctrlr;
5201         struct be_eq_obj *pbe_eq;
5202         struct beiscsi_hba *phba;
5203         unsigned int pps, delta;
5204         struct be_aic_obj *aic;
5205         int eqd, i, num = 0;
5206         unsigned long now;
5207
5208         phba = container_of(work, struct beiscsi_hba, eqd_update.work);
5209         if (!beiscsi_hba_is_online(phba))
5210                 return;
5211
5212         phwi_ctrlr = phba->phwi_ctrlr;
5213         phwi_context = phwi_ctrlr->phwi_ctxt;
5214
5215         for (i = 0; i <= phba->num_cpus; i++) {
5216                 aic = &phba->aic_obj[i];
5217                 pbe_eq = &phwi_context->be_eq[i];
5218                 now = jiffies;
5219                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
5220                     pbe_eq->cq_count < aic->eq_prev) {
5221                         aic->jiffies = now;
5222                         aic->eq_prev = pbe_eq->cq_count;
5223                         continue;
5224                 }
5225                 delta = jiffies_to_msecs(now - aic->jiffies);
5226                 pps = (((u32)(pbe_eq->cq_count - aic->eq_prev) * 1000) / delta);
5227                 eqd = (pps / 1500) << 2;
5228
5229                 if (eqd < 8)
5230                         eqd = 0;
5231                 eqd = min_t(u32, eqd, BEISCSI_EQ_DELAY_MAX);
5232                 eqd = max_t(u32, eqd, BEISCSI_EQ_DELAY_MIN);
5233
5234                 aic->jiffies = now;
5235                 aic->eq_prev = pbe_eq->cq_count;
5236
5237                 if (eqd != aic->prev_eqd) {
5238                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
5239                         set_eqd[num].eq_id = pbe_eq->q.id;
5240                         aic->prev_eqd = eqd;
5241                         num++;
5242                 }
5243         }
5244         if (num)
5245                 /* completion of this is ignored */
5246                 beiscsi_modify_eq_delay(phba, set_eqd, num);
5247
5248         schedule_delayed_work(&phba->eqd_update,
5249                               msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
5250 }
5251
5252 static void beiscsi_hw_tpe_check(struct timer_list *t)
5253 {
5254         struct beiscsi_hba *phba = from_timer(phba, t, hw_check);
5255         u32 wait;
5256
5257         /* if not TPE, do nothing */
5258         if (!beiscsi_detect_tpe(phba))
5259                 return;
5260
5261         /* wait default 4000ms before recovering */
5262         wait = 4000;
5263         if (phba->ue2rp > BEISCSI_UE_DETECT_INTERVAL)
5264                 wait = phba->ue2rp - BEISCSI_UE_DETECT_INTERVAL;
5265         queue_delayed_work(phba->wq, &phba->recover_port,
5266                            msecs_to_jiffies(wait));
5267 }
5268
5269 static void beiscsi_hw_health_check(struct timer_list *t)
5270 {
5271         struct beiscsi_hba *phba = from_timer(phba, t, hw_check);
5272
5273         beiscsi_detect_ue(phba);
5274         if (beiscsi_detect_ue(phba)) {
5275                 __beiscsi_log(phba, KERN_ERR,
5276                               "BM_%d : port in error: %lx\n", phba->state);
5277                 /* sessions are no longer valid, so first fail the sessions */
5278                 queue_work(phba->wq, &phba->sess_work);
5279
5280                 /* detect UER supported */
5281                 if (!test_bit(BEISCSI_HBA_UER_SUPP, &phba->state))
5282                         return;
5283                 /* modify this timer to check TPE */
5284                 phba->hw_check.function = beiscsi_hw_tpe_check;
5285         }
5286
5287         mod_timer(&phba->hw_check,
5288                   jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
5289 }
5290
5291 /*
5292  * beiscsi_enable_port()- Enables the disabled port.
5293  * Only port resources freed in disable function are reallocated.
5294  * This is called in HBA error handling path.
5295  *
5296  * @phba: Instance of driver private structure
5297  *
5298  **/
5299 static int beiscsi_enable_port(struct beiscsi_hba *phba)
5300 {
5301         struct hwi_context_memory *phwi_context;
5302         struct hwi_controller *phwi_ctrlr;
5303         struct be_eq_obj *pbe_eq;
5304         int ret, i;
5305
5306         if (test_bit(BEISCSI_HBA_ONLINE, &phba->state)) {
5307                 __beiscsi_log(phba, KERN_ERR,
5308                               "BM_%d : %s : port is online %lx\n",
5309                               __func__, phba->state);
5310                 return 0;
5311         }
5312
5313         ret = beiscsi_init_sliport(phba);
5314         if (ret)
5315                 return ret;
5316
5317         be2iscsi_enable_msix(phba);
5318
5319         beiscsi_get_params(phba);
5320         beiscsi_set_host_data(phba);
5321         /* Re-enable UER. If different TPE occurs then it is recoverable. */
5322         beiscsi_set_uer_feature(phba);
5323
5324         phba->shost->max_id = phba->params.cxns_per_ctrl;
5325         phba->shost->can_queue = phba->params.ios_per_ctrl;
5326         ret = beiscsi_init_port(phba);
5327         if (ret < 0) {
5328                 __beiscsi_log(phba, KERN_ERR,
5329                               "BM_%d : init port failed\n");
5330                 goto disable_msix;
5331         }
5332
5333         for (i = 0; i < MAX_MCC_CMD; i++) {
5334                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5335                 phba->ctrl.mcc_tag[i] = i + 1;
5336                 phba->ctrl.mcc_tag_status[i + 1] = 0;
5337                 phba->ctrl.mcc_tag_available++;
5338         }
5339
5340         phwi_ctrlr = phba->phwi_ctrlr;
5341         phwi_context = phwi_ctrlr->phwi_ctxt;
5342         for (i = 0; i < phba->num_cpus; i++) {
5343                 pbe_eq = &phwi_context->be_eq[i];
5344                 irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);
5345         }
5346
5347         i = (phba->pcidev->msix_enabled) ? i : 0;
5348         /* Work item for MCC handling */
5349         pbe_eq = &phwi_context->be_eq[i];
5350         INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);
5351
5352         ret = beiscsi_init_irqs(phba);
5353         if (ret < 0) {
5354                 __beiscsi_log(phba, KERN_ERR,
5355                               "BM_%d : setup IRQs failed %d\n", ret);
5356                 goto cleanup_port;
5357         }
5358         hwi_enable_intr(phba);
5359         /* port operational: clear all error bits */
5360         set_bit(BEISCSI_HBA_ONLINE, &phba->state);
5361         __beiscsi_log(phba, KERN_INFO,
5362                       "BM_%d : port online: 0x%lx\n", phba->state);
5363
5364         /* start hw_check timer and eqd_update work */
5365         schedule_delayed_work(&phba->eqd_update,
5366                               msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
5367
5368         /**
5369          * Timer function gets modified for TPE detection.
5370          * Always reinit to do health check first.
5371          */
5372         phba->hw_check.function = beiscsi_hw_health_check;
5373         mod_timer(&phba->hw_check,
5374                   jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
5375         return 0;
5376
5377 cleanup_port:
5378         for (i = 0; i < phba->num_cpus; i++) {
5379                 pbe_eq = &phwi_context->be_eq[i];
5380                 irq_poll_disable(&pbe_eq->iopoll);
5381         }
5382         hwi_cleanup_port(phba);
5383
5384 disable_msix:
5385         pci_free_irq_vectors(phba->pcidev);
5386         return ret;
5387 }
5388
5389 /*
5390  * beiscsi_disable_port()- Disable port and cleanup driver resources.
5391  * This is called in HBA error handling and driver removal.
5392  * @phba: Instance Priv structure
5393  * @unload: indicate driver is unloading
5394  *
5395  * Free the OS and HW resources held by the driver
5396  **/
5397 static void beiscsi_disable_port(struct beiscsi_hba *phba, int unload)
5398 {
5399         struct hwi_context_memory *phwi_context;
5400         struct hwi_controller *phwi_ctrlr;
5401         struct be_eq_obj *pbe_eq;
5402         unsigned int i;
5403
5404         if (!test_and_clear_bit(BEISCSI_HBA_ONLINE, &phba->state))
5405                 return;
5406
5407         phwi_ctrlr = phba->phwi_ctrlr;
5408         phwi_context = phwi_ctrlr->phwi_ctxt;
5409         hwi_disable_intr(phba);
5410         beiscsi_free_irqs(phba);
5411         pci_free_irq_vectors(phba->pcidev);
5412
5413         for (i = 0; i < phba->num_cpus; i++) {
5414                 pbe_eq = &phwi_context->be_eq[i];
5415                 irq_poll_disable(&pbe_eq->iopoll);
5416         }
5417         cancel_delayed_work_sync(&phba->eqd_update);
5418         cancel_work_sync(&phba->boot_work);
5419         /* WQ might be running cancel queued mcc_work if we are not exiting */
5420         if (!unload && beiscsi_hba_in_error(phba)) {
5421                 pbe_eq = &phwi_context->be_eq[i];
5422                 cancel_work_sync(&pbe_eq->mcc_work);
5423         }
5424         hwi_cleanup_port(phba);
5425         beiscsi_cleanup_port(phba);
5426 }
5427
5428 static void beiscsi_sess_work(struct work_struct *work)
5429 {
5430         struct beiscsi_hba *phba;
5431
5432         phba = container_of(work, struct beiscsi_hba, sess_work);
5433         /*
5434          * This work gets scheduled only in case of HBA error.
5435          * Old sessions are gone so need to be re-established.
5436          * iscsi_session_failure needs process context hence this work.
5437          */
5438         iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);
5439 }
5440
5441 static void beiscsi_recover_port(struct work_struct *work)
5442 {
5443         struct beiscsi_hba *phba;
5444
5445         phba = container_of(work, struct beiscsi_hba, recover_port.work);
5446         beiscsi_disable_port(phba, 0);
5447         beiscsi_enable_port(phba);
5448 }
5449
5450 static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev,
5451                 pci_channel_state_t state)
5452 {
5453         struct beiscsi_hba *phba = NULL;
5454
5455         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5456         set_bit(BEISCSI_HBA_PCI_ERR, &phba->state);
5457
5458         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5459                     "BM_%d : EEH error detected\n");
5460
5461         /* first stop UE detection when PCI error detected */
5462         del_timer_sync(&phba->hw_check);
5463         cancel_delayed_work_sync(&phba->recover_port);
5464
5465         /* sessions are no longer valid, so first fail the sessions */
5466         iscsi_host_for_each_session(phba->shost, beiscsi_session_fail);
5467         beiscsi_disable_port(phba, 0);
5468
5469         if (state == pci_channel_io_perm_failure) {
5470                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5471                             "BM_%d : EEH : State PERM Failure");
5472                 return PCI_ERS_RESULT_DISCONNECT;
5473         }
5474
5475         pci_disable_device(pdev);
5476
5477         /* The error could cause the FW to trigger a flash debug dump.
5478          * Resetting the card while flash dump is in progress
5479          * can cause it not to recover; wait for it to finish.
5480          * Wait only for first function as it is needed only once per
5481          * adapter.
5482          **/
5483         if (pdev->devfn == 0)
5484                 ssleep(30);
5485
5486         return PCI_ERS_RESULT_NEED_RESET;
5487 }
5488
5489 static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev)
5490 {
5491         struct beiscsi_hba *phba = NULL;
5492         int status = 0;
5493
5494         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5495
5496         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5497                     "BM_%d : EEH Reset\n");
5498
5499         status = pci_enable_device(pdev);
5500         if (status)
5501                 return PCI_ERS_RESULT_DISCONNECT;
5502
5503         pci_set_master(pdev);
5504         pci_set_power_state(pdev, PCI_D0);
5505         pci_restore_state(pdev);
5506
5507         status = beiscsi_check_fw_rdy(phba);
5508         if (status) {
5509                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5510                             "BM_%d : EEH Reset Completed\n");
5511         } else {
5512                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5513                             "BM_%d : EEH Reset Completion Failure\n");
5514                 return PCI_ERS_RESULT_DISCONNECT;
5515         }
5516
5517         pci_cleanup_aer_uncorrect_error_status(pdev);
5518         return PCI_ERS_RESULT_RECOVERED;
5519 }
5520
5521 static void beiscsi_eeh_resume(struct pci_dev *pdev)
5522 {
5523         struct beiscsi_hba *phba;
5524         int ret;
5525
5526         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5527         pci_save_state(pdev);
5528
5529         ret = beiscsi_enable_port(phba);
5530         if (ret)
5531                 __beiscsi_log(phba, KERN_ERR,
5532                               "BM_%d : AER EEH resume failed\n");
5533 }
5534
5535 static int beiscsi_dev_probe(struct pci_dev *pcidev,
5536                              const struct pci_device_id *id)
5537 {
5538         struct hwi_context_memory *phwi_context;
5539         struct hwi_controller *phwi_ctrlr;
5540         struct beiscsi_hba *phba = NULL;
5541         struct be_eq_obj *pbe_eq;
5542         unsigned int s_handle;
5543         char wq_name[20];
5544         int ret, i;
5545
5546         ret = beiscsi_enable_pci(pcidev);
5547         if (ret < 0) {
5548                 dev_err(&pcidev->dev,
5549                         "beiscsi_dev_probe - Failed to enable pci device\n");
5550                 return ret;
5551         }
5552
5553         phba = beiscsi_hba_alloc(pcidev);
5554         if (!phba) {
5555                 dev_err(&pcidev->dev,
5556                         "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
5557                 ret = -ENOMEM;
5558                 goto disable_pci;
5559         }
5560
5561         /* Enable EEH reporting */
5562         ret = pci_enable_pcie_error_reporting(pcidev);
5563         if (ret)
5564                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5565                             "BM_%d : PCIe Error Reporting "
5566                             "Enabling Failed\n");
5567
5568         pci_save_state(pcidev);
5569
5570         /* Initialize Driver configuration Paramters */
5571         beiscsi_hba_attrs_init(phba);
5572
5573         phba->mac_addr_set = false;
5574
5575         switch (pcidev->device) {
5576         case BE_DEVICE_ID1:
5577         case OC_DEVICE_ID1:
5578         case OC_DEVICE_ID2:
5579                 phba->generation = BE_GEN2;
5580                 phba->iotask_fn = beiscsi_iotask;
5581                 dev_warn(&pcidev->dev,
5582                          "Obsolete/Unsupported BE2 Adapter Family\n");
5583                 break;
5584         case BE_DEVICE_ID2:
5585         case OC_DEVICE_ID3:
5586                 phba->generation = BE_GEN3;
5587                 phba->iotask_fn = beiscsi_iotask;
5588                 break;
5589         case OC_SKH_ID1:
5590                 phba->generation = BE_GEN4;
5591                 phba->iotask_fn = beiscsi_iotask_v2;
5592                 break;
5593         default:
5594                 phba->generation = 0;
5595         }
5596
5597         ret = be_ctrl_init(phba, pcidev);
5598         if (ret) {
5599                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5600                             "BM_%d : be_ctrl_init failed\n");
5601                 goto free_hba;
5602         }
5603
5604         ret = beiscsi_init_sliport(phba);
5605         if (ret)
5606                 goto free_hba;
5607
5608         spin_lock_init(&phba->io_sgl_lock);
5609         spin_lock_init(&phba->mgmt_sgl_lock);
5610         spin_lock_init(&phba->async_pdu_lock);
5611         ret = beiscsi_get_fw_config(&phba->ctrl, phba);
5612         if (ret != 0) {
5613                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5614                             "BM_%d : Error getting fw config\n");
5615                 goto free_port;
5616         }
5617         beiscsi_get_port_name(&phba->ctrl, phba);
5618         beiscsi_get_params(phba);
5619         beiscsi_set_host_data(phba);
5620         beiscsi_set_uer_feature(phba);
5621
5622         be2iscsi_enable_msix(phba);
5623
5624         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5625                     "BM_%d : num_cpus = %d\n",
5626                     phba->num_cpus);
5627
5628         phba->shost->max_id = phba->params.cxns_per_ctrl;
5629         phba->shost->can_queue = phba->params.ios_per_ctrl;
5630         ret = beiscsi_get_memory(phba);
5631         if (ret < 0) {
5632                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5633                             "BM_%d : alloc host mem failed\n");
5634                 goto free_port;
5635         }
5636
5637         ret = beiscsi_init_port(phba);
5638         if (ret < 0) {
5639                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5640                             "BM_%d : init port failed\n");
5641                 beiscsi_free_mem(phba);
5642                 goto free_port;
5643         }
5644
5645         for (i = 0; i < MAX_MCC_CMD; i++) {
5646                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5647                 phba->ctrl.mcc_tag[i] = i + 1;
5648                 phba->ctrl.mcc_tag_status[i + 1] = 0;
5649                 phba->ctrl.mcc_tag_available++;
5650                 memset(&phba->ctrl.ptag_state[i].tag_mem_state, 0,
5651                        sizeof(struct be_dma_mem));
5652         }
5653
5654         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
5655
5656         snprintf(wq_name, sizeof(wq_name), "beiscsi_%02x_wq",
5657                  phba->shost->host_no);
5658         phba->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 1, wq_name);
5659         if (!phba->wq) {
5660                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5661                             "BM_%d : beiscsi_dev_probe-"
5662                             "Failed to allocate work queue\n");
5663                 ret = -ENOMEM;
5664                 goto free_twq;
5665         }
5666
5667         INIT_DELAYED_WORK(&phba->eqd_update, beiscsi_eqd_update_work);
5668
5669         phwi_ctrlr = phba->phwi_ctrlr;
5670         phwi_context = phwi_ctrlr->phwi_ctxt;
5671
5672         for (i = 0; i < phba->num_cpus; i++) {
5673                 pbe_eq = &phwi_context->be_eq[i];
5674                 irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget, be_iopoll);
5675         }
5676
5677         i = (phba->pcidev->msix_enabled) ? i : 0;
5678         /* Work item for MCC handling */
5679         pbe_eq = &phwi_context->be_eq[i];
5680         INIT_WORK(&pbe_eq->mcc_work, beiscsi_mcc_work);
5681
5682         ret = beiscsi_init_irqs(phba);
5683         if (ret < 0) {
5684                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5685                             "BM_%d : beiscsi_dev_probe-"
5686                             "Failed to beiscsi_init_irqs\n");
5687                 goto disable_iopoll;
5688         }
5689         hwi_enable_intr(phba);
5690
5691         ret = iscsi_host_add(phba->shost, &phba->pcidev->dev);
5692         if (ret)
5693                 goto free_irqs;
5694
5695         /* set online bit after port is operational */
5696         set_bit(BEISCSI_HBA_ONLINE, &phba->state);
5697         __beiscsi_log(phba, KERN_INFO,
5698                       "BM_%d : port online: 0x%lx\n", phba->state);
5699
5700         INIT_WORK(&phba->boot_work, beiscsi_boot_work);
5701         ret = beiscsi_boot_get_shandle(phba, &s_handle);
5702         if (ret > 0) {
5703                 beiscsi_start_boot_work(phba, s_handle);
5704                 /**
5705                  * Set this bit after starting the work to let
5706                  * probe handle it first.
5707                  * ASYNC event can too schedule this work.
5708                  */
5709                 set_bit(BEISCSI_HBA_BOOT_FOUND, &phba->state);
5710         }
5711
5712         beiscsi_iface_create_default(phba);
5713         schedule_delayed_work(&phba->eqd_update,
5714                               msecs_to_jiffies(BEISCSI_EQD_UPDATE_INTERVAL));
5715
5716         INIT_WORK(&phba->sess_work, beiscsi_sess_work);
5717         INIT_DELAYED_WORK(&phba->recover_port, beiscsi_recover_port);
5718         /**
5719          * Start UE detection here. UE before this will cause stall in probe
5720          * and eventually fail the probe.
5721          */
5722         timer_setup(&phba->hw_check, beiscsi_hw_health_check, 0);
5723         mod_timer(&phba->hw_check,
5724                   jiffies + msecs_to_jiffies(BEISCSI_UE_DETECT_INTERVAL));
5725         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5726                     "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
5727         return 0;
5728
5729 free_irqs:
5730         hwi_disable_intr(phba);
5731         beiscsi_free_irqs(phba);
5732 disable_iopoll:
5733         for (i = 0; i < phba->num_cpus; i++) {
5734                 pbe_eq = &phwi_context->be_eq[i];
5735                 irq_poll_disable(&pbe_eq->iopoll);
5736         }
5737         destroy_workqueue(phba->wq);
5738 free_twq:
5739         hwi_cleanup_port(phba);
5740         beiscsi_cleanup_port(phba);
5741         beiscsi_free_mem(phba);
5742 free_port:
5743         pci_free_consistent(phba->pcidev,
5744                             phba->ctrl.mbox_mem_alloced.size,
5745                             phba->ctrl.mbox_mem_alloced.va,
5746                             phba->ctrl.mbox_mem_alloced.dma);
5747         beiscsi_unmap_pci_function(phba);
5748 free_hba:
5749         pci_disable_msix(phba->pcidev);
5750         pci_dev_put(phba->pcidev);
5751         iscsi_host_free(phba->shost);
5752         pci_set_drvdata(pcidev, NULL);
5753 disable_pci:
5754         pci_release_regions(pcidev);
5755         pci_disable_device(pcidev);
5756         return ret;
5757 }
5758
5759 static void beiscsi_remove(struct pci_dev *pcidev)
5760 {
5761         struct beiscsi_hba *phba = NULL;
5762
5763         phba = pci_get_drvdata(pcidev);
5764         if (!phba) {
5765                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
5766                 return;
5767         }
5768
5769         /* first stop UE detection before unloading */
5770         del_timer_sync(&phba->hw_check);
5771         cancel_delayed_work_sync(&phba->recover_port);
5772         cancel_work_sync(&phba->sess_work);
5773
5774         beiscsi_iface_destroy_default(phba);
5775         iscsi_host_remove(phba->shost);
5776         beiscsi_disable_port(phba, 1);
5777
5778         /* after cancelling boot_work */
5779         iscsi_boot_destroy_kset(phba->boot_struct.boot_kset);
5780
5781         /* free all resources */
5782         destroy_workqueue(phba->wq);
5783         beiscsi_free_mem(phba);
5784
5785         /* ctrl uninit */
5786         beiscsi_unmap_pci_function(phba);
5787         pci_free_consistent(phba->pcidev,
5788                             phba->ctrl.mbox_mem_alloced.size,
5789                             phba->ctrl.mbox_mem_alloced.va,
5790                             phba->ctrl.mbox_mem_alloced.dma);
5791
5792         pci_dev_put(phba->pcidev);
5793         iscsi_host_free(phba->shost);
5794         pci_disable_pcie_error_reporting(pcidev);
5795         pci_set_drvdata(pcidev, NULL);
5796         pci_release_regions(pcidev);
5797         pci_disable_device(pcidev);
5798 }
5799
5800
5801 static struct pci_error_handlers beiscsi_eeh_handlers = {
5802         .error_detected = beiscsi_eeh_err_detected,
5803         .slot_reset = beiscsi_eeh_reset,
5804         .resume = beiscsi_eeh_resume,
5805 };
5806
5807 struct iscsi_transport beiscsi_iscsi_transport = {
5808         .owner = THIS_MODULE,
5809         .name = DRV_NAME,
5810         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
5811                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
5812         .create_session = beiscsi_session_create,
5813         .destroy_session = beiscsi_session_destroy,
5814         .create_conn = beiscsi_conn_create,
5815         .bind_conn = beiscsi_conn_bind,
5816         .destroy_conn = iscsi_conn_teardown,
5817         .attr_is_visible = beiscsi_attr_is_visible,
5818         .set_iface_param = beiscsi_iface_set_param,
5819         .get_iface_param = beiscsi_iface_get_param,
5820         .set_param = beiscsi_set_param,
5821         .get_conn_param = iscsi_conn_get_param,
5822         .get_session_param = iscsi_session_get_param,
5823         .get_host_param = beiscsi_get_host_param,
5824         .start_conn = beiscsi_conn_start,
5825         .stop_conn = iscsi_conn_stop,
5826         .send_pdu = iscsi_conn_send_pdu,
5827         .xmit_task = beiscsi_task_xmit,
5828         .cleanup_task = beiscsi_cleanup_task,
5829         .alloc_pdu = beiscsi_alloc_pdu,
5830         .parse_pdu_itt = beiscsi_parse_pdu,
5831         .get_stats = beiscsi_conn_get_stats,
5832         .get_ep_param = beiscsi_ep_get_param,
5833         .ep_connect = beiscsi_ep_connect,
5834         .ep_poll = beiscsi_ep_poll,
5835         .ep_disconnect = beiscsi_ep_disconnect,
5836         .session_recovery_timedout = iscsi_session_recovery_timedout,
5837         .bsg_request = beiscsi_bsg_request,
5838 };
5839
5840 static struct pci_driver beiscsi_pci_driver = {
5841         .name = DRV_NAME,
5842         .probe = beiscsi_dev_probe,
5843         .remove = beiscsi_remove,
5844         .id_table = beiscsi_pci_id_table,
5845         .err_handler = &beiscsi_eeh_handlers
5846 };
5847
5848 static int __init beiscsi_module_init(void)
5849 {
5850         int ret;
5851
5852         beiscsi_scsi_transport =
5853                         iscsi_register_transport(&beiscsi_iscsi_transport);
5854         if (!beiscsi_scsi_transport) {
5855                 printk(KERN_ERR
5856                        "beiscsi_module_init - Unable to  register beiscsi transport.\n");
5857                 return -ENOMEM;
5858         }
5859         printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5860                &beiscsi_iscsi_transport);
5861
5862         ret = pci_register_driver(&beiscsi_pci_driver);
5863         if (ret) {
5864                 printk(KERN_ERR
5865                        "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
5866                 goto unregister_iscsi_transport;
5867         }
5868         return 0;
5869
5870 unregister_iscsi_transport:
5871         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5872         return ret;
5873 }
5874
5875 static void __exit beiscsi_module_exit(void)
5876 {
5877         pci_unregister_driver(&beiscsi_pci_driver);
5878         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5879 }
5880
5881 module_init(beiscsi_module_init);
5882 module_exit(beiscsi_module_exit);