drivers/infiniband/ulp/srp/ib_srp.c

   1 /*
   2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/init.h>
  37 #include <linux/slab.h>
  38 #include <linux/err.h>
  39 #include <linux/string.h>
  40 #include <linux/parser.h>
  41 #include <linux/random.h>
  42 #include <linux/jiffies.h>
  43
  44 #include <linux/atomic.h>
  45
  46 #include <scsi/scsi.h>
  47 #include <scsi/scsi_device.h>
  48 #include <scsi/scsi_dbg.h>
  49 #include <scsi/scsi_tcq.h>
  50 #include <scsi/srp.h>
  51 #include <scsi/scsi_transport_srp.h>
  52
  53 #include "ib_srp.h"
  54
  55 #define DRV_NAME        "ib_srp"
  56 #define PFX             DRV_NAME ": "
  57 #define DRV_VERSION     "1.0"
  58 #define DRV_RELDATE     "July 1, 2013"
  59
  60 MODULE_AUTHOR("Roland Dreier");
  61 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator "
  62                    "v" DRV_VERSION " (" DRV_RELDATE ")");
  63 MODULE_LICENSE("Dual BSD/GPL");
  64
  65 static unsigned int srp_sg_tablesize;
  66 static unsigned int cmd_sg_entries;
  67 static unsigned int indirect_sg_entries;
  68 static bool allow_ext_sg;
  69 static bool prefer_fr;
  70 static bool register_always;
  71 static int topspin_workarounds = 1;
  72
  73 module_param(srp_sg_tablesize, uint, 0444);
  74 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
  75
  76 module_param(cmd_sg_entries, uint, 0444);
  77 MODULE_PARM_DESC(cmd_sg_entries,
  78                  "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
  79
  80 module_param(indirect_sg_entries, uint, 0444);
  81 MODULE_PARM_DESC(indirect_sg_entries,
  82                  "Default max number of gather/scatter entries (default is 12, max is " __stringify(SCSI_MAX_SG_CHAIN_SEGMENTS) ")");
  83
  84 module_param(allow_ext_sg, bool, 0444);
  85 MODULE_PARM_DESC(allow_ext_sg,
  86                   "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
  87
  88 module_param(topspin_workarounds, int, 0444);
  89 MODULE_PARM_DESC(topspin_workarounds,
  90                  "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
  91
  92 module_param(prefer_fr, bool, 0444);
  93 MODULE_PARM_DESC(prefer_fr,
  94 "Whether to use fast registration if both FMR and fast registration are supported");
  95
  96 module_param(register_always, bool, 0444);
  97 MODULE_PARM_DESC(register_always,
  98                  "Use memory registration even for contiguous memory regions");
  99
 100 static struct kernel_param_ops srp_tmo_ops;
 101
 102 static int srp_reconnect_delay = 10;
 103 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
 104                 S_IRUGO | S_IWUSR);
 105 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
 106
 107 static int srp_fast_io_fail_tmo = 15;
 108 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
 109                 S_IRUGO | S_IWUSR);
 110 MODULE_PARM_DESC(fast_io_fail_tmo,
 111                  "Number of seconds between the observation of a transport"
 112                  " layer error and failing all I/O. \"off\" means that this"
 113                  " functionality is disabled.");
 114
 115 static int srp_dev_loss_tmo = 600;
 116 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
 117                 S_IRUGO | S_IWUSR);
 118 MODULE_PARM_DESC(dev_loss_tmo,
 119                  "Maximum number of seconds that the SRP transport should"
 120                  " insulate transport layer errors. After this time has been"
 121                  " exceeded the SCSI host is removed. Should be"
 122                  " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
 123                  " if fast_io_fail_tmo has not been set. \"off\" means that"
 124                  " this functionality is disabled.");
 125
 126 static void srp_add_one(struct ib_device *device);
 127 static void srp_remove_one(struct ib_device *device);
 128 static void srp_recv_completion(struct ib_cq *cq, void *target_ptr);
 129 static void srp_send_completion(struct ib_cq *cq, void *target_ptr);
 130 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event);
 131
 132 static struct scsi_transport_template *ib_srp_transport_template;
 133
 134 static struct ib_client srp_client = {
 135         .name   = "srp",
 136         .add    = srp_add_one,
 137         .remove = srp_remove_one
 138 };
 139
 140 static struct ib_sa_client srp_sa_client;
 141
 142 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
 143 {
 144         int tmo = *(int *)kp->arg;
 145
 146         if (tmo >= 0)
 147                 return sprintf(buffer, "%d", tmo);
 148         else
 149                 return sprintf(buffer, "off");
 150 }
 151
 152 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
 153 {
 154         int tmo, res;
 155
 156         if (strncmp(val, "off", 3) != 0) {
 157                 res = kstrtoint(val, 0, &tmo);
 158                 if (res)
 159                         goto out;
 160         } else {
 161                 tmo = -1;
 162         }
 163         if (kp->arg == &srp_reconnect_delay)
 164                 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
 165                                     srp_dev_loss_tmo);
 166         else if (kp->arg == &srp_fast_io_fail_tmo)
 167                 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
 168         else
 169                 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
 170                                     tmo);
 171         if (res)
 172                 goto out;
 173         *(int *)kp->arg = tmo;
 174
 175 out:
 176         return res;
 177 }
 178
 179 static struct kernel_param_ops srp_tmo_ops = {
 180         .get = srp_tmo_get,
 181         .set = srp_tmo_set,
 182 };
 183
 184 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 185 {
 186         return (struct srp_target_port *) host->hostdata;
 187 }
 188
 189 static const char *srp_target_info(struct Scsi_Host *host)
 190 {
 191         return host_to_target(host)->target_name;
 192 }
 193
 194 static int srp_target_is_topspin(struct srp_target_port *target)
 195 {
 196         static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
 197         static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
 198
 199         return topspin_workarounds &&
 200                 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
 201                  !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
 202 }
 203
 204 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
 205                                    gfp_t gfp_mask,
 206                                    enum dma_data_direction direction)
 207 {
 208         struct srp_iu *iu;
 209
 210         iu = kmalloc(sizeof *iu, gfp_mask);
 211         if (!iu)
 212                 goto out;
 213
 214         iu->buf = kzalloc(size, gfp_mask);
 215         if (!iu->buf)
 216                 goto out_free_iu;
 217
 218         iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
 219                                     direction);
 220         if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
 221                 goto out_free_buf;
 222
 223         iu->size      = size;
 224         iu->direction = direction;
 225
 226         return iu;
 227
 228 out_free_buf:
 229         kfree(iu->buf);
 230 out_free_iu:
 231         kfree(iu);
 232 out:
 233         return NULL;
 234 }
 235
 236 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
 237 {
 238         if (!iu)
 239                 return;
 240
 241         ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
 242                             iu->direction);
 243         kfree(iu->buf);
 244         kfree(iu);
 245 }
 246
 247 static void srp_qp_event(struct ib_event *event, void *context)
 248 {
 249         pr_debug("QP event %d\n", event->event);
 250 }
 251
 252 static int srp_init_qp(struct srp_target_port *target,
 253                        struct ib_qp *qp)
 254 {
 255         struct ib_qp_attr *attr;
 256         int ret;
 257
 258         attr = kmalloc(sizeof *attr, GFP_KERNEL);
 259         if (!attr)
 260                 return -ENOMEM;
 261
 262         ret = ib_find_pkey(target->srp_host->srp_dev->dev,
 263                            target->srp_host->port,
 264                            be16_to_cpu(target->path.pkey),
 265                            &attr->pkey_index);
 266         if (ret)
 267                 goto out;
 268
 269         attr->qp_state        = IB_QPS_INIT;
 270         attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
 271                                     IB_ACCESS_REMOTE_WRITE);
 272         attr->port_num        = target->srp_host->port;
 273
 274         ret = ib_modify_qp(qp, attr,
 275                            IB_QP_STATE          |
 276                            IB_QP_PKEY_INDEX     |
 277                            IB_QP_ACCESS_FLAGS   |
 278                            IB_QP_PORT);
 279
 280 out:
 281         kfree(attr);
 282         return ret;
 283 }
 284
 285 static int srp_new_cm_id(struct srp_target_port *target)
 286 {
 287         struct ib_cm_id *new_cm_id;
 288
 289         new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
 290                                     srp_cm_handler, target);
 291         if (IS_ERR(new_cm_id))
 292                 return PTR_ERR(new_cm_id);
 293
 294         if (target->cm_id)
 295                 ib_destroy_cm_id(target->cm_id);
 296         target->cm_id = new_cm_id;
 297
 298         return 0;
 299 }
 300
 301 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target)
 302 {
 303         struct srp_device *dev = target->srp_host->srp_dev;
 304         struct ib_fmr_pool_param fmr_param;
 305
 306         memset(&fmr_param, 0, sizeof(fmr_param));
 307         fmr_param.pool_size         = target->scsi_host->can_queue;
 308         fmr_param.dirty_watermark   = fmr_param.pool_size / 4;
 309         fmr_param.cache             = 1;
 310         fmr_param.max_pages_per_fmr = dev->max_pages_per_mr;
 311         fmr_param.page_shift        = ilog2(dev->mr_page_size);
 312         fmr_param.access            = (IB_ACCESS_LOCAL_WRITE |
 313                                        IB_ACCESS_REMOTE_WRITE |
 314                                        IB_ACCESS_REMOTE_READ);
 315
 316         return ib_create_fmr_pool(dev->pd, &fmr_param);
 317 }
 318
 319 /**
 320  * srp_destroy_fr_pool() - free the resources owned by a pool
 321  * @pool: Fast registration pool to be destroyed.
 322  */
 323 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
 324 {
 325         int i;
 326         struct srp_fr_desc *d;
 327
 328         if (!pool)
 329                 return;
 330
 331         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 332                 if (d->frpl)
 333                         ib_free_fast_reg_page_list(d->frpl);
 334                 if (d->mr)
 335                         ib_dereg_mr(d->mr);
 336         }
 337         kfree(pool);
 338 }
 339
 340 /**
 341  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
 342  * @device:            IB device to allocate fast registration descriptors for.
 343  * @pd:                Protection domain associated with the FR descriptors.
 344  * @pool_size:         Number of descriptors to allocate.
 345  * @max_page_list_len: Maximum fast registration work request page list length.
 346  */
 347 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
 348                                               struct ib_pd *pd, int pool_size,
 349                                               int max_page_list_len)
 350 {
 351         struct srp_fr_pool *pool;
 352         struct srp_fr_desc *d;
 353         struct ib_mr *mr;
 354         struct ib_fast_reg_page_list *frpl;
 355         int i, ret = -EINVAL;
 356
 357         if (pool_size <= 0)
 358                 goto err;
 359         ret = -ENOMEM;
 360         pool = kzalloc(sizeof(struct srp_fr_pool) +
 361                        pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL);
 362         if (!pool)
 363                 goto err;
 364         pool->size = pool_size;
 365         pool->max_page_list_len = max_page_list_len;
 366         spin_lock_init(&pool->lock);
 367         INIT_LIST_HEAD(&pool->free_list);
 368
 369         for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
 370                 mr = ib_alloc_fast_reg_mr(pd, max_page_list_len);
 371                 if (IS_ERR(mr)) {
 372                         ret = PTR_ERR(mr);
 373                         goto destroy_pool;
 374                 }
 375                 d->mr = mr;
 376                 frpl = ib_alloc_fast_reg_page_list(device, max_page_list_len);
 377                 if (IS_ERR(frpl)) {
 378                         ret = PTR_ERR(frpl);
 379                         goto destroy_pool;
 380                 }
 381                 d->frpl = frpl;
 382                 list_add_tail(&d->entry, &pool->free_list);
 383         }
 384
 385 out:
 386         return pool;
 387
 388 destroy_pool:
 389         srp_destroy_fr_pool(pool);
 390
 391 err:
 392         pool = ERR_PTR(ret);
 393         goto out;
 394 }
 395
 396 /**
 397  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
 398  * @pool: Pool to obtain descriptor from.
 399  */
 400 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
 401 {
 402         struct srp_fr_desc *d = NULL;
 403         unsigned long flags;
 404
 405         spin_lock_irqsave(&pool->lock, flags);
 406         if (!list_empty(&pool->free_list)) {
 407                 d = list_first_entry(&pool->free_list, typeof(*d), entry);
 408                 list_del(&d->entry);
 409         }
 410         spin_unlock_irqrestore(&pool->lock, flags);
 411
 412         return d;
 413 }
 414
 415 /**
 416  * srp_fr_pool_put() - put an FR descriptor back in the free list
 417  * @pool: Pool the descriptor was allocated from.
 418  * @desc: Pointer to an array of fast registration descriptor pointers.
 419  * @n:    Number of descriptors to put back.
 420  *
 421  * Note: The caller must already have queued an invalidation request for
 422  * desc->mr->rkey before calling this function.
 423  */
 424 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
 425                             int n)
 426 {
 427         unsigned long flags;
 428         int i;
 429
 430         spin_lock_irqsave(&pool->lock, flags);
 431         for (i = 0; i < n; i++)
 432                 list_add(&desc[i]->entry, &pool->free_list);
 433         spin_unlock_irqrestore(&pool->lock, flags);
 434 }
 435
 436 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
 437 {
 438         struct srp_device *dev = target->srp_host->srp_dev;
 439
 440         return srp_create_fr_pool(dev->dev, dev->pd,
 441                                   target->scsi_host->can_queue,
 442                                   dev->max_pages_per_mr);
 443 }
 444
 445 static int srp_create_target_ib(struct srp_target_port *target)
 446 {
 447         struct srp_device *dev = target->srp_host->srp_dev;
 448         struct ib_qp_init_attr *init_attr;
 449         struct ib_cq *recv_cq, *send_cq;
 450         struct ib_qp *qp;
 451         struct ib_fmr_pool *fmr_pool = NULL;
 452         struct srp_fr_pool *fr_pool = NULL;
 453         const int m = 1 + dev->use_fast_reg;
 454         int ret;
 455
 456         init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
 457         if (!init_attr)
 458                 return -ENOMEM;
 459
 460         recv_cq = ib_create_cq(dev->dev, srp_recv_completion, NULL, target,
 461                                target->queue_size, target->comp_vector);
 462         if (IS_ERR(recv_cq)) {
 463                 ret = PTR_ERR(recv_cq);
 464                 goto err;
 465         }
 466
 467         send_cq = ib_create_cq(dev->dev, srp_send_completion, NULL, target,
 468                                m * target->queue_size, target->comp_vector);
 469         if (IS_ERR(send_cq)) {
 470                 ret = PTR_ERR(send_cq);
 471                 goto err_recv_cq;
 472         }
 473
 474         ib_req_notify_cq(recv_cq, IB_CQ_NEXT_COMP);
 475
 476         init_attr->event_handler       = srp_qp_event;
 477         init_attr->cap.max_send_wr     = m * target->queue_size;
 478         init_attr->cap.max_recv_wr     = target->queue_size;
 479         init_attr->cap.max_recv_sge    = 1;
 480         init_attr->cap.max_send_sge    = 1;
 481         init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
 482         init_attr->qp_type             = IB_QPT_RC;
 483         init_attr->send_cq             = send_cq;
 484         init_attr->recv_cq             = recv_cq;
 485
 486         qp = ib_create_qp(dev->pd, init_attr);
 487         if (IS_ERR(qp)) {
 488                 ret = PTR_ERR(qp);
 489                 goto err_send_cq;
 490         }
 491
 492         ret = srp_init_qp(target, qp);
 493         if (ret)
 494                 goto err_qp;
 495
 496         if (dev->use_fast_reg && dev->has_fr) {
 497                 fr_pool = srp_alloc_fr_pool(target);
 498                 if (IS_ERR(fr_pool)) {
 499                         ret = PTR_ERR(fr_pool);
 500                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 501                                      "FR pool allocation failed (%d)\n", ret);
 502                         goto err_qp;
 503                 }
 504                 if (target->fr_pool)
 505                         srp_destroy_fr_pool(target->fr_pool);
 506                 target->fr_pool = fr_pool;
 507         } else if (!dev->use_fast_reg && dev->has_fmr) {
 508                 fmr_pool = srp_alloc_fmr_pool(target);
 509                 if (IS_ERR(fmr_pool)) {
 510                         ret = PTR_ERR(fmr_pool);
 511                         shost_printk(KERN_WARNING, target->scsi_host, PFX
 512                                      "FMR pool allocation failed (%d)\n", ret);
 513                         goto err_qp;
 514                 }
 515                 if (target->fmr_pool)
 516                         ib_destroy_fmr_pool(target->fmr_pool);
 517                 target->fmr_pool = fmr_pool;
 518         }
 519
 520         if (target->qp)
 521                 ib_destroy_qp(target->qp);
 522         if (target->recv_cq)
 523                 ib_destroy_cq(target->recv_cq);
 524         if (target->send_cq)
 525                 ib_destroy_cq(target->send_cq);
 526
 527         target->qp = qp;
 528         target->recv_cq = recv_cq;
 529         target->send_cq = send_cq;
 530
 531         kfree(init_attr);
 532         return 0;
 533
 534 err_qp:
 535         ib_destroy_qp(qp);
 536
 537 err_send_cq:
 538         ib_destroy_cq(send_cq);
 539
 540 err_recv_cq:
 541         ib_destroy_cq(recv_cq);
 542
 543 err:
 544         kfree(init_attr);
 545         return ret;
 546 }
 547
 548 /*
 549  * Note: this function may be called without srp_alloc_iu_bufs() having been
 550  * invoked. Hence the target->[rt]x_ring checks.
 551  */
 552 static void srp_free_target_ib(struct srp_target_port *target)
 553 {
 554         struct srp_device *dev = target->srp_host->srp_dev;
 555         int i;
 556
 557         if (dev->use_fast_reg) {
 558                 if (target->fr_pool)
 559                         srp_destroy_fr_pool(target->fr_pool);
 560         } else {
 561                 if (target->fmr_pool)
 562                         ib_destroy_fmr_pool(target->fmr_pool);
 563         }
 564         ib_destroy_qp(target->qp);
 565         ib_destroy_cq(target->send_cq);
 566         ib_destroy_cq(target->recv_cq);
 567
 568         target->qp = NULL;
 569         target->send_cq = target->recv_cq = NULL;
 570
 571         if (target->rx_ring) {
 572                 for (i = 0; i < target->queue_size; ++i)
 573                         srp_free_iu(target->srp_host, target->rx_ring[i]);
 574                 kfree(target->rx_ring);
 575                 target->rx_ring = NULL;
 576         }
 577         if (target->tx_ring) {
 578                 for (i = 0; i < target->queue_size; ++i)
 579                         srp_free_iu(target->srp_host, target->tx_ring[i]);
 580                 kfree(target->tx_ring);
 581                 target->tx_ring = NULL;
 582         }
 583 }
 584
 585 static void srp_path_rec_completion(int status,
 586                                     struct ib_sa_path_rec *pathrec,
 587                                     void *target_ptr)
 588 {
 589         struct srp_target_port *target = target_ptr;
 590
 591         target->status = status;
 592         if (status)
 593                 shost_printk(KERN_ERR, target->scsi_host,
 594                              PFX "Got failed path rec status %d\n", status);
 595         else
 596                 target->path = *pathrec;
 597         complete(&target->done);
 598 }
 599
 600 static int srp_lookup_path(struct srp_target_port *target)
 601 {
 602         int ret;
 603
 604         target->path.numb_path = 1;
 605
 606         init_completion(&target->done);
 607
 608         target->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
 609                                                    target->srp_host->srp_dev->dev,
 610                                                    target->srp_host->port,
 611                                                    &target->path,
 612                                                    IB_SA_PATH_REC_SERVICE_ID    |
 613                                                    IB_SA_PATH_REC_DGID          |
 614                                                    IB_SA_PATH_REC_SGID          |
 615                                                    IB_SA_PATH_REC_NUMB_PATH     |
 616                                                    IB_SA_PATH_REC_PKEY,
 617                                                    SRP_PATH_REC_TIMEOUT_MS,
 618                                                    GFP_KERNEL,
 619                                                    srp_path_rec_completion,
 620                                                    target, &target->path_query);
 621         if (target->path_query_id < 0)
 622                 return target->path_query_id;
 623
 624         ret = wait_for_completion_interruptible(&target->done);
 625         if (ret < 0)
 626                 return ret;
 627
 628         if (target->status < 0)
 629                 shost_printk(KERN_WARNING, target->scsi_host,
 630                              PFX "Path record query failed\n");
 631
 632         return target->status;
 633 }
 634
 635 static int srp_send_req(struct srp_target_port *target)
 636 {
 637         struct {
 638                 struct ib_cm_req_param param;
 639                 struct srp_login_req   priv;
 640         } *req = NULL;
 641         int status;
 642
 643         req = kzalloc(sizeof *req, GFP_KERNEL);
 644         if (!req)
 645                 return -ENOMEM;
 646
 647         req->param.primary_path               = &target->path;
 648         req->param.alternate_path             = NULL;
 649         req->param.service_id                 = target->service_id;
 650         req->param.qp_num                     = target->qp->qp_num;
 651         req->param.qp_type                    = target->qp->qp_type;
 652         req->param.private_data               = &req->priv;
 653         req->param.private_data_len           = sizeof req->priv;
 654         req->param.flow_control               = 1;
 655
 656         get_random_bytes(&req->param.starting_psn, 4);
 657         req->param.starting_psn              &= 0xffffff;
 658
 659         /*
 660          * Pick some arbitrary defaults here; we could make these
 661          * module parameters if anyone cared about setting them.
 662          */
 663         req->param.responder_resources        = 4;
 664         req->param.remote_cm_response_timeout = 20;
 665         req->param.local_cm_response_timeout  = 20;
 666         req->param.retry_count                = target->tl_retry_count;
 667         req->param.rnr_retry_count            = 7;
 668         req->param.max_cm_retries             = 15;
 669
 670         req->priv.opcode        = SRP_LOGIN_REQ;
 671         req->priv.tag           = 0;
 672         req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
 673         req->priv.req_buf_fmt   = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 674                                               SRP_BUF_FORMAT_INDIRECT);
 675         /*
 676          * In the published SRP specification (draft rev. 16a), the
 677          * port identifier format is 8 bytes of ID extension followed
 678          * by 8 bytes of GUID.  Older drafts put the two halves in the
 679          * opposite order, so that the GUID comes first.
 680          *
 681          * Targets conforming to these obsolete drafts can be
 682          * recognized by the I/O Class they report.
 683          */
 684         if (target->io_class == SRP_REV10_IB_IO_CLASS) {
 685                 memcpy(req->priv.initiator_port_id,
 686                        &target->path.sgid.global.interface_id, 8);
 687                 memcpy(req->priv.initiator_port_id + 8,
 688                        &target->initiator_ext, 8);
 689                 memcpy(req->priv.target_port_id,     &target->ioc_guid, 8);
 690                 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8);
 691         } else {
 692                 memcpy(req->priv.initiator_port_id,
 693                        &target->initiator_ext, 8);
 694                 memcpy(req->priv.initiator_port_id + 8,
 695                        &target->path.sgid.global.interface_id, 8);
 696                 memcpy(req->priv.target_port_id,     &target->id_ext, 8);
 697                 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8);
 698         }
 699
 700         /*
 701          * Topspin/Cisco SRP targets will reject our login unless we
 702          * zero out the first 8 bytes of our initiator port ID and set
 703          * the second 8 bytes to the local node GUID.
 704          */
 705         if (srp_target_is_topspin(target)) {
 706                 shost_printk(KERN_DEBUG, target->scsi_host,
 707                              PFX "Topspin/Cisco initiator port ID workaround "
 708                              "activated for target GUID %016llx\n",
 709                              (unsigned long long) be64_to_cpu(target->ioc_guid));
 710                 memset(req->priv.initiator_port_id, 0, 8);
 711                 memcpy(req->priv.initiator_port_id + 8,
 712                        &target->srp_host->srp_dev->dev->node_guid, 8);
 713         }
 714
 715         status = ib_send_cm_req(target->cm_id, &req->param);
 716
 717         kfree(req);
 718
 719         return status;
 720 }
 721
 722 static bool srp_queue_remove_work(struct srp_target_port *target)
 723 {
 724         bool changed = false;
 725
 726         spin_lock_irq(&target->lock);
 727         if (target->state != SRP_TARGET_REMOVED) {
 728                 target->state = SRP_TARGET_REMOVED;
 729                 changed = true;
 730         }
 731         spin_unlock_irq(&target->lock);
 732
 733         if (changed)
 734                 queue_work(system_long_wq, &target->remove_work);
 735
 736         return changed;
 737 }
 738
 739 static bool srp_change_conn_state(struct srp_target_port *target,
 740                                   bool connected)
 741 {
 742         bool changed = false;
 743
 744         spin_lock_irq(&target->lock);
 745         if (target->connected != connected) {
 746                 target->connected = connected;
 747                 changed = true;
 748         }
 749         spin_unlock_irq(&target->lock);
 750
 751         return changed;
 752 }
 753
 754 static void srp_disconnect_target(struct srp_target_port *target)
 755 {
 756         if (srp_change_conn_state(target, false)) {
 757                 /* XXX should send SRP_I_LOGOUT request */
 758
 759                 if (ib_send_cm_dreq(target->cm_id, NULL, 0)) {
 760                         shost_printk(KERN_DEBUG, target->scsi_host,
 761                                      PFX "Sending CM DREQ failed\n");
 762                 }
 763         }
 764 }
 765
 766 static void srp_free_req_data(struct srp_target_port *target)
 767 {
 768         struct srp_device *dev = target->srp_host->srp_dev;
 769         struct ib_device *ibdev = dev->dev;
 770         struct srp_request *req;
 771         int i;
 772
 773         if (!target->req_ring)
 774                 return;
 775
 776         for (i = 0; i < target->req_ring_size; ++i) {
 777                 req = &target->req_ring[i];
 778                 if (dev->use_fast_reg)
 779                         kfree(req->fr_list);
 780                 else
 781                         kfree(req->fmr_list);
 782                 kfree(req->map_page);
 783                 if (req->indirect_dma_addr) {
 784                         ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
 785                                             target->indirect_size,
 786                                             DMA_TO_DEVICE);
 787                 }
 788                 kfree(req->indirect_desc);
 789         }
 790
 791         kfree(target->req_ring);
 792         target->req_ring = NULL;
 793 }
 794
 795 static int srp_alloc_req_data(struct srp_target_port *target)
 796 {
 797         struct srp_device *srp_dev = target->srp_host->srp_dev;
 798         struct ib_device *ibdev = srp_dev->dev;
 799         struct srp_request *req;
 800         void *mr_list;
 801         dma_addr_t dma_addr;
 802         int i, ret = -ENOMEM;
 803
 804         INIT_LIST_HEAD(&target->free_reqs);
 805
 806         target->req_ring = kzalloc(target->req_ring_size *
 807                                    sizeof(*target->req_ring), GFP_KERNEL);
 808         if (!target->req_ring)
 809                 goto out;
 810
 811         for (i = 0; i < target->req_ring_size; ++i) {
 812                 req = &target->req_ring[i];
 813                 mr_list = kmalloc(target->cmd_sg_cnt * sizeof(void *),
 814                                   GFP_KERNEL);
 815                 if (!mr_list)
 816                         goto out;
 817                 if (srp_dev->use_fast_reg)
 818                         req->fr_list = mr_list;
 819                 else
 820                         req->fmr_list = mr_list;
 821                 req->map_page = kmalloc(srp_dev->max_pages_per_mr *
 822                                         sizeof(void *), GFP_KERNEL);
 823                 if (!req->map_page)
 824                         goto out;
 825                 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
 826                 if (!req->indirect_desc)
 827                         goto out;
 828
 829                 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
 830                                              target->indirect_size,
 831                                              DMA_TO_DEVICE);
 832                 if (ib_dma_mapping_error(ibdev, dma_addr))
 833                         goto out;
 834
 835                 req->indirect_dma_addr = dma_addr;
 836                 req->index = i;
 837                 list_add_tail(&req->list, &target->free_reqs);
 838         }
 839         ret = 0;
 840
 841 out:
 842         return ret;
 843 }
 844
 845 /**
 846  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
 847  * @shost: SCSI host whose attributes to remove from sysfs.
 848  *
 849  * Note: Any attributes defined in the host template and that did not exist
 850  * before invocation of this function will be ignored.
 851  */
 852 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
 853 {
 854         struct device_attribute **attr;
 855
 856         for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr)
 857                 device_remove_file(&shost->shost_dev, *attr);
 858 }
 859
 860 static void srp_remove_target(struct srp_target_port *target)
 861 {
 862         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 863
 864         srp_del_scsi_host_attr(target->scsi_host);
 865         srp_rport_get(target->rport);
 866         srp_remove_host(target->scsi_host);
 867         scsi_remove_host(target->scsi_host);
 868         srp_stop_rport_timers(target->rport);
 869         srp_disconnect_target(target);
 870         ib_destroy_cm_id(target->cm_id);
 871         srp_free_target_ib(target);
 872         cancel_work_sync(&target->tl_err_work);
 873         srp_rport_put(target->rport);
 874         srp_free_req_data(target);
 875
 876         spin_lock(&target->srp_host->target_lock);
 877         list_del(&target->list);
 878         spin_unlock(&target->srp_host->target_lock);
 879
 880         scsi_host_put(target->scsi_host);
 881 }
 882
 883 static void srp_remove_work(struct work_struct *work)
 884 {
 885         struct srp_target_port *target =
 886                 container_of(work, struct srp_target_port, remove_work);
 887
 888         WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
 889
 890         srp_remove_target(target);
 891 }
 892
 893 static void srp_rport_delete(struct srp_rport *rport)
 894 {
 895         struct srp_target_port *target = rport->lld_data;
 896
 897         srp_queue_remove_work(target);
 898 }
 899
 900 static int srp_connect_target(struct srp_target_port *target)
 901 {
 902         int retries = 3;
 903         int ret;
 904
 905         WARN_ON_ONCE(target->connected);
 906
 907         target->qp_in_error = false;
 908
 909         ret = srp_lookup_path(target);
 910         if (ret)
 911                 return ret;
 912
 913         while (1) {
 914                 init_completion(&target->done);
 915                 ret = srp_send_req(target);
 916                 if (ret)
 917                         return ret;
 918                 ret = wait_for_completion_interruptible(&target->done);
 919                 if (ret < 0)
 920                         return ret;
 921
 922                 /*
 923                  * The CM event handling code will set status to
 924                  * SRP_PORT_REDIRECT if we get a port redirect REJ
 925                  * back, or SRP_DLID_REDIRECT if we get a lid/qp
 926                  * redirect REJ back.
 927                  */
 928                 switch (target->status) {
 929                 case 0:
 930                         srp_change_conn_state(target, true);
 931                         return 0;
 932
 933                 case SRP_PORT_REDIRECT:
 934                         ret = srp_lookup_path(target);
 935                         if (ret)
 936                                 return ret;
 937                         break;
 938
 939                 case SRP_DLID_REDIRECT:
 940                         break;
 941
 942                 case SRP_STALE_CONN:
 943                         /* Our current CM id was stale, and is now in timewait.
 944                          * Try to reconnect with a new one.
 945                          */
 946                         if (!retries-- || srp_new_cm_id(target)) {
 947                                 shost_printk(KERN_ERR, target->scsi_host, PFX
 948                                              "giving up on stale connection\n");
 949                                 target->status = -ECONNRESET;
 950                                 return target->status;
 951                         }
 952
 953                         shost_printk(KERN_ERR, target->scsi_host, PFX
 954                                      "retrying stale connection\n");
 955                         break;
 956
 957                 default:
 958                         return target->status;
 959                 }
 960         }
 961 }
 962
 963 static int srp_inv_rkey(struct srp_target_port *target, u32 rkey)
 964 {
 965         struct ib_send_wr *bad_wr;
 966         struct ib_send_wr wr = {
 967                 .opcode             = IB_WR_LOCAL_INV,
 968                 .wr_id              = LOCAL_INV_WR_ID_MASK,
 969                 .next               = NULL,
 970                 .num_sge            = 0,
 971                 .send_flags         = 0,
 972                 .ex.invalidate_rkey = rkey,
 973         };
 974
 975         return ib_post_send(target->qp, &wr, &bad_wr);
 976 }
 977
 978 static void srp_unmap_data(struct scsi_cmnd *scmnd,
 979                            struct srp_target_port *target,
 980                            struct srp_request *req)
 981 {
 982         struct srp_device *dev = target->srp_host->srp_dev;
 983         struct ib_device *ibdev = dev->dev;
 984         int i, res;
 985
 986         if (!scsi_sglist(scmnd) ||
 987             (scmnd->sc_data_direction != DMA_TO_DEVICE &&
 988              scmnd->sc_data_direction != DMA_FROM_DEVICE))
 989                 return;
 990
 991         if (dev->use_fast_reg) {
 992                 struct srp_fr_desc **pfr;
 993
 994                 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
 995                         res = srp_inv_rkey(target, (*pfr)->mr->rkey);
 996                         if (res < 0) {
 997                                 shost_printk(KERN_ERR, target->scsi_host, PFX
 998                                   "Queueing INV WR for rkey %#x failed (%d)\n",
 999                                   (*pfr)->mr->rkey, res);
1000                                 queue_work(system_long_wq,
1001                                            &target->tl_err_work);
1002                         }
1003                 }
1004                 if (req->nmdesc)
1005                         srp_fr_pool_put(target->fr_pool, req->fr_list,
1006                                         req->nmdesc);
1007         } else {
1008                 struct ib_pool_fmr **pfmr;
1009
1010                 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++)
1011                         ib_fmr_pool_unmap(*pfmr);
1012         }
1013
1014         ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1015                         scmnd->sc_data_direction);
1016 }
1017
1018 /**
1019  * srp_claim_req - Take ownership of the scmnd associated with a request.
1020  * @target: SRP target port.
1021  * @req: SRP request.
1022  * @sdev: If not NULL, only take ownership for this SCSI device.
1023  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1024  *         ownership of @req->scmnd if it equals @scmnd.
1025  *
1026  * Return value:
1027  * Either NULL or a pointer to the SCSI command the caller became owner of.
1028  */
1029 static struct scsi_cmnd *srp_claim_req(struct srp_target_port *target,
1030                                        struct srp_request *req,
1031                                        struct scsi_device *sdev,
1032                                        struct scsi_cmnd *scmnd)
1033 {
1034         unsigned long flags;
1035
1036         spin_lock_irqsave(&target->lock, flags);
1037         if (req->scmnd &&
1038             (!sdev || req->scmnd->device == sdev) &&
1039             (!scmnd || req->scmnd == scmnd)) {
1040                 scmnd = req->scmnd;
1041                 req->scmnd = NULL;
1042         } else {
1043                 scmnd = NULL;
1044         }
1045         spin_unlock_irqrestore(&target->lock, flags);
1046
1047         return scmnd;
1048 }
1049
1050 /**
1051  * srp_free_req() - Unmap data and add request to the free request list.
1052  * @target: SRP target port.
1053  * @req:    Request to be freed.
1054  * @scmnd:  SCSI command associated with @req.
1055  * @req_lim_delta: Amount to be added to @target->req_lim.
1056  */
1057 static void srp_free_req(struct srp_target_port *target,
1058                          struct srp_request *req, struct scsi_cmnd *scmnd,
1059                          s32 req_lim_delta)
1060 {
1061         unsigned long flags;
1062
1063         srp_unmap_data(scmnd, target, req);
1064
1065         spin_lock_irqsave(&target->lock, flags);
1066         target->req_lim += req_lim_delta;
1067         list_add_tail(&req->list, &target->free_reqs);
1068         spin_unlock_irqrestore(&target->lock, flags);
1069 }
1070
1071 static void srp_finish_req(struct srp_target_port *target,
1072                            struct srp_request *req, struct scsi_device *sdev,
1073                            int result)
1074 {
1075         struct scsi_cmnd *scmnd = srp_claim_req(target, req, sdev, NULL);
1076
1077         if (scmnd) {
1078                 srp_free_req(target, req, scmnd, 0);
1079                 scmnd->result = result;
1080                 scmnd->scsi_done(scmnd);
1081         }
1082 }
1083
1084 static void srp_terminate_io(struct srp_rport *rport)
1085 {
1086         struct srp_target_port *target = rport->lld_data;
1087         struct Scsi_Host *shost = target->scsi_host;
1088         struct scsi_device *sdev;
1089         int i;
1090
1091         /*
1092          * Invoking srp_terminate_io() while srp_queuecommand() is running
1093          * is not safe. Hence the warning statement below.
1094          */
1095         shost_for_each_device(sdev, shost)
1096                 WARN_ON_ONCE(sdev->request_queue->request_fn_active);
1097
1098         for (i = 0; i < target->req_ring_size; ++i) {
1099                 struct srp_request *req = &target->req_ring[i];
1100                 srp_finish_req(target, req, NULL, DID_TRANSPORT_FAILFAST << 16);
1101         }
1102 }
1103
1104 /*
1105  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1106  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1107  * srp_reset_device() or srp_reset_host() calls will occur while this function
1108  * is in progress. One way to realize that is not to call this function
1109  * directly but to call srp_reconnect_rport() instead since that last function
1110  * serializes calls of this function via rport->mutex and also blocks
1111  * srp_queuecommand() calls before invoking this function.
1112  */
1113 static int srp_rport_reconnect(struct srp_rport *rport)
1114 {
1115         struct srp_target_port *target = rport->lld_data;
1116         int i, ret;
1117
1118         srp_disconnect_target(target);
1119         /*
1120          * Now get a new local CM ID so that we avoid confusing the target in
1121          * case things are really fouled up. Doing so also ensures that all CM
1122          * callbacks will have finished before a new QP is allocated.
1123          */
1124         ret = srp_new_cm_id(target);
1125
1126         for (i = 0; i < target->req_ring_size; ++i) {
1127                 struct srp_request *req = &target->req_ring[i];
1128                 srp_finish_req(target, req, NULL, DID_RESET << 16);
1129         }
1130
1131         /*
1132          * Whether or not creating a new CM ID succeeded, create a new
1133          * QP. This guarantees that all callback functions for the old QP have
1134          * finished before any send requests are posted on the new QP.
1135          */
1136         ret += srp_create_target_ib(target);
1137
1138         INIT_LIST_HEAD(&target->free_tx);
1139         for (i = 0; i < target->queue_size; ++i)
1140                 list_add(&target->tx_ring[i]->list, &target->free_tx);
1141
1142         if (ret == 0)
1143                 ret = srp_connect_target(target);
1144
1145         if (ret == 0)
1146                 shost_printk(KERN_INFO, target->scsi_host,
1147                              PFX "reconnect succeeded\n");
1148
1149         return ret;
1150 }
1151
1152 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1153                          unsigned int dma_len, u32 rkey)
1154 {
1155         struct srp_direct_buf *desc = state->desc;
1156
1157         desc->va = cpu_to_be64(dma_addr);
1158         desc->key = cpu_to_be32(rkey);
1159         desc->len = cpu_to_be32(dma_len);
1160
1161         state->total_len += dma_len;
1162         state->desc++;
1163         state->ndesc++;
1164 }
1165
1166 static int srp_map_finish_fmr(struct srp_map_state *state,
1167                               struct srp_target_port *target)
1168 {
1169         struct ib_pool_fmr *fmr;
1170         u64 io_addr = 0;
1171
1172         fmr = ib_fmr_pool_map_phys(target->fmr_pool, state->pages,
1173                                    state->npages, io_addr);
1174         if (IS_ERR(fmr))
1175                 return PTR_ERR(fmr);
1176
1177         *state->next_fmr++ = fmr;
1178         state->nmdesc++;
1179
1180         srp_map_desc(state, 0, state->dma_len, fmr->fmr->rkey);
1181
1182         return 0;
1183 }
1184
1185 static int srp_map_finish_fr(struct srp_map_state *state,
1186                              struct srp_target_port *target)
1187 {
1188         struct srp_device *dev = target->srp_host->srp_dev;
1189         struct ib_send_wr *bad_wr;
1190         struct ib_send_wr wr;
1191         struct srp_fr_desc *desc;
1192         u32 rkey;
1193
1194         desc = srp_fr_pool_get(target->fr_pool);
1195         if (!desc)
1196                 return -ENOMEM;
1197
1198         rkey = ib_inc_rkey(desc->mr->rkey);
1199         ib_update_fast_reg_key(desc->mr, rkey);
1200
1201         memcpy(desc->frpl->page_list, state->pages,
1202                sizeof(state->pages[0]) * state->npages);
1203
1204         memset(&wr, 0, sizeof(wr));
1205         wr.opcode = IB_WR_FAST_REG_MR;
1206         wr.wr_id = FAST_REG_WR_ID_MASK;
1207         wr.wr.fast_reg.iova_start = state->base_dma_addr;
1208         wr.wr.fast_reg.page_list = desc->frpl;
1209         wr.wr.fast_reg.page_list_len = state->npages;
1210         wr.wr.fast_reg.page_shift = ilog2(dev->mr_page_size);
1211         wr.wr.fast_reg.length = state->dma_len;
1212         wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
1213                                        IB_ACCESS_REMOTE_READ |
1214                                        IB_ACCESS_REMOTE_WRITE);
1215         wr.wr.fast_reg.rkey = desc->mr->lkey;
1216
1217         *state->next_fr++ = desc;
1218         state->nmdesc++;
1219
1220         srp_map_desc(state, state->base_dma_addr, state->dma_len,
1221                      desc->mr->rkey);
1222
1223         return ib_post_send(target->qp, &wr, &bad_wr);
1224 }
1225
1226 static int srp_finish_mapping(struct srp_map_state *state,
1227                               struct srp_target_port *target)
1228 {
1229         int ret = 0;
1230
1231         if (state->npages == 0)
1232                 return 0;
1233
1234         if (state->npages == 1 && !register_always)
1235                 srp_map_desc(state, state->base_dma_addr, state->dma_len,
1236                              target->rkey);
1237         else
1238                 ret = target->srp_host->srp_dev->use_fast_reg ?
1239                         srp_map_finish_fr(state, target) :
1240                         srp_map_finish_fmr(state, target);
1241
1242         if (ret == 0) {
1243                 state->npages = 0;
1244                 state->dma_len = 0;
1245         }
1246
1247         return ret;
1248 }
1249
1250 static void srp_map_update_start(struct srp_map_state *state,
1251                                  struct scatterlist *sg, int sg_index,
1252                                  dma_addr_t dma_addr)
1253 {
1254         state->unmapped_sg = sg;
1255         state->unmapped_index = sg_index;
1256         state->unmapped_addr = dma_addr;
1257 }
1258
1259 static int srp_map_sg_entry(struct srp_map_state *state,
1260                             struct srp_target_port *target,
1261                             struct scatterlist *sg, int sg_index,
1262                             bool use_mr)
1263 {
1264         struct srp_device *dev = target->srp_host->srp_dev;
1265         struct ib_device *ibdev = dev->dev;
1266         dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg);
1267         unsigned int dma_len = ib_sg_dma_len(ibdev, sg);
1268         unsigned int len;
1269         int ret;
1270
1271         if (!dma_len)
1272                 return 0;
1273
1274         if (!use_mr) {
1275                 /*
1276                  * Once we're in direct map mode for a request, we don't
1277                  * go back to FMR or FR mode, so no need to update anything
1278                  * other than the descriptor.
1279                  */
1280                 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1281                 return 0;
1282         }
1283
1284         /*
1285          * Since not all RDMA HW drivers support non-zero page offsets for
1286          * FMR, if we start at an offset into a page, don't merge into the
1287          * current FMR mapping. Finish it out, and use the kernel's MR for
1288          * this sg entry.
1289          */
1290         if ((!dev->use_fast_reg && dma_addr & ~dev->mr_page_mask) ||
1291             dma_len > dev->mr_max_size) {
1292                 ret = srp_finish_mapping(state, target);
1293                 if (ret)
1294                         return ret;
1295
1296                 srp_map_desc(state, dma_addr, dma_len, target->rkey);
1297                 srp_map_update_start(state, NULL, 0, 0);
1298                 return 0;
1299         }
1300
1301         /*
1302          * If this is the first sg that will be mapped via FMR or via FR, save
1303          * our position. We need to know the first unmapped entry, its index,
1304          * and the first unmapped address within that entry to be able to
1305          * restart mapping after an error.
1306          */
1307         if (!state->unmapped_sg)
1308                 srp_map_update_start(state, sg, sg_index, dma_addr);
1309
1310         while (dma_len) {
1311                 unsigned offset = dma_addr & ~dev->mr_page_mask;
1312                 if (state->npages == dev->max_pages_per_mr || offset != 0) {
1313                         ret = srp_finish_mapping(state, target);
1314                         if (ret)
1315                                 return ret;
1316
1317                         srp_map_update_start(state, sg, sg_index, dma_addr);
1318                 }
1319
1320                 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset);
1321
1322                 if (!state->npages)
1323                         state->base_dma_addr = dma_addr;
1324                 state->pages[state->npages++] = dma_addr & dev->mr_page_mask;
1325                 state->dma_len += len;
1326                 dma_addr += len;
1327                 dma_len -= len;
1328         }
1329
1330         /*
1331          * If the last entry of the MR wasn't a full page, then we need to
1332          * close it out and start a new one -- we can only merge at page
1333          * boundries.
1334          */
1335         ret = 0;
1336         if (len != dev->mr_page_size) {
1337                 ret = srp_finish_mapping(state, target);
1338                 if (!ret)
1339                         srp_map_update_start(state, NULL, 0, 0);
1340         }
1341         return ret;
1342 }
1343
1344 static int srp_map_sg(struct srp_map_state *state,
1345                       struct srp_target_port *target, struct srp_request *req,
1346                       struct scatterlist *scat, int count)
1347 {
1348         struct srp_device *dev = target->srp_host->srp_dev;
1349         struct ib_device *ibdev = dev->dev;
1350         struct scatterlist *sg;
1351         int i;
1352         bool use_mr;
1353
1354         state->desc     = req->indirect_desc;
1355         state->pages    = req->map_page;
1356         if (dev->use_fast_reg) {
1357                 state->next_fr = req->fr_list;
1358                 use_mr = !!target->fr_pool;
1359         } else {
1360                 state->next_fmr = req->fmr_list;
1361                 use_mr = !!target->fmr_pool;
1362         }
1363
1364         for_each_sg(scat, sg, count, i) {
1365                 if (srp_map_sg_entry(state, target, sg, i, use_mr)) {
1366                         /*
1367                          * Memory registration failed, so backtrack to the
1368                          * first unmapped entry and continue on without using
1369                          * memory registration.
1370                          */
1371                         dma_addr_t dma_addr;
1372                         unsigned int dma_len;
1373
1374 backtrack:
1375                         sg = state->unmapped_sg;
1376                         i = state->unmapped_index;
1377
1378                         dma_addr = ib_sg_dma_address(ibdev, sg);
1379                         dma_len = ib_sg_dma_len(ibdev, sg);
1380                         dma_len -= (state->unmapped_addr - dma_addr);
1381                         dma_addr = state->unmapped_addr;
1382                         use_mr = false;
1383                         srp_map_desc(state, dma_addr, dma_len, target->rkey);
1384                 }
1385         }
1386
1387         if (use_mr && srp_finish_mapping(state, target))
1388                 goto backtrack;
1389
1390         req->nmdesc = state->nmdesc;
1391
1392         return 0;
1393 }
1394
1395 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_target_port *target,
1396                         struct srp_request *req)
1397 {
1398         struct scatterlist *scat;
1399         struct srp_cmd *cmd = req->cmd->buf;
1400         int len, nents, count;
1401         struct srp_device *dev;
1402         struct ib_device *ibdev;
1403         struct srp_map_state state;
1404         struct srp_indirect_buf *indirect_hdr;
1405         u32 table_len;
1406         u8 fmt;
1407
1408         if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1409                 return sizeof (struct srp_cmd);
1410
1411         if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1412             scmnd->sc_data_direction != DMA_TO_DEVICE) {
1413                 shost_printk(KERN_WARNING, target->scsi_host,
1414                              PFX "Unhandled data direction %d\n",
1415                              scmnd->sc_data_direction);
1416                 return -EINVAL;
1417         }
1418
1419         nents = scsi_sg_count(scmnd);
1420         scat  = scsi_sglist(scmnd);
1421
1422         dev = target->srp_host->srp_dev;
1423         ibdev = dev->dev;
1424
1425         count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1426         if (unlikely(count == 0))
1427                 return -EIO;
1428
1429         fmt = SRP_DATA_DESC_DIRECT;
1430         len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf);
1431
1432         if (count == 1 && !register_always) {
1433                 /*
1434                  * The midlayer only generated a single gather/scatter
1435                  * entry, or DMA mapping coalesced everything to a
1436                  * single entry.  So a direct descriptor along with
1437                  * the DMA MR suffices.
1438                  */
1439                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1440
1441                 buf->va  = cpu_to_be64(ib_sg_dma_address(ibdev, scat));
1442                 buf->key = cpu_to_be32(target->rkey);
1443                 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat));
1444
1445                 req->nmdesc = 0;
1446                 goto map_complete;
1447         }
1448
1449         /*
1450          * We have more than one scatter/gather entry, so build our indirect
1451          * descriptor table, trying to merge as many entries as we can.
1452          */
1453         indirect_hdr = (void *) cmd->add_data;
1454
1455         ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1456                                    target->indirect_size, DMA_TO_DEVICE);
1457
1458         memset(&state, 0, sizeof(state));
1459         srp_map_sg(&state, target, req, scat, count);
1460
1461         /* We've mapped the request, now pull as much of the indirect
1462          * descriptor table as we can into the command buffer. If this
1463          * target is not using an external indirect table, we are
1464          * guaranteed to fit into the command, as the SCSI layer won't
1465          * give us more S/G entries than we allow.
1466          */
1467         if (state.ndesc == 1) {
1468                 /*
1469                  * Memory registration collapsed the sg-list into one entry,
1470                  * so use a direct descriptor.
1471                  */
1472                 struct srp_direct_buf *buf = (void *) cmd->add_data;
1473
1474                 *buf = req->indirect_desc[0];
1475                 goto map_complete;
1476         }
1477
1478         if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1479                                                 !target->allow_ext_sg)) {
1480                 shost_printk(KERN_ERR, target->scsi_host,
1481                              "Could not fit S/G list into SRP_CMD\n");
1482                 return -EIO;
1483         }
1484
1485         count = min(state.ndesc, target->cmd_sg_cnt);
1486         table_len = state.ndesc * sizeof (struct srp_direct_buf);
1487
1488         fmt = SRP_DATA_DESC_INDIRECT;
1489         len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf);
1490         len += count * sizeof (struct srp_direct_buf);
1491
1492         memcpy(indirect_hdr->desc_list, req->indirect_desc,
1493                count * sizeof (struct srp_direct_buf));
1494
1495         indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1496         indirect_hdr->table_desc.key = cpu_to_be32(target->rkey);
1497         indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1498         indirect_hdr->len = cpu_to_be32(state.total_len);
1499
1500         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1501                 cmd->data_out_desc_cnt = count;
1502         else
1503                 cmd->data_in_desc_cnt = count;
1504
1505         ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1506                                       DMA_TO_DEVICE);
1507
1508 map_complete:
1509         if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1510                 cmd->buf_fmt = fmt << 4;
1511         else
1512                 cmd->buf_fmt = fmt;
1513
1514         return len;
1515 }
1516
1517 /*
1518  * Return an IU and possible credit to the free pool
1519  */
1520 static void srp_put_tx_iu(struct srp_target_port *target, struct srp_iu *iu,
1521                           enum srp_iu_type iu_type)
1522 {
1523         unsigned long flags;
1524
1525         spin_lock_irqsave(&target->lock, flags);
1526         list_add(&iu->list, &target->free_tx);
1527         if (iu_type != SRP_IU_RSP)
1528                 ++target->req_lim;
1529         spin_unlock_irqrestore(&target->lock, flags);
1530 }
1531
1532 /*
1533  * Must be called with target->lock held to protect req_lim and free_tx.
1534  * If IU is not sent, it must be returned using srp_put_tx_iu().
1535  *
1536  * Note:
1537  * An upper limit for the number of allocated information units for each
1538  * request type is:
1539  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1540  *   more than Scsi_Host.can_queue requests.
1541  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1542  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1543  *   one unanswered SRP request to an initiator.
1544  */
1545 static struct srp_iu *__srp_get_tx_iu(struct srp_target_port *target,
1546                                       enum srp_iu_type iu_type)
1547 {
1548         s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1549         struct srp_iu *iu;
1550
1551         srp_send_completion(target->send_cq, target);
1552
1553         if (list_empty(&target->free_tx))
1554                 return NULL;
1555
1556         /* Initiator responses to target requests do not consume credits */
1557         if (iu_type != SRP_IU_RSP) {
1558                 if (target->req_lim <= rsv) {
1559                         ++target->zero_req_lim;
1560                         return NULL;
1561                 }
1562
1563                 --target->req_lim;
1564         }
1565
1566         iu = list_first_entry(&target->free_tx, struct srp_iu, list);
1567         list_del(&iu->list);
1568         return iu;
1569 }
1570
1571 static int srp_post_send(struct srp_target_port *target,
1572                          struct srp_iu *iu, int len)
1573 {
1574         struct ib_sge list;
1575         struct ib_send_wr wr, *bad_wr;
1576
1577         list.addr   = iu->dma;
1578         list.length = len;
1579         list.lkey   = target->lkey;
1580
1581         wr.next       = NULL;
1582         wr.wr_id      = (uintptr_t) iu;
1583         wr.sg_list    = &list;
1584         wr.num_sge    = 1;
1585         wr.opcode     = IB_WR_SEND;
1586         wr.send_flags = IB_SEND_SIGNALED;
1587
1588         return ib_post_send(target->qp, &wr, &bad_wr);
1589 }
1590
1591 static int srp_post_recv(struct srp_target_port *target, struct srp_iu *iu)
1592 {
1593         struct ib_recv_wr wr, *bad_wr;
1594         struct ib_sge list;
1595
1596         list.addr   = iu->dma;
1597         list.length = iu->size;
1598         list.lkey   = target->lkey;
1599
1600         wr.next     = NULL;
1601         wr.wr_id    = (uintptr_t) iu;
1602         wr.sg_list  = &list;
1603         wr.num_sge  = 1;
1604
1605         return ib_post_recv(target->qp, &wr, &bad_wr);
1606 }
1607
1608 static void srp_process_rsp(struct srp_target_port *target, struct srp_rsp *rsp)
1609 {
1610         struct srp_request *req;
1611         struct scsi_cmnd *scmnd;
1612         unsigned long flags;
1613
1614         if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1615                 spin_lock_irqsave(&target->lock, flags);
1616                 target->req_lim += be32_to_cpu(rsp->req_lim_delta);
1617                 spin_unlock_irqrestore(&target->lock, flags);
1618
1619                 target->tsk_mgmt_status = -1;
1620                 if (be32_to_cpu(rsp->resp_data_len) >= 4)
1621                         target->tsk_mgmt_status = rsp->data[3];
1622                 complete(&target->tsk_mgmt_done);
1623         } else {
1624                 req = &target->req_ring[rsp->tag];
1625                 scmnd = srp_claim_req(target, req, NULL, NULL);
1626                 if (!scmnd) {
1627                         shost_printk(KERN_ERR, target->scsi_host,
1628                                      "Null scmnd for RSP w/tag %016llx\n",
1629                                      (unsigned long long) rsp->tag);
1630
1631                         spin_lock_irqsave(&target->lock, flags);
1632                         target->req_lim += be32_to_cpu(rsp->req_lim_delta);
1633                         spin_unlock_irqrestore(&target->lock, flags);
1634
1635                         return;
1636                 }
1637                 scmnd->result = rsp->status;
1638
1639                 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1640                         memcpy(scmnd->sense_buffer, rsp->data +
1641                                be32_to_cpu(rsp->resp_data_len),
1642                                min_t(int, be32_to_cpu(rsp->sense_data_len),
1643                                      SCSI_SENSE_BUFFERSIZE));
1644                 }
1645
1646                 if (rsp->flags & (SRP_RSP_FLAG_DOOVER | SRP_RSP_FLAG_DOUNDER))
1647                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1648                 else if (rsp->flags & (SRP_RSP_FLAG_DIOVER | SRP_RSP_FLAG_DIUNDER))
1649                         scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1650
1651                 srp_free_req(target, req, scmnd,
1652                              be32_to_cpu(rsp->req_lim_delta));
1653
1654                 scmnd->host_scribble = NULL;
1655                 scmnd->scsi_done(scmnd);
1656         }
1657 }
1658
1659 static int srp_response_common(struct srp_target_port *target, s32 req_delta,
1660                                void *rsp, int len)
1661 {
1662         struct ib_device *dev = target->srp_host->srp_dev->dev;
1663         unsigned long flags;
1664         struct srp_iu *iu;
1665         int err;
1666
1667         spin_lock_irqsave(&target->lock, flags);
1668         target->req_lim += req_delta;
1669         iu = __srp_get_tx_iu(target, SRP_IU_RSP);
1670         spin_unlock_irqrestore(&target->lock, flags);
1671
1672         if (!iu) {
1673                 shost_printk(KERN_ERR, target->scsi_host, PFX
1674                              "no IU available to send response\n");
1675                 return 1;
1676         }
1677
1678         ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
1679         memcpy(iu->buf, rsp, len);
1680         ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
1681
1682         err = srp_post_send(target, iu, len);
1683         if (err) {
1684                 shost_printk(KERN_ERR, target->scsi_host, PFX
1685                              "unable to post response: %d\n", err);
1686                 srp_put_tx_iu(target, iu, SRP_IU_RSP);
1687         }
1688
1689         return err;
1690 }
1691
1692 static void srp_process_cred_req(struct srp_target_port *target,
1693                                  struct srp_cred_req *req)
1694 {
1695         struct srp_cred_rsp rsp = {
1696                 .opcode = SRP_CRED_RSP,
1697                 .tag = req->tag,
1698         };
1699         s32 delta = be32_to_cpu(req->req_lim_delta);
1700
1701         if (srp_response_common(target, delta, &rsp, sizeof rsp))
1702                 shost_printk(KERN_ERR, target->scsi_host, PFX
1703                              "problems processing SRP_CRED_REQ\n");
1704 }
1705
1706 static void srp_process_aer_req(struct srp_target_port *target,
1707                                 struct srp_aer_req *req)
1708 {
1709         struct srp_aer_rsp rsp = {
1710                 .opcode = SRP_AER_RSP,
1711                 .tag = req->tag,
1712         };
1713         s32 delta = be32_to_cpu(req->req_lim_delta);
1714
1715         shost_printk(KERN_ERR, target->scsi_host, PFX
1716                      "ignoring AER for LUN %llu\n", be64_to_cpu(req->lun));
1717
1718         if (srp_response_common(target, delta, &rsp, sizeof rsp))
1719                 shost_printk(KERN_ERR, target->scsi_host, PFX
1720                              "problems processing SRP_AER_REQ\n");
1721 }
1722
1723 static void srp_handle_recv(struct srp_target_port *target, struct ib_wc *wc)
1724 {
1725         struct ib_device *dev = target->srp_host->srp_dev->dev;
1726         struct srp_iu *iu = (struct srp_iu *) (uintptr_t) wc->wr_id;
1727         int res;
1728         u8 opcode;
1729
1730         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_ti_iu_len,
1731                                    DMA_FROM_DEVICE);
1732
1733         opcode = *(u8 *) iu->buf;
1734
1735         if (0) {
1736                 shost_printk(KERN_ERR, target->scsi_host,
1737                              PFX "recv completion, opcode 0x%02x\n", opcode);
1738                 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
1739                                iu->buf, wc->byte_len, true);
1740         }
1741
1742         switch (opcode) {
1743         case SRP_RSP:
1744                 srp_process_rsp(target, iu->buf);
1745                 break;
1746
1747         case SRP_CRED_REQ:
1748                 srp_process_cred_req(target, iu->buf);
1749                 break;
1750
1751         case SRP_AER_REQ:
1752                 srp_process_aer_req(target, iu->buf);
1753                 break;
1754
1755         case SRP_T_LOGOUT:
1756                 /* XXX Handle target logout */
1757                 shost_printk(KERN_WARNING, target->scsi_host,
1758                              PFX "Got target logout request\n");
1759                 break;
1760
1761         default:
1762                 shost_printk(KERN_WARNING, target->scsi_host,
1763                              PFX "Unhandled SRP opcode 0x%02x\n", opcode);
1764                 break;
1765         }
1766
1767         ib_dma_sync_single_for_device(dev, iu->dma, target->max_ti_iu_len,
1768                                       DMA_FROM_DEVICE);
1769
1770         res = srp_post_recv(target, iu);
1771         if (res != 0)
1772                 shost_printk(KERN_ERR, target->scsi_host,
1773                              PFX "Recv failed with error code %d\n", res);
1774 }
1775
1776 /**
1777  * srp_tl_err_work() - handle a transport layer error
1778  * @work: Work structure embedded in an SRP target port.
1779  *
1780  * Note: This function may get invoked before the rport has been created,
1781  * hence the target->rport test.
1782  */
1783 static void srp_tl_err_work(struct work_struct *work)
1784 {
1785         struct srp_target_port *target;
1786
1787         target = container_of(work, struct srp_target_port, tl_err_work);
1788         if (target->rport)
1789                 srp_start_tl_fail_timers(target->rport);
1790 }
1791
1792 static void srp_handle_qp_err(u64 wr_id, enum ib_wc_status wc_status,
1793                               bool send_err, struct srp_target_port *target)
1794 {
1795         if (target->connected && !target->qp_in_error) {
1796                 if (wr_id & LOCAL_INV_WR_ID_MASK) {
1797                         shost_printk(KERN_ERR, target->scsi_host, PFX
1798                                      "LOCAL_INV failed with status %d\n",
1799                                      wc_status);
1800                 } else if (wr_id & FAST_REG_WR_ID_MASK) {
1801                         shost_printk(KERN_ERR, target->scsi_host, PFX
1802                                      "FAST_REG_MR failed status %d\n",
1803                                      wc_status);
1804                 } else {
1805                         shost_printk(KERN_ERR, target->scsi_host,
1806                                      PFX "failed %s status %d for iu %p\n",
1807                                      send_err ? "send" : "receive",
1808                                      wc_status, (void *)(uintptr_t)wr_id);
1809                 }
1810                 queue_work(system_long_wq, &target->tl_err_work);
1811         }
1812         target->qp_in_error = true;
1813 }
1814
1815 static void srp_recv_completion(struct ib_cq *cq, void *target_ptr)
1816 {
1817         struct srp_target_port *target = target_ptr;
1818         struct ib_wc wc;
1819
1820         ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1821         while (ib_poll_cq(cq, 1, &wc) > 0) {
1822                 if (likely(wc.status == IB_WC_SUCCESS)) {
1823                         srp_handle_recv(target, &wc);
1824                 } else {
1825                         srp_handle_qp_err(wc.wr_id, wc.status, false, target);
1826                 }
1827         }
1828 }
1829
1830 static void srp_send_completion(struct ib_cq *cq, void *target_ptr)
1831 {
1832         struct srp_target_port *target = target_ptr;
1833         struct ib_wc wc;
1834         struct srp_iu *iu;
1835
1836         while (ib_poll_cq(cq, 1, &wc) > 0) {
1837                 if (likely(wc.status == IB_WC_SUCCESS)) {
1838                         iu = (struct srp_iu *) (uintptr_t) wc.wr_id;
1839                         list_add(&iu->list, &target->free_tx);
1840                 } else {
1841                         srp_handle_qp_err(wc.wr_id, wc.status, true, target);
1842                 }
1843         }
1844 }
1845
1846 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
1847 {
1848         struct srp_target_port *target = host_to_target(shost);
1849         struct srp_rport *rport = target->rport;
1850         struct srp_request *req;
1851         struct srp_iu *iu;
1852         struct srp_cmd *cmd;
1853         struct ib_device *dev;
1854         unsigned long flags;
1855         int len, ret;
1856         const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler;
1857
1858         /*
1859          * The SCSI EH thread is the only context from which srp_queuecommand()
1860          * can get invoked for blocked devices (SDEV_BLOCK /
1861          * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by
1862          * locking the rport mutex if invoked from inside the SCSI EH.
1863          */
1864         if (in_scsi_eh)
1865                 mutex_lock(&rport->mutex);
1866
1867         scmnd->result = srp_chkready(target->rport);
1868         if (unlikely(scmnd->result))
1869                 goto err;
1870
1871         spin_lock_irqsave(&target->lock, flags);
1872         iu = __srp_get_tx_iu(target, SRP_IU_CMD);
1873         if (!iu)
1874                 goto err_unlock;
1875
1876         req = list_first_entry(&target->free_reqs, struct srp_request, list);
1877         list_del(&req->list);
1878         spin_unlock_irqrestore(&target->lock, flags);
1879
1880         dev = target->srp_host->srp_dev->dev;
1881         ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
1882                                    DMA_TO_DEVICE);
1883
1884         scmnd->host_scribble = (void *) req;
1885
1886         cmd = iu->buf;
1887         memset(cmd, 0, sizeof *cmd);
1888
1889         cmd->opcode = SRP_CMD;
1890         cmd->lun    = cpu_to_be64((u64) scmnd->device->lun << 48);
1891         cmd->tag    = req->index;
1892         memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
1893
1894         req->scmnd    = scmnd;
1895         req->cmd      = iu;
1896
1897         len = srp_map_data(scmnd, target, req);
1898         if (len < 0) {
1899                 shost_printk(KERN_ERR, target->scsi_host,
1900                              PFX "Failed to map data (%d)\n", len);
1901                 /*
1902                  * If we ran out of memory descriptors (-ENOMEM) because an
1903                  * application is queuing many requests with more than
1904                  * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
1905                  * to reduce queue depth temporarily.
1906                  */
1907                 scmnd->result = len == -ENOMEM ?
1908                         DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16;
1909                 goto err_iu;
1910         }
1911
1912         ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
1913                                       DMA_TO_DEVICE);
1914
1915         if (srp_post_send(target, iu, len)) {
1916                 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
1917                 goto err_unmap;
1918         }
1919
1920         ret = 0;
1921
1922 unlock_rport:
1923         if (in_scsi_eh)
1924                 mutex_unlock(&rport->mutex);
1925
1926         return ret;
1927
1928 err_unmap:
1929         srp_unmap_data(scmnd, target, req);
1930
1931 err_iu:
1932         srp_put_tx_iu(target, iu, SRP_IU_CMD);
1933
1934         /*
1935          * Avoid that the loops that iterate over the request ring can
1936          * encounter a dangling SCSI command pointer.
1937          */
1938         req->scmnd = NULL;
1939
1940         spin_lock_irqsave(&target->lock, flags);
1941         list_add(&req->list, &target->free_reqs);
1942
1943 err_unlock:
1944         spin_unlock_irqrestore(&target->lock, flags);
1945
1946 err:
1947         if (scmnd->result) {
1948                 scmnd->scsi_done(scmnd);
1949                 ret = 0;
1950         } else {
1951                 ret = SCSI_MLQUEUE_HOST_BUSY;
1952         }
1953
1954         goto unlock_rport;
1955 }
1956
1957 /*
1958  * Note: the resources allocated in this function are freed in
1959  * srp_free_target_ib().
1960  */
1961 static int srp_alloc_iu_bufs(struct srp_target_port *target)
1962 {
1963         int i;
1964
1965         target->rx_ring = kzalloc(target->queue_size * sizeof(*target->rx_ring),
1966                                   GFP_KERNEL);
1967         if (!target->rx_ring)
1968                 goto err_no_ring;
1969         target->tx_ring = kzalloc(target->queue_size * sizeof(*target->tx_ring),
1970                                   GFP_KERNEL);
1971         if (!target->tx_ring)
1972                 goto err_no_ring;
1973
1974         for (i = 0; i < target->queue_size; ++i) {
1975                 target->rx_ring[i] = srp_alloc_iu(target->srp_host,
1976                                                   target->max_ti_iu_len,
1977                                                   GFP_KERNEL, DMA_FROM_DEVICE);
1978                 if (!target->rx_ring[i])
1979                         goto err;
1980         }
1981
1982         for (i = 0; i < target->queue_size; ++i) {
1983                 target->tx_ring[i] = srp_alloc_iu(target->srp_host,
1984                                                   target->max_iu_len,
1985                                                   GFP_KERNEL, DMA_TO_DEVICE);
1986                 if (!target->tx_ring[i])
1987                         goto err;
1988
1989                 list_add(&target->tx_ring[i]->list, &target->free_tx);
1990         }
1991
1992         return 0;
1993
1994 err:
1995         for (i = 0; i < target->queue_size; ++i) {
1996                 srp_free_iu(target->srp_host, target->rx_ring[i]);
1997                 srp_free_iu(target->srp_host, target->tx_ring[i]);
1998         }
1999
2000
2001 err_no_ring:
2002         kfree(target->tx_ring);
2003         target->tx_ring = NULL;
2004         kfree(target->rx_ring);
2005         target->rx_ring = NULL;
2006
2007         return -ENOMEM;
2008 }
2009
2010 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2011 {
2012         uint64_t T_tr_ns, max_compl_time_ms;
2013         uint32_t rq_tmo_jiffies;
2014
2015         /*
2016          * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2017          * table 91), both the QP timeout and the retry count have to be set
2018          * for RC QP's during the RTR to RTS transition.
2019          */
2020         WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2021                      (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2022
2023         /*
2024          * Set target->rq_tmo_jiffies to one second more than the largest time
2025          * it can take before an error completion is generated. See also
2026          * C9-140..142 in the IBTA spec for more information about how to
2027          * convert the QP Local ACK Timeout value to nanoseconds.
2028          */
2029         T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2030         max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2031         do_div(max_compl_time_ms, NSEC_PER_MSEC);
2032         rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2033
2034         return rq_tmo_jiffies;
2035 }
2036
2037 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2038                                struct srp_login_rsp *lrsp,
2039                                struct srp_target_port *target)
2040 {
2041         struct ib_qp_attr *qp_attr = NULL;
2042         int attr_mask = 0;
2043         int ret;
2044         int i;
2045
2046         if (lrsp->opcode == SRP_LOGIN_RSP) {
2047                 target->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2048                 target->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2049
2050                 /*
2051                  * Reserve credits for task management so we don't
2052                  * bounce requests back to the SCSI mid-layer.
2053                  */
2054                 target->scsi_host->can_queue
2055                         = min(target->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2056                               target->scsi_host->can_queue);
2057                 target->scsi_host->cmd_per_lun
2058                         = min_t(int, target->scsi_host->can_queue,
2059                                 target->scsi_host->cmd_per_lun);
2060         } else {
2061                 shost_printk(KERN_WARNING, target->scsi_host,
2062                              PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2063                 ret = -ECONNRESET;
2064                 goto error;
2065         }
2066
2067         if (!target->rx_ring) {
2068                 ret = srp_alloc_iu_bufs(target);
2069                 if (ret)
2070                         goto error;
2071         }
2072
2073         ret = -ENOMEM;
2074         qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
2075         if (!qp_attr)
2076                 goto error;
2077
2078         qp_attr->qp_state = IB_QPS_RTR;
2079         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2080         if (ret)
2081                 goto error_free;
2082
2083         ret = ib_modify_qp(target->qp, qp_attr, attr_mask);
2084         if (ret)
2085                 goto error_free;
2086
2087         for (i = 0; i < target->queue_size; i++) {
2088                 struct srp_iu *iu = target->rx_ring[i];
2089                 ret = srp_post_recv(target, iu);
2090                 if (ret)
2091                         goto error_free;
2092         }
2093
2094         qp_attr->qp_state = IB_QPS_RTS;
2095         ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2096         if (ret)
2097                 goto error_free;
2098
2099         target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2100
2101         ret = ib_modify_qp(target->qp, qp_attr, attr_mask);
2102         if (ret)
2103                 goto error_free;
2104
2105         ret = ib_send_cm_rtu(cm_id, NULL, 0);
2106
2107 error_free:
2108         kfree(qp_attr);
2109
2110 error:
2111         target->status = ret;
2112 }
2113
2114 static void srp_cm_rej_handler(struct ib_cm_id *cm_id,
2115                                struct ib_cm_event *event,
2116                                struct srp_target_port *target)
2117 {
2118         struct Scsi_Host *shost = target->scsi_host;
2119         struct ib_class_port_info *cpi;
2120         int opcode;
2121
2122         switch (event->param.rej_rcvd.reason) {
2123         case IB_CM_REJ_PORT_CM_REDIRECT:
2124                 cpi = event->param.rej_rcvd.ari;
2125                 target->path.dlid = cpi->redirect_lid;
2126                 target->path.pkey = cpi->redirect_pkey;
2127                 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2128                 memcpy(target->path.dgid.raw, cpi->redirect_gid, 16);
2129
2130                 target->status = target->path.dlid ?
2131                         SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2132                 break;
2133
2134         case IB_CM_REJ_PORT_REDIRECT:
2135                 if (srp_target_is_topspin(target)) {
2136                         /*
2137                          * Topspin/Cisco SRP gateways incorrectly send
2138                          * reject reason code 25 when they mean 24
2139                          * (port redirect).
2140                          */
2141                         memcpy(target->path.dgid.raw,
2142                                event->param.rej_rcvd.ari, 16);
2143
2144                         shost_printk(KERN_DEBUG, shost,
2145                                      PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2146                                      (unsigned long long) be64_to_cpu(target->path.dgid.global.subnet_prefix),
2147                                      (unsigned long long) be64_to_cpu(target->path.dgid.global.interface_id));
2148
2149                         target->status = SRP_PORT_REDIRECT;
2150                 } else {
2151                         shost_printk(KERN_WARNING, shost,
2152                                      "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2153                         target->status = -ECONNRESET;
2154                 }
2155                 break;
2156
2157         case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2158                 shost_printk(KERN_WARNING, shost,
2159                             "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2160                 target->status = -ECONNRESET;
2161                 break;
2162
2163         case IB_CM_REJ_CONSUMER_DEFINED:
2164                 opcode = *(u8 *) event->private_data;
2165                 if (opcode == SRP_LOGIN_REJ) {
2166                         struct srp_login_rej *rej = event->private_data;
2167                         u32 reason = be32_to_cpu(rej->reason);
2168
2169                         if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2170                                 shost_printk(KERN_WARNING, shost,
2171                                              PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2172                         else
2173                                 shost_printk(KERN_WARNING, shost, PFX
2174                                              "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2175                                              target->path.sgid.raw,
2176                                              target->orig_dgid, reason);
2177                 } else
2178                         shost_printk(KERN_WARNING, shost,
2179                                      "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2180                                      " opcode 0x%02x\n", opcode);
2181                 target->status = -ECONNRESET;
2182                 break;
2183
2184         case IB_CM_REJ_STALE_CONN:
2185                 shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2186                 target->status = SRP_STALE_CONN;
2187                 break;
2188
2189         default:
2190                 shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2191                              event->param.rej_rcvd.reason);
2192                 target->status = -ECONNRESET;
2193         }
2194 }
2195
2196 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2197 {
2198         struct srp_target_port *target = cm_id->context;
2199         int comp = 0;
2200
2201         switch (event->event) {
2202         case IB_CM_REQ_ERROR:
2203                 shost_printk(KERN_DEBUG, target->scsi_host,
2204                              PFX "Sending CM REQ failed\n");
2205                 comp = 1;
2206                 target->status = -ECONNRESET;
2207                 break;
2208
2209         case IB_CM_REP_RECEIVED:
2210                 comp = 1;
2211                 srp_cm_rep_handler(cm_id, event->private_data, target);
2212                 break;
2213
2214         case IB_CM_REJ_RECEIVED:
2215                 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2216                 comp = 1;
2217
2218                 srp_cm_rej_handler(cm_id, event, target);
2219                 break;
2220
2221         case IB_CM_DREQ_RECEIVED:
2222                 shost_printk(KERN_WARNING, target->scsi_host,
2223                              PFX "DREQ received - connection closed\n");
2224                 srp_change_conn_state(target, false);
2225                 if (ib_send_cm_drep(cm_id, NULL, 0))
2226                         shost_printk(KERN_ERR, target->scsi_host,
2227                                      PFX "Sending CM DREP failed\n");
2228                 queue_work(system_long_wq, &target->tl_err_work);
2229                 break;
2230
2231         case IB_CM_TIMEWAIT_EXIT:
2232                 shost_printk(KERN_ERR, target->scsi_host,
2233                              PFX "connection closed\n");
2234                 comp = 1;
2235
2236                 target->status = 0;
2237                 break;
2238
2239         case IB_CM_MRA_RECEIVED:
2240         case IB_CM_DREQ_ERROR:
2241         case IB_CM_DREP_RECEIVED:
2242                 break;
2243
2244         default:
2245                 shost_printk(KERN_WARNING, target->scsi_host,
2246                              PFX "Unhandled CM event %d\n", event->event);
2247                 break;
2248         }
2249
2250         if (comp)
2251                 complete(&target->done);
2252
2253         return 0;
2254 }
2255
2256 /**
2257  * srp_change_queue_type - changing device queue tag type
2258  * @sdev: scsi device struct
2259  * @tag_type: requested tag type
2260  *
2261  * Returns queue tag type.
2262  */
2263 static int
2264 srp_change_queue_type(struct scsi_device *sdev, int tag_type)
2265 {
2266         if (sdev->tagged_supported) {
2267                 scsi_set_tag_type(sdev, tag_type);
2268                 if (tag_type)
2269                         scsi_activate_tcq(sdev, sdev->queue_depth);
2270                 else
2271                         scsi_deactivate_tcq(sdev, sdev->queue_depth);
2272         } else
2273                 tag_type = 0;
2274
2275         return tag_type;
2276 }
2277
2278 /**
2279  * srp_change_queue_depth - setting device queue depth
2280  * @sdev: scsi device struct
2281  * @qdepth: requested queue depth
2282  * @reason: SCSI_QDEPTH_DEFAULT/SCSI_QDEPTH_QFULL/SCSI_QDEPTH_RAMP_UP
2283  * (see include/scsi/scsi_host.h for definition)
2284  *
2285  * Returns queue depth.
2286  */
2287 static int
2288 srp_change_queue_depth(struct scsi_device *sdev, int qdepth, int reason)
2289 {
2290         struct Scsi_Host *shost = sdev->host;
2291         int max_depth;
2292         if (reason == SCSI_QDEPTH_DEFAULT || reason == SCSI_QDEPTH_RAMP_UP) {
2293                 max_depth = shost->can_queue;
2294                 if (!sdev->tagged_supported)
2295                         max_depth = 1;
2296                 if (qdepth > max_depth)
2297                         qdepth = max_depth;
2298                 scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), qdepth);
2299         } else if (reason == SCSI_QDEPTH_QFULL)
2300                 scsi_track_queue_full(sdev, qdepth);
2301         else
2302                 return -EOPNOTSUPP;
2303
2304         return sdev->queue_depth;
2305 }
2306
2307 static int srp_send_tsk_mgmt(struct srp_target_port *target,
2308                              u64 req_tag, unsigned int lun, u8 func)
2309 {
2310         struct srp_rport *rport = target->rport;
2311         struct ib_device *dev = target->srp_host->srp_dev->dev;
2312         struct srp_iu *iu;
2313         struct srp_tsk_mgmt *tsk_mgmt;
2314
2315         if (!target->connected || target->qp_in_error)
2316                 return -1;
2317
2318         init_completion(&target->tsk_mgmt_done);
2319
2320         /*
2321          * Lock the rport mutex to avoid that srp_create_target_ib() is
2322          * invoked while a task management function is being sent.
2323          */
2324         mutex_lock(&rport->mutex);
2325         spin_lock_irq(&target->lock);
2326         iu = __srp_get_tx_iu(target, SRP_IU_TSK_MGMT);
2327         spin_unlock_irq(&target->lock);
2328
2329         if (!iu) {
2330                 mutex_unlock(&rport->mutex);
2331
2332                 return -1;
2333         }
2334
2335         ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2336                                    DMA_TO_DEVICE);
2337         tsk_mgmt = iu->buf;
2338         memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2339
2340         tsk_mgmt->opcode        = SRP_TSK_MGMT;
2341         tsk_mgmt->lun           = cpu_to_be64((u64) lun << 48);
2342         tsk_mgmt->tag           = req_tag | SRP_TAG_TSK_MGMT;
2343         tsk_mgmt->tsk_mgmt_func = func;
2344         tsk_mgmt->task_tag      = req_tag;
2345
2346         ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2347                                       DMA_TO_DEVICE);
2348         if (srp_post_send(target, iu, sizeof *tsk_mgmt)) {
2349                 srp_put_tx_iu(target, iu, SRP_IU_TSK_MGMT);
2350                 mutex_unlock(&rport->mutex);
2351
2352                 return -1;
2353         }
2354         mutex_unlock(&rport->mutex);
2355
2356         if (!wait_for_completion_timeout(&target->tsk_mgmt_done,
2357                                          msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)))
2358                 return -1;
2359
2360         return 0;
2361 }
2362
2363 static int srp_abort(struct scsi_cmnd *scmnd)
2364 {
2365         struct srp_target_port *target = host_to_target(scmnd->device->host);
2366         struct srp_request *req = (struct srp_request *) scmnd->host_scribble;
2367         int ret;
2368
2369         shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2370
2371         if (!req || !srp_claim_req(target, req, NULL, scmnd))
2372                 return SUCCESS;
2373         if (srp_send_tsk_mgmt(target, req->index, scmnd->device->lun,
2374                               SRP_TSK_ABORT_TASK) == 0)
2375                 ret = SUCCESS;
2376         else if (target->rport->state == SRP_RPORT_LOST)
2377                 ret = FAST_IO_FAIL;
2378         else
2379                 ret = FAILED;
2380         srp_free_req(target, req, scmnd, 0);
2381         scmnd->result = DID_ABORT << 16;
2382         scmnd->scsi_done(scmnd);
2383
2384         return ret;
2385 }
2386
2387 static int srp_reset_device(struct scsi_cmnd *scmnd)
2388 {
2389         struct srp_target_port *target = host_to_target(scmnd->device->host);
2390         int i;
2391
2392         shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2393
2394         if (srp_send_tsk_mgmt(target, SRP_TAG_NO_REQ, scmnd->device->lun,
2395                               SRP_TSK_LUN_RESET))
2396                 return FAILED;
2397         if (target->tsk_mgmt_status)
2398                 return FAILED;
2399
2400         for (i = 0; i < target->req_ring_size; ++i) {
2401                 struct srp_request *req = &target->req_ring[i];
2402                 srp_finish_req(target, req, scmnd->device, DID_RESET << 16);
2403         }
2404
2405         return SUCCESS;
2406 }
2407
2408 static int srp_reset_host(struct scsi_cmnd *scmnd)
2409 {
2410         struct srp_target_port *target = host_to_target(scmnd->device->host);
2411
2412         shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2413
2414         return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2415 }
2416
2417 static int srp_slave_configure(struct scsi_device *sdev)
2418 {
2419         struct Scsi_Host *shost = sdev->host;
2420         struct srp_target_port *target = host_to_target(shost);
2421         struct request_queue *q = sdev->request_queue;
2422         unsigned long timeout;
2423
2424         if (sdev->type == TYPE_DISK) {
2425                 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2426                 blk_queue_rq_timeout(q, timeout);
2427         }
2428
2429         return 0;
2430 }
2431
2432 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr,
2433                            char *buf)
2434 {
2435         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2436
2437         return sprintf(buf, "0x%016llx\n",
2438                        (unsigned long long) be64_to_cpu(target->id_ext));
2439 }
2440
2441 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr,
2442                              char *buf)
2443 {
2444         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2445
2446         return sprintf(buf, "0x%016llx\n",
2447                        (unsigned long long) be64_to_cpu(target->ioc_guid));
2448 }
2449
2450 static ssize_t show_service_id(struct device *dev,
2451                                struct device_attribute *attr, char *buf)
2452 {
2453         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2454
2455         return sprintf(buf, "0x%016llx\n",
2456                        (unsigned long long) be64_to_cpu(target->service_id));
2457 }
2458
2459 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr,
2460                          char *buf)
2461 {
2462         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2463
2464         return sprintf(buf, "0x%04x\n", be16_to_cpu(target->path.pkey));
2465 }
2466
2467 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr,
2468                          char *buf)
2469 {
2470         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2471
2472         return sprintf(buf, "%pI6\n", target->path.sgid.raw);
2473 }
2474
2475 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr,
2476                          char *buf)
2477 {
2478         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2479
2480         return sprintf(buf, "%pI6\n", target->path.dgid.raw);
2481 }
2482
2483 static ssize_t show_orig_dgid(struct device *dev,
2484                               struct device_attribute *attr, char *buf)
2485 {
2486         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2487
2488         return sprintf(buf, "%pI6\n", target->orig_dgid);
2489 }
2490
2491 static ssize_t show_req_lim(struct device *dev,
2492                             struct device_attribute *attr, char *buf)
2493 {
2494         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2495
2496         return sprintf(buf, "%d\n", target->req_lim);
2497 }
2498
2499 static ssize_t show_zero_req_lim(struct device *dev,
2500                                  struct device_attribute *attr, char *buf)
2501 {
2502         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2503
2504         return sprintf(buf, "%d\n", target->zero_req_lim);
2505 }
2506
2507 static ssize_t show_local_ib_port(struct device *dev,
2508                                   struct device_attribute *attr, char *buf)
2509 {
2510         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2511
2512         return sprintf(buf, "%d\n", target->srp_host->port);
2513 }
2514
2515 static ssize_t show_local_ib_device(struct device *dev,
2516                                     struct device_attribute *attr, char *buf)
2517 {
2518         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2519
2520         return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
2521 }
2522
2523 static ssize_t show_comp_vector(struct device *dev,
2524                                 struct device_attribute *attr, char *buf)
2525 {
2526         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2527
2528         return sprintf(buf, "%d\n", target->comp_vector);
2529 }
2530
2531 static ssize_t show_tl_retry_count(struct device *dev,
2532                                    struct device_attribute *attr, char *buf)
2533 {
2534         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2535
2536         return sprintf(buf, "%d\n", target->tl_retry_count);
2537 }
2538
2539 static ssize_t show_cmd_sg_entries(struct device *dev,
2540                                    struct device_attribute *attr, char *buf)
2541 {
2542         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2543
2544         return sprintf(buf, "%u\n", target->cmd_sg_cnt);
2545 }
2546
2547 static ssize_t show_allow_ext_sg(struct device *dev,
2548                                  struct device_attribute *attr, char *buf)
2549 {
2550         struct srp_target_port *target = host_to_target(class_to_shost(dev));
2551
2552         return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
2553 }
2554
2555 static DEVICE_ATTR(id_ext,          S_IRUGO, show_id_ext,          NULL);
2556 static DEVICE_ATTR(ioc_guid,        S_IRUGO, show_ioc_guid,        NULL);
2557 static DEVICE_ATTR(service_id,      S_IRUGO, show_service_id,      NULL);
2558 static DEVICE_ATTR(pkey,            S_IRUGO, show_pkey,            NULL);
2559 static DEVICE_ATTR(sgid,            S_IRUGO, show_sgid,            NULL);
2560 static DEVICE_ATTR(dgid,            S_IRUGO, show_dgid,            NULL);
2561 static DEVICE_ATTR(orig_dgid,       S_IRUGO, show_orig_dgid,       NULL);
2562 static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
2563 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,    NULL);
2564 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
2565 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
2566 static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
2567 static DEVICE_ATTR(tl_retry_count,  S_IRUGO, show_tl_retry_count,  NULL);
2568 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
2569 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
2570
2571 static struct device_attribute *srp_host_attrs[] = {
2572         &dev_attr_id_ext,
2573         &dev_attr_ioc_guid,
2574         &dev_attr_service_id,
2575         &dev_attr_pkey,
2576         &dev_attr_sgid,
2577         &dev_attr_dgid,
2578         &dev_attr_orig_dgid,
2579         &dev_attr_req_lim,
2580         &dev_attr_zero_req_lim,
2581         &dev_attr_local_ib_port,
2582         &dev_attr_local_ib_device,
2583         &dev_attr_comp_vector,
2584         &dev_attr_tl_retry_count,
2585         &dev_attr_cmd_sg_entries,
2586         &dev_attr_allow_ext_sg,
2587         NULL
2588 };
2589
2590 static struct scsi_host_template srp_template = {
2591         .module                         = THIS_MODULE,
2592         .name                           = "InfiniBand SRP initiator",
2593         .proc_name                      = DRV_NAME,
2594         .slave_configure                = srp_slave_configure,
2595         .info                           = srp_target_info,
2596         .queuecommand                   = srp_queuecommand,
2597         .change_queue_depth             = srp_change_queue_depth,
2598         .change_queue_type              = srp_change_queue_type,
2599         .eh_abort_handler               = srp_abort,
2600         .eh_device_reset_handler        = srp_reset_device,
2601         .eh_host_reset_handler          = srp_reset_host,
2602         .skip_settle_delay              = true,
2603         .sg_tablesize                   = SRP_DEF_SG_TABLESIZE,
2604         .can_queue                      = SRP_DEFAULT_CMD_SQ_SIZE,
2605         .this_id                        = -1,
2606         .cmd_per_lun                    = SRP_DEFAULT_CMD_SQ_SIZE,
2607         .use_clustering                 = ENABLE_CLUSTERING,
2608         .shost_attrs                    = srp_host_attrs
2609 };
2610
2611 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
2612 {
2613         struct srp_rport_identifiers ids;
2614         struct srp_rport *rport;
2615
2616         sprintf(target->target_name, "SRP.T10:%016llX",
2617                  (unsigned long long) be64_to_cpu(target->id_ext));
2618
2619         if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device))
2620                 return -ENODEV;
2621
2622         memcpy(ids.port_id, &target->id_ext, 8);
2623         memcpy(ids.port_id + 8, &target->ioc_guid, 8);
2624         ids.roles = SRP_RPORT_ROLE_TARGET;
2625         rport = srp_rport_add(target->scsi_host, &ids);
2626         if (IS_ERR(rport)) {
2627                 scsi_remove_host(target->scsi_host);
2628                 return PTR_ERR(rport);
2629         }
2630
2631         rport->lld_data = target;
2632         target->rport = rport;
2633
2634         spin_lock(&host->target_lock);
2635         list_add_tail(&target->list, &host->target_list);
2636         spin_unlock(&host->target_lock);
2637
2638         target->state = SRP_TARGET_LIVE;
2639
2640         scsi_scan_target(&target->scsi_host->shost_gendev,
2641                          0, target->scsi_id, SCAN_WILD_CARD, 0);
2642
2643         return 0;
2644 }
2645
2646 static void srp_release_dev(struct device *dev)
2647 {
2648         struct srp_host *host =
2649                 container_of(dev, struct srp_host, dev);
2650
2651         complete(&host->released);
2652 }
2653
2654 static struct class srp_class = {
2655         .name    = "infiniband_srp",
2656         .dev_release = srp_release_dev
2657 };
2658
2659 /**
2660  * srp_conn_unique() - check whether the connection to a target is unique
2661  * @host:   SRP host.
2662  * @target: SRP target port.
2663  */
2664 static bool srp_conn_unique(struct srp_host *host,
2665                             struct srp_target_port *target)
2666 {
2667         struct srp_target_port *t;
2668         bool ret = false;
2669
2670         if (target->state == SRP_TARGET_REMOVED)
2671                 goto out;
2672
2673         ret = true;
2674
2675         spin_lock(&host->target_lock);
2676         list_for_each_entry(t, &host->target_list, list) {
2677                 if (t != target &&
2678                     target->id_ext == t->id_ext &&
2679                     target->ioc_guid == t->ioc_guid &&
2680                     target->initiator_ext == t->initiator_ext) {
2681                         ret = false;
2682                         break;
2683                 }
2684         }
2685         spin_unlock(&host->target_lock);
2686
2687 out:
2688         return ret;
2689 }
2690
2691 /*
2692  * Target ports are added by writing
2693  *
2694  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
2695  *     pkey=<P_Key>,service_id=<service ID>
2696  *
2697  * to the add_target sysfs attribute.
2698  */
2699 enum {
2700         SRP_OPT_ERR             = 0,
2701         SRP_OPT_ID_EXT          = 1 << 0,
2702         SRP_OPT_IOC_GUID        = 1 << 1,
2703         SRP_OPT_DGID            = 1 << 2,
2704         SRP_OPT_PKEY            = 1 << 3,
2705         SRP_OPT_SERVICE_ID      = 1 << 4,
2706         SRP_OPT_MAX_SECT        = 1 << 5,
2707         SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
2708         SRP_OPT_IO_CLASS        = 1 << 7,
2709         SRP_OPT_INITIATOR_EXT   = 1 << 8,
2710         SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
2711         SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
2712         SRP_OPT_SG_TABLESIZE    = 1 << 11,
2713         SRP_OPT_COMP_VECTOR     = 1 << 12,
2714         SRP_OPT_TL_RETRY_COUNT  = 1 << 13,
2715         SRP_OPT_QUEUE_SIZE      = 1 << 14,
2716         SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
2717                                    SRP_OPT_IOC_GUID     |
2718                                    SRP_OPT_DGID         |
2719                                    SRP_OPT_PKEY         |
2720                                    SRP_OPT_SERVICE_ID),
2721 };
2722
2723 static const match_table_t srp_opt_tokens = {
2724         { SRP_OPT_ID_EXT,               "id_ext=%s"             },
2725         { SRP_OPT_IOC_GUID,             "ioc_guid=%s"           },
2726         { SRP_OPT_DGID,                 "dgid=%s"               },
2727         { SRP_OPT_PKEY,                 "pkey=%x"               },
2728         { SRP_OPT_SERVICE_ID,           "service_id=%s"         },
2729         { SRP_OPT_MAX_SECT,             "max_sect=%d"           },
2730         { SRP_OPT_MAX_CMD_PER_LUN,      "max_cmd_per_lun=%d"    },
2731         { SRP_OPT_IO_CLASS,             "io_class=%x"           },
2732         { SRP_OPT_INITIATOR_EXT,        "initiator_ext=%s"      },
2733         { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
2734         { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
2735         { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
2736         { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
2737         { SRP_OPT_TL_RETRY_COUNT,       "tl_retry_count=%u"     },
2738         { SRP_OPT_QUEUE_SIZE,           "queue_size=%d"         },
2739         { SRP_OPT_ERR,                  NULL                    }
2740 };
2741
2742 static int srp_parse_options(const char *buf, struct srp_target_port *target)
2743 {
2744         char *options, *sep_opt;
2745         char *p;
2746         char dgid[3];
2747         substring_t args[MAX_OPT_ARGS];
2748         int opt_mask = 0;
2749         int token;
2750         int ret = -EINVAL;
2751         int i;
2752
2753         options = kstrdup(buf, GFP_KERNEL);
2754         if (!options)
2755                 return -ENOMEM;
2756
2757         sep_opt = options;
2758         while ((p = strsep(&sep_opt, ",")) != NULL) {
2759                 if (!*p)
2760                         continue;
2761
2762                 token = match_token(p, srp_opt_tokens, args);
2763                 opt_mask |= token;
2764
2765                 switch (token) {
2766                 case SRP_OPT_ID_EXT:
2767                         p = match_strdup(args);
2768                         if (!p) {
2769                                 ret = -ENOMEM;
2770                                 goto out;
2771                         }
2772                         target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2773                         kfree(p);
2774                         break;
2775
2776                 case SRP_OPT_IOC_GUID:
2777                         p = match_strdup(args);
2778                         if (!p) {
2779                                 ret = -ENOMEM;
2780                                 goto out;
2781                         }
2782                         target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16));
2783                         kfree(p);
2784                         break;
2785
2786                 case SRP_OPT_DGID:
2787                         p = match_strdup(args);
2788                         if (!p) {
2789                                 ret = -ENOMEM;
2790                                 goto out;
2791                         }
2792                         if (strlen(p) != 32) {
2793                                 pr_warn("bad dest GID parameter '%s'\n", p);
2794                                 kfree(p);
2795                                 goto out;
2796                         }
2797
2798                         for (i = 0; i < 16; ++i) {
2799                                 strlcpy(dgid, p + i * 2, 3);
2800                                 target->path.dgid.raw[i] = simple_strtoul(dgid, NULL, 16);
2801                         }
2802                         kfree(p);
2803                         memcpy(target->orig_dgid, target->path.dgid.raw, 16);
2804                         break;
2805
2806                 case SRP_OPT_PKEY:
2807                         if (match_hex(args, &token)) {
2808                                 pr_warn("bad P_Key parameter '%s'\n", p);
2809                                 goto out;
2810                         }
2811                         target->path.pkey = cpu_to_be16(token);
2812                         break;
2813
2814                 case SRP_OPT_SERVICE_ID:
2815                         p = match_strdup(args);
2816                         if (!p) {
2817                                 ret = -ENOMEM;
2818                                 goto out;
2819                         }
2820                         target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16));
2821                         target->path.service_id = target->service_id;
2822                         kfree(p);
2823                         break;
2824
2825                 case SRP_OPT_MAX_SECT:
2826                         if (match_int(args, &token)) {
2827                                 pr_warn("bad max sect parameter '%s'\n", p);
2828                                 goto out;
2829                         }
2830                         target->scsi_host->max_sectors = token;
2831                         break;
2832
2833                 case SRP_OPT_QUEUE_SIZE:
2834                         if (match_int(args, &token) || token < 1) {
2835                                 pr_warn("bad queue_size parameter '%s'\n", p);
2836                                 goto out;
2837                         }
2838                         target->scsi_host->can_queue = token;
2839                         target->queue_size = token + SRP_RSP_SQ_SIZE +
2840                                              SRP_TSK_MGMT_SQ_SIZE;
2841                         if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
2842                                 target->scsi_host->cmd_per_lun = token;
2843                         break;
2844
2845                 case SRP_OPT_MAX_CMD_PER_LUN:
2846                         if (match_int(args, &token) || token < 1) {
2847                                 pr_warn("bad max cmd_per_lun parameter '%s'\n",
2848                                         p);
2849                                 goto out;
2850                         }
2851                         target->scsi_host->cmd_per_lun = token;
2852                         break;
2853
2854                 case SRP_OPT_IO_CLASS:
2855                         if (match_hex(args, &token)) {
2856                                 pr_warn("bad IO class parameter '%s'\n", p);
2857                                 goto out;
2858                         }
2859                         if (token != SRP_REV10_IB_IO_CLASS &&
2860                             token != SRP_REV16A_IB_IO_CLASS) {
2861                                 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
2862                                         token, SRP_REV10_IB_IO_CLASS,
2863                                         SRP_REV16A_IB_IO_CLASS);
2864                                 goto out;
2865                         }
2866                         target->io_class = token;
2867                         break;
2868
2869                 case SRP_OPT_INITIATOR_EXT:
2870                         p = match_strdup(args);
2871                         if (!p) {
2872                                 ret = -ENOMEM;
2873                                 goto out;
2874                         }
2875                         target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16));
2876                         kfree(p);
2877                         break;
2878
2879                 case SRP_OPT_CMD_SG_ENTRIES:
2880                         if (match_int(args, &token) || token < 1 || token > 255) {
2881                                 pr_warn("bad max cmd_sg_entries parameter '%s'\n",
2882                                         p);
2883                                 goto out;
2884                         }
2885                         target->cmd_sg_cnt = token;
2886                         break;
2887
2888                 case SRP_OPT_ALLOW_EXT_SG:
2889                         if (match_int(args, &token)) {
2890                                 pr_warn("bad allow_ext_sg parameter '%s'\n", p);
2891                                 goto out;
2892                         }
2893                         target->allow_ext_sg = !!token;
2894                         break;
2895
2896                 case SRP_OPT_SG_TABLESIZE:
2897                         if (match_int(args, &token) || token < 1 ||
2898                                         token > SCSI_MAX_SG_CHAIN_SEGMENTS) {
2899                                 pr_warn("bad max sg_tablesize parameter '%s'\n",
2900                                         p);
2901                                 goto out;
2902                         }
2903                         target->sg_tablesize = token;
2904                         break;
2905
2906                 case SRP_OPT_COMP_VECTOR:
2907                         if (match_int(args, &token) || token < 0) {
2908                                 pr_warn("bad comp_vector parameter '%s'\n", p);
2909                                 goto out;
2910                         }
2911                         target->comp_vector = token;
2912                         break;
2913
2914                 case SRP_OPT_TL_RETRY_COUNT:
2915                         if (match_int(args, &token) || token < 2 || token > 7) {
2916                                 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
2917                                         p);
2918                                 goto out;
2919                         }
2920                         target->tl_retry_count = token;
2921                         break;
2922
2923                 default:
2924                         pr_warn("unknown parameter or missing value '%s' in target creation request\n",
2925                                 p);
2926                         goto out;
2927                 }
2928         }
2929
2930         if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL)
2931                 ret = 0;
2932         else
2933                 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i)
2934                         if ((srp_opt_tokens[i].token & SRP_OPT_ALL) &&
2935                             !(srp_opt_tokens[i].token & opt_mask))
2936                                 pr_warn("target creation request is missing parameter '%s'\n",
2937                                         srp_opt_tokens[i].pattern);
2938
2939         if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
2940             && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
2941                 pr_warn("cmd_per_lun = %d > queue_size = %d\n",
2942                         target->scsi_host->cmd_per_lun,
2943                         target->scsi_host->can_queue);
2944
2945 out:
2946         kfree(options);
2947         return ret;
2948 }
2949
2950 static ssize_t srp_create_target(struct device *dev,
2951                                  struct device_attribute *attr,
2952                                  const char *buf, size_t count)
2953 {
2954         struct srp_host *host =
2955                 container_of(dev, struct srp_host, dev);
2956         struct Scsi_Host *target_host;
2957         struct srp_target_port *target;
2958         struct srp_device *srp_dev = host->srp_dev;
2959         struct ib_device *ibdev = srp_dev->dev;
2960         int ret;
2961
2962         target_host = scsi_host_alloc(&srp_template,
2963                                       sizeof (struct srp_target_port));
2964         if (!target_host)
2965                 return -ENOMEM;
2966
2967         target_host->transportt  = ib_srp_transport_template;
2968         target_host->max_channel = 0;
2969         target_host->max_id      = 1;
2970         target_host->max_lun     = SRP_MAX_LUN;
2971         target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
2972
2973         target = host_to_target(target_host);
2974
2975         target->io_class        = SRP_REV16A_IB_IO_CLASS;
2976         target->scsi_host       = target_host;
2977         target->srp_host        = host;
2978         target->lkey            = host->srp_dev->mr->lkey;
2979         target->rkey            = host->srp_dev->mr->rkey;
2980         target->cmd_sg_cnt      = cmd_sg_entries;
2981         target->sg_tablesize    = indirect_sg_entries ? : cmd_sg_entries;
2982         target->allow_ext_sg    = allow_ext_sg;
2983         target->tl_retry_count  = 7;
2984         target->queue_size      = SRP_DEFAULT_QUEUE_SIZE;
2985
2986         mutex_lock(&host->add_target_mutex);
2987
2988         ret = srp_parse_options(buf, target);
2989         if (ret)
2990                 goto err;
2991
2992         target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE;
2993
2994         if (!srp_conn_unique(target->srp_host, target)) {
2995                 shost_printk(KERN_INFO, target->scsi_host,
2996                              PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
2997                              be64_to_cpu(target->id_ext),
2998                              be64_to_cpu(target->ioc_guid),
2999                              be64_to_cpu(target->initiator_ext));
3000                 ret = -EEXIST;
3001                 goto err;
3002         }
3003
3004         if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg &&
3005             target->cmd_sg_cnt < target->sg_tablesize) {
3006                 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3007                 target->sg_tablesize = target->cmd_sg_cnt;
3008         }
3009
3010         target_host->sg_tablesize = target->sg_tablesize;
3011         target->indirect_size = target->sg_tablesize *
3012                                 sizeof (struct srp_direct_buf);
3013         target->max_iu_len = sizeof (struct srp_cmd) +
3014                              sizeof (struct srp_indirect_buf) +
3015                              target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
3016
3017         INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3018         INIT_WORK(&target->remove_work, srp_remove_work);
3019         spin_lock_init(&target->lock);
3020         INIT_LIST_HEAD(&target->free_tx);
3021         ret = srp_alloc_req_data(target);
3022         if (ret)
3023                 goto err_free_mem;
3024
3025         ret = ib_query_gid(ibdev, host->port, 0, &target->path.sgid);
3026         if (ret)
3027                 goto err_free_mem;
3028
3029         ret = srp_create_target_ib(target);
3030         if (ret)
3031                 goto err_free_mem;
3032
3033         ret = srp_new_cm_id(target);
3034         if (ret)
3035                 goto err_free_ib;
3036
3037         ret = srp_connect_target(target);
3038         if (ret) {
3039                 shost_printk(KERN_ERR, target->scsi_host,
3040                              PFX "Connection failed\n");
3041                 goto err_cm_id;
3042         }
3043
3044         ret = srp_add_target(host, target);
3045         if (ret)
3046                 goto err_disconnect;
3047
3048         shost_printk(KERN_DEBUG, target->scsi_host, PFX
3049                      "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3050                      be64_to_cpu(target->id_ext),
3051                      be64_to_cpu(target->ioc_guid),
3052                      be16_to_cpu(target->path.pkey),
3053                      be64_to_cpu(target->service_id),
3054                      target->path.sgid.raw, target->path.dgid.raw);
3055
3056         ret = count;
3057
3058 out:
3059         mutex_unlock(&host->add_target_mutex);
3060         return ret;
3061
3062 err_disconnect:
3063         srp_disconnect_target(target);
3064
3065 err_cm_id:
3066         ib_destroy_cm_id(target->cm_id);
3067
3068 err_free_ib:
3069         srp_free_target_ib(target);
3070
3071 err_free_mem:
3072         srp_free_req_data(target);
3073
3074 err:
3075         scsi_host_put(target_host);
3076         goto out;
3077 }
3078
3079 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target);
3080
3081 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr,
3082                           char *buf)
3083 {
3084         struct srp_host *host = container_of(dev, struct srp_host, dev);
3085
3086         return sprintf(buf, "%s\n", host->srp_dev->dev->name);
3087 }
3088
3089 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL);
3090
3091 static ssize_t show_port(struct device *dev, struct device_attribute *attr,
3092                          char *buf)
3093 {
3094         struct srp_host *host = container_of(dev, struct srp_host, dev);
3095
3096         return sprintf(buf, "%d\n", host->port);
3097 }
3098
3099 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL);
3100
3101 static struct srp_host *srp_add_port(struct srp_device *device, u8 port)
3102 {
3103         struct srp_host *host;
3104
3105         host = kzalloc(sizeof *host, GFP_KERNEL);
3106         if (!host)
3107                 return NULL;
3108
3109         INIT_LIST_HEAD(&host->target_list);
3110         spin_lock_init(&host->target_lock);
3111         init_completion(&host->released);
3112         mutex_init(&host->add_target_mutex);
3113         host->srp_dev = device;
3114         host->port = port;
3115
3116         host->dev.class = &srp_class;
3117         host->dev.parent = device->dev->dma_device;
3118         dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port);
3119
3120         if (device_register(&host->dev))
3121                 goto free_host;
3122         if (device_create_file(&host->dev, &dev_attr_add_target))
3123                 goto err_class;
3124         if (device_create_file(&host->dev, &dev_attr_ibdev))
3125                 goto err_class;
3126         if (device_create_file(&host->dev, &dev_attr_port))
3127                 goto err_class;
3128
3129         return host;
3130
3131 err_class:
3132         device_unregister(&host->dev);
3133
3134 free_host:
3135         kfree(host);
3136
3137         return NULL;
3138 }
3139
3140 static void srp_add_one(struct ib_device *device)
3141 {
3142         struct srp_device *srp_dev;
3143         struct ib_device_attr *dev_attr;
3144         struct srp_host *host;
3145         int mr_page_shift, s, e, p;
3146         u64 max_pages_per_mr;
3147
3148         dev_attr = kmalloc(sizeof *dev_attr, GFP_KERNEL);
3149         if (!dev_attr)
3150                 return;
3151
3152         if (ib_query_device(device, dev_attr)) {
3153                 pr_warn("Query device failed for %s\n", device->name);
3154                 goto free_attr;
3155         }
3156
3157         srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL);
3158         if (!srp_dev)
3159                 goto free_attr;
3160
3161         srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr &&
3162                             device->map_phys_fmr && device->unmap_fmr);
3163         srp_dev->has_fr = (dev_attr->device_cap_flags &
3164                            IB_DEVICE_MEM_MGT_EXTENSIONS);
3165         if (!srp_dev->has_fmr && !srp_dev->has_fr)
3166                 dev_warn(&device->dev, "neither FMR nor FR is supported\n");
3167
3168         srp_dev->use_fast_reg = (srp_dev->has_fr &&
3169                                  (!srp_dev->has_fmr || prefer_fr));
3170
3171         /*
3172          * Use the smallest page size supported by the HCA, down to a
3173          * minimum of 4096 bytes. We're unlikely to build large sglists
3174          * out of smaller entries.
3175          */
3176         mr_page_shift           = max(12, ffs(dev_attr->page_size_cap) - 1);
3177         srp_dev->mr_page_size   = 1 << mr_page_shift;
3178         srp_dev->mr_page_mask   = ~((u64) srp_dev->mr_page_size - 1);
3179         max_pages_per_mr        = dev_attr->max_mr_size;
3180         do_div(max_pages_per_mr, srp_dev->mr_page_size);
3181         srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
3182                                           max_pages_per_mr);
3183         if (srp_dev->use_fast_reg) {
3184                 srp_dev->max_pages_per_mr =
3185                         min_t(u32, srp_dev->max_pages_per_mr,
3186                               dev_attr->max_fast_reg_page_list_len);
3187         }
3188         srp_dev->mr_max_size    = srp_dev->mr_page_size *
3189                                    srp_dev->max_pages_per_mr;
3190         pr_debug("%s: mr_page_shift = %d, dev_attr->max_mr_size = %#llx, dev_attr->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
3191                  device->name, mr_page_shift, dev_attr->max_mr_size,
3192                  dev_attr->max_fast_reg_page_list_len,
3193                  srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
3194
3195         INIT_LIST_HEAD(&srp_dev->dev_list);
3196
3197         srp_dev->dev = device;
3198         srp_dev->pd  = ib_alloc_pd(device);
3199         if (IS_ERR(srp_dev->pd))
3200                 goto free_dev;
3201
3202         srp_dev->mr = ib_get_dma_mr(srp_dev->pd,
3203                                     IB_ACCESS_LOCAL_WRITE |
3204                                     IB_ACCESS_REMOTE_READ |
3205                                     IB_ACCESS_REMOTE_WRITE);
3206         if (IS_ERR(srp_dev->mr))
3207                 goto err_pd;
3208
3209         if (device->node_type == RDMA_NODE_IB_SWITCH) {
3210                 s = 0;
3211                 e = 0;
3212         } else {
3213                 s = 1;
3214                 e = device->phys_port_cnt;
3215         }
3216
3217         for (p = s; p <= e; ++p) {
3218                 host = srp_add_port(srp_dev, p);
3219                 if (host)
3220                         list_add_tail(&host->list, &srp_dev->dev_list);
3221         }
3222
3223         ib_set_client_data(device, &srp_client, srp_dev);
3224
3225         goto free_attr;
3226
3227 err_pd:
3228         ib_dealloc_pd(srp_dev->pd);
3229
3230 free_dev:
3231         kfree(srp_dev);
3232
3233 free_attr:
3234         kfree(dev_attr);
3235 }
3236
3237 static void srp_remove_one(struct ib_device *device)
3238 {
3239         struct srp_device *srp_dev;
3240         struct srp_host *host, *tmp_host;
3241         struct srp_target_port *target;
3242
3243         srp_dev = ib_get_client_data(device, &srp_client);
3244         if (!srp_dev)
3245                 return;
3246
3247         list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3248                 device_unregister(&host->dev);
3249                 /*
3250                  * Wait for the sysfs entry to go away, so that no new
3251                  * target ports can be created.
3252                  */
3253                 wait_for_completion(&host->released);
3254
3255                 /*
3256                  * Remove all target ports.
3257                  */
3258                 spin_lock(&host->target_lock);
3259                 list_for_each_entry(target, &host->target_list, list)
3260                         srp_queue_remove_work(target);
3261                 spin_unlock(&host->target_lock);
3262
3263                 /*
3264                  * Wait for target port removal tasks.
3265                  */
3266                 flush_workqueue(system_long_wq);
3267
3268                 kfree(host);
3269         }
3270
3271         ib_dereg_mr(srp_dev->mr);
3272         ib_dealloc_pd(srp_dev->pd);
3273
3274         kfree(srp_dev);
3275 }
3276
3277 static struct srp_function_template ib_srp_transport_functions = {
3278         .has_rport_state         = true,
3279         .reset_timer_if_blocked  = true,
3280         .reconnect_delay         = &srp_reconnect_delay,
3281         .fast_io_fail_tmo        = &srp_fast_io_fail_tmo,
3282         .dev_loss_tmo            = &srp_dev_loss_tmo,
3283         .reconnect               = srp_rport_reconnect,
3284         .rport_delete            = srp_rport_delete,
3285         .terminate_rport_io      = srp_terminate_io,
3286 };
3287
3288 static int __init srp_init_module(void)
3289 {
3290         int ret;
3291
3292         BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));
3293
3294         if (srp_sg_tablesize) {
3295                 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
3296                 if (!cmd_sg_entries)
3297                         cmd_sg_entries = srp_sg_tablesize;
3298         }
3299
3300         if (!cmd_sg_entries)
3301                 cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
3302
3303         if (cmd_sg_entries > 255) {
3304                 pr_warn("Clamping cmd_sg_entries to 255\n");
3305                 cmd_sg_entries = 255;
3306         }
3307
3308         if (!indirect_sg_entries)
3309                 indirect_sg_entries = cmd_sg_entries;
3310         else if (indirect_sg_entries < cmd_sg_entries) {
3311                 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
3312                         cmd_sg_entries);
3313                 indirect_sg_entries = cmd_sg_entries;
3314         }
3315
3316         ib_srp_transport_template =
3317                 srp_attach_transport(&ib_srp_transport_functions);
3318         if (!ib_srp_transport_template)
3319                 return -ENOMEM;
3320
3321         ret = class_register(&srp_class);
3322         if (ret) {
3323                 pr_err("couldn't register class infiniband_srp\n");
3324                 srp_release_transport(ib_srp_transport_template);
3325                 return ret;
3326         }
3327
3328         ib_sa_register_client(&srp_sa_client);
3329
3330         ret = ib_register_client(&srp_client);
3331         if (ret) {
3332                 pr_err("couldn't register IB client\n");
3333                 srp_release_transport(ib_srp_transport_template);
3334                 ib_sa_unregister_client(&srp_sa_client);
3335                 class_unregister(&srp_class);
3336                 return ret;
3337         }
3338
3339         return 0;
3340 }
3341
3342 static void __exit srp_cleanup_module(void)
3343 {
3344         ib_unregister_client(&srp_client);
3345         ib_sa_unregister_client(&srp_sa_client);
3346         class_unregister(&srp_class);
3347         srp_release_transport(ib_srp_transport_template);
3348 }
3349
3350 module_init(srp_init_module);
3351 module_exit(srp_cleanup_module);