drivers/nvme/host/fc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016 Avago Technologies.  All rights reserved.
   4  */
   5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   6 #include <linux/module.h>
   7 #include <linux/parser.h>
   8 #include <uapi/scsi/fc/fc_fs.h>
   9 #include <uapi/scsi/fc/fc_els.h>
  10 #include <linux/delay.h>
  11 #include <linux/overflow.h>
  12 #include <linux/blk-cgroup.h>
  13 #include "nvme.h"
  14 #include "fabrics.h"
  15 #include <linux/nvme-fc-driver.h>
  16 #include <linux/nvme-fc.h>
  17 #include "fc.h"
  18 #include <scsi/scsi_transport_fc.h>
  19 #include <linux/blk-mq-pci.h>
  20
  21 /* *************************** Data Structures/Defines ****************** */
  22
  23
  24 enum nvme_fc_queue_flags {
  25         NVME_FC_Q_CONNECTED = 0,
  26         NVME_FC_Q_LIVE,
  27 };
  28
  29 #define NVME_FC_DEFAULT_DEV_LOSS_TMO    60      /* seconds */
  30 #define NVME_FC_DEFAULT_RECONNECT_TMO   2       /* delay between reconnects
  31                                                  * when connected and a
  32                                                  * connection failure.
  33                                                  */
  34
  35 struct nvme_fc_queue {
  36         struct nvme_fc_ctrl     *ctrl;
  37         struct device           *dev;
  38         struct blk_mq_hw_ctx    *hctx;
  39         void                    *lldd_handle;
  40         size_t                  cmnd_capsule_len;
  41         u32                     qnum;
  42         u32                     rqcnt;
  43         u32                     seqno;
  44
  45         u64                     connection_id;
  46         atomic_t                csn;
  47
  48         unsigned long           flags;
  49 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
  50
  51 enum nvme_fcop_flags {
  52         FCOP_FLAGS_TERMIO       = (1 << 0),
  53         FCOP_FLAGS_AEN          = (1 << 1),
  54 };
  55
  56 struct nvmefc_ls_req_op {
  57         struct nvmefc_ls_req    ls_req;
  58
  59         struct nvme_fc_rport    *rport;
  60         struct nvme_fc_queue    *queue;
  61         struct request          *rq;
  62         u32                     flags;
  63
  64         int                     ls_error;
  65         struct completion       ls_done;
  66         struct list_head        lsreq_list;     /* rport->ls_req_list */
  67         bool                    req_queued;
  68 };
  69
  70 struct nvmefc_ls_rcv_op {
  71         struct nvme_fc_rport            *rport;
  72         struct nvmefc_ls_rsp            *lsrsp;
  73         union nvmefc_ls_requests        *rqstbuf;
  74         union nvmefc_ls_responses       *rspbuf;
  75         u16                             rqstdatalen;
  76         bool                            handled;
  77         dma_addr_t                      rspdma;
  78         struct list_head                lsrcv_list;     /* rport->ls_rcv_list */
  79 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
  80
  81 enum nvme_fcpop_state {
  82         FCPOP_STATE_UNINIT      = 0,
  83         FCPOP_STATE_IDLE        = 1,
  84         FCPOP_STATE_ACTIVE      = 2,
  85         FCPOP_STATE_ABORTED     = 3,
  86         FCPOP_STATE_COMPLETE    = 4,
  87 };
  88
  89 struct nvme_fc_fcp_op {
  90         struct nvme_request     nreq;           /*
  91                                                  * nvme/host/core.c
  92                                                  * requires this to be
  93                                                  * the 1st element in the
  94                                                  * private structure
  95                                                  * associated with the
  96                                                  * request.
  97                                                  */
  98         struct nvmefc_fcp_req   fcp_req;
  99
 100         struct nvme_fc_ctrl     *ctrl;
 101         struct nvme_fc_queue    *queue;
 102         struct request          *rq;
 103
 104         atomic_t                state;
 105         u32                     flags;
 106         u32                     rqno;
 107         u32                     nents;
 108
 109         struct nvme_fc_cmd_iu   cmd_iu;
 110         struct nvme_fc_ersp_iu  rsp_iu;
 111 };
 112
 113 struct nvme_fcp_op_w_sgl {
 114         struct nvme_fc_fcp_op   op;
 115         struct scatterlist      sgl[NVME_INLINE_SG_CNT];
 116         uint8_t                 priv[];
 117 };
 118
 119 struct nvme_fc_lport {
 120         struct nvme_fc_local_port       localport;
 121
 122         struct ida                      endp_cnt;
 123         struct list_head                port_list;      /* nvme_fc_port_list */
 124         struct list_head                endp_list;
 125         struct device                   *dev;   /* physical device for dma */
 126         struct nvme_fc_port_template    *ops;
 127         struct kref                     ref;
 128         atomic_t                        act_rport_cnt;
 129 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
 130
 131 struct nvme_fc_rport {
 132         struct nvme_fc_remote_port      remoteport;
 133
 134         struct list_head                endp_list; /* for lport->endp_list */
 135         struct list_head                ctrl_list;
 136         struct list_head                ls_req_list;
 137         struct list_head                ls_rcv_list;
 138         struct list_head                disc_list;
 139         struct device                   *dev;   /* physical device for dma */
 140         struct nvme_fc_lport            *lport;
 141         spinlock_t                      lock;
 142         struct kref                     ref;
 143         atomic_t                        act_ctrl_cnt;
 144         unsigned long                   dev_loss_end;
 145         struct work_struct              lsrcv_work;
 146 } __aligned(sizeof(u64));       /* alignment for other things alloc'd with */
 147
 148 /* fc_ctrl flags values - specified as bit positions */
 149 #define ASSOC_ACTIVE            0
 150 #define ASSOC_FAILED            1
 151 #define FCCTRL_TERMIO           2
 152
 153 struct nvme_fc_ctrl {
 154         spinlock_t              lock;
 155         struct nvme_fc_queue    *queues;
 156         struct device           *dev;
 157         struct nvme_fc_lport    *lport;
 158         struct nvme_fc_rport    *rport;
 159         u32                     cnum;
 160
 161         bool                    ioq_live;
 162         u64                     association_id;
 163         struct nvmefc_ls_rcv_op *rcv_disconn;
 164
 165         struct list_head        ctrl_list;      /* rport->ctrl_list */
 166
 167         struct blk_mq_tag_set   admin_tag_set;
 168         struct blk_mq_tag_set   tag_set;
 169
 170         struct work_struct      ioerr_work;
 171         struct delayed_work     connect_work;
 172
 173         struct kref             ref;
 174         unsigned long           flags;
 175         u32                     iocnt;
 176         wait_queue_head_t       ioabort_wait;
 177
 178         struct nvme_fc_fcp_op   aen_ops[NVME_NR_AEN_COMMANDS];
 179
 180         struct nvme_ctrl        ctrl;
 181 };
 182
 183 static inline struct nvme_fc_ctrl *
 184 to_fc_ctrl(struct nvme_ctrl *ctrl)
 185 {
 186         return container_of(ctrl, struct nvme_fc_ctrl, ctrl);
 187 }
 188
 189 static inline struct nvme_fc_lport *
 190 localport_to_lport(struct nvme_fc_local_port *portptr)
 191 {
 192         return container_of(portptr, struct nvme_fc_lport, localport);
 193 }
 194
 195 static inline struct nvme_fc_rport *
 196 remoteport_to_rport(struct nvme_fc_remote_port *portptr)
 197 {
 198         return container_of(portptr, struct nvme_fc_rport, remoteport);
 199 }
 200
 201 static inline struct nvmefc_ls_req_op *
 202 ls_req_to_lsop(struct nvmefc_ls_req *lsreq)
 203 {
 204         return container_of(lsreq, struct nvmefc_ls_req_op, ls_req);
 205 }
 206
 207 static inline struct nvme_fc_fcp_op *
 208 fcp_req_to_fcp_op(struct nvmefc_fcp_req *fcpreq)
 209 {
 210         return container_of(fcpreq, struct nvme_fc_fcp_op, fcp_req);
 211 }
 212
 213
 214
 215 /* *************************** Globals **************************** */
 216
 217
 218 static DEFINE_SPINLOCK(nvme_fc_lock);
 219
 220 static LIST_HEAD(nvme_fc_lport_list);
 221 static DEFINE_IDA(nvme_fc_local_port_cnt);
 222 static DEFINE_IDA(nvme_fc_ctrl_cnt);
 223
 224 /*
 225  * These items are short-term. They will eventually be moved into
 226  * a generic FC class. See comments in module init.
 227  */
 228 static struct device *fc_udev_device;
 229
 230 static void nvme_fc_complete_rq(struct request *rq);
 231
 232 /* *********************** FC-NVME Port Management ************************ */
 233
 234 static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *,
 235                         struct nvme_fc_queue *, unsigned int);
 236
 237 static void nvme_fc_handle_ls_rqst_work(struct work_struct *work);
 238
 239
 240 static void
 241 nvme_fc_free_lport(struct kref *ref)
 242 {
 243         struct nvme_fc_lport *lport =
 244                 container_of(ref, struct nvme_fc_lport, ref);
 245         unsigned long flags;
 246
 247         WARN_ON(lport->localport.port_state != FC_OBJSTATE_DELETED);
 248         WARN_ON(!list_empty(&lport->endp_list));
 249
 250         /* remove from transport list */
 251         spin_lock_irqsave(&nvme_fc_lock, flags);
 252         list_del(&lport->port_list);
 253         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 254
 255         ida_free(&nvme_fc_local_port_cnt, lport->localport.port_num);
 256         ida_destroy(&lport->endp_cnt);
 257
 258         put_device(lport->dev);
 259
 260         kfree(lport);
 261 }
 262
 263 static void
 264 nvme_fc_lport_put(struct nvme_fc_lport *lport)
 265 {
 266         kref_put(&lport->ref, nvme_fc_free_lport);
 267 }
 268
 269 static int
 270 nvme_fc_lport_get(struct nvme_fc_lport *lport)
 271 {
 272         return kref_get_unless_zero(&lport->ref);
 273 }
 274
 275
 276 static struct nvme_fc_lport *
 277 nvme_fc_attach_to_unreg_lport(struct nvme_fc_port_info *pinfo,
 278                         struct nvme_fc_port_template *ops,
 279                         struct device *dev)
 280 {
 281         struct nvme_fc_lport *lport;
 282         unsigned long flags;
 283
 284         spin_lock_irqsave(&nvme_fc_lock, flags);
 285
 286         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
 287                 if (lport->localport.node_name != pinfo->node_name ||
 288                     lport->localport.port_name != pinfo->port_name)
 289                         continue;
 290
 291                 if (lport->dev != dev) {
 292                         lport = ERR_PTR(-EXDEV);
 293                         goto out_done;
 294                 }
 295
 296                 if (lport->localport.port_state != FC_OBJSTATE_DELETED) {
 297                         lport = ERR_PTR(-EEXIST);
 298                         goto out_done;
 299                 }
 300
 301                 if (!nvme_fc_lport_get(lport)) {
 302                         /*
 303                          * fails if ref cnt already 0. If so,
 304                          * act as if lport already deleted
 305                          */
 306                         lport = NULL;
 307                         goto out_done;
 308                 }
 309
 310                 /* resume the lport */
 311
 312                 lport->ops = ops;
 313                 lport->localport.port_role = pinfo->port_role;
 314                 lport->localport.port_id = pinfo->port_id;
 315                 lport->localport.port_state = FC_OBJSTATE_ONLINE;
 316
 317                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 318
 319                 return lport;
 320         }
 321
 322         lport = NULL;
 323
 324 out_done:
 325         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 326
 327         return lport;
 328 }
 329
 330 /**
 331  * nvme_fc_register_localport - transport entry point called by an
 332  *                              LLDD to register the existence of a NVME
 333  *                              host FC port.
 334  * @pinfo:     pointer to information about the port to be registered
 335  * @template:  LLDD entrypoints and operational parameters for the port
 336  * @dev:       physical hardware device node port corresponds to. Will be
 337  *             used for DMA mappings
 338  * @portptr:   pointer to a local port pointer. Upon success, the routine
 339  *             will allocate a nvme_fc_local_port structure and place its
 340  *             address in the local port pointer. Upon failure, local port
 341  *             pointer will be set to 0.
 342  *
 343  * Returns:
 344  * a completion status. Must be 0 upon success; a negative errno
 345  * (ex: -ENXIO) upon failure.
 346  */
 347 int
 348 nvme_fc_register_localport(struct nvme_fc_port_info *pinfo,
 349                         struct nvme_fc_port_template *template,
 350                         struct device *dev,
 351                         struct nvme_fc_local_port **portptr)
 352 {
 353         struct nvme_fc_lport *newrec;
 354         unsigned long flags;
 355         int ret, idx;
 356
 357         if (!template->localport_delete || !template->remoteport_delete ||
 358             !template->ls_req || !template->fcp_io ||
 359             !template->ls_abort || !template->fcp_abort ||
 360             !template->max_hw_queues || !template->max_sgl_segments ||
 361             !template->max_dif_sgl_segments || !template->dma_boundary) {
 362                 ret = -EINVAL;
 363                 goto out_reghost_failed;
 364         }
 365
 366         /*
 367          * look to see if there is already a localport that had been
 368          * deregistered and in the process of waiting for all the
 369          * references to fully be removed.  If the references haven't
 370          * expired, we can simply re-enable the localport. Remoteports
 371          * and controller reconnections should resume naturally.
 372          */
 373         newrec = nvme_fc_attach_to_unreg_lport(pinfo, template, dev);
 374
 375         /* found an lport, but something about its state is bad */
 376         if (IS_ERR(newrec)) {
 377                 ret = PTR_ERR(newrec);
 378                 goto out_reghost_failed;
 379
 380         /* found existing lport, which was resumed */
 381         } else if (newrec) {
 382                 *portptr = &newrec->localport;
 383                 return 0;
 384         }
 385
 386         /* nothing found - allocate a new localport struct */
 387
 388         newrec = kmalloc((sizeof(*newrec) + template->local_priv_sz),
 389                          GFP_KERNEL);
 390         if (!newrec) {
 391                 ret = -ENOMEM;
 392                 goto out_reghost_failed;
 393         }
 394
 395         idx = ida_alloc(&nvme_fc_local_port_cnt, GFP_KERNEL);
 396         if (idx < 0) {
 397                 ret = -ENOSPC;
 398                 goto out_fail_kfree;
 399         }
 400
 401         if (!get_device(dev) && dev) {
 402                 ret = -ENODEV;
 403                 goto out_ida_put;
 404         }
 405
 406         INIT_LIST_HEAD(&newrec->port_list);
 407         INIT_LIST_HEAD(&newrec->endp_list);
 408         kref_init(&newrec->ref);
 409         atomic_set(&newrec->act_rport_cnt, 0);
 410         newrec->ops = template;
 411         newrec->dev = dev;
 412         ida_init(&newrec->endp_cnt);
 413         if (template->local_priv_sz)
 414                 newrec->localport.private = &newrec[1];
 415         else
 416                 newrec->localport.private = NULL;
 417         newrec->localport.node_name = pinfo->node_name;
 418         newrec->localport.port_name = pinfo->port_name;
 419         newrec->localport.port_role = pinfo->port_role;
 420         newrec->localport.port_id = pinfo->port_id;
 421         newrec->localport.port_state = FC_OBJSTATE_ONLINE;
 422         newrec->localport.port_num = idx;
 423
 424         spin_lock_irqsave(&nvme_fc_lock, flags);
 425         list_add_tail(&newrec->port_list, &nvme_fc_lport_list);
 426         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 427
 428         if (dev)
 429                 dma_set_seg_boundary(dev, template->dma_boundary);
 430
 431         *portptr = &newrec->localport;
 432         return 0;
 433
 434 out_ida_put:
 435         ida_free(&nvme_fc_local_port_cnt, idx);
 436 out_fail_kfree:
 437         kfree(newrec);
 438 out_reghost_failed:
 439         *portptr = NULL;
 440
 441         return ret;
 442 }
 443 EXPORT_SYMBOL_GPL(nvme_fc_register_localport);
 444
 445 /**
 446  * nvme_fc_unregister_localport - transport entry point called by an
 447  *                              LLDD to deregister/remove a previously
 448  *                              registered a NVME host FC port.
 449  * @portptr: pointer to the (registered) local port that is to be deregistered.
 450  *
 451  * Returns:
 452  * a completion status. Must be 0 upon success; a negative errno
 453  * (ex: -ENXIO) upon failure.
 454  */
 455 int
 456 nvme_fc_unregister_localport(struct nvme_fc_local_port *portptr)
 457 {
 458         struct nvme_fc_lport *lport = localport_to_lport(portptr);
 459         unsigned long flags;
 460
 461         if (!portptr)
 462                 return -EINVAL;
 463
 464         spin_lock_irqsave(&nvme_fc_lock, flags);
 465
 466         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 467                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 468                 return -EINVAL;
 469         }
 470         portptr->port_state = FC_OBJSTATE_DELETED;
 471
 472         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 473
 474         if (atomic_read(&lport->act_rport_cnt) == 0)
 475                 lport->ops->localport_delete(&lport->localport);
 476
 477         nvme_fc_lport_put(lport);
 478
 479         return 0;
 480 }
 481 EXPORT_SYMBOL_GPL(nvme_fc_unregister_localport);
 482
 483 /*
 484  * TRADDR strings, per FC-NVME are fixed format:
 485  *   "nn-0x<16hexdigits>:pn-0x<16hexdigits>" - 43 characters
 486  * udev event will only differ by prefix of what field is
 487  * being specified:
 488  *    "NVMEFC_HOST_TRADDR=" or "NVMEFC_TRADDR=" - 19 max characters
 489  *  19 + 43 + null_fudge = 64 characters
 490  */
 491 #define FCNVME_TRADDR_LENGTH            64
 492
 493 static void
 494 nvme_fc_signal_discovery_scan(struct nvme_fc_lport *lport,
 495                 struct nvme_fc_rport *rport)
 496 {
 497         char hostaddr[FCNVME_TRADDR_LENGTH];    /* NVMEFC_HOST_TRADDR=...*/
 498         char tgtaddr[FCNVME_TRADDR_LENGTH];     /* NVMEFC_TRADDR=...*/
 499         char *envp[4] = { "FC_EVENT=nvmediscovery", hostaddr, tgtaddr, NULL };
 500
 501         if (!(rport->remoteport.port_role & FC_PORT_ROLE_NVME_DISCOVERY))
 502                 return;
 503
 504         snprintf(hostaddr, sizeof(hostaddr),
 505                 "NVMEFC_HOST_TRADDR=nn-0x%016llx:pn-0x%016llx",
 506                 lport->localport.node_name, lport->localport.port_name);
 507         snprintf(tgtaddr, sizeof(tgtaddr),
 508                 "NVMEFC_TRADDR=nn-0x%016llx:pn-0x%016llx",
 509                 rport->remoteport.node_name, rport->remoteport.port_name);
 510         kobject_uevent_env(&fc_udev_device->kobj, KOBJ_CHANGE, envp);
 511 }
 512
 513 static void
 514 nvme_fc_free_rport(struct kref *ref)
 515 {
 516         struct nvme_fc_rport *rport =
 517                 container_of(ref, struct nvme_fc_rport, ref);
 518         struct nvme_fc_lport *lport =
 519                         localport_to_lport(rport->remoteport.localport);
 520         unsigned long flags;
 521
 522         WARN_ON(rport->remoteport.port_state != FC_OBJSTATE_DELETED);
 523         WARN_ON(!list_empty(&rport->ctrl_list));
 524
 525         /* remove from lport list */
 526         spin_lock_irqsave(&nvme_fc_lock, flags);
 527         list_del(&rport->endp_list);
 528         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 529
 530         WARN_ON(!list_empty(&rport->disc_list));
 531         ida_free(&lport->endp_cnt, rport->remoteport.port_num);
 532
 533         kfree(rport);
 534
 535         nvme_fc_lport_put(lport);
 536 }
 537
 538 static void
 539 nvme_fc_rport_put(struct nvme_fc_rport *rport)
 540 {
 541         kref_put(&rport->ref, nvme_fc_free_rport);
 542 }
 543
 544 static int
 545 nvme_fc_rport_get(struct nvme_fc_rport *rport)
 546 {
 547         return kref_get_unless_zero(&rport->ref);
 548 }
 549
 550 static void
 551 nvme_fc_resume_controller(struct nvme_fc_ctrl *ctrl)
 552 {
 553         switch (nvme_ctrl_state(&ctrl->ctrl)) {
 554         case NVME_CTRL_NEW:
 555         case NVME_CTRL_CONNECTING:
 556                 /*
 557                  * As all reconnects were suppressed, schedule a
 558                  * connect.
 559                  */
 560                 dev_info(ctrl->ctrl.device,
 561                         "NVME-FC{%d}: connectivity re-established. "
 562                         "Attempting reconnect\n", ctrl->cnum);
 563
 564                 queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
 565                 break;
 566
 567         case NVME_CTRL_RESETTING:
 568                 /*
 569                  * Controller is already in the process of terminating the
 570                  * association. No need to do anything further. The reconnect
 571                  * step will naturally occur after the reset completes.
 572                  */
 573                 break;
 574
 575         default:
 576                 /* no action to take - let it delete */
 577                 break;
 578         }
 579 }
 580
 581 static struct nvme_fc_rport *
 582 nvme_fc_attach_to_suspended_rport(struct nvme_fc_lport *lport,
 583                                 struct nvme_fc_port_info *pinfo)
 584 {
 585         struct nvme_fc_rport *rport;
 586         struct nvme_fc_ctrl *ctrl;
 587         unsigned long flags;
 588
 589         spin_lock_irqsave(&nvme_fc_lock, flags);
 590
 591         list_for_each_entry(rport, &lport->endp_list, endp_list) {
 592                 if (rport->remoteport.node_name != pinfo->node_name ||
 593                     rport->remoteport.port_name != pinfo->port_name)
 594                         continue;
 595
 596                 if (!nvme_fc_rport_get(rport)) {
 597                         rport = ERR_PTR(-ENOLCK);
 598                         goto out_done;
 599                 }
 600
 601                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
 602
 603                 spin_lock_irqsave(&rport->lock, flags);
 604
 605                 /* has it been unregistered */
 606                 if (rport->remoteport.port_state != FC_OBJSTATE_DELETED) {
 607                         /* means lldd called us twice */
 608                         spin_unlock_irqrestore(&rport->lock, flags);
 609                         nvme_fc_rport_put(rport);
 610                         return ERR_PTR(-ESTALE);
 611                 }
 612
 613                 rport->remoteport.port_role = pinfo->port_role;
 614                 rport->remoteport.port_id = pinfo->port_id;
 615                 rport->remoteport.port_state = FC_OBJSTATE_ONLINE;
 616                 rport->dev_loss_end = 0;
 617
 618                 /*
 619                  * kick off a reconnect attempt on all associations to the
 620                  * remote port. A successful reconnects will resume i/o.
 621                  */
 622                 list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list)
 623                         nvme_fc_resume_controller(ctrl);
 624
 625                 spin_unlock_irqrestore(&rport->lock, flags);
 626
 627                 return rport;
 628         }
 629
 630         rport = NULL;
 631
 632 out_done:
 633         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 634
 635         return rport;
 636 }
 637
 638 static inline void
 639 __nvme_fc_set_dev_loss_tmo(struct nvme_fc_rport *rport,
 640                         struct nvme_fc_port_info *pinfo)
 641 {
 642         if (pinfo->dev_loss_tmo)
 643                 rport->remoteport.dev_loss_tmo = pinfo->dev_loss_tmo;
 644         else
 645                 rport->remoteport.dev_loss_tmo = NVME_FC_DEFAULT_DEV_LOSS_TMO;
 646 }
 647
 648 /**
 649  * nvme_fc_register_remoteport - transport entry point called by an
 650  *                              LLDD to register the existence of a NVME
 651  *                              subsystem FC port on its fabric.
 652  * @localport: pointer to the (registered) local port that the remote
 653  *             subsystem port is connected to.
 654  * @pinfo:     pointer to information about the port to be registered
 655  * @portptr:   pointer to a remote port pointer. Upon success, the routine
 656  *             will allocate a nvme_fc_remote_port structure and place its
 657  *             address in the remote port pointer. Upon failure, remote port
 658  *             pointer will be set to 0.
 659  *
 660  * Returns:
 661  * a completion status. Must be 0 upon success; a negative errno
 662  * (ex: -ENXIO) upon failure.
 663  */
 664 int
 665 nvme_fc_register_remoteport(struct nvme_fc_local_port *localport,
 666                                 struct nvme_fc_port_info *pinfo,
 667                                 struct nvme_fc_remote_port **portptr)
 668 {
 669         struct nvme_fc_lport *lport = localport_to_lport(localport);
 670         struct nvme_fc_rport *newrec;
 671         unsigned long flags;
 672         int ret, idx;
 673
 674         if (!nvme_fc_lport_get(lport)) {
 675                 ret = -ESHUTDOWN;
 676                 goto out_reghost_failed;
 677         }
 678
 679         /*
 680          * look to see if there is already a remoteport that is waiting
 681          * for a reconnect (within dev_loss_tmo) with the same WWN's.
 682          * If so, transition to it and reconnect.
 683          */
 684         newrec = nvme_fc_attach_to_suspended_rport(lport, pinfo);
 685
 686         /* found an rport, but something about its state is bad */
 687         if (IS_ERR(newrec)) {
 688                 ret = PTR_ERR(newrec);
 689                 goto out_lport_put;
 690
 691         /* found existing rport, which was resumed */
 692         } else if (newrec) {
 693                 nvme_fc_lport_put(lport);
 694                 __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 695                 nvme_fc_signal_discovery_scan(lport, newrec);
 696                 *portptr = &newrec->remoteport;
 697                 return 0;
 698         }
 699
 700         /* nothing found - allocate a new remoteport struct */
 701
 702         newrec = kmalloc((sizeof(*newrec) + lport->ops->remote_priv_sz),
 703                          GFP_KERNEL);
 704         if (!newrec) {
 705                 ret = -ENOMEM;
 706                 goto out_lport_put;
 707         }
 708
 709         idx = ida_alloc(&lport->endp_cnt, GFP_KERNEL);
 710         if (idx < 0) {
 711                 ret = -ENOSPC;
 712                 goto out_kfree_rport;
 713         }
 714
 715         INIT_LIST_HEAD(&newrec->endp_list);
 716         INIT_LIST_HEAD(&newrec->ctrl_list);
 717         INIT_LIST_HEAD(&newrec->ls_req_list);
 718         INIT_LIST_HEAD(&newrec->disc_list);
 719         kref_init(&newrec->ref);
 720         atomic_set(&newrec->act_ctrl_cnt, 0);
 721         spin_lock_init(&newrec->lock);
 722         newrec->remoteport.localport = &lport->localport;
 723         INIT_LIST_HEAD(&newrec->ls_rcv_list);
 724         newrec->dev = lport->dev;
 725         newrec->lport = lport;
 726         if (lport->ops->remote_priv_sz)
 727                 newrec->remoteport.private = &newrec[1];
 728         else
 729                 newrec->remoteport.private = NULL;
 730         newrec->remoteport.port_role = pinfo->port_role;
 731         newrec->remoteport.node_name = pinfo->node_name;
 732         newrec->remoteport.port_name = pinfo->port_name;
 733         newrec->remoteport.port_id = pinfo->port_id;
 734         newrec->remoteport.port_state = FC_OBJSTATE_ONLINE;
 735         newrec->remoteport.port_num = idx;
 736         __nvme_fc_set_dev_loss_tmo(newrec, pinfo);
 737         INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work);
 738
 739         spin_lock_irqsave(&nvme_fc_lock, flags);
 740         list_add_tail(&newrec->endp_list, &lport->endp_list);
 741         spin_unlock_irqrestore(&nvme_fc_lock, flags);
 742
 743         nvme_fc_signal_discovery_scan(lport, newrec);
 744
 745         *portptr = &newrec->remoteport;
 746         return 0;
 747
 748 out_kfree_rport:
 749         kfree(newrec);
 750 out_lport_put:
 751         nvme_fc_lport_put(lport);
 752 out_reghost_failed:
 753         *portptr = NULL;
 754         return ret;
 755 }
 756 EXPORT_SYMBOL_GPL(nvme_fc_register_remoteport);
 757
 758 static int
 759 nvme_fc_abort_lsops(struct nvme_fc_rport *rport)
 760 {
 761         struct nvmefc_ls_req_op *lsop;
 762         unsigned long flags;
 763
 764 restart:
 765         spin_lock_irqsave(&rport->lock, flags);
 766
 767         list_for_each_entry(lsop, &rport->ls_req_list, lsreq_list) {
 768                 if (!(lsop->flags & FCOP_FLAGS_TERMIO)) {
 769                         lsop->flags |= FCOP_FLAGS_TERMIO;
 770                         spin_unlock_irqrestore(&rport->lock, flags);
 771                         rport->lport->ops->ls_abort(&rport->lport->localport,
 772                                                 &rport->remoteport,
 773                                                 &lsop->ls_req);
 774                         goto restart;
 775                 }
 776         }
 777         spin_unlock_irqrestore(&rport->lock, flags);
 778
 779         return 0;
 780 }
 781
 782 static void
 783 nvme_fc_ctrl_connectivity_loss(struct nvme_fc_ctrl *ctrl)
 784 {
 785         dev_info(ctrl->ctrl.device,
 786                 "NVME-FC{%d}: controller connectivity lost. Awaiting "
 787                 "Reconnect", ctrl->cnum);
 788
 789         switch (nvme_ctrl_state(&ctrl->ctrl)) {
 790         case NVME_CTRL_NEW:
 791         case NVME_CTRL_LIVE:
 792                 /*
 793                  * Schedule a controller reset. The reset will terminate the
 794                  * association and schedule the reconnect timer.  Reconnects
 795                  * will be attempted until either the ctlr_loss_tmo
 796                  * (max_retries * connect_delay) expires or the remoteport's
 797                  * dev_loss_tmo expires.
 798                  */
 799                 if (nvme_reset_ctrl(&ctrl->ctrl)) {
 800                         dev_warn(ctrl->ctrl.device,
 801                                 "NVME-FC{%d}: Couldn't schedule reset.\n",
 802                                 ctrl->cnum);
 803                         nvme_delete_ctrl(&ctrl->ctrl);
 804                 }
 805                 break;
 806
 807         case NVME_CTRL_CONNECTING:
 808                 /*
 809                  * The association has already been terminated and the
 810                  * controller is attempting reconnects.  No need to do anything
 811                  * futher.  Reconnects will be attempted until either the
 812                  * ctlr_loss_tmo (max_retries * connect_delay) expires or the
 813                  * remoteport's dev_loss_tmo expires.
 814                  */
 815                 break;
 816
 817         case NVME_CTRL_RESETTING:
 818                 /*
 819                  * Controller is already in the process of terminating the
 820                  * association.  No need to do anything further. The reconnect
 821                  * step will kick in naturally after the association is
 822                  * terminated.
 823                  */
 824                 break;
 825
 826         case NVME_CTRL_DELETING:
 827         case NVME_CTRL_DELETING_NOIO:
 828         default:
 829                 /* no action to take - let it delete */
 830                 break;
 831         }
 832 }
 833
 834 /**
 835  * nvme_fc_unregister_remoteport - transport entry point called by an
 836  *                              LLDD to deregister/remove a previously
 837  *                              registered a NVME subsystem FC port.
 838  * @portptr: pointer to the (registered) remote port that is to be
 839  *           deregistered.
 840  *
 841  * Returns:
 842  * a completion status. Must be 0 upon success; a negative errno
 843  * (ex: -ENXIO) upon failure.
 844  */
 845 int
 846 nvme_fc_unregister_remoteport(struct nvme_fc_remote_port *portptr)
 847 {
 848         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 849         struct nvme_fc_ctrl *ctrl;
 850         unsigned long flags;
 851
 852         if (!portptr)
 853                 return -EINVAL;
 854
 855         spin_lock_irqsave(&rport->lock, flags);
 856
 857         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 858                 spin_unlock_irqrestore(&rport->lock, flags);
 859                 return -EINVAL;
 860         }
 861         portptr->port_state = FC_OBJSTATE_DELETED;
 862
 863         rport->dev_loss_end = jiffies + (portptr->dev_loss_tmo * HZ);
 864
 865         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
 866                 /* if dev_loss_tmo==0, dev loss is immediate */
 867                 if (!portptr->dev_loss_tmo) {
 868                         dev_warn(ctrl->ctrl.device,
 869                                 "NVME-FC{%d}: controller connectivity lost.\n",
 870                                 ctrl->cnum);
 871                         nvme_delete_ctrl(&ctrl->ctrl);
 872                 } else
 873                         nvme_fc_ctrl_connectivity_loss(ctrl);
 874         }
 875
 876         spin_unlock_irqrestore(&rport->lock, flags);
 877
 878         nvme_fc_abort_lsops(rport);
 879
 880         if (atomic_read(&rport->act_ctrl_cnt) == 0)
 881                 rport->lport->ops->remoteport_delete(portptr);
 882
 883         /*
 884          * release the reference, which will allow, if all controllers
 885          * go away, which should only occur after dev_loss_tmo occurs,
 886          * for the rport to be torn down.
 887          */
 888         nvme_fc_rport_put(rport);
 889
 890         return 0;
 891 }
 892 EXPORT_SYMBOL_GPL(nvme_fc_unregister_remoteport);
 893
 894 /**
 895  * nvme_fc_rescan_remoteport - transport entry point called by an
 896  *                              LLDD to request a nvme device rescan.
 897  * @remoteport: pointer to the (registered) remote port that is to be
 898  *              rescanned.
 899  *
 900  * Returns: N/A
 901  */
 902 void
 903 nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport)
 904 {
 905         struct nvme_fc_rport *rport = remoteport_to_rport(remoteport);
 906
 907         nvme_fc_signal_discovery_scan(rport->lport, rport);
 908 }
 909 EXPORT_SYMBOL_GPL(nvme_fc_rescan_remoteport);
 910
 911 int
 912 nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *portptr,
 913                         u32 dev_loss_tmo)
 914 {
 915         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
 916         unsigned long flags;
 917
 918         spin_lock_irqsave(&rport->lock, flags);
 919
 920         if (portptr->port_state != FC_OBJSTATE_ONLINE) {
 921                 spin_unlock_irqrestore(&rport->lock, flags);
 922                 return -EINVAL;
 923         }
 924
 925         /* a dev_loss_tmo of 0 (immediate) is allowed to be set */
 926         rport->remoteport.dev_loss_tmo = dev_loss_tmo;
 927
 928         spin_unlock_irqrestore(&rport->lock, flags);
 929
 930         return 0;
 931 }
 932 EXPORT_SYMBOL_GPL(nvme_fc_set_remoteport_devloss);
 933
 934
 935 /* *********************** FC-NVME DMA Handling **************************** */
 936
 937 /*
 938  * The fcloop device passes in a NULL device pointer. Real LLD's will
 939  * pass in a valid device pointer. If NULL is passed to the dma mapping
 940  * routines, depending on the platform, it may or may not succeed, and
 941  * may crash.
 942  *
 943  * As such:
 944  * Wrapper all the dma routines and check the dev pointer.
 945  *
 946  * If simple mappings (return just a dma address, we'll noop them,
 947  * returning a dma address of 0.
 948  *
 949  * On more complex mappings (dma_map_sg), a pseudo routine fills
 950  * in the scatter list, setting all dma addresses to 0.
 951  */
 952
 953 static inline dma_addr_t
 954 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
 955                 enum dma_data_direction dir)
 956 {
 957         return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
 958 }
 959
 960 static inline int
 961 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 962 {
 963         return dev ? dma_mapping_error(dev, dma_addr) : 0;
 964 }
 965
 966 static inline void
 967 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
 968         enum dma_data_direction dir)
 969 {
 970         if (dev)
 971                 dma_unmap_single(dev, addr, size, dir);
 972 }
 973
 974 static inline void
 975 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
 976                 enum dma_data_direction dir)
 977 {
 978         if (dev)
 979                 dma_sync_single_for_cpu(dev, addr, size, dir);
 980 }
 981
 982 static inline void
 983 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
 984                 enum dma_data_direction dir)
 985 {
 986         if (dev)
 987                 dma_sync_single_for_device(dev, addr, size, dir);
 988 }
 989
 990 /* pseudo dma_map_sg call */
 991 static int
 992 fc_map_sg(struct scatterlist *sg, int nents)
 993 {
 994         struct scatterlist *s;
 995         int i;
 996
 997         WARN_ON(nents == 0 || sg[0].length == 0);
 998
 999         for_each_sg(sg, s, nents, i) {
1000                 s->dma_address = 0L;
1001 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1002                 s->dma_length = s->length;
1003 #endif
1004         }
1005         return nents;
1006 }
1007
1008 static inline int
1009 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1010                 enum dma_data_direction dir)
1011 {
1012         return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
1013 }
1014
1015 static inline void
1016 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1017                 enum dma_data_direction dir)
1018 {
1019         if (dev)
1020                 dma_unmap_sg(dev, sg, nents, dir);
1021 }
1022
1023 /* *********************** FC-NVME LS Handling **************************** */
1024
1025 static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
1026 static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
1027
1028 static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
1029
1030 static void
1031 __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
1032 {
1033         struct nvme_fc_rport *rport = lsop->rport;
1034         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1035         unsigned long flags;
1036
1037         spin_lock_irqsave(&rport->lock, flags);
1038
1039         if (!lsop->req_queued) {
1040                 spin_unlock_irqrestore(&rport->lock, flags);
1041                 return;
1042         }
1043
1044         list_del(&lsop->lsreq_list);
1045
1046         lsop->req_queued = false;
1047
1048         spin_unlock_irqrestore(&rport->lock, flags);
1049
1050         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1051                                   (lsreq->rqstlen + lsreq->rsplen),
1052                                   DMA_BIDIRECTIONAL);
1053
1054         nvme_fc_rport_put(rport);
1055 }
1056
1057 static int
1058 __nvme_fc_send_ls_req(struct nvme_fc_rport *rport,
1059                 struct nvmefc_ls_req_op *lsop,
1060                 void (*done)(struct nvmefc_ls_req *req, int status))
1061 {
1062         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1063         unsigned long flags;
1064         int ret = 0;
1065
1066         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
1067                 return -ECONNREFUSED;
1068
1069         if (!nvme_fc_rport_get(rport))
1070                 return -ESHUTDOWN;
1071
1072         lsreq->done = done;
1073         lsop->rport = rport;
1074         lsop->req_queued = false;
1075         INIT_LIST_HEAD(&lsop->lsreq_list);
1076         init_completion(&lsop->ls_done);
1077
1078         lsreq->rqstdma = fc_dma_map_single(rport->dev, lsreq->rqstaddr,
1079                                   lsreq->rqstlen + lsreq->rsplen,
1080                                   DMA_BIDIRECTIONAL);
1081         if (fc_dma_mapping_error(rport->dev, lsreq->rqstdma)) {
1082                 ret = -EFAULT;
1083                 goto out_putrport;
1084         }
1085         lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen;
1086
1087         spin_lock_irqsave(&rport->lock, flags);
1088
1089         list_add_tail(&lsop->lsreq_list, &rport->ls_req_list);
1090
1091         lsop->req_queued = true;
1092
1093         spin_unlock_irqrestore(&rport->lock, flags);
1094
1095         ret = rport->lport->ops->ls_req(&rport->lport->localport,
1096                                         &rport->remoteport, lsreq);
1097         if (ret)
1098                 goto out_unlink;
1099
1100         return 0;
1101
1102 out_unlink:
1103         lsop->ls_error = ret;
1104         spin_lock_irqsave(&rport->lock, flags);
1105         lsop->req_queued = false;
1106         list_del(&lsop->lsreq_list);
1107         spin_unlock_irqrestore(&rport->lock, flags);
1108         fc_dma_unmap_single(rport->dev, lsreq->rqstdma,
1109                                   (lsreq->rqstlen + lsreq->rsplen),
1110                                   DMA_BIDIRECTIONAL);
1111 out_putrport:
1112         nvme_fc_rport_put(rport);
1113
1114         return ret;
1115 }
1116
1117 static void
1118 nvme_fc_send_ls_req_done(struct nvmefc_ls_req *lsreq, int status)
1119 {
1120         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1121
1122         lsop->ls_error = status;
1123         complete(&lsop->ls_done);
1124 }
1125
1126 static int
1127 nvme_fc_send_ls_req(struct nvme_fc_rport *rport, struct nvmefc_ls_req_op *lsop)
1128 {
1129         struct nvmefc_ls_req *lsreq = &lsop->ls_req;
1130         struct fcnvme_ls_rjt *rjt = lsreq->rspaddr;
1131         int ret;
1132
1133         ret = __nvme_fc_send_ls_req(rport, lsop, nvme_fc_send_ls_req_done);
1134
1135         if (!ret) {
1136                 /*
1137                  * No timeout/not interruptible as we need the struct
1138                  * to exist until the lldd calls us back. Thus mandate
1139                  * wait until driver calls back. lldd responsible for
1140                  * the timeout action
1141                  */
1142                 wait_for_completion(&lsop->ls_done);
1143
1144                 __nvme_fc_finish_ls_req(lsop);
1145
1146                 ret = lsop->ls_error;
1147         }
1148
1149         if (ret)
1150                 return ret;
1151
1152         /* ACC or RJT payload ? */
1153         if (rjt->w0.ls_cmd == FCNVME_LS_RJT)
1154                 return -ENXIO;
1155
1156         return 0;
1157 }
1158
1159 static int
1160 nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport,
1161                 struct nvmefc_ls_req_op *lsop,
1162                 void (*done)(struct nvmefc_ls_req *req, int status))
1163 {
1164         /* don't wait for completion */
1165
1166         return __nvme_fc_send_ls_req(rport, lsop, done);
1167 }
1168
1169 static int
1170 nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl,
1171         struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio)
1172 {
1173         struct nvmefc_ls_req_op *lsop;
1174         struct nvmefc_ls_req *lsreq;
1175         struct fcnvme_ls_cr_assoc_rqst *assoc_rqst;
1176         struct fcnvme_ls_cr_assoc_acc *assoc_acc;
1177         unsigned long flags;
1178         int ret, fcret = 0;
1179
1180         lsop = kzalloc((sizeof(*lsop) +
1181                          sizeof(*assoc_rqst) + sizeof(*assoc_acc) +
1182                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1183         if (!lsop) {
1184                 dev_info(ctrl->ctrl.device,
1185                         "NVME-FC{%d}: send Create Association failed: ENOMEM\n",
1186                         ctrl->cnum);
1187                 ret = -ENOMEM;
1188                 goto out_no_memory;
1189         }
1190
1191         assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1];
1192         assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1];
1193         lsreq = &lsop->ls_req;
1194         if (ctrl->lport->ops->lsrqst_priv_sz)
1195                 lsreq->private = &assoc_acc[1];
1196         else
1197                 lsreq->private = NULL;
1198
1199         assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION;
1200         assoc_rqst->desc_list_len =
1201                         cpu_to_be32(sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1202
1203         assoc_rqst->assoc_cmd.desc_tag =
1204                         cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD);
1205         assoc_rqst->assoc_cmd.desc_len =
1206                         fcnvme_lsdesc_len(
1207                                 sizeof(struct fcnvme_lsdesc_cr_assoc_cmd));
1208
1209         assoc_rqst->assoc_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1210         assoc_rqst->assoc_cmd.sqsize = cpu_to_be16(qsize - 1);
1211         /* Linux supports only Dynamic controllers */
1212         assoc_rqst->assoc_cmd.cntlid = cpu_to_be16(0xffff);
1213         uuid_copy(&assoc_rqst->assoc_cmd.hostid, &ctrl->ctrl.opts->host->id);
1214         strscpy(assoc_rqst->assoc_cmd.hostnqn, ctrl->ctrl.opts->host->nqn,
1215                 sizeof(assoc_rqst->assoc_cmd.hostnqn));
1216         strscpy(assoc_rqst->assoc_cmd.subnqn, ctrl->ctrl.opts->subsysnqn,
1217                 sizeof(assoc_rqst->assoc_cmd.subnqn));
1218
1219         lsop->queue = queue;
1220         lsreq->rqstaddr = assoc_rqst;
1221         lsreq->rqstlen = sizeof(*assoc_rqst);
1222         lsreq->rspaddr = assoc_acc;
1223         lsreq->rsplen = sizeof(*assoc_acc);
1224         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1225
1226         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1227         if (ret)
1228                 goto out_free_buffer;
1229
1230         /* process connect LS completion */
1231
1232         /* validate the ACC response */
1233         if (assoc_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1234                 fcret = VERR_LSACC;
1235         else if (assoc_acc->hdr.desc_list_len !=
1236                         fcnvme_lsdesc_len(
1237                                 sizeof(struct fcnvme_ls_cr_assoc_acc)))
1238                 fcret = VERR_CR_ASSOC_ACC_LEN;
1239         else if (assoc_acc->hdr.rqst.desc_tag !=
1240                         cpu_to_be32(FCNVME_LSDESC_RQST))
1241                 fcret = VERR_LSDESC_RQST;
1242         else if (assoc_acc->hdr.rqst.desc_len !=
1243                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1244                 fcret = VERR_LSDESC_RQST_LEN;
1245         else if (assoc_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_ASSOCIATION)
1246                 fcret = VERR_CR_ASSOC;
1247         else if (assoc_acc->associd.desc_tag !=
1248                         cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1249                 fcret = VERR_ASSOC_ID;
1250         else if (assoc_acc->associd.desc_len !=
1251                         fcnvme_lsdesc_len(
1252                                 sizeof(struct fcnvme_lsdesc_assoc_id)))
1253                 fcret = VERR_ASSOC_ID_LEN;
1254         else if (assoc_acc->connectid.desc_tag !=
1255                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1256                 fcret = VERR_CONN_ID;
1257         else if (assoc_acc->connectid.desc_len !=
1258                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1259                 fcret = VERR_CONN_ID_LEN;
1260
1261         if (fcret) {
1262                 ret = -EBADF;
1263                 dev_err(ctrl->dev,
1264                         "q %d Create Association LS failed: %s\n",
1265                         queue->qnum, validation_errors[fcret]);
1266         } else {
1267                 spin_lock_irqsave(&ctrl->lock, flags);
1268                 ctrl->association_id =
1269                         be64_to_cpu(assoc_acc->associd.association_id);
1270                 queue->connection_id =
1271                         be64_to_cpu(assoc_acc->connectid.connection_id);
1272                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1273                 spin_unlock_irqrestore(&ctrl->lock, flags);
1274         }
1275
1276 out_free_buffer:
1277         kfree(lsop);
1278 out_no_memory:
1279         if (ret)
1280                 dev_err(ctrl->dev,
1281                         "queue %d connect admin queue failed (%d).\n",
1282                         queue->qnum, ret);
1283         return ret;
1284 }
1285
1286 static int
1287 nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
1288                         u16 qsize, u16 ersp_ratio)
1289 {
1290         struct nvmefc_ls_req_op *lsop;
1291         struct nvmefc_ls_req *lsreq;
1292         struct fcnvme_ls_cr_conn_rqst *conn_rqst;
1293         struct fcnvme_ls_cr_conn_acc *conn_acc;
1294         int ret, fcret = 0;
1295
1296         lsop = kzalloc((sizeof(*lsop) +
1297                          sizeof(*conn_rqst) + sizeof(*conn_acc) +
1298                          ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1299         if (!lsop) {
1300                 dev_info(ctrl->ctrl.device,
1301                         "NVME-FC{%d}: send Create Connection failed: ENOMEM\n",
1302                         ctrl->cnum);
1303                 ret = -ENOMEM;
1304                 goto out_no_memory;
1305         }
1306
1307         conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1];
1308         conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1];
1309         lsreq = &lsop->ls_req;
1310         if (ctrl->lport->ops->lsrqst_priv_sz)
1311                 lsreq->private = (void *)&conn_acc[1];
1312         else
1313                 lsreq->private = NULL;
1314
1315         conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION;
1316         conn_rqst->desc_list_len = cpu_to_be32(
1317                                 sizeof(struct fcnvme_lsdesc_assoc_id) +
1318                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1319
1320         conn_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1321         conn_rqst->associd.desc_len =
1322                         fcnvme_lsdesc_len(
1323                                 sizeof(struct fcnvme_lsdesc_assoc_id));
1324         conn_rqst->associd.association_id = cpu_to_be64(ctrl->association_id);
1325         conn_rqst->connect_cmd.desc_tag =
1326                         cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD);
1327         conn_rqst->connect_cmd.desc_len =
1328                         fcnvme_lsdesc_len(
1329                                 sizeof(struct fcnvme_lsdesc_cr_conn_cmd));
1330         conn_rqst->connect_cmd.ersp_ratio = cpu_to_be16(ersp_ratio);
1331         conn_rqst->connect_cmd.qid  = cpu_to_be16(queue->qnum);
1332         conn_rqst->connect_cmd.sqsize = cpu_to_be16(qsize - 1);
1333
1334         lsop->queue = queue;
1335         lsreq->rqstaddr = conn_rqst;
1336         lsreq->rqstlen = sizeof(*conn_rqst);
1337         lsreq->rspaddr = conn_acc;
1338         lsreq->rsplen = sizeof(*conn_acc);
1339         lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC;
1340
1341         ret = nvme_fc_send_ls_req(ctrl->rport, lsop);
1342         if (ret)
1343                 goto out_free_buffer;
1344
1345         /* process connect LS completion */
1346
1347         /* validate the ACC response */
1348         if (conn_acc->hdr.w0.ls_cmd != FCNVME_LS_ACC)
1349                 fcret = VERR_LSACC;
1350         else if (conn_acc->hdr.desc_list_len !=
1351                         fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)))
1352                 fcret = VERR_CR_CONN_ACC_LEN;
1353         else if (conn_acc->hdr.rqst.desc_tag != cpu_to_be32(FCNVME_LSDESC_RQST))
1354                 fcret = VERR_LSDESC_RQST;
1355         else if (conn_acc->hdr.rqst.desc_len !=
1356                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)))
1357                 fcret = VERR_LSDESC_RQST_LEN;
1358         else if (conn_acc->hdr.rqst.w0.ls_cmd != FCNVME_LS_CREATE_CONNECTION)
1359                 fcret = VERR_CR_CONN;
1360         else if (conn_acc->connectid.desc_tag !=
1361                         cpu_to_be32(FCNVME_LSDESC_CONN_ID))
1362                 fcret = VERR_CONN_ID;
1363         else if (conn_acc->connectid.desc_len !=
1364                         fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_conn_id)))
1365                 fcret = VERR_CONN_ID_LEN;
1366
1367         if (fcret) {
1368                 ret = -EBADF;
1369                 dev_err(ctrl->dev,
1370                         "q %d Create I/O Connection LS failed: %s\n",
1371                         queue->qnum, validation_errors[fcret]);
1372         } else {
1373                 queue->connection_id =
1374                         be64_to_cpu(conn_acc->connectid.connection_id);
1375                 set_bit(NVME_FC_Q_CONNECTED, &queue->flags);
1376         }
1377
1378 out_free_buffer:
1379         kfree(lsop);
1380 out_no_memory:
1381         if (ret)
1382                 dev_err(ctrl->dev,
1383                         "queue %d connect I/O queue failed (%d).\n",
1384                         queue->qnum, ret);
1385         return ret;
1386 }
1387
1388 static void
1389 nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status)
1390 {
1391         struct nvmefc_ls_req_op *lsop = ls_req_to_lsop(lsreq);
1392
1393         __nvme_fc_finish_ls_req(lsop);
1394
1395         /* fc-nvme initiator doesn't care about success or failure of cmd */
1396
1397         kfree(lsop);
1398 }
1399
1400 /*
1401  * This routine sends a FC-NVME LS to disconnect (aka terminate)
1402  * the FC-NVME Association.  Terminating the association also
1403  * terminates the FC-NVME connections (per queue, both admin and io
1404  * queues) that are part of the association. E.g. things are torn
1405  * down, and the related FC-NVME Association ID and Connection IDs
1406  * become invalid.
1407  *
1408  * The behavior of the fc-nvme initiator is such that it's
1409  * understanding of the association and connections will implicitly
1410  * be torn down. The action is implicit as it may be due to a loss of
1411  * connectivity with the fc-nvme target, so you may never get a
1412  * response even if you tried.  As such, the action of this routine
1413  * is to asynchronously send the LS, ignore any results of the LS, and
1414  * continue on with terminating the association. If the fc-nvme target
1415  * is present and receives the LS, it too can tear down.
1416  */
1417 static void
1418 nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl)
1419 {
1420         struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst;
1421         struct fcnvme_ls_disconnect_assoc_acc *discon_acc;
1422         struct nvmefc_ls_req_op *lsop;
1423         struct nvmefc_ls_req *lsreq;
1424         int ret;
1425
1426         lsop = kzalloc((sizeof(*lsop) +
1427                         sizeof(*discon_rqst) + sizeof(*discon_acc) +
1428                         ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL);
1429         if (!lsop) {
1430                 dev_info(ctrl->ctrl.device,
1431                         "NVME-FC{%d}: send Disconnect Association "
1432                         "failed: ENOMEM\n",
1433                         ctrl->cnum);
1434                 return;
1435         }
1436
1437         discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1];
1438         discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1];
1439         lsreq = &lsop->ls_req;
1440         if (ctrl->lport->ops->lsrqst_priv_sz)
1441                 lsreq->private = (void *)&discon_acc[1];
1442         else
1443                 lsreq->private = NULL;
1444
1445         nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc,
1446                                 ctrl->association_id);
1447
1448         ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop,
1449                                 nvme_fc_disconnect_assoc_done);
1450         if (ret)
1451                 kfree(lsop);
1452 }
1453
1454 static void
1455 nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp)
1456 {
1457         struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private;
1458         struct nvme_fc_rport *rport = lsop->rport;
1459         struct nvme_fc_lport *lport = rport->lport;
1460         unsigned long flags;
1461
1462         spin_lock_irqsave(&rport->lock, flags);
1463         list_del(&lsop->lsrcv_list);
1464         spin_unlock_irqrestore(&rport->lock, flags);
1465
1466         fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma,
1467                                 sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1468         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1469                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1470
1471         kfree(lsop->rspbuf);
1472         kfree(lsop->rqstbuf);
1473         kfree(lsop);
1474
1475         nvme_fc_rport_put(rport);
1476 }
1477
1478 static void
1479 nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop)
1480 {
1481         struct nvme_fc_rport *rport = lsop->rport;
1482         struct nvme_fc_lport *lport = rport->lport;
1483         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1484         int ret;
1485
1486         fc_dma_sync_single_for_device(lport->dev, lsop->rspdma,
1487                                   sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1488
1489         ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport,
1490                                      lsop->lsrsp);
1491         if (ret) {
1492                 dev_warn(lport->dev,
1493                         "LLDD rejected LS RSP xmt: LS %d status %d\n",
1494                         w0->ls_cmd, ret);
1495                 nvme_fc_xmt_ls_rsp_done(lsop->lsrsp);
1496                 return;
1497         }
1498 }
1499
1500 static struct nvme_fc_ctrl *
1501 nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport,
1502                       struct nvmefc_ls_rcv_op *lsop)
1503 {
1504         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1505                                         &lsop->rqstbuf->rq_dis_assoc;
1506         struct nvme_fc_ctrl *ctrl, *ret = NULL;
1507         struct nvmefc_ls_rcv_op *oldls = NULL;
1508         u64 association_id = be64_to_cpu(rqst->associd.association_id);
1509         unsigned long flags;
1510
1511         spin_lock_irqsave(&rport->lock, flags);
1512
1513         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
1514                 if (!nvme_fc_ctrl_get(ctrl))
1515                         continue;
1516                 spin_lock(&ctrl->lock);
1517                 if (association_id == ctrl->association_id) {
1518                         oldls = ctrl->rcv_disconn;
1519                         ctrl->rcv_disconn = lsop;
1520                         ret = ctrl;
1521                 }
1522                 spin_unlock(&ctrl->lock);
1523                 if (ret)
1524                         /* leave the ctrl get reference */
1525                         break;
1526                 nvme_fc_ctrl_put(ctrl);
1527         }
1528
1529         spin_unlock_irqrestore(&rport->lock, flags);
1530
1531         /* transmit a response for anything that was pending */
1532         if (oldls) {
1533                 dev_info(rport->lport->dev,
1534                         "NVME-FC{%d}: Multiple Disconnect Association "
1535                         "LS's received\n", ctrl->cnum);
1536                 /* overwrite good response with bogus failure */
1537                 oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf,
1538                                                 sizeof(*oldls->rspbuf),
1539                                                 rqst->w0.ls_cmd,
1540                                                 FCNVME_RJT_RC_UNAB,
1541                                                 FCNVME_RJT_EXP_NONE, 0);
1542                 nvme_fc_xmt_ls_rsp(oldls);
1543         }
1544
1545         return ret;
1546 }
1547
1548 /*
1549  * returns true to mean LS handled and ls_rsp can be sent
1550  * returns false to defer ls_rsp xmt (will be done as part of
1551  *     association termination)
1552  */
1553 static bool
1554 nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop)
1555 {
1556         struct nvme_fc_rport *rport = lsop->rport;
1557         struct fcnvme_ls_disconnect_assoc_rqst *rqst =
1558                                         &lsop->rqstbuf->rq_dis_assoc;
1559         struct fcnvme_ls_disconnect_assoc_acc *acc =
1560                                         &lsop->rspbuf->rsp_dis_assoc;
1561         struct nvme_fc_ctrl *ctrl = NULL;
1562         int ret = 0;
1563
1564         memset(acc, 0, sizeof(*acc));
1565
1566         ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst);
1567         if (!ret) {
1568                 /* match an active association */
1569                 ctrl = nvme_fc_match_disconn_ls(rport, lsop);
1570                 if (!ctrl)
1571                         ret = VERR_NO_ASSOC;
1572         }
1573
1574         if (ret) {
1575                 dev_info(rport->lport->dev,
1576                         "Disconnect LS failed: %s\n",
1577                         validation_errors[ret]);
1578                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc,
1579                                         sizeof(*acc), rqst->w0.ls_cmd,
1580                                         (ret == VERR_NO_ASSOC) ?
1581                                                 FCNVME_RJT_RC_INV_ASSOC :
1582                                                 FCNVME_RJT_RC_LOGIC,
1583                                         FCNVME_RJT_EXP_NONE, 0);
1584                 return true;
1585         }
1586
1587         /* format an ACCept response */
1588
1589         lsop->lsrsp->rsplen = sizeof(*acc);
1590
1591         nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1592                         fcnvme_lsdesc_len(
1593                                 sizeof(struct fcnvme_ls_disconnect_assoc_acc)),
1594                         FCNVME_LS_DISCONNECT_ASSOC);
1595
1596         /*
1597          * the transmit of the response will occur after the exchanges
1598          * for the association have been ABTS'd by
1599          * nvme_fc_delete_association().
1600          */
1601
1602         /* fail the association */
1603         nvme_fc_error_recovery(ctrl, "Disconnect Association LS received");
1604
1605         /* release the reference taken by nvme_fc_match_disconn_ls() */
1606         nvme_fc_ctrl_put(ctrl);
1607
1608         return false;
1609 }
1610
1611 /*
1612  * Actual Processing routine for received FC-NVME LS Requests from the LLD
1613  * returns true if a response should be sent afterward, false if rsp will
1614  * be sent asynchronously.
1615  */
1616 static bool
1617 nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop)
1618 {
1619         struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0;
1620         bool ret = true;
1621
1622         lsop->lsrsp->nvme_fc_private = lsop;
1623         lsop->lsrsp->rspbuf = lsop->rspbuf;
1624         lsop->lsrsp->rspdma = lsop->rspdma;
1625         lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done;
1626         /* Be preventative. handlers will later set to valid length */
1627         lsop->lsrsp->rsplen = 0;
1628
1629         /*
1630          * handlers:
1631          *   parse request input, execute the request, and format the
1632          *   LS response
1633          */
1634         switch (w0->ls_cmd) {
1635         case FCNVME_LS_DISCONNECT_ASSOC:
1636                 ret = nvme_fc_ls_disconnect_assoc(lsop);
1637                 break;
1638         case FCNVME_LS_DISCONNECT_CONN:
1639                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1640                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1641                                 FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0);
1642                 break;
1643         case FCNVME_LS_CREATE_ASSOCIATION:
1644         case FCNVME_LS_CREATE_CONNECTION:
1645                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1646                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1647                                 FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0);
1648                 break;
1649         default:
1650                 lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf,
1651                                 sizeof(*lsop->rspbuf), w0->ls_cmd,
1652                                 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1653                 break;
1654         }
1655
1656         return(ret);
1657 }
1658
1659 static void
1660 nvme_fc_handle_ls_rqst_work(struct work_struct *work)
1661 {
1662         struct nvme_fc_rport *rport =
1663                 container_of(work, struct nvme_fc_rport, lsrcv_work);
1664         struct fcnvme_ls_rqst_w0 *w0;
1665         struct nvmefc_ls_rcv_op *lsop;
1666         unsigned long flags;
1667         bool sendrsp;
1668
1669 restart:
1670         sendrsp = true;
1671         spin_lock_irqsave(&rport->lock, flags);
1672         list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) {
1673                 if (lsop->handled)
1674                         continue;
1675
1676                 lsop->handled = true;
1677                 if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
1678                         spin_unlock_irqrestore(&rport->lock, flags);
1679                         sendrsp = nvme_fc_handle_ls_rqst(lsop);
1680                 } else {
1681                         spin_unlock_irqrestore(&rport->lock, flags);
1682                         w0 = &lsop->rqstbuf->w0;
1683                         lsop->lsrsp->rsplen = nvme_fc_format_rjt(
1684                                                 lsop->rspbuf,
1685                                                 sizeof(*lsop->rspbuf),
1686                                                 w0->ls_cmd,
1687                                                 FCNVME_RJT_RC_UNAB,
1688                                                 FCNVME_RJT_EXP_NONE, 0);
1689                 }
1690                 if (sendrsp)
1691                         nvme_fc_xmt_ls_rsp(lsop);
1692                 goto restart;
1693         }
1694         spin_unlock_irqrestore(&rport->lock, flags);
1695 }
1696
1697 static
1698 void nvme_fc_rcv_ls_req_err_msg(struct nvme_fc_lport *lport,
1699                                 struct fcnvme_ls_rqst_w0 *w0)
1700 {
1701         dev_info(lport->dev, "RCV %s LS failed: No memory\n",
1702                 (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1703                         nvmefc_ls_names[w0->ls_cmd] : "");
1704 }
1705
1706 /**
1707  * nvme_fc_rcv_ls_req - transport entry point called by an LLDD
1708  *                       upon the reception of a NVME LS request.
1709  *
1710  * The nvme-fc layer will copy payload to an internal structure for
1711  * processing.  As such, upon completion of the routine, the LLDD may
1712  * immediately free/reuse the LS request buffer passed in the call.
1713  *
1714  * If this routine returns error, the LLDD should abort the exchange.
1715  *
1716  * @portptr:    pointer to the (registered) remote port that the LS
1717  *              was received from. The remoteport is associated with
1718  *              a specific localport.
1719  * @lsrsp:      pointer to a nvmefc_ls_rsp response structure to be
1720  *              used to reference the exchange corresponding to the LS
1721  *              when issuing an ls response.
1722  * @lsreqbuf:   pointer to the buffer containing the LS Request
1723  * @lsreqbuf_len: length, in bytes, of the received LS request
1724  */
1725 int
1726 nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr,
1727                         struct nvmefc_ls_rsp *lsrsp,
1728                         void *lsreqbuf, u32 lsreqbuf_len)
1729 {
1730         struct nvme_fc_rport *rport = remoteport_to_rport(portptr);
1731         struct nvme_fc_lport *lport = rport->lport;
1732         struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf;
1733         struct nvmefc_ls_rcv_op *lsop;
1734         unsigned long flags;
1735         int ret;
1736
1737         nvme_fc_rport_get(rport);
1738
1739         /* validate there's a routine to transmit a response */
1740         if (!lport->ops->xmt_ls_rsp) {
1741                 dev_info(lport->dev,
1742                         "RCV %s LS failed: no LLDD xmt_ls_rsp\n",
1743                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1744                                 nvmefc_ls_names[w0->ls_cmd] : "");
1745                 ret = -EINVAL;
1746                 goto out_put;
1747         }
1748
1749         if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) {
1750                 dev_info(lport->dev,
1751                         "RCV %s LS failed: payload too large\n",
1752                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1753                                 nvmefc_ls_names[w0->ls_cmd] : "");
1754                 ret = -E2BIG;
1755                 goto out_put;
1756         }
1757
1758         lsop = kzalloc(sizeof(*lsop), GFP_KERNEL);
1759         if (!lsop) {
1760                 nvme_fc_rcv_ls_req_err_msg(lport, w0);
1761                 ret = -ENOMEM;
1762                 goto out_put;
1763         }
1764
1765         lsop->rqstbuf = kzalloc(sizeof(*lsop->rqstbuf), GFP_KERNEL);
1766         lsop->rspbuf = kzalloc(sizeof(*lsop->rspbuf), GFP_KERNEL);
1767         if (!lsop->rqstbuf || !lsop->rspbuf) {
1768                 nvme_fc_rcv_ls_req_err_msg(lport, w0);
1769                 ret = -ENOMEM;
1770                 goto out_free;
1771         }
1772
1773         lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf,
1774                                         sizeof(*lsop->rspbuf),
1775                                         DMA_TO_DEVICE);
1776         if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) {
1777                 dev_info(lport->dev,
1778                         "RCV %s LS failed: DMA mapping failure\n",
1779                         (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ?
1780                                 nvmefc_ls_names[w0->ls_cmd] : "");
1781                 ret = -EFAULT;
1782                 goto out_free;
1783         }
1784
1785         lsop->rport = rport;
1786         lsop->lsrsp = lsrsp;
1787
1788         memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len);
1789         lsop->rqstdatalen = lsreqbuf_len;
1790
1791         spin_lock_irqsave(&rport->lock, flags);
1792         if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) {
1793                 spin_unlock_irqrestore(&rport->lock, flags);
1794                 ret = -ENOTCONN;
1795                 goto out_unmap;
1796         }
1797         list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list);
1798         spin_unlock_irqrestore(&rport->lock, flags);
1799
1800         schedule_work(&rport->lsrcv_work);
1801
1802         return 0;
1803
1804 out_unmap:
1805         fc_dma_unmap_single(lport->dev, lsop->rspdma,
1806                         sizeof(*lsop->rspbuf), DMA_TO_DEVICE);
1807 out_free:
1808         kfree(lsop->rspbuf);
1809         kfree(lsop->rqstbuf);
1810         kfree(lsop);
1811 out_put:
1812         nvme_fc_rport_put(rport);
1813         return ret;
1814 }
1815 EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req);
1816
1817
1818 /* *********************** NVME Ctrl Routines **************************** */
1819
1820 static void
1821 __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl,
1822                 struct nvme_fc_fcp_op *op)
1823 {
1824         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.rspdma,
1825                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1826         fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma,
1827                                 sizeof(op->cmd_iu), DMA_TO_DEVICE);
1828
1829         atomic_set(&op->state, FCPOP_STATE_UNINIT);
1830 }
1831
1832 static void
1833 nvme_fc_exit_request(struct blk_mq_tag_set *set, struct request *rq,
1834                 unsigned int hctx_idx)
1835 {
1836         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
1837
1838         return __nvme_fc_exit_request(to_fc_ctrl(set->driver_data), op);
1839 }
1840
1841 static int
1842 __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op)
1843 {
1844         unsigned long flags;
1845         int opstate;
1846
1847         spin_lock_irqsave(&ctrl->lock, flags);
1848         opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED);
1849         if (opstate != FCPOP_STATE_ACTIVE)
1850                 atomic_set(&op->state, opstate);
1851         else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) {
1852                 op->flags |= FCOP_FLAGS_TERMIO;
1853                 ctrl->iocnt++;
1854         }
1855         spin_unlock_irqrestore(&ctrl->lock, flags);
1856
1857         if (opstate != FCPOP_STATE_ACTIVE)
1858                 return -ECANCELED;
1859
1860         ctrl->lport->ops->fcp_abort(&ctrl->lport->localport,
1861                                         &ctrl->rport->remoteport,
1862                                         op->queue->lldd_handle,
1863                                         &op->fcp_req);
1864
1865         return 0;
1866 }
1867
1868 static void
1869 nvme_fc_abort_aen_ops(struct nvme_fc_ctrl *ctrl)
1870 {
1871         struct nvme_fc_fcp_op *aen_op = ctrl->aen_ops;
1872         int i;
1873
1874         /* ensure we've initialized the ops once */
1875         if (!(aen_op->flags & FCOP_FLAGS_AEN))
1876                 return;
1877
1878         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++)
1879                 __nvme_fc_abort_op(ctrl, aen_op);
1880 }
1881
1882 static inline void
1883 __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
1884                 struct nvme_fc_fcp_op *op, int opstate)
1885 {
1886         unsigned long flags;
1887
1888         if (opstate == FCPOP_STATE_ABORTED) {
1889                 spin_lock_irqsave(&ctrl->lock, flags);
1890                 if (test_bit(FCCTRL_TERMIO, &ctrl->flags) &&
1891                     op->flags & FCOP_FLAGS_TERMIO) {
1892                         if (!--ctrl->iocnt)
1893                                 wake_up(&ctrl->ioabort_wait);
1894                 }
1895                 spin_unlock_irqrestore(&ctrl->lock, flags);
1896         }
1897 }
1898
1899 static void
1900 nvme_fc_ctrl_ioerr_work(struct work_struct *work)
1901 {
1902         struct nvme_fc_ctrl *ctrl =
1903                         container_of(work, struct nvme_fc_ctrl, ioerr_work);
1904
1905         nvme_fc_error_recovery(ctrl, "transport detected io error");
1906 }
1907
1908 /*
1909  * nvme_fc_io_getuuid - Routine called to get the appid field
1910  * associated with request by the lldd
1911  * @req:IO request from nvme fc to driver
1912  * Returns: UUID if there is an appid associated with VM or
1913  * NULL if the user/libvirt has not set the appid to VM
1914  */
1915 char *nvme_fc_io_getuuid(struct nvmefc_fcp_req *req)
1916 {
1917         struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1918         struct request *rq = op->rq;
1919
1920         if (!IS_ENABLED(CONFIG_BLK_CGROUP_FC_APPID) || !rq || !rq->bio)
1921                 return NULL;
1922         return blkcg_get_fc_appid(rq->bio);
1923 }
1924 EXPORT_SYMBOL_GPL(nvme_fc_io_getuuid);
1925
1926 static void
1927 nvme_fc_fcpio_done(struct nvmefc_fcp_req *req)
1928 {
1929         struct nvme_fc_fcp_op *op = fcp_req_to_fcp_op(req);
1930         struct request *rq = op->rq;
1931         struct nvmefc_fcp_req *freq = &op->fcp_req;
1932         struct nvme_fc_ctrl *ctrl = op->ctrl;
1933         struct nvme_fc_queue *queue = op->queue;
1934         struct nvme_completion *cqe = &op->rsp_iu.cqe;
1935         struct nvme_command *sqe = &op->cmd_iu.sqe;
1936         __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1);
1937         union nvme_result result;
1938         bool terminate_assoc = true;
1939         int opstate;
1940
1941         /*
1942          * WARNING:
1943          * The current linux implementation of a nvme controller
1944          * allocates a single tag set for all io queues and sizes
1945          * the io queues to fully hold all possible tags. Thus, the
1946          * implementation does not reference or care about the sqhd
1947          * value as it never needs to use the sqhd/sqtail pointers
1948          * for submission pacing.
1949          *
1950          * This affects the FC-NVME implementation in two ways:
1951          * 1) As the value doesn't matter, we don't need to waste
1952          *    cycles extracting it from ERSPs and stamping it in the
1953          *    cases where the transport fabricates CQEs on successful
1954          *    completions.
1955          * 2) The FC-NVME implementation requires that delivery of
1956          *    ERSP completions are to go back to the nvme layer in order
1957          *    relative to the rsn, such that the sqhd value will always
1958          *    be "in order" for the nvme layer. As the nvme layer in
1959          *    linux doesn't care about sqhd, there's no need to return
1960          *    them in order.
1961          *
1962          * Additionally:
1963          * As the core nvme layer in linux currently does not look at
1964          * every field in the cqe - in cases where the FC transport must
1965          * fabricate a CQE, the following fields will not be set as they
1966          * are not referenced:
1967          *      cqe.sqid,  cqe.sqhd,  cqe.command_id
1968          *
1969          * Failure or error of an individual i/o, in a transport
1970          * detected fashion unrelated to the nvme completion status,
1971          * potentially cause the initiator and target sides to get out
1972          * of sync on SQ head/tail (aka outstanding io count allowed).
1973          * Per FC-NVME spec, failure of an individual command requires
1974          * the connection to be terminated, which in turn requires the
1975          * association to be terminated.
1976          */
1977
1978         opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
1979
1980         fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma,
1981                                 sizeof(op->rsp_iu), DMA_FROM_DEVICE);
1982
1983         if (opstate == FCPOP_STATE_ABORTED)
1984                 status = cpu_to_le16(NVME_SC_HOST_ABORTED_CMD << 1);
1985         else if (freq->status) {
1986                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
1987                 dev_info(ctrl->ctrl.device,
1988                         "NVME-FC{%d}: io failed due to lldd error %d\n",
1989                         ctrl->cnum, freq->status);
1990         }
1991
1992         /*
1993          * For the linux implementation, if we have an unsuccesful
1994          * status, they blk-mq layer can typically be called with the
1995          * non-zero status and the content of the cqe isn't important.
1996          */
1997         if (status)
1998                 goto done;
1999
2000         /*
2001          * command completed successfully relative to the wire
2002          * protocol. However, validate anything received and
2003          * extract the status and result from the cqe (create it
2004          * where necessary).
2005          */
2006
2007         switch (freq->rcv_rsplen) {
2008
2009         case 0:
2010         case NVME_FC_SIZEOF_ZEROS_RSP:
2011                 /*
2012                  * No response payload or 12 bytes of payload (which
2013                  * should all be zeros) are considered successful and
2014                  * no payload in the CQE by the transport.
2015                  */
2016                 if (freq->transferred_length !=
2017                     be32_to_cpu(op->cmd_iu.data_len)) {
2018                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2019                         dev_info(ctrl->ctrl.device,
2020                                 "NVME-FC{%d}: io failed due to bad transfer "
2021                                 "length: %d vs expected %d\n",
2022                                 ctrl->cnum, freq->transferred_length,
2023                                 be32_to_cpu(op->cmd_iu.data_len));
2024                         goto done;
2025                 }
2026                 result.u64 = 0;
2027                 break;
2028
2029         case sizeof(struct nvme_fc_ersp_iu):
2030                 /*
2031                  * The ERSP IU contains a full completion with CQE.
2032                  * Validate ERSP IU and look at cqe.
2033                  */
2034                 if (unlikely(be16_to_cpu(op->rsp_iu.iu_len) !=
2035                                         (freq->rcv_rsplen / 4) ||
2036                              be32_to_cpu(op->rsp_iu.xfrd_len) !=
2037                                         freq->transferred_length ||
2038                              op->rsp_iu.ersp_result ||
2039                              sqe->common.command_id != cqe->command_id)) {
2040                         status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2041                         dev_info(ctrl->ctrl.device,
2042                                 "NVME-FC{%d}: io failed due to bad NVMe_ERSP: "
2043                                 "iu len %d, xfr len %d vs %d, status code "
2044                                 "%d, cmdid %d vs %d\n",
2045                                 ctrl->cnum, be16_to_cpu(op->rsp_iu.iu_len),
2046                                 be32_to_cpu(op->rsp_iu.xfrd_len),
2047                                 freq->transferred_length,
2048                                 op->rsp_iu.ersp_result,
2049                                 sqe->common.command_id,
2050                                 cqe->command_id);
2051                         goto done;
2052                 }
2053                 result = cqe->result;
2054                 status = cqe->status;
2055                 break;
2056
2057         default:
2058                 status = cpu_to_le16(NVME_SC_HOST_PATH_ERROR << 1);
2059                 dev_info(ctrl->ctrl.device,
2060                         "NVME-FC{%d}: io failed due to odd NVMe_xRSP iu "
2061                         "len %d\n",
2062                         ctrl->cnum, freq->rcv_rsplen);
2063                 goto done;
2064         }
2065
2066         terminate_assoc = false;
2067
2068 done:
2069         if (op->flags & FCOP_FLAGS_AEN) {
2070                 nvme_complete_async_event(&queue->ctrl->ctrl, status, &result);
2071                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2072                 atomic_set(&op->state, FCPOP_STATE_IDLE);
2073                 op->flags = FCOP_FLAGS_AEN;     /* clear other flags */
2074                 nvme_fc_ctrl_put(ctrl);
2075                 goto check_error;
2076         }
2077
2078         __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2079         if (!nvme_try_complete_req(rq, status, result))
2080                 nvme_fc_complete_rq(rq);
2081
2082 check_error:
2083         if (terminate_assoc && ctrl->ctrl.state != NVME_CTRL_RESETTING)
2084                 queue_work(nvme_reset_wq, &ctrl->ioerr_work);
2085 }
2086
2087 static int
2088 __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl,
2089                 struct nvme_fc_queue *queue, struct nvme_fc_fcp_op *op,
2090                 struct request *rq, u32 rqno)
2091 {
2092         struct nvme_fcp_op_w_sgl *op_w_sgl =
2093                 container_of(op, typeof(*op_w_sgl), op);
2094         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2095         int ret = 0;
2096
2097         memset(op, 0, sizeof(*op));
2098         op->fcp_req.cmdaddr = &op->cmd_iu;
2099         op->fcp_req.cmdlen = sizeof(op->cmd_iu);
2100         op->fcp_req.rspaddr = &op->rsp_iu;
2101         op->fcp_req.rsplen = sizeof(op->rsp_iu);
2102         op->fcp_req.done = nvme_fc_fcpio_done;
2103         op->ctrl = ctrl;
2104         op->queue = queue;
2105         op->rq = rq;
2106         op->rqno = rqno;
2107
2108         cmdiu->format_id = NVME_CMD_FORMAT_ID;
2109         cmdiu->fc_id = NVME_CMD_FC_ID;
2110         cmdiu->iu_len = cpu_to_be16(sizeof(*cmdiu) / sizeof(u32));
2111         if (queue->qnum)
2112                 cmdiu->rsv_cat = fccmnd_set_cat_css(0,
2113                                         (NVME_CC_CSS_NVM >> NVME_CC_CSS_SHIFT));
2114         else
2115                 cmdiu->rsv_cat = fccmnd_set_cat_admin(0);
2116
2117         op->fcp_req.cmddma = fc_dma_map_single(ctrl->lport->dev,
2118                                 &op->cmd_iu, sizeof(op->cmd_iu), DMA_TO_DEVICE);
2119         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.cmddma)) {
2120                 dev_err(ctrl->dev,
2121                         "FCP Op failed - cmdiu dma mapping failed.\n");
2122                 ret = -EFAULT;
2123                 goto out_on_error;
2124         }
2125
2126         op->fcp_req.rspdma = fc_dma_map_single(ctrl->lport->dev,
2127                                 &op->rsp_iu, sizeof(op->rsp_iu),
2128                                 DMA_FROM_DEVICE);
2129         if (fc_dma_mapping_error(ctrl->lport->dev, op->fcp_req.rspdma)) {
2130                 dev_err(ctrl->dev,
2131                         "FCP Op failed - rspiu dma mapping failed.\n");
2132                 ret = -EFAULT;
2133         }
2134
2135         atomic_set(&op->state, FCPOP_STATE_IDLE);
2136 out_on_error:
2137         return ret;
2138 }
2139
2140 static int
2141 nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq,
2142                 unsigned int hctx_idx, unsigned int numa_node)
2143 {
2144         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2145         struct nvme_fcp_op_w_sgl *op = blk_mq_rq_to_pdu(rq);
2146         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
2147         struct nvme_fc_queue *queue = &ctrl->queues[queue_idx];
2148         int res;
2149
2150         res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++);
2151         if (res)
2152                 return res;
2153         op->op.fcp_req.first_sgl = op->sgl;
2154         op->op.fcp_req.private = &op->priv[0];
2155         nvme_req(rq)->ctrl = &ctrl->ctrl;
2156         nvme_req(rq)->cmd = &op->op.cmd_iu.sqe;
2157         return res;
2158 }
2159
2160 static int
2161 nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl)
2162 {
2163         struct nvme_fc_fcp_op *aen_op;
2164         struct nvme_fc_cmd_iu *cmdiu;
2165         struct nvme_command *sqe;
2166         void *private = NULL;
2167         int i, ret;
2168
2169         aen_op = ctrl->aen_ops;
2170         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2171                 if (ctrl->lport->ops->fcprqst_priv_sz) {
2172                         private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz,
2173                                                 GFP_KERNEL);
2174                         if (!private)
2175                                 return -ENOMEM;
2176                 }
2177
2178                 cmdiu = &aen_op->cmd_iu;
2179                 sqe = &cmdiu->sqe;
2180                 ret = __nvme_fc_init_request(ctrl, &ctrl->queues[0],
2181                                 aen_op, (struct request *)NULL,
2182                                 (NVME_AQ_BLK_MQ_DEPTH + i));
2183                 if (ret) {
2184                         kfree(private);
2185                         return ret;
2186                 }
2187
2188                 aen_op->flags = FCOP_FLAGS_AEN;
2189                 aen_op->fcp_req.private = private;
2190
2191                 memset(sqe, 0, sizeof(*sqe));
2192                 sqe->common.opcode = nvme_admin_async_event;
2193                 /* Note: core layer may overwrite the sqe.command_id value */
2194                 sqe->common.command_id = NVME_AQ_BLK_MQ_DEPTH + i;
2195         }
2196         return 0;
2197 }
2198
2199 static void
2200 nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl)
2201 {
2202         struct nvme_fc_fcp_op *aen_op;
2203         int i;
2204
2205         cancel_work_sync(&ctrl->ctrl.async_event_work);
2206         aen_op = ctrl->aen_ops;
2207         for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) {
2208                 __nvme_fc_exit_request(ctrl, aen_op);
2209
2210                 kfree(aen_op->fcp_req.private);
2211                 aen_op->fcp_req.private = NULL;
2212         }
2213 }
2214
2215 static inline int
2216 __nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int qidx)
2217 {
2218         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(data);
2219         struct nvme_fc_queue *queue = &ctrl->queues[qidx];
2220
2221         hctx->driver_data = queue;
2222         queue->hctx = hctx;
2223         return 0;
2224 }
2225
2226 static int
2227 nvme_fc_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, unsigned int hctx_idx)
2228 {
2229         return __nvme_fc_init_hctx(hctx, data, hctx_idx + 1);
2230 }
2231
2232 static int
2233 nvme_fc_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2234                 unsigned int hctx_idx)
2235 {
2236         return __nvme_fc_init_hctx(hctx, data, hctx_idx);
2237 }
2238
2239 static void
2240 nvme_fc_init_queue(struct nvme_fc_ctrl *ctrl, int idx)
2241 {
2242         struct nvme_fc_queue *queue;
2243
2244         queue = &ctrl->queues[idx];
2245         memset(queue, 0, sizeof(*queue));
2246         queue->ctrl = ctrl;
2247         queue->qnum = idx;
2248         atomic_set(&queue->csn, 0);
2249         queue->dev = ctrl->dev;
2250
2251         if (idx > 0)
2252                 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
2253         else
2254                 queue->cmnd_capsule_len = sizeof(struct nvme_command);
2255
2256         /*
2257          * Considered whether we should allocate buffers for all SQEs
2258          * and CQEs and dma map them - mapping their respective entries
2259          * into the request structures (kernel vm addr and dma address)
2260          * thus the driver could use the buffers/mappings directly.
2261          * It only makes sense if the LLDD would use them for its
2262          * messaging api. It's very unlikely most adapter api's would use
2263          * a native NVME sqe/cqe. More reasonable if FC-NVME IU payload
2264          * structures were used instead.
2265          */
2266 }
2267
2268 /*
2269  * This routine terminates a queue at the transport level.
2270  * The transport has already ensured that all outstanding ios on
2271  * the queue have been terminated.
2272  * The transport will send a Disconnect LS request to terminate
2273  * the queue's connection. Termination of the admin queue will also
2274  * terminate the association at the target.
2275  */
2276 static void
2277 nvme_fc_free_queue(struct nvme_fc_queue *queue)
2278 {
2279         if (!test_and_clear_bit(NVME_FC_Q_CONNECTED, &queue->flags))
2280                 return;
2281
2282         clear_bit(NVME_FC_Q_LIVE, &queue->flags);
2283         /*
2284          * Current implementation never disconnects a single queue.
2285          * It always terminates a whole association. So there is never
2286          * a disconnect(queue) LS sent to the target.
2287          */
2288
2289         queue->connection_id = 0;
2290         atomic_set(&queue->csn, 0);
2291 }
2292
2293 static void
2294 __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *ctrl,
2295         struct nvme_fc_queue *queue, unsigned int qidx)
2296 {
2297         if (ctrl->lport->ops->delete_queue)
2298                 ctrl->lport->ops->delete_queue(&ctrl->lport->localport, qidx,
2299                                 queue->lldd_handle);
2300         queue->lldd_handle = NULL;
2301 }
2302
2303 static void
2304 nvme_fc_free_io_queues(struct nvme_fc_ctrl *ctrl)
2305 {
2306         int i;
2307
2308         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2309                 nvme_fc_free_queue(&ctrl->queues[i]);
2310 }
2311
2312 static int
2313 __nvme_fc_create_hw_queue(struct nvme_fc_ctrl *ctrl,
2314         struct nvme_fc_queue *queue, unsigned int qidx, u16 qsize)
2315 {
2316         int ret = 0;
2317
2318         queue->lldd_handle = NULL;
2319         if (ctrl->lport->ops->create_queue)
2320                 ret = ctrl->lport->ops->create_queue(&ctrl->lport->localport,
2321                                 qidx, qsize, &queue->lldd_handle);
2322
2323         return ret;
2324 }
2325
2326 static void
2327 nvme_fc_delete_hw_io_queues(struct nvme_fc_ctrl *ctrl)
2328 {
2329         struct nvme_fc_queue *queue = &ctrl->queues[ctrl->ctrl.queue_count - 1];
2330         int i;
2331
2332         for (i = ctrl->ctrl.queue_count - 1; i >= 1; i--, queue--)
2333                 __nvme_fc_delete_hw_queue(ctrl, queue, i);
2334 }
2335
2336 static int
2337 nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2338 {
2339         struct nvme_fc_queue *queue = &ctrl->queues[1];
2340         int i, ret;
2341
2342         for (i = 1; i < ctrl->ctrl.queue_count; i++, queue++) {
2343                 ret = __nvme_fc_create_hw_queue(ctrl, queue, i, qsize);
2344                 if (ret)
2345                         goto delete_queues;
2346         }
2347
2348         return 0;
2349
2350 delete_queues:
2351         for (; i > 0; i--)
2352                 __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i);
2353         return ret;
2354 }
2355
2356 static int
2357 nvme_fc_connect_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize)
2358 {
2359         int i, ret = 0;
2360
2361         for (i = 1; i < ctrl->ctrl.queue_count; i++) {
2362                 ret = nvme_fc_connect_queue(ctrl, &ctrl->queues[i], qsize,
2363                                         (qsize / 5));
2364                 if (ret)
2365                         break;
2366                 ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
2367                 if (ret)
2368                         break;
2369
2370                 set_bit(NVME_FC_Q_LIVE, &ctrl->queues[i].flags);
2371         }
2372
2373         return ret;
2374 }
2375
2376 static void
2377 nvme_fc_init_io_queues(struct nvme_fc_ctrl *ctrl)
2378 {
2379         int i;
2380
2381         for (i = 1; i < ctrl->ctrl.queue_count; i++)
2382                 nvme_fc_init_queue(ctrl, i);
2383 }
2384
2385 static void
2386 nvme_fc_ctrl_free(struct kref *ref)
2387 {
2388         struct nvme_fc_ctrl *ctrl =
2389                 container_of(ref, struct nvme_fc_ctrl, ref);
2390         unsigned long flags;
2391
2392         if (ctrl->ctrl.tagset)
2393                 nvme_remove_io_tag_set(&ctrl->ctrl);
2394
2395         /* remove from rport list */
2396         spin_lock_irqsave(&ctrl->rport->lock, flags);
2397         list_del(&ctrl->ctrl_list);
2398         spin_unlock_irqrestore(&ctrl->rport->lock, flags);
2399
2400         nvme_unquiesce_admin_queue(&ctrl->ctrl);
2401         nvme_remove_admin_tag_set(&ctrl->ctrl);
2402
2403         kfree(ctrl->queues);
2404
2405         put_device(ctrl->dev);
2406         nvme_fc_rport_put(ctrl->rport);
2407
2408         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
2409         if (ctrl->ctrl.opts)
2410                 nvmf_free_options(ctrl->ctrl.opts);
2411         kfree(ctrl);
2412 }
2413
2414 static void
2415 nvme_fc_ctrl_put(struct nvme_fc_ctrl *ctrl)
2416 {
2417         kref_put(&ctrl->ref, nvme_fc_ctrl_free);
2418 }
2419
2420 static int
2421 nvme_fc_ctrl_get(struct nvme_fc_ctrl *ctrl)
2422 {
2423         return kref_get_unless_zero(&ctrl->ref);
2424 }
2425
2426 /*
2427  * All accesses from nvme core layer done - can now free the
2428  * controller. Called after last nvme_put_ctrl() call
2429  */
2430 static void
2431 nvme_fc_nvme_ctrl_freed(struct nvme_ctrl *nctrl)
2432 {
2433         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2434
2435         WARN_ON(nctrl != &ctrl->ctrl);
2436
2437         nvme_fc_ctrl_put(ctrl);
2438 }
2439
2440 /*
2441  * This routine is used by the transport when it needs to find active
2442  * io on a queue that is to be terminated. The transport uses
2443  * blk_mq_tagset_busy_itr() to find the busy requests, which then invoke
2444  * this routine to kill them on a 1 by 1 basis.
2445  *
2446  * As FC allocates FC exchange for each io, the transport must contact
2447  * the LLDD to terminate the exchange, thus releasing the FC exchange.
2448  * After terminating the exchange the LLDD will call the transport's
2449  * normal io done path for the request, but it will have an aborted
2450  * status. The done path will return the io request back to the block
2451  * layer with an error status.
2452  */
2453 static bool nvme_fc_terminate_exchange(struct request *req, void *data)
2454 {
2455         struct nvme_ctrl *nctrl = data;
2456         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
2457         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(req);
2458
2459         op->nreq.flags |= NVME_REQ_CANCELLED;
2460         __nvme_fc_abort_op(ctrl, op);
2461         return true;
2462 }
2463
2464 /*
2465  * This routine runs through all outstanding commands on the association
2466  * and aborts them.  This routine is typically be called by the
2467  * delete_association routine. It is also called due to an error during
2468  * reconnect. In that scenario, it is most likely a command that initializes
2469  * the controller, including fabric Connect commands on io queues, that
2470  * may have timed out or failed thus the io must be killed for the connect
2471  * thread to see the error.
2472  */
2473 static void
2474 __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
2475 {
2476         int q;
2477
2478         /*
2479          * if aborting io, the queues are no longer good, mark them
2480          * all as not live.
2481          */
2482         if (ctrl->ctrl.queue_count > 1) {
2483                 for (q = 1; q < ctrl->ctrl.queue_count; q++)
2484                         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[q].flags);
2485         }
2486         clear_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
2487
2488         /*
2489          * If io queues are present, stop them and terminate all outstanding
2490          * ios on them. As FC allocates FC exchange for each io, the
2491          * transport must contact the LLDD to terminate the exchange,
2492          * thus releasing the FC exchange. We use blk_mq_tagset_busy_itr()
2493          * to tell us what io's are busy and invoke a transport routine
2494          * to kill them with the LLDD.  After terminating the exchange
2495          * the LLDD will call the transport's normal io done path, but it
2496          * will have an aborted status. The done path will return the
2497          * io requests back to the block layer as part of normal completions
2498          * (but with error status).
2499          */
2500         if (ctrl->ctrl.queue_count > 1) {
2501                 nvme_quiesce_io_queues(&ctrl->ctrl);
2502                 nvme_sync_io_queues(&ctrl->ctrl);
2503                 blk_mq_tagset_busy_iter(&ctrl->tag_set,
2504                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2505                 blk_mq_tagset_wait_completed_request(&ctrl->tag_set);
2506                 if (start_queues)
2507                         nvme_unquiesce_io_queues(&ctrl->ctrl);
2508         }
2509
2510         /*
2511          * Other transports, which don't have link-level contexts bound
2512          * to sqe's, would try to gracefully shutdown the controller by
2513          * writing the registers for shutdown and polling (call
2514          * nvme_disable_ctrl()). Given a bunch of i/o was potentially
2515          * just aborted and we will wait on those contexts, and given
2516          * there was no indication of how live the controlelr is on the
2517          * link, don't send more io to create more contexts for the
2518          * shutdown. Let the controller fail via keepalive failure if
2519          * its still present.
2520          */
2521
2522         /*
2523          * clean up the admin queue. Same thing as above.
2524          */
2525         nvme_quiesce_admin_queue(&ctrl->ctrl);
2526         blk_sync_queue(ctrl->ctrl.admin_q);
2527         blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
2528                                 nvme_fc_terminate_exchange, &ctrl->ctrl);
2529         blk_mq_tagset_wait_completed_request(&ctrl->admin_tag_set);
2530         if (start_queues)
2531                 nvme_unquiesce_admin_queue(&ctrl->ctrl);
2532 }
2533
2534 static void
2535 nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
2536 {
2537         /*
2538          * if an error (io timeout, etc) while (re)connecting, the remote
2539          * port requested terminating of the association (disconnect_ls)
2540          * or an error (timeout or abort) occurred on an io while creating
2541          * the controller.  Abort any ios on the association and let the
2542          * create_association error path resolve things.
2543          */
2544         if (ctrl->ctrl.state == NVME_CTRL_CONNECTING) {
2545                 __nvme_fc_abort_outstanding_ios(ctrl, true);
2546                 set_bit(ASSOC_FAILED, &ctrl->flags);
2547                 dev_warn(ctrl->ctrl.device,
2548                         "NVME-FC{%d}: transport error during (re)connect\n",
2549                         ctrl->cnum);
2550                 return;
2551         }
2552
2553         /* Otherwise, only proceed if in LIVE state - e.g. on first error */
2554         if (ctrl->ctrl.state != NVME_CTRL_LIVE)
2555                 return;
2556
2557         dev_warn(ctrl->ctrl.device,
2558                 "NVME-FC{%d}: transport association event: %s\n",
2559                 ctrl->cnum, errmsg);
2560         dev_warn(ctrl->ctrl.device,
2561                 "NVME-FC{%d}: resetting controller\n", ctrl->cnum);
2562
2563         nvme_reset_ctrl(&ctrl->ctrl);
2564 }
2565
2566 static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
2567 {
2568         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2569         struct nvme_fc_ctrl *ctrl = op->ctrl;
2570         u16 qnum = op->queue->qnum;
2571         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2572         struct nvme_command *sqe = &cmdiu->sqe;
2573
2574         /*
2575          * Attempt to abort the offending command. Command completion
2576          * will detect the aborted io and will fail the connection.
2577          */
2578         dev_info(ctrl->ctrl.device,
2579                 "NVME-FC{%d.%d}: io timeout: opcode %d fctype %d (%s) w10/11: "
2580                 "x%08x/x%08x\n",
2581                 ctrl->cnum, qnum, sqe->common.opcode, sqe->fabrics.fctype,
2582                 nvme_fabrics_opcode_str(qnum, sqe),
2583                 sqe->common.cdw10, sqe->common.cdw11);
2584         if (__nvme_fc_abort_op(ctrl, op))
2585                 nvme_fc_error_recovery(ctrl, "io timeout abort failed");
2586
2587         /*
2588          * the io abort has been initiated. Have the reset timer
2589          * restarted and the abort completion will complete the io
2590          * shortly. Avoids a synchronous wait while the abort finishes.
2591          */
2592         return BLK_EH_RESET_TIMER;
2593 }
2594
2595 static int
2596 nvme_fc_map_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2597                 struct nvme_fc_fcp_op *op)
2598 {
2599         struct nvmefc_fcp_req *freq = &op->fcp_req;
2600         int ret;
2601
2602         freq->sg_cnt = 0;
2603
2604         if (!blk_rq_nr_phys_segments(rq))
2605                 return 0;
2606
2607         freq->sg_table.sgl = freq->first_sgl;
2608         ret = sg_alloc_table_chained(&freq->sg_table,
2609                         blk_rq_nr_phys_segments(rq), freq->sg_table.sgl,
2610                         NVME_INLINE_SG_CNT);
2611         if (ret)
2612                 return -ENOMEM;
2613
2614         op->nents = blk_rq_map_sg(rq->q, rq, freq->sg_table.sgl);
2615         WARN_ON(op->nents > blk_rq_nr_phys_segments(rq));
2616         freq->sg_cnt = fc_dma_map_sg(ctrl->lport->dev, freq->sg_table.sgl,
2617                                 op->nents, rq_dma_dir(rq));
2618         if (unlikely(freq->sg_cnt <= 0)) {
2619                 sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2620                 freq->sg_cnt = 0;
2621                 return -EFAULT;
2622         }
2623
2624         /*
2625          * TODO: blk_integrity_rq(rq)  for DIF
2626          */
2627         return 0;
2628 }
2629
2630 static void
2631 nvme_fc_unmap_data(struct nvme_fc_ctrl *ctrl, struct request *rq,
2632                 struct nvme_fc_fcp_op *op)
2633 {
2634         struct nvmefc_fcp_req *freq = &op->fcp_req;
2635
2636         if (!freq->sg_cnt)
2637                 return;
2638
2639         fc_dma_unmap_sg(ctrl->lport->dev, freq->sg_table.sgl, op->nents,
2640                         rq_dma_dir(rq));
2641
2642         sg_free_table_chained(&freq->sg_table, NVME_INLINE_SG_CNT);
2643
2644         freq->sg_cnt = 0;
2645 }
2646
2647 /*
2648  * In FC, the queue is a logical thing. At transport connect, the target
2649  * creates its "queue" and returns a handle that is to be given to the
2650  * target whenever it posts something to the corresponding SQ.  When an
2651  * SQE is sent on a SQ, FC effectively considers the SQE, or rather the
2652  * command contained within the SQE, an io, and assigns a FC exchange
2653  * to it. The SQE and the associated SQ handle are sent in the initial
2654  * CMD IU sents on the exchange. All transfers relative to the io occur
2655  * as part of the exchange.  The CQE is the last thing for the io,
2656  * which is transferred (explicitly or implicitly) with the RSP IU
2657  * sent on the exchange. After the CQE is received, the FC exchange is
2658  * terminaed and the Exchange may be used on a different io.
2659  *
2660  * The transport to LLDD api has the transport making a request for a
2661  * new fcp io request to the LLDD. The LLDD then allocates a FC exchange
2662  * resource and transfers the command. The LLDD will then process all
2663  * steps to complete the io. Upon completion, the transport done routine
2664  * is called.
2665  *
2666  * So - while the operation is outstanding to the LLDD, there is a link
2667  * level FC exchange resource that is also outstanding. This must be
2668  * considered in all cleanup operations.
2669  */
2670 static blk_status_t
2671 nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue,
2672         struct nvme_fc_fcp_op *op, u32 data_len,
2673         enum nvmefc_fcp_datadir io_dir)
2674 {
2675         struct nvme_fc_cmd_iu *cmdiu = &op->cmd_iu;
2676         struct nvme_command *sqe = &cmdiu->sqe;
2677         int ret, opstate;
2678
2679         /*
2680          * before attempting to send the io, check to see if we believe
2681          * the target device is present
2682          */
2683         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
2684                 return BLK_STS_RESOURCE;
2685
2686         if (!nvme_fc_ctrl_get(ctrl))
2687                 return BLK_STS_IOERR;
2688
2689         /* format the FC-NVME CMD IU and fcp_req */
2690         cmdiu->connection_id = cpu_to_be64(queue->connection_id);
2691         cmdiu->data_len = cpu_to_be32(data_len);
2692         switch (io_dir) {
2693         case NVMEFC_FCP_WRITE:
2694                 cmdiu->flags = FCNVME_CMD_FLAGS_WRITE;
2695                 break;
2696         case NVMEFC_FCP_READ:
2697                 cmdiu->flags = FCNVME_CMD_FLAGS_READ;
2698                 break;
2699         case NVMEFC_FCP_NODATA:
2700                 cmdiu->flags = 0;
2701                 break;
2702         }
2703         op->fcp_req.payload_length = data_len;
2704         op->fcp_req.io_dir = io_dir;
2705         op->fcp_req.transferred_length = 0;
2706         op->fcp_req.rcv_rsplen = 0;
2707         op->fcp_req.status = NVME_SC_SUCCESS;
2708         op->fcp_req.sqid = cpu_to_le16(queue->qnum);
2709
2710         /*
2711          * validate per fabric rules, set fields mandated by fabric spec
2712          * as well as those by FC-NVME spec.
2713          */
2714         WARN_ON_ONCE(sqe->common.metadata);
2715         sqe->common.flags |= NVME_CMD_SGL_METABUF;
2716
2717         /*
2718          * format SQE DPTR field per FC-NVME rules:
2719          *    type=0x5     Transport SGL Data Block Descriptor
2720          *    subtype=0xA  Transport-specific value
2721          *    address=0
2722          *    length=length of the data series
2723          */
2724         sqe->rw.dptr.sgl.type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2725                                         NVME_SGL_FMT_TRANSPORT_A;
2726         sqe->rw.dptr.sgl.length = cpu_to_le32(data_len);
2727         sqe->rw.dptr.sgl.addr = 0;
2728
2729         if (!(op->flags & FCOP_FLAGS_AEN)) {
2730                 ret = nvme_fc_map_data(ctrl, op->rq, op);
2731                 if (ret < 0) {
2732                         nvme_cleanup_cmd(op->rq);
2733                         nvme_fc_ctrl_put(ctrl);
2734                         if (ret == -ENOMEM || ret == -EAGAIN)
2735                                 return BLK_STS_RESOURCE;
2736                         return BLK_STS_IOERR;
2737                 }
2738         }
2739
2740         fc_dma_sync_single_for_device(ctrl->lport->dev, op->fcp_req.cmddma,
2741                                   sizeof(op->cmd_iu), DMA_TO_DEVICE);
2742
2743         atomic_set(&op->state, FCPOP_STATE_ACTIVE);
2744
2745         if (!(op->flags & FCOP_FLAGS_AEN))
2746                 nvme_start_request(op->rq);
2747
2748         cmdiu->csn = cpu_to_be32(atomic_inc_return(&queue->csn));
2749         ret = ctrl->lport->ops->fcp_io(&ctrl->lport->localport,
2750                                         &ctrl->rport->remoteport,
2751                                         queue->lldd_handle, &op->fcp_req);
2752
2753         if (ret) {
2754                 /*
2755                  * If the lld fails to send the command is there an issue with
2756                  * the csn value?  If the command that fails is the Connect,
2757                  * no - as the connection won't be live.  If it is a command
2758                  * post-connect, it's possible a gap in csn may be created.
2759                  * Does this matter?  As Linux initiators don't send fused
2760                  * commands, no.  The gap would exist, but as there's nothing
2761                  * that depends on csn order to be delivered on the target
2762                  * side, it shouldn't hurt.  It would be difficult for a
2763                  * target to even detect the csn gap as it has no idea when the
2764                  * cmd with the csn was supposed to arrive.
2765                  */
2766                 opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE);
2767                 __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate);
2768
2769                 if (!(op->flags & FCOP_FLAGS_AEN)) {
2770                         nvme_fc_unmap_data(ctrl, op->rq, op);
2771                         nvme_cleanup_cmd(op->rq);
2772                 }
2773
2774                 nvme_fc_ctrl_put(ctrl);
2775
2776                 if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE &&
2777                                 ret != -EBUSY)
2778                         return BLK_STS_IOERR;
2779
2780                 return BLK_STS_RESOURCE;
2781         }
2782
2783         return BLK_STS_OK;
2784 }
2785
2786 static blk_status_t
2787 nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx,
2788                         const struct blk_mq_queue_data *bd)
2789 {
2790         struct nvme_ns *ns = hctx->queue->queuedata;
2791         struct nvme_fc_queue *queue = hctx->driver_data;
2792         struct nvme_fc_ctrl *ctrl = queue->ctrl;
2793         struct request *rq = bd->rq;
2794         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2795         enum nvmefc_fcp_datadir io_dir;
2796         bool queue_ready = test_bit(NVME_FC_Q_LIVE, &queue->flags);
2797         u32 data_len;
2798         blk_status_t ret;
2799
2800         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE ||
2801             !nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2802                 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2803
2804         ret = nvme_setup_cmd(ns, rq);
2805         if (ret)
2806                 return ret;
2807
2808         /*
2809          * nvme core doesn't quite treat the rq opaquely. Commands such
2810          * as WRITE ZEROES will return a non-zero rq payload_bytes yet
2811          * there is no actual payload to be transferred.
2812          * To get it right, key data transmission on there being 1 or
2813          * more physical segments in the sg list. If there is no
2814          * physical segments, there is no payload.
2815          */
2816         if (blk_rq_nr_phys_segments(rq)) {
2817                 data_len = blk_rq_payload_bytes(rq);
2818                 io_dir = ((rq_data_dir(rq) == WRITE) ?
2819                                         NVMEFC_FCP_WRITE : NVMEFC_FCP_READ);
2820         } else {
2821                 data_len = 0;
2822                 io_dir = NVMEFC_FCP_NODATA;
2823         }
2824
2825
2826         return nvme_fc_start_fcp_op(ctrl, queue, op, data_len, io_dir);
2827 }
2828
2829 static void
2830 nvme_fc_submit_async_event(struct nvme_ctrl *arg)
2831 {
2832         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg);
2833         struct nvme_fc_fcp_op *aen_op;
2834         blk_status_t ret;
2835
2836         if (test_bit(FCCTRL_TERMIO, &ctrl->flags))
2837                 return;
2838
2839         aen_op = &ctrl->aen_ops[0];
2840
2841         ret = nvme_fc_start_fcp_op(ctrl, aen_op->queue, aen_op, 0,
2842                                         NVMEFC_FCP_NODATA);
2843         if (ret)
2844                 dev_err(ctrl->ctrl.device,
2845                         "failed async event work\n");
2846 }
2847
2848 static void
2849 nvme_fc_complete_rq(struct request *rq)
2850 {
2851         struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
2852         struct nvme_fc_ctrl *ctrl = op->ctrl;
2853
2854         atomic_set(&op->state, FCPOP_STATE_IDLE);
2855         op->flags &= ~FCOP_FLAGS_TERMIO;
2856
2857         nvme_fc_unmap_data(ctrl, rq, op);
2858         nvme_complete_rq(rq);
2859         nvme_fc_ctrl_put(ctrl);
2860 }
2861
2862 static void nvme_fc_map_queues(struct blk_mq_tag_set *set)
2863 {
2864         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(set->driver_data);
2865         int i;
2866
2867         for (i = 0; i < set->nr_maps; i++) {
2868                 struct blk_mq_queue_map *map = &set->map[i];
2869
2870                 if (!map->nr_queues) {
2871                         WARN_ON(i == HCTX_TYPE_DEFAULT);
2872                         continue;
2873                 }
2874
2875                 /* Call LLDD map queue functionality if defined */
2876                 if (ctrl->lport->ops->map_queues)
2877                         ctrl->lport->ops->map_queues(&ctrl->lport->localport,
2878                                                      map);
2879                 else
2880                         blk_mq_map_queues(map);
2881         }
2882 }
2883
2884 static const struct blk_mq_ops nvme_fc_mq_ops = {
2885         .queue_rq       = nvme_fc_queue_rq,
2886         .complete       = nvme_fc_complete_rq,
2887         .init_request   = nvme_fc_init_request,
2888         .exit_request   = nvme_fc_exit_request,
2889         .init_hctx      = nvme_fc_init_hctx,
2890         .timeout        = nvme_fc_timeout,
2891         .map_queues     = nvme_fc_map_queues,
2892 };
2893
2894 static int
2895 nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl)
2896 {
2897         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2898         unsigned int nr_io_queues;
2899         int ret;
2900
2901         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2902                                 ctrl->lport->ops->max_hw_queues);
2903         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2904         if (ret) {
2905                 dev_info(ctrl->ctrl.device,
2906                         "set_queue_count failed: %d\n", ret);
2907                 return ret;
2908         }
2909
2910         ctrl->ctrl.queue_count = nr_io_queues + 1;
2911         if (!nr_io_queues)
2912                 return 0;
2913
2914         nvme_fc_init_io_queues(ctrl);
2915
2916         ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
2917                         &nvme_fc_mq_ops, 1,
2918                         struct_size_t(struct nvme_fcp_op_w_sgl, priv,
2919                                       ctrl->lport->ops->fcprqst_priv_sz));
2920         if (ret)
2921                 return ret;
2922
2923         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2924         if (ret)
2925                 goto out_cleanup_tagset;
2926
2927         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2928         if (ret)
2929                 goto out_delete_hw_queues;
2930
2931         ctrl->ioq_live = true;
2932
2933         return 0;
2934
2935 out_delete_hw_queues:
2936         nvme_fc_delete_hw_io_queues(ctrl);
2937 out_cleanup_tagset:
2938         nvme_remove_io_tag_set(&ctrl->ctrl);
2939         nvme_fc_free_io_queues(ctrl);
2940
2941         /* force put free routine to ignore io queues */
2942         ctrl->ctrl.tagset = NULL;
2943
2944         return ret;
2945 }
2946
2947 static int
2948 nvme_fc_recreate_io_queues(struct nvme_fc_ctrl *ctrl)
2949 {
2950         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2951         u32 prior_ioq_cnt = ctrl->ctrl.queue_count - 1;
2952         unsigned int nr_io_queues;
2953         int ret;
2954
2955         nr_io_queues = min(min(opts->nr_io_queues, num_online_cpus()),
2956                                 ctrl->lport->ops->max_hw_queues);
2957         ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
2958         if (ret) {
2959                 dev_info(ctrl->ctrl.device,
2960                         "set_queue_count failed: %d\n", ret);
2961                 return ret;
2962         }
2963
2964         if (!nr_io_queues && prior_ioq_cnt) {
2965                 dev_info(ctrl->ctrl.device,
2966                         "Fail Reconnect: At least 1 io queue "
2967                         "required (was %d)\n", prior_ioq_cnt);
2968                 return -ENOSPC;
2969         }
2970
2971         ctrl->ctrl.queue_count = nr_io_queues + 1;
2972         /* check for io queues existing */
2973         if (ctrl->ctrl.queue_count == 1)
2974                 return 0;
2975
2976         if (prior_ioq_cnt != nr_io_queues) {
2977                 dev_info(ctrl->ctrl.device,
2978                         "reconnect: revising io queue count from %d to %d\n",
2979                         prior_ioq_cnt, nr_io_queues);
2980                 blk_mq_update_nr_hw_queues(&ctrl->tag_set, nr_io_queues);
2981         }
2982
2983         ret = nvme_fc_create_hw_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2984         if (ret)
2985                 goto out_free_io_queues;
2986
2987         ret = nvme_fc_connect_io_queues(ctrl, ctrl->ctrl.sqsize + 1);
2988         if (ret)
2989                 goto out_delete_hw_queues;
2990
2991         return 0;
2992
2993 out_delete_hw_queues:
2994         nvme_fc_delete_hw_io_queues(ctrl);
2995 out_free_io_queues:
2996         nvme_fc_free_io_queues(ctrl);
2997         return ret;
2998 }
2999
3000 static void
3001 nvme_fc_rport_active_on_lport(struct nvme_fc_rport *rport)
3002 {
3003         struct nvme_fc_lport *lport = rport->lport;
3004
3005         atomic_inc(&lport->act_rport_cnt);
3006 }
3007
3008 static void
3009 nvme_fc_rport_inactive_on_lport(struct nvme_fc_rport *rport)
3010 {
3011         struct nvme_fc_lport *lport = rport->lport;
3012         u32 cnt;
3013
3014         cnt = atomic_dec_return(&lport->act_rport_cnt);
3015         if (cnt == 0 && lport->localport.port_state == FC_OBJSTATE_DELETED)
3016                 lport->ops->localport_delete(&lport->localport);
3017 }
3018
3019 static int
3020 nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl)
3021 {
3022         struct nvme_fc_rport *rport = ctrl->rport;
3023         u32 cnt;
3024
3025         if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags))
3026                 return 1;
3027
3028         cnt = atomic_inc_return(&rport->act_ctrl_cnt);
3029         if (cnt == 1)
3030                 nvme_fc_rport_active_on_lport(rport);
3031
3032         return 0;
3033 }
3034
3035 static int
3036 nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl)
3037 {
3038         struct nvme_fc_rport *rport = ctrl->rport;
3039         struct nvme_fc_lport *lport = rport->lport;
3040         u32 cnt;
3041
3042         /* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */
3043
3044         cnt = atomic_dec_return(&rport->act_ctrl_cnt);
3045         if (cnt == 0) {
3046                 if (rport->remoteport.port_state == FC_OBJSTATE_DELETED)
3047                         lport->ops->remoteport_delete(&rport->remoteport);
3048                 nvme_fc_rport_inactive_on_lport(rport);
3049         }
3050
3051         return 0;
3052 }
3053
3054 /*
3055  * This routine restarts the controller on the host side, and
3056  * on the link side, recreates the controller association.
3057  */
3058 static int
3059 nvme_fc_create_association(struct nvme_fc_ctrl *ctrl)
3060 {
3061         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
3062         struct nvmefc_ls_rcv_op *disls = NULL;
3063         unsigned long flags;
3064         int ret;
3065         bool changed;
3066
3067         ++ctrl->ctrl.nr_reconnects;
3068
3069         if (ctrl->rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3070                 return -ENODEV;
3071
3072         if (nvme_fc_ctlr_active_on_rport(ctrl))
3073                 return -ENOTUNIQ;
3074
3075         dev_info(ctrl->ctrl.device,
3076                 "NVME-FC{%d}: create association : host wwpn 0x%016llx "
3077                 " rport wwpn 0x%016llx: NQN \"%s\"\n",
3078                 ctrl->cnum, ctrl->lport->localport.port_name,
3079                 ctrl->rport->remoteport.port_name, ctrl->ctrl.opts->subsysnqn);
3080
3081         clear_bit(ASSOC_FAILED, &ctrl->flags);
3082
3083         /*
3084          * Create the admin queue
3085          */
3086
3087         ret = __nvme_fc_create_hw_queue(ctrl, &ctrl->queues[0], 0,
3088                                 NVME_AQ_DEPTH);
3089         if (ret)
3090                 goto out_free_queue;
3091
3092         ret = nvme_fc_connect_admin_queue(ctrl, &ctrl->queues[0],
3093                                 NVME_AQ_DEPTH, (NVME_AQ_DEPTH / 4));
3094         if (ret)
3095                 goto out_delete_hw_queue;
3096
3097         ret = nvmf_connect_admin_queue(&ctrl->ctrl);
3098         if (ret)
3099                 goto out_disconnect_admin_queue;
3100
3101         set_bit(NVME_FC_Q_LIVE, &ctrl->queues[0].flags);
3102
3103         /*
3104          * Check controller capabilities
3105          *
3106          * todo:- add code to check if ctrl attributes changed from
3107          * prior connection values
3108          */
3109
3110         ret = nvme_enable_ctrl(&ctrl->ctrl);
3111         if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3112                 ret = -EIO;
3113         if (ret)
3114                 goto out_disconnect_admin_queue;
3115
3116         ctrl->ctrl.max_segments = ctrl->lport->ops->max_sgl_segments;
3117         ctrl->ctrl.max_hw_sectors = ctrl->ctrl.max_segments <<
3118                                                 (ilog2(SZ_4K) - 9);
3119
3120         nvme_unquiesce_admin_queue(&ctrl->ctrl);
3121
3122         ret = nvme_init_ctrl_finish(&ctrl->ctrl, false);
3123         if (ret)
3124                 goto out_disconnect_admin_queue;
3125         if (test_bit(ASSOC_FAILED, &ctrl->flags)) {
3126                 ret = -EIO;
3127                 goto out_stop_keep_alive;
3128         }
3129         /* sanity checks */
3130
3131         /* FC-NVME does not have other data in the capsule */
3132         if (ctrl->ctrl.icdoff) {
3133                 dev_err(ctrl->ctrl.device, "icdoff %d is not supported!\n",
3134                                 ctrl->ctrl.icdoff);
3135                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3136                 goto out_stop_keep_alive;
3137         }
3138
3139         /* FC-NVME supports normal SGL Data Block Descriptors */
3140         if (!nvme_ctrl_sgl_supported(&ctrl->ctrl)) {
3141                 dev_err(ctrl->ctrl.device,
3142                         "Mandatory sgls are not supported!\n");
3143                 ret = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
3144                 goto out_stop_keep_alive;
3145         }
3146
3147         if (opts->queue_size > ctrl->ctrl.maxcmd) {
3148                 /* warn if maxcmd is lower than queue_size */
3149                 dev_warn(ctrl->ctrl.device,
3150                         "queue_size %zu > ctrl maxcmd %u, reducing "
3151                         "to maxcmd\n",
3152                         opts->queue_size, ctrl->ctrl.maxcmd);
3153                 opts->queue_size = ctrl->ctrl.maxcmd;
3154                 ctrl->ctrl.sqsize = opts->queue_size - 1;
3155         }
3156
3157         ret = nvme_fc_init_aen_ops(ctrl);
3158         if (ret)
3159                 goto out_term_aen_ops;
3160
3161         /*
3162          * Create the io queues
3163          */
3164
3165         if (ctrl->ctrl.queue_count > 1) {
3166                 if (!ctrl->ioq_live)
3167                         ret = nvme_fc_create_io_queues(ctrl);
3168                 else
3169                         ret = nvme_fc_recreate_io_queues(ctrl);
3170         }
3171         if (!ret && test_bit(ASSOC_FAILED, &ctrl->flags))
3172                 ret = -EIO;
3173         if (ret)
3174                 goto out_term_aen_ops;
3175
3176         changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
3177
3178         ctrl->ctrl.nr_reconnects = 0;
3179
3180         if (changed)
3181                 nvme_start_ctrl(&ctrl->ctrl);
3182
3183         return 0;       /* Success */
3184
3185 out_term_aen_ops:
3186         nvme_fc_term_aen_ops(ctrl);
3187 out_stop_keep_alive:
3188         nvme_stop_keep_alive(&ctrl->ctrl);
3189 out_disconnect_admin_queue:
3190         dev_warn(ctrl->ctrl.device,
3191                 "NVME-FC{%d}: create_assoc failed, assoc_id %llx ret %d\n",
3192                 ctrl->cnum, ctrl->association_id, ret);
3193         /* send a Disconnect(association) LS to fc-nvme target */
3194         nvme_fc_xmt_disconnect_assoc(ctrl);
3195         spin_lock_irqsave(&ctrl->lock, flags);
3196         ctrl->association_id = 0;
3197         disls = ctrl->rcv_disconn;
3198         ctrl->rcv_disconn = NULL;
3199         spin_unlock_irqrestore(&ctrl->lock, flags);
3200         if (disls)
3201                 nvme_fc_xmt_ls_rsp(disls);
3202 out_delete_hw_queue:
3203         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3204 out_free_queue:
3205         nvme_fc_free_queue(&ctrl->queues[0]);
3206         clear_bit(ASSOC_ACTIVE, &ctrl->flags);
3207         nvme_fc_ctlr_inactive_on_rport(ctrl);
3208
3209         return ret;
3210 }
3211
3212
3213 /*
3214  * This routine stops operation of the controller on the host side.
3215  * On the host os stack side: Admin and IO queues are stopped,
3216  *   outstanding ios on them terminated via FC ABTS.
3217  * On the link side: the association is terminated.
3218  */
3219 static void
3220 nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl)
3221 {
3222         struct nvmefc_ls_rcv_op *disls = NULL;
3223         unsigned long flags;
3224
3225         if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags))
3226                 return;
3227
3228         spin_lock_irqsave(&ctrl->lock, flags);
3229         set_bit(FCCTRL_TERMIO, &ctrl->flags);
3230         ctrl->iocnt = 0;
3231         spin_unlock_irqrestore(&ctrl->lock, flags);
3232
3233         __nvme_fc_abort_outstanding_ios(ctrl, false);
3234
3235         /* kill the aens as they are a separate path */
3236         nvme_fc_abort_aen_ops(ctrl);
3237
3238         /* wait for all io that had to be aborted */
3239         spin_lock_irq(&ctrl->lock);
3240         wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock);
3241         clear_bit(FCCTRL_TERMIO, &ctrl->flags);
3242         spin_unlock_irq(&ctrl->lock);
3243
3244         nvme_fc_term_aen_ops(ctrl);
3245
3246         /*
3247          * send a Disconnect(association) LS to fc-nvme target
3248          * Note: could have been sent at top of process, but
3249          * cleaner on link traffic if after the aborts complete.
3250          * Note: if association doesn't exist, association_id will be 0
3251          */
3252         if (ctrl->association_id)
3253                 nvme_fc_xmt_disconnect_assoc(ctrl);
3254
3255         spin_lock_irqsave(&ctrl->lock, flags);
3256         ctrl->association_id = 0;
3257         disls = ctrl->rcv_disconn;
3258         ctrl->rcv_disconn = NULL;
3259         spin_unlock_irqrestore(&ctrl->lock, flags);
3260         if (disls)
3261                 /*
3262                  * if a Disconnect Request was waiting for a response, send
3263                  * now that all ABTS's have been issued (and are complete).
3264                  */
3265                 nvme_fc_xmt_ls_rsp(disls);
3266
3267         if (ctrl->ctrl.tagset) {
3268                 nvme_fc_delete_hw_io_queues(ctrl);
3269                 nvme_fc_free_io_queues(ctrl);
3270         }
3271
3272         __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0);
3273         nvme_fc_free_queue(&ctrl->queues[0]);
3274
3275         /* re-enable the admin_q so anything new can fast fail */
3276         nvme_unquiesce_admin_queue(&ctrl->ctrl);
3277
3278         /* resume the io queues so that things will fast fail */
3279         nvme_unquiesce_io_queues(&ctrl->ctrl);
3280
3281         nvme_fc_ctlr_inactive_on_rport(ctrl);
3282 }
3283
3284 static void
3285 nvme_fc_delete_ctrl(struct nvme_ctrl *nctrl)
3286 {
3287         struct nvme_fc_ctrl *ctrl = to_fc_ctrl(nctrl);
3288
3289         cancel_work_sync(&ctrl->ioerr_work);
3290         cancel_delayed_work_sync(&ctrl->connect_work);
3291         /*
3292          * kill the association on the link side.  this will block
3293          * waiting for io to terminate
3294          */
3295         nvme_fc_delete_association(ctrl);
3296 }
3297
3298 static void
3299 nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status)
3300 {
3301         struct nvme_fc_rport *rport = ctrl->rport;
3302         struct nvme_fc_remote_port *portptr = &rport->remoteport;
3303         unsigned long recon_delay = ctrl->ctrl.opts->reconnect_delay * HZ;
3304         bool recon = true;
3305
3306         if (nvme_ctrl_state(&ctrl->ctrl) != NVME_CTRL_CONNECTING)
3307                 return;
3308
3309         if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3310                 dev_info(ctrl->ctrl.device,
3311                         "NVME-FC{%d}: reset: Reconnect attempt failed (%d)\n",
3312                         ctrl->cnum, status);
3313                 if (status > 0 && (status & NVME_SC_DNR))
3314                         recon = false;
3315         } else if (time_after_eq(jiffies, rport->dev_loss_end))
3316                 recon = false;
3317
3318         if (recon && nvmf_should_reconnect(&ctrl->ctrl)) {
3319                 if (portptr->port_state == FC_OBJSTATE_ONLINE)
3320                         dev_info(ctrl->ctrl.device,
3321                                 "NVME-FC{%d}: Reconnect attempt in %ld "
3322                                 "seconds\n",
3323                                 ctrl->cnum, recon_delay / HZ);
3324                 else if (time_after(jiffies + recon_delay, rport->dev_loss_end))
3325                         recon_delay = rport->dev_loss_end - jiffies;
3326
3327                 queue_delayed_work(nvme_wq, &ctrl->connect_work, recon_delay);
3328         } else {
3329                 if (portptr->port_state == FC_OBJSTATE_ONLINE) {
3330                         if (status > 0 && (status & NVME_SC_DNR))
3331                                 dev_warn(ctrl->ctrl.device,
3332                                          "NVME-FC{%d}: reconnect failure\n",
3333                                          ctrl->cnum);
3334                         else
3335                                 dev_warn(ctrl->ctrl.device,
3336                                          "NVME-FC{%d}: Max reconnect attempts "
3337                                          "(%d) reached.\n",
3338                                          ctrl->cnum, ctrl->ctrl.nr_reconnects);
3339                 } else
3340                         dev_warn(ctrl->ctrl.device,
3341                                 "NVME-FC{%d}: dev_loss_tmo (%d) expired "
3342                                 "while waiting for remoteport connectivity.\n",
3343                                 ctrl->cnum, min_t(int, portptr->dev_loss_tmo,
3344                                         (ctrl->ctrl.opts->max_reconnects *
3345                                          ctrl->ctrl.opts->reconnect_delay)));
3346                 WARN_ON(nvme_delete_ctrl(&ctrl->ctrl));
3347         }
3348 }
3349
3350 static void
3351 nvme_fc_reset_ctrl_work(struct work_struct *work)
3352 {
3353         struct nvme_fc_ctrl *ctrl =
3354                 container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
3355
3356         nvme_stop_ctrl(&ctrl->ctrl);
3357
3358         /* will block will waiting for io to terminate */
3359         nvme_fc_delete_association(ctrl);
3360
3361         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
3362                 dev_err(ctrl->ctrl.device,
3363                         "NVME-FC{%d}: error_recovery: Couldn't change state "
3364                         "to CONNECTING\n", ctrl->cnum);
3365
3366         if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
3367                 if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3368                         dev_err(ctrl->ctrl.device,
3369                                 "NVME-FC{%d}: failed to schedule connect "
3370                                 "after reset\n", ctrl->cnum);
3371                 } else {
3372                         flush_delayed_work(&ctrl->connect_work);
3373                 }
3374         } else {
3375                 nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);
3376         }
3377 }
3378
3379
3380 static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
3381         .name                   = "fc",
3382         .module                 = THIS_MODULE,
3383         .flags                  = NVME_F_FABRICS,
3384         .reg_read32             = nvmf_reg_read32,
3385         .reg_read64             = nvmf_reg_read64,
3386         .reg_write32            = nvmf_reg_write32,
3387         .free_ctrl              = nvme_fc_nvme_ctrl_freed,
3388         .submit_async_event     = nvme_fc_submit_async_event,
3389         .delete_ctrl            = nvme_fc_delete_ctrl,
3390         .get_address            = nvmf_get_address,
3391 };
3392
3393 static void
3394 nvme_fc_connect_ctrl_work(struct work_struct *work)
3395 {
3396         int ret;
3397
3398         struct nvme_fc_ctrl *ctrl =
3399                         container_of(to_delayed_work(work),
3400                                 struct nvme_fc_ctrl, connect_work);
3401
3402         ret = nvme_fc_create_association(ctrl);
3403         if (ret)
3404                 nvme_fc_reconnect_or_delete(ctrl, ret);
3405         else
3406                 dev_info(ctrl->ctrl.device,
3407                         "NVME-FC{%d}: controller connect complete\n",
3408                         ctrl->cnum);
3409 }
3410
3411
3412 static const struct blk_mq_ops nvme_fc_admin_mq_ops = {
3413         .queue_rq       = nvme_fc_queue_rq,
3414         .complete       = nvme_fc_complete_rq,
3415         .init_request   = nvme_fc_init_request,
3416         .exit_request   = nvme_fc_exit_request,
3417         .init_hctx      = nvme_fc_init_admin_hctx,
3418         .timeout        = nvme_fc_timeout,
3419 };
3420
3421
3422 /*
3423  * Fails a controller request if it matches an existing controller
3424  * (association) with the same tuple:
3425  * <Host NQN, Host ID, local FC port, remote FC port, SUBSYS NQN>
3426  *
3427  * The ports don't need to be compared as they are intrinsically
3428  * already matched by the port pointers supplied.
3429  */
3430 static bool
3431 nvme_fc_existing_controller(struct nvme_fc_rport *rport,
3432                 struct nvmf_ctrl_options *opts)
3433 {
3434         struct nvme_fc_ctrl *ctrl;
3435         unsigned long flags;
3436         bool found = false;
3437
3438         spin_lock_irqsave(&rport->lock, flags);
3439         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3440                 found = nvmf_ctlr_matches_baseopts(&ctrl->ctrl, opts);
3441                 if (found)
3442                         break;
3443         }
3444         spin_unlock_irqrestore(&rport->lock, flags);
3445
3446         return found;
3447 }
3448
3449 static struct nvme_ctrl *
3450 nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
3451         struct nvme_fc_lport *lport, struct nvme_fc_rport *rport)
3452 {
3453         struct nvme_fc_ctrl *ctrl;
3454         unsigned long flags;
3455         int ret, idx, ctrl_loss_tmo;
3456
3457         if (!(rport->remoteport.port_role &
3458             (FC_PORT_ROLE_NVME_DISCOVERY | FC_PORT_ROLE_NVME_TARGET))) {
3459                 ret = -EBADR;
3460                 goto out_fail;
3461         }
3462
3463         if (!opts->duplicate_connect &&
3464             nvme_fc_existing_controller(rport, opts)) {
3465                 ret = -EALREADY;
3466                 goto out_fail;
3467         }
3468
3469         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
3470         if (!ctrl) {
3471                 ret = -ENOMEM;
3472                 goto out_fail;
3473         }
3474
3475         idx = ida_alloc(&nvme_fc_ctrl_cnt, GFP_KERNEL);
3476         if (idx < 0) {
3477                 ret = -ENOSPC;
3478                 goto out_free_ctrl;
3479         }
3480
3481         /*
3482          * if ctrl_loss_tmo is being enforced and the default reconnect delay
3483          * is being used, change to a shorter reconnect delay for FC.
3484          */
3485         if (opts->max_reconnects != -1 &&
3486             opts->reconnect_delay == NVMF_DEF_RECONNECT_DELAY &&
3487             opts->reconnect_delay > NVME_FC_DEFAULT_RECONNECT_TMO) {
3488                 ctrl_loss_tmo = opts->max_reconnects * opts->reconnect_delay;
3489                 opts->reconnect_delay = NVME_FC_DEFAULT_RECONNECT_TMO;
3490                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3491                                                 opts->reconnect_delay);
3492         }
3493
3494         ctrl->ctrl.opts = opts;
3495         ctrl->ctrl.nr_reconnects = 0;
3496         INIT_LIST_HEAD(&ctrl->ctrl_list);
3497         ctrl->lport = lport;
3498         ctrl->rport = rport;
3499         ctrl->dev = lport->dev;
3500         ctrl->cnum = idx;
3501         ctrl->ioq_live = false;
3502         init_waitqueue_head(&ctrl->ioabort_wait);
3503
3504         get_device(ctrl->dev);
3505         kref_init(&ctrl->ref);
3506
3507         INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
3508         INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
3509         INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
3510         spin_lock_init(&ctrl->lock);
3511
3512         /* io queue count */
3513         ctrl->ctrl.queue_count = min_t(unsigned int,
3514                                 opts->nr_io_queues,
3515                                 lport->ops->max_hw_queues);
3516         ctrl->ctrl.queue_count++;       /* +1 for admin queue */
3517
3518         ctrl->ctrl.sqsize = opts->queue_size - 1;
3519         ctrl->ctrl.kato = opts->kato;
3520         ctrl->ctrl.cntlid = 0xffff;
3521
3522         ret = -ENOMEM;
3523         ctrl->queues = kcalloc(ctrl->ctrl.queue_count,
3524                                 sizeof(struct nvme_fc_queue), GFP_KERNEL);
3525         if (!ctrl->queues)
3526                 goto out_free_ida;
3527
3528         nvme_fc_init_queue(ctrl, 0);
3529
3530         /*
3531          * Would have been nice to init io queues tag set as well.
3532          * However, we require interaction from the controller
3533          * for max io queue count before we can do so.
3534          * Defer this to the connect path.
3535          */
3536
3537         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_fc_ctrl_ops, 0);
3538         if (ret)
3539                 goto out_free_queues;
3540         if (lport->dev)
3541                 ctrl->ctrl.numa_node = dev_to_node(lport->dev);
3542
3543         /* at this point, teardown path changes to ref counting on nvme ctrl */
3544
3545         ret = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
3546                         &nvme_fc_admin_mq_ops,
3547                         struct_size_t(struct nvme_fcp_op_w_sgl, priv,
3548                                       ctrl->lport->ops->fcprqst_priv_sz));
3549         if (ret)
3550                 goto fail_ctrl;
3551
3552         spin_lock_irqsave(&rport->lock, flags);
3553         list_add_tail(&ctrl->ctrl_list, &rport->ctrl_list);
3554         spin_unlock_irqrestore(&rport->lock, flags);
3555
3556         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING) ||
3557             !nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
3558                 dev_err(ctrl->ctrl.device,
3559                         "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum);
3560                 goto fail_ctrl;
3561         }
3562
3563         if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) {
3564                 dev_err(ctrl->ctrl.device,
3565                         "NVME-FC{%d}: failed to schedule initial connect\n",
3566                         ctrl->cnum);
3567                 goto fail_ctrl;
3568         }
3569
3570         flush_delayed_work(&ctrl->connect_work);
3571
3572         dev_info(ctrl->ctrl.device,
3573                 "NVME-FC{%d}: new ctrl: NQN \"%s\"\n",
3574                 ctrl->cnum, nvmf_ctrl_subsysnqn(&ctrl->ctrl));
3575
3576         return &ctrl->ctrl;
3577
3578 fail_ctrl:
3579         nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING);
3580         cancel_work_sync(&ctrl->ioerr_work);
3581         cancel_work_sync(&ctrl->ctrl.reset_work);
3582         cancel_delayed_work_sync(&ctrl->connect_work);
3583
3584         ctrl->ctrl.opts = NULL;
3585
3586         /* initiate nvme ctrl ref counting teardown */
3587         nvme_uninit_ctrl(&ctrl->ctrl);
3588
3589         /* Remove core ctrl ref. */
3590         nvme_put_ctrl(&ctrl->ctrl);
3591
3592         /* as we're past the point where we transition to the ref
3593          * counting teardown path, if we return a bad pointer here,
3594          * the calling routine, thinking it's prior to the
3595          * transition, will do an rport put. Since the teardown
3596          * path also does a rport put, we do an extra get here to
3597          * so proper order/teardown happens.
3598          */
3599         nvme_fc_rport_get(rport);
3600
3601         return ERR_PTR(-EIO);
3602
3603 out_free_queues:
3604         kfree(ctrl->queues);
3605 out_free_ida:
3606         put_device(ctrl->dev);
3607         ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum);
3608 out_free_ctrl:
3609         kfree(ctrl);
3610 out_fail:
3611         /* exit via here doesn't follow ctlr ref points */
3612         return ERR_PTR(ret);
3613 }
3614
3615
3616 struct nvmet_fc_traddr {
3617         u64     nn;
3618         u64     pn;
3619 };
3620
3621 static int
3622 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
3623 {
3624         u64 token64;
3625
3626         if (match_u64(sstr, &token64))
3627                 return -EINVAL;
3628         *val = token64;
3629
3630         return 0;
3631 }
3632
3633 /*
3634  * This routine validates and extracts the WWN's from the TRADDR string.
3635  * As kernel parsers need the 0x to determine number base, universally
3636  * build string to parse with 0x prefix before parsing name strings.
3637  */
3638 static int
3639 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
3640 {
3641         char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
3642         substring_t wwn = { name, &name[sizeof(name)-1] };
3643         int nnoffset, pnoffset;
3644
3645         /* validate if string is one of the 2 allowed formats */
3646         if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
3647                         !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
3648                         !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
3649                                 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
3650                 nnoffset = NVME_FC_TRADDR_OXNNLEN;
3651                 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
3652                                                 NVME_FC_TRADDR_OXNNLEN;
3653         } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
3654                         !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
3655                         !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
3656                                 "pn-", NVME_FC_TRADDR_NNLEN))) {
3657                 nnoffset = NVME_FC_TRADDR_NNLEN;
3658                 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
3659         } else
3660                 goto out_einval;
3661
3662         name[0] = '0';
3663         name[1] = 'x';
3664         name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
3665
3666         memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3667         if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
3668                 goto out_einval;
3669
3670         memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
3671         if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
3672                 goto out_einval;
3673
3674         return 0;
3675
3676 out_einval:
3677         pr_warn("%s: bad traddr string\n", __func__);
3678         return -EINVAL;
3679 }
3680
3681 static struct nvme_ctrl *
3682 nvme_fc_create_ctrl(struct device *dev, struct nvmf_ctrl_options *opts)
3683 {
3684         struct nvme_fc_lport *lport;
3685         struct nvme_fc_rport *rport;
3686         struct nvme_ctrl *ctrl;
3687         struct nvmet_fc_traddr laddr = { 0L, 0L };
3688         struct nvmet_fc_traddr raddr = { 0L, 0L };
3689         unsigned long flags;
3690         int ret;
3691
3692         ret = nvme_fc_parse_traddr(&raddr, opts->traddr, NVMF_TRADDR_SIZE);
3693         if (ret || !raddr.nn || !raddr.pn)
3694                 return ERR_PTR(-EINVAL);
3695
3696         ret = nvme_fc_parse_traddr(&laddr, opts->host_traddr, NVMF_TRADDR_SIZE);
3697         if (ret || !laddr.nn || !laddr.pn)
3698                 return ERR_PTR(-EINVAL);
3699
3700         /* find the host and remote ports to connect together */
3701         spin_lock_irqsave(&nvme_fc_lock, flags);
3702         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3703                 if (lport->localport.node_name != laddr.nn ||
3704                     lport->localport.port_name != laddr.pn ||
3705                     lport->localport.port_state != FC_OBJSTATE_ONLINE)
3706                         continue;
3707
3708                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3709                         if (rport->remoteport.node_name != raddr.nn ||
3710                             rport->remoteport.port_name != raddr.pn ||
3711                             rport->remoteport.port_state != FC_OBJSTATE_ONLINE)
3712                                 continue;
3713
3714                         /* if fail to get reference fall through. Will error */
3715                         if (!nvme_fc_rport_get(rport))
3716                                 break;
3717
3718                         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3719
3720                         ctrl = nvme_fc_init_ctrl(dev, opts, lport, rport);
3721                         if (IS_ERR(ctrl))
3722                                 nvme_fc_rport_put(rport);
3723                         return ctrl;
3724                 }
3725         }
3726         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3727
3728         pr_warn("%s: %s - %s combination not found\n",
3729                 __func__, opts->traddr, opts->host_traddr);
3730         return ERR_PTR(-ENOENT);
3731 }
3732
3733
3734 static struct nvmf_transport_ops nvme_fc_transport = {
3735         .name           = "fc",
3736         .module         = THIS_MODULE,
3737         .required_opts  = NVMF_OPT_TRADDR | NVMF_OPT_HOST_TRADDR,
3738         .allowed_opts   = NVMF_OPT_RECONNECT_DELAY | NVMF_OPT_CTRL_LOSS_TMO,
3739         .create_ctrl    = nvme_fc_create_ctrl,
3740 };
3741
3742 /* Arbitrary successive failures max. With lots of subsystems could be high */
3743 #define DISCOVERY_MAX_FAIL      20
3744
3745 static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
3746                 struct device_attribute *attr, const char *buf, size_t count)
3747 {
3748         unsigned long flags;
3749         LIST_HEAD(local_disc_list);
3750         struct nvme_fc_lport *lport;
3751         struct nvme_fc_rport *rport;
3752         int failcnt = 0;
3753
3754         spin_lock_irqsave(&nvme_fc_lock, flags);
3755 restart:
3756         list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
3757                 list_for_each_entry(rport, &lport->endp_list, endp_list) {
3758                         if (!nvme_fc_lport_get(lport))
3759                                 continue;
3760                         if (!nvme_fc_rport_get(rport)) {
3761                                 /*
3762                                  * This is a temporary condition. Upon restart
3763                                  * this rport will be gone from the list.
3764                                  *
3765                                  * Revert the lport put and retry.  Anything
3766                                  * added to the list already will be skipped (as
3767                                  * they are no longer list_empty).  Loops should
3768                                  * resume at rports that were not yet seen.
3769                                  */
3770                                 nvme_fc_lport_put(lport);
3771
3772                                 if (failcnt++ < DISCOVERY_MAX_FAIL)
3773                                         goto restart;
3774
3775                                 pr_err("nvme_discovery: too many reference "
3776                                        "failures\n");
3777                                 goto process_local_list;
3778                         }
3779                         if (list_empty(&rport->disc_list))
3780                                 list_add_tail(&rport->disc_list,
3781                                               &local_disc_list);
3782                 }
3783         }
3784
3785 process_local_list:
3786         while (!list_empty(&local_disc_list)) {
3787                 rport = list_first_entry(&local_disc_list,
3788                                          struct nvme_fc_rport, disc_list);
3789                 list_del_init(&rport->disc_list);
3790                 spin_unlock_irqrestore(&nvme_fc_lock, flags);
3791
3792                 lport = rport->lport;
3793                 /* signal discovery. Won't hurt if it repeats */
3794                 nvme_fc_signal_discovery_scan(lport, rport);
3795                 nvme_fc_rport_put(rport);
3796                 nvme_fc_lport_put(lport);
3797
3798                 spin_lock_irqsave(&nvme_fc_lock, flags);
3799         }
3800         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3801
3802         return count;
3803 }
3804
3805 static DEVICE_ATTR(nvme_discovery, 0200, NULL, nvme_fc_nvme_discovery_store);
3806
3807 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3808 /* Parse the cgroup id from a buf and return the length of cgrpid */
3809 static int fc_parse_cgrpid(const char *buf, u64 *id)
3810 {
3811         char cgrp_id[16+1];
3812         int cgrpid_len, j;
3813
3814         memset(cgrp_id, 0x0, sizeof(cgrp_id));
3815         for (cgrpid_len = 0, j = 0; cgrpid_len < 17; cgrpid_len++) {
3816                 if (buf[cgrpid_len] != ':')
3817                         cgrp_id[cgrpid_len] = buf[cgrpid_len];
3818                 else {
3819                         j = 1;
3820                         break;
3821                 }
3822         }
3823         if (!j)
3824                 return -EINVAL;
3825         if (kstrtou64(cgrp_id, 16, id) < 0)
3826                 return -EINVAL;
3827         return cgrpid_len;
3828 }
3829
3830 /*
3831  * Parse and update the appid in the blkcg associated with the cgroupid.
3832  */
3833 static ssize_t fc_appid_store(struct device *dev,
3834                 struct device_attribute *attr, const char *buf, size_t count)
3835 {
3836         size_t orig_count = count;
3837         u64 cgrp_id;
3838         int appid_len = 0;
3839         int cgrpid_len = 0;
3840         char app_id[FC_APPID_LEN];
3841         int ret = 0;
3842
3843         if (buf[count-1] == '\n')
3844                 count--;
3845
3846         if ((count > (16+1+FC_APPID_LEN)) || (!strchr(buf, ':')))
3847                 return -EINVAL;
3848
3849         cgrpid_len = fc_parse_cgrpid(buf, &cgrp_id);
3850         if (cgrpid_len < 0)
3851                 return -EINVAL;
3852         appid_len = count - cgrpid_len - 1;
3853         if (appid_len > FC_APPID_LEN)
3854                 return -EINVAL;
3855
3856         memset(app_id, 0x0, sizeof(app_id));
3857         memcpy(app_id, &buf[cgrpid_len+1], appid_len);
3858         ret = blkcg_set_fc_appid(app_id, cgrp_id, sizeof(app_id));
3859         if (ret < 0)
3860                 return ret;
3861         return orig_count;
3862 }
3863 static DEVICE_ATTR(appid_store, 0200, NULL, fc_appid_store);
3864 #endif /* CONFIG_BLK_CGROUP_FC_APPID */
3865
3866 static struct attribute *nvme_fc_attrs[] = {
3867         &dev_attr_nvme_discovery.attr,
3868 #ifdef CONFIG_BLK_CGROUP_FC_APPID
3869         &dev_attr_appid_store.attr,
3870 #endif
3871         NULL
3872 };
3873
3874 static const struct attribute_group nvme_fc_attr_group = {
3875         .attrs = nvme_fc_attrs,
3876 };
3877
3878 static const struct attribute_group *nvme_fc_attr_groups[] = {
3879         &nvme_fc_attr_group,
3880         NULL
3881 };
3882
3883 static struct class fc_class = {
3884         .name = "fc",
3885         .dev_groups = nvme_fc_attr_groups,
3886 };
3887
3888 static int __init nvme_fc_init_module(void)
3889 {
3890         int ret;
3891
3892         /*
3893          * NOTE:
3894          * It is expected that in the future the kernel will combine
3895          * the FC-isms that are currently under scsi and now being
3896          * added to by NVME into a new standalone FC class. The SCSI
3897          * and NVME protocols and their devices would be under this
3898          * new FC class.
3899          *
3900          * As we need something to post FC-specific udev events to,
3901          * specifically for nvme probe events, start by creating the
3902          * new device class.  When the new standalone FC class is
3903          * put in place, this code will move to a more generic
3904          * location for the class.
3905          */
3906         ret = class_register(&fc_class);
3907         if (ret) {
3908                 pr_err("couldn't register class fc\n");
3909                 return ret;
3910         }
3911
3912         /*
3913          * Create a device for the FC-centric udev events
3914          */
3915         fc_udev_device = device_create(&fc_class, NULL, MKDEV(0, 0), NULL,
3916                                 "fc_udev_device");
3917         if (IS_ERR(fc_udev_device)) {
3918                 pr_err("couldn't create fc_udev device!\n");
3919                 ret = PTR_ERR(fc_udev_device);
3920                 goto out_destroy_class;
3921         }
3922
3923         ret = nvmf_register_transport(&nvme_fc_transport);
3924         if (ret)
3925                 goto out_destroy_device;
3926
3927         return 0;
3928
3929 out_destroy_device:
3930         device_destroy(&fc_class, MKDEV(0, 0));
3931 out_destroy_class:
3932         class_unregister(&fc_class);
3933
3934         return ret;
3935 }
3936
3937 static void
3938 nvme_fc_delete_controllers(struct nvme_fc_rport *rport)
3939 {
3940         struct nvme_fc_ctrl *ctrl;
3941
3942         spin_lock(&rport->lock);
3943         list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) {
3944                 dev_warn(ctrl->ctrl.device,
3945                         "NVME-FC{%d}: transport unloading: deleting ctrl\n",
3946                         ctrl->cnum);
3947                 nvme_delete_ctrl(&ctrl->ctrl);
3948         }
3949         spin_unlock(&rport->lock);
3950 }
3951
3952 static void __exit nvme_fc_exit_module(void)
3953 {
3954         struct nvme_fc_lport *lport;
3955         struct nvme_fc_rport *rport;
3956         unsigned long flags;
3957
3958         spin_lock_irqsave(&nvme_fc_lock, flags);
3959         list_for_each_entry(lport, &nvme_fc_lport_list, port_list)
3960                 list_for_each_entry(rport, &lport->endp_list, endp_list)
3961                         nvme_fc_delete_controllers(rport);
3962         spin_unlock_irqrestore(&nvme_fc_lock, flags);
3963         flush_workqueue(nvme_delete_wq);
3964
3965         nvmf_unregister_transport(&nvme_fc_transport);
3966
3967         device_destroy(&fc_class, MKDEV(0, 0));
3968         class_unregister(&fc_class);
3969 }
3970
3971 module_init(nvme_fc_init_module);
3972 module_exit(nvme_fc_exit_module);
3973
3974 MODULE_DESCRIPTION("NVMe host FC transport driver");
3975 MODULE_LICENSE("GPL v2");