drivers/nvme/host/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVM Express device driver
   4  * Copyright (c) 2011-2014, Intel Corporation.
   5  */
   6
   7 #include <linux/blkdev.h>
   8 #include <linux/blk-mq.h>
   9 #include <linux/compat.h>
  10 #include <linux/delay.h>
  11 #include <linux/errno.h>
  12 #include <linux/hdreg.h>
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/backing-dev.h>
  16 #include <linux/list_sort.h>
  17 #include <linux/slab.h>
  18 #include <linux/types.h>
  19 #include <linux/pr.h>
  20 #include <linux/ptrace.h>
  21 #include <linux/nvme_ioctl.h>
  22 #include <linux/pm_qos.h>
  23 #include <asm/unaligned.h>
  24
  25 #include "nvme.h"
  26 #include "fabrics.h"
  27
  28 #define CREATE_TRACE_POINTS
  29 #include "trace.h"
  30
  31 #define NVME_MINORS             (1U << MINORBITS)
  32
  33 unsigned int admin_timeout = 60;
  34 module_param(admin_timeout, uint, 0644);
  35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  36 EXPORT_SYMBOL_GPL(admin_timeout);
  37
  38 unsigned int nvme_io_timeout = 30;
  39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
  40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  41 EXPORT_SYMBOL_GPL(nvme_io_timeout);
  42
  43 static unsigned char shutdown_timeout = 5;
  44 module_param(shutdown_timeout, byte, 0644);
  45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  46
  47 static u8 nvme_max_retries = 5;
  48 module_param_named(max_retries, nvme_max_retries, byte, 0644);
  49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
  50
  51 static unsigned long default_ps_max_latency_us = 100000;
  52 module_param(default_ps_max_latency_us, ulong, 0644);
  53 MODULE_PARM_DESC(default_ps_max_latency_us,
  54                  "max power saving latency for new devices; use PM QOS to change per device");
  55
  56 static bool force_apst;
  57 module_param(force_apst, bool, 0644);
  58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
  59
  60 static bool streams;
  61 module_param(streams, bool, 0644);
  62 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
  63
  64 /*
  65  * nvme_wq - hosts nvme related works that are not reset or delete
  66  * nvme_reset_wq - hosts nvme reset works
  67  * nvme_delete_wq - hosts nvme delete works
  68  *
  69  * nvme_wq will host works such as scan, aen handling, fw activation,
  70  * keep-alive, periodic reconnects etc. nvme_reset_wq
  71  * runs reset works which also flush works hosted on nvme_wq for
  72  * serialization purposes. nvme_delete_wq host controller deletion
  73  * works which flush reset works for serialization.
  74  */
  75 struct workqueue_struct *nvme_wq;
  76 EXPORT_SYMBOL_GPL(nvme_wq);
  77
  78 struct workqueue_struct *nvme_reset_wq;
  79 EXPORT_SYMBOL_GPL(nvme_reset_wq);
  80
  81 struct workqueue_struct *nvme_delete_wq;
  82 EXPORT_SYMBOL_GPL(nvme_delete_wq);
  83
  84 static LIST_HEAD(nvme_subsystems);
  85 static DEFINE_MUTEX(nvme_subsystems_lock);
  86
  87 static DEFINE_IDA(nvme_instance_ida);
  88 static dev_t nvme_chr_devt;
  89 static struct class *nvme_class;
  90 static struct class *nvme_subsys_class;
  91
  92 static int _nvme_revalidate_disk(struct gendisk *disk);
  93 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
  94 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
  95                                            unsigned nsid);
  96
  97 static void nvme_update_bdev_size(struct gendisk *disk)
  98 {
  99         struct block_device *bdev = bdget_disk(disk, 0);
 100
 101         if (bdev) {
 102                 bd_set_nr_sectors(bdev, get_capacity(disk));
 103                 bdput(bdev);
 104         }
 105 }
 106
 107 /*
 108  * Prepare a queue for teardown.
 109  *
 110  * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
 111  * the capacity to 0 after that to avoid blocking dispatchers that may be
 112  * holding bd_butex.  This will end buffered writers dirtying pages that can't
 113  * be synced.
 114  */
 115 static void nvme_set_queue_dying(struct nvme_ns *ns)
 116 {
 117         if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
 118                 return;
 119
 120         blk_set_queue_dying(ns->queue);
 121         blk_mq_unquiesce_queue(ns->queue);
 122
 123         set_capacity(ns->disk, 0);
 124         nvme_update_bdev_size(ns->disk);
 125 }
 126
 127 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
 128 {
 129         /*
 130          * Only new queue scan work when admin and IO queues are both alive
 131          */
 132         if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
 133                 queue_work(nvme_wq, &ctrl->scan_work);
 134 }
 135
 136 /*
 137  * Use this function to proceed with scheduling reset_work for a controller
 138  * that had previously been set to the resetting state. This is intended for
 139  * code paths that can't be interrupted by other reset attempts. A hot removal
 140  * may prevent this from succeeding.
 141  */
 142 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
 143 {
 144         if (ctrl->state != NVME_CTRL_RESETTING)
 145                 return -EBUSY;
 146         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 147                 return -EBUSY;
 148         return 0;
 149 }
 150 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
 151
 152 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
 153 {
 154         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 155                 return -EBUSY;
 156         if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
 157                 return -EBUSY;
 158         return 0;
 159 }
 160 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
 161
 162 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
 163 {
 164         int ret;
 165
 166         ret = nvme_reset_ctrl(ctrl);
 167         if (!ret) {
 168                 flush_work(&ctrl->reset_work);
 169                 if (ctrl->state != NVME_CTRL_LIVE)
 170                         ret = -ENETRESET;
 171         }
 172
 173         return ret;
 174 }
 175 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
 176
 177 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
 178 {
 179         dev_info(ctrl->device,
 180                  "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
 181
 182         flush_work(&ctrl->reset_work);
 183         nvme_stop_ctrl(ctrl);
 184         nvme_remove_namespaces(ctrl);
 185         ctrl->ops->delete_ctrl(ctrl);
 186         nvme_uninit_ctrl(ctrl);
 187 }
 188
 189 static void nvme_delete_ctrl_work(struct work_struct *work)
 190 {
 191         struct nvme_ctrl *ctrl =
 192                 container_of(work, struct nvme_ctrl, delete_work);
 193
 194         nvme_do_delete_ctrl(ctrl);
 195 }
 196
 197 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
 198 {
 199         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 200                 return -EBUSY;
 201         if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
 202                 return -EBUSY;
 203         return 0;
 204 }
 205 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
 206
 207 static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
 208 {
 209         /*
 210          * Keep a reference until nvme_do_delete_ctrl() complete,
 211          * since ->delete_ctrl can free the controller.
 212          */
 213         nvme_get_ctrl(ctrl);
 214         if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
 215                 nvme_do_delete_ctrl(ctrl);
 216         nvme_put_ctrl(ctrl);
 217 }
 218
 219 static blk_status_t nvme_error_status(u16 status)
 220 {
 221         switch (status & 0x7ff) {
 222         case NVME_SC_SUCCESS:
 223                 return BLK_STS_OK;
 224         case NVME_SC_CAP_EXCEEDED:
 225                 return BLK_STS_NOSPC;
 226         case NVME_SC_LBA_RANGE:
 227         case NVME_SC_CMD_INTERRUPTED:
 228         case NVME_SC_NS_NOT_READY:
 229                 return BLK_STS_TARGET;
 230         case NVME_SC_BAD_ATTRIBUTES:
 231         case NVME_SC_ONCS_NOT_SUPPORTED:
 232         case NVME_SC_INVALID_OPCODE:
 233         case NVME_SC_INVALID_FIELD:
 234         case NVME_SC_INVALID_NS:
 235                 return BLK_STS_NOTSUPP;
 236         case NVME_SC_WRITE_FAULT:
 237         case NVME_SC_READ_ERROR:
 238         case NVME_SC_UNWRITTEN_BLOCK:
 239         case NVME_SC_ACCESS_DENIED:
 240         case NVME_SC_READ_ONLY:
 241         case NVME_SC_COMPARE_FAILED:
 242                 return BLK_STS_MEDIUM;
 243         case NVME_SC_GUARD_CHECK:
 244         case NVME_SC_APPTAG_CHECK:
 245         case NVME_SC_REFTAG_CHECK:
 246         case NVME_SC_INVALID_PI:
 247                 return BLK_STS_PROTECTION;
 248         case NVME_SC_RESERVATION_CONFLICT:
 249                 return BLK_STS_NEXUS;
 250         case NVME_SC_HOST_PATH_ERROR:
 251                 return BLK_STS_TRANSPORT;
 252         default:
 253                 return BLK_STS_IOERR;
 254         }
 255 }
 256
 257 static void nvme_retry_req(struct request *req)
 258 {
 259         struct nvme_ns *ns = req->q->queuedata;
 260         unsigned long delay = 0;
 261         u16 crd;
 262
 263         /* The mask and shift result must be <= 3 */
 264         crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
 265         if (ns && crd)
 266                 delay = ns->ctrl->crdt[crd - 1] * 100;
 267
 268         nvme_req(req)->retries++;
 269         blk_mq_requeue_request(req, false);
 270         blk_mq_delay_kick_requeue_list(req->q, delay);
 271 }
 272
 273 enum nvme_disposition {
 274         COMPLETE,
 275         RETRY,
 276         FAILOVER,
 277 };
 278
 279 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
 280 {
 281         if (likely(nvme_req(req)->status == 0))
 282                 return COMPLETE;
 283
 284         if (blk_noretry_request(req) ||
 285             (nvme_req(req)->status & NVME_SC_DNR) ||
 286             nvme_req(req)->retries >= nvme_max_retries)
 287                 return COMPLETE;
 288
 289         if (req->cmd_flags & REQ_NVME_MPATH) {
 290                 if (nvme_is_path_error(nvme_req(req)->status) ||
 291                     blk_queue_dying(req->q))
 292                         return FAILOVER;
 293         } else {
 294                 if (blk_queue_dying(req->q))
 295                         return COMPLETE;
 296         }
 297
 298         return RETRY;
 299 }
 300
 301 static inline void nvme_end_req(struct request *req)
 302 {
 303         blk_status_t status = nvme_error_status(nvme_req(req)->status);
 304
 305         if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
 306             req_op(req) == REQ_OP_ZONE_APPEND)
 307                 req->__sector = nvme_lba_to_sect(req->q->queuedata,
 308                         le64_to_cpu(nvme_req(req)->result.u64));
 309
 310         nvme_trace_bio_complete(req, status);
 311         blk_mq_end_request(req, status);
 312 }
 313
 314 void nvme_complete_rq(struct request *req)
 315 {
 316         trace_nvme_complete_rq(req);
 317         nvme_cleanup_cmd(req);
 318
 319         if (nvme_req(req)->ctrl->kas)
 320                 nvme_req(req)->ctrl->comp_seen = true;
 321
 322         switch (nvme_decide_disposition(req)) {
 323         case COMPLETE:
 324                 nvme_end_req(req);
 325                 return;
 326         case RETRY:
 327                 nvme_retry_req(req);
 328                 return;
 329         case FAILOVER:
 330                 nvme_failover_req(req);
 331                 return;
 332         }
 333 }
 334 EXPORT_SYMBOL_GPL(nvme_complete_rq);
 335
 336 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
 337 {
 338         dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
 339                                 "Cancelling I/O %d", req->tag);
 340
 341         /* don't abort one completed request */
 342         if (blk_mq_request_completed(req))
 343                 return true;
 344
 345         nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
 346         blk_mq_complete_request(req);
 347         return true;
 348 }
 349 EXPORT_SYMBOL_GPL(nvme_cancel_request);
 350
 351 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
 352                 enum nvme_ctrl_state new_state)
 353 {
 354         enum nvme_ctrl_state old_state;
 355         unsigned long flags;
 356         bool changed = false;
 357
 358         spin_lock_irqsave(&ctrl->lock, flags);
 359
 360         old_state = ctrl->state;
 361         switch (new_state) {
 362         case NVME_CTRL_LIVE:
 363                 switch (old_state) {
 364                 case NVME_CTRL_NEW:
 365                 case NVME_CTRL_RESETTING:
 366                 case NVME_CTRL_CONNECTING:
 367                         changed = true;
 368                         fallthrough;
 369                 default:
 370                         break;
 371                 }
 372                 break;
 373         case NVME_CTRL_RESETTING:
 374                 switch (old_state) {
 375                 case NVME_CTRL_NEW:
 376                 case NVME_CTRL_LIVE:
 377                         changed = true;
 378                         fallthrough;
 379                 default:
 380                         break;
 381                 }
 382                 break;
 383         case NVME_CTRL_CONNECTING:
 384                 switch (old_state) {
 385                 case NVME_CTRL_NEW:
 386                 case NVME_CTRL_RESETTING:
 387                         changed = true;
 388                         fallthrough;
 389                 default:
 390                         break;
 391                 }
 392                 break;
 393         case NVME_CTRL_DELETING:
 394                 switch (old_state) {
 395                 case NVME_CTRL_LIVE:
 396                 case NVME_CTRL_RESETTING:
 397                 case NVME_CTRL_CONNECTING:
 398                         changed = true;
 399                         fallthrough;
 400                 default:
 401                         break;
 402                 }
 403                 break;
 404         case NVME_CTRL_DELETING_NOIO:
 405                 switch (old_state) {
 406                 case NVME_CTRL_DELETING:
 407                 case NVME_CTRL_DEAD:
 408                         changed = true;
 409                         fallthrough;
 410                 default:
 411                         break;
 412                 }
 413                 break;
 414         case NVME_CTRL_DEAD:
 415                 switch (old_state) {
 416                 case NVME_CTRL_DELETING:
 417                         changed = true;
 418                         fallthrough;
 419                 default:
 420                         break;
 421                 }
 422                 break;
 423         default:
 424                 break;
 425         }
 426
 427         if (changed) {
 428                 ctrl->state = new_state;
 429                 wake_up_all(&ctrl->state_wq);
 430         }
 431
 432         spin_unlock_irqrestore(&ctrl->lock, flags);
 433         if (changed && ctrl->state == NVME_CTRL_LIVE)
 434                 nvme_kick_requeue_lists(ctrl);
 435         return changed;
 436 }
 437 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
 438
 439 /*
 440  * Returns true for sink states that can't ever transition back to live.
 441  */
 442 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
 443 {
 444         switch (ctrl->state) {
 445         case NVME_CTRL_NEW:
 446         case NVME_CTRL_LIVE:
 447         case NVME_CTRL_RESETTING:
 448         case NVME_CTRL_CONNECTING:
 449                 return false;
 450         case NVME_CTRL_DELETING:
 451         case NVME_CTRL_DELETING_NOIO:
 452         case NVME_CTRL_DEAD:
 453                 return true;
 454         default:
 455                 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
 456                 return true;
 457         }
 458 }
 459
 460 /*
 461  * Waits for the controller state to be resetting, or returns false if it is
 462  * not possible to ever transition to that state.
 463  */
 464 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
 465 {
 466         wait_event(ctrl->state_wq,
 467                    nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
 468                    nvme_state_terminal(ctrl));
 469         return ctrl->state == NVME_CTRL_RESETTING;
 470 }
 471 EXPORT_SYMBOL_GPL(nvme_wait_reset);
 472
 473 static void nvme_free_ns_head(struct kref *ref)
 474 {
 475         struct nvme_ns_head *head =
 476                 container_of(ref, struct nvme_ns_head, ref);
 477
 478         nvme_mpath_remove_disk(head);
 479         ida_simple_remove(&head->subsys->ns_ida, head->instance);
 480         cleanup_srcu_struct(&head->srcu);
 481         nvme_put_subsystem(head->subsys);
 482         kfree(head);
 483 }
 484
 485 static void nvme_put_ns_head(struct nvme_ns_head *head)
 486 {
 487         kref_put(&head->ref, nvme_free_ns_head);
 488 }
 489
 490 static void nvme_free_ns(struct kref *kref)
 491 {
 492         struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
 493
 494         if (ns->ndev)
 495                 nvme_nvm_unregister(ns);
 496
 497         put_disk(ns->disk);
 498         nvme_put_ns_head(ns->head);
 499         nvme_put_ctrl(ns->ctrl);
 500         kfree(ns);
 501 }
 502
 503 void nvme_put_ns(struct nvme_ns *ns)
 504 {
 505         kref_put(&ns->kref, nvme_free_ns);
 506 }
 507 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
 508
 509 static inline void nvme_clear_nvme_request(struct request *req)
 510 {
 511         if (!(req->rq_flags & RQF_DONTPREP)) {
 512                 nvme_req(req)->retries = 0;
 513                 nvme_req(req)->flags = 0;
 514                 req->rq_flags |= RQF_DONTPREP;
 515         }
 516 }
 517
 518 struct request *nvme_alloc_request(struct request_queue *q,
 519                 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
 520 {
 521         unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
 522         struct request *req;
 523
 524         if (qid == NVME_QID_ANY) {
 525                 req = blk_mq_alloc_request(q, op, flags);
 526         } else {
 527                 req = blk_mq_alloc_request_hctx(q, op, flags,
 528                                 qid ? qid - 1 : 0);
 529         }
 530         if (IS_ERR(req))
 531                 return req;
 532
 533         req->cmd_flags |= REQ_FAILFAST_DRIVER;
 534         nvme_clear_nvme_request(req);
 535         nvme_req(req)->cmd = cmd;
 536
 537         return req;
 538 }
 539 EXPORT_SYMBOL_GPL(nvme_alloc_request);
 540
 541 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
 542 {
 543         struct nvme_command c;
 544
 545         memset(&c, 0, sizeof(c));
 546
 547         c.directive.opcode = nvme_admin_directive_send;
 548         c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
 549         c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
 550         c.directive.dtype = NVME_DIR_IDENTIFY;
 551         c.directive.tdtype = NVME_DIR_STREAMS;
 552         c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
 553
 554         return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
 555 }
 556
 557 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
 558 {
 559         return nvme_toggle_streams(ctrl, false);
 560 }
 561
 562 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
 563 {
 564         return nvme_toggle_streams(ctrl, true);
 565 }
 566
 567 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
 568                                   struct streams_directive_params *s, u32 nsid)
 569 {
 570         struct nvme_command c;
 571
 572         memset(&c, 0, sizeof(c));
 573         memset(s, 0, sizeof(*s));
 574
 575         c.directive.opcode = nvme_admin_directive_recv;
 576         c.directive.nsid = cpu_to_le32(nsid);
 577         c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s)));
 578         c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
 579         c.directive.dtype = NVME_DIR_STREAMS;
 580
 581         return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
 582 }
 583
 584 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
 585 {
 586         struct streams_directive_params s;
 587         int ret;
 588
 589         if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
 590                 return 0;
 591         if (!streams)
 592                 return 0;
 593
 594         ret = nvme_enable_streams(ctrl);
 595         if (ret)
 596                 return ret;
 597
 598         ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
 599         if (ret)
 600                 goto out_disable_stream;
 601
 602         ctrl->nssa = le16_to_cpu(s.nssa);
 603         if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
 604                 dev_info(ctrl->device, "too few streams (%u) available\n",
 605                                         ctrl->nssa);
 606                 goto out_disable_stream;
 607         }
 608
 609         ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
 610         dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
 611         return 0;
 612
 613 out_disable_stream:
 614         nvme_disable_streams(ctrl);
 615         return ret;
 616 }
 617
 618 /*
 619  * Check if 'req' has a write hint associated with it. If it does, assign
 620  * a valid namespace stream to the write.
 621  */
 622 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
 623                                      struct request *req, u16 *control,
 624                                      u32 *dsmgmt)
 625 {
 626         enum rw_hint streamid = req->write_hint;
 627
 628         if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
 629                 streamid = 0;
 630         else {
 631                 streamid--;
 632                 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
 633                         return;
 634
 635                 *control |= NVME_RW_DTYPE_STREAMS;
 636                 *dsmgmt |= streamid << 16;
 637         }
 638
 639         if (streamid < ARRAY_SIZE(req->q->write_hints))
 640                 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
 641 }
 642
 643 static void nvme_setup_passthrough(struct request *req,
 644                 struct nvme_command *cmd)
 645 {
 646         memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
 647         /* passthru commands should let the driver set the SGL flags */
 648         cmd->common.flags &= ~NVME_CMD_SGL_ALL;
 649 }
 650
 651 static inline void nvme_setup_flush(struct nvme_ns *ns,
 652                 struct nvme_command *cmnd)
 653 {
 654         cmnd->common.opcode = nvme_cmd_flush;
 655         cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
 656 }
 657
 658 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
 659                 struct nvme_command *cmnd)
 660 {
 661         unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
 662         struct nvme_dsm_range *range;
 663         struct bio *bio;
 664
 665         /*
 666          * Some devices do not consider the DSM 'Number of Ranges' field when
 667          * determining how much data to DMA. Always allocate memory for maximum
 668          * number of segments to prevent device reading beyond end of buffer.
 669          */
 670         static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
 671
 672         range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
 673         if (!range) {
 674                 /*
 675                  * If we fail allocation our range, fallback to the controller
 676                  * discard page. If that's also busy, it's safe to return
 677                  * busy, as we know we can make progress once that's freed.
 678                  */
 679                 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
 680                         return BLK_STS_RESOURCE;
 681
 682                 range = page_address(ns->ctrl->discard_page);
 683         }
 684
 685         __rq_for_each_bio(bio, req) {
 686                 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
 687                 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
 688
 689                 if (n < segments) {
 690                         range[n].cattr = cpu_to_le32(0);
 691                         range[n].nlb = cpu_to_le32(nlb);
 692                         range[n].slba = cpu_to_le64(slba);
 693                 }
 694                 n++;
 695         }
 696
 697         if (WARN_ON_ONCE(n != segments)) {
 698                 if (virt_to_page(range) == ns->ctrl->discard_page)
 699                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 700                 else
 701                         kfree(range);
 702                 return BLK_STS_IOERR;
 703         }
 704
 705         cmnd->dsm.opcode = nvme_cmd_dsm;
 706         cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
 707         cmnd->dsm.nr = cpu_to_le32(segments - 1);
 708         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 709
 710         req->special_vec.bv_page = virt_to_page(range);
 711         req->special_vec.bv_offset = offset_in_page(range);
 712         req->special_vec.bv_len = alloc_size;
 713         req->rq_flags |= RQF_SPECIAL_PAYLOAD;
 714
 715         return BLK_STS_OK;
 716 }
 717
 718 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
 719                 struct request *req, struct nvme_command *cmnd)
 720 {
 721         if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
 722                 return nvme_setup_discard(ns, req, cmnd);
 723
 724         cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
 725         cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
 726         cmnd->write_zeroes.slba =
 727                 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 728         cmnd->write_zeroes.length =
 729                 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 730         cmnd->write_zeroes.control = 0;
 731         return BLK_STS_OK;
 732 }
 733
 734 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
 735                 struct request *req, struct nvme_command *cmnd,
 736                 enum nvme_opcode op)
 737 {
 738         struct nvme_ctrl *ctrl = ns->ctrl;
 739         u16 control = 0;
 740         u32 dsmgmt = 0;
 741
 742         if (req->cmd_flags & REQ_FUA)
 743                 control |= NVME_RW_FUA;
 744         if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
 745                 control |= NVME_RW_LR;
 746
 747         if (req->cmd_flags & REQ_RAHEAD)
 748                 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
 749
 750         cmnd->rw.opcode = op;
 751         cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
 752         cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
 753         cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
 754
 755         if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
 756                 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
 757
 758         if (ns->ms) {
 759                 /*
 760                  * If formated with metadata, the block layer always provides a
 761                  * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
 762                  * we enable the PRACT bit for protection information or set the
 763                  * namespace capacity to zero to prevent any I/O.
 764                  */
 765                 if (!blk_integrity_rq(req)) {
 766                         if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
 767                                 return BLK_STS_NOTSUPP;
 768                         control |= NVME_RW_PRINFO_PRACT;
 769                 }
 770
 771                 switch (ns->pi_type) {
 772                 case NVME_NS_DPS_PI_TYPE3:
 773                         control |= NVME_RW_PRINFO_PRCHK_GUARD;
 774                         break;
 775                 case NVME_NS_DPS_PI_TYPE1:
 776                 case NVME_NS_DPS_PI_TYPE2:
 777                         control |= NVME_RW_PRINFO_PRCHK_GUARD |
 778                                         NVME_RW_PRINFO_PRCHK_REF;
 779                         if (op == nvme_cmd_zone_append)
 780                                 control |= NVME_RW_APPEND_PIREMAP;
 781                         cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
 782                         break;
 783                 }
 784         }
 785
 786         cmnd->rw.control = cpu_to_le16(control);
 787         cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
 788         return 0;
 789 }
 790
 791 void nvme_cleanup_cmd(struct request *req)
 792 {
 793         if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
 794                 struct nvme_ns *ns = req->rq_disk->private_data;
 795                 struct page *page = req->special_vec.bv_page;
 796
 797                 if (page == ns->ctrl->discard_page)
 798                         clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
 799                 else
 800                         kfree(page_address(page) + req->special_vec.bv_offset);
 801         }
 802 }
 803 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
 804
 805 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
 806                 struct nvme_command *cmd)
 807 {
 808         blk_status_t ret = BLK_STS_OK;
 809
 810         nvme_clear_nvme_request(req);
 811
 812         memset(cmd, 0, sizeof(*cmd));
 813         switch (req_op(req)) {
 814         case REQ_OP_DRV_IN:
 815         case REQ_OP_DRV_OUT:
 816                 nvme_setup_passthrough(req, cmd);
 817                 break;
 818         case REQ_OP_FLUSH:
 819                 nvme_setup_flush(ns, cmd);
 820                 break;
 821         case REQ_OP_ZONE_RESET_ALL:
 822         case REQ_OP_ZONE_RESET:
 823                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
 824                 break;
 825         case REQ_OP_ZONE_OPEN:
 826                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
 827                 break;
 828         case REQ_OP_ZONE_CLOSE:
 829                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
 830                 break;
 831         case REQ_OP_ZONE_FINISH:
 832                 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
 833                 break;
 834         case REQ_OP_WRITE_ZEROES:
 835                 ret = nvme_setup_write_zeroes(ns, req, cmd);
 836                 break;
 837         case REQ_OP_DISCARD:
 838                 ret = nvme_setup_discard(ns, req, cmd);
 839                 break;
 840         case REQ_OP_READ:
 841                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
 842                 break;
 843         case REQ_OP_WRITE:
 844                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
 845                 break;
 846         case REQ_OP_ZONE_APPEND:
 847                 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
 848                 break;
 849         default:
 850                 WARN_ON_ONCE(1);
 851                 return BLK_STS_IOERR;
 852         }
 853
 854         cmd->common.command_id = req->tag;
 855         trace_nvme_setup_cmd(req, cmd);
 856         return ret;
 857 }
 858 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
 859
 860 static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
 861 {
 862         struct completion *waiting = rq->end_io_data;
 863
 864         rq->end_io_data = NULL;
 865         complete(waiting);
 866 }
 867
 868 static void nvme_execute_rq_polled(struct request_queue *q,
 869                 struct gendisk *bd_disk, struct request *rq, int at_head)
 870 {
 871         DECLARE_COMPLETION_ONSTACK(wait);
 872
 873         WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
 874
 875         rq->cmd_flags |= REQ_HIPRI;
 876         rq->end_io_data = &wait;
 877         blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
 878
 879         while (!completion_done(&wait)) {
 880                 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
 881                 cond_resched();
 882         }
 883 }
 884
 885 /*
 886  * Returns 0 on success.  If the result is negative, it's a Linux error code;
 887  * if the result is positive, it's an NVM Express status code
 888  */
 889 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 890                 union nvme_result *result, void *buffer, unsigned bufflen,
 891                 unsigned timeout, int qid, int at_head,
 892                 blk_mq_req_flags_t flags, bool poll)
 893 {
 894         struct request *req;
 895         int ret;
 896
 897         req = nvme_alloc_request(q, cmd, flags, qid);
 898         if (IS_ERR(req))
 899                 return PTR_ERR(req);
 900
 901         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
 902
 903         if (buffer && bufflen) {
 904                 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
 905                 if (ret)
 906                         goto out;
 907         }
 908
 909         if (poll)
 910                 nvme_execute_rq_polled(req->q, NULL, req, at_head);
 911         else
 912                 blk_execute_rq(req->q, NULL, req, at_head);
 913         if (result)
 914                 *result = nvme_req(req)->result;
 915         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
 916                 ret = -EINTR;
 917         else
 918                 ret = nvme_req(req)->status;
 919  out:
 920         blk_mq_free_request(req);
 921         return ret;
 922 }
 923 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
 924
 925 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
 926                 void *buffer, unsigned bufflen)
 927 {
 928         return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
 929                         NVME_QID_ANY, 0, 0, false);
 930 }
 931 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
 932
 933 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
 934                 unsigned len, u32 seed, bool write)
 935 {
 936         struct bio_integrity_payload *bip;
 937         int ret = -ENOMEM;
 938         void *buf;
 939
 940         buf = kmalloc(len, GFP_KERNEL);
 941         if (!buf)
 942                 goto out;
 943
 944         ret = -EFAULT;
 945         if (write && copy_from_user(buf, ubuf, len))
 946                 goto out_free_meta;
 947
 948         bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
 949         if (IS_ERR(bip)) {
 950                 ret = PTR_ERR(bip);
 951                 goto out_free_meta;
 952         }
 953
 954         bip->bip_iter.bi_size = len;
 955         bip->bip_iter.bi_sector = seed;
 956         ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
 957                         offset_in_page(buf));
 958         if (ret == len)
 959                 return buf;
 960         ret = -ENOMEM;
 961 out_free_meta:
 962         kfree(buf);
 963 out:
 964         return ERR_PTR(ret);
 965 }
 966
 967 static u32 nvme_known_admin_effects(u8 opcode)
 968 {
 969         switch (opcode) {
 970         case nvme_admin_format_nvm:
 971                 return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
 972                         NVME_CMD_EFFECTS_CSE_MASK;
 973         case nvme_admin_sanitize_nvm:
 974                 return NVME_CMD_EFFECTS_CSE_MASK;
 975         default:
 976                 break;
 977         }
 978         return 0;
 979 }
 980
 981 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
 982 {
 983         u32 effects = 0;
 984
 985         if (ns) {
 986                 if (ns->head->effects)
 987                         effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
 988                 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
 989                         dev_warn(ctrl->device,
 990                                  "IO command:%02x has unhandled effects:%08x\n",
 991                                  opcode, effects);
 992                 return 0;
 993         }
 994
 995         if (ctrl->effects)
 996                 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
 997         effects |= nvme_known_admin_effects(opcode);
 998
 999         return effects;
1000 }
1001 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1002
1003 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1004                                u8 opcode)
1005 {
1006         u32 effects = nvme_command_effects(ctrl, ns, opcode);
1007
1008         /*
1009          * For simplicity, IO to all namespaces is quiesced even if the command
1010          * effects say only one namespace is affected.
1011          */
1012         if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1013                 mutex_lock(&ctrl->scan_lock);
1014                 mutex_lock(&ctrl->subsys->lock);
1015                 nvme_mpath_start_freeze(ctrl->subsys);
1016                 nvme_mpath_wait_freeze(ctrl->subsys);
1017                 nvme_start_freeze(ctrl);
1018                 nvme_wait_freeze(ctrl);
1019         }
1020         return effects;
1021 }
1022
1023 static void nvme_update_formats(struct nvme_ctrl *ctrl, u32 *effects)
1024 {
1025         struct nvme_ns *ns;
1026
1027         down_read(&ctrl->namespaces_rwsem);
1028         list_for_each_entry(ns, &ctrl->namespaces, list)
1029                 if (_nvme_revalidate_disk(ns->disk))
1030                         nvme_set_queue_dying(ns);
1031                 else if (blk_queue_is_zoned(ns->disk->queue)) {
1032                         /*
1033                          * IO commands are required to fully revalidate a zoned
1034                          * device. Force the command effects to trigger rescan
1035                          * work so report zones can run in a context with
1036                          * unfrozen IO queues.
1037                          */
1038                         *effects |= NVME_CMD_EFFECTS_NCC;
1039                 }
1040         up_read(&ctrl->namespaces_rwsem);
1041 }
1042
1043 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1044 {
1045         /*
1046          * Revalidate LBA changes prior to unfreezing. This is necessary to
1047          * prevent memory corruption if a logical block size was changed by
1048          * this command.
1049          */
1050         if (effects & NVME_CMD_EFFECTS_LBCC)
1051                 nvme_update_formats(ctrl, &effects);
1052         if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1053                 nvme_unfreeze(ctrl);
1054                 nvme_mpath_unfreeze(ctrl->subsys);
1055                 mutex_unlock(&ctrl->subsys->lock);
1056                 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1057                 mutex_unlock(&ctrl->scan_lock);
1058         }
1059         if (effects & NVME_CMD_EFFECTS_CCC)
1060                 nvme_init_identify(ctrl);
1061         if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1062                 nvme_queue_scan(ctrl);
1063                 flush_work(&ctrl->scan_work);
1064         }
1065 }
1066
1067 void nvme_execute_passthru_rq(struct request *rq)
1068 {
1069         struct nvme_command *cmd = nvme_req(rq)->cmd;
1070         struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1071         struct nvme_ns *ns = rq->q->queuedata;
1072         struct gendisk *disk = ns ? ns->disk : NULL;
1073         u32 effects;
1074
1075         effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1076         blk_execute_rq(rq->q, disk, rq, 0);
1077         nvme_passthru_end(ctrl, effects);
1078 }
1079 EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1080
1081 static int nvme_submit_user_cmd(struct request_queue *q,
1082                 struct nvme_command *cmd, void __user *ubuffer,
1083                 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
1084                 u32 meta_seed, u64 *result, unsigned timeout)
1085 {
1086         bool write = nvme_is_write(cmd);
1087         struct nvme_ns *ns = q->queuedata;
1088         struct gendisk *disk = ns ? ns->disk : NULL;
1089         struct request *req;
1090         struct bio *bio = NULL;
1091         void *meta = NULL;
1092         int ret;
1093
1094         req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
1095         if (IS_ERR(req))
1096                 return PTR_ERR(req);
1097
1098         req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
1099         nvme_req(req)->flags |= NVME_REQ_USERCMD;
1100
1101         if (ubuffer && bufflen) {
1102                 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
1103                                 GFP_KERNEL);
1104                 if (ret)
1105                         goto out;
1106                 bio = req->bio;
1107                 bio->bi_disk = disk;
1108                 if (disk && meta_buffer && meta_len) {
1109                         meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
1110                                         meta_seed, write);
1111                         if (IS_ERR(meta)) {
1112                                 ret = PTR_ERR(meta);
1113                                 goto out_unmap;
1114                         }
1115                         req->cmd_flags |= REQ_INTEGRITY;
1116                 }
1117         }
1118
1119         nvme_execute_passthru_rq(req);
1120         if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
1121                 ret = -EINTR;
1122         else
1123                 ret = nvme_req(req)->status;
1124         if (result)
1125                 *result = le64_to_cpu(nvme_req(req)->result.u64);
1126         if (meta && !ret && !write) {
1127                 if (copy_to_user(meta_buffer, meta, meta_len))
1128                         ret = -EFAULT;
1129         }
1130         kfree(meta);
1131  out_unmap:
1132         if (bio)
1133                 blk_rq_unmap_user(bio);
1134  out:
1135         blk_mq_free_request(req);
1136         return ret;
1137 }
1138
1139 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
1140 {
1141         struct nvme_ctrl *ctrl = rq->end_io_data;
1142         unsigned long flags;
1143         bool startka = false;
1144
1145         blk_mq_free_request(rq);
1146
1147         if (status) {
1148                 dev_err(ctrl->device,
1149                         "failed nvme_keep_alive_end_io error=%d\n",
1150                                 status);
1151                 return;
1152         }
1153
1154         ctrl->comp_seen = false;
1155         spin_lock_irqsave(&ctrl->lock, flags);
1156         if (ctrl->state == NVME_CTRL_LIVE ||
1157             ctrl->state == NVME_CTRL_CONNECTING)
1158                 startka = true;
1159         spin_unlock_irqrestore(&ctrl->lock, flags);
1160         if (startka)
1161                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1162 }
1163
1164 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
1165 {
1166         struct request *rq;
1167
1168         rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
1169                         NVME_QID_ANY);
1170         if (IS_ERR(rq))
1171                 return PTR_ERR(rq);
1172
1173         rq->timeout = ctrl->kato * HZ;
1174         rq->end_io_data = ctrl;
1175
1176         blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
1177
1178         return 0;
1179 }
1180
1181 static void nvme_keep_alive_work(struct work_struct *work)
1182 {
1183         struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1184                         struct nvme_ctrl, ka_work);
1185         bool comp_seen = ctrl->comp_seen;
1186
1187         if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1188                 dev_dbg(ctrl->device,
1189                         "reschedule traffic based keep-alive timer\n");
1190                 ctrl->comp_seen = false;
1191                 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1192                 return;
1193         }
1194
1195         if (nvme_keep_alive(ctrl)) {
1196                 /* allocation failure, reset the controller */
1197                 dev_err(ctrl->device, "keep-alive failed\n");
1198                 nvme_reset_ctrl(ctrl);
1199                 return;
1200         }
1201 }
1202
1203 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1204 {
1205         if (unlikely(ctrl->kato == 0))
1206                 return;
1207
1208         queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
1209 }
1210
1211 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1212 {
1213         if (unlikely(ctrl->kato == 0))
1214                 return;
1215
1216         cancel_delayed_work_sync(&ctrl->ka_work);
1217 }
1218 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1219
1220 /*
1221  * In NVMe 1.0 the CNS field was just a binary controller or namespace
1222  * flag, thus sending any new CNS opcodes has a big chance of not working.
1223  * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1224  * (but not for any later version).
1225  */
1226 static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1227 {
1228         if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1229                 return ctrl->vs < NVME_VS(1, 2, 0);
1230         return ctrl->vs < NVME_VS(1, 1, 0);
1231 }
1232
1233 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1234 {
1235         struct nvme_command c = { };
1236         int error;
1237
1238         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1239         c.identify.opcode = nvme_admin_identify;
1240         c.identify.cns = NVME_ID_CNS_CTRL;
1241
1242         *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1243         if (!*id)
1244                 return -ENOMEM;
1245
1246         error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1247                         sizeof(struct nvme_id_ctrl));
1248         if (error)
1249                 kfree(*id);
1250         return error;
1251 }
1252
1253 static bool nvme_multi_css(struct nvme_ctrl *ctrl)
1254 {
1255         return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
1256 }
1257
1258 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1259                 struct nvme_ns_id_desc *cur, bool *csi_seen)
1260 {
1261         const char *warn_str = "ctrl returned bogus length:";
1262         void *data = cur;
1263
1264         switch (cur->nidt) {
1265         case NVME_NIDT_EUI64:
1266                 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1267                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1268                                  warn_str, cur->nidl);
1269                         return -1;
1270                 }
1271                 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1272                 return NVME_NIDT_EUI64_LEN;
1273         case NVME_NIDT_NGUID:
1274                 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1275                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1276                                  warn_str, cur->nidl);
1277                         return -1;
1278                 }
1279                 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1280                 return NVME_NIDT_NGUID_LEN;
1281         case NVME_NIDT_UUID:
1282                 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1283                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1284                                  warn_str, cur->nidl);
1285                         return -1;
1286                 }
1287                 uuid_copy(&ids->uuid, data + sizeof(*cur));
1288                 return NVME_NIDT_UUID_LEN;
1289         case NVME_NIDT_CSI:
1290                 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1291                         dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1292                                  warn_str, cur->nidl);
1293                         return -1;
1294                 }
1295                 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1296                 *csi_seen = true;
1297                 return NVME_NIDT_CSI_LEN;
1298         default:
1299                 /* Skip unknown types */
1300                 return cur->nidl;
1301         }
1302 }
1303
1304 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1305                 struct nvme_ns_ids *ids)
1306 {
1307         struct nvme_command c = { };
1308         bool csi_seen = false;
1309         int status, pos, len;
1310         void *data;
1311
1312         if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1313                 return 0;
1314
1315         c.identify.opcode = nvme_admin_identify;
1316         c.identify.nsid = cpu_to_le32(nsid);
1317         c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1318
1319         data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1320         if (!data)
1321                 return -ENOMEM;
1322
1323         status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1324                                       NVME_IDENTIFY_DATA_SIZE);
1325         if (status) {
1326                 dev_warn(ctrl->device,
1327                         "Identify Descriptors failed (%d)\n", status);
1328                 goto free_data;
1329         }
1330
1331         for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1332                 struct nvme_ns_id_desc *cur = data + pos;
1333
1334                 if (cur->nidl == 0)
1335                         break;
1336
1337                 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
1338                 if (len < 0)
1339                         break;
1340
1341                 len += sizeof(*cur);
1342         }
1343
1344         if (nvme_multi_css(ctrl) && !csi_seen) {
1345                 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1346                          nsid);
1347                 status = -EINVAL;
1348         }
1349
1350 free_data:
1351         kfree(data);
1352         return status;
1353 }
1354
1355 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
1356 {
1357         struct nvme_command c = { };
1358
1359         c.identify.opcode = nvme_admin_identify;
1360         c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
1361         c.identify.nsid = cpu_to_le32(nsid);
1362         return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
1363                                     NVME_IDENTIFY_DATA_SIZE);
1364 }
1365
1366 static int nvme_identify_ns(struct nvme_ctrl *ctrl,
1367                 unsigned nsid, struct nvme_id_ns **id)
1368 {
1369         struct nvme_command c = { };
1370         int error;
1371
1372         /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1373         c.identify.opcode = nvme_admin_identify;
1374         c.identify.nsid = cpu_to_le32(nsid);
1375         c.identify.cns = NVME_ID_CNS_NS;
1376
1377         *id = kmalloc(sizeof(**id), GFP_KERNEL);
1378         if (!*id)
1379                 return -ENOMEM;
1380
1381         error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1382         if (error) {
1383                 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1384                 kfree(*id);
1385         }
1386
1387         return error;
1388 }
1389
1390 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1391                 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1392 {
1393         union nvme_result res = { 0 };
1394         struct nvme_command c;
1395         int ret;
1396
1397         memset(&c, 0, sizeof(c));
1398         c.features.opcode = op;
1399         c.features.fid = cpu_to_le32(fid);
1400         c.features.dword11 = cpu_to_le32(dword11);
1401
1402         ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1403                         buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1404         if (ret >= 0 && result)
1405                 *result = le32_to_cpu(res.u32);
1406         return ret;
1407 }
1408
1409 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1410                       unsigned int dword11, void *buffer, size_t buflen,
1411                       u32 *result)
1412 {
1413         return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1414                              buflen, result);
1415 }
1416 EXPORT_SYMBOL_GPL(nvme_set_features);
1417
1418 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1419                       unsigned int dword11, void *buffer, size_t buflen,
1420                       u32 *result)
1421 {
1422         return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1423                              buflen, result);
1424 }
1425 EXPORT_SYMBOL_GPL(nvme_get_features);
1426
1427 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1428 {
1429         u32 q_count = (*count - 1) | ((*count - 1) << 16);
1430         u32 result;
1431         int status, nr_io_queues;
1432
1433         status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1434                         &result);
1435         if (status < 0)
1436                 return status;
1437
1438         /*
1439          * Degraded controllers might return an error when setting the queue
1440          * count.  We still want to be able to bring them online and offer
1441          * access to the admin queue, as that might be only way to fix them up.
1442          */
1443         if (status > 0) {
1444                 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1445                 *count = 0;
1446         } else {
1447                 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1448                 *count = min(*count, nr_io_queues);
1449         }
1450
1451         return 0;
1452 }
1453 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1454
1455 #define NVME_AEN_SUPPORTED \
1456         (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1457          NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1458
1459 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1460 {
1461         u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1462         int status;
1463
1464         if (!supported_aens)
1465                 return;
1466
1467         status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1468                         NULL, 0, &result);
1469         if (status)
1470                 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1471                          supported_aens);
1472
1473         queue_work(nvme_wq, &ctrl->async_event_work);
1474 }
1475
1476 /*
1477  * Convert integer values from ioctl structures to user pointers, silently
1478  * ignoring the upper bits in the compat case to match behaviour of 32-bit
1479  * kernels.
1480  */
1481 static void __user *nvme_to_user_ptr(uintptr_t ptrval)
1482 {
1483         if (in_compat_syscall())
1484                 ptrval = (compat_uptr_t)ptrval;
1485         return (void __user *)ptrval;
1486 }
1487
1488 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1489 {
1490         struct nvme_user_io io;
1491         struct nvme_command c;
1492         unsigned length, meta_len;
1493         void __user *metadata;
1494
1495         if (copy_from_user(&io, uio, sizeof(io)))
1496                 return -EFAULT;
1497         if (io.flags)
1498                 return -EINVAL;
1499
1500         switch (io.opcode) {
1501         case nvme_cmd_write:
1502         case nvme_cmd_read:
1503         case nvme_cmd_compare:
1504                 break;
1505         default:
1506                 return -EINVAL;
1507         }
1508
1509         length = (io.nblocks + 1) << ns->lba_shift;
1510         meta_len = (io.nblocks + 1) * ns->ms;
1511         metadata = nvme_to_user_ptr(io.metadata);
1512
1513         if (ns->features & NVME_NS_EXT_LBAS) {
1514                 length += meta_len;
1515                 meta_len = 0;
1516         } else if (meta_len) {
1517                 if ((io.metadata & 3) || !io.metadata)
1518                         return -EINVAL;
1519         }
1520
1521         memset(&c, 0, sizeof(c));
1522         c.rw.opcode = io.opcode;
1523         c.rw.flags = io.flags;
1524         c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1525         c.rw.slba = cpu_to_le64(io.slba);
1526         c.rw.length = cpu_to_le16(io.nblocks);
1527         c.rw.control = cpu_to_le16(io.control);
1528         c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1529         c.rw.reftag = cpu_to_le32(io.reftag);
1530         c.rw.apptag = cpu_to_le16(io.apptag);
1531         c.rw.appmask = cpu_to_le16(io.appmask);
1532
1533         return nvme_submit_user_cmd(ns->queue, &c,
1534                         nvme_to_user_ptr(io.addr), length,
1535                         metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1536 }
1537
1538 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1539                         struct nvme_passthru_cmd __user *ucmd)
1540 {
1541         struct nvme_passthru_cmd cmd;
1542         struct nvme_command c;
1543         unsigned timeout = 0;
1544         u64 result;
1545         int status;
1546
1547         if (!capable(CAP_SYS_ADMIN))
1548                 return -EACCES;
1549         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1550                 return -EFAULT;
1551         if (cmd.flags)
1552                 return -EINVAL;
1553
1554         memset(&c, 0, sizeof(c));
1555         c.common.opcode = cmd.opcode;
1556         c.common.flags = cmd.flags;
1557         c.common.nsid = cpu_to_le32(cmd.nsid);
1558         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1559         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1560         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1561         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1562         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1563         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1564         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1565         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1566
1567         if (cmd.timeout_ms)
1568                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1569
1570         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1571                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1572                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1573                         0, &result, timeout);
1574
1575         if (status >= 0) {
1576                 if (put_user(result, &ucmd->result))
1577                         return -EFAULT;
1578         }
1579
1580         return status;
1581 }
1582
1583 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1584                         struct nvme_passthru_cmd64 __user *ucmd)
1585 {
1586         struct nvme_passthru_cmd64 cmd;
1587         struct nvme_command c;
1588         unsigned timeout = 0;
1589         int status;
1590
1591         if (!capable(CAP_SYS_ADMIN))
1592                 return -EACCES;
1593         if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1594                 return -EFAULT;
1595         if (cmd.flags)
1596                 return -EINVAL;
1597
1598         memset(&c, 0, sizeof(c));
1599         c.common.opcode = cmd.opcode;
1600         c.common.flags = cmd.flags;
1601         c.common.nsid = cpu_to_le32(cmd.nsid);
1602         c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1603         c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1604         c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1605         c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1606         c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1607         c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1608         c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1609         c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1610
1611         if (cmd.timeout_ms)
1612                 timeout = msecs_to_jiffies(cmd.timeout_ms);
1613
1614         status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1615                         nvme_to_user_ptr(cmd.addr), cmd.data_len,
1616                         nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1617                         0, &cmd.result, timeout);
1618
1619         if (status >= 0) {
1620                 if (put_user(cmd.result, &ucmd->result))
1621                         return -EFAULT;
1622         }
1623
1624         return status;
1625 }
1626
1627 /*
1628  * Issue ioctl requests on the first available path.  Note that unlike normal
1629  * block layer requests we will not retry failed request on another controller.
1630  */
1631 struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
1632                 struct nvme_ns_head **head, int *srcu_idx)
1633 {
1634 #ifdef CONFIG_NVME_MULTIPATH
1635         if (disk->fops == &nvme_ns_head_ops) {
1636                 struct nvme_ns *ns;
1637
1638                 *head = disk->private_data;
1639                 *srcu_idx = srcu_read_lock(&(*head)->srcu);
1640                 ns = nvme_find_path(*head);
1641                 if (!ns)
1642                         srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1643                 return ns;
1644         }
1645 #endif
1646         *head = NULL;
1647         *srcu_idx = -1;
1648         return disk->private_data;
1649 }
1650
1651 void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1652 {
1653         if (head)
1654                 srcu_read_unlock(&head->srcu, idx);
1655 }
1656
1657 static bool is_ctrl_ioctl(unsigned int cmd)
1658 {
1659         if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1660                 return true;
1661         if (is_sed_ioctl(cmd))
1662                 return true;
1663         return false;
1664 }
1665
1666 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1667                                   void __user *argp,
1668                                   struct nvme_ns_head *head,
1669                                   int srcu_idx)
1670 {
1671         struct nvme_ctrl *ctrl = ns->ctrl;
1672         int ret;
1673
1674         nvme_get_ctrl(ns->ctrl);
1675         nvme_put_ns_from_disk(head, srcu_idx);
1676
1677         switch (cmd) {
1678         case NVME_IOCTL_ADMIN_CMD:
1679                 ret = nvme_user_cmd(ctrl, NULL, argp);
1680                 break;
1681         case NVME_IOCTL_ADMIN64_CMD:
1682                 ret = nvme_user_cmd64(ctrl, NULL, argp);
1683                 break;
1684         default:
1685                 ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1686                 break;
1687         }
1688         nvme_put_ctrl(ctrl);
1689         return ret;
1690 }
1691
1692 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1693                 unsigned int cmd, unsigned long arg)
1694 {
1695         struct nvme_ns_head *head = NULL;
1696         void __user *argp = (void __user *)arg;
1697         struct nvme_ns *ns;
1698         int srcu_idx, ret;
1699
1700         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1701         if (unlikely(!ns))
1702                 return -EWOULDBLOCK;
1703
1704         /*
1705          * Handle ioctls that apply to the controller instead of the namespace
1706          * seperately and drop the ns SRCU reference early.  This avoids a
1707          * deadlock when deleting namespaces using the passthrough interface.
1708          */
1709         if (is_ctrl_ioctl(cmd))
1710                 return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1711
1712         switch (cmd) {
1713         case NVME_IOCTL_ID:
1714                 force_successful_syscall_return();
1715                 ret = ns->head->ns_id;
1716                 break;
1717         case NVME_IOCTL_IO_CMD:
1718                 ret = nvme_user_cmd(ns->ctrl, ns, argp);
1719                 break;
1720         case NVME_IOCTL_SUBMIT_IO:
1721                 ret = nvme_submit_io(ns, argp);
1722                 break;
1723         case NVME_IOCTL_IO64_CMD:
1724                 ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1725                 break;
1726         default:
1727                 if (ns->ndev)
1728                         ret = nvme_nvm_ioctl(ns, cmd, arg);
1729                 else
1730                         ret = -ENOTTY;
1731         }
1732
1733         nvme_put_ns_from_disk(head, srcu_idx);
1734         return ret;
1735 }
1736
1737 #ifdef CONFIG_COMPAT
1738 struct nvme_user_io32 {
1739         __u8    opcode;
1740         __u8    flags;
1741         __u16   control;
1742         __u16   nblocks;
1743         __u16   rsvd;
1744         __u64   metadata;
1745         __u64   addr;
1746         __u64   slba;
1747         __u32   dsmgmt;
1748         __u32   reftag;
1749         __u16   apptag;
1750         __u16   appmask;
1751 } __attribute__((__packed__));
1752
1753 #define NVME_IOCTL_SUBMIT_IO32  _IOW('N', 0x42, struct nvme_user_io32)
1754
1755 static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1756                 unsigned int cmd, unsigned long arg)
1757 {
1758         /*
1759          * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO
1760          * between 32 bit programs and 64 bit kernel.
1761          * The cause is that the results of sizeof(struct nvme_user_io),
1762          * which is used to define NVME_IOCTL_SUBMIT_IO,
1763          * are not same between 32 bit compiler and 64 bit compiler.
1764          * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling
1765          * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs.
1766          * Other IOCTL numbers are same between 32 bit and 64 bit.
1767          * So there is nothing to do regarding to other IOCTL numbers.
1768          */
1769         if (cmd == NVME_IOCTL_SUBMIT_IO32)
1770                 return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg);
1771
1772         return nvme_ioctl(bdev, mode, cmd, arg);
1773 }
1774 #else
1775 #define nvme_compat_ioctl       NULL
1776 #endif /* CONFIG_COMPAT */
1777
1778 static int nvme_open(struct block_device *bdev, fmode_t mode)
1779 {
1780         struct nvme_ns *ns = bdev->bd_disk->private_data;
1781
1782 #ifdef CONFIG_NVME_MULTIPATH
1783         /* should never be called due to GENHD_FL_HIDDEN */
1784         if (WARN_ON_ONCE(ns->head->disk))
1785                 goto fail;
1786 #endif
1787         if (!kref_get_unless_zero(&ns->kref))
1788                 goto fail;
1789         if (!try_module_get(ns->ctrl->ops->module))
1790                 goto fail_put_ns;
1791
1792         return 0;
1793
1794 fail_put_ns:
1795         nvme_put_ns(ns);
1796 fail:
1797         return -ENXIO;
1798 }
1799
1800 static void nvme_release(struct gendisk *disk, fmode_t mode)
1801 {
1802         struct nvme_ns *ns = disk->private_data;
1803
1804         module_put(ns->ctrl->ops->module);
1805         nvme_put_ns(ns);
1806 }
1807
1808 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1809 {
1810         /* some standard values */
1811         geo->heads = 1 << 6;
1812         geo->sectors = 1 << 5;
1813         geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1814         return 0;
1815 }
1816
1817 #ifdef CONFIG_BLK_DEV_INTEGRITY
1818 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1819                                 u32 max_integrity_segments)
1820 {
1821         struct blk_integrity integrity;
1822
1823         memset(&integrity, 0, sizeof(integrity));
1824         switch (pi_type) {
1825         case NVME_NS_DPS_PI_TYPE3:
1826                 integrity.profile = &t10_pi_type3_crc;
1827                 integrity.tag_size = sizeof(u16) + sizeof(u32);
1828                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1829                 break;
1830         case NVME_NS_DPS_PI_TYPE1:
1831         case NVME_NS_DPS_PI_TYPE2:
1832                 integrity.profile = &t10_pi_type1_crc;
1833                 integrity.tag_size = sizeof(u16);
1834                 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1835                 break;
1836         default:
1837                 integrity.profile = NULL;
1838                 break;
1839         }
1840         integrity.tuple_size = ms;
1841         blk_integrity_register(disk, &integrity);
1842         blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1843 }
1844 #else
1845 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1846                                 u32 max_integrity_segments)
1847 {
1848 }
1849 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1850
1851 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1852 {
1853         struct nvme_ctrl *ctrl = ns->ctrl;
1854         struct request_queue *queue = disk->queue;
1855         u32 size = queue_logical_block_size(queue);
1856
1857         if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1858                 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1859                 return;
1860         }
1861
1862         if (ctrl->nr_streams && ns->sws && ns->sgs)
1863                 size *= ns->sws * ns->sgs;
1864
1865         BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1866                         NVME_DSM_MAX_RANGES);
1867
1868         queue->limits.discard_alignment = 0;
1869         queue->limits.discard_granularity = size;
1870
1871         /* If discard is already enabled, don't reset queue limits */
1872         if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1873                 return;
1874
1875         blk_queue_max_discard_sectors(queue, UINT_MAX);
1876         blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1877
1878         if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1879                 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1880 }
1881
1882 static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns)
1883 {
1884         u64 max_blocks;
1885
1886         if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) ||
1887             (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
1888                 return;
1889         /*
1890          * Even though NVMe spec explicitly states that MDTS is not
1891          * applicable to the write-zeroes:- "The restriction does not apply to
1892          * commands that do not transfer data between the host and the
1893          * controller (e.g., Write Uncorrectable ro Write Zeroes command).".
1894          * In order to be more cautious use controller's max_hw_sectors value
1895          * to configure the maximum sectors for the write-zeroes which is
1896          * configured based on the controller's MDTS field in the
1897          * nvme_init_identify() if available.
1898          */
1899         if (ns->ctrl->max_hw_sectors == UINT_MAX)
1900                 max_blocks = (u64)USHRT_MAX + 1;
1901         else
1902                 max_blocks = ns->ctrl->max_hw_sectors + 1;
1903
1904         blk_queue_max_write_zeroes_sectors(disk->queue,
1905                                            nvme_lba_to_sect(ns, max_blocks));
1906 }
1907
1908 static int nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1909                 struct nvme_id_ns *id, struct nvme_ns_ids *ids)
1910 {
1911         memset(ids, 0, sizeof(*ids));
1912
1913         if (ctrl->vs >= NVME_VS(1, 1, 0))
1914                 memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
1915         if (ctrl->vs >= NVME_VS(1, 2, 0))
1916                 memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
1917         if (ctrl->vs >= NVME_VS(1, 3, 0) || nvme_multi_css(ctrl))
1918                 return nvme_identify_ns_descs(ctrl, nsid, ids);
1919         return 0;
1920 }
1921
1922 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1923 {
1924         return !uuid_is_null(&ids->uuid) ||
1925                 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1926                 memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1927 }
1928
1929 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1930 {
1931         return uuid_equal(&a->uuid, &b->uuid) &&
1932                 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1933                 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1934                 a->csi == b->csi;
1935 }
1936
1937 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1938                                  u32 *phys_bs, u32 *io_opt)
1939 {
1940         struct streams_directive_params s;
1941         int ret;
1942
1943         if (!ctrl->nr_streams)
1944                 return 0;
1945
1946         ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
1947         if (ret)
1948                 return ret;
1949
1950         ns->sws = le32_to_cpu(s.sws);
1951         ns->sgs = le16_to_cpu(s.sgs);
1952
1953         if (ns->sws) {
1954                 *phys_bs = ns->sws * (1 << ns->lba_shift);
1955                 if (ns->sgs)
1956                         *io_opt = *phys_bs * ns->sgs;
1957         }
1958
1959         return 0;
1960 }
1961
1962 static void nvme_update_disk_info(struct gendisk *disk,
1963                 struct nvme_ns *ns, struct nvme_id_ns *id)
1964 {
1965         sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1966         unsigned short bs = 1 << ns->lba_shift;
1967         u32 atomic_bs, phys_bs, io_opt = 0;
1968
1969         if (ns->lba_shift > PAGE_SHIFT) {
1970                 /* unsupported block size, set capacity to 0 later */
1971                 bs = (1 << 9);
1972         }
1973         blk_mq_freeze_queue(disk->queue);
1974         blk_integrity_unregister(disk);
1975
1976         atomic_bs = phys_bs = bs;
1977         nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt);
1978         if (id->nabo == 0) {
1979                 /*
1980                  * Bit 1 indicates whether NAWUPF is defined for this namespace
1981                  * and whether it should be used instead of AWUPF. If NAWUPF ==
1982                  * 0 then AWUPF must be used instead.
1983                  */
1984                 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
1985                         atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1986                 else
1987                         atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1988         }
1989
1990         if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
1991                 /* NPWG = Namespace Preferred Write Granularity */
1992                 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
1993                 /* NOWS = Namespace Optimal Write Size */
1994                 io_opt = bs * (1 + le16_to_cpu(id->nows));
1995         }
1996
1997         blk_queue_logical_block_size(disk->queue, bs);
1998         /*
1999          * Linux filesystems assume writing a single physical block is
2000          * an atomic operation. Hence limit the physical block size to the
2001          * value of the Atomic Write Unit Power Fail parameter.
2002          */
2003         blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
2004         blk_queue_io_min(disk->queue, phys_bs);
2005         blk_queue_io_opt(disk->queue, io_opt);
2006
2007         /*
2008          * The block layer can't support LBA sizes larger than the page size
2009          * yet, so catch this early and don't allow block I/O.
2010          */
2011         if (ns->lba_shift > PAGE_SHIFT)
2012                 capacity = 0;
2013
2014         /*
2015          * Register a metadata profile for PI, or the plain non-integrity NVMe
2016          * metadata masquerading as Type 0 if supported, otherwise reject block
2017          * I/O to namespaces with metadata except when the namespace supports
2018          * PI, as it can strip/insert in that case.
2019          */
2020         if (ns->ms) {
2021                 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
2022                     (ns->features & NVME_NS_METADATA_SUPPORTED))
2023                         nvme_init_integrity(disk, ns->ms, ns->pi_type,
2024                                             ns->ctrl->max_integrity_segments);
2025                 else if (!nvme_ns_has_pi(ns))
2026                         capacity = 0;
2027         }
2028
2029         set_capacity_revalidate_and_notify(disk, capacity, false);
2030
2031         nvme_config_discard(disk, ns);
2032         nvme_config_write_zeroes(disk, ns);
2033
2034         if (id->nsattr & NVME_NS_ATTR_RO)
2035                 set_disk_ro(disk, true);
2036         else
2037                 set_disk_ro(disk, false);
2038
2039         blk_mq_unfreeze_queue(disk->queue);
2040 }
2041
2042 static inline bool nvme_first_scan(struct gendisk *disk)
2043 {
2044         /* nvme_alloc_ns() scans the disk prior to adding it */
2045         return !(disk->flags & GENHD_FL_UP);
2046 }
2047
2048 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
2049 {
2050         struct nvme_ctrl *ctrl = ns->ctrl;
2051         u32 iob;
2052
2053         if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2054             is_power_of_2(ctrl->max_hw_sectors))
2055                 iob = ctrl->max_hw_sectors;
2056         else
2057                 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
2058
2059         if (!iob)
2060                 return;
2061
2062         if (!is_power_of_2(iob)) {
2063                 if (nvme_first_scan(ns->disk))
2064                         pr_warn("%s: ignoring unaligned IO boundary:%u\n",
2065                                 ns->disk->disk_name, iob);
2066                 return;
2067         }
2068
2069         if (blk_queue_is_zoned(ns->disk->queue)) {
2070                 if (nvme_first_scan(ns->disk))
2071                         pr_warn("%s: ignoring zoned namespace IO boundary\n",
2072                                 ns->disk->disk_name);
2073                 return;
2074         }
2075
2076         blk_queue_chunk_sectors(ns->queue, iob);
2077 }
2078
2079 static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
2080 {
2081         unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
2082         struct nvme_ns *ns = disk->private_data;
2083         struct nvme_ctrl *ctrl = ns->ctrl;
2084         int ret;
2085
2086         /*
2087          * If identify namespace failed, use default 512 byte block size so
2088          * block layer can use before failing read/write for 0 capacity.
2089          */
2090         ns->lba_shift = id->lbaf[lbaf].ds;
2091         if (ns->lba_shift == 0)
2092                 ns->lba_shift = 9;
2093
2094         switch (ns->head->ids.csi) {
2095         case NVME_CSI_NVM:
2096                 break;
2097         case NVME_CSI_ZNS:
2098                 ret = nvme_update_zone_info(disk, ns, lbaf);
2099                 if (ret) {
2100                         dev_warn(ctrl->device,
2101                                 "failed to add zoned namespace:%u ret:%d\n",
2102                                 ns->head->ns_id, ret);
2103                         return ret;
2104                 }
2105                 break;
2106         default:
2107                 dev_warn(ctrl->device, "unknown csi:%u ns:%u\n",
2108                         ns->head->ids.csi, ns->head->ns_id);
2109                 return -ENODEV;
2110         }
2111
2112         ns->features = 0;
2113         ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
2114         /* the PI implementation requires metadata equal t10 pi tuple size */
2115         if (ns->ms == sizeof(struct t10_pi_tuple))
2116                 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
2117         else
2118                 ns->pi_type = 0;
2119
2120         if (ns->ms) {
2121                 /*
2122                  * For PCIe only the separate metadata pointer is supported,
2123                  * as the block layer supplies metadata in a separate bio_vec
2124                  * chain. For Fabrics, only metadata as part of extended data
2125                  * LBA is supported on the wire per the Fabrics specification,
2126                  * but the HBA/HCA will do the remapping from the separate
2127                  * metadata buffers for us.
2128                  */
2129                 if (id->flbas & NVME_NS_FLBAS_META_EXT) {
2130                         ns->features |= NVME_NS_EXT_LBAS;
2131                         if ((ctrl->ops->flags & NVME_F_FABRICS) &&
2132                             (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED) &&
2133                             ctrl->max_integrity_segments)
2134                                 ns->features |= NVME_NS_METADATA_SUPPORTED;
2135                 } else {
2136                         if (WARN_ON_ONCE(ctrl->ops->flags & NVME_F_FABRICS))
2137                                 return -EINVAL;
2138                         if (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)
2139                                 ns->features |= NVME_NS_METADATA_SUPPORTED;
2140                 }
2141         }
2142
2143         nvme_set_chunk_sectors(ns, id);
2144         nvme_update_disk_info(disk, ns, id);
2145 #ifdef CONFIG_NVME_MULTIPATH
2146         if (ns->head->disk) {
2147                 nvme_update_disk_info(ns->head->disk, ns, id);
2148                 blk_stack_limits(&ns->head->disk->queue->limits,
2149                                  &ns->queue->limits, 0);
2150                 blk_queue_update_readahead(ns->head->disk->queue);
2151                 nvme_update_bdev_size(ns->head->disk);
2152         }
2153 #endif
2154         return 0;
2155 }
2156
2157 static int _nvme_revalidate_disk(struct gendisk *disk)
2158 {
2159         struct nvme_ns *ns = disk->private_data;
2160         struct nvme_ctrl *ctrl = ns->ctrl;
2161         struct nvme_id_ns *id;
2162         struct nvme_ns_ids ids;
2163         int ret = 0;
2164
2165         if (test_bit(NVME_NS_DEAD, &ns->flags)) {
2166                 set_capacity(disk, 0);
2167                 return -ENODEV;
2168         }
2169
2170         ret = nvme_identify_ns(ctrl, ns->head->ns_id, &id);
2171         if (ret)
2172                 goto out;
2173
2174         if (id->ncap == 0) {
2175                 ret = -ENODEV;
2176                 goto free_id;
2177         }
2178
2179         ret = nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
2180         if (ret)
2181                 goto free_id;
2182
2183         if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
2184                 dev_err(ctrl->device,
2185                         "identifiers changed for nsid %d\n", ns->head->ns_id);
2186                 ret = -ENODEV;
2187                 goto free_id;
2188         }
2189
2190         ret = __nvme_revalidate_disk(disk, id);
2191 free_id:
2192         kfree(id);
2193 out:
2194         /*
2195          * Only fail the function if we got a fatal error back from the
2196          * device, otherwise ignore the error and just move on.
2197          */
2198         if (ret == -ENOMEM || (ret > 0 && !(ret & NVME_SC_DNR)))
2199                 ret = 0;
2200         else if (ret > 0)
2201                 ret = blk_status_to_errno(nvme_error_status(ret));
2202         return ret;
2203 }
2204
2205 static int nvme_revalidate_disk(struct gendisk *disk)
2206 {
2207         int ret;
2208
2209         ret = _nvme_revalidate_disk(disk);
2210         if (ret)
2211                 return ret;
2212
2213 #ifdef CONFIG_BLK_DEV_ZONED
2214         if (blk_queue_is_zoned(disk->queue)) {
2215                 struct nvme_ns *ns = disk->private_data;
2216                 struct nvme_ctrl *ctrl = ns->ctrl;
2217
2218                 ret = blk_revalidate_disk_zones(disk, NULL);
2219                 if (!ret)
2220                         blk_queue_max_zone_append_sectors(disk->queue,
2221                                                           ctrl->max_zone_append);
2222         }
2223 #endif
2224         return ret;
2225 }
2226
2227 static char nvme_pr_type(enum pr_type type)
2228 {
2229         switch (type) {
2230         case PR_WRITE_EXCLUSIVE:
2231                 return 1;
2232         case PR_EXCLUSIVE_ACCESS:
2233                 return 2;
2234         case PR_WRITE_EXCLUSIVE_REG_ONLY:
2235                 return 3;
2236         case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2237                 return 4;
2238         case PR_WRITE_EXCLUSIVE_ALL_REGS:
2239                 return 5;
2240         case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2241                 return 6;
2242         default:
2243                 return 0;
2244         }
2245 };
2246
2247 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2248                                 u64 key, u64 sa_key, u8 op)
2249 {
2250         struct nvme_ns_head *head = NULL;
2251         struct nvme_ns *ns;
2252         struct nvme_command c;
2253         int srcu_idx, ret;
2254         u8 data[16] = { 0, };
2255
2256         ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
2257         if (unlikely(!ns))
2258                 return -EWOULDBLOCK;
2259
2260         put_unaligned_le64(key, &data[0]);
2261         put_unaligned_le64(sa_key, &data[8]);
2262
2263         memset(&c, 0, sizeof(c));
2264         c.common.opcode = op;
2265         c.common.nsid = cpu_to_le32(ns->head->ns_id);
2266         c.common.cdw10 = cpu_to_le32(cdw10);
2267
2268         ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
2269         nvme_put_ns_from_disk(head, srcu_idx);
2270         return ret;
2271 }
2272
2273 static int nvme_pr_register(struct block_device *bdev, u64 old,
2274                 u64 new, unsigned flags)
2275 {
2276         u32 cdw10;
2277
2278         if (flags & ~PR_FL_IGNORE_KEY)
2279                 return -EOPNOTSUPP;
2280
2281         cdw10 = old ? 2 : 0;
2282         cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2283         cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2284         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2285 }
2286
2287 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2288                 enum pr_type type, unsigned flags)
2289 {
2290         u32 cdw10;
2291
2292         if (flags & ~PR_FL_IGNORE_KEY)
2293                 return -EOPNOTSUPP;
2294
2295         cdw10 = nvme_pr_type(type) << 8;
2296         cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2297         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2298 }
2299
2300 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2301                 enum pr_type type, bool abort)
2302 {
2303         u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
2304         return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2305 }
2306
2307 static int nvme_pr_clear(struct block_device *bdev, u64 key)
2308 {
2309         u32 cdw10 = 1 | (key ? 1 << 3 : 0);
2310         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
2311 }
2312
2313 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2314 {
2315         u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
2316         return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2317 }
2318
2319 static const struct pr_ops nvme_pr_ops = {
2320         .pr_register    = nvme_pr_register,
2321         .pr_reserve     = nvme_pr_reserve,
2322         .pr_release     = nvme_pr_release,
2323         .pr_preempt     = nvme_pr_preempt,
2324         .pr_clear       = nvme_pr_clear,
2325 };
2326
2327 #ifdef CONFIG_BLK_SED_OPAL
2328 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2329                 bool send)
2330 {
2331         struct nvme_ctrl *ctrl = data;
2332         struct nvme_command cmd;
2333
2334         memset(&cmd, 0, sizeof(cmd));
2335         if (send)
2336                 cmd.common.opcode = nvme_admin_security_send;
2337         else
2338                 cmd.common.opcode = nvme_admin_security_recv;
2339         cmd.common.nsid = 0;
2340         cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2341         cmd.common.cdw11 = cpu_to_le32(len);
2342
2343         return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2344                                       ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
2345 }
2346 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2347 #endif /* CONFIG_BLK_SED_OPAL */
2348
2349 static const struct block_device_operations nvme_fops = {
2350         .owner          = THIS_MODULE,
2351         .ioctl          = nvme_ioctl,
2352         .compat_ioctl   = nvme_compat_ioctl,
2353         .open           = nvme_open,
2354         .release        = nvme_release,
2355         .getgeo         = nvme_getgeo,
2356         .report_zones   = nvme_report_zones,
2357         .pr_ops         = &nvme_pr_ops,
2358 };
2359
2360 #ifdef CONFIG_NVME_MULTIPATH
2361 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2362 {
2363         struct nvme_ns_head *head = bdev->bd_disk->private_data;
2364
2365         if (!kref_get_unless_zero(&head->ref))
2366                 return -ENXIO;
2367         return 0;
2368 }
2369
2370 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2371 {
2372         nvme_put_ns_head(disk->private_data);
2373 }
2374
2375 const struct block_device_operations nvme_ns_head_ops = {
2376         .owner          = THIS_MODULE,
2377         .submit_bio     = nvme_ns_head_submit_bio,
2378         .open           = nvme_ns_head_open,
2379         .release        = nvme_ns_head_release,
2380         .ioctl          = nvme_ioctl,
2381         .compat_ioctl   = nvme_compat_ioctl,
2382         .getgeo         = nvme_getgeo,
2383         .report_zones   = nvme_report_zones,
2384         .pr_ops         = &nvme_pr_ops,
2385 };
2386 #endif /* CONFIG_NVME_MULTIPATH */
2387
2388 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2389 {
2390         unsigned long timeout =
2391                 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2392         u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2393         int ret;
2394
2395         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2396                 if (csts == ~0)
2397                         return -ENODEV;
2398                 if ((csts & NVME_CSTS_RDY) == bit)
2399                         break;
2400
2401                 usleep_range(1000, 2000);
2402                 if (fatal_signal_pending(current))
2403                         return -EINTR;
2404                 if (time_after(jiffies, timeout)) {
2405                         dev_err(ctrl->device,
2406                                 "Device not ready; aborting %s, CSTS=0x%x\n",
2407                                 enabled ? "initialisation" : "reset", csts);
2408                         return -ENODEV;
2409                 }
2410         }
2411
2412         return ret;
2413 }
2414
2415 /*
2416  * If the device has been passed off to us in an enabled state, just clear
2417  * the enabled bit.  The spec says we should set the 'shutdown notification
2418  * bits', but doing so may cause the device to complete commands to the
2419  * admin queue ... and we don't know what memory that might be pointing at!
2420  */
2421 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2422 {
2423         int ret;
2424
2425         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2426         ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2427
2428         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2429         if (ret)
2430                 return ret;
2431
2432         if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2433                 msleep(NVME_QUIRK_DELAY_AMOUNT);
2434
2435         return nvme_wait_ready(ctrl, ctrl->cap, false);
2436 }
2437 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2438
2439 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2440 {
2441         unsigned dev_page_min;
2442         int ret;
2443
2444         ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2445         if (ret) {
2446                 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2447                 return ret;
2448         }
2449         dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2450
2451         if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2452                 dev_err(ctrl->device,
2453                         "Minimum device page size %u too large for host (%u)\n",
2454                         1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2455                 return -ENODEV;
2456         }
2457
2458         if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2459                 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2460         else
2461                 ctrl->ctrl_config = NVME_CC_CSS_NVM;
2462         ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2463         ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2464         ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2465         ctrl->ctrl_config |= NVME_CC_ENABLE;
2466
2467         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2468         if (ret)
2469                 return ret;
2470         return nvme_wait_ready(ctrl, ctrl->cap, true);
2471 }
2472 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2473
2474 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2475 {
2476         unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2477         u32 csts;
2478         int ret;
2479
2480         ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2481         ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2482
2483         ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2484         if (ret)
2485                 return ret;
2486
2487         while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2488                 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2489                         break;
2490
2491                 msleep(100);
2492                 if (fatal_signal_pending(current))
2493                         return -EINTR;
2494                 if (time_after(jiffies, timeout)) {
2495                         dev_err(ctrl->device,
2496                                 "Device shutdown incomplete; abort shutdown\n");
2497                         return -ENODEV;
2498                 }
2499         }
2500
2501         return ret;
2502 }
2503 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2504
2505 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
2506                 struct request_queue *q)
2507 {
2508         bool vwc = false;
2509
2510         if (ctrl->max_hw_sectors) {
2511                 u32 max_segments =
2512                         (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
2513
2514                 max_segments = min_not_zero(max_segments, ctrl->max_segments);
2515                 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
2516                 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
2517         }
2518         blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
2519         blk_queue_dma_alignment(q, 7);
2520         if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
2521                 vwc = true;
2522         blk_queue_write_cache(q, vwc, vwc);
2523 }
2524
2525 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2526 {
2527         __le64 ts;
2528         int ret;
2529
2530         if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2531                 return 0;
2532
2533         ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2534         ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2535                         NULL);
2536         if (ret)
2537                 dev_warn_once(ctrl->device,
2538                         "could not set timestamp (%d)\n", ret);
2539         return ret;
2540 }
2541
2542 static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2543 {
2544         struct nvme_feat_host_behavior *host;
2545         int ret;
2546
2547         /* Don't bother enabling the feature if retry delay is not reported */
2548         if (!ctrl->crdt[0])
2549                 return 0;
2550
2551         host = kzalloc(sizeof(*host), GFP_KERNEL);
2552         if (!host)
2553                 return 0;
2554
2555         host->acre = NVME_ENABLE_ACRE;
2556         ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2557                                 host, sizeof(*host), NULL);
2558         kfree(host);
2559         return ret;
2560 }
2561
2562 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2563 {
2564         /*
2565          * APST (Autonomous Power State Transition) lets us program a
2566          * table of power state transitions that the controller will
2567          * perform automatically.  We configure it with a simple
2568          * heuristic: we are willing to spend at most 2% of the time
2569          * transitioning between power states.  Therefore, when running
2570          * in any given state, we will enter the next lower-power
2571          * non-operational state after waiting 50 * (enlat + exlat)
2572          * microseconds, as long as that state's exit latency is under
2573          * the requested maximum latency.
2574          *
2575          * We will not autonomously enter any non-operational state for
2576          * which the total latency exceeds ps_max_latency_us.  Users
2577          * can set ps_max_latency_us to zero to turn off APST.
2578          */
2579
2580         unsigned apste;
2581         struct nvme_feat_auto_pst *table;
2582         u64 max_lat_us = 0;
2583         int max_ps = -1;
2584         int ret;
2585
2586         /*
2587          * If APST isn't supported or if we haven't been initialized yet,
2588          * then don't do anything.
2589          */
2590         if (!ctrl->apsta)
2591                 return 0;
2592
2593         if (ctrl->npss > 31) {
2594                 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2595                 return 0;
2596         }
2597
2598         table = kzalloc(sizeof(*table), GFP_KERNEL);
2599         if (!table)
2600                 return 0;
2601
2602         if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2603                 /* Turn off APST. */
2604                 apste = 0;
2605                 dev_dbg(ctrl->device, "APST disabled\n");
2606         } else {
2607                 __le64 target = cpu_to_le64(0);
2608                 int state;
2609
2610                 /*
2611                  * Walk through all states from lowest- to highest-power.
2612                  * According to the spec, lower-numbered states use more
2613                  * power.  NPSS, despite the name, is the index of the
2614                  * lowest-power state, not the number of states.
2615                  */
2616                 for (state = (int)ctrl->npss; state >= 0; state--) {
2617                         u64 total_latency_us, exit_latency_us, transition_ms;
2618
2619                         if (target)
2620                                 table->entries[state] = target;
2621
2622                         /*
2623                          * Don't allow transitions to the deepest state
2624                          * if it's quirked off.
2625                          */
2626                         if (state == ctrl->npss &&
2627                             (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2628                                 continue;
2629
2630                         /*
2631                          * Is this state a useful non-operational state for
2632                          * higher-power states to autonomously transition to?
2633                          */
2634                         if (!(ctrl->psd[state].flags &
2635                               NVME_PS_FLAGS_NON_OP_STATE))
2636                                 continue;
2637
2638                         exit_latency_us =
2639                                 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2640                         if (exit_latency_us > ctrl->ps_max_latency_us)
2641                                 continue;
2642
2643                         total_latency_us =
2644                                 exit_latency_us +
2645                                 le32_to_cpu(ctrl->psd[state].entry_lat);
2646
2647                         /*
2648                          * This state is good.  Use it as the APST idle
2649                          * target for higher power states.
2650                          */
2651                         transition_ms = total_latency_us + 19;
2652                         do_div(transition_ms, 20);
2653                         if (transition_ms > (1 << 24) - 1)
2654                                 transition_ms = (1 << 24) - 1;
2655
2656                         target = cpu_to_le64((state << 3) |
2657                                              (transition_ms << 8));
2658
2659                         if (max_ps == -1)
2660                                 max_ps = state;
2661
2662                         if (total_latency_us > max_lat_us)
2663                                 max_lat_us = total_latency_us;
2664                 }
2665
2666                 apste = 1;
2667
2668                 if (max_ps == -1) {
2669                         dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2670                 } else {
2671                         dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2672                                 max_ps, max_lat_us, (int)sizeof(*table), table);
2673                 }
2674         }
2675
2676         ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2677                                 table, sizeof(*table), NULL);
2678         if (ret)
2679                 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2680
2681         kfree(table);
2682         return ret;
2683 }
2684
2685 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2686 {
2687         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2688         u64 latency;
2689
2690         switch (val) {
2691         case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2692         case PM_QOS_LATENCY_ANY:
2693                 latency = U64_MAX;
2694                 break;
2695
2696         default:
2697                 latency = val;
2698         }
2699
2700         if (ctrl->ps_max_latency_us != latency) {
2701                 ctrl->ps_max_latency_us = latency;
2702                 nvme_configure_apst(ctrl);
2703         }
2704 }
2705
2706 struct nvme_core_quirk_entry {
2707         /*
2708          * NVMe model and firmware strings are padded with spaces.  For
2709          * simplicity, strings in the quirk table are padded with NULLs
2710          * instead.
2711          */
2712         u16 vid;
2713         const char *mn;
2714         const char *fr;
2715         unsigned long quirks;
2716 };
2717
2718 static const struct nvme_core_quirk_entry core_quirks[] = {
2719         {
2720                 /*
2721                  * This Toshiba device seems to die using any APST states.  See:
2722                  * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2723                  */
2724                 .vid = 0x1179,
2725                 .mn = "THNSF5256GPUK TOSHIBA",
2726                 .quirks = NVME_QUIRK_NO_APST,
2727         },
2728         {
2729                 /*
2730                  * This LiteON CL1-3D*-Q11 firmware version has a race
2731                  * condition associated with actions related to suspend to idle
2732                  * LiteON has resolved the problem in future firmware
2733                  */
2734                 .vid = 0x14a4,
2735                 .fr = "22301111",
2736                 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2737         }
2738 };
2739
2740 /* match is null-terminated but idstr is space-padded. */
2741 static bool string_matches(const char *idstr, const char *match, size_t len)
2742 {
2743         size_t matchlen;
2744
2745         if (!match)
2746                 return true;
2747
2748         matchlen = strlen(match);
2749         WARN_ON_ONCE(matchlen > len);
2750
2751         if (memcmp(idstr, match, matchlen))
2752                 return false;
2753
2754         for (; matchlen < len; matchlen++)
2755                 if (idstr[matchlen] != ' ')
2756                         return false;
2757
2758         return true;
2759 }
2760
2761 static bool quirk_matches(const struct nvme_id_ctrl *id,
2762                           const struct nvme_core_quirk_entry *q)
2763 {
2764         return q->vid == le16_to_cpu(id->vid) &&
2765                 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2766                 string_matches(id->fr, q->fr, sizeof(id->fr));
2767 }
2768
2769 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2770                 struct nvme_id_ctrl *id)
2771 {
2772         size_t nqnlen;
2773         int off;
2774
2775         if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2776                 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2777                 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2778                         strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2779                         return;
2780                 }
2781
2782                 if (ctrl->vs >= NVME_VS(1, 2, 1))
2783                         dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2784         }
2785
2786         /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2787         off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2788                         "nqn.2014.08.org.nvmexpress:%04x%04x",
2789                         le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2790         memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2791         off += sizeof(id->sn);
2792         memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2793         off += sizeof(id->mn);
2794         memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2795 }
2796
2797 static void nvme_release_subsystem(struct device *dev)
2798 {
2799         struct nvme_subsystem *subsys =
2800                 container_of(dev, struct nvme_subsystem, dev);
2801
2802         if (subsys->instance >= 0)
2803                 ida_simple_remove(&nvme_instance_ida, subsys->instance);
2804         kfree(subsys);
2805 }
2806
2807 static void nvme_destroy_subsystem(struct kref *ref)
2808 {
2809         struct nvme_subsystem *subsys =
2810                         container_of(ref, struct nvme_subsystem, ref);
2811
2812         mutex_lock(&nvme_subsystems_lock);
2813         list_del(&subsys->entry);
2814         mutex_unlock(&nvme_subsystems_lock);
2815
2816         ida_destroy(&subsys->ns_ida);
2817         device_del(&subsys->dev);
2818         put_device(&subsys->dev);
2819 }
2820
2821 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2822 {
2823         kref_put(&subsys->ref, nvme_destroy_subsystem);
2824 }
2825
2826 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2827 {
2828         struct nvme_subsystem *subsys;
2829
2830         lockdep_assert_held(&nvme_subsystems_lock);
2831
2832         /*
2833          * Fail matches for discovery subsystems. This results
2834          * in each discovery controller bound to a unique subsystem.
2835          * This avoids issues with validating controller values
2836          * that can only be true when there is a single unique subsystem.
2837          * There may be multiple and completely independent entities
2838          * that provide discovery controllers.
2839          */
2840         if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2841                 return NULL;
2842
2843         list_for_each_entry(subsys, &nvme_subsystems, entry) {
2844                 if (strcmp(subsys->subnqn, subsysnqn))
2845                         continue;
2846                 if (!kref_get_unless_zero(&subsys->ref))
2847                         continue;
2848                 return subsys;
2849         }
2850
2851         return NULL;
2852 }
2853
2854 #define SUBSYS_ATTR_RO(_name, _mode, _show)                     \
2855         struct device_attribute subsys_attr_##_name = \
2856                 __ATTR(_name, _mode, _show, NULL)
2857
2858 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2859                                     struct device_attribute *attr,
2860                                     char *buf)
2861 {
2862         struct nvme_subsystem *subsys =
2863                 container_of(dev, struct nvme_subsystem, dev);
2864
2865         return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2866 }
2867 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2868
2869 #define nvme_subsys_show_str_function(field)                            \
2870 static ssize_t subsys_##field##_show(struct device *dev,                \
2871                             struct device_attribute *attr, char *buf)   \
2872 {                                                                       \
2873         struct nvme_subsystem *subsys =                                 \
2874                 container_of(dev, struct nvme_subsystem, dev);          \
2875         return sprintf(buf, "%.*s\n",                                   \
2876                        (int)sizeof(subsys->field), subsys->field);      \
2877 }                                                                       \
2878 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2879
2880 nvme_subsys_show_str_function(model);
2881 nvme_subsys_show_str_function(serial);
2882 nvme_subsys_show_str_function(firmware_rev);
2883
2884 static struct attribute *nvme_subsys_attrs[] = {
2885         &subsys_attr_model.attr,
2886         &subsys_attr_serial.attr,
2887         &subsys_attr_firmware_rev.attr,
2888         &subsys_attr_subsysnqn.attr,
2889 #ifdef CONFIG_NVME_MULTIPATH
2890         &subsys_attr_iopolicy.attr,
2891 #endif
2892         NULL,
2893 };
2894
2895 static struct attribute_group nvme_subsys_attrs_group = {
2896         .attrs = nvme_subsys_attrs,
2897 };
2898
2899 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2900         &nvme_subsys_attrs_group,
2901         NULL,
2902 };
2903
2904 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2905                 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2906 {
2907         struct nvme_ctrl *tmp;
2908
2909         lockdep_assert_held(&nvme_subsystems_lock);
2910
2911         list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2912                 if (nvme_state_terminal(tmp))
2913                         continue;
2914
2915                 if (tmp->cntlid == ctrl->cntlid) {
2916                         dev_err(ctrl->device,
2917                                 "Duplicate cntlid %u with %s, rejecting\n",
2918                                 ctrl->cntlid, dev_name(tmp->device));
2919                         return false;
2920                 }
2921
2922                 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2923                     (ctrl->opts && ctrl->opts->discovery_nqn))
2924                         continue;
2925
2926                 dev_err(ctrl->device,
2927                         "Subsystem does not support multiple controllers\n");
2928                 return false;
2929         }
2930
2931         return true;
2932 }
2933
2934 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2935 {
2936         struct nvme_subsystem *subsys, *found;
2937         int ret;
2938
2939         subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2940         if (!subsys)
2941                 return -ENOMEM;
2942
2943         subsys->instance = -1;
2944         mutex_init(&subsys->lock);
2945         kref_init(&subsys->ref);
2946         INIT_LIST_HEAD(&subsys->ctrls);
2947         INIT_LIST_HEAD(&subsys->nsheads);
2948         nvme_init_subnqn(subsys, ctrl, id);
2949         memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2950         memcpy(subsys->model, id->mn, sizeof(subsys->model));
2951         memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2952         subsys->vendor_id = le16_to_cpu(id->vid);
2953         subsys->cmic = id->cmic;
2954         subsys->awupf = le16_to_cpu(id->awupf);
2955 #ifdef CONFIG_NVME_MULTIPATH
2956         subsys->iopolicy = NVME_IOPOLICY_NUMA;
2957 #endif
2958
2959         subsys->dev.class = nvme_subsys_class;
2960         subsys->dev.release = nvme_release_subsystem;
2961         subsys->dev.groups = nvme_subsys_attrs_groups;
2962         dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2963         device_initialize(&subsys->dev);
2964
2965         mutex_lock(&nvme_subsystems_lock);
2966         found = __nvme_find_get_subsystem(subsys->subnqn);
2967         if (found) {
2968                 put_device(&subsys->dev);
2969                 subsys = found;
2970
2971                 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2972                         ret = -EINVAL;
2973                         goto out_put_subsystem;
2974                 }
2975         } else {
2976                 ret = device_add(&subsys->dev);
2977                 if (ret) {
2978                         dev_err(ctrl->device,
2979                                 "failed to register subsystem device.\n");
2980                         put_device(&subsys->dev);
2981                         goto out_unlock;
2982                 }
2983                 ida_init(&subsys->ns_ida);
2984                 list_add_tail(&subsys->entry, &nvme_subsystems);
2985         }
2986
2987         ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2988                                 dev_name(ctrl->device));
2989         if (ret) {
2990                 dev_err(ctrl->device,
2991                         "failed to create sysfs link from subsystem.\n");
2992                 goto out_put_subsystem;
2993         }
2994
2995         if (!found)
2996                 subsys->instance = ctrl->instance;
2997         ctrl->subsys = subsys;
2998         list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2999         mutex_unlock(&nvme_subsystems_lock);
3000         return 0;
3001
3002 out_put_subsystem:
3003         nvme_put_subsystem(subsys);
3004 out_unlock:
3005         mutex_unlock(&nvme_subsystems_lock);
3006         return ret;
3007 }
3008
3009 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
3010                 void *log, size_t size, u64 offset)
3011 {
3012         struct nvme_command c = { };
3013         u32 dwlen = nvme_bytes_to_numd(size);
3014
3015         c.get_log_page.opcode = nvme_admin_get_log_page;
3016         c.get_log_page.nsid = cpu_to_le32(nsid);
3017         c.get_log_page.lid = log_page;
3018         c.get_log_page.lsp = lsp;
3019         c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
3020         c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
3021         c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
3022         c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
3023         c.get_log_page.csi = csi;
3024
3025         return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
3026 }
3027
3028 static struct nvme_cel *nvme_find_cel(struct nvme_ctrl *ctrl, u8 csi)
3029 {
3030         struct nvme_cel *cel, *ret = NULL;
3031
3032         spin_lock_irq(&ctrl->lock);
3033         list_for_each_entry(cel, &ctrl->cels, entry) {
3034                 if (cel->csi == csi) {
3035                         ret = cel;
3036                         break;
3037                 }
3038         }
3039         spin_unlock_irq(&ctrl->lock);
3040
3041         return ret;
3042 }
3043
3044 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
3045                                 struct nvme_effects_log **log)
3046 {
3047         struct nvme_cel *cel = nvme_find_cel(ctrl, csi);
3048         int ret;
3049
3050         if (cel)
3051                 goto out;
3052
3053         cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3054         if (!cel)
3055                 return -ENOMEM;
3056
3057         ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0, csi,
3058                         &cel->log, sizeof(cel->log), 0);
3059         if (ret) {
3060                 kfree(cel);
3061                 return ret;
3062         }
3063
3064         cel->csi = csi;
3065
3066         spin_lock_irq(&ctrl->lock);
3067         list_add_tail(&cel->entry, &ctrl->cels);
3068         spin_unlock_irq(&ctrl->lock);
3069 out:
3070         *log = &cel->log;
3071         return 0;
3072 }
3073
3074 /*
3075  * Initialize the cached copies of the Identify data and various controller
3076  * register in our nvme_ctrl structure.  This should be called as soon as
3077  * the admin queue is fully up and running.
3078  */
3079 int nvme_init_identify(struct nvme_ctrl *ctrl)
3080 {
3081         struct nvme_id_ctrl *id;
3082         int ret, page_shift;
3083         u32 max_hw_sectors;
3084         bool prev_apst_enabled;
3085
3086         ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3087         if (ret) {
3088                 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3089                 return ret;
3090         }
3091         page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
3092         ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3093
3094         if (ctrl->vs >= NVME_VS(1, 1, 0))
3095                 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3096
3097         ret = nvme_identify_ctrl(ctrl, &id);
3098         if (ret) {
3099                 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3100                 return -EIO;
3101         }
3102
3103         if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3104                 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3105                 if (ret < 0)
3106                         goto out_free;
3107         }
3108
3109         if (!(ctrl->ops->flags & NVME_F_FABRICS))
3110                 ctrl->cntlid = le16_to_cpu(id->cntlid);
3111
3112         if (!ctrl->identified) {
3113                 int i;
3114
3115                 ret = nvme_init_subsystem(ctrl, id);
3116                 if (ret)
3117                         goto out_free;
3118
3119                 /*
3120                  * Check for quirks.  Quirk can depend on firmware version,
3121                  * so, in principle, the set of quirks present can change
3122                  * across a reset.  As a possible future enhancement, we
3123                  * could re-scan for quirks every time we reinitialize
3124                  * the device, but we'd have to make sure that the driver
3125                  * behaves intelligently if the quirks change.
3126                  */
3127                 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3128                         if (quirk_matches(id, &core_quirks[i]))
3129                                 ctrl->quirks |= core_quirks[i].quirks;
3130                 }
3131         }
3132
3133         if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3134                 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3135                 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3136         }
3137
3138         ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3139         ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3140         ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3141
3142         ctrl->oacs = le16_to_cpu(id->oacs);
3143         ctrl->oncs = le16_to_cpu(id->oncs);
3144         ctrl->mtfa = le16_to_cpu(id->mtfa);
3145         ctrl->oaes = le32_to_cpu(id->oaes);
3146         ctrl->wctemp = le16_to_cpu(id->wctemp);
3147         ctrl->cctemp = le16_to_cpu(id->cctemp);
3148
3149         atomic_set(&ctrl->abort_limit, id->acl + 1);
3150         ctrl->vwc = id->vwc;
3151         if (id->mdts)
3152                 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
3153         else
3154                 max_hw_sectors = UINT_MAX;
3155         ctrl->max_hw_sectors =
3156                 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3157
3158         nvme_set_queue_limits(ctrl, ctrl->admin_q);
3159         ctrl->sgls = le32_to_cpu(id->sgls);
3160         ctrl->kas = le16_to_cpu(id->kas);
3161         ctrl->max_namespaces = le32_to_cpu(id->mnan);
3162         ctrl->ctratt = le32_to_cpu(id->ctratt);
3163
3164         if (id->rtd3e) {
3165                 /* us -> s */
3166                 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3167
3168                 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3169                                                  shutdown_timeout, 60);
3170
3171                 if (ctrl->shutdown_timeout != shutdown_timeout)
3172                         dev_info(ctrl->device,
3173                                  "Shutdown timeout set to %u seconds\n",
3174                                  ctrl->shutdown_timeout);
3175         } else
3176                 ctrl->shutdown_timeout = shutdown_timeout;
3177
3178         ctrl->npss = id->npss;
3179         ctrl->apsta = id->apsta;
3180         prev_apst_enabled = ctrl->apst_enabled;
3181         if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3182                 if (force_apst && id->apsta) {
3183                         dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3184                         ctrl->apst_enabled = true;
3185                 } else {
3186                         ctrl->apst_enabled = false;
3187                 }
3188         } else {
3189                 ctrl->apst_enabled = id->apsta;
3190         }
3191         memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3192
3193         if (ctrl->ops->flags & NVME_F_FABRICS) {
3194                 ctrl->icdoff = le16_to_cpu(id->icdoff);
3195                 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3196                 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3197                 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3198
3199                 /*
3200                  * In fabrics we need to verify the cntlid matches the
3201                  * admin connect
3202                  */
3203                 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3204                         dev_err(ctrl->device,
3205                                 "Mismatching cntlid: Connect %u vs Identify "
3206                                 "%u, rejecting\n",
3207                                 ctrl->cntlid, le16_to_cpu(id->cntlid));
3208                         ret = -EINVAL;
3209                         goto out_free;
3210                 }
3211
3212                 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
3213                         dev_err(ctrl->device,
3214                                 "keep-alive support is mandatory for fabrics\n");
3215                         ret = -EINVAL;
3216                         goto out_free;
3217                 }
3218         } else {
3219                 ctrl->hmpre = le32_to_cpu(id->hmpre);
3220                 ctrl->hmmin = le32_to_cpu(id->hmmin);
3221                 ctrl->hmminds = le32_to_cpu(id->hmminds);
3222                 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3223         }
3224
3225         ret = nvme_mpath_init(ctrl, id);
3226         kfree(id);
3227
3228         if (ret < 0)
3229                 return ret;
3230
3231         if (ctrl->apst_enabled && !prev_apst_enabled)
3232                 dev_pm_qos_expose_latency_tolerance(ctrl->device);
3233         else if (!ctrl->apst_enabled && prev_apst_enabled)
3234                 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3235
3236         ret = nvme_configure_apst(ctrl);
3237         if (ret < 0)
3238                 return ret;
3239
3240         ret = nvme_configure_timestamp(ctrl);
3241         if (ret < 0)
3242                 return ret;
3243
3244         ret = nvme_configure_directives(ctrl);
3245         if (ret < 0)
3246                 return ret;
3247
3248         ret = nvme_configure_acre(ctrl);
3249         if (ret < 0)
3250                 return ret;
3251
3252         if (!ctrl->identified)
3253                 nvme_hwmon_init(ctrl);
3254
3255         ctrl->identified = true;
3256
3257         return 0;
3258
3259 out_free:
3260         kfree(id);
3261         return ret;
3262 }
3263 EXPORT_SYMBOL_GPL(nvme_init_identify);
3264
3265 static int nvme_dev_open(struct inode *inode, struct file *file)
3266 {
3267         struct nvme_ctrl *ctrl =
3268                 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3269
3270         switch (ctrl->state) {
3271         case NVME_CTRL_LIVE:
3272                 break;
3273         default:
3274                 return -EWOULDBLOCK;
3275         }
3276
3277         file->private_data = ctrl;
3278         return 0;
3279 }
3280
3281 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
3282 {
3283         struct nvme_ns *ns;
3284         int ret;
3285
3286         down_read(&ctrl->namespaces_rwsem);
3287         if (list_empty(&ctrl->namespaces)) {
3288                 ret = -ENOTTY;
3289                 goto out_unlock;
3290         }
3291
3292         ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
3293         if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
3294                 dev_warn(ctrl->device,
3295                         "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
3296                 ret = -EINVAL;
3297                 goto out_unlock;
3298         }
3299
3300         dev_warn(ctrl->device,
3301                 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
3302         kref_get(&ns->kref);
3303         up_read(&ctrl->namespaces_rwsem);
3304
3305         ret = nvme_user_cmd(ctrl, ns, argp);
3306         nvme_put_ns(ns);
3307         return ret;
3308
3309 out_unlock:
3310         up_read(&ctrl->namespaces_rwsem);
3311         return ret;
3312 }
3313
3314 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
3315                 unsigned long arg)
3316 {
3317         struct nvme_ctrl *ctrl = file->private_data;
3318         void __user *argp = (void __user *)arg;
3319
3320         switch (cmd) {
3321         case NVME_IOCTL_ADMIN_CMD:
3322                 return nvme_user_cmd(ctrl, NULL, argp);
3323         case NVME_IOCTL_ADMIN64_CMD:
3324                 return nvme_user_cmd64(ctrl, NULL, argp);
3325         case NVME_IOCTL_IO_CMD:
3326                 return nvme_dev_user_cmd(ctrl, argp);
3327         case NVME_IOCTL_RESET:
3328                 dev_warn(ctrl->device, "resetting controller\n");
3329                 return nvme_reset_ctrl_sync(ctrl);
3330         case NVME_IOCTL_SUBSYS_RESET:
3331                 return nvme_reset_subsystem(ctrl);
3332         case NVME_IOCTL_RESCAN:
3333                 nvme_queue_scan(ctrl);
3334                 return 0;
3335         default:
3336                 return -ENOTTY;
3337         }
3338 }
3339
3340 static const struct file_operations nvme_dev_fops = {
3341         .owner          = THIS_MODULE,
3342         .open           = nvme_dev_open,
3343         .unlocked_ioctl = nvme_dev_ioctl,
3344         .compat_ioctl   = compat_ptr_ioctl,
3345 };
3346
3347 static ssize_t nvme_sysfs_reset(struct device *dev,
3348                                 struct device_attribute *attr, const char *buf,
3349                                 size_t count)
3350 {
3351         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3352         int ret;
3353
3354         ret = nvme_reset_ctrl_sync(ctrl);
3355         if (ret < 0)
3356                 return ret;
3357         return count;
3358 }
3359 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3360
3361 static ssize_t nvme_sysfs_rescan(struct device *dev,
3362                                 struct device_attribute *attr, const char *buf,
3363                                 size_t count)
3364 {
3365         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3366
3367         nvme_queue_scan(ctrl);
3368         return count;
3369 }
3370 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3371
3372 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3373 {
3374         struct gendisk *disk = dev_to_disk(dev);
3375
3376         if (disk->fops == &nvme_fops)
3377                 return nvme_get_ns_from_dev(dev)->head;
3378         else
3379                 return disk->private_data;
3380 }
3381
3382 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3383                 char *buf)
3384 {
3385         struct nvme_ns_head *head = dev_to_ns_head(dev);
3386         struct nvme_ns_ids *ids = &head->ids;
3387         struct nvme_subsystem *subsys = head->subsys;
3388         int serial_len = sizeof(subsys->serial);
3389         int model_len = sizeof(subsys->model);
3390
3391         if (!uuid_is_null(&ids->uuid))
3392                 return sprintf(buf, "uuid.%pU\n", &ids->uuid);
3393
3394         if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3395                 return sprintf(buf, "eui.%16phN\n", ids->nguid);
3396
3397         if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3398                 return sprintf(buf, "eui.%8phN\n", ids->eui64);
3399
3400         while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3401                                   subsys->serial[serial_len - 1] == '\0'))
3402                 serial_len--;
3403         while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3404                                  subsys->model[model_len - 1] == '\0'))
3405                 model_len--;
3406
3407         return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3408                 serial_len, subsys->serial, model_len, subsys->model,
3409                 head->ns_id);
3410 }
3411 static DEVICE_ATTR_RO(wwid);
3412
3413 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3414                 char *buf)
3415 {
3416         return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3417 }
3418 static DEVICE_ATTR_RO(nguid);
3419
3420 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3421                 char *buf)
3422 {
3423         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3424
3425         /* For backward compatibility expose the NGUID to userspace if
3426          * we have no UUID set
3427          */
3428         if (uuid_is_null(&ids->uuid)) {
3429                 printk_ratelimited(KERN_WARNING
3430                                    "No UUID available providing old NGUID\n");
3431                 return sprintf(buf, "%pU\n", ids->nguid);
3432         }
3433         return sprintf(buf, "%pU\n", &ids->uuid);
3434 }
3435 static DEVICE_ATTR_RO(uuid);
3436
3437 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3438                 char *buf)
3439 {
3440         return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3441 }
3442 static DEVICE_ATTR_RO(eui);
3443
3444 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3445                 char *buf)
3446 {
3447         return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3448 }
3449 static DEVICE_ATTR_RO(nsid);
3450
3451 static struct attribute *nvme_ns_id_attrs[] = {
3452         &dev_attr_wwid.attr,
3453         &dev_attr_uuid.attr,
3454         &dev_attr_nguid.attr,
3455         &dev_attr_eui.attr,
3456         &dev_attr_nsid.attr,
3457 #ifdef CONFIG_NVME_MULTIPATH
3458         &dev_attr_ana_grpid.attr,
3459         &dev_attr_ana_state.attr,
3460 #endif
3461         NULL,
3462 };
3463
3464 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3465                 struct attribute *a, int n)
3466 {
3467         struct device *dev = container_of(kobj, struct device, kobj);
3468         struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3469
3470         if (a == &dev_attr_uuid.attr) {
3471                 if (uuid_is_null(&ids->uuid) &&
3472                     !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3473                         return 0;
3474         }
3475         if (a == &dev_attr_nguid.attr) {
3476                 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3477                         return 0;
3478         }
3479         if (a == &dev_attr_eui.attr) {
3480                 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3481                         return 0;
3482         }
3483 #ifdef CONFIG_NVME_MULTIPATH
3484         if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3485                 if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3486                         return 0;
3487                 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3488                         return 0;
3489         }
3490 #endif
3491         return a->mode;
3492 }
3493
3494 static const struct attribute_group nvme_ns_id_attr_group = {
3495         .attrs          = nvme_ns_id_attrs,
3496         .is_visible     = nvme_ns_id_attrs_are_visible,
3497 };
3498
3499 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3500         &nvme_ns_id_attr_group,
3501 #ifdef CONFIG_NVM
3502         &nvme_nvm_attr_group,
3503 #endif
3504         NULL,
3505 };
3506
3507 #define nvme_show_str_function(field)                                           \
3508 static ssize_t  field##_show(struct device *dev,                                \
3509                             struct device_attribute *attr, char *buf)           \
3510 {                                                                               \
3511         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3512         return sprintf(buf, "%.*s\n",                                           \
3513                 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field);         \
3514 }                                                                               \
3515 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3516
3517 nvme_show_str_function(model);
3518 nvme_show_str_function(serial);
3519 nvme_show_str_function(firmware_rev);
3520
3521 #define nvme_show_int_function(field)                                           \
3522 static ssize_t  field##_show(struct device *dev,                                \
3523                             struct device_attribute *attr, char *buf)           \
3524 {                                                                               \
3525         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);                          \
3526         return sprintf(buf, "%d\n", ctrl->field);       \
3527 }                                                                               \
3528 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3529
3530 nvme_show_int_function(cntlid);
3531 nvme_show_int_function(numa_node);
3532 nvme_show_int_function(queue_count);
3533 nvme_show_int_function(sqsize);
3534
3535 static ssize_t nvme_sysfs_delete(struct device *dev,
3536                                 struct device_attribute *attr, const char *buf,
3537                                 size_t count)
3538 {
3539         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3540
3541         /* Can't delete non-created controllers */
3542         if (!ctrl->created)
3543                 return -EBUSY;
3544
3545         if (device_remove_file_self(dev, attr))
3546                 nvme_delete_ctrl_sync(ctrl);
3547         return count;
3548 }
3549 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3550
3551 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3552                                          struct device_attribute *attr,
3553                                          char *buf)
3554 {
3555         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3556
3557         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3558 }
3559 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3560
3561 static ssize_t nvme_sysfs_show_state(struct device *dev,
3562                                      struct device_attribute *attr,
3563                                      char *buf)
3564 {
3565         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3566         static const char *const state_name[] = {
3567                 [NVME_CTRL_NEW]         = "new",
3568                 [NVME_CTRL_LIVE]        = "live",
3569                 [NVME_CTRL_RESETTING]   = "resetting",
3570                 [NVME_CTRL_CONNECTING]  = "connecting",
3571                 [NVME_CTRL_DELETING]    = "deleting",
3572                 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
3573                 [NVME_CTRL_DEAD]        = "dead",
3574         };
3575
3576         if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3577             state_name[ctrl->state])
3578                 return sprintf(buf, "%s\n", state_name[ctrl->state]);
3579
3580         return sprintf(buf, "unknown state\n");
3581 }
3582
3583 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3584
3585 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3586                                          struct device_attribute *attr,
3587                                          char *buf)
3588 {
3589         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3590
3591         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
3592 }
3593 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3594
3595 static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3596                                         struct device_attribute *attr,
3597                                         char *buf)
3598 {
3599         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3600
3601         return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn);
3602 }
3603 static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3604
3605 static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3606                                         struct device_attribute *attr,
3607                                         char *buf)
3608 {
3609         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3610
3611         return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id);
3612 }
3613 static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3614
3615 static ssize_t nvme_sysfs_show_address(struct device *dev,
3616                                          struct device_attribute *attr,
3617                                          char *buf)
3618 {
3619         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3620
3621         return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3622 }
3623 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3624
3625 static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3626                 struct device_attribute *attr, char *buf)
3627 {
3628         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3629         struct nvmf_ctrl_options *opts = ctrl->opts;
3630
3631         if (ctrl->opts->max_reconnects == -1)
3632                 return sprintf(buf, "off\n");
3633         return sprintf(buf, "%d\n",
3634                         opts->max_reconnects * opts->reconnect_delay);
3635 }
3636
3637 static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3638                 struct device_attribute *attr, const char *buf, size_t count)
3639 {
3640         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3641         struct nvmf_ctrl_options *opts = ctrl->opts;
3642         int ctrl_loss_tmo, err;
3643
3644         err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3645         if (err)
3646                 return -EINVAL;
3647
3648         else if (ctrl_loss_tmo < 0)
3649                 opts->max_reconnects = -1;
3650         else
3651                 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3652                                                 opts->reconnect_delay);
3653         return count;
3654 }
3655 static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3656         nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3657
3658 static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3659                 struct device_attribute *attr, char *buf)
3660 {
3661         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3662
3663         if (ctrl->opts->reconnect_delay == -1)
3664                 return sprintf(buf, "off\n");
3665         return sprintf(buf, "%d\n", ctrl->opts->reconnect_delay);
3666 }
3667
3668 static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3669                 struct device_attribute *attr, const char *buf, size_t count)
3670 {
3671         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3672         unsigned int v;
3673         int err;
3674
3675         err = kstrtou32(buf, 10, &v);
3676         if (err)
3677                 return err;
3678
3679         ctrl->opts->reconnect_delay = v;
3680         return count;
3681 }
3682 static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3683         nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3684
3685 static struct attribute *nvme_dev_attrs[] = {
3686         &dev_attr_reset_controller.attr,
3687         &dev_attr_rescan_controller.attr,
3688         &dev_attr_model.attr,
3689         &dev_attr_serial.attr,
3690         &dev_attr_firmware_rev.attr,
3691         &dev_attr_cntlid.attr,
3692         &dev_attr_delete_controller.attr,
3693         &dev_attr_transport.attr,
3694         &dev_attr_subsysnqn.attr,
3695         &dev_attr_address.attr,
3696         &dev_attr_state.attr,
3697         &dev_attr_numa_node.attr,
3698         &dev_attr_queue_count.attr,
3699         &dev_attr_sqsize.attr,
3700         &dev_attr_hostnqn.attr,
3701         &dev_attr_hostid.attr,
3702         &dev_attr_ctrl_loss_tmo.attr,
3703         &dev_attr_reconnect_delay.attr,
3704         NULL
3705 };
3706
3707 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3708                 struct attribute *a, int n)
3709 {
3710         struct device *dev = container_of(kobj, struct device, kobj);
3711         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3712
3713         if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3714                 return 0;
3715         if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3716                 return 0;
3717         if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3718                 return 0;
3719         if (a == &dev_attr_hostid.attr && !ctrl->opts)
3720                 return 0;
3721         if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3722                 return 0;
3723         if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3724                 return 0;
3725
3726         return a->mode;
3727 }
3728
3729 static struct attribute_group nvme_dev_attrs_group = {
3730         .attrs          = nvme_dev_attrs,
3731         .is_visible     = nvme_dev_attrs_are_visible,
3732 };
3733
3734 static const struct attribute_group *nvme_dev_attr_groups[] = {
3735         &nvme_dev_attrs_group,
3736         NULL,
3737 };
3738
3739 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
3740                 unsigned nsid)
3741 {
3742         struct nvme_ns_head *h;
3743
3744         lockdep_assert_held(&subsys->lock);
3745
3746         list_for_each_entry(h, &subsys->nsheads, entry) {
3747                 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3748                         return h;
3749         }
3750
3751         return NULL;
3752 }
3753
3754 static int __nvme_check_ids(struct nvme_subsystem *subsys,
3755                 struct nvme_ns_head *new)
3756 {
3757         struct nvme_ns_head *h;
3758
3759         lockdep_assert_held(&subsys->lock);
3760
3761         list_for_each_entry(h, &subsys->nsheads, entry) {
3762                 if (nvme_ns_ids_valid(&new->ids) &&
3763                     nvme_ns_ids_equal(&new->ids, &h->ids))
3764                         return -EINVAL;
3765         }
3766
3767         return 0;
3768 }
3769
3770 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3771                 unsigned nsid, struct nvme_ns_ids *ids)
3772 {
3773         struct nvme_ns_head *head;
3774         size_t size = sizeof(*head);
3775         int ret = -ENOMEM;
3776
3777 #ifdef CONFIG_NVME_MULTIPATH
3778         size += num_possible_nodes() * sizeof(struct nvme_ns *);
3779 #endif
3780
3781         head = kzalloc(size, GFP_KERNEL);
3782         if (!head)
3783                 goto out;
3784         ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3785         if (ret < 0)
3786                 goto out_free_head;
3787         head->instance = ret;
3788         INIT_LIST_HEAD(&head->list);
3789         ret = init_srcu_struct(&head->srcu);
3790         if (ret)
3791                 goto out_ida_remove;
3792         head->subsys = ctrl->subsys;
3793         head->ns_id = nsid;
3794         head->ids = *ids;
3795         kref_init(&head->ref);
3796
3797         ret = __nvme_check_ids(ctrl->subsys, head);
3798         if (ret) {
3799                 dev_err(ctrl->device,
3800                         "duplicate IDs for nsid %d\n", nsid);
3801                 goto out_cleanup_srcu;
3802         }
3803
3804         if (head->ids.csi) {
3805                 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3806                 if (ret)
3807                         goto out_cleanup_srcu;
3808         } else
3809                 head->effects = ctrl->effects;
3810
3811         ret = nvme_mpath_alloc_disk(ctrl, head);
3812         if (ret)
3813                 goto out_cleanup_srcu;
3814
3815         list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3816
3817         kref_get(&ctrl->subsys->ref);
3818
3819         return head;
3820 out_cleanup_srcu:
3821         cleanup_srcu_struct(&head->srcu);
3822 out_ida_remove:
3823         ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3824 out_free_head:
3825         kfree(head);
3826 out:
3827         if (ret > 0)
3828                 ret = blk_status_to_errno(nvme_error_status(ret));
3829         return ERR_PTR(ret);
3830 }
3831
3832 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3833                 struct nvme_id_ns *id)
3834 {
3835         struct nvme_ctrl *ctrl = ns->ctrl;
3836         bool is_shared = id->nmic & NVME_NS_NMIC_SHARED;
3837         struct nvme_ns_head *head = NULL;
3838         struct nvme_ns_ids ids;
3839         int ret = 0;
3840
3841         ret = nvme_report_ns_ids(ctrl, nsid, id, &ids);
3842         if (ret) {
3843                 if (ret < 0)
3844                         return ret;
3845                 return blk_status_to_errno(nvme_error_status(ret));
3846         }
3847
3848         mutex_lock(&ctrl->subsys->lock);
3849         head = nvme_find_ns_head(ctrl->subsys, nsid);
3850         if (!head) {
3851                 head = nvme_alloc_ns_head(ctrl, nsid, &ids);
3852                 if (IS_ERR(head)) {
3853                         ret = PTR_ERR(head);
3854                         goto out_unlock;
3855                 }
3856                 head->shared = is_shared;
3857         } else {
3858                 ret = -EINVAL;
3859                 if (!is_shared || !head->shared) {
3860                         dev_err(ctrl->device,
3861                                 "Duplicate unshared namespace %d\n", nsid);
3862                         goto out_put_ns_head;
3863                 }
3864                 if (!nvme_ns_ids_equal(&head->ids, &ids)) {
3865                         dev_err(ctrl->device,
3866                                 "IDs don't match for shared namespace %d\n",
3867                                         nsid);
3868                         goto out_put_ns_head;
3869                 }
3870         }
3871
3872         list_add_tail(&ns->siblings, &head->list);
3873         ns->head = head;
3874         mutex_unlock(&ctrl->subsys->lock);
3875         return 0;
3876
3877 out_put_ns_head:
3878         nvme_put_ns_head(head);
3879 out_unlock:
3880         mutex_unlock(&ctrl->subsys->lock);
3881         return ret;
3882 }
3883
3884 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
3885 {
3886         struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3887         struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3888
3889         return nsa->head->ns_id - nsb->head->ns_id;
3890 }
3891
3892 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3893 {
3894         struct nvme_ns *ns, *ret = NULL;
3895
3896         down_read(&ctrl->namespaces_rwsem);
3897         list_for_each_entry(ns, &ctrl->namespaces, list) {
3898                 if (ns->head->ns_id == nsid) {
3899                         if (!kref_get_unless_zero(&ns->kref))
3900                                 continue;
3901                         ret = ns;
3902                         break;
3903                 }
3904                 if (ns->head->ns_id > nsid)
3905                         break;
3906         }
3907         up_read(&ctrl->namespaces_rwsem);
3908         return ret;
3909 }
3910 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3911
3912 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3913 {
3914         struct nvme_ns *ns;
3915         struct gendisk *disk;
3916         struct nvme_id_ns *id;
3917         char disk_name[DISK_NAME_LEN];
3918         int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
3919
3920         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3921         if (!ns)
3922                 return;
3923
3924         ns->queue = blk_mq_init_queue(ctrl->tagset);
3925         if (IS_ERR(ns->queue))
3926                 goto out_free_ns;
3927
3928         if (ctrl->opts && ctrl->opts->data_digest)
3929                 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
3930
3931         blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3932         if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3933                 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3934
3935         ns->queue->queuedata = ns;
3936         ns->ctrl = ctrl;
3937
3938         kref_init(&ns->kref);
3939         ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
3940
3941         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
3942         nvme_set_queue_limits(ctrl, ns->queue);
3943
3944         ret = nvme_identify_ns(ctrl, nsid, &id);
3945         if (ret)
3946                 goto out_free_queue;
3947
3948         if (id->ncap == 0)      /* no namespace (legacy quirk) */
3949                 goto out_free_id;
3950
3951         ret = nvme_init_ns_head(ns, nsid, id);
3952         if (ret)
3953                 goto out_free_id;
3954         nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3955
3956         disk = alloc_disk_node(0, node);
3957         if (!disk)
3958                 goto out_unlink_ns;
3959
3960         disk->fops = &nvme_fops;
3961         disk->private_data = ns;
3962         disk->queue = ns->queue;
3963         disk->flags = flags;
3964         memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3965         ns->disk = disk;
3966
3967         if (__nvme_revalidate_disk(disk, id))
3968                 goto out_put_disk;
3969
3970         if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3971                 ret = nvme_nvm_register(ns, disk_name, node);
3972                 if (ret) {
3973                         dev_warn(ctrl->device, "LightNVM init failure\n");
3974                         goto out_put_disk;
3975                 }
3976         }
3977
3978         down_write(&ctrl->namespaces_rwsem);
3979         list_add_tail(&ns->list, &ctrl->namespaces);
3980         up_write(&ctrl->namespaces_rwsem);
3981
3982         nvme_get_ctrl(ctrl);
3983
3984         device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3985
3986         nvme_mpath_add_disk(ns, id);
3987         nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3988         kfree(id);
3989
3990         return;
3991  out_put_disk:
3992         /* prevent double queue cleanup */
3993         ns->disk->queue = NULL;
3994         put_disk(ns->disk);
3995  out_unlink_ns:
3996         mutex_lock(&ctrl->subsys->lock);
3997         list_del_rcu(&ns->siblings);
3998         if (list_empty(&ns->head->list))
3999                 list_del_init(&ns->head->entry);
4000         mutex_unlock(&ctrl->subsys->lock);
4001         nvme_put_ns_head(ns->head);
4002  out_free_id:
4003         kfree(id);
4004  out_free_queue:
4005         blk_cleanup_queue(ns->queue);
4006  out_free_ns:
4007         kfree(ns);
4008 }
4009
4010 static void nvme_ns_remove(struct nvme_ns *ns)
4011 {
4012         if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4013                 return;
4014
4015         nvme_fault_inject_fini(&ns->fault_inject);
4016
4017         mutex_lock(&ns->ctrl->subsys->lock);
4018         list_del_rcu(&ns->siblings);
4019         if (list_empty(&ns->head->list))
4020                 list_del_init(&ns->head->entry);
4021         mutex_unlock(&ns->ctrl->subsys->lock);
4022
4023         synchronize_rcu(); /* guarantee not available in head->list */
4024         nvme_mpath_clear_current_path(ns);
4025         synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
4026
4027         if (ns->disk->flags & GENHD_FL_UP) {
4028                 del_gendisk(ns->disk);
4029                 blk_cleanup_queue(ns->queue);
4030                 if (blk_get_integrity(ns->disk))
4031                         blk_integrity_unregister(ns->disk);
4032         }
4033
4034         down_write(&ns->ctrl->namespaces_rwsem);
4035         list_del_init(&ns->list);
4036         up_write(&ns->ctrl->namespaces_rwsem);
4037
4038         nvme_mpath_check_last_path(ns);
4039         nvme_put_ns(ns);
4040 }
4041
4042 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4043 {
4044         struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4045
4046         if (ns) {
4047                 nvme_ns_remove(ns);
4048                 nvme_put_ns(ns);
4049         }
4050 }
4051
4052 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4053 {
4054         struct nvme_ns *ns;
4055         int ret;
4056
4057         ns = nvme_find_get_ns(ctrl, nsid);
4058         if (!ns) {
4059                 nvme_alloc_ns(ctrl, nsid);
4060                 return;
4061         }
4062
4063         ret = nvme_revalidate_disk(ns->disk);
4064         revalidate_disk_size(ns->disk, ret == 0);
4065         if (ret)
4066                 nvme_ns_remove(ns);
4067         nvme_put_ns(ns);
4068 }
4069
4070 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4071                                         unsigned nsid)
4072 {
4073         struct nvme_ns *ns, *next;
4074         LIST_HEAD(rm_list);
4075
4076         down_write(&ctrl->namespaces_rwsem);
4077         list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4078                 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
4079                         list_move_tail(&ns->list, &rm_list);
4080         }
4081         up_write(&ctrl->namespaces_rwsem);
4082
4083         list_for_each_entry_safe(ns, next, &rm_list, list)
4084                 nvme_ns_remove(ns);
4085
4086 }
4087
4088 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4089 {
4090         const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4091         __le32 *ns_list;
4092         u32 prev = 0;
4093         int ret = 0, i;
4094
4095         if (nvme_ctrl_limited_cns(ctrl))
4096                 return -EOPNOTSUPP;
4097
4098         ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4099         if (!ns_list)
4100                 return -ENOMEM;
4101
4102         for (;;) {
4103                 ret = nvme_identify_ns_list(ctrl, prev, ns_list);
4104                 if (ret)
4105                         goto free;
4106
4107                 for (i = 0; i < nr_entries; i++) {
4108                         u32 nsid = le32_to_cpu(ns_list[i]);
4109
4110                         if (!nsid)      /* end of the list? */
4111                                 goto out;
4112                         nvme_validate_ns(ctrl, nsid);
4113                         while (++prev < nsid)
4114                                 nvme_ns_remove_by_nsid(ctrl, prev);
4115                 }
4116         }
4117  out:
4118         nvme_remove_invalid_namespaces(ctrl, prev);
4119  free:
4120         kfree(ns_list);
4121         return ret;
4122 }
4123
4124 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4125 {
4126         struct nvme_id_ctrl *id;
4127         u32 nn, i;
4128
4129         if (nvme_identify_ctrl(ctrl, &id))
4130                 return;
4131         nn = le32_to_cpu(id->nn);
4132         kfree(id);
4133
4134         for (i = 1; i <= nn; i++)
4135                 nvme_validate_ns(ctrl, i);
4136
4137         nvme_remove_invalid_namespaces(ctrl, nn);
4138 }
4139
4140 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4141 {
4142         size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4143         __le32 *log;
4144         int error;
4145
4146         log = kzalloc(log_size, GFP_KERNEL);
4147         if (!log)
4148                 return;
4149
4150         /*
4151          * We need to read the log to clear the AEN, but we don't want to rely
4152          * on it for the changed namespace information as userspace could have
4153          * raced with us in reading the log page, which could cause us to miss
4154          * updates.
4155          */
4156         error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4157                         NVME_CSI_NVM, log, log_size, 0);
4158         if (error)
4159                 dev_warn(ctrl->device,
4160                         "reading changed ns log failed: %d\n", error);
4161
4162         kfree(log);
4163 }
4164
4165 static void nvme_scan_work(struct work_struct *work)
4166 {
4167         struct nvme_ctrl *ctrl =
4168                 container_of(work, struct nvme_ctrl, scan_work);
4169
4170         /* No tagset on a live ctrl means IO queues could not created */
4171         if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
4172                 return;
4173
4174         if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4175                 dev_info(ctrl->device, "rescanning namespaces.\n");
4176                 nvme_clear_changed_ns_log(ctrl);
4177         }
4178
4179         mutex_lock(&ctrl->scan_lock);
4180         if (nvme_scan_ns_list(ctrl) != 0)
4181                 nvme_scan_ns_sequential(ctrl);
4182         mutex_unlock(&ctrl->scan_lock);
4183
4184         down_write(&ctrl->namespaces_rwsem);
4185         list_sort(NULL, &ctrl->namespaces, ns_cmp);
4186         up_write(&ctrl->namespaces_rwsem);
4187 }
4188
4189 /*
4190  * This function iterates the namespace list unlocked to allow recovery from
4191  * controller failure. It is up to the caller to ensure the namespace list is
4192  * not modified by scan work while this function is executing.
4193  */
4194 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4195 {
4196         struct nvme_ns *ns, *next;
4197         LIST_HEAD(ns_list);
4198
4199         /*
4200          * make sure to requeue I/O to all namespaces as these
4201          * might result from the scan itself and must complete
4202          * for the scan_work to make progress
4203          */
4204         nvme_mpath_clear_ctrl_paths(ctrl);
4205
4206         /* prevent racing with ns scanning */
4207         flush_work(&ctrl->scan_work);
4208
4209         /*
4210          * The dead states indicates the controller was not gracefully
4211          * disconnected. In that case, we won't be able to flush any data while
4212          * removing the namespaces' disks; fail all the queues now to avoid
4213          * potentially having to clean up the failed sync later.
4214          */
4215         if (ctrl->state == NVME_CTRL_DEAD)
4216                 nvme_kill_queues(ctrl);
4217
4218         /* this is a no-op when called from the controller reset handler */
4219         nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4220
4221         down_write(&ctrl->namespaces_rwsem);
4222         list_splice_init(&ctrl->namespaces, &ns_list);
4223         up_write(&ctrl->namespaces_rwsem);
4224
4225         list_for_each_entry_safe(ns, next, &ns_list, list)
4226                 nvme_ns_remove(ns);
4227 }
4228 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4229
4230 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4231 {
4232         struct nvme_ctrl *ctrl =
4233                 container_of(dev, struct nvme_ctrl, ctrl_device);
4234         struct nvmf_ctrl_options *opts = ctrl->opts;
4235         int ret;
4236
4237         ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4238         if (ret)
4239                 return ret;
4240
4241         if (opts) {
4242                 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4243                 if (ret)
4244                         return ret;
4245
4246                 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4247                                 opts->trsvcid ?: "none");
4248                 if (ret)
4249                         return ret;
4250
4251                 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4252                                 opts->host_traddr ?: "none");
4253         }
4254         return ret;
4255 }
4256
4257 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4258 {
4259         char *envp[2] = { NULL, NULL };
4260         u32 aen_result = ctrl->aen_result;
4261
4262         ctrl->aen_result = 0;
4263         if (!aen_result)
4264                 return;
4265
4266         envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4267         if (!envp[0])
4268                 return;
4269         kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4270         kfree(envp[0]);
4271 }
4272
4273 static void nvme_async_event_work(struct work_struct *work)
4274 {
4275         struct nvme_ctrl *ctrl =
4276                 container_of(work, struct nvme_ctrl, async_event_work);
4277
4278         nvme_aen_uevent(ctrl);
4279         ctrl->ops->submit_async_event(ctrl);
4280 }
4281
4282 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4283 {
4284
4285         u32 csts;
4286
4287         if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4288                 return false;
4289
4290         if (csts == ~0)
4291                 return false;
4292
4293         return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4294 }
4295
4296 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4297 {
4298         struct nvme_fw_slot_info_log *log;
4299
4300         log = kmalloc(sizeof(*log), GFP_KERNEL);
4301         if (!log)
4302                 return;
4303
4304         if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4305                         log, sizeof(*log), 0))
4306                 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4307         kfree(log);
4308 }
4309
4310 static void nvme_fw_act_work(struct work_struct *work)
4311 {
4312         struct nvme_ctrl *ctrl = container_of(work,
4313                                 struct nvme_ctrl, fw_act_work);
4314         unsigned long fw_act_timeout;
4315
4316         if (ctrl->mtfa)
4317                 fw_act_timeout = jiffies +
4318                                 msecs_to_jiffies(ctrl->mtfa * 100);
4319         else
4320                 fw_act_timeout = jiffies +
4321                                 msecs_to_jiffies(admin_timeout * 1000);
4322
4323         nvme_stop_queues(ctrl);
4324         while (nvme_ctrl_pp_status(ctrl)) {
4325                 if (time_after(jiffies, fw_act_timeout)) {
4326                         dev_warn(ctrl->device,
4327                                 "Fw activation timeout, reset controller\n");
4328                         nvme_try_sched_reset(ctrl);
4329                         return;
4330                 }
4331                 msleep(100);
4332         }
4333
4334         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4335                 return;
4336
4337         nvme_start_queues(ctrl);
4338         /* read FW slot information to clear the AER */
4339         nvme_get_fw_slot_info(ctrl);
4340 }
4341
4342 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4343 {
4344         u32 aer_notice_type = (result & 0xff00) >> 8;
4345
4346         trace_nvme_async_event(ctrl, aer_notice_type);
4347
4348         switch (aer_notice_type) {
4349         case NVME_AER_NOTICE_NS_CHANGED:
4350                 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4351                 nvme_queue_scan(ctrl);
4352                 break;
4353         case NVME_AER_NOTICE_FW_ACT_STARTING:
4354                 /*
4355                  * We are (ab)using the RESETTING state to prevent subsequent
4356                  * recovery actions from interfering with the controller's
4357                  * firmware activation.
4358                  */
4359                 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4360                         queue_work(nvme_wq, &ctrl->fw_act_work);
4361                 break;
4362 #ifdef CONFIG_NVME_MULTIPATH
4363         case NVME_AER_NOTICE_ANA:
4364                 if (!ctrl->ana_log_buf)
4365                         break;
4366                 queue_work(nvme_wq, &ctrl->ana_work);
4367                 break;
4368 #endif
4369         case NVME_AER_NOTICE_DISC_CHANGED:
4370                 ctrl->aen_result = result;
4371                 break;
4372         default:
4373                 dev_warn(ctrl->device, "async event result %08x\n", result);
4374         }
4375 }
4376
4377 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4378                 volatile union nvme_result *res)
4379 {
4380         u32 result = le32_to_cpu(res->u32);
4381         u32 aer_type = result & 0x07;
4382
4383         if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4384                 return;
4385
4386         switch (aer_type) {
4387         case NVME_AER_NOTICE:
4388                 nvme_handle_aen_notice(ctrl, result);
4389                 break;
4390         case NVME_AER_ERROR:
4391         case NVME_AER_SMART:
4392         case NVME_AER_CSS:
4393         case NVME_AER_VS:
4394                 trace_nvme_async_event(ctrl, aer_type);
4395                 ctrl->aen_result = result;
4396                 break;
4397         default:
4398                 break;
4399         }
4400         queue_work(nvme_wq, &ctrl->async_event_work);
4401 }
4402 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4403
4404 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4405 {
4406         nvme_mpath_stop(ctrl);
4407         nvme_stop_keep_alive(ctrl);
4408         flush_work(&ctrl->async_event_work);
4409         cancel_work_sync(&ctrl->fw_act_work);
4410 }
4411 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4412
4413 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4414 {
4415         nvme_start_keep_alive(ctrl);
4416
4417         nvme_enable_aen(ctrl);
4418
4419         if (ctrl->queue_count > 1) {
4420                 nvme_queue_scan(ctrl);
4421                 nvme_start_queues(ctrl);
4422         }
4423         ctrl->created = true;
4424 }
4425 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4426
4427 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4428 {
4429         nvme_fault_inject_fini(&ctrl->fault_inject);
4430         dev_pm_qos_hide_latency_tolerance(ctrl->device);
4431         cdev_device_del(&ctrl->cdev, ctrl->device);
4432         nvme_put_ctrl(ctrl);
4433 }
4434 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4435
4436 static void nvme_free_ctrl(struct device *dev)
4437 {
4438         struct nvme_ctrl *ctrl =
4439                 container_of(dev, struct nvme_ctrl, ctrl_device);
4440         struct nvme_subsystem *subsys = ctrl->subsys;
4441         struct nvme_cel *cel, *next;
4442
4443         if (!subsys || ctrl->instance != subsys->instance)
4444                 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4445
4446         list_for_each_entry_safe(cel, next, &ctrl->cels, entry) {
4447                 list_del(&cel->entry);
4448                 kfree(cel);
4449         }
4450
4451         nvme_mpath_uninit(ctrl);
4452         __free_page(ctrl->discard_page);
4453
4454         if (subsys) {
4455                 mutex_lock(&nvme_subsystems_lock);
4456                 list_del(&ctrl->subsys_entry);
4457                 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4458                 mutex_unlock(&nvme_subsystems_lock);
4459         }
4460
4461         ctrl->ops->free_ctrl(ctrl);
4462
4463         if (subsys)
4464                 nvme_put_subsystem(subsys);
4465 }
4466
4467 /*
4468  * Initialize a NVMe controller structures.  This needs to be called during
4469  * earliest initialization so that we have the initialized structured around
4470  * during probing.
4471  */
4472 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4473                 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4474 {
4475         int ret;
4476
4477         ctrl->state = NVME_CTRL_NEW;
4478         spin_lock_init(&ctrl->lock);
4479         mutex_init(&ctrl->scan_lock);
4480         INIT_LIST_HEAD(&ctrl->namespaces);
4481         INIT_LIST_HEAD(&ctrl->cels);
4482         init_rwsem(&ctrl->namespaces_rwsem);
4483         ctrl->dev = dev;
4484         ctrl->ops = ops;
4485         ctrl->quirks = quirks;
4486         ctrl->numa_node = NUMA_NO_NODE;
4487         INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4488         INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4489         INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4490         INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4491         init_waitqueue_head(&ctrl->state_wq);
4492
4493         INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4494         memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4495         ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4496
4497         BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4498                         PAGE_SIZE);
4499         ctrl->discard_page = alloc_page(GFP_KERNEL);
4500         if (!ctrl->discard_page) {
4501                 ret = -ENOMEM;
4502                 goto out;
4503         }
4504
4505         ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4506         if (ret < 0)
4507                 goto out;
4508         ctrl->instance = ret;
4509
4510         device_initialize(&ctrl->ctrl_device);
4511         ctrl->device = &ctrl->ctrl_device;
4512         ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
4513         ctrl->device->class = nvme_class;
4514         ctrl->device->parent = ctrl->dev;
4515         ctrl->device->groups = nvme_dev_attr_groups;
4516         ctrl->device->release = nvme_free_ctrl;
4517         dev_set_drvdata(ctrl->device, ctrl);
4518         ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4519         if (ret)
4520                 goto out_release_instance;
4521
4522         nvme_get_ctrl(ctrl);
4523         cdev_init(&ctrl->cdev, &nvme_dev_fops);
4524         ctrl->cdev.owner = ops->module;
4525         ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4526         if (ret)
4527                 goto out_free_name;
4528
4529         /*
4530          * Initialize latency tolerance controls.  The sysfs files won't
4531          * be visible to userspace unless the device actually supports APST.
4532          */
4533         ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4534         dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4535                 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4536
4537         nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4538
4539         return 0;
4540 out_free_name:
4541         nvme_put_ctrl(ctrl);
4542         kfree_const(ctrl->device->kobj.name);
4543 out_release_instance:
4544         ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4545 out:
4546         if (ctrl->discard_page)
4547                 __free_page(ctrl->discard_page);
4548         return ret;
4549 }
4550 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4551
4552 /**
4553  * nvme_kill_queues(): Ends all namespace queues
4554  * @ctrl: the dead controller that needs to end
4555  *
4556  * Call this function when the driver determines it is unable to get the
4557  * controller in a state capable of servicing IO.
4558  */
4559 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4560 {
4561         struct nvme_ns *ns;
4562
4563         down_read(&ctrl->namespaces_rwsem);
4564
4565         /* Forcibly unquiesce queues to avoid blocking dispatch */
4566         if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4567                 blk_mq_unquiesce_queue(ctrl->admin_q);
4568
4569         list_for_each_entry(ns, &ctrl->namespaces, list)
4570                 nvme_set_queue_dying(ns);
4571
4572         up_read(&ctrl->namespaces_rwsem);
4573 }
4574 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4575
4576 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4577 {
4578         struct nvme_ns *ns;
4579
4580         down_read(&ctrl->namespaces_rwsem);
4581         list_for_each_entry(ns, &ctrl->namespaces, list)
4582                 blk_mq_unfreeze_queue(ns->queue);
4583         up_read(&ctrl->namespaces_rwsem);
4584 }
4585 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4586
4587 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4588 {
4589         struct nvme_ns *ns;
4590
4591         down_read(&ctrl->namespaces_rwsem);
4592         list_for_each_entry(ns, &ctrl->namespaces, list) {
4593                 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4594                 if (timeout <= 0)
4595                         break;
4596         }
4597         up_read(&ctrl->namespaces_rwsem);
4598         return timeout;
4599 }
4600 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4601
4602 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4603 {
4604         struct nvme_ns *ns;
4605
4606         down_read(&ctrl->namespaces_rwsem);
4607         list_for_each_entry(ns, &ctrl->namespaces, list)
4608                 blk_mq_freeze_queue_wait(ns->queue);
4609         up_read(&ctrl->namespaces_rwsem);
4610 }
4611 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4612
4613 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4614 {
4615         struct nvme_ns *ns;
4616
4617         down_read(&ctrl->namespaces_rwsem);
4618         list_for_each_entry(ns, &ctrl->namespaces, list)
4619                 blk_freeze_queue_start(ns->queue);
4620         up_read(&ctrl->namespaces_rwsem);
4621 }
4622 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4623
4624 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4625 {
4626         struct nvme_ns *ns;
4627
4628         down_read(&ctrl->namespaces_rwsem);
4629         list_for_each_entry(ns, &ctrl->namespaces, list)
4630                 blk_mq_quiesce_queue(ns->queue);
4631         up_read(&ctrl->namespaces_rwsem);
4632 }
4633 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4634
4635 void nvme_start_queues(struct nvme_ctrl *ctrl)
4636 {
4637         struct nvme_ns *ns;
4638
4639         down_read(&ctrl->namespaces_rwsem);
4640         list_for_each_entry(ns, &ctrl->namespaces, list)
4641                 blk_mq_unquiesce_queue(ns->queue);
4642         up_read(&ctrl->namespaces_rwsem);
4643 }
4644 EXPORT_SYMBOL_GPL(nvme_start_queues);
4645
4646
4647 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4648 {
4649         struct nvme_ns *ns;
4650
4651         down_read(&ctrl->namespaces_rwsem);
4652         list_for_each_entry(ns, &ctrl->namespaces, list)
4653                 blk_sync_queue(ns->queue);
4654         up_read(&ctrl->namespaces_rwsem);
4655
4656         if (ctrl->admin_q)
4657                 blk_sync_queue(ctrl->admin_q);
4658 }
4659 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4660
4661 struct nvme_ctrl *nvme_ctrl_get_by_path(const char *path)
4662 {
4663         struct nvme_ctrl *ctrl;
4664         struct file *f;
4665
4666         f = filp_open(path, O_RDWR, 0);
4667         if (IS_ERR(f))
4668                 return ERR_CAST(f);
4669
4670         if (f->f_op != &nvme_dev_fops) {
4671                 ctrl = ERR_PTR(-EINVAL);
4672                 goto out_close;
4673         }
4674
4675         ctrl = f->private_data;
4676         nvme_get_ctrl(ctrl);
4677
4678 out_close:
4679         filp_close(f, NULL);
4680         return ctrl;
4681 }
4682 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_get_by_path, NVME_TARGET_PASSTHRU);
4683
4684 /*
4685  * Check we didn't inadvertently grow the command structure sizes:
4686  */
4687 static inline void _nvme_check_size(void)
4688 {
4689         BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4690         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4691         BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4692         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4693         BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4694         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4695         BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4696         BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4697         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4698         BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4699         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4700         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4701         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4702         BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4703         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
4704         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4705         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4706         BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4707         BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4708 }
4709
4710
4711 static int __init nvme_core_init(void)
4712 {
4713         int result = -ENOMEM;
4714
4715         _nvme_check_size();
4716
4717         nvme_wq = alloc_workqueue("nvme-wq",
4718                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4719         if (!nvme_wq)
4720                 goto out;
4721
4722         nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4723                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4724         if (!nvme_reset_wq)
4725                 goto destroy_wq;
4726
4727         nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4728                         WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4729         if (!nvme_delete_wq)
4730                 goto destroy_reset_wq;
4731
4732         result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
4733         if (result < 0)
4734                 goto destroy_delete_wq;
4735
4736         nvme_class = class_create(THIS_MODULE, "nvme");
4737         if (IS_ERR(nvme_class)) {
4738                 result = PTR_ERR(nvme_class);
4739                 goto unregister_chrdev;
4740         }
4741         nvme_class->dev_uevent = nvme_class_uevent;
4742
4743         nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4744         if (IS_ERR(nvme_subsys_class)) {
4745                 result = PTR_ERR(nvme_subsys_class);
4746                 goto destroy_class;
4747         }
4748         return 0;
4749
4750 destroy_class:
4751         class_destroy(nvme_class);
4752 unregister_chrdev:
4753         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4754 destroy_delete_wq:
4755         destroy_workqueue(nvme_delete_wq);
4756 destroy_reset_wq:
4757         destroy_workqueue(nvme_reset_wq);
4758 destroy_wq:
4759         destroy_workqueue(nvme_wq);
4760 out:
4761         return result;
4762 }
4763
4764 static void __exit nvme_core_exit(void)
4765 {
4766         class_destroy(nvme_subsys_class);
4767         class_destroy(nvme_class);
4768         unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4769         destroy_workqueue(nvme_delete_wq);
4770         destroy_workqueue(nvme_reset_wq);
4771         destroy_workqueue(nvme_wq);
4772         ida_destroy(&nvme_instance_ida);
4773 }
4774
4775 MODULE_LICENSE("GPL");
4776 MODULE_VERSION("1.0");
4777 module_init(nvme_core_init);
4778 module_exit(nvme_core_exit);