drivers/nvme/host/pci.c

   1 /*
   2  * NVM Express device driver
   3  * Copyright (c) 2011-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #include <linux/bitops.h>
  16 #include <linux/blkdev.h>
  17 #include <linux/blk-mq.h>
  18 #include <linux/cpu.h>
  19 #include <linux/delay.h>
  20 #include <linux/errno.h>
  21 #include <linux/fs.h>
  22 #include <linux/genhd.h>
  23 #include <linux/hdreg.h>
  24 #include <linux/idr.h>
  25 #include <linux/init.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/io.h>
  28 #include <linux/kdev_t.h>
  29 #include <linux/kthread.h>
  30 #include <linux/kernel.h>
  31 #include <linux/mm.h>
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/mutex.h>
  35 #include <linux/pci.h>
  36 #include <linux/poison.h>
  37 #include <linux/ptrace.h>
  38 #include <linux/sched.h>
  39 #include <linux/slab.h>
  40 #include <linux/t10-pi.h>
  41 #include <linux/types.h>
  42 #include <linux/io-64-nonatomic-lo-hi.h>
  43 #include <asm/unaligned.h>
  44
  45 #include "nvme.h"
  46
  47 #define NVME_Q_DEPTH            1024
  48 #define NVME_AQ_DEPTH           256
  49 #define SQ_SIZE(depth)          (depth * sizeof(struct nvme_command))
  50 #define CQ_SIZE(depth)          (depth * sizeof(struct nvme_completion))
  51
  52 /*
  53  * We handle AEN commands ourselves and don't even let the
  54  * block layer know about them.
  55  */
  56 #define NVME_NR_AEN_COMMANDS    1
  57 #define NVME_AQ_BLKMQ_DEPTH     (NVME_AQ_DEPTH - NVME_NR_AEN_COMMANDS)
  58
  59 unsigned char admin_timeout = 60;
  60 module_param(admin_timeout, byte, 0644);
  61 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
  62
  63 unsigned char nvme_io_timeout = 30;
  64 module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
  65 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
  66
  67 unsigned char shutdown_timeout = 5;
  68 module_param(shutdown_timeout, byte, 0644);
  69 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
  70
  71 static int use_threaded_interrupts;
  72 module_param(use_threaded_interrupts, int, 0);
  73
  74 static bool use_cmb_sqes = true;
  75 module_param(use_cmb_sqes, bool, 0644);
  76 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
  77
  78 static LIST_HEAD(dev_list);
  79 static struct task_struct *nvme_thread;
  80 static struct workqueue_struct *nvme_workq;
  81 static wait_queue_head_t nvme_kthread_wait;
  82
  83 struct nvme_dev;
  84 struct nvme_queue;
  85 struct nvme_iod;
  86
  87 static int nvme_reset(struct nvme_dev *dev);
  88 static void nvme_process_cq(struct nvme_queue *nvmeq);
  89 static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod);
  90 static void nvme_remove_dead_ctrl(struct nvme_dev *dev);
  91 static void nvme_dev_shutdown(struct nvme_dev *dev);
  92
  93 struct async_cmd_info {
  94         struct kthread_work work;
  95         struct kthread_worker *worker;
  96         int status;
  97         void *ctx;
  98 };
  99
 100 /*
 101  * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 102  */
 103 struct nvme_dev {
 104         struct list_head node;
 105         struct nvme_queue **queues;
 106         struct blk_mq_tag_set tagset;
 107         struct blk_mq_tag_set admin_tagset;
 108         u32 __iomem *dbs;
 109         struct device *dev;
 110         struct dma_pool *prp_page_pool;
 111         struct dma_pool *prp_small_pool;
 112         unsigned queue_count;
 113         unsigned online_queues;
 114         unsigned max_qid;
 115         int q_depth;
 116         u32 db_stride;
 117         struct msix_entry *entry;
 118         void __iomem *bar;
 119         struct work_struct reset_work;
 120         struct work_struct scan_work;
 121         struct work_struct remove_work;
 122         struct mutex shutdown_lock;
 123         bool subsystem;
 124         void __iomem *cmb;
 125         dma_addr_t cmb_dma_addr;
 126         u64 cmb_size;
 127         u32 cmbsz;
 128         unsigned long flags;
 129 #define NVME_CTRL_RESETTING    0
 130
 131         struct nvme_ctrl ctrl;
 132 };
 133
 134 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
 135 {
 136         return container_of(ctrl, struct nvme_dev, ctrl);
 137 }
 138
 139 /*
 140  * An NVM Express queue.  Each device has at least two (one for admin
 141  * commands and one for I/O commands).
 142  */
 143 struct nvme_queue {
 144         struct device *q_dmadev;
 145         struct nvme_dev *dev;
 146         char irqname[24];       /* nvme4294967295-65535\0 */
 147         spinlock_t q_lock;
 148         struct nvme_command *sq_cmds;
 149         struct nvme_command __iomem *sq_cmds_io;
 150         volatile struct nvme_completion *cqes;
 151         struct blk_mq_tags **tags;
 152         dma_addr_t sq_dma_addr;
 153         dma_addr_t cq_dma_addr;
 154         u32 __iomem *q_db;
 155         u16 q_depth;
 156         s16 cq_vector;
 157         u16 sq_head;
 158         u16 sq_tail;
 159         u16 cq_head;
 160         u16 qid;
 161         u8 cq_phase;
 162         u8 cqe_seen;
 163         struct async_cmd_info cmdinfo;
 164 };
 165
 166 /*
 167  * The nvme_iod describes the data in an I/O, including the list of PRP
 168  * entries.  You can't see it in this data structure because C doesn't let
 169  * me express that.  Use nvme_alloc_iod to ensure there's enough space
 170  * allocated to store the PRP list.
 171  */
 172 struct nvme_iod {
 173         unsigned long private;  /* For the use of the submitter of the I/O */
 174         int npages;             /* In the PRP list. 0 means small pool in use */
 175         int offset;             /* Of PRP list */
 176         int nents;              /* Used in scatterlist */
 177         int length;             /* Of data, in bytes */
 178         dma_addr_t first_dma;
 179         struct scatterlist meta_sg[1]; /* metadata requires single contiguous buffer */
 180         struct scatterlist sg[0];
 181 };
 182
 183 /*
 184  * Check we didin't inadvertently grow the command struct
 185  */
 186 static inline void _nvme_check_size(void)
 187 {
 188         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
 189         BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
 190         BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
 191         BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
 192         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
 193         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
 194         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
 195         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
 196         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
 197         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
 198         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
 199         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
 200 }
 201
 202 struct nvme_cmd_info {
 203         int aborted;
 204         struct nvme_queue *nvmeq;
 205         struct nvme_iod *iod;
 206         struct nvme_iod __iod;
 207 };
 208
 209 /*
 210  * Max size of iod being embedded in the request payload
 211  */
 212 #define NVME_INT_PAGES          2
 213 #define NVME_INT_BYTES(dev)     (NVME_INT_PAGES * (dev)->ctrl.page_size)
 214 #define NVME_INT_MASK           0x01
 215
 216 /*
 217  * Will slightly overestimate the number of pages needed.  This is OK
 218  * as it only leads to a small amount of wasted memory for the lifetime of
 219  * the I/O.
 220  */
 221 static int nvme_npages(unsigned size, struct nvme_dev *dev)
 222 {
 223         unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
 224                                       dev->ctrl.page_size);
 225         return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
 226 }
 227
 228 static unsigned int nvme_cmd_size(struct nvme_dev *dev)
 229 {
 230         unsigned int ret = sizeof(struct nvme_cmd_info);
 231
 232         ret += sizeof(struct nvme_iod);
 233         ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
 234         ret += sizeof(struct scatterlist) * NVME_INT_PAGES;
 235
 236         return ret;
 237 }
 238
 239 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 240                                 unsigned int hctx_idx)
 241 {
 242         struct nvme_dev *dev = data;
 243         struct nvme_queue *nvmeq = dev->queues[0];
 244
 245         WARN_ON(hctx_idx != 0);
 246         WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
 247         WARN_ON(nvmeq->tags);
 248
 249         hctx->driver_data = nvmeq;
 250         nvmeq->tags = &dev->admin_tagset.tags[0];
 251         return 0;
 252 }
 253
 254 static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
 255 {
 256         struct nvme_queue *nvmeq = hctx->driver_data;
 257
 258         nvmeq->tags = NULL;
 259 }
 260
 261 static int nvme_admin_init_request(void *data, struct request *req,
 262                                 unsigned int hctx_idx, unsigned int rq_idx,
 263                                 unsigned int numa_node)
 264 {
 265         struct nvme_dev *dev = data;
 266         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
 267         struct nvme_queue *nvmeq = dev->queues[0];
 268
 269         BUG_ON(!nvmeq);
 270         cmd->nvmeq = nvmeq;
 271         return 0;
 272 }
 273
 274 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 275                           unsigned int hctx_idx)
 276 {
 277         struct nvme_dev *dev = data;
 278         struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
 279
 280         if (!nvmeq->tags)
 281                 nvmeq->tags = &dev->tagset.tags[hctx_idx];
 282
 283         WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
 284         hctx->driver_data = nvmeq;
 285         return 0;
 286 }
 287
 288 static int nvme_init_request(void *data, struct request *req,
 289                                 unsigned int hctx_idx, unsigned int rq_idx,
 290                                 unsigned int numa_node)
 291 {
 292         struct nvme_dev *dev = data;
 293         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
 294         struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
 295
 296         BUG_ON(!nvmeq);
 297         cmd->nvmeq = nvmeq;
 298         return 0;
 299 }
 300
 301 static void *iod_get_private(struct nvme_iod *iod)
 302 {
 303         return (void *) (iod->private & ~0x1UL);
 304 }
 305
 306 /*
 307  * If bit 0 is set, the iod is embedded in the request payload.
 308  */
 309 static bool iod_should_kfree(struct nvme_iod *iod)
 310 {
 311         return (iod->private & NVME_INT_MASK) == 0;
 312 }
 313
 314 static void nvme_complete_async_event(struct nvme_dev *dev,
 315                 struct nvme_completion *cqe)
 316 {
 317         u16 status = le16_to_cpu(cqe->status) >> 1;
 318         u32 result = le32_to_cpu(cqe->result);
 319
 320         if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
 321                 ++dev->ctrl.event_limit;
 322         if (status != NVME_SC_SUCCESS)
 323                 return;
 324
 325         switch (result & 0xff07) {
 326         case NVME_AER_NOTICE_NS_CHANGED:
 327                 dev_info(dev->dev, "rescanning\n");
 328                 queue_work(nvme_workq, &dev->scan_work);
 329         default:
 330                 dev_warn(dev->dev, "async event result %08x\n", result);
 331         }
 332 }
 333
 334 /**
 335  * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
 336  * @nvmeq: The queue to use
 337  * @cmd: The command to send
 338  *
 339  * Safe to use from interrupt context
 340  */
 341 static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
 342                                                 struct nvme_command *cmd)
 343 {
 344         u16 tail = nvmeq->sq_tail;
 345
 346         if (nvmeq->sq_cmds_io)
 347                 memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
 348         else
 349                 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
 350
 351         if (++tail == nvmeq->q_depth)
 352                 tail = 0;
 353         writel(tail, nvmeq->q_db);
 354         nvmeq->sq_tail = tail;
 355 }
 356
 357 static __le64 **iod_list(struct nvme_iod *iod)
 358 {
 359         return ((void *)iod) + iod->offset;
 360 }
 361
 362 static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
 363                             unsigned nseg, unsigned long private)
 364 {
 365         iod->private = private;
 366         iod->offset = offsetof(struct nvme_iod, sg[nseg]);
 367         iod->npages = -1;
 368         iod->length = nbytes;
 369         iod->nents = 0;
 370 }
 371
 372 static struct nvme_iod *
 373 __nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
 374                  unsigned long priv, gfp_t gfp)
 375 {
 376         struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
 377                                 sizeof(__le64 *) * nvme_npages(bytes, dev) +
 378                                 sizeof(struct scatterlist) * nseg, gfp);
 379
 380         if (iod)
 381                 iod_init(iod, bytes, nseg, priv);
 382
 383         return iod;
 384 }
 385
 386 static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
 387                                        gfp_t gfp)
 388 {
 389         unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
 390                                                 sizeof(struct nvme_dsm_range);
 391         struct nvme_iod *iod;
 392
 393         if (rq->nr_phys_segments <= NVME_INT_PAGES &&
 394             size <= NVME_INT_BYTES(dev)) {
 395                 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);
 396
 397                 iod = &cmd->__iod;
 398                 iod_init(iod, size, rq->nr_phys_segments,
 399                                 (unsigned long) rq | NVME_INT_MASK);
 400                 return iod;
 401         }
 402
 403         return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
 404                                 (unsigned long) rq, gfp);
 405 }
 406
 407 static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
 408 {
 409         const int last_prp = dev->ctrl.page_size / 8 - 1;
 410         int i;
 411         __le64 **list = iod_list(iod);
 412         dma_addr_t prp_dma = iod->first_dma;
 413
 414         if (iod->npages == 0)
 415                 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
 416         for (i = 0; i < iod->npages; i++) {
 417                 __le64 *prp_list = list[i];
 418                 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
 419                 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
 420                 prp_dma = next_prp_dma;
 421         }
 422
 423         if (iod_should_kfree(iod))
 424                 kfree(iod);
 425 }
 426
 427 #ifdef CONFIG_BLK_DEV_INTEGRITY
 428 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
 429 {
 430         if (be32_to_cpu(pi->ref_tag) == v)
 431                 pi->ref_tag = cpu_to_be32(p);
 432 }
 433
 434 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
 435 {
 436         if (be32_to_cpu(pi->ref_tag) == p)
 437                 pi->ref_tag = cpu_to_be32(v);
 438 }
 439
 440 /**
 441  * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 442  *
 443  * The virtual start sector is the one that was originally submitted by the
 444  * block layer. Due to partitioning, MD/DM cloning, etc. the actual physical
 445  * start sector may be different. Remap protection information to match the
 446  * physical LBA on writes, and back to the original seed on reads.
 447  *
 448  * Type 0 and 3 do not have a ref tag, so no remapping required.
 449  */
 450 static void nvme_dif_remap(struct request *req,
 451                         void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
 452 {
 453         struct nvme_ns *ns = req->rq_disk->private_data;
 454         struct bio_integrity_payload *bip;
 455         struct t10_pi_tuple *pi;
 456         void *p, *pmap;
 457         u32 i, nlb, ts, phys, virt;
 458
 459         if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
 460                 return;
 461
 462         bip = bio_integrity(req->bio);
 463         if (!bip)
 464                 return;
 465
 466         pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;
 467
 468         p = pmap;
 469         virt = bip_get_seed(bip);
 470         phys = nvme_block_nr(ns, blk_rq_pos(req));
 471         nlb = (blk_rq_bytes(req) >> ns->lba_shift);
 472         ts = ns->disk->queue->integrity.tuple_size;
 473
 474         for (i = 0; i < nlb; i++, virt++, phys++) {
 475                 pi = (struct t10_pi_tuple *)p;
 476                 dif_swap(phys, virt, pi);
 477                 p += ts;
 478         }
 479         kunmap_atomic(pmap);
 480 }
 481 #else /* CONFIG_BLK_DEV_INTEGRITY */
 482 static void nvme_dif_remap(struct request *req,
 483                         void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
 484 {
 485 }
 486 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
 487 {
 488 }
 489 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
 490 {
 491 }
 492 #endif
 493
 494 static void req_completion(struct nvme_queue *nvmeq, struct nvme_completion *cqe)
 495 {
 496         struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
 497         struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
 498         struct nvme_iod *iod = cmd_rq->iod;
 499         u16 status = le16_to_cpup(&cqe->status) >> 1;
 500         int error = 0;
 501
 502         if (unlikely(status)) {
 503                 if (nvme_req_needs_retry(req, status)) {
 504                         nvme_unmap_data(nvmeq->dev, iod);
 505                         nvme_requeue_req(req);
 506                         return;
 507                 }
 508
 509                 if (req->cmd_type == REQ_TYPE_DRV_PRIV)
 510                         error = status;
 511                 else
 512                         error = nvme_error_status(status);
 513         }
 514
 515         if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
 516                 u32 result = le32_to_cpup(&cqe->result);
 517                 req->special = (void *)(uintptr_t)result;
 518         }
 519
 520         if (cmd_rq->aborted)
 521                 dev_warn(nvmeq->dev->dev,
 522                         "completing aborted command with status:%04x\n",
 523                         error);
 524
 525         nvme_unmap_data(nvmeq->dev, iod);
 526         blk_mq_complete_request(req, error);
 527 }
 528
 529 static bool nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
 530                 int total_len)
 531 {
 532         struct dma_pool *pool;
 533         int length = total_len;
 534         struct scatterlist *sg = iod->sg;
 535         int dma_len = sg_dma_len(sg);
 536         u64 dma_addr = sg_dma_address(sg);
 537         u32 page_size = dev->ctrl.page_size;
 538         int offset = dma_addr & (page_size - 1);
 539         __le64 *prp_list;
 540         __le64 **list = iod_list(iod);
 541         dma_addr_t prp_dma;
 542         int nprps, i;
 543
 544         length -= (page_size - offset);
 545         if (length <= 0)
 546                 return true;
 547
 548         dma_len -= (page_size - offset);
 549         if (dma_len) {
 550                 dma_addr += (page_size - offset);
 551         } else {
 552                 sg = sg_next(sg);
 553                 dma_addr = sg_dma_address(sg);
 554                 dma_len = sg_dma_len(sg);
 555         }
 556
 557         if (length <= page_size) {
 558                 iod->first_dma = dma_addr;
 559                 return true;
 560         }
 561
 562         nprps = DIV_ROUND_UP(length, page_size);
 563         if (nprps <= (256 / 8)) {
 564                 pool = dev->prp_small_pool;
 565                 iod->npages = 0;
 566         } else {
 567                 pool = dev->prp_page_pool;
 568                 iod->npages = 1;
 569         }
 570
 571         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 572         if (!prp_list) {
 573                 iod->first_dma = dma_addr;
 574                 iod->npages = -1;
 575                 return false;
 576         }
 577         list[0] = prp_list;
 578         iod->first_dma = prp_dma;
 579         i = 0;
 580         for (;;) {
 581                 if (i == page_size >> 3) {
 582                         __le64 *old_prp_list = prp_list;
 583                         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
 584                         if (!prp_list)
 585                                 return false;
 586                         list[iod->npages++] = prp_list;
 587                         prp_list[0] = old_prp_list[i - 1];
 588                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
 589                         i = 1;
 590                 }
 591                 prp_list[i++] = cpu_to_le64(dma_addr);
 592                 dma_len -= page_size;
 593                 dma_addr += page_size;
 594                 length -= page_size;
 595                 if (length <= 0)
 596                         break;
 597                 if (dma_len > 0)
 598                         continue;
 599                 BUG_ON(dma_len < 0);
 600                 sg = sg_next(sg);
 601                 dma_addr = sg_dma_address(sg);
 602                 dma_len = sg_dma_len(sg);
 603         }
 604
 605         return true;
 606 }
 607
 608 static int nvme_map_data(struct nvme_dev *dev, struct nvme_iod *iod,
 609                 struct nvme_command *cmnd)
 610 {
 611         struct request *req = iod_get_private(iod);
 612         struct request_queue *q = req->q;
 613         enum dma_data_direction dma_dir = rq_data_dir(req) ?
 614                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
 615         int ret = BLK_MQ_RQ_QUEUE_ERROR;
 616
 617         sg_init_table(iod->sg, req->nr_phys_segments);
 618         iod->nents = blk_rq_map_sg(q, req, iod->sg);
 619         if (!iod->nents)
 620                 goto out;
 621
 622         ret = BLK_MQ_RQ_QUEUE_BUSY;
 623         if (!dma_map_sg(dev->dev, iod->sg, iod->nents, dma_dir))
 624                 goto out;
 625
 626         if (!nvme_setup_prps(dev, iod, blk_rq_bytes(req)))
 627                 goto out_unmap;
 628
 629         ret = BLK_MQ_RQ_QUEUE_ERROR;
 630         if (blk_integrity_rq(req)) {
 631                 if (blk_rq_count_integrity_sg(q, req->bio) != 1)
 632                         goto out_unmap;
 633
 634                 sg_init_table(iod->meta_sg, 1);
 635                 if (blk_rq_map_integrity_sg(q, req->bio, iod->meta_sg) != 1)
 636                         goto out_unmap;
 637
 638                 if (rq_data_dir(req))
 639                         nvme_dif_remap(req, nvme_dif_prep);
 640
 641                 if (!dma_map_sg(dev->dev, iod->meta_sg, 1, dma_dir))
 642                         goto out_unmap;
 643         }
 644
 645         cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
 646         cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
 647         if (blk_integrity_rq(req))
 648                 cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));
 649         return BLK_MQ_RQ_QUEUE_OK;
 650
 651 out_unmap:
 652         dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
 653 out:
 654         return ret;
 655 }
 656
 657 static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod)
 658 {
 659         struct request *req = iod_get_private(iod);
 660         enum dma_data_direction dma_dir = rq_data_dir(req) ?
 661                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
 662
 663         if (iod->nents) {
 664                 dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
 665                 if (blk_integrity_rq(req)) {
 666                         if (!rq_data_dir(req))
 667                                 nvme_dif_remap(req, nvme_dif_complete);
 668                         dma_unmap_sg(dev->dev, iod->meta_sg, 1, dma_dir);
 669                 }
 670         }
 671
 672         nvme_free_iod(dev, iod);
 673 }
 674
 675 /*
 676  * We reuse the small pool to allocate the 16-byte range here as it is not
 677  * worth having a special pool for these or additional cases to handle freeing
 678  * the iod.
 679  */
 680 static int nvme_setup_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
 681                 struct nvme_iod *iod, struct nvme_command *cmnd)
 682 {
 683         struct request *req = iod_get_private(iod);
 684         struct nvme_dsm_range *range;
 685
 686         range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
 687                                                 &iod->first_dma);
 688         if (!range)
 689                 return BLK_MQ_RQ_QUEUE_BUSY;
 690         iod_list(iod)[0] = (__le64 *)range;
 691         iod->npages = 0;
 692
 693         range->cattr = cpu_to_le32(0);
 694         range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
 695         range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
 696
 697         memset(cmnd, 0, sizeof(*cmnd));
 698         cmnd->dsm.opcode = nvme_cmd_dsm;
 699         cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
 700         cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
 701         cmnd->dsm.nr = 0;
 702         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
 703         return BLK_MQ_RQ_QUEUE_OK;
 704 }
 705
 706 /*
 707  * NOTE: ns is NULL when called on the admin queue.
 708  */
 709 static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 710                          const struct blk_mq_queue_data *bd)
 711 {
 712         struct nvme_ns *ns = hctx->queue->queuedata;
 713         struct nvme_queue *nvmeq = hctx->driver_data;
 714         struct nvme_dev *dev = nvmeq->dev;
 715         struct request *req = bd->rq;
 716         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
 717         struct nvme_iod *iod;
 718         struct nvme_command cmnd;
 719         int ret = BLK_MQ_RQ_QUEUE_OK;
 720
 721         /*
 722          * If formated with metadata, require the block layer provide a buffer
 723          * unless this namespace is formated such that the metadata can be
 724          * stripped/generated by the controller with PRACT=1.
 725          */
 726         if (ns && ns->ms && !blk_integrity_rq(req)) {
 727                 if (!(ns->pi_type && ns->ms == 8) &&
 728                                         req->cmd_type != REQ_TYPE_DRV_PRIV) {
 729                         blk_mq_complete_request(req, -EFAULT);
 730                         return BLK_MQ_RQ_QUEUE_OK;
 731                 }
 732         }
 733
 734         iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
 735         if (!iod)
 736                 return BLK_MQ_RQ_QUEUE_BUSY;
 737
 738         if (req->cmd_flags & REQ_DISCARD) {
 739                 ret = nvme_setup_discard(nvmeq, ns, iod, &cmnd);
 740         } else {
 741                 if (req->cmd_type == REQ_TYPE_DRV_PRIV)
 742                         memcpy(&cmnd, req->cmd, sizeof(cmnd));
 743                 else if (req->cmd_flags & REQ_FLUSH)
 744                         nvme_setup_flush(ns, &cmnd);
 745                 else
 746                         nvme_setup_rw(ns, req, &cmnd);
 747
 748                 if (req->nr_phys_segments)
 749                         ret = nvme_map_data(dev, iod, &cmnd);
 750         }
 751
 752         if (ret)
 753                 goto out;
 754
 755         cmd->iod = iod;
 756         cmd->aborted = 0;
 757         cmnd.common.command_id = req->tag;
 758         blk_mq_start_request(req);
 759
 760         spin_lock_irq(&nvmeq->q_lock);
 761         __nvme_submit_cmd(nvmeq, &cmnd);
 762         nvme_process_cq(nvmeq);
 763         spin_unlock_irq(&nvmeq->q_lock);
 764         return BLK_MQ_RQ_QUEUE_OK;
 765 out:
 766         nvme_free_iod(dev, iod);
 767         return ret;
 768 }
 769
 770 static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
 771 {
 772         u16 head, phase;
 773
 774         head = nvmeq->cq_head;
 775         phase = nvmeq->cq_phase;
 776
 777         for (;;) {
 778                 struct nvme_completion cqe = nvmeq->cqes[head];
 779                 u16 status = le16_to_cpu(cqe.status);
 780
 781                 if ((status & 1) != phase)
 782                         break;
 783                 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
 784                 if (++head == nvmeq->q_depth) {
 785                         head = 0;
 786                         phase = !phase;
 787                 }
 788
 789                 if (tag && *tag == cqe.command_id)
 790                         *tag = -1;
 791
 792                 if (unlikely(cqe.command_id >= nvmeq->q_depth)) {
 793                         dev_warn(nvmeq->q_dmadev,
 794                                 "invalid id %d completed on queue %d\n",
 795                                 cqe.command_id, le16_to_cpu(cqe.sq_id));
 796                         continue;
 797                 }
 798
 799                 /*
 800                  * AEN requests are special as they don't time out and can
 801                  * survive any kind of queue freeze and often don't respond to
 802                  * aborts.  We don't even bother to allocate a struct request
 803                  * for them but rather special case them here.
 804                  */
 805                 if (unlikely(nvmeq->qid == 0 &&
 806                                 cqe.command_id >= NVME_AQ_BLKMQ_DEPTH)) {
 807                         nvme_complete_async_event(nvmeq->dev, &cqe);
 808                         continue;
 809                 }
 810
 811                 req_completion(nvmeq, &cqe);
 812         }
 813
 814         /* If the controller ignores the cq head doorbell and continuously
 815          * writes to the queue, it is theoretically possible to wrap around
 816          * the queue twice and mistakenly return IRQ_NONE.  Linux only
 817          * requires that 0.1% of your interrupts are handled, so this isn't
 818          * a big problem.
 819          */
 820         if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
 821                 return;
 822
 823         if (likely(nvmeq->cq_vector >= 0))
 824                 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
 825         nvmeq->cq_head = head;
 826         nvmeq->cq_phase = phase;
 827
 828         nvmeq->cqe_seen = 1;
 829 }
 830
 831 static void nvme_process_cq(struct nvme_queue *nvmeq)
 832 {
 833         __nvme_process_cq(nvmeq, NULL);
 834 }
 835
 836 static irqreturn_t nvme_irq(int irq, void *data)
 837 {
 838         irqreturn_t result;
 839         struct nvme_queue *nvmeq = data;
 840         spin_lock(&nvmeq->q_lock);
 841         nvme_process_cq(nvmeq);
 842         result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
 843         nvmeq->cqe_seen = 0;
 844         spin_unlock(&nvmeq->q_lock);
 845         return result;
 846 }
 847
 848 static irqreturn_t nvme_irq_check(int irq, void *data)
 849 {
 850         struct nvme_queue *nvmeq = data;
 851         struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
 852         if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
 853                 return IRQ_NONE;
 854         return IRQ_WAKE_THREAD;
 855 }
 856
 857 static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
 858 {
 859         struct nvme_queue *nvmeq = hctx->driver_data;
 860
 861         if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
 862             nvmeq->cq_phase) {
 863                 spin_lock_irq(&nvmeq->q_lock);
 864                 __nvme_process_cq(nvmeq, &tag);
 865                 spin_unlock_irq(&nvmeq->q_lock);
 866
 867                 if (tag == -1)
 868                         return 1;
 869         }
 870
 871         return 0;
 872 }
 873
 874 static void nvme_submit_async_event(struct nvme_dev *dev)
 875 {
 876         struct nvme_command c;
 877
 878         memset(&c, 0, sizeof(c));
 879         c.common.opcode = nvme_admin_async_event;
 880         c.common.command_id = NVME_AQ_BLKMQ_DEPTH + --dev->ctrl.event_limit;
 881
 882         __nvme_submit_cmd(dev->queues[0], &c);
 883 }
 884
 885 static void async_cmd_info_endio(struct request *req, int error)
 886 {
 887         struct async_cmd_info *cmdinfo = req->end_io_data;
 888
 889         cmdinfo->status = req->errors;
 890         queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
 891         blk_mq_free_request(req);
 892 }
 893
 894 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
 895 {
 896         struct nvme_command c;
 897
 898         memset(&c, 0, sizeof(c));
 899         c.delete_queue.opcode = opcode;
 900         c.delete_queue.qid = cpu_to_le16(id);
 901
 902         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 903 }
 904
 905 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
 906                                                 struct nvme_queue *nvmeq)
 907 {
 908         struct nvme_command c;
 909         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
 910
 911         /*
 912          * Note: we (ab)use the fact the the prp fields survive if no data
 913          * is attached to the request.
 914          */
 915         memset(&c, 0, sizeof(c));
 916         c.create_cq.opcode = nvme_admin_create_cq;
 917         c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
 918         c.create_cq.cqid = cpu_to_le16(qid);
 919         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 920         c.create_cq.cq_flags = cpu_to_le16(flags);
 921         c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
 922
 923         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 924 }
 925
 926 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
 927                                                 struct nvme_queue *nvmeq)
 928 {
 929         struct nvme_command c;
 930         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
 931
 932         /*
 933          * Note: we (ab)use the fact the the prp fields survive if no data
 934          * is attached to the request.
 935          */
 936         memset(&c, 0, sizeof(c));
 937         c.create_sq.opcode = nvme_admin_create_sq;
 938         c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
 939         c.create_sq.sqid = cpu_to_le16(qid);
 940         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
 941         c.create_sq.sq_flags = cpu_to_le16(flags);
 942         c.create_sq.cqid = cpu_to_le16(qid);
 943
 944         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
 945 }
 946
 947 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
 948 {
 949         return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
 950 }
 951
 952 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
 953 {
 954         return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
 955 }
 956
 957 static void abort_endio(struct request *req, int error)
 958 {
 959         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
 960         struct nvme_queue *nvmeq = cmd->nvmeq;
 961         u32 result = (u32)(uintptr_t)req->special;
 962         u16 status = req->errors;
 963
 964         dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
 965         atomic_inc(&nvmeq->dev->ctrl.abort_limit);
 966
 967         blk_mq_free_request(req);
 968 }
 969
 970 static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
 971 {
 972         struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
 973         struct nvme_queue *nvmeq = cmd_rq->nvmeq;
 974         struct nvme_dev *dev = nvmeq->dev;
 975         struct request *abort_req;
 976         struct nvme_command cmd;
 977
 978         /*
 979          * Shutdown immediately if controller times out while starting. The
 980          * reset work will see the pci device disabled when it gets the forced
 981          * cancellation error. All outstanding requests are completed on
 982          * shutdown, so we return BLK_EH_HANDLED.
 983          */
 984         if (test_bit(NVME_CTRL_RESETTING, &dev->flags)) {
 985                 dev_warn(dev->dev,
 986                          "I/O %d QID %d timeout, disable controller\n",
 987                          req->tag, nvmeq->qid);
 988                 nvme_dev_shutdown(dev);
 989                 req->errors = NVME_SC_CANCELLED;
 990                 return BLK_EH_HANDLED;
 991         }
 992
 993         /*
 994          * Shutdown the controller immediately and schedule a reset if the
 995          * command was already aborted once before and still hasn't been
 996          * returned to the driver, or if this is the admin queue.
 997          */
 998         if (!nvmeq->qid || cmd_rq->aborted) {
 999                 dev_warn(dev->dev,
1000                          "I/O %d QID %d timeout, reset controller\n",
1001                          req->tag, nvmeq->qid);
1002                 nvme_dev_shutdown(dev);
1003                 queue_work(nvme_workq, &dev->reset_work);
1004
1005                 /*
1006                  * Mark the request as handled, since the inline shutdown
1007                  * forces all outstanding requests to complete.
1008                  */
1009                 req->errors = NVME_SC_CANCELLED;
1010                 return BLK_EH_HANDLED;
1011         }
1012
1013         cmd_rq->aborted = 1;
1014
1015         if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1016                 atomic_inc(&dev->ctrl.abort_limit);
1017                 return BLK_EH_RESET_TIMER;
1018         }
1019
1020         memset(&cmd, 0, sizeof(cmd));
1021         cmd.abort.opcode = nvme_admin_abort_cmd;
1022         cmd.abort.cid = req->tag;
1023         cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1024
1025         dev_warn(nvmeq->q_dmadev, "I/O %d QID %d timeout, aborting\n",
1026                                  req->tag, nvmeq->qid);
1027
1028         abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
1029                         BLK_MQ_REQ_NOWAIT);
1030         if (IS_ERR(abort_req)) {
1031                 atomic_inc(&dev->ctrl.abort_limit);
1032                 return BLK_EH_RESET_TIMER;
1033         }
1034
1035         abort_req->timeout = ADMIN_TIMEOUT;
1036         abort_req->end_io_data = NULL;
1037         blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
1038
1039         /*
1040          * The aborted req will be completed on receiving the abort req.
1041          * We enable the timer again. If hit twice, it'll cause a device reset,
1042          * as the device then is in a faulty state.
1043          */
1044         return BLK_EH_RESET_TIMER;
1045 }
1046
1047 static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
1048 {
1049         struct nvme_queue *nvmeq = data;
1050         int status;
1051
1052         if (!blk_mq_request_started(req))
1053                 return;
1054
1055         dev_warn(nvmeq->q_dmadev,
1056                  "Cancelling I/O %d QID %d\n", req->tag, nvmeq->qid);
1057
1058         status = NVME_SC_CANCELLED;
1059         if (blk_queue_dying(req->q))
1060                 status |= NVME_SC_DNR;
1061         blk_mq_complete_request(req, status);
1062 }
1063
1064 static void nvme_free_queue(struct nvme_queue *nvmeq)
1065 {
1066         dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1067                                 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1068         if (nvmeq->sq_cmds)
1069                 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1070                                         nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1071         kfree(nvmeq);
1072 }
1073
1074 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1075 {
1076         int i;
1077
1078         for (i = dev->queue_count - 1; i >= lowest; i--) {
1079                 struct nvme_queue *nvmeq = dev->queues[i];
1080                 dev->queue_count--;
1081                 dev->queues[i] = NULL;
1082                 nvme_free_queue(nvmeq);
1083         }
1084 }
1085
1086 /**
1087  * nvme_suspend_queue - put queue into suspended state
1088  * @nvmeq - queue to suspend
1089  */
1090 static int nvme_suspend_queue(struct nvme_queue *nvmeq)
1091 {
1092         int vector;
1093
1094         spin_lock_irq(&nvmeq->q_lock);
1095         if (nvmeq->cq_vector == -1) {
1096                 spin_unlock_irq(&nvmeq->q_lock);
1097                 return 1;
1098         }
1099         vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
1100         nvmeq->dev->online_queues--;
1101         nvmeq->cq_vector = -1;
1102         spin_unlock_irq(&nvmeq->q_lock);
1103
1104         if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1105                 blk_mq_freeze_queue_start(nvmeq->dev->ctrl.admin_q);
1106
1107         irq_set_affinity_hint(vector, NULL);
1108         free_irq(vector, nvmeq);
1109
1110         return 0;
1111 }
1112
1113 static void nvme_clear_queue(struct nvme_queue *nvmeq)
1114 {
1115         spin_lock_irq(&nvmeq->q_lock);
1116         if (nvmeq->tags && *nvmeq->tags)
1117                 blk_mq_all_tag_busy_iter(*nvmeq->tags, nvme_cancel_queue_ios, nvmeq);
1118         spin_unlock_irq(&nvmeq->q_lock);
1119 }
1120
1121 static void nvme_disable_queue(struct nvme_dev *dev, int qid)
1122 {
1123         struct nvme_queue *nvmeq = dev->queues[qid];
1124
1125         if (!nvmeq)
1126                 return;
1127         if (nvme_suspend_queue(nvmeq))
1128                 return;
1129
1130         /* Don't tell the adapter to delete the admin queue.
1131          * Don't tell a removed adapter to delete IO queues. */
1132         if (qid && readl(dev->bar + NVME_REG_CSTS) != -1) {
1133                 adapter_delete_sq(dev, qid);
1134                 adapter_delete_cq(dev, qid);
1135         }
1136
1137         spin_lock_irq(&nvmeq->q_lock);
1138         nvme_process_cq(nvmeq);
1139         spin_unlock_irq(&nvmeq->q_lock);
1140 }
1141
1142 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1143                                 int entry_size)
1144 {
1145         int q_depth = dev->q_depth;
1146         unsigned q_size_aligned = roundup(q_depth * entry_size,
1147                                           dev->ctrl.page_size);
1148
1149         if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1150                 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1151                 mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1152                 q_depth = div_u64(mem_per_q, entry_size);
1153
1154                 /*
1155                  * Ensure the reduced q_depth is above some threshold where it
1156                  * would be better to map queues in system memory with the
1157                  * original depth
1158                  */
1159                 if (q_depth < 64)
1160                         return -ENOMEM;
1161         }
1162
1163         return q_depth;
1164 }
1165
1166 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1167                                 int qid, int depth)
1168 {
1169         if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
1170                 unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
1171                                                       dev->ctrl.page_size);
1172                 nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
1173                 nvmeq->sq_cmds_io = dev->cmb + offset;
1174         } else {
1175                 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
1176                                         &nvmeq->sq_dma_addr, GFP_KERNEL);
1177                 if (!nvmeq->sq_cmds)
1178                         return -ENOMEM;
1179         }
1180
1181         return 0;
1182 }
1183
1184 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1185                                                         int depth)
1186 {
1187         struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
1188         if (!nvmeq)
1189                 return NULL;
1190
1191         nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
1192                                           &nvmeq->cq_dma_addr, GFP_KERNEL);
1193         if (!nvmeq->cqes)
1194                 goto free_nvmeq;
1195
1196         if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
1197                 goto free_cqdma;
1198
1199         nvmeq->q_dmadev = dev->dev;
1200         nvmeq->dev = dev;
1201         snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1202                         dev->ctrl.instance, qid);
1203         spin_lock_init(&nvmeq->q_lock);
1204         nvmeq->cq_head = 0;
1205         nvmeq->cq_phase = 1;
1206         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1207         nvmeq->q_depth = depth;
1208         nvmeq->qid = qid;
1209         nvmeq->cq_vector = -1;
1210         dev->queues[qid] = nvmeq;
1211
1212         /* make sure queue descriptor is set before queue count, for kthread */
1213         mb();
1214         dev->queue_count++;
1215
1216         return nvmeq;
1217
1218  free_cqdma:
1219         dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
1220                                                         nvmeq->cq_dma_addr);
1221  free_nvmeq:
1222         kfree(nvmeq);
1223         return NULL;
1224 }
1225
1226 static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1227                                                         const char *name)
1228 {
1229         if (use_threaded_interrupts)
1230                 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1231                                         nvme_irq_check, nvme_irq, IRQF_SHARED,
1232                                         name, nvmeq);
1233         return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1234                                 IRQF_SHARED, name, nvmeq);
1235 }
1236
1237 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1238 {
1239         struct nvme_dev *dev = nvmeq->dev;
1240
1241         spin_lock_irq(&nvmeq->q_lock);
1242         nvmeq->sq_tail = 0;
1243         nvmeq->cq_head = 0;
1244         nvmeq->cq_phase = 1;
1245         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1246         memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1247         dev->online_queues++;
1248         spin_unlock_irq(&nvmeq->q_lock);
1249 }
1250
1251 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1252 {
1253         struct nvme_dev *dev = nvmeq->dev;
1254         int result;
1255
1256         nvmeq->cq_vector = qid - 1;
1257         result = adapter_alloc_cq(dev, qid, nvmeq);
1258         if (result < 0)
1259                 return result;
1260
1261         result = adapter_alloc_sq(dev, qid, nvmeq);
1262         if (result < 0)
1263                 goto release_cq;
1264
1265         result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1266         if (result < 0)
1267                 goto release_sq;
1268
1269         nvme_init_queue(nvmeq, qid);
1270         return result;
1271
1272  release_sq:
1273         adapter_delete_sq(dev, qid);
1274  release_cq:
1275         adapter_delete_cq(dev, qid);
1276         return result;
1277 }
1278
1279 static struct blk_mq_ops nvme_mq_admin_ops = {
1280         .queue_rq       = nvme_queue_rq,
1281         .map_queue      = blk_mq_map_queue,
1282         .init_hctx      = nvme_admin_init_hctx,
1283         .exit_hctx      = nvme_admin_exit_hctx,
1284         .init_request   = nvme_admin_init_request,
1285         .timeout        = nvme_timeout,
1286 };
1287
1288 static struct blk_mq_ops nvme_mq_ops = {
1289         .queue_rq       = nvme_queue_rq,
1290         .map_queue      = blk_mq_map_queue,
1291         .init_hctx      = nvme_init_hctx,
1292         .init_request   = nvme_init_request,
1293         .timeout        = nvme_timeout,
1294         .poll           = nvme_poll,
1295 };
1296
1297 static void nvme_dev_remove_admin(struct nvme_dev *dev)
1298 {
1299         if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1300                 blk_cleanup_queue(dev->ctrl.admin_q);
1301                 blk_mq_free_tag_set(&dev->admin_tagset);
1302         }
1303 }
1304
1305 static int nvme_alloc_admin_tags(struct nvme_dev *dev)
1306 {
1307         if (!dev->ctrl.admin_q) {
1308                 dev->admin_tagset.ops = &nvme_mq_admin_ops;
1309                 dev->admin_tagset.nr_hw_queues = 1;
1310                 dev->admin_tagset.queue_depth = NVME_AQ_BLKMQ_DEPTH;
1311                 dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1312                 dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1313                 dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
1314                 dev->admin_tagset.driver_data = dev;
1315
1316                 if (blk_mq_alloc_tag_set(&dev->admin_tagset))
1317                         return -ENOMEM;
1318
1319                 dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
1320                 if (IS_ERR(dev->ctrl.admin_q)) {
1321                         blk_mq_free_tag_set(&dev->admin_tagset);
1322                         return -ENOMEM;
1323                 }
1324                 if (!blk_get_queue(dev->ctrl.admin_q)) {
1325                         nvme_dev_remove_admin(dev);
1326                         dev->ctrl.admin_q = NULL;
1327                         return -ENODEV;
1328                 }
1329         } else
1330                 blk_mq_unfreeze_queue(dev->ctrl.admin_q);
1331
1332         return 0;
1333 }
1334
1335 static int nvme_configure_admin_queue(struct nvme_dev *dev)
1336 {
1337         int result;
1338         u32 aqa;
1339         u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
1340         struct nvme_queue *nvmeq;
1341
1342         dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
1343                                                 NVME_CAP_NSSRC(cap) : 0;
1344
1345         if (dev->subsystem &&
1346             (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
1347                 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1348
1349         result = nvme_disable_ctrl(&dev->ctrl, cap);
1350         if (result < 0)
1351                 return result;
1352
1353         nvmeq = dev->queues[0];
1354         if (!nvmeq) {
1355                 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1356                 if (!nvmeq)
1357                         return -ENOMEM;
1358         }
1359
1360         aqa = nvmeq->q_depth - 1;
1361         aqa |= aqa << 16;
1362
1363         writel(aqa, dev->bar + NVME_REG_AQA);
1364         lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
1365         lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
1366
1367         result = nvme_enable_ctrl(&dev->ctrl, cap);
1368         if (result)
1369                 goto free_nvmeq;
1370
1371         nvmeq->cq_vector = 0;
1372         result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1373         if (result) {
1374                 nvmeq->cq_vector = -1;
1375                 goto free_nvmeq;
1376         }
1377
1378         return result;
1379
1380  free_nvmeq:
1381         nvme_free_queues(dev, 0);
1382         return result;
1383 }
1384
1385 static int nvme_kthread(void *data)
1386 {
1387         struct nvme_dev *dev, *next;
1388
1389         while (!kthread_should_stop()) {
1390                 set_current_state(TASK_INTERRUPTIBLE);
1391                 spin_lock(&dev_list_lock);
1392                 list_for_each_entry_safe(dev, next, &dev_list, node) {
1393                         int i;
1394                         u32 csts = readl(dev->bar + NVME_REG_CSTS);
1395
1396                         /*
1397                          * Skip controllers currently under reset.
1398                          */
1399                         if (work_pending(&dev->reset_work) || work_busy(&dev->reset_work))
1400                                 continue;
1401
1402                         if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
1403                                                         csts & NVME_CSTS_CFS) {
1404                                 if (queue_work(nvme_workq, &dev->reset_work)) {
1405                                         dev_warn(dev->dev,
1406                                                 "Failed status: %x, reset controller\n",
1407                                                 readl(dev->bar + NVME_REG_CSTS));
1408                                 }
1409                                 continue;
1410                         }
1411                         for (i = 0; i < dev->queue_count; i++) {
1412                                 struct nvme_queue *nvmeq = dev->queues[i];
1413                                 if (!nvmeq)
1414                                         continue;
1415                                 spin_lock_irq(&nvmeq->q_lock);
1416                                 nvme_process_cq(nvmeq);
1417
1418                                 while (i == 0 && dev->ctrl.event_limit > 0)
1419                                         nvme_submit_async_event(dev);
1420                                 spin_unlock_irq(&nvmeq->q_lock);
1421                         }
1422                 }
1423                 spin_unlock(&dev_list_lock);
1424                 schedule_timeout(round_jiffies_relative(HZ));
1425         }
1426         return 0;
1427 }
1428
1429 static int nvme_create_io_queues(struct nvme_dev *dev)
1430 {
1431         unsigned i;
1432         int ret = 0;
1433
1434         for (i = dev->queue_count; i <= dev->max_qid; i++) {
1435                 if (!nvme_alloc_queue(dev, i, dev->q_depth)) {
1436                         ret = -ENOMEM;
1437                         break;
1438                 }
1439         }
1440
1441         for (i = dev->online_queues; i <= dev->queue_count - 1; i++) {
1442                 ret = nvme_create_queue(dev->queues[i], i);
1443                 if (ret) {
1444                         nvme_free_queues(dev, i);
1445                         break;
1446                 }
1447         }
1448
1449         /*
1450          * Ignore failing Create SQ/CQ commands, we can continue with less
1451          * than the desired aount of queues, and even a controller without
1452          * I/O queues an still be used to issue admin commands.  This might
1453          * be useful to upgrade a buggy firmware for example.
1454          */
1455         return ret >= 0 ? 0 : ret;
1456 }
1457
1458 static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
1459 {
1460         u64 szu, size, offset;
1461         u32 cmbloc;
1462         resource_size_t bar_size;
1463         struct pci_dev *pdev = to_pci_dev(dev->dev);
1464         void __iomem *cmb;
1465         dma_addr_t dma_addr;
1466
1467         if (!use_cmb_sqes)
1468                 return NULL;
1469
1470         dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1471         if (!(NVME_CMB_SZ(dev->cmbsz)))
1472                 return NULL;
1473
1474         cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1475
1476         szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
1477         size = szu * NVME_CMB_SZ(dev->cmbsz);
1478         offset = szu * NVME_CMB_OFST(cmbloc);
1479         bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc));
1480
1481         if (offset > bar_size)
1482                 return NULL;
1483
1484         /*
1485          * Controllers may support a CMB size larger than their BAR,
1486          * for example, due to being behind a bridge. Reduce the CMB to
1487          * the reported size of the BAR
1488          */
1489         if (size > bar_size - offset)
1490                 size = bar_size - offset;
1491
1492         dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset;
1493         cmb = ioremap_wc(dma_addr, size);
1494         if (!cmb)
1495                 return NULL;
1496
1497         dev->cmb_dma_addr = dma_addr;
1498         dev->cmb_size = size;
1499         return cmb;
1500 }
1501
1502 static inline void nvme_release_cmb(struct nvme_dev *dev)
1503 {
1504         if (dev->cmb) {
1505                 iounmap(dev->cmb);
1506                 dev->cmb = NULL;
1507         }
1508 }
1509
1510 static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1511 {
1512         return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
1513 }
1514
1515 static int nvme_setup_io_queues(struct nvme_dev *dev)
1516 {
1517         struct nvme_queue *adminq = dev->queues[0];
1518         struct pci_dev *pdev = to_pci_dev(dev->dev);
1519         int result, i, vecs, nr_io_queues, size;
1520
1521         nr_io_queues = num_possible_cpus();
1522         result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
1523         if (result < 0)
1524                 return result;
1525
1526         /*
1527          * Degraded controllers might return an error when setting the queue
1528          * count.  We still want to be able to bring them online and offer
1529          * access to the admin queue, as that might be only way to fix them up.
1530          */
1531         if (result > 0) {
1532                 dev_err(dev->dev, "Could not set queue count (%d)\n", result);
1533                 nr_io_queues = 0;
1534                 result = 0;
1535         }
1536
1537         if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
1538                 result = nvme_cmb_qdepth(dev, nr_io_queues,
1539                                 sizeof(struct nvme_command));
1540                 if (result > 0)
1541                         dev->q_depth = result;
1542                 else
1543                         nvme_release_cmb(dev);
1544         }
1545
1546         size = db_bar_size(dev, nr_io_queues);
1547         if (size > 8192) {
1548                 iounmap(dev->bar);
1549                 do {
1550                         dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1551                         if (dev->bar)
1552                                 break;
1553                         if (!--nr_io_queues)
1554                                 return -ENOMEM;
1555                         size = db_bar_size(dev, nr_io_queues);
1556                 } while (1);
1557                 dev->dbs = dev->bar + 4096;
1558                 adminq->q_db = dev->dbs;
1559         }
1560
1561         /* Deregister the admin queue's interrupt */
1562         free_irq(dev->entry[0].vector, adminq);
1563
1564         /*
1565          * If we enable msix early due to not intx, disable it again before
1566          * setting up the full range we need.
1567          */
1568         if (!pdev->irq)
1569                 pci_disable_msix(pdev);
1570
1571         for (i = 0; i < nr_io_queues; i++)
1572                 dev->entry[i].entry = i;
1573         vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
1574         if (vecs < 0) {
1575                 vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
1576                 if (vecs < 0) {
1577                         vecs = 1;
1578                 } else {
1579                         for (i = 0; i < vecs; i++)
1580                                 dev->entry[i].vector = i + pdev->irq;
1581                 }
1582         }
1583
1584         /*
1585          * Should investigate if there's a performance win from allocating
1586          * more queues than interrupt vectors; it might allow the submission
1587          * path to scale better, even if the receive path is limited by the
1588          * number of interrupts.
1589          */
1590         nr_io_queues = vecs;
1591         dev->max_qid = nr_io_queues;
1592
1593         result = queue_request_irq(dev, adminq, adminq->irqname);
1594         if (result) {
1595                 adminq->cq_vector = -1;
1596                 goto free_queues;
1597         }
1598
1599         /* Free previously allocated queues that are no longer usable */
1600         nvme_free_queues(dev, nr_io_queues + 1);
1601         return nvme_create_io_queues(dev);
1602
1603  free_queues:
1604         nvme_free_queues(dev, 1);
1605         return result;
1606 }
1607
1608 static void nvme_set_irq_hints(struct nvme_dev *dev)
1609 {
1610         struct nvme_queue *nvmeq;
1611         int i;
1612
1613         for (i = 0; i < dev->online_queues; i++) {
1614                 nvmeq = dev->queues[i];
1615
1616                 if (!nvmeq->tags || !(*nvmeq->tags))
1617                         continue;
1618
1619                 irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
1620                                         blk_mq_tags_cpumask(*nvmeq->tags));
1621         }
1622 }
1623
1624 static void nvme_dev_scan(struct work_struct *work)
1625 {
1626         struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
1627
1628         if (!dev->tagset.tags)
1629                 return;
1630         nvme_scan_namespaces(&dev->ctrl);
1631         nvme_set_irq_hints(dev);
1632 }
1633
1634 /*
1635  * Return: error value if an error occurred setting up the queues or calling
1636  * Identify Device.  0 if these succeeded, even if adding some of the
1637  * namespaces failed.  At the moment, these failures are silent.  TBD which
1638  * failures should be reported.
1639  */
1640 static int nvme_dev_add(struct nvme_dev *dev)
1641 {
1642         if (!dev->ctrl.tagset) {
1643                 dev->tagset.ops = &nvme_mq_ops;
1644                 dev->tagset.nr_hw_queues = dev->online_queues - 1;
1645                 dev->tagset.timeout = NVME_IO_TIMEOUT;
1646                 dev->tagset.numa_node = dev_to_node(dev->dev);
1647                 dev->tagset.queue_depth =
1648                                 min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
1649                 dev->tagset.cmd_size = nvme_cmd_size(dev);
1650                 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
1651                 dev->tagset.driver_data = dev;
1652
1653                 if (blk_mq_alloc_tag_set(&dev->tagset))
1654                         return 0;
1655                 dev->ctrl.tagset = &dev->tagset;
1656         }
1657         queue_work(nvme_workq, &dev->scan_work);
1658         return 0;
1659 }
1660
1661 static int nvme_dev_map(struct nvme_dev *dev)
1662 {
1663         u64 cap;
1664         int bars, result = -ENOMEM;
1665         struct pci_dev *pdev = to_pci_dev(dev->dev);
1666
1667         if (pci_enable_device_mem(pdev))
1668                 return result;
1669
1670         dev->entry[0].vector = pdev->irq;
1671         pci_set_master(pdev);
1672         bars = pci_select_bars(pdev, IORESOURCE_MEM);
1673         if (!bars)
1674                 goto disable_pci;
1675
1676         if (pci_request_selected_regions(pdev, bars, "nvme"))
1677                 goto disable_pci;
1678
1679         if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
1680             dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
1681                 goto disable;
1682
1683         dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
1684         if (!dev->bar)
1685                 goto disable;
1686
1687         if (readl(dev->bar + NVME_REG_CSTS) == -1) {
1688                 result = -ENODEV;
1689                 goto unmap;
1690         }
1691
1692         /*
1693          * Some devices don't advertse INTx interrupts, pre-enable a single
1694          * MSIX vec for setup. We'll adjust this later.
1695          */
1696         if (!pdev->irq) {
1697                 result = pci_enable_msix(pdev, dev->entry, 1);
1698                 if (result < 0)
1699                         goto unmap;
1700         }
1701
1702         cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
1703
1704         dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
1705         dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
1706         dev->dbs = dev->bar + 4096;
1707         if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2))
1708                 dev->cmb = nvme_map_cmb(dev);
1709
1710         return 0;
1711
1712  unmap:
1713         iounmap(dev->bar);
1714         dev->bar = NULL;
1715  disable:
1716         pci_release_regions(pdev);
1717  disable_pci:
1718         pci_disable_device(pdev);
1719         return result;
1720 }
1721
1722 static void nvme_dev_unmap(struct nvme_dev *dev)
1723 {
1724         struct pci_dev *pdev = to_pci_dev(dev->dev);
1725
1726         if (pdev->msi_enabled)
1727                 pci_disable_msi(pdev);
1728         else if (pdev->msix_enabled)
1729                 pci_disable_msix(pdev);
1730
1731         if (dev->bar) {
1732                 iounmap(dev->bar);
1733                 dev->bar = NULL;
1734                 pci_release_regions(pdev);
1735         }
1736
1737         if (pci_is_enabled(pdev))
1738                 pci_disable_device(pdev);
1739 }
1740
1741 struct nvme_delq_ctx {
1742         struct task_struct *waiter;
1743         struct kthread_worker *worker;
1744         atomic_t refcount;
1745 };
1746
1747 static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
1748 {
1749         dq->waiter = current;
1750         mb();
1751
1752         for (;;) {
1753                 set_current_state(TASK_KILLABLE);
1754                 if (!atomic_read(&dq->refcount))
1755                         break;
1756                 if (!schedule_timeout(ADMIN_TIMEOUT) ||
1757                                         fatal_signal_pending(current)) {
1758                         /*
1759                          * Disable the controller first since we can't trust it
1760                          * at this point, but leave the admin queue enabled
1761                          * until all queue deletion requests are flushed.
1762                          * FIXME: This may take a while if there are more h/w
1763                          * queues than admin tags.
1764                          */
1765                         set_current_state(TASK_RUNNING);
1766                         nvme_disable_ctrl(&dev->ctrl,
1767                                 lo_hi_readq(dev->bar + NVME_REG_CAP));
1768                         nvme_clear_queue(dev->queues[0]);
1769                         flush_kthread_worker(dq->worker);
1770                         nvme_disable_queue(dev, 0);
1771                         return;
1772                 }
1773         }
1774         set_current_state(TASK_RUNNING);
1775 }
1776
1777 static void nvme_put_dq(struct nvme_delq_ctx *dq)
1778 {
1779         atomic_dec(&dq->refcount);
1780         if (dq->waiter)
1781                 wake_up_process(dq->waiter);
1782 }
1783
1784 static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
1785 {
1786         atomic_inc(&dq->refcount);
1787         return dq;
1788 }
1789
1790 static void nvme_del_queue_end(struct nvme_queue *nvmeq)
1791 {
1792         struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
1793         nvme_put_dq(dq);
1794
1795         spin_lock_irq(&nvmeq->q_lock);
1796         nvme_process_cq(nvmeq);
1797         spin_unlock_irq(&nvmeq->q_lock);
1798 }
1799
1800 static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
1801                                                 kthread_work_func_t fn)
1802 {
1803         struct request *req;
1804         struct nvme_command c;
1805
1806         memset(&c, 0, sizeof(c));
1807         c.delete_queue.opcode = opcode;
1808         c.delete_queue.qid = cpu_to_le16(nvmeq->qid);
1809
1810         init_kthread_work(&nvmeq->cmdinfo.work, fn);
1811
1812         req = nvme_alloc_request(nvmeq->dev->ctrl.admin_q, &c, 0);
1813         if (IS_ERR(req))
1814                 return PTR_ERR(req);
1815
1816         req->timeout = ADMIN_TIMEOUT;
1817         req->end_io_data = &nvmeq->cmdinfo;
1818         blk_execute_rq_nowait(req->q, NULL, req, 0, async_cmd_info_endio);
1819         return 0;
1820 }
1821
1822 static void nvme_del_cq_work_handler(struct kthread_work *work)
1823 {
1824         struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
1825                                                         cmdinfo.work);
1826         nvme_del_queue_end(nvmeq);
1827 }
1828
1829 static int nvme_delete_cq(struct nvme_queue *nvmeq)
1830 {
1831         return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
1832                                                 nvme_del_cq_work_handler);
1833 }
1834
1835 static void nvme_del_sq_work_handler(struct kthread_work *work)
1836 {
1837         struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
1838                                                         cmdinfo.work);
1839         int status = nvmeq->cmdinfo.status;
1840
1841         if (!status)
1842                 status = nvme_delete_cq(nvmeq);
1843         if (status)
1844                 nvme_del_queue_end(nvmeq);
1845 }
1846
1847 static int nvme_delete_sq(struct nvme_queue *nvmeq)
1848 {
1849         return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
1850                                                 nvme_del_sq_work_handler);
1851 }
1852
1853 static void nvme_del_queue_start(struct kthread_work *work)
1854 {
1855         struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
1856                                                         cmdinfo.work);
1857         if (nvme_delete_sq(nvmeq))
1858                 nvme_del_queue_end(nvmeq);
1859 }
1860
1861 static void nvme_disable_io_queues(struct nvme_dev *dev)
1862 {
1863         int i;
1864         DEFINE_KTHREAD_WORKER_ONSTACK(worker);
1865         struct nvme_delq_ctx dq;
1866         struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
1867                                         &worker, "nvme%d", dev->ctrl.instance);
1868
1869         if (IS_ERR(kworker_task)) {
1870                 dev_err(dev->dev,
1871                         "Failed to create queue del task\n");
1872                 for (i = dev->queue_count - 1; i > 0; i--)
1873                         nvme_disable_queue(dev, i);
1874                 return;
1875         }
1876
1877         dq.waiter = NULL;
1878         atomic_set(&dq.refcount, 0);
1879         dq.worker = &worker;
1880         for (i = dev->queue_count - 1; i > 0; i--) {
1881                 struct nvme_queue *nvmeq = dev->queues[i];
1882
1883                 if (nvme_suspend_queue(nvmeq))
1884                         continue;
1885                 nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
1886                 nvmeq->cmdinfo.worker = dq.worker;
1887                 init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
1888                 queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
1889         }
1890         nvme_wait_dq(&dq, dev);
1891         kthread_stop(kworker_task);
1892 }
1893
1894 static int nvme_dev_list_add(struct nvme_dev *dev)
1895 {
1896         bool start_thread = false;
1897
1898         spin_lock(&dev_list_lock);
1899         if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
1900                 start_thread = true;
1901                 nvme_thread = NULL;
1902         }
1903         list_add(&dev->node, &dev_list);
1904         spin_unlock(&dev_list_lock);
1905
1906         if (start_thread) {
1907                 nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
1908                 wake_up_all(&nvme_kthread_wait);
1909         } else
1910                 wait_event_killable(nvme_kthread_wait, nvme_thread);
1911
1912         if (IS_ERR_OR_NULL(nvme_thread))
1913                 return nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
1914
1915         return 0;
1916 }
1917
1918 /*
1919 * Remove the node from the device list and check
1920 * for whether or not we need to stop the nvme_thread.
1921 */
1922 static void nvme_dev_list_remove(struct nvme_dev *dev)
1923 {
1924         struct task_struct *tmp = NULL;
1925
1926         spin_lock(&dev_list_lock);
1927         list_del_init(&dev->node);
1928         if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
1929                 tmp = nvme_thread;
1930                 nvme_thread = NULL;
1931         }
1932         spin_unlock(&dev_list_lock);
1933
1934         if (tmp)
1935                 kthread_stop(tmp);
1936 }
1937
1938 static void nvme_freeze_queues(struct nvme_dev *dev)
1939 {
1940         struct nvme_ns *ns;
1941
1942         list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
1943                 blk_mq_freeze_queue_start(ns->queue);
1944
1945                 spin_lock_irq(ns->queue->queue_lock);
1946                 queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
1947                 spin_unlock_irq(ns->queue->queue_lock);
1948
1949                 blk_mq_cancel_requeue_work(ns->queue);
1950                 blk_mq_stop_hw_queues(ns->queue);
1951         }
1952 }
1953
1954 static void nvme_unfreeze_queues(struct nvme_dev *dev)
1955 {
1956         struct nvme_ns *ns;
1957
1958         list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
1959                 queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
1960                 blk_mq_unfreeze_queue(ns->queue);
1961                 blk_mq_start_stopped_hw_queues(ns->queue, true);
1962                 blk_mq_kick_requeue_list(ns->queue);
1963         }
1964 }
1965
1966 static void nvme_dev_shutdown(struct nvme_dev *dev)
1967 {
1968         int i;
1969         u32 csts = -1;
1970
1971         nvme_dev_list_remove(dev);
1972
1973         mutex_lock(&dev->shutdown_lock);
1974         if (dev->bar) {
1975                 nvme_freeze_queues(dev);
1976                 csts = readl(dev->bar + NVME_REG_CSTS);
1977         }
1978         if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
1979                 for (i = dev->queue_count - 1; i >= 0; i--) {
1980                         struct nvme_queue *nvmeq = dev->queues[i];
1981                         nvme_suspend_queue(nvmeq);
1982                 }
1983         } else {
1984                 nvme_disable_io_queues(dev);
1985                 nvme_shutdown_ctrl(&dev->ctrl);
1986                 nvme_disable_queue(dev, 0);
1987         }
1988         nvme_dev_unmap(dev);
1989
1990         for (i = dev->queue_count - 1; i >= 0; i--)
1991                 nvme_clear_queue(dev->queues[i]);
1992         mutex_unlock(&dev->shutdown_lock);
1993 }
1994
1995 static int nvme_setup_prp_pools(struct nvme_dev *dev)
1996 {
1997         dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
1998                                                 PAGE_SIZE, PAGE_SIZE, 0);
1999         if (!dev->prp_page_pool)
2000                 return -ENOMEM;
2001
2002         /* Optimisation for I/Os between 4k and 128k */
2003         dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2004                                                 256, 256, 0);
2005         if (!dev->prp_small_pool) {
2006                 dma_pool_destroy(dev->prp_page_pool);
2007                 return -ENOMEM;
2008         }
2009         return 0;
2010 }
2011
2012 static void nvme_release_prp_pools(struct nvme_dev *dev)
2013 {
2014         dma_pool_destroy(dev->prp_page_pool);
2015         dma_pool_destroy(dev->prp_small_pool);
2016 }
2017
2018 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2019 {
2020         struct nvme_dev *dev = to_nvme_dev(ctrl);
2021
2022         put_device(dev->dev);
2023         if (dev->tagset.tags)
2024                 blk_mq_free_tag_set(&dev->tagset);
2025         if (dev->ctrl.admin_q)
2026                 blk_put_queue(dev->ctrl.admin_q);
2027         kfree(dev->queues);
2028         kfree(dev->entry);
2029         kfree(dev);
2030 }
2031
2032 static void nvme_reset_work(struct work_struct *work)
2033 {
2034         struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
2035         int result;
2036
2037         if (WARN_ON(test_bit(NVME_CTRL_RESETTING, &dev->flags)))
2038                 goto out;
2039
2040         /*
2041          * If we're called to reset a live controller first shut it down before
2042          * moving on.
2043          */
2044         if (dev->bar)
2045                 nvme_dev_shutdown(dev);
2046
2047         set_bit(NVME_CTRL_RESETTING, &dev->flags);
2048
2049         result = nvme_dev_map(dev);
2050         if (result)
2051                 goto out;
2052
2053         result = nvme_configure_admin_queue(dev);
2054         if (result)
2055                 goto unmap;
2056
2057         nvme_init_queue(dev->queues[0], 0);
2058         result = nvme_alloc_admin_tags(dev);
2059         if (result)
2060                 goto disable;
2061
2062         result = nvme_init_identify(&dev->ctrl);
2063         if (result)
2064                 goto free_tags;
2065
2066         result = nvme_setup_io_queues(dev);
2067         if (result)
2068                 goto free_tags;
2069
2070         dev->ctrl.event_limit = NVME_NR_AEN_COMMANDS;
2071
2072         result = nvme_dev_list_add(dev);
2073         if (result)
2074                 goto remove;
2075
2076         /*
2077          * Keep the controller around but remove all namespaces if we don't have
2078          * any working I/O queue.
2079          */
2080         if (dev->online_queues < 2) {
2081                 dev_warn(dev->dev, "IO queues not created\n");
2082                 nvme_remove_namespaces(&dev->ctrl);
2083         } else {
2084                 nvme_unfreeze_queues(dev);
2085                 nvme_dev_add(dev);
2086         }
2087
2088         clear_bit(NVME_CTRL_RESETTING, &dev->flags);
2089         return;
2090
2091  remove:
2092         nvme_dev_list_remove(dev);
2093  free_tags:
2094         nvme_dev_remove_admin(dev);
2095         blk_put_queue(dev->ctrl.admin_q);
2096         dev->ctrl.admin_q = NULL;
2097         dev->queues[0]->tags = NULL;
2098  disable:
2099         nvme_disable_queue(dev, 0);
2100  unmap:
2101         nvme_dev_unmap(dev);
2102  out:
2103         nvme_remove_dead_ctrl(dev);
2104 }
2105
2106 static void nvme_remove_dead_ctrl_work(struct work_struct *work)
2107 {
2108         struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2109         struct pci_dev *pdev = to_pci_dev(dev->dev);
2110
2111         if (pci_get_drvdata(pdev))
2112                 pci_stop_and_remove_bus_device_locked(pdev);
2113         nvme_put_ctrl(&dev->ctrl);
2114 }
2115
2116 static void nvme_remove_dead_ctrl(struct nvme_dev *dev)
2117 {
2118         dev_warn(dev->dev, "Removing after probe failure\n");
2119         kref_get(&dev->ctrl.kref);
2120         if (!schedule_work(&dev->remove_work))
2121                 nvme_put_ctrl(&dev->ctrl);
2122 }
2123
2124 static int nvme_reset(struct nvme_dev *dev)
2125 {
2126         if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q))
2127                 return -ENODEV;
2128
2129         if (!queue_work(nvme_workq, &dev->reset_work))
2130                 return -EBUSY;
2131
2132         flush_work(&dev->reset_work);
2133         return 0;
2134 }
2135
2136 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
2137 {
2138         *val = readl(to_nvme_dev(ctrl)->bar + off);
2139         return 0;
2140 }
2141
2142 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2143 {
2144         writel(val, to_nvme_dev(ctrl)->bar + off);
2145         return 0;
2146 }
2147
2148 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
2149 {
2150         *val = readq(to_nvme_dev(ctrl)->bar + off);
2151         return 0;
2152 }
2153
2154 static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl)
2155 {
2156         struct nvme_dev *dev = to_nvme_dev(ctrl);
2157
2158         return !dev->bar || dev->online_queues < 2;
2159 }
2160
2161 static int nvme_pci_reset_ctrl(struct nvme_ctrl *ctrl)
2162 {
2163         return nvme_reset(to_nvme_dev(ctrl));
2164 }
2165
2166 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
2167         .reg_read32             = nvme_pci_reg_read32,
2168         .reg_write32            = nvme_pci_reg_write32,
2169         .reg_read64             = nvme_pci_reg_read64,
2170         .io_incapable           = nvme_pci_io_incapable,
2171         .reset_ctrl             = nvme_pci_reset_ctrl,
2172         .free_ctrl              = nvme_pci_free_ctrl,
2173 };
2174
2175 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2176 {
2177         int node, result = -ENOMEM;
2178         struct nvme_dev *dev;
2179
2180         node = dev_to_node(&pdev->dev);
2181         if (node == NUMA_NO_NODE)
2182                 set_dev_node(&pdev->dev, 0);
2183
2184         dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
2185         if (!dev)
2186                 return -ENOMEM;
2187         dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
2188                                                         GFP_KERNEL, node);
2189         if (!dev->entry)
2190                 goto free;
2191         dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
2192                                                         GFP_KERNEL, node);
2193         if (!dev->queues)
2194                 goto free;
2195
2196         dev->dev = get_device(&pdev->dev);
2197         pci_set_drvdata(pdev, dev);
2198
2199         INIT_LIST_HEAD(&dev->node);
2200         INIT_WORK(&dev->scan_work, nvme_dev_scan);
2201         INIT_WORK(&dev->reset_work, nvme_reset_work);
2202         INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2203         mutex_init(&dev->shutdown_lock);
2204
2205         result = nvme_setup_prp_pools(dev);
2206         if (result)
2207                 goto put_pci;
2208
2209         result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
2210                         id->driver_data);
2211         if (result)
2212                 goto release_pools;
2213
2214         queue_work(nvme_workq, &dev->reset_work);
2215         return 0;
2216
2217  release_pools:
2218         nvme_release_prp_pools(dev);
2219  put_pci:
2220         put_device(dev->dev);
2221  free:
2222         kfree(dev->queues);
2223         kfree(dev->entry);
2224         kfree(dev);
2225         return result;
2226 }
2227
2228 static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
2229 {
2230         struct nvme_dev *dev = pci_get_drvdata(pdev);
2231
2232         if (prepare)
2233                 nvme_dev_shutdown(dev);
2234         else
2235                 queue_work(nvme_workq, &dev->reset_work);
2236 }
2237
2238 static void nvme_shutdown(struct pci_dev *pdev)
2239 {
2240         struct nvme_dev *dev = pci_get_drvdata(pdev);
2241         nvme_dev_shutdown(dev);
2242 }
2243
2244 static void nvme_remove(struct pci_dev *pdev)
2245 {
2246         struct nvme_dev *dev = pci_get_drvdata(pdev);
2247
2248         spin_lock(&dev_list_lock);
2249         list_del_init(&dev->node);
2250         spin_unlock(&dev_list_lock);
2251
2252         pci_set_drvdata(pdev, NULL);
2253         flush_work(&dev->reset_work);
2254         flush_work(&dev->scan_work);
2255         nvme_remove_namespaces(&dev->ctrl);
2256         nvme_uninit_ctrl(&dev->ctrl);
2257         nvme_dev_shutdown(dev);
2258         nvme_dev_remove_admin(dev);
2259         nvme_free_queues(dev, 0);
2260         nvme_release_cmb(dev);
2261         nvme_release_prp_pools(dev);
2262         nvme_put_ctrl(&dev->ctrl);
2263 }
2264
2265 /* These functions are yet to be implemented */
2266 #define nvme_error_detected NULL
2267 #define nvme_dump_registers NULL
2268 #define nvme_link_reset NULL
2269 #define nvme_slot_reset NULL
2270 #define nvme_error_resume NULL
2271
2272 #ifdef CONFIG_PM_SLEEP
2273 static int nvme_suspend(struct device *dev)
2274 {
2275         struct pci_dev *pdev = to_pci_dev(dev);
2276         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2277
2278         nvme_dev_shutdown(ndev);
2279         return 0;
2280 }
2281
2282 static int nvme_resume(struct device *dev)
2283 {
2284         struct pci_dev *pdev = to_pci_dev(dev);
2285         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2286
2287         queue_work(nvme_workq, &ndev->reset_work);
2288         return 0;
2289 }
2290 #endif
2291
2292 static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
2293
2294 static const struct pci_error_handlers nvme_err_handler = {
2295         .error_detected = nvme_error_detected,
2296         .mmio_enabled   = nvme_dump_registers,
2297         .link_reset     = nvme_link_reset,
2298         .slot_reset     = nvme_slot_reset,
2299         .resume         = nvme_error_resume,
2300         .reset_notify   = nvme_reset_notify,
2301 };
2302
2303 /* Move to pci_ids.h later */
2304 #define PCI_CLASS_STORAGE_EXPRESS       0x010802
2305
2306 static const struct pci_device_id nvme_id_table[] = {
2307         { PCI_VDEVICE(INTEL, 0x0953),
2308                 .driver_data = NVME_QUIRK_STRIPE_SIZE, },
2309         { PCI_VDEVICE(INTEL, 0x5845),   /* Qemu emulated controller */
2310                 .driver_data = NVME_QUIRK_IDENTIFY_CNS, },
2311         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2312         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
2313         { 0, }
2314 };
2315 MODULE_DEVICE_TABLE(pci, nvme_id_table);
2316
2317 static struct pci_driver nvme_driver = {
2318         .name           = "nvme",
2319         .id_table       = nvme_id_table,
2320         .probe          = nvme_probe,
2321         .remove         = nvme_remove,
2322         .shutdown       = nvme_shutdown,
2323         .driver         = {
2324                 .pm     = &nvme_dev_pm_ops,
2325         },
2326         .err_handler    = &nvme_err_handler,
2327 };
2328
2329 static int __init nvme_init(void)
2330 {
2331         int result;
2332
2333         init_waitqueue_head(&nvme_kthread_wait);
2334
2335         nvme_workq = alloc_workqueue("nvme", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
2336         if (!nvme_workq)
2337                 return -ENOMEM;
2338
2339         result = nvme_core_init();
2340         if (result < 0)
2341                 goto kill_workq;
2342
2343         result = pci_register_driver(&nvme_driver);
2344         if (result)
2345                 goto core_exit;
2346         return 0;
2347
2348  core_exit:
2349         nvme_core_exit();
2350  kill_workq:
2351         destroy_workqueue(nvme_workq);
2352         return result;
2353 }
2354
2355 static void __exit nvme_exit(void)
2356 {
2357         pci_unregister_driver(&nvme_driver);
2358         nvme_core_exit();
2359         destroy_workqueue(nvme_workq);
2360         BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
2361         _nvme_check_size();
2362 }
2363
2364 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2365 MODULE_LICENSE("GPL");
2366 MODULE_VERSION("1.0");
2367 module_init(nvme_init);
2368 module_exit(nvme_exit);