nvme: move nvme_{enable,disable,shutdown}_ctrl to common code
[linux-2.6-block.git] / drivers / nvme / host / pci.c
1 /*
2  * NVM Express device driver
3  * Copyright (c) 2011-2014, Intel Corporation.
4  *
5  * This program is free software; you can redistribute it and/or modify it
6  * under the terms and conditions of the GNU General Public License,
7  * version 2, as published by the Free Software Foundation.
8  *
9  * This program is distributed in the hope it will be useful, but WITHOUT
10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
12  * more details.
13  */
14
15 #include <linux/bitops.h>
16 #include <linux/blkdev.h>
17 #include <linux/blk-mq.h>
18 #include <linux/cpu.h>
19 #include <linux/delay.h>
20 #include <linux/errno.h>
21 #include <linux/fs.h>
22 #include <linux/genhd.h>
23 #include <linux/hdreg.h>
24 #include <linux/idr.h>
25 #include <linux/init.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/kdev_t.h>
29 #include <linux/kthread.h>
30 #include <linux/kernel.h>
31 #include <linux/list_sort.h>
32 #include <linux/mm.h>
33 #include <linux/module.h>
34 #include <linux/moduleparam.h>
35 #include <linux/pci.h>
36 #include <linux/poison.h>
37 #include <linux/ptrace.h>
38 #include <linux/sched.h>
39 #include <linux/slab.h>
40 #include <linux/t10-pi.h>
41 #include <linux/types.h>
42 #include <linux/pr.h>
43 #include <scsi/sg.h>
44 #include <linux/io-64-nonatomic-lo-hi.h>
45 #include <asm/unaligned.h>
46
47 #include <uapi/linux/nvme_ioctl.h>
48 #include "nvme.h"
49
50 #define NVME_MINORS             (1U << MINORBITS)
51 #define NVME_Q_DEPTH            1024
52 #define NVME_AQ_DEPTH           256
53 #define SQ_SIZE(depth)          (depth * sizeof(struct nvme_command))
54 #define CQ_SIZE(depth)          (depth * sizeof(struct nvme_completion))
55
56 unsigned char admin_timeout = 60;
57 module_param(admin_timeout, byte, 0644);
58 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
59
60 unsigned char nvme_io_timeout = 30;
61 module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
62 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
63
64 unsigned char shutdown_timeout = 5;
65 module_param(shutdown_timeout, byte, 0644);
66 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
67
68 static int nvme_major;
69 module_param(nvme_major, int, 0);
70
71 static int nvme_char_major;
72 module_param(nvme_char_major, int, 0);
73
74 static int use_threaded_interrupts;
75 module_param(use_threaded_interrupts, int, 0);
76
77 static bool use_cmb_sqes = true;
78 module_param(use_cmb_sqes, bool, 0644);
79 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
80
81 static LIST_HEAD(dev_list);
82 static struct task_struct *nvme_thread;
83 static struct workqueue_struct *nvme_workq;
84 static wait_queue_head_t nvme_kthread_wait;
85
86 static struct class *nvme_class;
87
88 struct nvme_dev;
89 struct nvme_queue;
90 struct nvme_iod;
91
92 static int __nvme_reset(struct nvme_dev *dev);
93 static int nvme_reset(struct nvme_dev *dev);
94 static void nvme_process_cq(struct nvme_queue *nvmeq);
95 static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod);
96 static void nvme_dead_ctrl(struct nvme_dev *dev);
97
98 struct async_cmd_info {
99         struct kthread_work work;
100         struct kthread_worker *worker;
101         struct request *req;
102         u32 result;
103         int status;
104         void *ctx;
105 };
106
107 /*
108  * Represents an NVM Express device.  Each nvme_dev is a PCI function.
109  */
110 struct nvme_dev {
111         struct list_head node;
112         struct nvme_queue **queues;
113         struct blk_mq_tag_set tagset;
114         struct blk_mq_tag_set admin_tagset;
115         u32 __iomem *dbs;
116         struct device *dev;
117         struct dma_pool *prp_page_pool;
118         struct dma_pool *prp_small_pool;
119         unsigned queue_count;
120         unsigned online_queues;
121         unsigned max_qid;
122         int q_depth;
123         u32 db_stride;
124         struct msix_entry *entry;
125         void __iomem *bar;
126         struct list_head namespaces;
127         struct device *device;
128         struct work_struct reset_work;
129         struct work_struct probe_work;
130         struct work_struct scan_work;
131         bool subsystem;
132         u32 max_hw_sectors;
133         u32 stripe_size;
134         void __iomem *cmb;
135         dma_addr_t cmb_dma_addr;
136         u64 cmb_size;
137         u32 cmbsz;
138
139         struct nvme_ctrl ctrl;
140 };
141
142 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
143 {
144         return container_of(ctrl, struct nvme_dev, ctrl);
145 }
146
147 /*
148  * An NVM Express queue.  Each device has at least two (one for admin
149  * commands and one for I/O commands).
150  */
151 struct nvme_queue {
152         struct device *q_dmadev;
153         struct nvme_dev *dev;
154         char irqname[24];       /* nvme4294967295-65535\0 */
155         spinlock_t q_lock;
156         struct nvme_command *sq_cmds;
157         struct nvme_command __iomem *sq_cmds_io;
158         volatile struct nvme_completion *cqes;
159         struct blk_mq_tags **tags;
160         dma_addr_t sq_dma_addr;
161         dma_addr_t cq_dma_addr;
162         u32 __iomem *q_db;
163         u16 q_depth;
164         s16 cq_vector;
165         u16 sq_head;
166         u16 sq_tail;
167         u16 cq_head;
168         u16 qid;
169         u8 cq_phase;
170         u8 cqe_seen;
171         struct async_cmd_info cmdinfo;
172 };
173
174 /*
175  * The nvme_iod describes the data in an I/O, including the list of PRP
176  * entries.  You can't see it in this data structure because C doesn't let
177  * me express that.  Use nvme_alloc_iod to ensure there's enough space
178  * allocated to store the PRP list.
179  */
180 struct nvme_iod {
181         unsigned long private;  /* For the use of the submitter of the I/O */
182         int npages;             /* In the PRP list. 0 means small pool in use */
183         int offset;             /* Of PRP list */
184         int nents;              /* Used in scatterlist */
185         int length;             /* Of data, in bytes */
186         dma_addr_t first_dma;
187         struct scatterlist meta_sg[1]; /* metadata requires single contiguous buffer */
188         struct scatterlist sg[0];
189 };
190
191 /*
192  * Check we didin't inadvertently grow the command struct
193  */
194 static inline void _nvme_check_size(void)
195 {
196         BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
197         BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
198         BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
199         BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
200         BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
201         BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
202         BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
203         BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
204         BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
205         BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
206         BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
207         BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
208 }
209
210 typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
211                                                 struct nvme_completion *);
212
213 struct nvme_cmd_info {
214         nvme_completion_fn fn;
215         void *ctx;
216         int aborted;
217         struct nvme_queue *nvmeq;
218         struct nvme_iod iod[0];
219 };
220
221 /*
222  * Max size of iod being embedded in the request payload
223  */
224 #define NVME_INT_PAGES          2
225 #define NVME_INT_BYTES(dev)     (NVME_INT_PAGES * (dev)->ctrl.page_size)
226 #define NVME_INT_MASK           0x01
227
228 /*
229  * Will slightly overestimate the number of pages needed.  This is OK
230  * as it only leads to a small amount of wasted memory for the lifetime of
231  * the I/O.
232  */
233 static int nvme_npages(unsigned size, struct nvme_dev *dev)
234 {
235         unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
236                                       dev->ctrl.page_size);
237         return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
238 }
239
240 static unsigned int nvme_cmd_size(struct nvme_dev *dev)
241 {
242         unsigned int ret = sizeof(struct nvme_cmd_info);
243
244         ret += sizeof(struct nvme_iod);
245         ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
246         ret += sizeof(struct scatterlist) * NVME_INT_PAGES;
247
248         return ret;
249 }
250
251 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
252                                 unsigned int hctx_idx)
253 {
254         struct nvme_dev *dev = data;
255         struct nvme_queue *nvmeq = dev->queues[0];
256
257         WARN_ON(hctx_idx != 0);
258         WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
259         WARN_ON(nvmeq->tags);
260
261         hctx->driver_data = nvmeq;
262         nvmeq->tags = &dev->admin_tagset.tags[0];
263         return 0;
264 }
265
266 static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
267 {
268         struct nvme_queue *nvmeq = hctx->driver_data;
269
270         nvmeq->tags = NULL;
271 }
272
273 static int nvme_admin_init_request(void *data, struct request *req,
274                                 unsigned int hctx_idx, unsigned int rq_idx,
275                                 unsigned int numa_node)
276 {
277         struct nvme_dev *dev = data;
278         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
279         struct nvme_queue *nvmeq = dev->queues[0];
280
281         BUG_ON(!nvmeq);
282         cmd->nvmeq = nvmeq;
283         return 0;
284 }
285
286 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
287                           unsigned int hctx_idx)
288 {
289         struct nvme_dev *dev = data;
290         struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
291
292         if (!nvmeq->tags)
293                 nvmeq->tags = &dev->tagset.tags[hctx_idx];
294
295         WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
296         hctx->driver_data = nvmeq;
297         return 0;
298 }
299
300 static int nvme_init_request(void *data, struct request *req,
301                                 unsigned int hctx_idx, unsigned int rq_idx,
302                                 unsigned int numa_node)
303 {
304         struct nvme_dev *dev = data;
305         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
306         struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
307
308         BUG_ON(!nvmeq);
309         cmd->nvmeq = nvmeq;
310         return 0;
311 }
312
313 static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
314                                 nvme_completion_fn handler)
315 {
316         cmd->fn = handler;
317         cmd->ctx = ctx;
318         cmd->aborted = 0;
319         blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
320 }
321
322 static void *iod_get_private(struct nvme_iod *iod)
323 {
324         return (void *) (iod->private & ~0x1UL);
325 }
326
327 /*
328  * If bit 0 is set, the iod is embedded in the request payload.
329  */
330 static bool iod_should_kfree(struct nvme_iod *iod)
331 {
332         return (iod->private & NVME_INT_MASK) == 0;
333 }
334
335 /* Special values must be less than 0x1000 */
336 #define CMD_CTX_BASE            ((void *)POISON_POINTER_DELTA)
337 #define CMD_CTX_CANCELLED       (0x30C + CMD_CTX_BASE)
338 #define CMD_CTX_COMPLETED       (0x310 + CMD_CTX_BASE)
339 #define CMD_CTX_INVALID         (0x314 + CMD_CTX_BASE)
340
341 static void special_completion(struct nvme_queue *nvmeq, void *ctx,
342                                                 struct nvme_completion *cqe)
343 {
344         if (ctx == CMD_CTX_CANCELLED)
345                 return;
346         if (ctx == CMD_CTX_COMPLETED) {
347                 dev_warn(nvmeq->q_dmadev,
348                                 "completed id %d twice on queue %d\n",
349                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
350                 return;
351         }
352         if (ctx == CMD_CTX_INVALID) {
353                 dev_warn(nvmeq->q_dmadev,
354                                 "invalid id %d completed on queue %d\n",
355                                 cqe->command_id, le16_to_cpup(&cqe->sq_id));
356                 return;
357         }
358         dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
359 }
360
361 static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
362 {
363         void *ctx;
364
365         if (fn)
366                 *fn = cmd->fn;
367         ctx = cmd->ctx;
368         cmd->fn = special_completion;
369         cmd->ctx = CMD_CTX_CANCELLED;
370         return ctx;
371 }
372
373 static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
374                                                 struct nvme_completion *cqe)
375 {
376         u32 result = le32_to_cpup(&cqe->result);
377         u16 status = le16_to_cpup(&cqe->status) >> 1;
378
379         if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
380                 ++nvmeq->dev->ctrl.event_limit;
381         if (status != NVME_SC_SUCCESS)
382                 return;
383
384         switch (result & 0xff07) {
385         case NVME_AER_NOTICE_NS_CHANGED:
386                 dev_info(nvmeq->q_dmadev, "rescanning\n");
387                 schedule_work(&nvmeq->dev->scan_work);
388         default:
389                 dev_warn(nvmeq->q_dmadev, "async event result %08x\n", result);
390         }
391 }
392
393 static void abort_completion(struct nvme_queue *nvmeq, void *ctx,
394                                                 struct nvme_completion *cqe)
395 {
396         struct request *req = ctx;
397
398         u16 status = le16_to_cpup(&cqe->status) >> 1;
399         u32 result = le32_to_cpup(&cqe->result);
400
401         blk_mq_free_request(req);
402
403         dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
404         ++nvmeq->dev->ctrl.abort_limit;
405 }
406
407 static void async_completion(struct nvme_queue *nvmeq, void *ctx,
408                                                 struct nvme_completion *cqe)
409 {
410         struct async_cmd_info *cmdinfo = ctx;
411         cmdinfo->result = le32_to_cpup(&cqe->result);
412         cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
413         queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
414         blk_mq_free_request(cmdinfo->req);
415 }
416
417 static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
418                                   unsigned int tag)
419 {
420         struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, tag);
421
422         return blk_mq_rq_to_pdu(req);
423 }
424
425 /*
426  * Called with local interrupts disabled and the q_lock held.  May not sleep.
427  */
428 static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
429                                                 nvme_completion_fn *fn)
430 {
431         struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
432         void *ctx;
433         if (tag >= nvmeq->q_depth) {
434                 *fn = special_completion;
435                 return CMD_CTX_INVALID;
436         }
437         if (fn)
438                 *fn = cmd->fn;
439         ctx = cmd->ctx;
440         cmd->fn = special_completion;
441         cmd->ctx = CMD_CTX_COMPLETED;
442         return ctx;
443 }
444
445 /**
446  * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
447  * @nvmeq: The queue to use
448  * @cmd: The command to send
449  *
450  * Safe to use from interrupt context
451  */
452 static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
453                                                 struct nvme_command *cmd)
454 {
455         u16 tail = nvmeq->sq_tail;
456
457         if (nvmeq->sq_cmds_io)
458                 memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
459         else
460                 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
461
462         if (++tail == nvmeq->q_depth)
463                 tail = 0;
464         writel(tail, nvmeq->q_db);
465         nvmeq->sq_tail = tail;
466 }
467
468 static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
469 {
470         unsigned long flags;
471         spin_lock_irqsave(&nvmeq->q_lock, flags);
472         __nvme_submit_cmd(nvmeq, cmd);
473         spin_unlock_irqrestore(&nvmeq->q_lock, flags);
474 }
475
476 static __le64 **iod_list(struct nvme_iod *iod)
477 {
478         return ((void *)iod) + iod->offset;
479 }
480
481 static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
482                             unsigned nseg, unsigned long private)
483 {
484         iod->private = private;
485         iod->offset = offsetof(struct nvme_iod, sg[nseg]);
486         iod->npages = -1;
487         iod->length = nbytes;
488         iod->nents = 0;
489 }
490
491 static struct nvme_iod *
492 __nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
493                  unsigned long priv, gfp_t gfp)
494 {
495         struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
496                                 sizeof(__le64 *) * nvme_npages(bytes, dev) +
497                                 sizeof(struct scatterlist) * nseg, gfp);
498
499         if (iod)
500                 iod_init(iod, bytes, nseg, priv);
501
502         return iod;
503 }
504
505 static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
506                                        gfp_t gfp)
507 {
508         unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
509                                                 sizeof(struct nvme_dsm_range);
510         struct nvme_iod *iod;
511
512         if (rq->nr_phys_segments <= NVME_INT_PAGES &&
513             size <= NVME_INT_BYTES(dev)) {
514                 struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);
515
516                 iod = cmd->iod;
517                 iod_init(iod, size, rq->nr_phys_segments,
518                                 (unsigned long) rq | NVME_INT_MASK);
519                 return iod;
520         }
521
522         return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
523                                 (unsigned long) rq, gfp);
524 }
525
526 static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
527 {
528         const int last_prp = dev->ctrl.page_size / 8 - 1;
529         int i;
530         __le64 **list = iod_list(iod);
531         dma_addr_t prp_dma = iod->first_dma;
532
533         if (iod->npages == 0)
534                 dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
535         for (i = 0; i < iod->npages; i++) {
536                 __le64 *prp_list = list[i];
537                 dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
538                 dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
539                 prp_dma = next_prp_dma;
540         }
541
542         if (iod_should_kfree(iod))
543                 kfree(iod);
544 }
545
546 #ifdef CONFIG_BLK_DEV_INTEGRITY
547 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
548 {
549         if (be32_to_cpu(pi->ref_tag) == v)
550                 pi->ref_tag = cpu_to_be32(p);
551 }
552
553 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
554 {
555         if (be32_to_cpu(pi->ref_tag) == p)
556                 pi->ref_tag = cpu_to_be32(v);
557 }
558
559 /**
560  * nvme_dif_remap - remaps ref tags to bip seed and physical lba
561  *
562  * The virtual start sector is the one that was originally submitted by the
563  * block layer. Due to partitioning, MD/DM cloning, etc. the actual physical
564  * start sector may be different. Remap protection information to match the
565  * physical LBA on writes, and back to the original seed on reads.
566  *
567  * Type 0 and 3 do not have a ref tag, so no remapping required.
568  */
569 static void nvme_dif_remap(struct request *req,
570                         void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
571 {
572         struct nvme_ns *ns = req->rq_disk->private_data;
573         struct bio_integrity_payload *bip;
574         struct t10_pi_tuple *pi;
575         void *p, *pmap;
576         u32 i, nlb, ts, phys, virt;
577
578         if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
579                 return;
580
581         bip = bio_integrity(req->bio);
582         if (!bip)
583                 return;
584
585         pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;
586
587         p = pmap;
588         virt = bip_get_seed(bip);
589         phys = nvme_block_nr(ns, blk_rq_pos(req));
590         nlb = (blk_rq_bytes(req) >> ns->lba_shift);
591         ts = ns->disk->queue->integrity.tuple_size;
592
593         for (i = 0; i < nlb; i++, virt++, phys++) {
594                 pi = (struct t10_pi_tuple *)p;
595                 dif_swap(phys, virt, pi);
596                 p += ts;
597         }
598         kunmap_atomic(pmap);
599 }
600 #else /* CONFIG_BLK_DEV_INTEGRITY */
601 static void nvme_dif_remap(struct request *req,
602                         void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
603 {
604 }
605 static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
606 {
607 }
608 static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
609 {
610 }
611 #endif
612
613 static void req_completion(struct nvme_queue *nvmeq, void *ctx,
614                                                 struct nvme_completion *cqe)
615 {
616         struct nvme_iod *iod = ctx;
617         struct request *req = iod_get_private(iod);
618         struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
619         u16 status = le16_to_cpup(&cqe->status) >> 1;
620         int error = 0;
621
622         if (unlikely(status)) {
623                 if (!(status & NVME_SC_DNR || blk_noretry_request(req))
624                     && (jiffies - req->start_time) < req->timeout) {
625                         unsigned long flags;
626
627                         nvme_unmap_data(nvmeq->dev, iod);
628
629                         blk_mq_requeue_request(req);
630                         spin_lock_irqsave(req->q->queue_lock, flags);
631                         if (!blk_queue_stopped(req->q))
632                                 blk_mq_kick_requeue_list(req->q);
633                         spin_unlock_irqrestore(req->q->queue_lock, flags);
634                         return;
635                 }
636
637                 if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
638                         if (cmd_rq->ctx == CMD_CTX_CANCELLED)
639                                 error = -EINTR;
640                         else
641                                 error = status;
642                 } else {
643                         error = nvme_error_status(status);
644                 }
645         }
646
647         if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
648                 u32 result = le32_to_cpup(&cqe->result);
649                 req->special = (void *)(uintptr_t)result;
650         }
651
652         if (cmd_rq->aborted)
653                 dev_warn(nvmeq->dev->dev,
654                         "completing aborted command with status:%04x\n",
655                         error);
656
657         nvme_unmap_data(nvmeq->dev, iod);
658         blk_mq_complete_request(req, error);
659 }
660
661 static bool nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
662                 int total_len)
663 {
664         struct dma_pool *pool;
665         int length = total_len;
666         struct scatterlist *sg = iod->sg;
667         int dma_len = sg_dma_len(sg);
668         u64 dma_addr = sg_dma_address(sg);
669         u32 page_size = dev->ctrl.page_size;
670         int offset = dma_addr & (page_size - 1);
671         __le64 *prp_list;
672         __le64 **list = iod_list(iod);
673         dma_addr_t prp_dma;
674         int nprps, i;
675
676         length -= (page_size - offset);
677         if (length <= 0)
678                 return true;
679
680         dma_len -= (page_size - offset);
681         if (dma_len) {
682                 dma_addr += (page_size - offset);
683         } else {
684                 sg = sg_next(sg);
685                 dma_addr = sg_dma_address(sg);
686                 dma_len = sg_dma_len(sg);
687         }
688
689         if (length <= page_size) {
690                 iod->first_dma = dma_addr;
691                 return true;
692         }
693
694         nprps = DIV_ROUND_UP(length, page_size);
695         if (nprps <= (256 / 8)) {
696                 pool = dev->prp_small_pool;
697                 iod->npages = 0;
698         } else {
699                 pool = dev->prp_page_pool;
700                 iod->npages = 1;
701         }
702
703         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
704         if (!prp_list) {
705                 iod->first_dma = dma_addr;
706                 iod->npages = -1;
707                 return false;
708         }
709         list[0] = prp_list;
710         iod->first_dma = prp_dma;
711         i = 0;
712         for (;;) {
713                 if (i == page_size >> 3) {
714                         __le64 *old_prp_list = prp_list;
715                         prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
716                         if (!prp_list)
717                                 return false;
718                         list[iod->npages++] = prp_list;
719                         prp_list[0] = old_prp_list[i - 1];
720                         old_prp_list[i - 1] = cpu_to_le64(prp_dma);
721                         i = 1;
722                 }
723                 prp_list[i++] = cpu_to_le64(dma_addr);
724                 dma_len -= page_size;
725                 dma_addr += page_size;
726                 length -= page_size;
727                 if (length <= 0)
728                         break;
729                 if (dma_len > 0)
730                         continue;
731                 BUG_ON(dma_len < 0);
732                 sg = sg_next(sg);
733                 dma_addr = sg_dma_address(sg);
734                 dma_len = sg_dma_len(sg);
735         }
736
737         return true;
738 }
739
740 static int nvme_map_data(struct nvme_dev *dev, struct nvme_iod *iod,
741                 struct nvme_command *cmnd)
742 {
743         struct request *req = iod_get_private(iod);
744         struct request_queue *q = req->q;
745         enum dma_data_direction dma_dir = rq_data_dir(req) ?
746                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
747         int ret = BLK_MQ_RQ_QUEUE_ERROR;
748
749         sg_init_table(iod->sg, req->nr_phys_segments);
750         iod->nents = blk_rq_map_sg(q, req, iod->sg);
751         if (!iod->nents)
752                 goto out;
753
754         ret = BLK_MQ_RQ_QUEUE_BUSY;
755         if (!dma_map_sg(dev->dev, iod->sg, iod->nents, dma_dir))
756                 goto out;
757
758         if (!nvme_setup_prps(dev, iod, blk_rq_bytes(req)))
759                 goto out_unmap;
760
761         ret = BLK_MQ_RQ_QUEUE_ERROR;
762         if (blk_integrity_rq(req)) {
763                 if (blk_rq_count_integrity_sg(q, req->bio) != 1)
764                         goto out_unmap;
765
766                 sg_init_table(iod->meta_sg, 1);
767                 if (blk_rq_map_integrity_sg(q, req->bio, iod->meta_sg) != 1)
768                         goto out_unmap;
769
770                 if (rq_data_dir(req))
771                         nvme_dif_remap(req, nvme_dif_prep);
772
773                 if (!dma_map_sg(dev->dev, iod->meta_sg, 1, dma_dir))
774                         goto out_unmap;
775         }
776
777         cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
778         cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
779         if (blk_integrity_rq(req))
780                 cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));
781         return BLK_MQ_RQ_QUEUE_OK;
782
783 out_unmap:
784         dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
785 out:
786         return ret;
787 }
788
789 static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod)
790 {
791         struct request *req = iod_get_private(iod);
792         enum dma_data_direction dma_dir = rq_data_dir(req) ?
793                         DMA_TO_DEVICE : DMA_FROM_DEVICE;
794
795         if (iod->nents) {
796                 dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
797                 if (blk_integrity_rq(req)) {
798                         if (!rq_data_dir(req))
799                                 nvme_dif_remap(req, nvme_dif_complete);
800                         dma_unmap_sg(dev->dev, iod->meta_sg, 1, dma_dir);
801                 }
802         }
803
804         nvme_free_iod(dev, iod);
805 }
806
807 /*
808  * We reuse the small pool to allocate the 16-byte range here as it is not
809  * worth having a special pool for these or additional cases to handle freeing
810  * the iod.
811  */
812 static int nvme_setup_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
813                 struct nvme_iod *iod, struct nvme_command *cmnd)
814 {
815         struct request *req = iod_get_private(iod);
816         struct nvme_dsm_range *range;
817
818         range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
819                                                 &iod->first_dma);
820         if (!range)
821                 return BLK_MQ_RQ_QUEUE_BUSY;
822         iod_list(iod)[0] = (__le64 *)range;
823         iod->npages = 0;
824
825         range->cattr = cpu_to_le32(0);
826         range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
827         range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
828
829         memset(cmnd, 0, sizeof(*cmnd));
830         cmnd->dsm.opcode = nvme_cmd_dsm;
831         cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
832         cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
833         cmnd->dsm.nr = 0;
834         cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
835         return BLK_MQ_RQ_QUEUE_OK;
836 }
837
838 /*
839  * NOTE: ns is NULL when called on the admin queue.
840  */
841 static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
842                          const struct blk_mq_queue_data *bd)
843 {
844         struct nvme_ns *ns = hctx->queue->queuedata;
845         struct nvme_queue *nvmeq = hctx->driver_data;
846         struct nvme_dev *dev = nvmeq->dev;
847         struct request *req = bd->rq;
848         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
849         struct nvme_iod *iod;
850         struct nvme_command cmnd;
851         int ret = BLK_MQ_RQ_QUEUE_OK;
852
853         /*
854          * If formated with metadata, require the block layer provide a buffer
855          * unless this namespace is formated such that the metadata can be
856          * stripped/generated by the controller with PRACT=1.
857          */
858         if (ns && ns->ms && !blk_integrity_rq(req)) {
859                 if (!(ns->pi_type && ns->ms == 8) &&
860                                         req->cmd_type != REQ_TYPE_DRV_PRIV) {
861                         blk_mq_complete_request(req, -EFAULT);
862                         return BLK_MQ_RQ_QUEUE_OK;
863                 }
864         }
865
866         iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
867         if (!iod)
868                 return BLK_MQ_RQ_QUEUE_BUSY;
869
870         if (req->cmd_flags & REQ_DISCARD) {
871                 ret = nvme_setup_discard(nvmeq, ns, iod, &cmnd);
872         } else {
873                 if (req->cmd_type == REQ_TYPE_DRV_PRIV)
874                         memcpy(&cmnd, req->cmd, sizeof(cmnd));
875                 else if (req->cmd_flags & REQ_FLUSH)
876                         nvme_setup_flush(ns, &cmnd);
877                 else
878                         nvme_setup_rw(ns, req, &cmnd);
879
880                 if (req->nr_phys_segments)
881                         ret = nvme_map_data(dev, iod, &cmnd);
882         }
883
884         if (ret)
885                 goto out;
886
887         cmnd.common.command_id = req->tag;
888         nvme_set_info(cmd, iod, req_completion);
889
890         spin_lock_irq(&nvmeq->q_lock);
891         __nvme_submit_cmd(nvmeq, &cmnd);
892         nvme_process_cq(nvmeq);
893         spin_unlock_irq(&nvmeq->q_lock);
894         return BLK_MQ_RQ_QUEUE_OK;
895 out:
896         nvme_free_iod(dev, iod);
897         return ret;
898 }
899
900 static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
901 {
902         u16 head, phase;
903
904         head = nvmeq->cq_head;
905         phase = nvmeq->cq_phase;
906
907         for (;;) {
908                 void *ctx;
909                 nvme_completion_fn fn;
910                 struct nvme_completion cqe = nvmeq->cqes[head];
911                 if ((le16_to_cpu(cqe.status) & 1) != phase)
912                         break;
913                 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
914                 if (++head == nvmeq->q_depth) {
915                         head = 0;
916                         phase = !phase;
917                 }
918                 if (tag && *tag == cqe.command_id)
919                         *tag = -1;
920                 ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
921                 fn(nvmeq, ctx, &cqe);
922         }
923
924         /* If the controller ignores the cq head doorbell and continuously
925          * writes to the queue, it is theoretically possible to wrap around
926          * the queue twice and mistakenly return IRQ_NONE.  Linux only
927          * requires that 0.1% of your interrupts are handled, so this isn't
928          * a big problem.
929          */
930         if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
931                 return;
932
933         if (likely(nvmeq->cq_vector >= 0))
934                 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
935         nvmeq->cq_head = head;
936         nvmeq->cq_phase = phase;
937
938         nvmeq->cqe_seen = 1;
939 }
940
941 static void nvme_process_cq(struct nvme_queue *nvmeq)
942 {
943         __nvme_process_cq(nvmeq, NULL);
944 }
945
946 static irqreturn_t nvme_irq(int irq, void *data)
947 {
948         irqreturn_t result;
949         struct nvme_queue *nvmeq = data;
950         spin_lock(&nvmeq->q_lock);
951         nvme_process_cq(nvmeq);
952         result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
953         nvmeq->cqe_seen = 0;
954         spin_unlock(&nvmeq->q_lock);
955         return result;
956 }
957
958 static irqreturn_t nvme_irq_check(int irq, void *data)
959 {
960         struct nvme_queue *nvmeq = data;
961         struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
962         if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
963                 return IRQ_NONE;
964         return IRQ_WAKE_THREAD;
965 }
966
967 static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
968 {
969         struct nvme_queue *nvmeq = hctx->driver_data;
970
971         if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
972             nvmeq->cq_phase) {
973                 spin_lock_irq(&nvmeq->q_lock);
974                 __nvme_process_cq(nvmeq, &tag);
975                 spin_unlock_irq(&nvmeq->q_lock);
976
977                 if (tag == -1)
978                         return 1;
979         }
980
981         return 0;
982 }
983
984 static int nvme_submit_async_admin_req(struct nvme_dev *dev)
985 {
986         struct nvme_queue *nvmeq = dev->queues[0];
987         struct nvme_command c;
988         struct nvme_cmd_info *cmd_info;
989         struct request *req;
990
991         req = blk_mq_alloc_request(dev->ctrl.admin_q, WRITE,
992                         BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED);
993         if (IS_ERR(req))
994                 return PTR_ERR(req);
995
996         req->cmd_flags |= REQ_NO_TIMEOUT;
997         cmd_info = blk_mq_rq_to_pdu(req);
998         nvme_set_info(cmd_info, NULL, async_req_completion);
999
1000         memset(&c, 0, sizeof(c));
1001         c.common.opcode = nvme_admin_async_event;
1002         c.common.command_id = req->tag;
1003
1004         blk_mq_free_request(req);
1005         __nvme_submit_cmd(nvmeq, &c);
1006         return 0;
1007 }
1008
1009 static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
1010                         struct nvme_command *cmd,
1011                         struct async_cmd_info *cmdinfo, unsigned timeout)
1012 {
1013         struct nvme_queue *nvmeq = dev->queues[0];
1014         struct request *req;
1015         struct nvme_cmd_info *cmd_rq;
1016
1017         req = blk_mq_alloc_request(dev->ctrl.admin_q, WRITE, 0);
1018         if (IS_ERR(req))
1019                 return PTR_ERR(req);
1020
1021         req->timeout = timeout;
1022         cmd_rq = blk_mq_rq_to_pdu(req);
1023         cmdinfo->req = req;
1024         nvme_set_info(cmd_rq, cmdinfo, async_completion);
1025         cmdinfo->status = -EINTR;
1026
1027         cmd->common.command_id = req->tag;
1028
1029         nvme_submit_cmd(nvmeq, cmd);
1030         return 0;
1031 }
1032
1033 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1034 {
1035         struct nvme_command c;
1036
1037         memset(&c, 0, sizeof(c));
1038         c.delete_queue.opcode = opcode;
1039         c.delete_queue.qid = cpu_to_le16(id);
1040
1041         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1042 }
1043
1044 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1045                                                 struct nvme_queue *nvmeq)
1046 {
1047         struct nvme_command c;
1048         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
1049
1050         /*
1051          * Note: we (ab)use the fact the the prp fields survive if no data
1052          * is attached to the request.
1053          */
1054         memset(&c, 0, sizeof(c));
1055         c.create_cq.opcode = nvme_admin_create_cq;
1056         c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
1057         c.create_cq.cqid = cpu_to_le16(qid);
1058         c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1059         c.create_cq.cq_flags = cpu_to_le16(flags);
1060         c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
1061
1062         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1063 }
1064
1065 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1066                                                 struct nvme_queue *nvmeq)
1067 {
1068         struct nvme_command c;
1069         int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
1070
1071         /*
1072          * Note: we (ab)use the fact the the prp fields survive if no data
1073          * is attached to the request.
1074          */
1075         memset(&c, 0, sizeof(c));
1076         c.create_sq.opcode = nvme_admin_create_sq;
1077         c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
1078         c.create_sq.sqid = cpu_to_le16(qid);
1079         c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1080         c.create_sq.sq_flags = cpu_to_le16(flags);
1081         c.create_sq.cqid = cpu_to_le16(qid);
1082
1083         return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1084 }
1085
1086 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1087 {
1088         return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
1089 }
1090
1091 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1092 {
1093         return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
1094 }
1095
1096 /**
1097  * nvme_abort_req - Attempt aborting a request
1098  *
1099  * Schedule controller reset if the command was already aborted once before and
1100  * still hasn't been returned to the driver, or if this is the admin queue.
1101  */
1102 static void nvme_abort_req(struct request *req)
1103 {
1104         struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
1105         struct nvme_queue *nvmeq = cmd_rq->nvmeq;
1106         struct nvme_dev *dev = nvmeq->dev;
1107         struct request *abort_req;
1108         struct nvme_cmd_info *abort_cmd;
1109         struct nvme_command cmd;
1110
1111         if (!nvmeq->qid || cmd_rq->aborted) {
1112                 spin_lock(&dev_list_lock);
1113                 if (!__nvme_reset(dev)) {
1114                         dev_warn(dev->dev,
1115                                  "I/O %d QID %d timeout, reset controller\n",
1116                                  req->tag, nvmeq->qid);
1117                 }
1118                 spin_unlock(&dev_list_lock);
1119                 return;
1120         }
1121
1122         if (!dev->ctrl.abort_limit)
1123                 return;
1124
1125         abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, WRITE,
1126                         BLK_MQ_REQ_NOWAIT);
1127         if (IS_ERR(abort_req))
1128                 return;
1129
1130         abort_cmd = blk_mq_rq_to_pdu(abort_req);
1131         nvme_set_info(abort_cmd, abort_req, abort_completion);
1132
1133         memset(&cmd, 0, sizeof(cmd));
1134         cmd.abort.opcode = nvme_admin_abort_cmd;
1135         cmd.abort.cid = req->tag;
1136         cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1137         cmd.abort.command_id = abort_req->tag;
1138
1139         --dev->ctrl.abort_limit;
1140         cmd_rq->aborted = 1;
1141
1142         dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag,
1143                                                         nvmeq->qid);
1144         nvme_submit_cmd(dev->queues[0], &cmd);
1145 }
1146
1147 static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
1148 {
1149         struct nvme_queue *nvmeq = data;
1150         void *ctx;
1151         nvme_completion_fn fn;
1152         struct nvme_cmd_info *cmd;
1153         struct nvme_completion cqe;
1154
1155         if (!blk_mq_request_started(req))
1156                 return;
1157
1158         cmd = blk_mq_rq_to_pdu(req);
1159
1160         if (cmd->ctx == CMD_CTX_CANCELLED)
1161                 return;
1162
1163         if (blk_queue_dying(req->q))
1164                 cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
1165         else
1166                 cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);
1167
1168
1169         dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
1170                                                 req->tag, nvmeq->qid);
1171         ctx = cancel_cmd_info(cmd, &fn);
1172         fn(nvmeq, ctx, &cqe);
1173 }
1174
1175 static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
1176 {
1177         struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
1178         struct nvme_queue *nvmeq = cmd->nvmeq;
1179
1180         dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag,
1181                                                         nvmeq->qid);
1182         spin_lock_irq(&nvmeq->q_lock);
1183         nvme_abort_req(req);
1184         spin_unlock_irq(&nvmeq->q_lock);
1185
1186         /*
1187          * The aborted req will be completed on receiving the abort req.
1188          * We enable the timer again. If hit twice, it'll cause a device reset,
1189          * as the device then is in a faulty state.
1190          */
1191         return BLK_EH_RESET_TIMER;
1192 }
1193
1194 static void nvme_free_queue(struct nvme_queue *nvmeq)
1195 {
1196         dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
1197                                 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1198         if (nvmeq->sq_cmds)
1199                 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1200                                         nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1201         kfree(nvmeq);
1202 }
1203
1204 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1205 {
1206         int i;
1207
1208         for (i = dev->queue_count - 1; i >= lowest; i--) {
1209                 struct nvme_queue *nvmeq = dev->queues[i];
1210                 dev->queue_count--;
1211                 dev->queues[i] = NULL;
1212                 nvme_free_queue(nvmeq);
1213         }
1214 }
1215
1216 /**
1217  * nvme_suspend_queue - put queue into suspended state
1218  * @nvmeq - queue to suspend
1219  */
1220 static int nvme_suspend_queue(struct nvme_queue *nvmeq)
1221 {
1222         int vector;
1223
1224         spin_lock_irq(&nvmeq->q_lock);
1225         if (nvmeq->cq_vector == -1) {
1226                 spin_unlock_irq(&nvmeq->q_lock);
1227                 return 1;
1228         }
1229         vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
1230         nvmeq->dev->online_queues--;
1231         nvmeq->cq_vector = -1;
1232         spin_unlock_irq(&nvmeq->q_lock);
1233
1234         if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1235                 blk_mq_freeze_queue_start(nvmeq->dev->ctrl.admin_q);
1236
1237         irq_set_affinity_hint(vector, NULL);
1238         free_irq(vector, nvmeq);
1239
1240         return 0;
1241 }
1242
1243 static void nvme_clear_queue(struct nvme_queue *nvmeq)
1244 {
1245         spin_lock_irq(&nvmeq->q_lock);
1246         if (nvmeq->tags && *nvmeq->tags)
1247                 blk_mq_all_tag_busy_iter(*nvmeq->tags, nvme_cancel_queue_ios, nvmeq);
1248         spin_unlock_irq(&nvmeq->q_lock);
1249 }
1250
1251 static void nvme_disable_queue(struct nvme_dev *dev, int qid)
1252 {
1253         struct nvme_queue *nvmeq = dev->queues[qid];
1254
1255         if (!nvmeq)
1256                 return;
1257         if (nvme_suspend_queue(nvmeq))
1258                 return;
1259
1260         /* Don't tell the adapter to delete the admin queue.
1261          * Don't tell a removed adapter to delete IO queues. */
1262         if (qid && readl(dev->bar + NVME_REG_CSTS) != -1) {
1263                 adapter_delete_sq(dev, qid);
1264                 adapter_delete_cq(dev, qid);
1265         }
1266
1267         spin_lock_irq(&nvmeq->q_lock);
1268         nvme_process_cq(nvmeq);
1269         spin_unlock_irq(&nvmeq->q_lock);
1270 }
1271
1272 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1273                                 int entry_size)
1274 {
1275         int q_depth = dev->q_depth;
1276         unsigned q_size_aligned = roundup(q_depth * entry_size,
1277                                           dev->ctrl.page_size);
1278
1279         if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1280                 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1281                 mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1282                 q_depth = div_u64(mem_per_q, entry_size);
1283
1284                 /*
1285                  * Ensure the reduced q_depth is above some threshold where it
1286                  * would be better to map queues in system memory with the
1287                  * original depth
1288                  */
1289                 if (q_depth < 64)
1290                         return -ENOMEM;
1291         }
1292
1293         return q_depth;
1294 }
1295
1296 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1297                                 int qid, int depth)
1298 {
1299         if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
1300                 unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
1301                                                       dev->ctrl.page_size);
1302                 nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
1303                 nvmeq->sq_cmds_io = dev->cmb + offset;
1304         } else {
1305                 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
1306                                         &nvmeq->sq_dma_addr, GFP_KERNEL);
1307                 if (!nvmeq->sq_cmds)
1308                         return -ENOMEM;
1309         }
1310
1311         return 0;
1312 }
1313
1314 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
1315                                                         int depth)
1316 {
1317         struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
1318         if (!nvmeq)
1319                 return NULL;
1320
1321         nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
1322                                           &nvmeq->cq_dma_addr, GFP_KERNEL);
1323         if (!nvmeq->cqes)
1324                 goto free_nvmeq;
1325
1326         if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
1327                 goto free_cqdma;
1328
1329         nvmeq->q_dmadev = dev->dev;
1330         nvmeq->dev = dev;
1331         snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1332                         dev->ctrl.instance, qid);
1333         spin_lock_init(&nvmeq->q_lock);
1334         nvmeq->cq_head = 0;
1335         nvmeq->cq_phase = 1;
1336         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1337         nvmeq->q_depth = depth;
1338         nvmeq->qid = qid;
1339         nvmeq->cq_vector = -1;
1340         dev->queues[qid] = nvmeq;
1341
1342         /* make sure queue descriptor is set before queue count, for kthread */
1343         mb();
1344         dev->queue_count++;
1345
1346         return nvmeq;
1347
1348  free_cqdma:
1349         dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
1350                                                         nvmeq->cq_dma_addr);
1351  free_nvmeq:
1352         kfree(nvmeq);
1353         return NULL;
1354 }
1355
1356 static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1357                                                         const char *name)
1358 {
1359         if (use_threaded_interrupts)
1360                 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1361                                         nvme_irq_check, nvme_irq, IRQF_SHARED,
1362                                         name, nvmeq);
1363         return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1364                                 IRQF_SHARED, name, nvmeq);
1365 }
1366
1367 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1368 {
1369         struct nvme_dev *dev = nvmeq->dev;
1370
1371         spin_lock_irq(&nvmeq->q_lock);
1372         nvmeq->sq_tail = 0;
1373         nvmeq->cq_head = 0;
1374         nvmeq->cq_phase = 1;
1375         nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1376         memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1377         dev->online_queues++;
1378         spin_unlock_irq(&nvmeq->q_lock);
1379 }
1380
1381 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
1382 {
1383         struct nvme_dev *dev = nvmeq->dev;
1384         int result;
1385
1386         nvmeq->cq_vector = qid - 1;
1387         result = adapter_alloc_cq(dev, qid, nvmeq);
1388         if (result < 0)
1389                 return result;
1390
1391         result = adapter_alloc_sq(dev, qid, nvmeq);
1392         if (result < 0)
1393                 goto release_cq;
1394
1395         result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1396         if (result < 0)
1397                 goto release_sq;
1398
1399         nvme_init_queue(nvmeq, qid);
1400         return result;
1401
1402  release_sq:
1403         adapter_delete_sq(dev, qid);
1404  release_cq:
1405         adapter_delete_cq(dev, qid);
1406         return result;
1407 }
1408
1409 static struct blk_mq_ops nvme_mq_admin_ops = {
1410         .queue_rq       = nvme_queue_rq,
1411         .map_queue      = blk_mq_map_queue,
1412         .init_hctx      = nvme_admin_init_hctx,
1413         .exit_hctx      = nvme_admin_exit_hctx,
1414         .init_request   = nvme_admin_init_request,
1415         .timeout        = nvme_timeout,
1416 };
1417
1418 static struct blk_mq_ops nvme_mq_ops = {
1419         .queue_rq       = nvme_queue_rq,
1420         .map_queue      = blk_mq_map_queue,
1421         .init_hctx      = nvme_init_hctx,
1422         .init_request   = nvme_init_request,
1423         .timeout        = nvme_timeout,
1424         .poll           = nvme_poll,
1425 };
1426
1427 static void nvme_dev_remove_admin(struct nvme_dev *dev)
1428 {
1429         if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1430                 blk_cleanup_queue(dev->ctrl.admin_q);
1431                 blk_mq_free_tag_set(&dev->admin_tagset);
1432         }
1433 }
1434
1435 static int nvme_alloc_admin_tags(struct nvme_dev *dev)
1436 {
1437         if (!dev->ctrl.admin_q) {
1438                 dev->admin_tagset.ops = &nvme_mq_admin_ops;
1439                 dev->admin_tagset.nr_hw_queues = 1;
1440                 dev->admin_tagset.queue_depth = NVME_AQ_DEPTH - 1;
1441                 dev->admin_tagset.reserved_tags = 1;
1442                 dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1443                 dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1444                 dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
1445                 dev->admin_tagset.driver_data = dev;
1446
1447                 if (blk_mq_alloc_tag_set(&dev->admin_tagset))
1448                         return -ENOMEM;
1449
1450                 dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
1451                 if (IS_ERR(dev->ctrl.admin_q)) {
1452                         blk_mq_free_tag_set(&dev->admin_tagset);
1453                         return -ENOMEM;
1454                 }
1455                 if (!blk_get_queue(dev->ctrl.admin_q)) {
1456                         nvme_dev_remove_admin(dev);
1457                         dev->ctrl.admin_q = NULL;
1458                         return -ENODEV;
1459                 }
1460         } else
1461                 blk_mq_unfreeze_queue(dev->ctrl.admin_q);
1462
1463         return 0;
1464 }
1465
1466 static int nvme_configure_admin_queue(struct nvme_dev *dev)
1467 {
1468         int result;
1469         u32 aqa;
1470         u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
1471         struct nvme_queue *nvmeq;
1472
1473         dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
1474                                                 NVME_CAP_NSSRC(cap) : 0;
1475
1476         if (dev->subsystem &&
1477             (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
1478                 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1479
1480         result = nvme_disable_ctrl(&dev->ctrl, cap);
1481         if (result < 0)
1482                 return result;
1483
1484         nvmeq = dev->queues[0];
1485         if (!nvmeq) {
1486                 nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1487                 if (!nvmeq)
1488                         return -ENOMEM;
1489         }
1490
1491         aqa = nvmeq->q_depth - 1;
1492         aqa |= aqa << 16;
1493
1494         writel(aqa, dev->bar + NVME_REG_AQA);
1495         lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
1496         lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
1497
1498         result = nvme_enable_ctrl(&dev->ctrl, cap);
1499         if (result)
1500                 goto free_nvmeq;
1501
1502         nvmeq->cq_vector = 0;
1503         result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1504         if (result) {
1505                 nvmeq->cq_vector = -1;
1506                 goto free_nvmeq;
1507         }
1508
1509         return result;
1510
1511  free_nvmeq:
1512         nvme_free_queues(dev, 0);
1513         return result;
1514 }
1515
1516 static int nvme_subsys_reset(struct nvme_dev *dev)
1517 {
1518         if (!dev->subsystem)
1519                 return -ENOTTY;
1520
1521         writel(0x4E564D65, dev->bar + NVME_REG_NSSR); /* "NVMe" */
1522         return 0;
1523 }
1524
1525 static int nvme_kthread(void *data)
1526 {
1527         struct nvme_dev *dev, *next;
1528
1529         while (!kthread_should_stop()) {
1530                 set_current_state(TASK_INTERRUPTIBLE);
1531                 spin_lock(&dev_list_lock);
1532                 list_for_each_entry_safe(dev, next, &dev_list, node) {
1533                         int i;
1534                         u32 csts = readl(dev->bar + NVME_REG_CSTS);
1535
1536                         if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
1537                                                         csts & NVME_CSTS_CFS) {
1538                                 if (!__nvme_reset(dev)) {
1539                                         dev_warn(dev->dev,
1540                                                 "Failed status: %x, reset controller\n",
1541                                                 readl(dev->bar + NVME_REG_CSTS));
1542                                 }
1543                                 continue;
1544                         }
1545                         for (i = 0; i < dev->queue_count; i++) {
1546                                 struct nvme_queue *nvmeq = dev->queues[i];
1547                                 if (!nvmeq)
1548                                         continue;
1549                                 spin_lock_irq(&nvmeq->q_lock);
1550                                 nvme_process_cq(nvmeq);
1551
1552                                 while (i == 0 && dev->ctrl.event_limit > 0) {
1553                                         if (nvme_submit_async_admin_req(dev))
1554                                                 break;
1555                                         dev->ctrl.event_limit--;
1556                                 }
1557                                 spin_unlock_irq(&nvmeq->q_lock);
1558                         }
1559                 }
1560                 spin_unlock(&dev_list_lock);
1561                 schedule_timeout(round_jiffies_relative(HZ));
1562         }
1563         return 0;
1564 }
1565
1566 static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
1567 {
1568         struct nvme_ns *ns;
1569         struct gendisk *disk;
1570         int node = dev_to_node(dev->dev);
1571
1572         ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
1573         if (!ns)
1574                 return;
1575
1576         ns->queue = blk_mq_init_queue(&dev->tagset);
1577         if (IS_ERR(ns->queue))
1578                 goto out_free_ns;
1579         queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
1580         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
1581         ns->ctrl = &dev->ctrl;
1582         ns->queue->queuedata = ns;
1583
1584         disk = alloc_disk_node(0, node);
1585         if (!disk)
1586                 goto out_free_queue;
1587
1588         kref_init(&ns->kref);
1589         ns->ns_id = nsid;
1590         ns->disk = disk;
1591         ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
1592         list_add_tail(&ns->list, &dev->namespaces);
1593
1594         blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
1595         if (dev->max_hw_sectors) {
1596                 blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
1597                 blk_queue_max_segments(ns->queue,
1598                         (dev->max_hw_sectors / (dev->ctrl.page_size >> 9)) + 1);
1599         }
1600         if (dev->stripe_size)
1601                 blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
1602         if (dev->ctrl.vwc & NVME_CTRL_VWC_PRESENT)
1603                 blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
1604         blk_queue_virt_boundary(ns->queue, dev->ctrl.page_size - 1);
1605
1606         disk->major = nvme_major;
1607         disk->first_minor = 0;
1608         disk->fops = &nvme_fops;
1609         disk->private_data = ns;
1610         disk->queue = ns->queue;
1611         disk->driverfs_dev = dev->device;
1612         disk->flags = GENHD_FL_EXT_DEVT;
1613         sprintf(disk->disk_name, "nvme%dn%d", dev->ctrl.instance, nsid);
1614
1615         /*
1616          * Initialize capacity to 0 until we establish the namespace format and
1617          * setup integrity extentions if necessary. The revalidate_disk after
1618          * add_disk allows the driver to register with integrity if the format
1619          * requires it.
1620          */
1621         set_capacity(disk, 0);
1622         if (nvme_revalidate_disk(ns->disk))
1623                 goto out_free_disk;
1624
1625         kref_get(&dev->ctrl.kref);
1626         if (ns->type != NVME_NS_LIGHTNVM) {
1627                 add_disk(ns->disk);
1628                 if (ns->ms) {
1629                         struct block_device *bd = bdget_disk(ns->disk, 0);
1630                         if (!bd)
1631                                 return;
1632                         if (blkdev_get(bd, FMODE_READ, NULL)) {
1633                                 bdput(bd);
1634                                 return;
1635                         }
1636                         blkdev_reread_part(bd);
1637                         blkdev_put(bd, FMODE_READ);
1638                 }
1639         }
1640         return;
1641  out_free_disk:
1642         kfree(disk);
1643         list_del(&ns->list);
1644  out_free_queue:
1645         blk_cleanup_queue(ns->queue);
1646  out_free_ns:
1647         kfree(ns);
1648 }
1649
1650 /*
1651  * Create I/O queues.  Failing to create an I/O queue is not an issue,
1652  * we can continue with less than the desired amount of queues, and
1653  * even a controller without I/O queues an still be used to issue
1654  * admin commands.  This might be useful to upgrade a buggy firmware
1655  * for example.
1656  */
1657 static void nvme_create_io_queues(struct nvme_dev *dev)
1658 {
1659         unsigned i;
1660
1661         for (i = dev->queue_count; i <= dev->max_qid; i++)
1662                 if (!nvme_alloc_queue(dev, i, dev->q_depth))
1663                         break;
1664
1665         for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
1666                 if (nvme_create_queue(dev->queues[i], i)) {
1667                         nvme_free_queues(dev, i);
1668                         break;
1669                 }
1670 }
1671
1672 static int set_queue_count(struct nvme_dev *dev, int count)
1673 {
1674         int status;
1675         u32 result;
1676         u32 q_count = (count - 1) | ((count - 1) << 16);
1677
1678         status = nvme_set_features(&dev->ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0,
1679                                                                 &result);
1680         if (status < 0)
1681                 return status;
1682         if (status > 0) {
1683                 dev_err(dev->dev, "Could not set queue count (%d)\n", status);
1684                 return 0;
1685         }
1686         return min(result & 0xffff, result >> 16) + 1;
1687 }
1688
1689 static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
1690 {
1691         u64 szu, size, offset;
1692         u32 cmbloc;
1693         resource_size_t bar_size;
1694         struct pci_dev *pdev = to_pci_dev(dev->dev);
1695         void __iomem *cmb;
1696         dma_addr_t dma_addr;
1697
1698         if (!use_cmb_sqes)
1699                 return NULL;
1700
1701         dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1702         if (!(NVME_CMB_SZ(dev->cmbsz)))
1703                 return NULL;
1704
1705         cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1706
1707         szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
1708         size = szu * NVME_CMB_SZ(dev->cmbsz);
1709         offset = szu * NVME_CMB_OFST(cmbloc);
1710         bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc));
1711
1712         if (offset > bar_size)
1713                 return NULL;
1714
1715         /*
1716          * Controllers may support a CMB size larger than their BAR,
1717          * for example, due to being behind a bridge. Reduce the CMB to
1718          * the reported size of the BAR
1719          */
1720         if (size > bar_size - offset)
1721                 size = bar_size - offset;
1722
1723         dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset;
1724         cmb = ioremap_wc(dma_addr, size);
1725         if (!cmb)
1726                 return NULL;
1727
1728         dev->cmb_dma_addr = dma_addr;
1729         dev->cmb_size = size;
1730         return cmb;
1731 }
1732
1733 static inline void nvme_release_cmb(struct nvme_dev *dev)
1734 {
1735         if (dev->cmb) {
1736                 iounmap(dev->cmb);
1737                 dev->cmb = NULL;
1738         }
1739 }
1740
1741 static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1742 {
1743         return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
1744 }
1745
1746 static int nvme_setup_io_queues(struct nvme_dev *dev)
1747 {
1748         struct nvme_queue *adminq = dev->queues[0];
1749         struct pci_dev *pdev = to_pci_dev(dev->dev);
1750         int result, i, vecs, nr_io_queues, size;
1751
1752         nr_io_queues = num_possible_cpus();
1753         result = set_queue_count(dev, nr_io_queues);
1754         if (result <= 0)
1755                 return result;
1756         if (result < nr_io_queues)
1757                 nr_io_queues = result;
1758
1759         if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
1760                 result = nvme_cmb_qdepth(dev, nr_io_queues,
1761                                 sizeof(struct nvme_command));
1762                 if (result > 0)
1763                         dev->q_depth = result;
1764                 else
1765                         nvme_release_cmb(dev);
1766         }
1767
1768         size = db_bar_size(dev, nr_io_queues);
1769         if (size > 8192) {
1770                 iounmap(dev->bar);
1771                 do {
1772                         dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1773                         if (dev->bar)
1774                                 break;
1775                         if (!--nr_io_queues)
1776                                 return -ENOMEM;
1777                         size = db_bar_size(dev, nr_io_queues);
1778                 } while (1);
1779                 dev->dbs = dev->bar + 4096;
1780                 adminq->q_db = dev->dbs;
1781         }
1782
1783         /* Deregister the admin queue's interrupt */
1784         free_irq(dev->entry[0].vector, adminq);
1785
1786         /*
1787          * If we enable msix early due to not intx, disable it again before
1788          * setting up the full range we need.
1789          */
1790         if (!pdev->irq)
1791                 pci_disable_msix(pdev);
1792
1793         for (i = 0; i < nr_io_queues; i++)
1794                 dev->entry[i].entry = i;
1795         vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
1796         if (vecs < 0) {
1797                 vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
1798                 if (vecs < 0) {
1799                         vecs = 1;
1800                 } else {
1801                         for (i = 0; i < vecs; i++)
1802                                 dev->entry[i].vector = i + pdev->irq;
1803                 }
1804         }
1805
1806         /*
1807          * Should investigate if there's a performance win from allocating
1808          * more queues than interrupt vectors; it might allow the submission
1809          * path to scale better, even if the receive path is limited by the
1810          * number of interrupts.
1811          */
1812         nr_io_queues = vecs;
1813         dev->max_qid = nr_io_queues;
1814
1815         result = queue_request_irq(dev, adminq, adminq->irqname);
1816         if (result) {
1817                 adminq->cq_vector = -1;
1818                 goto free_queues;
1819         }
1820
1821         /* Free previously allocated queues that are no longer usable */
1822         nvme_free_queues(dev, nr_io_queues + 1);
1823         nvme_create_io_queues(dev);
1824
1825         return 0;
1826
1827  free_queues:
1828         nvme_free_queues(dev, 1);
1829         return result;
1830 }
1831
1832 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
1833 {
1834         struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
1835         struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
1836
1837         return nsa->ns_id - nsb->ns_id;
1838 }
1839
1840 static struct nvme_ns *nvme_find_ns(struct nvme_dev *dev, unsigned nsid)
1841 {
1842         struct nvme_ns *ns;
1843
1844         list_for_each_entry(ns, &dev->namespaces, list) {
1845                 if (ns->ns_id == nsid)
1846                         return ns;
1847                 if (ns->ns_id > nsid)
1848                         break;
1849         }
1850         return NULL;
1851 }
1852
1853 static inline bool nvme_io_incapable(struct nvme_dev *dev)
1854 {
1855         return (!dev->bar ||
1856                 readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_CFS ||
1857                 dev->online_queues < 2);
1858 }
1859
1860 static void nvme_ns_remove(struct nvme_ns *ns)
1861 {
1862         bool kill = nvme_io_incapable(to_nvme_dev(ns->ctrl)) &&
1863                         !blk_queue_dying(ns->queue);
1864
1865         if (kill)
1866                 blk_set_queue_dying(ns->queue);
1867         if (ns->disk->flags & GENHD_FL_UP)
1868                 del_gendisk(ns->disk);
1869         if (kill || !blk_queue_dying(ns->queue)) {
1870                 blk_mq_abort_requeue_list(ns->queue);
1871                 blk_cleanup_queue(ns->queue);
1872         }
1873         list_del_init(&ns->list);
1874         nvme_put_ns(ns);
1875 }
1876
1877 static void nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn)
1878 {
1879         struct nvme_ns *ns, *next;
1880         unsigned i;
1881
1882         for (i = 1; i <= nn; i++) {
1883                 ns = nvme_find_ns(dev, i);
1884                 if (ns) {
1885                         if (revalidate_disk(ns->disk))
1886                                 nvme_ns_remove(ns);
1887                 } else
1888                         nvme_alloc_ns(dev, i);
1889         }
1890         list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
1891                 if (ns->ns_id > nn)
1892                         nvme_ns_remove(ns);
1893         }
1894         list_sort(NULL, &dev->namespaces, ns_cmp);
1895 }
1896
1897 static void nvme_set_irq_hints(struct nvme_dev *dev)
1898 {
1899         struct nvme_queue *nvmeq;
1900         int i;
1901
1902         for (i = 0; i < dev->online_queues; i++) {
1903                 nvmeq = dev->queues[i];
1904
1905                 if (!nvmeq->tags || !(*nvmeq->tags))
1906                         continue;
1907
1908                 irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
1909                                         blk_mq_tags_cpumask(*nvmeq->tags));
1910         }
1911 }
1912
1913 static void nvme_dev_scan(struct work_struct *work)
1914 {
1915         struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
1916         struct nvme_id_ctrl *ctrl;
1917
1918         if (!dev->tagset.tags)
1919                 return;
1920         if (nvme_identify_ctrl(&dev->ctrl, &ctrl))
1921                 return;
1922         nvme_scan_namespaces(dev, le32_to_cpup(&ctrl->nn));
1923         kfree(ctrl);
1924         nvme_set_irq_hints(dev);
1925 }
1926
1927 /*
1928  * Return: error value if an error occurred setting up the queues or calling
1929  * Identify Device.  0 if these succeeded, even if adding some of the
1930  * namespaces failed.  At the moment, these failures are silent.  TBD which
1931  * failures should be reported.
1932  */
1933 static int nvme_dev_add(struct nvme_dev *dev)
1934 {
1935         int res;
1936         struct nvme_id_ctrl *ctrl;
1937         int shift = NVME_CAP_MPSMIN(lo_hi_readq(dev->bar + NVME_REG_CAP)) + 12;
1938
1939         res = nvme_identify_ctrl(&dev->ctrl, &ctrl);
1940         if (res) {
1941                 dev_err(dev->dev, "Identify Controller failed (%d)\n", res);
1942                 return -EIO;
1943         }
1944
1945         dev->ctrl.oncs = le16_to_cpup(&ctrl->oncs);
1946         dev->ctrl.abort_limit = ctrl->acl + 1;
1947         dev->ctrl.vwc = ctrl->vwc;
1948         memcpy(dev->ctrl.serial, ctrl->sn, sizeof(ctrl->sn));
1949         memcpy(dev->ctrl.model, ctrl->mn, sizeof(ctrl->mn));
1950         memcpy(dev->ctrl.firmware_rev, ctrl->fr, sizeof(ctrl->fr));
1951         if (ctrl->mdts)
1952                 dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
1953         else
1954                 dev->max_hw_sectors = UINT_MAX;
1955
1956         if ((dev->ctrl.quirks & NVME_QUIRK_STRIPE_SIZE) && ctrl->vs[3]) {
1957                 unsigned int max_hw_sectors;
1958
1959                 dev->stripe_size = 1 << (ctrl->vs[3] + shift);
1960                 max_hw_sectors = dev->stripe_size >> (shift - 9);
1961                 if (dev->max_hw_sectors) {
1962                         dev->max_hw_sectors = min(max_hw_sectors,
1963                                                         dev->max_hw_sectors);
1964                 } else
1965                         dev->max_hw_sectors = max_hw_sectors;
1966         }
1967         kfree(ctrl);
1968
1969         if (!dev->tagset.tags) {
1970                 dev->tagset.ops = &nvme_mq_ops;
1971                 dev->tagset.nr_hw_queues = dev->online_queues - 1;
1972                 dev->tagset.timeout = NVME_IO_TIMEOUT;
1973                 dev->tagset.numa_node = dev_to_node(dev->dev);
1974                 dev->tagset.queue_depth =
1975                                 min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
1976                 dev->tagset.cmd_size = nvme_cmd_size(dev);
1977                 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
1978                 dev->tagset.driver_data = dev;
1979
1980                 if (blk_mq_alloc_tag_set(&dev->tagset))
1981                         return 0;
1982         }
1983         schedule_work(&dev->scan_work);
1984         return 0;
1985 }
1986
1987 static int nvme_dev_map(struct nvme_dev *dev)
1988 {
1989         u64 cap;
1990         int bars, result = -ENOMEM;
1991         struct pci_dev *pdev = to_pci_dev(dev->dev);
1992
1993         if (pci_enable_device_mem(pdev))
1994                 return result;
1995
1996         dev->entry[0].vector = pdev->irq;
1997         pci_set_master(pdev);
1998         bars = pci_select_bars(pdev, IORESOURCE_MEM);
1999         if (!bars)
2000                 goto disable_pci;
2001
2002         if (pci_request_selected_regions(pdev, bars, "nvme"))
2003                 goto disable_pci;
2004
2005         if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
2006             dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
2007                 goto disable;
2008
2009         dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
2010         if (!dev->bar)
2011                 goto disable;
2012
2013         if (readl(dev->bar + NVME_REG_CSTS) == -1) {
2014                 result = -ENODEV;
2015                 goto unmap;
2016         }
2017
2018         /*
2019          * Some devices don't advertse INTx interrupts, pre-enable a single
2020          * MSIX vec for setup. We'll adjust this later.
2021          */
2022         if (!pdev->irq) {
2023                 result = pci_enable_msix(pdev, dev->entry, 1);
2024                 if (result < 0)
2025                         goto unmap;
2026         }
2027
2028         cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
2029
2030         dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
2031         dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
2032         dev->dbs = dev->bar + 4096;
2033         if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2))
2034                 dev->cmb = nvme_map_cmb(dev);
2035
2036         return 0;
2037
2038  unmap:
2039         iounmap(dev->bar);
2040         dev->bar = NULL;
2041  disable:
2042         pci_release_regions(pdev);
2043  disable_pci:
2044         pci_disable_device(pdev);
2045         return result;
2046 }
2047
2048 static void nvme_dev_unmap(struct nvme_dev *dev)
2049 {
2050         struct pci_dev *pdev = to_pci_dev(dev->dev);
2051
2052         if (pdev->msi_enabled)
2053                 pci_disable_msi(pdev);
2054         else if (pdev->msix_enabled)
2055                 pci_disable_msix(pdev);
2056
2057         if (dev->bar) {
2058                 iounmap(dev->bar);
2059                 dev->bar = NULL;
2060                 pci_release_regions(pdev);
2061         }
2062
2063         if (pci_is_enabled(pdev))
2064                 pci_disable_device(pdev);
2065 }
2066
2067 struct nvme_delq_ctx {
2068         struct task_struct *waiter;
2069         struct kthread_worker *worker;
2070         atomic_t refcount;
2071 };
2072
2073 static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
2074 {
2075         dq->waiter = current;
2076         mb();
2077
2078         for (;;) {
2079                 set_current_state(TASK_KILLABLE);
2080                 if (!atomic_read(&dq->refcount))
2081                         break;
2082                 if (!schedule_timeout(ADMIN_TIMEOUT) ||
2083                                         fatal_signal_pending(current)) {
2084                         /*
2085                          * Disable the controller first since we can't trust it
2086                          * at this point, but leave the admin queue enabled
2087                          * until all queue deletion requests are flushed.
2088                          * FIXME: This may take a while if there are more h/w
2089                          * queues than admin tags.
2090                          */
2091                         set_current_state(TASK_RUNNING);
2092                         nvme_disable_ctrl(&dev->ctrl,
2093                                 lo_hi_readq(dev->bar + NVME_REG_CAP));
2094                         nvme_clear_queue(dev->queues[0]);
2095                         flush_kthread_worker(dq->worker);
2096                         nvme_disable_queue(dev, 0);
2097                         return;
2098                 }
2099         }
2100         set_current_state(TASK_RUNNING);
2101 }
2102
2103 static void nvme_put_dq(struct nvme_delq_ctx *dq)
2104 {
2105         atomic_dec(&dq->refcount);
2106         if (dq->waiter)
2107                 wake_up_process(dq->waiter);
2108 }
2109
2110 static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
2111 {
2112         atomic_inc(&dq->refcount);
2113         return dq;
2114 }
2115
2116 static void nvme_del_queue_end(struct nvme_queue *nvmeq)
2117 {
2118         struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
2119         nvme_put_dq(dq);
2120
2121         spin_lock_irq(&nvmeq->q_lock);
2122         nvme_process_cq(nvmeq);
2123         spin_unlock_irq(&nvmeq->q_lock);
2124 }
2125
2126 static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
2127                                                 kthread_work_func_t fn)
2128 {
2129         struct nvme_command c;
2130
2131         memset(&c, 0, sizeof(c));
2132         c.delete_queue.opcode = opcode;
2133         c.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2134
2135         init_kthread_work(&nvmeq->cmdinfo.work, fn);
2136         return nvme_submit_admin_async_cmd(nvmeq->dev, &c, &nvmeq->cmdinfo,
2137                                                                 ADMIN_TIMEOUT);
2138 }
2139
2140 static void nvme_del_cq_work_handler(struct kthread_work *work)
2141 {
2142         struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2143                                                         cmdinfo.work);
2144         nvme_del_queue_end(nvmeq);
2145 }
2146
2147 static int nvme_delete_cq(struct nvme_queue *nvmeq)
2148 {
2149         return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
2150                                                 nvme_del_cq_work_handler);
2151 }
2152
2153 static void nvme_del_sq_work_handler(struct kthread_work *work)
2154 {
2155         struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2156                                                         cmdinfo.work);
2157         int status = nvmeq->cmdinfo.status;
2158
2159         if (!status)
2160                 status = nvme_delete_cq(nvmeq);
2161         if (status)
2162                 nvme_del_queue_end(nvmeq);
2163 }
2164
2165 static int nvme_delete_sq(struct nvme_queue *nvmeq)
2166 {
2167         return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
2168                                                 nvme_del_sq_work_handler);
2169 }
2170
2171 static void nvme_del_queue_start(struct kthread_work *work)
2172 {
2173         struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
2174                                                         cmdinfo.work);
2175         if (nvme_delete_sq(nvmeq))
2176                 nvme_del_queue_end(nvmeq);
2177 }
2178
2179 static void nvme_disable_io_queues(struct nvme_dev *dev)
2180 {
2181         int i;
2182         DEFINE_KTHREAD_WORKER_ONSTACK(worker);
2183         struct nvme_delq_ctx dq;
2184         struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
2185                                         &worker, "nvme%d", dev->ctrl.instance);
2186
2187         if (IS_ERR(kworker_task)) {
2188                 dev_err(dev->dev,
2189                         "Failed to create queue del task\n");
2190                 for (i = dev->queue_count - 1; i > 0; i--)
2191                         nvme_disable_queue(dev, i);
2192                 return;
2193         }
2194
2195         dq.waiter = NULL;
2196         atomic_set(&dq.refcount, 0);
2197         dq.worker = &worker;
2198         for (i = dev->queue_count - 1; i > 0; i--) {
2199                 struct nvme_queue *nvmeq = dev->queues[i];
2200
2201                 if (nvme_suspend_queue(nvmeq))
2202                         continue;
2203                 nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
2204                 nvmeq->cmdinfo.worker = dq.worker;
2205                 init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
2206                 queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
2207         }
2208         nvme_wait_dq(&dq, dev);
2209         kthread_stop(kworker_task);
2210 }
2211
2212 /*
2213 * Remove the node from the device list and check
2214 * for whether or not we need to stop the nvme_thread.
2215 */
2216 static void nvme_dev_list_remove(struct nvme_dev *dev)
2217 {
2218         struct task_struct *tmp = NULL;
2219
2220         spin_lock(&dev_list_lock);
2221         list_del_init(&dev->node);
2222         if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
2223                 tmp = nvme_thread;
2224                 nvme_thread = NULL;
2225         }
2226         spin_unlock(&dev_list_lock);
2227
2228         if (tmp)
2229                 kthread_stop(tmp);
2230 }
2231
2232 static void nvme_freeze_queues(struct nvme_dev *dev)
2233 {
2234         struct nvme_ns *ns;
2235
2236         list_for_each_entry(ns, &dev->namespaces, list) {
2237                 blk_mq_freeze_queue_start(ns->queue);
2238
2239                 spin_lock_irq(ns->queue->queue_lock);
2240                 queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
2241                 spin_unlock_irq(ns->queue->queue_lock);
2242
2243                 blk_mq_cancel_requeue_work(ns->queue);
2244                 blk_mq_stop_hw_queues(ns->queue);
2245         }
2246 }
2247
2248 static void nvme_unfreeze_queues(struct nvme_dev *dev)
2249 {
2250         struct nvme_ns *ns;
2251
2252         list_for_each_entry(ns, &dev->namespaces, list) {
2253                 queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
2254                 blk_mq_unfreeze_queue(ns->queue);
2255                 blk_mq_start_stopped_hw_queues(ns->queue, true);
2256                 blk_mq_kick_requeue_list(ns->queue);
2257         }
2258 }
2259
2260 static void nvme_dev_shutdown(struct nvme_dev *dev)
2261 {
2262         int i;
2263         u32 csts = -1;
2264
2265         nvme_dev_list_remove(dev);
2266
2267         if (dev->bar) {
2268                 nvme_freeze_queues(dev);
2269                 csts = readl(dev->bar + NVME_REG_CSTS);
2270         }
2271         if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
2272                 for (i = dev->queue_count - 1; i >= 0; i--) {
2273                         struct nvme_queue *nvmeq = dev->queues[i];
2274                         nvme_suspend_queue(nvmeq);
2275                 }
2276         } else {
2277                 nvme_disable_io_queues(dev);
2278                 nvme_shutdown_ctrl(&dev->ctrl);
2279                 nvme_disable_queue(dev, 0);
2280         }
2281         nvme_dev_unmap(dev);
2282
2283         for (i = dev->queue_count - 1; i >= 0; i--)
2284                 nvme_clear_queue(dev->queues[i]);
2285 }
2286
2287 static void nvme_dev_remove(struct nvme_dev *dev)
2288 {
2289         struct nvme_ns *ns, *next;
2290
2291         list_for_each_entry_safe(ns, next, &dev->namespaces, list)
2292                 nvme_ns_remove(ns);
2293 }
2294
2295 static int nvme_setup_prp_pools(struct nvme_dev *dev)
2296 {
2297         dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
2298                                                 PAGE_SIZE, PAGE_SIZE, 0);
2299         if (!dev->prp_page_pool)
2300                 return -ENOMEM;
2301
2302         /* Optimisation for I/Os between 4k and 128k */
2303         dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2304                                                 256, 256, 0);
2305         if (!dev->prp_small_pool) {
2306                 dma_pool_destroy(dev->prp_page_pool);
2307                 return -ENOMEM;
2308         }
2309         return 0;
2310 }
2311
2312 static void nvme_release_prp_pools(struct nvme_dev *dev)
2313 {
2314         dma_pool_destroy(dev->prp_page_pool);
2315         dma_pool_destroy(dev->prp_small_pool);
2316 }
2317
2318 static DEFINE_IDA(nvme_instance_ida);
2319
2320 static int nvme_set_instance(struct nvme_dev *dev)
2321 {
2322         int instance, error;
2323
2324         do {
2325                 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
2326                         return -ENODEV;
2327
2328                 spin_lock(&dev_list_lock);
2329                 error = ida_get_new(&nvme_instance_ida, &instance);
2330                 spin_unlock(&dev_list_lock);
2331         } while (error == -EAGAIN);
2332
2333         if (error)
2334                 return -ENODEV;
2335
2336         dev->ctrl.instance = instance;
2337         return 0;
2338 }
2339
2340 static void nvme_release_instance(struct nvme_dev *dev)
2341 {
2342         spin_lock(&dev_list_lock);
2343         ida_remove(&nvme_instance_ida, dev->ctrl.instance);
2344         spin_unlock(&dev_list_lock);
2345 }
2346
2347 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2348 {
2349         struct nvme_dev *dev = to_nvme_dev(ctrl);
2350
2351         put_device(dev->dev);
2352         put_device(dev->device);
2353         nvme_release_instance(dev);
2354         if (dev->tagset.tags)
2355                 blk_mq_free_tag_set(&dev->tagset);
2356         if (dev->ctrl.admin_q)
2357                 blk_put_queue(dev->ctrl.admin_q);
2358         kfree(dev->queues);
2359         kfree(dev->entry);
2360         kfree(dev);
2361 }
2362
2363 static int nvme_dev_open(struct inode *inode, struct file *f)
2364 {
2365         struct nvme_dev *dev;
2366         int instance = iminor(inode);
2367         int ret = -ENODEV;
2368
2369         spin_lock(&dev_list_lock);
2370         list_for_each_entry(dev, &dev_list, node) {
2371                 if (dev->ctrl.instance == instance) {
2372                         if (!dev->ctrl.admin_q) {
2373                                 ret = -EWOULDBLOCK;
2374                                 break;
2375                         }
2376                         if (!kref_get_unless_zero(&dev->ctrl.kref))
2377                                 break;
2378                         f->private_data = dev;
2379                         ret = 0;
2380                         break;
2381                 }
2382         }
2383         spin_unlock(&dev_list_lock);
2384
2385         return ret;
2386 }
2387
2388 static int nvme_dev_release(struct inode *inode, struct file *f)
2389 {
2390         struct nvme_dev *dev = f->private_data;
2391         nvme_put_ctrl(&dev->ctrl);
2392         return 0;
2393 }
2394
2395 static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
2396 {
2397         struct nvme_dev *dev = f->private_data;
2398         struct nvme_ns *ns;
2399
2400         switch (cmd) {
2401         case NVME_IOCTL_ADMIN_CMD:
2402                 return nvme_user_cmd(&dev->ctrl, NULL, (void __user *)arg);
2403         case NVME_IOCTL_IO_CMD:
2404                 if (list_empty(&dev->namespaces))
2405                         return -ENOTTY;
2406                 ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
2407                 return nvme_user_cmd(&dev->ctrl, ns, (void __user *)arg);
2408         case NVME_IOCTL_RESET:
2409                 dev_warn(dev->dev, "resetting controller\n");
2410                 return nvme_reset(dev);
2411         case NVME_IOCTL_SUBSYS_RESET:
2412                 return nvme_subsys_reset(dev);
2413         default:
2414                 return -ENOTTY;
2415         }
2416 }
2417
2418 static const struct file_operations nvme_dev_fops = {
2419         .owner          = THIS_MODULE,
2420         .open           = nvme_dev_open,
2421         .release        = nvme_dev_release,
2422         .unlocked_ioctl = nvme_dev_ioctl,
2423         .compat_ioctl   = nvme_dev_ioctl,
2424 };
2425
2426 static void nvme_probe_work(struct work_struct *work)
2427 {
2428         struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
2429         bool start_thread = false;
2430         int result;
2431
2432         result = nvme_dev_map(dev);
2433         if (result)
2434                 goto out;
2435
2436         result = nvme_configure_admin_queue(dev);
2437         if (result)
2438                 goto unmap;
2439
2440         spin_lock(&dev_list_lock);
2441         if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
2442                 start_thread = true;
2443                 nvme_thread = NULL;
2444         }
2445         list_add(&dev->node, &dev_list);
2446         spin_unlock(&dev_list_lock);
2447
2448         if (start_thread) {
2449                 nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
2450                 wake_up_all(&nvme_kthread_wait);
2451         } else
2452                 wait_event_killable(nvme_kthread_wait, nvme_thread);
2453
2454         if (IS_ERR_OR_NULL(nvme_thread)) {
2455                 result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
2456                 goto disable;
2457         }
2458
2459         nvme_init_queue(dev->queues[0], 0);
2460         result = nvme_alloc_admin_tags(dev);
2461         if (result)
2462                 goto disable;
2463
2464         result = nvme_setup_io_queues(dev);
2465         if (result)
2466                 goto free_tags;
2467
2468         dev->ctrl.event_limit = 1;
2469
2470         /*
2471          * Keep the controller around but remove all namespaces if we don't have
2472          * any working I/O queue.
2473          */
2474         if (dev->online_queues < 2) {
2475                 dev_warn(dev->dev, "IO queues not created\n");
2476                 nvme_dev_remove(dev);
2477         } else {
2478                 nvme_unfreeze_queues(dev);
2479                 nvme_dev_add(dev);
2480         }
2481
2482         return;
2483
2484  free_tags:
2485         nvme_dev_remove_admin(dev);
2486         blk_put_queue(dev->ctrl.admin_q);
2487         dev->ctrl.admin_q = NULL;
2488         dev->queues[0]->tags = NULL;
2489  disable:
2490         nvme_disable_queue(dev, 0);
2491         nvme_dev_list_remove(dev);
2492  unmap:
2493         nvme_dev_unmap(dev);
2494  out:
2495         if (!work_busy(&dev->reset_work))
2496                 nvme_dead_ctrl(dev);
2497 }
2498
2499 static int nvme_remove_dead_ctrl(void *arg)
2500 {
2501         struct nvme_dev *dev = (struct nvme_dev *)arg;
2502         struct pci_dev *pdev = to_pci_dev(dev->dev);
2503
2504         if (pci_get_drvdata(pdev))
2505                 pci_stop_and_remove_bus_device_locked(pdev);
2506         nvme_put_ctrl(&dev->ctrl);
2507         return 0;
2508 }
2509
2510 static void nvme_dead_ctrl(struct nvme_dev *dev)
2511 {
2512         dev_warn(dev->dev, "Device failed to resume\n");
2513         kref_get(&dev->ctrl.kref);
2514         if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
2515                                                 dev->ctrl.instance))) {
2516                 dev_err(dev->dev,
2517                         "Failed to start controller remove task\n");
2518                 nvme_put_ctrl(&dev->ctrl);
2519         }
2520 }
2521
2522 static void nvme_reset_work(struct work_struct *ws)
2523 {
2524         struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
2525         bool in_probe = work_busy(&dev->probe_work);
2526
2527         nvme_dev_shutdown(dev);
2528
2529         /* Synchronize with device probe so that work will see failure status
2530          * and exit gracefully without trying to schedule another reset */
2531         flush_work(&dev->probe_work);
2532
2533         /* Fail this device if reset occured during probe to avoid
2534          * infinite initialization loops. */
2535         if (in_probe) {
2536                 nvme_dead_ctrl(dev);
2537                 return;
2538         }
2539         /* Schedule device resume asynchronously so the reset work is available
2540          * to cleanup errors that may occur during reinitialization */
2541         schedule_work(&dev->probe_work);
2542 }
2543
2544 static int __nvme_reset(struct nvme_dev *dev)
2545 {
2546         if (work_pending(&dev->reset_work))
2547                 return -EBUSY;
2548         list_del_init(&dev->node);
2549         queue_work(nvme_workq, &dev->reset_work);
2550         return 0;
2551 }
2552
2553 static int nvme_reset(struct nvme_dev *dev)
2554 {
2555         int ret;
2556
2557         if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q))
2558                 return -ENODEV;
2559
2560         spin_lock(&dev_list_lock);
2561         ret = __nvme_reset(dev);
2562         spin_unlock(&dev_list_lock);
2563
2564         if (!ret) {
2565                 flush_work(&dev->reset_work);
2566                 flush_work(&dev->probe_work);
2567                 return 0;
2568         }
2569
2570         return ret;
2571 }
2572
2573 static ssize_t nvme_sysfs_reset(struct device *dev,
2574                                 struct device_attribute *attr, const char *buf,
2575                                 size_t count)
2576 {
2577         struct nvme_dev *ndev = dev_get_drvdata(dev);
2578         int ret;
2579
2580         ret = nvme_reset(ndev);
2581         if (ret < 0)
2582                 return ret;
2583
2584         return count;
2585 }
2586 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
2587
2588 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
2589 {
2590         *val = readl(to_nvme_dev(ctrl)->bar + off);
2591         return 0;
2592 }
2593
2594 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2595 {
2596         writel(val, to_nvme_dev(ctrl)->bar + off);
2597         return 0;
2598 }
2599
2600 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
2601         .reg_read32             = nvme_pci_reg_read32,
2602         .reg_write32            = nvme_pci_reg_write32,
2603         .free_ctrl              = nvme_pci_free_ctrl,
2604 };
2605
2606 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2607 {
2608         int node, result = -ENOMEM;
2609         struct nvme_dev *dev;
2610
2611         node = dev_to_node(&pdev->dev);
2612         if (node == NUMA_NO_NODE)
2613                 set_dev_node(&pdev->dev, 0);
2614
2615         dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
2616         if (!dev)
2617                 return -ENOMEM;
2618         dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
2619                                                         GFP_KERNEL, node);
2620         if (!dev->entry)
2621                 goto free;
2622         dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
2623                                                         GFP_KERNEL, node);
2624         if (!dev->queues)
2625                 goto free;
2626
2627         INIT_LIST_HEAD(&dev->namespaces);
2628         INIT_WORK(&dev->reset_work, nvme_reset_work);
2629         dev->dev = get_device(&pdev->dev);
2630         pci_set_drvdata(pdev, dev);
2631
2632         dev->ctrl.ops = &nvme_pci_ctrl_ops;
2633         dev->ctrl.dev = dev->dev;
2634         dev->ctrl.quirks = id->driver_data;
2635
2636         result = nvme_set_instance(dev);
2637         if (result)
2638                 goto put_pci;
2639
2640         result = nvme_setup_prp_pools(dev);
2641         if (result)
2642                 goto release;
2643
2644         kref_init(&dev->ctrl.kref);
2645         dev->device = device_create(nvme_class, &pdev->dev,
2646                                 MKDEV(nvme_char_major, dev->ctrl.instance),
2647                                 dev, "nvme%d", dev->ctrl.instance);
2648         if (IS_ERR(dev->device)) {
2649                 result = PTR_ERR(dev->device);
2650                 goto release_pools;
2651         }
2652         get_device(dev->device);
2653         dev_set_drvdata(dev->device, dev);
2654
2655         result = device_create_file(dev->device, &dev_attr_reset_controller);
2656         if (result)
2657                 goto put_dev;
2658
2659         INIT_LIST_HEAD(&dev->node);
2660         INIT_WORK(&dev->scan_work, nvme_dev_scan);
2661         INIT_WORK(&dev->probe_work, nvme_probe_work);
2662         schedule_work(&dev->probe_work);
2663         return 0;
2664
2665  put_dev:
2666         device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
2667         put_device(dev->device);
2668  release_pools:
2669         nvme_release_prp_pools(dev);
2670  release:
2671         nvme_release_instance(dev);
2672  put_pci:
2673         put_device(dev->dev);
2674  free:
2675         kfree(dev->queues);
2676         kfree(dev->entry);
2677         kfree(dev);
2678         return result;
2679 }
2680
2681 static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
2682 {
2683         struct nvme_dev *dev = pci_get_drvdata(pdev);
2684
2685         if (prepare)
2686                 nvme_dev_shutdown(dev);
2687         else
2688                 schedule_work(&dev->probe_work);
2689 }
2690
2691 static void nvme_shutdown(struct pci_dev *pdev)
2692 {
2693         struct nvme_dev *dev = pci_get_drvdata(pdev);
2694         nvme_dev_shutdown(dev);
2695 }
2696
2697 static void nvme_remove(struct pci_dev *pdev)
2698 {
2699         struct nvme_dev *dev = pci_get_drvdata(pdev);
2700
2701         spin_lock(&dev_list_lock);
2702         list_del_init(&dev->node);
2703         spin_unlock(&dev_list_lock);
2704
2705         pci_set_drvdata(pdev, NULL);
2706         flush_work(&dev->probe_work);
2707         flush_work(&dev->reset_work);
2708         flush_work(&dev->scan_work);
2709         device_remove_file(dev->device, &dev_attr_reset_controller);
2710         nvme_dev_remove(dev);
2711         nvme_dev_shutdown(dev);
2712         nvme_dev_remove_admin(dev);
2713         device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
2714         nvme_free_queues(dev, 0);
2715         nvme_release_cmb(dev);
2716         nvme_release_prp_pools(dev);
2717         nvme_put_ctrl(&dev->ctrl);
2718 }
2719
2720 /* These functions are yet to be implemented */
2721 #define nvme_error_detected NULL
2722 #define nvme_dump_registers NULL
2723 #define nvme_link_reset NULL
2724 #define nvme_slot_reset NULL
2725 #define nvme_error_resume NULL
2726
2727 #ifdef CONFIG_PM_SLEEP
2728 static int nvme_suspend(struct device *dev)
2729 {
2730         struct pci_dev *pdev = to_pci_dev(dev);
2731         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2732
2733         nvme_dev_shutdown(ndev);
2734         return 0;
2735 }
2736
2737 static int nvme_resume(struct device *dev)
2738 {
2739         struct pci_dev *pdev = to_pci_dev(dev);
2740         struct nvme_dev *ndev = pci_get_drvdata(pdev);
2741
2742         schedule_work(&ndev->probe_work);
2743         return 0;
2744 }
2745 #endif
2746
2747 static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
2748
2749 static const struct pci_error_handlers nvme_err_handler = {
2750         .error_detected = nvme_error_detected,
2751         .mmio_enabled   = nvme_dump_registers,
2752         .link_reset     = nvme_link_reset,
2753         .slot_reset     = nvme_slot_reset,
2754         .resume         = nvme_error_resume,
2755         .reset_notify   = nvme_reset_notify,
2756 };
2757
2758 /* Move to pci_ids.h later */
2759 #define PCI_CLASS_STORAGE_EXPRESS       0x010802
2760
2761 static const struct pci_device_id nvme_id_table[] = {
2762         { PCI_VDEVICE(INTEL, 0x0953),
2763                 .driver_data = NVME_QUIRK_STRIPE_SIZE, },
2764         { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2765         { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
2766         { 0, }
2767 };
2768 MODULE_DEVICE_TABLE(pci, nvme_id_table);
2769
2770 static struct pci_driver nvme_driver = {
2771         .name           = "nvme",
2772         .id_table       = nvme_id_table,
2773         .probe          = nvme_probe,
2774         .remove         = nvme_remove,
2775         .shutdown       = nvme_shutdown,
2776         .driver         = {
2777                 .pm     = &nvme_dev_pm_ops,
2778         },
2779         .err_handler    = &nvme_err_handler,
2780 };
2781
2782 static int __init nvme_init(void)
2783 {
2784         int result;
2785
2786         init_waitqueue_head(&nvme_kthread_wait);
2787
2788         nvme_workq = create_singlethread_workqueue("nvme");
2789         if (!nvme_workq)
2790                 return -ENOMEM;
2791
2792         result = register_blkdev(nvme_major, "nvme");
2793         if (result < 0)
2794                 goto kill_workq;
2795         else if (result > 0)
2796                 nvme_major = result;
2797
2798         result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
2799                                                         &nvme_dev_fops);
2800         if (result < 0)
2801                 goto unregister_blkdev;
2802         else if (result > 0)
2803                 nvme_char_major = result;
2804
2805         nvme_class = class_create(THIS_MODULE, "nvme");
2806         if (IS_ERR(nvme_class)) {
2807                 result = PTR_ERR(nvme_class);
2808                 goto unregister_chrdev;
2809         }
2810
2811         result = pci_register_driver(&nvme_driver);
2812         if (result)
2813                 goto destroy_class;
2814         return 0;
2815
2816  destroy_class:
2817         class_destroy(nvme_class);
2818  unregister_chrdev:
2819         __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
2820  unregister_blkdev:
2821         unregister_blkdev(nvme_major, "nvme");
2822  kill_workq:
2823         destroy_workqueue(nvme_workq);
2824         return result;
2825 }
2826
2827 static void __exit nvme_exit(void)
2828 {
2829         pci_unregister_driver(&nvme_driver);
2830         unregister_blkdev(nvme_major, "nvme");
2831         destroy_workqueue(nvme_workq);
2832         class_destroy(nvme_class);
2833         __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
2834         BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
2835         _nvme_check_size();
2836 }
2837
2838 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
2839 MODULE_LICENSE("GPL");
2840 MODULE_VERSION("1.0");
2841 module_init(nvme_init);
2842 module_exit(nvme_exit);