projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
NVMe: Detect command IDs completing that are out of range
[linux-2.6-block.git] / drivers / block / nvme.c
CommitLineData
b60503ba
MW		 1/*
2 * NVM Express device driver
3 * Copyright (c) 2011, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 */
18
19#include <linux/nvme.h>
20#include <linux/bio.h>
21#include <linux/blkdev.h>
22#include <linux/errno.h>
23#include <linux/fs.h>
24#include <linux/genhd.h>
25#include <linux/init.h>
26#include <linux/interrupt.h>
27#include <linux/io.h>
28#include <linux/kdev_t.h>
29#include <linux/kernel.h>
30#include <linux/mm.h>
31#include <linux/module.h>
32#include <linux/moduleparam.h>
33#include <linux/pci.h>
be7b6275 34#include <linux/poison.h>
b60503ba
MW		 35#include <linux/sched.h>
36#include <linux/slab.h>
37#include <linux/types.h>
38#include <linux/version.h>
39
40#define NVME_Q_DEPTH 1024
41#define SQ_SIZE(depth) (depth * sizeof(struct nvme_command))
42#define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion))
43#define NVME_MINORS 64
44
45static int nvme_major;
46module_param(nvme_major, int, 0);
47
58ffacb5
MW		 48static int use_threaded_interrupts;
49module_param(use_threaded_interrupts, int, 0);
50
b60503ba
MW		 51/*
52 * Represents an NVM Express device. Each nvme_dev is a PCI function.
53 */
54struct nvme_dev {
b60503ba
MW		 55 struct nvme_queue **queues;
56 u32 __iomem *dbs;
57 struct pci_dev *pci_dev;
58 int instance;
59 int queue_count;
60 u32 ctrl_config;
61 struct msix_entry *entry;
62 struct nvme_bar __iomem *bar;
63 struct list_head namespaces;
51814232
MW		 64 char serial[20];
65 char model[40];
66 char firmware_rev[8];
b60503ba
MW		 67};
68
69/*
70 * An NVM Express namespace is equivalent to a SCSI LUN
71 */
72struct nvme_ns {
73 struct list_head list;
74
75 struct nvme_dev *dev;
76 struct request_queue *queue;
77 struct gendisk *disk;
78
79 int ns_id;
80 int lba_shift;
81};
82
83/*
84 * An NVM Express queue. Each device has at least two (one for admin
85 * commands and one for I/O commands).
86 */
87struct nvme_queue {
88 struct device *q_dmadev;
89 spinlock_t q_lock;
90 struct nvme_command *sq_cmds;
91 volatile struct nvme_completion *cqes;
92 dma_addr_t sq_dma_addr;
93 dma_addr_t cq_dma_addr;
94 wait_queue_head_t sq_full;
95 struct bio_list sq_cong;
96 u32 __iomem *q_db;
97 u16 q_depth;
98 u16 cq_vector;
99 u16 sq_head;
100 u16 sq_tail;
101 u16 cq_head;
82123460 102 u16 cq_phase;
b60503ba
MW		 103 unsigned long cmdid_data[];
104};
105
106/*
107 * Check we didin't inadvertently grow the command struct
108 */
109static inline void _nvme_check_size(void)
110{
111 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
112 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
113 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
114 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
115 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
116 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
117 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
118 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
119 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
120}
121
122/**
123 * alloc_cmdid - Allocate a Command ID
124 * @param nvmeq The queue that will be used for this command
125 * @param ctx A pointer that will be passed to the handler
126 * @param handler The ID of the handler to call
127 *
128 * Allocate a Command ID for a queue. The data passed in will
129 * be passed to the completion handler. This is implemented by using
130 * the bottom two bits of the ctx pointer to store the handler ID.
131 * Passing in a pointer that's not 4-byte aligned will cause a BUG.
132 * We can change this if it becomes a problem.
133 */
134static int alloc_cmdid(struct nvme_queue *nvmeq, void *ctx, int handler)
135{
136 int depth = nvmeq->q_depth;
137 unsigned long data = (unsigned long)ctx | handler;
138 int cmdid;
139
140 BUG_ON((unsigned long)ctx & 3);
141
142 do {
143 cmdid = find_first_zero_bit(nvmeq->cmdid_data, depth);
144 if (cmdid >= depth)
145 return -EBUSY;
146 } while (test_and_set_bit(cmdid, nvmeq->cmdid_data));
147
148 nvmeq->cmdid_data[cmdid + BITS_TO_LONGS(depth)] = data;
149 return cmdid;
150}
151
152static int alloc_cmdid_killable(struct nvme_queue *nvmeq, void *ctx,
153 int handler)
154{
155 int cmdid;
156 wait_event_killable(nvmeq->sq_full,
157 (cmdid = alloc_cmdid(nvmeq, ctx, handler)) >= 0);
158 return (cmdid < 0) ? -EINTR : cmdid;
159}
160
161/* If you need more than four handlers, you'll need to change how
be7b6275
MW		 162 * alloc_cmdid and nvme_process_cq work. Consider using a special
163 * CMD_CTX value instead, if that works for your situation.
b60503ba
MW		 164 */
165enum {
166 sync_completion_id = 0,
167 bio_completion_id,
168};
169
be7b6275
MW		 170#define CMD_CTX_BASE (POISON_POINTER_DELTA + sync_completion_id)
171#define CMD_CTX_CANCELLED (0x2008 + CMD_CTX_BASE)
b36235df 172#define CMD_CTX_COMPLETED (0x2010 + CMD_CTX_BASE)
48e3d398 173#define CMD_CTX_INVALID (0x2014 + CMD_CTX_BASE)
be7b6275 174
b60503ba
MW		 175static unsigned long free_cmdid(struct nvme_queue *nvmeq, int cmdid)
176{
177 unsigned long data;
b36235df 178 unsigned offset = cmdid + BITS_TO_LONGS(nvmeq->q_depth);
b60503ba 179
48e3d398
MW		 180 if (cmdid > nvmeq->q_depth)
181 return CMD_CTX_INVALID;
b36235df
MW		 182 data = nvmeq->cmdid_data[offset];
183 nvmeq->cmdid_data[offset] = CMD_CTX_COMPLETED;
b60503ba
MW		 184 clear_bit(cmdid, nvmeq->cmdid_data);
185 wake_up(&nvmeq->sq_full);
186 return data;
187}
188
be7b6275 189static void cancel_cmdid_data(struct nvme_queue *nvmeq, int cmdid)
3c0cf138 190{
b36235df
MW		 191 unsigned offset = cmdid + BITS_TO_LONGS(nvmeq->q_depth);
192 nvmeq->cmdid_data[offset] = CMD_CTX_CANCELLED;
3c0cf138
MW		 193}
194
b60503ba
MW		 195static struct nvme_queue *get_nvmeq(struct nvme_ns *ns)
196{
1b23484b
MW		 197 int qid, cpu = get_cpu();
198 if (cpu < ns->dev->queue_count)
199 qid = cpu + 1;
200 else
201 qid = (cpu % rounddown_pow_of_two(ns->dev->queue_count)) + 1;
202 return ns->dev->queues[qid];
b60503ba
MW		 203}
204
205static void put_nvmeq(struct nvme_queue *nvmeq)
206{
1b23484b 207 put_cpu();
b60503ba
MW		 208}
209
210/**
211 * nvme_submit_cmd: Copy a command into a queue and ring the doorbell
212 * @nvmeq: The queue to use
213 * @cmd: The command to send
214 *
215 * Safe to use from interrupt context
216 */
217static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
218{
219 unsigned long flags;
220 u16 tail;
221 /* XXX: Need to check tail isn't going to overrun head */
222 spin_lock_irqsave(&nvmeq->q_lock, flags);
223 tail = nvmeq->sq_tail;
224 memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
225 writel(tail, nvmeq->q_db);
226 if (++tail == nvmeq->q_depth)
227 tail = 0;
228 nvmeq->sq_tail = tail;
229 spin_unlock_irqrestore(&nvmeq->q_lock, flags);
230
231 return 0;
232}
233
234struct nvme_req_info {
235 struct bio *bio;
236 int nents;
237 struct scatterlist sg[0];
238};
239
240/* XXX: use a mempool */
241static struct nvme_req_info *alloc_info(unsigned nseg, gfp_t gfp)
242{
243 return kmalloc(sizeof(struct nvme_req_info) +
244 sizeof(struct scatterlist) * nseg, gfp);
245}
246
247static void free_info(struct nvme_req_info *info)
248{
249 kfree(info);
250}
251
252static void bio_completion(struct nvme_queue *nvmeq, void *ctx,
253 struct nvme_completion *cqe)
254{
255 struct nvme_req_info *info = ctx;
256 struct bio *bio = info->bio;
257 u16 status = le16_to_cpup(&cqe->status) >> 1;
258
259 dma_unmap_sg(nvmeq->q_dmadev, info->sg, info->nents,
260 bio_data_dir(bio) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
261 free_info(info);
262 bio_endio(bio, status ? -EIO : 0);
263}
264
ff22b54f
MW		 265/* length is in bytes */
266static void nvme_setup_prps(struct nvme_common_command *cmd,
267 struct scatterlist *sg, int length)
268{
269 int dma_len = sg_dma_len(sg);
270 u64 dma_addr = sg_dma_address(sg);
271 int offset = offset_in_page(dma_addr);
272
273 cmd->prp1 = cpu_to_le64(dma_addr);
274 length -= (PAGE_SIZE - offset);
275 if (length <= 0)
276 return;
277
278 dma_len -= (PAGE_SIZE - offset);
279 if (dma_len) {
280 dma_addr += (PAGE_SIZE - offset);
281 } else {
282 sg = sg_next(sg);
283 dma_addr = sg_dma_address(sg);
284 dma_len = sg_dma_len(sg);
285 }
286
287 if (length <= PAGE_SIZE) {
288 cmd->prp2 = cpu_to_le64(dma_addr);
289 return;
290 }
291
292 /* XXX: support PRP lists */
293}
294
b60503ba
MW		 295static int nvme_map_bio(struct device *dev, struct nvme_req_info *info,
296 struct bio *bio, enum dma_data_direction dma_dir, int psegs)
297{
298 struct bio_vec *bvec;
299 struct scatterlist *sg = info->sg;
300 int i, nsegs;
301
302 sg_init_table(sg, psegs);
303 bio_for_each_segment(bvec, bio, i) {
304 sg_set_page(sg, bvec->bv_page, bvec->bv_len, bvec->bv_offset);
305 /* XXX: handle non-mergable here */
306 nsegs++;
307 }
308 info->nents = nsegs;
309
310 return dma_map_sg(dev, info->sg, info->nents, dma_dir);
311}
312
313static int nvme_submit_bio_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns,
314 struct bio *bio)
315{
ff22b54f 316 struct nvme_command *cmnd;
b60503ba
MW		 317 struct nvme_req_info *info;
318 enum dma_data_direction dma_dir;
319 int cmdid;
320 u16 control;
321 u32 dsmgmt;
322 unsigned long flags;
323 int psegs = bio_phys_segments(ns->queue, bio);
324
325 info = alloc_info(psegs, GFP_NOIO);
326 if (!info)
327 goto congestion;
328 info->bio = bio;
329
330 cmdid = alloc_cmdid(nvmeq, info, bio_completion_id);
331 if (unlikely(cmdid < 0))
332 goto free_info;
333
334 control = 0;
335 if (bio->bi_rw & REQ_FUA)
336 control |= NVME_RW_FUA;
337 if (bio->bi_rw & (REQ_FAILFAST_DEV | REQ_RAHEAD))
338 control |= NVME_RW_LR;
339
340 dsmgmt = 0;
341 if (bio->bi_rw & REQ_RAHEAD)
342 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
343
344 spin_lock_irqsave(&nvmeq->q_lock, flags);
ff22b54f 345 cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
b60503ba 346
b8deb62c 347 memset(cmnd, 0, sizeof(*cmnd));
b60503ba 348 if (bio_data_dir(bio)) {
ff22b54f 349 cmnd->rw.opcode = nvme_cmd_write;
b60503ba
MW		 350 dma_dir = DMA_TO_DEVICE;
351 } else {
ff22b54f 352 cmnd->rw.opcode = nvme_cmd_read;
b60503ba
MW		 353 dma_dir = DMA_FROM_DEVICE;
354 }
355
356 nvme_map_bio(nvmeq->q_dmadev, info, bio, dma_dir, psegs);
357
ff22b54f
MW		 358 cmnd->rw.flags = 1;
359 cmnd->rw.command_id = cmdid;
360 cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
361 nvme_setup_prps(&cmnd->common, info->sg, bio->bi_size);
362 cmnd->rw.slba = cpu_to_le64(bio->bi_sector >> (ns->lba_shift - 9));
363 cmnd->rw.length = cpu_to_le16((bio->bi_size >> ns->lba_shift) - 1);
364 cmnd->rw.control = cpu_to_le16(control);
365 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
b60503ba
MW		 366
367 writel(nvmeq->sq_tail, nvmeq->q_db);
368 if (++nvmeq->sq_tail == nvmeq->q_depth)
369 nvmeq->sq_tail = 0;
370
371 spin_unlock_irqrestore(&nvmeq->q_lock, flags);
372
373 return 0;
374
375 free_info:
376 free_info(info);
377 congestion:
378 return -EBUSY;
379}
380
381/*
382 * NB: return value of non-zero would mean that we were a stacking driver.
383 * make_request must always succeed.
384 */
385static int nvme_make_request(struct request_queue *q, struct bio *bio)
386{
387 struct nvme_ns *ns = q->queuedata;
388 struct nvme_queue *nvmeq = get_nvmeq(ns);
389
390 if (nvme_submit_bio_queue(nvmeq, ns, bio)) {
391 blk_set_queue_congested(q, rw_is_sync(bio->bi_rw));
392 bio_list_add(&nvmeq->sq_cong, bio);
393 }
394 put_nvmeq(nvmeq);
395
396 return 0;
397}
398
399struct sync_cmd_info {
400 struct task_struct *task;
401 u32 result;
402 int status;
403};
404
405static void sync_completion(struct nvme_queue *nvmeq, void *ctx,
406 struct nvme_completion *cqe)
407{
408 struct sync_cmd_info *cmdinfo = ctx;
be7b6275
MW		 409 if ((unsigned long)cmdinfo == CMD_CTX_CANCELLED)
410 return;
b36235df
MW		 411 if (unlikely((unsigned long)cmdinfo == CMD_CTX_COMPLETED)) {
412 dev_warn(nvmeq->q_dmadev,
413 "completed id %d twice on queue %d\n",
414 cqe->command_id, le16_to_cpup(&cqe->sq_id));
415 return;
416 }
48e3d398
MW		 417 if (unlikely((unsigned long)cmdinfo == CMD_CTX_INVALID)) {
418 dev_warn(nvmeq->q_dmadev,
419 "invalid id %d completed on queue %d\n",
420 cqe->command_id, le16_to_cpup(&cqe->sq_id));
421 return;
422 }
b60503ba
MW		 423 cmdinfo->result = le32_to_cpup(&cqe->result);
424 cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
425 wake_up_process(cmdinfo->task);
426}
427
428typedef void (*completion_fn)(struct nvme_queue *, void *,
429 struct nvme_completion *);
430
431static irqreturn_t nvme_process_cq(struct nvme_queue *nvmeq)
432{
82123460 433 u16 head, phase;
b60503ba
MW		 434
435 static const completion_fn completions[4] = {
436 [sync_completion_id] = sync_completion,
437 [bio_completion_id] = bio_completion,
438 };
439
440 head = nvmeq->cq_head;
82123460 441 phase = nvmeq->cq_phase;
b60503ba
MW		 442
443 for (;;) {
444 unsigned long data;
445 void *ptr;
446 unsigned char handler;
447 struct nvme_completion cqe = nvmeq->cqes[head];
82123460 448 if ((le16_to_cpu(cqe.status) & 1) != phase)
b60503ba
MW		 449 break;
450 nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
451 if (++head == nvmeq->q_depth) {
452 head = 0;
82123460 453 phase = !phase;
b60503ba
MW		 454 }
455
456 data = free_cmdid(nvmeq, cqe.command_id);
457 handler = data & 3;
458 ptr = (void *)(data & ~3UL);
459 completions[handler](nvmeq, ptr, &cqe);
460 }
461
462 /* If the controller ignores the cq head doorbell and continuously
463 * writes to the queue, it is theoretically possible to wrap around
464 * the queue twice and mistakenly return IRQ_NONE. Linux only
465 * requires that 0.1% of your interrupts are handled, so this isn't
466 * a big problem.
467 */
82123460 468 if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
b60503ba
MW		 469 return IRQ_NONE;
470
471 writel(head, nvmeq->q_db + 1);
472 nvmeq->cq_head = head;
82123460 473 nvmeq->cq_phase = phase;
b60503ba
MW		 474
475 return IRQ_HANDLED;
476}
477
478static irqreturn_t nvme_irq(int irq, void *data)
479{
480 return nvme_process_cq(data);
481}
482
58ffacb5
MW		 483static irqreturn_t nvme_irq_thread(int irq, void *data)
484{
485 irqreturn_t result;
486 struct nvme_queue *nvmeq = data;
487 spin_lock(&nvmeq->q_lock);
488 result = nvme_process_cq(nvmeq);
489 spin_unlock(&nvmeq->q_lock);
490 return result;
491}
492
493static irqreturn_t nvme_irq_check(int irq, void *data)
494{
495 struct nvme_queue *nvmeq = data;
496 struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
497 if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
498 return IRQ_NONE;
499 return IRQ_WAKE_THREAD;
500}
501
3c0cf138
MW		 502static void nvme_abort_command(struct nvme_queue *nvmeq, int cmdid)
503{
504 spin_lock_irq(&nvmeq->q_lock);
be7b6275 505 cancel_cmdid_data(nvmeq, cmdid);
3c0cf138
MW		 506 spin_unlock_irq(&nvmeq->q_lock);
507}
508
b60503ba
MW		 509/*
510 * Returns 0 on success. If the result is negative, it's a Linux error code;
511 * if the result is positive, it's an NVM Express status code
512 */
3c0cf138
MW		 513static int nvme_submit_sync_cmd(struct nvme_queue *nvmeq,
514 struct nvme_command *cmd, u32 *result)
b60503ba
MW		 515{
516 int cmdid;
517 struct sync_cmd_info cmdinfo;
518
519 cmdinfo.task = current;
520 cmdinfo.status = -EINTR;
521
3c0cf138 522 cmdid = alloc_cmdid_killable(nvmeq, &cmdinfo, sync_completion_id);
b60503ba
MW		 523 if (cmdid < 0)
524 return cmdid;
525 cmd->common.command_id = cmdid;
526
3c0cf138
MW		 527 set_current_state(TASK_KILLABLE);
528 nvme_submit_cmd(nvmeq, cmd);
b60503ba
MW		 529 schedule();
530
3c0cf138
MW		 531 if (cmdinfo.status == -EINTR) {
532 nvme_abort_command(nvmeq, cmdid);
533 return -EINTR;
534 }
535
b60503ba
MW		 536 if (result)
537 *result = cmdinfo.result;
538
539 return cmdinfo.status;
540}
541
542static int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
543 u32 *result)
544{
545 return nvme_submit_sync_cmd(dev->queues[0], cmd, result);
546}
547
548static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
549{
550 int status;
551 struct nvme_command c;
552
553 memset(&c, 0, sizeof(c));
554 c.delete_queue.opcode = opcode;
555 c.delete_queue.qid = cpu_to_le16(id);
556
557 status = nvme_submit_admin_cmd(dev, &c, NULL);
558 if (status)
559 return -EIO;
560 return 0;
561}
562
563static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
564 struct nvme_queue *nvmeq)
565{
566 int status;
567 struct nvme_command c;
568 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;
569
570 memset(&c, 0, sizeof(c));
571 c.create_cq.opcode = nvme_admin_create_cq;
572 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
573 c.create_cq.cqid = cpu_to_le16(qid);
574 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
575 c.create_cq.cq_flags = cpu_to_le16(flags);
576 c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);
577
578 status = nvme_submit_admin_cmd(dev, &c, NULL);
579 if (status)
580 return -EIO;
581 return 0;
582}
583
584static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
585 struct nvme_queue *nvmeq)
586{
587 int status;
588 struct nvme_command c;
589 int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;
590
591 memset(&c, 0, sizeof(c));
592 c.create_sq.opcode = nvme_admin_create_sq;
593 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
594 c.create_sq.sqid = cpu_to_le16(qid);
595 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
596 c.create_sq.sq_flags = cpu_to_le16(flags);
597 c.create_sq.cqid = cpu_to_le16(qid);
598
599 status = nvme_submit_admin_cmd(dev, &c, NULL);
600 if (status)
601 return -EIO;
602 return 0;
603}
604
605static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
606{
607 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
608}
609
610static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
611{
612 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
613}
614
615static void nvme_free_queue(struct nvme_dev *dev, int qid)
616{
617 struct nvme_queue *nvmeq = dev->queues[qid];
618
619 free_irq(dev->entry[nvmeq->cq_vector].vector, nvmeq);
620
621 /* Don't tell the adapter to delete the admin queue */
622 if (qid) {
623 adapter_delete_sq(dev, qid);
624 adapter_delete_cq(dev, qid);
625 }
626
627 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
628 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
629 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
630 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
631 kfree(nvmeq);
632}
633
634static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
635 int depth, int vector)
636{
637 struct device *dmadev = &dev->pci_dev->dev;
638 unsigned extra = (depth + BITS_TO_LONGS(depth)) * sizeof(long);
639 struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
640 if (!nvmeq)
641 return NULL;
642
643 nvmeq->cqes = dma_alloc_coherent(dmadev, CQ_SIZE(depth),
644 &nvmeq->cq_dma_addr, GFP_KERNEL);
645 if (!nvmeq->cqes)
646 goto free_nvmeq;
647 memset((void *)nvmeq->cqes, 0, CQ_SIZE(depth));
648
649 nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
650 &nvmeq->sq_dma_addr, GFP_KERNEL);
651 if (!nvmeq->sq_cmds)
652 goto free_cqdma;
653
654 nvmeq->q_dmadev = dmadev;
655 spin_lock_init(&nvmeq->q_lock);
656 nvmeq->cq_head = 0;
82123460 657 nvmeq->cq_phase = 1;
b60503ba
MW		 658 init_waitqueue_head(&nvmeq->sq_full);
659 bio_list_init(&nvmeq->sq_cong);
660 nvmeq->q_db = &dev->dbs[qid * 2];
661 nvmeq->q_depth = depth;
662 nvmeq->cq_vector = vector;
663
664 return nvmeq;
665
666 free_cqdma:
667 dma_free_coherent(dmadev, CQ_SIZE(nvmeq->q_depth), (void *)nvmeq->cqes,
668 nvmeq->cq_dma_addr);
669 free_nvmeq:
670 kfree(nvmeq);
671 return NULL;
672}
673
3001082c
MW		 674static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
675 const char *name)
676{
58ffacb5
MW		 677 if (use_threaded_interrupts)
678 return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
679 nvme_irq_check, nvme_irq_thread,
680 IRQF_DISABLED | IRQF_SHARED,
681 name, nvmeq);
3001082c
MW		 682 return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
683 IRQF_DISABLED | IRQF_SHARED, name, nvmeq);
684}
685
b60503ba
MW		 686static __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev,
687 int qid, int cq_size, int vector)
688{
689 int result;
690 struct nvme_queue *nvmeq = nvme_alloc_queue(dev, qid, cq_size, vector);
691
3f85d50b
MW		 692 if (!nvmeq)
693 return NULL;
694
b60503ba
MW		 695 result = adapter_alloc_cq(dev, qid, nvmeq);
696 if (result < 0)
697 goto free_nvmeq;
698
699 result = adapter_alloc_sq(dev, qid, nvmeq);
700 if (result < 0)
701 goto release_cq;
702
3001082c 703 result = queue_request_irq(dev, nvmeq, "nvme");
b60503ba
MW		 704 if (result < 0)
705 goto release_sq;
706
707 return nvmeq;
708
709 release_sq:
710 adapter_delete_sq(dev, qid);
711 release_cq:
712 adapter_delete_cq(dev, qid);
713 free_nvmeq:
714 dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
715 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
716 dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
717 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
718 kfree(nvmeq);
719 return NULL;
720}
721
722static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev)
723{
724 int result;
725 u32 aqa;
726 struct nvme_queue *nvmeq;
727
728 dev->dbs = ((void __iomem *)dev->bar) + 4096;
729
730 nvmeq = nvme_alloc_queue(dev, 0, 64, 0);
3f85d50b
MW		 731 if (!nvmeq)
732 return -ENOMEM;
b60503ba
MW		 733
734 aqa = nvmeq->q_depth - 1;
735 aqa |= aqa << 16;
736
737 dev->ctrl_config = NVME_CC_ENABLE | NVME_CC_CSS_NVM;
738 dev->ctrl_config |= (PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
739 dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
740
5911f200 741 writel(0, &dev->bar->cc);
b60503ba
MW		 742 writel(aqa, &dev->bar->aqa);
743 writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
744 writeq(nvmeq->cq_dma_addr, &dev->bar->acq);
745 writel(dev->ctrl_config, &dev->bar->cc);
746
747 while (!(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {
748 msleep(100);
749 if (fatal_signal_pending(current))
750 return -EINTR;
751 }
752
3001082c 753 result = queue_request_irq(dev, nvmeq, "nvme admin");
b60503ba
MW		 754 dev->queues[0] = nvmeq;
755 return result;
756}
757
7fc3cdab
MW		 758static int nvme_map_user_pages(struct nvme_dev *dev, int write,
759 unsigned long addr, unsigned length,
760 struct scatterlist **sgp)
b60503ba 761{
36c14ed9 762 int i, err, count, nents, offset;
7fc3cdab
MW		 763 struct scatterlist *sg;
764 struct page **pages;
36c14ed9
MW		 765
766 if (addr & 3)
767 return -EINVAL;
7fc3cdab
MW		 768 if (!length)
769 return -EINVAL;
770
36c14ed9 771 offset = offset_in_page(addr);
7fc3cdab
MW		 772 count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
773 pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
36c14ed9
MW		 774
775 err = get_user_pages_fast(addr, count, 1, pages);
776 if (err < count) {
777 count = err;
778 err = -EFAULT;
779 goto put_pages;
780 }
7fc3cdab
MW		 781
782 sg = kcalloc(count, sizeof(*sg), GFP_KERNEL);
36c14ed9 783 sg_init_table(sg, count);
ff22b54f 784 sg_set_page(&sg[0], pages[0], PAGE_SIZE - offset, offset);
7fc3cdab
MW		 785 length -= (PAGE_SIZE - offset);
786 for (i = 1; i < count; i++) {
787 sg_set_page(&sg[i], pages[i], min_t(int, length, PAGE_SIZE), 0);
788 length -= PAGE_SIZE;
789 }
790
791 err = -ENOMEM;
792 nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
793 write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
36c14ed9
MW		 794 if (!nents)
795 goto put_pages;
b60503ba 796
7fc3cdab
MW		 797 kfree(pages);
798 *sgp = sg;
799 return nents;
b60503ba 800
7fc3cdab
MW		 801 put_pages:
802 for (i = 0; i < count; i++)
803 put_page(pages[i]);
804 kfree(pages);
805 return err;
806}
b60503ba 807
7fc3cdab
MW		 808static void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
809 unsigned long addr, int length,
810 struct scatterlist *sg, int nents)
811{
812 int i, count;
b60503ba 813
7fc3cdab 814 count = DIV_ROUND_UP(offset_in_page(addr) + length, PAGE_SIZE);
36c14ed9 815 dma_unmap_sg(&dev->pci_dev->dev, sg, nents, DMA_FROM_DEVICE);
7fc3cdab 816
36c14ed9 817 for (i = 0; i < count; i++)
7fc3cdab
MW		 818 put_page(sg_page(&sg[i]));
819}
b60503ba 820
7fc3cdab
MW		 821static int nvme_submit_user_admin_command(struct nvme_dev *dev,
822 unsigned long addr, unsigned length,
823 struct nvme_command *cmd)
824{
825 int err, nents;
826 struct scatterlist *sg;
827
828 nents = nvme_map_user_pages(dev, 0, addr, length, &sg);
829 if (nents < 0)
830 return nents;
831 nvme_setup_prps(&cmd->common, sg, length);
832 err = nvme_submit_admin_cmd(dev, cmd, NULL);
833 nvme_unmap_user_pages(dev, 0, addr, length, sg, nents);
834 return err ? -EIO : 0;
b60503ba
MW		 835}
836
bd38c555 837static int nvme_identify(struct nvme_ns *ns, unsigned long addr, int cns)
b60503ba 838{
b60503ba 839 struct nvme_command c;
b60503ba 840
bd38c555
MW		 841 memset(&c, 0, sizeof(c));
842 c.identify.opcode = nvme_admin_identify;
843 c.identify.nsid = cns ? 0 : cpu_to_le32(ns->ns_id);
844 c.identify.cns = cpu_to_le32(cns);
845
846 return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c);
847}
848
849static int nvme_get_range_type(struct nvme_ns *ns, unsigned long addr)
850{
851 struct nvme_command c;
b60503ba
MW		 852
853 memset(&c, 0, sizeof(c));
854 c.features.opcode = nvme_admin_get_features;
855 c.features.nsid = cpu_to_le32(ns->ns_id);
b60503ba
MW		 856 c.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE);
857
bd38c555 858 return nvme_submit_user_admin_command(ns->dev, addr, 4096, &c);
b60503ba
MW		 859}
860
a53295b6
MW		 861static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
862{
863 struct nvme_dev *dev = ns->dev;
864 struct nvme_queue *nvmeq;
865 struct nvme_user_io io;
866 struct nvme_command c;
867 unsigned length;
868 u32 result;
869 int nents, status;
870 struct scatterlist *sg;
871
872 if (copy_from_user(&io, uio, sizeof(io)))
873 return -EFAULT;
874 length = io.nblocks << io.block_shift;
875 nents = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length, &sg);
876 if (nents < 0)
877 return nents;
878
879 memset(&c, 0, sizeof(c));
880 c.rw.opcode = io.opcode;
881 c.rw.flags = io.flags;
882 c.rw.nsid = cpu_to_le32(io.nsid);
883 c.rw.slba = cpu_to_le64(io.slba);
884 c.rw.length = cpu_to_le16(io.nblocks - 1);
885 c.rw.control = cpu_to_le16(io.control);
886 c.rw.dsmgmt = cpu_to_le16(io.dsmgmt);
887 c.rw.reftag = cpu_to_le32(io.reftag); /* XXX: endian? */
888 c.rw.apptag = cpu_to_le16(io.apptag);
889 c.rw.appmask = cpu_to_le16(io.appmask);
890 /* XXX: metadata */
891 nvme_setup_prps(&c.common, sg, length);
892
893 nvmeq = get_nvmeq(ns);
b1ad37ef
MW		 894 /* Since nvme_submit_sync_cmd sleeps, we can't keep preemption
895 * disabled. We may be preempted at any point, and be rescheduled
896 * to a different CPU. That will cause cacheline bouncing, but no
897 * additional races since q_lock already protects against other CPUs.
898 */
a53295b6 899 put_nvmeq(nvmeq);
b1ad37ef 900 status = nvme_submit_sync_cmd(nvmeq, &c, &result);
a53295b6
MW		 901
902 nvme_unmap_user_pages(dev, io.opcode & 1, io.addr, length, sg, nents);
903 put_user(result, &uio->result);
904 return status;
905}
906
6ee44cdc
MW		 907static int nvme_download_firmware(struct nvme_ns *ns,
908 struct nvme_dlfw __user *udlfw)
909{
910 struct nvme_dev *dev = ns->dev;
911 struct nvme_dlfw dlfw;
912 struct nvme_command c;
913 int nents, status;
914 struct scatterlist *sg;
915
916 if (copy_from_user(&dlfw, udlfw, sizeof(dlfw)))
917 return -EFAULT;
918 if (dlfw.length >= (1 << 30))
919 return -EINVAL;
920
921 nents = nvme_map_user_pages(dev, 1, dlfw.addr, dlfw.length * 4, &sg);
922 if (nents < 0)
923 return nents;
924
925 memset(&c, 0, sizeof(c));
926 c.dlfw.opcode = nvme_admin_download_fw;
927 c.dlfw.numd = cpu_to_le32(dlfw.length);
928 c.dlfw.offset = cpu_to_le32(dlfw.offset);
929 nvme_setup_prps(&c.common, sg, dlfw.length * 4);
930
931 status = nvme_submit_admin_cmd(dev, &c, NULL);
932 nvme_unmap_user_pages(dev, 0, dlfw.addr, dlfw.length * 4, sg, nents);
933 return status;
934}
935
936static int nvme_activate_firmware(struct nvme_ns *ns, unsigned long arg)
937{
938 struct nvme_dev *dev = ns->dev;
939 struct nvme_command c;
940
941 memset(&c, 0, sizeof(c));
942 c.common.opcode = nvme_admin_activate_fw;
943 c.common.rsvd10[0] = cpu_to_le32(arg);
944
945 return nvme_submit_admin_cmd(dev, &c, NULL);
946}
947
b60503ba
MW		 948static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
949 unsigned long arg)
950{
951 struct nvme_ns *ns = bdev->bd_disk->private_data;
952
953 switch (cmd) {
954 case NVME_IOCTL_IDENTIFY_NS:
36c14ed9 955 return nvme_identify(ns, arg, 0);
b60503ba 956 case NVME_IOCTL_IDENTIFY_CTRL:
36c14ed9 957 return nvme_identify(ns, arg, 1);
b60503ba 958 case NVME_IOCTL_GET_RANGE_TYPE:
bd38c555 959 return nvme_get_range_type(ns, arg);
a53295b6
MW		 960 case NVME_IOCTL_SUBMIT_IO:
961 return nvme_submit_io(ns, (void __user *)arg);
6ee44cdc
MW		 962 case NVME_IOCTL_DOWNLOAD_FW:
963 return nvme_download_firmware(ns, (void __user *)arg);
964 case NVME_IOCTL_ACTIVATE_FW:
965 return nvme_activate_firmware(ns, arg);
b60503ba
MW		 966 default:
967 return -ENOTTY;
968 }
969}
970
971static const struct block_device_operations nvme_fops = {
972 .owner = THIS_MODULE,
973 .ioctl = nvme_ioctl,
974};
975
976static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int index,
977 struct nvme_id_ns *id, struct nvme_lba_range_type *rt)
978{
979 struct nvme_ns *ns;
980 struct gendisk *disk;
981 int lbaf;
982
983 if (rt->attributes & NVME_LBART_ATTRIB_HIDE)
984 return NULL;
985
986 ns = kzalloc(sizeof(*ns), GFP_KERNEL);
987 if (!ns)
988 return NULL;
989 ns->queue = blk_alloc_queue(GFP_KERNEL);
990 if (!ns->queue)
991 goto out_free_ns;
992 ns->queue->queue_flags = QUEUE_FLAG_DEFAULT | QUEUE_FLAG_NOMERGES |
993 QUEUE_FLAG_NONROT | QUEUE_FLAG_DISCARD;
994 blk_queue_make_request(ns->queue, nvme_make_request);
995 ns->dev = dev;
996 ns->queue->queuedata = ns;
997
998 disk = alloc_disk(NVME_MINORS);
999 if (!disk)
1000 goto out_free_queue;
1001 ns->ns_id = index;
1002 ns->disk = disk;
1003 lbaf = id->flbas & 0xf;
1004 ns->lba_shift = id->lbaf[lbaf].ds;
1005
1006 disk->major = nvme_major;
1007 disk->minors = NVME_MINORS;
1008 disk->first_minor = NVME_MINORS * index;
1009 disk->fops = &nvme_fops;
1010 disk->private_data = ns;
1011 disk->queue = ns->queue;
388f037f 1012 disk->driverfs_dev = &dev->pci_dev->dev;
b60503ba
MW		 1013 sprintf(disk->disk_name, "nvme%dn%d", dev->instance, index);
1014 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
1015
1016 return ns;
1017
1018 out_free_queue:
1019 blk_cleanup_queue(ns->queue);
1020 out_free_ns:
1021 kfree(ns);
1022 return NULL;
1023}
1024
1025static void nvme_ns_free(struct nvme_ns *ns)
1026{
1027 put_disk(ns->disk);
1028 blk_cleanup_queue(ns->queue);
1029 kfree(ns);
1030}
1031
b3b06812 1032static int set_queue_count(struct nvme_dev *dev, int count)
b60503ba
MW		 1033{
1034 int status;
1035 u32 result;
1036 struct nvme_command c;
b3b06812 1037 u32 q_count = (count - 1) | ((count - 1) << 16);
b60503ba
MW		 1038
1039 memset(&c, 0, sizeof(c));
1040 c.features.opcode = nvme_admin_get_features;
1041 c.features.fid = cpu_to_le32(NVME_FEAT_NUM_QUEUES);
1042 c.features.dword11 = cpu_to_le32(q_count);
1043
1044 status = nvme_submit_admin_cmd(dev, &c, &result);
1045 if (status)
1046 return -EIO;
1047 return min(result & 0xffff, result >> 16) + 1;
1048}
1049
b60503ba
MW		 1050static int __devinit nvme_setup_io_queues(struct nvme_dev *dev)
1051{
1b23484b 1052 int result, cpu, i, nr_queues;
b60503ba 1053
1b23484b
MW		 1054 nr_queues = num_online_cpus();
1055 result = set_queue_count(dev, nr_queues);
1056 if (result < 0)
1057 return result;
1058 if (result < nr_queues)
1059 nr_queues = result;
b60503ba 1060
1b23484b
MW		 1061 /* Deregister the admin queue's interrupt */
1062 free_irq(dev->entry[0].vector, dev->queues[0]);
1063
1064 for (i = 0; i < nr_queues; i++)
1065 dev->entry[i].entry = i;
1066 for (;;) {
1067 result = pci_enable_msix(dev->pci_dev, dev->entry, nr_queues);
1068 if (result == 0) {
1069 break;
1070 } else if (result > 0) {
1071 nr_queues = result;
1072 continue;
1073 } else {
1074 nr_queues = 1;
1075 break;
1076 }
1077 }
1078
1079 result = queue_request_irq(dev, dev->queues[0], "nvme admin");
1080 /* XXX: handle failure here */
1081
1082 cpu = cpumask_first(cpu_online_mask);
1083 for (i = 0; i < nr_queues; i++) {
1084 irq_set_affinity_hint(dev->entry[i].vector, get_cpu_mask(cpu));
1085 cpu = cpumask_next(cpu, cpu_online_mask);
1086 }
1087
1088 for (i = 0; i < nr_queues; i++) {
1089 dev->queues[i + 1] = nvme_create_queue(dev, i + 1,
1090 NVME_Q_DEPTH, i);
1091 if (!dev->queues[i + 1])
1092 return -ENOMEM;
1093 dev->queue_count++;
1094 }
b60503ba
MW		 1095
1096 return 0;
1097}
1098
1099static void nvme_free_queues(struct nvme_dev *dev)
1100{
1101 int i;
1102
1103 for (i = dev->queue_count - 1; i >= 0; i--)
1104 nvme_free_queue(dev, i);
1105}
1106
1107static int __devinit nvme_dev_add(struct nvme_dev *dev)
1108{
1109 int res, nn, i;
1110 struct nvme_ns *ns, *next;
51814232 1111 struct nvme_id_ctrl *ctrl;
b60503ba
MW		 1112 void *id;
1113 dma_addr_t dma_addr;
1114 struct nvme_command cid, crt;
1115
1116 res = nvme_setup_io_queues(dev);
1117 if (res)
1118 return res;
1119
1120 /* XXX: Switch to a SG list once prp2 works */
1121 id = dma_alloc_coherent(&dev->pci_dev->dev, 8192, &dma_addr,
1122 GFP_KERNEL);
1123
1124 memset(&cid, 0, sizeof(cid));
1125 cid.identify.opcode = nvme_admin_identify;
1126 cid.identify.nsid = 0;
1127 cid.identify.prp1 = cpu_to_le64(dma_addr);
1128 cid.identify.cns = cpu_to_le32(1);
1129
1130 res = nvme_submit_admin_cmd(dev, &cid, NULL);
1131 if (res) {
1132 res = -EIO;
1133 goto out_free;
1134 }
1135
51814232
MW		 1136 ctrl = id;
1137 nn = le32_to_cpup(&ctrl->nn);
1138 memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
1139 memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
1140 memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
b60503ba
MW		 1141
1142 cid.identify.cns = 0;
1143 memset(&crt, 0, sizeof(crt));
1144 crt.features.opcode = nvme_admin_get_features;
1145 crt.features.prp1 = cpu_to_le64(dma_addr + 4096);
1146 crt.features.fid = cpu_to_le32(NVME_FEAT_LBA_RANGE);
1147
1148 for (i = 0; i < nn; i++) {
1149 cid.identify.nsid = cpu_to_le32(i);
1150 res = nvme_submit_admin_cmd(dev, &cid, NULL);
1151 if (res)
1152 continue;
1153
1154 if (((struct nvme_id_ns *)id)->ncap == 0)
1155 continue;
1156
1157 crt.features.nsid = cpu_to_le32(i);
1158 res = nvme_submit_admin_cmd(dev, &crt, NULL);
1159 if (res)
1160 continue;
1161
1162 ns = nvme_alloc_ns(dev, i, id, id + 4096);
1163 if (ns)
1164 list_add_tail(&ns->list, &dev->namespaces);
1165 }
1166 list_for_each_entry(ns, &dev->namespaces, list)
1167 add_disk(ns->disk);
1168
1169 dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr);
1170 return 0;
1171
1172 out_free:
1173 list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
1174 list_del(&ns->list);
1175 nvme_ns_free(ns);
1176 }
1177
1178 dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr);
1179 return res;
1180}
1181
1182static int nvme_dev_remove(struct nvme_dev *dev)
1183{
1184 struct nvme_ns *ns, *next;
1185
1186 /* TODO: wait all I/O finished or cancel them */
1187
1188 list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
1189 list_del(&ns->list);
1190 del_gendisk(ns->disk);
1191 nvme_ns_free(ns);
1192 }
1193
1194 nvme_free_queues(dev);
1195
1196 return 0;
1197}
1198
1199/* XXX: Use an ida or something to let remove / add work correctly */
1200static void nvme_set_instance(struct nvme_dev *dev)
1201{
1202 static int instance;
1203 dev->instance = instance++;
1204}
1205
1206static void nvme_release_instance(struct nvme_dev *dev)
1207{
1208}
1209
1210static int __devinit nvme_probe(struct pci_dev *pdev,
1211 const struct pci_device_id *id)
1212{
574e8b95 1213 int bars, result = -ENOMEM;
b60503ba
MW		 1214 struct nvme_dev *dev;
1215
1216 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1217 if (!dev)
1218 return -ENOMEM;
1219 dev->entry = kcalloc(num_possible_cpus(), sizeof(*dev->entry),
1220 GFP_KERNEL);
1221 if (!dev->entry)
1222 goto free;
1b23484b
MW		 1223 dev->queues = kcalloc(num_possible_cpus() + 1, sizeof(void *),
1224 GFP_KERNEL);
b60503ba
MW		 1225 if (!dev->queues)
1226 goto free;
1227
0ee5a7d7
SMM		 1228 if (pci_enable_device_mem(pdev))
1229 goto free;
f64d3365 1230 pci_set_master(pdev);
574e8b95
MW		 1231 bars = pci_select_bars(pdev, IORESOURCE_MEM);
1232 if (pci_request_selected_regions(pdev, bars, "nvme"))
1233 goto disable;
0ee5a7d7 1234
b60503ba
MW		 1235 INIT_LIST_HEAD(&dev->namespaces);
1236 dev->pci_dev = pdev;
1237 pci_set_drvdata(pdev, dev);
2930353f
MW		 1238 dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1239 dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
b60503ba 1240 nvme_set_instance(dev);
53c9577e 1241 dev->entry[0].vector = pdev->irq;
b60503ba
MW		 1242
1243 dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
1244 if (!dev->bar) {
1245 result = -ENOMEM;
574e8b95 1246 goto disable_msix;
b60503ba
MW		 1247 }
1248
1249 result = nvme_configure_admin_queue(dev);
1250 if (result)
1251 goto unmap;
1252 dev->queue_count++;
1253
1254 result = nvme_dev_add(dev);
1255 if (result)
1256 goto delete;
1257 return 0;
1258
1259 delete:
1260 nvme_free_queues(dev);
1261 unmap:
1262 iounmap(dev->bar);
574e8b95 1263 disable_msix:
b60503ba
MW		 1264 pci_disable_msix(pdev);
1265 nvme_release_instance(dev);
574e8b95 1266 disable:
0ee5a7d7 1267 pci_disable_device(pdev);
574e8b95 1268 pci_release_regions(pdev);
b60503ba
MW		 1269 free:
1270 kfree(dev->queues);
1271 kfree(dev->entry);
1272 kfree(dev);
1273 return result;
1274}
1275
1276static void __devexit nvme_remove(struct pci_dev *pdev)
1277{
1278 struct nvme_dev *dev = pci_get_drvdata(pdev);
1279 nvme_dev_remove(dev);
1280 pci_disable_msix(pdev);
1281 iounmap(dev->bar);
1282 nvme_release_instance(dev);
0ee5a7d7 1283 pci_disable_device(pdev);
574e8b95 1284 pci_release_regions(pdev);
b60503ba
MW		 1285 kfree(dev->queues);
1286 kfree(dev->entry);
1287 kfree(dev);
1288}
1289
1290/* These functions are yet to be implemented */
1291#define nvme_error_detected NULL
1292#define nvme_dump_registers NULL
1293#define nvme_link_reset NULL
1294#define nvme_slot_reset NULL
1295#define nvme_error_resume NULL
1296#define nvme_suspend NULL
1297#define nvme_resume NULL
1298
1299static struct pci_error_handlers nvme_err_handler = {
1300 .error_detected = nvme_error_detected,
1301 .mmio_enabled = nvme_dump_registers,
1302 .link_reset = nvme_link_reset,
1303 .slot_reset = nvme_slot_reset,
1304 .resume = nvme_error_resume,
1305};
1306
1307/* Move to pci_ids.h later */
1308#define PCI_CLASS_STORAGE_EXPRESS 0x010802
1309
1310static DEFINE_PCI_DEVICE_TABLE(nvme_id_table) = {
1311 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
1312 { 0, }
1313};
1314MODULE_DEVICE_TABLE(pci, nvme_id_table);
1315
1316static struct pci_driver nvme_driver = {
1317 .name = "nvme",
1318 .id_table = nvme_id_table,
1319 .probe = nvme_probe,
1320 .remove = __devexit_p(nvme_remove),
1321 .suspend = nvme_suspend,
1322 .resume = nvme_resume,
1323 .err_handler = &nvme_err_handler,
1324};
1325
1326static int __init nvme_init(void)
1327{
1328 int result;
1329
1330 nvme_major = register_blkdev(nvme_major, "nvme");
1331 if (nvme_major <= 0)
1332 return -EBUSY;
1333
1334 result = pci_register_driver(&nvme_driver);
1335 if (!result)
1336 return 0;
1337
1338 unregister_blkdev(nvme_major, "nvme");
1339 return result;
1340}
1341
1342static void __exit nvme_exit(void)
1343{
1344 pci_unregister_driver(&nvme_driver);
1345 unregister_blkdev(nvme_major, "nvme");
1346}
1347
1348MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
1349MODULE_LICENSE("GPL");
db5d0c19 1350MODULE_VERSION("0.2");
b60503ba
MW		 1351module_init(nvme_init);
1352module_exit(nvme_exit);
Linux 6.x block layer and io_uring tree(s)