projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
NVMe: Fix check_flush_dependency warning
[linux-2.6-block.git] / drivers / nvme / host / core.c
CommitLineData
21d34711
CH		 1/*
2 * NVM Express device driver
3 * Copyright (c) 2011-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/blkdev.h>
16#include <linux/blk-mq.h>
5fd4ce1b 17#include <linux/delay.h>
21d34711 18#include <linux/errno.h>
1673f1f0 19#include <linux/hdreg.h>
21d34711 20#include <linux/kernel.h>
5bae7f73
CH		 21#include <linux/module.h>
22#include <linux/list_sort.h>
21d34711
CH		 23#include <linux/slab.h>
24#include <linux/types.h>
1673f1f0
CH		 25#include <linux/pr.h>
26#include <linux/ptrace.h>
27#include <linux/nvme_ioctl.h>
28#include <linux/t10-pi.h>
29#include <scsi/sg.h>
30#include <asm/unaligned.h>
21d34711
CH		 31
32#include "nvme.h"
33
f3ca80fc
CH		 34#define NVME_MINORS (1U << MINORBITS)
35
ba0ba7d3
ML		 36unsigned char admin_timeout = 60;
37module_param(admin_timeout, byte, 0644);
38MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
576d55d6 39EXPORT_SYMBOL_GPL(admin_timeout);
ba0ba7d3
ML		 40
41unsigned char nvme_io_timeout = 30;
42module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
43MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
576d55d6 44EXPORT_SYMBOL_GPL(nvme_io_timeout);
ba0ba7d3
ML		 45
46unsigned char shutdown_timeout = 5;
47module_param(shutdown_timeout, byte, 0644);
48MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
49
5bae7f73
CH		 50static int nvme_major;
51module_param(nvme_major, int, 0);
52
f3ca80fc
CH		 53static int nvme_char_major;
54module_param(nvme_char_major, int, 0);
55
56static LIST_HEAD(nvme_ctrl_list);
9f2482b9 57static DEFINE_SPINLOCK(dev_list_lock);
1673f1f0 58
f3ca80fc
CH		 59static struct class *nvme_class;
60
1673f1f0
CH		 61static void nvme_free_ns(struct kref *kref)
62{
63 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
64
65 if (ns->type == NVME_NS_LIGHTNVM)
66 nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
67
68 spin_lock(&dev_list_lock);
69 ns->disk->private_data = NULL;
70 spin_unlock(&dev_list_lock);
71
1673f1f0 72 put_disk(ns->disk);
075790eb
KB		 73 ida_simple_remove(&ns->ctrl->ns_ida, ns->instance);
74 nvme_put_ctrl(ns->ctrl);
1673f1f0
CH		 75 kfree(ns);
76}
77
5bae7f73 78static void nvme_put_ns(struct nvme_ns *ns)
1673f1f0
CH		 79{
80 kref_put(&ns->kref, nvme_free_ns);
81}
82
83static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk)
84{
85 struct nvme_ns *ns;
86
87 spin_lock(&dev_list_lock);
88 ns = disk->private_data;
e439bb12
SG		 89 if (ns) {
90 if (!kref_get_unless_zero(&ns->kref))
91 goto fail;
92 if (!try_module_get(ns->ctrl->ops->module))
93 goto fail_put_ns;
94 }
1673f1f0
CH		 95 spin_unlock(&dev_list_lock);
96
97 return ns;
e439bb12
SG		 98
99fail_put_ns:
100 kref_put(&ns->kref, nvme_free_ns);
101fail:
102 spin_unlock(&dev_list_lock);
103 return NULL;
1673f1f0
CH		 104}
105
7688faa6
CH		 106void nvme_requeue_req(struct request *req)
107{
108 unsigned long flags;
109
110 blk_mq_requeue_request(req);
111 spin_lock_irqsave(req->q->queue_lock, flags);
112 if (!blk_queue_stopped(req->q))
113 blk_mq_kick_requeue_list(req->q);
114 spin_unlock_irqrestore(req->q->queue_lock, flags);
115}
576d55d6 116EXPORT_SYMBOL_GPL(nvme_requeue_req);
7688faa6 117
4160982e
CH		 118struct request *nvme_alloc_request(struct request_queue *q,
119 struct nvme_command *cmd, unsigned int flags)
21d34711
CH		 120{
121 bool write = cmd->common.opcode & 1;
21d34711 122 struct request *req;
21d34711 123
4160982e 124 req = blk_mq_alloc_request(q, write, flags);
21d34711 125 if (IS_ERR(req))
4160982e 126 return req;
21d34711
CH		 127
128 req->cmd_type = REQ_TYPE_DRV_PRIV;
129 req->cmd_flags |= REQ_FAILFAST_DRIVER;
130 req->__data_len = 0;
131 req->__sector = (sector_t) -1;
132 req->bio = req->biotail = NULL;
133
21d34711
CH		 134 req->cmd = (unsigned char *)cmd;
135 req->cmd_len = sizeof(struct nvme_command);
21d34711 136
4160982e
CH		 137 return req;
138}
576d55d6 139EXPORT_SYMBOL_GPL(nvme_alloc_request);
4160982e 140
8093f7ca
ML		 141static inline void nvme_setup_flush(struct nvme_ns *ns,
142 struct nvme_command *cmnd)
143{
144 memset(cmnd, 0, sizeof(*cmnd));
145 cmnd->common.opcode = nvme_cmd_flush;
146 cmnd->common.nsid = cpu_to_le32(ns->ns_id);
147}
148
149static inline int nvme_setup_discard(struct nvme_ns *ns, struct request *req,
150 struct nvme_command *cmnd)
151{
152 struct nvme_dsm_range *range;
153 struct page *page;
154 int offset;
155 unsigned int nr_bytes = blk_rq_bytes(req);
156
157 range = kmalloc(sizeof(*range), GFP_ATOMIC);
158 if (!range)
159 return BLK_MQ_RQ_QUEUE_BUSY;
160
161 range->cattr = cpu_to_le32(0);
162 range->nlb = cpu_to_le32(nr_bytes >> ns->lba_shift);
163 range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
164
165 memset(cmnd, 0, sizeof(*cmnd));
166 cmnd->dsm.opcode = nvme_cmd_dsm;
167 cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
168 cmnd->dsm.nr = 0;
169 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
170
171 req->completion_data = range;
172 page = virt_to_page(range);
173 offset = offset_in_page(range);
174 blk_add_request_payload(req, page, offset, sizeof(*range));
175
176 /*
177 * we set __data_len back to the size of the area to be discarded
178 * on disk. This allows us to report completion on the full amount
179 * of blocks described by the request.
180 */
181 req->__data_len = nr_bytes;
182
183 return 0;
184}
185
186static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
187 struct nvme_command *cmnd)
188{
189 u16 control = 0;
190 u32 dsmgmt = 0;
191
192 if (req->cmd_flags & REQ_FUA)
193 control |= NVME_RW_FUA;
194 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
195 control |= NVME_RW_LR;
196
197 if (req->cmd_flags & REQ_RAHEAD)
198 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
199
200 memset(cmnd, 0, sizeof(*cmnd));
201 cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
202 cmnd->rw.command_id = req->tag;
203 cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
204 cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
205 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
206
207 if (ns->ms) {
208 switch (ns->pi_type) {
209 case NVME_NS_DPS_PI_TYPE3:
210 control |= NVME_RW_PRINFO_PRCHK_GUARD;
211 break;
212 case NVME_NS_DPS_PI_TYPE1:
213 case NVME_NS_DPS_PI_TYPE2:
214 control |= NVME_RW_PRINFO_PRCHK_GUARD |
215 NVME_RW_PRINFO_PRCHK_REF;
216 cmnd->rw.reftag = cpu_to_le32(
217 nvme_block_nr(ns, blk_rq_pos(req)));
218 break;
219 }
220 if (!blk_integrity_rq(req))
221 control |= NVME_RW_PRINFO_PRACT;
222 }
223
224 cmnd->rw.control = cpu_to_le16(control);
225 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
226}
227
228int nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
229 struct nvme_command *cmd)
230{
231 int ret = 0;
232
233 if (req->cmd_type == REQ_TYPE_DRV_PRIV)
234 memcpy(cmd, req->cmd, sizeof(*cmd));
235 else if (req->cmd_flags & REQ_FLUSH)
236 nvme_setup_flush(ns, cmd);
237 else if (req->cmd_flags & REQ_DISCARD)
238 ret = nvme_setup_discard(ns, req, cmd);
239 else
240 nvme_setup_rw(ns, req, cmd);
241
242 return ret;
243}
244EXPORT_SYMBOL_GPL(nvme_setup_cmd);
245
4160982e
CH		 246/*
247 * Returns 0 on success. If the result is negative, it's a Linux error code;
248 * if the result is positive, it's an NVM Express status code
249 */
250int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1cb3cce5
CH		 251 struct nvme_completion *cqe, void *buffer, unsigned bufflen,
252 unsigned timeout)
4160982e
CH		 253{
254 struct request *req;
255 int ret;
256
257 req = nvme_alloc_request(q, cmd, 0);
258 if (IS_ERR(req))
259 return PTR_ERR(req);
260
261 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
1cb3cce5 262 req->special = cqe;
4160982e 263
21d34711
CH		 264 if (buffer && bufflen) {
265 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
266 if (ret)
267 goto out;
4160982e
CH		 268 }
269
270 blk_execute_rq(req->q, NULL, req, 0);
4160982e
CH		 271 ret = req->errors;
272 out:
273 blk_mq_free_request(req);
274 return ret;
275}
276
277int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
278 void *buffer, unsigned bufflen)
279{
1cb3cce5 280 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0);
4160982e 281}
576d55d6 282EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
4160982e 283
0b7f1f26
KB		 284int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
285 void __user *ubuffer, unsigned bufflen,
286 void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
287 u32 *result, unsigned timeout)
4160982e 288{
0b7f1f26 289 bool write = cmd->common.opcode & 1;
1cb3cce5 290 struct nvme_completion cqe;
0b7f1f26
KB		 291 struct nvme_ns *ns = q->queuedata;
292 struct gendisk *disk = ns ? ns->disk : NULL;
4160982e 293 struct request *req;
0b7f1f26
KB		 294 struct bio *bio = NULL;
295 void *meta = NULL;
4160982e
CH		 296 int ret;
297
298 req = nvme_alloc_request(q, cmd, 0);
299 if (IS_ERR(req))
300 return PTR_ERR(req);
301
302 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
1cb3cce5 303 req->special = &cqe;
4160982e
CH		 304
305 if (ubuffer && bufflen) {
21d34711
CH		 306 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
307 GFP_KERNEL);
308 if (ret)
309 goto out;
310 bio = req->bio;
21d34711 311
0b7f1f26
KB		 312 if (!disk)
313 goto submit;
314 bio->bi_bdev = bdget_disk(disk, 0);
315 if (!bio->bi_bdev) {
316 ret = -ENODEV;
317 goto out_unmap;
318 }
319
e9fc63d6 320 if (meta_buffer && meta_len) {
0b7f1f26
KB		 321 struct bio_integrity_payload *bip;
322
323 meta = kmalloc(meta_len, GFP_KERNEL);
324 if (!meta) {
325 ret = -ENOMEM;
326 goto out_unmap;
327 }
328
329 if (write) {
330 if (copy_from_user(meta, meta_buffer,
331 meta_len)) {
332 ret = -EFAULT;
333 goto out_free_meta;
334 }
335 }
336
337 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
06c1e390
KB		 338 if (IS_ERR(bip)) {
339 ret = PTR_ERR(bip);
0b7f1f26
KB		 340 goto out_free_meta;
341 }
342
343 bip->bip_iter.bi_size = meta_len;
344 bip->bip_iter.bi_sector = meta_seed;
345
346 ret = bio_integrity_add_page(bio, virt_to_page(meta),
347 meta_len, offset_in_page(meta));
348 if (ret != meta_len) {
349 ret = -ENOMEM;
350 goto out_free_meta;
351 }
352 }
353 }
354 submit:
355 blk_execute_rq(req->q, disk, req, 0);
356 ret = req->errors;
21d34711 357 if (result)
1cb3cce5 358 *result = le32_to_cpu(cqe.result);
0b7f1f26
KB		 359 if (meta && !ret && !write) {
360 if (copy_to_user(meta_buffer, meta, meta_len))
361 ret = -EFAULT;
362 }
363 out_free_meta:
364 kfree(meta);
365 out_unmap:
366 if (bio) {
367 if (disk && bio->bi_bdev)
368 bdput(bio->bi_bdev);
369 blk_rq_unmap_user(bio);
370 }
21d34711
CH		 371 out:
372 blk_mq_free_request(req);
373 return ret;
374}
375
0b7f1f26
KB		 376int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
377 void __user *ubuffer, unsigned bufflen, u32 *result,
378 unsigned timeout)
379{
380 return __nvme_submit_user_cmd(q, cmd, ubuffer, bufflen, NULL, 0, 0,
381 result, timeout);
382}
383
1c63dc66 384int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
21d34711
CH		 385{
386 struct nvme_command c = { };
387 int error;
388
389 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
390 c.identify.opcode = nvme_admin_identify;
391 c.identify.cns = cpu_to_le32(1);
392
393 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
394 if (!*id)
395 return -ENOMEM;
396
397 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
398 sizeof(struct nvme_id_ctrl));
399 if (error)
400 kfree(*id);
401 return error;
402}
403
540c801c
KB		 404static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
405{
406 struct nvme_command c = { };
407
408 c.identify.opcode = nvme_admin_identify;
409 c.identify.cns = cpu_to_le32(2);
410 c.identify.nsid = cpu_to_le32(nsid);
411 return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
412}
413
1c63dc66 414int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
21d34711
CH		 415 struct nvme_id_ns **id)
416{
417 struct nvme_command c = { };
418 int error;
419
420 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
421 c.identify.opcode = nvme_admin_identify,
422 c.identify.nsid = cpu_to_le32(nsid),
423
424 *id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
425 if (!*id)
426 return -ENOMEM;
427
428 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
429 sizeof(struct nvme_id_ns));
430 if (error)
431 kfree(*id);
432 return error;
433}
434
1c63dc66 435int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
21d34711
CH		 436 dma_addr_t dma_addr, u32 *result)
437{
438 struct nvme_command c;
1cb3cce5
CH		 439 struct nvme_completion cqe;
440 int ret;
21d34711
CH		 441
442 memset(&c, 0, sizeof(c));
443 c.features.opcode = nvme_admin_get_features;
444 c.features.nsid = cpu_to_le32(nsid);
445 c.features.prp1 = cpu_to_le64(dma_addr);
446 c.features.fid = cpu_to_le32(fid);
447
1cb3cce5
CH		 448 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0);
449 if (ret >= 0)
450 *result = le32_to_cpu(cqe.result);
451 return ret;
21d34711
CH		 452}
453
1c63dc66 454int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
21d34711
CH		 455 dma_addr_t dma_addr, u32 *result)
456{
457 struct nvme_command c;
1cb3cce5
CH		 458 struct nvme_completion cqe;
459 int ret;
21d34711
CH		 460
461 memset(&c, 0, sizeof(c));
462 c.features.opcode = nvme_admin_set_features;
463 c.features.prp1 = cpu_to_le64(dma_addr);
464 c.features.fid = cpu_to_le32(fid);
465 c.features.dword11 = cpu_to_le32(dword11);
466
1cb3cce5
CH		 467 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0);
468 if (ret >= 0)
469 *result = le32_to_cpu(cqe.result);
470 return ret;
21d34711
CH		 471}
472
1c63dc66 473int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
21d34711
CH		 474{
475 struct nvme_command c = { };
476 int error;
477
478 c.common.opcode = nvme_admin_get_log_page,
479 c.common.nsid = cpu_to_le32(0xFFFFFFFF),
480 c.common.cdw10[0] = cpu_to_le32(
481 (((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
482 NVME_LOG_SMART),
483
484 *log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
485 if (!*log)
486 return -ENOMEM;
487
488 error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
489 sizeof(struct nvme_smart_log));
490 if (error)
491 kfree(*log);
492 return error;
493}
1673f1f0 494
9a0be7ab
CH		 495int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
496{
497 u32 q_count = (*count - 1) | ((*count - 1) << 16);
498 u32 result;
499 int status, nr_io_queues;
500
501 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0,
502 &result);
503 if (status)
504 return status;
505
506 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
507 *count = min(*count, nr_io_queues);
508 return 0;
509}
576d55d6 510EXPORT_SYMBOL_GPL(nvme_set_queue_count);
9a0be7ab 511
1673f1f0
CH		 512static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
513{
514 struct nvme_user_io io;
515 struct nvme_command c;
516 unsigned length, meta_len;
517 void __user *metadata;
518
519 if (copy_from_user(&io, uio, sizeof(io)))
520 return -EFAULT;
63088ec7
KB		 521 if (io.flags)
522 return -EINVAL;
1673f1f0
CH		 523
524 switch (io.opcode) {
525 case nvme_cmd_write:
526 case nvme_cmd_read:
527 case nvme_cmd_compare:
528 break;
529 default:
530 return -EINVAL;
531 }
532
533 length = (io.nblocks + 1) << ns->lba_shift;
534 meta_len = (io.nblocks + 1) * ns->ms;
535 metadata = (void __user *)(uintptr_t)io.metadata;
536
537 if (ns->ext) {
538 length += meta_len;
539 meta_len = 0;
540 } else if (meta_len) {
541 if ((io.metadata & 3) || !io.metadata)
542 return -EINVAL;
543 }
544
545 memset(&c, 0, sizeof(c));
546 c.rw.opcode = io.opcode;
547 c.rw.flags = io.flags;
548 c.rw.nsid = cpu_to_le32(ns->ns_id);
549 c.rw.slba = cpu_to_le64(io.slba);
550 c.rw.length = cpu_to_le16(io.nblocks);
551 c.rw.control = cpu_to_le16(io.control);
552 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
553 c.rw.reftag = cpu_to_le32(io.reftag);
554 c.rw.apptag = cpu_to_le16(io.apptag);
555 c.rw.appmask = cpu_to_le16(io.appmask);
556
557 return __nvme_submit_user_cmd(ns->queue, &c,
558 (void __user *)(uintptr_t)io.addr, length,
559 metadata, meta_len, io.slba, NULL, 0);
560}
561
f3ca80fc 562static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1673f1f0
CH		 563 struct nvme_passthru_cmd __user *ucmd)
564{
565 struct nvme_passthru_cmd cmd;
566 struct nvme_command c;
567 unsigned timeout = 0;
568 int status;
569
570 if (!capable(CAP_SYS_ADMIN))
571 return -EACCES;
572 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
573 return -EFAULT;
63088ec7
KB		 574 if (cmd.flags)
575 return -EINVAL;
1673f1f0
CH		 576
577 memset(&c, 0, sizeof(c));
578 c.common.opcode = cmd.opcode;
579 c.common.flags = cmd.flags;
580 c.common.nsid = cpu_to_le32(cmd.nsid);
581 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
582 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
583 c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
584 c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
585 c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
586 c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
587 c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
588 c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
589
590 if (cmd.timeout_ms)
591 timeout = msecs_to_jiffies(cmd.timeout_ms);
592
593 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
d1ea7be5 594 (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1673f1f0
CH		 595 &cmd.result, timeout);
596 if (status >= 0) {
597 if (put_user(cmd.result, &ucmd->result))
598 return -EFAULT;
599 }
600
601 return status;
602}
603
604static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
605 unsigned int cmd, unsigned long arg)
606{
607 struct nvme_ns *ns = bdev->bd_disk->private_data;
608
609 switch (cmd) {
610 case NVME_IOCTL_ID:
611 force_successful_syscall_return();
612 return ns->ns_id;
613 case NVME_IOCTL_ADMIN_CMD:
614 return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
615 case NVME_IOCTL_IO_CMD:
616 return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
617 case NVME_IOCTL_SUBMIT_IO:
618 return nvme_submit_io(ns, (void __user *)arg);
44907332 619#ifdef CONFIG_BLK_DEV_NVME_SCSI
1673f1f0
CH		 620 case SG_GET_VERSION_NUM:
621 return nvme_sg_get_version_num((void __user *)arg);
622 case SG_IO:
623 return nvme_sg_io(ns, (void __user *)arg);
44907332 624#endif
1673f1f0
CH		 625 default:
626 return -ENOTTY;
627 }
628}
629
630#ifdef CONFIG_COMPAT
631static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
632 unsigned int cmd, unsigned long arg)
633{
634 switch (cmd) {
635 case SG_IO:
636 return -ENOIOCTLCMD;
637 }
638 return nvme_ioctl(bdev, mode, cmd, arg);
639}
640#else
641#define nvme_compat_ioctl NULL
642#endif
643
644static int nvme_open(struct block_device *bdev, fmode_t mode)
645{
646 return nvme_get_ns_from_disk(bdev->bd_disk) ? 0 : -ENXIO;
647}
648
649static void nvme_release(struct gendisk *disk, fmode_t mode)
650{
e439bb12
SG		 651 struct nvme_ns *ns = disk->private_data;
652
653 module_put(ns->ctrl->ops->module);
654 nvme_put_ns(ns);
1673f1f0
CH		 655}
656
657static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
658{
659 /* some standard values */
660 geo->heads = 1 << 6;
661 geo->sectors = 1 << 5;
662 geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
663 return 0;
664}
665
666#ifdef CONFIG_BLK_DEV_INTEGRITY
667static void nvme_init_integrity(struct nvme_ns *ns)
668{
669 struct blk_integrity integrity;
670
671 switch (ns->pi_type) {
672 case NVME_NS_DPS_PI_TYPE3:
673 integrity.profile = &t10_pi_type3_crc;
674 break;
675 case NVME_NS_DPS_PI_TYPE1:
676 case NVME_NS_DPS_PI_TYPE2:
677 integrity.profile = &t10_pi_type1_crc;
678 break;
679 default:
680 integrity.profile = NULL;
681 break;
682 }
683 integrity.tuple_size = ns->ms;
684 blk_integrity_register(ns->disk, &integrity);
685 blk_queue_max_integrity_segments(ns->queue, 1);
686}
687#else
688static void nvme_init_integrity(struct nvme_ns *ns)
689{
690}
691#endif /* CONFIG_BLK_DEV_INTEGRITY */
692
693static void nvme_config_discard(struct nvme_ns *ns)
694{
08095e70 695 struct nvme_ctrl *ctrl = ns->ctrl;
1673f1f0 696 u32 logical_block_size = queue_logical_block_size(ns->queue);
08095e70
KB		 697
698 if (ctrl->quirks & NVME_QUIRK_DISCARD_ZEROES)
699 ns->queue->limits.discard_zeroes_data = 1;
700 else
701 ns->queue->limits.discard_zeroes_data = 0;
702
1673f1f0
CH		 703 ns->queue->limits.discard_alignment = logical_block_size;
704 ns->queue->limits.discard_granularity = logical_block_size;
705 blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
706 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
707}
708
5bae7f73 709static int nvme_revalidate_disk(struct gendisk *disk)
1673f1f0
CH		 710{
711 struct nvme_ns *ns = disk->private_data;
712 struct nvme_id_ns *id;
713 u8 lbaf, pi_type;
714 u16 old_ms;
715 unsigned short bs;
716
69d9a99c
KB		 717 if (test_bit(NVME_NS_DEAD, &ns->flags)) {
718 set_capacity(disk, 0);
719 return -ENODEV;
720 }
1673f1f0 721 if (nvme_identify_ns(ns->ctrl, ns->ns_id, &id)) {
1b3c47c1
SG		 722 dev_warn(disk_to_dev(ns->disk), "%s: Identify failure\n",
723 __func__);
1673f1f0
CH		 724 return -ENODEV;
725 }
726 if (id->ncap == 0) {
727 kfree(id);
728 return -ENODEV;
729 }
730
731 if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
732 if (nvme_nvm_register(ns->queue, disk->disk_name)) {
1b3c47c1 733 dev_warn(disk_to_dev(ns->disk),
1673f1f0
CH		 734 "%s: LightNVM init failure\n", __func__);
735 kfree(id);
736 return -ENODEV;
737 }
738 ns->type = NVME_NS_LIGHTNVM;
739 }
740
2b9b6e86
KB		 741 if (ns->ctrl->vs >= NVME_VS(1, 1))
742 memcpy(ns->eui, id->eui64, sizeof(ns->eui));
743 if (ns->ctrl->vs >= NVME_VS(1, 2))
744 memcpy(ns->uuid, id->nguid, sizeof(ns->uuid));
745
1673f1f0
CH		 746 old_ms = ns->ms;
747 lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
748 ns->lba_shift = id->lbaf[lbaf].ds;
749 ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
750 ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
751
752 /*
753 * If identify namespace failed, use default 512 byte block size so
754 * block layer can use before failing read/write for 0 capacity.
755 */
756 if (ns->lba_shift == 0)
757 ns->lba_shift = 9;
758 bs = 1 << ns->lba_shift;
1673f1f0
CH		 759 /* XXX: PI implementation requires metadata equal t10 pi tuple size */
760 pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
761 id->dps & NVME_NS_DPS_PI_MASK : 0;
762
763 blk_mq_freeze_queue(disk->queue);
764 if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
765 ns->ms != old_ms ||
766 bs != queue_logical_block_size(disk->queue) ||
767 (ns->ms && ns->ext)))
768 blk_integrity_unregister(disk);
769
770 ns->pi_type = pi_type;
771 blk_queue_logical_block_size(ns->queue, bs);
772
4b9d5b15 773 if (ns->ms && !blk_get_integrity(disk) && !ns->ext)
1673f1f0 774 nvme_init_integrity(ns);
1673f1f0
CH		 775 if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
776 set_capacity(disk, 0);
777 else
778 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
779
780 if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
781 nvme_config_discard(ns);
782 blk_mq_unfreeze_queue(disk->queue);
783
784 kfree(id);
785 return 0;
786}
787
788static char nvme_pr_type(enum pr_type type)
789{
790 switch (type) {
791 case PR_WRITE_EXCLUSIVE:
792 return 1;
793 case PR_EXCLUSIVE_ACCESS:
794 return 2;
795 case PR_WRITE_EXCLUSIVE_REG_ONLY:
796 return 3;
797 case PR_EXCLUSIVE_ACCESS_REG_ONLY:
798 return 4;
799 case PR_WRITE_EXCLUSIVE_ALL_REGS:
800 return 5;
801 case PR_EXCLUSIVE_ACCESS_ALL_REGS:
802 return 6;
803 default:
804 return 0;
805 }
806};
807
808static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
809 u64 key, u64 sa_key, u8 op)
810{
811 struct nvme_ns *ns = bdev->bd_disk->private_data;
812 struct nvme_command c;
813 u8 data[16] = { 0, };
814
815 put_unaligned_le64(key, &data[0]);
816 put_unaligned_le64(sa_key, &data[8]);
817
818 memset(&c, 0, sizeof(c));
819 c.common.opcode = op;
820 c.common.nsid = cpu_to_le32(ns->ns_id);
821 c.common.cdw10[0] = cpu_to_le32(cdw10);
822
823 return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
824}
825
826static int nvme_pr_register(struct block_device *bdev, u64 old,
827 u64 new, unsigned flags)
828{
829 u32 cdw10;
830
831 if (flags & ~PR_FL_IGNORE_KEY)
832 return -EOPNOTSUPP;
833
834 cdw10 = old ? 2 : 0;
835 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
836 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
837 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
838}
839
840static int nvme_pr_reserve(struct block_device *bdev, u64 key,
841 enum pr_type type, unsigned flags)
842{
843 u32 cdw10;
844
845 if (flags & ~PR_FL_IGNORE_KEY)
846 return -EOPNOTSUPP;
847
848 cdw10 = nvme_pr_type(type) << 8;
849 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
850 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
851}
852
853static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
854 enum pr_type type, bool abort)
855{
856 u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
857 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
858}
859
860static int nvme_pr_clear(struct block_device *bdev, u64 key)
861{
8c0b3915 862 u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1673f1f0
CH		 863 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
864}
865
866static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
867{
868 u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
869 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
870}
871
872static const struct pr_ops nvme_pr_ops = {
873 .pr_register = nvme_pr_register,
874 .pr_reserve = nvme_pr_reserve,
875 .pr_release = nvme_pr_release,
876 .pr_preempt = nvme_pr_preempt,
877 .pr_clear = nvme_pr_clear,
878};
879
5bae7f73 880static const struct block_device_operations nvme_fops = {
1673f1f0
CH		 881 .owner = THIS_MODULE,
882 .ioctl = nvme_ioctl,
883 .compat_ioctl = nvme_compat_ioctl,
884 .open = nvme_open,
885 .release = nvme_release,
886 .getgeo = nvme_getgeo,
887 .revalidate_disk= nvme_revalidate_disk,
888 .pr_ops = &nvme_pr_ops,
889};
890
5fd4ce1b
CH		 891static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
892{
893 unsigned long timeout =
894 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
895 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
896 int ret;
897
898 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
899 if ((csts & NVME_CSTS_RDY) == bit)
900 break;
901
902 msleep(100);
903 if (fatal_signal_pending(current))
904 return -EINTR;
905 if (time_after(jiffies, timeout)) {
1b3c47c1 906 dev_err(ctrl->device,
5fd4ce1b
CH		 907 "Device not ready; aborting %s\n", enabled ?
908 "initialisation" : "reset");
909 return -ENODEV;
910 }
911 }
912
913 return ret;
914}
915
916/*
917 * If the device has been passed off to us in an enabled state, just clear
918 * the enabled bit. The spec says we should set the 'shutdown notification
919 * bits', but doing so may cause the device to complete commands to the
920 * admin queue ... and we don't know what memory that might be pointing at!
921 */
922int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
923{
924 int ret;
925
926 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
927 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
928
929 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
930 if (ret)
931 return ret;
932 return nvme_wait_ready(ctrl, cap, false);
933}
576d55d6 934EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
5fd4ce1b
CH		 935
936int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
937{
938 /*
939 * Default to a 4K page size, with the intention to update this
940 * path in the future to accomodate architectures with differing
941 * kernel and IO page sizes.
942 */
943 unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
944 int ret;
945
946 if (page_shift < dev_page_min) {
1b3c47c1 947 dev_err(ctrl->device,
5fd4ce1b
CH		 948 "Minimum device page size %u too large for host (%u)\n",
949 1 << dev_page_min, 1 << page_shift);
950 return -ENODEV;
951 }
952
953 ctrl->page_size = 1 << page_shift;
954
955 ctrl->ctrl_config = NVME_CC_CSS_NVM;
956 ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
957 ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
958 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
959 ctrl->ctrl_config |= NVME_CC_ENABLE;
960
961 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
962 if (ret)
963 return ret;
964 return nvme_wait_ready(ctrl, cap, true);
965}
576d55d6 966EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
5fd4ce1b
CH		 967
968int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
969{
970 unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
971 u32 csts;
972 int ret;
973
974 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
975 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
976
977 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
978 if (ret)
979 return ret;
980
981 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
982 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
983 break;
984
985 msleep(100);
986 if (fatal_signal_pending(current))
987 return -EINTR;
988 if (time_after(jiffies, timeout)) {
1b3c47c1 989 dev_err(ctrl->device,
5fd4ce1b
CH		 990 "Device shutdown incomplete; abort shutdown\n");
991 return -ENODEV;
992 }
993 }
994
995 return ret;
996}
576d55d6 997EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
5fd4ce1b 998
da35825d
CH		 999static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1000 struct request_queue *q)
1001{
7c88cb00
JA		 1002 bool vwc = false;
1003
da35825d 1004 if (ctrl->max_hw_sectors) {
45686b61
CH		 1005 u32 max_segments =
1006 (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
1007
da35825d 1008 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
45686b61 1009 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
da35825d
CH		 1010 }
1011 if (ctrl->stripe_size)
1012 blk_queue_chunk_sectors(q, ctrl->stripe_size >> 9);
da35825d 1013 blk_queue_virt_boundary(q, ctrl->page_size - 1);
7c88cb00
JA		 1014 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
1015 vwc = true;
1016 blk_queue_write_cache(q, vwc, vwc);
da35825d
CH		 1017}
1018
7fd8930f
CH		 1019/*
1020 * Initialize the cached copies of the Identify data and various controller
1021 * register in our nvme_ctrl structure. This should be called as soon as
1022 * the admin queue is fully up and running.
1023 */
1024int nvme_init_identify(struct nvme_ctrl *ctrl)
1025{
1026 struct nvme_id_ctrl *id;
1027 u64 cap;
1028 int ret, page_shift;
1029
f3ca80fc
CH		 1030 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
1031 if (ret) {
1b3c47c1 1032 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
f3ca80fc
CH		 1033 return ret;
1034 }
1035
7fd8930f
CH		 1036 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
1037 if (ret) {
1b3c47c1 1038 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
7fd8930f
CH		 1039 return ret;
1040 }
1041 page_shift = NVME_CAP_MPSMIN(cap) + 12;
1042
f3ca80fc
CH		 1043 if (ctrl->vs >= NVME_VS(1, 1))
1044 ctrl->subsystem = NVME_CAP_NSSRC(cap);
1045
7fd8930f
CH		 1046 ret = nvme_identify_ctrl(ctrl, &id);
1047 if (ret) {
1b3c47c1 1048 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
7fd8930f
CH		 1049 return -EIO;
1050 }
1051
118472ab 1052 ctrl->vid = le16_to_cpu(id->vid);
7fd8930f 1053 ctrl->oncs = le16_to_cpup(&id->oncs);
6bf25d16 1054 atomic_set(&ctrl->abort_limit, id->acl + 1);
7fd8930f 1055 ctrl->vwc = id->vwc;
931e1c22 1056 ctrl->cntlid = le16_to_cpup(&id->cntlid);
7fd8930f
CH		 1057 memcpy(ctrl->serial, id->sn, sizeof(id->sn));
1058 memcpy(ctrl->model, id->mn, sizeof(id->mn));
1059 memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
1060 if (id->mdts)
1061 ctrl->max_hw_sectors = 1 << (id->mdts + page_shift - 9);
1062 else
1063 ctrl->max_hw_sectors = UINT_MAX;
1064
1065 if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && id->vs[3]) {
1066 unsigned int max_hw_sectors;
1067
1068 ctrl->stripe_size = 1 << (id->vs[3] + page_shift);
1069 max_hw_sectors = ctrl->stripe_size >> (page_shift - 9);
1070 if (ctrl->max_hw_sectors) {
1071 ctrl->max_hw_sectors = min(max_hw_sectors,
1072 ctrl->max_hw_sectors);
1073 } else {
1074 ctrl->max_hw_sectors = max_hw_sectors;
1075 }
1076 }
1077
da35825d
CH		 1078 nvme_set_queue_limits(ctrl, ctrl->admin_q);
1079
7fd8930f
CH		 1080 kfree(id);
1081 return 0;
1082}
576d55d6 1083EXPORT_SYMBOL_GPL(nvme_init_identify);
7fd8930f 1084
f3ca80fc 1085static int nvme_dev_open(struct inode *inode, struct file *file)
1673f1f0 1086{
f3ca80fc
CH		 1087 struct nvme_ctrl *ctrl;
1088 int instance = iminor(inode);
1089 int ret = -ENODEV;
1673f1f0 1090
f3ca80fc
CH		 1091 spin_lock(&dev_list_lock);
1092 list_for_each_entry(ctrl, &nvme_ctrl_list, node) {
1093 if (ctrl->instance != instance)
1094 continue;
1095
1096 if (!ctrl->admin_q) {
1097 ret = -EWOULDBLOCK;
1098 break;
1099 }
1100 if (!kref_get_unless_zero(&ctrl->kref))
1101 break;
1102 file->private_data = ctrl;
1103 ret = 0;
1104 break;
1105 }
1106 spin_unlock(&dev_list_lock);
1107
1108 return ret;
1673f1f0
CH		 1109}
1110
f3ca80fc 1111static int nvme_dev_release(struct inode *inode, struct file *file)
1673f1f0 1112{
f3ca80fc
CH		 1113 nvme_put_ctrl(file->private_data);
1114 return 0;
1115}
1116
bfd89471
CH		 1117static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
1118{
1119 struct nvme_ns *ns;
1120 int ret;
1121
1122 mutex_lock(&ctrl->namespaces_mutex);
1123 if (list_empty(&ctrl->namespaces)) {
1124 ret = -ENOTTY;
1125 goto out_unlock;
1126 }
1127
1128 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
1129 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
1b3c47c1 1130 dev_warn(ctrl->device,
bfd89471
CH		 1131 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
1132 ret = -EINVAL;
1133 goto out_unlock;
1134 }
1135
1b3c47c1 1136 dev_warn(ctrl->device,
bfd89471
CH		 1137 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
1138 kref_get(&ns->kref);
1139 mutex_unlock(&ctrl->namespaces_mutex);
1140
1141 ret = nvme_user_cmd(ctrl, ns, argp);
1142 nvme_put_ns(ns);
1143 return ret;
1144
1145out_unlock:
1146 mutex_unlock(&ctrl->namespaces_mutex);
1147 return ret;
1148}
1149
f3ca80fc
CH		 1150static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
1151 unsigned long arg)
1152{
1153 struct nvme_ctrl *ctrl = file->private_data;
1154 void __user *argp = (void __user *)arg;
f3ca80fc
CH		 1155
1156 switch (cmd) {
1157 case NVME_IOCTL_ADMIN_CMD:
1158 return nvme_user_cmd(ctrl, NULL, argp);
1159 case NVME_IOCTL_IO_CMD:
bfd89471 1160 return nvme_dev_user_cmd(ctrl, argp);
f3ca80fc 1161 case NVME_IOCTL_RESET:
1b3c47c1 1162 dev_warn(ctrl->device, "resetting controller\n");
f3ca80fc
CH		 1163 return ctrl->ops->reset_ctrl(ctrl);
1164 case NVME_IOCTL_SUBSYS_RESET:
1165 return nvme_reset_subsystem(ctrl);
1166 default:
1167 return -ENOTTY;
1168 }
1169}
1170
1171static const struct file_operations nvme_dev_fops = {
1172 .owner = THIS_MODULE,
1173 .open = nvme_dev_open,
1174 .release = nvme_dev_release,
1175 .unlocked_ioctl = nvme_dev_ioctl,
1176 .compat_ioctl = nvme_dev_ioctl,
1177};
1178
1179static ssize_t nvme_sysfs_reset(struct device *dev,
1180 struct device_attribute *attr, const char *buf,
1181 size_t count)
1182{
1183 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
1184 int ret;
1185
1186 ret = ctrl->ops->reset_ctrl(ctrl);
1187 if (ret < 0)
1188 return ret;
1189 return count;
1673f1f0 1190}
f3ca80fc 1191static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
1673f1f0 1192
118472ab
KB		 1193static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
1194 char *buf)
1195{
1196 struct nvme_ns *ns = dev_to_disk(dev)->private_data;
1197 struct nvme_ctrl *ctrl = ns->ctrl;
1198 int serial_len = sizeof(ctrl->serial);
1199 int model_len = sizeof(ctrl->model);
1200
1201 if (memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
1202 return sprintf(buf, "eui.%16phN\n", ns->uuid);
1203
1204 if (memchr_inv(ns->eui, 0, sizeof(ns->eui)))
1205 return sprintf(buf, "eui.%8phN\n", ns->eui);
1206
1207 while (ctrl->serial[serial_len - 1] == ' ')
1208 serial_len--;
1209 while (ctrl->model[model_len - 1] == ' ')
1210 model_len--;
1211
1212 return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", ctrl->vid,
1213 serial_len, ctrl->serial, model_len, ctrl->model, ns->ns_id);
1214}
1215static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);
1216
2b9b6e86
KB		 1217static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
1218 char *buf)
1219{
1220 struct nvme_ns *ns = dev_to_disk(dev)->private_data;
1221 return sprintf(buf, "%pU\n", ns->uuid);
1222}
1223static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);
1224
1225static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
1226 char *buf)
1227{
1228 struct nvme_ns *ns = dev_to_disk(dev)->private_data;
1229 return sprintf(buf, "%8phd\n", ns->eui);
1230}
1231static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);
1232
1233static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
1234 char *buf)
1235{
1236 struct nvme_ns *ns = dev_to_disk(dev)->private_data;
1237 return sprintf(buf, "%d\n", ns->ns_id);
1238}
1239static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);
1240
1241static struct attribute *nvme_ns_attrs[] = {
118472ab 1242 &dev_attr_wwid.attr,
2b9b6e86
KB		 1243 &dev_attr_uuid.attr,
1244 &dev_attr_eui.attr,
1245 &dev_attr_nsid.attr,
1246 NULL,
1247};
1248
1249static umode_t nvme_attrs_are_visible(struct kobject *kobj,
1250 struct attribute *a, int n)
1251{
1252 struct device *dev = container_of(kobj, struct device, kobj);
1253 struct nvme_ns *ns = dev_to_disk(dev)->private_data;
1254
1255 if (a == &dev_attr_uuid.attr) {
1256 if (!memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
1257 return 0;
1258 }
1259 if (a == &dev_attr_eui.attr) {
1260 if (!memchr_inv(ns->eui, 0, sizeof(ns->eui)))
1261 return 0;
1262 }
1263 return a->mode;
1264}
1265
1266static const struct attribute_group nvme_ns_attr_group = {
1267 .attrs = nvme_ns_attrs,
1268 .is_visible = nvme_attrs_are_visible,
1269};
1270
931e1c22 1271#define nvme_show_str_function(field) \
779ff756
KB		 1272static ssize_t field##_show(struct device *dev, \
1273 struct device_attribute *attr, char *buf) \
1274{ \
1275 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
1276 return sprintf(buf, "%.*s\n", (int)sizeof(ctrl->field), ctrl->field); \
1277} \
1278static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
1279
931e1c22
ML		 1280#define nvme_show_int_function(field) \
1281static ssize_t field##_show(struct device *dev, \
1282 struct device_attribute *attr, char *buf) \
1283{ \
1284 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
1285 return sprintf(buf, "%d\n", ctrl->field); \
1286} \
1287static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
1288
1289nvme_show_str_function(model);
1290nvme_show_str_function(serial);
1291nvme_show_str_function(firmware_rev);
1292nvme_show_int_function(cntlid);
779ff756
KB		 1293
1294static struct attribute *nvme_dev_attrs[] = {
1295 &dev_attr_reset_controller.attr,
1296 &dev_attr_model.attr,
1297 &dev_attr_serial.attr,
1298 &dev_attr_firmware_rev.attr,
931e1c22 1299 &dev_attr_cntlid.attr,
779ff756
KB		 1300 NULL
1301};
1302
1303static struct attribute_group nvme_dev_attrs_group = {
1304 .attrs = nvme_dev_attrs,
1305};
1306
1307static const struct attribute_group *nvme_dev_attr_groups[] = {
1308 &nvme_dev_attrs_group,
1309 NULL,
1310};
1311
5bae7f73
CH		 1312static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
1313{
1314 struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
1315 struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
1316
1317 return nsa->ns_id - nsb->ns_id;
1318}
1319
1320static struct nvme_ns *nvme_find_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1321{
1322 struct nvme_ns *ns;
1323
69d3b8ac
CH		 1324 lockdep_assert_held(&ctrl->namespaces_mutex);
1325
5bae7f73
CH		 1326 list_for_each_entry(ns, &ctrl->namespaces, list) {
1327 if (ns->ns_id == nsid)
1328 return ns;
1329 if (ns->ns_id > nsid)
1330 break;
1331 }
1332 return NULL;
1333}
1334
1335static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1336{
1337 struct nvme_ns *ns;
1338 struct gendisk *disk;
1339 int node = dev_to_node(ctrl->dev);
1340
69d3b8ac
CH		 1341 lockdep_assert_held(&ctrl->namespaces_mutex);
1342
5bae7f73
CH		 1343 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
1344 if (!ns)
1345 return;
1346
075790eb
KB		 1347 ns->instance = ida_simple_get(&ctrl->ns_ida, 1, 0, GFP_KERNEL);
1348 if (ns->instance < 0)
1349 goto out_free_ns;
1350
5bae7f73
CH		 1351 ns->queue = blk_mq_init_queue(ctrl->tagset);
1352 if (IS_ERR(ns->queue))
075790eb 1353 goto out_release_instance;
5bae7f73
CH		 1354 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
1355 ns->queue->queuedata = ns;
1356 ns->ctrl = ctrl;
1357
1358 disk = alloc_disk_node(0, node);
1359 if (!disk)
1360 goto out_free_queue;
1361
1362 kref_init(&ns->kref);
1363 ns->ns_id = nsid;
1364 ns->disk = disk;
1365 ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
5bae7f73 1366
da35825d 1367
5bae7f73 1368 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
da35825d 1369 nvme_set_queue_limits(ctrl, ns->queue);
5bae7f73
CH		 1370
1371 disk->major = nvme_major;
1372 disk->first_minor = 0;
1373 disk->fops = &nvme_fops;
1374 disk->private_data = ns;
1375 disk->queue = ns->queue;
1376 disk->driverfs_dev = ctrl->device;
1377 disk->flags = GENHD_FL_EXT_DEVT;
075790eb 1378 sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
5bae7f73 1379
5bae7f73
CH		 1380 if (nvme_revalidate_disk(ns->disk))
1381 goto out_free_disk;
1382
4b9d5b15 1383 list_add_tail(&ns->list, &ctrl->namespaces);
5bae7f73 1384 kref_get(&ctrl->kref);
2b9b6e86
KB		 1385 if (ns->type == NVME_NS_LIGHTNVM)
1386 return;
5bae7f73 1387
2b9b6e86
KB		 1388 add_disk(ns->disk);
1389 if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
1390 &nvme_ns_attr_group))
1391 pr_warn("%s: failed to create sysfs group for identification\n",
1392 ns->disk->disk_name);
5bae7f73
CH		 1393 return;
1394 out_free_disk:
1395 kfree(disk);
5bae7f73
CH		 1396 out_free_queue:
1397 blk_cleanup_queue(ns->queue);
075790eb
KB		 1398 out_release_instance:
1399 ida_simple_remove(&ctrl->ns_ida, ns->instance);
5bae7f73
CH		 1400 out_free_ns:
1401 kfree(ns);
1402}
1403
1404static void nvme_ns_remove(struct nvme_ns *ns)
1405{
646017a6
KB		 1406 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
1407 return;
69d3b8ac 1408
5bae7f73
CH		 1409 if (ns->disk->flags & GENHD_FL_UP) {
1410 if (blk_get_integrity(ns->disk))
1411 blk_integrity_unregister(ns->disk);
2b9b6e86
KB		 1412 sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
1413 &nvme_ns_attr_group);
5bae7f73 1414 del_gendisk(ns->disk);
5bae7f73
CH		 1415 blk_mq_abort_requeue_list(ns->queue);
1416 blk_cleanup_queue(ns->queue);
1417 }
646017a6 1418 mutex_lock(&ns->ctrl->namespaces_mutex);
5bae7f73 1419 list_del_init(&ns->list);
646017a6 1420 mutex_unlock(&ns->ctrl->namespaces_mutex);
5bae7f73
CH		 1421 nvme_put_ns(ns);
1422}
1423
540c801c
KB		 1424static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1425{
1426 struct nvme_ns *ns;
1427
1428 ns = nvme_find_ns(ctrl, nsid);
1429 if (ns) {
1430 if (revalidate_disk(ns->disk))
1431 nvme_ns_remove(ns);
1432 } else
1433 nvme_alloc_ns(ctrl, nsid);
1434}
1435
1436static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
1437{
1438 struct nvme_ns *ns;
1439 __le32 *ns_list;
1440 unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
1441 int ret = 0;
1442
1443 ns_list = kzalloc(0x1000, GFP_KERNEL);
1444 if (!ns_list)
1445 return -ENOMEM;
1446
1447 for (i = 0; i < num_lists; i++) {
1448 ret = nvme_identify_ns_list(ctrl, prev, ns_list);
1449 if (ret)
1450 goto out;
1451
1452 for (j = 0; j < min(nn, 1024U); j++) {
1453 nsid = le32_to_cpu(ns_list[j]);
1454 if (!nsid)
1455 goto out;
1456
1457 nvme_validate_ns(ctrl, nsid);
1458
1459 while (++prev < nsid) {
1460 ns = nvme_find_ns(ctrl, prev);
1461 if (ns)
1462 nvme_ns_remove(ns);
1463 }
1464 }
1465 nn -= j;
1466 }
1467 out:
1468 kfree(ns_list);
1469 return ret;
1470}
1471
5bae7f73
CH		 1472static void __nvme_scan_namespaces(struct nvme_ctrl *ctrl, unsigned nn)
1473{
1474 struct nvme_ns *ns, *next;
1475 unsigned i;
1476
69d3b8ac
CH		 1477 lockdep_assert_held(&ctrl->namespaces_mutex);
1478
540c801c
KB		 1479 for (i = 1; i <= nn; i++)
1480 nvme_validate_ns(ctrl, i);
1481
5bae7f73
CH		 1482 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
1483 if (ns->ns_id > nn)
1484 nvme_ns_remove(ns);
1485 }
5bae7f73
CH		 1486}
1487
1488void nvme_scan_namespaces(struct nvme_ctrl *ctrl)
1489{
1490 struct nvme_id_ctrl *id;
540c801c 1491 unsigned nn;
5bae7f73
CH		 1492
1493 if (nvme_identify_ctrl(ctrl, &id))
1494 return;
540c801c 1495
69d3b8ac 1496 mutex_lock(&ctrl->namespaces_mutex);
540c801c
KB		 1497 nn = le32_to_cpu(id->nn);
1498 if (ctrl->vs >= NVME_VS(1, 1) &&
1499 !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
1500 if (!nvme_scan_ns_list(ctrl, nn))
1501 goto done;
1502 }
5bae7f73 1503 __nvme_scan_namespaces(ctrl, le32_to_cpup(&id->nn));
540c801c
KB		 1504 done:
1505 list_sort(NULL, &ctrl->namespaces, ns_cmp);
69d3b8ac 1506 mutex_unlock(&ctrl->namespaces_mutex);
5bae7f73
CH		 1507 kfree(id);
1508}
576d55d6 1509EXPORT_SYMBOL_GPL(nvme_scan_namespaces);
5bae7f73
CH		 1510
1511void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
1512{
1513 struct nvme_ns *ns, *next;
1514
1515 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
1516 nvme_ns_remove(ns);
1517}
576d55d6 1518EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
5bae7f73 1519
f3ca80fc
CH		 1520static DEFINE_IDA(nvme_instance_ida);
1521
1522static int nvme_set_instance(struct nvme_ctrl *ctrl)
1523{
1524 int instance, error;
1525
1526 do {
1527 if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
1528 return -ENODEV;
1529
1530 spin_lock(&dev_list_lock);
1531 error = ida_get_new(&nvme_instance_ida, &instance);
1532 spin_unlock(&dev_list_lock);
1533 } while (error == -EAGAIN);
1534
1535 if (error)
1536 return -ENODEV;
1537
1538 ctrl->instance = instance;
1539 return 0;
1540}
1541
1542static void nvme_release_instance(struct nvme_ctrl *ctrl)
1543{
1544 spin_lock(&dev_list_lock);
1545 ida_remove(&nvme_instance_ida, ctrl->instance);
1546 spin_unlock(&dev_list_lock);
1547}
1548
53029b04 1549void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
576d55d6 1550{
53029b04 1551 device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
f3ca80fc
CH		 1552
1553 spin_lock(&dev_list_lock);
1554 list_del(&ctrl->node);
1555 spin_unlock(&dev_list_lock);
53029b04 1556}
576d55d6 1557EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
53029b04
KB		 1558
1559static void nvme_free_ctrl(struct kref *kref)
1560{
1561 struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
f3ca80fc
CH		 1562
1563 put_device(ctrl->device);
1564 nvme_release_instance(ctrl);
075790eb 1565 ida_destroy(&ctrl->ns_ida);
f3ca80fc
CH		 1566
1567 ctrl->ops->free_ctrl(ctrl);
1568}
1569
1570void nvme_put_ctrl(struct nvme_ctrl *ctrl)
1571{
1572 kref_put(&ctrl->kref, nvme_free_ctrl);
1573}
576d55d6 1574EXPORT_SYMBOL_GPL(nvme_put_ctrl);
f3ca80fc
CH		 1575
1576/*
1577 * Initialize a NVMe controller structures. This needs to be called during
1578 * earliest initialization so that we have the initialized structured around
1579 * during probing.
1580 */
1581int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
1582 const struct nvme_ctrl_ops *ops, unsigned long quirks)
1583{
1584 int ret;
1585
1586 INIT_LIST_HEAD(&ctrl->namespaces);
69d3b8ac 1587 mutex_init(&ctrl->namespaces_mutex);
f3ca80fc
CH		 1588 kref_init(&ctrl->kref);
1589 ctrl->dev = dev;
1590 ctrl->ops = ops;
1591 ctrl->quirks = quirks;
1592
1593 ret = nvme_set_instance(ctrl);
1594 if (ret)
1595 goto out;
1596
779ff756 1597 ctrl->device = device_create_with_groups(nvme_class, ctrl->dev,
f3ca80fc 1598 MKDEV(nvme_char_major, ctrl->instance),
f4f0f63e 1599 ctrl, nvme_dev_attr_groups,
779ff756 1600 "nvme%d", ctrl->instance);
f3ca80fc
CH		 1601 if (IS_ERR(ctrl->device)) {
1602 ret = PTR_ERR(ctrl->device);
1603 goto out_release_instance;
1604 }
1605 get_device(ctrl->device);
075790eb 1606 ida_init(&ctrl->ns_ida);
f3ca80fc 1607
f3ca80fc
CH		 1608 spin_lock(&dev_list_lock);
1609 list_add_tail(&ctrl->node, &nvme_ctrl_list);
1610 spin_unlock(&dev_list_lock);
1611
1612 return 0;
f3ca80fc
CH		 1613out_release_instance:
1614 nvme_release_instance(ctrl);
1615out:
1616 return ret;
1617}
576d55d6 1618EXPORT_SYMBOL_GPL(nvme_init_ctrl);
f3ca80fc 1619
69d9a99c
KB		 1620/**
1621 * nvme_kill_queues(): Ends all namespace queues
1622 * @ctrl: the dead controller that needs to end
1623 *
1624 * Call this function when the driver determines it is unable to get the
1625 * controller in a state capable of servicing IO.
1626 */
1627void nvme_kill_queues(struct nvme_ctrl *ctrl)
1628{
1629 struct nvme_ns *ns;
1630
1631 mutex_lock(&ctrl->namespaces_mutex);
1632 list_for_each_entry(ns, &ctrl->namespaces, list) {
1633 if (!kref_get_unless_zero(&ns->kref))
1634 continue;
1635
1636 /*
1637 * Revalidating a dead namespace sets capacity to 0. This will
1638 * end buffered writers dirtying pages that can't be synced.
1639 */
1640 if (!test_and_set_bit(NVME_NS_DEAD, &ns->flags))
1641 revalidate_disk(ns->disk);
1642
1643 blk_set_queue_dying(ns->queue);
1644 blk_mq_abort_requeue_list(ns->queue);
1645 blk_mq_start_stopped_hw_queues(ns->queue, true);
1646
1647 nvme_put_ns(ns);
1648 }
1649 mutex_unlock(&ctrl->namespaces_mutex);
1650}
237045fc 1651EXPORT_SYMBOL_GPL(nvme_kill_queues);
69d9a99c 1652
25646264 1653void nvme_stop_queues(struct nvme_ctrl *ctrl)
363c9aac
SG		 1654{
1655 struct nvme_ns *ns;
1656
69d3b8ac 1657 mutex_lock(&ctrl->namespaces_mutex);
363c9aac 1658 list_for_each_entry(ns, &ctrl->namespaces, list) {
363c9aac
SG		 1659 spin_lock_irq(ns->queue->queue_lock);
1660 queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
1661 spin_unlock_irq(ns->queue->queue_lock);
1662
1663 blk_mq_cancel_requeue_work(ns->queue);
1664 blk_mq_stop_hw_queues(ns->queue);
1665 }
69d3b8ac 1666 mutex_unlock(&ctrl->namespaces_mutex);
363c9aac 1667}
576d55d6 1668EXPORT_SYMBOL_GPL(nvme_stop_queues);
363c9aac 1669
25646264 1670void nvme_start_queues(struct nvme_ctrl *ctrl)
363c9aac
SG		 1671{
1672 struct nvme_ns *ns;
1673
69d3b8ac 1674 mutex_lock(&ctrl->namespaces_mutex);
363c9aac
SG		 1675 list_for_each_entry(ns, &ctrl->namespaces, list) {
1676 queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
363c9aac
SG		 1677 blk_mq_start_stopped_hw_queues(ns->queue, true);
1678 blk_mq_kick_requeue_list(ns->queue);
1679 }
69d3b8ac 1680 mutex_unlock(&ctrl->namespaces_mutex);
363c9aac 1681}
576d55d6 1682EXPORT_SYMBOL_GPL(nvme_start_queues);
363c9aac 1683
5bae7f73
CH		 1684int __init nvme_core_init(void)
1685{
1686 int result;
1687
1688 result = register_blkdev(nvme_major, "nvme");
1689 if (result < 0)
1690 return result;
1691 else if (result > 0)
1692 nvme_major = result;
1693
f3ca80fc
CH		 1694 result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
1695 &nvme_dev_fops);
1696 if (result < 0)
1697 goto unregister_blkdev;
1698 else if (result > 0)
1699 nvme_char_major = result;
1700
1701 nvme_class = class_create(THIS_MODULE, "nvme");
1702 if (IS_ERR(nvme_class)) {
1703 result = PTR_ERR(nvme_class);
1704 goto unregister_chrdev;
1705 }
1706
5bae7f73 1707 return 0;
f3ca80fc
CH		 1708
1709 unregister_chrdev:
1710 __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
1711 unregister_blkdev:
1712 unregister_blkdev(nvme_major, "nvme");
1713 return result;
5bae7f73
CH		 1714}
1715
1716void nvme_core_exit(void)
1717{
1718 unregister_blkdev(nvme_major, "nvme");
f3ca80fc
CH		 1719 class_destroy(nvme_class);
1720 __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
5bae7f73 1721}
576d55d6
ML		 1722
1723MODULE_LICENSE("GPL");
1724MODULE_VERSION("1.0");
1725module_init(nvme_core_init);
1726module_exit(nvme_core_exit);
Linux 6.x block layer and io_uring tree(s)