projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nvme: add a quirk to disable namespace identifiers
[linux-2.6-block.git] / drivers / nvme / host / core.c
CommitLineData
bc50ad75 1// SPDX-License-Identifier: GPL-2.0
21d34711
CH		 2/*
3 * NVM Express device driver
4 * Copyright (c) 2011-2014, Intel Corporation.
21d34711
CH		 5 */
6
7#include <linux/blkdev.h>
8#include <linux/blk-mq.h>
fe45e630 9#include <linux/blk-integrity.h>
c95b708d 10#include <linux/compat.h>
5fd4ce1b 11#include <linux/delay.h>
21d34711 12#include <linux/errno.h>
1673f1f0 13#include <linux/hdreg.h>
21d34711 14#include <linux/kernel.h>
5bae7f73 15#include <linux/module.h>
958f2a0f 16#include <linux/backing-dev.h>
21d34711
CH		 17#include <linux/slab.h>
18#include <linux/types.h>
1673f1f0
CH		 19#include <linux/pr.h>
20#include <linux/ptrace.h>
21#include <linux/nvme_ioctl.h>
c5552fde 22#include <linux/pm_qos.h>
1673f1f0 23#include <asm/unaligned.h>
21d34711
CH		 24
25#include "nvme.h"
038bd4cb 26#include "fabrics.h"
21d34711 27
35fe0d12
HR		 28#define CREATE_TRACE_POINTS
29#include "trace.h"
30
f3ca80fc
CH		 31#define NVME_MINORS (1U << MINORBITS)
32
8ae4e447
MO		 33unsigned int admin_timeout = 60;
34module_param(admin_timeout, uint, 0644);
ba0ba7d3 35MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
576d55d6 36EXPORT_SYMBOL_GPL(admin_timeout);
ba0ba7d3 37
8ae4e447
MO		 38unsigned int nvme_io_timeout = 30;
39module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
ba0ba7d3 40MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
576d55d6 41EXPORT_SYMBOL_GPL(nvme_io_timeout);
ba0ba7d3 42
b3b1b0b0 43static unsigned char shutdown_timeout = 5;
ba0ba7d3
ML		 44module_param(shutdown_timeout, byte, 0644);
45MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
46
44e44b29
CH		 47static u8 nvme_max_retries = 5;
48module_param_named(max_retries, nvme_max_retries, byte, 0644);
f80ec966 49MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
5bae7f73 50
9947d6a0 51static unsigned long default_ps_max_latency_us = 100000;
c5552fde
AL		 52module_param(default_ps_max_latency_us, ulong, 0644);
53MODULE_PARM_DESC(default_ps_max_latency_us,
54 "max power saving latency for new devices; use PM QOS to change per device");
55
c35e30b4
AL		 56static bool force_apst;
57module_param(force_apst, bool, 0644);
58MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
59
ebd8a93a
AB		 60static unsigned long apst_primary_timeout_ms = 100;
61module_param(apst_primary_timeout_ms, ulong, 0644);
62MODULE_PARM_DESC(apst_primary_timeout_ms,
63 "primary APST timeout in ms");
64
65static unsigned long apst_secondary_timeout_ms = 2000;
66module_param(apst_secondary_timeout_ms, ulong, 0644);
67MODULE_PARM_DESC(apst_secondary_timeout_ms,
68 "secondary APST timeout in ms");
69
70static unsigned long apst_primary_latency_tol_us = 15000;
71module_param(apst_primary_latency_tol_us, ulong, 0644);
72MODULE_PARM_DESC(apst_primary_latency_tol_us,
73 "primary APST latency tolerance in us");
74
75static unsigned long apst_secondary_latency_tol_us = 100000;
76module_param(apst_secondary_latency_tol_us, ulong, 0644);
77MODULE_PARM_DESC(apst_secondary_latency_tol_us,
78 "secondary APST latency tolerance in us");
79
b227c59b
RS		 80/*
81 * nvme_wq - hosts nvme related works that are not reset or delete
82 * nvme_reset_wq - hosts nvme reset works
83 * nvme_delete_wq - hosts nvme delete works
84 *
97b2512a
NK		 85 * nvme_wq will host works such as scan, aen handling, fw activation,
86 * keep-alive, periodic reconnects etc. nvme_reset_wq
b227c59b
RS		 87 * runs reset works which also flush works hosted on nvme_wq for
88 * serialization purposes. nvme_delete_wq host controller deletion
89 * works which flush reset works for serialization.
90 */
9a6327d2
SG		 91struct workqueue_struct *nvme_wq;
92EXPORT_SYMBOL_GPL(nvme_wq);
93
b227c59b
RS		 94struct workqueue_struct *nvme_reset_wq;
95EXPORT_SYMBOL_GPL(nvme_reset_wq);
96
97struct workqueue_struct *nvme_delete_wq;
98EXPORT_SYMBOL_GPL(nvme_delete_wq);
99
ab9e00cc
CH		 100static LIST_HEAD(nvme_subsystems);
101static DEFINE_MUTEX(nvme_subsystems_lock);
1673f1f0 102
9843f685 103static DEFINE_IDA(nvme_instance_ida);
f68abd9c 104static dev_t nvme_ctrl_base_chr_devt;
f3ca80fc 105static struct class *nvme_class;
ab9e00cc 106static struct class *nvme_subsys_class;
f3ca80fc 107
2637baed
MI		 108static DEFINE_IDA(nvme_ns_chr_minor_ida);
109static dev_t nvme_ns_chr_devt;
110static struct class *nvme_ns_chr_class;
111
12d9f070 112static void nvme_put_subsystem(struct nvme_subsystem *subsys);
cf39a6bc
SB		 113static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
114 unsigned nsid);
b58da2d2
TS		 115static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
116 struct nvme_command *cmd);
cf39a6bc 117
2405252a 118void nvme_queue_scan(struct nvme_ctrl *ctrl)
50e8d8ee
CH		 119{
120 /*
121 * Only new queue scan work when admin and IO queues are both alive
122 */
5d02a5c1 123 if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
50e8d8ee
CH		 124 queue_work(nvme_wq, &ctrl->scan_work);
125}
126
4c75f877
KB		 127/*
128 * Use this function to proceed with scheduling reset_work for a controller
129 * that had previously been set to the resetting state. This is intended for
130 * code paths that can't be interrupted by other reset attempts. A hot removal
131 * may prevent this from succeeding.
132 */
c1ac9a4b 133int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
4c75f877
KB		 134{
135 if (ctrl->state != NVME_CTRL_RESETTING)
136 return -EBUSY;
137 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
138 return -EBUSY;
139 return 0;
140}
c1ac9a4b 141EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
4c75f877 142
8c4dfea9
VG		 143static void nvme_failfast_work(struct work_struct *work)
144{
145 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
146 struct nvme_ctrl, failfast_work);
147
148 if (ctrl->state != NVME_CTRL_CONNECTING)
149 return;
150
151 set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
152 dev_info(ctrl->device, "failfast expired\n");
153 nvme_kick_requeue_lists(ctrl);
154}
155
156static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
157{
158 if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
159 return;
160
161 schedule_delayed_work(&ctrl->failfast_work,
162 ctrl->opts->fast_io_fail_tmo * HZ);
163}
164
165static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
166{
167 if (!ctrl->opts)
168 return;
169
170 cancel_delayed_work_sync(&ctrl->failfast_work);
171 clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
172}
173
174
d86c4d8e
CH		 175int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
176{
177 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
178 return -EBUSY;
b227c59b 179 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
d86c4d8e
CH		 180 return -EBUSY;
181 return 0;
182}
183EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
184
2405252a 185int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
d86c4d8e
CH		 186{
187 int ret;
188
189 ret = nvme_reset_ctrl(ctrl);
8000d1fd 190 if (!ret) {
d86c4d8e 191 flush_work(&ctrl->reset_work);
5d02a5c1 192 if (ctrl->state != NVME_CTRL_LIVE)
8000d1fd
NC		 193 ret = -ENETRESET;
194 }
195
d86c4d8e
CH		 196 return ret;
197}
198
a686ed75 199static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
c5017e85 200{
77d0612d 201 dev_info(ctrl->device,
e5ea42fa 202 "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
77d0612d 203
4054637c 204 flush_work(&ctrl->reset_work);
6cd53d14
CH		 205 nvme_stop_ctrl(ctrl);
206 nvme_remove_namespaces(ctrl);
c5017e85 207 ctrl->ops->delete_ctrl(ctrl);
6cd53d14 208 nvme_uninit_ctrl(ctrl);
c5017e85
CH		 209}
210
a686ed75
BVA		 211static void nvme_delete_ctrl_work(struct work_struct *work)
212{
213 struct nvme_ctrl *ctrl =
214 container_of(work, struct nvme_ctrl, delete_work);
215
216 nvme_do_delete_ctrl(ctrl);
217}
218
c5017e85
CH		 219int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
220{
221 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
222 return -EBUSY;
b227c59b 223 if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
c5017e85
CH		 224 return -EBUSY;
225 return 0;
226}
227EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
228
6721c18a 229static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
c5017e85 230{
c5017e85 231 /*
01fc08ff
YY		 232 * Keep a reference until nvme_do_delete_ctrl() complete,
233 * since ->delete_ctrl can free the controller.
c5017e85
CH		 234 */
235 nvme_get_ctrl(ctrl);
6721c18a 236 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
b9c77583 237 nvme_do_delete_ctrl(ctrl);
c5017e85 238 nvme_put_ctrl(ctrl);
c5017e85 239}
c5017e85 240
2f9c1736 241static blk_status_t nvme_error_status(u16 status)
27fa9bc5 242{
2f9c1736 243 switch (status & 0x7ff) {
27fa9bc5 244 case NVME_SC_SUCCESS:
2a842aca 245 return BLK_STS_OK;
27fa9bc5 246 case NVME_SC_CAP_EXCEEDED:
2a842aca 247 return BLK_STS_NOSPC;
e96fef2c 248 case NVME_SC_LBA_RANGE:
35038bff
KB		 249 case NVME_SC_CMD_INTERRUPTED:
250 case NVME_SC_NS_NOT_READY:
e96fef2c
KB		 251 return BLK_STS_TARGET;
252 case NVME_SC_BAD_ATTRIBUTES:
e02ab023 253 case NVME_SC_ONCS_NOT_SUPPORTED:
e96fef2c
KB		 254 case NVME_SC_INVALID_OPCODE:
255 case NVME_SC_INVALID_FIELD:
256 case NVME_SC_INVALID_NS:
2a842aca 257 return BLK_STS_NOTSUPP;
e02ab023
JG		 258 case NVME_SC_WRITE_FAULT:
259 case NVME_SC_READ_ERROR:
260 case NVME_SC_UNWRITTEN_BLOCK:
a751da33
CH		 261 case NVME_SC_ACCESS_DENIED:
262 case NVME_SC_READ_ONLY:
e96fef2c 263 case NVME_SC_COMPARE_FAILED:
2a842aca 264 return BLK_STS_MEDIUM;
a751da33
CH		 265 case NVME_SC_GUARD_CHECK:
266 case NVME_SC_APPTAG_CHECK:
267 case NVME_SC_REFTAG_CHECK:
268 case NVME_SC_INVALID_PI:
269 return BLK_STS_PROTECTION;
270 case NVME_SC_RESERVATION_CONFLICT:
271 return BLK_STS_NEXUS;
1c0d12c0
SG		 272 case NVME_SC_HOST_PATH_ERROR:
273 return BLK_STS_TRANSPORT;
afaf5c6c
KB		 274 case NVME_SC_ZONE_TOO_MANY_ACTIVE:
275 return BLK_STS_ZONE_ACTIVE_RESOURCE;
276 case NVME_SC_ZONE_TOO_MANY_OPEN:
277 return BLK_STS_ZONE_OPEN_RESOURCE;
2a842aca
CH		 278 default:
279 return BLK_STS_IOERR;
27fa9bc5
CH		 280 }
281}
27fa9bc5 282
49cd84b6
KB		 283static void nvme_retry_req(struct request *req)
284{
49cd84b6
KB		 285 unsigned long delay = 0;
286 u16 crd;
287
288 /* The mask and shift result must be <= 3 */
289 crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
f9063a53
MI		 290 if (crd)
291 delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
49cd84b6
KB		 292
293 nvme_req(req)->retries++;
294 blk_mq_requeue_request(req, false);
295 blk_mq_delay_kick_requeue_list(req->q, delay);
296}
297
bd83fe6f
AA		 298static void nvme_log_error(struct request *req)
299{
300 struct nvme_ns *ns = req->q->queuedata;
301 struct nvme_request *nr = nvme_req(req);
302
303 if (ns) {
304 pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %llu blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
305 ns->disk ? ns->disk->disk_name : "?",
306 nvme_get_opcode_str(nr->cmd->common.opcode),
307 nr->cmd->common.opcode,
308 (unsigned long long)nvme_sect_to_lba(ns, blk_rq_pos(req)),
309 (unsigned long long)blk_rq_bytes(req) >> ns->lba_shift,
310 nvme_get_error_status_str(nr->status),
311 nr->status >> 8 & 7, /* Status Code Type */
312 nr->status & 0xff, /* Status Code */
313 nr->status & NVME_SC_MORE ? "MORE " : "",
314 nr->status & NVME_SC_DNR ? "DNR " : "");
315 return;
316 }
317
318 pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
319 dev_name(nr->ctrl->device),
320 nvme_get_admin_opcode_str(nr->cmd->common.opcode),
321 nr->cmd->common.opcode,
322 nvme_get_error_status_str(nr->status),
323 nr->status >> 8 & 7, /* Status Code Type */
324 nr->status & 0xff, /* Status Code */
325 nr->status & NVME_SC_MORE ? "MORE " : "",
326 nr->status & NVME_SC_DNR ? "DNR " : "");
327}
328
5ddaabe8
CH		 329enum nvme_disposition {
330 COMPLETE,
331 RETRY,
332 FAILOVER,
333};
334
335static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
77f02a7a 336{
5ddaabe8
CH		 337 if (likely(nvme_req(req)->status == 0))
338 return COMPLETE;
908e4564 339
5ddaabe8
CH		 340 if (blk_noretry_request(req) ||
341 (nvme_req(req)->status & NVME_SC_DNR) ||
342 nvme_req(req)->retries >= nvme_max_retries)
343 return COMPLETE;
ca5554a6 344
5ddaabe8 345 if (req->cmd_flags & REQ_NVME_MPATH) {
5eac5f33
CL		 346 if (nvme_is_path_error(nvme_req(req)->status) ||
347 blk_queue_dying(req->q))
5ddaabe8 348 return FAILOVER;
5eac5f33
CL		 349 } else {
350 if (blk_queue_dying(req->q))
351 return COMPLETE;
5ddaabe8 352 }
16686f3a 353
5ddaabe8
CH		 354 return RETRY;
355}
6e3ca03e 356
c234a653 357static inline void nvme_end_req_zoned(struct request *req)
5ddaabe8 358{
5ddaabe8
CH		 359 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
360 req_op(req) == REQ_OP_ZONE_APPEND)
240e6ee2
KB		 361 req->__sector = nvme_lba_to_sect(req->q->queuedata,
362 le64_to_cpu(nvme_req(req)->result.u64));
c234a653
JA		 363}
364
365static inline void nvme_end_req(struct request *req)
366{
367 blk_status_t status = nvme_error_status(nvme_req(req)->status);
35fe0d12 368
b42b6f44 369 if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET)))
bd83fe6f 370 nvme_log_error(req);
c234a653 371 nvme_end_req_zoned(req);
2b59787a 372 nvme_trace_bio_complete(req);
908e4564 373 blk_mq_end_request(req, status);
77f02a7a 374}
5ddaabe8
CH		 375
376void nvme_complete_rq(struct request *req)
377{
378 trace_nvme_complete_rq(req);
379 nvme_cleanup_cmd(req);
380
381 if (nvme_req(req)->ctrl->kas)
382 nvme_req(req)->ctrl->comp_seen = true;
383
384 switch (nvme_decide_disposition(req)) {
385 case COMPLETE:
386 nvme_end_req(req);
387 return;
388 case RETRY:
389 nvme_retry_req(req);
390 return;
391 case FAILOVER:
392 nvme_failover_req(req);
393 return;
394 }
395}
77f02a7a
CH		 396EXPORT_SYMBOL_GPL(nvme_complete_rq);
397
c234a653
JA		 398void nvme_complete_batch_req(struct request *req)
399{
00e757b6 400 trace_nvme_complete_rq(req);
c234a653
JA		 401 nvme_cleanup_cmd(req);
402 nvme_end_req_zoned(req);
403}
404EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
405
dda3248e
CL		 406/*
407 * Called to unwind from ->queue_rq on a failed command submission so that the
408 * multipathing code gets called to potentially failover to another path.
409 * The caller needs to unwind all transport specific resource allocations and
410 * must return propagate the return value.
411 */
412blk_status_t nvme_host_path_error(struct request *req)
413{
414 nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
415 blk_mq_set_request_complete(req);
416 nvme_complete_rq(req);
417 return BLK_STS_OK;
418}
419EXPORT_SYMBOL_GPL(nvme_host_path_error);
420
7baa8572 421bool nvme_cancel_request(struct request *req, void *data, bool reserved)
c55a2fd4 422{
c55a2fd4
ML		 423 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
424 "Cancelling I/O %d", req->tag);
425
78ca4072
ML		 426 /* don't abort one completed request */
427 if (blk_mq_request_completed(req))
428 return true;
429
2dc3947b 430 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
d3589381 431 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
15f73f5b 432 blk_mq_complete_request(req);
7baa8572 433 return true;
c55a2fd4
ML		 434}
435EXPORT_SYMBOL_GPL(nvme_cancel_request);
436
25479069
CL		 437void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
438{
439 if (ctrl->tagset) {
440 blk_mq_tagset_busy_iter(ctrl->tagset,
441 nvme_cancel_request, ctrl);
442 blk_mq_tagset_wait_completed_request(ctrl->tagset);
443 }
444}
445EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
446
447void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
448{
449 if (ctrl->admin_tagset) {
450 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
451 nvme_cancel_request, ctrl);
452 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
453 }
454}
455EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
456
bb8d261e
CH		 457bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
458 enum nvme_ctrl_state new_state)
459{
f6b6a28e 460 enum nvme_ctrl_state old_state;
0a72bbba 461 unsigned long flags;
bb8d261e
CH		 462 bool changed = false;
463
0a72bbba 464 spin_lock_irqsave(&ctrl->lock, flags);
f6b6a28e
GKB		 465
466 old_state = ctrl->state;
bb8d261e
CH		 467 switch (new_state) {
468 case NVME_CTRL_LIVE:
469 switch (old_state) {
7d2e8008 470 case NVME_CTRL_NEW:
bb8d261e 471 case NVME_CTRL_RESETTING:
ad6a0a52 472 case NVME_CTRL_CONNECTING:
bb8d261e 473 changed = true;
df561f66 474 fallthrough;
bb8d261e
CH		 475 default:
476 break;
477 }
478 break;
479 case NVME_CTRL_RESETTING:
480 switch (old_state) {
481 case NVME_CTRL_NEW:
def61eca 482 case NVME_CTRL_LIVE:
def61eca 483 changed = true;
df561f66 484 fallthrough;
def61eca
CH		 485 default:
486 break;
487 }
488 break;
ad6a0a52 489 case NVME_CTRL_CONNECTING:
def61eca 490 switch (old_state) {
b754a32c 491 case NVME_CTRL_NEW:
3cec7f9d 492 case NVME_CTRL_RESETTING:
bb8d261e 493 changed = true;
df561f66 494 fallthrough;
bb8d261e
CH		 495 default:
496 break;
497 }
498 break;
499 case NVME_CTRL_DELETING:
500 switch (old_state) {
501 case NVME_CTRL_LIVE:
502 case NVME_CTRL_RESETTING:
ad6a0a52 503 case NVME_CTRL_CONNECTING:
bb8d261e 504 changed = true;
df561f66 505 fallthrough;
bb8d261e
CH		 506 default:
507 break;
508 }
509 break;
ecca390e
SG		 510 case NVME_CTRL_DELETING_NOIO:
511 switch (old_state) {
512 case NVME_CTRL_DELETING:
513 case NVME_CTRL_DEAD:
514 changed = true;
df561f66 515 fallthrough;
ecca390e
SG		 516 default:
517 break;
518 }
519 break;
0ff9d4e1
KB		 520 case NVME_CTRL_DEAD:
521 switch (old_state) {
522 case NVME_CTRL_DELETING:
523 changed = true;
df561f66 524 fallthrough;
0ff9d4e1
KB		 525 default:
526 break;
527 }
528 break;
bb8d261e
CH		 529 default:
530 break;
531 }
bb8d261e 532
c1ac9a4b 533 if (changed) {
bb8d261e 534 ctrl->state = new_state;
c1ac9a4b
KB		 535 wake_up_all(&ctrl->state_wq);
536 }
bb8d261e 537
0a72bbba 538 spin_unlock_irqrestore(&ctrl->lock, flags);
8c4dfea9
VG		 539 if (!changed)
540 return false;
541
542 if (ctrl->state == NVME_CTRL_LIVE) {
543 if (old_state == NVME_CTRL_CONNECTING)
544 nvme_stop_failfast_work(ctrl);
32acab31 545 nvme_kick_requeue_lists(ctrl);
8c4dfea9
VG		 546 } else if (ctrl->state == NVME_CTRL_CONNECTING &&
547 old_state == NVME_CTRL_RESETTING) {
548 nvme_start_failfast_work(ctrl);
549 }
bb8d261e
CH		 550 return changed;
551}
552EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
553
c1ac9a4b
KB		 554/*
555 * Returns true for sink states that can't ever transition back to live.
556 */
557static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
558{
559 switch (ctrl->state) {
560 case NVME_CTRL_NEW:
561 case NVME_CTRL_LIVE:
562 case NVME_CTRL_RESETTING:
563 case NVME_CTRL_CONNECTING:
564 return false;
565 case NVME_CTRL_DELETING:
ecca390e 566 case NVME_CTRL_DELETING_NOIO:
c1ac9a4b
KB		 567 case NVME_CTRL_DEAD:
568 return true;
569 default:
570 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
571 return true;
572 }
573}
574
575/*
576 * Waits for the controller state to be resetting, or returns false if it is
577 * not possible to ever transition to that state.
578 */
579bool nvme_wait_reset(struct nvme_ctrl *ctrl)
580{
581 wait_event(ctrl->state_wq,
582 nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
583 nvme_state_terminal(ctrl));
584 return ctrl->state == NVME_CTRL_RESETTING;
585}
586EXPORT_SYMBOL_GPL(nvme_wait_reset);
587
ed754e5d
CH		 588static void nvme_free_ns_head(struct kref *ref)
589{
590 struct nvme_ns_head *head =
591 container_of(ref, struct nvme_ns_head, ref);
592
32acab31 593 nvme_mpath_remove_disk(head);
8b850475 594 ida_free(&head->subsys->ns_ida, head->instance);
f5ad3991 595 cleanup_srcu_struct(&head->srcu);
12d9f070 596 nvme_put_subsystem(head->subsys);
ed754e5d
CH		 597 kfree(head);
598}
599
1496bd49 600bool nvme_tryget_ns_head(struct nvme_ns_head *head)
871ca3ef
CH		 601{
602 return kref_get_unless_zero(&head->ref);
603}
604
1496bd49 605void nvme_put_ns_head(struct nvme_ns_head *head)
ed754e5d
CH		 606{
607 kref_put(&head->ref, nvme_free_ns_head);
608}
609
1673f1f0
CH		 610static void nvme_free_ns(struct kref *kref)
611{
612 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
613
1673f1f0 614 put_disk(ns->disk);
ed754e5d 615 nvme_put_ns_head(ns->head);
075790eb 616 nvme_put_ctrl(ns->ctrl);
1673f1f0
CH		 617 kfree(ns);
618}
619
4c74d1f8
KJ		 620static inline bool nvme_get_ns(struct nvme_ns *ns)
621{
622 return kref_get_unless_zero(&ns->kref);
623}
624
24493b8b 625void nvme_put_ns(struct nvme_ns *ns)
1673f1f0
CH		 626{
627 kref_put(&ns->kref, nvme_free_ns);
628}
24493b8b 629EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
1673f1f0 630
bb06ec31
JS		 631static inline void nvme_clear_nvme_request(struct request *req)
632{
ae5e6886 633 nvme_req(req)->status = 0;
c03fd85d
CK		 634 nvme_req(req)->retries = 0;
635 nvme_req(req)->flags = 0;
636 req->rq_flags |= RQF_DONTPREP;
bb06ec31
JS		 637}
638
e559398f
CH		 639/* initialize a passthrough request */
640void nvme_init_request(struct request *req, struct nvme_command *cmd)
39dfe844 641{
0d2e7c84
CK		 642 if (req->q->queuedata)
643 req->timeout = NVME_IO_TIMEOUT;
644 else /* no queuedata implies admin queue */
dc96f938 645 req->timeout = NVME_ADMIN_TIMEOUT;
21d34711 646
f4b9e6c9
KB		 647 /* passthru commands should let the driver set the SGL flags */
648 cmd->common.flags &= ~NVME_CMD_SGL_ALL;
649
21d34711 650 req->cmd_flags |= REQ_FAILFAST_DRIVER;
be42a33b 651 if (req->mq_hctx->type == HCTX_TYPE_POLL)
6ce913fe 652 req->cmd_flags |= REQ_POLLED;
bb06ec31 653 nvme_clear_nvme_request(req);
f4b9e6c9 654 memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd));
39dfe844 655}
e559398f 656EXPORT_SYMBOL_GPL(nvme_init_request);
39dfe844 657
a9715744
TC		 658/*
659 * For something we're not in a state to send to the device the default action
660 * is to busy it and retry it after the controller state is recovered. However,
661 * if the controller is deleting or if anything is marked for failfast or
662 * nvme multipath it is immediately failed.
663 *
664 * Note: commands used to initialize the controller will be marked for failfast.
665 * Note: nvme cli/ioctl commands are marked for failfast.
666 */
667blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
668 struct request *rq)
669{
670 if (ctrl->state != NVME_CTRL_DELETING_NOIO &&
8b77fa6f 671 ctrl->state != NVME_CTRL_DELETING &&
a9715744
TC		 672 ctrl->state != NVME_CTRL_DEAD &&
673 !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
674 !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
675 return BLK_STS_RESOURCE;
676 return nvme_host_path_error(rq);
677}
678EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
679
680bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
681 bool queue_live)
682{
683 struct nvme_request *req = nvme_req(rq);
684
685 /*
686 * currently we have a problem sending passthru commands
687 * on the admin_q if the controller is not LIVE because we can't
688 * make sure that they are going out after the admin connect,
689 * controller enable and/or other commands in the initialization
690 * sequence. until the controller will be LIVE, fail with
691 * BLK_STS_RESOURCE so that they will be rescheduled.
692 */
693 if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
694 return false;
695
696 if (ctrl->ops->flags & NVME_F_FABRICS) {
697 /*
698 * Only allow commands on a live queue, except for the connect
699 * command, which is require to set the queue live in the
700 * appropinquate states.
701 */
702 switch (ctrl->state) {
703 case NVME_CTRL_CONNECTING:
704 if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
705 req->cmd->fabrics.fctype == nvme_fabrics_type_connect)
706 return true;
707 break;
708 default:
709 break;
710 case NVME_CTRL_DEAD:
711 return false;
712 }
713 }
714
715 return queue_live;
716}
717EXPORT_SYMBOL_GPL(__nvme_check_ready);
718
8093f7ca
ML		 719static inline void nvme_setup_flush(struct nvme_ns *ns,
720 struct nvme_command *cmnd)
721{
9c3d2929 722 memset(cmnd, 0, sizeof(*cmnd));
8093f7ca 723 cmnd->common.opcode = nvme_cmd_flush;
ed754e5d 724 cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
8093f7ca
ML		 725}
726
fc17b653 727static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
8093f7ca
ML		 728 struct nvme_command *cmnd)
729{
b35ba01e 730 unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
8093f7ca 731 struct nvme_dsm_range *range;
b35ba01e 732 struct bio *bio;
8093f7ca 733
530436c4
EH		 734 /*
735 * Some devices do not consider the DSM 'Number of Ranges' field when
736 * determining how much data to DMA. Always allocate memory for maximum
737 * number of segments to prevent device reading beyond end of buffer.
738 */
739 static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
740
741 range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
cb5b7262
JA		 742 if (!range) {
743 /*
744 * If we fail allocation our range, fallback to the controller
745 * discard page. If that's also busy, it's safe to return
746 * busy, as we know we can make progress once that's freed.
747 */
748 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
749 return BLK_STS_RESOURCE;
750
751 range = page_address(ns->ctrl->discard_page);
752 }
8093f7ca 753
b35ba01e 754 __rq_for_each_bio(bio, req) {
314d48dd 755 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
b35ba01e
CH		 756 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
757
8cb6af7b
KB		 758 if (n < segments) {
759 range[n].cattr = cpu_to_le32(0);
760 range[n].nlb = cpu_to_le32(nlb);
761 range[n].slba = cpu_to_le64(slba);
762 }
b35ba01e
CH		 763 n++;
764 }
765
766 if (WARN_ON_ONCE(n != segments)) {
cb5b7262
JA		 767 if (virt_to_page(range) == ns->ctrl->discard_page)
768 clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
769 else
770 kfree(range);
fc17b653 771 return BLK_STS_IOERR;
b35ba01e 772 }
8093f7ca 773
9c3d2929 774 memset(cmnd, 0, sizeof(*cmnd));
8093f7ca 775 cmnd->dsm.opcode = nvme_cmd_dsm;
ed754e5d 776 cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
f1dd03a8 777 cmnd->dsm.nr = cpu_to_le32(segments - 1);
8093f7ca
ML		 778 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
779
f9d03f96
CH		 780 req->special_vec.bv_page = virt_to_page(range);
781 req->special_vec.bv_offset = offset_in_page(range);
530436c4 782 req->special_vec.bv_len = alloc_size;
f9d03f96 783 req->rq_flags |= RQF_SPECIAL_PAYLOAD;
8093f7ca 784
fc17b653 785 return BLK_STS_OK;
8093f7ca 786}
8093f7ca 787
4020aad8
KB		 788static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
789 struct request *req)
790{
791 u32 upper, lower;
792 u64 ref48;
793
794 /* both rw and write zeroes share the same reftag format */
795 switch (ns->guard_type) {
796 case NVME_NVM_NS_16B_GUARD:
797 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
798 break;
799 case NVME_NVM_NS_64B_GUARD:
800 ref48 = ext_pi_ref_tag(req);
801 lower = lower_32_bits(ref48);
802 upper = upper_32_bits(ref48);
803
804 cmnd->rw.reftag = cpu_to_le32(lower);
805 cmnd->rw.cdw3 = cpu_to_le32(upper);
806 break;
807 default:
808 break;
809 }
810}
811
6e02318e
CK		 812static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
813 struct request *req, struct nvme_command *cmnd)
814{
9c3d2929
JA		 815 memset(cmnd, 0, sizeof(*cmnd));
816
6e02318e
CK		 817 if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
818 return nvme_setup_discard(ns, req, cmnd);
819
820 cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
821 cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
822 cmnd->write_zeroes.slba =
314d48dd 823 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
6e02318e
CK		 824 cmnd->write_zeroes.length =
825 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
00b33cf3
KJ		 826
827 if (nvme_ns_has_pi(ns)) {
aaeb7bb0 828 cmnd->write_zeroes.control = cpu_to_le16(NVME_RW_PRINFO_PRACT);
00b33cf3
KJ		 829
830 switch (ns->pi_type) {
831 case NVME_NS_DPS_PI_TYPE1:
832 case NVME_NS_DPS_PI_TYPE2:
4020aad8 833 nvme_set_ref_tag(ns, cmnd, req);
00b33cf3
KJ		 834 break;
835 }
836 }
837
6e02318e
CK		 838 return BLK_STS_OK;
839}
840
ebe6d874 841static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
240e6ee2
KB		 842 struct request *req, struct nvme_command *cmnd,
843 enum nvme_opcode op)
8093f7ca
ML		 844{
845 u16 control = 0;
846 u32 dsmgmt = 0;
847
848 if (req->cmd_flags & REQ_FUA)
849 control |= NVME_RW_FUA;
850 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
851 control |= NVME_RW_LR;
852
853 if (req->cmd_flags & REQ_RAHEAD)
854 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
855
240e6ee2 856 cmnd->rw.opcode = op;
a9a7e30f 857 cmnd->rw.flags = 0;
ed754e5d 858 cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
4020aad8
KB		 859 cmnd->rw.cdw2 = 0;
860 cmnd->rw.cdw3 = 0;
a9a7e30f 861 cmnd->rw.metadata = 0;
314d48dd 862 cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
8093f7ca 863 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
a9a7e30f
JA		 864 cmnd->rw.reftag = 0;
865 cmnd->rw.apptag = 0;
866 cmnd->rw.appmask = 0;
8093f7ca
ML		 867
868 if (ns->ms) {
715ea9e0
CH		 869 /*
870 * If formated with metadata, the block layer always provides a
871 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else
872 * we enable the PRACT bit for protection information or set the
873 * namespace capacity to zero to prevent any I/O.
874 */
875 if (!blk_integrity_rq(req)) {
876 if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
877 return BLK_STS_NOTSUPP;
878 control |= NVME_RW_PRINFO_PRACT;
879 }
880
8093f7ca
ML		 881 switch (ns->pi_type) {
882 case NVME_NS_DPS_PI_TYPE3:
883 control |= NVME_RW_PRINFO_PRCHK_GUARD;
884 break;
885 case NVME_NS_DPS_PI_TYPE1:
886 case NVME_NS_DPS_PI_TYPE2:
887 control |= NVME_RW_PRINFO_PRCHK_GUARD |
888 NVME_RW_PRINFO_PRCHK_REF;
240e6ee2
KB		 889 if (op == nvme_cmd_zone_append)
890 control |= NVME_RW_APPEND_PIREMAP;
4020aad8 891 nvme_set_ref_tag(ns, cmnd, req);
8093f7ca
ML		 892 break;
893 }
8093f7ca
ML		 894 }
895
896 cmnd->rw.control = cpu_to_le16(control);
897 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
ebe6d874 898 return 0;
8093f7ca
ML		 899}
900
f7f1fc36
MG		 901void nvme_cleanup_cmd(struct request *req)
902{
f7f1fc36 903 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
fc97e942 904 struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
cb5b7262 905
3973e15f 906 if (req->special_vec.bv_page == ctrl->discard_page)
fc97e942 907 clear_bit_unlock(0, &ctrl->discard_page_busy);
cb5b7262 908 else
3973e15f 909 kfree(bvec_virt(&req->special_vec));
f7f1fc36
MG		 910 }
911}
912EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
913
f4b9e6c9 914blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
8093f7ca 915{
f4b9e6c9 916 struct nvme_command *cmd = nvme_req(req)->cmd;
fc17b653 917 blk_status_t ret = BLK_STS_OK;
8093f7ca 918
9c3d2929 919 if (!(req->rq_flags & RQF_DONTPREP))
c03fd85d 920 nvme_clear_nvme_request(req);
987f699a 921
aebf526b
CH		 922 switch (req_op(req)) {
923 case REQ_OP_DRV_IN:
924 case REQ_OP_DRV_OUT:
f4b9e6c9 925 /* these are setup prior to execution in nvme_init_request() */
aebf526b
CH		 926 break;
927 case REQ_OP_FLUSH:
8093f7ca 928 nvme_setup_flush(ns, cmd);
aebf526b 929 break;
240e6ee2
KB		 930 case REQ_OP_ZONE_RESET_ALL:
931 case REQ_OP_ZONE_RESET:
932 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
933 break;
934 case REQ_OP_ZONE_OPEN:
935 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
936 break;
937 case REQ_OP_ZONE_CLOSE:
938 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
939 break;
940 case REQ_OP_ZONE_FINISH:
941 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
942 break;
e850fd16 943 case REQ_OP_WRITE_ZEROES:
6e02318e
CK		 944 ret = nvme_setup_write_zeroes(ns, req, cmd);
945 break;
aebf526b 946 case REQ_OP_DISCARD:
8093f7ca 947 ret = nvme_setup_discard(ns, req, cmd);
aebf526b
CH		 948 break;
949 case REQ_OP_READ:
240e6ee2
KB		 950 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
951 break;
aebf526b 952 case REQ_OP_WRITE:
240e6ee2
KB		 953 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
954 break;
955 case REQ_OP_ZONE_APPEND:
956 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
aebf526b
CH		 957 break;
958 default:
959 WARN_ON_ONCE(1);
fc17b653 960 return BLK_STS_IOERR;
aebf526b 961 }
8093f7ca 962
e7006de6 963 cmd->common.command_id = nvme_cid(req);
5d87eb94 964 trace_nvme_setup_cmd(req, cmd);
8093f7ca
ML		 965 return ret;
966}
967EXPORT_SYMBOL_GPL(nvme_setup_cmd);
968
ae5e6886
KB		 969/*
970 * Return values:
971 * 0: success
972 * >0: nvme controller's cqe status response
973 * <0: kernel error in lieu of controller response
974 */
ba326643 975static int nvme_execute_rq(struct request *rq, bool at_head)
ae5e6886
KB		 976{
977 blk_status_t status;
978
b84ba30b 979 status = blk_execute_rq(rq, at_head);
ae5e6886
KB		 980 if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
981 return -EINTR;
982 if (nvme_req(rq)->status)
983 return nvme_req(rq)->status;
984 return blk_status_to_errno(status);
985}
986
4160982e
CH		 987/*
988 * Returns 0 on success. If the result is negative, it's a Linux error code;
989 * if the result is positive, it's an NVM Express status code
990 */
991int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
d49187e9 992 union nvme_result *result, void *buffer, unsigned bufflen,
9a95e4ef 993 unsigned timeout, int qid, int at_head,
be42a33b 994 blk_mq_req_flags_t flags)
4160982e
CH		 995{
996 struct request *req;
997 int ret;
998
39dfe844 999 if (qid == NVME_QID_ANY)
e559398f 1000 req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
39dfe844 1001 else
e559398f
CH		 1002 req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
1003 qid ? qid - 1 : 0);
1004
4160982e
CH		 1005 if (IS_ERR(req))
1006 return PTR_ERR(req);
e559398f 1007 nvme_init_request(req, cmd);
4160982e 1008
0d2e7c84
CK		 1009 if (timeout)
1010 req->timeout = timeout;
4160982e 1011
21d34711
CH		 1012 if (buffer && bufflen) {
1013 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1014 if (ret)
1015 goto out;
4160982e
CH		 1016 }
1017
b42b6f44 1018 req->rq_flags |= RQF_QUIET;
ba326643 1019 ret = nvme_execute_rq(req, at_head);
ae5e6886 1020 if (result && ret >= 0)
d49187e9 1021 *result = nvme_req(req)->result;
4160982e
CH		 1022 out:
1023 blk_mq_free_request(req);
1024 return ret;
1025}
eb71f435 1026EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
4160982e
CH		 1027
1028int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1029 void *buffer, unsigned bufflen)
1030{
eb71f435 1031 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
be42a33b 1032 NVME_QID_ANY, 0, 0);
4160982e 1033}
576d55d6 1034EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
4160982e 1035
df21b6b1
LG		 1036static u32 nvme_known_admin_effects(u8 opcode)
1037{
1038 switch (opcode) {
1039 case nvme_admin_format_nvm:
75eb779e 1040 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_NCC |
df21b6b1
LG		 1041 NVME_CMD_EFFECTS_CSE_MASK;
1042 case nvme_admin_sanitize_nvm:
75eb779e 1043 return NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK;
df21b6b1
LG		 1044 default:
1045 break;
1046 }
1047 return 0;
1048}
1049
1050u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1051{
1052 u32 effects = 0;
1053
1054 if (ns) {
1055 if (ns->head->effects)
1056 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1057 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
ed4a854b
KB		 1058 dev_warn_once(ctrl->device,
1059 "IO command:%02x has unhandled effects:%08x\n",
1060 opcode, effects);
df21b6b1
LG		 1061 return 0;
1062 }
1063
1064 if (ctrl->effects)
1065 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1066 effects |= nvme_known_admin_effects(opcode);
1067
1068 return effects;
1069}
1070EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1071
1072static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1073 u8 opcode)
1074{
1075 u32 effects = nvme_command_effects(ctrl, ns, opcode);
1076
1077 /*
1078 * For simplicity, IO to all namespaces is quiesced even if the command
1079 * effects say only one namespace is affected.
1080 */
af0f446d 1081 if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
df21b6b1
LG		 1082 mutex_lock(&ctrl->scan_lock);
1083 mutex_lock(&ctrl->subsys->lock);
1084 nvme_mpath_start_freeze(ctrl->subsys);
1085 nvme_mpath_wait_freeze(ctrl->subsys);
1086 nvme_start_freeze(ctrl);
1087 nvme_wait_freeze(ctrl);
1088 }
1089 return effects;
1090}
1091
b58da2d2
TS		 1092static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects,
1093 struct nvme_command *cmd, int status)
df21b6b1 1094{
af0f446d 1095 if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
df21b6b1
LG		 1096 nvme_unfreeze(ctrl);
1097 nvme_mpath_unfreeze(ctrl->subsys);
1098 mutex_unlock(&ctrl->subsys->lock);
1099 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1100 mutex_unlock(&ctrl->scan_lock);
1101 }
1102 if (effects & NVME_CMD_EFFECTS_CCC)
f21c4769 1103 nvme_init_ctrl_finish(ctrl);
df21b6b1
LG		 1104 if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1105 nvme_queue_scan(ctrl);
1106 flush_work(&ctrl->scan_work);
1107 }
b58da2d2
TS		 1108
1109 switch (cmd->common.opcode) {
1110 case nvme_admin_set_features:
1111 switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1112 case NVME_FEAT_KATO:
1113 /*
1114 * Keep alive commands interval on the host should be
1115 * updated when KATO is modified by Set Features
1116 * commands.
1117 */
1118 if (!status)
1119 nvme_update_keep_alive(ctrl, cmd);
1120 break;
1121 default:
1122 break;
1123 }
1124 break;
1125 default:
1126 break;
1127 }
df21b6b1
LG		 1128}
1129
ae5e6886 1130int nvme_execute_passthru_rq(struct request *rq)
17365ae6
LG		 1131{
1132 struct nvme_command *cmd = nvme_req(rq)->cmd;
1133 struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1134 struct nvme_ns *ns = rq->q->queuedata;
17365ae6 1135 u32 effects;
ae5e6886 1136 int ret;
17365ae6
LG		 1137
1138 effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
ba326643 1139 ret = nvme_execute_rq(rq, false);
18479ddb 1140 if (effects) /* nothing to be done for zero cmd effects */
b58da2d2 1141 nvme_passthru_end(ctrl, effects, cmd, ret);
ae5e6886
KB		 1142
1143 return ret;
17365ae6
LG		 1144}
1145EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1146
a70b81bd
HR		 1147/*
1148 * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1149 *
1150 * The host should send Keep Alive commands at half of the Keep Alive Timeout
1151 * accounting for transport roundtrip times [..].
1152 */
1153static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
4160982e 1154{
a70b81bd 1155 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ / 2);
21d34711
CH		 1156}
1157
2a842aca 1158static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
038bd4cb
SG		 1159{
1160 struct nvme_ctrl *ctrl = rq->end_io_data;
86880d64
JS		 1161 unsigned long flags;
1162 bool startka = false;
038bd4cb
SG		 1163
1164 blk_mq_free_request(rq);
1165
2a842aca 1166 if (status) {
038bd4cb 1167 dev_err(ctrl->device,
2a842aca
CH		 1168 "failed nvme_keep_alive_end_io error=%d\n",
1169 status);
038bd4cb
SG		 1170 return;
1171 }
1172
6e3ca03e 1173 ctrl->comp_seen = false;
86880d64
JS		 1174 spin_lock_irqsave(&ctrl->lock, flags);
1175 if (ctrl->state == NVME_CTRL_LIVE ||
1176 ctrl->state == NVME_CTRL_CONNECTING)
1177 startka = true;
1178 spin_unlock_irqrestore(&ctrl->lock, flags);
1179 if (startka)
a70b81bd 1180 nvme_queue_keep_alive_work(ctrl);
038bd4cb
SG		 1181}
1182
038bd4cb
SG		 1183static void nvme_keep_alive_work(struct work_struct *work)
1184{
1185 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1186 struct nvme_ctrl, ka_work);
6e3ca03e 1187 bool comp_seen = ctrl->comp_seen;
06c3c336 1188 struct request *rq;
6e3ca03e
SG		 1189
1190 if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1191 dev_dbg(ctrl->device,
1192 "reschedule traffic based keep-alive timer\n");
1193 ctrl->comp_seen = false;
a70b81bd 1194 nvme_queue_keep_alive_work(ctrl);
6e3ca03e
SG		 1195 return;
1196 }
038bd4cb 1197
e559398f
CH		 1198 rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1199 BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
06c3c336 1200 if (IS_ERR(rq)) {
038bd4cb 1201 /* allocation failure, reset the controller */
985c5a32 1202 dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
39bdc590 1203 nvme_reset_ctrl(ctrl);
038bd4cb
SG		 1204 return;
1205 }
e559398f 1206 nvme_init_request(rq, &ctrl->ka_cmd);
06c3c336
CH		 1207
1208 rq->timeout = ctrl->kato * HZ;
1209 rq->end_io_data = ctrl;
b84ba30b 1210 blk_execute_rq_nowait(rq, false, nvme_keep_alive_end_io);
038bd4cb
SG		 1211}
1212
00b683db 1213static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
038bd4cb
SG		 1214{
1215 if (unlikely(ctrl->kato == 0))
1216 return;
1217
a70b81bd 1218 nvme_queue_keep_alive_work(ctrl);
038bd4cb 1219}
038bd4cb
SG		 1220
1221void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1222{
1223 if (unlikely(ctrl->kato == 0))
1224 return;
1225
1226 cancel_delayed_work_sync(&ctrl->ka_work);
1227}
1228EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1229
b58da2d2
TS		 1230static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1231 struct nvme_command *cmd)
1232{
1233 unsigned int new_kato =
1234 DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1235
1236 dev_info(ctrl->device,
1237 "keep alive interval updated from %u ms to %u ms\n",
1238 ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1239
1240 nvme_stop_keep_alive(ctrl);
1241 ctrl->kato = new_kato;
1242 nvme_start_keep_alive(ctrl);
1243}
1244
b9a5c3d4
CH		 1245/*
1246 * In NVMe 1.0 the CNS field was just a binary controller or namespace
1247 * flag, thus sending any new CNS opcodes has a big chance of not working.
1248 * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1249 * (but not for any later version).
1250 */
1251static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1252{
1253 if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1254 return ctrl->vs < NVME_VS(1, 2, 0);
1255 return ctrl->vs < NVME_VS(1, 1, 0);
1256}
1257
3f7f25a9 1258static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
21d34711
CH		 1259{
1260 struct nvme_command c = { };
1261 int error;
1262
1263 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1264 c.identify.opcode = nvme_admin_identify;
986994a2 1265 c.identify.cns = NVME_ID_CNS_CTRL;
21d34711
CH		 1266
1267 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1268 if (!*id)
1269 return -ENOMEM;
1270
1271 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1272 sizeof(struct nvme_id_ctrl));
1273 if (error)
1274 kfree(*id);
1275 return error;
1276}
1277
ad95a613 1278static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
71010c30 1279 struct nvme_ns_id_desc *cur, bool *csi_seen)
ad95a613
CK		 1280{
1281 const char *warn_str = "ctrl returned bogus length:";
1282 void *data = cur;
1283
1284 switch (cur->nidt) {
1285 case NVME_NIDT_EUI64:
1286 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1287 dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1288 warn_str, cur->nidl);
1289 return -1;
1290 }
00ff400e
CH		 1291 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1292 return NVME_NIDT_EUI64_LEN;
ad95a613
CK		 1293 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1294 return NVME_NIDT_EUI64_LEN;
1295 case NVME_NIDT_NGUID:
1296 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1297 dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1298 warn_str, cur->nidl);
1299 return -1;
1300 }
00ff400e
CH		 1301 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1302 return NVME_NIDT_NGUID_LEN;
ad95a613
CK		 1303 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1304 return NVME_NIDT_NGUID_LEN;
1305 case NVME_NIDT_UUID:
1306 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1307 dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1308 warn_str, cur->nidl);
1309 return -1;
1310 }
00ff400e
CH		 1311 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1312 return NVME_NIDT_UUID_LEN;
ad95a613
CK		 1313 uuid_copy(&ids->uuid, data + sizeof(*cur));
1314 return NVME_NIDT_UUID_LEN;
71010c30
NC		 1315 case NVME_NIDT_CSI:
1316 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1317 dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1318 warn_str, cur->nidl);
1319 return -1;
1320 }
1321 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1322 *csi_seen = true;
1323 return NVME_NIDT_CSI_LEN;
ad95a613
CK		 1324 default:
1325 /* Skip unknown types */
1326 return cur->nidl;
1327 }
1328}
1329
cdbff4f2 1330static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
002fab04 1331 struct nvme_ns_ids *ids)
3b22ba26
JT		 1332{
1333 struct nvme_command c = { };
71010c30
NC		 1334 bool csi_seen = false;
1335 int status, pos, len;
3b22ba26 1336 void *data;
3b22ba26 1337
8b7c0ff2
CH		 1338 if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1339 return 0;
5bedd3af
CH		 1340 if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1341 return 0;
1342
3b22ba26
JT		 1343 c.identify.opcode = nvme_admin_identify;
1344 c.identify.nsid = cpu_to_le32(nsid);
1345 c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1346
1347 data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1348 if (!data)
1349 return -ENOMEM;
1350
cdbff4f2 1351 status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
3b22ba26 1352 NVME_IDENTIFY_DATA_SIZE);
fb314eb0
CH		 1353 if (status) {
1354 dev_warn(ctrl->device,
aa9d7295
MI		 1355 "Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1356 nsid, status);
3b22ba26 1357 goto free_data;
fb314eb0 1358 }
3b22ba26
JT		 1359
1360 for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1361 struct nvme_ns_id_desc *cur = data + pos;
1362
1363 if (cur->nidl == 0)
1364 break;
1365
71010c30 1366 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
ad95a613 1367 if (len < 0)
71010c30 1368 break;
3b22ba26
JT		 1369
1370 len += sizeof(*cur);
1371 }
71010c30
NC		 1372
1373 if (nvme_multi_css(ctrl) && !csi_seen) {
1374 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1375 nsid);
1376 status = -EINVAL;
1377 }
1378
3b22ba26
JT		 1379free_data:
1380 kfree(data);
1381 return status;
1382}
1383
8b7c0ff2
CH		 1384static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1385 struct nvme_ns_ids *ids, struct nvme_id_ns **id)
21d34711
CH		 1386{
1387 struct nvme_command c = { };
1388 int error;
1389
1390 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
778f067c
MG		 1391 c.identify.opcode = nvme_admin_identify;
1392 c.identify.nsid = cpu_to_le32(nsid);
986994a2 1393 c.identify.cns = NVME_ID_CNS_NS;
21d34711 1394
331813f6
SG		 1395 *id = kmalloc(sizeof(**id), GFP_KERNEL);
1396 if (!*id)
1397 return -ENOMEM;
21d34711 1398
331813f6 1399 error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
cdbff4f2 1400 if (error) {
d0de579c 1401 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
b8b8cd01 1402 goto out_free_id;
cdbff4f2
CH		 1403 }
1404
d95c1f41 1405 error = NVME_SC_INVALID_NS | NVME_SC_DNR;
b8b8cd01
CH		 1406 if ((*id)->ncap == 0) /* namespace not allocated or attached */
1407 goto out_free_id;
8b7c0ff2 1408
00ff400e
CH		 1409
1410 if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1411 dev_info(ctrl->device,
1412 "Ignoring bogus Namespace Identifiers\n");
1413 } else {
1414 if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1415 !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1416 memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
1417 if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1418 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1419 memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
1420 }
8b7c0ff2 1421
b8b8cd01
CH		 1422 return 0;
1423
1424out_free_id:
1425 kfree(*id);
331813f6 1426 return error;
21d34711
CH		 1427}
1428
1a87ee65
KB		 1429static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1430 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
21d34711 1431{
15755854 1432 union nvme_result res = { 0 };
cc72c442 1433 struct nvme_command c = { };
1cb3cce5 1434 int ret;
21d34711 1435
1a87ee65 1436 c.features.opcode = op;
21d34711
CH		 1437 c.features.fid = cpu_to_le32(fid);
1438 c.features.dword11 = cpu_to_le32(dword11);
1439
d49187e9 1440 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
be42a33b 1441 buffer, buflen, 0, NVME_QID_ANY, 0, 0);
9b47f77a 1442 if (ret >= 0 && result)
d49187e9 1443 *result = le32_to_cpu(res.u32);
1cb3cce5 1444 return ret;
21d34711
CH		 1445}
1446
1a87ee65
KB		 1447int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1448 unsigned int dword11, void *buffer, size_t buflen,
1449 u32 *result)
1450{
1451 return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1452 buflen, result);
1453}
1454EXPORT_SYMBOL_GPL(nvme_set_features);
1455
1456int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1457 unsigned int dword11, void *buffer, size_t buflen,
1458 u32 *result)
1459{
1460 return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1461 buflen, result);
1462}
1463EXPORT_SYMBOL_GPL(nvme_get_features);
1464
9a0be7ab
CH		 1465int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1466{
1467 u32 q_count = (*count - 1) | ((*count - 1) << 16);
1468 u32 result;
1469 int status, nr_io_queues;
1470
1a6fe74d 1471 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
9a0be7ab 1472 &result);
f5fa90dc 1473 if (status < 0)
9a0be7ab
CH		 1474 return status;
1475
f5fa90dc
CH		 1476 /*
1477 * Degraded controllers might return an error when setting the queue
1478 * count. We still want to be able to bring them online and offer
1479 * access to the admin queue, as that might be only way to fix them up.
1480 */
1481 if (status > 0) {
f0425db0 1482 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
f5fa90dc
CH		 1483 *count = 0;
1484 } else {
1485 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1486 *count = min(*count, nr_io_queues);
1487 }
1488
9a0be7ab
CH		 1489 return 0;
1490}
576d55d6 1491EXPORT_SYMBOL_GPL(nvme_set_queue_count);
9a0be7ab 1492
c0561f82 1493#define NVME_AEN_SUPPORTED \
85f8a435
SG		 1494 (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1495 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
c0561f82
HR		 1496
1497static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1498{
fa441b71 1499 u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
c0561f82
HR		 1500 int status;
1501
fa441b71
WZ		 1502 if (!supported_aens)
1503 return;
1504
1505 status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1506 NULL, 0, &result);
c0561f82
HR		 1507 if (status)
1508 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
fa441b71 1509 supported_aens);
93da4023
SG		 1510
1511 queue_work(nvme_wq, &ctrl->async_event_work);
c0561f82
HR		 1512}
1513
f5b9a51d 1514static int nvme_ns_open(struct nvme_ns *ns)
c225b610 1515{
c225b610 1516
32acab31 1517 /* should never be called due to GENHD_FL_HIDDEN */
30897388 1518 if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
85088c4a 1519 goto fail;
4c74d1f8 1520 if (!nvme_get_ns(ns))
85088c4a
NC		 1521 goto fail;
1522 if (!try_module_get(ns->ctrl->ops->module))
1523 goto fail_put_ns;
1524
c6424a90 1525 return 0;
85088c4a
NC		 1526
1527fail_put_ns:
1528 nvme_put_ns(ns);
1529fail:
1530 return -ENXIO;
1673f1f0
CH		 1531}
1532
f5b9a51d 1533static void nvme_ns_release(struct nvme_ns *ns)
1673f1f0 1534{
85088c4a
NC		 1535
1536 module_put(ns->ctrl->ops->module);
1537 nvme_put_ns(ns);
1673f1f0
CH		 1538}
1539
f5b9a51d
CH		 1540static int nvme_open(struct block_device *bdev, fmode_t mode)
1541{
1542 return nvme_ns_open(bdev->bd_disk->private_data);
1543}
1544
1545static void nvme_release(struct gendisk *disk, fmode_t mode)
1546{
1547 nvme_ns_release(disk->private_data);
1548}
1549
1496bd49 1550int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1673f1f0
CH		 1551{
1552 /* some standard values */
1553 geo->heads = 1 << 6;
1554 geo->sectors = 1 << 5;
1555 geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1556 return 0;
1557}
1558
1559#ifdef CONFIG_BLK_DEV_INTEGRITY
4020aad8 1560static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
95093350 1561 u32 max_integrity_segments)
1673f1f0 1562{
cc72c442 1563 struct blk_integrity integrity = { };
1673f1f0 1564
4020aad8 1565 switch (ns->pi_type) {
1673f1f0 1566 case NVME_NS_DPS_PI_TYPE3:
4020aad8
KB		 1567 switch (ns->guard_type) {
1568 case NVME_NVM_NS_16B_GUARD:
1569 integrity.profile = &t10_pi_type3_crc;
1570 integrity.tag_size = sizeof(u16) + sizeof(u32);
1571 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1572 break;
1573 case NVME_NVM_NS_64B_GUARD:
1574 integrity.profile = &ext_pi_type3_crc64;
1575 integrity.tag_size = sizeof(u16) + 6;
1576 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1577 break;
1578 default:
1579 integrity.profile = NULL;
1580 break;
1581 }
1673f1f0
CH		 1582 break;
1583 case NVME_NS_DPS_PI_TYPE1:
1584 case NVME_NS_DPS_PI_TYPE2:
4020aad8
KB		 1585 switch (ns->guard_type) {
1586 case NVME_NVM_NS_16B_GUARD:
1587 integrity.profile = &t10_pi_type1_crc;
1588 integrity.tag_size = sizeof(u16);
1589 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1590 break;
1591 case NVME_NVM_NS_64B_GUARD:
1592 integrity.profile = &ext_pi_type1_crc64;
1593 integrity.tag_size = sizeof(u16);
1594 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1595 break;
1596 default:
1597 integrity.profile = NULL;
1598 break;
1599 }
1673f1f0
CH		 1600 break;
1601 default:
1602 integrity.profile = NULL;
1603 break;
1604 }
4020aad8
KB		 1605
1606 integrity.tuple_size = ns->ms;
39b7baa4 1607 blk_integrity_register(disk, &integrity);
95093350 1608 blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1673f1f0
CH		 1609}
1610#else
4020aad8 1611static void nvme_init_integrity(struct gendisk *disk, struct nvme_ns *ns,
95093350 1612 u32 max_integrity_segments)
1673f1f0
CH		 1613{
1614}
1615#endif /* CONFIG_BLK_DEV_INTEGRITY */
1616
26318571 1617static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1673f1f0 1618{
3831761e 1619 struct nvme_ctrl *ctrl = ns->ctrl;
26318571 1620 struct request_queue *queue = disk->queue;
30e5e929
CH		 1621 u32 size = queue_logical_block_size(queue);
1622
5befc7c2 1623 if (ctrl->max_discard_sectors == 0) {
3831761e
JA		 1624 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1625 return;
1626 }
1627
b35ba01e
CH		 1628 BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1629 NVME_DSM_MAX_RANGES);
1630
b224f613 1631 queue->limits.discard_alignment = 0;
30e5e929 1632 queue->limits.discard_granularity = size;
f5d11840 1633
3831761e
JA		 1634 /* If discard is already enabled, don't reset queue limits */
1635 if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1636 return;
1637
5befc7c2
KB		 1638 blk_queue_max_discard_sectors(queue, ctrl->max_discard_sectors);
1639 blk_queue_max_discard_segments(queue, ctrl->max_discard_segments);
e850fd16
CH		 1640
1641 if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
30e5e929 1642 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1673f1f0
CH		 1643}
1644
002fab04
CH		 1645static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1646{
1647 return uuid_equal(&a->uuid, &b->uuid) &&
1648 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
71010c30
NC		 1649 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1650 a->csi == b->csi;
002fab04
CH		 1651}
1652
4020aad8 1653static int nvme_init_ms(struct nvme_ns *ns, struct nvme_id_ns *id)
d4609ea8 1654{
4020aad8
KB		 1655 bool first = id->dps & NVME_NS_DPS_PI_FIRST;
1656 unsigned lbaf = nvme_lbaf_index(id->flbas);
d4609ea8 1657 struct nvme_ctrl *ctrl = ns->ctrl;
4020aad8
KB		 1658 struct nvme_command c = { };
1659 struct nvme_id_ns_nvm *nvm;
1660 int ret = 0;
1661 u32 elbaf;
1662
1663 ns->pi_size = 0;
1664 ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1665 if (!(ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1666 ns->pi_size = sizeof(struct t10_pi_tuple);
1667 ns->guard_type = NVME_NVM_NS_16B_GUARD;
1668 goto set_pi;
1669 }
d4609ea8 1670
4020aad8
KB		 1671 nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1672 if (!nvm)
1673 return -ENOMEM;
d4609ea8 1674
4020aad8
KB		 1675 c.identify.opcode = nvme_admin_identify;
1676 c.identify.nsid = cpu_to_le32(ns->head->ns_id);
1677 c.identify.cns = NVME_ID_CNS_CS_NS;
1678 c.identify.csi = NVME_CSI_NVM;
1679
1680 ret = nvme_submit_sync_cmd(ns->ctrl->admin_q, &c, nvm, sizeof(*nvm));
1681 if (ret)
1682 goto free_data;
1683
1684 elbaf = le32_to_cpu(nvm->elbaf[lbaf]);
1685
1686 /* no support for storage tag formats right now */
1687 if (nvme_elbaf_sts(elbaf))
1688 goto free_data;
1689
1690 ns->guard_type = nvme_elbaf_guard_type(elbaf);
1691 switch (ns->guard_type) {
1692 case NVME_NVM_NS_64B_GUARD:
1693 ns->pi_size = sizeof(struct crc64_pi_tuple);
1694 break;
1695 case NVME_NVM_NS_16B_GUARD:
1696 ns->pi_size = sizeof(struct t10_pi_tuple);
1697 break;
1698 default:
1699 break;
1700 }
1701
1702free_data:
1703 kfree(nvm);
1704set_pi:
1705 if (ns->pi_size && (first || ns->ms == ns->pi_size))
d4609ea8
CH		 1706 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1707 else
1708 ns->pi_type = 0;
1709
4020aad8
KB		 1710 return ret;
1711}
1712
1713static void nvme_configure_metadata(struct nvme_ns *ns, struct nvme_id_ns *id)
1714{
1715 struct nvme_ctrl *ctrl = ns->ctrl;
1716
1717 if (nvme_init_ms(ns, id))
1718 return;
1719
d4609ea8
CH		 1720 ns->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1721 if (!ns->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
363f6368
CH		 1722 return;
1723
d4609ea8
CH		 1724 if (ctrl->ops->flags & NVME_F_FABRICS) {
1725 /*
1726 * The NVMe over Fabrics specification only supports metadata as
1727 * part of the extended data LBA. We rely on HCA/HBA support to
1728 * remap the separate metadata buffer from the block layer.
1729 */
1730 if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
363f6368 1731 return;
d39ad2a4
KB		 1732
1733 ns->features |= NVME_NS_EXT_LBAS;
1734
1735 /*
1736 * The current fabrics transport drivers support namespace
1737 * metadata formats only if nvme_ns_has_pi() returns true.
1738 * Suppress support for all other formats so the namespace will
1739 * have a 0 capacity and not be usable through the block stack.
1740 *
1741 * Note, this check will need to be modified if any drivers
1742 * gain the ability to use other metadata formats.
1743 */
1744 if (ctrl->max_integrity_segments && nvme_ns_has_pi(ns))
1745 ns->features |= NVME_NS_METADATA_SUPPORTED;
d4609ea8
CH		 1746 } else {
1747 /*
1748 * For PCIe controllers, we can't easily remap the separate
1749 * metadata buffer from the block layer and thus require a
1750 * separate metadata buffer for block layer metadata/PI support.
1751 * We allow extended LBAs for the passthrough interface, though.
1752 */
1753 if (id->flbas & NVME_NS_FLBAS_META_EXT)
1754 ns->features |= NVME_NS_EXT_LBAS;
1755 else
1756 ns->features |= NVME_NS_METADATA_SUPPORTED;
1757 }
d4609ea8
CH		 1758}
1759
658d9f7c
CH		 1760static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1761 struct request_queue *q)
1762{
c4485252 1763 bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
658d9f7c
CH		 1764
1765 if (ctrl->max_hw_sectors) {
1766 u32 max_segments =
1767 (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
1768
1769 max_segments = min_not_zero(max_segments, ctrl->max_segments);
1770 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1771 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1772 }
1773 blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
1774 blk_queue_dma_alignment(q, 7);
658d9f7c
CH		 1775 blk_queue_write_cache(q, vwc, vwc);
1776}
1777
24b0b58c
CH		 1778static void nvme_update_disk_info(struct gendisk *disk,
1779 struct nvme_ns *ns, struct nvme_id_ns *id)
1780{
e08f2ae8 1781 sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
cee160fd 1782 unsigned short bs = 1 << ns->lba_shift;
68ab60ca 1783 u32 atomic_bs, phys_bs, io_opt = 0;
24b0b58c 1784
13f0b26b
CH		 1785 /*
1786 * The block layer can't support LBA sizes larger than the page size
1787 * yet, so catch this early and don't allow block I/O.
1788 */
01fa0174 1789 if (ns->lba_shift > PAGE_SHIFT) {
13f0b26b 1790 capacity = 0;
01fa0174
SG		 1791 bs = (1 << 9);
1792 }
f9d5f457 1793
24b0b58c
CH		 1794 blk_integrity_unregister(disk);
1795
68ab60ca 1796 atomic_bs = phys_bs = bs;
81adb863
BVA		 1797 if (id->nabo == 0) {
1798 /*
1799 * Bit 1 indicates whether NAWUPF is defined for this namespace
1800 * and whether it should be used instead of AWUPF. If NAWUPF ==
1801 * 0 then AWUPF must be used instead.
1802 */
92decf11 1803 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
81adb863
BVA		 1804 atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1805 else
1806 atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
81adb863 1807 }
31fdad7b 1808
92decf11 1809 if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
81adb863 1810 /* NPWG = Namespace Preferred Write Granularity */
31fdad7b 1811 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
81adb863 1812 /* NOWS = Namespace Optimal Write Size */
31fdad7b 1813 io_opt = bs * (1 + le16_to_cpu(id->nows));
81adb863
BVA		 1814 }
1815
cee160fd 1816 blk_queue_logical_block_size(disk->queue, bs);
81adb863
BVA		 1817 /*
1818 * Linux filesystems assume writing a single physical block is
1819 * an atomic operation. Hence limit the physical block size to the
1820 * value of the Atomic Write Unit Power Fail parameter.
1821 */
1822 blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1823 blk_queue_io_min(disk->queue, phys_bs);
1824 blk_queue_io_opt(disk->queue, io_opt);
cee160fd 1825
b29f8485
MG		 1826 /*
1827 * Register a metadata profile for PI, or the plain non-integrity NVMe
1828 * metadata masquerading as Type 0 if supported, otherwise reject block
1829 * I/O to namespaces with metadata except when the namespace supports
1830 * PI, as it can strip/insert in that case.
1831 */
1832 if (ns->ms) {
1833 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1834 (ns->features & NVME_NS_METADATA_SUPPORTED))
4020aad8 1835 nvme_init_integrity(disk, ns,
95093350 1836 ns->ctrl->max_integrity_segments);
b29f8485
MG		 1837 else if (!nvme_ns_has_pi(ns))
1838 capacity = 0;
1839 }
1840
449f4ec9 1841 set_capacity_and_notify(disk, capacity);
b1aafb35 1842
26318571 1843 nvme_config_discard(disk, ns);
5befc7c2
KB		 1844 blk_queue_max_write_zeroes_sectors(disk->queue,
1845 ns->ctrl->max_zeroes_sectors);
24b0b58c
CH		 1846}
1847
e83d776f
KB		 1848static inline bool nvme_first_scan(struct gendisk *disk)
1849{
1850 /* nvme_alloc_ns() scans the disk prior to adding it */
50b4aecf 1851 return !disk_live(disk);
e83d776f
KB		 1852}
1853
1854static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id)
1855{
1856 struct nvme_ctrl *ctrl = ns->ctrl;
1857 u32 iob;
1858
1859 if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
1860 is_power_of_2(ctrl->max_hw_sectors))
1861 iob = ctrl->max_hw_sectors;
1862 else
1863 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
1864
1865 if (!iob)
1866 return;
1867
1868 if (!is_power_of_2(iob)) {
1869 if (nvme_first_scan(ns->disk))
1870 pr_warn("%s: ignoring unaligned IO boundary:%u\n",
1871 ns->disk->disk_name, iob);
1872 return;
1873 }
1874
1875 if (blk_queue_is_zoned(ns->disk->queue)) {
1876 if (nvme_first_scan(ns->disk))
1877 pr_warn("%s: ignoring zoned namespace IO boundary\n",
1878 ns->disk->disk_name);
1879 return;
1880 }
1881
1882 blk_queue_chunk_sectors(ns->queue, iob);
1883}
1884
81382f17 1885static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_id_ns *id)
ac81bfa9 1886{
4020aad8 1887 unsigned lbaf = nvme_lbaf_index(id->flbas);
240e6ee2 1888 int ret;
1673f1f0 1889
f9d5f457 1890 blk_mq_freeze_queue(ns->disk->queue);
240e6ee2 1891 ns->lba_shift = id->lbaf[lbaf].ds;
8b7c0ff2 1892 nvme_set_queue_limits(ns->ctrl, ns->queue);
38adf94e 1893
363f6368 1894 nvme_configure_metadata(ns, id);
73d90386
DLM		 1895 nvme_set_chunk_sectors(ns, id);
1896 nvme_update_disk_info(ns->disk, ns, id);
1897
8b7c0ff2 1898 if (ns->head->ids.csi == NVME_CSI_ZNS) {
d525c3c0 1899 ret = nvme_update_zone_info(ns, lbaf);
8b7c0ff2 1900 if (ret)
f9d5f457 1901 goto out_unfreeze;
71010c30
NC		 1902 }
1903
726be2c7
PR		 1904 set_disk_ro(ns->disk, (id->nsattr & NVME_NS_ATTR_RO) ||
1905 test_bit(NVME_NS_FORCE_RO, &ns->flags));
e7d65803 1906 set_bit(NVME_NS_READY, &ns->flags);
f9d5f457 1907 blk_mq_unfreeze_queue(ns->disk->queue);
1673f1f0 1908
3a9967ba
CH		 1909 if (blk_queue_is_zoned(ns->queue)) {
1910 ret = nvme_revalidate_zones(ns);
8685699c 1911 if (ret && !nvme_first_scan(ns->disk))
602e57c9 1912 return ret;
b29f8485
MG		 1913 }
1914
30897388 1915 if (nvme_ns_head_multipath(ns->head)) {
f9d5f457 1916 blk_mq_freeze_queue(ns->head->disk->queue);
32acab31 1917 nvme_update_disk_info(ns->head->disk, ns, id);
726be2c7
PR		 1918 set_disk_ro(ns->head->disk,
1919 (id->nsattr & NVME_NS_ATTR_RO) ||
1920 test_bit(NVME_NS_FORCE_RO, &ns->flags));
e7d65803 1921 nvme_mpath_revalidate_paths(ns);
b9b1a5d7
CH		 1922 blk_stack_limits(&ns->head->disk->queue->limits,
1923 &ns->queue->limits, 0);
471aa704 1924 disk_update_readahead(ns->head->disk);
f9d5f457 1925 blk_mq_unfreeze_queue(ns->head->disk->queue);
8f676b85 1926 }
33cfdc2a 1927 return 0;
ac81bfa9 1928
f9d5f457 1929out_unfreeze:
a9e0e6bc
CH		 1930 /*
1931 * If probing fails due an unsupported feature, hide the block device,
1932 * but still allow other access.
1933 */
1934 if (ret == -ENODEV) {
1935 ns->disk->flags |= GENHD_FL_HIDDEN;
602e57c9 1936 set_bit(NVME_NS_READY, &ns->flags);
a9e0e6bc
CH		 1937 ret = 0;
1938 }
602e57c9 1939 blk_mq_unfreeze_queue(ns->disk->queue);
240e6ee2
KB		 1940 return ret;
1941}
1942
1673f1f0
CH		 1943static char nvme_pr_type(enum pr_type type)
1944{
1945 switch (type) {
1946 case PR_WRITE_EXCLUSIVE:
1947 return 1;
1948 case PR_EXCLUSIVE_ACCESS:
1949 return 2;
1950 case PR_WRITE_EXCLUSIVE_REG_ONLY:
1951 return 3;
1952 case PR_EXCLUSIVE_ACCESS_REG_ONLY:
1953 return 4;
1954 case PR_WRITE_EXCLUSIVE_ALL_REGS:
1955 return 5;
1956 case PR_EXCLUSIVE_ACCESS_ALL_REGS:
1957 return 6;
1958 default:
1959 return 0;
1960 }
50ab19d8 1961}
1673f1f0 1962
f1cf35e1
CH		 1963static int nvme_send_ns_head_pr_command(struct block_device *bdev,
1964 struct nvme_command *c, u8 data[16])
1965{
1966 struct nvme_ns_head *head = bdev->bd_disk->private_data;
1967 int srcu_idx = srcu_read_lock(&head->srcu);
1968 struct nvme_ns *ns = nvme_find_path(head);
1969 int ret = -EWOULDBLOCK;
1970
1971 if (ns) {
1972 c->common.nsid = cpu_to_le32(ns->head->ns_id);
1973 ret = nvme_submit_sync_cmd(ns->queue, c, data, 16);
1974 }
1975 srcu_read_unlock(&head->srcu, srcu_idx);
1976 return ret;
1977}
1978
1979static int nvme_send_ns_pr_command(struct nvme_ns *ns, struct nvme_command *c,
1980 u8 data[16])
1981{
1982 c->common.nsid = cpu_to_le32(ns->head->ns_id);
1983 return nvme_submit_sync_cmd(ns->queue, c, data, 16);
1984}
1985
1673f1f0
CH		 1986static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
1987 u64 key, u64 sa_key, u8 op)
1988{
cc72c442 1989 struct nvme_command c = { };
1673f1f0
CH		 1990 u8 data[16] = { 0, };
1991
1992 put_unaligned_le64(key, &data[0]);
1993 put_unaligned_le64(sa_key, &data[8]);
1994
1673f1f0 1995 c.common.opcode = op;
b7c8f366 1996 c.common.cdw10 = cpu_to_le32(cdw10);
1673f1f0 1997
f1cf35e1
CH		 1998 if (IS_ENABLED(CONFIG_NVME_MULTIPATH) &&
1999 bdev->bd_disk->fops == &nvme_ns_head_ops)
2000 return nvme_send_ns_head_pr_command(bdev, &c, data);
2001 return nvme_send_ns_pr_command(bdev->bd_disk->private_data, &c, data);
1673f1f0
CH		 2002}
2003
2004static int nvme_pr_register(struct block_device *bdev, u64 old,
2005 u64 new, unsigned flags)
2006{
2007 u32 cdw10;
2008
2009 if (flags & ~PR_FL_IGNORE_KEY)
2010 return -EOPNOTSUPP;
2011
2012 cdw10 = old ? 2 : 0;
2013 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2014 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2015 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2016}
2017
2018static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2019 enum pr_type type, unsigned flags)
2020{
2021 u32 cdw10;
2022
2023 if (flags & ~PR_FL_IGNORE_KEY)
2024 return -EOPNOTSUPP;
2025
2026 cdw10 = nvme_pr_type(type) << 8;
2027 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2028 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2029}
2030
2031static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2032 enum pr_type type, bool abort)
2033{
e9a9853c 2034 u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
f1c772d5 2035
1673f1f0
CH		 2036 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2037}
2038
2039static int nvme_pr_clear(struct block_device *bdev, u64 key)
2040{
8c0b3915 2041 u32 cdw10 = 1 | (key ? 1 << 3 : 0);
f1c772d5 2042
1673f1f0
CH		 2043 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
2044}
2045
2046static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2047{
e9a9853c 2048 u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
f1c772d5 2049
1673f1f0
CH		 2050 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2051}
2052
1496bd49 2053const struct pr_ops nvme_pr_ops = {
1673f1f0
CH		 2054 .pr_register = nvme_pr_register,
2055 .pr_reserve = nvme_pr_reserve,
2056 .pr_release = nvme_pr_release,
2057 .pr_preempt = nvme_pr_preempt,
2058 .pr_clear = nvme_pr_clear,
2059};
2060
a98e58e5 2061#ifdef CONFIG_BLK_SED_OPAL
4f1244c8
CH		 2062int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2063 bool send)
a98e58e5 2064{
4f1244c8 2065 struct nvme_ctrl *ctrl = data;
cc72c442 2066 struct nvme_command cmd = { };
a98e58e5 2067
a98e58e5
SB		 2068 if (send)
2069 cmd.common.opcode = nvme_admin_security_send;
2070 else
2071 cmd.common.opcode = nvme_admin_security_recv;
a98e58e5 2072 cmd.common.nsid = 0;
b7c8f366
CK		 2073 cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2074 cmd.common.cdw11 = cpu_to_le32(len);
a98e58e5 2075
dc96f938 2076 return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len, 0,
be42a33b 2077 NVME_QID_ANY, 1, 0);
a98e58e5
SB		 2078}
2079EXPORT_SYMBOL_GPL(nvme_sec_submit);
2080#endif /* CONFIG_BLK_SED_OPAL */
2081
8b4fb0f9
CH		 2082#ifdef CONFIG_BLK_DEV_ZONED
2083static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2084 unsigned int nr_zones, report_zones_cb cb, void *data)
2085{
2086 return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2087 data);
2088}
2089#else
2090#define nvme_report_zones NULL
2091#endif /* CONFIG_BLK_DEV_ZONED */
2092
ba4fb320 2093static const struct block_device_operations nvme_bdev_ops = {
1673f1f0
CH		 2094 .owner = THIS_MODULE,
2095 .ioctl = nvme_ioctl,
1673f1f0
CH		 2096 .open = nvme_open,
2097 .release = nvme_release,
2098 .getgeo = nvme_getgeo,
240e6ee2 2099 .report_zones = nvme_report_zones,
1673f1f0
CH		 2100 .pr_ops = &nvme_pr_ops,
2101};
2102
5fd4ce1b
CH		 2103static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2104{
2105 unsigned long timeout =
2106 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2107 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2108 int ret;
2109
2110 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
0df1e4f5
KB		 2111 if (csts == ~0)
2112 return -ENODEV;
5fd4ce1b
CH		 2113 if ((csts & NVME_CSTS_RDY) == bit)
2114 break;
2115
3e98c244 2116 usleep_range(1000, 2000);
5fd4ce1b
CH		 2117 if (fatal_signal_pending(current))
2118 return -EINTR;
2119 if (time_after(jiffies, timeout)) {
1b3c47c1 2120 dev_err(ctrl->device,
94d2e705
RG		 2121 "Device not ready; aborting %s, CSTS=0x%x\n",
2122 enabled ? "initialisation" : "reset", csts);
5fd4ce1b
CH		 2123 return -ENODEV;
2124 }
2125 }
2126
2127 return ret;
2128}
2129
2130/*
2131 * If the device has been passed off to us in an enabled state, just clear
2132 * the enabled bit. The spec says we should set the 'shutdown notification
2133 * bits', but doing so may cause the device to complete commands to the
2134 * admin queue ... and we don't know what memory that might be pointing at!
2135 */
b5b05048 2136int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
5fd4ce1b
CH		 2137{
2138 int ret;
2139
2140 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2141 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2142
2143 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2144 if (ret)
2145 return ret;
54adc010 2146
b5a10c5f 2147 if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
54adc010
GP		 2148 msleep(NVME_QUIRK_DELAY_AMOUNT);
2149
b5b05048 2150 return nvme_wait_ready(ctrl, ctrl->cap, false);
5fd4ce1b 2151}
576d55d6 2152EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
5fd4ce1b 2153
c0f2f45b 2154int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
5fd4ce1b 2155{
6c3c05b0 2156 unsigned dev_page_min;
5fd4ce1b
CH		 2157 int ret;
2158
c0f2f45b
SG		 2159 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2160 if (ret) {
2161 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2162 return ret;
2163 }
2164 dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2165
6c3c05b0 2166 if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
1b3c47c1 2167 dev_err(ctrl->device,
5fd4ce1b 2168 "Minimum device page size %u too large for host (%u)\n",
6c3c05b0 2169 1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
5fd4ce1b
CH		 2170 return -ENODEV;
2171 }
2172
71010c30
NC		 2173 if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2174 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2175 else
2176 ctrl->ctrl_config = NVME_CC_CSS_NVM;
6c3c05b0 2177 ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
60b43f62 2178 ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
5fd4ce1b
CH		 2179 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2180 ctrl->ctrl_config |= NVME_CC_ENABLE;
2181
2182 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2183 if (ret)
2184 return ret;
c0f2f45b 2185 return nvme_wait_ready(ctrl, ctrl->cap, true);
5fd4ce1b 2186}
576d55d6 2187EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
5fd4ce1b
CH		 2188
2189int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2190{
07fbd32a 2191 unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
5fd4ce1b
CH		 2192 u32 csts;
2193 int ret;
2194
2195 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2196 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2197
2198 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2199 if (ret)
2200 return ret;
2201
2202 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2203 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2204 break;
2205
2206 msleep(100);
2207 if (fatal_signal_pending(current))
2208 return -EINTR;
2209 if (time_after(jiffies, timeout)) {
1b3c47c1 2210 dev_err(ctrl->device,
5fd4ce1b
CH		 2211 "Device shutdown incomplete; abort shutdown\n");
2212 return -ENODEV;
2213 }
2214 }
2215
2216 return ret;
2217}
576d55d6 2218EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
5fd4ce1b 2219
dbf86b39
JD		 2220static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2221{
2222 __le64 ts;
2223 int ret;
2224
2225 if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2226 return 0;
2227
2228 ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2229 ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2230 NULL);
2231 if (ret)
2232 dev_warn_once(ctrl->device,
2233 "could not set timestamp (%d)\n", ret);
2234 return ret;
2235}
2236
4020aad8 2237static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
49cd84b6
KB		 2238{
2239 struct nvme_feat_host_behavior *host;
4020aad8 2240 u8 acre = 0, lbafee = 0;
49cd84b6
KB		 2241 int ret;
2242
2243 /* Don't bother enabling the feature if retry delay is not reported */
4020aad8
KB		 2244 if (ctrl->crdt[0])
2245 acre = NVME_ENABLE_ACRE;
2246 if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2247 lbafee = NVME_ENABLE_LBAFEE;
2248
2249 if (!acre && !lbafee)
49cd84b6
KB		 2250 return 0;
2251
2252 host = kzalloc(sizeof(*host), GFP_KERNEL);
2253 if (!host)
2254 return 0;
2255
4020aad8
KB		 2256 host->acre = acre;
2257 host->lbafee = lbafee;
49cd84b6
KB		 2258 ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2259 host, sizeof(*host), NULL);
2260 kfree(host);
2261 return ret;
2262}
2263
ebd8a93a
AB		 2264/*
2265 * The function checks whether the given total (exlat + enlat) latency of
2266 * a power state allows the latter to be used as an APST transition target.
2267 * It does so by comparing the latency to the primary and secondary latency
2268 * tolerances defined by module params. If there's a match, the corresponding
2269 * timeout value is returned and the matching tolerance index (1 or 2) is
2270 * reported.
2271 */
2272static bool nvme_apst_get_transition_time(u64 total_latency,
2273 u64 *transition_time, unsigned *last_index)
2274{
2275 if (total_latency <= apst_primary_latency_tol_us) {
2276 if (*last_index == 1)
2277 return false;
2278 *last_index = 1;
2279 *transition_time = apst_primary_timeout_ms;
2280 return true;
2281 }
2282 if (apst_secondary_timeout_ms &&
2283 total_latency <= apst_secondary_latency_tol_us) {
2284 if (*last_index <= 2)
2285 return false;
2286 *last_index = 2;
2287 *transition_time = apst_secondary_timeout_ms;
2288 return true;
2289 }
2290 return false;
2291}
2292
60df5de9
CH		 2293/*
2294 * APST (Autonomous Power State Transition) lets us program a table of power
2295 * state transitions that the controller will perform automatically.
ebd8a93a
AB		 2296 *
2297 * Depending on module params, one of the two supported techniques will be used:
2298 *
2299 * - If the parameters provide explicit timeouts and tolerances, they will be
2300 * used to build a table with up to 2 non-operational states to transition to.
2301 * The default parameter values were selected based on the values used by
2302 * Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2303 * regeneration of the APST table in the event of switching between external
2304 * and battery power, the timeouts and tolerances reflect a compromise
2305 * between values used by Microsoft for AC and battery scenarios.
2306 * - If not, we'll configure the table with a simple heuristic: we are willing
2307 * to spend at most 2% of the time transitioning between power states.
2308 * Therefore, when running in any given state, we will enter the next
2309 * lower-power non-operational state after waiting 50 * (enlat + exlat)
2310 * microseconds, as long as that state's exit latency is under the requested
2311 * maximum latency.
60df5de9
CH		 2312 *
2313 * We will not autonomously enter any non-operational state for which the total
2314 * latency exceeds ps_max_latency_us.
2315 *
2316 * Users can set ps_max_latency_us to zero to turn off APST.
2317 */
634b8325 2318static int nvme_configure_apst(struct nvme_ctrl *ctrl)
c5552fde 2319{
c5552fde 2320 struct nvme_feat_auto_pst *table;
60df5de9 2321 unsigned apste = 0;
fb0dc399 2322 u64 max_lat_us = 0;
60df5de9 2323 __le64 target = 0;
fb0dc399 2324 int max_ps = -1;
60df5de9 2325 int state;
c5552fde 2326 int ret;
ebd8a93a 2327 unsigned last_lt_index = UINT_MAX;
c5552fde
AL		 2328
2329 /*
2330 * If APST isn't supported or if we haven't been initialized yet,
2331 * then don't do anything.
2332 */
2333 if (!ctrl->apsta)
634b8325 2334 return 0;
c5552fde
AL		 2335
2336 if (ctrl->npss > 31) {
2337 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
634b8325 2338 return 0;
c5552fde
AL		 2339 }
2340
2341 table = kzalloc(sizeof(*table), GFP_KERNEL);
2342 if (!table)
634b8325 2343 return 0;
c5552fde 2344
76a5af84 2345 if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
c5552fde 2346 /* Turn off APST. */
fb0dc399 2347 dev_dbg(ctrl->device, "APST disabled\n");
60df5de9
CH		 2348 goto done;
2349 }
c5552fde 2350
60df5de9
CH		 2351 /*
2352 * Walk through all states from lowest- to highest-power.
2353 * According to the spec, lower-numbered states use more power. NPSS,
2354 * despite the name, is the index of the lowest-power state, not the
2355 * number of states.
2356 */
2357 for (state = (int)ctrl->npss; state >= 0; state--) {
2358 u64 total_latency_us, exit_latency_us, transition_ms;
da87591b 2359
60df5de9
CH		 2360 if (target)
2361 table->entries[state] = target;
c5552fde 2362
c5552fde 2363 /*
60df5de9
CH		 2364 * Don't allow transitions to the deepest state if it's quirked
2365 * off.
c5552fde 2366 */
60df5de9
CH		 2367 if (state == ctrl->npss &&
2368 (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2369 continue;
fb0dc399 2370
60df5de9
CH		 2371 /*
2372 * Is this state a useful non-operational state for higher-power
2373 * states to autonomously transition to?
2374 */
2375 if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2376 continue;
fb0dc399 2377
60df5de9
CH		 2378 exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2379 if (exit_latency_us > ctrl->ps_max_latency_us)
2380 continue;
c5552fde 2381
60df5de9
CH		 2382 total_latency_us = exit_latency_us +
2383 le32_to_cpu(ctrl->psd[state].entry_lat);
fb0dc399 2384
60df5de9 2385 /*
ebd8a93a
AB		 2386 * This state is good. It can be used as the APST idle target
2387 * for higher power states.
60df5de9 2388 */
ebd8a93a
AB		 2389 if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2390 if (!nvme_apst_get_transition_time(total_latency_us,
2391 &transition_ms, &last_lt_index))
2392 continue;
2393 } else {
2394 transition_ms = total_latency_us + 19;
2395 do_div(transition_ms, 20);
2396 if (transition_ms > (1 << 24) - 1)
2397 transition_ms = (1 << 24) - 1;
2398 }
60df5de9
CH		 2399
2400 target = cpu_to_le64((state << 3) | (transition_ms << 8));
2401 if (max_ps == -1)
2402 max_ps = state;
2403 if (total_latency_us > max_lat_us)
2404 max_lat_us = total_latency_us;
c5552fde
AL		 2405 }
2406
60df5de9
CH		 2407 if (max_ps == -1)
2408 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2409 else
2410 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2411 max_ps, max_lat_us, (int)sizeof(*table), table);
2412 apste = 1;
2413
2414done:
c5552fde
AL		 2415 ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2416 table, sizeof(*table), NULL);
2417 if (ret)
2418 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
c5552fde 2419 kfree(table);
634b8325 2420 return ret;
c5552fde
AL		 2421}
2422
2423static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2424{
2425 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2426 u64 latency;
2427
2428 switch (val) {
2429 case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2430 case PM_QOS_LATENCY_ANY:
2431 latency = U64_MAX;
2432 break;
2433
2434 default:
2435 latency = val;
2436 }
2437
2438 if (ctrl->ps_max_latency_us != latency) {
2439 ctrl->ps_max_latency_us = latency;
53fe2a30
CH		 2440 if (ctrl->state == NVME_CTRL_LIVE)
2441 nvme_configure_apst(ctrl);
c5552fde
AL		 2442 }
2443}
2444
bd4da3ab
AL		 2445struct nvme_core_quirk_entry {
2446 /*
2447 * NVMe model and firmware strings are padded with spaces. For
2448 * simplicity, strings in the quirk table are padded with NULLs
2449 * instead.
2450 */
2451 u16 vid;
2452 const char *mn;
2453 const char *fr;
2454 unsigned long quirks;
2455};
2456
2457static const struct nvme_core_quirk_entry core_quirks[] = {
c5552fde 2458 {
be56945c
AL		 2459 /*
2460 * This Toshiba device seems to die using any APST states. See:
2461 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2462 */
2463 .vid = 0x1179,
2464 .mn = "THNSF5256GPUK TOSHIBA",
c5552fde 2465 .quirks = NVME_QUIRK_NO_APST,
cb32de1b
ML		 2466 },
2467 {
2468 /*
2469 * This LiteON CL1-3D*-Q11 firmware version has a race
2470 * condition associated with actions related to suspend to idle
2471 * LiteON has resolved the problem in future firmware
2472 */
2473 .vid = 0x14a4,
2474 .fr = "22301111",
2475 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
5a6254d5
EM		 2476 },
2477 {
2478 /*
2479 * This Kioxia CD6-V Series / HPE PE8030 device times out and
2480 * aborts I/O during any load, but more easily reproducible
2481 * with discards (fstrim).
2482 *
2483 * The device is left in a state where it is also not possible
2484 * to use "nvme set-feature" to disable APST, but booting with
2485 * nvme_core.default_ps_max_latency=0 works.
2486 */
2487 .vid = 0x1e0f,
2488 .mn = "KCD6XVUL6T40",
2489 .quirks = NVME_QUIRK_NO_APST,
be56945c 2490 }
bd4da3ab
AL		 2491};
2492
2493/* match is null-terminated but idstr is space-padded. */
2494static bool string_matches(const char *idstr, const char *match, size_t len)
2495{
2496 size_t matchlen;
2497
2498 if (!match)
2499 return true;
2500
2501 matchlen = strlen(match);
2502 WARN_ON_ONCE(matchlen > len);
2503
2504 if (memcmp(idstr, match, matchlen))
2505 return false;
2506
2507 for (; matchlen < len; matchlen++)
2508 if (idstr[matchlen] != ' ')
2509 return false;
2510
2511 return true;
2512}
2513
2514static bool quirk_matches(const struct nvme_id_ctrl *id,
2515 const struct nvme_core_quirk_entry *q)
2516{
2517 return q->vid == le16_to_cpu(id->vid) &&
2518 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2519 string_matches(id->fr, q->fr, sizeof(id->fr));
2520}
2521
ab9e00cc
CH		 2522static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2523 struct nvme_id_ctrl *id)
180de007
CH		 2524{
2525 size_t nqnlen;
2526 int off;
2527
6299358d
JD		 2528 if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2529 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2530 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2531 strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2532 return;
2533 }
180de007 2534
6299358d
JD		 2535 if (ctrl->vs >= NVME_VS(1, 2, 1))
2536 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2537 }
180de007
CH		 2538
2539 /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
ab9e00cc 2540 off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
3da584f5 2541 "nqn.2014.08.org.nvmexpress:%04x%04x",
180de007 2542 le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
ab9e00cc 2543 memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
180de007 2544 off += sizeof(id->sn);
ab9e00cc 2545 memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
180de007 2546 off += sizeof(id->mn);
ab9e00cc
CH		 2547 memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2548}
2549
e654dfd3 2550static void nvme_release_subsystem(struct device *dev)
ab9e00cc 2551{
e654dfd3
LG		 2552 struct nvme_subsystem *subsys =
2553 container_of(dev, struct nvme_subsystem, dev);
2554
733e4b69 2555 if (subsys->instance >= 0)
8b850475 2556 ida_free(&nvme_instance_ida, subsys->instance);
ab9e00cc
CH		 2557 kfree(subsys);
2558}
2559
ab9e00cc
CH		 2560static void nvme_destroy_subsystem(struct kref *ref)
2561{
2562 struct nvme_subsystem *subsys =
2563 container_of(ref, struct nvme_subsystem, ref);
2564
2565 mutex_lock(&nvme_subsystems_lock);
2566 list_del(&subsys->entry);
2567 mutex_unlock(&nvme_subsystems_lock);
2568
ed754e5d 2569 ida_destroy(&subsys->ns_ida);
ab9e00cc
CH		 2570 device_del(&subsys->dev);
2571 put_device(&subsys->dev);
2572}
2573
2574static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2575{
2576 kref_put(&subsys->ref, nvme_destroy_subsystem);
2577}
2578
2579static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2580{
2581 struct nvme_subsystem *subsys;
2582
2583 lockdep_assert_held(&nvme_subsystems_lock);
2584
c26aa572
JS		 2585 /*
2586 * Fail matches for discovery subsystems. This results
2587 * in each discovery controller bound to a unique subsystem.
2588 * This avoids issues with validating controller values
2589 * that can only be true when there is a single unique subsystem.
2590 * There may be multiple and completely independent entities
2591 * that provide discovery controllers.
2592 */
2593 if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2594 return NULL;
2595
ab9e00cc
CH		 2596 list_for_each_entry(subsys, &nvme_subsystems, entry) {
2597 if (strcmp(subsys->subnqn, subsysnqn))
2598 continue;
2599 if (!kref_get_unless_zero(&subsys->ref))
2600 continue;
2601 return subsys;
2602 }
2603
2604 return NULL;
2605}
2606
1e496938
HR		 2607#define SUBSYS_ATTR_RO(_name, _mode, _show) \
2608 struct device_attribute subsys_attr_##_name = \
2609 __ATTR(_name, _mode, _show, NULL)
2610
2611static ssize_t nvme_subsys_show_nqn(struct device *dev,
2612 struct device_attribute *attr,
2613 char *buf)
2614{
2615 struct nvme_subsystem *subsys =
2616 container_of(dev, struct nvme_subsystem, dev);
2617
f720a8ed 2618 return sysfs_emit(buf, "%s\n", subsys->subnqn);
1e496938
HR		 2619}
2620static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2621
954ae166
HR		 2622static ssize_t nvme_subsys_show_type(struct device *dev,
2623 struct device_attribute *attr,
2624 char *buf)
2625{
2626 struct nvme_subsystem *subsys =
2627 container_of(dev, struct nvme_subsystem, dev);
2628
2629 switch (subsys->subtype) {
2630 case NVME_NQN_DISC:
2631 return sysfs_emit(buf, "discovery\n");
2632 case NVME_NQN_NVME:
2633 return sysfs_emit(buf, "nvm\n");
2634 default:
2635 return sysfs_emit(buf, "reserved\n");
2636 }
2637}
2638static SUBSYS_ATTR_RO(subsystype, S_IRUGO, nvme_subsys_show_type);
2639
1e496938
HR		 2640#define nvme_subsys_show_str_function(field) \
2641static ssize_t subsys_##field##_show(struct device *dev, \
2642 struct device_attribute *attr, char *buf) \
2643{ \
2644 struct nvme_subsystem *subsys = \
2645 container_of(dev, struct nvme_subsystem, dev); \
bff4bcf3
DW		 2646 return sysfs_emit(buf, "%.*s\n", \
2647 (int)sizeof(subsys->field), subsys->field); \
1e496938
HR		 2648} \
2649static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2650
2651nvme_subsys_show_str_function(model);
2652nvme_subsys_show_str_function(serial);
2653nvme_subsys_show_str_function(firmware_rev);
2654
2655static struct attribute *nvme_subsys_attrs[] = {
2656 &subsys_attr_model.attr,
2657 &subsys_attr_serial.attr,
2658 &subsys_attr_firmware_rev.attr,
2659 &subsys_attr_subsysnqn.attr,
954ae166 2660 &subsys_attr_subsystype.attr,
75c10e73
HR		 2661#ifdef CONFIG_NVME_MULTIPATH
2662 &subsys_attr_iopolicy.attr,
2663#endif
1e496938
HR		 2664 NULL,
2665};
2666
60b152a5 2667static const struct attribute_group nvme_subsys_attrs_group = {
1e496938
HR		 2668 .attrs = nvme_subsys_attrs,
2669};
2670
2671static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2672 &nvme_subsys_attrs_group,
2673 NULL,
2674};
2675
5ab25a32
SG		 2676static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2677{
2678 return ctrl->opts && ctrl->opts->discovery_nqn;
2679}
2680
1b1031ca
CH		 2681static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2682 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
b837b283 2683{
1b1031ca 2684 struct nvme_ctrl *tmp;
b837b283 2685
32fd90c4
CH		 2686 lockdep_assert_held(&nvme_subsystems_lock);
2687
1b1031ca 2688 list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
e7c43fea 2689 if (nvme_state_terminal(tmp))
1b1031ca
CH		 2690 continue;
2691
2692 if (tmp->cntlid == ctrl->cntlid) {
2693 dev_err(ctrl->device,
16cc33b2
KB		 2694 "Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2695 ctrl->cntlid, dev_name(tmp->device),
2696 subsys->subnqn);
1b1031ca
CH		 2697 return false;
2698 }
b837b283 2699
92decf11 2700 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
5ab25a32 2701 nvme_discovery_ctrl(ctrl))
1b1031ca
CH		 2702 continue;
2703
2704 dev_err(ctrl->device,
2705 "Subsystem does not support multiple controllers\n");
2706 return false;
b837b283 2707 }
b837b283 2708
1b1031ca 2709 return true;
b837b283
IR		 2710}
2711
ab9e00cc
CH		 2712static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2713{
2714 struct nvme_subsystem *subsys, *found;
2715 int ret;
2716
2717 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2718 if (!subsys)
2719 return -ENOMEM;
733e4b69
KB		 2720
2721 subsys->instance = -1;
ab9e00cc
CH		 2722 mutex_init(&subsys->lock);
2723 kref_init(&subsys->ref);
2724 INIT_LIST_HEAD(&subsys->ctrls);
ed754e5d 2725 INIT_LIST_HEAD(&subsys->nsheads);
ab9e00cc
CH		 2726 nvme_init_subnqn(subsys, ctrl, id);
2727 memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2728 memcpy(subsys->model, id->mn, sizeof(subsys->model));
2729 memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2730 subsys->vendor_id = le16_to_cpu(id->vid);
2731 subsys->cmic = id->cmic;
954ae166
HR		 2732
2733 /* Versions prior to 1.4 don't necessarily report a valid type */
2734 if (id->cntrltype == NVME_CTRL_DISC ||
2735 !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2736 subsys->subtype = NVME_NQN_DISC;
2737 else
2738 subsys->subtype = NVME_NQN_NVME;
2739
20e8b689
HR		 2740 if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
2741 dev_err(ctrl->device,
2742 "Subsystem %s is not a discovery controller",
2743 subsys->subnqn);
2744 kfree(subsys);
2745 return -EINVAL;
2746 }
81adb863 2747 subsys->awupf = le16_to_cpu(id->awupf);
e3d34794 2748 nvme_mpath_default_iopolicy(subsys);
ab9e00cc
CH		 2749
2750 subsys->dev.class = nvme_subsys_class;
2751 subsys->dev.release = nvme_release_subsystem;
1e496938 2752 subsys->dev.groups = nvme_subsys_attrs_groups;
733e4b69 2753 dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
ab9e00cc
CH		 2754 device_initialize(&subsys->dev);
2755
2756 mutex_lock(&nvme_subsystems_lock);
2757 found = __nvme_find_get_subsystem(subsys->subnqn);
2758 if (found) {
e654dfd3 2759 put_device(&subsys->dev);
ab9e00cc 2760 subsys = found;
32fd90c4 2761
1b1031ca 2762 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
ab9e00cc 2763 ret = -EINVAL;
32fd90c4 2764 goto out_put_subsystem;
ab9e00cc 2765 }
ab9e00cc
CH		 2766 } else {
2767 ret = device_add(&subsys->dev);
2768 if (ret) {
2769 dev_err(ctrl->device,
2770 "failed to register subsystem device.\n");
8c36e66f 2771 put_device(&subsys->dev);
ab9e00cc
CH		 2772 goto out_unlock;
2773 }
ed754e5d 2774 ida_init(&subsys->ns_ida);
ab9e00cc
CH		 2775 list_add_tail(&subsys->entry, &nvme_subsystems);
2776 }
2777
bc4f6e06
DC		 2778 ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2779 dev_name(ctrl->device));
2780 if (ret) {
ab9e00cc
CH		 2781 dev_err(ctrl->device,
2782 "failed to create sysfs link from subsystem.\n");
32fd90c4 2783 goto out_put_subsystem;
ab9e00cc
CH		 2784 }
2785
733e4b69
KB		 2786 if (!found)
2787 subsys->instance = ctrl->instance;
32fd90c4 2788 ctrl->subsys = subsys;
ab9e00cc 2789 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
32fd90c4 2790 mutex_unlock(&nvme_subsystems_lock);
ab9e00cc
CH		 2791 return 0;
2792
32fd90c4
CH		 2793out_put_subsystem:
2794 nvme_put_subsystem(subsys);
ab9e00cc
CH		 2795out_unlock:
2796 mutex_unlock(&nvme_subsystems_lock);
ab9e00cc 2797 return ret;
180de007
CH		 2798}
2799
be93e87e 2800int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
0e98719b 2801 void *log, size_t size, u64 offset)
c627c487
KB		 2802{
2803 struct nvme_command c = { };
71fb90eb 2804 u32 dwlen = nvme_bytes_to_numd(size);
70da6094
MB		 2805
2806 c.get_log_page.opcode = nvme_admin_get_log_page;
0e98719b 2807 c.get_log_page.nsid = cpu_to_le32(nsid);
70da6094 2808 c.get_log_page.lid = log_page;
0e98719b 2809 c.get_log_page.lsp = lsp;
70da6094
MB		 2810 c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2811 c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
7ec6074f
MB		 2812 c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2813 c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
be93e87e 2814 c.get_log_page.csi = csi;
c627c487
KB		 2815
2816 return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2817}
2818
be93e87e
KB		 2819static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
2820 struct nvme_effects_log **log)
84fef62d 2821{
f6224b86 2822 struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
84fef62d
KB		 2823 int ret;
2824
be93e87e
KB		 2825 if (cel)
2826 goto out;
84fef62d 2827
be93e87e
KB		 2828 cel = kzalloc(sizeof(*cel), GFP_KERNEL);
2829 if (!cel)
2830 return -ENOMEM;
84fef62d 2831
46d2613e 2832 ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
f6224b86 2833 cel, sizeof(*cel), 0);
84fef62d 2834 if (ret) {
be93e87e
KB		 2835 kfree(cel);
2836 return ret;
84fef62d 2837 }
be93e87e 2838
f6224b86 2839 xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
be93e87e 2840out:
f6224b86 2841 *log = cel;
be93e87e 2842 return 0;
180de007
CH		 2843}
2844
5befc7c2 2845static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
7fd8930f 2846{
8609c63f 2847 u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
7fd8930f 2848
8609c63f
BVA		 2849 if (check_shl_overflow(1U, units + page_shift - 9, &val))
2850 return UINT_MAX;
2851 return val;
5befc7c2
KB		 2852}
2853
2854static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
2855{
2856 struct nvme_command c = { };
2857 struct nvme_id_ctrl_nvm *id;
2858 int ret;
2859
2860 if (ctrl->oncs & NVME_CTRL_ONCS_DSM) {
2861 ctrl->max_discard_sectors = UINT_MAX;
2862 ctrl->max_discard_segments = NVME_DSM_MAX_RANGES;
2863 } else {
2864 ctrl->max_discard_sectors = 0;
2865 ctrl->max_discard_segments = 0;
f3ca80fc 2866 }
7fd8930f 2867
5befc7c2
KB		 2868 /*
2869 * Even though NVMe spec explicitly states that MDTS is not applicable
2870 * to the write-zeroes, we are cautious and limit the size to the
2871 * controllers max_hw_sectors value, which is based on the MDTS field
2872 * and possibly other limiting factors.
2873 */
2874 if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
2875 !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
2876 ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
2877 else
2878 ctrl->max_zeroes_sectors = 0;
2879
2880 if (nvme_ctrl_limited_cns(ctrl))
2881 return 0;
2882
2883 id = kzalloc(sizeof(*id), GFP_KERNEL);
2884 if (!id)
2885 return 0;
2886
2887 c.identify.opcode = nvme_admin_identify;
2888 c.identify.cns = NVME_ID_CNS_CS_CTRL;
2889 c.identify.csi = NVME_CSI_NVM;
2890
2891 ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
2892 if (ret)
2893 goto free_data;
2894
2895 if (id->dmrl)
2896 ctrl->max_discard_segments = id->dmrl;
2897 if (id->dmrsl)
2898 ctrl->max_discard_sectors = le32_to_cpu(id->dmrsl);
2899 if (id->wzsl)
2900 ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
2901
2902free_data:
2903 kfree(id);
2904 return ret;
2905}
2906
44ef5611 2907static int nvme_init_identify(struct nvme_ctrl *ctrl)
7fd8930f
CH		 2908{
2909 struct nvme_id_ctrl *id;
a229dbf6 2910 u32 max_hw_sectors;
76a5af84 2911 bool prev_apst_enabled;
5befc7c2 2912 int ret;
f3ca80fc 2913
7fd8930f
CH		 2914 ret = nvme_identify_ctrl(ctrl, &id);
2915 if (ret) {
1b3c47c1 2916 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
7fd8930f
CH		 2917 return -EIO;
2918 }
2919
84fef62d 2920 if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
be93e87e 2921 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
84fef62d 2922 if (ret < 0)
75c8b19a 2923 goto out_free;
84fef62d 2924 }
180de007 2925
a89fcca8
GP		 2926 if (!(ctrl->ops->flags & NVME_F_FABRICS))
2927 ctrl->cntlid = le16_to_cpu(id->cntlid);
2928
bd4da3ab 2929 if (!ctrl->identified) {
44ef5611 2930 unsigned int i;
ab9e00cc
CH		 2931
2932 ret = nvme_init_subsystem(ctrl, id);
2933 if (ret)
2934 goto out_free;
2935
bd4da3ab
AL		 2936 /*
2937 * Check for quirks. Quirk can depend on firmware version,
2938 * so, in principle, the set of quirks present can change
2939 * across a reset. As a possible future enhancement, we
2940 * could re-scan for quirks every time we reinitialize
2941 * the device, but we'd have to make sure that the driver
2942 * behaves intelligently if the quirks change.
2943 */
bd4da3ab
AL		 2944 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
2945 if (quirk_matches(id, &core_quirks[i]))
2946 ctrl->quirks |= core_quirks[i].quirks;
2947 }
2948 }
2949
c35e30b4 2950 if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
f0425db0 2951 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
c35e30b4
AL		 2952 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
2953 }
2954
49cd84b6
KB		 2955 ctrl->crdt[0] = le16_to_cpu(id->crdt1);
2956 ctrl->crdt[1] = le16_to_cpu(id->crdt2);
2957 ctrl->crdt[2] = le16_to_cpu(id->crdt3);
2958
8a9ae523 2959 ctrl->oacs = le16_to_cpu(id->oacs);
43e2d08d 2960 ctrl->oncs = le16_to_cpu(id->oncs);
2d466c7a 2961 ctrl->mtfa = le16_to_cpu(id->mtfa);
c0561f82 2962 ctrl->oaes = le32_to_cpu(id->oaes);
400b6a7b
GR		 2963 ctrl->wctemp = le16_to_cpu(id->wctemp);
2964 ctrl->cctemp = le16_to_cpu(id->cctemp);
2965
6bf25d16 2966 atomic_set(&ctrl->abort_limit, id->acl + 1);
7fd8930f 2967 ctrl->vwc = id->vwc;
7fd8930f 2968 if (id->mdts)
5befc7c2 2969 max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
7fd8930f 2970 else
a229dbf6
CH		 2971 max_hw_sectors = UINT_MAX;
2972 ctrl->max_hw_sectors =
2973 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
7fd8930f 2974
da35825d 2975 nvme_set_queue_limits(ctrl, ctrl->admin_q);
07bfcd09 2976 ctrl->sgls = le32_to_cpu(id->sgls);
038bd4cb 2977 ctrl->kas = le16_to_cpu(id->kas);
0d0b660f 2978 ctrl->max_namespaces = le32_to_cpu(id->mnan);
3e53ba38 2979 ctrl->ctratt = le32_to_cpu(id->ctratt);
07bfcd09 2980
86c2457a
MB		 2981 ctrl->cntrltype = id->cntrltype;
2982 ctrl->dctype = id->dctype;
2983
07fbd32a
MP		 2984 if (id->rtd3e) {
2985 /* us -> s */
f5af577d 2986 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
07fbd32a
MP		 2987
2988 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
2989 shutdown_timeout, 60);
2990
2991 if (ctrl->shutdown_timeout != shutdown_timeout)
1a3838d7 2992 dev_info(ctrl->device,
07fbd32a
MP		 2993 "Shutdown timeout set to %u seconds\n",
2994 ctrl->shutdown_timeout);
2995 } else
2996 ctrl->shutdown_timeout = shutdown_timeout;
2997
c5552fde 2998 ctrl->npss = id->npss;
76a5af84
KHF		 2999 ctrl->apsta = id->apsta;
3000 prev_apst_enabled = ctrl->apst_enabled;
c35e30b4
AL		 3001 if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3002 if (force_apst && id->apsta) {
f0425db0 3003 dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
76a5af84 3004 ctrl->apst_enabled = true;
c35e30b4 3005 } else {
76a5af84 3006 ctrl->apst_enabled = false;
c35e30b4
AL		 3007 }
3008 } else {
76a5af84 3009 ctrl->apst_enabled = id->apsta;
c35e30b4 3010 }
c5552fde
AL		 3011 memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3012
d3d5b87d 3013 if (ctrl->ops->flags & NVME_F_FABRICS) {
07bfcd09
CH		 3014 ctrl->icdoff = le16_to_cpu(id->icdoff);
3015 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3016 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3017 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3018
3019 /*
3020 * In fabrics we need to verify the cntlid matches the
3021 * admin connect
3022 */
634b8325 3023 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
a8157ff3
JS		 3024 dev_err(ctrl->device,
3025 "Mismatching cntlid: Connect %u vs Identify "
3026 "%u, rejecting\n",
3027 ctrl->cntlid, le16_to_cpu(id->cntlid));
07bfcd09 3028 ret = -EINVAL;
634b8325
KB		 3029 goto out_free;
3030 }
038bd4cb 3031
5ab25a32 3032 if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
f0425db0 3033 dev_err(ctrl->device,
038bd4cb
SG		 3034 "keep-alive support is mandatory for fabrics\n");
3035 ret = -EINVAL;
634b8325 3036 goto out_free;
038bd4cb 3037 }
07bfcd09 3038 } else {
fe6d53c9
CH		 3039 ctrl->hmpre = le32_to_cpu(id->hmpre);
3040 ctrl->hmmin = le32_to_cpu(id->hmmin);
044a9df1
CH		 3041 ctrl->hmminds = le32_to_cpu(id->hmminds);
3042 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
07bfcd09 3043 }
da35825d 3044
5e1f6899 3045 ret = nvme_mpath_init_identify(ctrl, id);
0d0b660f 3046 if (ret < 0)
44ef5611 3047 goto out_free;
0d0b660f 3048
76a5af84 3049 if (ctrl->apst_enabled && !prev_apst_enabled)
c5552fde 3050 dev_pm_qos_expose_latency_tolerance(ctrl->device);
76a5af84 3051 else if (!ctrl->apst_enabled && prev_apst_enabled)
c5552fde
AL		 3052 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3053
44ef5611
CK		 3054out_free:
3055 kfree(id);
3056 return ret;
3057}
3058
3059/*
3060 * Initialize the cached copies of the Identify data and various controller
3061 * register in our nvme_ctrl structure. This should be called as soon as
3062 * the admin queue is fully up and running.
3063 */
3064int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl)
3065{
3066 int ret;
3067
3068 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3069 if (ret) {
3070 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3071 return ret;
3072 }
3073
3074 ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3075
3076 if (ctrl->vs >= NVME_VS(1, 1, 0))
3077 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3078
3079 ret = nvme_init_identify(ctrl);
3080 if (ret)
3081 return ret;
3082
5befc7c2
KB		 3083 ret = nvme_init_non_mdts_limits(ctrl);
3084 if (ret < 0)
3085 return ret;
3086
634b8325
KB		 3087 ret = nvme_configure_apst(ctrl);
3088 if (ret < 0)
3089 return ret;
95d54bd1 3090
dbf86b39
JD		 3091 ret = nvme_configure_timestamp(ctrl);
3092 if (ret < 0)
3093 return ret;
634b8325 3094
4020aad8 3095 ret = nvme_configure_host_options(ctrl);
49cd84b6
KB		 3096 if (ret < 0)
3097 return ret;
3098
5ab25a32 3099 if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
59e330f8
KB		 3100 ret = nvme_hwmon_init(ctrl);
3101 if (ret < 0)
3102 return ret;
3103 }
400b6a7b 3104
bd4da3ab 3105 ctrl->identified = true;
c5552fde 3106
634b8325 3107 return 0;
7fd8930f 3108}
f21c4769 3109EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
7fd8930f 3110
f3ca80fc 3111static int nvme_dev_open(struct inode *inode, struct file *file)
1673f1f0 3112{
a6a5149b
CH		 3113 struct nvme_ctrl *ctrl =
3114 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
1673f1f0 3115
2b1b7e78
JW		 3116 switch (ctrl->state) {
3117 case NVME_CTRL_LIVE:
2b1b7e78
JW		 3118 break;
3119 default:
a6a5149b 3120 return -EWOULDBLOCK;
2b1b7e78
JW		 3121 }
3122
52a3974f 3123 nvme_get_ctrl(ctrl);
4bab6909
CK		 3124 if (!try_module_get(ctrl->ops->module)) {
3125 nvme_put_ctrl(ctrl);
52a3974f 3126 return -EINVAL;
4bab6909 3127 }
52a3974f 3128
a6a5149b 3129 file->private_data = ctrl;
f3ca80fc
CH		 3130 return 0;
3131}
3132
52a3974f
CK		 3133static int nvme_dev_release(struct inode *inode, struct file *file)
3134{
3135 struct nvme_ctrl *ctrl =
3136 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3137
3138 module_put(ctrl->ops->module);
3139 nvme_put_ctrl(ctrl);
3140 return 0;
3141}
3142
f3ca80fc
CH		 3143static const struct file_operations nvme_dev_fops = {
3144 .owner = THIS_MODULE,
3145 .open = nvme_dev_open,
52a3974f 3146 .release = nvme_dev_release,
f3ca80fc 3147 .unlocked_ioctl = nvme_dev_ioctl,
1832f2d8 3148 .compat_ioctl = compat_ptr_ioctl,
f3ca80fc
CH		 3149};
3150
3151static ssize_t nvme_sysfs_reset(struct device *dev,
3152 struct device_attribute *attr, const char *buf,
3153 size_t count)
3154{
3155 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3156 int ret;
3157
d86c4d8e 3158 ret = nvme_reset_ctrl_sync(ctrl);
f3ca80fc
CH		 3159 if (ret < 0)
3160 return ret;
3161 return count;
1673f1f0 3162}
f3ca80fc 3163static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
1673f1f0 3164
9ec3bb2f
KB		 3165static ssize_t nvme_sysfs_rescan(struct device *dev,
3166 struct device_attribute *attr, const char *buf,
3167 size_t count)
3168{
3169 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3170
3171 nvme_queue_scan(ctrl);
3172 return count;
3173}
3174static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3175
5b85b826
CH		 3176static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3177{
3178 struct gendisk *disk = dev_to_disk(dev);
3179
ba4fb320 3180 if (disk->fops == &nvme_bdev_ops)
5b85b826
CH		 3181 return nvme_get_ns_from_dev(dev)->head;
3182 else
3183 return disk->private_data;
3184}
3185
118472ab 3186static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
5b85b826 3187 char *buf)
118472ab 3188{
5b85b826
CH		 3189 struct nvme_ns_head *head = dev_to_ns_head(dev);
3190 struct nvme_ns_ids *ids = &head->ids;
3191 struct nvme_subsystem *subsys = head->subsys;
ab9e00cc
CH		 3192 int serial_len = sizeof(subsys->serial);
3193 int model_len = sizeof(subsys->model);
118472ab 3194
002fab04 3195 if (!uuid_is_null(&ids->uuid))
bff4bcf3 3196 return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
6484f5d1 3197
002fab04 3198 if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
bff4bcf3 3199 return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
118472ab 3200
002fab04 3201 if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
bff4bcf3 3202 return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
118472ab 3203
ab9e00cc
CH		 3204 while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3205 subsys->serial[serial_len - 1] == '\0'))
118472ab 3206 serial_len--;
ab9e00cc
CH		 3207 while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3208 subsys->model[model_len - 1] == '\0'))
118472ab
KB		 3209 model_len--;
3210
bff4bcf3 3211 return sysfs_emit(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
ab9e00cc 3212 serial_len, subsys->serial, model_len, subsys->model,
5b85b826 3213 head->ns_id);
118472ab 3214}
c828a892 3215static DEVICE_ATTR_RO(wwid);
118472ab 3216
d934f984 3217static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
5b85b826 3218 char *buf)
d934f984 3219{
bff4bcf3 3220 return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
d934f984 3221}
c828a892 3222static DEVICE_ATTR_RO(nguid);
d934f984 3223
2b9b6e86 3224static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
5b85b826 3225 char *buf)
2b9b6e86 3226{
5b85b826 3227 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
d934f984
JT		 3228
3229 /* For backward compatibility expose the NGUID to userspace if
3230 * we have no UUID set
3231 */
002fab04 3232 if (uuid_is_null(&ids->uuid)) {
d934f984
JT		 3233 printk_ratelimited(KERN_WARNING
3234 "No UUID available providing old NGUID\n");
bff4bcf3 3235 return sysfs_emit(buf, "%pU\n", ids->nguid);
d934f984 3236 }
bff4bcf3 3237 return sysfs_emit(buf, "%pU\n", &ids->uuid);
2b9b6e86 3238}
c828a892 3239static DEVICE_ATTR_RO(uuid);
2b9b6e86
KB		 3240
3241static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
5b85b826 3242 char *buf)
2b9b6e86 3243{
bff4bcf3 3244 return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
2b9b6e86 3245}
c828a892 3246static DEVICE_ATTR_RO(eui);
2b9b6e86
KB		 3247
3248static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
5b85b826 3249 char *buf)
2b9b6e86 3250{
bff4bcf3 3251 return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
2b9b6e86 3252}
c828a892 3253static DEVICE_ATTR_RO(nsid);
2b9b6e86 3254
5b85b826 3255static struct attribute *nvme_ns_id_attrs[] = {
118472ab 3256 &dev_attr_wwid.attr,
2b9b6e86 3257 &dev_attr_uuid.attr,
d934f984 3258 &dev_attr_nguid.attr,
2b9b6e86
KB		 3259 &dev_attr_eui.attr,
3260 &dev_attr_nsid.attr,
0d0b660f
CH		 3261#ifdef CONFIG_NVME_MULTIPATH
3262 &dev_attr_ana_grpid.attr,
3263 &dev_attr_ana_state.attr,
3264#endif
2b9b6e86
KB		 3265 NULL,
3266};
3267
5b85b826 3268static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
2b9b6e86
KB		 3269 struct attribute *a, int n)
3270{
3271 struct device *dev = container_of(kobj, struct device, kobj);
5b85b826 3272 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
2b9b6e86
KB		 3273
3274 if (a == &dev_attr_uuid.attr) {
a04b5de5 3275 if (uuid_is_null(&ids->uuid) &&
002fab04 3276 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
d934f984
JT		 3277 return 0;
3278 }
3279 if (a == &dev_attr_nguid.attr) {
002fab04 3280 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
2b9b6e86
KB		 3281 return 0;
3282 }
3283 if (a == &dev_attr_eui.attr) {
002fab04 3284 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
2b9b6e86
KB		 3285 return 0;
3286 }
0d0b660f
CH		 3287#ifdef CONFIG_NVME_MULTIPATH
3288 if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
ba4fb320 3289 if (dev_to_disk(dev)->fops != &nvme_bdev_ops) /* per-path attr */
0d0b660f
CH		 3290 return 0;
3291 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3292 return 0;
3293 }
3294#endif
2b9b6e86
KB		 3295 return a->mode;
3296}
3297
eb090c4c 3298static const struct attribute_group nvme_ns_id_attr_group = {
5b85b826
CH		 3299 .attrs = nvme_ns_id_attrs,
3300 .is_visible = nvme_ns_id_attrs_are_visible,
2b9b6e86
KB		 3301};
3302
33b14f67
HR		 3303const struct attribute_group *nvme_ns_id_attr_groups[] = {
3304 &nvme_ns_id_attr_group,
33b14f67
HR		 3305 NULL,
3306};
3307
931e1c22 3308#define nvme_show_str_function(field) \
779ff756
KB		 3309static ssize_t field##_show(struct device *dev, \
3310 struct device_attribute *attr, char *buf) \
3311{ \
3312 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
bff4bcf3 3313 return sysfs_emit(buf, "%.*s\n", \
ab9e00cc 3314 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \
779ff756
KB		 3315} \
3316static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3317
ab9e00cc
CH		 3318nvme_show_str_function(model);
3319nvme_show_str_function(serial);
3320nvme_show_str_function(firmware_rev);
3321
931e1c22
ML		 3322#define nvme_show_int_function(field) \
3323static ssize_t field##_show(struct device *dev, \
3324 struct device_attribute *attr, char *buf) \
3325{ \
3326 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
bff4bcf3 3327 return sysfs_emit(buf, "%d\n", ctrl->field); \
931e1c22
ML		 3328} \
3329static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3330
931e1c22 3331nvme_show_int_function(cntlid);
103e515e 3332nvme_show_int_function(numa_node);
2b1ff255
JS		 3333nvme_show_int_function(queue_count);
3334nvme_show_int_function(sqsize);
74c22990 3335nvme_show_int_function(kato);
779ff756 3336
1a353d85
ML		 3337static ssize_t nvme_sysfs_delete(struct device *dev,
3338 struct device_attribute *attr, const char *buf,
3339 size_t count)
3340{
3341 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3342
3343 if (device_remove_file_self(dev, attr))
c5017e85 3344 nvme_delete_ctrl_sync(ctrl);
1a353d85
ML		 3345 return count;
3346}
3347static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3348
3349static ssize_t nvme_sysfs_show_transport(struct device *dev,
3350 struct device_attribute *attr,
3351 char *buf)
3352{
3353 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3354
f720a8ed 3355 return sysfs_emit(buf, "%s\n", ctrl->ops->name);
1a353d85
ML		 3356}
3357static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3358
8432bdb2
SG		 3359static ssize_t nvme_sysfs_show_state(struct device *dev,
3360 struct device_attribute *attr,
3361 char *buf)
3362{
3363 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3364 static const char *const state_name[] = {
3365 [NVME_CTRL_NEW] = "new",
3366 [NVME_CTRL_LIVE] = "live",
3367 [NVME_CTRL_RESETTING] = "resetting",
ad6a0a52 3368 [NVME_CTRL_CONNECTING] = "connecting",
8432bdb2 3369 [NVME_CTRL_DELETING] = "deleting",
ecca390e 3370 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
8432bdb2
SG		 3371 [NVME_CTRL_DEAD] = "dead",
3372 };
3373
3374 if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3375 state_name[ctrl->state])
bff4bcf3 3376 return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
8432bdb2 3377
bff4bcf3 3378 return sysfs_emit(buf, "unknown state\n");
8432bdb2
SG		 3379}
3380
3381static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3382
1a353d85
ML		 3383static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3384 struct device_attribute *attr,
3385 char *buf)
3386{
3387 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3388
f720a8ed 3389 return sysfs_emit(buf, "%s\n", ctrl->subsys->subnqn);
1a353d85
ML		 3390}
3391static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3392
76171c6c
SG		 3393static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3394 struct device_attribute *attr,
3395 char *buf)
3396{
3397 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3398
f720a8ed 3399 return sysfs_emit(buf, "%s\n", ctrl->opts->host->nqn);
76171c6c
SG		 3400}
3401static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3402
45fb19f7
SG		 3403static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3404 struct device_attribute *attr,
3405 char *buf)
3406{
3407 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3408
f720a8ed 3409 return sysfs_emit(buf, "%pU\n", &ctrl->opts->host->id);
45fb19f7
SG		 3410}
3411static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3412
1a353d85
ML		 3413static ssize_t nvme_sysfs_show_address(struct device *dev,
3414 struct device_attribute *attr,
3415 char *buf)
3416{
3417 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3418
3419 return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3420}
3421static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3422
764075fd
SG		 3423static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3424 struct device_attribute *attr, char *buf)
3425{
3426 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3427 struct nvmf_ctrl_options *opts = ctrl->opts;
3428
3429 if (ctrl->opts->max_reconnects == -1)
bff4bcf3
DW		 3430 return sysfs_emit(buf, "off\n");
3431 return sysfs_emit(buf, "%d\n",
3432 opts->max_reconnects * opts->reconnect_delay);
764075fd
SG		 3433}
3434
3435static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3436 struct device_attribute *attr, const char *buf, size_t count)
3437{
3438 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3439 struct nvmf_ctrl_options *opts = ctrl->opts;
3440 int ctrl_loss_tmo, err;
3441
3442 err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3443 if (err)
3444 return -EINVAL;
3445
25a64e4e 3446 if (ctrl_loss_tmo < 0)
764075fd
SG		 3447 opts->max_reconnects = -1;
3448 else
3449 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3450 opts->reconnect_delay);
3451 return count;
3452}
3453static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3454 nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3455
3456static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3457 struct device_attribute *attr, char *buf)
3458{
3459 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3460
3461 if (ctrl->opts->reconnect_delay == -1)
bff4bcf3
DW		 3462 return sysfs_emit(buf, "off\n");
3463 return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
764075fd
SG		 3464}
3465
3466static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3467 struct device_attribute *attr, const char *buf, size_t count)
3468{
3469 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3470 unsigned int v;
3471 int err;
3472
3473 err = kstrtou32(buf, 10, &v);
eca9e827
DC		 3474 if (err)
3475 return err;
764075fd
SG		 3476
3477 ctrl->opts->reconnect_delay = v;
3478 return count;
3479}
3480static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3481 nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3482
09fbed63
DW		 3483static ssize_t nvme_ctrl_fast_io_fail_tmo_show(struct device *dev,
3484 struct device_attribute *attr, char *buf)
3485{
3486 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3487
3488 if (ctrl->opts->fast_io_fail_tmo == -1)
3489 return sysfs_emit(buf, "off\n");
3490 return sysfs_emit(buf, "%d\n", ctrl->opts->fast_io_fail_tmo);
3491}
3492
3493static ssize_t nvme_ctrl_fast_io_fail_tmo_store(struct device *dev,
3494 struct device_attribute *attr, const char *buf, size_t count)
3495{
3496 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3497 struct nvmf_ctrl_options *opts = ctrl->opts;
3498 int fast_io_fail_tmo, err;
3499
3500 err = kstrtoint(buf, 10, &fast_io_fail_tmo);
3501 if (err)
3502 return -EINVAL;
3503
3504 if (fast_io_fail_tmo < 0)
3505 opts->fast_io_fail_tmo = -1;
3506 else
3507 opts->fast_io_fail_tmo = fast_io_fail_tmo;
3508 return count;
3509}
3510static DEVICE_ATTR(fast_io_fail_tmo, S_IRUGO | S_IWUSR,
3511 nvme_ctrl_fast_io_fail_tmo_show, nvme_ctrl_fast_io_fail_tmo_store);
3512
86c2457a
MB		 3513static ssize_t cntrltype_show(struct device *dev,
3514 struct device_attribute *attr, char *buf)
3515{
3516 static const char * const type[] = {
3517 [NVME_CTRL_IO] = "io\n",
3518 [NVME_CTRL_DISC] = "discovery\n",
3519 [NVME_CTRL_ADMIN] = "admin\n",
3520 };
3521 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3522
3523 if (ctrl->cntrltype > NVME_CTRL_ADMIN || !type[ctrl->cntrltype])
3524 return sysfs_emit(buf, "reserved\n");
3525
3526 return sysfs_emit(buf, type[ctrl->cntrltype]);
3527}
3528static DEVICE_ATTR_RO(cntrltype);
3529
3530static ssize_t dctype_show(struct device *dev,
3531 struct device_attribute *attr, char *buf)
3532{
3533 static const char * const type[] = {
3534 [NVME_DCTYPE_NOT_REPORTED] = "none\n",
3535 [NVME_DCTYPE_DDC] = "ddc\n",
3536 [NVME_DCTYPE_CDC] = "cdc\n",
3537 };
3538 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3539
3540 if (ctrl->dctype > NVME_DCTYPE_CDC || !type[ctrl->dctype])
3541 return sysfs_emit(buf, "reserved\n");
3542
3543 return sysfs_emit(buf, type[ctrl->dctype]);
3544}
3545static DEVICE_ATTR_RO(dctype);
3546
779ff756
KB		 3547static struct attribute *nvme_dev_attrs[] = {
3548 &dev_attr_reset_controller.attr,
9ec3bb2f 3549 &dev_attr_rescan_controller.attr,
779ff756
KB		 3550 &dev_attr_model.attr,
3551 &dev_attr_serial.attr,
3552 &dev_attr_firmware_rev.attr,
931e1c22 3553 &dev_attr_cntlid.attr,
1a353d85
ML		 3554 &dev_attr_delete_controller.attr,
3555 &dev_attr_transport.attr,
3556 &dev_attr_subsysnqn.attr,
3557 &dev_attr_address.attr,
8432bdb2 3558 &dev_attr_state.attr,
103e515e 3559 &dev_attr_numa_node.attr,
2b1ff255
JS		 3560 &dev_attr_queue_count.attr,
3561 &dev_attr_sqsize.attr,
76171c6c 3562 &dev_attr_hostnqn.attr,
45fb19f7 3563 &dev_attr_hostid.attr,
764075fd
SG		 3564 &dev_attr_ctrl_loss_tmo.attr,
3565 &dev_attr_reconnect_delay.attr,
09fbed63 3566 &dev_attr_fast_io_fail_tmo.attr,
74c22990 3567 &dev_attr_kato.attr,
86c2457a
MB		 3568 &dev_attr_cntrltype.attr,
3569 &dev_attr_dctype.attr,
779ff756
KB		 3570 NULL
3571};
3572
1a353d85
ML		 3573static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3574 struct attribute *a, int n)
3575{
3576 struct device *dev = container_of(kobj, struct device, kobj);
3577 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3578
49d3d50b
CH		 3579 if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3580 return 0;
3581 if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3582 return 0;
76171c6c
SG		 3583 if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3584 return 0;
45fb19f7
SG		 3585 if (a == &dev_attr_hostid.attr && !ctrl->opts)
3586 return 0;
7cd49f75
SG		 3587 if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
3588 return 0;
3589 if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
3590 return 0;
d6609084
GT		 3591 if (a == &dev_attr_fast_io_fail_tmo.attr && !ctrl->opts)
3592 return 0;
1a353d85
ML		 3593
3594 return a->mode;
3595}
3596
60b152a5 3597static const struct attribute_group nvme_dev_attrs_group = {
1a353d85
ML		 3598 .attrs = nvme_dev_attrs,
3599 .is_visible = nvme_dev_attrs_are_visible,
779ff756
KB		 3600};
3601
3602static const struct attribute_group *nvme_dev_attr_groups[] = {
3603 &nvme_dev_attrs_group,
3604 NULL,
3605};
3606
5974ea7c 3607static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
ed754e5d
CH		 3608 unsigned nsid)
3609{
3610 struct nvme_ns_head *h;
3611
5974ea7c 3612 lockdep_assert_held(&ctrl->subsys->lock);
ed754e5d 3613
5974ea7c
SM		 3614 list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3615 /*
3616 * Private namespaces can share NSIDs under some conditions.
3617 * In that case we can't use the same ns_head for namespaces
3618 * with the same NSID.
3619 */
3620 if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
9edceaf4
DW		 3621 continue;
3622 if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
ed754e5d
CH		 3623 return h;
3624 }
3625
3626 return NULL;
3627}
3628
fd8099e7
CH		 3629static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3630 struct nvme_ns_ids *ids)
ed754e5d 3631{
e2724cb9
CH		 3632 bool has_uuid = !uuid_is_null(&ids->uuid);
3633 bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
3634 bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
ed754e5d
CH		 3635 struct nvme_ns_head *h;
3636
3637 lockdep_assert_held(&subsys->lock);
3638
3639 list_for_each_entry(h, &subsys->nsheads, entry) {
e2724cb9
CH		 3640 if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
3641 return -EINVAL;
3642 if (has_nguid &&
3643 memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
3644 return -EINVAL;
3645 if (has_eui64 &&
3646 memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
ed754e5d
CH		 3647 return -EINVAL;
3648 }
3649
3650 return 0;
3651}
3652
be5eb933
AM		 3653static void nvme_cdev_rel(struct device *dev)
3654{
8b850475 3655 ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
be5eb933
AM		 3656}
3657
2637baed
MI		 3658void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
3659{
3660 cdev_device_del(cdev, cdev_device);
be5eb933 3661 put_device(cdev_device);
2637baed
MI		 3662}
3663
3664int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
3665 const struct file_operations *fops, struct module *owner)
3666{
3667 int minor, ret;
3668
8b850475 3669 minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
2637baed
MI		 3670 if (minor < 0)
3671 return minor;
3672 cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
3673 cdev_device->class = nvme_ns_chr_class;
be5eb933 3674 cdev_device->release = nvme_cdev_rel;
2637baed
MI		 3675 device_initialize(cdev_device);
3676 cdev_init(cdev, fops);
3677 cdev->owner = owner;
3678 ret = cdev_device_add(cdev, cdev_device);
be5eb933 3679 if (ret)
3596a065 3680 put_device(cdev_device);
be5eb933 3681
2637baed
MI		 3682 return ret;
3683}
3684
3685static int nvme_ns_chr_open(struct inode *inode, struct file *file)
3686{
3687 return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
3688}
3689
3690static int nvme_ns_chr_release(struct inode *inode, struct file *file)
3691{
3692 nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
3693 return 0;
3694}
3695
3696static const struct file_operations nvme_ns_chr_fops = {
3697 .owner = THIS_MODULE,
3698 .open = nvme_ns_chr_open,
3699 .release = nvme_ns_chr_release,
3700 .unlocked_ioctl = nvme_ns_chr_ioctl,
3701 .compat_ioctl = compat_ptr_ioctl,
3702};
3703
3704static int nvme_add_ns_cdev(struct nvme_ns *ns)
3705{
3706 int ret;
3707
3708 ns->cdev_device.parent = ns->ctrl->device;
3709 ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
3710 ns->ctrl->instance, ns->head->instance);
3711 if (ret)
3712 return ret;
be5eb933
AM		 3713
3714 return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
3715 ns->ctrl->ops->module);
2637baed
MI		 3716}
3717
ed754e5d 3718static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
03f8cebc 3719 unsigned nsid, struct nvme_ns_ids *ids)
ed754e5d
CH		 3720{
3721 struct nvme_ns_head *head;
f3334447 3722 size_t size = sizeof(*head);
ed754e5d
CH		 3723 int ret = -ENOMEM;
3724
f3334447
CH		 3725#ifdef CONFIG_NVME_MULTIPATH
3726 size += num_possible_nodes() * sizeof(struct nvme_ns *);
3727#endif
3728
3729 head = kzalloc(size, GFP_KERNEL);
ed754e5d
CH		 3730 if (!head)
3731 goto out;
8b850475 3732 ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
ed754e5d
CH		 3733 if (ret < 0)
3734 goto out_free_head;
3735 head->instance = ret;
3736 INIT_LIST_HEAD(&head->list);
fd92c77f
MG		 3737 ret = init_srcu_struct(&head->srcu);
3738 if (ret)
3739 goto out_ida_remove;
ed754e5d
CH		 3740 head->subsys = ctrl->subsys;
3741 head->ns_id = nsid;
43fcd9e1 3742 head->ids = *ids;
ed754e5d
CH		 3743 kref_init(&head->ref);
3744
be93e87e
KB		 3745 if (head->ids.csi) {
3746 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3747 if (ret)
3748 goto out_cleanup_srcu;
3749 } else
3750 head->effects = ctrl->effects;
3751
32acab31
CH		 3752 ret = nvme_mpath_alloc_disk(ctrl, head);
3753 if (ret)
3754 goto out_cleanup_srcu;
3755
ed754e5d 3756 list_add_tail(&head->entry, &ctrl->subsys->nsheads);
12d9f070
JW		 3757
3758 kref_get(&ctrl->subsys->ref);
3759
ed754e5d
CH		 3760 return head;
3761out_cleanup_srcu:
3762 cleanup_srcu_struct(&head->srcu);
fd92c77f 3763out_ida_remove:
8b850475 3764 ida_free(&ctrl->subsys->ns_ida, head->instance);
ed754e5d
CH		 3765out_free_head:
3766 kfree(head);
3767out:
538af88e
SG		 3768 if (ret > 0)
3769 ret = blk_status_to_errno(nvme_error_status(ret));
ed754e5d
CH		 3770 return ERR_PTR(ret);
3771}
3772
2079f41e
CH		 3773static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
3774 struct nvme_ns_ids *ids)
3775{
3776 struct nvme_subsystem *s;
3777 int ret = 0;
3778
3779 /*
3780 * Note that this check is racy as we try to avoid holding the global
3781 * lock over the whole ns_head creation. But it is only intended as
3782 * a sanity check anyway.
3783 */
3784 mutex_lock(&nvme_subsystems_lock);
3785 list_for_each_entry(s, &nvme_subsystems, entry) {
3786 if (s == this)
3787 continue;
3788 mutex_lock(&s->lock);
3789 ret = nvme_subsys_check_duplicate_ids(s, ids);
3790 mutex_unlock(&s->lock);
3791 if (ret)
3792 break;
3793 }
3794 mutex_unlock(&nvme_subsystems_lock);
3795
3796 return ret;
3797}
3798
ed754e5d 3799static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
8b7c0ff2 3800 struct nvme_ns_ids *ids, bool is_shared)
ed754e5d
CH		 3801{
3802 struct nvme_ctrl *ctrl = ns->ctrl;
ed754e5d 3803 struct nvme_ns_head *head = NULL;
2079f41e
CH		 3804 int ret;
3805
3806 ret = nvme_global_check_duplicate_ids(ctrl->subsys, ids);
3807 if (ret) {
3808 dev_err(ctrl->device,
3809 "globally duplicate IDs for nsid %d\n", nsid);
3810 return ret;
3811 }
ed754e5d
CH		 3812
3813 mutex_lock(&ctrl->subsys->lock);
5974ea7c 3814 head = nvme_find_ns_head(ctrl, nsid);
ed754e5d 3815 if (!head) {
e2d77d2e
CH		 3816 ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, ids);
3817 if (ret) {
3818 dev_err(ctrl->device,
2079f41e
CH		 3819 "duplicate IDs in subsystem for nsid %d\n",
3820 nsid);
e2d77d2e
CH		 3821 goto out_unlock;
3822 }
8b7c0ff2 3823 head = nvme_alloc_ns_head(ctrl, nsid, ids);
ed754e5d
CH		 3824 if (IS_ERR(head)) {
3825 ret = PTR_ERR(head);
3826 goto out_unlock;
3827 }
0c284db7 3828 head->shared = is_shared;
ed754e5d 3829 } else {
6623c5b3 3830 ret = -EINVAL;
0c284db7 3831 if (!is_shared || !head->shared) {
9ad1927a 3832 dev_err(ctrl->device,
6623c5b3
CH		 3833 "Duplicate unshared namespace %d\n", nsid);
3834 goto out_put_ns_head;
9ad1927a 3835 }
8b7c0ff2 3836 if (!nvme_ns_ids_equal(&head->ids, ids)) {
ed754e5d
CH		 3837 dev_err(ctrl->device,
3838 "IDs don't match for shared namespace %d\n",
3839 nsid);
6623c5b3 3840 goto out_put_ns_head;
ed754e5d 3841 }
ce8d7861
CH		 3842
3843 if (!multipath && !list_empty(&head->list)) {
3844 dev_warn(ctrl->device,
3845 "Found shared namespace %d, but multipathing not supported.\n",
3846 nsid);
3847 dev_warn_once(ctrl->device,
3848 "Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0\n.");
3849 }
ed754e5d
CH		 3850 }
3851
772ea326 3852 list_add_tail_rcu(&ns->siblings, &head->list);
ed754e5d 3853 ns->head = head;
6623c5b3
CH		 3854 mutex_unlock(&ctrl->subsys->lock);
3855 return 0;
ed754e5d 3856
6623c5b3
CH		 3857out_put_ns_head:
3858 nvme_put_ns_head(head);
ed754e5d
CH		 3859out_unlock:
3860 mutex_unlock(&ctrl->subsys->lock);
3861 return ret;
3862}
3863
24493b8b 3864struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
5bae7f73 3865{
32f0c4af 3866 struct nvme_ns *ns, *ret = NULL;
69d3b8ac 3867
765cc031 3868 down_read(&ctrl->namespaces_rwsem);
5bae7f73 3869 list_for_each_entry(ns, &ctrl->namespaces, list) {
ed754e5d 3870 if (ns->head->ns_id == nsid) {
4c74d1f8 3871 if (!nvme_get_ns(ns))
2dd41228 3872 continue;
32f0c4af
KB		 3873 ret = ns;
3874 break;
3875 }
ed754e5d 3876 if (ns->head->ns_id > nsid)
5bae7f73
CH		 3877 break;
3878 }
765cc031 3879 up_read(&ctrl->namespaces_rwsem);
32f0c4af 3880 return ret;
5bae7f73 3881}
24493b8b 3882EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
5bae7f73 3883
298ba0e3
CH		 3884/*
3885 * Add the namespace to the controller list while keeping the list ordered.
3886 */
3887static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
3888{
3889 struct nvme_ns *tmp;
3890
3891 list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
3892 if (tmp->head->ns_id < ns->head->ns_id) {
3893 list_add(&ns->list, &tmp->list);
3894 return;
3895 }
3896 }
3897 list_add(&ns->list, &ns->ctrl->namespaces);
3898}
3899
8b7c0ff2
CH		 3900static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
3901 struct nvme_ns_ids *ids)
5bae7f73
CH		 3902{
3903 struct nvme_ns *ns;
3904 struct gendisk *disk;
ac81bfa9 3905 struct nvme_id_ns *id;
9953ab0c 3906 int node = ctrl->numa_node;
5bae7f73 3907
8b7c0ff2 3908 if (nvme_identify_ns(ctrl, nsid, ids, &id))
fab72f5a
CH		 3909 return;
3910
5bae7f73
CH		 3911 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3912 if (!ns)
fab72f5a 3913 goto out_free_id;
5bae7f73 3914
5f432cce
CH		 3915 disk = blk_mq_alloc_disk(ctrl->tagset, ns);
3916 if (IS_ERR(disk))
ed754e5d 3917 goto out_free_ns;
5f432cce
CH		 3918 disk->fops = &nvme_bdev_ops;
3919 disk->private_data = ns;
3920
3921 ns->disk = disk;
3922 ns->queue = disk->queue;
e0596ab2 3923
7d30c81b 3924 if (ctrl->opts && ctrl->opts->data_digest)
1cb039f3 3925 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
958f2a0f 3926
8b904b5b 3927 blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
e0596ab2
LG		 3928 if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3929 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3930
5bae7f73 3931 ns->ctrl = ctrl;
5bae7f73 3932 kref_init(&ns->kref);
5bae7f73 3933
e1aaf5ca 3934 if (nvme_init_ns_head(ns, nsid, ids, id->nmic & NVME_NS_NMIC_SHARED))
5f432cce 3935 goto out_cleanup_disk;
ac81bfa9 3936
9953ab0c 3937 /*
b739e137
CH		 3938 * If multipathing is enabled, the device name for all disks and not
3939 * just those that represent shared namespaces needs to be based on the
3940 * subsystem instance. Using the controller instance for private
3941 * namespaces could lead to naming collisions between shared and private
3942 * namespaces if they don't use a common numbering scheme.
3943 *
3944 * If multipathing is not enabled, disk names must use the controller
3945 * instance as shared namespaces will show up as multiple block
3946 * devices.
9953ab0c 3947 */
b739e137
CH		 3948 if (ns->head->disk) {
3949 sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
3950 ctrl->instance, ns->head->instance);
3951 disk->flags |= GENHD_FL_HIDDEN;
3952 } else if (multipath) {
3953 sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
3954 ns->head->instance);
3955 } else {
9953ab0c
CH		 3956 sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
3957 ns->head->instance);
b739e137 3958 }
3dc87dd0 3959
81382f17 3960 if (nvme_update_ns_info(ns, id))
5f432cce 3961 goto out_unlink_ns;
5bae7f73 3962
765cc031 3963 down_write(&ctrl->namespaces_rwsem);
298ba0e3 3964 nvme_ns_add_to_ctrl_list(ns);
765cc031 3965 up_write(&ctrl->namespaces_rwsem);
d22524a4 3966 nvme_get_ctrl(ctrl);
ac81bfa9 3967
ab3994f6
LC		 3968 if (device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups))
3969 goto out_cleanup_ns_from_list;
3970
2637baed
MI		 3971 if (!nvme_ns_head_multipath(ns->head))
3972 nvme_add_ns_cdev(ns);
32acab31 3973
0d0b660f 3974 nvme_mpath_add_disk(ns, id);
a3646451 3975 nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
0d0b660f
CH		 3976 kfree(id);
3977
adce7e98 3978 return;
5f432cce 3979
ab3994f6
LC		 3980 out_cleanup_ns_from_list:
3981 nvme_put_ctrl(ctrl);
3982 down_write(&ctrl->namespaces_rwsem);
3983 list_del_init(&ns->list);
3984 up_write(&ctrl->namespaces_rwsem);
ed754e5d
CH		 3985 out_unlink_ns:
3986 mutex_lock(&ctrl->subsys->lock);
3987 list_del_rcu(&ns->siblings);
d5675729
KB		 3988 if (list_empty(&ns->head->list))
3989 list_del_init(&ns->head->entry);
ed754e5d 3990 mutex_unlock(&ctrl->subsys->lock);
a63b8370 3991 nvme_put_ns_head(ns->head);
5f432cce
CH		 3992 out_cleanup_disk:
3993 blk_cleanup_disk(disk);
5bae7f73
CH		 3994 out_free_ns:
3995 kfree(ns);
fab72f5a
CH		 3996 out_free_id:
3997 kfree(id);
5bae7f73
CH		 3998}
3999
4000static void nvme_ns_remove(struct nvme_ns *ns)
4001{
5396fdac
HR		 4002 bool last_path = false;
4003
646017a6
KB		 4004 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
4005 return;
69d3b8ac 4006
e7d65803 4007 clear_bit(NVME_NS_READY, &ns->flags);
0a05226a 4008 set_capacity(ns->disk, 0);
a3646451 4009 nvme_fault_inject_fini(&ns->fault_inject);
2181e455 4010
d6d67427
CL		 4011 /*
4012 * Ensure that !NVME_NS_READY is seen by other threads to prevent
4013 * this ns going back into current_path.
4014 */
4015 synchronize_srcu(&ns->head->srcu);
4016
4017 /* wait for concurrent submissions */
4018 if (nvme_mpath_clear_current_path(ns))
4019 synchronize_srcu(&ns->head->srcu);
4020
2181e455
AE		 4021 mutex_lock(&ns->ctrl->subsys->lock);
4022 list_del_rcu(&ns->siblings);
9edceaf4
DW		 4023 if (list_empty(&ns->head->list)) {
4024 list_del_init(&ns->head->entry);
4025 last_path = true;
4026 }
2181e455 4027 mutex_unlock(&ns->ctrl->subsys->lock);
d5675729 4028
041bd1a1
DW		 4029 /* guarantee not available in head->list */
4030 synchronize_rcu();
4031
5eba2005
CH		 4032 if (!nvme_ns_head_multipath(ns->head))
4033 nvme_cdev_del(&ns->cdev, &ns->cdev_device);
4034 del_gendisk(ns->disk);
4035 blk_cleanup_queue(ns->queue);
32f0c4af 4036
765cc031 4037 down_write(&ns->ctrl->namespaces_rwsem);
5bae7f73 4038 list_del_init(&ns->list);
765cc031 4039 up_write(&ns->ctrl->namespaces_rwsem);
32f0c4af 4040
5396fdac
HR		 4041 if (last_path)
4042 nvme_mpath_shutdown_disk(ns->head);
5bae7f73
CH		 4043 nvme_put_ns(ns);
4044}
4045
4450ba3b
CH		 4046static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
4047{
4048 struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
4049
4050 if (ns) {
4051 nvme_ns_remove(ns);
4052 nvme_put_ns(ns);
4053 }
4054}
4055
0a05226a 4056static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_ids *ids)
b2dc748a 4057{
b2dc748a 4058 struct nvme_id_ns *id;
d95c1f41 4059 int ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
b2dc748a 4060
0a05226a
CH		 4061 if (test_bit(NVME_NS_DEAD, &ns->flags))
4062 goto out;
b2dc748a 4063
af5d6f7b 4064 ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, ids, &id);
b2dc748a
CH		 4065 if (ret)
4066 goto out;
4067
d95c1f41 4068 ret = NVME_SC_INVALID_NS | NVME_SC_DNR;
b2dc748a 4069 if (!nvme_ns_ids_equal(&ns->head->ids, ids)) {
af5d6f7b 4070 dev_err(ns->ctrl->device,
b2dc748a 4071 "identifiers changed for nsid %d\n", ns->head->ns_id);
0a05226a 4072 goto out_free_id;
b2dc748a
CH		 4073 }
4074
4075 ret = nvme_update_ns_info(ns, id);
0a05226a
CH		 4076
4077out_free_id:
b2dc748a
CH		 4078 kfree(id);
4079out:
4080 /*
0a05226a 4081 * Only remove the namespace if we got a fatal error back from the
b2dc748a 4082 * device, otherwise ignore the error and just move on.
0a05226a
CH		 4083 *
4084 * TODO: we should probably schedule a delayed retry here.
b2dc748a 4085 */
d95c1f41 4086 if (ret > 0 && (ret & NVME_SC_DNR))
0a05226a 4087 nvme_ns_remove(ns);
b2dc748a
CH		 4088}
4089
eba9bcf7 4090static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
540c801c 4091{
8b7c0ff2 4092 struct nvme_ns_ids ids = { };
540c801c
KB		 4093 struct nvme_ns *ns;
4094
8b7c0ff2
CH		 4095 if (nvme_identify_ns_descs(ctrl, nsid, &ids))
4096 return;
540c801c 4097
32f0c4af 4098 ns = nvme_find_get_ns(ctrl, nsid);
8b7c0ff2 4099 if (ns) {
0a05226a 4100 nvme_validate_ns(ns, &ids);
8b7c0ff2 4101 nvme_put_ns(ns);
b55d3d21
CH		 4102 return;
4103 }
4104
8b7c0ff2
CH		 4105 switch (ids.csi) {
4106 case NVME_CSI_NVM:
4107 nvme_alloc_ns(ctrl, nsid, &ids);
4108 break;
4109 case NVME_CSI_ZNS:
4110 if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
4111 dev_warn(ctrl->device,
4112 "nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
4113 nsid);
4114 break;
4115 }
0ec84df4
CK		 4116 if (!nvme_multi_css(ctrl)) {
4117 dev_warn(ctrl->device,
4118 "command set not reported for nsid: %d\n",
f4f9fc29 4119 nsid);
0ec84df4
CK		 4120 break;
4121 }
8b7c0ff2
CH		 4122 nvme_alloc_ns(ctrl, nsid, &ids);
4123 break;
4124 default:
4125 dev_warn(ctrl->device, "unknown csi %u for nsid %u\n",
4126 ids.csi, nsid);
4127 break;
4128 }
540c801c
KB		 4129}
4130
47b0e50a
SB		 4131static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4132 unsigned nsid)
4133{
4134 struct nvme_ns *ns, *next;
6f8e0d78 4135 LIST_HEAD(rm_list);
47b0e50a 4136
765cc031 4137 down_write(&ctrl->namespaces_rwsem);
47b0e50a 4138 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
cf39a6bc 4139 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
6f8e0d78 4140 list_move_tail(&ns->list, &rm_list);
47b0e50a 4141 }
765cc031 4142 up_write(&ctrl->namespaces_rwsem);
6f8e0d78
JW		 4143
4144 list_for_each_entry_safe(ns, next, &rm_list, list)
4145 nvme_ns_remove(ns);
4146
47b0e50a
SB		 4147}
4148
4005f28d 4149static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
540c801c 4150{
aec459b4 4151 const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
540c801c 4152 __le32 *ns_list;
4005f28d
CH		 4153 u32 prev = 0;
4154 int ret = 0, i;
540c801c 4155
25dcaa92
CH		 4156 if (nvme_ctrl_limited_cns(ctrl))
4157 return -EOPNOTSUPP;
540c801c 4158
42595eb7 4159 ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
540c801c
KB		 4160 if (!ns_list)
4161 return -ENOMEM;
4162
4005f28d 4163 for (;;) {
7b153362
CH		 4164 struct nvme_command cmd = {
4165 .identify.opcode = nvme_admin_identify,
4166 .identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST,
4167 .identify.nsid = cpu_to_le32(prev),
4168 };
4169
4170 ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4171 NVME_IDENTIFY_DATA_SIZE);
f781f3dd
MI		 4172 if (ret) {
4173 dev_warn(ctrl->device,
4174 "Identify NS List failed (status=0x%x)\n", ret);
47b0e50a 4175 goto free;
f781f3dd 4176 }
540c801c 4177
aec459b4 4178 for (i = 0; i < nr_entries; i++) {
4005f28d 4179 u32 nsid = le32_to_cpu(ns_list[i]);
540c801c 4180
4005f28d
CH		 4181 if (!nsid) /* end of the list? */
4182 goto out;
eba9bcf7 4183 nvme_validate_or_alloc_ns(ctrl, nsid);
4450ba3b
CH		 4184 while (++prev < nsid)
4185 nvme_ns_remove_by_nsid(ctrl, prev);
540c801c 4186 }
540c801c
KB		 4187 }
4188 out:
47b0e50a
SB		 4189 nvme_remove_invalid_namespaces(ctrl, prev);
4190 free:
540c801c
KB		 4191 kfree(ns_list);
4192 return ret;
4193}
4194
4005f28d 4195static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
5bae7f73 4196{
4005f28d
CH		 4197 struct nvme_id_ctrl *id;
4198 u32 nn, i;
4199
4200 if (nvme_identify_ctrl(ctrl, &id))
4201 return;
4202 nn = le32_to_cpu(id->nn);
4203 kfree(id);
5bae7f73 4204
540c801c 4205 for (i = 1; i <= nn; i++)
eba9bcf7 4206 nvme_validate_or_alloc_ns(ctrl, i);
540c801c 4207
47b0e50a 4208 nvme_remove_invalid_namespaces(ctrl, nn);
5bae7f73
CH		 4209}
4210
f493af37 4211static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
30d90964
CH		 4212{
4213 size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4214 __le32 *log;
f493af37 4215 int error;
30d90964
CH		 4216
4217 log = kzalloc(log_size, GFP_KERNEL);
4218 if (!log)
f493af37 4219 return;
30d90964 4220
f493af37
CH		 4221 /*
4222 * We need to read the log to clear the AEN, but we don't want to rely
4223 * on it for the changed namespace information as userspace could have
4224 * raced with us in reading the log page, which could cause us to miss
4225 * updates.
4226 */
be93e87e
KB		 4227 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4228 NVME_CSI_NVM, log, log_size, 0);
f493af37 4229 if (error)
30d90964
CH		 4230 dev_warn(ctrl->device,
4231 "reading changed ns log failed: %d\n", error);
30d90964 4232
30d90964 4233 kfree(log);
30d90964
CH		 4234}
4235
5955be21 4236static void nvme_scan_work(struct work_struct *work)
5bae7f73 4237{
5955be21
CH		 4238 struct nvme_ctrl *ctrl =
4239 container_of(work, struct nvme_ctrl, scan_work);
5bae7f73 4240
5d02a5c1
KB		 4241 /* No tagset on a live ctrl means IO queues could not created */
4242 if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
5955be21
CH		 4243 return;
4244
77016199 4245 if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
30d90964 4246 dev_info(ctrl->device, "rescanning namespaces.\n");
f493af37 4247 nvme_clear_changed_ns_log(ctrl);
30d90964
CH		 4248 }
4249
e7ad43c3 4250 mutex_lock(&ctrl->scan_lock);
4005f28d
CH		 4251 if (nvme_scan_ns_list(ctrl) != 0)
4252 nvme_scan_ns_sequential(ctrl);
e7ad43c3 4253 mutex_unlock(&ctrl->scan_lock);
5955be21 4254}
5bae7f73 4255
32f0c4af
KB		 4256/*
4257 * This function iterates the namespace list unlocked to allow recovery from
4258 * controller failure. It is up to the caller to ensure the namespace list is
4259 * not modified by scan work while this function is executing.
4260 */
5bae7f73
CH		 4261void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4262{
4263 struct nvme_ns *ns, *next;
6f8e0d78 4264 LIST_HEAD(ns_list);
5bae7f73 4265
0157ec8d
SG		 4266 /*
4267 * make sure to requeue I/O to all namespaces as these
4268 * might result from the scan itself and must complete
4269 * for the scan_work to make progress
4270 */
4271 nvme_mpath_clear_ctrl_paths(ctrl);
4272
f6c8e432
SG		 4273 /* prevent racing with ns scanning */
4274 flush_work(&ctrl->scan_work);
4275
0ff9d4e1
KB		 4276 /*
4277 * The dead states indicates the controller was not gracefully
4278 * disconnected. In that case, we won't be able to flush any data while
4279 * removing the namespaces' disks; fail all the queues now to avoid
4280 * potentially having to clean up the failed sync later.
4281 */
4282 if (ctrl->state == NVME_CTRL_DEAD)
4283 nvme_kill_queues(ctrl);
4284
ecca390e
SG		 4285 /* this is a no-op when called from the controller reset handler */
4286 nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4287
765cc031 4288 down_write(&ctrl->namespaces_rwsem);
6f8e0d78 4289 list_splice_init(&ctrl->namespaces, &ns_list);
765cc031 4290 up_write(&ctrl->namespaces_rwsem);
6f8e0d78
JW		 4291
4292 list_for_each_entry_safe(ns, next, &ns_list, list)
5bae7f73
CH		 4293 nvme_ns_remove(ns);
4294}
576d55d6 4295EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
5bae7f73 4296
a42f42e5
SG		 4297static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4298{
4299 struct nvme_ctrl *ctrl =
4300 container_of(dev, struct nvme_ctrl, ctrl_device);
4301 struct nvmf_ctrl_options *opts = ctrl->opts;
4302 int ret;
4303
4304 ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4305 if (ret)
4306 return ret;
4307
4308 if (opts) {
4309 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4310 if (ret)
4311 return ret;
4312
4313 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4314 opts->trsvcid ?: "none");
4315 if (ret)
4316 return ret;
4317
4318 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4319 opts->host_traddr ?: "none");
3ede8f72
MB		 4320 if (ret)
4321 return ret;
4322
4323 ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4324 opts->host_iface ?: "none");
a42f42e5
SG		 4325 }
4326 return ret;
4327}
4328
20d64911
MB		 4329static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4330{
4331 char *envp[2] = { envdata, NULL };
4332
4333 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4334}
4335
e3d7874d
KB		 4336static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4337{
4338 char *envp[2] = { NULL, NULL };
4339 u32 aen_result = ctrl->aen_result;
4340
4341 ctrl->aen_result = 0;
4342 if (!aen_result)
4343 return;
4344
4345 envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4346 if (!envp[0])
4347 return;
4348 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4349 kfree(envp[0]);
4350}
4351
f866fc42
CH		 4352static void nvme_async_event_work(struct work_struct *work)
4353{
4354 struct nvme_ctrl *ctrl =
4355 container_of(work, struct nvme_ctrl, async_event_work);
4356
e3d7874d 4357 nvme_aen_uevent(ctrl);
0fa0f99f
SG		 4358
4359 /*
4360 * The transport drivers must guarantee AER submission here is safe by
4361 * flushing ctrl async_event_work after changing the controller state
4362 * from LIVE and before freeing the admin queue.
4363 */
4364 if (ctrl->state == NVME_CTRL_LIVE)
4365 ctrl->ops->submit_async_event(ctrl);
f866fc42
CH		 4366}
4367
b6dccf7f
AD		 4368static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4369{
4370
4371 u32 csts;
4372
4373 if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4374 return false;
4375
4376 if (csts == ~0)
4377 return false;
4378
4379 return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4380}
4381
4382static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4383{
b6dccf7f
AD		 4384 struct nvme_fw_slot_info_log *log;
4385
4386 log = kmalloc(sizeof(*log), GFP_KERNEL);
4387 if (!log)
4388 return;
4389
be93e87e
KB		 4390 if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4391 log, sizeof(*log), 0))
0e98719b 4392 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
b6dccf7f
AD		 4393 kfree(log);
4394}
4395
4396static void nvme_fw_act_work(struct work_struct *work)
4397{
4398 struct nvme_ctrl *ctrl = container_of(work,
4399 struct nvme_ctrl, fw_act_work);
4400 unsigned long fw_act_timeout;
4401
4402 if (ctrl->mtfa)
4403 fw_act_timeout = jiffies +
4404 msecs_to_jiffies(ctrl->mtfa * 100);
4405 else
4406 fw_act_timeout = jiffies +
4407 msecs_to_jiffies(admin_timeout * 1000);
4408
4409 nvme_stop_queues(ctrl);
4410 while (nvme_ctrl_pp_status(ctrl)) {
4411 if (time_after(jiffies, fw_act_timeout)) {
4412 dev_warn(ctrl->device,
4413 "Fw activation timeout, reset controller\n");
4c75f877
KB		 4414 nvme_try_sched_reset(ctrl);
4415 return;
b6dccf7f
AD		 4416 }
4417 msleep(100);
4418 }
4419
4c75f877 4420 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
b6dccf7f
AD		 4421 return;
4422
4423 nvme_start_queues(ctrl);
a806c6c8 4424 /* read FW slot information to clear the AER */
b6dccf7f
AD		 4425 nvme_get_fw_slot_info(ctrl);
4426}
4427
868c2392
CH		 4428static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4429{
09bd1ff4
CK		 4430 u32 aer_notice_type = (result & 0xff00) >> 8;
4431
521cfb8e
CK		 4432 trace_nvme_async_event(ctrl, aer_notice_type);
4433
09bd1ff4 4434 switch (aer_notice_type) {
868c2392 4435 case NVME_AER_NOTICE_NS_CHANGED:
77016199 4436 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
868c2392
CH		 4437 nvme_queue_scan(ctrl);
4438 break;
4439 case NVME_AER_NOTICE_FW_ACT_STARTING:
4c75f877
KB		 4440 /*
4441 * We are (ab)using the RESETTING state to prevent subsequent
4442 * recovery actions from interfering with the controller's
4443 * firmware activation.
4444 */
4445 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4446 queue_work(nvme_wq, &ctrl->fw_act_work);
868c2392 4447 break;
0d0b660f
CH		 4448#ifdef CONFIG_NVME_MULTIPATH
4449 case NVME_AER_NOTICE_ANA:
4450 if (!ctrl->ana_log_buf)
4451 break;
4452 queue_work(nvme_wq, &ctrl->ana_work);
4453 break;
4454#endif
85f8a435
SG		 4455 case NVME_AER_NOTICE_DISC_CHANGED:
4456 ctrl->aen_result = result;
4457 break;
868c2392
CH		 4458 default:
4459 dev_warn(ctrl->device, "async event result %08x\n", result);
4460 }
4461}
4462
7bf58533 4463void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
287a63eb 4464 volatile union nvme_result *res)
f866fc42 4465{
7bf58533 4466 u32 result = le32_to_cpu(res->u32);
09bd1ff4 4467 u32 aer_type = result & 0x07;
f866fc42 4468
ad22c355 4469 if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
f866fc42
CH		 4470 return;
4471
09bd1ff4 4472 switch (aer_type) {
868c2392
CH		 4473 case NVME_AER_NOTICE:
4474 nvme_handle_aen_notice(ctrl, result);
4475 break;
e3d7874d
KB		 4476 case NVME_AER_ERROR:
4477 case NVME_AER_SMART:
4478 case NVME_AER_CSS:
4479 case NVME_AER_VS:
09bd1ff4 4480 trace_nvme_async_event(ctrl, aer_type);
e3d7874d 4481 ctrl->aen_result = result;
7bf58533
CH		 4482 break;
4483 default:
4484 break;
f866fc42 4485 }
c669ccdc 4486 queue_work(nvme_wq, &ctrl->async_event_work);
f866fc42 4487}
f866fc42 4488EXPORT_SYMBOL_GPL(nvme_complete_async_event);
f3ca80fc 4489
d09f2b45 4490void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
576d55d6 4491{
0d0b660f 4492 nvme_mpath_stop(ctrl);
d09f2b45 4493 nvme_stop_keep_alive(ctrl);
8c4dfea9 4494 nvme_stop_failfast_work(ctrl);
f866fc42 4495 flush_work(&ctrl->async_event_work);
b6dccf7f 4496 cancel_work_sync(&ctrl->fw_act_work);
d09f2b45
SG		 4497}
4498EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4499
4500void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4501{
5887450b 4502 nvme_start_keep_alive(ctrl);
d09f2b45 4503
93da4023
SG		 4504 nvme_enable_aen(ctrl);
4505
d09f2b45
SG		 4506 if (ctrl->queue_count > 1) {
4507 nvme_queue_scan(ctrl);
d09f2b45 4508 nvme_start_queues(ctrl);
a4a6f3c8 4509 nvme_mpath_update(ctrl);
d09f2b45 4510 }
20d64911
MB		 4511
4512 nvme_change_uevent(ctrl, "NVME_EVENT=connected");
d09f2b45
SG		 4513}
4514EXPORT_SYMBOL_GPL(nvme_start_ctrl);
5955be21 4515
d09f2b45
SG		 4516void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4517{
ed7770f6 4518 nvme_hwmon_exit(ctrl);
f79d5fda 4519 nvme_fault_inject_fini(&ctrl->fault_inject);
510a405d 4520 dev_pm_qos_hide_latency_tolerance(ctrl->device);
a6a5149b 4521 cdev_device_del(&ctrl->cdev, ctrl->device);
726612b6 4522 nvme_put_ctrl(ctrl);
53029b04 4523}
576d55d6 4524EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
53029b04 4525
8168d23f
KB		 4526static void nvme_free_cels(struct nvme_ctrl *ctrl)
4527{
4528 struct nvme_effects_log *cel;
4529 unsigned long i;
4530
8f8ea928 4531 xa_for_each(&ctrl->cels, i, cel) {
8168d23f
KB		 4532 xa_erase(&ctrl->cels, i);
4533 kfree(cel);
4534 }
4535
4536 xa_destroy(&ctrl->cels);
4537}
4538
d22524a4 4539static void nvme_free_ctrl(struct device *dev)
53029b04 4540{
d22524a4
CH		 4541 struct nvme_ctrl *ctrl =
4542 container_of(dev, struct nvme_ctrl, ctrl_device);
ab9e00cc 4543 struct nvme_subsystem *subsys = ctrl->subsys;
f3ca80fc 4544
192f6c29 4545 if (!subsys || ctrl->instance != subsys->instance)
8b850475 4546 ida_free(&nvme_instance_ida, ctrl->instance);
733e4b69 4547
8168d23f 4548 nvme_free_cels(ctrl);
0d0b660f 4549 nvme_mpath_uninit(ctrl);
092ff052 4550 __free_page(ctrl->discard_page);
f3ca80fc 4551
ab9e00cc 4552 if (subsys) {
32fd90c4 4553 mutex_lock(&nvme_subsystems_lock);
ab9e00cc 4554 list_del(&ctrl->subsys_entry);
ab9e00cc 4555 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
32fd90c4 4556 mutex_unlock(&nvme_subsystems_lock);
ab9e00cc 4557 }
f3ca80fc
CH		 4558
4559 ctrl->ops->free_ctrl(ctrl);
f3ca80fc 4560
ab9e00cc
CH		 4561 if (subsys)
4562 nvme_put_subsystem(subsys);
f3ca80fc
CH		 4563}
4564
4565/*
4566 * Initialize a NVMe controller structures. This needs to be called during
4567 * earliest initialization so that we have the initialized structured around
4568 * during probing.
4569 */
4570int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4571 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4572{
4573 int ret;
4574
bb8d261e 4575 ctrl->state = NVME_CTRL_NEW;
8c4dfea9 4576 clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
bb8d261e 4577 spin_lock_init(&ctrl->lock);
e7ad43c3 4578 mutex_init(&ctrl->scan_lock);
f3ca80fc 4579 INIT_LIST_HEAD(&ctrl->namespaces);
1cf7a12e 4580 xa_init(&ctrl->cels);
765cc031 4581 init_rwsem(&ctrl->namespaces_rwsem);
f3ca80fc
CH		 4582 ctrl->dev = dev;
4583 ctrl->ops = ops;
4584 ctrl->quirks = quirks;
4fea243e 4585 ctrl->numa_node = NUMA_NO_NODE;
5955be21 4586 INIT_WORK(&ctrl->scan_work, nvme_scan_work);
f866fc42 4587 INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
b6dccf7f 4588 INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
c5017e85 4589 INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
c1ac9a4b 4590 init_waitqueue_head(&ctrl->state_wq);
f3ca80fc 4591
230f1f9e 4592 INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
8c4dfea9 4593 INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
230f1f9e
JS		 4594 memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4595 ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4596
cb5b7262
JA		 4597 BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4598 PAGE_SIZE);
4599 ctrl->discard_page = alloc_page(GFP_KERNEL);
4600 if (!ctrl->discard_page) {
4601 ret = -ENOMEM;
4602 goto out;
4603 }
4604
8b850475 4605 ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
9843f685 4606 if (ret < 0)
f3ca80fc 4607 goto out;
9843f685 4608 ctrl->instance = ret;
f3ca80fc 4609
d22524a4
CH		 4610 device_initialize(&ctrl->ctrl_device);
4611 ctrl->device = &ctrl->ctrl_device;
f68abd9c
JG		 4612 ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
4613 ctrl->instance);
d22524a4
CH		 4614 ctrl->device->class = nvme_class;
4615 ctrl->device->parent = ctrl->dev;
4616 ctrl->device->groups = nvme_dev_attr_groups;
4617 ctrl->device->release = nvme_free_ctrl;
4618 dev_set_drvdata(ctrl->device, ctrl);
4619 ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4620 if (ret)
f3ca80fc 4621 goto out_release_instance;
f3ca80fc 4622
b780d741 4623 nvme_get_ctrl(ctrl);
a6a5149b
CH		 4624 cdev_init(&ctrl->cdev, &nvme_dev_fops);
4625 ctrl->cdev.owner = ops->module;
4626 ret = cdev_device_add(&ctrl->cdev, ctrl->device);
d22524a4
CH		 4627 if (ret)
4628 goto out_free_name;
f3ca80fc 4629
c5552fde
AL		 4630 /*
4631 * Initialize latency tolerance controls. The sysfs files won't
4632 * be visible to userspace unless the device actually supports APST.
4633 */
4634 ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4635 dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4636 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4637
f79d5fda 4638 nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
5e1f6899 4639 nvme_mpath_init_ctrl(ctrl);
f79d5fda 4640
f3ca80fc 4641 return 0;
d22524a4 4642out_free_name:
b780d741 4643 nvme_put_ctrl(ctrl);
d6a2b953 4644 kfree_const(ctrl->device->kobj.name);
f3ca80fc 4645out_release_instance:
8b850475 4646 ida_free(&nvme_instance_ida, ctrl->instance);
f3ca80fc 4647out:
cb5b7262
JA		 4648 if (ctrl->discard_page)
4649 __free_page(ctrl->discard_page);
f3ca80fc
CH		 4650 return ret;
4651}
576d55d6 4652EXPORT_SYMBOL_GPL(nvme_init_ctrl);
f3ca80fc 4653
ebc9b952
ML		 4654static void nvme_start_ns_queue(struct nvme_ns *ns)
4655{
9e6a6b12
ML		 4656 if (test_and_clear_bit(NVME_NS_STOPPED, &ns->flags))
4657 blk_mq_unquiesce_queue(ns->queue);
ebc9b952
ML		 4658}
4659
4660static void nvme_stop_ns_queue(struct nvme_ns *ns)
4661{
9e6a6b12
ML		 4662 if (!test_and_set_bit(NVME_NS_STOPPED, &ns->flags))
4663 blk_mq_quiesce_queue(ns->queue);
26af1cd0
ML		 4664 else
4665 blk_mq_wait_quiesce_done(ns->queue);
ebc9b952
ML		 4666}
4667
4668/*
4669 * Prepare a queue for teardown.
4670 *
4671 * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
4672 * the capacity to 0 after that to avoid blocking dispatchers that may be
4673 * holding bd_butex. This will end buffered writers dirtying pages that can't
4674 * be synced.
4675 */
4676static void nvme_set_queue_dying(struct nvme_ns *ns)
4677{
4678 if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
4679 return;
4680
7a5428dc 4681 blk_mark_disk_dead(ns->disk);
ebc9b952
ML		 4682 nvme_start_ns_queue(ns);
4683
4684 set_capacity_and_notify(ns->disk, 0);
4685}
4686
69d9a99c
KB		 4687/**
4688 * nvme_kill_queues(): Ends all namespace queues
4689 * @ctrl: the dead controller that needs to end
4690 *
4691 * Call this function when the driver determines it is unable to get the
4692 * controller in a state capable of servicing IO.
4693 */
4694void nvme_kill_queues(struct nvme_ctrl *ctrl)
4695{
4696 struct nvme_ns *ns;
4697
765cc031 4698 down_read(&ctrl->namespaces_rwsem);
82654b6b 4699
443bd90f 4700 /* Forcibly unquiesce queues to avoid blocking dispatch */
751a0cc0 4701 if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
6ca1d902 4702 nvme_start_admin_queue(ctrl);
443bd90f 4703
cf39a6bc
SB		 4704 list_for_each_entry(ns, &ctrl->namespaces, list)
4705 nvme_set_queue_dying(ns);
806f026f 4706
765cc031 4707 up_read(&ctrl->namespaces_rwsem);
69d9a99c 4708}
237045fc 4709EXPORT_SYMBOL_GPL(nvme_kill_queues);
69d9a99c 4710
302ad8cc
KB		 4711void nvme_unfreeze(struct nvme_ctrl *ctrl)
4712{
4713 struct nvme_ns *ns;
4714
765cc031 4715 down_read(&ctrl->namespaces_rwsem);
302ad8cc
KB		 4716 list_for_each_entry(ns, &ctrl->namespaces, list)
4717 blk_mq_unfreeze_queue(ns->queue);
765cc031 4718 up_read(&ctrl->namespaces_rwsem);
302ad8cc
KB		 4719}
4720EXPORT_SYMBOL_GPL(nvme_unfreeze);
4721
7cf0d7c0 4722int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
302ad8cc
KB		 4723{
4724 struct nvme_ns *ns;
4725
765cc031 4726 down_read(&ctrl->namespaces_rwsem);
302ad8cc
KB		 4727 list_for_each_entry(ns, &ctrl->namespaces, list) {
4728 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4729 if (timeout <= 0)
4730 break;
4731 }
765cc031 4732 up_read(&ctrl->namespaces_rwsem);
7cf0d7c0 4733 return timeout;
302ad8cc
KB		 4734}
4735EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4736
4737void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4738{
4739 struct nvme_ns *ns;
4740
765cc031 4741 down_read(&ctrl->namespaces_rwsem);
302ad8cc
KB		 4742 list_for_each_entry(ns, &ctrl->namespaces, list)
4743 blk_mq_freeze_queue_wait(ns->queue);
765cc031 4744 up_read(&ctrl->namespaces_rwsem);
302ad8cc
KB		 4745}
4746EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4747
4748void nvme_start_freeze(struct nvme_ctrl *ctrl)
4749{
4750 struct nvme_ns *ns;
4751
765cc031 4752 down_read(&ctrl->namespaces_rwsem);
302ad8cc 4753 list_for_each_entry(ns, &ctrl->namespaces, list)
1671d522 4754 blk_freeze_queue_start(ns->queue);
765cc031 4755 up_read(&ctrl->namespaces_rwsem);
302ad8cc
KB		 4756}
4757EXPORT_SYMBOL_GPL(nvme_start_freeze);
4758
25646264 4759void nvme_stop_queues(struct nvme_ctrl *ctrl)
363c9aac
SG		 4760{
4761 struct nvme_ns *ns;
4762
765cc031 4763 down_read(&ctrl->namespaces_rwsem);
a6eaa884 4764 list_for_each_entry(ns, &ctrl->namespaces, list)
ebc9b952 4765 nvme_stop_ns_queue(ns);
765cc031 4766 up_read(&ctrl->namespaces_rwsem);
363c9aac 4767}
576d55d6 4768EXPORT_SYMBOL_GPL(nvme_stop_queues);
363c9aac 4769
25646264 4770void nvme_start_queues(struct nvme_ctrl *ctrl)
363c9aac
SG		 4771{
4772 struct nvme_ns *ns;
4773
765cc031 4774 down_read(&ctrl->namespaces_rwsem);
8d7b8faf 4775 list_for_each_entry(ns, &ctrl->namespaces, list)
ebc9b952 4776 nvme_start_ns_queue(ns);
765cc031 4777 up_read(&ctrl->namespaces_rwsem);
363c9aac 4778}
576d55d6 4779EXPORT_SYMBOL_GPL(nvme_start_queues);
363c9aac 4780
a277654b
ML		 4781void nvme_stop_admin_queue(struct nvme_ctrl *ctrl)
4782{
9e6a6b12
ML		 4783 if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4784 blk_mq_quiesce_queue(ctrl->admin_q);
26af1cd0
ML		 4785 else
4786 blk_mq_wait_quiesce_done(ctrl->admin_q);
a277654b
ML		 4787}
4788EXPORT_SYMBOL_GPL(nvme_stop_admin_queue);
4789
4790void nvme_start_admin_queue(struct nvme_ctrl *ctrl)
4791{
9e6a6b12
ML		 4792 if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4793 blk_mq_unquiesce_queue(ctrl->admin_q);
a277654b
ML		 4794}
4795EXPORT_SYMBOL_GPL(nvme_start_admin_queue);
4796
04800fbf 4797void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
d6135c3a
KB		 4798{
4799 struct nvme_ns *ns;
4800
4801 down_read(&ctrl->namespaces_rwsem);
4802 list_for_each_entry(ns, &ctrl->namespaces, list)
4803 blk_sync_queue(ns->queue);
4804 up_read(&ctrl->namespaces_rwsem);
04800fbf
CL		 4805}
4806EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
03894b7a 4807
04800fbf
CL		 4808void nvme_sync_queues(struct nvme_ctrl *ctrl)
4809{
4810 nvme_sync_io_queues(ctrl);
03894b7a
EN		 4811 if (ctrl->admin_q)
4812 blk_sync_queue(ctrl->admin_q);
d6135c3a
KB		 4813}
4814EXPORT_SYMBOL_GPL(nvme_sync_queues);
4815
b2702aaa 4816struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
f783f444 4817{
b2702aaa
CK		 4818 if (file->f_op != &nvme_dev_fops)
4819 return NULL;
4820 return file->private_data;
f783f444 4821}
b2702aaa 4822EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
f783f444 4823
81101540
CH		 4824/*
4825 * Check we didn't inadvertently grow the command structure sizes:
4826 */
4827static inline void _nvme_check_size(void)
4828{
4829 BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4830 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4831 BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4832 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4833 BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4834 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4835 BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4836 BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4837 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4838 BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4839 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4840 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4841 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
240e6ee2 4842 BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4020aad8 4843 BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
240e6ee2 4844 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
5befc7c2 4845 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
81101540
CH		 4846 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4847 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4848 BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4849 BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4020aad8 4850 BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
81101540
CH		 4851}
4852
4853
893a74b7 4854static int __init nvme_core_init(void)
5bae7f73 4855{
b227c59b 4856 int result = -ENOMEM;
5bae7f73 4857
81101540
CH		 4858 _nvme_check_size();
4859
9a6327d2
SG		 4860 nvme_wq = alloc_workqueue("nvme-wq",
4861 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4862 if (!nvme_wq)
b227c59b
RS		 4863 goto out;
4864
4865 nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4866 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4867 if (!nvme_reset_wq)
4868 goto destroy_wq;
4869
4870 nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4871 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4872 if (!nvme_delete_wq)
4873 goto destroy_reset_wq;
9a6327d2 4874
f68abd9c
JG		 4875 result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
4876 NVME_MINORS, "nvme");
f3ca80fc 4877 if (result < 0)
b227c59b 4878 goto destroy_delete_wq;
f3ca80fc
CH		 4879
4880 nvme_class = class_create(THIS_MODULE, "nvme");
4881 if (IS_ERR(nvme_class)) {
4882 result = PTR_ERR(nvme_class);
4883 goto unregister_chrdev;
4884 }
a42f42e5 4885 nvme_class->dev_uevent = nvme_class_uevent;
f3ca80fc 4886
ab9e00cc
CH		 4887 nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4888 if (IS_ERR(nvme_subsys_class)) {
4889 result = PTR_ERR(nvme_subsys_class);
4890 goto destroy_class;
4891 }
2637baed
MI		 4892
4893 result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
4894 "nvme-generic");
4895 if (result < 0)
4896 goto destroy_subsys_class;
4897
4898 nvme_ns_chr_class = class_create(THIS_MODULE, "nvme-generic");
4899 if (IS_ERR(nvme_ns_chr_class)) {
4900 result = PTR_ERR(nvme_ns_chr_class);
4901 goto unregister_generic_ns;
4902 }
4903
5bae7f73 4904 return 0;
f3ca80fc 4905
2637baed
MI		 4906unregister_generic_ns:
4907 unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
4908destroy_subsys_class:
4909 class_destroy(nvme_subsys_class);
ab9e00cc
CH		 4910destroy_class:
4911 class_destroy(nvme_class);
9a6327d2 4912unregister_chrdev:
f68abd9c 4913 unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
b227c59b
RS		 4914destroy_delete_wq:
4915 destroy_workqueue(nvme_delete_wq);
4916destroy_reset_wq:
4917 destroy_workqueue(nvme_reset_wq);
9a6327d2
SG		 4918destroy_wq:
4919 destroy_workqueue(nvme_wq);
b227c59b 4920out:
f3ca80fc 4921 return result;
5bae7f73
CH		 4922}
4923
893a74b7 4924static void __exit nvme_core_exit(void)
5bae7f73 4925{
2637baed 4926 class_destroy(nvme_ns_chr_class);
ab9e00cc 4927 class_destroy(nvme_subsys_class);
f3ca80fc 4928 class_destroy(nvme_class);
2637baed 4929 unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
f68abd9c 4930 unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
b227c59b
RS		 4931 destroy_workqueue(nvme_delete_wq);
4932 destroy_workqueue(nvme_reset_wq);
9a6327d2 4933 destroy_workqueue(nvme_wq);
2637baed 4934 ida_destroy(&nvme_ns_chr_minor_ida);
f41cfd5d 4935 ida_destroy(&nvme_instance_ida);
5bae7f73 4936}
576d55d6
ML		 4937
4938MODULE_LICENSE("GPL");
4939MODULE_VERSION("1.0");
4940module_init(nvme_core_init);
4941module_exit(nvme_core_exit);
Linux 6.x block layer and io_uring tree(s)