Commit | Line | Data |
---|---|---|
bc50ad75 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
f11bb3e2 CH |
2 | /* |
3 | * Copyright (c) 2011-2014, Intel Corporation. | |
f11bb3e2 CH |
4 | */ |
5 | ||
6 | #ifndef _NVME_H | |
7 | #define _NVME_H | |
8 | ||
9 | #include <linux/nvme.h> | |
a6a5149b | 10 | #include <linux/cdev.h> |
f11bb3e2 CH |
11 | #include <linux/pci.h> |
12 | #include <linux/kref.h> | |
13 | #include <linux/blk-mq.h> | |
a98e58e5 | 14 | #include <linux/sed-opal.h> |
b9e03857 | 15 | #include <linux/fault-inject.h> |
978628ec | 16 | #include <linux/rcupdate.h> |
c1ac9a4b | 17 | #include <linux/wait.h> |
4d2ce688 | 18 | #include <linux/t10-pi.h> |
a1a825ab | 19 | #include <linux/ratelimit_types.h> |
f11bb3e2 | 20 | |
35fe0d12 HR |
21 | #include <trace/events/block.h> |
22 | ||
b668f2f5 MC |
23 | extern const struct pr_ops nvme_pr_ops; |
24 | ||
8ae4e447 | 25 | extern unsigned int nvme_io_timeout; |
f11bb3e2 CH |
26 | #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) |
27 | ||
8ae4e447 | 28 | extern unsigned int admin_timeout; |
dc96f938 | 29 | #define NVME_ADMIN_TIMEOUT (admin_timeout * HZ) |
21d34711 | 30 | |
038bd4cb | 31 | #define NVME_DEFAULT_KATO 5 |
038bd4cb | 32 | |
38e18002 IR |
33 | #ifdef CONFIG_ARCH_NO_SG_CHAIN |
34 | #define NVME_INLINE_SG_CNT 0 | |
ba7ca2ae | 35 | #define NVME_INLINE_METADATA_SG_CNT 0 |
38e18002 IR |
36 | #else |
37 | #define NVME_INLINE_SG_CNT 2 | |
ba7ca2ae | 38 | #define NVME_INLINE_METADATA_SG_CNT 1 |
38e18002 IR |
39 | #endif |
40 | ||
6c3c05b0 CK |
41 | /* |
42 | * Default to a 4K page size, with the intention to update this | |
43 | * path in the future to accommodate architectures with differing | |
44 | * kernel and IO page sizes. | |
45 | */ | |
46 | #define NVME_CTRL_PAGE_SHIFT 12 | |
47 | #define NVME_CTRL_PAGE_SIZE (1 << NVME_CTRL_PAGE_SHIFT) | |
48 | ||
9a6327d2 | 49 | extern struct workqueue_struct *nvme_wq; |
b227c59b RS |
50 | extern struct workqueue_struct *nvme_reset_wq; |
51 | extern struct workqueue_struct *nvme_delete_wq; | |
f227345f | 52 | extern struct mutex nvme_subsystems_lock; |
9a6327d2 | 53 | |
f11bb3e2 | 54 | /* |
106198ed CH |
55 | * List of workarounds for devices that required behavior not specified in |
56 | * the standard. | |
f11bb3e2 | 57 | */ |
106198ed CH |
58 | enum nvme_quirks { |
59 | /* | |
60 | * Prefers I/O aligned to a stripe size specified in a vendor | |
61 | * specific Identify field. | |
62 | */ | |
63 | NVME_QUIRK_STRIPE_SIZE = (1 << 0), | |
540c801c KB |
64 | |
65 | /* | |
66 | * The controller doesn't handle Identify value others than 0 or 1 | |
67 | * correctly. | |
68 | */ | |
69 | NVME_QUIRK_IDENTIFY_CNS = (1 << 1), | |
08095e70 KB |
70 | |
71 | /* | |
e850fd16 CH |
72 | * The controller deterministically returns O's on reads to |
73 | * logical blocks that deallocate was called on. | |
08095e70 | 74 | */ |
e850fd16 | 75 | NVME_QUIRK_DEALLOCATE_ZEROES = (1 << 2), |
54adc010 GP |
76 | |
77 | /* | |
78 | * The controller needs a delay before starts checking the device | |
79 | * readiness, which is done by reading the NVME_CSTS_RDY bit. | |
80 | */ | |
81 | NVME_QUIRK_DELAY_BEFORE_CHK_RDY = (1 << 3), | |
c5552fde AL |
82 | |
83 | /* | |
84 | * APST should not be used. | |
85 | */ | |
86 | NVME_QUIRK_NO_APST = (1 << 4), | |
ff5350a8 AL |
87 | |
88 | /* | |
89 | * The deepest sleep state should not be used. | |
90 | */ | |
91 | NVME_QUIRK_NO_DEEPEST_PS = (1 << 5), | |
608cc4b1 | 92 | |
83bdfcbd KB |
93 | /* |
94 | * Problems seen with concurrent commands | |
95 | */ | |
96 | NVME_QUIRK_QDEPTH_ONE = (1 << 6), | |
97 | ||
9abd68ef JA |
98 | /* |
99 | * Set MEDIUM priority on SQ creation | |
100 | */ | |
101 | NVME_QUIRK_MEDIUM_PRIO_SQ = (1 << 7), | |
6299358d JD |
102 | |
103 | /* | |
104 | * Ignore device provided subnqn. | |
105 | */ | |
106 | NVME_QUIRK_IGNORE_DEV_SUBNQN = (1 << 8), | |
7b210e4e CH |
107 | |
108 | /* | |
109 | * Broken Write Zeroes. | |
110 | */ | |
111 | NVME_QUIRK_DISABLE_WRITE_ZEROES = (1 << 9), | |
cb32de1b ML |
112 | |
113 | /* | |
114 | * Force simple suspend/resume path. | |
115 | */ | |
116 | NVME_QUIRK_SIMPLE_SUSPEND = (1 << 10), | |
7ad67ca5 | 117 | |
66341331 BH |
118 | /* |
119 | * Use only one interrupt vector for all queues | |
120 | */ | |
7ad67ca5 | 121 | NVME_QUIRK_SINGLE_VECTOR = (1 << 11), |
66341331 BH |
122 | |
123 | /* | |
124 | * Use non-standard 128 bytes SQEs. | |
125 | */ | |
7ad67ca5 | 126 | NVME_QUIRK_128_BYTES_SQES = (1 << 12), |
d38e9f04 BH |
127 | |
128 | /* | |
129 | * Prevent tag overlap between queues | |
130 | */ | |
7ad67ca5 | 131 | NVME_QUIRK_SHARED_TAGS = (1 << 13), |
6c6aa2f2 AM |
132 | |
133 | /* | |
134 | * Don't change the value of the temperature threshold feature | |
135 | */ | |
136 | NVME_QUIRK_NO_TEMP_THRESH_CHANGE = (1 << 14), | |
5bedd3af CH |
137 | |
138 | /* | |
139 | * The controller doesn't handle the Identify Namespace | |
140 | * Identification Descriptor list subcommand despite claiming | |
141 | * NVMe 1.3 compliance. | |
142 | */ | |
143 | NVME_QUIRK_NO_NS_DESC_LIST = (1 << 15), | |
4bdf2603 FS |
144 | |
145 | /* | |
146 | * The controller does not properly handle DMA addresses over | |
147 | * 48 bits. | |
148 | */ | |
149 | NVME_QUIRK_DMA_ADDRESS_BITS_48 = (1 << 16), | |
a2941f6a KB |
150 | |
151 | /* | |
b7df575f | 152 | * The controller requires the command_id value be limited, so skip |
a2941f6a KB |
153 | * encoding the generation sequence number. |
154 | */ | |
155 | NVME_QUIRK_SKIP_CID_GEN = (1 << 17), | |
00ff400e CH |
156 | |
157 | /* | |
158 | * Reports garbage in the namespace identifiers (eui64, nguid, uuid). | |
159 | */ | |
160 | NVME_QUIRK_BOGUS_NID = (1 << 18), | |
bd375fee HV |
161 | |
162 | /* | |
163 | * No temperature thresholds for channels other than 0 (Composite). | |
164 | */ | |
165 | NVME_QUIRK_NO_SECONDARY_TEMP_THRESH = (1 << 19), | |
107b4e06 GG |
166 | |
167 | /* | |
168 | * Disables simple suspend/resume path. | |
169 | */ | |
170 | NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND = (1 << 20), | |
d5887dc6 SA |
171 | |
172 | /* | |
173 | * MSI (but not MSI-X) interrupts are broken and never fire. | |
174 | */ | |
175 | NVME_QUIRK_BROKEN_MSI = (1 << 21), | |
106198ed CH |
176 | }; |
177 | ||
d49187e9 CH |
178 | /* |
179 | * Common request structure for NVMe passthrough. All drivers must have | |
180 | * this structure as the first member of their request-private data. | |
181 | */ | |
182 | struct nvme_request { | |
183 | struct nvme_command *cmd; | |
184 | union nvme_result result; | |
e7006de6 | 185 | u8 genctr; |
44e44b29 | 186 | u8 retries; |
27fa9bc5 CH |
187 | u8 flags; |
188 | u16 status; | |
d4d957b5 SG |
189 | #ifdef CONFIG_NVME_MULTIPATH |
190 | unsigned long start_time; | |
191 | #endif | |
59e29ce6 | 192 | struct nvme_ctrl *ctrl; |
27fa9bc5 CH |
193 | }; |
194 | ||
32acab31 CH |
195 | /* |
196 | * Mark a bio as coming in through the mpath node. | |
197 | */ | |
198 | #define REQ_NVME_MPATH REQ_DRV | |
199 | ||
27fa9bc5 CH |
200 | enum { |
201 | NVME_REQ_CANCELLED = (1 << 0), | |
bb06ec31 | 202 | NVME_REQ_USERCMD = (1 << 1), |
d4d957b5 | 203 | NVME_MPATH_IO_STATS = (1 << 2), |
f227345f | 204 | NVME_MPATH_CNT_ACTIVE = (1 << 3), |
d49187e9 CH |
205 | }; |
206 | ||
207 | static inline struct nvme_request *nvme_req(struct request *req) | |
208 | { | |
209 | return blk_mq_rq_to_pdu(req); | |
210 | } | |
211 | ||
5d87eb94 KB |
212 | static inline u16 nvme_req_qid(struct request *req) |
213 | { | |
643c476d | 214 | if (!req->q->queuedata) |
5d87eb94 | 215 | return 0; |
84115d6d BW |
216 | |
217 | return req->mq_hctx->queue_num + 1; | |
5d87eb94 KB |
218 | } |
219 | ||
54adc010 GP |
220 | /* The below value is the specific amount of delay needed before checking |
221 | * readiness in case of the PCI_DEVICE(0x1c58, 0x0003), which needs the | |
222 | * NVME_QUIRK_DELAY_BEFORE_CHK_RDY quirk enabled. The value (in ms) was | |
223 | * found empirically. | |
224 | */ | |
8c97eecc | 225 | #define NVME_QUIRK_DELAY_AMOUNT 2300 |
54adc010 | 226 | |
4212f4e9 SG |
227 | /* |
228 | * enum nvme_ctrl_state: Controller state | |
229 | * | |
230 | * @NVME_CTRL_NEW: New controller just allocated, initial state | |
231 | * @NVME_CTRL_LIVE: Controller is connected and I/O capable | |
232 | * @NVME_CTRL_RESETTING: Controller is resetting (or scheduled reset) | |
233 | * @NVME_CTRL_CONNECTING: Controller is disconnected, now connecting the | |
234 | * transport | |
235 | * @NVME_CTRL_DELETING: Controller is deleting (or scheduled deletion) | |
ecca390e SG |
236 | * @NVME_CTRL_DELETING_NOIO: Controller is deleting and I/O is not |
237 | * disabled/failed immediately. This state comes | |
238 | * after all async event processing took place and | |
239 | * before ns removal and the controller deletion | |
240 | * progress | |
4212f4e9 SG |
241 | * @NVME_CTRL_DEAD: Controller is non-present/unresponsive during |
242 | * shutdown or removal. In this case we forcibly | |
243 | * kill all inflight I/O as they have no chance to | |
244 | * complete | |
245 | */ | |
bb8d261e CH |
246 | enum nvme_ctrl_state { |
247 | NVME_CTRL_NEW, | |
248 | NVME_CTRL_LIVE, | |
249 | NVME_CTRL_RESETTING, | |
ad6a0a52 | 250 | NVME_CTRL_CONNECTING, |
bb8d261e | 251 | NVME_CTRL_DELETING, |
ecca390e | 252 | NVME_CTRL_DELETING_NOIO, |
0ff9d4e1 | 253 | NVME_CTRL_DEAD, |
bb8d261e CH |
254 | }; |
255 | ||
a3646451 AM |
256 | struct nvme_fault_inject { |
257 | #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS | |
258 | struct fault_attr attr; | |
259 | struct dentry *parent; | |
260 | bool dont_retry; /* DNR, do not retry */ | |
261 | u16 status; /* status code */ | |
262 | #endif | |
263 | }; | |
264 | ||
bf093d97 SG |
265 | enum nvme_ctrl_flags { |
266 | NVME_CTRL_FAILFAST_EXPIRED = 0, | |
267 | NVME_CTRL_ADMIN_Q_STOPPED = 1, | |
f46ef9e8 | 268 | NVME_CTRL_STARTED_ONCE = 2, |
98d81f0d | 269 | NVME_CTRL_STOPPED = 3, |
c917dd96 | 270 | NVME_CTRL_SKIP_ID_CNS_CS = 4, |
d0dd594b | 271 | NVME_CTRL_DIRTY_CAPABILITY = 5, |
839a40d1 | 272 | NVME_CTRL_FROZEN = 6, |
bf093d97 SG |
273 | }; |
274 | ||
1c63dc66 | 275 | struct nvme_ctrl { |
6e3ca03e | 276 | bool comp_seen; |
bd4da3ab | 277 | bool identified; |
9f079dda | 278 | bool passthru_err_log_enabled; |
9d217fb0 | 279 | enum nvme_ctrl_state state; |
bb8d261e | 280 | spinlock_t lock; |
e7ad43c3 | 281 | struct mutex scan_lock; |
1c63dc66 | 282 | const struct nvme_ctrl_ops *ops; |
f11bb3e2 | 283 | struct request_queue *admin_q; |
07bfcd09 | 284 | struct request_queue *connect_q; |
e7832cb4 | 285 | struct request_queue *fabrics_q; |
f11bb3e2 | 286 | struct device *dev; |
f11bb3e2 | 287 | int instance; |
103e515e | 288 | int numa_node; |
5bae7f73 | 289 | struct blk_mq_tag_set *tagset; |
34b6c231 | 290 | struct blk_mq_tag_set *admin_tagset; |
f11bb3e2 | 291 | struct list_head namespaces; |
be647e2c KB |
292 | struct mutex namespaces_lock; |
293 | struct srcu_struct srcu; | |
d22524a4 | 294 | struct device ctrl_device; |
5bae7f73 | 295 | struct device *device; /* char device */ |
ed7770f6 HR |
296 | #ifdef CONFIG_NVME_HWMON |
297 | struct device *hwmon_device; | |
298 | #endif | |
a6a5149b | 299 | struct cdev cdev; |
d86c4d8e | 300 | struct work_struct reset_work; |
c5017e85 | 301 | struct work_struct delete_work; |
c1ac9a4b | 302 | wait_queue_head_t state_wq; |
1c63dc66 | 303 | |
ab9e00cc CH |
304 | struct nvme_subsystem *subsys; |
305 | struct list_head subsys_entry; | |
306 | ||
4f1244c8 | 307 | struct opal_dev *opal_dev; |
a98e58e5 | 308 | |
76e3914a | 309 | u16 cntlid; |
5fd4ce1b | 310 | |
b6dccf7f | 311 | u16 mtfa; |
9d217fb0 | 312 | u32 ctrl_config; |
d858e5f0 | 313 | u32 queue_count; |
5fd4ce1b | 314 | |
20d0dfe6 | 315 | u64 cap; |
f11bb3e2 | 316 | u32 max_hw_sectors; |
943e942e | 317 | u32 max_segments; |
95093350 | 318 | u32 max_integrity_segments; |
5befc7c2 | 319 | u32 max_zeroes_sectors; |
240e6ee2 KB |
320 | #ifdef CONFIG_BLK_DEV_ZONED |
321 | u32 max_zone_append; | |
322 | #endif | |
49cd84b6 | 323 | u16 crdt[3]; |
f11bb3e2 | 324 | u16 oncs; |
3b946fe1 | 325 | u8 dmrl; |
1a86924e | 326 | u32 dmrsl; |
8a9ae523 | 327 | u16 oacs; |
f968688f | 328 | u16 sqsize; |
0d0b660f | 329 | u32 max_namespaces; |
6bf25d16 | 330 | atomic_t abort_limit; |
f11bb3e2 | 331 | u8 vwc; |
f3ca80fc | 332 | u32 vs; |
07bfcd09 | 333 | u32 sgls; |
038bd4cb | 334 | u16 kas; |
c5552fde AL |
335 | u8 npss; |
336 | u8 apsta; | |
400b6a7b GR |
337 | u16 wctemp; |
338 | u16 cctemp; | |
c0561f82 | 339 | u32 oaes; |
e3d7874d | 340 | u32 aen_result; |
3e53ba38 | 341 | u32 ctratt; |
07fbd32a | 342 | unsigned int shutdown_timeout; |
038bd4cb | 343 | unsigned int kato; |
f3ca80fc | 344 | bool subsystem; |
106198ed | 345 | unsigned long quirks; |
c5552fde | 346 | struct nvme_id_power_state psd[32]; |
84fef62d | 347 | struct nvme_effects_log *effects; |
1cf7a12e | 348 | struct xarray cels; |
5955be21 | 349 | struct work_struct scan_work; |
f866fc42 | 350 | struct work_struct async_event_work; |
038bd4cb | 351 | struct delayed_work ka_work; |
8c4dfea9 | 352 | struct delayed_work failfast_work; |
0a34e466 | 353 | struct nvme_command ka_cmd; |
774a9636 | 354 | unsigned long ka_last_check_time; |
b6dccf7f | 355 | struct work_struct fw_act_work; |
30d90964 | 356 | unsigned long events; |
07bfcd09 | 357 | |
0d0b660f CH |
358 | #ifdef CONFIG_NVME_MULTIPATH |
359 | /* asymmetric namespace access: */ | |
360 | u8 anacap; | |
361 | u8 anatt; | |
362 | u32 anagrpmax; | |
363 | u32 nanagrpid; | |
364 | struct mutex ana_lock; | |
365 | struct nvme_ana_rsp_hdr *ana_log_buf; | |
366 | size_t ana_log_size; | |
367 | struct timer_list anatt_timer; | |
368 | struct work_struct ana_work; | |
f227345f | 369 | atomic_t nr_active; |
0d0b660f CH |
370 | #endif |
371 | ||
d6800634 | 372 | #ifdef CONFIG_NVME_HOST_AUTH |
f50fff73 | 373 | struct work_struct dhchap_auth_work; |
f50fff73 | 374 | struct mutex dhchap_auth_mutex; |
aa36d711 | 375 | struct nvme_dhchap_queue_context *dhchap_ctxs; |
f50fff73 HR |
376 | struct nvme_dhchap_key *host_key; |
377 | struct nvme_dhchap_key *ctrl_key; | |
378 | u16 transaction; | |
379 | #endif | |
36389576 | 380 | key_serial_t tls_pskid; |
f50fff73 | 381 | |
c5552fde AL |
382 | /* Power saving configuration */ |
383 | u64 ps_max_latency_us; | |
76a5af84 | 384 | bool apst_enabled; |
c5552fde | 385 | |
044a9df1 | 386 | /* PCIe only: */ |
9d217fb0 | 387 | u16 hmmaxd; |
fe6d53c9 CH |
388 | u32 hmpre; |
389 | u32 hmmin; | |
044a9df1 | 390 | u32 hmminds; |
fe6d53c9 | 391 | |
07bfcd09 | 392 | /* Fabrics only */ |
07bfcd09 CH |
393 | u32 ioccsz; |
394 | u32 iorcsz; | |
395 | u16 icdoff; | |
396 | u16 maxcmd; | |
fdf9dfa8 | 397 | int nr_reconnects; |
8c4dfea9 | 398 | unsigned long flags; |
07bfcd09 | 399 | struct nvmf_ctrl_options *opts; |
cb5b7262 JA |
400 | |
401 | struct page *discard_page; | |
402 | unsigned long discard_page_busy; | |
f79d5fda AM |
403 | |
404 | struct nvme_fault_inject fault_inject; | |
86c2457a MB |
405 | |
406 | enum nvme_ctrl_type cntrltype; | |
407 | enum nvme_dctype dctype; | |
f11bb3e2 CH |
408 | }; |
409 | ||
5c687c28 KB |
410 | static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl) |
411 | { | |
412 | return READ_ONCE(ctrl->state); | |
413 | } | |
414 | ||
75c10e73 HR |
415 | enum nvme_iopolicy { |
416 | NVME_IOPOLICY_NUMA, | |
417 | NVME_IOPOLICY_RR, | |
f227345f | 418 | NVME_IOPOLICY_QD, |
75c10e73 HR |
419 | }; |
420 | ||
ab9e00cc CH |
421 | struct nvme_subsystem { |
422 | int instance; | |
423 | struct device dev; | |
424 | /* | |
425 | * Because we unregister the device on the last put we need | |
426 | * a separate refcount. | |
427 | */ | |
428 | struct kref ref; | |
429 | struct list_head entry; | |
430 | struct mutex lock; | |
431 | struct list_head ctrls; | |
ed754e5d | 432 | struct list_head nsheads; |
ab9e00cc CH |
433 | char subnqn[NVMF_NQN_SIZE]; |
434 | char serial[20]; | |
435 | char model[40]; | |
436 | char firmware_rev[8]; | |
437 | u8 cmic; | |
954ae166 | 438 | enum nvme_subsys_type subtype; |
ab9e00cc | 439 | u16 vendor_id; |
81adb863 | 440 | u16 awupf; /* 0's based awupf value. */ |
ed754e5d | 441 | struct ida ns_ida; |
75c10e73 HR |
442 | #ifdef CONFIG_NVME_MULTIPATH |
443 | enum nvme_iopolicy iopolicy; | |
444 | #endif | |
ab9e00cc CH |
445 | }; |
446 | ||
002fab04 CH |
447 | /* |
448 | * Container structure for uniqueue namespace identifiers. | |
449 | */ | |
450 | struct nvme_ns_ids { | |
451 | u8 eui64[8]; | |
452 | u8 nguid[16]; | |
453 | uuid_t uuid; | |
71010c30 | 454 | u8 csi; |
002fab04 CH |
455 | }; |
456 | ||
ed754e5d CH |
457 | /* |
458 | * Anchor structure for namespaces. There is one for each namespace in a | |
459 | * NVMe subsystem that any of our controllers can see, and the namespace | |
460 | * structure for each controller is chained of it. For private namespaces | |
461 | * there is a 1:1 relation to our namespace structures, that is ->list | |
462 | * only ever has a single entry for private namespaces. | |
463 | */ | |
464 | struct nvme_ns_head { | |
465 | struct list_head list; | |
466 | struct srcu_struct srcu; | |
467 | struct nvme_subsystem *subsys; | |
ed754e5d | 468 | struct nvme_ns_ids ids; |
b4c1f33a KJ |
469 | u8 lba_shift; |
470 | u16 ms; | |
471 | u16 pi_size; | |
472 | u8 pi_type; | |
473 | u8 guard_type; | |
ed754e5d CH |
474 | struct list_head entry; |
475 | struct kref ref; | |
0c284db7 | 476 | bool shared; |
1f4137e8 | 477 | bool passthru_err_log_enabled; |
be93e87e | 478 | struct nvme_effects_log *effects; |
96392961 DW |
479 | u64 nuse; |
480 | unsigned ns_id; | |
b4c1f33a | 481 | int instance; |
9419e71b DW |
482 | #ifdef CONFIG_BLK_DEV_ZONED |
483 | u64 zsze; | |
484 | #endif | |
485 | unsigned long features; | |
2637baed | 486 | |
a1a825ab | 487 | struct ratelimit_state rs_nuse; |
2637baed MI |
488 | |
489 | struct cdev cdev; | |
490 | struct device cdev_device; | |
491 | ||
f3334447 | 492 | struct gendisk *disk; |
30897388 | 493 | #ifdef CONFIG_NVME_MULTIPATH |
f3334447 CH |
494 | struct bio_list requeue_list; |
495 | spinlock_t requeue_lock; | |
496 | struct work_struct requeue_work; | |
497 | struct mutex lock; | |
d8a22f85 AE |
498 | unsigned long flags; |
499 | #define NVME_NSHEAD_DISK_LIVE 0 | |
f3334447 CH |
500 | struct nvme_ns __rcu *current_path[]; |
501 | #endif | |
ed754e5d CH |
502 | }; |
503 | ||
30897388 MI |
504 | static inline bool nvme_ns_head_multipath(struct nvme_ns_head *head) |
505 | { | |
506 | return IS_ENABLED(CONFIG_NVME_MULTIPATH) && head->disk; | |
507 | } | |
508 | ||
ffc89b1d MG |
509 | enum nvme_ns_features { |
510 | NVME_NS_EXT_LBAS = 1 << 0, /* support extended LBA format */ | |
b29f8485 | 511 | NVME_NS_METADATA_SUPPORTED = 1 << 1, /* support getting generated md */ |
9570a488 | 512 | NVME_NS_DEAC = 1 << 2, /* DEAC bit in Write Zeores supported */ |
ffc89b1d MG |
513 | }; |
514 | ||
f11bb3e2 CH |
515 | struct nvme_ns { |
516 | struct list_head list; | |
517 | ||
1c63dc66 | 518 | struct nvme_ctrl *ctrl; |
f11bb3e2 CH |
519 | struct request_queue *queue; |
520 | struct gendisk *disk; | |
0d0b660f CH |
521 | #ifdef CONFIG_NVME_MULTIPATH |
522 | enum nvme_ana_state ana_state; | |
523 | u32 ana_grpid; | |
524 | #endif | |
ed754e5d | 525 | struct list_head siblings; |
f11bb3e2 | 526 | struct kref kref; |
ed754e5d | 527 | struct nvme_ns_head *head; |
f11bb3e2 | 528 | |
646017a6 | 529 | unsigned long flags; |
0d0b660f | 530 | #define NVME_NS_REMOVING 0 |
0d0b660f | 531 | #define NVME_NS_ANA_PENDING 2 |
2f4c9ba2 | 532 | #define NVME_NS_FORCE_RO 3 |
e7d65803 | 533 | #define NVME_NS_READY 4 |
b9e03857 | 534 | |
2637baed MI |
535 | struct cdev cdev; |
536 | struct device cdev_device; | |
537 | ||
b9e03857 | 538 | struct nvme_fault_inject fault_inject; |
f11bb3e2 CH |
539 | }; |
540 | ||
4d2ce688 | 541 | /* NVMe ns supports metadata actions by the controller (generate/strip) */ |
0372dd4e | 542 | static inline bool nvme_ns_has_pi(struct nvme_ns_head *head) |
4d2ce688 | 543 | { |
0372dd4e | 544 | return head->pi_type && head->ms == head->pi_size; |
4d2ce688 JS |
545 | } |
546 | ||
1c63dc66 | 547 | struct nvme_ctrl_ops { |
1a353d85 | 548 | const char *name; |
e439bb12 | 549 | struct module *module; |
d3d5b87d CH |
550 | unsigned int flags; |
551 | #define NVME_F_FABRICS (1 << 0) | |
c81bfba9 | 552 | #define NVME_F_METADATA_SUPPORTED (1 << 1) |
db45e1a5 CH |
553 | #define NVME_F_BLOCKING (1 << 2) |
554 | ||
86adbf0c | 555 | const struct attribute_group **dev_attr_groups; |
1c63dc66 | 556 | int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val); |
5fd4ce1b | 557 | int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val); |
7fd8930f | 558 | int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val); |
1673f1f0 | 559 | void (*free_ctrl)(struct nvme_ctrl *ctrl); |
ad22c355 | 560 | void (*submit_async_event)(struct nvme_ctrl *ctrl); |
210b1f65 | 561 | int (*subsystem_reset)(struct nvme_ctrl *ctrl); |
c5017e85 | 562 | void (*delete_ctrl)(struct nvme_ctrl *ctrl); |
f7f70f4a | 563 | void (*stop_ctrl)(struct nvme_ctrl *ctrl); |
1a353d85 | 564 | int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size); |
2f0dad17 | 565 | void (*print_device_info)(struct nvme_ctrl *ctrl); |
2f859441 | 566 | bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl); |
f11bb3e2 CH |
567 | }; |
568 | ||
e7006de6 SG |
569 | /* |
570 | * nvme command_id is constructed as such: | |
571 | * | xxxx | xxxxxxxxxxxx | | |
572 | * gen request tag | |
573 | */ | |
574 | #define nvme_genctr_mask(gen) (gen & 0xf) | |
575 | #define nvme_cid_install_genctr(gen) (nvme_genctr_mask(gen) << 12) | |
576 | #define nvme_genctr_from_cid(cid) ((cid & 0xf000) >> 12) | |
577 | #define nvme_tag_from_cid(cid) (cid & 0xfff) | |
578 | ||
579 | static inline u16 nvme_cid(struct request *rq) | |
580 | { | |
581 | return nvme_cid_install_genctr(nvme_req(rq)->genctr) | rq->tag; | |
582 | } | |
583 | ||
584 | static inline struct request *nvme_find_rq(struct blk_mq_tags *tags, | |
585 | u16 command_id) | |
586 | { | |
587 | u8 genctr = nvme_genctr_from_cid(command_id); | |
588 | u16 tag = nvme_tag_from_cid(command_id); | |
589 | struct request *rq; | |
590 | ||
591 | rq = blk_mq_tag_to_rq(tags, tag); | |
592 | if (unlikely(!rq)) { | |
593 | pr_err("could not locate request for tag %#x\n", | |
594 | tag); | |
595 | return NULL; | |
596 | } | |
597 | if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) { | |
598 | dev_err(nvme_req(rq)->ctrl->device, | |
599 | "request %#x genctr mismatch (got %#x expected %#x)\n", | |
600 | tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr)); | |
601 | return NULL; | |
602 | } | |
603 | return rq; | |
604 | } | |
605 | ||
606 | static inline struct request *nvme_cid_to_rq(struct blk_mq_tags *tags, | |
607 | u16 command_id) | |
608 | { | |
609 | return blk_mq_tag_to_rq(tags, nvme_tag_from_cid(command_id)); | |
610 | } | |
611 | ||
2f0dad17 KB |
612 | /* |
613 | * Return the length of the string without the space padding | |
614 | */ | |
615 | static inline int nvme_strlen(char *s, int len) | |
616 | { | |
617 | while (s[len - 1] == ' ') | |
618 | len--; | |
619 | return len; | |
620 | } | |
621 | ||
622 | static inline void nvme_print_device_info(struct nvme_ctrl *ctrl) | |
623 | { | |
624 | struct nvme_subsystem *subsys = ctrl->subsys; | |
625 | ||
626 | if (ctrl->ops->print_device_info) { | |
627 | ctrl->ops->print_device_info(ctrl); | |
628 | return; | |
629 | } | |
630 | ||
631 | dev_err(ctrl->device, | |
632 | "VID:%04x model:%.*s firmware:%.*s\n", subsys->vendor_id, | |
633 | nvme_strlen(subsys->model, sizeof(subsys->model)), | |
634 | subsys->model, nvme_strlen(subsys->firmware_rev, | |
635 | sizeof(subsys->firmware_rev)), | |
636 | subsys->firmware_rev); | |
637 | } | |
638 | ||
b9e03857 | 639 | #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS |
a3646451 AM |
640 | void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj, |
641 | const char *dev_name); | |
642 | void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject); | |
b9e03857 TT |
643 | void nvme_should_fail(struct request *req); |
644 | #else | |
a3646451 AM |
645 | static inline void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj, |
646 | const char *dev_name) | |
647 | { | |
648 | } | |
649 | static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj) | |
650 | { | |
651 | } | |
b9e03857 TT |
652 | static inline void nvme_should_fail(struct request *req) {} |
653 | #endif | |
654 | ||
1e866afd KB |
655 | bool nvme_wait_reset(struct nvme_ctrl *ctrl); |
656 | int nvme_try_sched_reset(struct nvme_ctrl *ctrl); | |
657 | ||
f3ca80fc CH |
658 | static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl) |
659 | { | |
210b1f65 | 660 | if (!ctrl->subsystem || !ctrl->ops->subsystem_reset) |
f3ca80fc | 661 | return -ENOTTY; |
210b1f65 | 662 | return ctrl->ops->subsystem_reset(ctrl); |
f3ca80fc CH |
663 | } |
664 | ||
314d48dd DLM |
665 | /* |
666 | * Convert a 512B sector number to a device logical block number. | |
667 | */ | |
0372dd4e | 668 | static inline u64 nvme_sect_to_lba(struct nvme_ns_head *head, sector_t sector) |
f11bb3e2 | 669 | { |
0372dd4e | 670 | return sector >> (head->lba_shift - SECTOR_SHIFT); |
f11bb3e2 CH |
671 | } |
672 | ||
e08f2ae8 DLM |
673 | /* |
674 | * Convert a device logical block number to a 512B sector number. | |
675 | */ | |
0372dd4e | 676 | static inline sector_t nvme_lba_to_sect(struct nvme_ns_head *head, u64 lba) |
f11bb3e2 | 677 | { |
0372dd4e | 678 | return lba << (head->lba_shift - SECTOR_SHIFT); |
f11bb3e2 CH |
679 | } |
680 | ||
71fb90eb KB |
681 | /* |
682 | * Convert byte length to nvme's 0-based num dwords | |
683 | */ | |
684 | static inline u32 nvme_bytes_to_numd(size_t len) | |
685 | { | |
686 | return (len >> 2) - 1; | |
687 | } | |
688 | ||
5ddaabe8 CH |
689 | static inline bool nvme_is_ana_error(u16 status) |
690 | { | |
d89a5c67 | 691 | switch (status & NVME_SCT_SC_MASK) { |
5ddaabe8 CH |
692 | case NVME_SC_ANA_TRANSITION: |
693 | case NVME_SC_ANA_INACCESSIBLE: | |
694 | case NVME_SC_ANA_PERSISTENT_LOSS: | |
695 | return true; | |
696 | default: | |
697 | return false; | |
698 | } | |
699 | } | |
700 | ||
701 | static inline bool nvme_is_path_error(u16 status) | |
702 | { | |
1e41f3bd | 703 | /* check for a status code type of 'path related status' */ |
d89a5c67 | 704 | return (status & NVME_SCT_MASK) == NVME_SCT_PATH; |
5ddaabe8 CH |
705 | } |
706 | ||
2eb81a33 CH |
707 | /* |
708 | * Fill in the status and result information from the CQE, and then figure out | |
709 | * if blk-mq will need to use IPI magic to complete the request, and if yes do | |
710 | * so. If not let the caller complete the request without an indirect function | |
711 | * call. | |
712 | */ | |
713 | static inline bool nvme_try_complete_req(struct request *req, __le16 status, | |
27fa9bc5 | 714 | union nvme_result result) |
15a190f7 | 715 | { |
27fa9bc5 | 716 | struct nvme_request *rq = nvme_req(req); |
e4fdb2b1 KB |
717 | struct nvme_ctrl *ctrl = rq->ctrl; |
718 | ||
719 | if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN)) | |
720 | rq->genctr++; | |
15a190f7 | 721 | |
27fa9bc5 CH |
722 | rq->status = le16_to_cpu(status) >> 1; |
723 | rq->result = result; | |
b9e03857 TT |
724 | /* inject error when permitted by fault injection framework */ |
725 | nvme_should_fail(req); | |
ff029451 CH |
726 | if (unlikely(blk_should_fake_timeout(req->q))) |
727 | return true; | |
728 | return blk_mq_complete_request_remote(req); | |
7688faa6 CH |
729 | } |
730 | ||
d22524a4 CH |
731 | static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl) |
732 | { | |
733 | get_device(ctrl->device); | |
734 | } | |
735 | ||
736 | static inline void nvme_put_ctrl(struct nvme_ctrl *ctrl) | |
737 | { | |
738 | put_device(ctrl->device); | |
739 | } | |
740 | ||
58a8df67 IR |
741 | static inline bool nvme_is_aen_req(u16 qid, __u16 command_id) |
742 | { | |
e7006de6 SG |
743 | return !qid && |
744 | nvme_tag_from_cid(command_id) >= NVME_AQ_BLK_MQ_DEPTH; | |
58a8df67 IR |
745 | } |
746 | ||
25bb3534 NS |
747 | /* |
748 | * Returns true for sink states that can't ever transition back to live. | |
749 | */ | |
750 | static inline bool nvme_state_terminal(struct nvme_ctrl *ctrl) | |
751 | { | |
752 | switch (nvme_ctrl_state(ctrl)) { | |
753 | case NVME_CTRL_NEW: | |
754 | case NVME_CTRL_LIVE: | |
755 | case NVME_CTRL_RESETTING: | |
756 | case NVME_CTRL_CONNECTING: | |
757 | return false; | |
758 | case NVME_CTRL_DELETING: | |
759 | case NVME_CTRL_DELETING_NOIO: | |
760 | case NVME_CTRL_DEAD: | |
761 | return true; | |
762 | default: | |
763 | WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state); | |
764 | return true; | |
765 | } | |
766 | } | |
767 | ||
a2e4c5f5 | 768 | void nvme_end_req(struct request *req); |
77f02a7a | 769 | void nvme_complete_rq(struct request *req); |
c234a653 JA |
770 | void nvme_complete_batch_req(struct request *req); |
771 | ||
772 | static __always_inline void nvme_complete_batch(struct io_comp_batch *iob, | |
773 | void (*fn)(struct request *rq)) | |
774 | { | |
775 | struct request *req; | |
776 | ||
777 | rq_list_for_each(&iob->req_list, req) { | |
778 | fn(req); | |
779 | nvme_complete_batch_req(req); | |
780 | } | |
781 | blk_mq_end_request_batch(iob); | |
782 | } | |
783 | ||
dda3248e | 784 | blk_status_t nvme_host_path_error(struct request *req); |
2dd6532e | 785 | bool nvme_cancel_request(struct request *req, void *data); |
25479069 CL |
786 | void nvme_cancel_tagset(struct nvme_ctrl *ctrl); |
787 | void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl); | |
bb8d261e CH |
788 | bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl, |
789 | enum nvme_ctrl_state new_state); | |
285b6e9b | 790 | int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown); |
c0f2f45b | 791 | int nvme_enable_ctrl(struct nvme_ctrl *ctrl); |
f3ca80fc CH |
792 | int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, |
793 | const struct nvme_ctrl_ops *ops, unsigned long quirks); | |
1a9e2181 | 794 | int nvme_add_ctrl(struct nvme_ctrl *ctrl); |
53029b04 | 795 | void nvme_uninit_ctrl(struct nvme_ctrl *ctrl); |
d09f2b45 SG |
796 | void nvme_start_ctrl(struct nvme_ctrl *ctrl); |
797 | void nvme_stop_ctrl(struct nvme_ctrl *ctrl); | |
94cc781f | 798 | int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended); |
fe60e8c5 | 799 | int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, |
db45e1a5 | 800 | const struct blk_mq_ops *ops, unsigned int cmd_size); |
fe60e8c5 CH |
801 | void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl); |
802 | int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set, | |
db45e1a5 CH |
803 | const struct blk_mq_ops *ops, unsigned int nr_maps, |
804 | unsigned int cmd_size); | |
fe60e8c5 | 805 | void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl); |
5bae7f73 | 806 | |
5bae7f73 | 807 | void nvme_remove_namespaces(struct nvme_ctrl *ctrl); |
1673f1f0 | 808 | |
7bf58533 | 809 | void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status, |
287a63eb | 810 | volatile union nvme_result *res); |
f866fc42 | 811 | |
9f27bd70 CH |
812 | void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl); |
813 | void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl); | |
814 | void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl); | |
815 | void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl); | |
cd50f9b2 | 816 | void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl); |
d6135c3a | 817 | void nvme_sync_queues(struct nvme_ctrl *ctrl); |
04800fbf | 818 | void nvme_sync_io_queues(struct nvme_ctrl *ctrl); |
302ad8cc KB |
819 | void nvme_unfreeze(struct nvme_ctrl *ctrl); |
820 | void nvme_wait_freeze(struct nvme_ctrl *ctrl); | |
7cf0d7c0 | 821 | int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout); |
302ad8cc | 822 | void nvme_start_freeze(struct nvme_ctrl *ctrl); |
363c9aac | 823 | |
f9ed86dc | 824 | static inline enum req_op nvme_req_op(struct nvme_command *cmd) |
e559398f CH |
825 | { |
826 | return nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN; | |
827 | } | |
828 | ||
eb71f435 | 829 | #define NVME_QID_ANY -1 |
e559398f | 830 | void nvme_init_request(struct request *req, struct nvme_command *cmd); |
f7f1fc36 | 831 | void nvme_cleanup_cmd(struct request *req); |
f4b9e6c9 | 832 | blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req); |
a9715744 TC |
833 | blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl, |
834 | struct request *req); | |
835 | bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq, | |
6d3c7fb1 | 836 | bool queue_live, enum nvme_ctrl_state state); |
a9715744 TC |
837 | |
838 | static inline bool nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq, | |
839 | bool queue_live) | |
840 | { | |
6d3c7fb1 KB |
841 | enum nvme_ctrl_state state = nvme_ctrl_state(ctrl); |
842 | ||
843 | if (likely(state == NVME_CTRL_LIVE)) | |
a9715744 | 844 | return true; |
6d3c7fb1 | 845 | if (ctrl->ops->flags & NVME_F_FABRICS && state == NVME_CTRL_DELETING) |
8b77fa6f | 846 | return queue_live; |
6d3c7fb1 | 847 | return __nvme_check_ready(ctrl, rq, queue_live, state); |
a9715744 | 848 | } |
5974ea7c SM |
849 | |
850 | /* | |
851 | * NSID shall be unique for all shared namespaces, or if at least one of the | |
852 | * following conditions is met: | |
853 | * 1. Namespace Management is supported by the controller | |
854 | * 2. ANA is supported by the controller | |
855 | * 3. NVM Set are supported by the controller | |
856 | * | |
857 | * In other case, private namespace are not required to report a unique NSID. | |
858 | */ | |
859 | static inline bool nvme_is_unique_nsid(struct nvme_ctrl *ctrl, | |
860 | struct nvme_ns_head *head) | |
861 | { | |
862 | return head->shared || | |
863 | (ctrl->oacs & NVME_CTRL_OACS_NS_MNGT_SUPP) || | |
864 | (ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA) || | |
865 | (ctrl->ctratt & NVME_CTRL_CTRATT_NVM_SETS); | |
866 | } | |
867 | ||
bd2687f2 HR |
868 | /* |
869 | * Flags for __nvme_submit_sync_cmd() | |
870 | */ | |
871 | typedef __u32 __bitwise nvme_submit_flags_t; | |
872 | ||
873 | enum { | |
874 | /* Insert request at the head of the queue */ | |
875 | NVME_SUBMIT_AT_HEAD = (__force nvme_submit_flags_t)(1 << 0), | |
876 | /* Set BLK_MQ_REQ_NOWAIT when allocating request */ | |
877 | NVME_SUBMIT_NOWAIT = (__force nvme_submit_flags_t)(1 << 1), | |
878 | /* Set BLK_MQ_REQ_RESERVED when allocating request */ | |
879 | NVME_SUBMIT_RESERVED = (__force nvme_submit_flags_t)(1 << 2), | |
dd0b0a4a | 880 | /* Retry command when NVME_STATUS_DNR is not set in the result */ |
48dae466 | 881 | NVME_SUBMIT_RETRY = (__force nvme_submit_flags_t)(1 << 3), |
bd2687f2 HR |
882 | }; |
883 | ||
f11bb3e2 CH |
884 | int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, |
885 | void *buf, unsigned bufflen); | |
886 | int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, | |
d49187e9 | 887 | union nvme_result *result, void *buffer, unsigned bufflen, |
bd2687f2 | 888 | int qid, nvme_submit_flags_t flags); |
1a87ee65 KB |
889 | int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid, |
890 | unsigned int dword11, void *buffer, size_t buflen, | |
891 | u32 *result); | |
892 | int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid, | |
893 | unsigned int dword11, void *buffer, size_t buflen, | |
894 | u32 *result); | |
9a0be7ab | 895 | int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); |
038bd4cb | 896 | void nvme_stop_keep_alive(struct nvme_ctrl *ctrl); |
d86c4d8e | 897 | int nvme_reset_ctrl(struct nvme_ctrl *ctrl); |
2405252a | 898 | int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl); |
c5017e85 | 899 | int nvme_delete_ctrl(struct nvme_ctrl *ctrl); |
2405252a | 900 | void nvme_queue_scan(struct nvme_ctrl *ctrl); |
be93e87e | 901 | int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, |
0e98719b | 902 | void *log, size_t size, u64 offset); |
1496bd49 CH |
903 | bool nvme_tryget_ns_head(struct nvme_ns_head *head); |
904 | void nvme_put_ns_head(struct nvme_ns_head *head); | |
2637baed MI |
905 | int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, |
906 | const struct file_operations *fops, struct module *owner); | |
907 | void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device); | |
05bdb996 | 908 | int nvme_ioctl(struct block_device *bdev, blk_mode_t mode, |
2405252a | 909 | unsigned int cmd, unsigned long arg); |
2637baed | 910 | long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg); |
05bdb996 | 911 | int nvme_ns_head_ioctl(struct block_device *bdev, blk_mode_t mode, |
2405252a | 912 | unsigned int cmd, unsigned long arg); |
2637baed MI |
913 | long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd, |
914 | unsigned long arg); | |
2405252a CH |
915 | long nvme_dev_ioctl(struct file *file, unsigned int cmd, |
916 | unsigned long arg); | |
de97fcb3 JA |
917 | int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, |
918 | struct io_comp_batch *iob, unsigned int poll_flags); | |
456cba38 KJ |
919 | int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, |
920 | unsigned int issue_flags); | |
921 | int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd, | |
922 | unsigned int issue_flags); | |
a1a825ab DW |
923 | int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid, |
924 | struct nvme_id_ns **id); | |
1496bd49 | 925 | int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo); |
58e5bdeb | 926 | int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags); |
d558fb51 | 927 | |
83ac678e | 928 | extern const struct attribute_group *nvme_ns_attr_groups[]; |
1496bd49 | 929 | extern const struct pr_ops nvme_pr_ops; |
32acab31 | 930 | extern const struct block_device_operations nvme_ns_head_ops; |
86adbf0c | 931 | extern const struct attribute_group nvme_dev_attrs_group; |
942e21c0 MG |
932 | extern const struct attribute_group *nvme_subsys_attrs_groups[]; |
933 | extern const struct attribute_group *nvme_dev_attr_groups[]; | |
934 | extern const struct block_device_operations nvme_bdev_ops; | |
32acab31 | 935 | |
942e21c0 | 936 | void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl); |
f1cf35e1 | 937 | struct nvme_ns *nvme_find_path(struct nvme_ns_head *head); |
32acab31 | 938 | #ifdef CONFIG_NVME_MULTIPATH |
66b20ac0 MR |
939 | static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) |
940 | { | |
941 | return ctrl->ana_log_buf != NULL; | |
942 | } | |
943 | ||
b9156dae SG |
944 | void nvme_mpath_unfreeze(struct nvme_subsystem *subsys); |
945 | void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys); | |
946 | void nvme_mpath_start_freeze(struct nvme_subsystem *subsys); | |
e3d34794 | 947 | void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys); |
5ddaabe8 | 948 | void nvme_failover_req(struct request *req); |
32acab31 CH |
949 | void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl); |
950 | int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head); | |
c13cf14f | 951 | void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid); |
32acab31 | 952 | void nvme_mpath_remove_disk(struct nvme_ns_head *head); |
5e1f6899 CH |
953 | int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id); |
954 | void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl); | |
a4a6f3c8 | 955 | void nvme_mpath_update(struct nvme_ctrl *ctrl); |
0d0b660f CH |
956 | void nvme_mpath_uninit(struct nvme_ctrl *ctrl); |
957 | void nvme_mpath_stop(struct nvme_ctrl *ctrl); | |
0157ec8d | 958 | bool nvme_mpath_clear_current_path(struct nvme_ns *ns); |
e7d65803 | 959 | void nvme_mpath_revalidate_paths(struct nvme_ns *ns); |
0157ec8d | 960 | void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl); |
5396fdac | 961 | void nvme_mpath_shutdown_disk(struct nvme_ns_head *head); |
d4d957b5 SG |
962 | void nvme_mpath_start_request(struct request *rq); |
963 | void nvme_mpath_end_request(struct request *rq); | |
479a322f | 964 | |
2b59787a | 965 | static inline void nvme_trace_bio_complete(struct request *req) |
35fe0d12 HR |
966 | { |
967 | struct nvme_ns *ns = req->q->queuedata; | |
968 | ||
3659fb5a | 969 | if ((req->cmd_flags & REQ_NVME_MPATH) && req->bio) |
d24de76a | 970 | trace_block_bio_complete(ns->head->disk->queue, req->bio); |
35fe0d12 HR |
971 | } |
972 | ||
b739e137 | 973 | extern bool multipath; |
0d0b660f CH |
974 | extern struct device_attribute dev_attr_ana_grpid; |
975 | extern struct device_attribute dev_attr_ana_state; | |
75c10e73 | 976 | extern struct device_attribute subsys_attr_iopolicy; |
0d0b660f | 977 | |
bafd5909 GL |
978 | static inline bool nvme_disk_is_ns_head(struct gendisk *disk) |
979 | { | |
980 | return disk->fops == &nvme_ns_head_ops; | |
981 | } | |
32acab31 | 982 | #else |
b739e137 | 983 | #define multipath false |
0d0b660f CH |
984 | static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) |
985 | { | |
986 | return false; | |
987 | } | |
5ddaabe8 | 988 | static inline void nvme_failover_req(struct request *req) |
32acab31 CH |
989 | { |
990 | } | |
32acab31 CH |
991 | static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl) |
992 | { | |
993 | } | |
994 | static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, | |
995 | struct nvme_ns_head *head) | |
996 | { | |
997 | return 0; | |
998 | } | |
c13cf14f | 999 | static inline void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) |
32acab31 CH |
1000 | { |
1001 | } | |
1002 | static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head) | |
1003 | { | |
1004 | } | |
0157ec8d SG |
1005 | static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns) |
1006 | { | |
1007 | return false; | |
1008 | } | |
e7d65803 HR |
1009 | static inline void nvme_mpath_revalidate_paths(struct nvme_ns *ns) |
1010 | { | |
1011 | } | |
0157ec8d | 1012 | static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl) |
479a322f SG |
1013 | { |
1014 | } | |
5396fdac | 1015 | static inline void nvme_mpath_shutdown_disk(struct nvme_ns_head *head) |
32acab31 CH |
1016 | { |
1017 | } | |
2b59787a | 1018 | static inline void nvme_trace_bio_complete(struct request *req) |
35fe0d12 HR |
1019 | { |
1020 | } | |
5e1f6899 CH |
1021 | static inline void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl) |
1022 | { | |
1023 | } | |
1024 | static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, | |
0d0b660f CH |
1025 | struct nvme_id_ctrl *id) |
1026 | { | |
2bd64307 | 1027 | if (ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA) |
14a1336e CH |
1028 | dev_warn(ctrl->device, |
1029 | "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n"); | |
0d0b660f CH |
1030 | return 0; |
1031 | } | |
a4a6f3c8 AE |
1032 | static inline void nvme_mpath_update(struct nvme_ctrl *ctrl) |
1033 | { | |
1034 | } | |
0d0b660f CH |
1035 | static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl) |
1036 | { | |
1037 | } | |
1038 | static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl) | |
1039 | { | |
1040 | } | |
b9156dae SG |
1041 | static inline void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) |
1042 | { | |
1043 | } | |
1044 | static inline void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) | |
1045 | { | |
1046 | } | |
1047 | static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) | |
1048 | { | |
1049 | } | |
e3d34794 HR |
1050 | static inline void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys) |
1051 | { | |
1052 | } | |
d4d957b5 SG |
1053 | static inline void nvme_mpath_start_request(struct request *rq) |
1054 | { | |
1055 | } | |
1056 | static inline void nvme_mpath_end_request(struct request *rq) | |
1057 | { | |
1058 | } | |
bafd5909 GL |
1059 | static inline bool nvme_disk_is_ns_head(struct gendisk *disk) |
1060 | { | |
1061 | return false; | |
1062 | } | |
32acab31 CH |
1063 | #endif /* CONFIG_NVME_MULTIPATH */ |
1064 | ||
18f03a06 CH |
1065 | int nvme_ns_get_unique_id(struct nvme_ns *ns, u8 id[16], |
1066 | enum blk_unique_id type); | |
1067 | ||
c85c9ab9 CH |
1068 | struct nvme_zone_info { |
1069 | u64 zone_size; | |
1070 | unsigned int max_open_zones; | |
1071 | unsigned int max_active_zones; | |
1072 | }; | |
1073 | ||
8b4fb0f9 CH |
1074 | int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, |
1075 | unsigned int nr_zones, report_zones_cb cb, void *data); | |
c85c9ab9 CH |
1076 | int nvme_query_zone_info(struct nvme_ns *ns, unsigned lbaf, |
1077 | struct nvme_zone_info *zi); | |
1078 | void nvme_update_zone_info(struct nvme_ns *ns, struct queue_limits *lim, | |
1079 | struct nvme_zone_info *zi); | |
240e6ee2 | 1080 | #ifdef CONFIG_BLK_DEV_ZONED |
240e6ee2 KB |
1081 | blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, struct request *req, |
1082 | struct nvme_command *cmnd, | |
1083 | enum nvme_zone_mgmt_action action); | |
1084 | #else | |
240e6ee2 KB |
1085 | static inline blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, |
1086 | struct request *req, struct nvme_command *cmnd, | |
1087 | enum nvme_zone_mgmt_action action) | |
1088 | { | |
1089 | return BLK_STS_NOTSUPP; | |
1090 | } | |
240e6ee2 KB |
1091 | #endif |
1092 | ||
40267efd SL |
1093 | static inline struct nvme_ns *nvme_get_ns_from_dev(struct device *dev) |
1094 | { | |
bafd5909 GL |
1095 | struct gendisk *disk = dev_to_disk(dev); |
1096 | ||
1097 | WARN_ON(nvme_disk_is_ns_head(disk)); | |
1098 | return disk->private_data; | |
40267efd | 1099 | } |
ca064085 | 1100 | |
400b6a7b | 1101 | #ifdef CONFIG_NVME_HWMON |
59e330f8 | 1102 | int nvme_hwmon_init(struct nvme_ctrl *ctrl); |
ed7770f6 | 1103 | void nvme_hwmon_exit(struct nvme_ctrl *ctrl); |
400b6a7b | 1104 | #else |
59e330f8 KB |
1105 | static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl) |
1106 | { | |
1107 | return 0; | |
1108 | } | |
ed7770f6 HR |
1109 | |
1110 | static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl) | |
1111 | { | |
1112 | } | |
400b6a7b GR |
1113 | #endif |
1114 | ||
6887fc64 SG |
1115 | static inline void nvme_start_request(struct request *rq) |
1116 | { | |
d4d957b5 SG |
1117 | if (rq->cmd_flags & REQ_NVME_MPATH) |
1118 | nvme_mpath_start_request(rq); | |
6887fc64 SG |
1119 | blk_mq_start_request(rq); |
1120 | } | |
1121 | ||
73eefc27 CK |
1122 | static inline bool nvme_ctrl_sgl_supported(struct nvme_ctrl *ctrl) |
1123 | { | |
1124 | return ctrl->sgls & ((1 << 0) | (1 << 1)); | |
1125 | } | |
1126 | ||
d6800634 | 1127 | #ifdef CONFIG_NVME_HOST_AUTH |
e481fc0a SG |
1128 | int __init nvme_init_auth(void); |
1129 | void __exit nvme_exit_auth(void); | |
193a8c7e | 1130 | int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl); |
f50fff73 HR |
1131 | void nvme_auth_stop(struct nvme_ctrl *ctrl); |
1132 | int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid); | |
1133 | int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid); | |
f50fff73 HR |
1134 | void nvme_auth_free(struct nvme_ctrl *ctrl); |
1135 | #else | |
193a8c7e SG |
1136 | static inline int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) |
1137 | { | |
1138 | return 0; | |
1139 | } | |
e481fc0a SG |
1140 | static inline int __init nvme_init_auth(void) |
1141 | { | |
1142 | return 0; | |
1143 | } | |
1144 | static inline void __exit nvme_exit_auth(void) | |
1145 | { | |
1146 | } | |
f50fff73 HR |
1147 | static inline void nvme_auth_stop(struct nvme_ctrl *ctrl) {}; |
1148 | static inline int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) | |
1149 | { | |
1150 | return -EPROTONOSUPPORT; | |
1151 | } | |
1152 | static inline int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) | |
1153 | { | |
44350336 | 1154 | return -EPROTONOSUPPORT; |
f50fff73 HR |
1155 | } |
1156 | static inline void nvme_auth_free(struct nvme_ctrl *ctrl) {}; | |
1157 | #endif | |
1158 | ||
df21b6b1 LG |
1159 | u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, |
1160 | u8 opcode); | |
62281b9e CH |
1161 | u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); |
1162 | int nvme_execute_rq(struct request *rq, bool at_head); | |
31a59782 | 1163 | void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, |
bc8fb906 | 1164 | struct nvme_command *cmd, int status); |
b2702aaa | 1165 | struct nvme_ctrl *nvme_ctrl_from_file(struct file *file); |
24493b8b | 1166 | struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid); |
be647e2c | 1167 | bool nvme_get_ns(struct nvme_ns *ns); |
24493b8b | 1168 | void nvme_put_ns(struct nvme_ns *ns); |
df21b6b1 | 1169 | |
43dc9878 AM |
1170 | static inline bool nvme_multi_css(struct nvme_ctrl *ctrl) |
1171 | { | |
1172 | return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI; | |
1173 | } | |
1174 | ||
bd83fe6f | 1175 | #ifdef CONFIG_NVME_VERBOSE_ERRORS |
4b682194 CS |
1176 | const char *nvme_get_error_status_str(u16 status); |
1177 | const char *nvme_get_opcode_str(u8 opcode); | |
1178 | const char *nvme_get_admin_opcode_str(u8 opcode); | |
1179 | const char *nvme_get_fabrics_opcode_str(u8 opcode); | |
bd83fe6f | 1180 | #else /* CONFIG_NVME_VERBOSE_ERRORS */ |
4b682194 | 1181 | static inline const char *nvme_get_error_status_str(u16 status) |
bd83fe6f AA |
1182 | { |
1183 | return "I/O Error"; | |
1184 | } | |
4b682194 | 1185 | static inline const char *nvme_get_opcode_str(u8 opcode) |
bd83fe6f AA |
1186 | { |
1187 | return "I/O Cmd"; | |
1188 | } | |
4b682194 | 1189 | static inline const char *nvme_get_admin_opcode_str(u8 opcode) |
bd83fe6f AA |
1190 | { |
1191 | return "Admin Cmd"; | |
1192 | } | |
567da14d | 1193 | |
4b682194 | 1194 | static inline const char *nvme_get_fabrics_opcode_str(u8 opcode) |
567da14d AE |
1195 | { |
1196 | return "Fabrics Cmd"; | |
1197 | } | |
bd83fe6f AA |
1198 | #endif /* CONFIG_NVME_VERBOSE_ERRORS */ |
1199 | ||
7d23e836 | 1200 | static inline const char *nvme_opcode_str(int qid, u8 opcode) |
567da14d | 1201 | { |
567da14d AE |
1202 | return qid ? nvme_get_opcode_str(opcode) : |
1203 | nvme_get_admin_opcode_str(opcode); | |
1204 | } | |
7d23e836 CS |
1205 | |
1206 | static inline const char *nvme_fabrics_opcode_str( | |
1207 | int qid, const struct nvme_command *cmd) | |
1208 | { | |
1209 | if (nvme_is_fabrics(cmd)) | |
1210 | return nvme_get_fabrics_opcode_str(cmd->fabrics.fctype); | |
1211 | ||
1212 | return nvme_opcode_str(qid, cmd->common.opcode); | |
1213 | } | |
f11bb3e2 | 1214 | #endif /* _NVME_H */ |