Commit | Line | Data |
---|---|---|
77141dc6 | 1 | // SPDX-License-Identifier: GPL-2.0 |
a07b4970 CH |
2 | /* |
3 | * Common code for the NVMe target. | |
4 | * Copyright (c) 2015-2016 HGST, a Western Digital Company. | |
a07b4970 CH |
5 | */ |
6 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
7 | #include <linux/module.h> | |
28b89118 | 8 | #include <linux/random.h> |
b2d09103 | 9 | #include <linux/rculist.h> |
c6925093 | 10 | #include <linux/pci-p2pdma.h> |
a5dffbb6 | 11 | #include <linux/scatterlist.h> |
b2d09103 | 12 | |
a5448fdc MI |
13 | #define CREATE_TRACE_POINTS |
14 | #include "trace.h" | |
15 | ||
a07b4970 CH |
16 | #include "nvmet.h" |
17 | ||
55eb942e | 18 | struct workqueue_struct *buffered_io_wq; |
aaf2e048 | 19 | struct workqueue_struct *zbd_wq; |
e929f06d | 20 | static const struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX]; |
15fbad96 | 21 | static DEFINE_IDA(cntlid_ida); |
a07b4970 | 22 | |
8832cf92 SG |
23 | struct workqueue_struct *nvmet_wq; |
24 | EXPORT_SYMBOL_GPL(nvmet_wq); | |
25 | ||
a07b4970 CH |
26 | /* |
27 | * This read/write semaphore is used to synchronize access to configuration | |
28 | * information on a target system that will result in discovery log page | |
29 | * information change for at least one host. | |
30 | * The full list of resources to protected by this semaphore is: | |
31 | * | |
32 | * - subsystems list | |
33 | * - per-subsystem allowed hosts list | |
34 | * - allow_any_host subsystem attribute | |
35 | * - nvmet_genctr | |
36 | * - the nvmet_transports array | |
37 | * | |
38 | * When updating any of those lists/structures write lock should be obtained, | |
39 | * while when reading (popolating discovery log page or checking host-subsystem | |
40 | * link) read lock is obtained to allow concurrent reads. | |
41 | */ | |
42 | DECLARE_RWSEM(nvmet_config_sem); | |
43 | ||
72efd25d CH |
44 | u32 nvmet_ana_group_enabled[NVMET_MAX_ANAGRPS + 1]; |
45 | u64 nvmet_ana_chgcnt; | |
46 | DECLARE_RWSEM(nvmet_ana_sem); | |
47 | ||
c6aa3542 CK |
48 | inline u16 errno_to_nvme_status(struct nvmet_req *req, int errno) |
49 | { | |
c6aa3542 | 50 | switch (errno) { |
cfc1a1af | 51 | case 0: |
7860569a | 52 | return NVME_SC_SUCCESS; |
c6aa3542 CK |
53 | case -ENOSPC: |
54 | req->error_loc = offsetof(struct nvme_rw_command, length); | |
7860569a | 55 | return NVME_SC_CAP_EXCEEDED | NVME_SC_DNR; |
c6aa3542 CK |
56 | case -EREMOTEIO: |
57 | req->error_loc = offsetof(struct nvme_rw_command, slba); | |
7860569a | 58 | return NVME_SC_LBA_RANGE | NVME_SC_DNR; |
c6aa3542 CK |
59 | case -EOPNOTSUPP: |
60 | req->error_loc = offsetof(struct nvme_common_command, opcode); | |
61 | switch (req->cmd->common.opcode) { | |
62 | case nvme_cmd_dsm: | |
63 | case nvme_cmd_write_zeroes: | |
7860569a | 64 | return NVME_SC_ONCS_NOT_SUPPORTED | NVME_SC_DNR; |
c6aa3542 | 65 | default: |
7860569a | 66 | return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; |
c6aa3542 CK |
67 | } |
68 | break; | |
69 | case -ENODATA: | |
70 | req->error_loc = offsetof(struct nvme_rw_command, nsid); | |
7860569a | 71 | return NVME_SC_ACCESS_DENIED; |
c6aa3542 | 72 | case -EIO: |
df561f66 | 73 | fallthrough; |
c6aa3542 CK |
74 | default: |
75 | req->error_loc = offsetof(struct nvme_common_command, opcode); | |
7860569a | 76 | return NVME_SC_INTERNAL | NVME_SC_DNR; |
c6aa3542 | 77 | } |
c6aa3542 CK |
78 | } |
79 | ||
d81d57cf CK |
80 | u16 nvmet_report_invalid_opcode(struct nvmet_req *req) |
81 | { | |
82 | pr_debug("unhandled cmd %d on qid %d\n", req->cmd->common.opcode, | |
83 | req->sq->qid); | |
84 | ||
85 | req->error_loc = offsetof(struct nvme_common_command, opcode); | |
86 | return NVME_SC_INVALID_OPCODE | NVME_SC_DNR; | |
87 | } | |
88 | ||
a07b4970 CH |
89 | static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, |
90 | const char *subsysnqn); | |
91 | ||
92 | u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf, | |
93 | size_t len) | |
94 | { | |
e81446af CK |
95 | if (sg_pcopy_from_buffer(req->sg, req->sg_cnt, buf, len, off) != len) { |
96 | req->error_loc = offsetof(struct nvme_common_command, dptr); | |
a07b4970 | 97 | return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR; |
e81446af | 98 | } |
a07b4970 CH |
99 | return 0; |
100 | } | |
101 | ||
102 | u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, size_t len) | |
103 | { | |
e81446af CK |
104 | if (sg_pcopy_to_buffer(req->sg, req->sg_cnt, buf, len, off) != len) { |
105 | req->error_loc = offsetof(struct nvme_common_command, dptr); | |
a07b4970 | 106 | return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR; |
e81446af | 107 | } |
a07b4970 CH |
108 | return 0; |
109 | } | |
110 | ||
c7759fff CH |
111 | u16 nvmet_zero_sgl(struct nvmet_req *req, off_t off, size_t len) |
112 | { | |
e81446af CK |
113 | if (sg_zero_buffer(req->sg, req->sg_cnt, len, off) != len) { |
114 | req->error_loc = offsetof(struct nvme_common_command, dptr); | |
c7759fff | 115 | return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR; |
e81446af | 116 | } |
c7759fff CH |
117 | return 0; |
118 | } | |
119 | ||
245067e3 | 120 | static u32 nvmet_max_nsid(struct nvmet_subsys *subsys) |
ba2dec35 | 121 | { |
7774e77e CK |
122 | struct nvmet_ns *cur; |
123 | unsigned long idx; | |
245067e3 | 124 | u32 nsid = 0; |
ba2dec35 | 125 | |
7774e77e CK |
126 | xa_for_each(&subsys->namespaces, idx, cur) |
127 | nsid = cur->nsid; | |
ba2dec35 | 128 | |
7774e77e | 129 | return nsid; |
ba2dec35 RS |
130 | } |
131 | ||
a07b4970 CH |
132 | static u32 nvmet_async_event_result(struct nvmet_async_event *aen) |
133 | { | |
134 | return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16); | |
135 | } | |
136 | ||
819f7b88 CK |
137 | static void nvmet_async_events_failall(struct nvmet_ctrl *ctrl) |
138 | { | |
819f7b88 CK |
139 | struct nvmet_req *req; |
140 | ||
141 | mutex_lock(&ctrl->lock); | |
142 | while (ctrl->nr_async_event_cmds) { | |
143 | req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; | |
144 | mutex_unlock(&ctrl->lock); | |
8bb6cb9b | 145 | nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR); |
819f7b88 CK |
146 | mutex_lock(&ctrl->lock); |
147 | } | |
148 | mutex_unlock(&ctrl->lock); | |
149 | } | |
150 | ||
151 | static void nvmet_async_events_process(struct nvmet_ctrl *ctrl) | |
a07b4970 | 152 | { |
a07b4970 CH |
153 | struct nvmet_async_event *aen; |
154 | struct nvmet_req *req; | |
155 | ||
1cdf9f76 DM |
156 | mutex_lock(&ctrl->lock); |
157 | while (ctrl->nr_async_event_cmds && !list_empty(&ctrl->async_events)) { | |
158 | aen = list_first_entry(&ctrl->async_events, | |
159 | struct nvmet_async_event, entry); | |
a07b4970 | 160 | req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; |
819f7b88 | 161 | nvmet_set_result(req, nvmet_async_event_result(aen)); |
a07b4970 CH |
162 | |
163 | list_del(&aen->entry); | |
164 | kfree(aen); | |
165 | ||
166 | mutex_unlock(&ctrl->lock); | |
696ece75 | 167 | trace_nvmet_async_event(ctrl, req->cqe->result.u32); |
819f7b88 | 168 | nvmet_req_complete(req, 0); |
1cdf9f76 | 169 | mutex_lock(&ctrl->lock); |
a07b4970 | 170 | } |
1cdf9f76 | 171 | mutex_unlock(&ctrl->lock); |
a07b4970 CH |
172 | } |
173 | ||
0f5be6a4 DW |
174 | static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) |
175 | { | |
64f5e9cd | 176 | struct nvmet_async_event *aen, *tmp; |
0f5be6a4 DW |
177 | |
178 | mutex_lock(&ctrl->lock); | |
64f5e9cd SG |
179 | list_for_each_entry_safe(aen, tmp, &ctrl->async_events, entry) { |
180 | list_del(&aen->entry); | |
181 | kfree(aen); | |
0f5be6a4 DW |
182 | } |
183 | mutex_unlock(&ctrl->lock); | |
184 | } | |
185 | ||
186 | static void nvmet_async_event_work(struct work_struct *work) | |
187 | { | |
188 | struct nvmet_ctrl *ctrl = | |
189 | container_of(work, struct nvmet_ctrl, async_event_work); | |
190 | ||
819f7b88 | 191 | nvmet_async_events_process(ctrl); |
0f5be6a4 DW |
192 | } |
193 | ||
b662a078 | 194 | void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, |
a07b4970 CH |
195 | u8 event_info, u8 log_page) |
196 | { | |
197 | struct nvmet_async_event *aen; | |
198 | ||
199 | aen = kmalloc(sizeof(*aen), GFP_KERNEL); | |
200 | if (!aen) | |
201 | return; | |
202 | ||
203 | aen->event_type = event_type; | |
204 | aen->event_info = event_info; | |
205 | aen->log_page = log_page; | |
206 | ||
207 | mutex_lock(&ctrl->lock); | |
208 | list_add_tail(&aen->entry, &ctrl->async_events); | |
209 | mutex_unlock(&ctrl->lock); | |
210 | ||
8832cf92 | 211 | queue_work(nvmet_wq, &ctrl->async_event_work); |
a07b4970 CH |
212 | } |
213 | ||
c16734ea CH |
214 | static void nvmet_add_to_changed_ns_log(struct nvmet_ctrl *ctrl, __le32 nsid) |
215 | { | |
216 | u32 i; | |
217 | ||
218 | mutex_lock(&ctrl->lock); | |
219 | if (ctrl->nr_changed_ns > NVME_MAX_CHANGED_NAMESPACES) | |
220 | goto out_unlock; | |
221 | ||
222 | for (i = 0; i < ctrl->nr_changed_ns; i++) { | |
223 | if (ctrl->changed_ns_list[i] == nsid) | |
224 | goto out_unlock; | |
225 | } | |
226 | ||
227 | if (ctrl->nr_changed_ns == NVME_MAX_CHANGED_NAMESPACES) { | |
228 | ctrl->changed_ns_list[0] = cpu_to_le32(0xffffffff); | |
229 | ctrl->nr_changed_ns = U32_MAX; | |
230 | goto out_unlock; | |
231 | } | |
232 | ||
233 | ctrl->changed_ns_list[ctrl->nr_changed_ns++] = nsid; | |
234 | out_unlock: | |
235 | mutex_unlock(&ctrl->lock); | |
236 | } | |
237 | ||
dedf0be5 | 238 | void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid) |
c16734ea CH |
239 | { |
240 | struct nvmet_ctrl *ctrl; | |
241 | ||
013a63ef MG |
242 | lockdep_assert_held(&subsys->lock); |
243 | ||
c16734ea CH |
244 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { |
245 | nvmet_add_to_changed_ns_log(ctrl, cpu_to_le32(nsid)); | |
7114ddeb | 246 | if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_NS_ATTR)) |
c86b8f7b | 247 | continue; |
c16734ea CH |
248 | nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, |
249 | NVME_AER_NOTICE_NS_CHANGED, | |
250 | NVME_LOG_CHANGED_NS); | |
251 | } | |
252 | } | |
253 | ||
62ac0d32 CH |
254 | void nvmet_send_ana_event(struct nvmet_subsys *subsys, |
255 | struct nvmet_port *port) | |
256 | { | |
257 | struct nvmet_ctrl *ctrl; | |
258 | ||
259 | mutex_lock(&subsys->lock); | |
260 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { | |
261 | if (port && ctrl->port != port) | |
262 | continue; | |
7114ddeb | 263 | if (nvmet_aen_bit_disabled(ctrl, NVME_AEN_BIT_ANA_CHANGE)) |
62ac0d32 CH |
264 | continue; |
265 | nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, | |
266 | NVME_AER_NOTICE_ANA, NVME_LOG_ANA); | |
267 | } | |
268 | mutex_unlock(&subsys->lock); | |
269 | } | |
270 | ||
271 | void nvmet_port_send_ana_event(struct nvmet_port *port) | |
272 | { | |
273 | struct nvmet_subsys_link *p; | |
274 | ||
275 | down_read(&nvmet_config_sem); | |
276 | list_for_each_entry(p, &port->subsystems, entry) | |
277 | nvmet_send_ana_event(p->subsys, port); | |
278 | up_read(&nvmet_config_sem); | |
279 | } | |
280 | ||
e929f06d | 281 | int nvmet_register_transport(const struct nvmet_fabrics_ops *ops) |
a07b4970 CH |
282 | { |
283 | int ret = 0; | |
284 | ||
285 | down_write(&nvmet_config_sem); | |
286 | if (nvmet_transports[ops->type]) | |
287 | ret = -EINVAL; | |
288 | else | |
289 | nvmet_transports[ops->type] = ops; | |
290 | up_write(&nvmet_config_sem); | |
291 | ||
292 | return ret; | |
293 | } | |
294 | EXPORT_SYMBOL_GPL(nvmet_register_transport); | |
295 | ||
e929f06d | 296 | void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops) |
a07b4970 CH |
297 | { |
298 | down_write(&nvmet_config_sem); | |
299 | nvmet_transports[ops->type] = NULL; | |
300 | up_write(&nvmet_config_sem); | |
301 | } | |
302 | EXPORT_SYMBOL_GPL(nvmet_unregister_transport); | |
303 | ||
3aed8673 LG |
304 | void nvmet_port_del_ctrls(struct nvmet_port *port, struct nvmet_subsys *subsys) |
305 | { | |
306 | struct nvmet_ctrl *ctrl; | |
307 | ||
308 | mutex_lock(&subsys->lock); | |
309 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { | |
310 | if (ctrl->port == port) | |
311 | ctrl->ops->delete_ctrl(ctrl); | |
312 | } | |
313 | mutex_unlock(&subsys->lock); | |
314 | } | |
315 | ||
a07b4970 CH |
316 | int nvmet_enable_port(struct nvmet_port *port) |
317 | { | |
e929f06d | 318 | const struct nvmet_fabrics_ops *ops; |
a07b4970 CH |
319 | int ret; |
320 | ||
321 | lockdep_assert_held(&nvmet_config_sem); | |
322 | ||
323 | ops = nvmet_transports[port->disc_addr.trtype]; | |
324 | if (!ops) { | |
325 | up_write(&nvmet_config_sem); | |
326 | request_module("nvmet-transport-%d", port->disc_addr.trtype); | |
327 | down_write(&nvmet_config_sem); | |
328 | ops = nvmet_transports[port->disc_addr.trtype]; | |
329 | if (!ops) { | |
330 | pr_err("transport type %d not supported\n", | |
331 | port->disc_addr.trtype); | |
332 | return -EINVAL; | |
333 | } | |
334 | } | |
335 | ||
336 | if (!try_module_get(ops->owner)) | |
337 | return -EINVAL; | |
338 | ||
ea52ac1c IR |
339 | /* |
340 | * If the user requested PI support and the transport isn't pi capable, | |
341 | * don't enable the port. | |
342 | */ | |
6fa350f7 | 343 | if (port->pi_enable && !(ops->flags & NVMF_METADATA_SUPPORTED)) { |
ea52ac1c IR |
344 | pr_err("T10-PI is not supported by transport type %d\n", |
345 | port->disc_addr.trtype); | |
346 | ret = -EINVAL; | |
347 | goto out_put; | |
a07b4970 CH |
348 | } |
349 | ||
ea52ac1c IR |
350 | ret = ops->add_port(port); |
351 | if (ret) | |
352 | goto out_put; | |
353 | ||
0d5ee2b2 SW |
354 | /* If the transport didn't set inline_data_size, then disable it. */ |
355 | if (port->inline_data_size < 0) | |
356 | port->inline_data_size = 0; | |
357 | ||
a07b4970 | 358 | port->enabled = true; |
9d09dd8d | 359 | port->tr_ops = ops; |
a07b4970 | 360 | return 0; |
ea52ac1c IR |
361 | |
362 | out_put: | |
363 | module_put(ops->owner); | |
364 | return ret; | |
a07b4970 CH |
365 | } |
366 | ||
367 | void nvmet_disable_port(struct nvmet_port *port) | |
368 | { | |
e929f06d | 369 | const struct nvmet_fabrics_ops *ops; |
a07b4970 CH |
370 | |
371 | lockdep_assert_held(&nvmet_config_sem); | |
372 | ||
373 | port->enabled = false; | |
9d09dd8d | 374 | port->tr_ops = NULL; |
a07b4970 CH |
375 | |
376 | ops = nvmet_transports[port->disc_addr.trtype]; | |
377 | ops->remove_port(port); | |
378 | module_put(ops->owner); | |
379 | } | |
380 | ||
381 | static void nvmet_keep_alive_timer(struct work_struct *work) | |
382 | { | |
383 | struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work), | |
384 | struct nvmet_ctrl, ka_work); | |
aaeadd70 | 385 | bool reset_tbkas = ctrl->reset_tbkas; |
c09305ae | 386 | |
aaeadd70 SG |
387 | ctrl->reset_tbkas = false; |
388 | if (reset_tbkas) { | |
c09305ae SG |
389 | pr_debug("ctrl %d reschedule traffic based keep-alive timer\n", |
390 | ctrl->cntlid); | |
8832cf92 | 391 | queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); |
c09305ae SG |
392 | return; |
393 | } | |
a07b4970 CH |
394 | |
395 | pr_err("ctrl %d keep-alive timer (%d seconds) expired!\n", | |
396 | ctrl->cntlid, ctrl->kato); | |
397 | ||
23a8ed4a | 398 | nvmet_ctrl_fatal_error(ctrl); |
a07b4970 CH |
399 | } |
400 | ||
4e683c48 | 401 | void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl) |
a07b4970 | 402 | { |
0d3b6a8d AE |
403 | if (unlikely(ctrl->kato == 0)) |
404 | return; | |
405 | ||
a07b4970 CH |
406 | pr_debug("ctrl %d start keep-alive timer for %d secs\n", |
407 | ctrl->cntlid, ctrl->kato); | |
408 | ||
8832cf92 | 409 | queue_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); |
a07b4970 CH |
410 | } |
411 | ||
4e683c48 | 412 | void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl) |
a07b4970 | 413 | { |
0d3b6a8d AE |
414 | if (unlikely(ctrl->kato == 0)) |
415 | return; | |
416 | ||
a07b4970 CH |
417 | pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid); |
418 | ||
419 | cancel_delayed_work_sync(&ctrl->ka_work); | |
420 | } | |
421 | ||
3a1f7c79 | 422 | u16 nvmet_req_find_ns(struct nvmet_req *req) |
a07b4970 | 423 | { |
3a1f7c79 | 424 | u32 nsid = le32_to_cpu(req->cmd->common.nsid); |
a07b4970 | 425 | |
20c2c3bb | 426 | req->ns = xa_load(&nvmet_req_subsys(req)->namespaces, nsid); |
3a1f7c79 CK |
427 | if (unlikely(!req->ns)) { |
428 | req->error_loc = offsetof(struct nvme_common_command, nsid); | |
429 | return NVME_SC_INVALID_NS | NVME_SC_DNR; | |
430 | } | |
a07b4970 | 431 | |
3a1f7c79 CK |
432 | percpu_ref_get(&req->ns->ref); |
433 | return NVME_SC_SUCCESS; | |
a07b4970 CH |
434 | } |
435 | ||
436 | static void nvmet_destroy_namespace(struct percpu_ref *ref) | |
437 | { | |
438 | struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref); | |
439 | ||
440 | complete(&ns->disable_done); | |
441 | } | |
442 | ||
443 | void nvmet_put_namespace(struct nvmet_ns *ns) | |
444 | { | |
445 | percpu_ref_put(&ns->ref); | |
446 | } | |
447 | ||
d5eff33e CK |
448 | static void nvmet_ns_dev_disable(struct nvmet_ns *ns) |
449 | { | |
450 | nvmet_bdev_ns_disable(ns); | |
451 | nvmet_file_ns_disable(ns); | |
452 | } | |
453 | ||
c6925093 LG |
454 | static int nvmet_p2pmem_ns_enable(struct nvmet_ns *ns) |
455 | { | |
456 | int ret; | |
457 | struct pci_dev *p2p_dev; | |
458 | ||
459 | if (!ns->use_p2pmem) | |
460 | return 0; | |
461 | ||
462 | if (!ns->bdev) { | |
463 | pr_err("peer-to-peer DMA is not supported by non-block device namespaces\n"); | |
464 | return -EINVAL; | |
465 | } | |
466 | ||
e556f6ba | 467 | if (!blk_queue_pci_p2pdma(ns->bdev->bd_disk->queue)) { |
c6925093 LG |
468 | pr_err("peer-to-peer DMA is not supported by the driver of %s\n", |
469 | ns->device_path); | |
470 | return -EINVAL; | |
471 | } | |
472 | ||
473 | if (ns->p2p_dev) { | |
474 | ret = pci_p2pdma_distance(ns->p2p_dev, nvmet_ns_dev(ns), true); | |
475 | if (ret < 0) | |
476 | return -EINVAL; | |
477 | } else { | |
478 | /* | |
479 | * Right now we just check that there is p2pmem available so | |
480 | * we can report an error to the user right away if there | |
481 | * is not. We'll find the actual device to use once we | |
482 | * setup the controller when the port's device is available. | |
483 | */ | |
484 | ||
485 | p2p_dev = pci_p2pmem_find(nvmet_ns_dev(ns)); | |
486 | if (!p2p_dev) { | |
487 | pr_err("no peer-to-peer memory is available for %s\n", | |
488 | ns->device_path); | |
489 | return -EINVAL; | |
490 | } | |
491 | ||
492 | pci_dev_put(p2p_dev); | |
493 | } | |
494 | ||
495 | return 0; | |
496 | } | |
497 | ||
498 | /* | |
499 | * Note: ctrl->subsys->lock should be held when calling this function | |
500 | */ | |
501 | static void nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl *ctrl, | |
502 | struct nvmet_ns *ns) | |
503 | { | |
504 | struct device *clients[2]; | |
505 | struct pci_dev *p2p_dev; | |
506 | int ret; | |
507 | ||
21d3bbdd | 508 | if (!ctrl->p2p_client || !ns->use_p2pmem) |
c6925093 LG |
509 | return; |
510 | ||
511 | if (ns->p2p_dev) { | |
512 | ret = pci_p2pdma_distance(ns->p2p_dev, ctrl->p2p_client, true); | |
513 | if (ret < 0) | |
514 | return; | |
515 | ||
516 | p2p_dev = pci_dev_get(ns->p2p_dev); | |
517 | } else { | |
518 | clients[0] = ctrl->p2p_client; | |
519 | clients[1] = nvmet_ns_dev(ns); | |
520 | ||
521 | p2p_dev = pci_p2pmem_find_many(clients, ARRAY_SIZE(clients)); | |
522 | if (!p2p_dev) { | |
523 | pr_err("no peer-to-peer memory is available that's supported by %s and %s\n", | |
524 | dev_name(ctrl->p2p_client), ns->device_path); | |
525 | return; | |
526 | } | |
527 | } | |
528 | ||
529 | ret = radix_tree_insert(&ctrl->p2p_ns_map, ns->nsid, p2p_dev); | |
530 | if (ret < 0) | |
531 | pci_dev_put(p2p_dev); | |
532 | ||
533 | pr_info("using p2pmem on %s for nsid %d\n", pci_name(p2p_dev), | |
534 | ns->nsid); | |
535 | } | |
536 | ||
da783733 | 537 | bool nvmet_ns_revalidate(struct nvmet_ns *ns) |
463c5fab | 538 | { |
de124f42 CK |
539 | loff_t oldsize = ns->size; |
540 | ||
463c5fab CK |
541 | if (ns->bdev) |
542 | nvmet_bdev_ns_revalidate(ns); | |
543 | else | |
544 | nvmet_file_ns_revalidate(ns); | |
de124f42 | 545 | |
da783733 | 546 | return oldsize != ns->size; |
463c5fab CK |
547 | } |
548 | ||
a07b4970 CH |
549 | int nvmet_ns_enable(struct nvmet_ns *ns) |
550 | { | |
551 | struct nvmet_subsys *subsys = ns->subsys; | |
c6925093 | 552 | struct nvmet_ctrl *ctrl; |
793c7cfc | 553 | int ret; |
a07b4970 CH |
554 | |
555 | mutex_lock(&subsys->lock); | |
793c7cfc | 556 | ret = 0; |
ba76af67 | 557 | |
ab7a2737 | 558 | if (nvmet_is_passthru_subsys(subsys)) { |
ba76af67 LG |
559 | pr_info("cannot enable both passthru and regular namespaces for a single subsystem"); |
560 | goto out_unlock; | |
561 | } | |
562 | ||
e4fcf07c | 563 | if (ns->enabled) |
a07b4970 CH |
564 | goto out_unlock; |
565 | ||
e84c2091 MG |
566 | ret = -EMFILE; |
567 | if (subsys->nr_namespaces == NVMET_MAX_NAMESPACES) | |
568 | goto out_unlock; | |
569 | ||
d5eff33e | 570 | ret = nvmet_bdev_ns_enable(ns); |
405a7519 | 571 | if (ret == -ENOTBLK) |
d5eff33e CK |
572 | ret = nvmet_file_ns_enable(ns); |
573 | if (ret) | |
a07b4970 | 574 | goto out_unlock; |
a07b4970 | 575 | |
c6925093 LG |
576 | ret = nvmet_p2pmem_ns_enable(ns); |
577 | if (ret) | |
a536b497 | 578 | goto out_dev_disable; |
c6925093 LG |
579 | |
580 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) | |
581 | nvmet_p2pmem_ns_add_p2p(ctrl, ns); | |
582 | ||
a07b4970 CH |
583 | ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, |
584 | 0, GFP_KERNEL); | |
585 | if (ret) | |
d5eff33e | 586 | goto out_dev_put; |
a07b4970 CH |
587 | |
588 | if (ns->nsid > subsys->max_nsid) | |
589 | subsys->max_nsid = ns->nsid; | |
590 | ||
7774e77e CK |
591 | ret = xa_insert(&subsys->namespaces, ns->nsid, ns, GFP_KERNEL); |
592 | if (ret) | |
593 | goto out_restore_subsys_maxnsid; | |
a07b4970 | 594 | |
793c7cfc | 595 | subsys->nr_namespaces++; |
a07b4970 | 596 | |
c16734ea | 597 | nvmet_ns_changed(subsys, ns->nsid); |
e4fcf07c | 598 | ns->enabled = true; |
a07b4970 CH |
599 | ret = 0; |
600 | out_unlock: | |
601 | mutex_unlock(&subsys->lock); | |
602 | return ret; | |
7774e77e CK |
603 | |
604 | out_restore_subsys_maxnsid: | |
605 | subsys->max_nsid = nvmet_max_nsid(subsys); | |
606 | percpu_ref_exit(&ns->ref); | |
d5eff33e | 607 | out_dev_put: |
c6925093 LG |
608 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) |
609 | pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid)); | |
a536b497 | 610 | out_dev_disable: |
d5eff33e | 611 | nvmet_ns_dev_disable(ns); |
a07b4970 CH |
612 | goto out_unlock; |
613 | } | |
614 | ||
615 | void nvmet_ns_disable(struct nvmet_ns *ns) | |
616 | { | |
617 | struct nvmet_subsys *subsys = ns->subsys; | |
c6925093 | 618 | struct nvmet_ctrl *ctrl; |
a07b4970 CH |
619 | |
620 | mutex_lock(&subsys->lock); | |
e4fcf07c SA |
621 | if (!ns->enabled) |
622 | goto out_unlock; | |
623 | ||
624 | ns->enabled = false; | |
7774e77e | 625 | xa_erase(&ns->subsys->namespaces, ns->nsid); |
ba2dec35 RS |
626 | if (ns->nsid == subsys->max_nsid) |
627 | subsys->max_nsid = nvmet_max_nsid(subsys); | |
c6925093 LG |
628 | |
629 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) | |
630 | pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid)); | |
631 | ||
a07b4970 CH |
632 | mutex_unlock(&subsys->lock); |
633 | ||
634 | /* | |
635 | * Now that we removed the namespaces from the lookup list, we | |
636 | * can kill the per_cpu ref and wait for any remaining references | |
637 | * to be dropped, as well as a RCU grace period for anyone only | |
638 | * using the namepace under rcu_read_lock(). Note that we can't | |
639 | * use call_rcu here as we need to ensure the namespaces have | |
640 | * been fully destroyed before unloading the module. | |
641 | */ | |
642 | percpu_ref_kill(&ns->ref); | |
643 | synchronize_rcu(); | |
644 | wait_for_completion(&ns->disable_done); | |
645 | percpu_ref_exit(&ns->ref); | |
646 | ||
647 | mutex_lock(&subsys->lock); | |
c6925093 | 648 | |
793c7cfc | 649 | subsys->nr_namespaces--; |
c16734ea | 650 | nvmet_ns_changed(subsys, ns->nsid); |
d5eff33e | 651 | nvmet_ns_dev_disable(ns); |
e4fcf07c | 652 | out_unlock: |
a07b4970 CH |
653 | mutex_unlock(&subsys->lock); |
654 | } | |
655 | ||
656 | void nvmet_ns_free(struct nvmet_ns *ns) | |
657 | { | |
658 | nvmet_ns_disable(ns); | |
659 | ||
72efd25d CH |
660 | down_write(&nvmet_ana_sem); |
661 | nvmet_ana_group_enabled[ns->anagrpid]--; | |
662 | up_write(&nvmet_ana_sem); | |
663 | ||
a07b4970 CH |
664 | kfree(ns->device_path); |
665 | kfree(ns); | |
666 | } | |
667 | ||
668 | struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid) | |
669 | { | |
670 | struct nvmet_ns *ns; | |
671 | ||
672 | ns = kzalloc(sizeof(*ns), GFP_KERNEL); | |
673 | if (!ns) | |
674 | return NULL; | |
675 | ||
a07b4970 CH |
676 | init_completion(&ns->disable_done); |
677 | ||
678 | ns->nsid = nsid; | |
679 | ns->subsys = subsys; | |
72efd25d CH |
680 | |
681 | down_write(&nvmet_ana_sem); | |
682 | ns->anagrpid = NVMET_DEFAULT_ANA_GRPID; | |
683 | nvmet_ana_group_enabled[ns->anagrpid]++; | |
684 | up_write(&nvmet_ana_sem); | |
685 | ||
637dc0f3 | 686 | uuid_gen(&ns->uuid); |
55eb942e | 687 | ns->buffered_io = false; |
ab5d0b38 | 688 | ns->csi = NVME_CSI_NVM; |
a07b4970 CH |
689 | |
690 | return ns; | |
691 | } | |
692 | ||
e6a622fd | 693 | static void nvmet_update_sq_head(struct nvmet_req *req) |
a07b4970 | 694 | { |
f9cf2a64 | 695 | if (req->sq->size) { |
e6a622fd SG |
696 | u32 old_sqhd, new_sqhd; |
697 | ||
bbf5410b | 698 | old_sqhd = READ_ONCE(req->sq->sqhd); |
f9cf2a64 | 699 | do { |
f9cf2a64 | 700 | new_sqhd = (old_sqhd + 1) % req->sq->size; |
bbf5410b | 701 | } while (!try_cmpxchg(&req->sq->sqhd, &old_sqhd, new_sqhd)); |
f9cf2a64 | 702 | } |
fc6c9730 | 703 | req->cqe->sq_head = cpu_to_le16(req->sq->sqhd & 0x0000FFFF); |
e6a622fd SG |
704 | } |
705 | ||
76574f37 CK |
706 | static void nvmet_set_error(struct nvmet_req *req, u16 status) |
707 | { | |
708 | struct nvmet_ctrl *ctrl = req->sq->ctrl; | |
709 | struct nvme_error_slot *new_error_slot; | |
710 | unsigned long flags; | |
711 | ||
fc6c9730 | 712 | req->cqe->status = cpu_to_le16(status << 1); |
76574f37 | 713 | |
5698b805 | 714 | if (!ctrl || req->error_loc == NVMET_NO_ERROR_LOC) |
76574f37 CK |
715 | return; |
716 | ||
717 | spin_lock_irqsave(&ctrl->error_lock, flags); | |
718 | ctrl->err_counter++; | |
719 | new_error_slot = | |
720 | &ctrl->slots[ctrl->err_counter % NVMET_ERROR_LOG_SLOTS]; | |
721 | ||
722 | new_error_slot->error_count = cpu_to_le64(ctrl->err_counter); | |
723 | new_error_slot->sqid = cpu_to_le16(req->sq->qid); | |
724 | new_error_slot->cmdid = cpu_to_le16(req->cmd->common.command_id); | |
725 | new_error_slot->status_field = cpu_to_le16(status << 1); | |
726 | new_error_slot->param_error_location = cpu_to_le16(req->error_loc); | |
727 | new_error_slot->lba = cpu_to_le64(req->error_slba); | |
728 | new_error_slot->nsid = req->cmd->common.nsid; | |
729 | spin_unlock_irqrestore(&ctrl->error_lock, flags); | |
730 | ||
731 | /* set the more bit for this request */ | |
fc6c9730 | 732 | req->cqe->status |= cpu_to_le16(1 << 14); |
76574f37 CK |
733 | } |
734 | ||
e6a622fd SG |
735 | static void __nvmet_req_complete(struct nvmet_req *req, u16 status) |
736 | { | |
6a02a61e BVA |
737 | struct nvmet_ns *ns = req->ns; |
738 | ||
e6a622fd SG |
739 | if (!req->sq->sqhd_disabled) |
740 | nvmet_update_sq_head(req); | |
fc6c9730 MG |
741 | req->cqe->sq_id = cpu_to_le16(req->sq->qid); |
742 | req->cqe->command_id = req->cmd->common.command_id; | |
76574f37 | 743 | |
cb019da3 | 744 | if (unlikely(status)) |
76574f37 | 745 | nvmet_set_error(req, status); |
a5448fdc MI |
746 | |
747 | trace_nvmet_req_complete(req); | |
748 | ||
a07b4970 | 749 | req->ops->queue_response(req); |
6a02a61e BVA |
750 | if (ns) |
751 | nvmet_put_namespace(ns); | |
a07b4970 CH |
752 | } |
753 | ||
754 | void nvmet_req_complete(struct nvmet_req *req, u16 status) | |
755 | { | |
756 | __nvmet_req_complete(req, status); | |
757 | percpu_ref_put(&req->sq->ref); | |
758 | } | |
759 | EXPORT_SYMBOL_GPL(nvmet_req_complete); | |
760 | ||
761 | void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, | |
762 | u16 qid, u16 size) | |
763 | { | |
764 | cq->qid = qid; | |
765 | cq->size = size; | |
a07b4970 CH |
766 | } |
767 | ||
768 | void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, | |
769 | u16 qid, u16 size) | |
770 | { | |
bb1cc747 | 771 | sq->sqhd = 0; |
a07b4970 CH |
772 | sq->qid = qid; |
773 | sq->size = size; | |
774 | ||
775 | ctrl->sqs[qid] = sq; | |
776 | } | |
777 | ||
427242ce SG |
778 | static void nvmet_confirm_sq(struct percpu_ref *ref) |
779 | { | |
780 | struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref); | |
781 | ||
782 | complete(&sq->confirm_done); | |
783 | } | |
784 | ||
a07b4970 CH |
785 | void nvmet_sq_destroy(struct nvmet_sq *sq) |
786 | { | |
0f5be6a4 DW |
787 | struct nvmet_ctrl *ctrl = sq->ctrl; |
788 | ||
a07b4970 CH |
789 | /* |
790 | * If this is the admin queue, complete all AERs so that our | |
791 | * queue doesn't have outstanding requests on it. | |
792 | */ | |
64f5e9cd | 793 | if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq) |
819f7b88 | 794 | nvmet_async_events_failall(ctrl); |
427242ce SG |
795 | percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq); |
796 | wait_for_completion(&sq->confirm_done); | |
a07b4970 CH |
797 | wait_for_completion(&sq->free_done); |
798 | percpu_ref_exit(&sq->ref); | |
db1312dd | 799 | nvmet_auth_sq_free(sq); |
a07b4970 | 800 | |
0f5be6a4 | 801 | if (ctrl) { |
aaeadd70 SG |
802 | /* |
803 | * The teardown flow may take some time, and the host may not | |
804 | * send us keep-alive during this period, hence reset the | |
805 | * traffic based keep-alive timer so we don't trigger a | |
806 | * controller teardown as a result of a keep-alive expiration. | |
807 | */ | |
808 | ctrl->reset_tbkas = true; | |
b71df126 | 809 | sq->ctrl->sqs[sq->qid] = NULL; |
0f5be6a4 | 810 | nvmet_ctrl_put(ctrl); |
a07b4970 CH |
811 | sq->ctrl = NULL; /* allows reusing the queue later */ |
812 | } | |
813 | } | |
814 | EXPORT_SYMBOL_GPL(nvmet_sq_destroy); | |
815 | ||
816 | static void nvmet_sq_free(struct percpu_ref *ref) | |
817 | { | |
818 | struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref); | |
819 | ||
820 | complete(&sq->free_done); | |
821 | } | |
822 | ||
823 | int nvmet_sq_init(struct nvmet_sq *sq) | |
824 | { | |
825 | int ret; | |
826 | ||
827 | ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL); | |
828 | if (ret) { | |
829 | pr_err("percpu_ref init failed!\n"); | |
830 | return ret; | |
831 | } | |
832 | init_completion(&sq->free_done); | |
427242ce | 833 | init_completion(&sq->confirm_done); |
1befd944 | 834 | nvmet_auth_sq_init(sq); |
a07b4970 CH |
835 | |
836 | return 0; | |
837 | } | |
838 | EXPORT_SYMBOL_GPL(nvmet_sq_init); | |
839 | ||
72efd25d CH |
840 | static inline u16 nvmet_check_ana_state(struct nvmet_port *port, |
841 | struct nvmet_ns *ns) | |
842 | { | |
843 | enum nvme_ana_state state = port->ana_state[ns->anagrpid]; | |
844 | ||
845 | if (unlikely(state == NVME_ANA_INACCESSIBLE)) | |
846 | return NVME_SC_ANA_INACCESSIBLE; | |
847 | if (unlikely(state == NVME_ANA_PERSISTENT_LOSS)) | |
848 | return NVME_SC_ANA_PERSISTENT_LOSS; | |
849 | if (unlikely(state == NVME_ANA_CHANGE)) | |
850 | return NVME_SC_ANA_TRANSITION; | |
851 | return 0; | |
852 | } | |
853 | ||
dedf0be5 CK |
854 | static inline u16 nvmet_io_cmd_check_access(struct nvmet_req *req) |
855 | { | |
856 | if (unlikely(req->ns->readonly)) { | |
857 | switch (req->cmd->common.opcode) { | |
858 | case nvme_cmd_read: | |
859 | case nvme_cmd_flush: | |
860 | break; | |
861 | default: | |
862 | return NVME_SC_NS_WRITE_PROTECTED; | |
863 | } | |
864 | } | |
865 | ||
866 | return 0; | |
867 | } | |
868 | ||
d5eff33e CK |
869 | static u16 nvmet_parse_io_cmd(struct nvmet_req *req) |
870 | { | |
6490c9ed | 871 | struct nvme_command *cmd = req->cmd; |
d5eff33e CK |
872 | u16 ret; |
873 | ||
6490c9ed HR |
874 | if (nvme_is_fabrics(cmd)) |
875 | return nvmet_parse_fabrics_io_cmd(req); | |
876 | ||
db1312dd HR |
877 | if (unlikely(!nvmet_check_auth_status(req))) |
878 | return NVME_SC_AUTH_REQUIRED | NVME_SC_DNR; | |
879 | ||
7798df6f | 880 | ret = nvmet_check_ctrl_status(req); |
d5eff33e CK |
881 | if (unlikely(ret)) |
882 | return ret; | |
883 | ||
ab7a2737 | 884 | if (nvmet_is_passthru_req(req)) |
c1fef73f LG |
885 | return nvmet_parse_passthru_io_cmd(req); |
886 | ||
3a1f7c79 CK |
887 | ret = nvmet_req_find_ns(req); |
888 | if (unlikely(ret)) | |
889 | return ret; | |
890 | ||
72efd25d | 891 | ret = nvmet_check_ana_state(req->port, req->ns); |
e81446af CK |
892 | if (unlikely(ret)) { |
893 | req->error_loc = offsetof(struct nvme_common_command, nsid); | |
dedf0be5 | 894 | return ret; |
e81446af | 895 | } |
dedf0be5 | 896 | ret = nvmet_io_cmd_check_access(req); |
e81446af CK |
897 | if (unlikely(ret)) { |
898 | req->error_loc = offsetof(struct nvme_common_command, nsid); | |
72efd25d | 899 | return ret; |
e81446af | 900 | } |
d5eff33e | 901 | |
ab5d0b38 CK |
902 | switch (req->ns->csi) { |
903 | case NVME_CSI_NVM: | |
904 | if (req->ns->file) | |
905 | return nvmet_file_parse_io_cmd(req); | |
906 | return nvmet_bdev_parse_io_cmd(req); | |
aaf2e048 CK |
907 | case NVME_CSI_ZNS: |
908 | if (IS_ENABLED(CONFIG_BLK_DEV_ZONED)) | |
909 | return nvmet_bdev_zns_parse_io_cmd(req); | |
910 | return NVME_SC_INVALID_IO_CMD_SET; | |
ab5d0b38 CK |
911 | default: |
912 | return NVME_SC_INVALID_IO_CMD_SET; | |
913 | } | |
d5eff33e CK |
914 | } |
915 | ||
a07b4970 | 916 | bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq, |
e929f06d | 917 | struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops) |
a07b4970 CH |
918 | { |
919 | u8 flags = req->cmd->common.flags; | |
920 | u16 status; | |
921 | ||
922 | req->cq = cq; | |
923 | req->sq = sq; | |
924 | req->ops = ops; | |
925 | req->sg = NULL; | |
c6e3f133 | 926 | req->metadata_sg = NULL; |
a07b4970 | 927 | req->sg_cnt = 0; |
c6e3f133 | 928 | req->metadata_sg_cnt = 0; |
5e62d5c9 | 929 | req->transfer_len = 0; |
c6e3f133 | 930 | req->metadata_len = 0; |
fc6c9730 MG |
931 | req->cqe->status = 0; |
932 | req->cqe->sq_head = 0; | |
423b4487 | 933 | req->ns = NULL; |
5698b805 | 934 | req->error_loc = NVMET_NO_ERROR_LOC; |
e4a97625 | 935 | req->error_slba = 0; |
a07b4970 CH |
936 | |
937 | /* no support for fused commands yet */ | |
938 | if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) { | |
e81446af | 939 | req->error_loc = offsetof(struct nvme_common_command, flags); |
a07b4970 CH |
940 | status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; |
941 | goto fail; | |
942 | } | |
943 | ||
bffd2b61 MG |
944 | /* |
945 | * For fabrics, PSDT field shall describe metadata pointer (MPTR) that | |
946 | * contains an address of a single contiguous physical buffer that is | |
947 | * byte aligned. | |
948 | */ | |
949 | if (unlikely((flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METABUF)) { | |
e81446af | 950 | req->error_loc = offsetof(struct nvme_common_command, flags); |
a07b4970 CH |
951 | status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; |
952 | goto fail; | |
953 | } | |
954 | ||
955 | if (unlikely(!req->sq->ctrl)) | |
d84dd8cd | 956 | /* will return an error for any non-connect command: */ |
a07b4970 CH |
957 | status = nvmet_parse_connect_cmd(req); |
958 | else if (likely(req->sq->qid != 0)) | |
959 | status = nvmet_parse_io_cmd(req); | |
a07b4970 CH |
960 | else |
961 | status = nvmet_parse_admin_cmd(req); | |
962 | ||
963 | if (status) | |
964 | goto fail; | |
965 | ||
3c3751f2 CK |
966 | trace_nvmet_req_init(req, req->cmd); |
967 | ||
a07b4970 CH |
968 | if (unlikely(!percpu_ref_tryget_live(&sq->ref))) { |
969 | status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; | |
970 | goto fail; | |
971 | } | |
972 | ||
c09305ae | 973 | if (sq->ctrl) |
aaeadd70 | 974 | sq->ctrl->reset_tbkas = true; |
c09305ae | 975 | |
a07b4970 CH |
976 | return true; |
977 | ||
978 | fail: | |
979 | __nvmet_req_complete(req, status); | |
980 | return false; | |
981 | } | |
982 | EXPORT_SYMBOL_GPL(nvmet_req_init); | |
983 | ||
549f01ae VI |
984 | void nvmet_req_uninit(struct nvmet_req *req) |
985 | { | |
986 | percpu_ref_put(&req->sq->ref); | |
423b4487 SG |
987 | if (req->ns) |
988 | nvmet_put_namespace(req->ns); | |
549f01ae VI |
989 | } |
990 | EXPORT_SYMBOL_GPL(nvmet_req_uninit); | |
991 | ||
136cc1ff | 992 | bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len) |
5e62d5c9 | 993 | { |
136cc1ff | 994 | if (unlikely(len != req->transfer_len)) { |
e81446af | 995 | req->error_loc = offsetof(struct nvme_common_command, dptr); |
5e62d5c9 | 996 | nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR); |
e9061c39 CH |
997 | return false; |
998 | } | |
999 | ||
1000 | return true; | |
1001 | } | |
136cc1ff | 1002 | EXPORT_SYMBOL_GPL(nvmet_check_transfer_len); |
e9061c39 | 1003 | |
b716e688 SG |
1004 | bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len) |
1005 | { | |
1006 | if (unlikely(data_len > req->transfer_len)) { | |
1007 | req->error_loc = offsetof(struct nvme_common_command, dptr); | |
1008 | nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR); | |
1009 | return false; | |
1010 | } | |
1011 | ||
1012 | return true; | |
1013 | } | |
1014 | ||
c6e3f133 | 1015 | static unsigned int nvmet_data_transfer_len(struct nvmet_req *req) |
5b2322e4 | 1016 | { |
c6e3f133 IR |
1017 | return req->transfer_len - req->metadata_len; |
1018 | } | |
c6925093 | 1019 | |
bcd9a079 MG |
1020 | static int nvmet_req_alloc_p2pmem_sgls(struct pci_dev *p2p_dev, |
1021 | struct nvmet_req *req) | |
c6e3f133 | 1022 | { |
bcd9a079 | 1023 | req->sg = pci_p2pmem_alloc_sgl(p2p_dev, &req->sg_cnt, |
c6e3f133 IR |
1024 | nvmet_data_transfer_len(req)); |
1025 | if (!req->sg) | |
1026 | goto out_err; | |
1027 | ||
1028 | if (req->metadata_len) { | |
bcd9a079 | 1029 | req->metadata_sg = pci_p2pmem_alloc_sgl(p2p_dev, |
c6e3f133 IR |
1030 | &req->metadata_sg_cnt, req->metadata_len); |
1031 | if (!req->metadata_sg) | |
1032 | goto out_free_sg; | |
1033 | } | |
bcd9a079 MG |
1034 | |
1035 | req->p2p_dev = p2p_dev; | |
1036 | ||
c6e3f133 IR |
1037 | return 0; |
1038 | out_free_sg: | |
1039 | pci_p2pmem_free_sgl(req->p2p_dev, req->sg); | |
1040 | out_err: | |
1041 | return -ENOMEM; | |
1042 | } | |
1043 | ||
bcd9a079 | 1044 | static struct pci_dev *nvmet_req_find_p2p_dev(struct nvmet_req *req) |
c6e3f133 | 1045 | { |
bcd9a079 MG |
1046 | if (!IS_ENABLED(CONFIG_PCI_P2PDMA) || |
1047 | !req->sq->ctrl || !req->sq->qid || !req->ns) | |
1048 | return NULL; | |
1049 | return radix_tree_lookup(&req->sq->ctrl->p2p_ns_map, req->ns->nsid); | |
c6e3f133 IR |
1050 | } |
1051 | ||
1052 | int nvmet_req_alloc_sgls(struct nvmet_req *req) | |
1053 | { | |
bcd9a079 MG |
1054 | struct pci_dev *p2p_dev = nvmet_req_find_p2p_dev(req); |
1055 | ||
1056 | if (p2p_dev && !nvmet_req_alloc_p2pmem_sgls(p2p_dev, req)) | |
c6e3f133 IR |
1057 | return 0; |
1058 | ||
1059 | req->sg = sgl_alloc(nvmet_data_transfer_len(req), GFP_KERNEL, | |
1060 | &req->sg_cnt); | |
e522f446 | 1061 | if (unlikely(!req->sg)) |
c6e3f133 IR |
1062 | goto out; |
1063 | ||
1064 | if (req->metadata_len) { | |
1065 | req->metadata_sg = sgl_alloc(req->metadata_len, GFP_KERNEL, | |
1066 | &req->metadata_sg_cnt); | |
1067 | if (unlikely(!req->metadata_sg)) | |
1068 | goto out_free; | |
1069 | } | |
5b2322e4 LG |
1070 | |
1071 | return 0; | |
c6e3f133 IR |
1072 | out_free: |
1073 | sgl_free(req->sg); | |
1074 | out: | |
1075 | return -ENOMEM; | |
5b2322e4 | 1076 | } |
c6e3f133 | 1077 | EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgls); |
5b2322e4 | 1078 | |
c6e3f133 | 1079 | void nvmet_req_free_sgls(struct nvmet_req *req) |
5b2322e4 | 1080 | { |
c6e3f133 | 1081 | if (req->p2p_dev) { |
c6925093 | 1082 | pci_p2pmem_free_sgl(req->p2p_dev, req->sg); |
c6e3f133 IR |
1083 | if (req->metadata_sg) |
1084 | pci_p2pmem_free_sgl(req->p2p_dev, req->metadata_sg); | |
bcd9a079 | 1085 | req->p2p_dev = NULL; |
c6e3f133 | 1086 | } else { |
c6925093 | 1087 | sgl_free(req->sg); |
c6e3f133 IR |
1088 | if (req->metadata_sg) |
1089 | sgl_free(req->metadata_sg); | |
1090 | } | |
c6925093 | 1091 | |
5b2322e4 | 1092 | req->sg = NULL; |
c6e3f133 | 1093 | req->metadata_sg = NULL; |
5b2322e4 | 1094 | req->sg_cnt = 0; |
c6e3f133 | 1095 | req->metadata_sg_cnt = 0; |
5b2322e4 | 1096 | } |
c6e3f133 | 1097 | EXPORT_SYMBOL_GPL(nvmet_req_free_sgls); |
5b2322e4 | 1098 | |
a07b4970 CH |
1099 | static inline bool nvmet_cc_en(u32 cc) |
1100 | { | |
ad4e05b2 | 1101 | return (cc >> NVME_CC_EN_SHIFT) & 0x1; |
a07b4970 CH |
1102 | } |
1103 | ||
1104 | static inline u8 nvmet_cc_css(u32 cc) | |
1105 | { | |
ad4e05b2 | 1106 | return (cc >> NVME_CC_CSS_SHIFT) & 0x7; |
a07b4970 CH |
1107 | } |
1108 | ||
1109 | static inline u8 nvmet_cc_mps(u32 cc) | |
1110 | { | |
ad4e05b2 | 1111 | return (cc >> NVME_CC_MPS_SHIFT) & 0xf; |
a07b4970 CH |
1112 | } |
1113 | ||
1114 | static inline u8 nvmet_cc_ams(u32 cc) | |
1115 | { | |
ad4e05b2 | 1116 | return (cc >> NVME_CC_AMS_SHIFT) & 0x7; |
a07b4970 CH |
1117 | } |
1118 | ||
1119 | static inline u8 nvmet_cc_shn(u32 cc) | |
1120 | { | |
ad4e05b2 | 1121 | return (cc >> NVME_CC_SHN_SHIFT) & 0x3; |
a07b4970 CH |
1122 | } |
1123 | ||
1124 | static inline u8 nvmet_cc_iosqes(u32 cc) | |
1125 | { | |
ad4e05b2 | 1126 | return (cc >> NVME_CC_IOSQES_SHIFT) & 0xf; |
a07b4970 CH |
1127 | } |
1128 | ||
1129 | static inline u8 nvmet_cc_iocqes(u32 cc) | |
1130 | { | |
ad4e05b2 | 1131 | return (cc >> NVME_CC_IOCQES_SHIFT) & 0xf; |
a07b4970 CH |
1132 | } |
1133 | ||
ab5d0b38 CK |
1134 | static inline bool nvmet_css_supported(u8 cc_css) |
1135 | { | |
63bc732c | 1136 | switch (cc_css << NVME_CC_CSS_SHIFT) { |
ab5d0b38 CK |
1137 | case NVME_CC_CSS_NVM: |
1138 | case NVME_CC_CSS_CSI: | |
1139 | return true; | |
1140 | default: | |
1141 | return false; | |
1142 | } | |
1143 | } | |
1144 | ||
a07b4970 CH |
1145 | static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl) |
1146 | { | |
1147 | lockdep_assert_held(&ctrl->lock); | |
1148 | ||
d218a8a3 SG |
1149 | /* |
1150 | * Only I/O controllers should verify iosqes,iocqes. | |
1151 | * Strictly speaking, the spec says a discovery controller | |
1152 | * should verify iosqes,iocqes are zeroed, however that | |
1153 | * would break backwards compatibility, so don't enforce it. | |
1154 | */ | |
a294711e | 1155 | if (!nvmet_is_disc_subsys(ctrl->subsys) && |
d218a8a3 SG |
1156 | (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES || |
1157 | nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES)) { | |
1158 | ctrl->csts = NVME_CSTS_CFS; | |
1159 | return; | |
1160 | } | |
1161 | ||
1162 | if (nvmet_cc_mps(ctrl->cc) != 0 || | |
a07b4970 | 1163 | nvmet_cc_ams(ctrl->cc) != 0 || |
ab5d0b38 | 1164 | !nvmet_css_supported(nvmet_cc_css(ctrl->cc))) { |
a07b4970 CH |
1165 | ctrl->csts = NVME_CSTS_CFS; |
1166 | return; | |
1167 | } | |
1168 | ||
1169 | ctrl->csts = NVME_CSTS_RDY; | |
d68a90e1 MG |
1170 | |
1171 | /* | |
1172 | * Controllers that are not yet enabled should not really enforce the | |
1173 | * keep alive timeout, but we still want to track a timeout and cleanup | |
1174 | * in case a host died before it enabled the controller. Hence, simply | |
1175 | * reset the keep alive timer when the controller is enabled. | |
1176 | */ | |
85bd23f3 | 1177 | if (ctrl->kato) |
ddd2b8de | 1178 | mod_delayed_work(nvmet_wq, &ctrl->ka_work, ctrl->kato * HZ); |
a07b4970 CH |
1179 | } |
1180 | ||
1181 | static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl) | |
1182 | { | |
1183 | lockdep_assert_held(&ctrl->lock); | |
1184 | ||
1185 | /* XXX: tear down queues? */ | |
1186 | ctrl->csts &= ~NVME_CSTS_RDY; | |
1187 | ctrl->cc = 0; | |
1188 | } | |
1189 | ||
1190 | void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new) | |
1191 | { | |
1192 | u32 old; | |
1193 | ||
1194 | mutex_lock(&ctrl->lock); | |
1195 | old = ctrl->cc; | |
1196 | ctrl->cc = new; | |
1197 | ||
1198 | if (nvmet_cc_en(new) && !nvmet_cc_en(old)) | |
1199 | nvmet_start_ctrl(ctrl); | |
1200 | if (!nvmet_cc_en(new) && nvmet_cc_en(old)) | |
1201 | nvmet_clear_ctrl(ctrl); | |
1202 | if (nvmet_cc_shn(new) && !nvmet_cc_shn(old)) { | |
1203 | nvmet_clear_ctrl(ctrl); | |
1204 | ctrl->csts |= NVME_CSTS_SHST_CMPLT; | |
1205 | } | |
1206 | if (!nvmet_cc_shn(new) && nvmet_cc_shn(old)) | |
1207 | ctrl->csts &= ~NVME_CSTS_SHST_CMPLT; | |
1208 | mutex_unlock(&ctrl->lock); | |
1209 | } | |
1210 | ||
1211 | static void nvmet_init_cap(struct nvmet_ctrl *ctrl) | |
1212 | { | |
1213 | /* command sets supported: NVMe command set: */ | |
1214 | ctrl->cap = (1ULL << 37); | |
ab5d0b38 CK |
1215 | /* Controller supports one or more I/O Command Sets */ |
1216 | ctrl->cap |= (1ULL << 43); | |
a07b4970 CH |
1217 | /* CC.EN timeout in 500msec units: */ |
1218 | ctrl->cap |= (15ULL << 24); | |
1219 | /* maximum queue entries supported: */ | |
6d1555cc MG |
1220 | if (ctrl->ops->get_max_queue_size) |
1221 | ctrl->cap |= ctrl->ops->get_max_queue_size(ctrl) - 1; | |
1222 | else | |
1223 | ctrl->cap |= NVMET_QUEUE_SIZE - 1; | |
77d651a6 | 1224 | |
ab7a2737 | 1225 | if (nvmet_is_passthru_subsys(ctrl->subsys)) |
77d651a6 | 1226 | nvmet_passthrough_override_cap(ctrl); |
a07b4970 CH |
1227 | } |
1228 | ||
de587804 CK |
1229 | struct nvmet_ctrl *nvmet_ctrl_find_get(const char *subsysnqn, |
1230 | const char *hostnqn, u16 cntlid, | |
1231 | struct nvmet_req *req) | |
a07b4970 | 1232 | { |
de587804 | 1233 | struct nvmet_ctrl *ctrl = NULL; |
a07b4970 | 1234 | struct nvmet_subsys *subsys; |
a07b4970 CH |
1235 | |
1236 | subsys = nvmet_find_get_subsys(req->port, subsysnqn); | |
1237 | if (!subsys) { | |
1238 | pr_warn("connect request for invalid subsystem %s!\n", | |
1239 | subsysnqn); | |
fc6c9730 | 1240 | req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn); |
de587804 | 1241 | goto out; |
a07b4970 CH |
1242 | } |
1243 | ||
1244 | mutex_lock(&subsys->lock); | |
1245 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { | |
1246 | if (ctrl->cntlid == cntlid) { | |
1247 | if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) { | |
1248 | pr_warn("hostnqn mismatch.\n"); | |
1249 | continue; | |
1250 | } | |
1251 | if (!kref_get_unless_zero(&ctrl->ref)) | |
1252 | continue; | |
1253 | ||
de587804 CK |
1254 | /* ctrl found */ |
1255 | goto found; | |
a07b4970 CH |
1256 | } |
1257 | } | |
1258 | ||
de587804 | 1259 | ctrl = NULL; /* ctrl not found */ |
a07b4970 CH |
1260 | pr_warn("could not find controller %d for subsys %s / host %s\n", |
1261 | cntlid, subsysnqn, hostnqn); | |
fc6c9730 | 1262 | req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid); |
a07b4970 | 1263 | |
de587804 | 1264 | found: |
a07b4970 CH |
1265 | mutex_unlock(&subsys->lock); |
1266 | nvmet_subsys_put(subsys); | |
de587804 CK |
1267 | out: |
1268 | return ctrl; | |
a07b4970 CH |
1269 | } |
1270 | ||
7798df6f | 1271 | u16 nvmet_check_ctrl_status(struct nvmet_req *req) |
64a0ca88 PP |
1272 | { |
1273 | if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) { | |
b40b83e3 | 1274 | pr_err("got cmd %d while CC.EN == 0 on qid = %d\n", |
7798df6f | 1275 | req->cmd->common.opcode, req->sq->qid); |
64a0ca88 PP |
1276 | return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR; |
1277 | } | |
1278 | ||
1279 | if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) { | |
b40b83e3 | 1280 | pr_err("got cmd %d while CSTS.RDY == 0 on qid = %d\n", |
7798df6f | 1281 | req->cmd->common.opcode, req->sq->qid); |
64a0ca88 PP |
1282 | return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR; |
1283 | } | |
db1312dd HR |
1284 | |
1285 | if (unlikely(!nvmet_check_auth_status(req))) { | |
1286 | pr_warn("qid %d not authenticated\n", req->sq->qid); | |
1287 | return NVME_SC_AUTH_REQUIRED | NVME_SC_DNR; | |
1288 | } | |
64a0ca88 PP |
1289 | return 0; |
1290 | } | |
1291 | ||
253928ee | 1292 | bool nvmet_host_allowed(struct nvmet_subsys *subsys, const char *hostnqn) |
a07b4970 CH |
1293 | { |
1294 | struct nvmet_host_link *p; | |
1295 | ||
253928ee SG |
1296 | lockdep_assert_held(&nvmet_config_sem); |
1297 | ||
a07b4970 CH |
1298 | if (subsys->allow_any_host) |
1299 | return true; | |
1300 | ||
a294711e | 1301 | if (nvmet_is_disc_subsys(subsys)) /* allow all access to disc subsys */ |
253928ee SG |
1302 | return true; |
1303 | ||
a07b4970 CH |
1304 | list_for_each_entry(p, &subsys->hosts, entry) { |
1305 | if (!strcmp(nvmet_host_name(p->host), hostnqn)) | |
1306 | return true; | |
1307 | } | |
1308 | ||
1309 | return false; | |
1310 | } | |
1311 | ||
c6925093 LG |
1312 | /* |
1313 | * Note: ctrl->subsys->lock should be held when calling this function | |
1314 | */ | |
1315 | static void nvmet_setup_p2p_ns_map(struct nvmet_ctrl *ctrl, | |
1316 | struct nvmet_req *req) | |
1317 | { | |
1318 | struct nvmet_ns *ns; | |
7774e77e | 1319 | unsigned long idx; |
c6925093 LG |
1320 | |
1321 | if (!req->p2p_client) | |
1322 | return; | |
1323 | ||
1324 | ctrl->p2p_client = get_device(req->p2p_client); | |
1325 | ||
7774e77e | 1326 | xa_for_each(&ctrl->subsys->namespaces, idx, ns) |
c6925093 LG |
1327 | nvmet_p2pmem_ns_add_p2p(ctrl, ns); |
1328 | } | |
1329 | ||
1330 | /* | |
1331 | * Note: ctrl->subsys->lock should be held when calling this function | |
1332 | */ | |
1333 | static void nvmet_release_p2p_ns_map(struct nvmet_ctrl *ctrl) | |
1334 | { | |
1335 | struct radix_tree_iter iter; | |
1336 | void __rcu **slot; | |
1337 | ||
1338 | radix_tree_for_each_slot(slot, &ctrl->p2p_ns_map, &iter, 0) | |
1339 | pci_dev_put(radix_tree_deref_slot(slot)); | |
1340 | ||
1341 | put_device(ctrl->p2p_client); | |
1342 | } | |
1343 | ||
d11de63f YY |
1344 | static void nvmet_fatal_error_handler(struct work_struct *work) |
1345 | { | |
1346 | struct nvmet_ctrl *ctrl = | |
1347 | container_of(work, struct nvmet_ctrl, fatal_err_work); | |
1348 | ||
1349 | pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid); | |
1350 | ctrl->ops->delete_ctrl(ctrl); | |
1351 | } | |
1352 | ||
a07b4970 CH |
1353 | u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, |
1354 | struct nvmet_req *req, u32 kato, struct nvmet_ctrl **ctrlp) | |
1355 | { | |
1356 | struct nvmet_subsys *subsys; | |
1357 | struct nvmet_ctrl *ctrl; | |
1358 | int ret; | |
1359 | u16 status; | |
1360 | ||
1361 | status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; | |
1362 | subsys = nvmet_find_get_subsys(req->port, subsysnqn); | |
1363 | if (!subsys) { | |
1364 | pr_warn("connect request for invalid subsystem %s!\n", | |
1365 | subsysnqn); | |
fc6c9730 | 1366 | req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn); |
a56f14c2 | 1367 | req->error_loc = offsetof(struct nvme_common_command, dptr); |
a07b4970 CH |
1368 | goto out; |
1369 | } | |
1370 | ||
a07b4970 | 1371 | down_read(&nvmet_config_sem); |
253928ee | 1372 | if (!nvmet_host_allowed(subsys, hostnqn)) { |
a07b4970 CH |
1373 | pr_info("connect by host %s for subsystem %s not allowed\n", |
1374 | hostnqn, subsysnqn); | |
fc6c9730 | 1375 | req->cqe->result.u32 = IPO_IATTR_CONNECT_DATA(hostnqn); |
a07b4970 | 1376 | up_read(&nvmet_config_sem); |
130c24b5 | 1377 | status = NVME_SC_CONNECT_INVALID_HOST | NVME_SC_DNR; |
a56f14c2 | 1378 | req->error_loc = offsetof(struct nvme_common_command, dptr); |
a07b4970 CH |
1379 | goto out_put_subsystem; |
1380 | } | |
1381 | up_read(&nvmet_config_sem); | |
1382 | ||
1383 | status = NVME_SC_INTERNAL; | |
1384 | ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); | |
1385 | if (!ctrl) | |
1386 | goto out_put_subsystem; | |
1387 | mutex_init(&ctrl->lock); | |
1388 | ||
4ee43280 | 1389 | ctrl->port = req->port; |
6d1555cc | 1390 | ctrl->ops = req->ops; |
4ee43280 | 1391 | |
34ad6151 AA |
1392 | #ifdef CONFIG_NVME_TARGET_PASSTHRU |
1393 | /* By default, set loop targets to clear IDS by default */ | |
1394 | if (ctrl->port->disc_addr.trtype == NVMF_TRTYPE_LOOP) | |
1395 | subsys->clear_ids = 1; | |
1396 | #endif | |
1397 | ||
a07b4970 CH |
1398 | INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work); |
1399 | INIT_LIST_HEAD(&ctrl->async_events); | |
c6925093 | 1400 | INIT_RADIX_TREE(&ctrl->p2p_ns_map, GFP_KERNEL); |
d11de63f | 1401 | INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler); |
f6e8bd59 | 1402 | INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer); |
a07b4970 CH |
1403 | |
1404 | memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE); | |
1405 | memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE); | |
1406 | ||
1407 | kref_init(&ctrl->ref); | |
1408 | ctrl->subsys = subsys; | |
77d651a6 | 1409 | nvmet_init_cap(ctrl); |
c86b8f7b | 1410 | WRITE_ONCE(ctrl->aen_enabled, NVMET_AEN_CFG_OPTIONAL); |
a07b4970 | 1411 | |
c16734ea CH |
1412 | ctrl->changed_ns_list = kmalloc_array(NVME_MAX_CHANGED_NAMESPACES, |
1413 | sizeof(__le32), GFP_KERNEL); | |
1414 | if (!ctrl->changed_ns_list) | |
1415 | goto out_free_ctrl; | |
1416 | ||
a07b4970 CH |
1417 | ctrl->sqs = kcalloc(subsys->max_qid + 1, |
1418 | sizeof(struct nvmet_sq *), | |
1419 | GFP_KERNEL); | |
1420 | if (!ctrl->sqs) | |
6d65aeab | 1421 | goto out_free_changed_ns_list; |
a07b4970 | 1422 | |
94a39d61 | 1423 | if (subsys->cntlid_min > subsys->cntlid_max) |
fec356a6 | 1424 | goto out_free_sqs; |
94a39d61 | 1425 | |
22027a98 | 1426 | ret = ida_alloc_range(&cntlid_ida, |
94a39d61 | 1427 | subsys->cntlid_min, subsys->cntlid_max, |
a07b4970 CH |
1428 | GFP_KERNEL); |
1429 | if (ret < 0) { | |
1430 | status = NVME_SC_CONNECT_CTRL_BUSY | NVME_SC_DNR; | |
1431 | goto out_free_sqs; | |
1432 | } | |
1433 | ctrl->cntlid = ret; | |
1434 | ||
f9362ac1 JS |
1435 | /* |
1436 | * Discovery controllers may use some arbitrary high value | |
1437 | * in order to cleanup stale discovery sessions | |
1438 | */ | |
a294711e | 1439 | if (nvmet_is_disc_subsys(ctrl->subsys) && !kato) |
f9362ac1 JS |
1440 | kato = NVMET_DISC_KATO_MS; |
1441 | ||
1442 | /* keep-alive timeout in seconds */ | |
1443 | ctrl->kato = DIV_ROUND_UP(kato, 1000); | |
1444 | ||
e4a97625 CK |
1445 | ctrl->err_counter = 0; |
1446 | spin_lock_init(&ctrl->error_lock); | |
1447 | ||
a07b4970 CH |
1448 | nvmet_start_keep_alive_timer(ctrl); |
1449 | ||
1450 | mutex_lock(&subsys->lock); | |
1451 | list_add_tail(&ctrl->subsys_entry, &subsys->ctrls); | |
c6925093 | 1452 | nvmet_setup_p2p_ns_map(ctrl, req); |
a07b4970 CH |
1453 | mutex_unlock(&subsys->lock); |
1454 | ||
1455 | *ctrlp = ctrl; | |
1456 | return 0; | |
1457 | ||
1458 | out_free_sqs: | |
1459 | kfree(ctrl->sqs); | |
c16734ea CH |
1460 | out_free_changed_ns_list: |
1461 | kfree(ctrl->changed_ns_list); | |
a07b4970 CH |
1462 | out_free_ctrl: |
1463 | kfree(ctrl); | |
1464 | out_put_subsystem: | |
1465 | nvmet_subsys_put(subsys); | |
1466 | out: | |
1467 | return status; | |
1468 | } | |
1469 | ||
1470 | static void nvmet_ctrl_free(struct kref *ref) | |
1471 | { | |
1472 | struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref); | |
1473 | struct nvmet_subsys *subsys = ctrl->subsys; | |
1474 | ||
a07b4970 | 1475 | mutex_lock(&subsys->lock); |
c6925093 | 1476 | nvmet_release_p2p_ns_map(ctrl); |
a07b4970 CH |
1477 | list_del(&ctrl->subsys_entry); |
1478 | mutex_unlock(&subsys->lock); | |
1479 | ||
6b1943af IR |
1480 | nvmet_stop_keep_alive_timer(ctrl); |
1481 | ||
06406d81 SG |
1482 | flush_work(&ctrl->async_event_work); |
1483 | cancel_work_sync(&ctrl->fatal_err_work); | |
1484 | ||
db1312dd HR |
1485 | nvmet_destroy_auth(ctrl); |
1486 | ||
22027a98 | 1487 | ida_free(&cntlid_ida, ctrl->cntlid); |
a07b4970 | 1488 | |
64f5e9cd | 1489 | nvmet_async_events_free(ctrl); |
a07b4970 | 1490 | kfree(ctrl->sqs); |
c16734ea | 1491 | kfree(ctrl->changed_ns_list); |
a07b4970 | 1492 | kfree(ctrl); |
6b1943af IR |
1493 | |
1494 | nvmet_subsys_put(subsys); | |
a07b4970 CH |
1495 | } |
1496 | ||
1497 | void nvmet_ctrl_put(struct nvmet_ctrl *ctrl) | |
1498 | { | |
1499 | kref_put(&ctrl->ref, nvmet_ctrl_free); | |
1500 | } | |
1501 | ||
a07b4970 CH |
1502 | void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl) |
1503 | { | |
8242ddac SG |
1504 | mutex_lock(&ctrl->lock); |
1505 | if (!(ctrl->csts & NVME_CSTS_CFS)) { | |
1506 | ctrl->csts |= NVME_CSTS_CFS; | |
8832cf92 | 1507 | queue_work(nvmet_wq, &ctrl->fatal_err_work); |
8242ddac SG |
1508 | } |
1509 | mutex_unlock(&ctrl->lock); | |
a07b4970 CH |
1510 | } |
1511 | EXPORT_SYMBOL_GPL(nvmet_ctrl_fatal_error); | |
1512 | ||
1513 | static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, | |
1514 | const char *subsysnqn) | |
1515 | { | |
1516 | struct nvmet_subsys_link *p; | |
1517 | ||
1518 | if (!port) | |
1519 | return NULL; | |
1520 | ||
0c48645a | 1521 | if (!strcmp(NVME_DISC_SUBSYS_NAME, subsysnqn)) { |
a07b4970 CH |
1522 | if (!kref_get_unless_zero(&nvmet_disc_subsys->ref)) |
1523 | return NULL; | |
1524 | return nvmet_disc_subsys; | |
1525 | } | |
1526 | ||
1527 | down_read(&nvmet_config_sem); | |
1528 | list_for_each_entry(p, &port->subsystems, entry) { | |
1529 | if (!strncmp(p->subsys->subsysnqn, subsysnqn, | |
1530 | NVMF_NQN_SIZE)) { | |
1531 | if (!kref_get_unless_zero(&p->subsys->ref)) | |
1532 | break; | |
1533 | up_read(&nvmet_config_sem); | |
1534 | return p->subsys; | |
1535 | } | |
1536 | } | |
1537 | up_read(&nvmet_config_sem); | |
1538 | return NULL; | |
1539 | } | |
1540 | ||
1541 | struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn, | |
1542 | enum nvme_subsys_type type) | |
1543 | { | |
1544 | struct nvmet_subsys *subsys; | |
e13b0615 | 1545 | char serial[NVMET_SN_MAX_SIZE / 2]; |
0d148efd | 1546 | int ret; |
a07b4970 CH |
1547 | |
1548 | subsys = kzalloc(sizeof(*subsys), GFP_KERNEL); | |
1549 | if (!subsys) | |
6b7e631b | 1550 | return ERR_PTR(-ENOMEM); |
a07b4970 | 1551 | |
ba76af67 | 1552 | subsys->ver = NVMET_DEFAULT_VS; |
2e7f5d2a | 1553 | /* generate a random serial number as our controllers are ephemeral: */ |
e13b0615 NG |
1554 | get_random_bytes(&serial, sizeof(serial)); |
1555 | bin2hex(subsys->serial, &serial, sizeof(serial)); | |
a07b4970 | 1556 | |
0d148efd NG |
1557 | subsys->model_number = kstrdup(NVMET_DEFAULT_CTRL_MODEL, GFP_KERNEL); |
1558 | if (!subsys->model_number) { | |
1559 | ret = -ENOMEM; | |
1560 | goto free_subsys; | |
1561 | } | |
a07b4970 CH |
1562 | |
1563 | switch (type) { | |
1564 | case NVME_NQN_NVME: | |
1565 | subsys->max_qid = NVMET_NR_QUEUES; | |
1566 | break; | |
1567 | case NVME_NQN_DISC: | |
2953b30b | 1568 | case NVME_NQN_CURR: |
a07b4970 CH |
1569 | subsys->max_qid = 0; |
1570 | break; | |
1571 | default: | |
1572 | pr_err("%s: Unknown Subsystem type - %d\n", __func__, type); | |
0d148efd NG |
1573 | ret = -EINVAL; |
1574 | goto free_mn; | |
a07b4970 CH |
1575 | } |
1576 | subsys->type = type; | |
1577 | subsys->subsysnqn = kstrndup(subsysnqn, NVMF_NQN_SIZE, | |
1578 | GFP_KERNEL); | |
69555af2 | 1579 | if (!subsys->subsysnqn) { |
0d148efd NG |
1580 | ret = -ENOMEM; |
1581 | goto free_mn; | |
a07b4970 | 1582 | } |
94a39d61 CK |
1583 | subsys->cntlid_min = NVME_CNTLID_MIN; |
1584 | subsys->cntlid_max = NVME_CNTLID_MAX; | |
a07b4970 CH |
1585 | kref_init(&subsys->ref); |
1586 | ||
1587 | mutex_init(&subsys->lock); | |
7774e77e | 1588 | xa_init(&subsys->namespaces); |
a07b4970 | 1589 | INIT_LIST_HEAD(&subsys->ctrls); |
a07b4970 CH |
1590 | INIT_LIST_HEAD(&subsys->hosts); |
1591 | ||
1592 | return subsys; | |
0d148efd NG |
1593 | |
1594 | free_mn: | |
1595 | kfree(subsys->model_number); | |
1596 | free_subsys: | |
1597 | kfree(subsys); | |
1598 | return ERR_PTR(ret); | |
a07b4970 CH |
1599 | } |
1600 | ||
1601 | static void nvmet_subsys_free(struct kref *ref) | |
1602 | { | |
1603 | struct nvmet_subsys *subsys = | |
1604 | container_of(ref, struct nvmet_subsys, ref); | |
1605 | ||
7774e77e | 1606 | WARN_ON_ONCE(!xa_empty(&subsys->namespaces)); |
a07b4970 | 1607 | |
7774e77e | 1608 | xa_destroy(&subsys->namespaces); |
ba76af67 LG |
1609 | nvmet_passthru_subsys_free(subsys); |
1610 | ||
a07b4970 | 1611 | kfree(subsys->subsysnqn); |
d9f273b7 | 1612 | kfree(subsys->model_number); |
a07b4970 CH |
1613 | kfree(subsys); |
1614 | } | |
1615 | ||
344770b0 SG |
1616 | void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys) |
1617 | { | |
1618 | struct nvmet_ctrl *ctrl; | |
1619 | ||
1620 | mutex_lock(&subsys->lock); | |
1621 | list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) | |
1622 | ctrl->ops->delete_ctrl(ctrl); | |
1623 | mutex_unlock(&subsys->lock); | |
1624 | } | |
1625 | ||
a07b4970 CH |
1626 | void nvmet_subsys_put(struct nvmet_subsys *subsys) |
1627 | { | |
1628 | kref_put(&subsys->ref, nvmet_subsys_free); | |
1629 | } | |
1630 | ||
1631 | static int __init nvmet_init(void) | |
1632 | { | |
1633 | int error; | |
1634 | ||
72efd25d CH |
1635 | nvmet_ana_group_enabled[NVMET_DEFAULT_ANA_GRPID] = 1; |
1636 | ||
aaf2e048 CK |
1637 | zbd_wq = alloc_workqueue("nvmet-zbd-wq", WQ_MEM_RECLAIM, 0); |
1638 | if (!zbd_wq) | |
1639 | return -ENOMEM; | |
1640 | ||
55eb942e CK |
1641 | buffered_io_wq = alloc_workqueue("nvmet-buffered-io-wq", |
1642 | WQ_MEM_RECLAIM, 0); | |
1643 | if (!buffered_io_wq) { | |
1644 | error = -ENOMEM; | |
aaf2e048 | 1645 | goto out_free_zbd_work_queue; |
55eb942e | 1646 | } |
72efd25d | 1647 | |
8832cf92 SG |
1648 | nvmet_wq = alloc_workqueue("nvmet-wq", WQ_MEM_RECLAIM, 0); |
1649 | if (!nvmet_wq) { | |
1650 | error = -ENOMEM; | |
1651 | goto out_free_buffered_work_queue; | |
1652 | } | |
1653 | ||
a07b4970 CH |
1654 | error = nvmet_init_discovery(); |
1655 | if (error) | |
8832cf92 | 1656 | goto out_free_nvmet_work_queue; |
a07b4970 CH |
1657 | |
1658 | error = nvmet_init_configfs(); | |
1659 | if (error) | |
1660 | goto out_exit_discovery; | |
1661 | return 0; | |
1662 | ||
1663 | out_exit_discovery: | |
1664 | nvmet_exit_discovery(); | |
8832cf92 SG |
1665 | out_free_nvmet_work_queue: |
1666 | destroy_workqueue(nvmet_wq); | |
1667 | out_free_buffered_work_queue: | |
04db0e5e | 1668 | destroy_workqueue(buffered_io_wq); |
aaf2e048 CK |
1669 | out_free_zbd_work_queue: |
1670 | destroy_workqueue(zbd_wq); | |
a07b4970 CH |
1671 | return error; |
1672 | } | |
1673 | ||
1674 | static void __exit nvmet_exit(void) | |
1675 | { | |
1676 | nvmet_exit_configfs(); | |
1677 | nvmet_exit_discovery(); | |
15fbad96 | 1678 | ida_destroy(&cntlid_ida); |
8832cf92 | 1679 | destroy_workqueue(nvmet_wq); |
55eb942e | 1680 | destroy_workqueue(buffered_io_wq); |
aaf2e048 | 1681 | destroy_workqueue(zbd_wq); |
a07b4970 CH |
1682 | |
1683 | BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_entry) != 1024); | |
1684 | BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_hdr) != 1024); | |
1685 | } | |
1686 | ||
1687 | module_init(nvmet_init); | |
1688 | module_exit(nvmet_exit); | |
1689 | ||
1690 | MODULE_LICENSE("GPL v2"); |