projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
nvme: implement In-Band authentication
[linux-2.6-block.git] / drivers / nvme / host / rdma.c
CommitLineData
5d8762d5 1// SPDX-License-Identifier: GPL-2.0
71102307
CH		 2/*
3 * NVMe over Fabrics RDMA host code.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
71102307
CH		 5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
71102307
CH		 7#include <linux/module.h>
8#include <linux/init.h>
9#include <linux/slab.h>
f41725bb 10#include <rdma/mr_pool.h>
71102307
CH		 11#include <linux/err.h>
12#include <linux/string.h>
71102307
CH		 13#include <linux/atomic.h>
14#include <linux/blk-mq.h>
0b36658c 15#include <linux/blk-mq-rdma.h>
fe45e630 16#include <linux/blk-integrity.h>
71102307
CH		 17#include <linux/types.h>
18#include <linux/list.h>
19#include <linux/mutex.h>
20#include <linux/scatterlist.h>
21#include <linux/nvme.h>
71102307
CH		 22#include <asm/unaligned.h>
23
24#include <rdma/ib_verbs.h>
25#include <rdma/rdma_cm.h>
71102307
CH		 26#include <linux/nvme-rdma.h>
27
28#include "nvme.h"
29#include "fabrics.h"
30
31
782d820c 32#define NVME_RDMA_CONNECT_TIMEOUT_MS 3000 /* 3 second */
71102307 33
71102307
CH		 34#define NVME_RDMA_MAX_SEGMENTS 256
35
64a741c1 36#define NVME_RDMA_MAX_INLINE_SEGMENTS 4
71102307 37
5ec5d3bd
MG		 38#define NVME_RDMA_DATA_SGL_SIZE \
39 (sizeof(struct scatterlist) * NVME_INLINE_SG_CNT)
40#define NVME_RDMA_METADATA_SGL_SIZE \
41 (sizeof(struct scatterlist) * NVME_INLINE_METADATA_SG_CNT)
42
71102307 43struct nvme_rdma_device {
f87c89ad
MG		 44 struct ib_device *dev;
45 struct ib_pd *pd;
71102307
CH		 46 struct kref ref;
47 struct list_head entry;
64a741c1 48 unsigned int num_inline_segments;
71102307
CH		 49};
50
51struct nvme_rdma_qe {
52 struct ib_cqe cqe;
53 void *data;
54 u64 dma;
55};
56
324d9e78
IR		 57struct nvme_rdma_sgl {
58 int nents;
59 struct sg_table sg_table;
60};
61
71102307
CH		 62struct nvme_rdma_queue;
63struct nvme_rdma_request {
d49187e9 64 struct nvme_request req;
71102307
CH		 65 struct ib_mr *mr;
66 struct nvme_rdma_qe sqe;
4af7f7ff
SG		 67 union nvme_result result;
68 __le16 status;
69 refcount_t ref;
71102307
CH		 70 struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS];
71 u32 num_sge;
71102307
CH		 72 struct ib_reg_wr reg_wr;
73 struct ib_cqe reg_cqe;
74 struct nvme_rdma_queue *queue;
324d9e78 75 struct nvme_rdma_sgl data_sgl;
5ec5d3bd
MG		 76 struct nvme_rdma_sgl *metadata_sgl;
77 bool use_sig_mr;
71102307
CH		 78};
79
80enum nvme_rdma_queue_flags {
5013e98b
SG		 81 NVME_RDMA_Q_ALLOCATED = 0,
82 NVME_RDMA_Q_LIVE = 1,
eb1bd249 83 NVME_RDMA_Q_TR_READY = 2,
71102307
CH		 84};
85
86struct nvme_rdma_queue {
87 struct nvme_rdma_qe *rsp_ring;
71102307
CH		 88 int queue_size;
89 size_t cmnd_capsule_len;
90 struct nvme_rdma_ctrl *ctrl;
91 struct nvme_rdma_device *device;
92 struct ib_cq *ib_cq;
93 struct ib_qp *qp;
94
95 unsigned long flags;
96 struct rdma_cm_id *cm_id;
97 int cm_error;
98 struct completion cm_done;
5ec5d3bd 99 bool pi_support;
287f329e 100 int cq_size;
7674073b 101 struct mutex queue_lock;
71102307
CH		 102};
103
104struct nvme_rdma_ctrl {
71102307
CH		 105 /* read only in the hot path */
106 struct nvme_rdma_queue *queues;
71102307
CH		 107
108 /* other member variables */
71102307 109 struct blk_mq_tag_set tag_set;
71102307
CH		 110 struct work_struct err_work;
111
112 struct nvme_rdma_qe async_event_sqe;
113
71102307
CH		 114 struct delayed_work reconnect_work;
115
116 struct list_head list;
117
118 struct blk_mq_tag_set admin_tag_set;
119 struct nvme_rdma_device *device;
120
71102307
CH		 121 u32 max_fr_pages;
122
0928f9b4
SG		 123 struct sockaddr_storage addr;
124 struct sockaddr_storage src_addr;
71102307
CH		 125
126 struct nvme_ctrl ctrl;
64a741c1 127 bool use_inline_data;
b1064d3e 128 u32 io_queues[HCTX_MAX_TYPES];
71102307
CH		 129};
130
131static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
132{
133 return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
134}
135
136static LIST_HEAD(device_list);
137static DEFINE_MUTEX(device_list_mutex);
138
139static LIST_HEAD(nvme_rdma_ctrl_list);
140static DEFINE_MUTEX(nvme_rdma_ctrl_mutex);
141
71102307
CH		 142/*
143 * Disabling this option makes small I/O goes faster, but is fundamentally
144 * unsafe. With it turned off we will have to register a global rkey that
145 * allows read and write access to all physical memory.
146 */
147static bool register_always = true;
148module_param(register_always, bool, 0444);
149MODULE_PARM_DESC(register_always,
150 "Use memory registration even for contiguous memory regions");
151
152static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
153 struct rdma_cm_event *event);
154static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc);
ff029451 155static void nvme_rdma_complete_rq(struct request *rq);
71102307 156
90af3512
SG		 157static const struct blk_mq_ops nvme_rdma_mq_ops;
158static const struct blk_mq_ops nvme_rdma_admin_mq_ops;
159
71102307
CH		 160static inline int nvme_rdma_queue_idx(struct nvme_rdma_queue *queue)
161{
162 return queue - queue->ctrl->queues;
163}
164
ff8519f9
SG		 165static bool nvme_rdma_poll_queue(struct nvme_rdma_queue *queue)
166{
167 return nvme_rdma_queue_idx(queue) >
b1064d3e
SG		 168 queue->ctrl->io_queues[HCTX_TYPE_DEFAULT] +
169 queue->ctrl->io_queues[HCTX_TYPE_READ];
ff8519f9
SG		 170}
171
71102307
CH		 172static inline size_t nvme_rdma_inline_data_size(struct nvme_rdma_queue *queue)
173{
174 return queue->cmnd_capsule_len - sizeof(struct nvme_command);
175}
176
177static void nvme_rdma_free_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
178 size_t capsule_size, enum dma_data_direction dir)
179{
180 ib_dma_unmap_single(ibdev, qe->dma, capsule_size, dir);
181 kfree(qe->data);
182}
183
184static int nvme_rdma_alloc_qe(struct ib_device *ibdev, struct nvme_rdma_qe *qe,
185 size_t capsule_size, enum dma_data_direction dir)
186{
187 qe->data = kzalloc(capsule_size, GFP_KERNEL);
188 if (!qe->data)
189 return -ENOMEM;
190
191 qe->dma = ib_dma_map_single(ibdev, qe->data, capsule_size, dir);
192 if (ib_dma_mapping_error(ibdev, qe->dma)) {
193 kfree(qe->data);
6344d02d 194 qe->data = NULL;
71102307
CH		 195 return -ENOMEM;
196 }
197
198 return 0;
199}
200
201static void nvme_rdma_free_ring(struct ib_device *ibdev,
202 struct nvme_rdma_qe *ring, size_t ib_queue_size,
203 size_t capsule_size, enum dma_data_direction dir)
204{
205 int i;
206
207 for (i = 0; i < ib_queue_size; i++)
208 nvme_rdma_free_qe(ibdev, &ring[i], capsule_size, dir);
209 kfree(ring);
210}
211
212static struct nvme_rdma_qe *nvme_rdma_alloc_ring(struct ib_device *ibdev,
213 size_t ib_queue_size, size_t capsule_size,
214 enum dma_data_direction dir)
215{
216 struct nvme_rdma_qe *ring;
217 int i;
218
219 ring = kcalloc(ib_queue_size, sizeof(struct nvme_rdma_qe), GFP_KERNEL);
220 if (!ring)
221 return NULL;
222
62f99b62
MG		 223 /*
224 * Bind the CQEs (post recv buffers) DMA mapping to the RDMA queue
225 * lifetime. It's safe, since any chage in the underlying RDMA device
226 * will issue error recovery and queue re-creation.
227 */
71102307
CH		 228 for (i = 0; i < ib_queue_size; i++) {
229 if (nvme_rdma_alloc_qe(ibdev, &ring[i], capsule_size, dir))
230 goto out_free_ring;
231 }
232
233 return ring;
234
235out_free_ring:
236 nvme_rdma_free_ring(ibdev, ring, i, capsule_size, dir);
237 return NULL;
238}
239
240static void nvme_rdma_qp_event(struct ib_event *event, void *context)
241{
27a4beef
MG		 242 pr_debug("QP event %s (%d)\n",
243 ib_event_msg(event->event), event->event);
244
71102307
CH		 245}
246
247static int nvme_rdma_wait_for_cm(struct nvme_rdma_queue *queue)
248{
35da77d5
BVA		 249 int ret;
250
251 ret = wait_for_completion_interruptible_timeout(&queue->cm_done,
71102307 252 msecs_to_jiffies(NVME_RDMA_CONNECT_TIMEOUT_MS) + 1);
35da77d5
BVA		 253 if (ret < 0)
254 return ret;
255 if (ret == 0)
256 return -ETIMEDOUT;
257 WARN_ON_ONCE(queue->cm_error > 0);
71102307
CH		 258 return queue->cm_error;
259}
260
261static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor)
262{
263 struct nvme_rdma_device *dev = queue->device;
264 struct ib_qp_init_attr init_attr;
265 int ret;
266
267 memset(&init_attr, 0, sizeof(init_attr));
268 init_attr.event_handler = nvme_rdma_qp_event;
269 /* +1 for drain */
270 init_attr.cap.max_send_wr = factor * queue->queue_size + 1;
271 /* +1 for drain */
272 init_attr.cap.max_recv_wr = queue->queue_size + 1;
273 init_attr.cap.max_recv_sge = 1;
64a741c1 274 init_attr.cap.max_send_sge = 1 + dev->num_inline_segments;
71102307
CH		 275 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
276 init_attr.qp_type = IB_QPT_RC;
277 init_attr.send_cq = queue->ib_cq;
278 init_attr.recv_cq = queue->ib_cq;
5ec5d3bd
MG		 279 if (queue->pi_support)
280 init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN;
287f329e 281 init_attr.qp_context = queue;
71102307
CH		 282
283 ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr);
284
285 queue->qp = queue->cm_id->qp;
286 return ret;
287}
288
385475ee
CH		 289static void nvme_rdma_exit_request(struct blk_mq_tag_set *set,
290 struct request *rq, unsigned int hctx_idx)
71102307
CH		 291{
292 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
71102307 293
62f99b62 294 kfree(req->sqe.data);
71102307
CH		 295}
296
385475ee
CH		 297static int nvme_rdma_init_request(struct blk_mq_tag_set *set,
298 struct request *rq, unsigned int hctx_idx,
299 unsigned int numa_node)
71102307 300{
385475ee 301 struct nvme_rdma_ctrl *ctrl = set->driver_data;
71102307 302 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
385475ee 303 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
71102307 304 struct nvme_rdma_queue *queue = &ctrl->queues[queue_idx];
71102307 305
59e29ce6 306 nvme_req(rq)->ctrl = &ctrl->ctrl;
62f99b62
MG		 307 req->sqe.data = kzalloc(sizeof(struct nvme_command), GFP_KERNEL);
308 if (!req->sqe.data)
309 return -ENOMEM;
71102307 310
5ec5d3bd
MG		 311 /* metadata nvme_rdma_sgl struct is located after command's data SGL */
312 if (queue->pi_support)
313 req->metadata_sgl = (void *)nvme_req(rq) +
314 sizeof(struct nvme_rdma_request) +
315 NVME_RDMA_DATA_SGL_SIZE;
316
71102307 317 req->queue = queue;
f4b9e6c9 318 nvme_req(rq)->cmd = req->sqe.data;
71102307
CH		 319
320 return 0;
71102307
CH		 321}
322
71102307
CH		 323static int nvme_rdma_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
324 unsigned int hctx_idx)
325{
326 struct nvme_rdma_ctrl *ctrl = data;
327 struct nvme_rdma_queue *queue = &ctrl->queues[hctx_idx + 1];
328
d858e5f0 329 BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
71102307
CH		 330
331 hctx->driver_data = queue;
332 return 0;
333}
334
335static int nvme_rdma_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
336 unsigned int hctx_idx)
337{
338 struct nvme_rdma_ctrl *ctrl = data;
339 struct nvme_rdma_queue *queue = &ctrl->queues[0];
340
341 BUG_ON(hctx_idx != 0);
342
343 hctx->driver_data = queue;
344 return 0;
345}
346
347static void nvme_rdma_free_dev(struct kref *ref)
348{
349 struct nvme_rdma_device *ndev =
350 container_of(ref, struct nvme_rdma_device, ref);
351
352 mutex_lock(&device_list_mutex);
353 list_del(&ndev->entry);
354 mutex_unlock(&device_list_mutex);
355
71102307 356 ib_dealloc_pd(ndev->pd);
71102307
CH		 357 kfree(ndev);
358}
359
360static void nvme_rdma_dev_put(struct nvme_rdma_device *dev)
361{
362 kref_put(&dev->ref, nvme_rdma_free_dev);
363}
364
365static int nvme_rdma_dev_get(struct nvme_rdma_device *dev)
366{
367 return kref_get_unless_zero(&dev->ref);
368}
369
370static struct nvme_rdma_device *
371nvme_rdma_find_get_device(struct rdma_cm_id *cm_id)
372{
373 struct nvme_rdma_device *ndev;
374
375 mutex_lock(&device_list_mutex);
376 list_for_each_entry(ndev, &device_list, entry) {
377 if (ndev->dev->node_guid == cm_id->device->node_guid &&
378 nvme_rdma_dev_get(ndev))
379 goto out_unlock;
380 }
381
382 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
383 if (!ndev)
384 goto out_err;
385
386 ndev->dev = cm_id->device;
387 kref_init(&ndev->ref);
388
11975e01
CH		 389 ndev->pd = ib_alloc_pd(ndev->dev,
390 register_always ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
71102307
CH		 391 if (IS_ERR(ndev->pd))
392 goto out_free_dev;
393
71102307
CH		 394 if (!(ndev->dev->attrs.device_cap_flags &
395 IB_DEVICE_MEM_MGT_EXTENSIONS)) {
396 dev_err(&ndev->dev->dev,
397 "Memory registrations not supported.\n");
11975e01 398 goto out_free_pd;
71102307
CH		 399 }
400
64a741c1 401 ndev->num_inline_segments = min(NVME_RDMA_MAX_INLINE_SEGMENTS,
0a3173a5 402 ndev->dev->attrs.max_send_sge - 1);
71102307
CH		 403 list_add(&ndev->entry, &device_list);
404out_unlock:
405 mutex_unlock(&device_list_mutex);
406 return ndev;
407
71102307
CH		 408out_free_pd:
409 ib_dealloc_pd(ndev->pd);
410out_free_dev:
411 kfree(ndev);
412out_err:
413 mutex_unlock(&device_list_mutex);
414 return NULL;
415}
416
287f329e
YF		 417static void nvme_rdma_free_cq(struct nvme_rdma_queue *queue)
418{
419 if (nvme_rdma_poll_queue(queue))
420 ib_free_cq(queue->ib_cq);
421 else
422 ib_cq_pool_put(queue->ib_cq, queue->cq_size);
423}
424
71102307
CH		 425static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue)
426{
eb1bd249
MG		 427 struct nvme_rdma_device *dev;
428 struct ib_device *ibdev;
429
430 if (!test_and_clear_bit(NVME_RDMA_Q_TR_READY, &queue->flags))
431 return;
432
433 dev = queue->device;
434 ibdev = dev->dev;
71102307 435
5ec5d3bd
MG		 436 if (queue->pi_support)
437 ib_mr_pool_destroy(queue->qp, &queue->qp->sig_mrs);
f41725bb
IR		 438 ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs);
439
eb1bd249
MG		 440 /*
441 * The cm_id object might have been destroyed during RDMA connection
442 * establishment error flow to avoid getting other cma events, thus
443 * the destruction of the QP shouldn't use rdma_cm API.
444 */
445 ib_destroy_qp(queue->qp);
287f329e 446 nvme_rdma_free_cq(queue);
71102307
CH		 447
448 nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
449 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
450
451 nvme_rdma_dev_put(dev);
452}
453
5ec5d3bd 454static int nvme_rdma_get_max_fr_pages(struct ib_device *ibdev, bool pi_support)
f41725bb 455{
5ec5d3bd
MG		 456 u32 max_page_list_len;
457
458 if (pi_support)
459 max_page_list_len = ibdev->attrs.max_pi_fast_reg_page_list_len;
460 else
461 max_page_list_len = ibdev->attrs.max_fast_reg_page_list_len;
462
463 return min_t(u32, NVME_RDMA_MAX_SEGMENTS, max_page_list_len - 1);
f41725bb
IR		 464}
465
287f329e
YF		 466static int nvme_rdma_create_cq(struct ib_device *ibdev,
467 struct nvme_rdma_queue *queue)
468{
469 int ret, comp_vector, idx = nvme_rdma_queue_idx(queue);
470 enum ib_poll_context poll_ctx;
471
472 /*
473 * Spread I/O queues completion vectors according their queue index.
474 * Admin queues can always go on completion vector 0.
475 */
476 comp_vector = (idx == 0 ? idx : idx - 1) % ibdev->num_comp_vectors;
477
478 /* Polling queues need direct cq polling context */
479 if (nvme_rdma_poll_queue(queue)) {
480 poll_ctx = IB_POLL_DIRECT;
481 queue->ib_cq = ib_alloc_cq(ibdev, queue, queue->cq_size,
482 comp_vector, poll_ctx);
483 } else {
484 poll_ctx = IB_POLL_SOFTIRQ;
485 queue->ib_cq = ib_cq_pool_get(ibdev, queue->cq_size,
486 comp_vector, poll_ctx);
487 }
488
489 if (IS_ERR(queue->ib_cq)) {
490 ret = PTR_ERR(queue->ib_cq);
491 return ret;
492 }
493
494 return 0;
495}
496
ca6e95bb 497static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
71102307 498{
ca6e95bb 499 struct ib_device *ibdev;
71102307
CH		 500 const int send_wr_factor = 3; /* MR, SEND, INV */
501 const int cq_factor = send_wr_factor + 1; /* + RECV */
ff13c1b8 502 int ret, pages_per_mr;
71102307 503
ca6e95bb
SG		 504 queue->device = nvme_rdma_find_get_device(queue->cm_id);
505 if (!queue->device) {
506 dev_err(queue->cm_id->device->dev.parent,
507 "no client data found!\n");
508 return -ECONNREFUSED;
509 }
510 ibdev = queue->device->dev;
71102307 511
71102307 512 /* +1 for ib_stop_cq */
287f329e
YF		 513 queue->cq_size = cq_factor * queue->queue_size + 1;
514
515 ret = nvme_rdma_create_cq(ibdev, queue);
516 if (ret)
ca6e95bb 517 goto out_put_dev;
71102307
CH		 518
519 ret = nvme_rdma_create_qp(queue, send_wr_factor);
520 if (ret)
521 goto out_destroy_ib_cq;
522
523 queue->rsp_ring = nvme_rdma_alloc_ring(ibdev, queue->queue_size,
524 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
525 if (!queue->rsp_ring) {
526 ret = -ENOMEM;
527 goto out_destroy_qp;
528 }
529
ff13c1b8
MG		 530 /*
531 * Currently we don't use SG_GAPS MR's so if the first entry is
532 * misaligned we'll end up using two entries for a single data page,
533 * so one additional entry is required.
534 */
5ec5d3bd 535 pages_per_mr = nvme_rdma_get_max_fr_pages(ibdev, queue->pi_support) + 1;
f41725bb
IR		 536 ret = ib_mr_pool_init(queue->qp, &queue->qp->rdma_mrs,
537 queue->queue_size,
538 IB_MR_TYPE_MEM_REG,
ff13c1b8 539 pages_per_mr, 0);
f41725bb
IR		 540 if (ret) {
541 dev_err(queue->ctrl->ctrl.device,
542 "failed to initialize MR pool sized %d for QID %d\n",
287f329e 543 queue->queue_size, nvme_rdma_queue_idx(queue));
f41725bb
IR		 544 goto out_destroy_ring;
545 }
546
5ec5d3bd
MG		 547 if (queue->pi_support) {
548 ret = ib_mr_pool_init(queue->qp, &queue->qp->sig_mrs,
549 queue->queue_size, IB_MR_TYPE_INTEGRITY,
550 pages_per_mr, pages_per_mr);
551 if (ret) {
552 dev_err(queue->ctrl->ctrl.device,
553 "failed to initialize PI MR pool sized %d for QID %d\n",
287f329e 554 queue->queue_size, nvme_rdma_queue_idx(queue));
5ec5d3bd
MG		 555 goto out_destroy_mr_pool;
556 }
557 }
558
eb1bd249
MG		 559 set_bit(NVME_RDMA_Q_TR_READY, &queue->flags);
560
71102307
CH		 561 return 0;
562
5ec5d3bd
MG		 563out_destroy_mr_pool:
564 ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs);
f41725bb
IR		 565out_destroy_ring:
566 nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size,
567 sizeof(struct nvme_completion), DMA_FROM_DEVICE);
71102307 568out_destroy_qp:
1f61def9 569 rdma_destroy_qp(queue->cm_id);
71102307 570out_destroy_ib_cq:
287f329e 571 nvme_rdma_free_cq(queue);
ca6e95bb
SG		 572out_put_dev:
573 nvme_rdma_dev_put(queue->device);
71102307
CH		 574 return ret;
575}
576
41e8cfa1 577static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
71102307
CH		 578 int idx, size_t queue_size)
579{
580 struct nvme_rdma_queue *queue;
8f4e8dac 581 struct sockaddr *src_addr = NULL;
71102307
CH		 582 int ret;
583
584 queue = &ctrl->queues[idx];
7674073b 585 mutex_init(&queue->queue_lock);
71102307 586 queue->ctrl = ctrl;
5ec5d3bd
MG		 587 if (idx && ctrl->ctrl.max_integrity_segments)
588 queue->pi_support = true;
589 else
590 queue->pi_support = false;
71102307
CH		 591 init_completion(&queue->cm_done);
592
593 if (idx > 0)
594 queue->cmnd_capsule_len = ctrl->ctrl.ioccsz * 16;
595 else
596 queue->cmnd_capsule_len = sizeof(struct nvme_command);
597
598 queue->queue_size = queue_size;
599
600 queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue,
601 RDMA_PS_TCP, IB_QPT_RC);
602 if (IS_ERR(queue->cm_id)) {
603 dev_info(ctrl->ctrl.device,
604 "failed to create CM ID: %ld\n", PTR_ERR(queue->cm_id));
7674073b
CL		 605 ret = PTR_ERR(queue->cm_id);
606 goto out_destroy_mutex;
71102307
CH		 607 }
608
8f4e8dac 609 if (ctrl->ctrl.opts->mask & NVMF_OPT_HOST_TRADDR)
0928f9b4 610 src_addr = (struct sockaddr *)&ctrl->src_addr;
8f4e8dac 611
0928f9b4
SG		 612 queue->cm_error = -ETIMEDOUT;
613 ret = rdma_resolve_addr(queue->cm_id, src_addr,
614 (struct sockaddr *)&ctrl->addr,
71102307
CH		 615 NVME_RDMA_CONNECT_TIMEOUT_MS);
616 if (ret) {
617 dev_info(ctrl->ctrl.device,
618 "rdma_resolve_addr failed (%d).\n", ret);
619 goto out_destroy_cm_id;
620 }
621
622 ret = nvme_rdma_wait_for_cm(queue);
623 if (ret) {
624 dev_info(ctrl->ctrl.device,
d8bfceeb 625 "rdma connection establishment failed (%d)\n", ret);
71102307
CH		 626 goto out_destroy_cm_id;
627 }
628
5013e98b 629 set_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags);
71102307
CH		 630
631 return 0;
632
633out_destroy_cm_id:
634 rdma_destroy_id(queue->cm_id);
eb1bd249 635 nvme_rdma_destroy_queue_ib(queue);
7674073b
CL		 636out_destroy_mutex:
637 mutex_destroy(&queue->queue_lock);
71102307
CH		 638 return ret;
639}
640
d94211b8
SG		 641static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
642{
643 rdma_disconnect(queue->cm_id);
644 ib_drain_qp(queue->qp);
645}
646
71102307
CH		 647static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
648{
7674073b
CL		 649 mutex_lock(&queue->queue_lock);
650 if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
651 __nvme_rdma_stop_queue(queue);
652 mutex_unlock(&queue->queue_lock);
71102307
CH		 653}
654
655static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
656{
5013e98b 657 if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
a57bd541
SG		 658 return;
659
71102307 660 rdma_destroy_id(queue->cm_id);
9817d763 661 nvme_rdma_destroy_queue_ib(queue);
7674073b 662 mutex_destroy(&queue->queue_lock);
71102307
CH		 663}
664
a57bd541 665static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
71102307 666{
a57bd541
SG		 667 int i;
668
669 for (i = 1; i < ctrl->ctrl.queue_count; i++)
670 nvme_rdma_free_queue(&ctrl->queues[i]);
71102307
CH		 671}
672
a57bd541 673static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
71102307
CH		 674{
675 int i;
676
d858e5f0 677 for (i = 1; i < ctrl->ctrl.queue_count; i++)
a57bd541 678 nvme_rdma_stop_queue(&ctrl->queues[i]);
71102307
CH		 679}
680
68e16fcf
SG		 681static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
682{
ff8519f9 683 struct nvme_rdma_queue *queue = &ctrl->queues[idx];
68e16fcf
SG		 684 int ret;
685
686 if (idx)
be42a33b 687 ret = nvmf_connect_io_queue(&ctrl->ctrl, idx);
68e16fcf
SG		 688 else
689 ret = nvmf_connect_admin_queue(&ctrl->ctrl);
690
d94211b8 691 if (!ret) {
ff8519f9 692 set_bit(NVME_RDMA_Q_LIVE, &queue->flags);
d94211b8 693 } else {
67b483dd
SG		 694 if (test_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
695 __nvme_rdma_stop_queue(queue);
68e16fcf
SG		 696 dev_info(ctrl->ctrl.device,
697 "failed to connect queue: %d ret=%d\n", idx, ret);
d94211b8 698 }
68e16fcf
SG		 699 return ret;
700}
701
702static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl)
71102307
CH		 703{
704 int i, ret = 0;
705
d858e5f0 706 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
68e16fcf
SG		 707 ret = nvme_rdma_start_queue(ctrl, i);
708 if (ret)
a57bd541 709 goto out_stop_queues;
71102307
CH		 710 }
711
c8dbc37c
SW		 712 return 0;
713
a57bd541 714out_stop_queues:
68e16fcf
SG		 715 for (i--; i >= 1; i--)
716 nvme_rdma_stop_queue(&ctrl->queues[i]);
71102307
CH		 717 return ret;
718}
719
41e8cfa1 720static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
71102307 721{
c248c643 722 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
0b36658c 723 struct ib_device *ibdev = ctrl->device->dev;
5651cd3c
SG		 724 unsigned int nr_io_queues, nr_default_queues;
725 unsigned int nr_read_queues, nr_poll_queues;
71102307
CH		 726 int i, ret;
727
5651cd3c
SG		 728 nr_read_queues = min_t(unsigned int, ibdev->num_comp_vectors,
729 min(opts->nr_io_queues, num_online_cpus()));
730 nr_default_queues = min_t(unsigned int, ibdev->num_comp_vectors,
731 min(opts->nr_write_queues, num_online_cpus()));
732 nr_poll_queues = min(opts->nr_poll_queues, num_online_cpus());
733 nr_io_queues = nr_read_queues + nr_default_queues + nr_poll_queues;
b65bb777 734
c248c643
SG		 735 ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
736 if (ret)
737 return ret;
738
85032874 739 if (nr_io_queues == 0) {
c4c6df5f
SG		 740 dev_err(ctrl->ctrl.device,
741 "unable to set any I/O queues\n");
742 return -ENOMEM;
743 }
c248c643 744
85032874 745 ctrl->ctrl.queue_count = nr_io_queues + 1;
c248c643
SG		 746 dev_info(ctrl->ctrl.device,
747 "creating %d I/O queues.\n", nr_io_queues);
748
5651cd3c
SG		 749 if (opts->nr_write_queues && nr_read_queues < nr_io_queues) {
750 /*
751 * separate read/write queues
752 * hand out dedicated default queues only after we have
753 * sufficient read queues.
754 */
755 ctrl->io_queues[HCTX_TYPE_READ] = nr_read_queues;
756 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
757 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
758 min(nr_default_queues, nr_io_queues);
759 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
760 } else {
761 /*
762 * shared read/write queues
763 * either no write queues were requested, or we don't have
764 * sufficient queue count to have dedicated default queues.
765 */
766 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
767 min(nr_read_queues, nr_io_queues);
768 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
769 }
770
771 if (opts->nr_poll_queues && nr_io_queues) {
772 /* map dedicated poll queues only if we have queues left */
773 ctrl->io_queues[HCTX_TYPE_POLL] =
774 min(nr_poll_queues, nr_io_queues);
775 }
776
d858e5f0 777 for (i = 1; i < ctrl->ctrl.queue_count; i++) {
41e8cfa1
SG		 778 ret = nvme_rdma_alloc_queue(ctrl, i,
779 ctrl->ctrl.sqsize + 1);
780 if (ret)
71102307 781 goto out_free_queues;
71102307
CH		 782 }
783
784 return 0;
785
786out_free_queues:
f361e5a0 787 for (i--; i >= 1; i--)
a57bd541 788 nvme_rdma_free_queue(&ctrl->queues[i]);
71102307
CH		 789
790 return ret;
791}
792
b28a308e
SG		 793static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl,
794 bool admin)
795{
796 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
797 struct blk_mq_tag_set *set;
798 int ret;
799
800 if (admin) {
801 set = &ctrl->admin_tag_set;
802 memset(set, 0, sizeof(*set));
803 set->ops = &nvme_rdma_admin_mq_ops;
38dabe21 804 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
ed01fee2 805 set->reserved_tags = NVMF_RESERVED_TAGS;
103e515e 806 set->numa_node = nctrl->numa_node;
b28a308e 807 set->cmd_size = sizeof(struct nvme_rdma_request) +
5ec5d3bd 808 NVME_RDMA_DATA_SGL_SIZE;
b28a308e
SG		 809 set->driver_data = ctrl;
810 set->nr_hw_queues = 1;
dc96f938 811 set->timeout = NVME_ADMIN_TIMEOUT;
94f29d4f 812 set->flags = BLK_MQ_F_NO_SCHED;
b28a308e
SG		 813 } else {
814 set = &ctrl->tag_set;
815 memset(set, 0, sizeof(*set));
816 set->ops = &nvme_rdma_mq_ops;
5e77d61c 817 set->queue_depth = nctrl->sqsize + 1;
ed01fee2 818 set->reserved_tags = NVMF_RESERVED_TAGS;
103e515e 819 set->numa_node = nctrl->numa_node;
b28a308e
SG		 820 set->flags = BLK_MQ_F_SHOULD_MERGE;
821 set->cmd_size = sizeof(struct nvme_rdma_request) +
5ec5d3bd
MG		 822 NVME_RDMA_DATA_SGL_SIZE;
823 if (nctrl->max_integrity_segments)
824 set->cmd_size += sizeof(struct nvme_rdma_sgl) +
825 NVME_RDMA_METADATA_SGL_SIZE;
b28a308e
SG		 826 set->driver_data = ctrl;
827 set->nr_hw_queues = nctrl->queue_count - 1;
828 set->timeout = NVME_IO_TIMEOUT;
ff8519f9 829 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
b28a308e
SG		 830 }
831
832 ret = blk_mq_alloc_tag_set(set);
833 if (ret)
87fd1253 834 return ERR_PTR(ret);
b28a308e
SG		 835
836 return set;
b28a308e
SG		 837}
838
3f02fffb
SG		 839static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl,
840 bool remove)
71102307 841{
3f02fffb 842 if (remove) {
6f8191fd
CH		 843 blk_mq_destroy_queue(ctrl->ctrl.admin_q);
844 blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
87fd1253 845 blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
3f02fffb 846 }
682630f0 847 if (ctrl->async_event_sqe.data) {
925dd04c 848 cancel_work_sync(&ctrl->ctrl.async_event_work);
682630f0
SG		 849 nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe,
850 sizeof(struct nvme_command), DMA_TO_DEVICE);
851 ctrl->async_event_sqe.data = NULL;
852 }
a57bd541 853 nvme_rdma_free_queue(&ctrl->queues[0]);
71102307
CH		 854}
855
3f02fffb
SG		 856static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
857 bool new)
90af3512 858{
5ec5d3bd 859 bool pi_capable = false;
90af3512
SG		 860 int error;
861
41e8cfa1 862 error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
90af3512
SG		 863 if (error)
864 return error;
865
866 ctrl->device = ctrl->queues[0].device;
22dd4c70 867 ctrl->ctrl.numa_node = ibdev_to_node(ctrl->device->dev);
90af3512 868
5ec5d3bd 869 /* T10-PI support */
e945c653
JG		 870 if (ctrl->device->dev->attrs.kernel_cap_flags &
871 IBK_INTEGRITY_HANDOVER)
5ec5d3bd
MG		 872 pi_capable = true;
873
874 ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev,
875 pi_capable);
90af3512 876
62f99b62
MG		 877 /*
878 * Bind the async event SQE DMA mapping to the admin queue lifetime.
879 * It's safe, since any chage in the underlying RDMA device will issue
880 * error recovery and queue re-creation.
881 */
94e42213
SG		 882 error = nvme_rdma_alloc_qe(ctrl->device->dev, &ctrl->async_event_sqe,
883 sizeof(struct nvme_command), DMA_TO_DEVICE);
884 if (error)
885 goto out_free_queue;
886
3f02fffb
SG		 887 if (new) {
888 ctrl->ctrl.admin_tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, true);
f04b9cc8
SG		 889 if (IS_ERR(ctrl->ctrl.admin_tagset)) {
890 error = PTR_ERR(ctrl->ctrl.admin_tagset);
94e42213 891 goto out_free_async_qe;
f04b9cc8 892 }
90af3512 893
e7832cb4
SG		 894 ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set);
895 if (IS_ERR(ctrl->ctrl.fabrics_q)) {
896 error = PTR_ERR(ctrl->ctrl.fabrics_q);
897 goto out_free_tagset;
898 }
899
3f02fffb
SG		 900 ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
901 if (IS_ERR(ctrl->ctrl.admin_q)) {
902 error = PTR_ERR(ctrl->ctrl.admin_q);
e7832cb4 903 goto out_cleanup_fabrics_q;
3f02fffb 904 }
90af3512
SG		 905 }
906
68e16fcf 907 error = nvme_rdma_start_queue(ctrl, 0);
90af3512
SG		 908 if (error)
909 goto out_cleanup_queue;
910
c0f2f45b 911 error = nvme_enable_ctrl(&ctrl->ctrl);
90af3512 912 if (error)
2e050f00 913 goto out_stop_queue;
90af3512 914
ff13c1b8
MG		 915 ctrl->ctrl.max_segments = ctrl->max_fr_pages;
916 ctrl->ctrl.max_hw_sectors = ctrl->max_fr_pages << (ilog2(SZ_4K) - 9);
5ec5d3bd
MG		 917 if (pi_capable)
918 ctrl->ctrl.max_integrity_segments = ctrl->max_fr_pages;
919 else
920 ctrl->ctrl.max_integrity_segments = 0;
90af3512 921
6ca1d902 922 nvme_start_admin_queue(&ctrl->ctrl);
e7832cb4 923
f21c4769 924 error = nvme_init_ctrl_finish(&ctrl->ctrl);
90af3512 925 if (error)
958dc1d3 926 goto out_quiesce_queue;
90af3512 927
90af3512
SG		 928 return 0;
929
958dc1d3 930out_quiesce_queue:
6ca1d902 931 nvme_stop_admin_queue(&ctrl->ctrl);
958dc1d3 932 blk_sync_queue(ctrl->ctrl.admin_q);
2e050f00
JW		 933out_stop_queue:
934 nvme_rdma_stop_queue(&ctrl->queues[0]);
958dc1d3 935 nvme_cancel_admin_tagset(&ctrl->ctrl);
90af3512 936out_cleanup_queue:
3f02fffb 937 if (new)
6f8191fd 938 blk_mq_destroy_queue(ctrl->ctrl.admin_q);
e7832cb4
SG		 939out_cleanup_fabrics_q:
940 if (new)
6f8191fd 941 blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
90af3512 942out_free_tagset:
3f02fffb 943 if (new)
87fd1253 944 blk_mq_free_tag_set(ctrl->ctrl.admin_tagset);
94e42213 945out_free_async_qe:
9134ae2a
PS		 946 if (ctrl->async_event_sqe.data) {
947 nvme_rdma_free_qe(ctrl->device->dev, &ctrl->async_event_sqe,
948 sizeof(struct nvme_command), DMA_TO_DEVICE);
949 ctrl->async_event_sqe.data = NULL;
950 }
90af3512
SG		 951out_free_queue:
952 nvme_rdma_free_queue(&ctrl->queues[0]);
953 return error;
954}
955
a57bd541
SG		 956static void nvme_rdma_destroy_io_queues(struct nvme_rdma_ctrl *ctrl,
957 bool remove)
958{
a57bd541 959 if (remove) {
6f8191fd 960 blk_mq_destroy_queue(ctrl->ctrl.connect_q);
87fd1253 961 blk_mq_free_tag_set(ctrl->ctrl.tagset);
a57bd541
SG		 962 }
963 nvme_rdma_free_io_queues(ctrl);
964}
965
966static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new)
967{
968 int ret;
969
41e8cfa1 970 ret = nvme_rdma_alloc_io_queues(ctrl);
a57bd541
SG		 971 if (ret)
972 return ret;
973
974 if (new) {
975 ctrl->ctrl.tagset = nvme_rdma_alloc_tagset(&ctrl->ctrl, false);
f04b9cc8
SG		 976 if (IS_ERR(ctrl->ctrl.tagset)) {
977 ret = PTR_ERR(ctrl->ctrl.tagset);
a57bd541 978 goto out_free_io_queues;
f04b9cc8 979 }
a57bd541 980
72e8b5cd
CK		 981 ret = nvme_ctrl_init_connect_q(&(ctrl->ctrl));
982 if (ret)
a57bd541 983 goto out_free_tag_set;
a57bd541
SG		 984 }
985
68e16fcf 986 ret = nvme_rdma_start_io_queues(ctrl);
a57bd541
SG		 987 if (ret)
988 goto out_cleanup_connect_q;
989
9f98772b
SG		 990 if (!new) {
991 nvme_start_queues(&ctrl->ctrl);
2362acb6
SG		 992 if (!nvme_wait_freeze_timeout(&ctrl->ctrl, NVME_IO_TIMEOUT)) {
993 /*
994 * If we timed out waiting for freeze we are likely to
995 * be stuck. Fail the controller initialization just
996 * to be safe.
997 */
998 ret = -ENODEV;
999 goto out_wait_freeze_timed_out;
1000 }
9f98772b
SG		 1001 blk_mq_update_nr_hw_queues(ctrl->ctrl.tagset,
1002 ctrl->ctrl.queue_count - 1);
1003 nvme_unfreeze(&ctrl->ctrl);
1004 }
1005
a57bd541
SG		 1006 return 0;
1007
2362acb6
SG		 1008out_wait_freeze_timed_out:
1009 nvme_stop_queues(&ctrl->ctrl);
958dc1d3 1010 nvme_sync_io_queues(&ctrl->ctrl);
2362acb6 1011 nvme_rdma_stop_io_queues(ctrl);
a57bd541 1012out_cleanup_connect_q:
958dc1d3 1013 nvme_cancel_tagset(&ctrl->ctrl);
a57bd541 1014 if (new)
6f8191fd 1015 blk_mq_destroy_queue(ctrl->ctrl.connect_q);
a57bd541
SG		 1016out_free_tag_set:
1017 if (new)
87fd1253 1018 blk_mq_free_tag_set(ctrl->ctrl.tagset);
a57bd541
SG		 1019out_free_io_queues:
1020 nvme_rdma_free_io_queues(ctrl);
1021 return ret;
71102307
CH		 1022}
1023
75862c72
SG		 1024static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl,
1025 bool remove)
1026{
6ca1d902 1027 nvme_stop_admin_queue(&ctrl->ctrl);
3017013d 1028 blk_sync_queue(ctrl->ctrl.admin_q);
75862c72 1029 nvme_rdma_stop_queue(&ctrl->queues[0]);
c4189d68 1030 nvme_cancel_admin_tagset(&ctrl->ctrl);
e7832cb4 1031 if (remove)
6ca1d902 1032 nvme_start_admin_queue(&ctrl->ctrl);
75862c72
SG		 1033 nvme_rdma_destroy_admin_queue(ctrl, remove);
1034}
1035
1036static void nvme_rdma_teardown_io_queues(struct nvme_rdma_ctrl *ctrl,
1037 bool remove)
1038{
1039 if (ctrl->ctrl.queue_count > 1) {
9f98772b 1040 nvme_start_freeze(&ctrl->ctrl);
75862c72 1041 nvme_stop_queues(&ctrl->ctrl);
3017013d 1042 nvme_sync_io_queues(&ctrl->ctrl);
75862c72 1043 nvme_rdma_stop_io_queues(ctrl);
c4189d68 1044 nvme_cancel_tagset(&ctrl->ctrl);
75862c72
SG		 1045 if (remove)
1046 nvme_start_queues(&ctrl->ctrl);
1047 nvme_rdma_destroy_io_queues(ctrl, remove);
1048 }
1049}
1050
f7f70f4a
RL		 1051static void nvme_rdma_stop_ctrl(struct nvme_ctrl *nctrl)
1052{
1053 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1054
1055 cancel_work_sync(&ctrl->err_work);
1056 cancel_delayed_work_sync(&ctrl->reconnect_work);
1057}
1058
71102307
CH		 1059static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl)
1060{
1061 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
1062
1063 if (list_empty(&ctrl->list))
1064 goto free_ctrl;
1065
1066 mutex_lock(&nvme_rdma_ctrl_mutex);
1067 list_del(&ctrl->list);
1068 mutex_unlock(&nvme_rdma_ctrl_mutex);
1069
71102307
CH		 1070 nvmf_free_options(nctrl->opts);
1071free_ctrl:
3d064101 1072 kfree(ctrl->queues);
71102307
CH		 1073 kfree(ctrl);
1074}
1075
fd8563ce
SG		 1076static void nvme_rdma_reconnect_or_remove(struct nvme_rdma_ctrl *ctrl)
1077{
1078 /* If we are resetting/deleting then do nothing */
ad6a0a52 1079 if (ctrl->ctrl.state != NVME_CTRL_CONNECTING) {
fd8563ce
SG		 1080 WARN_ON_ONCE(ctrl->ctrl.state == NVME_CTRL_NEW ||
1081 ctrl->ctrl.state == NVME_CTRL_LIVE);
1082 return;
1083 }
1084
1085 if (nvmf_should_reconnect(&ctrl->ctrl)) {
1086 dev_info(ctrl->ctrl.device, "Reconnecting in %d seconds...\n",
1087 ctrl->ctrl.opts->reconnect_delay);
9a6327d2 1088 queue_delayed_work(nvme_wq, &ctrl->reconnect_work,
fd8563ce
SG		 1089 ctrl->ctrl.opts->reconnect_delay * HZ);
1090 } else {
12fa1304 1091 nvme_delete_ctrl(&ctrl->ctrl);
fd8563ce
SG		 1092 }
1093}
1094
c66e2998 1095static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
71102307 1096{
13ce7e62 1097 int ret;
71102307 1098 bool changed;
71102307 1099
c66e2998 1100 ret = nvme_rdma_configure_admin_queue(ctrl, new);
71102307 1101 if (ret)
c66e2998
SG		 1102 return ret;
1103
1104 if (ctrl->ctrl.icdoff) {
09748122 1105 ret = -EOPNOTSUPP;
c66e2998
SG		 1106 dev_err(ctrl->ctrl.device, "icdoff is not supported!\n");
1107 goto destroy_admin;
1108 }
1109
1110 if (!(ctrl->ctrl.sgls & (1 << 2))) {
09748122 1111 ret = -EOPNOTSUPP;
c66e2998
SG		 1112 dev_err(ctrl->ctrl.device,
1113 "Mandatory keyed sgls are not supported!\n");
1114 goto destroy_admin;
1115 }
1116
1117 if (ctrl->ctrl.opts->queue_size > ctrl->ctrl.sqsize + 1) {
1118 dev_warn(ctrl->ctrl.device,
1119 "queue_size %zu > ctrl sqsize %u, clamping down\n",
1120 ctrl->ctrl.opts->queue_size, ctrl->ctrl.sqsize + 1);
1121 }
1122
44c3c625
MG		 1123 if (ctrl->ctrl.sqsize + 1 > NVME_RDMA_MAX_QUEUE_SIZE) {
1124 dev_warn(ctrl->ctrl.device,
1125 "ctrl sqsize %u > max queue size %u, clamping down\n",
1126 ctrl->ctrl.sqsize + 1, NVME_RDMA_MAX_QUEUE_SIZE);
1127 ctrl->ctrl.sqsize = NVME_RDMA_MAX_QUEUE_SIZE - 1;
1128 }
1129
c66e2998
SG		 1130 if (ctrl->ctrl.sqsize + 1 > ctrl->ctrl.maxcmd) {
1131 dev_warn(ctrl->ctrl.device,
1132 "sqsize %u > ctrl maxcmd %u, clamping down\n",
1133 ctrl->ctrl.sqsize + 1, ctrl->ctrl.maxcmd);
1134 ctrl->ctrl.sqsize = ctrl->ctrl.maxcmd - 1;
1135 }
71102307 1136
64a741c1
SW		 1137 if (ctrl->ctrl.sgls & (1 << 20))
1138 ctrl->use_inline_data = true;
71102307 1139
d858e5f0 1140 if (ctrl->ctrl.queue_count > 1) {
c66e2998 1141 ret = nvme_rdma_configure_io_queues(ctrl, new);
71102307 1142 if (ret)
5e1fe61d 1143 goto destroy_admin;
71102307
CH		 1144 }
1145
1146 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
0a960afd 1147 if (!changed) {
96135862 1148 /*
ecca390e 1149 * state change failure is ok if we started ctrl delete,
96135862
IR		 1150 * unless we're during creation of a new controller to
1151 * avoid races with teardown flow.
1152 */
ecca390e
SG		 1153 WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING &&
1154 ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO);
96135862 1155 WARN_ON_ONCE(new);
c66e2998
SG		 1156 ret = -EINVAL;
1157 goto destroy_io;
0a960afd
SG		 1158 }
1159
d09f2b45 1160 nvme_start_ctrl(&ctrl->ctrl);
c66e2998
SG		 1161 return 0;
1162
1163destroy_io:
958dc1d3
CL		 1164 if (ctrl->ctrl.queue_count > 1) {
1165 nvme_stop_queues(&ctrl->ctrl);
1166 nvme_sync_io_queues(&ctrl->ctrl);
1167 nvme_rdma_stop_io_queues(ctrl);
1168 nvme_cancel_tagset(&ctrl->ctrl);
c66e2998 1169 nvme_rdma_destroy_io_queues(ctrl, new);
958dc1d3 1170 }
c66e2998 1171destroy_admin:
6ca1d902 1172 nvme_stop_admin_queue(&ctrl->ctrl);
958dc1d3 1173 blk_sync_queue(ctrl->ctrl.admin_q);
c66e2998 1174 nvme_rdma_stop_queue(&ctrl->queues[0]);
958dc1d3 1175 nvme_cancel_admin_tagset(&ctrl->ctrl);
c66e2998
SG		 1176 nvme_rdma_destroy_admin_queue(ctrl, new);
1177 return ret;
1178}
1179
1180static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work)
1181{
1182 struct nvme_rdma_ctrl *ctrl = container_of(to_delayed_work(work),
1183 struct nvme_rdma_ctrl, reconnect_work);
1184
1185 ++ctrl->ctrl.nr_reconnects;
1186
1187 if (nvme_rdma_setup_ctrl(ctrl, false))
1188 goto requeue;
71102307 1189
5e1fe61d
SG		 1190 dev_info(ctrl->ctrl.device, "Successfully reconnected (%d attempts)\n",
1191 ctrl->ctrl.nr_reconnects);
1192
1193 ctrl->ctrl.nr_reconnects = 0;
71102307
CH		 1194
1195 return;
1196
71102307 1197requeue:
fd8563ce 1198 dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
fdf9dfa8 1199 ctrl->ctrl.nr_reconnects);
fd8563ce 1200 nvme_rdma_reconnect_or_remove(ctrl);
71102307
CH		 1201}
1202
1203static void nvme_rdma_error_recovery_work(struct work_struct *work)
1204{
1205 struct nvme_rdma_ctrl *ctrl = container_of(work,
1206 struct nvme_rdma_ctrl, err_work);
1207
f50fff73 1208 nvme_auth_stop(&ctrl->ctrl);
e4d753d7 1209 nvme_stop_keep_alive(&ctrl->ctrl);
b6bb1722 1210 flush_work(&ctrl->ctrl.async_event_work);
75862c72 1211 nvme_rdma_teardown_io_queues(ctrl, false);
e818a5b4 1212 nvme_start_queues(&ctrl->ctrl);
75862c72 1213 nvme_rdma_teardown_admin_queue(ctrl, false);
6ca1d902 1214 nvme_start_admin_queue(&ctrl->ctrl);
e818a5b4 1215
ad6a0a52 1216 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
ecca390e
SG		 1217 /* state change failure is ok if we started ctrl delete */
1218 WARN_ON_ONCE(ctrl->ctrl.state != NVME_CTRL_DELETING &&
1219 ctrl->ctrl.state != NVME_CTRL_DELETING_NOIO);
d5bf4b7f
SG		 1220 return;
1221 }
1222
fd8563ce 1223 nvme_rdma_reconnect_or_remove(ctrl);
71102307
CH		 1224}
1225
1226static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl)
1227{
d5bf4b7f 1228 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_RESETTING))
71102307
CH		 1229 return;
1230
0475a8dc 1231 dev_warn(ctrl->ctrl.device, "starting error recovery\n");
97b2512a 1232 queue_work(nvme_reset_wq, &ctrl->err_work);
71102307
CH		 1233}
1234
8446546c
CH		 1235static void nvme_rdma_end_request(struct nvme_rdma_request *req)
1236{
1237 struct request *rq = blk_mq_rq_from_pdu(req);
1238
1239 if (!refcount_dec_and_test(&req->ref))
1240 return;
2eb81a33 1241 if (!nvme_try_complete_req(rq, req->status, req->result))
ff029451 1242 nvme_rdma_complete_rq(rq);
8446546c
CH		 1243}
1244
71102307
CH		 1245static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc,
1246 const char *op)
1247{
287f329e 1248 struct nvme_rdma_queue *queue = wc->qp->qp_context;
71102307
CH		 1249 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1250
1251 if (ctrl->ctrl.state == NVME_CTRL_LIVE)
1252 dev_info(ctrl->ctrl.device,
1253 "%s for CQE 0x%p failed with status %s (%d)\n",
1254 op, wc->wr_cqe,
1255 ib_wc_status_msg(wc->status), wc->status);
1256 nvme_rdma_error_recovery(ctrl);
1257}
1258
1259static void nvme_rdma_memreg_done(struct ib_cq *cq, struct ib_wc *wc)
1260{
1261 if (unlikely(wc->status != IB_WC_SUCCESS))
1262 nvme_rdma_wr_error(cq, wc, "MEMREG");
1263}
1264
1265static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc)
1266{
2f122e4f
SG		 1267 struct nvme_rdma_request *req =
1268 container_of(wc->wr_cqe, struct nvme_rdma_request, reg_cqe);
2f122e4f 1269
8446546c 1270 if (unlikely(wc->status != IB_WC_SUCCESS))
71102307 1271 nvme_rdma_wr_error(cq, wc, "LOCAL_INV");
8446546c
CH		 1272 else
1273 nvme_rdma_end_request(req);
71102307
CH		 1274}
1275
1276static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue,
1277 struct nvme_rdma_request *req)
1278{
71102307
CH		 1279 struct ib_send_wr wr = {
1280 .opcode = IB_WR_LOCAL_INV,
1281 .next = NULL,
1282 .num_sge = 0,
2f122e4f 1283 .send_flags = IB_SEND_SIGNALED,
71102307
CH		 1284 .ex.invalidate_rkey = req->mr->rkey,
1285 };
1286
1287 req->reg_cqe.done = nvme_rdma_inv_rkey_done;
1288 wr.wr_cqe = &req->reg_cqe;
1289
45e3cc1a 1290 return ib_post_send(queue->qp, &wr, NULL);
71102307
CH		 1291}
1292
4686af88
MG		 1293static void nvme_rdma_dma_unmap_req(struct ib_device *ibdev, struct request *rq)
1294{
1295 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1296
1297 if (blk_integrity_rq(rq)) {
1298 ib_dma_unmap_sg(ibdev, req->metadata_sgl->sg_table.sgl,
1299 req->metadata_sgl->nents, rq_dma_dir(rq));
1300 sg_free_table_chained(&req->metadata_sgl->sg_table,
1301 NVME_INLINE_METADATA_SG_CNT);
1302 }
1303
1304 ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents,
1305 rq_dma_dir(rq));
1306 sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT);
1307}
1308
71102307
CH		 1309static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue,
1310 struct request *rq)
1311{
1312 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
71102307
CH		 1313 struct nvme_rdma_device *dev = queue->device;
1314 struct ib_device *ibdev = dev->dev;
5ec5d3bd 1315 struct list_head *pool = &queue->qp->rdma_mrs;
71102307 1316
34e08191 1317 if (!blk_rq_nr_phys_segments(rq))
71102307
CH		 1318 return;
1319
5ec5d3bd
MG		 1320 if (req->use_sig_mr)
1321 pool = &queue->qp->sig_mrs;
1322
f41725bb 1323 if (req->mr) {
5ec5d3bd 1324 ib_mr_pool_put(queue->qp, pool, req->mr);
f41725bb
IR		 1325 req->mr = NULL;
1326 }
1327
4686af88 1328 nvme_rdma_dma_unmap_req(ibdev, rq);
71102307
CH		 1329}
1330
1331static int nvme_rdma_set_sg_null(struct nvme_command *c)
1332{
1333 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1334
1335 sg->addr = 0;
1336 put_unaligned_le24(0, sg->length);
1337 put_unaligned_le32(0, sg->key);
1338 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1339 return 0;
1340}
1341
1342static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue,
64a741c1
SW		 1343 struct nvme_rdma_request *req, struct nvme_command *c,
1344 int count)
71102307
CH		 1345{
1346 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
64a741c1 1347 struct ib_sge *sge = &req->sge[1];
12b2aaad 1348 struct scatterlist *sgl;
64a741c1
SW		 1349 u32 len = 0;
1350 int i;
71102307 1351
12b2aaad 1352 for_each_sg(req->data_sgl.sg_table.sgl, sgl, count, i) {
64a741c1
SW		 1353 sge->addr = sg_dma_address(sgl);
1354 sge->length = sg_dma_len(sgl);
1355 sge->lkey = queue->device->pd->local_dma_lkey;
1356 len += sge->length;
12b2aaad 1357 sge++;
64a741c1 1358 }
71102307
CH		 1359
1360 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
64a741c1 1361 sg->length = cpu_to_le32(len);
71102307
CH		 1362 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
1363
64a741c1 1364 req->num_sge += count;
71102307
CH		 1365 return 0;
1366}
1367
1368static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue,
1369 struct nvme_rdma_request *req, struct nvme_command *c)
1370{
1371 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1372
324d9e78
IR		 1373 sg->addr = cpu_to_le64(sg_dma_address(req->data_sgl.sg_table.sgl));
1374 put_unaligned_le24(sg_dma_len(req->data_sgl.sg_table.sgl), sg->length);
11975e01 1375 put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key);
71102307
CH		 1376 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1377 return 0;
1378}
1379
1380static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue,
1381 struct nvme_rdma_request *req, struct nvme_command *c,
1382 int count)
1383{
1384 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1385 int nr;
1386
f41725bb
IR		 1387 req->mr = ib_mr_pool_get(queue->qp, &queue->qp->rdma_mrs);
1388 if (WARN_ON_ONCE(!req->mr))
1389 return -EAGAIN;
1390
b925a2dc
MG		 1391 /*
1392 * Align the MR to a 4K page size to match the ctrl page size and
1393 * the block virtual boundary.
1394 */
324d9e78
IR		 1395 nr = ib_map_mr_sg(req->mr, req->data_sgl.sg_table.sgl, count, NULL,
1396 SZ_4K);
a7b7c7a1 1397 if (unlikely(nr < count)) {
f41725bb
IR		 1398 ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr);
1399 req->mr = NULL;
71102307
CH		 1400 if (nr < 0)
1401 return nr;
1402 return -EINVAL;
1403 }
1404
1405 ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
1406
1407 req->reg_cqe.done = nvme_rdma_memreg_done;
1408 memset(&req->reg_wr, 0, sizeof(req->reg_wr));
1409 req->reg_wr.wr.opcode = IB_WR_REG_MR;
1410 req->reg_wr.wr.wr_cqe = &req->reg_cqe;
1411 req->reg_wr.wr.num_sge = 0;
1412 req->reg_wr.mr = req->mr;
1413 req->reg_wr.key = req->mr->rkey;
1414 req->reg_wr.access = IB_ACCESS_LOCAL_WRITE |
1415 IB_ACCESS_REMOTE_READ |
1416 IB_ACCESS_REMOTE_WRITE;
1417
71102307
CH		 1418 sg->addr = cpu_to_le64(req->mr->iova);
1419 put_unaligned_le24(req->mr->length, sg->length);
1420 put_unaligned_le32(req->mr->rkey, sg->key);
1421 sg->type = (NVME_KEY_SGL_FMT_DATA_DESC << 4) |
1422 NVME_SGL_FMT_INVALIDATE;
1423
1424 return 0;
1425}
1426
5ec5d3bd
MG		 1427static void nvme_rdma_set_sig_domain(struct blk_integrity *bi,
1428 struct nvme_command *cmd, struct ib_sig_domain *domain,
1429 u16 control, u8 pi_type)
1430{
1431 domain->sig_type = IB_SIG_TYPE_T10_DIF;
1432 domain->sig.dif.bg_type = IB_T10DIF_CRC;
1433 domain->sig.dif.pi_interval = 1 << bi->interval_exp;
1434 domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag);
1435 if (control & NVME_RW_PRINFO_PRCHK_REF)
1436 domain->sig.dif.ref_remap = true;
1437
1438 domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.apptag);
1439 domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.appmask);
1440 domain->sig.dif.app_escape = true;
1441 if (pi_type == NVME_NS_DPS_PI_TYPE3)
1442 domain->sig.dif.ref_escape = true;
1443}
1444
1445static void nvme_rdma_set_sig_attrs(struct blk_integrity *bi,
1446 struct nvme_command *cmd, struct ib_sig_attrs *sig_attrs,
1447 u8 pi_type)
1448{
1449 u16 control = le16_to_cpu(cmd->rw.control);
1450
1451 memset(sig_attrs, 0, sizeof(*sig_attrs));
1452 if (control & NVME_RW_PRINFO_PRACT) {
1453 /* for WRITE_INSERT/READ_STRIP no memory domain */
1454 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
1455 nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control,
1456 pi_type);
1457 /* Clear the PRACT bit since HCA will generate/verify the PI */
1458 control &= ~NVME_RW_PRINFO_PRACT;
1459 cmd->rw.control = cpu_to_le16(control);
1460 } else {
1461 /* for WRITE_PASS/READ_PASS both wire/memory domains exist */
1462 nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control,
1463 pi_type);
1464 nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control,
1465 pi_type);
1466 }
1467}
1468
1469static void nvme_rdma_set_prot_checks(struct nvme_command *cmd, u8 *mask)
1470{
1471 *mask = 0;
1472 if (le16_to_cpu(cmd->rw.control) & NVME_RW_PRINFO_PRCHK_REF)
1473 *mask |= IB_SIG_CHECK_REFTAG;
1474 if (le16_to_cpu(cmd->rw.control) & NVME_RW_PRINFO_PRCHK_GUARD)
1475 *mask |= IB_SIG_CHECK_GUARD;
1476}
1477
1478static void nvme_rdma_sig_done(struct ib_cq *cq, struct ib_wc *wc)
1479{
1480 if (unlikely(wc->status != IB_WC_SUCCESS))
1481 nvme_rdma_wr_error(cq, wc, "SIG");
1482}
1483
1484static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue,
1485 struct nvme_rdma_request *req, struct nvme_command *c,
1486 int count, int pi_count)
1487{
1488 struct nvme_rdma_sgl *sgl = &req->data_sgl;
1489 struct ib_reg_wr *wr = &req->reg_wr;
1490 struct request *rq = blk_mq_rq_from_pdu(req);
1491 struct nvme_ns *ns = rq->q->queuedata;
1492 struct bio *bio = rq->bio;
1493 struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl;
1494 int nr;
1495
1496 req->mr = ib_mr_pool_get(queue->qp, &queue->qp->sig_mrs);
1497 if (WARN_ON_ONCE(!req->mr))
1498 return -EAGAIN;
1499
1500 nr = ib_map_mr_sg_pi(req->mr, sgl->sg_table.sgl, count, NULL,
1501 req->metadata_sgl->sg_table.sgl, pi_count, NULL,
1502 SZ_4K);
1503 if (unlikely(nr))
1504 goto mr_put;
1505
309dca30 1506 nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_bdev->bd_disk), c,
5ec5d3bd
MG		 1507 req->mr->sig_attrs, ns->pi_type);
1508 nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask);
1509
1510 ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey));
1511
1512 req->reg_cqe.done = nvme_rdma_sig_done;
1513 memset(wr, 0, sizeof(*wr));
1514 wr->wr.opcode = IB_WR_REG_MR_INTEGRITY;
1515 wr->wr.wr_cqe = &req->reg_cqe;
1516 wr->wr.num_sge = 0;
1517 wr->wr.send_flags = 0;
1518 wr->mr = req->mr;
1519 wr->key = req->mr->rkey;
1520 wr->access = IB_ACCESS_LOCAL_WRITE |
1521 IB_ACCESS_REMOTE_READ |
1522 IB_ACCESS_REMOTE_WRITE;
1523
1524 sg->addr = cpu_to_le64(req->mr->iova);
1525 put_unaligned_le24(req->mr->length, sg->length);
1526 put_unaligned_le32(req->mr->rkey, sg->key);
1527 sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4;
1528
1529 return 0;
1530
1531mr_put:
1532 ib_mr_pool_put(queue->qp, &queue->qp->sig_mrs, req->mr);
1533 req->mr = NULL;
1534 if (nr < 0)
1535 return nr;
1536 return -EINVAL;
1537}
1538
4686af88
MG		 1539static int nvme_rdma_dma_map_req(struct ib_device *ibdev, struct request *rq,
1540 int *count, int *pi_count)
71102307
CH		 1541{
1542 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
4686af88 1543 int ret;
71102307 1544
324d9e78
IR		 1545 req->data_sgl.sg_table.sgl = (struct scatterlist *)(req + 1);
1546 ret = sg_alloc_table_chained(&req->data_sgl.sg_table,
1547 blk_rq_nr_phys_segments(rq), req->data_sgl.sg_table.sgl,
38e18002 1548 NVME_INLINE_SG_CNT);
71102307
CH		 1549 if (ret)
1550 return -ENOMEM;
1551
324d9e78
IR		 1552 req->data_sgl.nents = blk_rq_map_sg(rq->q, rq,
1553 req->data_sgl.sg_table.sgl);
71102307 1554
4686af88
MG		 1555 *count = ib_dma_map_sg(ibdev, req->data_sgl.sg_table.sgl,
1556 req->data_sgl.nents, rq_dma_dir(rq));
1557 if (unlikely(*count <= 0)) {
94423a8f
MG		 1558 ret = -EIO;
1559 goto out_free_table;
71102307
CH		 1560 }
1561
5ec5d3bd
MG		 1562 if (blk_integrity_rq(rq)) {
1563 req->metadata_sgl->sg_table.sgl =
1564 (struct scatterlist *)(req->metadata_sgl + 1);
1565 ret = sg_alloc_table_chained(&req->metadata_sgl->sg_table,
1566 blk_rq_count_integrity_sg(rq->q, rq->bio),
1567 req->metadata_sgl->sg_table.sgl,
1568 NVME_INLINE_METADATA_SG_CNT);
1569 if (unlikely(ret)) {
1570 ret = -ENOMEM;
1571 goto out_unmap_sg;
1572 }
1573
1574 req->metadata_sgl->nents = blk_rq_map_integrity_sg(rq->q,
1575 rq->bio, req->metadata_sgl->sg_table.sgl);
4686af88
MG		 1576 *pi_count = ib_dma_map_sg(ibdev,
1577 req->metadata_sgl->sg_table.sgl,
1578 req->metadata_sgl->nents,
1579 rq_dma_dir(rq));
1580 if (unlikely(*pi_count <= 0)) {
5ec5d3bd
MG		 1581 ret = -EIO;
1582 goto out_free_pi_table;
1583 }
1584 }
1585
4686af88
MG		 1586 return 0;
1587
1588out_free_pi_table:
1589 sg_free_table_chained(&req->metadata_sgl->sg_table,
1590 NVME_INLINE_METADATA_SG_CNT);
1591out_unmap_sg:
1592 ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents,
1593 rq_dma_dir(rq));
1594out_free_table:
1595 sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT);
1596 return ret;
1597}
1598
1599static int nvme_rdma_map_data(struct nvme_rdma_queue *queue,
1600 struct request *rq, struct nvme_command *c)
1601{
1602 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
1603 struct nvme_rdma_device *dev = queue->device;
1604 struct ib_device *ibdev = dev->dev;
1605 int pi_count = 0;
1606 int count, ret;
1607
1608 req->num_sge = 1;
1609 refcount_set(&req->ref, 2); /* send and recv completions */
1610
1611 c->common.flags |= NVME_CMD_SGL_METABUF;
1612
1613 if (!blk_rq_nr_phys_segments(rq))
1614 return nvme_rdma_set_sg_null(c);
1615
1616 ret = nvme_rdma_dma_map_req(ibdev, rq, &count, &pi_count);
1617 if (unlikely(ret))
1618 return ret;
1619
5ec5d3bd
MG		 1620 if (req->use_sig_mr) {
1621 ret = nvme_rdma_map_sg_pi(queue, req, c, count, pi_count);
1622 goto out;
1623 }
1624
64a741c1 1625 if (count <= dev->num_inline_segments) {
b131c61d 1626 if (rq_data_dir(rq) == WRITE && nvme_rdma_queue_idx(queue) &&
64a741c1 1627 queue->ctrl->use_inline_data &&
b131c61d 1628 blk_rq_payload_bytes(rq) <=
94423a8f 1629 nvme_rdma_inline_data_size(queue)) {
64a741c1 1630 ret = nvme_rdma_map_sg_inline(queue, req, c, count);
94423a8f
MG		 1631 goto out;
1632 }
71102307 1633
64a741c1 1634 if (count == 1 && dev->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
94423a8f
MG		 1635 ret = nvme_rdma_map_sg_single(queue, req, c);
1636 goto out;
1637 }
71102307
CH		 1638 }
1639
94423a8f
MG		 1640 ret = nvme_rdma_map_sg_fr(queue, req, c, count);
1641out:
1642 if (unlikely(ret))
4686af88 1643 goto out_dma_unmap_req;
94423a8f
MG		 1644
1645 return 0;
1646
4686af88
MG		 1647out_dma_unmap_req:
1648 nvme_rdma_dma_unmap_req(ibdev, rq);
94423a8f 1649 return ret;
71102307
CH		 1650}
1651
1652static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc)
1653{
4af7f7ff
SG		 1654 struct nvme_rdma_qe *qe =
1655 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
1656 struct nvme_rdma_request *req =
1657 container_of(qe, struct nvme_rdma_request, sqe);
4af7f7ff 1658
8446546c 1659 if (unlikely(wc->status != IB_WC_SUCCESS))
71102307 1660 nvme_rdma_wr_error(cq, wc, "SEND");
8446546c
CH		 1661 else
1662 nvme_rdma_end_request(req);
71102307
CH		 1663}
1664
1665static int nvme_rdma_post_send(struct nvme_rdma_queue *queue,
1666 struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge,
b4b591c8 1667 struct ib_send_wr *first)
71102307 1668{
45e3cc1a 1669 struct ib_send_wr wr;
71102307
CH		 1670 int ret;
1671
1672 sge->addr = qe->dma;
a62315b8 1673 sge->length = sizeof(struct nvme_command);
71102307
CH		 1674 sge->lkey = queue->device->pd->local_dma_lkey;
1675
71102307
CH		 1676 wr.next = NULL;
1677 wr.wr_cqe = &qe->cqe;
1678 wr.sg_list = sge;
1679 wr.num_sge = num_sge;
1680 wr.opcode = IB_WR_SEND;
b4b591c8 1681 wr.send_flags = IB_SEND_SIGNALED;
71102307
CH		 1682
1683 if (first)
1684 first->next = &wr;
1685 else
1686 first = &wr;
1687
45e3cc1a 1688 ret = ib_post_send(queue->qp, first, NULL);
a7b7c7a1 1689 if (unlikely(ret)) {
71102307
CH		 1690 dev_err(queue->ctrl->ctrl.device,
1691 "%s failed with error code %d\n", __func__, ret);
1692 }
1693 return ret;
1694}
1695
1696static int nvme_rdma_post_recv(struct nvme_rdma_queue *queue,
1697 struct nvme_rdma_qe *qe)
1698{
45e3cc1a 1699 struct ib_recv_wr wr;
71102307
CH		 1700 struct ib_sge list;
1701 int ret;
1702
1703 list.addr = qe->dma;
1704 list.length = sizeof(struct nvme_completion);
1705 list.lkey = queue->device->pd->local_dma_lkey;
1706
1707 qe->cqe.done = nvme_rdma_recv_done;
1708
1709 wr.next = NULL;
1710 wr.wr_cqe = &qe->cqe;
1711 wr.sg_list = &list;
1712 wr.num_sge = 1;
1713
45e3cc1a 1714 ret = ib_post_recv(queue->qp, &wr, NULL);
a7b7c7a1 1715 if (unlikely(ret)) {
71102307
CH		 1716 dev_err(queue->ctrl->ctrl.device,
1717 "%s failed with error code %d\n", __func__, ret);
1718 }
1719 return ret;
1720}
1721
1722static struct blk_mq_tags *nvme_rdma_tagset(struct nvme_rdma_queue *queue)
1723{
1724 u32 queue_idx = nvme_rdma_queue_idx(queue);
1725
1726 if (queue_idx == 0)
1727 return queue->ctrl->admin_tag_set.tags[queue_idx];
1728 return queue->ctrl->tag_set.tags[queue_idx - 1];
1729}
1730
b4b591c8
SG		 1731static void nvme_rdma_async_done(struct ib_cq *cq, struct ib_wc *wc)
1732{
1733 if (unlikely(wc->status != IB_WC_SUCCESS))
1734 nvme_rdma_wr_error(cq, wc, "ASYNC");
1735}
1736
ad22c355 1737static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg)
71102307
CH		 1738{
1739 struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(arg);
1740 struct nvme_rdma_queue *queue = &ctrl->queues[0];
1741 struct ib_device *dev = queue->device->dev;
1742 struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe;
1743 struct nvme_command *cmd = sqe->data;
1744 struct ib_sge sge;
1745 int ret;
1746
71102307
CH		 1747 ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE);
1748
1749 memset(cmd, 0, sizeof(*cmd));
1750 cmd->common.opcode = nvme_admin_async_event;
38dabe21 1751 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
71102307
CH		 1752 cmd->common.flags |= NVME_CMD_SGL_METABUF;
1753 nvme_rdma_set_sg_null(cmd);
1754
b4b591c8
SG		 1755 sqe->cqe.done = nvme_rdma_async_done;
1756
71102307
CH		 1757 ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd),
1758 DMA_TO_DEVICE);
1759
b4b591c8 1760 ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL);
71102307
CH		 1761 WARN_ON_ONCE(ret);
1762}
1763
1052b8ac
JA		 1764static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue,
1765 struct nvme_completion *cqe, struct ib_wc *wc)
71102307 1766{
71102307
CH		 1767 struct request *rq;
1768 struct nvme_rdma_request *req;
71102307 1769
e7006de6 1770 rq = nvme_find_rq(nvme_rdma_tagset(queue), cqe->command_id);
71102307
CH		 1771 if (!rq) {
1772 dev_err(queue->ctrl->ctrl.device,
e7006de6 1773 "got bad command_id %#x on QP %#x\n",
71102307
CH		 1774 cqe->command_id, queue->qp->qp_num);
1775 nvme_rdma_error_recovery(queue->ctrl);
1052b8ac 1776 return;
71102307
CH		 1777 }
1778 req = blk_mq_rq_to_pdu(rq);
1779
4af7f7ff
SG		 1780 req->status = cqe->status;
1781 req->result = cqe->result;
71102307 1782
3ef0279b 1783 if (wc->wc_flags & IB_WC_WITH_INVALIDATE) {
a87da50f
CL		 1784 if (unlikely(!req->mr ||
1785 wc->ex.invalidate_rkey != req->mr->rkey)) {
3ef0279b
SG		 1786 dev_err(queue->ctrl->ctrl.device,
1787 "Bogus remote invalidation for rkey %#x\n",
a87da50f 1788 req->mr ? req->mr->rkey : 0);
3ef0279b
SG		 1789 nvme_rdma_error_recovery(queue->ctrl);
1790 }
f41725bb 1791 } else if (req->mr) {
1052b8ac
JA		 1792 int ret;
1793
2f122e4f
SG		 1794 ret = nvme_rdma_inv_rkey(queue, req);
1795 if (unlikely(ret < 0)) {
1796 dev_err(queue->ctrl->ctrl.device,
1797 "Queueing INV WR for rkey %#x failed (%d)\n",
1798 req->mr->rkey, ret);
1799 nvme_rdma_error_recovery(queue->ctrl);
1800 }
1801 /* the local invalidation completion will end the request */
7a804c34 1802 return;
2f122e4f 1803 }
7a804c34
CH		 1804
1805 nvme_rdma_end_request(req);
71102307
CH		 1806}
1807
1052b8ac 1808static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc)
71102307
CH		 1809{
1810 struct nvme_rdma_qe *qe =
1811 container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe);
287f329e 1812 struct nvme_rdma_queue *queue = wc->qp->qp_context;
71102307
CH		 1813 struct ib_device *ibdev = queue->device->dev;
1814 struct nvme_completion *cqe = qe->data;
1815 const size_t len = sizeof(struct nvme_completion);
71102307
CH		 1816
1817 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1818 nvme_rdma_wr_error(cq, wc, "RECV");
1052b8ac 1819 return;
71102307
CH		 1820 }
1821
25c1ca6e 1822 /* sanity checking for received data length */
1823 if (unlikely(wc->byte_len < len)) {
1824 dev_err(queue->ctrl->ctrl.device,
1825 "Unexpected nvme completion length(%d)\n", wc->byte_len);
1826 nvme_rdma_error_recovery(queue->ctrl);
1827 return;
1828 }
1829
71102307
CH		 1830 ib_dma_sync_single_for_cpu(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1831 /*
1832 * AEN requests are special as they don't time out and can
1833 * survive any kind of queue freeze and often don't respond to
1834 * aborts. We don't even bother to allocate a struct request
1835 * for them but rather special case them here.
1836 */
58a8df67
IR		 1837 if (unlikely(nvme_is_aen_req(nvme_rdma_queue_idx(queue),
1838 cqe->command_id)))
7bf58533
CH		 1839 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
1840 &cqe->result);
71102307 1841 else
1052b8ac 1842 nvme_rdma_process_nvme_rsp(queue, cqe, wc);
71102307
CH		 1843 ib_dma_sync_single_for_device(ibdev, qe->dma, len, DMA_FROM_DEVICE);
1844
1845 nvme_rdma_post_recv(queue, qe);
71102307
CH		 1846}
1847
1848static int nvme_rdma_conn_established(struct nvme_rdma_queue *queue)
1849{
1850 int ret, i;
1851
1852 for (i = 0; i < queue->queue_size; i++) {
1853 ret = nvme_rdma_post_recv(queue, &queue->rsp_ring[i]);
1854 if (ret)
9817d763 1855 return ret;
71102307
CH		 1856 }
1857
1858 return 0;
71102307
CH		 1859}
1860
1861static int nvme_rdma_conn_rejected(struct nvme_rdma_queue *queue,
1862 struct rdma_cm_event *ev)
1863{
7f03953c
SW		 1864 struct rdma_cm_id *cm_id = queue->cm_id;
1865 int status = ev->status;
1866 const char *rej_msg;
1867 const struct nvme_rdma_cm_rej *rej_data;
1868 u8 rej_data_len;
1869
1870 rej_msg = rdma_reject_msg(cm_id, status);
1871 rej_data = rdma_consumer_reject_data(cm_id, ev, &rej_data_len);
1872
1873 if (rej_data && rej_data_len >= sizeof(u16)) {
1874 u16 sts = le16_to_cpu(rej_data->sts);
71102307
CH		 1875
1876 dev_err(queue->ctrl->ctrl.device,
7f03953c
SW		 1877 "Connect rejected: status %d (%s) nvme status %d (%s).\n",
1878 status, rej_msg, sts, nvme_rdma_cm_msg(sts));
71102307
CH		 1879 } else {
1880 dev_err(queue->ctrl->ctrl.device,
7f03953c 1881 "Connect rejected: status %d (%s).\n", status, rej_msg);
71102307
CH		 1882 }
1883
1884 return -ECONNRESET;
1885}
1886
1887static int nvme_rdma_addr_resolved(struct nvme_rdma_queue *queue)
1888{
e63440d6 1889 struct nvme_ctrl *ctrl = &queue->ctrl->ctrl;
71102307
CH		 1890 int ret;
1891
ca6e95bb
SG		 1892 ret = nvme_rdma_create_queue_ib(queue);
1893 if (ret)
1894 return ret;
71102307 1895
e63440d6
IR		 1896 if (ctrl->opts->tos >= 0)
1897 rdma_set_service_type(queue->cm_id, ctrl->opts->tos);
71102307
CH		 1898 ret = rdma_resolve_route(queue->cm_id, NVME_RDMA_CONNECT_TIMEOUT_MS);
1899 if (ret) {
e63440d6 1900 dev_err(ctrl->device, "rdma_resolve_route failed (%d).\n",
71102307
CH		 1901 queue->cm_error);
1902 goto out_destroy_queue;
1903 }
1904
1905 return 0;
1906
1907out_destroy_queue:
1908 nvme_rdma_destroy_queue_ib(queue);
71102307
CH		 1909 return ret;
1910}
1911
1912static int nvme_rdma_route_resolved(struct nvme_rdma_queue *queue)
1913{
1914 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
1915 struct rdma_conn_param param = { };
0b857b44 1916 struct nvme_rdma_cm_req priv = { };
71102307
CH		 1917 int ret;
1918
1919 param.qp_num = queue->qp->qp_num;
1920 param.flow_control = 1;
1921
1922 param.responder_resources = queue->device->dev->attrs.max_qp_rd_atom;
2ac17c28
SG		 1923 /* maximum retry count */
1924 param.retry_count = 7;
71102307
CH		 1925 param.rnr_retry_count = 7;
1926 param.private_data = &priv;
1927 param.private_data_len = sizeof(priv);
1928
1929 priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0);
1930 priv.qid = cpu_to_le16(nvme_rdma_queue_idx(queue));
f994d9dc
JF		 1931 /*
1932 * set the admin queue depth to the minimum size
1933 * specified by the Fabrics standard.
1934 */
1935 if (priv.qid == 0) {
7aa1f427
SG		 1936 priv.hrqsize = cpu_to_le16(NVME_AQ_DEPTH);
1937 priv.hsqsize = cpu_to_le16(NVME_AQ_DEPTH - 1);
f994d9dc 1938 } else {
c5af8654
JF		 1939 /*
1940 * current interpretation of the fabrics spec
1941 * is at minimum you make hrqsize sqsize+1, or a
1942 * 1's based representation of sqsize.
1943 */
f994d9dc 1944 priv.hrqsize = cpu_to_le16(queue->queue_size);
c5af8654 1945 priv.hsqsize = cpu_to_le16(queue->ctrl->ctrl.sqsize);
f994d9dc 1946 }
71102307 1947
071ba4cc 1948 ret = rdma_connect_locked(queue->cm_id, &param);
71102307
CH		 1949 if (ret) {
1950 dev_err(ctrl->ctrl.device,
071ba4cc 1951 "rdma_connect_locked failed (%d).\n", ret);
9817d763 1952 return ret;
71102307
CH		 1953 }
1954
1955 return 0;
71102307
CH		 1956}
1957
71102307
CH		 1958static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
1959 struct rdma_cm_event *ev)
1960{
1961 struct nvme_rdma_queue *queue = cm_id->context;
1962 int cm_error = 0;
1963
1964 dev_dbg(queue->ctrl->ctrl.device, "%s (%d): status %d id %p\n",
1965 rdma_event_msg(ev->event), ev->event,
1966 ev->status, cm_id);
1967
1968 switch (ev->event) {
1969 case RDMA_CM_EVENT_ADDR_RESOLVED:
1970 cm_error = nvme_rdma_addr_resolved(queue);
1971 break;
1972 case RDMA_CM_EVENT_ROUTE_RESOLVED:
1973 cm_error = nvme_rdma_route_resolved(queue);
1974 break;
1975 case RDMA_CM_EVENT_ESTABLISHED:
1976 queue->cm_error = nvme_rdma_conn_established(queue);
1977 /* complete cm_done regardless of success/failure */
1978 complete(&queue->cm_done);
1979 return 0;
1980 case RDMA_CM_EVENT_REJECTED:
1981 cm_error = nvme_rdma_conn_rejected(queue, ev);
1982 break;
71102307
CH		 1983 case RDMA_CM_EVENT_ROUTE_ERROR:
1984 case RDMA_CM_EVENT_CONNECT_ERROR:
1985 case RDMA_CM_EVENT_UNREACHABLE:
abf87d5e 1986 case RDMA_CM_EVENT_ADDR_ERROR:
71102307
CH		 1987 dev_dbg(queue->ctrl->ctrl.device,
1988 "CM error event %d\n", ev->event);
1989 cm_error = -ECONNRESET;
1990 break;
1991 case RDMA_CM_EVENT_DISCONNECTED:
1992 case RDMA_CM_EVENT_ADDR_CHANGE:
1993 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
1994 dev_dbg(queue->ctrl->ctrl.device,
1995 "disconnect received - connection closed\n");
1996 nvme_rdma_error_recovery(queue->ctrl);
1997 break;
1998 case RDMA_CM_EVENT_DEVICE_REMOVAL:
e87a911f
SW		 1999 /* device removal is handled via the ib_client API */
2000 break;
71102307
CH		 2001 default:
2002 dev_err(queue->ctrl->ctrl.device,
2003 "Unexpected RDMA CM event (%d)\n", ev->event);
2004 nvme_rdma_error_recovery(queue->ctrl);
2005 break;
2006 }
2007
2008 if (cm_error) {
2009 queue->cm_error = cm_error;
2010 complete(&queue->cm_done);
2011 }
2012
2013 return 0;
2014}
2015
0475a8dc
SG		 2016static void nvme_rdma_complete_timed_out(struct request *rq)
2017{
2018 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
2019 struct nvme_rdma_queue *queue = req->queue;
0475a8dc 2020
0475a8dc 2021 nvme_rdma_stop_queue(queue);
93ba75c9 2022 nvmf_complete_timed_out_request(rq);
0475a8dc
SG		 2023}
2024
9bdb4833 2025static enum blk_eh_timer_return nvme_rdma_timeout(struct request *rq)
71102307
CH		 2026{
2027 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
4c174e63
SG		 2028 struct nvme_rdma_queue *queue = req->queue;
2029 struct nvme_rdma_ctrl *ctrl = queue->ctrl;
71102307 2030
4c174e63
SG		 2031 dev_warn(ctrl->ctrl.device, "I/O %d QID %d timeout\n",
2032 rq->tag, nvme_rdma_queue_idx(queue));
e62a538d 2033
4c174e63
SG		 2034 if (ctrl->ctrl.state != NVME_CTRL_LIVE) {
2035 /*
0475a8dc
SG		 2036 * If we are resetting, connecting or deleting we should
2037 * complete immediately because we may block controller
2038 * teardown or setup sequence
2039 * - ctrl disable/shutdown fabrics requests
2040 * - connect requests
2041 * - initialization admin requests
2042 * - I/O requests that entered after unquiescing and
2043 * the controller stopped responding
2044 *
2045 * All other requests should be cancelled by the error
2046 * recovery work, so it's fine that we fail it here.
4c174e63 2047 */
0475a8dc 2048 nvme_rdma_complete_timed_out(rq);
4c174e63
SG		 2049 return BLK_EH_DONE;
2050 }
71102307 2051
0475a8dc
SG		 2052 /*
2053 * LIVE state should trigger the normal error recovery which will
2054 * handle completing this request.
2055 */
4c174e63 2056 nvme_rdma_error_recovery(ctrl);
4c174e63 2057 return BLK_EH_RESET_TIMER;
71102307
CH		 2058}
2059
fc17b653 2060static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx,
71102307
CH		 2061 const struct blk_mq_queue_data *bd)
2062{
2063 struct nvme_ns *ns = hctx->queue->queuedata;
2064 struct nvme_rdma_queue *queue = hctx->driver_data;
2065 struct request *rq = bd->rq;
2066 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
2067 struct nvme_rdma_qe *sqe = &req->sqe;
f4b9e6c9 2068 struct nvme_command *c = nvme_req(rq)->cmd;
71102307 2069 struct ib_device *dev;
3bc32bb1 2070 bool queue_ready = test_bit(NVME_RDMA_Q_LIVE, &queue->flags);
fc17b653
CH		 2071 blk_status_t ret;
2072 int err;
71102307
CH		 2073
2074 WARN_ON_ONCE(rq->tag < 0);
2075
a9715744
TC		 2076 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2077 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
553cd9ef 2078
71102307 2079 dev = queue->device->dev;
62f99b62
MG		 2080
2081 req->sqe.dma = ib_dma_map_single(dev, req->sqe.data,
2082 sizeof(struct nvme_command),
2083 DMA_TO_DEVICE);
2084 err = ib_dma_mapping_error(dev, req->sqe.dma);
2085 if (unlikely(err))
2086 return BLK_STS_RESOURCE;
2087
71102307
CH		 2088 ib_dma_sync_single_for_cpu(dev, sqe->dma,
2089 sizeof(struct nvme_command), DMA_TO_DEVICE);
2090
f4b9e6c9 2091 ret = nvme_setup_cmd(ns, rq);
fc17b653 2092 if (ret)
62f99b62 2093 goto unmap_qe;
71102307 2094
71102307
CH		 2095 blk_mq_start_request(rq);
2096
5ec5d3bd
MG		 2097 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
2098 queue->pi_support &&
2099 (c->common.opcode == nvme_cmd_write ||
2100 c->common.opcode == nvme_cmd_read) &&
2101 nvme_ns_has_pi(ns))
2102 req->use_sig_mr = true;
2103 else
2104 req->use_sig_mr = false;
2105
fc17b653 2106 err = nvme_rdma_map_data(queue, rq, c);
a7b7c7a1 2107 if (unlikely(err < 0)) {
71102307 2108 dev_err(queue->ctrl->ctrl.device,
fc17b653 2109 "Failed to map data (%d)\n", err);
71102307
CH		 2110 goto err;
2111 }
2112
b4b591c8
SG		 2113 sqe->cqe.done = nvme_rdma_send_done;
2114
71102307
CH		 2115 ib_dma_sync_single_for_device(dev, sqe->dma,
2116 sizeof(struct nvme_command), DMA_TO_DEVICE);
2117
fc17b653 2118 err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge,
f41725bb 2119 req->mr ? &req->reg_wr.wr : NULL);
16686f3a
MG		 2120 if (unlikely(err))
2121 goto err_unmap;
71102307 2122
fc17b653 2123 return BLK_STS_OK;
62f99b62 2124
16686f3a
MG		 2125err_unmap:
2126 nvme_rdma_unmap_data(queue, rq);
71102307 2127err:
62eca397
CL		 2128 if (err == -EIO)
2129 ret = nvme_host_path_error(rq);
2130 else if (err == -ENOMEM || err == -EAGAIN)
62f99b62
MG		 2131 ret = BLK_STS_RESOURCE;
2132 else
2133 ret = BLK_STS_IOERR;
16686f3a 2134 nvme_cleanup_cmd(rq);
62f99b62
MG		 2135unmap_qe:
2136 ib_dma_unmap_single(dev, req->sqe.dma, sizeof(struct nvme_command),
2137 DMA_TO_DEVICE);
2138 return ret;
71102307
CH		 2139}
2140
5a72e899 2141static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
ff8519f9
SG		 2142{
2143 struct nvme_rdma_queue *queue = hctx->driver_data;
2144
2145 return ib_process_cq_direct(queue->ib_cq, -1);
2146}
2147
5ec5d3bd
MG		 2148static void nvme_rdma_check_pi_status(struct nvme_rdma_request *req)
2149{
2150 struct request *rq = blk_mq_rq_from_pdu(req);
2151 struct ib_mr_status mr_status;
2152 int ret;
2153
2154 ret = ib_check_mr_status(req->mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
2155 if (ret) {
2156 pr_err("ib_check_mr_status failed, ret %d\n", ret);
2157 nvme_req(rq)->status = NVME_SC_INVALID_PI;
2158 return;
2159 }
2160
2161 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
2162 switch (mr_status.sig_err.err_type) {
2163 case IB_SIG_BAD_GUARD:
2164 nvme_req(rq)->status = NVME_SC_GUARD_CHECK;
2165 break;
2166 case IB_SIG_BAD_REFTAG:
2167 nvme_req(rq)->status = NVME_SC_REFTAG_CHECK;
2168 break;
2169 case IB_SIG_BAD_APPTAG:
2170 nvme_req(rq)->status = NVME_SC_APPTAG_CHECK;
2171 break;
2172 }
2173 pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n",
2174 mr_status.sig_err.err_type, mr_status.sig_err.expected,
2175 mr_status.sig_err.actual);
2176 }
2177}
2178
71102307
CH		 2179static void nvme_rdma_complete_rq(struct request *rq)
2180{
2181 struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
62f99b62
MG		 2182 struct nvme_rdma_queue *queue = req->queue;
2183 struct ib_device *ibdev = queue->device->dev;
71102307 2184
5ec5d3bd
MG		 2185 if (req->use_sig_mr)
2186 nvme_rdma_check_pi_status(req);
2187
62f99b62
MG		 2188 nvme_rdma_unmap_data(queue, rq);
2189 ib_dma_unmap_single(ibdev, req->sqe.dma, sizeof(struct nvme_command),
2190 DMA_TO_DEVICE);
77f02a7a 2191 nvme_complete_rq(rq);
71102307
CH		 2192}
2193
0b36658c
SG		 2194static int nvme_rdma_map_queues(struct blk_mq_tag_set *set)
2195{
2196 struct nvme_rdma_ctrl *ctrl = set->driver_data;
5651cd3c 2197 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
0b36658c 2198
5651cd3c 2199 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
b65bb777 2200 /* separate read/write queues */
5651cd3c
SG		 2201 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2202 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2203 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2204 set->map[HCTX_TYPE_READ].nr_queues =
2205 ctrl->io_queues[HCTX_TYPE_READ];
b65bb777 2206 set->map[HCTX_TYPE_READ].queue_offset =
5651cd3c 2207 ctrl->io_queues[HCTX_TYPE_DEFAULT];
b65bb777 2208 } else {
5651cd3c
SG		 2209 /* shared read/write queues */
2210 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2211 ctrl->io_queues[HCTX_TYPE_DEFAULT];
2212 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2213 set->map[HCTX_TYPE_READ].nr_queues =
2214 ctrl->io_queues[HCTX_TYPE_DEFAULT];
b65bb777
SG		 2215 set->map[HCTX_TYPE_READ].queue_offset = 0;
2216 }
2217 blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_DEFAULT],
2218 ctrl->device->dev, 0);
2219 blk_mq_rdma_map_queues(&set->map[HCTX_TYPE_READ],
2220 ctrl->device->dev, 0);
ff8519f9 2221
5651cd3c
SG		 2222 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
2223 /* map dedicated poll queues only if we have queues left */
ff8519f9 2224 set->map[HCTX_TYPE_POLL].nr_queues =
b1064d3e 2225 ctrl->io_queues[HCTX_TYPE_POLL];
ff8519f9 2226 set->map[HCTX_TYPE_POLL].queue_offset =
5651cd3c
SG		 2227 ctrl->io_queues[HCTX_TYPE_DEFAULT] +
2228 ctrl->io_queues[HCTX_TYPE_READ];
ff8519f9
SG		 2229 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
2230 }
5651cd3c
SG		 2231
2232 dev_info(ctrl->ctrl.device,
2233 "mapped %d/%d/%d default/read/poll queues.\n",
2234 ctrl->io_queues[HCTX_TYPE_DEFAULT],
2235 ctrl->io_queues[HCTX_TYPE_READ],
2236 ctrl->io_queues[HCTX_TYPE_POLL]);
2237
b65bb777 2238 return 0;
0b36658c
SG		 2239}
2240
f363b089 2241static const struct blk_mq_ops nvme_rdma_mq_ops = {
71102307
CH		 2242 .queue_rq = nvme_rdma_queue_rq,
2243 .complete = nvme_rdma_complete_rq,
71102307
CH		 2244 .init_request = nvme_rdma_init_request,
2245 .exit_request = nvme_rdma_exit_request,
71102307 2246 .init_hctx = nvme_rdma_init_hctx,
71102307 2247 .timeout = nvme_rdma_timeout,
0b36658c 2248 .map_queues = nvme_rdma_map_queues,
ff8519f9 2249 .poll = nvme_rdma_poll,
71102307
CH		 2250};
2251
f363b089 2252static const struct blk_mq_ops nvme_rdma_admin_mq_ops = {
71102307
CH		 2253 .queue_rq = nvme_rdma_queue_rq,
2254 .complete = nvme_rdma_complete_rq,
385475ee
CH		 2255 .init_request = nvme_rdma_init_request,
2256 .exit_request = nvme_rdma_exit_request,
71102307
CH		 2257 .init_hctx = nvme_rdma_init_admin_hctx,
2258 .timeout = nvme_rdma_timeout,
2259};
2260
18398af2 2261static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
71102307 2262{
75862c72 2263 nvme_rdma_teardown_io_queues(ctrl, shutdown);
6ca1d902 2264 nvme_stop_admin_queue(&ctrl->ctrl);
18398af2 2265 if (shutdown)
71102307 2266 nvme_shutdown_ctrl(&ctrl->ctrl);
18398af2 2267 else
b5b05048 2268 nvme_disable_ctrl(&ctrl->ctrl);
75862c72 2269 nvme_rdma_teardown_admin_queue(ctrl, shutdown);
71102307
CH		 2270}
2271
c5017e85 2272static void nvme_rdma_delete_ctrl(struct nvme_ctrl *ctrl)
2461a8dd 2273{
e9bc2587 2274 nvme_rdma_shutdown_ctrl(to_rdma_ctrl(ctrl), true);
71102307
CH		 2275}
2276
71102307
CH		 2277static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
2278{
d86c4d8e
CH		 2279 struct nvme_rdma_ctrl *ctrl =
2280 container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work);
71102307 2281
d09f2b45 2282 nvme_stop_ctrl(&ctrl->ctrl);
18398af2 2283 nvme_rdma_shutdown_ctrl(ctrl, false);
71102307 2284
ad6a0a52 2285 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
d5bf4b7f
SG		 2286 /* state change failure should never happen */
2287 WARN_ON_ONCE(1);
2288 return;
2289 }
2290
c66e2998 2291 if (nvme_rdma_setup_ctrl(ctrl, false))
370ae6e4 2292 goto out_fail;
71102307 2293
71102307
CH		 2294 return;
2295
370ae6e4 2296out_fail:
8000d1fd
NC		 2297 ++ctrl->ctrl.nr_reconnects;
2298 nvme_rdma_reconnect_or_remove(ctrl);
71102307
CH		 2299}
2300
71102307
CH		 2301static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = {
2302 .name = "rdma",
2303 .module = THIS_MODULE,
5ec5d3bd 2304 .flags = NVME_F_FABRICS | NVME_F_METADATA_SUPPORTED,
71102307
CH		 2305 .reg_read32 = nvmf_reg_read32,
2306 .reg_read64 = nvmf_reg_read64,
2307 .reg_write32 = nvmf_reg_write32,
71102307
CH		 2308 .free_ctrl = nvme_rdma_free_ctrl,
2309 .submit_async_event = nvme_rdma_submit_async_event,
c5017e85 2310 .delete_ctrl = nvme_rdma_delete_ctrl,
71102307 2311 .get_address = nvmf_get_address,
f7f70f4a 2312 .stop_ctrl = nvme_rdma_stop_ctrl,
71102307
CH		 2313};
2314
36e835f2
JS		 2315/*
2316 * Fails a connection request if it matches an existing controller
2317 * (association) with the same tuple:
2318 * <Host NQN, Host ID, local address, remote address, remote port, SUBSYS NQN>
2319 *
2320 * if local address is not specified in the request, it will match an
2321 * existing controller with all the other parameters the same and no
2322 * local port address specified as well.
2323 *
2324 * The ports don't need to be compared as they are intrinsically
2325 * already matched by the port pointers supplied.
2326 */
2327static bool
2328nvme_rdma_existing_controller(struct nvmf_ctrl_options *opts)
2329{
2330 struct nvme_rdma_ctrl *ctrl;
2331 bool found = false;
2332
2333 mutex_lock(&nvme_rdma_ctrl_mutex);
2334 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
b7c7be6f 2335 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
36e835f2
JS		 2336 if (found)
2337 break;
2338 }
2339 mutex_unlock(&nvme_rdma_ctrl_mutex);
2340
2341 return found;
2342}
2343
71102307
CH		 2344static struct nvme_ctrl *nvme_rdma_create_ctrl(struct device *dev,
2345 struct nvmf_ctrl_options *opts)
2346{
2347 struct nvme_rdma_ctrl *ctrl;
2348 int ret;
2349 bool changed;
2350
2351 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2352 if (!ctrl)
2353 return ERR_PTR(-ENOMEM);
2354 ctrl->ctrl.opts = opts;
2355 INIT_LIST_HEAD(&ctrl->list);
2356
bb59b8e5
SG		 2357 if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2358 opts->trsvcid =
2359 kstrdup(__stringify(NVME_RDMA_IP_PORT), GFP_KERNEL);
2360 if (!opts->trsvcid) {
2361 ret = -ENOMEM;
2362 goto out_free_ctrl;
2363 }
2364 opts->mask |= NVMF_OPT_TRSVCID;
2365 }
0928f9b4
SG		 2366
2367 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
bb59b8e5 2368 opts->traddr, opts->trsvcid, &ctrl->addr);
71102307 2369 if (ret) {
bb59b8e5
SG		 2370 pr_err("malformed address passed: %s:%s\n",
2371 opts->traddr, opts->trsvcid);
71102307
CH		 2372 goto out_free_ctrl;
2373 }
2374
8f4e8dac 2375 if (opts->mask & NVMF_OPT_HOST_TRADDR) {
0928f9b4
SG		 2376 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2377 opts->host_traddr, NULL, &ctrl->src_addr);
8f4e8dac 2378 if (ret) {
0928f9b4 2379 pr_err("malformed src address passed: %s\n",
8f4e8dac
MG		 2380 opts->host_traddr);
2381 goto out_free_ctrl;
2382 }
2383 }
2384
36e835f2
JS		 2385 if (!opts->duplicate_connect && nvme_rdma_existing_controller(opts)) {
2386 ret = -EALREADY;
2387 goto out_free_ctrl;
2388 }
2389
71102307
CH		 2390 INIT_DELAYED_WORK(&ctrl->reconnect_work,
2391 nvme_rdma_reconnect_ctrl_work);
2392 INIT_WORK(&ctrl->err_work, nvme_rdma_error_recovery_work);
d86c4d8e 2393 INIT_WORK(&ctrl->ctrl.reset_work, nvme_rdma_reset_ctrl_work);
71102307 2394
ff8519f9
SG		 2395 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2396 opts->nr_poll_queues + 1;
c5af8654 2397 ctrl->ctrl.sqsize = opts->queue_size - 1;
71102307
CH		 2398 ctrl->ctrl.kato = opts->kato;
2399
2400 ret = -ENOMEM;
d858e5f0 2401 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
71102307
CH		 2402 GFP_KERNEL);
2403 if (!ctrl->queues)
3d064101
SG		 2404 goto out_free_ctrl;
2405
2406 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
2407 0 /* no quirks, we're perfect! */);
2408 if (ret)
2409 goto out_kfree_queues;
71102307 2410
b754a32c
MG		 2411 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING);
2412 WARN_ON_ONCE(!changed);
2413
c66e2998 2414 ret = nvme_rdma_setup_ctrl(ctrl, true);
71102307 2415 if (ret)
3d064101 2416 goto out_uninit_ctrl;
71102307 2417
0928f9b4 2418 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISpcs\n",
e5ea42fa 2419 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr);
71102307 2420
71102307
CH		 2421 mutex_lock(&nvme_rdma_ctrl_mutex);
2422 list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
2423 mutex_unlock(&nvme_rdma_ctrl_mutex);
2424
71102307
CH		 2425 return &ctrl->ctrl;
2426
71102307
CH		 2427out_uninit_ctrl:
2428 nvme_uninit_ctrl(&ctrl->ctrl);
2429 nvme_put_ctrl(&ctrl->ctrl);
2430 if (ret > 0)
2431 ret = -EIO;
2432 return ERR_PTR(ret);
3d064101
SG		 2433out_kfree_queues:
2434 kfree(ctrl->queues);
71102307
CH		 2435out_free_ctrl:
2436 kfree(ctrl);
2437 return ERR_PTR(ret);
2438}
2439
2440static struct nvmf_transport_ops nvme_rdma_transport = {
2441 .name = "rdma",
0de5cd36 2442 .module = THIS_MODULE,
71102307 2443 .required_opts = NVMF_OPT_TRADDR,
8f4e8dac 2444 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
b65bb777 2445 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
e63440d6
IR		 2446 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2447 NVMF_OPT_TOS,
71102307
CH		 2448 .create_ctrl = nvme_rdma_create_ctrl,
2449};
2450
e87a911f
SW		 2451static void nvme_rdma_remove_one(struct ib_device *ib_device, void *client_data)
2452{
2453 struct nvme_rdma_ctrl *ctrl;
9bad0404
MG		 2454 struct nvme_rdma_device *ndev;
2455 bool found = false;
2456
2457 mutex_lock(&device_list_mutex);
2458 list_for_each_entry(ndev, &device_list, entry) {
2459 if (ndev->dev == ib_device) {
2460 found = true;
2461 break;
2462 }
2463 }
2464 mutex_unlock(&device_list_mutex);
2465
2466 if (!found)
2467 return;
e87a911f
SW		 2468
2469 /* Delete all controllers using this device */
2470 mutex_lock(&nvme_rdma_ctrl_mutex);
2471 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list) {
2472 if (ctrl->device->dev != ib_device)
2473 continue;
c5017e85 2474 nvme_delete_ctrl(&ctrl->ctrl);
e87a911f
SW		 2475 }
2476 mutex_unlock(&nvme_rdma_ctrl_mutex);
2477
b227c59b 2478 flush_workqueue(nvme_delete_wq);
e87a911f
SW		 2479}
2480
2481static struct ib_client nvme_rdma_ib_client = {
2482 .name = "nvme_rdma",
e87a911f
SW		 2483 .remove = nvme_rdma_remove_one
2484};
2485
71102307
CH		 2486static int __init nvme_rdma_init_module(void)
2487{
e87a911f
SW		 2488 int ret;
2489
e87a911f 2490 ret = ib_register_client(&nvme_rdma_ib_client);
a56c79cf 2491 if (ret)
9a6327d2 2492 return ret;
a56c79cf
SG		 2493
2494 ret = nvmf_register_transport(&nvme_rdma_transport);
2495 if (ret)
2496 goto err_unreg_client;
e87a911f 2497
a56c79cf 2498 return 0;
e87a911f 2499
a56c79cf
SG		 2500err_unreg_client:
2501 ib_unregister_client(&nvme_rdma_ib_client);
a56c79cf 2502 return ret;
71102307
CH		 2503}
2504
2505static void __exit nvme_rdma_cleanup_module(void)
2506{
9ad9e8d6
MG		 2507 struct nvme_rdma_ctrl *ctrl;
2508
71102307 2509 nvmf_unregister_transport(&nvme_rdma_transport);
e87a911f 2510 ib_unregister_client(&nvme_rdma_ib_client);
9ad9e8d6
MG		 2511
2512 mutex_lock(&nvme_rdma_ctrl_mutex);
2513 list_for_each_entry(ctrl, &nvme_rdma_ctrl_list, list)
2514 nvme_delete_ctrl(&ctrl->ctrl);
2515 mutex_unlock(&nvme_rdma_ctrl_mutex);
2516 flush_workqueue(nvme_delete_wq);
71102307
CH		 2517}
2518
2519module_init(nvme_rdma_init_module);
2520module_exit(nvme_rdma_cleanup_module);
2521
2522MODULE_LICENSE("GPL v2");
Linux 6.x block layer and io_uring tree(s)