projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ksmbd: remove unneeded NULL check in for_each_netdev
[linux-block.git] / fs / ksmbd / transport_rdma.c
CommitLineData
0626e664
NJ		 1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2017, Microsoft Corporation.
4 * Copyright (C) 2018, LG Electronics.
5 *
6 * Author(s): Long Li <longli@microsoft.com>,
7 * Hyunchul Lee <hyc.lee@gmail.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
17 * the GNU General Public License for more details.
18 */
19
20#define SUBMOD_NAME "smb_direct"
21
22#include <linux/kthread.h>
23#include <linux/rwlock.h>
24#include <linux/list.h>
25#include <linux/mempool.h>
26#include <linux/highmem.h>
27#include <linux/scatterlist.h>
28#include <rdma/ib_verbs.h>
29#include <rdma/rdma_cm.h>
30#include <rdma/rw.h>
31
32#include "glob.h"
33#include "connection.h"
34#include "smb_common.h"
35#include "smbstatus.h"
0626e664
NJ		 36#include "transport_rdma.h"
37
38#define SMB_DIRECT_PORT 5445
39
40#define SMB_DIRECT_VERSION_LE cpu_to_le16(0x0100)
41
42/* SMB_DIRECT negotiation timeout in seconds */
43#define SMB_DIRECT_NEGOTIATE_TIMEOUT 120
44
45#define SMB_DIRECT_MAX_SEND_SGES 8
46#define SMB_DIRECT_MAX_RECV_SGES 1
47
48/*
49 * Default maximum number of RDMA read/write outstanding on this connection
50 * This value is possibly decreased during QP creation on hardware limit
51 */
52#define SMB_DIRECT_CM_INITIATOR_DEPTH 8
53
54/* Maximum number of retries on data transfer operations */
55#define SMB_DIRECT_CM_RETRY 6
56/* No need to retry on Receiver Not Ready since SMB_DIRECT manages credits */
57#define SMB_DIRECT_CM_RNR_RETRY 0
58
59/*
60 * User configurable initial values per SMB_DIRECT transport connection
61 * as defined in [MS-KSMBD] 3.1.1.1
62 * Those may change after a SMB_DIRECT negotiation
63 */
64/* The local peer's maximum number of credits to grant to the peer */
65static int smb_direct_receive_credit_max = 255;
66
67/* The remote peer's credit request of local peer */
68static int smb_direct_send_credit_target = 255;
69
70/* The maximum single message size can be sent to remote peer */
71static int smb_direct_max_send_size = 8192;
72
73/* The maximum fragmented upper-layer payload receive size supported */
74static int smb_direct_max_fragmented_recv_size = 1024 * 1024;
75
76/* The maximum single-message size which can be received */
77static int smb_direct_max_receive_size = 8192;
78
79static int smb_direct_max_read_write_size = 1024 * 1024;
80
81static int smb_direct_max_outstanding_rw_ops = 8;
82
83static struct smb_direct_listener {
84 struct rdma_cm_id *cm_id;
85} smb_direct_listener;
86
0626e664
NJ		 87static struct workqueue_struct *smb_direct_wq;
88
89enum smb_direct_status {
90 SMB_DIRECT_CS_NEW = 0,
91 SMB_DIRECT_CS_CONNECTED,
92 SMB_DIRECT_CS_DISCONNECTING,
93 SMB_DIRECT_CS_DISCONNECTED,
94};
95
96struct smb_direct_transport {
97 struct ksmbd_transport transport;
98
99 enum smb_direct_status status;
100 bool full_packet_received;
101 wait_queue_head_t wait_status;
102
103 struct rdma_cm_id *cm_id;
104 struct ib_cq *send_cq;
105 struct ib_cq *recv_cq;
106 struct ib_pd *pd;
107 struct ib_qp *qp;
108
109 int max_send_size;
110 int max_recv_size;
111 int max_fragmented_send_size;
112 int max_fragmented_recv_size;
113 int max_rdma_rw_size;
114
115 spinlock_t reassembly_queue_lock;
116 struct list_head reassembly_queue;
117 int reassembly_data_length;
118 int reassembly_queue_length;
119 int first_entry_offset;
120 wait_queue_head_t wait_reassembly_queue;
121
122 spinlock_t receive_credit_lock;
123 int recv_credits;
124 int count_avail_recvmsg;
125 int recv_credit_max;
126 int recv_credit_target;
127
128 spinlock_t recvmsg_queue_lock;
129 struct list_head recvmsg_queue;
130
131 spinlock_t empty_recvmsg_queue_lock;
132 struct list_head empty_recvmsg_queue;
133
134 int send_credit_target;
135 atomic_t send_credits;
136 spinlock_t lock_new_recv_credits;
137 int new_recv_credits;
138 atomic_t rw_avail_ops;
139
140 wait_queue_head_t wait_send_credits;
141 wait_queue_head_t wait_rw_avail_ops;
142
143 mempool_t *sendmsg_mempool;
144 struct kmem_cache *sendmsg_cache;
145 mempool_t *recvmsg_mempool;
146 struct kmem_cache *recvmsg_cache;
147
148 wait_queue_head_t wait_send_payload_pending;
149 atomic_t send_payload_pending;
150 wait_queue_head_t wait_send_pending;
151 atomic_t send_pending;
152
153 struct delayed_work post_recv_credits_work;
154 struct work_struct send_immediate_work;
155 struct work_struct disconnect_work;
156
157 bool negotiation_requested;
158};
159
160#define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport))
0626e664
NJ		 161
162enum {
163 SMB_DIRECT_MSG_NEGOTIATE_REQ = 0,
164 SMB_DIRECT_MSG_DATA_TRANSFER
165};
166
167static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops;
168
169struct smb_direct_send_ctx {
170 struct list_head msg_list;
171 int wr_cnt;
172 bool need_invalidate_rkey;
173 unsigned int remote_key;
174};
175
176struct smb_direct_sendmsg {
177 struct smb_direct_transport *transport;
178 struct ib_send_wr wr;
179 struct list_head list;
180 int num_sge;
181 struct ib_sge sge[SMB_DIRECT_MAX_SEND_SGES];
182 struct ib_cqe cqe;
183 u8 packet[];
184};
185
186struct smb_direct_recvmsg {
187 struct smb_direct_transport *transport;
188 struct list_head list;
189 int type;
190 struct ib_sge sge;
191 struct ib_cqe cqe;
192 bool first_segment;
193 u8 packet[];
194};
195
196struct smb_direct_rdma_rw_msg {
197 struct smb_direct_transport *t;
198 struct ib_cqe cqe;
199 struct completion *completion;
200 struct rdma_rw_ctx rw_ctx;
201 struct sg_table sgt;
202 struct scatterlist sg_list[0];
203};
204
8ad8dc34
HL		 205static inline int get_buf_page_count(void *buf, int size)
206{
207 return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) -
208 (uintptr_t)buf / PAGE_SIZE;
209}
0626e664
NJ		 210
211static void smb_direct_destroy_pools(struct smb_direct_transport *transport);
212static void smb_direct_post_recv_credits(struct work_struct *work);
213static int smb_direct_post_send_data(struct smb_direct_transport *t,
070fb21e
NJ		 214 struct smb_direct_send_ctx *send_ctx,
215 struct kvec *iov, int niov,
216 int remaining_data_length);
0626e664 217
02d4b4aa
NJ		 218static inline struct smb_direct_transport *
219smb_trans_direct_transfort(struct ksmbd_transport *t)
220{
221 return container_of(t, struct smb_direct_transport, transport);
222}
223
0626e664
NJ		 224static inline void
225*smb_direct_recvmsg_payload(struct smb_direct_recvmsg *recvmsg)
226{
227 return (void *)recvmsg->packet;
228}
229
230static inline bool is_receive_credit_post_required(int receive_credits,
070fb21e 231 int avail_recvmsg_count)
0626e664
NJ		 232{
233 return receive_credits <= (smb_direct_receive_credit_max >> 3) &&
234 avail_recvmsg_count >= (receive_credits >> 2);
235}
236
237static struct
238smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t)
239{
240 struct smb_direct_recvmsg *recvmsg = NULL;
241
242 spin_lock(&t->recvmsg_queue_lock);
243 if (!list_empty(&t->recvmsg_queue)) {
244 recvmsg = list_first_entry(&t->recvmsg_queue,
245 struct smb_direct_recvmsg,
246 list);
247 list_del(&recvmsg->list);
248 }
249 spin_unlock(&t->recvmsg_queue_lock);
250 return recvmsg;
251}
252
253static void put_recvmsg(struct smb_direct_transport *t,
070fb21e 254 struct smb_direct_recvmsg *recvmsg)
0626e664
NJ		 255{
256 ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
070fb21e 257 recvmsg->sge.length, DMA_FROM_DEVICE);
0626e664
NJ		 258
259 spin_lock(&t->recvmsg_queue_lock);
260 list_add(&recvmsg->list, &t->recvmsg_queue);
261 spin_unlock(&t->recvmsg_queue_lock);
0626e664
NJ		 262}
263
264static struct
265smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t)
266{
267 struct smb_direct_recvmsg *recvmsg = NULL;
268
269 spin_lock(&t->empty_recvmsg_queue_lock);
270 if (!list_empty(&t->empty_recvmsg_queue)) {
64b39f4a 271 recvmsg = list_first_entry(&t->empty_recvmsg_queue,
070fb21e 272 struct smb_direct_recvmsg, list);
0626e664
NJ		 273 list_del(&recvmsg->list);
274 }
275 spin_unlock(&t->empty_recvmsg_queue_lock);
276 return recvmsg;
277}
278
279static void put_empty_recvmsg(struct smb_direct_transport *t,
070fb21e 280 struct smb_direct_recvmsg *recvmsg)
0626e664
NJ		 281{
282 ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr,
070fb21e 283 recvmsg->sge.length, DMA_FROM_DEVICE);
0626e664
NJ		 284
285 spin_lock(&t->empty_recvmsg_queue_lock);
286 list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue);
287 spin_unlock(&t->empty_recvmsg_queue_lock);
288}
289
290static void enqueue_reassembly(struct smb_direct_transport *t,
070fb21e
NJ		 291 struct smb_direct_recvmsg *recvmsg,
292 int data_length)
0626e664
NJ		 293{
294 spin_lock(&t->reassembly_queue_lock);
295 list_add_tail(&recvmsg->list, &t->reassembly_queue);
296 t->reassembly_queue_length++;
297 /*
298 * Make sure reassembly_data_length is updated after list and
299 * reassembly_queue_length are updated. On the dequeue side
300 * reassembly_data_length is checked without a lock to determine
301 * if reassembly_queue_length and list is up to date
302 */
303 virt_wmb();
304 t->reassembly_data_length += data_length;
305 spin_unlock(&t->reassembly_queue_lock);
0626e664
NJ		 306}
307
64b39f4a 308static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t)
0626e664
NJ		 309{
310 if (!list_empty(&t->reassembly_queue))
311 return list_first_entry(&t->reassembly_queue,
312 struct smb_direct_recvmsg, list);
313 else
314 return NULL;
315}
316
317static void smb_direct_disconnect_rdma_work(struct work_struct *work)
318{
319 struct smb_direct_transport *t =
320 container_of(work, struct smb_direct_transport,
321 disconnect_work);
322
323 if (t->status == SMB_DIRECT_CS_CONNECTED) {
324 t->status = SMB_DIRECT_CS_DISCONNECTING;
325 rdma_disconnect(t->cm_id);
326 }
327}
328
329static void
330smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t)
331{
332 queue_work(smb_direct_wq, &t->disconnect_work);
333}
334
335static void smb_direct_send_immediate_work(struct work_struct *work)
336{
337 struct smb_direct_transport *t = container_of(work,
338 struct smb_direct_transport, send_immediate_work);
339
340 if (t->status != SMB_DIRECT_CS_CONNECTED)
341 return;
342
343 smb_direct_post_send_data(t, NULL, NULL, 0, 0);
344}
345
346static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
347{
348 struct smb_direct_transport *t;
349 struct ksmbd_conn *conn;
350
351 t = kzalloc(sizeof(*t), GFP_KERNEL);
352 if (!t)
353 return NULL;
354
355 t->cm_id = cm_id;
356 cm_id->context = t;
357
358 t->status = SMB_DIRECT_CS_NEW;
359 init_waitqueue_head(&t->wait_status);
360
361 spin_lock_init(&t->reassembly_queue_lock);
362 INIT_LIST_HEAD(&t->reassembly_queue);
363 t->reassembly_data_length = 0;
364 t->reassembly_queue_length = 0;
365 init_waitqueue_head(&t->wait_reassembly_queue);
366 init_waitqueue_head(&t->wait_send_credits);
367 init_waitqueue_head(&t->wait_rw_avail_ops);
368
369 spin_lock_init(&t->receive_credit_lock);
370 spin_lock_init(&t->recvmsg_queue_lock);
371 INIT_LIST_HEAD(&t->recvmsg_queue);
372
373 spin_lock_init(&t->empty_recvmsg_queue_lock);
374 INIT_LIST_HEAD(&t->empty_recvmsg_queue);
375
376 init_waitqueue_head(&t->wait_send_payload_pending);
377 atomic_set(&t->send_payload_pending, 0);
378 init_waitqueue_head(&t->wait_send_pending);
379 atomic_set(&t->send_pending, 0);
380
381 spin_lock_init(&t->lock_new_recv_credits);
382
383 INIT_DELAYED_WORK(&t->post_recv_credits_work,
384 smb_direct_post_recv_credits);
385 INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work);
386 INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work);
387
388 conn = ksmbd_conn_alloc();
389 if (!conn)
390 goto err;
391 conn->transport = KSMBD_TRANS(t);
392 KSMBD_TRANS(t)->conn = conn;
393 KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops;
394 return t;
395err:
396 kfree(t);
397 return NULL;
398}
399
400static void free_transport(struct smb_direct_transport *t)
401{
402 struct smb_direct_recvmsg *recvmsg;
403
404 wake_up_interruptible(&t->wait_send_credits);
405
406 ksmbd_debug(RDMA, "wait for all send posted to IB to finish\n");
407 wait_event(t->wait_send_payload_pending,
070fb21e 408 atomic_read(&t->send_payload_pending) == 0);
0626e664 409 wait_event(t->wait_send_pending,
070fb21e 410 atomic_read(&t->send_pending) == 0);
0626e664
NJ		 411
412 cancel_work_sync(&t->disconnect_work);
413 cancel_delayed_work_sync(&t->post_recv_credits_work);
414 cancel_work_sync(&t->send_immediate_work);
415
416 if (t->qp) {
417 ib_drain_qp(t->qp);
418 ib_destroy_qp(t->qp);
419 }
420
421 ksmbd_debug(RDMA, "drain the reassembly queue\n");
422 do {
423 spin_lock(&t->reassembly_queue_lock);
424 recvmsg = get_first_reassembly(t);
425 if (recvmsg) {
426 list_del(&recvmsg->list);
64b39f4a 427 spin_unlock(&t->reassembly_queue_lock);
0626e664 428 put_recvmsg(t, recvmsg);
64b39f4a 429 } else {
0626e664 430 spin_unlock(&t->reassembly_queue_lock);
64b39f4a 431 }
0626e664
NJ		 432 } while (recvmsg);
433 t->reassembly_data_length = 0;
434
435 if (t->send_cq)
436 ib_free_cq(t->send_cq);
437 if (t->recv_cq)
438 ib_free_cq(t->recv_cq);
439 if (t->pd)
440 ib_dealloc_pd(t->pd);
441 if (t->cm_id)
442 rdma_destroy_id(t->cm_id);
443
444 smb_direct_destroy_pools(t);
445 ksmbd_conn_free(KSMBD_TRANS(t)->conn);
446 kfree(t);
447}
448
449static struct smb_direct_sendmsg
450*smb_direct_alloc_sendmsg(struct smb_direct_transport *t)
451{
452 struct smb_direct_sendmsg *msg;
453
454 msg = mempool_alloc(t->sendmsg_mempool, GFP_KERNEL);
455 if (!msg)
456 return ERR_PTR(-ENOMEM);
457 msg->transport = t;
458 INIT_LIST_HEAD(&msg->list);
459 msg->num_sge = 0;
460 return msg;
461}
462
463static void smb_direct_free_sendmsg(struct smb_direct_transport *t,
070fb21e 464 struct smb_direct_sendmsg *msg)
0626e664
NJ		 465{
466 int i;
467
468 if (msg->num_sge > 0) {
469 ib_dma_unmap_single(t->cm_id->device,
070fb21e
NJ		 470 msg->sge[0].addr, msg->sge[0].length,
471 DMA_TO_DEVICE);
0626e664
NJ		 472 for (i = 1; i < msg->num_sge; i++)
473 ib_dma_unmap_page(t->cm_id->device,
070fb21e
NJ		 474 msg->sge[i].addr, msg->sge[i].length,
475 DMA_TO_DEVICE);
0626e664
NJ		 476 }
477 mempool_free(msg, t->sendmsg_mempool);
478}
479
480static int smb_direct_check_recvmsg(struct smb_direct_recvmsg *recvmsg)
481{
482 switch (recvmsg->type) {
483 case SMB_DIRECT_MSG_DATA_TRANSFER: {
484 struct smb_direct_data_transfer *req =
64b39f4a
NJ		 485 (struct smb_direct_data_transfer *)recvmsg->packet;
486 struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet
0626e664
NJ		 487 + le32_to_cpu(req->data_offset) - 4);
488 ksmbd_debug(RDMA,
070fb21e
NJ		 489 "CreditGranted: %u, CreditRequested: %u, DataLength: %u, RemainingDataLength: %u, SMB: %x, Command: %u\n",
490 le16_to_cpu(req->credits_granted),
491 le16_to_cpu(req->credits_requested),
492 req->data_length, req->remaining_data_length,
493 hdr->ProtocolId, hdr->Command);
0626e664
NJ		 494 break;
495 }
496 case SMB_DIRECT_MSG_NEGOTIATE_REQ: {
497 struct smb_direct_negotiate_req *req =
498 (struct smb_direct_negotiate_req *)recvmsg->packet;
499 ksmbd_debug(RDMA,
070fb21e
NJ		 500 "MinVersion: %u, MaxVersion: %u, CreditRequested: %u, MaxSendSize: %u, MaxRecvSize: %u, MaxFragmentedSize: %u\n",
501 le16_to_cpu(req->min_version),
502 le16_to_cpu(req->max_version),
503 le16_to_cpu(req->credits_requested),
504 le32_to_cpu(req->preferred_send_size),
505 le32_to_cpu(req->max_receive_size),
506 le32_to_cpu(req->max_fragmented_size));
0626e664 507 if (le16_to_cpu(req->min_version) > 0x0100 ||
64b39f4a 508 le16_to_cpu(req->max_version) < 0x0100)
0626e664
NJ		 509 return -EOPNOTSUPP;
510 if (le16_to_cpu(req->credits_requested) <= 0 ||
64b39f4a
NJ		 511 le32_to_cpu(req->max_receive_size) <= 128 ||
512 le32_to_cpu(req->max_fragmented_size) <=
513 128 * 1024)
0626e664
NJ		 514 return -ECONNABORTED;
515
516 break;
517 }
518 default:
519 return -EINVAL;
520 }
521 return 0;
522}
523
524static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
525{
526 struct smb_direct_recvmsg *recvmsg;
527 struct smb_direct_transport *t;
528
529 recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe);
530 t = recvmsg->transport;
531
532 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) {
533 if (wc->status != IB_WC_WR_FLUSH_ERR) {
bde1694a
NJ		 534 pr_err("Recv error. status='%s (%d)' opcode=%d\n",
535 ib_wc_status_msg(wc->status), wc->status,
536 wc->opcode);
0626e664
NJ		 537 smb_direct_disconnect_rdma_connection(t);
538 }
539 put_empty_recvmsg(t, recvmsg);
540 return;
541 }
542
543 ksmbd_debug(RDMA, "Recv completed. status='%s (%d)', opcode=%d\n",
070fb21e
NJ		 544 ib_wc_status_msg(wc->status), wc->status,
545 wc->opcode);
0626e664
NJ		 546
547 ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr,
070fb21e 548 recvmsg->sge.length, DMA_FROM_DEVICE);
0626e664
NJ		 549
550 switch (recvmsg->type) {
551 case SMB_DIRECT_MSG_NEGOTIATE_REQ:
552 t->negotiation_requested = true;
553 t->full_packet_received = true;
554 wake_up_interruptible(&t->wait_status);
555 break;
556 case SMB_DIRECT_MSG_DATA_TRANSFER: {
557 struct smb_direct_data_transfer *data_transfer =
558 (struct smb_direct_data_transfer *)recvmsg->packet;
559 int data_length = le32_to_cpu(data_transfer->data_length);
560 int avail_recvmsg_count, receive_credits;
561
562 if (data_length) {
563 if (t->full_packet_received)
564 recvmsg->first_segment = true;
565
566 if (le32_to_cpu(data_transfer->remaining_data_length))
567 t->full_packet_received = false;
568 else
569 t->full_packet_received = true;
570
571 enqueue_reassembly(t, recvmsg, data_length);
572 wake_up_interruptible(&t->wait_reassembly_queue);
573
574 spin_lock(&t->receive_credit_lock);
575 receive_credits = --(t->recv_credits);
576 avail_recvmsg_count = t->count_avail_recvmsg;
577 spin_unlock(&t->receive_credit_lock);
578 } else {
579 put_empty_recvmsg(t, recvmsg);
580
581 spin_lock(&t->receive_credit_lock);
582 receive_credits = --(t->recv_credits);
583 avail_recvmsg_count = ++(t->count_avail_recvmsg);
584 spin_unlock(&t->receive_credit_lock);
585 }
586
587 t->recv_credit_target =
588 le16_to_cpu(data_transfer->credits_requested);
589 atomic_add(le16_to_cpu(data_transfer->credits_granted),
070fb21e 590 &t->send_credits);
0626e664
NJ		 591
592 if (le16_to_cpu(data_transfer->flags) &
070fb21e 593 SMB_DIRECT_RESPONSE_REQUESTED)
0626e664
NJ		 594 queue_work(smb_direct_wq, &t->send_immediate_work);
595
596 if (atomic_read(&t->send_credits) > 0)
597 wake_up_interruptible(&t->wait_send_credits);
598
64b39f4a 599 if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count))
0626e664 600 mod_delayed_work(smb_direct_wq,
070fb21e 601 &t->post_recv_credits_work, 0);
0626e664
NJ		 602 break;
603 }
604 default:
605 break;
606 }
607}
608
609static int smb_direct_post_recv(struct smb_direct_transport *t,
070fb21e 610 struct smb_direct_recvmsg *recvmsg)
0626e664
NJ		 611{
612 struct ib_recv_wr wr;
613 int ret;
614
615 recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device,
070fb21e
NJ		 616 recvmsg->packet, t->max_recv_size,
617 DMA_FROM_DEVICE);
0626e664
NJ		 618 ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr);
619 if (ret)
620 return ret;
621 recvmsg->sge.length = t->max_recv_size;
622 recvmsg->sge.lkey = t->pd->local_dma_lkey;
623 recvmsg->cqe.done = recv_done;
624
625 wr.wr_cqe = &recvmsg->cqe;
626 wr.next = NULL;
627 wr.sg_list = &recvmsg->sge;
628 wr.num_sge = 1;
629
630 ret = ib_post_recv(t->qp, &wr, NULL);
631 if (ret) {
bde1694a 632 pr_err("Can't post recv: %d\n", ret);
0626e664 633 ib_dma_unmap_single(t->cm_id->device,
070fb21e
NJ		 634 recvmsg->sge.addr, recvmsg->sge.length,
635 DMA_FROM_DEVICE);
0626e664
NJ		 636 smb_direct_disconnect_rdma_connection(t);
637 return ret;
638 }
639 return ret;
640}
641
642static int smb_direct_read(struct ksmbd_transport *t, char *buf,
070fb21e 643 unsigned int size)
0626e664
NJ		 644{
645 struct smb_direct_recvmsg *recvmsg;
646 struct smb_direct_data_transfer *data_transfer;
647 int to_copy, to_read, data_read, offset;
648 u32 data_length, remaining_data_length, data_offset;
649 int rc;
02d4b4aa 650 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
0626e664
NJ		 651
652again:
653 if (st->status != SMB_DIRECT_CS_CONNECTED) {
bde1694a 654 pr_err("disconnected\n");
0626e664
NJ		 655 return -ENOTCONN;
656 }
657
658 /*
659 * No need to hold the reassembly queue lock all the time as we are
660 * the only one reading from the front of the queue. The transport
661 * may add more entries to the back of the queue at the same time
662 */
663 if (st->reassembly_data_length >= size) {
664 int queue_length;
665 int queue_removed = 0;
666
667 /*
668 * Need to make sure reassembly_data_length is read before
669 * reading reassembly_queue_length and calling
670 * get_first_reassembly. This call is lock free
671 * as we never read at the end of the queue which are being
672 * updated in SOFTIRQ as more data is received
673 */
674 virt_rmb();
675 queue_length = st->reassembly_queue_length;
676 data_read = 0;
677 to_read = size;
678 offset = st->first_entry_offset;
679 while (data_read < size) {
680 recvmsg = get_first_reassembly(st);
681 data_transfer = smb_direct_recvmsg_payload(recvmsg);
682 data_length = le32_to_cpu(data_transfer->data_length);
683 remaining_data_length =
64b39f4a 684 le32_to_cpu(data_transfer->remaining_data_length);
0626e664
NJ		 685 data_offset = le32_to_cpu(data_transfer->data_offset);
686
687 /*
688 * The upper layer expects RFC1002 length at the
689 * beginning of the payload. Return it to indicate
690 * the total length of the packet. This minimize the
691 * change to upper layer packet processing logic. This
692 * will be eventually remove when an intermediate
693 * transport layer is added
694 */
695 if (recvmsg->first_segment && size == 4) {
696 unsigned int rfc1002_len =
697 data_length + remaining_data_length;
698 *((__be32 *)buf) = cpu_to_be32(rfc1002_len);
699 data_read = 4;
700 recvmsg->first_segment = false;
701 ksmbd_debug(RDMA,
070fb21e
NJ		 702 "returning rfc1002 length %d\n",
703 rfc1002_len);
0626e664
NJ		 704 goto read_rfc1002_done;
705 }
706
707 to_copy = min_t(int, data_length - offset, to_read);
64b39f4a 708 memcpy(buf + data_read, (char *)data_transfer + data_offset + offset,
070fb21e 709 to_copy);
0626e664
NJ		 710
711 /* move on to the next buffer? */
712 if (to_copy == data_length - offset) {
713 queue_length--;
714 /*
715 * No need to lock if we are not at the
716 * end of the queue
717 */
64b39f4a 718 if (queue_length) {
0626e664 719 list_del(&recvmsg->list);
64b39f4a
NJ		 720 } else {
721 spin_lock_irq(&st->reassembly_queue_lock);
0626e664 722 list_del(&recvmsg->list);
64b39f4a 723 spin_unlock_irq(&st->reassembly_queue_lock);
0626e664
NJ		 724 }
725 queue_removed++;
726 put_recvmsg(st, recvmsg);
727 offset = 0;
64b39f4a 728 } else {
0626e664 729 offset += to_copy;
64b39f4a 730 }
0626e664
NJ		 731
732 to_read -= to_copy;
733 data_read += to_copy;
734 }
735
736 spin_lock_irq(&st->reassembly_queue_lock);
737 st->reassembly_data_length -= data_read;
738 st->reassembly_queue_length -= queue_removed;
739 spin_unlock_irq(&st->reassembly_queue_lock);
740
741 spin_lock(&st->receive_credit_lock);
742 st->count_avail_recvmsg += queue_removed;
64b39f4a 743 if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) {
0626e664
NJ		 744 spin_unlock(&st->receive_credit_lock);
745 mod_delayed_work(smb_direct_wq,
070fb21e 746 &st->post_recv_credits_work, 0);
64b39f4a 747 } else {
0626e664 748 spin_unlock(&st->receive_credit_lock);
64b39f4a 749 }
0626e664
NJ		 750
751 st->first_entry_offset = offset;
752 ksmbd_debug(RDMA,
070fb21e
NJ		 753 "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n",
754 data_read, st->reassembly_data_length,
755 st->first_entry_offset);
0626e664
NJ		 756read_rfc1002_done:
757 return data_read;
758 }
759
760 ksmbd_debug(RDMA, "wait_event on more data\n");
64b39f4a 761 rc = wait_event_interruptible(st->wait_reassembly_queue,
070fb21e
NJ		 762 st->reassembly_data_length >= size ||
763 st->status != SMB_DIRECT_CS_CONNECTED);
0626e664
NJ		 764 if (rc)
765 return -EINTR;
766
767 goto again;
768}
769
770static void smb_direct_post_recv_credits(struct work_struct *work)
771{
772 struct smb_direct_transport *t = container_of(work,
773 struct smb_direct_transport, post_recv_credits_work.work);
774 struct smb_direct_recvmsg *recvmsg;
775 int receive_credits, credits = 0;
776 int ret;
777 int use_free = 1;
778
779 spin_lock(&t->receive_credit_lock);
780 receive_credits = t->recv_credits;
781 spin_unlock(&t->receive_credit_lock);
782
783 if (receive_credits < t->recv_credit_target) {
784 while (true) {
785 if (use_free)
786 recvmsg = get_free_recvmsg(t);
787 else
788 recvmsg = get_empty_recvmsg(t);
789 if (!recvmsg) {
790 if (use_free) {
791 use_free = 0;
792 continue;
64b39f4a 793 } else {
0626e664 794 break;
64b39f4a 795 }
0626e664
NJ		 796 }
797
798 recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER;
799 recvmsg->first_segment = false;
800
801 ret = smb_direct_post_recv(t, recvmsg);
802 if (ret) {
bde1694a 803 pr_err("Can't post recv: %d\n", ret);
0626e664
NJ		 804 put_recvmsg(t, recvmsg);
805 break;
806 }
807 credits++;
808 }
809 }
810
811 spin_lock(&t->receive_credit_lock);
812 t->recv_credits += credits;
813 t->count_avail_recvmsg -= credits;
814 spin_unlock(&t->receive_credit_lock);
815
816 spin_lock(&t->lock_new_recv_credits);
817 t->new_recv_credits += credits;
818 spin_unlock(&t->lock_new_recv_credits);
819
820 if (credits)
821 queue_work(smb_direct_wq, &t->send_immediate_work);
822}
823
824static void send_done(struct ib_cq *cq, struct ib_wc *wc)
825{
826 struct smb_direct_sendmsg *sendmsg, *sibling;
827 struct smb_direct_transport *t;
828 struct list_head *pos, *prev, *end;
829
830 sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe);
831 t = sendmsg->transport;
832
833 ksmbd_debug(RDMA, "Send completed. status='%s (%d)', opcode=%d\n",
070fb21e
NJ		 834 ib_wc_status_msg(wc->status), wc->status,
835 wc->opcode);
0626e664
NJ		 836
837 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) {
bde1694a
NJ		 838 pr_err("Send error. status='%s (%d)', opcode=%d\n",
839 ib_wc_status_msg(wc->status), wc->status,
840 wc->opcode);
0626e664
NJ		 841 smb_direct_disconnect_rdma_connection(t);
842 }
843
844 if (sendmsg->num_sge > 1) {
845 if (atomic_dec_and_test(&t->send_payload_pending))
846 wake_up(&t->wait_send_payload_pending);
847 } else {
848 if (atomic_dec_and_test(&t->send_pending))
849 wake_up(&t->wait_send_pending);
850 }
851
852 /* iterate and free the list of messages in reverse. the list's head
853 * is invalid.
854 */
855 for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next;
070fb21e 856 prev != end; pos = prev, prev = prev->prev) {
0626e664
NJ		 857 sibling = container_of(pos, struct smb_direct_sendmsg, list);
858 smb_direct_free_sendmsg(t, sibling);
859 }
860
861 sibling = container_of(pos, struct smb_direct_sendmsg, list);
862 smb_direct_free_sendmsg(t, sibling);
863}
864
865static int manage_credits_prior_sending(struct smb_direct_transport *t)
866{
867 int new_credits;
868
869 spin_lock(&t->lock_new_recv_credits);
870 new_credits = t->new_recv_credits;
871 t->new_recv_credits = 0;
872 spin_unlock(&t->lock_new_recv_credits);
873
874 return new_credits;
875}
876
877static int smb_direct_post_send(struct smb_direct_transport *t,
070fb21e 878 struct ib_send_wr *wr)
0626e664
NJ		 879{
880 int ret;
881
882 if (wr->num_sge > 1)
883 atomic_inc(&t->send_payload_pending);
884 else
885 atomic_inc(&t->send_pending);
886
887 ret = ib_post_send(t->qp, wr, NULL);
888 if (ret) {
bde1694a 889 pr_err("failed to post send: %d\n", ret);
0626e664
NJ		 890 if (wr->num_sge > 1) {
891 if (atomic_dec_and_test(&t->send_payload_pending))
892 wake_up(&t->wait_send_payload_pending);
893 } else {
894 if (atomic_dec_and_test(&t->send_pending))
895 wake_up(&t->wait_send_pending);
896 }
897 smb_direct_disconnect_rdma_connection(t);
898 }
899 return ret;
900}
901
902static void smb_direct_send_ctx_init(struct smb_direct_transport *t,
070fb21e
NJ		 903 struct smb_direct_send_ctx *send_ctx,
904 bool need_invalidate_rkey,
905 unsigned int remote_key)
0626e664
NJ		 906{
907 INIT_LIST_HEAD(&send_ctx->msg_list);
908 send_ctx->wr_cnt = 0;
909 send_ctx->need_invalidate_rkey = need_invalidate_rkey;
910 send_ctx->remote_key = remote_key;
911}
912
913static int smb_direct_flush_send_list(struct smb_direct_transport *t,
070fb21e
NJ		 914 struct smb_direct_send_ctx *send_ctx,
915 bool is_last)
0626e664
NJ		 916{
917 struct smb_direct_sendmsg *first, *last;
918 int ret;
919
920 if (list_empty(&send_ctx->msg_list))
921 return 0;
922
923 first = list_first_entry(&send_ctx->msg_list,
070fb21e
NJ		 924 struct smb_direct_sendmsg,
925 list);
0626e664 926 last = list_last_entry(&send_ctx->msg_list,
070fb21e
NJ		 927 struct smb_direct_sendmsg,
928 list);
0626e664
NJ		 929
930 last->wr.send_flags = IB_SEND_SIGNALED;
931 last->wr.wr_cqe = &last->cqe;
932 if (is_last && send_ctx->need_invalidate_rkey) {
933 last->wr.opcode = IB_WR_SEND_WITH_INV;
934 last->wr.ex.invalidate_rkey = send_ctx->remote_key;
935 }
936
937 ret = smb_direct_post_send(t, &first->wr);
938 if (!ret) {
939 smb_direct_send_ctx_init(t, send_ctx,
070fb21e
NJ		 940 send_ctx->need_invalidate_rkey,
941 send_ctx->remote_key);
0626e664
NJ		 942 } else {
943 atomic_add(send_ctx->wr_cnt, &t->send_credits);
944 wake_up(&t->wait_send_credits);
945 list_for_each_entry_safe(first, last, &send_ctx->msg_list,
070fb21e 946 list) {
0626e664
NJ		 947 smb_direct_free_sendmsg(t, first);
948 }
949 }
950 return ret;
951}
952
953static int wait_for_credits(struct smb_direct_transport *t,
070fb21e 954 wait_queue_head_t *waitq, atomic_t *credits)
0626e664
NJ		 955{
956 int ret;
957
958 do {
959 if (atomic_dec_return(credits) >= 0)
960 return 0;
961
962 atomic_inc(credits);
963 ret = wait_event_interruptible(*waitq,
070fb21e
NJ		 964 atomic_read(credits) > 0 ||
965 t->status != SMB_DIRECT_CS_CONNECTED);
0626e664
NJ		 966
967 if (t->status != SMB_DIRECT_CS_CONNECTED)
968 return -ENOTCONN;
969 else if (ret < 0)
970 return ret;
971 } while (true);
972}
973
974static int wait_for_send_credits(struct smb_direct_transport *t,
070fb21e 975 struct smb_direct_send_ctx *send_ctx)
0626e664
NJ		 976{
977 int ret;
978
070fb21e
NJ		 979 if (send_ctx &&
980 (send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) {
0626e664
NJ		 981 ret = smb_direct_flush_send_list(t, send_ctx, false);
982 if (ret)
983 return ret;
984 }
985
986 return wait_for_credits(t, &t->wait_send_credits, &t->send_credits);
987}
988
989static int smb_direct_create_header(struct smb_direct_transport *t,
070fb21e
NJ		 990 int size, int remaining_data_length,
991 struct smb_direct_sendmsg **sendmsg_out)
0626e664
NJ		 992{
993 struct smb_direct_sendmsg *sendmsg;
994 struct smb_direct_data_transfer *packet;
995 int header_length;
996 int ret;
997
998 sendmsg = smb_direct_alloc_sendmsg(t);
8ef32967
DC		 999 if (IS_ERR(sendmsg))
1000 return PTR_ERR(sendmsg);
0626e664
NJ		 1001
1002 /* Fill in the packet header */
1003 packet = (struct smb_direct_data_transfer *)sendmsg->packet;
1004 packet->credits_requested = cpu_to_le16(t->send_credit_target);
1005 packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
1006
1007 packet->flags = 0;
1008 packet->reserved = 0;
1009 if (!size)
1010 packet->data_offset = 0;
1011 else
1012 packet->data_offset = cpu_to_le32(24);
1013 packet->data_length = cpu_to_le32(size);
1014 packet->remaining_data_length = cpu_to_le32(remaining_data_length);
1015 packet->padding = 0;
1016
1017 ksmbd_debug(RDMA,
070fb21e
NJ		 1018 "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n",
1019 le16_to_cpu(packet->credits_requested),
1020 le16_to_cpu(packet->credits_granted),
1021 le32_to_cpu(packet->data_offset),
1022 le32_to_cpu(packet->data_length),
1023 le32_to_cpu(packet->remaining_data_length));
0626e664
NJ		 1024
1025 /* Map the packet to DMA */
1026 header_length = sizeof(struct smb_direct_data_transfer);
1027 /* If this is a packet without payload, don't send padding */
1028 if (!size)
1029 header_length =
1030 offsetof(struct smb_direct_data_transfer, padding);
1031
1032 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
1033 (void *)packet,
1034 header_length,
1035 DMA_TO_DEVICE);
1036 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
1037 if (ret) {
1038 smb_direct_free_sendmsg(t, sendmsg);
1039 return ret;
1040 }
1041
1042 sendmsg->num_sge = 1;
1043 sendmsg->sge[0].length = header_length;
1044 sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1045
1046 *sendmsg_out = sendmsg;
1047 return 0;
1048}
1049
64b39f4a 1050static int get_sg_list(void *buf, int size, struct scatterlist *sg_list, int nentries)
0626e664
NJ		 1051{
1052 bool high = is_vmalloc_addr(buf);
1053 struct page *page;
1054 int offset, len;
1055 int i = 0;
1056
8ad8dc34 1057 if (nentries < get_buf_page_count(buf, size))
0626e664
NJ		 1058 return -EINVAL;
1059
1060 offset = offset_in_page(buf);
1061 buf -= offset;
1062 while (size > 0) {
1063 len = min_t(int, PAGE_SIZE - offset, size);
1064 if (high)
1065 page = vmalloc_to_page(buf);
1066 else
1067 page = kmap_to_page(buf);
1068
1069 if (!sg_list)
1070 return -EINVAL;
1071 sg_set_page(sg_list, page, len, offset);
1072 sg_list = sg_next(sg_list);
1073
1074 buf += PAGE_SIZE;
1075 size -= len;
1076 offset = 0;
1077 i++;
1078 }
1079 return i;
1080}
1081
1082static int get_mapped_sg_list(struct ib_device *device, void *buf, int size,
070fb21e
NJ		 1083 struct scatterlist *sg_list, int nentries,
1084 enum dma_data_direction dir)
0626e664
NJ		 1085{
1086 int npages;
1087
1088 npages = get_sg_list(buf, size, sg_list, nentries);
1089 if (npages <= 0)
1090 return -EINVAL;
1091 return ib_dma_map_sg(device, sg_list, npages, dir);
1092}
1093
1094static int post_sendmsg(struct smb_direct_transport *t,
070fb21e
NJ		 1095 struct smb_direct_send_ctx *send_ctx,
1096 struct smb_direct_sendmsg *msg)
0626e664
NJ		 1097{
1098 int i;
1099
1100 for (i = 0; i < msg->num_sge; i++)
1101 ib_dma_sync_single_for_device(t->cm_id->device,
070fb21e
NJ		 1102 msg->sge[i].addr, msg->sge[i].length,
1103 DMA_TO_DEVICE);
0626e664
NJ		 1104
1105 msg->cqe.done = send_done;
1106 msg->wr.opcode = IB_WR_SEND;
1107 msg->wr.sg_list = &msg->sge[0];
1108 msg->wr.num_sge = msg->num_sge;
1109 msg->wr.next = NULL;
1110
1111 if (send_ctx) {
1112 msg->wr.wr_cqe = NULL;
1113 msg->wr.send_flags = 0;
1114 if (!list_empty(&send_ctx->msg_list)) {
1115 struct smb_direct_sendmsg *last;
1116
1117 last = list_last_entry(&send_ctx->msg_list,
1118 struct smb_direct_sendmsg,
1119 list);
1120 last->wr.next = &msg->wr;
1121 }
1122 list_add_tail(&msg->list, &send_ctx->msg_list);
1123 send_ctx->wr_cnt++;
1124 return 0;
1125 }
1126
1127 msg->wr.wr_cqe = &msg->cqe;
1128 msg->wr.send_flags = IB_SEND_SIGNALED;
1129 return smb_direct_post_send(t, &msg->wr);
1130}
1131
1132static int smb_direct_post_send_data(struct smb_direct_transport *t,
070fb21e
NJ		 1133 struct smb_direct_send_ctx *send_ctx,
1134 struct kvec *iov, int niov,
1135 int remaining_data_length)
0626e664
NJ		 1136{
1137 int i, j, ret;
1138 struct smb_direct_sendmsg *msg;
1139 int data_length;
64b39f4a 1140 struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1];
0626e664
NJ		 1141
1142 ret = wait_for_send_credits(t, send_ctx);
1143 if (ret)
1144 return ret;
1145
1146 data_length = 0;
1147 for (i = 0; i < niov; i++)
1148 data_length += iov[i].iov_len;
1149
1150 ret = smb_direct_create_header(t, data_length, remaining_data_length,
1151 &msg);
1152 if (ret) {
1153 atomic_inc(&t->send_credits);
1154 return ret;
1155 }
1156
1157 for (i = 0; i < niov; i++) {
1158 struct ib_sge *sge;
1159 int sg_cnt;
1160
64b39f4a 1161 sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1);
0626e664 1162 sg_cnt = get_mapped_sg_list(t->cm_id->device,
070fb21e
NJ		 1163 iov[i].iov_base, iov[i].iov_len,
1164 sg, SMB_DIRECT_MAX_SEND_SGES - 1,
1165 DMA_TO_DEVICE);
0626e664 1166 if (sg_cnt <= 0) {
bde1694a 1167 pr_err("failed to map buffer\n");
bc3fcc94 1168 ret = -ENOMEM;
0626e664 1169 goto err;
64b39f4a 1170 } else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES - 1) {
bde1694a 1171 pr_err("buffer not fitted into sges\n");
0626e664
NJ		 1172 ret = -E2BIG;
1173 ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt,
1174 DMA_TO_DEVICE);
1175 goto err;
1176 }
1177
1178 for (j = 0; j < sg_cnt; j++) {
1179 sge = &msg->sge[msg->num_sge];
1180 sge->addr = sg_dma_address(&sg[j]);
1181 sge->length = sg_dma_len(&sg[j]);
1182 sge->lkey = t->pd->local_dma_lkey;
1183 msg->num_sge++;
1184 }
1185 }
1186
1187 ret = post_sendmsg(t, send_ctx, msg);
1188 if (ret)
1189 goto err;
1190 return 0;
1191err:
1192 smb_direct_free_sendmsg(t, msg);
1193 atomic_inc(&t->send_credits);
1194 return ret;
1195}
1196
1197static int smb_direct_writev(struct ksmbd_transport *t,
070fb21e
NJ		 1198 struct kvec *iov, int niovs, int buflen,
1199 bool need_invalidate, unsigned int remote_key)
0626e664 1200{
02d4b4aa 1201 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
0626e664
NJ		 1202 int remaining_data_length;
1203 int start, i, j;
1204 int max_iov_size = st->max_send_size -
1205 sizeof(struct smb_direct_data_transfer);
1206 int ret;
1207 struct kvec vec;
1208 struct smb_direct_send_ctx send_ctx;
1209
1210 if (st->status != SMB_DIRECT_CS_CONNECTED) {
1211 ret = -ENOTCONN;
1212 goto done;
1213 }
1214
1215 //FIXME: skip RFC1002 header..
1216 buflen -= 4;
1217 iov[0].iov_base += 4;
1218 iov[0].iov_len -= 4;
1219
1220 remaining_data_length = buflen;
1221 ksmbd_debug(RDMA, "Sending smb (RDMA): smb_len=%u\n", buflen);
1222
1223 smb_direct_send_ctx_init(st, &send_ctx, need_invalidate, remote_key);
1224 start = i = 0;
1225 buflen = 0;
1226 while (true) {
1227 buflen += iov[i].iov_len;
1228 if (buflen > max_iov_size) {
1229 if (i > start) {
1230 remaining_data_length -=
64b39f4a 1231 (buflen - iov[i].iov_len);
0626e664 1232 ret = smb_direct_post_send_data(st, &send_ctx,
070fb21e
NJ		 1233 &iov[start], i - start,
1234 remaining_data_length);
0626e664
NJ		 1235 if (ret)
1236 goto done;
1237 } else {
1238 /* iov[start] is too big, break it */
64b39f4a 1239 int nvec = (buflen + max_iov_size - 1) /
0626e664
NJ		 1240 max_iov_size;
1241
1242 for (j = 0; j < nvec; j++) {
1243 vec.iov_base =
1244 (char *)iov[start].iov_base +
64b39f4a 1245 j * max_iov_size;
0626e664
NJ		 1246 vec.iov_len =
1247 min_t(int, max_iov_size,
070fb21e 1248 buflen - max_iov_size * j);
0626e664 1249 remaining_data_length -= vec.iov_len;
070fb21e
NJ		 1250 ret = smb_direct_post_send_data(st, &send_ctx, &vec, 1,
1251 remaining_data_length);
0626e664
NJ		 1252 if (ret)
1253 goto done;
1254 }
1255 i++;
1256 if (i == niovs)
1257 break;
1258 }
1259 start = i;
1260 buflen = 0;
1261 } else {
1262 i++;
1263 if (i == niovs) {
1264 /* send out all remaining vecs */
1265 remaining_data_length -= buflen;
1266 ret = smb_direct_post_send_data(st, &send_ctx,
070fb21e
NJ		 1267 &iov[start], i - start,
1268 remaining_data_length);
0626e664
NJ		 1269 if (ret)
1270 goto done;
1271 break;
1272 }
1273 }
1274 }
1275
1276done:
1277 ret = smb_direct_flush_send_list(st, &send_ctx, true);
1278
1279 /*
1280 * As an optimization, we don't wait for individual I/O to finish
1281 * before sending the next one.
1282 * Send them all and wait for pending send count to get to 0
1283 * that means all the I/Os have been out and we are good to return
1284 */
1285
1286 wait_event(st->wait_send_payload_pending,
070fb21e 1287 atomic_read(&st->send_payload_pending) == 0);
0626e664
NJ		 1288 return ret;
1289}
1290
1291static void read_write_done(struct ib_cq *cq, struct ib_wc *wc,
070fb21e 1292 enum dma_data_direction dir)
0626e664
NJ		 1293{
1294 struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe,
070fb21e 1295 struct smb_direct_rdma_rw_msg, cqe);
0626e664
NJ		 1296 struct smb_direct_transport *t = msg->t;
1297
1298 if (wc->status != IB_WC_SUCCESS) {
bde1694a
NJ		 1299 pr_err("read/write error. opcode = %d, status = %s(%d)\n",
1300 wc->opcode, ib_wc_status_msg(wc->status), wc->status);
0626e664
NJ		 1301 smb_direct_disconnect_rdma_connection(t);
1302 }
1303
1304 if (atomic_inc_return(&t->rw_avail_ops) > 0)
1305 wake_up(&t->wait_rw_avail_ops);
1306
1307 rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,
070fb21e 1308 msg->sg_list, msg->sgt.nents, dir);
0626e664
NJ		 1309 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1310 complete(msg->completion);
1311 kfree(msg);
1312}
1313
1314static void read_done(struct ib_cq *cq, struct ib_wc *wc)
1315{
1316 read_write_done(cq, wc, DMA_FROM_DEVICE);
1317}
1318
1319static void write_done(struct ib_cq *cq, struct ib_wc *wc)
1320{
1321 read_write_done(cq, wc, DMA_TO_DEVICE);
1322}
1323
1324static int smb_direct_rdma_xmit(struct smb_direct_transport *t, void *buf,
070fb21e
NJ		 1325 int buf_len, u32 remote_key, u64 remote_offset,
1326 u32 remote_len, bool is_read)
0626e664
NJ		 1327{
1328 struct smb_direct_rdma_rw_msg *msg;
1329 int ret;
1330 DECLARE_COMPLETION_ONSTACK(completion);
1331 struct ib_send_wr *first_wr = NULL;
1332
1333 ret = wait_for_credits(t, &t->wait_rw_avail_ops, &t->rw_avail_ops);
1334 if (ret < 0)
1335 return ret;
1336
1337 /* TODO: mempool */
1338 msg = kmalloc(offsetof(struct smb_direct_rdma_rw_msg, sg_list) +
070fb21e 1339 sizeof(struct scatterlist) * SG_CHUNK_SIZE, GFP_KERNEL);
0626e664
NJ		 1340 if (!msg) {
1341 atomic_inc(&t->rw_avail_ops);
1342 return -ENOMEM;
1343 }
1344
1345 msg->sgt.sgl = &msg->sg_list[0];
1346 ret = sg_alloc_table_chained(&msg->sgt,
8ad8dc34 1347 get_buf_page_count(buf, buf_len),
070fb21e 1348 msg->sg_list, SG_CHUNK_SIZE);
0626e664
NJ		 1349 if (ret) {
1350 atomic_inc(&t->rw_avail_ops);
1351 kfree(msg);
1352 return -ENOMEM;
1353 }
1354
1355 ret = get_sg_list(buf, buf_len, msg->sgt.sgl, msg->sgt.orig_nents);
1356 if (ret <= 0) {
bde1694a 1357 pr_err("failed to get pages\n");
0626e664
NJ		 1358 goto err;
1359 }
1360
1361 ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port,
8ad8dc34 1362 msg->sg_list, get_buf_page_count(buf, buf_len),
070fb21e
NJ		 1363 0, remote_offset, remote_key,
1364 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
0626e664 1365 if (ret < 0) {
bde1694a 1366 pr_err("failed to init rdma_rw_ctx: %d\n", ret);
0626e664
NJ		 1367 goto err;
1368 }
1369
1370 msg->t = t;
1371 msg->cqe.done = is_read ? read_done : write_done;
1372 msg->completion = &completion;
1373 first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port,
070fb21e 1374 &msg->cqe, NULL);
0626e664
NJ		 1375
1376 ret = ib_post_send(t->qp, first_wr, NULL);
1377 if (ret) {
bde1694a 1378 pr_err("failed to post send wr: %d\n", ret);
0626e664
NJ		 1379 goto err;
1380 }
1381
1382 wait_for_completion(&completion);
1383 return 0;
1384
1385err:
1386 atomic_inc(&t->rw_avail_ops);
1387 if (first_wr)
1388 rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,
070fb21e
NJ		 1389 msg->sg_list, msg->sgt.nents,
1390 is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
0626e664
NJ		 1391 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);
1392 kfree(msg);
1393 return ret;
0626e664
NJ		 1394}
1395
64b39f4a 1396static int smb_direct_rdma_write(struct ksmbd_transport *t, void *buf,
070fb21e
NJ		 1397 unsigned int buflen, u32 remote_key,
1398 u64 remote_offset, u32 remote_len)
0626e664 1399{
02d4b4aa 1400 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
070fb21e
NJ		 1401 remote_key, remote_offset,
1402 remote_len, false);
0626e664
NJ		 1403}
1404
64b39f4a 1405static int smb_direct_rdma_read(struct ksmbd_transport *t, void *buf,
070fb21e
NJ		 1406 unsigned int buflen, u32 remote_key,
1407 u64 remote_offset, u32 remote_len)
0626e664 1408{
02d4b4aa 1409 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen,
070fb21e
NJ		 1410 remote_key, remote_offset,
1411 remote_len, true);
0626e664
NJ		 1412}
1413
1414static void smb_direct_disconnect(struct ksmbd_transport *t)
1415{
02d4b4aa 1416 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
0626e664
NJ		 1417
1418 ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id);
1419
1420 smb_direct_disconnect_rdma_connection(st);
1421 wait_event_interruptible(st->wait_status,
070fb21e 1422 st->status == SMB_DIRECT_CS_DISCONNECTED);
0626e664
NJ		 1423 free_transport(st);
1424}
1425
1426static int smb_direct_cm_handler(struct rdma_cm_id *cm_id,
070fb21e 1427 struct rdma_cm_event *event)
0626e664
NJ		 1428{
1429 struct smb_direct_transport *t = cm_id->context;
1430
1431 ksmbd_debug(RDMA, "RDMA CM event. cm_id=%p event=%s (%d)\n",
070fb21e 1432 cm_id, rdma_event_msg(event->event), event->event);
0626e664
NJ		 1433
1434 switch (event->event) {
1435 case RDMA_CM_EVENT_ESTABLISHED: {
1436 t->status = SMB_DIRECT_CS_CONNECTED;
1437 wake_up_interruptible(&t->wait_status);
1438 break;
1439 }
1440 case RDMA_CM_EVENT_DEVICE_REMOVAL:
1441 case RDMA_CM_EVENT_DISCONNECTED: {
1442 t->status = SMB_DIRECT_CS_DISCONNECTED;
1443 wake_up_interruptible(&t->wait_status);
1444 wake_up_interruptible(&t->wait_reassembly_queue);
1445 wake_up(&t->wait_send_credits);
1446 break;
1447 }
1448 case RDMA_CM_EVENT_CONNECT_ERROR: {
1449 t->status = SMB_DIRECT_CS_DISCONNECTED;
1450 wake_up_interruptible(&t->wait_status);
1451 break;
1452 }
1453 default:
bde1694a
NJ		 1454 pr_err("Unexpected RDMA CM event. cm_id=%p, event=%s (%d)\n",
1455 cm_id, rdma_event_msg(event->event),
1456 event->event);
0626e664
NJ		 1457 break;
1458 }
1459 return 0;
1460}
1461
1462static void smb_direct_qpair_handler(struct ib_event *event, void *context)
1463{
1464 struct smb_direct_transport *t = context;
1465
1466 ksmbd_debug(RDMA, "Received QP event. cm_id=%p, event=%s (%d)\n",
070fb21e 1467 t->cm_id, ib_event_msg(event->event), event->event);
0626e664
NJ		 1468
1469 switch (event->event) {
1470 case IB_EVENT_CQ_ERR:
1471 case IB_EVENT_QP_FATAL:
1472 smb_direct_disconnect_rdma_connection(t);
1473 break;
1474 default:
1475 break;
1476 }
1477}
1478
1479static int smb_direct_send_negotiate_response(struct smb_direct_transport *t,
070fb21e 1480 int failed)
0626e664
NJ		 1481{
1482 struct smb_direct_sendmsg *sendmsg;
1483 struct smb_direct_negotiate_resp *resp;
1484 int ret;
1485
1486 sendmsg = smb_direct_alloc_sendmsg(t);
1487 if (IS_ERR(sendmsg))
1488 return -ENOMEM;
1489
1490 resp = (struct smb_direct_negotiate_resp *)sendmsg->packet;
1491 if (failed) {
1492 memset(resp, 0, sizeof(*resp));
1493 resp->min_version = cpu_to_le16(0x0100);
1494 resp->max_version = cpu_to_le16(0x0100);
1495 resp->status = STATUS_NOT_SUPPORTED;
1496 } else {
1497 resp->status = STATUS_SUCCESS;
1498 resp->min_version = SMB_DIRECT_VERSION_LE;
1499 resp->max_version = SMB_DIRECT_VERSION_LE;
1500 resp->negotiated_version = SMB_DIRECT_VERSION_LE;
1501 resp->reserved = 0;
1502 resp->credits_requested =
1503 cpu_to_le16(t->send_credit_target);
64b39f4a 1504 resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t));
0626e664
NJ		 1505 resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size);
1506 resp->preferred_send_size = cpu_to_le32(t->max_send_size);
1507 resp->max_receive_size = cpu_to_le32(t->max_recv_size);
1508 resp->max_fragmented_size =
1509 cpu_to_le32(t->max_fragmented_recv_size);
1510 }
1511
1512 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device,
070fb21e
NJ		 1513 (void *)resp, sizeof(*resp),
1514 DMA_TO_DEVICE);
1515 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr);
0626e664
NJ		 1516 if (ret) {
1517 smb_direct_free_sendmsg(t, sendmsg);
1518 return ret;
1519 }
1520
1521 sendmsg->num_sge = 1;
1522 sendmsg->sge[0].length = sizeof(*resp);
1523 sendmsg->sge[0].lkey = t->pd->local_dma_lkey;
1524
1525 ret = post_sendmsg(t, NULL, sendmsg);
1526 if (ret) {
1527 smb_direct_free_sendmsg(t, sendmsg);
1528 return ret;
1529 }
1530
1531 wait_event(t->wait_send_pending,
070fb21e 1532 atomic_read(&t->send_pending) == 0);
0626e664
NJ		 1533 return 0;
1534}
1535
1536static int smb_direct_accept_client(struct smb_direct_transport *t)
1537{
1538 struct rdma_conn_param conn_param;
1539 struct ib_port_immutable port_immutable;
1540 u32 ird_ord_hdr[2];
1541 int ret;
1542
1543 memset(&conn_param, 0, sizeof(conn_param));
070fb21e
NJ		 1544 conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom,
1545 SMB_DIRECT_CM_INITIATOR_DEPTH);
0626e664
NJ		 1546 conn_param.responder_resources = 0;
1547
1548 t->cm_id->device->ops.get_port_immutable(t->cm_id->device,
070fb21e
NJ		 1549 t->cm_id->port_num,
1550 &port_immutable);
0626e664
NJ		 1551 if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) {
1552 ird_ord_hdr[0] = conn_param.responder_resources;
1553 ird_ord_hdr[1] = 1;
1554 conn_param.private_data = ird_ord_hdr;
1555 conn_param.private_data_len = sizeof(ird_ord_hdr);
1556 } else {
1557 conn_param.private_data = NULL;
1558 conn_param.private_data_len = 0;
1559 }
1560 conn_param.retry_count = SMB_DIRECT_CM_RETRY;
1561 conn_param.rnr_retry_count = SMB_DIRECT_CM_RNR_RETRY;
1562 conn_param.flow_control = 0;
1563
1564 ret = rdma_accept(t->cm_id, &conn_param);
1565 if (ret) {
bde1694a 1566 pr_err("error at rdma_accept: %d\n", ret);
0626e664
NJ		 1567 return ret;
1568 }
1569
1570 wait_event_interruptible(t->wait_status,
1571 t->status != SMB_DIRECT_CS_NEW);
1572 if (t->status != SMB_DIRECT_CS_CONNECTED)
1573 return -ENOTCONN;
1574 return 0;
1575}
1576
1577static int smb_direct_negotiate(struct smb_direct_transport *t)
1578{
1579 int ret;
1580 struct smb_direct_recvmsg *recvmsg;
1581 struct smb_direct_negotiate_req *req;
1582
1583 recvmsg = get_free_recvmsg(t);
1584 if (!recvmsg)
1585 return -ENOMEM;
1586 recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ;
1587
1588 ret = smb_direct_post_recv(t, recvmsg);
1589 if (ret) {
bde1694a 1590 pr_err("Can't post recv: %d\n", ret);
0626e664
NJ		 1591 goto out;
1592 }
1593
1594 t->negotiation_requested = false;
1595 ret = smb_direct_accept_client(t);
1596 if (ret) {
bde1694a 1597 pr_err("Can't accept client\n");
0626e664
NJ		 1598 goto out;
1599 }
1600
1601 smb_direct_post_recv_credits(&t->post_recv_credits_work.work);
1602
1603 ksmbd_debug(RDMA, "Waiting for SMB_DIRECT negotiate request\n");
1604 ret = wait_event_interruptible_timeout(t->wait_status,
070fb21e
NJ		 1605 t->negotiation_requested ||
1606 t->status == SMB_DIRECT_CS_DISCONNECTED,
1607 SMB_DIRECT_NEGOTIATE_TIMEOUT * HZ);
0626e664
NJ		 1608 if (ret <= 0 || t->status == SMB_DIRECT_CS_DISCONNECTED) {
1609 ret = ret < 0 ? ret : -ETIMEDOUT;
1610 goto out;
1611 }
1612
1613 ret = smb_direct_check_recvmsg(recvmsg);
1614 if (ret == -ECONNABORTED)
1615 goto out;
1616
1617 req = (struct smb_direct_negotiate_req *)recvmsg->packet;
1618 t->max_recv_size = min_t(int, t->max_recv_size,
070fb21e 1619 le32_to_cpu(req->preferred_send_size));
0626e664 1620 t->max_send_size = min_t(int, t->max_send_size,
070fb21e 1621 le32_to_cpu(req->max_receive_size));
0626e664
NJ		 1622 t->max_fragmented_send_size =
1623 le32_to_cpu(req->max_fragmented_size);
1624
1625 ret = smb_direct_send_negotiate_response(t, ret);
1626out:
1627 if (recvmsg)
1628 put_recvmsg(t, recvmsg);
1629 return ret;
1630}
1631
1632static int smb_direct_init_params(struct smb_direct_transport *t,
070fb21e 1633 struct ib_qp_cap *cap)
0626e664
NJ		 1634{
1635 struct ib_device *device = t->cm_id->device;
1636 int max_send_sges, max_pages, max_rw_wrs, max_send_wrs;
1637
1638 /* need 2 more sge. because a SMB_DIRECT header will be mapped,
1639 * and maybe a send buffer could be not page aligned.
1640 */
1641 t->max_send_size = smb_direct_max_send_size;
1642 max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 2;
1643 if (max_send_sges > SMB_DIRECT_MAX_SEND_SGES) {
bde1694a 1644 pr_err("max_send_size %d is too large\n", t->max_send_size);
0626e664
NJ		 1645 return -EINVAL;
1646 }
1647
1648 /*
1649 * allow smb_direct_max_outstanding_rw_ops of in-flight RDMA
1650 * read/writes. HCA guarantees at least max_send_sge of sges for
1651 * a RDMA read/write work request, and if memory registration is used,
1652 * we need reg_mr, local_inv wrs for each read/write.
1653 */
1654 t->max_rdma_rw_size = smb_direct_max_read_write_size;
1655 max_pages = DIV_ROUND_UP(t->max_rdma_rw_size, PAGE_SIZE) + 1;
1656 max_rw_wrs = DIV_ROUND_UP(max_pages, SMB_DIRECT_MAX_SEND_SGES);
1657 max_rw_wrs += rdma_rw_mr_factor(device, t->cm_id->port_num,
1658 max_pages) * 2;
1659 max_rw_wrs *= smb_direct_max_outstanding_rw_ops;
1660
1661 max_send_wrs = smb_direct_send_credit_target + max_rw_wrs;
1662 if (max_send_wrs > device->attrs.max_cqe ||
64b39f4a 1663 max_send_wrs > device->attrs.max_qp_wr) {
bde1694a
NJ		 1664 pr_err("consider lowering send_credit_target = %d, or max_outstanding_rw_ops = %d\n",
1665 smb_direct_send_credit_target,
1666 smb_direct_max_outstanding_rw_ops);
1667 pr_err("Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n",
1668 device->attrs.max_cqe, device->attrs.max_qp_wr);
0626e664
NJ		 1669 return -EINVAL;
1670 }
1671
1672 if (smb_direct_receive_credit_max > device->attrs.max_cqe ||
1673 smb_direct_receive_credit_max > device->attrs.max_qp_wr) {
bde1694a
NJ		 1674 pr_err("consider lowering receive_credit_max = %d\n",
1675 smb_direct_receive_credit_max);
1676 pr_err("Possible CQE overrun, device reporting max_cpe %d max_qp_wr %d\n",
1677 device->attrs.max_cqe, device->attrs.max_qp_wr);
0626e664
NJ		 1678 return -EINVAL;
1679 }
1680
1681 if (device->attrs.max_send_sge < SMB_DIRECT_MAX_SEND_SGES) {
bde1694a
NJ		 1682 pr_err("warning: device max_send_sge = %d too small\n",
1683 device->attrs.max_send_sge);
0626e664
NJ		 1684 return -EINVAL;
1685 }
1686 if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) {
bde1694a
NJ		 1687 pr_err("warning: device max_recv_sge = %d too small\n",
1688 device->attrs.max_recv_sge);
0626e664
NJ		 1689 return -EINVAL;
1690 }
1691
1692 t->recv_credits = 0;
1693 t->count_avail_recvmsg = 0;
1694
1695 t->recv_credit_max = smb_direct_receive_credit_max;
1696 t->recv_credit_target = 10;
1697 t->new_recv_credits = 0;
1698
1699 t->send_credit_target = smb_direct_send_credit_target;
1700 atomic_set(&t->send_credits, 0);
1701 atomic_set(&t->rw_avail_ops, smb_direct_max_outstanding_rw_ops);
1702
1703 t->max_send_size = smb_direct_max_send_size;
1704 t->max_recv_size = smb_direct_max_receive_size;
1705 t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size;
1706
1707 cap->max_send_wr = max_send_wrs;
1708 cap->max_recv_wr = t->recv_credit_max;
1709 cap->max_send_sge = SMB_DIRECT_MAX_SEND_SGES;
1710 cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES;
1711 cap->max_inline_data = 0;
1712 cap->max_rdma_ctxs = 0;
1713 return 0;
1714}
1715
1716static void smb_direct_destroy_pools(struct smb_direct_transport *t)
1717{
1718 struct smb_direct_recvmsg *recvmsg;
1719
1720 while ((recvmsg = get_free_recvmsg(t)))
1721 mempool_free(recvmsg, t->recvmsg_mempool);
1722 while ((recvmsg = get_empty_recvmsg(t)))
1723 mempool_free(recvmsg, t->recvmsg_mempool);
1724
1725 mempool_destroy(t->recvmsg_mempool);
1726 t->recvmsg_mempool = NULL;
1727
1728 kmem_cache_destroy(t->recvmsg_cache);
1729 t->recvmsg_cache = NULL;
1730
1731 mempool_destroy(t->sendmsg_mempool);
1732 t->sendmsg_mempool = NULL;
1733
1734 kmem_cache_destroy(t->sendmsg_cache);
1735 t->sendmsg_cache = NULL;
1736}
1737
1738static int smb_direct_create_pools(struct smb_direct_transport *t)
1739{
1740 char name[80];
1741 int i;
1742 struct smb_direct_recvmsg *recvmsg;
1743
1744 snprintf(name, sizeof(name), "smb_direct_rqst_pool_%p", t);
1745 t->sendmsg_cache = kmem_cache_create(name,
070fb21e
NJ		 1746 sizeof(struct smb_direct_sendmsg) +
1747 sizeof(struct smb_direct_negotiate_resp),
1748 0, SLAB_HWCACHE_ALIGN, NULL);
0626e664
NJ		 1749 if (!t->sendmsg_cache)
1750 return -ENOMEM;
1751
1752 t->sendmsg_mempool = mempool_create(t->send_credit_target,
070fb21e
NJ		 1753 mempool_alloc_slab, mempool_free_slab,
1754 t->sendmsg_cache);
0626e664
NJ		 1755 if (!t->sendmsg_mempool)
1756 goto err;
1757
1758 snprintf(name, sizeof(name), "smb_direct_resp_%p", t);
1759 t->recvmsg_cache = kmem_cache_create(name,
070fb21e
NJ		 1760 sizeof(struct smb_direct_recvmsg) +
1761 t->max_recv_size,
1762 0, SLAB_HWCACHE_ALIGN, NULL);
0626e664
NJ		 1763 if (!t->recvmsg_cache)
1764 goto err;
1765
1766 t->recvmsg_mempool =
1767 mempool_create(t->recv_credit_max, mempool_alloc_slab,
070fb21e 1768 mempool_free_slab, t->recvmsg_cache);
0626e664
NJ		 1769 if (!t->recvmsg_mempool)
1770 goto err;
1771
1772 INIT_LIST_HEAD(&t->recvmsg_queue);
1773
1774 for (i = 0; i < t->recv_credit_max; i++) {
1775 recvmsg = mempool_alloc(t->recvmsg_mempool, GFP_KERNEL);
1776 if (!recvmsg)
1777 goto err;
1778 recvmsg->transport = t;
1779 list_add(&recvmsg->list, &t->recvmsg_queue);
1780 }
1781 t->count_avail_recvmsg = t->recv_credit_max;
1782
1783 return 0;
1784err:
1785 smb_direct_destroy_pools(t);
1786 return -ENOMEM;
1787}
1788
1789static int smb_direct_create_qpair(struct smb_direct_transport *t,
070fb21e 1790 struct ib_qp_cap *cap)
0626e664
NJ		 1791{
1792 int ret;
1793 struct ib_qp_init_attr qp_attr;
1794
1795 t->pd = ib_alloc_pd(t->cm_id->device, 0);
1796 if (IS_ERR(t->pd)) {
bde1694a 1797 pr_err("Can't create RDMA PD\n");
0626e664
NJ		 1798 ret = PTR_ERR(t->pd);
1799 t->pd = NULL;
1800 return ret;
1801 }
1802
1803 t->send_cq = ib_alloc_cq(t->cm_id->device, t,
070fb21e 1804 t->send_credit_target, 0, IB_POLL_WORKQUEUE);
0626e664 1805 if (IS_ERR(t->send_cq)) {
bde1694a 1806 pr_err("Can't create RDMA send CQ\n");
0626e664
NJ		 1807 ret = PTR_ERR(t->send_cq);
1808 t->send_cq = NULL;
1809 goto err;
1810 }
1811
1812 t->recv_cq = ib_alloc_cq(t->cm_id->device, t,
070fb21e
NJ		 1813 cap->max_send_wr + cap->max_rdma_ctxs,
1814 0, IB_POLL_WORKQUEUE);
0626e664 1815 if (IS_ERR(t->recv_cq)) {
bde1694a 1816 pr_err("Can't create RDMA recv CQ\n");
0626e664
NJ		 1817 ret = PTR_ERR(t->recv_cq);
1818 t->recv_cq = NULL;
1819 goto err;
1820 }
1821
1822 memset(&qp_attr, 0, sizeof(qp_attr));
1823 qp_attr.event_handler = smb_direct_qpair_handler;
1824 qp_attr.qp_context = t;
1825 qp_attr.cap = *cap;
1826 qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
1827 qp_attr.qp_type = IB_QPT_RC;
1828 qp_attr.send_cq = t->send_cq;
1829 qp_attr.recv_cq = t->recv_cq;
1830 qp_attr.port_num = ~0;
1831
1832 ret = rdma_create_qp(t->cm_id, t->pd, &qp_attr);
1833 if (ret) {
bde1694a 1834 pr_err("Can't create RDMA QP: %d\n", ret);
0626e664
NJ		 1835 goto err;
1836 }
1837
1838 t->qp = t->cm_id->qp;
1839 t->cm_id->event_handler = smb_direct_cm_handler;
1840
1841 return 0;
1842err:
1843 if (t->qp) {
1844 ib_destroy_qp(t->qp);
1845 t->qp = NULL;
1846 }
1847 if (t->recv_cq) {
1848 ib_destroy_cq(t->recv_cq);
1849 t->recv_cq = NULL;
1850 }
1851 if (t->send_cq) {
1852 ib_destroy_cq(t->send_cq);
1853 t->send_cq = NULL;
1854 }
1855 if (t->pd) {
1856 ib_dealloc_pd(t->pd);
1857 t->pd = NULL;
1858 }
1859 return ret;
1860}
1861
1862static int smb_direct_prepare(struct ksmbd_transport *t)
1863{
02d4b4aa 1864 struct smb_direct_transport *st = smb_trans_direct_transfort(t);
0626e664
NJ		 1865 int ret;
1866 struct ib_qp_cap qp_cap;
1867
1868 ret = smb_direct_init_params(st, &qp_cap);
1869 if (ret) {
bde1694a 1870 pr_err("Can't configure RDMA parameters\n");
0626e664
NJ		 1871 return ret;
1872 }
1873
1874 ret = smb_direct_create_pools(st);
1875 if (ret) {
bde1694a 1876 pr_err("Can't init RDMA pool: %d\n", ret);
0626e664
NJ		 1877 return ret;
1878 }
1879
1880 ret = smb_direct_create_qpair(st, &qp_cap);
1881 if (ret) {
bde1694a 1882 pr_err("Can't accept RDMA client: %d\n", ret);
0626e664
NJ		 1883 return ret;
1884 }
1885
1886 ret = smb_direct_negotiate(st);
1887 if (ret) {
bde1694a 1888 pr_err("Can't negotiate: %d\n", ret);
0626e664
NJ		 1889 return ret;
1890 }
1891
1892 st->status = SMB_DIRECT_CS_CONNECTED;
1893 return 0;
1894}
1895
1896static bool rdma_frwr_is_supported(struct ib_device_attr *attrs)
1897{
1898 if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
1899 return false;
1900 if (attrs->max_fast_reg_page_list_len == 0)
1901 return false;
1902 return true;
1903}
1904
1905static int smb_direct_handle_connect_request(struct rdma_cm_id *new_cm_id)
1906{
1907 struct smb_direct_transport *t;
1908
1909 if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) {
1910 ksmbd_debug(RDMA,
070fb21e
NJ		 1911 "Fast Registration Work Requests is not supported. device capabilities=%llx\n",
1912 new_cm_id->device->attrs.device_cap_flags);
0626e664
NJ		 1913 return -EPROTONOSUPPORT;
1914 }
1915
1916 t = alloc_transport(new_cm_id);
1917 if (!t)
1918 return -ENOMEM;
1919
1920 KSMBD_TRANS(t)->handler = kthread_run(ksmbd_conn_handler_loop,
070fb21e
NJ		 1921 KSMBD_TRANS(t)->conn, "ksmbd:r%u",
1922 SMB_DIRECT_PORT);
0626e664
NJ		 1923 if (IS_ERR(KSMBD_TRANS(t)->handler)) {
1924 int ret = PTR_ERR(KSMBD_TRANS(t)->handler);
1925
bde1694a 1926 pr_err("Can't start thread\n");
0626e664
NJ		 1927 free_transport(t);
1928 return ret;
1929 }
1930
1931 return 0;
1932}
1933
1934static int smb_direct_listen_handler(struct rdma_cm_id *cm_id,
070fb21e 1935 struct rdma_cm_event *event)
0626e664
NJ		 1936{
1937 switch (event->event) {
1938 case RDMA_CM_EVENT_CONNECT_REQUEST: {
1939 int ret = smb_direct_handle_connect_request(cm_id);
1940
1941 if (ret) {
bde1694a 1942 pr_err("Can't create transport: %d\n", ret);
0626e664
NJ		 1943 return ret;
1944 }
1945
1946 ksmbd_debug(RDMA, "Received connection request. cm_id=%p\n",
070fb21e 1947 cm_id);
0626e664
NJ		 1948 break;
1949 }
1950 default:
bde1694a
NJ		 1951 pr_err("Unexpected listen event. cm_id=%p, event=%s (%d)\n",
1952 cm_id, rdma_event_msg(event->event), event->event);
0626e664
NJ		 1953 break;
1954 }
1955 return 0;
1956}
1957
1958static int smb_direct_listen(int port)
1959{
1960 int ret;
1961 struct rdma_cm_id *cm_id;
1962 struct sockaddr_in sin = {
1963 .sin_family = AF_INET,
1964 .sin_addr.s_addr = htonl(INADDR_ANY),
1965 .sin_port = htons(port),
1966 };
1967
1968 cm_id = rdma_create_id(&init_net, smb_direct_listen_handler,
070fb21e 1969 &smb_direct_listener, RDMA_PS_TCP, IB_QPT_RC);
0626e664 1970 if (IS_ERR(cm_id)) {
bde1694a 1971 pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id));
0626e664
NJ		 1972 return PTR_ERR(cm_id);
1973 }
1974
1975 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&sin);
1976 if (ret) {
bde1694a 1977 pr_err("Can't bind: %d\n", ret);
0626e664
NJ		 1978 goto err;
1979 }
1980
1981 smb_direct_listener.cm_id = cm_id;
1982
1983 ret = rdma_listen(cm_id, 10);
1984 if (ret) {
bde1694a 1985 pr_err("Can't listen: %d\n", ret);
0626e664
NJ		 1986 goto err;
1987 }
1988 return 0;
1989err:
1990 smb_direct_listener.cm_id = NULL;
1991 rdma_destroy_id(cm_id);
1992 return ret;
1993}
1994
1995int ksmbd_rdma_init(void)
1996{
1997 int ret;
1998
1999 smb_direct_listener.cm_id = NULL;
2000
2001 /* When a client is running out of send credits, the credits are
2002 * granted by the server's sending a packet using this queue.
2003 * This avoids the situation that a clients cannot send packets
2004 * for lack of credits
2005 */
2006 smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq",
070fb21e 2007 WQ_HIGHPRI | WQ_MEM_RECLAIM, 0);
0626e664
NJ		 2008 if (!smb_direct_wq)
2009 return -ENOMEM;
2010
2011 ret = smb_direct_listen(SMB_DIRECT_PORT);
2012 if (ret) {
2013 destroy_workqueue(smb_direct_wq);
2014 smb_direct_wq = NULL;
bde1694a 2015 pr_err("Can't listen: %d\n", ret);
0626e664
NJ		 2016 return ret;
2017 }
2018
2019 ksmbd_debug(RDMA, "init RDMA listener. cm_id=%p\n",
070fb21e 2020 smb_direct_listener.cm_id);
0626e664
NJ		 2021 return 0;
2022}
2023
2024int ksmbd_rdma_destroy(void)
2025{
2026 if (smb_direct_listener.cm_id)
2027 rdma_destroy_id(smb_direct_listener.cm_id);
2028 smb_direct_listener.cm_id = NULL;
2029
2030 if (smb_direct_wq) {
2031 flush_workqueue(smb_direct_wq);
2032 destroy_workqueue(smb_direct_wq);
2033 smb_direct_wq = NULL;
2034 }
2035 return 0;
2036}
2037
2038static struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = {
2039 .prepare = smb_direct_prepare,
2040 .disconnect = smb_direct_disconnect,
2041 .writev = smb_direct_writev,
2042 .read = smb_direct_read,
2043 .rdma_read = smb_direct_rdma_read,
2044 .rdma_write = smb_direct_rdma_write,
2045};
Linux 6.x block layer and io_uring tree(s)