projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: kzalloc() -> kcalloc()
[linux-2.6-block.git] / net / smc / smc_wr.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
f38ba179
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Work Requests exploiting Infiniband API
6 *
7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
8 * are submitted to either RC SQ or RC RQ respectively
9 * (reliably connected send/receive queue)
10 * and become work queue entries (WQEs).
11 * While an SQ WR/WQE is pending, we track it until transmission completion.
12 * Through a send or receive completion queue (CQ) respectively,
13 * we get completion queue entries (CQEs) [aka work completions (WCs)].
14 * Since the CQ callback is called from IRQ context, we split work by using
15 * bottom halves implemented by tasklets.
16 *
17 * SMC uses this to exchange LLC (link layer control)
18 * and CDC (connection data control) messages.
19 *
20 * Copyright IBM Corp. 2016
21 *
22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com>
23 */
24
25#include <linux/atomic.h>
26#include <linux/hashtable.h>
27#include <linux/wait.h>
28#include <rdma/ib_verbs.h>
29#include <asm/div64.h>
30
31#include "smc.h"
32#include "smc_wr.h"
33
34#define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */
35
36#define SMC_WR_RX_HASH_BITS 4
37static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
38static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
39
40struct smc_wr_tx_pend { /* control data for a pending send request */
41 u64 wr_id; /* work request id sent */
42 smc_wr_tx_handler handler;
43 enum ib_wc_status wc_status; /* CQE status */
44 struct smc_link *link;
45 u32 idx;
46 struct smc_wr_tx_pend_priv priv;
47};
48
49/******************************** send queue *********************************/
50
51/*------------------------------- completion --------------------------------*/
52
53static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
54{
55 u32 i;
56
57 for (i = 0; i < link->wr_tx_cnt; i++) {
58 if (link->wr_tx_pends[i].wr_id == wr_id)
59 return i;
60 }
61 return link->wr_tx_cnt;
62}
63
64static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
65{
66 struct smc_wr_tx_pend pnd_snd;
67 struct smc_link *link;
68 u32 pnd_snd_idx;
69 int i;
70
71 link = wc->qp->qp_context;
652a1e41
UB		 72
73 if (wc->opcode == IB_WC_REG_MR) {
74 if (wc->status)
75 link->wr_reg_state = FAILED;
76 else
77 link->wr_reg_state = CONFIRMED;
78 wake_up(&link->wr_reg_wait);
79 return;
80 }
81
f38ba179
UB		 82 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
83 if (pnd_snd_idx == link->wr_tx_cnt)
84 return;
85 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
86 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
87 /* clear the full struct smc_wr_tx_pend including .priv */
88 memset(&link->wr_tx_pends[pnd_snd_idx], 0,
89 sizeof(link->wr_tx_pends[pnd_snd_idx]));
90 memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
91 sizeof(link->wr_tx_bufs[pnd_snd_idx]));
92 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
93 return;
94 if (wc->status) {
b38d7324
UB		 95 struct smc_link_group *lgr;
96
f38ba179
UB		 97 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
98 /* clear full struct smc_wr_tx_pend including .priv */
99 memset(&link->wr_tx_pends[i], 0,
100 sizeof(link->wr_tx_pends[i]));
101 memset(&link->wr_tx_bufs[i], 0,
102 sizeof(link->wr_tx_bufs[i]));
103 clear_bit(i, link->wr_tx_mask);
104 }
b38d7324
UB		 105 /* terminate connections of this link group abnormally */
106 lgr = container_of(link, struct smc_link_group,
107 lnk[SMC_SINGLE_LINK]);
108 smc_lgr_terminate(lgr);
f38ba179
UB		 109 }
110 if (pnd_snd.handler)
111 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
112 wake_up(&link->wr_tx_wait);
113}
114
115static void smc_wr_tx_tasklet_fn(unsigned long data)
116{
117 struct smc_ib_device *dev = (struct smc_ib_device *)data;
118 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
119 int i = 0, rc;
120 int polled = 0;
121
122again:
123 polled++;
124 do {
86e780d3 125 memset(&wc, 0, sizeof(wc));
f38ba179
UB		 126 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
127 if (polled == 1) {
128 ib_req_notify_cq(dev->roce_cq_send,
129 IB_CQ_NEXT_COMP |
130 IB_CQ_REPORT_MISSED_EVENTS);
131 }
132 if (!rc)
133 break;
134 for (i = 0; i < rc; i++)
135 smc_wr_tx_process_cqe(&wc[i]);
136 } while (rc > 0);
137 if (polled == 1)
138 goto again;
139}
140
141void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
142{
143 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
144
145 tasklet_schedule(&dev->send_tasklet);
146}
147
148/*---------------------------- request submission ---------------------------*/
149
150static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
151{
152 *idx = link->wr_tx_cnt;
153 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
154 if (!test_and_set_bit(*idx, link->wr_tx_mask))
155 return 0;
156 }
157 *idx = link->wr_tx_cnt;
158 return -EBUSY;
159}
160
161/**
162 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
163 * and sets info for pending transmit tracking
164 * @link: Pointer to smc_link used to later send the message.
165 * @handler: Send completion handler function pointer.
166 * @wr_buf: Out value returns pointer to message buffer.
167 * @wr_pend_priv: Out value returns pointer serving as handler context.
168 *
169 * Return: 0 on success, or -errno on error.
170 */
171int smc_wr_tx_get_free_slot(struct smc_link *link,
172 smc_wr_tx_handler handler,
173 struct smc_wr_buf **wr_buf,
174 struct smc_wr_tx_pend_priv **wr_pend_priv)
175{
176 struct smc_wr_tx_pend *wr_pend;
1a0a04c7 177 u32 idx = link->wr_tx_cnt;
f38ba179
UB		 178 struct ib_send_wr *wr_ib;
179 u64 wr_id;
f38ba179
UB		 180 int rc;
181
182 *wr_buf = NULL;
183 *wr_pend_priv = NULL;
184 if (in_softirq()) {
185 rc = smc_wr_tx_get_free_slot_index(link, &idx);
186 if (rc)
187 return rc;
188 } else {
1a0a04c7
UB		 189 struct smc_link_group *lgr;
190
191 lgr = container_of(link, struct smc_link_group,
192 lnk[SMC_SINGLE_LINK]);
86e780d3 193 rc = wait_event_timeout(
f38ba179 194 link->wr_tx_wait,
1a0a04c7 195 list_empty(&lgr->list) || /* lgr terminated */
f38ba179
UB		 196 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
197 SMC_WR_TX_WAIT_FREE_SLOT_TIME);
198 if (!rc) {
b38d7324 199 /* timeout - terminate connections */
b38d7324 200 smc_lgr_terminate(lgr);
f38ba179
UB		 201 return -EPIPE;
202 }
f38ba179
UB		 203 if (idx == link->wr_tx_cnt)
204 return -EPIPE;
205 }
206 wr_id = smc_wr_tx_get_next_wr_id(link);
207 wr_pend = &link->wr_tx_pends[idx];
208 wr_pend->wr_id = wr_id;
209 wr_pend->handler = handler;
210 wr_pend->link = link;
211 wr_pend->idx = idx;
212 wr_ib = &link->wr_tx_ibs[idx];
213 wr_ib->wr_id = wr_id;
214 *wr_buf = &link->wr_tx_bufs[idx];
215 *wr_pend_priv = &wr_pend->priv;
216 return 0;
217}
218
219int smc_wr_tx_put_slot(struct smc_link *link,
220 struct smc_wr_tx_pend_priv *wr_pend_priv)
221{
222 struct smc_wr_tx_pend *pend;
223
224 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
225 if (pend->idx < link->wr_tx_cnt) {
226 /* clear the full struct smc_wr_tx_pend including .priv */
227 memset(&link->wr_tx_pends[pend->idx], 0,
228 sizeof(link->wr_tx_pends[pend->idx]));
229 memset(&link->wr_tx_bufs[pend->idx], 0,
230 sizeof(link->wr_tx_bufs[pend->idx]));
231 test_and_clear_bit(pend->idx, link->wr_tx_mask);
232 return 1;
233 }
234
235 return 0;
236}
237
238/* Send prepared WR slot via ib_post_send.
239 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
240 */
241int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
242{
243 struct ib_send_wr *failed_wr = NULL;
244 struct smc_wr_tx_pend *pend;
245 int rc;
246
247 ib_req_notify_cq(link->smcibdev->roce_cq_send,
8301fa44 248 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
f38ba179
UB		 249 pend = container_of(priv, struct smc_wr_tx_pend, priv);
250 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx],
251 &failed_wr);
b4772b3a
UB		 252 if (rc) {
253 struct smc_link_group *lgr =
254 container_of(link, struct smc_link_group,
255 lnk[SMC_SINGLE_LINK]);
256
f38ba179 257 smc_wr_tx_put_slot(link, priv);
b4772b3a
UB		 258 smc_lgr_terminate(lgr);
259 }
f38ba179
UB		 260 return rc;
261}
262
652a1e41
UB		 263/* Register a memory region and wait for result. */
264int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
265{
266 struct ib_send_wr *failed_wr = NULL;
267 int rc;
268
269 ib_req_notify_cq(link->smcibdev->roce_cq_send,
270 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
271 link->wr_reg_state = POSTED;
272 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
273 link->wr_reg.mr = mr;
274 link->wr_reg.key = mr->rkey;
275 failed_wr = &link->wr_reg.wr;
276 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr);
277 WARN_ON(failed_wr != &link->wr_reg.wr);
278 if (rc)
279 return rc;
280
281 rc = wait_event_interruptible_timeout(link->wr_reg_wait,
282 (link->wr_reg_state != POSTED),
283 SMC_WR_REG_MR_WAIT_TIME);
284 if (!rc) {
285 /* timeout - terminate connections */
286 struct smc_link_group *lgr;
287
288 lgr = container_of(link, struct smc_link_group,
289 lnk[SMC_SINGLE_LINK]);
290 smc_lgr_terminate(lgr);
291 return -EPIPE;
292 }
293 if (rc == -ERESTARTSYS)
294 return -EINTR;
295 switch (link->wr_reg_state) {
296 case CONFIRMED:
297 rc = 0;
298 break;
299 case FAILED:
300 rc = -EIO;
301 break;
302 case POSTED:
303 rc = -EPIPE;
304 break;
305 }
306 return rc;
307}
308
86e780d3 309void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
5f08318f
UB		 310 smc_wr_tx_filter filter,
311 smc_wr_tx_dismisser dismisser,
312 unsigned long data)
313{
314 struct smc_wr_tx_pend_priv *tx_pend;
86e780d3 315 struct smc_wr_rx_hdr *wr_tx;
5f08318f
UB		 316 int i;
317
318 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
86e780d3
UB		 319 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
320 if (wr_tx->type != wr_tx_hdr_type)
5f08318f
UB		 321 continue;
322 tx_pend = &link->wr_tx_pends[i].priv;
323 if (filter(tx_pend, data))
324 dismisser(tx_pend);
325 }
326}
327
f38ba179
UB		 328/****************************** receive queue ********************************/
329
330int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
331{
332 struct smc_wr_rx_handler *h_iter;
333 int rc = 0;
334
335 spin_lock(&smc_wr_rx_hash_lock);
336 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
337 if (h_iter->type == handler->type) {
338 rc = -EEXIST;
339 goto out_unlock;
340 }
341 }
342 hash_add(smc_wr_rx_hash, &handler->list, handler->type);
343out_unlock:
344 spin_unlock(&smc_wr_rx_hash_lock);
345 return rc;
346}
347
348/* Demultiplex a received work request based on the message type to its handler.
349 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
350 * and not being modified any more afterwards so we don't need to lock it.
351 */
352static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
353{
354 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
355 struct smc_wr_rx_handler *handler;
356 struct smc_wr_rx_hdr *wr_rx;
357 u64 temp_wr_id;
358 u32 index;
359
360 if (wc->byte_len < sizeof(*wr_rx))
361 return; /* short message */
362 temp_wr_id = wc->wr_id;
363 index = do_div(temp_wr_id, link->wr_rx_cnt);
364 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
365 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
366 if (handler->type == wr_rx->type)
367 handler->handler(wc, wr_rx);
368 }
369}
370
371static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
372{
373 struct smc_link *link;
374 int i;
375
376 for (i = 0; i < num; i++) {
377 link = wc[i].qp->qp_context;
378 if (wc[i].status == IB_WC_SUCCESS) {
877ae5be 379 link->wr_rx_tstamp = jiffies;
f38ba179
UB		 380 smc_wr_rx_demultiplex(&wc[i]);
381 smc_wr_rx_post(link); /* refill WR RX */
382 } else {
b38d7324
UB		 383 struct smc_link_group *lgr;
384
f38ba179
UB		 385 /* handle status errors */
386 switch (wc[i].status) {
387 case IB_WC_RETRY_EXC_ERR:
388 case IB_WC_RNR_RETRY_EXC_ERR:
389 case IB_WC_WR_FLUSH_ERR:
b38d7324
UB		 390 /* terminate connections of this link group
391 * abnormally
392 */
393 lgr = container_of(link, struct smc_link_group,
394 lnk[SMC_SINGLE_LINK]);
395 smc_lgr_terminate(lgr);
f38ba179
UB		 396 break;
397 default:
398 smc_wr_rx_post(link); /* refill WR RX */
399 break;
400 }
401 }
402 }
403}
404
405static void smc_wr_rx_tasklet_fn(unsigned long data)
406{
407 struct smc_ib_device *dev = (struct smc_ib_device *)data;
408 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
409 int polled = 0;
410 int rc;
411
412again:
413 polled++;
414 do {
415 memset(&wc, 0, sizeof(wc));
416 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
417 if (polled == 1) {
418 ib_req_notify_cq(dev->roce_cq_recv,
419 IB_CQ_SOLICITED_MASK
420 | IB_CQ_REPORT_MISSED_EVENTS);
421 }
422 if (!rc)
423 break;
424 smc_wr_rx_process_cqes(&wc[0], rc);
425 } while (rc > 0);
426 if (polled == 1)
427 goto again;
428}
429
430void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
431{
432 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
433
434 tasklet_schedule(&dev->recv_tasklet);
435}
436
437int smc_wr_rx_post_init(struct smc_link *link)
438{
439 u32 i;
440 int rc = 0;
441
442 for (i = 0; i < link->wr_rx_cnt; i++)
443 rc = smc_wr_rx_post(link);
444 return rc;
445}
446
447/***************************** init, exit, misc ******************************/
448
449void smc_wr_remember_qp_attr(struct smc_link *lnk)
450{
451 struct ib_qp_attr *attr = &lnk->qp_attr;
452 struct ib_qp_init_attr init_attr;
453
454 memset(attr, 0, sizeof(*attr));
455 memset(&init_attr, 0, sizeof(init_attr));
456 ib_query_qp(lnk->roce_qp, attr,
457 IB_QP_STATE |
458 IB_QP_CUR_STATE |
459 IB_QP_PKEY_INDEX |
460 IB_QP_PORT |
461 IB_QP_QKEY |
462 IB_QP_AV |
463 IB_QP_PATH_MTU |
464 IB_QP_TIMEOUT |
465 IB_QP_RETRY_CNT |
466 IB_QP_RNR_RETRY |
467 IB_QP_RQ_PSN |
468 IB_QP_ALT_PATH |
469 IB_QP_MIN_RNR_TIMER |
470 IB_QP_SQ_PSN |
471 IB_QP_PATH_MIG_STATE |
472 IB_QP_CAP |
473 IB_QP_DEST_QPN,
474 &init_attr);
475
476 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
477 lnk->qp_attr.cap.max_send_wr);
478 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
479 lnk->qp_attr.cap.max_recv_wr);
480}
481
482static void smc_wr_init_sge(struct smc_link *lnk)
483{
484 u32 i;
485
486 for (i = 0; i < lnk->wr_tx_cnt; i++) {
487 lnk->wr_tx_sges[i].addr =
488 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
489 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
bd4ad577 490 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
f38ba179
UB		 491 lnk->wr_tx_ibs[i].next = NULL;
492 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
493 lnk->wr_tx_ibs[i].num_sge = 1;
494 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
495 lnk->wr_tx_ibs[i].send_flags =
2c9c1682 496 IB_SEND_SIGNALED | IB_SEND_SOLICITED;
f38ba179
UB		 497 }
498 for (i = 0; i < lnk->wr_rx_cnt; i++) {
499 lnk->wr_rx_sges[i].addr =
500 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
501 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
bd4ad577 502 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
f38ba179
UB		 503 lnk->wr_rx_ibs[i].next = NULL;
504 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
505 lnk->wr_rx_ibs[i].num_sge = 1;
506 }
652a1e41
UB		 507 lnk->wr_reg.wr.next = NULL;
508 lnk->wr_reg.wr.num_sge = 0;
509 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
510 lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
511 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
f38ba179
UB		 512}
513
514void smc_wr_free_link(struct smc_link *lnk)
515{
516 struct ib_device *ibdev;
517
518 memset(lnk->wr_tx_mask, 0,
519 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
520
521 if (!lnk->smcibdev)
522 return;
523 ibdev = lnk->smcibdev->ibdev;
524
525 if (lnk->wr_rx_dma_addr) {
526 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
527 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
528 DMA_FROM_DEVICE);
529 lnk->wr_rx_dma_addr = 0;
530 }
531 if (lnk->wr_tx_dma_addr) {
532 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
533 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
534 DMA_TO_DEVICE);
535 lnk->wr_tx_dma_addr = 0;
536 }
537}
538
539void smc_wr_free_link_mem(struct smc_link *lnk)
540{
541 kfree(lnk->wr_tx_pends);
542 lnk->wr_tx_pends = NULL;
543 kfree(lnk->wr_tx_mask);
544 lnk->wr_tx_mask = NULL;
545 kfree(lnk->wr_tx_sges);
546 lnk->wr_tx_sges = NULL;
547 kfree(lnk->wr_rx_sges);
548 lnk->wr_rx_sges = NULL;
549 kfree(lnk->wr_rx_ibs);
550 lnk->wr_rx_ibs = NULL;
551 kfree(lnk->wr_tx_ibs);
552 lnk->wr_tx_ibs = NULL;
553 kfree(lnk->wr_tx_bufs);
554 lnk->wr_tx_bufs = NULL;
555 kfree(lnk->wr_rx_bufs);
556 lnk->wr_rx_bufs = NULL;
557}
558
559int smc_wr_alloc_link_mem(struct smc_link *link)
560{
561 /* allocate link related memory */
562 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
563 if (!link->wr_tx_bufs)
564 goto no_mem;
565 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
566 GFP_KERNEL);
567 if (!link->wr_rx_bufs)
568 goto no_mem_wr_tx_bufs;
569 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
570 GFP_KERNEL);
571 if (!link->wr_tx_ibs)
572 goto no_mem_wr_rx_bufs;
573 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
574 sizeof(link->wr_rx_ibs[0]),
575 GFP_KERNEL);
576 if (!link->wr_rx_ibs)
577 goto no_mem_wr_tx_ibs;
578 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
579 GFP_KERNEL);
580 if (!link->wr_tx_sges)
581 goto no_mem_wr_rx_ibs;
582 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
583 sizeof(link->wr_rx_sges[0]),
584 GFP_KERNEL);
585 if (!link->wr_rx_sges)
586 goto no_mem_wr_tx_sges;
6396bb22
KC		 587 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
588 sizeof(*link->wr_tx_mask),
589 GFP_KERNEL);
f38ba179
UB		 590 if (!link->wr_tx_mask)
591 goto no_mem_wr_rx_sges;
592 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
593 sizeof(link->wr_tx_pends[0]),
594 GFP_KERNEL);
595 if (!link->wr_tx_pends)
596 goto no_mem_wr_tx_mask;
597 return 0;
598
599no_mem_wr_tx_mask:
600 kfree(link->wr_tx_mask);
601no_mem_wr_rx_sges:
602 kfree(link->wr_rx_sges);
603no_mem_wr_tx_sges:
604 kfree(link->wr_tx_sges);
605no_mem_wr_rx_ibs:
606 kfree(link->wr_rx_ibs);
607no_mem_wr_tx_ibs:
608 kfree(link->wr_tx_ibs);
609no_mem_wr_rx_bufs:
610 kfree(link->wr_rx_bufs);
611no_mem_wr_tx_bufs:
612 kfree(link->wr_tx_bufs);
613no_mem:
614 return -ENOMEM;
615}
616
617void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
618{
619 tasklet_kill(&smcibdev->recv_tasklet);
620 tasklet_kill(&smcibdev->send_tasklet);
621}
622
623void smc_wr_add_dev(struct smc_ib_device *smcibdev)
624{
625 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
626 (unsigned long)smcibdev);
627 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
628 (unsigned long)smcibdev);
629}
630
631int smc_wr_create_link(struct smc_link *lnk)
632{
633 struct ib_device *ibdev = lnk->smcibdev->ibdev;
634 int rc = 0;
635
636 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
637 lnk->wr_rx_id = 0;
638 lnk->wr_rx_dma_addr = ib_dma_map_single(
639 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
640 DMA_FROM_DEVICE);
641 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
642 lnk->wr_rx_dma_addr = 0;
643 rc = -EIO;
644 goto out;
645 }
646 lnk->wr_tx_dma_addr = ib_dma_map_single(
647 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
648 DMA_TO_DEVICE);
649 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
650 rc = -EIO;
651 goto dma_unmap;
652 }
653 smc_wr_init_sge(lnk);
654 memset(lnk->wr_tx_mask, 0,
655 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
652a1e41
UB		 656 init_waitqueue_head(&lnk->wr_tx_wait);
657 init_waitqueue_head(&lnk->wr_reg_wait);
f38ba179
UB		 658 return rc;
659
660dma_unmap:
661 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
662 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
663 DMA_FROM_DEVICE);
664 lnk->wr_rx_dma_addr = 0;
665out:
666 return rc;
667}
Linux 6.x block layer and io_uring tree(s)