projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/smc: use mutex instead of rwlock_t to protect buffers
[linux-block.git] / net / smc / smc_llc.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
9bf9abea
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
9bf9abea
UB		 7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10 * Ursula Braun <ubraun@linux.vnet.ibm.com>
11 */
12
13#include <net/tcp.h>
14#include <rdma/ib_verbs.h>
15
16#include "smc.h"
17#include "smc_core.h"
18#include "smc_clc.h"
19#include "smc_llc.h"
20
0f627126
SR		 21#define SMC_LLC_DATA_LEN 40
22
23struct smc_llc_hdr {
24 struct smc_wr_rx_hdr common;
25 u8 length; /* 44 */
52bedf37
KG		 26#if defined(__BIG_ENDIAN_BITFIELD)
27 u8 reserved:4,
28 add_link_rej_rsn:4;
29#elif defined(__LITTLE_ENDIAN_BITFIELD)
30 u8 add_link_rej_rsn:4,
31 reserved:4;
32#endif
0f627126
SR		 33 u8 flags;
34};
35
75d320d6
KG		 36#define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
37
0f627126
SR		 38struct smc_llc_msg_confirm_link { /* type 0x01 */
39 struct smc_llc_hdr hd;
40 u8 sender_mac[ETH_ALEN];
41 u8 sender_gid[SMC_GID_SIZE];
42 u8 sender_qp_num[3];
43 u8 link_num;
44 u8 link_uid[SMC_LGR_ID_SIZE];
45 u8 max_links;
46 u8 reserved[9];
47};
48
52bedf37
KG		 49#define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
50#define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
51
52#define SMC_LLC_ADD_LNK_MAX_LINKS 2
53
54struct smc_llc_msg_add_link { /* type 0x02 */
55 struct smc_llc_hdr hd;
56 u8 sender_mac[ETH_ALEN];
57 u8 reserved2[2];
58 u8 sender_gid[SMC_GID_SIZE];
59 u8 sender_qp_num[3];
60 u8 link_num;
61 u8 flags2; /* QP mtu */
62 u8 initial_psn[3];
63 u8 reserved[8];
64};
65
66#define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
67#define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
68
69struct smc_llc_msg_del_link { /* type 0x04 */
70 struct smc_llc_hdr hd;
71 u8 link_num;
72 __be32 reason;
73 u8 reserved[35];
74} __packed; /* format defined in RFC7609 */
75
313164da
KG		 76struct smc_llc_msg_test_link { /* type 0x07 */
77 struct smc_llc_hdr hd;
78 u8 user_data[16];
79 u8 reserved[24];
80};
81
4ed75de5
KG		 82struct smc_rmb_rtoken {
83 union {
84 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
85 /* is actually the num of rtokens, first */
86 /* rtoken is always for the current link */
87 u8 link_id; /* link id of the rtoken */
88 };
89 __be32 rmb_key;
90 __be64 rmb_vaddr;
91} __packed; /* format defined in RFC7609 */
92
93#define SMC_LLC_RKEYS_PER_MSG 3
94
95struct smc_llc_msg_confirm_rkey { /* type 0x06 */
96 struct smc_llc_hdr hd;
97 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
98 u8 reserved;
99};
100
101struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */
102 struct smc_llc_hdr hd;
103 u8 num_rkeys;
104 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
105};
106
107#define SMC_LLC_DEL_RKEY_MAX 8
108#define SMC_LLC_FLAG_RKEY_NEG 0x20
109
110struct smc_llc_msg_delete_rkey { /* type 0x09 */
111 struct smc_llc_hdr hd;
112 u8 num_rkeys;
113 u8 err_mask;
114 u8 reserved[2];
115 __be32 rkey[8];
116 u8 reserved2[4];
117};
118
0f627126
SR		 119union smc_llc_msg {
120 struct smc_llc_msg_confirm_link confirm_link;
52bedf37
KG		 121 struct smc_llc_msg_add_link add_link;
122 struct smc_llc_msg_del_link delete_link;
4ed75de5
KG		 123
124 struct smc_llc_msg_confirm_rkey confirm_rkey;
125 struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont;
126 struct smc_llc_msg_delete_rkey delete_rkey;
127
313164da 128 struct smc_llc_msg_test_link test_link;
0f627126
SR		 129 struct {
130 struct smc_llc_hdr hdr;
131 u8 data[SMC_LLC_DATA_LEN];
132 } raw;
133};
134
135#define SMC_LLC_FLAG_RESP 0x80
136
6c8968c4
KG		 137struct smc_llc_qentry {
138 struct list_head list;
139 struct smc_link *link;
140 union smc_llc_msg msg;
141};
142
9bf9abea
UB		 143/********************************** send *************************************/
144
145struct smc_llc_tx_pend {
146};
147
148/* handler for send/transmission completion of an LLC msg */
149static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
150 struct smc_link *link,
151 enum ib_wc_status wc_status)
152{
153 /* future work: handle wc_status error for recovery and failover */
154}
155
156/**
157 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
158 * @link: Pointer to SMC link used for sending LLC control message.
159 * @wr_buf: Out variable returning pointer to work request payload buffer.
160 * @pend: Out variable returning pointer to private pending WR tracking.
161 * It's the context the transmit complete handler will get.
162 *
163 * Reserves and pre-fills an entry for a pending work request send/tx.
164 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
165 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
166 *
167 * Return: 0 on success, otherwise an error value.
168 */
169static int smc_llc_add_pending_send(struct smc_link *link,
170 struct smc_wr_buf **wr_buf,
171 struct smc_wr_tx_pend_priv **pend)
172{
173 int rc;
174
ad6f317f
UB		 175 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
176 pend);
9bf9abea
UB		 177 if (rc < 0)
178 return rc;
179 BUILD_BUG_ON_MSG(
180 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
181 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
182 BUILD_BUG_ON_MSG(
183 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
184 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
185 BUILD_BUG_ON_MSG(
186 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
187 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
188 return 0;
189}
190
191/* high-level API to send LLC confirm link */
947541f3 192int smc_llc_send_confirm_link(struct smc_link *link,
9bf9abea
UB		 193 enum smc_llc_reqresp reqresp)
194{
00e5fb26 195 struct smc_link_group *lgr = smc_get_lgr(link);
9bf9abea
UB		 196 struct smc_llc_msg_confirm_link *confllc;
197 struct smc_wr_tx_pend_priv *pend;
198 struct smc_wr_buf *wr_buf;
199 int rc;
200
201 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
202 if (rc)
203 return rc;
204 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
205 memset(confllc, 0, sizeof(*confllc));
206 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
207 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
75d320d6 208 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
9bf9abea
UB		 209 if (reqresp == SMC_LLC_RESP)
210 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
947541f3
UB		 211 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
212 ETH_ALEN);
7005ada6 213 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
9bf9abea 214 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
2be922f3 215 confllc->link_num = link->link_id;
9bf9abea 216 memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
52bedf37
KG		 217 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
218 /* send llc message */
219 rc = smc_wr_tx_send(link, pend);
220 return rc;
221}
222
44aa81ce
KG		 223/* send LLC confirm rkey request */
224static int smc_llc_send_confirm_rkey(struct smc_link *link,
225 struct smc_buf_desc *rmb_desc)
226{
227 struct smc_llc_msg_confirm_rkey *rkeyllc;
228 struct smc_wr_tx_pend_priv *pend;
229 struct smc_wr_buf *wr_buf;
230 int rc;
231
232 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
233 if (rc)
234 return rc;
235 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
236 memset(rkeyllc, 0, sizeof(*rkeyllc));
237 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
238 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
239 rkeyllc->rtoken[0].rmb_key =
387707fd 240 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
44aa81ce 241 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
387707fd 242 (u64)sg_dma_address(rmb_desc->sgt[link->link_idx].sgl));
44aa81ce
KG		 243 /* send llc message */
244 rc = smc_wr_tx_send(link, pend);
245 return rc;
246}
247
60e03c62
KG		 248/* send LLC delete rkey request */
249static int smc_llc_send_delete_rkey(struct smc_link *link,
250 struct smc_buf_desc *rmb_desc)
251{
252 struct smc_llc_msg_delete_rkey *rkeyllc;
253 struct smc_wr_tx_pend_priv *pend;
254 struct smc_wr_buf *wr_buf;
255 int rc;
256
257 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
258 if (rc)
259 return rc;
260 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
261 memset(rkeyllc, 0, sizeof(*rkeyllc));
262 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
263 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
264 rkeyllc->num_rkeys = 1;
387707fd 265 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
60e03c62
KG		 266 /* send llc message */
267 rc = smc_wr_tx_send(link, pend);
268 return rc;
269}
270
2a4c57a9
KG		 271/* prepare an add link message */
272static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
7005ada6 273 struct smc_link *link, u8 mac[], u8 gid[],
2a4c57a9
KG		 274 enum smc_llc_reqresp reqresp)
275{
276 memset(addllc, 0, sizeof(*addllc));
277 addllc->hd.common.type = SMC_LLC_ADD_LINK;
278 addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
279 if (reqresp == SMC_LLC_RESP) {
280 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
281 /* always reject more links for now */
282 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
283 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
284 }
285 memcpy(addllc->sender_mac, mac, ETH_ALEN);
286 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
287}
288
52bedf37 289/* send ADD LINK request or response */
7005ada6 290int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
52bedf37
KG		 291 enum smc_llc_reqresp reqresp)
292{
293 struct smc_llc_msg_add_link *addllc;
294 struct smc_wr_tx_pend_priv *pend;
295 struct smc_wr_buf *wr_buf;
296 int rc;
297
298 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
299 if (rc)
300 return rc;
301 addllc = (struct smc_llc_msg_add_link *)wr_buf;
2a4c57a9 302 smc_llc_prep_add_link(addllc, link, mac, gid, reqresp);
52bedf37
KG		 303 /* send llc message */
304 rc = smc_wr_tx_send(link, pend);
305 return rc;
306}
307
2a4c57a9
KG		 308/* prepare a delete link message */
309static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
310 struct smc_link *link,
0d18a0cb 311 enum smc_llc_reqresp reqresp, bool orderly)
2a4c57a9
KG		 312{
313 memset(delllc, 0, sizeof(*delllc));
314 delllc->hd.common.type = SMC_LLC_DELETE_LINK;
315 delllc->hd.length = sizeof(struct smc_llc_msg_add_link);
316 if (reqresp == SMC_LLC_RESP)
317 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
318 /* DEL_LINK_ALL because only 1 link supported */
319 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
0d18a0cb
KG		 320 if (orderly)
321 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
2a4c57a9
KG		 322 delllc->link_num = link->link_id;
323}
324
52bedf37
KG		 325/* send DELETE LINK request or response */
326int smc_llc_send_delete_link(struct smc_link *link,
0d18a0cb 327 enum smc_llc_reqresp reqresp, bool orderly)
52bedf37
KG		 328{
329 struct smc_llc_msg_del_link *delllc;
330 struct smc_wr_tx_pend_priv *pend;
331 struct smc_wr_buf *wr_buf;
332 int rc;
333
334 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
335 if (rc)
336 return rc;
337 delllc = (struct smc_llc_msg_del_link *)wr_buf;
0d18a0cb 338 smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
9bf9abea
UB		 339 /* send llc message */
340 rc = smc_wr_tx_send(link, pend);
341 return rc;
342}
343
d97935fa
KG		 344/* send LLC test link request */
345static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
313164da
KG		 346{
347 struct smc_llc_msg_test_link *testllc;
348 struct smc_wr_tx_pend_priv *pend;
349 struct smc_wr_buf *wr_buf;
350 int rc;
351
352 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
353 if (rc)
354 return rc;
355 testllc = (struct smc_llc_msg_test_link *)wr_buf;
356 memset(testllc, 0, sizeof(*testllc));
357 testllc->hd.common.type = SMC_LLC_TEST_LINK;
358 testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
313164da
KG		 359 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
360 /* send llc message */
361 rc = smc_wr_tx_send(link, pend);
362 return rc;
363}
364
6c8968c4
KG		 365/* schedule an llc send on link, may wait for buffers */
366static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
4ed75de5
KG		 367{
368 struct smc_wr_tx_pend_priv *pend;
369 struct smc_wr_buf *wr_buf;
370 int rc;
371
6c8968c4
KG		 372 if (!smc_link_usable(link))
373 return -ENOLINK;
374 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
4ed75de5 375 if (rc)
6c8968c4
KG		 376 return rc;
377 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
378 return smc_wr_tx_send(link, pend);
4ed75de5
KG		 379}
380
9bf9abea
UB		 381/********************************* receive ***********************************/
382
383static void smc_llc_rx_confirm_link(struct smc_link *link,
384 struct smc_llc_msg_confirm_link *llc)
385{
00e5fb26 386 struct smc_link_group *lgr = smc_get_lgr(link);
ef79d439 387 int conf_rc = 0;
9bf9abea 388
75d320d6 389 /* RMBE eyecatchers are not supported */
ef79d439 390 if (!(llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
75d320d6
KG		 391 conf_rc = ENOTSUPP;
392
ef79d439
KG		 393 if (lgr->role == SMC_CLNT &&
394 link->state == SMC_LNK_ACTIVATING) {
395 link->llc_confirm_rc = conf_rc;
396 link->link_id = llc->link_num;
397 complete(&link->llc_confirm);
9bf9abea
UB		 398 }
399}
400
52bedf37
KG		 401static void smc_llc_rx_add_link(struct smc_link *link,
402 struct smc_llc_msg_add_link *llc)
403{
00e5fb26 404 struct smc_link_group *lgr = smc_get_lgr(link);
52bedf37 405
ef79d439
KG		 406 if (link->state == SMC_LNK_ACTIVATING) {
407 complete(&link->llc_add);
408 return;
409 }
52bedf37 410
ef79d439
KG		 411 if (lgr->role == SMC_SERV) {
412 smc_llc_prep_add_link(llc, link,
413 link->smcibdev->mac[link->ibport - 1],
414 link->gid, SMC_LLC_REQ);
52bedf37 415
ef79d439
KG		 416 } else {
417 smc_llc_prep_add_link(llc, link,
418 link->smcibdev->mac[link->ibport - 1],
419 link->gid, SMC_LLC_RESP);
52bedf37 420 }
ef79d439 421 smc_llc_send_message(link, llc);
52bedf37
KG		 422}
423
424static void smc_llc_rx_delete_link(struct smc_link *link,
425 struct smc_llc_msg_del_link *llc)
426{
00e5fb26 427 struct smc_link_group *lgr = smc_get_lgr(link);
52bedf37 428
ef79d439
KG		 429 smc_lgr_forget(lgr);
430 smc_llc_link_deleting(link);
431 if (lgr->role == SMC_SERV) {
432 /* client asks to delete this link, send request */
433 smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
52bedf37 434 } else {
ef79d439
KG		 435 /* server requests to delete this link, send response */
436 smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
52bedf37 437 }
ef79d439
KG		 438 smc_llc_send_message(link, llc);
439 smc_lgr_terminate_sched(lgr);
52bedf37
KG		 440}
441
313164da
KG		 442static void smc_llc_rx_test_link(struct smc_link *link,
443 struct smc_llc_msg_test_link *llc)
444{
ef79d439
KG		 445 llc->hd.flags |= SMC_LLC_FLAG_RESP;
446 smc_llc_send_message(link, llc);
313164da
KG		 447}
448
4ed75de5
KG		 449static void smc_llc_rx_confirm_rkey(struct smc_link *link,
450 struct smc_llc_msg_confirm_rkey *llc)
451{
4ed75de5
KG		 452 int rc;
453
ef79d439
KG		 454 rc = smc_rtoken_add(link,
455 llc->rtoken[0].rmb_vaddr,
456 llc->rtoken[0].rmb_key);
4ed75de5 457
ef79d439 458 /* ignore rtokens for other links, we have only one link */
4ed75de5 459
ef79d439
KG		 460 llc->hd.flags |= SMC_LLC_FLAG_RESP;
461 if (rc < 0)
462 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
463 smc_llc_send_message(link, llc);
4ed75de5
KG		 464}
465
466static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link,
467 struct smc_llc_msg_confirm_rkey_cont *llc)
468{
ef79d439
KG		 469 /* ignore rtokens for other links, we have only one link */
470 llc->hd.flags |= SMC_LLC_FLAG_RESP;
471 smc_llc_send_message(link, llc);
4ed75de5
KG		 472}
473
474static void smc_llc_rx_delete_rkey(struct smc_link *link,
475 struct smc_llc_msg_delete_rkey *llc)
476{
4ed75de5
KG		 477 u8 err_mask = 0;
478 int i, max;
479
ef79d439
KG		 480 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
481 for (i = 0; i < max; i++) {
482 if (smc_rtoken_delete(link, llc->rkey[i]))
483 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
484 }
4ed75de5 485
ef79d439
KG		 486 if (err_mask) {
487 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
488 llc->err_mask = err_mask;
4ed75de5 489 }
ef79d439
KG		 490
491 llc->hd.flags |= SMC_LLC_FLAG_RESP;
492 smc_llc_send_message(link, llc);
4ed75de5
KG		 493}
494
6c8968c4
KG		 495/* flush the llc event queue */
496void smc_llc_event_flush(struct smc_link_group *lgr)
9bf9abea 497{
6c8968c4
KG		 498 struct smc_llc_qentry *qentry, *q;
499
500 spin_lock_bh(&lgr->llc_event_q_lock);
501 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
502 list_del_init(&qentry->list);
503 kfree(qentry);
504 }
505 spin_unlock_bh(&lgr->llc_event_q_lock);
506}
507
508static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
509{
510 union smc_llc_msg *llc = &qentry->msg;
511 struct smc_link *link = qentry->link;
9bf9abea 512
d854fcbf 513 if (!smc_link_usable(link))
6c8968c4 514 goto out;
313164da
KG		 515
516 switch (llc->raw.hdr.common.type) {
517 case SMC_LLC_TEST_LINK:
518 smc_llc_rx_test_link(link, &llc->test_link);
519 break;
520 case SMC_LLC_CONFIRM_LINK:
9bf9abea 521 smc_llc_rx_confirm_link(link, &llc->confirm_link);
313164da 522 break;
52bedf37
KG		 523 case SMC_LLC_ADD_LINK:
524 smc_llc_rx_add_link(link, &llc->add_link);
525 break;
526 case SMC_LLC_DELETE_LINK:
527 smc_llc_rx_delete_link(link, &llc->delete_link);
528 break;
4ed75de5
KG		 529 case SMC_LLC_CONFIRM_RKEY:
530 smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey);
531 break;
532 case SMC_LLC_CONFIRM_RKEY_CONT:
533 smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont);
534 break;
535 case SMC_LLC_DELETE_RKEY:
536 smc_llc_rx_delete_rkey(link, &llc->delete_rkey);
537 break;
313164da 538 }
6c8968c4
KG		 539out:
540 kfree(qentry);
541}
542
543/* worker to process llc messages on the event queue */
544static void smc_llc_event_work(struct work_struct *work)
545{
546 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
547 llc_event_work);
548 struct smc_llc_qentry *qentry;
549
550again:
551 spin_lock_bh(&lgr->llc_event_q_lock);
552 if (!list_empty(&lgr->llc_event_q)) {
553 qentry = list_first_entry(&lgr->llc_event_q,
554 struct smc_llc_qentry, list);
555 list_del_init(&qentry->list);
556 spin_unlock_bh(&lgr->llc_event_q_lock);
557 smc_llc_event_handler(qentry);
558 goto again;
559 }
560 spin_unlock_bh(&lgr->llc_event_q_lock);
561}
562
ef79d439
KG		 563/* process llc responses in tasklet context */
564static void smc_llc_rx_response(struct smc_link *link, union smc_llc_msg *llc)
565{
566 int rc = 0;
567
568 switch (llc->raw.hdr.common.type) {
569 case SMC_LLC_TEST_LINK:
570 if (link->state == SMC_LNK_ACTIVE)
571 complete(&link->llc_testlink_resp);
572 break;
573 case SMC_LLC_CONFIRM_LINK:
574 if (!(llc->raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
575 rc = ENOTSUPP;
576 if (link->lgr->role == SMC_SERV &&
577 link->state == SMC_LNK_ACTIVATING) {
578 link->llc_confirm_resp_rc = rc;
579 complete(&link->llc_confirm_resp);
580 }
581 break;
582 case SMC_LLC_ADD_LINK:
583 if (link->state == SMC_LNK_ACTIVATING)
584 complete(&link->llc_add_resp);
585 break;
586 case SMC_LLC_DELETE_LINK:
587 if (link->lgr->role == SMC_SERV)
588 smc_lgr_schedule_free_work_fast(link->lgr);
589 break;
590 case SMC_LLC_CONFIRM_RKEY:
591 link->llc_confirm_rkey_resp_rc = llc->raw.hdr.flags &
592 SMC_LLC_FLAG_RKEY_NEG;
593 complete(&link->llc_confirm_rkey_resp);
594 break;
595 case SMC_LLC_CONFIRM_RKEY_CONT:
596 /* unused as long as we don't send this type of msg */
597 break;
598 case SMC_LLC_DELETE_RKEY:
599 link->llc_delete_rkey_resp_rc = llc->raw.hdr.flags &
600 SMC_LLC_FLAG_RKEY_NEG;
601 complete(&link->llc_delete_rkey_resp);
602 break;
603 }
604}
605
6c8968c4
KG		 606/* copy received msg and add it to the event queue */
607static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
608{
609 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
610 struct smc_link_group *lgr = link->lgr;
611 struct smc_llc_qentry *qentry;
612 union smc_llc_msg *llc = buf;
613 unsigned long flags;
614
615 if (wc->byte_len < sizeof(*llc))
616 return; /* short message */
617 if (llc->raw.hdr.length != sizeof(*llc))
618 return; /* invalid message */
619
ef79d439
KG		 620 /* process responses immediately */
621 if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
622 smc_llc_rx_response(link, llc);
623 return;
624 }
625
6c8968c4
KG		 626 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
627 if (!qentry)
628 return;
629 qentry->link = link;
630 INIT_LIST_HEAD(&qentry->list);
631 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
632 spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
633 list_add_tail(&qentry->list, &lgr->llc_event_q);
634 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
635 schedule_work(&link->lgr->llc_event_work);
9bf9abea
UB		 636}
637
44aa81ce 638/***************************** worker, utils *********************************/
877ae5be
KG		 639
640static void smc_llc_testlink_work(struct work_struct *work)
641{
642 struct smc_link *link = container_of(to_delayed_work(work),
643 struct smc_link, llc_testlink_wrk);
644 unsigned long next_interval;
877ae5be
KG		 645 unsigned long expire_time;
646 u8 user_data[16] = { 0 };
647 int rc;
648
877ae5be
KG		 649 if (link->state != SMC_LNK_ACTIVE)
650 return; /* don't reschedule worker */
651 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
652 if (time_is_after_jiffies(expire_time)) {
653 next_interval = expire_time - jiffies;
654 goto out;
655 }
656 reinit_completion(&link->llc_testlink_resp);
d97935fa 657 smc_llc_send_test_link(link, user_data);
877ae5be
KG		 658 /* receive TEST LINK response over RoCE fabric */
659 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
660 SMC_LLC_WAIT_TIME);
1020e1ef
KG		 661 if (link->state != SMC_LNK_ACTIVE)
662 return; /* link state changed */
877ae5be 663 if (rc <= 0) {
5f78fe96 664 smc_lgr_terminate_sched(smc_get_lgr(link));
877ae5be
KG		 665 return;
666 }
667 next_interval = link->llc_testlink_time;
668out:
1020e1ef 669 schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
877ae5be
KG		 670}
671
2a4c57a9 672int smc_llc_link_init(struct smc_link *link)
877ae5be 673{
b32cf4ab
KG		 674 init_completion(&link->llc_confirm);
675 init_completion(&link->llc_confirm_resp);
676 init_completion(&link->llc_add);
677 init_completion(&link->llc_add_resp);
ef79d439
KG		 678 init_completion(&link->llc_confirm_rkey_resp);
679 init_completion(&link->llc_delete_rkey_resp);
60e03c62 680 mutex_init(&link->llc_delete_rkey_mutex);
877ae5be 681 init_completion(&link->llc_testlink_resp);
6c8968c4 682 INIT_WORK(&link->lgr->llc_event_work, smc_llc_event_work);
877ae5be 683 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2a4c57a9 684 return 0;
b32cf4ab
KG		 685}
686
687void smc_llc_link_active(struct smc_link *link, int testlink_time)
688{
877ae5be
KG		 689 link->state = SMC_LNK_ACTIVE;
690 if (testlink_time) {
691 link->llc_testlink_time = testlink_time * HZ;
1020e1ef
KG		 692 schedule_delayed_work(&link->llc_testlink_wrk,
693 link->llc_testlink_time);
877ae5be
KG		 694 }
695}
696
0d18a0cb
KG		 697void smc_llc_link_deleting(struct smc_link *link)
698{
699 link->state = SMC_LNK_DELETING;
15e1b99a 700 smc_wr_wakeup_tx_wait(link);
0d18a0cb
KG		 701}
702
877ae5be 703/* called in worker context */
2a4c57a9 704void smc_llc_link_clear(struct smc_link *link)
877ae5be 705{
2140ac26
KG		 706 complete(&link->llc_testlink_resp);
707 cancel_delayed_work_sync(&link->llc_testlink_wrk);
708 smc_wr_wakeup_reg_wait(link);
709 smc_wr_wakeup_tx_wait(link);
877ae5be
KG		 710}
711
44aa81ce
KG		 712/* register a new rtoken at the remote peer */
713int smc_llc_do_confirm_rkey(struct smc_link *link,
714 struct smc_buf_desc *rmb_desc)
715{
716 int rc;
717
60e03c62 718 /* protected by mutex smc_create_lgr_pending */
ef79d439 719 reinit_completion(&link->llc_confirm_rkey_resp);
4600cfc3
KG		 720 rc = smc_llc_send_confirm_rkey(link, rmb_desc);
721 if (rc)
722 return rc;
44aa81ce 723 /* receive CONFIRM RKEY response from server over RoCE fabric */
ef79d439
KG		 724 rc = wait_for_completion_interruptible_timeout(
725 &link->llc_confirm_rkey_resp, SMC_LLC_WAIT_TIME);
726 if (rc <= 0 || link->llc_confirm_rkey_resp_rc)
44aa81ce
KG		 727 return -EFAULT;
728 return 0;
729}
730
60e03c62
KG		 731/* unregister an rtoken at the remote peer */
732int smc_llc_do_delete_rkey(struct smc_link *link,
733 struct smc_buf_desc *rmb_desc)
734{
0b29ec64 735 int rc = 0;
60e03c62
KG		 736
737 mutex_lock(&link->llc_delete_rkey_mutex);
0b29ec64
UB		 738 if (link->state != SMC_LNK_ACTIVE)
739 goto out;
ef79d439 740 reinit_completion(&link->llc_delete_rkey_resp);
60e03c62
KG		 741 rc = smc_llc_send_delete_rkey(link, rmb_desc);
742 if (rc)
743 goto out;
744 /* receive DELETE RKEY response from server over RoCE fabric */
ef79d439
KG		 745 rc = wait_for_completion_interruptible_timeout(
746 &link->llc_delete_rkey_resp, SMC_LLC_WAIT_TIME);
747 if (rc <= 0 || link->llc_delete_rkey_resp_rc)
60e03c62
KG		 748 rc = -EFAULT;
749 else
750 rc = 0;
751out:
752 mutex_unlock(&link->llc_delete_rkey_mutex);
753 return rc;
754}
755
9bf9abea
UB		 756/***************************** init, exit, misc ******************************/
757
758static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
759 {
760 .handler = smc_llc_rx_handler,
761 .type = SMC_LLC_CONFIRM_LINK
762 },
313164da
KG		 763 {
764 .handler = smc_llc_rx_handler,
765 .type = SMC_LLC_TEST_LINK
766 },
52bedf37
KG		 767 {
768 .handler = smc_llc_rx_handler,
769 .type = SMC_LLC_ADD_LINK
770 },
771 {
772 .handler = smc_llc_rx_handler,
773 .type = SMC_LLC_DELETE_LINK
774 },
4ed75de5
KG		 775 {
776 .handler = smc_llc_rx_handler,
777 .type = SMC_LLC_CONFIRM_RKEY
778 },
779 {
780 .handler = smc_llc_rx_handler,
781 .type = SMC_LLC_CONFIRM_RKEY_CONT
782 },
783 {
784 .handler = smc_llc_rx_handler,
785 .type = SMC_LLC_DELETE_RKEY
786 },
9bf9abea
UB		 787 {
788 .handler = NULL,
789 }
790};
791
792int __init smc_llc_init(void)
793{
794 struct smc_wr_rx_handler *handler;
795 int rc = 0;
796
797 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
798 INIT_HLIST_NODE(&handler->list);
799 rc = smc_wr_rx_register_handler(handler);
800 if (rc)
801 break;
802 }
803 return rc;
804}
Linux 6.x block layer and io_uring tree(s)