projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/smc: add event-based llc_flow framework
[linux-block.git] / net / smc / smc_llc.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
9bf9abea
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
9bf9abea
UB		 7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10 * Ursula Braun <ubraun@linux.vnet.ibm.com>
11 */
12
13#include <net/tcp.h>
14#include <rdma/ib_verbs.h>
15
16#include "smc.h"
17#include "smc_core.h"
18#include "smc_clc.h"
19#include "smc_llc.h"
20
0f627126
SR		 21#define SMC_LLC_DATA_LEN 40
22
23struct smc_llc_hdr {
24 struct smc_wr_rx_hdr common;
25 u8 length; /* 44 */
52bedf37
KG		 26#if defined(__BIG_ENDIAN_BITFIELD)
27 u8 reserved:4,
28 add_link_rej_rsn:4;
29#elif defined(__LITTLE_ENDIAN_BITFIELD)
30 u8 add_link_rej_rsn:4,
31 reserved:4;
32#endif
0f627126
SR		 33 u8 flags;
34};
35
75d320d6
KG		 36#define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
37
0f627126
SR		 38struct smc_llc_msg_confirm_link { /* type 0x01 */
39 struct smc_llc_hdr hd;
40 u8 sender_mac[ETH_ALEN];
41 u8 sender_gid[SMC_GID_SIZE];
42 u8 sender_qp_num[3];
43 u8 link_num;
44 u8 link_uid[SMC_LGR_ID_SIZE];
45 u8 max_links;
46 u8 reserved[9];
47};
48
52bedf37
KG		 49#define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
50#define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
51
52#define SMC_LLC_ADD_LNK_MAX_LINKS 2
53
54struct smc_llc_msg_add_link { /* type 0x02 */
55 struct smc_llc_hdr hd;
56 u8 sender_mac[ETH_ALEN];
57 u8 reserved2[2];
58 u8 sender_gid[SMC_GID_SIZE];
59 u8 sender_qp_num[3];
60 u8 link_num;
61 u8 flags2; /* QP mtu */
62 u8 initial_psn[3];
63 u8 reserved[8];
64};
65
66#define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
67#define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
68
69struct smc_llc_msg_del_link { /* type 0x04 */
70 struct smc_llc_hdr hd;
71 u8 link_num;
72 __be32 reason;
73 u8 reserved[35];
74} __packed; /* format defined in RFC7609 */
75
313164da
KG		 76struct smc_llc_msg_test_link { /* type 0x07 */
77 struct smc_llc_hdr hd;
78 u8 user_data[16];
79 u8 reserved[24];
80};
81
4ed75de5
KG		 82struct smc_rmb_rtoken {
83 union {
84 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
85 /* is actually the num of rtokens, first */
86 /* rtoken is always for the current link */
87 u8 link_id; /* link id of the rtoken */
88 };
89 __be32 rmb_key;
90 __be64 rmb_vaddr;
91} __packed; /* format defined in RFC7609 */
92
93#define SMC_LLC_RKEYS_PER_MSG 3
94
95struct smc_llc_msg_confirm_rkey { /* type 0x06 */
96 struct smc_llc_hdr hd;
97 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
98 u8 reserved;
99};
100
101struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */
102 struct smc_llc_hdr hd;
103 u8 num_rkeys;
104 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
105};
106
107#define SMC_LLC_DEL_RKEY_MAX 8
108#define SMC_LLC_FLAG_RKEY_NEG 0x20
109
110struct smc_llc_msg_delete_rkey { /* type 0x09 */
111 struct smc_llc_hdr hd;
112 u8 num_rkeys;
113 u8 err_mask;
114 u8 reserved[2];
115 __be32 rkey[8];
116 u8 reserved2[4];
117};
118
0f627126
SR		 119union smc_llc_msg {
120 struct smc_llc_msg_confirm_link confirm_link;
52bedf37
KG		 121 struct smc_llc_msg_add_link add_link;
122 struct smc_llc_msg_del_link delete_link;
4ed75de5
KG		 123
124 struct smc_llc_msg_confirm_rkey confirm_rkey;
125 struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont;
126 struct smc_llc_msg_delete_rkey delete_rkey;
127
313164da 128 struct smc_llc_msg_test_link test_link;
0f627126
SR		 129 struct {
130 struct smc_llc_hdr hdr;
131 u8 data[SMC_LLC_DATA_LEN];
132 } raw;
133};
134
135#define SMC_LLC_FLAG_RESP 0x80
136
6c8968c4
KG		 137struct smc_llc_qentry {
138 struct list_head list;
139 struct smc_link *link;
140 union smc_llc_msg msg;
141};
142
555da9af
KG		 143struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
144{
145 struct smc_llc_qentry *qentry = flow->qentry;
146
147 flow->qentry = NULL;
148 return qentry;
149}
150
151void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
152{
153 struct smc_llc_qentry *qentry;
154
155 if (flow->qentry) {
156 qentry = flow->qentry;
157 flow->qentry = NULL;
158 kfree(qentry);
159 }
160}
161
162static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
163 struct smc_llc_qentry *qentry)
164{
165 flow->qentry = qentry;
166}
167
168/* try to start a new llc flow, initiated by an incoming llc msg */
169static bool smc_llc_flow_start(struct smc_llc_flow *flow,
170 struct smc_llc_qentry *qentry)
171{
172 struct smc_link_group *lgr = qentry->link->lgr;
173
174 spin_lock_bh(&lgr->llc_flow_lock);
175 if (flow->type) {
176 /* a flow is already active */
177 if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK ||
178 qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) &&
179 !lgr->delayed_event) {
180 lgr->delayed_event = qentry;
181 } else {
182 /* forget this llc request */
183 kfree(qentry);
184 }
185 spin_unlock_bh(&lgr->llc_flow_lock);
186 return false;
187 }
188 switch (qentry->msg.raw.hdr.common.type) {
189 case SMC_LLC_ADD_LINK:
190 flow->type = SMC_LLC_FLOW_ADD_LINK;
191 break;
192 case SMC_LLC_DELETE_LINK:
193 flow->type = SMC_LLC_FLOW_DEL_LINK;
194 break;
195 case SMC_LLC_CONFIRM_RKEY:
196 case SMC_LLC_DELETE_RKEY:
197 flow->type = SMC_LLC_FLOW_RKEY;
198 break;
199 default:
200 flow->type = SMC_LLC_FLOW_NONE;
201 }
202 if (qentry == lgr->delayed_event)
203 lgr->delayed_event = NULL;
204 spin_unlock_bh(&lgr->llc_flow_lock);
205 smc_llc_flow_qentry_set(flow, qentry);
206 return true;
207}
208
209/* start a new local llc flow, wait till current flow finished */
210int smc_llc_flow_initiate(struct smc_link_group *lgr,
211 enum smc_llc_flowtype type)
212{
213 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
214 int rc;
215
216 /* all flows except confirm_rkey and delete_rkey are exclusive,
217 * confirm/delete rkey flows can run concurrently (local and remote)
218 */
219 if (type == SMC_LLC_FLOW_RKEY)
220 allowed_remote = SMC_LLC_FLOW_RKEY;
221again:
222 if (list_empty(&lgr->list))
223 return -ENODEV;
224 spin_lock_bh(&lgr->llc_flow_lock);
225 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
226 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
227 lgr->llc_flow_rmt.type == allowed_remote)) {
228 lgr->llc_flow_lcl.type = type;
229 spin_unlock_bh(&lgr->llc_flow_lock);
230 return 0;
231 }
232 spin_unlock_bh(&lgr->llc_flow_lock);
233 rc = wait_event_interruptible_timeout(lgr->llc_waiter,
234 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
235 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
236 lgr->llc_flow_rmt.type == allowed_remote)),
237 SMC_LLC_WAIT_TIME);
238 if (!rc)
239 return -ETIMEDOUT;
240 goto again;
241}
242
243/* finish the current llc flow */
244void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
245{
246 spin_lock_bh(&lgr->llc_flow_lock);
247 memset(flow, 0, sizeof(*flow));
248 flow->type = SMC_LLC_FLOW_NONE;
249 spin_unlock_bh(&lgr->llc_flow_lock);
250 if (!list_empty(&lgr->list) && lgr->delayed_event &&
251 flow == &lgr->llc_flow_lcl)
252 schedule_work(&lgr->llc_event_work);
253 else
254 wake_up_interruptible(&lgr->llc_waiter);
255}
256
257/* lnk is optional and used for early wakeup when link goes down, useful in
258 * cases where we wait for a response on the link after we sent a request
259 */
260struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
261 struct smc_link *lnk,
262 int time_out, u8 exp_msg)
263{
264 struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
265
266 wait_event_interruptible_timeout(lgr->llc_waiter,
267 (flow->qentry ||
268 (lnk && !smc_link_usable(lnk)) ||
269 list_empty(&lgr->list)),
270 time_out);
271 if (!flow->qentry ||
272 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
273 smc_llc_flow_qentry_del(flow);
274 goto out;
275 }
276 if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) {
277 if (exp_msg == SMC_LLC_ADD_LINK &&
278 flow->qentry->msg.raw.hdr.common.type ==
279 SMC_LLC_DELETE_LINK) {
280 /* flow_start will delay the unexpected msg */
281 smc_llc_flow_start(&lgr->llc_flow_lcl,
282 smc_llc_flow_qentry_clr(flow));
283 return NULL;
284 }
285 smc_llc_flow_qentry_del(flow);
286 }
287out:
288 return flow->qentry;
289}
290
9bf9abea
UB		 291/********************************** send *************************************/
292
293struct smc_llc_tx_pend {
294};
295
296/* handler for send/transmission completion of an LLC msg */
297static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
298 struct smc_link *link,
299 enum ib_wc_status wc_status)
300{
301 /* future work: handle wc_status error for recovery and failover */
302}
303
304/**
305 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
306 * @link: Pointer to SMC link used for sending LLC control message.
307 * @wr_buf: Out variable returning pointer to work request payload buffer.
308 * @pend: Out variable returning pointer to private pending WR tracking.
309 * It's the context the transmit complete handler will get.
310 *
311 * Reserves and pre-fills an entry for a pending work request send/tx.
312 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
313 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
314 *
315 * Return: 0 on success, otherwise an error value.
316 */
317static int smc_llc_add_pending_send(struct smc_link *link,
318 struct smc_wr_buf **wr_buf,
319 struct smc_wr_tx_pend_priv **pend)
320{
321 int rc;
322
ad6f317f
UB		 323 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
324 pend);
9bf9abea
UB		 325 if (rc < 0)
326 return rc;
327 BUILD_BUG_ON_MSG(
328 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
329 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
330 BUILD_BUG_ON_MSG(
331 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
332 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
333 BUILD_BUG_ON_MSG(
334 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
335 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
336 return 0;
337}
338
339/* high-level API to send LLC confirm link */
947541f3 340int smc_llc_send_confirm_link(struct smc_link *link,
9bf9abea
UB		 341 enum smc_llc_reqresp reqresp)
342{
00e5fb26 343 struct smc_link_group *lgr = smc_get_lgr(link);
9bf9abea
UB		 344 struct smc_llc_msg_confirm_link *confllc;
345 struct smc_wr_tx_pend_priv *pend;
346 struct smc_wr_buf *wr_buf;
347 int rc;
348
349 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
350 if (rc)
351 return rc;
352 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
353 memset(confllc, 0, sizeof(*confllc));
354 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
355 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
75d320d6 356 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
9bf9abea
UB		 357 if (reqresp == SMC_LLC_RESP)
358 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
947541f3
UB		 359 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
360 ETH_ALEN);
7005ada6 361 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
9bf9abea 362 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
2be922f3 363 confllc->link_num = link->link_id;
9bf9abea 364 memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
52bedf37
KG		 365 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
366 /* send llc message */
367 rc = smc_wr_tx_send(link, pend);
368 return rc;
369}
370
44aa81ce
KG		 371/* send LLC confirm rkey request */
372static int smc_llc_send_confirm_rkey(struct smc_link *link,
373 struct smc_buf_desc *rmb_desc)
374{
375 struct smc_llc_msg_confirm_rkey *rkeyllc;
376 struct smc_wr_tx_pend_priv *pend;
377 struct smc_wr_buf *wr_buf;
378 int rc;
379
380 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
381 if (rc)
382 return rc;
383 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
384 memset(rkeyllc, 0, sizeof(*rkeyllc));
385 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
386 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
387 rkeyllc->rtoken[0].rmb_key =
387707fd 388 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
44aa81ce 389 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
387707fd 390 (u64)sg_dma_address(rmb_desc->sgt[link->link_idx].sgl));
44aa81ce
KG		 391 /* send llc message */
392 rc = smc_wr_tx_send(link, pend);
393 return rc;
394}
395
60e03c62
KG		 396/* send LLC delete rkey request */
397static int smc_llc_send_delete_rkey(struct smc_link *link,
398 struct smc_buf_desc *rmb_desc)
399{
400 struct smc_llc_msg_delete_rkey *rkeyllc;
401 struct smc_wr_tx_pend_priv *pend;
402 struct smc_wr_buf *wr_buf;
403 int rc;
404
405 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
406 if (rc)
407 return rc;
408 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
409 memset(rkeyllc, 0, sizeof(*rkeyllc));
410 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
411 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
412 rkeyllc->num_rkeys = 1;
387707fd 413 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
60e03c62
KG		 414 /* send llc message */
415 rc = smc_wr_tx_send(link, pend);
416 return rc;
417}
418
2a4c57a9
KG		 419/* prepare an add link message */
420static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc,
7005ada6 421 struct smc_link *link, u8 mac[], u8 gid[],
2a4c57a9
KG		 422 enum smc_llc_reqresp reqresp)
423{
424 memset(addllc, 0, sizeof(*addllc));
425 addllc->hd.common.type = SMC_LLC_ADD_LINK;
426 addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
427 if (reqresp == SMC_LLC_RESP) {
428 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
429 /* always reject more links for now */
430 addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
431 addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
432 }
433 memcpy(addllc->sender_mac, mac, ETH_ALEN);
434 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
435}
436
52bedf37 437/* send ADD LINK request or response */
7005ada6 438int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
52bedf37
KG		 439 enum smc_llc_reqresp reqresp)
440{
441 struct smc_llc_msg_add_link *addllc;
442 struct smc_wr_tx_pend_priv *pend;
443 struct smc_wr_buf *wr_buf;
444 int rc;
445
446 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
447 if (rc)
448 return rc;
449 addllc = (struct smc_llc_msg_add_link *)wr_buf;
2a4c57a9 450 smc_llc_prep_add_link(addllc, link, mac, gid, reqresp);
52bedf37
KG		 451 /* send llc message */
452 rc = smc_wr_tx_send(link, pend);
453 return rc;
454}
455
2a4c57a9
KG		 456/* prepare a delete link message */
457static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc,
458 struct smc_link *link,
0d18a0cb 459 enum smc_llc_reqresp reqresp, bool orderly)
2a4c57a9
KG		 460{
461 memset(delllc, 0, sizeof(*delllc));
462 delllc->hd.common.type = SMC_LLC_DELETE_LINK;
463 delllc->hd.length = sizeof(struct smc_llc_msg_add_link);
464 if (reqresp == SMC_LLC_RESP)
465 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
466 /* DEL_LINK_ALL because only 1 link supported */
467 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
0d18a0cb
KG		 468 if (orderly)
469 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
2a4c57a9
KG		 470 delllc->link_num = link->link_id;
471}
472
52bedf37
KG		 473/* send DELETE LINK request or response */
474int smc_llc_send_delete_link(struct smc_link *link,
0d18a0cb 475 enum smc_llc_reqresp reqresp, bool orderly)
52bedf37
KG		 476{
477 struct smc_llc_msg_del_link *delllc;
478 struct smc_wr_tx_pend_priv *pend;
479 struct smc_wr_buf *wr_buf;
480 int rc;
481
482 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
483 if (rc)
484 return rc;
485 delllc = (struct smc_llc_msg_del_link *)wr_buf;
0d18a0cb 486 smc_llc_prep_delete_link(delllc, link, reqresp, orderly);
9bf9abea
UB		 487 /* send llc message */
488 rc = smc_wr_tx_send(link, pend);
489 return rc;
490}
491
d97935fa
KG		 492/* send LLC test link request */
493static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
313164da
KG		 494{
495 struct smc_llc_msg_test_link *testllc;
496 struct smc_wr_tx_pend_priv *pend;
497 struct smc_wr_buf *wr_buf;
498 int rc;
499
500 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
501 if (rc)
502 return rc;
503 testllc = (struct smc_llc_msg_test_link *)wr_buf;
504 memset(testllc, 0, sizeof(*testllc));
505 testllc->hd.common.type = SMC_LLC_TEST_LINK;
506 testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
313164da
KG		 507 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
508 /* send llc message */
509 rc = smc_wr_tx_send(link, pend);
510 return rc;
511}
512
6c8968c4
KG		 513/* schedule an llc send on link, may wait for buffers */
514static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
4ed75de5
KG		 515{
516 struct smc_wr_tx_pend_priv *pend;
517 struct smc_wr_buf *wr_buf;
518 int rc;
519
6c8968c4
KG		 520 if (!smc_link_usable(link))
521 return -ENOLINK;
522 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
4ed75de5 523 if (rc)
6c8968c4
KG		 524 return rc;
525 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
526 return smc_wr_tx_send(link, pend);
4ed75de5
KG		 527}
528
9bf9abea
UB		 529/********************************* receive ***********************************/
530
531static void smc_llc_rx_confirm_link(struct smc_link *link,
532 struct smc_llc_msg_confirm_link *llc)
533{
00e5fb26 534 struct smc_link_group *lgr = smc_get_lgr(link);
ef79d439 535 int conf_rc = 0;
9bf9abea 536
75d320d6 537 /* RMBE eyecatchers are not supported */
ef79d439 538 if (!(llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
75d320d6
KG		 539 conf_rc = ENOTSUPP;
540
ef79d439
KG		 541 if (lgr->role == SMC_CLNT &&
542 link->state == SMC_LNK_ACTIVATING) {
543 link->llc_confirm_rc = conf_rc;
544 link->link_id = llc->link_num;
545 complete(&link->llc_confirm);
9bf9abea
UB		 546 }
547}
548
52bedf37
KG		 549static void smc_llc_rx_add_link(struct smc_link *link,
550 struct smc_llc_msg_add_link *llc)
551{
00e5fb26 552 struct smc_link_group *lgr = smc_get_lgr(link);
52bedf37 553
ef79d439
KG		 554 if (link->state == SMC_LNK_ACTIVATING) {
555 complete(&link->llc_add);
556 return;
557 }
52bedf37 558
ef79d439
KG		 559 if (lgr->role == SMC_SERV) {
560 smc_llc_prep_add_link(llc, link,
561 link->smcibdev->mac[link->ibport - 1],
562 link->gid, SMC_LLC_REQ);
52bedf37 563
ef79d439
KG		 564 } else {
565 smc_llc_prep_add_link(llc, link,
566 link->smcibdev->mac[link->ibport - 1],
567 link->gid, SMC_LLC_RESP);
52bedf37 568 }
ef79d439 569 smc_llc_send_message(link, llc);
52bedf37
KG		 570}
571
572static void smc_llc_rx_delete_link(struct smc_link *link,
573 struct smc_llc_msg_del_link *llc)
574{
00e5fb26 575 struct smc_link_group *lgr = smc_get_lgr(link);
52bedf37 576
ef79d439
KG		 577 smc_lgr_forget(lgr);
578 smc_llc_link_deleting(link);
579 if (lgr->role == SMC_SERV) {
580 /* client asks to delete this link, send request */
581 smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true);
52bedf37 582 } else {
ef79d439
KG		 583 /* server requests to delete this link, send response */
584 smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true);
52bedf37 585 }
ef79d439
KG		 586 smc_llc_send_message(link, llc);
587 smc_lgr_terminate_sched(lgr);
52bedf37
KG		 588}
589
313164da
KG		 590static void smc_llc_rx_test_link(struct smc_link *link,
591 struct smc_llc_msg_test_link *llc)
592{
ef79d439
KG		 593 llc->hd.flags |= SMC_LLC_FLAG_RESP;
594 smc_llc_send_message(link, llc);
313164da
KG		 595}
596
4ed75de5
KG		 597static void smc_llc_rx_confirm_rkey(struct smc_link *link,
598 struct smc_llc_msg_confirm_rkey *llc)
599{
4ed75de5
KG		 600 int rc;
601
ef79d439
KG		 602 rc = smc_rtoken_add(link,
603 llc->rtoken[0].rmb_vaddr,
604 llc->rtoken[0].rmb_key);
4ed75de5 605
ef79d439 606 /* ignore rtokens for other links, we have only one link */
4ed75de5 607
ef79d439
KG		 608 llc->hd.flags |= SMC_LLC_FLAG_RESP;
609 if (rc < 0)
610 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
611 smc_llc_send_message(link, llc);
4ed75de5
KG		 612}
613
614static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link,
615 struct smc_llc_msg_confirm_rkey_cont *llc)
616{
ef79d439
KG		 617 /* ignore rtokens for other links, we have only one link */
618 llc->hd.flags |= SMC_LLC_FLAG_RESP;
619 smc_llc_send_message(link, llc);
4ed75de5
KG		 620}
621
622static void smc_llc_rx_delete_rkey(struct smc_link *link,
623 struct smc_llc_msg_delete_rkey *llc)
624{
4ed75de5
KG		 625 u8 err_mask = 0;
626 int i, max;
627
ef79d439
KG		 628 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
629 for (i = 0; i < max; i++) {
630 if (smc_rtoken_delete(link, llc->rkey[i]))
631 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
632 }
4ed75de5 633
ef79d439
KG		 634 if (err_mask) {
635 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
636 llc->err_mask = err_mask;
4ed75de5 637 }
ef79d439
KG		 638
639 llc->hd.flags |= SMC_LLC_FLAG_RESP;
640 smc_llc_send_message(link, llc);
4ed75de5
KG		 641}
642
6c8968c4 643/* flush the llc event queue */
00a049cf 644static void smc_llc_event_flush(struct smc_link_group *lgr)
9bf9abea 645{
6c8968c4
KG		 646 struct smc_llc_qentry *qentry, *q;
647
648 spin_lock_bh(&lgr->llc_event_q_lock);
649 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
650 list_del_init(&qentry->list);
651 kfree(qentry);
652 }
653 spin_unlock_bh(&lgr->llc_event_q_lock);
654}
655
656static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
657{
658 union smc_llc_msg *llc = &qentry->msg;
659 struct smc_link *link = qentry->link;
9bf9abea 660
d854fcbf 661 if (!smc_link_usable(link))
6c8968c4 662 goto out;
313164da
KG		 663
664 switch (llc->raw.hdr.common.type) {
665 case SMC_LLC_TEST_LINK:
666 smc_llc_rx_test_link(link, &llc->test_link);
667 break;
668 case SMC_LLC_CONFIRM_LINK:
9bf9abea 669 smc_llc_rx_confirm_link(link, &llc->confirm_link);
313164da 670 break;
52bedf37
KG		 671 case SMC_LLC_ADD_LINK:
672 smc_llc_rx_add_link(link, &llc->add_link);
673 break;
674 case SMC_LLC_DELETE_LINK:
675 smc_llc_rx_delete_link(link, &llc->delete_link);
676 break;
4ed75de5
KG		 677 case SMC_LLC_CONFIRM_RKEY:
678 smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey);
679 break;
680 case SMC_LLC_CONFIRM_RKEY_CONT:
681 smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont);
682 break;
683 case SMC_LLC_DELETE_RKEY:
684 smc_llc_rx_delete_rkey(link, &llc->delete_rkey);
685 break;
313164da 686 }
6c8968c4
KG		 687out:
688 kfree(qentry);
689}
690
691/* worker to process llc messages on the event queue */
692static void smc_llc_event_work(struct work_struct *work)
693{
694 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
695 llc_event_work);
696 struct smc_llc_qentry *qentry;
697
555da9af
KG		 698 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
699 if (smc_link_usable(lgr->delayed_event->link)) {
700 smc_llc_event_handler(lgr->delayed_event);
701 } else {
702 qentry = lgr->delayed_event;
703 lgr->delayed_event = NULL;
704 kfree(qentry);
705 }
706 }
707
6c8968c4
KG		 708again:
709 spin_lock_bh(&lgr->llc_event_q_lock);
710 if (!list_empty(&lgr->llc_event_q)) {
711 qentry = list_first_entry(&lgr->llc_event_q,
712 struct smc_llc_qentry, list);
713 list_del_init(&qentry->list);
714 spin_unlock_bh(&lgr->llc_event_q_lock);
715 smc_llc_event_handler(qentry);
716 goto again;
717 }
718 spin_unlock_bh(&lgr->llc_event_q_lock);
719}
720
ef79d439
KG		 721/* process llc responses in tasklet context */
722static void smc_llc_rx_response(struct smc_link *link, union smc_llc_msg *llc)
723{
724 int rc = 0;
725
726 switch (llc->raw.hdr.common.type) {
727 case SMC_LLC_TEST_LINK:
728 if (link->state == SMC_LNK_ACTIVE)
729 complete(&link->llc_testlink_resp);
730 break;
731 case SMC_LLC_CONFIRM_LINK:
732 if (!(llc->raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
733 rc = ENOTSUPP;
734 if (link->lgr->role == SMC_SERV &&
735 link->state == SMC_LNK_ACTIVATING) {
736 link->llc_confirm_resp_rc = rc;
737 complete(&link->llc_confirm_resp);
738 }
739 break;
740 case SMC_LLC_ADD_LINK:
741 if (link->state == SMC_LNK_ACTIVATING)
742 complete(&link->llc_add_resp);
743 break;
744 case SMC_LLC_DELETE_LINK:
745 if (link->lgr->role == SMC_SERV)
746 smc_lgr_schedule_free_work_fast(link->lgr);
747 break;
748 case SMC_LLC_CONFIRM_RKEY:
749 link->llc_confirm_rkey_resp_rc = llc->raw.hdr.flags &
750 SMC_LLC_FLAG_RKEY_NEG;
751 complete(&link->llc_confirm_rkey_resp);
752 break;
753 case SMC_LLC_CONFIRM_RKEY_CONT:
754 /* unused as long as we don't send this type of msg */
755 break;
756 case SMC_LLC_DELETE_RKEY:
757 link->llc_delete_rkey_resp_rc = llc->raw.hdr.flags &
758 SMC_LLC_FLAG_RKEY_NEG;
759 complete(&link->llc_delete_rkey_resp);
760 break;
761 }
762}
763
6c8968c4
KG		 764/* copy received msg and add it to the event queue */
765static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
766{
767 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
768 struct smc_link_group *lgr = link->lgr;
769 struct smc_llc_qentry *qentry;
770 union smc_llc_msg *llc = buf;
771 unsigned long flags;
772
773 if (wc->byte_len < sizeof(*llc))
774 return; /* short message */
775 if (llc->raw.hdr.length != sizeof(*llc))
776 return; /* invalid message */
777
ef79d439
KG		 778 /* process responses immediately */
779 if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
780 smc_llc_rx_response(link, llc);
781 return;
782 }
783
6c8968c4
KG		 784 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
785 if (!qentry)
786 return;
787 qentry->link = link;
788 INIT_LIST_HEAD(&qentry->list);
789 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
790 spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
791 list_add_tail(&qentry->list, &lgr->llc_event_q);
792 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
793 schedule_work(&link->lgr->llc_event_work);
9bf9abea
UB		 794}
795
44aa81ce 796/***************************** worker, utils *********************************/
877ae5be
KG		 797
798static void smc_llc_testlink_work(struct work_struct *work)
799{
800 struct smc_link *link = container_of(to_delayed_work(work),
801 struct smc_link, llc_testlink_wrk);
802 unsigned long next_interval;
877ae5be
KG		 803 unsigned long expire_time;
804 u8 user_data[16] = { 0 };
805 int rc;
806
877ae5be
KG		 807 if (link->state != SMC_LNK_ACTIVE)
808 return; /* don't reschedule worker */
809 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
810 if (time_is_after_jiffies(expire_time)) {
811 next_interval = expire_time - jiffies;
812 goto out;
813 }
814 reinit_completion(&link->llc_testlink_resp);
d97935fa 815 smc_llc_send_test_link(link, user_data);
877ae5be
KG		 816 /* receive TEST LINK response over RoCE fabric */
817 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
818 SMC_LLC_WAIT_TIME);
1020e1ef
KG		 819 if (link->state != SMC_LNK_ACTIVE)
820 return; /* link state changed */
877ae5be 821 if (rc <= 0) {
5f78fe96 822 smc_lgr_terminate_sched(smc_get_lgr(link));
877ae5be
KG		 823 return;
824 }
825 next_interval = link->llc_testlink_time;
826out:
1020e1ef 827 schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
877ae5be
KG		 828}
829
00a049cf
KG		 830void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
831{
832 struct net *net = sock_net(smc->clcsock->sk);
833
834 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
835 INIT_LIST_HEAD(&lgr->llc_event_q);
836 spin_lock_init(&lgr->llc_event_q_lock);
555da9af
KG		 837 spin_lock_init(&lgr->llc_flow_lock);
838 init_waitqueue_head(&lgr->llc_waiter);
00a049cf
KG		 839 lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
840}
841
842/* called after lgr was removed from lgr_list */
843void smc_llc_lgr_clear(struct smc_link_group *lgr)
844{
845 smc_llc_event_flush(lgr);
555da9af 846 wake_up_interruptible_all(&lgr->llc_waiter);
00a049cf 847 cancel_work_sync(&lgr->llc_event_work);
555da9af
KG		 848 if (lgr->delayed_event) {
849 kfree(lgr->delayed_event);
850 lgr->delayed_event = NULL;
851 }
00a049cf
KG		 852}
853
2a4c57a9 854int smc_llc_link_init(struct smc_link *link)
877ae5be 855{
b32cf4ab
KG		 856 init_completion(&link->llc_confirm);
857 init_completion(&link->llc_confirm_resp);
858 init_completion(&link->llc_add);
859 init_completion(&link->llc_add_resp);
ef79d439
KG		 860 init_completion(&link->llc_confirm_rkey_resp);
861 init_completion(&link->llc_delete_rkey_resp);
60e03c62 862 mutex_init(&link->llc_delete_rkey_mutex);
877ae5be
KG		 863 init_completion(&link->llc_testlink_resp);
864 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2a4c57a9 865 return 0;
b32cf4ab
KG		 866}
867
00a049cf 868void smc_llc_link_active(struct smc_link *link)
b32cf4ab 869{
877ae5be 870 link->state = SMC_LNK_ACTIVE;
00a049cf
KG		 871 if (link->lgr->llc_testlink_time) {
872 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ;
1020e1ef
KG		 873 schedule_delayed_work(&link->llc_testlink_wrk,
874 link->llc_testlink_time);
877ae5be
KG		 875 }
876}
877
0d18a0cb
KG		 878void smc_llc_link_deleting(struct smc_link *link)
879{
880 link->state = SMC_LNK_DELETING;
15e1b99a 881 smc_wr_wakeup_tx_wait(link);
0d18a0cb
KG		 882}
883
877ae5be 884/* called in worker context */
2a4c57a9 885void smc_llc_link_clear(struct smc_link *link)
877ae5be 886{
2140ac26
KG		 887 complete(&link->llc_testlink_resp);
888 cancel_delayed_work_sync(&link->llc_testlink_wrk);
889 smc_wr_wakeup_reg_wait(link);
890 smc_wr_wakeup_tx_wait(link);
877ae5be
KG		 891}
892
44aa81ce
KG		 893/* register a new rtoken at the remote peer */
894int smc_llc_do_confirm_rkey(struct smc_link *link,
895 struct smc_buf_desc *rmb_desc)
896{
897 int rc;
898
60e03c62 899 /* protected by mutex smc_create_lgr_pending */
ef79d439 900 reinit_completion(&link->llc_confirm_rkey_resp);
4600cfc3
KG		 901 rc = smc_llc_send_confirm_rkey(link, rmb_desc);
902 if (rc)
903 return rc;
44aa81ce 904 /* receive CONFIRM RKEY response from server over RoCE fabric */
ef79d439
KG		 905 rc = wait_for_completion_interruptible_timeout(
906 &link->llc_confirm_rkey_resp, SMC_LLC_WAIT_TIME);
907 if (rc <= 0 || link->llc_confirm_rkey_resp_rc)
44aa81ce
KG		 908 return -EFAULT;
909 return 0;
910}
911
60e03c62
KG		 912/* unregister an rtoken at the remote peer */
913int smc_llc_do_delete_rkey(struct smc_link *link,
914 struct smc_buf_desc *rmb_desc)
915{
0b29ec64 916 int rc = 0;
60e03c62
KG		 917
918 mutex_lock(&link->llc_delete_rkey_mutex);
0b29ec64
UB		 919 if (link->state != SMC_LNK_ACTIVE)
920 goto out;
ef79d439 921 reinit_completion(&link->llc_delete_rkey_resp);
60e03c62
KG		 922 rc = smc_llc_send_delete_rkey(link, rmb_desc);
923 if (rc)
924 goto out;
925 /* receive DELETE RKEY response from server over RoCE fabric */
ef79d439
KG		 926 rc = wait_for_completion_interruptible_timeout(
927 &link->llc_delete_rkey_resp, SMC_LLC_WAIT_TIME);
928 if (rc <= 0 || link->llc_delete_rkey_resp_rc)
60e03c62
KG		 929 rc = -EFAULT;
930 else
931 rc = 0;
932out:
933 mutex_unlock(&link->llc_delete_rkey_mutex);
934 return rc;
935}
936
9bf9abea
UB		 937/***************************** init, exit, misc ******************************/
938
939static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
940 {
941 .handler = smc_llc_rx_handler,
942 .type = SMC_LLC_CONFIRM_LINK
943 },
313164da
KG		 944 {
945 .handler = smc_llc_rx_handler,
946 .type = SMC_LLC_TEST_LINK
947 },
52bedf37
KG		 948 {
949 .handler = smc_llc_rx_handler,
950 .type = SMC_LLC_ADD_LINK
951 },
952 {
953 .handler = smc_llc_rx_handler,
954 .type = SMC_LLC_DELETE_LINK
955 },
4ed75de5
KG		 956 {
957 .handler = smc_llc_rx_handler,
958 .type = SMC_LLC_CONFIRM_RKEY
959 },
960 {
961 .handler = smc_llc_rx_handler,
962 .type = SMC_LLC_CONFIRM_RKEY_CONT
963 },
964 {
965 .handler = smc_llc_rx_handler,
966 .type = SMC_LLC_DELETE_RKEY
967 },
9bf9abea
UB		 968 {
969 .handler = NULL,
970 }
971};
972
973int __init smc_llc_init(void)
974{
975 struct smc_wr_rx_handler *handler;
976 int rc = 0;
977
978 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
979 INIT_HLIST_NODE(&handler->list);
980 rc = smc_wr_rx_register_handler(handler);
981 if (rc)
982 break;
983 }
984 return rc;
985}
Linux 6.x block layer and io_uring tree(s)