projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/smc: allocate index for a new link
[linux-block.git] / net / smc / smc_llc.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
9bf9abea
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
9bf9abea
UB		 7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10 * Ursula Braun <ubraun@linux.vnet.ibm.com>
11 */
12
13#include <net/tcp.h>
14#include <rdma/ib_verbs.h>
15
16#include "smc.h"
17#include "smc_core.h"
18#include "smc_clc.h"
19#include "smc_llc.h"
20
0f627126
SR		 21#define SMC_LLC_DATA_LEN 40
22
23struct smc_llc_hdr {
24 struct smc_wr_rx_hdr common;
25 u8 length; /* 44 */
52bedf37
KG		 26#if defined(__BIG_ENDIAN_BITFIELD)
27 u8 reserved:4,
28 add_link_rej_rsn:4;
29#elif defined(__LITTLE_ENDIAN_BITFIELD)
30 u8 add_link_rej_rsn:4,
31 reserved:4;
32#endif
0f627126
SR		 33 u8 flags;
34};
35
75d320d6
KG		 36#define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
37
0f627126
SR		 38struct smc_llc_msg_confirm_link { /* type 0x01 */
39 struct smc_llc_hdr hd;
40 u8 sender_mac[ETH_ALEN];
41 u8 sender_gid[SMC_GID_SIZE];
42 u8 sender_qp_num[3];
43 u8 link_num;
44 u8 link_uid[SMC_LGR_ID_SIZE];
45 u8 max_links;
46 u8 reserved[9];
47};
48
52bedf37
KG		 49#define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
50#define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
51
52#define SMC_LLC_ADD_LNK_MAX_LINKS 2
53
54struct smc_llc_msg_add_link { /* type 0x02 */
55 struct smc_llc_hdr hd;
56 u8 sender_mac[ETH_ALEN];
57 u8 reserved2[2];
58 u8 sender_gid[SMC_GID_SIZE];
59 u8 sender_qp_num[3];
60 u8 link_num;
fbed3b37
KG		 61#if defined(__BIG_ENDIAN_BITFIELD)
62 u8 reserved3 : 4,
63 qp_mtu : 4;
64#elif defined(__LITTLE_ENDIAN_BITFIELD)
65 u8 qp_mtu : 4,
66 reserved3 : 4;
67#endif
52bedf37
KG		 68 u8 initial_psn[3];
69 u8 reserved[8];
70};
71
72#define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
73#define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
74
75struct smc_llc_msg_del_link { /* type 0x04 */
76 struct smc_llc_hdr hd;
77 u8 link_num;
78 __be32 reason;
79 u8 reserved[35];
80} __packed; /* format defined in RFC7609 */
81
313164da
KG		 82struct smc_llc_msg_test_link { /* type 0x07 */
83 struct smc_llc_hdr hd;
84 u8 user_data[16];
85 u8 reserved[24];
86};
87
4ed75de5
KG		 88struct smc_rmb_rtoken {
89 union {
90 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
91 /* is actually the num of rtokens, first */
92 /* rtoken is always for the current link */
93 u8 link_id; /* link id of the rtoken */
94 };
95 __be32 rmb_key;
96 __be64 rmb_vaddr;
97} __packed; /* format defined in RFC7609 */
98
99#define SMC_LLC_RKEYS_PER_MSG 3
100
101struct smc_llc_msg_confirm_rkey { /* type 0x06 */
102 struct smc_llc_hdr hd;
103 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
104 u8 reserved;
105};
106
4ed75de5 107#define SMC_LLC_DEL_RKEY_MAX 8
3bc67e09 108#define SMC_LLC_FLAG_RKEY_RETRY 0x10
4ed75de5
KG		 109#define SMC_LLC_FLAG_RKEY_NEG 0x20
110
111struct smc_llc_msg_delete_rkey { /* type 0x09 */
112 struct smc_llc_hdr hd;
113 u8 num_rkeys;
114 u8 err_mask;
115 u8 reserved[2];
116 __be32 rkey[8];
117 u8 reserved2[4];
118};
119
0f627126
SR		 120union smc_llc_msg {
121 struct smc_llc_msg_confirm_link confirm_link;
52bedf37
KG		 122 struct smc_llc_msg_add_link add_link;
123 struct smc_llc_msg_del_link delete_link;
4ed75de5
KG		 124
125 struct smc_llc_msg_confirm_rkey confirm_rkey;
4ed75de5
KG		 126 struct smc_llc_msg_delete_rkey delete_rkey;
127
313164da 128 struct smc_llc_msg_test_link test_link;
0f627126
SR		 129 struct {
130 struct smc_llc_hdr hdr;
131 u8 data[SMC_LLC_DATA_LEN];
132 } raw;
133};
134
135#define SMC_LLC_FLAG_RESP 0x80
136
6c8968c4
KG		 137struct smc_llc_qentry {
138 struct list_head list;
139 struct smc_link *link;
140 union smc_llc_msg msg;
141};
142
555da9af
KG		 143struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
144{
145 struct smc_llc_qentry *qentry = flow->qentry;
146
147 flow->qentry = NULL;
148 return qentry;
149}
150
151void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
152{
153 struct smc_llc_qentry *qentry;
154
155 if (flow->qentry) {
156 qentry = flow->qentry;
157 flow->qentry = NULL;
158 kfree(qentry);
159 }
160}
161
162static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
163 struct smc_llc_qentry *qentry)
164{
165 flow->qentry = qentry;
166}
167
168/* try to start a new llc flow, initiated by an incoming llc msg */
169static bool smc_llc_flow_start(struct smc_llc_flow *flow,
170 struct smc_llc_qentry *qentry)
171{
172 struct smc_link_group *lgr = qentry->link->lgr;
173
174 spin_lock_bh(&lgr->llc_flow_lock);
175 if (flow->type) {
176 /* a flow is already active */
177 if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK ||
178 qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) &&
179 !lgr->delayed_event) {
180 lgr->delayed_event = qentry;
181 } else {
182 /* forget this llc request */
183 kfree(qentry);
184 }
185 spin_unlock_bh(&lgr->llc_flow_lock);
186 return false;
187 }
188 switch (qentry->msg.raw.hdr.common.type) {
189 case SMC_LLC_ADD_LINK:
190 flow->type = SMC_LLC_FLOW_ADD_LINK;
191 break;
192 case SMC_LLC_DELETE_LINK:
193 flow->type = SMC_LLC_FLOW_DEL_LINK;
194 break;
195 case SMC_LLC_CONFIRM_RKEY:
196 case SMC_LLC_DELETE_RKEY:
197 flow->type = SMC_LLC_FLOW_RKEY;
198 break;
199 default:
200 flow->type = SMC_LLC_FLOW_NONE;
201 }
202 if (qentry == lgr->delayed_event)
203 lgr->delayed_event = NULL;
204 spin_unlock_bh(&lgr->llc_flow_lock);
205 smc_llc_flow_qentry_set(flow, qentry);
206 return true;
207}
208
209/* start a new local llc flow, wait till current flow finished */
210int smc_llc_flow_initiate(struct smc_link_group *lgr,
211 enum smc_llc_flowtype type)
212{
213 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
214 int rc;
215
216 /* all flows except confirm_rkey and delete_rkey are exclusive,
217 * confirm/delete rkey flows can run concurrently (local and remote)
218 */
219 if (type == SMC_LLC_FLOW_RKEY)
220 allowed_remote = SMC_LLC_FLOW_RKEY;
221again:
222 if (list_empty(&lgr->list))
223 return -ENODEV;
224 spin_lock_bh(&lgr->llc_flow_lock);
225 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
226 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
227 lgr->llc_flow_rmt.type == allowed_remote)) {
228 lgr->llc_flow_lcl.type = type;
229 spin_unlock_bh(&lgr->llc_flow_lock);
230 return 0;
231 }
232 spin_unlock_bh(&lgr->llc_flow_lock);
233 rc = wait_event_interruptible_timeout(lgr->llc_waiter,
234 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
235 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
236 lgr->llc_flow_rmt.type == allowed_remote)),
237 SMC_LLC_WAIT_TIME);
238 if (!rc)
239 return -ETIMEDOUT;
240 goto again;
241}
242
243/* finish the current llc flow */
244void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
245{
246 spin_lock_bh(&lgr->llc_flow_lock);
247 memset(flow, 0, sizeof(*flow));
248 flow->type = SMC_LLC_FLOW_NONE;
249 spin_unlock_bh(&lgr->llc_flow_lock);
250 if (!list_empty(&lgr->list) && lgr->delayed_event &&
251 flow == &lgr->llc_flow_lcl)
252 schedule_work(&lgr->llc_event_work);
253 else
254 wake_up_interruptible(&lgr->llc_waiter);
255}
256
257/* lnk is optional and used for early wakeup when link goes down, useful in
258 * cases where we wait for a response on the link after we sent a request
259 */
260struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
261 struct smc_link *lnk,
262 int time_out, u8 exp_msg)
263{
264 struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
265
266 wait_event_interruptible_timeout(lgr->llc_waiter,
267 (flow->qentry ||
268 (lnk && !smc_link_usable(lnk)) ||
269 list_empty(&lgr->list)),
270 time_out);
271 if (!flow->qentry ||
272 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
273 smc_llc_flow_qentry_del(flow);
274 goto out;
275 }
276 if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) {
277 if (exp_msg == SMC_LLC_ADD_LINK &&
278 flow->qentry->msg.raw.hdr.common.type ==
279 SMC_LLC_DELETE_LINK) {
280 /* flow_start will delay the unexpected msg */
281 smc_llc_flow_start(&lgr->llc_flow_lcl,
282 smc_llc_flow_qentry_clr(flow));
283 return NULL;
284 }
285 smc_llc_flow_qentry_del(flow);
286 }
287out:
288 return flow->qentry;
289}
290
9bf9abea
UB		 291/********************************** send *************************************/
292
293struct smc_llc_tx_pend {
294};
295
296/* handler for send/transmission completion of an LLC msg */
297static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
298 struct smc_link *link,
299 enum ib_wc_status wc_status)
300{
301 /* future work: handle wc_status error for recovery and failover */
302}
303
304/**
305 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
306 * @link: Pointer to SMC link used for sending LLC control message.
307 * @wr_buf: Out variable returning pointer to work request payload buffer.
308 * @pend: Out variable returning pointer to private pending WR tracking.
309 * It's the context the transmit complete handler will get.
310 *
311 * Reserves and pre-fills an entry for a pending work request send/tx.
312 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
313 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
314 *
315 * Return: 0 on success, otherwise an error value.
316 */
317static int smc_llc_add_pending_send(struct smc_link *link,
318 struct smc_wr_buf **wr_buf,
319 struct smc_wr_tx_pend_priv **pend)
320{
321 int rc;
322
ad6f317f
UB		 323 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
324 pend);
9bf9abea
UB		 325 if (rc < 0)
326 return rc;
327 BUILD_BUG_ON_MSG(
328 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
329 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
330 BUILD_BUG_ON_MSG(
331 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
332 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
333 BUILD_BUG_ON_MSG(
334 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
335 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
336 return 0;
337}
338
339/* high-level API to send LLC confirm link */
947541f3 340int smc_llc_send_confirm_link(struct smc_link *link,
9bf9abea
UB		 341 enum smc_llc_reqresp reqresp)
342{
00e5fb26 343 struct smc_link_group *lgr = smc_get_lgr(link);
9bf9abea
UB		 344 struct smc_llc_msg_confirm_link *confllc;
345 struct smc_wr_tx_pend_priv *pend;
346 struct smc_wr_buf *wr_buf;
347 int rc;
348
349 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
350 if (rc)
351 return rc;
352 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
353 memset(confllc, 0, sizeof(*confllc));
354 confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
355 confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
75d320d6 356 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
9bf9abea
UB		 357 if (reqresp == SMC_LLC_RESP)
358 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
947541f3
UB		 359 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
360 ETH_ALEN);
7005ada6 361 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
9bf9abea 362 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
2be922f3 363 confllc->link_num = link->link_id;
9bf9abea 364 memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE);
52bedf37
KG		 365 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */
366 /* send llc message */
367 rc = smc_wr_tx_send(link, pend);
368 return rc;
369}
370
44aa81ce 371/* send LLC confirm rkey request */
3d88a21b 372static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
44aa81ce
KG		 373 struct smc_buf_desc *rmb_desc)
374{
375 struct smc_llc_msg_confirm_rkey *rkeyllc;
376 struct smc_wr_tx_pend_priv *pend;
377 struct smc_wr_buf *wr_buf;
3d88a21b
KG		 378 struct smc_link *link;
379 int i, rc, rtok_ix;
44aa81ce 380
3d88a21b 381 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
44aa81ce
KG		 382 if (rc)
383 return rc;
384 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
385 memset(rkeyllc, 0, sizeof(*rkeyllc));
386 rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
387 rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
3d88a21b
KG		 388
389 rtok_ix = 1;
390 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
391 link = &send_link->lgr->lnk[i];
392 if (link->state == SMC_LNK_ACTIVE && link != send_link) {
393 rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
394 rkeyllc->rtoken[rtok_ix].rmb_key =
395 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
396 rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64(
397 (u64)sg_dma_address(
398 rmb_desc->sgt[link->link_idx].sgl));
399 rtok_ix++;
400 }
401 }
402 /* rkey of send_link is in rtoken[0] */
403 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
44aa81ce 404 rkeyllc->rtoken[0].rmb_key =
3d88a21b 405 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey);
44aa81ce 406 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
3d88a21b 407 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
44aa81ce 408 /* send llc message */
3d88a21b 409 rc = smc_wr_tx_send(send_link, pend);
44aa81ce
KG		 410 return rc;
411}
412
60e03c62
KG		 413/* send LLC delete rkey request */
414static int smc_llc_send_delete_rkey(struct smc_link *link,
415 struct smc_buf_desc *rmb_desc)
416{
417 struct smc_llc_msg_delete_rkey *rkeyllc;
418 struct smc_wr_tx_pend_priv *pend;
419 struct smc_wr_buf *wr_buf;
420 int rc;
421
422 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
423 if (rc)
424 return rc;
425 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
426 memset(rkeyllc, 0, sizeof(*rkeyllc));
427 rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
428 rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
429 rkeyllc->num_rkeys = 1;
387707fd 430 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
60e03c62
KG		 431 /* send llc message */
432 rc = smc_wr_tx_send(link, pend);
433 return rc;
434}
435
52bedf37 436/* send ADD LINK request or response */
7005ada6 437int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
fbed3b37 438 struct smc_link *link_new,
52bedf37
KG		 439 enum smc_llc_reqresp reqresp)
440{
441 struct smc_llc_msg_add_link *addllc;
442 struct smc_wr_tx_pend_priv *pend;
443 struct smc_wr_buf *wr_buf;
444 int rc;
445
446 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
447 if (rc)
448 return rc;
449 addllc = (struct smc_llc_msg_add_link *)wr_buf;
fbed3b37
KG		 450
451 memset(addllc, 0, sizeof(*addllc));
452 addllc->hd.common.type = SMC_LLC_ADD_LINK;
453 addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
454 if (reqresp == SMC_LLC_RESP)
455 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
456 memcpy(addllc->sender_mac, mac, ETH_ALEN);
457 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
458 if (link_new) {
459 addllc->link_num = link_new->link_id;
460 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
461 hton24(addllc->initial_psn, link_new->psn_initial);
462 if (reqresp == SMC_LLC_REQ)
463 addllc->qp_mtu = link_new->path_mtu;
464 else
465 addllc->qp_mtu = min(link_new->path_mtu,
466 link_new->peer_mtu);
467 }
52bedf37
KG		 468 /* send llc message */
469 rc = smc_wr_tx_send(link, pend);
470 return rc;
471}
472
473/* send DELETE LINK request or response */
fbed3b37
KG		 474int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
475 enum smc_llc_reqresp reqresp, bool orderly,
476 u32 reason)
52bedf37
KG		 477{
478 struct smc_llc_msg_del_link *delllc;
479 struct smc_wr_tx_pend_priv *pend;
480 struct smc_wr_buf *wr_buf;
481 int rc;
482
483 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
484 if (rc)
485 return rc;
486 delllc = (struct smc_llc_msg_del_link *)wr_buf;
fbed3b37
KG		 487
488 memset(delllc, 0, sizeof(*delllc));
489 delllc->hd.common.type = SMC_LLC_DELETE_LINK;
490 delllc->hd.length = sizeof(struct smc_llc_msg_del_link);
491 if (reqresp == SMC_LLC_RESP)
492 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
493 if (orderly)
494 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
495 if (link_del_id)
496 delllc->link_num = link_del_id;
497 else
498 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
499 delllc->reason = htonl(reason);
9bf9abea
UB		 500 /* send llc message */
501 rc = smc_wr_tx_send(link, pend);
502 return rc;
503}
504
d97935fa
KG		 505/* send LLC test link request */
506static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
313164da
KG		 507{
508 struct smc_llc_msg_test_link *testllc;
509 struct smc_wr_tx_pend_priv *pend;
510 struct smc_wr_buf *wr_buf;
511 int rc;
512
513 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
514 if (rc)
515 return rc;
516 testllc = (struct smc_llc_msg_test_link *)wr_buf;
517 memset(testllc, 0, sizeof(*testllc));
518 testllc->hd.common.type = SMC_LLC_TEST_LINK;
519 testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
313164da
KG		 520 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
521 /* send llc message */
522 rc = smc_wr_tx_send(link, pend);
523 return rc;
524}
525
6c8968c4
KG		 526/* schedule an llc send on link, may wait for buffers */
527static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
4ed75de5
KG		 528{
529 struct smc_wr_tx_pend_priv *pend;
530 struct smc_wr_buf *wr_buf;
531 int rc;
532
6c8968c4
KG		 533 if (!smc_link_usable(link))
534 return -ENOLINK;
535 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
4ed75de5 536 if (rc)
6c8968c4
KG		 537 return rc;
538 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
539 return smc_wr_tx_send(link, pend);
4ed75de5
KG		 540}
541
9bf9abea
UB		 542/********************************* receive ***********************************/
543
8574cf40
KG		 544static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
545 enum smc_lgr_type lgr_new_t)
546{
547 int i;
548
549 if (lgr->type == SMC_LGR_SYMMETRIC ||
550 (lgr->type != SMC_LGR_SINGLE &&
551 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
552 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
553 return -EMLINK;
554
555 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
556 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
557 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
558 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
559 return i;
560 } else {
561 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
562 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
563 return i;
564 }
565 return -EMLINK;
566}
567
52bedf37
KG		 568static void smc_llc_rx_delete_link(struct smc_link *link,
569 struct smc_llc_msg_del_link *llc)
570{
00e5fb26 571 struct smc_link_group *lgr = smc_get_lgr(link);
52bedf37 572
ef79d439 573 smc_lgr_forget(lgr);
ef79d439
KG		 574 if (lgr->role == SMC_SERV) {
575 /* client asks to delete this link, send request */
fbed3b37
KG		 576 smc_llc_send_delete_link(link, 0, SMC_LLC_REQ, true,
577 SMC_LLC_DEL_PROG_INIT_TERM);
52bedf37 578 } else {
ef79d439 579 /* server requests to delete this link, send response */
fbed3b37
KG		 580 smc_llc_send_delete_link(link, 0, SMC_LLC_RESP, true,
581 SMC_LLC_DEL_PROG_INIT_TERM);
52bedf37 582 }
87523930 583 smcr_link_down_cond(link);
52bedf37
KG		 584}
585
3bc67e09
KG		 586/* process a confirm_rkey request from peer, remote flow */
587static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
4ed75de5 588{
3bc67e09
KG		 589 struct smc_llc_msg_confirm_rkey *llc;
590 struct smc_llc_qentry *qentry;
591 struct smc_link *link;
592 int num_entries;
593 int rk_idx;
594 int i;
595
596 qentry = lgr->llc_flow_rmt.qentry;
597 llc = &qentry->msg.confirm_rkey;
598 link = qentry->link;
599
600 num_entries = llc->rtoken[0].num_rkeys;
601 /* first rkey entry is for receiving link */
602 rk_idx = smc_rtoken_add(link,
603 llc->rtoken[0].rmb_vaddr,
604 llc->rtoken[0].rmb_key);
605 if (rk_idx < 0)
606 goto out_err;
607
608 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
609 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
610 llc->rtoken[i].rmb_vaddr,
611 llc->rtoken[i].rmb_key);
612 /* max links is 3 so there is no need to support conf_rkey_cont msgs */
613 goto out;
614out_err:
615 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
616 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
617out:
ef79d439 618 llc->hd.flags |= SMC_LLC_FLAG_RESP;
3bc67e09
KG		 619 smc_llc_send_message(link, &qentry->msg);
620 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
4ed75de5
KG		 621}
622
218b24fe
KG		 623/* process a delete_rkey request from peer, remote flow */
624static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
4ed75de5 625{
218b24fe
KG		 626 struct smc_llc_msg_delete_rkey *llc;
627 struct smc_llc_qentry *qentry;
628 struct smc_link *link;
4ed75de5
KG		 629 u8 err_mask = 0;
630 int i, max;
631
218b24fe
KG		 632 qentry = lgr->llc_flow_rmt.qentry;
633 llc = &qentry->msg.delete_rkey;
634 link = qentry->link;
635
ef79d439
KG		 636 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
637 for (i = 0; i < max; i++) {
638 if (smc_rtoken_delete(link, llc->rkey[i]))
639 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
640 }
ef79d439
KG		 641 if (err_mask) {
642 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
643 llc->err_mask = err_mask;
4ed75de5 644 }
218b24fe
KG		 645 llc->hd.flags |= SMC_LLC_FLAG_RESP;
646 smc_llc_send_message(link, &qentry->msg);
647 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
648}
ef79d439 649
6c8968c4 650/* flush the llc event queue */
00a049cf 651static void smc_llc_event_flush(struct smc_link_group *lgr)
9bf9abea 652{
6c8968c4
KG		 653 struct smc_llc_qentry *qentry, *q;
654
655 spin_lock_bh(&lgr->llc_event_q_lock);
656 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
657 list_del_init(&qentry->list);
658 kfree(qentry);
659 }
660 spin_unlock_bh(&lgr->llc_event_q_lock);
661}
662
663static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
664{
665 union smc_llc_msg *llc = &qentry->msg;
666 struct smc_link *link = qentry->link;
0fb0b02b 667 struct smc_link_group *lgr = link->lgr;
9bf9abea 668
d854fcbf 669 if (!smc_link_usable(link))
6c8968c4 670 goto out;
313164da
KG		 671
672 switch (llc->raw.hdr.common.type) {
673 case SMC_LLC_TEST_LINK:
56e8091c
KG		 674 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
675 smc_llc_send_message(link, llc);
313164da 676 break;
52bedf37 677 case SMC_LLC_ADD_LINK:
0fb0b02b
KG		 678 if (list_empty(&lgr->list))
679 goto out; /* lgr is terminating */
680 if (lgr->role == SMC_CLNT) {
681 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) {
682 /* a flow is waiting for this message */
683 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
684 qentry);
685 wake_up_interruptible(&lgr->llc_waiter);
686 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
687 qentry)) {
688 /* tbd: schedule_work(&lgr->llc_add_link_work); */
689 }
690 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
691 /* as smc server, handle client suggestion */
692 /* tbd: schedule_work(&lgr->llc_add_link_work); */
693 }
694 return;
695 case SMC_LLC_CONFIRM_LINK:
696 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
697 /* a flow is waiting for this message */
698 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
699 wake_up_interruptible(&lgr->llc_waiter);
700 return;
701 }
52bedf37
KG		 702 break;
703 case SMC_LLC_DELETE_LINK:
704 smc_llc_rx_delete_link(link, &llc->delete_link);
705 break;
4ed75de5 706 case SMC_LLC_CONFIRM_RKEY:
3bc67e09
KG		 707 /* new request from remote, assign to remote flow */
708 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
709 /* process here, does not wait for more llc msgs */
710 smc_llc_rmt_conf_rkey(lgr);
711 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
712 }
713 return;
4ed75de5 714 case SMC_LLC_CONFIRM_RKEY_CONT:
42d18acc
KG		 715 /* not used because max links is 3, and 3 rkeys fit into
716 * one CONFIRM_RKEY message
717 */
4ed75de5
KG		 718 break;
719 case SMC_LLC_DELETE_RKEY:
218b24fe
KG		 720 /* new request from remote, assign to remote flow */
721 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
722 /* process here, does not wait for more llc msgs */
723 smc_llc_rmt_delete_rkey(lgr);
724 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
725 }
726 return;
313164da 727 }
6c8968c4
KG		 728out:
729 kfree(qentry);
730}
731
732/* worker to process llc messages on the event queue */
733static void smc_llc_event_work(struct work_struct *work)
734{
735 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
736 llc_event_work);
737 struct smc_llc_qentry *qentry;
738
555da9af
KG		 739 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
740 if (smc_link_usable(lgr->delayed_event->link)) {
741 smc_llc_event_handler(lgr->delayed_event);
742 } else {
743 qentry = lgr->delayed_event;
744 lgr->delayed_event = NULL;
745 kfree(qentry);
746 }
747 }
748
6c8968c4
KG		 749again:
750 spin_lock_bh(&lgr->llc_event_q_lock);
751 if (!list_empty(&lgr->llc_event_q)) {
752 qentry = list_first_entry(&lgr->llc_event_q,
753 struct smc_llc_qentry, list);
754 list_del_init(&qentry->list);
755 spin_unlock_bh(&lgr->llc_event_q_lock);
756 smc_llc_event_handler(qentry);
757 goto again;
758 }
759 spin_unlock_bh(&lgr->llc_event_q_lock);
760}
761
ef79d439 762/* process llc responses in tasklet context */
a6688d91
KG		 763static void smc_llc_rx_response(struct smc_link *link,
764 struct smc_llc_qentry *qentry)
ef79d439 765{
a6688d91 766 u8 llc_type = qentry->msg.raw.hdr.common.type;
ef79d439 767
a6688d91 768 switch (llc_type) {
ef79d439
KG		 769 case SMC_LLC_TEST_LINK:
770 if (link->state == SMC_LNK_ACTIVE)
771 complete(&link->llc_testlink_resp);
772 break;
ef79d439 773 case SMC_LLC_ADD_LINK:
4667bb4a 774 case SMC_LLC_CONFIRM_LINK:
3d88a21b 775 case SMC_LLC_CONFIRM_RKEY:
6d74c3a8 776 case SMC_LLC_DELETE_RKEY:
4667bb4a
KG		 777 /* assign responses to the local flow, we requested them */
778 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
779 wake_up_interruptible(&link->lgr->llc_waiter);
780 return;
ef79d439
KG		 781 case SMC_LLC_DELETE_LINK:
782 if (link->lgr->role == SMC_SERV)
783 smc_lgr_schedule_free_work_fast(link->lgr);
784 break;
ef79d439 785 case SMC_LLC_CONFIRM_RKEY_CONT:
42d18acc 786 /* not used because max links is 3 */
ef79d439 787 break;
ef79d439 788 }
a6688d91 789 kfree(qentry);
ef79d439
KG		 790}
791
a6688d91 792static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
6c8968c4 793{
6c8968c4
KG		 794 struct smc_link_group *lgr = link->lgr;
795 struct smc_llc_qentry *qentry;
6c8968c4
KG		 796 unsigned long flags;
797
6c8968c4
KG		 798 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
799 if (!qentry)
800 return;
801 qentry->link = link;
802 INIT_LIST_HEAD(&qentry->list);
803 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
a6688d91
KG		 804
805 /* process responses immediately */
806 if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
807 smc_llc_rx_response(link, qentry);
808 return;
809 }
810
811 /* add requests to event queue */
6c8968c4
KG		 812 spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
813 list_add_tail(&qentry->list, &lgr->llc_event_q);
814 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
815 schedule_work(&link->lgr->llc_event_work);
9bf9abea
UB		 816}
817
a6688d91
KG		 818/* copy received msg and add it to the event queue */
819static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
820{
821 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
822 union smc_llc_msg *llc = buf;
823
824 if (wc->byte_len < sizeof(*llc))
825 return; /* short message */
826 if (llc->raw.hdr.length != sizeof(*llc))
827 return; /* invalid message */
828
829 smc_llc_enqueue(link, llc);
830}
831
44aa81ce 832/***************************** worker, utils *********************************/
877ae5be
KG		 833
834static void smc_llc_testlink_work(struct work_struct *work)
835{
836 struct smc_link *link = container_of(to_delayed_work(work),
837 struct smc_link, llc_testlink_wrk);
838 unsigned long next_interval;
877ae5be
KG		 839 unsigned long expire_time;
840 u8 user_data[16] = { 0 };
841 int rc;
842
877ae5be
KG		 843 if (link->state != SMC_LNK_ACTIVE)
844 return; /* don't reschedule worker */
845 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
846 if (time_is_after_jiffies(expire_time)) {
847 next_interval = expire_time - jiffies;
848 goto out;
849 }
850 reinit_completion(&link->llc_testlink_resp);
d97935fa 851 smc_llc_send_test_link(link, user_data);
877ae5be
KG		 852 /* receive TEST LINK response over RoCE fabric */
853 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
854 SMC_LLC_WAIT_TIME);
1020e1ef
KG		 855 if (link->state != SMC_LNK_ACTIVE)
856 return; /* link state changed */
877ae5be 857 if (rc <= 0) {
87523930 858 smcr_link_down_cond_sched(link);
877ae5be
KG		 859 return;
860 }
861 next_interval = link->llc_testlink_time;
862out:
1020e1ef 863 schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
877ae5be
KG		 864}
865
00a049cf
KG		 866void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
867{
868 struct net *net = sock_net(smc->clcsock->sk);
869
870 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
871 INIT_LIST_HEAD(&lgr->llc_event_q);
872 spin_lock_init(&lgr->llc_event_q_lock);
555da9af
KG		 873 spin_lock_init(&lgr->llc_flow_lock);
874 init_waitqueue_head(&lgr->llc_waiter);
d5500667 875 mutex_init(&lgr->llc_conf_mutex);
00a049cf
KG		 876 lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
877}
878
879/* called after lgr was removed from lgr_list */
880void smc_llc_lgr_clear(struct smc_link_group *lgr)
881{
882 smc_llc_event_flush(lgr);
555da9af 883 wake_up_interruptible_all(&lgr->llc_waiter);
00a049cf 884 cancel_work_sync(&lgr->llc_event_work);
555da9af
KG		 885 if (lgr->delayed_event) {
886 kfree(lgr->delayed_event);
887 lgr->delayed_event = NULL;
888 }
00a049cf
KG		 889}
890
2a4c57a9 891int smc_llc_link_init(struct smc_link *link)
877ae5be
KG		 892{
893 init_completion(&link->llc_testlink_resp);
894 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2a4c57a9 895 return 0;
b32cf4ab
KG		 896}
897
00a049cf 898void smc_llc_link_active(struct smc_link *link)
b32cf4ab 899{
877ae5be 900 link->state = SMC_LNK_ACTIVE;
00a049cf
KG		 901 if (link->lgr->llc_testlink_time) {
902 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ;
1020e1ef
KG		 903 schedule_delayed_work(&link->llc_testlink_wrk,
904 link->llc_testlink_time);
877ae5be
KG		 905 }
906}
907
877ae5be 908/* called in worker context */
2a4c57a9 909void smc_llc_link_clear(struct smc_link *link)
877ae5be 910{
2140ac26
KG		 911 complete(&link->llc_testlink_resp);
912 cancel_delayed_work_sync(&link->llc_testlink_wrk);
913 smc_wr_wakeup_reg_wait(link);
914 smc_wr_wakeup_tx_wait(link);
877ae5be
KG		 915}
916
3d88a21b
KG		 917/* register a new rtoken at the remote peer (for all links) */
918int smc_llc_do_confirm_rkey(struct smc_link *send_link,
44aa81ce
KG		 919 struct smc_buf_desc *rmb_desc)
920{
3d88a21b
KG		 921 struct smc_link_group *lgr = send_link->lgr;
922 struct smc_llc_qentry *qentry = NULL;
923 int rc = 0;
44aa81ce 924
3d88a21b
KG		 925 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
926 if (rc)
927 goto out;
44aa81ce 928 /* receive CONFIRM RKEY response from server over RoCE fabric */
3d88a21b
KG		 929 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
930 SMC_LLC_CONFIRM_RKEY);
931 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
932 rc = -EFAULT;
933out:
934 if (qentry)
935 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
3d88a21b 936 return rc;
44aa81ce
KG		 937}
938
60e03c62 939/* unregister an rtoken at the remote peer */
6d74c3a8 940int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
60e03c62
KG		 941 struct smc_buf_desc *rmb_desc)
942{
6d74c3a8
KG		 943 struct smc_llc_qentry *qentry = NULL;
944 struct smc_link *send_link;
0b29ec64 945 int rc = 0;
60e03c62 946
6d74c3a8
KG		 947 send_link = smc_llc_usable_link(lgr);
948 if (!send_link)
949 return -ENOLINK;
950
6d74c3a8
KG		 951 /* protected by llc_flow control */
952 rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
60e03c62
KG		 953 if (rc)
954 goto out;
955 /* receive DELETE RKEY response from server over RoCE fabric */
6d74c3a8
KG		 956 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
957 SMC_LLC_DELETE_RKEY);
958 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
60e03c62 959 rc = -EFAULT;
60e03c62 960out:
6d74c3a8
KG		 961 if (qentry)
962 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
60e03c62
KG		 963 return rc;
964}
965
92334cfc
KG		 966/* evaluate confirm link request or response */
967int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
968 enum smc_llc_reqresp type)
969{
970 if (type == SMC_LLC_REQ) /* SMC server assigns link_id */
971 qentry->link->link_id = qentry->msg.confirm_link.link_num;
972 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
973 return -ENOTSUPP;
974 return 0;
975}
976
9bf9abea
UB		 977/***************************** init, exit, misc ******************************/
978
979static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
980 {
981 .handler = smc_llc_rx_handler,
982 .type = SMC_LLC_CONFIRM_LINK
983 },
313164da
KG		 984 {
985 .handler = smc_llc_rx_handler,
986 .type = SMC_LLC_TEST_LINK
987 },
52bedf37
KG		 988 {
989 .handler = smc_llc_rx_handler,
990 .type = SMC_LLC_ADD_LINK
991 },
992 {
993 .handler = smc_llc_rx_handler,
994 .type = SMC_LLC_DELETE_LINK
995 },
4ed75de5
KG		 996 {
997 .handler = smc_llc_rx_handler,
998 .type = SMC_LLC_CONFIRM_RKEY
999 },
1000 {
1001 .handler = smc_llc_rx_handler,
1002 .type = SMC_LLC_CONFIRM_RKEY_CONT
1003 },
1004 {
1005 .handler = smc_llc_rx_handler,
1006 .type = SMC_LLC_DELETE_RKEY
1007 },
9bf9abea
UB		 1008 {
1009 .handler = NULL,
1010 }
1011};
1012
1013int __init smc_llc_init(void)
1014{
1015 struct smc_wr_rx_handler *handler;
1016 int rc = 0;
1017
1018 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
1019 INIT_HLIST_NODE(&handler->list);
1020 rc = smc_wr_rx_register_handler(handler);
1021 if (rc)
1022 break;
1023 }
1024 return rc;
1025}
Linux 6.x block layer and io_uring tree(s)