projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'devicetree-for-5.18' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-block.git] / net / smc / smc_llc.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
9bf9abea
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Link Layer Control (LLC)
6 *
9bf9abea
UB		 7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com>
10 * Ursula Braun <ubraun@linux.vnet.ibm.com>
11 */
12
13#include <net/tcp.h>
14#include <rdma/ib_verbs.h>
15
16#include "smc.h"
17#include "smc_core.h"
18#include "smc_clc.h"
19#include "smc_llc.h"
336ba09f 20#include "smc_pnet.h"
9bf9abea 21
0f627126
SR		 22#define SMC_LLC_DATA_LEN 40
23
24struct smc_llc_hdr {
25 struct smc_wr_rx_hdr common;
b4ba4652
KG		 26 union {
27 struct {
28 u8 length; /* 44 */
29 #if defined(__BIG_ENDIAN_BITFIELD)
30 u8 reserved:4,
31 add_link_rej_rsn:4;
52bedf37 32#elif defined(__LITTLE_ENDIAN_BITFIELD)
b4ba4652
KG		 33 u8 add_link_rej_rsn:4,
34 reserved:4;
52bedf37 35#endif
b4ba4652
KG		 36 };
37 u16 length_v2; /* 44 - 8192*/
38 };
0f627126 39 u8 flags;
b4ba4652
KG		 40} __packed; /* format defined in
41 * IBM Shared Memory Communications Version 2
42 * (https://www.ibm.com/support/pages/node/6326337)
43 */
0f627126 44
75d320d6
KG		 45#define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03
46
0f627126
SR		 47struct smc_llc_msg_confirm_link { /* type 0x01 */
48 struct smc_llc_hdr hd;
49 u8 sender_mac[ETH_ALEN];
50 u8 sender_gid[SMC_GID_SIZE];
51 u8 sender_qp_num[3];
52 u8 link_num;
53 u8 link_uid[SMC_LGR_ID_SIZE];
54 u8 max_links;
55 u8 reserved[9];
56};
57
52bedf37
KG		 58#define SMC_LLC_FLAG_ADD_LNK_REJ 0x40
59#define SMC_LLC_REJ_RSN_NO_ALT_PATH 1
60
61#define SMC_LLC_ADD_LNK_MAX_LINKS 2
62
63struct smc_llc_msg_add_link { /* type 0x02 */
64 struct smc_llc_hdr hd;
65 u8 sender_mac[ETH_ALEN];
66 u8 reserved2[2];
67 u8 sender_gid[SMC_GID_SIZE];
68 u8 sender_qp_num[3];
69 u8 link_num;
fbed3b37
KG		 70#if defined(__BIG_ENDIAN_BITFIELD)
71 u8 reserved3 : 4,
72 qp_mtu : 4;
73#elif defined(__LITTLE_ENDIAN_BITFIELD)
74 u8 qp_mtu : 4,
75 reserved3 : 4;
76#endif
52bedf37
KG		 77 u8 initial_psn[3];
78 u8 reserved[8];
79};
80
87f88cda
KG		 81struct smc_llc_msg_add_link_cont_rt {
82 __be32 rmb_key;
83 __be32 rmb_key_new;
84 __be64 rmb_vaddr_new;
85};
86
b4ba4652
KG		 87struct smc_llc_msg_add_link_v2_ext {
88#if defined(__BIG_ENDIAN_BITFIELD)
89 u8 v2_direct : 1,
90 reserved : 7;
91#elif defined(__LITTLE_ENDIAN_BITFIELD)
92 u8 reserved : 7,
93 v2_direct : 1;
94#endif
95 u8 reserved2;
96 u8 client_target_gid[SMC_GID_SIZE];
97 u8 reserved3[8];
98 u16 num_rkeys;
99 struct smc_llc_msg_add_link_cont_rt rt[];
100} __packed; /* format defined in
101 * IBM Shared Memory Communications Version 2
102 * (https://www.ibm.com/support/pages/node/6326337)
103 */
104
105struct smc_llc_msg_req_add_link_v2 {
106 struct smc_llc_hdr hd;
107 u8 reserved[20];
108 u8 gid_cnt;
109 u8 reserved2[3];
110 u8 gid[][SMC_GID_SIZE];
111};
112
87f88cda
KG		 113#define SMC_LLC_RKEYS_PER_CONT_MSG 2
114
115struct smc_llc_msg_add_link_cont { /* type 0x03 */
116 struct smc_llc_hdr hd;
117 u8 link_num;
118 u8 num_rkeys;
119 u8 reserved2[2];
120 struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
121 u8 reserved[4];
122} __packed; /* format defined in RFC7609 */
123
52bedf37
KG		 124#define SMC_LLC_FLAG_DEL_LINK_ALL 0x40
125#define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20
126
127struct smc_llc_msg_del_link { /* type 0x04 */
128 struct smc_llc_hdr hd;
129 u8 link_num;
130 __be32 reason;
131 u8 reserved[35];
132} __packed; /* format defined in RFC7609 */
133
313164da
KG		 134struct smc_llc_msg_test_link { /* type 0x07 */
135 struct smc_llc_hdr hd;
136 u8 user_data[16];
137 u8 reserved[24];
138};
139
4ed75de5
KG		 140struct smc_rmb_rtoken {
141 union {
142 u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */
143 /* is actually the num of rtokens, first */
144 /* rtoken is always for the current link */
145 u8 link_id; /* link id of the rtoken */
146 };
147 __be32 rmb_key;
148 __be64 rmb_vaddr;
149} __packed; /* format defined in RFC7609 */
150
b4ba4652
KG		 151#define SMC_LLC_RKEYS_PER_MSG 3
152#define SMC_LLC_RKEYS_PER_MSG_V2 255
4ed75de5
KG		 153
154struct smc_llc_msg_confirm_rkey { /* type 0x06 */
155 struct smc_llc_hdr hd;
156 struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
157 u8 reserved;
158};
159
4ed75de5 160#define SMC_LLC_DEL_RKEY_MAX 8
3bc67e09 161#define SMC_LLC_FLAG_RKEY_RETRY 0x10
4ed75de5
KG		 162#define SMC_LLC_FLAG_RKEY_NEG 0x20
163
164struct smc_llc_msg_delete_rkey { /* type 0x09 */
165 struct smc_llc_hdr hd;
166 u8 num_rkeys;
167 u8 err_mask;
168 u8 reserved[2];
169 __be32 rkey[8];
170 u8 reserved2[4];
171};
172
b4ba4652
KG		 173struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */
174 struct smc_llc_hdr hd;
175 u8 num_rkeys;
176 u8 num_inval_rkeys;
177 u8 reserved[2];
178 __be32 rkey[];
179};
180
0f627126
SR		 181union smc_llc_msg {
182 struct smc_llc_msg_confirm_link confirm_link;
52bedf37 183 struct smc_llc_msg_add_link add_link;
b4ba4652 184 struct smc_llc_msg_req_add_link_v2 req_add_link;
87f88cda 185 struct smc_llc_msg_add_link_cont add_link_cont;
52bedf37 186 struct smc_llc_msg_del_link delete_link;
4ed75de5
KG		 187
188 struct smc_llc_msg_confirm_rkey confirm_rkey;
4ed75de5
KG		 189 struct smc_llc_msg_delete_rkey delete_rkey;
190
313164da 191 struct smc_llc_msg_test_link test_link;
0f627126
SR		 192 struct {
193 struct smc_llc_hdr hdr;
194 u8 data[SMC_LLC_DATA_LEN];
195 } raw;
196};
197
198#define SMC_LLC_FLAG_RESP 0x80
199
6c8968c4
KG		 200struct smc_llc_qentry {
201 struct list_head list;
202 struct smc_link *link;
203 union smc_llc_msg msg;
204};
205
4dadd151
KG		 206static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
207
555da9af
KG		 208struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
209{
210 struct smc_llc_qentry *qentry = flow->qentry;
211
212 flow->qentry = NULL;
213 return qentry;
214}
215
216void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
217{
218 struct smc_llc_qentry *qentry;
219
220 if (flow->qentry) {
221 qentry = flow->qentry;
222 flow->qentry = NULL;
223 kfree(qentry);
224 }
225}
226
227static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
228 struct smc_llc_qentry *qentry)
229{
230 flow->qentry = qentry;
231}
232
6778a6be
KG		 233static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type,
234 struct smc_llc_qentry *qentry)
235{
b4ba4652 236 u8 msg_type = qentry->msg.raw.hdr.common.llc_type;
6778a6be
KG		 237
238 if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) &&
239 flow_type != msg_type && !lgr->delayed_event) {
240 lgr->delayed_event = qentry;
241 return;
242 }
243 /* drop parallel or already-in-progress llc requests */
244 if (flow_type != msg_type)
de2fea7b 245 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped parallel "
6778a6be
KG		 246 "LLC msg: msg %d flow %d role %d\n",
247 SMC_LGR_ID_SIZE, &lgr->id,
de2fea7b 248 lgr->net->net_cookie,
6778a6be
KG		 249 qentry->msg.raw.hdr.common.type,
250 flow_type, lgr->role);
251 kfree(qentry);
252}
253
555da9af
KG		 254/* try to start a new llc flow, initiated by an incoming llc msg */
255static bool smc_llc_flow_start(struct smc_llc_flow *flow,
256 struct smc_llc_qentry *qentry)
257{
258 struct smc_link_group *lgr = qentry->link->lgr;
259
260 spin_lock_bh(&lgr->llc_flow_lock);
261 if (flow->type) {
262 /* a flow is already active */
6778a6be 263 smc_llc_flow_parallel(lgr, flow->type, qentry);
555da9af
KG		 264 spin_unlock_bh(&lgr->llc_flow_lock);
265 return false;
266 }
b4ba4652 267 switch (qentry->msg.raw.hdr.common.llc_type) {
555da9af
KG		 268 case SMC_LLC_ADD_LINK:
269 flow->type = SMC_LLC_FLOW_ADD_LINK;
270 break;
271 case SMC_LLC_DELETE_LINK:
272 flow->type = SMC_LLC_FLOW_DEL_LINK;
273 break;
274 case SMC_LLC_CONFIRM_RKEY:
275 case SMC_LLC_DELETE_RKEY:
276 flow->type = SMC_LLC_FLOW_RKEY;
277 break;
278 default:
279 flow->type = SMC_LLC_FLOW_NONE;
280 }
555da9af 281 smc_llc_flow_qentry_set(flow, qentry);
6778a6be 282 spin_unlock_bh(&lgr->llc_flow_lock);
555da9af
KG		 283 return true;
284}
285
286/* start a new local llc flow, wait till current flow finished */
287int smc_llc_flow_initiate(struct smc_link_group *lgr,
288 enum smc_llc_flowtype type)
289{
290 enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
291 int rc;
292
293 /* all flows except confirm_rkey and delete_rkey are exclusive,
294 * confirm/delete rkey flows can run concurrently (local and remote)
295 */
296 if (type == SMC_LLC_FLOW_RKEY)
297 allowed_remote = SMC_LLC_FLOW_RKEY;
298again:
299 if (list_empty(&lgr->list))
300 return -ENODEV;
301 spin_lock_bh(&lgr->llc_flow_lock);
302 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
303 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
304 lgr->llc_flow_rmt.type == allowed_remote)) {
305 lgr->llc_flow_lcl.type = type;
306 spin_unlock_bh(&lgr->llc_flow_lock);
307 return 0;
308 }
309 spin_unlock_bh(&lgr->llc_flow_lock);
6778a6be
KG		 310 rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) ||
311 (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
312 (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
313 lgr->llc_flow_rmt.type == allowed_remote))),
314 SMC_LLC_WAIT_TIME * 10);
555da9af
KG		 315 if (!rc)
316 return -ETIMEDOUT;
317 goto again;
318}
319
320/* finish the current llc flow */
321void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
322{
323 spin_lock_bh(&lgr->llc_flow_lock);
324 memset(flow, 0, sizeof(*flow));
325 flow->type = SMC_LLC_FLOW_NONE;
326 spin_unlock_bh(&lgr->llc_flow_lock);
327 if (!list_empty(&lgr->list) && lgr->delayed_event &&
328 flow == &lgr->llc_flow_lcl)
329 schedule_work(&lgr->llc_event_work);
330 else
6778a6be 331 wake_up(&lgr->llc_flow_waiter);
555da9af
KG		 332}
333
334/* lnk is optional and used for early wakeup when link goes down, useful in
335 * cases where we wait for a response on the link after we sent a request
336 */
337struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
338 struct smc_link *lnk,
339 int time_out, u8 exp_msg)
340{
341 struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
6778a6be 342 u8 rcv_msg;
555da9af 343
6778a6be
KG		 344 wait_event_timeout(lgr->llc_msg_waiter,
345 (flow->qentry ||
346 (lnk && !smc_link_usable(lnk)) ||
347 list_empty(&lgr->list)),
348 time_out);
555da9af
KG		 349 if (!flow->qentry ||
350 (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
351 smc_llc_flow_qentry_del(flow);
352 goto out;
353 }
b4ba4652 354 rcv_msg = flow->qentry->msg.raw.hdr.common.llc_type;
6778a6be 355 if (exp_msg && rcv_msg != exp_msg) {
555da9af 356 if (exp_msg == SMC_LLC_ADD_LINK &&
6778a6be 357 rcv_msg == SMC_LLC_DELETE_LINK) {
555da9af
KG		 358 /* flow_start will delay the unexpected msg */
359 smc_llc_flow_start(&lgr->llc_flow_lcl,
360 smc_llc_flow_qentry_clr(flow));
361 return NULL;
362 }
de2fea7b 363 pr_warn_once("smc: SMC-R lg %*phN net %llu dropped unexpected LLC msg: "
6778a6be 364 "msg %d exp %d flow %d role %d flags %x\n",
de2fea7b
TL		 365 SMC_LGR_ID_SIZE, &lgr->id, lgr->net->net_cookie,
366 rcv_msg, exp_msg,
6778a6be
KG		 367 flow->type, lgr->role,
368 flow->qentry->msg.raw.hdr.flags);
555da9af
KG		 369 smc_llc_flow_qentry_del(flow);
370 }
371out:
372 return flow->qentry;
373}
374
9bf9abea
UB		 375/********************************** send *************************************/
376
377struct smc_llc_tx_pend {
378};
379
380/* handler for send/transmission completion of an LLC msg */
381static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
382 struct smc_link *link,
383 enum ib_wc_status wc_status)
384{
385 /* future work: handle wc_status error for recovery and failover */
386}
387
388/**
389 * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
390 * @link: Pointer to SMC link used for sending LLC control message.
391 * @wr_buf: Out variable returning pointer to work request payload buffer.
392 * @pend: Out variable returning pointer to private pending WR tracking.
393 * It's the context the transmit complete handler will get.
394 *
395 * Reserves and pre-fills an entry for a pending work request send/tx.
396 * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
397 * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
398 *
399 * Return: 0 on success, otherwise an error value.
400 */
401static int smc_llc_add_pending_send(struct smc_link *link,
402 struct smc_wr_buf **wr_buf,
403 struct smc_wr_tx_pend_priv **pend)
404{
405 int rc;
406
ad6f317f
UB		 407 rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
408 pend);
9bf9abea
UB		 409 if (rc < 0)
410 return rc;
411 BUILD_BUG_ON_MSG(
412 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
413 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
414 BUILD_BUG_ON_MSG(
415 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
416 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
417 BUILD_BUG_ON_MSG(
418 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
419 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
420 return 0;
421}
422
b4ba4652
KG		 423static int smc_llc_add_pending_send_v2(struct smc_link *link,
424 struct smc_wr_v2_buf **wr_buf,
425 struct smc_wr_tx_pend_priv **pend)
426{
427 int rc;
428
429 rc = smc_wr_tx_get_v2_slot(link, smc_llc_tx_handler, wr_buf, pend);
430 if (rc < 0)
431 return rc;
432 return 0;
433}
434
435static void smc_llc_init_msg_hdr(struct smc_llc_hdr *hdr,
436 struct smc_link_group *lgr, size_t len)
437{
438 if (lgr->smc_version == SMC_V2) {
439 hdr->common.llc_version = SMC_V2;
440 hdr->length_v2 = len;
441 } else {
442 hdr->common.llc_version = 0;
443 hdr->length = len;
444 }
445}
446
9bf9abea 447/* high-level API to send LLC confirm link */
947541f3 448int smc_llc_send_confirm_link(struct smc_link *link,
9bf9abea
UB		 449 enum smc_llc_reqresp reqresp)
450{
9bf9abea
UB		 451 struct smc_llc_msg_confirm_link *confllc;
452 struct smc_wr_tx_pend_priv *pend;
453 struct smc_wr_buf *wr_buf;
454 int rc;
455
95f7f3e7
KG		 456 if (!smc_wr_tx_link_hold(link))
457 return -ENOLINK;
9bf9abea
UB		 458 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
459 if (rc)
95f7f3e7 460 goto put_out;
9bf9abea
UB		 461 confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
462 memset(confllc, 0, sizeof(*confllc));
b4ba4652
KG		 463 confllc->hd.common.llc_type = SMC_LLC_CONFIRM_LINK;
464 smc_llc_init_msg_hdr(&confllc->hd, link->lgr, sizeof(*confllc));
75d320d6 465 confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
9bf9abea
UB		 466 if (reqresp == SMC_LLC_RESP)
467 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
947541f3
UB		 468 memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
469 ETH_ALEN);
7005ada6 470 memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
9bf9abea 471 hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
2be922f3 472 confllc->link_num = link->link_id;
45fa8da0 473 memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
b1570a87 474 confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
52bedf37
KG		 475 /* send llc message */
476 rc = smc_wr_tx_send(link, pend);
95f7f3e7
KG		 477put_out:
478 smc_wr_tx_link_put(link);
52bedf37
KG		 479 return rc;
480}
481
44aa81ce 482/* send LLC confirm rkey request */
3d88a21b 483static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
44aa81ce
KG		 484 struct smc_buf_desc *rmb_desc)
485{
486 struct smc_llc_msg_confirm_rkey *rkeyllc;
487 struct smc_wr_tx_pend_priv *pend;
488 struct smc_wr_buf *wr_buf;
3d88a21b
KG		 489 struct smc_link *link;
490 int i, rc, rtok_ix;
44aa81ce 491
95f7f3e7
KG		 492 if (!smc_wr_tx_link_hold(send_link))
493 return -ENOLINK;
3d88a21b 494 rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
44aa81ce 495 if (rc)
95f7f3e7 496 goto put_out;
44aa81ce
KG		 497 rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
498 memset(rkeyllc, 0, sizeof(*rkeyllc));
b4ba4652
KG		 499 rkeyllc->hd.common.llc_type = SMC_LLC_CONFIRM_RKEY;
500 smc_llc_init_msg_hdr(&rkeyllc->hd, send_link->lgr, sizeof(*rkeyllc));
3d88a21b
KG		 501
502 rtok_ix = 1;
503 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
504 link = &send_link->lgr->lnk[i];
741a49a4 505 if (smc_link_active(link) && link != send_link) {
3d88a21b
KG		 506 rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
507 rkeyllc->rtoken[rtok_ix].rmb_key =
508 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
509 rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64(
510 (u64)sg_dma_address(
511 rmb_desc->sgt[link->link_idx].sgl));
512 rtok_ix++;
513 }
514 }
515 /* rkey of send_link is in rtoken[0] */
516 rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
44aa81ce 517 rkeyllc->rtoken[0].rmb_key =
3d88a21b 518 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey);
44aa81ce 519 rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
3d88a21b 520 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
44aa81ce 521 /* send llc message */
3d88a21b 522 rc = smc_wr_tx_send(send_link, pend);
95f7f3e7
KG		 523put_out:
524 smc_wr_tx_link_put(send_link);
44aa81ce
KG		 525 return rc;
526}
527
60e03c62
KG		 528/* send LLC delete rkey request */
529static int smc_llc_send_delete_rkey(struct smc_link *link,
530 struct smc_buf_desc *rmb_desc)
531{
532 struct smc_llc_msg_delete_rkey *rkeyllc;
533 struct smc_wr_tx_pend_priv *pend;
534 struct smc_wr_buf *wr_buf;
535 int rc;
536
95f7f3e7
KG		 537 if (!smc_wr_tx_link_hold(link))
538 return -ENOLINK;
60e03c62
KG		 539 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
540 if (rc)
95f7f3e7 541 goto put_out;
60e03c62
KG		 542 rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
543 memset(rkeyllc, 0, sizeof(*rkeyllc));
b4ba4652
KG		 544 rkeyllc->hd.common.llc_type = SMC_LLC_DELETE_RKEY;
545 smc_llc_init_msg_hdr(&rkeyllc->hd, link->lgr, sizeof(*rkeyllc));
60e03c62 546 rkeyllc->num_rkeys = 1;
387707fd 547 rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
60e03c62
KG		 548 /* send llc message */
549 rc = smc_wr_tx_send(link, pend);
95f7f3e7
KG		 550put_out:
551 smc_wr_tx_link_put(link);
60e03c62
KG		 552 return rc;
553}
554
b4ba4652
KG		 555/* return first buffer from any of the next buf lists */
556static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
557 int *buf_lst)
558{
559 struct smc_buf_desc *buf_pos;
560
561 while (*buf_lst < SMC_RMBE_SIZES) {
562 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
563 struct smc_buf_desc, list);
564 if (buf_pos)
565 return buf_pos;
566 (*buf_lst)++;
567 }
568 return NULL;
569}
570
571/* return next rmb from buffer lists */
572static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
573 int *buf_lst,
574 struct smc_buf_desc *buf_pos)
575{
576 struct smc_buf_desc *buf_next;
577
578 if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
579 (*buf_lst)++;
580 return _smc_llc_get_next_rmb(lgr, buf_lst);
581 }
582 buf_next = list_next_entry(buf_pos, list);
583 return buf_next;
584}
585
586static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
587 int *buf_lst)
588{
589 *buf_lst = 0;
590 return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
591}
592
593static int smc_llc_fill_ext_v2(struct smc_llc_msg_add_link_v2_ext *ext,
594 struct smc_link *link, struct smc_link *link_new)
595{
596 struct smc_link_group *lgr = link->lgr;
597 struct smc_buf_desc *buf_pos;
598 int prim_lnk_idx, lnk_idx, i;
599 struct smc_buf_desc *rmb;
600 int len = sizeof(*ext);
601 int buf_lst;
602
603 ext->v2_direct = !lgr->uses_gateway;
604 memcpy(ext->client_target_gid, link_new->gid, SMC_GID_SIZE);
605
606 prim_lnk_idx = link->link_idx;
607 lnk_idx = link_new->link_idx;
608 mutex_lock(&lgr->rmbs_lock);
609 ext->num_rkeys = lgr->conns_num;
610 if (!ext->num_rkeys)
611 goto out;
612 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
613 for (i = 0; i < ext->num_rkeys; i++) {
614 if (!buf_pos)
615 break;
616 rmb = buf_pos;
617 ext->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey);
618 ext->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey);
619 ext->rt[i].rmb_vaddr_new =
620 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
621 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
622 while (buf_pos && !(buf_pos)->used)
623 buf_pos = smc_llc_get_next_rmb(lgr, &buf_lst, buf_pos);
624 }
625 len += i * sizeof(ext->rt[0]);
626out:
627 mutex_unlock(&lgr->rmbs_lock);
628 return len;
629}
630
52bedf37 631/* send ADD LINK request or response */
7005ada6 632int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
fbed3b37 633 struct smc_link *link_new,
52bedf37
KG		 634 enum smc_llc_reqresp reqresp)
635{
b4ba4652 636 struct smc_llc_msg_add_link_v2_ext *ext = NULL;
52bedf37
KG		 637 struct smc_llc_msg_add_link *addllc;
638 struct smc_wr_tx_pend_priv *pend;
b4ba4652 639 int len = sizeof(*addllc);
52bedf37
KG		 640 int rc;
641
95f7f3e7
KG		 642 if (!smc_wr_tx_link_hold(link))
643 return -ENOLINK;
b4ba4652
KG		 644 if (link->lgr->smc_version == SMC_V2) {
645 struct smc_wr_v2_buf *wr_buf;
646
647 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
648 if (rc)
649 goto put_out;
650 addllc = (struct smc_llc_msg_add_link *)wr_buf;
651 ext = (struct smc_llc_msg_add_link_v2_ext *)
652 &wr_buf->raw[sizeof(*addllc)];
653 memset(ext, 0, SMC_WR_TX_SIZE);
654 } else {
655 struct smc_wr_buf *wr_buf;
656
657 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
658 if (rc)
659 goto put_out;
660 addllc = (struct smc_llc_msg_add_link *)wr_buf;
661 }
fbed3b37
KG		 662
663 memset(addllc, 0, sizeof(*addllc));
b4ba4652 664 addllc->hd.common.llc_type = SMC_LLC_ADD_LINK;
fbed3b37
KG		 665 if (reqresp == SMC_LLC_RESP)
666 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
667 memcpy(addllc->sender_mac, mac, ETH_ALEN);
668 memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
669 if (link_new) {
670 addllc->link_num = link_new->link_id;
671 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
672 hton24(addllc->initial_psn, link_new->psn_initial);
673 if (reqresp == SMC_LLC_REQ)
674 addllc->qp_mtu = link_new->path_mtu;
675 else
676 addllc->qp_mtu = min(link_new->path_mtu,
677 link_new->peer_mtu);
678 }
b4ba4652
KG		 679 if (ext && link_new)
680 len += smc_llc_fill_ext_v2(ext, link, link_new);
681 smc_llc_init_msg_hdr(&addllc->hd, link->lgr, len);
52bedf37 682 /* send llc message */
b4ba4652
KG		 683 if (link->lgr->smc_version == SMC_V2)
684 rc = smc_wr_tx_v2_send(link, pend, len);
685 else
686 rc = smc_wr_tx_send(link, pend);
95f7f3e7
KG		 687put_out:
688 smc_wr_tx_link_put(link);
52bedf37
KG		 689 return rc;
690}
691
692/* send DELETE LINK request or response */
fbed3b37
KG		 693int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
694 enum smc_llc_reqresp reqresp, bool orderly,
695 u32 reason)
52bedf37
KG		 696{
697 struct smc_llc_msg_del_link *delllc;
698 struct smc_wr_tx_pend_priv *pend;
699 struct smc_wr_buf *wr_buf;
700 int rc;
701
95f7f3e7
KG		 702 if (!smc_wr_tx_link_hold(link))
703 return -ENOLINK;
52bedf37
KG		 704 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
705 if (rc)
95f7f3e7 706 goto put_out;
52bedf37 707 delllc = (struct smc_llc_msg_del_link *)wr_buf;
fbed3b37
KG		 708
709 memset(delllc, 0, sizeof(*delllc));
b4ba4652
KG		 710 delllc->hd.common.llc_type = SMC_LLC_DELETE_LINK;
711 smc_llc_init_msg_hdr(&delllc->hd, link->lgr, sizeof(*delllc));
fbed3b37
KG		 712 if (reqresp == SMC_LLC_RESP)
713 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
714 if (orderly)
715 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
716 if (link_del_id)
717 delllc->link_num = link_del_id;
718 else
719 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
720 delllc->reason = htonl(reason);
9bf9abea
UB		 721 /* send llc message */
722 rc = smc_wr_tx_send(link, pend);
95f7f3e7
KG		 723put_out:
724 smc_wr_tx_link_put(link);
9bf9abea
UB		 725 return rc;
726}
727
d97935fa
KG		 728/* send LLC test link request */
729static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
313164da
KG		 730{
731 struct smc_llc_msg_test_link *testllc;
732 struct smc_wr_tx_pend_priv *pend;
733 struct smc_wr_buf *wr_buf;
734 int rc;
735
95f7f3e7
KG		 736 if (!smc_wr_tx_link_hold(link))
737 return -ENOLINK;
313164da
KG		 738 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
739 if (rc)
95f7f3e7 740 goto put_out;
313164da
KG		 741 testllc = (struct smc_llc_msg_test_link *)wr_buf;
742 memset(testllc, 0, sizeof(*testllc));
b4ba4652
KG		 743 testllc->hd.common.llc_type = SMC_LLC_TEST_LINK;
744 smc_llc_init_msg_hdr(&testllc->hd, link->lgr, sizeof(*testllc));
313164da
KG		 745 memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
746 /* send llc message */
747 rc = smc_wr_tx_send(link, pend);
95f7f3e7
KG		 748put_out:
749 smc_wr_tx_link_put(link);
313164da
KG		 750 return rc;
751}
752
6c8968c4
KG		 753/* schedule an llc send on link, may wait for buffers */
754static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
4ed75de5
KG		 755{
756 struct smc_wr_tx_pend_priv *pend;
757 struct smc_wr_buf *wr_buf;
758 int rc;
759
95f7f3e7 760 if (!smc_wr_tx_link_hold(link))
6c8968c4
KG		 761 return -ENOLINK;
762 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
4ed75de5 763 if (rc)
95f7f3e7 764 goto put_out;
6c8968c4 765 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
95f7f3e7
KG		 766 rc = smc_wr_tx_send(link, pend);
767put_out:
768 smc_wr_tx_link_put(link);
769 return rc;
4ed75de5
KG		 770}
771
f3811fd7
KG		 772/* schedule an llc send on link, may wait for buffers,
773 * and wait for send completion notification.
774 * @return 0 on success
775 */
776static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
777{
778 struct smc_wr_tx_pend_priv *pend;
779 struct smc_wr_buf *wr_buf;
780 int rc;
781
95f7f3e7 782 if (!smc_wr_tx_link_hold(link))
f3811fd7
KG		 783 return -ENOLINK;
784 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
785 if (rc)
95f7f3e7 786 goto put_out;
f3811fd7 787 memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
95f7f3e7
KG		 788 rc = smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
789put_out:
790 smc_wr_tx_link_put(link);
791 return rc;
f3811fd7
KG		 792}
793
9bf9abea
UB		 794/********************************* receive ***********************************/
795
336ba09f
KG		 796static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
797 enum smc_lgr_type lgr_new_t)
798{
799 int i;
800
801 if (lgr->type == SMC_LGR_SYMMETRIC ||
802 (lgr->type != SMC_LGR_SINGLE &&
803 (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
804 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
805 return -EMLINK;
806
807 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
808 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
809 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
810 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
811 return i;
812 } else {
813 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
814 if (lgr->lnk[i].state == SMC_LNK_UNUSED)
815 return i;
816 }
817 return -EMLINK;
818}
819
87f88cda
KG		 820/* send one add_link_continue msg */
821static int smc_llc_add_link_cont(struct smc_link *link,
822 struct smc_link *link_new, u8 *num_rkeys_todo,
823 int *buf_lst, struct smc_buf_desc **buf_pos)
824{
825 struct smc_llc_msg_add_link_cont *addc_llc;
826 struct smc_link_group *lgr = link->lgr;
827 int prim_lnk_idx, lnk_idx, i, rc;
828 struct smc_wr_tx_pend_priv *pend;
829 struct smc_wr_buf *wr_buf;
830 struct smc_buf_desc *rmb;
831 u8 n;
832
95f7f3e7
KG		 833 if (!smc_wr_tx_link_hold(link))
834 return -ENOLINK;
87f88cda
KG		 835 rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
836 if (rc)
95f7f3e7 837 goto put_out;
87f88cda
KG		 838 addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
839 memset(addc_llc, 0, sizeof(*addc_llc));
840
841 prim_lnk_idx = link->link_idx;
842 lnk_idx = link_new->link_idx;
843 addc_llc->link_num = link_new->link_id;
844 addc_llc->num_rkeys = *num_rkeys_todo;
845 n = *num_rkeys_todo;
846 for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
847 if (!*buf_pos) {
848 addc_llc->num_rkeys = addc_llc->num_rkeys -
849 *num_rkeys_todo;
850 *num_rkeys_todo = 0;
851 break;
852 }
853 rmb = *buf_pos;
854
855 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey);
856 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey);
857 addc_llc->rt[i].rmb_vaddr_new =
858 cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
859
860 (*num_rkeys_todo)--;
861 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
862 while (*buf_pos && !(*buf_pos)->used)
863 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
864 }
b4ba4652 865 addc_llc->hd.common.llc_type = SMC_LLC_ADD_LINK_CONT;
87f88cda
KG		 866 addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
867 if (lgr->role == SMC_CLNT)
868 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
95f7f3e7
KG		 869 rc = smc_wr_tx_send(link, pend);
870put_out:
871 smc_wr_tx_link_put(link);
872 return rc;
87f88cda
KG		 873}
874
875static int smc_llc_cli_rkey_exchange(struct smc_link *link,
876 struct smc_link *link_new)
877{
878 struct smc_llc_msg_add_link_cont *addc_llc;
879 struct smc_link_group *lgr = link->lgr;
880 u8 max, num_rkeys_send, num_rkeys_recv;
881 struct smc_llc_qentry *qentry;
882 struct smc_buf_desc *buf_pos;
883 int buf_lst;
884 int rc = 0;
885 int i;
886
887 mutex_lock(&lgr->rmbs_lock);
888 num_rkeys_send = lgr->conns_num;
889 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
890 do {
891 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
892 SMC_LLC_ADD_LINK_CONT);
893 if (!qentry) {
894 rc = -ETIMEDOUT;
895 break;
896 }
897 addc_llc = &qentry->msg.add_link_cont;
898 num_rkeys_recv = addc_llc->num_rkeys;
899 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
900 for (i = 0; i < max; i++) {
901 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
902 addc_llc->rt[i].rmb_key,
903 addc_llc->rt[i].rmb_vaddr_new,
904 addc_llc->rt[i].rmb_key_new);
905 num_rkeys_recv--;
906 }
907 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
908 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
909 &buf_lst, &buf_pos);
910 if (rc)
911 break;
912 } while (num_rkeys_send || num_rkeys_recv);
913
914 mutex_unlock(&lgr->rmbs_lock);
915 return rc;
916}
917
336ba09f
KG		 918/* prepare and send an add link reject response */
919static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
920{
921 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
922 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
923 qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
b4ba4652
KG		 924 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
925 sizeof(qentry->msg));
336ba09f
KG		 926 return smc_llc_send_message(qentry->link, &qentry->msg);
927}
928
b1570a87
KG		 929static int smc_llc_cli_conf_link(struct smc_link *link,
930 struct smc_init_info *ini,
931 struct smc_link *link_new,
932 enum smc_lgr_type lgr_new_t)
933{
934 struct smc_link_group *lgr = link->lgr;
b1570a87
KG		 935 struct smc_llc_qentry *qentry = NULL;
936 int rc = 0;
937
938 /* receive CONFIRM LINK request over RoCE fabric */
939 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
940 if (!qentry) {
941 rc = smc_llc_send_delete_link(link, link_new->link_id,
942 SMC_LLC_REQ, false,
943 SMC_LLC_DEL_LOST_PATH);
944 return -ENOLINK;
945 }
b4ba4652 946 if (qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
b1570a87 947 /* received DELETE_LINK instead */
b1570a87
KG		 948 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
949 smc_llc_send_message(link, &qentry->msg);
950 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
951 return -ENOLINK;
952 }
649758ff 953 smc_llc_save_peer_uid(qentry);
b1570a87
KG		 954 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
955
956 rc = smc_ib_modify_qp_rts(link_new);
957 if (rc) {
958 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
959 false, SMC_LLC_DEL_LOST_PATH);
960 return -ENOLINK;
961 }
962 smc_wr_remember_qp_attr(link_new);
963
964 rc = smcr_buf_reg_lgr(link_new);
965 if (rc) {
966 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
967 false, SMC_LLC_DEL_LOST_PATH);
968 return -ENOLINK;
969 }
970
971 /* send CONFIRM LINK response over RoCE fabric */
972 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
973 if (rc) {
974 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
975 false, SMC_LLC_DEL_LOST_PATH);
976 return -ENOLINK;
977 }
978 smc_llc_link_active(link_new);
ad6c111b
KG		 979 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
980 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
981 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
982 else
983 smcr_lgr_set_type(lgr, lgr_new_t);
b1570a87
KG		 984 return 0;
985}
986
b4ba4652
KG		 987static void smc_llc_save_add_link_rkeys(struct smc_link *link,
988 struct smc_link *link_new)
989{
990 struct smc_llc_msg_add_link_v2_ext *ext;
991 struct smc_link_group *lgr = link->lgr;
992 int max, i;
993
994 ext = (struct smc_llc_msg_add_link_v2_ext *)((u8 *)lgr->wr_rx_buf_v2 +
995 SMC_WR_TX_SIZE);
996 max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
997 mutex_lock(&lgr->rmbs_lock);
998 for (i = 0; i < max; i++) {
999 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1000 ext->rt[i].rmb_key,
1001 ext->rt[i].rmb_vaddr_new,
1002 ext->rt[i].rmb_key_new);
1003 }
1004 mutex_unlock(&lgr->rmbs_lock);
1005}
1006
336ba09f
KG		 1007static void smc_llc_save_add_link_info(struct smc_link *link,
1008 struct smc_llc_msg_add_link *add_llc)
1009{
1010 link->peer_qpn = ntoh24(add_llc->sender_qp_num);
1011 memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
1012 memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
1013 link->peer_psn = ntoh24(add_llc->initial_psn);
1014 link->peer_mtu = add_llc->qp_mtu;
1015}
1016
1017/* as an SMC client, process an add link request */
1018int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
1019{
1020 struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
1021 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1022 struct smc_link_group *lgr = smc_get_lgr(link);
ed990df2 1023 struct smc_init_info *ini = NULL;
336ba09f 1024 struct smc_link *lnk_new = NULL;
336ba09f
KG		 1025 int lnk_idx, rc = 0;
1026
fffe83c8
KG		 1027 if (!llc->qp_mtu)
1028 goto out_reject;
1029
ed990df2
KG		 1030 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1031 if (!ini) {
1032 rc = -ENOMEM;
1033 goto out_reject;
1034 }
1035
1036 ini->vlan_id = lgr->vlan_id;
b4ba4652
KG		 1037 if (lgr->smc_version == SMC_V2) {
1038 ini->check_smcrv2 = true;
1039 ini->smcrv2.saddr = lgr->saddr;
1040 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(llc->sender_gid);
1041 }
ed990df2 1042 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
336ba09f 1043 if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
b4ba4652
KG		 1044 (lgr->smc_version == SMC_V2 ||
1045 !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1046 if (!ini->ib_dev && !ini->smcrv2.ib_dev_v2)
336ba09f
KG		 1047 goto out_reject;
1048 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1049 }
b4ba4652
KG		 1050 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1051 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1052 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1053 ini->smcrv2.ib_port_v2 = link->ibport;
1054 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
336ba09f 1055 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
ed990df2
KG		 1056 ini->ib_dev = link->smcibdev;
1057 ini->ib_port = link->ibport;
336ba09f
KG		 1058 }
1059 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1060 if (lnk_idx < 0)
1061 goto out_reject;
1062 lnk_new = &lgr->lnk[lnk_idx];
ed990df2 1063 rc = smcr_link_init(lgr, lnk_new, lnk_idx, ini);
336ba09f
KG		 1064 if (rc)
1065 goto out_reject;
1066 smc_llc_save_add_link_info(lnk_new, llc);
45fa8da0
KG		 1067 lnk_new->link_id = llc->link_num; /* SMC server assigns link id */
1068 smc_llc_link_set_uid(lnk_new);
336ba09f
KG		 1069
1070 rc = smc_ib_ready_link(lnk_new);
1071 if (rc)
1072 goto out_clear_lnk;
1073
1074 rc = smcr_buf_map_lgr(lnk_new);
1075 if (rc)
1076 goto out_clear_lnk;
1077
1078 rc = smc_llc_send_add_link(link,
b4ba4652 1079 lnk_new->smcibdev->mac[lnk_new->ibport - 1],
336ba09f
KG		 1080 lnk_new->gid, lnk_new, SMC_LLC_RESP);
1081 if (rc)
1082 goto out_clear_lnk;
b4ba4652
KG		 1083 if (lgr->smc_version == SMC_V2) {
1084 smc_llc_save_add_link_rkeys(link, lnk_new);
1085 } else {
1086 rc = smc_llc_cli_rkey_exchange(link, lnk_new);
1087 if (rc) {
1088 rc = 0;
1089 goto out_clear_lnk;
1090 }
336ba09f 1091 }
ed990df2 1092 rc = smc_llc_cli_conf_link(link, ini, lnk_new, lgr_new_t);
336ba09f
KG		 1093 if (!rc)
1094 goto out;
1095out_clear_lnk:
8f3d65c1 1096 lnk_new->state = SMC_LNK_INACTIVE;
0a99be43 1097 smcr_link_clear(lnk_new, false);
336ba09f
KG		 1098out_reject:
1099 smc_llc_cli_add_link_reject(qentry);
1100out:
ed990df2 1101 kfree(ini);
336ba09f
KG		 1102 kfree(qentry);
1103 return rc;
1104}
1105
b4ba4652
KG		 1106static void smc_llc_send_request_add_link(struct smc_link *link)
1107{
1108 struct smc_llc_msg_req_add_link_v2 *llc;
1109 struct smc_wr_tx_pend_priv *pend;
1110 struct smc_wr_v2_buf *wr_buf;
1111 struct smc_gidlist gidlist;
1112 int rc, len, i;
1113
1114 if (!smc_wr_tx_link_hold(link))
1115 return;
1116 if (link->lgr->type == SMC_LGR_SYMMETRIC ||
1117 link->lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1118 goto put_out;
1119
1120 smc_fill_gid_list(link->lgr, &gidlist, link->smcibdev, link->gid);
1121 if (gidlist.len <= 1)
1122 goto put_out;
1123
1124 rc = smc_llc_add_pending_send_v2(link, &wr_buf, &pend);
1125 if (rc)
1126 goto put_out;
1127 llc = (struct smc_llc_msg_req_add_link_v2 *)wr_buf;
1128 memset(llc, 0, SMC_WR_TX_SIZE);
1129
1130 llc->hd.common.llc_type = SMC_LLC_REQ_ADD_LINK;
1131 for (i = 0; i < gidlist.len; i++)
1132 memcpy(llc->gid[i], gidlist.list[i], sizeof(gidlist.list[0]));
1133 llc->gid_cnt = gidlist.len;
1134 len = sizeof(*llc) + (gidlist.len * sizeof(gidlist.list[0]));
1135 smc_llc_init_msg_hdr(&llc->hd, link->lgr, len);
1136 rc = smc_wr_tx_v2_send(link, pend, len);
1137 if (!rc)
1138 /* set REQ_ADD_LINK flow and wait for response from peer */
1139 link->lgr->llc_flow_lcl.type = SMC_LLC_FLOW_REQ_ADD_LINK;
1140put_out:
1141 smc_wr_tx_link_put(link);
1142}
1143
c48254fa
KG		 1144/* as an SMC client, invite server to start the add_link processing */
1145static void smc_llc_cli_add_link_invite(struct smc_link *link,
1146 struct smc_llc_qentry *qentry)
1147{
1148 struct smc_link_group *lgr = smc_get_lgr(link);
ed990df2 1149 struct smc_init_info *ini = NULL;
c48254fa 1150
b4ba4652
KG		 1151 if (lgr->smc_version == SMC_V2) {
1152 smc_llc_send_request_add_link(link);
1153 goto out;
1154 }
1155
c48254fa
KG		 1156 if (lgr->type == SMC_LGR_SYMMETRIC ||
1157 lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1158 goto out;
1159
ed990df2
KG		 1160 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1161 if (!ini)
1162 goto out;
1163
1164 ini->vlan_id = lgr->vlan_id;
1165 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
1166 if (!ini->ib_dev)
c48254fa
KG		 1167 goto out;
1168
ed990df2
KG		 1169 smc_llc_send_add_link(link, ini->ib_dev->mac[ini->ib_port - 1],
1170 ini->ib_gid, NULL, SMC_LLC_REQ);
c48254fa 1171out:
ed990df2 1172 kfree(ini);
c48254fa
KG		 1173 kfree(qentry);
1174}
1175
fffe83c8
KG		 1176static bool smc_llc_is_empty_llc_message(union smc_llc_msg *llc)
1177{
1178 int i;
1179
1180 for (i = 0; i < ARRAY_SIZE(llc->raw.data); i++)
1181 if (llc->raw.data[i])
1182 return false;
1183 return true;
1184}
1185
c48254fa
KG		 1186static bool smc_llc_is_local_add_link(union smc_llc_msg *llc)
1187{
b4ba4652 1188 if (llc->raw.hdr.common.llc_type == SMC_LLC_ADD_LINK &&
fffe83c8 1189 smc_llc_is_empty_llc_message(llc))
c48254fa
KG		 1190 return true;
1191 return false;
1192}
1193
b1570a87
KG		 1194static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
1195{
1196 struct smc_llc_qentry *qentry;
1197
1198 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1199
1200 mutex_lock(&lgr->llc_conf_mutex);
c48254fa
KG		 1201 if (smc_llc_is_local_add_link(&qentry->msg))
1202 smc_llc_cli_add_link_invite(qentry->link, qentry);
1203 else
1204 smc_llc_cli_add_link(qentry->link, qentry);
b1570a87
KG		 1205 mutex_unlock(&lgr->llc_conf_mutex);
1206}
1207
9c416878
KG		 1208static int smc_llc_active_link_count(struct smc_link_group *lgr)
1209{
1210 int i, link_count = 0;
1211
1212 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
741a49a4 1213 if (!smc_link_active(&lgr->lnk[i]))
9c416878
KG		 1214 continue;
1215 link_count++;
1216 }
1217 return link_count;
1218}
1219
c9a5d243
KG		 1220/* find the asymmetric link when 3 links are established */
1221static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
1222{
1223 int asym_idx = -ENOENT;
1224 int i, j, k;
1225 bool found;
1226
1227 /* determine asymmetric link */
1228 found = false;
1229 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1230 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
1231 if (!smc_link_usable(&lgr->lnk[i]) ||
1232 !smc_link_usable(&lgr->lnk[j]))
1233 continue;
1234 if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
1235 SMC_GID_SIZE)) {
1236 found = true; /* asym_lnk is i or j */
1237 break;
1238 }
1239 }
1240 if (found)
1241 break;
1242 }
1243 if (!found)
1244 goto out; /* no asymmetric link */
1245 for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
1246 if (!smc_link_usable(&lgr->lnk[k]))
1247 continue;
1248 if (k != i &&
1249 !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
1250 SMC_GID_SIZE)) {
1251 asym_idx = i;
1252 break;
1253 }
1254 if (k != j &&
1255 !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
1256 SMC_GID_SIZE)) {
1257 asym_idx = j;
1258 break;
1259 }
1260 }
1261out:
1262 return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
1263}
1264
1265static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
1266{
1267 struct smc_link *lnk_new = NULL, *lnk_asym;
1268 struct smc_llc_qentry *qentry;
1269 int rc;
1270
1271 lnk_asym = smc_llc_find_asym_link(lgr);
1272 if (!lnk_asym)
1273 return; /* no asymmetric link */
1274 if (!smc_link_downing(&lnk_asym->state))
1275 return;
c6f02ebe 1276 lnk_new = smc_switch_conns(lgr, lnk_asym, false);
c9a5d243
KG		 1277 smc_wr_tx_wait_no_pending_sends(lnk_asym);
1278 if (!lnk_new)
1279 goto out_free;
1280 /* change flow type from ADD_LINK into DEL_LINK */
1281 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
1282 rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
1283 true, SMC_LLC_DEL_NO_ASYM_NEEDED);
1284 if (rc) {
1285 smcr_link_down_cond(lnk_new);
1286 goto out_free;
1287 }
1288 qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
1289 SMC_LLC_DELETE_LINK);
1290 if (!qentry) {
1291 smcr_link_down_cond(lnk_new);
1292 goto out_free;
1293 }
1294 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1295out_free:
0a99be43 1296 smcr_link_clear(lnk_asym, true);
c9a5d243
KG		 1297}
1298
57b49924
KG		 1299static int smc_llc_srv_rkey_exchange(struct smc_link *link,
1300 struct smc_link *link_new)
1301{
1302 struct smc_llc_msg_add_link_cont *addc_llc;
1303 struct smc_link_group *lgr = link->lgr;
1304 u8 max, num_rkeys_send, num_rkeys_recv;
1305 struct smc_llc_qentry *qentry = NULL;
1306 struct smc_buf_desc *buf_pos;
1307 int buf_lst;
1308 int rc = 0;
1309 int i;
1310
1311 mutex_lock(&lgr->rmbs_lock);
1312 num_rkeys_send = lgr->conns_num;
1313 buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
1314 do {
1315 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
1316 &buf_lst, &buf_pos);
1317 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1318 SMC_LLC_ADD_LINK_CONT);
1319 if (!qentry) {
1320 rc = -ETIMEDOUT;
1321 goto out;
1322 }
1323 addc_llc = &qentry->msg.add_link_cont;
1324 num_rkeys_recv = addc_llc->num_rkeys;
1325 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1326 for (i = 0; i < max; i++) {
1327 smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1328 addc_llc->rt[i].rmb_key,
1329 addc_llc->rt[i].rmb_vaddr_new,
1330 addc_llc->rt[i].rmb_key_new);
1331 num_rkeys_recv--;
1332 }
1333 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1334 } while (num_rkeys_send || num_rkeys_recv);
1335out:
1336 mutex_unlock(&lgr->rmbs_lock);
1337 return rc;
1338}
1339
1551c95b
KG		 1340static int smc_llc_srv_conf_link(struct smc_link *link,
1341 struct smc_link *link_new,
1342 enum smc_lgr_type lgr_new_t)
1343{
1344 struct smc_link_group *lgr = link->lgr;
1345 struct smc_llc_qentry *qentry = NULL;
1346 int rc;
1347
1348 /* send CONFIRM LINK request over the RoCE fabric */
1349 rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1350 if (rc)
1351 return -ENOLINK;
1352 /* receive CONFIRM LINK response over the RoCE fabric */
a35fffbf
KG		 1353 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, 0);
1354 if (!qentry ||
b4ba4652 1355 qentry->msg.raw.hdr.common.llc_type != SMC_LLC_CONFIRM_LINK) {
1551c95b
KG		 1356 /* send DELETE LINK */
1357 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1358 false, SMC_LLC_DEL_LOST_PATH);
a35fffbf
KG		 1359 if (qentry)
1360 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1551c95b
KG		 1361 return -ENOLINK;
1362 }
649758ff 1363 smc_llc_save_peer_uid(qentry);
1551c95b 1364 smc_llc_link_active(link_new);
ad6c111b
KG		 1365 if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1366 lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1367 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1368 else
1369 smcr_lgr_set_type(lgr, lgr_new_t);
1551c95b
KG		 1370 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1371 return 0;
1372}
1373
b4ba4652
KG		 1374static void smc_llc_send_req_add_link_response(struct smc_llc_qentry *qentry)
1375{
1376 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
1377 smc_llc_init_msg_hdr(&qentry->msg.raw.hdr, qentry->link->lgr,
1378 sizeof(qentry->msg));
1379 memset(&qentry->msg.raw.data, 0, sizeof(qentry->msg.raw.data));
1380 smc_llc_send_message(qentry->link, &qentry->msg);
1381}
1382
1383int smc_llc_srv_add_link(struct smc_link *link,
1384 struct smc_llc_qentry *req_qentry)
2d2209f2
KG		 1385{
1386 enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1387 struct smc_link_group *lgr = link->lgr;
1388 struct smc_llc_msg_add_link *add_llc;
1389 struct smc_llc_qentry *qentry = NULL;
b4ba4652 1390 bool send_req_add_link_resp = false;
ed990df2 1391 struct smc_link *link_new = NULL;
b4ba4652 1392 struct smc_init_info *ini = NULL;
2d2209f2
KG		 1393 int lnk_idx, rc = 0;
1394
b4ba4652
KG		 1395 if (req_qentry &&
1396 req_qentry->msg.raw.hdr.common.llc_type == SMC_LLC_REQ_ADD_LINK)
1397 send_req_add_link_resp = true;
1398
ed990df2 1399 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
b4ba4652
KG		 1400 if (!ini) {
1401 rc = -ENOMEM;
1402 goto out;
1403 }
ed990df2 1404
2d2209f2 1405 /* ignore client add link recommendation, start new flow */
ed990df2 1406 ini->vlan_id = lgr->vlan_id;
b4ba4652
KG		 1407 if (lgr->smc_version == SMC_V2) {
1408 ini->check_smcrv2 = true;
1409 ini->smcrv2.saddr = lgr->saddr;
1410 if (send_req_add_link_resp) {
1411 struct smc_llc_msg_req_add_link_v2 *req_add =
1412 &req_qentry->msg.req_add_link;
1413
1414 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(req_add->gid[0]);
1415 }
1416 }
ed990df2 1417 smc_pnet_find_alt_roce(lgr, ini, link->smcibdev);
b4ba4652
KG		 1418 if (lgr->smc_version == SMC_V2 && !ini->smcrv2.ib_dev_v2) {
1419 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1420 ini->smcrv2.ib_dev_v2 = link->smcibdev;
1421 ini->smcrv2.ib_port_v2 = link->ibport;
1422 } else if (lgr->smc_version < SMC_V2 && !ini->ib_dev) {
2d2209f2 1423 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
ed990df2
KG		 1424 ini->ib_dev = link->smcibdev;
1425 ini->ib_port = link->ibport;
2d2209f2
KG		 1426 }
1427 lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
ed990df2
KG		 1428 if (lnk_idx < 0) {
1429 rc = 0;
1430 goto out;
1431 }
2d2209f2 1432
ed990df2 1433 rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, ini);
2d2209f2 1434 if (rc)
ed990df2 1435 goto out;
2d2209f2 1436 link_new = &lgr->lnk[lnk_idx];
b4ba4652
KG		 1437
1438 rc = smcr_buf_map_lgr(link_new);
1439 if (rc)
1440 goto out_err;
1441
2d2209f2 1442 rc = smc_llc_send_add_link(link,
b4ba4652 1443 link_new->smcibdev->mac[link_new->ibport-1],
2d2209f2
KG		 1444 link_new->gid, link_new, SMC_LLC_REQ);
1445 if (rc)
1446 goto out_err;
b4ba4652 1447 send_req_add_link_resp = false;
2d2209f2
KG		 1448 /* receive ADD LINK response over the RoCE fabric */
1449 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1450 if (!qentry) {
1451 rc = -ETIMEDOUT;
1452 goto out_err;
1453 }
1454 add_llc = &qentry->msg.add_link;
1455 if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1456 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1457 rc = -ENOLINK;
1458 goto out_err;
1459 }
1460 if (lgr->type == SMC_LGR_SINGLE &&
1461 (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
b4ba4652
KG		 1462 (lgr->smc_version == SMC_V2 ||
1463 !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN)))) {
2d2209f2
KG		 1464 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1465 }
1466 smc_llc_save_add_link_info(link_new, add_llc);
1467 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1468
1469 rc = smc_ib_ready_link(link_new);
2d2209f2
KG		 1470 if (rc)
1471 goto out_err;
1472 rc = smcr_buf_reg_lgr(link_new);
1473 if (rc)
1474 goto out_err;
b4ba4652
KG		 1475 if (lgr->smc_version == SMC_V2) {
1476 smc_llc_save_add_link_rkeys(link, link_new);
1477 } else {
1478 rc = smc_llc_srv_rkey_exchange(link, link_new);
1479 if (rc)
1480 goto out_err;
1481 }
1551c95b 1482 rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
2d2209f2
KG		 1483 if (rc)
1484 goto out_err;
ed990df2 1485 kfree(ini);
2d2209f2
KG		 1486 return 0;
1487out_err:
ed990df2
KG		 1488 if (link_new) {
1489 link_new->state = SMC_LNK_INACTIVE;
1490 smcr_link_clear(link_new, false);
1491 }
1492out:
1493 kfree(ini);
b4ba4652
KG		 1494 if (send_req_add_link_resp)
1495 smc_llc_send_req_add_link_response(req_qentry);
2d2209f2
KG		 1496 return rc;
1497}
1498
1499static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1500{
1501 struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
b4ba4652 1502 struct smc_llc_qentry *qentry;
2d2209f2
KG		 1503 int rc;
1504
b4ba4652 1505 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
2d2209f2
KG		 1506
1507 mutex_lock(&lgr->llc_conf_mutex);
b4ba4652 1508 rc = smc_llc_srv_add_link(link, qentry);
2d2209f2
KG		 1509 if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1510 /* delete any asymmetric link */
c9a5d243 1511 smc_llc_delete_asym_link(lgr);
2d2209f2
KG		 1512 }
1513 mutex_unlock(&lgr->llc_conf_mutex);
b4ba4652 1514 kfree(qentry);
2d2209f2
KG		 1515}
1516
c48254fa
KG		 1517/* enqueue a local add_link req to trigger a new add_link flow */
1518void smc_llc_add_link_local(struct smc_link *link)
4dadd151 1519{
16cb3653 1520 struct smc_llc_msg_add_link add_llc = {};
4dadd151 1521
b4ba4652
KG		 1522 add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK;
1523 smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc));
c48254fa 1524 /* no dev and port needed */
4dadd151
KG		 1525 smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1526}
1527
b45e7f98
KG		 1528/* worker to process an add link message */
1529static void smc_llc_add_link_work(struct work_struct *work)
1530{
1531 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1532 llc_add_link_work);
1533
1534 if (list_empty(&lgr->list)) {
1535 /* link group is terminating */
1536 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1537 goto out;
1538 }
1539
b1570a87
KG		 1540 if (lgr->role == SMC_CLNT)
1541 smc_llc_process_cli_add_link(lgr);
2d2209f2
KG		 1542 else
1543 smc_llc_process_srv_add_link(lgr);
b45e7f98 1544out:
b4ba4652
KG		 1545 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_REQ_ADD_LINK)
1546 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
b45e7f98
KG		 1547}
1548
4dadd151
KG		 1549/* enqueue a local del_link msg to trigger a new del_link flow,
1550 * called only for role SMC_SERV
1551 */
1552void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1553{
16cb3653 1554 struct smc_llc_msg_del_link del_llc = {};
4dadd151 1555
b4ba4652
KG		 1556 del_llc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1557 smc_llc_init_msg_hdr(&del_llc.hd, link->lgr, sizeof(del_llc));
4dadd151
KG		 1558 del_llc.link_num = del_link_id;
1559 del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1560 del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1561 smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1562}
1563
9c416878
KG		 1564static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1565{
1566 struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1567 struct smc_llc_msg_del_link *del_llc;
1568 struct smc_llc_qentry *qentry;
1569 int active_links;
1570 int lnk_idx;
1571
1572 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1573 lnk = qentry->link;
1574 del_llc = &qentry->msg.delete_link;
1575
1576 if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1577 smc_lgr_terminate_sched(lgr);
1578 goto out;
1579 }
1580 mutex_lock(&lgr->llc_conf_mutex);
1581 /* delete single link */
1582 for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1583 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1584 continue;
1585 lnk_del = &lgr->lnk[lnk_idx];
1586 break;
1587 }
1588 del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1589 if (!lnk_del) {
1590 /* link was not found */
1591 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1592 smc_llc_send_message(lnk, &qentry->msg);
1593 goto out_unlock;
1594 }
1595 lnk_asym = smc_llc_find_asym_link(lgr);
1596
1597 del_llc->reason = 0;
1598 smc_llc_send_message(lnk, &qentry->msg); /* response */
1599
8f3d65c1
KG		 1600 if (smc_link_downing(&lnk_del->state))
1601 smc_switch_conns(lgr, lnk_del, false);
0a99be43 1602 smcr_link_clear(lnk_del, true);
9c416878
KG		 1603
1604 active_links = smc_llc_active_link_count(lgr);
1605 if (lnk_del == lnk_asym) {
1606 /* expected deletion of asym link, don't change lgr state */
1607 } else if (active_links == 1) {
ad6c111b 1608 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
9c416878 1609 } else if (!active_links) {
ad6c111b 1610 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
9c416878
KG		 1611 smc_lgr_terminate_sched(lgr);
1612 }
1613out_unlock:
1614 mutex_unlock(&lgr->llc_conf_mutex);
1615out:
1616 kfree(qentry);
1617}
1618
f3811fd7
KG		 1619/* try to send a DELETE LINK ALL request on any active link,
1620 * waiting for send completion
1621 */
1622void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1623{
7e94e46c 1624 struct smc_llc_msg_del_link delllc = {};
f3811fd7
KG		 1625 int i;
1626
b4ba4652
KG		 1627 delllc.hd.common.llc_type = SMC_LLC_DELETE_LINK;
1628 smc_llc_init_msg_hdr(&delllc.hd, lgr, sizeof(delllc));
f3811fd7
KG		 1629 if (ord)
1630 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1631 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1632 delllc.reason = htonl(rsn);
1633
1634 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
90cee52f 1635 if (!smc_link_sendable(&lgr->lnk[i]))
f3811fd7
KG		 1636 continue;
1637 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1638 break;
1639 }
1640}
1641
08ae27dd
KG		 1642static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1643{
1644 struct smc_llc_msg_del_link *del_llc;
1645 struct smc_link *lnk, *lnk_del;
1646 struct smc_llc_qentry *qentry;
1647 int active_links;
1648 int i;
1649
1650 mutex_lock(&lgr->llc_conf_mutex);
1651 qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1652 lnk = qentry->link;
1653 del_llc = &qentry->msg.delete_link;
1654
1655 if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1656 /* delete entire lgr */
f3811fd7
KG		 1657 smc_llc_send_link_delete_all(lgr, true, ntohl(
1658 qentry->msg.delete_link.reason));
08ae27dd
KG		 1659 smc_lgr_terminate_sched(lgr);
1660 goto out;
1661 }
1662 /* delete single link */
1663 lnk_del = NULL;
1664 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1665 if (lgr->lnk[i].link_id == del_llc->link_num) {
1666 lnk_del = &lgr->lnk[i];
1667 break;
1668 }
1669 }
1670 if (!lnk_del)
1671 goto out; /* asymmetric link already deleted */
1672
1673 if (smc_link_downing(&lnk_del->state)) {
b7eede75
KG		 1674 if (smc_switch_conns(lgr, lnk_del, false))
1675 smc_wr_tx_wait_no_pending_sends(lnk_del);
08ae27dd
KG		 1676 }
1677 if (!list_empty(&lgr->list)) {
1678 /* qentry is either a request from peer (send it back to
1679 * initiate the DELETE_LINK processing), or a locally
1680 * enqueued DELETE_LINK request (forward it)
1681 */
1682 if (!smc_llc_send_message(lnk, &qentry->msg)) {
08ae27dd
KG		 1683 struct smc_llc_qentry *qentry2;
1684
1685 qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1686 SMC_LLC_DELETE_LINK);
ca7e3edc 1687 if (qentry2)
08ae27dd 1688 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
08ae27dd
KG		 1689 }
1690 }
0a99be43 1691 smcr_link_clear(lnk_del, true);
08ae27dd
KG		 1692
1693 active_links = smc_llc_active_link_count(lgr);
1694 if (active_links == 1) {
ad6c111b 1695 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
08ae27dd 1696 } else if (!active_links) {
ad6c111b 1697 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
08ae27dd
KG		 1698 smc_lgr_terminate_sched(lgr);
1699 }
1700
1701 if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1702 /* trigger setup of asymm alt link */
c48254fa 1703 smc_llc_add_link_local(lnk);
08ae27dd
KG		 1704 }
1705out:
1706 mutex_unlock(&lgr->llc_conf_mutex);
1707 kfree(qentry);
1708}
1709
9ec6bf19 1710static void smc_llc_delete_link_work(struct work_struct *work)
52bedf37 1711{
9ec6bf19
KG		 1712 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1713 llc_del_link_work);
52bedf37 1714
9ec6bf19
KG		 1715 if (list_empty(&lgr->list)) {
1716 /* link group is terminating */
1717 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1718 goto out;
52bedf37 1719 }
9c416878
KG		 1720
1721 if (lgr->role == SMC_CLNT)
1722 smc_llc_process_cli_delete_link(lgr);
08ae27dd
KG		 1723 else
1724 smc_llc_process_srv_delete_link(lgr);
9ec6bf19
KG		 1725out:
1726 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
52bedf37
KG		 1727}
1728
3bc67e09
KG		 1729/* process a confirm_rkey request from peer, remote flow */
1730static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
4ed75de5 1731{
3bc67e09
KG		 1732 struct smc_llc_msg_confirm_rkey *llc;
1733 struct smc_llc_qentry *qentry;
1734 struct smc_link *link;
1735 int num_entries;
1736 int rk_idx;
1737 int i;
1738
1739 qentry = lgr->llc_flow_rmt.qentry;
1740 llc = &qentry->msg.confirm_rkey;
1741 link = qentry->link;
1742
1743 num_entries = llc->rtoken[0].num_rkeys;
b4ba4652
KG		 1744 if (num_entries > SMC_LLC_RKEYS_PER_MSG)
1745 goto out_err;
3bc67e09
KG		 1746 /* first rkey entry is for receiving link */
1747 rk_idx = smc_rtoken_add(link,
1748 llc->rtoken[0].rmb_vaddr,
1749 llc->rtoken[0].rmb_key);
1750 if (rk_idx < 0)
1751 goto out_err;
1752
1753 for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1754 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1755 llc->rtoken[i].rmb_vaddr,
1756 llc->rtoken[i].rmb_key);
1757 /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1758 goto out;
1759out_err:
1760 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1761 llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1762out:
ef79d439 1763 llc->hd.flags |= SMC_LLC_FLAG_RESP;
b4ba4652 1764 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
3bc67e09
KG		 1765 smc_llc_send_message(link, &qentry->msg);
1766 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
4ed75de5
KG		 1767}
1768
218b24fe
KG		 1769/* process a delete_rkey request from peer, remote flow */
1770static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
4ed75de5 1771{
218b24fe
KG		 1772 struct smc_llc_msg_delete_rkey *llc;
1773 struct smc_llc_qentry *qentry;
1774 struct smc_link *link;
4ed75de5
KG		 1775 u8 err_mask = 0;
1776 int i, max;
1777
218b24fe
KG		 1778 qentry = lgr->llc_flow_rmt.qentry;
1779 llc = &qentry->msg.delete_rkey;
1780 link = qentry->link;
1781
b4ba4652
KG		 1782 if (lgr->smc_version == SMC_V2) {
1783 struct smc_llc_msg_delete_rkey_v2 *llcv2;
1784
1785 memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc));
1786 llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2;
1787 llcv2->num_inval_rkeys = 0;
1788
1789 max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2);
1790 for (i = 0; i < max; i++) {
1791 if (smc_rtoken_delete(link, llcv2->rkey[i]))
1792 llcv2->num_inval_rkeys++;
1793 }
1794 memset(&llc->rkey[0], 0, sizeof(llc->rkey));
1795 memset(&llc->reserved2, 0, sizeof(llc->reserved2));
1796 smc_llc_init_msg_hdr(&llc->hd, link->lgr, sizeof(*llc));
1797 if (llcv2->num_inval_rkeys) {
1798 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1799 llc->err_mask = llcv2->num_inval_rkeys;
1800 }
1801 goto finish;
1802 }
1803
ef79d439
KG		 1804 max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1805 for (i = 0; i < max; i++) {
1806 if (smc_rtoken_delete(link, llc->rkey[i]))
1807 err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1808 }
ef79d439
KG		 1809 if (err_mask) {
1810 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1811 llc->err_mask = err_mask;
4ed75de5 1812 }
b4ba4652 1813finish:
218b24fe
KG		 1814 llc->hd.flags |= SMC_LLC_FLAG_RESP;
1815 smc_llc_send_message(link, &qentry->msg);
1816 smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1817}
ef79d439 1818
3e0c40af
KG		 1819static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1820{
de2fea7b
TL		 1821 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu LLC protocol violation: "
1822 "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id,
1823 lgr->net->net_cookie, type);
3e0c40af
KG		 1824 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1825 smc_lgr_terminate_sched(lgr);
1826}
1827
6c8968c4 1828/* flush the llc event queue */
00a049cf 1829static void smc_llc_event_flush(struct smc_link_group *lgr)
9bf9abea 1830{
6c8968c4
KG		 1831 struct smc_llc_qentry *qentry, *q;
1832
1833 spin_lock_bh(&lgr->llc_event_q_lock);
1834 list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1835 list_del_init(&qentry->list);
1836 kfree(qentry);
1837 }
1838 spin_unlock_bh(&lgr->llc_event_q_lock);
1839}
1840
1841static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1842{
1843 union smc_llc_msg *llc = &qentry->msg;
1844 struct smc_link *link = qentry->link;
0fb0b02b 1845 struct smc_link_group *lgr = link->lgr;
9bf9abea 1846
d854fcbf 1847 if (!smc_link_usable(link))
6c8968c4 1848 goto out;
313164da 1849
b4ba4652 1850 switch (llc->raw.hdr.common.llc_type) {
313164da 1851 case SMC_LLC_TEST_LINK:
56e8091c
KG		 1852 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1853 smc_llc_send_message(link, llc);
313164da 1854 break;
52bedf37 1855 case SMC_LLC_ADD_LINK:
0fb0b02b
KG		 1856 if (list_empty(&lgr->list))
1857 goto out; /* lgr is terminating */
1858 if (lgr->role == SMC_CLNT) {
c48254fa
KG		 1859 if (smc_llc_is_local_add_link(llc)) {
1860 if (lgr->llc_flow_lcl.type ==
1861 SMC_LLC_FLOW_ADD_LINK)
1862 break; /* add_link in progress */
1863 if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1864 qentry)) {
1865 schedule_work(&lgr->llc_add_link_work);
1866 }
1867 return;
1868 }
1869 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1870 !lgr->llc_flow_lcl.qentry) {
0fb0b02b
KG		 1871 /* a flow is waiting for this message */
1872 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1873 qentry);
6778a6be 1874 wake_up(&lgr->llc_msg_waiter);
b4ba4652
KG		 1875 return;
1876 }
1877 if (lgr->llc_flow_lcl.type ==
1878 SMC_LLC_FLOW_REQ_ADD_LINK) {
1879 /* server started add_link processing */
1880 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1881 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1882 qentry);
1883 schedule_work(&lgr->llc_add_link_work);
1884 return;
1885 }
1886 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
b45e7f98 1887 schedule_work(&lgr->llc_add_link_work);
0fb0b02b
KG		 1888 }
1889 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1890 /* as smc server, handle client suggestion */
b45e7f98 1891 schedule_work(&lgr->llc_add_link_work);
0fb0b02b
KG		 1892 }
1893 return;
1894 case SMC_LLC_CONFIRM_LINK:
87f88cda 1895 case SMC_LLC_ADD_LINK_CONT:
0fb0b02b
KG		 1896 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1897 /* a flow is waiting for this message */
1898 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
6778a6be 1899 wake_up(&lgr->llc_msg_waiter);
0fb0b02b
KG		 1900 return;
1901 }
52bedf37
KG		 1902 break;
1903 case SMC_LLC_DELETE_LINK:
b9979c2e
KG		 1904 if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1905 !lgr->llc_flow_lcl.qentry) {
1906 /* DEL LINK REQ during ADD LINK SEQ */
1907 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1908 wake_up(&lgr->llc_msg_waiter);
1909 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1910 schedule_work(&lgr->llc_del_link_work);
9ec6bf19
KG		 1911 }
1912 return;
4ed75de5 1913 case SMC_LLC_CONFIRM_RKEY:
3bc67e09
KG		 1914 /* new request from remote, assign to remote flow */
1915 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1916 /* process here, does not wait for more llc msgs */
1917 smc_llc_rmt_conf_rkey(lgr);
1918 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1919 }
1920 return;
4ed75de5 1921 case SMC_LLC_CONFIRM_RKEY_CONT:
42d18acc
KG		 1922 /* not used because max links is 3, and 3 rkeys fit into
1923 * one CONFIRM_RKEY message
1924 */
4ed75de5
KG		 1925 break;
1926 case SMC_LLC_DELETE_RKEY:
218b24fe
KG		 1927 /* new request from remote, assign to remote flow */
1928 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1929 /* process here, does not wait for more llc msgs */
1930 smc_llc_rmt_delete_rkey(lgr);
1931 smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1932 }
1933 return;
b4ba4652
KG		 1934 case SMC_LLC_REQ_ADD_LINK:
1935 /* handle response here, smc_llc_flow_stop() cannot be called
1936 * in tasklet context
1937 */
1938 if (lgr->role == SMC_CLNT &&
1939 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK &&
1940 (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP)) {
1941 smc_llc_flow_stop(link->lgr, &lgr->llc_flow_lcl);
1942 } else if (lgr->role == SMC_SERV) {
1943 if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1944 /* as smc server, handle client suggestion */
1945 lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK;
1946 schedule_work(&lgr->llc_add_link_work);
1947 }
1948 return;
1949 }
1950 break;
3e0c40af
KG		 1951 default:
1952 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1953 break;
313164da 1954 }
6c8968c4
KG		 1955out:
1956 kfree(qentry);
1957}
1958
1959/* worker to process llc messages on the event queue */
1960static void smc_llc_event_work(struct work_struct *work)
1961{
1962 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1963 llc_event_work);
1964 struct smc_llc_qentry *qentry;
1965
555da9af 1966 if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
d535ca13
KG		 1967 qentry = lgr->delayed_event;
1968 lgr->delayed_event = NULL;
1969 if (smc_link_usable(qentry->link))
1970 smc_llc_event_handler(qentry);
1971 else
555da9af 1972 kfree(qentry);
555da9af
KG		 1973 }
1974
6c8968c4
KG		 1975again:
1976 spin_lock_bh(&lgr->llc_event_q_lock);
1977 if (!list_empty(&lgr->llc_event_q)) {
1978 qentry = list_first_entry(&lgr->llc_event_q,
1979 struct smc_llc_qentry, list);
1980 list_del_init(&qentry->list);
1981 spin_unlock_bh(&lgr->llc_event_q_lock);
1982 smc_llc_event_handler(qentry);
1983 goto again;
1984 }
1985 spin_unlock_bh(&lgr->llc_event_q_lock);
1986}
1987
ef79d439 1988/* process llc responses in tasklet context */
a6688d91
KG		 1989static void smc_llc_rx_response(struct smc_link *link,
1990 struct smc_llc_qentry *qentry)
ef79d439 1991{
2ff08678
KG		 1992 enum smc_llc_flowtype flowtype = link->lgr->llc_flow_lcl.type;
1993 struct smc_llc_flow *flow = &link->lgr->llc_flow_lcl;
b4ba4652 1994 u8 llc_type = qentry->msg.raw.hdr.common.llc_type;
ef79d439 1995
a6688d91 1996 switch (llc_type) {
ef79d439 1997 case SMC_LLC_TEST_LINK:
741a49a4 1998 if (smc_link_active(link))
ef79d439
KG		 1999 complete(&link->llc_testlink_resp);
2000 break;
ef79d439 2001 case SMC_LLC_ADD_LINK:
87f88cda 2002 case SMC_LLC_ADD_LINK_CONT:
2ff08678
KG		 2003 case SMC_LLC_CONFIRM_LINK:
2004 if (flowtype != SMC_LLC_FLOW_ADD_LINK || flow->qentry)
2005 break; /* drop out-of-flow response */
2006 goto assign;
2007 case SMC_LLC_DELETE_LINK:
2008 if (flowtype != SMC_LLC_FLOW_DEL_LINK || flow->qentry)
2009 break; /* drop out-of-flow response */
2010 goto assign;
3d88a21b 2011 case SMC_LLC_CONFIRM_RKEY:
6d74c3a8 2012 case SMC_LLC_DELETE_RKEY:
2ff08678
KG		 2013 if (flowtype != SMC_LLC_FLOW_RKEY || flow->qentry)
2014 break; /* drop out-of-flow response */
2015 goto assign;
ef79d439 2016 case SMC_LLC_CONFIRM_RKEY_CONT:
42d18acc 2017 /* not used because max links is 3 */
ef79d439 2018 break;
3e0c40af 2019 default:
b4ba4652
KG		 2020 smc_llc_protocol_violation(link->lgr,
2021 qentry->msg.raw.hdr.common.type);
3e0c40af 2022 break;
ef79d439 2023 }
a6688d91 2024 kfree(qentry);
2ff08678
KG		 2025 return;
2026assign:
2027 /* assign responses to the local flow, we requested them */
2028 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
2029 wake_up(&link->lgr->llc_msg_waiter);
ef79d439
KG		 2030}
2031
a6688d91 2032static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
6c8968c4 2033{
6c8968c4
KG		 2034 struct smc_link_group *lgr = link->lgr;
2035 struct smc_llc_qentry *qentry;
6c8968c4
KG		 2036 unsigned long flags;
2037
6c8968c4
KG		 2038 qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
2039 if (!qentry)
2040 return;
2041 qentry->link = link;
2042 INIT_LIST_HEAD(&qentry->list);
2043 memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
a6688d91
KG		 2044
2045 /* process responses immediately */
b4ba4652
KG		 2046 if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) &&
2047 llc->raw.hdr.common.llc_type != SMC_LLC_REQ_ADD_LINK) {
a6688d91
KG		 2048 smc_llc_rx_response(link, qentry);
2049 return;
2050 }
2051
2052 /* add requests to event queue */
6c8968c4
KG		 2053 spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
2054 list_add_tail(&qentry->list, &lgr->llc_event_q);
2055 spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
22ef473d 2056 queue_work(system_highpri_wq, &lgr->llc_event_work);
9bf9abea
UB		 2057}
2058
a6688d91
KG		 2059/* copy received msg and add it to the event queue */
2060static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
2061{
2062 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
2063 union smc_llc_msg *llc = buf;
2064
2065 if (wc->byte_len < sizeof(*llc))
2066 return; /* short message */
b4ba4652
KG		 2067 if (!llc->raw.hdr.common.llc_version) {
2068 if (llc->raw.hdr.length != sizeof(*llc))
2069 return; /* invalid message */
2070 } else {
2071 if (llc->raw.hdr.length_v2 < sizeof(*llc))
2072 return; /* invalid message */
2073 }
a6688d91
KG		 2074
2075 smc_llc_enqueue(link, llc);
2076}
2077
44aa81ce 2078/***************************** worker, utils *********************************/
877ae5be
KG		 2079
2080static void smc_llc_testlink_work(struct work_struct *work)
2081{
2082 struct smc_link *link = container_of(to_delayed_work(work),
2083 struct smc_link, llc_testlink_wrk);
2084 unsigned long next_interval;
877ae5be
KG		 2085 unsigned long expire_time;
2086 u8 user_data[16] = { 0 };
2087 int rc;
2088
741a49a4 2089 if (!smc_link_active(link))
877ae5be
KG		 2090 return; /* don't reschedule worker */
2091 expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
2092 if (time_is_after_jiffies(expire_time)) {
2093 next_interval = expire_time - jiffies;
2094 goto out;
2095 }
2096 reinit_completion(&link->llc_testlink_resp);
d97935fa 2097 smc_llc_send_test_link(link, user_data);
877ae5be
KG		 2098 /* receive TEST LINK response over RoCE fabric */
2099 rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
2100 SMC_LLC_WAIT_TIME);
741a49a4 2101 if (!smc_link_active(link))
1020e1ef 2102 return; /* link state changed */
877ae5be 2103 if (rc <= 0) {
87523930 2104 smcr_link_down_cond_sched(link);
877ae5be
KG		 2105 return;
2106 }
2107 next_interval = link->llc_testlink_time;
2108out:
1020e1ef 2109 schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
877ae5be
KG		 2110}
2111
00a049cf
KG		 2112void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
2113{
2114 struct net *net = sock_net(smc->clcsock->sk);
2115
2116 INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
b45e7f98 2117 INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
9ec6bf19 2118 INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
00a049cf
KG		 2119 INIT_LIST_HEAD(&lgr->llc_event_q);
2120 spin_lock_init(&lgr->llc_event_q_lock);
555da9af 2121 spin_lock_init(&lgr->llc_flow_lock);
6778a6be
KG		 2122 init_waitqueue_head(&lgr->llc_flow_waiter);
2123 init_waitqueue_head(&lgr->llc_msg_waiter);
d5500667 2124 mutex_init(&lgr->llc_conf_mutex);
00a049cf
KG		 2125 lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
2126}
2127
2128/* called after lgr was removed from lgr_list */
2129void smc_llc_lgr_clear(struct smc_link_group *lgr)
2130{
2131 smc_llc_event_flush(lgr);
6778a6be
KG		 2132 wake_up_all(&lgr->llc_flow_waiter);
2133 wake_up_all(&lgr->llc_msg_waiter);
00a049cf 2134 cancel_work_sync(&lgr->llc_event_work);
b45e7f98 2135 cancel_work_sync(&lgr->llc_add_link_work);
9ec6bf19 2136 cancel_work_sync(&lgr->llc_del_link_work);
555da9af
KG		 2137 if (lgr->delayed_event) {
2138 kfree(lgr->delayed_event);
2139 lgr->delayed_event = NULL;
2140 }
00a049cf
KG		 2141}
2142
2a4c57a9 2143int smc_llc_link_init(struct smc_link *link)
877ae5be
KG		 2144{
2145 init_completion(&link->llc_testlink_resp);
2146 INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
2a4c57a9 2147 return 0;
b32cf4ab
KG		 2148}
2149
00a049cf 2150void smc_llc_link_active(struct smc_link *link)
b32cf4ab 2151{
de2fea7b 2152 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link added: id %*phN, "
0a99be43
KG		 2153 "peerid %*phN, ibdev %s, ibport %d\n",
2154 SMC_LGR_ID_SIZE, &link->lgr->id,
de2fea7b 2155 link->lgr->net->net_cookie,
0a99be43
KG		 2156 SMC_LGR_ID_SIZE, &link->link_uid,
2157 SMC_LGR_ID_SIZE, &link->peer_link_uid,
2158 link->smcibdev->ibdev->name, link->ibport);
877ae5be 2159 link->state = SMC_LNK_ACTIVE;
00a049cf 2160 if (link->lgr->llc_testlink_time) {
c4a146c7 2161 link->llc_testlink_time = link->lgr->llc_testlink_time;
1020e1ef
KG		 2162 schedule_delayed_work(&link->llc_testlink_wrk,
2163 link->llc_testlink_time);
877ae5be
KG		 2164 }
2165}
2166
877ae5be 2167/* called in worker context */
0a99be43 2168void smc_llc_link_clear(struct smc_link *link, bool log)
877ae5be 2169{
0a99be43 2170 if (log)
de2fea7b 2171 pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu link removed: id %*phN"
0a99be43
KG		 2172 ", peerid %*phN, ibdev %s, ibport %d\n",
2173 SMC_LGR_ID_SIZE, &link->lgr->id,
de2fea7b 2174 link->lgr->net->net_cookie,
0a99be43
KG		 2175 SMC_LGR_ID_SIZE, &link->link_uid,
2176 SMC_LGR_ID_SIZE, &link->peer_link_uid,
2177 link->smcibdev->ibdev->name, link->ibport);
2140ac26
KG		 2178 complete(&link->llc_testlink_resp);
2179 cancel_delayed_work_sync(&link->llc_testlink_wrk);
877ae5be
KG		 2180}
2181
3d88a21b
KG		 2182/* register a new rtoken at the remote peer (for all links) */
2183int smc_llc_do_confirm_rkey(struct smc_link *send_link,
44aa81ce
KG		 2184 struct smc_buf_desc *rmb_desc)
2185{
3d88a21b
KG		 2186 struct smc_link_group *lgr = send_link->lgr;
2187 struct smc_llc_qentry *qentry = NULL;
2188 int rc = 0;
44aa81ce 2189
3d88a21b
KG		 2190 rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
2191 if (rc)
2192 goto out;
44aa81ce 2193 /* receive CONFIRM RKEY response from server over RoCE fabric */
3d88a21b
KG		 2194 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2195 SMC_LLC_CONFIRM_RKEY);
2196 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
2197 rc = -EFAULT;
2198out:
2199 if (qentry)
2200 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
3d88a21b 2201 return rc;
44aa81ce
KG		 2202}
2203
60e03c62 2204/* unregister an rtoken at the remote peer */
6d74c3a8 2205int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
60e03c62
KG		 2206 struct smc_buf_desc *rmb_desc)
2207{
6d74c3a8
KG		 2208 struct smc_llc_qentry *qentry = NULL;
2209 struct smc_link *send_link;
0b29ec64 2210 int rc = 0;
60e03c62 2211
6d74c3a8
KG		 2212 send_link = smc_llc_usable_link(lgr);
2213 if (!send_link)
2214 return -ENOLINK;
2215
6d74c3a8
KG		 2216 /* protected by llc_flow control */
2217 rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
60e03c62
KG		 2218 if (rc)
2219 goto out;
2220 /* receive DELETE RKEY response from server over RoCE fabric */
6d74c3a8
KG		 2221 qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
2222 SMC_LLC_DELETE_RKEY);
2223 if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
60e03c62 2224 rc = -EFAULT;
60e03c62 2225out:
6d74c3a8
KG		 2226 if (qentry)
2227 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
60e03c62
KG		 2228 return rc;
2229}
2230
45fa8da0
KG		 2231void smc_llc_link_set_uid(struct smc_link *link)
2232{
2233 __be32 link_uid;
2234
2235 link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
2236 memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
2237}
2238
649758ff
KG		 2239/* save peers link user id, used for debug purposes */
2240void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
2241{
2242 memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
2243 SMC_LGR_ID_SIZE);
2244}
2245
92334cfc
KG		 2246/* evaluate confirm link request or response */
2247int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
2248 enum smc_llc_reqresp type)
2249{
45fa8da0 2250 if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */
92334cfc 2251 qentry->link->link_id = qentry->msg.confirm_link.link_num;
45fa8da0
KG		 2252 smc_llc_link_set_uid(qentry->link);
2253 }
92334cfc
KG		 2254 if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
2255 return -ENOTSUPP;
2256 return 0;
2257}
2258
9bf9abea
UB		 2259/***************************** init, exit, misc ******************************/
2260
2261static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
2262 {
2263 .handler = smc_llc_rx_handler,
2264 .type = SMC_LLC_CONFIRM_LINK
2265 },
313164da
KG		 2266 {
2267 .handler = smc_llc_rx_handler,
2268 .type = SMC_LLC_TEST_LINK
2269 },
52bedf37
KG		 2270 {
2271 .handler = smc_llc_rx_handler,
2272 .type = SMC_LLC_ADD_LINK
2273 },
87f88cda
KG		 2274 {
2275 .handler = smc_llc_rx_handler,
2276 .type = SMC_LLC_ADD_LINK_CONT
2277 },
52bedf37
KG		 2278 {
2279 .handler = smc_llc_rx_handler,
2280 .type = SMC_LLC_DELETE_LINK
2281 },
4ed75de5
KG		 2282 {
2283 .handler = smc_llc_rx_handler,
2284 .type = SMC_LLC_CONFIRM_RKEY
2285 },
2286 {
2287 .handler = smc_llc_rx_handler,
2288 .type = SMC_LLC_CONFIRM_RKEY_CONT
2289 },
2290 {
2291 .handler = smc_llc_rx_handler,
2292 .type = SMC_LLC_DELETE_RKEY
2293 },
b4ba4652
KG		 2294 /* V2 types */
2295 {
2296 .handler = smc_llc_rx_handler,
2297 .type = SMC_LLC_CONFIRM_LINK_V2
2298 },
2299 {
2300 .handler = smc_llc_rx_handler,
2301 .type = SMC_LLC_TEST_LINK_V2
2302 },
2303 {
2304 .handler = smc_llc_rx_handler,
2305 .type = SMC_LLC_ADD_LINK_V2
2306 },
2307 {
2308 .handler = smc_llc_rx_handler,
2309 .type = SMC_LLC_DELETE_LINK_V2
2310 },
2311 {
2312 .handler = smc_llc_rx_handler,
2313 .type = SMC_LLC_REQ_ADD_LINK_V2
2314 },
2315 {
2316 .handler = smc_llc_rx_handler,
2317 .type = SMC_LLC_CONFIRM_RKEY_V2
2318 },
2319 {
2320 .handler = smc_llc_rx_handler,
2321 .type = SMC_LLC_DELETE_RKEY_V2
2322 },
9bf9abea
UB		 2323 {
2324 .handler = NULL,
2325 }
2326};
2327
2328int __init smc_llc_init(void)
2329{
2330 struct smc_wr_rx_handler *handler;
2331 int rc = 0;
2332
2333 for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
2334 INIT_HLIST_NODE(&handler->list);
2335 rc = smc_wr_rx_register_handler(handler);
2336 if (rc)
2337 break;
2338 }
2339 return rc;
2340}
Linux 6.x block layer and io_uring tree(s)