Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9bf9abea UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Link Layer Control (LLC) | |
6 | * | |
9bf9abea UB |
7 | * Copyright IBM Corp. 2016 |
8 | * | |
9 | * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> | |
10 | * Ursula Braun <ubraun@linux.vnet.ibm.com> | |
11 | */ | |
12 | ||
13 | #include <net/tcp.h> | |
14 | #include <rdma/ib_verbs.h> | |
15 | ||
16 | #include "smc.h" | |
17 | #include "smc_core.h" | |
18 | #include "smc_clc.h" | |
19 | #include "smc_llc.h" | |
336ba09f | 20 | #include "smc_pnet.h" |
9bf9abea | 21 | |
0f627126 SR |
22 | #define SMC_LLC_DATA_LEN 40 |
23 | ||
24 | struct smc_llc_hdr { | |
25 | struct smc_wr_rx_hdr common; | |
26 | u8 length; /* 44 */ | |
52bedf37 KG |
27 | #if defined(__BIG_ENDIAN_BITFIELD) |
28 | u8 reserved:4, | |
29 | add_link_rej_rsn:4; | |
30 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
31 | u8 add_link_rej_rsn:4, | |
32 | reserved:4; | |
33 | #endif | |
0f627126 SR |
34 | u8 flags; |
35 | }; | |
36 | ||
75d320d6 KG |
37 | #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 |
38 | ||
0f627126 SR |
39 | struct smc_llc_msg_confirm_link { /* type 0x01 */ |
40 | struct smc_llc_hdr hd; | |
41 | u8 sender_mac[ETH_ALEN]; | |
42 | u8 sender_gid[SMC_GID_SIZE]; | |
43 | u8 sender_qp_num[3]; | |
44 | u8 link_num; | |
45 | u8 link_uid[SMC_LGR_ID_SIZE]; | |
46 | u8 max_links; | |
47 | u8 reserved[9]; | |
48 | }; | |
49 | ||
52bedf37 KG |
50 | #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 |
51 | #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 | |
52 | ||
53 | #define SMC_LLC_ADD_LNK_MAX_LINKS 2 | |
54 | ||
55 | struct smc_llc_msg_add_link { /* type 0x02 */ | |
56 | struct smc_llc_hdr hd; | |
57 | u8 sender_mac[ETH_ALEN]; | |
58 | u8 reserved2[2]; | |
59 | u8 sender_gid[SMC_GID_SIZE]; | |
60 | u8 sender_qp_num[3]; | |
61 | u8 link_num; | |
fbed3b37 KG |
62 | #if defined(__BIG_ENDIAN_BITFIELD) |
63 | u8 reserved3 : 4, | |
64 | qp_mtu : 4; | |
65 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
66 | u8 qp_mtu : 4, | |
67 | reserved3 : 4; | |
68 | #endif | |
52bedf37 KG |
69 | u8 initial_psn[3]; |
70 | u8 reserved[8]; | |
71 | }; | |
72 | ||
87f88cda KG |
73 | struct smc_llc_msg_add_link_cont_rt { |
74 | __be32 rmb_key; | |
75 | __be32 rmb_key_new; | |
76 | __be64 rmb_vaddr_new; | |
77 | }; | |
78 | ||
79 | #define SMC_LLC_RKEYS_PER_CONT_MSG 2 | |
80 | ||
81 | struct smc_llc_msg_add_link_cont { /* type 0x03 */ | |
82 | struct smc_llc_hdr hd; | |
83 | u8 link_num; | |
84 | u8 num_rkeys; | |
85 | u8 reserved2[2]; | |
86 | struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; | |
87 | u8 reserved[4]; | |
88 | } __packed; /* format defined in RFC7609 */ | |
89 | ||
52bedf37 KG |
90 | #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 |
91 | #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 | |
92 | ||
93 | struct smc_llc_msg_del_link { /* type 0x04 */ | |
94 | struct smc_llc_hdr hd; | |
95 | u8 link_num; | |
96 | __be32 reason; | |
97 | u8 reserved[35]; | |
98 | } __packed; /* format defined in RFC7609 */ | |
99 | ||
313164da KG |
100 | struct smc_llc_msg_test_link { /* type 0x07 */ |
101 | struct smc_llc_hdr hd; | |
102 | u8 user_data[16]; | |
103 | u8 reserved[24]; | |
104 | }; | |
105 | ||
4ed75de5 KG |
106 | struct smc_rmb_rtoken { |
107 | union { | |
108 | u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ | |
109 | /* is actually the num of rtokens, first */ | |
110 | /* rtoken is always for the current link */ | |
111 | u8 link_id; /* link id of the rtoken */ | |
112 | }; | |
113 | __be32 rmb_key; | |
114 | __be64 rmb_vaddr; | |
115 | } __packed; /* format defined in RFC7609 */ | |
116 | ||
117 | #define SMC_LLC_RKEYS_PER_MSG 3 | |
118 | ||
119 | struct smc_llc_msg_confirm_rkey { /* type 0x06 */ | |
120 | struct smc_llc_hdr hd; | |
121 | struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; | |
122 | u8 reserved; | |
123 | }; | |
124 | ||
4ed75de5 | 125 | #define SMC_LLC_DEL_RKEY_MAX 8 |
3bc67e09 | 126 | #define SMC_LLC_FLAG_RKEY_RETRY 0x10 |
4ed75de5 KG |
127 | #define SMC_LLC_FLAG_RKEY_NEG 0x20 |
128 | ||
129 | struct smc_llc_msg_delete_rkey { /* type 0x09 */ | |
130 | struct smc_llc_hdr hd; | |
131 | u8 num_rkeys; | |
132 | u8 err_mask; | |
133 | u8 reserved[2]; | |
134 | __be32 rkey[8]; | |
135 | u8 reserved2[4]; | |
136 | }; | |
137 | ||
0f627126 SR |
138 | union smc_llc_msg { |
139 | struct smc_llc_msg_confirm_link confirm_link; | |
52bedf37 | 140 | struct smc_llc_msg_add_link add_link; |
87f88cda | 141 | struct smc_llc_msg_add_link_cont add_link_cont; |
52bedf37 | 142 | struct smc_llc_msg_del_link delete_link; |
4ed75de5 KG |
143 | |
144 | struct smc_llc_msg_confirm_rkey confirm_rkey; | |
4ed75de5 KG |
145 | struct smc_llc_msg_delete_rkey delete_rkey; |
146 | ||
313164da | 147 | struct smc_llc_msg_test_link test_link; |
0f627126 SR |
148 | struct { |
149 | struct smc_llc_hdr hdr; | |
150 | u8 data[SMC_LLC_DATA_LEN]; | |
151 | } raw; | |
152 | }; | |
153 | ||
154 | #define SMC_LLC_FLAG_RESP 0x80 | |
155 | ||
6c8968c4 KG |
156 | struct smc_llc_qentry { |
157 | struct list_head list; | |
158 | struct smc_link *link; | |
159 | union smc_llc_msg msg; | |
160 | }; | |
161 | ||
4dadd151 KG |
162 | static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); |
163 | ||
555da9af KG |
164 | struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) |
165 | { | |
166 | struct smc_llc_qentry *qentry = flow->qentry; | |
167 | ||
168 | flow->qentry = NULL; | |
169 | return qentry; | |
170 | } | |
171 | ||
172 | void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) | |
173 | { | |
174 | struct smc_llc_qentry *qentry; | |
175 | ||
176 | if (flow->qentry) { | |
177 | qentry = flow->qentry; | |
178 | flow->qentry = NULL; | |
179 | kfree(qentry); | |
180 | } | |
181 | } | |
182 | ||
183 | static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, | |
184 | struct smc_llc_qentry *qentry) | |
185 | { | |
186 | flow->qentry = qentry; | |
187 | } | |
188 | ||
189 | /* try to start a new llc flow, initiated by an incoming llc msg */ | |
190 | static bool smc_llc_flow_start(struct smc_llc_flow *flow, | |
191 | struct smc_llc_qentry *qentry) | |
192 | { | |
193 | struct smc_link_group *lgr = qentry->link->lgr; | |
194 | ||
195 | spin_lock_bh(&lgr->llc_flow_lock); | |
196 | if (flow->type) { | |
197 | /* a flow is already active */ | |
198 | if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK || | |
199 | qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) && | |
200 | !lgr->delayed_event) { | |
201 | lgr->delayed_event = qentry; | |
202 | } else { | |
203 | /* forget this llc request */ | |
204 | kfree(qentry); | |
205 | } | |
206 | spin_unlock_bh(&lgr->llc_flow_lock); | |
207 | return false; | |
208 | } | |
209 | switch (qentry->msg.raw.hdr.common.type) { | |
210 | case SMC_LLC_ADD_LINK: | |
211 | flow->type = SMC_LLC_FLOW_ADD_LINK; | |
212 | break; | |
213 | case SMC_LLC_DELETE_LINK: | |
214 | flow->type = SMC_LLC_FLOW_DEL_LINK; | |
215 | break; | |
216 | case SMC_LLC_CONFIRM_RKEY: | |
217 | case SMC_LLC_DELETE_RKEY: | |
218 | flow->type = SMC_LLC_FLOW_RKEY; | |
219 | break; | |
220 | default: | |
221 | flow->type = SMC_LLC_FLOW_NONE; | |
222 | } | |
223 | if (qentry == lgr->delayed_event) | |
224 | lgr->delayed_event = NULL; | |
225 | spin_unlock_bh(&lgr->llc_flow_lock); | |
226 | smc_llc_flow_qentry_set(flow, qentry); | |
227 | return true; | |
228 | } | |
229 | ||
230 | /* start a new local llc flow, wait till current flow finished */ | |
231 | int smc_llc_flow_initiate(struct smc_link_group *lgr, | |
232 | enum smc_llc_flowtype type) | |
233 | { | |
234 | enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; | |
235 | int rc; | |
236 | ||
237 | /* all flows except confirm_rkey and delete_rkey are exclusive, | |
238 | * confirm/delete rkey flows can run concurrently (local and remote) | |
239 | */ | |
240 | if (type == SMC_LLC_FLOW_RKEY) | |
241 | allowed_remote = SMC_LLC_FLOW_RKEY; | |
242 | again: | |
243 | if (list_empty(&lgr->list)) | |
244 | return -ENODEV; | |
245 | spin_lock_bh(&lgr->llc_flow_lock); | |
246 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
247 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
248 | lgr->llc_flow_rmt.type == allowed_remote)) { | |
249 | lgr->llc_flow_lcl.type = type; | |
250 | spin_unlock_bh(&lgr->llc_flow_lock); | |
251 | return 0; | |
252 | } | |
253 | spin_unlock_bh(&lgr->llc_flow_lock); | |
254 | rc = wait_event_interruptible_timeout(lgr->llc_waiter, | |
255 | (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
256 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
257 | lgr->llc_flow_rmt.type == allowed_remote)), | |
258 | SMC_LLC_WAIT_TIME); | |
259 | if (!rc) | |
260 | return -ETIMEDOUT; | |
261 | goto again; | |
262 | } | |
263 | ||
264 | /* finish the current llc flow */ | |
265 | void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) | |
266 | { | |
267 | spin_lock_bh(&lgr->llc_flow_lock); | |
268 | memset(flow, 0, sizeof(*flow)); | |
269 | flow->type = SMC_LLC_FLOW_NONE; | |
270 | spin_unlock_bh(&lgr->llc_flow_lock); | |
271 | if (!list_empty(&lgr->list) && lgr->delayed_event && | |
272 | flow == &lgr->llc_flow_lcl) | |
273 | schedule_work(&lgr->llc_event_work); | |
274 | else | |
275 | wake_up_interruptible(&lgr->llc_waiter); | |
276 | } | |
277 | ||
278 | /* lnk is optional and used for early wakeup when link goes down, useful in | |
279 | * cases where we wait for a response on the link after we sent a request | |
280 | */ | |
281 | struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, | |
282 | struct smc_link *lnk, | |
283 | int time_out, u8 exp_msg) | |
284 | { | |
285 | struct smc_llc_flow *flow = &lgr->llc_flow_lcl; | |
286 | ||
287 | wait_event_interruptible_timeout(lgr->llc_waiter, | |
288 | (flow->qentry || | |
289 | (lnk && !smc_link_usable(lnk)) || | |
290 | list_empty(&lgr->list)), | |
291 | time_out); | |
292 | if (!flow->qentry || | |
293 | (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { | |
294 | smc_llc_flow_qentry_del(flow); | |
295 | goto out; | |
296 | } | |
297 | if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) { | |
298 | if (exp_msg == SMC_LLC_ADD_LINK && | |
299 | flow->qentry->msg.raw.hdr.common.type == | |
300 | SMC_LLC_DELETE_LINK) { | |
301 | /* flow_start will delay the unexpected msg */ | |
302 | smc_llc_flow_start(&lgr->llc_flow_lcl, | |
303 | smc_llc_flow_qentry_clr(flow)); | |
304 | return NULL; | |
305 | } | |
306 | smc_llc_flow_qentry_del(flow); | |
307 | } | |
308 | out: | |
309 | return flow->qentry; | |
310 | } | |
311 | ||
9bf9abea UB |
312 | /********************************** send *************************************/ |
313 | ||
314 | struct smc_llc_tx_pend { | |
315 | }; | |
316 | ||
317 | /* handler for send/transmission completion of an LLC msg */ | |
318 | static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, | |
319 | struct smc_link *link, | |
320 | enum ib_wc_status wc_status) | |
321 | { | |
322 | /* future work: handle wc_status error for recovery and failover */ | |
323 | } | |
324 | ||
325 | /** | |
326 | * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits | |
327 | * @link: Pointer to SMC link used for sending LLC control message. | |
328 | * @wr_buf: Out variable returning pointer to work request payload buffer. | |
329 | * @pend: Out variable returning pointer to private pending WR tracking. | |
330 | * It's the context the transmit complete handler will get. | |
331 | * | |
332 | * Reserves and pre-fills an entry for a pending work request send/tx. | |
333 | * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. | |
334 | * Can sleep due to smc_get_ctrl_buf (if not in softirq context). | |
335 | * | |
336 | * Return: 0 on success, otherwise an error value. | |
337 | */ | |
338 | static int smc_llc_add_pending_send(struct smc_link *link, | |
339 | struct smc_wr_buf **wr_buf, | |
340 | struct smc_wr_tx_pend_priv **pend) | |
341 | { | |
342 | int rc; | |
343 | ||
ad6f317f UB |
344 | rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, |
345 | pend); | |
9bf9abea UB |
346 | if (rc < 0) |
347 | return rc; | |
348 | BUILD_BUG_ON_MSG( | |
349 | sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, | |
350 | "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); | |
351 | BUILD_BUG_ON_MSG( | |
352 | sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, | |
353 | "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); | |
354 | BUILD_BUG_ON_MSG( | |
355 | sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, | |
356 | "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); | |
357 | return 0; | |
358 | } | |
359 | ||
360 | /* high-level API to send LLC confirm link */ | |
947541f3 | 361 | int smc_llc_send_confirm_link(struct smc_link *link, |
9bf9abea UB |
362 | enum smc_llc_reqresp reqresp) |
363 | { | |
00e5fb26 | 364 | struct smc_link_group *lgr = smc_get_lgr(link); |
9bf9abea UB |
365 | struct smc_llc_msg_confirm_link *confllc; |
366 | struct smc_wr_tx_pend_priv *pend; | |
367 | struct smc_wr_buf *wr_buf; | |
368 | int rc; | |
369 | ||
370 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
371 | if (rc) | |
372 | return rc; | |
373 | confllc = (struct smc_llc_msg_confirm_link *)wr_buf; | |
374 | memset(confllc, 0, sizeof(*confllc)); | |
375 | confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; | |
376 | confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); | |
75d320d6 | 377 | confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; |
9bf9abea UB |
378 | if (reqresp == SMC_LLC_RESP) |
379 | confllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
947541f3 UB |
380 | memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], |
381 | ETH_ALEN); | |
7005ada6 | 382 | memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); |
9bf9abea | 383 | hton24(confllc->sender_qp_num, link->roce_qp->qp_num); |
2be922f3 | 384 | confllc->link_num = link->link_id; |
9bf9abea | 385 | memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); |
b1570a87 | 386 | confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; |
52bedf37 KG |
387 | /* send llc message */ |
388 | rc = smc_wr_tx_send(link, pend); | |
389 | return rc; | |
390 | } | |
391 | ||
44aa81ce | 392 | /* send LLC confirm rkey request */ |
3d88a21b | 393 | static int smc_llc_send_confirm_rkey(struct smc_link *send_link, |
44aa81ce KG |
394 | struct smc_buf_desc *rmb_desc) |
395 | { | |
396 | struct smc_llc_msg_confirm_rkey *rkeyllc; | |
397 | struct smc_wr_tx_pend_priv *pend; | |
398 | struct smc_wr_buf *wr_buf; | |
3d88a21b KG |
399 | struct smc_link *link; |
400 | int i, rc, rtok_ix; | |
44aa81ce | 401 | |
3d88a21b | 402 | rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); |
44aa81ce KG |
403 | if (rc) |
404 | return rc; | |
405 | rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; | |
406 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
407 | rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; | |
408 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); | |
3d88a21b KG |
409 | |
410 | rtok_ix = 1; | |
411 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
412 | link = &send_link->lgr->lnk[i]; | |
413 | if (link->state == SMC_LNK_ACTIVE && link != send_link) { | |
414 | rkeyllc->rtoken[rtok_ix].link_id = link->link_id; | |
415 | rkeyllc->rtoken[rtok_ix].rmb_key = | |
416 | htonl(rmb_desc->mr_rx[link->link_idx]->rkey); | |
417 | rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( | |
418 | (u64)sg_dma_address( | |
419 | rmb_desc->sgt[link->link_idx].sgl)); | |
420 | rtok_ix++; | |
421 | } | |
422 | } | |
423 | /* rkey of send_link is in rtoken[0] */ | |
424 | rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; | |
44aa81ce | 425 | rkeyllc->rtoken[0].rmb_key = |
3d88a21b | 426 | htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); |
44aa81ce | 427 | rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( |
3d88a21b | 428 | (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); |
44aa81ce | 429 | /* send llc message */ |
3d88a21b | 430 | rc = smc_wr_tx_send(send_link, pend); |
44aa81ce KG |
431 | return rc; |
432 | } | |
433 | ||
60e03c62 KG |
434 | /* send LLC delete rkey request */ |
435 | static int smc_llc_send_delete_rkey(struct smc_link *link, | |
436 | struct smc_buf_desc *rmb_desc) | |
437 | { | |
438 | struct smc_llc_msg_delete_rkey *rkeyllc; | |
439 | struct smc_wr_tx_pend_priv *pend; | |
440 | struct smc_wr_buf *wr_buf; | |
441 | int rc; | |
442 | ||
443 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
444 | if (rc) | |
445 | return rc; | |
446 | rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; | |
447 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
448 | rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; | |
449 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); | |
450 | rkeyllc->num_rkeys = 1; | |
387707fd | 451 | rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); |
60e03c62 KG |
452 | /* send llc message */ |
453 | rc = smc_wr_tx_send(link, pend); | |
454 | return rc; | |
455 | } | |
456 | ||
52bedf37 | 457 | /* send ADD LINK request or response */ |
7005ada6 | 458 | int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], |
fbed3b37 | 459 | struct smc_link *link_new, |
52bedf37 KG |
460 | enum smc_llc_reqresp reqresp) |
461 | { | |
462 | struct smc_llc_msg_add_link *addllc; | |
463 | struct smc_wr_tx_pend_priv *pend; | |
464 | struct smc_wr_buf *wr_buf; | |
465 | int rc; | |
466 | ||
467 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
468 | if (rc) | |
469 | return rc; | |
470 | addllc = (struct smc_llc_msg_add_link *)wr_buf; | |
fbed3b37 KG |
471 | |
472 | memset(addllc, 0, sizeof(*addllc)); | |
473 | addllc->hd.common.type = SMC_LLC_ADD_LINK; | |
474 | addllc->hd.length = sizeof(struct smc_llc_msg_add_link); | |
475 | if (reqresp == SMC_LLC_RESP) | |
476 | addllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
477 | memcpy(addllc->sender_mac, mac, ETH_ALEN); | |
478 | memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); | |
479 | if (link_new) { | |
480 | addllc->link_num = link_new->link_id; | |
481 | hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); | |
482 | hton24(addllc->initial_psn, link_new->psn_initial); | |
483 | if (reqresp == SMC_LLC_REQ) | |
484 | addllc->qp_mtu = link_new->path_mtu; | |
485 | else | |
486 | addllc->qp_mtu = min(link_new->path_mtu, | |
487 | link_new->peer_mtu); | |
488 | } | |
52bedf37 KG |
489 | /* send llc message */ |
490 | rc = smc_wr_tx_send(link, pend); | |
491 | return rc; | |
492 | } | |
493 | ||
494 | /* send DELETE LINK request or response */ | |
fbed3b37 KG |
495 | int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, |
496 | enum smc_llc_reqresp reqresp, bool orderly, | |
497 | u32 reason) | |
52bedf37 KG |
498 | { |
499 | struct smc_llc_msg_del_link *delllc; | |
500 | struct smc_wr_tx_pend_priv *pend; | |
501 | struct smc_wr_buf *wr_buf; | |
502 | int rc; | |
503 | ||
504 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
505 | if (rc) | |
506 | return rc; | |
507 | delllc = (struct smc_llc_msg_del_link *)wr_buf; | |
fbed3b37 KG |
508 | |
509 | memset(delllc, 0, sizeof(*delllc)); | |
510 | delllc->hd.common.type = SMC_LLC_DELETE_LINK; | |
511 | delllc->hd.length = sizeof(struct smc_llc_msg_del_link); | |
512 | if (reqresp == SMC_LLC_RESP) | |
513 | delllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
514 | if (orderly) | |
515 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
516 | if (link_del_id) | |
517 | delllc->link_num = link_del_id; | |
518 | else | |
519 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; | |
520 | delllc->reason = htonl(reason); | |
9bf9abea UB |
521 | /* send llc message */ |
522 | rc = smc_wr_tx_send(link, pend); | |
523 | return rc; | |
524 | } | |
525 | ||
d97935fa KG |
526 | /* send LLC test link request */ |
527 | static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) | |
313164da KG |
528 | { |
529 | struct smc_llc_msg_test_link *testllc; | |
530 | struct smc_wr_tx_pend_priv *pend; | |
531 | struct smc_wr_buf *wr_buf; | |
532 | int rc; | |
533 | ||
534 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
535 | if (rc) | |
536 | return rc; | |
537 | testllc = (struct smc_llc_msg_test_link *)wr_buf; | |
538 | memset(testllc, 0, sizeof(*testllc)); | |
539 | testllc->hd.common.type = SMC_LLC_TEST_LINK; | |
540 | testllc->hd.length = sizeof(struct smc_llc_msg_test_link); | |
313164da KG |
541 | memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); |
542 | /* send llc message */ | |
543 | rc = smc_wr_tx_send(link, pend); | |
544 | return rc; | |
545 | } | |
546 | ||
6c8968c4 KG |
547 | /* schedule an llc send on link, may wait for buffers */ |
548 | static int smc_llc_send_message(struct smc_link *link, void *llcbuf) | |
4ed75de5 KG |
549 | { |
550 | struct smc_wr_tx_pend_priv *pend; | |
551 | struct smc_wr_buf *wr_buf; | |
552 | int rc; | |
553 | ||
6c8968c4 KG |
554 | if (!smc_link_usable(link)) |
555 | return -ENOLINK; | |
556 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
4ed75de5 | 557 | if (rc) |
6c8968c4 KG |
558 | return rc; |
559 | memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); | |
560 | return smc_wr_tx_send(link, pend); | |
4ed75de5 KG |
561 | } |
562 | ||
f3811fd7 KG |
563 | /* schedule an llc send on link, may wait for buffers, |
564 | * and wait for send completion notification. | |
565 | * @return 0 on success | |
566 | */ | |
567 | static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) | |
568 | { | |
569 | struct smc_wr_tx_pend_priv *pend; | |
570 | struct smc_wr_buf *wr_buf; | |
571 | int rc; | |
572 | ||
573 | if (!smc_link_usable(link)) | |
574 | return -ENOLINK; | |
575 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
576 | if (rc) | |
577 | return rc; | |
578 | memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); | |
579 | return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); | |
580 | } | |
581 | ||
9bf9abea UB |
582 | /********************************* receive ***********************************/ |
583 | ||
336ba09f KG |
584 | static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, |
585 | enum smc_lgr_type lgr_new_t) | |
586 | { | |
587 | int i; | |
588 | ||
589 | if (lgr->type == SMC_LGR_SYMMETRIC || | |
590 | (lgr->type != SMC_LGR_SINGLE && | |
591 | (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
592 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) | |
593 | return -EMLINK; | |
594 | ||
595 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
596 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { | |
597 | for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) | |
598 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
599 | return i; | |
600 | } else { | |
601 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) | |
602 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
603 | return i; | |
604 | } | |
605 | return -EMLINK; | |
606 | } | |
607 | ||
87f88cda KG |
608 | /* return first buffer from any of the next buf lists */ |
609 | static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
610 | int *buf_lst) | |
611 | { | |
612 | struct smc_buf_desc *buf_pos; | |
613 | ||
614 | while (*buf_lst < SMC_RMBE_SIZES) { | |
615 | buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], | |
616 | struct smc_buf_desc, list); | |
617 | if (buf_pos) | |
618 | return buf_pos; | |
619 | (*buf_lst)++; | |
620 | } | |
621 | return NULL; | |
622 | } | |
623 | ||
624 | /* return next rmb from buffer lists */ | |
625 | static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
626 | int *buf_lst, | |
627 | struct smc_buf_desc *buf_pos) | |
628 | { | |
629 | struct smc_buf_desc *buf_next; | |
630 | ||
631 | if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { | |
632 | (*buf_lst)++; | |
633 | return _smc_llc_get_next_rmb(lgr, buf_lst); | |
634 | } | |
635 | buf_next = list_next_entry(buf_pos, list); | |
636 | return buf_next; | |
637 | } | |
638 | ||
639 | static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, | |
640 | int *buf_lst) | |
641 | { | |
642 | *buf_lst = 0; | |
643 | return smc_llc_get_next_rmb(lgr, buf_lst, NULL); | |
644 | } | |
645 | ||
646 | /* send one add_link_continue msg */ | |
647 | static int smc_llc_add_link_cont(struct smc_link *link, | |
648 | struct smc_link *link_new, u8 *num_rkeys_todo, | |
649 | int *buf_lst, struct smc_buf_desc **buf_pos) | |
650 | { | |
651 | struct smc_llc_msg_add_link_cont *addc_llc; | |
652 | struct smc_link_group *lgr = link->lgr; | |
653 | int prim_lnk_idx, lnk_idx, i, rc; | |
654 | struct smc_wr_tx_pend_priv *pend; | |
655 | struct smc_wr_buf *wr_buf; | |
656 | struct smc_buf_desc *rmb; | |
657 | u8 n; | |
658 | ||
659 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
660 | if (rc) | |
661 | return rc; | |
662 | addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; | |
663 | memset(addc_llc, 0, sizeof(*addc_llc)); | |
664 | ||
665 | prim_lnk_idx = link->link_idx; | |
666 | lnk_idx = link_new->link_idx; | |
667 | addc_llc->link_num = link_new->link_id; | |
668 | addc_llc->num_rkeys = *num_rkeys_todo; | |
669 | n = *num_rkeys_todo; | |
670 | for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { | |
671 | if (!*buf_pos) { | |
672 | addc_llc->num_rkeys = addc_llc->num_rkeys - | |
673 | *num_rkeys_todo; | |
674 | *num_rkeys_todo = 0; | |
675 | break; | |
676 | } | |
677 | rmb = *buf_pos; | |
678 | ||
679 | addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); | |
680 | addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); | |
681 | addc_llc->rt[i].rmb_vaddr_new = | |
682 | cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); | |
683 | ||
684 | (*num_rkeys_todo)--; | |
685 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
686 | while (*buf_pos && !(*buf_pos)->used) | |
687 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
688 | } | |
689 | addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; | |
690 | addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); | |
691 | if (lgr->role == SMC_CLNT) | |
692 | addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
693 | return smc_wr_tx_send(link, pend); | |
694 | } | |
695 | ||
696 | static int smc_llc_cli_rkey_exchange(struct smc_link *link, | |
697 | struct smc_link *link_new) | |
698 | { | |
699 | struct smc_llc_msg_add_link_cont *addc_llc; | |
700 | struct smc_link_group *lgr = link->lgr; | |
701 | u8 max, num_rkeys_send, num_rkeys_recv; | |
702 | struct smc_llc_qentry *qentry; | |
703 | struct smc_buf_desc *buf_pos; | |
704 | int buf_lst; | |
705 | int rc = 0; | |
706 | int i; | |
707 | ||
708 | mutex_lock(&lgr->rmbs_lock); | |
709 | num_rkeys_send = lgr->conns_num; | |
710 | buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); | |
711 | do { | |
712 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, | |
713 | SMC_LLC_ADD_LINK_CONT); | |
714 | if (!qentry) { | |
715 | rc = -ETIMEDOUT; | |
716 | break; | |
717 | } | |
718 | addc_llc = &qentry->msg.add_link_cont; | |
719 | num_rkeys_recv = addc_llc->num_rkeys; | |
720 | max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); | |
721 | for (i = 0; i < max; i++) { | |
722 | smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, | |
723 | addc_llc->rt[i].rmb_key, | |
724 | addc_llc->rt[i].rmb_vaddr_new, | |
725 | addc_llc->rt[i].rmb_key_new); | |
726 | num_rkeys_recv--; | |
727 | } | |
728 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
729 | rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, | |
730 | &buf_lst, &buf_pos); | |
731 | if (rc) | |
732 | break; | |
733 | } while (num_rkeys_send || num_rkeys_recv); | |
734 | ||
735 | mutex_unlock(&lgr->rmbs_lock); | |
736 | return rc; | |
737 | } | |
738 | ||
336ba09f KG |
739 | /* prepare and send an add link reject response */ |
740 | static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) | |
741 | { | |
742 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; | |
743 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; | |
744 | qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; | |
745 | return smc_llc_send_message(qentry->link, &qentry->msg); | |
746 | } | |
747 | ||
b1570a87 KG |
748 | static int smc_llc_cli_conf_link(struct smc_link *link, |
749 | struct smc_init_info *ini, | |
750 | struct smc_link *link_new, | |
751 | enum smc_lgr_type lgr_new_t) | |
752 | { | |
753 | struct smc_link_group *lgr = link->lgr; | |
754 | struct smc_llc_msg_del_link *del_llc; | |
755 | struct smc_llc_qentry *qentry = NULL; | |
756 | int rc = 0; | |
757 | ||
758 | /* receive CONFIRM LINK request over RoCE fabric */ | |
759 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); | |
760 | if (!qentry) { | |
761 | rc = smc_llc_send_delete_link(link, link_new->link_id, | |
762 | SMC_LLC_REQ, false, | |
763 | SMC_LLC_DEL_LOST_PATH); | |
764 | return -ENOLINK; | |
765 | } | |
766 | if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { | |
767 | /* received DELETE_LINK instead */ | |
768 | del_llc = &qentry->msg.delete_link; | |
769 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; | |
770 | smc_llc_send_message(link, &qentry->msg); | |
771 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
772 | return -ENOLINK; | |
773 | } | |
774 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
775 | ||
776 | rc = smc_ib_modify_qp_rts(link_new); | |
777 | if (rc) { | |
778 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
779 | false, SMC_LLC_DEL_LOST_PATH); | |
780 | return -ENOLINK; | |
781 | } | |
782 | smc_wr_remember_qp_attr(link_new); | |
783 | ||
784 | rc = smcr_buf_reg_lgr(link_new); | |
785 | if (rc) { | |
786 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
787 | false, SMC_LLC_DEL_LOST_PATH); | |
788 | return -ENOLINK; | |
789 | } | |
790 | ||
791 | /* send CONFIRM LINK response over RoCE fabric */ | |
792 | rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); | |
793 | if (rc) { | |
794 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
795 | false, SMC_LLC_DEL_LOST_PATH); | |
796 | return -ENOLINK; | |
797 | } | |
798 | smc_llc_link_active(link_new); | |
ad6c111b KG |
799 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || |
800 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) | |
801 | smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); | |
802 | else | |
803 | smcr_lgr_set_type(lgr, lgr_new_t); | |
b1570a87 KG |
804 | return 0; |
805 | } | |
806 | ||
336ba09f KG |
807 | static void smc_llc_save_add_link_info(struct smc_link *link, |
808 | struct smc_llc_msg_add_link *add_llc) | |
809 | { | |
810 | link->peer_qpn = ntoh24(add_llc->sender_qp_num); | |
811 | memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); | |
812 | memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); | |
813 | link->peer_psn = ntoh24(add_llc->initial_psn); | |
814 | link->peer_mtu = add_llc->qp_mtu; | |
815 | } | |
816 | ||
817 | /* as an SMC client, process an add link request */ | |
818 | int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) | |
819 | { | |
820 | struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; | |
821 | enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; | |
822 | struct smc_link_group *lgr = smc_get_lgr(link); | |
823 | struct smc_link *lnk_new = NULL; | |
824 | struct smc_init_info ini; | |
825 | int lnk_idx, rc = 0; | |
826 | ||
827 | ini.vlan_id = lgr->vlan_id; | |
828 | smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); | |
829 | if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && | |
830 | !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { | |
831 | if (!ini.ib_dev) | |
832 | goto out_reject; | |
833 | lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; | |
834 | } | |
835 | if (!ini.ib_dev) { | |
836 | lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; | |
837 | ini.ib_dev = link->smcibdev; | |
838 | ini.ib_port = link->ibport; | |
839 | } | |
840 | lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); | |
841 | if (lnk_idx < 0) | |
842 | goto out_reject; | |
843 | lnk_new = &lgr->lnk[lnk_idx]; | |
844 | rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); | |
845 | if (rc) | |
846 | goto out_reject; | |
847 | smc_llc_save_add_link_info(lnk_new, llc); | |
848 | lnk_new->link_id = llc->link_num; | |
849 | ||
850 | rc = smc_ib_ready_link(lnk_new); | |
851 | if (rc) | |
852 | goto out_clear_lnk; | |
853 | ||
854 | rc = smcr_buf_map_lgr(lnk_new); | |
855 | if (rc) | |
856 | goto out_clear_lnk; | |
857 | ||
858 | rc = smc_llc_send_add_link(link, | |
859 | lnk_new->smcibdev->mac[ini.ib_port - 1], | |
860 | lnk_new->gid, lnk_new, SMC_LLC_RESP); | |
861 | if (rc) | |
862 | goto out_clear_lnk; | |
87f88cda | 863 | rc = smc_llc_cli_rkey_exchange(link, lnk_new); |
336ba09f KG |
864 | if (rc) { |
865 | rc = 0; | |
866 | goto out_clear_lnk; | |
867 | } | |
b1570a87 | 868 | rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); |
336ba09f KG |
869 | if (!rc) |
870 | goto out; | |
871 | out_clear_lnk: | |
872 | smcr_link_clear(lnk_new); | |
873 | out_reject: | |
874 | smc_llc_cli_add_link_reject(qentry); | |
875 | out: | |
876 | kfree(qentry); | |
877 | return rc; | |
878 | } | |
879 | ||
b1570a87 KG |
880 | static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) |
881 | { | |
882 | struct smc_llc_qentry *qentry; | |
883 | ||
884 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
885 | ||
886 | mutex_lock(&lgr->llc_conf_mutex); | |
887 | smc_llc_cli_add_link(qentry->link, qentry); | |
888 | mutex_unlock(&lgr->llc_conf_mutex); | |
889 | } | |
890 | ||
9c416878 KG |
891 | static int smc_llc_active_link_count(struct smc_link_group *lgr) |
892 | { | |
893 | int i, link_count = 0; | |
894 | ||
895 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
896 | if (!smc_link_usable(&lgr->lnk[i])) | |
897 | continue; | |
898 | link_count++; | |
899 | } | |
900 | return link_count; | |
901 | } | |
902 | ||
c9a5d243 KG |
903 | /* find the asymmetric link when 3 links are established */ |
904 | static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) | |
905 | { | |
906 | int asym_idx = -ENOENT; | |
907 | int i, j, k; | |
908 | bool found; | |
909 | ||
910 | /* determine asymmetric link */ | |
911 | found = false; | |
912 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
913 | for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { | |
914 | if (!smc_link_usable(&lgr->lnk[i]) || | |
915 | !smc_link_usable(&lgr->lnk[j])) | |
916 | continue; | |
917 | if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, | |
918 | SMC_GID_SIZE)) { | |
919 | found = true; /* asym_lnk is i or j */ | |
920 | break; | |
921 | } | |
922 | } | |
923 | if (found) | |
924 | break; | |
925 | } | |
926 | if (!found) | |
927 | goto out; /* no asymmetric link */ | |
928 | for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { | |
929 | if (!smc_link_usable(&lgr->lnk[k])) | |
930 | continue; | |
931 | if (k != i && | |
932 | !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, | |
933 | SMC_GID_SIZE)) { | |
934 | asym_idx = i; | |
935 | break; | |
936 | } | |
937 | if (k != j && | |
938 | !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, | |
939 | SMC_GID_SIZE)) { | |
940 | asym_idx = j; | |
941 | break; | |
942 | } | |
943 | } | |
944 | out: | |
945 | return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; | |
946 | } | |
947 | ||
948 | static void smc_llc_delete_asym_link(struct smc_link_group *lgr) | |
949 | { | |
950 | struct smc_link *lnk_new = NULL, *lnk_asym; | |
951 | struct smc_llc_qentry *qentry; | |
952 | int rc; | |
953 | ||
954 | lnk_asym = smc_llc_find_asym_link(lgr); | |
955 | if (!lnk_asym) | |
956 | return; /* no asymmetric link */ | |
957 | if (!smc_link_downing(&lnk_asym->state)) | |
958 | return; | |
c6f02ebe | 959 | lnk_new = smc_switch_conns(lgr, lnk_asym, false); |
c9a5d243 KG |
960 | smc_wr_tx_wait_no_pending_sends(lnk_asym); |
961 | if (!lnk_new) | |
962 | goto out_free; | |
963 | /* change flow type from ADD_LINK into DEL_LINK */ | |
964 | lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; | |
965 | rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, | |
966 | true, SMC_LLC_DEL_NO_ASYM_NEEDED); | |
967 | if (rc) { | |
968 | smcr_link_down_cond(lnk_new); | |
969 | goto out_free; | |
970 | } | |
971 | qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, | |
972 | SMC_LLC_DELETE_LINK); | |
973 | if (!qentry) { | |
974 | smcr_link_down_cond(lnk_new); | |
975 | goto out_free; | |
976 | } | |
977 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
978 | out_free: | |
979 | smcr_link_clear(lnk_asym); | |
980 | } | |
981 | ||
57b49924 KG |
982 | static int smc_llc_srv_rkey_exchange(struct smc_link *link, |
983 | struct smc_link *link_new) | |
984 | { | |
985 | struct smc_llc_msg_add_link_cont *addc_llc; | |
986 | struct smc_link_group *lgr = link->lgr; | |
987 | u8 max, num_rkeys_send, num_rkeys_recv; | |
988 | struct smc_llc_qentry *qentry = NULL; | |
989 | struct smc_buf_desc *buf_pos; | |
990 | int buf_lst; | |
991 | int rc = 0; | |
992 | int i; | |
993 | ||
994 | mutex_lock(&lgr->rmbs_lock); | |
995 | num_rkeys_send = lgr->conns_num; | |
996 | buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); | |
997 | do { | |
998 | smc_llc_add_link_cont(link, link_new, &num_rkeys_send, | |
999 | &buf_lst, &buf_pos); | |
1000 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, | |
1001 | SMC_LLC_ADD_LINK_CONT); | |
1002 | if (!qentry) { | |
1003 | rc = -ETIMEDOUT; | |
1004 | goto out; | |
1005 | } | |
1006 | addc_llc = &qentry->msg.add_link_cont; | |
1007 | num_rkeys_recv = addc_llc->num_rkeys; | |
1008 | max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); | |
1009 | for (i = 0; i < max; i++) { | |
1010 | smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, | |
1011 | addc_llc->rt[i].rmb_key, | |
1012 | addc_llc->rt[i].rmb_vaddr_new, | |
1013 | addc_llc->rt[i].rmb_key_new); | |
1014 | num_rkeys_recv--; | |
1015 | } | |
1016 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1017 | } while (num_rkeys_send || num_rkeys_recv); | |
1018 | out: | |
1019 | mutex_unlock(&lgr->rmbs_lock); | |
1020 | return rc; | |
1021 | } | |
1022 | ||
1551c95b KG |
1023 | static int smc_llc_srv_conf_link(struct smc_link *link, |
1024 | struct smc_link *link_new, | |
1025 | enum smc_lgr_type lgr_new_t) | |
1026 | { | |
1027 | struct smc_link_group *lgr = link->lgr; | |
1028 | struct smc_llc_qentry *qentry = NULL; | |
1029 | int rc; | |
1030 | ||
1031 | /* send CONFIRM LINK request over the RoCE fabric */ | |
1032 | rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); | |
1033 | if (rc) | |
1034 | return -ENOLINK; | |
1035 | /* receive CONFIRM LINK response over the RoCE fabric */ | |
1036 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, | |
1037 | SMC_LLC_CONFIRM_LINK); | |
1038 | if (!qentry) { | |
1039 | /* send DELETE LINK */ | |
1040 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
1041 | false, SMC_LLC_DEL_LOST_PATH); | |
1042 | return -ENOLINK; | |
1043 | } | |
1044 | smc_llc_link_active(link_new); | |
ad6c111b KG |
1045 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || |
1046 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) | |
1047 | smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); | |
1048 | else | |
1049 | smcr_lgr_set_type(lgr, lgr_new_t); | |
1551c95b KG |
1050 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); |
1051 | return 0; | |
1052 | } | |
1053 | ||
2d2209f2 KG |
1054 | int smc_llc_srv_add_link(struct smc_link *link) |
1055 | { | |
1056 | enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; | |
1057 | struct smc_link_group *lgr = link->lgr; | |
1058 | struct smc_llc_msg_add_link *add_llc; | |
1059 | struct smc_llc_qentry *qentry = NULL; | |
1060 | struct smc_link *link_new; | |
1061 | struct smc_init_info ini; | |
1062 | int lnk_idx, rc = 0; | |
1063 | ||
1064 | /* ignore client add link recommendation, start new flow */ | |
1065 | ini.vlan_id = lgr->vlan_id; | |
1066 | smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); | |
1067 | if (!ini.ib_dev) { | |
1068 | lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; | |
1069 | ini.ib_dev = link->smcibdev; | |
1070 | ini.ib_port = link->ibport; | |
1071 | } | |
1072 | lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); | |
1073 | if (lnk_idx < 0) | |
1074 | return 0; | |
1075 | ||
1076 | rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); | |
1077 | if (rc) | |
1078 | return rc; | |
1079 | link_new = &lgr->lnk[lnk_idx]; | |
1080 | rc = smc_llc_send_add_link(link, | |
1081 | link_new->smcibdev->mac[ini.ib_port - 1], | |
1082 | link_new->gid, link_new, SMC_LLC_REQ); | |
1083 | if (rc) | |
1084 | goto out_err; | |
1085 | /* receive ADD LINK response over the RoCE fabric */ | |
1086 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); | |
1087 | if (!qentry) { | |
1088 | rc = -ETIMEDOUT; | |
1089 | goto out_err; | |
1090 | } | |
1091 | add_llc = &qentry->msg.add_link; | |
1092 | if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { | |
1093 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1094 | rc = -ENOLINK; | |
1095 | goto out_err; | |
1096 | } | |
1097 | if (lgr->type == SMC_LGR_SINGLE && | |
1098 | (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && | |
1099 | !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { | |
1100 | lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; | |
1101 | } | |
1102 | smc_llc_save_add_link_info(link_new, add_llc); | |
1103 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1104 | ||
1105 | rc = smc_ib_ready_link(link_new); | |
1106 | if (rc) | |
1107 | goto out_err; | |
1108 | rc = smcr_buf_map_lgr(link_new); | |
1109 | if (rc) | |
1110 | goto out_err; | |
1111 | rc = smcr_buf_reg_lgr(link_new); | |
1112 | if (rc) | |
1113 | goto out_err; | |
57b49924 | 1114 | rc = smc_llc_srv_rkey_exchange(link, link_new); |
2d2209f2 KG |
1115 | if (rc) |
1116 | goto out_err; | |
1551c95b | 1117 | rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); |
2d2209f2 KG |
1118 | if (rc) |
1119 | goto out_err; | |
1120 | return 0; | |
1121 | out_err: | |
1122 | smcr_link_clear(link_new); | |
1123 | return rc; | |
1124 | } | |
1125 | ||
1126 | static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) | |
1127 | { | |
1128 | struct smc_link *link = lgr->llc_flow_lcl.qentry->link; | |
1129 | int rc; | |
1130 | ||
1131 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1132 | ||
1133 | mutex_lock(&lgr->llc_conf_mutex); | |
1134 | rc = smc_llc_srv_add_link(link); | |
1135 | if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { | |
1136 | /* delete any asymmetric link */ | |
c9a5d243 | 1137 | smc_llc_delete_asym_link(lgr); |
2d2209f2 KG |
1138 | } |
1139 | mutex_unlock(&lgr->llc_conf_mutex); | |
1140 | } | |
1141 | ||
4dadd151 KG |
1142 | /* enqueue a local add_link req to trigger a new add_link flow, only as SERV */ |
1143 | void smc_llc_srv_add_link_local(struct smc_link *link) | |
1144 | { | |
1145 | struct smc_llc_msg_add_link add_llc = {0}; | |
1146 | ||
1147 | add_llc.hd.length = sizeof(add_llc); | |
1148 | add_llc.hd.common.type = SMC_LLC_ADD_LINK; | |
1149 | /* no dev and port needed, we as server ignore client data anyway */ | |
1150 | smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); | |
1151 | } | |
1152 | ||
b45e7f98 KG |
1153 | /* worker to process an add link message */ |
1154 | static void smc_llc_add_link_work(struct work_struct *work) | |
1155 | { | |
1156 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
1157 | llc_add_link_work); | |
1158 | ||
1159 | if (list_empty(&lgr->list)) { | |
1160 | /* link group is terminating */ | |
1161 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1162 | goto out; | |
1163 | } | |
1164 | ||
b1570a87 KG |
1165 | if (lgr->role == SMC_CLNT) |
1166 | smc_llc_process_cli_add_link(lgr); | |
2d2209f2 KG |
1167 | else |
1168 | smc_llc_process_srv_add_link(lgr); | |
b45e7f98 KG |
1169 | out: |
1170 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); | |
1171 | } | |
1172 | ||
4dadd151 KG |
1173 | /* enqueue a local del_link msg to trigger a new del_link flow, |
1174 | * called only for role SMC_SERV | |
1175 | */ | |
1176 | void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) | |
1177 | { | |
1178 | struct smc_llc_msg_del_link del_llc = {0}; | |
1179 | ||
1180 | del_llc.hd.length = sizeof(del_llc); | |
1181 | del_llc.hd.common.type = SMC_LLC_DELETE_LINK; | |
1182 | del_llc.link_num = del_link_id; | |
1183 | del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); | |
1184 | del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
1185 | smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); | |
1186 | } | |
1187 | ||
9c416878 KG |
1188 | static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) |
1189 | { | |
1190 | struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; | |
1191 | struct smc_llc_msg_del_link *del_llc; | |
1192 | struct smc_llc_qentry *qentry; | |
1193 | int active_links; | |
1194 | int lnk_idx; | |
1195 | ||
1196 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
1197 | lnk = qentry->link; | |
1198 | del_llc = &qentry->msg.delete_link; | |
1199 | ||
1200 | if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { | |
1201 | smc_lgr_terminate_sched(lgr); | |
1202 | goto out; | |
1203 | } | |
1204 | mutex_lock(&lgr->llc_conf_mutex); | |
1205 | /* delete single link */ | |
1206 | for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { | |
1207 | if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) | |
1208 | continue; | |
1209 | lnk_del = &lgr->lnk[lnk_idx]; | |
1210 | break; | |
1211 | } | |
1212 | del_llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
1213 | if (!lnk_del) { | |
1214 | /* link was not found */ | |
1215 | del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); | |
1216 | smc_llc_send_message(lnk, &qentry->msg); | |
1217 | goto out_unlock; | |
1218 | } | |
1219 | lnk_asym = smc_llc_find_asym_link(lgr); | |
1220 | ||
1221 | del_llc->reason = 0; | |
1222 | smc_llc_send_message(lnk, &qentry->msg); /* response */ | |
1223 | ||
1224 | if (smc_link_downing(&lnk_del->state)) { | |
c6f02ebe | 1225 | smc_switch_conns(lgr, lnk_del, false); |
9c416878 KG |
1226 | smc_wr_tx_wait_no_pending_sends(lnk_del); |
1227 | } | |
1228 | smcr_link_clear(lnk_del); | |
1229 | ||
1230 | active_links = smc_llc_active_link_count(lgr); | |
1231 | if (lnk_del == lnk_asym) { | |
1232 | /* expected deletion of asym link, don't change lgr state */ | |
1233 | } else if (active_links == 1) { | |
ad6c111b | 1234 | smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); |
9c416878 | 1235 | } else if (!active_links) { |
ad6c111b | 1236 | smcr_lgr_set_type(lgr, SMC_LGR_NONE); |
9c416878 KG |
1237 | smc_lgr_terminate_sched(lgr); |
1238 | } | |
1239 | out_unlock: | |
1240 | mutex_unlock(&lgr->llc_conf_mutex); | |
1241 | out: | |
1242 | kfree(qentry); | |
1243 | } | |
1244 | ||
f3811fd7 KG |
1245 | /* try to send a DELETE LINK ALL request on any active link, |
1246 | * waiting for send completion | |
1247 | */ | |
1248 | void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) | |
1249 | { | |
1250 | struct smc_llc_msg_del_link delllc = {0}; | |
1251 | int i; | |
1252 | ||
1253 | delllc.hd.common.type = SMC_LLC_DELETE_LINK; | |
1254 | delllc.hd.length = sizeof(delllc); | |
1255 | if (ord) | |
1256 | delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
1257 | delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; | |
1258 | delllc.reason = htonl(rsn); | |
1259 | ||
1260 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
1261 | if (!smc_link_usable(&lgr->lnk[i])) | |
1262 | continue; | |
1263 | if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) | |
1264 | break; | |
1265 | } | |
1266 | } | |
1267 | ||
08ae27dd KG |
1268 | static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) |
1269 | { | |
1270 | struct smc_llc_msg_del_link *del_llc; | |
1271 | struct smc_link *lnk, *lnk_del; | |
1272 | struct smc_llc_qentry *qentry; | |
1273 | int active_links; | |
1274 | int i; | |
1275 | ||
1276 | mutex_lock(&lgr->llc_conf_mutex); | |
1277 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
1278 | lnk = qentry->link; | |
1279 | del_llc = &qentry->msg.delete_link; | |
1280 | ||
1281 | if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { | |
1282 | /* delete entire lgr */ | |
f3811fd7 KG |
1283 | smc_llc_send_link_delete_all(lgr, true, ntohl( |
1284 | qentry->msg.delete_link.reason)); | |
08ae27dd KG |
1285 | smc_lgr_terminate_sched(lgr); |
1286 | goto out; | |
1287 | } | |
1288 | /* delete single link */ | |
1289 | lnk_del = NULL; | |
1290 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
1291 | if (lgr->lnk[i].link_id == del_llc->link_num) { | |
1292 | lnk_del = &lgr->lnk[i]; | |
1293 | break; | |
1294 | } | |
1295 | } | |
1296 | if (!lnk_del) | |
1297 | goto out; /* asymmetric link already deleted */ | |
1298 | ||
1299 | if (smc_link_downing(&lnk_del->state)) { | |
c6f02ebe | 1300 | smc_switch_conns(lgr, lnk_del, false); |
08ae27dd KG |
1301 | smc_wr_tx_wait_no_pending_sends(lnk_del); |
1302 | } | |
1303 | if (!list_empty(&lgr->list)) { | |
1304 | /* qentry is either a request from peer (send it back to | |
1305 | * initiate the DELETE_LINK processing), or a locally | |
1306 | * enqueued DELETE_LINK request (forward it) | |
1307 | */ | |
1308 | if (!smc_llc_send_message(lnk, &qentry->msg)) { | |
1309 | struct smc_llc_msg_del_link *del_llc_resp; | |
1310 | struct smc_llc_qentry *qentry2; | |
1311 | ||
1312 | qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, | |
1313 | SMC_LLC_DELETE_LINK); | |
1314 | if (!qentry2) { | |
1315 | } else { | |
1316 | del_llc_resp = &qentry2->msg.delete_link; | |
1317 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1318 | } | |
1319 | } | |
1320 | } | |
1321 | smcr_link_clear(lnk_del); | |
1322 | ||
1323 | active_links = smc_llc_active_link_count(lgr); | |
1324 | if (active_links == 1) { | |
ad6c111b | 1325 | smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); |
08ae27dd | 1326 | } else if (!active_links) { |
ad6c111b | 1327 | smcr_lgr_set_type(lgr, SMC_LGR_NONE); |
08ae27dd KG |
1328 | smc_lgr_terminate_sched(lgr); |
1329 | } | |
1330 | ||
1331 | if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { | |
1332 | /* trigger setup of asymm alt link */ | |
4dadd151 | 1333 | smc_llc_srv_add_link_local(lnk); |
08ae27dd KG |
1334 | } |
1335 | out: | |
1336 | mutex_unlock(&lgr->llc_conf_mutex); | |
1337 | kfree(qentry); | |
1338 | } | |
1339 | ||
9ec6bf19 | 1340 | static void smc_llc_delete_link_work(struct work_struct *work) |
52bedf37 | 1341 | { |
9ec6bf19 KG |
1342 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, |
1343 | llc_del_link_work); | |
52bedf37 | 1344 | |
9ec6bf19 KG |
1345 | if (list_empty(&lgr->list)) { |
1346 | /* link group is terminating */ | |
1347 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1348 | goto out; | |
52bedf37 | 1349 | } |
9c416878 KG |
1350 | |
1351 | if (lgr->role == SMC_CLNT) | |
1352 | smc_llc_process_cli_delete_link(lgr); | |
08ae27dd KG |
1353 | else |
1354 | smc_llc_process_srv_delete_link(lgr); | |
9ec6bf19 KG |
1355 | out: |
1356 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); | |
52bedf37 KG |
1357 | } |
1358 | ||
3bc67e09 KG |
1359 | /* process a confirm_rkey request from peer, remote flow */ |
1360 | static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 1361 | { |
3bc67e09 KG |
1362 | struct smc_llc_msg_confirm_rkey *llc; |
1363 | struct smc_llc_qentry *qentry; | |
1364 | struct smc_link *link; | |
1365 | int num_entries; | |
1366 | int rk_idx; | |
1367 | int i; | |
1368 | ||
1369 | qentry = lgr->llc_flow_rmt.qentry; | |
1370 | llc = &qentry->msg.confirm_rkey; | |
1371 | link = qentry->link; | |
1372 | ||
1373 | num_entries = llc->rtoken[0].num_rkeys; | |
1374 | /* first rkey entry is for receiving link */ | |
1375 | rk_idx = smc_rtoken_add(link, | |
1376 | llc->rtoken[0].rmb_vaddr, | |
1377 | llc->rtoken[0].rmb_key); | |
1378 | if (rk_idx < 0) | |
1379 | goto out_err; | |
1380 | ||
1381 | for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) | |
1382 | smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, | |
1383 | llc->rtoken[i].rmb_vaddr, | |
1384 | llc->rtoken[i].rmb_key); | |
1385 | /* max links is 3 so there is no need to support conf_rkey_cont msgs */ | |
1386 | goto out; | |
1387 | out_err: | |
1388 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
1389 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; | |
1390 | out: | |
ef79d439 | 1391 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
3bc67e09 KG |
1392 | smc_llc_send_message(link, &qentry->msg); |
1393 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
4ed75de5 KG |
1394 | } |
1395 | ||
218b24fe KG |
1396 | /* process a delete_rkey request from peer, remote flow */ |
1397 | static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 1398 | { |
218b24fe KG |
1399 | struct smc_llc_msg_delete_rkey *llc; |
1400 | struct smc_llc_qentry *qentry; | |
1401 | struct smc_link *link; | |
4ed75de5 KG |
1402 | u8 err_mask = 0; |
1403 | int i, max; | |
1404 | ||
218b24fe KG |
1405 | qentry = lgr->llc_flow_rmt.qentry; |
1406 | llc = &qentry->msg.delete_rkey; | |
1407 | link = qentry->link; | |
1408 | ||
ef79d439 KG |
1409 | max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); |
1410 | for (i = 0; i < max; i++) { | |
1411 | if (smc_rtoken_delete(link, llc->rkey[i])) | |
1412 | err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); | |
1413 | } | |
ef79d439 KG |
1414 | if (err_mask) { |
1415 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
1416 | llc->err_mask = err_mask; | |
4ed75de5 | 1417 | } |
218b24fe KG |
1418 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
1419 | smc_llc_send_message(link, &qentry->msg); | |
1420 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
1421 | } | |
ef79d439 | 1422 | |
6c8968c4 | 1423 | /* flush the llc event queue */ |
00a049cf | 1424 | static void smc_llc_event_flush(struct smc_link_group *lgr) |
9bf9abea | 1425 | { |
6c8968c4 KG |
1426 | struct smc_llc_qentry *qentry, *q; |
1427 | ||
1428 | spin_lock_bh(&lgr->llc_event_q_lock); | |
1429 | list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { | |
1430 | list_del_init(&qentry->list); | |
1431 | kfree(qentry); | |
1432 | } | |
1433 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1434 | } | |
1435 | ||
1436 | static void smc_llc_event_handler(struct smc_llc_qentry *qentry) | |
1437 | { | |
1438 | union smc_llc_msg *llc = &qentry->msg; | |
1439 | struct smc_link *link = qentry->link; | |
0fb0b02b | 1440 | struct smc_link_group *lgr = link->lgr; |
9bf9abea | 1441 | |
d854fcbf | 1442 | if (!smc_link_usable(link)) |
6c8968c4 | 1443 | goto out; |
313164da KG |
1444 | |
1445 | switch (llc->raw.hdr.common.type) { | |
1446 | case SMC_LLC_TEST_LINK: | |
56e8091c KG |
1447 | llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; |
1448 | smc_llc_send_message(link, llc); | |
313164da | 1449 | break; |
52bedf37 | 1450 | case SMC_LLC_ADD_LINK: |
0fb0b02b KG |
1451 | if (list_empty(&lgr->list)) |
1452 | goto out; /* lgr is terminating */ | |
1453 | if (lgr->role == SMC_CLNT) { | |
1454 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { | |
1455 | /* a flow is waiting for this message */ | |
1456 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1457 | qentry); | |
1458 | wake_up_interruptible(&lgr->llc_waiter); | |
1459 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, | |
1460 | qentry)) { | |
b45e7f98 | 1461 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1462 | } |
1463 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { | |
1464 | /* as smc server, handle client suggestion */ | |
b45e7f98 | 1465 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1466 | } |
1467 | return; | |
1468 | case SMC_LLC_CONFIRM_LINK: | |
87f88cda | 1469 | case SMC_LLC_ADD_LINK_CONT: |
0fb0b02b KG |
1470 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { |
1471 | /* a flow is waiting for this message */ | |
1472 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); | |
1473 | wake_up_interruptible(&lgr->llc_waiter); | |
1474 | return; | |
1475 | } | |
52bedf37 KG |
1476 | break; |
1477 | case SMC_LLC_DELETE_LINK: | |
9ec6bf19 KG |
1478 | if (lgr->role == SMC_CLNT) { |
1479 | /* server requests to delete this link, send response */ | |
1480 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { | |
1481 | /* DEL LINK REQ during ADD LINK SEQ */ | |
1482 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1483 | qentry); | |
1484 | wake_up_interruptible(&lgr->llc_waiter); | |
1485 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, | |
1486 | qentry)) { | |
1487 | schedule_work(&lgr->llc_del_link_work); | |
1488 | } | |
1489 | } else { | |
1490 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && | |
1491 | !lgr->llc_flow_lcl.qentry) { | |
1492 | /* DEL LINK REQ during ADD LINK SEQ */ | |
1493 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1494 | qentry); | |
1495 | wake_up_interruptible(&lgr->llc_waiter); | |
1496 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, | |
1497 | qentry)) { | |
1498 | schedule_work(&lgr->llc_del_link_work); | |
1499 | } | |
1500 | } | |
1501 | return; | |
4ed75de5 | 1502 | case SMC_LLC_CONFIRM_RKEY: |
3bc67e09 KG |
1503 | /* new request from remote, assign to remote flow */ |
1504 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1505 | /* process here, does not wait for more llc msgs */ | |
1506 | smc_llc_rmt_conf_rkey(lgr); | |
1507 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1508 | } | |
1509 | return; | |
4ed75de5 | 1510 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc KG |
1511 | /* not used because max links is 3, and 3 rkeys fit into |
1512 | * one CONFIRM_RKEY message | |
1513 | */ | |
4ed75de5 KG |
1514 | break; |
1515 | case SMC_LLC_DELETE_RKEY: | |
218b24fe KG |
1516 | /* new request from remote, assign to remote flow */ |
1517 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1518 | /* process here, does not wait for more llc msgs */ | |
1519 | smc_llc_rmt_delete_rkey(lgr); | |
1520 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1521 | } | |
1522 | return; | |
313164da | 1523 | } |
6c8968c4 KG |
1524 | out: |
1525 | kfree(qentry); | |
1526 | } | |
1527 | ||
1528 | /* worker to process llc messages on the event queue */ | |
1529 | static void smc_llc_event_work(struct work_struct *work) | |
1530 | { | |
1531 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
1532 | llc_event_work); | |
1533 | struct smc_llc_qentry *qentry; | |
1534 | ||
555da9af KG |
1535 | if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { |
1536 | if (smc_link_usable(lgr->delayed_event->link)) { | |
1537 | smc_llc_event_handler(lgr->delayed_event); | |
1538 | } else { | |
1539 | qentry = lgr->delayed_event; | |
1540 | lgr->delayed_event = NULL; | |
1541 | kfree(qentry); | |
1542 | } | |
1543 | } | |
1544 | ||
6c8968c4 KG |
1545 | again: |
1546 | spin_lock_bh(&lgr->llc_event_q_lock); | |
1547 | if (!list_empty(&lgr->llc_event_q)) { | |
1548 | qentry = list_first_entry(&lgr->llc_event_q, | |
1549 | struct smc_llc_qentry, list); | |
1550 | list_del_init(&qentry->list); | |
1551 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1552 | smc_llc_event_handler(qentry); | |
1553 | goto again; | |
1554 | } | |
1555 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1556 | } | |
1557 | ||
ef79d439 | 1558 | /* process llc responses in tasklet context */ |
a6688d91 KG |
1559 | static void smc_llc_rx_response(struct smc_link *link, |
1560 | struct smc_llc_qentry *qentry) | |
ef79d439 | 1561 | { |
a6688d91 | 1562 | u8 llc_type = qentry->msg.raw.hdr.common.type; |
ef79d439 | 1563 | |
a6688d91 | 1564 | switch (llc_type) { |
ef79d439 KG |
1565 | case SMC_LLC_TEST_LINK: |
1566 | if (link->state == SMC_LNK_ACTIVE) | |
1567 | complete(&link->llc_testlink_resp); | |
1568 | break; | |
ef79d439 | 1569 | case SMC_LLC_ADD_LINK: |
9ec6bf19 | 1570 | case SMC_LLC_DELETE_LINK: |
4667bb4a | 1571 | case SMC_LLC_CONFIRM_LINK: |
87f88cda | 1572 | case SMC_LLC_ADD_LINK_CONT: |
3d88a21b | 1573 | case SMC_LLC_CONFIRM_RKEY: |
6d74c3a8 | 1574 | case SMC_LLC_DELETE_RKEY: |
4667bb4a KG |
1575 | /* assign responses to the local flow, we requested them */ |
1576 | smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); | |
1577 | wake_up_interruptible(&link->lgr->llc_waiter); | |
1578 | return; | |
ef79d439 | 1579 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc | 1580 | /* not used because max links is 3 */ |
ef79d439 | 1581 | break; |
ef79d439 | 1582 | } |
a6688d91 | 1583 | kfree(qentry); |
ef79d439 KG |
1584 | } |
1585 | ||
a6688d91 | 1586 | static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) |
6c8968c4 | 1587 | { |
6c8968c4 KG |
1588 | struct smc_link_group *lgr = link->lgr; |
1589 | struct smc_llc_qentry *qentry; | |
6c8968c4 KG |
1590 | unsigned long flags; |
1591 | ||
6c8968c4 KG |
1592 | qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); |
1593 | if (!qentry) | |
1594 | return; | |
1595 | qentry->link = link; | |
1596 | INIT_LIST_HEAD(&qentry->list); | |
1597 | memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); | |
a6688d91 KG |
1598 | |
1599 | /* process responses immediately */ | |
1600 | if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { | |
1601 | smc_llc_rx_response(link, qentry); | |
1602 | return; | |
1603 | } | |
1604 | ||
1605 | /* add requests to event queue */ | |
6c8968c4 KG |
1606 | spin_lock_irqsave(&lgr->llc_event_q_lock, flags); |
1607 | list_add_tail(&qentry->list, &lgr->llc_event_q); | |
1608 | spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); | |
1609 | schedule_work(&link->lgr->llc_event_work); | |
9bf9abea UB |
1610 | } |
1611 | ||
a6688d91 KG |
1612 | /* copy received msg and add it to the event queue */ |
1613 | static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) | |
1614 | { | |
1615 | struct smc_link *link = (struct smc_link *)wc->qp->qp_context; | |
1616 | union smc_llc_msg *llc = buf; | |
1617 | ||
1618 | if (wc->byte_len < sizeof(*llc)) | |
1619 | return; /* short message */ | |
1620 | if (llc->raw.hdr.length != sizeof(*llc)) | |
1621 | return; /* invalid message */ | |
1622 | ||
1623 | smc_llc_enqueue(link, llc); | |
1624 | } | |
1625 | ||
44aa81ce | 1626 | /***************************** worker, utils *********************************/ |
877ae5be KG |
1627 | |
1628 | static void smc_llc_testlink_work(struct work_struct *work) | |
1629 | { | |
1630 | struct smc_link *link = container_of(to_delayed_work(work), | |
1631 | struct smc_link, llc_testlink_wrk); | |
1632 | unsigned long next_interval; | |
877ae5be KG |
1633 | unsigned long expire_time; |
1634 | u8 user_data[16] = { 0 }; | |
1635 | int rc; | |
1636 | ||
877ae5be KG |
1637 | if (link->state != SMC_LNK_ACTIVE) |
1638 | return; /* don't reschedule worker */ | |
1639 | expire_time = link->wr_rx_tstamp + link->llc_testlink_time; | |
1640 | if (time_is_after_jiffies(expire_time)) { | |
1641 | next_interval = expire_time - jiffies; | |
1642 | goto out; | |
1643 | } | |
1644 | reinit_completion(&link->llc_testlink_resp); | |
d97935fa | 1645 | smc_llc_send_test_link(link, user_data); |
877ae5be KG |
1646 | /* receive TEST LINK response over RoCE fabric */ |
1647 | rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, | |
1648 | SMC_LLC_WAIT_TIME); | |
1020e1ef KG |
1649 | if (link->state != SMC_LNK_ACTIVE) |
1650 | return; /* link state changed */ | |
877ae5be | 1651 | if (rc <= 0) { |
87523930 | 1652 | smcr_link_down_cond_sched(link); |
877ae5be KG |
1653 | return; |
1654 | } | |
1655 | next_interval = link->llc_testlink_time; | |
1656 | out: | |
1020e1ef | 1657 | schedule_delayed_work(&link->llc_testlink_wrk, next_interval); |
877ae5be KG |
1658 | } |
1659 | ||
00a049cf KG |
1660 | void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) |
1661 | { | |
1662 | struct net *net = sock_net(smc->clcsock->sk); | |
1663 | ||
1664 | INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); | |
b45e7f98 | 1665 | INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); |
9ec6bf19 | 1666 | INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); |
00a049cf KG |
1667 | INIT_LIST_HEAD(&lgr->llc_event_q); |
1668 | spin_lock_init(&lgr->llc_event_q_lock); | |
555da9af KG |
1669 | spin_lock_init(&lgr->llc_flow_lock); |
1670 | init_waitqueue_head(&lgr->llc_waiter); | |
d5500667 | 1671 | mutex_init(&lgr->llc_conf_mutex); |
00a049cf KG |
1672 | lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; |
1673 | } | |
1674 | ||
1675 | /* called after lgr was removed from lgr_list */ | |
1676 | void smc_llc_lgr_clear(struct smc_link_group *lgr) | |
1677 | { | |
1678 | smc_llc_event_flush(lgr); | |
555da9af | 1679 | wake_up_interruptible_all(&lgr->llc_waiter); |
00a049cf | 1680 | cancel_work_sync(&lgr->llc_event_work); |
b45e7f98 | 1681 | cancel_work_sync(&lgr->llc_add_link_work); |
9ec6bf19 | 1682 | cancel_work_sync(&lgr->llc_del_link_work); |
555da9af KG |
1683 | if (lgr->delayed_event) { |
1684 | kfree(lgr->delayed_event); | |
1685 | lgr->delayed_event = NULL; | |
1686 | } | |
00a049cf KG |
1687 | } |
1688 | ||
2a4c57a9 | 1689 | int smc_llc_link_init(struct smc_link *link) |
877ae5be KG |
1690 | { |
1691 | init_completion(&link->llc_testlink_resp); | |
1692 | INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); | |
2a4c57a9 | 1693 | return 0; |
b32cf4ab KG |
1694 | } |
1695 | ||
00a049cf | 1696 | void smc_llc_link_active(struct smc_link *link) |
b32cf4ab | 1697 | { |
877ae5be | 1698 | link->state = SMC_LNK_ACTIVE; |
00a049cf KG |
1699 | if (link->lgr->llc_testlink_time) { |
1700 | link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; | |
1020e1ef KG |
1701 | schedule_delayed_work(&link->llc_testlink_wrk, |
1702 | link->llc_testlink_time); | |
877ae5be KG |
1703 | } |
1704 | } | |
1705 | ||
877ae5be | 1706 | /* called in worker context */ |
2a4c57a9 | 1707 | void smc_llc_link_clear(struct smc_link *link) |
877ae5be | 1708 | { |
2140ac26 KG |
1709 | complete(&link->llc_testlink_resp); |
1710 | cancel_delayed_work_sync(&link->llc_testlink_wrk); | |
1711 | smc_wr_wakeup_reg_wait(link); | |
1712 | smc_wr_wakeup_tx_wait(link); | |
877ae5be KG |
1713 | } |
1714 | ||
3d88a21b KG |
1715 | /* register a new rtoken at the remote peer (for all links) */ |
1716 | int smc_llc_do_confirm_rkey(struct smc_link *send_link, | |
44aa81ce KG |
1717 | struct smc_buf_desc *rmb_desc) |
1718 | { | |
3d88a21b KG |
1719 | struct smc_link_group *lgr = send_link->lgr; |
1720 | struct smc_llc_qentry *qentry = NULL; | |
1721 | int rc = 0; | |
44aa81ce | 1722 | |
3d88a21b KG |
1723 | rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); |
1724 | if (rc) | |
1725 | goto out; | |
44aa81ce | 1726 | /* receive CONFIRM RKEY response from server over RoCE fabric */ |
3d88a21b KG |
1727 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1728 | SMC_LLC_CONFIRM_RKEY); | |
1729 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
1730 | rc = -EFAULT; | |
1731 | out: | |
1732 | if (qentry) | |
1733 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
3d88a21b | 1734 | return rc; |
44aa81ce KG |
1735 | } |
1736 | ||
60e03c62 | 1737 | /* unregister an rtoken at the remote peer */ |
6d74c3a8 | 1738 | int smc_llc_do_delete_rkey(struct smc_link_group *lgr, |
60e03c62 KG |
1739 | struct smc_buf_desc *rmb_desc) |
1740 | { | |
6d74c3a8 KG |
1741 | struct smc_llc_qentry *qentry = NULL; |
1742 | struct smc_link *send_link; | |
0b29ec64 | 1743 | int rc = 0; |
60e03c62 | 1744 | |
6d74c3a8 KG |
1745 | send_link = smc_llc_usable_link(lgr); |
1746 | if (!send_link) | |
1747 | return -ENOLINK; | |
1748 | ||
6d74c3a8 KG |
1749 | /* protected by llc_flow control */ |
1750 | rc = smc_llc_send_delete_rkey(send_link, rmb_desc); | |
60e03c62 KG |
1751 | if (rc) |
1752 | goto out; | |
1753 | /* receive DELETE RKEY response from server over RoCE fabric */ | |
6d74c3a8 KG |
1754 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1755 | SMC_LLC_DELETE_RKEY); | |
1756 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
60e03c62 | 1757 | rc = -EFAULT; |
60e03c62 | 1758 | out: |
6d74c3a8 KG |
1759 | if (qentry) |
1760 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
60e03c62 KG |
1761 | return rc; |
1762 | } | |
1763 | ||
92334cfc KG |
1764 | /* evaluate confirm link request or response */ |
1765 | int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, | |
1766 | enum smc_llc_reqresp type) | |
1767 | { | |
1768 | if (type == SMC_LLC_REQ) /* SMC server assigns link_id */ | |
1769 | qentry->link->link_id = qentry->msg.confirm_link.link_num; | |
1770 | if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) | |
1771 | return -ENOTSUPP; | |
1772 | return 0; | |
1773 | } | |
1774 | ||
9bf9abea UB |
1775 | /***************************** init, exit, misc ******************************/ |
1776 | ||
1777 | static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { | |
1778 | { | |
1779 | .handler = smc_llc_rx_handler, | |
1780 | .type = SMC_LLC_CONFIRM_LINK | |
1781 | }, | |
313164da KG |
1782 | { |
1783 | .handler = smc_llc_rx_handler, | |
1784 | .type = SMC_LLC_TEST_LINK | |
1785 | }, | |
52bedf37 KG |
1786 | { |
1787 | .handler = smc_llc_rx_handler, | |
1788 | .type = SMC_LLC_ADD_LINK | |
1789 | }, | |
87f88cda KG |
1790 | { |
1791 | .handler = smc_llc_rx_handler, | |
1792 | .type = SMC_LLC_ADD_LINK_CONT | |
1793 | }, | |
52bedf37 KG |
1794 | { |
1795 | .handler = smc_llc_rx_handler, | |
1796 | .type = SMC_LLC_DELETE_LINK | |
1797 | }, | |
4ed75de5 KG |
1798 | { |
1799 | .handler = smc_llc_rx_handler, | |
1800 | .type = SMC_LLC_CONFIRM_RKEY | |
1801 | }, | |
1802 | { | |
1803 | .handler = smc_llc_rx_handler, | |
1804 | .type = SMC_LLC_CONFIRM_RKEY_CONT | |
1805 | }, | |
1806 | { | |
1807 | .handler = smc_llc_rx_handler, | |
1808 | .type = SMC_LLC_DELETE_RKEY | |
1809 | }, | |
9bf9abea UB |
1810 | { |
1811 | .handler = NULL, | |
1812 | } | |
1813 | }; | |
1814 | ||
1815 | int __init smc_llc_init(void) | |
1816 | { | |
1817 | struct smc_wr_rx_handler *handler; | |
1818 | int rc = 0; | |
1819 | ||
1820 | for (handler = smc_llc_rx_handlers; handler->handler; handler++) { | |
1821 | INIT_HLIST_NODE(&handler->list); | |
1822 | rc = smc_wr_rx_register_handler(handler); | |
1823 | if (rc) | |
1824 | break; | |
1825 | } | |
1826 | return rc; | |
1827 | } |