Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9bf9abea UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Link Layer Control (LLC) | |
6 | * | |
9bf9abea UB |
7 | * Copyright IBM Corp. 2016 |
8 | * | |
9 | * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> | |
10 | * Ursula Braun <ubraun@linux.vnet.ibm.com> | |
11 | */ | |
12 | ||
13 | #include <net/tcp.h> | |
14 | #include <rdma/ib_verbs.h> | |
15 | ||
16 | #include "smc.h" | |
17 | #include "smc_core.h" | |
18 | #include "smc_clc.h" | |
19 | #include "smc_llc.h" | |
336ba09f | 20 | #include "smc_pnet.h" |
9bf9abea | 21 | |
0f627126 SR |
22 | #define SMC_LLC_DATA_LEN 40 |
23 | ||
24 | struct smc_llc_hdr { | |
25 | struct smc_wr_rx_hdr common; | |
26 | u8 length; /* 44 */ | |
52bedf37 KG |
27 | #if defined(__BIG_ENDIAN_BITFIELD) |
28 | u8 reserved:4, | |
29 | add_link_rej_rsn:4; | |
30 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
31 | u8 add_link_rej_rsn:4, | |
32 | reserved:4; | |
33 | #endif | |
0f627126 SR |
34 | u8 flags; |
35 | }; | |
36 | ||
75d320d6 KG |
37 | #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 |
38 | ||
0f627126 SR |
39 | struct smc_llc_msg_confirm_link { /* type 0x01 */ |
40 | struct smc_llc_hdr hd; | |
41 | u8 sender_mac[ETH_ALEN]; | |
42 | u8 sender_gid[SMC_GID_SIZE]; | |
43 | u8 sender_qp_num[3]; | |
44 | u8 link_num; | |
45 | u8 link_uid[SMC_LGR_ID_SIZE]; | |
46 | u8 max_links; | |
47 | u8 reserved[9]; | |
48 | }; | |
49 | ||
52bedf37 KG |
50 | #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 |
51 | #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 | |
52 | ||
53 | #define SMC_LLC_ADD_LNK_MAX_LINKS 2 | |
54 | ||
55 | struct smc_llc_msg_add_link { /* type 0x02 */ | |
56 | struct smc_llc_hdr hd; | |
57 | u8 sender_mac[ETH_ALEN]; | |
58 | u8 reserved2[2]; | |
59 | u8 sender_gid[SMC_GID_SIZE]; | |
60 | u8 sender_qp_num[3]; | |
61 | u8 link_num; | |
fbed3b37 KG |
62 | #if defined(__BIG_ENDIAN_BITFIELD) |
63 | u8 reserved3 : 4, | |
64 | qp_mtu : 4; | |
65 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
66 | u8 qp_mtu : 4, | |
67 | reserved3 : 4; | |
68 | #endif | |
52bedf37 KG |
69 | u8 initial_psn[3]; |
70 | u8 reserved[8]; | |
71 | }; | |
72 | ||
87f88cda KG |
73 | struct smc_llc_msg_add_link_cont_rt { |
74 | __be32 rmb_key; | |
75 | __be32 rmb_key_new; | |
76 | __be64 rmb_vaddr_new; | |
77 | }; | |
78 | ||
79 | #define SMC_LLC_RKEYS_PER_CONT_MSG 2 | |
80 | ||
81 | struct smc_llc_msg_add_link_cont { /* type 0x03 */ | |
82 | struct smc_llc_hdr hd; | |
83 | u8 link_num; | |
84 | u8 num_rkeys; | |
85 | u8 reserved2[2]; | |
86 | struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; | |
87 | u8 reserved[4]; | |
88 | } __packed; /* format defined in RFC7609 */ | |
89 | ||
52bedf37 KG |
90 | #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 |
91 | #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 | |
92 | ||
93 | struct smc_llc_msg_del_link { /* type 0x04 */ | |
94 | struct smc_llc_hdr hd; | |
95 | u8 link_num; | |
96 | __be32 reason; | |
97 | u8 reserved[35]; | |
98 | } __packed; /* format defined in RFC7609 */ | |
99 | ||
313164da KG |
100 | struct smc_llc_msg_test_link { /* type 0x07 */ |
101 | struct smc_llc_hdr hd; | |
102 | u8 user_data[16]; | |
103 | u8 reserved[24]; | |
104 | }; | |
105 | ||
4ed75de5 KG |
106 | struct smc_rmb_rtoken { |
107 | union { | |
108 | u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ | |
109 | /* is actually the num of rtokens, first */ | |
110 | /* rtoken is always for the current link */ | |
111 | u8 link_id; /* link id of the rtoken */ | |
112 | }; | |
113 | __be32 rmb_key; | |
114 | __be64 rmb_vaddr; | |
115 | } __packed; /* format defined in RFC7609 */ | |
116 | ||
117 | #define SMC_LLC_RKEYS_PER_MSG 3 | |
118 | ||
119 | struct smc_llc_msg_confirm_rkey { /* type 0x06 */ | |
120 | struct smc_llc_hdr hd; | |
121 | struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; | |
122 | u8 reserved; | |
123 | }; | |
124 | ||
4ed75de5 | 125 | #define SMC_LLC_DEL_RKEY_MAX 8 |
3bc67e09 | 126 | #define SMC_LLC_FLAG_RKEY_RETRY 0x10 |
4ed75de5 KG |
127 | #define SMC_LLC_FLAG_RKEY_NEG 0x20 |
128 | ||
129 | struct smc_llc_msg_delete_rkey { /* type 0x09 */ | |
130 | struct smc_llc_hdr hd; | |
131 | u8 num_rkeys; | |
132 | u8 err_mask; | |
133 | u8 reserved[2]; | |
134 | __be32 rkey[8]; | |
135 | u8 reserved2[4]; | |
136 | }; | |
137 | ||
0f627126 SR |
138 | union smc_llc_msg { |
139 | struct smc_llc_msg_confirm_link confirm_link; | |
52bedf37 | 140 | struct smc_llc_msg_add_link add_link; |
87f88cda | 141 | struct smc_llc_msg_add_link_cont add_link_cont; |
52bedf37 | 142 | struct smc_llc_msg_del_link delete_link; |
4ed75de5 KG |
143 | |
144 | struct smc_llc_msg_confirm_rkey confirm_rkey; | |
4ed75de5 KG |
145 | struct smc_llc_msg_delete_rkey delete_rkey; |
146 | ||
313164da | 147 | struct smc_llc_msg_test_link test_link; |
0f627126 SR |
148 | struct { |
149 | struct smc_llc_hdr hdr; | |
150 | u8 data[SMC_LLC_DATA_LEN]; | |
151 | } raw; | |
152 | }; | |
153 | ||
154 | #define SMC_LLC_FLAG_RESP 0x80 | |
155 | ||
6c8968c4 KG |
156 | struct smc_llc_qentry { |
157 | struct list_head list; | |
158 | struct smc_link *link; | |
159 | union smc_llc_msg msg; | |
160 | }; | |
161 | ||
555da9af KG |
162 | struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) |
163 | { | |
164 | struct smc_llc_qentry *qentry = flow->qentry; | |
165 | ||
166 | flow->qentry = NULL; | |
167 | return qentry; | |
168 | } | |
169 | ||
170 | void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) | |
171 | { | |
172 | struct smc_llc_qentry *qentry; | |
173 | ||
174 | if (flow->qentry) { | |
175 | qentry = flow->qentry; | |
176 | flow->qentry = NULL; | |
177 | kfree(qentry); | |
178 | } | |
179 | } | |
180 | ||
181 | static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, | |
182 | struct smc_llc_qentry *qentry) | |
183 | { | |
184 | flow->qentry = qentry; | |
185 | } | |
186 | ||
187 | /* try to start a new llc flow, initiated by an incoming llc msg */ | |
188 | static bool smc_llc_flow_start(struct smc_llc_flow *flow, | |
189 | struct smc_llc_qentry *qentry) | |
190 | { | |
191 | struct smc_link_group *lgr = qentry->link->lgr; | |
192 | ||
193 | spin_lock_bh(&lgr->llc_flow_lock); | |
194 | if (flow->type) { | |
195 | /* a flow is already active */ | |
196 | if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK || | |
197 | qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) && | |
198 | !lgr->delayed_event) { | |
199 | lgr->delayed_event = qentry; | |
200 | } else { | |
201 | /* forget this llc request */ | |
202 | kfree(qentry); | |
203 | } | |
204 | spin_unlock_bh(&lgr->llc_flow_lock); | |
205 | return false; | |
206 | } | |
207 | switch (qentry->msg.raw.hdr.common.type) { | |
208 | case SMC_LLC_ADD_LINK: | |
209 | flow->type = SMC_LLC_FLOW_ADD_LINK; | |
210 | break; | |
211 | case SMC_LLC_DELETE_LINK: | |
212 | flow->type = SMC_LLC_FLOW_DEL_LINK; | |
213 | break; | |
214 | case SMC_LLC_CONFIRM_RKEY: | |
215 | case SMC_LLC_DELETE_RKEY: | |
216 | flow->type = SMC_LLC_FLOW_RKEY; | |
217 | break; | |
218 | default: | |
219 | flow->type = SMC_LLC_FLOW_NONE; | |
220 | } | |
221 | if (qentry == lgr->delayed_event) | |
222 | lgr->delayed_event = NULL; | |
223 | spin_unlock_bh(&lgr->llc_flow_lock); | |
224 | smc_llc_flow_qentry_set(flow, qentry); | |
225 | return true; | |
226 | } | |
227 | ||
228 | /* start a new local llc flow, wait till current flow finished */ | |
229 | int smc_llc_flow_initiate(struct smc_link_group *lgr, | |
230 | enum smc_llc_flowtype type) | |
231 | { | |
232 | enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; | |
233 | int rc; | |
234 | ||
235 | /* all flows except confirm_rkey and delete_rkey are exclusive, | |
236 | * confirm/delete rkey flows can run concurrently (local and remote) | |
237 | */ | |
238 | if (type == SMC_LLC_FLOW_RKEY) | |
239 | allowed_remote = SMC_LLC_FLOW_RKEY; | |
240 | again: | |
241 | if (list_empty(&lgr->list)) | |
242 | return -ENODEV; | |
243 | spin_lock_bh(&lgr->llc_flow_lock); | |
244 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
245 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
246 | lgr->llc_flow_rmt.type == allowed_remote)) { | |
247 | lgr->llc_flow_lcl.type = type; | |
248 | spin_unlock_bh(&lgr->llc_flow_lock); | |
249 | return 0; | |
250 | } | |
251 | spin_unlock_bh(&lgr->llc_flow_lock); | |
252 | rc = wait_event_interruptible_timeout(lgr->llc_waiter, | |
253 | (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
254 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
255 | lgr->llc_flow_rmt.type == allowed_remote)), | |
256 | SMC_LLC_WAIT_TIME); | |
257 | if (!rc) | |
258 | return -ETIMEDOUT; | |
259 | goto again; | |
260 | } | |
261 | ||
262 | /* finish the current llc flow */ | |
263 | void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) | |
264 | { | |
265 | spin_lock_bh(&lgr->llc_flow_lock); | |
266 | memset(flow, 0, sizeof(*flow)); | |
267 | flow->type = SMC_LLC_FLOW_NONE; | |
268 | spin_unlock_bh(&lgr->llc_flow_lock); | |
269 | if (!list_empty(&lgr->list) && lgr->delayed_event && | |
270 | flow == &lgr->llc_flow_lcl) | |
271 | schedule_work(&lgr->llc_event_work); | |
272 | else | |
273 | wake_up_interruptible(&lgr->llc_waiter); | |
274 | } | |
275 | ||
276 | /* lnk is optional and used for early wakeup when link goes down, useful in | |
277 | * cases where we wait for a response on the link after we sent a request | |
278 | */ | |
279 | struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, | |
280 | struct smc_link *lnk, | |
281 | int time_out, u8 exp_msg) | |
282 | { | |
283 | struct smc_llc_flow *flow = &lgr->llc_flow_lcl; | |
284 | ||
285 | wait_event_interruptible_timeout(lgr->llc_waiter, | |
286 | (flow->qentry || | |
287 | (lnk && !smc_link_usable(lnk)) || | |
288 | list_empty(&lgr->list)), | |
289 | time_out); | |
290 | if (!flow->qentry || | |
291 | (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { | |
292 | smc_llc_flow_qentry_del(flow); | |
293 | goto out; | |
294 | } | |
295 | if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) { | |
296 | if (exp_msg == SMC_LLC_ADD_LINK && | |
297 | flow->qentry->msg.raw.hdr.common.type == | |
298 | SMC_LLC_DELETE_LINK) { | |
299 | /* flow_start will delay the unexpected msg */ | |
300 | smc_llc_flow_start(&lgr->llc_flow_lcl, | |
301 | smc_llc_flow_qentry_clr(flow)); | |
302 | return NULL; | |
303 | } | |
304 | smc_llc_flow_qentry_del(flow); | |
305 | } | |
306 | out: | |
307 | return flow->qentry; | |
308 | } | |
309 | ||
9bf9abea UB |
310 | /********************************** send *************************************/ |
311 | ||
312 | struct smc_llc_tx_pend { | |
313 | }; | |
314 | ||
315 | /* handler for send/transmission completion of an LLC msg */ | |
316 | static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, | |
317 | struct smc_link *link, | |
318 | enum ib_wc_status wc_status) | |
319 | { | |
320 | /* future work: handle wc_status error for recovery and failover */ | |
321 | } | |
322 | ||
323 | /** | |
324 | * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits | |
325 | * @link: Pointer to SMC link used for sending LLC control message. | |
326 | * @wr_buf: Out variable returning pointer to work request payload buffer. | |
327 | * @pend: Out variable returning pointer to private pending WR tracking. | |
328 | * It's the context the transmit complete handler will get. | |
329 | * | |
330 | * Reserves and pre-fills an entry for a pending work request send/tx. | |
331 | * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. | |
332 | * Can sleep due to smc_get_ctrl_buf (if not in softirq context). | |
333 | * | |
334 | * Return: 0 on success, otherwise an error value. | |
335 | */ | |
336 | static int smc_llc_add_pending_send(struct smc_link *link, | |
337 | struct smc_wr_buf **wr_buf, | |
338 | struct smc_wr_tx_pend_priv **pend) | |
339 | { | |
340 | int rc; | |
341 | ||
ad6f317f UB |
342 | rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, |
343 | pend); | |
9bf9abea UB |
344 | if (rc < 0) |
345 | return rc; | |
346 | BUILD_BUG_ON_MSG( | |
347 | sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, | |
348 | "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); | |
349 | BUILD_BUG_ON_MSG( | |
350 | sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, | |
351 | "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); | |
352 | BUILD_BUG_ON_MSG( | |
353 | sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, | |
354 | "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); | |
355 | return 0; | |
356 | } | |
357 | ||
358 | /* high-level API to send LLC confirm link */ | |
947541f3 | 359 | int smc_llc_send_confirm_link(struct smc_link *link, |
9bf9abea UB |
360 | enum smc_llc_reqresp reqresp) |
361 | { | |
00e5fb26 | 362 | struct smc_link_group *lgr = smc_get_lgr(link); |
9bf9abea UB |
363 | struct smc_llc_msg_confirm_link *confllc; |
364 | struct smc_wr_tx_pend_priv *pend; | |
365 | struct smc_wr_buf *wr_buf; | |
366 | int rc; | |
367 | ||
368 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
369 | if (rc) | |
370 | return rc; | |
371 | confllc = (struct smc_llc_msg_confirm_link *)wr_buf; | |
372 | memset(confllc, 0, sizeof(*confllc)); | |
373 | confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; | |
374 | confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); | |
75d320d6 | 375 | confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; |
9bf9abea UB |
376 | if (reqresp == SMC_LLC_RESP) |
377 | confllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
947541f3 UB |
378 | memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], |
379 | ETH_ALEN); | |
7005ada6 | 380 | memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); |
9bf9abea | 381 | hton24(confllc->sender_qp_num, link->roce_qp->qp_num); |
2be922f3 | 382 | confllc->link_num = link->link_id; |
9bf9abea | 383 | memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); |
b1570a87 | 384 | confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; |
52bedf37 KG |
385 | /* send llc message */ |
386 | rc = smc_wr_tx_send(link, pend); | |
387 | return rc; | |
388 | } | |
389 | ||
44aa81ce | 390 | /* send LLC confirm rkey request */ |
3d88a21b | 391 | static int smc_llc_send_confirm_rkey(struct smc_link *send_link, |
44aa81ce KG |
392 | struct smc_buf_desc *rmb_desc) |
393 | { | |
394 | struct smc_llc_msg_confirm_rkey *rkeyllc; | |
395 | struct smc_wr_tx_pend_priv *pend; | |
396 | struct smc_wr_buf *wr_buf; | |
3d88a21b KG |
397 | struct smc_link *link; |
398 | int i, rc, rtok_ix; | |
44aa81ce | 399 | |
3d88a21b | 400 | rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); |
44aa81ce KG |
401 | if (rc) |
402 | return rc; | |
403 | rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; | |
404 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
405 | rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; | |
406 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); | |
3d88a21b KG |
407 | |
408 | rtok_ix = 1; | |
409 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
410 | link = &send_link->lgr->lnk[i]; | |
411 | if (link->state == SMC_LNK_ACTIVE && link != send_link) { | |
412 | rkeyllc->rtoken[rtok_ix].link_id = link->link_id; | |
413 | rkeyllc->rtoken[rtok_ix].rmb_key = | |
414 | htonl(rmb_desc->mr_rx[link->link_idx]->rkey); | |
415 | rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( | |
416 | (u64)sg_dma_address( | |
417 | rmb_desc->sgt[link->link_idx].sgl)); | |
418 | rtok_ix++; | |
419 | } | |
420 | } | |
421 | /* rkey of send_link is in rtoken[0] */ | |
422 | rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; | |
44aa81ce | 423 | rkeyllc->rtoken[0].rmb_key = |
3d88a21b | 424 | htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); |
44aa81ce | 425 | rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( |
3d88a21b | 426 | (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); |
44aa81ce | 427 | /* send llc message */ |
3d88a21b | 428 | rc = smc_wr_tx_send(send_link, pend); |
44aa81ce KG |
429 | return rc; |
430 | } | |
431 | ||
60e03c62 KG |
432 | /* send LLC delete rkey request */ |
433 | static int smc_llc_send_delete_rkey(struct smc_link *link, | |
434 | struct smc_buf_desc *rmb_desc) | |
435 | { | |
436 | struct smc_llc_msg_delete_rkey *rkeyllc; | |
437 | struct smc_wr_tx_pend_priv *pend; | |
438 | struct smc_wr_buf *wr_buf; | |
439 | int rc; | |
440 | ||
441 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
442 | if (rc) | |
443 | return rc; | |
444 | rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; | |
445 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
446 | rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; | |
447 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); | |
448 | rkeyllc->num_rkeys = 1; | |
387707fd | 449 | rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); |
60e03c62 KG |
450 | /* send llc message */ |
451 | rc = smc_wr_tx_send(link, pend); | |
452 | return rc; | |
453 | } | |
454 | ||
52bedf37 | 455 | /* send ADD LINK request or response */ |
7005ada6 | 456 | int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], |
fbed3b37 | 457 | struct smc_link *link_new, |
52bedf37 KG |
458 | enum smc_llc_reqresp reqresp) |
459 | { | |
460 | struct smc_llc_msg_add_link *addllc; | |
461 | struct smc_wr_tx_pend_priv *pend; | |
462 | struct smc_wr_buf *wr_buf; | |
463 | int rc; | |
464 | ||
465 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
466 | if (rc) | |
467 | return rc; | |
468 | addllc = (struct smc_llc_msg_add_link *)wr_buf; | |
fbed3b37 KG |
469 | |
470 | memset(addllc, 0, sizeof(*addllc)); | |
471 | addllc->hd.common.type = SMC_LLC_ADD_LINK; | |
472 | addllc->hd.length = sizeof(struct smc_llc_msg_add_link); | |
473 | if (reqresp == SMC_LLC_RESP) | |
474 | addllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
475 | memcpy(addllc->sender_mac, mac, ETH_ALEN); | |
476 | memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); | |
477 | if (link_new) { | |
478 | addllc->link_num = link_new->link_id; | |
479 | hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); | |
480 | hton24(addllc->initial_psn, link_new->psn_initial); | |
481 | if (reqresp == SMC_LLC_REQ) | |
482 | addllc->qp_mtu = link_new->path_mtu; | |
483 | else | |
484 | addllc->qp_mtu = min(link_new->path_mtu, | |
485 | link_new->peer_mtu); | |
486 | } | |
52bedf37 KG |
487 | /* send llc message */ |
488 | rc = smc_wr_tx_send(link, pend); | |
489 | return rc; | |
490 | } | |
491 | ||
492 | /* send DELETE LINK request or response */ | |
fbed3b37 KG |
493 | int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, |
494 | enum smc_llc_reqresp reqresp, bool orderly, | |
495 | u32 reason) | |
52bedf37 KG |
496 | { |
497 | struct smc_llc_msg_del_link *delllc; | |
498 | struct smc_wr_tx_pend_priv *pend; | |
499 | struct smc_wr_buf *wr_buf; | |
500 | int rc; | |
501 | ||
502 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
503 | if (rc) | |
504 | return rc; | |
505 | delllc = (struct smc_llc_msg_del_link *)wr_buf; | |
fbed3b37 KG |
506 | |
507 | memset(delllc, 0, sizeof(*delllc)); | |
508 | delllc->hd.common.type = SMC_LLC_DELETE_LINK; | |
509 | delllc->hd.length = sizeof(struct smc_llc_msg_del_link); | |
510 | if (reqresp == SMC_LLC_RESP) | |
511 | delllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
512 | if (orderly) | |
513 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
514 | if (link_del_id) | |
515 | delllc->link_num = link_del_id; | |
516 | else | |
517 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; | |
518 | delllc->reason = htonl(reason); | |
9bf9abea UB |
519 | /* send llc message */ |
520 | rc = smc_wr_tx_send(link, pend); | |
521 | return rc; | |
522 | } | |
523 | ||
d97935fa KG |
524 | /* send LLC test link request */ |
525 | static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) | |
313164da KG |
526 | { |
527 | struct smc_llc_msg_test_link *testllc; | |
528 | struct smc_wr_tx_pend_priv *pend; | |
529 | struct smc_wr_buf *wr_buf; | |
530 | int rc; | |
531 | ||
532 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
533 | if (rc) | |
534 | return rc; | |
535 | testllc = (struct smc_llc_msg_test_link *)wr_buf; | |
536 | memset(testllc, 0, sizeof(*testllc)); | |
537 | testllc->hd.common.type = SMC_LLC_TEST_LINK; | |
538 | testllc->hd.length = sizeof(struct smc_llc_msg_test_link); | |
313164da KG |
539 | memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); |
540 | /* send llc message */ | |
541 | rc = smc_wr_tx_send(link, pend); | |
542 | return rc; | |
543 | } | |
544 | ||
6c8968c4 KG |
545 | /* schedule an llc send on link, may wait for buffers */ |
546 | static int smc_llc_send_message(struct smc_link *link, void *llcbuf) | |
4ed75de5 KG |
547 | { |
548 | struct smc_wr_tx_pend_priv *pend; | |
549 | struct smc_wr_buf *wr_buf; | |
550 | int rc; | |
551 | ||
6c8968c4 KG |
552 | if (!smc_link_usable(link)) |
553 | return -ENOLINK; | |
554 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
4ed75de5 | 555 | if (rc) |
6c8968c4 KG |
556 | return rc; |
557 | memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); | |
558 | return smc_wr_tx_send(link, pend); | |
4ed75de5 KG |
559 | } |
560 | ||
9bf9abea UB |
561 | /********************************* receive ***********************************/ |
562 | ||
336ba09f KG |
563 | static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, |
564 | enum smc_lgr_type lgr_new_t) | |
565 | { | |
566 | int i; | |
567 | ||
568 | if (lgr->type == SMC_LGR_SYMMETRIC || | |
569 | (lgr->type != SMC_LGR_SINGLE && | |
570 | (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
571 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) | |
572 | return -EMLINK; | |
573 | ||
574 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
575 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { | |
576 | for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) | |
577 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
578 | return i; | |
579 | } else { | |
580 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) | |
581 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
582 | return i; | |
583 | } | |
584 | return -EMLINK; | |
585 | } | |
586 | ||
87f88cda KG |
587 | /* return first buffer from any of the next buf lists */ |
588 | static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
589 | int *buf_lst) | |
590 | { | |
591 | struct smc_buf_desc *buf_pos; | |
592 | ||
593 | while (*buf_lst < SMC_RMBE_SIZES) { | |
594 | buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], | |
595 | struct smc_buf_desc, list); | |
596 | if (buf_pos) | |
597 | return buf_pos; | |
598 | (*buf_lst)++; | |
599 | } | |
600 | return NULL; | |
601 | } | |
602 | ||
603 | /* return next rmb from buffer lists */ | |
604 | static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
605 | int *buf_lst, | |
606 | struct smc_buf_desc *buf_pos) | |
607 | { | |
608 | struct smc_buf_desc *buf_next; | |
609 | ||
610 | if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { | |
611 | (*buf_lst)++; | |
612 | return _smc_llc_get_next_rmb(lgr, buf_lst); | |
613 | } | |
614 | buf_next = list_next_entry(buf_pos, list); | |
615 | return buf_next; | |
616 | } | |
617 | ||
618 | static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, | |
619 | int *buf_lst) | |
620 | { | |
621 | *buf_lst = 0; | |
622 | return smc_llc_get_next_rmb(lgr, buf_lst, NULL); | |
623 | } | |
624 | ||
625 | /* send one add_link_continue msg */ | |
626 | static int smc_llc_add_link_cont(struct smc_link *link, | |
627 | struct smc_link *link_new, u8 *num_rkeys_todo, | |
628 | int *buf_lst, struct smc_buf_desc **buf_pos) | |
629 | { | |
630 | struct smc_llc_msg_add_link_cont *addc_llc; | |
631 | struct smc_link_group *lgr = link->lgr; | |
632 | int prim_lnk_idx, lnk_idx, i, rc; | |
633 | struct smc_wr_tx_pend_priv *pend; | |
634 | struct smc_wr_buf *wr_buf; | |
635 | struct smc_buf_desc *rmb; | |
636 | u8 n; | |
637 | ||
638 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
639 | if (rc) | |
640 | return rc; | |
641 | addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; | |
642 | memset(addc_llc, 0, sizeof(*addc_llc)); | |
643 | ||
644 | prim_lnk_idx = link->link_idx; | |
645 | lnk_idx = link_new->link_idx; | |
646 | addc_llc->link_num = link_new->link_id; | |
647 | addc_llc->num_rkeys = *num_rkeys_todo; | |
648 | n = *num_rkeys_todo; | |
649 | for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { | |
650 | if (!*buf_pos) { | |
651 | addc_llc->num_rkeys = addc_llc->num_rkeys - | |
652 | *num_rkeys_todo; | |
653 | *num_rkeys_todo = 0; | |
654 | break; | |
655 | } | |
656 | rmb = *buf_pos; | |
657 | ||
658 | addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); | |
659 | addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); | |
660 | addc_llc->rt[i].rmb_vaddr_new = | |
661 | cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); | |
662 | ||
663 | (*num_rkeys_todo)--; | |
664 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
665 | while (*buf_pos && !(*buf_pos)->used) | |
666 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
667 | } | |
668 | addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; | |
669 | addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); | |
670 | if (lgr->role == SMC_CLNT) | |
671 | addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
672 | return smc_wr_tx_send(link, pend); | |
673 | } | |
674 | ||
675 | static int smc_llc_cli_rkey_exchange(struct smc_link *link, | |
676 | struct smc_link *link_new) | |
677 | { | |
678 | struct smc_llc_msg_add_link_cont *addc_llc; | |
679 | struct smc_link_group *lgr = link->lgr; | |
680 | u8 max, num_rkeys_send, num_rkeys_recv; | |
681 | struct smc_llc_qentry *qentry; | |
682 | struct smc_buf_desc *buf_pos; | |
683 | int buf_lst; | |
684 | int rc = 0; | |
685 | int i; | |
686 | ||
687 | mutex_lock(&lgr->rmbs_lock); | |
688 | num_rkeys_send = lgr->conns_num; | |
689 | buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); | |
690 | do { | |
691 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, | |
692 | SMC_LLC_ADD_LINK_CONT); | |
693 | if (!qentry) { | |
694 | rc = -ETIMEDOUT; | |
695 | break; | |
696 | } | |
697 | addc_llc = &qentry->msg.add_link_cont; | |
698 | num_rkeys_recv = addc_llc->num_rkeys; | |
699 | max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); | |
700 | for (i = 0; i < max; i++) { | |
701 | smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, | |
702 | addc_llc->rt[i].rmb_key, | |
703 | addc_llc->rt[i].rmb_vaddr_new, | |
704 | addc_llc->rt[i].rmb_key_new); | |
705 | num_rkeys_recv--; | |
706 | } | |
707 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
708 | rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, | |
709 | &buf_lst, &buf_pos); | |
710 | if (rc) | |
711 | break; | |
712 | } while (num_rkeys_send || num_rkeys_recv); | |
713 | ||
714 | mutex_unlock(&lgr->rmbs_lock); | |
715 | return rc; | |
716 | } | |
717 | ||
336ba09f KG |
718 | /* prepare and send an add link reject response */ |
719 | static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) | |
720 | { | |
721 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; | |
722 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; | |
723 | qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; | |
724 | return smc_llc_send_message(qentry->link, &qentry->msg); | |
725 | } | |
726 | ||
b1570a87 KG |
727 | static int smc_llc_cli_conf_link(struct smc_link *link, |
728 | struct smc_init_info *ini, | |
729 | struct smc_link *link_new, | |
730 | enum smc_lgr_type lgr_new_t) | |
731 | { | |
732 | struct smc_link_group *lgr = link->lgr; | |
733 | struct smc_llc_msg_del_link *del_llc; | |
734 | struct smc_llc_qentry *qentry = NULL; | |
735 | int rc = 0; | |
736 | ||
737 | /* receive CONFIRM LINK request over RoCE fabric */ | |
738 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); | |
739 | if (!qentry) { | |
740 | rc = smc_llc_send_delete_link(link, link_new->link_id, | |
741 | SMC_LLC_REQ, false, | |
742 | SMC_LLC_DEL_LOST_PATH); | |
743 | return -ENOLINK; | |
744 | } | |
745 | if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { | |
746 | /* received DELETE_LINK instead */ | |
747 | del_llc = &qentry->msg.delete_link; | |
748 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; | |
749 | smc_llc_send_message(link, &qentry->msg); | |
750 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
751 | return -ENOLINK; | |
752 | } | |
753 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
754 | ||
755 | rc = smc_ib_modify_qp_rts(link_new); | |
756 | if (rc) { | |
757 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
758 | false, SMC_LLC_DEL_LOST_PATH); | |
759 | return -ENOLINK; | |
760 | } | |
761 | smc_wr_remember_qp_attr(link_new); | |
762 | ||
763 | rc = smcr_buf_reg_lgr(link_new); | |
764 | if (rc) { | |
765 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
766 | false, SMC_LLC_DEL_LOST_PATH); | |
767 | return -ENOLINK; | |
768 | } | |
769 | ||
770 | /* send CONFIRM LINK response over RoCE fabric */ | |
771 | rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); | |
772 | if (rc) { | |
773 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
774 | false, SMC_LLC_DEL_LOST_PATH); | |
775 | return -ENOLINK; | |
776 | } | |
777 | smc_llc_link_active(link_new); | |
778 | lgr->type = lgr_new_t; | |
779 | return 0; | |
780 | } | |
781 | ||
336ba09f KG |
782 | static void smc_llc_save_add_link_info(struct smc_link *link, |
783 | struct smc_llc_msg_add_link *add_llc) | |
784 | { | |
785 | link->peer_qpn = ntoh24(add_llc->sender_qp_num); | |
786 | memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); | |
787 | memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); | |
788 | link->peer_psn = ntoh24(add_llc->initial_psn); | |
789 | link->peer_mtu = add_llc->qp_mtu; | |
790 | } | |
791 | ||
792 | /* as an SMC client, process an add link request */ | |
793 | int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) | |
794 | { | |
795 | struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; | |
796 | enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; | |
797 | struct smc_link_group *lgr = smc_get_lgr(link); | |
798 | struct smc_link *lnk_new = NULL; | |
799 | struct smc_init_info ini; | |
800 | int lnk_idx, rc = 0; | |
801 | ||
802 | ini.vlan_id = lgr->vlan_id; | |
803 | smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); | |
804 | if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && | |
805 | !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { | |
806 | if (!ini.ib_dev) | |
807 | goto out_reject; | |
808 | lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; | |
809 | } | |
810 | if (!ini.ib_dev) { | |
811 | lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; | |
812 | ini.ib_dev = link->smcibdev; | |
813 | ini.ib_port = link->ibport; | |
814 | } | |
815 | lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); | |
816 | if (lnk_idx < 0) | |
817 | goto out_reject; | |
818 | lnk_new = &lgr->lnk[lnk_idx]; | |
819 | rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); | |
820 | if (rc) | |
821 | goto out_reject; | |
822 | smc_llc_save_add_link_info(lnk_new, llc); | |
823 | lnk_new->link_id = llc->link_num; | |
824 | ||
825 | rc = smc_ib_ready_link(lnk_new); | |
826 | if (rc) | |
827 | goto out_clear_lnk; | |
828 | ||
829 | rc = smcr_buf_map_lgr(lnk_new); | |
830 | if (rc) | |
831 | goto out_clear_lnk; | |
832 | ||
833 | rc = smc_llc_send_add_link(link, | |
834 | lnk_new->smcibdev->mac[ini.ib_port - 1], | |
835 | lnk_new->gid, lnk_new, SMC_LLC_RESP); | |
836 | if (rc) | |
837 | goto out_clear_lnk; | |
87f88cda | 838 | rc = smc_llc_cli_rkey_exchange(link, lnk_new); |
336ba09f KG |
839 | if (rc) { |
840 | rc = 0; | |
841 | goto out_clear_lnk; | |
842 | } | |
b1570a87 | 843 | rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); |
336ba09f KG |
844 | if (!rc) |
845 | goto out; | |
846 | out_clear_lnk: | |
847 | smcr_link_clear(lnk_new); | |
848 | out_reject: | |
849 | smc_llc_cli_add_link_reject(qentry); | |
850 | out: | |
851 | kfree(qentry); | |
852 | return rc; | |
853 | } | |
854 | ||
b1570a87 KG |
855 | static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) |
856 | { | |
857 | struct smc_llc_qentry *qentry; | |
858 | ||
859 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
860 | ||
861 | mutex_lock(&lgr->llc_conf_mutex); | |
862 | smc_llc_cli_add_link(qentry->link, qentry); | |
863 | mutex_unlock(&lgr->llc_conf_mutex); | |
864 | } | |
865 | ||
b45e7f98 KG |
866 | /* worker to process an add link message */ |
867 | static void smc_llc_add_link_work(struct work_struct *work) | |
868 | { | |
869 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
870 | llc_add_link_work); | |
871 | ||
872 | if (list_empty(&lgr->list)) { | |
873 | /* link group is terminating */ | |
874 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
875 | goto out; | |
876 | } | |
877 | ||
b1570a87 KG |
878 | if (lgr->role == SMC_CLNT) |
879 | smc_llc_process_cli_add_link(lgr); | |
b45e7f98 KG |
880 | /* tbd: call smc_llc_process_srv_add_link(lgr); */ |
881 | out: | |
882 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); | |
883 | } | |
884 | ||
52bedf37 KG |
885 | static void smc_llc_rx_delete_link(struct smc_link *link, |
886 | struct smc_llc_msg_del_link *llc) | |
887 | { | |
00e5fb26 | 888 | struct smc_link_group *lgr = smc_get_lgr(link); |
52bedf37 | 889 | |
ef79d439 | 890 | smc_lgr_forget(lgr); |
ef79d439 KG |
891 | if (lgr->role == SMC_SERV) { |
892 | /* client asks to delete this link, send request */ | |
fbed3b37 KG |
893 | smc_llc_send_delete_link(link, 0, SMC_LLC_REQ, true, |
894 | SMC_LLC_DEL_PROG_INIT_TERM); | |
52bedf37 | 895 | } else { |
ef79d439 | 896 | /* server requests to delete this link, send response */ |
fbed3b37 KG |
897 | smc_llc_send_delete_link(link, 0, SMC_LLC_RESP, true, |
898 | SMC_LLC_DEL_PROG_INIT_TERM); | |
52bedf37 | 899 | } |
87523930 | 900 | smcr_link_down_cond(link); |
52bedf37 KG |
901 | } |
902 | ||
3bc67e09 KG |
903 | /* process a confirm_rkey request from peer, remote flow */ |
904 | static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 905 | { |
3bc67e09 KG |
906 | struct smc_llc_msg_confirm_rkey *llc; |
907 | struct smc_llc_qentry *qentry; | |
908 | struct smc_link *link; | |
909 | int num_entries; | |
910 | int rk_idx; | |
911 | int i; | |
912 | ||
913 | qentry = lgr->llc_flow_rmt.qentry; | |
914 | llc = &qentry->msg.confirm_rkey; | |
915 | link = qentry->link; | |
916 | ||
917 | num_entries = llc->rtoken[0].num_rkeys; | |
918 | /* first rkey entry is for receiving link */ | |
919 | rk_idx = smc_rtoken_add(link, | |
920 | llc->rtoken[0].rmb_vaddr, | |
921 | llc->rtoken[0].rmb_key); | |
922 | if (rk_idx < 0) | |
923 | goto out_err; | |
924 | ||
925 | for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) | |
926 | smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, | |
927 | llc->rtoken[i].rmb_vaddr, | |
928 | llc->rtoken[i].rmb_key); | |
929 | /* max links is 3 so there is no need to support conf_rkey_cont msgs */ | |
930 | goto out; | |
931 | out_err: | |
932 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
933 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; | |
934 | out: | |
ef79d439 | 935 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
3bc67e09 KG |
936 | smc_llc_send_message(link, &qentry->msg); |
937 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
4ed75de5 KG |
938 | } |
939 | ||
218b24fe KG |
940 | /* process a delete_rkey request from peer, remote flow */ |
941 | static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 942 | { |
218b24fe KG |
943 | struct smc_llc_msg_delete_rkey *llc; |
944 | struct smc_llc_qentry *qentry; | |
945 | struct smc_link *link; | |
4ed75de5 KG |
946 | u8 err_mask = 0; |
947 | int i, max; | |
948 | ||
218b24fe KG |
949 | qentry = lgr->llc_flow_rmt.qentry; |
950 | llc = &qentry->msg.delete_rkey; | |
951 | link = qentry->link; | |
952 | ||
ef79d439 KG |
953 | max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); |
954 | for (i = 0; i < max; i++) { | |
955 | if (smc_rtoken_delete(link, llc->rkey[i])) | |
956 | err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); | |
957 | } | |
ef79d439 KG |
958 | if (err_mask) { |
959 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
960 | llc->err_mask = err_mask; | |
4ed75de5 | 961 | } |
218b24fe KG |
962 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
963 | smc_llc_send_message(link, &qentry->msg); | |
964 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
965 | } | |
ef79d439 | 966 | |
6c8968c4 | 967 | /* flush the llc event queue */ |
00a049cf | 968 | static void smc_llc_event_flush(struct smc_link_group *lgr) |
9bf9abea | 969 | { |
6c8968c4 KG |
970 | struct smc_llc_qentry *qentry, *q; |
971 | ||
972 | spin_lock_bh(&lgr->llc_event_q_lock); | |
973 | list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { | |
974 | list_del_init(&qentry->list); | |
975 | kfree(qentry); | |
976 | } | |
977 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
978 | } | |
979 | ||
980 | static void smc_llc_event_handler(struct smc_llc_qentry *qentry) | |
981 | { | |
982 | union smc_llc_msg *llc = &qentry->msg; | |
983 | struct smc_link *link = qentry->link; | |
0fb0b02b | 984 | struct smc_link_group *lgr = link->lgr; |
9bf9abea | 985 | |
d854fcbf | 986 | if (!smc_link_usable(link)) |
6c8968c4 | 987 | goto out; |
313164da KG |
988 | |
989 | switch (llc->raw.hdr.common.type) { | |
990 | case SMC_LLC_TEST_LINK: | |
56e8091c KG |
991 | llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; |
992 | smc_llc_send_message(link, llc); | |
313164da | 993 | break; |
52bedf37 | 994 | case SMC_LLC_ADD_LINK: |
0fb0b02b KG |
995 | if (list_empty(&lgr->list)) |
996 | goto out; /* lgr is terminating */ | |
997 | if (lgr->role == SMC_CLNT) { | |
998 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { | |
999 | /* a flow is waiting for this message */ | |
1000 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1001 | qentry); | |
1002 | wake_up_interruptible(&lgr->llc_waiter); | |
1003 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, | |
1004 | qentry)) { | |
b45e7f98 | 1005 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1006 | } |
1007 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { | |
1008 | /* as smc server, handle client suggestion */ | |
b45e7f98 | 1009 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1010 | } |
1011 | return; | |
1012 | case SMC_LLC_CONFIRM_LINK: | |
87f88cda | 1013 | case SMC_LLC_ADD_LINK_CONT: |
0fb0b02b KG |
1014 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { |
1015 | /* a flow is waiting for this message */ | |
1016 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); | |
1017 | wake_up_interruptible(&lgr->llc_waiter); | |
1018 | return; | |
1019 | } | |
52bedf37 KG |
1020 | break; |
1021 | case SMC_LLC_DELETE_LINK: | |
1022 | smc_llc_rx_delete_link(link, &llc->delete_link); | |
1023 | break; | |
4ed75de5 | 1024 | case SMC_LLC_CONFIRM_RKEY: |
3bc67e09 KG |
1025 | /* new request from remote, assign to remote flow */ |
1026 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1027 | /* process here, does not wait for more llc msgs */ | |
1028 | smc_llc_rmt_conf_rkey(lgr); | |
1029 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1030 | } | |
1031 | return; | |
4ed75de5 | 1032 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc KG |
1033 | /* not used because max links is 3, and 3 rkeys fit into |
1034 | * one CONFIRM_RKEY message | |
1035 | */ | |
4ed75de5 KG |
1036 | break; |
1037 | case SMC_LLC_DELETE_RKEY: | |
218b24fe KG |
1038 | /* new request from remote, assign to remote flow */ |
1039 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1040 | /* process here, does not wait for more llc msgs */ | |
1041 | smc_llc_rmt_delete_rkey(lgr); | |
1042 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1043 | } | |
1044 | return; | |
313164da | 1045 | } |
6c8968c4 KG |
1046 | out: |
1047 | kfree(qentry); | |
1048 | } | |
1049 | ||
1050 | /* worker to process llc messages on the event queue */ | |
1051 | static void smc_llc_event_work(struct work_struct *work) | |
1052 | { | |
1053 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
1054 | llc_event_work); | |
1055 | struct smc_llc_qentry *qentry; | |
1056 | ||
555da9af KG |
1057 | if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { |
1058 | if (smc_link_usable(lgr->delayed_event->link)) { | |
1059 | smc_llc_event_handler(lgr->delayed_event); | |
1060 | } else { | |
1061 | qentry = lgr->delayed_event; | |
1062 | lgr->delayed_event = NULL; | |
1063 | kfree(qentry); | |
1064 | } | |
1065 | } | |
1066 | ||
6c8968c4 KG |
1067 | again: |
1068 | spin_lock_bh(&lgr->llc_event_q_lock); | |
1069 | if (!list_empty(&lgr->llc_event_q)) { | |
1070 | qentry = list_first_entry(&lgr->llc_event_q, | |
1071 | struct smc_llc_qentry, list); | |
1072 | list_del_init(&qentry->list); | |
1073 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1074 | smc_llc_event_handler(qentry); | |
1075 | goto again; | |
1076 | } | |
1077 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1078 | } | |
1079 | ||
ef79d439 | 1080 | /* process llc responses in tasklet context */ |
a6688d91 KG |
1081 | static void smc_llc_rx_response(struct smc_link *link, |
1082 | struct smc_llc_qentry *qentry) | |
ef79d439 | 1083 | { |
a6688d91 | 1084 | u8 llc_type = qentry->msg.raw.hdr.common.type; |
ef79d439 | 1085 | |
a6688d91 | 1086 | switch (llc_type) { |
ef79d439 KG |
1087 | case SMC_LLC_TEST_LINK: |
1088 | if (link->state == SMC_LNK_ACTIVE) | |
1089 | complete(&link->llc_testlink_resp); | |
1090 | break; | |
ef79d439 | 1091 | case SMC_LLC_ADD_LINK: |
4667bb4a | 1092 | case SMC_LLC_CONFIRM_LINK: |
87f88cda | 1093 | case SMC_LLC_ADD_LINK_CONT: |
3d88a21b | 1094 | case SMC_LLC_CONFIRM_RKEY: |
6d74c3a8 | 1095 | case SMC_LLC_DELETE_RKEY: |
4667bb4a KG |
1096 | /* assign responses to the local flow, we requested them */ |
1097 | smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); | |
1098 | wake_up_interruptible(&link->lgr->llc_waiter); | |
1099 | return; | |
ef79d439 KG |
1100 | case SMC_LLC_DELETE_LINK: |
1101 | if (link->lgr->role == SMC_SERV) | |
1102 | smc_lgr_schedule_free_work_fast(link->lgr); | |
1103 | break; | |
ef79d439 | 1104 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc | 1105 | /* not used because max links is 3 */ |
ef79d439 | 1106 | break; |
ef79d439 | 1107 | } |
a6688d91 | 1108 | kfree(qentry); |
ef79d439 KG |
1109 | } |
1110 | ||
a6688d91 | 1111 | static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) |
6c8968c4 | 1112 | { |
6c8968c4 KG |
1113 | struct smc_link_group *lgr = link->lgr; |
1114 | struct smc_llc_qentry *qentry; | |
6c8968c4 KG |
1115 | unsigned long flags; |
1116 | ||
6c8968c4 KG |
1117 | qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); |
1118 | if (!qentry) | |
1119 | return; | |
1120 | qentry->link = link; | |
1121 | INIT_LIST_HEAD(&qentry->list); | |
1122 | memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); | |
a6688d91 KG |
1123 | |
1124 | /* process responses immediately */ | |
1125 | if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { | |
1126 | smc_llc_rx_response(link, qentry); | |
1127 | return; | |
1128 | } | |
1129 | ||
1130 | /* add requests to event queue */ | |
6c8968c4 KG |
1131 | spin_lock_irqsave(&lgr->llc_event_q_lock, flags); |
1132 | list_add_tail(&qentry->list, &lgr->llc_event_q); | |
1133 | spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); | |
1134 | schedule_work(&link->lgr->llc_event_work); | |
9bf9abea UB |
1135 | } |
1136 | ||
a6688d91 KG |
1137 | /* copy received msg and add it to the event queue */ |
1138 | static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) | |
1139 | { | |
1140 | struct smc_link *link = (struct smc_link *)wc->qp->qp_context; | |
1141 | union smc_llc_msg *llc = buf; | |
1142 | ||
1143 | if (wc->byte_len < sizeof(*llc)) | |
1144 | return; /* short message */ | |
1145 | if (llc->raw.hdr.length != sizeof(*llc)) | |
1146 | return; /* invalid message */ | |
1147 | ||
1148 | smc_llc_enqueue(link, llc); | |
1149 | } | |
1150 | ||
44aa81ce | 1151 | /***************************** worker, utils *********************************/ |
877ae5be KG |
1152 | |
1153 | static void smc_llc_testlink_work(struct work_struct *work) | |
1154 | { | |
1155 | struct smc_link *link = container_of(to_delayed_work(work), | |
1156 | struct smc_link, llc_testlink_wrk); | |
1157 | unsigned long next_interval; | |
877ae5be KG |
1158 | unsigned long expire_time; |
1159 | u8 user_data[16] = { 0 }; | |
1160 | int rc; | |
1161 | ||
877ae5be KG |
1162 | if (link->state != SMC_LNK_ACTIVE) |
1163 | return; /* don't reschedule worker */ | |
1164 | expire_time = link->wr_rx_tstamp + link->llc_testlink_time; | |
1165 | if (time_is_after_jiffies(expire_time)) { | |
1166 | next_interval = expire_time - jiffies; | |
1167 | goto out; | |
1168 | } | |
1169 | reinit_completion(&link->llc_testlink_resp); | |
d97935fa | 1170 | smc_llc_send_test_link(link, user_data); |
877ae5be KG |
1171 | /* receive TEST LINK response over RoCE fabric */ |
1172 | rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, | |
1173 | SMC_LLC_WAIT_TIME); | |
1020e1ef KG |
1174 | if (link->state != SMC_LNK_ACTIVE) |
1175 | return; /* link state changed */ | |
877ae5be | 1176 | if (rc <= 0) { |
87523930 | 1177 | smcr_link_down_cond_sched(link); |
877ae5be KG |
1178 | return; |
1179 | } | |
1180 | next_interval = link->llc_testlink_time; | |
1181 | out: | |
1020e1ef | 1182 | schedule_delayed_work(&link->llc_testlink_wrk, next_interval); |
877ae5be KG |
1183 | } |
1184 | ||
00a049cf KG |
1185 | void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) |
1186 | { | |
1187 | struct net *net = sock_net(smc->clcsock->sk); | |
1188 | ||
1189 | INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); | |
b45e7f98 | 1190 | INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); |
00a049cf KG |
1191 | INIT_LIST_HEAD(&lgr->llc_event_q); |
1192 | spin_lock_init(&lgr->llc_event_q_lock); | |
555da9af KG |
1193 | spin_lock_init(&lgr->llc_flow_lock); |
1194 | init_waitqueue_head(&lgr->llc_waiter); | |
d5500667 | 1195 | mutex_init(&lgr->llc_conf_mutex); |
00a049cf KG |
1196 | lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; |
1197 | } | |
1198 | ||
1199 | /* called after lgr was removed from lgr_list */ | |
1200 | void smc_llc_lgr_clear(struct smc_link_group *lgr) | |
1201 | { | |
1202 | smc_llc_event_flush(lgr); | |
555da9af | 1203 | wake_up_interruptible_all(&lgr->llc_waiter); |
00a049cf | 1204 | cancel_work_sync(&lgr->llc_event_work); |
b45e7f98 | 1205 | cancel_work_sync(&lgr->llc_add_link_work); |
555da9af KG |
1206 | if (lgr->delayed_event) { |
1207 | kfree(lgr->delayed_event); | |
1208 | lgr->delayed_event = NULL; | |
1209 | } | |
00a049cf KG |
1210 | } |
1211 | ||
2a4c57a9 | 1212 | int smc_llc_link_init(struct smc_link *link) |
877ae5be KG |
1213 | { |
1214 | init_completion(&link->llc_testlink_resp); | |
1215 | INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); | |
2a4c57a9 | 1216 | return 0; |
b32cf4ab KG |
1217 | } |
1218 | ||
00a049cf | 1219 | void smc_llc_link_active(struct smc_link *link) |
b32cf4ab | 1220 | { |
877ae5be | 1221 | link->state = SMC_LNK_ACTIVE; |
00a049cf KG |
1222 | if (link->lgr->llc_testlink_time) { |
1223 | link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; | |
1020e1ef KG |
1224 | schedule_delayed_work(&link->llc_testlink_wrk, |
1225 | link->llc_testlink_time); | |
877ae5be KG |
1226 | } |
1227 | } | |
1228 | ||
877ae5be | 1229 | /* called in worker context */ |
2a4c57a9 | 1230 | void smc_llc_link_clear(struct smc_link *link) |
877ae5be | 1231 | { |
2140ac26 KG |
1232 | complete(&link->llc_testlink_resp); |
1233 | cancel_delayed_work_sync(&link->llc_testlink_wrk); | |
1234 | smc_wr_wakeup_reg_wait(link); | |
1235 | smc_wr_wakeup_tx_wait(link); | |
877ae5be KG |
1236 | } |
1237 | ||
3d88a21b KG |
1238 | /* register a new rtoken at the remote peer (for all links) */ |
1239 | int smc_llc_do_confirm_rkey(struct smc_link *send_link, | |
44aa81ce KG |
1240 | struct smc_buf_desc *rmb_desc) |
1241 | { | |
3d88a21b KG |
1242 | struct smc_link_group *lgr = send_link->lgr; |
1243 | struct smc_llc_qentry *qentry = NULL; | |
1244 | int rc = 0; | |
44aa81ce | 1245 | |
3d88a21b KG |
1246 | rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); |
1247 | if (rc) | |
1248 | goto out; | |
44aa81ce | 1249 | /* receive CONFIRM RKEY response from server over RoCE fabric */ |
3d88a21b KG |
1250 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1251 | SMC_LLC_CONFIRM_RKEY); | |
1252 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
1253 | rc = -EFAULT; | |
1254 | out: | |
1255 | if (qentry) | |
1256 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
3d88a21b | 1257 | return rc; |
44aa81ce KG |
1258 | } |
1259 | ||
60e03c62 | 1260 | /* unregister an rtoken at the remote peer */ |
6d74c3a8 | 1261 | int smc_llc_do_delete_rkey(struct smc_link_group *lgr, |
60e03c62 KG |
1262 | struct smc_buf_desc *rmb_desc) |
1263 | { | |
6d74c3a8 KG |
1264 | struct smc_llc_qentry *qentry = NULL; |
1265 | struct smc_link *send_link; | |
0b29ec64 | 1266 | int rc = 0; |
60e03c62 | 1267 | |
6d74c3a8 KG |
1268 | send_link = smc_llc_usable_link(lgr); |
1269 | if (!send_link) | |
1270 | return -ENOLINK; | |
1271 | ||
6d74c3a8 KG |
1272 | /* protected by llc_flow control */ |
1273 | rc = smc_llc_send_delete_rkey(send_link, rmb_desc); | |
60e03c62 KG |
1274 | if (rc) |
1275 | goto out; | |
1276 | /* receive DELETE RKEY response from server over RoCE fabric */ | |
6d74c3a8 KG |
1277 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1278 | SMC_LLC_DELETE_RKEY); | |
1279 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
60e03c62 | 1280 | rc = -EFAULT; |
60e03c62 | 1281 | out: |
6d74c3a8 KG |
1282 | if (qentry) |
1283 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
60e03c62 KG |
1284 | return rc; |
1285 | } | |
1286 | ||
92334cfc KG |
1287 | /* evaluate confirm link request or response */ |
1288 | int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, | |
1289 | enum smc_llc_reqresp type) | |
1290 | { | |
1291 | if (type == SMC_LLC_REQ) /* SMC server assigns link_id */ | |
1292 | qentry->link->link_id = qentry->msg.confirm_link.link_num; | |
1293 | if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) | |
1294 | return -ENOTSUPP; | |
1295 | return 0; | |
1296 | } | |
1297 | ||
9bf9abea UB |
1298 | /***************************** init, exit, misc ******************************/ |
1299 | ||
1300 | static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { | |
1301 | { | |
1302 | .handler = smc_llc_rx_handler, | |
1303 | .type = SMC_LLC_CONFIRM_LINK | |
1304 | }, | |
313164da KG |
1305 | { |
1306 | .handler = smc_llc_rx_handler, | |
1307 | .type = SMC_LLC_TEST_LINK | |
1308 | }, | |
52bedf37 KG |
1309 | { |
1310 | .handler = smc_llc_rx_handler, | |
1311 | .type = SMC_LLC_ADD_LINK | |
1312 | }, | |
87f88cda KG |
1313 | { |
1314 | .handler = smc_llc_rx_handler, | |
1315 | .type = SMC_LLC_ADD_LINK_CONT | |
1316 | }, | |
52bedf37 KG |
1317 | { |
1318 | .handler = smc_llc_rx_handler, | |
1319 | .type = SMC_LLC_DELETE_LINK | |
1320 | }, | |
4ed75de5 KG |
1321 | { |
1322 | .handler = smc_llc_rx_handler, | |
1323 | .type = SMC_LLC_CONFIRM_RKEY | |
1324 | }, | |
1325 | { | |
1326 | .handler = smc_llc_rx_handler, | |
1327 | .type = SMC_LLC_CONFIRM_RKEY_CONT | |
1328 | }, | |
1329 | { | |
1330 | .handler = smc_llc_rx_handler, | |
1331 | .type = SMC_LLC_DELETE_RKEY | |
1332 | }, | |
9bf9abea UB |
1333 | { |
1334 | .handler = NULL, | |
1335 | } | |
1336 | }; | |
1337 | ||
1338 | int __init smc_llc_init(void) | |
1339 | { | |
1340 | struct smc_wr_rx_handler *handler; | |
1341 | int rc = 0; | |
1342 | ||
1343 | for (handler = smc_llc_rx_handlers; handler->handler; handler++) { | |
1344 | INIT_HLIST_NODE(&handler->list); | |
1345 | rc = smc_wr_rx_register_handler(handler); | |
1346 | if (rc) | |
1347 | break; | |
1348 | } | |
1349 | return rc; | |
1350 | } |