Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9bf9abea UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Link Layer Control (LLC) | |
6 | * | |
9bf9abea UB |
7 | * Copyright IBM Corp. 2016 |
8 | * | |
9 | * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> | |
10 | * Ursula Braun <ubraun@linux.vnet.ibm.com> | |
11 | */ | |
12 | ||
13 | #include <net/tcp.h> | |
14 | #include <rdma/ib_verbs.h> | |
15 | ||
16 | #include "smc.h" | |
17 | #include "smc_core.h" | |
18 | #include "smc_clc.h" | |
19 | #include "smc_llc.h" | |
336ba09f | 20 | #include "smc_pnet.h" |
9bf9abea | 21 | |
0f627126 SR |
22 | #define SMC_LLC_DATA_LEN 40 |
23 | ||
24 | struct smc_llc_hdr { | |
25 | struct smc_wr_rx_hdr common; | |
26 | u8 length; /* 44 */ | |
52bedf37 KG |
27 | #if defined(__BIG_ENDIAN_BITFIELD) |
28 | u8 reserved:4, | |
29 | add_link_rej_rsn:4; | |
30 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
31 | u8 add_link_rej_rsn:4, | |
32 | reserved:4; | |
33 | #endif | |
0f627126 SR |
34 | u8 flags; |
35 | }; | |
36 | ||
75d320d6 KG |
37 | #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 |
38 | ||
0f627126 SR |
39 | struct smc_llc_msg_confirm_link { /* type 0x01 */ |
40 | struct smc_llc_hdr hd; | |
41 | u8 sender_mac[ETH_ALEN]; | |
42 | u8 sender_gid[SMC_GID_SIZE]; | |
43 | u8 sender_qp_num[3]; | |
44 | u8 link_num; | |
45 | u8 link_uid[SMC_LGR_ID_SIZE]; | |
46 | u8 max_links; | |
47 | u8 reserved[9]; | |
48 | }; | |
49 | ||
52bedf37 KG |
50 | #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 |
51 | #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 | |
52 | ||
53 | #define SMC_LLC_ADD_LNK_MAX_LINKS 2 | |
54 | ||
55 | struct smc_llc_msg_add_link { /* type 0x02 */ | |
56 | struct smc_llc_hdr hd; | |
57 | u8 sender_mac[ETH_ALEN]; | |
58 | u8 reserved2[2]; | |
59 | u8 sender_gid[SMC_GID_SIZE]; | |
60 | u8 sender_qp_num[3]; | |
61 | u8 link_num; | |
fbed3b37 KG |
62 | #if defined(__BIG_ENDIAN_BITFIELD) |
63 | u8 reserved3 : 4, | |
64 | qp_mtu : 4; | |
65 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
66 | u8 qp_mtu : 4, | |
67 | reserved3 : 4; | |
68 | #endif | |
52bedf37 KG |
69 | u8 initial_psn[3]; |
70 | u8 reserved[8]; | |
71 | }; | |
72 | ||
87f88cda KG |
73 | struct smc_llc_msg_add_link_cont_rt { |
74 | __be32 rmb_key; | |
75 | __be32 rmb_key_new; | |
76 | __be64 rmb_vaddr_new; | |
77 | }; | |
78 | ||
79 | #define SMC_LLC_RKEYS_PER_CONT_MSG 2 | |
80 | ||
81 | struct smc_llc_msg_add_link_cont { /* type 0x03 */ | |
82 | struct smc_llc_hdr hd; | |
83 | u8 link_num; | |
84 | u8 num_rkeys; | |
85 | u8 reserved2[2]; | |
86 | struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; | |
87 | u8 reserved[4]; | |
88 | } __packed; /* format defined in RFC7609 */ | |
89 | ||
52bedf37 KG |
90 | #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 |
91 | #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 | |
92 | ||
93 | struct smc_llc_msg_del_link { /* type 0x04 */ | |
94 | struct smc_llc_hdr hd; | |
95 | u8 link_num; | |
96 | __be32 reason; | |
97 | u8 reserved[35]; | |
98 | } __packed; /* format defined in RFC7609 */ | |
99 | ||
313164da KG |
100 | struct smc_llc_msg_test_link { /* type 0x07 */ |
101 | struct smc_llc_hdr hd; | |
102 | u8 user_data[16]; | |
103 | u8 reserved[24]; | |
104 | }; | |
105 | ||
4ed75de5 KG |
106 | struct smc_rmb_rtoken { |
107 | union { | |
108 | u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ | |
109 | /* is actually the num of rtokens, first */ | |
110 | /* rtoken is always for the current link */ | |
111 | u8 link_id; /* link id of the rtoken */ | |
112 | }; | |
113 | __be32 rmb_key; | |
114 | __be64 rmb_vaddr; | |
115 | } __packed; /* format defined in RFC7609 */ | |
116 | ||
117 | #define SMC_LLC_RKEYS_PER_MSG 3 | |
118 | ||
119 | struct smc_llc_msg_confirm_rkey { /* type 0x06 */ | |
120 | struct smc_llc_hdr hd; | |
121 | struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; | |
122 | u8 reserved; | |
123 | }; | |
124 | ||
4ed75de5 | 125 | #define SMC_LLC_DEL_RKEY_MAX 8 |
3bc67e09 | 126 | #define SMC_LLC_FLAG_RKEY_RETRY 0x10 |
4ed75de5 KG |
127 | #define SMC_LLC_FLAG_RKEY_NEG 0x20 |
128 | ||
129 | struct smc_llc_msg_delete_rkey { /* type 0x09 */ | |
130 | struct smc_llc_hdr hd; | |
131 | u8 num_rkeys; | |
132 | u8 err_mask; | |
133 | u8 reserved[2]; | |
134 | __be32 rkey[8]; | |
135 | u8 reserved2[4]; | |
136 | }; | |
137 | ||
0f627126 SR |
138 | union smc_llc_msg { |
139 | struct smc_llc_msg_confirm_link confirm_link; | |
52bedf37 | 140 | struct smc_llc_msg_add_link add_link; |
87f88cda | 141 | struct smc_llc_msg_add_link_cont add_link_cont; |
52bedf37 | 142 | struct smc_llc_msg_del_link delete_link; |
4ed75de5 KG |
143 | |
144 | struct smc_llc_msg_confirm_rkey confirm_rkey; | |
4ed75de5 KG |
145 | struct smc_llc_msg_delete_rkey delete_rkey; |
146 | ||
313164da | 147 | struct smc_llc_msg_test_link test_link; |
0f627126 SR |
148 | struct { |
149 | struct smc_llc_hdr hdr; | |
150 | u8 data[SMC_LLC_DATA_LEN]; | |
151 | } raw; | |
152 | }; | |
153 | ||
154 | #define SMC_LLC_FLAG_RESP 0x80 | |
155 | ||
6c8968c4 KG |
156 | struct smc_llc_qentry { |
157 | struct list_head list; | |
158 | struct smc_link *link; | |
159 | union smc_llc_msg msg; | |
160 | }; | |
161 | ||
555da9af KG |
162 | struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) |
163 | { | |
164 | struct smc_llc_qentry *qentry = flow->qentry; | |
165 | ||
166 | flow->qentry = NULL; | |
167 | return qentry; | |
168 | } | |
169 | ||
170 | void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) | |
171 | { | |
172 | struct smc_llc_qentry *qentry; | |
173 | ||
174 | if (flow->qentry) { | |
175 | qentry = flow->qentry; | |
176 | flow->qentry = NULL; | |
177 | kfree(qentry); | |
178 | } | |
179 | } | |
180 | ||
181 | static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, | |
182 | struct smc_llc_qentry *qentry) | |
183 | { | |
184 | flow->qentry = qentry; | |
185 | } | |
186 | ||
187 | /* try to start a new llc flow, initiated by an incoming llc msg */ | |
188 | static bool smc_llc_flow_start(struct smc_llc_flow *flow, | |
189 | struct smc_llc_qentry *qentry) | |
190 | { | |
191 | struct smc_link_group *lgr = qentry->link->lgr; | |
192 | ||
193 | spin_lock_bh(&lgr->llc_flow_lock); | |
194 | if (flow->type) { | |
195 | /* a flow is already active */ | |
196 | if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK || | |
197 | qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) && | |
198 | !lgr->delayed_event) { | |
199 | lgr->delayed_event = qentry; | |
200 | } else { | |
201 | /* forget this llc request */ | |
202 | kfree(qentry); | |
203 | } | |
204 | spin_unlock_bh(&lgr->llc_flow_lock); | |
205 | return false; | |
206 | } | |
207 | switch (qentry->msg.raw.hdr.common.type) { | |
208 | case SMC_LLC_ADD_LINK: | |
209 | flow->type = SMC_LLC_FLOW_ADD_LINK; | |
210 | break; | |
211 | case SMC_LLC_DELETE_LINK: | |
212 | flow->type = SMC_LLC_FLOW_DEL_LINK; | |
213 | break; | |
214 | case SMC_LLC_CONFIRM_RKEY: | |
215 | case SMC_LLC_DELETE_RKEY: | |
216 | flow->type = SMC_LLC_FLOW_RKEY; | |
217 | break; | |
218 | default: | |
219 | flow->type = SMC_LLC_FLOW_NONE; | |
220 | } | |
221 | if (qentry == lgr->delayed_event) | |
222 | lgr->delayed_event = NULL; | |
223 | spin_unlock_bh(&lgr->llc_flow_lock); | |
224 | smc_llc_flow_qentry_set(flow, qentry); | |
225 | return true; | |
226 | } | |
227 | ||
228 | /* start a new local llc flow, wait till current flow finished */ | |
229 | int smc_llc_flow_initiate(struct smc_link_group *lgr, | |
230 | enum smc_llc_flowtype type) | |
231 | { | |
232 | enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; | |
233 | int rc; | |
234 | ||
235 | /* all flows except confirm_rkey and delete_rkey are exclusive, | |
236 | * confirm/delete rkey flows can run concurrently (local and remote) | |
237 | */ | |
238 | if (type == SMC_LLC_FLOW_RKEY) | |
239 | allowed_remote = SMC_LLC_FLOW_RKEY; | |
240 | again: | |
241 | if (list_empty(&lgr->list)) | |
242 | return -ENODEV; | |
243 | spin_lock_bh(&lgr->llc_flow_lock); | |
244 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
245 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
246 | lgr->llc_flow_rmt.type == allowed_remote)) { | |
247 | lgr->llc_flow_lcl.type = type; | |
248 | spin_unlock_bh(&lgr->llc_flow_lock); | |
249 | return 0; | |
250 | } | |
251 | spin_unlock_bh(&lgr->llc_flow_lock); | |
252 | rc = wait_event_interruptible_timeout(lgr->llc_waiter, | |
253 | (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
254 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
255 | lgr->llc_flow_rmt.type == allowed_remote)), | |
256 | SMC_LLC_WAIT_TIME); | |
257 | if (!rc) | |
258 | return -ETIMEDOUT; | |
259 | goto again; | |
260 | } | |
261 | ||
262 | /* finish the current llc flow */ | |
263 | void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) | |
264 | { | |
265 | spin_lock_bh(&lgr->llc_flow_lock); | |
266 | memset(flow, 0, sizeof(*flow)); | |
267 | flow->type = SMC_LLC_FLOW_NONE; | |
268 | spin_unlock_bh(&lgr->llc_flow_lock); | |
269 | if (!list_empty(&lgr->list) && lgr->delayed_event && | |
270 | flow == &lgr->llc_flow_lcl) | |
271 | schedule_work(&lgr->llc_event_work); | |
272 | else | |
273 | wake_up_interruptible(&lgr->llc_waiter); | |
274 | } | |
275 | ||
276 | /* lnk is optional and used for early wakeup when link goes down, useful in | |
277 | * cases where we wait for a response on the link after we sent a request | |
278 | */ | |
279 | struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, | |
280 | struct smc_link *lnk, | |
281 | int time_out, u8 exp_msg) | |
282 | { | |
283 | struct smc_llc_flow *flow = &lgr->llc_flow_lcl; | |
284 | ||
285 | wait_event_interruptible_timeout(lgr->llc_waiter, | |
286 | (flow->qentry || | |
287 | (lnk && !smc_link_usable(lnk)) || | |
288 | list_empty(&lgr->list)), | |
289 | time_out); | |
290 | if (!flow->qentry || | |
291 | (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { | |
292 | smc_llc_flow_qentry_del(flow); | |
293 | goto out; | |
294 | } | |
295 | if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) { | |
296 | if (exp_msg == SMC_LLC_ADD_LINK && | |
297 | flow->qentry->msg.raw.hdr.common.type == | |
298 | SMC_LLC_DELETE_LINK) { | |
299 | /* flow_start will delay the unexpected msg */ | |
300 | smc_llc_flow_start(&lgr->llc_flow_lcl, | |
301 | smc_llc_flow_qentry_clr(flow)); | |
302 | return NULL; | |
303 | } | |
304 | smc_llc_flow_qentry_del(flow); | |
305 | } | |
306 | out: | |
307 | return flow->qentry; | |
308 | } | |
309 | ||
9bf9abea UB |
310 | /********************************** send *************************************/ |
311 | ||
312 | struct smc_llc_tx_pend { | |
313 | }; | |
314 | ||
315 | /* handler for send/transmission completion of an LLC msg */ | |
316 | static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, | |
317 | struct smc_link *link, | |
318 | enum ib_wc_status wc_status) | |
319 | { | |
320 | /* future work: handle wc_status error for recovery and failover */ | |
321 | } | |
322 | ||
323 | /** | |
324 | * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits | |
325 | * @link: Pointer to SMC link used for sending LLC control message. | |
326 | * @wr_buf: Out variable returning pointer to work request payload buffer. | |
327 | * @pend: Out variable returning pointer to private pending WR tracking. | |
328 | * It's the context the transmit complete handler will get. | |
329 | * | |
330 | * Reserves and pre-fills an entry for a pending work request send/tx. | |
331 | * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. | |
332 | * Can sleep due to smc_get_ctrl_buf (if not in softirq context). | |
333 | * | |
334 | * Return: 0 on success, otherwise an error value. | |
335 | */ | |
336 | static int smc_llc_add_pending_send(struct smc_link *link, | |
337 | struct smc_wr_buf **wr_buf, | |
338 | struct smc_wr_tx_pend_priv **pend) | |
339 | { | |
340 | int rc; | |
341 | ||
ad6f317f UB |
342 | rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, |
343 | pend); | |
9bf9abea UB |
344 | if (rc < 0) |
345 | return rc; | |
346 | BUILD_BUG_ON_MSG( | |
347 | sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, | |
348 | "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); | |
349 | BUILD_BUG_ON_MSG( | |
350 | sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, | |
351 | "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); | |
352 | BUILD_BUG_ON_MSG( | |
353 | sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, | |
354 | "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); | |
355 | return 0; | |
356 | } | |
357 | ||
358 | /* high-level API to send LLC confirm link */ | |
947541f3 | 359 | int smc_llc_send_confirm_link(struct smc_link *link, |
9bf9abea UB |
360 | enum smc_llc_reqresp reqresp) |
361 | { | |
00e5fb26 | 362 | struct smc_link_group *lgr = smc_get_lgr(link); |
9bf9abea UB |
363 | struct smc_llc_msg_confirm_link *confllc; |
364 | struct smc_wr_tx_pend_priv *pend; | |
365 | struct smc_wr_buf *wr_buf; | |
366 | int rc; | |
367 | ||
368 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
369 | if (rc) | |
370 | return rc; | |
371 | confllc = (struct smc_llc_msg_confirm_link *)wr_buf; | |
372 | memset(confllc, 0, sizeof(*confllc)); | |
373 | confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; | |
374 | confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); | |
75d320d6 | 375 | confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; |
9bf9abea UB |
376 | if (reqresp == SMC_LLC_RESP) |
377 | confllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
947541f3 UB |
378 | memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], |
379 | ETH_ALEN); | |
7005ada6 | 380 | memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); |
9bf9abea | 381 | hton24(confllc->sender_qp_num, link->roce_qp->qp_num); |
2be922f3 | 382 | confllc->link_num = link->link_id; |
9bf9abea | 383 | memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); |
b1570a87 | 384 | confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; |
52bedf37 KG |
385 | /* send llc message */ |
386 | rc = smc_wr_tx_send(link, pend); | |
387 | return rc; | |
388 | } | |
389 | ||
44aa81ce | 390 | /* send LLC confirm rkey request */ |
3d88a21b | 391 | static int smc_llc_send_confirm_rkey(struct smc_link *send_link, |
44aa81ce KG |
392 | struct smc_buf_desc *rmb_desc) |
393 | { | |
394 | struct smc_llc_msg_confirm_rkey *rkeyllc; | |
395 | struct smc_wr_tx_pend_priv *pend; | |
396 | struct smc_wr_buf *wr_buf; | |
3d88a21b KG |
397 | struct smc_link *link; |
398 | int i, rc, rtok_ix; | |
44aa81ce | 399 | |
3d88a21b | 400 | rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); |
44aa81ce KG |
401 | if (rc) |
402 | return rc; | |
403 | rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; | |
404 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
405 | rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; | |
406 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); | |
3d88a21b KG |
407 | |
408 | rtok_ix = 1; | |
409 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
410 | link = &send_link->lgr->lnk[i]; | |
411 | if (link->state == SMC_LNK_ACTIVE && link != send_link) { | |
412 | rkeyllc->rtoken[rtok_ix].link_id = link->link_id; | |
413 | rkeyllc->rtoken[rtok_ix].rmb_key = | |
414 | htonl(rmb_desc->mr_rx[link->link_idx]->rkey); | |
415 | rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( | |
416 | (u64)sg_dma_address( | |
417 | rmb_desc->sgt[link->link_idx].sgl)); | |
418 | rtok_ix++; | |
419 | } | |
420 | } | |
421 | /* rkey of send_link is in rtoken[0] */ | |
422 | rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; | |
44aa81ce | 423 | rkeyllc->rtoken[0].rmb_key = |
3d88a21b | 424 | htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); |
44aa81ce | 425 | rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( |
3d88a21b | 426 | (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); |
44aa81ce | 427 | /* send llc message */ |
3d88a21b | 428 | rc = smc_wr_tx_send(send_link, pend); |
44aa81ce KG |
429 | return rc; |
430 | } | |
431 | ||
60e03c62 KG |
432 | /* send LLC delete rkey request */ |
433 | static int smc_llc_send_delete_rkey(struct smc_link *link, | |
434 | struct smc_buf_desc *rmb_desc) | |
435 | { | |
436 | struct smc_llc_msg_delete_rkey *rkeyllc; | |
437 | struct smc_wr_tx_pend_priv *pend; | |
438 | struct smc_wr_buf *wr_buf; | |
439 | int rc; | |
440 | ||
441 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
442 | if (rc) | |
443 | return rc; | |
444 | rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; | |
445 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
446 | rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; | |
447 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); | |
448 | rkeyllc->num_rkeys = 1; | |
387707fd | 449 | rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); |
60e03c62 KG |
450 | /* send llc message */ |
451 | rc = smc_wr_tx_send(link, pend); | |
452 | return rc; | |
453 | } | |
454 | ||
52bedf37 | 455 | /* send ADD LINK request or response */ |
7005ada6 | 456 | int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], |
fbed3b37 | 457 | struct smc_link *link_new, |
52bedf37 KG |
458 | enum smc_llc_reqresp reqresp) |
459 | { | |
460 | struct smc_llc_msg_add_link *addllc; | |
461 | struct smc_wr_tx_pend_priv *pend; | |
462 | struct smc_wr_buf *wr_buf; | |
463 | int rc; | |
464 | ||
465 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
466 | if (rc) | |
467 | return rc; | |
468 | addllc = (struct smc_llc_msg_add_link *)wr_buf; | |
fbed3b37 KG |
469 | |
470 | memset(addllc, 0, sizeof(*addllc)); | |
471 | addllc->hd.common.type = SMC_LLC_ADD_LINK; | |
472 | addllc->hd.length = sizeof(struct smc_llc_msg_add_link); | |
473 | if (reqresp == SMC_LLC_RESP) | |
474 | addllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
475 | memcpy(addllc->sender_mac, mac, ETH_ALEN); | |
476 | memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); | |
477 | if (link_new) { | |
478 | addllc->link_num = link_new->link_id; | |
479 | hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); | |
480 | hton24(addllc->initial_psn, link_new->psn_initial); | |
481 | if (reqresp == SMC_LLC_REQ) | |
482 | addllc->qp_mtu = link_new->path_mtu; | |
483 | else | |
484 | addllc->qp_mtu = min(link_new->path_mtu, | |
485 | link_new->peer_mtu); | |
486 | } | |
52bedf37 KG |
487 | /* send llc message */ |
488 | rc = smc_wr_tx_send(link, pend); | |
489 | return rc; | |
490 | } | |
491 | ||
492 | /* send DELETE LINK request or response */ | |
fbed3b37 KG |
493 | int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, |
494 | enum smc_llc_reqresp reqresp, bool orderly, | |
495 | u32 reason) | |
52bedf37 KG |
496 | { |
497 | struct smc_llc_msg_del_link *delllc; | |
498 | struct smc_wr_tx_pend_priv *pend; | |
499 | struct smc_wr_buf *wr_buf; | |
500 | int rc; | |
501 | ||
502 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
503 | if (rc) | |
504 | return rc; | |
505 | delllc = (struct smc_llc_msg_del_link *)wr_buf; | |
fbed3b37 KG |
506 | |
507 | memset(delllc, 0, sizeof(*delllc)); | |
508 | delllc->hd.common.type = SMC_LLC_DELETE_LINK; | |
509 | delllc->hd.length = sizeof(struct smc_llc_msg_del_link); | |
510 | if (reqresp == SMC_LLC_RESP) | |
511 | delllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
512 | if (orderly) | |
513 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
514 | if (link_del_id) | |
515 | delllc->link_num = link_del_id; | |
516 | else | |
517 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; | |
518 | delllc->reason = htonl(reason); | |
9bf9abea UB |
519 | /* send llc message */ |
520 | rc = smc_wr_tx_send(link, pend); | |
521 | return rc; | |
522 | } | |
523 | ||
d97935fa KG |
524 | /* send LLC test link request */ |
525 | static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) | |
313164da KG |
526 | { |
527 | struct smc_llc_msg_test_link *testllc; | |
528 | struct smc_wr_tx_pend_priv *pend; | |
529 | struct smc_wr_buf *wr_buf; | |
530 | int rc; | |
531 | ||
532 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
533 | if (rc) | |
534 | return rc; | |
535 | testllc = (struct smc_llc_msg_test_link *)wr_buf; | |
536 | memset(testllc, 0, sizeof(*testllc)); | |
537 | testllc->hd.common.type = SMC_LLC_TEST_LINK; | |
538 | testllc->hd.length = sizeof(struct smc_llc_msg_test_link); | |
313164da KG |
539 | memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); |
540 | /* send llc message */ | |
541 | rc = smc_wr_tx_send(link, pend); | |
542 | return rc; | |
543 | } | |
544 | ||
6c8968c4 KG |
545 | /* schedule an llc send on link, may wait for buffers */ |
546 | static int smc_llc_send_message(struct smc_link *link, void *llcbuf) | |
4ed75de5 KG |
547 | { |
548 | struct smc_wr_tx_pend_priv *pend; | |
549 | struct smc_wr_buf *wr_buf; | |
550 | int rc; | |
551 | ||
6c8968c4 KG |
552 | if (!smc_link_usable(link)) |
553 | return -ENOLINK; | |
554 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
4ed75de5 | 555 | if (rc) |
6c8968c4 KG |
556 | return rc; |
557 | memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); | |
558 | return smc_wr_tx_send(link, pend); | |
4ed75de5 KG |
559 | } |
560 | ||
9bf9abea UB |
561 | /********************************* receive ***********************************/ |
562 | ||
336ba09f KG |
563 | static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, |
564 | enum smc_lgr_type lgr_new_t) | |
565 | { | |
566 | int i; | |
567 | ||
568 | if (lgr->type == SMC_LGR_SYMMETRIC || | |
569 | (lgr->type != SMC_LGR_SINGLE && | |
570 | (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
571 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) | |
572 | return -EMLINK; | |
573 | ||
574 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
575 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { | |
576 | for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) | |
577 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
578 | return i; | |
579 | } else { | |
580 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) | |
581 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
582 | return i; | |
583 | } | |
584 | return -EMLINK; | |
585 | } | |
586 | ||
87f88cda KG |
587 | /* return first buffer from any of the next buf lists */ |
588 | static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
589 | int *buf_lst) | |
590 | { | |
591 | struct smc_buf_desc *buf_pos; | |
592 | ||
593 | while (*buf_lst < SMC_RMBE_SIZES) { | |
594 | buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], | |
595 | struct smc_buf_desc, list); | |
596 | if (buf_pos) | |
597 | return buf_pos; | |
598 | (*buf_lst)++; | |
599 | } | |
600 | return NULL; | |
601 | } | |
602 | ||
603 | /* return next rmb from buffer lists */ | |
604 | static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
605 | int *buf_lst, | |
606 | struct smc_buf_desc *buf_pos) | |
607 | { | |
608 | struct smc_buf_desc *buf_next; | |
609 | ||
610 | if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { | |
611 | (*buf_lst)++; | |
612 | return _smc_llc_get_next_rmb(lgr, buf_lst); | |
613 | } | |
614 | buf_next = list_next_entry(buf_pos, list); | |
615 | return buf_next; | |
616 | } | |
617 | ||
618 | static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, | |
619 | int *buf_lst) | |
620 | { | |
621 | *buf_lst = 0; | |
622 | return smc_llc_get_next_rmb(lgr, buf_lst, NULL); | |
623 | } | |
624 | ||
625 | /* send one add_link_continue msg */ | |
626 | static int smc_llc_add_link_cont(struct smc_link *link, | |
627 | struct smc_link *link_new, u8 *num_rkeys_todo, | |
628 | int *buf_lst, struct smc_buf_desc **buf_pos) | |
629 | { | |
630 | struct smc_llc_msg_add_link_cont *addc_llc; | |
631 | struct smc_link_group *lgr = link->lgr; | |
632 | int prim_lnk_idx, lnk_idx, i, rc; | |
633 | struct smc_wr_tx_pend_priv *pend; | |
634 | struct smc_wr_buf *wr_buf; | |
635 | struct smc_buf_desc *rmb; | |
636 | u8 n; | |
637 | ||
638 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
639 | if (rc) | |
640 | return rc; | |
641 | addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; | |
642 | memset(addc_llc, 0, sizeof(*addc_llc)); | |
643 | ||
644 | prim_lnk_idx = link->link_idx; | |
645 | lnk_idx = link_new->link_idx; | |
646 | addc_llc->link_num = link_new->link_id; | |
647 | addc_llc->num_rkeys = *num_rkeys_todo; | |
648 | n = *num_rkeys_todo; | |
649 | for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { | |
650 | if (!*buf_pos) { | |
651 | addc_llc->num_rkeys = addc_llc->num_rkeys - | |
652 | *num_rkeys_todo; | |
653 | *num_rkeys_todo = 0; | |
654 | break; | |
655 | } | |
656 | rmb = *buf_pos; | |
657 | ||
658 | addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); | |
659 | addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); | |
660 | addc_llc->rt[i].rmb_vaddr_new = | |
661 | cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); | |
662 | ||
663 | (*num_rkeys_todo)--; | |
664 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
665 | while (*buf_pos && !(*buf_pos)->used) | |
666 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
667 | } | |
668 | addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; | |
669 | addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); | |
670 | if (lgr->role == SMC_CLNT) | |
671 | addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
672 | return smc_wr_tx_send(link, pend); | |
673 | } | |
674 | ||
675 | static int smc_llc_cli_rkey_exchange(struct smc_link *link, | |
676 | struct smc_link *link_new) | |
677 | { | |
678 | struct smc_llc_msg_add_link_cont *addc_llc; | |
679 | struct smc_link_group *lgr = link->lgr; | |
680 | u8 max, num_rkeys_send, num_rkeys_recv; | |
681 | struct smc_llc_qentry *qentry; | |
682 | struct smc_buf_desc *buf_pos; | |
683 | int buf_lst; | |
684 | int rc = 0; | |
685 | int i; | |
686 | ||
687 | mutex_lock(&lgr->rmbs_lock); | |
688 | num_rkeys_send = lgr->conns_num; | |
689 | buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); | |
690 | do { | |
691 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, | |
692 | SMC_LLC_ADD_LINK_CONT); | |
693 | if (!qentry) { | |
694 | rc = -ETIMEDOUT; | |
695 | break; | |
696 | } | |
697 | addc_llc = &qentry->msg.add_link_cont; | |
698 | num_rkeys_recv = addc_llc->num_rkeys; | |
699 | max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); | |
700 | for (i = 0; i < max; i++) { | |
701 | smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, | |
702 | addc_llc->rt[i].rmb_key, | |
703 | addc_llc->rt[i].rmb_vaddr_new, | |
704 | addc_llc->rt[i].rmb_key_new); | |
705 | num_rkeys_recv--; | |
706 | } | |
707 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
708 | rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, | |
709 | &buf_lst, &buf_pos); | |
710 | if (rc) | |
711 | break; | |
712 | } while (num_rkeys_send || num_rkeys_recv); | |
713 | ||
714 | mutex_unlock(&lgr->rmbs_lock); | |
715 | return rc; | |
716 | } | |
717 | ||
336ba09f KG |
718 | /* prepare and send an add link reject response */ |
719 | static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) | |
720 | { | |
721 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; | |
722 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; | |
723 | qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; | |
724 | return smc_llc_send_message(qentry->link, &qentry->msg); | |
725 | } | |
726 | ||
b1570a87 KG |
727 | static int smc_llc_cli_conf_link(struct smc_link *link, |
728 | struct smc_init_info *ini, | |
729 | struct smc_link *link_new, | |
730 | enum smc_lgr_type lgr_new_t) | |
731 | { | |
732 | struct smc_link_group *lgr = link->lgr; | |
733 | struct smc_llc_msg_del_link *del_llc; | |
734 | struct smc_llc_qentry *qentry = NULL; | |
735 | int rc = 0; | |
736 | ||
737 | /* receive CONFIRM LINK request over RoCE fabric */ | |
738 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); | |
739 | if (!qentry) { | |
740 | rc = smc_llc_send_delete_link(link, link_new->link_id, | |
741 | SMC_LLC_REQ, false, | |
742 | SMC_LLC_DEL_LOST_PATH); | |
743 | return -ENOLINK; | |
744 | } | |
745 | if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { | |
746 | /* received DELETE_LINK instead */ | |
747 | del_llc = &qentry->msg.delete_link; | |
748 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; | |
749 | smc_llc_send_message(link, &qentry->msg); | |
750 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
751 | return -ENOLINK; | |
752 | } | |
753 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
754 | ||
755 | rc = smc_ib_modify_qp_rts(link_new); | |
756 | if (rc) { | |
757 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
758 | false, SMC_LLC_DEL_LOST_PATH); | |
759 | return -ENOLINK; | |
760 | } | |
761 | smc_wr_remember_qp_attr(link_new); | |
762 | ||
763 | rc = smcr_buf_reg_lgr(link_new); | |
764 | if (rc) { | |
765 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
766 | false, SMC_LLC_DEL_LOST_PATH); | |
767 | return -ENOLINK; | |
768 | } | |
769 | ||
770 | /* send CONFIRM LINK response over RoCE fabric */ | |
771 | rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); | |
772 | if (rc) { | |
773 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
774 | false, SMC_LLC_DEL_LOST_PATH); | |
775 | return -ENOLINK; | |
776 | } | |
777 | smc_llc_link_active(link_new); | |
778 | lgr->type = lgr_new_t; | |
779 | return 0; | |
780 | } | |
781 | ||
336ba09f KG |
782 | static void smc_llc_save_add_link_info(struct smc_link *link, |
783 | struct smc_llc_msg_add_link *add_llc) | |
784 | { | |
785 | link->peer_qpn = ntoh24(add_llc->sender_qp_num); | |
786 | memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); | |
787 | memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); | |
788 | link->peer_psn = ntoh24(add_llc->initial_psn); | |
789 | link->peer_mtu = add_llc->qp_mtu; | |
790 | } | |
791 | ||
792 | /* as an SMC client, process an add link request */ | |
793 | int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) | |
794 | { | |
795 | struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; | |
796 | enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; | |
797 | struct smc_link_group *lgr = smc_get_lgr(link); | |
798 | struct smc_link *lnk_new = NULL; | |
799 | struct smc_init_info ini; | |
800 | int lnk_idx, rc = 0; | |
801 | ||
802 | ini.vlan_id = lgr->vlan_id; | |
803 | smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); | |
804 | if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && | |
805 | !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { | |
806 | if (!ini.ib_dev) | |
807 | goto out_reject; | |
808 | lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; | |
809 | } | |
810 | if (!ini.ib_dev) { | |
811 | lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; | |
812 | ini.ib_dev = link->smcibdev; | |
813 | ini.ib_port = link->ibport; | |
814 | } | |
815 | lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); | |
816 | if (lnk_idx < 0) | |
817 | goto out_reject; | |
818 | lnk_new = &lgr->lnk[lnk_idx]; | |
819 | rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); | |
820 | if (rc) | |
821 | goto out_reject; | |
822 | smc_llc_save_add_link_info(lnk_new, llc); | |
823 | lnk_new->link_id = llc->link_num; | |
824 | ||
825 | rc = smc_ib_ready_link(lnk_new); | |
826 | if (rc) | |
827 | goto out_clear_lnk; | |
828 | ||
829 | rc = smcr_buf_map_lgr(lnk_new); | |
830 | if (rc) | |
831 | goto out_clear_lnk; | |
832 | ||
833 | rc = smc_llc_send_add_link(link, | |
834 | lnk_new->smcibdev->mac[ini.ib_port - 1], | |
835 | lnk_new->gid, lnk_new, SMC_LLC_RESP); | |
836 | if (rc) | |
837 | goto out_clear_lnk; | |
87f88cda | 838 | rc = smc_llc_cli_rkey_exchange(link, lnk_new); |
336ba09f KG |
839 | if (rc) { |
840 | rc = 0; | |
841 | goto out_clear_lnk; | |
842 | } | |
b1570a87 | 843 | rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); |
336ba09f KG |
844 | if (!rc) |
845 | goto out; | |
846 | out_clear_lnk: | |
847 | smcr_link_clear(lnk_new); | |
848 | out_reject: | |
849 | smc_llc_cli_add_link_reject(qentry); | |
850 | out: | |
851 | kfree(qentry); | |
852 | return rc; | |
853 | } | |
854 | ||
b1570a87 KG |
855 | static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) |
856 | { | |
857 | struct smc_llc_qentry *qentry; | |
858 | ||
859 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
860 | ||
861 | mutex_lock(&lgr->llc_conf_mutex); | |
862 | smc_llc_cli_add_link(qentry->link, qentry); | |
863 | mutex_unlock(&lgr->llc_conf_mutex); | |
864 | } | |
865 | ||
c9a5d243 KG |
866 | /* find the asymmetric link when 3 links are established */ |
867 | static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) | |
868 | { | |
869 | int asym_idx = -ENOENT; | |
870 | int i, j, k; | |
871 | bool found; | |
872 | ||
873 | /* determine asymmetric link */ | |
874 | found = false; | |
875 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
876 | for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { | |
877 | if (!smc_link_usable(&lgr->lnk[i]) || | |
878 | !smc_link_usable(&lgr->lnk[j])) | |
879 | continue; | |
880 | if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, | |
881 | SMC_GID_SIZE)) { | |
882 | found = true; /* asym_lnk is i or j */ | |
883 | break; | |
884 | } | |
885 | } | |
886 | if (found) | |
887 | break; | |
888 | } | |
889 | if (!found) | |
890 | goto out; /* no asymmetric link */ | |
891 | for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { | |
892 | if (!smc_link_usable(&lgr->lnk[k])) | |
893 | continue; | |
894 | if (k != i && | |
895 | !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, | |
896 | SMC_GID_SIZE)) { | |
897 | asym_idx = i; | |
898 | break; | |
899 | } | |
900 | if (k != j && | |
901 | !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, | |
902 | SMC_GID_SIZE)) { | |
903 | asym_idx = j; | |
904 | break; | |
905 | } | |
906 | } | |
907 | out: | |
908 | return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; | |
909 | } | |
910 | ||
911 | static void smc_llc_delete_asym_link(struct smc_link_group *lgr) | |
912 | { | |
913 | struct smc_link *lnk_new = NULL, *lnk_asym; | |
914 | struct smc_llc_qentry *qentry; | |
915 | int rc; | |
916 | ||
917 | lnk_asym = smc_llc_find_asym_link(lgr); | |
918 | if (!lnk_asym) | |
919 | return; /* no asymmetric link */ | |
920 | if (!smc_link_downing(&lnk_asym->state)) | |
921 | return; | |
922 | /* tbd: lnk_new = smc_switch_conns(lgr, lnk_asym, false); */ | |
923 | smc_wr_tx_wait_no_pending_sends(lnk_asym); | |
924 | if (!lnk_new) | |
925 | goto out_free; | |
926 | /* change flow type from ADD_LINK into DEL_LINK */ | |
927 | lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; | |
928 | rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, | |
929 | true, SMC_LLC_DEL_NO_ASYM_NEEDED); | |
930 | if (rc) { | |
931 | smcr_link_down_cond(lnk_new); | |
932 | goto out_free; | |
933 | } | |
934 | qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, | |
935 | SMC_LLC_DELETE_LINK); | |
936 | if (!qentry) { | |
937 | smcr_link_down_cond(lnk_new); | |
938 | goto out_free; | |
939 | } | |
940 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
941 | out_free: | |
942 | smcr_link_clear(lnk_asym); | |
943 | } | |
944 | ||
57b49924 KG |
945 | static int smc_llc_srv_rkey_exchange(struct smc_link *link, |
946 | struct smc_link *link_new) | |
947 | { | |
948 | struct smc_llc_msg_add_link_cont *addc_llc; | |
949 | struct smc_link_group *lgr = link->lgr; | |
950 | u8 max, num_rkeys_send, num_rkeys_recv; | |
951 | struct smc_llc_qentry *qentry = NULL; | |
952 | struct smc_buf_desc *buf_pos; | |
953 | int buf_lst; | |
954 | int rc = 0; | |
955 | int i; | |
956 | ||
957 | mutex_lock(&lgr->rmbs_lock); | |
958 | num_rkeys_send = lgr->conns_num; | |
959 | buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); | |
960 | do { | |
961 | smc_llc_add_link_cont(link, link_new, &num_rkeys_send, | |
962 | &buf_lst, &buf_pos); | |
963 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, | |
964 | SMC_LLC_ADD_LINK_CONT); | |
965 | if (!qentry) { | |
966 | rc = -ETIMEDOUT; | |
967 | goto out; | |
968 | } | |
969 | addc_llc = &qentry->msg.add_link_cont; | |
970 | num_rkeys_recv = addc_llc->num_rkeys; | |
971 | max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); | |
972 | for (i = 0; i < max; i++) { | |
973 | smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, | |
974 | addc_llc->rt[i].rmb_key, | |
975 | addc_llc->rt[i].rmb_vaddr_new, | |
976 | addc_llc->rt[i].rmb_key_new); | |
977 | num_rkeys_recv--; | |
978 | } | |
979 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
980 | } while (num_rkeys_send || num_rkeys_recv); | |
981 | out: | |
982 | mutex_unlock(&lgr->rmbs_lock); | |
983 | return rc; | |
984 | } | |
985 | ||
1551c95b KG |
986 | static int smc_llc_srv_conf_link(struct smc_link *link, |
987 | struct smc_link *link_new, | |
988 | enum smc_lgr_type lgr_new_t) | |
989 | { | |
990 | struct smc_link_group *lgr = link->lgr; | |
991 | struct smc_llc_qentry *qentry = NULL; | |
992 | int rc; | |
993 | ||
994 | /* send CONFIRM LINK request over the RoCE fabric */ | |
995 | rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); | |
996 | if (rc) | |
997 | return -ENOLINK; | |
998 | /* receive CONFIRM LINK response over the RoCE fabric */ | |
999 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, | |
1000 | SMC_LLC_CONFIRM_LINK); | |
1001 | if (!qentry) { | |
1002 | /* send DELETE LINK */ | |
1003 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
1004 | false, SMC_LLC_DEL_LOST_PATH); | |
1005 | return -ENOLINK; | |
1006 | } | |
1007 | smc_llc_link_active(link_new); | |
1008 | lgr->type = lgr_new_t; | |
1009 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1010 | return 0; | |
1011 | } | |
1012 | ||
2d2209f2 KG |
1013 | int smc_llc_srv_add_link(struct smc_link *link) |
1014 | { | |
1015 | enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; | |
1016 | struct smc_link_group *lgr = link->lgr; | |
1017 | struct smc_llc_msg_add_link *add_llc; | |
1018 | struct smc_llc_qentry *qentry = NULL; | |
1019 | struct smc_link *link_new; | |
1020 | struct smc_init_info ini; | |
1021 | int lnk_idx, rc = 0; | |
1022 | ||
1023 | /* ignore client add link recommendation, start new flow */ | |
1024 | ini.vlan_id = lgr->vlan_id; | |
1025 | smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); | |
1026 | if (!ini.ib_dev) { | |
1027 | lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; | |
1028 | ini.ib_dev = link->smcibdev; | |
1029 | ini.ib_port = link->ibport; | |
1030 | } | |
1031 | lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); | |
1032 | if (lnk_idx < 0) | |
1033 | return 0; | |
1034 | ||
1035 | rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); | |
1036 | if (rc) | |
1037 | return rc; | |
1038 | link_new = &lgr->lnk[lnk_idx]; | |
1039 | rc = smc_llc_send_add_link(link, | |
1040 | link_new->smcibdev->mac[ini.ib_port - 1], | |
1041 | link_new->gid, link_new, SMC_LLC_REQ); | |
1042 | if (rc) | |
1043 | goto out_err; | |
1044 | /* receive ADD LINK response over the RoCE fabric */ | |
1045 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); | |
1046 | if (!qentry) { | |
1047 | rc = -ETIMEDOUT; | |
1048 | goto out_err; | |
1049 | } | |
1050 | add_llc = &qentry->msg.add_link; | |
1051 | if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { | |
1052 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1053 | rc = -ENOLINK; | |
1054 | goto out_err; | |
1055 | } | |
1056 | if (lgr->type == SMC_LGR_SINGLE && | |
1057 | (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && | |
1058 | !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { | |
1059 | lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; | |
1060 | } | |
1061 | smc_llc_save_add_link_info(link_new, add_llc); | |
1062 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1063 | ||
1064 | rc = smc_ib_ready_link(link_new); | |
1065 | if (rc) | |
1066 | goto out_err; | |
1067 | rc = smcr_buf_map_lgr(link_new); | |
1068 | if (rc) | |
1069 | goto out_err; | |
1070 | rc = smcr_buf_reg_lgr(link_new); | |
1071 | if (rc) | |
1072 | goto out_err; | |
57b49924 | 1073 | rc = smc_llc_srv_rkey_exchange(link, link_new); |
2d2209f2 KG |
1074 | if (rc) |
1075 | goto out_err; | |
1551c95b | 1076 | rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); |
2d2209f2 KG |
1077 | if (rc) |
1078 | goto out_err; | |
1079 | return 0; | |
1080 | out_err: | |
1081 | smcr_link_clear(link_new); | |
1082 | return rc; | |
1083 | } | |
1084 | ||
1085 | static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) | |
1086 | { | |
1087 | struct smc_link *link = lgr->llc_flow_lcl.qentry->link; | |
1088 | int rc; | |
1089 | ||
1090 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1091 | ||
1092 | mutex_lock(&lgr->llc_conf_mutex); | |
1093 | rc = smc_llc_srv_add_link(link); | |
1094 | if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { | |
1095 | /* delete any asymmetric link */ | |
c9a5d243 | 1096 | smc_llc_delete_asym_link(lgr); |
2d2209f2 KG |
1097 | } |
1098 | mutex_unlock(&lgr->llc_conf_mutex); | |
1099 | } | |
1100 | ||
b45e7f98 KG |
1101 | /* worker to process an add link message */ |
1102 | static void smc_llc_add_link_work(struct work_struct *work) | |
1103 | { | |
1104 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
1105 | llc_add_link_work); | |
1106 | ||
1107 | if (list_empty(&lgr->list)) { | |
1108 | /* link group is terminating */ | |
1109 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1110 | goto out; | |
1111 | } | |
1112 | ||
b1570a87 KG |
1113 | if (lgr->role == SMC_CLNT) |
1114 | smc_llc_process_cli_add_link(lgr); | |
2d2209f2 KG |
1115 | else |
1116 | smc_llc_process_srv_add_link(lgr); | |
b45e7f98 KG |
1117 | out: |
1118 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); | |
1119 | } | |
1120 | ||
9ec6bf19 | 1121 | static void smc_llc_delete_link_work(struct work_struct *work) |
52bedf37 | 1122 | { |
9ec6bf19 KG |
1123 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, |
1124 | llc_del_link_work); | |
52bedf37 | 1125 | |
9ec6bf19 KG |
1126 | if (list_empty(&lgr->list)) { |
1127 | /* link group is terminating */ | |
1128 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1129 | goto out; | |
52bedf37 | 1130 | } |
9ec6bf19 KG |
1131 | out: |
1132 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); | |
52bedf37 KG |
1133 | } |
1134 | ||
3bc67e09 KG |
1135 | /* process a confirm_rkey request from peer, remote flow */ |
1136 | static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 1137 | { |
3bc67e09 KG |
1138 | struct smc_llc_msg_confirm_rkey *llc; |
1139 | struct smc_llc_qentry *qentry; | |
1140 | struct smc_link *link; | |
1141 | int num_entries; | |
1142 | int rk_idx; | |
1143 | int i; | |
1144 | ||
1145 | qentry = lgr->llc_flow_rmt.qentry; | |
1146 | llc = &qentry->msg.confirm_rkey; | |
1147 | link = qentry->link; | |
1148 | ||
1149 | num_entries = llc->rtoken[0].num_rkeys; | |
1150 | /* first rkey entry is for receiving link */ | |
1151 | rk_idx = smc_rtoken_add(link, | |
1152 | llc->rtoken[0].rmb_vaddr, | |
1153 | llc->rtoken[0].rmb_key); | |
1154 | if (rk_idx < 0) | |
1155 | goto out_err; | |
1156 | ||
1157 | for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) | |
1158 | smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, | |
1159 | llc->rtoken[i].rmb_vaddr, | |
1160 | llc->rtoken[i].rmb_key); | |
1161 | /* max links is 3 so there is no need to support conf_rkey_cont msgs */ | |
1162 | goto out; | |
1163 | out_err: | |
1164 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
1165 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; | |
1166 | out: | |
ef79d439 | 1167 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
3bc67e09 KG |
1168 | smc_llc_send_message(link, &qentry->msg); |
1169 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
4ed75de5 KG |
1170 | } |
1171 | ||
218b24fe KG |
1172 | /* process a delete_rkey request from peer, remote flow */ |
1173 | static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 1174 | { |
218b24fe KG |
1175 | struct smc_llc_msg_delete_rkey *llc; |
1176 | struct smc_llc_qentry *qentry; | |
1177 | struct smc_link *link; | |
4ed75de5 KG |
1178 | u8 err_mask = 0; |
1179 | int i, max; | |
1180 | ||
218b24fe KG |
1181 | qentry = lgr->llc_flow_rmt.qentry; |
1182 | llc = &qentry->msg.delete_rkey; | |
1183 | link = qentry->link; | |
1184 | ||
ef79d439 KG |
1185 | max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); |
1186 | for (i = 0; i < max; i++) { | |
1187 | if (smc_rtoken_delete(link, llc->rkey[i])) | |
1188 | err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); | |
1189 | } | |
ef79d439 KG |
1190 | if (err_mask) { |
1191 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
1192 | llc->err_mask = err_mask; | |
4ed75de5 | 1193 | } |
218b24fe KG |
1194 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
1195 | smc_llc_send_message(link, &qentry->msg); | |
1196 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
1197 | } | |
ef79d439 | 1198 | |
6c8968c4 | 1199 | /* flush the llc event queue */ |
00a049cf | 1200 | static void smc_llc_event_flush(struct smc_link_group *lgr) |
9bf9abea | 1201 | { |
6c8968c4 KG |
1202 | struct smc_llc_qentry *qentry, *q; |
1203 | ||
1204 | spin_lock_bh(&lgr->llc_event_q_lock); | |
1205 | list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { | |
1206 | list_del_init(&qentry->list); | |
1207 | kfree(qentry); | |
1208 | } | |
1209 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1210 | } | |
1211 | ||
1212 | static void smc_llc_event_handler(struct smc_llc_qentry *qentry) | |
1213 | { | |
1214 | union smc_llc_msg *llc = &qentry->msg; | |
1215 | struct smc_link *link = qentry->link; | |
0fb0b02b | 1216 | struct smc_link_group *lgr = link->lgr; |
9bf9abea | 1217 | |
d854fcbf | 1218 | if (!smc_link_usable(link)) |
6c8968c4 | 1219 | goto out; |
313164da KG |
1220 | |
1221 | switch (llc->raw.hdr.common.type) { | |
1222 | case SMC_LLC_TEST_LINK: | |
56e8091c KG |
1223 | llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; |
1224 | smc_llc_send_message(link, llc); | |
313164da | 1225 | break; |
52bedf37 | 1226 | case SMC_LLC_ADD_LINK: |
0fb0b02b KG |
1227 | if (list_empty(&lgr->list)) |
1228 | goto out; /* lgr is terminating */ | |
1229 | if (lgr->role == SMC_CLNT) { | |
1230 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { | |
1231 | /* a flow is waiting for this message */ | |
1232 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1233 | qentry); | |
1234 | wake_up_interruptible(&lgr->llc_waiter); | |
1235 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, | |
1236 | qentry)) { | |
b45e7f98 | 1237 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1238 | } |
1239 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { | |
1240 | /* as smc server, handle client suggestion */ | |
b45e7f98 | 1241 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1242 | } |
1243 | return; | |
1244 | case SMC_LLC_CONFIRM_LINK: | |
87f88cda | 1245 | case SMC_LLC_ADD_LINK_CONT: |
0fb0b02b KG |
1246 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { |
1247 | /* a flow is waiting for this message */ | |
1248 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); | |
1249 | wake_up_interruptible(&lgr->llc_waiter); | |
1250 | return; | |
1251 | } | |
52bedf37 KG |
1252 | break; |
1253 | case SMC_LLC_DELETE_LINK: | |
9ec6bf19 KG |
1254 | if (lgr->role == SMC_CLNT) { |
1255 | /* server requests to delete this link, send response */ | |
1256 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { | |
1257 | /* DEL LINK REQ during ADD LINK SEQ */ | |
1258 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1259 | qentry); | |
1260 | wake_up_interruptible(&lgr->llc_waiter); | |
1261 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, | |
1262 | qentry)) { | |
1263 | schedule_work(&lgr->llc_del_link_work); | |
1264 | } | |
1265 | } else { | |
1266 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && | |
1267 | !lgr->llc_flow_lcl.qentry) { | |
1268 | /* DEL LINK REQ during ADD LINK SEQ */ | |
1269 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1270 | qentry); | |
1271 | wake_up_interruptible(&lgr->llc_waiter); | |
1272 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, | |
1273 | qentry)) { | |
1274 | schedule_work(&lgr->llc_del_link_work); | |
1275 | } | |
1276 | } | |
1277 | return; | |
4ed75de5 | 1278 | case SMC_LLC_CONFIRM_RKEY: |
3bc67e09 KG |
1279 | /* new request from remote, assign to remote flow */ |
1280 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1281 | /* process here, does not wait for more llc msgs */ | |
1282 | smc_llc_rmt_conf_rkey(lgr); | |
1283 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1284 | } | |
1285 | return; | |
4ed75de5 | 1286 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc KG |
1287 | /* not used because max links is 3, and 3 rkeys fit into |
1288 | * one CONFIRM_RKEY message | |
1289 | */ | |
4ed75de5 KG |
1290 | break; |
1291 | case SMC_LLC_DELETE_RKEY: | |
218b24fe KG |
1292 | /* new request from remote, assign to remote flow */ |
1293 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1294 | /* process here, does not wait for more llc msgs */ | |
1295 | smc_llc_rmt_delete_rkey(lgr); | |
1296 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1297 | } | |
1298 | return; | |
313164da | 1299 | } |
6c8968c4 KG |
1300 | out: |
1301 | kfree(qentry); | |
1302 | } | |
1303 | ||
1304 | /* worker to process llc messages on the event queue */ | |
1305 | static void smc_llc_event_work(struct work_struct *work) | |
1306 | { | |
1307 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
1308 | llc_event_work); | |
1309 | struct smc_llc_qentry *qentry; | |
1310 | ||
555da9af KG |
1311 | if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { |
1312 | if (smc_link_usable(lgr->delayed_event->link)) { | |
1313 | smc_llc_event_handler(lgr->delayed_event); | |
1314 | } else { | |
1315 | qentry = lgr->delayed_event; | |
1316 | lgr->delayed_event = NULL; | |
1317 | kfree(qentry); | |
1318 | } | |
1319 | } | |
1320 | ||
6c8968c4 KG |
1321 | again: |
1322 | spin_lock_bh(&lgr->llc_event_q_lock); | |
1323 | if (!list_empty(&lgr->llc_event_q)) { | |
1324 | qentry = list_first_entry(&lgr->llc_event_q, | |
1325 | struct smc_llc_qentry, list); | |
1326 | list_del_init(&qentry->list); | |
1327 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1328 | smc_llc_event_handler(qentry); | |
1329 | goto again; | |
1330 | } | |
1331 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1332 | } | |
1333 | ||
ef79d439 | 1334 | /* process llc responses in tasklet context */ |
a6688d91 KG |
1335 | static void smc_llc_rx_response(struct smc_link *link, |
1336 | struct smc_llc_qentry *qentry) | |
ef79d439 | 1337 | { |
a6688d91 | 1338 | u8 llc_type = qentry->msg.raw.hdr.common.type; |
ef79d439 | 1339 | |
a6688d91 | 1340 | switch (llc_type) { |
ef79d439 KG |
1341 | case SMC_LLC_TEST_LINK: |
1342 | if (link->state == SMC_LNK_ACTIVE) | |
1343 | complete(&link->llc_testlink_resp); | |
1344 | break; | |
ef79d439 | 1345 | case SMC_LLC_ADD_LINK: |
9ec6bf19 | 1346 | case SMC_LLC_DELETE_LINK: |
4667bb4a | 1347 | case SMC_LLC_CONFIRM_LINK: |
87f88cda | 1348 | case SMC_LLC_ADD_LINK_CONT: |
3d88a21b | 1349 | case SMC_LLC_CONFIRM_RKEY: |
6d74c3a8 | 1350 | case SMC_LLC_DELETE_RKEY: |
4667bb4a KG |
1351 | /* assign responses to the local flow, we requested them */ |
1352 | smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); | |
1353 | wake_up_interruptible(&link->lgr->llc_waiter); | |
1354 | return; | |
ef79d439 | 1355 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc | 1356 | /* not used because max links is 3 */ |
ef79d439 | 1357 | break; |
ef79d439 | 1358 | } |
a6688d91 | 1359 | kfree(qentry); |
ef79d439 KG |
1360 | } |
1361 | ||
a6688d91 | 1362 | static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) |
6c8968c4 | 1363 | { |
6c8968c4 KG |
1364 | struct smc_link_group *lgr = link->lgr; |
1365 | struct smc_llc_qentry *qentry; | |
6c8968c4 KG |
1366 | unsigned long flags; |
1367 | ||
6c8968c4 KG |
1368 | qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); |
1369 | if (!qentry) | |
1370 | return; | |
1371 | qentry->link = link; | |
1372 | INIT_LIST_HEAD(&qentry->list); | |
1373 | memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); | |
a6688d91 KG |
1374 | |
1375 | /* process responses immediately */ | |
1376 | if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { | |
1377 | smc_llc_rx_response(link, qentry); | |
1378 | return; | |
1379 | } | |
1380 | ||
1381 | /* add requests to event queue */ | |
6c8968c4 KG |
1382 | spin_lock_irqsave(&lgr->llc_event_q_lock, flags); |
1383 | list_add_tail(&qentry->list, &lgr->llc_event_q); | |
1384 | spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); | |
1385 | schedule_work(&link->lgr->llc_event_work); | |
9bf9abea UB |
1386 | } |
1387 | ||
a6688d91 KG |
1388 | /* copy received msg and add it to the event queue */ |
1389 | static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) | |
1390 | { | |
1391 | struct smc_link *link = (struct smc_link *)wc->qp->qp_context; | |
1392 | union smc_llc_msg *llc = buf; | |
1393 | ||
1394 | if (wc->byte_len < sizeof(*llc)) | |
1395 | return; /* short message */ | |
1396 | if (llc->raw.hdr.length != sizeof(*llc)) | |
1397 | return; /* invalid message */ | |
1398 | ||
1399 | smc_llc_enqueue(link, llc); | |
1400 | } | |
1401 | ||
44aa81ce | 1402 | /***************************** worker, utils *********************************/ |
877ae5be KG |
1403 | |
1404 | static void smc_llc_testlink_work(struct work_struct *work) | |
1405 | { | |
1406 | struct smc_link *link = container_of(to_delayed_work(work), | |
1407 | struct smc_link, llc_testlink_wrk); | |
1408 | unsigned long next_interval; | |
877ae5be KG |
1409 | unsigned long expire_time; |
1410 | u8 user_data[16] = { 0 }; | |
1411 | int rc; | |
1412 | ||
877ae5be KG |
1413 | if (link->state != SMC_LNK_ACTIVE) |
1414 | return; /* don't reschedule worker */ | |
1415 | expire_time = link->wr_rx_tstamp + link->llc_testlink_time; | |
1416 | if (time_is_after_jiffies(expire_time)) { | |
1417 | next_interval = expire_time - jiffies; | |
1418 | goto out; | |
1419 | } | |
1420 | reinit_completion(&link->llc_testlink_resp); | |
d97935fa | 1421 | smc_llc_send_test_link(link, user_data); |
877ae5be KG |
1422 | /* receive TEST LINK response over RoCE fabric */ |
1423 | rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, | |
1424 | SMC_LLC_WAIT_TIME); | |
1020e1ef KG |
1425 | if (link->state != SMC_LNK_ACTIVE) |
1426 | return; /* link state changed */ | |
877ae5be | 1427 | if (rc <= 0) { |
87523930 | 1428 | smcr_link_down_cond_sched(link); |
877ae5be KG |
1429 | return; |
1430 | } | |
1431 | next_interval = link->llc_testlink_time; | |
1432 | out: | |
1020e1ef | 1433 | schedule_delayed_work(&link->llc_testlink_wrk, next_interval); |
877ae5be KG |
1434 | } |
1435 | ||
00a049cf KG |
1436 | void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) |
1437 | { | |
1438 | struct net *net = sock_net(smc->clcsock->sk); | |
1439 | ||
1440 | INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); | |
b45e7f98 | 1441 | INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); |
9ec6bf19 | 1442 | INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); |
00a049cf KG |
1443 | INIT_LIST_HEAD(&lgr->llc_event_q); |
1444 | spin_lock_init(&lgr->llc_event_q_lock); | |
555da9af KG |
1445 | spin_lock_init(&lgr->llc_flow_lock); |
1446 | init_waitqueue_head(&lgr->llc_waiter); | |
d5500667 | 1447 | mutex_init(&lgr->llc_conf_mutex); |
00a049cf KG |
1448 | lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; |
1449 | } | |
1450 | ||
1451 | /* called after lgr was removed from lgr_list */ | |
1452 | void smc_llc_lgr_clear(struct smc_link_group *lgr) | |
1453 | { | |
1454 | smc_llc_event_flush(lgr); | |
555da9af | 1455 | wake_up_interruptible_all(&lgr->llc_waiter); |
00a049cf | 1456 | cancel_work_sync(&lgr->llc_event_work); |
b45e7f98 | 1457 | cancel_work_sync(&lgr->llc_add_link_work); |
9ec6bf19 | 1458 | cancel_work_sync(&lgr->llc_del_link_work); |
555da9af KG |
1459 | if (lgr->delayed_event) { |
1460 | kfree(lgr->delayed_event); | |
1461 | lgr->delayed_event = NULL; | |
1462 | } | |
00a049cf KG |
1463 | } |
1464 | ||
2a4c57a9 | 1465 | int smc_llc_link_init(struct smc_link *link) |
877ae5be KG |
1466 | { |
1467 | init_completion(&link->llc_testlink_resp); | |
1468 | INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); | |
2a4c57a9 | 1469 | return 0; |
b32cf4ab KG |
1470 | } |
1471 | ||
00a049cf | 1472 | void smc_llc_link_active(struct smc_link *link) |
b32cf4ab | 1473 | { |
877ae5be | 1474 | link->state = SMC_LNK_ACTIVE; |
00a049cf KG |
1475 | if (link->lgr->llc_testlink_time) { |
1476 | link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; | |
1020e1ef KG |
1477 | schedule_delayed_work(&link->llc_testlink_wrk, |
1478 | link->llc_testlink_time); | |
877ae5be KG |
1479 | } |
1480 | } | |
1481 | ||
877ae5be | 1482 | /* called in worker context */ |
2a4c57a9 | 1483 | void smc_llc_link_clear(struct smc_link *link) |
877ae5be | 1484 | { |
2140ac26 KG |
1485 | complete(&link->llc_testlink_resp); |
1486 | cancel_delayed_work_sync(&link->llc_testlink_wrk); | |
1487 | smc_wr_wakeup_reg_wait(link); | |
1488 | smc_wr_wakeup_tx_wait(link); | |
877ae5be KG |
1489 | } |
1490 | ||
3d88a21b KG |
1491 | /* register a new rtoken at the remote peer (for all links) */ |
1492 | int smc_llc_do_confirm_rkey(struct smc_link *send_link, | |
44aa81ce KG |
1493 | struct smc_buf_desc *rmb_desc) |
1494 | { | |
3d88a21b KG |
1495 | struct smc_link_group *lgr = send_link->lgr; |
1496 | struct smc_llc_qentry *qentry = NULL; | |
1497 | int rc = 0; | |
44aa81ce | 1498 | |
3d88a21b KG |
1499 | rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); |
1500 | if (rc) | |
1501 | goto out; | |
44aa81ce | 1502 | /* receive CONFIRM RKEY response from server over RoCE fabric */ |
3d88a21b KG |
1503 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1504 | SMC_LLC_CONFIRM_RKEY); | |
1505 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
1506 | rc = -EFAULT; | |
1507 | out: | |
1508 | if (qentry) | |
1509 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
3d88a21b | 1510 | return rc; |
44aa81ce KG |
1511 | } |
1512 | ||
60e03c62 | 1513 | /* unregister an rtoken at the remote peer */ |
6d74c3a8 | 1514 | int smc_llc_do_delete_rkey(struct smc_link_group *lgr, |
60e03c62 KG |
1515 | struct smc_buf_desc *rmb_desc) |
1516 | { | |
6d74c3a8 KG |
1517 | struct smc_llc_qentry *qentry = NULL; |
1518 | struct smc_link *send_link; | |
0b29ec64 | 1519 | int rc = 0; |
60e03c62 | 1520 | |
6d74c3a8 KG |
1521 | send_link = smc_llc_usable_link(lgr); |
1522 | if (!send_link) | |
1523 | return -ENOLINK; | |
1524 | ||
6d74c3a8 KG |
1525 | /* protected by llc_flow control */ |
1526 | rc = smc_llc_send_delete_rkey(send_link, rmb_desc); | |
60e03c62 KG |
1527 | if (rc) |
1528 | goto out; | |
1529 | /* receive DELETE RKEY response from server over RoCE fabric */ | |
6d74c3a8 KG |
1530 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1531 | SMC_LLC_DELETE_RKEY); | |
1532 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
60e03c62 | 1533 | rc = -EFAULT; |
60e03c62 | 1534 | out: |
6d74c3a8 KG |
1535 | if (qentry) |
1536 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
60e03c62 KG |
1537 | return rc; |
1538 | } | |
1539 | ||
92334cfc KG |
1540 | /* evaluate confirm link request or response */ |
1541 | int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, | |
1542 | enum smc_llc_reqresp type) | |
1543 | { | |
1544 | if (type == SMC_LLC_REQ) /* SMC server assigns link_id */ | |
1545 | qentry->link->link_id = qentry->msg.confirm_link.link_num; | |
1546 | if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) | |
1547 | return -ENOTSUPP; | |
1548 | return 0; | |
1549 | } | |
1550 | ||
9bf9abea UB |
1551 | /***************************** init, exit, misc ******************************/ |
1552 | ||
1553 | static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { | |
1554 | { | |
1555 | .handler = smc_llc_rx_handler, | |
1556 | .type = SMC_LLC_CONFIRM_LINK | |
1557 | }, | |
313164da KG |
1558 | { |
1559 | .handler = smc_llc_rx_handler, | |
1560 | .type = SMC_LLC_TEST_LINK | |
1561 | }, | |
52bedf37 KG |
1562 | { |
1563 | .handler = smc_llc_rx_handler, | |
1564 | .type = SMC_LLC_ADD_LINK | |
1565 | }, | |
87f88cda KG |
1566 | { |
1567 | .handler = smc_llc_rx_handler, | |
1568 | .type = SMC_LLC_ADD_LINK_CONT | |
1569 | }, | |
52bedf37 KG |
1570 | { |
1571 | .handler = smc_llc_rx_handler, | |
1572 | .type = SMC_LLC_DELETE_LINK | |
1573 | }, | |
4ed75de5 KG |
1574 | { |
1575 | .handler = smc_llc_rx_handler, | |
1576 | .type = SMC_LLC_CONFIRM_RKEY | |
1577 | }, | |
1578 | { | |
1579 | .handler = smc_llc_rx_handler, | |
1580 | .type = SMC_LLC_CONFIRM_RKEY_CONT | |
1581 | }, | |
1582 | { | |
1583 | .handler = smc_llc_rx_handler, | |
1584 | .type = SMC_LLC_DELETE_RKEY | |
1585 | }, | |
9bf9abea UB |
1586 | { |
1587 | .handler = NULL, | |
1588 | } | |
1589 | }; | |
1590 | ||
1591 | int __init smc_llc_init(void) | |
1592 | { | |
1593 | struct smc_wr_rx_handler *handler; | |
1594 | int rc = 0; | |
1595 | ||
1596 | for (handler = smc_llc_rx_handlers; handler->handler; handler++) { | |
1597 | INIT_HLIST_NODE(&handler->list); | |
1598 | rc = smc_wr_rx_register_handler(handler); | |
1599 | if (rc) | |
1600 | break; | |
1601 | } | |
1602 | return rc; | |
1603 | } |