Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9bf9abea UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Link Layer Control (LLC) | |
6 | * | |
9bf9abea UB |
7 | * Copyright IBM Corp. 2016 |
8 | * | |
9 | * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> | |
10 | * Ursula Braun <ubraun@linux.vnet.ibm.com> | |
11 | */ | |
12 | ||
13 | #include <net/tcp.h> | |
14 | #include <rdma/ib_verbs.h> | |
15 | ||
16 | #include "smc.h" | |
17 | #include "smc_core.h" | |
18 | #include "smc_clc.h" | |
19 | #include "smc_llc.h" | |
20 | ||
0f627126 SR |
21 | #define SMC_LLC_DATA_LEN 40 |
22 | ||
23 | struct smc_llc_hdr { | |
24 | struct smc_wr_rx_hdr common; | |
25 | u8 length; /* 44 */ | |
52bedf37 KG |
26 | #if defined(__BIG_ENDIAN_BITFIELD) |
27 | u8 reserved:4, | |
28 | add_link_rej_rsn:4; | |
29 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
30 | u8 add_link_rej_rsn:4, | |
31 | reserved:4; | |
32 | #endif | |
0f627126 SR |
33 | u8 flags; |
34 | }; | |
35 | ||
75d320d6 KG |
36 | #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 |
37 | ||
0f627126 SR |
38 | struct smc_llc_msg_confirm_link { /* type 0x01 */ |
39 | struct smc_llc_hdr hd; | |
40 | u8 sender_mac[ETH_ALEN]; | |
41 | u8 sender_gid[SMC_GID_SIZE]; | |
42 | u8 sender_qp_num[3]; | |
43 | u8 link_num; | |
44 | u8 link_uid[SMC_LGR_ID_SIZE]; | |
45 | u8 max_links; | |
46 | u8 reserved[9]; | |
47 | }; | |
48 | ||
52bedf37 KG |
49 | #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 |
50 | #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 | |
51 | ||
52 | #define SMC_LLC_ADD_LNK_MAX_LINKS 2 | |
53 | ||
54 | struct smc_llc_msg_add_link { /* type 0x02 */ | |
55 | struct smc_llc_hdr hd; | |
56 | u8 sender_mac[ETH_ALEN]; | |
57 | u8 reserved2[2]; | |
58 | u8 sender_gid[SMC_GID_SIZE]; | |
59 | u8 sender_qp_num[3]; | |
60 | u8 link_num; | |
61 | u8 flags2; /* QP mtu */ | |
62 | u8 initial_psn[3]; | |
63 | u8 reserved[8]; | |
64 | }; | |
65 | ||
66 | #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 | |
67 | #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 | |
68 | ||
69 | struct smc_llc_msg_del_link { /* type 0x04 */ | |
70 | struct smc_llc_hdr hd; | |
71 | u8 link_num; | |
72 | __be32 reason; | |
73 | u8 reserved[35]; | |
74 | } __packed; /* format defined in RFC7609 */ | |
75 | ||
313164da KG |
76 | struct smc_llc_msg_test_link { /* type 0x07 */ |
77 | struct smc_llc_hdr hd; | |
78 | u8 user_data[16]; | |
79 | u8 reserved[24]; | |
80 | }; | |
81 | ||
4ed75de5 KG |
82 | struct smc_rmb_rtoken { |
83 | union { | |
84 | u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ | |
85 | /* is actually the num of rtokens, first */ | |
86 | /* rtoken is always for the current link */ | |
87 | u8 link_id; /* link id of the rtoken */ | |
88 | }; | |
89 | __be32 rmb_key; | |
90 | __be64 rmb_vaddr; | |
91 | } __packed; /* format defined in RFC7609 */ | |
92 | ||
93 | #define SMC_LLC_RKEYS_PER_MSG 3 | |
94 | ||
95 | struct smc_llc_msg_confirm_rkey { /* type 0x06 */ | |
96 | struct smc_llc_hdr hd; | |
97 | struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; | |
98 | u8 reserved; | |
99 | }; | |
100 | ||
101 | struct smc_llc_msg_confirm_rkey_cont { /* type 0x08 */ | |
102 | struct smc_llc_hdr hd; | |
103 | u8 num_rkeys; | |
104 | struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; | |
105 | }; | |
106 | ||
107 | #define SMC_LLC_DEL_RKEY_MAX 8 | |
108 | #define SMC_LLC_FLAG_RKEY_NEG 0x20 | |
109 | ||
110 | struct smc_llc_msg_delete_rkey { /* type 0x09 */ | |
111 | struct smc_llc_hdr hd; | |
112 | u8 num_rkeys; | |
113 | u8 err_mask; | |
114 | u8 reserved[2]; | |
115 | __be32 rkey[8]; | |
116 | u8 reserved2[4]; | |
117 | }; | |
118 | ||
0f627126 SR |
119 | union smc_llc_msg { |
120 | struct smc_llc_msg_confirm_link confirm_link; | |
52bedf37 KG |
121 | struct smc_llc_msg_add_link add_link; |
122 | struct smc_llc_msg_del_link delete_link; | |
4ed75de5 KG |
123 | |
124 | struct smc_llc_msg_confirm_rkey confirm_rkey; | |
125 | struct smc_llc_msg_confirm_rkey_cont confirm_rkey_cont; | |
126 | struct smc_llc_msg_delete_rkey delete_rkey; | |
127 | ||
313164da | 128 | struct smc_llc_msg_test_link test_link; |
0f627126 SR |
129 | struct { |
130 | struct smc_llc_hdr hdr; | |
131 | u8 data[SMC_LLC_DATA_LEN]; | |
132 | } raw; | |
133 | }; | |
134 | ||
135 | #define SMC_LLC_FLAG_RESP 0x80 | |
136 | ||
9bf9abea UB |
137 | /********************************** send *************************************/ |
138 | ||
139 | struct smc_llc_tx_pend { | |
140 | }; | |
141 | ||
142 | /* handler for send/transmission completion of an LLC msg */ | |
143 | static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, | |
144 | struct smc_link *link, | |
145 | enum ib_wc_status wc_status) | |
146 | { | |
147 | /* future work: handle wc_status error for recovery and failover */ | |
148 | } | |
149 | ||
150 | /** | |
151 | * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits | |
152 | * @link: Pointer to SMC link used for sending LLC control message. | |
153 | * @wr_buf: Out variable returning pointer to work request payload buffer. | |
154 | * @pend: Out variable returning pointer to private pending WR tracking. | |
155 | * It's the context the transmit complete handler will get. | |
156 | * | |
157 | * Reserves and pre-fills an entry for a pending work request send/tx. | |
158 | * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. | |
159 | * Can sleep due to smc_get_ctrl_buf (if not in softirq context). | |
160 | * | |
161 | * Return: 0 on success, otherwise an error value. | |
162 | */ | |
163 | static int smc_llc_add_pending_send(struct smc_link *link, | |
164 | struct smc_wr_buf **wr_buf, | |
165 | struct smc_wr_tx_pend_priv **pend) | |
166 | { | |
167 | int rc; | |
168 | ||
ad6f317f UB |
169 | rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, |
170 | pend); | |
9bf9abea UB |
171 | if (rc < 0) |
172 | return rc; | |
173 | BUILD_BUG_ON_MSG( | |
174 | sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, | |
175 | "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); | |
176 | BUILD_BUG_ON_MSG( | |
177 | sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, | |
178 | "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); | |
179 | BUILD_BUG_ON_MSG( | |
180 | sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, | |
181 | "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); | |
182 | return 0; | |
183 | } | |
184 | ||
185 | /* high-level API to send LLC confirm link */ | |
947541f3 | 186 | int smc_llc_send_confirm_link(struct smc_link *link, |
9bf9abea UB |
187 | enum smc_llc_reqresp reqresp) |
188 | { | |
00e5fb26 | 189 | struct smc_link_group *lgr = smc_get_lgr(link); |
9bf9abea UB |
190 | struct smc_llc_msg_confirm_link *confllc; |
191 | struct smc_wr_tx_pend_priv *pend; | |
192 | struct smc_wr_buf *wr_buf; | |
193 | int rc; | |
194 | ||
195 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
196 | if (rc) | |
197 | return rc; | |
198 | confllc = (struct smc_llc_msg_confirm_link *)wr_buf; | |
199 | memset(confllc, 0, sizeof(*confllc)); | |
200 | confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; | |
201 | confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); | |
75d320d6 | 202 | confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; |
9bf9abea UB |
203 | if (reqresp == SMC_LLC_RESP) |
204 | confllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
947541f3 UB |
205 | memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], |
206 | ETH_ALEN); | |
7005ada6 | 207 | memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); |
9bf9abea | 208 | hton24(confllc->sender_qp_num, link->roce_qp->qp_num); |
2be922f3 | 209 | confllc->link_num = link->link_id; |
9bf9abea | 210 | memcpy(confllc->link_uid, lgr->id, SMC_LGR_ID_SIZE); |
52bedf37 KG |
211 | confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; /* enforce peer resp. */ |
212 | /* send llc message */ | |
213 | rc = smc_wr_tx_send(link, pend); | |
214 | return rc; | |
215 | } | |
216 | ||
44aa81ce KG |
217 | /* send LLC confirm rkey request */ |
218 | static int smc_llc_send_confirm_rkey(struct smc_link *link, | |
219 | struct smc_buf_desc *rmb_desc) | |
220 | { | |
221 | struct smc_llc_msg_confirm_rkey *rkeyllc; | |
222 | struct smc_wr_tx_pend_priv *pend; | |
223 | struct smc_wr_buf *wr_buf; | |
224 | int rc; | |
225 | ||
226 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
227 | if (rc) | |
228 | return rc; | |
229 | rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; | |
230 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
231 | rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; | |
232 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); | |
233 | rkeyllc->rtoken[0].rmb_key = | |
234 | htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey); | |
235 | rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( | |
236 | (u64)sg_dma_address(rmb_desc->sgt[SMC_SINGLE_LINK].sgl)); | |
237 | /* send llc message */ | |
238 | rc = smc_wr_tx_send(link, pend); | |
239 | return rc; | |
240 | } | |
241 | ||
60e03c62 KG |
242 | /* send LLC delete rkey request */ |
243 | static int smc_llc_send_delete_rkey(struct smc_link *link, | |
244 | struct smc_buf_desc *rmb_desc) | |
245 | { | |
246 | struct smc_llc_msg_delete_rkey *rkeyllc; | |
247 | struct smc_wr_tx_pend_priv *pend; | |
248 | struct smc_wr_buf *wr_buf; | |
249 | int rc; | |
250 | ||
251 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
252 | if (rc) | |
253 | return rc; | |
254 | rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; | |
255 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
256 | rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; | |
257 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); | |
258 | rkeyllc->num_rkeys = 1; | |
259 | rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[SMC_SINGLE_LINK]->rkey); | |
260 | /* send llc message */ | |
261 | rc = smc_wr_tx_send(link, pend); | |
262 | return rc; | |
263 | } | |
264 | ||
2a4c57a9 KG |
265 | /* prepare an add link message */ |
266 | static void smc_llc_prep_add_link(struct smc_llc_msg_add_link *addllc, | |
7005ada6 | 267 | struct smc_link *link, u8 mac[], u8 gid[], |
2a4c57a9 KG |
268 | enum smc_llc_reqresp reqresp) |
269 | { | |
270 | memset(addllc, 0, sizeof(*addllc)); | |
271 | addllc->hd.common.type = SMC_LLC_ADD_LINK; | |
272 | addllc->hd.length = sizeof(struct smc_llc_msg_add_link); | |
273 | if (reqresp == SMC_LLC_RESP) { | |
274 | addllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
275 | /* always reject more links for now */ | |
276 | addllc->hd.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; | |
277 | addllc->hd.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; | |
278 | } | |
279 | memcpy(addllc->sender_mac, mac, ETH_ALEN); | |
280 | memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); | |
281 | } | |
282 | ||
52bedf37 | 283 | /* send ADD LINK request or response */ |
7005ada6 | 284 | int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], |
52bedf37 KG |
285 | enum smc_llc_reqresp reqresp) |
286 | { | |
287 | struct smc_llc_msg_add_link *addllc; | |
288 | struct smc_wr_tx_pend_priv *pend; | |
289 | struct smc_wr_buf *wr_buf; | |
290 | int rc; | |
291 | ||
292 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
293 | if (rc) | |
294 | return rc; | |
295 | addllc = (struct smc_llc_msg_add_link *)wr_buf; | |
2a4c57a9 | 296 | smc_llc_prep_add_link(addllc, link, mac, gid, reqresp); |
52bedf37 KG |
297 | /* send llc message */ |
298 | rc = smc_wr_tx_send(link, pend); | |
299 | return rc; | |
300 | } | |
301 | ||
2a4c57a9 KG |
302 | /* prepare a delete link message */ |
303 | static void smc_llc_prep_delete_link(struct smc_llc_msg_del_link *delllc, | |
304 | struct smc_link *link, | |
0d18a0cb | 305 | enum smc_llc_reqresp reqresp, bool orderly) |
2a4c57a9 KG |
306 | { |
307 | memset(delllc, 0, sizeof(*delllc)); | |
308 | delllc->hd.common.type = SMC_LLC_DELETE_LINK; | |
309 | delllc->hd.length = sizeof(struct smc_llc_msg_add_link); | |
310 | if (reqresp == SMC_LLC_RESP) | |
311 | delllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
312 | /* DEL_LINK_ALL because only 1 link supported */ | |
313 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; | |
0d18a0cb KG |
314 | if (orderly) |
315 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
2a4c57a9 KG |
316 | delllc->link_num = link->link_id; |
317 | } | |
318 | ||
52bedf37 KG |
319 | /* send DELETE LINK request or response */ |
320 | int smc_llc_send_delete_link(struct smc_link *link, | |
0d18a0cb | 321 | enum smc_llc_reqresp reqresp, bool orderly) |
52bedf37 KG |
322 | { |
323 | struct smc_llc_msg_del_link *delllc; | |
324 | struct smc_wr_tx_pend_priv *pend; | |
325 | struct smc_wr_buf *wr_buf; | |
326 | int rc; | |
327 | ||
328 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
329 | if (rc) | |
330 | return rc; | |
331 | delllc = (struct smc_llc_msg_del_link *)wr_buf; | |
0d18a0cb | 332 | smc_llc_prep_delete_link(delllc, link, reqresp, orderly); |
9bf9abea UB |
333 | /* send llc message */ |
334 | rc = smc_wr_tx_send(link, pend); | |
335 | return rc; | |
336 | } | |
337 | ||
d97935fa KG |
338 | /* send LLC test link request */ |
339 | static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) | |
313164da KG |
340 | { |
341 | struct smc_llc_msg_test_link *testllc; | |
342 | struct smc_wr_tx_pend_priv *pend; | |
343 | struct smc_wr_buf *wr_buf; | |
344 | int rc; | |
345 | ||
346 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
347 | if (rc) | |
348 | return rc; | |
349 | testllc = (struct smc_llc_msg_test_link *)wr_buf; | |
350 | memset(testllc, 0, sizeof(*testllc)); | |
351 | testllc->hd.common.type = SMC_LLC_TEST_LINK; | |
352 | testllc->hd.length = sizeof(struct smc_llc_msg_test_link); | |
313164da KG |
353 | memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); |
354 | /* send llc message */ | |
355 | rc = smc_wr_tx_send(link, pend); | |
356 | return rc; | |
357 | } | |
358 | ||
2a4c57a9 KG |
359 | struct smc_llc_send_work { |
360 | struct work_struct work; | |
361 | struct smc_link *link; | |
362 | int llclen; | |
363 | union smc_llc_msg llcbuf; | |
364 | }; | |
365 | ||
366 | /* worker that sends a prepared message */ | |
367 | static void smc_llc_send_message_work(struct work_struct *work) | |
4ed75de5 | 368 | { |
2a4c57a9 KG |
369 | struct smc_llc_send_work *llcwrk = container_of(work, |
370 | struct smc_llc_send_work, work); | |
4ed75de5 KG |
371 | struct smc_wr_tx_pend_priv *pend; |
372 | struct smc_wr_buf *wr_buf; | |
373 | int rc; | |
374 | ||
2a4c57a9 KG |
375 | if (llcwrk->link->state == SMC_LNK_INACTIVE) |
376 | goto out; | |
377 | rc = smc_llc_add_pending_send(llcwrk->link, &wr_buf, &pend); | |
4ed75de5 | 378 | if (rc) |
2a4c57a9 KG |
379 | goto out; |
380 | memcpy(wr_buf, &llcwrk->llcbuf, llcwrk->llclen); | |
381 | smc_wr_tx_send(llcwrk->link, pend); | |
382 | out: | |
383 | kfree(llcwrk); | |
384 | } | |
385 | ||
386 | /* copy llcbuf and schedule an llc send on link */ | |
387 | static int smc_llc_send_message(struct smc_link *link, void *llcbuf, int llclen) | |
388 | { | |
389 | struct smc_llc_send_work *wrk = kmalloc(sizeof(*wrk), GFP_ATOMIC); | |
390 | ||
391 | if (!wrk) | |
392 | return -ENOMEM; | |
393 | INIT_WORK(&wrk->work, smc_llc_send_message_work); | |
394 | wrk->link = link; | |
395 | wrk->llclen = llclen; | |
396 | memcpy(&wrk->llcbuf, llcbuf, llclen); | |
397 | queue_work(link->llc_wq, &wrk->work); | |
398 | return 0; | |
4ed75de5 KG |
399 | } |
400 | ||
9bf9abea UB |
401 | /********************************* receive ***********************************/ |
402 | ||
403 | static void smc_llc_rx_confirm_link(struct smc_link *link, | |
404 | struct smc_llc_msg_confirm_link *llc) | |
405 | { | |
00e5fb26 | 406 | struct smc_link_group *lgr = smc_get_lgr(link); |
75d320d6 | 407 | int conf_rc; |
9bf9abea | 408 | |
75d320d6 KG |
409 | /* RMBE eyecatchers are not supported */ |
410 | if (llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC) | |
411 | conf_rc = 0; | |
412 | else | |
413 | conf_rc = ENOTSUPP; | |
414 | ||
9bf9abea | 415 | if (llc->hd.flags & SMC_LLC_FLAG_RESP) { |
52bedf37 KG |
416 | if (lgr->role == SMC_SERV && |
417 | link->state == SMC_LNK_ACTIVATING) { | |
75d320d6 | 418 | link->llc_confirm_resp_rc = conf_rc; |
9bf9abea | 419 | complete(&link->llc_confirm_resp); |
75d320d6 | 420 | } |
9bf9abea | 421 | } else { |
52bedf37 KG |
422 | if (lgr->role == SMC_CLNT && |
423 | link->state == SMC_LNK_ACTIVATING) { | |
75d320d6 | 424 | link->llc_confirm_rc = conf_rc; |
9bf9abea UB |
425 | link->link_id = llc->link_num; |
426 | complete(&link->llc_confirm); | |
427 | } | |
428 | } | |
429 | } | |
430 | ||
52bedf37 KG |
431 | static void smc_llc_rx_add_link(struct smc_link *link, |
432 | struct smc_llc_msg_add_link *llc) | |
433 | { | |
00e5fb26 | 434 | struct smc_link_group *lgr = smc_get_lgr(link); |
52bedf37 KG |
435 | |
436 | if (llc->hd.flags & SMC_LLC_FLAG_RESP) { | |
437 | if (link->state == SMC_LNK_ACTIVATING) | |
438 | complete(&link->llc_add_resp); | |
439 | } else { | |
440 | if (link->state == SMC_LNK_ACTIVATING) { | |
441 | complete(&link->llc_add); | |
442 | return; | |
443 | } | |
444 | ||
445 | if (lgr->role == SMC_SERV) { | |
2a4c57a9 | 446 | smc_llc_prep_add_link(llc, link, |
52bedf37 | 447 | link->smcibdev->mac[link->ibport - 1], |
7005ada6 | 448 | link->gid, SMC_LLC_REQ); |
52bedf37 KG |
449 | |
450 | } else { | |
2a4c57a9 | 451 | smc_llc_prep_add_link(llc, link, |
52bedf37 | 452 | link->smcibdev->mac[link->ibport - 1], |
7005ada6 | 453 | link->gid, SMC_LLC_RESP); |
52bedf37 | 454 | } |
2a4c57a9 | 455 | smc_llc_send_message(link, llc, sizeof(*llc)); |
52bedf37 KG |
456 | } |
457 | } | |
458 | ||
459 | static void smc_llc_rx_delete_link(struct smc_link *link, | |
460 | struct smc_llc_msg_del_link *llc) | |
461 | { | |
00e5fb26 | 462 | struct smc_link_group *lgr = smc_get_lgr(link); |
52bedf37 KG |
463 | |
464 | if (llc->hd.flags & SMC_LLC_FLAG_RESP) { | |
465 | if (lgr->role == SMC_SERV) | |
0d18a0cb | 466 | smc_lgr_schedule_free_work_fast(lgr); |
52bedf37 | 467 | } else { |
0d18a0cb KG |
468 | smc_lgr_forget(lgr); |
469 | smc_llc_link_deleting(link); | |
52bedf37 | 470 | if (lgr->role == SMC_SERV) { |
0d18a0cb KG |
471 | /* client asks to delete this link, send request */ |
472 | smc_llc_prep_delete_link(llc, link, SMC_LLC_REQ, true); | |
52bedf37 | 473 | } else { |
0d18a0cb KG |
474 | /* server requests to delete this link, send response */ |
475 | smc_llc_prep_delete_link(llc, link, SMC_LLC_RESP, true); | |
52bedf37 | 476 | } |
0d18a0cb KG |
477 | smc_llc_send_message(link, llc, sizeof(*llc)); |
478 | smc_lgr_schedule_free_work_fast(lgr); | |
52bedf37 KG |
479 | } |
480 | } | |
481 | ||
313164da KG |
482 | static void smc_llc_rx_test_link(struct smc_link *link, |
483 | struct smc_llc_msg_test_link *llc) | |
484 | { | |
485 | if (llc->hd.flags & SMC_LLC_FLAG_RESP) { | |
877ae5be KG |
486 | if (link->state == SMC_LNK_ACTIVE) |
487 | complete(&link->llc_testlink_resp); | |
313164da | 488 | } else { |
d97935fa KG |
489 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
490 | smc_llc_send_message(link, llc, sizeof(*llc)); | |
313164da KG |
491 | } |
492 | } | |
493 | ||
4ed75de5 KG |
494 | static void smc_llc_rx_confirm_rkey(struct smc_link *link, |
495 | struct smc_llc_msg_confirm_rkey *llc) | |
496 | { | |
4ed75de5 KG |
497 | int rc; |
498 | ||
4ed75de5 | 499 | if (llc->hd.flags & SMC_LLC_FLAG_RESP) { |
44aa81ce KG |
500 | link->llc_confirm_rkey_rc = llc->hd.flags & |
501 | SMC_LLC_FLAG_RKEY_NEG; | |
502 | complete(&link->llc_confirm_rkey); | |
4ed75de5 | 503 | } else { |
00e5fb26 | 504 | rc = smc_rtoken_add(smc_get_lgr(link), |
4ed75de5 KG |
505 | llc->rtoken[0].rmb_vaddr, |
506 | llc->rtoken[0].rmb_key); | |
507 | ||
508 | /* ignore rtokens for other links, we have only one link */ | |
509 | ||
510 | llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
511 | if (rc < 0) | |
512 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
9fcdf8e9 | 513 | smc_llc_send_message(link, llc, sizeof(*llc)); |
4ed75de5 KG |
514 | } |
515 | } | |
516 | ||
517 | static void smc_llc_rx_confirm_rkey_cont(struct smc_link *link, | |
518 | struct smc_llc_msg_confirm_rkey_cont *llc) | |
519 | { | |
520 | if (llc->hd.flags & SMC_LLC_FLAG_RESP) { | |
521 | /* unused as long as we don't send this type of msg */ | |
522 | } else { | |
523 | /* ignore rtokens for other links, we have only one link */ | |
524 | llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
9fcdf8e9 | 525 | smc_llc_send_message(link, llc, sizeof(*llc)); |
4ed75de5 KG |
526 | } |
527 | } | |
528 | ||
529 | static void smc_llc_rx_delete_rkey(struct smc_link *link, | |
530 | struct smc_llc_msg_delete_rkey *llc) | |
531 | { | |
4ed75de5 KG |
532 | u8 err_mask = 0; |
533 | int i, max; | |
534 | ||
4ed75de5 | 535 | if (llc->hd.flags & SMC_LLC_FLAG_RESP) { |
60e03c62 KG |
536 | link->llc_delete_rkey_rc = llc->hd.flags & |
537 | SMC_LLC_FLAG_RKEY_NEG; | |
538 | complete(&link->llc_delete_rkey); | |
4ed75de5 KG |
539 | } else { |
540 | max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); | |
541 | for (i = 0; i < max; i++) { | |
00e5fb26 | 542 | if (smc_rtoken_delete(smc_get_lgr(link), llc->rkey[i])) |
4ed75de5 KG |
543 | err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); |
544 | } | |
545 | ||
546 | if (err_mask) { | |
547 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
548 | llc->err_mask = err_mask; | |
549 | } | |
550 | ||
551 | llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
9fcdf8e9 | 552 | smc_llc_send_message(link, llc, sizeof(*llc)); |
4ed75de5 KG |
553 | } |
554 | } | |
555 | ||
9bf9abea UB |
556 | static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) |
557 | { | |
558 | struct smc_link *link = (struct smc_link *)wc->qp->qp_context; | |
559 | union smc_llc_msg *llc = buf; | |
560 | ||
561 | if (wc->byte_len < sizeof(*llc)) | |
562 | return; /* short message */ | |
563 | if (llc->raw.hdr.length != sizeof(*llc)) | |
564 | return; /* invalid message */ | |
8f332a74 KG |
565 | if (link->state == SMC_LNK_INACTIVE) |
566 | return; /* link not active, drop msg */ | |
313164da KG |
567 | |
568 | switch (llc->raw.hdr.common.type) { | |
569 | case SMC_LLC_TEST_LINK: | |
570 | smc_llc_rx_test_link(link, &llc->test_link); | |
571 | break; | |
572 | case SMC_LLC_CONFIRM_LINK: | |
9bf9abea | 573 | smc_llc_rx_confirm_link(link, &llc->confirm_link); |
313164da | 574 | break; |
52bedf37 KG |
575 | case SMC_LLC_ADD_LINK: |
576 | smc_llc_rx_add_link(link, &llc->add_link); | |
577 | break; | |
578 | case SMC_LLC_DELETE_LINK: | |
579 | smc_llc_rx_delete_link(link, &llc->delete_link); | |
580 | break; | |
4ed75de5 KG |
581 | case SMC_LLC_CONFIRM_RKEY: |
582 | smc_llc_rx_confirm_rkey(link, &llc->confirm_rkey); | |
583 | break; | |
584 | case SMC_LLC_CONFIRM_RKEY_CONT: | |
585 | smc_llc_rx_confirm_rkey_cont(link, &llc->confirm_rkey_cont); | |
586 | break; | |
587 | case SMC_LLC_DELETE_RKEY: | |
588 | smc_llc_rx_delete_rkey(link, &llc->delete_rkey); | |
589 | break; | |
313164da | 590 | } |
9bf9abea UB |
591 | } |
592 | ||
44aa81ce | 593 | /***************************** worker, utils *********************************/ |
877ae5be KG |
594 | |
595 | static void smc_llc_testlink_work(struct work_struct *work) | |
596 | { | |
597 | struct smc_link *link = container_of(to_delayed_work(work), | |
598 | struct smc_link, llc_testlink_wrk); | |
599 | unsigned long next_interval; | |
877ae5be KG |
600 | unsigned long expire_time; |
601 | u8 user_data[16] = { 0 }; | |
602 | int rc; | |
603 | ||
877ae5be KG |
604 | if (link->state != SMC_LNK_ACTIVE) |
605 | return; /* don't reschedule worker */ | |
606 | expire_time = link->wr_rx_tstamp + link->llc_testlink_time; | |
607 | if (time_is_after_jiffies(expire_time)) { | |
608 | next_interval = expire_time - jiffies; | |
609 | goto out; | |
610 | } | |
611 | reinit_completion(&link->llc_testlink_resp); | |
d97935fa | 612 | smc_llc_send_test_link(link, user_data); |
877ae5be KG |
613 | /* receive TEST LINK response over RoCE fabric */ |
614 | rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, | |
615 | SMC_LLC_WAIT_TIME); | |
616 | if (rc <= 0) { | |
00e5fb26 | 617 | smc_lgr_terminate(smc_get_lgr(link)); |
877ae5be KG |
618 | return; |
619 | } | |
620 | next_interval = link->llc_testlink_time; | |
621 | out: | |
2a4c57a9 KG |
622 | queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk, |
623 | next_interval); | |
877ae5be KG |
624 | } |
625 | ||
2a4c57a9 | 626 | int smc_llc_link_init(struct smc_link *link) |
877ae5be | 627 | { |
00e5fb26 | 628 | struct smc_link_group *lgr = smc_get_lgr(link); |
2a4c57a9 KG |
629 | link->llc_wq = alloc_ordered_workqueue("llc_wq-%x:%x)", WQ_MEM_RECLAIM, |
630 | *((u32 *)lgr->id), | |
631 | link->link_id); | |
632 | if (!link->llc_wq) | |
633 | return -ENOMEM; | |
b32cf4ab KG |
634 | init_completion(&link->llc_confirm); |
635 | init_completion(&link->llc_confirm_resp); | |
636 | init_completion(&link->llc_add); | |
637 | init_completion(&link->llc_add_resp); | |
638 | init_completion(&link->llc_confirm_rkey); | |
60e03c62 KG |
639 | init_completion(&link->llc_delete_rkey); |
640 | mutex_init(&link->llc_delete_rkey_mutex); | |
877ae5be KG |
641 | init_completion(&link->llc_testlink_resp); |
642 | INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); | |
2a4c57a9 | 643 | return 0; |
b32cf4ab KG |
644 | } |
645 | ||
646 | void smc_llc_link_active(struct smc_link *link, int testlink_time) | |
647 | { | |
877ae5be KG |
648 | link->state = SMC_LNK_ACTIVE; |
649 | if (testlink_time) { | |
650 | link->llc_testlink_time = testlink_time * HZ; | |
2a4c57a9 KG |
651 | queue_delayed_work(link->llc_wq, &link->llc_testlink_wrk, |
652 | link->llc_testlink_time); | |
877ae5be KG |
653 | } |
654 | } | |
655 | ||
0d18a0cb KG |
656 | void smc_llc_link_deleting(struct smc_link *link) |
657 | { | |
658 | link->state = SMC_LNK_DELETING; | |
659 | } | |
660 | ||
877ae5be KG |
661 | /* called in tasklet context */ |
662 | void smc_llc_link_inactive(struct smc_link *link) | |
663 | { | |
664 | link->state = SMC_LNK_INACTIVE; | |
665 | cancel_delayed_work(&link->llc_testlink_wrk); | |
666 | } | |
667 | ||
668 | /* called in worker context */ | |
2a4c57a9 | 669 | void smc_llc_link_clear(struct smc_link *link) |
877ae5be | 670 | { |
2a4c57a9 KG |
671 | flush_workqueue(link->llc_wq); |
672 | destroy_workqueue(link->llc_wq); | |
877ae5be KG |
673 | } |
674 | ||
44aa81ce KG |
675 | /* register a new rtoken at the remote peer */ |
676 | int smc_llc_do_confirm_rkey(struct smc_link *link, | |
677 | struct smc_buf_desc *rmb_desc) | |
678 | { | |
679 | int rc; | |
680 | ||
60e03c62 | 681 | /* protected by mutex smc_create_lgr_pending */ |
44aa81ce | 682 | reinit_completion(&link->llc_confirm_rkey); |
4600cfc3 KG |
683 | rc = smc_llc_send_confirm_rkey(link, rmb_desc); |
684 | if (rc) | |
685 | return rc; | |
44aa81ce KG |
686 | /* receive CONFIRM RKEY response from server over RoCE fabric */ |
687 | rc = wait_for_completion_interruptible_timeout(&link->llc_confirm_rkey, | |
688 | SMC_LLC_WAIT_TIME); | |
689 | if (rc <= 0 || link->llc_confirm_rkey_rc) | |
690 | return -EFAULT; | |
691 | return 0; | |
692 | } | |
693 | ||
60e03c62 KG |
694 | /* unregister an rtoken at the remote peer */ |
695 | int smc_llc_do_delete_rkey(struct smc_link *link, | |
696 | struct smc_buf_desc *rmb_desc) | |
697 | { | |
698 | int rc; | |
699 | ||
700 | mutex_lock(&link->llc_delete_rkey_mutex); | |
701 | reinit_completion(&link->llc_delete_rkey); | |
702 | rc = smc_llc_send_delete_rkey(link, rmb_desc); | |
703 | if (rc) | |
704 | goto out; | |
705 | /* receive DELETE RKEY response from server over RoCE fabric */ | |
706 | rc = wait_for_completion_interruptible_timeout(&link->llc_delete_rkey, | |
707 | SMC_LLC_WAIT_TIME); | |
708 | if (rc <= 0 || link->llc_delete_rkey_rc) | |
709 | rc = -EFAULT; | |
710 | else | |
711 | rc = 0; | |
712 | out: | |
713 | mutex_unlock(&link->llc_delete_rkey_mutex); | |
714 | return rc; | |
715 | } | |
716 | ||
9bf9abea UB |
717 | /***************************** init, exit, misc ******************************/ |
718 | ||
719 | static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { | |
720 | { | |
721 | .handler = smc_llc_rx_handler, | |
722 | .type = SMC_LLC_CONFIRM_LINK | |
723 | }, | |
313164da KG |
724 | { |
725 | .handler = smc_llc_rx_handler, | |
726 | .type = SMC_LLC_TEST_LINK | |
727 | }, | |
52bedf37 KG |
728 | { |
729 | .handler = smc_llc_rx_handler, | |
730 | .type = SMC_LLC_ADD_LINK | |
731 | }, | |
732 | { | |
733 | .handler = smc_llc_rx_handler, | |
734 | .type = SMC_LLC_DELETE_LINK | |
735 | }, | |
4ed75de5 KG |
736 | { |
737 | .handler = smc_llc_rx_handler, | |
738 | .type = SMC_LLC_CONFIRM_RKEY | |
739 | }, | |
740 | { | |
741 | .handler = smc_llc_rx_handler, | |
742 | .type = SMC_LLC_CONFIRM_RKEY_CONT | |
743 | }, | |
744 | { | |
745 | .handler = smc_llc_rx_handler, | |
746 | .type = SMC_LLC_DELETE_RKEY | |
747 | }, | |
9bf9abea UB |
748 | { |
749 | .handler = NULL, | |
750 | } | |
751 | }; | |
752 | ||
753 | int __init smc_llc_init(void) | |
754 | { | |
755 | struct smc_wr_rx_handler *handler; | |
756 | int rc = 0; | |
757 | ||
758 | for (handler = smc_llc_rx_handlers; handler->handler; handler++) { | |
759 | INIT_HLIST_NODE(&handler->list); | |
760 | rc = smc_wr_rx_register_handler(handler); | |
761 | if (rc) | |
762 | break; | |
763 | } | |
764 | return rc; | |
765 | } |