Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5f08318f UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Connection Data Control (CDC) | |
6 | * handles flow control | |
7 | * | |
8 | * Copyright IBM Corp. 2016 | |
9 | * | |
10 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> | |
11 | */ | |
12 | ||
13 | #include <linux/spinlock.h> | |
14 | ||
15 | #include "smc.h" | |
16 | #include "smc_wr.h" | |
17 | #include "smc_cdc.h" | |
e6727f39 | 18 | #include "smc_tx.h" |
952310cc | 19 | #include "smc_rx.h" |
b38d7324 | 20 | #include "smc_close.h" |
cc0ab806 | 21 | #include "smc_ism.h" |
5f08318f UB |
22 | |
23 | /********************************** send *************************************/ | |
24 | ||
5f08318f UB |
25 | /* handler for send/transmission completion of a CDC msg */ |
26 | static void smc_cdc_tx_handler(struct smc_wr_tx_pend_priv *pnd_snd, | |
27 | struct smc_link *link, | |
28 | enum ib_wc_status wc_status) | |
29 | { | |
30 | struct smc_cdc_tx_pend *cdcpend = (struct smc_cdc_tx_pend *)pnd_snd; | |
bac6de7b | 31 | struct smc_connection *conn = cdcpend->conn; |
c5bf605b | 32 | struct smc_buf_desc *sndbuf_desc; |
5f08318f UB |
33 | struct smc_sock *smc; |
34 | int diff; | |
35 | ||
c5bf605b | 36 | sndbuf_desc = conn->sndbuf_desc; |
bac6de7b | 37 | smc = container_of(conn, struct smc_sock, conn); |
5f08318f | 38 | bh_lock_sock(&smc->sk); |
c5bf605b W |
39 | if (!wc_status && sndbuf_desc) { |
40 | diff = smc_curs_diff(sndbuf_desc->len, | |
5f08318f UB |
41 | &cdcpend->conn->tx_curs_fin, |
42 | &cdcpend->cursor); | |
43 | /* sndbuf_space is decreased in smc_sendmsg */ | |
44 | smp_mb__before_atomic(); | |
45 | atomic_add(diff, &cdcpend->conn->sndbuf_space); | |
69cb7dc0 | 46 | /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */ |
5f08318f | 47 | smp_mb__after_atomic(); |
bac6de7b | 48 | smc_curs_copy(&conn->tx_curs_fin, &cdcpend->cursor, conn); |
f0ec4f1d KG |
49 | smc_curs_copy(&conn->local_tx_ctrl_fin, &cdcpend->p_cursor, |
50 | conn); | |
51 | conn->tx_cdc_seq_fin = cdcpend->ctrl_seq; | |
5f08318f | 52 | } |
349d4312 | 53 | |
dcd2cf5f | 54 | if (atomic_dec_and_test(&conn->cdc_pend_tx_wr)) { |
6b88af83 DL |
55 | /* If user owns the sock_lock, mark the connection need sending. |
56 | * User context will later try to send when it release sock_lock | |
57 | * in smc_release_cb() | |
dcd2cf5f | 58 | */ |
6b88af83 DL |
59 | if (sock_owned_by_user(&smc->sk)) |
60 | conn->tx_in_release_sock = true; | |
61 | else | |
62 | smc_tx_pending(conn); | |
63 | ||
dcd2cf5f DL |
64 | if (unlikely(wq_has_sleeper(&conn->cdc_pend_tx_wq))) |
65 | wake_up(&conn->cdc_pend_tx_wq); | |
66 | } | |
349d4312 DL |
67 | WARN_ON(atomic_read(&conn->cdc_pend_tx_wr) < 0); |
68 | ||
e6727f39 | 69 | smc_tx_sndbuf_nonfull(smc); |
5f08318f UB |
70 | bh_unlock_sock(&smc->sk); |
71 | } | |
72 | ||
51957bc5 | 73 | int smc_cdc_get_free_slot(struct smc_connection *conn, |
c6f02ebe | 74 | struct smc_link *link, |
5f08318f | 75 | struct smc_wr_buf **wr_buf, |
ad6f317f | 76 | struct smc_rdma_wr **wr_rdma_buf, |
5f08318f UB |
77 | struct smc_cdc_tx_pend **pend) |
78 | { | |
1a0a04c7 | 79 | int rc; |
51957bc5 | 80 | |
1a0a04c7 | 81 | rc = smc_wr_tx_get_free_slot(link, smc_cdc_tx_handler, wr_buf, |
ad6f317f | 82 | wr_rdma_buf, |
1a0a04c7 | 83 | (struct smc_wr_tx_pend_priv **)pend); |
2bced6ae | 84 | if (conn->killed) { |
1a0a04c7 | 85 | /* abnormal termination */ |
2bced6ae KG |
86 | if (!rc) |
87 | smc_wr_tx_put_slot(link, | |
e225c9a5 | 88 | (struct smc_wr_tx_pend_priv *)(*pend)); |
1a0a04c7 | 89 | rc = -EPIPE; |
2bced6ae | 90 | } |
1a0a04c7 | 91 | return rc; |
5f08318f UB |
92 | } |
93 | ||
94 | static inline void smc_cdc_add_pending_send(struct smc_connection *conn, | |
95 | struct smc_cdc_tx_pend *pend) | |
96 | { | |
97 | BUILD_BUG_ON_MSG( | |
98 | sizeof(struct smc_cdc_msg) > SMC_WR_BUF_SIZE, | |
99 | "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_cdc_msg)"); | |
100 | BUILD_BUG_ON_MSG( | |
b9a22dd9 | 101 | offsetofend(struct smc_cdc_msg, reserved) > SMC_WR_TX_SIZE, |
5f08318f UB |
102 | "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_cdc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); |
103 | BUILD_BUG_ON_MSG( | |
104 | sizeof(struct smc_cdc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, | |
105 | "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_cdc_tx_pend)"); | |
106 | pend->conn = conn; | |
107 | pend->cursor = conn->tx_curs_sent; | |
108 | pend->p_cursor = conn->local_tx_ctrl.prod; | |
109 | pend->ctrl_seq = conn->tx_cdc_seq; | |
110 | } | |
111 | ||
112 | int smc_cdc_msg_send(struct smc_connection *conn, | |
113 | struct smc_wr_buf *wr_buf, | |
114 | struct smc_cdc_tx_pend *pend) | |
115 | { | |
387707fd | 116 | struct smc_link *link = conn->lnk; |
b8649efa | 117 | union smc_host_cursor cfed; |
5f08318f UB |
118 | int rc; |
119 | ||
5f08318f UB |
120 | smc_cdc_add_pending_send(conn, pend); |
121 | ||
122 | conn->tx_cdc_seq++; | |
123 | conn->local_tx_ctrl.seqno = conn->tx_cdc_seq; | |
ccc8ca9b | 124 | smc_host_msg_to_cdc((struct smc_cdc_msg *)wr_buf, conn, &cfed); |
349d4312 DL |
125 | |
126 | atomic_inc(&conn->cdc_pend_tx_wr); | |
127 | smp_mb__after_atomic(); /* Make sure cdc_pend_tx_wr added before post */ | |
128 | ||
5f08318f | 129 | rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend); |
4dff63c2 | 130 | if (!rc) { |
b8649efa | 131 | smc_curs_copy(&conn->rx_curs_confirmed, &cfed, conn); |
4dff63c2 | 132 | conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; |
f0ec4f1d KG |
133 | } else { |
134 | conn->tx_cdc_seq--; | |
135 | conn->local_tx_ctrl.seqno = conn->tx_cdc_seq; | |
349d4312 | 136 | atomic_dec(&conn->cdc_pend_tx_wr); |
4dff63c2 | 137 | } |
5f08318f UB |
138 | |
139 | return rc; | |
140 | } | |
141 | ||
29bd73db | 142 | /* send a validation msg indicating the move of a conn to an other QP link */ |
b8ded9de KG |
143 | int smcr_cdc_msg_send_validation(struct smc_connection *conn, |
144 | struct smc_cdc_tx_pend *pend, | |
145 | struct smc_wr_buf *wr_buf) | |
29bd73db KG |
146 | { |
147 | struct smc_host_cdc_msg *local = &conn->local_tx_ctrl; | |
148 | struct smc_link *link = conn->lnk; | |
29bd73db KG |
149 | struct smc_cdc_msg *peer; |
150 | int rc; | |
151 | ||
29bd73db KG |
152 | peer = (struct smc_cdc_msg *)wr_buf; |
153 | peer->common.type = local->common.type; | |
154 | peer->len = local->len; | |
155 | peer->seqno = htons(conn->tx_cdc_seq_fin); /* seqno last compl. tx */ | |
156 | peer->token = htonl(local->token); | |
157 | peer->prod_flags.failover_validation = 1; | |
158 | ||
349d4312 DL |
159 | /* We need to set pend->conn here to make sure smc_cdc_tx_handler() |
160 | * can handle properly | |
161 | */ | |
162 | smc_cdc_add_pending_send(conn, pend); | |
163 | ||
164 | atomic_inc(&conn->cdc_pend_tx_wr); | |
165 | smp_mb__after_atomic(); /* Make sure cdc_pend_tx_wr added before post */ | |
166 | ||
29bd73db | 167 | rc = smc_wr_tx_send(link, (struct smc_wr_tx_pend_priv *)pend); |
349d4312 DL |
168 | if (unlikely(rc)) |
169 | atomic_dec(&conn->cdc_pend_tx_wr); | |
170 | ||
29bd73db KG |
171 | return rc; |
172 | } | |
173 | ||
be244f28 | 174 | static int smcr_cdc_get_slot_and_msg_send(struct smc_connection *conn) |
5f08318f UB |
175 | { |
176 | struct smc_cdc_tx_pend *pend; | |
177 | struct smc_wr_buf *wr_buf; | |
c6f02ebe KG |
178 | struct smc_link *link; |
179 | bool again = false; | |
5f08318f UB |
180 | int rc; |
181 | ||
c6f02ebe KG |
182 | again: |
183 | link = conn->lnk; | |
95f7f3e7 KG |
184 | if (!smc_wr_tx_link_hold(link)) |
185 | return -ENOLINK; | |
c6f02ebe | 186 | rc = smc_cdc_get_free_slot(conn, link, &wr_buf, NULL, &pend); |
5f08318f | 187 | if (rc) |
95f7f3e7 | 188 | goto put_out; |
5f08318f | 189 | |
2dee25af | 190 | spin_lock_bh(&conn->send_lock); |
c6f02ebe KG |
191 | if (link != conn->lnk) { |
192 | /* link of connection changed, try again one time*/ | |
193 | spin_unlock_bh(&conn->send_lock); | |
194 | smc_wr_tx_put_slot(link, | |
195 | (struct smc_wr_tx_pend_priv *)pend); | |
95f7f3e7 | 196 | smc_wr_tx_link_put(link); |
c6f02ebe KG |
197 | if (again) |
198 | return -ENOLINK; | |
199 | again = true; | |
200 | goto again; | |
201 | } | |
2dee25af KG |
202 | rc = smc_cdc_msg_send(conn, wr_buf, pend); |
203 | spin_unlock_bh(&conn->send_lock); | |
95f7f3e7 KG |
204 | put_out: |
205 | smc_wr_tx_link_put(link); | |
2dee25af | 206 | return rc; |
5f08318f UB |
207 | } |
208 | ||
be244f28 HW |
209 | int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn) |
210 | { | |
211 | int rc; | |
212 | ||
ea89c6c0 WG |
213 | if (!smc_conn_lgr_valid(conn) || |
214 | (conn->lgr->is_smcd && conn->lgr->peer_shutdown)) | |
50c6b20e UB |
215 | return -EPIPE; |
216 | ||
be244f28 HW |
217 | if (conn->lgr->is_smcd) { |
218 | spin_lock_bh(&conn->send_lock); | |
219 | rc = smcd_cdc_msg_send(conn); | |
220 | spin_unlock_bh(&conn->send_lock); | |
221 | } else { | |
222 | rc = smcr_cdc_get_slot_and_msg_send(conn); | |
223 | } | |
224 | ||
225 | return rc; | |
226 | } | |
227 | ||
349d4312 | 228 | void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn) |
5f08318f | 229 | { |
349d4312 | 230 | wait_event(conn->cdc_pend_tx_wq, !atomic_read(&conn->cdc_pend_tx_wr)); |
5f08318f UB |
231 | } |
232 | ||
be244f28 HW |
233 | /* Send a SMC-D CDC header. |
234 | * This increments the free space available in our send buffer. | |
235 | * Also update the confirmed receive buffer with what was sent to the peer. | |
236 | */ | |
237 | int smcd_cdc_msg_send(struct smc_connection *conn) | |
238 | { | |
239 | struct smc_sock *smc = container_of(conn, struct smc_sock, conn); | |
b9a22dd9 | 240 | union smc_host_cursor curs; |
be244f28 HW |
241 | struct smcd_cdc_msg cdc; |
242 | int rc, diff; | |
243 | ||
244 | memset(&cdc, 0, sizeof(cdc)); | |
245 | cdc.common.type = SMC_CDC_MSG_TYPE; | |
b9a22dd9 UB |
246 | curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.prod.acurs); |
247 | cdc.prod.wrap = curs.wrap; | |
248 | cdc.prod.count = curs.count; | |
249 | curs.acurs.counter = atomic64_read(&conn->local_tx_ctrl.cons.acurs); | |
250 | cdc.cons.wrap = curs.wrap; | |
251 | cdc.cons.count = curs.count; | |
252 | cdc.cons.prod_flags = conn->local_tx_ctrl.prod_flags; | |
253 | cdc.cons.conn_state_flags = conn->local_tx_ctrl.conn_state_flags; | |
be244f28 HW |
254 | rc = smcd_tx_ism_write(conn, &cdc, sizeof(cdc), 0, 1); |
255 | if (rc) | |
256 | return rc; | |
b9a22dd9 | 257 | smc_curs_copy(&conn->rx_curs_confirmed, &curs, conn); |
4dff63c2 | 258 | conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0; |
cc0ab806 WG |
259 | |
260 | if (smc_ism_support_dmb_nocopy(conn->lgr->smcd)) | |
261 | /* if local sndbuf shares the same memory region with | |
262 | * peer DMB, then don't update the tx_curs_fin | |
263 | * and sndbuf_space until peer has consumed the data. | |
264 | */ | |
265 | return 0; | |
266 | ||
be244f28 HW |
267 | /* Calculate transmitted data and increment free send buffer space */ |
268 | diff = smc_curs_diff(conn->sndbuf_desc->len, &conn->tx_curs_fin, | |
269 | &conn->tx_curs_sent); | |
270 | /* increased by confirmed number of bytes */ | |
271 | smp_mb__before_atomic(); | |
272 | atomic_add(diff, &conn->sndbuf_space); | |
273 | /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */ | |
274 | smp_mb__after_atomic(); | |
bac6de7b | 275 | smc_curs_copy(&conn->tx_curs_fin, &conn->tx_curs_sent, conn); |
be244f28 HW |
276 | |
277 | smc_tx_sndbuf_nonfull(smc); | |
cc0ab806 | 278 | return 0; |
be244f28 HW |
279 | } |
280 | ||
5f08318f UB |
281 | /********************************* receive ***********************************/ |
282 | ||
283 | static inline bool smc_cdc_before(u16 seq1, u16 seq2) | |
284 | { | |
285 | return (s16)(seq1 - seq2) < 0; | |
286 | } | |
287 | ||
de8474eb SR |
288 | static void smc_cdc_handle_urg_data_arrival(struct smc_sock *smc, |
289 | int *diff_prod) | |
290 | { | |
291 | struct smc_connection *conn = &smc->conn; | |
292 | char *base; | |
293 | ||
294 | /* new data included urgent business */ | |
bac6de7b | 295 | smc_curs_copy(&conn->urg_curs, &conn->local_rx_ctrl.prod, conn); |
de8474eb SR |
296 | conn->urg_state = SMC_URG_VALID; |
297 | if (!sock_flag(&smc->sk, SOCK_URGINLINE)) | |
298 | /* we'll skip the urgent byte, so don't account for it */ | |
299 | (*diff_prod)--; | |
be244f28 | 300 | base = (char *)conn->rmb_desc->cpu_addr + conn->rx_off; |
de8474eb SR |
301 | if (conn->urg_curs.count) |
302 | conn->urg_rx_byte = *(base + conn->urg_curs.count - 1); | |
303 | else | |
304 | conn->urg_rx_byte = *(base + conn->rmb_desc->len - 1); | |
305 | sk_send_sigurg(&smc->sk); | |
306 | } | |
307 | ||
b286a065 KG |
308 | static void smc_cdc_msg_validate(struct smc_sock *smc, struct smc_cdc_msg *cdc, |
309 | struct smc_link *link) | |
310 | { | |
311 | struct smc_connection *conn = &smc->conn; | |
312 | u16 recv_seq = ntohs(cdc->seqno); | |
313 | s16 diff; | |
314 | ||
315 | /* check that seqnum was seen before */ | |
316 | diff = conn->local_rx_ctrl.seqno - recv_seq; | |
317 | if (diff < 0) { /* diff larger than 0x7fff */ | |
318 | /* drop connection */ | |
319 | conn->out_of_sync = 1; /* prevent any further receives */ | |
320 | spin_lock_bh(&conn->send_lock); | |
321 | conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; | |
322 | conn->lnk = link; | |
323 | spin_unlock_bh(&conn->send_lock); | |
324 | sock_hold(&smc->sk); /* sock_put in abort_work */ | |
22ef473d | 325 | if (!queue_work(smc_close_wq, &conn->abort_work)) |
b286a065 KG |
326 | sock_put(&smc->sk); |
327 | } | |
328 | } | |
329 | ||
5f08318f | 330 | static void smc_cdc_msg_recv_action(struct smc_sock *smc, |
5f08318f UB |
331 | struct smc_cdc_msg *cdc) |
332 | { | |
333 | union smc_host_cursor cons_old, prod_old; | |
334 | struct smc_connection *conn = &smc->conn; | |
cc0ab806 | 335 | int diff_cons, diff_prod, diff_tx; |
5f08318f | 336 | |
bac6de7b SR |
337 | smc_curs_copy(&prod_old, &conn->local_rx_ctrl.prod, conn); |
338 | smc_curs_copy(&cons_old, &conn->local_rx_ctrl.cons, conn); | |
5f08318f UB |
339 | smc_cdc_msg_to_host(&conn->local_rx_ctrl, cdc, conn); |
340 | ||
341 | diff_cons = smc_curs_diff(conn->peer_rmbe_size, &cons_old, | |
342 | &conn->local_rx_ctrl.cons); | |
343 | if (diff_cons) { | |
344 | /* peer_rmbe_space is decreased during data transfer with RDMA | |
345 | * write | |
346 | */ | |
347 | smp_mb__before_atomic(); | |
348 | atomic_add(diff_cons, &conn->peer_rmbe_space); | |
349 | /* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */ | |
350 | smp_mb__after_atomic(); | |
cc0ab806 WG |
351 | |
352 | /* if local sndbuf shares the same memory region with | |
353 | * peer RMB, then update tx_curs_fin and sndbuf_space | |
354 | * here since peer has already consumed the data. | |
355 | */ | |
356 | if (conn->lgr->is_smcd && | |
357 | smc_ism_support_dmb_nocopy(conn->lgr->smcd)) { | |
358 | /* Calculate consumed data and | |
359 | * increment free send buffer space. | |
360 | */ | |
361 | diff_tx = smc_curs_diff(conn->sndbuf_desc->len, | |
362 | &conn->tx_curs_fin, | |
363 | &conn->local_rx_ctrl.cons); | |
364 | /* increase local sndbuf space and fin_curs */ | |
365 | smp_mb__before_atomic(); | |
366 | atomic_add(diff_tx, &conn->sndbuf_space); | |
367 | /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */ | |
368 | smp_mb__after_atomic(); | |
369 | smc_curs_copy(&conn->tx_curs_fin, | |
370 | &conn->local_rx_ctrl.cons, conn); | |
371 | ||
372 | smc_tx_sndbuf_nonfull(smc); | |
373 | } | |
5f08318f UB |
374 | } |
375 | ||
69cb7dc0 | 376 | diff_prod = smc_curs_diff(conn->rmb_desc->len, &prod_old, |
5f08318f UB |
377 | &conn->local_rx_ctrl.prod); |
378 | if (diff_prod) { | |
de8474eb SR |
379 | if (conn->local_rx_ctrl.prod_flags.urg_data_present) |
380 | smc_cdc_handle_urg_data_arrival(smc, &diff_prod); | |
5f08318f UB |
381 | /* bytes_to_rcv is decreased in smc_recvmsg */ |
382 | smp_mb__before_atomic(); | |
383 | atomic_add(diff_prod, &conn->bytes_to_rcv); | |
69cb7dc0 | 384 | /* guarantee 0 <= bytes_to_rcv <= rmb_desc->len */ |
5f08318f | 385 | smp_mb__after_atomic(); |
952310cc | 386 | smc->sk.sk_data_ready(&smc->sk); |
de8474eb | 387 | } else { |
cf0cfe53 KG |
388 | if (conn->local_rx_ctrl.prod_flags.write_blocked) |
389 | smc->sk.sk_data_ready(&smc->sk); | |
390 | if (conn->local_rx_ctrl.prod_flags.urg_data_pending) | |
391 | conn->urg_state = SMC_URG_NOTYET; | |
5f08318f UB |
392 | } |
393 | ||
51f1de79 | 394 | /* trigger sndbuf consumer: RDMA write into peer RMBE and CDC */ |
cf0cfe53 KG |
395 | if ((diff_cons && smc_tx_prepared_sends(conn)) || |
396 | conn->local_rx_ctrl.prod_flags.cons_curs_upd_req || | |
6b88af83 DL |
397 | conn->local_rx_ctrl.prod_flags.urg_data_pending) { |
398 | if (!sock_owned_by_user(&smc->sk)) | |
399 | smc_tx_pending(conn); | |
400 | else | |
401 | conn->tx_in_release_sock = true; | |
402 | } | |
cf0cfe53 | 403 | |
de8474eb SR |
404 | if (diff_cons && conn->urg_tx_pend && |
405 | atomic_read(&conn->peer_rmbe_space) == conn->peer_rmbe_size) { | |
406 | /* urg data confirmed by peer, indicate we're ready for more */ | |
407 | conn->urg_tx_pend = false; | |
408 | smc->sk.sk_write_space(&smc->sk); | |
409 | } | |
51f1de79 | 410 | |
b38d7324 | 411 | if (conn->local_rx_ctrl.conn_state_flags.peer_conn_abort) { |
5f08318f | 412 | smc->sk.sk_err = ECONNRESET; |
b38d7324 UB |
413 | conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; |
414 | } | |
46c28dbd UB |
415 | if (smc_cdc_rxed_any_close_or_senddone(conn)) { |
416 | smc->sk.sk_shutdown |= RCV_SHUTDOWN; | |
417 | if (smc->clcsock && smc->clcsock->sk) | |
418 | smc->clcsock->sk->sk_shutdown |= RCV_SHUTDOWN; | |
5211c972 | 419 | smc_sock_set_flag(&smc->sk, SOCK_DONE); |
51f1de79 | 420 | sock_hold(&smc->sk); /* sock_put in close_work */ |
22ef473d | 421 | if (!queue_work(smc_close_wq, &conn->close_work)) |
51f1de79 | 422 | sock_put(&smc->sk); |
b38d7324 | 423 | } |
5f08318f UB |
424 | } |
425 | ||
426 | /* called under tasklet context */ | |
d7b0e37c | 427 | static void smc_cdc_msg_recv(struct smc_sock *smc, struct smc_cdc_msg *cdc) |
5f08318f | 428 | { |
5f08318f | 429 | sock_hold(&smc->sk); |
5f08318f | 430 | bh_lock_sock(&smc->sk); |
d7b0e37c | 431 | smc_cdc_msg_recv_action(smc, cdc); |
5f08318f UB |
432 | bh_unlock_sock(&smc->sk); |
433 | sock_put(&smc->sk); /* no free sk in softirq-context */ | |
434 | } | |
435 | ||
be244f28 HW |
436 | /* Schedule a tasklet for this connection. Triggered from the ISM device IRQ |
437 | * handler to indicate update in the DMBE. | |
438 | * | |
439 | * Context: | |
440 | * - tasklet context | |
441 | */ | |
fcb8e3a3 | 442 | static void smcd_cdc_rx_tsklet(struct tasklet_struct *t) |
be244f28 | 443 | { |
fcb8e3a3 | 444 | struct smc_connection *conn = from_tasklet(conn, t, rx_tsklet); |
b9a22dd9 | 445 | struct smcd_cdc_msg *data_cdc; |
be244f28 HW |
446 | struct smcd_cdc_msg cdc; |
447 | struct smc_sock *smc; | |
448 | ||
b2900980 | 449 | if (!conn || conn->killed) |
be244f28 HW |
450 | return; |
451 | ||
b9a22dd9 UB |
452 | data_cdc = (struct smcd_cdc_msg *)conn->rmb_desc->cpu_addr; |
453 | smcd_curs_copy(&cdc.prod, &data_cdc->prod, conn); | |
454 | smcd_curs_copy(&cdc.cons, &data_cdc->cons, conn); | |
be244f28 HW |
455 | smc = container_of(conn, struct smc_sock, conn); |
456 | smc_cdc_msg_recv(smc, (struct smc_cdc_msg *)&cdc); | |
457 | } | |
458 | ||
459 | /* Initialize receive tasklet. Called from ISM device IRQ handler to start | |
460 | * receiver side. | |
461 | */ | |
462 | void smcd_cdc_rx_init(struct smc_connection *conn) | |
463 | { | |
fcb8e3a3 | 464 | tasklet_setup(&conn->rx_tsklet, smcd_cdc_rx_tsklet); |
be244f28 HW |
465 | } |
466 | ||
5f08318f UB |
467 | /***************************** init, exit, misc ******************************/ |
468 | ||
469 | static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) | |
470 | { | |
471 | struct smc_link *link = (struct smc_link *)wc->qp->qp_context; | |
472 | struct smc_cdc_msg *cdc = buf; | |
d7b0e37c HW |
473 | struct smc_connection *conn; |
474 | struct smc_link_group *lgr; | |
475 | struct smc_sock *smc; | |
5f08318f UB |
476 | |
477 | if (wc->byte_len < offsetof(struct smc_cdc_msg, reserved)) | |
478 | return; /* short message */ | |
cbba07a7 | 479 | if (cdc->len != SMC_WR_TX_SIZE) |
5f08318f | 480 | return; /* invalid message */ |
d7b0e37c HW |
481 | |
482 | /* lookup connection */ | |
00e5fb26 | 483 | lgr = smc_get_lgr(link); |
d7b0e37c HW |
484 | read_lock_bh(&lgr->conns_lock); |
485 | conn = smc_lgr_find_conn(ntohl(cdc->token), lgr); | |
486 | read_unlock_bh(&lgr->conns_lock); | |
b286a065 | 487 | if (!conn || conn->out_of_sync) |
d7b0e37c HW |
488 | return; |
489 | smc = container_of(conn, struct smc_sock, conn); | |
490 | ||
b286a065 KG |
491 | if (cdc->prod_flags.failover_validation) { |
492 | smc_cdc_msg_validate(smc, cdc, link); | |
493 | return; | |
d7b0e37c | 494 | } |
b286a065 KG |
495 | if (smc_cdc_before(ntohs(cdc->seqno), |
496 | conn->local_rx_ctrl.seqno)) | |
497 | /* received seqno is old */ | |
498 | return; | |
499 | ||
d7b0e37c | 500 | smc_cdc_msg_recv(smc, cdc); |
5f08318f UB |
501 | } |
502 | ||
503 | static struct smc_wr_rx_handler smc_cdc_rx_handlers[] = { | |
504 | { | |
505 | .handler = smc_cdc_rx_handler, | |
506 | .type = SMC_CDC_MSG_TYPE | |
507 | }, | |
508 | { | |
509 | .handler = NULL, | |
510 | } | |
511 | }; | |
512 | ||
513 | int __init smc_cdc_init(void) | |
514 | { | |
515 | struct smc_wr_rx_handler *handler; | |
516 | int rc = 0; | |
517 | ||
518 | for (handler = smc_cdc_rx_handlers; handler->handler; handler++) { | |
519 | INIT_HLIST_NODE(&handler->list); | |
520 | rc = smc_wr_rx_register_handler(handler); | |
521 | if (rc) | |
522 | break; | |
523 | } | |
524 | return rc; | |
525 | } |