Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
5f08318f UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Connection Data Control (CDC) | |
6 | * | |
7 | * Copyright IBM Corp. 2016 | |
8 | * | |
9 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> | |
10 | */ | |
11 | ||
12 | #ifndef SMC_CDC_H | |
13 | #define SMC_CDC_H | |
14 | ||
15 | #include <linux/kernel.h> /* max_t */ | |
16 | #include <linux/atomic.h> | |
17 | #include <linux/in.h> | |
18 | #include <linux/compiler.h> | |
19 | ||
20 | #include "smc.h" | |
21 | #include "smc_core.h" | |
22 | #include "smc_wr.h" | |
23 | ||
24 | #define SMC_CDC_MSG_TYPE 0xFE | |
25 | ||
26 | /* in network byte order */ | |
27 | union smc_cdc_cursor { /* SMC cursor */ | |
28 | struct { | |
29 | __be16 reserved; | |
30 | __be16 wrap; | |
31 | __be32 count; | |
32 | }; | |
33 | #ifdef KERNEL_HAS_ATOMIC64 | |
34 | atomic64_t acurs; /* for atomic processing */ | |
35 | #else | |
36 | u64 acurs; /* for atomic processing */ | |
37 | #endif | |
38 | } __aligned(8); | |
39 | ||
40 | /* in network byte order */ | |
41 | struct smc_cdc_msg { | |
42 | struct smc_wr_rx_hdr common; /* .type = 0xFE */ | |
43 | u8 len; /* 44 */ | |
44 | __be16 seqno; | |
45 | __be32 token; | |
46 | union smc_cdc_cursor prod; | |
47 | union smc_cdc_cursor cons; /* piggy backed "ack" */ | |
48 | struct smc_cdc_producer_flags prod_flags; | |
49 | struct smc_cdc_conn_state_flags conn_state_flags; | |
50 | u8 reserved[18]; | |
b9a22dd9 UB |
51 | }; |
52 | ||
53 | /* SMC-D cursor format */ | |
54 | union smcd_cdc_cursor { | |
55 | struct { | |
56 | u16 wrap; | |
57 | u32 count; | |
58 | struct smc_cdc_producer_flags prod_flags; | |
59 | struct smc_cdc_conn_state_flags conn_state_flags; | |
60 | } __packed; | |
61 | #ifdef KERNEL_HAS_ATOMIC64 | |
62 | atomic64_t acurs; /* for atomic processing */ | |
63 | #else | |
64 | u64 acurs; /* for atomic processing */ | |
65 | #endif | |
66 | } __aligned(8); | |
5f08318f | 67 | |
be244f28 HW |
68 | /* CDC message for SMC-D */ |
69 | struct smcd_cdc_msg { | |
70 | struct smc_wr_rx_hdr common; /* Type = 0xFE */ | |
71 | u8 res1[7]; | |
b9a22dd9 UB |
72 | union smcd_cdc_cursor prod; |
73 | union smcd_cdc_cursor cons; | |
be244f28 | 74 | u8 res3[8]; |
b9a22dd9 | 75 | } __aligned(8); |
be244f28 | 76 | |
5f08318f UB |
77 | static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn) |
78 | { | |
79 | return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort || | |
80 | conn->local_rx_ctrl.conn_state_flags.peer_conn_closed; | |
81 | } | |
82 | ||
83 | static inline bool smc_cdc_rxed_any_close_or_senddone( | |
84 | struct smc_connection *conn) | |
85 | { | |
86 | return smc_cdc_rxed_any_close(conn) || | |
87 | conn->local_rx_ctrl.conn_state_flags.peer_done_writing; | |
88 | } | |
89 | ||
90 | static inline void smc_curs_add(int size, union smc_host_cursor *curs, | |
91 | int value) | |
92 | { | |
93 | curs->count += value; | |
94 | if (curs->count >= size) { | |
95 | curs->wrap++; | |
96 | curs->count -= size; | |
97 | } | |
98 | } | |
99 | ||
100 | /* SMC cursors are 8 bytes long and require atomic reading and writing */ | |
101 | static inline u64 smc_curs_read(union smc_host_cursor *curs, | |
102 | struct smc_connection *conn) | |
103 | { | |
104 | #ifndef KERNEL_HAS_ATOMIC64 | |
105 | unsigned long flags; | |
106 | u64 ret; | |
107 | ||
108 | spin_lock_irqsave(&conn->acurs_lock, flags); | |
109 | ret = curs->acurs; | |
110 | spin_unlock_irqrestore(&conn->acurs_lock, flags); | |
111 | return ret; | |
112 | #else | |
113 | return atomic64_read(&curs->acurs); | |
114 | #endif | |
115 | } | |
116 | ||
bac6de7b SR |
117 | /* Copy cursor src into tgt */ |
118 | static inline void smc_curs_copy(union smc_host_cursor *tgt, | |
119 | union smc_host_cursor *src, | |
120 | struct smc_connection *conn) | |
5f08318f UB |
121 | { |
122 | #ifndef KERNEL_HAS_ATOMIC64 | |
123 | unsigned long flags; | |
124 | ||
125 | spin_lock_irqsave(&conn->acurs_lock, flags); | |
bac6de7b | 126 | tgt->acurs = src->acurs; |
5f08318f UB |
127 | spin_unlock_irqrestore(&conn->acurs_lock, flags); |
128 | #else | |
bac6de7b | 129 | atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); |
5f08318f UB |
130 | #endif |
131 | } | |
132 | ||
bac6de7b SR |
133 | static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt, |
134 | union smc_cdc_cursor *src, | |
135 | struct smc_connection *conn) | |
5f08318f UB |
136 | { |
137 | #ifndef KERNEL_HAS_ATOMIC64 | |
138 | unsigned long flags; | |
139 | ||
140 | spin_lock_irqsave(&conn->acurs_lock, flags); | |
bac6de7b | 141 | tgt->acurs = src->acurs; |
5f08318f UB |
142 | spin_unlock_irqrestore(&conn->acurs_lock, flags); |
143 | #else | |
bac6de7b | 144 | atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); |
5f08318f UB |
145 | #endif |
146 | } | |
147 | ||
b9a22dd9 UB |
148 | static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt, |
149 | union smcd_cdc_cursor *src, | |
150 | struct smc_connection *conn) | |
151 | { | |
152 | #ifndef KERNEL_HAS_ATOMIC64 | |
153 | unsigned long flags; | |
154 | ||
155 | spin_lock_irqsave(&conn->acurs_lock, flags); | |
156 | tgt->acurs = src->acurs; | |
157 | spin_unlock_irqrestore(&conn->acurs_lock, flags); | |
158 | #else | |
159 | atomic64_set(&tgt->acurs, atomic64_read(&src->acurs)); | |
160 | #endif | |
161 | } | |
162 | ||
b8649efa UB |
163 | /* calculate cursor difference between old and new, where old <= new and |
164 | * difference cannot exceed size | |
165 | */ | |
5f08318f UB |
166 | static inline int smc_curs_diff(unsigned int size, |
167 | union smc_host_cursor *old, | |
168 | union smc_host_cursor *new) | |
169 | { | |
170 | if (old->wrap != new->wrap) | |
171 | return max_t(int, 0, | |
172 | ((size - old->count) + new->count)); | |
173 | ||
174 | return max_t(int, 0, (new->count - old->count)); | |
175 | } | |
176 | ||
de8474eb SR |
177 | /* calculate cursor difference between old and new - returns negative |
178 | * value in case old > new | |
179 | */ | |
180 | static inline int smc_curs_comp(unsigned int size, | |
181 | union smc_host_cursor *old, | |
182 | union smc_host_cursor *new) | |
183 | { | |
184 | if (old->wrap > new->wrap || | |
185 | (old->wrap == new->wrap && old->count > new->count)) | |
186 | return -smc_curs_diff(size, new, old); | |
187 | return smc_curs_diff(size, old, new); | |
188 | } | |
189 | ||
b8649efa UB |
190 | /* calculate cursor difference between old and new, where old <= new and |
191 | * difference may exceed size | |
192 | */ | |
193 | static inline int smc_curs_diff_large(unsigned int size, | |
194 | union smc_host_cursor *old, | |
195 | union smc_host_cursor *new) | |
196 | { | |
197 | if (old->wrap < new->wrap) | |
198 | return min_t(int, | |
199 | (size - old->count) + new->count + | |
200 | (new->wrap - old->wrap - 1) * size, | |
201 | size); | |
202 | ||
203 | if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */ | |
204 | return min_t(int, | |
205 | (size - old->count) + new->count + | |
206 | (new->wrap + 0xffff - old->wrap) * size, | |
207 | size); | |
208 | ||
209 | return max_t(int, 0, (new->count - old->count)); | |
210 | } | |
211 | ||
5f08318f UB |
212 | static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer, |
213 | union smc_host_cursor *local, | |
ccc8ca9b | 214 | union smc_host_cursor *save, |
5f08318f UB |
215 | struct smc_connection *conn) |
216 | { | |
ccc8ca9b UB |
217 | smc_curs_copy(save, local, conn); |
218 | peer->count = htonl(save->count); | |
219 | peer->wrap = htons(save->wrap); | |
5f08318f UB |
220 | /* peer->reserved = htons(0); must be ensured by caller */ |
221 | } | |
222 | ||
223 | static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer, | |
ccc8ca9b UB |
224 | struct smc_connection *conn, |
225 | union smc_host_cursor *save) | |
5f08318f | 226 | { |
ccc8ca9b UB |
227 | struct smc_host_cdc_msg *local = &conn->local_tx_ctrl; |
228 | ||
5f08318f UB |
229 | peer->common.type = local->common.type; |
230 | peer->len = local->len; | |
231 | peer->seqno = htons(local->seqno); | |
232 | peer->token = htonl(local->token); | |
ccc8ca9b UB |
233 | smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn); |
234 | smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn); | |
5f08318f UB |
235 | peer->prod_flags = local->prod_flags; |
236 | peer->conn_state_flags = local->conn_state_flags; | |
237 | } | |
238 | ||
239 | static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local, | |
240 | union smc_cdc_cursor *peer, | |
241 | struct smc_connection *conn) | |
242 | { | |
243 | union smc_host_cursor temp, old; | |
244 | union smc_cdc_cursor net; | |
245 | ||
bac6de7b SR |
246 | smc_curs_copy(&old, local, conn); |
247 | smc_curs_copy_net(&net, peer, conn); | |
5f08318f UB |
248 | temp.count = ntohl(net.count); |
249 | temp.wrap = ntohs(net.wrap); | |
250 | if ((old.wrap > temp.wrap) && temp.wrap) | |
251 | return; | |
252 | if ((old.wrap == temp.wrap) && | |
253 | (old.count > temp.count)) | |
254 | return; | |
bac6de7b | 255 | smc_curs_copy(local, &temp, conn); |
5f08318f UB |
256 | } |
257 | ||
be244f28 HW |
258 | static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local, |
259 | struct smc_cdc_msg *peer, | |
260 | struct smc_connection *conn) | |
5f08318f UB |
261 | { |
262 | local->common.type = peer->common.type; | |
263 | local->len = peer->len; | |
264 | local->seqno = ntohs(peer->seqno); | |
265 | local->token = ntohl(peer->token); | |
266 | smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn); | |
267 | smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn); | |
268 | local->prod_flags = peer->prod_flags; | |
269 | local->conn_state_flags = peer->conn_state_flags; | |
270 | } | |
271 | ||
be244f28 | 272 | static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local, |
a225d2cd UB |
273 | struct smcd_cdc_msg *peer, |
274 | struct smc_connection *conn) | |
be244f28 | 275 | { |
b9a22dd9 UB |
276 | union smc_host_cursor temp; |
277 | ||
278 | temp.wrap = peer->prod.wrap; | |
279 | temp.count = peer->prod.count; | |
a225d2cd | 280 | smc_curs_copy(&local->prod, &temp, conn); |
b9a22dd9 UB |
281 | |
282 | temp.wrap = peer->cons.wrap; | |
283 | temp.count = peer->cons.count; | |
a225d2cd | 284 | smc_curs_copy(&local->cons, &temp, conn); |
b9a22dd9 UB |
285 | local->prod_flags = peer->cons.prod_flags; |
286 | local->conn_state_flags = peer->cons.conn_state_flags; | |
be244f28 HW |
287 | } |
288 | ||
289 | static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local, | |
290 | struct smc_cdc_msg *peer, | |
291 | struct smc_connection *conn) | |
292 | { | |
293 | if (conn->lgr->is_smcd) | |
a225d2cd | 294 | smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn); |
be244f28 HW |
295 | else |
296 | smcr_cdc_msg_to_host(local, peer, conn); | |
297 | } | |
298 | ||
ad6f317f UB |
299 | struct smc_cdc_tx_pend { |
300 | struct smc_connection *conn; /* socket connection */ | |
301 | union smc_host_cursor cursor; /* tx sndbuf cursor sent */ | |
302 | union smc_host_cursor p_cursor; /* rx RMBE cursor produced */ | |
303 | u16 ctrl_seq; /* conn. tx sequence # */ | |
304 | }; | |
5f08318f | 305 | |
51957bc5 UB |
306 | int smc_cdc_get_free_slot(struct smc_connection *conn, |
307 | struct smc_wr_buf **wr_buf, | |
ad6f317f | 308 | struct smc_rdma_wr **wr_rdma_buf, |
5f08318f UB |
309 | struct smc_cdc_tx_pend **pend); |
310 | void smc_cdc_tx_dismiss_slots(struct smc_connection *conn); | |
311 | int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf, | |
312 | struct smc_cdc_tx_pend *pend); | |
313 | int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn); | |
be244f28 | 314 | int smcd_cdc_msg_send(struct smc_connection *conn); |
5f08318f | 315 | int smc_cdc_init(void) __init; |
be244f28 | 316 | void smcd_cdc_rx_init(struct smc_connection *conn); |
5f08318f UB |
317 | |
318 | #endif /* SMC_CDC_H */ |