Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
ac713874 UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Definitions for the SMC module (socket related) | |
6 | * | |
7 | * Copyright IBM Corp. 2016 | |
8 | * | |
9 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> | |
10 | */ | |
11 | #ifndef __SMC_H | |
12 | #define __SMC_H | |
13 | ||
14 | #include <linux/socket.h> | |
15 | #include <linux/types.h> | |
f38ba179 | 16 | #include <linux/compiler.h> /* __aligned */ |
f9496b7c | 17 | #include <net/genetlink.h> |
ac713874 UB |
18 | #include <net/sock.h> |
19 | ||
0cfdd8f9 UB |
20 | #include "smc_ib.h" |
21 | ||
5ac54d87 | 22 | #define SMC_V1 1 /* SMC version V1 */ |
d70bf4f7 | 23 | #define SMC_V2 2 /* SMC version V2 */ |
8c3dca34 | 24 | #define SMC_RELEASE 0 |
5ac54d87 | 25 | |
aaa4d33f KG |
26 | #define SMCPROTO_SMC 0 /* SMC protocol, IPv4 */ |
27 | #define SMCPROTO_SMC6 1 /* SMC protocol, IPv6 */ | |
ac713874 | 28 | |
3fc64937 UB |
29 | #define SMC_MAX_ISM_DEVS 8 /* max # of proposed non-native ISM |
30 | * devices | |
31 | */ | |
dcd2cf5f | 32 | #define SMC_AUTOCORKING_DEFAULT_SIZE 0x10000 /* 64K by default */ |
3fc64937 | 33 | |
f16a7dd5 | 34 | extern struct proto smc_proto; |
aaa4d33f | 35 | extern struct proto smc_proto6; |
f16a7dd5 | 36 | |
5f08318f UB |
37 | #ifdef ATOMIC64_INIT |
38 | #define KERNEL_HAS_ATOMIC64 | |
39 | #endif | |
40 | ||
ac713874 UB |
41 | enum smc_state { /* possible states of an SMC socket */ |
42 | SMC_ACTIVE = 1, | |
43 | SMC_INIT = 2, | |
44 | SMC_CLOSED = 7, | |
45 | SMC_LISTEN = 10, | |
b38d7324 UB |
46 | /* normal close */ |
47 | SMC_PEERCLOSEWAIT1 = 20, | |
48 | SMC_PEERCLOSEWAIT2 = 21, | |
49 | SMC_APPFINCLOSEWAIT = 24, | |
50 | SMC_APPCLOSEWAIT1 = 22, | |
51 | SMC_APPCLOSEWAIT2 = 23, | |
52 | SMC_PEERFINCLOSEWAIT = 25, | |
53 | /* abnormal close */ | |
54 | SMC_PEERABORTWAIT = 26, | |
55 | SMC_PROCESSABORT = 27, | |
ac713874 UB |
56 | }; |
57 | ||
0cfdd8f9 UB |
58 | struct smc_link_group; |
59 | ||
f38ba179 | 60 | struct smc_wr_rx_hdr { /* common prefix part of LLC and CDC to demultiplex */ |
b4ba4652 KG |
61 | union { |
62 | u8 type; | |
63 | #if defined(__BIG_ENDIAN_BITFIELD) | |
64 | struct { | |
65 | u8 llc_version:4, | |
66 | llc_type:4; | |
67 | }; | |
68 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
69 | struct { | |
70 | u8 llc_type:4, | |
71 | llc_version:4; | |
72 | }; | |
73 | #endif | |
74 | }; | |
f38ba179 UB |
75 | } __aligned(1); |
76 | ||
5f08318f UB |
77 | struct smc_cdc_conn_state_flags { |
78 | #if defined(__BIG_ENDIAN_BITFIELD) | |
79 | u8 peer_done_writing : 1; /* Sending done indicator */ | |
80 | u8 peer_conn_closed : 1; /* Peer connection closed indicator */ | |
81 | u8 peer_conn_abort : 1; /* Abnormal close indicator */ | |
82 | u8 reserved : 5; | |
83 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
84 | u8 reserved : 5; | |
85 | u8 peer_conn_abort : 1; | |
86 | u8 peer_conn_closed : 1; | |
87 | u8 peer_done_writing : 1; | |
88 | #endif | |
89 | }; | |
90 | ||
91 | struct smc_cdc_producer_flags { | |
92 | #if defined(__BIG_ENDIAN_BITFIELD) | |
93 | u8 write_blocked : 1; /* Writing Blocked, no rx buf space */ | |
94 | u8 urg_data_pending : 1; /* Urgent Data Pending */ | |
95 | u8 urg_data_present : 1; /* Urgent Data Present */ | |
96 | u8 cons_curs_upd_req : 1; /* cursor update requested */ | |
97 | u8 failover_validation : 1;/* message replay due to failover */ | |
98 | u8 reserved : 3; | |
99 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
100 | u8 reserved : 3; | |
101 | u8 failover_validation : 1; | |
102 | u8 cons_curs_upd_req : 1; | |
103 | u8 urg_data_present : 1; | |
104 | u8 urg_data_pending : 1; | |
105 | u8 write_blocked : 1; | |
106 | #endif | |
107 | }; | |
108 | ||
109 | /* in host byte order */ | |
110 | union smc_host_cursor { /* SMC cursor - an offset in an RMBE */ | |
111 | struct { | |
112 | u16 reserved; | |
113 | u16 wrap; /* window wrap sequence number */ | |
114 | u32 count; /* cursor (= offset) part */ | |
115 | }; | |
116 | #ifdef KERNEL_HAS_ATOMIC64 | |
117 | atomic64_t acurs; /* for atomic processing */ | |
118 | #else | |
119 | u64 acurs; /* for atomic processing */ | |
120 | #endif | |
121 | } __aligned(8); | |
122 | ||
123 | /* in host byte order, except for flag bitfields in network byte order */ | |
124 | struct smc_host_cdc_msg { /* Connection Data Control message */ | |
125 | struct smc_wr_rx_hdr common; /* .type = 0xFE */ | |
126 | u8 len; /* length = 44 */ | |
127 | u16 seqno; /* connection seq # */ | |
128 | u32 token; /* alert_token */ | |
129 | union smc_host_cursor prod; /* producer cursor */ | |
130 | union smc_host_cursor cons; /* consumer cursor, | |
131 | * piggy backed "ack" | |
132 | */ | |
133 | struct smc_cdc_producer_flags prod_flags; /* conn. tx/rx status */ | |
134 | struct smc_cdc_conn_state_flags conn_state_flags; /* peer conn. status*/ | |
135 | u8 reserved[18]; | |
136 | } __aligned(8); | |
137 | ||
de8474eb | 138 | enum smc_urg_state { |
d7cf4a3b UB |
139 | SMC_URG_VALID = 1, /* data present */ |
140 | SMC_URG_NOTYET = 2, /* data pending */ | |
141 | SMC_URG_READ = 3, /* data was already read */ | |
de8474eb SR |
142 | }; |
143 | ||
341adeec WG |
144 | struct smc_mark_woken { |
145 | bool woken; | |
146 | void *key; | |
147 | wait_queue_entry_t wait_entry; | |
148 | }; | |
149 | ||
0cfdd8f9 UB |
150 | struct smc_connection { |
151 | struct rb_node alert_node; | |
152 | struct smc_link_group *lgr; /* link group of connection */ | |
387707fd | 153 | struct smc_link *lnk; /* assigned SMC-R link */ |
0cfdd8f9 | 154 | u32 alert_token_local; /* unique conn. id */ |
92a138e3 | 155 | u8 peer_rmbe_idx; /* from tcp handshake */ |
cd6851f3 UB |
156 | int peer_rmbe_size; /* size of peer rx buffer */ |
157 | atomic_t peer_rmbe_space;/* remaining free bytes in peer | |
158 | * rmbe | |
159 | */ | |
bd4ad577 | 160 | int rtoken_idx; /* idx to peer RMB rkey/addr */ |
cd6851f3 UB |
161 | |
162 | struct smc_buf_desc *sndbuf_desc; /* send buffer descriptor */ | |
cd6851f3 | 163 | struct smc_buf_desc *rmb_desc; /* RMBE descriptor */ |
cd6851f3 | 164 | int rmbe_size_short;/* compressed notation */ |
952310cc UB |
165 | int rmbe_update_limit; |
166 | /* lower limit for consumer | |
167 | * cursor update | |
168 | */ | |
5f08318f UB |
169 | |
170 | struct smc_host_cdc_msg local_tx_ctrl; /* host byte order staging | |
171 | * buffer for CDC msg send | |
172 | * .prod cf. TCP snd_nxt | |
173 | * .cons cf. TCP sends ack | |
174 | */ | |
f0ec4f1d KG |
175 | union smc_host_cursor local_tx_ctrl_fin; |
176 | /* prod crsr - confirmed by peer | |
177 | */ | |
5f08318f UB |
178 | union smc_host_cursor tx_curs_prep; /* tx - prepared data |
179 | * snd_max..wmem_alloc | |
180 | */ | |
181 | union smc_host_cursor tx_curs_sent; /* tx - sent data | |
182 | * snd_nxt ? | |
183 | */ | |
184 | union smc_host_cursor tx_curs_fin; /* tx - confirmed by peer | |
185 | * snd-wnd-begin ? | |
186 | */ | |
187 | atomic_t sndbuf_space; /* remaining space in sndbuf */ | |
188 | u16 tx_cdc_seq; /* sequence # for CDC send */ | |
f0ec4f1d | 189 | u16 tx_cdc_seq_fin; /* sequence # - tx completed */ |
5f08318f | 190 | spinlock_t send_lock; /* protect wr_sends */ |
349d4312 DL |
191 | atomic_t cdc_pend_tx_wr; /* number of pending tx CDC wqe |
192 | * - inc when post wqe, | |
193 | * - dec on polled tx cqe | |
194 | */ | |
195 | wait_queue_head_t cdc_pend_tx_wq; /* wakeup on no cdc_pend_tx_wr*/ | |
dcd2cf5f | 196 | atomic_t tx_pushing; /* nr_threads trying tx push */ |
18e537cd | 197 | struct delayed_work tx_work; /* retry of smc_cdc_msg_send */ |
95d8d263 | 198 | u32 tx_off; /* base offset in peer rmb */ |
5f08318f UB |
199 | |
200 | struct smc_host_cdc_msg local_rx_ctrl; /* filled during event_handl. | |
201 | * .prod cf. TCP rcv_nxt | |
202 | * .cons cf. TCP snd_una | |
203 | */ | |
204 | union smc_host_cursor rx_curs_confirmed; /* confirmed to peer | |
205 | * source of snd_una ? | |
206 | */ | |
de8474eb SR |
207 | union smc_host_cursor urg_curs; /* points at urgent byte */ |
208 | enum smc_urg_state urg_state; | |
209 | bool urg_tx_pend; /* urgent data staged */ | |
210 | bool urg_rx_skip_pend; | |
211 | /* indicate urgent oob data | |
212 | * read, but previous regular | |
213 | * data still pending | |
214 | */ | |
215 | char urg_rx_byte; /* urgent byte */ | |
6b88af83 DL |
216 | bool tx_in_release_sock; |
217 | /* flush pending tx data in | |
218 | * sock release_cb() | |
219 | */ | |
5f08318f UB |
220 | atomic_t bytes_to_rcv; /* arrived data, |
221 | * not yet received | |
222 | */ | |
9014db20 SR |
223 | atomic_t splice_pending; /* number of spliced bytes |
224 | * pending processing | |
225 | */ | |
5f08318f UB |
226 | #ifndef KERNEL_HAS_ATOMIC64 |
227 | spinlock_t acurs_lock; /* protect cursors */ | |
228 | #endif | |
46c28dbd | 229 | struct work_struct close_work; /* peer sent some closing */ |
b286a065 | 230 | struct work_struct abort_work; /* abort the connection */ |
be244f28 HW |
231 | struct tasklet_struct rx_tsklet; /* Receiver tasklet for SMC-D */ |
232 | u8 rx_off; /* receive offset: | |
233 | * 0 for SMC-R, 32 for SMC-D | |
234 | */ | |
235 | u64 peer_token; /* SMC-D token of peer */ | |
b2900980 | 236 | u8 killed : 1; /* abnormal termination */ |
61f434b0 | 237 | u8 freed : 1; /* normal termiation */ |
b286a065 | 238 | u8 out_of_sync : 1; /* out of sync with peer */ |
0cfdd8f9 UB |
239 | }; |
240 | ||
ac713874 UB |
241 | struct smc_sock { /* smc sock container */ |
242 | struct sock sk; | |
243 | struct socket *clcsock; /* internal tcp socket */ | |
341adeec WG |
244 | void (*clcsk_state_change)(struct sock *sk); |
245 | /* original stat_change fct. */ | |
a60a2b1e | 246 | void (*clcsk_data_ready)(struct sock *sk); |
341adeec WG |
247 | /* original data_ready fct. */ |
248 | void (*clcsk_write_space)(struct sock *sk); | |
249 | /* original write_space fct. */ | |
250 | void (*clcsk_error_report)(struct sock *sk); | |
251 | /* original error_report fct. */ | |
0cfdd8f9 | 252 | struct smc_connection conn; /* smc connection */ |
a046d57d | 253 | struct smc_sock *listen_smc; /* listen parent */ |
24ac3a08 | 254 | struct work_struct connect_work; /* handle non-blocking connect*/ |
a046d57d UB |
255 | struct work_struct tcp_listen_work;/* handle tcp socket accepts */ |
256 | struct work_struct smc_listen_work;/* prepare new accept socket */ | |
257 | struct list_head accept_q; /* sockets to be accepted */ | |
258 | spinlock_t accept_q_lock; /* protects accept_q */ | |
a6a6fe27 | 259 | bool limit_smc_hs; /* put constraint on handshake */ |
ac713874 | 260 | bool use_fallback; /* fallback to tcp */ |
603cc149 KG |
261 | int fallback_rsn; /* reason for fallback */ |
262 | u32 peer_diagnosis; /* decline reason from peer */ | |
8270d9c2 W |
263 | atomic_t queued_smc_hs; /* queued smc handshakes */ |
264 | struct inet_connection_sock_af_ops af_ops; | |
265 | const struct inet_connection_sock_af_ops *ori_af_ops; | |
266 | /* original af ops */ | |
abb190f1 UB |
267 | int sockopt_defer_accept; |
268 | /* sockopt TCP_DEFER_ACCEPT | |
269 | * value | |
270 | */ | |
b38d7324 UB |
271 | u8 wait_close_tx_prepared : 1; |
272 | /* shutdown wr or close | |
273 | * started, waiting for unsent | |
274 | * data to be sent | |
275 | */ | |
50717a37 UB |
276 | u8 connect_nonblock : 1; |
277 | /* non-blocking connect in | |
278 | * flight | |
279 | */ | |
78abe3d0 MJ |
280 | struct mutex clcsock_release_lock; |
281 | /* protects clcsock of a listen | |
282 | * socket | |
283 | * */ | |
ac713874 UB |
284 | }; |
285 | ||
407db475 | 286 | #define smc_sk(ptr) container_of_const(ptr, struct smc_sock, sk) |
ac713874 | 287 | |
97b9af7a WG |
288 | static inline void smc_init_saved_callbacks(struct smc_sock *smc) |
289 | { | |
290 | smc->clcsk_state_change = NULL; | |
291 | smc->clcsk_data_ready = NULL; | |
292 | smc->clcsk_write_space = NULL; | |
293 | smc->clcsk_error_report = NULL; | |
294 | } | |
295 | ||
8270d9c2 | 296 | static inline struct smc_sock *smc_clcsock_user_data(const struct sock *clcsk) |
341adeec WG |
297 | { |
298 | return (struct smc_sock *) | |
299 | ((uintptr_t)clcsk->sk_user_data & ~SK_USER_DATA_NOCOPY); | |
300 | } | |
301 | ||
97b9af7a WG |
302 | /* save target_cb in saved_cb, and replace target_cb with new_cb */ |
303 | static inline void smc_clcsock_replace_cb(void (**target_cb)(struct sock *), | |
304 | void (*new_cb)(struct sock *), | |
305 | void (**saved_cb)(struct sock *)) | |
306 | { | |
307 | /* only save once */ | |
308 | if (!*saved_cb) | |
309 | *saved_cb = *target_cb; | |
310 | *target_cb = new_cb; | |
311 | } | |
312 | ||
313 | /* restore target_cb to saved_cb, and reset saved_cb to NULL */ | |
314 | static inline void smc_clcsock_restore_cb(void (**target_cb)(struct sock *), | |
315 | void (**saved_cb)(struct sock *)) | |
316 | { | |
317 | if (!*saved_cb) | |
318 | return; | |
319 | *target_cb = *saved_cb; | |
320 | *saved_cb = NULL; | |
321 | } | |
322 | ||
22ef473d KG |
323 | extern struct workqueue_struct *smc_hs_wq; /* wq for handshake work */ |
324 | extern struct workqueue_struct *smc_close_wq; /* wq for close work */ | |
325 | ||
a4cf0443 UB |
326 | #define SMC_SYSTEMID_LEN 8 |
327 | ||
328 | extern u8 local_systemid[SMC_SYSTEMID_LEN]; /* unique system identifier */ | |
329 | ||
8c3dca34 UB |
330 | #define ntohll(x) be64_to_cpu(x) |
331 | #define htonll(x) cpu_to_be64(x) | |
332 | ||
0cfdd8f9 UB |
333 | /* convert an u32 value into network byte order, store it into a 3 byte field */ |
334 | static inline void hton24(u8 *net, u32 host) | |
335 | { | |
336 | __be32 t; | |
337 | ||
338 | t = cpu_to_be32(host); | |
339 | memcpy(net, ((u8 *)&t) + 1, 3); | |
340 | } | |
341 | ||
342 | /* convert a received 3 byte field into host byte order*/ | |
343 | static inline u32 ntoh24(u8 *net) | |
344 | { | |
345 | __be32 t = 0; | |
346 | ||
347 | memcpy(((u8 *)&t) + 1, net, 3); | |
348 | return be32_to_cpu(t); | |
349 | } | |
350 | ||
a046d57d UB |
351 | #ifdef CONFIG_XFRM |
352 | static inline bool using_ipsec(struct smc_sock *smc) | |
353 | { | |
354 | return (smc->clcsock->sk->sk_policy[0] || | |
a8fbf8e7 | 355 | smc->clcsock->sk->sk_policy[1]) ? true : false; |
a046d57d UB |
356 | } |
357 | #else | |
358 | static inline bool using_ipsec(struct smc_sock *smc) | |
359 | { | |
a8fbf8e7 | 360 | return false; |
a046d57d UB |
361 | } |
362 | #endif | |
363 | ||
b4ba4652 KG |
364 | struct smc_gidlist; |
365 | ||
b38d7324 UB |
366 | struct sock *smc_accept_dequeue(struct sock *parent, struct socket *new_sock); |
367 | void smc_close_non_accepted(struct sock *sk); | |
b4ba4652 KG |
368 | void smc_fill_gid_list(struct smc_link_group *lgr, |
369 | struct smc_gidlist *gidlist, | |
370 | struct smc_ib_device *known_dev, u8 *known_gid); | |
a046d57d | 371 | |
f9496b7c W |
372 | /* smc handshake limitation interface for netlink */ |
373 | int smc_nl_dump_hs_limitation(struct sk_buff *skb, struct netlink_callback *cb); | |
374 | int smc_nl_enable_hs_limitation(struct sk_buff *skb, struct genl_info *info); | |
375 | int smc_nl_disable_hs_limitation(struct sk_buff *skb, struct genl_info *info); | |
376 | ||
ac713874 | 377 | #endif /* __SMC_H */ |