Commit | Line | Data |
---|---|---|
70041088 | 1 | /* |
1e2b44e7 | 2 | * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved. |
70041088 AG |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | * | |
32 | */ | |
33 | #include <linux/kernel.h> | |
5a0e3ad6 | 34 | #include <linux/slab.h> |
70041088 | 35 | #include <linux/in.h> |
3a9a231d | 36 | #include <linux/module.h> |
70041088 | 37 | #include <net/tcp.h> |
467fa153 SV |
38 | #include <net/net_namespace.h> |
39 | #include <net/netns/generic.h> | |
eee2fa6a | 40 | #include <net/addrconf.h> |
70041088 AG |
41 | |
42 | #include "rds.h" | |
43 | #include "tcp.h" | |
44 | ||
45 | /* only for info exporting */ | |
46 | static DEFINE_SPINLOCK(rds_tcp_tc_list_lock); | |
47 | static LIST_HEAD(rds_tcp_tc_list); | |
1e2b44e7 KCP |
48 | |
49 | /* rds_tcp_tc_count counts only IPv4 connections. | |
50 | * rds6_tcp_tc_count counts both IPv4 and IPv6 connections. | |
51 | */ | |
ff51bf84 | 52 | static unsigned int rds_tcp_tc_count; |
e65d4d96 | 53 | #if IS_ENABLED(CONFIG_IPV6) |
1e2b44e7 | 54 | static unsigned int rds6_tcp_tc_count; |
e65d4d96 | 55 | #endif |
70041088 AG |
56 | |
57 | /* Track rds_tcp_connection structs so they can be cleaned up */ | |
58 | static DEFINE_SPINLOCK(rds_tcp_conn_lock); | |
59 | static LIST_HEAD(rds_tcp_conn_list); | |
ebeeb1ad | 60 | static atomic_t rds_tcp_unloading = ATOMIC_INIT(0); |
70041088 AG |
61 | |
62 | static struct kmem_cache *rds_tcp_conn_slab; | |
63 | ||
c6a58ffe SV |
64 | static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write, |
65 | void __user *buffer, size_t *lenp, | |
66 | loff_t *fpos); | |
67 | ||
af73e72d WY |
68 | static int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF; |
69 | static int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF; | |
c6a58ffe SV |
70 | |
71 | static struct ctl_table rds_tcp_sysctl_table[] = { | |
72 | #define RDS_TCP_SNDBUF 0 | |
73 | { | |
74 | .procname = "rds_tcp_sndbuf", | |
75 | /* data is per-net pointer */ | |
76 | .maxlen = sizeof(int), | |
77 | .mode = 0644, | |
78 | .proc_handler = rds_tcp_skbuf_handler, | |
79 | .extra1 = &rds_tcp_min_sndbuf, | |
80 | }, | |
81 | #define RDS_TCP_RCVBUF 1 | |
82 | { | |
83 | .procname = "rds_tcp_rcvbuf", | |
84 | /* data is per-net pointer */ | |
85 | .maxlen = sizeof(int), | |
86 | .mode = 0644, | |
87 | .proc_handler = rds_tcp_skbuf_handler, | |
88 | .extra1 = &rds_tcp_min_rcvbuf, | |
89 | }, | |
90 | { } | |
91 | }; | |
92 | ||
70041088 AG |
93 | /* doing it this way avoids calling tcp_sk() */ |
94 | void rds_tcp_nonagle(struct socket *sock) | |
95 | { | |
70041088 AG |
96 | int val = 1; |
97 | ||
e73a67f7 | 98 | kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&val, |
70041088 | 99 | sizeof(val)); |
70041088 AG |
100 | } |
101 | ||
b589513e | 102 | u32 rds_tcp_write_seq(struct rds_tcp_connection *tc) |
70041088 | 103 | { |
b589513e SV |
104 | /* seq# of the last byte of data in tcp send buffer */ |
105 | return tcp_sk(tc->t_sock->sk)->write_seq; | |
70041088 AG |
106 | } |
107 | ||
108 | u32 rds_tcp_snd_una(struct rds_tcp_connection *tc) | |
109 | { | |
110 | return tcp_sk(tc->t_sock->sk)->snd_una; | |
111 | } | |
112 | ||
113 | void rds_tcp_restore_callbacks(struct socket *sock, | |
114 | struct rds_tcp_connection *tc) | |
115 | { | |
116 | rdsdebug("restoring sock %p callbacks from tc %p\n", sock, tc); | |
117 | write_lock_bh(&sock->sk->sk_callback_lock); | |
118 | ||
119 | /* done under the callback_lock to serialize with write_space */ | |
120 | spin_lock(&rds_tcp_tc_list_lock); | |
121 | list_del_init(&tc->t_list_item); | |
e65d4d96 | 122 | #if IS_ENABLED(CONFIG_IPV6) |
1e2b44e7 | 123 | rds6_tcp_tc_count--; |
e65d4d96 | 124 | #endif |
1e2b44e7 KCP |
125 | if (!tc->t_cpath->cp_conn->c_isv6) |
126 | rds_tcp_tc_count--; | |
70041088 AG |
127 | spin_unlock(&rds_tcp_tc_list_lock); |
128 | ||
129 | tc->t_sock = NULL; | |
130 | ||
131 | sock->sk->sk_write_space = tc->t_orig_write_space; | |
132 | sock->sk->sk_data_ready = tc->t_orig_data_ready; | |
133 | sock->sk->sk_state_change = tc->t_orig_state_change; | |
134 | sock->sk->sk_user_data = NULL; | |
135 | ||
136 | write_unlock_bh(&sock->sk->sk_callback_lock); | |
137 | } | |
138 | ||
139 | /* | |
335b48d9 SV |
140 | * rds_tcp_reset_callbacks() switches the to the new sock and |
141 | * returns the existing tc->t_sock. | |
142 | * | |
143 | * The only functions that set tc->t_sock are rds_tcp_set_callbacks | |
144 | * and rds_tcp_reset_callbacks. Send and receive trust that | |
145 | * it is set. The absence of RDS_CONN_UP bit protects those paths | |
146 | * from being called while it isn't set. | |
147 | */ | |
148 | void rds_tcp_reset_callbacks(struct socket *sock, | |
ea3b1ea5 | 149 | struct rds_conn_path *cp) |
335b48d9 | 150 | { |
ea3b1ea5 | 151 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
335b48d9 SV |
152 | struct socket *osock = tc->t_sock; |
153 | ||
154 | if (!osock) | |
155 | goto newsock; | |
156 | ||
157 | /* Need to resolve a duelling SYN between peers. | |
158 | * We have an outstanding SYN to this peer, which may | |
159 | * potentially have transitioned to the RDS_CONN_UP state, | |
160 | * so we must quiesce any send threads before resetting | |
ea3b1ea5 SV |
161 | * cp_transport_data. We quiesce these threads by setting |
162 | * cp_state to something other than RDS_CONN_UP, and then | |
335b48d9 SV |
163 | * waiting for any existing threads in rds_send_xmit to |
164 | * complete release_in_xmit(). (Subsequent threads entering | |
165 | * rds_send_xmit() will bail on !rds_conn_up(). | |
9c79440e SV |
166 | * |
167 | * However an incoming syn-ack at this point would end up | |
168 | * marking the conn as RDS_CONN_UP, and would again permit | |
169 | * rds_send_xmi() threads through, so ideally we would | |
170 | * synchronize on RDS_CONN_UP after lock_sock(), but cannot | |
171 | * do that: waiting on !RDS_IN_XMIT after lock_sock() may | |
172 | * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT | |
173 | * would not get set. As a result, we set c_state to | |
174 | * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change | |
175 | * cannot mark rds_conn_path_up() in the window before lock_sock() | |
335b48d9 | 176 | */ |
ea3b1ea5 SV |
177 | atomic_set(&cp->cp_state, RDS_CONN_RESETTING); |
178 | wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags)); | |
335b48d9 SV |
179 | lock_sock(osock->sk); |
180 | /* reset receive side state for rds_tcp_data_recv() for osock */ | |
ac3615e7 SV |
181 | cancel_delayed_work_sync(&cp->cp_send_w); |
182 | cancel_delayed_work_sync(&cp->cp_recv_w); | |
335b48d9 SV |
183 | if (tc->t_tinc) { |
184 | rds_inc_put(&tc->t_tinc->ti_inc); | |
185 | tc->t_tinc = NULL; | |
186 | } | |
187 | tc->t_tinc_hdr_rem = sizeof(struct rds_header); | |
188 | tc->t_tinc_data_rem = 0; | |
ac3615e7 | 189 | rds_tcp_restore_callbacks(osock, tc); |
335b48d9 SV |
190 | release_sock(osock->sk); |
191 | sock_release(osock); | |
192 | newsock: | |
ea3b1ea5 | 193 | rds_send_path_reset(cp); |
335b48d9 | 194 | lock_sock(sock->sk); |
ac3615e7 | 195 | rds_tcp_set_callbacks(sock, cp); |
335b48d9 SV |
196 | release_sock(sock->sk); |
197 | } | |
198 | ||
199 | /* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments | |
200 | * above rds_tcp_reset_callbacks for notes about synchronization | |
201 | * with data path | |
70041088 | 202 | */ |
ea3b1ea5 | 203 | void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp) |
70041088 | 204 | { |
ea3b1ea5 | 205 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
70041088 AG |
206 | |
207 | rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc); | |
208 | write_lock_bh(&sock->sk->sk_callback_lock); | |
209 | ||
210 | /* done under the callback_lock to serialize with write_space */ | |
211 | spin_lock(&rds_tcp_tc_list_lock); | |
212 | list_add_tail(&tc->t_list_item, &rds_tcp_tc_list); | |
e65d4d96 | 213 | #if IS_ENABLED(CONFIG_IPV6) |
1e2b44e7 | 214 | rds6_tcp_tc_count++; |
e65d4d96 | 215 | #endif |
1e2b44e7 KCP |
216 | if (!tc->t_cpath->cp_conn->c_isv6) |
217 | rds_tcp_tc_count++; | |
70041088 AG |
218 | spin_unlock(&rds_tcp_tc_list_lock); |
219 | ||
220 | /* accepted sockets need our listen data ready undone */ | |
221 | if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready) | |
222 | sock->sk->sk_data_ready = sock->sk->sk_user_data; | |
223 | ||
224 | tc->t_sock = sock; | |
ea3b1ea5 | 225 | tc->t_cpath = cp; |
70041088 AG |
226 | tc->t_orig_data_ready = sock->sk->sk_data_ready; |
227 | tc->t_orig_write_space = sock->sk->sk_write_space; | |
228 | tc->t_orig_state_change = sock->sk->sk_state_change; | |
229 | ||
ea3b1ea5 | 230 | sock->sk->sk_user_data = cp; |
70041088 AG |
231 | sock->sk->sk_data_ready = rds_tcp_data_ready; |
232 | sock->sk->sk_write_space = rds_tcp_write_space; | |
233 | sock->sk->sk_state_change = rds_tcp_state_change; | |
234 | ||
235 | write_unlock_bh(&sock->sk->sk_callback_lock); | |
236 | } | |
237 | ||
1e2b44e7 KCP |
238 | /* Handle RDS_INFO_TCP_SOCKETS socket option. It only returns IPv4 |
239 | * connections for backward compatibility. | |
240 | */ | |
1ac507d4 | 241 | static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len, |
70041088 AG |
242 | struct rds_info_iterator *iter, |
243 | struct rds_info_lengths *lens) | |
244 | { | |
245 | struct rds_info_tcp_socket tsinfo; | |
246 | struct rds_tcp_connection *tc; | |
247 | unsigned long flags; | |
70041088 AG |
248 | |
249 | spin_lock_irqsave(&rds_tcp_tc_list_lock, flags); | |
250 | ||
251 | if (len / sizeof(tsinfo) < rds_tcp_tc_count) | |
252 | goto out; | |
253 | ||
254 | list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) { | |
1e2b44e7 | 255 | struct inet_sock *inet = inet_sk(tc->t_sock->sk); |
70041088 | 256 | |
1e2b44e7 KCP |
257 | if (tc->t_cpath->cp_conn->c_isv6) |
258 | continue; | |
259 | ||
260 | tsinfo.local_addr = inet->inet_saddr; | |
261 | tsinfo.local_port = inet->inet_sport; | |
262 | tsinfo.peer_addr = inet->inet_daddr; | |
263 | tsinfo.peer_port = inet->inet_dport; | |
70041088 AG |
264 | |
265 | tsinfo.hdr_rem = tc->t_tinc_hdr_rem; | |
266 | tsinfo.data_rem = tc->t_tinc_data_rem; | |
267 | tsinfo.last_sent_nxt = tc->t_last_sent_nxt; | |
268 | tsinfo.last_expected_una = tc->t_last_expected_una; | |
269 | tsinfo.last_seen_una = tc->t_last_seen_una; | |
270 | ||
271 | rds_info_copy(iter, &tsinfo, sizeof(tsinfo)); | |
272 | } | |
273 | ||
274 | out: | |
275 | lens->nr = rds_tcp_tc_count; | |
276 | lens->each = sizeof(tsinfo); | |
277 | ||
278 | spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags); | |
279 | } | |
280 | ||
e65d4d96 | 281 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
282 | /* Handle RDS6_INFO_TCP_SOCKETS socket option. It returns both IPv4 and |
283 | * IPv6 connections. IPv4 connection address is returned in an IPv4 mapped | |
284 | * address. | |
285 | */ | |
286 | static void rds6_tcp_tc_info(struct socket *sock, unsigned int len, | |
287 | struct rds_info_iterator *iter, | |
288 | struct rds_info_lengths *lens) | |
289 | { | |
290 | struct rds6_info_tcp_socket tsinfo6; | |
291 | struct rds_tcp_connection *tc; | |
292 | unsigned long flags; | |
293 | ||
294 | spin_lock_irqsave(&rds_tcp_tc_list_lock, flags); | |
295 | ||
296 | if (len / sizeof(tsinfo6) < rds6_tcp_tc_count) | |
297 | goto out; | |
298 | ||
299 | list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) { | |
300 | struct sock *sk = tc->t_sock->sk; | |
301 | struct inet_sock *inet = inet_sk(sk); | |
302 | ||
303 | tsinfo6.local_addr = sk->sk_v6_rcv_saddr; | |
304 | tsinfo6.local_port = inet->inet_sport; | |
305 | tsinfo6.peer_addr = sk->sk_v6_daddr; | |
306 | tsinfo6.peer_port = inet->inet_dport; | |
307 | ||
308 | tsinfo6.hdr_rem = tc->t_tinc_hdr_rem; | |
309 | tsinfo6.data_rem = tc->t_tinc_data_rem; | |
310 | tsinfo6.last_sent_nxt = tc->t_last_sent_nxt; | |
311 | tsinfo6.last_expected_una = tc->t_last_expected_una; | |
312 | tsinfo6.last_seen_una = tc->t_last_seen_una; | |
313 | ||
314 | rds_info_copy(iter, &tsinfo6, sizeof(tsinfo6)); | |
315 | } | |
316 | ||
317 | out: | |
318 | lens->nr = rds6_tcp_tc_count; | |
319 | lens->each = sizeof(tsinfo6); | |
320 | ||
321 | spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags); | |
322 | } | |
e65d4d96 | 323 | #endif |
b7ff8b10 | 324 | |
eee2fa6a KCP |
325 | static int rds_tcp_laddr_check(struct net *net, const struct in6_addr *addr, |
326 | __u32 scope_id) | |
70041088 | 327 | { |
eee2fa6a | 328 | struct net_device *dev = NULL; |
e65d4d96 | 329 | #if IS_ENABLED(CONFIG_IPV6) |
eee2fa6a | 330 | int ret; |
e65d4d96 | 331 | #endif |
eee2fa6a KCP |
332 | |
333 | if (ipv6_addr_v4mapped(addr)) { | |
334 | if (inet_addr_type(net, addr->s6_addr32[3]) == RTN_LOCAL) | |
335 | return 0; | |
336 | return -EADDRNOTAVAIL; | |
337 | } | |
338 | ||
339 | /* If the scope_id is specified, check only those addresses | |
340 | * hosted on the specified interface. | |
341 | */ | |
342 | if (scope_id != 0) { | |
343 | rcu_read_lock(); | |
344 | dev = dev_get_by_index_rcu(net, scope_id); | |
345 | /* scope_id is not valid... */ | |
346 | if (!dev) { | |
347 | rcu_read_unlock(); | |
348 | return -EADDRNOTAVAIL; | |
349 | } | |
350 | rcu_read_unlock(); | |
351 | } | |
e65d4d96 | 352 | #if IS_ENABLED(CONFIG_IPV6) |
eee2fa6a KCP |
353 | ret = ipv6_chk_addr(net, addr, dev, 0); |
354 | if (ret) | |
70041088 | 355 | return 0; |
e65d4d96 | 356 | #endif |
70041088 AG |
357 | return -EADDRNOTAVAIL; |
358 | } | |
359 | ||
66261da1 SV |
360 | static void rds_tcp_conn_free(void *arg) |
361 | { | |
362 | struct rds_tcp_connection *tc = arg; | |
53d0e83f | 363 | unsigned long flags; |
66261da1 SV |
364 | |
365 | rdsdebug("freeing tc %p\n", tc); | |
366 | ||
53d0e83f | 367 | spin_lock_irqsave(&rds_tcp_conn_lock, flags); |
66261da1 SV |
368 | if (!tc->t_tcp_node_detached) |
369 | list_del(&tc->t_tcp_node); | |
53d0e83f | 370 | spin_unlock_irqrestore(&rds_tcp_conn_lock, flags); |
66261da1 SV |
371 | |
372 | kmem_cache_free(rds_tcp_conn_slab, tc); | |
373 | } | |
374 | ||
70041088 AG |
375 | static int rds_tcp_conn_alloc(struct rds_connection *conn, gfp_t gfp) |
376 | { | |
377 | struct rds_tcp_connection *tc; | |
66261da1 SV |
378 | int i, j; |
379 | int ret = 0; | |
70041088 | 380 | |
02105b2c SV |
381 | for (i = 0; i < RDS_MPATH_WORKERS; i++) { |
382 | tc = kmem_cache_alloc(rds_tcp_conn_slab, gfp); | |
66261da1 SV |
383 | if (!tc) { |
384 | ret = -ENOMEM; | |
ebeeb1ad | 385 | goto fail; |
66261da1 | 386 | } |
02105b2c SV |
387 | mutex_init(&tc->t_conn_path_lock); |
388 | tc->t_sock = NULL; | |
389 | tc->t_tinc = NULL; | |
390 | tc->t_tinc_hdr_rem = sizeof(struct rds_header); | |
391 | tc->t_tinc_data_rem = 0; | |
70041088 | 392 | |
02105b2c SV |
393 | conn->c_path[i].cp_transport_data = tc; |
394 | tc->t_cpath = &conn->c_path[i]; | |
ebeeb1ad | 395 | tc->t_tcp_node_detached = true; |
70041088 | 396 | |
02105b2c SV |
397 | rdsdebug("rds_conn_path [%d] tc %p\n", i, |
398 | conn->c_path[i].cp_transport_data); | |
399 | } | |
53d0e83f | 400 | spin_lock_irq(&rds_tcp_conn_lock); |
ebeeb1ad SV |
401 | for (i = 0; i < RDS_MPATH_WORKERS; i++) { |
402 | tc = conn->c_path[i].cp_transport_data; | |
403 | tc->t_tcp_node_detached = false; | |
404 | list_add_tail(&tc->t_tcp_node, &rds_tcp_conn_list); | |
405 | } | |
53d0e83f | 406 | spin_unlock_irq(&rds_tcp_conn_lock); |
ebeeb1ad | 407 | fail: |
66261da1 SV |
408 | if (ret) { |
409 | for (j = 0; j < i; j++) | |
410 | rds_tcp_conn_free(conn->c_path[j].cp_transport_data); | |
411 | } | |
412 | return ret; | |
70041088 AG |
413 | } |
414 | ||
afb4164d SV |
415 | static bool list_has_conn(struct list_head *list, struct rds_connection *conn) |
416 | { | |
417 | struct rds_tcp_connection *tc, *_tc; | |
418 | ||
419 | list_for_each_entry_safe(tc, _tc, list, t_tcp_node) { | |
420 | if (tc->t_cpath->cp_conn == conn) | |
421 | return true; | |
422 | } | |
423 | return false; | |
424 | } | |
425 | ||
ebeeb1ad SV |
426 | static void rds_tcp_set_unloading(void) |
427 | { | |
428 | atomic_set(&rds_tcp_unloading, 1); | |
429 | } | |
430 | ||
431 | static bool rds_tcp_is_unloading(struct rds_connection *conn) | |
432 | { | |
433 | return atomic_read(&rds_tcp_unloading) != 0; | |
434 | } | |
435 | ||
70041088 AG |
436 | static void rds_tcp_destroy_conns(void) |
437 | { | |
438 | struct rds_tcp_connection *tc, *_tc; | |
439 | LIST_HEAD(tmp_list); | |
440 | ||
441 | /* avoid calling conn_destroy with irqs off */ | |
442 | spin_lock_irq(&rds_tcp_conn_lock); | |
afb4164d SV |
443 | list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) { |
444 | if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn)) | |
445 | list_move_tail(&tc->t_tcp_node, &tmp_list); | |
446 | } | |
70041088 AG |
447 | spin_unlock_irq(&rds_tcp_conn_lock); |
448 | ||
26e4e6bb | 449 | list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node) |
02105b2c | 450 | rds_conn_destroy(tc->t_cpath->cp_conn); |
70041088 AG |
451 | } |
452 | ||
467fa153 | 453 | static void rds_tcp_exit(void); |
70041088 AG |
454 | |
455 | struct rds_transport rds_tcp_transport = { | |
456 | .laddr_check = rds_tcp_laddr_check, | |
226f7a7d SV |
457 | .xmit_path_prepare = rds_tcp_xmit_path_prepare, |
458 | .xmit_path_complete = rds_tcp_xmit_path_complete, | |
70041088 | 459 | .xmit = rds_tcp_xmit, |
2da43c4a | 460 | .recv_path = rds_tcp_recv_path, |
70041088 AG |
461 | .conn_alloc = rds_tcp_conn_alloc, |
462 | .conn_free = rds_tcp_conn_free, | |
b04e8554 | 463 | .conn_path_connect = rds_tcp_conn_path_connect, |
226f7a7d | 464 | .conn_path_shutdown = rds_tcp_conn_path_shutdown, |
70041088 | 465 | .inc_copy_to_user = rds_tcp_inc_copy_to_user, |
70041088 AG |
466 | .inc_free = rds_tcp_inc_free, |
467 | .stats_info_copy = rds_tcp_stats_info_copy, | |
468 | .exit = rds_tcp_exit, | |
469 | .t_owner = THIS_MODULE, | |
470 | .t_name = "tcp", | |
335776bd | 471 | .t_type = RDS_TRANS_TCP, |
70041088 | 472 | .t_prefer_loopback = 1, |
5916e2c1 | 473 | .t_mp_capable = 1, |
ebeeb1ad | 474 | .t_unloading = rds_tcp_is_unloading, |
70041088 AG |
475 | }; |
476 | ||
c7d03a00 | 477 | static unsigned int rds_tcp_netid; |
467fa153 SV |
478 | |
479 | /* per-network namespace private data for this module */ | |
480 | struct rds_tcp_net { | |
481 | struct socket *rds_tcp_listen_sock; | |
482 | struct work_struct rds_tcp_accept_w; | |
c6a58ffe SV |
483 | struct ctl_table_header *rds_tcp_sysctl; |
484 | struct ctl_table *ctl_table; | |
485 | int sndbuf_size; | |
486 | int rcvbuf_size; | |
467fa153 SV |
487 | }; |
488 | ||
c6a58ffe SV |
489 | /* All module specific customizations to the RDS-TCP socket should be done in |
490 | * rds_tcp_tune() and applied after socket creation. | |
491 | */ | |
492 | void rds_tcp_tune(struct socket *sock) | |
493 | { | |
494 | struct sock *sk = sock->sk; | |
495 | struct net *net = sock_net(sk); | |
496 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
497 | ||
498 | rds_tcp_nonagle(sock); | |
499 | lock_sock(sk); | |
500 | if (rtn->sndbuf_size > 0) { | |
501 | sk->sk_sndbuf = rtn->sndbuf_size; | |
502 | sk->sk_userlocks |= SOCK_SNDBUF_LOCK; | |
503 | } | |
504 | if (rtn->rcvbuf_size > 0) { | |
505 | sk->sk_sndbuf = rtn->rcvbuf_size; | |
506 | sk->sk_userlocks |= SOCK_RCVBUF_LOCK; | |
507 | } | |
508 | release_sock(sk); | |
509 | } | |
510 | ||
467fa153 SV |
511 | static void rds_tcp_accept_worker(struct work_struct *work) |
512 | { | |
513 | struct rds_tcp_net *rtn = container_of(work, | |
514 | struct rds_tcp_net, | |
515 | rds_tcp_accept_w); | |
516 | ||
517 | while (rds_tcp_accept_one(rtn->rds_tcp_listen_sock) == 0) | |
518 | cond_resched(); | |
519 | } | |
520 | ||
521 | void rds_tcp_accept_work(struct sock *sk) | |
522 | { | |
523 | struct net *net = sock_net(sk); | |
524 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
525 | ||
526 | queue_work(rds_wq, &rtn->rds_tcp_accept_w); | |
527 | } | |
528 | ||
529 | static __net_init int rds_tcp_init_net(struct net *net) | |
530 | { | |
531 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
c6a58ffe SV |
532 | struct ctl_table *tbl; |
533 | int err = 0; | |
467fa153 | 534 | |
c6a58ffe SV |
535 | memset(rtn, 0, sizeof(*rtn)); |
536 | ||
537 | /* {snd, rcv}buf_size default to 0, which implies we let the | |
538 | * stack pick the value, and permit auto-tuning of buffer size. | |
539 | */ | |
540 | if (net == &init_net) { | |
541 | tbl = rds_tcp_sysctl_table; | |
542 | } else { | |
543 | tbl = kmemdup(rds_tcp_sysctl_table, | |
544 | sizeof(rds_tcp_sysctl_table), GFP_KERNEL); | |
545 | if (!tbl) { | |
546 | pr_warn("could not set allocate syctl table\n"); | |
547 | return -ENOMEM; | |
548 | } | |
549 | rtn->ctl_table = tbl; | |
550 | } | |
551 | tbl[RDS_TCP_SNDBUF].data = &rtn->sndbuf_size; | |
552 | tbl[RDS_TCP_RCVBUF].data = &rtn->rcvbuf_size; | |
553 | rtn->rds_tcp_sysctl = register_net_sysctl(net, "net/rds/tcp", tbl); | |
554 | if (!rtn->rds_tcp_sysctl) { | |
555 | pr_warn("could not register sysctl\n"); | |
556 | err = -ENOMEM; | |
557 | goto fail; | |
558 | } | |
e65d4d96 KCP |
559 | |
560 | #if IS_ENABLED(CONFIG_IPV6) | |
1e2b44e7 | 561 | rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, true); |
e65d4d96 KCP |
562 | #else |
563 | rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, false); | |
564 | #endif | |
467fa153 | 565 | if (!rtn->rds_tcp_listen_sock) { |
1e2b44e7 KCP |
566 | pr_warn("could not set up IPv6 listen sock\n"); |
567 | ||
e65d4d96 | 568 | #if IS_ENABLED(CONFIG_IPV6) |
1e2b44e7 KCP |
569 | /* Try IPv4 as some systems disable IPv6 */ |
570 | rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net, false); | |
571 | if (!rtn->rds_tcp_listen_sock) { | |
e65d4d96 | 572 | #endif |
1e2b44e7 KCP |
573 | unregister_net_sysctl_table(rtn->rds_tcp_sysctl); |
574 | rtn->rds_tcp_sysctl = NULL; | |
575 | err = -EAFNOSUPPORT; | |
576 | goto fail; | |
e65d4d96 | 577 | #if IS_ENABLED(CONFIG_IPV6) |
1e2b44e7 | 578 | } |
e65d4d96 | 579 | #endif |
467fa153 SV |
580 | } |
581 | INIT_WORK(&rtn->rds_tcp_accept_w, rds_tcp_accept_worker); | |
582 | return 0; | |
c6a58ffe SV |
583 | |
584 | fail: | |
585 | if (net != &init_net) | |
586 | kfree(tbl); | |
587 | return err; | |
467fa153 SV |
588 | } |
589 | ||
467fa153 SV |
590 | static void rds_tcp_kill_sock(struct net *net) |
591 | { | |
592 | struct rds_tcp_connection *tc, *_tc; | |
467fa153 SV |
593 | LIST_HEAD(tmp_list); |
594 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
b21dd450 | 595 | struct socket *lsock = rtn->rds_tcp_listen_sock; |
467fa153 | 596 | |
467fa153 | 597 | rtn->rds_tcp_listen_sock = NULL; |
b21dd450 | 598 | rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w); |
53d0e83f | 599 | spin_lock_irq(&rds_tcp_conn_lock); |
467fa153 | 600 | list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) { |
681648e6 | 601 | struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net); |
467fa153 SV |
602 | |
603 | if (net != c_net || !tc->t_sock) | |
604 | continue; | |
f10b4cff | 605 | if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn)) { |
afb4164d | 606 | list_move_tail(&tc->t_tcp_node, &tmp_list); |
f10b4cff SV |
607 | } else { |
608 | list_del(&tc->t_tcp_node); | |
609 | tc->t_tcp_node_detached = true; | |
610 | } | |
467fa153 | 611 | } |
53d0e83f | 612 | spin_unlock_irq(&rds_tcp_conn_lock); |
2d746c93 | 613 | list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node) |
02105b2c | 614 | rds_conn_destroy(tc->t_cpath->cp_conn); |
467fa153 SV |
615 | } |
616 | ||
bdf5bd7f | 617 | static void __net_exit rds_tcp_exit_net(struct net *net) |
a93d01f5 SV |
618 | { |
619 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
b21dd450 | 620 | |
bdf5bd7f | 621 | rds_tcp_kill_sock(net); |
a93d01f5 | 622 | |
bdf5bd7f SV |
623 | if (rtn->rds_tcp_sysctl) |
624 | unregister_net_sysctl_table(rtn->rds_tcp_sysctl); | |
625 | ||
626 | if (net != &init_net && rtn->ctl_table) | |
627 | kfree(rtn->ctl_table); | |
a93d01f5 SV |
628 | } |
629 | ||
bdf5bd7f SV |
630 | static struct pernet_operations rds_tcp_net_ops = { |
631 | .init = rds_tcp_init_net, | |
632 | .exit = rds_tcp_exit_net, | |
633 | .id = &rds_tcp_netid, | |
634 | .size = sizeof(struct rds_tcp_net), | |
bdf5bd7f SV |
635 | }; |
636 | ||
637 | void *rds_tcp_listen_sock_def_readable(struct net *net) | |
467fa153 | 638 | { |
bdf5bd7f SV |
639 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); |
640 | struct socket *lsock = rtn->rds_tcp_listen_sock; | |
467fa153 | 641 | |
bdf5bd7f SV |
642 | if (!lsock) |
643 | return NULL; | |
467fa153 | 644 | |
bdf5bd7f | 645 | return lsock->sk->sk_user_data; |
467fa153 SV |
646 | } |
647 | ||
c6a58ffe SV |
648 | /* when sysctl is used to modify some kernel socket parameters,this |
649 | * function resets the RDS connections in that netns so that we can | |
650 | * restart with new parameters. The assumption is that such reset | |
651 | * events are few and far-between. | |
652 | */ | |
653 | static void rds_tcp_sysctl_reset(struct net *net) | |
654 | { | |
655 | struct rds_tcp_connection *tc, *_tc; | |
656 | ||
53d0e83f | 657 | spin_lock_irq(&rds_tcp_conn_lock); |
c6a58ffe | 658 | list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) { |
681648e6 | 659 | struct net *c_net = read_pnet(&tc->t_cpath->cp_conn->c_net); |
c6a58ffe SV |
660 | |
661 | if (net != c_net || !tc->t_sock) | |
662 | continue; | |
663 | ||
02105b2c | 664 | /* reconnect with new parameters */ |
aed20a53 | 665 | rds_conn_path_drop(tc->t_cpath, false); |
c6a58ffe | 666 | } |
53d0e83f | 667 | spin_unlock_irq(&rds_tcp_conn_lock); |
c6a58ffe SV |
668 | } |
669 | ||
670 | static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write, | |
671 | void __user *buffer, size_t *lenp, | |
672 | loff_t *fpos) | |
673 | { | |
674 | struct net *net = current->nsproxy->net_ns; | |
675 | int err; | |
676 | ||
677 | err = proc_dointvec_minmax(ctl, write, buffer, lenp, fpos); | |
678 | if (err < 0) { | |
679 | pr_warn("Invalid input. Must be >= %d\n", | |
680 | *(int *)(ctl->extra1)); | |
681 | return err; | |
682 | } | |
683 | if (write) | |
684 | rds_tcp_sysctl_reset(net); | |
685 | return 0; | |
686 | } | |
687 | ||
467fa153 SV |
688 | static void rds_tcp_exit(void) |
689 | { | |
ebeeb1ad SV |
690 | rds_tcp_set_unloading(); |
691 | synchronize_rcu(); | |
467fa153 | 692 | rds_info_deregister_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info); |
e65d4d96 | 693 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 | 694 | rds_info_deregister_func(RDS6_INFO_TCP_SOCKETS, rds6_tcp_tc_info); |
e65d4d96 | 695 | #endif |
bdf5bd7f | 696 | unregister_pernet_device(&rds_tcp_net_ops); |
467fa153 SV |
697 | rds_tcp_destroy_conns(); |
698 | rds_trans_unregister(&rds_tcp_transport); | |
699 | rds_tcp_recv_exit(); | |
700 | kmem_cache_destroy(rds_tcp_conn_slab); | |
701 | } | |
702 | module_exit(rds_tcp_exit); | |
703 | ||
ff51bf84 | 704 | static int rds_tcp_init(void) |
70041088 AG |
705 | { |
706 | int ret; | |
707 | ||
708 | rds_tcp_conn_slab = kmem_cache_create("rds_tcp_connection", | |
709 | sizeof(struct rds_tcp_connection), | |
710 | 0, 0, NULL); | |
8690bfa1 | 711 | if (!rds_tcp_conn_slab) { |
70041088 AG |
712 | ret = -ENOMEM; |
713 | goto out; | |
714 | } | |
715 | ||
16c09b1c SV |
716 | ret = rds_tcp_recv_init(); |
717 | if (ret) | |
3b5923f0 | 718 | goto out_slab; |
467fa153 | 719 | |
bdf5bd7f | 720 | ret = register_pernet_device(&rds_tcp_net_ops); |
467fa153 | 721 | if (ret) |
16c09b1c | 722 | goto out_recv; |
467fa153 | 723 | |
a8d63a53 | 724 | rds_trans_register(&rds_tcp_transport); |
70041088 | 725 | |
70041088 | 726 | rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info); |
e65d4d96 | 727 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 | 728 | rds_info_register_func(RDS6_INFO_TCP_SOCKETS, rds6_tcp_tc_info); |
e65d4d96 | 729 | #endif |
70041088 AG |
730 | |
731 | goto out; | |
16c09b1c SV |
732 | out_recv: |
733 | rds_tcp_recv_exit(); | |
3b5923f0 | 734 | out_slab: |
70041088 AG |
735 | kmem_cache_destroy(rds_tcp_conn_slab); |
736 | out: | |
737 | return ret; | |
738 | } | |
739 | module_init(rds_tcp_init); | |
740 | ||
741 | MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>"); | |
742 | MODULE_DESCRIPTION("RDS: TCP transport"); | |
743 | MODULE_LICENSE("Dual BSD/GPL"); |