Commit | Line | Data |
---|---|---|
70041088 AG |
1 | /* |
2 | * Copyright (c) 2006 Oracle. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | * | |
32 | */ | |
33 | #include <linux/kernel.h> | |
5a0e3ad6 | 34 | #include <linux/slab.h> |
70041088 | 35 | #include <linux/in.h> |
3a9a231d | 36 | #include <linux/module.h> |
70041088 | 37 | #include <net/tcp.h> |
467fa153 SV |
38 | #include <net/net_namespace.h> |
39 | #include <net/netns/generic.h> | |
70041088 AG |
40 | |
41 | #include "rds.h" | |
42 | #include "tcp.h" | |
43 | ||
44 | /* only for info exporting */ | |
45 | static DEFINE_SPINLOCK(rds_tcp_tc_list_lock); | |
46 | static LIST_HEAD(rds_tcp_tc_list); | |
ff51bf84 | 47 | static unsigned int rds_tcp_tc_count; |
70041088 AG |
48 | |
49 | /* Track rds_tcp_connection structs so they can be cleaned up */ | |
50 | static DEFINE_SPINLOCK(rds_tcp_conn_lock); | |
51 | static LIST_HEAD(rds_tcp_conn_list); | |
52 | ||
53 | static struct kmem_cache *rds_tcp_conn_slab; | |
54 | ||
c6a58ffe SV |
55 | static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write, |
56 | void __user *buffer, size_t *lenp, | |
57 | loff_t *fpos); | |
58 | ||
af73e72d WY |
59 | static int rds_tcp_min_sndbuf = SOCK_MIN_SNDBUF; |
60 | static int rds_tcp_min_rcvbuf = SOCK_MIN_RCVBUF; | |
c6a58ffe SV |
61 | |
62 | static struct ctl_table rds_tcp_sysctl_table[] = { | |
63 | #define RDS_TCP_SNDBUF 0 | |
64 | { | |
65 | .procname = "rds_tcp_sndbuf", | |
66 | /* data is per-net pointer */ | |
67 | .maxlen = sizeof(int), | |
68 | .mode = 0644, | |
69 | .proc_handler = rds_tcp_skbuf_handler, | |
70 | .extra1 = &rds_tcp_min_sndbuf, | |
71 | }, | |
72 | #define RDS_TCP_RCVBUF 1 | |
73 | { | |
74 | .procname = "rds_tcp_rcvbuf", | |
75 | /* data is per-net pointer */ | |
76 | .maxlen = sizeof(int), | |
77 | .mode = 0644, | |
78 | .proc_handler = rds_tcp_skbuf_handler, | |
79 | .extra1 = &rds_tcp_min_rcvbuf, | |
80 | }, | |
81 | { } | |
82 | }; | |
83 | ||
70041088 AG |
84 | /* doing it this way avoids calling tcp_sk() */ |
85 | void rds_tcp_nonagle(struct socket *sock) | |
86 | { | |
70041088 AG |
87 | int val = 1; |
88 | ||
e73a67f7 | 89 | kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY, (void *)&val, |
70041088 | 90 | sizeof(val)); |
70041088 AG |
91 | } |
92 | ||
70041088 AG |
93 | u32 rds_tcp_snd_nxt(struct rds_tcp_connection *tc) |
94 | { | |
95 | return tcp_sk(tc->t_sock->sk)->snd_nxt; | |
96 | } | |
97 | ||
98 | u32 rds_tcp_snd_una(struct rds_tcp_connection *tc) | |
99 | { | |
100 | return tcp_sk(tc->t_sock->sk)->snd_una; | |
101 | } | |
102 | ||
103 | void rds_tcp_restore_callbacks(struct socket *sock, | |
104 | struct rds_tcp_connection *tc) | |
105 | { | |
106 | rdsdebug("restoring sock %p callbacks from tc %p\n", sock, tc); | |
107 | write_lock_bh(&sock->sk->sk_callback_lock); | |
108 | ||
109 | /* done under the callback_lock to serialize with write_space */ | |
110 | spin_lock(&rds_tcp_tc_list_lock); | |
111 | list_del_init(&tc->t_list_item); | |
112 | rds_tcp_tc_count--; | |
113 | spin_unlock(&rds_tcp_tc_list_lock); | |
114 | ||
115 | tc->t_sock = NULL; | |
116 | ||
117 | sock->sk->sk_write_space = tc->t_orig_write_space; | |
118 | sock->sk->sk_data_ready = tc->t_orig_data_ready; | |
119 | sock->sk->sk_state_change = tc->t_orig_state_change; | |
120 | sock->sk->sk_user_data = NULL; | |
121 | ||
122 | write_unlock_bh(&sock->sk->sk_callback_lock); | |
123 | } | |
124 | ||
125 | /* | |
335b48d9 SV |
126 | * rds_tcp_reset_callbacks() switches the to the new sock and |
127 | * returns the existing tc->t_sock. | |
128 | * | |
129 | * The only functions that set tc->t_sock are rds_tcp_set_callbacks | |
130 | * and rds_tcp_reset_callbacks. Send and receive trust that | |
131 | * it is set. The absence of RDS_CONN_UP bit protects those paths | |
132 | * from being called while it isn't set. | |
133 | */ | |
134 | void rds_tcp_reset_callbacks(struct socket *sock, | |
ea3b1ea5 | 135 | struct rds_conn_path *cp) |
335b48d9 | 136 | { |
ea3b1ea5 | 137 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
335b48d9 SV |
138 | struct socket *osock = tc->t_sock; |
139 | ||
140 | if (!osock) | |
141 | goto newsock; | |
142 | ||
143 | /* Need to resolve a duelling SYN between peers. | |
144 | * We have an outstanding SYN to this peer, which may | |
145 | * potentially have transitioned to the RDS_CONN_UP state, | |
146 | * so we must quiesce any send threads before resetting | |
ea3b1ea5 SV |
147 | * cp_transport_data. We quiesce these threads by setting |
148 | * cp_state to something other than RDS_CONN_UP, and then | |
335b48d9 SV |
149 | * waiting for any existing threads in rds_send_xmit to |
150 | * complete release_in_xmit(). (Subsequent threads entering | |
151 | * rds_send_xmit() will bail on !rds_conn_up(). | |
9c79440e SV |
152 | * |
153 | * However an incoming syn-ack at this point would end up | |
154 | * marking the conn as RDS_CONN_UP, and would again permit | |
155 | * rds_send_xmi() threads through, so ideally we would | |
156 | * synchronize on RDS_CONN_UP after lock_sock(), but cannot | |
157 | * do that: waiting on !RDS_IN_XMIT after lock_sock() may | |
158 | * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT | |
159 | * would not get set. As a result, we set c_state to | |
160 | * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change | |
161 | * cannot mark rds_conn_path_up() in the window before lock_sock() | |
335b48d9 | 162 | */ |
ea3b1ea5 SV |
163 | atomic_set(&cp->cp_state, RDS_CONN_RESETTING); |
164 | wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags)); | |
335b48d9 SV |
165 | lock_sock(osock->sk); |
166 | /* reset receive side state for rds_tcp_data_recv() for osock */ | |
ac3615e7 SV |
167 | cancel_delayed_work_sync(&cp->cp_send_w); |
168 | cancel_delayed_work_sync(&cp->cp_recv_w); | |
335b48d9 SV |
169 | if (tc->t_tinc) { |
170 | rds_inc_put(&tc->t_tinc->ti_inc); | |
171 | tc->t_tinc = NULL; | |
172 | } | |
173 | tc->t_tinc_hdr_rem = sizeof(struct rds_header); | |
174 | tc->t_tinc_data_rem = 0; | |
ac3615e7 | 175 | rds_tcp_restore_callbacks(osock, tc); |
335b48d9 SV |
176 | release_sock(osock->sk); |
177 | sock_release(osock); | |
178 | newsock: | |
ea3b1ea5 | 179 | rds_send_path_reset(cp); |
335b48d9 | 180 | lock_sock(sock->sk); |
ac3615e7 | 181 | rds_tcp_set_callbacks(sock, cp); |
335b48d9 SV |
182 | release_sock(sock->sk); |
183 | } | |
184 | ||
185 | /* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments | |
186 | * above rds_tcp_reset_callbacks for notes about synchronization | |
187 | * with data path | |
70041088 | 188 | */ |
ea3b1ea5 | 189 | void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp) |
70041088 | 190 | { |
ea3b1ea5 | 191 | struct rds_tcp_connection *tc = cp->cp_transport_data; |
70041088 AG |
192 | |
193 | rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc); | |
194 | write_lock_bh(&sock->sk->sk_callback_lock); | |
195 | ||
196 | /* done under the callback_lock to serialize with write_space */ | |
197 | spin_lock(&rds_tcp_tc_list_lock); | |
198 | list_add_tail(&tc->t_list_item, &rds_tcp_tc_list); | |
199 | rds_tcp_tc_count++; | |
200 | spin_unlock(&rds_tcp_tc_list_lock); | |
201 | ||
202 | /* accepted sockets need our listen data ready undone */ | |
203 | if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready) | |
204 | sock->sk->sk_data_ready = sock->sk->sk_user_data; | |
205 | ||
206 | tc->t_sock = sock; | |
ea3b1ea5 | 207 | tc->t_cpath = cp; |
70041088 AG |
208 | tc->t_orig_data_ready = sock->sk->sk_data_ready; |
209 | tc->t_orig_write_space = sock->sk->sk_write_space; | |
210 | tc->t_orig_state_change = sock->sk->sk_state_change; | |
211 | ||
ea3b1ea5 | 212 | sock->sk->sk_user_data = cp; |
70041088 AG |
213 | sock->sk->sk_data_ready = rds_tcp_data_ready; |
214 | sock->sk->sk_write_space = rds_tcp_write_space; | |
215 | sock->sk->sk_state_change = rds_tcp_state_change; | |
216 | ||
217 | write_unlock_bh(&sock->sk->sk_callback_lock); | |
218 | } | |
219 | ||
1ac507d4 | 220 | static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len, |
70041088 AG |
221 | struct rds_info_iterator *iter, |
222 | struct rds_info_lengths *lens) | |
223 | { | |
224 | struct rds_info_tcp_socket tsinfo; | |
225 | struct rds_tcp_connection *tc; | |
226 | unsigned long flags; | |
227 | struct sockaddr_in sin; | |
228 | int sinlen; | |
1ac507d4 | 229 | struct socket *sock; |
70041088 AG |
230 | |
231 | spin_lock_irqsave(&rds_tcp_tc_list_lock, flags); | |
232 | ||
233 | if (len / sizeof(tsinfo) < rds_tcp_tc_count) | |
234 | goto out; | |
235 | ||
236 | list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) { | |
237 | ||
1ac507d4 SV |
238 | sock = tc->t_sock; |
239 | if (sock) { | |
240 | sock->ops->getname(sock, (struct sockaddr *)&sin, | |
241 | &sinlen, 0); | |
242 | tsinfo.local_addr = sin.sin_addr.s_addr; | |
243 | tsinfo.local_port = sin.sin_port; | |
244 | sock->ops->getname(sock, (struct sockaddr *)&sin, | |
245 | &sinlen, 1); | |
246 | tsinfo.peer_addr = sin.sin_addr.s_addr; | |
247 | tsinfo.peer_port = sin.sin_port; | |
248 | } | |
70041088 AG |
249 | |
250 | tsinfo.hdr_rem = tc->t_tinc_hdr_rem; | |
251 | tsinfo.data_rem = tc->t_tinc_data_rem; | |
252 | tsinfo.last_sent_nxt = tc->t_last_sent_nxt; | |
253 | tsinfo.last_expected_una = tc->t_last_expected_una; | |
254 | tsinfo.last_seen_una = tc->t_last_seen_una; | |
255 | ||
256 | rds_info_copy(iter, &tsinfo, sizeof(tsinfo)); | |
257 | } | |
258 | ||
259 | out: | |
260 | lens->nr = rds_tcp_tc_count; | |
261 | lens->each = sizeof(tsinfo); | |
262 | ||
263 | spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags); | |
264 | } | |
265 | ||
d5a8ac28 | 266 | static int rds_tcp_laddr_check(struct net *net, __be32 addr) |
70041088 | 267 | { |
d5a8ac28 | 268 | if (inet_addr_type(net, addr) == RTN_LOCAL) |
70041088 AG |
269 | return 0; |
270 | return -EADDRNOTAVAIL; | |
271 | } | |
272 | ||
273 | static int rds_tcp_conn_alloc(struct rds_connection *conn, gfp_t gfp) | |
274 | { | |
275 | struct rds_tcp_connection *tc; | |
02105b2c | 276 | int i; |
70041088 | 277 | |
02105b2c SV |
278 | for (i = 0; i < RDS_MPATH_WORKERS; i++) { |
279 | tc = kmem_cache_alloc(rds_tcp_conn_slab, gfp); | |
280 | if (!tc) | |
281 | return -ENOMEM; | |
70041088 | 282 | |
02105b2c SV |
283 | mutex_init(&tc->t_conn_path_lock); |
284 | tc->t_sock = NULL; | |
285 | tc->t_tinc = NULL; | |
286 | tc->t_tinc_hdr_rem = sizeof(struct rds_header); | |
287 | tc->t_tinc_data_rem = 0; | |
70041088 | 288 | |
02105b2c SV |
289 | conn->c_path[i].cp_transport_data = tc; |
290 | tc->t_cpath = &conn->c_path[i]; | |
70041088 | 291 | |
02105b2c SV |
292 | spin_lock_irq(&rds_tcp_conn_lock); |
293 | list_add_tail(&tc->t_tcp_node, &rds_tcp_conn_list); | |
294 | spin_unlock_irq(&rds_tcp_conn_lock); | |
295 | rdsdebug("rds_conn_path [%d] tc %p\n", i, | |
296 | conn->c_path[i].cp_transport_data); | |
297 | } | |
70041088 | 298 | |
70041088 AG |
299 | return 0; |
300 | } | |
301 | ||
302 | static void rds_tcp_conn_free(void *arg) | |
303 | { | |
304 | struct rds_tcp_connection *tc = arg; | |
8200a59f | 305 | unsigned long flags; |
70041088 | 306 | rdsdebug("freeing tc %p\n", tc); |
8200a59f PE |
307 | |
308 | spin_lock_irqsave(&rds_tcp_conn_lock, flags); | |
309 | list_del(&tc->t_tcp_node); | |
310 | spin_unlock_irqrestore(&rds_tcp_conn_lock, flags); | |
311 | ||
70041088 AG |
312 | kmem_cache_free(rds_tcp_conn_slab, tc); |
313 | } | |
314 | ||
afb4164d SV |
315 | static bool list_has_conn(struct list_head *list, struct rds_connection *conn) |
316 | { | |
317 | struct rds_tcp_connection *tc, *_tc; | |
318 | ||
319 | list_for_each_entry_safe(tc, _tc, list, t_tcp_node) { | |
320 | if (tc->t_cpath->cp_conn == conn) | |
321 | return true; | |
322 | } | |
323 | return false; | |
324 | } | |
325 | ||
70041088 AG |
326 | static void rds_tcp_destroy_conns(void) |
327 | { | |
328 | struct rds_tcp_connection *tc, *_tc; | |
329 | LIST_HEAD(tmp_list); | |
330 | ||
331 | /* avoid calling conn_destroy with irqs off */ | |
332 | spin_lock_irq(&rds_tcp_conn_lock); | |
afb4164d SV |
333 | list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) { |
334 | if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn)) | |
335 | list_move_tail(&tc->t_tcp_node, &tmp_list); | |
336 | } | |
70041088 AG |
337 | spin_unlock_irq(&rds_tcp_conn_lock); |
338 | ||
26e4e6bb | 339 | list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node) |
02105b2c | 340 | rds_conn_destroy(tc->t_cpath->cp_conn); |
70041088 AG |
341 | } |
342 | ||
467fa153 | 343 | static void rds_tcp_exit(void); |
70041088 AG |
344 | |
345 | struct rds_transport rds_tcp_transport = { | |
346 | .laddr_check = rds_tcp_laddr_check, | |
226f7a7d SV |
347 | .xmit_path_prepare = rds_tcp_xmit_path_prepare, |
348 | .xmit_path_complete = rds_tcp_xmit_path_complete, | |
70041088 | 349 | .xmit = rds_tcp_xmit, |
2da43c4a | 350 | .recv_path = rds_tcp_recv_path, |
70041088 AG |
351 | .conn_alloc = rds_tcp_conn_alloc, |
352 | .conn_free = rds_tcp_conn_free, | |
b04e8554 | 353 | .conn_path_connect = rds_tcp_conn_path_connect, |
226f7a7d | 354 | .conn_path_shutdown = rds_tcp_conn_path_shutdown, |
70041088 | 355 | .inc_copy_to_user = rds_tcp_inc_copy_to_user, |
70041088 AG |
356 | .inc_free = rds_tcp_inc_free, |
357 | .stats_info_copy = rds_tcp_stats_info_copy, | |
358 | .exit = rds_tcp_exit, | |
359 | .t_owner = THIS_MODULE, | |
360 | .t_name = "tcp", | |
335776bd | 361 | .t_type = RDS_TRANS_TCP, |
70041088 | 362 | .t_prefer_loopback = 1, |
5916e2c1 | 363 | .t_mp_capable = 1, |
70041088 AG |
364 | }; |
365 | ||
c7d03a00 | 366 | static unsigned int rds_tcp_netid; |
467fa153 SV |
367 | |
368 | /* per-network namespace private data for this module */ | |
369 | struct rds_tcp_net { | |
370 | struct socket *rds_tcp_listen_sock; | |
371 | struct work_struct rds_tcp_accept_w; | |
c6a58ffe SV |
372 | struct ctl_table_header *rds_tcp_sysctl; |
373 | struct ctl_table *ctl_table; | |
374 | int sndbuf_size; | |
375 | int rcvbuf_size; | |
467fa153 SV |
376 | }; |
377 | ||
c6a58ffe SV |
378 | /* All module specific customizations to the RDS-TCP socket should be done in |
379 | * rds_tcp_tune() and applied after socket creation. | |
380 | */ | |
381 | void rds_tcp_tune(struct socket *sock) | |
382 | { | |
383 | struct sock *sk = sock->sk; | |
384 | struct net *net = sock_net(sk); | |
385 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
386 | ||
387 | rds_tcp_nonagle(sock); | |
388 | lock_sock(sk); | |
389 | if (rtn->sndbuf_size > 0) { | |
390 | sk->sk_sndbuf = rtn->sndbuf_size; | |
391 | sk->sk_userlocks |= SOCK_SNDBUF_LOCK; | |
392 | } | |
393 | if (rtn->rcvbuf_size > 0) { | |
394 | sk->sk_sndbuf = rtn->rcvbuf_size; | |
395 | sk->sk_userlocks |= SOCK_RCVBUF_LOCK; | |
396 | } | |
397 | release_sock(sk); | |
398 | } | |
399 | ||
467fa153 SV |
400 | static void rds_tcp_accept_worker(struct work_struct *work) |
401 | { | |
402 | struct rds_tcp_net *rtn = container_of(work, | |
403 | struct rds_tcp_net, | |
404 | rds_tcp_accept_w); | |
405 | ||
406 | while (rds_tcp_accept_one(rtn->rds_tcp_listen_sock) == 0) | |
407 | cond_resched(); | |
408 | } | |
409 | ||
410 | void rds_tcp_accept_work(struct sock *sk) | |
411 | { | |
412 | struct net *net = sock_net(sk); | |
413 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
414 | ||
415 | queue_work(rds_wq, &rtn->rds_tcp_accept_w); | |
416 | } | |
417 | ||
418 | static __net_init int rds_tcp_init_net(struct net *net) | |
419 | { | |
420 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
c6a58ffe SV |
421 | struct ctl_table *tbl; |
422 | int err = 0; | |
467fa153 | 423 | |
c6a58ffe SV |
424 | memset(rtn, 0, sizeof(*rtn)); |
425 | ||
426 | /* {snd, rcv}buf_size default to 0, which implies we let the | |
427 | * stack pick the value, and permit auto-tuning of buffer size. | |
428 | */ | |
429 | if (net == &init_net) { | |
430 | tbl = rds_tcp_sysctl_table; | |
431 | } else { | |
432 | tbl = kmemdup(rds_tcp_sysctl_table, | |
433 | sizeof(rds_tcp_sysctl_table), GFP_KERNEL); | |
434 | if (!tbl) { | |
435 | pr_warn("could not set allocate syctl table\n"); | |
436 | return -ENOMEM; | |
437 | } | |
438 | rtn->ctl_table = tbl; | |
439 | } | |
440 | tbl[RDS_TCP_SNDBUF].data = &rtn->sndbuf_size; | |
441 | tbl[RDS_TCP_RCVBUF].data = &rtn->rcvbuf_size; | |
442 | rtn->rds_tcp_sysctl = register_net_sysctl(net, "net/rds/tcp", tbl); | |
443 | if (!rtn->rds_tcp_sysctl) { | |
444 | pr_warn("could not register sysctl\n"); | |
445 | err = -ENOMEM; | |
446 | goto fail; | |
447 | } | |
467fa153 SV |
448 | rtn->rds_tcp_listen_sock = rds_tcp_listen_init(net); |
449 | if (!rtn->rds_tcp_listen_sock) { | |
450 | pr_warn("could not set up listen sock\n"); | |
c6a58ffe SV |
451 | unregister_net_sysctl_table(rtn->rds_tcp_sysctl); |
452 | rtn->rds_tcp_sysctl = NULL; | |
453 | err = -EAFNOSUPPORT; | |
454 | goto fail; | |
467fa153 SV |
455 | } |
456 | INIT_WORK(&rtn->rds_tcp_accept_w, rds_tcp_accept_worker); | |
457 | return 0; | |
c6a58ffe SV |
458 | |
459 | fail: | |
460 | if (net != &init_net) | |
461 | kfree(tbl); | |
462 | return err; | |
467fa153 SV |
463 | } |
464 | ||
465 | static void __net_exit rds_tcp_exit_net(struct net *net) | |
466 | { | |
467 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
468 | ||
c6a58ffe SV |
469 | if (rtn->rds_tcp_sysctl) |
470 | unregister_net_sysctl_table(rtn->rds_tcp_sysctl); | |
471 | ||
472 | if (net != &init_net && rtn->ctl_table) | |
473 | kfree(rtn->ctl_table); | |
474 | ||
467fa153 SV |
475 | /* If rds_tcp_exit_net() is called as a result of netns deletion, |
476 | * the rds_tcp_kill_sock() device notifier would already have cleaned | |
477 | * up the listen socket, thus there is no work to do in this function. | |
478 | * | |
479 | * If rds_tcp_exit_net() is called as a result of module unload, | |
480 | * i.e., due to rds_tcp_exit() -> unregister_pernet_subsys(), then | |
481 | * we do need to clean up the listen socket here. | |
482 | */ | |
483 | if (rtn->rds_tcp_listen_sock) { | |
b21dd450 SV |
484 | struct socket *lsock = rtn->rds_tcp_listen_sock; |
485 | ||
467fa153 | 486 | rtn->rds_tcp_listen_sock = NULL; |
b21dd450 | 487 | rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w); |
467fa153 SV |
488 | } |
489 | } | |
490 | ||
491 | static struct pernet_operations rds_tcp_net_ops = { | |
492 | .init = rds_tcp_init_net, | |
493 | .exit = rds_tcp_exit_net, | |
494 | .id = &rds_tcp_netid, | |
495 | .size = sizeof(struct rds_tcp_net), | |
496 | }; | |
497 | ||
afb4164d SV |
498 | /* explicitly send a RST on each socket, thereby releasing any socket refcnts |
499 | * that may otherwise hold up netns deletion. | |
500 | */ | |
501 | static void rds_tcp_conn_paths_destroy(struct rds_connection *conn) | |
502 | { | |
503 | struct rds_conn_path *cp; | |
504 | struct rds_tcp_connection *tc; | |
505 | int i; | |
506 | struct sock *sk; | |
507 | ||
508 | for (i = 0; i < RDS_MPATH_WORKERS; i++) { | |
509 | cp = &conn->c_path[i]; | |
510 | tc = cp->cp_transport_data; | |
511 | if (!tc->t_sock) | |
512 | continue; | |
513 | sk = tc->t_sock->sk; | |
514 | sk->sk_prot->disconnect(sk, 0); | |
515 | tcp_done(sk); | |
516 | } | |
517 | } | |
518 | ||
467fa153 SV |
519 | static void rds_tcp_kill_sock(struct net *net) |
520 | { | |
521 | struct rds_tcp_connection *tc, *_tc; | |
467fa153 SV |
522 | LIST_HEAD(tmp_list); |
523 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
b21dd450 | 524 | struct socket *lsock = rtn->rds_tcp_listen_sock; |
467fa153 | 525 | |
467fa153 | 526 | rtn->rds_tcp_listen_sock = NULL; |
b21dd450 | 527 | rds_tcp_listen_stop(lsock, &rtn->rds_tcp_accept_w); |
467fa153 SV |
528 | spin_lock_irq(&rds_tcp_conn_lock); |
529 | list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) { | |
8edc3aff | 530 | struct net *c_net = tc->t_cpath->cp_conn->c_net; |
467fa153 SV |
531 | |
532 | if (net != c_net || !tc->t_sock) | |
533 | continue; | |
afb4164d SV |
534 | if (!list_has_conn(&tmp_list, tc->t_cpath->cp_conn)) |
535 | list_move_tail(&tc->t_tcp_node, &tmp_list); | |
467fa153 SV |
536 | } |
537 | spin_unlock_irq(&rds_tcp_conn_lock); | |
538 | list_for_each_entry_safe(tc, _tc, &tmp_list, t_tcp_node) { | |
afb4164d | 539 | rds_tcp_conn_paths_destroy(tc->t_cpath->cp_conn); |
02105b2c | 540 | rds_conn_destroy(tc->t_cpath->cp_conn); |
467fa153 SV |
541 | } |
542 | } | |
543 | ||
a93d01f5 SV |
544 | void *rds_tcp_listen_sock_def_readable(struct net *net) |
545 | { | |
546 | struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); | |
b21dd450 SV |
547 | struct socket *lsock = rtn->rds_tcp_listen_sock; |
548 | ||
549 | if (!lsock) | |
550 | return NULL; | |
a93d01f5 | 551 | |
b21dd450 | 552 | return lsock->sk->sk_user_data; |
a93d01f5 SV |
553 | } |
554 | ||
467fa153 SV |
555 | static int rds_tcp_dev_event(struct notifier_block *this, |
556 | unsigned long event, void *ptr) | |
557 | { | |
558 | struct net_device *dev = netdev_notifier_info_to_dev(ptr); | |
559 | ||
560 | /* rds-tcp registers as a pernet subys, so the ->exit will only | |
561 | * get invoked after network acitivity has quiesced. We need to | |
562 | * clean up all sockets to quiesce network activity, and use | |
563 | * the unregistration of the per-net loopback device as a trigger | |
564 | * to start that cleanup. | |
565 | */ | |
566 | if (event == NETDEV_UNREGISTER_FINAL && | |
567 | dev->ifindex == LOOPBACK_IFINDEX) | |
568 | rds_tcp_kill_sock(dev_net(dev)); | |
569 | ||
570 | return NOTIFY_DONE; | |
571 | } | |
572 | ||
573 | static struct notifier_block rds_tcp_dev_notifier = { | |
574 | .notifier_call = rds_tcp_dev_event, | |
575 | .priority = -10, /* must be called after other network notifiers */ | |
576 | }; | |
577 | ||
c6a58ffe SV |
578 | /* when sysctl is used to modify some kernel socket parameters,this |
579 | * function resets the RDS connections in that netns so that we can | |
580 | * restart with new parameters. The assumption is that such reset | |
581 | * events are few and far-between. | |
582 | */ | |
583 | static void rds_tcp_sysctl_reset(struct net *net) | |
584 | { | |
585 | struct rds_tcp_connection *tc, *_tc; | |
586 | ||
587 | spin_lock_irq(&rds_tcp_conn_lock); | |
588 | list_for_each_entry_safe(tc, _tc, &rds_tcp_conn_list, t_tcp_node) { | |
8edc3aff | 589 | struct net *c_net = tc->t_cpath->cp_conn->c_net; |
c6a58ffe SV |
590 | |
591 | if (net != c_net || !tc->t_sock) | |
592 | continue; | |
593 | ||
02105b2c SV |
594 | /* reconnect with new parameters */ |
595 | rds_conn_path_drop(tc->t_cpath); | |
c6a58ffe SV |
596 | } |
597 | spin_unlock_irq(&rds_tcp_conn_lock); | |
598 | } | |
599 | ||
600 | static int rds_tcp_skbuf_handler(struct ctl_table *ctl, int write, | |
601 | void __user *buffer, size_t *lenp, | |
602 | loff_t *fpos) | |
603 | { | |
604 | struct net *net = current->nsproxy->net_ns; | |
605 | int err; | |
606 | ||
607 | err = proc_dointvec_minmax(ctl, write, buffer, lenp, fpos); | |
608 | if (err < 0) { | |
609 | pr_warn("Invalid input. Must be >= %d\n", | |
610 | *(int *)(ctl->extra1)); | |
611 | return err; | |
612 | } | |
613 | if (write) | |
614 | rds_tcp_sysctl_reset(net); | |
615 | return 0; | |
616 | } | |
617 | ||
467fa153 SV |
618 | static void rds_tcp_exit(void) |
619 | { | |
620 | rds_info_deregister_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info); | |
621 | unregister_pernet_subsys(&rds_tcp_net_ops); | |
622 | if (unregister_netdevice_notifier(&rds_tcp_dev_notifier)) | |
623 | pr_warn("could not unregister rds_tcp_dev_notifier\n"); | |
624 | rds_tcp_destroy_conns(); | |
625 | rds_trans_unregister(&rds_tcp_transport); | |
626 | rds_tcp_recv_exit(); | |
627 | kmem_cache_destroy(rds_tcp_conn_slab); | |
628 | } | |
629 | module_exit(rds_tcp_exit); | |
630 | ||
ff51bf84 | 631 | static int rds_tcp_init(void) |
70041088 AG |
632 | { |
633 | int ret; | |
634 | ||
635 | rds_tcp_conn_slab = kmem_cache_create("rds_tcp_connection", | |
636 | sizeof(struct rds_tcp_connection), | |
637 | 0, 0, NULL); | |
8690bfa1 | 638 | if (!rds_tcp_conn_slab) { |
70041088 AG |
639 | ret = -ENOMEM; |
640 | goto out; | |
641 | } | |
642 | ||
16c09b1c SV |
643 | ret = rds_tcp_recv_init(); |
644 | if (ret) | |
3b5923f0 | 645 | goto out_slab; |
467fa153 SV |
646 | |
647 | ret = register_pernet_subsys(&rds_tcp_net_ops); | |
648 | if (ret) | |
16c09b1c | 649 | goto out_recv; |
467fa153 | 650 | |
16c09b1c SV |
651 | ret = register_netdevice_notifier(&rds_tcp_dev_notifier); |
652 | if (ret) { | |
653 | pr_warn("could not register rds_tcp_dev_notifier\n"); | |
3dad5424 | 654 | goto out_pernet; |
16c09b1c | 655 | } |
70041088 | 656 | |
a8d63a53 | 657 | rds_trans_register(&rds_tcp_transport); |
70041088 | 658 | |
70041088 AG |
659 | rds_info_register_func(RDS_INFO_TCP_SOCKETS, rds_tcp_tc_info); |
660 | ||
661 | goto out; | |
662 | ||
3dad5424 | 663 | out_pernet: |
467fa153 | 664 | unregister_pernet_subsys(&rds_tcp_net_ops); |
16c09b1c SV |
665 | out_recv: |
666 | rds_tcp_recv_exit(); | |
3b5923f0 | 667 | out_slab: |
70041088 AG |
668 | kmem_cache_destroy(rds_tcp_conn_slab); |
669 | out: | |
670 | return ret; | |
671 | } | |
672 | module_init(rds_tcp_init); | |
673 | ||
674 | MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>"); | |
675 | MODULE_DESCRIPTION("RDS: TCP transport"); | |
676 | MODULE_LICENSE("Dual BSD/GPL"); | |
677 |