Commit | Line | Data |
---|---|---|
00e0f34c | 1 | /* |
1e2b44e7 | 2 | * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved. |
00e0f34c AG |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | * | |
32 | */ | |
33 | #include <linux/kernel.h> | |
34 | #include <linux/list.h> | |
5a0e3ad6 | 35 | #include <linux/slab.h> |
bc3b2d7f | 36 | #include <linux/export.h> |
eee2fa6a KCP |
37 | #include <net/ipv6.h> |
38 | #include <net/inet6_hashtables.h> | |
1e2b44e7 | 39 | #include <net/addrconf.h> |
00e0f34c AG |
40 | |
41 | #include "rds.h" | |
42 | #include "loop.h" | |
00e0f34c AG |
43 | |
44 | #define RDS_CONNECTION_HASH_BITS 12 | |
45 | #define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS) | |
46 | #define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1) | |
47 | ||
48 | /* converting this to RCU is a chore for another day.. */ | |
49 | static DEFINE_SPINLOCK(rds_conn_lock); | |
50 | static unsigned long rds_conn_count; | |
51 | static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES]; | |
52 | static struct kmem_cache *rds_conn_slab; | |
53 | ||
eee2fa6a KCP |
54 | static struct hlist_head *rds_conn_bucket(const struct in6_addr *laddr, |
55 | const struct in6_addr *faddr) | |
00e0f34c | 56 | { |
eee2fa6a | 57 | static u32 rds6_hash_secret __read_mostly; |
1bbdceef HFS |
58 | static u32 rds_hash_secret __read_mostly; |
59 | ||
eee2fa6a | 60 | u32 lhash, fhash, hash; |
1bbdceef HFS |
61 | |
62 | net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret)); | |
eee2fa6a KCP |
63 | net_get_random_once(&rds6_hash_secret, sizeof(rds6_hash_secret)); |
64 | ||
65 | lhash = (__force u32)laddr->s6_addr32[3]; | |
e65d4d96 | 66 | #if IS_ENABLED(CONFIG_IPV6) |
eee2fa6a | 67 | fhash = __ipv6_addr_jhash(faddr, rds6_hash_secret); |
e65d4d96 KCP |
68 | #else |
69 | fhash = (__force u32)faddr->s6_addr32[3]; | |
70 | #endif | |
71 | hash = __inet_ehashfn(lhash, 0, fhash, 0, rds_hash_secret); | |
1bbdceef | 72 | |
00e0f34c AG |
73 | return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK]; |
74 | } | |
75 | ||
76 | #define rds_conn_info_set(var, test, suffix) do { \ | |
77 | if (test) \ | |
78 | var |= RDS_INFO_CONNECTION_FLAG_##suffix; \ | |
79 | } while (0) | |
80 | ||
bcf50ef2 | 81 | /* rcu read lock must be held or the connection spinlock */ |
8f384c01 SV |
82 | static struct rds_connection *rds_conn_lookup(struct net *net, |
83 | struct hlist_head *head, | |
eee2fa6a KCP |
84 | const struct in6_addr *laddr, |
85 | const struct in6_addr *faddr, | |
86 | struct rds_transport *trans, | |
87 | int dev_if) | |
00e0f34c AG |
88 | { |
89 | struct rds_connection *conn, *ret = NULL; | |
00e0f34c | 90 | |
b67bfe0d | 91 | hlist_for_each_entry_rcu(conn, head, c_hash_node) { |
eee2fa6a KCP |
92 | if (ipv6_addr_equal(&conn->c_faddr, faddr) && |
93 | ipv6_addr_equal(&conn->c_laddr, laddr) && | |
94 | conn->c_trans == trans && | |
95 | net == rds_conn_net(conn) && | |
96 | conn->c_dev_if == dev_if) { | |
00e0f34c AG |
97 | ret = conn; |
98 | break; | |
99 | } | |
100 | } | |
eee2fa6a KCP |
101 | rdsdebug("returning conn %p for %pI6c -> %pI6c\n", ret, |
102 | laddr, faddr); | |
00e0f34c AG |
103 | return ret; |
104 | } | |
105 | ||
106 | /* | |
107 | * This is called by transports as they're bringing down a connection. | |
108 | * It clears partial message state so that the transport can start sending | |
109 | * and receiving over this connection again in the future. It is up to | |
110 | * the transport to have serialized this call with its send and recv. | |
111 | */ | |
d769ef81 | 112 | static void rds_conn_path_reset(struct rds_conn_path *cp) |
00e0f34c | 113 | { |
d769ef81 SV |
114 | struct rds_connection *conn = cp->cp_conn; |
115 | ||
eee2fa6a KCP |
116 | rdsdebug("connection %pI6c to %pI6c reset\n", |
117 | &conn->c_laddr, &conn->c_faddr); | |
00e0f34c AG |
118 | |
119 | rds_stats_inc(s_conn_reset); | |
d769ef81 SV |
120 | rds_send_path_reset(cp); |
121 | cp->cp_flags = 0; | |
00e0f34c AG |
122 | |
123 | /* Do not clear next_rx_seq here, else we cannot distinguish | |
124 | * retransmitted packets from new packets, and will hand all | |
125 | * of them to the application. That is not consistent with the | |
126 | * reliability guarantees of RDS. */ | |
127 | } | |
128 | ||
1c5113cf SV |
129 | static void __rds_conn_path_init(struct rds_connection *conn, |
130 | struct rds_conn_path *cp, bool is_outgoing) | |
131 | { | |
132 | spin_lock_init(&cp->cp_lock); | |
133 | cp->cp_next_tx_seq = 1; | |
134 | init_waitqueue_head(&cp->cp_waitq); | |
135 | INIT_LIST_HEAD(&cp->cp_send_queue); | |
136 | INIT_LIST_HEAD(&cp->cp_retrans); | |
137 | ||
138 | cp->cp_conn = conn; | |
139 | atomic_set(&cp->cp_state, RDS_CONN_DOWN); | |
140 | cp->cp_send_gen = 0; | |
1c5113cf | 141 | cp->cp_reconnect_jiffies = 0; |
cdc306a5 | 142 | cp->cp_conn->c_proposed_version = RDS_PROTOCOL_VERSION; |
1c5113cf SV |
143 | INIT_DELAYED_WORK(&cp->cp_send_w, rds_send_worker); |
144 | INIT_DELAYED_WORK(&cp->cp_recv_w, rds_recv_worker); | |
145 | INIT_DELAYED_WORK(&cp->cp_conn_w, rds_connect_worker); | |
146 | INIT_WORK(&cp->cp_down_w, rds_shutdown_worker); | |
147 | mutex_init(&cp->cp_cm_lock); | |
148 | cp->cp_flags = 0; | |
149 | } | |
150 | ||
00e0f34c AG |
151 | /* |
152 | * There is only every one 'conn' for a given pair of addresses in the | |
153 | * system at a time. They contain messages to be retransmitted and so | |
154 | * span the lifetime of the actual underlying transport connections. | |
155 | * | |
156 | * For now they are not garbage collected once they're created. They | |
157 | * are torn down as the module is removed, if ever. | |
158 | */ | |
d5a8ac28 | 159 | static struct rds_connection *__rds_conn_create(struct net *net, |
eee2fa6a KCP |
160 | const struct in6_addr *laddr, |
161 | const struct in6_addr *faddr, | |
162 | struct rds_transport *trans, | |
163 | gfp_t gfp, | |
164 | int is_outgoing, | |
165 | int dev_if) | |
00e0f34c | 166 | { |
cb24405e | 167 | struct rds_connection *conn, *parent = NULL; |
00e0f34c | 168 | struct hlist_head *head = rds_conn_bucket(laddr, faddr); |
5adb5bc6 | 169 | struct rds_transport *loop_trans; |
00e0f34c | 170 | unsigned long flags; |
5916e2c1 | 171 | int ret, i; |
840df162 | 172 | int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); |
00e0f34c | 173 | |
bcf50ef2 | 174 | rcu_read_lock(); |
eee2fa6a KCP |
175 | conn = rds_conn_lookup(net, head, laddr, faddr, trans, dev_if); |
176 | if (conn && | |
177 | conn->c_loopback && | |
178 | conn->c_trans != &rds_loop_transport && | |
179 | ipv6_addr_equal(laddr, faddr) && | |
180 | !is_outgoing) { | |
00e0f34c AG |
181 | /* This is a looped back IB connection, and we're |
182 | * called by the code handling the incoming connect. | |
183 | * We need a second connection object into which we | |
184 | * can stick the other QP. */ | |
185 | parent = conn; | |
186 | conn = parent->c_passive; | |
187 | } | |
bcf50ef2 | 188 | rcu_read_unlock(); |
00e0f34c AG |
189 | if (conn) |
190 | goto out; | |
191 | ||
05a178ec | 192 | conn = kmem_cache_zalloc(rds_conn_slab, gfp); |
8690bfa1 | 193 | if (!conn) { |
00e0f34c AG |
194 | conn = ERR_PTR(-ENOMEM); |
195 | goto out; | |
196 | } | |
840df162 SV |
197 | conn->c_path = kcalloc(npaths, sizeof(struct rds_conn_path), gfp); |
198 | if (!conn->c_path) { | |
199 | kmem_cache_free(rds_conn_slab, conn); | |
200 | conn = ERR_PTR(-ENOMEM); | |
201 | goto out; | |
202 | } | |
00e0f34c | 203 | |
00e0f34c | 204 | INIT_HLIST_NODE(&conn->c_hash_node); |
eee2fa6a KCP |
205 | conn->c_laddr = *laddr; |
206 | conn->c_isv6 = !ipv6_addr_v4mapped(laddr); | |
207 | conn->c_faddr = *faddr; | |
208 | conn->c_dev_if = dev_if; | |
e65d4d96 KCP |
209 | |
210 | #if IS_ENABLED(CONFIG_IPV6) | |
1e2b44e7 KCP |
211 | /* If the local address is link local, set c_bound_if to be the |
212 | * index used for this connection. Otherwise, set it to 0 as | |
213 | * the socket is not bound to an interface. c_bound_if is used | |
214 | * to look up a socket when a packet is received | |
215 | */ | |
216 | if (ipv6_addr_type(laddr) & IPV6_ADDR_LINKLOCAL) | |
217 | conn->c_bound_if = dev_if; | |
218 | else | |
e65d4d96 | 219 | #endif |
1e2b44e7 | 220 | conn->c_bound_if = 0; |
00e0f34c | 221 | |
1c5113cf | 222 | rds_conn_net_set(conn, net); |
00e0f34c AG |
223 | |
224 | ret = rds_cong_get_maps(conn); | |
225 | if (ret) { | |
840df162 | 226 | kfree(conn->c_path); |
00e0f34c AG |
227 | kmem_cache_free(rds_conn_slab, conn); |
228 | conn = ERR_PTR(ret); | |
229 | goto out; | |
230 | } | |
231 | ||
232 | /* | |
233 | * This is where a connection becomes loopback. If *any* RDS sockets | |
234 | * can bind to the destination address then we'd rather the messages | |
235 | * flow through loopback rather than either transport. | |
236 | */ | |
eee2fa6a | 237 | loop_trans = rds_trans_get_preferred(net, faddr, conn->c_dev_if); |
5adb5bc6 ZB |
238 | if (loop_trans) { |
239 | rds_trans_put(loop_trans); | |
00e0f34c AG |
240 | conn->c_loopback = 1; |
241 | if (is_outgoing && trans->t_prefer_loopback) { | |
242 | /* "outgoing" connection - and the transport | |
243 | * says it wants the connection handled by the | |
244 | * loopback transport. This is what TCP does. | |
245 | */ | |
246 | trans = &rds_loop_transport; | |
247 | } | |
248 | } | |
249 | ||
250 | conn->c_trans = trans; | |
251 | ||
5916e2c1 | 252 | init_waitqueue_head(&conn->c_hs_waitq); |
840df162 | 253 | for (i = 0; i < npaths; i++) { |
5916e2c1 SV |
254 | __rds_conn_path_init(conn, &conn->c_path[i], |
255 | is_outgoing); | |
256 | conn->c_path[i].cp_index = i; | |
257 | } | |
ebeeb1ad SV |
258 | rcu_read_lock(); |
259 | if (rds_destroy_pending(conn)) | |
260 | ret = -ENETDOWN; | |
261 | else | |
d4014d8c | 262 | ret = trans->conn_alloc(conn, GFP_ATOMIC); |
00e0f34c | 263 | if (ret) { |
ebeeb1ad | 264 | rcu_read_unlock(); |
840df162 | 265 | kfree(conn->c_path); |
00e0f34c AG |
266 | kmem_cache_free(rds_conn_slab, conn); |
267 | conn = ERR_PTR(ret); | |
268 | goto out; | |
269 | } | |
270 | ||
eee2fa6a KCP |
271 | rdsdebug("allocated conn %p for %pI6c -> %pI6c over %s %s\n", |
272 | conn, laddr, faddr, | |
273 | strnlen(trans->t_name, sizeof(trans->t_name)) ? | |
274 | trans->t_name : "[unknown]", is_outgoing ? "(outgoing)" : ""); | |
00e0f34c | 275 | |
cb24405e AG |
276 | /* |
277 | * Since we ran without holding the conn lock, someone could | |
278 | * have created the same conn (either normal or passive) in the | |
279 | * interim. We check while holding the lock. If we won, we complete | |
280 | * init and return our conn. If we lost, we rollback and return the | |
281 | * other one. | |
282 | */ | |
00e0f34c | 283 | spin_lock_irqsave(&rds_conn_lock, flags); |
cb24405e AG |
284 | if (parent) { |
285 | /* Creating passive conn */ | |
286 | if (parent->c_passive) { | |
1c5113cf | 287 | trans->conn_free(conn->c_path[0].cp_transport_data); |
840df162 | 288 | kfree(conn->c_path); |
cb24405e AG |
289 | kmem_cache_free(rds_conn_slab, conn); |
290 | conn = parent->c_passive; | |
291 | } else { | |
00e0f34c | 292 | parent->c_passive = conn; |
cb24405e AG |
293 | rds_cong_add_conn(conn); |
294 | rds_conn_count++; | |
295 | } | |
00e0f34c | 296 | } else { |
cb24405e AG |
297 | /* Creating normal conn */ |
298 | struct rds_connection *found; | |
299 | ||
eee2fa6a KCP |
300 | found = rds_conn_lookup(net, head, laddr, faddr, trans, |
301 | dev_if); | |
cb24405e | 302 | if (found) { |
1c5113cf SV |
303 | struct rds_conn_path *cp; |
304 | int i; | |
305 | ||
840df162 | 306 | for (i = 0; i < npaths; i++) { |
1c5113cf | 307 | cp = &conn->c_path[i]; |
02105b2c SV |
308 | /* The ->conn_alloc invocation may have |
309 | * allocated resource for all paths, so all | |
310 | * of them may have to be freed here. | |
311 | */ | |
312 | if (cp->cp_transport_data) | |
313 | trans->conn_free(cp->cp_transport_data); | |
1c5113cf | 314 | } |
840df162 | 315 | kfree(conn->c_path); |
cb24405e AG |
316 | kmem_cache_free(rds_conn_slab, conn); |
317 | conn = found; | |
318 | } else { | |
905dd418 SV |
319 | conn->c_my_gen_num = rds_gen_num; |
320 | conn->c_peer_gen_num = 0; | |
3b20fc38 | 321 | hlist_add_head_rcu(&conn->c_hash_node, head); |
cb24405e AG |
322 | rds_cong_add_conn(conn); |
323 | rds_conn_count++; | |
324 | } | |
00e0f34c | 325 | } |
00e0f34c | 326 | spin_unlock_irqrestore(&rds_conn_lock, flags); |
ebeeb1ad | 327 | rcu_read_unlock(); |
00e0f34c AG |
328 | |
329 | out: | |
330 | return conn; | |
331 | } | |
332 | ||
d5a8ac28 | 333 | struct rds_connection *rds_conn_create(struct net *net, |
eee2fa6a KCP |
334 | const struct in6_addr *laddr, |
335 | const struct in6_addr *faddr, | |
336 | struct rds_transport *trans, gfp_t gfp, | |
337 | int dev_if) | |
00e0f34c | 338 | { |
eee2fa6a | 339 | return __rds_conn_create(net, laddr, faddr, trans, gfp, 0, dev_if); |
00e0f34c | 340 | } |
616b757a | 341 | EXPORT_SYMBOL_GPL(rds_conn_create); |
00e0f34c | 342 | |
d5a8ac28 | 343 | struct rds_connection *rds_conn_create_outgoing(struct net *net, |
eee2fa6a KCP |
344 | const struct in6_addr *laddr, |
345 | const struct in6_addr *faddr, | |
346 | struct rds_transport *trans, | |
347 | gfp_t gfp, int dev_if) | |
00e0f34c | 348 | { |
eee2fa6a | 349 | return __rds_conn_create(net, laddr, faddr, trans, gfp, 1, dev_if); |
00e0f34c | 350 | } |
616b757a | 351 | EXPORT_SYMBOL_GPL(rds_conn_create_outgoing); |
00e0f34c | 352 | |
d769ef81 | 353 | void rds_conn_shutdown(struct rds_conn_path *cp) |
2dc39357 | 354 | { |
d769ef81 SV |
355 | struct rds_connection *conn = cp->cp_conn; |
356 | ||
2dc39357 | 357 | /* shut it down unless it's down already */ |
d769ef81 | 358 | if (!rds_conn_path_transition(cp, RDS_CONN_DOWN, RDS_CONN_DOWN)) { |
2dc39357 AG |
359 | /* |
360 | * Quiesce the connection mgmt handlers before we start tearing | |
361 | * things down. We don't hold the mutex for the entire | |
362 | * duration of the shutdown operation, else we may be | |
363 | * deadlocking with the CM handler. Instead, the CM event | |
364 | * handler is supposed to check for state DISCONNECTING | |
365 | */ | |
d769ef81 SV |
366 | mutex_lock(&cp->cp_cm_lock); |
367 | if (!rds_conn_path_transition(cp, RDS_CONN_UP, | |
368 | RDS_CONN_DISCONNECTING) && | |
369 | !rds_conn_path_transition(cp, RDS_CONN_ERROR, | |
370 | RDS_CONN_DISCONNECTING)) { | |
371 | rds_conn_path_error(cp, | |
372 | "shutdown called in state %d\n", | |
373 | atomic_read(&cp->cp_state)); | |
374 | mutex_unlock(&cp->cp_cm_lock); | |
2dc39357 AG |
375 | return; |
376 | } | |
d769ef81 | 377 | mutex_unlock(&cp->cp_cm_lock); |
2dc39357 | 378 | |
d769ef81 SV |
379 | wait_event(cp->cp_waitq, |
380 | !test_bit(RDS_IN_XMIT, &cp->cp_flags)); | |
381 | wait_event(cp->cp_waitq, | |
382 | !test_bit(RDS_RECV_REFILL, &cp->cp_flags)); | |
7e3f2952 | 383 | |
226f7a7d | 384 | conn->c_trans->conn_path_shutdown(cp); |
d769ef81 | 385 | rds_conn_path_reset(cp); |
2dc39357 | 386 | |
d769ef81 | 387 | if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING, |
e97656d0 SV |
388 | RDS_CONN_DOWN) && |
389 | !rds_conn_path_transition(cp, RDS_CONN_ERROR, | |
d769ef81 | 390 | RDS_CONN_DOWN)) { |
2dc39357 AG |
391 | /* This can happen - eg when we're in the middle of tearing |
392 | * down the connection, and someone unloads the rds module. | |
e97656d0 | 393 | * Quite reproducible with loopback connections. |
2dc39357 | 394 | * Mostly harmless. |
e97656d0 SV |
395 | * |
396 | * Note that this also happens with rds-tcp because | |
397 | * we could have triggered rds_conn_path_drop in irq | |
398 | * mode from rds_tcp_state change on the receipt of | |
399 | * a FIN, thus we need to recheck for RDS_CONN_ERROR | |
400 | * here. | |
2dc39357 | 401 | */ |
d769ef81 SV |
402 | rds_conn_path_error(cp, "%s: failed to transition " |
403 | "to state DOWN, current state " | |
404 | "is %d\n", __func__, | |
405 | atomic_read(&cp->cp_state)); | |
2dc39357 AG |
406 | return; |
407 | } | |
408 | } | |
409 | ||
410 | /* Then reconnect if it's still live. | |
411 | * The passive side of an IB loopback connection is never added | |
412 | * to the conn hash, so we never trigger a reconnect on this | |
413 | * conn - the reconnect is always triggered by the active peer. */ | |
d769ef81 | 414 | cancel_delayed_work_sync(&cp->cp_conn_w); |
bcf50ef2 CM |
415 | rcu_read_lock(); |
416 | if (!hlist_unhashed(&conn->c_hash_node)) { | |
417 | rcu_read_unlock(); | |
8315011a | 418 | rds_queue_reconnect(cp); |
bcf50ef2 CM |
419 | } else { |
420 | rcu_read_unlock(); | |
421 | } | |
2dc39357 AG |
422 | } |
423 | ||
3ecc5693 SV |
424 | /* destroy a single rds_conn_path. rds_conn_destroy() iterates over |
425 | * all paths using rds_conn_path_destroy() | |
426 | */ | |
427 | static void rds_conn_path_destroy(struct rds_conn_path *cp) | |
428 | { | |
429 | struct rds_message *rm, *rtmp; | |
430 | ||
02105b2c SV |
431 | if (!cp->cp_transport_data) |
432 | return; | |
433 | ||
3ecc5693 SV |
434 | /* make sure lingering queued work won't try to ref the conn */ |
435 | cancel_delayed_work_sync(&cp->cp_send_w); | |
436 | cancel_delayed_work_sync(&cp->cp_recv_w); | |
437 | ||
aed20a53 SV |
438 | rds_conn_path_drop(cp, true); |
439 | flush_work(&cp->cp_down_w); | |
440 | ||
3ecc5693 SV |
441 | /* tear down queued messages */ |
442 | list_for_each_entry_safe(rm, rtmp, | |
443 | &cp->cp_send_queue, | |
444 | m_conn_item) { | |
445 | list_del_init(&rm->m_conn_item); | |
446 | BUG_ON(!list_empty(&rm->m_sock_item)); | |
447 | rds_message_put(rm); | |
448 | } | |
449 | if (cp->cp_xmit_rm) | |
450 | rds_message_put(cp->cp_xmit_rm); | |
451 | ||
3db6e0d1 SV |
452 | WARN_ON(delayed_work_pending(&cp->cp_send_w)); |
453 | WARN_ON(delayed_work_pending(&cp->cp_recv_w)); | |
454 | WARN_ON(delayed_work_pending(&cp->cp_conn_w)); | |
455 | WARN_ON(work_pending(&cp->cp_down_w)); | |
456 | ||
3ecc5693 SV |
457 | cp->cp_conn->c_trans->conn_free(cp->cp_transport_data); |
458 | } | |
459 | ||
2dc39357 AG |
460 | /* |
461 | * Stop and free a connection. | |
ffcec0e1 ZB |
462 | * |
463 | * This can only be used in very limited circumstances. It assumes that once | |
464 | * the conn has been shutdown that no one else is referencing the connection. | |
465 | * We can only ensure this in the rmmod path in the current code. | |
2dc39357 | 466 | */ |
00e0f34c AG |
467 | void rds_conn_destroy(struct rds_connection *conn) |
468 | { | |
fe8ff6b5 | 469 | unsigned long flags; |
02105b2c SV |
470 | int i; |
471 | struct rds_conn_path *cp; | |
840df162 | 472 | int npaths = (conn->c_trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); |
00e0f34c AG |
473 | |
474 | rdsdebug("freeing conn %p for %pI4 -> " | |
475 | "%pI4\n", conn, &conn->c_laddr, | |
476 | &conn->c_faddr); | |
477 | ||
abf45439 CM |
478 | /* Ensure conn will not be scheduled for reconnect */ |
479 | spin_lock_irq(&rds_conn_lock); | |
bcf50ef2 | 480 | hlist_del_init_rcu(&conn->c_hash_node); |
abf45439 | 481 | spin_unlock_irq(&rds_conn_lock); |
bcf50ef2 CM |
482 | synchronize_rcu(); |
483 | ||
ffcec0e1 | 484 | /* shut the connection down */ |
840df162 | 485 | for (i = 0; i < npaths; i++) { |
02105b2c SV |
486 | cp = &conn->c_path[i]; |
487 | rds_conn_path_destroy(cp); | |
488 | BUG_ON(!list_empty(&cp->cp_retrans)); | |
00e0f34c | 489 | } |
00e0f34c AG |
490 | |
491 | /* | |
492 | * The congestion maps aren't freed up here. They're | |
493 | * freed by rds_cong_exit() after all the connections | |
494 | * have been freed. | |
495 | */ | |
496 | rds_cong_remove_conn(conn); | |
497 | ||
840df162 | 498 | kfree(conn->c_path); |
00e0f34c AG |
499 | kmem_cache_free(rds_conn_slab, conn); |
500 | ||
fe8ff6b5 | 501 | spin_lock_irqsave(&rds_conn_lock, flags); |
00e0f34c | 502 | rds_conn_count--; |
fe8ff6b5 | 503 | spin_unlock_irqrestore(&rds_conn_lock, flags); |
00e0f34c | 504 | } |
616b757a | 505 | EXPORT_SYMBOL_GPL(rds_conn_destroy); |
00e0f34c | 506 | |
1e2b44e7 KCP |
507 | static void __rds_inc_msg_cp(struct rds_incoming *inc, |
508 | struct rds_info_iterator *iter, | |
b7ff8b10 | 509 | void *saddr, void *daddr, int flip, bool isv6) |
1e2b44e7 | 510 | { |
e65d4d96 | 511 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
512 | if (isv6) |
513 | rds6_inc_info_copy(inc, iter, saddr, daddr, flip); | |
514 | else | |
e65d4d96 | 515 | #endif |
b7ff8b10 KCP |
516 | rds_inc_info_copy(inc, iter, *(__be32 *)saddr, |
517 | *(__be32 *)daddr, flip); | |
1e2b44e7 KCP |
518 | } |
519 | ||
520 | static void rds_conn_message_info_cmn(struct socket *sock, unsigned int len, | |
521 | struct rds_info_iterator *iter, | |
522 | struct rds_info_lengths *lens, | |
b7ff8b10 | 523 | int want_send, bool isv6) |
00e0f34c AG |
524 | { |
525 | struct hlist_head *head; | |
00e0f34c AG |
526 | struct list_head *list; |
527 | struct rds_connection *conn; | |
528 | struct rds_message *rm; | |
00e0f34c | 529 | unsigned int total = 0; |
501dcccd | 530 | unsigned long flags; |
00e0f34c | 531 | size_t i; |
992c9ec5 | 532 | int j; |
00e0f34c | 533 | |
b7ff8b10 KCP |
534 | if (isv6) |
535 | len /= sizeof(struct rds6_info_message); | |
536 | else | |
537 | len /= sizeof(struct rds_info_message); | |
00e0f34c | 538 | |
bcf50ef2 | 539 | rcu_read_lock(); |
00e0f34c AG |
540 | |
541 | for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); | |
542 | i++, head++) { | |
b67bfe0d | 543 | hlist_for_each_entry_rcu(conn, head, c_hash_node) { |
992c9ec5 | 544 | struct rds_conn_path *cp; |
840df162 SV |
545 | int npaths; |
546 | ||
b7ff8b10 KCP |
547 | if (!isv6 && conn->c_isv6) |
548 | continue; | |
549 | ||
840df162 SV |
550 | npaths = (conn->c_trans->t_mp_capable ? |
551 | RDS_MPATH_WORKERS : 1); | |
992c9ec5 | 552 | |
840df162 | 553 | for (j = 0; j < npaths; j++) { |
992c9ec5 SV |
554 | cp = &conn->c_path[j]; |
555 | if (want_send) | |
556 | list = &cp->cp_send_queue; | |
557 | else | |
558 | list = &cp->cp_retrans; | |
559 | ||
560 | spin_lock_irqsave(&cp->cp_lock, flags); | |
561 | ||
562 | /* XXX too lazy to maintain counts.. */ | |
563 | list_for_each_entry(rm, list, m_conn_item) { | |
564 | total++; | |
565 | if (total <= len) | |
1e2b44e7 KCP |
566 | __rds_inc_msg_cp(&rm->m_inc, |
567 | iter, | |
568 | &conn->c_laddr, | |
569 | &conn->c_faddr, | |
b7ff8b10 | 570 | 0, isv6); |
992c9ec5 SV |
571 | } |
572 | ||
573 | spin_unlock_irqrestore(&cp->cp_lock, flags); | |
00e0f34c | 574 | } |
00e0f34c AG |
575 | } |
576 | } | |
bcf50ef2 | 577 | rcu_read_unlock(); |
00e0f34c AG |
578 | |
579 | lens->nr = total; | |
b7ff8b10 KCP |
580 | if (isv6) |
581 | lens->each = sizeof(struct rds6_info_message); | |
582 | else | |
583 | lens->each = sizeof(struct rds_info_message); | |
00e0f34c AG |
584 | } |
585 | ||
1e2b44e7 KCP |
586 | static void rds_conn_message_info(struct socket *sock, unsigned int len, |
587 | struct rds_info_iterator *iter, | |
588 | struct rds_info_lengths *lens, | |
589 | int want_send) | |
590 | { | |
b7ff8b10 KCP |
591 | rds_conn_message_info_cmn(sock, len, iter, lens, want_send, false); |
592 | } | |
593 | ||
e65d4d96 | 594 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
595 | static void rds6_conn_message_info(struct socket *sock, unsigned int len, |
596 | struct rds_info_iterator *iter, | |
597 | struct rds_info_lengths *lens, | |
598 | int want_send) | |
599 | { | |
600 | rds_conn_message_info_cmn(sock, len, iter, lens, want_send, true); | |
1e2b44e7 | 601 | } |
e65d4d96 | 602 | #endif |
1e2b44e7 | 603 | |
00e0f34c AG |
604 | static void rds_conn_message_info_send(struct socket *sock, unsigned int len, |
605 | struct rds_info_iterator *iter, | |
606 | struct rds_info_lengths *lens) | |
607 | { | |
608 | rds_conn_message_info(sock, len, iter, lens, 1); | |
609 | } | |
610 | ||
e65d4d96 | 611 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
612 | static void rds6_conn_message_info_send(struct socket *sock, unsigned int len, |
613 | struct rds_info_iterator *iter, | |
614 | struct rds_info_lengths *lens) | |
615 | { | |
616 | rds6_conn_message_info(sock, len, iter, lens, 1); | |
617 | } | |
e65d4d96 | 618 | #endif |
b7ff8b10 | 619 | |
00e0f34c AG |
620 | static void rds_conn_message_info_retrans(struct socket *sock, |
621 | unsigned int len, | |
622 | struct rds_info_iterator *iter, | |
623 | struct rds_info_lengths *lens) | |
624 | { | |
625 | rds_conn_message_info(sock, len, iter, lens, 0); | |
626 | } | |
627 | ||
e65d4d96 | 628 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
629 | static void rds6_conn_message_info_retrans(struct socket *sock, |
630 | unsigned int len, | |
631 | struct rds_info_iterator *iter, | |
632 | struct rds_info_lengths *lens) | |
633 | { | |
634 | rds6_conn_message_info(sock, len, iter, lens, 0); | |
635 | } | |
e65d4d96 | 636 | #endif |
b7ff8b10 | 637 | |
00e0f34c AG |
638 | void rds_for_each_conn_info(struct socket *sock, unsigned int len, |
639 | struct rds_info_iterator *iter, | |
640 | struct rds_info_lengths *lens, | |
641 | int (*visitor)(struct rds_connection *, void *), | |
f1cb9d68 | 642 | u64 *buffer, |
00e0f34c AG |
643 | size_t item_len) |
644 | { | |
00e0f34c | 645 | struct hlist_head *head; |
00e0f34c | 646 | struct rds_connection *conn; |
00e0f34c AG |
647 | size_t i; |
648 | ||
bcf50ef2 | 649 | rcu_read_lock(); |
00e0f34c AG |
650 | |
651 | lens->nr = 0; | |
652 | lens->each = item_len; | |
653 | ||
654 | for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); | |
655 | i++, head++) { | |
b67bfe0d | 656 | hlist_for_each_entry_rcu(conn, head, c_hash_node) { |
00e0f34c AG |
657 | |
658 | /* XXX no c_lock usage.. */ | |
659 | if (!visitor(conn, buffer)) | |
660 | continue; | |
661 | ||
662 | /* We copy as much as we can fit in the buffer, | |
663 | * but we count all items so that the caller | |
664 | * can resize the buffer. */ | |
665 | if (len >= item_len) { | |
666 | rds_info_copy(iter, buffer, item_len); | |
667 | len -= item_len; | |
668 | } | |
669 | lens->nr++; | |
670 | } | |
671 | } | |
bcf50ef2 | 672 | rcu_read_unlock(); |
00e0f34c | 673 | } |
616b757a | 674 | EXPORT_SYMBOL_GPL(rds_for_each_conn_info); |
00e0f34c | 675 | |
bb789763 SS |
676 | static void rds_walk_conn_path_info(struct socket *sock, unsigned int len, |
677 | struct rds_info_iterator *iter, | |
678 | struct rds_info_lengths *lens, | |
679 | int (*visitor)(struct rds_conn_path *, void *), | |
b2c9272a | 680 | u64 *buffer, |
bb789763 | 681 | size_t item_len) |
992c9ec5 | 682 | { |
992c9ec5 SV |
683 | struct hlist_head *head; |
684 | struct rds_connection *conn; | |
685 | size_t i; | |
992c9ec5 SV |
686 | |
687 | rcu_read_lock(); | |
688 | ||
689 | lens->nr = 0; | |
690 | lens->each = item_len; | |
691 | ||
692 | for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); | |
693 | i++, head++) { | |
694 | hlist_for_each_entry_rcu(conn, head, c_hash_node) { | |
695 | struct rds_conn_path *cp; | |
696 | ||
eee2fa6a KCP |
697 | /* XXX We only copy the information from the first |
698 | * path for now. The problem is that if there are | |
699 | * more than one underlying paths, we cannot report | |
700 | * information of all of them using the existing | |
701 | * API. For example, there is only one next_tx_seq, | |
702 | * which path's next_tx_seq should we report? It is | |
703 | * a bug in the design of MPRDS. | |
704 | */ | |
705 | cp = conn->c_path; | |
992c9ec5 | 706 | |
eee2fa6a KCP |
707 | /* XXX no cp_lock usage.. */ |
708 | if (!visitor(cp, buffer)) | |
709 | continue; | |
992c9ec5 SV |
710 | |
711 | /* We copy as much as we can fit in the buffer, | |
712 | * but we count all items so that the caller | |
713 | * can resize the buffer. | |
714 | */ | |
715 | if (len >= item_len) { | |
716 | rds_info_copy(iter, buffer, item_len); | |
717 | len -= item_len; | |
718 | } | |
719 | lens->nr++; | |
720 | } | |
721 | } | |
722 | rcu_read_unlock(); | |
723 | } | |
724 | ||
725 | static int rds_conn_info_visitor(struct rds_conn_path *cp, void *buffer) | |
00e0f34c AG |
726 | { |
727 | struct rds_info_connection *cinfo = buffer; | |
eee2fa6a | 728 | struct rds_connection *conn = cp->cp_conn; |
00e0f34c | 729 | |
1e2b44e7 KCP |
730 | if (conn->c_isv6) |
731 | return 0; | |
732 | ||
992c9ec5 SV |
733 | cinfo->next_tx_seq = cp->cp_next_tx_seq; |
734 | cinfo->next_rx_seq = cp->cp_next_rx_seq; | |
eee2fa6a KCP |
735 | cinfo->laddr = conn->c_laddr.s6_addr32[3]; |
736 | cinfo->faddr = conn->c_faddr.s6_addr32[3]; | |
737 | strncpy(cinfo->transport, conn->c_trans->t_name, | |
00e0f34c AG |
738 | sizeof(cinfo->transport)); |
739 | cinfo->flags = 0; | |
740 | ||
992c9ec5 | 741 | rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags), |
0f4b1c7e | 742 | SENDING); |
00e0f34c AG |
743 | /* XXX Future: return the state rather than these funky bits */ |
744 | rds_conn_info_set(cinfo->flags, | |
992c9ec5 | 745 | atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING, |
00e0f34c AG |
746 | CONNECTING); |
747 | rds_conn_info_set(cinfo->flags, | |
992c9ec5 | 748 | atomic_read(&cp->cp_state) == RDS_CONN_UP, |
00e0f34c AG |
749 | CONNECTED); |
750 | return 1; | |
751 | } | |
752 | ||
e65d4d96 | 753 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
754 | static int rds6_conn_info_visitor(struct rds_conn_path *cp, void *buffer) |
755 | { | |
756 | struct rds6_info_connection *cinfo6 = buffer; | |
757 | struct rds_connection *conn = cp->cp_conn; | |
758 | ||
759 | cinfo6->next_tx_seq = cp->cp_next_tx_seq; | |
760 | cinfo6->next_rx_seq = cp->cp_next_rx_seq; | |
761 | cinfo6->laddr = conn->c_laddr; | |
762 | cinfo6->faddr = conn->c_faddr; | |
763 | strncpy(cinfo6->transport, conn->c_trans->t_name, | |
764 | sizeof(cinfo6->transport)); | |
765 | cinfo6->flags = 0; | |
766 | ||
767 | rds_conn_info_set(cinfo6->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags), | |
768 | SENDING); | |
769 | /* XXX Future: return the state rather than these funky bits */ | |
770 | rds_conn_info_set(cinfo6->flags, | |
771 | atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING, | |
772 | CONNECTING); | |
773 | rds_conn_info_set(cinfo6->flags, | |
774 | atomic_read(&cp->cp_state) == RDS_CONN_UP, | |
775 | CONNECTED); | |
776 | /* Just return 1 as there is no error case. This is a helper function | |
777 | * for rds_walk_conn_path_info() and it wants a return value. | |
778 | */ | |
779 | return 1; | |
780 | } | |
e65d4d96 | 781 | #endif |
b7ff8b10 | 782 | |
00e0f34c AG |
783 | static void rds_conn_info(struct socket *sock, unsigned int len, |
784 | struct rds_info_iterator *iter, | |
785 | struct rds_info_lengths *lens) | |
786 | { | |
b2c9272a SM |
787 | u64 buffer[(sizeof(struct rds_info_connection) + 7) / 8]; |
788 | ||
992c9ec5 | 789 | rds_walk_conn_path_info(sock, len, iter, lens, |
00e0f34c | 790 | rds_conn_info_visitor, |
b2c9272a | 791 | buffer, |
00e0f34c AG |
792 | sizeof(struct rds_info_connection)); |
793 | } | |
794 | ||
e65d4d96 | 795 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
796 | static void rds6_conn_info(struct socket *sock, unsigned int len, |
797 | struct rds_info_iterator *iter, | |
798 | struct rds_info_lengths *lens) | |
799 | { | |
800 | u64 buffer[(sizeof(struct rds6_info_connection) + 7) / 8]; | |
801 | ||
802 | rds_walk_conn_path_info(sock, len, iter, lens, | |
803 | rds6_conn_info_visitor, | |
804 | buffer, | |
805 | sizeof(struct rds6_info_connection)); | |
806 | } | |
e65d4d96 | 807 | #endif |
b7ff8b10 | 808 | |
ef87b7ea | 809 | int rds_conn_init(void) |
00e0f34c | 810 | { |
c809195f SV |
811 | int ret; |
812 | ||
813 | ret = rds_loop_net_init(); /* register pernet callback */ | |
814 | if (ret) | |
815 | return ret; | |
816 | ||
00e0f34c AG |
817 | rds_conn_slab = kmem_cache_create("rds_connection", |
818 | sizeof(struct rds_connection), | |
819 | 0, 0, NULL); | |
c809195f SV |
820 | if (!rds_conn_slab) { |
821 | rds_loop_net_exit(); | |
00e0f34c | 822 | return -ENOMEM; |
c809195f | 823 | } |
00e0f34c AG |
824 | |
825 | rds_info_register_func(RDS_INFO_CONNECTIONS, rds_conn_info); | |
826 | rds_info_register_func(RDS_INFO_SEND_MESSAGES, | |
827 | rds_conn_message_info_send); | |
828 | rds_info_register_func(RDS_INFO_RETRANS_MESSAGES, | |
829 | rds_conn_message_info_retrans); | |
e65d4d96 | 830 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
831 | rds_info_register_func(RDS6_INFO_CONNECTIONS, rds6_conn_info); |
832 | rds_info_register_func(RDS6_INFO_SEND_MESSAGES, | |
833 | rds6_conn_message_info_send); | |
834 | rds_info_register_func(RDS6_INFO_RETRANS_MESSAGES, | |
835 | rds6_conn_message_info_retrans); | |
e65d4d96 | 836 | #endif |
00e0f34c AG |
837 | return 0; |
838 | } | |
839 | ||
840 | void rds_conn_exit(void) | |
841 | { | |
c809195f | 842 | rds_loop_net_exit(); /* unregister pernet callback */ |
00e0f34c AG |
843 | rds_loop_exit(); |
844 | ||
845 | WARN_ON(!hlist_empty(rds_conn_hash)); | |
846 | ||
847 | kmem_cache_destroy(rds_conn_slab); | |
848 | ||
849 | rds_info_deregister_func(RDS_INFO_CONNECTIONS, rds_conn_info); | |
850 | rds_info_deregister_func(RDS_INFO_SEND_MESSAGES, | |
851 | rds_conn_message_info_send); | |
852 | rds_info_deregister_func(RDS_INFO_RETRANS_MESSAGES, | |
853 | rds_conn_message_info_retrans); | |
e65d4d96 | 854 | #if IS_ENABLED(CONFIG_IPV6) |
b7ff8b10 KCP |
855 | rds_info_deregister_func(RDS6_INFO_CONNECTIONS, rds6_conn_info); |
856 | rds_info_deregister_func(RDS6_INFO_SEND_MESSAGES, | |
857 | rds6_conn_message_info_send); | |
858 | rds_info_deregister_func(RDS6_INFO_RETRANS_MESSAGES, | |
859 | rds6_conn_message_info_retrans); | |
e65d4d96 | 860 | #endif |
00e0f34c AG |
861 | } |
862 | ||
863 | /* | |
864 | * Force a disconnect | |
865 | */ | |
aed20a53 | 866 | void rds_conn_path_drop(struct rds_conn_path *cp, bool destroy) |
0cb43965 SV |
867 | { |
868 | atomic_set(&cp->cp_state, RDS_CONN_ERROR); | |
aed20a53 | 869 | |
3db6e0d1 | 870 | rcu_read_lock(); |
ebeeb1ad | 871 | if (!destroy && rds_destroy_pending(cp->cp_conn)) { |
3db6e0d1 | 872 | rcu_read_unlock(); |
aed20a53 | 873 | return; |
3db6e0d1 | 874 | } |
0cb43965 | 875 | queue_work(rds_wq, &cp->cp_down_w); |
3db6e0d1 | 876 | rcu_read_unlock(); |
0cb43965 SV |
877 | } |
878 | EXPORT_SYMBOL_GPL(rds_conn_path_drop); | |
879 | ||
00e0f34c AG |
880 | void rds_conn_drop(struct rds_connection *conn) |
881 | { | |
5916e2c1 | 882 | WARN_ON(conn->c_trans->t_mp_capable); |
aed20a53 | 883 | rds_conn_path_drop(&conn->c_path[0], false); |
00e0f34c | 884 | } |
616b757a | 885 | EXPORT_SYMBOL_GPL(rds_conn_drop); |
00e0f34c | 886 | |
f3c6808d ZB |
887 | /* |
888 | * If the connection is down, trigger a connect. We may have scheduled a | |
889 | * delayed reconnect however - in this case we should not interfere. | |
890 | */ | |
3c0a5900 SV |
891 | void rds_conn_path_connect_if_down(struct rds_conn_path *cp) |
892 | { | |
3db6e0d1 | 893 | rcu_read_lock(); |
ebeeb1ad | 894 | if (rds_destroy_pending(cp->cp_conn)) { |
3db6e0d1 SV |
895 | rcu_read_unlock(); |
896 | return; | |
897 | } | |
3c0a5900 SV |
898 | if (rds_conn_path_state(cp) == RDS_CONN_DOWN && |
899 | !test_and_set_bit(RDS_RECONNECT_PENDING, &cp->cp_flags)) | |
900 | queue_delayed_work(rds_wq, &cp->cp_conn_w, 0); | |
3db6e0d1 | 901 | rcu_read_unlock(); |
3c0a5900 | 902 | } |
1a0e100f | 903 | EXPORT_SYMBOL_GPL(rds_conn_path_connect_if_down); |
3c0a5900 | 904 | |
f3c6808d ZB |
905 | void rds_conn_connect_if_down(struct rds_connection *conn) |
906 | { | |
3c0a5900 SV |
907 | WARN_ON(conn->c_trans->t_mp_capable); |
908 | rds_conn_path_connect_if_down(&conn->c_path[0]); | |
f3c6808d ZB |
909 | } |
910 | EXPORT_SYMBOL_GPL(rds_conn_connect_if_down); | |
911 | ||
fb1b3dc4 SV |
912 | void |
913 | __rds_conn_path_error(struct rds_conn_path *cp, const char *fmt, ...) | |
914 | { | |
915 | va_list ap; | |
916 | ||
917 | va_start(ap, fmt); | |
918 | vprintk(fmt, ap); | |
919 | va_end(ap); | |
920 | ||
aed20a53 | 921 | rds_conn_path_drop(cp, false); |
fb1b3dc4 | 922 | } |