Commit | Line | Data |
---|---|---|
b4d0d230 | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
4a3388c8 DH |
2 | /* Client connection-specific management code. |
3 | * | |
245500d8 | 4 | * Copyright (C) 2016, 2020 Red Hat, Inc. All Rights Reserved. |
4a3388c8 DH |
5 | * Written by David Howells (dhowells@redhat.com) |
6 | * | |
45025bce DH |
7 | * Client connections need to be cached for a little while after they've made a |
8 | * call so as to handle retransmitted DATA packets in case the server didn't | |
9 | * receive the final ACK or terminating ABORT we sent it. | |
10 | * | |
45025bce DH |
11 | * There are flags of relevance to the cache: |
12 | * | |
45025bce DH |
13 | * (2) DONT_REUSE - The connection should be discarded as soon as possible and |
14 | * should not be reused. This is set when an exclusive connection is used | |
15 | * or a call ID counter overflows. | |
16 | * | |
17 | * The caching state may only be changed if the cache lock is held. | |
18 | * | |
19 | * There are two idle client connection expiry durations. If the total number | |
20 | * of connections is below the reap threshold, we use the normal duration; if | |
21 | * it's above, we use the fast duration. | |
4a3388c8 DH |
22 | */ |
23 | ||
24 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
25 | ||
26 | #include <linux/slab.h> | |
27 | #include <linux/idr.h> | |
28 | #include <linux/timer.h> | |
174cd4b1 IM |
29 | #include <linux/sched/signal.h> |
30 | ||
4a3388c8 DH |
31 | #include "ar-internal.h" |
32 | ||
45025bce | 33 | __read_mostly unsigned int rxrpc_reap_client_connections = 900; |
a158bdd3 DH |
34 | __read_mostly unsigned long rxrpc_conn_idle_client_expiry = 2 * 60 * HZ; |
35 | __read_mostly unsigned long rxrpc_conn_idle_client_fast_expiry = 2 * HZ; | |
45025bce | 36 | |
1bab27af DH |
37 | static void rxrpc_activate_bundle(struct rxrpc_bundle *bundle) |
38 | { | |
39 | atomic_inc(&bundle->active); | |
40 | } | |
3bcd6c7e | 41 | |
4a3388c8 | 42 | /* |
f06cb291 | 43 | * Release a connection ID for a client connection. |
4a3388c8 | 44 | */ |
f06cb291 DH |
45 | static void rxrpc_put_client_connection_id(struct rxrpc_local *local, |
46 | struct rxrpc_connection *conn) | |
4a3388c8 | 47 | { |
9d35d880 | 48 | idr_remove(&local->conn_ids, conn->proto.cid >> RXRPC_CIDSHIFT); |
4a3388c8 | 49 | } |
eb9b9d22 DH |
50 | |
51 | /* | |
52 | * Destroy the client connection ID tree. | |
53 | */ | |
9d35d880 | 54 | static void rxrpc_destroy_client_conn_ids(struct rxrpc_local *local) |
eb9b9d22 DH |
55 | { |
56 | struct rxrpc_connection *conn; | |
57 | int id; | |
58 | ||
f06cb291 DH |
59 | if (!idr_is_empty(&local->conn_ids)) { |
60 | idr_for_each_entry(&local->conn_ids, conn, id) { | |
eb9b9d22 | 61 | pr_err("AF_RXRPC: Leaked client conn %p {%d}\n", |
a0575429 | 62 | conn, refcount_read(&conn->ref)); |
eb9b9d22 DH |
63 | } |
64 | BUG(); | |
65 | } | |
66 | ||
f06cb291 | 67 | idr_destroy(&local->conn_ids); |
eb9b9d22 | 68 | } |
c6d2b8d7 | 69 | |
245500d8 DH |
70 | /* |
71 | * Allocate a connection bundle. | |
72 | */ | |
1bab27af | 73 | static struct rxrpc_bundle *rxrpc_alloc_bundle(struct rxrpc_call *call, |
245500d8 DH |
74 | gfp_t gfp) |
75 | { | |
0c3bd086 | 76 | static atomic_t rxrpc_bundle_id; |
245500d8 DH |
77 | struct rxrpc_bundle *bundle; |
78 | ||
79 | bundle = kzalloc(sizeof(*bundle), gfp); | |
80 | if (bundle) { | |
1bab27af DH |
81 | bundle->local = call->local; |
82 | bundle->peer = rxrpc_get_peer(call->peer, rxrpc_peer_get_bundle); | |
83 | bundle->key = key_get(call->key); | |
84 | bundle->security = call->security; | |
85 | bundle->exclusive = test_bit(RXRPC_CALL_EXCLUSIVE, &call->flags); | |
86 | bundle->upgrade = test_bit(RXRPC_CALL_UPGRADE, &call->flags); | |
87 | bundle->service_id = call->dest_srx.srx_service; | |
88 | bundle->security_level = call->security_level; | |
0c3bd086 | 89 | bundle->debug_id = atomic_inc_return(&rxrpc_bundle_id); |
a0575429 | 90 | refcount_set(&bundle->ref, 1); |
3bcd6c7e | 91 | atomic_set(&bundle->active, 1); |
245500d8 | 92 | INIT_LIST_HEAD(&bundle->waiting_calls); |
fa3492ab | 93 | trace_rxrpc_bundle(bundle->debug_id, 1, rxrpc_bundle_new); |
245500d8 DH |
94 | } |
95 | return bundle; | |
96 | } | |
97 | ||
fa3492ab DH |
98 | struct rxrpc_bundle *rxrpc_get_bundle(struct rxrpc_bundle *bundle, |
99 | enum rxrpc_bundle_trace why) | |
245500d8 | 100 | { |
fa3492ab DH |
101 | int r; |
102 | ||
103 | __refcount_inc(&bundle->ref, &r); | |
104 | trace_rxrpc_bundle(bundle->debug_id, r + 1, why); | |
245500d8 DH |
105 | return bundle; |
106 | } | |
107 | ||
ca77fba8 ET |
108 | static void rxrpc_free_bundle(struct rxrpc_bundle *bundle) |
109 | { | |
0c3bd086 DH |
110 | trace_rxrpc_bundle(bundle->debug_id, refcount_read(&bundle->ref), |
111 | rxrpc_bundle_free); | |
47c810a7 | 112 | rxrpc_put_peer(bundle->peer, rxrpc_peer_put_bundle); |
1bab27af | 113 | key_put(bundle->key); |
ca77fba8 ET |
114 | kfree(bundle); |
115 | } | |
116 | ||
fa3492ab | 117 | void rxrpc_put_bundle(struct rxrpc_bundle *bundle, enum rxrpc_bundle_trace why) |
245500d8 | 118 | { |
1bab27af | 119 | unsigned int id; |
a0575429 DH |
120 | bool dead; |
121 | int r; | |
245500d8 | 122 | |
1bab27af DH |
123 | if (bundle) { |
124 | id = bundle->debug_id; | |
125 | dead = __refcount_dec_and_test(&bundle->ref, &r); | |
126 | trace_rxrpc_bundle(id, r - 1, why); | |
127 | if (dead) | |
128 | rxrpc_free_bundle(bundle); | |
129 | } | |
245500d8 DH |
130 | } |
131 | ||
9d35d880 DH |
132 | /* |
133 | * Get rid of outstanding client connection preallocations when a local | |
134 | * endpoint is destroyed. | |
135 | */ | |
136 | void rxrpc_purge_client_connections(struct rxrpc_local *local) | |
137 | { | |
138 | rxrpc_destroy_client_conn_ids(local); | |
139 | } | |
140 | ||
c6d2b8d7 | 141 | /* |
45025bce | 142 | * Allocate a client connection. |
c6d2b8d7 DH |
143 | */ |
144 | static struct rxrpc_connection * | |
9d35d880 | 145 | rxrpc_alloc_client_connection(struct rxrpc_bundle *bundle) |
c6d2b8d7 DH |
146 | { |
147 | struct rxrpc_connection *conn; | |
9d35d880 DH |
148 | struct rxrpc_local *local = bundle->local; |
149 | struct rxrpc_net *rxnet = local->rxnet; | |
150 | int id; | |
c6d2b8d7 DH |
151 | |
152 | _enter(""); | |
153 | ||
9d35d880 DH |
154 | conn = rxrpc_alloc_connection(rxnet, GFP_ATOMIC | __GFP_NOWARN); |
155 | if (!conn) | |
c6d2b8d7 | 156 | return ERR_PTR(-ENOMEM); |
9d35d880 DH |
157 | |
158 | id = idr_alloc_cyclic(&local->conn_ids, conn, 1, 0x40000000, | |
159 | GFP_ATOMIC | __GFP_NOWARN); | |
160 | if (id < 0) { | |
161 | kfree(conn); | |
162 | return ERR_PTR(id); | |
c6d2b8d7 DH |
163 | } |
164 | ||
a0575429 | 165 | refcount_set(&conn->ref, 1); |
9d35d880 DH |
166 | conn->proto.cid = id << RXRPC_CIDSHIFT; |
167 | conn->proto.epoch = local->rxnet->epoch; | |
168 | conn->out_clientflag = RXRPC_CLIENT_INITIATED; | |
169 | conn->bundle = rxrpc_get_bundle(bundle, rxrpc_bundle_get_client_conn); | |
170 | conn->local = rxrpc_get_local(bundle->local, rxrpc_local_get_client_conn); | |
171 | conn->peer = rxrpc_get_peer(bundle->peer, rxrpc_peer_get_client_conn); | |
172 | conn->key = key_get(bundle->key); | |
173 | conn->security = bundle->security; | |
2cc80086 DH |
174 | conn->exclusive = bundle->exclusive; |
175 | conn->upgrade = bundle->upgrade; | |
176 | conn->orig_service_id = bundle->service_id; | |
177 | conn->security_level = bundle->security_level; | |
9d35d880 | 178 | conn->state = RXRPC_CONN_CLIENT_UNSECURED; |
2cc80086 | 179 | conn->service_id = conn->orig_service_id; |
c6d2b8d7 | 180 | |
9d35d880 DH |
181 | if (conn->security == &rxrpc_no_security) |
182 | conn->state = RXRPC_CONN_CLIENT; | |
c6d2b8d7 | 183 | |
31f5f9a1 | 184 | atomic_inc(&rxnet->nr_conns); |
2baec2c3 DH |
185 | write_lock(&rxnet->conn_lock); |
186 | list_add_tail(&conn->proc_link, &rxnet->conn_proc_list); | |
187 | write_unlock(&rxnet->conn_lock); | |
c6d2b8d7 | 188 | |
9d35d880 | 189 | rxrpc_see_connection(conn, rxrpc_conn_new_client); |
245500d8 DH |
190 | |
191 | atomic_inc(&rxnet->nr_client_conns); | |
363deeab | 192 | trace_rxrpc_client(conn, -1, rxrpc_client_alloc); |
c6d2b8d7 | 193 | return conn; |
c6d2b8d7 DH |
194 | } |
195 | ||
196 | /* | |
45025bce | 197 | * Determine if a connection may be reused. |
c6d2b8d7 | 198 | */ |
45025bce DH |
199 | static bool rxrpc_may_reuse_conn(struct rxrpc_connection *conn) |
200 | { | |
245500d8 | 201 | struct rxrpc_net *rxnet; |
45025bce DH |
202 | int id_cursor, id, distance, limit; |
203 | ||
245500d8 DH |
204 | if (!conn) |
205 | goto dont_reuse; | |
206 | ||
3cec055c | 207 | rxnet = conn->rxnet; |
45025bce DH |
208 | if (test_bit(RXRPC_CONN_DONT_REUSE, &conn->flags)) |
209 | goto dont_reuse; | |
210 | ||
9d35d880 DH |
211 | if ((conn->state != RXRPC_CONN_CLIENT_UNSECURED && |
212 | conn->state != RXRPC_CONN_CLIENT) || | |
245500d8 | 213 | conn->proto.epoch != rxnet->epoch) |
45025bce DH |
214 | goto mark_dont_reuse; |
215 | ||
216 | /* The IDR tree gets very expensive on memory if the connection IDs are | |
217 | * widely scattered throughout the number space, so we shall want to | |
218 | * kill off connections that, say, have an ID more than about four | |
219 | * times the maximum number of client conns away from the current | |
220 | * allocation point to try and keep the IDs concentrated. | |
221 | */ | |
f06cb291 | 222 | id_cursor = idr_get_cursor(&conn->local->conn_ids); |
45025bce DH |
223 | id = conn->proto.cid >> RXRPC_CIDSHIFT; |
224 | distance = id - id_cursor; | |
225 | if (distance < 0) | |
226 | distance = -distance; | |
245500d8 | 227 | limit = max_t(unsigned long, atomic_read(&rxnet->nr_conns) * 4, 1024); |
45025bce DH |
228 | if (distance > limit) |
229 | goto mark_dont_reuse; | |
230 | ||
231 | return true; | |
232 | ||
233 | mark_dont_reuse: | |
234 | set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags); | |
235 | dont_reuse: | |
236 | return false; | |
237 | } | |
238 | ||
239 | /* | |
245500d8 DH |
240 | * Look up the conn bundle that matches the connection parameters, adding it if |
241 | * it doesn't yet exist. | |
45025bce | 242 | */ |
9d35d880 | 243 | int rxrpc_look_up_bundle(struct rxrpc_call *call, gfp_t gfp) |
c6d2b8d7 | 244 | { |
245500d8 | 245 | struct rxrpc_bundle *bundle, *candidate; |
1bab27af | 246 | struct rxrpc_local *local = call->local; |
c6d2b8d7 DH |
247 | struct rb_node *p, **pp, *parent; |
248 | long diff; | |
1bab27af | 249 | bool upgrade = test_bit(RXRPC_CALL_UPGRADE, &call->flags); |
c6d2b8d7 | 250 | |
245500d8 | 251 | _enter("{%px,%x,%u,%u}", |
1bab27af DH |
252 | call->peer, key_serial(call->key), call->security_level, |
253 | upgrade); | |
c6d2b8d7 | 254 | |
1bab27af DH |
255 | if (test_bit(RXRPC_CALL_EXCLUSIVE, &call->flags)) { |
256 | call->bundle = rxrpc_alloc_bundle(call, gfp); | |
9d35d880 | 257 | return call->bundle ? 0 : -ENOMEM; |
1bab27af | 258 | } |
c6d2b8d7 | 259 | |
245500d8 DH |
260 | /* First, see if the bundle is already there. */ |
261 | _debug("search 1"); | |
262 | spin_lock(&local->client_bundles_lock); | |
263 | p = local->client_bundles.rb_node; | |
264 | while (p) { | |
265 | bundle = rb_entry(p, struct rxrpc_bundle, local_node); | |
f7aec129 | 266 | |
1bab27af DH |
267 | #define cmp(X, Y) ((long)(X) - (long)(Y)) |
268 | diff = (cmp(bundle->peer, call->peer) ?: | |
269 | cmp(bundle->key, call->key) ?: | |
270 | cmp(bundle->security_level, call->security_level) ?: | |
271 | cmp(bundle->upgrade, upgrade)); | |
45025bce | 272 | #undef cmp |
245500d8 DH |
273 | if (diff < 0) |
274 | p = p->rb_left; | |
275 | else if (diff > 0) | |
276 | p = p->rb_right; | |
277 | else | |
278 | goto found_bundle; | |
c6d2b8d7 | 279 | } |
245500d8 DH |
280 | spin_unlock(&local->client_bundles_lock); |
281 | _debug("not found"); | |
c6d2b8d7 | 282 | |
245500d8 | 283 | /* It wasn't. We need to add one. */ |
1bab27af | 284 | candidate = rxrpc_alloc_bundle(call, gfp); |
245500d8 | 285 | if (!candidate) |
9d35d880 | 286 | return -ENOMEM; |
c6d2b8d7 | 287 | |
c6d2b8d7 | 288 | _debug("search 2"); |
245500d8 DH |
289 | spin_lock(&local->client_bundles_lock); |
290 | pp = &local->client_bundles.rb_node; | |
c6d2b8d7 DH |
291 | parent = NULL; |
292 | while (*pp) { | |
293 | parent = *pp; | |
245500d8 | 294 | bundle = rb_entry(parent, struct rxrpc_bundle, local_node); |
c6d2b8d7 | 295 | |
1bab27af DH |
296 | #define cmp(X, Y) ((long)(X) - (long)(Y)) |
297 | diff = (cmp(bundle->peer, call->peer) ?: | |
298 | cmp(bundle->key, call->key) ?: | |
299 | cmp(bundle->security_level, call->security_level) ?: | |
300 | cmp(bundle->upgrade, upgrade)); | |
45025bce | 301 | #undef cmp |
245500d8 | 302 | if (diff < 0) |
c6d2b8d7 | 303 | pp = &(*pp)->rb_left; |
245500d8 | 304 | else if (diff > 0) |
c6d2b8d7 | 305 | pp = &(*pp)->rb_right; |
245500d8 DH |
306 | else |
307 | goto found_bundle_free; | |
c6d2b8d7 DH |
308 | } |
309 | ||
245500d8 | 310 | _debug("new bundle"); |
245500d8 DH |
311 | rb_link_node(&candidate->local_node, parent, pp); |
312 | rb_insert_color(&candidate->local_node, &local->client_bundles); | |
1bab27af | 313 | call->bundle = rxrpc_get_bundle(candidate, rxrpc_bundle_get_client_call); |
245500d8 | 314 | spin_unlock(&local->client_bundles_lock); |
1bab27af | 315 | _leave(" = B=%u [new]", call->bundle->debug_id); |
9d35d880 | 316 | return 0; |
245500d8 DH |
317 | |
318 | found_bundle_free: | |
ca77fba8 | 319 | rxrpc_free_bundle(candidate); |
245500d8 | 320 | found_bundle: |
1bab27af DH |
321 | call->bundle = rxrpc_get_bundle(bundle, rxrpc_bundle_get_client_call); |
322 | rxrpc_activate_bundle(bundle); | |
245500d8 | 323 | spin_unlock(&local->client_bundles_lock); |
1bab27af | 324 | _leave(" = B=%u [found]", call->bundle->debug_id); |
9d35d880 | 325 | return 0; |
45025bce | 326 | } |
c6d2b8d7 | 327 | |
45025bce | 328 | /* |
245500d8 | 329 | * Allocate a new connection and add it into a bundle. |
45025bce | 330 | */ |
9d35d880 DH |
331 | static bool rxrpc_add_conn_to_bundle(struct rxrpc_bundle *bundle, |
332 | unsigned int slot) | |
45025bce | 333 | { |
9d35d880 DH |
334 | struct rxrpc_connection *conn, *old; |
335 | unsigned int shift = slot * RXRPC_MAXCALLS; | |
336 | unsigned int i; | |
45025bce | 337 | |
9d35d880 DH |
338 | old = bundle->conns[slot]; |
339 | if (old) { | |
340 | bundle->conns[slot] = NULL; | |
341 | trace_rxrpc_client(old, -1, rxrpc_client_replace); | |
342 | rxrpc_put_connection(old, rxrpc_conn_put_noreuse); | |
363deeab | 343 | } |
45025bce | 344 | |
9d35d880 DH |
345 | conn = rxrpc_alloc_client_connection(bundle); |
346 | if (IS_ERR(conn)) { | |
347 | bundle->alloc_error = PTR_ERR(conn); | |
348 | return false; | |
001c1122 DH |
349 | } |
350 | ||
9d35d880 DH |
351 | rxrpc_activate_bundle(bundle); |
352 | conn->bundle_shift = shift; | |
353 | bundle->conns[slot] = conn; | |
354 | for (i = 0; i < RXRPC_MAXCALLS; i++) | |
355 | set_bit(shift + i, &bundle->avail_chans); | |
356 | return true; | |
45025bce DH |
357 | } |
358 | ||
359 | /* | |
245500d8 DH |
360 | * Add a connection to a bundle if there are no usable connections or we have |
361 | * connections waiting for extra capacity. | |
45025bce | 362 | */ |
9d35d880 | 363 | static bool rxrpc_bundle_has_space(struct rxrpc_bundle *bundle) |
45025bce | 364 | { |
9d35d880 | 365 | int slot = -1, i, usable; |
45025bce | 366 | |
245500d8 DH |
367 | _enter(""); |
368 | ||
9d35d880 | 369 | bundle->alloc_error = 0; |
245500d8 DH |
370 | |
371 | /* See if there are any usable connections. */ | |
372 | usable = 0; | |
9d35d880 | 373 | for (i = 0; i < ARRAY_SIZE(bundle->conns); i++) { |
245500d8 DH |
374 | if (rxrpc_may_reuse_conn(bundle->conns[i])) |
375 | usable++; | |
9d35d880 DH |
376 | else if (slot == -1) |
377 | slot = i; | |
245500d8 DH |
378 | } |
379 | ||
9d35d880 DH |
380 | if (!usable && bundle->upgrade) |
381 | bundle->try_upgrade = true; | |
382 | ||
245500d8 DH |
383 | if (!usable) |
384 | goto alloc_conn; | |
385 | ||
288827d5 DH |
386 | if (!bundle->avail_chans && |
387 | !bundle->try_upgrade && | |
288827d5 DH |
388 | usable < ARRAY_SIZE(bundle->conns)) |
389 | goto alloc_conn; | |
390 | ||
245500d8 | 391 | _leave(""); |
9d35d880 | 392 | return usable; |
245500d8 DH |
393 | |
394 | alloc_conn: | |
9d35d880 | 395 | return slot >= 0 ? rxrpc_add_conn_to_bundle(bundle, slot) : false; |
45025bce DH |
396 | } |
397 | ||
398 | /* | |
399 | * Assign a channel to the call at the front of the queue and wake the call up. | |
400 | * We don't increment the callNumber counter until this number has been exposed | |
401 | * to the world. | |
402 | */ | |
403 | static void rxrpc_activate_one_channel(struct rxrpc_connection *conn, | |
404 | unsigned int channel) | |
405 | { | |
406 | struct rxrpc_channel *chan = &conn->channels[channel]; | |
245500d8 DH |
407 | struct rxrpc_bundle *bundle = conn->bundle; |
408 | struct rxrpc_call *call = list_entry(bundle->waiting_calls.next, | |
9d35d880 | 409 | struct rxrpc_call, wait_link); |
45025bce DH |
410 | u32 call_id = chan->call_counter + 1; |
411 | ||
245500d8 DH |
412 | _enter("C=%x,%u", conn->debug_id, channel); |
413 | ||
9d35d880 DH |
414 | list_del_init(&call->wait_link); |
415 | ||
363deeab DH |
416 | trace_rxrpc_client(conn, channel, rxrpc_client_chan_activate); |
417 | ||
3136ef49 DH |
418 | /* Cancel the final ACK on the previous call if it hasn't been sent yet |
419 | * as the DATA packet will implicitly ACK it. | |
420 | */ | |
421 | clear_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags); | |
245500d8 | 422 | clear_bit(conn->bundle_shift + channel, &bundle->avail_chans); |
af338a9e | 423 | |
cb0fc0c9 | 424 | rxrpc_see_call(call, rxrpc_call_see_activate_client); |
7fa25105 | 425 | call->conn = rxrpc_get_connection(conn, rxrpc_conn_get_activate_call); |
45025bce DH |
426 | call->cid = conn->proto.cid | channel; |
427 | call->call_id = call_id; | |
f3441d41 | 428 | call->dest_srx.srx_service = conn->service_id; |
9d35d880 DH |
429 | call->cong_ssthresh = call->peer->cong_ssthresh; |
430 | if (call->cong_cwnd >= call->cong_ssthresh) | |
431 | call->cong_mode = RXRPC_CALL_CONGEST_AVOIDANCE; | |
432 | else | |
433 | call->cong_mode = RXRPC_CALL_SLOW_START; | |
245500d8 DH |
434 | |
435 | chan->call_id = call_id; | |
436 | chan->call_debug_id = call->debug_id; | |
9d35d880 DH |
437 | chan->call = call; |
438 | ||
439 | rxrpc_see_call(call, rxrpc_call_see_connected); | |
440 | trace_rxrpc_connect_call(call); | |
441 | call->tx_last_sent = ktime_get_real(); | |
442 | rxrpc_start_call_timer(call); | |
443 | rxrpc_set_call_state(call, RXRPC_CALL_CLIENT_SEND_REQUEST); | |
45025bce DH |
444 | wake_up(&call->waitq); |
445 | } | |
446 | ||
245500d8 DH |
447 | /* |
448 | * Remove a connection from the idle list if it's on it. | |
449 | */ | |
9d35d880 | 450 | static void rxrpc_unidle_conn(struct rxrpc_connection *conn) |
245500d8 | 451 | { |
245500d8 | 452 | if (!list_empty(&conn->cache_link)) { |
9d35d880 DH |
453 | list_del_init(&conn->cache_link); |
454 | rxrpc_put_connection(conn, rxrpc_conn_put_unidle); | |
245500d8 DH |
455 | } |
456 | } | |
457 | ||
2629c7fa | 458 | /* |
9d35d880 | 459 | * Assign channels and callNumbers to waiting calls. |
2629c7fa | 460 | */ |
9d35d880 | 461 | static void rxrpc_activate_channels(struct rxrpc_bundle *bundle) |
2629c7fa | 462 | { |
245500d8 DH |
463 | struct rxrpc_connection *conn; |
464 | unsigned long avail, mask; | |
465 | unsigned int channel, slot; | |
2629c7fa | 466 | |
9d35d880 DH |
467 | trace_rxrpc_client(NULL, -1, rxrpc_client_activate_chans); |
468 | ||
245500d8 DH |
469 | if (bundle->try_upgrade) |
470 | mask = 1; | |
471 | else | |
472 | mask = ULONG_MAX; | |
473 | ||
474 | while (!list_empty(&bundle->waiting_calls)) { | |
475 | avail = bundle->avail_chans & mask; | |
476 | if (!avail) | |
477 | break; | |
478 | channel = __ffs(avail); | |
479 | clear_bit(channel, &bundle->avail_chans); | |
480 | ||
481 | slot = channel / RXRPC_MAXCALLS; | |
482 | conn = bundle->conns[slot]; | |
483 | if (!conn) | |
484 | break; | |
485 | ||
486 | if (bundle->try_upgrade) | |
487 | set_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags); | |
9d35d880 | 488 | rxrpc_unidle_conn(conn); |
245500d8 DH |
489 | |
490 | channel &= (RXRPC_MAXCALLS - 1); | |
491 | conn->act_chans |= 1 << channel; | |
492 | rxrpc_activate_one_channel(conn, channel); | |
493 | } | |
2629c7fa DH |
494 | } |
495 | ||
45025bce | 496 | /* |
9d35d880 | 497 | * Connect waiting channels (called from the I/O thread). |
45025bce | 498 | */ |
9d35d880 | 499 | void rxrpc_connect_client_calls(struct rxrpc_local *local) |
45025bce | 500 | { |
9d35d880 | 501 | struct rxrpc_call *call; |
45025bce | 502 | |
9d35d880 DH |
503 | while ((call = list_first_entry_or_null(&local->new_client_calls, |
504 | struct rxrpc_call, wait_link)) | |
505 | ) { | |
506 | struct rxrpc_bundle *bundle = call->bundle; | |
45025bce | 507 | |
9d35d880 DH |
508 | spin_lock(&local->client_call_lock); |
509 | list_move_tail(&call->wait_link, &bundle->waiting_calls); | |
510 | spin_unlock(&local->client_call_lock); | |
245500d8 | 511 | |
9d35d880 DH |
512 | if (rxrpc_bundle_has_space(bundle)) |
513 | rxrpc_activate_channels(bundle); | |
363deeab | 514 | } |
45025bce DH |
515 | } |
516 | ||
517 | /* | |
518 | * Note that a call, and thus a connection, is about to be exposed to the | |
519 | * world. | |
520 | */ | |
521 | void rxrpc_expose_client_call(struct rxrpc_call *call) | |
522 | { | |
363deeab | 523 | unsigned int channel = call->cid & RXRPC_CHANNELMASK; |
45025bce | 524 | struct rxrpc_connection *conn = call->conn; |
363deeab | 525 | struct rxrpc_channel *chan = &conn->channels[channel]; |
45025bce DH |
526 | |
527 | if (!test_and_set_bit(RXRPC_CALL_EXPOSED, &call->flags)) { | |
528 | /* Mark the call ID as being used. If the callNumber counter | |
529 | * exceeds ~2 billion, we kill the connection after its | |
530 | * outstanding calls have finished so that the counter doesn't | |
531 | * wrap. | |
532 | */ | |
533 | chan->call_counter++; | |
534 | if (chan->call_counter >= INT_MAX) | |
535 | set_bit(RXRPC_CONN_DONT_REUSE, &conn->flags); | |
245500d8 | 536 | trace_rxrpc_client(conn, channel, rxrpc_client_exposed); |
29fb4ec3 DH |
537 | |
538 | spin_lock(&call->peer->lock); | |
539 | hlist_add_head(&call->error_link, &call->peer->error_targets); | |
540 | spin_unlock(&call->peer->lock); | |
45025bce DH |
541 | } |
542 | } | |
543 | ||
3d18cbb7 DH |
544 | /* |
545 | * Set the reap timer. | |
546 | */ | |
0d6bf319 | 547 | static void rxrpc_set_client_reap_timer(struct rxrpc_local *local) |
3d18cbb7 | 548 | { |
0d6bf319 | 549 | if (!local->kill_all_client_conns) { |
245500d8 DH |
550 | unsigned long now = jiffies; |
551 | unsigned long reap_at = now + rxrpc_conn_idle_client_expiry; | |
3d18cbb7 | 552 | |
0d6bf319 DH |
553 | if (local->rxnet->live) |
554 | timer_reduce(&local->client_conn_reap_timer, reap_at); | |
245500d8 | 555 | } |
3d18cbb7 DH |
556 | } |
557 | ||
45025bce DH |
558 | /* |
559 | * Disconnect a client call. | |
560 | */ | |
245500d8 | 561 | void rxrpc_disconnect_client_call(struct rxrpc_bundle *bundle, struct rxrpc_call *call) |
45025bce | 562 | { |
245500d8 | 563 | struct rxrpc_connection *conn; |
930c9f91 | 564 | struct rxrpc_channel *chan = NULL; |
0d6bf319 | 565 | struct rxrpc_local *local = bundle->local; |
245500d8 DH |
566 | unsigned int channel; |
567 | bool may_reuse; | |
930c9f91 | 568 | u32 cid; |
45025bce | 569 | |
245500d8 | 570 | _enter("c=%x", call->debug_id); |
930c9f91 | 571 | |
45025bce | 572 | /* Calls that have never actually been assigned a channel can simply be |
245500d8 | 573 | * discarded. |
45025bce | 574 | */ |
245500d8 DH |
575 | conn = call->conn; |
576 | if (!conn) { | |
45025bce DH |
577 | _debug("call is waiting"); |
578 | ASSERTCMP(call->call_id, ==, 0); | |
579 | ASSERT(!test_bit(RXRPC_CALL_EXPOSED, &call->flags)); | |
9d35d880 DH |
580 | list_del_init(&call->wait_link); |
581 | return; | |
45025bce DH |
582 | } |
583 | ||
245500d8 DH |
584 | cid = call->cid; |
585 | channel = cid & RXRPC_CHANNELMASK; | |
586 | chan = &conn->channels[channel]; | |
587 | trace_rxrpc_client(conn, channel, rxrpc_client_chan_disconnect); | |
588 | ||
9d35d880 DH |
589 | if (WARN_ON(chan->call != call)) |
590 | return; | |
45025bce | 591 | |
245500d8 DH |
592 | may_reuse = rxrpc_may_reuse_conn(conn); |
593 | ||
45025bce DH |
594 | /* If a client call was exposed to the world, we save the result for |
595 | * retransmission. | |
596 | * | |
597 | * We use a barrier here so that the call number and abort code can be | |
598 | * read without needing to take a lock. | |
599 | * | |
600 | * TODO: Make the incoming packet handler check this and handle | |
601 | * terminal retransmission without requiring access to the call. | |
602 | */ | |
603 | if (test_bit(RXRPC_CALL_EXPOSED, &call->flags)) { | |
f5c17aae | 604 | _debug("exposed %u,%u", call->call_id, call->abort_code); |
45025bce | 605 | __rxrpc_disconnect_call(conn, call); |
245500d8 DH |
606 | |
607 | if (test_and_clear_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags)) { | |
608 | trace_rxrpc_client(conn, channel, rxrpc_client_to_active); | |
609 | bundle->try_upgrade = false; | |
610 | if (may_reuse) | |
9d35d880 | 611 | rxrpc_activate_channels(bundle); |
245500d8 | 612 | } |
45025bce DH |
613 | } |
614 | ||
615 | /* See if we can pass the channel directly to another call. */ | |
245500d8 | 616 | if (may_reuse && !list_empty(&bundle->waiting_calls)) { |
363deeab | 617 | trace_rxrpc_client(conn, channel, rxrpc_client_chan_pass); |
45025bce | 618 | rxrpc_activate_one_channel(conn, channel); |
9d35d880 | 619 | return; |
45025bce DH |
620 | } |
621 | ||
3136ef49 DH |
622 | /* Schedule the final ACK to be transmitted in a short while so that it |
623 | * can be skipped if we find a follow-on call. The first DATA packet | |
624 | * of the follow on call will implicitly ACK this call. | |
625 | */ | |
17e9e23b DH |
626 | if (call->completion == RXRPC_CALL_SUCCEEDED && |
627 | test_bit(RXRPC_CALL_EXPOSED, &call->flags)) { | |
3136ef49 DH |
628 | unsigned long final_ack_at = jiffies + 2; |
629 | ||
630 | WRITE_ONCE(chan->final_ack_at, final_ack_at); | |
631 | smp_wmb(); /* vs rxrpc_process_delayed_final_acks() */ | |
632 | set_bit(RXRPC_CONN_FINAL_ACK_0 + channel, &conn->flags); | |
633 | rxrpc_reduce_conn_timer(conn, final_ack_at); | |
634 | } | |
635 | ||
245500d8 | 636 | /* Deactivate the channel. */ |
9d35d880 | 637 | chan->call = NULL; |
245500d8 DH |
638 | set_bit(conn->bundle_shift + channel, &conn->bundle->avail_chans); |
639 | conn->act_chans &= ~(1 << channel); | |
45025bce | 640 | |
245500d8 DH |
641 | /* If no channels remain active, then put the connection on the idle |
642 | * list for a short while. Give it a ref to stop it going away if it | |
643 | * becomes unbundled. | |
644 | */ | |
645 | if (!conn->act_chans) { | |
646 | trace_rxrpc_client(conn, channel, rxrpc_client_to_idle); | |
647 | conn->idle_timestamp = jiffies; | |
45025bce | 648 | |
7fa25105 | 649 | rxrpc_get_connection(conn, rxrpc_conn_get_idle); |
0d6bf319 | 650 | list_move_tail(&conn->cache_link, &local->idle_client_conns); |
45025bce | 651 | |
0d6bf319 | 652 | rxrpc_set_client_reap_timer(local); |
45025bce | 653 | } |
245500d8 | 654 | } |
45025bce | 655 | |
245500d8 DH |
656 | /* |
657 | * Remove a connection from a bundle. | |
658 | */ | |
659 | static void rxrpc_unbundle_conn(struct rxrpc_connection *conn) | |
660 | { | |
661 | struct rxrpc_bundle *bundle = conn->bundle; | |
245500d8 | 662 | unsigned int bindex; |
245500d8 DH |
663 | int i; |
664 | ||
665 | _enter("C=%x", conn->debug_id); | |
666 | ||
ddc7834a DH |
667 | if (conn->flags & RXRPC_CONN_FINAL_ACK_MASK) |
668 | rxrpc_process_delayed_final_acks(conn, true); | |
669 | ||
245500d8 DH |
670 | bindex = conn->bundle_shift / RXRPC_MAXCALLS; |
671 | if (bundle->conns[bindex] == conn) { | |
672 | _debug("clear slot %u", bindex); | |
673 | bundle->conns[bindex] = NULL; | |
674 | for (i = 0; i < RXRPC_MAXCALLS; i++) | |
675 | clear_bit(conn->bundle_shift + i, &bundle->avail_chans); | |
9d35d880 | 676 | rxrpc_put_client_connection_id(bundle->local, conn); |
3bcd6c7e | 677 | rxrpc_deactivate_bundle(bundle); |
7fa25105 | 678 | rxrpc_put_connection(conn, rxrpc_conn_put_unbundle); |
3bcd6c7e DH |
679 | } |
680 | } | |
245500d8 | 681 | |
3bcd6c7e DH |
682 | /* |
683 | * Drop the active count on a bundle. | |
684 | */ | |
1bab27af | 685 | void rxrpc_deactivate_bundle(struct rxrpc_bundle *bundle) |
3bcd6c7e | 686 | { |
1bab27af | 687 | struct rxrpc_local *local; |
3bcd6c7e DH |
688 | bool need_put = false; |
689 | ||
1bab27af DH |
690 | if (!bundle) |
691 | return; | |
692 | ||
693 | local = bundle->local; | |
3bcd6c7e | 694 | if (atomic_dec_and_lock(&bundle->active, &local->client_bundles_lock)) { |
2cc80086 | 695 | if (!bundle->exclusive) { |
245500d8 DH |
696 | _debug("erase bundle"); |
697 | rb_erase(&bundle->local_node, &local->client_bundles); | |
f3af4ad1 | 698 | need_put = true; |
245500d8 DH |
699 | } |
700 | ||
701 | spin_unlock(&local->client_bundles_lock); | |
f3af4ad1 | 702 | if (need_put) |
fa3492ab | 703 | rxrpc_put_bundle(bundle, rxrpc_bundle_put_discard); |
245500d8 | 704 | } |
c6d2b8d7 | 705 | } |
001c1122 DH |
706 | |
707 | /* | |
45025bce | 708 | * Clean up a dead client connection. |
001c1122 | 709 | */ |
3cec055c | 710 | void rxrpc_kill_client_conn(struct rxrpc_connection *conn) |
001c1122 | 711 | { |
2cc80086 | 712 | struct rxrpc_local *local = conn->local; |
2baec2c3 | 713 | struct rxrpc_net *rxnet = local->rxnet; |
001c1122 | 714 | |
245500d8 | 715 | _enter("C=%x", conn->debug_id); |
363deeab | 716 | |
245500d8 DH |
717 | trace_rxrpc_client(conn, -1, rxrpc_client_cleanup); |
718 | atomic_dec(&rxnet->nr_client_conns); | |
001c1122 | 719 | |
f06cb291 | 720 | rxrpc_put_client_connection_id(local, conn); |
45025bce DH |
721 | } |
722 | ||
723 | /* | |
724 | * Discard expired client connections from the idle list. Each conn in the | |
725 | * idle list has been exposed and holds an extra ref because of that. | |
726 | * | |
727 | * This may be called from conn setup or from a work item so cannot be | |
728 | * considered non-reentrant. | |
729 | */ | |
0d6bf319 | 730 | void rxrpc_discard_expired_client_conns(struct rxrpc_local *local) |
45025bce DH |
731 | { |
732 | struct rxrpc_connection *conn; | |
733 | unsigned long expiry, conn_expires_at, now; | |
734 | unsigned int nr_conns; | |
45025bce | 735 | |
2baec2c3 | 736 | _enter(""); |
45025bce | 737 | |
45025bce DH |
738 | /* We keep an estimate of what the number of conns ought to be after |
739 | * we've discarded some so that we don't overdo the discarding. | |
740 | */ | |
0d6bf319 | 741 | nr_conns = atomic_read(&local->rxnet->nr_client_conns); |
45025bce DH |
742 | |
743 | next: | |
9d35d880 DH |
744 | conn = list_first_entry_or_null(&local->idle_client_conns, |
745 | struct rxrpc_connection, cache_link); | |
746 | if (!conn) | |
747 | return; | |
45025bce | 748 | |
0d6bf319 | 749 | if (!local->kill_all_client_conns) { |
45025bce DH |
750 | /* If the number of connections is over the reap limit, we |
751 | * expedite discard by reducing the expiry timeout. We must, | |
752 | * however, have at least a short grace period to be able to do | |
753 | * final-ACK or ABORT retransmission. | |
754 | */ | |
755 | expiry = rxrpc_conn_idle_client_expiry; | |
756 | if (nr_conns > rxrpc_reap_client_connections) | |
757 | expiry = rxrpc_conn_idle_client_fast_expiry; | |
2cc80086 | 758 | if (conn->local->service_closed) |
f859ab61 | 759 | expiry = rxrpc_closed_conn_expiry * HZ; |
45025bce DH |
760 | |
761 | conn_expires_at = conn->idle_timestamp + expiry; | |
762 | ||
763 | now = READ_ONCE(jiffies); | |
764 | if (time_after(conn_expires_at, now)) | |
765 | goto not_yet_expired; | |
766 | } | |
767 | ||
3cec055c | 768 | atomic_dec(&conn->active); |
363deeab | 769 | trace_rxrpc_client(conn, -1, rxrpc_client_discard); |
45025bce DH |
770 | list_del_init(&conn->cache_link); |
771 | ||
245500d8 | 772 | rxrpc_unbundle_conn(conn); |
7fa25105 DH |
773 | /* Drop the ->cache_link ref */ |
774 | rxrpc_put_connection(conn, rxrpc_conn_put_discard_idle); | |
245500d8 | 775 | |
45025bce DH |
776 | nr_conns--; |
777 | goto next; | |
778 | ||
779 | not_yet_expired: | |
780 | /* The connection at the front of the queue hasn't yet expired, so | |
781 | * schedule the work item for that point if we discarded something. | |
782 | * | |
783 | * We don't worry if the work item is already scheduled - it can look | |
784 | * after rescheduling itself at a later time. We could cancel it, but | |
785 | * then things get messier. | |
786 | */ | |
787 | _debug("not yet"); | |
0d6bf319 DH |
788 | if (!local->kill_all_client_conns) |
789 | timer_reduce(&local->client_conn_reap_timer, conn_expires_at); | |
45025bce | 790 | |
45025bce | 791 | _leave(""); |
001c1122 | 792 | } |
d12040b6 DH |
793 | |
794 | /* | |
795 | * Clean up the client connections on a local endpoint. | |
796 | */ | |
797 | void rxrpc_clean_up_local_conns(struct rxrpc_local *local) | |
798 | { | |
9d35d880 | 799 | struct rxrpc_connection *conn; |
d12040b6 DH |
800 | |
801 | _enter(""); | |
802 | ||
0d6bf319 | 803 | local->kill_all_client_conns = true; |
0d6bf319 DH |
804 | |
805 | del_timer_sync(&local->client_conn_reap_timer); | |
806 | ||
9d35d880 DH |
807 | while ((conn = list_first_entry_or_null(&local->idle_client_conns, |
808 | struct rxrpc_connection, cache_link))) { | |
d12040b6 | 809 | list_del_init(&conn->cache_link); |
9d35d880 DH |
810 | atomic_dec(&conn->active); |
811 | trace_rxrpc_client(conn, -1, rxrpc_client_discard); | |
546a4241 | 812 | rxrpc_unbundle_conn(conn); |
7fa25105 | 813 | rxrpc_put_connection(conn, rxrpc_conn_put_local_dead); |
d12040b6 DH |
814 | } |
815 | ||
816 | _leave(" [culled]"); | |
817 | } |