Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
80b14d5e TM |
2 | /* |
3 | * Multipath support for RPC | |
4 | * | |
5 | * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved. | |
6 | * | |
7 | * Trond Myklebust <trond.myklebust@primarydata.com> | |
8 | * | |
9 | */ | |
10 | #include <linux/types.h> | |
11 | #include <linux/kref.h> | |
12 | #include <linux/list.h> | |
13 | #include <linux/rcupdate.h> | |
14 | #include <linux/rculist.h> | |
15 | #include <linux/slab.h> | |
16 | #include <asm/cmpxchg.h> | |
17 | #include <linux/spinlock.h> | |
18 | #include <linux/sunrpc/xprt.h> | |
39e5d2df | 19 | #include <linux/sunrpc/addr.h> |
80b14d5e TM |
20 | #include <linux/sunrpc/xprtmultipath.h> |
21 | ||
22 | typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head, | |
23 | const struct rpc_xprt *cur); | |
24 | ||
25 | static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular; | |
26 | static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin; | |
27 | static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall; | |
28 | ||
29 | static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps, | |
30 | struct rpc_xprt *xprt) | |
31 | { | |
32 | if (unlikely(xprt_get(xprt) == NULL)) | |
33 | return; | |
34 | list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list); | |
35 | smp_wmb(); | |
36 | if (xps->xps_nxprts == 0) | |
37 | xps->xps_net = xprt->xprt_net; | |
38 | xps->xps_nxprts++; | |
39 | } | |
40 | ||
41 | /** | |
42 | * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch | |
43 | * @xps: pointer to struct rpc_xprt_switch | |
44 | * @xprt: pointer to struct rpc_xprt | |
45 | * | |
46 | * Adds xprt to the end of the list of struct rpc_xprt in xps. | |
47 | */ | |
48 | void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps, | |
49 | struct rpc_xprt *xprt) | |
50 | { | |
51 | if (xprt == NULL) | |
52 | return; | |
53 | spin_lock(&xps->xps_lock); | |
39e5d2df AA |
54 | if ((xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) && |
55 | !rpc_xprt_switch_has_addr(xps, (struct sockaddr *)&xprt->addr)) | |
80b14d5e TM |
56 | xprt_switch_add_xprt_locked(xps, xprt); |
57 | spin_unlock(&xps->xps_lock); | |
58 | } | |
59 | ||
60 | static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps, | |
61 | struct rpc_xprt *xprt) | |
62 | { | |
63 | if (unlikely(xprt == NULL)) | |
64 | return; | |
65 | xps->xps_nxprts--; | |
66 | if (xps->xps_nxprts == 0) | |
67 | xps->xps_net = NULL; | |
68 | smp_wmb(); | |
69 | list_del_rcu(&xprt->xprt_switch); | |
70 | } | |
71 | ||
72 | /** | |
73 | * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch | |
74 | * @xps: pointer to struct rpc_xprt_switch | |
75 | * @xprt: pointer to struct rpc_xprt | |
76 | * | |
77 | * Removes xprt from the list of struct rpc_xprt in xps. | |
78 | */ | |
79 | void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps, | |
80 | struct rpc_xprt *xprt) | |
81 | { | |
82 | spin_lock(&xps->xps_lock); | |
83 | xprt_switch_remove_xprt_locked(xps, xprt); | |
84 | spin_unlock(&xps->xps_lock); | |
85 | xprt_put(xprt); | |
86 | } | |
87 | ||
88 | /** | |
89 | * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch | |
90 | * @xprt: pointer to struct rpc_xprt | |
91 | * @gfp_flags: allocation flags | |
92 | * | |
93 | * On success, returns an initialised struct rpc_xprt_switch, containing | |
94 | * the entry xprt. Returns NULL on failure. | |
95 | */ | |
96 | struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt, | |
97 | gfp_t gfp_flags) | |
98 | { | |
99 | struct rpc_xprt_switch *xps; | |
100 | ||
101 | xps = kmalloc(sizeof(*xps), gfp_flags); | |
102 | if (xps != NULL) { | |
103 | spin_lock_init(&xps->xps_lock); | |
104 | kref_init(&xps->xps_kref); | |
105 | xps->xps_nxprts = 0; | |
106 | INIT_LIST_HEAD(&xps->xps_xprt_list); | |
107 | xps->xps_iter_ops = &rpc_xprt_iter_singular; | |
108 | xprt_switch_add_xprt_locked(xps, xprt); | |
109 | } | |
110 | ||
111 | return xps; | |
112 | } | |
113 | ||
114 | static void xprt_switch_free_entries(struct rpc_xprt_switch *xps) | |
115 | { | |
116 | spin_lock(&xps->xps_lock); | |
117 | while (!list_empty(&xps->xps_xprt_list)) { | |
118 | struct rpc_xprt *xprt; | |
119 | ||
120 | xprt = list_first_entry(&xps->xps_xprt_list, | |
121 | struct rpc_xprt, xprt_switch); | |
122 | xprt_switch_remove_xprt_locked(xps, xprt); | |
123 | spin_unlock(&xps->xps_lock); | |
124 | xprt_put(xprt); | |
125 | spin_lock(&xps->xps_lock); | |
126 | } | |
127 | spin_unlock(&xps->xps_lock); | |
128 | } | |
129 | ||
130 | static void xprt_switch_free(struct kref *kref) | |
131 | { | |
132 | struct rpc_xprt_switch *xps = container_of(kref, | |
133 | struct rpc_xprt_switch, xps_kref); | |
134 | ||
135 | xprt_switch_free_entries(xps); | |
136 | kfree_rcu(xps, xps_rcu); | |
137 | } | |
138 | ||
139 | /** | |
140 | * xprt_switch_get - Return a reference to a rpc_xprt_switch | |
141 | * @xps: pointer to struct rpc_xprt_switch | |
142 | * | |
143 | * Returns a reference to xps unless the refcount is already zero. | |
144 | */ | |
145 | struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps) | |
146 | { | |
147 | if (xps != NULL && kref_get_unless_zero(&xps->xps_kref)) | |
148 | return xps; | |
149 | return NULL; | |
150 | } | |
151 | ||
152 | /** | |
153 | * xprt_switch_put - Release a reference to a rpc_xprt_switch | |
154 | * @xps: pointer to struct rpc_xprt_switch | |
155 | * | |
156 | * Release the reference to xps, and free it once the refcount is zero. | |
157 | */ | |
158 | void xprt_switch_put(struct rpc_xprt_switch *xps) | |
159 | { | |
160 | if (xps != NULL) | |
161 | kref_put(&xps->xps_kref, xprt_switch_free); | |
162 | } | |
163 | ||
164 | /** | |
165 | * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch | |
166 | * @xps: pointer to struct rpc_xprt_switch | |
167 | * | |
168 | * Sets a round-robin default policy for iterators acting on xps. | |
169 | */ | |
170 | void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps) | |
171 | { | |
172 | if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin) | |
173 | WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin); | |
174 | } | |
175 | ||
176 | static | |
177 | const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi) | |
178 | { | |
179 | if (xpi->xpi_ops != NULL) | |
180 | return xpi->xpi_ops; | |
181 | return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops; | |
182 | } | |
183 | ||
184 | static | |
185 | void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi) | |
186 | { | |
187 | } | |
188 | ||
189 | static | |
190 | void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi) | |
191 | { | |
192 | WRITE_ONCE(xpi->xpi_cursor, NULL); | |
193 | } | |
194 | ||
195 | static | |
196 | struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head) | |
197 | { | |
198 | return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch); | |
199 | } | |
200 | ||
201 | static | |
202 | struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi) | |
203 | { | |
204 | struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); | |
205 | ||
206 | if (xps == NULL) | |
207 | return NULL; | |
208 | return xprt_switch_find_first_entry(&xps->xps_xprt_list); | |
209 | } | |
210 | ||
211 | static | |
212 | struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head, | |
213 | const struct rpc_xprt *cur) | |
214 | { | |
215 | struct rpc_xprt *pos; | |
216 | ||
217 | list_for_each_entry_rcu(pos, head, xprt_switch) { | |
218 | if (cur == pos) | |
219 | return pos; | |
220 | } | |
221 | return NULL; | |
222 | } | |
223 | ||
224 | static | |
225 | struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi) | |
226 | { | |
227 | struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); | |
228 | struct list_head *head; | |
229 | ||
230 | if (xps == NULL) | |
231 | return NULL; | |
232 | head = &xps->xps_xprt_list; | |
233 | if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2) | |
234 | return xprt_switch_find_first_entry(head); | |
235 | return xprt_switch_find_current_entry(head, xpi->xpi_cursor); | |
236 | } | |
237 | ||
39e5d2df AA |
238 | bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps, |
239 | const struct sockaddr *sap) | |
240 | { | |
241 | struct list_head *head; | |
242 | struct rpc_xprt *pos; | |
243 | ||
244 | if (xps == NULL || sap == NULL) | |
245 | return false; | |
246 | ||
247 | head = &xps->xps_xprt_list; | |
248 | list_for_each_entry_rcu(pos, head, xprt_switch) { | |
249 | if (rpc_cmp_addr_port(sap, (struct sockaddr *)&pos->addr)) { | |
250 | pr_info("RPC: addr %s already in xprt switch\n", | |
251 | pos->address_strings[RPC_DISPLAY_ADDR]); | |
252 | return true; | |
253 | } | |
254 | } | |
255 | return false; | |
256 | } | |
257 | ||
80b14d5e TM |
258 | static |
259 | struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head, | |
260 | const struct rpc_xprt *cur) | |
261 | { | |
262 | struct rpc_xprt *pos, *prev = NULL; | |
263 | ||
264 | list_for_each_entry_rcu(pos, head, xprt_switch) { | |
265 | if (cur == prev) | |
266 | return pos; | |
267 | prev = pos; | |
268 | } | |
269 | return NULL; | |
270 | } | |
271 | ||
272 | static | |
273 | struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head, | |
274 | struct rpc_xprt **cursor, | |
275 | xprt_switch_find_xprt_t find_next) | |
276 | { | |
277 | struct rpc_xprt *cur, *pos, *old; | |
278 | ||
279 | cur = READ_ONCE(*cursor); | |
280 | for (;;) { | |
281 | old = cur; | |
282 | pos = find_next(head, old); | |
283 | if (pos == NULL) | |
284 | break; | |
285 | cur = cmpxchg_relaxed(cursor, old, pos); | |
286 | if (cur == old) | |
287 | break; | |
288 | } | |
289 | return pos; | |
290 | } | |
291 | ||
292 | static | |
293 | struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi, | |
294 | xprt_switch_find_xprt_t find_next) | |
295 | { | |
296 | struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); | |
80b14d5e TM |
297 | |
298 | if (xps == NULL) | |
299 | return NULL; | |
bdc54d8e TM |
300 | return xprt_switch_set_next_cursor(&xps->xps_xprt_list, |
301 | &xpi->xpi_cursor, | |
302 | find_next); | |
80b14d5e TM |
303 | } |
304 | ||
305 | static | |
306 | struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head, | |
307 | const struct rpc_xprt *cur) | |
308 | { | |
309 | struct rpc_xprt *ret; | |
310 | ||
311 | ret = xprt_switch_find_next_entry(head, cur); | |
312 | if (ret != NULL) | |
313 | return ret; | |
314 | return xprt_switch_find_first_entry(head); | |
315 | } | |
316 | ||
317 | static | |
318 | struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi) | |
319 | { | |
320 | return xprt_iter_next_entry_multiple(xpi, | |
321 | xprt_switch_find_next_entry_roundrobin); | |
322 | } | |
323 | ||
324 | static | |
325 | struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi) | |
326 | { | |
327 | return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry); | |
328 | } | |
329 | ||
330 | /* | |
331 | * xprt_iter_rewind - Resets the xprt iterator | |
332 | * @xpi: pointer to rpc_xprt_iter | |
333 | * | |
334 | * Resets xpi to ensure that it points to the first entry in the list | |
335 | * of transports. | |
336 | */ | |
337 | static | |
338 | void xprt_iter_rewind(struct rpc_xprt_iter *xpi) | |
339 | { | |
340 | rcu_read_lock(); | |
341 | xprt_iter_ops(xpi)->xpi_rewind(xpi); | |
342 | rcu_read_unlock(); | |
343 | } | |
344 | ||
345 | static void __xprt_iter_init(struct rpc_xprt_iter *xpi, | |
346 | struct rpc_xprt_switch *xps, | |
347 | const struct rpc_xprt_iter_ops *ops) | |
348 | { | |
349 | rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps)); | |
350 | xpi->xpi_cursor = NULL; | |
351 | xpi->xpi_ops = ops; | |
352 | } | |
353 | ||
354 | /** | |
355 | * xprt_iter_init - Initialise an xprt iterator | |
356 | * @xpi: pointer to rpc_xprt_iter | |
357 | * @xps: pointer to rpc_xprt_switch | |
358 | * | |
359 | * Initialises the iterator to use the default iterator ops | |
360 | * as set in xps. This function is mainly intended for internal | |
361 | * use in the rpc_client. | |
362 | */ | |
363 | void xprt_iter_init(struct rpc_xprt_iter *xpi, | |
364 | struct rpc_xprt_switch *xps) | |
365 | { | |
366 | __xprt_iter_init(xpi, xps, NULL); | |
367 | } | |
368 | ||
369 | /** | |
370 | * xprt_iter_init_listall - Initialise an xprt iterator | |
371 | * @xpi: pointer to rpc_xprt_iter | |
372 | * @xps: pointer to rpc_xprt_switch | |
373 | * | |
374 | * Initialises the iterator to iterate once through the entire list | |
375 | * of entries in xps. | |
376 | */ | |
377 | void xprt_iter_init_listall(struct rpc_xprt_iter *xpi, | |
378 | struct rpc_xprt_switch *xps) | |
379 | { | |
380 | __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall); | |
381 | } | |
382 | ||
383 | /** | |
384 | * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch | |
385 | * @xpi: pointer to rpc_xprt_iter | |
386 | * @xps: pointer to a new rpc_xprt_switch or NULL | |
387 | * | |
388 | * Swaps out the existing xpi->xpi_xpswitch with a new value. | |
389 | */ | |
390 | struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi, | |
391 | struct rpc_xprt_switch *newswitch) | |
392 | { | |
393 | struct rpc_xprt_switch __rcu *oldswitch; | |
394 | ||
395 | /* Atomically swap out the old xpswitch */ | |
396 | oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch)); | |
397 | if (newswitch != NULL) | |
398 | xprt_iter_rewind(xpi); | |
399 | return rcu_dereference_protected(oldswitch, true); | |
400 | } | |
401 | ||
402 | /** | |
403 | * xprt_iter_destroy - Destroys the xprt iterator | |
404 | * @xpi pointer to rpc_xprt_iter | |
405 | */ | |
406 | void xprt_iter_destroy(struct rpc_xprt_iter *xpi) | |
407 | { | |
408 | xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL)); | |
409 | } | |
410 | ||
411 | /** | |
412 | * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor | |
413 | * @xpi: pointer to rpc_xprt_iter | |
414 | * | |
415 | * Returns a pointer to the struct rpc_xprt that is currently | |
416 | * pointed to by the cursor. | |
417 | * Caller must be holding rcu_read_lock(). | |
418 | */ | |
419 | struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi) | |
420 | { | |
421 | WARN_ON_ONCE(!rcu_read_lock_held()); | |
422 | return xprt_iter_ops(xpi)->xpi_xprt(xpi); | |
423 | } | |
424 | ||
425 | static | |
426 | struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi, | |
427 | struct rpc_xprt *(*fn)(struct rpc_xprt_iter *)) | |
428 | { | |
429 | struct rpc_xprt *ret; | |
430 | ||
431 | do { | |
432 | ret = fn(xpi); | |
433 | if (ret == NULL) | |
434 | break; | |
435 | ret = xprt_get(ret); | |
436 | } while (ret == NULL); | |
437 | return ret; | |
438 | } | |
439 | ||
440 | /** | |
441 | * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor | |
442 | * @xpi: pointer to rpc_xprt_iter | |
443 | * | |
444 | * Returns a reference to the struct rpc_xprt that is currently | |
445 | * pointed to by the cursor. | |
446 | */ | |
447 | struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi) | |
448 | { | |
449 | struct rpc_xprt *xprt; | |
450 | ||
451 | rcu_read_lock(); | |
452 | xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt); | |
453 | rcu_read_unlock(); | |
454 | return xprt; | |
455 | } | |
456 | ||
457 | /** | |
458 | * xprt_iter_get_next - Returns the next rpc_xprt following the cursor | |
459 | * @xpi: pointer to rpc_xprt_iter | |
460 | * | |
461 | * Returns a reference to the struct rpc_xprt that immediately follows the | |
462 | * entry pointed to by the cursor. | |
463 | */ | |
464 | struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi) | |
465 | { | |
466 | struct rpc_xprt *xprt; | |
467 | ||
468 | rcu_read_lock(); | |
469 | xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next); | |
470 | rcu_read_unlock(); | |
471 | return xprt; | |
472 | } | |
473 | ||
474 | /* Policy for always returning the first entry in the rpc_xprt_switch */ | |
475 | static | |
476 | const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = { | |
477 | .xpi_rewind = xprt_iter_no_rewind, | |
478 | .xpi_xprt = xprt_iter_first_entry, | |
479 | .xpi_next = xprt_iter_first_entry, | |
480 | }; | |
481 | ||
482 | /* Policy for round-robin iteration of entries in the rpc_xprt_switch */ | |
483 | static | |
484 | const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = { | |
485 | .xpi_rewind = xprt_iter_default_rewind, | |
486 | .xpi_xprt = xprt_iter_current_entry, | |
487 | .xpi_next = xprt_iter_next_entry_roundrobin, | |
488 | }; | |
489 | ||
490 | /* Policy for once-through iteration of entries in the rpc_xprt_switch */ | |
491 | static | |
492 | const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = { | |
493 | .xpi_rewind = xprt_iter_default_rewind, | |
494 | .xpi_xprt = xprt_iter_current_entry, | |
495 | .xpi_next = xprt_iter_next_entry_all, | |
496 | }; |