projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
SUNRPC: Don't leak sockets in xs_local_connect()
[linux-block.git] / net / sunrpc / xprtsock.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
a246b010
CL		 2/*
3 * linux/net/sunrpc/xprtsock.c
4 *
5 * Client-side transport implementation for sockets.
6 *
113aa838
AC		 7 * TCP callback races fixes (C) 1998 Red Hat
8 * TCP send fixes (C) 1998 Red Hat
a246b010
CL		 9 * TCP NFS related read + write fixes
10 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
11 *
12 * Rewrite of larges part of the code in order to stabilize TCP stuff.
13 * Fix behaviour when socket buffer is full.
14 * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no>
55aa4f58
CL		 15 *
16 * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com>
8f9d5b1a
CL		 17 *
18 * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005.
19 * <gilles.quillard@bull.net>
a246b010
CL		 20 */
21
22#include <linux/types.h>
176e21ee 23#include <linux/string.h>
a246b010 24#include <linux/slab.h>
bc25571e 25#include <linux/module.h>
a246b010 26#include <linux/capability.h>
a246b010
CL		 27#include <linux/pagemap.h>
28#include <linux/errno.h>
29#include <linux/socket.h>
30#include <linux/in.h>
31#include <linux/net.h>
32#include <linux/mm.h>
176e21ee 33#include <linux/un.h>
a246b010
CL		 34#include <linux/udp.h>
35#include <linux/tcp.h>
36#include <linux/sunrpc/clnt.h>
5976687a 37#include <linux/sunrpc/addr.h>
02107148 38#include <linux/sunrpc/sched.h>
4cfc7e60 39#include <linux/sunrpc/svcsock.h>
49c36fcc 40#include <linux/sunrpc/xprtsock.h>
a246b010 41#include <linux/file.h>
9e00abc3 42#ifdef CONFIG_SUNRPC_BACKCHANNEL
44b98efd
RL		 43#include <linux/sunrpc/bc_xprt.h>
44#endif
a246b010
CL		 45
46#include <net/sock.h>
47#include <net/checksum.h>
48#include <net/udp.h>
49#include <net/tcp.h>
277e4ab7 50#include <linux/bvec.h>
6a829eb8 51#include <linux/highmem.h>
277e4ab7 52#include <linux/uio.h>
a1231fda 53#include <linux/sched/mm.h>
a246b010 54
40b5ea0c
TM		 55#include <trace/events/sunrpc.h>
56
9e55eef4 57#include "socklib.h"
4cfc7e60 58#include "sunrpc.h"
176e21ee
CL		 59
60static void xs_close(struct rpc_xprt *xprt);
3b21f757 61static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock);
7196dbb0
TM		 62static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
63 struct socket *sock);
176e21ee 64
c556b754
CL		 65/*
66 * xprtsock tunables
67 */
09acfea5
TM		 68static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE;
69static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE;
70static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE;
c556b754 71
09acfea5
TM		 72static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT;
73static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT;
c556b754 74
7d1e8255 75#define XS_TCP_LINGER_TO (15U * HZ)
25fe6142 76static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;
7d1e8255 77
fbf76683
CL		 78/*
79 * We can register our own files under /proc/sys/sunrpc by
80 * calling register_sysctl_table() again. The files in that
81 * directory become the union of all files registered there.
82 *
83 * We simply need to make sure that we don't collide with
84 * someone else's file names!
85 */
86
fbf76683
CL		 87static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE;
88static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE;
d9ba131d 89static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT;
fbf76683
CL		 90static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT;
91static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT;
92
93static struct ctl_table_header *sunrpc_table_header;
94
d3abc739
OK		 95static struct xprt_class xs_local_transport;
96static struct xprt_class xs_udp_transport;
97static struct xprt_class xs_tcp_transport;
98static struct xprt_class xs_bc_tcp_transport;
99
fbf76683
CL		 100/*
101 * FIXME: changing the UDP slot table size should also resize the UDP
102 * socket buffers for existing UDP transports
103 */
fe2c6338 104static struct ctl_table xs_tunables_table[] = {
fbf76683 105 {
fbf76683
CL		 106 .procname = "udp_slot_table_entries",
107 .data = &xprt_udp_slot_table_entries,
108 .maxlen = sizeof(unsigned int),
109 .mode = 0644,
6d456111 110 .proc_handler = proc_dointvec_minmax,
fbf76683
CL		 111 .extra1 = &min_slot_table_size,
112 .extra2 = &max_slot_table_size
113 },
114 {
fbf76683
CL		 115 .procname = "tcp_slot_table_entries",
116 .data = &xprt_tcp_slot_table_entries,
117 .maxlen = sizeof(unsigned int),
118 .mode = 0644,
6d456111 119 .proc_handler = proc_dointvec_minmax,
fbf76683
CL		 120 .extra1 = &min_slot_table_size,
121 .extra2 = &max_slot_table_size
122 },
d9ba131d
TM		 123 {
124 .procname = "tcp_max_slot_table_entries",
125 .data = &xprt_max_tcp_slot_table_entries,
126 .maxlen = sizeof(unsigned int),
127 .mode = 0644,
128 .proc_handler = proc_dointvec_minmax,
129 .extra1 = &min_slot_table_size,
130 .extra2 = &max_tcp_slot_table_limit
131 },
fbf76683 132 {
fbf76683
CL		 133 .procname = "min_resvport",
134 .data = &xprt_min_resvport,
135 .maxlen = sizeof(unsigned int),
136 .mode = 0644,
6d456111 137 .proc_handler = proc_dointvec_minmax,
fbf76683 138 .extra1 = &xprt_min_resvport_limit,
826799e6 139 .extra2 = &xprt_max_resvport_limit
fbf76683
CL		 140 },
141 {
fbf76683
CL		 142 .procname = "max_resvport",
143 .data = &xprt_max_resvport,
144 .maxlen = sizeof(unsigned int),
145 .mode = 0644,
6d456111 146 .proc_handler = proc_dointvec_minmax,
826799e6 147 .extra1 = &xprt_min_resvport_limit,
fbf76683
CL		 148 .extra2 = &xprt_max_resvport_limit
149 },
25fe6142
TM		 150 {
151 .procname = "tcp_fin_timeout",
152 .data = &xs_tcp_fin_timeout,
153 .maxlen = sizeof(xs_tcp_fin_timeout),
154 .mode = 0644,
6d456111 155 .proc_handler = proc_dointvec_jiffies,
fbf76683 156 },
f8572d8f 157 { },
fbf76683
CL		 158};
159
fe2c6338 160static struct ctl_table sunrpc_table[] = {
fbf76683 161 {
fbf76683
CL		 162 .procname = "sunrpc",
163 .mode = 0555,
164 .child = xs_tunables_table
165 },
f8572d8f 166 { },
fbf76683
CL		 167};
168
03bf4b70
CL		 169/*
170 * Wait duration for a reply from the RPC portmapper.
171 */
172#define XS_BIND_TO (60U * HZ)
173
174/*
175 * Delay if a UDP socket connect error occurs. This is most likely some
176 * kind of resource problem on the local host.
177 */
178#define XS_UDP_REEST_TO (2U * HZ)
179
180/*
181 * The reestablish timeout allows clients to delay for a bit before attempting
182 * to reconnect to a server that just dropped our connection.
183 *
184 * We implement an exponential backoff when trying to reestablish a TCP
185 * transport connection with the server. Some servers like to drop a TCP
186 * connection when they are overworked, so we start with a short timeout and
187 * increase over time if the server is down or not responding.
188 */
189#define XS_TCP_INIT_REEST_TO (3U * HZ)
03bf4b70
CL		 190
191/*
192 * TCP idle timeout; client drops the transport socket if it is idle
193 * for this long. Note that we also timeout UDP sockets to prevent
194 * holding port numbers when there is no RPC traffic.
195 */
196#define XS_IDLE_DISC_TO (5U * 60 * HZ)
197
f895b252 198#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
a246b010 199# undef RPC_DEBUG_DATA
9903cd1c 200# define RPCDBG_FACILITY RPCDBG_TRANS
a246b010
CL		 201#endif
202
a246b010 203#ifdef RPC_DEBUG_DATA
9903cd1c 204static void xs_pktdump(char *msg, u32 *packet, unsigned int count)
a246b010 205{
9903cd1c
CL		 206 u8 *buf = (u8 *) packet;
207 int j;
a246b010 208
46121cf7 209 dprintk("RPC: %s\n", msg);
a246b010
CL		 210 for (j = 0; j < count && j < 128; j += 4) {
211 if (!(j & 31)) {
212 if (j)
213 dprintk("\n");
214 dprintk("0x%04x ", j);
215 }
216 dprintk("%02x%02x%02x%02x ",
217 buf[j], buf[j+1], buf[j+2], buf[j+3]);
218 }
219 dprintk("\n");
220}
221#else
9903cd1c 222static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count)
a246b010
CL		 223{
224 /* NOP */
225}
226#endif
227
2118071d
TM		 228static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)
229{
230 return (struct rpc_xprt *) sk->sk_user_data;
231}
232
95392c59
CL		 233static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt)
234{
235 return (struct sockaddr *) &xprt->addr;
236}
237
176e21ee
CL		 238static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt)
239{
240 return (struct sockaddr_un *) &xprt->addr;
241}
242
95392c59 243static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt)
edb267a6 244{
95392c59
CL		 245 return (struct sockaddr_in *) &xprt->addr;
246}
247
248static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt)
249{
250 return (struct sockaddr_in6 *) &xprt->addr;
251}
252
c877b849 253static void xs_format_common_peer_addresses(struct rpc_xprt *xprt)
edb267a6 254{
c877b849 255 struct sockaddr *sap = xs_addr(xprt);
9dc3b095
CL		 256 struct sockaddr_in6 *sin6;
257 struct sockaddr_in *sin;
176e21ee 258 struct sockaddr_un *sun;
c877b849 259 char buf[128];
edb267a6 260
9dc3b095 261 switch (sap->sa_family) {
176e21ee
CL		 262 case AF_LOCAL:
263 sun = xs_addr_un(xprt);
264 strlcpy(buf, sun->sun_path, sizeof(buf));
265 xprt->address_strings[RPC_DISPLAY_ADDR] =
266 kstrdup(buf, GFP_KERNEL);
267 break;
9dc3b095 268 case AF_INET:
176e21ee
CL		 269 (void)rpc_ntop(sap, buf, sizeof(buf));
270 xprt->address_strings[RPC_DISPLAY_ADDR] =
271 kstrdup(buf, GFP_KERNEL);
9dc3b095 272 sin = xs_addr_in(xprt);
fc0b5791 273 snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));
9dc3b095
CL		 274 break;
275 case AF_INET6:
176e21ee
CL		 276 (void)rpc_ntop(sap, buf, sizeof(buf));
277 xprt->address_strings[RPC_DISPLAY_ADDR] =
278 kstrdup(buf, GFP_KERNEL);
9dc3b095 279 sin6 = xs_addr_in6(xprt);
fc0b5791 280 snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);
9dc3b095
CL		 281 break;
282 default:
283 BUG();
756805e7 284 }
176e21ee 285
9dc3b095 286 xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);
edb267a6
CL		 287}
288
9dc3b095 289static void xs_format_common_peer_ports(struct rpc_xprt *xprt)
4b6473fb 290{
9dc3b095
CL		 291 struct sockaddr *sap = xs_addr(xprt);
292 char buf[128];
4b6473fb 293
81160e66 294 snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap));
c877b849 295 xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);
4b6473fb 296
81160e66 297 snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap));
c877b849
CL		 298 xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
299}
4b6473fb 300
9dc3b095
CL		 301static void xs_format_peer_addresses(struct rpc_xprt *xprt,
302 const char *protocol,
303 const char *netid)
c877b849 304{
b454ae90 305 xprt->address_strings[RPC_DISPLAY_PROTO] = protocol;
b454ae90 306 xprt->address_strings[RPC_DISPLAY_NETID] = netid;
c877b849 307 xs_format_common_peer_addresses(xprt);
9dc3b095 308 xs_format_common_peer_ports(xprt);
edb267a6 309}
4b6473fb 310
9dc3b095 311static void xs_update_peer_port(struct rpc_xprt *xprt)
4b6473fb 312{
9dc3b095
CL		 313 kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
314 kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
4417c8c4 315
9dc3b095 316 xs_format_common_peer_ports(xprt);
edb267a6
CL		 317}
318
319static void xs_free_peer_addresses(struct rpc_xprt *xprt)
320{
33e01dc7
CL		 321 unsigned int i;
322
323 for (i = 0; i < RPC_DISPLAY_MAX; i++)
324 switch (i) {
325 case RPC_DISPLAY_PROTO:
326 case RPC_DISPLAY_NETID:
327 continue;
328 default:
329 kfree(xprt->address_strings[i]);
330 }
edb267a6
CL		 331}
332
277e4ab7
TM		 333static size_t
334xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp)
335{
336 size_t i,n;
337
16e5e90f 338 if (!want || !(buf->flags & XDRBUF_SPARSE_PAGES))
277e4ab7 339 return want;
277e4ab7
TM		 340 n = (buf->page_base + want + PAGE_SIZE - 1) >> PAGE_SHIFT;
341 for (i = 0; i < n; i++) {
342 if (buf->pages[i])
343 continue;
344 buf->bvec[i].bv_page = buf->pages[i] = alloc_page(gfp);
345 if (!buf->pages[i]) {
16e5e90f
TM		 346 i *= PAGE_SIZE;
347 return i > buf->page_base ? i - buf->page_base : 0;
277e4ab7
TM		 348 }
349 }
350 return want;
351}
352
353static ssize_t
354xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek)
355{
356 ssize_t ret;
357 if (seek != 0)
358 iov_iter_advance(&msg->msg_iter, seek);
359 ret = sock_recvmsg(sock, msg, flags);
360 return ret > 0 ? ret + seek : ret;
361}
362
363static ssize_t
364xs_read_kvec(struct socket *sock, struct msghdr *msg, int flags,
365 struct kvec *kvec, size_t count, size_t seek)
366{
0e9b4a82 367 iov_iter_kvec(&msg->msg_iter, READ, kvec, 1, count);
277e4ab7
TM		 368 return xs_sock_recvmsg(sock, msg, flags, seek);
369}
370
371static ssize_t
372xs_read_bvec(struct socket *sock, struct msghdr *msg, int flags,
373 struct bio_vec *bvec, unsigned long nr, size_t count,
374 size_t seek)
375{
0e9b4a82 376 iov_iter_bvec(&msg->msg_iter, READ, bvec, nr, count);
277e4ab7
TM		 377 return xs_sock_recvmsg(sock, msg, flags, seek);
378}
379
380static ssize_t
381xs_read_discard(struct socket *sock, struct msghdr *msg, int flags,
382 size_t count)
383{
b76a5afd
TM		 384 iov_iter_discard(&msg->msg_iter, READ, count);
385 return sock_recvmsg(sock, msg, flags);
277e4ab7
TM		 386}
387
6a829eb8
TM		 388#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
389static void
390xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek)
391{
392 struct bvec_iter bi = {
393 .bi_size = count,
394 };
395 struct bio_vec bv;
396
397 bvec_iter_advance(bvec, &bi, seek & PAGE_MASK);
398 for_each_bvec(bv, bvec, bi, bi)
399 flush_dcache_page(bv.bv_page);
400}
401#else
402static inline void
403xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek)
404{
405}
406#endif
407
277e4ab7
TM		 408static ssize_t
409xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags,
410 struct xdr_buf *buf, size_t count, size_t seek, size_t *read)
411{
412 size_t want, seek_init = seek, offset = 0;
413 ssize_t ret;
414
e92053a5
TM		 415 want = min_t(size_t, count, buf->head[0].iov_len);
416 if (seek < want) {
277e4ab7
TM		 417 ret = xs_read_kvec(sock, msg, flags, &buf->head[0], want, seek);
418 if (ret <= 0)
419 goto sock_err;
420 offset += ret;
421 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
422 goto out;
423 if (ret != want)
c4433055 424 goto out;
277e4ab7
TM		 425 seek = 0;
426 } else {
e92053a5
TM		 427 seek -= want;
428 offset += want;
277e4ab7 429 }
16e5e90f 430
b2648015
TM		 431 want = xs_alloc_sparse_pages(
432 buf, min_t(size_t, count - offset, buf->page_len),
433 GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
16e5e90f 434 if (seek < want) {
277e4ab7
TM		 435 ret = xs_read_bvec(sock, msg, flags, buf->bvec,
436 xdr_buf_pagecount(buf),
437 want + buf->page_base,
438 seek + buf->page_base);
439 if (ret <= 0)
440 goto sock_err;
6a829eb8 441 xs_flush_bvec(buf->bvec, ret, seek + buf->page_base);
ac9645c8
DA		 442 ret -= buf->page_base;
443 offset += ret;
277e4ab7
TM		 444 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
445 goto out;
446 if (ret != want)
c4433055 447 goto out;
277e4ab7
TM		 448 seek = 0;
449 } else {
16e5e90f
TM		 450 seek -= want;
451 offset += want;
277e4ab7 452 }
16e5e90f 453
e92053a5
TM		 454 want = min_t(size_t, count - offset, buf->tail[0].iov_len);
455 if (seek < want) {
277e4ab7
TM		 456 ret = xs_read_kvec(sock, msg, flags, &buf->tail[0], want, seek);
457 if (ret <= 0)
458 goto sock_err;
459 offset += ret;
460 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
461 goto out;
462 if (ret != want)
c4433055 463 goto out;
9734ad57 464 } else if (offset < seek_init)
e92053a5 465 offset = seek_init;
277e4ab7 466 ret = -EMSGSIZE;
277e4ab7
TM		 467out:
468 *read = offset - seek_init;
469 return ret;
277e4ab7
TM		 470sock_err:
471 offset += seek;
472 goto out;
473}
474
475static void
476xs_read_header(struct sock_xprt *transport, struct xdr_buf *buf)
477{
478 if (!transport->recv.copied) {
479 if (buf->head[0].iov_len >= transport->recv.offset)
480 memcpy(buf->head[0].iov_base,
481 &transport->recv.xid,
482 transport->recv.offset);
483 transport->recv.copied = transport->recv.offset;
484 }
485}
486
487static bool
488xs_read_stream_request_done(struct sock_xprt *transport)
489{
490 return transport->recv.fraghdr & cpu_to_be32(RPC_LAST_STREAM_FRAGMENT);
491}
492
e92053a5
TM		 493static void
494xs_read_stream_check_eor(struct sock_xprt *transport,
495 struct msghdr *msg)
496{
497 if (xs_read_stream_request_done(transport))
498 msg->msg_flags |= MSG_EOR;
499}
500
277e4ab7
TM		 501static ssize_t
502xs_read_stream_request(struct sock_xprt *transport, struct msghdr *msg,
503 int flags, struct rpc_rqst *req)
504{
505 struct xdr_buf *buf = &req->rq_private_buf;
3f649ab7
KC		 506 size_t want, read;
507 ssize_t ret;
277e4ab7
TM		 508
509 xs_read_header(transport, buf);
510
511 want = transport->recv.len - transport->recv.offset;
e92053a5
TM		 512 if (want != 0) {
513 ret = xs_read_xdr_buf(transport->sock, msg, flags, buf,
514 transport->recv.copied + want,
515 transport->recv.copied,
516 &read);
517 transport->recv.offset += read;
518 transport->recv.copied += read;
727fcc64 519 }
e92053a5 520
727fcc64 521 if (transport->recv.offset == transport->recv.len)
e92053a5 522 xs_read_stream_check_eor(transport, msg);
277e4ab7 523
e92053a5
TM		 524 if (want == 0)
525 return 0;
526
277e4ab7 527 switch (ret) {
c4433055
TM		 528 default:
529 break;
26781eab 530 case -EFAULT:
277e4ab7 531 case -EMSGSIZE:
26781eab 532 msg->msg_flags |= MSG_TRUNC;
c4433055 533 return read;
277e4ab7
TM		 534 case 0:
535 return -ESHUTDOWN;
277e4ab7 536 }
c4433055 537 return ret < 0 ? ret : read;
277e4ab7
TM		 538}
539
540static size_t
541xs_read_stream_headersize(bool isfrag)
542{
543 if (isfrag)
544 return sizeof(__be32);
545 return 3 * sizeof(__be32);
546}
547
548static ssize_t
549xs_read_stream_header(struct sock_xprt *transport, struct msghdr *msg,
550 int flags, size_t want, size_t seek)
551{
552 struct kvec kvec = {
553 .iov_base = &transport->recv.fraghdr,
554 .iov_len = want,
555 };
556 return xs_read_kvec(transport->sock, msg, flags, &kvec, want, seek);
557}
558
559#if defined(CONFIG_SUNRPC_BACKCHANNEL)
560static ssize_t
561xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
562{
563 struct rpc_xprt *xprt = &transport->xprt;
564 struct rpc_rqst *req;
565 ssize_t ret;
566
98b5cee3
BC		 567 /* Is this transport associated with the backchannel? */
568 if (!xprt->bc_serv)
569 return -ESHUTDOWN;
570
277e4ab7
TM		 571 /* Look up and lock the request corresponding to the given XID */
572 req = xprt_lookup_bc_request(xprt, transport->recv.xid);
573 if (!req) {
574 printk(KERN_WARNING "Callback slot table overflowed\n");
575 return -ESHUTDOWN;
576 }
45835a63
TM		 577 if (transport->recv.copied && !req->rq_private_buf.len)
578 return -ESHUTDOWN;
277e4ab7
TM		 579
580 ret = xs_read_stream_request(transport, msg, flags, req);
581 if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
c4433055 582 xprt_complete_bc_request(req, transport->recv.copied);
45835a63
TM		 583 else
584 req->rq_private_buf.len = transport->recv.copied;
277e4ab7
TM		 585
586 return ret;
587}
588#else /* CONFIG_SUNRPC_BACKCHANNEL */
589static ssize_t
590xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
591{
592 return -ESHUTDOWN;
593}
594#endif /* CONFIG_SUNRPC_BACKCHANNEL */
595
596static ssize_t
597xs_read_stream_reply(struct sock_xprt *transport, struct msghdr *msg, int flags)
598{
599 struct rpc_xprt *xprt = &transport->xprt;
600 struct rpc_rqst *req;
601 ssize_t ret = 0;
602
603 /* Look up and lock the request corresponding to the given XID */
604 spin_lock(&xprt->queue_lock);
605 req = xprt_lookup_rqst(xprt, transport->recv.xid);
45835a63 606 if (!req || (transport->recv.copied && !req->rq_private_buf.len)) {
277e4ab7
TM		 607 msg->msg_flags |= MSG_TRUNC;
608 goto out;
609 }
610 xprt_pin_rqst(req);
611 spin_unlock(&xprt->queue_lock);
612
613 ret = xs_read_stream_request(transport, msg, flags, req);
614
615 spin_lock(&xprt->queue_lock);
616 if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
c4433055 617 xprt_complete_rqst(req->rq_task, transport->recv.copied);
45835a63
TM		 618 else
619 req->rq_private_buf.len = transport->recv.copied;
277e4ab7
TM		 620 xprt_unpin_rqst(req);
621out:
622 spin_unlock(&xprt->queue_lock);
623 return ret;
624}
625
626static ssize_t
627xs_read_stream(struct sock_xprt *transport, int flags)
628{
629 struct msghdr msg = { 0 };
630 size_t want, read = 0;
631 ssize_t ret = 0;
632
633 if (transport->recv.len == 0) {
634 want = xs_read_stream_headersize(transport->recv.copied != 0);
635 ret = xs_read_stream_header(transport, &msg, flags, want,
636 transport->recv.offset);
637 if (ret <= 0)
638 goto out_err;
639 transport->recv.offset = ret;
c4433055
TM		 640 if (transport->recv.offset != want)
641 return transport->recv.offset;
277e4ab7
TM		 642 transport->recv.len = be32_to_cpu(transport->recv.fraghdr) &
643 RPC_FRAGMENT_SIZE_MASK;
644 transport->recv.offset -= sizeof(transport->recv.fraghdr);
645 read = ret;
646 }
647
648 switch (be32_to_cpu(transport->recv.calldir)) {
c4433055
TM		 649 default:
650 msg.msg_flags |= MSG_TRUNC;
651 break;
277e4ab7
TM		 652 case RPC_CALL:
653 ret = xs_read_stream_call(transport, &msg, flags);
654 break;
655 case RPC_REPLY:
656 ret = xs_read_stream_reply(transport, &msg, flags);
657 }
658 if (msg.msg_flags & MSG_TRUNC) {
659 transport->recv.calldir = cpu_to_be32(-1);
660 transport->recv.copied = -1;
661 }
662 if (ret < 0)
663 goto out_err;
664 read += ret;
665 if (transport->recv.offset < transport->recv.len) {
c4433055
TM		 666 if (!(msg.msg_flags & MSG_TRUNC))
667 return read;
b76a5afd 668 msg.msg_flags = 0;
277e4ab7
TM		 669 ret = xs_read_discard(transport->sock, &msg, flags,
670 transport->recv.len - transport->recv.offset);
671 if (ret <= 0)
672 goto out_err;
673 transport->recv.offset += ret;
674 read += ret;
675 if (transport->recv.offset != transport->recv.len)
c4433055 676 return read;
277e4ab7
TM		 677 }
678 if (xs_read_stream_request_done(transport)) {
c50b8ee0 679 trace_xs_stream_read_request(transport);
277e4ab7
TM		 680 transport->recv.copied = 0;
681 }
682 transport->recv.offset = 0;
683 transport->recv.len = 0;
684 return read;
685out_err:
79462857 686 return ret != 0 ? ret : -ESHUTDOWN;
277e4ab7
TM		 687}
688
0ffe86f4
TM		 689static __poll_t xs_poll_socket(struct sock_xprt *transport)
690{
a73881c9
TM		 691 return transport->sock->ops->poll(transport->file, transport->sock,
692 NULL);
0ffe86f4
TM		 693}
694
695static bool xs_poll_socket_readable(struct sock_xprt *transport)
696{
697 __poll_t events = xs_poll_socket(transport);
698
699 return (events & (EPOLLIN | EPOLLRDNORM)) && !(events & EPOLLRDHUP);
700}
701
702static void xs_poll_check_readable(struct sock_xprt *transport)
703{
704
705 clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
706 if (!xs_poll_socket_readable(transport))
707 return;
708 if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
709 queue_work(xprtiod_workqueue, &transport->recv_worker);
710}
711
c50b8ee0
TM		 712static void xs_stream_data_receive(struct sock_xprt *transport)
713{
714 size_t read = 0;
715 ssize_t ret = 0;
716
717 mutex_lock(&transport->recv_mutex);
718 if (transport->sock == NULL)
719 goto out;
c50b8ee0
TM		 720 for (;;) {
721 ret = xs_read_stream(transport, MSG_DONTWAIT);
c4433055 722 if (ret < 0)
c50b8ee0
TM		 723 break;
724 read += ret;
725 cond_resched();
726 }
5f52a9d4
TM		 727 if (ret == -ESHUTDOWN)
728 kernel_sock_shutdown(transport->sock, SHUT_RDWR);
729 else
730 xs_poll_check_readable(transport);
c50b8ee0
TM		 731out:
732 mutex_unlock(&transport->recv_mutex);
733 trace_xs_stream_read_data(&transport->xprt, ret, read);
734}
735
736static void xs_stream_data_receive_workfn(struct work_struct *work)
737{
738 struct sock_xprt *transport =
739 container_of(work, struct sock_xprt, recv_worker);
a1231fda
TM		 740 unsigned int pflags = memalloc_nofs_save();
741
c50b8ee0 742 xs_stream_data_receive(transport);
a1231fda 743 memalloc_nofs_restore(pflags);
c50b8ee0
TM		 744}
745
550aebfe
TM		 746static void
747xs_stream_reset_connect(struct sock_xprt *transport)
748{
749 transport->recv.offset = 0;
750 transport->recv.len = 0;
751 transport->recv.copied = 0;
752 transport->xmit.offset = 0;
ae053551
TM		 753}
754
755static void
756xs_stream_start_connect(struct sock_xprt *transport)
757{
550aebfe
TM		 758 transport->xprt.stat.connect_count++;
759 transport->xprt.stat.connect_start = jiffies;
760}
761
b4b5cc85
CL		 762#define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
763
9903cd1c 764/**
c544577d 765 * xs_nospace - handle transmit was incomplete
50f484e2 766 * @req: pointer to RPC request
7496b59f 767 * @transport: pointer to struct sock_xprt
9903cd1c 768 *
a246b010 769 */
7496b59f 770static int xs_nospace(struct rpc_rqst *req, struct sock_xprt *transport)
a246b010 771{
7496b59f 772 struct rpc_xprt *xprt = &transport->xprt;
06ea0bfe 773 struct sock *sk = transport->inet;
24ca9a84 774 int ret = -EAGAIN;
a246b010 775
015747d2 776 trace_rpc_socket_nospace(req, transport);
262965f5 777
b6ddf64f 778 /* Protect against races with write_space */
b5e92419 779 spin_lock(&xprt->transport_lock);
b6ddf64f
TM		 780
781 /* Don't race with disconnect */
782 if (xprt_connected(xprt)) {
13331a55 783 /* wait for more buffer space */
2790a624 784 set_bit(XPRT_SOCK_NOSPACE, &transport->sock_state);
7496b59f 785 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
13331a55 786 sk->sk_write_pending++;
c544577d 787 xprt_wait_for_buffer_space(xprt);
13331a55 788 } else
5e3771ce 789 ret = -ENOTCONN;
262965f5 790
b5e92419 791 spin_unlock(&xprt->transport_lock);
7496b59f
TM		 792 return ret;
793}
06ea0bfe 794
7496b59f
TM		 795static int xs_sock_nospace(struct rpc_rqst *req)
796{
797 struct sock_xprt *transport =
798 container_of(req->rq_xprt, struct sock_xprt, xprt);
799 struct sock *sk = transport->inet;
800 int ret = -EAGAIN;
d48f9ce7 801
7496b59f
TM		 802 lock_sock(sk);
803 if (!sock_writeable(sk))
804 ret = xs_nospace(req, transport);
805 release_sock(sk);
806 return ret;
807}
d48f9ce7 808
d0afde5f 809static int xs_stream_nospace(struct rpc_rqst *req, bool vm_wait)
7496b59f
TM		 810{
811 struct sock_xprt *transport =
812 container_of(req->rq_xprt, struct sock_xprt, xprt);
813 struct sock *sk = transport->inet;
814 int ret = -EAGAIN;
815
d0afde5f
TM		 816 if (vm_wait)
817 return -ENOBUFS;
7496b59f
TM		 818 lock_sock(sk);
819 if (!sk_stream_memory_free(sk))
820 ret = xs_nospace(req, transport);
821 release_sock(sk);
5e3771ce 822 return ret;
262965f5
CL		 823}
824
eb07d5a4 825static int
277e4ab7
TM		 826xs_stream_prepare_request(struct rpc_rqst *req)
827{
ff053dbb
TM		 828 gfp_t gfp = rpc_task_gfp_mask();
829 int ret;
830
831 ret = xdr_alloc_bvec(&req->rq_snd_buf, gfp);
832 if (ret < 0)
833 return ret;
75369089 834 xdr_free_bvec(&req->rq_rcv_buf);
ff053dbb 835 return xdr_alloc_bvec(&req->rq_rcv_buf, gfp);
277e4ab7
TM		 836}
837
4cd34e7c
TM		 838/*
839 * Determine if the previous message in the stream was aborted before it
840 * could complete transmission.
841 */
842static bool
843xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req)
844{
845 return transport->xmit.offset != 0 && req->rq_bytes_sent == 0;
846}
847
61677eee 848/*
06b5fc3a 849 * Return the stream record marker field for a record of length < 2^31-1
61677eee 850 */
06b5fc3a
TM		 851static rpc_fraghdr
852xs_stream_record_marker(struct xdr_buf *xdr)
61677eee 853{
06b5fc3a
TM		 854 if (!xdr->len)
855 return 0;
856 return cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | (u32)xdr->len);
61677eee
CL		 857}
858
176e21ee
CL		 859/**
860 * xs_local_send_request - write an RPC request to an AF_LOCAL socket
50f484e2 861 * @req: pointer to RPC request
176e21ee
CL		 862 *
863 * Return values:
864 * 0: The request has been sent
865 * EAGAIN: The socket was blocked, please call again later to
866 * complete the request
867 * ENOTCONN: Caller needs to invoke connect logic then call again
12b20ce3 868 * other: Some other error occurred, the request was not sent
176e21ee 869 */
adfa7144 870static int xs_local_send_request(struct rpc_rqst *req)
176e21ee 871{
176e21ee
CL		 872 struct rpc_xprt *xprt = req->rq_xprt;
873 struct sock_xprt *transport =
874 container_of(xprt, struct sock_xprt, xprt);
875 struct xdr_buf *xdr = &req->rq_snd_buf;
7e3d3620
TM		 876 rpc_fraghdr rm = xs_stream_record_marker(xdr);
877 unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen;
9e55eef4
CL		 878 struct msghdr msg = {
879 .msg_flags = XS_SENDMSG_FLAGS,
880 };
d0afde5f 881 bool vm_wait;
3f649ab7 882 unsigned int sent;
176e21ee
CL		 883 int status;
884
4cd34e7c
TM		 885 /* Close the stream if the previous transmission was incomplete */
886 if (xs_send_request_was_aborted(transport, req)) {
f0043206 887 xprt_force_disconnect(xprt);
4cd34e7c
TM		 888 return -ENOTCONN;
889 }
890
176e21ee
CL		 891 xs_pktdump("packet data:",
892 req->rq_svec->iov_base, req->rq_svec->iov_len);
893
d0afde5f
TM		 894 vm_wait = sk_stream_is_writeable(transport->inet) ? true : false;
895
78215759 896 req->rq_xtime = ktime_get();
9e55eef4
CL		 897 status = xprt_sock_sendmsg(transport->sock, &msg, xdr,
898 transport->xmit.offset, rm, &sent);
176e21ee 899 dprintk("RPC: %s(%u) = %d\n",
6c7a64e5 900 __func__, xdr->len - transport->xmit.offset, status);
743c69e7 901
f279cd00 902 if (likely(sent > 0) || status == 0) {
6c7a64e5
TM		 903 transport->xmit.offset += sent;
904 req->rq_bytes_sent = transport->xmit.offset;
7e3d3620 905 if (likely(req->rq_bytes_sent >= msglen)) {
6c7a64e5 906 req->rq_xmit_bytes_sent += transport->xmit.offset;
6c7a64e5 907 transport->xmit.offset = 0;
176e21ee
CL		 908 return 0;
909 }
910 status = -EAGAIN;
d0afde5f 911 vm_wait = false;
176e21ee
CL		 912 }
913
914 switch (status) {
915 case -EAGAIN:
d0afde5f 916 status = xs_stream_nospace(req, vm_wait);
176e21ee
CL		 917 break;
918 default:
919 dprintk("RPC: sendmsg returned unrecognized error %d\n",
920 -status);
df561f66 921 fallthrough;
176e21ee 922 case -EPIPE:
f0043206 923 xprt_force_disconnect(xprt);
176e21ee
CL		 924 status = -ENOTCONN;
925 }
926
927 return status;
928}
929
262965f5
CL		 930/**
931 * xs_udp_send_request - write an RPC request to a UDP socket
50f484e2 932 * @req: pointer to RPC request
262965f5
CL		 933 *
934 * Return values:
935 * 0: The request has been sent
936 * EAGAIN: The socket was blocked, please call again later to
937 * complete the request
938 * ENOTCONN: Caller needs to invoke connect logic then call again
25985edc 939 * other: Some other error occurred, the request was not sent
262965f5 940 */
adfa7144 941static int xs_udp_send_request(struct rpc_rqst *req)
262965f5 942{
262965f5 943 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 944 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262965f5 945 struct xdr_buf *xdr = &req->rq_snd_buf;
9e55eef4
CL		 946 struct msghdr msg = {
947 .msg_name = xs_addr(xprt),
948 .msg_namelen = xprt->addrlen,
949 .msg_flags = XS_SENDMSG_FLAGS,
950 };
3f649ab7 951 unsigned int sent;
262965f5 952 int status;
a246b010 953
9903cd1c 954 xs_pktdump("packet data:",
a246b010
CL		 955 req->rq_svec->iov_base,
956 req->rq_svec->iov_len);
957
01d37c42
TM		 958 if (!xprt_bound(xprt))
959 return -ENOTCONN;
75891f50
TM		 960
961 if (!xprt_request_get_cong(xprt, req))
962 return -EBADSLT;
963
ff053dbb
TM		 964 status = xdr_alloc_bvec(xdr, rpc_task_gfp_mask());
965 if (status < 0)
966 return status;
78215759 967 req->rq_xtime = ktime_get();
9e55eef4 968 status = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, 0, &sent);
a246b010 969
46121cf7 970 dprintk("RPC: xs_udp_send_request(%u) = %d\n",
6c7a64e5 971 xdr->len, status);
a246b010 972
3dedbb5c
JB		 973 /* firewall is blocking us, don't return -EAGAIN or we end up looping */
974 if (status == -EPERM)
975 goto process_status;
976
743c69e7
N		 977 if (status == -EAGAIN && sock_writeable(transport->inet))
978 status = -ENOBUFS;
979
f279cd00
JB		 980 if (sent > 0 || status == 0) {
981 req->rq_xmit_bytes_sent += sent;
982 if (sent >= req->rq_slen)
2199700f
TM		 983 return 0;
984 /* Still some bytes left; set up for a retry later. */
262965f5 985 status = -EAGAIN;
2199700f 986 }
a246b010 987
3dedbb5c 988process_status:
262965f5 989 switch (status) {
fba91afb
TM		 990 case -ENOTSOCK:
991 status = -ENOTCONN;
992 /* Should we call xs_close() here? */
993 break;
b6ddf64f 994 case -EAGAIN:
7496b59f 995 status = xs_sock_nospace(req);
b6ddf64f 996 break;
262965f5 997 case -ENETUNREACH:
3601c4a9 998 case -ENOBUFS:
262965f5 999 case -EPIPE:
a246b010 1000 case -ECONNREFUSED:
3dedbb5c 1001 case -EPERM:
a246b010 1002 /* When the server has died, an ICMP port unreachable message
9903cd1c 1003 * prompts ECONNREFUSED. */
13331a55
TM		 1004 break;
1005 default:
1006 dprintk("RPC: sendmsg returned unrecognized error %d\n",
1007 -status);
a246b010 1008 }
5fe46e9d 1009
262965f5 1010 return status;
a246b010
CL		 1011}
1012
9903cd1c 1013/**
262965f5 1014 * xs_tcp_send_request - write an RPC request to a TCP socket
50f484e2 1015 * @req: pointer to RPC request
9903cd1c
CL		 1016 *
1017 * Return values:
262965f5
CL		 1018 * 0: The request has been sent
1019 * EAGAIN: The socket was blocked, please call again later to
1020 * complete the request
1021 * ENOTCONN: Caller needs to invoke connect logic then call again
25985edc 1022 * other: Some other error occurred, the request was not sent
9903cd1c
CL		 1023 *
1024 * XXX: In the case of soft timeouts, should we eventually give up
262965f5 1025 * if sendmsg is not able to make progress?
9903cd1c 1026 */
adfa7144 1027static int xs_tcp_send_request(struct rpc_rqst *req)
a246b010 1028{
a246b010 1029 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 1030 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262965f5 1031 struct xdr_buf *xdr = &req->rq_snd_buf;
7e3d3620
TM		 1032 rpc_fraghdr rm = xs_stream_record_marker(xdr);
1033 unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen;
9e55eef4
CL		 1034 struct msghdr msg = {
1035 .msg_flags = XS_SENDMSG_FLAGS,
1036 };
d0afde5f 1037 bool vm_wait;
3f649ab7 1038 unsigned int sent;
b595bb15 1039 int status;
a246b010 1040
4cd34e7c
TM		 1041 /* Close the stream if the previous transmission was incomplete */
1042 if (xs_send_request_was_aborted(transport, req)) {
1043 if (transport->sock != NULL)
1044 kernel_sock_shutdown(transport->sock, SHUT_RDWR);
1045 return -ENOTCONN;
1046 }
d275880a
TM		 1047 if (!transport->inet)
1048 return -ENOTCONN;
4cd34e7c 1049
262965f5
CL		 1050 xs_pktdump("packet data:",
1051 req->rq_svec->iov_base,
1052 req->rq_svec->iov_len);
a246b010 1053
7196dbb0
TM		 1054 if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state))
1055 xs_tcp_set_socket_timeouts(xprt, transport->sock);
1056
3b21f757
TM		 1057 xs_set_srcport(transport, transport->sock);
1058
a246b010
CL		 1059 /* Continue transmitting the packet/record. We must be careful
1060 * to cope with writespace callbacks arriving _after_ we have
262965f5 1061 * called sendmsg(). */
78215759 1062 req->rq_xtime = ktime_get();
d737e5d4 1063 tcp_sock_set_cork(transport->inet, true);
d0afde5f
TM		 1064
1065 vm_wait = sk_stream_is_writeable(transport->inet) ? true : false;
1066
1067 do {
9e55eef4
CL		 1068 status = xprt_sock_sendmsg(transport->sock, &msg, xdr,
1069 transport->xmit.offset, rm, &sent);
a246b010 1070
46121cf7 1071 dprintk("RPC: xs_tcp_send_request(%u) = %d\n",
6c7a64e5 1072 xdr->len - transport->xmit.offset, status);
a246b010 1073
262965f5
CL		 1074 /* If we've sent the entire packet, immediately
1075 * reset the count of bytes sent. */
6c7a64e5
TM		 1076 transport->xmit.offset += sent;
1077 req->rq_bytes_sent = transport->xmit.offset;
7e3d3620 1078 if (likely(req->rq_bytes_sent >= msglen)) {
6c7a64e5 1079 req->rq_xmit_bytes_sent += transport->xmit.offset;
6c7a64e5 1080 transport->xmit.offset = 0;
d737e5d4
TM		 1081 if (atomic_long_read(&xprt->xmit_queuelen) == 1)
1082 tcp_sock_set_cork(transport->inet, false);
262965f5
CL		 1083 return 0;
1084 }
a246b010 1085
9ffadfbc
TM		 1086 WARN_ON_ONCE(sent == 0 && status == 0);
1087
d0afde5f
TM		 1088 if (sent > 0)
1089 vm_wait = false;
1090
1091 } while (status == 0);
a246b010 1092
262965f5 1093 switch (status) {
fba91afb
TM		 1094 case -ENOTSOCK:
1095 status = -ENOTCONN;
1096 /* Should we call xs_close() here? */
1097 break;
262965f5 1098 case -EAGAIN:
d0afde5f 1099 status = xs_stream_nospace(req, vm_wait);
262965f5 1100 break;
262965f5 1101 case -ECONNRESET:
2a9e1cfa 1102 case -ECONNREFUSED:
262965f5 1103 case -ENOTCONN:
3913c78c 1104 case -EADDRINUSE:
b5872f0c 1105 case -ENOBUFS:
b9d2bb2e 1106 case -EPIPE:
13331a55
TM		 1107 break;
1108 default:
1109 dprintk("RPC: sendmsg returned unrecognized error %d\n",
1110 -status);
a246b010 1111 }
5fe46e9d 1112
a246b010
CL		 1113 return status;
1114}
1115
2a9e1cfa
TM		 1116static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)
1117{
1118 transport->old_data_ready = sk->sk_data_ready;
1119 transport->old_state_change = sk->sk_state_change;
1120 transport->old_write_space = sk->sk_write_space;
2118071d 1121 transport->old_error_report = sk->sk_error_report;
2a9e1cfa
TM		 1122}
1123
1124static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)
1125{
1126 sk->sk_data_ready = transport->old_data_ready;
1127 sk->sk_state_change = transport->old_state_change;
1128 sk->sk_write_space = transport->old_write_space;
2118071d
TM		 1129 sk->sk_error_report = transport->old_error_report;
1130}
1131
42d42a5b
TM		 1132static void xs_sock_reset_state_flags(struct rpc_xprt *xprt)
1133{
1134 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1135
1136 clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
4f8943f8
TM		 1137 clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state);
1138 clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state);
1139 clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state);
2790a624 1140 clear_bit(XPRT_SOCK_NOSPACE, &transport->sock_state);
4f8943f8
TM		 1141}
1142
1143static void xs_run_error_worker(struct sock_xprt *transport, unsigned int nr)
1144{
1145 set_bit(nr, &transport->sock_state);
1146 queue_work(xprtiod_workqueue, &transport->error_worker);
42d42a5b
TM		 1147}
1148
b70ae915
TM		 1149static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt)
1150{
d896ba83 1151 xprt->connect_cookie++;
b70ae915
TM		 1152 smp_mb__before_atomic();
1153 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
1154 clear_bit(XPRT_CLOSING, &xprt->state);
42d42a5b 1155 xs_sock_reset_state_flags(xprt);
b70ae915
TM		 1156 smp_mb__after_atomic();
1157}
1158
2118071d
TM		 1159/**
1160 * xs_error_report - callback to handle TCP socket state errors
1161 * @sk: socket
1162 *
1163 * Note: we don't call sock_error() since there may be a rpc_task
1164 * using the socket, and so we don't want to clear sk->sk_err.
1165 */
1166static void xs_error_report(struct sock *sk)
1167{
4f8943f8 1168 struct sock_xprt *transport;
2118071d 1169 struct rpc_xprt *xprt;
2118071d 1170
2118071d 1171 if (!(xprt = xprt_from_sock(sk)))
ea9afca8 1172 return;
2118071d 1173
4f8943f8 1174 transport = container_of(xprt, struct sock_xprt, xprt);
af84537d
BC		 1175 transport->xprt_err = -sk->sk_err;
1176 if (transport->xprt_err == 0)
ea9afca8 1177 return;
2118071d 1178 dprintk("RPC: xs_error_report client %p, error=%d...\n",
af84537d
BC		 1179 xprt, -transport->xprt_err);
1180 trace_rpc_socket_error(xprt, sk->sk_socket, transport->xprt_err);
1181
1182 /* barrier ensures xprt_err is set before XPRT_SOCK_WAKE_ERROR */
1183 smp_mb__before_atomic();
4f8943f8 1184 xs_run_error_worker(transport, XPRT_SOCK_WAKE_ERROR);
2a9e1cfa
TM		 1185}
1186
fe315e76 1187static void xs_reset_transport(struct sock_xprt *transport)
a246b010 1188{
ee0ac0c2
CL		 1189 struct socket *sock = transport->sock;
1190 struct sock *sk = transport->inet;
6cc7e908 1191 struct rpc_xprt *xprt = &transport->xprt;
a73881c9 1192 struct file *filp = transport->file;
a246b010 1193
fe315e76
CL		 1194 if (sk == NULL)
1195 return;
f0043206
TM		 1196 /*
1197 * Make sure we're calling this in a context from which it is safe
1198 * to call __fput_sync(). In practice that means rpciod and the
1199 * system workqueue.
1200 */
1201 if (!(current->flags & PF_WQ_WORKER)) {
1202 WARN_ON_ONCE(1);
1203 set_bit(XPRT_CLOSE_WAIT, &xprt->state);
1204 return;
1205 }
9903cd1c 1206
264d1df3
JL		 1207 if (atomic_read(&transport->xprt.swapper))
1208 sk_clear_memalloc(sk);
1209
09939204
TM		 1210 kernel_sock_shutdown(sock, SHUT_RDWR);
1211
edc1b01c 1212 mutex_lock(&transport->recv_mutex);
ea9afca8 1213 lock_sock(sk);
ee0ac0c2
CL		 1214 transport->inet = NULL;
1215 transport->sock = NULL;
a73881c9 1216 transport->file = NULL;
a246b010 1217
9903cd1c 1218 sk->sk_user_data = NULL;
2a9e1cfa
TM		 1219
1220 xs_restore_old_callbacks(transport, sk);
0c78789e 1221 xprt_clear_connected(xprt);
6cc7e908 1222 xs_sock_reset_connection_flags(xprt);
ae053551
TM		 1223 /* Reset stream record info */
1224 xs_stream_reset_connect(transport);
ea9afca8 1225 release_sock(sk);
edc1b01c 1226 mutex_unlock(&transport->recv_mutex);
a246b010 1227
6cc7e908 1228 trace_rpc_socket_close(xprt, sock);
f0043206 1229 __fput_sync(filp);
0445f92c
TM		 1230
1231 xprt_disconnect_done(xprt);
fe315e76
CL		 1232}
1233
1234/**
1235 * xs_close - close a socket
1236 * @xprt: transport
1237 *
1238 * This is used when all requests are complete; ie, no DRC state remains
1239 * on the server we want to save.
f75e6745
TM		 1240 *
1241 * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with
1242 * xs_reset_transport() zeroing the socket from underneath a writer.
fe315e76
CL		 1243 */
1244static void xs_close(struct rpc_xprt *xprt)
1245{
1246 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1247
1248 dprintk("RPC: xs_close xprt %p\n", xprt);
1249
1250 xs_reset_transport(transport);
61d0a8e6 1251 xprt->reestablish_timeout = 0;
a246b010
CL		 1252}
1253
4a068258
CL		 1254static void xs_inject_disconnect(struct rpc_xprt *xprt)
1255{
1256 dprintk("RPC: injecting transport disconnect on xprt=%p\n",
1257 xprt);
1258 xprt_disconnect_done(xprt);
1259}
1260
315f3812
KM		 1261static void xs_xprt_free(struct rpc_xprt *xprt)
1262{
1263 xs_free_peer_addresses(xprt);
1264 xprt_free(xprt);
1265}
1266
9903cd1c
CL		 1267/**
1268 * xs_destroy - prepare to shutdown a transport
1269 * @xprt: doomed transport
1270 *
1271 */
1272static void xs_destroy(struct rpc_xprt *xprt)
a246b010 1273{
03c78827
TM		 1274 struct sock_xprt *transport = container_of(xprt,
1275 struct sock_xprt, xprt);
46121cf7 1276 dprintk("RPC: xs_destroy xprt %p\n", xprt);
9903cd1c 1277
03c78827 1278 cancel_delayed_work_sync(&transport->connect_worker);
a1311d87 1279 xs_close(xprt);
edc1b01c 1280 cancel_work_sync(&transport->recv_worker);
b5e92419 1281 cancel_work_sync(&transport->error_worker);
315f3812 1282 xs_xprt_free(xprt);
a1311d87 1283 module_put(THIS_MODULE);
a246b010
CL		 1284}
1285
9903cd1c 1286/**
f9b2ee71
TM		 1287 * xs_udp_data_read_skb - receive callback for UDP sockets
1288 * @xprt: transport
1289 * @sk: socket
1290 * @skb: skbuff
9903cd1c 1291 *
a246b010 1292 */
f9b2ee71
TM		 1293static void xs_udp_data_read_skb(struct rpc_xprt *xprt,
1294 struct sock *sk,
1295 struct sk_buff *skb)
a246b010 1296{
9903cd1c 1297 struct rpc_task *task;
a246b010 1298 struct rpc_rqst *rovr;
f9b2ee71 1299 int repsize, copied;
d8ed029d
AD		 1300 u32 _xid;
1301 __be32 *xp;
a246b010 1302
1da8c681 1303 repsize = skb->len;
a246b010 1304 if (repsize < 4) {
46121cf7 1305 dprintk("RPC: impossible RPC reply size %d!\n", repsize);
f9b2ee71 1306 return;
a246b010
CL		 1307 }
1308
1309 /* Copy the XID from the skb... */
1da8c681 1310 xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid);
a246b010 1311 if (xp == NULL)
f9b2ee71 1312 return;
a246b010
CL		 1313
1314 /* Look up and lock the request corresponding to the given XID */
75c84151 1315 spin_lock(&xprt->queue_lock);
a246b010
CL		 1316 rovr = xprt_lookup_rqst(xprt, *xp);
1317 if (!rovr)
1318 goto out_unlock;
729749bb 1319 xprt_pin_rqst(rovr);
ecd465ee 1320 xprt_update_rtt(rovr->rq_task);
75c84151 1321 spin_unlock(&xprt->queue_lock);
a246b010
CL		 1322 task = rovr->rq_task;
1323
a246b010
CL		 1324 if ((copied = rovr->rq_private_buf.buflen) > repsize)
1325 copied = repsize;
1326
1327 /* Suck it into the iovec, verify checksum if not done by hw. */
1781f7f5 1328 if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) {
75c84151 1329 spin_lock(&xprt->queue_lock);
0afa6b44 1330 __UDPX_INC_STATS(sk, UDP_MIB_INERRORS);
729749bb 1331 goto out_unpin;
1781f7f5
HX		 1332 }
1333
a246b010 1334
b5e92419 1335 spin_lock(&xprt->transport_lock);
6a24dfb6 1336 xprt_adjust_cwnd(xprt, task, copied);
b5e92419 1337 spin_unlock(&xprt->transport_lock);
75c84151 1338 spin_lock(&xprt->queue_lock);
1570c1e4 1339 xprt_complete_rqst(task, copied);
0afa6b44 1340 __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS);
729749bb
TM		 1341out_unpin:
1342 xprt_unpin_rqst(rovr);
a246b010 1343 out_unlock:
75c84151 1344 spin_unlock(&xprt->queue_lock);
f9b2ee71
TM		 1345}
1346
1347static void xs_udp_data_receive(struct sock_xprt *transport)
1348{
1349 struct sk_buff *skb;
1350 struct sock *sk;
1351 int err;
1352
1353 mutex_lock(&transport->recv_mutex);
1354 sk = transport->inet;
1355 if (sk == NULL)
1356 goto out;
1357 for (;;) {
7c13f97f 1358 skb = skb_recv_udp(sk, 0, 1, &err);
4f546149 1359 if (skb == NULL)
f9b2ee71 1360 break;
4f546149
TM		 1361 xs_udp_data_read_skb(&transport->xprt, sk, skb);
1362 consume_skb(skb);
1363 cond_resched();
f9b2ee71 1364 }
0ffe86f4 1365 xs_poll_check_readable(transport);
f9b2ee71
TM		 1366out:
1367 mutex_unlock(&transport->recv_mutex);
1368}
1369
1370static void xs_udp_data_receive_workfn(struct work_struct *work)
1371{
1372 struct sock_xprt *transport =
1373 container_of(work, struct sock_xprt, recv_worker);
a1231fda
TM		 1374 unsigned int pflags = memalloc_nofs_save();
1375
f9b2ee71 1376 xs_udp_data_receive(transport);
a1231fda 1377 memalloc_nofs_restore(pflags);
f9b2ee71
TM		 1378}
1379
1380/**
1381 * xs_data_ready - "data ready" callback for UDP sockets
1382 * @sk: socket with data to read
1383 *
1384 */
1385static void xs_data_ready(struct sock *sk)
1386{
1387 struct rpc_xprt *xprt;
1388
f9b2ee71
TM		 1389 dprintk("RPC: xs_data_ready...\n");
1390 xprt = xprt_from_sock(sk);
1391 if (xprt != NULL) {
1392 struct sock_xprt *transport = container_of(xprt,
1393 struct sock_xprt, xprt);
5157b956
TM		 1394 transport->old_data_ready(sk);
1395 /* Any data means we had a useful conversation, so
1396 * then we don't need to delay the next reconnect
1397 */
1398 if (xprt->reestablish_timeout)
1399 xprt->reestablish_timeout = 0;
42d42a5b 1400 if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
40a5f1b1 1401 queue_work(xprtiod_workqueue, &transport->recv_worker);
f9b2ee71 1402 }
a246b010
CL		 1403}
1404
a519fc7a
TM		 1405/*
1406 * Helper function to force a TCP close if the server is sending
1407 * junk and/or it has put us in CLOSE_WAIT
1408 */
1409static void xs_tcp_force_close(struct rpc_xprt *xprt)
1410{
a519fc7a
TM		 1411 xprt_force_disconnect(xprt);
1412}
1413
9e00abc3 1414#if defined(CONFIG_SUNRPC_BACKCHANNEL)
6b26cc8c
CL		 1415static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt)
1416{
1417 return PAGE_SIZE;
1418}
9e00abc3 1419#endif /* CONFIG_SUNRPC_BACKCHANNEL */
44b98efd 1420
9903cd1c
CL		 1421/**
1422 * xs_tcp_state_change - callback to handle TCP socket state changes
1423 * @sk: socket whose state has changed
1424 *
1425 */
1426static void xs_tcp_state_change(struct sock *sk)
a246b010 1427{
9903cd1c 1428 struct rpc_xprt *xprt;
0fdea1e8 1429 struct sock_xprt *transport;
a246b010 1430
a246b010 1431 if (!(xprt = xprt_from_sock(sk)))
ea9afca8 1432 return;
46121cf7 1433 dprintk("RPC: xs_tcp_state_change client %p...\n", xprt);
669502ff 1434 dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n",
46121cf7
CL		 1435 sk->sk_state, xprt_connected(xprt),
1436 sock_flag(sk, SOCK_DEAD),
669502ff
AC		 1437 sock_flag(sk, SOCK_ZAPPED),
1438 sk->sk_shutdown);
a246b010 1439
0fdea1e8 1440 transport = container_of(xprt, struct sock_xprt, xprt);
40b5ea0c 1441 trace_rpc_socket_state_change(xprt, sk->sk_socket);
a246b010
CL		 1442 switch (sk->sk_state) {
1443 case TCP_ESTABLISHED:
a246b010 1444 if (!xprt_test_and_set_connected(xprt)) {
8b71798c 1445 xprt->connect_cookie++;
0fdea1e8
TM		 1446 clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
1447 xprt_clear_connecting(xprt);
51971139 1448
3968a8a5
CL		 1449 xprt->stat.connect_count++;
1450 xprt->stat.connect_time += (long)jiffies -
1451 xprt->stat.connect_start;
4f8943f8 1452 xs_run_error_worker(transport, XPRT_SOCK_WAKE_PENDING);
a246b010 1453 }
a246b010 1454 break;
3b948ae5
TM		 1455 case TCP_FIN_WAIT1:
1456 /* The client initiated a shutdown of the socket */
7c1d71cf 1457 xprt->connect_cookie++;
663b8858 1458 xprt->reestablish_timeout = 0;
3b948ae5 1459 set_bit(XPRT_CLOSING, &xprt->state);
4e857c58 1460 smp_mb__before_atomic();
3b948ae5 1461 clear_bit(XPRT_CONNECTED, &xprt->state);
ef803670 1462 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
4e857c58 1463 smp_mb__after_atomic();
a246b010 1464 break;
632e3bdc 1465 case TCP_CLOSE_WAIT:
3b948ae5 1466 /* The server initiated a shutdown of the socket */
7c1d71cf 1467 xprt->connect_cookie++;
d0bea455 1468 clear_bit(XPRT_CONNECTED, &xprt->state);
4f8943f8 1469 xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT);
df561f66 1470 fallthrough;
663b8858
TM		 1471 case TCP_CLOSING:
1472 /*
1473 * If the server closed down the connection, make sure that
1474 * we back off before reconnecting
1475 */
1476 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
1477 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
3b948ae5
TM		 1478 break;
1479 case TCP_LAST_ACK:
670f9457 1480 set_bit(XPRT_CLOSING, &xprt->state);
4e857c58 1481 smp_mb__before_atomic();
3b948ae5 1482 clear_bit(XPRT_CONNECTED, &xprt->state);
4e857c58 1483 smp_mb__after_atomic();
3b948ae5
TM		 1484 break;
1485 case TCP_CLOSE:
0fdea1e8
TM		 1486 if (test_and_clear_bit(XPRT_SOCK_CONNECTING,
1487 &transport->sock_state))
1488 xprt_clear_connecting(xprt);
9b30889c 1489 clear_bit(XPRT_CLOSING, &xprt->state);
9b30889c 1490 /* Trigger the socket release */
4f8943f8 1491 xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT);
a246b010 1492 }
a246b010
CL		 1493}
1494
1f0fa154
IJ		 1495static void xs_write_space(struct sock *sk)
1496{
4f8943f8 1497 struct sock_xprt *transport;
1f0fa154
IJ		 1498 struct rpc_xprt *xprt;
1499
13331a55 1500 if (!sk->sk_socket)
1f0fa154 1501 return;
13331a55 1502 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1f0fa154
IJ		 1503
1504 if (unlikely(!(xprt = xprt_from_sock(sk))))
1505 return;
4f8943f8 1506 transport = container_of(xprt, struct sock_xprt, xprt);
2790a624
TM		 1507 if (!test_and_clear_bit(XPRT_SOCK_NOSPACE, &transport->sock_state))
1508 return;
4f8943f8
TM		 1509 xs_run_error_worker(transport, XPRT_SOCK_WAKE_WRITE);
1510 sk->sk_write_pending--;
1f0fa154
IJ		 1511}
1512
9903cd1c 1513/**
c7b2cae8
CL		 1514 * xs_udp_write_space - callback invoked when socket buffer space
1515 * becomes available
9903cd1c
CL		 1516 * @sk: socket whose state has changed
1517 *
a246b010
CL		 1518 * Called when more output buffer space is available for this socket.
1519 * We try not to wake our writers until they can make "significant"
c7b2cae8 1520 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
a246b010
CL		 1521 * with a bunch of small requests.
1522 */
c7b2cae8 1523static void xs_udp_write_space(struct sock *sk)
a246b010 1524{
c7b2cae8 1525 /* from net/core/sock.c:sock_def_write_space */
1f0fa154
IJ		 1526 if (sock_writeable(sk))
1527 xs_write_space(sk);
c7b2cae8 1528}
a246b010 1529
c7b2cae8
CL		 1530/**
1531 * xs_tcp_write_space - callback invoked when socket buffer space
1532 * becomes available
1533 * @sk: socket whose state has changed
1534 *
1535 * Called when more output buffer space is available for this socket.
1536 * We try not to wake our writers until they can make "significant"
1537 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
1538 * with a bunch of small requests.
1539 */
1540static void xs_tcp_write_space(struct sock *sk)
1541{
c7b2cae8 1542 /* from net/core/stream.c:sk_stream_write_space */
64dc6130 1543 if (sk_stream_is_writeable(sk))
1f0fa154 1544 xs_write_space(sk);
a246b010
CL		 1545}
1546
470056c2 1547static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt)
a246b010 1548{
ee0ac0c2
CL		 1549 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1550 struct sock *sk = transport->inet;
a246b010 1551
7c6e066e 1552 if (transport->rcvsize) {
a246b010 1553 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
7c6e066e 1554 sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;
a246b010 1555 }
7c6e066e 1556 if (transport->sndsize) {
a246b010 1557 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
7c6e066e 1558 sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;
a246b010
CL		 1559 sk->sk_write_space(sk);
1560 }
1561}
1562
43118c29 1563/**
470056c2 1564 * xs_udp_set_buffer_size - set send and receive limits
43118c29 1565 * @xprt: generic transport
470056c2
CL		 1566 * @sndsize: requested size of send buffer, in bytes
1567 * @rcvsize: requested size of receive buffer, in bytes
43118c29 1568 *
470056c2 1569 * Set socket send and receive buffer size limits.
43118c29 1570 */
470056c2 1571static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize)
43118c29 1572{
7c6e066e
CL		 1573 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1574
1575 transport->sndsize = 0;
470056c2 1576 if (sndsize)
7c6e066e
CL		 1577 transport->sndsize = sndsize + 1024;
1578 transport->rcvsize = 0;
470056c2 1579 if (rcvsize)
7c6e066e 1580 transport->rcvsize = rcvsize + 1024;
470056c2
CL		 1581
1582 xs_udp_do_set_buffer_size(xprt);
43118c29
CL		 1583}
1584
46c0ee8b
CL		 1585/**
1586 * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport
acf0a39f 1587 * @xprt: controlling transport
46c0ee8b
CL		 1588 * @task: task that timed out
1589 *
1590 * Adjust the congestion window after a retransmit timeout has occurred.
1591 */
6a24dfb6 1592static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task)
46c0ee8b 1593{
b5e92419 1594 spin_lock(&xprt->transport_lock);
6a24dfb6 1595 xprt_adjust_cwnd(xprt, task, -ETIMEDOUT);
b5e92419 1596 spin_unlock(&xprt->transport_lock);
46c0ee8b
CL		 1597}
1598
826799e6 1599static int xs_get_random_port(void)
b85d8806 1600{
826799e6
BF		 1601 unsigned short min = xprt_min_resvport, max = xprt_max_resvport;
1602 unsigned short range;
1603 unsigned short rand;
1604
1605 if (max < min)
1606 return -EADDRINUSE;
1607 range = max - min + 1;
1608 rand = (unsigned short) prandom_u32() % range;
1609 return rand + min;
b85d8806
CL		 1610}
1611
4dda9c8a
TM		 1612static unsigned short xs_sock_getport(struct socket *sock)
1613{
1614 struct sockaddr_storage buf;
4dda9c8a
TM		 1615 unsigned short port = 0;
1616
9b2c45d4 1617 if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0)
4dda9c8a
TM		 1618 goto out;
1619 switch (buf.ss_family) {
1620 case AF_INET6:
1621 port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port);
1622 break;
1623 case AF_INET:
1624 port = ntohs(((struct sockaddr_in *)&buf)->sin_port);
1625 }
1626out:
1627 return port;
1628}
1629
92200412
CL		 1630/**
1631 * xs_set_port - reset the port number in the remote endpoint address
1632 * @xprt: generic transport
1633 * @port: new port number
1634 *
1635 */
1636static void xs_set_port(struct rpc_xprt *xprt, unsigned short port)
1637{
46121cf7 1638 dprintk("RPC: setting port for xprt %p to %u\n", xprt, port);
c4efcb1d 1639
9dc3b095
CL		 1640 rpc_set_port(xs_addr(xprt), port);
1641 xs_update_peer_port(xprt);
92200412
CL		 1642}
1643
4dda9c8a
TM		 1644static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)
1645{
e6237b6f 1646 if (transport->srcport == 0 && transport->xprt.reuseport)
4dda9c8a
TM		 1647 transport->srcport = xs_sock_getport(sock);
1648}
1649
826799e6 1650static int xs_get_srcport(struct sock_xprt *transport)
67a391d7 1651{
826799e6 1652 int port = transport->srcport;
67a391d7
TM		 1653
1654 if (port == 0 && transport->xprt.resvport)
1655 port = xs_get_random_port();
1656 return port;
1657}
1658
421ab1be 1659static unsigned short xs_sock_srcport(struct rpc_xprt *xprt)
a8482488
OK		 1660{
1661 struct sock_xprt *sock = container_of(xprt, struct sock_xprt, xprt);
b49ea673
N		 1662 unsigned short ret = 0;
1663 mutex_lock(&sock->recv_mutex);
1664 if (sock->sock)
1665 ret = xs_sock_getport(sock->sock);
1666 mutex_unlock(&sock->recv_mutex);
1667 return ret;
a8482488 1668}
421ab1be
TM		 1669
1670static int xs_sock_srcaddr(struct rpc_xprt *xprt, char *buf, size_t buflen)
1671{
1672 struct sock_xprt *sock = container_of(xprt, struct sock_xprt, xprt);
1673 union {
1674 struct sockaddr sa;
1675 struct sockaddr_storage st;
1676 } saddr;
1677 int ret = -ENOTCONN;
1678
1679 mutex_lock(&sock->recv_mutex);
1680 if (sock->sock) {
1681 ret = kernel_getsockname(sock->sock, &saddr.sa);
1682 if (ret >= 0)
1683 ret = snprintf(buf, buflen, "%pISc", &saddr.sa);
1684 }
1685 mutex_unlock(&sock->recv_mutex);
1686 return ret;
1687}
a8482488 1688
baaf4e48 1689static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port)
67a391d7 1690{
fbfffbd5
CL		 1691 if (transport->srcport != 0)
1692 transport->srcport = 0;
67a391d7
TM		 1693 if (!transport->xprt.resvport)
1694 return 0;
1695 if (port <= xprt_min_resvport || port > xprt_max_resvport)
1696 return xprt_max_resvport;
1697 return --port;
1698}
beb59b68 1699static int xs_bind(struct sock_xprt *transport, struct socket *sock)
a246b010 1700{
beb59b68 1701 struct sockaddr_storage myaddr;
67a391d7 1702 int err, nloop = 0;
826799e6 1703 int port = xs_get_srcport(transport);
67a391d7 1704 unsigned short last;
a246b010 1705
0f7a622c
CP		 1706 /*
1707 * If we are asking for any ephemeral port (i.e. port == 0 &&
1708 * transport->xprt.resvport == 0), don't bind. Let the local
1709 * port selection happen implicitly when the socket is used
1710 * (for example at connect time).
1711 *
1712 * This ensures that we can continue to establish TCP
1713 * connections even when all local ephemeral ports are already
1714 * a part of some TCP connection. This makes no difference
12b20ce3 1715 * for UDP sockets, but also doesn't harm them.
0f7a622c
CP		 1716 *
1717 * If we're asking for any reserved port (i.e. port == 0 &&
1718 * transport->xprt.resvport == 1) xs_get_srcport above will
1719 * ensure that port is non-zero and we will bind as needed.
1720 */
826799e6
BF		 1721 if (port <= 0)
1722 return port;
0f7a622c 1723
beb59b68 1724 memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen);
a246b010 1725 do {
beb59b68
PE		 1726 rpc_set_port((struct sockaddr *)&myaddr, port);
1727 err = kernel_bind(sock, (struct sockaddr *)&myaddr,
1728 transport->xprt.addrlen);
a246b010 1729 if (err == 0) {
bc1c56e9
N		 1730 if (transport->xprt.reuseport)
1731 transport->srcport = port;
d3bc9a1d 1732 break;
a246b010 1733 }
67a391d7 1734 last = port;
baaf4e48 1735 port = xs_next_srcport(transport, port);
67a391d7
TM		 1736 if (port > last)
1737 nloop++;
1738 } while (err == -EADDRINUSE && nloop != 2);
90058d37 1739
4232e863 1740 if (myaddr.ss_family == AF_INET)
beb59b68
PE		 1741 dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__,
1742 &((struct sockaddr_in *)&myaddr)->sin_addr,
1743 port, err ? "failed" : "ok", err);
1744 else
1745 dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__,
1746 &((struct sockaddr_in6 *)&myaddr)->sin6_addr,
1747 port, err ? "failed" : "ok", err);
a246b010
CL		 1748 return err;
1749}
1750
176e21ee
CL		 1751/*
1752 * We don't support autobind on AF_LOCAL sockets
1753 */
1754static void xs_local_rpcbind(struct rpc_task *task)
1755{
fb43d172 1756 xprt_set_bound(task->tk_xprt);
176e21ee
CL		 1757}
1758
1759static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port)
1760{
1761}
beb59b68 1762
ed07536e 1763#ifdef CONFIG_DEBUG_LOCK_ALLOC
064a9177
N		 1764static struct lock_class_key xs_key[3];
1765static struct lock_class_key xs_slock_key[3];
ed07536e 1766
176e21ee
CL		 1767static inline void xs_reclassify_socketu(struct socket *sock)
1768{
1769 struct sock *sk = sock->sk;
1770
176e21ee 1771 sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC",
064a9177 1772 &xs_slock_key[0], "sk_lock-AF_LOCAL-RPC", &xs_key[0]);
176e21ee
CL		 1773}
1774
8945ee5e 1775static inline void xs_reclassify_socket4(struct socket *sock)
ed07536e
PZ		 1776{
1777 struct sock *sk = sock->sk;
8945ee5e 1778
8945ee5e 1779 sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC",
064a9177 1780 &xs_slock_key[1], "sk_lock-AF_INET-RPC", &xs_key[1]);
8945ee5e 1781}
ed07536e 1782
8945ee5e
CL		 1783static inline void xs_reclassify_socket6(struct socket *sock)
1784{
1785 struct sock *sk = sock->sk;
ed07536e 1786
8945ee5e 1787 sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC",
064a9177 1788 &xs_slock_key[2], "sk_lock-AF_INET6-RPC", &xs_key[2]);
ed07536e 1789}
6bc9638a
PE		 1790
1791static inline void xs_reclassify_socket(int family, struct socket *sock)
1792{
fafc4e1e 1793 if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk)))
1b7a1819
WAA		 1794 return;
1795
4232e863 1796 switch (family) {
176e21ee
CL		 1797 case AF_LOCAL:
1798 xs_reclassify_socketu(sock);
1799 break;
4232e863 1800 case AF_INET:
6bc9638a 1801 xs_reclassify_socket4(sock);
4232e863
CL		 1802 break;
1803 case AF_INET6:
6bc9638a 1804 xs_reclassify_socket6(sock);
4232e863
CL		 1805 break;
1806 }
6bc9638a 1807}
ed07536e 1808#else
6bc9638a
PE		 1809static inline void xs_reclassify_socket(int family, struct socket *sock)
1810{
1811}
ed07536e
PZ		 1812#endif
1813
93dc41bd
N		 1814static void xs_dummy_setup_socket(struct work_struct *work)
1815{
1816}
1817
6bc9638a 1818static struct socket *xs_create_sock(struct rpc_xprt *xprt,
4dda9c8a
TM		 1819 struct sock_xprt *transport, int family, int type,
1820 int protocol, bool reuseport)
22f79326 1821{
a73881c9 1822 struct file *filp;
22f79326
PE		 1823 struct socket *sock;
1824 int err;
1825
6bc9638a 1826 err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1);
22f79326
PE		 1827 if (err < 0) {
1828 dprintk("RPC: can't create %d transport socket (%d).\n",
1829 protocol, -err);
1830 goto out;
1831 }
6bc9638a 1832 xs_reclassify_socket(family, sock);
22f79326 1833
4dda9c8a 1834 if (reuseport)
fe31a326 1835 sock_set_reuseport(sock->sk);
4dda9c8a 1836
4cea288a
BH		 1837 err = xs_bind(transport, sock);
1838 if (err) {
22f79326
PE		 1839 sock_release(sock);
1840 goto out;
1841 }
1842
a73881c9
TM		 1843 filp = sock_alloc_file(sock, O_NONBLOCK, NULL);
1844 if (IS_ERR(filp))
1845 return ERR_CAST(filp);
1846 transport->file = filp;
1847
22f79326
PE		 1848 return sock;
1849out:
1850 return ERR_PTR(err);
1851}
1852
176e21ee
CL		 1853static int xs_local_finish_connecting(struct rpc_xprt *xprt,
1854 struct socket *sock)
1855{
1856 struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
1857 xprt);
1858
1859 if (!transport->inet) {
1860 struct sock *sk = sock->sk;
1861
ea9afca8 1862 lock_sock(sk);
176e21ee
CL		 1863
1864 xs_save_old_callbacks(transport, sk);
1865
1866 sk->sk_user_data = xprt;
a2648094 1867 sk->sk_data_ready = xs_data_ready;
176e21ee 1868 sk->sk_write_space = xs_udp_write_space;
2118071d 1869 sk->sk_error_report = xs_error_report;
176e21ee
CL		 1870
1871 xprt_clear_connected(xprt);
1872
1873 /* Reset to new socket */
1874 transport->sock = sock;
1875 transport->inet = sk;
1876
ea9afca8 1877 release_sock(sk);
176e21ee
CL		 1878 }
1879
ae053551 1880 xs_stream_start_connect(transport);
6c7a64e5 1881
176e21ee
CL		 1882 return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0);
1883}
1884
1885/**
1886 * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint
176e21ee 1887 * @transport: socket transport to connect
176e21ee 1888 */
dc107402 1889static int xs_local_setup_socket(struct sock_xprt *transport)
176e21ee 1890{
176e21ee 1891 struct rpc_xprt *xprt = &transport->xprt;
a73881c9 1892 struct file *filp;
176e21ee 1893 struct socket *sock;
68e9a246 1894 int status;
176e21ee 1895
176e21ee
CL		 1896 status = __sock_create(xprt->xprt_net, AF_LOCAL,
1897 SOCK_STREAM, 0, &sock, 1);
1898 if (status < 0) {
1899 dprintk("RPC: can't create AF_LOCAL "
1900 "transport socket (%d).\n", -status);
1901 goto out;
1902 }
d1358917 1903 xs_reclassify_socket(AF_LOCAL, sock);
176e21ee 1904
a73881c9
TM		 1905 filp = sock_alloc_file(sock, O_NONBLOCK, NULL);
1906 if (IS_ERR(filp)) {
1907 status = PTR_ERR(filp);
1908 goto out;
1909 }
1910 transport->file = filp;
1911
176e21ee
CL		 1912 dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n",
1913 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
1914
1915 status = xs_local_finish_connecting(xprt, sock);
40b5ea0c 1916 trace_rpc_socket_connect(xprt, sock, status);
176e21ee
CL		 1917 switch (status) {
1918 case 0:
1919 dprintk("RPC: xprt %p connected to %s\n",
1920 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
3968a8a5
CL		 1921 xprt->stat.connect_count++;
1922 xprt->stat.connect_time += (long)jiffies -
1923 xprt->stat.connect_start;
176e21ee 1924 xprt_set_connected(xprt);
93f479d3 1925 break;
3601c4a9 1926 case -ENOBUFS:
176e21ee
CL		 1927 break;
1928 case -ENOENT:
1929 dprintk("RPC: xprt %p: socket %s does not exist\n",
1930 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
1931 break;
4a20a988
TM		 1932 case -ECONNREFUSED:
1933 dprintk("RPC: xprt %p: connection refused for %s\n",
1934 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
1935 break;
176e21ee
CL		 1936 default:
1937 printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n",
1938 __func__, -status,
1939 xprt->address_strings[RPC_DISPLAY_ADDR]);
1940 }
1941
1942out:
1943 xprt_clear_connecting(xprt);
1944 xprt_wake_pending_tasks(xprt, status);
dc107402
BF		 1945 return status;
1946}
1947
b6669737 1948static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task)
dc107402 1949{
dc107402
BF		 1950 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1951 int ret;
1952
aad41a7d
TM		 1953 if (transport->file)
1954 goto force_disconnect;
1955
c4f03966 1956 if (RPC_IS_ASYNC(task)) {
dc107402
BF		 1957 /*
1958 * We want the AF_LOCAL connect to be resolved in the
1959 * filesystem namespace of the process making the rpc
1960 * call. Thus we connect synchronously.
1961 *
1962 * If we want to support asynchronous AF_LOCAL calls,
1963 * we'll need to figure out how to pass a namespace to
1964 * connect.
1965 */
5ad64b36 1966 task->tk_rpc_status = -ENOTCONN;
dc107402 1967 rpc_exit(task, -ENOTCONN);
aad41a7d 1968 goto out_wake;
dc107402
BF		 1969 }
1970 ret = xs_local_setup_socket(transport);
1971 if (ret && !RPC_IS_SOFTCONN(task))
1972 msleep_interruptible(15000);
aad41a7d
TM		 1973 return;
1974force_disconnect:
1975 xprt_force_disconnect(xprt);
1976out_wake:
1977 xprt_clear_connecting(xprt);
1978 xprt_wake_pending_tasks(xprt, -ENOTCONN);
176e21ee
CL		 1979}
1980
3c87ef6e 1981#if IS_ENABLED(CONFIG_SUNRPC_SWAP)
d6e971d8 1982/*
693486d5
N		 1983 * Note that this should be called with XPRT_LOCKED held, or recv_mutex
1984 * held, or when we otherwise know that we have exclusive access to the
1985 * socket, to guard against races with xs_reset_transport.
d6e971d8 1986 */
a564b8f0
MG		 1987static void xs_set_memalloc(struct rpc_xprt *xprt)
1988{
1989 struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
1990 xprt);
1991
d6e971d8
JL		 1992 /*
1993 * If there's no sock, then we have nothing to set. The
1994 * reconnecting process will get it for us.
1995 */
1996 if (!transport->inet)
1997 return;
8e228133 1998 if (atomic_read(&xprt->swapper))
a564b8f0
MG		 1999 sk_set_memalloc(transport->inet);
2000}
2001
2002/**
d67fa4d8 2003 * xs_enable_swap - Tag this transport as being used for swap.
a564b8f0 2004 * @xprt: transport to tag
a564b8f0 2005 *
8e228133
JL		 2006 * Take a reference to this transport on behalf of the rpc_clnt, and
2007 * optionally mark it for swapping if it wasn't already.
a564b8f0 2008 */
d67fa4d8
JL		 2009static int
2010xs_enable_swap(struct rpc_xprt *xprt)
a564b8f0 2011{
d6e971d8 2012 struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
a564b8f0 2013
693486d5
N		 2014 mutex_lock(&xs->recv_mutex);
2015 if (atomic_inc_return(&xprt->swapper) == 1 &&
2016 xs->inet)
d6e971d8 2017 sk_set_memalloc(xs->inet);
693486d5 2018 mutex_unlock(&xs->recv_mutex);
8e228133
JL		 2019 return 0;
2020}
a564b8f0 2021
8e228133 2022/**
d67fa4d8 2023 * xs_disable_swap - Untag this transport as being used for swap.
8e228133
JL		 2024 * @xprt: transport to tag
2025 *
2026 * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the
2027 * swapper refcount goes to 0, untag the socket as a memalloc socket.
2028 */
d67fa4d8
JL		 2029static void
2030xs_disable_swap(struct rpc_xprt *xprt)
8e228133 2031{
d6e971d8 2032 struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
8e228133 2033
693486d5
N		 2034 mutex_lock(&xs->recv_mutex);
2035 if (atomic_dec_and_test(&xprt->swapper) &&
2036 xs->inet)
d6e971d8 2037 sk_clear_memalloc(xs->inet);
693486d5 2038 mutex_unlock(&xs->recv_mutex);
a564b8f0 2039}
a564b8f0
MG		 2040#else
2041static void xs_set_memalloc(struct rpc_xprt *xprt)
2042{
2043}
d67fa4d8
JL		 2044
2045static int
2046xs_enable_swap(struct rpc_xprt *xprt)
2047{
2048 return -EINVAL;
2049}
2050
2051static void
2052xs_disable_swap(struct rpc_xprt *xprt)
2053{
2054}
a564b8f0
MG		 2055#endif
2056
16be2d20
CL		 2057static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
2058{
2059 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2060
2061 if (!transport->inet) {
2062 struct sock *sk = sock->sk;
2063
ea9afca8 2064 lock_sock(sk);
16be2d20 2065
2a9e1cfa
TM		 2066 xs_save_old_callbacks(transport, sk);
2067
16be2d20 2068 sk->sk_user_data = xprt;
f9b2ee71 2069 sk->sk_data_ready = xs_data_ready;
16be2d20 2070 sk->sk_write_space = xs_udp_write_space;
16be2d20
CL		 2071
2072 xprt_set_connected(xprt);
2073
2074 /* Reset to new socket */
2075 transport->sock = sock;
2076 transport->inet = sk;
2077
a564b8f0
MG		 2078 xs_set_memalloc(xprt);
2079
ea9afca8 2080 release_sock(sk);
16be2d20
CL		 2081 }
2082 xs_udp_do_set_buffer_size(xprt);
02910177
TM		 2083
2084 xprt->stat.connect_start = jiffies;
16be2d20
CL		 2085}
2086
8c14ff2a 2087static void xs_udp_setup_socket(struct work_struct *work)
a246b010 2088{
34161db6
TM		 2089 struct sock_xprt *transport =
2090 container_of(work, struct sock_xprt, connect_worker.work);
c8475461 2091 struct rpc_xprt *xprt = &transport->xprt;
d099b8af 2092 struct socket *sock;
b65c0310 2093 int status = -EIO;
8db55a03 2094 unsigned int pflags = current->flags;
9903cd1c 2095
8db55a03
N		 2096 if (atomic_read(&xprt->swapper))
2097 current->flags |= PF_MEMALLOC;
8c14ff2a 2098 sock = xs_create_sock(xprt, transport,
4dda9c8a
TM		 2099 xs_addr(xprt)->sa_family, SOCK_DGRAM,
2100 IPPROTO_UDP, false);
b65c0310 2101 if (IS_ERR(sock))
b0d93ad5 2102 goto out;
68e220bd 2103
c740eff8
CL		 2104 dprintk("RPC: worker connecting xprt %p via %s to "
2105 "%s (port %s)\n", xprt,
2106 xprt->address_strings[RPC_DISPLAY_PROTO],
2107 xprt->address_strings[RPC_DISPLAY_ADDR],
2108 xprt->address_strings[RPC_DISPLAY_PORT]);
68e220bd
CL		 2109
2110 xs_udp_finish_connecting(xprt, sock);
40b5ea0c 2111 trace_rpc_socket_connect(xprt, sock, 0);
b0d93ad5
CL		 2112 status = 0;
2113out:
b0d93ad5 2114 xprt_clear_connecting(xprt);
cf76785d 2115 xprt_unlock_connect(xprt, transport);
7d1e8255 2116 xprt_wake_pending_tasks(xprt, status);
8db55a03 2117 current_restore_flags(pflags, PF_MEMALLOC);
a246b010
CL		 2118}
2119
4876cc77
TM		 2120/**
2121 * xs_tcp_shutdown - gracefully shut down a TCP socket
2122 * @xprt: transport
2123 *
2124 * Initiates a graceful shutdown of the TCP socket by calling the
2125 * equivalent of shutdown(SHUT_RDWR);
2126 */
2127static void xs_tcp_shutdown(struct rpc_xprt *xprt)
2128{
2129 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2130 struct socket *sock = transport->sock;
9b30889c 2131 int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE;
4876cc77
TM		 2132
2133 if (sock == NULL)
2134 return;
0a6ff58e
TM		 2135 if (!xprt->reuseport) {
2136 xs_close(xprt);
2137 return;
2138 }
9b30889c 2139 switch (skst) {
7c81e6a9
TM		 2140 case TCP_FIN_WAIT1:
2141 case TCP_FIN_WAIT2:
2142 break;
2143 case TCP_ESTABLISHED:
2144 case TCP_CLOSE_WAIT:
4876cc77
TM		 2145 kernel_sock_shutdown(sock, SHUT_RDWR);
2146 trace_rpc_socket_shutdown(xprt, sock);
9b30889c 2147 break;
7c81e6a9 2148 default:
4876cc77 2149 xs_reset_transport(transport);
9b30889c 2150 }
4876cc77
TM		 2151}
2152
8d1b8c62
TM		 2153static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
2154 struct socket *sock)
2155{
7196dbb0
TM		 2156 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2157 unsigned int keepidle;
2158 unsigned int keepcnt;
8d1b8c62
TM		 2159 unsigned int timeo;
2160
b5e92419 2161 spin_lock(&xprt->transport_lock);
7196dbb0
TM		 2162 keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ);
2163 keepcnt = xprt->timeout->to_retries + 1;
2164 timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
2165 (xprt->timeout->to_retries + 1);
2166 clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
b5e92419 2167 spin_unlock(&xprt->transport_lock);
7196dbb0 2168
8d1b8c62 2169 /* TCP Keepalive options */
ce3d9544 2170 sock_set_keepalive(sock->sk);
71c48eb8 2171 tcp_sock_set_keepidle(sock->sk, keepidle);
d41ecaac 2172 tcp_sock_set_keepintvl(sock->sk, keepidle);
480aeb96 2173 tcp_sock_set_keepcnt(sock->sk, keepcnt);
8d1b8c62
TM		 2174
2175 /* TCP user timeout (see RFC5482) */
c488aead 2176 tcp_sock_set_user_timeout(sock->sk, timeo);
8d1b8c62
TM		 2177}
2178
7196dbb0
TM		 2179static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt,
2180 unsigned long connect_timeout,
2181 unsigned long reconnect_timeout)
2182{
2183 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2184 struct rpc_timeout to;
2185 unsigned long initval;
2186
b5e92419 2187 spin_lock(&xprt->transport_lock);
7196dbb0
TM		 2188 if (reconnect_timeout < xprt->max_reconnect_timeout)
2189 xprt->max_reconnect_timeout = reconnect_timeout;
2190 if (connect_timeout < xprt->connect_timeout) {
2191 memcpy(&to, xprt->timeout, sizeof(to));
2192 initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1);
2193 /* Arbitrary lower limit */
2194 if (initval < XS_TCP_INIT_REEST_TO << 1)
2195 initval = XS_TCP_INIT_REEST_TO << 1;
2196 to.to_initval = initval;
2197 to.to_maxval = initval;
2198 memcpy(&transport->tcp_timeout, &to,
2199 sizeof(transport->tcp_timeout));
2200 xprt->timeout = &transport->tcp_timeout;
2201 xprt->connect_timeout = connect_timeout;
2202 }
2203 set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
b5e92419 2204 spin_unlock(&xprt->transport_lock);
7196dbb0
TM		 2205}
2206
16be2d20 2207static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
a246b010 2208{
16be2d20 2209 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
edb267a6 2210
ee0ac0c2 2211 if (!transport->inet) {
b0d93ad5 2212 struct sock *sk = sock->sk;
7f260e85 2213
d88e4d82
N		 2214 /* Avoid temporary address, they are bad for long-lived
2215 * connections such as NFS mounts.
2216 * RFC4941, section 3.6 suggests that:
2217 * Individual applications, which have specific
2218 * knowledge about the normal duration of connections,
2219 * MAY override this as appropriate.
2220 */
18d5ad62
CH		 2221 if (xs_addr(xprt)->sa_family == PF_INET6) {
2222 ip6_sock_set_addr_preferences(sk,
2223 IPV6_PREFER_SRC_PUBLIC);
2224 }
d88e4d82 2225
8d1b8c62 2226 xs_tcp_set_socket_timeouts(xprt, sock);
d737e5d4 2227 tcp_sock_set_nodelay(sk);
775f06ab 2228
ea9afca8 2229 lock_sock(sk);
b0d93ad5 2230
2a9e1cfa
TM		 2231 xs_save_old_callbacks(transport, sk);
2232
b0d93ad5 2233 sk->sk_user_data = xprt;
5157b956 2234 sk->sk_data_ready = xs_data_ready;
b0d93ad5
CL		 2235 sk->sk_state_change = xs_tcp_state_change;
2236 sk->sk_write_space = xs_tcp_write_space;
2118071d 2237 sk->sk_error_report = xs_error_report;
3167e12c
CL		 2238
2239 /* socket options */
3167e12c 2240 sock_reset_flag(sk, SOCK_LINGER);
b0d93ad5
CL		 2241
2242 xprt_clear_connected(xprt);
2243
2244 /* Reset to new socket */
ee0ac0c2
CL		 2245 transport->sock = sock;
2246 transport->inet = sk;
b0d93ad5 2247
ea9afca8 2248 release_sock(sk);
b0d93ad5
CL		 2249 }
2250
01d37c42 2251 if (!xprt_bound(xprt))
280254b6 2252 return -ENOTCONN;
01d37c42 2253
a564b8f0
MG		 2254 xs_set_memalloc(xprt);
2255
ae053551 2256 xs_stream_start_connect(transport);
e1806c7b 2257
b0d93ad5 2258 /* Tell the socket layer to start connecting... */
0fdea1e8 2259 set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
280254b6 2260 return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);
16be2d20
CL		 2261}
2262
9903cd1c 2263/**
b61d59ff 2264 * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint
acf0a39f 2265 * @work: queued work item
9903cd1c
CL		 2266 *
2267 * Invoked by a work queue tasklet.
a246b010 2268 */
cdd518d5 2269static void xs_tcp_setup_socket(struct work_struct *work)
a246b010 2270{
cdd518d5
PE		 2271 struct sock_xprt *transport =
2272 container_of(work, struct sock_xprt, connect_worker.work);
ee0ac0c2 2273 struct socket *sock = transport->sock;
a9f5f0f7 2274 struct rpc_xprt *xprt = &transport->xprt;
280254b6 2275 int status;
8db55a03 2276 unsigned int pflags = current->flags;
a246b010 2277
8db55a03
N		 2278 if (atomic_read(&xprt->swapper))
2279 current->flags |= PF_MEMALLOC;
89f42494
TM		 2280
2281 if (xprt_connected(xprt))
2282 goto out;
2283 if (test_and_clear_bit(XPRT_SOCK_CONNECT_SENT,
2284 &transport->sock_state) ||
2285 !sock) {
2286 xs_reset_transport(transport);
2287 sock = xs_create_sock(xprt, transport, xs_addr(xprt)->sa_family,
2288 SOCK_STREAM, IPPROTO_TCP, true);
b61d59ff 2289 if (IS_ERR(sock)) {
280254b6 2290 xprt_wake_pending_tasks(xprt, PTR_ERR(sock));
3167e12c
CL		 2291 goto out;
2292 }
7d1e8255 2293 }
a246b010 2294
c740eff8
CL		 2295 dprintk("RPC: worker connecting xprt %p via %s to "
2296 "%s (port %s)\n", xprt,
2297 xprt->address_strings[RPC_DISPLAY_PROTO],
2298 xprt->address_strings[RPC_DISPLAY_ADDR],
2299 xprt->address_strings[RPC_DISPLAY_PORT]);
edb267a6 2300
16be2d20 2301 status = xs_tcp_finish_connecting(xprt, sock);
40b5ea0c 2302 trace_rpc_socket_connect(xprt, sock, status);
46121cf7
CL		 2303 dprintk("RPC: %p connect status %d connected %d sock state %d\n",
2304 xprt, -status, xprt_connected(xprt),
2305 sock->sk->sk_state);
2a491991
TM		 2306 switch (status) {
2307 case 0:
2308 case -EINPROGRESS:
280254b6 2309 /* SYN_SENT! */
89f42494 2310 set_bit(XPRT_SOCK_CONNECT_SENT, &transport->sock_state);
280254b6
TM		 2311 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
2312 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2313 fallthrough;
2a491991 2314 case -EALREADY:
280254b6
TM		 2315 goto out_unlock;
2316 case -EADDRNOTAVAIL:
2317 /* Source port number is unavailable. Try a new one! */
2318 transport->srcport = 0;
2319 status = -EAGAIN;
2320 break;
9fcfe0c8
TM		 2321 case -EINVAL:
2322 /* Happens, for instance, if the user specified a link
2323 * local IPv6 address without a scope-id.
2324 */
3ed5e2a2
TM		 2325 case -ECONNREFUSED:
2326 case -ECONNRESET:
eb5b46fa 2327 case -ENETDOWN:
3ed5e2a2 2328 case -ENETUNREACH:
4ba161a7 2329 case -EHOSTUNREACH:
3913c78c 2330 case -EADDRINUSE:
3601c4a9 2331 case -ENOBUFS:
280254b6
TM		 2332 break;
2333 default:
2334 printk("%s: connect returned unhandled error %d\n",
2335 __func__, status);
2336 status = -EAGAIN;
a246b010 2337 }
280254b6
TM		 2338
2339 /* xs_tcp_force_close() wakes tasks with a fixed error code.
2340 * We need to wake them first to ensure the correct error code.
2341 */
2342 xprt_wake_pending_tasks(xprt, status);
2343 xs_tcp_force_close(xprt);
a246b010 2344out:
68e220bd 2345 xprt_clear_connecting(xprt);
280254b6 2346out_unlock:
cf76785d 2347 xprt_unlock_connect(xprt, transport);
8db55a03 2348 current_restore_flags(pflags, PF_MEMALLOC);
68e220bd 2349}
b0d93ad5 2350
9903cd1c
CL		 2351/**
2352 * xs_connect - connect a socket to a remote endpoint
1b092092 2353 * @xprt: pointer to transport structure
9903cd1c
CL		 2354 * @task: address of RPC task that manages state of connect request
2355 *
2356 * TCP: If the remote end dropped the connection, delay reconnecting.
03bf4b70
CL		 2357 *
2358 * UDP socket connects are synchronous, but we use a work queue anyway
2359 * to guarantee that even unprivileged user processes can set up a
2360 * socket on a privileged port.
2361 *
2362 * If a UDP socket connect fails, the delay behavior here prevents
2363 * retry floods (hard mounts).
9903cd1c 2364 */
1b092092 2365static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
a246b010 2366{
ee0ac0c2 2367 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
02910177 2368 unsigned long delay = 0;
a246b010 2369
718ba5b8
TM		 2370 WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));
2371
89f42494 2372 if (transport->sock != NULL) {
46121cf7 2373 dprintk("RPC: xs_connect delayed xprt %p for %lu "
89f42494 2374 "seconds\n", xprt, xprt->reestablish_timeout / HZ);
99b1a4c3 2375
675dd90a
CL		 2376 delay = xprt_reconnect_delay(xprt);
2377 xprt_reconnect_backoff(xprt, XS_TCP_INIT_REEST_TO);
02910177 2378
02910177 2379 } else
46121cf7 2380 dprintk("RPC: xs_connect scheduled xprt %p\n", xprt);
02910177
TM		 2381
2382 queue_delayed_work(xprtiod_workqueue,
2383 &transport->connect_worker,
2384 delay);
a246b010
CL		 2385}
2386
4f8943f8
TM		 2387static void xs_wake_disconnect(struct sock_xprt *transport)
2388{
2389 if (test_and_clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state))
2390 xs_tcp_force_close(&transport->xprt);
2391}
2392
2393static void xs_wake_write(struct sock_xprt *transport)
2394{
2395 if (test_and_clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state))
2396 xprt_write_space(&transport->xprt);
2397}
2398
2399static void xs_wake_error(struct sock_xprt *transport)
2400{
2401 int sockerr;
4f8943f8
TM		 2402
2403 if (!test_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state))
2404 return;
2405 mutex_lock(&transport->recv_mutex);
2406 if (transport->sock == NULL)
2407 goto out;
2408 if (!test_and_clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state))
2409 goto out;
af84537d 2410 sockerr = xchg(&transport->xprt_err, 0);
4f8943f8
TM		 2411 if (sockerr < 0)
2412 xprt_wake_pending_tasks(&transport->xprt, sockerr);
2413out:
2414 mutex_unlock(&transport->recv_mutex);
2415}
2416
2417static void xs_wake_pending(struct sock_xprt *transport)
2418{
2419 if (test_and_clear_bit(XPRT_SOCK_WAKE_PENDING, &transport->sock_state))
2420 xprt_wake_pending_tasks(&transport->xprt, -EAGAIN);
2421}
2422
2423static void xs_error_handle(struct work_struct *work)
2424{
2425 struct sock_xprt *transport = container_of(work,
2426 struct sock_xprt, error_worker);
2427
2428 xs_wake_disconnect(transport);
2429 xs_wake_write(transport);
2430 xs_wake_error(transport);
2431 xs_wake_pending(transport);
2432}
2433
176e21ee 2434/**
12b20ce3 2435 * xs_local_print_stats - display AF_LOCAL socket-specific stats
176e21ee
CL		 2436 * @xprt: rpc_xprt struct containing statistics
2437 * @seq: output file
2438 *
2439 */
2440static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2441{
2442 long idle_time = 0;
2443
2444 if (xprt_connected(xprt))
2445 idle_time = (long)(jiffies - xprt->last_used) / HZ;
2446
2447 seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu "
15a45206 2448 "%llu %llu %lu %llu %llu\n",
176e21ee
CL		 2449 xprt->stat.bind_count,
2450 xprt->stat.connect_count,
8440a886 2451 xprt->stat.connect_time / HZ,
176e21ee
CL		 2452 idle_time,
2453 xprt->stat.sends,
2454 xprt->stat.recvs,
2455 xprt->stat.bad_xids,
2456 xprt->stat.req_u,
15a45206
AA		 2457 xprt->stat.bklog_u,
2458 xprt->stat.max_slots,
2459 xprt->stat.sending_u,
2460 xprt->stat.pending_u);
176e21ee
CL		 2461}
2462
262ca07d 2463/**
12b20ce3 2464 * xs_udp_print_stats - display UDP socket-specific stats
262ca07d
CL		 2465 * @xprt: rpc_xprt struct containing statistics
2466 * @seq: output file
2467 *
2468 */
2469static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2470{
c8475461
CL		 2471 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2472
15a45206
AA		 2473 seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu "
2474 "%lu %llu %llu\n",
fbfffbd5 2475 transport->srcport,
262ca07d
CL		 2476 xprt->stat.bind_count,
2477 xprt->stat.sends,
2478 xprt->stat.recvs,
2479 xprt->stat.bad_xids,
2480 xprt->stat.req_u,
15a45206
AA		 2481 xprt->stat.bklog_u,
2482 xprt->stat.max_slots,
2483 xprt->stat.sending_u,
2484 xprt->stat.pending_u);
262ca07d
CL		 2485}
2486
2487/**
12b20ce3 2488 * xs_tcp_print_stats - display TCP socket-specific stats
262ca07d
CL		 2489 * @xprt: rpc_xprt struct containing statistics
2490 * @seq: output file
2491 *
2492 */
2493static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2494{
c8475461 2495 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262ca07d
CL		 2496 long idle_time = 0;
2497
2498 if (xprt_connected(xprt))
2499 idle_time = (long)(jiffies - xprt->last_used) / HZ;
2500
15a45206
AA		 2501 seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu "
2502 "%llu %llu %lu %llu %llu\n",
fbfffbd5 2503 transport->srcport,
262ca07d
CL		 2504 xprt->stat.bind_count,
2505 xprt->stat.connect_count,
8440a886 2506 xprt->stat.connect_time / HZ,
262ca07d
CL		 2507 idle_time,
2508 xprt->stat.sends,
2509 xprt->stat.recvs,
2510 xprt->stat.bad_xids,
2511 xprt->stat.req_u,
15a45206
AA		 2512 xprt->stat.bklog_u,
2513 xprt->stat.max_slots,
2514 xprt->stat.sending_u,
2515 xprt->stat.pending_u);
262ca07d
CL		 2516}
2517
4cfc7e60
RI		 2518/*
2519 * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason
2520 * we allocate pages instead doing a kmalloc like rpc_malloc is because we want
2521 * to use the server side send routines.
2522 */
5fe6eaa1 2523static int bc_malloc(struct rpc_task *task)
4cfc7e60 2524{
5fe6eaa1
CL		 2525 struct rpc_rqst *rqst = task->tk_rqstp;
2526 size_t size = rqst->rq_callsize;
4cfc7e60
RI		 2527 struct page *page;
2528 struct rpc_buffer *buf;
2529
5fe6eaa1
CL		 2530 if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) {
2531 WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n",
2532 size);
2533 return -EINVAL;
2534 }
4cfc7e60 2535
b2648015 2536 page = alloc_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
4cfc7e60 2537 if (!page)
5fe6eaa1 2538 return -ENOMEM;
4cfc7e60
RI		 2539
2540 buf = page_address(page);
2541 buf->len = PAGE_SIZE;
2542
5fe6eaa1 2543 rqst->rq_buffer = buf->data;
18e601d6 2544 rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize;
5fe6eaa1 2545 return 0;
4cfc7e60
RI		 2546}
2547
2548/*
2549 * Free the space allocated in the bc_alloc routine
2550 */
3435c74a 2551static void bc_free(struct rpc_task *task)
4cfc7e60 2552{
3435c74a 2553 void *buffer = task->tk_rqstp->rq_buffer;
4cfc7e60
RI		 2554 struct rpc_buffer *buf;
2555
4cfc7e60
RI		 2556 buf = container_of(buffer, struct rpc_buffer, data);
2557 free_page((unsigned long)buf);
2558}
2559
4cfc7e60
RI		 2560static int bc_sendto(struct rpc_rqst *req)
2561{
da1661b9 2562 struct xdr_buf *xdr = &req->rq_snd_buf;
4cfc7e60 2563 struct sock_xprt *transport =
067fb11b 2564 container_of(req->rq_xprt, struct sock_xprt, xprt);
067fb11b 2565 struct msghdr msg = {
da1661b9 2566 .msg_flags = 0,
067fb11b
CL		 2567 };
2568 rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT |
da1661b9
CL		 2569 (u32)xdr->len);
2570 unsigned int sent = 0;
2571 int err;
4cfc7e60 2572
8729aaba 2573 req->rq_xtime = ktime_get();
ff053dbb
TM		 2574 err = xdr_alloc_bvec(xdr, rpc_task_gfp_mask());
2575 if (err < 0)
2576 return err;
da1661b9
CL		 2577 err = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, marker, &sent);
2578 xdr_free_bvec(xdr);
2579 if (err < 0 || sent != (xdr->len + sizeof(marker)))
067fb11b 2580 return -EAGAIN;
da1661b9 2581 return sent;
4cfc7e60
RI		 2582}
2583
ca4faf54
CL		 2584/**
2585 * bc_send_request - Send a backchannel Call on a TCP socket
2586 * @req: rpc_rqst containing Call message to be sent
2587 *
2588 * xpt_mutex ensures @rqstp's whole message is written to the socket
2589 * without interruption.
2590 *
2591 * Return values:
2592 * %0 if the message was sent successfully
2593 * %ENOTCONN if the message was not sent
4cfc7e60 2594 */
adfa7144 2595static int bc_send_request(struct rpc_rqst *req)
4cfc7e60 2596{
4cfc7e60 2597 struct svc_xprt *xprt;
7fc56136 2598 int len;
4cfc7e60 2599
4cfc7e60
RI		 2600 /*
2601 * Get the server socket associated with this callback xprt
2602 */
2603 xprt = req->rq_xprt->bc_xprt;
4cfc7e60
RI		 2604
2605 /*
2606 * Grab the mutex to serialize data as the connection is shared
2607 * with the fore channel
2608 */
c544577d 2609 mutex_lock(&xprt->xpt_mutex);
4cfc7e60
RI		 2610 if (test_bit(XPT_DEAD, &xprt->xpt_flags))
2611 len = -ENOTCONN;
2612 else
2613 len = bc_sendto(req);
2614 mutex_unlock(&xprt->xpt_mutex);
2615
2616 if (len > 0)
2617 len = 0;
2618
2619 return len;
2620}
2621
2622/*
2623 * The close routine. Since this is client initiated, we do nothing
2624 */
2625
2626static void bc_close(struct rpc_xprt *xprt)
2627{
6221f1d9 2628 xprt_disconnect_done(xprt);
4cfc7e60
RI		 2629}
2630
2631/*
2632 * The xprt destroy routine. Again, because this connection is client
2633 * initiated, we do nothing
2634 */
2635
2636static void bc_destroy(struct rpc_xprt *xprt)
2637{
47f72efa
KM		 2638 dprintk("RPC: bc_destroy xprt %p\n", xprt);
2639
2640 xs_xprt_free(xprt);
2641 module_put(THIS_MODULE);
4cfc7e60
RI		 2642}
2643
d31ae254 2644static const struct rpc_xprt_ops xs_local_ops = {
176e21ee 2645 .reserve_xprt = xprt_reserve_xprt,
4cd34e7c 2646 .release_xprt = xprt_release_xprt,
f39c1bfb 2647 .alloc_slot = xprt_alloc_slot,
a9cde23a 2648 .free_slot = xprt_free_slot,
176e21ee
CL		 2649 .rpcbind = xs_local_rpcbind,
2650 .set_port = xs_local_set_port,
dc107402 2651 .connect = xs_local_connect,
176e21ee
CL		 2652 .buf_alloc = rpc_malloc,
2653 .buf_free = rpc_free,
550aebfe 2654 .prepare_request = xs_stream_prepare_request,
176e21ee 2655 .send_request = xs_local_send_request,
8ba6a92d 2656 .wait_for_reply_request = xprt_wait_for_reply_request_def,
176e21ee 2657 .close = xs_close,
a1311d87 2658 .destroy = xs_destroy,
176e21ee 2659 .print_stats = xs_local_print_stats,
d67fa4d8
JL		 2660 .enable_swap = xs_enable_swap,
2661 .disable_swap = xs_disable_swap,
176e21ee
CL		 2662};
2663
d31ae254 2664static const struct rpc_xprt_ops xs_udp_ops = {
43118c29 2665 .set_buffer_size = xs_udp_set_buffer_size,
12a80469 2666 .reserve_xprt = xprt_reserve_xprt_cong,
49e9a890 2667 .release_xprt = xprt_release_xprt_cong,
f39c1bfb 2668 .alloc_slot = xprt_alloc_slot,
a9cde23a 2669 .free_slot = xprt_free_slot,
45160d62 2670 .rpcbind = rpcb_getport_async,
92200412 2671 .set_port = xs_set_port,
262965f5 2672 .connect = xs_connect,
421ab1be
TM		 2673 .get_srcaddr = xs_sock_srcaddr,
2674 .get_srcport = xs_sock_srcport,
02107148
CL		 2675 .buf_alloc = rpc_malloc,
2676 .buf_free = rpc_free,
262965f5 2677 .send_request = xs_udp_send_request,
8ba6a92d 2678 .wait_for_reply_request = xprt_wait_for_reply_request_rtt,
46c0ee8b 2679 .timer = xs_udp_timer,
a58dd398 2680 .release_request = xprt_release_rqst_cong,
262965f5
CL		 2681 .close = xs_close,
2682 .destroy = xs_destroy,
262ca07d 2683 .print_stats = xs_udp_print_stats,
d67fa4d8
JL		 2684 .enable_swap = xs_enable_swap,
2685 .disable_swap = xs_disable_swap,
4a068258 2686 .inject_disconnect = xs_inject_disconnect,
262965f5
CL		 2687};
2688
d31ae254 2689static const struct rpc_xprt_ops xs_tcp_ops = {
12a80469 2690 .reserve_xprt = xprt_reserve_xprt,
4cd34e7c 2691 .release_xprt = xprt_release_xprt,
36bd7de9 2692 .alloc_slot = xprt_alloc_slot,
a9cde23a 2693 .free_slot = xprt_free_slot,
45160d62 2694 .rpcbind = rpcb_getport_async,
92200412 2695 .set_port = xs_set_port,
0b9e7943 2696 .connect = xs_connect,
421ab1be
TM		 2697 .get_srcaddr = xs_sock_srcaddr,
2698 .get_srcport = xs_sock_srcport,
02107148
CL		 2699 .buf_alloc = rpc_malloc,
2700 .buf_free = rpc_free,
277e4ab7 2701 .prepare_request = xs_stream_prepare_request,
262965f5 2702 .send_request = xs_tcp_send_request,
8ba6a92d 2703 .wait_for_reply_request = xprt_wait_for_reply_request_def,
c627d31b 2704 .close = xs_tcp_shutdown,
9903cd1c 2705 .destroy = xs_destroy,
7196dbb0 2706 .set_connect_timeout = xs_tcp_set_connect_timeout,
262ca07d 2707 .print_stats = xs_tcp_print_stats,
d67fa4d8
JL		 2708 .enable_swap = xs_enable_swap,
2709 .disable_swap = xs_disable_swap,
4a068258 2710 .inject_disconnect = xs_inject_disconnect,
42e5c3e2
CL		 2711#ifdef CONFIG_SUNRPC_BACKCHANNEL
2712 .bc_setup = xprt_setup_bc,
6b26cc8c 2713 .bc_maxpayload = xs_tcp_bc_maxpayload,
7402a4fe 2714 .bc_num_slots = xprt_bc_max_slots,
42e5c3e2
CL		 2715 .bc_free_rqst = xprt_free_bc_rqst,
2716 .bc_destroy = xprt_destroy_bc,
2717#endif
a246b010
CL		 2718};
2719
4cfc7e60
RI		 2720/*
2721 * The rpc_xprt_ops for the server backchannel
2722 */
2723
d31ae254 2724static const struct rpc_xprt_ops bc_tcp_ops = {
4cfc7e60
RI		 2725 .reserve_xprt = xprt_reserve_xprt,
2726 .release_xprt = xprt_release_xprt,
84e28a30 2727 .alloc_slot = xprt_alloc_slot,
a9cde23a 2728 .free_slot = xprt_free_slot,
4cfc7e60
RI		 2729 .buf_alloc = bc_malloc,
2730 .buf_free = bc_free,
2731 .send_request = bc_send_request,
8ba6a92d 2732 .wait_for_reply_request = xprt_wait_for_reply_request_def,
4cfc7e60
RI		 2733 .close = bc_close,
2734 .destroy = bc_destroy,
2735 .print_stats = xs_tcp_print_stats,
d67fa4d8
JL		 2736 .enable_swap = xs_enable_swap,
2737 .disable_swap = xs_disable_swap,
4a068258 2738 .inject_disconnect = xs_inject_disconnect,
4cfc7e60
RI		 2739};
2740
92476850
CL		 2741static int xs_init_anyaddr(const int family, struct sockaddr *sap)
2742{
2743 static const struct sockaddr_in sin = {
2744 .sin_family = AF_INET,
2745 .sin_addr.s_addr = htonl(INADDR_ANY),
2746 };
2747 static const struct sockaddr_in6 sin6 = {
2748 .sin6_family = AF_INET6,
2749 .sin6_addr = IN6ADDR_ANY_INIT,
2750 };
2751
2752 switch (family) {
176e21ee
CL		 2753 case AF_LOCAL:
2754 break;
92476850
CL		 2755 case AF_INET:
2756 memcpy(sap, &sin, sizeof(sin));
2757 break;
2758 case AF_INET6:
2759 memcpy(sap, &sin6, sizeof(sin6));
2760 break;
2761 default:
2762 dprintk("RPC: %s: Bad address family\n", __func__);
2763 return -EAFNOSUPPORT;
2764 }
2765 return 0;
2766}
2767
3c341b0b 2768static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args,
d9ba131d
TM		 2769 unsigned int slot_table_size,
2770 unsigned int max_slot_table_size)
c8541ecd
CL		 2771{
2772 struct rpc_xprt *xprt;
ffc2e518 2773 struct sock_xprt *new;
c8541ecd 2774
96802a09 2775 if (args->addrlen > sizeof(xprt->addr)) {
46121cf7 2776 dprintk("RPC: xs_setup_xprt: address too large\n");
c8541ecd
CL		 2777 return ERR_PTR(-EBADF);
2778 }
2779
d9ba131d
TM		 2780 xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size,
2781 max_slot_table_size);
bd1722d4 2782 if (xprt == NULL) {
46121cf7
CL		 2783 dprintk("RPC: xs_setup_xprt: couldn't allocate "
2784 "rpc_xprt\n");
c8541ecd
CL		 2785 return ERR_PTR(-ENOMEM);
2786 }
c8541ecd 2787
bd1722d4 2788 new = container_of(xprt, struct sock_xprt, xprt);
edc1b01c 2789 mutex_init(&new->recv_mutex);
96802a09
FM		 2790 memcpy(&xprt->addr, args->dstaddr, args->addrlen);
2791 xprt->addrlen = args->addrlen;
d3bc9a1d 2792 if (args->srcaddr)
fbfffbd5 2793 memcpy(&new->srcaddr, args->srcaddr, args->addrlen);
92476850
CL		 2794 else {
2795 int err;
2796 err = xs_init_anyaddr(args->dstaddr->sa_family,
2797 (struct sockaddr *)&new->srcaddr);
2aa13531
SK		 2798 if (err != 0) {
2799 xprt_free(xprt);
92476850 2800 return ERR_PTR(err);
2aa13531 2801 }
92476850 2802 }
c8541ecd
CL		 2803
2804 return xprt;
2805}
2806
176e21ee
CL		 2807static const struct rpc_timeout xs_local_default_timeout = {
2808 .to_initval = 10 * HZ,
2809 .to_maxval = 10 * HZ,
2810 .to_retries = 2,
2811};
2812
2813/**
2814 * xs_setup_local - Set up transport to use an AF_LOCAL socket
2815 * @args: rpc transport creation arguments
2816 *
2817 * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP
2818 */
2819static struct rpc_xprt *xs_setup_local(struct xprt_create *args)
2820{
2821 struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr;
2822 struct sock_xprt *transport;
2823 struct rpc_xprt *xprt;
2824 struct rpc_xprt *ret;
2825
d9ba131d
TM		 2826 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
2827 xprt_max_tcp_slot_table_entries);
176e21ee
CL		 2828 if (IS_ERR(xprt))
2829 return xprt;
2830 transport = container_of(xprt, struct sock_xprt, xprt);
2831
2832 xprt->prot = 0;
d3abc739 2833 xprt->xprt_class = &xs_local_transport;
176e21ee
CL		 2834 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
2835
2836 xprt->bind_timeout = XS_BIND_TO;
2837 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2838 xprt->idle_timeout = XS_IDLE_DISC_TO;
2839
2840 xprt->ops = &xs_local_ops;
2841 xprt->timeout = &xs_local_default_timeout;
2842
550aebfe 2843 INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn);
4f8943f8 2844 INIT_WORK(&transport->error_worker, xs_error_handle);
550aebfe 2845 INIT_DELAYED_WORK(&transport->connect_worker, xs_dummy_setup_socket);
93dc41bd 2846
176e21ee
CL		 2847 switch (sun->sun_family) {
2848 case AF_LOCAL:
2849 if (sun->sun_path[0] != '/') {
2850 dprintk("RPC: bad AF_LOCAL address: %s\n",
2851 sun->sun_path);
2852 ret = ERR_PTR(-EINVAL);
2853 goto out_err;
2854 }
2855 xprt_set_bound(xprt);
176e21ee 2856 xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL);
7073ea87
BF		 2857 ret = ERR_PTR(xs_local_setup_socket(transport));
2858 if (ret)
2859 goto out_err;
176e21ee
CL		 2860 break;
2861 default:
2862 ret = ERR_PTR(-EAFNOSUPPORT);
2863 goto out_err;
2864 }
2865
2866 dprintk("RPC: set up xprt to %s via AF_LOCAL\n",
2867 xprt->address_strings[RPC_DISPLAY_ADDR]);
2868
2869 if (try_module_get(THIS_MODULE))
2870 return xprt;
2871 ret = ERR_PTR(-EINVAL);
2872out_err:
315f3812 2873 xs_xprt_free(xprt);
176e21ee
CL		 2874 return ret;
2875}
2876
2881ae74
TM		 2877static const struct rpc_timeout xs_udp_default_timeout = {
2878 .to_initval = 5 * HZ,
2879 .to_maxval = 30 * HZ,
2880 .to_increment = 5 * HZ,
2881 .to_retries = 5,
2882};
2883
9903cd1c
CL		 2884/**
2885 * xs_setup_udp - Set up transport to use a UDP socket
96802a09 2886 * @args: rpc transport creation arguments
9903cd1c
CL		 2887 *
2888 */
483066d6 2889static struct rpc_xprt *xs_setup_udp(struct xprt_create *args)
a246b010 2890{
8f9d5b1a 2891 struct sockaddr *addr = args->dstaddr;
c8541ecd 2892 struct rpc_xprt *xprt;
c8475461 2893 struct sock_xprt *transport;
0a68b0be 2894 struct rpc_xprt *ret;
a246b010 2895
d9ba131d
TM		 2896 xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries,
2897 xprt_udp_slot_table_entries);
c8541ecd
CL		 2898 if (IS_ERR(xprt))
2899 return xprt;
c8475461 2900 transport = container_of(xprt, struct sock_xprt, xprt);
a246b010 2901
ec739ef0 2902 xprt->prot = IPPROTO_UDP;
d3abc739 2903 xprt->xprt_class = &xs_udp_transport;
a246b010
CL		 2904 /* XXX: header size can vary due to auth type, IPv6, etc. */
2905 xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
2906
03bf4b70 2907 xprt->bind_timeout = XS_BIND_TO;
03bf4b70
CL		 2908 xprt->reestablish_timeout = XS_UDP_REEST_TO;
2909 xprt->idle_timeout = XS_IDLE_DISC_TO;
a246b010 2910
262965f5 2911 xprt->ops = &xs_udp_ops;
a246b010 2912
ba7392bb 2913 xprt->timeout = &xs_udp_default_timeout;
a246b010 2914
f9b2ee71 2915 INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn);
4f8943f8 2916 INIT_WORK(&transport->error_worker, xs_error_handle);
edc1b01c
TM		 2917 INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket);
2918
8f9d5b1a
CL		 2919 switch (addr->sa_family) {
2920 case AF_INET:
2921 if (((struct sockaddr_in *)addr)->sin_port != htons(0))
2922 xprt_set_bound(xprt);
2923
9dc3b095 2924 xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP);
8f9d5b1a
CL		 2925 break;
2926 case AF_INET6:
2927 if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
2928 xprt_set_bound(xprt);
2929
9dc3b095 2930 xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6);
8f9d5b1a
CL		 2931 break;
2932 default:
0a68b0be
BF		 2933 ret = ERR_PTR(-EAFNOSUPPORT);
2934 goto out_err;
8f9d5b1a
CL		 2935 }
2936
c740eff8
CL		 2937 if (xprt_bound(xprt))
2938 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
2939 xprt->address_strings[RPC_DISPLAY_ADDR],
2940 xprt->address_strings[RPC_DISPLAY_PORT],
2941 xprt->address_strings[RPC_DISPLAY_PROTO]);
2942 else
2943 dprintk("RPC: set up xprt to %s (autobind) via %s\n",
2944 xprt->address_strings[RPC_DISPLAY_ADDR],
2945 xprt->address_strings[RPC_DISPLAY_PROTO]);
edb267a6 2946
bc25571e
TT		 2947 if (try_module_get(THIS_MODULE))
2948 return xprt;
0a68b0be
BF		 2949 ret = ERR_PTR(-EINVAL);
2950out_err:
315f3812 2951 xs_xprt_free(xprt);
0a68b0be 2952 return ret;
a246b010
CL		 2953}
2954
2881ae74
TM		 2955static const struct rpc_timeout xs_tcp_default_timeout = {
2956 .to_initval = 60 * HZ,
2957 .to_maxval = 60 * HZ,
2958 .to_retries = 2,
2959};
2960
9903cd1c
CL		 2961/**
2962 * xs_setup_tcp - Set up transport to use a TCP socket
96802a09 2963 * @args: rpc transport creation arguments
9903cd1c
CL		 2964 *
2965 */
483066d6 2966static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args)
a246b010 2967{
8f9d5b1a 2968 struct sockaddr *addr = args->dstaddr;
c8541ecd 2969 struct rpc_xprt *xprt;
c8475461 2970 struct sock_xprt *transport;
0a68b0be 2971 struct rpc_xprt *ret;
b7993ceb
TM		 2972 unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries;
2973
2974 if (args->flags & XPRT_CREATE_INFINITE_SLOTS)
2975 max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT;
a246b010 2976
d9ba131d 2977 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
b7993ceb 2978 max_slot_table_size);
c8541ecd
CL		 2979 if (IS_ERR(xprt))
2980 return xprt;
c8475461 2981 transport = container_of(xprt, struct sock_xprt, xprt);
a246b010 2982
ec739ef0 2983 xprt->prot = IPPROTO_TCP;
d3abc739 2984 xprt->xprt_class = &xs_tcp_transport;
808012fb 2985 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
a246b010 2986
03bf4b70 2987 xprt->bind_timeout = XS_BIND_TO;
03bf4b70
CL		 2988 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2989 xprt->idle_timeout = XS_IDLE_DISC_TO;
a246b010 2990
262965f5 2991 xprt->ops = &xs_tcp_ops;
ba7392bb 2992 xprt->timeout = &xs_tcp_default_timeout;
a246b010 2993
3851f1cd 2994 xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
7196dbb0
TM		 2995 xprt->connect_timeout = xprt->timeout->to_initval *
2996 (xprt->timeout->to_retries + 1);
3851f1cd 2997
c50b8ee0 2998 INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn);
4f8943f8 2999 INIT_WORK(&transport->error_worker, xs_error_handle);
edc1b01c
TM		 3000 INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket);
3001
8f9d5b1a
CL		 3002 switch (addr->sa_family) {
3003 case AF_INET:
3004 if (((struct sockaddr_in *)addr)->sin_port != htons(0))
3005 xprt_set_bound(xprt);
3006
9dc3b095 3007 xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP);
8f9d5b1a
CL		 3008 break;
3009 case AF_INET6:
3010 if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
3011 xprt_set_bound(xprt);
3012
9dc3b095 3013 xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6);
8f9d5b1a
CL		 3014 break;
3015 default:
0a68b0be
BF		 3016 ret = ERR_PTR(-EAFNOSUPPORT);
3017 goto out_err;
8f9d5b1a
CL		 3018 }
3019
c740eff8
CL		 3020 if (xprt_bound(xprt))
3021 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3022 xprt->address_strings[RPC_DISPLAY_ADDR],
3023 xprt->address_strings[RPC_DISPLAY_PORT],
3024 xprt->address_strings[RPC_DISPLAY_PROTO]);
3025 else
3026 dprintk("RPC: set up xprt to %s (autobind) via %s\n",
3027 xprt->address_strings[RPC_DISPLAY_ADDR],
3028 xprt->address_strings[RPC_DISPLAY_PROTO]);
3029
bc25571e
TT		 3030 if (try_module_get(THIS_MODULE))
3031 return xprt;
0a68b0be
BF		 3032 ret = ERR_PTR(-EINVAL);
3033out_err:
315f3812 3034 xs_xprt_free(xprt);
0a68b0be 3035 return ret;
a246b010 3036}
282b32e1 3037
f300baba
AB		 3038/**
3039 * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket
3040 * @args: rpc transport creation arguments
3041 *
3042 */
3043static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args)
3044{
3045 struct sockaddr *addr = args->dstaddr;
3046 struct rpc_xprt *xprt;
3047 struct sock_xprt *transport;
3048 struct svc_sock *bc_sock;
0a68b0be 3049 struct rpc_xprt *ret;
f300baba 3050
d9ba131d
TM		 3051 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
3052 xprt_tcp_slot_table_entries);
f300baba
AB		 3053 if (IS_ERR(xprt))
3054 return xprt;
3055 transport = container_of(xprt, struct sock_xprt, xprt);
3056
3057 xprt->prot = IPPROTO_TCP;
d3abc739 3058 xprt->xprt_class = &xs_bc_tcp_transport;
f300baba
AB		 3059 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
3060 xprt->timeout = &xs_tcp_default_timeout;
3061
3062 /* backchannel */
3063 xprt_set_bound(xprt);
3064 xprt->bind_timeout = 0;
f300baba
AB		 3065 xprt->reestablish_timeout = 0;
3066 xprt->idle_timeout = 0;
3067
f300baba
AB		 3068 xprt->ops = &bc_tcp_ops;
3069
3070 switch (addr->sa_family) {
3071 case AF_INET:
3072 xs_format_peer_addresses(xprt, "tcp",
3073 RPCBIND_NETID_TCP);
3074 break;
3075 case AF_INET6:
3076 xs_format_peer_addresses(xprt, "tcp",
3077 RPCBIND_NETID_TCP6);
3078 break;
3079 default:
0a68b0be
BF		 3080 ret = ERR_PTR(-EAFNOSUPPORT);
3081 goto out_err;
f300baba
AB		 3082 }
3083
50fa0d40
PE		 3084 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3085 xprt->address_strings[RPC_DISPLAY_ADDR],
3086 xprt->address_strings[RPC_DISPLAY_PORT],
3087 xprt->address_strings[RPC_DISPLAY_PROTO]);
f300baba 3088
99de8ea9
BF		 3089 /*
3090 * Once we've associated a backchannel xprt with a connection,
28303ca3
WM		 3091 * we want to keep it around as long as the connection lasts,
3092 * in case we need to start using it for a backchannel again;
3093 * this reference won't be dropped until bc_xprt is destroyed.
99de8ea9
BF		 3094 */
3095 xprt_get(xprt);
3096 args->bc_xprt->xpt_bc_xprt = xprt;
3097 xprt->bc_xprt = args->bc_xprt;
3098 bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt);
3099 transport->sock = bc_sock->sk_sock;
3100 transport->inet = bc_sock->sk_sk;
3101
f300baba
AB		 3102 /*
3103 * Since we don't want connections for the backchannel, we set
3104 * the xprt status to connected
3105 */
3106 xprt_set_connected(xprt);
3107
f300baba
AB		 3108 if (try_module_get(THIS_MODULE))
3109 return xprt;
642aab58
KM		 3110
3111 args->bc_xprt->xpt_bc_xprt = NULL;
39a9beab 3112 args->bc_xprt->xpt_bc_xps = NULL;
99de8ea9 3113 xprt_put(xprt);
0a68b0be
BF		 3114 ret = ERR_PTR(-EINVAL);
3115out_err:
315f3812 3116 xs_xprt_free(xprt);
0a68b0be 3117 return ret;
f300baba
AB		 3118}
3119
176e21ee
CL		 3120static struct xprt_class xs_local_transport = {
3121 .list = LIST_HEAD_INIT(xs_local_transport.list),
3122 .name = "named UNIX socket",
3123 .owner = THIS_MODULE,
3124 .ident = XPRT_TRANSPORT_LOCAL,
3125 .setup = xs_setup_local,
d5aa6b22 3126 .netid = { "" },
176e21ee
CL		 3127};
3128
bc25571e
TT		 3129static struct xprt_class xs_udp_transport = {
3130 .list = LIST_HEAD_INIT(xs_udp_transport.list),
3131 .name = "udp",
3132 .owner = THIS_MODULE,
f300baba 3133 .ident = XPRT_TRANSPORT_UDP,
bc25571e 3134 .setup = xs_setup_udp,
d5aa6b22 3135 .netid = { "udp", "udp6", "" },
bc25571e
TT		 3136};
3137
3138static struct xprt_class xs_tcp_transport = {
3139 .list = LIST_HEAD_INIT(xs_tcp_transport.list),
3140 .name = "tcp",
3141 .owner = THIS_MODULE,
f300baba 3142 .ident = XPRT_TRANSPORT_TCP,
bc25571e 3143 .setup = xs_setup_tcp,
d5aa6b22 3144 .netid = { "tcp", "tcp6", "" },
bc25571e
TT		 3145};
3146
f300baba
AB		 3147static struct xprt_class xs_bc_tcp_transport = {
3148 .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list),
3149 .name = "tcp NFSv4.1 backchannel",
3150 .owner = THIS_MODULE,
3151 .ident = XPRT_TRANSPORT_BC_TCP,
3152 .setup = xs_setup_bc_tcp,
d5aa6b22 3153 .netid = { "" },
f300baba
AB		 3154};
3155
282b32e1 3156/**
bc25571e 3157 * init_socket_xprt - set up xprtsock's sysctls, register with RPC client
282b32e1
CL		 3158 *
3159 */
3160int init_socket_xprt(void)
3161{
2b1bec5f 3162 if (!sunrpc_table_header)
0b4d4147 3163 sunrpc_table_header = register_sysctl_table(sunrpc_table);
fbf76683 3164
176e21ee 3165 xprt_register_transport(&xs_local_transport);
bc25571e
TT		 3166 xprt_register_transport(&xs_udp_transport);
3167 xprt_register_transport(&xs_tcp_transport);
f300baba 3168 xprt_register_transport(&xs_bc_tcp_transport);
bc25571e 3169
282b32e1
CL		 3170 return 0;
3171}
3172
3173/**
bc25571e 3174 * cleanup_socket_xprt - remove xprtsock's sysctls, unregister
282b32e1
CL		 3175 *
3176 */
3177void cleanup_socket_xprt(void)
3178{
fbf76683
CL		 3179 if (sunrpc_table_header) {
3180 unregister_sysctl_table(sunrpc_table_header);
3181 sunrpc_table_header = NULL;
3182 }
bc25571e 3183
176e21ee 3184 xprt_unregister_transport(&xs_local_transport);
bc25571e
TT		 3185 xprt_unregister_transport(&xs_udp_transport);
3186 xprt_unregister_transport(&xs_tcp_transport);
f300baba 3187 xprt_unregister_transport(&xs_bc_tcp_transport);
282b32e1 3188}
cbf11071 3189
9bbb9e5a 3190static int param_set_portnr(const char *val, const struct kernel_param *kp)
cbf11071 3191{
ffb6ca33 3192 return param_set_uint_minmax(val, kp,
826799e6 3193 RPC_MIN_RESVPORT,
cbf11071
TM		 3194 RPC_MAX_RESVPORT);
3195}
3196
9c27847d 3197static const struct kernel_param_ops param_ops_portnr = {
9bbb9e5a
RR		 3198 .set = param_set_portnr,
3199 .get = param_get_uint,
3200};
3201
cbf11071
TM		 3202#define param_check_portnr(name, p) \
3203 __param_check(name, p, unsigned int);
3204
3205module_param_named(min_resvport, xprt_min_resvport, portnr, 0644);
3206module_param_named(max_resvport, xprt_max_resvport, portnr, 0644);
3207
9bbb9e5a
RR		 3208static int param_set_slot_table_size(const char *val,
3209 const struct kernel_param *kp)
cbf11071
TM		 3210{
3211 return param_set_uint_minmax(val, kp,
3212 RPC_MIN_SLOT_TABLE,
3213 RPC_MAX_SLOT_TABLE);
3214}
3215
9c27847d 3216static const struct kernel_param_ops param_ops_slot_table_size = {
9bbb9e5a
RR		 3217 .set = param_set_slot_table_size,
3218 .get = param_get_uint,
3219};
3220
cbf11071
TM		 3221#define param_check_slot_table_size(name, p) \
3222 __param_check(name, p, unsigned int);
3223
d9ba131d
TM		 3224static int param_set_max_slot_table_size(const char *val,
3225 const struct kernel_param *kp)
3226{
3227 return param_set_uint_minmax(val, kp,
3228 RPC_MIN_SLOT_TABLE,
3229 RPC_MAX_SLOT_TABLE_LIMIT);
3230}
3231
9c27847d 3232static const struct kernel_param_ops param_ops_max_slot_table_size = {
d9ba131d
TM		 3233 .set = param_set_max_slot_table_size,
3234 .get = param_get_uint,
3235};
3236
3237#define param_check_max_slot_table_size(name, p) \
3238 __param_check(name, p, unsigned int);
3239
cbf11071
TM		 3240module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries,
3241 slot_table_size, 0644);
d9ba131d
TM		 3242module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries,
3243 max_slot_table_size, 0644);
cbf11071
TM		 3244module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries,
3245 slot_table_size, 0644);
Linux 6.x block layer and io_uring tree(s)