projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
SUNRPC display xprt's main value in sysfs's xprt_info
[linux-block.git] / net / sunrpc / xprtsock.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
a246b010
CL		 2/*
3 * linux/net/sunrpc/xprtsock.c
4 *
5 * Client-side transport implementation for sockets.
6 *
113aa838
AC		 7 * TCP callback races fixes (C) 1998 Red Hat
8 * TCP send fixes (C) 1998 Red Hat
a246b010
CL		 9 * TCP NFS related read + write fixes
10 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
11 *
12 * Rewrite of larges part of the code in order to stabilize TCP stuff.
13 * Fix behaviour when socket buffer is full.
14 * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no>
55aa4f58
CL		 15 *
16 * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com>
8f9d5b1a
CL		 17 *
18 * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005.
19 * <gilles.quillard@bull.net>
a246b010
CL		 20 */
21
22#include <linux/types.h>
176e21ee 23#include <linux/string.h>
a246b010 24#include <linux/slab.h>
bc25571e 25#include <linux/module.h>
a246b010 26#include <linux/capability.h>
a246b010
CL		 27#include <linux/pagemap.h>
28#include <linux/errno.h>
29#include <linux/socket.h>
30#include <linux/in.h>
31#include <linux/net.h>
32#include <linux/mm.h>
176e21ee 33#include <linux/un.h>
a246b010
CL		 34#include <linux/udp.h>
35#include <linux/tcp.h>
36#include <linux/sunrpc/clnt.h>
5976687a 37#include <linux/sunrpc/addr.h>
02107148 38#include <linux/sunrpc/sched.h>
4cfc7e60 39#include <linux/sunrpc/svcsock.h>
49c36fcc 40#include <linux/sunrpc/xprtsock.h>
a246b010 41#include <linux/file.h>
9e00abc3 42#ifdef CONFIG_SUNRPC_BACKCHANNEL
44b98efd
RL		 43#include <linux/sunrpc/bc_xprt.h>
44#endif
a246b010
CL		 45
46#include <net/sock.h>
47#include <net/checksum.h>
48#include <net/udp.h>
49#include <net/tcp.h>
277e4ab7 50#include <linux/bvec.h>
6a829eb8 51#include <linux/highmem.h>
277e4ab7 52#include <linux/uio.h>
a1231fda 53#include <linux/sched/mm.h>
a246b010 54
40b5ea0c
TM		 55#include <trace/events/sunrpc.h>
56
9e55eef4 57#include "socklib.h"
4cfc7e60 58#include "sunrpc.h"
176e21ee
CL		 59
60static void xs_close(struct rpc_xprt *xprt);
7196dbb0
TM		 61static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
62 struct socket *sock);
176e21ee 63
c556b754
CL		 64/*
65 * xprtsock tunables
66 */
09acfea5
TM		 67static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE;
68static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE;
69static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE;
c556b754 70
09acfea5
TM		 71static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT;
72static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT;
c556b754 73
7d1e8255 74#define XS_TCP_LINGER_TO (15U * HZ)
25fe6142 75static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;
7d1e8255 76
fbf76683
CL		 77/*
78 * We can register our own files under /proc/sys/sunrpc by
79 * calling register_sysctl_table() again. The files in that
80 * directory become the union of all files registered there.
81 *
82 * We simply need to make sure that we don't collide with
83 * someone else's file names!
84 */
85
fbf76683
CL		 86static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE;
87static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE;
d9ba131d 88static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT;
fbf76683
CL		 89static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT;
90static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT;
91
92static struct ctl_table_header *sunrpc_table_header;
93
d3abc739
OK		 94static struct xprt_class xs_local_transport;
95static struct xprt_class xs_udp_transport;
96static struct xprt_class xs_tcp_transport;
97static struct xprt_class xs_bc_tcp_transport;
98
fbf76683
CL		 99/*
100 * FIXME: changing the UDP slot table size should also resize the UDP
101 * socket buffers for existing UDP transports
102 */
fe2c6338 103static struct ctl_table xs_tunables_table[] = {
fbf76683 104 {
fbf76683
CL		 105 .procname = "udp_slot_table_entries",
106 .data = &xprt_udp_slot_table_entries,
107 .maxlen = sizeof(unsigned int),
108 .mode = 0644,
6d456111 109 .proc_handler = proc_dointvec_minmax,
fbf76683
CL		 110 .extra1 = &min_slot_table_size,
111 .extra2 = &max_slot_table_size
112 },
113 {
fbf76683
CL		 114 .procname = "tcp_slot_table_entries",
115 .data = &xprt_tcp_slot_table_entries,
116 .maxlen = sizeof(unsigned int),
117 .mode = 0644,
6d456111 118 .proc_handler = proc_dointvec_minmax,
fbf76683
CL		 119 .extra1 = &min_slot_table_size,
120 .extra2 = &max_slot_table_size
121 },
d9ba131d
TM		 122 {
123 .procname = "tcp_max_slot_table_entries",
124 .data = &xprt_max_tcp_slot_table_entries,
125 .maxlen = sizeof(unsigned int),
126 .mode = 0644,
127 .proc_handler = proc_dointvec_minmax,
128 .extra1 = &min_slot_table_size,
129 .extra2 = &max_tcp_slot_table_limit
130 },
fbf76683 131 {
fbf76683
CL		 132 .procname = "min_resvport",
133 .data = &xprt_min_resvport,
134 .maxlen = sizeof(unsigned int),
135 .mode = 0644,
6d456111 136 .proc_handler = proc_dointvec_minmax,
fbf76683 137 .extra1 = &xprt_min_resvport_limit,
826799e6 138 .extra2 = &xprt_max_resvport_limit
fbf76683
CL		 139 },
140 {
fbf76683
CL		 141 .procname = "max_resvport",
142 .data = &xprt_max_resvport,
143 .maxlen = sizeof(unsigned int),
144 .mode = 0644,
6d456111 145 .proc_handler = proc_dointvec_minmax,
826799e6 146 .extra1 = &xprt_min_resvport_limit,
fbf76683
CL		 147 .extra2 = &xprt_max_resvport_limit
148 },
25fe6142
TM		 149 {
150 .procname = "tcp_fin_timeout",
151 .data = &xs_tcp_fin_timeout,
152 .maxlen = sizeof(xs_tcp_fin_timeout),
153 .mode = 0644,
6d456111 154 .proc_handler = proc_dointvec_jiffies,
fbf76683 155 },
f8572d8f 156 { },
fbf76683
CL		 157};
158
fe2c6338 159static struct ctl_table sunrpc_table[] = {
fbf76683 160 {
fbf76683
CL		 161 .procname = "sunrpc",
162 .mode = 0555,
163 .child = xs_tunables_table
164 },
f8572d8f 165 { },
fbf76683
CL		 166};
167
03bf4b70
CL		 168/*
169 * Wait duration for a reply from the RPC portmapper.
170 */
171#define XS_BIND_TO (60U * HZ)
172
173/*
174 * Delay if a UDP socket connect error occurs. This is most likely some
175 * kind of resource problem on the local host.
176 */
177#define XS_UDP_REEST_TO (2U * HZ)
178
179/*
180 * The reestablish timeout allows clients to delay for a bit before attempting
181 * to reconnect to a server that just dropped our connection.
182 *
183 * We implement an exponential backoff when trying to reestablish a TCP
184 * transport connection with the server. Some servers like to drop a TCP
185 * connection when they are overworked, so we start with a short timeout and
186 * increase over time if the server is down or not responding.
187 */
188#define XS_TCP_INIT_REEST_TO (3U * HZ)
03bf4b70
CL		 189
190/*
191 * TCP idle timeout; client drops the transport socket if it is idle
192 * for this long. Note that we also timeout UDP sockets to prevent
193 * holding port numbers when there is no RPC traffic.
194 */
195#define XS_IDLE_DISC_TO (5U * 60 * HZ)
196
f895b252 197#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
a246b010 198# undef RPC_DEBUG_DATA
9903cd1c 199# define RPCDBG_FACILITY RPCDBG_TRANS
a246b010
CL		 200#endif
201
a246b010 202#ifdef RPC_DEBUG_DATA
9903cd1c 203static void xs_pktdump(char *msg, u32 *packet, unsigned int count)
a246b010 204{
9903cd1c
CL		 205 u8 *buf = (u8 *) packet;
206 int j;
a246b010 207
46121cf7 208 dprintk("RPC: %s\n", msg);
a246b010
CL		 209 for (j = 0; j < count && j < 128; j += 4) {
210 if (!(j & 31)) {
211 if (j)
212 dprintk("\n");
213 dprintk("0x%04x ", j);
214 }
215 dprintk("%02x%02x%02x%02x ",
216 buf[j], buf[j+1], buf[j+2], buf[j+3]);
217 }
218 dprintk("\n");
219}
220#else
9903cd1c 221static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count)
a246b010
CL		 222{
223 /* NOP */
224}
225#endif
226
2118071d
TM		 227static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)
228{
229 return (struct rpc_xprt *) sk->sk_user_data;
230}
231
95392c59
CL		 232static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt)
233{
234 return (struct sockaddr *) &xprt->addr;
235}
236
176e21ee
CL		 237static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt)
238{
239 return (struct sockaddr_un *) &xprt->addr;
240}
241
95392c59 242static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt)
edb267a6 243{
95392c59
CL		 244 return (struct sockaddr_in *) &xprt->addr;
245}
246
247static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt)
248{
249 return (struct sockaddr_in6 *) &xprt->addr;
250}
251
c877b849 252static void xs_format_common_peer_addresses(struct rpc_xprt *xprt)
edb267a6 253{
c877b849 254 struct sockaddr *sap = xs_addr(xprt);
9dc3b095
CL		 255 struct sockaddr_in6 *sin6;
256 struct sockaddr_in *sin;
176e21ee 257 struct sockaddr_un *sun;
c877b849 258 char buf[128];
edb267a6 259
9dc3b095 260 switch (sap->sa_family) {
176e21ee
CL		 261 case AF_LOCAL:
262 sun = xs_addr_un(xprt);
263 strlcpy(buf, sun->sun_path, sizeof(buf));
264 xprt->address_strings[RPC_DISPLAY_ADDR] =
265 kstrdup(buf, GFP_KERNEL);
266 break;
9dc3b095 267 case AF_INET:
176e21ee
CL		 268 (void)rpc_ntop(sap, buf, sizeof(buf));
269 xprt->address_strings[RPC_DISPLAY_ADDR] =
270 kstrdup(buf, GFP_KERNEL);
9dc3b095 271 sin = xs_addr_in(xprt);
fc0b5791 272 snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));
9dc3b095
CL		 273 break;
274 case AF_INET6:
176e21ee
CL		 275 (void)rpc_ntop(sap, buf, sizeof(buf));
276 xprt->address_strings[RPC_DISPLAY_ADDR] =
277 kstrdup(buf, GFP_KERNEL);
9dc3b095 278 sin6 = xs_addr_in6(xprt);
fc0b5791 279 snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);
9dc3b095
CL		 280 break;
281 default:
282 BUG();
756805e7 283 }
176e21ee 284
9dc3b095 285 xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);
edb267a6
CL		 286}
287
9dc3b095 288static void xs_format_common_peer_ports(struct rpc_xprt *xprt)
4b6473fb 289{
9dc3b095
CL		 290 struct sockaddr *sap = xs_addr(xprt);
291 char buf[128];
4b6473fb 292
81160e66 293 snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap));
c877b849 294 xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);
4b6473fb 295
81160e66 296 snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap));
c877b849
CL		 297 xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
298}
4b6473fb 299
9dc3b095
CL		 300static void xs_format_peer_addresses(struct rpc_xprt *xprt,
301 const char *protocol,
302 const char *netid)
c877b849 303{
b454ae90 304 xprt->address_strings[RPC_DISPLAY_PROTO] = protocol;
b454ae90 305 xprt->address_strings[RPC_DISPLAY_NETID] = netid;
c877b849 306 xs_format_common_peer_addresses(xprt);
9dc3b095 307 xs_format_common_peer_ports(xprt);
edb267a6 308}
4b6473fb 309
9dc3b095 310static void xs_update_peer_port(struct rpc_xprt *xprt)
4b6473fb 311{
9dc3b095
CL		 312 kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
313 kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
4417c8c4 314
9dc3b095 315 xs_format_common_peer_ports(xprt);
edb267a6
CL		 316}
317
318static void xs_free_peer_addresses(struct rpc_xprt *xprt)
319{
33e01dc7
CL		 320 unsigned int i;
321
322 for (i = 0; i < RPC_DISPLAY_MAX; i++)
323 switch (i) {
324 case RPC_DISPLAY_PROTO:
325 case RPC_DISPLAY_NETID:
326 continue;
327 default:
328 kfree(xprt->address_strings[i]);
329 }
edb267a6
CL		 330}
331
277e4ab7
TM		 332static size_t
333xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp)
334{
335 size_t i,n;
336
16e5e90f 337 if (!want || !(buf->flags & XDRBUF_SPARSE_PAGES))
277e4ab7 338 return want;
277e4ab7
TM		 339 n = (buf->page_base + want + PAGE_SIZE - 1) >> PAGE_SHIFT;
340 for (i = 0; i < n; i++) {
341 if (buf->pages[i])
342 continue;
343 buf->bvec[i].bv_page = buf->pages[i] = alloc_page(gfp);
344 if (!buf->pages[i]) {
16e5e90f
TM		 345 i *= PAGE_SIZE;
346 return i > buf->page_base ? i - buf->page_base : 0;
277e4ab7
TM		 347 }
348 }
349 return want;
350}
351
352static ssize_t
353xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek)
354{
355 ssize_t ret;
356 if (seek != 0)
357 iov_iter_advance(&msg->msg_iter, seek);
358 ret = sock_recvmsg(sock, msg, flags);
359 return ret > 0 ? ret + seek : ret;
360}
361
362static ssize_t
363xs_read_kvec(struct socket *sock, struct msghdr *msg, int flags,
364 struct kvec *kvec, size_t count, size_t seek)
365{
0e9b4a82 366 iov_iter_kvec(&msg->msg_iter, READ, kvec, 1, count);
277e4ab7
TM		 367 return xs_sock_recvmsg(sock, msg, flags, seek);
368}
369
370static ssize_t
371xs_read_bvec(struct socket *sock, struct msghdr *msg, int flags,
372 struct bio_vec *bvec, unsigned long nr, size_t count,
373 size_t seek)
374{
0e9b4a82 375 iov_iter_bvec(&msg->msg_iter, READ, bvec, nr, count);
277e4ab7
TM		 376 return xs_sock_recvmsg(sock, msg, flags, seek);
377}
378
379static ssize_t
380xs_read_discard(struct socket *sock, struct msghdr *msg, int flags,
381 size_t count)
382{
b76a5afd
TM		 383 iov_iter_discard(&msg->msg_iter, READ, count);
384 return sock_recvmsg(sock, msg, flags);
277e4ab7
TM		 385}
386
6a829eb8
TM		 387#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
388static void
389xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek)
390{
391 struct bvec_iter bi = {
392 .bi_size = count,
393 };
394 struct bio_vec bv;
395
396 bvec_iter_advance(bvec, &bi, seek & PAGE_MASK);
397 for_each_bvec(bv, bvec, bi, bi)
398 flush_dcache_page(bv.bv_page);
399}
400#else
401static inline void
402xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek)
403{
404}
405#endif
406
277e4ab7
TM		 407static ssize_t
408xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags,
409 struct xdr_buf *buf, size_t count, size_t seek, size_t *read)
410{
411 size_t want, seek_init = seek, offset = 0;
412 ssize_t ret;
413
e92053a5
TM		 414 want = min_t(size_t, count, buf->head[0].iov_len);
415 if (seek < want) {
277e4ab7
TM		 416 ret = xs_read_kvec(sock, msg, flags, &buf->head[0], want, seek);
417 if (ret <= 0)
418 goto sock_err;
419 offset += ret;
420 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
421 goto out;
422 if (ret != want)
c4433055 423 goto out;
277e4ab7
TM		 424 seek = 0;
425 } else {
e92053a5
TM		 426 seek -= want;
427 offset += want;
277e4ab7 428 }
16e5e90f
TM		 429
430 want = xs_alloc_sparse_pages(buf,
431 min_t(size_t, count - offset, buf->page_len),
12a3ad61 432 GFP_KERNEL);
16e5e90f 433 if (seek < want) {
277e4ab7
TM		 434 ret = xs_read_bvec(sock, msg, flags, buf->bvec,
435 xdr_buf_pagecount(buf),
436 want + buf->page_base,
437 seek + buf->page_base);
438 if (ret <= 0)
439 goto sock_err;
6a829eb8 440 xs_flush_bvec(buf->bvec, ret, seek + buf->page_base);
ac9645c8
DA		 441 ret -= buf->page_base;
442 offset += ret;
277e4ab7
TM		 443 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
444 goto out;
445 if (ret != want)
c4433055 446 goto out;
277e4ab7
TM		 447 seek = 0;
448 } else {
16e5e90f
TM		 449 seek -= want;
450 offset += want;
277e4ab7 451 }
16e5e90f 452
e92053a5
TM		 453 want = min_t(size_t, count - offset, buf->tail[0].iov_len);
454 if (seek < want) {
277e4ab7
TM		 455 ret = xs_read_kvec(sock, msg, flags, &buf->tail[0], want, seek);
456 if (ret <= 0)
457 goto sock_err;
458 offset += ret;
459 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
460 goto out;
461 if (ret != want)
c4433055 462 goto out;
9734ad57 463 } else if (offset < seek_init)
e92053a5 464 offset = seek_init;
277e4ab7 465 ret = -EMSGSIZE;
277e4ab7
TM		 466out:
467 *read = offset - seek_init;
468 return ret;
277e4ab7
TM		 469sock_err:
470 offset += seek;
471 goto out;
472}
473
474static void
475xs_read_header(struct sock_xprt *transport, struct xdr_buf *buf)
476{
477 if (!transport->recv.copied) {
478 if (buf->head[0].iov_len >= transport->recv.offset)
479 memcpy(buf->head[0].iov_base,
480 &transport->recv.xid,
481 transport->recv.offset);
482 transport->recv.copied = transport->recv.offset;
483 }
484}
485
486static bool
487xs_read_stream_request_done(struct sock_xprt *transport)
488{
489 return transport->recv.fraghdr & cpu_to_be32(RPC_LAST_STREAM_FRAGMENT);
490}
491
e92053a5
TM		 492static void
493xs_read_stream_check_eor(struct sock_xprt *transport,
494 struct msghdr *msg)
495{
496 if (xs_read_stream_request_done(transport))
497 msg->msg_flags |= MSG_EOR;
498}
499
277e4ab7
TM		 500static ssize_t
501xs_read_stream_request(struct sock_xprt *transport, struct msghdr *msg,
502 int flags, struct rpc_rqst *req)
503{
504 struct xdr_buf *buf = &req->rq_private_buf;
3f649ab7
KC		 505 size_t want, read;
506 ssize_t ret;
277e4ab7
TM		 507
508 xs_read_header(transport, buf);
509
510 want = transport->recv.len - transport->recv.offset;
e92053a5
TM		 511 if (want != 0) {
512 ret = xs_read_xdr_buf(transport->sock, msg, flags, buf,
513 transport->recv.copied + want,
514 transport->recv.copied,
515 &read);
516 transport->recv.offset += read;
517 transport->recv.copied += read;
727fcc64 518 }
e92053a5 519
727fcc64 520 if (transport->recv.offset == transport->recv.len)
e92053a5 521 xs_read_stream_check_eor(transport, msg);
277e4ab7 522
e92053a5
TM		 523 if (want == 0)
524 return 0;
525
277e4ab7 526 switch (ret) {
c4433055
TM		 527 default:
528 break;
26781eab 529 case -EFAULT:
277e4ab7 530 case -EMSGSIZE:
26781eab 531 msg->msg_flags |= MSG_TRUNC;
c4433055 532 return read;
277e4ab7
TM		 533 case 0:
534 return -ESHUTDOWN;
277e4ab7 535 }
c4433055 536 return ret < 0 ? ret : read;
277e4ab7
TM		 537}
538
539static size_t
540xs_read_stream_headersize(bool isfrag)
541{
542 if (isfrag)
543 return sizeof(__be32);
544 return 3 * sizeof(__be32);
545}
546
547static ssize_t
548xs_read_stream_header(struct sock_xprt *transport, struct msghdr *msg,
549 int flags, size_t want, size_t seek)
550{
551 struct kvec kvec = {
552 .iov_base = &transport->recv.fraghdr,
553 .iov_len = want,
554 };
555 return xs_read_kvec(transport->sock, msg, flags, &kvec, want, seek);
556}
557
558#if defined(CONFIG_SUNRPC_BACKCHANNEL)
559static ssize_t
560xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
561{
562 struct rpc_xprt *xprt = &transport->xprt;
563 struct rpc_rqst *req;
564 ssize_t ret;
565
98b5cee3
BC		 566 /* Is this transport associated with the backchannel? */
567 if (!xprt->bc_serv)
568 return -ESHUTDOWN;
569
277e4ab7
TM		 570 /* Look up and lock the request corresponding to the given XID */
571 req = xprt_lookup_bc_request(xprt, transport->recv.xid);
572 if (!req) {
573 printk(KERN_WARNING "Callback slot table overflowed\n");
574 return -ESHUTDOWN;
575 }
45835a63
TM		 576 if (transport->recv.copied && !req->rq_private_buf.len)
577 return -ESHUTDOWN;
277e4ab7
TM		 578
579 ret = xs_read_stream_request(transport, msg, flags, req);
580 if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
c4433055 581 xprt_complete_bc_request(req, transport->recv.copied);
45835a63
TM		 582 else
583 req->rq_private_buf.len = transport->recv.copied;
277e4ab7
TM		 584
585 return ret;
586}
587#else /* CONFIG_SUNRPC_BACKCHANNEL */
588static ssize_t
589xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
590{
591 return -ESHUTDOWN;
592}
593#endif /* CONFIG_SUNRPC_BACKCHANNEL */
594
595static ssize_t
596xs_read_stream_reply(struct sock_xprt *transport, struct msghdr *msg, int flags)
597{
598 struct rpc_xprt *xprt = &transport->xprt;
599 struct rpc_rqst *req;
600 ssize_t ret = 0;
601
602 /* Look up and lock the request corresponding to the given XID */
603 spin_lock(&xprt->queue_lock);
604 req = xprt_lookup_rqst(xprt, transport->recv.xid);
45835a63 605 if (!req || (transport->recv.copied && !req->rq_private_buf.len)) {
277e4ab7
TM		 606 msg->msg_flags |= MSG_TRUNC;
607 goto out;
608 }
609 xprt_pin_rqst(req);
610 spin_unlock(&xprt->queue_lock);
611
612 ret = xs_read_stream_request(transport, msg, flags, req);
613
614 spin_lock(&xprt->queue_lock);
615 if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
c4433055 616 xprt_complete_rqst(req->rq_task, transport->recv.copied);
45835a63
TM		 617 else
618 req->rq_private_buf.len = transport->recv.copied;
277e4ab7
TM		 619 xprt_unpin_rqst(req);
620out:
621 spin_unlock(&xprt->queue_lock);
622 return ret;
623}
624
625static ssize_t
626xs_read_stream(struct sock_xprt *transport, int flags)
627{
628 struct msghdr msg = { 0 };
629 size_t want, read = 0;
630 ssize_t ret = 0;
631
632 if (transport->recv.len == 0) {
633 want = xs_read_stream_headersize(transport->recv.copied != 0);
634 ret = xs_read_stream_header(transport, &msg, flags, want,
635 transport->recv.offset);
636 if (ret <= 0)
637 goto out_err;
638 transport->recv.offset = ret;
c4433055
TM		 639 if (transport->recv.offset != want)
640 return transport->recv.offset;
277e4ab7
TM		 641 transport->recv.len = be32_to_cpu(transport->recv.fraghdr) &
642 RPC_FRAGMENT_SIZE_MASK;
643 transport->recv.offset -= sizeof(transport->recv.fraghdr);
644 read = ret;
645 }
646
647 switch (be32_to_cpu(transport->recv.calldir)) {
c4433055
TM		 648 default:
649 msg.msg_flags |= MSG_TRUNC;
650 break;
277e4ab7
TM		 651 case RPC_CALL:
652 ret = xs_read_stream_call(transport, &msg, flags);
653 break;
654 case RPC_REPLY:
655 ret = xs_read_stream_reply(transport, &msg, flags);
656 }
657 if (msg.msg_flags & MSG_TRUNC) {
658 transport->recv.calldir = cpu_to_be32(-1);
659 transport->recv.copied = -1;
660 }
661 if (ret < 0)
662 goto out_err;
663 read += ret;
664 if (transport->recv.offset < transport->recv.len) {
c4433055
TM		 665 if (!(msg.msg_flags & MSG_TRUNC))
666 return read;
b76a5afd 667 msg.msg_flags = 0;
277e4ab7
TM		 668 ret = xs_read_discard(transport->sock, &msg, flags,
669 transport->recv.len - transport->recv.offset);
670 if (ret <= 0)
671 goto out_err;
672 transport->recv.offset += ret;
673 read += ret;
674 if (transport->recv.offset != transport->recv.len)
c4433055 675 return read;
277e4ab7
TM		 676 }
677 if (xs_read_stream_request_done(transport)) {
c50b8ee0 678 trace_xs_stream_read_request(transport);
277e4ab7
TM		 679 transport->recv.copied = 0;
680 }
681 transport->recv.offset = 0;
682 transport->recv.len = 0;
683 return read;
684out_err:
79462857 685 return ret != 0 ? ret : -ESHUTDOWN;
277e4ab7
TM		 686}
687
0ffe86f4
TM		 688static __poll_t xs_poll_socket(struct sock_xprt *transport)
689{
a73881c9
TM		 690 return transport->sock->ops->poll(transport->file, transport->sock,
691 NULL);
0ffe86f4
TM		 692}
693
694static bool xs_poll_socket_readable(struct sock_xprt *transport)
695{
696 __poll_t events = xs_poll_socket(transport);
697
698 return (events & (EPOLLIN | EPOLLRDNORM)) && !(events & EPOLLRDHUP);
699}
700
701static void xs_poll_check_readable(struct sock_xprt *transport)
702{
703
704 clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
705 if (!xs_poll_socket_readable(transport))
706 return;
707 if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
708 queue_work(xprtiod_workqueue, &transport->recv_worker);
709}
710
c50b8ee0
TM		 711static void xs_stream_data_receive(struct sock_xprt *transport)
712{
713 size_t read = 0;
714 ssize_t ret = 0;
715
716 mutex_lock(&transport->recv_mutex);
717 if (transport->sock == NULL)
718 goto out;
c50b8ee0
TM		 719 for (;;) {
720 ret = xs_read_stream(transport, MSG_DONTWAIT);
c4433055 721 if (ret < 0)
c50b8ee0
TM		 722 break;
723 read += ret;
724 cond_resched();
725 }
5f52a9d4
TM		 726 if (ret == -ESHUTDOWN)
727 kernel_sock_shutdown(transport->sock, SHUT_RDWR);
728 else
729 xs_poll_check_readable(transport);
c50b8ee0
TM		 730out:
731 mutex_unlock(&transport->recv_mutex);
732 trace_xs_stream_read_data(&transport->xprt, ret, read);
733}
734
735static void xs_stream_data_receive_workfn(struct work_struct *work)
736{
737 struct sock_xprt *transport =
738 container_of(work, struct sock_xprt, recv_worker);
a1231fda
TM		 739 unsigned int pflags = memalloc_nofs_save();
740
c50b8ee0 741 xs_stream_data_receive(transport);
a1231fda 742 memalloc_nofs_restore(pflags);
c50b8ee0
TM		 743}
744
550aebfe
TM		 745static void
746xs_stream_reset_connect(struct sock_xprt *transport)
747{
748 transport->recv.offset = 0;
749 transport->recv.len = 0;
750 transport->recv.copied = 0;
751 transport->xmit.offset = 0;
ae053551
TM		 752}
753
754static void
755xs_stream_start_connect(struct sock_xprt *transport)
756{
550aebfe
TM		 757 transport->xprt.stat.connect_count++;
758 transport->xprt.stat.connect_start = jiffies;
759}
760
b4b5cc85
CL		 761#define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
762
9903cd1c 763/**
c544577d 764 * xs_nospace - handle transmit was incomplete
50f484e2 765 * @req: pointer to RPC request
9903cd1c 766 *
a246b010 767 */
c544577d 768static int xs_nospace(struct rpc_rqst *req)
a246b010 769{
262965f5 770 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 771 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
06ea0bfe 772 struct sock *sk = transport->inet;
24ca9a84 773 int ret = -EAGAIN;
a246b010 774
015747d2 775 trace_rpc_socket_nospace(req, transport);
262965f5 776
b6ddf64f 777 /* Protect against races with write_space */
b5e92419 778 spin_lock(&xprt->transport_lock);
b6ddf64f
TM		 779
780 /* Don't race with disconnect */
781 if (xprt_connected(xprt)) {
13331a55
TM		 782 /* wait for more buffer space */
783 sk->sk_write_pending++;
c544577d 784 xprt_wait_for_buffer_space(xprt);
13331a55 785 } else
5e3771ce 786 ret = -ENOTCONN;
262965f5 787
b5e92419 788 spin_unlock(&xprt->transport_lock);
06ea0bfe
TM		 789
790 /* Race breaker in case memory is freed before above code is called */
d48f9ce7
DV		 791 if (ret == -EAGAIN) {
792 struct socket_wq *wq;
793
794 rcu_read_lock();
795 wq = rcu_dereference(sk->sk_wq);
796 set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags);
797 rcu_read_unlock();
798
799 sk->sk_write_space(sk);
800 }
5e3771ce 801 return ret;
262965f5
CL		 802}
803
277e4ab7
TM		 804static void
805xs_stream_prepare_request(struct rpc_rqst *req)
806{
75369089 807 xdr_free_bvec(&req->rq_rcv_buf);
12a3ad61 808 req->rq_task->tk_status = xdr_alloc_bvec(&req->rq_rcv_buf, GFP_KERNEL);
277e4ab7
TM		 809}
810
4cd34e7c
TM		 811/*
812 * Determine if the previous message in the stream was aborted before it
813 * could complete transmission.
814 */
815static bool
816xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req)
817{
818 return transport->xmit.offset != 0 && req->rq_bytes_sent == 0;
819}
820
61677eee 821/*
06b5fc3a 822 * Return the stream record marker field for a record of length < 2^31-1
61677eee 823 */
06b5fc3a
TM		 824static rpc_fraghdr
825xs_stream_record_marker(struct xdr_buf *xdr)
61677eee 826{
06b5fc3a
TM		 827 if (!xdr->len)
828 return 0;
829 return cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | (u32)xdr->len);
61677eee
CL		 830}
831
176e21ee
CL		 832/**
833 * xs_local_send_request - write an RPC request to an AF_LOCAL socket
50f484e2 834 * @req: pointer to RPC request
176e21ee
CL		 835 *
836 * Return values:
837 * 0: The request has been sent
838 * EAGAIN: The socket was blocked, please call again later to
839 * complete the request
840 * ENOTCONN: Caller needs to invoke connect logic then call again
12b20ce3 841 * other: Some other error occurred, the request was not sent
176e21ee 842 */
adfa7144 843static int xs_local_send_request(struct rpc_rqst *req)
176e21ee 844{
176e21ee
CL		 845 struct rpc_xprt *xprt = req->rq_xprt;
846 struct sock_xprt *transport =
847 container_of(xprt, struct sock_xprt, xprt);
848 struct xdr_buf *xdr = &req->rq_snd_buf;
7e3d3620
TM		 849 rpc_fraghdr rm = xs_stream_record_marker(xdr);
850 unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen;
9e55eef4
CL		 851 struct msghdr msg = {
852 .msg_flags = XS_SENDMSG_FLAGS,
853 };
3f649ab7 854 unsigned int sent;
176e21ee
CL		 855 int status;
856
4cd34e7c
TM		 857 /* Close the stream if the previous transmission was incomplete */
858 if (xs_send_request_was_aborted(transport, req)) {
859 xs_close(xprt);
860 return -ENOTCONN;
861 }
862
176e21ee
CL		 863 xs_pktdump("packet data:",
864 req->rq_svec->iov_base, req->rq_svec->iov_len);
865
78215759 866 req->rq_xtime = ktime_get();
9e55eef4
CL		 867 status = xprt_sock_sendmsg(transport->sock, &msg, xdr,
868 transport->xmit.offset, rm, &sent);
176e21ee 869 dprintk("RPC: %s(%u) = %d\n",
6c7a64e5 870 __func__, xdr->len - transport->xmit.offset, status);
743c69e7
N		 871
872 if (status == -EAGAIN && sock_writeable(transport->inet))
873 status = -ENOBUFS;
874
f279cd00 875 if (likely(sent > 0) || status == 0) {
6c7a64e5
TM		 876 transport->xmit.offset += sent;
877 req->rq_bytes_sent = transport->xmit.offset;
7e3d3620 878 if (likely(req->rq_bytes_sent >= msglen)) {
6c7a64e5 879 req->rq_xmit_bytes_sent += transport->xmit.offset;
6c7a64e5 880 transport->xmit.offset = 0;
176e21ee
CL		 881 return 0;
882 }
883 status = -EAGAIN;
884 }
885
886 switch (status) {
3601c4a9 887 case -ENOBUFS:
b5872f0c 888 break;
176e21ee 889 case -EAGAIN:
c544577d 890 status = xs_nospace(req);
176e21ee
CL		 891 break;
892 default:
893 dprintk("RPC: sendmsg returned unrecognized error %d\n",
894 -status);
df561f66 895 fallthrough;
176e21ee
CL		 896 case -EPIPE:
897 xs_close(xprt);
898 status = -ENOTCONN;
899 }
900
901 return status;
902}
903
262965f5
CL		 904/**
905 * xs_udp_send_request - write an RPC request to a UDP socket
50f484e2 906 * @req: pointer to RPC request
262965f5
CL		 907 *
908 * Return values:
909 * 0: The request has been sent
910 * EAGAIN: The socket was blocked, please call again later to
911 * complete the request
912 * ENOTCONN: Caller needs to invoke connect logic then call again
25985edc 913 * other: Some other error occurred, the request was not sent
262965f5 914 */
adfa7144 915static int xs_udp_send_request(struct rpc_rqst *req)
262965f5 916{
262965f5 917 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 918 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262965f5 919 struct xdr_buf *xdr = &req->rq_snd_buf;
9e55eef4
CL		 920 struct msghdr msg = {
921 .msg_name = xs_addr(xprt),
922 .msg_namelen = xprt->addrlen,
923 .msg_flags = XS_SENDMSG_FLAGS,
924 };
3f649ab7 925 unsigned int sent;
262965f5 926 int status;
a246b010 927
9903cd1c 928 xs_pktdump("packet data:",
a246b010
CL		 929 req->rq_svec->iov_base,
930 req->rq_svec->iov_len);
931
01d37c42
TM		 932 if (!xprt_bound(xprt))
933 return -ENOTCONN;
75891f50
TM		 934
935 if (!xprt_request_get_cong(xprt, req))
936 return -EBADSLT;
937
78215759 938 req->rq_xtime = ktime_get();
9e55eef4 939 status = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, 0, &sent);
a246b010 940
46121cf7 941 dprintk("RPC: xs_udp_send_request(%u) = %d\n",
6c7a64e5 942 xdr->len, status);
a246b010 943
3dedbb5c
JB		 944 /* firewall is blocking us, don't return -EAGAIN or we end up looping */
945 if (status == -EPERM)
946 goto process_status;
947
743c69e7
N		 948 if (status == -EAGAIN && sock_writeable(transport->inet))
949 status = -ENOBUFS;
950
f279cd00
JB		 951 if (sent > 0 || status == 0) {
952 req->rq_xmit_bytes_sent += sent;
953 if (sent >= req->rq_slen)
2199700f
TM		 954 return 0;
955 /* Still some bytes left; set up for a retry later. */
262965f5 956 status = -EAGAIN;
2199700f 957 }
a246b010 958
3dedbb5c 959process_status:
262965f5 960 switch (status) {
fba91afb
TM		 961 case -ENOTSOCK:
962 status = -ENOTCONN;
963 /* Should we call xs_close() here? */
964 break;
b6ddf64f 965 case -EAGAIN:
c544577d 966 status = xs_nospace(req);
b6ddf64f 967 break;
262965f5 968 case -ENETUNREACH:
3601c4a9 969 case -ENOBUFS:
262965f5 970 case -EPIPE:
a246b010 971 case -ECONNREFUSED:
3dedbb5c 972 case -EPERM:
a246b010 973 /* When the server has died, an ICMP port unreachable message
9903cd1c 974 * prompts ECONNREFUSED. */
13331a55
TM		 975 break;
976 default:
977 dprintk("RPC: sendmsg returned unrecognized error %d\n",
978 -status);
a246b010 979 }
5fe46e9d 980
262965f5 981 return status;
a246b010
CL		 982}
983
9903cd1c 984/**
262965f5 985 * xs_tcp_send_request - write an RPC request to a TCP socket
50f484e2 986 * @req: pointer to RPC request
9903cd1c
CL		 987 *
988 * Return values:
262965f5
CL		 989 * 0: The request has been sent
990 * EAGAIN: The socket was blocked, please call again later to
991 * complete the request
992 * ENOTCONN: Caller needs to invoke connect logic then call again
25985edc 993 * other: Some other error occurred, the request was not sent
9903cd1c
CL		 994 *
995 * XXX: In the case of soft timeouts, should we eventually give up
262965f5 996 * if sendmsg is not able to make progress?
9903cd1c 997 */
adfa7144 998static int xs_tcp_send_request(struct rpc_rqst *req)
a246b010 999{
a246b010 1000 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 1001 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262965f5 1002 struct xdr_buf *xdr = &req->rq_snd_buf;
7e3d3620
TM		 1003 rpc_fraghdr rm = xs_stream_record_marker(xdr);
1004 unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen;
9e55eef4
CL		 1005 struct msghdr msg = {
1006 .msg_flags = XS_SENDMSG_FLAGS,
1007 };
9ffadfbc 1008 bool vm_wait = false;
3f649ab7 1009 unsigned int sent;
b595bb15 1010 int status;
a246b010 1011
4cd34e7c
TM		 1012 /* Close the stream if the previous transmission was incomplete */
1013 if (xs_send_request_was_aborted(transport, req)) {
1014 if (transport->sock != NULL)
1015 kernel_sock_shutdown(transport->sock, SHUT_RDWR);
1016 return -ENOTCONN;
1017 }
d275880a
TM		 1018 if (!transport->inet)
1019 return -ENOTCONN;
4cd34e7c 1020
262965f5
CL		 1021 xs_pktdump("packet data:",
1022 req->rq_svec->iov_base,
1023 req->rq_svec->iov_len);
a246b010 1024
7196dbb0
TM		 1025 if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state))
1026 xs_tcp_set_socket_timeouts(xprt, transport->sock);
1027
a246b010
CL		 1028 /* Continue transmitting the packet/record. We must be careful
1029 * to cope with writespace callbacks arriving _after_ we have
262965f5 1030 * called sendmsg(). */
78215759 1031 req->rq_xtime = ktime_get();
d737e5d4 1032 tcp_sock_set_cork(transport->inet, true);
a246b010 1033 while (1) {
9e55eef4
CL		 1034 status = xprt_sock_sendmsg(transport->sock, &msg, xdr,
1035 transport->xmit.offset, rm, &sent);
a246b010 1036
46121cf7 1037 dprintk("RPC: xs_tcp_send_request(%u) = %d\n",
6c7a64e5 1038 xdr->len - transport->xmit.offset, status);
a246b010 1039
262965f5
CL		 1040 /* If we've sent the entire packet, immediately
1041 * reset the count of bytes sent. */
6c7a64e5
TM		 1042 transport->xmit.offset += sent;
1043 req->rq_bytes_sent = transport->xmit.offset;
7e3d3620 1044 if (likely(req->rq_bytes_sent >= msglen)) {
6c7a64e5 1045 req->rq_xmit_bytes_sent += transport->xmit.offset;
6c7a64e5 1046 transport->xmit.offset = 0;
d737e5d4
TM		 1047 if (atomic_long_read(&xprt->xmit_queuelen) == 1)
1048 tcp_sock_set_cork(transport->inet, false);
262965f5
CL		 1049 return 0;
1050 }
a246b010 1051
9ffadfbc
TM		 1052 WARN_ON_ONCE(sent == 0 && status == 0);
1053
1054 if (status == -EAGAIN ) {
1055 /*
1056 * Return EAGAIN if we're sure we're hitting the
1057 * socket send buffer limits.
1058 */
1059 if (test_bit(SOCK_NOSPACE, &transport->sock->flags))
1060 break;
1061 /*
1062 * Did we hit a memory allocation failure?
1063 */
1064 if (sent == 0) {
1065 status = -ENOBUFS;
1066 if (vm_wait)
1067 break;
1068 /* Retry, knowing now that we're below the
1069 * socket send buffer limit
1070 */
1071 vm_wait = true;
1072 }
1073 continue;
1074 }
f580dd04
TM		 1075 if (status < 0)
1076 break;
9ffadfbc 1077 vm_wait = false;
a246b010
CL		 1078 }
1079
262965f5 1080 switch (status) {
fba91afb
TM		 1081 case -ENOTSOCK:
1082 status = -ENOTCONN;
1083 /* Should we call xs_close() here? */
1084 break;
262965f5 1085 case -EAGAIN:
c544577d 1086 status = xs_nospace(req);
262965f5 1087 break;
262965f5 1088 case -ECONNRESET:
2a9e1cfa 1089 case -ECONNREFUSED:
262965f5 1090 case -ENOTCONN:
3913c78c 1091 case -EADDRINUSE:
b5872f0c 1092 case -ENOBUFS:
b9d2bb2e 1093 case -EPIPE:
13331a55
TM		 1094 break;
1095 default:
1096 dprintk("RPC: sendmsg returned unrecognized error %d\n",
1097 -status);
a246b010 1098 }
5fe46e9d 1099
a246b010
CL		 1100 return status;
1101}
1102
2a9e1cfa
TM		 1103static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)
1104{
1105 transport->old_data_ready = sk->sk_data_ready;
1106 transport->old_state_change = sk->sk_state_change;
1107 transport->old_write_space = sk->sk_write_space;
2118071d 1108 transport->old_error_report = sk->sk_error_report;
2a9e1cfa
TM		 1109}
1110
1111static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)
1112{
1113 sk->sk_data_ready = transport->old_data_ready;
1114 sk->sk_state_change = transport->old_state_change;
1115 sk->sk_write_space = transport->old_write_space;
2118071d
TM		 1116 sk->sk_error_report = transport->old_error_report;
1117}
1118
42d42a5b
TM		 1119static void xs_sock_reset_state_flags(struct rpc_xprt *xprt)
1120{
1121 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1122
1123 clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
4f8943f8
TM		 1124 clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state);
1125 clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state);
1126 clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state);
1127}
1128
1129static void xs_run_error_worker(struct sock_xprt *transport, unsigned int nr)
1130{
1131 set_bit(nr, &transport->sock_state);
1132 queue_work(xprtiod_workqueue, &transport->error_worker);
42d42a5b
TM		 1133}
1134
b70ae915
TM		 1135static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt)
1136{
1137 smp_mb__before_atomic();
1138 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
1139 clear_bit(XPRT_CLOSING, &xprt->state);
42d42a5b 1140 xs_sock_reset_state_flags(xprt);
b70ae915
TM		 1141 smp_mb__after_atomic();
1142}
1143
2118071d
TM		 1144/**
1145 * xs_error_report - callback to handle TCP socket state errors
1146 * @sk: socket
1147 *
1148 * Note: we don't call sock_error() since there may be a rpc_task
1149 * using the socket, and so we don't want to clear sk->sk_err.
1150 */
1151static void xs_error_report(struct sock *sk)
1152{
4f8943f8 1153 struct sock_xprt *transport;
2118071d 1154 struct rpc_xprt *xprt;
2118071d
TM		 1155
1156 read_lock_bh(&sk->sk_callback_lock);
1157 if (!(xprt = xprt_from_sock(sk)))
1158 goto out;
1159
4f8943f8 1160 transport = container_of(xprt, struct sock_xprt, xprt);
af84537d
BC		 1161 transport->xprt_err = -sk->sk_err;
1162 if (transport->xprt_err == 0)
2118071d
TM		 1163 goto out;
1164 dprintk("RPC: xs_error_report client %p, error=%d...\n",
af84537d
BC		 1165 xprt, -transport->xprt_err);
1166 trace_rpc_socket_error(xprt, sk->sk_socket, transport->xprt_err);
1167
1168 /* barrier ensures xprt_err is set before XPRT_SOCK_WAKE_ERROR */
1169 smp_mb__before_atomic();
4f8943f8 1170 xs_run_error_worker(transport, XPRT_SOCK_WAKE_ERROR);
2118071d
TM		 1171 out:
1172 read_unlock_bh(&sk->sk_callback_lock);
2a9e1cfa
TM		 1173}
1174
fe315e76 1175static void xs_reset_transport(struct sock_xprt *transport)
a246b010 1176{
ee0ac0c2
CL		 1177 struct socket *sock = transport->sock;
1178 struct sock *sk = transport->inet;
6cc7e908 1179 struct rpc_xprt *xprt = &transport->xprt;
a73881c9 1180 struct file *filp = transport->file;
a246b010 1181
fe315e76
CL		 1182 if (sk == NULL)
1183 return;
9903cd1c 1184
264d1df3
JL		 1185 if (atomic_read(&transport->xprt.swapper))
1186 sk_clear_memalloc(sk);
1187
09939204
TM		 1188 kernel_sock_shutdown(sock, SHUT_RDWR);
1189
edc1b01c 1190 mutex_lock(&transport->recv_mutex);
a246b010 1191 write_lock_bh(&sk->sk_callback_lock);
ee0ac0c2
CL		 1192 transport->inet = NULL;
1193 transport->sock = NULL;
a73881c9 1194 transport->file = NULL;
a246b010 1195
9903cd1c 1196 sk->sk_user_data = NULL;
2a9e1cfa
TM		 1197
1198 xs_restore_old_callbacks(transport, sk);
0c78789e 1199 xprt_clear_connected(xprt);
a246b010 1200 write_unlock_bh(&sk->sk_callback_lock);
6cc7e908 1201 xs_sock_reset_connection_flags(xprt);
ae053551
TM		 1202 /* Reset stream record info */
1203 xs_stream_reset_connect(transport);
edc1b01c 1204 mutex_unlock(&transport->recv_mutex);
a246b010 1205
6cc7e908 1206 trace_rpc_socket_close(xprt, sock);
a73881c9 1207 fput(filp);
0445f92c
TM		 1208
1209 xprt_disconnect_done(xprt);
fe315e76
CL		 1210}
1211
1212/**
1213 * xs_close - close a socket
1214 * @xprt: transport
1215 *
1216 * This is used when all requests are complete; ie, no DRC state remains
1217 * on the server we want to save.
f75e6745
TM		 1218 *
1219 * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with
1220 * xs_reset_transport() zeroing the socket from underneath a writer.
fe315e76
CL		 1221 */
1222static void xs_close(struct rpc_xprt *xprt)
1223{
1224 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1225
1226 dprintk("RPC: xs_close xprt %p\n", xprt);
1227
1228 xs_reset_transport(transport);
61d0a8e6 1229 xprt->reestablish_timeout = 0;
a246b010
CL		 1230}
1231
4a068258
CL		 1232static void xs_inject_disconnect(struct rpc_xprt *xprt)
1233{
1234 dprintk("RPC: injecting transport disconnect on xprt=%p\n",
1235 xprt);
1236 xprt_disconnect_done(xprt);
1237}
1238
315f3812
KM		 1239static void xs_xprt_free(struct rpc_xprt *xprt)
1240{
1241 xs_free_peer_addresses(xprt);
1242 xprt_free(xprt);
1243}
1244
9903cd1c
CL		 1245/**
1246 * xs_destroy - prepare to shutdown a transport
1247 * @xprt: doomed transport
1248 *
1249 */
1250static void xs_destroy(struct rpc_xprt *xprt)
a246b010 1251{
03c78827
TM		 1252 struct sock_xprt *transport = container_of(xprt,
1253 struct sock_xprt, xprt);
46121cf7 1254 dprintk("RPC: xs_destroy xprt %p\n", xprt);
9903cd1c 1255
03c78827 1256 cancel_delayed_work_sync(&transport->connect_worker);
a1311d87 1257 xs_close(xprt);
edc1b01c 1258 cancel_work_sync(&transport->recv_worker);
b5e92419 1259 cancel_work_sync(&transport->error_worker);
315f3812 1260 xs_xprt_free(xprt);
a1311d87 1261 module_put(THIS_MODULE);
a246b010
CL		 1262}
1263
9903cd1c 1264/**
f9b2ee71
TM		 1265 * xs_udp_data_read_skb - receive callback for UDP sockets
1266 * @xprt: transport
1267 * @sk: socket
1268 * @skb: skbuff
9903cd1c 1269 *
a246b010 1270 */
f9b2ee71
TM		 1271static void xs_udp_data_read_skb(struct rpc_xprt *xprt,
1272 struct sock *sk,
1273 struct sk_buff *skb)
a246b010 1274{
9903cd1c 1275 struct rpc_task *task;
a246b010 1276 struct rpc_rqst *rovr;
f9b2ee71 1277 int repsize, copied;
d8ed029d
AD		 1278 u32 _xid;
1279 __be32 *xp;
a246b010 1280
1da8c681 1281 repsize = skb->len;
a246b010 1282 if (repsize < 4) {
46121cf7 1283 dprintk("RPC: impossible RPC reply size %d!\n", repsize);
f9b2ee71 1284 return;
a246b010
CL		 1285 }
1286
1287 /* Copy the XID from the skb... */
1da8c681 1288 xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid);
a246b010 1289 if (xp == NULL)
f9b2ee71 1290 return;
a246b010
CL		 1291
1292 /* Look up and lock the request corresponding to the given XID */
75c84151 1293 spin_lock(&xprt->queue_lock);
a246b010
CL		 1294 rovr = xprt_lookup_rqst(xprt, *xp);
1295 if (!rovr)
1296 goto out_unlock;
729749bb 1297 xprt_pin_rqst(rovr);
ecd465ee 1298 xprt_update_rtt(rovr->rq_task);
75c84151 1299 spin_unlock(&xprt->queue_lock);
a246b010
CL		 1300 task = rovr->rq_task;
1301
a246b010
CL		 1302 if ((copied = rovr->rq_private_buf.buflen) > repsize)
1303 copied = repsize;
1304
1305 /* Suck it into the iovec, verify checksum if not done by hw. */
1781f7f5 1306 if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) {
75c84151 1307 spin_lock(&xprt->queue_lock);
0afa6b44 1308 __UDPX_INC_STATS(sk, UDP_MIB_INERRORS);
729749bb 1309 goto out_unpin;
1781f7f5
HX		 1310 }
1311
a246b010 1312
b5e92419 1313 spin_lock(&xprt->transport_lock);
6a24dfb6 1314 xprt_adjust_cwnd(xprt, task, copied);
b5e92419 1315 spin_unlock(&xprt->transport_lock);
75c84151 1316 spin_lock(&xprt->queue_lock);
1570c1e4 1317 xprt_complete_rqst(task, copied);
0afa6b44 1318 __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS);
729749bb
TM		 1319out_unpin:
1320 xprt_unpin_rqst(rovr);
a246b010 1321 out_unlock:
75c84151 1322 spin_unlock(&xprt->queue_lock);
f9b2ee71
TM		 1323}
1324
1325static void xs_udp_data_receive(struct sock_xprt *transport)
1326{
1327 struct sk_buff *skb;
1328 struct sock *sk;
1329 int err;
1330
1331 mutex_lock(&transport->recv_mutex);
1332 sk = transport->inet;
1333 if (sk == NULL)
1334 goto out;
1335 for (;;) {
7c13f97f 1336 skb = skb_recv_udp(sk, 0, 1, &err);
4f546149 1337 if (skb == NULL)
f9b2ee71 1338 break;
4f546149
TM		 1339 xs_udp_data_read_skb(&transport->xprt, sk, skb);
1340 consume_skb(skb);
1341 cond_resched();
f9b2ee71 1342 }
0ffe86f4 1343 xs_poll_check_readable(transport);
f9b2ee71
TM		 1344out:
1345 mutex_unlock(&transport->recv_mutex);
1346}
1347
1348static void xs_udp_data_receive_workfn(struct work_struct *work)
1349{
1350 struct sock_xprt *transport =
1351 container_of(work, struct sock_xprt, recv_worker);
a1231fda
TM		 1352 unsigned int pflags = memalloc_nofs_save();
1353
f9b2ee71 1354 xs_udp_data_receive(transport);
a1231fda 1355 memalloc_nofs_restore(pflags);
f9b2ee71
TM		 1356}
1357
1358/**
1359 * xs_data_ready - "data ready" callback for UDP sockets
1360 * @sk: socket with data to read
1361 *
1362 */
1363static void xs_data_ready(struct sock *sk)
1364{
1365 struct rpc_xprt *xprt;
1366
1367 read_lock_bh(&sk->sk_callback_lock);
1368 dprintk("RPC: xs_data_ready...\n");
1369 xprt = xprt_from_sock(sk);
1370 if (xprt != NULL) {
1371 struct sock_xprt *transport = container_of(xprt,
1372 struct sock_xprt, xprt);
5157b956
TM		 1373 transport->old_data_ready(sk);
1374 /* Any data means we had a useful conversation, so
1375 * then we don't need to delay the next reconnect
1376 */
1377 if (xprt->reestablish_timeout)
1378 xprt->reestablish_timeout = 0;
42d42a5b 1379 if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
40a5f1b1 1380 queue_work(xprtiod_workqueue, &transport->recv_worker);
f9b2ee71 1381 }
f064af1e 1382 read_unlock_bh(&sk->sk_callback_lock);
a246b010
CL		 1383}
1384
a519fc7a
TM		 1385/*
1386 * Helper function to force a TCP close if the server is sending
1387 * junk and/or it has put us in CLOSE_WAIT
1388 */
1389static void xs_tcp_force_close(struct rpc_xprt *xprt)
1390{
a519fc7a
TM		 1391 xprt_force_disconnect(xprt);
1392}
1393
9e00abc3 1394#if defined(CONFIG_SUNRPC_BACKCHANNEL)
6b26cc8c
CL		 1395static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt)
1396{
1397 return PAGE_SIZE;
1398}
9e00abc3 1399#endif /* CONFIG_SUNRPC_BACKCHANNEL */
44b98efd 1400
9903cd1c
CL		 1401/**
1402 * xs_tcp_state_change - callback to handle TCP socket state changes
1403 * @sk: socket whose state has changed
1404 *
1405 */
1406static void xs_tcp_state_change(struct sock *sk)
a246b010 1407{
9903cd1c 1408 struct rpc_xprt *xprt;
0fdea1e8 1409 struct sock_xprt *transport;
a246b010 1410
f064af1e 1411 read_lock_bh(&sk->sk_callback_lock);
a246b010
CL		 1412 if (!(xprt = xprt_from_sock(sk)))
1413 goto out;
46121cf7 1414 dprintk("RPC: xs_tcp_state_change client %p...\n", xprt);
669502ff 1415 dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n",
46121cf7
CL		 1416 sk->sk_state, xprt_connected(xprt),
1417 sock_flag(sk, SOCK_DEAD),
669502ff
AC		 1418 sock_flag(sk, SOCK_ZAPPED),
1419 sk->sk_shutdown);
a246b010 1420
0fdea1e8 1421 transport = container_of(xprt, struct sock_xprt, xprt);
40b5ea0c 1422 trace_rpc_socket_state_change(xprt, sk->sk_socket);
a246b010
CL		 1423 switch (sk->sk_state) {
1424 case TCP_ESTABLISHED:
a246b010 1425 if (!xprt_test_and_set_connected(xprt)) {
8b71798c 1426 xprt->connect_cookie++;
0fdea1e8
TM		 1427 clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
1428 xprt_clear_connecting(xprt);
51971139 1429
3968a8a5
CL		 1430 xprt->stat.connect_count++;
1431 xprt->stat.connect_time += (long)jiffies -
1432 xprt->stat.connect_start;
4f8943f8 1433 xs_run_error_worker(transport, XPRT_SOCK_WAKE_PENDING);
a246b010 1434 }
a246b010 1435 break;
3b948ae5
TM		 1436 case TCP_FIN_WAIT1:
1437 /* The client initiated a shutdown of the socket */
7c1d71cf 1438 xprt->connect_cookie++;
663b8858 1439 xprt->reestablish_timeout = 0;
3b948ae5 1440 set_bit(XPRT_CLOSING, &xprt->state);
4e857c58 1441 smp_mb__before_atomic();
3b948ae5 1442 clear_bit(XPRT_CONNECTED, &xprt->state);
ef803670 1443 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
4e857c58 1444 smp_mb__after_atomic();
a246b010 1445 break;
632e3bdc 1446 case TCP_CLOSE_WAIT:
3b948ae5 1447 /* The server initiated a shutdown of the socket */
7c1d71cf 1448 xprt->connect_cookie++;
d0bea455 1449 clear_bit(XPRT_CONNECTED, &xprt->state);
4f8943f8 1450 xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT);
df561f66 1451 fallthrough;
663b8858
TM		 1452 case TCP_CLOSING:
1453 /*
1454 * If the server closed down the connection, make sure that
1455 * we back off before reconnecting
1456 */
1457 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
1458 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
3b948ae5
TM		 1459 break;
1460 case TCP_LAST_ACK:
670f9457 1461 set_bit(XPRT_CLOSING, &xprt->state);
4e857c58 1462 smp_mb__before_atomic();
3b948ae5 1463 clear_bit(XPRT_CONNECTED, &xprt->state);
4e857c58 1464 smp_mb__after_atomic();
3b948ae5
TM		 1465 break;
1466 case TCP_CLOSE:
0fdea1e8
TM		 1467 if (test_and_clear_bit(XPRT_SOCK_CONNECTING,
1468 &transport->sock_state))
1469 xprt_clear_connecting(xprt);
9b30889c 1470 clear_bit(XPRT_CLOSING, &xprt->state);
9b30889c 1471 /* Trigger the socket release */
4f8943f8 1472 xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT);
a246b010
CL		 1473 }
1474 out:
f064af1e 1475 read_unlock_bh(&sk->sk_callback_lock);
a246b010
CL		 1476}
1477
1f0fa154
IJ		 1478static void xs_write_space(struct sock *sk)
1479{
13331a55 1480 struct socket_wq *wq;
4f8943f8 1481 struct sock_xprt *transport;
1f0fa154
IJ		 1482 struct rpc_xprt *xprt;
1483
13331a55 1484 if (!sk->sk_socket)
1f0fa154 1485 return;
13331a55 1486 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1f0fa154
IJ		 1487
1488 if (unlikely(!(xprt = xprt_from_sock(sk))))
1489 return;
4f8943f8 1490 transport = container_of(xprt, struct sock_xprt, xprt);
13331a55
TM		 1491 rcu_read_lock();
1492 wq = rcu_dereference(sk->sk_wq);
1493 if (!wq || test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags) == 0)
1494 goto out;
1f0fa154 1495
4f8943f8
TM		 1496 xs_run_error_worker(transport, XPRT_SOCK_WAKE_WRITE);
1497 sk->sk_write_pending--;
13331a55
TM		 1498out:
1499 rcu_read_unlock();
1f0fa154
IJ		 1500}
1501
9903cd1c 1502/**
c7b2cae8
CL		 1503 * xs_udp_write_space - callback invoked when socket buffer space
1504 * becomes available
9903cd1c
CL		 1505 * @sk: socket whose state has changed
1506 *
a246b010
CL		 1507 * Called when more output buffer space is available for this socket.
1508 * We try not to wake our writers until they can make "significant"
c7b2cae8 1509 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
a246b010
CL		 1510 * with a bunch of small requests.
1511 */
c7b2cae8 1512static void xs_udp_write_space(struct sock *sk)
a246b010 1513{
f064af1e 1514 read_lock_bh(&sk->sk_callback_lock);
a246b010 1515
c7b2cae8 1516 /* from net/core/sock.c:sock_def_write_space */
1f0fa154
IJ		 1517 if (sock_writeable(sk))
1518 xs_write_space(sk);
a246b010 1519
f064af1e 1520 read_unlock_bh(&sk->sk_callback_lock);
c7b2cae8 1521}
a246b010 1522
c7b2cae8
CL		 1523/**
1524 * xs_tcp_write_space - callback invoked when socket buffer space
1525 * becomes available
1526 * @sk: socket whose state has changed
1527 *
1528 * Called when more output buffer space is available for this socket.
1529 * We try not to wake our writers until they can make "significant"
1530 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
1531 * with a bunch of small requests.
1532 */
1533static void xs_tcp_write_space(struct sock *sk)
1534{
f064af1e 1535 read_lock_bh(&sk->sk_callback_lock);
c7b2cae8
CL		 1536
1537 /* from net/core/stream.c:sk_stream_write_space */
64dc6130 1538 if (sk_stream_is_writeable(sk))
1f0fa154 1539 xs_write_space(sk);
b6ddf64f 1540
f064af1e 1541 read_unlock_bh(&sk->sk_callback_lock);
a246b010
CL		 1542}
1543
470056c2 1544static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt)
a246b010 1545{
ee0ac0c2
CL		 1546 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1547 struct sock *sk = transport->inet;
a246b010 1548
7c6e066e 1549 if (transport->rcvsize) {
a246b010 1550 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
7c6e066e 1551 sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;
a246b010 1552 }
7c6e066e 1553 if (transport->sndsize) {
a246b010 1554 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
7c6e066e 1555 sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;
a246b010
CL		 1556 sk->sk_write_space(sk);
1557 }
1558}
1559
43118c29 1560/**
470056c2 1561 * xs_udp_set_buffer_size - set send and receive limits
43118c29 1562 * @xprt: generic transport
470056c2
CL		 1563 * @sndsize: requested size of send buffer, in bytes
1564 * @rcvsize: requested size of receive buffer, in bytes
43118c29 1565 *
470056c2 1566 * Set socket send and receive buffer size limits.
43118c29 1567 */
470056c2 1568static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize)
43118c29 1569{
7c6e066e
CL		 1570 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1571
1572 transport->sndsize = 0;
470056c2 1573 if (sndsize)
7c6e066e
CL		 1574 transport->sndsize = sndsize + 1024;
1575 transport->rcvsize = 0;
470056c2 1576 if (rcvsize)
7c6e066e 1577 transport->rcvsize = rcvsize + 1024;
470056c2
CL		 1578
1579 xs_udp_do_set_buffer_size(xprt);
43118c29
CL		 1580}
1581
46c0ee8b
CL		 1582/**
1583 * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport
acf0a39f 1584 * @xprt: controlling transport
46c0ee8b
CL		 1585 * @task: task that timed out
1586 *
1587 * Adjust the congestion window after a retransmit timeout has occurred.
1588 */
6a24dfb6 1589static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task)
46c0ee8b 1590{
b5e92419 1591 spin_lock(&xprt->transport_lock);
6a24dfb6 1592 xprt_adjust_cwnd(xprt, task, -ETIMEDOUT);
b5e92419 1593 spin_unlock(&xprt->transport_lock);
46c0ee8b
CL		 1594}
1595
826799e6 1596static int xs_get_random_port(void)
b85d8806 1597{
826799e6
BF		 1598 unsigned short min = xprt_min_resvport, max = xprt_max_resvport;
1599 unsigned short range;
1600 unsigned short rand;
1601
1602 if (max < min)
1603 return -EADDRINUSE;
1604 range = max - min + 1;
1605 rand = (unsigned short) prandom_u32() % range;
1606 return rand + min;
b85d8806
CL		 1607}
1608
4dda9c8a
TM		 1609static unsigned short xs_sock_getport(struct socket *sock)
1610{
1611 struct sockaddr_storage buf;
4dda9c8a
TM		 1612 unsigned short port = 0;
1613
9b2c45d4 1614 if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0)
4dda9c8a
TM		 1615 goto out;
1616 switch (buf.ss_family) {
1617 case AF_INET6:
1618 port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port);
1619 break;
1620 case AF_INET:
1621 port = ntohs(((struct sockaddr_in *)&buf)->sin_port);
1622 }
1623out:
1624 return port;
1625}
1626
92200412
CL		 1627/**
1628 * xs_set_port - reset the port number in the remote endpoint address
1629 * @xprt: generic transport
1630 * @port: new port number
1631 *
1632 */
1633static void xs_set_port(struct rpc_xprt *xprt, unsigned short port)
1634{
46121cf7 1635 dprintk("RPC: setting port for xprt %p to %u\n", xprt, port);
c4efcb1d 1636
9dc3b095
CL		 1637 rpc_set_port(xs_addr(xprt), port);
1638 xs_update_peer_port(xprt);
92200412
CL		 1639}
1640
4dda9c8a
TM		 1641static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)
1642{
e6237b6f 1643 if (transport->srcport == 0 && transport->xprt.reuseport)
4dda9c8a
TM		 1644 transport->srcport = xs_sock_getport(sock);
1645}
1646
826799e6 1647static int xs_get_srcport(struct sock_xprt *transport)
67a391d7 1648{
826799e6 1649 int port = transport->srcport;
67a391d7
TM		 1650
1651 if (port == 0 && transport->xprt.resvport)
1652 port = xs_get_random_port();
1653 return port;
1654}
1655
baaf4e48 1656static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port)
67a391d7 1657{
fbfffbd5
CL		 1658 if (transport->srcport != 0)
1659 transport->srcport = 0;
67a391d7
TM		 1660 if (!transport->xprt.resvport)
1661 return 0;
1662 if (port <= xprt_min_resvport || port > xprt_max_resvport)
1663 return xprt_max_resvport;
1664 return --port;
1665}
beb59b68 1666static int xs_bind(struct sock_xprt *transport, struct socket *sock)
a246b010 1667{
beb59b68 1668 struct sockaddr_storage myaddr;
67a391d7 1669 int err, nloop = 0;
826799e6 1670 int port = xs_get_srcport(transport);
67a391d7 1671 unsigned short last;
a246b010 1672
0f7a622c
CP		 1673 /*
1674 * If we are asking for any ephemeral port (i.e. port == 0 &&
1675 * transport->xprt.resvport == 0), don't bind. Let the local
1676 * port selection happen implicitly when the socket is used
1677 * (for example at connect time).
1678 *
1679 * This ensures that we can continue to establish TCP
1680 * connections even when all local ephemeral ports are already
1681 * a part of some TCP connection. This makes no difference
12b20ce3 1682 * for UDP sockets, but also doesn't harm them.
0f7a622c
CP		 1683 *
1684 * If we're asking for any reserved port (i.e. port == 0 &&
1685 * transport->xprt.resvport == 1) xs_get_srcport above will
1686 * ensure that port is non-zero and we will bind as needed.
1687 */
826799e6
BF		 1688 if (port <= 0)
1689 return port;
0f7a622c 1690
beb59b68 1691 memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen);
a246b010 1692 do {
beb59b68
PE		 1693 rpc_set_port((struct sockaddr *)&myaddr, port);
1694 err = kernel_bind(sock, (struct sockaddr *)&myaddr,
1695 transport->xprt.addrlen);
a246b010 1696 if (err == 0) {
fbfffbd5 1697 transport->srcport = port;
d3bc9a1d 1698 break;
a246b010 1699 }
67a391d7 1700 last = port;
baaf4e48 1701 port = xs_next_srcport(transport, port);
67a391d7
TM		 1702 if (port > last)
1703 nloop++;
1704 } while (err == -EADDRINUSE && nloop != 2);
90058d37 1705
4232e863 1706 if (myaddr.ss_family == AF_INET)
beb59b68
PE		 1707 dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__,
1708 &((struct sockaddr_in *)&myaddr)->sin_addr,
1709 port, err ? "failed" : "ok", err);
1710 else
1711 dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__,
1712 &((struct sockaddr_in6 *)&myaddr)->sin6_addr,
1713 port, err ? "failed" : "ok", err);
a246b010
CL		 1714 return err;
1715}
1716
176e21ee
CL		 1717/*
1718 * We don't support autobind on AF_LOCAL sockets
1719 */
1720static void xs_local_rpcbind(struct rpc_task *task)
1721{
fb43d172 1722 xprt_set_bound(task->tk_xprt);
176e21ee
CL		 1723}
1724
1725static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port)
1726{
1727}
beb59b68 1728
ed07536e
PZ		 1729#ifdef CONFIG_DEBUG_LOCK_ALLOC
1730static struct lock_class_key xs_key[2];
1731static struct lock_class_key xs_slock_key[2];
1732
176e21ee
CL		 1733static inline void xs_reclassify_socketu(struct socket *sock)
1734{
1735 struct sock *sk = sock->sk;
1736
176e21ee
CL		 1737 sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC",
1738 &xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]);
1739}
1740
8945ee5e 1741static inline void xs_reclassify_socket4(struct socket *sock)
ed07536e
PZ		 1742{
1743 struct sock *sk = sock->sk;
8945ee5e 1744
8945ee5e
CL		 1745 sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC",
1746 &xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]);
1747}
ed07536e 1748
8945ee5e
CL		 1749static inline void xs_reclassify_socket6(struct socket *sock)
1750{
1751 struct sock *sk = sock->sk;
ed07536e 1752
8945ee5e
CL		 1753 sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC",
1754 &xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]);
ed07536e 1755}
6bc9638a
PE		 1756
1757static inline void xs_reclassify_socket(int family, struct socket *sock)
1758{
fafc4e1e 1759 if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk)))
1b7a1819
WAA		 1760 return;
1761
4232e863 1762 switch (family) {
176e21ee
CL		 1763 case AF_LOCAL:
1764 xs_reclassify_socketu(sock);
1765 break;
4232e863 1766 case AF_INET:
6bc9638a 1767 xs_reclassify_socket4(sock);
4232e863
CL		 1768 break;
1769 case AF_INET6:
6bc9638a 1770 xs_reclassify_socket6(sock);
4232e863
CL		 1771 break;
1772 }
6bc9638a 1773}
ed07536e 1774#else
6bc9638a
PE		 1775static inline void xs_reclassify_socket(int family, struct socket *sock)
1776{
1777}
ed07536e
PZ		 1778#endif
1779
93dc41bd
N		 1780static void xs_dummy_setup_socket(struct work_struct *work)
1781{
1782}
1783
6bc9638a 1784static struct socket *xs_create_sock(struct rpc_xprt *xprt,
4dda9c8a
TM		 1785 struct sock_xprt *transport, int family, int type,
1786 int protocol, bool reuseport)
22f79326 1787{
a73881c9 1788 struct file *filp;
22f79326
PE		 1789 struct socket *sock;
1790 int err;
1791
6bc9638a 1792 err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1);
22f79326
PE		 1793 if (err < 0) {
1794 dprintk("RPC: can't create %d transport socket (%d).\n",
1795 protocol, -err);
1796 goto out;
1797 }
6bc9638a 1798 xs_reclassify_socket(family, sock);
22f79326 1799
4dda9c8a 1800 if (reuseport)
fe31a326 1801 sock_set_reuseport(sock->sk);
4dda9c8a 1802
4cea288a
BH		 1803 err = xs_bind(transport, sock);
1804 if (err) {
22f79326
PE		 1805 sock_release(sock);
1806 goto out;
1807 }
1808
a73881c9
TM		 1809 filp = sock_alloc_file(sock, O_NONBLOCK, NULL);
1810 if (IS_ERR(filp))
1811 return ERR_CAST(filp);
1812 transport->file = filp;
1813
22f79326
PE		 1814 return sock;
1815out:
1816 return ERR_PTR(err);
1817}
1818
176e21ee
CL		 1819static int xs_local_finish_connecting(struct rpc_xprt *xprt,
1820 struct socket *sock)
1821{
1822 struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
1823 xprt);
1824
1825 if (!transport->inet) {
1826 struct sock *sk = sock->sk;
1827
1828 write_lock_bh(&sk->sk_callback_lock);
1829
1830 xs_save_old_callbacks(transport, sk);
1831
1832 sk->sk_user_data = xprt;
a2648094 1833 sk->sk_data_ready = xs_data_ready;
176e21ee 1834 sk->sk_write_space = xs_udp_write_space;
b4411457 1835 sock_set_flag(sk, SOCK_FASYNC);
2118071d 1836 sk->sk_error_report = xs_error_report;
176e21ee
CL		 1837
1838 xprt_clear_connected(xprt);
1839
1840 /* Reset to new socket */
1841 transport->sock = sock;
1842 transport->inet = sk;
1843
1844 write_unlock_bh(&sk->sk_callback_lock);
1845 }
1846
ae053551 1847 xs_stream_start_connect(transport);
6c7a64e5 1848
176e21ee
CL		 1849 return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0);
1850}
1851
1852/**
1853 * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint
176e21ee 1854 * @transport: socket transport to connect
176e21ee 1855 */
dc107402 1856static int xs_local_setup_socket(struct sock_xprt *transport)
176e21ee 1857{
176e21ee 1858 struct rpc_xprt *xprt = &transport->xprt;
a73881c9 1859 struct file *filp;
176e21ee 1860 struct socket *sock;
68e9a246 1861 int status;
176e21ee 1862
176e21ee
CL		 1863 status = __sock_create(xprt->xprt_net, AF_LOCAL,
1864 SOCK_STREAM, 0, &sock, 1);
1865 if (status < 0) {
1866 dprintk("RPC: can't create AF_LOCAL "
1867 "transport socket (%d).\n", -status);
1868 goto out;
1869 }
d1358917 1870 xs_reclassify_socket(AF_LOCAL, sock);
176e21ee 1871
a73881c9
TM		 1872 filp = sock_alloc_file(sock, O_NONBLOCK, NULL);
1873 if (IS_ERR(filp)) {
1874 status = PTR_ERR(filp);
1875 goto out;
1876 }
1877 transport->file = filp;
1878
176e21ee
CL		 1879 dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n",
1880 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
1881
1882 status = xs_local_finish_connecting(xprt, sock);
40b5ea0c 1883 trace_rpc_socket_connect(xprt, sock, status);
176e21ee
CL		 1884 switch (status) {
1885 case 0:
1886 dprintk("RPC: xprt %p connected to %s\n",
1887 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
3968a8a5
CL		 1888 xprt->stat.connect_count++;
1889 xprt->stat.connect_time += (long)jiffies -
1890 xprt->stat.connect_start;
176e21ee 1891 xprt_set_connected(xprt);
93f479d3 1892 break;
3601c4a9 1893 case -ENOBUFS:
176e21ee
CL		 1894 break;
1895 case -ENOENT:
1896 dprintk("RPC: xprt %p: socket %s does not exist\n",
1897 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
1898 break;
4a20a988
TM		 1899 case -ECONNREFUSED:
1900 dprintk("RPC: xprt %p: connection refused for %s\n",
1901 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
1902 break;
176e21ee
CL		 1903 default:
1904 printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n",
1905 __func__, -status,
1906 xprt->address_strings[RPC_DISPLAY_ADDR]);
1907 }
1908
1909out:
1910 xprt_clear_connecting(xprt);
1911 xprt_wake_pending_tasks(xprt, status);
dc107402
BF		 1912 return status;
1913}
1914
b6669737 1915static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task)
dc107402 1916{
dc107402
BF		 1917 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1918 int ret;
1919
1920 if (RPC_IS_ASYNC(task)) {
1921 /*
1922 * We want the AF_LOCAL connect to be resolved in the
1923 * filesystem namespace of the process making the rpc
1924 * call. Thus we connect synchronously.
1925 *
1926 * If we want to support asynchronous AF_LOCAL calls,
1927 * we'll need to figure out how to pass a namespace to
1928 * connect.
1929 */
5ad64b36 1930 task->tk_rpc_status = -ENOTCONN;
dc107402
BF		 1931 rpc_exit(task, -ENOTCONN);
1932 return;
1933 }
1934 ret = xs_local_setup_socket(transport);
1935 if (ret && !RPC_IS_SOFTCONN(task))
1936 msleep_interruptible(15000);
176e21ee
CL		 1937}
1938
3c87ef6e 1939#if IS_ENABLED(CONFIG_SUNRPC_SWAP)
d6e971d8
JL		 1940/*
1941 * Note that this should be called with XPRT_LOCKED held (or when we otherwise
1942 * know that we have exclusive access to the socket), to guard against
1943 * races with xs_reset_transport.
1944 */
a564b8f0
MG		 1945static void xs_set_memalloc(struct rpc_xprt *xprt)
1946{
1947 struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
1948 xprt);
1949
d6e971d8
JL		 1950 /*
1951 * If there's no sock, then we have nothing to set. The
1952 * reconnecting process will get it for us.
1953 */
1954 if (!transport->inet)
1955 return;
8e228133 1956 if (atomic_read(&xprt->swapper))
a564b8f0
MG		 1957 sk_set_memalloc(transport->inet);
1958}
1959
1960/**
d67fa4d8 1961 * xs_enable_swap - Tag this transport as being used for swap.
a564b8f0 1962 * @xprt: transport to tag
a564b8f0 1963 *
8e228133
JL		 1964 * Take a reference to this transport on behalf of the rpc_clnt, and
1965 * optionally mark it for swapping if it wasn't already.
a564b8f0 1966 */
d67fa4d8
JL		 1967static int
1968xs_enable_swap(struct rpc_xprt *xprt)
a564b8f0 1969{
d6e971d8 1970 struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
a564b8f0 1971
d6e971d8
JL		 1972 if (atomic_inc_return(&xprt->swapper) != 1)
1973 return 0;
1974 if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE))
1975 return -ERESTARTSYS;
1976 if (xs->inet)
1977 sk_set_memalloc(xs->inet);
1978 xprt_release_xprt(xprt, NULL);
8e228133
JL		 1979 return 0;
1980}
a564b8f0 1981
8e228133 1982/**
d67fa4d8 1983 * xs_disable_swap - Untag this transport as being used for swap.
8e228133
JL		 1984 * @xprt: transport to tag
1985 *
1986 * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the
1987 * swapper refcount goes to 0, untag the socket as a memalloc socket.
1988 */
d67fa4d8
JL		 1989static void
1990xs_disable_swap(struct rpc_xprt *xprt)
8e228133 1991{
d6e971d8 1992 struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
8e228133 1993
d6e971d8
JL		 1994 if (!atomic_dec_and_test(&xprt->swapper))
1995 return;
1996 if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE))
1997 return;
1998 if (xs->inet)
1999 sk_clear_memalloc(xs->inet);
2000 xprt_release_xprt(xprt, NULL);
a564b8f0 2001}
a564b8f0
MG		 2002#else
2003static void xs_set_memalloc(struct rpc_xprt *xprt)
2004{
2005}
d67fa4d8
JL		 2006
2007static int
2008xs_enable_swap(struct rpc_xprt *xprt)
2009{
2010 return -EINVAL;
2011}
2012
2013static void
2014xs_disable_swap(struct rpc_xprt *xprt)
2015{
2016}
a564b8f0
MG		 2017#endif
2018
16be2d20
CL		 2019static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
2020{
2021 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2022
2023 if (!transport->inet) {
2024 struct sock *sk = sock->sk;
2025
2026 write_lock_bh(&sk->sk_callback_lock);
2027
2a9e1cfa
TM		 2028 xs_save_old_callbacks(transport, sk);
2029
16be2d20 2030 sk->sk_user_data = xprt;
f9b2ee71 2031 sk->sk_data_ready = xs_data_ready;
16be2d20 2032 sk->sk_write_space = xs_udp_write_space;
b4411457 2033 sock_set_flag(sk, SOCK_FASYNC);
16be2d20
CL		 2034
2035 xprt_set_connected(xprt);
2036
2037 /* Reset to new socket */
2038 transport->sock = sock;
2039 transport->inet = sk;
2040
a564b8f0
MG		 2041 xs_set_memalloc(xprt);
2042
16be2d20
CL		 2043 write_unlock_bh(&sk->sk_callback_lock);
2044 }
2045 xs_udp_do_set_buffer_size(xprt);
02910177
TM		 2046
2047 xprt->stat.connect_start = jiffies;
16be2d20
CL		 2048}
2049
8c14ff2a 2050static void xs_udp_setup_socket(struct work_struct *work)
a246b010 2051{
34161db6
TM		 2052 struct sock_xprt *transport =
2053 container_of(work, struct sock_xprt, connect_worker.work);
c8475461 2054 struct rpc_xprt *xprt = &transport->xprt;
d099b8af 2055 struct socket *sock;
b65c0310 2056 int status = -EIO;
9903cd1c 2057
8c14ff2a 2058 sock = xs_create_sock(xprt, transport,
4dda9c8a
TM		 2059 xs_addr(xprt)->sa_family, SOCK_DGRAM,
2060 IPPROTO_UDP, false);
b65c0310 2061 if (IS_ERR(sock))
b0d93ad5 2062 goto out;
68e220bd 2063
c740eff8
CL		 2064 dprintk("RPC: worker connecting xprt %p via %s to "
2065 "%s (port %s)\n", xprt,
2066 xprt->address_strings[RPC_DISPLAY_PROTO],
2067 xprt->address_strings[RPC_DISPLAY_ADDR],
2068 xprt->address_strings[RPC_DISPLAY_PORT]);
68e220bd
CL		 2069
2070 xs_udp_finish_connecting(xprt, sock);
40b5ea0c 2071 trace_rpc_socket_connect(xprt, sock, 0);
b0d93ad5
CL		 2072 status = 0;
2073out:
b0d93ad5 2074 xprt_clear_connecting(xprt);
cf76785d 2075 xprt_unlock_connect(xprt, transport);
7d1e8255 2076 xprt_wake_pending_tasks(xprt, status);
a246b010
CL		 2077}
2078
4876cc77
TM		 2079/**
2080 * xs_tcp_shutdown - gracefully shut down a TCP socket
2081 * @xprt: transport
2082 *
2083 * Initiates a graceful shutdown of the TCP socket by calling the
2084 * equivalent of shutdown(SHUT_RDWR);
2085 */
2086static void xs_tcp_shutdown(struct rpc_xprt *xprt)
2087{
2088 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2089 struct socket *sock = transport->sock;
9b30889c 2090 int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE;
4876cc77
TM		 2091
2092 if (sock == NULL)
2093 return;
9b30889c
TM		 2094 switch (skst) {
2095 default:
4876cc77
TM		 2096 kernel_sock_shutdown(sock, SHUT_RDWR);
2097 trace_rpc_socket_shutdown(xprt, sock);
9b30889c
TM		 2098 break;
2099 case TCP_CLOSE:
2100 case TCP_TIME_WAIT:
4876cc77 2101 xs_reset_transport(transport);
9b30889c 2102 }
4876cc77
TM		 2103}
2104
8d1b8c62
TM		 2105static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
2106 struct socket *sock)
2107{
7196dbb0
TM		 2108 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2109 unsigned int keepidle;
2110 unsigned int keepcnt;
8d1b8c62
TM		 2111 unsigned int timeo;
2112
b5e92419 2113 spin_lock(&xprt->transport_lock);
7196dbb0
TM		 2114 keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ);
2115 keepcnt = xprt->timeout->to_retries + 1;
2116 timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
2117 (xprt->timeout->to_retries + 1);
2118 clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
b5e92419 2119 spin_unlock(&xprt->transport_lock);
7196dbb0 2120
8d1b8c62 2121 /* TCP Keepalive options */
ce3d9544 2122 sock_set_keepalive(sock->sk);
71c48eb8 2123 tcp_sock_set_keepidle(sock->sk, keepidle);
d41ecaac 2124 tcp_sock_set_keepintvl(sock->sk, keepidle);
480aeb96 2125 tcp_sock_set_keepcnt(sock->sk, keepcnt);
8d1b8c62
TM		 2126
2127 /* TCP user timeout (see RFC5482) */
c488aead 2128 tcp_sock_set_user_timeout(sock->sk, timeo);
8d1b8c62
TM		 2129}
2130
7196dbb0
TM		 2131static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt,
2132 unsigned long connect_timeout,
2133 unsigned long reconnect_timeout)
2134{
2135 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2136 struct rpc_timeout to;
2137 unsigned long initval;
2138
b5e92419 2139 spin_lock(&xprt->transport_lock);
7196dbb0
TM		 2140 if (reconnect_timeout < xprt->max_reconnect_timeout)
2141 xprt->max_reconnect_timeout = reconnect_timeout;
2142 if (connect_timeout < xprt->connect_timeout) {
2143 memcpy(&to, xprt->timeout, sizeof(to));
2144 initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1);
2145 /* Arbitrary lower limit */
2146 if (initval < XS_TCP_INIT_REEST_TO << 1)
2147 initval = XS_TCP_INIT_REEST_TO << 1;
2148 to.to_initval = initval;
2149 to.to_maxval = initval;
2150 memcpy(&transport->tcp_timeout, &to,
2151 sizeof(transport->tcp_timeout));
2152 xprt->timeout = &transport->tcp_timeout;
2153 xprt->connect_timeout = connect_timeout;
2154 }
2155 set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
b5e92419 2156 spin_unlock(&xprt->transport_lock);
7196dbb0
TM		 2157}
2158
16be2d20 2159static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
a246b010 2160{
16be2d20 2161 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
fe19a96b 2162 int ret = -ENOTCONN;
edb267a6 2163
ee0ac0c2 2164 if (!transport->inet) {
b0d93ad5 2165 struct sock *sk = sock->sk;
7f260e85 2166
d88e4d82
N		 2167 /* Avoid temporary address, they are bad for long-lived
2168 * connections such as NFS mounts.
2169 * RFC4941, section 3.6 suggests that:
2170 * Individual applications, which have specific
2171 * knowledge about the normal duration of connections,
2172 * MAY override this as appropriate.
2173 */
18d5ad62
CH		 2174 if (xs_addr(xprt)->sa_family == PF_INET6) {
2175 ip6_sock_set_addr_preferences(sk,
2176 IPV6_PREFER_SRC_PUBLIC);
2177 }
d88e4d82 2178
8d1b8c62 2179 xs_tcp_set_socket_timeouts(xprt, sock);
d737e5d4 2180 tcp_sock_set_nodelay(sk);
775f06ab 2181
b0d93ad5
CL		 2182 write_lock_bh(&sk->sk_callback_lock);
2183
2a9e1cfa
TM		 2184 xs_save_old_callbacks(transport, sk);
2185
b0d93ad5 2186 sk->sk_user_data = xprt;
5157b956 2187 sk->sk_data_ready = xs_data_ready;
b0d93ad5
CL		 2188 sk->sk_state_change = xs_tcp_state_change;
2189 sk->sk_write_space = xs_tcp_write_space;
b4411457 2190 sock_set_flag(sk, SOCK_FASYNC);
2118071d 2191 sk->sk_error_report = xs_error_report;
3167e12c
CL		 2192
2193 /* socket options */
3167e12c 2194 sock_reset_flag(sk, SOCK_LINGER);
b0d93ad5
CL		 2195
2196 xprt_clear_connected(xprt);
2197
2198 /* Reset to new socket */
ee0ac0c2
CL		 2199 transport->sock = sock;
2200 transport->inet = sk;
b0d93ad5
CL		 2201
2202 write_unlock_bh(&sk->sk_callback_lock);
2203 }
2204
01d37c42 2205 if (!xprt_bound(xprt))
fe19a96b 2206 goto out;
01d37c42 2207
a564b8f0
MG		 2208 xs_set_memalloc(xprt);
2209
ae053551 2210 xs_stream_start_connect(transport);
e1806c7b 2211
b0d93ad5 2212 /* Tell the socket layer to start connecting... */
0fdea1e8 2213 set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
fe19a96b
TM		 2214 ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);
2215 switch (ret) {
2216 case 0:
4dda9c8a 2217 xs_set_srcport(transport, sock);
df561f66 2218 fallthrough;
fe19a96b
TM		 2219 case -EINPROGRESS:
2220 /* SYN_SENT! */
fe19a96b
TM		 2221 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
2222 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
1f4c17a0
TM		 2223 break;
2224 case -EADDRNOTAVAIL:
2225 /* Source port number is unavailable. Try a new one! */
2226 transport->srcport = 0;
fe19a96b
TM		 2227 }
2228out:
2229 return ret;
16be2d20
CL		 2230}
2231
9903cd1c 2232/**
b61d59ff 2233 * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint
acf0a39f 2234 * @work: queued work item
9903cd1c
CL		 2235 *
2236 * Invoked by a work queue tasklet.
a246b010 2237 */
cdd518d5 2238static void xs_tcp_setup_socket(struct work_struct *work)
a246b010 2239{
cdd518d5
PE		 2240 struct sock_xprt *transport =
2241 container_of(work, struct sock_xprt, connect_worker.work);
ee0ac0c2 2242 struct socket *sock = transport->sock;
a9f5f0f7 2243 struct rpc_xprt *xprt = &transport->xprt;
b61d59ff 2244 int status = -EIO;
a246b010 2245
ee0ac0c2 2246 if (!sock) {
cdd518d5 2247 sock = xs_create_sock(xprt, transport,
4dda9c8a
TM		 2248 xs_addr(xprt)->sa_family, SOCK_STREAM,
2249 IPPROTO_TCP, true);
b61d59ff
TM		 2250 if (IS_ERR(sock)) {
2251 status = PTR_ERR(sock);
3167e12c
CL		 2252 goto out;
2253 }
7d1e8255 2254 }
a246b010 2255
c740eff8
CL		 2256 dprintk("RPC: worker connecting xprt %p via %s to "
2257 "%s (port %s)\n", xprt,
2258 xprt->address_strings[RPC_DISPLAY_PROTO],
2259 xprt->address_strings[RPC_DISPLAY_ADDR],
2260 xprt->address_strings[RPC_DISPLAY_PORT]);
edb267a6 2261
16be2d20 2262 status = xs_tcp_finish_connecting(xprt, sock);
40b5ea0c 2263 trace_rpc_socket_connect(xprt, sock, status);
46121cf7
CL		 2264 dprintk("RPC: %p connect status %d connected %d sock state %d\n",
2265 xprt, -status, xprt_connected(xprt),
2266 sock->sk->sk_state);
2a491991 2267 switch (status) {
f75e6745
TM		 2268 default:
2269 printk("%s: connect returned unhandled error %d\n",
2270 __func__, status);
df561f66 2271 fallthrough;
f75e6745
TM		 2272 case -EADDRNOTAVAIL:
2273 /* We're probably in TIME_WAIT. Get rid of existing socket,
2274 * and retry
2275 */
a519fc7a 2276 xs_tcp_force_close(xprt);
88b5ed73 2277 break;
2a491991
TM		 2278 case 0:
2279 case -EINPROGRESS:
2280 case -EALREADY:
718ba5b8 2281 xprt_unlock_connect(xprt, transport);
7d1e8255 2282 return;
9fcfe0c8
TM		 2283 case -EINVAL:
2284 /* Happens, for instance, if the user specified a link
2285 * local IPv6 address without a scope-id.
2286 */
3ed5e2a2
TM		 2287 case -ECONNREFUSED:
2288 case -ECONNRESET:
eb5b46fa 2289 case -ENETDOWN:
3ed5e2a2 2290 case -ENETUNREACH:
4ba161a7 2291 case -EHOSTUNREACH:
3913c78c 2292 case -EADDRINUSE:
3601c4a9 2293 case -ENOBUFS:
8c71139d
CM		 2294 /* xs_tcp_force_close() wakes tasks with a fixed error code.
2295 * We need to wake them first to ensure the correct error code.
6ea44adc
N		 2296 */
2297 xprt_wake_pending_tasks(xprt, status);
4efdd92c 2298 xs_tcp_force_close(xprt);
9fcfe0c8 2299 goto out;
a246b010 2300 }
2a491991 2301 status = -EAGAIN;
a246b010 2302out:
68e220bd 2303 xprt_clear_connecting(xprt);
cf76785d 2304 xprt_unlock_connect(xprt, transport);
7d1e8255 2305 xprt_wake_pending_tasks(xprt, status);
68e220bd 2306}
b0d93ad5 2307
9903cd1c
CL		 2308/**
2309 * xs_connect - connect a socket to a remote endpoint
1b092092 2310 * @xprt: pointer to transport structure
9903cd1c
CL		 2311 * @task: address of RPC task that manages state of connect request
2312 *
2313 * TCP: If the remote end dropped the connection, delay reconnecting.
03bf4b70
CL		 2314 *
2315 * UDP socket connects are synchronous, but we use a work queue anyway
2316 * to guarantee that even unprivileged user processes can set up a
2317 * socket on a privileged port.
2318 *
2319 * If a UDP socket connect fails, the delay behavior here prevents
2320 * retry floods (hard mounts).
9903cd1c 2321 */
1b092092 2322static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
a246b010 2323{
ee0ac0c2 2324 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
02910177 2325 unsigned long delay = 0;
a246b010 2326
718ba5b8
TM		 2327 WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));
2328
99b1a4c3 2329 if (transport->sock != NULL) {
46121cf7
CL		 2330 dprintk("RPC: xs_connect delayed xprt %p for %lu "
2331 "seconds\n",
03bf4b70 2332 xprt, xprt->reestablish_timeout / HZ);
99b1a4c3
TM		 2333
2334 /* Start by resetting any existing state */
2335 xs_reset_transport(transport);
2336
675dd90a
CL		 2337 delay = xprt_reconnect_delay(xprt);
2338 xprt_reconnect_backoff(xprt, XS_TCP_INIT_REEST_TO);
02910177 2339
02910177 2340 } else
46121cf7 2341 dprintk("RPC: xs_connect scheduled xprt %p\n", xprt);
02910177
TM		 2342
2343 queue_delayed_work(xprtiod_workqueue,
2344 &transport->connect_worker,
2345 delay);
a246b010
CL		 2346}
2347
4f8943f8
TM		 2348static void xs_wake_disconnect(struct sock_xprt *transport)
2349{
2350 if (test_and_clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state))
2351 xs_tcp_force_close(&transport->xprt);
2352}
2353
2354static void xs_wake_write(struct sock_xprt *transport)
2355{
2356 if (test_and_clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state))
2357 xprt_write_space(&transport->xprt);
2358}
2359
2360static void xs_wake_error(struct sock_xprt *transport)
2361{
2362 int sockerr;
4f8943f8
TM		 2363
2364 if (!test_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state))
2365 return;
2366 mutex_lock(&transport->recv_mutex);
2367 if (transport->sock == NULL)
2368 goto out;
2369 if (!test_and_clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state))
2370 goto out;
af84537d 2371 sockerr = xchg(&transport->xprt_err, 0);
4f8943f8
TM		 2372 if (sockerr < 0)
2373 xprt_wake_pending_tasks(&transport->xprt, sockerr);
2374out:
2375 mutex_unlock(&transport->recv_mutex);
2376}
2377
2378static void xs_wake_pending(struct sock_xprt *transport)
2379{
2380 if (test_and_clear_bit(XPRT_SOCK_WAKE_PENDING, &transport->sock_state))
2381 xprt_wake_pending_tasks(&transport->xprt, -EAGAIN);
2382}
2383
2384static void xs_error_handle(struct work_struct *work)
2385{
2386 struct sock_xprt *transport = container_of(work,
2387 struct sock_xprt, error_worker);
2388
2389 xs_wake_disconnect(transport);
2390 xs_wake_write(transport);
2391 xs_wake_error(transport);
2392 xs_wake_pending(transport);
2393}
2394
176e21ee 2395/**
12b20ce3 2396 * xs_local_print_stats - display AF_LOCAL socket-specific stats
176e21ee
CL		 2397 * @xprt: rpc_xprt struct containing statistics
2398 * @seq: output file
2399 *
2400 */
2401static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2402{
2403 long idle_time = 0;
2404
2405 if (xprt_connected(xprt))
2406 idle_time = (long)(jiffies - xprt->last_used) / HZ;
2407
2408 seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu "
15a45206 2409 "%llu %llu %lu %llu %llu\n",
176e21ee
CL		 2410 xprt->stat.bind_count,
2411 xprt->stat.connect_count,
8440a886 2412 xprt->stat.connect_time / HZ,
176e21ee
CL		 2413 idle_time,
2414 xprt->stat.sends,
2415 xprt->stat.recvs,
2416 xprt->stat.bad_xids,
2417 xprt->stat.req_u,
15a45206
AA		 2418 xprt->stat.bklog_u,
2419 xprt->stat.max_slots,
2420 xprt->stat.sending_u,
2421 xprt->stat.pending_u);
176e21ee
CL		 2422}
2423
262ca07d 2424/**
12b20ce3 2425 * xs_udp_print_stats - display UDP socket-specific stats
262ca07d
CL		 2426 * @xprt: rpc_xprt struct containing statistics
2427 * @seq: output file
2428 *
2429 */
2430static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2431{
c8475461
CL		 2432 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2433
15a45206
AA		 2434 seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu "
2435 "%lu %llu %llu\n",
fbfffbd5 2436 transport->srcport,
262ca07d
CL		 2437 xprt->stat.bind_count,
2438 xprt->stat.sends,
2439 xprt->stat.recvs,
2440 xprt->stat.bad_xids,
2441 xprt->stat.req_u,
15a45206
AA		 2442 xprt->stat.bklog_u,
2443 xprt->stat.max_slots,
2444 xprt->stat.sending_u,
2445 xprt->stat.pending_u);
262ca07d
CL		 2446}
2447
2448/**
12b20ce3 2449 * xs_tcp_print_stats - display TCP socket-specific stats
262ca07d
CL		 2450 * @xprt: rpc_xprt struct containing statistics
2451 * @seq: output file
2452 *
2453 */
2454static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2455{
c8475461 2456 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262ca07d
CL		 2457 long idle_time = 0;
2458
2459 if (xprt_connected(xprt))
2460 idle_time = (long)(jiffies - xprt->last_used) / HZ;
2461
15a45206
AA		 2462 seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu "
2463 "%llu %llu %lu %llu %llu\n",
fbfffbd5 2464 transport->srcport,
262ca07d
CL		 2465 xprt->stat.bind_count,
2466 xprt->stat.connect_count,
8440a886 2467 xprt->stat.connect_time / HZ,
262ca07d
CL		 2468 idle_time,
2469 xprt->stat.sends,
2470 xprt->stat.recvs,
2471 xprt->stat.bad_xids,
2472 xprt->stat.req_u,
15a45206
AA		 2473 xprt->stat.bklog_u,
2474 xprt->stat.max_slots,
2475 xprt->stat.sending_u,
2476 xprt->stat.pending_u);
262ca07d
CL		 2477}
2478
4cfc7e60
RI		 2479/*
2480 * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason
2481 * we allocate pages instead doing a kmalloc like rpc_malloc is because we want
2482 * to use the server side send routines.
2483 */
5fe6eaa1 2484static int bc_malloc(struct rpc_task *task)
4cfc7e60 2485{
5fe6eaa1
CL		 2486 struct rpc_rqst *rqst = task->tk_rqstp;
2487 size_t size = rqst->rq_callsize;
4cfc7e60
RI		 2488 struct page *page;
2489 struct rpc_buffer *buf;
2490
5fe6eaa1
CL		 2491 if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) {
2492 WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n",
2493 size);
2494 return -EINVAL;
2495 }
4cfc7e60 2496
b8a13d03 2497 page = alloc_page(GFP_KERNEL);
4cfc7e60 2498 if (!page)
5fe6eaa1 2499 return -ENOMEM;
4cfc7e60
RI		 2500
2501 buf = page_address(page);
2502 buf->len = PAGE_SIZE;
2503
5fe6eaa1 2504 rqst->rq_buffer = buf->data;
18e601d6 2505 rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize;
5fe6eaa1 2506 return 0;
4cfc7e60
RI		 2507}
2508
2509/*
2510 * Free the space allocated in the bc_alloc routine
2511 */
3435c74a 2512static void bc_free(struct rpc_task *task)
4cfc7e60 2513{
3435c74a 2514 void *buffer = task->tk_rqstp->rq_buffer;
4cfc7e60
RI		 2515 struct rpc_buffer *buf;
2516
4cfc7e60
RI		 2517 buf = container_of(buffer, struct rpc_buffer, data);
2518 free_page((unsigned long)buf);
2519}
2520
4cfc7e60
RI		 2521static int bc_sendto(struct rpc_rqst *req)
2522{
da1661b9 2523 struct xdr_buf *xdr = &req->rq_snd_buf;
4cfc7e60 2524 struct sock_xprt *transport =
067fb11b 2525 container_of(req->rq_xprt, struct sock_xprt, xprt);
067fb11b 2526 struct msghdr msg = {
da1661b9 2527 .msg_flags = 0,
067fb11b
CL		 2528 };
2529 rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT |
da1661b9
CL		 2530 (u32)xdr->len);
2531 unsigned int sent = 0;
2532 int err;
4cfc7e60 2533
8729aaba 2534 req->rq_xtime = ktime_get();
da1661b9
CL		 2535 err = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, marker, &sent);
2536 xdr_free_bvec(xdr);
2537 if (err < 0 || sent != (xdr->len + sizeof(marker)))
067fb11b 2538 return -EAGAIN;
da1661b9 2539 return sent;
4cfc7e60
RI		 2540}
2541
ca4faf54
CL		 2542/**
2543 * bc_send_request - Send a backchannel Call on a TCP socket
2544 * @req: rpc_rqst containing Call message to be sent
2545 *
2546 * xpt_mutex ensures @rqstp's whole message is written to the socket
2547 * without interruption.
2548 *
2549 * Return values:
2550 * %0 if the message was sent successfully
2551 * %ENOTCONN if the message was not sent
4cfc7e60 2552 */
adfa7144 2553static int bc_send_request(struct rpc_rqst *req)
4cfc7e60 2554{
4cfc7e60 2555 struct svc_xprt *xprt;
7fc56136 2556 int len;
4cfc7e60 2557
4cfc7e60
RI		 2558 /*
2559 * Get the server socket associated with this callback xprt
2560 */
2561 xprt = req->rq_xprt->bc_xprt;
4cfc7e60
RI		 2562
2563 /*
2564 * Grab the mutex to serialize data as the connection is shared
2565 * with the fore channel
2566 */
c544577d 2567 mutex_lock(&xprt->xpt_mutex);
4cfc7e60
RI		 2568 if (test_bit(XPT_DEAD, &xprt->xpt_flags))
2569 len = -ENOTCONN;
2570 else
2571 len = bc_sendto(req);
2572 mutex_unlock(&xprt->xpt_mutex);
2573
2574 if (len > 0)
2575 len = 0;
2576
2577 return len;
2578}
2579
2580/*
2581 * The close routine. Since this is client initiated, we do nothing
2582 */
2583
2584static void bc_close(struct rpc_xprt *xprt)
2585{
6221f1d9 2586 xprt_disconnect_done(xprt);
4cfc7e60
RI		 2587}
2588
2589/*
2590 * The xprt destroy routine. Again, because this connection is client
2591 * initiated, we do nothing
2592 */
2593
2594static void bc_destroy(struct rpc_xprt *xprt)
2595{
47f72efa
KM		 2596 dprintk("RPC: bc_destroy xprt %p\n", xprt);
2597
2598 xs_xprt_free(xprt);
2599 module_put(THIS_MODULE);
4cfc7e60
RI		 2600}
2601
d31ae254 2602static const struct rpc_xprt_ops xs_local_ops = {
176e21ee 2603 .reserve_xprt = xprt_reserve_xprt,
4cd34e7c 2604 .release_xprt = xprt_release_xprt,
f39c1bfb 2605 .alloc_slot = xprt_alloc_slot,
a9cde23a 2606 .free_slot = xprt_free_slot,
176e21ee
CL		 2607 .rpcbind = xs_local_rpcbind,
2608 .set_port = xs_local_set_port,
dc107402 2609 .connect = xs_local_connect,
176e21ee
CL		 2610 .buf_alloc = rpc_malloc,
2611 .buf_free = rpc_free,
550aebfe 2612 .prepare_request = xs_stream_prepare_request,
176e21ee 2613 .send_request = xs_local_send_request,
8ba6a92d 2614 .wait_for_reply_request = xprt_wait_for_reply_request_def,
176e21ee 2615 .close = xs_close,
a1311d87 2616 .destroy = xs_destroy,
176e21ee 2617 .print_stats = xs_local_print_stats,
d67fa4d8
JL		 2618 .enable_swap = xs_enable_swap,
2619 .disable_swap = xs_disable_swap,
176e21ee
CL		 2620};
2621
d31ae254 2622static const struct rpc_xprt_ops xs_udp_ops = {
43118c29 2623 .set_buffer_size = xs_udp_set_buffer_size,
12a80469 2624 .reserve_xprt = xprt_reserve_xprt_cong,
49e9a890 2625 .release_xprt = xprt_release_xprt_cong,
f39c1bfb 2626 .alloc_slot = xprt_alloc_slot,
a9cde23a 2627 .free_slot = xprt_free_slot,
45160d62 2628 .rpcbind = rpcb_getport_async,
92200412 2629 .set_port = xs_set_port,
262965f5 2630 .connect = xs_connect,
02107148
CL		 2631 .buf_alloc = rpc_malloc,
2632 .buf_free = rpc_free,
262965f5 2633 .send_request = xs_udp_send_request,
8ba6a92d 2634 .wait_for_reply_request = xprt_wait_for_reply_request_rtt,
46c0ee8b 2635 .timer = xs_udp_timer,
a58dd398 2636 .release_request = xprt_release_rqst_cong,
262965f5
CL		 2637 .close = xs_close,
2638 .destroy = xs_destroy,
262ca07d 2639 .print_stats = xs_udp_print_stats,
d67fa4d8
JL		 2640 .enable_swap = xs_enable_swap,
2641 .disable_swap = xs_disable_swap,
4a068258 2642 .inject_disconnect = xs_inject_disconnect,
262965f5
CL		 2643};
2644
d31ae254 2645static const struct rpc_xprt_ops xs_tcp_ops = {
12a80469 2646 .reserve_xprt = xprt_reserve_xprt,
4cd34e7c 2647 .release_xprt = xprt_release_xprt,
36bd7de9 2648 .alloc_slot = xprt_alloc_slot,
a9cde23a 2649 .free_slot = xprt_free_slot,
45160d62 2650 .rpcbind = rpcb_getport_async,
92200412 2651 .set_port = xs_set_port,
0b9e7943 2652 .connect = xs_connect,
02107148
CL		 2653 .buf_alloc = rpc_malloc,
2654 .buf_free = rpc_free,
277e4ab7 2655 .prepare_request = xs_stream_prepare_request,
262965f5 2656 .send_request = xs_tcp_send_request,
8ba6a92d 2657 .wait_for_reply_request = xprt_wait_for_reply_request_def,
c627d31b 2658 .close = xs_tcp_shutdown,
9903cd1c 2659 .destroy = xs_destroy,
7196dbb0 2660 .set_connect_timeout = xs_tcp_set_connect_timeout,
262ca07d 2661 .print_stats = xs_tcp_print_stats,
d67fa4d8
JL		 2662 .enable_swap = xs_enable_swap,
2663 .disable_swap = xs_disable_swap,
4a068258 2664 .inject_disconnect = xs_inject_disconnect,
42e5c3e2
CL		 2665#ifdef CONFIG_SUNRPC_BACKCHANNEL
2666 .bc_setup = xprt_setup_bc,
6b26cc8c 2667 .bc_maxpayload = xs_tcp_bc_maxpayload,
7402a4fe 2668 .bc_num_slots = xprt_bc_max_slots,
42e5c3e2
CL		 2669 .bc_free_rqst = xprt_free_bc_rqst,
2670 .bc_destroy = xprt_destroy_bc,
2671#endif
a246b010
CL		 2672};
2673
4cfc7e60
RI		 2674/*
2675 * The rpc_xprt_ops for the server backchannel
2676 */
2677
d31ae254 2678static const struct rpc_xprt_ops bc_tcp_ops = {
4cfc7e60
RI		 2679 .reserve_xprt = xprt_reserve_xprt,
2680 .release_xprt = xprt_release_xprt,
84e28a30 2681 .alloc_slot = xprt_alloc_slot,
a9cde23a 2682 .free_slot = xprt_free_slot,
4cfc7e60
RI		 2683 .buf_alloc = bc_malloc,
2684 .buf_free = bc_free,
2685 .send_request = bc_send_request,
8ba6a92d 2686 .wait_for_reply_request = xprt_wait_for_reply_request_def,
4cfc7e60
RI		 2687 .close = bc_close,
2688 .destroy = bc_destroy,
2689 .print_stats = xs_tcp_print_stats,
d67fa4d8
JL		 2690 .enable_swap = xs_enable_swap,
2691 .disable_swap = xs_disable_swap,
4a068258 2692 .inject_disconnect = xs_inject_disconnect,
4cfc7e60
RI		 2693};
2694
92476850
CL		 2695static int xs_init_anyaddr(const int family, struct sockaddr *sap)
2696{
2697 static const struct sockaddr_in sin = {
2698 .sin_family = AF_INET,
2699 .sin_addr.s_addr = htonl(INADDR_ANY),
2700 };
2701 static const struct sockaddr_in6 sin6 = {
2702 .sin6_family = AF_INET6,
2703 .sin6_addr = IN6ADDR_ANY_INIT,
2704 };
2705
2706 switch (family) {
176e21ee
CL		 2707 case AF_LOCAL:
2708 break;
92476850
CL		 2709 case AF_INET:
2710 memcpy(sap, &sin, sizeof(sin));
2711 break;
2712 case AF_INET6:
2713 memcpy(sap, &sin6, sizeof(sin6));
2714 break;
2715 default:
2716 dprintk("RPC: %s: Bad address family\n", __func__);
2717 return -EAFNOSUPPORT;
2718 }
2719 return 0;
2720}
2721
3c341b0b 2722static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args,
d9ba131d
TM		 2723 unsigned int slot_table_size,
2724 unsigned int max_slot_table_size)
c8541ecd
CL		 2725{
2726 struct rpc_xprt *xprt;
ffc2e518 2727 struct sock_xprt *new;
c8541ecd 2728
96802a09 2729 if (args->addrlen > sizeof(xprt->addr)) {
46121cf7 2730 dprintk("RPC: xs_setup_xprt: address too large\n");
c8541ecd
CL		 2731 return ERR_PTR(-EBADF);
2732 }
2733
d9ba131d
TM		 2734 xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size,
2735 max_slot_table_size);
bd1722d4 2736 if (xprt == NULL) {
46121cf7
CL		 2737 dprintk("RPC: xs_setup_xprt: couldn't allocate "
2738 "rpc_xprt\n");
c8541ecd
CL		 2739 return ERR_PTR(-ENOMEM);
2740 }
c8541ecd 2741
bd1722d4 2742 new = container_of(xprt, struct sock_xprt, xprt);
edc1b01c 2743 mutex_init(&new->recv_mutex);
96802a09
FM		 2744 memcpy(&xprt->addr, args->dstaddr, args->addrlen);
2745 xprt->addrlen = args->addrlen;
d3bc9a1d 2746 if (args->srcaddr)
fbfffbd5 2747 memcpy(&new->srcaddr, args->srcaddr, args->addrlen);
92476850
CL		 2748 else {
2749 int err;
2750 err = xs_init_anyaddr(args->dstaddr->sa_family,
2751 (struct sockaddr *)&new->srcaddr);
2aa13531
SK		 2752 if (err != 0) {
2753 xprt_free(xprt);
92476850 2754 return ERR_PTR(err);
2aa13531 2755 }
92476850 2756 }
c8541ecd
CL		 2757
2758 return xprt;
2759}
2760
176e21ee
CL		 2761static const struct rpc_timeout xs_local_default_timeout = {
2762 .to_initval = 10 * HZ,
2763 .to_maxval = 10 * HZ,
2764 .to_retries = 2,
2765};
2766
2767/**
2768 * xs_setup_local - Set up transport to use an AF_LOCAL socket
2769 * @args: rpc transport creation arguments
2770 *
2771 * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP
2772 */
2773static struct rpc_xprt *xs_setup_local(struct xprt_create *args)
2774{
2775 struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr;
2776 struct sock_xprt *transport;
2777 struct rpc_xprt *xprt;
2778 struct rpc_xprt *ret;
2779
d9ba131d
TM		 2780 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
2781 xprt_max_tcp_slot_table_entries);
176e21ee
CL		 2782 if (IS_ERR(xprt))
2783 return xprt;
2784 transport = container_of(xprt, struct sock_xprt, xprt);
2785
2786 xprt->prot = 0;
d3abc739 2787 xprt->xprt_class = &xs_local_transport;
176e21ee
CL		 2788 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
2789
2790 xprt->bind_timeout = XS_BIND_TO;
2791 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2792 xprt->idle_timeout = XS_IDLE_DISC_TO;
2793
2794 xprt->ops = &xs_local_ops;
2795 xprt->timeout = &xs_local_default_timeout;
2796
550aebfe 2797 INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn);
4f8943f8 2798 INIT_WORK(&transport->error_worker, xs_error_handle);
550aebfe 2799 INIT_DELAYED_WORK(&transport->connect_worker, xs_dummy_setup_socket);
93dc41bd 2800
176e21ee
CL		 2801 switch (sun->sun_family) {
2802 case AF_LOCAL:
2803 if (sun->sun_path[0] != '/') {
2804 dprintk("RPC: bad AF_LOCAL address: %s\n",
2805 sun->sun_path);
2806 ret = ERR_PTR(-EINVAL);
2807 goto out_err;
2808 }
2809 xprt_set_bound(xprt);
176e21ee 2810 xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL);
7073ea87
BF		 2811 ret = ERR_PTR(xs_local_setup_socket(transport));
2812 if (ret)
2813 goto out_err;
176e21ee
CL		 2814 break;
2815 default:
2816 ret = ERR_PTR(-EAFNOSUPPORT);
2817 goto out_err;
2818 }
2819
2820 dprintk("RPC: set up xprt to %s via AF_LOCAL\n",
2821 xprt->address_strings[RPC_DISPLAY_ADDR]);
2822
2823 if (try_module_get(THIS_MODULE))
2824 return xprt;
2825 ret = ERR_PTR(-EINVAL);
2826out_err:
315f3812 2827 xs_xprt_free(xprt);
176e21ee
CL		 2828 return ret;
2829}
2830
2881ae74
TM		 2831static const struct rpc_timeout xs_udp_default_timeout = {
2832 .to_initval = 5 * HZ,
2833 .to_maxval = 30 * HZ,
2834 .to_increment = 5 * HZ,
2835 .to_retries = 5,
2836};
2837
9903cd1c
CL		 2838/**
2839 * xs_setup_udp - Set up transport to use a UDP socket
96802a09 2840 * @args: rpc transport creation arguments
9903cd1c
CL		 2841 *
2842 */
483066d6 2843static struct rpc_xprt *xs_setup_udp(struct xprt_create *args)
a246b010 2844{
8f9d5b1a 2845 struct sockaddr *addr = args->dstaddr;
c8541ecd 2846 struct rpc_xprt *xprt;
c8475461 2847 struct sock_xprt *transport;
0a68b0be 2848 struct rpc_xprt *ret;
a246b010 2849
d9ba131d
TM		 2850 xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries,
2851 xprt_udp_slot_table_entries);
c8541ecd
CL		 2852 if (IS_ERR(xprt))
2853 return xprt;
c8475461 2854 transport = container_of(xprt, struct sock_xprt, xprt);
a246b010 2855
ec739ef0 2856 xprt->prot = IPPROTO_UDP;
d3abc739 2857 xprt->xprt_class = &xs_udp_transport;
a246b010
CL		 2858 /* XXX: header size can vary due to auth type, IPv6, etc. */
2859 xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
2860
03bf4b70 2861 xprt->bind_timeout = XS_BIND_TO;
03bf4b70
CL		 2862 xprt->reestablish_timeout = XS_UDP_REEST_TO;
2863 xprt->idle_timeout = XS_IDLE_DISC_TO;
a246b010 2864
262965f5 2865 xprt->ops = &xs_udp_ops;
a246b010 2866
ba7392bb 2867 xprt->timeout = &xs_udp_default_timeout;
a246b010 2868
f9b2ee71 2869 INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn);
4f8943f8 2870 INIT_WORK(&transport->error_worker, xs_error_handle);
edc1b01c
TM		 2871 INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket);
2872
8f9d5b1a
CL		 2873 switch (addr->sa_family) {
2874 case AF_INET:
2875 if (((struct sockaddr_in *)addr)->sin_port != htons(0))
2876 xprt_set_bound(xprt);
2877
9dc3b095 2878 xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP);
8f9d5b1a
CL		 2879 break;
2880 case AF_INET6:
2881 if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
2882 xprt_set_bound(xprt);
2883
9dc3b095 2884 xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6);
8f9d5b1a
CL		 2885 break;
2886 default:
0a68b0be
BF		 2887 ret = ERR_PTR(-EAFNOSUPPORT);
2888 goto out_err;
8f9d5b1a
CL		 2889 }
2890
c740eff8
CL		 2891 if (xprt_bound(xprt))
2892 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
2893 xprt->address_strings[RPC_DISPLAY_ADDR],
2894 xprt->address_strings[RPC_DISPLAY_PORT],
2895 xprt->address_strings[RPC_DISPLAY_PROTO]);
2896 else
2897 dprintk("RPC: set up xprt to %s (autobind) via %s\n",
2898 xprt->address_strings[RPC_DISPLAY_ADDR],
2899 xprt->address_strings[RPC_DISPLAY_PROTO]);
edb267a6 2900
bc25571e
TT		 2901 if (try_module_get(THIS_MODULE))
2902 return xprt;
0a68b0be
BF		 2903 ret = ERR_PTR(-EINVAL);
2904out_err:
315f3812 2905 xs_xprt_free(xprt);
0a68b0be 2906 return ret;
a246b010
CL		 2907}
2908
2881ae74
TM		 2909static const struct rpc_timeout xs_tcp_default_timeout = {
2910 .to_initval = 60 * HZ,
2911 .to_maxval = 60 * HZ,
2912 .to_retries = 2,
2913};
2914
9903cd1c
CL		 2915/**
2916 * xs_setup_tcp - Set up transport to use a TCP socket
96802a09 2917 * @args: rpc transport creation arguments
9903cd1c
CL		 2918 *
2919 */
483066d6 2920static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args)
a246b010 2921{
8f9d5b1a 2922 struct sockaddr *addr = args->dstaddr;
c8541ecd 2923 struct rpc_xprt *xprt;
c8475461 2924 struct sock_xprt *transport;
0a68b0be 2925 struct rpc_xprt *ret;
b7993ceb
TM		 2926 unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries;
2927
2928 if (args->flags & XPRT_CREATE_INFINITE_SLOTS)
2929 max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT;
a246b010 2930
d9ba131d 2931 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
b7993ceb 2932 max_slot_table_size);
c8541ecd
CL		 2933 if (IS_ERR(xprt))
2934 return xprt;
c8475461 2935 transport = container_of(xprt, struct sock_xprt, xprt);
a246b010 2936
ec739ef0 2937 xprt->prot = IPPROTO_TCP;
d3abc739 2938 xprt->xprt_class = &xs_tcp_transport;
808012fb 2939 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
a246b010 2940
03bf4b70 2941 xprt->bind_timeout = XS_BIND_TO;
03bf4b70
CL		 2942 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2943 xprt->idle_timeout = XS_IDLE_DISC_TO;
a246b010 2944
262965f5 2945 xprt->ops = &xs_tcp_ops;
ba7392bb 2946 xprt->timeout = &xs_tcp_default_timeout;
a246b010 2947
3851f1cd 2948 xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
7196dbb0
TM		 2949 xprt->connect_timeout = xprt->timeout->to_initval *
2950 (xprt->timeout->to_retries + 1);
3851f1cd 2951
c50b8ee0 2952 INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn);
4f8943f8 2953 INIT_WORK(&transport->error_worker, xs_error_handle);
edc1b01c
TM		 2954 INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket);
2955
8f9d5b1a
CL		 2956 switch (addr->sa_family) {
2957 case AF_INET:
2958 if (((struct sockaddr_in *)addr)->sin_port != htons(0))
2959 xprt_set_bound(xprt);
2960
9dc3b095 2961 xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP);
8f9d5b1a
CL		 2962 break;
2963 case AF_INET6:
2964 if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
2965 xprt_set_bound(xprt);
2966
9dc3b095 2967 xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6);
8f9d5b1a
CL		 2968 break;
2969 default:
0a68b0be
BF		 2970 ret = ERR_PTR(-EAFNOSUPPORT);
2971 goto out_err;
8f9d5b1a
CL		 2972 }
2973
c740eff8
CL		 2974 if (xprt_bound(xprt))
2975 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
2976 xprt->address_strings[RPC_DISPLAY_ADDR],
2977 xprt->address_strings[RPC_DISPLAY_PORT],
2978 xprt->address_strings[RPC_DISPLAY_PROTO]);
2979 else
2980 dprintk("RPC: set up xprt to %s (autobind) via %s\n",
2981 xprt->address_strings[RPC_DISPLAY_ADDR],
2982 xprt->address_strings[RPC_DISPLAY_PROTO]);
2983
bc25571e
TT		 2984 if (try_module_get(THIS_MODULE))
2985 return xprt;
0a68b0be
BF		 2986 ret = ERR_PTR(-EINVAL);
2987out_err:
315f3812 2988 xs_xprt_free(xprt);
0a68b0be 2989 return ret;
a246b010 2990}
282b32e1 2991
f300baba
AB		 2992/**
2993 * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket
2994 * @args: rpc transport creation arguments
2995 *
2996 */
2997static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args)
2998{
2999 struct sockaddr *addr = args->dstaddr;
3000 struct rpc_xprt *xprt;
3001 struct sock_xprt *transport;
3002 struct svc_sock *bc_sock;
0a68b0be 3003 struct rpc_xprt *ret;
f300baba 3004
d9ba131d
TM		 3005 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
3006 xprt_tcp_slot_table_entries);
f300baba
AB		 3007 if (IS_ERR(xprt))
3008 return xprt;
3009 transport = container_of(xprt, struct sock_xprt, xprt);
3010
3011 xprt->prot = IPPROTO_TCP;
d3abc739 3012 xprt->xprt_class = &xs_bc_tcp_transport;
f300baba
AB		 3013 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
3014 xprt->timeout = &xs_tcp_default_timeout;
3015
3016 /* backchannel */
3017 xprt_set_bound(xprt);
3018 xprt->bind_timeout = 0;
f300baba
AB		 3019 xprt->reestablish_timeout = 0;
3020 xprt->idle_timeout = 0;
3021
f300baba
AB		 3022 xprt->ops = &bc_tcp_ops;
3023
3024 switch (addr->sa_family) {
3025 case AF_INET:
3026 xs_format_peer_addresses(xprt, "tcp",
3027 RPCBIND_NETID_TCP);
3028 break;
3029 case AF_INET6:
3030 xs_format_peer_addresses(xprt, "tcp",
3031 RPCBIND_NETID_TCP6);
3032 break;
3033 default:
0a68b0be
BF		 3034 ret = ERR_PTR(-EAFNOSUPPORT);
3035 goto out_err;
f300baba
AB		 3036 }
3037
50fa0d40
PE		 3038 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3039 xprt->address_strings[RPC_DISPLAY_ADDR],
3040 xprt->address_strings[RPC_DISPLAY_PORT],
3041 xprt->address_strings[RPC_DISPLAY_PROTO]);
f300baba 3042
99de8ea9
BF		 3043 /*
3044 * Once we've associated a backchannel xprt with a connection,
28303ca3
WM		 3045 * we want to keep it around as long as the connection lasts,
3046 * in case we need to start using it for a backchannel again;
3047 * this reference won't be dropped until bc_xprt is destroyed.
99de8ea9
BF		 3048 */
3049 xprt_get(xprt);
3050 args->bc_xprt->xpt_bc_xprt = xprt;
3051 xprt->bc_xprt = args->bc_xprt;
3052 bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt);
3053 transport->sock = bc_sock->sk_sock;
3054 transport->inet = bc_sock->sk_sk;
3055
f300baba
AB		 3056 /*
3057 * Since we don't want connections for the backchannel, we set
3058 * the xprt status to connected
3059 */
3060 xprt_set_connected(xprt);
3061
f300baba
AB		 3062 if (try_module_get(THIS_MODULE))
3063 return xprt;
642aab58
KM		 3064
3065 args->bc_xprt->xpt_bc_xprt = NULL;
39a9beab 3066 args->bc_xprt->xpt_bc_xps = NULL;
99de8ea9 3067 xprt_put(xprt);
0a68b0be
BF		 3068 ret = ERR_PTR(-EINVAL);
3069out_err:
315f3812 3070 xs_xprt_free(xprt);
0a68b0be 3071 return ret;
f300baba
AB		 3072}
3073
176e21ee
CL		 3074static struct xprt_class xs_local_transport = {
3075 .list = LIST_HEAD_INIT(xs_local_transport.list),
3076 .name = "named UNIX socket",
3077 .owner = THIS_MODULE,
3078 .ident = XPRT_TRANSPORT_LOCAL,
3079 .setup = xs_setup_local,
d5aa6b22 3080 .netid = { "" },
176e21ee
CL		 3081};
3082
bc25571e
TT		 3083static struct xprt_class xs_udp_transport = {
3084 .list = LIST_HEAD_INIT(xs_udp_transport.list),
3085 .name = "udp",
3086 .owner = THIS_MODULE,
f300baba 3087 .ident = XPRT_TRANSPORT_UDP,
bc25571e 3088 .setup = xs_setup_udp,
d5aa6b22 3089 .netid = { "udp", "udp6", "" },
bc25571e
TT		 3090};
3091
3092static struct xprt_class xs_tcp_transport = {
3093 .list = LIST_HEAD_INIT(xs_tcp_transport.list),
3094 .name = "tcp",
3095 .owner = THIS_MODULE,
f300baba 3096 .ident = XPRT_TRANSPORT_TCP,
bc25571e 3097 .setup = xs_setup_tcp,
d5aa6b22 3098 .netid = { "tcp", "tcp6", "" },
bc25571e
TT		 3099};
3100
f300baba
AB		 3101static struct xprt_class xs_bc_tcp_transport = {
3102 .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list),
3103 .name = "tcp NFSv4.1 backchannel",
3104 .owner = THIS_MODULE,
3105 .ident = XPRT_TRANSPORT_BC_TCP,
3106 .setup = xs_setup_bc_tcp,
d5aa6b22 3107 .netid = { "" },
f300baba
AB		 3108};
3109
282b32e1 3110/**
bc25571e 3111 * init_socket_xprt - set up xprtsock's sysctls, register with RPC client
282b32e1
CL		 3112 *
3113 */
3114int init_socket_xprt(void)
3115{
2b1bec5f 3116 if (!sunrpc_table_header)
0b4d4147 3117 sunrpc_table_header = register_sysctl_table(sunrpc_table);
fbf76683 3118
176e21ee 3119 xprt_register_transport(&xs_local_transport);
bc25571e
TT		 3120 xprt_register_transport(&xs_udp_transport);
3121 xprt_register_transport(&xs_tcp_transport);
f300baba 3122 xprt_register_transport(&xs_bc_tcp_transport);
bc25571e 3123
282b32e1
CL		 3124 return 0;
3125}
3126
3127/**
bc25571e 3128 * cleanup_socket_xprt - remove xprtsock's sysctls, unregister
282b32e1
CL		 3129 *
3130 */
3131void cleanup_socket_xprt(void)
3132{
fbf76683
CL		 3133 if (sunrpc_table_header) {
3134 unregister_sysctl_table(sunrpc_table_header);
3135 sunrpc_table_header = NULL;
3136 }
bc25571e 3137
176e21ee 3138 xprt_unregister_transport(&xs_local_transport);
bc25571e
TT		 3139 xprt_unregister_transport(&xs_udp_transport);
3140 xprt_unregister_transport(&xs_tcp_transport);
f300baba 3141 xprt_unregister_transport(&xs_bc_tcp_transport);
282b32e1 3142}
cbf11071 3143
9bbb9e5a
RR		 3144static int param_set_uint_minmax(const char *val,
3145 const struct kernel_param *kp,
cbf11071
TM		 3146 unsigned int min, unsigned int max)
3147{
00cfaa94 3148 unsigned int num;
cbf11071
TM		 3149 int ret;
3150
3151 if (!val)
3152 return -EINVAL;
00cfaa94 3153 ret = kstrtouint(val, 0, &num);
9761a246
DC		 3154 if (ret)
3155 return ret;
3156 if (num < min || num > max)
cbf11071
TM		 3157 return -EINVAL;
3158 *((unsigned int *)kp->arg) = num;
3159 return 0;
3160}
3161
9bbb9e5a 3162static int param_set_portnr(const char *val, const struct kernel_param *kp)
cbf11071 3163{
ffb6ca33 3164 return param_set_uint_minmax(val, kp,
826799e6 3165 RPC_MIN_RESVPORT,
cbf11071
TM		 3166 RPC_MAX_RESVPORT);
3167}
3168
9c27847d 3169static const struct kernel_param_ops param_ops_portnr = {
9bbb9e5a
RR		 3170 .set = param_set_portnr,
3171 .get = param_get_uint,
3172};
3173
cbf11071
TM		 3174#define param_check_portnr(name, p) \
3175 __param_check(name, p, unsigned int);
3176
3177module_param_named(min_resvport, xprt_min_resvport, portnr, 0644);
3178module_param_named(max_resvport, xprt_max_resvport, portnr, 0644);
3179
9bbb9e5a
RR		 3180static int param_set_slot_table_size(const char *val,
3181 const struct kernel_param *kp)
cbf11071
TM		 3182{
3183 return param_set_uint_minmax(val, kp,
3184 RPC_MIN_SLOT_TABLE,
3185 RPC_MAX_SLOT_TABLE);
3186}
3187
9c27847d 3188static const struct kernel_param_ops param_ops_slot_table_size = {
9bbb9e5a
RR		 3189 .set = param_set_slot_table_size,
3190 .get = param_get_uint,
3191};
3192
cbf11071
TM		 3193#define param_check_slot_table_size(name, p) \
3194 __param_check(name, p, unsigned int);
3195
d9ba131d
TM		 3196static int param_set_max_slot_table_size(const char *val,
3197 const struct kernel_param *kp)
3198{
3199 return param_set_uint_minmax(val, kp,
3200 RPC_MIN_SLOT_TABLE,
3201 RPC_MAX_SLOT_TABLE_LIMIT);
3202}
3203
9c27847d 3204static const struct kernel_param_ops param_ops_max_slot_table_size = {
d9ba131d
TM		 3205 .set = param_set_max_slot_table_size,
3206 .get = param_get_uint,
3207};
3208
3209#define param_check_max_slot_table_size(name, p) \
3210 __param_check(name, p, unsigned int);
3211
cbf11071
TM		 3212module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries,
3213 slot_table_size, 0644);
d9ba131d
TM		 3214module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries,
3215 max_slot_table_size, 0644);
cbf11071
TM		 3216module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries,
3217 slot_table_size, 0644);
Linux 6.x block layer and io_uring tree(s)