projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
SUNRPC: Simplify TCP receive code by switching to using iterators
[linux-block.git] / net / sunrpc / xprtsock.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
a246b010
CL		 2/*
3 * linux/net/sunrpc/xprtsock.c
4 *
5 * Client-side transport implementation for sockets.
6 *
113aa838
AC		 7 * TCP callback races fixes (C) 1998 Red Hat
8 * TCP send fixes (C) 1998 Red Hat
a246b010
CL		 9 * TCP NFS related read + write fixes
10 * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
11 *
12 * Rewrite of larges part of the code in order to stabilize TCP stuff.
13 * Fix behaviour when socket buffer is full.
14 * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no>
55aa4f58
CL		 15 *
16 * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com>
8f9d5b1a
CL		 17 *
18 * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005.
19 * <gilles.quillard@bull.net>
a246b010
CL		 20 */
21
22#include <linux/types.h>
176e21ee 23#include <linux/string.h>
a246b010 24#include <linux/slab.h>
bc25571e 25#include <linux/module.h>
a246b010 26#include <linux/capability.h>
a246b010
CL		 27#include <linux/pagemap.h>
28#include <linux/errno.h>
29#include <linux/socket.h>
30#include <linux/in.h>
31#include <linux/net.h>
32#include <linux/mm.h>
176e21ee 33#include <linux/un.h>
a246b010
CL		 34#include <linux/udp.h>
35#include <linux/tcp.h>
36#include <linux/sunrpc/clnt.h>
5976687a 37#include <linux/sunrpc/addr.h>
02107148 38#include <linux/sunrpc/sched.h>
4cfc7e60 39#include <linux/sunrpc/svcsock.h>
49c36fcc 40#include <linux/sunrpc/xprtsock.h>
a246b010 41#include <linux/file.h>
9e00abc3 42#ifdef CONFIG_SUNRPC_BACKCHANNEL
44b98efd
RL		 43#include <linux/sunrpc/bc_xprt.h>
44#endif
a246b010
CL		 45
46#include <net/sock.h>
47#include <net/checksum.h>
48#include <net/udp.h>
49#include <net/tcp.h>
277e4ab7
TM		 50#include <linux/bvec.h>
51#include <linux/uio.h>
a246b010 52
40b5ea0c
TM		 53#include <trace/events/sunrpc.h>
54
4cfc7e60 55#include "sunrpc.h"
176e21ee
CL		 56
57static void xs_close(struct rpc_xprt *xprt);
7196dbb0
TM		 58static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
59 struct socket *sock);
176e21ee 60
c556b754
CL		 61/*
62 * xprtsock tunables
63 */
09acfea5
TM		 64static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE;
65static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE;
66static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE;
c556b754 67
09acfea5
TM		 68static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT;
69static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT;
c556b754 70
54c09874
TM		 71#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
72
7d1e8255 73#define XS_TCP_LINGER_TO (15U * HZ)
25fe6142 74static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;
7d1e8255 75
fbf76683
CL		 76/*
77 * We can register our own files under /proc/sys/sunrpc by
78 * calling register_sysctl_table() again. The files in that
79 * directory become the union of all files registered there.
80 *
81 * We simply need to make sure that we don't collide with
82 * someone else's file names!
83 */
84
fbf76683
CL		 85static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE;
86static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE;
d9ba131d 87static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT;
fbf76683
CL		 88static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT;
89static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT;
90
91static struct ctl_table_header *sunrpc_table_header;
92
93/*
94 * FIXME: changing the UDP slot table size should also resize the UDP
95 * socket buffers for existing UDP transports
96 */
fe2c6338 97static struct ctl_table xs_tunables_table[] = {
fbf76683 98 {
fbf76683
CL		 99 .procname = "udp_slot_table_entries",
100 .data = &xprt_udp_slot_table_entries,
101 .maxlen = sizeof(unsigned int),
102 .mode = 0644,
6d456111 103 .proc_handler = proc_dointvec_minmax,
fbf76683
CL		 104 .extra1 = &min_slot_table_size,
105 .extra2 = &max_slot_table_size
106 },
107 {
fbf76683
CL		 108 .procname = "tcp_slot_table_entries",
109 .data = &xprt_tcp_slot_table_entries,
110 .maxlen = sizeof(unsigned int),
111 .mode = 0644,
6d456111 112 .proc_handler = proc_dointvec_minmax,
fbf76683
CL		 113 .extra1 = &min_slot_table_size,
114 .extra2 = &max_slot_table_size
115 },
d9ba131d
TM		 116 {
117 .procname = "tcp_max_slot_table_entries",
118 .data = &xprt_max_tcp_slot_table_entries,
119 .maxlen = sizeof(unsigned int),
120 .mode = 0644,
121 .proc_handler = proc_dointvec_minmax,
122 .extra1 = &min_slot_table_size,
123 .extra2 = &max_tcp_slot_table_limit
124 },
fbf76683 125 {
fbf76683
CL		 126 .procname = "min_resvport",
127 .data = &xprt_min_resvport,
128 .maxlen = sizeof(unsigned int),
129 .mode = 0644,
6d456111 130 .proc_handler = proc_dointvec_minmax,
fbf76683 131 .extra1 = &xprt_min_resvport_limit,
e08ea3a9 132 .extra2 = &xprt_max_resvport
fbf76683
CL		 133 },
134 {
fbf76683
CL		 135 .procname = "max_resvport",
136 .data = &xprt_max_resvport,
137 .maxlen = sizeof(unsigned int),
138 .mode = 0644,
6d456111 139 .proc_handler = proc_dointvec_minmax,
e08ea3a9 140 .extra1 = &xprt_min_resvport,
fbf76683
CL		 141 .extra2 = &xprt_max_resvport_limit
142 },
25fe6142
TM		 143 {
144 .procname = "tcp_fin_timeout",
145 .data = &xs_tcp_fin_timeout,
146 .maxlen = sizeof(xs_tcp_fin_timeout),
147 .mode = 0644,
6d456111 148 .proc_handler = proc_dointvec_jiffies,
fbf76683 149 },
f8572d8f 150 { },
fbf76683
CL		 151};
152
fe2c6338 153static struct ctl_table sunrpc_table[] = {
fbf76683 154 {
fbf76683
CL		 155 .procname = "sunrpc",
156 .mode = 0555,
157 .child = xs_tunables_table
158 },
f8572d8f 159 { },
fbf76683
CL		 160};
161
162#endif
163
03bf4b70
CL		 164/*
165 * Wait duration for a reply from the RPC portmapper.
166 */
167#define XS_BIND_TO (60U * HZ)
168
169/*
170 * Delay if a UDP socket connect error occurs. This is most likely some
171 * kind of resource problem on the local host.
172 */
173#define XS_UDP_REEST_TO (2U * HZ)
174
175/*
176 * The reestablish timeout allows clients to delay for a bit before attempting
177 * to reconnect to a server that just dropped our connection.
178 *
179 * We implement an exponential backoff when trying to reestablish a TCP
180 * transport connection with the server. Some servers like to drop a TCP
181 * connection when they are overworked, so we start with a short timeout and
182 * increase over time if the server is down or not responding.
183 */
184#define XS_TCP_INIT_REEST_TO (3U * HZ)
03bf4b70
CL		 185
186/*
187 * TCP idle timeout; client drops the transport socket if it is idle
188 * for this long. Note that we also timeout UDP sockets to prevent
189 * holding port numbers when there is no RPC traffic.
190 */
191#define XS_IDLE_DISC_TO (5U * 60 * HZ)
192
f895b252 193#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
a246b010 194# undef RPC_DEBUG_DATA
9903cd1c 195# define RPCDBG_FACILITY RPCDBG_TRANS
a246b010
CL		 196#endif
197
a246b010 198#ifdef RPC_DEBUG_DATA
9903cd1c 199static void xs_pktdump(char *msg, u32 *packet, unsigned int count)
a246b010 200{
9903cd1c
CL		 201 u8 *buf = (u8 *) packet;
202 int j;
a246b010 203
46121cf7 204 dprintk("RPC: %s\n", msg);
a246b010
CL		 205 for (j = 0; j < count && j < 128; j += 4) {
206 if (!(j & 31)) {
207 if (j)
208 dprintk("\n");
209 dprintk("0x%04x ", j);
210 }
211 dprintk("%02x%02x%02x%02x ",
212 buf[j], buf[j+1], buf[j+2], buf[j+3]);
213 }
214 dprintk("\n");
215}
216#else
9903cd1c 217static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count)
a246b010
CL		 218{
219 /* NOP */
220}
221#endif
222
2118071d
TM		 223static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)
224{
225 return (struct rpc_xprt *) sk->sk_user_data;
226}
227
95392c59
CL		 228static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt)
229{
230 return (struct sockaddr *) &xprt->addr;
231}
232
176e21ee
CL		 233static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt)
234{
235 return (struct sockaddr_un *) &xprt->addr;
236}
237
95392c59 238static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt)
edb267a6 239{
95392c59
CL		 240 return (struct sockaddr_in *) &xprt->addr;
241}
242
243static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt)
244{
245 return (struct sockaddr_in6 *) &xprt->addr;
246}
247
c877b849 248static void xs_format_common_peer_addresses(struct rpc_xprt *xprt)
edb267a6 249{
c877b849 250 struct sockaddr *sap = xs_addr(xprt);
9dc3b095
CL		 251 struct sockaddr_in6 *sin6;
252 struct sockaddr_in *sin;
176e21ee 253 struct sockaddr_un *sun;
c877b849 254 char buf[128];
edb267a6 255
9dc3b095 256 switch (sap->sa_family) {
176e21ee
CL		 257 case AF_LOCAL:
258 sun = xs_addr_un(xprt);
259 strlcpy(buf, sun->sun_path, sizeof(buf));
260 xprt->address_strings[RPC_DISPLAY_ADDR] =
261 kstrdup(buf, GFP_KERNEL);
262 break;
9dc3b095 263 case AF_INET:
176e21ee
CL		 264 (void)rpc_ntop(sap, buf, sizeof(buf));
265 xprt->address_strings[RPC_DISPLAY_ADDR] =
266 kstrdup(buf, GFP_KERNEL);
9dc3b095 267 sin = xs_addr_in(xprt);
fc0b5791 268 snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));
9dc3b095
CL		 269 break;
270 case AF_INET6:
176e21ee
CL		 271 (void)rpc_ntop(sap, buf, sizeof(buf));
272 xprt->address_strings[RPC_DISPLAY_ADDR] =
273 kstrdup(buf, GFP_KERNEL);
9dc3b095 274 sin6 = xs_addr_in6(xprt);
fc0b5791 275 snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);
9dc3b095
CL		 276 break;
277 default:
278 BUG();
756805e7 279 }
176e21ee 280
9dc3b095 281 xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);
edb267a6
CL		 282}
283
9dc3b095 284static void xs_format_common_peer_ports(struct rpc_xprt *xprt)
4b6473fb 285{
9dc3b095
CL		 286 struct sockaddr *sap = xs_addr(xprt);
287 char buf[128];
4b6473fb 288
81160e66 289 snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap));
c877b849 290 xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);
4b6473fb 291
81160e66 292 snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap));
c877b849
CL		 293 xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
294}
4b6473fb 295
9dc3b095
CL		 296static void xs_format_peer_addresses(struct rpc_xprt *xprt,
297 const char *protocol,
298 const char *netid)
c877b849 299{
b454ae90 300 xprt->address_strings[RPC_DISPLAY_PROTO] = protocol;
b454ae90 301 xprt->address_strings[RPC_DISPLAY_NETID] = netid;
c877b849 302 xs_format_common_peer_addresses(xprt);
9dc3b095 303 xs_format_common_peer_ports(xprt);
edb267a6 304}
4b6473fb 305
9dc3b095 306static void xs_update_peer_port(struct rpc_xprt *xprt)
4b6473fb 307{
9dc3b095
CL		 308 kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
309 kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
4417c8c4 310
9dc3b095 311 xs_format_common_peer_ports(xprt);
edb267a6
CL		 312}
313
314static void xs_free_peer_addresses(struct rpc_xprt *xprt)
315{
33e01dc7
CL		 316 unsigned int i;
317
318 for (i = 0; i < RPC_DISPLAY_MAX; i++)
319 switch (i) {
320 case RPC_DISPLAY_PROTO:
321 case RPC_DISPLAY_NETID:
322 continue;
323 default:
324 kfree(xprt->address_strings[i]);
325 }
edb267a6
CL		 326}
327
277e4ab7
TM		 328static size_t
329xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp)
330{
331 size_t i,n;
332
333 if (!(buf->flags & XDRBUF_SPARSE_PAGES))
334 return want;
335 if (want > buf->page_len)
336 want = buf->page_len;
337 n = (buf->page_base + want + PAGE_SIZE - 1) >> PAGE_SHIFT;
338 for (i = 0; i < n; i++) {
339 if (buf->pages[i])
340 continue;
341 buf->bvec[i].bv_page = buf->pages[i] = alloc_page(gfp);
342 if (!buf->pages[i]) {
343 buf->page_len = (i * PAGE_SIZE) - buf->page_base;
344 return buf->page_len;
345 }
346 }
347 return want;
348}
349
350static ssize_t
351xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek)
352{
353 ssize_t ret;
354 if (seek != 0)
355 iov_iter_advance(&msg->msg_iter, seek);
356 ret = sock_recvmsg(sock, msg, flags);
357 return ret > 0 ? ret + seek : ret;
358}
359
360static ssize_t
361xs_read_kvec(struct socket *sock, struct msghdr *msg, int flags,
362 struct kvec *kvec, size_t count, size_t seek)
363{
364 iov_iter_kvec(&msg->msg_iter, READ | ITER_KVEC, kvec, 1, count);
365 return xs_sock_recvmsg(sock, msg, flags, seek);
366}
367
368static ssize_t
369xs_read_bvec(struct socket *sock, struct msghdr *msg, int flags,
370 struct bio_vec *bvec, unsigned long nr, size_t count,
371 size_t seek)
372{
373 iov_iter_bvec(&msg->msg_iter, READ | ITER_BVEC, bvec, nr, count);
374 return xs_sock_recvmsg(sock, msg, flags, seek);
375}
376
377static ssize_t
378xs_read_discard(struct socket *sock, struct msghdr *msg, int flags,
379 size_t count)
380{
381 struct kvec kvec = { 0 };
382 return xs_read_kvec(sock, msg, flags | MSG_TRUNC, &kvec, count, 0);
383}
384
385static ssize_t
386xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags,
387 struct xdr_buf *buf, size_t count, size_t seek, size_t *read)
388{
389 size_t want, seek_init = seek, offset = 0;
390 ssize_t ret;
391
392 if (seek < buf->head[0].iov_len) {
393 want = min_t(size_t, count, buf->head[0].iov_len);
394 ret = xs_read_kvec(sock, msg, flags, &buf->head[0], want, seek);
395 if (ret <= 0)
396 goto sock_err;
397 offset += ret;
398 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
399 goto out;
400 if (ret != want)
401 goto eagain;
402 seek = 0;
403 } else {
404 seek -= buf->head[0].iov_len;
405 offset += buf->head[0].iov_len;
406 }
407 if (seek < buf->page_len) {
408 want = xs_alloc_sparse_pages(buf,
409 min_t(size_t, count - offset, buf->page_len),
410 GFP_NOWAIT);
411 ret = xs_read_bvec(sock, msg, flags, buf->bvec,
412 xdr_buf_pagecount(buf),
413 want + buf->page_base,
414 seek + buf->page_base);
415 if (ret <= 0)
416 goto sock_err;
417 offset += ret - buf->page_base;
418 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
419 goto out;
420 if (ret != want)
421 goto eagain;
422 seek = 0;
423 } else {
424 seek -= buf->page_len;
425 offset += buf->page_len;
426 }
427 if (seek < buf->tail[0].iov_len) {
428 want = min_t(size_t, count - offset, buf->tail[0].iov_len);
429 ret = xs_read_kvec(sock, msg, flags, &buf->tail[0], want, seek);
430 if (ret <= 0)
431 goto sock_err;
432 offset += ret;
433 if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
434 goto out;
435 if (ret != want)
436 goto eagain;
437 } else
438 offset += buf->tail[0].iov_len;
439 ret = -EMSGSIZE;
440 msg->msg_flags |= MSG_TRUNC;
441out:
442 *read = offset - seek_init;
443 return ret;
444eagain:
445 ret = -EAGAIN;
446 goto out;
447sock_err:
448 offset += seek;
449 goto out;
450}
451
452static void
453xs_read_header(struct sock_xprt *transport, struct xdr_buf *buf)
454{
455 if (!transport->recv.copied) {
456 if (buf->head[0].iov_len >= transport->recv.offset)
457 memcpy(buf->head[0].iov_base,
458 &transport->recv.xid,
459 transport->recv.offset);
460 transport->recv.copied = transport->recv.offset;
461 }
462}
463
464static bool
465xs_read_stream_request_done(struct sock_xprt *transport)
466{
467 return transport->recv.fraghdr & cpu_to_be32(RPC_LAST_STREAM_FRAGMENT);
468}
469
470static ssize_t
471xs_read_stream_request(struct sock_xprt *transport, struct msghdr *msg,
472 int flags, struct rpc_rqst *req)
473{
474 struct xdr_buf *buf = &req->rq_private_buf;
475 size_t want, read;
476 ssize_t ret;
477
478 xs_read_header(transport, buf);
479
480 want = transport->recv.len - transport->recv.offset;
481 ret = xs_read_xdr_buf(transport->sock, msg, flags, buf,
482 transport->recv.copied + want, transport->recv.copied,
483 &read);
484 transport->recv.offset += read;
485 transport->recv.copied += read;
486 if (transport->recv.offset == transport->recv.len) {
487 if (xs_read_stream_request_done(transport))
488 msg->msg_flags |= MSG_EOR;
489 return transport->recv.copied;
490 }
491
492 switch (ret) {
493 case -EMSGSIZE:
494 return transport->recv.copied;
495 case 0:
496 return -ESHUTDOWN;
497 default:
498 if (ret < 0)
499 return ret;
500 }
501 return -EAGAIN;
502}
503
504static size_t
505xs_read_stream_headersize(bool isfrag)
506{
507 if (isfrag)
508 return sizeof(__be32);
509 return 3 * sizeof(__be32);
510}
511
512static ssize_t
513xs_read_stream_header(struct sock_xprt *transport, struct msghdr *msg,
514 int flags, size_t want, size_t seek)
515{
516 struct kvec kvec = {
517 .iov_base = &transport->recv.fraghdr,
518 .iov_len = want,
519 };
520 return xs_read_kvec(transport->sock, msg, flags, &kvec, want, seek);
521}
522
523#if defined(CONFIG_SUNRPC_BACKCHANNEL)
524static ssize_t
525xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
526{
527 struct rpc_xprt *xprt = &transport->xprt;
528 struct rpc_rqst *req;
529 ssize_t ret;
530
531 /* Look up and lock the request corresponding to the given XID */
532 req = xprt_lookup_bc_request(xprt, transport->recv.xid);
533 if (!req) {
534 printk(KERN_WARNING "Callback slot table overflowed\n");
535 return -ESHUTDOWN;
536 }
537
538 ret = xs_read_stream_request(transport, msg, flags, req);
539 if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
540 xprt_complete_bc_request(req, ret);
541
542 return ret;
543}
544#else /* CONFIG_SUNRPC_BACKCHANNEL */
545static ssize_t
546xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
547{
548 return -ESHUTDOWN;
549}
550#endif /* CONFIG_SUNRPC_BACKCHANNEL */
551
552static ssize_t
553xs_read_stream_reply(struct sock_xprt *transport, struct msghdr *msg, int flags)
554{
555 struct rpc_xprt *xprt = &transport->xprt;
556 struct rpc_rqst *req;
557 ssize_t ret = 0;
558
559 /* Look up and lock the request corresponding to the given XID */
560 spin_lock(&xprt->queue_lock);
561 req = xprt_lookup_rqst(xprt, transport->recv.xid);
562 if (!req) {
563 msg->msg_flags |= MSG_TRUNC;
564 goto out;
565 }
566 xprt_pin_rqst(req);
567 spin_unlock(&xprt->queue_lock);
568
569 ret = xs_read_stream_request(transport, msg, flags, req);
570
571 spin_lock(&xprt->queue_lock);
572 if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
573 xprt_complete_rqst(req->rq_task, ret);
574 xprt_unpin_rqst(req);
575out:
576 spin_unlock(&xprt->queue_lock);
577 return ret;
578}
579
580static ssize_t
581xs_read_stream(struct sock_xprt *transport, int flags)
582{
583 struct msghdr msg = { 0 };
584 size_t want, read = 0;
585 ssize_t ret = 0;
586
587 if (transport->recv.len == 0) {
588 want = xs_read_stream_headersize(transport->recv.copied != 0);
589 ret = xs_read_stream_header(transport, &msg, flags, want,
590 transport->recv.offset);
591 if (ret <= 0)
592 goto out_err;
593 transport->recv.offset = ret;
594 if (ret != want) {
595 ret = -EAGAIN;
596 goto out_err;
597 }
598 transport->recv.len = be32_to_cpu(transport->recv.fraghdr) &
599 RPC_FRAGMENT_SIZE_MASK;
600 transport->recv.offset -= sizeof(transport->recv.fraghdr);
601 read = ret;
602 }
603
604 switch (be32_to_cpu(transport->recv.calldir)) {
605 case RPC_CALL:
606 ret = xs_read_stream_call(transport, &msg, flags);
607 break;
608 case RPC_REPLY:
609 ret = xs_read_stream_reply(transport, &msg, flags);
610 }
611 if (msg.msg_flags & MSG_TRUNC) {
612 transport->recv.calldir = cpu_to_be32(-1);
613 transport->recv.copied = -1;
614 }
615 if (ret < 0)
616 goto out_err;
617 read += ret;
618 if (transport->recv.offset < transport->recv.len) {
619 ret = xs_read_discard(transport->sock, &msg, flags,
620 transport->recv.len - transport->recv.offset);
621 if (ret <= 0)
622 goto out_err;
623 transport->recv.offset += ret;
624 read += ret;
625 if (transport->recv.offset != transport->recv.len)
626 return -EAGAIN;
627 }
628 if (xs_read_stream_request_done(transport)) {
629 trace_xs_tcp_data_recv(transport);
630 transport->recv.copied = 0;
631 }
632 transport->recv.offset = 0;
633 transport->recv.len = 0;
634 return read;
635out_err:
636 switch (ret) {
637 case 0:
638 case -ESHUTDOWN:
639 xprt_force_disconnect(&transport->xprt);
640 return -ESHUTDOWN;
641 }
642 return ret;
643}
644
b4b5cc85
CL		 645#define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
646
24c5684b 647static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, unsigned int base, int more)
b4b5cc85 648{
b4b5cc85
CL		 649 struct msghdr msg = {
650 .msg_name = addr,
651 .msg_namelen = addrlen,
24c5684b
TM		 652 .msg_flags = XS_SENDMSG_FLAGS | (more ? MSG_MORE : 0),
653 };
654 struct kvec iov = {
655 .iov_base = vec->iov_base + base,
656 .iov_len = vec->iov_len - base,
b4b5cc85
CL		 657 };
658
24c5684b 659 if (iov.iov_len != 0)
b4b5cc85
CL		 660 return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len);
661 return kernel_sendmsg(sock, &msg, NULL, 0, 0);
662}
663
f279cd00 664static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, bool zerocopy, int *sent_p)
b4b5cc85 665{
a6b31d18
TM		 666 ssize_t (*do_sendpage)(struct socket *sock, struct page *page,
667 int offset, size_t size, int flags);
24c5684b
TM		 668 struct page **ppage;
669 unsigned int remainder;
f279cd00 670 int err;
24c5684b
TM		 671
672 remainder = xdr->page_len - base;
673 base += xdr->page_base;
674 ppage = xdr->pages + (base >> PAGE_SHIFT);
675 base &= ~PAGE_MASK;
a6b31d18
TM		 676 do_sendpage = sock->ops->sendpage;
677 if (!zerocopy)
678 do_sendpage = sock_no_sendpage;
24c5684b
TM		 679 for(;;) {
680 unsigned int len = min_t(unsigned int, PAGE_SIZE - base, remainder);
681 int flags = XS_SENDMSG_FLAGS;
b4b5cc85 682
24c5684b 683 remainder -= len;
31303d6c 684 if (more)
24c5684b 685 flags |= MSG_MORE;
31303d6c
TM		 686 if (remainder != 0)
687 flags |= MSG_SENDPAGE_NOTLAST | MSG_MORE;
a6b31d18 688 err = do_sendpage(sock, *ppage, base, len, flags);
24c5684b
TM		 689 if (remainder == 0 || err != len)
690 break;
f279cd00 691 *sent_p += err;
24c5684b
TM		 692 ppage++;
693 base = 0;
694 }
f279cd00
JB		 695 if (err > 0) {
696 *sent_p += err;
697 err = 0;
698 }
699 return err;
b4b5cc85
CL		 700}
701
9903cd1c
CL		 702/**
703 * xs_sendpages - write pages directly to a socket
704 * @sock: socket to send on
705 * @addr: UDP only -- address of destination
706 * @addrlen: UDP only -- length of destination address
707 * @xdr: buffer containing this request
708 * @base: starting position in the buffer
a6b31d18 709 * @zerocopy: true if it is safe to use sendpage()
f279cd00 710 * @sent_p: return the total number of bytes successfully queued for sending
9903cd1c 711 *
a246b010 712 */
f279cd00 713static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xdr, unsigned int base, bool zerocopy, int *sent_p)
a246b010 714{
24c5684b 715 unsigned int remainder = xdr->len - base;
f279cd00
JB		 716 int err = 0;
717 int sent = 0;
a246b010 718
262965f5 719 if (unlikely(!sock))
fba91afb 720 return -ENOTSOCK;
262965f5 721
24c5684b
TM		 722 if (base != 0) {
723 addr = NULL;
724 addrlen = 0;
725 }
262965f5 726
24c5684b
TM		 727 if (base < xdr->head[0].iov_len || addr != NULL) {
728 unsigned int len = xdr->head[0].iov_len - base;
729 remainder -= len;
730 err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0);
731 if (remainder == 0 || err != len)
a246b010 732 goto out;
f279cd00 733 *sent_p += err;
a246b010
CL		 734 base = 0;
735 } else
24c5684b 736 base -= xdr->head[0].iov_len;
a246b010 737
24c5684b
TM		 738 if (base < xdr->page_len) {
739 unsigned int len = xdr->page_len - base;
740 remainder -= len;
f279cd00
JB		 741 err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent);
742 *sent_p += sent;
743 if (remainder == 0 || sent != len)
a246b010
CL		 744 goto out;
745 base = 0;
24c5684b
TM		 746 } else
747 base -= xdr->page_len;
748
749 if (base >= xdr->tail[0].iov_len)
f279cd00 750 return 0;
24c5684b 751 err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0);
a246b010 752out:
f279cd00
JB		 753 if (err > 0) {
754 *sent_p += err;
755 err = 0;
756 }
757 return err;
a246b010
CL		 758}
759
9903cd1c 760/**
c544577d 761 * xs_nospace - handle transmit was incomplete
50f484e2 762 * @req: pointer to RPC request
9903cd1c 763 *
a246b010 764 */
c544577d 765static int xs_nospace(struct rpc_rqst *req)
a246b010 766{
262965f5 767 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 768 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
06ea0bfe 769 struct sock *sk = transport->inet;
24ca9a84 770 int ret = -EAGAIN;
a246b010 771
46121cf7 772 dprintk("RPC: %5u xmit incomplete (%u left of %u)\n",
c544577d
TM		 773 req->rq_task->tk_pid,
774 req->rq_slen - transport->xmit.offset,
262965f5
CL		 775 req->rq_slen);
776
b6ddf64f
TM		 777 /* Protect against races with write_space */
778 spin_lock_bh(&xprt->transport_lock);
779
780 /* Don't race with disconnect */
781 if (xprt_connected(xprt)) {
13331a55
TM		 782 /* wait for more buffer space */
783 sk->sk_write_pending++;
c544577d 784 xprt_wait_for_buffer_space(xprt);
13331a55 785 } else
5e3771ce 786 ret = -ENOTCONN;
262965f5 787
b6ddf64f 788 spin_unlock_bh(&xprt->transport_lock);
06ea0bfe
TM		 789
790 /* Race breaker in case memory is freed before above code is called */
d48f9ce7
DV		 791 if (ret == -EAGAIN) {
792 struct socket_wq *wq;
793
794 rcu_read_lock();
795 wq = rcu_dereference(sk->sk_wq);
796 set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags);
797 rcu_read_unlock();
798
799 sk->sk_write_space(sk);
800 }
5e3771ce 801 return ret;
262965f5
CL		 802}
803
277e4ab7
TM		 804static void
805xs_stream_prepare_request(struct rpc_rqst *req)
806{
807 req->rq_task->tk_status = xdr_alloc_bvec(&req->rq_rcv_buf, GFP_NOIO);
808}
809
4cd34e7c
TM		 810/*
811 * Determine if the previous message in the stream was aborted before it
812 * could complete transmission.
813 */
814static bool
815xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req)
816{
817 return transport->xmit.offset != 0 && req->rq_bytes_sent == 0;
818}
819
61677eee
CL		 820/*
821 * Construct a stream transport record marker in @buf.
822 */
823static inline void xs_encode_stream_record_marker(struct xdr_buf *buf)
824{
825 u32 reclen = buf->len - sizeof(rpc_fraghdr);
826 rpc_fraghdr *base = buf->head[0].iov_base;
827 *base = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | reclen);
828}
829
176e21ee
CL		 830/**
831 * xs_local_send_request - write an RPC request to an AF_LOCAL socket
50f484e2 832 * @req: pointer to RPC request
176e21ee
CL		 833 *
834 * Return values:
835 * 0: The request has been sent
836 * EAGAIN: The socket was blocked, please call again later to
837 * complete the request
838 * ENOTCONN: Caller needs to invoke connect logic then call again
839 * other: Some other error occured, the request was not sent
840 */
adfa7144 841static int xs_local_send_request(struct rpc_rqst *req)
176e21ee 842{
176e21ee
CL		 843 struct rpc_xprt *xprt = req->rq_xprt;
844 struct sock_xprt *transport =
845 container_of(xprt, struct sock_xprt, xprt);
846 struct xdr_buf *xdr = &req->rq_snd_buf;
847 int status;
f279cd00 848 int sent = 0;
176e21ee 849
4cd34e7c
TM		 850 /* Close the stream if the previous transmission was incomplete */
851 if (xs_send_request_was_aborted(transport, req)) {
852 xs_close(xprt);
853 return -ENOTCONN;
854 }
855
176e21ee
CL		 856 xs_encode_stream_record_marker(&req->rq_snd_buf);
857
858 xs_pktdump("packet data:",
859 req->rq_svec->iov_base, req->rq_svec->iov_len);
860
78215759 861 req->rq_xtime = ktime_get();
6c7a64e5
TM		 862 status = xs_sendpages(transport->sock, NULL, 0, xdr,
863 transport->xmit.offset,
f279cd00 864 true, &sent);
176e21ee 865 dprintk("RPC: %s(%u) = %d\n",
6c7a64e5 866 __func__, xdr->len - transport->xmit.offset, status);
743c69e7
N		 867
868 if (status == -EAGAIN && sock_writeable(transport->inet))
869 status = -ENOBUFS;
870
f279cd00 871 if (likely(sent > 0) || status == 0) {
6c7a64e5
TM		 872 transport->xmit.offset += sent;
873 req->rq_bytes_sent = transport->xmit.offset;
176e21ee 874 if (likely(req->rq_bytes_sent >= req->rq_slen)) {
6c7a64e5 875 req->rq_xmit_bytes_sent += transport->xmit.offset;
176e21ee 876 req->rq_bytes_sent = 0;
6c7a64e5 877 transport->xmit.offset = 0;
176e21ee
CL		 878 return 0;
879 }
880 status = -EAGAIN;
881 }
882
883 switch (status) {
3601c4a9 884 case -ENOBUFS:
b5872f0c 885 break;
176e21ee 886 case -EAGAIN:
c544577d 887 status = xs_nospace(req);
176e21ee
CL		 888 break;
889 default:
890 dprintk("RPC: sendmsg returned unrecognized error %d\n",
891 -status);
e9d47639 892 /* fall through */
176e21ee
CL		 893 case -EPIPE:
894 xs_close(xprt);
895 status = -ENOTCONN;
896 }
897
898 return status;
899}
900
262965f5
CL		 901/**
902 * xs_udp_send_request - write an RPC request to a UDP socket
50f484e2 903 * @req: pointer to RPC request
262965f5
CL		 904 *
905 * Return values:
906 * 0: The request has been sent
907 * EAGAIN: The socket was blocked, please call again later to
908 * complete the request
909 * ENOTCONN: Caller needs to invoke connect logic then call again
25985edc 910 * other: Some other error occurred, the request was not sent
262965f5 911 */
adfa7144 912static int xs_udp_send_request(struct rpc_rqst *req)
262965f5 913{
262965f5 914 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 915 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262965f5 916 struct xdr_buf *xdr = &req->rq_snd_buf;
f279cd00 917 int sent = 0;
262965f5 918 int status;
a246b010 919
9903cd1c 920 xs_pktdump("packet data:",
a246b010
CL		 921 req->rq_svec->iov_base,
922 req->rq_svec->iov_len);
923
01d37c42
TM		 924 if (!xprt_bound(xprt))
925 return -ENOTCONN;
75891f50
TM		 926
927 if (!xprt_request_get_cong(xprt, req))
928 return -EBADSLT;
929
78215759 930 req->rq_xtime = ktime_get();
f279cd00 931 status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen,
6c7a64e5 932 xdr, 0, true, &sent);
a246b010 933
46121cf7 934 dprintk("RPC: xs_udp_send_request(%u) = %d\n",
6c7a64e5 935 xdr->len, status);
a246b010 936
3dedbb5c
JB		 937 /* firewall is blocking us, don't return -EAGAIN or we end up looping */
938 if (status == -EPERM)
939 goto process_status;
940
743c69e7
N		 941 if (status == -EAGAIN && sock_writeable(transport->inet))
942 status = -ENOBUFS;
943
f279cd00
JB		 944 if (sent > 0 || status == 0) {
945 req->rq_xmit_bytes_sent += sent;
946 if (sent >= req->rq_slen)
2199700f
TM		 947 return 0;
948 /* Still some bytes left; set up for a retry later. */
262965f5 949 status = -EAGAIN;
2199700f 950 }
a246b010 951
3dedbb5c 952process_status:
262965f5 953 switch (status) {
fba91afb
TM		 954 case -ENOTSOCK:
955 status = -ENOTCONN;
956 /* Should we call xs_close() here? */
957 break;
b6ddf64f 958 case -EAGAIN:
c544577d 959 status = xs_nospace(req);
b6ddf64f 960 break;
262965f5 961 case -ENETUNREACH:
3601c4a9 962 case -ENOBUFS:
262965f5 963 case -EPIPE:
a246b010 964 case -ECONNREFUSED:
3dedbb5c 965 case -EPERM:
a246b010 966 /* When the server has died, an ICMP port unreachable message
9903cd1c 967 * prompts ECONNREFUSED. */
13331a55
TM		 968 break;
969 default:
970 dprintk("RPC: sendmsg returned unrecognized error %d\n",
971 -status);
a246b010 972 }
5fe46e9d 973
262965f5 974 return status;
a246b010
CL		 975}
976
9903cd1c 977/**
262965f5 978 * xs_tcp_send_request - write an RPC request to a TCP socket
50f484e2 979 * @req: pointer to RPC request
9903cd1c
CL		 980 *
981 * Return values:
262965f5
CL		 982 * 0: The request has been sent
983 * EAGAIN: The socket was blocked, please call again later to
984 * complete the request
985 * ENOTCONN: Caller needs to invoke connect logic then call again
25985edc 986 * other: Some other error occurred, the request was not sent
9903cd1c
CL		 987 *
988 * XXX: In the case of soft timeouts, should we eventually give up
262965f5 989 * if sendmsg is not able to make progress?
9903cd1c 990 */
adfa7144 991static int xs_tcp_send_request(struct rpc_rqst *req)
a246b010 992{
a246b010 993 struct rpc_xprt *xprt = req->rq_xprt;
ee0ac0c2 994 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262965f5 995 struct xdr_buf *xdr = &req->rq_snd_buf;
a6b31d18 996 bool zerocopy = true;
9ffadfbc 997 bool vm_wait = false;
b595bb15 998 int status;
f279cd00 999 int sent;
a246b010 1000
4cd34e7c
TM		 1001 /* Close the stream if the previous transmission was incomplete */
1002 if (xs_send_request_was_aborted(transport, req)) {
1003 if (transport->sock != NULL)
1004 kernel_sock_shutdown(transport->sock, SHUT_RDWR);
1005 return -ENOTCONN;
1006 }
1007
61677eee 1008 xs_encode_stream_record_marker(&req->rq_snd_buf);
a246b010 1009
262965f5
CL		 1010 xs_pktdump("packet data:",
1011 req->rq_svec->iov_base,
1012 req->rq_svec->iov_len);
a6b31d18
TM		 1013 /* Don't use zero copy if this is a resend. If the RPC call
1014 * completes while the socket holds a reference to the pages,
1015 * then we may end up resending corrupted data.
1016 */
50f484e2 1017 if (req->rq_task->tk_flags & RPC_TASK_SENT)
a6b31d18 1018 zerocopy = false;
a246b010 1019
7196dbb0
TM		 1020 if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state))
1021 xs_tcp_set_socket_timeouts(xprt, transport->sock);
1022
a246b010
CL		 1023 /* Continue transmitting the packet/record. We must be careful
1024 * to cope with writespace callbacks arriving _after_ we have
262965f5 1025 * called sendmsg(). */
78215759 1026 req->rq_xtime = ktime_get();
a246b010 1027 while (1) {
f279cd00
JB		 1028 sent = 0;
1029 status = xs_sendpages(transport->sock, NULL, 0, xdr,
6c7a64e5
TM		 1030 transport->xmit.offset,
1031 zerocopy, &sent);
a246b010 1032
46121cf7 1033 dprintk("RPC: xs_tcp_send_request(%u) = %d\n",
6c7a64e5 1034 xdr->len - transport->xmit.offset, status);
a246b010 1035
262965f5
CL		 1036 /* If we've sent the entire packet, immediately
1037 * reset the count of bytes sent. */
6c7a64e5
TM		 1038 transport->xmit.offset += sent;
1039 req->rq_bytes_sent = transport->xmit.offset;
262965f5 1040 if (likely(req->rq_bytes_sent >= req->rq_slen)) {
6c7a64e5 1041 req->rq_xmit_bytes_sent += transport->xmit.offset;
262965f5 1042 req->rq_bytes_sent = 0;
6c7a64e5 1043 transport->xmit.offset = 0;
262965f5
CL		 1044 return 0;
1045 }
a246b010 1046
9ffadfbc
TM		 1047 WARN_ON_ONCE(sent == 0 && status == 0);
1048
1049 if (status == -EAGAIN ) {
1050 /*
1051 * Return EAGAIN if we're sure we're hitting the
1052 * socket send buffer limits.
1053 */
1054 if (test_bit(SOCK_NOSPACE, &transport->sock->flags))
1055 break;
1056 /*
1057 * Did we hit a memory allocation failure?
1058 */
1059 if (sent == 0) {
1060 status = -ENOBUFS;
1061 if (vm_wait)
1062 break;
1063 /* Retry, knowing now that we're below the
1064 * socket send buffer limit
1065 */
1066 vm_wait = true;
1067 }
1068 continue;
1069 }
f580dd04
TM		 1070 if (status < 0)
1071 break;
9ffadfbc 1072 vm_wait = false;
a246b010
CL		 1073 }
1074
262965f5 1075 switch (status) {
fba91afb
TM		 1076 case -ENOTSOCK:
1077 status = -ENOTCONN;
1078 /* Should we call xs_close() here? */
1079 break;
262965f5 1080 case -EAGAIN:
c544577d 1081 status = xs_nospace(req);
262965f5 1082 break;
262965f5 1083 case -ECONNRESET:
2a9e1cfa 1084 case -ECONNREFUSED:
262965f5 1085 case -ENOTCONN:
3913c78c 1086 case -EADDRINUSE:
b5872f0c 1087 case -ENOBUFS:
b9d2bb2e 1088 case -EPIPE:
13331a55
TM		 1089 break;
1090 default:
1091 dprintk("RPC: sendmsg returned unrecognized error %d\n",
1092 -status);
a246b010 1093 }
5fe46e9d 1094
a246b010
CL		 1095 return status;
1096}
1097
2a9e1cfa
TM		 1098static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)
1099{
1100 transport->old_data_ready = sk->sk_data_ready;
1101 transport->old_state_change = sk->sk_state_change;
1102 transport->old_write_space = sk->sk_write_space;
2118071d 1103 transport->old_error_report = sk->sk_error_report;
2a9e1cfa
TM		 1104}
1105
1106static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)
1107{
1108 sk->sk_data_ready = transport->old_data_ready;
1109 sk->sk_state_change = transport->old_state_change;
1110 sk->sk_write_space = transport->old_write_space;
2118071d
TM		 1111 sk->sk_error_report = transport->old_error_report;
1112}
1113
42d42a5b
TM		 1114static void xs_sock_reset_state_flags(struct rpc_xprt *xprt)
1115{
1116 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1117
1118 clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
1119}
1120
b70ae915
TM		 1121static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt)
1122{
1123 smp_mb__before_atomic();
1124 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
1125 clear_bit(XPRT_CLOSING, &xprt->state);
42d42a5b 1126 xs_sock_reset_state_flags(xprt);
b70ae915
TM		 1127 smp_mb__after_atomic();
1128}
1129
2118071d
TM		 1130/**
1131 * xs_error_report - callback to handle TCP socket state errors
1132 * @sk: socket
1133 *
1134 * Note: we don't call sock_error() since there may be a rpc_task
1135 * using the socket, and so we don't want to clear sk->sk_err.
1136 */
1137static void xs_error_report(struct sock *sk)
1138{
1139 struct rpc_xprt *xprt;
1140 int err;
1141
1142 read_lock_bh(&sk->sk_callback_lock);
1143 if (!(xprt = xprt_from_sock(sk)))
1144 goto out;
1145
1146 err = -sk->sk_err;
1147 if (err == 0)
1148 goto out;
1149 dprintk("RPC: xs_error_report client %p, error=%d...\n",
1150 xprt, -err);
e8353c76 1151 trace_rpc_socket_error(xprt, sk->sk_socket, err);
2118071d
TM		 1152 xprt_wake_pending_tasks(xprt, err);
1153 out:
1154 read_unlock_bh(&sk->sk_callback_lock);
2a9e1cfa
TM		 1155}
1156
fe315e76 1157static void xs_reset_transport(struct sock_xprt *transport)
a246b010 1158{
ee0ac0c2
CL		 1159 struct socket *sock = transport->sock;
1160 struct sock *sk = transport->inet;
6cc7e908 1161 struct rpc_xprt *xprt = &transport->xprt;
a246b010 1162
fe315e76
CL		 1163 if (sk == NULL)
1164 return;
9903cd1c 1165
264d1df3
JL		 1166 if (atomic_read(&transport->xprt.swapper))
1167 sk_clear_memalloc(sk);
1168
09939204
TM		 1169 kernel_sock_shutdown(sock, SHUT_RDWR);
1170
edc1b01c 1171 mutex_lock(&transport->recv_mutex);
a246b010 1172 write_lock_bh(&sk->sk_callback_lock);
ee0ac0c2
CL		 1173 transport->inet = NULL;
1174 transport->sock = NULL;
a246b010 1175
9903cd1c 1176 sk->sk_user_data = NULL;
2a9e1cfa
TM		 1177
1178 xs_restore_old_callbacks(transport, sk);
0c78789e 1179 xprt_clear_connected(xprt);
a246b010 1180 write_unlock_bh(&sk->sk_callback_lock);
6cc7e908 1181 xs_sock_reset_connection_flags(xprt);
edc1b01c 1182 mutex_unlock(&transport->recv_mutex);
a246b010 1183
6cc7e908 1184 trace_rpc_socket_close(xprt, sock);
a246b010 1185 sock_release(sock);
fe315e76
CL		 1186}
1187
1188/**
1189 * xs_close - close a socket
1190 * @xprt: transport
1191 *
1192 * This is used when all requests are complete; ie, no DRC state remains
1193 * on the server we want to save.
f75e6745
TM		 1194 *
1195 * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with
1196 * xs_reset_transport() zeroing the socket from underneath a writer.
fe315e76
CL		 1197 */
1198static void xs_close(struct rpc_xprt *xprt)
1199{
1200 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1201
1202 dprintk("RPC: xs_close xprt %p\n", xprt);
1203
1204 xs_reset_transport(transport);
61d0a8e6 1205 xprt->reestablish_timeout = 0;
fe315e76 1206
62da3b24 1207 xprt_disconnect_done(xprt);
a246b010
CL		 1208}
1209
4a068258
CL		 1210static void xs_inject_disconnect(struct rpc_xprt *xprt)
1211{
1212 dprintk("RPC: injecting transport disconnect on xprt=%p\n",
1213 xprt);
1214 xprt_disconnect_done(xprt);
1215}
1216
315f3812
KM		 1217static void xs_xprt_free(struct rpc_xprt *xprt)
1218{
1219 xs_free_peer_addresses(xprt);
1220 xprt_free(xprt);
1221}
1222
9903cd1c
CL		 1223/**
1224 * xs_destroy - prepare to shutdown a transport
1225 * @xprt: doomed transport
1226 *
1227 */
1228static void xs_destroy(struct rpc_xprt *xprt)
a246b010 1229{
03c78827
TM		 1230 struct sock_xprt *transport = container_of(xprt,
1231 struct sock_xprt, xprt);
46121cf7 1232 dprintk("RPC: xs_destroy xprt %p\n", xprt);
9903cd1c 1233
03c78827 1234 cancel_delayed_work_sync(&transport->connect_worker);
a1311d87 1235 xs_close(xprt);
edc1b01c 1236 cancel_work_sync(&transport->recv_worker);
315f3812 1237 xs_xprt_free(xprt);
a1311d87 1238 module_put(THIS_MODULE);
a246b010
CL		 1239}
1240
176e21ee
CL		 1241static int xs_local_copy_to_xdr(struct xdr_buf *xdr, struct sk_buff *skb)
1242{
1243 struct xdr_skb_reader desc = {
1244 .skb = skb,
1245 .offset = sizeof(rpc_fraghdr),
1246 .count = skb->len - sizeof(rpc_fraghdr),
1247 };
1248
1249 if (xdr_partial_copy_from_skb(xdr, 0, &desc, xdr_skb_read_bits) < 0)
1250 return -1;
1251 if (desc.count)
1252 return -1;
1253 return 0;
1254}
1255
1256/**
a2648094
TM		 1257 * xs_local_data_read_skb
1258 * @xprt: transport
1259 * @sk: socket
1260 * @skb: skbuff
176e21ee
CL		 1261 *
1262 * Currently this assumes we can read the whole reply in a single gulp.
1263 */
a2648094
TM		 1264static void xs_local_data_read_skb(struct rpc_xprt *xprt,
1265 struct sock *sk,
1266 struct sk_buff *skb)
176e21ee
CL		 1267{
1268 struct rpc_task *task;
176e21ee 1269 struct rpc_rqst *rovr;
a2648094 1270 int repsize, copied;
176e21ee
CL		 1271 u32 _xid;
1272 __be32 *xp;
1273
176e21ee
CL		 1274 repsize = skb->len - sizeof(rpc_fraghdr);
1275 if (repsize < 4) {
1276 dprintk("RPC: impossible RPC reply size %d\n", repsize);
a2648094 1277 return;
176e21ee
CL		 1278 }
1279
1280 /* Copy the XID from the skb... */
1281 xp = skb_header_pointer(skb, sizeof(rpc_fraghdr), sizeof(_xid), &_xid);
1282 if (xp == NULL)
a2648094 1283 return;
176e21ee
CL		 1284
1285 /* Look up and lock the request corresponding to the given XID */
75c84151 1286 spin_lock(&xprt->queue_lock);
176e21ee
CL		 1287 rovr = xprt_lookup_rqst(xprt, *xp);
1288 if (!rovr)
1289 goto out_unlock;
729749bb 1290 xprt_pin_rqst(rovr);
75c84151 1291 spin_unlock(&xprt->queue_lock);
176e21ee
CL		 1292 task = rovr->rq_task;
1293
1294 copied = rovr->rq_private_buf.buflen;
1295 if (copied > repsize)
1296 copied = repsize;
1297
1298 if (xs_local_copy_to_xdr(&rovr->rq_private_buf, skb)) {
1299 dprintk("RPC: sk_buff copy failed\n");
75c84151 1300 spin_lock(&xprt->queue_lock);
729749bb 1301 goto out_unpin;
176e21ee
CL		 1302 }
1303
75c84151 1304 spin_lock(&xprt->queue_lock);
176e21ee 1305 xprt_complete_rqst(task, copied);
729749bb
TM		 1306out_unpin:
1307 xprt_unpin_rqst(rovr);
176e21ee 1308 out_unlock:
75c84151 1309 spin_unlock(&xprt->queue_lock);
a2648094
TM		 1310}
1311
1312static void xs_local_data_receive(struct sock_xprt *transport)
1313{
1314 struct sk_buff *skb;
1315 struct sock *sk;
1316 int err;
1317
0af3442a 1318restart:
a2648094
TM		 1319 mutex_lock(&transport->recv_mutex);
1320 sk = transport->inet;
1321 if (sk == NULL)
1322 goto out;
1323 for (;;) {
1324 skb = skb_recv_datagram(sk, 0, 1, &err);
42d42a5b
TM		 1325 if (skb != NULL) {
1326 xs_local_data_read_skb(&transport->xprt, sk, skb);
1327 skb_free_datagram(sk, skb);
1328 continue;
1329 }
1330 if (!test_and_clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
a2648094 1331 break;
0af3442a
TM		 1332 if (need_resched()) {
1333 mutex_unlock(&transport->recv_mutex);
1334 cond_resched();
1335 goto restart;
1336 }
a2648094
TM		 1337 }
1338out:
1339 mutex_unlock(&transport->recv_mutex);
1340}
1341
1342static void xs_local_data_receive_workfn(struct work_struct *work)
1343{
1344 struct sock_xprt *transport =
1345 container_of(work, struct sock_xprt, recv_worker);
1346 xs_local_data_receive(transport);
176e21ee
CL		 1347}
1348
9903cd1c 1349/**
f9b2ee71
TM		 1350 * xs_udp_data_read_skb - receive callback for UDP sockets
1351 * @xprt: transport
1352 * @sk: socket
1353 * @skb: skbuff
9903cd1c 1354 *
a246b010 1355 */
f9b2ee71
TM		 1356static void xs_udp_data_read_skb(struct rpc_xprt *xprt,
1357 struct sock *sk,
1358 struct sk_buff *skb)
a246b010 1359{
9903cd1c 1360 struct rpc_task *task;
a246b010 1361 struct rpc_rqst *rovr;
f9b2ee71 1362 int repsize, copied;
d8ed029d
AD		 1363 u32 _xid;
1364 __be32 *xp;
a246b010 1365
1da8c681 1366 repsize = skb->len;
a246b010 1367 if (repsize < 4) {
46121cf7 1368 dprintk("RPC: impossible RPC reply size %d!\n", repsize);
f9b2ee71 1369 return;
a246b010
CL		 1370 }
1371
1372 /* Copy the XID from the skb... */
1da8c681 1373 xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid);
a246b010 1374 if (xp == NULL)
f9b2ee71 1375 return;
a246b010
CL		 1376
1377 /* Look up and lock the request corresponding to the given XID */
75c84151 1378 spin_lock(&xprt->queue_lock);
a246b010
CL		 1379 rovr = xprt_lookup_rqst(xprt, *xp);
1380 if (!rovr)
1381 goto out_unlock;
729749bb 1382 xprt_pin_rqst(rovr);
ecd465ee 1383 xprt_update_rtt(rovr->rq_task);
75c84151 1384 spin_unlock(&xprt->queue_lock);
a246b010
CL		 1385 task = rovr->rq_task;
1386
a246b010
CL		 1387 if ((copied = rovr->rq_private_buf.buflen) > repsize)
1388 copied = repsize;
1389
1390 /* Suck it into the iovec, verify checksum if not done by hw. */
1781f7f5 1391 if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) {
75c84151 1392 spin_lock(&xprt->queue_lock);
0afa6b44 1393 __UDPX_INC_STATS(sk, UDP_MIB_INERRORS);
729749bb 1394 goto out_unpin;
1781f7f5
HX		 1395 }
1396
a246b010 1397
729749bb 1398 spin_lock_bh(&xprt->transport_lock);
6a24dfb6 1399 xprt_adjust_cwnd(xprt, task, copied);
ce7c252a 1400 spin_unlock_bh(&xprt->transport_lock);
75c84151 1401 spin_lock(&xprt->queue_lock);
1570c1e4 1402 xprt_complete_rqst(task, copied);
0afa6b44 1403 __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS);
729749bb
TM		 1404out_unpin:
1405 xprt_unpin_rqst(rovr);
a246b010 1406 out_unlock:
75c84151 1407 spin_unlock(&xprt->queue_lock);
f9b2ee71
TM		 1408}
1409
1410static void xs_udp_data_receive(struct sock_xprt *transport)
1411{
1412 struct sk_buff *skb;
1413 struct sock *sk;
1414 int err;
1415
0af3442a 1416restart:
f9b2ee71
TM		 1417 mutex_lock(&transport->recv_mutex);
1418 sk = transport->inet;
1419 if (sk == NULL)
1420 goto out;
1421 for (;;) {
7c13f97f 1422 skb = skb_recv_udp(sk, 0, 1, &err);
42d42a5b
TM		 1423 if (skb != NULL) {
1424 xs_udp_data_read_skb(&transport->xprt, sk, skb);
850cbadd 1425 consume_skb(skb);
42d42a5b
TM		 1426 continue;
1427 }
1428 if (!test_and_clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
f9b2ee71 1429 break;
0af3442a
TM		 1430 if (need_resched()) {
1431 mutex_unlock(&transport->recv_mutex);
1432 cond_resched();
1433 goto restart;
1434 }
f9b2ee71
TM		 1435 }
1436out:
1437 mutex_unlock(&transport->recv_mutex);
1438}
1439
1440static void xs_udp_data_receive_workfn(struct work_struct *work)
1441{
1442 struct sock_xprt *transport =
1443 container_of(work, struct sock_xprt, recv_worker);
1444 xs_udp_data_receive(transport);
1445}
1446
1447/**
1448 * xs_data_ready - "data ready" callback for UDP sockets
1449 * @sk: socket with data to read
1450 *
1451 */
1452static void xs_data_ready(struct sock *sk)
1453{
1454 struct rpc_xprt *xprt;
1455
1456 read_lock_bh(&sk->sk_callback_lock);
1457 dprintk("RPC: xs_data_ready...\n");
1458 xprt = xprt_from_sock(sk);
1459 if (xprt != NULL) {
1460 struct sock_xprt *transport = container_of(xprt,
1461 struct sock_xprt, xprt);
5157b956
TM		 1462 transport->old_data_ready(sk);
1463 /* Any data means we had a useful conversation, so
1464 * then we don't need to delay the next reconnect
1465 */
1466 if (xprt->reestablish_timeout)
1467 xprt->reestablish_timeout = 0;
42d42a5b 1468 if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
40a5f1b1 1469 queue_work(xprtiod_workqueue, &transport->recv_worker);
f9b2ee71 1470 }
f064af1e 1471 read_unlock_bh(&sk->sk_callback_lock);
a246b010
CL		 1472}
1473
a519fc7a
TM		 1474/*
1475 * Helper function to force a TCP close if the server is sending
1476 * junk and/or it has put us in CLOSE_WAIT
1477 */
1478static void xs_tcp_force_close(struct rpc_xprt *xprt)
1479{
a519fc7a
TM		 1480 xprt_force_disconnect(xprt);
1481}
1482
9e00abc3 1483#if defined(CONFIG_SUNRPC_BACKCHANNEL)
76566773
CL		 1484static int xs_tcp_bc_up(struct svc_serv *serv, struct net *net)
1485{
1486 int ret;
1487
1488 ret = svc_create_xprt(serv, "tcp-bc", net, PF_INET, 0,
1489 SVC_SOCK_ANONYMOUS);
1490 if (ret < 0)
1491 return ret;
1492 return 0;
1493}
6b26cc8c
CL		 1494
1495static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt)
1496{
1497 return PAGE_SIZE;
1498}
9e00abc3 1499#endif /* CONFIG_SUNRPC_BACKCHANNEL */
44b98efd 1500
66d7a56a
TM		 1501static void xs_tcp_data_receive(struct sock_xprt *transport)
1502{
1503 struct rpc_xprt *xprt = &transport->xprt;
1504 struct sock *sk;
277e4ab7
TM		 1505 size_t read = 0;
1506 ssize_t ret = 0;
66d7a56a 1507
0af3442a 1508restart:
edc1b01c 1509 mutex_lock(&transport->recv_mutex);
66d7a56a 1510 sk = transport->inet;
edc1b01c
TM		 1511 if (sk == NULL)
1512 goto out;
66d7a56a 1513
0af3442a 1514 for (;;) {
277e4ab7
TM		 1515 clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
1516 ret = xs_read_stream(transport, MSG_DONTWAIT | MSG_NOSIGNAL);
1517 if (ret < 0)
8d6f97d6 1518 break;
277e4ab7 1519 read += ret;
0af3442a
TM		 1520 if (need_resched()) {
1521 mutex_unlock(&transport->recv_mutex);
1522 cond_resched();
1523 goto restart;
1524 }
66d7a56a 1525 }
8d6f97d6
TM		 1526 if (test_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
1527 queue_work(xprtiod_workqueue, &transport->recv_worker);
edc1b01c
TM		 1528out:
1529 mutex_unlock(&transport->recv_mutex);
277e4ab7 1530 trace_xs_tcp_data_ready(xprt, ret, read);
66d7a56a
TM		 1531}
1532
edc1b01c
TM		 1533static void xs_tcp_data_receive_workfn(struct work_struct *work)
1534{
1535 struct sock_xprt *transport =
1536 container_of(work, struct sock_xprt, recv_worker);
1537 xs_tcp_data_receive(transport);
1538}
1539
9903cd1c
CL		 1540/**
1541 * xs_tcp_state_change - callback to handle TCP socket state changes
1542 * @sk: socket whose state has changed
1543 *
1544 */
1545static void xs_tcp_state_change(struct sock *sk)
a246b010 1546{
9903cd1c 1547 struct rpc_xprt *xprt;
0fdea1e8 1548 struct sock_xprt *transport;
a246b010 1549
f064af1e 1550 read_lock_bh(&sk->sk_callback_lock);
a246b010
CL		 1551 if (!(xprt = xprt_from_sock(sk)))
1552 goto out;
46121cf7 1553 dprintk("RPC: xs_tcp_state_change client %p...\n", xprt);
669502ff 1554 dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n",
46121cf7
CL		 1555 sk->sk_state, xprt_connected(xprt),
1556 sock_flag(sk, SOCK_DEAD),
669502ff
AC		 1557 sock_flag(sk, SOCK_ZAPPED),
1558 sk->sk_shutdown);
a246b010 1559
0fdea1e8 1560 transport = container_of(xprt, struct sock_xprt, xprt);
40b5ea0c 1561 trace_rpc_socket_state_change(xprt, sk->sk_socket);
a246b010
CL		 1562 switch (sk->sk_state) {
1563 case TCP_ESTABLISHED:
f064af1e 1564 spin_lock(&xprt->transport_lock);
a246b010 1565 if (!xprt_test_and_set_connected(xprt)) {
8b71798c 1566 xprt->connect_cookie++;
0fdea1e8
TM		 1567 clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
1568 xprt_clear_connecting(xprt);
51971139 1569
2a491991 1570 xprt_wake_pending_tasks(xprt, -EAGAIN);
a246b010 1571 }
f064af1e 1572 spin_unlock(&xprt->transport_lock);
a246b010 1573 break;
3b948ae5
TM		 1574 case TCP_FIN_WAIT1:
1575 /* The client initiated a shutdown of the socket */
7c1d71cf 1576 xprt->connect_cookie++;
663b8858 1577 xprt->reestablish_timeout = 0;
3b948ae5 1578 set_bit(XPRT_CLOSING, &xprt->state);
4e857c58 1579 smp_mb__before_atomic();
3b948ae5 1580 clear_bit(XPRT_CONNECTED, &xprt->state);
ef803670 1581 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
4e857c58 1582 smp_mb__after_atomic();
a246b010 1583 break;
632e3bdc 1584 case TCP_CLOSE_WAIT:
3b948ae5 1585 /* The server initiated a shutdown of the socket */
7c1d71cf 1586 xprt->connect_cookie++;
d0bea455 1587 clear_bit(XPRT_CONNECTED, &xprt->state);
a519fc7a 1588 xs_tcp_force_close(xprt);
e9d47639 1589 /* fall through */
663b8858
TM		 1590 case TCP_CLOSING:
1591 /*
1592 * If the server closed down the connection, make sure that
1593 * we back off before reconnecting
1594 */
1595 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
1596 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
3b948ae5
TM		 1597 break;
1598 case TCP_LAST_ACK:
670f9457 1599 set_bit(XPRT_CLOSING, &xprt->state);
4e857c58 1600 smp_mb__before_atomic();
3b948ae5 1601 clear_bit(XPRT_CONNECTED, &xprt->state);
4e857c58 1602 smp_mb__after_atomic();
3b948ae5
TM		 1603 break;
1604 case TCP_CLOSE:
0fdea1e8
TM		 1605 if (test_and_clear_bit(XPRT_SOCK_CONNECTING,
1606 &transport->sock_state))
1607 xprt_clear_connecting(xprt);
9b30889c 1608 clear_bit(XPRT_CLOSING, &xprt->state);
3ffbc1d6
N		 1609 if (sk->sk_err)
1610 xprt_wake_pending_tasks(xprt, -sk->sk_err);
9b30889c
TM		 1611 /* Trigger the socket release */
1612 xs_tcp_force_close(xprt);
a246b010
CL		 1613 }
1614 out:
f064af1e 1615 read_unlock_bh(&sk->sk_callback_lock);
a246b010
CL		 1616}
1617
1f0fa154
IJ		 1618static void xs_write_space(struct sock *sk)
1619{
13331a55 1620 struct socket_wq *wq;
1f0fa154
IJ		 1621 struct rpc_xprt *xprt;
1622
13331a55 1623 if (!sk->sk_socket)
1f0fa154 1624 return;
13331a55 1625 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1f0fa154
IJ		 1626
1627 if (unlikely(!(xprt = xprt_from_sock(sk))))
1628 return;
13331a55
TM		 1629 rcu_read_lock();
1630 wq = rcu_dereference(sk->sk_wq);
1631 if (!wq || test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags) == 0)
1632 goto out;
1f0fa154 1633
c544577d
TM		 1634 if (xprt_write_space(xprt))
1635 sk->sk_write_pending--;
13331a55
TM		 1636out:
1637 rcu_read_unlock();
1f0fa154
IJ		 1638}
1639
9903cd1c 1640/**
c7b2cae8
CL		 1641 * xs_udp_write_space - callback invoked when socket buffer space
1642 * becomes available
9903cd1c
CL		 1643 * @sk: socket whose state has changed
1644 *
a246b010
CL		 1645 * Called when more output buffer space is available for this socket.
1646 * We try not to wake our writers until they can make "significant"
c7b2cae8 1647 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
a246b010
CL		 1648 * with a bunch of small requests.
1649 */
c7b2cae8 1650static void xs_udp_write_space(struct sock *sk)
a246b010 1651{
f064af1e 1652 read_lock_bh(&sk->sk_callback_lock);
a246b010 1653
c7b2cae8 1654 /* from net/core/sock.c:sock_def_write_space */
1f0fa154
IJ		 1655 if (sock_writeable(sk))
1656 xs_write_space(sk);
a246b010 1657
f064af1e 1658 read_unlock_bh(&sk->sk_callback_lock);
c7b2cae8 1659}
a246b010 1660
c7b2cae8
CL		 1661/**
1662 * xs_tcp_write_space - callback invoked when socket buffer space
1663 * becomes available
1664 * @sk: socket whose state has changed
1665 *
1666 * Called when more output buffer space is available for this socket.
1667 * We try not to wake our writers until they can make "significant"
1668 * progress, otherwise we'll waste resources thrashing kernel_sendmsg
1669 * with a bunch of small requests.
1670 */
1671static void xs_tcp_write_space(struct sock *sk)
1672{
f064af1e 1673 read_lock_bh(&sk->sk_callback_lock);
c7b2cae8
CL		 1674
1675 /* from net/core/stream.c:sk_stream_write_space */
64dc6130 1676 if (sk_stream_is_writeable(sk))
1f0fa154 1677 xs_write_space(sk);
b6ddf64f 1678
f064af1e 1679 read_unlock_bh(&sk->sk_callback_lock);
a246b010
CL		 1680}
1681
470056c2 1682static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt)
a246b010 1683{
ee0ac0c2
CL		 1684 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1685 struct sock *sk = transport->inet;
a246b010 1686
7c6e066e 1687 if (transport->rcvsize) {
a246b010 1688 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
7c6e066e 1689 sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;
a246b010 1690 }
7c6e066e 1691 if (transport->sndsize) {
a246b010 1692 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
7c6e066e 1693 sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;
a246b010
CL		 1694 sk->sk_write_space(sk);
1695 }
1696}
1697
43118c29 1698/**
470056c2 1699 * xs_udp_set_buffer_size - set send and receive limits
43118c29 1700 * @xprt: generic transport
470056c2
CL		 1701 * @sndsize: requested size of send buffer, in bytes
1702 * @rcvsize: requested size of receive buffer, in bytes
43118c29 1703 *
470056c2 1704 * Set socket send and receive buffer size limits.
43118c29 1705 */
470056c2 1706static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize)
43118c29 1707{
7c6e066e
CL		 1708 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1709
1710 transport->sndsize = 0;
470056c2 1711 if (sndsize)
7c6e066e
CL		 1712 transport->sndsize = sndsize + 1024;
1713 transport->rcvsize = 0;
470056c2 1714 if (rcvsize)
7c6e066e 1715 transport->rcvsize = rcvsize + 1024;
470056c2
CL		 1716
1717 xs_udp_do_set_buffer_size(xprt);
43118c29
CL		 1718}
1719
46c0ee8b
CL		 1720/**
1721 * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport
1722 * @task: task that timed out
1723 *
1724 * Adjust the congestion window after a retransmit timeout has occurred.
1725 */
6a24dfb6 1726static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task)
46c0ee8b 1727{
b977b644 1728 spin_lock_bh(&xprt->transport_lock);
6a24dfb6 1729 xprt_adjust_cwnd(xprt, task, -ETIMEDOUT);
b977b644 1730 spin_unlock_bh(&xprt->transport_lock);
46c0ee8b
CL		 1731}
1732
b85d8806
CL		 1733static unsigned short xs_get_random_port(void)
1734{
5d71899a 1735 unsigned short range = xprt_max_resvport - xprt_min_resvport + 1;
63862b5b 1736 unsigned short rand = (unsigned short) prandom_u32() % range;
b85d8806
CL		 1737 return rand + xprt_min_resvport;
1738}
1739
4dda9c8a
TM		 1740/**
1741 * xs_set_reuseaddr_port - set the socket's port and address reuse options
1742 * @sock: socket
1743 *
1744 * Note that this function has to be called on all sockets that share the
1745 * same port, and it must be called before binding.
1746 */
1747static void xs_sock_set_reuseport(struct socket *sock)
1748{
402e23b4 1749 int opt = 1;
4dda9c8a 1750
402e23b4
TM		 1751 kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT,
1752 (char *)&opt, sizeof(opt));
4dda9c8a
TM		 1753}
1754
1755static unsigned short xs_sock_getport(struct socket *sock)
1756{
1757 struct sockaddr_storage buf;
4dda9c8a
TM		 1758 unsigned short port = 0;
1759
9b2c45d4 1760 if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0)
4dda9c8a
TM		 1761 goto out;
1762 switch (buf.ss_family) {
1763 case AF_INET6:
1764 port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port);
1765 break;
1766 case AF_INET:
1767 port = ntohs(((struct sockaddr_in *)&buf)->sin_port);
1768 }
1769out:
1770 return port;
1771}
1772
92200412
CL		 1773/**
1774 * xs_set_port - reset the port number in the remote endpoint address
1775 * @xprt: generic transport
1776 * @port: new port number
1777 *
1778 */
1779static void xs_set_port(struct rpc_xprt *xprt, unsigned short port)
1780{
46121cf7 1781 dprintk("RPC: setting port for xprt %p to %u\n", xprt, port);
c4efcb1d 1782
9dc3b095
CL		 1783 rpc_set_port(xs_addr(xprt), port);
1784 xs_update_peer_port(xprt);
92200412
CL		 1785}
1786
4dda9c8a
TM		 1787static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)
1788{
1789 if (transport->srcport == 0)
1790 transport->srcport = xs_sock_getport(sock);
1791}
1792
5d4ec932 1793static unsigned short xs_get_srcport(struct sock_xprt *transport)
67a391d7 1794{
fbfffbd5 1795 unsigned short port = transport->srcport;
67a391d7
TM		 1796
1797 if (port == 0 && transport->xprt.resvport)
1798 port = xs_get_random_port();
1799 return port;
1800}
1801
baaf4e48 1802static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port)
67a391d7 1803{
fbfffbd5
CL		 1804 if (transport->srcport != 0)
1805 transport->srcport = 0;
67a391d7
TM		 1806 if (!transport->xprt.resvport)
1807 return 0;
1808 if (port <= xprt_min_resvport || port > xprt_max_resvport)
1809 return xprt_max_resvport;
1810 return --port;
1811}
beb59b68 1812static int xs_bind(struct sock_xprt *transport, struct socket *sock)
a246b010 1813{
beb59b68 1814 struct sockaddr_storage myaddr;
67a391d7 1815 int err, nloop = 0;
5d4ec932 1816 unsigned short port = xs_get_srcport(transport);
67a391d7 1817 unsigned short last;
a246b010 1818
0f7a622c
CP		 1819 /*
1820 * If we are asking for any ephemeral port (i.e. port == 0 &&
1821 * transport->xprt.resvport == 0), don't bind. Let the local
1822 * port selection happen implicitly when the socket is used
1823 * (for example at connect time).
1824 *
1825 * This ensures that we can continue to establish TCP
1826 * connections even when all local ephemeral ports are already
1827 * a part of some TCP connection. This makes no difference
1828 * for UDP sockets, but also doens't harm them.
1829 *
1830 * If we're asking for any reserved port (i.e. port == 0 &&
1831 * transport->xprt.resvport == 1) xs_get_srcport above will
1832 * ensure that port is non-zero and we will bind as needed.
1833 */
1834 if (port == 0)
1835 return 0;
1836
beb59b68 1837 memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen);
a246b010 1838 do {
beb59b68
PE		 1839 rpc_set_port((struct sockaddr *)&myaddr, port);
1840 err = kernel_bind(sock, (struct sockaddr *)&myaddr,
1841 transport->xprt.addrlen);
a246b010 1842 if (err == 0) {
fbfffbd5 1843 transport->srcport = port;
d3bc9a1d 1844 break;
a246b010 1845 }
67a391d7 1846 last = port;
baaf4e48 1847 port = xs_next_srcport(transport, port);
67a391d7
TM		 1848 if (port > last)
1849 nloop++;
1850 } while (err == -EADDRINUSE && nloop != 2);
90058d37 1851
4232e863 1852 if (myaddr.ss_family == AF_INET)
beb59b68
PE		 1853 dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__,
1854 &((struct sockaddr_in *)&myaddr)->sin_addr,
1855 port, err ? "failed" : "ok", err);
1856 else
1857 dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__,
1858 &((struct sockaddr_in6 *)&myaddr)->sin6_addr,
1859 port, err ? "failed" : "ok", err);
a246b010
CL		 1860 return err;
1861}
1862
176e21ee
CL		 1863/*
1864 * We don't support autobind on AF_LOCAL sockets
1865 */
1866static void xs_local_rpcbind(struct rpc_task *task)
1867{
fb43d172 1868 xprt_set_bound(task->tk_xprt);
176e21ee
CL		 1869}
1870
1871static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port)
1872{
1873}
beb59b68 1874
ed07536e
PZ		 1875#ifdef CONFIG_DEBUG_LOCK_ALLOC
1876static struct lock_class_key xs_key[2];
1877static struct lock_class_key xs_slock_key[2];
1878
176e21ee
CL		 1879static inline void xs_reclassify_socketu(struct socket *sock)
1880{
1881 struct sock *sk = sock->sk;
1882
176e21ee
CL		 1883 sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC",
1884 &xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]);
1885}
1886
8945ee5e 1887static inline void xs_reclassify_socket4(struct socket *sock)
ed07536e
PZ		 1888{
1889 struct sock *sk = sock->sk;
8945ee5e 1890
8945ee5e
CL		 1891 sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC",
1892 &xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]);
1893}
ed07536e 1894
8945ee5e
CL		 1895static inline void xs_reclassify_socket6(struct socket *sock)
1896{
1897 struct sock *sk = sock->sk;
ed07536e 1898
8945ee5e
CL		 1899 sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC",
1900 &xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]);
ed07536e 1901}
6bc9638a
PE		 1902
1903static inline void xs_reclassify_socket(int family, struct socket *sock)
1904{
fafc4e1e 1905 if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk)))
1b7a1819
WAA		 1906 return;
1907
4232e863 1908 switch (family) {
176e21ee
CL		 1909 case AF_LOCAL:
1910 xs_reclassify_socketu(sock);
1911 break;
4232e863 1912 case AF_INET:
6bc9638a 1913 xs_reclassify_socket4(sock);
4232e863
CL		 1914 break;
1915 case AF_INET6:
6bc9638a 1916 xs_reclassify_socket6(sock);
4232e863
CL		 1917 break;
1918 }
6bc9638a 1919}
ed07536e 1920#else
6bc9638a
PE		 1921static inline void xs_reclassify_socket(int family, struct socket *sock)
1922{
1923}
ed07536e
PZ		 1924#endif
1925
93dc41bd
N		 1926static void xs_dummy_setup_socket(struct work_struct *work)
1927{
1928}
1929
6bc9638a 1930static struct socket *xs_create_sock(struct rpc_xprt *xprt,
4dda9c8a
TM		 1931 struct sock_xprt *transport, int family, int type,
1932 int protocol, bool reuseport)
22f79326
PE		 1933{
1934 struct socket *sock;
1935 int err;
1936
6bc9638a 1937 err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1);
22f79326
PE		 1938 if (err < 0) {
1939 dprintk("RPC: can't create %d transport socket (%d).\n",
1940 protocol, -err);
1941 goto out;
1942 }
6bc9638a 1943 xs_reclassify_socket(family, sock);
22f79326 1944
4dda9c8a
TM		 1945 if (reuseport)
1946 xs_sock_set_reuseport(sock);
1947
4cea288a
BH		 1948 err = xs_bind(transport, sock);
1949 if (err) {
22f79326
PE		 1950 sock_release(sock);
1951 goto out;
1952 }
1953
1954 return sock;
1955out:
1956 return ERR_PTR(err);
1957}
1958
176e21ee
CL		 1959static int xs_local_finish_connecting(struct rpc_xprt *xprt,
1960 struct socket *sock)
1961{
1962 struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
1963 xprt);
1964
1965 if (!transport->inet) {
1966 struct sock *sk = sock->sk;
1967
1968 write_lock_bh(&sk->sk_callback_lock);
1969
1970 xs_save_old_callbacks(transport, sk);
1971
1972 sk->sk_user_data = xprt;
a2648094 1973 sk->sk_data_ready = xs_data_ready;
176e21ee 1974 sk->sk_write_space = xs_udp_write_space;
b4411457 1975 sock_set_flag(sk, SOCK_FASYNC);
2118071d 1976 sk->sk_error_report = xs_error_report;
c2126157 1977 sk->sk_allocation = GFP_NOIO;
176e21ee
CL		 1978
1979 xprt_clear_connected(xprt);
1980
1981 /* Reset to new socket */
1982 transport->sock = sock;
1983 transport->inet = sk;
1984
1985 write_unlock_bh(&sk->sk_callback_lock);
1986 }
1987
6c7a64e5
TM		 1988 transport->xmit.offset = 0;
1989
176e21ee
CL		 1990 /* Tell the socket layer to start connecting... */
1991 xprt->stat.connect_count++;
1992 xprt->stat.connect_start = jiffies;
1993 return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0);
1994}
1995
1996/**
1997 * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint
176e21ee 1998 * @transport: socket transport to connect
176e21ee 1999 */
dc107402 2000static int xs_local_setup_socket(struct sock_xprt *transport)
176e21ee 2001{
176e21ee
CL		 2002 struct rpc_xprt *xprt = &transport->xprt;
2003 struct socket *sock;
2004 int status = -EIO;
2005
176e21ee
CL		 2006 status = __sock_create(xprt->xprt_net, AF_LOCAL,
2007 SOCK_STREAM, 0, &sock, 1);
2008 if (status < 0) {
2009 dprintk("RPC: can't create AF_LOCAL "
2010 "transport socket (%d).\n", -status);
2011 goto out;
2012 }
d1358917 2013 xs_reclassify_socket(AF_LOCAL, sock);
176e21ee
CL		 2014
2015 dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n",
2016 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
2017
2018 status = xs_local_finish_connecting(xprt, sock);
40b5ea0c 2019 trace_rpc_socket_connect(xprt, sock, status);
176e21ee
CL		 2020 switch (status) {
2021 case 0:
2022 dprintk("RPC: xprt %p connected to %s\n",
2023 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
2024 xprt_set_connected(xprt);
3601c4a9 2025 case -ENOBUFS:
176e21ee
CL		 2026 break;
2027 case -ENOENT:
2028 dprintk("RPC: xprt %p: socket %s does not exist\n",
2029 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
2030 break;
4a20a988
TM		 2031 case -ECONNREFUSED:
2032 dprintk("RPC: xprt %p: connection refused for %s\n",
2033 xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
2034 break;
176e21ee
CL		 2035 default:
2036 printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n",
2037 __func__, -status,
2038 xprt->address_strings[RPC_DISPLAY_ADDR]);
2039 }
2040
2041out:
2042 xprt_clear_connecting(xprt);
2043 xprt_wake_pending_tasks(xprt, status);
dc107402
BF		 2044 return status;
2045}
2046
b6669737 2047static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task)
dc107402 2048{
dc107402
BF		 2049 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2050 int ret;
2051
2052 if (RPC_IS_ASYNC(task)) {
2053 /*
2054 * We want the AF_LOCAL connect to be resolved in the
2055 * filesystem namespace of the process making the rpc
2056 * call. Thus we connect synchronously.
2057 *
2058 * If we want to support asynchronous AF_LOCAL calls,
2059 * we'll need to figure out how to pass a namespace to
2060 * connect.
2061 */
2062 rpc_exit(task, -ENOTCONN);
2063 return;
2064 }
2065 ret = xs_local_setup_socket(transport);
2066 if (ret && !RPC_IS_SOFTCONN(task))
2067 msleep_interruptible(15000);
176e21ee
CL		 2068}
2069
3c87ef6e 2070#if IS_ENABLED(CONFIG_SUNRPC_SWAP)
d6e971d8
JL		 2071/*
2072 * Note that this should be called with XPRT_LOCKED held (or when we otherwise
2073 * know that we have exclusive access to the socket), to guard against
2074 * races with xs_reset_transport.
2075 */
a564b8f0
MG		 2076static void xs_set_memalloc(struct rpc_xprt *xprt)
2077{
2078 struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
2079 xprt);
2080
d6e971d8
JL		 2081 /*
2082 * If there's no sock, then we have nothing to set. The
2083 * reconnecting process will get it for us.
2084 */
2085 if (!transport->inet)
2086 return;
8e228133 2087 if (atomic_read(&xprt->swapper))
a564b8f0
MG		 2088 sk_set_memalloc(transport->inet);
2089}
2090
2091/**
d67fa4d8 2092 * xs_enable_swap - Tag this transport as being used for swap.
a564b8f0 2093 * @xprt: transport to tag
a564b8f0 2094 *
8e228133
JL		 2095 * Take a reference to this transport on behalf of the rpc_clnt, and
2096 * optionally mark it for swapping if it wasn't already.
a564b8f0 2097 */
d67fa4d8
JL		 2098static int
2099xs_enable_swap(struct rpc_xprt *xprt)
a564b8f0 2100{
d6e971d8 2101 struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
a564b8f0 2102
d6e971d8
JL		 2103 if (atomic_inc_return(&xprt->swapper) != 1)
2104 return 0;
2105 if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE))
2106 return -ERESTARTSYS;
2107 if (xs->inet)
2108 sk_set_memalloc(xs->inet);
2109 xprt_release_xprt(xprt, NULL);
8e228133
JL		 2110 return 0;
2111}
a564b8f0 2112
8e228133 2113/**
d67fa4d8 2114 * xs_disable_swap - Untag this transport as being used for swap.
8e228133
JL		 2115 * @xprt: transport to tag
2116 *
2117 * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the
2118 * swapper refcount goes to 0, untag the socket as a memalloc socket.
2119 */
d67fa4d8
JL		 2120static void
2121xs_disable_swap(struct rpc_xprt *xprt)
8e228133 2122{
d6e971d8 2123 struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
8e228133 2124
d6e971d8
JL		 2125 if (!atomic_dec_and_test(&xprt->swapper))
2126 return;
2127 if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE))
2128 return;
2129 if (xs->inet)
2130 sk_clear_memalloc(xs->inet);
2131 xprt_release_xprt(xprt, NULL);
a564b8f0 2132}
a564b8f0
MG		 2133#else
2134static void xs_set_memalloc(struct rpc_xprt *xprt)
2135{
2136}
d67fa4d8
JL		 2137
2138static int
2139xs_enable_swap(struct rpc_xprt *xprt)
2140{
2141 return -EINVAL;
2142}
2143
2144static void
2145xs_disable_swap(struct rpc_xprt *xprt)
2146{
2147}
a564b8f0
MG		 2148#endif
2149
16be2d20
CL		 2150static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
2151{
2152 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2153
2154 if (!transport->inet) {
2155 struct sock *sk = sock->sk;
2156
2157 write_lock_bh(&sk->sk_callback_lock);
2158
2a9e1cfa
TM		 2159 xs_save_old_callbacks(transport, sk);
2160
16be2d20 2161 sk->sk_user_data = xprt;
f9b2ee71 2162 sk->sk_data_ready = xs_data_ready;
16be2d20 2163 sk->sk_write_space = xs_udp_write_space;
b4411457 2164 sock_set_flag(sk, SOCK_FASYNC);
c2126157 2165 sk->sk_allocation = GFP_NOIO;
16be2d20
CL		 2166
2167 xprt_set_connected(xprt);
2168
2169 /* Reset to new socket */
2170 transport->sock = sock;
2171 transport->inet = sk;
2172
a564b8f0
MG		 2173 xs_set_memalloc(xprt);
2174
16be2d20
CL		 2175 write_unlock_bh(&sk->sk_callback_lock);
2176 }
2177 xs_udp_do_set_buffer_size(xprt);
02910177
TM		 2178
2179 xprt->stat.connect_start = jiffies;
16be2d20
CL		 2180}
2181
8c14ff2a 2182static void xs_udp_setup_socket(struct work_struct *work)
a246b010 2183{
34161db6
TM		 2184 struct sock_xprt *transport =
2185 container_of(work, struct sock_xprt, connect_worker.work);
c8475461 2186 struct rpc_xprt *xprt = &transport->xprt;
d099b8af 2187 struct socket *sock;
b65c0310 2188 int status = -EIO;
9903cd1c 2189
8c14ff2a 2190 sock = xs_create_sock(xprt, transport,
4dda9c8a
TM		 2191 xs_addr(xprt)->sa_family, SOCK_DGRAM,
2192 IPPROTO_UDP, false);
b65c0310 2193 if (IS_ERR(sock))
b0d93ad5 2194 goto out;
68e220bd 2195
c740eff8
CL		 2196 dprintk("RPC: worker connecting xprt %p via %s to "
2197 "%s (port %s)\n", xprt,
2198 xprt->address_strings[RPC_DISPLAY_PROTO],
2199 xprt->address_strings[RPC_DISPLAY_ADDR],
2200 xprt->address_strings[RPC_DISPLAY_PORT]);
68e220bd
CL		 2201
2202 xs_udp_finish_connecting(xprt, sock);
40b5ea0c 2203 trace_rpc_socket_connect(xprt, sock, 0);
b0d93ad5
CL		 2204 status = 0;
2205out:
718ba5b8 2206 xprt_unlock_connect(xprt, transport);
b0d93ad5 2207 xprt_clear_connecting(xprt);
7d1e8255 2208 xprt_wake_pending_tasks(xprt, status);
a246b010
CL		 2209}
2210
4876cc77
TM		 2211/**
2212 * xs_tcp_shutdown - gracefully shut down a TCP socket
2213 * @xprt: transport
2214 *
2215 * Initiates a graceful shutdown of the TCP socket by calling the
2216 * equivalent of shutdown(SHUT_RDWR);
2217 */
2218static void xs_tcp_shutdown(struct rpc_xprt *xprt)
2219{
2220 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2221 struct socket *sock = transport->sock;
9b30889c 2222 int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE;
4876cc77
TM		 2223
2224 if (sock == NULL)
2225 return;
9b30889c
TM		 2226 switch (skst) {
2227 default:
4876cc77
TM		 2228 kernel_sock_shutdown(sock, SHUT_RDWR);
2229 trace_rpc_socket_shutdown(xprt, sock);
9b30889c
TM		 2230 break;
2231 case TCP_CLOSE:
2232 case TCP_TIME_WAIT:
4876cc77 2233 xs_reset_transport(transport);
9b30889c 2234 }
4876cc77
TM		 2235}
2236
8d1b8c62
TM		 2237static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
2238 struct socket *sock)
2239{
7196dbb0
TM		 2240 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2241 unsigned int keepidle;
2242 unsigned int keepcnt;
8d1b8c62
TM		 2243 unsigned int opt_on = 1;
2244 unsigned int timeo;
2245
7196dbb0
TM		 2246 spin_lock_bh(&xprt->transport_lock);
2247 keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ);
2248 keepcnt = xprt->timeout->to_retries + 1;
2249 timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
2250 (xprt->timeout->to_retries + 1);
2251 clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
2252 spin_unlock_bh(&xprt->transport_lock);
2253
8d1b8c62
TM		 2254 /* TCP Keepalive options */
2255 kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,
2256 (char *)&opt_on, sizeof(opt_on));
2257 kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE,
2258 (char *)&keepidle, sizeof(keepidle));
2259 kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL,
2260 (char *)&keepidle, sizeof(keepidle));
2261 kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT,
2262 (char *)&keepcnt, sizeof(keepcnt));
2263
2264 /* TCP user timeout (see RFC5482) */
8d1b8c62
TM		 2265 kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT,
2266 (char *)&timeo, sizeof(timeo));
2267}
2268
7196dbb0
TM		 2269static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt,
2270 unsigned long connect_timeout,
2271 unsigned long reconnect_timeout)
2272{
2273 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2274 struct rpc_timeout to;
2275 unsigned long initval;
2276
2277 spin_lock_bh(&xprt->transport_lock);
2278 if (reconnect_timeout < xprt->max_reconnect_timeout)
2279 xprt->max_reconnect_timeout = reconnect_timeout;
2280 if (connect_timeout < xprt->connect_timeout) {
2281 memcpy(&to, xprt->timeout, sizeof(to));
2282 initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1);
2283 /* Arbitrary lower limit */
2284 if (initval < XS_TCP_INIT_REEST_TO << 1)
2285 initval = XS_TCP_INIT_REEST_TO << 1;
2286 to.to_initval = initval;
2287 to.to_maxval = initval;
2288 memcpy(&transport->tcp_timeout, &to,
2289 sizeof(transport->tcp_timeout));
2290 xprt->timeout = &transport->tcp_timeout;
2291 xprt->connect_timeout = connect_timeout;
2292 }
2293 set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
2294 spin_unlock_bh(&xprt->transport_lock);
2295}
2296
16be2d20 2297static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
a246b010 2298{
16be2d20 2299 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
fe19a96b 2300 int ret = -ENOTCONN;
edb267a6 2301
ee0ac0c2 2302 if (!transport->inet) {
b0d93ad5 2303 struct sock *sk = sock->sk;
d88e4d82 2304 unsigned int addr_pref = IPV6_PREFER_SRC_PUBLIC;
7f260e85 2305
d88e4d82
N		 2306 /* Avoid temporary address, they are bad for long-lived
2307 * connections such as NFS mounts.
2308 * RFC4941, section 3.6 suggests that:
2309 * Individual applications, which have specific
2310 * knowledge about the normal duration of connections,
2311 * MAY override this as appropriate.
2312 */
2313 kernel_setsockopt(sock, SOL_IPV6, IPV6_ADDR_PREFERENCES,
2314 (char *)&addr_pref, sizeof(addr_pref));
2315
8d1b8c62 2316 xs_tcp_set_socket_timeouts(xprt, sock);
775f06ab 2317
b0d93ad5
CL		 2318 write_lock_bh(&sk->sk_callback_lock);
2319
2a9e1cfa
TM		 2320 xs_save_old_callbacks(transport, sk);
2321
b0d93ad5 2322 sk->sk_user_data = xprt;
5157b956 2323 sk->sk_data_ready = xs_data_ready;
b0d93ad5
CL		 2324 sk->sk_state_change = xs_tcp_state_change;
2325 sk->sk_write_space = xs_tcp_write_space;
b4411457 2326 sock_set_flag(sk, SOCK_FASYNC);
2118071d 2327 sk->sk_error_report = xs_error_report;
c2126157 2328 sk->sk_allocation = GFP_NOIO;
3167e12c
CL		 2329
2330 /* socket options */
3167e12c 2331 sock_reset_flag(sk, SOCK_LINGER);
3167e12c 2332 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF;
b0d93ad5
CL		 2333
2334 xprt_clear_connected(xprt);
2335
2336 /* Reset to new socket */
ee0ac0c2
CL		 2337 transport->sock = sock;
2338 transport->inet = sk;
b0d93ad5
CL		 2339
2340 write_unlock_bh(&sk->sk_callback_lock);
2341 }
2342
01d37c42 2343 if (!xprt_bound(xprt))
fe19a96b 2344 goto out;
01d37c42 2345
a564b8f0
MG		 2346 xs_set_memalloc(xprt);
2347
e1806c7b
TM		 2348 /* Reset TCP record info */
2349 transport->recv.offset = 0;
2350 transport->recv.len = 0;
2351 transport->recv.copied = 0;
6c7a64e5 2352 transport->xmit.offset = 0;
e1806c7b 2353
b0d93ad5 2354 /* Tell the socket layer to start connecting... */
262ca07d
CL		 2355 xprt->stat.connect_count++;
2356 xprt->stat.connect_start = jiffies;
0fdea1e8 2357 set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
fe19a96b
TM		 2358 ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);
2359 switch (ret) {
2360 case 0:
4dda9c8a 2361 xs_set_srcport(transport, sock);
e9d47639 2362 /* fall through */
fe19a96b
TM		 2363 case -EINPROGRESS:
2364 /* SYN_SENT! */
fe19a96b
TM		 2365 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
2366 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
1f4c17a0
TM		 2367 break;
2368 case -EADDRNOTAVAIL:
2369 /* Source port number is unavailable. Try a new one! */
2370 transport->srcport = 0;
fe19a96b
TM		 2371 }
2372out:
2373 return ret;
16be2d20
CL		 2374}
2375
9903cd1c 2376/**
b61d59ff 2377 * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint
9903cd1c
CL		 2378 *
2379 * Invoked by a work queue tasklet.
a246b010 2380 */
cdd518d5 2381static void xs_tcp_setup_socket(struct work_struct *work)
a246b010 2382{
cdd518d5
PE		 2383 struct sock_xprt *transport =
2384 container_of(work, struct sock_xprt, connect_worker.work);
ee0ac0c2 2385 struct socket *sock = transport->sock;
a9f5f0f7 2386 struct rpc_xprt *xprt = &transport->xprt;
b61d59ff 2387 int status = -EIO;
a246b010 2388
ee0ac0c2 2389 if (!sock) {
cdd518d5 2390 sock = xs_create_sock(xprt, transport,
4dda9c8a
TM		 2391 xs_addr(xprt)->sa_family, SOCK_STREAM,
2392 IPPROTO_TCP, true);
b61d59ff
TM		 2393 if (IS_ERR(sock)) {
2394 status = PTR_ERR(sock);
3167e12c
CL		 2395 goto out;
2396 }
7d1e8255 2397 }
a246b010 2398
c740eff8
CL		 2399 dprintk("RPC: worker connecting xprt %p via %s to "
2400 "%s (port %s)\n", xprt,
2401 xprt->address_strings[RPC_DISPLAY_PROTO],
2402 xprt->address_strings[RPC_DISPLAY_ADDR],
2403 xprt->address_strings[RPC_DISPLAY_PORT]);
edb267a6 2404
16be2d20 2405 status = xs_tcp_finish_connecting(xprt, sock);
40b5ea0c 2406 trace_rpc_socket_connect(xprt, sock, status);
46121cf7
CL		 2407 dprintk("RPC: %p connect status %d connected %d sock state %d\n",
2408 xprt, -status, xprt_connected(xprt),
2409 sock->sk->sk_state);
2a491991 2410 switch (status) {
f75e6745
TM		 2411 default:
2412 printk("%s: connect returned unhandled error %d\n",
2413 __func__, status);
e9d47639 2414 /* fall through */
f75e6745
TM		 2415 case -EADDRNOTAVAIL:
2416 /* We're probably in TIME_WAIT. Get rid of existing socket,
2417 * and retry
2418 */
a519fc7a 2419 xs_tcp_force_close(xprt);
88b5ed73 2420 break;
2a491991
TM		 2421 case 0:
2422 case -EINPROGRESS:
2423 case -EALREADY:
718ba5b8 2424 xprt_unlock_connect(xprt, transport);
7d1e8255 2425 return;
9fcfe0c8
TM		 2426 case -EINVAL:
2427 /* Happens, for instance, if the user specified a link
2428 * local IPv6 address without a scope-id.
2429 */
3ed5e2a2
TM		 2430 case -ECONNREFUSED:
2431 case -ECONNRESET:
eb5b46fa 2432 case -ENETDOWN:
3ed5e2a2 2433 case -ENETUNREACH:
4ba161a7 2434 case -EHOSTUNREACH:
3913c78c 2435 case -EADDRINUSE:
3601c4a9 2436 case -ENOBUFS:
6ea44adc
N		 2437 /*
2438 * xs_tcp_force_close() wakes tasks with -EIO.
2439 * We need to wake them first to ensure the
2440 * correct error code.
2441 */
2442 xprt_wake_pending_tasks(xprt, status);
4efdd92c 2443 xs_tcp_force_close(xprt);
9fcfe0c8 2444 goto out;
a246b010 2445 }
2a491991 2446 status = -EAGAIN;
a246b010 2447out:
718ba5b8 2448 xprt_unlock_connect(xprt, transport);
68e220bd 2449 xprt_clear_connecting(xprt);
7d1e8255 2450 xprt_wake_pending_tasks(xprt, status);
68e220bd 2451}
b0d93ad5 2452
02910177
TM		 2453static unsigned long xs_reconnect_delay(const struct rpc_xprt *xprt)
2454{
2455 unsigned long start, now = jiffies;
2456
2457 start = xprt->stat.connect_start + xprt->reestablish_timeout;
2458 if (time_after(start, now))
2459 return start - now;
2460 return 0;
2461}
2462
3851f1cd
TM		 2463static void xs_reconnect_backoff(struct rpc_xprt *xprt)
2464{
2465 xprt->reestablish_timeout <<= 1;
2466 if (xprt->reestablish_timeout > xprt->max_reconnect_timeout)
2467 xprt->reestablish_timeout = xprt->max_reconnect_timeout;
2468 if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
2469 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2470}
2471
9903cd1c
CL		 2472/**
2473 * xs_connect - connect a socket to a remote endpoint
1b092092 2474 * @xprt: pointer to transport structure
9903cd1c
CL		 2475 * @task: address of RPC task that manages state of connect request
2476 *
2477 * TCP: If the remote end dropped the connection, delay reconnecting.
03bf4b70
CL		 2478 *
2479 * UDP socket connects are synchronous, but we use a work queue anyway
2480 * to guarantee that even unprivileged user processes can set up a
2481 * socket on a privileged port.
2482 *
2483 * If a UDP socket connect fails, the delay behavior here prevents
2484 * retry floods (hard mounts).
9903cd1c 2485 */
1b092092 2486static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
a246b010 2487{
ee0ac0c2 2488 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
02910177 2489 unsigned long delay = 0;
a246b010 2490
718ba5b8
TM		 2491 WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));
2492
99b1a4c3 2493 if (transport->sock != NULL) {
46121cf7
CL		 2494 dprintk("RPC: xs_connect delayed xprt %p for %lu "
2495 "seconds\n",
03bf4b70 2496 xprt, xprt->reestablish_timeout / HZ);
99b1a4c3
TM		 2497
2498 /* Start by resetting any existing state */
2499 xs_reset_transport(transport);
2500
02910177 2501 delay = xs_reconnect_delay(xprt);
3851f1cd 2502 xs_reconnect_backoff(xprt);
02910177 2503
02910177 2504 } else
46121cf7 2505 dprintk("RPC: xs_connect scheduled xprt %p\n", xprt);
02910177
TM		 2506
2507 queue_delayed_work(xprtiod_workqueue,
2508 &transport->connect_worker,
2509 delay);
a246b010
CL		 2510}
2511
176e21ee
CL		 2512/**
2513 * xs_local_print_stats - display AF_LOCAL socket-specifc stats
2514 * @xprt: rpc_xprt struct containing statistics
2515 * @seq: output file
2516 *
2517 */
2518static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2519{
2520 long idle_time = 0;
2521
2522 if (xprt_connected(xprt))
2523 idle_time = (long)(jiffies - xprt->last_used) / HZ;
2524
2525 seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu "
15a45206 2526 "%llu %llu %lu %llu %llu\n",
176e21ee
CL		 2527 xprt->stat.bind_count,
2528 xprt->stat.connect_count,
2529 xprt->stat.connect_time,
2530 idle_time,
2531 xprt->stat.sends,
2532 xprt->stat.recvs,
2533 xprt->stat.bad_xids,
2534 xprt->stat.req_u,
15a45206
AA		 2535 xprt->stat.bklog_u,
2536 xprt->stat.max_slots,
2537 xprt->stat.sending_u,
2538 xprt->stat.pending_u);
176e21ee
CL		 2539}
2540
262ca07d
CL		 2541/**
2542 * xs_udp_print_stats - display UDP socket-specifc stats
2543 * @xprt: rpc_xprt struct containing statistics
2544 * @seq: output file
2545 *
2546 */
2547static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2548{
c8475461
CL		 2549 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2550
15a45206
AA		 2551 seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu "
2552 "%lu %llu %llu\n",
fbfffbd5 2553 transport->srcport,
262ca07d
CL		 2554 xprt->stat.bind_count,
2555 xprt->stat.sends,
2556 xprt->stat.recvs,
2557 xprt->stat.bad_xids,
2558 xprt->stat.req_u,
15a45206
AA		 2559 xprt->stat.bklog_u,
2560 xprt->stat.max_slots,
2561 xprt->stat.sending_u,
2562 xprt->stat.pending_u);
262ca07d
CL		 2563}
2564
2565/**
2566 * xs_tcp_print_stats - display TCP socket-specifc stats
2567 * @xprt: rpc_xprt struct containing statistics
2568 * @seq: output file
2569 *
2570 */
2571static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2572{
c8475461 2573 struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
262ca07d
CL		 2574 long idle_time = 0;
2575
2576 if (xprt_connected(xprt))
2577 idle_time = (long)(jiffies - xprt->last_used) / HZ;
2578
15a45206
AA		 2579 seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu "
2580 "%llu %llu %lu %llu %llu\n",
fbfffbd5 2581 transport->srcport,
262ca07d
CL		 2582 xprt->stat.bind_count,
2583 xprt->stat.connect_count,
2584 xprt->stat.connect_time,
2585 idle_time,
2586 xprt->stat.sends,
2587 xprt->stat.recvs,
2588 xprt->stat.bad_xids,
2589 xprt->stat.req_u,
15a45206
AA		 2590 xprt->stat.bklog_u,
2591 xprt->stat.max_slots,
2592 xprt->stat.sending_u,
2593 xprt->stat.pending_u);
262ca07d
CL		 2594}
2595
4cfc7e60
RI		 2596/*
2597 * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason
2598 * we allocate pages instead doing a kmalloc like rpc_malloc is because we want
2599 * to use the server side send routines.
2600 */
5fe6eaa1 2601static int bc_malloc(struct rpc_task *task)
4cfc7e60 2602{
5fe6eaa1
CL		 2603 struct rpc_rqst *rqst = task->tk_rqstp;
2604 size_t size = rqst->rq_callsize;
4cfc7e60
RI		 2605 struct page *page;
2606 struct rpc_buffer *buf;
2607
5fe6eaa1
CL		 2608 if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) {
2609 WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n",
2610 size);
2611 return -EINVAL;
2612 }
4cfc7e60 2613
b8a13d03 2614 page = alloc_page(GFP_KERNEL);
4cfc7e60 2615 if (!page)
5fe6eaa1 2616 return -ENOMEM;
4cfc7e60
RI		 2617
2618 buf = page_address(page);
2619 buf->len = PAGE_SIZE;
2620
5fe6eaa1 2621 rqst->rq_buffer = buf->data;
18e601d6 2622 rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize;
5fe6eaa1 2623 return 0;
4cfc7e60
RI		 2624}
2625
2626/*
2627 * Free the space allocated in the bc_alloc routine
2628 */
3435c74a 2629static void bc_free(struct rpc_task *task)
4cfc7e60 2630{
3435c74a 2631 void *buffer = task->tk_rqstp->rq_buffer;
4cfc7e60
RI		 2632 struct rpc_buffer *buf;
2633
4cfc7e60
RI		 2634 buf = container_of(buffer, struct rpc_buffer, data);
2635 free_page((unsigned long)buf);
2636}
2637
2638/*
2639 * Use the svc_sock to send the callback. Must be called with svsk->sk_mutex
2640 * held. Borrows heavily from svc_tcp_sendto and xs_tcp_send_request.
2641 */
2642static int bc_sendto(struct rpc_rqst *req)
2643{
2644 int len;
2645 struct xdr_buf *xbufp = &req->rq_snd_buf;
2646 struct rpc_xprt *xprt = req->rq_xprt;
2647 struct sock_xprt *transport =
2648 container_of(xprt, struct sock_xprt, xprt);
2649 struct socket *sock = transport->sock;
2650 unsigned long headoff;
2651 unsigned long tailoff;
2652
61677eee 2653 xs_encode_stream_record_marker(xbufp);
4cfc7e60
RI		 2654
2655 tailoff = (unsigned long)xbufp->tail[0].iov_base & ~PAGE_MASK;
2656 headoff = (unsigned long)xbufp->head[0].iov_base & ~PAGE_MASK;
2657 len = svc_send_common(sock, xbufp,
2658 virt_to_page(xbufp->head[0].iov_base), headoff,
2659 xbufp->tail[0].iov_base, tailoff);
2660
2661 if (len != xbufp->len) {
2662 printk(KERN_NOTICE "Error sending entire callback!\n");
2663 len = -EAGAIN;
2664 }
2665
2666 return len;
2667}
2668
2669/*
2670 * The send routine. Borrows from svc_send
2671 */
adfa7144 2672static int bc_send_request(struct rpc_rqst *req)
4cfc7e60 2673{
4cfc7e60 2674 struct svc_xprt *xprt;
7fc56136 2675 int len;
4cfc7e60
RI		 2676
2677 dprintk("sending request with xid: %08x\n", ntohl(req->rq_xid));
2678 /*
2679 * Get the server socket associated with this callback xprt
2680 */
2681 xprt = req->rq_xprt->bc_xprt;
4cfc7e60
RI		 2682
2683 /*
2684 * Grab the mutex to serialize data as the connection is shared
2685 * with the fore channel
2686 */
c544577d 2687 mutex_lock(&xprt->xpt_mutex);
4cfc7e60
RI		 2688 if (test_bit(XPT_DEAD, &xprt->xpt_flags))
2689 len = -ENOTCONN;
2690 else
2691 len = bc_sendto(req);
2692 mutex_unlock(&xprt->xpt_mutex);
2693
2694 if (len > 0)
2695 len = 0;
2696
2697 return len;
2698}
2699
2700/*
2701 * The close routine. Since this is client initiated, we do nothing
2702 */
2703
2704static void bc_close(struct rpc_xprt *xprt)
2705{
4cfc7e60
RI		 2706}
2707
2708/*
2709 * The xprt destroy routine. Again, because this connection is client
2710 * initiated, we do nothing
2711 */
2712
2713static void bc_destroy(struct rpc_xprt *xprt)
2714{
47f72efa
KM		 2715 dprintk("RPC: bc_destroy xprt %p\n", xprt);
2716
2717 xs_xprt_free(xprt);
2718 module_put(THIS_MODULE);
4cfc7e60
RI		 2719}
2720
d31ae254 2721static const struct rpc_xprt_ops xs_local_ops = {
176e21ee 2722 .reserve_xprt = xprt_reserve_xprt,
4cd34e7c 2723 .release_xprt = xprt_release_xprt,
f39c1bfb 2724 .alloc_slot = xprt_alloc_slot,
a9cde23a 2725 .free_slot = xprt_free_slot,
176e21ee
CL		 2726 .rpcbind = xs_local_rpcbind,
2727 .set_port = xs_local_set_port,
dc107402 2728 .connect = xs_local_connect,
176e21ee
CL		 2729 .buf_alloc = rpc_malloc,
2730 .buf_free = rpc_free,
2731 .send_request = xs_local_send_request,
2732 .set_retrans_timeout = xprt_set_retrans_timeout_def,
2733 .close = xs_close,
a1311d87 2734 .destroy = xs_destroy,
176e21ee 2735 .print_stats = xs_local_print_stats,
d67fa4d8
JL		 2736 .enable_swap = xs_enable_swap,
2737 .disable_swap = xs_disable_swap,
176e21ee
CL		 2738};
2739
d31ae254 2740static const struct rpc_xprt_ops xs_udp_ops = {
43118c29 2741 .set_buffer_size = xs_udp_set_buffer_size,
12a80469 2742 .reserve_xprt = xprt_reserve_xprt_cong,
49e9a890 2743 .release_xprt = xprt_release_xprt_cong,
f39c1bfb 2744 .alloc_slot = xprt_alloc_slot,
a9cde23a 2745 .free_slot = xprt_free_slot,
45160d62 2746 .rpcbind = rpcb_getport_async,
92200412 2747 .set_port = xs_set_port,
262965f5 2748 .connect = xs_connect,
02107148
CL		 2749 .buf_alloc = rpc_malloc,
2750 .buf_free = rpc_free,
262965f5 2751 .send_request = xs_udp_send_request,
fe3aca29 2752 .set_retrans_timeout = xprt_set_retrans_timeout_rtt,
46c0ee8b 2753 .timer = xs_udp_timer,
a58dd398 2754 .release_request = xprt_release_rqst_cong,
262965f5
CL		 2755 .close = xs_close,
2756 .destroy = xs_destroy,
262ca07d 2757 .print_stats = xs_udp_print_stats,
d67fa4d8
JL		 2758 .enable_swap = xs_enable_swap,
2759 .disable_swap = xs_disable_swap,
4a068258 2760 .inject_disconnect = xs_inject_disconnect,
262965f5
CL		 2761};
2762
d31ae254 2763static const struct rpc_xprt_ops xs_tcp_ops = {
12a80469 2764 .reserve_xprt = xprt_reserve_xprt,
4cd34e7c 2765 .release_xprt = xprt_release_xprt,
36bd7de9 2766 .alloc_slot = xprt_alloc_slot,
a9cde23a 2767 .free_slot = xprt_free_slot,
45160d62 2768 .rpcbind = rpcb_getport_async,
92200412 2769 .set_port = xs_set_port,
0b9e7943 2770 .connect = xs_connect,
02107148
CL		 2771 .buf_alloc = rpc_malloc,
2772 .buf_free = rpc_free,
277e4ab7 2773 .prepare_request = xs_stream_prepare_request,
262965f5 2774 .send_request = xs_tcp_send_request,
fe3aca29 2775 .set_retrans_timeout = xprt_set_retrans_timeout_def,
c627d31b 2776 .close = xs_tcp_shutdown,
9903cd1c 2777 .destroy = xs_destroy,
7196dbb0 2778 .set_connect_timeout = xs_tcp_set_connect_timeout,
262ca07d 2779 .print_stats = xs_tcp_print_stats,
d67fa4d8
JL		 2780 .enable_swap = xs_enable_swap,
2781 .disable_swap = xs_disable_swap,
4a068258 2782 .inject_disconnect = xs_inject_disconnect,
42e5c3e2
CL		 2783#ifdef CONFIG_SUNRPC_BACKCHANNEL
2784 .bc_setup = xprt_setup_bc,
76566773 2785 .bc_up = xs_tcp_bc_up,
6b26cc8c 2786 .bc_maxpayload = xs_tcp_bc_maxpayload,
42e5c3e2
CL		 2787 .bc_free_rqst = xprt_free_bc_rqst,
2788 .bc_destroy = xprt_destroy_bc,
2789#endif
a246b010
CL		 2790};
2791
4cfc7e60
RI		 2792/*
2793 * The rpc_xprt_ops for the server backchannel
2794 */
2795
d31ae254 2796static const struct rpc_xprt_ops bc_tcp_ops = {
4cfc7e60
RI		 2797 .reserve_xprt = xprt_reserve_xprt,
2798 .release_xprt = xprt_release_xprt,
84e28a30 2799 .alloc_slot = xprt_alloc_slot,
a9cde23a 2800 .free_slot = xprt_free_slot,
4cfc7e60
RI		 2801 .buf_alloc = bc_malloc,
2802 .buf_free = bc_free,
2803 .send_request = bc_send_request,
2804 .set_retrans_timeout = xprt_set_retrans_timeout_def,
2805 .close = bc_close,
2806 .destroy = bc_destroy,
2807 .print_stats = xs_tcp_print_stats,
d67fa4d8
JL		 2808 .enable_swap = xs_enable_swap,
2809 .disable_swap = xs_disable_swap,
4a068258 2810 .inject_disconnect = xs_inject_disconnect,
4cfc7e60
RI		 2811};
2812
92476850
CL		 2813static int xs_init_anyaddr(const int family, struct sockaddr *sap)
2814{
2815 static const struct sockaddr_in sin = {
2816 .sin_family = AF_INET,
2817 .sin_addr.s_addr = htonl(INADDR_ANY),
2818 };
2819 static const struct sockaddr_in6 sin6 = {
2820 .sin6_family = AF_INET6,
2821 .sin6_addr = IN6ADDR_ANY_INIT,
2822 };
2823
2824 switch (family) {
176e21ee
CL		 2825 case AF_LOCAL:
2826 break;
92476850
CL		 2827 case AF_INET:
2828 memcpy(sap, &sin, sizeof(sin));
2829 break;
2830 case AF_INET6:
2831 memcpy(sap, &sin6, sizeof(sin6));
2832 break;
2833 default:
2834 dprintk("RPC: %s: Bad address family\n", __func__);
2835 return -EAFNOSUPPORT;
2836 }
2837 return 0;
2838}
2839
3c341b0b 2840static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args,
d9ba131d
TM		 2841 unsigned int slot_table_size,
2842 unsigned int max_slot_table_size)
c8541ecd
CL		 2843{
2844 struct rpc_xprt *xprt;
ffc2e518 2845 struct sock_xprt *new;
c8541ecd 2846
96802a09 2847 if (args->addrlen > sizeof(xprt->addr)) {
46121cf7 2848 dprintk("RPC: xs_setup_xprt: address too large\n");
c8541ecd
CL		 2849 return ERR_PTR(-EBADF);
2850 }
2851
d9ba131d
TM		 2852 xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size,
2853 max_slot_table_size);
bd1722d4 2854 if (xprt == NULL) {
46121cf7
CL		 2855 dprintk("RPC: xs_setup_xprt: couldn't allocate "
2856 "rpc_xprt\n");
c8541ecd
CL		 2857 return ERR_PTR(-ENOMEM);
2858 }
c8541ecd 2859
bd1722d4 2860 new = container_of(xprt, struct sock_xprt, xprt);
edc1b01c 2861 mutex_init(&new->recv_mutex);
96802a09
FM		 2862 memcpy(&xprt->addr, args->dstaddr, args->addrlen);
2863 xprt->addrlen = args->addrlen;
d3bc9a1d 2864 if (args->srcaddr)
fbfffbd5 2865 memcpy(&new->srcaddr, args->srcaddr, args->addrlen);
92476850
CL		 2866 else {
2867 int err;
2868 err = xs_init_anyaddr(args->dstaddr->sa_family,
2869 (struct sockaddr *)&new->srcaddr);
2aa13531
SK		 2870 if (err != 0) {
2871 xprt_free(xprt);
92476850 2872 return ERR_PTR(err);
2aa13531 2873 }
92476850 2874 }
c8541ecd
CL		 2875
2876 return xprt;
2877}
2878
176e21ee
CL		 2879static const struct rpc_timeout xs_local_default_timeout = {
2880 .to_initval = 10 * HZ,
2881 .to_maxval = 10 * HZ,
2882 .to_retries = 2,
2883};
2884
2885/**
2886 * xs_setup_local - Set up transport to use an AF_LOCAL socket
2887 * @args: rpc transport creation arguments
2888 *
2889 * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP
2890 */
2891static struct rpc_xprt *xs_setup_local(struct xprt_create *args)
2892{
2893 struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr;
2894 struct sock_xprt *transport;
2895 struct rpc_xprt *xprt;
2896 struct rpc_xprt *ret;
2897
d9ba131d
TM		 2898 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
2899 xprt_max_tcp_slot_table_entries);
176e21ee
CL		 2900 if (IS_ERR(xprt))
2901 return xprt;
2902 transport = container_of(xprt, struct sock_xprt, xprt);
2903
2904 xprt->prot = 0;
2905 xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32);
2906 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
2907
2908 xprt->bind_timeout = XS_BIND_TO;
2909 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2910 xprt->idle_timeout = XS_IDLE_DISC_TO;
2911
2912 xprt->ops = &xs_local_ops;
2913 xprt->timeout = &xs_local_default_timeout;
2914
a2648094 2915 INIT_WORK(&transport->recv_worker, xs_local_data_receive_workfn);
93dc41bd
N		 2916 INIT_DELAYED_WORK(&transport->connect_worker,
2917 xs_dummy_setup_socket);
2918
176e21ee
CL		 2919 switch (sun->sun_family) {
2920 case AF_LOCAL:
2921 if (sun->sun_path[0] != '/') {
2922 dprintk("RPC: bad AF_LOCAL address: %s\n",
2923 sun->sun_path);
2924 ret = ERR_PTR(-EINVAL);
2925 goto out_err;
2926 }
2927 xprt_set_bound(xprt);
176e21ee 2928 xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL);
7073ea87
BF		 2929 ret = ERR_PTR(xs_local_setup_socket(transport));
2930 if (ret)
2931 goto out_err;
176e21ee
CL		 2932 break;
2933 default:
2934 ret = ERR_PTR(-EAFNOSUPPORT);
2935 goto out_err;
2936 }
2937
2938 dprintk("RPC: set up xprt to %s via AF_LOCAL\n",
2939 xprt->address_strings[RPC_DISPLAY_ADDR]);
2940
2941 if (try_module_get(THIS_MODULE))
2942 return xprt;
2943 ret = ERR_PTR(-EINVAL);
2944out_err:
315f3812 2945 xs_xprt_free(xprt);
176e21ee
CL		 2946 return ret;
2947}
2948
2881ae74
TM		 2949static const struct rpc_timeout xs_udp_default_timeout = {
2950 .to_initval = 5 * HZ,
2951 .to_maxval = 30 * HZ,
2952 .to_increment = 5 * HZ,
2953 .to_retries = 5,
2954};
2955
9903cd1c
CL		 2956/**
2957 * xs_setup_udp - Set up transport to use a UDP socket
96802a09 2958 * @args: rpc transport creation arguments
9903cd1c
CL		 2959 *
2960 */
483066d6 2961static struct rpc_xprt *xs_setup_udp(struct xprt_create *args)
a246b010 2962{
8f9d5b1a 2963 struct sockaddr *addr = args->dstaddr;
c8541ecd 2964 struct rpc_xprt *xprt;
c8475461 2965 struct sock_xprt *transport;
0a68b0be 2966 struct rpc_xprt *ret;
a246b010 2967
d9ba131d
TM		 2968 xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries,
2969 xprt_udp_slot_table_entries);
c8541ecd
CL		 2970 if (IS_ERR(xprt))
2971 return xprt;
c8475461 2972 transport = container_of(xprt, struct sock_xprt, xprt);
a246b010 2973
ec739ef0 2974 xprt->prot = IPPROTO_UDP;
808012fb 2975 xprt->tsh_size = 0;
a246b010
CL		 2976 /* XXX: header size can vary due to auth type, IPv6, etc. */
2977 xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
2978
03bf4b70 2979 xprt->bind_timeout = XS_BIND_TO;
03bf4b70
CL		 2980 xprt->reestablish_timeout = XS_UDP_REEST_TO;
2981 xprt->idle_timeout = XS_IDLE_DISC_TO;
a246b010 2982
262965f5 2983 xprt->ops = &xs_udp_ops;
a246b010 2984
ba7392bb 2985 xprt->timeout = &xs_udp_default_timeout;
a246b010 2986
f9b2ee71 2987 INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn);
edc1b01c
TM		 2988 INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket);
2989
8f9d5b1a
CL		 2990 switch (addr->sa_family) {
2991 case AF_INET:
2992 if (((struct sockaddr_in *)addr)->sin_port != htons(0))
2993 xprt_set_bound(xprt);
2994
9dc3b095 2995 xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP);
8f9d5b1a
CL		 2996 break;
2997 case AF_INET6:
2998 if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
2999 xprt_set_bound(xprt);
3000
9dc3b095 3001 xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6);
8f9d5b1a
CL		 3002 break;
3003 default:
0a68b0be
BF		 3004 ret = ERR_PTR(-EAFNOSUPPORT);
3005 goto out_err;
8f9d5b1a
CL		 3006 }
3007
c740eff8
CL		 3008 if (xprt_bound(xprt))
3009 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3010 xprt->address_strings[RPC_DISPLAY_ADDR],
3011 xprt->address_strings[RPC_DISPLAY_PORT],
3012 xprt->address_strings[RPC_DISPLAY_PROTO]);
3013 else
3014 dprintk("RPC: set up xprt to %s (autobind) via %s\n",
3015 xprt->address_strings[RPC_DISPLAY_ADDR],
3016 xprt->address_strings[RPC_DISPLAY_PROTO]);
edb267a6 3017
bc25571e
TT		 3018 if (try_module_get(THIS_MODULE))
3019 return xprt;
0a68b0be
BF		 3020 ret = ERR_PTR(-EINVAL);
3021out_err:
315f3812 3022 xs_xprt_free(xprt);
0a68b0be 3023 return ret;
a246b010
CL		 3024}
3025
2881ae74
TM		 3026static const struct rpc_timeout xs_tcp_default_timeout = {
3027 .to_initval = 60 * HZ,
3028 .to_maxval = 60 * HZ,
3029 .to_retries = 2,
3030};
3031
9903cd1c
CL		 3032/**
3033 * xs_setup_tcp - Set up transport to use a TCP socket
96802a09 3034 * @args: rpc transport creation arguments
9903cd1c
CL		 3035 *
3036 */
483066d6 3037static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args)
a246b010 3038{
8f9d5b1a 3039 struct sockaddr *addr = args->dstaddr;
c8541ecd 3040 struct rpc_xprt *xprt;
c8475461 3041 struct sock_xprt *transport;
0a68b0be 3042 struct rpc_xprt *ret;
b7993ceb
TM		 3043 unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries;
3044
3045 if (args->flags & XPRT_CREATE_INFINITE_SLOTS)
3046 max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT;
a246b010 3047
d9ba131d 3048 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
b7993ceb 3049 max_slot_table_size);
c8541ecd
CL		 3050 if (IS_ERR(xprt))
3051 return xprt;
c8475461 3052 transport = container_of(xprt, struct sock_xprt, xprt);
a246b010 3053
ec739ef0 3054 xprt->prot = IPPROTO_TCP;
808012fb 3055 xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32);
808012fb 3056 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
a246b010 3057
03bf4b70 3058 xprt->bind_timeout = XS_BIND_TO;
03bf4b70
CL		 3059 xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
3060 xprt->idle_timeout = XS_IDLE_DISC_TO;
a246b010 3061
262965f5 3062 xprt->ops = &xs_tcp_ops;
ba7392bb 3063 xprt->timeout = &xs_tcp_default_timeout;
a246b010 3064
3851f1cd 3065 xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
7196dbb0
TM		 3066 xprt->connect_timeout = xprt->timeout->to_initval *
3067 (xprt->timeout->to_retries + 1);
3851f1cd 3068
edc1b01c
TM		 3069 INIT_WORK(&transport->recv_worker, xs_tcp_data_receive_workfn);
3070 INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket);
3071
8f9d5b1a
CL		 3072 switch (addr->sa_family) {
3073 case AF_INET:
3074 if (((struct sockaddr_in *)addr)->sin_port != htons(0))
3075 xprt_set_bound(xprt);
3076
9dc3b095 3077 xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP);
8f9d5b1a
CL		 3078 break;
3079 case AF_INET6:
3080 if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
3081 xprt_set_bound(xprt);
3082
9dc3b095 3083 xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6);
8f9d5b1a
CL		 3084 break;
3085 default:
0a68b0be
BF		 3086 ret = ERR_PTR(-EAFNOSUPPORT);
3087 goto out_err;
8f9d5b1a
CL		 3088 }
3089
c740eff8
CL		 3090 if (xprt_bound(xprt))
3091 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3092 xprt->address_strings[RPC_DISPLAY_ADDR],
3093 xprt->address_strings[RPC_DISPLAY_PORT],
3094 xprt->address_strings[RPC_DISPLAY_PROTO]);
3095 else
3096 dprintk("RPC: set up xprt to %s (autobind) via %s\n",
3097 xprt->address_strings[RPC_DISPLAY_ADDR],
3098 xprt->address_strings[RPC_DISPLAY_PROTO]);
3099
bc25571e
TT		 3100 if (try_module_get(THIS_MODULE))
3101 return xprt;
0a68b0be
BF		 3102 ret = ERR_PTR(-EINVAL);
3103out_err:
315f3812 3104 xs_xprt_free(xprt);
0a68b0be 3105 return ret;
a246b010 3106}
282b32e1 3107
f300baba
AB		 3108/**
3109 * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket
3110 * @args: rpc transport creation arguments
3111 *
3112 */
3113static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args)
3114{
3115 struct sockaddr *addr = args->dstaddr;
3116 struct rpc_xprt *xprt;
3117 struct sock_xprt *transport;
3118 struct svc_sock *bc_sock;
0a68b0be 3119 struct rpc_xprt *ret;
f300baba 3120
d9ba131d
TM		 3121 xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
3122 xprt_tcp_slot_table_entries);
f300baba
AB		 3123 if (IS_ERR(xprt))
3124 return xprt;
3125 transport = container_of(xprt, struct sock_xprt, xprt);
3126
3127 xprt->prot = IPPROTO_TCP;
3128 xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32);
3129 xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
3130 xprt->timeout = &xs_tcp_default_timeout;
3131
3132 /* backchannel */
3133 xprt_set_bound(xprt);
3134 xprt->bind_timeout = 0;
f300baba
AB		 3135 xprt->reestablish_timeout = 0;
3136 xprt->idle_timeout = 0;
3137
f300baba
AB		 3138 xprt->ops = &bc_tcp_ops;
3139
3140 switch (addr->sa_family) {
3141 case AF_INET:
3142 xs_format_peer_addresses(xprt, "tcp",
3143 RPCBIND_NETID_TCP);
3144 break;
3145 case AF_INET6:
3146 xs_format_peer_addresses(xprt, "tcp",
3147 RPCBIND_NETID_TCP6);
3148 break;
3149 default:
0a68b0be
BF		 3150 ret = ERR_PTR(-EAFNOSUPPORT);
3151 goto out_err;
f300baba
AB		 3152 }
3153
50fa0d40
PE		 3154 dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3155 xprt->address_strings[RPC_DISPLAY_ADDR],
3156 xprt->address_strings[RPC_DISPLAY_PORT],
3157 xprt->address_strings[RPC_DISPLAY_PROTO]);
f300baba 3158
99de8ea9
BF		 3159 /*
3160 * Once we've associated a backchannel xprt with a connection,
28303ca3
WM		 3161 * we want to keep it around as long as the connection lasts,
3162 * in case we need to start using it for a backchannel again;
3163 * this reference won't be dropped until bc_xprt is destroyed.
99de8ea9
BF		 3164 */
3165 xprt_get(xprt);
3166 args->bc_xprt->xpt_bc_xprt = xprt;
3167 xprt->bc_xprt = args->bc_xprt;
3168 bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt);
3169 transport->sock = bc_sock->sk_sock;
3170 transport->inet = bc_sock->sk_sk;
3171
f300baba
AB		 3172 /*
3173 * Since we don't want connections for the backchannel, we set
3174 * the xprt status to connected
3175 */
3176 xprt_set_connected(xprt);
3177
f300baba
AB		 3178 if (try_module_get(THIS_MODULE))
3179 return xprt;
642aab58
KM		 3180
3181 args->bc_xprt->xpt_bc_xprt = NULL;
39a9beab 3182 args->bc_xprt->xpt_bc_xps = NULL;
99de8ea9 3183 xprt_put(xprt);
0a68b0be
BF		 3184 ret = ERR_PTR(-EINVAL);
3185out_err:
315f3812 3186 xs_xprt_free(xprt);
0a68b0be 3187 return ret;
f300baba
AB		 3188}
3189
176e21ee
CL		 3190static struct xprt_class xs_local_transport = {
3191 .list = LIST_HEAD_INIT(xs_local_transport.list),
3192 .name = "named UNIX socket",
3193 .owner = THIS_MODULE,
3194 .ident = XPRT_TRANSPORT_LOCAL,
3195 .setup = xs_setup_local,
3196};
3197
bc25571e
TT		 3198static struct xprt_class xs_udp_transport = {
3199 .list = LIST_HEAD_INIT(xs_udp_transport.list),
3200 .name = "udp",
3201 .owner = THIS_MODULE,
f300baba 3202 .ident = XPRT_TRANSPORT_UDP,
bc25571e
TT		 3203 .setup = xs_setup_udp,
3204};
3205
3206static struct xprt_class xs_tcp_transport = {
3207 .list = LIST_HEAD_INIT(xs_tcp_transport.list),
3208 .name = "tcp",
3209 .owner = THIS_MODULE,
f300baba 3210 .ident = XPRT_TRANSPORT_TCP,
bc25571e
TT		 3211 .setup = xs_setup_tcp,
3212};
3213
f300baba
AB		 3214static struct xprt_class xs_bc_tcp_transport = {
3215 .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list),
3216 .name = "tcp NFSv4.1 backchannel",
3217 .owner = THIS_MODULE,
3218 .ident = XPRT_TRANSPORT_BC_TCP,
3219 .setup = xs_setup_bc_tcp,
3220};
3221
282b32e1 3222/**
bc25571e 3223 * init_socket_xprt - set up xprtsock's sysctls, register with RPC client
282b32e1
CL		 3224 *
3225 */
3226int init_socket_xprt(void)
3227{
f895b252 3228#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
2b1bec5f 3229 if (!sunrpc_table_header)
0b4d4147 3230 sunrpc_table_header = register_sysctl_table(sunrpc_table);
fbf76683
CL		 3231#endif
3232
176e21ee 3233 xprt_register_transport(&xs_local_transport);
bc25571e
TT		 3234 xprt_register_transport(&xs_udp_transport);
3235 xprt_register_transport(&xs_tcp_transport);
f300baba 3236 xprt_register_transport(&xs_bc_tcp_transport);
bc25571e 3237
282b32e1
CL		 3238 return 0;
3239}
3240
3241/**
bc25571e 3242 * cleanup_socket_xprt - remove xprtsock's sysctls, unregister
282b32e1
CL		 3243 *
3244 */
3245void cleanup_socket_xprt(void)
3246{
f895b252 3247#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
fbf76683
CL		 3248 if (sunrpc_table_header) {
3249 unregister_sysctl_table(sunrpc_table_header);
3250 sunrpc_table_header = NULL;
3251 }
3252#endif
bc25571e 3253
176e21ee 3254 xprt_unregister_transport(&xs_local_transport);
bc25571e
TT		 3255 xprt_unregister_transport(&xs_udp_transport);
3256 xprt_unregister_transport(&xs_tcp_transport);
f300baba 3257 xprt_unregister_transport(&xs_bc_tcp_transport);
282b32e1 3258}
cbf11071 3259
9bbb9e5a
RR		 3260static int param_set_uint_minmax(const char *val,
3261 const struct kernel_param *kp,
cbf11071
TM		 3262 unsigned int min, unsigned int max)
3263{
00cfaa94 3264 unsigned int num;
cbf11071
TM		 3265 int ret;
3266
3267 if (!val)
3268 return -EINVAL;
00cfaa94 3269 ret = kstrtouint(val, 0, &num);
9761a246
DC		 3270 if (ret)
3271 return ret;
3272 if (num < min || num > max)
cbf11071
TM		 3273 return -EINVAL;
3274 *((unsigned int *)kp->arg) = num;
3275 return 0;
3276}
3277
9bbb9e5a 3278static int param_set_portnr(const char *val, const struct kernel_param *kp)
cbf11071 3279{
ffb6ca33
FS		 3280 if (kp->arg == &xprt_min_resvport)
3281 return param_set_uint_minmax(val, kp,
cbf11071 3282 RPC_MIN_RESVPORT,
ffb6ca33
FS		 3283 xprt_max_resvport);
3284 return param_set_uint_minmax(val, kp,
3285 xprt_min_resvport,
cbf11071
TM		 3286 RPC_MAX_RESVPORT);
3287}
3288
9c27847d 3289static const struct kernel_param_ops param_ops_portnr = {
9bbb9e5a
RR		 3290 .set = param_set_portnr,
3291 .get = param_get_uint,
3292};
3293
cbf11071
TM		 3294#define param_check_portnr(name, p) \
3295 __param_check(name, p, unsigned int);
3296
3297module_param_named(min_resvport, xprt_min_resvport, portnr, 0644);
3298module_param_named(max_resvport, xprt_max_resvport, portnr, 0644);
3299
9bbb9e5a
RR		 3300static int param_set_slot_table_size(const char *val,
3301 const struct kernel_param *kp)
cbf11071
TM		 3302{
3303 return param_set_uint_minmax(val, kp,
3304 RPC_MIN_SLOT_TABLE,
3305 RPC_MAX_SLOT_TABLE);
3306}
3307
9c27847d 3308static const struct kernel_param_ops param_ops_slot_table_size = {
9bbb9e5a
RR		 3309 .set = param_set_slot_table_size,
3310 .get = param_get_uint,
3311};
3312
cbf11071
TM		 3313#define param_check_slot_table_size(name, p) \
3314 __param_check(name, p, unsigned int);
3315
d9ba131d
TM		 3316static int param_set_max_slot_table_size(const char *val,
3317 const struct kernel_param *kp)
3318{
3319 return param_set_uint_minmax(val, kp,
3320 RPC_MIN_SLOT_TABLE,
3321 RPC_MAX_SLOT_TABLE_LIMIT);
3322}
3323
9c27847d 3324static const struct kernel_param_ops param_ops_max_slot_table_size = {
d9ba131d
TM		 3325 .set = param_set_max_slot_table_size,
3326 .get = param_get_uint,
3327};
3328
3329#define param_check_max_slot_table_size(name, p) \
3330 __param_check(name, p, unsigned int);
3331
cbf11071
TM		 3332module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries,
3333 slot_table_size, 0644);
d9ba131d
TM		 3334module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries,
3335 max_slot_table_size, 0644);
cbf11071
TM		 3336module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries,
3337 slot_table_size, 0644);
Linux 6.x block layer and io_uring tree(s)