Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1da177e4 LT |
2 | /* |
3 | * linux/net/sunrpc/svcsock.c | |
4 | * | |
5 | * These are the RPC server socket internals. | |
6 | * | |
7 | * The server scheduling algorithm does not always distribute the load | |
8 | * evenly when servicing a single client. May need to modify the | |
f6150c3c | 9 | * svc_xprt_enqueue procedure... |
1da177e4 LT |
10 | * |
11 | * TCP support is largely untested and may be a little slow. The problem | |
12 | * is that we currently do two separate recvfrom's, one for the 4-byte | |
13 | * record length, and the second for the actual record. This could possibly | |
14 | * be improved by always reading a minimum size of around 100 bytes and | |
15 | * tucking any superfluous bytes away in a temporary store. Still, that | |
16 | * leaves write requests out in the rain. An alternative may be to peek at | |
17 | * the first skb in the queue, and if it matches the next TCP sequence | |
18 | * number, to extract the record marker. Yuck. | |
19 | * | |
20 | * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> | |
21 | */ | |
22 | ||
172589cc | 23 | #include <linux/kernel.h> |
1da177e4 | 24 | #include <linux/sched.h> |
3a9a231d | 25 | #include <linux/module.h> |
1da177e4 LT |
26 | #include <linux/errno.h> |
27 | #include <linux/fcntl.h> | |
28 | #include <linux/net.h> | |
29 | #include <linux/in.h> | |
30 | #include <linux/inet.h> | |
31 | #include <linux/udp.h> | |
91483c4b | 32 | #include <linux/tcp.h> |
1da177e4 LT |
33 | #include <linux/unistd.h> |
34 | #include <linux/slab.h> | |
35 | #include <linux/netdevice.h> | |
36 | #include <linux/skbuff.h> | |
b41b66d6 | 37 | #include <linux/file.h> |
7dfb7103 | 38 | #include <linux/freezer.h> |
2eb2b935 CL |
39 | #include <linux/bvec.h> |
40 | ||
1da177e4 LT |
41 | #include <net/sock.h> |
42 | #include <net/checksum.h> | |
43 | #include <net/ip.h> | |
b92503b2 | 44 | #include <net/ipv6.h> |
850cbadd | 45 | #include <net/udp.h> |
b7872fe8 | 46 | #include <net/tcp.h> |
c752f073 | 47 | #include <net/tcp_states.h> |
6a7eccef | 48 | #include <net/tls_prot.h> |
b3cbf98e | 49 | #include <net/handshake.h> |
7c0f6ba6 | 50 | #include <linux/uaccess.h> |
becd2014 | 51 | #include <linux/highmem.h> |
1da177e4 | 52 | #include <asm/ioctls.h> |
b3cbf98e | 53 | #include <linux/key.h> |
1da177e4 LT |
54 | |
55 | #include <linux/sunrpc/types.h> | |
ad06e4bd | 56 | #include <linux/sunrpc/clnt.h> |
1da177e4 | 57 | #include <linux/sunrpc/xdr.h> |
c0401ea0 | 58 | #include <linux/sunrpc/msg_prot.h> |
1da177e4 LT |
59 | #include <linux/sunrpc/svcsock.h> |
60 | #include <linux/sunrpc/stats.h> | |
4cfc7e60 | 61 | #include <linux/sunrpc/xprt.h> |
1da177e4 | 62 | |
40e0b090 | 63 | #include <trace/events/sock.h> |
998024de CL |
64 | #include <trace/events/sunrpc.h> |
65 | ||
9e55eef4 | 66 | #include "socklib.h" |
177e4f99 HS |
67 | #include "sunrpc.h" |
68 | ||
360d8738 | 69 | #define RPCDBG_FACILITY RPCDBG_SVCXPRT |
1da177e4 | 70 | |
b3cbf98e CL |
71 | /* To-do: to avoid tying up an nfsd thread while waiting for a |
72 | * handshake request, the request could instead be deferred. | |
73 | */ | |
74 | enum { | |
75 | SVC_HANDSHAKE_TO = 5U * HZ | |
76 | }; | |
1da177e4 LT |
77 | |
78 | static struct svc_sock *svc_setup_socket(struct svc_serv *, struct socket *, | |
72c35376 | 79 | int flags); |
1da177e4 LT |
80 | static int svc_udp_recvfrom(struct svc_rqst *); |
81 | static int svc_udp_sendto(struct svc_rqst *); | |
755cceab | 82 | static void svc_sock_detach(struct svc_xprt *); |
69b6ba37 | 83 | static void svc_tcp_sock_detach(struct svc_xprt *); |
755cceab | 84 | static void svc_sock_free(struct svc_xprt *); |
1da177e4 | 85 | |
b700cbb1 | 86 | static struct svc_xprt *svc_create_socket(struct svc_serv *, int, |
62832c03 PE |
87 | struct net *, struct sockaddr *, |
88 | int, int); | |
ed07536e PZ |
89 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
90 | static struct lock_class_key svc_key[2]; | |
91 | static struct lock_class_key svc_slock_key[2]; | |
92 | ||
0f0257ea | 93 | static void svc_reclassify_socket(struct socket *sock) |
ed07536e PZ |
94 | { |
95 | struct sock *sk = sock->sk; | |
1b7a1819 | 96 | |
fafc4e1e | 97 | if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) |
1b7a1819 WAA |
98 | return; |
99 | ||
ed07536e PZ |
100 | switch (sk->sk_family) { |
101 | case AF_INET: | |
102 | sock_lock_init_class_and_name(sk, "slock-AF_INET-NFSD", | |
def13d74 TT |
103 | &svc_slock_key[0], |
104 | "sk_xprt.xpt_lock-AF_INET-NFSD", | |
105 | &svc_key[0]); | |
ed07536e PZ |
106 | break; |
107 | ||
108 | case AF_INET6: | |
109 | sock_lock_init_class_and_name(sk, "slock-AF_INET6-NFSD", | |
def13d74 TT |
110 | &svc_slock_key[1], |
111 | "sk_xprt.xpt_lock-AF_INET6-NFSD", | |
112 | &svc_key[1]); | |
ed07536e PZ |
113 | break; |
114 | ||
115 | default: | |
116 | BUG(); | |
117 | } | |
118 | } | |
119 | #else | |
0f0257ea | 120 | static void svc_reclassify_socket(struct socket *sock) |
ed07536e PZ |
121 | { |
122 | } | |
123 | #endif | |
124 | ||
cca557a5 | 125 | /** |
948f072a N |
126 | * svc_tcp_release_ctxt - Release transport-related resources |
127 | * @xprt: the transport which owned the context | |
128 | * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt | |
cca557a5 | 129 | * |
1da177e4 | 130 | */ |
948f072a | 131 | static void svc_tcp_release_ctxt(struct svc_xprt *xprt, void *ctxt) |
1da177e4 | 132 | { |
1da177e4 LT |
133 | } |
134 | ||
cca557a5 | 135 | /** |
948f072a N |
136 | * svc_udp_release_ctxt - Release transport-related resources |
137 | * @xprt: the transport which owned the context | |
138 | * @ctxt: the context from rqstp->rq_xprt_ctxt or dr->xprt_ctxt | |
cca557a5 CL |
139 | * |
140 | */ | |
948f072a | 141 | static void svc_udp_release_ctxt(struct svc_xprt *xprt, void *ctxt) |
850cbadd | 142 | { |
948f072a | 143 | struct sk_buff *skb = ctxt; |
850cbadd | 144 | |
948f072a | 145 | if (skb) |
850cbadd | 146 | consume_skb(skb); |
850cbadd PA |
147 | } |
148 | ||
b92503b2 CL |
149 | union svc_pktinfo_u { |
150 | struct in_pktinfo pkti; | |
b92503b2 | 151 | struct in6_pktinfo pkti6; |
b92503b2 | 152 | }; |
bc375ea7 DM |
153 | #define SVC_PKTINFO_SPACE \ |
154 | CMSG_SPACE(sizeof(union svc_pktinfo_u)) | |
b92503b2 CL |
155 | |
156 | static void svc_set_cmsg_data(struct svc_rqst *rqstp, struct cmsghdr *cmh) | |
157 | { | |
57b1d3ba TT |
158 | struct svc_sock *svsk = |
159 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); | |
160 | switch (svsk->sk_sk->sk_family) { | |
b92503b2 CL |
161 | case AF_INET: { |
162 | struct in_pktinfo *pki = CMSG_DATA(cmh); | |
163 | ||
164 | cmh->cmsg_level = SOL_IP; | |
165 | cmh->cmsg_type = IP_PKTINFO; | |
166 | pki->ipi_ifindex = 0; | |
849a1cf1 MJ |
167 | pki->ipi_spec_dst.s_addr = |
168 | svc_daddr_in(rqstp)->sin_addr.s_addr; | |
b92503b2 CL |
169 | cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); |
170 | } | |
171 | break; | |
5a05ed73 | 172 | |
b92503b2 CL |
173 | case AF_INET6: { |
174 | struct in6_pktinfo *pki = CMSG_DATA(cmh); | |
849a1cf1 | 175 | struct sockaddr_in6 *daddr = svc_daddr_in6(rqstp); |
b92503b2 CL |
176 | |
177 | cmh->cmsg_level = SOL_IPV6; | |
178 | cmh->cmsg_type = IPV6_PKTINFO; | |
849a1cf1 | 179 | pki->ipi6_ifindex = daddr->sin6_scope_id; |
4e3fd7a0 | 180 | pki->ipi6_addr = daddr->sin6_addr; |
b92503b2 CL |
181 | cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); |
182 | } | |
183 | break; | |
b92503b2 | 184 | } |
b92503b2 CL |
185 | } |
186 | ||
03493bca CL |
187 | static int svc_sock_result_payload(struct svc_rqst *rqstp, unsigned int offset, |
188 | unsigned int length) | |
41205539 CL |
189 | { |
190 | return 0; | |
191 | } | |
192 | ||
80212d59 N |
193 | /* |
194 | * Report socket names for nfsdfs | |
195 | */ | |
e7942b9f | 196 | static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining) |
80212d59 | 197 | { |
017cb47f CL |
198 | const struct sock *sk = svsk->sk_sk; |
199 | const char *proto_name = sk->sk_protocol == IPPROTO_UDP ? | |
200 | "udp" : "tcp"; | |
80212d59 N |
201 | int len; |
202 | ||
017cb47f | 203 | switch (sk->sk_family) { |
e7942b9f CL |
204 | case PF_INET: |
205 | len = snprintf(buf, remaining, "ipv4 %s %pI4 %d\n", | |
017cb47f | 206 | proto_name, |
c720c7e8 ED |
207 | &inet_sk(sk)->inet_rcv_saddr, |
208 | inet_sk(sk)->inet_num); | |
80212d59 | 209 | break; |
c2bb06db | 210 | #if IS_ENABLED(CONFIG_IPV6) |
58de2f86 CL |
211 | case PF_INET6: |
212 | len = snprintf(buf, remaining, "ipv6 %s %pI6 %d\n", | |
017cb47f | 213 | proto_name, |
efe4208f | 214 | &sk->sk_v6_rcv_saddr, |
c720c7e8 | 215 | inet_sk(sk)->inet_num); |
80212d59 | 216 | break; |
c2bb06db | 217 | #endif |
80212d59 | 218 | default: |
e7942b9f | 219 | len = snprintf(buf, remaining, "*unknown-%d*\n", |
017cb47f | 220 | sk->sk_family); |
80212d59 | 221 | } |
e7942b9f CL |
222 | |
223 | if (len >= remaining) { | |
224 | *buf = '\0'; | |
225 | return -ENAMETOOLONG; | |
80212d59 N |
226 | } |
227 | return len; | |
228 | } | |
229 | ||
5e052dda | 230 | static int |
39067dda | 231 | svc_tcp_sock_process_cmsg(struct socket *sock, struct msghdr *msg, |
5e052dda CL |
232 | struct cmsghdr *cmsg, int ret) |
233 | { | |
39067dda CL |
234 | u8 content_type = tls_get_record_type(sock->sk, cmsg); |
235 | u8 level, description; | |
236 | ||
237 | switch (content_type) { | |
238 | case 0: | |
239 | break; | |
240 | case TLS_RECORD_TYPE_DATA: | |
241 | /* TLS sets EOR at the end of each application data | |
242 | * record, even though there might be more frames | |
243 | * waiting to be decrypted. | |
244 | */ | |
245 | msg->msg_flags &= ~MSG_EOR; | |
246 | break; | |
247 | case TLS_RECORD_TYPE_ALERT: | |
248 | tls_alert_recv(sock->sk, msg, &level, &description); | |
249 | ret = (level == TLS_ALERT_LEVEL_FATAL) ? | |
250 | -ENOTCONN : -EAGAIN; | |
251 | break; | |
252 | default: | |
253 | /* discard this record type */ | |
254 | ret = -EAGAIN; | |
5e052dda CL |
255 | } |
256 | return ret; | |
257 | } | |
258 | ||
259 | static int | |
260 | svc_tcp_sock_recv_cmsg(struct svc_sock *svsk, struct msghdr *msg) | |
261 | { | |
262 | union { | |
263 | struct cmsghdr cmsg; | |
264 | u8 buf[CMSG_SPACE(sizeof(u8))]; | |
265 | } u; | |
39067dda | 266 | struct socket *sock = svsk->sk_sock; |
5e052dda CL |
267 | int ret; |
268 | ||
269 | msg->msg_control = &u; | |
270 | msg->msg_controllen = sizeof(u); | |
39067dda | 271 | ret = sock_recvmsg(sock, msg, MSG_DONTWAIT); |
5e052dda | 272 | if (unlikely(msg->msg_controllen != sizeof(u))) |
39067dda | 273 | ret = svc_tcp_sock_process_cmsg(sock, msg, &u.cmsg, ret); |
5e052dda CL |
274 | return ret; |
275 | } | |
276 | ||
ca07eda3 CL |
277 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE |
278 | static void svc_flush_bvec(const struct bio_vec *bvec, size_t size, size_t seek) | |
279 | { | |
280 | struct bvec_iter bi = { | |
13a9a9d7 | 281 | .bi_size = size + seek, |
ca07eda3 CL |
282 | }; |
283 | struct bio_vec bv; | |
284 | ||
285 | bvec_iter_advance(bvec, &bi, seek & PAGE_MASK); | |
286 | for_each_bvec(bv, bvec, bi, bi) | |
287 | flush_dcache_page(bv.bv_page); | |
288 | } | |
289 | #else | |
290 | static inline void svc_flush_bvec(const struct bio_vec *bvec, size_t size, | |
291 | size_t seek) | |
292 | { | |
293 | } | |
294 | #endif | |
295 | ||
1da177e4 | 296 | /* |
ca07eda3 CL |
297 | * Read from @rqstp's transport socket. The incoming message fills whole |
298 | * pages in @rqstp's rq_pages array until the last page of the message | |
299 | * has been received into a partial page. | |
1da177e4 | 300 | */ |
ca07eda3 CL |
301 | static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen, |
302 | size_t seek) | |
1da177e4 | 303 | { |
57b1d3ba TT |
304 | struct svc_sock *svsk = |
305 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); | |
ca07eda3 | 306 | struct bio_vec *bvec = rqstp->rq_bvec; |
4c8e5537 | 307 | struct msghdr msg = { NULL }; |
ca07eda3 | 308 | unsigned int i; |
4c8e5537 | 309 | ssize_t len; |
ca07eda3 | 310 | size_t t; |
1da177e4 | 311 | |
f8d1ff47 | 312 | clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); |
ca07eda3 | 313 | |
9088151f CH |
314 | for (i = 0, t = 0; t < buflen; i++, t += PAGE_SIZE) |
315 | bvec_set_page(&bvec[i], rqstp->rq_pages[i], PAGE_SIZE, 0); | |
ca07eda3 CL |
316 | rqstp->rq_respages = &rqstp->rq_pages[i]; |
317 | rqstp->rq_next_page = rqstp->rq_respages + 1; | |
318 | ||
de4eda9d | 319 | iov_iter_bvec(&msg.msg_iter, ITER_DEST, bvec, i, buflen); |
ca07eda3 CL |
320 | if (seek) { |
321 | iov_iter_advance(&msg.msg_iter, seek); | |
322 | buflen -= seek; | |
4c8e5537 | 323 | } |
5e052dda | 324 | len = svc_tcp_sock_recv_cmsg(svsk, &msg); |
ca07eda3 CL |
325 | if (len > 0) |
326 | svc_flush_bvec(bvec, len, seek); | |
327 | ||
f8d1ff47 TM |
328 | /* If we read a full record, then assume there may be more |
329 | * data to read (stream based sockets only!) | |
330 | */ | |
331 | if (len == buflen) | |
332 | set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); | |
1da177e4 | 333 | |
1da177e4 LT |
334 | return len; |
335 | } | |
336 | ||
337 | /* | |
338 | * Set socket snd and rcv buffer lengths | |
339 | */ | |
b7e5034c | 340 | static void svc_sock_setbufsize(struct svc_sock *svsk, unsigned int nreqs) |
1da177e4 | 341 | { |
b7e5034c BF |
342 | unsigned int max_mesg = svsk->sk_xprt.xpt_server->sv_max_mesg; |
343 | struct socket *sock = svsk->sk_sock; | |
344 | ||
345 | nreqs = min(nreqs, INT_MAX / 2 / max_mesg); | |
346 | ||
1da177e4 | 347 | lock_sock(sock->sk); |
b7e5034c BF |
348 | sock->sk->sk_sndbuf = nreqs * max_mesg * 2; |
349 | sock->sk->sk_rcvbuf = nreqs * max_mesg * 2; | |
47fcb03f | 350 | sock->sk->sk_write_space(sock->sk); |
1da177e4 | 351 | release_sock(sock->sk); |
1da177e4 | 352 | } |
16e4d93f | 353 | |
989f881e | 354 | static void svc_sock_secure_port(struct svc_rqst *rqstp) |
16e4d93f | 355 | { |
989f881e | 356 | if (svc_port_is_privileged(svc_addr(rqstp))) |
7827c81f | 357 | set_bit(RQ_SECURE, &rqstp->rq_flags); |
989f881e | 358 | else |
7827c81f | 359 | clear_bit(RQ_SECURE, &rqstp->rq_flags); |
16e4d93f CL |
360 | } |
361 | ||
1da177e4 LT |
362 | /* |
363 | * INET callback when data has been received on the socket. | |
364 | */ | |
fa9251af | 365 | static void svc_data_ready(struct sock *sk) |
1da177e4 | 366 | { |
939bb7ef | 367 | struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; |
1da177e4 | 368 | |
40e0b090 PY |
369 | trace_sk_data_ready(sk); |
370 | ||
939bb7ef | 371 | if (svsk) { |
eebe53e8 VL |
372 | /* Refer to svc_setup_socket() for details. */ |
373 | rmb(); | |
fa9251af | 374 | svsk->sk_odata(sk); |
998024de | 375 | trace_svcsock_data_ready(&svsk->sk_xprt, 0); |
b3cbf98e CL |
376 | if (test_bit(XPT_HANDSHAKE, &svsk->sk_xprt.xpt_flags)) |
377 | return; | |
4720b070 TM |
378 | if (!test_and_set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags)) |
379 | svc_xprt_enqueue(&svsk->sk_xprt); | |
939bb7ef | 380 | } |
1da177e4 LT |
381 | } |
382 | ||
383 | /* | |
384 | * INET callback when space is newly available on the socket. | |
385 | */ | |
0f0257ea | 386 | static void svc_write_space(struct sock *sk) |
1da177e4 LT |
387 | { |
388 | struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data); | |
389 | ||
390 | if (svsk) { | |
eebe53e8 VL |
391 | /* Refer to svc_setup_socket() for details. */ |
392 | rmb(); | |
998024de | 393 | trace_svcsock_write_space(&svsk->sk_xprt, 0); |
fa9251af | 394 | svsk->sk_owspace(sk); |
f6150c3c | 395 | svc_xprt_enqueue(&svsk->sk_xprt); |
1da177e4 | 396 | } |
1da177e4 LT |
397 | } |
398 | ||
c7fb3f06 TM |
399 | static int svc_tcp_has_wspace(struct svc_xprt *xprt) |
400 | { | |
637600f3 | 401 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
c7fb3f06 TM |
402 | |
403 | if (test_bit(XPT_LISTENER, &xprt->xpt_flags)) | |
404 | return 1; | |
637600f3 | 405 | return !test_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); |
51877680 TM |
406 | } |
407 | ||
ea08e392 SM |
408 | static void svc_tcp_kill_temp_xprt(struct svc_xprt *xprt) |
409 | { | |
c433594c CH |
410 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
411 | ||
412 | sock_no_linger(svsk->sk_sock->sk); | |
ea08e392 SM |
413 | } |
414 | ||
b3cbf98e CL |
415 | /** |
416 | * svc_tcp_handshake_done - Handshake completion handler | |
417 | * @data: address of xprt to wake | |
418 | * @status: status of handshake | |
419 | * @peerid: serial number of key containing the remote peer's identity | |
420 | * | |
421 | * If a security policy is specified as an export option, we don't | |
422 | * have a specific export here to check. So we set a "TLS session | |
423 | * is present" flag on the xprt and let an upper layer enforce local | |
424 | * security policy. | |
425 | */ | |
426 | static void svc_tcp_handshake_done(void *data, int status, key_serial_t peerid) | |
427 | { | |
428 | struct svc_xprt *xprt = data; | |
429 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
430 | ||
431 | if (!status) { | |
432 | if (peerid != TLS_NO_PEERID) | |
433 | set_bit(XPT_PEER_AUTH, &xprt->xpt_flags); | |
434 | set_bit(XPT_TLS_SESSION, &xprt->xpt_flags); | |
435 | } | |
436 | clear_bit(XPT_HANDSHAKE, &xprt->xpt_flags); | |
437 | complete_all(&svsk->sk_handshake_done); | |
438 | } | |
439 | ||
440 | /** | |
441 | * svc_tcp_handshake - Perform a transport-layer security handshake | |
442 | * @xprt: connected transport endpoint | |
443 | * | |
444 | */ | |
445 | static void svc_tcp_handshake(struct svc_xprt *xprt) | |
446 | { | |
447 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
448 | struct sock *sk = svsk->sk_sock->sk; | |
449 | struct tls_handshake_args args = { | |
450 | .ta_sock = svsk->sk_sock, | |
451 | .ta_done = svc_tcp_handshake_done, | |
452 | .ta_data = xprt, | |
453 | }; | |
454 | int ret; | |
455 | ||
456 | trace_svc_tls_upcall(xprt); | |
457 | ||
458 | clear_bit(XPT_TLS_SESSION, &xprt->xpt_flags); | |
459 | init_completion(&svsk->sk_handshake_done); | |
460 | ||
461 | ret = tls_server_hello_x509(&args, GFP_KERNEL); | |
462 | if (ret) { | |
463 | trace_svc_tls_not_started(xprt); | |
464 | goto out_failed; | |
465 | } | |
466 | ||
467 | ret = wait_for_completion_interruptible_timeout(&svsk->sk_handshake_done, | |
468 | SVC_HANDSHAKE_TO); | |
469 | if (ret <= 0) { | |
470 | if (tls_handshake_cancel(sk)) { | |
471 | trace_svc_tls_timed_out(xprt); | |
472 | goto out_close; | |
473 | } | |
474 | } | |
475 | ||
476 | if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) { | |
477 | trace_svc_tls_unavailable(xprt); | |
478 | goto out_close; | |
479 | } | |
480 | ||
481 | /* Mark the transport ready in case the remote sent RPC | |
482 | * traffic before the kernel received the handshake | |
483 | * completion downcall. | |
484 | */ | |
485 | set_bit(XPT_DATA, &xprt->xpt_flags); | |
486 | svc_xprt_enqueue(xprt); | |
487 | return; | |
488 | ||
489 | out_close: | |
490 | set_bit(XPT_CLOSE, &xprt->xpt_flags); | |
491 | out_failed: | |
492 | clear_bit(XPT_HANDSHAKE, &xprt->xpt_flags); | |
493 | set_bit(XPT_DATA, &xprt->xpt_flags); | |
494 | svc_xprt_enqueue(xprt); | |
495 | } | |
496 | ||
7702ce40 CL |
497 | /* |
498 | * See net/ipv6/ip_sockglue.c : ip_cmsg_recv_pktinfo | |
499 | */ | |
500 | static int svc_udp_get_dest_address4(struct svc_rqst *rqstp, | |
501 | struct cmsghdr *cmh) | |
502 | { | |
503 | struct in_pktinfo *pki = CMSG_DATA(cmh); | |
849a1cf1 MJ |
504 | struct sockaddr_in *daddr = svc_daddr_in(rqstp); |
505 | ||
7702ce40 CL |
506 | if (cmh->cmsg_type != IP_PKTINFO) |
507 | return 0; | |
849a1cf1 MJ |
508 | |
509 | daddr->sin_family = AF_INET; | |
510 | daddr->sin_addr.s_addr = pki->ipi_spec_dst.s_addr; | |
7702ce40 CL |
511 | return 1; |
512 | } | |
513 | ||
514 | /* | |
73df66f8 | 515 | * See net/ipv6/datagram.c : ip6_datagram_recv_ctl |
7702ce40 CL |
516 | */ |
517 | static int svc_udp_get_dest_address6(struct svc_rqst *rqstp, | |
518 | struct cmsghdr *cmh) | |
519 | { | |
520 | struct in6_pktinfo *pki = CMSG_DATA(cmh); | |
849a1cf1 MJ |
521 | struct sockaddr_in6 *daddr = svc_daddr_in6(rqstp); |
522 | ||
7702ce40 CL |
523 | if (cmh->cmsg_type != IPV6_PKTINFO) |
524 | return 0; | |
849a1cf1 MJ |
525 | |
526 | daddr->sin6_family = AF_INET6; | |
4e3fd7a0 | 527 | daddr->sin6_addr = pki->ipi6_addr; |
849a1cf1 | 528 | daddr->sin6_scope_id = pki->ipi6_ifindex; |
7702ce40 CL |
529 | return 1; |
530 | } | |
531 | ||
9dbc240f TT |
532 | /* |
533 | * Copy the UDP datagram's destination address to the rqstp structure. | |
534 | * The 'destination' address in this case is the address to which the | |
535 | * peer sent the datagram, i.e. our local address. For multihomed | |
536 | * hosts, this can change from msg to msg. Note that only the IP | |
537 | * address changes, the port number should remain the same. | |
538 | */ | |
7702ce40 CL |
539 | static int svc_udp_get_dest_address(struct svc_rqst *rqstp, |
540 | struct cmsghdr *cmh) | |
95756482 | 541 | { |
7702ce40 CL |
542 | switch (cmh->cmsg_level) { |
543 | case SOL_IP: | |
544 | return svc_udp_get_dest_address4(rqstp, cmh); | |
545 | case SOL_IPV6: | |
546 | return svc_udp_get_dest_address6(rqstp, cmh); | |
95756482 | 547 | } |
7702ce40 CL |
548 | |
549 | return 0; | |
95756482 CL |
550 | } |
551 | ||
fff1ebb2 CL |
552 | /** |
553 | * svc_udp_recvfrom - Receive a datagram from a UDP socket. | |
554 | * @rqstp: request structure into which to receive an RPC Call | |
555 | * | |
556 | * Called in a loop when XPT_DATA has been set. | |
557 | * | |
558 | * Returns: | |
559 | * On success, the number of bytes in a received RPC Call, or | |
560 | * %0 if a complete RPC Call message was not ready to return | |
1da177e4 | 561 | */ |
0f0257ea | 562 | static int svc_udp_recvfrom(struct svc_rqst *rqstp) |
1da177e4 | 563 | { |
57b1d3ba TT |
564 | struct svc_sock *svsk = |
565 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); | |
bb5cf160 | 566 | struct svc_serv *serv = svsk->sk_xprt.xpt_server; |
1da177e4 | 567 | struct sk_buff *skb; |
bc375ea7 DM |
568 | union { |
569 | struct cmsghdr hdr; | |
570 | long all[SVC_PKTINFO_SPACE / sizeof(long)]; | |
571 | } buffer; | |
572 | struct cmsghdr *cmh = &buffer.hdr; | |
7a37f578 N |
573 | struct msghdr msg = { |
574 | .msg_name = svc_addr(rqstp), | |
575 | .msg_control = cmh, | |
576 | .msg_controllen = sizeof(buffer), | |
577 | .msg_flags = MSG_DONTWAIT, | |
578 | }; | |
abc5c44d CL |
579 | size_t len; |
580 | int err; | |
1da177e4 | 581 | |
02fc6c36 | 582 | if (test_and_clear_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags)) |
1da177e4 LT |
583 | /* udp sockets need large rcvbuf as all pending |
584 | * requests are still in that buffer. sndbuf must | |
585 | * also be large enough that there is enough space | |
3262c816 GB |
586 | * for one reply per thread. We count all threads |
587 | * rather than threads in a particular pool, which | |
588 | * provides an upper bound on the number of threads | |
589 | * which will access the socket. | |
1da177e4 | 590 | */ |
b7e5034c | 591 | svc_sock_setbufsize(svsk, serv->sv_nrthreads + 3); |
1da177e4 | 592 | |
02fc6c36 | 593 | clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); |
05ed690e N |
594 | err = kernel_recvmsg(svsk->sk_sock, &msg, NULL, |
595 | 0, 0, MSG_PEEK | MSG_DONTWAIT); | |
fff1ebb2 CL |
596 | if (err < 0) |
597 | goto out_recv_err; | |
ec095263 | 598 | skb = skb_recv_udp(svsk->sk_sk, MSG_DONTWAIT, &err); |
fff1ebb2 CL |
599 | if (!skb) |
600 | goto out_recv_err; | |
601 | ||
9dbc240f | 602 | len = svc_addr_len(svc_addr(rqstp)); |
9dbc240f | 603 | rqstp->rq_addrlen = len; |
2456e855 | 604 | if (skb->tstamp == 0) { |
b7aa0bf7 | 605 | skb->tstamp = ktime_get_real(); |
cca5172a | 606 | /* Don't enable netstamp, sunrpc doesn't |
1da177e4 LT |
607 | need that much accuracy */ |
608 | } | |
3a0ed3e9 | 609 | sock_write_timestamp(svsk->sk_sk, skb->tstamp); |
02fc6c36 | 610 | set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); /* there may be more data... */ |
1da177e4 | 611 | |
fff1ebb2 | 612 | len = skb->len; |
1da177e4 | 613 | rqstp->rq_arg.len = len; |
fff1ebb2 | 614 | trace_svcsock_udp_recv(&svsk->sk_xprt, len); |
1da177e4 | 615 | |
95756482 | 616 | rqstp->rq_prot = IPPROTO_UDP; |
27459f09 | 617 | |
fff1ebb2 CL |
618 | if (!svc_udp_get_dest_address(rqstp, cmh)) |
619 | goto out_cmsg_err; | |
849a1cf1 | 620 | rqstp->rq_daddrlen = svc_addr_len(svc_daddr(rqstp)); |
1da177e4 LT |
621 | |
622 | if (skb_is_nonlinear(skb)) { | |
623 | /* we have to copy */ | |
624 | local_bh_disable(); | |
fff1ebb2 CL |
625 | if (csum_partial_copy_to_xdr(&rqstp->rq_arg, skb)) |
626 | goto out_bh_enable; | |
1da177e4 | 627 | local_bh_enable(); |
850cbadd | 628 | consume_skb(skb); |
1da177e4 LT |
629 | } else { |
630 | /* we can use it in-place */ | |
1da8c681 | 631 | rqstp->rq_arg.head[0].iov_base = skb->data; |
1da177e4 | 632 | rqstp->rq_arg.head[0].iov_len = len; |
22911fc5 ED |
633 | if (skb_checksum_complete(skb)) |
634 | goto out_free; | |
5148bf4e | 635 | rqstp->rq_xprt_ctxt = skb; |
1da177e4 LT |
636 | } |
637 | ||
638 | rqstp->rq_arg.page_base = 0; | |
639 | if (len <= rqstp->rq_arg.head[0].iov_len) { | |
640 | rqstp->rq_arg.head[0].iov_len = len; | |
641 | rqstp->rq_arg.page_len = 0; | |
44524359 | 642 | rqstp->rq_respages = rqstp->rq_pages+1; |
1da177e4 LT |
643 | } else { |
644 | rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len; | |
44524359 | 645 | rqstp->rq_respages = rqstp->rq_pages + 1 + |
172589cc | 646 | DIV_ROUND_UP(rqstp->rq_arg.page_len, PAGE_SIZE); |
1da177e4 | 647 | } |
afc59400 | 648 | rqstp->rq_next_page = rqstp->rq_respages+1; |
1da177e4 LT |
649 | |
650 | if (serv->sv_stats) | |
651 | serv->sv_stats->netudpcnt++; | |
652 | ||
319951eb | 653 | svc_sock_secure_port(rqstp); |
82011c80 | 654 | svc_xprt_received(rqstp->rq_xprt); |
1da177e4 | 655 | return len; |
fff1ebb2 CL |
656 | |
657 | out_recv_err: | |
658 | if (err != -EAGAIN) { | |
659 | /* possibly an icmp error */ | |
660 | set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); | |
661 | } | |
662 | trace_svcsock_udp_recv_err(&svsk->sk_xprt, err); | |
82011c80 | 663 | goto out_clear_busy; |
fff1ebb2 CL |
664 | out_cmsg_err: |
665 | net_warn_ratelimited("svc: received unknown control message %d/%d; dropping RPC reply datagram\n", | |
666 | cmh->cmsg_level, cmh->cmsg_type); | |
667 | goto out_free; | |
668 | out_bh_enable: | |
669 | local_bh_enable(); | |
f23abfdb | 670 | out_free: |
850cbadd | 671 | kfree_skb(skb); |
82011c80 CL |
672 | out_clear_busy: |
673 | svc_xprt_received(rqstp->rq_xprt); | |
f23abfdb | 674 | return 0; |
1da177e4 LT |
675 | } |
676 | ||
da1661b9 CL |
677 | /** |
678 | * svc_udp_sendto - Send out a reply on a UDP socket | |
679 | * @rqstp: completed svc_rqst | |
680 | * | |
ca4faf54 CL |
681 | * xpt_mutex ensures @rqstp's whole message is written to the socket |
682 | * without interruption. | |
683 | * | |
da1661b9 CL |
684 | * Returns the number of bytes sent, or a negative errno. |
685 | */ | |
686 | static int svc_udp_sendto(struct svc_rqst *rqstp) | |
1da177e4 | 687 | { |
da1661b9 CL |
688 | struct svc_xprt *xprt = rqstp->rq_xprt; |
689 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
690 | struct xdr_buf *xdr = &rqstp->rq_res; | |
691 | union { | |
692 | struct cmsghdr hdr; | |
693 | long all[SVC_PKTINFO_SPACE / sizeof(long)]; | |
694 | } buffer; | |
695 | struct cmsghdr *cmh = &buffer.hdr; | |
696 | struct msghdr msg = { | |
697 | .msg_name = &rqstp->rq_addr, | |
698 | .msg_namelen = rqstp->rq_addrlen, | |
699 | .msg_control = cmh, | |
baabf59c | 700 | .msg_flags = MSG_SPLICE_PAGES, |
da1661b9 CL |
701 | .msg_controllen = sizeof(buffer), |
702 | }; | |
baabf59c | 703 | unsigned int count; |
da1661b9 | 704 | int err; |
1da177e4 | 705 | |
948f072a N |
706 | svc_udp_release_ctxt(xprt, rqstp->rq_xprt_ctxt); |
707 | rqstp->rq_xprt_ctxt = NULL; | |
23cf1ee1 | 708 | |
da1661b9 | 709 | svc_set_cmsg_data(rqstp, cmh); |
1da177e4 | 710 | |
ca4faf54 CL |
711 | mutex_lock(&xprt->xpt_mutex); |
712 | ||
713 | if (svc_xprt_is_dead(xprt)) | |
714 | goto out_notconn; | |
715 | ||
baabf59c CL |
716 | count = xdr_buf_to_bvec(rqstp->rq_bvec, |
717 | ARRAY_SIZE(rqstp->rq_bvec), xdr); | |
ff053dbb | 718 | |
baabf59c | 719 | iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, |
1d9cabe2 | 720 | count, rqstp->rq_res.len); |
baabf59c | 721 | err = sock_sendmsg(svsk->sk_sock, &msg); |
da1661b9 CL |
722 | if (err == -ECONNREFUSED) { |
723 | /* ICMP error on earlier request. */ | |
baabf59c | 724 | iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, |
1d9cabe2 | 725 | count, rqstp->rq_res.len); |
baabf59c | 726 | err = sock_sendmsg(svsk->sk_sock, &msg); |
da1661b9 | 727 | } |
baabf59c | 728 | |
998024de | 729 | trace_svcsock_udp_send(xprt, err); |
baabf59c | 730 | |
ca4faf54 | 731 | mutex_unlock(&xprt->xpt_mutex); |
baabf59c | 732 | return err; |
ca4faf54 CL |
733 | |
734 | out_notconn: | |
735 | mutex_unlock(&xprt->xpt_mutex); | |
736 | return -ENOTCONN; | |
1da177e4 LT |
737 | } |
738 | ||
323bee32 TT |
739 | static int svc_udp_has_wspace(struct svc_xprt *xprt) |
740 | { | |
741 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
bb5cf160 | 742 | struct svc_serv *serv = xprt->xpt_server; |
323bee32 TT |
743 | unsigned long required; |
744 | ||
745 | /* | |
746 | * Set the SOCK_NOSPACE flag before checking the available | |
747 | * sock space. | |
748 | */ | |
749 | set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); | |
7a90e8cc | 750 | required = atomic_read(&svsk->sk_xprt.xpt_reserved) + serv->sv_max_mesg; |
323bee32 TT |
751 | if (required*2 > sock_wspace(svsk->sk_sk)) |
752 | return 0; | |
753 | clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); | |
754 | return 1; | |
755 | } | |
756 | ||
38a417cc TT |
757 | static struct svc_xprt *svc_udp_accept(struct svc_xprt *xprt) |
758 | { | |
759 | BUG(); | |
760 | return NULL; | |
761 | } | |
762 | ||
ea08e392 SM |
763 | static void svc_udp_kill_temp_xprt(struct svc_xprt *xprt) |
764 | { | |
765 | } | |
766 | ||
b700cbb1 | 767 | static struct svc_xprt *svc_udp_create(struct svc_serv *serv, |
62832c03 | 768 | struct net *net, |
b700cbb1 TT |
769 | struct sockaddr *sa, int salen, |
770 | int flags) | |
771 | { | |
62832c03 | 772 | return svc_create_socket(serv, IPPROTO_UDP, net, sa, salen, flags); |
b700cbb1 TT |
773 | } |
774 | ||
2412e927 | 775 | static const struct svc_xprt_ops svc_udp_ops = { |
b700cbb1 | 776 | .xpo_create = svc_udp_create, |
5d137990 TT |
777 | .xpo_recvfrom = svc_udp_recvfrom, |
778 | .xpo_sendto = svc_udp_sendto, | |
03493bca | 779 | .xpo_result_payload = svc_sock_result_payload, |
948f072a | 780 | .xpo_release_ctxt = svc_udp_release_ctxt, |
755cceab TT |
781 | .xpo_detach = svc_sock_detach, |
782 | .xpo_free = svc_sock_free, | |
323bee32 | 783 | .xpo_has_wspace = svc_udp_has_wspace, |
38a417cc | 784 | .xpo_accept = svc_udp_accept, |
ea08e392 | 785 | .xpo_kill_temp_xprt = svc_udp_kill_temp_xprt, |
360d8738 TT |
786 | }; |
787 | ||
788 | static struct svc_xprt_class svc_udp_class = { | |
789 | .xcl_name = "udp", | |
b700cbb1 | 790 | .xcl_owner = THIS_MODULE, |
360d8738 | 791 | .xcl_ops = &svc_udp_ops, |
49023155 | 792 | .xcl_max_payload = RPCSVC_MAXPAYLOAD_UDP, |
3c45ddf8 | 793 | .xcl_ident = XPRT_TRANSPORT_UDP, |
360d8738 TT |
794 | }; |
795 | ||
bb5cf160 | 796 | static void svc_udp_init(struct svc_sock *svsk, struct svc_serv *serv) |
1da177e4 | 797 | { |
bd4620dd SK |
798 | svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_udp_class, |
799 | &svsk->sk_xprt, serv); | |
def13d74 | 800 | clear_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags); |
fa9251af | 801 | svsk->sk_sk->sk_data_ready = svc_data_ready; |
1da177e4 | 802 | svsk->sk_sk->sk_write_space = svc_write_space; |
1da177e4 LT |
803 | |
804 | /* initialise setting must have enough space to | |
cca5172a | 805 | * receive and respond to one request. |
1da177e4 LT |
806 | * svc_udp_recvfrom will re-adjust if necessary |
807 | */ | |
b7e5034c | 808 | svc_sock_setbufsize(svsk, 3); |
1da177e4 | 809 | |
0f0257ea TT |
810 | /* data might have come in before data_ready set up */ |
811 | set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); | |
02fc6c36 | 812 | set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags); |
7a37f578 | 813 | |
7a37f578 | 814 | /* make sure we get destination address info */ |
7702ce40 CL |
815 | switch (svsk->sk_sk->sk_family) { |
816 | case AF_INET: | |
c1f9ec57 | 817 | ip_sock_set_pktinfo(svsk->sk_sock->sk); |
7d7207c2 | 818 | break; |
7702ce40 | 819 | case AF_INET6: |
7d7207c2 | 820 | ip6_sock_set_recvpktinfo(svsk->sk_sock->sk); |
7702ce40 CL |
821 | break; |
822 | default: | |
823 | BUG(); | |
824 | } | |
1da177e4 LT |
825 | } |
826 | ||
827 | /* | |
828 | * A data_ready event on a listening socket means there's a connection | |
829 | * pending. Do not use state_change as a substitute for it. | |
830 | */ | |
676d2369 | 831 | static void svc_tcp_listen_data_ready(struct sock *sk) |
1da177e4 | 832 | { |
939bb7ef | 833 | struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; |
1da177e4 | 834 | |
40e0b090 PY |
835 | trace_sk_data_ready(sk); |
836 | ||
939bb7ef NB |
837 | /* |
838 | * This callback may called twice when a new connection | |
839 | * is established as a child socket inherits everything | |
840 | * from a parent LISTEN socket. | |
841 | * 1) data_ready method of the parent socket will be called | |
842 | * when one of child sockets become ESTABLISHED. | |
843 | * 2) data_ready method of the child socket may be called | |
844 | * when it receives data before the socket is accepted. | |
fc80fc2d DH |
845 | * In case of 2, we should ignore it silently and DO NOT |
846 | * dereference svsk. | |
939bb7ef | 847 | */ |
fc80fc2d DH |
848 | if (sk->sk_state != TCP_LISTEN) |
849 | return; | |
850 | ||
851 | if (svsk) { | |
852 | /* Refer to svc_setup_socket() for details. */ | |
853 | rmb(); | |
854 | svsk->sk_odata(sk); | |
855 | set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); | |
856 | svc_xprt_enqueue(&svsk->sk_xprt); | |
1da177e4 | 857 | } |
1da177e4 LT |
858 | } |
859 | ||
860 | /* | |
861 | * A state change on a connected socket means it's dying or dead. | |
862 | */ | |
0f0257ea | 863 | static void svc_tcp_state_change(struct sock *sk) |
1da177e4 | 864 | { |
939bb7ef | 865 | struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; |
1da177e4 | 866 | |
a0469f46 | 867 | if (svsk) { |
eebe53e8 VL |
868 | /* Refer to svc_setup_socket() for details. */ |
869 | rmb(); | |
fa9251af | 870 | svsk->sk_ostate(sk); |
a0469f46 | 871 | trace_svcsock_tcp_state(&svsk->sk_xprt, svsk->sk_sock); |
e844d307 CL |
872 | if (sk->sk_state != TCP_ESTABLISHED) |
873 | svc_xprt_deferred_close(&svsk->sk_xprt); | |
1da177e4 | 874 | } |
1da177e4 LT |
875 | } |
876 | ||
877 | /* | |
878 | * Accept a TCP connection | |
879 | */ | |
38a417cc | 880 | static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt) |
1da177e4 | 881 | { |
38a417cc | 882 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); |
cdd88b9f | 883 | struct sockaddr_storage addr; |
884 | struct sockaddr *sin = (struct sockaddr *) &addr; | |
bb5cf160 | 885 | struct svc_serv *serv = svsk->sk_xprt.xpt_server; |
1da177e4 LT |
886 | struct socket *sock = svsk->sk_sock; |
887 | struct socket *newsock; | |
1da177e4 LT |
888 | struct svc_sock *newsvsk; |
889 | int err, slen; | |
890 | ||
1da177e4 | 891 | if (!sock) |
38a417cc | 892 | return NULL; |
1da177e4 | 893 | |
02fc6c36 | 894 | clear_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); |
e6242e92 SS |
895 | err = kernel_accept(sock, &newsock, O_NONBLOCK); |
896 | if (err < 0) { | |
d7900dae CL |
897 | if (err != -EAGAIN) |
898 | trace_svcsock_accept_err(xprt, serv->sv_name, err); | |
38a417cc | 899 | return NULL; |
1da177e4 | 900 | } |
319050d4 CL |
901 | if (IS_ERR(sock_alloc_file(newsock, O_NONBLOCK, NULL))) |
902 | return NULL; | |
903 | ||
02fc6c36 | 904 | set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); |
1da177e4 | 905 | |
9b2c45d4 | 906 | err = kernel_getpeername(newsock, sin); |
1da177e4 | 907 | if (err < 0) { |
a0469f46 | 908 | trace_svcsock_getpeername_err(xprt, serv->sv_name, err); |
1da177e4 LT |
909 | goto failed; /* aborted connection or whatever */ |
910 | } | |
9b2c45d4 | 911 | slen = err; |
1da177e4 | 912 | |
fa9251af TM |
913 | /* Reset the inherited callbacks before calling svc_setup_socket */ |
914 | newsock->sk->sk_state_change = svsk->sk_ostate; | |
915 | newsock->sk->sk_data_ready = svsk->sk_odata; | |
916 | newsock->sk->sk_write_space = svsk->sk_owspace; | |
917 | ||
1da177e4 LT |
918 | /* make sure that a write doesn't block forever when |
919 | * low on memory | |
920 | */ | |
921 | newsock->sk->sk_sndtimeo = HZ*30; | |
922 | ||
72c35376 BF |
923 | newsvsk = svc_setup_socket(serv, newsock, |
924 | (SVC_SOCK_ANONYMOUS | SVC_SOCK_TEMPORARY)); | |
925 | if (IS_ERR(newsvsk)) | |
1da177e4 | 926 | goto failed; |
9dbc240f | 927 | svc_xprt_set_remote(&newsvsk->sk_xprt, sin, slen); |
9b2c45d4 DV |
928 | err = kernel_getsockname(newsock, sin); |
929 | slen = err; | |
a0469f46 | 930 | if (unlikely(err < 0)) |
a9747692 | 931 | slen = offsetof(struct sockaddr, sa_data); |
9dbc240f | 932 | svc_xprt_set_local(&newsvsk->sk_xprt, sin, slen); |
067d7817 | 933 | |
ef11ce24 N |
934 | if (sock_is_loopback(newsock->sk)) |
935 | set_bit(XPT_LOCAL, &newsvsk->sk_xprt.xpt_flags); | |
936 | else | |
937 | clear_bit(XPT_LOCAL, &newsvsk->sk_xprt.xpt_flags); | |
f9f3cc4f TT |
938 | if (serv->sv_stats) |
939 | serv->sv_stats->nettcpconn++; | |
940 | ||
941 | return &newsvsk->sk_xprt; | |
942 | ||
943 | failed: | |
319050d4 | 944 | sockfd_put(newsock); |
f9f3cc4f TT |
945 | return NULL; |
946 | } | |
947 | ||
ca07eda3 CL |
948 | static size_t svc_tcp_restore_pages(struct svc_sock *svsk, |
949 | struct svc_rqst *rqstp) | |
31d68ef6 | 950 | { |
ca07eda3 CL |
951 | size_t len = svsk->sk_datalen; |
952 | unsigned int i, npages; | |
31d68ef6 | 953 | |
ca07eda3 | 954 | if (!len) |
31d68ef6 | 955 | return 0; |
31d68ef6 BF |
956 | npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
957 | for (i = 0; i < npages; i++) { | |
958 | if (rqstp->rq_pages[i] != NULL) | |
959 | put_page(rqstp->rq_pages[i]); | |
960 | BUG_ON(svsk->sk_pages[i] == NULL); | |
961 | rqstp->rq_pages[i] = svsk->sk_pages[i]; | |
962 | svsk->sk_pages[i] = NULL; | |
963 | } | |
964 | rqstp->rq_arg.head[0].iov_base = page_address(rqstp->rq_pages[0]); | |
965 | return len; | |
966 | } | |
967 | ||
968 | static void svc_tcp_save_pages(struct svc_sock *svsk, struct svc_rqst *rqstp) | |
969 | { | |
970 | unsigned int i, len, npages; | |
971 | ||
8af345f5 | 972 | if (svsk->sk_datalen == 0) |
31d68ef6 | 973 | return; |
8af345f5 | 974 | len = svsk->sk_datalen; |
31d68ef6 BF |
975 | npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
976 | for (i = 0; i < npages; i++) { | |
977 | svsk->sk_pages[i] = rqstp->rq_pages[i]; | |
978 | rqstp->rq_pages[i] = NULL; | |
979 | } | |
980 | } | |
981 | ||
982 | static void svc_tcp_clear_pages(struct svc_sock *svsk) | |
983 | { | |
984 | unsigned int i, len, npages; | |
985 | ||
8af345f5 | 986 | if (svsk->sk_datalen == 0) |
31d68ef6 | 987 | goto out; |
8af345f5 | 988 | len = svsk->sk_datalen; |
31d68ef6 BF |
989 | npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
990 | for (i = 0; i < npages; i++) { | |
cf3aa02c BF |
991 | if (svsk->sk_pages[i] == NULL) { |
992 | WARN_ON_ONCE(1); | |
993 | continue; | |
994 | } | |
31d68ef6 BF |
995 | put_page(svsk->sk_pages[i]); |
996 | svsk->sk_pages[i] = NULL; | |
997 | } | |
998 | out: | |
999 | svsk->sk_tcplen = 0; | |
8af345f5 | 1000 | svsk->sk_datalen = 0; |
31d68ef6 BF |
1001 | } |
1002 | ||
1da177e4 | 1003 | /* |
a5cda73e | 1004 | * Receive fragment record header into sk_marker. |
1da177e4 | 1005 | */ |
a5cda73e CL |
1006 | static ssize_t svc_tcp_read_marker(struct svc_sock *svsk, |
1007 | struct svc_rqst *rqstp) | |
1da177e4 | 1008 | { |
a5cda73e | 1009 | ssize_t want, len; |
1da177e4 | 1010 | |
a5cda73e CL |
1011 | /* If we haven't gotten the record length yet, |
1012 | * get the next four bytes. | |
1013 | */ | |
c0401ea0 | 1014 | if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) { |
a5cda73e | 1015 | struct msghdr msg = { NULL }; |
1da177e4 LT |
1016 | struct kvec iov; |
1017 | ||
5ee78d48 | 1018 | want = sizeof(rpc_fraghdr) - svsk->sk_tcplen; |
02648908 | 1019 | iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen; |
1da177e4 | 1020 | iov.iov_len = want; |
de4eda9d | 1021 | iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, want); |
5e052dda | 1022 | len = svc_tcp_sock_recv_cmsg(svsk, &msg); |
4c8e5537 | 1023 | if (len < 0) |
a5cda73e | 1024 | return len; |
1da177e4 | 1025 | svsk->sk_tcplen += len; |
1da177e4 | 1026 | if (len < want) { |
a5cda73e CL |
1027 | /* call again to read the remaining bytes */ |
1028 | goto err_short; | |
1da177e4 | 1029 | } |
a5cda73e | 1030 | trace_svcsock_marker(&svsk->sk_xprt, svsk->sk_marker); |
836fbadb | 1031 | if (svc_sock_reclen(svsk) + svsk->sk_datalen > |
a5cda73e CL |
1032 | svsk->sk_xprt.xpt_server->sv_max_mesg) |
1033 | goto err_too_large; | |
1da177e4 | 1034 | } |
cc248d4b | 1035 | return svc_sock_reclen(svsk); |
a5cda73e CL |
1036 | |
1037 | err_too_large: | |
1038 | net_notice_ratelimited("svc: %s %s RPC fragment too large: %d\n", | |
1039 | __func__, svsk->sk_xprt.xpt_server->sv_name, | |
1040 | svc_sock_reclen(svsk)); | |
e844d307 | 1041 | svc_xprt_deferred_close(&svsk->sk_xprt); |
a5cda73e | 1042 | err_short: |
8f55f3c0 AB |
1043 | return -EAGAIN; |
1044 | } | |
1045 | ||
586c52cc | 1046 | static int receive_cb_reply(struct svc_sock *svsk, struct svc_rqst *rqstp) |
4cfc7e60 | 1047 | { |
586c52cc | 1048 | struct rpc_xprt *bc_xprt = svsk->sk_xprt.xpt_bc_xprt; |
4cfc7e60 | 1049 | struct rpc_rqst *req = NULL; |
586c52cc TM |
1050 | struct kvec *src, *dst; |
1051 | __be32 *p = (__be32 *)rqstp->rq_arg.head[0].iov_base; | |
05a4b583 | 1052 | __be32 xid = *p; |
4cfc7e60 | 1053 | |
093a1468 | 1054 | if (!bc_xprt) |
586c52cc | 1055 | return -EAGAIN; |
75c84151 | 1056 | spin_lock(&bc_xprt->queue_lock); |
093a1468 TM |
1057 | req = xprt_lookup_rqst(bc_xprt, xid); |
1058 | if (!req) | |
05a4b583 | 1059 | goto unlock_eagain; |
586c52cc TM |
1060 | |
1061 | memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(struct xdr_buf)); | |
1062 | /* | |
1063 | * XXX!: cheating for now! Only copying HEAD. | |
1064 | * But we know this is good enough for now (in fact, for any | |
1065 | * callback reply in the forseeable future). | |
1066 | */ | |
1067 | dst = &req->rq_private_buf.head[0]; | |
1068 | src = &rqstp->rq_arg.head[0]; | |
1069 | if (dst->iov_len < src->iov_len) | |
093a1468 | 1070 | goto unlock_eagain; /* whatever; just giving up. */ |
586c52cc | 1071 | memcpy(dst->iov_base, src->iov_base, src->iov_len); |
cc248d4b | 1072 | xprt_complete_rqst(req->rq_task, rqstp->rq_arg.len); |
586c52cc | 1073 | rqstp->rq_arg.len = 0; |
75c84151 | 1074 | spin_unlock(&bc_xprt->queue_lock); |
586c52cc | 1075 | return 0; |
093a1468 | 1076 | unlock_eagain: |
75c84151 | 1077 | spin_unlock(&bc_xprt->queue_lock); |
093a1468 | 1078 | return -EAGAIN; |
586c52cc TM |
1079 | } |
1080 | ||
836fbadb BF |
1081 | static void svc_tcp_fragment_received(struct svc_sock *svsk) |
1082 | { | |
1083 | /* If we have more data, signal svc_xprt_enqueue() to try again */ | |
836fbadb | 1084 | svsk->sk_tcplen = 0; |
02648908 | 1085 | svsk->sk_marker = xdr_zero; |
2b877fc5 CL |
1086 | |
1087 | smp_wmb(); | |
1088 | tcp_set_rcvlowat(svsk->sk_sk, 1); | |
836fbadb | 1089 | } |
31d68ef6 | 1090 | |
ca07eda3 CL |
1091 | /** |
1092 | * svc_tcp_recvfrom - Receive data from a TCP socket | |
1093 | * @rqstp: request structure into which to receive an RPC Call | |
1094 | * | |
1095 | * Called in a loop when XPT_DATA has been set. | |
1096 | * | |
1097 | * Read the 4-byte stream record marker, then use the record length | |
1098 | * in that marker to set up exactly the resources needed to receive | |
1099 | * the next RPC message into @rqstp. | |
1100 | * | |
1101 | * Returns: | |
1102 | * On success, the number of bytes in a received RPC Call, or | |
1103 | * %0 if a complete RPC Call message was not ready to return | |
1104 | * | |
1105 | * The zero return case handles partial receives and callback Replies. | |
1106 | * The state of a partial receive is preserved in the svc_sock for | |
1107 | * the next call to svc_tcp_recvfrom. | |
8f55f3c0 AB |
1108 | */ |
1109 | static int svc_tcp_recvfrom(struct svc_rqst *rqstp) | |
1110 | { | |
1111 | struct svc_sock *svsk = | |
1112 | container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); | |
1113 | struct svc_serv *serv = svsk->sk_xprt.xpt_server; | |
ca07eda3 CL |
1114 | size_t want, base; |
1115 | ssize_t len; | |
586c52cc TM |
1116 | __be32 *p; |
1117 | __be32 calldir; | |
8f55f3c0 | 1118 | |
a5cda73e CL |
1119 | clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); |
1120 | len = svc_tcp_read_marker(svsk, rqstp); | |
8f55f3c0 AB |
1121 | if (len < 0) |
1122 | goto error; | |
1123 | ||
31d68ef6 | 1124 | base = svc_tcp_restore_pages(svsk, rqstp); |
a5cda73e | 1125 | want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr)); |
ca07eda3 | 1126 | len = svc_tcp_read_msg(rqstp, base + want, base); |
8af345f5 | 1127 | if (len >= 0) { |
7dae1dd7 | 1128 | trace_svcsock_tcp_recv(&svsk->sk_xprt, len); |
31d68ef6 | 1129 | svsk->sk_tcplen += len; |
8af345f5 BF |
1130 | svsk->sk_datalen += len; |
1131 | } | |
6be8c594 CL |
1132 | if (len != want || !svc_sock_final_rec(svsk)) |
1133 | goto err_incomplete; | |
1134 | if (svsk->sk_datalen < 8) | |
1135 | goto err_nuts; | |
ad46ccf0 | 1136 | |
836fbadb | 1137 | rqstp->rq_arg.len = svsk->sk_datalen; |
1da177e4 | 1138 | rqstp->rq_arg.page_base = 0; |
5ee78d48 TM |
1139 | if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) { |
1140 | rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len; | |
1da177e4 | 1141 | rqstp->rq_arg.page_len = 0; |
5ee78d48 TM |
1142 | } else |
1143 | rqstp->rq_arg.page_len = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len; | |
1da177e4 | 1144 | |
5148bf4e | 1145 | rqstp->rq_xprt_ctxt = NULL; |
1da177e4 | 1146 | rqstp->rq_prot = IPPROTO_TCP; |
7501cc2b | 1147 | if (test_bit(XPT_LOCAL, &svsk->sk_xprt.xpt_flags)) |
7827c81f | 1148 | set_bit(RQ_LOCAL, &rqstp->rq_flags); |
7501cc2b | 1149 | else |
7827c81f | 1150 | clear_bit(RQ_LOCAL, &rqstp->rq_flags); |
1da177e4 | 1151 | |
586c52cc TM |
1152 | p = (__be32 *)rqstp->rq_arg.head[0].iov_base; |
1153 | calldir = p[1]; | |
8985ef0b | 1154 | if (calldir) |
586c52cc | 1155 | len = receive_cb_reply(svsk, rqstp); |
586c52cc | 1156 | |
1da177e4 | 1157 | /* Reset TCP read info */ |
8af345f5 | 1158 | svsk->sk_datalen = 0; |
836fbadb | 1159 | svc_tcp_fragment_received(svsk); |
0601f793 | 1160 | |
8985ef0b BF |
1161 | if (len < 0) |
1162 | goto error; | |
1da177e4 | 1163 | |
9dbc240f | 1164 | svc_xprt_copy_addrs(rqstp, &svsk->sk_xprt); |
1da177e4 LT |
1165 | if (serv->sv_stats) |
1166 | serv->sv_stats->nettcpcnt++; | |
1167 | ||
319951eb | 1168 | svc_sock_secure_port(rqstp); |
82011c80 | 1169 | svc_xprt_received(rqstp->rq_xprt); |
5ee78d48 | 1170 | return rqstp->rq_arg.len; |
1da177e4 | 1171 | |
6be8c594 CL |
1172 | err_incomplete: |
1173 | svc_tcp_save_pages(svsk, rqstp); | |
1174 | if (len < 0 && len != -EAGAIN) | |
1175 | goto err_delete; | |
1176 | if (len == want) | |
1177 | svc_tcp_fragment_received(svsk); | |
2b877fc5 CL |
1178 | else { |
1179 | /* Avoid more ->sk_data_ready() calls until the rest | |
1180 | * of the message has arrived. This reduces service | |
1181 | * thread wake-ups on large incoming messages. */ | |
1182 | tcp_set_rcvlowat(svsk->sk_sk, | |
1183 | svc_sock_reclen(svsk) - svsk->sk_tcplen); | |
1184 | ||
6be8c594 CL |
1185 | trace_svcsock_tcp_recv_short(&svsk->sk_xprt, |
1186 | svc_sock_reclen(svsk), | |
1187 | svsk->sk_tcplen - sizeof(rpc_fraghdr)); | |
2b877fc5 | 1188 | } |
6be8c594 | 1189 | goto err_noclose; |
8f55f3c0 | 1190 | error: |
31d68ef6 | 1191 | if (len != -EAGAIN) |
ad46ccf0 | 1192 | goto err_delete; |
6be8c594 | 1193 | trace_svcsock_tcp_recv_eagain(&svsk->sk_xprt, 0); |
82011c80 | 1194 | goto err_noclose; |
6be8c594 CL |
1195 | err_nuts: |
1196 | svsk->sk_datalen = 0; | |
ad46ccf0 | 1197 | err_delete: |
6be8c594 | 1198 | trace_svcsock_tcp_recv_err(&svsk->sk_xprt, len); |
e844d307 | 1199 | svc_xprt_deferred_close(&svsk->sk_xprt); |
31d68ef6 | 1200 | err_noclose: |
82011c80 | 1201 | svc_xprt_received(rqstp->rq_xprt); |
9f9d2ebe | 1202 | return 0; /* record not complete */ |
1da177e4 LT |
1203 | } |
1204 | ||
4a85a6a3 | 1205 | /* |
5df5dd03 | 1206 | * MSG_SPLICE_PAGES is used exclusively to reduce the number of |
4a85a6a3 CL |
1207 | * copy operations in this path. Therefore the caller must ensure |
1208 | * that the pages backing @xdr are unchanging. | |
4a85a6a3 | 1209 | */ |
2eb2b935 | 1210 | static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp, |
4d12b727 | 1211 | rpc_fraghdr marker, unsigned int *sentp) |
4a85a6a3 | 1212 | { |
4d12b727 | 1213 | struct msghdr msg = { |
e18e157b | 1214 | .msg_flags = MSG_SPLICE_PAGES, |
4d12b727 | 1215 | }; |
2eb2b935 | 1216 | unsigned int count; |
e18e157b | 1217 | void *buf; |
987c7b1d | 1218 | int ret; |
4a85a6a3 CL |
1219 | |
1220 | *sentp = 0; | |
4a85a6a3 | 1221 | |
e18e157b CL |
1222 | /* The stream record marker is copied into a temporary page |
1223 | * fragment buffer so that it can be included in rq_bvec. | |
1224 | */ | |
1225 | buf = page_frag_alloc(&svsk->sk_frag_cache, sizeof(marker), | |
1226 | GFP_KERNEL); | |
1227 | if (!buf) | |
1228 | return -ENOMEM; | |
1229 | memcpy(buf, &marker, sizeof(marker)); | |
1230 | bvec_set_virt(rqstp->rq_bvec, buf, sizeof(marker)); | |
4a85a6a3 | 1231 | |
e18e157b CL |
1232 | count = xdr_buf_to_bvec(rqstp->rq_bvec + 1, |
1233 | ARRAY_SIZE(rqstp->rq_bvec) - 1, &rqstp->rq_res); | |
c96e2a69 | 1234 | |
2eb2b935 | 1235 | iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, |
e18e157b | 1236 | 1 + count, sizeof(marker) + rqstp->rq_res.len); |
2eb2b935 | 1237 | ret = sock_sendmsg(svsk->sk_sock, &msg); |
05258a0a | 1238 | page_frag_free(buf); |
5df5dd03 DH |
1239 | if (ret < 0) |
1240 | return ret; | |
1241 | *sentp += ret; | |
4a85a6a3 CL |
1242 | return 0; |
1243 | } | |
1244 | ||
da1661b9 CL |
1245 | /** |
1246 | * svc_tcp_sendto - Send out a reply on a TCP socket | |
1247 | * @rqstp: completed svc_rqst | |
1248 | * | |
ca4faf54 CL |
1249 | * xpt_mutex ensures @rqstp's whole message is written to the socket |
1250 | * without interruption. | |
1251 | * | |
da1661b9 | 1252 | * Returns the number of bytes sent, or a negative errno. |
1da177e4 | 1253 | */ |
0f0257ea | 1254 | static int svc_tcp_sendto(struct svc_rqst *rqstp) |
1da177e4 | 1255 | { |
da1661b9 CL |
1256 | struct svc_xprt *xprt = rqstp->rq_xprt; |
1257 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
1258 | struct xdr_buf *xdr = &rqstp->rq_res; | |
1259 | rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | | |
1260 | (u32)xdr->len); | |
3f649ab7 | 1261 | unsigned int sent; |
da1661b9 | 1262 | int err; |
1da177e4 | 1263 | |
948f072a N |
1264 | svc_tcp_release_ctxt(xprt, rqstp->rq_xprt_ctxt); |
1265 | rqstp->rq_xprt_ctxt = NULL; | |
23cf1ee1 | 1266 | |
ca4faf54 CL |
1267 | mutex_lock(&xprt->xpt_mutex); |
1268 | if (svc_xprt_is_dead(xprt)) | |
1269 | goto out_notconn; | |
2eb2b935 | 1270 | err = svc_tcp_sendmsg(svsk, rqstp, marker, &sent); |
cb579086 | 1271 | trace_svcsock_tcp_send(xprt, err < 0 ? (long)err : sent); |
da1661b9 CL |
1272 | if (err < 0 || sent != (xdr->len + sizeof(marker))) |
1273 | goto out_close; | |
ca4faf54 | 1274 | mutex_unlock(&xprt->xpt_mutex); |
1da177e4 | 1275 | return sent; |
da1661b9 | 1276 | |
ca4faf54 CL |
1277 | out_notconn: |
1278 | mutex_unlock(&xprt->xpt_mutex); | |
1279 | return -ENOTCONN; | |
da1661b9 CL |
1280 | out_close: |
1281 | pr_notice("rpc-srv/tcp: %s: %s %d when sending %d bytes - shutting down socket\n", | |
1282 | xprt->xpt_server->sv_name, | |
1283 | (err < 0) ? "got error" : "sent", | |
1284 | (err < 0) ? err : sent, xdr->len); | |
e844d307 | 1285 | svc_xprt_deferred_close(xprt); |
ca4faf54 | 1286 | mutex_unlock(&xprt->xpt_mutex); |
da1661b9 | 1287 | return -EAGAIN; |
1da177e4 LT |
1288 | } |
1289 | ||
b700cbb1 | 1290 | static struct svc_xprt *svc_tcp_create(struct svc_serv *serv, |
62832c03 | 1291 | struct net *net, |
b700cbb1 TT |
1292 | struct sockaddr *sa, int salen, |
1293 | int flags) | |
1294 | { | |
62832c03 | 1295 | return svc_create_socket(serv, IPPROTO_TCP, net, sa, salen, flags); |
b700cbb1 TT |
1296 | } |
1297 | ||
2412e927 | 1298 | static const struct svc_xprt_ops svc_tcp_ops = { |
b700cbb1 | 1299 | .xpo_create = svc_tcp_create, |
5d137990 TT |
1300 | .xpo_recvfrom = svc_tcp_recvfrom, |
1301 | .xpo_sendto = svc_tcp_sendto, | |
03493bca | 1302 | .xpo_result_payload = svc_sock_result_payload, |
948f072a | 1303 | .xpo_release_ctxt = svc_tcp_release_ctxt, |
69b6ba37 | 1304 | .xpo_detach = svc_tcp_sock_detach, |
755cceab | 1305 | .xpo_free = svc_sock_free, |
323bee32 | 1306 | .xpo_has_wspace = svc_tcp_has_wspace, |
38a417cc | 1307 | .xpo_accept = svc_tcp_accept, |
ea08e392 | 1308 | .xpo_kill_temp_xprt = svc_tcp_kill_temp_xprt, |
b3cbf98e | 1309 | .xpo_handshake = svc_tcp_handshake, |
360d8738 TT |
1310 | }; |
1311 | ||
1312 | static struct svc_xprt_class svc_tcp_class = { | |
1313 | .xcl_name = "tcp", | |
b700cbb1 | 1314 | .xcl_owner = THIS_MODULE, |
360d8738 | 1315 | .xcl_ops = &svc_tcp_ops, |
49023155 | 1316 | .xcl_max_payload = RPCSVC_MAXPAYLOAD_TCP, |
3c45ddf8 | 1317 | .xcl_ident = XPRT_TRANSPORT_TCP, |
360d8738 TT |
1318 | }; |
1319 | ||
1320 | void svc_init_xprt_sock(void) | |
1321 | { | |
1322 | svc_reg_xprt_class(&svc_tcp_class); | |
1323 | svc_reg_xprt_class(&svc_udp_class); | |
1324 | } | |
1325 | ||
1326 | void svc_cleanup_xprt_sock(void) | |
1327 | { | |
1328 | svc_unreg_xprt_class(&svc_tcp_class); | |
1329 | svc_unreg_xprt_class(&svc_udp_class); | |
1330 | } | |
1331 | ||
bb5cf160 | 1332 | static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv) |
1da177e4 LT |
1333 | { |
1334 | struct sock *sk = svsk->sk_sk; | |
1da177e4 | 1335 | |
bd4620dd SK |
1336 | svc_xprt_init(sock_net(svsk->sk_sock->sk), &svc_tcp_class, |
1337 | &svsk->sk_xprt, serv); | |
def13d74 | 1338 | set_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags); |
362142b2 | 1339 | set_bit(XPT_CONG_CTRL, &svsk->sk_xprt.xpt_flags); |
1da177e4 | 1340 | if (sk->sk_state == TCP_LISTEN) { |
ece200dd | 1341 | strcpy(svsk->sk_xprt.xpt_remotebuf, "listener"); |
02fc6c36 | 1342 | set_bit(XPT_LISTENER, &svsk->sk_xprt.xpt_flags); |
1da177e4 | 1343 | sk->sk_data_ready = svc_tcp_listen_data_ready; |
02fc6c36 | 1344 | set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); |
1da177e4 | 1345 | } else { |
1da177e4 | 1346 | sk->sk_state_change = svc_tcp_state_change; |
fa9251af | 1347 | sk->sk_data_ready = svc_data_ready; |
637600f3 | 1348 | sk->sk_write_space = svc_write_space; |
1da177e4 | 1349 | |
02648908 | 1350 | svsk->sk_marker = xdr_zero; |
1da177e4 | 1351 | svsk->sk_tcplen = 0; |
8af345f5 | 1352 | svsk->sk_datalen = 0; |
31d68ef6 | 1353 | memset(&svsk->sk_pages[0], 0, sizeof(svsk->sk_pages)); |
1da177e4 | 1354 | |
e0a912e8 | 1355 | tcp_sock_set_nodelay(sk); |
1da177e4 | 1356 | |
02fc6c36 | 1357 | set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); |
b2f21f7d TM |
1358 | switch (sk->sk_state) { |
1359 | case TCP_SYN_RECV: | |
1360 | case TCP_ESTABLISHED: | |
1361 | break; | |
1362 | default: | |
e844d307 | 1363 | svc_xprt_deferred_close(&svsk->sk_xprt); |
b2f21f7d | 1364 | } |
1da177e4 LT |
1365 | } |
1366 | } | |
1367 | ||
0f0257ea | 1368 | void svc_sock_update_bufs(struct svc_serv *serv) |
1da177e4 LT |
1369 | { |
1370 | /* | |
1371 | * The number of server threads has changed. Update | |
1372 | * rcvbuf and sndbuf accordingly on all sockets | |
1373 | */ | |
8f3a6de3 | 1374 | struct svc_sock *svsk; |
1da177e4 LT |
1375 | |
1376 | spin_lock_bh(&serv->sv_lock); | |
8f3a6de3 | 1377 | list_for_each_entry(svsk, &serv->sv_permsocks, sk_xprt.xpt_list) |
02fc6c36 | 1378 | set_bit(XPT_CHNGBUF, &svsk->sk_xprt.xpt_flags); |
1da177e4 LT |
1379 | spin_unlock_bh(&serv->sv_lock); |
1380 | } | |
1381 | ||
1da177e4 LT |
1382 | /* |
1383 | * Initialize socket for RPC use and create svc_sock struct | |
1da177e4 | 1384 | */ |
6b174337 CL |
1385 | static struct svc_sock *svc_setup_socket(struct svc_serv *serv, |
1386 | struct socket *sock, | |
72c35376 | 1387 | int flags) |
1da177e4 LT |
1388 | { |
1389 | struct svc_sock *svsk; | |
1390 | struct sock *inet; | |
6b174337 | 1391 | int pmap_register = !(flags & SVC_SOCK_ANONYMOUS); |
1da177e4 | 1392 | |
72c35376 BF |
1393 | svsk = kzalloc(sizeof(*svsk), GFP_KERNEL); |
1394 | if (!svsk) | |
1395 | return ERR_PTR(-ENOMEM); | |
1da177e4 LT |
1396 | |
1397 | inet = sock->sk; | |
1398 | ||
ae0d7770 CL |
1399 | if (pmap_register) { |
1400 | int err; | |
1401 | ||
72c35376 | 1402 | err = svc_register(serv, sock_net(sock->sk), inet->sk_family, |
5247fab5 | 1403 | inet->sk_protocol, |
c720c7e8 | 1404 | ntohs(inet_sk(inet)->inet_sport)); |
ae0d7770 | 1405 | if (err < 0) { |
ae0d7770 CL |
1406 | kfree(svsk); |
1407 | return ERR_PTR(err); | |
1408 | } | |
1da177e4 LT |
1409 | } |
1410 | ||
1da177e4 LT |
1411 | svsk->sk_sock = sock; |
1412 | svsk->sk_sk = inet; | |
1413 | svsk->sk_ostate = inet->sk_state_change; | |
1414 | svsk->sk_odata = inet->sk_data_ready; | |
1415 | svsk->sk_owspace = inet->sk_write_space; | |
eebe53e8 VL |
1416 | /* |
1417 | * This barrier is necessary in order to prevent race condition | |
e8277327 | 1418 | * with svc_data_ready(), svc_tcp_listen_data_ready(), and others |
eebe53e8 VL |
1419 | * when calling callbacks above. |
1420 | */ | |
1421 | wmb(); | |
1422 | inet->sk_user_data = svsk; | |
1da177e4 LT |
1423 | |
1424 | /* Initialize the socket */ | |
1425 | if (sock->type == SOCK_DGRAM) | |
bb5cf160 | 1426 | svc_udp_init(svsk, serv); |
637600f3 | 1427 | else |
bb5cf160 | 1428 | svc_tcp_init(svsk, serv); |
1da177e4 | 1429 | |
c42bebca | 1430 | trace_svcsock_new(svsk, sock); |
1da177e4 LT |
1431 | return svsk; |
1432 | } | |
1433 | ||
bfba9ab4 CL |
1434 | /** |
1435 | * svc_addsock - add a listener socket to an RPC service | |
1436 | * @serv: pointer to RPC service to which to add a new listener | |
c034203b | 1437 | * @net: caller's network namespace |
bfba9ab4 CL |
1438 | * @fd: file descriptor of the new listener |
1439 | * @name_return: pointer to buffer to fill in with name of listener | |
1440 | * @len: size of the buffer | |
4df493a2 | 1441 | * @cred: credential |
bfba9ab4 CL |
1442 | * |
1443 | * Fills in socket name and returns positive length of name if successful. | |
1444 | * Name is terminated with '\n'. On error, returns a negative errno | |
1445 | * value. | |
1446 | */ | |
c034203b DC |
1447 | int svc_addsock(struct svc_serv *serv, struct net *net, const int fd, |
1448 | char *name_return, const size_t len, const struct cred *cred) | |
b41b66d6 N |
1449 | { |
1450 | int err = 0; | |
1451 | struct socket *so = sockfd_lookup(fd, &err); | |
1452 | struct svc_sock *svsk = NULL; | |
a8e10078 BF |
1453 | struct sockaddr_storage addr; |
1454 | struct sockaddr *sin = (struct sockaddr *)&addr; | |
1455 | int salen; | |
b41b66d6 N |
1456 | |
1457 | if (!so) | |
1458 | return err; | |
c034203b DC |
1459 | err = -EINVAL; |
1460 | if (sock_net(so->sk) != net) | |
1461 | goto out; | |
a8e10078 | 1462 | err = -EAFNOSUPPORT; |
205ba423 | 1463 | if ((so->sk->sk_family != PF_INET) && (so->sk->sk_family != PF_INET6)) |
a8e10078 BF |
1464 | goto out; |
1465 | err = -EPROTONOSUPPORT; | |
1466 | if (so->sk->sk_protocol != IPPROTO_TCP && | |
b41b66d6 | 1467 | so->sk->sk_protocol != IPPROTO_UDP) |
a8e10078 BF |
1468 | goto out; |
1469 | err = -EISCONN; | |
1470 | if (so->state > SS_UNCONNECTED) | |
1471 | goto out; | |
1472 | err = -ENOENT; | |
1473 | if (!try_module_get(THIS_MODULE)) | |
1474 | goto out; | |
1475 | svsk = svc_setup_socket(serv, so, SVC_SOCK_DEFAULTS); | |
1476 | if (IS_ERR(svsk)) { | |
1477 | module_put(THIS_MODULE); | |
1478 | err = PTR_ERR(svsk); | |
1479 | goto out; | |
b41b66d6 | 1480 | } |
9b2c45d4 DV |
1481 | salen = kernel_getsockname(svsk->sk_sock, sin); |
1482 | if (salen >= 0) | |
a8e10078 | 1483 | svc_xprt_set_local(&svsk->sk_xprt, sin, salen); |
4df493a2 | 1484 | svsk->sk_xprt.xpt_cred = get_cred(cred); |
39b55301 | 1485 | svc_add_new_perm_xprt(serv, &svsk->sk_xprt); |
e7942b9f | 1486 | return svc_one_sock_name(svsk, name_return, len); |
a8e10078 BF |
1487 | out: |
1488 | sockfd_put(so); | |
1489 | return err; | |
b41b66d6 N |
1490 | } |
1491 | EXPORT_SYMBOL_GPL(svc_addsock); | |
1492 | ||
1da177e4 LT |
1493 | /* |
1494 | * Create socket for RPC service. | |
1495 | */ | |
b700cbb1 TT |
1496 | static struct svc_xprt *svc_create_socket(struct svc_serv *serv, |
1497 | int protocol, | |
62832c03 | 1498 | struct net *net, |
b700cbb1 TT |
1499 | struct sockaddr *sin, int len, |
1500 | int flags) | |
1da177e4 LT |
1501 | { |
1502 | struct svc_sock *svsk; | |
1503 | struct socket *sock; | |
1504 | int error; | |
1505 | int type; | |
9dbc240f TT |
1506 | struct sockaddr_storage addr; |
1507 | struct sockaddr *newsin = (struct sockaddr *)&addr; | |
1508 | int newlen; | |
c69da774 | 1509 | int family; |
1da177e4 LT |
1510 | |
1511 | if (protocol != IPPROTO_UDP && protocol != IPPROTO_TCP) { | |
1512 | printk(KERN_WARNING "svc: only UDP and TCP " | |
1513 | "sockets supported\n"); | |
b700cbb1 | 1514 | return ERR_PTR(-EINVAL); |
1da177e4 | 1515 | } |
c69da774 | 1516 | |
1da177e4 | 1517 | type = (protocol == IPPROTO_UDP)? SOCK_DGRAM : SOCK_STREAM; |
c69da774 TM |
1518 | switch (sin->sa_family) { |
1519 | case AF_INET6: | |
1520 | family = PF_INET6; | |
1521 | break; | |
1522 | case AF_INET: | |
1523 | family = PF_INET; | |
1524 | break; | |
1525 | default: | |
1526 | return ERR_PTR(-EINVAL); | |
1527 | } | |
1da177e4 | 1528 | |
14ec63c3 | 1529 | error = __sock_create(net, family, type, protocol, &sock, 1); |
77f1f67a | 1530 | if (error < 0) |
b700cbb1 | 1531 | return ERR_PTR(error); |
1da177e4 | 1532 | |
ed07536e PZ |
1533 | svc_reclassify_socket(sock); |
1534 | ||
c69da774 TM |
1535 | /* |
1536 | * If this is an PF_INET6 listener, we want to avoid | |
1537 | * getting requests from IPv4 remotes. Those should | |
1538 | * be shunted to a PF_INET listener via rpcbind. | |
1539 | */ | |
c69da774 | 1540 | if (family == PF_INET6) |
9b115749 | 1541 | ip6_sock_set_v6only(sock->sk); |
18114746 | 1542 | if (type == SOCK_STREAM) |
4a17fd52 | 1543 | sock->sk->sk_reuse = SK_CAN_REUSE; /* allow address reuse */ |
77f1f67a | 1544 | error = kernel_bind(sock, sin, len); |
18114746 ES |
1545 | if (error < 0) |
1546 | goto bummer; | |
1da177e4 | 1547 | |
9b2c45d4 | 1548 | error = kernel_getsockname(sock, newsin); |
9dbc240f TT |
1549 | if (error < 0) |
1550 | goto bummer; | |
9b2c45d4 | 1551 | newlen = error; |
9dbc240f | 1552 | |
1da177e4 | 1553 | if (protocol == IPPROTO_TCP) { |
e6242e92 | 1554 | if ((error = kernel_listen(sock, 64)) < 0) |
1da177e4 LT |
1555 | goto bummer; |
1556 | } | |
1557 | ||
72c35376 | 1558 | svsk = svc_setup_socket(serv, sock, flags); |
a8e10078 BF |
1559 | if (IS_ERR(svsk)) { |
1560 | error = PTR_ERR(svsk); | |
1561 | goto bummer; | |
e79eff1f | 1562 | } |
a8e10078 BF |
1563 | svc_xprt_set_local(&svsk->sk_xprt, newsin, newlen); |
1564 | return (struct svc_xprt *)svsk; | |
1da177e4 | 1565 | bummer: |
1da177e4 | 1566 | sock_release(sock); |
b700cbb1 | 1567 | return ERR_PTR(error); |
1da177e4 LT |
1568 | } |
1569 | ||
755cceab TT |
1570 | /* |
1571 | * Detach the svc_sock from the socket so that no | |
1572 | * more callbacks occur. | |
1573 | */ | |
1574 | static void svc_sock_detach(struct svc_xprt *xprt) | |
1575 | { | |
1576 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
1577 | struct sock *sk = svsk->sk_sk; | |
1578 | ||
755cceab | 1579 | /* put back the old socket callbacks */ |
069c225b | 1580 | lock_sock(sk); |
755cceab TT |
1581 | sk->sk_state_change = svsk->sk_ostate; |
1582 | sk->sk_data_ready = svsk->sk_odata; | |
1583 | sk->sk_write_space = svsk->sk_owspace; | |
069c225b TM |
1584 | sk->sk_user_data = NULL; |
1585 | release_sock(sk); | |
69b6ba37 TM |
1586 | } |
1587 | ||
1588 | /* | |
1589 | * Disconnect the socket, and reset the callbacks | |
1590 | */ | |
1591 | static void svc_tcp_sock_detach(struct svc_xprt *xprt) | |
1592 | { | |
1593 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
1594 | ||
5dd5ad68 CL |
1595 | tls_handshake_close(svsk->sk_sock); |
1596 | ||
69b6ba37 TM |
1597 | svc_sock_detach(xprt); |
1598 | ||
31d68ef6 BF |
1599 | if (!test_bit(XPT_LISTENER, &xprt->xpt_flags)) { |
1600 | svc_tcp_clear_pages(svsk); | |
69b6ba37 | 1601 | kernel_sock_shutdown(svsk->sk_sock, SHUT_RDWR); |
31d68ef6 | 1602 | } |
755cceab TT |
1603 | } |
1604 | ||
1605 | /* | |
1606 | * Free the svc_sock's socket resources and the svc_sock itself. | |
1607 | */ | |
1608 | static void svc_sock_free(struct svc_xprt *xprt) | |
1609 | { | |
1610 | struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); | |
e18e157b | 1611 | struct page_frag_cache *pfc = &svsk->sk_frag_cache; |
b3cbf98e | 1612 | struct socket *sock = svsk->sk_sock; |
755cceab | 1613 | |
c42bebca CL |
1614 | trace_svcsock_free(svsk, sock); |
1615 | ||
b3cbf98e CL |
1616 | tls_handshake_cancel(sock->sk); |
1617 | if (sock->file) | |
1618 | sockfd_put(sock); | |
755cceab | 1619 | else |
b3cbf98e | 1620 | sock_release(sock); |
e18e157b CL |
1621 | if (pfc->va) |
1622 | __page_frag_cache_drain(virt_to_head_page(pfc->va), | |
1623 | pfc->pagecnt_bias); | |
755cceab TT |
1624 | kfree(svsk); |
1625 | } |