projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'bitmap-for-5.19-rc1' of https://github.com/norov/linux
[linux-2.6-block.git] / net / core / sock.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Generic socket support routines. Memory allocators, socket lock/release
8 * handler for protocols to use and generic option handler.
9 *
02c30a84 10 * Authors: Ross Biro
1da177e4
LT		 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
4ec93edb 35 * code. The ACK stuff can wait and needs major
1da177e4
LT		 36 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
1da177e4
LT		 84 */
85
e005d193
JP		 86#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
87
80b14dee 88#include <asm/unaligned.h>
4fc268d2 89#include <linux/capability.h>
1da177e4 90#include <linux/errno.h>
cb820f8e 91#include <linux/errqueue.h>
1da177e4
LT		 92#include <linux/types.h>
93#include <linux/socket.h>
94#include <linux/in.h>
95#include <linux/kernel.h>
1da177e4
LT		 96#include <linux/module.h>
97#include <linux/proc_fs.h>
98#include <linux/seq_file.h>
99#include <linux/sched.h>
f1083048 100#include <linux/sched/mm.h>
1da177e4
LT		 101#include <linux/timer.h>
102#include <linux/string.h>
103#include <linux/sockios.h>
104#include <linux/net.h>
105#include <linux/mm.h>
106#include <linux/slab.h>
107#include <linux/interrupt.h>
108#include <linux/poll.h>
109#include <linux/tcp.h>
110#include <linux/init.h>
a1f8e7f7 111#include <linux/highmem.h>
3f551f94 112#include <linux/user_namespace.h>
c5905afb 113#include <linux/static_key.h>
3969eb38 114#include <linux/memcontrol.h>
8c1ae10d 115#include <linux/prefetch.h>
a6c0d093 116#include <linux/compat.h>
1da177e4 117
7c0f6ba6 118#include <linux/uaccess.h>
1da177e4
LT		 119
120#include <linux/netdevice.h>
121#include <net/protocol.h>
122#include <linux/skbuff.h>
457c4cbc 123#include <net/net_namespace.h>
2e6599cb 124#include <net/request_sock.h>
1da177e4 125#include <net/sock.h>
20d49473 126#include <linux/net_tstamp.h>
1da177e4
LT		 127#include <net/xfrm.h>
128#include <linux/ipsec.h>
f8451725 129#include <net/cls_cgroup.h>
5bc1421e 130#include <net/netprio_cgroup.h>
eb4cb008 131#include <linux/sock_diag.h>
1da177e4
LT		 132
133#include <linux/filter.h>
538950a1 134#include <net/sock_reuseport.h>
6ac99e8f 135#include <net/bpf_sk_storage.h>
1da177e4 136
3847ce32
SM		 137#include <trace/events/sock.h>
138
1da177e4 139#include <net/tcp.h>
076bb0c8 140#include <net/busy_poll.h>
06021292 141
d463126e
YL		 142#include <linux/ethtool.h>
143
6264f58c
JK		 144#include "dev.h"
145
36b77a52 146static DEFINE_MUTEX(proto_list_mutex);
d1a4c0b3
GC		 147static LIST_HEAD(proto_list);
148
0a8afd9f 149static void sock_def_write_space_wfree(struct sock *sk);
052ada09
PB		 150static void sock_def_write_space(struct sock *sk);
151
a3b299da
EB		 152/**
153 * sk_ns_capable - General socket capability test
154 * @sk: Socket to use a capability on or through
155 * @user_ns: The user namespace of the capability to use
156 * @cap: The capability to use
157 *
158 * Test to see if the opener of the socket had when the socket was
159 * created and the current process has the capability @cap in the user
160 * namespace @user_ns.
161 */
162bool sk_ns_capable(const struct sock *sk,
163 struct user_namespace *user_ns, int cap)
164{
165 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
166 ns_capable(user_ns, cap);
167}
168EXPORT_SYMBOL(sk_ns_capable);
169
170/**
171 * sk_capable - Socket global capability test
172 * @sk: Socket to use a capability on or through
e793c0f7 173 * @cap: The global capability to use
a3b299da
EB		 174 *
175 * Test to see if the opener of the socket had when the socket was
176 * created and the current process has the capability @cap in all user
177 * namespaces.
178 */
179bool sk_capable(const struct sock *sk, int cap)
180{
181 return sk_ns_capable(sk, &init_user_ns, cap);
182}
183EXPORT_SYMBOL(sk_capable);
184
185/**
186 * sk_net_capable - Network namespace socket capability test
187 * @sk: Socket to use a capability on or through
188 * @cap: The capability to use
189 *
e793c0f7 190 * Test to see if the opener of the socket had when the socket was created
a3b299da
EB		 191 * and the current process has the capability @cap over the network namespace
192 * the socket is a member of.
193 */
194bool sk_net_capable(const struct sock *sk, int cap)
195{
196 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
197}
198EXPORT_SYMBOL(sk_net_capable);
199
da21f24d
IM		 200/*
201 * Each address family might have different locking rules, so we have
cdfbabfb
DH		 202 * one slock key per address family and separate keys for internal and
203 * userspace sockets.
da21f24d 204 */
a5b5bb9a 205static struct lock_class_key af_family_keys[AF_MAX];
cdfbabfb 206static struct lock_class_key af_family_kern_keys[AF_MAX];
a5b5bb9a 207static struct lock_class_key af_family_slock_keys[AF_MAX];
cdfbabfb 208static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
a5b5bb9a 209
a5b5bb9a
IM		 210/*
211 * Make lock validator output more readable. (we pre-construct these
212 * strings build-time, so that runtime initialization of socket
213 * locks is fast):
214 */
cdfbabfb
DH		 215
216#define _sock_locks(x) \
217 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \
218 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \
219 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \
220 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \
221 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \
222 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \
223 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \
224 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \
225 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \
226 x "27" , x "28" , x "AF_CAN" , \
227 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \
228 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
229 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
230 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
68e8b849 231 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \
bc49d816 232 x "AF_MCTP" , \
68e8b849 233 x "AF_MAX"
cdfbabfb 234
36cbd3dc 235static const char *const af_family_key_strings[AF_MAX+1] = {
cdfbabfb 236 _sock_locks("sk_lock-")
a5b5bb9a 237};
36cbd3dc 238static const char *const af_family_slock_key_strings[AF_MAX+1] = {
cdfbabfb 239 _sock_locks("slock-")
a5b5bb9a 240};
36cbd3dc 241static const char *const af_family_clock_key_strings[AF_MAX+1] = {
cdfbabfb
DH		 242 _sock_locks("clock-")
243};
244
245static const char *const af_family_kern_key_strings[AF_MAX+1] = {
246 _sock_locks("k-sk_lock-")
247};
248static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
249 _sock_locks("k-slock-")
250};
251static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
252 _sock_locks("k-clock-")
443aef0e 253};
581319c5 254static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
6b431d50 255 _sock_locks("rlock-")
581319c5
PA		 256};
257static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
6b431d50 258 _sock_locks("wlock-")
581319c5
PA		 259};
260static const char *const af_family_elock_key_strings[AF_MAX+1] = {
6b431d50 261 _sock_locks("elock-")
581319c5 262};
da21f24d
IM		 263
264/*
581319c5 265 * sk_callback_lock and sk queues locking rules are per-address-family,
da21f24d
IM		 266 * so split the lock classes by using a per-AF key:
267 */
268static struct lock_class_key af_callback_keys[AF_MAX];
581319c5
PA		 269static struct lock_class_key af_rlock_keys[AF_MAX];
270static struct lock_class_key af_wlock_keys[AF_MAX];
271static struct lock_class_key af_elock_keys[AF_MAX];
cdfbabfb 272static struct lock_class_key af_kern_callback_keys[AF_MAX];
da21f24d 273
1da177e4 274/* Run time adjustable parameters. */
ab32ea5d 275__u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
6d8ebc8a 276EXPORT_SYMBOL(sysctl_wmem_max);
ab32ea5d 277__u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
6d8ebc8a 278EXPORT_SYMBOL(sysctl_rmem_max);
ab32ea5d
BH		 279__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
280__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
1da177e4 281
25985edc 282/* Maximal space eaten by iovec or ancillary data plus some space */
ab32ea5d 283int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
2a91525c 284EXPORT_SYMBOL(sysctl_optmem_max);
1da177e4 285
b245be1f
WB		 286int sysctl_tstamp_allow_data __read_mostly = 1;
287
a7950ae8
DB		 288DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
289EXPORT_SYMBOL_GPL(memalloc_socks_key);
c93bdd0e 290
7cb02404
MG		 291/**
292 * sk_set_memalloc - sets %SOCK_MEMALLOC
293 * @sk: socket to set it on
294 *
295 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
296 * It's the responsibility of the admin to adjust min_free_kbytes
297 * to meet the requirements
298 */
299void sk_set_memalloc(struct sock *sk)
300{
301 sock_set_flag(sk, SOCK_MEMALLOC);
302 sk->sk_allocation |= __GFP_MEMALLOC;
a7950ae8 303 static_branch_inc(&memalloc_socks_key);
7cb02404
MG		 304}
305EXPORT_SYMBOL_GPL(sk_set_memalloc);
306
307void sk_clear_memalloc(struct sock *sk)
308{
309 sock_reset_flag(sk, SOCK_MEMALLOC);
310 sk->sk_allocation &= ~__GFP_MEMALLOC;
a7950ae8 311 static_branch_dec(&memalloc_socks_key);
c76562b6
MG		 312
313 /*
314 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
5d753610
MG		 315 * progress of swapping. SOCK_MEMALLOC may be cleared while
316 * it has rmem allocations due to the last swapfile being deactivated
317 * but there is a risk that the socket is unusable due to exceeding
318 * the rmem limits. Reclaim the reserves and obey rmem limits again.
c76562b6 319 */
5d753610 320 sk_mem_reclaim(sk);
7cb02404
MG		 321}
322EXPORT_SYMBOL_GPL(sk_clear_memalloc);
323
b4b9e355
MG		 324int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
325{
326 int ret;
f1083048 327 unsigned int noreclaim_flag;
b4b9e355
MG		 328
329 /* these should have been dropped before queueing */
330 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
331
f1083048 332 noreclaim_flag = memalloc_noreclaim_save();
d2489c7b
ED		 333 ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv,
334 tcp_v6_do_rcv,
335 tcp_v4_do_rcv,
336 sk, skb);
f1083048 337 memalloc_noreclaim_restore(noreclaim_flag);
b4b9e355
MG		 338
339 return ret;
340}
341EXPORT_SYMBOL(__sk_backlog_rcv);
342
e3ae2365
AA		 343void sk_error_report(struct sock *sk)
344{
345 sk->sk_error_report(sk);
e6a3e443
AA		 346
347 switch (sk->sk_family) {
348 case AF_INET:
349 fallthrough;
350 case AF_INET6:
351 trace_inet_sk_error_report(sk);
352 break;
353 default:
354 break;
355 }
e3ae2365
AA		 356}
357EXPORT_SYMBOL(sk_error_report);
358
4c1e34c0 359int sock_get_timeout(long timeo, void *optval, bool old_timeval)
fe0c72f3 360{
a9beb86a 361 struct __kernel_sock_timeval tv;
fe0c72f3
AB		 362
363 if (timeo == MAX_SCHEDULE_TIMEOUT) {
364 tv.tv_sec = 0;
365 tv.tv_usec = 0;
366 } else {
367 tv.tv_sec = timeo / HZ;
368 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ;
369 }
370
e6986423 371 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
fe0c72f3
AB		 372 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec };
373 *(struct old_timeval32 *)optval = tv32;
374 return sizeof(tv32);
375 }
376
a9beb86a
DD		 377 if (old_timeval) {
378 struct __kernel_old_timeval old_tv;
379 old_tv.tv_sec = tv.tv_sec;
380 old_tv.tv_usec = tv.tv_usec;
381 *(struct __kernel_old_timeval *)optval = old_tv;
28e72b26 382 return sizeof(old_tv);
a9beb86a
DD		 383 }
384
28e72b26
VC		 385 *(struct __kernel_sock_timeval *)optval = tv;
386 return sizeof(tv);
fe0c72f3 387}
4c1e34c0 388EXPORT_SYMBOL(sock_get_timeout);
fe0c72f3 389
4c1e34c0
RP		 390int sock_copy_user_timeval(struct __kernel_sock_timeval *tv,
391 sockptr_t optval, int optlen, bool old_timeval)
1da177e4 392{
e6986423 393 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
fe0c72f3
AB		 394 struct old_timeval32 tv32;
395
396 if (optlen < sizeof(tv32))
397 return -EINVAL;
398
c34645ac 399 if (copy_from_sockptr(&tv32, optval, sizeof(tv32)))
fe0c72f3 400 return -EFAULT;
4c1e34c0
RP		 401 tv->tv_sec = tv32.tv_sec;
402 tv->tv_usec = tv32.tv_usec;
a9beb86a
DD		 403 } else if (old_timeval) {
404 struct __kernel_old_timeval old_tv;
405
406 if (optlen < sizeof(old_tv))
407 return -EINVAL;
c34645ac 408 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv)))
a9beb86a 409 return -EFAULT;
4c1e34c0
RP		 410 tv->tv_sec = old_tv.tv_sec;
411 tv->tv_usec = old_tv.tv_usec;
fe0c72f3 412 } else {
4c1e34c0 413 if (optlen < sizeof(*tv))
fe0c72f3 414 return -EINVAL;
4c1e34c0 415 if (copy_from_sockptr(tv, optval, sizeof(*tv)))
fe0c72f3
AB		 416 return -EFAULT;
417 }
4c1e34c0
RP		 418
419 return 0;
420}
421EXPORT_SYMBOL(sock_copy_user_timeval);
422
423static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen,
424 bool old_timeval)
425{
426 struct __kernel_sock_timeval tv;
427 int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval);
428
429 if (err)
430 return err;
431
ba78073e
VA		 432 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
433 return -EDOM;
1da177e4 434
ba78073e 435 if (tv.tv_sec < 0) {
6f11df83
AM		 436 static int warned __read_mostly;
437
ba78073e 438 *timeo_p = 0;
50aab54f 439 if (warned < 10 && net_ratelimit()) {
ba78073e 440 warned++;
e005d193
JP		 441 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
442 __func__, current->comm, task_pid_nr(current));
50aab54f 443 }
ba78073e
VA		 444 return 0;
445 }
1da177e4
LT		 446 *timeo_p = MAX_SCHEDULE_TIMEOUT;
447 if (tv.tv_sec == 0 && tv.tv_usec == 0)
448 return 0;
a9beb86a
DD		 449 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))
450 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ);
1da177e4
LT		 451 return 0;
452}
453
080a270f
HFS		 454static bool sock_needs_netstamp(const struct sock *sk)
455{
456 switch (sk->sk_family) {
457 case AF_UNSPEC:
458 case AF_UNIX:
459 return false;
460 default:
461 return true;
462 }
463}
464
08e29af3 465static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
4ec93edb 466{
08e29af3
ED		 467 if (sk->sk_flags & flags) {
468 sk->sk_flags &= ~flags;
080a270f
HFS		 469 if (sock_needs_netstamp(sk) &&
470 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
20d49473 471 net_disable_timestamp();
1da177e4
LT		 472 }
473}
474
475
e6afc8ac 476int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
f0088a50 477{
3b885787
NH		 478 unsigned long flags;
479 struct sk_buff_head *list = &sk->sk_receive_queue;
f0088a50 480
0fd7bac6 481 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
766e9037 482 atomic_inc(&sk->sk_drops);
3847ce32 483 trace_sock_rcvqueue_full(sk, skb);
766e9037 484 return -ENOMEM;
f0088a50
DV		 485 }
486
c76562b6 487 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
766e9037
ED		 488 atomic_inc(&sk->sk_drops);
489 return -ENOBUFS;
3ab224be
HA		 490 }
491
f0088a50
DV		 492 skb->dev = NULL;
493 skb_set_owner_r(skb, sk);
49ad9599 494
7fee226a
ED		 495 /* we escape from rcu protected region, make sure we dont leak
496 * a norefcounted dst
497 */
498 skb_dst_force(skb);
499
3b885787 500 spin_lock_irqsave(&list->lock, flags);
3bc3b96f 501 sock_skb_set_dropcount(sk, skb);
3b885787
NH		 502 __skb_queue_tail(list, skb);
503 spin_unlock_irqrestore(&list->lock, flags);
f0088a50
DV		 504
505 if (!sock_flag(sk, SOCK_DEAD))
676d2369 506 sk->sk_data_ready(sk);
766e9037 507 return 0;
f0088a50 508}
e6afc8ac 509EXPORT_SYMBOL(__sock_queue_rcv_skb);
510
c1b8a567
MD		 511int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb,
512 enum skb_drop_reason *reason)
e6afc8ac 513{
c1b8a567 514 enum skb_drop_reason drop_reason;
e6afc8ac 515 int err;
516
517 err = sk_filter(sk, skb);
c1b8a567
MD		 518 if (err) {
519 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
520 goto out;
521 }
522 err = __sock_queue_rcv_skb(sk, skb);
523 switch (err) {
524 case -ENOMEM:
525 drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
526 break;
527 case -ENOBUFS:
528 drop_reason = SKB_DROP_REASON_PROTO_MEM;
529 break;
530 default:
531 drop_reason = SKB_NOT_DROPPED_YET;
532 break;
533 }
534out:
535 if (reason)
536 *reason = drop_reason;
537 return err;
e6afc8ac 538}
c1b8a567 539EXPORT_SYMBOL(sock_queue_rcv_skb_reason);
f0088a50 540
4f0c40d9 541int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
c3f24cfb 542 const int nested, unsigned int trim_cap, bool refcounted)
f0088a50
DV		 543{
544 int rc = NET_RX_SUCCESS;
545
4f0c40d9 546 if (sk_filter_trim_cap(sk, skb, trim_cap))
f0088a50
DV		 547 goto discard_and_relse;
548
549 skb->dev = NULL;
550
274f482d 551 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
c377411f
ED		 552 atomic_inc(&sk->sk_drops);
553 goto discard_and_relse;
554 }
58a5a7b9
ACM		 555 if (nested)
556 bh_lock_sock_nested(sk);
557 else
558 bh_lock_sock(sk);
a5b5bb9a
IM		 559 if (!sock_owned_by_user(sk)) {
560 /*
561 * trylock + unlock semantics:
562 */
563 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
564
c57943a1 565 rc = sk_backlog_rcv(sk, skb);
a5b5bb9a 566
5facae4f 567 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
8265792b 568 } else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) {
8eae939f
ZY		 569 bh_unlock_sock(sk);
570 atomic_inc(&sk->sk_drops);
571 goto discard_and_relse;
572 }
573
f0088a50
DV		 574 bh_unlock_sock(sk);
575out:
c3f24cfb
ED		 576 if (refcounted)
577 sock_put(sk);
f0088a50
DV		 578 return rc;
579discard_and_relse:
580 kfree_skb(skb);
581 goto out;
582}
4f0c40d9 583EXPORT_SYMBOL(__sk_receive_skb);
f0088a50 584
bbd807df
BV		 585INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
586 u32));
587INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
588 u32));
f0088a50
DV		 589struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
590{
b6c6712a 591 struct dst_entry *dst = __sk_dst_get(sk);
f0088a50 592
bbd807df
BV		 593 if (dst && dst->obsolete &&
594 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
595 dst, cookie) == NULL) {
e022f0b4 596 sk_tx_queue_clear(sk);
9b8805a3 597 sk->sk_dst_pending_confirm = 0;
a9b3cd7f 598 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
f0088a50
DV		 599 dst_release(dst);
600 return NULL;
601 }
602
603 return dst;
604}
605EXPORT_SYMBOL(__sk_dst_check);
606
607struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
608{
609 struct dst_entry *dst = sk_dst_get(sk);
610
bbd807df
BV		 611 if (dst && dst->obsolete &&
612 INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
613 dst, cookie) == NULL) {
f0088a50
DV		 614 sk_dst_reset(sk);
615 dst_release(dst);
616 return NULL;
617 }
618
619 return dst;
620}
621EXPORT_SYMBOL(sk_dst_check);
622
7594888c 623static int sock_bindtoindex_locked(struct sock *sk, int ifindex)
4878809f
DM		 624{
625 int ret = -ENOPROTOOPT;
626#ifdef CONFIG_NETDEVICES
3b1e0a65 627 struct net *net = sock_net(sk);
4878809f
DM		 628
629 /* Sorry... */
630 ret = -EPERM;
c427bfec 631 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW))
4878809f
DM		 632 goto out;
633
f5dd3d0c
DH		 634 ret = -EINVAL;
635 if (ifindex < 0)
636 goto out;
637
e5fccaa1
ED		 638 /* Paired with all READ_ONCE() done locklessly. */
639 WRITE_ONCE(sk->sk_bound_dev_if, ifindex);
640
f5dd3d0c
DH		 641 if (sk->sk_prot->rehash)
642 sk->sk_prot->rehash(sk);
643 sk_dst_reset(sk);
644
645 ret = 0;
646
647out:
648#endif
649
650 return ret;
651}
652
8ea204c2 653int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk)
7594888c
CH		 654{
655 int ret;
656
8ea204c2
FF		 657 if (lock_sk)
658 lock_sock(sk);
7594888c 659 ret = sock_bindtoindex_locked(sk, ifindex);
8ea204c2
FF		 660 if (lock_sk)
661 release_sock(sk);
7594888c
CH		 662
663 return ret;
664}
665EXPORT_SYMBOL(sock_bindtoindex);
666
5790642b 667static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen)
f5dd3d0c
DH		 668{
669 int ret = -ENOPROTOOPT;
670#ifdef CONFIG_NETDEVICES
671 struct net *net = sock_net(sk);
672 char devname[IFNAMSIZ];
673 int index;
674
4878809f
DM		 675 ret = -EINVAL;
676 if (optlen < 0)
677 goto out;
678
679 /* Bind this socket to a particular device like "eth0",
680 * as specified in the passed interface name. If the
681 * name is "" or the option length is zero the socket
682 * is not bound.
683 */
684 if (optlen > IFNAMSIZ - 1)
685 optlen = IFNAMSIZ - 1;
686 memset(devname, 0, sizeof(devname));
687
688 ret = -EFAULT;
5790642b 689 if (copy_from_sockptr(devname, optval, optlen))
4878809f
DM		 690 goto out;
691
000ba2e4
DM		 692 index = 0;
693 if (devname[0] != '\0') {
bf8e56bf 694 struct net_device *dev;
4878809f 695
bf8e56bf
ED		 696 rcu_read_lock();
697 dev = dev_get_by_name_rcu(net, devname);
698 if (dev)
699 index = dev->ifindex;
700 rcu_read_unlock();
4878809f
DM		 701 ret = -ENODEV;
702 if (!dev)
703 goto out;
4878809f
DM		 704 }
705
8ea204c2 706 return sock_bindtoindex(sk, index, true);
4878809f
DM		 707out:
708#endif
709
710 return ret;
711}
712
c91f6df2
BH		 713static int sock_getbindtodevice(struct sock *sk, char __user *optval,
714 int __user *optlen, int len)
715{
716 int ret = -ENOPROTOOPT;
717#ifdef CONFIG_NETDEVICES
e5fccaa1 718 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
c91f6df2 719 struct net *net = sock_net(sk);
c91f6df2 720 char devname[IFNAMSIZ];
c91f6df2 721
e5fccaa1 722 if (bound_dev_if == 0) {
c91f6df2
BH		 723 len = 0;
724 goto zero;
725 }
726
727 ret = -EINVAL;
728 if (len < IFNAMSIZ)
729 goto out;
730
e5fccaa1 731 ret = netdev_get_name(net, devname, bound_dev_if);
5dbe7c17 732 if (ret)
c91f6df2 733 goto out;
c91f6df2
BH		 734
735 len = strlen(devname) + 1;
736
737 ret = -EFAULT;
738 if (copy_to_user(optval, devname, len))
739 goto out;
740
741zero:
742 ret = -EFAULT;
743 if (put_user(len, optlen))
744 goto out;
745
746 ret = 0;
747
748out:
749#endif
750
751 return ret;
752}
753
f60e5990 754bool sk_mc_loop(struct sock *sk)
755{
756 if (dev_recursion_level())
757 return false;
758 if (!sk)
759 return true;
760 switch (sk->sk_family) {
761 case AF_INET:
762 return inet_sk(sk)->mc_loop;
763#if IS_ENABLED(CONFIG_IPV6)
764 case AF_INET6:
765 return inet6_sk(sk)->mc_loop;
766#endif
767 }
0ad6f6e7 768 WARN_ON_ONCE(1);
f60e5990 769 return true;
770}
771EXPORT_SYMBOL(sk_mc_loop);
772
b58f0e8f
CH		 773void sock_set_reuseaddr(struct sock *sk)
774{
775 lock_sock(sk);
776 sk->sk_reuse = SK_CAN_REUSE;
777 release_sock(sk);
778}
779EXPORT_SYMBOL(sock_set_reuseaddr);
780
fe31a326
CH		 781void sock_set_reuseport(struct sock *sk)
782{
783 lock_sock(sk);
784 sk->sk_reuseport = true;
785 release_sock(sk);
786}
787EXPORT_SYMBOL(sock_set_reuseport);
788
c433594c
CH		 789void sock_no_linger(struct sock *sk)
790{
791 lock_sock(sk);
792 sk->sk_lingertime = 0;
793 sock_set_flag(sk, SOCK_LINGER);
794 release_sock(sk);
795}
796EXPORT_SYMBOL(sock_no_linger);
797
6e434967
CH		 798void sock_set_priority(struct sock *sk, u32 priority)
799{
800 lock_sock(sk);
801 sk->sk_priority = priority;
802 release_sock(sk);
803}
804EXPORT_SYMBOL(sock_set_priority);
805
76ee0785
CH		 806void sock_set_sndtimeo(struct sock *sk, s64 secs)
807{
808 lock_sock(sk);
809 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1)
810 sk->sk_sndtimeo = secs * HZ;
811 else
812 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
813 release_sock(sk);
814}
815EXPORT_SYMBOL(sock_set_sndtimeo);
816
783da70e
CH		 817static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns)
818{
819 if (val) {
820 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new);
821 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns);
822 sock_set_flag(sk, SOCK_RCVTSTAMP);
823 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
824 } else {
825 sock_reset_flag(sk, SOCK_RCVTSTAMP);
826 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
783da70e
CH		 827 }
828}
829
830void sock_enable_timestamps(struct sock *sk)
831{
832 lock_sock(sk);
833 __sock_set_timestamps(sk, true, false, true);
834 release_sock(sk);
835}
836EXPORT_SYMBOL(sock_enable_timestamps);
837
371087aa
FW		 838void sock_set_timestamp(struct sock *sk, int optname, bool valbool)
839{
840 switch (optname) {
841 case SO_TIMESTAMP_OLD:
842 __sock_set_timestamps(sk, valbool, false, false);
843 break;
844 case SO_TIMESTAMP_NEW:
845 __sock_set_timestamps(sk, valbool, true, false);
846 break;
847 case SO_TIMESTAMPNS_OLD:
848 __sock_set_timestamps(sk, valbool, false, true);
849 break;
850 case SO_TIMESTAMPNS_NEW:
851 __sock_set_timestamps(sk, valbool, true, true);
852 break;
853 }
854}
855
d463126e
YL		 856static int sock_timestamping_bind_phc(struct sock *sk, int phc_index)
857{
858 struct net *net = sock_net(sk);
859 struct net_device *dev = NULL;
860 bool match = false;
861 int *vclock_index;
862 int i, num;
863
864 if (sk->sk_bound_dev_if)
865 dev = dev_get_by_index(net, sk->sk_bound_dev_if);
866
867 if (!dev) {
868 pr_err("%s: sock not bind to device\n", __func__);
869 return -EOPNOTSUPP;
870 }
871
872 num = ethtool_get_phc_vclocks(dev, &vclock_index);
2a4d75bf
ML		 873 dev_put(dev);
874
d463126e
YL		 875 for (i = 0; i < num; i++) {
876 if (*(vclock_index + i) == phc_index) {
877 match = true;
878 break;
879 }
880 }
881
882 if (num > 0)
883 kfree(vclock_index);
884
885 if (!match)
886 return -EINVAL;
887
888 sk->sk_bind_phc = phc_index;
889
890 return 0;
891}
892
893int sock_set_timestamping(struct sock *sk, int optname,
894 struct so_timestamping timestamping)
ced122d9 895{
d463126e
YL		 896 int val = timestamping.flags;
897 int ret;
898
ced122d9
FW		 899 if (val & ~SOF_TIMESTAMPING_MASK)
900 return -EINVAL;
901
902 if (val & SOF_TIMESTAMPING_OPT_ID &&
903 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
42f67eea 904 if (sk_is_tcp(sk)) {
ced122d9
FW		 905 if ((1 << sk->sk_state) &
906 (TCPF_CLOSE | TCPF_LISTEN))
907 return -EINVAL;
a1cdec57 908 atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una);
ced122d9 909 } else {
a1cdec57 910 atomic_set(&sk->sk_tskey, 0);
ced122d9
FW		 911 }
912 }
913
914 if (val & SOF_TIMESTAMPING_OPT_STATS &&
915 !(val & SOF_TIMESTAMPING_OPT_TSONLY))
916 return -EINVAL;
917
d463126e
YL		 918 if (val & SOF_TIMESTAMPING_BIND_PHC) {
919 ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc);
920 if (ret)
921 return ret;
922 }
923
ced122d9
FW		 924 sk->sk_tsflags = val;
925 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW);
926
927 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
928 sock_enable_timestamp(sk,
929 SOCK_TIMESTAMPING_RX_SOFTWARE);
930 else
931 sock_disable_timestamp(sk,
932 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
933 return 0;
934}
935
ce3d9544
CH		 936void sock_set_keepalive(struct sock *sk)
937{
938 lock_sock(sk);
939 if (sk->sk_prot->keepalive)
940 sk->sk_prot->keepalive(sk, true);
941 sock_valbool_flag(sk, SOCK_KEEPOPEN, true);
942 release_sock(sk);
943}
944EXPORT_SYMBOL(sock_set_keepalive);
945
26cfabf9
CH		 946static void __sock_set_rcvbuf(struct sock *sk, int val)
947{
948 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it
949 * as a negative value.
950 */
951 val = min_t(int, val, INT_MAX / 2);
952 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
953
954 /* We double it on the way in to account for "struct sk_buff" etc.
955 * overhead. Applications assume that the SO_RCVBUF setting they make
956 * will allow that much actual data to be received on that socket.
957 *
958 * Applications are unaware that "struct sk_buff" and other overheads
959 * allocate from the receive buffer during socket buffer allocation.
960 *
961 * And after considering the possible alternatives, returning the value
962 * we actually used in getsockopt is the most desirable behavior.
963 */
964 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF));
965}
966
967void sock_set_rcvbuf(struct sock *sk, int val)
968{
969 lock_sock(sk);
970 __sock_set_rcvbuf(sk, val);
971 release_sock(sk);
972}
973EXPORT_SYMBOL(sock_set_rcvbuf);
974
dd9082f4
AA		 975static void __sock_set_mark(struct sock *sk, u32 val)
976{
977 if (val != sk->sk_mark) {
978 sk->sk_mark = val;
979 sk_dst_reset(sk);
980 }
981}
982
84d1c617
AA		 983void sock_set_mark(struct sock *sk, u32 val)
984{
985 lock_sock(sk);
dd9082f4 986 __sock_set_mark(sk, val);
84d1c617
AA		 987 release_sock(sk);
988}
989EXPORT_SYMBOL(sock_set_mark);
990
2bb2f5fb
WW		 991static void sock_release_reserved_memory(struct sock *sk, int bytes)
992{
993 /* Round down bytes to multiple of pages */
994 bytes &= ~(SK_MEM_QUANTUM - 1);
995
996 WARN_ON(bytes > sk->sk_reserved_mem);
997 sk->sk_reserved_mem -= bytes;
998 sk_mem_reclaim(sk);
999}
1000
1001static int sock_reserve_memory(struct sock *sk, int bytes)
1002{
1003 long allocated;
1004 bool charged;
1005 int pages;
1006
d00c8ee3 1007 if (!mem_cgroup_sockets_enabled || !sk->sk_memcg || !sk_has_account(sk))
2bb2f5fb
WW		 1008 return -EOPNOTSUPP;
1009
1010 if (!bytes)
1011 return 0;
1012
1013 pages = sk_mem_pages(bytes);
1014
1015 /* pre-charge to memcg */
1016 charged = mem_cgroup_charge_skmem(sk->sk_memcg, pages,
1017 GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1018 if (!charged)
1019 return -ENOMEM;
1020
1021 /* pre-charge to forward_alloc */
1022 allocated = sk_memory_allocated_add(sk, pages);
1023 /* If the system goes into memory pressure with this
1024 * precharge, give up and return error.
1025 */
1026 if (allocated > sk_prot_mem_limits(sk, 1)) {
1027 sk_memory_allocated_sub(sk, pages);
1028 mem_cgroup_uncharge_skmem(sk->sk_memcg, pages);
1029 return -ENOMEM;
1030 }
1031 sk->sk_forward_alloc += pages << SK_MEM_QUANTUM_SHIFT;
1032
1033 sk->sk_reserved_mem += pages << SK_MEM_QUANTUM_SHIFT;
1034
1035 return 0;
1036}
1037
1da177e4
LT		 1038/*
1039 * This is meant for all protocols to use and covers goings on
1040 * at the socket level. Everything here is generic.
1041 */
1042
1043int sock_setsockopt(struct socket *sock, int level, int optname,
c8c1bbb6 1044 sockptr_t optval, unsigned int optlen)
1da177e4 1045{
d463126e 1046 struct so_timestamping timestamping;
80b14dee 1047 struct sock_txtime sk_txtime;
2a91525c 1048 struct sock *sk = sock->sk;
1da177e4
LT		 1049 int val;
1050 int valbool;
1051 struct linger ling;
1052 int ret = 0;
4ec93edb 1053
1da177e4
LT		 1054 /*
1055 * Options without arguments
1056 */
1057
4878809f 1058 if (optname == SO_BINDTODEVICE)
c91f6df2 1059 return sock_setbindtodevice(sk, optval, optlen);
4878809f 1060
e71a4783
SH		 1061 if (optlen < sizeof(int))
1062 return -EINVAL;
4ec93edb 1063
c8c1bbb6 1064 if (copy_from_sockptr(&val, optval, sizeof(val)))
1da177e4 1065 return -EFAULT;
4ec93edb 1066
2a91525c 1067 valbool = val ? 1 : 0;
1da177e4
LT		 1068
1069 lock_sock(sk);
1070
2a91525c 1071 switch (optname) {
e71a4783 1072 case SO_DEBUG:
2a91525c 1073 if (val && !capable(CAP_NET_ADMIN))
e71a4783 1074 ret = -EACCES;
2a91525c 1075 else
c0ef877b 1076 sock_valbool_flag(sk, SOCK_DBG, valbool);
e71a4783
SH		 1077 break;
1078 case SO_REUSEADDR:
cdb8744d 1079 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
e71a4783 1080 break;
055dc21a
TH		 1081 case SO_REUSEPORT:
1082 sk->sk_reuseport = valbool;
1083 break;
e71a4783 1084 case SO_TYPE:
49c794e9 1085 case SO_PROTOCOL:
0d6038ee 1086 case SO_DOMAIN:
e71a4783
SH		 1087 case SO_ERROR:
1088 ret = -ENOPROTOOPT;
1089 break;
1090 case SO_DONTROUTE:
c0ef877b 1091 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
0fbe82e6 1092 sk_dst_reset(sk);
e71a4783
SH		 1093 break;
1094 case SO_BROADCAST:
1095 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
1096 break;
1097 case SO_SNDBUF:
1098 /* Don't error on this BSD doesn't and if you think
82981930
ED		 1099 * about it this is right. Otherwise apps have to
1100 * play 'guess the biggest size' games. RCVBUF/SNDBUF
1101 * are treated in BSD as hints
1102 */
1103 val = min_t(u32, val, sysctl_wmem_max);
b0573dea 1104set_sndbuf:
4057765f
GN		 1105 /* Ensure val * 2 fits into an int, to prevent max_t()
1106 * from treating it as a negative value.
1107 */
1108 val = min_t(int, val, INT_MAX / 2);
e71a4783 1109 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
e292f05e
ED		 1110 WRITE_ONCE(sk->sk_sndbuf,
1111 max_t(int, val * 2, SOCK_MIN_SNDBUF));
82981930 1112 /* Wake up sending tasks if we upped the value. */
e71a4783
SH		 1113 sk->sk_write_space(sk);
1114 break;
1da177e4 1115
e71a4783
SH		 1116 case SO_SNDBUFFORCE:
1117 if (!capable(CAP_NET_ADMIN)) {
1118 ret = -EPERM;
1119 break;
1120 }
4057765f
GN		 1121
1122 /* No negative values (to prevent underflow, as val will be
1123 * multiplied by 2).
1124 */
1125 if (val < 0)
1126 val = 0;
e71a4783 1127 goto set_sndbuf;
b0573dea 1128
e71a4783
SH		 1129 case SO_RCVBUF:
1130 /* Don't error on this BSD doesn't and if you think
82981930
ED		 1131 * about it this is right. Otherwise apps have to
1132 * play 'guess the biggest size' games. RCVBUF/SNDBUF
1133 * are treated in BSD as hints
1134 */
26cfabf9 1135 __sock_set_rcvbuf(sk, min_t(u32, val, sysctl_rmem_max));
e71a4783
SH		 1136 break;
1137
1138 case SO_RCVBUFFORCE:
1139 if (!capable(CAP_NET_ADMIN)) {
1140 ret = -EPERM;
1da177e4 1141 break;
e71a4783 1142 }
4057765f
GN		 1143
1144 /* No negative values (to prevent underflow, as val will be
1145 * multiplied by 2).
1146 */
26cfabf9
CH		 1147 __sock_set_rcvbuf(sk, max(val, 0));
1148 break;
1da177e4 1149
e71a4783 1150 case SO_KEEPALIVE:
4b9d07a4
UB		 1151 if (sk->sk_prot->keepalive)
1152 sk->sk_prot->keepalive(sk, valbool);
e71a4783
SH		 1153 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
1154 break;
1155
1156 case SO_OOBINLINE:
1157 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
1158 break;
1159
1160 case SO_NO_CHECK:
28448b80 1161 sk->sk_no_check_tx = valbool;
e71a4783
SH		 1162 break;
1163
1164 case SO_PRIORITY:
5e1fccc0 1165 if ((val >= 0 && val <= 6) ||
a1b519b7 1166 ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) ||
5e1fccc0 1167 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
e71a4783
SH		 1168 sk->sk_priority = val;
1169 else
1170 ret = -EPERM;
1171 break;
1172
1173 case SO_LINGER:
1174 if (optlen < sizeof(ling)) {
1175 ret = -EINVAL; /* 1003.1g */
1da177e4 1176 break;
e71a4783 1177 }
c8c1bbb6 1178 if (copy_from_sockptr(&ling, optval, sizeof(ling))) {
e71a4783 1179 ret = -EFAULT;
1da177e4 1180 break;
e71a4783
SH		 1181 }
1182 if (!ling.l_onoff)
1183 sock_reset_flag(sk, SOCK_LINGER);
1184 else {
1da177e4 1185#if (BITS_PER_LONG == 32)
e71a4783
SH		 1186 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
1187 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
1da177e4 1188 else
e71a4783
SH		 1189#endif
1190 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
1191 sock_set_flag(sk, SOCK_LINGER);
1192 }
1193 break;
1194
1195 case SO_BSDCOMPAT:
e71a4783
SH		 1196 break;
1197
1198 case SO_PASSCRED:
1199 if (valbool)
1200 set_bit(SOCK_PASSCRED, &sock->flags);
1201 else
1202 clear_bit(SOCK_PASSCRED, &sock->flags);
1203 break;
1204
7f1bc6e9 1205 case SO_TIMESTAMP_OLD:
887feae3 1206 case SO_TIMESTAMP_NEW:
7f1bc6e9 1207 case SO_TIMESTAMPNS_OLD:
887feae3 1208 case SO_TIMESTAMPNS_NEW:
81b4a0cc 1209 sock_set_timestamp(sk, optname, valbool);
e71a4783 1210 break;
ced122d9 1211
9718475e 1212 case SO_TIMESTAMPING_NEW:
7f1bc6e9 1213 case SO_TIMESTAMPING_OLD:
d463126e
YL		 1214 if (optlen == sizeof(timestamping)) {
1215 if (copy_from_sockptr(&timestamping, optval,
271dbc31
DC		 1216 sizeof(timestamping))) {
1217 ret = -EFAULT;
1218 break;
1219 }
d463126e
YL		 1220 } else {
1221 memset(&timestamping, 0, sizeof(timestamping));
1222 timestamping.flags = val;
1223 }
1224 ret = sock_set_timestamping(sk, optname, timestamping);
20d49473
PO		 1225 break;
1226
e71a4783
SH		 1227 case SO_RCVLOWAT:
1228 if (val < 0)
1229 val = INT_MAX;
d1361840
ED		 1230 if (sock->ops->set_rcvlowat)
1231 ret = sock->ops->set_rcvlowat(sk, val);
1232 else
eac66402 1233 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
e71a4783
SH		 1234 break;
1235
45bdc661 1236 case SO_RCVTIMEO_OLD:
a9beb86a 1237 case SO_RCVTIMEO_NEW:
c8c1bbb6 1238 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval,
c34645ac 1239 optlen, optname == SO_RCVTIMEO_OLD);
e71a4783
SH		 1240 break;
1241
45bdc661 1242 case SO_SNDTIMEO_OLD:
a9beb86a 1243 case SO_SNDTIMEO_NEW:
c8c1bbb6 1244 ret = sock_set_timeout(&sk->sk_sndtimeo, optval,
c34645ac 1245 optlen, optname == SO_SNDTIMEO_OLD);
e71a4783 1246 break;
1da177e4 1247
4d295e54
CH		 1248 case SO_ATTACH_FILTER: {
1249 struct sock_fprog fprog;
e71a4783 1250
c8c1bbb6 1251 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
4d295e54 1252 if (!ret)
e71a4783 1253 ret = sk_attach_filter(&fprog, sk);
e71a4783 1254 break;
4d295e54 1255 }
89aa0758
AS		 1256 case SO_ATTACH_BPF:
1257 ret = -EINVAL;
1258 if (optlen == sizeof(u32)) {
1259 u32 ufd;
1260
1261 ret = -EFAULT;
c8c1bbb6 1262 if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
89aa0758
AS		 1263 break;
1264
1265 ret = sk_attach_bpf(ufd, sk);
1266 }
1267 break;
1268
4d295e54
CH		 1269 case SO_ATTACH_REUSEPORT_CBPF: {
1270 struct sock_fprog fprog;
538950a1 1271
c8c1bbb6 1272 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
4d295e54 1273 if (!ret)
538950a1 1274 ret = sk_reuseport_attach_filter(&fprog, sk);
538950a1 1275 break;
4d295e54 1276 }
538950a1
CG		 1277 case SO_ATTACH_REUSEPORT_EBPF:
1278 ret = -EINVAL;
1279 if (optlen == sizeof(u32)) {
1280 u32 ufd;
1281
1282 ret = -EFAULT;
c8c1bbb6 1283 if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
538950a1
CG		 1284 break;
1285
1286 ret = sk_reuseport_attach_bpf(ufd, sk);
1287 }
1288 break;
1289
99f3a064
MKL		 1290 case SO_DETACH_REUSEPORT_BPF:
1291 ret = reuseport_detach_prog(sk);
1292 break;
1293
e71a4783 1294 case SO_DETACH_FILTER:
55b33325 1295 ret = sk_detach_filter(sk);
e71a4783 1296 break;
1da177e4 1297
d59577b6
VB		 1298 case SO_LOCK_FILTER:
1299 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
1300 ret = -EPERM;
1301 else
1302 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
1303 break;
1304
e71a4783
SH		 1305 case SO_PASSSEC:
1306 if (valbool)
1307 set_bit(SOCK_PASSSEC, &sock->flags);
1308 else
1309 clear_bit(SOCK_PASSSEC, &sock->flags);
1310 break;
4a19ec58 1311 case SO_MARK:
079925cc
MŻ		 1312 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
1313 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
4a19ec58 1314 ret = -EPERM;
dd9082f4 1315 break;
50254256 1316 }
dd9082f4
AA		 1317
1318 __sock_set_mark(sk, val);
4a19ec58 1319 break;
6fd1d51c 1320 case SO_RCVMARK:
1f86123b
EB		 1321 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
1322 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1323 ret = -EPERM;
1324 break;
1325 }
1326
6fd1d51c
EM		 1327 sock_valbool_flag(sk, SOCK_RCVMARK, valbool);
1328 break;
877ce7c1 1329
3b885787 1330 case SO_RXQ_OVFL:
8083f0fc 1331 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
3b885787 1332 break;
6e3e939f
JB		 1333
1334 case SO_WIFI_STATUS:
1335 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1336 break;
1337
ef64a54f
PE		 1338 case SO_PEEK_OFF:
1339 if (sock->ops->set_peek_off)
12663bfc 1340 ret = sock->ops->set_peek_off(sk, val);
ef64a54f
PE		 1341 else
1342 ret = -EOPNOTSUPP;
1343 break;
3bdc0eba
BG		 1344
1345 case SO_NOFCS:
1346 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1347 break;
1348
7d4c04fc
KJ		 1349 case SO_SELECT_ERR_QUEUE:
1350 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1351 break;
1352
e0d1095a 1353#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1354 case SO_BUSY_POLL:
dafcc438
ET		 1355 /* allow unprivileged users to decrease the value */
1356 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
1357 ret = -EPERM;
1358 else {
1359 if (val < 0)
1360 ret = -EINVAL;
1361 else
0dbffbb5 1362 WRITE_ONCE(sk->sk_ll_usec, val);
dafcc438
ET		 1363 }
1364 break;
7fd3253a
BT		 1365 case SO_PREFER_BUSY_POLL:
1366 if (valbool && !capable(CAP_NET_ADMIN))
1367 ret = -EPERM;
1368 else
1369 WRITE_ONCE(sk->sk_prefer_busy_poll, valbool);
1370 break;
7c951caf
BT		 1371 case SO_BUSY_POLL_BUDGET:
1372 if (val > READ_ONCE(sk->sk_busy_poll_budget) && !capable(CAP_NET_ADMIN)) {
1373 ret = -EPERM;
1374 } else {
1375 if (val < 0 || val > U16_MAX)
1376 ret = -EINVAL;
1377 else
1378 WRITE_ONCE(sk->sk_busy_poll_budget, val);
1379 }
1380 break;
dafcc438 1381#endif
62748f32
ED		 1382
1383 case SO_MAX_PACING_RATE:
6bdef102 1384 {
700465fd 1385 unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val;
6bdef102
ED		 1386
1387 if (sizeof(ulval) != sizeof(val) &&
1388 optlen >= sizeof(ulval) &&
c8c1bbb6 1389 copy_from_sockptr(&ulval, optval, sizeof(ulval))) {
6bdef102
ED		 1390 ret = -EFAULT;
1391 break;
1392 }
1393 if (ulval != ~0UL)
218af599
ED		 1394 cmpxchg(&sk->sk_pacing_status,
1395 SK_PACING_NONE,
1396 SK_PACING_NEEDED);
6bdef102
ED		 1397 sk->sk_max_pacing_rate = ulval;
1398 sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval);
62748f32 1399 break;
6bdef102 1400 }
70da268b 1401 case SO_INCOMING_CPU:
7170a977 1402 WRITE_ONCE(sk->sk_incoming_cpu, val);
70da268b
ED		 1403 break;
1404
a87cb3e4
TH		 1405 case SO_CNX_ADVICE:
1406 if (val == 1)
1407 dst_negative_advice(sk);
1408 break;
76851d12
WB		 1409
1410 case SO_ZEROCOPY:
28190752 1411 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
42f67eea 1412 if (!(sk_is_tcp(sk) ||
b5947e5d
WB		 1413 (sk->sk_type == SOCK_DGRAM &&
1414 sk->sk_protocol == IPPROTO_UDP)))
869420a8 1415 ret = -EOPNOTSUPP;
28190752 1416 } else if (sk->sk_family != PF_RDS) {
869420a8 1417 ret = -EOPNOTSUPP;
28190752
SV		 1418 }
1419 if (!ret) {
1420 if (val < 0 || val > 1)
1421 ret = -EINVAL;
1422 else
1423 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
28190752 1424 }
334e6413
JSP		 1425 break;
1426
80b14dee 1427 case SO_TXTIME:
790709f2 1428 if (optlen != sizeof(struct sock_txtime)) {
80b14dee 1429 ret = -EINVAL;
790709f2 1430 break;
c8c1bbb6 1431 } else if (copy_from_sockptr(&sk_txtime, optval,
80b14dee
RC		 1432 sizeof(struct sock_txtime))) {
1433 ret = -EFAULT;
790709f2 1434 break;
80b14dee
RC		 1435 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
1436 ret = -EINVAL;
790709f2
ED		 1437 break;
1438 }
1439 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet
1440 * scheduler has enough safe guards.
1441 */
1442 if (sk_txtime.clockid != CLOCK_MONOTONIC &&
1443 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1444 ret = -EPERM;
1445 break;
80b14dee 1446 }
790709f2
ED		 1447 sock_valbool_flag(sk, SOCK_TXTIME, true);
1448 sk->sk_clockid = sk_txtime.clockid;
1449 sk->sk_txtime_deadline_mode =
1450 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
1451 sk->sk_txtime_report_errors =
1452 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
80b14dee
RC		 1453 break;
1454
f5dd3d0c 1455 case SO_BINDTOIFINDEX:
7594888c 1456 ret = sock_bindtoindex_locked(sk, val);
f5dd3d0c
DH		 1457 break;
1458
04190bf8
PT		 1459 case SO_BUF_LOCK:
1460 if (val & ~SOCK_BUF_LOCK_MASK) {
1461 ret = -EINVAL;
1462 break;
1463 }
1464 sk->sk_userlocks = val | (sk->sk_userlocks &
1465 ~SOCK_BUF_LOCK_MASK);
1466 break;
1467
2bb2f5fb
WW		 1468 case SO_RESERVE_MEM:
1469 {
1470 int delta;
1471
1472 if (val < 0) {
1473 ret = -EINVAL;
1474 break;
1475 }
1476
1477 delta = val - sk->sk_reserved_mem;
1478 if (delta < 0)
1479 sock_release_reserved_memory(sk, -delta);
1480 else
1481 ret = sock_reserve_memory(sk, delta);
1482 break;
1483 }
1484
26859240
AK		 1485 case SO_TXREHASH:
1486 if (val < -1 || val > 1) {
1487 ret = -EINVAL;
1488 break;
1489 }
cb6cd2ce
AK		 1490 /* Paired with READ_ONCE() in tcp_rtx_synack() */
1491 WRITE_ONCE(sk->sk_txrehash, (u8)val);
26859240
AK		 1492 break;
1493
e71a4783
SH		 1494 default:
1495 ret = -ENOPROTOOPT;
1496 break;
4ec93edb 1497 }
1da177e4
LT		 1498 release_sock(sk);
1499 return ret;
1500}
2a91525c 1501EXPORT_SYMBOL(sock_setsockopt);
1da177e4 1502
35306eb2
ED		 1503static const struct cred *sk_get_peer_cred(struct sock *sk)
1504{
1505 const struct cred *cred;
1506
1507 spin_lock(&sk->sk_peer_lock);
1508 cred = get_cred(sk->sk_peer_cred);
1509 spin_unlock(&sk->sk_peer_lock);
1510
1511 return cred;
1512}
1da177e4 1513
8f09898b 1514static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1515 struct ucred *ucred)
3f551f94
EB		 1516{
1517 ucred->pid = pid_vnr(pid);
1518 ucred->uid = ucred->gid = -1;
1519 if (cred) {
1520 struct user_namespace *current_ns = current_user_ns();
1521
b2e4f544
EB		 1522 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1523 ucred->gid = from_kgid_munged(current_ns, cred->egid);
3f551f94
EB		 1524 }
1525}
1526
28b5ba2a
DH		 1527static int groups_to_user(gid_t __user *dst, const struct group_info *src)
1528{
1529 struct user_namespace *user_ns = current_user_ns();
1530 int i;
1531
1532 for (i = 0; i < src->ngroups; i++)
1533 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
1534 return -EFAULT;
1535
1536 return 0;
1537}
1538
1da177e4
LT		 1539int sock_getsockopt(struct socket *sock, int level, int optname,
1540 char __user *optval, int __user *optlen)
1541{
1542 struct sock *sk = sock->sk;
4ec93edb 1543
e71a4783 1544 union {
4ec93edb 1545 int val;
5daab9db 1546 u64 val64;
677f136c 1547 unsigned long ulval;
4ec93edb 1548 struct linger ling;
fe0c72f3
AB		 1549 struct old_timeval32 tm32;
1550 struct __kernel_old_timeval tm;
a9beb86a 1551 struct __kernel_sock_timeval stm;
80b14dee 1552 struct sock_txtime txtime;
d463126e 1553 struct so_timestamping timestamping;
1da177e4 1554 } v;
4ec93edb 1555
4d0392be 1556 int lv = sizeof(int);
1da177e4 1557 int len;
4ec93edb 1558
e71a4783 1559 if (get_user(len, optlen))
4ec93edb 1560 return -EFAULT;
e71a4783 1561 if (len < 0)
1da177e4 1562 return -EINVAL;
4ec93edb 1563
50fee1de 1564 memset(&v, 0, sizeof(v));
df0bca04 1565
2a91525c 1566 switch (optname) {
e71a4783
SH		 1567 case SO_DEBUG:
1568 v.val = sock_flag(sk, SOCK_DBG);
1569 break;
1570
1571 case SO_DONTROUTE:
1572 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1573 break;
1574
1575 case SO_BROADCAST:
1b23a5df 1576 v.val = sock_flag(sk, SOCK_BROADCAST);
e71a4783
SH		 1577 break;
1578
1579 case SO_SNDBUF:
1580 v.val = sk->sk_sndbuf;
1581 break;
1582
1583 case SO_RCVBUF:
1584 v.val = sk->sk_rcvbuf;
1585 break;
1586
1587 case SO_REUSEADDR:
1588 v.val = sk->sk_reuse;
1589 break;
1590
055dc21a
TH		 1591 case SO_REUSEPORT:
1592 v.val = sk->sk_reuseport;
1593 break;
1594
e71a4783 1595 case SO_KEEPALIVE:
1b23a5df 1596 v.val = sock_flag(sk, SOCK_KEEPOPEN);
e71a4783
SH		 1597 break;
1598
1599 case SO_TYPE:
1600 v.val = sk->sk_type;
1601 break;
1602
49c794e9
JE		 1603 case SO_PROTOCOL:
1604 v.val = sk->sk_protocol;
1605 break;
1606
0d6038ee
JE		 1607 case SO_DOMAIN:
1608 v.val = sk->sk_family;
1609 break;
1610
e71a4783
SH		 1611 case SO_ERROR:
1612 v.val = -sock_error(sk);
2a91525c 1613 if (v.val == 0)
e71a4783
SH		 1614 v.val = xchg(&sk->sk_err_soft, 0);
1615 break;
1616
1617 case SO_OOBINLINE:
1b23a5df 1618 v.val = sock_flag(sk, SOCK_URGINLINE);
e71a4783
SH		 1619 break;
1620
1621 case SO_NO_CHECK:
28448b80 1622 v.val = sk->sk_no_check_tx;
e71a4783
SH		 1623 break;
1624
1625 case SO_PRIORITY:
1626 v.val = sk->sk_priority;
1627 break;
1628
1629 case SO_LINGER:
1630 lv = sizeof(v.ling);
1b23a5df 1631 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
e71a4783
SH		 1632 v.ling.l_linger = sk->sk_lingertime / HZ;
1633 break;
1634
1635 case SO_BSDCOMPAT:
e71a4783
SH		 1636 break;
1637
7f1bc6e9 1638 case SO_TIMESTAMP_OLD:
92f37fd2 1639 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
887feae3 1640 !sock_flag(sk, SOCK_TSTAMP_NEW) &&
92f37fd2
ED		 1641 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1642 break;
1643
7f1bc6e9 1644 case SO_TIMESTAMPNS_OLD:
887feae3
DD		 1645 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW);
1646 break;
1647
1648 case SO_TIMESTAMP_NEW:
1649 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW);
1650 break;
1651
1652 case SO_TIMESTAMPNS_NEW:
1653 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW);
e71a4783
SH		 1654 break;
1655
7f1bc6e9 1656 case SO_TIMESTAMPING_OLD:
d463126e
YL		 1657 lv = sizeof(v.timestamping);
1658 v.timestamping.flags = sk->sk_tsflags;
1659 v.timestamping.bind_phc = sk->sk_bind_phc;
20d49473
PO		 1660 break;
1661
a9beb86a
DD		 1662 case SO_RCVTIMEO_OLD:
1663 case SO_RCVTIMEO_NEW:
1664 lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname);
e71a4783
SH		 1665 break;
1666
a9beb86a
DD		 1667 case SO_SNDTIMEO_OLD:
1668 case SO_SNDTIMEO_NEW:
1669 lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname);
e71a4783 1670 break;
1da177e4 1671
e71a4783
SH		 1672 case SO_RCVLOWAT:
1673 v.val = sk->sk_rcvlowat;
1674 break;
1da177e4 1675
e71a4783 1676 case SO_SNDLOWAT:
2a91525c 1677 v.val = 1;
e71a4783 1678 break;
1da177e4 1679
e71a4783 1680 case SO_PASSCRED:
82981930 1681 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
e71a4783 1682 break;
1da177e4 1683
e71a4783 1684 case SO_PEERCRED:
109f6e39
EB		 1685 {
1686 struct ucred peercred;
1687 if (len > sizeof(peercred))
1688 len = sizeof(peercred);
35306eb2
ED		 1689
1690 spin_lock(&sk->sk_peer_lock);
109f6e39 1691 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
35306eb2
ED		 1692 spin_unlock(&sk->sk_peer_lock);
1693
109f6e39 1694 if (copy_to_user(optval, &peercred, len))
e71a4783
SH		 1695 return -EFAULT;
1696 goto lenout;
109f6e39 1697 }
1da177e4 1698
28b5ba2a
DH		 1699 case SO_PEERGROUPS:
1700 {
35306eb2 1701 const struct cred *cred;
28b5ba2a
DH		 1702 int ret, n;
1703
35306eb2
ED		 1704 cred = sk_get_peer_cred(sk);
1705 if (!cred)
28b5ba2a
DH		 1706 return -ENODATA;
1707
35306eb2 1708 n = cred->group_info->ngroups;
28b5ba2a
DH		 1709 if (len < n * sizeof(gid_t)) {
1710 len = n * sizeof(gid_t);
35306eb2 1711 put_cred(cred);
28b5ba2a
DH		 1712 return put_user(len, optlen) ? -EFAULT : -ERANGE;
1713 }
1714 len = n * sizeof(gid_t);
1715
35306eb2
ED		 1716 ret = groups_to_user((gid_t __user *)optval, cred->group_info);
1717 put_cred(cred);
28b5ba2a
DH		 1718 if (ret)
1719 return ret;
1720 goto lenout;
1721 }
1722
e71a4783
SH		 1723 case SO_PEERNAME:
1724 {
1725 char address[128];
1726
9b2c45d4
DV		 1727 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2);
1728 if (lv < 0)
e71a4783
SH		 1729 return -ENOTCONN;
1730 if (lv < len)
1731 return -EINVAL;
1732 if (copy_to_user(optval, address, len))
1733 return -EFAULT;
1734 goto lenout;
1735 }
1da177e4 1736
e71a4783
SH		 1737 /* Dubious BSD thing... Probably nobody even uses it, but
1738 * the UNIX standard wants it for whatever reason... -DaveM
1739 */
1740 case SO_ACCEPTCONN:
1741 v.val = sk->sk_state == TCP_LISTEN;
1742 break;
1da177e4 1743
e71a4783 1744 case SO_PASSSEC:
82981930 1745 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
e71a4783 1746 break;
877ce7c1 1747
e71a4783
SH		 1748 case SO_PEERSEC:
1749 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1da177e4 1750
4a19ec58
LAT		 1751 case SO_MARK:
1752 v.val = sk->sk_mark;
1753 break;
1754
6fd1d51c
EM		 1755 case SO_RCVMARK:
1756 v.val = sock_flag(sk, SOCK_RCVMARK);
1757 break;
1758
3b885787 1759 case SO_RXQ_OVFL:
1b23a5df 1760 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
3b885787
NH		 1761 break;
1762
6e3e939f 1763 case SO_WIFI_STATUS:
1b23a5df 1764 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
6e3e939f
JB		 1765 break;
1766
ef64a54f
PE		 1767 case SO_PEEK_OFF:
1768 if (!sock->ops->set_peek_off)
1769 return -EOPNOTSUPP;
1770
1771 v.val = sk->sk_peek_off;
1772 break;
bc2f7996 1773 case SO_NOFCS:
1b23a5df 1774 v.val = sock_flag(sk, SOCK_NOFCS);
bc2f7996 1775 break;
c91f6df2 1776
f7b86bfe 1777 case SO_BINDTODEVICE:
c91f6df2
BH		 1778 return sock_getbindtodevice(sk, optval, optlen, len);
1779
a8fc9277
PE		 1780 case SO_GET_FILTER:
1781 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1782 if (len < 0)
1783 return len;
1784
1785 goto lenout;
c91f6df2 1786
d59577b6
VB		 1787 case SO_LOCK_FILTER:
1788 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1789 break;
1790
ea02f941
MS		 1791 case SO_BPF_EXTENSIONS:
1792 v.val = bpf_tell_extensions();
1793 break;
1794
7d4c04fc
KJ		 1795 case SO_SELECT_ERR_QUEUE:
1796 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1797 break;
1798
e0d1095a 1799#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1800 case SO_BUSY_POLL:
dafcc438
ET		 1801 v.val = sk->sk_ll_usec;
1802 break;
7fd3253a
BT		 1803 case SO_PREFER_BUSY_POLL:
1804 v.val = READ_ONCE(sk->sk_prefer_busy_poll);
1805 break;
dafcc438
ET		 1806#endif
1807
62748f32 1808 case SO_MAX_PACING_RATE:
677f136c
ED		 1809 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) {
1810 lv = sizeof(v.ulval);
1811 v.ulval = sk->sk_max_pacing_rate;
1812 } else {
1813 /* 32bit version */
1814 v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);
1815 }
62748f32
ED		 1816 break;
1817
2c8c56e1 1818 case SO_INCOMING_CPU:
7170a977 1819 v.val = READ_ONCE(sk->sk_incoming_cpu);
2c8c56e1
ED		 1820 break;
1821
a2d133b1
JH		 1822 case SO_MEMINFO:
1823 {
1824 u32 meminfo[SK_MEMINFO_VARS];
1825
a2d133b1
JH		 1826 sk_get_meminfo(sk, meminfo);
1827
1828 len = min_t(unsigned int, len, sizeof(meminfo));
1829 if (copy_to_user(optval, &meminfo, len))
1830 return -EFAULT;
1831
1832 goto lenout;
1833 }
6d433902
SS		 1834
1835#ifdef CONFIG_NET_RX_BUSY_POLL
1836 case SO_INCOMING_NAPI_ID:
1837 v.val = READ_ONCE(sk->sk_napi_id);
1838
1839 /* aggregate non-NAPI IDs down to 0 */
1840 if (v.val < MIN_NAPI_ID)
1841 v.val = 0;
1842
1843 break;
1844#endif
1845
5daab9db
CF		 1846 case SO_COOKIE:
1847 lv = sizeof(u64);
1848 if (len < lv)
1849 return -EINVAL;
1850 v.val64 = sock_gen_cookie(sk);
1851 break;
1852
76851d12
WB		 1853 case SO_ZEROCOPY:
1854 v.val = sock_flag(sk, SOCK_ZEROCOPY);
1855 break;
1856
80b14dee
RC		 1857 case SO_TXTIME:
1858 lv = sizeof(v.txtime);
1859 v.txtime.clockid = sk->sk_clockid;
1860 v.txtime.flags |= sk->sk_txtime_deadline_mode ?
1861 SOF_TXTIME_DEADLINE_MODE : 0;
4b15c707
JSP		 1862 v.txtime.flags |= sk->sk_txtime_report_errors ?
1863 SOF_TXTIME_REPORT_ERRORS : 0;
80b14dee
RC		 1864 break;
1865
f5dd3d0c 1866 case SO_BINDTOIFINDEX:
e5fccaa1 1867 v.val = READ_ONCE(sk->sk_bound_dev_if);
f5dd3d0c
DH		 1868 break;
1869
e8b9eab9
MP		 1870 case SO_NETNS_COOKIE:
1871 lv = sizeof(u64);
1872 if (len != lv)
1873 return -EINVAL;
1874 v.val64 = sock_net(sk)->net_cookie;
1875 break;
1876
04190bf8
PT		 1877 case SO_BUF_LOCK:
1878 v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK;
1879 break;
1880
2bb2f5fb
WW		 1881 case SO_RESERVE_MEM:
1882 v.val = sk->sk_reserved_mem;
1883 break;
1884
26859240
AK		 1885 case SO_TXREHASH:
1886 v.val = sk->sk_txrehash;
1887 break;
1888
e71a4783 1889 default:
443b5991
YH		 1890 /* We implement the SO_SNDLOWAT etc to not be settable
1891 * (1003.1g 7).
1892 */
e71a4783 1893 return -ENOPROTOOPT;
1da177e4 1894 }
e71a4783 1895
1da177e4
LT		 1896 if (len > lv)
1897 len = lv;
1898 if (copy_to_user(optval, &v, len))
1899 return -EFAULT;
1900lenout:
4ec93edb
YH		 1901 if (put_user(len, optlen))
1902 return -EFAULT;
1903 return 0;
1da177e4
LT		 1904}
1905
a5b5bb9a
IM		 1906/*
1907 * Initialize an sk_lock.
1908 *
1909 * (We also register the sk_lock with the lock validator.)
1910 */
b6f99a21 1911static inline void sock_lock_init(struct sock *sk)
a5b5bb9a 1912{
cdfbabfb
DH		 1913 if (sk->sk_kern_sock)
1914 sock_lock_init_class_and_name(
1915 sk,
1916 af_family_kern_slock_key_strings[sk->sk_family],
1917 af_family_kern_slock_keys + sk->sk_family,
1918 af_family_kern_key_strings[sk->sk_family],
1919 af_family_kern_keys + sk->sk_family);
1920 else
1921 sock_lock_init_class_and_name(
1922 sk,
ed07536e
PZ		 1923 af_family_slock_key_strings[sk->sk_family],
1924 af_family_slock_keys + sk->sk_family,
1925 af_family_key_strings[sk->sk_family],
1926 af_family_keys + sk->sk_family);
a5b5bb9a
IM		 1927}
1928
4dc6dc71
ED		 1929/*
1930 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1931 * even temporarly, because of RCU lookups. sk_node should also be left as is.
68835aba 1932 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
4dc6dc71 1933 */
f1a6c4da
PE		 1934static void sock_copy(struct sock *nsk, const struct sock *osk)
1935{
b8e202d1 1936 const struct proto *prot = READ_ONCE(osk->sk_prot);
f1a6c4da
PE		 1937#ifdef CONFIG_SECURITY_NETWORK
1938 void *sptr = nsk->sk_security;
1939#endif
df610cd9
KI		 1940
1941 /* If we move sk_tx_queue_mapping out of the private section,
1942 * we must check if sk_tx_queue_clear() is called after
1943 * sock_copy() in sk_clone_lock().
1944 */
1945 BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) <
1946 offsetof(struct sock, sk_dontcopy_begin) ||
1947 offsetof(struct sock, sk_tx_queue_mapping) >=
1948 offsetof(struct sock, sk_dontcopy_end));
1949
68835aba
ED		 1950 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1951
1952 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
b8e202d1 1953 prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
68835aba 1954
f1a6c4da
PE		 1955#ifdef CONFIG_SECURITY_NETWORK
1956 nsk->sk_security = sptr;
1957 security_sk_clone(osk, nsk);
1958#endif
1959}
1960
2e4afe7b
PE		 1961static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1962 int family)
c308c1b2
PE		 1963{
1964 struct sock *sk;
1965 struct kmem_cache *slab;
1966
1967 slab = prot->slab;
e912b114
ED		 1968 if (slab != NULL) {
1969 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1970 if (!sk)
1971 return sk;
6471384a 1972 if (want_init_on_alloc(priority))
ba2489b0 1973 sk_prot_clear_nulls(sk, prot->obj_size);
fcbdf09d 1974 } else
c308c1b2
PE		 1975 sk = kmalloc(prot->obj_size, priority);
1976
2e4afe7b
PE		 1977 if (sk != NULL) {
1978 if (security_sk_alloc(sk, family, priority))
1979 goto out_free;
1980
1981 if (!try_module_get(prot->owner))
1982 goto out_free_sec;
1983 }
1984
c308c1b2 1985 return sk;
2e4afe7b
PE		 1986
1987out_free_sec:
1988 security_sk_free(sk);
1989out_free:
1990 if (slab != NULL)
1991 kmem_cache_free(slab, sk);
1992 else
1993 kfree(sk);
1994 return NULL;
c308c1b2
PE		 1995}
1996
1997static void sk_prot_free(struct proto *prot, struct sock *sk)
1998{
1999 struct kmem_cache *slab;
2e4afe7b 2000 struct module *owner;
c308c1b2 2001
2e4afe7b 2002 owner = prot->owner;
c308c1b2 2003 slab = prot->slab;
2e4afe7b 2004
bd1060a1 2005 cgroup_sk_free(&sk->sk_cgrp_data);
2d758073 2006 mem_cgroup_sk_free(sk);
2e4afe7b 2007 security_sk_free(sk);
c308c1b2
PE		 2008 if (slab != NULL)
2009 kmem_cache_free(slab, sk);
2010 else
2011 kfree(sk);
2e4afe7b 2012 module_put(owner);
c308c1b2
PE		 2013}
2014
1da177e4
LT		 2015/**
2016 * sk_alloc - All socket objects are allocated here
c4ea43c5 2017 * @net: the applicable net namespace
4dc3b16b
PP		 2018 * @family: protocol family
2019 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
2020 * @prot: struct proto associated with this new sock instance
11aa9c28 2021 * @kern: is this to be a kernel socket?
1da177e4 2022 */
1b8d7ae4 2023struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
11aa9c28 2024 struct proto *prot, int kern)
1da177e4 2025{
c308c1b2 2026 struct sock *sk;
1da177e4 2027
154adbc8 2028 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1da177e4 2029 if (sk) {
154adbc8
PE		 2030 sk->sk_family = family;
2031 /*
2032 * See comment in struct sock definition to understand
2033 * why we need sk_prot_creator -acme
2034 */
2035 sk->sk_prot = sk->sk_prot_creator = prot;
cdfbabfb 2036 sk->sk_kern_sock = kern;
154adbc8 2037 sock_lock_init(sk);
26abe143 2038 sk->sk_net_refcnt = kern ? 0 : 1;
648845ab 2039 if (likely(sk->sk_net_refcnt)) {
ffa84b5f 2040 get_net_track(net, &sk->ns_tracker, priority);
648845ab
TZ		 2041 sock_inuse_add(net, 1);
2042 }
2043
26abe143 2044 sock_net_set(sk, net);
14afee4b 2045 refcount_set(&sk->sk_wmem_alloc, 1);
f8451725 2046
2d758073 2047 mem_cgroup_sk_alloc(sk);
d979a39d 2048 cgroup_sk_alloc(&sk->sk_cgrp_data);
2a56a1fe
TH		 2049 sock_update_classid(&sk->sk_cgrp_data);
2050 sock_update_netprioidx(&sk->sk_cgrp_data);
41b14fb8 2051 sk_tx_queue_clear(sk);
1da177e4 2052 }
a79af59e 2053
2e4afe7b 2054 return sk;
1da177e4 2055}
2a91525c 2056EXPORT_SYMBOL(sk_alloc);
1da177e4 2057
a4298e45
ED		 2058/* Sockets having SOCK_RCU_FREE will call this function after one RCU
2059 * grace period. This is the case for UDP sockets and TCP listeners.
2060 */
2061static void __sk_destruct(struct rcu_head *head)
1da177e4 2062{
a4298e45 2063 struct sock *sk = container_of(head, struct sock, sk_rcu);
1da177e4 2064 struct sk_filter *filter;
1da177e4
LT		 2065
2066 if (sk->sk_destruct)
2067 sk->sk_destruct(sk);
2068
a898def2 2069 filter = rcu_dereference_check(sk->sk_filter,
14afee4b 2070 refcount_read(&sk->sk_wmem_alloc) == 0);
1da177e4 2071 if (filter) {
309dd5fc 2072 sk_filter_uncharge(sk, filter);
a9b3cd7f 2073 RCU_INIT_POINTER(sk->sk_filter, NULL);
1da177e4
LT		 2074 }
2075
08e29af3 2076 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1da177e4 2077
6ac99e8f
MKL		 2078#ifdef CONFIG_BPF_SYSCALL
2079 bpf_sk_storage_free(sk);
2080#endif
2081
1da177e4 2082 if (atomic_read(&sk->sk_omem_alloc))
e005d193
JP		 2083 pr_debug("%s: optmem leakage (%d bytes) detected\n",
2084 __func__, atomic_read(&sk->sk_omem_alloc));
1da177e4 2085
22a0e18e
ED		 2086 if (sk->sk_frag.page) {
2087 put_page(sk->sk_frag.page);
2088 sk->sk_frag.page = NULL;
2089 }
2090
35306eb2
ED		 2091 /* We do not need to acquire sk->sk_peer_lock, we are the last user. */
2092 put_cred(sk->sk_peer_cred);
109f6e39 2093 put_pid(sk->sk_peer_pid);
35306eb2 2094
26abe143 2095 if (likely(sk->sk_net_refcnt))
ffa84b5f 2096 put_net_track(sock_net(sk), &sk->ns_tracker);
c308c1b2 2097 sk_prot_free(sk->sk_prot_creator, sk);
1da177e4 2098}
2b85a34e 2099
a4298e45
ED		 2100void sk_destruct(struct sock *sk)
2101{
8c7138b3
MKL		 2102 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
2103
2104 if (rcu_access_pointer(sk->sk_reuseport_cb)) {
2105 reuseport_detach_sock(sk);
2106 use_call_rcu = true;
2107 }
2108
2109 if (use_call_rcu)
a4298e45
ED		 2110 call_rcu(&sk->sk_rcu, __sk_destruct);
2111 else
2112 __sk_destruct(&sk->sk_rcu);
2113}
2114
eb4cb008
CG		 2115static void __sk_free(struct sock *sk)
2116{
648845ab
TZ		 2117 if (likely(sk->sk_net_refcnt))
2118 sock_inuse_add(sock_net(sk), -1);
2119
9709020c 2120 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
eb4cb008
CG		 2121 sock_diag_broadcast_destroy(sk);
2122 else
2123 sk_destruct(sk);
2124}
2125
2b85a34e
ED		 2126void sk_free(struct sock *sk)
2127{
2128 /*
25985edc 2129 * We subtract one from sk_wmem_alloc and can know if
2b85a34e
ED		 2130 * some packets are still in some tx queue.
2131 * If not null, sock_wfree() will call __sk_free(sk) later
2132 */
14afee4b 2133 if (refcount_dec_and_test(&sk->sk_wmem_alloc))
2b85a34e
ED		 2134 __sk_free(sk);
2135}
2a91525c 2136EXPORT_SYMBOL(sk_free);
1da177e4 2137
581319c5
PA		 2138static void sk_init_common(struct sock *sk)
2139{
2140 skb_queue_head_init(&sk->sk_receive_queue);
2141 skb_queue_head_init(&sk->sk_write_queue);
2142 skb_queue_head_init(&sk->sk_error_queue);
2143
2144 rwlock_init(&sk->sk_callback_lock);
2145 lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
2146 af_rlock_keys + sk->sk_family,
2147 af_family_rlock_key_strings[sk->sk_family]);
2148 lockdep_set_class_and_name(&sk->sk_write_queue.lock,
2149 af_wlock_keys + sk->sk_family,
2150 af_family_wlock_key_strings[sk->sk_family]);
2151 lockdep_set_class_and_name(&sk->sk_error_queue.lock,
2152 af_elock_keys + sk->sk_family,
2153 af_family_elock_key_strings[sk->sk_family]);
2154 lockdep_set_class_and_name(&sk->sk_callback_lock,
2155 af_callback_keys + sk->sk_family,
2156 af_family_clock_key_strings[sk->sk_family]);
2157}
2158
e56c57d0
ED		 2159/**
2160 * sk_clone_lock - clone a socket, and lock its clone
2161 * @sk: the socket to clone
2162 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
2163 *
2164 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
2165 */
2166struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
87d11ceb 2167{
b8e202d1 2168 struct proto *prot = READ_ONCE(sk->sk_prot);
bbc20b70 2169 struct sk_filter *filter;
278571ba 2170 bool is_charged = true;
bbc20b70 2171 struct sock *newsk;
87d11ceb 2172
b8e202d1 2173 newsk = sk_prot_alloc(prot, priority, sk->sk_family);
bbc20b70
ED		 2174 if (!newsk)
2175 goto out;
87d11ceb 2176
bbc20b70 2177 sock_copy(newsk, sk);
9d538fa6 2178
bbc20b70 2179 newsk->sk_prot_creator = prot;
87d11ceb 2180
bbc20b70 2181 /* SANITY */
938cca9e 2182 if (likely(newsk->sk_net_refcnt)) {
ffa84b5f 2183 get_net_track(sock_net(newsk), &newsk->ns_tracker, priority);
938cca9e
TH		 2184 sock_inuse_add(sock_net(newsk), 1);
2185 }
bbc20b70
ED		 2186 sk_node_init(&newsk->sk_node);
2187 sock_lock_init(newsk);
2188 bh_lock_sock(newsk);
2189 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
2190 newsk->sk_backlog.len = 0;
87d11ceb 2191
bbc20b70 2192 atomic_set(&newsk->sk_rmem_alloc, 0);
87d11ceb 2193
bbc20b70
ED		 2194 /* sk_wmem_alloc set to one (see sk_free() and sock_wfree()) */
2195 refcount_set(&newsk->sk_wmem_alloc, 1);
d752a498 2196
bbc20b70
ED		 2197 atomic_set(&newsk->sk_omem_alloc, 0);
2198 sk_init_common(newsk);
d752a498 2199
bbc20b70
ED		 2200 newsk->sk_dst_cache = NULL;
2201 newsk->sk_dst_pending_confirm = 0;
2202 newsk->sk_wmem_queued = 0;
2203 newsk->sk_forward_alloc = 0;
2bb2f5fb 2204 newsk->sk_reserved_mem = 0;
bbc20b70
ED		 2205 atomic_set(&newsk->sk_drops, 0);
2206 newsk->sk_send_head = NULL;
2207 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
2208 atomic_set(&newsk->sk_zckey, 0);
87d11ceb 2209
bbc20b70 2210 sock_reset_flag(newsk, SOCK_DONE);
87d11ceb 2211
bbc20b70
ED		 2212 /* sk->sk_memcg will be populated at accept() time */
2213 newsk->sk_memcg = NULL;
8f51dfc7 2214
bbc20b70 2215 cgroup_sk_clone(&newsk->sk_cgrp_data);
87d11ceb 2216
bbc20b70
ED		 2217 rcu_read_lock();
2218 filter = rcu_dereference(sk->sk_filter);
2219 if (filter != NULL)
2220 /* though it's an empty new sock, the charging may fail
2221 * if sysctl_optmem_max was changed between creation of
2222 * original socket and cloning
2223 */
2224 is_charged = sk_filter_charge(newsk, filter);
2225 RCU_INIT_POINTER(newsk->sk_filter, filter);
2226 rcu_read_unlock();
2227
2228 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
2229 /* We need to make sure that we don't uncharge the new
2230 * socket if we couldn't charge it in the first place
2231 * as otherwise we uncharge the parent's filter.
f1ff5ce2 2232 */
bbc20b70
ED		 2233 if (!is_charged)
2234 RCU_INIT_POINTER(newsk->sk_filter, NULL);
2235 sk_free_unlock_clone(newsk);
2236 newsk = NULL;
2237 goto out;
2238 }
2239 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
f1ff5ce2 2240
bbc20b70
ED		 2241 if (bpf_sk_storage_clone(sk, newsk)) {
2242 sk_free_unlock_clone(newsk);
2243 newsk = NULL;
2244 goto out;
2245 }
d979a39d 2246
bbc20b70
ED		 2247 /* Clear sk_user_data if parent had the pointer tagged
2248 * as not suitable for copying when cloning.
2249 */
2250 if (sk_user_data_is_nocopy(newsk))
2251 newsk->sk_user_data = NULL;
2252
2253 newsk->sk_err = 0;
2254 newsk->sk_err_soft = 0;
2255 newsk->sk_priority = 0;
2256 newsk->sk_incoming_cpu = raw_smp_processor_id();
bbc20b70
ED		 2257
2258 /* Before updating sk_refcnt, we must commit prior changes to memory
2259 * (Documentation/RCU/rculist_nulls.rst for details)
2260 */
2261 smp_wmb();
2262 refcount_set(&newsk->sk_refcnt, 2);
87d11ceb 2263
bbc20b70
ED		 2264 /* Increment the counter in the same struct proto as the master
2265 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
2266 * is the same as sk->sk_prot->socks, as this field was copied
2267 * with memcpy).
2268 *
2269 * This _changes_ the previous behaviour, where
2270 * tcp_create_openreq_child always was incrementing the
2271 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
2272 * to be taken into account in all callers. -acme
2273 */
2274 sk_refcnt_debug_inc(newsk);
2275 sk_set_socket(newsk, NULL);
2276 sk_tx_queue_clear(newsk);
2277 RCU_INIT_POINTER(newsk->sk_wq, NULL);
87d11ceb 2278
bbc20b70
ED		 2279 if (newsk->sk_prot->sockets_allocated)
2280 sk_sockets_allocated_inc(newsk);
704da560 2281
bbc20b70
ED		 2282 if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP)
2283 net_enable_timestamp();
87d11ceb
ACM		 2284out:
2285 return newsk;
2286}
e56c57d0 2287EXPORT_SYMBOL_GPL(sk_clone_lock);
87d11ceb 2288
94352d45
ACM		 2289void sk_free_unlock_clone(struct sock *sk)
2290{
2291 /* It is still raw copy of parent, so invalidate
2292 * destructor and make plain sk_free() */
2293 sk->sk_destruct = NULL;
2294 bh_unlock_sock(sk);
2295 sk_free(sk);
2296}
2297EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
2298
7c4e983c
AD		 2299static void sk_trim_gso_size(struct sock *sk)
2300{
2301 if (sk->sk_gso_max_size <= GSO_LEGACY_MAX_SIZE)
2302 return;
2303#if IS_ENABLED(CONFIG_IPV6)
2304 if (sk->sk_family == AF_INET6 &&
2305 sk_is_tcp(sk) &&
2306 !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
2307 return;
2308#endif
2309 sk->sk_gso_max_size = GSO_LEGACY_MAX_SIZE;
2310}
2311
9958089a
AK		 2312void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
2313{
d6a4e26a
ED		 2314 u32 max_segs = 1;
2315
6bd4f355 2316 sk_dst_set(sk, dst);
d0d598ca
ED		 2317 sk->sk_route_caps = dst->dev->features;
2318 if (sk_is_tcp(sk))
2319 sk->sk_route_caps |= NETIF_F_GSO;
9958089a 2320 if (sk->sk_route_caps & NETIF_F_GSO)
4fcd6b99 2321 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
aba54656
ED		 2322 if (unlikely(sk->sk_gso_disabled))
2323 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
9958089a 2324 if (sk_can_gso(sk)) {
f70f250a 2325 if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
9958089a 2326 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
82cc1a7a 2327 } else {
9958089a 2328 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
4b66d216
ED		 2329 /* pairs with the WRITE_ONCE() in netif_set_gso_max_size() */
2330 sk->sk_gso_max_size = READ_ONCE(dst->dev->gso_max_size);
7c4e983c 2331 sk_trim_gso_size(sk);
ab14f180 2332 sk->sk_gso_max_size -= (MAX_TCP_HEADER + 1);
6d872df3
ED		 2333 /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */
2334 max_segs = max_t(u32, READ_ONCE(dst->dev->gso_max_segs), 1);
82cc1a7a 2335 }
9958089a 2336 }
d6a4e26a 2337 sk->sk_gso_max_segs = max_segs;
9958089a
AK		 2338}
2339EXPORT_SYMBOL_GPL(sk_setup_caps);
2340
1da177e4
LT		 2341/*
2342 * Simple resource managers for sockets.
2343 */
2344
2345
4ec93edb
YH		 2346/*
2347 * Write buffer destructor automatically called from kfree_skb.
1da177e4
LT		 2348 */
2349void sock_wfree(struct sk_buff *skb)
2350{
2351 struct sock *sk = skb->sk;
d99927f4 2352 unsigned int len = skb->truesize;
052ada09 2353 bool free;
1da177e4 2354
d99927f4 2355 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
052ada09
PB		 2356 if (sock_flag(sk, SOCK_RCU_FREE) &&
2357 sk->sk_write_space == sock_def_write_space) {
2358 rcu_read_lock();
2359 free = refcount_sub_and_test(len, &sk->sk_wmem_alloc);
0a8afd9f 2360 sock_def_write_space_wfree(sk);
052ada09
PB		 2361 rcu_read_unlock();
2362 if (unlikely(free))
2363 __sk_free(sk);
2364 return;
2365 }
2366
d99927f4
ED		 2367 /*
2368 * Keep a reference on sk_wmem_alloc, this will be released
2369 * after sk_write_space() call
2370 */
14afee4b 2371 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
1da177e4 2372 sk->sk_write_space(sk);
d99927f4
ED		 2373 len = 1;
2374 }
2b85a34e 2375 /*
d99927f4
ED		 2376 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
2377 * could not do because of in-flight packets
2b85a34e 2378 */
14afee4b 2379 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2b85a34e 2380 __sk_free(sk);
1da177e4 2381}
2a91525c 2382EXPORT_SYMBOL(sock_wfree);
1da177e4 2383
1d2077ac
ED		 2384/* This variant of sock_wfree() is used by TCP,
2385 * since it sets SOCK_USE_WRITE_QUEUE.
2386 */
2387void __sock_wfree(struct sk_buff *skb)
2388{
2389 struct sock *sk = skb->sk;
2390
14afee4b 2391 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1d2077ac
ED		 2392 __sk_free(sk);
2393}
2394
9e17f8a4
ED		 2395void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
2396{
2397 skb_orphan(skb);
2398 skb->sk = sk;
2399#ifdef CONFIG_INET
2400 if (unlikely(!sk_fullsock(sk))) {
2401 skb->destructor = sock_edemux;
2402 sock_hold(sk);
2403 return;
2404 }
2405#endif
2406 skb->destructor = sock_wfree;
2407 skb_set_hash_from_sk(skb, sk);
2408 /*
2409 * We used to take a refcount on sk, but following operation
2410 * is enough to guarantee sk_free() wont free this sock until
2411 * all in-flight packets are completed
2412 */
14afee4b 2413 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
9e17f8a4
ED		 2414}
2415EXPORT_SYMBOL(skb_set_owner_w);
2416
41477662
JK		 2417static bool can_skb_orphan_partial(const struct sk_buff *skb)
2418{
2419#ifdef CONFIG_TLS_DEVICE
2420 /* Drivers depend on in-order delivery for crypto offload,
2421 * partial orphan breaks out-of-order-OK logic.
2422 */
2423 if (skb->decrypted)
2424 return false;
2425#endif
2426 return (skb->destructor == sock_wfree ||
2427 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree));
2428}
2429
1d2077ac
ED		 2430/* This helper is used by netem, as it can hold packets in its
2431 * delay queue. We want to allow the owner socket to send more
2432 * packets, as if they were already TX completed by a typical driver.
2433 * But we also want to keep skb->sk set because some packet schedulers
f6ba8d33 2434 * rely on it (sch_fq for example).
1d2077ac 2435 */
f2f872f9
ED		 2436void skb_orphan_partial(struct sk_buff *skb)
2437{
f6ba8d33 2438 if (skb_is_tcp_pure_ack(skb))
1d2077ac
ED		 2439 return;
2440
098116e7
PA		 2441 if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk))
2442 return;
2443
2444 skb_orphan(skb);
f2f872f9
ED		 2445}
2446EXPORT_SYMBOL(skb_orphan_partial);
2447
4ec93edb
YH		 2448/*
2449 * Read buffer destructor automatically called from kfree_skb.
1da177e4
LT		 2450 */
2451void sock_rfree(struct sk_buff *skb)
2452{
2453 struct sock *sk = skb->sk;
d361fd59 2454 unsigned int len = skb->truesize;
1da177e4 2455
d361fd59
ED		 2456 atomic_sub(len, &sk->sk_rmem_alloc);
2457 sk_mem_uncharge(sk, len);
1da177e4 2458}
2a91525c 2459EXPORT_SYMBOL(sock_rfree);
1da177e4 2460
7768eed8
OH		 2461/*
2462 * Buffer destructor for skbs that are not used directly in read or write
2463 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
2464 */
62bccb8c
AD		 2465void sock_efree(struct sk_buff *skb)
2466{
2467 sock_put(skb->sk);
2468}
2469EXPORT_SYMBOL(sock_efree);
2470
cf7fbe66
JS		 2471/* Buffer destructor for prefetch/receive path where reference count may
2472 * not be held, e.g. for listen sockets.
2473 */
2474#ifdef CONFIG_INET
2475void sock_pfree(struct sk_buff *skb)
2476{
7ae215d2
JS		 2477 if (sk_is_refcounted(skb->sk))
2478 sock_gen_put(skb->sk);
cf7fbe66
JS		 2479}
2480EXPORT_SYMBOL(sock_pfree);
2481#endif /* CONFIG_INET */
2482
976d0201 2483kuid_t sock_i_uid(struct sock *sk)
1da177e4 2484{
976d0201 2485 kuid_t uid;
1da177e4 2486
f064af1e 2487 read_lock_bh(&sk->sk_callback_lock);
976d0201 2488 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
f064af1e 2489 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 2490 return uid;
2491}
2a91525c 2492EXPORT_SYMBOL(sock_i_uid);
1da177e4
LT		 2493
2494unsigned long sock_i_ino(struct sock *sk)
2495{
2496 unsigned long ino;
2497
f064af1e 2498 read_lock_bh(&sk->sk_callback_lock);
1da177e4 2499 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
f064af1e 2500 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 2501 return ino;
2502}
2a91525c 2503EXPORT_SYMBOL(sock_i_ino);
1da177e4
LT		 2504
2505/*
2506 * Allocate a skb from the socket's send buffer.
2507 */
86a76caf 2508struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
dd0fc66f 2509 gfp_t priority)
1da177e4 2510{
e292f05e
ED		 2511 if (force ||
2512 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
2a91525c 2513 struct sk_buff *skb = alloc_skb(size, priority);
e292f05e 2514
1da177e4
LT		 2515 if (skb) {
2516 skb_set_owner_w(skb, sk);
2517 return skb;
2518 }
2519 }
2520 return NULL;
2521}
2a91525c 2522EXPORT_SYMBOL(sock_wmalloc);
1da177e4 2523
98ba0bd5
WB		 2524static void sock_ofree(struct sk_buff *skb)
2525{
2526 struct sock *sk = skb->sk;
2527
2528 atomic_sub(skb->truesize, &sk->sk_omem_alloc);
2529}
2530
2531struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
2532 gfp_t priority)
2533{
2534 struct sk_buff *skb;
2535
2536 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
2537 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
2538 sysctl_optmem_max)
2539 return NULL;
2540
2541 skb = alloc_skb(size, priority);
2542 if (!skb)
2543 return NULL;
2544
2545 atomic_add(skb->truesize, &sk->sk_omem_alloc);
2546 skb->sk = sk;
2547 skb->destructor = sock_ofree;
2548 return skb;
2549}
2550
4ec93edb 2551/*
1da177e4 2552 * Allocate a memory block from the socket's option memory buffer.
4ec93edb 2553 */
dd0fc66f 2554void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
1da177e4 2555{
95c96174 2556 if ((unsigned int)size <= sysctl_optmem_max &&
1da177e4
LT		 2557 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
2558 void *mem;
2559 /* First do the add, to avoid the race if kmalloc
4ec93edb 2560 * might sleep.
1da177e4
LT		 2561 */
2562 atomic_add(size, &sk->sk_omem_alloc);
2563 mem = kmalloc(size, priority);
2564 if (mem)
2565 return mem;
2566 atomic_sub(size, &sk->sk_omem_alloc);
2567 }
2568 return NULL;
2569}
2a91525c 2570EXPORT_SYMBOL(sock_kmalloc);
1da177e4 2571
79e88659
DB		 2572/* Free an option memory block. Note, we actually want the inline
2573 * here as this allows gcc to detect the nullify and fold away the
2574 * condition entirely.
1da177e4 2575 */
79e88659
DB		 2576static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
2577 const bool nullify)
1da177e4 2578{
e53da5fb
DM		 2579 if (WARN_ON_ONCE(!mem))
2580 return;
79e88659 2581 if (nullify)
453431a5 2582 kfree_sensitive(mem);
79e88659
DB		 2583 else
2584 kfree(mem);
1da177e4
LT		 2585 atomic_sub(size, &sk->sk_omem_alloc);
2586}
79e88659
DB		 2587
2588void sock_kfree_s(struct sock *sk, void *mem, int size)
2589{
2590 __sock_kfree_s(sk, mem, size, false);
2591}
2a91525c 2592EXPORT_SYMBOL(sock_kfree_s);
1da177e4 2593
79e88659
DB		 2594void sock_kzfree_s(struct sock *sk, void *mem, int size)
2595{
2596 __sock_kfree_s(sk, mem, size, true);
2597}
2598EXPORT_SYMBOL(sock_kzfree_s);
2599
1da177e4
LT		 2600/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
2601 I think, these locks should be removed for datagram sockets.
2602 */
2a91525c 2603static long sock_wait_for_wmem(struct sock *sk, long timeo)
1da177e4
LT		 2604{
2605 DEFINE_WAIT(wait);
2606
9cd3e072 2607 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4
LT		 2608 for (;;) {
2609 if (!timeo)
2610 break;
2611 if (signal_pending(current))
2612 break;
2613 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
aa395145 2614 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
e292f05e 2615 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
1da177e4
LT		 2616 break;
2617 if (sk->sk_shutdown & SEND_SHUTDOWN)
2618 break;
2619 if (sk->sk_err)
2620 break;
2621 timeo = schedule_timeout(timeo);
2622 }
aa395145 2623 finish_wait(sk_sleep(sk), &wait);
1da177e4
LT		 2624 return timeo;
2625}
2626
2627
2628/*
2629 * Generic send/receive buffer handlers
2630 */
2631
4cc7f68d
HX		 2632struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
2633 unsigned long data_len, int noblock,
28d64271 2634 int *errcode, int max_page_order)
1da177e4 2635{
2e4e4410 2636 struct sk_buff *skb;
1da177e4
LT		 2637 long timeo;
2638 int err;
2639
1da177e4 2640 timeo = sock_sndtimeo(sk, noblock);
2e4e4410 2641 for (;;) {
1da177e4
LT		 2642 err = sock_error(sk);
2643 if (err != 0)
2644 goto failure;
2645
2646 err = -EPIPE;
2647 if (sk->sk_shutdown & SEND_SHUTDOWN)
2648 goto failure;
2649
e292f05e 2650 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
2e4e4410 2651 break;
28d64271 2652
9cd3e072 2653 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2e4e4410
ED		 2654 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2655 err = -EAGAIN;
2656 if (!timeo)
1da177e4 2657 goto failure;
2e4e4410
ED		 2658 if (signal_pending(current))
2659 goto interrupted;
2660 timeo = sock_wait_for_wmem(sk, timeo);
1da177e4 2661 }
2e4e4410
ED		 2662 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2663 errcode, sk->sk_allocation);
2664 if (skb)
2665 skb_set_owner_w(skb, sk);
1da177e4
LT		 2666 return skb;
2667
2668interrupted:
2669 err = sock_intr_errno(timeo);
2670failure:
2671 *errcode = err;
2672 return NULL;
2673}
4cc7f68d 2674EXPORT_SYMBOL(sock_alloc_send_pskb);
1da177e4 2675
39771b12
WB		 2676int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2677 struct sockcm_cookie *sockc)
2678{
3dd17e63
SHY		 2679 u32 tsflags;
2680
39771b12
WB		 2681 switch (cmsg->cmsg_type) {
2682 case SO_MARK:
91f0d8a4
JK		 2683 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
2684 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
39771b12
WB		 2685 return -EPERM;
2686 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2687 return -EINVAL;
2688 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2689 break;
7f1bc6e9 2690 case SO_TIMESTAMPING_OLD:
3dd17e63
SHY		 2691 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2692 return -EINVAL;
2693
2694 tsflags = *(u32 *)CMSG_DATA(cmsg);
2695 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2696 return -EINVAL;
2697
2698 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2699 sockc->tsflags |= tsflags;
2700 break;
80b14dee
RC		 2701 case SCM_TXTIME:
2702 if (!sock_flag(sk, SOCK_TXTIME))
2703 return -EINVAL;
2704 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
2705 return -EINVAL;
2706 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
2707 break;
779f1ede
SHY		 2708 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2709 case SCM_RIGHTS:
2710 case SCM_CREDENTIALS:
2711 break;
39771b12
WB		 2712 default:
2713 return -EINVAL;
2714 }
2715 return 0;
2716}
2717EXPORT_SYMBOL(__sock_cmsg_send);
2718
f28ea365
EJ		 2719int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2720 struct sockcm_cookie *sockc)
2721{
2722 struct cmsghdr *cmsg;
39771b12 2723 int ret;
f28ea365
EJ		 2724
2725 for_each_cmsghdr(cmsg, msg) {
2726 if (!CMSG_OK(msg, cmsg))
2727 return -EINVAL;
2728 if (cmsg->cmsg_level != SOL_SOCKET)
2729 continue;
39771b12
WB		 2730 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2731 if (ret)
2732 return ret;
f28ea365
EJ		 2733 }
2734 return 0;
2735}
2736EXPORT_SYMBOL(sock_cmsg_send);
2737
06044751
ED		 2738static void sk_enter_memory_pressure(struct sock *sk)
2739{
2740 if (!sk->sk_prot->enter_memory_pressure)
2741 return;
2742
2743 sk->sk_prot->enter_memory_pressure(sk);
2744}
2745
2746static void sk_leave_memory_pressure(struct sock *sk)
2747{
2748 if (sk->sk_prot->leave_memory_pressure) {
2749 sk->sk_prot->leave_memory_pressure(sk);
2750 } else {
2751 unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2752
503978ac
ED		 2753 if (memory_pressure && READ_ONCE(*memory_pressure))
2754 WRITE_ONCE(*memory_pressure, 0);
06044751
ED		 2755 }
2756}
2757
ce27ec60 2758DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
5640f768 2759
400dfd3a
ED		 2760/**
2761 * skb_page_frag_refill - check that a page_frag contains enough room
2762 * @sz: minimum size of the fragment we want to get
2763 * @pfrag: pointer to page_frag
82d5e2b8 2764 * @gfp: priority for memory allocation
400dfd3a
ED		 2765 *
2766 * Note: While this allocator tries to use high order pages, there is
2767 * no guarantee that allocations succeed. Therefore, @sz MUST be
2768 * less or equal than PAGE_SIZE.
2769 */
d9b2938a 2770bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
5640f768 2771{
5640f768 2772 if (pfrag->page) {
fe896d18 2773 if (page_ref_count(pfrag->page) == 1) {
5640f768
ED		 2774 pfrag->offset = 0;
2775 return true;
2776 }
400dfd3a 2777 if (pfrag->offset + sz <= pfrag->size)
5640f768
ED		 2778 return true;
2779 put_page(pfrag->page);
2780 }
2781
d9b2938a 2782 pfrag->offset = 0;
ce27ec60
ED		 2783 if (SKB_FRAG_PAGE_ORDER &&
2784 !static_branch_unlikely(&net_high_order_alloc_disable_key)) {
d0164adc
MG		 2785 /* Avoid direct reclaim but allow kswapd to wake */
2786 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2787 __GFP_COMP | __GFP_NOWARN |
2788 __GFP_NORETRY,
d9b2938a 2789 SKB_FRAG_PAGE_ORDER);
5640f768 2790 if (likely(pfrag->page)) {
d9b2938a 2791 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
5640f768
ED		 2792 return true;
2793 }
d9b2938a
ED		 2794 }
2795 pfrag->page = alloc_page(gfp);
2796 if (likely(pfrag->page)) {
2797 pfrag->size = PAGE_SIZE;
2798 return true;
2799 }
400dfd3a
ED		 2800 return false;
2801}
2802EXPORT_SYMBOL(skb_page_frag_refill);
2803
2804bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2805{
2806 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2807 return true;
2808
5640f768
ED		 2809 sk_enter_memory_pressure(sk);
2810 sk_stream_moderate_sndbuf(sk);
2811 return false;
2812}
2813EXPORT_SYMBOL(sk_page_frag_refill);
2814
ad80b0fc 2815void __lock_sock(struct sock *sk)
f39234d6
NK		 2816 __releases(&sk->sk_lock.slock)
2817 __acquires(&sk->sk_lock.slock)
1da177e4
LT		 2818{
2819 DEFINE_WAIT(wait);
2820
e71a4783 2821 for (;;) {
1da177e4
LT		 2822 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2823 TASK_UNINTERRUPTIBLE);
2824 spin_unlock_bh(&sk->sk_lock.slock);
2825 schedule();
2826 spin_lock_bh(&sk->sk_lock.slock);
e71a4783 2827 if (!sock_owned_by_user(sk))
1da177e4
LT		 2828 break;
2829 }
2830 finish_wait(&sk->sk_lock.wq, &wait);
2831}
2832
8873c064 2833void __release_sock(struct sock *sk)
f39234d6
NK		 2834 __releases(&sk->sk_lock.slock)
2835 __acquires(&sk->sk_lock.slock)
1da177e4 2836{
5413d1ba 2837 struct sk_buff *skb, *next;
1da177e4 2838
5413d1ba 2839 while ((skb = sk->sk_backlog.head) != NULL) {
1da177e4 2840 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
1da177e4 2841
5413d1ba 2842 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4 2843
5413d1ba
ED		 2844 do {
2845 next = skb->next;
e4cbb02a 2846 prefetch(next);
7fee226a 2847 WARN_ON_ONCE(skb_dst_is_noref(skb));
a8305bff 2848 skb_mark_not_on_list(skb);
c57943a1 2849 sk_backlog_rcv(sk, skb);
1da177e4 2850
5413d1ba 2851 cond_resched();
1da177e4
LT		 2852
2853 skb = next;
2854 } while (skb != NULL);
2855
5413d1ba
ED		 2856 spin_lock_bh(&sk->sk_lock.slock);
2857 }
8eae939f
ZY		 2858
2859 /*
2860 * Doing the zeroing here guarantee we can not loop forever
2861 * while a wild producer attempts to flood us.
2862 */
2863 sk->sk_backlog.len = 0;
1da177e4
LT		 2864}
2865
d41a69f1
ED		 2866void __sk_flush_backlog(struct sock *sk)
2867{
2868 spin_lock_bh(&sk->sk_lock.slock);
2869 __release_sock(sk);
2870 spin_unlock_bh(&sk->sk_lock.slock);
2871}
2872
1da177e4
LT		 2873/**
2874 * sk_wait_data - wait for data to arrive at sk_receive_queue
4dc3b16b
PP		 2875 * @sk: sock to wait on
2876 * @timeo: for how long
dfbafc99 2877 * @skb: last skb seen on sk_receive_queue
1da177e4
LT		 2878 *
2879 * Now socket state including sk->sk_err is changed only under lock,
2880 * hence we may omit checks after joining wait queue.
2881 * We check receive queue before schedule() only as optimization;
2882 * it is very likely that release_sock() added new data.
2883 */
dfbafc99 2884int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
1da177e4 2885{
d9dc8b0f 2886 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1da177e4 2887 int rc;
1da177e4 2888
d9dc8b0f 2889 add_wait_queue(sk_sleep(sk), &wait);
9cd3e072 2890 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2891 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
9cd3e072 2892 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2893 remove_wait_queue(sk_sleep(sk), &wait);
1da177e4
LT		 2894 return rc;
2895}
1da177e4
LT		 2896EXPORT_SYMBOL(sk_wait_data);
2897
3ab224be 2898/**
f8c3bf00 2899 * __sk_mem_raise_allocated - increase memory_allocated
3ab224be
HA		 2900 * @sk: socket
2901 * @size: memory size to allocate
f8c3bf00 2902 * @amt: pages to allocate
3ab224be
HA		 2903 * @kind: allocation type
2904 *
f8c3bf00 2905 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
3ab224be 2906 */
f8c3bf00 2907int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
3ab224be
HA		 2908{
2909 struct proto *prot = sk->sk_prot;
f8c3bf00 2910 long allocated = sk_memory_allocated_add(sk, amt);
4b1327be 2911 bool memcg_charge = mem_cgroup_sockets_enabled && sk->sk_memcg;
d6f19938 2912 bool charged = true;
e805605c 2913
4b1327be
WW		 2914 if (memcg_charge &&
2915 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt,
2916 gfp_memcg_charge())))
e805605c 2917 goto suppress_allocation;
3ab224be
HA		 2918
2919 /* Under limit. */
e805605c 2920 if (allocated <= sk_prot_mem_limits(sk, 0)) {
180d8cd9 2921 sk_leave_memory_pressure(sk);
3ab224be
HA		 2922 return 1;
2923 }
2924
e805605c
JW		 2925 /* Under pressure. */
2926 if (allocated > sk_prot_mem_limits(sk, 1))
180d8cd9 2927 sk_enter_memory_pressure(sk);
3ab224be 2928
e805605c
JW		 2929 /* Over hard limit. */
2930 if (allocated > sk_prot_mem_limits(sk, 2))
3ab224be
HA		 2931 goto suppress_allocation;
2932
2933 /* guarantee minimum buffer size under pressure */
2934 if (kind == SK_MEM_RECV) {
a3dcaf17 2935 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
3ab224be 2936 return 1;
180d8cd9 2937
3ab224be 2938 } else { /* SK_MEM_SEND */
a3dcaf17
ED		 2939 int wmem0 = sk_get_wmem0(sk, prot);
2940
3ab224be 2941 if (sk->sk_type == SOCK_STREAM) {
a3dcaf17 2942 if (sk->sk_wmem_queued < wmem0)
3ab224be 2943 return 1;
a3dcaf17 2944 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
3ab224be 2945 return 1;
a3dcaf17 2946 }
3ab224be
HA		 2947 }
2948
180d8cd9 2949 if (sk_has_memory_pressure(sk)) {
5bf325a5 2950 u64 alloc;
1748376b 2951
180d8cd9 2952 if (!sk_under_memory_pressure(sk))
1748376b 2953 return 1;
180d8cd9
GC		 2954 alloc = sk_sockets_allocated_read_positive(sk);
2955 if (sk_prot_mem_limits(sk, 2) > alloc *
3ab224be
HA		 2956 sk_mem_pages(sk->sk_wmem_queued +
2957 atomic_read(&sk->sk_rmem_alloc) +
2958 sk->sk_forward_alloc))
2959 return 1;
2960 }
2961
2962suppress_allocation:
2963
2964 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2965 sk_stream_moderate_sndbuf(sk);
2966
2967 /* Fail only if socket is _under_ its sndbuf.
2968 * In this case we cannot block, so that we have to fail.
2969 */
4b1327be
WW		 2970 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) {
2971 /* Force charge with __GFP_NOFAIL */
2972 if (memcg_charge && !charged) {
2973 mem_cgroup_charge_skmem(sk->sk_memcg, amt,
2974 gfp_memcg_charge() | __GFP_NOFAIL);
2975 }
3ab224be 2976 return 1;
4b1327be 2977 }
3ab224be
HA		 2978 }
2979
d6f19938
YS		 2980 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged))
2981 trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
3847ce32 2982
0e90b31f 2983 sk_memory_allocated_sub(sk, amt);
180d8cd9 2984
4b1327be 2985 if (memcg_charge && charged)
baac50bb 2986 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
e805605c 2987
3ab224be
HA		 2988 return 0;
2989}
f8c3bf00
PA		 2990EXPORT_SYMBOL(__sk_mem_raise_allocated);
2991
2992/**
2993 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2994 * @sk: socket
2995 * @size: memory size to allocate
2996 * @kind: allocation type
2997 *
2998 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2999 * rmem allocation. This function assumes that protocols which have
3000 * memory_pressure use sk_wmem_queued as write buffer accounting.
3001 */
3002int __sk_mem_schedule(struct sock *sk, int size, int kind)
3003{
3004 int ret, amt = sk_mem_pages(size);
3005
3006 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
3007 ret = __sk_mem_raise_allocated(sk, size, amt, kind);
3008 if (!ret)
3009 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
3010 return ret;
3011}
3ab224be
HA		 3012EXPORT_SYMBOL(__sk_mem_schedule);
3013
3014/**
f8c3bf00 3015 * __sk_mem_reduce_allocated - reclaim memory_allocated
3ab224be 3016 * @sk: socket
f8c3bf00
PA		 3017 * @amount: number of quanta
3018 *
3019 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3ab224be 3020 */
f8c3bf00 3021void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3ab224be 3022{
1a24e04e 3023 sk_memory_allocated_sub(sk, amount);
3ab224be 3024
baac50bb
JW		 3025 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
3026 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
e805605c 3027
180d8cd9
GC		 3028 if (sk_under_memory_pressure(sk) &&
3029 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
3030 sk_leave_memory_pressure(sk);
3ab224be 3031}
f8c3bf00
PA		 3032EXPORT_SYMBOL(__sk_mem_reduce_allocated);
3033
3034/**
3035 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
3036 * @sk: socket
3037 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
3038 */
3039void __sk_mem_reclaim(struct sock *sk, int amount)
3040{
3041 amount >>= SK_MEM_QUANTUM_SHIFT;
3042 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
3043 __sk_mem_reduce_allocated(sk, amount);
3044}
3ab224be
HA		 3045EXPORT_SYMBOL(__sk_mem_reclaim);
3046
627d2d6b 3047int sk_set_peek_off(struct sock *sk, int val)
3048{
627d2d6b 3049 sk->sk_peek_off = val;
3050 return 0;
3051}
3052EXPORT_SYMBOL_GPL(sk_set_peek_off);
3ab224be 3053
1da177e4
LT		 3054/*
3055 * Set of default routines for initialising struct proto_ops when
3056 * the protocol does not support a particular function. In certain
3057 * cases where it makes no sense for a protocol to have a "do nothing"
3058 * function, some default processing is provided.
3059 */
3060
3061int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
3062{
3063 return -EOPNOTSUPP;
3064}
2a91525c 3065EXPORT_SYMBOL(sock_no_bind);
1da177e4 3066
4ec93edb 3067int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
1da177e4
LT		 3068 int len, int flags)
3069{
3070 return -EOPNOTSUPP;
3071}
2a91525c 3072EXPORT_SYMBOL(sock_no_connect);
1da177e4
LT		 3073
3074int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
3075{
3076 return -EOPNOTSUPP;
3077}
2a91525c 3078EXPORT_SYMBOL(sock_no_socketpair);
1da177e4 3079
cdfbabfb
DH		 3080int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
3081 bool kern)
1da177e4
LT		 3082{
3083 return -EOPNOTSUPP;
3084}
2a91525c 3085EXPORT_SYMBOL(sock_no_accept);
1da177e4 3086
4ec93edb 3087int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
9b2c45d4 3088 int peer)
1da177e4
LT		 3089{
3090 return -EOPNOTSUPP;
3091}
2a91525c 3092EXPORT_SYMBOL(sock_no_getname);
1da177e4 3093
1da177e4
LT		 3094int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
3095{
3096 return -EOPNOTSUPP;
3097}
2a91525c 3098EXPORT_SYMBOL(sock_no_ioctl);
1da177e4
LT		 3099
3100int sock_no_listen(struct socket *sock, int backlog)
3101{
3102 return -EOPNOTSUPP;
3103}
2a91525c 3104EXPORT_SYMBOL(sock_no_listen);
1da177e4
LT		 3105
3106int sock_no_shutdown(struct socket *sock, int how)
3107{
3108 return -EOPNOTSUPP;
3109}
2a91525c 3110EXPORT_SYMBOL(sock_no_shutdown);
1da177e4 3111
1b784140 3112int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
1da177e4
LT		 3113{
3114 return -EOPNOTSUPP;
3115}
2a91525c 3116EXPORT_SYMBOL(sock_no_sendmsg);
1da177e4 3117
306b13eb
TH		 3118int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
3119{
3120 return -EOPNOTSUPP;
3121}
3122EXPORT_SYMBOL(sock_no_sendmsg_locked);
3123
1b784140
YX		 3124int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
3125 int flags)
1da177e4
LT		 3126{
3127 return -EOPNOTSUPP;
3128}
2a91525c 3129EXPORT_SYMBOL(sock_no_recvmsg);
1da177e4
LT		 3130
3131int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
3132{
3133 /* Mirror missing mmap method error code */
3134 return -ENODEV;
3135}
2a91525c 3136EXPORT_SYMBOL(sock_no_mmap);
1da177e4 3137
d9539752
KC		 3138/*
3139 * When a file is received (via SCM_RIGHTS, etc), we must bump the
3140 * various sock-based usage counts.
3141 */
3142void __receive_sock(struct file *file)
3143{
3144 struct socket *sock;
d9539752 3145
dba4a925 3146 sock = sock_from_file(file);
d9539752
KC		 3147 if (sock) {
3148 sock_update_netprioidx(&sock->sk->sk_cgrp_data);
3149 sock_update_classid(&sock->sk->sk_cgrp_data);
3150 }
3151}
3152
1da177e4
LT		 3153ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
3154{
3155 ssize_t res;
3156 struct msghdr msg = {.msg_flags = flags};
3157 struct kvec iov;
3158 char *kaddr = kmap(page);
3159 iov.iov_base = kaddr + offset;
3160 iov.iov_len = size;
3161 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
3162 kunmap(page);
3163 return res;
3164}
2a91525c 3165EXPORT_SYMBOL(sock_no_sendpage);
1da177e4 3166
306b13eb
TH		 3167ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
3168 int offset, size_t size, int flags)
3169{
3170 ssize_t res;
3171 struct msghdr msg = {.msg_flags = flags};
3172 struct kvec iov;
3173 char *kaddr = kmap(page);
3174
3175 iov.iov_base = kaddr + offset;
3176 iov.iov_len = size;
3177 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
3178 kunmap(page);
3179 return res;
3180}
3181EXPORT_SYMBOL(sock_no_sendpage_locked);
3182
1da177e4
LT		 3183/*
3184 * Default Socket Callbacks
3185 */
3186
3187static void sock_def_wakeup(struct sock *sk)
3188{
43815482
ED		 3189 struct socket_wq *wq;
3190
3191 rcu_read_lock();
3192 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 3193 if (skwq_has_sleeper(wq))
43815482
ED		 3194 wake_up_interruptible_all(&wq->wait);
3195 rcu_read_unlock();
1da177e4
LT		 3196}
3197
3198static void sock_def_error_report(struct sock *sk)
3199{
43815482
ED		 3200 struct socket_wq *wq;
3201
3202 rcu_read_lock();
3203 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 3204 if (skwq_has_sleeper(wq))
a9a08845 3205 wake_up_interruptible_poll(&wq->wait, EPOLLERR);
8d8ad9d7 3206 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
43815482 3207 rcu_read_unlock();
1da177e4
LT		 3208}
3209
43a825af 3210void sock_def_readable(struct sock *sk)
1da177e4 3211{
43815482
ED		 3212 struct socket_wq *wq;
3213
3214 rcu_read_lock();
3215 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 3216 if (skwq_has_sleeper(wq))
a9a08845
LT		 3217 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
3218 EPOLLRDNORM | EPOLLRDBAND);
8d8ad9d7 3219 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
43815482 3220 rcu_read_unlock();
1da177e4
LT		 3221}
3222
3223static void sock_def_write_space(struct sock *sk)
3224{
43815482
ED		 3225 struct socket_wq *wq;
3226
3227 rcu_read_lock();
1da177e4
LT		 3228
3229 /* Do not wake up a writer until he can make "significant"
3230 * progress. --DaveM
3231 */
14bfee9b 3232 if (sock_writeable(sk)) {
43815482 3233 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 3234 if (skwq_has_sleeper(wq))
a9a08845
LT		 3235 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
3236 EPOLLWRNORM | EPOLLWRBAND);
1da177e4
LT		 3237
3238 /* Should agree with poll, otherwise some programs break */
14bfee9b 3239 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
1da177e4
LT		 3240 }
3241
43815482 3242 rcu_read_unlock();
1da177e4
LT		 3243}
3244
0a8afd9f
PB		 3245/* An optimised version of sock_def_write_space(), should only be called
3246 * for SOCK_RCU_FREE sockets under RCU read section and after putting
3247 * ->sk_wmem_alloc.
3248 */
3249static void sock_def_write_space_wfree(struct sock *sk)
3250{
3251 /* Do not wake up a writer until he can make "significant"
3252 * progress. --DaveM
3253 */
3254 if (sock_writeable(sk)) {
3255 struct socket_wq *wq = rcu_dereference(sk->sk_wq);
3256
3257 /* rely on refcount_sub from sock_wfree() */
3258 smp_mb__after_atomic();
3259 if (wq && waitqueue_active(&wq->wait))
3260 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
3261 EPOLLWRNORM | EPOLLWRBAND);
3262
3263 /* Should agree with poll, otherwise some programs break */
3264 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
3265 }
3266}
3267
1da177e4
LT		 3268static void sock_def_destruct(struct sock *sk)
3269{
1da177e4
LT		 3270}
3271
3272void sk_send_sigurg(struct sock *sk)
3273{
3274 if (sk->sk_socket && sk->sk_socket->file)
3275 if (send_sigurg(&sk->sk_socket->file->f_owner))
8d8ad9d7 3276 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
1da177e4 3277}
2a91525c 3278EXPORT_SYMBOL(sk_send_sigurg);
1da177e4
LT		 3279
3280void sk_reset_timer(struct sock *sk, struct timer_list* timer,
3281 unsigned long expires)
3282{
3283 if (!mod_timer(timer, expires))
3284 sock_hold(sk);
3285}
1da177e4
LT		 3286EXPORT_SYMBOL(sk_reset_timer);
3287
3288void sk_stop_timer(struct sock *sk, struct timer_list* timer)
3289{
25cc4ae9 3290 if (del_timer(timer))
1da177e4
LT		 3291 __sock_put(sk);
3292}
1da177e4
LT		 3293EXPORT_SYMBOL(sk_stop_timer);
3294
08b81d87
GT		 3295void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer)
3296{
3297 if (del_timer_sync(timer))
3298 __sock_put(sk);
3299}
3300EXPORT_SYMBOL(sk_stop_timer_sync);
3301
1da177e4
LT		 3302void sock_init_data(struct socket *sock, struct sock *sk)
3303{
581319c5 3304 sk_init_common(sk);
1da177e4
LT		 3305 sk->sk_send_head = NULL;
3306
99767f27 3307 timer_setup(&sk->sk_timer, NULL, 0);
4ec93edb 3308
1da177e4
LT		 3309 sk->sk_allocation = GFP_KERNEL;
3310 sk->sk_rcvbuf = sysctl_rmem_default;
3311 sk->sk_sndbuf = sysctl_wmem_default;
3312 sk->sk_state = TCP_CLOSE;
972692e0 3313 sk_set_socket(sk, sock);
1da177e4
LT		 3314
3315 sock_set_flag(sk, SOCK_ZAPPED);
3316
e71a4783 3317 if (sock) {
1da177e4 3318 sk->sk_type = sock->type;
333f7909 3319 RCU_INIT_POINTER(sk->sk_wq, &sock->wq);
1da177e4 3320 sock->sk = sk;
86741ec2
LC		 3321 sk->sk_uid = SOCK_INODE(sock)->i_uid;
3322 } else {
c2f26e8f 3323 RCU_INIT_POINTER(sk->sk_wq, NULL);
86741ec2
LC		 3324 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
3325 }
1da177e4 3326
1da177e4 3327 rwlock_init(&sk->sk_callback_lock);
cdfbabfb
DH		 3328 if (sk->sk_kern_sock)
3329 lockdep_set_class_and_name(
3330 &sk->sk_callback_lock,
3331 af_kern_callback_keys + sk->sk_family,
3332 af_family_kern_clock_key_strings[sk->sk_family]);
3333 else
3334 lockdep_set_class_and_name(
3335 &sk->sk_callback_lock,
443aef0e
PZ		 3336 af_callback_keys + sk->sk_family,
3337 af_family_clock_key_strings[sk->sk_family]);
1da177e4
LT		 3338
3339 sk->sk_state_change = sock_def_wakeup;
3340 sk->sk_data_ready = sock_def_readable;
3341 sk->sk_write_space = sock_def_write_space;
3342 sk->sk_error_report = sock_def_error_report;
3343 sk->sk_destruct = sock_def_destruct;
3344
5640f768
ED		 3345 sk->sk_frag.page = NULL;
3346 sk->sk_frag.offset = 0;
ef64a54f 3347 sk->sk_peek_off = -1;
1da177e4 3348
109f6e39
EB		 3349 sk->sk_peer_pid = NULL;
3350 sk->sk_peer_cred = NULL;
35306eb2
ED		 3351 spin_lock_init(&sk->sk_peer_lock);
3352
1da177e4
LT		 3353 sk->sk_write_pending = 0;
3354 sk->sk_rcvlowat = 1;
3355 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
3356 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
3357
6c7c98ba 3358 sk->sk_stamp = SK_DEFAULT_STAMP;
3a0ed3e9
DD		 3359#if BITS_PER_LONG==32
3360 seqlock_init(&sk->sk_stamp_seq);
3361#endif
52267790 3362 atomic_set(&sk->sk_zckey, 0);
1da177e4 3363
e0d1095a 3364#ifdef CONFIG_NET_RX_BUSY_POLL
06021292 3365 sk->sk_napi_id = 0;
64b0dc51 3366 sk->sk_ll_usec = sysctl_net_busy_read;
06021292
ET		 3367#endif
3368
76a9ebe8
ED		 3369 sk->sk_max_pacing_rate = ~0UL;
3370 sk->sk_pacing_rate = ~0UL;
7c68fa2b 3371 WRITE_ONCE(sk->sk_pacing_shift, 10);
70da268b 3372 sk->sk_incoming_cpu = -1;
26859240 3373 sk->sk_txrehash = SOCK_TXREHASH_DEFAULT;
c6345ce7
AN		 3374
3375 sk_rx_queue_clear(sk);
4dc6dc71
ED		 3376 /*
3377 * Before updating sk_refcnt, we must commit prior changes to memory
2cdb54c9 3378 * (Documentation/RCU/rculist_nulls.rst for details)
4dc6dc71
ED		 3379 */
3380 smp_wmb();
41c6d650 3381 refcount_set(&sk->sk_refcnt, 1);
33c732c3 3382 atomic_set(&sk->sk_drops, 0);
1da177e4 3383}
2a91525c 3384EXPORT_SYMBOL(sock_init_data);
1da177e4 3385
b5606c2d 3386void lock_sock_nested(struct sock *sk, int subclass)
1da177e4 3387{
2dcb96ba
TG		 3388 /* The sk_lock has mutex_lock() semantics here. */
3389 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
3390
1da177e4 3391 might_sleep();
a5b5bb9a 3392 spin_lock_bh(&sk->sk_lock.slock);
33d60fbd 3393 if (sock_owned_by_user_nocheck(sk))
1da177e4 3394 __lock_sock(sk);
d2e9117c 3395 sk->sk_lock.owned = 1;
2dcb96ba 3396 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4 3397}
fcc70d5f 3398EXPORT_SYMBOL(lock_sock_nested);
1da177e4 3399
b5606c2d 3400void release_sock(struct sock *sk)
1da177e4 3401{
a5b5bb9a 3402 spin_lock_bh(&sk->sk_lock.slock);
1da177e4
LT		 3403 if (sk->sk_backlog.tail)
3404 __release_sock(sk);
46d3ceab 3405
c3f9b018
ED		 3406 /* Warning : release_cb() might need to release sk ownership,
3407 * ie call sock_release_ownership(sk) before us.
3408 */
46d3ceab
ED		 3409 if (sk->sk_prot->release_cb)
3410 sk->sk_prot->release_cb(sk);
3411
c3f9b018 3412 sock_release_ownership(sk);
a5b5bb9a
IM		 3413 if (waitqueue_active(&sk->sk_lock.wq))
3414 wake_up(&sk->sk_lock.wq);
3415 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4
LT		 3416}
3417EXPORT_SYMBOL(release_sock);
3418
49054556 3419bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock)
8a74ad60
ED		 3420{
3421 might_sleep();
3422 spin_lock_bh(&sk->sk_lock.slock);
3423
33d60fbd 3424 if (!sock_owned_by_user_nocheck(sk)) {
8a74ad60 3425 /*
2dcb96ba
TG		 3426 * Fast path return with bottom halves disabled and
3427 * sock::sk_lock.slock held.
3428 *
3429 * The 'mutex' is not contended and holding
3430 * sock::sk_lock.slock prevents all other lockers to
3431 * proceed so the corresponding unlock_sock_fast() can
3432 * avoid the slow path of release_sock() completely and
3433 * just release slock.
3434 *
3435 * From a semantical POV this is equivalent to 'acquiring'
3436 * the 'mutex', hence the corresponding lockdep
3437 * mutex_release() has to happen in the fast path of
3438 * unlock_sock_fast().
8a74ad60
ED		 3439 */
3440 return false;
2dcb96ba 3441 }
8a74ad60
ED		 3442
3443 __lock_sock(sk);
3444 sk->sk_lock.owned = 1;
12f4bd86 3445 __acquire(&sk->sk_lock.slock);
2dcb96ba 3446 spin_unlock_bh(&sk->sk_lock.slock);
8a74ad60
ED		 3447 return true;
3448}
49054556 3449EXPORT_SYMBOL(__lock_sock_fast);
8a74ad60 3450
c7cbdbf2
AB		 3451int sock_gettstamp(struct socket *sock, void __user *userstamp,
3452 bool timeval, bool time32)
4ec93edb 3453{
c7cbdbf2
AB		 3454 struct sock *sk = sock->sk;
3455 struct timespec64 ts;
9dae3497
YS		 3456
3457 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
c7cbdbf2
AB		 3458 ts = ktime_to_timespec64(sock_read_timestamp(sk));
3459 if (ts.tv_sec == -1)
1da177e4 3460 return -ENOENT;
c7cbdbf2 3461 if (ts.tv_sec == 0) {
3a0ed3e9 3462 ktime_t kt = ktime_get_real();
f95f96a4 3463 sock_write_timestamp(sk, kt);
c7cbdbf2 3464 ts = ktime_to_timespec64(kt);
b7aa0bf7 3465 }
1da177e4 3466
c7cbdbf2
AB		 3467 if (timeval)
3468 ts.tv_nsec /= 1000;
9dae3497 3469
c7cbdbf2
AB		 3470#ifdef CONFIG_COMPAT_32BIT_TIME
3471 if (time32)
3472 return put_old_timespec32(&ts, userstamp);
3473#endif
3474#ifdef CONFIG_SPARC64
3475 /* beware of padding in sparc64 timeval */
3476 if (timeval && !in_compat_syscall()) {
3477 struct __kernel_old_timeval __user tv = {
c98f4822
SR		 3478 .tv_sec = ts.tv_sec,
3479 .tv_usec = ts.tv_nsec,
c7cbdbf2 3480 };
c98f4822 3481 if (copy_to_user(userstamp, &tv, sizeof(tv)))
c7cbdbf2
AB		 3482 return -EFAULT;
3483 return 0;
ae40eb1e 3484 }
c7cbdbf2
AB		 3485#endif
3486 return put_timespec64(&ts, userstamp);
ae40eb1e 3487}
c7cbdbf2 3488EXPORT_SYMBOL(sock_gettstamp);
ae40eb1e 3489
193d357d 3490void sock_enable_timestamp(struct sock *sk, enum sock_flags flag)
4ec93edb 3491{
20d49473 3492 if (!sock_flag(sk, flag)) {
08e29af3
ED		 3493 unsigned long previous_flags = sk->sk_flags;
3494
20d49473
PO		 3495 sock_set_flag(sk, flag);
3496 /*
3497 * we just set one of the two flags which require net
3498 * time stamping, but time stamping might have been on
3499 * already because of the other one
3500 */
080a270f
HFS		 3501 if (sock_needs_netstamp(sk) &&
3502 !(previous_flags & SK_FLAGS_TIMESTAMP))
20d49473 3503 net_enable_timestamp();
1da177e4
LT		 3504 }
3505}
1da177e4 3506
cb820f8e
RC		 3507int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
3508 int level, int type)
3509{
3510 struct sock_exterr_skb *serr;
364a9e93 3511 struct sk_buff *skb;
cb820f8e
RC		 3512 int copied, err;
3513
3514 err = -EAGAIN;
364a9e93 3515 skb = sock_dequeue_err_skb(sk);
cb820f8e
RC		 3516 if (skb == NULL)
3517 goto out;
3518
3519 copied = skb->len;
3520 if (copied > len) {
3521 msg->msg_flags |= MSG_TRUNC;
3522 copied = len;
3523 }
51f3d02b 3524 err = skb_copy_datagram_msg(skb, 0, msg, copied);
cb820f8e
RC		 3525 if (err)
3526 goto out_free_skb;
3527
3528 sock_recv_timestamp(msg, sk, skb);
3529
3530 serr = SKB_EXT_ERR(skb);
3531 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
3532
3533 msg->msg_flags |= MSG_ERRQUEUE;
3534 err = copied;
3535
cb820f8e
RC		 3536out_free_skb:
3537 kfree_skb(skb);
3538out:
3539 return err;
3540}
3541EXPORT_SYMBOL(sock_recv_errqueue);
3542
1da177e4
LT		 3543/*
3544 * Get a socket option on an socket.
3545 *
3546 * FIX: POSIX 1003.1g is very ambiguous here. It states that
3547 * asynchronous errors should be reported by getsockopt. We assume
3548 * this means if you specify SO_ERROR (otherwise whats the point of it).
3549 */
3550int sock_common_getsockopt(struct socket *sock, int level, int optname,
3551 char __user *optval, int __user *optlen)
3552{
3553 struct sock *sk = sock->sk;
3554
3555 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
3556}
1da177e4
LT		 3557EXPORT_SYMBOL(sock_common_getsockopt);
3558
1b784140
YX		 3559int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
3560 int flags)
1da177e4
LT		 3561{
3562 struct sock *sk = sock->sk;
3563 int addr_len = 0;
3564 int err;
3565
ec095263 3566 err = sk->sk_prot->recvmsg(sk, msg, size, flags, &addr_len);
1da177e4
LT		 3567 if (err >= 0)
3568 msg->msg_namelen = addr_len;
3569 return err;
3570}
1da177e4
LT		 3571EXPORT_SYMBOL(sock_common_recvmsg);
3572
3573/*
3574 * Set socket options on an inet socket.
3575 */
3576int sock_common_setsockopt(struct socket *sock, int level, int optname,
a7b75c5a 3577 sockptr_t optval, unsigned int optlen)
1da177e4
LT		 3578{
3579 struct sock *sk = sock->sk;
3580
3581 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
3582}
1da177e4
LT		 3583EXPORT_SYMBOL(sock_common_setsockopt);
3584
3585void sk_common_release(struct sock *sk)
3586{
3587 if (sk->sk_prot->destroy)
3588 sk->sk_prot->destroy(sk);
3589
3590 /*
645f0897 3591 * Observation: when sk_common_release is called, processes have
1da177e4
LT		 3592 * no access to socket. But net still has.
3593 * Step one, detach it from networking:
3594 *
3595 * A. Remove from hash tables.
3596 */
3597
3598 sk->sk_prot->unhash(sk);
3599
3600 /*
3601 * In this point socket cannot receive new packets, but it is possible
3602 * that some packets are in flight because some CPU runs receiver and
3603 * did hash table lookup before we unhashed socket. They will achieve
3604 * receive queue and will be purged by socket destructor.
3605 *
3606 * Also we still have packets pending on receive queue and probably,
3607 * our own packets waiting in device queues. sock_destroy will drain
3608 * receive queue, but transmitted packets will delay socket destruction
3609 * until the last reference will be released.
3610 */
3611
3612 sock_orphan(sk);
3613
3614 xfrm_sk_free_policy(sk);
3615
e6848976 3616 sk_refcnt_debug_release(sk);
5640f768 3617
1da177e4
LT		 3618 sock_put(sk);
3619}
1da177e4
LT		 3620EXPORT_SYMBOL(sk_common_release);
3621
a2d133b1
JH		 3622void sk_get_meminfo(const struct sock *sk, u32 *mem)
3623{
3624 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
3625
3626 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
ebb3b78d 3627 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
a2d133b1 3628 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
e292f05e 3629 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
a2d133b1 3630 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
ab4e846a 3631 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
a2d133b1 3632 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
70c26558 3633 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
a2d133b1
JH		 3634 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
3635}
3636
13ff3d6f 3637#ifdef CONFIG_PROC_FS
13ff3d6f 3638static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
70ee1159 3639
70ee1159
PE		 3640int sock_prot_inuse_get(struct net *net, struct proto *prot)
3641{
3642 int cpu, idx = prot->inuse_idx;
3643 int res = 0;
3644
3645 for_each_possible_cpu(cpu)
08fc7f81 3646 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
70ee1159
PE		 3647
3648 return res >= 0 ? res : 0;
3649}
3650EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
3651
648845ab
TZ		 3652int sock_inuse_get(struct net *net)
3653{
3654 int cpu, res = 0;
3655
3656 for_each_possible_cpu(cpu)
4199bae1 3657 res += per_cpu_ptr(net->core.prot_inuse, cpu)->all;
648845ab
TZ		 3658
3659 return res;
3660}
3661
3662EXPORT_SYMBOL_GPL(sock_inuse_get);
3663
2c8c1e72 3664static int __net_init sock_inuse_init_net(struct net *net)
70ee1159 3665{
08fc7f81 3666 net->core.prot_inuse = alloc_percpu(struct prot_inuse);
648845ab
TZ		 3667 if (net->core.prot_inuse == NULL)
3668 return -ENOMEM;
648845ab 3669 return 0;
70ee1159
PE		 3670}
3671
2c8c1e72 3672static void __net_exit sock_inuse_exit_net(struct net *net)
70ee1159 3673{
08fc7f81 3674 free_percpu(net->core.prot_inuse);
70ee1159
PE		 3675}
3676
3677static struct pernet_operations net_inuse_ops = {
3678 .init = sock_inuse_init_net,
3679 .exit = sock_inuse_exit_net,
3680};
3681
3682static __init int net_inuse_init(void)
3683{
3684 if (register_pernet_subsys(&net_inuse_ops))
3685 panic("Cannot initialize net inuse counters");
3686
3687 return 0;
3688}
3689
3690core_initcall(net_inuse_init);
13ff3d6f 3691
b45ce321 3692static int assign_proto_idx(struct proto *prot)
13ff3d6f
PE		 3693{
3694 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3695
3696 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
e005d193 3697 pr_err("PROTO_INUSE_NR exhausted\n");
b45ce321 3698 return -ENOSPC;
13ff3d6f
PE		 3699 }
3700
3701 set_bit(prot->inuse_idx, proto_inuse_idx);
b45ce321 3702 return 0;
13ff3d6f
PE		 3703}
3704
3705static void release_proto_idx(struct proto *prot)
3706{
3707 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3708 clear_bit(prot->inuse_idx, proto_inuse_idx);
3709}
3710#else
b45ce321 3711static inline int assign_proto_idx(struct proto *prot)
13ff3d6f 3712{
b45ce321 3713 return 0;
13ff3d6f
PE		 3714}
3715
3716static inline void release_proto_idx(struct proto *prot)
3717{
3718}
648845ab 3719
13ff3d6f
PE		 3720#endif
3721
0f5907af
ML		 3722static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot)
3723{
3724 if (!twsk_prot)
3725 return;
3726 kfree(twsk_prot->twsk_slab_name);
3727 twsk_prot->twsk_slab_name = NULL;
3728 kmem_cache_destroy(twsk_prot->twsk_slab);
3729 twsk_prot->twsk_slab = NULL;
3730}
3731
b80350f3
TZ		 3732static int tw_prot_init(const struct proto *prot)
3733{
3734 struct timewait_sock_ops *twsk_prot = prot->twsk_prot;
3735
3736 if (!twsk_prot)
3737 return 0;
3738
3739 twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s",
3740 prot->name);
3741 if (!twsk_prot->twsk_slab_name)
3742 return -ENOMEM;
3743
3744 twsk_prot->twsk_slab =
3745 kmem_cache_create(twsk_prot->twsk_slab_name,
3746 twsk_prot->twsk_obj_size, 0,
3747 SLAB_ACCOUNT | prot->slab_flags,
3748 NULL);
3749 if (!twsk_prot->twsk_slab) {
3750 pr_crit("%s: Can't create timewait sock SLAB cache!\n",
3751 prot->name);
3752 return -ENOMEM;
3753 }
3754
3755 return 0;
3756}
3757
0159dfd3
ED		 3758static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3759{
3760 if (!rsk_prot)
3761 return;
3762 kfree(rsk_prot->slab_name);
3763 rsk_prot->slab_name = NULL;
adf78eda
JL		 3764 kmem_cache_destroy(rsk_prot->slab);
3765 rsk_prot->slab = NULL;
0159dfd3
ED		 3766}
3767
3768static int req_prot_init(const struct proto *prot)
3769{
3770 struct request_sock_ops *rsk_prot = prot->rsk_prot;
3771
3772 if (!rsk_prot)
3773 return 0;
3774
3775 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3776 prot->name);
3777 if (!rsk_prot->slab_name)
3778 return -ENOMEM;
3779
3780 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3781 rsk_prot->obj_size, 0,
e699e2c6
SB		 3782 SLAB_ACCOUNT | prot->slab_flags,
3783 NULL);
0159dfd3
ED		 3784
3785 if (!rsk_prot->slab) {
3786 pr_crit("%s: Can't create request sock SLAB cache!\n",
3787 prot->name);
3788 return -ENOMEM;
3789 }
3790 return 0;
3791}
3792
b733c007
PE		 3793int proto_register(struct proto *prot, int alloc_slab)
3794{
b45ce321 3795 int ret = -ENOBUFS;
3796
f20cfd66
ED		 3797 if (prot->memory_allocated && !prot->sysctl_mem) {
3798 pr_err("%s: missing sysctl_mem\n", prot->name);
3799 return -EINVAL;
3800 }
1da177e4 3801 if (alloc_slab) {
30c2c9f1
DW		 3802 prot->slab = kmem_cache_create_usercopy(prot->name,
3803 prot->obj_size, 0,
e699e2c6
SB		 3804 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
3805 prot->slab_flags,
289a4860 3806 prot->useroffset, prot->usersize,
271b72c7 3807 NULL);
1da177e4
LT		 3808
3809 if (prot->slab == NULL) {
e005d193
JP		 3810 pr_crit("%s: Can't create sock SLAB cache!\n",
3811 prot->name);
60e7663d 3812 goto out;
1da177e4 3813 }
2e6599cb 3814
0159dfd3
ED		 3815 if (req_prot_init(prot))
3816 goto out_free_request_sock_slab;
8feaf0c0 3817
b80350f3
TZ		 3818 if (tw_prot_init(prot))
3819 goto out_free_timewait_sock_slab;
1da177e4
LT		 3820 }
3821
36b77a52 3822 mutex_lock(&proto_list_mutex);
b45ce321 3823 ret = assign_proto_idx(prot);
3824 if (ret) {
3825 mutex_unlock(&proto_list_mutex);
0f5907af 3826 goto out_free_timewait_sock_slab;
b45ce321 3827 }
1da177e4 3828 list_add(&prot->node, &proto_list);
36b77a52 3829 mutex_unlock(&proto_list_mutex);
b45ce321 3830 return ret;
b733c007 3831
0f5907af 3832out_free_timewait_sock_slab:
ed744d81 3833 if (alloc_slab)
0f5907af 3834 tw_prot_cleanup(prot->twsk_prot);
8feaf0c0 3835out_free_request_sock_slab:
b45ce321 3836 if (alloc_slab) {
3837 req_prot_cleanup(prot->rsk_prot);
0159dfd3 3838
b45ce321 3839 kmem_cache_destroy(prot->slab);
3840 prot->slab = NULL;
3841 }
b733c007 3842out:
b45ce321 3843 return ret;
1da177e4 3844}
1da177e4
LT		 3845EXPORT_SYMBOL(proto_register);
3846
3847void proto_unregister(struct proto *prot)
3848{
36b77a52 3849 mutex_lock(&proto_list_mutex);
13ff3d6f 3850 release_proto_idx(prot);
0a3f4358 3851 list_del(&prot->node);
36b77a52 3852 mutex_unlock(&proto_list_mutex);
1da177e4 3853
adf78eda
JL		 3854 kmem_cache_destroy(prot->slab);
3855 prot->slab = NULL;
1da177e4 3856
0159dfd3 3857 req_prot_cleanup(prot->rsk_prot);
0f5907af 3858 tw_prot_cleanup(prot->twsk_prot);
1da177e4 3859}
1da177e4
LT		 3860EXPORT_SYMBOL(proto_unregister);
3861
bf2ae2e4
XL		 3862int sock_load_diag_module(int family, int protocol)
3863{
3864 if (!protocol) {
3865 if (!sock_is_registered(family))
3866 return -ENOENT;
3867
3868 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
3869 NETLINK_SOCK_DIAG, family);
3870 }
3871
3872#ifdef CONFIG_INET
3873 if (family == AF_INET &&
c34c1287 3874 protocol != IPPROTO_RAW &&
3f935c75 3875 protocol < MAX_INET_PROTOS &&
bf2ae2e4
XL		 3876 !rcu_access_pointer(inet_protos[protocol]))
3877 return -ENOENT;
3878#endif
3879
3880 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
3881 NETLINK_SOCK_DIAG, family, protocol);
3882}
3883EXPORT_SYMBOL(sock_load_diag_module);
3884
1da177e4 3885#ifdef CONFIG_PROC_FS
1da177e4 3886static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
36b77a52 3887 __acquires(proto_list_mutex)
1da177e4 3888{
36b77a52 3889 mutex_lock(&proto_list_mutex);
60f0438a 3890 return seq_list_start_head(&proto_list, *pos);
1da177e4
LT		 3891}
3892
3893static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3894{
60f0438a 3895 return seq_list_next(v, &proto_list, pos);
1da177e4
LT		 3896}
3897
3898static void proto_seq_stop(struct seq_file *seq, void *v)
36b77a52 3899 __releases(proto_list_mutex)
1da177e4 3900{
36b77a52 3901 mutex_unlock(&proto_list_mutex);
1da177e4
LT		 3902}
3903
3904static char proto_method_implemented(const void *method)
3905{
3906 return method == NULL ? 'n' : 'y';
3907}
180d8cd9
GC		 3908static long sock_prot_memory_allocated(struct proto *proto)
3909{
cb75a36c 3910 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
180d8cd9
GC		 3911}
3912
7a512eb8 3913static const char *sock_prot_memory_pressure(struct proto *proto)
180d8cd9
GC		 3914{
3915 return proto->memory_pressure != NULL ?
3916 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3917}
1da177e4
LT		 3918
3919static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3920{
180d8cd9 3921
8d987e5c 3922 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
1da177e4
LT		 3923 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3924 proto->name,
3925 proto->obj_size,
14e943db 3926 sock_prot_inuse_get(seq_file_net(seq), proto),
180d8cd9
GC		 3927 sock_prot_memory_allocated(proto),
3928 sock_prot_memory_pressure(proto),
1da177e4
LT		 3929 proto->max_header,
3930 proto->slab == NULL ? "no" : "yes",
3931 module_name(proto->owner),
3932 proto_method_implemented(proto->close),
3933 proto_method_implemented(proto->connect),
3934 proto_method_implemented(proto->disconnect),
3935 proto_method_implemented(proto->accept),
3936 proto_method_implemented(proto->ioctl),
3937 proto_method_implemented(proto->init),
3938 proto_method_implemented(proto->destroy),
3939 proto_method_implemented(proto->shutdown),
3940 proto_method_implemented(proto->setsockopt),
3941 proto_method_implemented(proto->getsockopt),
3942 proto_method_implemented(proto->sendmsg),
3943 proto_method_implemented(proto->recvmsg),
3944 proto_method_implemented(proto->sendpage),
3945 proto_method_implemented(proto->bind),
3946 proto_method_implemented(proto->backlog_rcv),
3947 proto_method_implemented(proto->hash),
3948 proto_method_implemented(proto->unhash),
3949 proto_method_implemented(proto->get_port),
3950 proto_method_implemented(proto->enter_memory_pressure));
3951}
3952
3953static int proto_seq_show(struct seq_file *seq, void *v)
3954{
60f0438a 3955 if (v == &proto_list)
1da177e4
LT		 3956 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3957 "protocol",
3958 "size",
3959 "sockets",
3960 "memory",
3961 "press",
3962 "maxhdr",
3963 "slab",
3964 "module",
3965 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3966 else
60f0438a 3967 proto_seq_printf(seq, list_entry(v, struct proto, node));
1da177e4
LT		 3968 return 0;
3969}
3970
f690808e 3971static const struct seq_operations proto_seq_ops = {
1da177e4
LT		 3972 .start = proto_seq_start,
3973 .next = proto_seq_next,
3974 .stop = proto_seq_stop,
3975 .show = proto_seq_show,
3976};
3977
14e943db
ED		 3978static __net_init int proto_init_net(struct net *net)
3979{
c3506372
CH		 3980 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
3981 sizeof(struct seq_net_private)))
14e943db
ED		 3982 return -ENOMEM;
3983
3984 return 0;
3985}
3986
3987static __net_exit void proto_exit_net(struct net *net)
3988{
ece31ffd 3989 remove_proc_entry("protocols", net->proc_net);
14e943db
ED		 3990}
3991
3992
3993static __net_initdata struct pernet_operations proto_net_ops = {
3994 .init = proto_init_net,
3995 .exit = proto_exit_net,
1da177e4
LT		 3996};
3997
3998static int __init proto_init(void)
3999{
14e943db 4000 return register_pernet_subsys(&proto_net_ops);
1da177e4
LT		 4001}
4002
4003subsys_initcall(proto_init);
4004
4005#endif /* PROC_FS */
7db6b048
SS		 4006
4007#ifdef CONFIG_NET_RX_BUSY_POLL
4008bool sk_busy_loop_end(void *p, unsigned long start_time)
4009{
4010 struct sock *sk = p;
4011
3f926af3 4012 return !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
7db6b048
SS		 4013 sk_busy_loop_timeout(sk, start_time);
4014}
4015EXPORT_SYMBOL(sk_busy_loop_end);
4016#endif /* CONFIG_NET_RX_BUSY_POLL */
c0425a42
CH		 4017
4018int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len)
4019{
4020 if (!sk->sk_prot->bind_add)
4021 return -EOPNOTSUPP;
4022 return sk->sk_prot->bind_add(sk, addr, addr_len);
4023}
4024EXPORT_SYMBOL(sock_bind_add);
Linux 6.x block layer and io_uring tree(s)