projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ocfs2: fix unbalanced locking
[linux-block.git] / net / core / sock.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Generic socket support routines. Memory allocators, socket lock/release
8 * handler for protocols to use and generic option handler.
9 *
02c30a84 10 * Authors: Ross Biro
1da177e4
LT		 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
4ec93edb 35 * code. The ACK stuff can wait and needs major
1da177e4
LT		 36 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
1da177e4
LT		 84 */
85
e005d193
JP		 86#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
87
80b14dee 88#include <asm/unaligned.h>
4fc268d2 89#include <linux/capability.h>
1da177e4 90#include <linux/errno.h>
cb820f8e 91#include <linux/errqueue.h>
1da177e4
LT		 92#include <linux/types.h>
93#include <linux/socket.h>
94#include <linux/in.h>
95#include <linux/kernel.h>
1da177e4
LT		 96#include <linux/module.h>
97#include <linux/proc_fs.h>
98#include <linux/seq_file.h>
99#include <linux/sched.h>
f1083048 100#include <linux/sched/mm.h>
1da177e4
LT		 101#include <linux/timer.h>
102#include <linux/string.h>
103#include <linux/sockios.h>
104#include <linux/net.h>
105#include <linux/mm.h>
106#include <linux/slab.h>
107#include <linux/interrupt.h>
108#include <linux/poll.h>
109#include <linux/tcp.h>
110#include <linux/init.h>
a1f8e7f7 111#include <linux/highmem.h>
3f551f94 112#include <linux/user_namespace.h>
c5905afb 113#include <linux/static_key.h>
3969eb38 114#include <linux/memcontrol.h>
8c1ae10d 115#include <linux/prefetch.h>
a6c0d093 116#include <linux/compat.h>
1da177e4 117
7c0f6ba6 118#include <linux/uaccess.h>
1da177e4
LT		 119
120#include <linux/netdevice.h>
121#include <net/protocol.h>
122#include <linux/skbuff.h>
457c4cbc 123#include <net/net_namespace.h>
2e6599cb 124#include <net/request_sock.h>
1da177e4 125#include <net/sock.h>
20d49473 126#include <linux/net_tstamp.h>
1da177e4
LT		 127#include <net/xfrm.h>
128#include <linux/ipsec.h>
f8451725 129#include <net/cls_cgroup.h>
5bc1421e 130#include <net/netprio_cgroup.h>
eb4cb008 131#include <linux/sock_diag.h>
1da177e4
LT		 132
133#include <linux/filter.h>
538950a1 134#include <net/sock_reuseport.h>
6ac99e8f 135#include <net/bpf_sk_storage.h>
1da177e4 136
3847ce32
SM		 137#include <trace/events/sock.h>
138
1da177e4 139#include <net/tcp.h>
076bb0c8 140#include <net/busy_poll.h>
06021292 141
36b77a52 142static DEFINE_MUTEX(proto_list_mutex);
d1a4c0b3
GC		 143static LIST_HEAD(proto_list);
144
648845ab
TZ		 145static void sock_inuse_add(struct net *net, int val);
146
a3b299da
EB		 147/**
148 * sk_ns_capable - General socket capability test
149 * @sk: Socket to use a capability on or through
150 * @user_ns: The user namespace of the capability to use
151 * @cap: The capability to use
152 *
153 * Test to see if the opener of the socket had when the socket was
154 * created and the current process has the capability @cap in the user
155 * namespace @user_ns.
156 */
157bool sk_ns_capable(const struct sock *sk,
158 struct user_namespace *user_ns, int cap)
159{
160 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
161 ns_capable(user_ns, cap);
162}
163EXPORT_SYMBOL(sk_ns_capable);
164
165/**
166 * sk_capable - Socket global capability test
167 * @sk: Socket to use a capability on or through
e793c0f7 168 * @cap: The global capability to use
a3b299da
EB		 169 *
170 * Test to see if the opener of the socket had when the socket was
171 * created and the current process has the capability @cap in all user
172 * namespaces.
173 */
174bool sk_capable(const struct sock *sk, int cap)
175{
176 return sk_ns_capable(sk, &init_user_ns, cap);
177}
178EXPORT_SYMBOL(sk_capable);
179
180/**
181 * sk_net_capable - Network namespace socket capability test
182 * @sk: Socket to use a capability on or through
183 * @cap: The capability to use
184 *
e793c0f7 185 * Test to see if the opener of the socket had when the socket was created
a3b299da
EB		 186 * and the current process has the capability @cap over the network namespace
187 * the socket is a member of.
188 */
189bool sk_net_capable(const struct sock *sk, int cap)
190{
191 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
192}
193EXPORT_SYMBOL(sk_net_capable);
194
da21f24d
IM		 195/*
196 * Each address family might have different locking rules, so we have
cdfbabfb
DH		 197 * one slock key per address family and separate keys for internal and
198 * userspace sockets.
da21f24d 199 */
a5b5bb9a 200static struct lock_class_key af_family_keys[AF_MAX];
cdfbabfb 201static struct lock_class_key af_family_kern_keys[AF_MAX];
a5b5bb9a 202static struct lock_class_key af_family_slock_keys[AF_MAX];
cdfbabfb 203static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
a5b5bb9a 204
a5b5bb9a
IM		 205/*
206 * Make lock validator output more readable. (we pre-construct these
207 * strings build-time, so that runtime initialization of socket
208 * locks is fast):
209 */
cdfbabfb
DH		 210
211#define _sock_locks(x) \
212 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \
213 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \
214 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \
215 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \
216 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \
217 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \
218 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \
219 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \
220 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \
221 x "27" , x "28" , x "AF_CAN" , \
222 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \
223 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
224 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
225 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
68e8b849
BT		 226 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \
227 x "AF_MAX"
cdfbabfb 228
36cbd3dc 229static const char *const af_family_key_strings[AF_MAX+1] = {
cdfbabfb 230 _sock_locks("sk_lock-")
a5b5bb9a 231};
36cbd3dc 232static const char *const af_family_slock_key_strings[AF_MAX+1] = {
cdfbabfb 233 _sock_locks("slock-")
a5b5bb9a 234};
36cbd3dc 235static const char *const af_family_clock_key_strings[AF_MAX+1] = {
cdfbabfb
DH		 236 _sock_locks("clock-")
237};
238
239static const char *const af_family_kern_key_strings[AF_MAX+1] = {
240 _sock_locks("k-sk_lock-")
241};
242static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
243 _sock_locks("k-slock-")
244};
245static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
246 _sock_locks("k-clock-")
443aef0e 247};
581319c5 248static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
6b431d50 249 _sock_locks("rlock-")
581319c5
PA		 250};
251static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
6b431d50 252 _sock_locks("wlock-")
581319c5
PA		 253};
254static const char *const af_family_elock_key_strings[AF_MAX+1] = {
6b431d50 255 _sock_locks("elock-")
581319c5 256};
da21f24d
IM		 257
258/*
581319c5 259 * sk_callback_lock and sk queues locking rules are per-address-family,
da21f24d
IM		 260 * so split the lock classes by using a per-AF key:
261 */
262static struct lock_class_key af_callback_keys[AF_MAX];
581319c5
PA		 263static struct lock_class_key af_rlock_keys[AF_MAX];
264static struct lock_class_key af_wlock_keys[AF_MAX];
265static struct lock_class_key af_elock_keys[AF_MAX];
cdfbabfb 266static struct lock_class_key af_kern_callback_keys[AF_MAX];
da21f24d 267
1da177e4 268/* Run time adjustable parameters. */
ab32ea5d 269__u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
6d8ebc8a 270EXPORT_SYMBOL(sysctl_wmem_max);
ab32ea5d 271__u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
6d8ebc8a 272EXPORT_SYMBOL(sysctl_rmem_max);
ab32ea5d
BH		 273__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
274__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
1da177e4 275
25985edc 276/* Maximal space eaten by iovec or ancillary data plus some space */
ab32ea5d 277int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
2a91525c 278EXPORT_SYMBOL(sysctl_optmem_max);
1da177e4 279
b245be1f
WB		 280int sysctl_tstamp_allow_data __read_mostly = 1;
281
a7950ae8
DB		 282DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
283EXPORT_SYMBOL_GPL(memalloc_socks_key);
c93bdd0e 284
7cb02404
MG		 285/**
286 * sk_set_memalloc - sets %SOCK_MEMALLOC
287 * @sk: socket to set it on
288 *
289 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
290 * It's the responsibility of the admin to adjust min_free_kbytes
291 * to meet the requirements
292 */
293void sk_set_memalloc(struct sock *sk)
294{
295 sock_set_flag(sk, SOCK_MEMALLOC);
296 sk->sk_allocation |= __GFP_MEMALLOC;
a7950ae8 297 static_branch_inc(&memalloc_socks_key);
7cb02404
MG		 298}
299EXPORT_SYMBOL_GPL(sk_set_memalloc);
300
301void sk_clear_memalloc(struct sock *sk)
302{
303 sock_reset_flag(sk, SOCK_MEMALLOC);
304 sk->sk_allocation &= ~__GFP_MEMALLOC;
a7950ae8 305 static_branch_dec(&memalloc_socks_key);
c76562b6
MG		 306
307 /*
308 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
5d753610
MG		 309 * progress of swapping. SOCK_MEMALLOC may be cleared while
310 * it has rmem allocations due to the last swapfile being deactivated
311 * but there is a risk that the socket is unusable due to exceeding
312 * the rmem limits. Reclaim the reserves and obey rmem limits again.
c76562b6 313 */
5d753610 314 sk_mem_reclaim(sk);
7cb02404
MG		 315}
316EXPORT_SYMBOL_GPL(sk_clear_memalloc);
317
b4b9e355
MG		 318int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
319{
320 int ret;
f1083048 321 unsigned int noreclaim_flag;
b4b9e355
MG		 322
323 /* these should have been dropped before queueing */
324 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
325
f1083048 326 noreclaim_flag = memalloc_noreclaim_save();
b4b9e355 327 ret = sk->sk_backlog_rcv(sk, skb);
f1083048 328 memalloc_noreclaim_restore(noreclaim_flag);
b4b9e355
MG		 329
330 return ret;
331}
332EXPORT_SYMBOL(__sk_backlog_rcv);
333
a9beb86a 334static int sock_get_timeout(long timeo, void *optval, bool old_timeval)
fe0c72f3 335{
a9beb86a 336 struct __kernel_sock_timeval tv;
fe0c72f3
AB		 337
338 if (timeo == MAX_SCHEDULE_TIMEOUT) {
339 tv.tv_sec = 0;
340 tv.tv_usec = 0;
341 } else {
342 tv.tv_sec = timeo / HZ;
343 tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ;
344 }
345
e6986423 346 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
fe0c72f3
AB		 347 struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec };
348 *(struct old_timeval32 *)optval = tv32;
349 return sizeof(tv32);
350 }
351
a9beb86a
DD		 352 if (old_timeval) {
353 struct __kernel_old_timeval old_tv;
354 old_tv.tv_sec = tv.tv_sec;
355 old_tv.tv_usec = tv.tv_usec;
356 *(struct __kernel_old_timeval *)optval = old_tv;
28e72b26 357 return sizeof(old_tv);
a9beb86a
DD		 358 }
359
28e72b26
VC		 360 *(struct __kernel_sock_timeval *)optval = tv;
361 return sizeof(tv);
fe0c72f3
AB		 362}
363
c34645ac
CH		 364static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen,
365 bool old_timeval)
1da177e4 366{
a9beb86a 367 struct __kernel_sock_timeval tv;
1da177e4 368
e6986423 369 if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
fe0c72f3
AB		 370 struct old_timeval32 tv32;
371
372 if (optlen < sizeof(tv32))
373 return -EINVAL;
374
c34645ac 375 if (copy_from_sockptr(&tv32, optval, sizeof(tv32)))
fe0c72f3
AB		 376 return -EFAULT;
377 tv.tv_sec = tv32.tv_sec;
378 tv.tv_usec = tv32.tv_usec;
a9beb86a
DD		 379 } else if (old_timeval) {
380 struct __kernel_old_timeval old_tv;
381
382 if (optlen < sizeof(old_tv))
383 return -EINVAL;
c34645ac 384 if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv)))
a9beb86a
DD		 385 return -EFAULT;
386 tv.tv_sec = old_tv.tv_sec;
387 tv.tv_usec = old_tv.tv_usec;
fe0c72f3
AB		 388 } else {
389 if (optlen < sizeof(tv))
390 return -EINVAL;
c34645ac 391 if (copy_from_sockptr(&tv, optval, sizeof(tv)))
fe0c72f3
AB		 392 return -EFAULT;
393 }
ba78073e
VA		 394 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
395 return -EDOM;
1da177e4 396
ba78073e 397 if (tv.tv_sec < 0) {
6f11df83
AM		 398 static int warned __read_mostly;
399
ba78073e 400 *timeo_p = 0;
50aab54f 401 if (warned < 10 && net_ratelimit()) {
ba78073e 402 warned++;
e005d193
JP		 403 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
404 __func__, current->comm, task_pid_nr(current));
50aab54f 405 }
ba78073e
VA		 406 return 0;
407 }
1da177e4
LT		 408 *timeo_p = MAX_SCHEDULE_TIMEOUT;
409 if (tv.tv_sec == 0 && tv.tv_usec == 0)
410 return 0;
a9beb86a
DD		 411 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))
412 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ);
1da177e4
LT		 413 return 0;
414}
415
416static void sock_warn_obsolete_bsdism(const char *name)
417{
418 static int warned;
419 static char warncomm[TASK_COMM_LEN];
4ec93edb
YH		 420 if (strcmp(warncomm, current->comm) && warned < 5) {
421 strcpy(warncomm, current->comm);
e005d193
JP		 422 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
423 warncomm, name);
1da177e4
LT		 424 warned++;
425 }
426}
427
080a270f
HFS		 428static bool sock_needs_netstamp(const struct sock *sk)
429{
430 switch (sk->sk_family) {
431 case AF_UNSPEC:
432 case AF_UNIX:
433 return false;
434 default:
435 return true;
436 }
437}
438
08e29af3 439static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
4ec93edb 440{
08e29af3
ED		 441 if (sk->sk_flags & flags) {
442 sk->sk_flags &= ~flags;
080a270f
HFS		 443 if (sock_needs_netstamp(sk) &&
444 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
20d49473 445 net_disable_timestamp();
1da177e4
LT		 446 }
447}
448
449
e6afc8ac 450int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
f0088a50 451{
3b885787
NH		 452 unsigned long flags;
453 struct sk_buff_head *list = &sk->sk_receive_queue;
f0088a50 454
0fd7bac6 455 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
766e9037 456 atomic_inc(&sk->sk_drops);
3847ce32 457 trace_sock_rcvqueue_full(sk, skb);
766e9037 458 return -ENOMEM;
f0088a50
DV		 459 }
460
c76562b6 461 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
766e9037
ED		 462 atomic_inc(&sk->sk_drops);
463 return -ENOBUFS;
3ab224be
HA		 464 }
465
f0088a50
DV		 466 skb->dev = NULL;
467 skb_set_owner_r(skb, sk);
49ad9599 468
7fee226a
ED		 469 /* we escape from rcu protected region, make sure we dont leak
470 * a norefcounted dst
471 */
472 skb_dst_force(skb);
473
3b885787 474 spin_lock_irqsave(&list->lock, flags);
3bc3b96f 475 sock_skb_set_dropcount(sk, skb);
3b885787
NH		 476 __skb_queue_tail(list, skb);
477 spin_unlock_irqrestore(&list->lock, flags);
f0088a50
DV		 478
479 if (!sock_flag(sk, SOCK_DEAD))
676d2369 480 sk->sk_data_ready(sk);
766e9037 481 return 0;
f0088a50 482}
e6afc8ac 483EXPORT_SYMBOL(__sock_queue_rcv_skb);
484
485int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
486{
487 int err;
488
489 err = sk_filter(sk, skb);
490 if (err)
491 return err;
492
493 return __sock_queue_rcv_skb(sk, skb);
494}
f0088a50
DV		 495EXPORT_SYMBOL(sock_queue_rcv_skb);
496
4f0c40d9 497int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
c3f24cfb 498 const int nested, unsigned int trim_cap, bool refcounted)
f0088a50
DV		 499{
500 int rc = NET_RX_SUCCESS;
501
4f0c40d9 502 if (sk_filter_trim_cap(sk, skb, trim_cap))
f0088a50
DV		 503 goto discard_and_relse;
504
505 skb->dev = NULL;
506
274f482d 507 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
c377411f
ED		 508 atomic_inc(&sk->sk_drops);
509 goto discard_and_relse;
510 }
58a5a7b9
ACM		 511 if (nested)
512 bh_lock_sock_nested(sk);
513 else
514 bh_lock_sock(sk);
a5b5bb9a
IM		 515 if (!sock_owned_by_user(sk)) {
516 /*
517 * trylock + unlock semantics:
518 */
519 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
520
c57943a1 521 rc = sk_backlog_rcv(sk, skb);
a5b5bb9a 522
5facae4f 523 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
8265792b 524 } else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) {
8eae939f
ZY		 525 bh_unlock_sock(sk);
526 atomic_inc(&sk->sk_drops);
527 goto discard_and_relse;
528 }
529
f0088a50
DV		 530 bh_unlock_sock(sk);
531out:
c3f24cfb
ED		 532 if (refcounted)
533 sock_put(sk);
f0088a50
DV		 534 return rc;
535discard_and_relse:
536 kfree_skb(skb);
537 goto out;
538}
4f0c40d9 539EXPORT_SYMBOL(__sk_receive_skb);
f0088a50
DV		 540
541struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
542{
b6c6712a 543 struct dst_entry *dst = __sk_dst_get(sk);
f0088a50
DV		 544
545 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
e022f0b4 546 sk_tx_queue_clear(sk);
9b8805a3 547 sk->sk_dst_pending_confirm = 0;
a9b3cd7f 548 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
f0088a50
DV		 549 dst_release(dst);
550 return NULL;
551 }
552
553 return dst;
554}
555EXPORT_SYMBOL(__sk_dst_check);
556
557struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
558{
559 struct dst_entry *dst = sk_dst_get(sk);
560
561 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
562 sk_dst_reset(sk);
563 dst_release(dst);
564 return NULL;
565 }
566
567 return dst;
568}
569EXPORT_SYMBOL(sk_dst_check);
570
7594888c 571static int sock_bindtoindex_locked(struct sock *sk, int ifindex)
4878809f
DM		 572{
573 int ret = -ENOPROTOOPT;
574#ifdef CONFIG_NETDEVICES
3b1e0a65 575 struct net *net = sock_net(sk);
4878809f
DM		 576
577 /* Sorry... */
578 ret = -EPERM;
c427bfec 579 if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW))
4878809f
DM		 580 goto out;
581
f5dd3d0c
DH		 582 ret = -EINVAL;
583 if (ifindex < 0)
584 goto out;
585
586 sk->sk_bound_dev_if = ifindex;
587 if (sk->sk_prot->rehash)
588 sk->sk_prot->rehash(sk);
589 sk_dst_reset(sk);
590
591 ret = 0;
592
593out:
594#endif
595
596 return ret;
597}
598
8ea204c2 599int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk)
7594888c
CH		 600{
601 int ret;
602
8ea204c2
FF		 603 if (lock_sk)
604 lock_sock(sk);
7594888c 605 ret = sock_bindtoindex_locked(sk, ifindex);
8ea204c2
FF		 606 if (lock_sk)
607 release_sock(sk);
7594888c
CH		 608
609 return ret;
610}
611EXPORT_SYMBOL(sock_bindtoindex);
612
5790642b 613static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen)
f5dd3d0c
DH		 614{
615 int ret = -ENOPROTOOPT;
616#ifdef CONFIG_NETDEVICES
617 struct net *net = sock_net(sk);
618 char devname[IFNAMSIZ];
619 int index;
620
4878809f
DM		 621 ret = -EINVAL;
622 if (optlen < 0)
623 goto out;
624
625 /* Bind this socket to a particular device like "eth0",
626 * as specified in the passed interface name. If the
627 * name is "" or the option length is zero the socket
628 * is not bound.
629 */
630 if (optlen > IFNAMSIZ - 1)
631 optlen = IFNAMSIZ - 1;
632 memset(devname, 0, sizeof(devname));
633
634 ret = -EFAULT;
5790642b 635 if (copy_from_sockptr(devname, optval, optlen))
4878809f
DM		 636 goto out;
637
000ba2e4
DM		 638 index = 0;
639 if (devname[0] != '\0') {
bf8e56bf 640 struct net_device *dev;
4878809f 641
bf8e56bf
ED		 642 rcu_read_lock();
643 dev = dev_get_by_name_rcu(net, devname);
644 if (dev)
645 index = dev->ifindex;
646 rcu_read_unlock();
4878809f
DM		 647 ret = -ENODEV;
648 if (!dev)
649 goto out;
4878809f
DM		 650 }
651
8ea204c2 652 return sock_bindtoindex(sk, index, true);
4878809f
DM		 653out:
654#endif
655
656 return ret;
657}
658
c91f6df2
BH		 659static int sock_getbindtodevice(struct sock *sk, char __user *optval,
660 int __user *optlen, int len)
661{
662 int ret = -ENOPROTOOPT;
663#ifdef CONFIG_NETDEVICES
664 struct net *net = sock_net(sk);
c91f6df2 665 char devname[IFNAMSIZ];
c91f6df2
BH		 666
667 if (sk->sk_bound_dev_if == 0) {
668 len = 0;
669 goto zero;
670 }
671
672 ret = -EINVAL;
673 if (len < IFNAMSIZ)
674 goto out;
675
5dbe7c17
NS		 676 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
677 if (ret)
c91f6df2 678 goto out;
c91f6df2
BH		 679
680 len = strlen(devname) + 1;
681
682 ret = -EFAULT;
683 if (copy_to_user(optval, devname, len))
684 goto out;
685
686zero:
687 ret = -EFAULT;
688 if (put_user(len, optlen))
689 goto out;
690
691 ret = 0;
692
693out:
694#endif
695
696 return ret;
697}
698
f60e5990 699bool sk_mc_loop(struct sock *sk)
700{
701 if (dev_recursion_level())
702 return false;
703 if (!sk)
704 return true;
705 switch (sk->sk_family) {
706 case AF_INET:
707 return inet_sk(sk)->mc_loop;
708#if IS_ENABLED(CONFIG_IPV6)
709 case AF_INET6:
710 return inet6_sk(sk)->mc_loop;
711#endif
712 }
0ad6f6e7 713 WARN_ON_ONCE(1);
f60e5990 714 return true;
715}
716EXPORT_SYMBOL(sk_mc_loop);
717
b58f0e8f
CH		 718void sock_set_reuseaddr(struct sock *sk)
719{
720 lock_sock(sk);
721 sk->sk_reuse = SK_CAN_REUSE;
722 release_sock(sk);
723}
724EXPORT_SYMBOL(sock_set_reuseaddr);
725
fe31a326
CH		 726void sock_set_reuseport(struct sock *sk)
727{
728 lock_sock(sk);
729 sk->sk_reuseport = true;
730 release_sock(sk);
731}
732EXPORT_SYMBOL(sock_set_reuseport);
733
c433594c
CH		 734void sock_no_linger(struct sock *sk)
735{
736 lock_sock(sk);
737 sk->sk_lingertime = 0;
738 sock_set_flag(sk, SOCK_LINGER);
739 release_sock(sk);
740}
741EXPORT_SYMBOL(sock_no_linger);
742
6e434967
CH		 743void sock_set_priority(struct sock *sk, u32 priority)
744{
745 lock_sock(sk);
746 sk->sk_priority = priority;
747 release_sock(sk);
748}
749EXPORT_SYMBOL(sock_set_priority);
750
76ee0785
CH		 751void sock_set_sndtimeo(struct sock *sk, s64 secs)
752{
753 lock_sock(sk);
754 if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1)
755 sk->sk_sndtimeo = secs * HZ;
756 else
757 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
758 release_sock(sk);
759}
760EXPORT_SYMBOL(sock_set_sndtimeo);
761
783da70e
CH		 762static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns)
763{
764 if (val) {
765 sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new);
766 sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns);
767 sock_set_flag(sk, SOCK_RCVTSTAMP);
768 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
769 } else {
770 sock_reset_flag(sk, SOCK_RCVTSTAMP);
771 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
772 sock_reset_flag(sk, SOCK_TSTAMP_NEW);
773 }
774}
775
776void sock_enable_timestamps(struct sock *sk)
777{
778 lock_sock(sk);
779 __sock_set_timestamps(sk, true, false, true);
780 release_sock(sk);
781}
782EXPORT_SYMBOL(sock_enable_timestamps);
783
ce3d9544
CH		 784void sock_set_keepalive(struct sock *sk)
785{
786 lock_sock(sk);
787 if (sk->sk_prot->keepalive)
788 sk->sk_prot->keepalive(sk, true);
789 sock_valbool_flag(sk, SOCK_KEEPOPEN, true);
790 release_sock(sk);
791}
792EXPORT_SYMBOL(sock_set_keepalive);
793
26cfabf9
CH		 794static void __sock_set_rcvbuf(struct sock *sk, int val)
795{
796 /* Ensure val * 2 fits into an int, to prevent max_t() from treating it
797 * as a negative value.
798 */
799 val = min_t(int, val, INT_MAX / 2);
800 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
801
802 /* We double it on the way in to account for "struct sk_buff" etc.
803 * overhead. Applications assume that the SO_RCVBUF setting they make
804 * will allow that much actual data to be received on that socket.
805 *
806 * Applications are unaware that "struct sk_buff" and other overheads
807 * allocate from the receive buffer during socket buffer allocation.
808 *
809 * And after considering the possible alternatives, returning the value
810 * we actually used in getsockopt is the most desirable behavior.
811 */
812 WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF));
813}
814
815void sock_set_rcvbuf(struct sock *sk, int val)
816{
817 lock_sock(sk);
818 __sock_set_rcvbuf(sk, val);
819 release_sock(sk);
820}
821EXPORT_SYMBOL(sock_set_rcvbuf);
822
1da177e4
LT		 823/*
824 * This is meant for all protocols to use and covers goings on
825 * at the socket level. Everything here is generic.
826 */
827
828int sock_setsockopt(struct socket *sock, int level, int optname,
c8c1bbb6 829 sockptr_t optval, unsigned int optlen)
1da177e4 830{
80b14dee 831 struct sock_txtime sk_txtime;
2a91525c 832 struct sock *sk = sock->sk;
1da177e4
LT		 833 int val;
834 int valbool;
835 struct linger ling;
836 int ret = 0;
4ec93edb 837
1da177e4
LT		 838 /*
839 * Options without arguments
840 */
841
4878809f 842 if (optname == SO_BINDTODEVICE)
c91f6df2 843 return sock_setbindtodevice(sk, optval, optlen);
4878809f 844
e71a4783
SH		 845 if (optlen < sizeof(int))
846 return -EINVAL;
4ec93edb 847
c8c1bbb6 848 if (copy_from_sockptr(&val, optval, sizeof(val)))
1da177e4 849 return -EFAULT;
4ec93edb 850
2a91525c 851 valbool = val ? 1 : 0;
1da177e4
LT		 852
853 lock_sock(sk);
854
2a91525c 855 switch (optname) {
e71a4783 856 case SO_DEBUG:
2a91525c 857 if (val && !capable(CAP_NET_ADMIN))
e71a4783 858 ret = -EACCES;
2a91525c 859 else
c0ef877b 860 sock_valbool_flag(sk, SOCK_DBG, valbool);
e71a4783
SH		 861 break;
862 case SO_REUSEADDR:
cdb8744d 863 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
e71a4783 864 break;
055dc21a
TH		 865 case SO_REUSEPORT:
866 sk->sk_reuseport = valbool;
867 break;
e71a4783 868 case SO_TYPE:
49c794e9 869 case SO_PROTOCOL:
0d6038ee 870 case SO_DOMAIN:
e71a4783
SH		 871 case SO_ERROR:
872 ret = -ENOPROTOOPT;
873 break;
874 case SO_DONTROUTE:
c0ef877b 875 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
0fbe82e6 876 sk_dst_reset(sk);
e71a4783
SH		 877 break;
878 case SO_BROADCAST:
879 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
880 break;
881 case SO_SNDBUF:
882 /* Don't error on this BSD doesn't and if you think
82981930
ED		 883 * about it this is right. Otherwise apps have to
884 * play 'guess the biggest size' games. RCVBUF/SNDBUF
885 * are treated in BSD as hints
886 */
887 val = min_t(u32, val, sysctl_wmem_max);
b0573dea 888set_sndbuf:
4057765f
GN		 889 /* Ensure val * 2 fits into an int, to prevent max_t()
890 * from treating it as a negative value.
891 */
892 val = min_t(int, val, INT_MAX / 2);
e71a4783 893 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
e292f05e
ED		 894 WRITE_ONCE(sk->sk_sndbuf,
895 max_t(int, val * 2, SOCK_MIN_SNDBUF));
82981930 896 /* Wake up sending tasks if we upped the value. */
e71a4783
SH		 897 sk->sk_write_space(sk);
898 break;
1da177e4 899
e71a4783
SH		 900 case SO_SNDBUFFORCE:
901 if (!capable(CAP_NET_ADMIN)) {
902 ret = -EPERM;
903 break;
904 }
4057765f
GN		 905
906 /* No negative values (to prevent underflow, as val will be
907 * multiplied by 2).
908 */
909 if (val < 0)
910 val = 0;
e71a4783 911 goto set_sndbuf;
b0573dea 912
e71a4783
SH		 913 case SO_RCVBUF:
914 /* Don't error on this BSD doesn't and if you think
82981930
ED		 915 * about it this is right. Otherwise apps have to
916 * play 'guess the biggest size' games. RCVBUF/SNDBUF
917 * are treated in BSD as hints
918 */
26cfabf9 919 __sock_set_rcvbuf(sk, min_t(u32, val, sysctl_rmem_max));
e71a4783
SH		 920 break;
921
922 case SO_RCVBUFFORCE:
923 if (!capable(CAP_NET_ADMIN)) {
924 ret = -EPERM;
1da177e4 925 break;
e71a4783 926 }
4057765f
GN		 927
928 /* No negative values (to prevent underflow, as val will be
929 * multiplied by 2).
930 */
26cfabf9
CH		 931 __sock_set_rcvbuf(sk, max(val, 0));
932 break;
1da177e4 933
e71a4783 934 case SO_KEEPALIVE:
4b9d07a4
UB		 935 if (sk->sk_prot->keepalive)
936 sk->sk_prot->keepalive(sk, valbool);
e71a4783
SH		 937 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
938 break;
939
940 case SO_OOBINLINE:
941 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
942 break;
943
944 case SO_NO_CHECK:
28448b80 945 sk->sk_no_check_tx = valbool;
e71a4783
SH		 946 break;
947
948 case SO_PRIORITY:
5e1fccc0
EB		 949 if ((val >= 0 && val <= 6) ||
950 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
e71a4783
SH		 951 sk->sk_priority = val;
952 else
953 ret = -EPERM;
954 break;
955
956 case SO_LINGER:
957 if (optlen < sizeof(ling)) {
958 ret = -EINVAL; /* 1003.1g */
1da177e4 959 break;
e71a4783 960 }
c8c1bbb6 961 if (copy_from_sockptr(&ling, optval, sizeof(ling))) {
e71a4783 962 ret = -EFAULT;
1da177e4 963 break;
e71a4783
SH		 964 }
965 if (!ling.l_onoff)
966 sock_reset_flag(sk, SOCK_LINGER);
967 else {
1da177e4 968#if (BITS_PER_LONG == 32)
e71a4783
SH		 969 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
970 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
1da177e4 971 else
e71a4783
SH		 972#endif
973 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
974 sock_set_flag(sk, SOCK_LINGER);
975 }
976 break;
977
978 case SO_BSDCOMPAT:
979 sock_warn_obsolete_bsdism("setsockopt");
980 break;
981
982 case SO_PASSCRED:
983 if (valbool)
984 set_bit(SOCK_PASSCRED, &sock->flags);
985 else
986 clear_bit(SOCK_PASSCRED, &sock->flags);
987 break;
988
7f1bc6e9 989 case SO_TIMESTAMP_OLD:
783da70e
CH		 990 __sock_set_timestamps(sk, valbool, false, false);
991 break;
887feae3 992 case SO_TIMESTAMP_NEW:
783da70e
CH		 993 __sock_set_timestamps(sk, valbool, true, false);
994 break;
7f1bc6e9 995 case SO_TIMESTAMPNS_OLD:
783da70e
CH		 996 __sock_set_timestamps(sk, valbool, false, true);
997 break;
887feae3 998 case SO_TIMESTAMPNS_NEW:
783da70e 999 __sock_set_timestamps(sk, valbool, true, true);
e71a4783 1000 break;
9718475e
DD		 1001 case SO_TIMESTAMPING_NEW:
1002 sock_set_flag(sk, SOCK_TSTAMP_NEW);
ff7653f9 1003 /* fall through */
7f1bc6e9 1004 case SO_TIMESTAMPING_OLD:
20d49473 1005 if (val & ~SOF_TIMESTAMPING_MASK) {
f249fb78 1006 ret = -EINVAL;
20d49473
PO		 1007 break;
1008 }
b245be1f 1009
09c2d251 1010 if (val & SOF_TIMESTAMPING_OPT_ID &&
4ed2d765 1011 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
ac5cc977
WC		 1012 if (sk->sk_protocol == IPPROTO_TCP &&
1013 sk->sk_type == SOCK_STREAM) {
6db8b963
SHY		 1014 if ((1 << sk->sk_state) &
1015 (TCPF_CLOSE | TCPF_LISTEN)) {
4ed2d765
WB		 1016 ret = -EINVAL;
1017 break;
1018 }
1019 sk->sk_tskey = tcp_sk(sk)->snd_una;
1020 } else {
1021 sk->sk_tskey = 0;
1022 }
1023 }
1c885808
FY		 1024
1025 if (val & SOF_TIMESTAMPING_OPT_STATS &&
1026 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
1027 ret = -EINVAL;
1028 break;
1029 }
1030
b9f40e21 1031 sk->sk_tsflags = val;
20d49473
PO		 1032 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
1033 sock_enable_timestamp(sk,
1034 SOCK_TIMESTAMPING_RX_SOFTWARE);
9718475e
DD		 1035 else {
1036 if (optname == SO_TIMESTAMPING_NEW)
1037 sock_reset_flag(sk, SOCK_TSTAMP_NEW);
1038
20d49473 1039 sock_disable_timestamp(sk,
08e29af3 1040 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
9718475e 1041 }
20d49473
PO		 1042 break;
1043
e71a4783
SH		 1044 case SO_RCVLOWAT:
1045 if (val < 0)
1046 val = INT_MAX;
d1361840
ED		 1047 if (sock->ops->set_rcvlowat)
1048 ret = sock->ops->set_rcvlowat(sk, val);
1049 else
eac66402 1050 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
e71a4783
SH		 1051 break;
1052
45bdc661 1053 case SO_RCVTIMEO_OLD:
a9beb86a 1054 case SO_RCVTIMEO_NEW:
c8c1bbb6 1055 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval,
c34645ac 1056 optlen, optname == SO_RCVTIMEO_OLD);
e71a4783
SH		 1057 break;
1058
45bdc661 1059 case SO_SNDTIMEO_OLD:
a9beb86a 1060 case SO_SNDTIMEO_NEW:
c8c1bbb6 1061 ret = sock_set_timeout(&sk->sk_sndtimeo, optval,
c34645ac 1062 optlen, optname == SO_SNDTIMEO_OLD);
e71a4783 1063 break;
1da177e4 1064
4d295e54
CH		 1065 case SO_ATTACH_FILTER: {
1066 struct sock_fprog fprog;
e71a4783 1067
c8c1bbb6 1068 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
4d295e54 1069 if (!ret)
e71a4783 1070 ret = sk_attach_filter(&fprog, sk);
e71a4783 1071 break;
4d295e54 1072 }
89aa0758
AS		 1073 case SO_ATTACH_BPF:
1074 ret = -EINVAL;
1075 if (optlen == sizeof(u32)) {
1076 u32 ufd;
1077
1078 ret = -EFAULT;
c8c1bbb6 1079 if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
89aa0758
AS		 1080 break;
1081
1082 ret = sk_attach_bpf(ufd, sk);
1083 }
1084 break;
1085
4d295e54
CH		 1086 case SO_ATTACH_REUSEPORT_CBPF: {
1087 struct sock_fprog fprog;
538950a1 1088
c8c1bbb6 1089 ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
4d295e54 1090 if (!ret)
538950a1 1091 ret = sk_reuseport_attach_filter(&fprog, sk);
538950a1 1092 break;
4d295e54 1093 }
538950a1
CG		 1094 case SO_ATTACH_REUSEPORT_EBPF:
1095 ret = -EINVAL;
1096 if (optlen == sizeof(u32)) {
1097 u32 ufd;
1098
1099 ret = -EFAULT;
c8c1bbb6 1100 if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
538950a1
CG		 1101 break;
1102
1103 ret = sk_reuseport_attach_bpf(ufd, sk);
1104 }
1105 break;
1106
99f3a064
MKL		 1107 case SO_DETACH_REUSEPORT_BPF:
1108 ret = reuseport_detach_prog(sk);
1109 break;
1110
e71a4783 1111 case SO_DETACH_FILTER:
55b33325 1112 ret = sk_detach_filter(sk);
e71a4783 1113 break;
1da177e4 1114
d59577b6
VB		 1115 case SO_LOCK_FILTER:
1116 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
1117 ret = -EPERM;
1118 else
1119 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
1120 break;
1121
e71a4783
SH		 1122 case SO_PASSSEC:
1123 if (valbool)
1124 set_bit(SOCK_PASSSEC, &sock->flags);
1125 else
1126 clear_bit(SOCK_PASSSEC, &sock->flags);
1127 break;
4a19ec58 1128 case SO_MARK:
50254256 1129 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
4a19ec58 1130 ret = -EPERM;
50254256 1131 } else if (val != sk->sk_mark) {
4a19ec58 1132 sk->sk_mark = val;
50254256
DB		 1133 sk_dst_reset(sk);
1134 }
4a19ec58 1135 break;
877ce7c1 1136
3b885787 1137 case SO_RXQ_OVFL:
8083f0fc 1138 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
3b885787 1139 break;
6e3e939f
JB		 1140
1141 case SO_WIFI_STATUS:
1142 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1143 break;
1144
ef64a54f
PE		 1145 case SO_PEEK_OFF:
1146 if (sock->ops->set_peek_off)
12663bfc 1147 ret = sock->ops->set_peek_off(sk, val);
ef64a54f
PE		 1148 else
1149 ret = -EOPNOTSUPP;
1150 break;
3bdc0eba
BG		 1151
1152 case SO_NOFCS:
1153 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1154 break;
1155
7d4c04fc
KJ		 1156 case SO_SELECT_ERR_QUEUE:
1157 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1158 break;
1159
e0d1095a 1160#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1161 case SO_BUSY_POLL:
dafcc438
ET		 1162 /* allow unprivileged users to decrease the value */
1163 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
1164 ret = -EPERM;
1165 else {
1166 if (val < 0)
1167 ret = -EINVAL;
1168 else
1169 sk->sk_ll_usec = val;
1170 }
1171 break;
1172#endif
62748f32
ED		 1173
1174 case SO_MAX_PACING_RATE:
6bdef102
ED		 1175 {
1176 unsigned long ulval = (val == ~0U) ? ~0UL : val;
1177
1178 if (sizeof(ulval) != sizeof(val) &&
1179 optlen >= sizeof(ulval) &&
c8c1bbb6 1180 copy_from_sockptr(&ulval, optval, sizeof(ulval))) {
6bdef102
ED		 1181 ret = -EFAULT;
1182 break;
1183 }
1184 if (ulval != ~0UL)
218af599
ED		 1185 cmpxchg(&sk->sk_pacing_status,
1186 SK_PACING_NONE,
1187 SK_PACING_NEEDED);
6bdef102
ED		 1188 sk->sk_max_pacing_rate = ulval;
1189 sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval);
62748f32 1190 break;
6bdef102 1191 }
70da268b 1192 case SO_INCOMING_CPU:
7170a977 1193 WRITE_ONCE(sk->sk_incoming_cpu, val);
70da268b
ED		 1194 break;
1195
a87cb3e4
TH		 1196 case SO_CNX_ADVICE:
1197 if (val == 1)
1198 dst_negative_advice(sk);
1199 break;
76851d12
WB		 1200
1201 case SO_ZEROCOPY:
28190752 1202 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
b5947e5d
WB		 1203 if (!((sk->sk_type == SOCK_STREAM &&
1204 sk->sk_protocol == IPPROTO_TCP) ||
1205 (sk->sk_type == SOCK_DGRAM &&
1206 sk->sk_protocol == IPPROTO_UDP)))
28190752 1207 ret = -ENOTSUPP;
28190752 1208 } else if (sk->sk_family != PF_RDS) {
76851d12 1209 ret = -ENOTSUPP;
28190752
SV		 1210 }
1211 if (!ret) {
1212 if (val < 0 || val > 1)
1213 ret = -EINVAL;
1214 else
1215 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
28190752 1216 }
334e6413
JSP		 1217 break;
1218
80b14dee 1219 case SO_TXTIME:
790709f2 1220 if (optlen != sizeof(struct sock_txtime)) {
80b14dee 1221 ret = -EINVAL;
790709f2 1222 break;
c8c1bbb6 1223 } else if (copy_from_sockptr(&sk_txtime, optval,
80b14dee
RC		 1224 sizeof(struct sock_txtime))) {
1225 ret = -EFAULT;
790709f2 1226 break;
80b14dee
RC		 1227 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
1228 ret = -EINVAL;
790709f2
ED		 1229 break;
1230 }
1231 /* CLOCK_MONOTONIC is only used by sch_fq, and this packet
1232 * scheduler has enough safe guards.
1233 */
1234 if (sk_txtime.clockid != CLOCK_MONOTONIC &&
1235 !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1236 ret = -EPERM;
1237 break;
80b14dee 1238 }
790709f2
ED		 1239 sock_valbool_flag(sk, SOCK_TXTIME, true);
1240 sk->sk_clockid = sk_txtime.clockid;
1241 sk->sk_txtime_deadline_mode =
1242 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
1243 sk->sk_txtime_report_errors =
1244 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
80b14dee
RC		 1245 break;
1246
f5dd3d0c 1247 case SO_BINDTOIFINDEX:
7594888c 1248 ret = sock_bindtoindex_locked(sk, val);
f5dd3d0c
DH		 1249 break;
1250
e71a4783
SH		 1251 default:
1252 ret = -ENOPROTOOPT;
1253 break;
4ec93edb 1254 }
1da177e4
LT		 1255 release_sock(sk);
1256 return ret;
1257}
2a91525c 1258EXPORT_SYMBOL(sock_setsockopt);
1da177e4
LT		 1259
1260
8f09898b 1261static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1262 struct ucred *ucred)
3f551f94
EB		 1263{
1264 ucred->pid = pid_vnr(pid);
1265 ucred->uid = ucred->gid = -1;
1266 if (cred) {
1267 struct user_namespace *current_ns = current_user_ns();
1268
b2e4f544
EB		 1269 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1270 ucred->gid = from_kgid_munged(current_ns, cred->egid);
3f551f94
EB		 1271 }
1272}
1273
28b5ba2a
DH		 1274static int groups_to_user(gid_t __user *dst, const struct group_info *src)
1275{
1276 struct user_namespace *user_ns = current_user_ns();
1277 int i;
1278
1279 for (i = 0; i < src->ngroups; i++)
1280 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
1281 return -EFAULT;
1282
1283 return 0;
1284}
1285
1da177e4
LT		 1286int sock_getsockopt(struct socket *sock, int level, int optname,
1287 char __user *optval, int __user *optlen)
1288{
1289 struct sock *sk = sock->sk;
4ec93edb 1290
e71a4783 1291 union {
4ec93edb 1292 int val;
5daab9db 1293 u64 val64;
677f136c 1294 unsigned long ulval;
4ec93edb 1295 struct linger ling;
fe0c72f3
AB		 1296 struct old_timeval32 tm32;
1297 struct __kernel_old_timeval tm;
a9beb86a 1298 struct __kernel_sock_timeval stm;
80b14dee 1299 struct sock_txtime txtime;
1da177e4 1300 } v;
4ec93edb 1301
4d0392be 1302 int lv = sizeof(int);
1da177e4 1303 int len;
4ec93edb 1304
e71a4783 1305 if (get_user(len, optlen))
4ec93edb 1306 return -EFAULT;
e71a4783 1307 if (len < 0)
1da177e4 1308 return -EINVAL;
4ec93edb 1309
50fee1de 1310 memset(&v, 0, sizeof(v));
df0bca04 1311
2a91525c 1312 switch (optname) {
e71a4783
SH		 1313 case SO_DEBUG:
1314 v.val = sock_flag(sk, SOCK_DBG);
1315 break;
1316
1317 case SO_DONTROUTE:
1318 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1319 break;
1320
1321 case SO_BROADCAST:
1b23a5df 1322 v.val = sock_flag(sk, SOCK_BROADCAST);
e71a4783
SH		 1323 break;
1324
1325 case SO_SNDBUF:
1326 v.val = sk->sk_sndbuf;
1327 break;
1328
1329 case SO_RCVBUF:
1330 v.val = sk->sk_rcvbuf;
1331 break;
1332
1333 case SO_REUSEADDR:
1334 v.val = sk->sk_reuse;
1335 break;
1336
055dc21a
TH		 1337 case SO_REUSEPORT:
1338 v.val = sk->sk_reuseport;
1339 break;
1340
e71a4783 1341 case SO_KEEPALIVE:
1b23a5df 1342 v.val = sock_flag(sk, SOCK_KEEPOPEN);
e71a4783
SH		 1343 break;
1344
1345 case SO_TYPE:
1346 v.val = sk->sk_type;
1347 break;
1348
49c794e9
JE		 1349 case SO_PROTOCOL:
1350 v.val = sk->sk_protocol;
1351 break;
1352
0d6038ee
JE		 1353 case SO_DOMAIN:
1354 v.val = sk->sk_family;
1355 break;
1356
e71a4783
SH		 1357 case SO_ERROR:
1358 v.val = -sock_error(sk);
2a91525c 1359 if (v.val == 0)
e71a4783
SH		 1360 v.val = xchg(&sk->sk_err_soft, 0);
1361 break;
1362
1363 case SO_OOBINLINE:
1b23a5df 1364 v.val = sock_flag(sk, SOCK_URGINLINE);
e71a4783
SH		 1365 break;
1366
1367 case SO_NO_CHECK:
28448b80 1368 v.val = sk->sk_no_check_tx;
e71a4783
SH		 1369 break;
1370
1371 case SO_PRIORITY:
1372 v.val = sk->sk_priority;
1373 break;
1374
1375 case SO_LINGER:
1376 lv = sizeof(v.ling);
1b23a5df 1377 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
e71a4783
SH		 1378 v.ling.l_linger = sk->sk_lingertime / HZ;
1379 break;
1380
1381 case SO_BSDCOMPAT:
1382 sock_warn_obsolete_bsdism("getsockopt");
1383 break;
1384
7f1bc6e9 1385 case SO_TIMESTAMP_OLD:
92f37fd2 1386 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
887feae3 1387 !sock_flag(sk, SOCK_TSTAMP_NEW) &&
92f37fd2
ED		 1388 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1389 break;
1390
7f1bc6e9 1391 case SO_TIMESTAMPNS_OLD:
887feae3
DD		 1392 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW);
1393 break;
1394
1395 case SO_TIMESTAMP_NEW:
1396 v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW);
1397 break;
1398
1399 case SO_TIMESTAMPNS_NEW:
1400 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW);
e71a4783
SH		 1401 break;
1402
7f1bc6e9 1403 case SO_TIMESTAMPING_OLD:
b9f40e21 1404 v.val = sk->sk_tsflags;
20d49473
PO		 1405 break;
1406
a9beb86a
DD		 1407 case SO_RCVTIMEO_OLD:
1408 case SO_RCVTIMEO_NEW:
1409 lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname);
e71a4783
SH		 1410 break;
1411
a9beb86a
DD		 1412 case SO_SNDTIMEO_OLD:
1413 case SO_SNDTIMEO_NEW:
1414 lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname);
e71a4783 1415 break;
1da177e4 1416
e71a4783
SH		 1417 case SO_RCVLOWAT:
1418 v.val = sk->sk_rcvlowat;
1419 break;
1da177e4 1420
e71a4783 1421 case SO_SNDLOWAT:
2a91525c 1422 v.val = 1;
e71a4783 1423 break;
1da177e4 1424
e71a4783 1425 case SO_PASSCRED:
82981930 1426 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
e71a4783 1427 break;
1da177e4 1428
e71a4783 1429 case SO_PEERCRED:
109f6e39
EB		 1430 {
1431 struct ucred peercred;
1432 if (len > sizeof(peercred))
1433 len = sizeof(peercred);
1434 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1435 if (copy_to_user(optval, &peercred, len))
e71a4783
SH		 1436 return -EFAULT;
1437 goto lenout;
109f6e39 1438 }
1da177e4 1439
28b5ba2a
DH		 1440 case SO_PEERGROUPS:
1441 {
1442 int ret, n;
1443
1444 if (!sk->sk_peer_cred)
1445 return -ENODATA;
1446
1447 n = sk->sk_peer_cred->group_info->ngroups;
1448 if (len < n * sizeof(gid_t)) {
1449 len = n * sizeof(gid_t);
1450 return put_user(len, optlen) ? -EFAULT : -ERANGE;
1451 }
1452 len = n * sizeof(gid_t);
1453
1454 ret = groups_to_user((gid_t __user *)optval,
1455 sk->sk_peer_cred->group_info);
1456 if (ret)
1457 return ret;
1458 goto lenout;
1459 }
1460
e71a4783
SH		 1461 case SO_PEERNAME:
1462 {
1463 char address[128];
1464
9b2c45d4
DV		 1465 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2);
1466 if (lv < 0)
e71a4783
SH		 1467 return -ENOTCONN;
1468 if (lv < len)
1469 return -EINVAL;
1470 if (copy_to_user(optval, address, len))
1471 return -EFAULT;
1472 goto lenout;
1473 }
1da177e4 1474
e71a4783
SH		 1475 /* Dubious BSD thing... Probably nobody even uses it, but
1476 * the UNIX standard wants it for whatever reason... -DaveM
1477 */
1478 case SO_ACCEPTCONN:
1479 v.val = sk->sk_state == TCP_LISTEN;
1480 break;
1da177e4 1481
e71a4783 1482 case SO_PASSSEC:
82981930 1483 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
e71a4783 1484 break;
877ce7c1 1485
e71a4783
SH		 1486 case SO_PEERSEC:
1487 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1da177e4 1488
4a19ec58
LAT		 1489 case SO_MARK:
1490 v.val = sk->sk_mark;
1491 break;
1492
3b885787 1493 case SO_RXQ_OVFL:
1b23a5df 1494 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
3b885787
NH		 1495 break;
1496
6e3e939f 1497 case SO_WIFI_STATUS:
1b23a5df 1498 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
6e3e939f
JB		 1499 break;
1500
ef64a54f
PE		 1501 case SO_PEEK_OFF:
1502 if (!sock->ops->set_peek_off)
1503 return -EOPNOTSUPP;
1504
1505 v.val = sk->sk_peek_off;
1506 break;
bc2f7996 1507 case SO_NOFCS:
1b23a5df 1508 v.val = sock_flag(sk, SOCK_NOFCS);
bc2f7996 1509 break;
c91f6df2 1510
f7b86bfe 1511 case SO_BINDTODEVICE:
c91f6df2
BH		 1512 return sock_getbindtodevice(sk, optval, optlen, len);
1513
a8fc9277
PE		 1514 case SO_GET_FILTER:
1515 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1516 if (len < 0)
1517 return len;
1518
1519 goto lenout;
c91f6df2 1520
d59577b6
VB		 1521 case SO_LOCK_FILTER:
1522 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1523 break;
1524
ea02f941
MS		 1525 case SO_BPF_EXTENSIONS:
1526 v.val = bpf_tell_extensions();
1527 break;
1528
7d4c04fc
KJ		 1529 case SO_SELECT_ERR_QUEUE:
1530 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1531 break;
1532
e0d1095a 1533#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1534 case SO_BUSY_POLL:
dafcc438
ET		 1535 v.val = sk->sk_ll_usec;
1536 break;
1537#endif
1538
62748f32 1539 case SO_MAX_PACING_RATE:
677f136c
ED		 1540 if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) {
1541 lv = sizeof(v.ulval);
1542 v.ulval = sk->sk_max_pacing_rate;
1543 } else {
1544 /* 32bit version */
1545 v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);
1546 }
62748f32
ED		 1547 break;
1548
2c8c56e1 1549 case SO_INCOMING_CPU:
7170a977 1550 v.val = READ_ONCE(sk->sk_incoming_cpu);
2c8c56e1
ED		 1551 break;
1552
a2d133b1
JH		 1553 case SO_MEMINFO:
1554 {
1555 u32 meminfo[SK_MEMINFO_VARS];
1556
a2d133b1
JH		 1557 sk_get_meminfo(sk, meminfo);
1558
1559 len = min_t(unsigned int, len, sizeof(meminfo));
1560 if (copy_to_user(optval, &meminfo, len))
1561 return -EFAULT;
1562
1563 goto lenout;
1564 }
6d433902
SS		 1565
1566#ifdef CONFIG_NET_RX_BUSY_POLL
1567 case SO_INCOMING_NAPI_ID:
1568 v.val = READ_ONCE(sk->sk_napi_id);
1569
1570 /* aggregate non-NAPI IDs down to 0 */
1571 if (v.val < MIN_NAPI_ID)
1572 v.val = 0;
1573
1574 break;
1575#endif
1576
5daab9db
CF		 1577 case SO_COOKIE:
1578 lv = sizeof(u64);
1579 if (len < lv)
1580 return -EINVAL;
1581 v.val64 = sock_gen_cookie(sk);
1582 break;
1583
76851d12
WB		 1584 case SO_ZEROCOPY:
1585 v.val = sock_flag(sk, SOCK_ZEROCOPY);
1586 break;
1587
80b14dee
RC		 1588 case SO_TXTIME:
1589 lv = sizeof(v.txtime);
1590 v.txtime.clockid = sk->sk_clockid;
1591 v.txtime.flags |= sk->sk_txtime_deadline_mode ?
1592 SOF_TXTIME_DEADLINE_MODE : 0;
4b15c707
JSP		 1593 v.txtime.flags |= sk->sk_txtime_report_errors ?
1594 SOF_TXTIME_REPORT_ERRORS : 0;
80b14dee
RC		 1595 break;
1596
f5dd3d0c
DH		 1597 case SO_BINDTOIFINDEX:
1598 v.val = sk->sk_bound_dev_if;
1599 break;
1600
e71a4783 1601 default:
443b5991
YH		 1602 /* We implement the SO_SNDLOWAT etc to not be settable
1603 * (1003.1g 7).
1604 */
e71a4783 1605 return -ENOPROTOOPT;
1da177e4 1606 }
e71a4783 1607
1da177e4
LT		 1608 if (len > lv)
1609 len = lv;
1610 if (copy_to_user(optval, &v, len))
1611 return -EFAULT;
1612lenout:
4ec93edb
YH		 1613 if (put_user(len, optlen))
1614 return -EFAULT;
1615 return 0;
1da177e4
LT		 1616}
1617
a5b5bb9a
IM		 1618/*
1619 * Initialize an sk_lock.
1620 *
1621 * (We also register the sk_lock with the lock validator.)
1622 */
b6f99a21 1623static inline void sock_lock_init(struct sock *sk)
a5b5bb9a 1624{
cdfbabfb
DH		 1625 if (sk->sk_kern_sock)
1626 sock_lock_init_class_and_name(
1627 sk,
1628 af_family_kern_slock_key_strings[sk->sk_family],
1629 af_family_kern_slock_keys + sk->sk_family,
1630 af_family_kern_key_strings[sk->sk_family],
1631 af_family_kern_keys + sk->sk_family);
1632 else
1633 sock_lock_init_class_and_name(
1634 sk,
ed07536e
PZ		 1635 af_family_slock_key_strings[sk->sk_family],
1636 af_family_slock_keys + sk->sk_family,
1637 af_family_key_strings[sk->sk_family],
1638 af_family_keys + sk->sk_family);
a5b5bb9a
IM		 1639}
1640
4dc6dc71
ED		 1641/*
1642 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1643 * even temporarly, because of RCU lookups. sk_node should also be left as is.
68835aba 1644 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
4dc6dc71 1645 */
f1a6c4da
PE		 1646static void sock_copy(struct sock *nsk, const struct sock *osk)
1647{
b8e202d1 1648 const struct proto *prot = READ_ONCE(osk->sk_prot);
f1a6c4da
PE		 1649#ifdef CONFIG_SECURITY_NETWORK
1650 void *sptr = nsk->sk_security;
1651#endif
68835aba
ED		 1652 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1653
1654 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
b8e202d1 1655 prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
68835aba 1656
f1a6c4da
PE		 1657#ifdef CONFIG_SECURITY_NETWORK
1658 nsk->sk_security = sptr;
1659 security_sk_clone(osk, nsk);
1660#endif
1661}
1662
2e4afe7b
PE		 1663static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1664 int family)
c308c1b2
PE		 1665{
1666 struct sock *sk;
1667 struct kmem_cache *slab;
1668
1669 slab = prot->slab;
e912b114
ED		 1670 if (slab != NULL) {
1671 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1672 if (!sk)
1673 return sk;
6471384a 1674 if (want_init_on_alloc(priority))
ba2489b0 1675 sk_prot_clear_nulls(sk, prot->obj_size);
fcbdf09d 1676 } else
c308c1b2
PE		 1677 sk = kmalloc(prot->obj_size, priority);
1678
2e4afe7b
PE		 1679 if (sk != NULL) {
1680 if (security_sk_alloc(sk, family, priority))
1681 goto out_free;
1682
1683 if (!try_module_get(prot->owner))
1684 goto out_free_sec;
e022f0b4 1685 sk_tx_queue_clear(sk);
2e4afe7b
PE		 1686 }
1687
c308c1b2 1688 return sk;
2e4afe7b
PE		 1689
1690out_free_sec:
1691 security_sk_free(sk);
1692out_free:
1693 if (slab != NULL)
1694 kmem_cache_free(slab, sk);
1695 else
1696 kfree(sk);
1697 return NULL;
c308c1b2
PE		 1698}
1699
1700static void sk_prot_free(struct proto *prot, struct sock *sk)
1701{
1702 struct kmem_cache *slab;
2e4afe7b 1703 struct module *owner;
c308c1b2 1704
2e4afe7b 1705 owner = prot->owner;
c308c1b2 1706 slab = prot->slab;
2e4afe7b 1707
bd1060a1 1708 cgroup_sk_free(&sk->sk_cgrp_data);
2d758073 1709 mem_cgroup_sk_free(sk);
2e4afe7b 1710 security_sk_free(sk);
c308c1b2
PE		 1711 if (slab != NULL)
1712 kmem_cache_free(slab, sk);
1713 else
1714 kfree(sk);
2e4afe7b 1715 module_put(owner);
c308c1b2
PE		 1716}
1717
1da177e4
LT		 1718/**
1719 * sk_alloc - All socket objects are allocated here
c4ea43c5 1720 * @net: the applicable net namespace
4dc3b16b
PP		 1721 * @family: protocol family
1722 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1723 * @prot: struct proto associated with this new sock instance
11aa9c28 1724 * @kern: is this to be a kernel socket?
1da177e4 1725 */
1b8d7ae4 1726struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
11aa9c28 1727 struct proto *prot, int kern)
1da177e4 1728{
c308c1b2 1729 struct sock *sk;
1da177e4 1730
154adbc8 1731 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1da177e4 1732 if (sk) {
154adbc8
PE		 1733 sk->sk_family = family;
1734 /*
1735 * See comment in struct sock definition to understand
1736 * why we need sk_prot_creator -acme
1737 */
1738 sk->sk_prot = sk->sk_prot_creator = prot;
cdfbabfb 1739 sk->sk_kern_sock = kern;
154adbc8 1740 sock_lock_init(sk);
26abe143 1741 sk->sk_net_refcnt = kern ? 0 : 1;
648845ab 1742 if (likely(sk->sk_net_refcnt)) {
26abe143 1743 get_net(net);
648845ab
TZ		 1744 sock_inuse_add(net, 1);
1745 }
1746
26abe143 1747 sock_net_set(sk, net);
14afee4b 1748 refcount_set(&sk->sk_wmem_alloc, 1);
f8451725 1749
2d758073 1750 mem_cgroup_sk_alloc(sk);
d979a39d 1751 cgroup_sk_alloc(&sk->sk_cgrp_data);
2a56a1fe
TH		 1752 sock_update_classid(&sk->sk_cgrp_data);
1753 sock_update_netprioidx(&sk->sk_cgrp_data);
41b14fb8 1754 sk_tx_queue_clear(sk);
1da177e4 1755 }
a79af59e 1756
2e4afe7b 1757 return sk;
1da177e4 1758}
2a91525c 1759EXPORT_SYMBOL(sk_alloc);
1da177e4 1760
a4298e45
ED		 1761/* Sockets having SOCK_RCU_FREE will call this function after one RCU
1762 * grace period. This is the case for UDP sockets and TCP listeners.
1763 */
1764static void __sk_destruct(struct rcu_head *head)
1da177e4 1765{
a4298e45 1766 struct sock *sk = container_of(head, struct sock, sk_rcu);
1da177e4 1767 struct sk_filter *filter;
1da177e4
LT		 1768
1769 if (sk->sk_destruct)
1770 sk->sk_destruct(sk);
1771
a898def2 1772 filter = rcu_dereference_check(sk->sk_filter,
14afee4b 1773 refcount_read(&sk->sk_wmem_alloc) == 0);
1da177e4 1774 if (filter) {
309dd5fc 1775 sk_filter_uncharge(sk, filter);
a9b3cd7f 1776 RCU_INIT_POINTER(sk->sk_filter, NULL);
1da177e4
LT		 1777 }
1778
08e29af3 1779 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1da177e4 1780
6ac99e8f
MKL		 1781#ifdef CONFIG_BPF_SYSCALL
1782 bpf_sk_storage_free(sk);
1783#endif
1784
1da177e4 1785 if (atomic_read(&sk->sk_omem_alloc))
e005d193
JP		 1786 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1787 __func__, atomic_read(&sk->sk_omem_alloc));
1da177e4 1788
22a0e18e
ED		 1789 if (sk->sk_frag.page) {
1790 put_page(sk->sk_frag.page);
1791 sk->sk_frag.page = NULL;
1792 }
1793
109f6e39
EB		 1794 if (sk->sk_peer_cred)
1795 put_cred(sk->sk_peer_cred);
1796 put_pid(sk->sk_peer_pid);
26abe143
EB		 1797 if (likely(sk->sk_net_refcnt))
1798 put_net(sock_net(sk));
c308c1b2 1799 sk_prot_free(sk->sk_prot_creator, sk);
1da177e4 1800}
2b85a34e 1801
a4298e45
ED		 1802void sk_destruct(struct sock *sk)
1803{
8c7138b3
MKL		 1804 bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
1805
1806 if (rcu_access_pointer(sk->sk_reuseport_cb)) {
1807 reuseport_detach_sock(sk);
1808 use_call_rcu = true;
1809 }
1810
1811 if (use_call_rcu)
a4298e45
ED		 1812 call_rcu(&sk->sk_rcu, __sk_destruct);
1813 else
1814 __sk_destruct(&sk->sk_rcu);
1815}
1816
eb4cb008
CG		 1817static void __sk_free(struct sock *sk)
1818{
648845ab
TZ		 1819 if (likely(sk->sk_net_refcnt))
1820 sock_inuse_add(sock_net(sk), -1);
1821
9709020c 1822 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
eb4cb008
CG		 1823 sock_diag_broadcast_destroy(sk);
1824 else
1825 sk_destruct(sk);
1826}
1827
2b85a34e
ED		 1828void sk_free(struct sock *sk)
1829{
1830 /*
25985edc 1831 * We subtract one from sk_wmem_alloc and can know if
2b85a34e
ED		 1832 * some packets are still in some tx queue.
1833 * If not null, sock_wfree() will call __sk_free(sk) later
1834 */
14afee4b 1835 if (refcount_dec_and_test(&sk->sk_wmem_alloc))
2b85a34e
ED		 1836 __sk_free(sk);
1837}
2a91525c 1838EXPORT_SYMBOL(sk_free);
1da177e4 1839
581319c5
PA		 1840static void sk_init_common(struct sock *sk)
1841{
1842 skb_queue_head_init(&sk->sk_receive_queue);
1843 skb_queue_head_init(&sk->sk_write_queue);
1844 skb_queue_head_init(&sk->sk_error_queue);
1845
1846 rwlock_init(&sk->sk_callback_lock);
1847 lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
1848 af_rlock_keys + sk->sk_family,
1849 af_family_rlock_key_strings[sk->sk_family]);
1850 lockdep_set_class_and_name(&sk->sk_write_queue.lock,
1851 af_wlock_keys + sk->sk_family,
1852 af_family_wlock_key_strings[sk->sk_family]);
1853 lockdep_set_class_and_name(&sk->sk_error_queue.lock,
1854 af_elock_keys + sk->sk_family,
1855 af_family_elock_key_strings[sk->sk_family]);
1856 lockdep_set_class_and_name(&sk->sk_callback_lock,
1857 af_callback_keys + sk->sk_family,
1858 af_family_clock_key_strings[sk->sk_family]);
1859}
1860
e56c57d0
ED		 1861/**
1862 * sk_clone_lock - clone a socket, and lock its clone
1863 * @sk: the socket to clone
1864 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1865 *
1866 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1867 */
1868struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
87d11ceb 1869{
b8e202d1 1870 struct proto *prot = READ_ONCE(sk->sk_prot);
8fd1d178 1871 struct sock *newsk;
278571ba 1872 bool is_charged = true;
87d11ceb 1873
b8e202d1 1874 newsk = sk_prot_alloc(prot, priority, sk->sk_family);
87d11ceb
ACM		 1875 if (newsk != NULL) {
1876 struct sk_filter *filter;
1877
892c141e 1878 sock_copy(newsk, sk);
87d11ceb 1879
b8e202d1 1880 newsk->sk_prot_creator = prot;
9d538fa6 1881
87d11ceb 1882 /* SANITY */
8a681736
SV		 1883 if (likely(newsk->sk_net_refcnt))
1884 get_net(sock_net(newsk));
87d11ceb
ACM		 1885 sk_node_init(&newsk->sk_node);
1886 sock_lock_init(newsk);
1887 bh_lock_sock(newsk);
fa438ccf 1888 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
8eae939f 1889 newsk->sk_backlog.len = 0;
87d11ceb
ACM		 1890
1891 atomic_set(&newsk->sk_rmem_alloc, 0);
2b85a34e
ED		 1892 /*
1893 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1894 */
14afee4b 1895 refcount_set(&newsk->sk_wmem_alloc, 1);
87d11ceb 1896 atomic_set(&newsk->sk_omem_alloc, 0);
581319c5 1897 sk_init_common(newsk);
87d11ceb
ACM		 1898
1899 newsk->sk_dst_cache = NULL;
9b8805a3 1900 newsk->sk_dst_pending_confirm = 0;
87d11ceb
ACM		 1901 newsk->sk_wmem_queued = 0;
1902 newsk->sk_forward_alloc = 0;
9caad864 1903 atomic_set(&newsk->sk_drops, 0);
87d11ceb 1904 newsk->sk_send_head = NULL;
87d11ceb 1905 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
52267790 1906 atomic_set(&newsk->sk_zckey, 0);
87d11ceb
ACM		 1907
1908 sock_reset_flag(newsk, SOCK_DONE);
d752a498
SB		 1909
1910 /* sk->sk_memcg will be populated at accept() time */
1911 newsk->sk_memcg = NULL;
1912
ad0f75e5 1913 cgroup_sk_clone(&newsk->sk_cgrp_data);
87d11ceb 1914
eefca20e
ED		 1915 rcu_read_lock();
1916 filter = rcu_dereference(sk->sk_filter);
87d11ceb 1917 if (filter != NULL)
278571ba
AS		 1918 /* though it's an empty new sock, the charging may fail
1919 * if sysctl_optmem_max was changed between creation of
1920 * original socket and cloning
1921 */
1922 is_charged = sk_filter_charge(newsk, filter);
eefca20e
ED		 1923 RCU_INIT_POINTER(newsk->sk_filter, filter);
1924 rcu_read_unlock();
87d11ceb 1925
d188ba86 1926 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
a97e50cc
DB		 1927 /* We need to make sure that we don't uncharge the new
1928 * socket if we couldn't charge it in the first place
1929 * as otherwise we uncharge the parent's filter.
1930 */
1931 if (!is_charged)
1932 RCU_INIT_POINTER(newsk->sk_filter, NULL);
94352d45 1933 sk_free_unlock_clone(newsk);
87d11ceb
ACM		 1934 newsk = NULL;
1935 goto out;
1936 }
fa463497 1937 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
8f51dfc7
SF		 1938
1939 if (bpf_sk_storage_clone(sk, newsk)) {
1940 sk_free_unlock_clone(newsk);
1941 newsk = NULL;
1942 goto out;
1943 }
87d11ceb 1944
f1ff5ce2
JS		 1945 /* Clear sk_user_data if parent had the pointer tagged
1946 * as not suitable for copying when cloning.
1947 */
1948 if (sk_user_data_is_nocopy(newsk))
7a1ca972 1949 newsk->sk_user_data = NULL;
f1ff5ce2 1950
87d11ceb 1951 newsk->sk_err = 0;
e551c32d 1952 newsk->sk_err_soft = 0;
87d11ceb 1953 newsk->sk_priority = 0;
2c8c56e1 1954 newsk->sk_incoming_cpu = raw_smp_processor_id();
648845ab
TZ		 1955 if (likely(newsk->sk_net_refcnt))
1956 sock_inuse_add(sock_net(newsk), 1);
d979a39d 1957
4dc6dc71
ED		 1958 /*
1959 * Before updating sk_refcnt, we must commit prior changes to memory
2cdb54c9 1960 * (Documentation/RCU/rculist_nulls.rst for details)
4dc6dc71
ED		 1961 */
1962 smp_wmb();
41c6d650 1963 refcount_set(&newsk->sk_refcnt, 2);
87d11ceb
ACM		 1964
1965 /*
1966 * Increment the counter in the same struct proto as the master
1967 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1968 * is the same as sk->sk_prot->socks, as this field was copied
1969 * with memcpy).
1970 *
1971 * This _changes_ the previous behaviour, where
1972 * tcp_create_openreq_child always was incrementing the
1973 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1974 * to be taken into account in all callers. -acme
1975 */
1976 sk_refcnt_debug_inc(newsk);
972692e0 1977 sk_set_socket(newsk, NULL);
41b14fb8 1978 sk_tx_queue_clear(newsk);
c2f26e8f 1979 RCU_INIT_POINTER(newsk->sk_wq, NULL);
87d11ceb
ACM		 1980
1981 if (newsk->sk_prot->sockets_allocated)
180d8cd9 1982 sk_sockets_allocated_inc(newsk);
704da560 1983
080a270f
HFS		 1984 if (sock_needs_netstamp(sk) &&
1985 newsk->sk_flags & SK_FLAGS_TIMESTAMP)
704da560 1986 net_enable_timestamp();
87d11ceb
ACM		 1987 }
1988out:
1989 return newsk;
1990}
e56c57d0 1991EXPORT_SYMBOL_GPL(sk_clone_lock);
87d11ceb 1992
94352d45
ACM		 1993void sk_free_unlock_clone(struct sock *sk)
1994{
1995 /* It is still raw copy of parent, so invalidate
1996 * destructor and make plain sk_free() */
1997 sk->sk_destruct = NULL;
1998 bh_unlock_sock(sk);
1999 sk_free(sk);
2000}
2001EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
2002
9958089a
AK		 2003void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
2004{
d6a4e26a
ED		 2005 u32 max_segs = 1;
2006
6bd4f355 2007 sk_dst_set(sk, dst);
0a6b2a1d 2008 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps;
9958089a 2009 if (sk->sk_route_caps & NETIF_F_GSO)
4fcd6b99 2010 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
a465419b 2011 sk->sk_route_caps &= ~sk->sk_route_nocaps;
9958089a 2012 if (sk_can_gso(sk)) {
f70f250a 2013 if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
9958089a 2014 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
82cc1a7a 2015 } else {
9958089a 2016 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
82cc1a7a 2017 sk->sk_gso_max_size = dst->dev->gso_max_size;
d6a4e26a 2018 max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
82cc1a7a 2019 }
9958089a 2020 }
d6a4e26a 2021 sk->sk_gso_max_segs = max_segs;
9958089a
AK		 2022}
2023EXPORT_SYMBOL_GPL(sk_setup_caps);
2024
1da177e4
LT		 2025/*
2026 * Simple resource managers for sockets.
2027 */
2028
2029
4ec93edb
YH		 2030/*
2031 * Write buffer destructor automatically called from kfree_skb.
1da177e4
LT		 2032 */
2033void sock_wfree(struct sk_buff *skb)
2034{
2035 struct sock *sk = skb->sk;
d99927f4 2036 unsigned int len = skb->truesize;
1da177e4 2037
d99927f4
ED		 2038 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
2039 /*
2040 * Keep a reference on sk_wmem_alloc, this will be released
2041 * after sk_write_space() call
2042 */
14afee4b 2043 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
1da177e4 2044 sk->sk_write_space(sk);
d99927f4
ED		 2045 len = 1;
2046 }
2b85a34e 2047 /*
d99927f4
ED		 2048 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
2049 * could not do because of in-flight packets
2b85a34e 2050 */
14afee4b 2051 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2b85a34e 2052 __sk_free(sk);
1da177e4 2053}
2a91525c 2054EXPORT_SYMBOL(sock_wfree);
1da177e4 2055
1d2077ac
ED		 2056/* This variant of sock_wfree() is used by TCP,
2057 * since it sets SOCK_USE_WRITE_QUEUE.
2058 */
2059void __sock_wfree(struct sk_buff *skb)
2060{
2061 struct sock *sk = skb->sk;
2062
14afee4b 2063 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1d2077ac
ED		 2064 __sk_free(sk);
2065}
2066
9e17f8a4
ED		 2067void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
2068{
2069 skb_orphan(skb);
2070 skb->sk = sk;
2071#ifdef CONFIG_INET
2072 if (unlikely(!sk_fullsock(sk))) {
2073 skb->destructor = sock_edemux;
2074 sock_hold(sk);
2075 return;
2076 }
2077#endif
2078 skb->destructor = sock_wfree;
2079 skb_set_hash_from_sk(skb, sk);
2080 /*
2081 * We used to take a refcount on sk, but following operation
2082 * is enough to guarantee sk_free() wont free this sock until
2083 * all in-flight packets are completed
2084 */
14afee4b 2085 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
9e17f8a4
ED		 2086}
2087EXPORT_SYMBOL(skb_set_owner_w);
2088
41477662
JK		 2089static bool can_skb_orphan_partial(const struct sk_buff *skb)
2090{
2091#ifdef CONFIG_TLS_DEVICE
2092 /* Drivers depend on in-order delivery for crypto offload,
2093 * partial orphan breaks out-of-order-OK logic.
2094 */
2095 if (skb->decrypted)
2096 return false;
2097#endif
2098 return (skb->destructor == sock_wfree ||
2099 (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree));
2100}
2101
1d2077ac
ED		 2102/* This helper is used by netem, as it can hold packets in its
2103 * delay queue. We want to allow the owner socket to send more
2104 * packets, as if they were already TX completed by a typical driver.
2105 * But we also want to keep skb->sk set because some packet schedulers
f6ba8d33 2106 * rely on it (sch_fq for example).
1d2077ac 2107 */
f2f872f9
ED		 2108void skb_orphan_partial(struct sk_buff *skb)
2109{
f6ba8d33 2110 if (skb_is_tcp_pure_ack(skb))
1d2077ac
ED		 2111 return;
2112
41477662 2113 if (can_skb_orphan_partial(skb)) {
f6ba8d33
ED		 2114 struct sock *sk = skb->sk;
2115
41c6d650 2116 if (refcount_inc_not_zero(&sk->sk_refcnt)) {
14afee4b 2117 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc));
f6ba8d33
ED		 2118 skb->destructor = sock_efree;
2119 }
f2f872f9
ED		 2120 } else {
2121 skb_orphan(skb);
2122 }
2123}
2124EXPORT_SYMBOL(skb_orphan_partial);
2125
4ec93edb
YH		 2126/*
2127 * Read buffer destructor automatically called from kfree_skb.
1da177e4
LT		 2128 */
2129void sock_rfree(struct sk_buff *skb)
2130{
2131 struct sock *sk = skb->sk;
d361fd59 2132 unsigned int len = skb->truesize;
1da177e4 2133
d361fd59
ED		 2134 atomic_sub(len, &sk->sk_rmem_alloc);
2135 sk_mem_uncharge(sk, len);
1da177e4 2136}
2a91525c 2137EXPORT_SYMBOL(sock_rfree);
1da177e4 2138
7768eed8
OH		 2139/*
2140 * Buffer destructor for skbs that are not used directly in read or write
2141 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
2142 */
62bccb8c
AD		 2143void sock_efree(struct sk_buff *skb)
2144{
2145 sock_put(skb->sk);
2146}
2147EXPORT_SYMBOL(sock_efree);
2148
cf7fbe66
JS		 2149/* Buffer destructor for prefetch/receive path where reference count may
2150 * not be held, e.g. for listen sockets.
2151 */
2152#ifdef CONFIG_INET
2153void sock_pfree(struct sk_buff *skb)
2154{
7ae215d2
JS		 2155 if (sk_is_refcounted(skb->sk))
2156 sock_gen_put(skb->sk);
cf7fbe66
JS		 2157}
2158EXPORT_SYMBOL(sock_pfree);
2159#endif /* CONFIG_INET */
2160
976d0201 2161kuid_t sock_i_uid(struct sock *sk)
1da177e4 2162{
976d0201 2163 kuid_t uid;
1da177e4 2164
f064af1e 2165 read_lock_bh(&sk->sk_callback_lock);
976d0201 2166 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
f064af1e 2167 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 2168 return uid;
2169}
2a91525c 2170EXPORT_SYMBOL(sock_i_uid);
1da177e4
LT		 2171
2172unsigned long sock_i_ino(struct sock *sk)
2173{
2174 unsigned long ino;
2175
f064af1e 2176 read_lock_bh(&sk->sk_callback_lock);
1da177e4 2177 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
f064af1e 2178 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 2179 return ino;
2180}
2a91525c 2181EXPORT_SYMBOL(sock_i_ino);
1da177e4
LT		 2182
2183/*
2184 * Allocate a skb from the socket's send buffer.
2185 */
86a76caf 2186struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
dd0fc66f 2187 gfp_t priority)
1da177e4 2188{
e292f05e
ED		 2189 if (force ||
2190 refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
2a91525c 2191 struct sk_buff *skb = alloc_skb(size, priority);
e292f05e 2192
1da177e4
LT		 2193 if (skb) {
2194 skb_set_owner_w(skb, sk);
2195 return skb;
2196 }
2197 }
2198 return NULL;
2199}
2a91525c 2200EXPORT_SYMBOL(sock_wmalloc);
1da177e4 2201
98ba0bd5
WB		 2202static void sock_ofree(struct sk_buff *skb)
2203{
2204 struct sock *sk = skb->sk;
2205
2206 atomic_sub(skb->truesize, &sk->sk_omem_alloc);
2207}
2208
2209struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
2210 gfp_t priority)
2211{
2212 struct sk_buff *skb;
2213
2214 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
2215 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
2216 sysctl_optmem_max)
2217 return NULL;
2218
2219 skb = alloc_skb(size, priority);
2220 if (!skb)
2221 return NULL;
2222
2223 atomic_add(skb->truesize, &sk->sk_omem_alloc);
2224 skb->sk = sk;
2225 skb->destructor = sock_ofree;
2226 return skb;
2227}
2228
4ec93edb 2229/*
1da177e4 2230 * Allocate a memory block from the socket's option memory buffer.
4ec93edb 2231 */
dd0fc66f 2232void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
1da177e4 2233{
95c96174 2234 if ((unsigned int)size <= sysctl_optmem_max &&
1da177e4
LT		 2235 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
2236 void *mem;
2237 /* First do the add, to avoid the race if kmalloc
4ec93edb 2238 * might sleep.
1da177e4
LT		 2239 */
2240 atomic_add(size, &sk->sk_omem_alloc);
2241 mem = kmalloc(size, priority);
2242 if (mem)
2243 return mem;
2244 atomic_sub(size, &sk->sk_omem_alloc);
2245 }
2246 return NULL;
2247}
2a91525c 2248EXPORT_SYMBOL(sock_kmalloc);
1da177e4 2249
79e88659
DB		 2250/* Free an option memory block. Note, we actually want the inline
2251 * here as this allows gcc to detect the nullify and fold away the
2252 * condition entirely.
1da177e4 2253 */
79e88659
DB		 2254static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
2255 const bool nullify)
1da177e4 2256{
e53da5fb
DM		 2257 if (WARN_ON_ONCE(!mem))
2258 return;
79e88659
DB		 2259 if (nullify)
2260 kzfree(mem);
2261 else
2262 kfree(mem);
1da177e4
LT		 2263 atomic_sub(size, &sk->sk_omem_alloc);
2264}
79e88659
DB		 2265
2266void sock_kfree_s(struct sock *sk, void *mem, int size)
2267{
2268 __sock_kfree_s(sk, mem, size, false);
2269}
2a91525c 2270EXPORT_SYMBOL(sock_kfree_s);
1da177e4 2271
79e88659
DB		 2272void sock_kzfree_s(struct sock *sk, void *mem, int size)
2273{
2274 __sock_kfree_s(sk, mem, size, true);
2275}
2276EXPORT_SYMBOL(sock_kzfree_s);
2277
1da177e4
LT		 2278/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
2279 I think, these locks should be removed for datagram sockets.
2280 */
2a91525c 2281static long sock_wait_for_wmem(struct sock *sk, long timeo)
1da177e4
LT		 2282{
2283 DEFINE_WAIT(wait);
2284
9cd3e072 2285 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4
LT		 2286 for (;;) {
2287 if (!timeo)
2288 break;
2289 if (signal_pending(current))
2290 break;
2291 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
aa395145 2292 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
e292f05e 2293 if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
1da177e4
LT		 2294 break;
2295 if (sk->sk_shutdown & SEND_SHUTDOWN)
2296 break;
2297 if (sk->sk_err)
2298 break;
2299 timeo = schedule_timeout(timeo);
2300 }
aa395145 2301 finish_wait(sk_sleep(sk), &wait);
1da177e4
LT		 2302 return timeo;
2303}
2304
2305
2306/*
2307 * Generic send/receive buffer handlers
2308 */
2309
4cc7f68d
HX		 2310struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
2311 unsigned long data_len, int noblock,
28d64271 2312 int *errcode, int max_page_order)
1da177e4 2313{
2e4e4410 2314 struct sk_buff *skb;
1da177e4
LT		 2315 long timeo;
2316 int err;
2317
1da177e4 2318 timeo = sock_sndtimeo(sk, noblock);
2e4e4410 2319 for (;;) {
1da177e4
LT		 2320 err = sock_error(sk);
2321 if (err != 0)
2322 goto failure;
2323
2324 err = -EPIPE;
2325 if (sk->sk_shutdown & SEND_SHUTDOWN)
2326 goto failure;
2327
e292f05e 2328 if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
2e4e4410 2329 break;
28d64271 2330
9cd3e072 2331 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2e4e4410
ED		 2332 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2333 err = -EAGAIN;
2334 if (!timeo)
1da177e4 2335 goto failure;
2e4e4410
ED		 2336 if (signal_pending(current))
2337 goto interrupted;
2338 timeo = sock_wait_for_wmem(sk, timeo);
1da177e4 2339 }
2e4e4410
ED		 2340 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2341 errcode, sk->sk_allocation);
2342 if (skb)
2343 skb_set_owner_w(skb, sk);
1da177e4
LT		 2344 return skb;
2345
2346interrupted:
2347 err = sock_intr_errno(timeo);
2348failure:
2349 *errcode = err;
2350 return NULL;
2351}
4cc7f68d 2352EXPORT_SYMBOL(sock_alloc_send_pskb);
1da177e4 2353
4ec93edb 2354struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1da177e4
LT		 2355 int noblock, int *errcode)
2356{
28d64271 2357 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
1da177e4 2358}
2a91525c 2359EXPORT_SYMBOL(sock_alloc_send_skb);
1da177e4 2360
39771b12
WB		 2361int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2362 struct sockcm_cookie *sockc)
2363{
3dd17e63
SHY		 2364 u32 tsflags;
2365
39771b12
WB		 2366 switch (cmsg->cmsg_type) {
2367 case SO_MARK:
2368 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
2369 return -EPERM;
2370 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2371 return -EINVAL;
2372 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2373 break;
7f1bc6e9 2374 case SO_TIMESTAMPING_OLD:
3dd17e63
SHY		 2375 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2376 return -EINVAL;
2377
2378 tsflags = *(u32 *)CMSG_DATA(cmsg);
2379 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2380 return -EINVAL;
2381
2382 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2383 sockc->tsflags |= tsflags;
2384 break;
80b14dee
RC		 2385 case SCM_TXTIME:
2386 if (!sock_flag(sk, SOCK_TXTIME))
2387 return -EINVAL;
2388 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
2389 return -EINVAL;
2390 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
2391 break;
779f1ede
SHY		 2392 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2393 case SCM_RIGHTS:
2394 case SCM_CREDENTIALS:
2395 break;
39771b12
WB		 2396 default:
2397 return -EINVAL;
2398 }
2399 return 0;
2400}
2401EXPORT_SYMBOL(__sock_cmsg_send);
2402
f28ea365
EJ		 2403int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2404 struct sockcm_cookie *sockc)
2405{
2406 struct cmsghdr *cmsg;
39771b12 2407 int ret;
f28ea365
EJ		 2408
2409 for_each_cmsghdr(cmsg, msg) {
2410 if (!CMSG_OK(msg, cmsg))
2411 return -EINVAL;
2412 if (cmsg->cmsg_level != SOL_SOCKET)
2413 continue;
39771b12
WB		 2414 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2415 if (ret)
2416 return ret;
f28ea365
EJ		 2417 }
2418 return 0;
2419}
2420EXPORT_SYMBOL(sock_cmsg_send);
2421
06044751
ED		 2422static void sk_enter_memory_pressure(struct sock *sk)
2423{
2424 if (!sk->sk_prot->enter_memory_pressure)
2425 return;
2426
2427 sk->sk_prot->enter_memory_pressure(sk);
2428}
2429
2430static void sk_leave_memory_pressure(struct sock *sk)
2431{
2432 if (sk->sk_prot->leave_memory_pressure) {
2433 sk->sk_prot->leave_memory_pressure(sk);
2434 } else {
2435 unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2436
503978ac
ED		 2437 if (memory_pressure && READ_ONCE(*memory_pressure))
2438 WRITE_ONCE(*memory_pressure, 0);
06044751
ED		 2439 }
2440}
2441
5640f768 2442#define SKB_FRAG_PAGE_ORDER get_order(32768)
ce27ec60 2443DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
5640f768 2444
400dfd3a
ED		 2445/**
2446 * skb_page_frag_refill - check that a page_frag contains enough room
2447 * @sz: minimum size of the fragment we want to get
2448 * @pfrag: pointer to page_frag
82d5e2b8 2449 * @gfp: priority for memory allocation
400dfd3a
ED		 2450 *
2451 * Note: While this allocator tries to use high order pages, there is
2452 * no guarantee that allocations succeed. Therefore, @sz MUST be
2453 * less or equal than PAGE_SIZE.
2454 */
d9b2938a 2455bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
5640f768 2456{
5640f768 2457 if (pfrag->page) {
fe896d18 2458 if (page_ref_count(pfrag->page) == 1) {
5640f768
ED		 2459 pfrag->offset = 0;
2460 return true;
2461 }
400dfd3a 2462 if (pfrag->offset + sz <= pfrag->size)
5640f768
ED		 2463 return true;
2464 put_page(pfrag->page);
2465 }
2466
d9b2938a 2467 pfrag->offset = 0;
ce27ec60
ED		 2468 if (SKB_FRAG_PAGE_ORDER &&
2469 !static_branch_unlikely(&net_high_order_alloc_disable_key)) {
d0164adc
MG		 2470 /* Avoid direct reclaim but allow kswapd to wake */
2471 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2472 __GFP_COMP | __GFP_NOWARN |
2473 __GFP_NORETRY,
d9b2938a 2474 SKB_FRAG_PAGE_ORDER);
5640f768 2475 if (likely(pfrag->page)) {
d9b2938a 2476 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
5640f768
ED		 2477 return true;
2478 }
d9b2938a
ED		 2479 }
2480 pfrag->page = alloc_page(gfp);
2481 if (likely(pfrag->page)) {
2482 pfrag->size = PAGE_SIZE;
2483 return true;
2484 }
400dfd3a
ED		 2485 return false;
2486}
2487EXPORT_SYMBOL(skb_page_frag_refill);
2488
2489bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2490{
2491 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2492 return true;
2493
5640f768
ED		 2494 sk_enter_memory_pressure(sk);
2495 sk_stream_moderate_sndbuf(sk);
2496 return false;
2497}
2498EXPORT_SYMBOL(sk_page_frag_refill);
2499
1da177e4 2500static void __lock_sock(struct sock *sk)
f39234d6
NK		 2501 __releases(&sk->sk_lock.slock)
2502 __acquires(&sk->sk_lock.slock)
1da177e4
LT		 2503{
2504 DEFINE_WAIT(wait);
2505
e71a4783 2506 for (;;) {
1da177e4
LT		 2507 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2508 TASK_UNINTERRUPTIBLE);
2509 spin_unlock_bh(&sk->sk_lock.slock);
2510 schedule();
2511 spin_lock_bh(&sk->sk_lock.slock);
e71a4783 2512 if (!sock_owned_by_user(sk))
1da177e4
LT		 2513 break;
2514 }
2515 finish_wait(&sk->sk_lock.wq, &wait);
2516}
2517
8873c064 2518void __release_sock(struct sock *sk)
f39234d6
NK		 2519 __releases(&sk->sk_lock.slock)
2520 __acquires(&sk->sk_lock.slock)
1da177e4 2521{
5413d1ba 2522 struct sk_buff *skb, *next;
1da177e4 2523
5413d1ba 2524 while ((skb = sk->sk_backlog.head) != NULL) {
1da177e4 2525 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
1da177e4 2526
5413d1ba 2527 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4 2528
5413d1ba
ED		 2529 do {
2530 next = skb->next;
e4cbb02a 2531 prefetch(next);
7fee226a 2532 WARN_ON_ONCE(skb_dst_is_noref(skb));
a8305bff 2533 skb_mark_not_on_list(skb);
c57943a1 2534 sk_backlog_rcv(sk, skb);
1da177e4 2535
5413d1ba 2536 cond_resched();
1da177e4
LT		 2537
2538 skb = next;
2539 } while (skb != NULL);
2540
5413d1ba
ED		 2541 spin_lock_bh(&sk->sk_lock.slock);
2542 }
8eae939f
ZY		 2543
2544 /*
2545 * Doing the zeroing here guarantee we can not loop forever
2546 * while a wild producer attempts to flood us.
2547 */
2548 sk->sk_backlog.len = 0;
1da177e4
LT		 2549}
2550
d41a69f1
ED		 2551void __sk_flush_backlog(struct sock *sk)
2552{
2553 spin_lock_bh(&sk->sk_lock.slock);
2554 __release_sock(sk);
2555 spin_unlock_bh(&sk->sk_lock.slock);
2556}
2557
1da177e4
LT		 2558/**
2559 * sk_wait_data - wait for data to arrive at sk_receive_queue
4dc3b16b
PP		 2560 * @sk: sock to wait on
2561 * @timeo: for how long
dfbafc99 2562 * @skb: last skb seen on sk_receive_queue
1da177e4
LT		 2563 *
2564 * Now socket state including sk->sk_err is changed only under lock,
2565 * hence we may omit checks after joining wait queue.
2566 * We check receive queue before schedule() only as optimization;
2567 * it is very likely that release_sock() added new data.
2568 */
dfbafc99 2569int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
1da177e4 2570{
d9dc8b0f 2571 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1da177e4 2572 int rc;
1da177e4 2573
d9dc8b0f 2574 add_wait_queue(sk_sleep(sk), &wait);
9cd3e072 2575 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2576 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
9cd3e072 2577 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2578 remove_wait_queue(sk_sleep(sk), &wait);
1da177e4
LT		 2579 return rc;
2580}
1da177e4
LT		 2581EXPORT_SYMBOL(sk_wait_data);
2582
3ab224be 2583/**
f8c3bf00 2584 * __sk_mem_raise_allocated - increase memory_allocated
3ab224be
HA		 2585 * @sk: socket
2586 * @size: memory size to allocate
f8c3bf00 2587 * @amt: pages to allocate
3ab224be
HA		 2588 * @kind: allocation type
2589 *
f8c3bf00 2590 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
3ab224be 2591 */
f8c3bf00 2592int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
3ab224be
HA		 2593{
2594 struct proto *prot = sk->sk_prot;
f8c3bf00 2595 long allocated = sk_memory_allocated_add(sk, amt);
d6f19938 2596 bool charged = true;
e805605c 2597
baac50bb 2598 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
d6f19938 2599 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt)))
e805605c 2600 goto suppress_allocation;
3ab224be
HA		 2601
2602 /* Under limit. */
e805605c 2603 if (allocated <= sk_prot_mem_limits(sk, 0)) {
180d8cd9 2604 sk_leave_memory_pressure(sk);
3ab224be
HA		 2605 return 1;
2606 }
2607
e805605c
JW		 2608 /* Under pressure. */
2609 if (allocated > sk_prot_mem_limits(sk, 1))
180d8cd9 2610 sk_enter_memory_pressure(sk);
3ab224be 2611
e805605c
JW		 2612 /* Over hard limit. */
2613 if (allocated > sk_prot_mem_limits(sk, 2))
3ab224be
HA		 2614 goto suppress_allocation;
2615
2616 /* guarantee minimum buffer size under pressure */
2617 if (kind == SK_MEM_RECV) {
a3dcaf17 2618 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
3ab224be 2619 return 1;
180d8cd9 2620
3ab224be 2621 } else { /* SK_MEM_SEND */
a3dcaf17
ED		 2622 int wmem0 = sk_get_wmem0(sk, prot);
2623
3ab224be 2624 if (sk->sk_type == SOCK_STREAM) {
a3dcaf17 2625 if (sk->sk_wmem_queued < wmem0)
3ab224be 2626 return 1;
a3dcaf17 2627 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
3ab224be 2628 return 1;
a3dcaf17 2629 }
3ab224be
HA		 2630 }
2631
180d8cd9 2632 if (sk_has_memory_pressure(sk)) {
5bf325a5 2633 u64 alloc;
1748376b 2634
180d8cd9 2635 if (!sk_under_memory_pressure(sk))
1748376b 2636 return 1;
180d8cd9
GC		 2637 alloc = sk_sockets_allocated_read_positive(sk);
2638 if (sk_prot_mem_limits(sk, 2) > alloc *
3ab224be
HA		 2639 sk_mem_pages(sk->sk_wmem_queued +
2640 atomic_read(&sk->sk_rmem_alloc) +
2641 sk->sk_forward_alloc))
2642 return 1;
2643 }
2644
2645suppress_allocation:
2646
2647 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2648 sk_stream_moderate_sndbuf(sk);
2649
2650 /* Fail only if socket is _under_ its sndbuf.
2651 * In this case we cannot block, so that we have to fail.
2652 */
2653 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2654 return 1;
2655 }
2656
d6f19938
YS		 2657 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged))
2658 trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
3847ce32 2659
0e90b31f 2660 sk_memory_allocated_sub(sk, amt);
180d8cd9 2661
baac50bb
JW		 2662 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2663 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
e805605c 2664
3ab224be
HA		 2665 return 0;
2666}
f8c3bf00
PA		 2667EXPORT_SYMBOL(__sk_mem_raise_allocated);
2668
2669/**
2670 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2671 * @sk: socket
2672 * @size: memory size to allocate
2673 * @kind: allocation type
2674 *
2675 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2676 * rmem allocation. This function assumes that protocols which have
2677 * memory_pressure use sk_wmem_queued as write buffer accounting.
2678 */
2679int __sk_mem_schedule(struct sock *sk, int size, int kind)
2680{
2681 int ret, amt = sk_mem_pages(size);
2682
2683 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
2684 ret = __sk_mem_raise_allocated(sk, size, amt, kind);
2685 if (!ret)
2686 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
2687 return ret;
2688}
3ab224be
HA		 2689EXPORT_SYMBOL(__sk_mem_schedule);
2690
2691/**
f8c3bf00 2692 * __sk_mem_reduce_allocated - reclaim memory_allocated
3ab224be 2693 * @sk: socket
f8c3bf00
PA		 2694 * @amount: number of quanta
2695 *
2696 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3ab224be 2697 */
f8c3bf00 2698void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3ab224be 2699{
1a24e04e 2700 sk_memory_allocated_sub(sk, amount);
3ab224be 2701
baac50bb
JW		 2702 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2703 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
e805605c 2704
180d8cd9
GC		 2705 if (sk_under_memory_pressure(sk) &&
2706 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2707 sk_leave_memory_pressure(sk);
3ab224be 2708}
f8c3bf00
PA		 2709EXPORT_SYMBOL(__sk_mem_reduce_allocated);
2710
2711/**
2712 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
2713 * @sk: socket
2714 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2715 */
2716void __sk_mem_reclaim(struct sock *sk, int amount)
2717{
2718 amount >>= SK_MEM_QUANTUM_SHIFT;
2719 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2720 __sk_mem_reduce_allocated(sk, amount);
2721}
3ab224be
HA		 2722EXPORT_SYMBOL(__sk_mem_reclaim);
2723
627d2d6b 2724int sk_set_peek_off(struct sock *sk, int val)
2725{
627d2d6b 2726 sk->sk_peek_off = val;
2727 return 0;
2728}
2729EXPORT_SYMBOL_GPL(sk_set_peek_off);
3ab224be 2730
1da177e4
LT		 2731/*
2732 * Set of default routines for initialising struct proto_ops when
2733 * the protocol does not support a particular function. In certain
2734 * cases where it makes no sense for a protocol to have a "do nothing"
2735 * function, some default processing is provided.
2736 */
2737
2738int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2739{
2740 return -EOPNOTSUPP;
2741}
2a91525c 2742EXPORT_SYMBOL(sock_no_bind);
1da177e4 2743
4ec93edb 2744int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
1da177e4
LT		 2745 int len, int flags)
2746{
2747 return -EOPNOTSUPP;
2748}
2a91525c 2749EXPORT_SYMBOL(sock_no_connect);
1da177e4
LT		 2750
2751int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2752{
2753 return -EOPNOTSUPP;
2754}
2a91525c 2755EXPORT_SYMBOL(sock_no_socketpair);
1da177e4 2756
cdfbabfb
DH		 2757int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
2758 bool kern)
1da177e4
LT		 2759{
2760 return -EOPNOTSUPP;
2761}
2a91525c 2762EXPORT_SYMBOL(sock_no_accept);
1da177e4 2763
4ec93edb 2764int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
9b2c45d4 2765 int peer)
1da177e4
LT		 2766{
2767 return -EOPNOTSUPP;
2768}
2a91525c 2769EXPORT_SYMBOL(sock_no_getname);
1da177e4 2770
1da177e4
LT		 2771int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2772{
2773 return -EOPNOTSUPP;
2774}
2a91525c 2775EXPORT_SYMBOL(sock_no_ioctl);
1da177e4
LT		 2776
2777int sock_no_listen(struct socket *sock, int backlog)
2778{
2779 return -EOPNOTSUPP;
2780}
2a91525c 2781EXPORT_SYMBOL(sock_no_listen);
1da177e4
LT		 2782
2783int sock_no_shutdown(struct socket *sock, int how)
2784{
2785 return -EOPNOTSUPP;
2786}
2a91525c 2787EXPORT_SYMBOL(sock_no_shutdown);
1da177e4 2788
1b784140 2789int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
1da177e4
LT		 2790{
2791 return -EOPNOTSUPP;
2792}
2a91525c 2793EXPORT_SYMBOL(sock_no_sendmsg);
1da177e4 2794
306b13eb
TH		 2795int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
2796{
2797 return -EOPNOTSUPP;
2798}
2799EXPORT_SYMBOL(sock_no_sendmsg_locked);
2800
1b784140
YX		 2801int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2802 int flags)
1da177e4
LT		 2803{
2804 return -EOPNOTSUPP;
2805}
2a91525c 2806EXPORT_SYMBOL(sock_no_recvmsg);
1da177e4
LT		 2807
2808int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2809{
2810 /* Mirror missing mmap method error code */
2811 return -ENODEV;
2812}
2a91525c 2813EXPORT_SYMBOL(sock_no_mmap);
1da177e4 2814
d9539752
KC		 2815/*
2816 * When a file is received (via SCM_RIGHTS, etc), we must bump the
2817 * various sock-based usage counts.
2818 */
2819void __receive_sock(struct file *file)
2820{
2821 struct socket *sock;
2822 int error;
2823
2824 /*
2825 * The resulting value of "error" is ignored here since we only
2826 * need to take action when the file is a socket and testing
2827 * "sock" for NULL is sufficient.
2828 */
2829 sock = sock_from_file(file, &error);
2830 if (sock) {
2831 sock_update_netprioidx(&sock->sk->sk_cgrp_data);
2832 sock_update_classid(&sock->sk->sk_cgrp_data);
2833 }
2834}
2835
1da177e4
LT		 2836ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2837{
2838 ssize_t res;
2839 struct msghdr msg = {.msg_flags = flags};
2840 struct kvec iov;
2841 char *kaddr = kmap(page);
2842 iov.iov_base = kaddr + offset;
2843 iov.iov_len = size;
2844 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2845 kunmap(page);
2846 return res;
2847}
2a91525c 2848EXPORT_SYMBOL(sock_no_sendpage);
1da177e4 2849
306b13eb
TH		 2850ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
2851 int offset, size_t size, int flags)
2852{
2853 ssize_t res;
2854 struct msghdr msg = {.msg_flags = flags};
2855 struct kvec iov;
2856 char *kaddr = kmap(page);
2857
2858 iov.iov_base = kaddr + offset;
2859 iov.iov_len = size;
2860 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
2861 kunmap(page);
2862 return res;
2863}
2864EXPORT_SYMBOL(sock_no_sendpage_locked);
2865
1da177e4
LT		 2866/*
2867 * Default Socket Callbacks
2868 */
2869
2870static void sock_def_wakeup(struct sock *sk)
2871{
43815482
ED		 2872 struct socket_wq *wq;
2873
2874 rcu_read_lock();
2875 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2876 if (skwq_has_sleeper(wq))
43815482
ED		 2877 wake_up_interruptible_all(&wq->wait);
2878 rcu_read_unlock();
1da177e4
LT		 2879}
2880
2881static void sock_def_error_report(struct sock *sk)
2882{
43815482
ED		 2883 struct socket_wq *wq;
2884
2885 rcu_read_lock();
2886 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2887 if (skwq_has_sleeper(wq))
a9a08845 2888 wake_up_interruptible_poll(&wq->wait, EPOLLERR);
8d8ad9d7 2889 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
43815482 2890 rcu_read_unlock();
1da177e4
LT		 2891}
2892
43a825af 2893void sock_def_readable(struct sock *sk)
1da177e4 2894{
43815482
ED		 2895 struct socket_wq *wq;
2896
2897 rcu_read_lock();
2898 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2899 if (skwq_has_sleeper(wq))
a9a08845
LT		 2900 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
2901 EPOLLRDNORM | EPOLLRDBAND);
8d8ad9d7 2902 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
43815482 2903 rcu_read_unlock();
1da177e4
LT		 2904}
2905
2906static void sock_def_write_space(struct sock *sk)
2907{
43815482
ED		 2908 struct socket_wq *wq;
2909
2910 rcu_read_lock();
1da177e4
LT		 2911
2912 /* Do not wake up a writer until he can make "significant"
2913 * progress. --DaveM
2914 */
e292f05e 2915 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= READ_ONCE(sk->sk_sndbuf)) {
43815482 2916 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2917 if (skwq_has_sleeper(wq))
a9a08845
LT		 2918 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
2919 EPOLLWRNORM | EPOLLWRBAND);
1da177e4
LT		 2920
2921 /* Should agree with poll, otherwise some programs break */
2922 if (sock_writeable(sk))
8d8ad9d7 2923 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
1da177e4
LT		 2924 }
2925
43815482 2926 rcu_read_unlock();
1da177e4
LT		 2927}
2928
2929static void sock_def_destruct(struct sock *sk)
2930{
1da177e4
LT		 2931}
2932
2933void sk_send_sigurg(struct sock *sk)
2934{
2935 if (sk->sk_socket && sk->sk_socket->file)
2936 if (send_sigurg(&sk->sk_socket->file->f_owner))
8d8ad9d7 2937 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
1da177e4 2938}
2a91525c 2939EXPORT_SYMBOL(sk_send_sigurg);
1da177e4
LT		 2940
2941void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2942 unsigned long expires)
2943{
2944 if (!mod_timer(timer, expires))
2945 sock_hold(sk);
2946}
1da177e4
LT		 2947EXPORT_SYMBOL(sk_reset_timer);
2948
2949void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2950{
25cc4ae9 2951 if (del_timer(timer))
1da177e4
LT		 2952 __sock_put(sk);
2953}
1da177e4
LT		 2954EXPORT_SYMBOL(sk_stop_timer);
2955
2956void sock_init_data(struct socket *sock, struct sock *sk)
2957{
581319c5 2958 sk_init_common(sk);
1da177e4
LT		 2959 sk->sk_send_head = NULL;
2960
99767f27 2961 timer_setup(&sk->sk_timer, NULL, 0);
4ec93edb 2962
1da177e4
LT		 2963 sk->sk_allocation = GFP_KERNEL;
2964 sk->sk_rcvbuf = sysctl_rmem_default;
2965 sk->sk_sndbuf = sysctl_wmem_default;
2966 sk->sk_state = TCP_CLOSE;
972692e0 2967 sk_set_socket(sk, sock);
1da177e4
LT		 2968
2969 sock_set_flag(sk, SOCK_ZAPPED);
2970
e71a4783 2971 if (sock) {
1da177e4 2972 sk->sk_type = sock->type;
333f7909 2973 RCU_INIT_POINTER(sk->sk_wq, &sock->wq);
1da177e4 2974 sock->sk = sk;
86741ec2
LC		 2975 sk->sk_uid = SOCK_INODE(sock)->i_uid;
2976 } else {
c2f26e8f 2977 RCU_INIT_POINTER(sk->sk_wq, NULL);
86741ec2
LC		 2978 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
2979 }
1da177e4 2980
1da177e4 2981 rwlock_init(&sk->sk_callback_lock);
cdfbabfb
DH		 2982 if (sk->sk_kern_sock)
2983 lockdep_set_class_and_name(
2984 &sk->sk_callback_lock,
2985 af_kern_callback_keys + sk->sk_family,
2986 af_family_kern_clock_key_strings[sk->sk_family]);
2987 else
2988 lockdep_set_class_and_name(
2989 &sk->sk_callback_lock,
443aef0e
PZ		 2990 af_callback_keys + sk->sk_family,
2991 af_family_clock_key_strings[sk->sk_family]);
1da177e4
LT		 2992
2993 sk->sk_state_change = sock_def_wakeup;
2994 sk->sk_data_ready = sock_def_readable;
2995 sk->sk_write_space = sock_def_write_space;
2996 sk->sk_error_report = sock_def_error_report;
2997 sk->sk_destruct = sock_def_destruct;
2998
5640f768
ED		 2999 sk->sk_frag.page = NULL;
3000 sk->sk_frag.offset = 0;
ef64a54f 3001 sk->sk_peek_off = -1;
1da177e4 3002
109f6e39
EB		 3003 sk->sk_peer_pid = NULL;
3004 sk->sk_peer_cred = NULL;
1da177e4
LT		 3005 sk->sk_write_pending = 0;
3006 sk->sk_rcvlowat = 1;
3007 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
3008 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
3009
6c7c98ba 3010 sk->sk_stamp = SK_DEFAULT_STAMP;
3a0ed3e9
DD		 3011#if BITS_PER_LONG==32
3012 seqlock_init(&sk->sk_stamp_seq);
3013#endif
52267790 3014 atomic_set(&sk->sk_zckey, 0);
1da177e4 3015
e0d1095a 3016#ifdef CONFIG_NET_RX_BUSY_POLL
06021292 3017 sk->sk_napi_id = 0;
64b0dc51 3018 sk->sk_ll_usec = sysctl_net_busy_read;
06021292
ET		 3019#endif
3020
76a9ebe8
ED		 3021 sk->sk_max_pacing_rate = ~0UL;
3022 sk->sk_pacing_rate = ~0UL;
7c68fa2b 3023 WRITE_ONCE(sk->sk_pacing_shift, 10);
70da268b 3024 sk->sk_incoming_cpu = -1;
c6345ce7
AN		 3025
3026 sk_rx_queue_clear(sk);
4dc6dc71
ED		 3027 /*
3028 * Before updating sk_refcnt, we must commit prior changes to memory
2cdb54c9 3029 * (Documentation/RCU/rculist_nulls.rst for details)
4dc6dc71
ED		 3030 */
3031 smp_wmb();
41c6d650 3032 refcount_set(&sk->sk_refcnt, 1);
33c732c3 3033 atomic_set(&sk->sk_drops, 0);
1da177e4 3034}
2a91525c 3035EXPORT_SYMBOL(sock_init_data);
1da177e4 3036
b5606c2d 3037void lock_sock_nested(struct sock *sk, int subclass)
1da177e4
LT		 3038{
3039 might_sleep();
a5b5bb9a 3040 spin_lock_bh(&sk->sk_lock.slock);
d2e9117c 3041 if (sk->sk_lock.owned)
1da177e4 3042 __lock_sock(sk);
d2e9117c 3043 sk->sk_lock.owned = 1;
a5b5bb9a
IM		 3044 spin_unlock(&sk->sk_lock.slock);
3045 /*
3046 * The sk_lock has mutex_lock() semantics here:
3047 */
fcc70d5f 3048 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
a5b5bb9a 3049 local_bh_enable();
1da177e4 3050}
fcc70d5f 3051EXPORT_SYMBOL(lock_sock_nested);
1da177e4 3052
b5606c2d 3053void release_sock(struct sock *sk)
1da177e4 3054{
a5b5bb9a 3055 spin_lock_bh(&sk->sk_lock.slock);
1da177e4
LT		 3056 if (sk->sk_backlog.tail)
3057 __release_sock(sk);
46d3ceab 3058
c3f9b018
ED		 3059 /* Warning : release_cb() might need to release sk ownership,
3060 * ie call sock_release_ownership(sk) before us.
3061 */
46d3ceab
ED		 3062 if (sk->sk_prot->release_cb)
3063 sk->sk_prot->release_cb(sk);
3064
c3f9b018 3065 sock_release_ownership(sk);
a5b5bb9a
IM		 3066 if (waitqueue_active(&sk->sk_lock.wq))
3067 wake_up(&sk->sk_lock.wq);
3068 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4
LT		 3069}
3070EXPORT_SYMBOL(release_sock);
3071
8a74ad60
ED		 3072/**
3073 * lock_sock_fast - fast version of lock_sock
3074 * @sk: socket
3075 *
3076 * This version should be used for very small section, where process wont block
d651983d
MCC		 3077 * return false if fast path is taken:
3078 *
8a74ad60 3079 * sk_lock.slock locked, owned = 0, BH disabled
d651983d
MCC		 3080 *
3081 * return true if slow path is taken:
3082 *
8a74ad60
ED		 3083 * sk_lock.slock unlocked, owned = 1, BH enabled
3084 */
3085bool lock_sock_fast(struct sock *sk)
3086{
3087 might_sleep();
3088 spin_lock_bh(&sk->sk_lock.slock);
3089
3090 if (!sk->sk_lock.owned)
3091 /*
3092 * Note : We must disable BH
3093 */
3094 return false;
3095
3096 __lock_sock(sk);
3097 sk->sk_lock.owned = 1;
3098 spin_unlock(&sk->sk_lock.slock);
3099 /*
3100 * The sk_lock has mutex_lock() semantics here:
3101 */
3102 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
3103 local_bh_enable();
3104 return true;
3105}
3106EXPORT_SYMBOL(lock_sock_fast);
3107
c7cbdbf2
AB		 3108int sock_gettstamp(struct socket *sock, void __user *userstamp,
3109 bool timeval, bool time32)
4ec93edb 3110{
c7cbdbf2
AB		 3111 struct sock *sk = sock->sk;
3112 struct timespec64 ts;
9dae3497
YS		 3113
3114 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
c7cbdbf2
AB		 3115 ts = ktime_to_timespec64(sock_read_timestamp(sk));
3116 if (ts.tv_sec == -1)
1da177e4 3117 return -ENOENT;
c7cbdbf2 3118 if (ts.tv_sec == 0) {
3a0ed3e9 3119 ktime_t kt = ktime_get_real();
f95f96a4 3120 sock_write_timestamp(sk, kt);
c7cbdbf2 3121 ts = ktime_to_timespec64(kt);
b7aa0bf7 3122 }
1da177e4 3123
c7cbdbf2
AB		 3124 if (timeval)
3125 ts.tv_nsec /= 1000;
9dae3497 3126
c7cbdbf2
AB		 3127#ifdef CONFIG_COMPAT_32BIT_TIME
3128 if (time32)
3129 return put_old_timespec32(&ts, userstamp);
3130#endif
3131#ifdef CONFIG_SPARC64
3132 /* beware of padding in sparc64 timeval */
3133 if (timeval && !in_compat_syscall()) {
3134 struct __kernel_old_timeval __user tv = {
c98f4822
SR		 3135 .tv_sec = ts.tv_sec,
3136 .tv_usec = ts.tv_nsec,
c7cbdbf2 3137 };
c98f4822 3138 if (copy_to_user(userstamp, &tv, sizeof(tv)))
c7cbdbf2
AB		 3139 return -EFAULT;
3140 return 0;
ae40eb1e 3141 }
c7cbdbf2
AB		 3142#endif
3143 return put_timespec64(&ts, userstamp);
ae40eb1e 3144}
c7cbdbf2 3145EXPORT_SYMBOL(sock_gettstamp);
ae40eb1e 3146
193d357d 3147void sock_enable_timestamp(struct sock *sk, enum sock_flags flag)
4ec93edb 3148{
20d49473 3149 if (!sock_flag(sk, flag)) {
08e29af3
ED		 3150 unsigned long previous_flags = sk->sk_flags;
3151
20d49473
PO		 3152 sock_set_flag(sk, flag);
3153 /*
3154 * we just set one of the two flags which require net
3155 * time stamping, but time stamping might have been on
3156 * already because of the other one
3157 */
080a270f
HFS		 3158 if (sock_needs_netstamp(sk) &&
3159 !(previous_flags & SK_FLAGS_TIMESTAMP))
20d49473 3160 net_enable_timestamp();
1da177e4
LT		 3161 }
3162}
1da177e4 3163
cb820f8e
RC		 3164int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
3165 int level, int type)
3166{
3167 struct sock_exterr_skb *serr;
364a9e93 3168 struct sk_buff *skb;
cb820f8e
RC		 3169 int copied, err;
3170
3171 err = -EAGAIN;
364a9e93 3172 skb = sock_dequeue_err_skb(sk);
cb820f8e
RC		 3173 if (skb == NULL)
3174 goto out;
3175
3176 copied = skb->len;
3177 if (copied > len) {
3178 msg->msg_flags |= MSG_TRUNC;
3179 copied = len;
3180 }
51f3d02b 3181 err = skb_copy_datagram_msg(skb, 0, msg, copied);
cb820f8e
RC		 3182 if (err)
3183 goto out_free_skb;
3184
3185 sock_recv_timestamp(msg, sk, skb);
3186
3187 serr = SKB_EXT_ERR(skb);
3188 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
3189
3190 msg->msg_flags |= MSG_ERRQUEUE;
3191 err = copied;
3192
cb820f8e
RC		 3193out_free_skb:
3194 kfree_skb(skb);
3195out:
3196 return err;
3197}
3198EXPORT_SYMBOL(sock_recv_errqueue);
3199
1da177e4
LT		 3200/*
3201 * Get a socket option on an socket.
3202 *
3203 * FIX: POSIX 1003.1g is very ambiguous here. It states that
3204 * asynchronous errors should be reported by getsockopt. We assume
3205 * this means if you specify SO_ERROR (otherwise whats the point of it).
3206 */
3207int sock_common_getsockopt(struct socket *sock, int level, int optname,
3208 char __user *optval, int __user *optlen)
3209{
3210 struct sock *sk = sock->sk;
3211
3212 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
3213}
1da177e4
LT		 3214EXPORT_SYMBOL(sock_common_getsockopt);
3215
1b784140
YX		 3216int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
3217 int flags)
1da177e4
LT		 3218{
3219 struct sock *sk = sock->sk;
3220 int addr_len = 0;
3221 int err;
3222
1b784140 3223 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
1da177e4
LT		 3224 flags & ~MSG_DONTWAIT, &addr_len);
3225 if (err >= 0)
3226 msg->msg_namelen = addr_len;
3227 return err;
3228}
1da177e4
LT		 3229EXPORT_SYMBOL(sock_common_recvmsg);
3230
3231/*
3232 * Set socket options on an inet socket.
3233 */
3234int sock_common_setsockopt(struct socket *sock, int level, int optname,
a7b75c5a 3235 sockptr_t optval, unsigned int optlen)
1da177e4
LT		 3236{
3237 struct sock *sk = sock->sk;
3238
3239 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
3240}
1da177e4
LT		 3241EXPORT_SYMBOL(sock_common_setsockopt);
3242
3243void sk_common_release(struct sock *sk)
3244{
3245 if (sk->sk_prot->destroy)
3246 sk->sk_prot->destroy(sk);
3247
3248 /*
3249 * Observation: when sock_common_release is called, processes have
3250 * no access to socket. But net still has.
3251 * Step one, detach it from networking:
3252 *
3253 * A. Remove from hash tables.
3254 */
3255
3256 sk->sk_prot->unhash(sk);
3257
3258 /*
3259 * In this point socket cannot receive new packets, but it is possible
3260 * that some packets are in flight because some CPU runs receiver and
3261 * did hash table lookup before we unhashed socket. They will achieve
3262 * receive queue and will be purged by socket destructor.
3263 *
3264 * Also we still have packets pending on receive queue and probably,
3265 * our own packets waiting in device queues. sock_destroy will drain
3266 * receive queue, but transmitted packets will delay socket destruction
3267 * until the last reference will be released.
3268 */
3269
3270 sock_orphan(sk);
3271
3272 xfrm_sk_free_policy(sk);
3273
e6848976 3274 sk_refcnt_debug_release(sk);
5640f768 3275
1da177e4
LT		 3276 sock_put(sk);
3277}
1da177e4
LT		 3278EXPORT_SYMBOL(sk_common_release);
3279
a2d133b1
JH		 3280void sk_get_meminfo(const struct sock *sk, u32 *mem)
3281{
3282 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
3283
3284 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
ebb3b78d 3285 mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
a2d133b1 3286 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
e292f05e 3287 mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
a2d133b1 3288 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
ab4e846a 3289 mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
a2d133b1 3290 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
70c26558 3291 mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
a2d133b1
JH		 3292 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
3293}
3294
13ff3d6f
PE		 3295#ifdef CONFIG_PROC_FS
3296#define PROTO_INUSE_NR 64 /* should be enough for the first time */
1338d466
PE		 3297struct prot_inuse {
3298 int val[PROTO_INUSE_NR];
3299};
13ff3d6f
PE		 3300
3301static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
70ee1159 3302
70ee1159
PE		 3303void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
3304{
08fc7f81 3305 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val);
70ee1159
PE		 3306}
3307EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
3308
3309int sock_prot_inuse_get(struct net *net, struct proto *prot)
3310{
3311 int cpu, idx = prot->inuse_idx;
3312 int res = 0;
3313
3314 for_each_possible_cpu(cpu)
08fc7f81 3315 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
70ee1159
PE		 3316
3317 return res >= 0 ? res : 0;
3318}
3319EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
3320
648845ab
TZ		 3321static void sock_inuse_add(struct net *net, int val)
3322{
3323 this_cpu_add(*net->core.sock_inuse, val);
3324}
3325
3326int sock_inuse_get(struct net *net)
3327{
3328 int cpu, res = 0;
3329
3330 for_each_possible_cpu(cpu)
3331 res += *per_cpu_ptr(net->core.sock_inuse, cpu);
3332
3333 return res;
3334}
3335
3336EXPORT_SYMBOL_GPL(sock_inuse_get);
3337
2c8c1e72 3338static int __net_init sock_inuse_init_net(struct net *net)
70ee1159 3339{
08fc7f81 3340 net->core.prot_inuse = alloc_percpu(struct prot_inuse);
648845ab
TZ		 3341 if (net->core.prot_inuse == NULL)
3342 return -ENOMEM;
3343
3344 net->core.sock_inuse = alloc_percpu(int);
3345 if (net->core.sock_inuse == NULL)
3346 goto out;
3347
3348 return 0;
3349
3350out:
3351 free_percpu(net->core.prot_inuse);
3352 return -ENOMEM;
70ee1159
PE		 3353}
3354
2c8c1e72 3355static void __net_exit sock_inuse_exit_net(struct net *net)
70ee1159 3356{
08fc7f81 3357 free_percpu(net->core.prot_inuse);
648845ab 3358 free_percpu(net->core.sock_inuse);
70ee1159
PE		 3359}
3360
3361static struct pernet_operations net_inuse_ops = {
3362 .init = sock_inuse_init_net,
3363 .exit = sock_inuse_exit_net,
3364};
3365
3366static __init int net_inuse_init(void)
3367{
3368 if (register_pernet_subsys(&net_inuse_ops))
3369 panic("Cannot initialize net inuse counters");
3370
3371 return 0;
3372}
3373
3374core_initcall(net_inuse_init);
13ff3d6f 3375
b45ce321 3376static int assign_proto_idx(struct proto *prot)
13ff3d6f
PE		 3377{
3378 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3379
3380 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
e005d193 3381 pr_err("PROTO_INUSE_NR exhausted\n");
b45ce321 3382 return -ENOSPC;
13ff3d6f
PE		 3383 }
3384
3385 set_bit(prot->inuse_idx, proto_inuse_idx);
b45ce321 3386 return 0;
13ff3d6f
PE		 3387}
3388
3389static void release_proto_idx(struct proto *prot)
3390{
3391 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3392 clear_bit(prot->inuse_idx, proto_inuse_idx);
3393}
3394#else
b45ce321 3395static inline int assign_proto_idx(struct proto *prot)
13ff3d6f 3396{
b45ce321 3397 return 0;
13ff3d6f
PE		 3398}
3399
3400static inline void release_proto_idx(struct proto *prot)
3401{
3402}
648845ab
TZ		 3403
3404static void sock_inuse_add(struct net *net, int val)
3405{
3406}
13ff3d6f
PE		 3407#endif
3408
0159dfd3
ED		 3409static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3410{
3411 if (!rsk_prot)
3412 return;
3413 kfree(rsk_prot->slab_name);
3414 rsk_prot->slab_name = NULL;
adf78eda
JL		 3415 kmem_cache_destroy(rsk_prot->slab);
3416 rsk_prot->slab = NULL;
0159dfd3
ED		 3417}
3418
3419static int req_prot_init(const struct proto *prot)
3420{
3421 struct request_sock_ops *rsk_prot = prot->rsk_prot;
3422
3423 if (!rsk_prot)
3424 return 0;
3425
3426 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3427 prot->name);
3428 if (!rsk_prot->slab_name)
3429 return -ENOMEM;
3430
3431 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3432 rsk_prot->obj_size, 0,
e699e2c6
SB		 3433 SLAB_ACCOUNT | prot->slab_flags,
3434 NULL);
0159dfd3
ED		 3435
3436 if (!rsk_prot->slab) {
3437 pr_crit("%s: Can't create request sock SLAB cache!\n",
3438 prot->name);
3439 return -ENOMEM;
3440 }
3441 return 0;
3442}
3443
b733c007
PE		 3444int proto_register(struct proto *prot, int alloc_slab)
3445{
b45ce321 3446 int ret = -ENOBUFS;
3447
1da177e4 3448 if (alloc_slab) {
30c2c9f1
DW		 3449 prot->slab = kmem_cache_create_usercopy(prot->name,
3450 prot->obj_size, 0,
e699e2c6
SB		 3451 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
3452 prot->slab_flags,
289a4860 3453 prot->useroffset, prot->usersize,
271b72c7 3454 NULL);
1da177e4
LT		 3455
3456 if (prot->slab == NULL) {
e005d193
JP		 3457 pr_crit("%s: Can't create sock SLAB cache!\n",
3458 prot->name);
60e7663d 3459 goto out;
1da177e4 3460 }
2e6599cb 3461
0159dfd3
ED		 3462 if (req_prot_init(prot))
3463 goto out_free_request_sock_slab;
8feaf0c0 3464
6d6ee43e 3465 if (prot->twsk_prot != NULL) {
faf23422 3466 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
8feaf0c0 3467
7e56b5d6 3468 if (prot->twsk_prot->twsk_slab_name == NULL)
8feaf0c0
ACM		 3469 goto out_free_request_sock_slab;
3470
6d6ee43e 3471 prot->twsk_prot->twsk_slab =
7e56b5d6 3472 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
6d6ee43e 3473 prot->twsk_prot->twsk_obj_size,
3ab5aee7 3474 0,
e699e2c6 3475 SLAB_ACCOUNT |
52db70dc 3476 prot->slab_flags,
20c2df83 3477 NULL);
6d6ee43e 3478 if (prot->twsk_prot->twsk_slab == NULL)
8feaf0c0
ACM		 3479 goto out_free_timewait_sock_slab_name;
3480 }
1da177e4
LT		 3481 }
3482
36b77a52 3483 mutex_lock(&proto_list_mutex);
b45ce321 3484 ret = assign_proto_idx(prot);
3485 if (ret) {
3486 mutex_unlock(&proto_list_mutex);
3487 goto out_free_timewait_sock_slab_name;
3488 }
1da177e4 3489 list_add(&prot->node, &proto_list);
36b77a52 3490 mutex_unlock(&proto_list_mutex);
b45ce321 3491 return ret;
b733c007 3492
8feaf0c0 3493out_free_timewait_sock_slab_name:
b45ce321 3494 if (alloc_slab && prot->twsk_prot)
3495 kfree(prot->twsk_prot->twsk_slab_name);
8feaf0c0 3496out_free_request_sock_slab:
b45ce321 3497 if (alloc_slab) {
3498 req_prot_cleanup(prot->rsk_prot);
0159dfd3 3499
b45ce321 3500 kmem_cache_destroy(prot->slab);
3501 prot->slab = NULL;
3502 }
b733c007 3503out:
b45ce321 3504 return ret;
1da177e4 3505}
1da177e4
LT		 3506EXPORT_SYMBOL(proto_register);
3507
3508void proto_unregister(struct proto *prot)
3509{
36b77a52 3510 mutex_lock(&proto_list_mutex);
13ff3d6f 3511 release_proto_idx(prot);
0a3f4358 3512 list_del(&prot->node);
36b77a52 3513 mutex_unlock(&proto_list_mutex);
1da177e4 3514
adf78eda
JL		 3515 kmem_cache_destroy(prot->slab);
3516 prot->slab = NULL;
1da177e4 3517
0159dfd3 3518 req_prot_cleanup(prot->rsk_prot);
2e6599cb 3519
6d6ee43e 3520 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
6d6ee43e 3521 kmem_cache_destroy(prot->twsk_prot->twsk_slab);
7e56b5d6 3522 kfree(prot->twsk_prot->twsk_slab_name);
6d6ee43e 3523 prot->twsk_prot->twsk_slab = NULL;
8feaf0c0 3524 }
1da177e4 3525}
1da177e4
LT		 3526EXPORT_SYMBOL(proto_unregister);
3527
bf2ae2e4
XL		 3528int sock_load_diag_module(int family, int protocol)
3529{
3530 if (!protocol) {
3531 if (!sock_is_registered(family))
3532 return -ENOENT;
3533
3534 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
3535 NETLINK_SOCK_DIAG, family);
3536 }
3537
3538#ifdef CONFIG_INET
3539 if (family == AF_INET &&
c34c1287 3540 protocol != IPPROTO_RAW &&
3f935c75 3541 protocol < MAX_INET_PROTOS &&
bf2ae2e4
XL		 3542 !rcu_access_pointer(inet_protos[protocol]))
3543 return -ENOENT;
3544#endif
3545
3546 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
3547 NETLINK_SOCK_DIAG, family, protocol);
3548}
3549EXPORT_SYMBOL(sock_load_diag_module);
3550
1da177e4 3551#ifdef CONFIG_PROC_FS
1da177e4 3552static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
36b77a52 3553 __acquires(proto_list_mutex)
1da177e4 3554{
36b77a52 3555 mutex_lock(&proto_list_mutex);
60f0438a 3556 return seq_list_start_head(&proto_list, *pos);
1da177e4
LT		 3557}
3558
3559static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3560{
60f0438a 3561 return seq_list_next(v, &proto_list, pos);
1da177e4
LT		 3562}
3563
3564static void proto_seq_stop(struct seq_file *seq, void *v)
36b77a52 3565 __releases(proto_list_mutex)
1da177e4 3566{
36b77a52 3567 mutex_unlock(&proto_list_mutex);
1da177e4
LT		 3568}
3569
3570static char proto_method_implemented(const void *method)
3571{
3572 return method == NULL ? 'n' : 'y';
3573}
180d8cd9
GC		 3574static long sock_prot_memory_allocated(struct proto *proto)
3575{
cb75a36c 3576 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
180d8cd9
GC		 3577}
3578
7a512eb8 3579static const char *sock_prot_memory_pressure(struct proto *proto)
180d8cd9
GC		 3580{
3581 return proto->memory_pressure != NULL ?
3582 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3583}
1da177e4
LT		 3584
3585static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3586{
180d8cd9 3587
8d987e5c 3588 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
1da177e4
LT		 3589 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3590 proto->name,
3591 proto->obj_size,
14e943db 3592 sock_prot_inuse_get(seq_file_net(seq), proto),
180d8cd9
GC		 3593 sock_prot_memory_allocated(proto),
3594 sock_prot_memory_pressure(proto),
1da177e4
LT		 3595 proto->max_header,
3596 proto->slab == NULL ? "no" : "yes",
3597 module_name(proto->owner),
3598 proto_method_implemented(proto->close),
3599 proto_method_implemented(proto->connect),
3600 proto_method_implemented(proto->disconnect),
3601 proto_method_implemented(proto->accept),
3602 proto_method_implemented(proto->ioctl),
3603 proto_method_implemented(proto->init),
3604 proto_method_implemented(proto->destroy),
3605 proto_method_implemented(proto->shutdown),
3606 proto_method_implemented(proto->setsockopt),
3607 proto_method_implemented(proto->getsockopt),
3608 proto_method_implemented(proto->sendmsg),
3609 proto_method_implemented(proto->recvmsg),
3610 proto_method_implemented(proto->sendpage),
3611 proto_method_implemented(proto->bind),
3612 proto_method_implemented(proto->backlog_rcv),
3613 proto_method_implemented(proto->hash),
3614 proto_method_implemented(proto->unhash),
3615 proto_method_implemented(proto->get_port),
3616 proto_method_implemented(proto->enter_memory_pressure));
3617}
3618
3619static int proto_seq_show(struct seq_file *seq, void *v)
3620{
60f0438a 3621 if (v == &proto_list)
1da177e4
LT		 3622 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3623 "protocol",
3624 "size",
3625 "sockets",
3626 "memory",
3627 "press",
3628 "maxhdr",
3629 "slab",
3630 "module",
3631 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3632 else
60f0438a 3633 proto_seq_printf(seq, list_entry(v, struct proto, node));
1da177e4
LT		 3634 return 0;
3635}
3636
f690808e 3637static const struct seq_operations proto_seq_ops = {
1da177e4
LT		 3638 .start = proto_seq_start,
3639 .next = proto_seq_next,
3640 .stop = proto_seq_stop,
3641 .show = proto_seq_show,
3642};
3643
14e943db
ED		 3644static __net_init int proto_init_net(struct net *net)
3645{
c3506372
CH		 3646 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
3647 sizeof(struct seq_net_private)))
14e943db
ED		 3648 return -ENOMEM;
3649
3650 return 0;
3651}
3652
3653static __net_exit void proto_exit_net(struct net *net)
3654{
ece31ffd 3655 remove_proc_entry("protocols", net->proc_net);
14e943db
ED		 3656}
3657
3658
3659static __net_initdata struct pernet_operations proto_net_ops = {
3660 .init = proto_init_net,
3661 .exit = proto_exit_net,
1da177e4
LT		 3662};
3663
3664static int __init proto_init(void)
3665{
14e943db 3666 return register_pernet_subsys(&proto_net_ops);
1da177e4
LT		 3667}
3668
3669subsys_initcall(proto_init);
3670
3671#endif /* PROC_FS */
7db6b048
SS		 3672
3673#ifdef CONFIG_NET_RX_BUSY_POLL
3674bool sk_busy_loop_end(void *p, unsigned long start_time)
3675{
3676 struct sock *sk = p;
3677
3f926af3 3678 return !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
7db6b048
SS		 3679 sk_busy_loop_timeout(sk, start_time);
3680}
3681EXPORT_SYMBOL(sk_busy_loop_end);
3682#endif /* CONFIG_NET_RX_BUSY_POLL */
c0425a42
CH		 3683
3684int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len)
3685{
3686 if (!sk->sk_prot->bind_add)
3687 return -EOPNOTSUPP;
3688 return sk->sk_prot->bind_add(sk, addr, addr_len);
3689}
3690EXPORT_SYMBOL(sock_bind_add);
Linux 6.x block layer and io_uring tree(s)