projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 's390-qeth-next'
[linux-2.6-block.git] / net / core / sock.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic socket support routines. Memory allocators, socket lock/release
7 * handler for protocols to use and generic option handler.
8 *
9 *
02c30a84 10 * Authors: Ross Biro
1da177e4
LT		 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
4ec93edb 35 * code. The ACK stuff can wait and needs major
1da177e4
LT		 36 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
84 *
85 *
86 * This program is free software; you can redistribute it and/or
87 * modify it under the terms of the GNU General Public License
88 * as published by the Free Software Foundation; either version
89 * 2 of the License, or (at your option) any later version.
90 */
91
e005d193
JP		 92#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
93
80b14dee 94#include <asm/unaligned.h>
4fc268d2 95#include <linux/capability.h>
1da177e4 96#include <linux/errno.h>
cb820f8e 97#include <linux/errqueue.h>
1da177e4
LT		 98#include <linux/types.h>
99#include <linux/socket.h>
100#include <linux/in.h>
101#include <linux/kernel.h>
1da177e4
LT		 102#include <linux/module.h>
103#include <linux/proc_fs.h>
104#include <linux/seq_file.h>
105#include <linux/sched.h>
f1083048 106#include <linux/sched/mm.h>
1da177e4
LT		 107#include <linux/timer.h>
108#include <linux/string.h>
109#include <linux/sockios.h>
110#include <linux/net.h>
111#include <linux/mm.h>
112#include <linux/slab.h>
113#include <linux/interrupt.h>
114#include <linux/poll.h>
115#include <linux/tcp.h>
116#include <linux/init.h>
a1f8e7f7 117#include <linux/highmem.h>
3f551f94 118#include <linux/user_namespace.h>
c5905afb 119#include <linux/static_key.h>
3969eb38 120#include <linux/memcontrol.h>
8c1ae10d 121#include <linux/prefetch.h>
1da177e4 122
7c0f6ba6 123#include <linux/uaccess.h>
1da177e4
LT		 124
125#include <linux/netdevice.h>
126#include <net/protocol.h>
127#include <linux/skbuff.h>
457c4cbc 128#include <net/net_namespace.h>
2e6599cb 129#include <net/request_sock.h>
1da177e4 130#include <net/sock.h>
20d49473 131#include <linux/net_tstamp.h>
1da177e4
LT		 132#include <net/xfrm.h>
133#include <linux/ipsec.h>
f8451725 134#include <net/cls_cgroup.h>
5bc1421e 135#include <net/netprio_cgroup.h>
eb4cb008 136#include <linux/sock_diag.h>
1da177e4
LT		 137
138#include <linux/filter.h>
538950a1 139#include <net/sock_reuseport.h>
1da177e4 140
3847ce32
SM		 141#include <trace/events/sock.h>
142
1da177e4 143#include <net/tcp.h>
076bb0c8 144#include <net/busy_poll.h>
06021292 145
36b77a52 146static DEFINE_MUTEX(proto_list_mutex);
d1a4c0b3
GC		 147static LIST_HEAD(proto_list);
148
648845ab
TZ		 149static void sock_inuse_add(struct net *net, int val);
150
a3b299da
EB		 151/**
152 * sk_ns_capable - General socket capability test
153 * @sk: Socket to use a capability on or through
154 * @user_ns: The user namespace of the capability to use
155 * @cap: The capability to use
156 *
157 * Test to see if the opener of the socket had when the socket was
158 * created and the current process has the capability @cap in the user
159 * namespace @user_ns.
160 */
161bool sk_ns_capable(const struct sock *sk,
162 struct user_namespace *user_ns, int cap)
163{
164 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
165 ns_capable(user_ns, cap);
166}
167EXPORT_SYMBOL(sk_ns_capable);
168
169/**
170 * sk_capable - Socket global capability test
171 * @sk: Socket to use a capability on or through
e793c0f7 172 * @cap: The global capability to use
a3b299da
EB		 173 *
174 * Test to see if the opener of the socket had when the socket was
175 * created and the current process has the capability @cap in all user
176 * namespaces.
177 */
178bool sk_capable(const struct sock *sk, int cap)
179{
180 return sk_ns_capable(sk, &init_user_ns, cap);
181}
182EXPORT_SYMBOL(sk_capable);
183
184/**
185 * sk_net_capable - Network namespace socket capability test
186 * @sk: Socket to use a capability on or through
187 * @cap: The capability to use
188 *
e793c0f7 189 * Test to see if the opener of the socket had when the socket was created
a3b299da
EB		 190 * and the current process has the capability @cap over the network namespace
191 * the socket is a member of.
192 */
193bool sk_net_capable(const struct sock *sk, int cap)
194{
195 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
196}
197EXPORT_SYMBOL(sk_net_capable);
198
da21f24d
IM		 199/*
200 * Each address family might have different locking rules, so we have
cdfbabfb
DH		 201 * one slock key per address family and separate keys for internal and
202 * userspace sockets.
da21f24d 203 */
a5b5bb9a 204static struct lock_class_key af_family_keys[AF_MAX];
cdfbabfb 205static struct lock_class_key af_family_kern_keys[AF_MAX];
a5b5bb9a 206static struct lock_class_key af_family_slock_keys[AF_MAX];
cdfbabfb 207static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
a5b5bb9a 208
a5b5bb9a
IM		 209/*
210 * Make lock validator output more readable. (we pre-construct these
211 * strings build-time, so that runtime initialization of socket
212 * locks is fast):
213 */
cdfbabfb
DH		 214
215#define _sock_locks(x) \
216 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \
217 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \
218 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \
219 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \
220 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \
221 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \
222 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \
223 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \
224 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \
225 x "27" , x "28" , x "AF_CAN" , \
226 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \
227 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
228 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
229 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
68e8b849
BT		 230 x "AF_QIPCRTR", x "AF_SMC" , x "AF_XDP" , \
231 x "AF_MAX"
cdfbabfb 232
36cbd3dc 233static const char *const af_family_key_strings[AF_MAX+1] = {
cdfbabfb 234 _sock_locks("sk_lock-")
a5b5bb9a 235};
36cbd3dc 236static const char *const af_family_slock_key_strings[AF_MAX+1] = {
cdfbabfb 237 _sock_locks("slock-")
a5b5bb9a 238};
36cbd3dc 239static const char *const af_family_clock_key_strings[AF_MAX+1] = {
cdfbabfb
DH		 240 _sock_locks("clock-")
241};
242
243static const char *const af_family_kern_key_strings[AF_MAX+1] = {
244 _sock_locks("k-sk_lock-")
245};
246static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
247 _sock_locks("k-slock-")
248};
249static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
250 _sock_locks("k-clock-")
443aef0e 251};
581319c5 252static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
6b431d50 253 _sock_locks("rlock-")
581319c5
PA		 254};
255static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
6b431d50 256 _sock_locks("wlock-")
581319c5
PA		 257};
258static const char *const af_family_elock_key_strings[AF_MAX+1] = {
6b431d50 259 _sock_locks("elock-")
581319c5 260};
da21f24d
IM		 261
262/*
581319c5 263 * sk_callback_lock and sk queues locking rules are per-address-family,
da21f24d
IM		 264 * so split the lock classes by using a per-AF key:
265 */
266static struct lock_class_key af_callback_keys[AF_MAX];
581319c5
PA		 267static struct lock_class_key af_rlock_keys[AF_MAX];
268static struct lock_class_key af_wlock_keys[AF_MAX];
269static struct lock_class_key af_elock_keys[AF_MAX];
cdfbabfb 270static struct lock_class_key af_kern_callback_keys[AF_MAX];
da21f24d 271
1da177e4 272/* Run time adjustable parameters. */
ab32ea5d 273__u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
6d8ebc8a 274EXPORT_SYMBOL(sysctl_wmem_max);
ab32ea5d 275__u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
6d8ebc8a 276EXPORT_SYMBOL(sysctl_rmem_max);
ab32ea5d
BH		 277__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
278__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
1da177e4 279
25985edc 280/* Maximal space eaten by iovec or ancillary data plus some space */
ab32ea5d 281int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
2a91525c 282EXPORT_SYMBOL(sysctl_optmem_max);
1da177e4 283
b245be1f
WB		 284int sysctl_tstamp_allow_data __read_mostly = 1;
285
a7950ae8
DB		 286DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
287EXPORT_SYMBOL_GPL(memalloc_socks_key);
c93bdd0e 288
7cb02404
MG		 289/**
290 * sk_set_memalloc - sets %SOCK_MEMALLOC
291 * @sk: socket to set it on
292 *
293 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
294 * It's the responsibility of the admin to adjust min_free_kbytes
295 * to meet the requirements
296 */
297void sk_set_memalloc(struct sock *sk)
298{
299 sock_set_flag(sk, SOCK_MEMALLOC);
300 sk->sk_allocation |= __GFP_MEMALLOC;
a7950ae8 301 static_branch_inc(&memalloc_socks_key);
7cb02404
MG		 302}
303EXPORT_SYMBOL_GPL(sk_set_memalloc);
304
305void sk_clear_memalloc(struct sock *sk)
306{
307 sock_reset_flag(sk, SOCK_MEMALLOC);
308 sk->sk_allocation &= ~__GFP_MEMALLOC;
a7950ae8 309 static_branch_dec(&memalloc_socks_key);
c76562b6
MG		 310
311 /*
312 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
5d753610
MG		 313 * progress of swapping. SOCK_MEMALLOC may be cleared while
314 * it has rmem allocations due to the last swapfile being deactivated
315 * but there is a risk that the socket is unusable due to exceeding
316 * the rmem limits. Reclaim the reserves and obey rmem limits again.
c76562b6 317 */
5d753610 318 sk_mem_reclaim(sk);
7cb02404
MG		 319}
320EXPORT_SYMBOL_GPL(sk_clear_memalloc);
321
b4b9e355
MG		 322int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
323{
324 int ret;
f1083048 325 unsigned int noreclaim_flag;
b4b9e355
MG		 326
327 /* these should have been dropped before queueing */
328 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
329
f1083048 330 noreclaim_flag = memalloc_noreclaim_save();
b4b9e355 331 ret = sk->sk_backlog_rcv(sk, skb);
f1083048 332 memalloc_noreclaim_restore(noreclaim_flag);
b4b9e355
MG		 333
334 return ret;
335}
336EXPORT_SYMBOL(__sk_backlog_rcv);
337
1da177e4
LT		 338static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
339{
340 struct timeval tv;
341
342 if (optlen < sizeof(tv))
343 return -EINVAL;
344 if (copy_from_user(&tv, optval, sizeof(tv)))
345 return -EFAULT;
ba78073e
VA		 346 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
347 return -EDOM;
1da177e4 348
ba78073e 349 if (tv.tv_sec < 0) {
6f11df83
AM		 350 static int warned __read_mostly;
351
ba78073e 352 *timeo_p = 0;
50aab54f 353 if (warned < 10 && net_ratelimit()) {
ba78073e 354 warned++;
e005d193
JP		 355 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
356 __func__, current->comm, task_pid_nr(current));
50aab54f 357 }
ba78073e
VA		 358 return 0;
359 }
1da177e4
LT		 360 *timeo_p = MAX_SCHEDULE_TIMEOUT;
361 if (tv.tv_sec == 0 && tv.tv_usec == 0)
362 return 0;
363 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1))
8ccde4c5 364 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ);
1da177e4
LT		 365 return 0;
366}
367
368static void sock_warn_obsolete_bsdism(const char *name)
369{
370 static int warned;
371 static char warncomm[TASK_COMM_LEN];
4ec93edb
YH		 372 if (strcmp(warncomm, current->comm) && warned < 5) {
373 strcpy(warncomm, current->comm);
e005d193
JP		 374 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
375 warncomm, name);
1da177e4
LT		 376 warned++;
377 }
378}
379
080a270f
HFS		 380static bool sock_needs_netstamp(const struct sock *sk)
381{
382 switch (sk->sk_family) {
383 case AF_UNSPEC:
384 case AF_UNIX:
385 return false;
386 default:
387 return true;
388 }
389}
390
08e29af3 391static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
4ec93edb 392{
08e29af3
ED		 393 if (sk->sk_flags & flags) {
394 sk->sk_flags &= ~flags;
080a270f
HFS		 395 if (sock_needs_netstamp(sk) &&
396 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
20d49473 397 net_disable_timestamp();
1da177e4
LT		 398 }
399}
400
401
e6afc8ac 402int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
f0088a50 403{
3b885787
NH		 404 unsigned long flags;
405 struct sk_buff_head *list = &sk->sk_receive_queue;
f0088a50 406
0fd7bac6 407 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
766e9037 408 atomic_inc(&sk->sk_drops);
3847ce32 409 trace_sock_rcvqueue_full(sk, skb);
766e9037 410 return -ENOMEM;
f0088a50
DV		 411 }
412
c76562b6 413 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
766e9037
ED		 414 atomic_inc(&sk->sk_drops);
415 return -ENOBUFS;
3ab224be
HA		 416 }
417
f0088a50
DV		 418 skb->dev = NULL;
419 skb_set_owner_r(skb, sk);
49ad9599 420
7fee226a
ED		 421 /* we escape from rcu protected region, make sure we dont leak
422 * a norefcounted dst
423 */
424 skb_dst_force(skb);
425
3b885787 426 spin_lock_irqsave(&list->lock, flags);
3bc3b96f 427 sock_skb_set_dropcount(sk, skb);
3b885787
NH		 428 __skb_queue_tail(list, skb);
429 spin_unlock_irqrestore(&list->lock, flags);
f0088a50
DV		 430
431 if (!sock_flag(sk, SOCK_DEAD))
676d2369 432 sk->sk_data_ready(sk);
766e9037 433 return 0;
f0088a50 434}
e6afc8ac 435EXPORT_SYMBOL(__sock_queue_rcv_skb);
436
437int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
438{
439 int err;
440
441 err = sk_filter(sk, skb);
442 if (err)
443 return err;
444
445 return __sock_queue_rcv_skb(sk, skb);
446}
f0088a50
DV		 447EXPORT_SYMBOL(sock_queue_rcv_skb);
448
4f0c40d9 449int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
c3f24cfb 450 const int nested, unsigned int trim_cap, bool refcounted)
f0088a50
DV		 451{
452 int rc = NET_RX_SUCCESS;
453
4f0c40d9 454 if (sk_filter_trim_cap(sk, skb, trim_cap))
f0088a50
DV		 455 goto discard_and_relse;
456
457 skb->dev = NULL;
458
274f482d 459 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
c377411f
ED		 460 atomic_inc(&sk->sk_drops);
461 goto discard_and_relse;
462 }
58a5a7b9
ACM		 463 if (nested)
464 bh_lock_sock_nested(sk);
465 else
466 bh_lock_sock(sk);
a5b5bb9a
IM		 467 if (!sock_owned_by_user(sk)) {
468 /*
469 * trylock + unlock semantics:
470 */
471 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
472
c57943a1 473 rc = sk_backlog_rcv(sk, skb);
a5b5bb9a
IM		 474
475 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
f545a38f 476 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
8eae939f
ZY		 477 bh_unlock_sock(sk);
478 atomic_inc(&sk->sk_drops);
479 goto discard_and_relse;
480 }
481
f0088a50
DV		 482 bh_unlock_sock(sk);
483out:
c3f24cfb
ED		 484 if (refcounted)
485 sock_put(sk);
f0088a50
DV		 486 return rc;
487discard_and_relse:
488 kfree_skb(skb);
489 goto out;
490}
4f0c40d9 491EXPORT_SYMBOL(__sk_receive_skb);
f0088a50
DV		 492
493struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
494{
b6c6712a 495 struct dst_entry *dst = __sk_dst_get(sk);
f0088a50
DV		 496
497 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
e022f0b4 498 sk_tx_queue_clear(sk);
9b8805a3 499 sk->sk_dst_pending_confirm = 0;
a9b3cd7f 500 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
f0088a50
DV		 501 dst_release(dst);
502 return NULL;
503 }
504
505 return dst;
506}
507EXPORT_SYMBOL(__sk_dst_check);
508
509struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
510{
511 struct dst_entry *dst = sk_dst_get(sk);
512
513 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
514 sk_dst_reset(sk);
515 dst_release(dst);
516 return NULL;
517 }
518
519 return dst;
520}
521EXPORT_SYMBOL(sk_dst_check);
522
c91f6df2
BH		 523static int sock_setbindtodevice(struct sock *sk, char __user *optval,
524 int optlen)
4878809f
DM		 525{
526 int ret = -ENOPROTOOPT;
527#ifdef CONFIG_NETDEVICES
3b1e0a65 528 struct net *net = sock_net(sk);
4878809f
DM		 529 char devname[IFNAMSIZ];
530 int index;
531
532 /* Sorry... */
533 ret = -EPERM;
5e1fccc0 534 if (!ns_capable(net->user_ns, CAP_NET_RAW))
4878809f
DM		 535 goto out;
536
537 ret = -EINVAL;
538 if (optlen < 0)
539 goto out;
540
541 /* Bind this socket to a particular device like "eth0",
542 * as specified in the passed interface name. If the
543 * name is "" or the option length is zero the socket
544 * is not bound.
545 */
546 if (optlen > IFNAMSIZ - 1)
547 optlen = IFNAMSIZ - 1;
548 memset(devname, 0, sizeof(devname));
549
550 ret = -EFAULT;
551 if (copy_from_user(devname, optval, optlen))
552 goto out;
553
000ba2e4
DM		 554 index = 0;
555 if (devname[0] != '\0') {
bf8e56bf 556 struct net_device *dev;
4878809f 557
bf8e56bf
ED		 558 rcu_read_lock();
559 dev = dev_get_by_name_rcu(net, devname);
560 if (dev)
561 index = dev->ifindex;
562 rcu_read_unlock();
4878809f
DM		 563 ret = -ENODEV;
564 if (!dev)
565 goto out;
4878809f
DM		 566 }
567
568 lock_sock(sk);
569 sk->sk_bound_dev_if = index;
6da5b0f0
MM		 570 if (sk->sk_prot->rehash)
571 sk->sk_prot->rehash(sk);
4878809f
DM		 572 sk_dst_reset(sk);
573 release_sock(sk);
574
575 ret = 0;
576
577out:
578#endif
579
580 return ret;
581}
582
c91f6df2
BH		 583static int sock_getbindtodevice(struct sock *sk, char __user *optval,
584 int __user *optlen, int len)
585{
586 int ret = -ENOPROTOOPT;
587#ifdef CONFIG_NETDEVICES
588 struct net *net = sock_net(sk);
c91f6df2 589 char devname[IFNAMSIZ];
c91f6df2
BH		 590
591 if (sk->sk_bound_dev_if == 0) {
592 len = 0;
593 goto zero;
594 }
595
596 ret = -EINVAL;
597 if (len < IFNAMSIZ)
598 goto out;
599
5dbe7c17
NS		 600 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
601 if (ret)
c91f6df2 602 goto out;
c91f6df2
BH		 603
604 len = strlen(devname) + 1;
605
606 ret = -EFAULT;
607 if (copy_to_user(optval, devname, len))
608 goto out;
609
610zero:
611 ret = -EFAULT;
612 if (put_user(len, optlen))
613 goto out;
614
615 ret = 0;
616
617out:
618#endif
619
620 return ret;
621}
622
c0ef877b
PE		 623static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool)
624{
625 if (valbool)
626 sock_set_flag(sk, bit);
627 else
628 sock_reset_flag(sk, bit);
629}
630
f60e5990 631bool sk_mc_loop(struct sock *sk)
632{
633 if (dev_recursion_level())
634 return false;
635 if (!sk)
636 return true;
637 switch (sk->sk_family) {
638 case AF_INET:
639 return inet_sk(sk)->mc_loop;
640#if IS_ENABLED(CONFIG_IPV6)
641 case AF_INET6:
642 return inet6_sk(sk)->mc_loop;
643#endif
644 }
645 WARN_ON(1);
646 return true;
647}
648EXPORT_SYMBOL(sk_mc_loop);
649
1da177e4
LT		 650/*
651 * This is meant for all protocols to use and covers goings on
652 * at the socket level. Everything here is generic.
653 */
654
655int sock_setsockopt(struct socket *sock, int level, int optname,
b7058842 656 char __user *optval, unsigned int optlen)
1da177e4 657{
80b14dee 658 struct sock_txtime sk_txtime;
2a91525c 659 struct sock *sk = sock->sk;
1da177e4
LT		 660 int val;
661 int valbool;
662 struct linger ling;
663 int ret = 0;
4ec93edb 664
1da177e4
LT		 665 /*
666 * Options without arguments
667 */
668
4878809f 669 if (optname == SO_BINDTODEVICE)
c91f6df2 670 return sock_setbindtodevice(sk, optval, optlen);
4878809f 671
e71a4783
SH		 672 if (optlen < sizeof(int))
673 return -EINVAL;
4ec93edb 674
1da177e4
LT		 675 if (get_user(val, (int __user *)optval))
676 return -EFAULT;
4ec93edb 677
2a91525c 678 valbool = val ? 1 : 0;
1da177e4
LT		 679
680 lock_sock(sk);
681
2a91525c 682 switch (optname) {
e71a4783 683 case SO_DEBUG:
2a91525c 684 if (val && !capable(CAP_NET_ADMIN))
e71a4783 685 ret = -EACCES;
2a91525c 686 else
c0ef877b 687 sock_valbool_flag(sk, SOCK_DBG, valbool);
e71a4783
SH		 688 break;
689 case SO_REUSEADDR:
cdb8744d 690 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
e71a4783 691 break;
055dc21a
TH		 692 case SO_REUSEPORT:
693 sk->sk_reuseport = valbool;
694 break;
e71a4783 695 case SO_TYPE:
49c794e9 696 case SO_PROTOCOL:
0d6038ee 697 case SO_DOMAIN:
e71a4783
SH		 698 case SO_ERROR:
699 ret = -ENOPROTOOPT;
700 break;
701 case SO_DONTROUTE:
c0ef877b 702 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
e71a4783
SH		 703 break;
704 case SO_BROADCAST:
705 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
706 break;
707 case SO_SNDBUF:
708 /* Don't error on this BSD doesn't and if you think
82981930
ED		 709 * about it this is right. Otherwise apps have to
710 * play 'guess the biggest size' games. RCVBUF/SNDBUF
711 * are treated in BSD as hints
712 */
713 val = min_t(u32, val, sysctl_wmem_max);
b0573dea 714set_sndbuf:
e71a4783 715 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
b98b0bc8 716 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);
82981930 717 /* Wake up sending tasks if we upped the value. */
e71a4783
SH		 718 sk->sk_write_space(sk);
719 break;
1da177e4 720
e71a4783
SH		 721 case SO_SNDBUFFORCE:
722 if (!capable(CAP_NET_ADMIN)) {
723 ret = -EPERM;
724 break;
725 }
726 goto set_sndbuf;
b0573dea 727
e71a4783
SH		 728 case SO_RCVBUF:
729 /* Don't error on this BSD doesn't and if you think
82981930
ED		 730 * about it this is right. Otherwise apps have to
731 * play 'guess the biggest size' games. RCVBUF/SNDBUF
732 * are treated in BSD as hints
733 */
734 val = min_t(u32, val, sysctl_rmem_max);
b0573dea 735set_rcvbuf:
e71a4783
SH		 736 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
737 /*
738 * We double it on the way in to account for
739 * "struct sk_buff" etc. overhead. Applications
740 * assume that the SO_RCVBUF setting they make will
741 * allow that much actual data to be received on that
742 * socket.
743 *
744 * Applications are unaware that "struct sk_buff" and
745 * other overheads allocate from the receive buffer
746 * during socket buffer allocation.
747 *
748 * And after considering the possible alternatives,
749 * returning the value we actually used in getsockopt
750 * is the most desirable behavior.
751 */
b98b0bc8 752 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF);
e71a4783
SH		 753 break;
754
755 case SO_RCVBUFFORCE:
756 if (!capable(CAP_NET_ADMIN)) {
757 ret = -EPERM;
1da177e4 758 break;
e71a4783
SH		 759 }
760 goto set_rcvbuf;
1da177e4 761
e71a4783 762 case SO_KEEPALIVE:
4b9d07a4
UB		 763 if (sk->sk_prot->keepalive)
764 sk->sk_prot->keepalive(sk, valbool);
e71a4783
SH		 765 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
766 break;
767
768 case SO_OOBINLINE:
769 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
770 break;
771
772 case SO_NO_CHECK:
28448b80 773 sk->sk_no_check_tx = valbool;
e71a4783
SH		 774 break;
775
776 case SO_PRIORITY:
5e1fccc0
EB		 777 if ((val >= 0 && val <= 6) ||
778 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
e71a4783
SH		 779 sk->sk_priority = val;
780 else
781 ret = -EPERM;
782 break;
783
784 case SO_LINGER:
785 if (optlen < sizeof(ling)) {
786 ret = -EINVAL; /* 1003.1g */
1da177e4 787 break;
e71a4783 788 }
2a91525c 789 if (copy_from_user(&ling, optval, sizeof(ling))) {
e71a4783 790 ret = -EFAULT;
1da177e4 791 break;
e71a4783
SH		 792 }
793 if (!ling.l_onoff)
794 sock_reset_flag(sk, SOCK_LINGER);
795 else {
1da177e4 796#if (BITS_PER_LONG == 32)
e71a4783
SH		 797 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
798 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
1da177e4 799 else
e71a4783
SH		 800#endif
801 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
802 sock_set_flag(sk, SOCK_LINGER);
803 }
804 break;
805
806 case SO_BSDCOMPAT:
807 sock_warn_obsolete_bsdism("setsockopt");
808 break;
809
810 case SO_PASSCRED:
811 if (valbool)
812 set_bit(SOCK_PASSCRED, &sock->flags);
813 else
814 clear_bit(SOCK_PASSCRED, &sock->flags);
815 break;
816
817 case SO_TIMESTAMP:
92f37fd2 818 case SO_TIMESTAMPNS:
e71a4783 819 if (valbool) {
92f37fd2
ED		 820 if (optname == SO_TIMESTAMP)
821 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
822 else
823 sock_set_flag(sk, SOCK_RCVTSTAMPNS);
e71a4783 824 sock_set_flag(sk, SOCK_RCVTSTAMP);
20d49473 825 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
92f37fd2 826 } else {
e71a4783 827 sock_reset_flag(sk, SOCK_RCVTSTAMP);
92f37fd2
ED		 828 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
829 }
e71a4783
SH		 830 break;
831
20d49473
PO		 832 case SO_TIMESTAMPING:
833 if (val & ~SOF_TIMESTAMPING_MASK) {
f249fb78 834 ret = -EINVAL;
20d49473
PO		 835 break;
836 }
b245be1f 837
09c2d251 838 if (val & SOF_TIMESTAMPING_OPT_ID &&
4ed2d765 839 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
ac5cc977
WC		 840 if (sk->sk_protocol == IPPROTO_TCP &&
841 sk->sk_type == SOCK_STREAM) {
6db8b963
SHY		 842 if ((1 << sk->sk_state) &
843 (TCPF_CLOSE | TCPF_LISTEN)) {
4ed2d765
WB		 844 ret = -EINVAL;
845 break;
846 }
847 sk->sk_tskey = tcp_sk(sk)->snd_una;
848 } else {
849 sk->sk_tskey = 0;
850 }
851 }
1c885808
FY		 852
853 if (val & SOF_TIMESTAMPING_OPT_STATS &&
854 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
855 ret = -EINVAL;
856 break;
857 }
858
b9f40e21 859 sk->sk_tsflags = val;
20d49473
PO		 860 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
861 sock_enable_timestamp(sk,
862 SOCK_TIMESTAMPING_RX_SOFTWARE);
863 else
864 sock_disable_timestamp(sk,
08e29af3 865 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
20d49473
PO		 866 break;
867
e71a4783
SH		 868 case SO_RCVLOWAT:
869 if (val < 0)
870 val = INT_MAX;
d1361840
ED		 871 if (sock->ops->set_rcvlowat)
872 ret = sock->ops->set_rcvlowat(sk, val);
873 else
874 sk->sk_rcvlowat = val ? : 1;
e71a4783
SH		 875 break;
876
877 case SO_RCVTIMEO:
878 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen);
879 break;
880
881 case SO_SNDTIMEO:
882 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen);
883 break;
1da177e4 884
e71a4783
SH		 885 case SO_ATTACH_FILTER:
886 ret = -EINVAL;
887 if (optlen == sizeof(struct sock_fprog)) {
888 struct sock_fprog fprog;
1da177e4 889
e71a4783
SH		 890 ret = -EFAULT;
891 if (copy_from_user(&fprog, optval, sizeof(fprog)))
1da177e4 892 break;
e71a4783
SH		 893
894 ret = sk_attach_filter(&fprog, sk);
895 }
896 break;
897
89aa0758
AS		 898 case SO_ATTACH_BPF:
899 ret = -EINVAL;
900 if (optlen == sizeof(u32)) {
901 u32 ufd;
902
903 ret = -EFAULT;
904 if (copy_from_user(&ufd, optval, sizeof(ufd)))
905 break;
906
907 ret = sk_attach_bpf(ufd, sk);
908 }
909 break;
910
538950a1
CG		 911 case SO_ATTACH_REUSEPORT_CBPF:
912 ret = -EINVAL;
913 if (optlen == sizeof(struct sock_fprog)) {
914 struct sock_fprog fprog;
915
916 ret = -EFAULT;
917 if (copy_from_user(&fprog, optval, sizeof(fprog)))
918 break;
919
920 ret = sk_reuseport_attach_filter(&fprog, sk);
921 }
922 break;
923
924 case SO_ATTACH_REUSEPORT_EBPF:
925 ret = -EINVAL;
926 if (optlen == sizeof(u32)) {
927 u32 ufd;
928
929 ret = -EFAULT;
930 if (copy_from_user(&ufd, optval, sizeof(ufd)))
931 break;
932
933 ret = sk_reuseport_attach_bpf(ufd, sk);
934 }
935 break;
936
e71a4783 937 case SO_DETACH_FILTER:
55b33325 938 ret = sk_detach_filter(sk);
e71a4783 939 break;
1da177e4 940
d59577b6
VB		 941 case SO_LOCK_FILTER:
942 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
943 ret = -EPERM;
944 else
945 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
946 break;
947
e71a4783
SH		 948 case SO_PASSSEC:
949 if (valbool)
950 set_bit(SOCK_PASSSEC, &sock->flags);
951 else
952 clear_bit(SOCK_PASSSEC, &sock->flags);
953 break;
4a19ec58 954 case SO_MARK:
5e1fccc0 955 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
4a19ec58 956 ret = -EPERM;
2a91525c 957 else
4a19ec58 958 sk->sk_mark = val;
4a19ec58 959 break;
877ce7c1 960
3b885787 961 case SO_RXQ_OVFL:
8083f0fc 962 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
3b885787 963 break;
6e3e939f
JB		 964
965 case SO_WIFI_STATUS:
966 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
967 break;
968
ef64a54f
PE		 969 case SO_PEEK_OFF:
970 if (sock->ops->set_peek_off)
12663bfc 971 ret = sock->ops->set_peek_off(sk, val);
ef64a54f
PE		 972 else
973 ret = -EOPNOTSUPP;
974 break;
3bdc0eba
BG		 975
976 case SO_NOFCS:
977 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
978 break;
979
7d4c04fc
KJ		 980 case SO_SELECT_ERR_QUEUE:
981 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
982 break;
983
e0d1095a 984#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 985 case SO_BUSY_POLL:
dafcc438
ET		 986 /* allow unprivileged users to decrease the value */
987 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
988 ret = -EPERM;
989 else {
990 if (val < 0)
991 ret = -EINVAL;
992 else
993 sk->sk_ll_usec = val;
994 }
995 break;
996#endif
62748f32
ED		 997
998 case SO_MAX_PACING_RATE:
218af599
ED		 999 if (val != ~0U)
1000 cmpxchg(&sk->sk_pacing_status,
1001 SK_PACING_NONE,
1002 SK_PACING_NEEDED);
76a9ebe8 1003 sk->sk_max_pacing_rate = (val == ~0U) ? ~0UL : val;
62748f32
ED		 1004 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
1005 sk->sk_max_pacing_rate);
1006 break;
1007
70da268b
ED		 1008 case SO_INCOMING_CPU:
1009 sk->sk_incoming_cpu = val;
1010 break;
1011
a87cb3e4
TH		 1012 case SO_CNX_ADVICE:
1013 if (val == 1)
1014 dst_negative_advice(sk);
1015 break;
76851d12
WB		 1016
1017 case SO_ZEROCOPY:
28190752
SV		 1018 if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
1019 if (sk->sk_protocol != IPPROTO_TCP)
1020 ret = -ENOTSUPP;
28190752 1021 } else if (sk->sk_family != PF_RDS) {
76851d12 1022 ret = -ENOTSUPP;
28190752
SV		 1023 }
1024 if (!ret) {
1025 if (val < 0 || val > 1)
1026 ret = -EINVAL;
1027 else
1028 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
28190752 1029 }
334e6413
JSP		 1030 break;
1031
80b14dee
RC		 1032 case SO_TXTIME:
1033 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1034 ret = -EPERM;
1035 } else if (optlen != sizeof(struct sock_txtime)) {
1036 ret = -EINVAL;
1037 } else if (copy_from_user(&sk_txtime, optval,
1038 sizeof(struct sock_txtime))) {
1039 ret = -EFAULT;
1040 } else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
1041 ret = -EINVAL;
1042 } else {
1043 sock_valbool_flag(sk, SOCK_TXTIME, true);
1044 sk->sk_clockid = sk_txtime.clockid;
1045 sk->sk_txtime_deadline_mode =
1046 !!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
4b15c707
JSP		 1047 sk->sk_txtime_report_errors =
1048 !!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
80b14dee
RC		 1049 }
1050 break;
1051
e71a4783
SH		 1052 default:
1053 ret = -ENOPROTOOPT;
1054 break;
4ec93edb 1055 }
1da177e4
LT		 1056 release_sock(sk);
1057 return ret;
1058}
2a91525c 1059EXPORT_SYMBOL(sock_setsockopt);
1da177e4
LT		 1060
1061
8f09898b 1062static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1063 struct ucred *ucred)
3f551f94
EB		 1064{
1065 ucred->pid = pid_vnr(pid);
1066 ucred->uid = ucred->gid = -1;
1067 if (cred) {
1068 struct user_namespace *current_ns = current_user_ns();
1069
b2e4f544
EB		 1070 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1071 ucred->gid = from_kgid_munged(current_ns, cred->egid);
3f551f94
EB		 1072 }
1073}
1074
28b5ba2a
DH		 1075static int groups_to_user(gid_t __user *dst, const struct group_info *src)
1076{
1077 struct user_namespace *user_ns = current_user_ns();
1078 int i;
1079
1080 for (i = 0; i < src->ngroups; i++)
1081 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
1082 return -EFAULT;
1083
1084 return 0;
1085}
1086
1da177e4
LT		 1087int sock_getsockopt(struct socket *sock, int level, int optname,
1088 char __user *optval, int __user *optlen)
1089{
1090 struct sock *sk = sock->sk;
4ec93edb 1091
e71a4783 1092 union {
4ec93edb 1093 int val;
5daab9db 1094 u64 val64;
4ec93edb 1095 struct linger ling;
1da177e4 1096 struct timeval tm;
80b14dee 1097 struct sock_txtime txtime;
1da177e4 1098 } v;
4ec93edb 1099
4d0392be 1100 int lv = sizeof(int);
1da177e4 1101 int len;
4ec93edb 1102
e71a4783 1103 if (get_user(len, optlen))
4ec93edb 1104 return -EFAULT;
e71a4783 1105 if (len < 0)
1da177e4 1106 return -EINVAL;
4ec93edb 1107
50fee1de 1108 memset(&v, 0, sizeof(v));
df0bca04 1109
2a91525c 1110 switch (optname) {
e71a4783
SH		 1111 case SO_DEBUG:
1112 v.val = sock_flag(sk, SOCK_DBG);
1113 break;
1114
1115 case SO_DONTROUTE:
1116 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1117 break;
1118
1119 case SO_BROADCAST:
1b23a5df 1120 v.val = sock_flag(sk, SOCK_BROADCAST);
e71a4783
SH		 1121 break;
1122
1123 case SO_SNDBUF:
1124 v.val = sk->sk_sndbuf;
1125 break;
1126
1127 case SO_RCVBUF:
1128 v.val = sk->sk_rcvbuf;
1129 break;
1130
1131 case SO_REUSEADDR:
1132 v.val = sk->sk_reuse;
1133 break;
1134
055dc21a
TH		 1135 case SO_REUSEPORT:
1136 v.val = sk->sk_reuseport;
1137 break;
1138
e71a4783 1139 case SO_KEEPALIVE:
1b23a5df 1140 v.val = sock_flag(sk, SOCK_KEEPOPEN);
e71a4783
SH		 1141 break;
1142
1143 case SO_TYPE:
1144 v.val = sk->sk_type;
1145 break;
1146
49c794e9
JE		 1147 case SO_PROTOCOL:
1148 v.val = sk->sk_protocol;
1149 break;
1150
0d6038ee
JE		 1151 case SO_DOMAIN:
1152 v.val = sk->sk_family;
1153 break;
1154
e71a4783
SH		 1155 case SO_ERROR:
1156 v.val = -sock_error(sk);
2a91525c 1157 if (v.val == 0)
e71a4783
SH		 1158 v.val = xchg(&sk->sk_err_soft, 0);
1159 break;
1160
1161 case SO_OOBINLINE:
1b23a5df 1162 v.val = sock_flag(sk, SOCK_URGINLINE);
e71a4783
SH		 1163 break;
1164
1165 case SO_NO_CHECK:
28448b80 1166 v.val = sk->sk_no_check_tx;
e71a4783
SH		 1167 break;
1168
1169 case SO_PRIORITY:
1170 v.val = sk->sk_priority;
1171 break;
1172
1173 case SO_LINGER:
1174 lv = sizeof(v.ling);
1b23a5df 1175 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
e71a4783
SH		 1176 v.ling.l_linger = sk->sk_lingertime / HZ;
1177 break;
1178
1179 case SO_BSDCOMPAT:
1180 sock_warn_obsolete_bsdism("getsockopt");
1181 break;
1182
1183 case SO_TIMESTAMP:
92f37fd2
ED		 1184 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1185 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1186 break;
1187
1188 case SO_TIMESTAMPNS:
1189 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
e71a4783
SH		 1190 break;
1191
20d49473 1192 case SO_TIMESTAMPING:
b9f40e21 1193 v.val = sk->sk_tsflags;
20d49473
PO		 1194 break;
1195
e71a4783 1196 case SO_RCVTIMEO:
2a91525c 1197 lv = sizeof(struct timeval);
e71a4783
SH		 1198 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
1199 v.tm.tv_sec = 0;
1200 v.tm.tv_usec = 0;
1201 } else {
1202 v.tm.tv_sec = sk->sk_rcvtimeo / HZ;
8ccde4c5 1203 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ;
e71a4783
SH		 1204 }
1205 break;
1206
1207 case SO_SNDTIMEO:
2a91525c 1208 lv = sizeof(struct timeval);
e71a4783
SH		 1209 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) {
1210 v.tm.tv_sec = 0;
1211 v.tm.tv_usec = 0;
1212 } else {
1213 v.tm.tv_sec = sk->sk_sndtimeo / HZ;
8ccde4c5 1214 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ;
e71a4783
SH		 1215 }
1216 break;
1da177e4 1217
e71a4783
SH		 1218 case SO_RCVLOWAT:
1219 v.val = sk->sk_rcvlowat;
1220 break;
1da177e4 1221
e71a4783 1222 case SO_SNDLOWAT:
2a91525c 1223 v.val = 1;
e71a4783 1224 break;
1da177e4 1225
e71a4783 1226 case SO_PASSCRED:
82981930 1227 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
e71a4783 1228 break;
1da177e4 1229
e71a4783 1230 case SO_PEERCRED:
109f6e39
EB		 1231 {
1232 struct ucred peercred;
1233 if (len > sizeof(peercred))
1234 len = sizeof(peercred);
1235 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1236 if (copy_to_user(optval, &peercred, len))
e71a4783
SH		 1237 return -EFAULT;
1238 goto lenout;
109f6e39 1239 }
1da177e4 1240
28b5ba2a
DH		 1241 case SO_PEERGROUPS:
1242 {
1243 int ret, n;
1244
1245 if (!sk->sk_peer_cred)
1246 return -ENODATA;
1247
1248 n = sk->sk_peer_cred->group_info->ngroups;
1249 if (len < n * sizeof(gid_t)) {
1250 len = n * sizeof(gid_t);
1251 return put_user(len, optlen) ? -EFAULT : -ERANGE;
1252 }
1253 len = n * sizeof(gid_t);
1254
1255 ret = groups_to_user((gid_t __user *)optval,
1256 sk->sk_peer_cred->group_info);
1257 if (ret)
1258 return ret;
1259 goto lenout;
1260 }
1261
e71a4783
SH		 1262 case SO_PEERNAME:
1263 {
1264 char address[128];
1265
9b2c45d4
DV		 1266 lv = sock->ops->getname(sock, (struct sockaddr *)address, 2);
1267 if (lv < 0)
e71a4783
SH		 1268 return -ENOTCONN;
1269 if (lv < len)
1270 return -EINVAL;
1271 if (copy_to_user(optval, address, len))
1272 return -EFAULT;
1273 goto lenout;
1274 }
1da177e4 1275
e71a4783
SH		 1276 /* Dubious BSD thing... Probably nobody even uses it, but
1277 * the UNIX standard wants it for whatever reason... -DaveM
1278 */
1279 case SO_ACCEPTCONN:
1280 v.val = sk->sk_state == TCP_LISTEN;
1281 break;
1da177e4 1282
e71a4783 1283 case SO_PASSSEC:
82981930 1284 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
e71a4783 1285 break;
877ce7c1 1286
e71a4783
SH		 1287 case SO_PEERSEC:
1288 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1da177e4 1289
4a19ec58
LAT		 1290 case SO_MARK:
1291 v.val = sk->sk_mark;
1292 break;
1293
3b885787 1294 case SO_RXQ_OVFL:
1b23a5df 1295 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
3b885787
NH		 1296 break;
1297
6e3e939f 1298 case SO_WIFI_STATUS:
1b23a5df 1299 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
6e3e939f
JB		 1300 break;
1301
ef64a54f
PE		 1302 case SO_PEEK_OFF:
1303 if (!sock->ops->set_peek_off)
1304 return -EOPNOTSUPP;
1305
1306 v.val = sk->sk_peek_off;
1307 break;
bc2f7996 1308 case SO_NOFCS:
1b23a5df 1309 v.val = sock_flag(sk, SOCK_NOFCS);
bc2f7996 1310 break;
c91f6df2 1311
f7b86bfe 1312 case SO_BINDTODEVICE:
c91f6df2
BH		 1313 return sock_getbindtodevice(sk, optval, optlen, len);
1314
a8fc9277
PE		 1315 case SO_GET_FILTER:
1316 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1317 if (len < 0)
1318 return len;
1319
1320 goto lenout;
c91f6df2 1321
d59577b6
VB		 1322 case SO_LOCK_FILTER:
1323 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1324 break;
1325
ea02f941
MS		 1326 case SO_BPF_EXTENSIONS:
1327 v.val = bpf_tell_extensions();
1328 break;
1329
7d4c04fc
KJ		 1330 case SO_SELECT_ERR_QUEUE:
1331 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1332 break;
1333
e0d1095a 1334#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1335 case SO_BUSY_POLL:
dafcc438
ET		 1336 v.val = sk->sk_ll_usec;
1337 break;
1338#endif
1339
62748f32 1340 case SO_MAX_PACING_RATE:
76a9ebe8
ED		 1341 /* 32bit version */
1342 v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);
62748f32
ED		 1343 break;
1344
2c8c56e1
ED		 1345 case SO_INCOMING_CPU:
1346 v.val = sk->sk_incoming_cpu;
1347 break;
1348
a2d133b1
JH		 1349 case SO_MEMINFO:
1350 {
1351 u32 meminfo[SK_MEMINFO_VARS];
1352
1353 if (get_user(len, optlen))
1354 return -EFAULT;
1355
1356 sk_get_meminfo(sk, meminfo);
1357
1358 len = min_t(unsigned int, len, sizeof(meminfo));
1359 if (copy_to_user(optval, &meminfo, len))
1360 return -EFAULT;
1361
1362 goto lenout;
1363 }
6d433902
SS		 1364
1365#ifdef CONFIG_NET_RX_BUSY_POLL
1366 case SO_INCOMING_NAPI_ID:
1367 v.val = READ_ONCE(sk->sk_napi_id);
1368
1369 /* aggregate non-NAPI IDs down to 0 */
1370 if (v.val < MIN_NAPI_ID)
1371 v.val = 0;
1372
1373 break;
1374#endif
1375
5daab9db
CF		 1376 case SO_COOKIE:
1377 lv = sizeof(u64);
1378 if (len < lv)
1379 return -EINVAL;
1380 v.val64 = sock_gen_cookie(sk);
1381 break;
1382
76851d12
WB		 1383 case SO_ZEROCOPY:
1384 v.val = sock_flag(sk, SOCK_ZEROCOPY);
1385 break;
1386
80b14dee
RC		 1387 case SO_TXTIME:
1388 lv = sizeof(v.txtime);
1389 v.txtime.clockid = sk->sk_clockid;
1390 v.txtime.flags |= sk->sk_txtime_deadline_mode ?
1391 SOF_TXTIME_DEADLINE_MODE : 0;
4b15c707
JSP		 1392 v.txtime.flags |= sk->sk_txtime_report_errors ?
1393 SOF_TXTIME_REPORT_ERRORS : 0;
80b14dee
RC		 1394 break;
1395
e71a4783 1396 default:
443b5991
YH		 1397 /* We implement the SO_SNDLOWAT etc to not be settable
1398 * (1003.1g 7).
1399 */
e71a4783 1400 return -ENOPROTOOPT;
1da177e4 1401 }
e71a4783 1402
1da177e4
LT		 1403 if (len > lv)
1404 len = lv;
1405 if (copy_to_user(optval, &v, len))
1406 return -EFAULT;
1407lenout:
4ec93edb
YH		 1408 if (put_user(len, optlen))
1409 return -EFAULT;
1410 return 0;
1da177e4
LT		 1411}
1412
a5b5bb9a
IM		 1413/*
1414 * Initialize an sk_lock.
1415 *
1416 * (We also register the sk_lock with the lock validator.)
1417 */
b6f99a21 1418static inline void sock_lock_init(struct sock *sk)
a5b5bb9a 1419{
cdfbabfb
DH		 1420 if (sk->sk_kern_sock)
1421 sock_lock_init_class_and_name(
1422 sk,
1423 af_family_kern_slock_key_strings[sk->sk_family],
1424 af_family_kern_slock_keys + sk->sk_family,
1425 af_family_kern_key_strings[sk->sk_family],
1426 af_family_kern_keys + sk->sk_family);
1427 else
1428 sock_lock_init_class_and_name(
1429 sk,
ed07536e
PZ		 1430 af_family_slock_key_strings[sk->sk_family],
1431 af_family_slock_keys + sk->sk_family,
1432 af_family_key_strings[sk->sk_family],
1433 af_family_keys + sk->sk_family);
a5b5bb9a
IM		 1434}
1435
4dc6dc71
ED		 1436/*
1437 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1438 * even temporarly, because of RCU lookups. sk_node should also be left as is.
68835aba 1439 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
4dc6dc71 1440 */
f1a6c4da
PE		 1441static void sock_copy(struct sock *nsk, const struct sock *osk)
1442{
1443#ifdef CONFIG_SECURITY_NETWORK
1444 void *sptr = nsk->sk_security;
1445#endif
68835aba
ED		 1446 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1447
1448 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1449 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1450
f1a6c4da
PE		 1451#ifdef CONFIG_SECURITY_NETWORK
1452 nsk->sk_security = sptr;
1453 security_sk_clone(osk, nsk);
1454#endif
1455}
1456
2e4afe7b
PE		 1457static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1458 int family)
c308c1b2
PE		 1459{
1460 struct sock *sk;
1461 struct kmem_cache *slab;
1462
1463 slab = prot->slab;
e912b114
ED		 1464 if (slab != NULL) {
1465 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1466 if (!sk)
1467 return sk;
ba2489b0
ED		 1468 if (priority & __GFP_ZERO)
1469 sk_prot_clear_nulls(sk, prot->obj_size);
fcbdf09d 1470 } else
c308c1b2
PE		 1471 sk = kmalloc(prot->obj_size, priority);
1472
2e4afe7b
PE		 1473 if (sk != NULL) {
1474 if (security_sk_alloc(sk, family, priority))
1475 goto out_free;
1476
1477 if (!try_module_get(prot->owner))
1478 goto out_free_sec;
e022f0b4 1479 sk_tx_queue_clear(sk);
2e4afe7b
PE		 1480 }
1481
c308c1b2 1482 return sk;
2e4afe7b
PE		 1483
1484out_free_sec:
1485 security_sk_free(sk);
1486out_free:
1487 if (slab != NULL)
1488 kmem_cache_free(slab, sk);
1489 else
1490 kfree(sk);
1491 return NULL;
c308c1b2
PE		 1492}
1493
1494static void sk_prot_free(struct proto *prot, struct sock *sk)
1495{
1496 struct kmem_cache *slab;
2e4afe7b 1497 struct module *owner;
c308c1b2 1498
2e4afe7b 1499 owner = prot->owner;
c308c1b2 1500 slab = prot->slab;
2e4afe7b 1501
bd1060a1 1502 cgroup_sk_free(&sk->sk_cgrp_data);
2d758073 1503 mem_cgroup_sk_free(sk);
2e4afe7b 1504 security_sk_free(sk);
c308c1b2
PE		 1505 if (slab != NULL)
1506 kmem_cache_free(slab, sk);
1507 else
1508 kfree(sk);
2e4afe7b 1509 module_put(owner);
c308c1b2
PE		 1510}
1511
1da177e4
LT		 1512/**
1513 * sk_alloc - All socket objects are allocated here
c4ea43c5 1514 * @net: the applicable net namespace
4dc3b16b
PP		 1515 * @family: protocol family
1516 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1517 * @prot: struct proto associated with this new sock instance
11aa9c28 1518 * @kern: is this to be a kernel socket?
1da177e4 1519 */
1b8d7ae4 1520struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
11aa9c28 1521 struct proto *prot, int kern)
1da177e4 1522{
c308c1b2 1523 struct sock *sk;
1da177e4 1524
154adbc8 1525 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1da177e4 1526 if (sk) {
154adbc8
PE		 1527 sk->sk_family = family;
1528 /*
1529 * See comment in struct sock definition to understand
1530 * why we need sk_prot_creator -acme
1531 */
1532 sk->sk_prot = sk->sk_prot_creator = prot;
cdfbabfb 1533 sk->sk_kern_sock = kern;
154adbc8 1534 sock_lock_init(sk);
26abe143 1535 sk->sk_net_refcnt = kern ? 0 : 1;
648845ab 1536 if (likely(sk->sk_net_refcnt)) {
26abe143 1537 get_net(net);
648845ab
TZ		 1538 sock_inuse_add(net, 1);
1539 }
1540
26abe143 1541 sock_net_set(sk, net);
14afee4b 1542 refcount_set(&sk->sk_wmem_alloc, 1);
f8451725 1543
2d758073 1544 mem_cgroup_sk_alloc(sk);
d979a39d 1545 cgroup_sk_alloc(&sk->sk_cgrp_data);
2a56a1fe
TH		 1546 sock_update_classid(&sk->sk_cgrp_data);
1547 sock_update_netprioidx(&sk->sk_cgrp_data);
1da177e4 1548 }
a79af59e 1549
2e4afe7b 1550 return sk;
1da177e4 1551}
2a91525c 1552EXPORT_SYMBOL(sk_alloc);
1da177e4 1553
a4298e45
ED		 1554/* Sockets having SOCK_RCU_FREE will call this function after one RCU
1555 * grace period. This is the case for UDP sockets and TCP listeners.
1556 */
1557static void __sk_destruct(struct rcu_head *head)
1da177e4 1558{
a4298e45 1559 struct sock *sk = container_of(head, struct sock, sk_rcu);
1da177e4 1560 struct sk_filter *filter;
1da177e4
LT		 1561
1562 if (sk->sk_destruct)
1563 sk->sk_destruct(sk);
1564
a898def2 1565 filter = rcu_dereference_check(sk->sk_filter,
14afee4b 1566 refcount_read(&sk->sk_wmem_alloc) == 0);
1da177e4 1567 if (filter) {
309dd5fc 1568 sk_filter_uncharge(sk, filter);
a9b3cd7f 1569 RCU_INIT_POINTER(sk->sk_filter, NULL);
1da177e4 1570 }
538950a1
CG		 1571 if (rcu_access_pointer(sk->sk_reuseport_cb))
1572 reuseport_detach_sock(sk);
1da177e4 1573
08e29af3 1574 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1da177e4
LT		 1575
1576 if (atomic_read(&sk->sk_omem_alloc))
e005d193
JP		 1577 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1578 __func__, atomic_read(&sk->sk_omem_alloc));
1da177e4 1579
22a0e18e
ED		 1580 if (sk->sk_frag.page) {
1581 put_page(sk->sk_frag.page);
1582 sk->sk_frag.page = NULL;
1583 }
1584
109f6e39
EB		 1585 if (sk->sk_peer_cred)
1586 put_cred(sk->sk_peer_cred);
1587 put_pid(sk->sk_peer_pid);
26abe143
EB		 1588 if (likely(sk->sk_net_refcnt))
1589 put_net(sock_net(sk));
c308c1b2 1590 sk_prot_free(sk->sk_prot_creator, sk);
1da177e4 1591}
2b85a34e 1592
a4298e45
ED		 1593void sk_destruct(struct sock *sk)
1594{
1595 if (sock_flag(sk, SOCK_RCU_FREE))
1596 call_rcu(&sk->sk_rcu, __sk_destruct);
1597 else
1598 __sk_destruct(&sk->sk_rcu);
1599}
1600
eb4cb008
CG		 1601static void __sk_free(struct sock *sk)
1602{
648845ab
TZ		 1603 if (likely(sk->sk_net_refcnt))
1604 sock_inuse_add(sock_net(sk), -1);
1605
9709020c 1606 if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
eb4cb008
CG		 1607 sock_diag_broadcast_destroy(sk);
1608 else
1609 sk_destruct(sk);
1610}
1611
2b85a34e
ED		 1612void sk_free(struct sock *sk)
1613{
1614 /*
25985edc 1615 * We subtract one from sk_wmem_alloc and can know if
2b85a34e
ED		 1616 * some packets are still in some tx queue.
1617 * If not null, sock_wfree() will call __sk_free(sk) later
1618 */
14afee4b 1619 if (refcount_dec_and_test(&sk->sk_wmem_alloc))
2b85a34e
ED		 1620 __sk_free(sk);
1621}
2a91525c 1622EXPORT_SYMBOL(sk_free);
1da177e4 1623
581319c5
PA		 1624static void sk_init_common(struct sock *sk)
1625{
1626 skb_queue_head_init(&sk->sk_receive_queue);
1627 skb_queue_head_init(&sk->sk_write_queue);
1628 skb_queue_head_init(&sk->sk_error_queue);
1629
1630 rwlock_init(&sk->sk_callback_lock);
1631 lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
1632 af_rlock_keys + sk->sk_family,
1633 af_family_rlock_key_strings[sk->sk_family]);
1634 lockdep_set_class_and_name(&sk->sk_write_queue.lock,
1635 af_wlock_keys + sk->sk_family,
1636 af_family_wlock_key_strings[sk->sk_family]);
1637 lockdep_set_class_and_name(&sk->sk_error_queue.lock,
1638 af_elock_keys + sk->sk_family,
1639 af_family_elock_key_strings[sk->sk_family]);
1640 lockdep_set_class_and_name(&sk->sk_callback_lock,
1641 af_callback_keys + sk->sk_family,
1642 af_family_clock_key_strings[sk->sk_family]);
1643}
1644
e56c57d0
ED		 1645/**
1646 * sk_clone_lock - clone a socket, and lock its clone
1647 * @sk: the socket to clone
1648 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1649 *
1650 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1651 */
1652struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
87d11ceb 1653{
8fd1d178 1654 struct sock *newsk;
278571ba 1655 bool is_charged = true;
87d11ceb 1656
8fd1d178 1657 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
87d11ceb
ACM		 1658 if (newsk != NULL) {
1659 struct sk_filter *filter;
1660
892c141e 1661 sock_copy(newsk, sk);
87d11ceb 1662
9d538fa6
CP		 1663 newsk->sk_prot_creator = sk->sk_prot;
1664
87d11ceb 1665 /* SANITY */
8a681736
SV		 1666 if (likely(newsk->sk_net_refcnt))
1667 get_net(sock_net(newsk));
87d11ceb
ACM		 1668 sk_node_init(&newsk->sk_node);
1669 sock_lock_init(newsk);
1670 bh_lock_sock(newsk);
fa438ccf 1671 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
8eae939f 1672 newsk->sk_backlog.len = 0;
87d11ceb
ACM		 1673
1674 atomic_set(&newsk->sk_rmem_alloc, 0);
2b85a34e
ED		 1675 /*
1676 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1677 */
14afee4b 1678 refcount_set(&newsk->sk_wmem_alloc, 1);
87d11ceb 1679 atomic_set(&newsk->sk_omem_alloc, 0);
581319c5 1680 sk_init_common(newsk);
87d11ceb
ACM		 1681
1682 newsk->sk_dst_cache = NULL;
9b8805a3 1683 newsk->sk_dst_pending_confirm = 0;
87d11ceb
ACM		 1684 newsk->sk_wmem_queued = 0;
1685 newsk->sk_forward_alloc = 0;
9caad864 1686 atomic_set(&newsk->sk_drops, 0);
87d11ceb 1687 newsk->sk_send_head = NULL;
87d11ceb 1688 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
52267790 1689 atomic_set(&newsk->sk_zckey, 0);
87d11ceb
ACM		 1690
1691 sock_reset_flag(newsk, SOCK_DONE);
edbe69ef 1692 mem_cgroup_sk_alloc(newsk);
c0576e39 1693 cgroup_sk_alloc(&newsk->sk_cgrp_data);
87d11ceb 1694
eefca20e
ED		 1695 rcu_read_lock();
1696 filter = rcu_dereference(sk->sk_filter);
87d11ceb 1697 if (filter != NULL)
278571ba
AS		 1698 /* though it's an empty new sock, the charging may fail
1699 * if sysctl_optmem_max was changed between creation of
1700 * original socket and cloning
1701 */
1702 is_charged = sk_filter_charge(newsk, filter);
eefca20e
ED		 1703 RCU_INIT_POINTER(newsk->sk_filter, filter);
1704 rcu_read_unlock();
87d11ceb 1705
d188ba86 1706 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
a97e50cc
DB		 1707 /* We need to make sure that we don't uncharge the new
1708 * socket if we couldn't charge it in the first place
1709 * as otherwise we uncharge the parent's filter.
1710 */
1711 if (!is_charged)
1712 RCU_INIT_POINTER(newsk->sk_filter, NULL);
94352d45 1713 sk_free_unlock_clone(newsk);
87d11ceb
ACM		 1714 newsk = NULL;
1715 goto out;
1716 }
fa463497 1717 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
87d11ceb
ACM		 1718
1719 newsk->sk_err = 0;
e551c32d 1720 newsk->sk_err_soft = 0;
87d11ceb 1721 newsk->sk_priority = 0;
2c8c56e1 1722 newsk->sk_incoming_cpu = raw_smp_processor_id();
33cf7c90 1723 atomic64_set(&newsk->sk_cookie, 0);
648845ab
TZ		 1724 if (likely(newsk->sk_net_refcnt))
1725 sock_inuse_add(sock_net(newsk), 1);
d979a39d 1726
4dc6dc71
ED		 1727 /*
1728 * Before updating sk_refcnt, we must commit prior changes to memory
1729 * (Documentation/RCU/rculist_nulls.txt for details)
1730 */
1731 smp_wmb();
41c6d650 1732 refcount_set(&newsk->sk_refcnt, 2);
87d11ceb
ACM		 1733
1734 /*
1735 * Increment the counter in the same struct proto as the master
1736 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1737 * is the same as sk->sk_prot->socks, as this field was copied
1738 * with memcpy).
1739 *
1740 * This _changes_ the previous behaviour, where
1741 * tcp_create_openreq_child always was incrementing the
1742 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1743 * to be taken into account in all callers. -acme
1744 */
1745 sk_refcnt_debug_inc(newsk);
972692e0 1746 sk_set_socket(newsk, NULL);
43815482 1747 newsk->sk_wq = NULL;
87d11ceb
ACM		 1748
1749 if (newsk->sk_prot->sockets_allocated)
180d8cd9 1750 sk_sockets_allocated_inc(newsk);
704da560 1751
080a270f
HFS		 1752 if (sock_needs_netstamp(sk) &&
1753 newsk->sk_flags & SK_FLAGS_TIMESTAMP)
704da560 1754 net_enable_timestamp();
87d11ceb
ACM		 1755 }
1756out:
1757 return newsk;
1758}
e56c57d0 1759EXPORT_SYMBOL_GPL(sk_clone_lock);
87d11ceb 1760
94352d45
ACM		 1761void sk_free_unlock_clone(struct sock *sk)
1762{
1763 /* It is still raw copy of parent, so invalidate
1764 * destructor and make plain sk_free() */
1765 sk->sk_destruct = NULL;
1766 bh_unlock_sock(sk);
1767 sk_free(sk);
1768}
1769EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
1770
9958089a
AK		 1771void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1772{
d6a4e26a
ED		 1773 u32 max_segs = 1;
1774
6bd4f355 1775 sk_dst_set(sk, dst);
0a6b2a1d 1776 sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps;
9958089a 1777 if (sk->sk_route_caps & NETIF_F_GSO)
4fcd6b99 1778 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
a465419b 1779 sk->sk_route_caps &= ~sk->sk_route_nocaps;
9958089a 1780 if (sk_can_gso(sk)) {
f70f250a 1781 if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
9958089a 1782 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
82cc1a7a 1783 } else {
9958089a 1784 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
82cc1a7a 1785 sk->sk_gso_max_size = dst->dev->gso_max_size;
d6a4e26a 1786 max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
82cc1a7a 1787 }
9958089a 1788 }
d6a4e26a 1789 sk->sk_gso_max_segs = max_segs;
9958089a
AK		 1790}
1791EXPORT_SYMBOL_GPL(sk_setup_caps);
1792
1da177e4
LT		 1793/*
1794 * Simple resource managers for sockets.
1795 */
1796
1797
4ec93edb
YH		 1798/*
1799 * Write buffer destructor automatically called from kfree_skb.
1da177e4
LT		 1800 */
1801void sock_wfree(struct sk_buff *skb)
1802{
1803 struct sock *sk = skb->sk;
d99927f4 1804 unsigned int len = skb->truesize;
1da177e4 1805
d99927f4
ED		 1806 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
1807 /*
1808 * Keep a reference on sk_wmem_alloc, this will be released
1809 * after sk_write_space() call
1810 */
14afee4b 1811 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
1da177e4 1812 sk->sk_write_space(sk);
d99927f4
ED		 1813 len = 1;
1814 }
2b85a34e 1815 /*
d99927f4
ED		 1816 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
1817 * could not do because of in-flight packets
2b85a34e 1818 */
14afee4b 1819 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2b85a34e 1820 __sk_free(sk);
1da177e4 1821}
2a91525c 1822EXPORT_SYMBOL(sock_wfree);
1da177e4 1823
1d2077ac
ED		 1824/* This variant of sock_wfree() is used by TCP,
1825 * since it sets SOCK_USE_WRITE_QUEUE.
1826 */
1827void __sock_wfree(struct sk_buff *skb)
1828{
1829 struct sock *sk = skb->sk;
1830
14afee4b 1831 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1d2077ac
ED		 1832 __sk_free(sk);
1833}
1834
9e17f8a4
ED		 1835void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1836{
1837 skb_orphan(skb);
1838 skb->sk = sk;
1839#ifdef CONFIG_INET
1840 if (unlikely(!sk_fullsock(sk))) {
1841 skb->destructor = sock_edemux;
1842 sock_hold(sk);
1843 return;
1844 }
1845#endif
1846 skb->destructor = sock_wfree;
1847 skb_set_hash_from_sk(skb, sk);
1848 /*
1849 * We used to take a refcount on sk, but following operation
1850 * is enough to guarantee sk_free() wont free this sock until
1851 * all in-flight packets are completed
1852 */
14afee4b 1853 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
9e17f8a4
ED		 1854}
1855EXPORT_SYMBOL(skb_set_owner_w);
1856
1d2077ac
ED		 1857/* This helper is used by netem, as it can hold packets in its
1858 * delay queue. We want to allow the owner socket to send more
1859 * packets, as if they were already TX completed by a typical driver.
1860 * But we also want to keep skb->sk set because some packet schedulers
f6ba8d33 1861 * rely on it (sch_fq for example).
1d2077ac 1862 */
f2f872f9
ED		 1863void skb_orphan_partial(struct sk_buff *skb)
1864{
f6ba8d33 1865 if (skb_is_tcp_pure_ack(skb))
1d2077ac
ED		 1866 return;
1867
f2f872f9
ED		 1868 if (skb->destructor == sock_wfree
1869#ifdef CONFIG_INET
1870 || skb->destructor == tcp_wfree
1871#endif
1872 ) {
f6ba8d33
ED		 1873 struct sock *sk = skb->sk;
1874
41c6d650 1875 if (refcount_inc_not_zero(&sk->sk_refcnt)) {
14afee4b 1876 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc));
f6ba8d33
ED		 1877 skb->destructor = sock_efree;
1878 }
f2f872f9
ED		 1879 } else {
1880 skb_orphan(skb);
1881 }
1882}
1883EXPORT_SYMBOL(skb_orphan_partial);
1884
4ec93edb
YH		 1885/*
1886 * Read buffer destructor automatically called from kfree_skb.
1da177e4
LT		 1887 */
1888void sock_rfree(struct sk_buff *skb)
1889{
1890 struct sock *sk = skb->sk;
d361fd59 1891 unsigned int len = skb->truesize;
1da177e4 1892
d361fd59
ED		 1893 atomic_sub(len, &sk->sk_rmem_alloc);
1894 sk_mem_uncharge(sk, len);
1da177e4 1895}
2a91525c 1896EXPORT_SYMBOL(sock_rfree);
1da177e4 1897
7768eed8
OH		 1898/*
1899 * Buffer destructor for skbs that are not used directly in read or write
1900 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
1901 */
62bccb8c
AD		 1902void sock_efree(struct sk_buff *skb)
1903{
1904 sock_put(skb->sk);
1905}
1906EXPORT_SYMBOL(sock_efree);
1907
976d0201 1908kuid_t sock_i_uid(struct sock *sk)
1da177e4 1909{
976d0201 1910 kuid_t uid;
1da177e4 1911
f064af1e 1912 read_lock_bh(&sk->sk_callback_lock);
976d0201 1913 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
f064af1e 1914 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 1915 return uid;
1916}
2a91525c 1917EXPORT_SYMBOL(sock_i_uid);
1da177e4
LT		 1918
1919unsigned long sock_i_ino(struct sock *sk)
1920{
1921 unsigned long ino;
1922
f064af1e 1923 read_lock_bh(&sk->sk_callback_lock);
1da177e4 1924 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
f064af1e 1925 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 1926 return ino;
1927}
2a91525c 1928EXPORT_SYMBOL(sock_i_ino);
1da177e4
LT		 1929
1930/*
1931 * Allocate a skb from the socket's send buffer.
1932 */
86a76caf 1933struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
dd0fc66f 1934 gfp_t priority)
1da177e4 1935{
14afee4b 1936 if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) {
2a91525c 1937 struct sk_buff *skb = alloc_skb(size, priority);
1da177e4
LT		 1938 if (skb) {
1939 skb_set_owner_w(skb, sk);
1940 return skb;
1941 }
1942 }
1943 return NULL;
1944}
2a91525c 1945EXPORT_SYMBOL(sock_wmalloc);
1da177e4 1946
98ba0bd5
WB		 1947static void sock_ofree(struct sk_buff *skb)
1948{
1949 struct sock *sk = skb->sk;
1950
1951 atomic_sub(skb->truesize, &sk->sk_omem_alloc);
1952}
1953
1954struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1955 gfp_t priority)
1956{
1957 struct sk_buff *skb;
1958
1959 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
1960 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
1961 sysctl_optmem_max)
1962 return NULL;
1963
1964 skb = alloc_skb(size, priority);
1965 if (!skb)
1966 return NULL;
1967
1968 atomic_add(skb->truesize, &sk->sk_omem_alloc);
1969 skb->sk = sk;
1970 skb->destructor = sock_ofree;
1971 return skb;
1972}
1973
4ec93edb 1974/*
1da177e4 1975 * Allocate a memory block from the socket's option memory buffer.
4ec93edb 1976 */
dd0fc66f 1977void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
1da177e4 1978{
95c96174 1979 if ((unsigned int)size <= sysctl_optmem_max &&
1da177e4
LT		 1980 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
1981 void *mem;
1982 /* First do the add, to avoid the race if kmalloc
4ec93edb 1983 * might sleep.
1da177e4
LT		 1984 */
1985 atomic_add(size, &sk->sk_omem_alloc);
1986 mem = kmalloc(size, priority);
1987 if (mem)
1988 return mem;
1989 atomic_sub(size, &sk->sk_omem_alloc);
1990 }
1991 return NULL;
1992}
2a91525c 1993EXPORT_SYMBOL(sock_kmalloc);
1da177e4 1994
79e88659
DB		 1995/* Free an option memory block. Note, we actually want the inline
1996 * here as this allows gcc to detect the nullify and fold away the
1997 * condition entirely.
1da177e4 1998 */
79e88659
DB		 1999static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
2000 const bool nullify)
1da177e4 2001{
e53da5fb
DM		 2002 if (WARN_ON_ONCE(!mem))
2003 return;
79e88659
DB		 2004 if (nullify)
2005 kzfree(mem);
2006 else
2007 kfree(mem);
1da177e4
LT		 2008 atomic_sub(size, &sk->sk_omem_alloc);
2009}
79e88659
DB		 2010
2011void sock_kfree_s(struct sock *sk, void *mem, int size)
2012{
2013 __sock_kfree_s(sk, mem, size, false);
2014}
2a91525c 2015EXPORT_SYMBOL(sock_kfree_s);
1da177e4 2016
79e88659
DB		 2017void sock_kzfree_s(struct sock *sk, void *mem, int size)
2018{
2019 __sock_kfree_s(sk, mem, size, true);
2020}
2021EXPORT_SYMBOL(sock_kzfree_s);
2022
1da177e4
LT		 2023/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
2024 I think, these locks should be removed for datagram sockets.
2025 */
2a91525c 2026static long sock_wait_for_wmem(struct sock *sk, long timeo)
1da177e4
LT		 2027{
2028 DEFINE_WAIT(wait);
2029
9cd3e072 2030 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4
LT		 2031 for (;;) {
2032 if (!timeo)
2033 break;
2034 if (signal_pending(current))
2035 break;
2036 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
aa395145 2037 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
14afee4b 2038 if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf)
1da177e4
LT		 2039 break;
2040 if (sk->sk_shutdown & SEND_SHUTDOWN)
2041 break;
2042 if (sk->sk_err)
2043 break;
2044 timeo = schedule_timeout(timeo);
2045 }
aa395145 2046 finish_wait(sk_sleep(sk), &wait);
1da177e4
LT		 2047 return timeo;
2048}
2049
2050
2051/*
2052 * Generic send/receive buffer handlers
2053 */
2054
4cc7f68d
HX		 2055struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
2056 unsigned long data_len, int noblock,
28d64271 2057 int *errcode, int max_page_order)
1da177e4 2058{
2e4e4410 2059 struct sk_buff *skb;
1da177e4
LT		 2060 long timeo;
2061 int err;
2062
1da177e4 2063 timeo = sock_sndtimeo(sk, noblock);
2e4e4410 2064 for (;;) {
1da177e4
LT		 2065 err = sock_error(sk);
2066 if (err != 0)
2067 goto failure;
2068
2069 err = -EPIPE;
2070 if (sk->sk_shutdown & SEND_SHUTDOWN)
2071 goto failure;
2072
2e4e4410
ED		 2073 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf)
2074 break;
28d64271 2075
9cd3e072 2076 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2e4e4410
ED		 2077 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2078 err = -EAGAIN;
2079 if (!timeo)
1da177e4 2080 goto failure;
2e4e4410
ED		 2081 if (signal_pending(current))
2082 goto interrupted;
2083 timeo = sock_wait_for_wmem(sk, timeo);
1da177e4 2084 }
2e4e4410
ED		 2085 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2086 errcode, sk->sk_allocation);
2087 if (skb)
2088 skb_set_owner_w(skb, sk);
1da177e4
LT		 2089 return skb;
2090
2091interrupted:
2092 err = sock_intr_errno(timeo);
2093failure:
2094 *errcode = err;
2095 return NULL;
2096}
4cc7f68d 2097EXPORT_SYMBOL(sock_alloc_send_pskb);
1da177e4 2098
4ec93edb 2099struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1da177e4
LT		 2100 int noblock, int *errcode)
2101{
28d64271 2102 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
1da177e4 2103}
2a91525c 2104EXPORT_SYMBOL(sock_alloc_send_skb);
1da177e4 2105
39771b12
WB		 2106int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2107 struct sockcm_cookie *sockc)
2108{
3dd17e63
SHY		 2109 u32 tsflags;
2110
39771b12
WB		 2111 switch (cmsg->cmsg_type) {
2112 case SO_MARK:
2113 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
2114 return -EPERM;
2115 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2116 return -EINVAL;
2117 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2118 break;
3dd17e63
SHY		 2119 case SO_TIMESTAMPING:
2120 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2121 return -EINVAL;
2122
2123 tsflags = *(u32 *)CMSG_DATA(cmsg);
2124 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2125 return -EINVAL;
2126
2127 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2128 sockc->tsflags |= tsflags;
2129 break;
80b14dee
RC		 2130 case SCM_TXTIME:
2131 if (!sock_flag(sk, SOCK_TXTIME))
2132 return -EINVAL;
2133 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
2134 return -EINVAL;
2135 sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
2136 break;
779f1ede
SHY		 2137 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2138 case SCM_RIGHTS:
2139 case SCM_CREDENTIALS:
2140 break;
39771b12
WB		 2141 default:
2142 return -EINVAL;
2143 }
2144 return 0;
2145}
2146EXPORT_SYMBOL(__sock_cmsg_send);
2147
f28ea365
EJ		 2148int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2149 struct sockcm_cookie *sockc)
2150{
2151 struct cmsghdr *cmsg;
39771b12 2152 int ret;
f28ea365
EJ		 2153
2154 for_each_cmsghdr(cmsg, msg) {
2155 if (!CMSG_OK(msg, cmsg))
2156 return -EINVAL;
2157 if (cmsg->cmsg_level != SOL_SOCKET)
2158 continue;
39771b12
WB		 2159 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2160 if (ret)
2161 return ret;
f28ea365
EJ		 2162 }
2163 return 0;
2164}
2165EXPORT_SYMBOL(sock_cmsg_send);
2166
06044751
ED		 2167static void sk_enter_memory_pressure(struct sock *sk)
2168{
2169 if (!sk->sk_prot->enter_memory_pressure)
2170 return;
2171
2172 sk->sk_prot->enter_memory_pressure(sk);
2173}
2174
2175static void sk_leave_memory_pressure(struct sock *sk)
2176{
2177 if (sk->sk_prot->leave_memory_pressure) {
2178 sk->sk_prot->leave_memory_pressure(sk);
2179 } else {
2180 unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2181
2182 if (memory_pressure && *memory_pressure)
2183 *memory_pressure = 0;
2184 }
2185}
2186
5640f768
ED		 2187/* On 32bit arches, an skb frag is limited to 2^15 */
2188#define SKB_FRAG_PAGE_ORDER get_order(32768)
2189
400dfd3a
ED		 2190/**
2191 * skb_page_frag_refill - check that a page_frag contains enough room
2192 * @sz: minimum size of the fragment we want to get
2193 * @pfrag: pointer to page_frag
82d5e2b8 2194 * @gfp: priority for memory allocation
400dfd3a
ED		 2195 *
2196 * Note: While this allocator tries to use high order pages, there is
2197 * no guarantee that allocations succeed. Therefore, @sz MUST be
2198 * less or equal than PAGE_SIZE.
2199 */
d9b2938a 2200bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
5640f768 2201{
5640f768 2202 if (pfrag->page) {
fe896d18 2203 if (page_ref_count(pfrag->page) == 1) {
5640f768
ED		 2204 pfrag->offset = 0;
2205 return true;
2206 }
400dfd3a 2207 if (pfrag->offset + sz <= pfrag->size)
5640f768
ED		 2208 return true;
2209 put_page(pfrag->page);
2210 }
2211
d9b2938a
ED		 2212 pfrag->offset = 0;
2213 if (SKB_FRAG_PAGE_ORDER) {
d0164adc
MG		 2214 /* Avoid direct reclaim but allow kswapd to wake */
2215 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2216 __GFP_COMP | __GFP_NOWARN |
2217 __GFP_NORETRY,
d9b2938a 2218 SKB_FRAG_PAGE_ORDER);
5640f768 2219 if (likely(pfrag->page)) {
d9b2938a 2220 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
5640f768
ED		 2221 return true;
2222 }
d9b2938a
ED		 2223 }
2224 pfrag->page = alloc_page(gfp);
2225 if (likely(pfrag->page)) {
2226 pfrag->size = PAGE_SIZE;
2227 return true;
2228 }
400dfd3a
ED		 2229 return false;
2230}
2231EXPORT_SYMBOL(skb_page_frag_refill);
2232
2233bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2234{
2235 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2236 return true;
2237
5640f768
ED		 2238 sk_enter_memory_pressure(sk);
2239 sk_stream_moderate_sndbuf(sk);
2240 return false;
2241}
2242EXPORT_SYMBOL(sk_page_frag_refill);
2243
1da177e4 2244static void __lock_sock(struct sock *sk)
f39234d6
NK		 2245 __releases(&sk->sk_lock.slock)
2246 __acquires(&sk->sk_lock.slock)
1da177e4
LT		 2247{
2248 DEFINE_WAIT(wait);
2249
e71a4783 2250 for (;;) {
1da177e4
LT		 2251 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2252 TASK_UNINTERRUPTIBLE);
2253 spin_unlock_bh(&sk->sk_lock.slock);
2254 schedule();
2255 spin_lock_bh(&sk->sk_lock.slock);
e71a4783 2256 if (!sock_owned_by_user(sk))
1da177e4
LT		 2257 break;
2258 }
2259 finish_wait(&sk->sk_lock.wq, &wait);
2260}
2261
8873c064 2262void __release_sock(struct sock *sk)
f39234d6
NK		 2263 __releases(&sk->sk_lock.slock)
2264 __acquires(&sk->sk_lock.slock)
1da177e4 2265{
5413d1ba 2266 struct sk_buff *skb, *next;
1da177e4 2267
5413d1ba 2268 while ((skb = sk->sk_backlog.head) != NULL) {
1da177e4 2269 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
1da177e4 2270
5413d1ba 2271 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4 2272
5413d1ba
ED		 2273 do {
2274 next = skb->next;
e4cbb02a 2275 prefetch(next);
7fee226a 2276 WARN_ON_ONCE(skb_dst_is_noref(skb));
a8305bff 2277 skb_mark_not_on_list(skb);
c57943a1 2278 sk_backlog_rcv(sk, skb);
1da177e4 2279
5413d1ba 2280 cond_resched();
1da177e4
LT		 2281
2282 skb = next;
2283 } while (skb != NULL);
2284
5413d1ba
ED		 2285 spin_lock_bh(&sk->sk_lock.slock);
2286 }
8eae939f
ZY		 2287
2288 /*
2289 * Doing the zeroing here guarantee we can not loop forever
2290 * while a wild producer attempts to flood us.
2291 */
2292 sk->sk_backlog.len = 0;
1da177e4
LT		 2293}
2294
d41a69f1
ED		 2295void __sk_flush_backlog(struct sock *sk)
2296{
2297 spin_lock_bh(&sk->sk_lock.slock);
2298 __release_sock(sk);
2299 spin_unlock_bh(&sk->sk_lock.slock);
2300}
2301
1da177e4
LT		 2302/**
2303 * sk_wait_data - wait for data to arrive at sk_receive_queue
4dc3b16b
PP		 2304 * @sk: sock to wait on
2305 * @timeo: for how long
dfbafc99 2306 * @skb: last skb seen on sk_receive_queue
1da177e4
LT		 2307 *
2308 * Now socket state including sk->sk_err is changed only under lock,
2309 * hence we may omit checks after joining wait queue.
2310 * We check receive queue before schedule() only as optimization;
2311 * it is very likely that release_sock() added new data.
2312 */
dfbafc99 2313int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
1da177e4 2314{
d9dc8b0f 2315 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1da177e4 2316 int rc;
1da177e4 2317
d9dc8b0f 2318 add_wait_queue(sk_sleep(sk), &wait);
9cd3e072 2319 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2320 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
9cd3e072 2321 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2322 remove_wait_queue(sk_sleep(sk), &wait);
1da177e4
LT		 2323 return rc;
2324}
1da177e4
LT		 2325EXPORT_SYMBOL(sk_wait_data);
2326
3ab224be 2327/**
f8c3bf00 2328 * __sk_mem_raise_allocated - increase memory_allocated
3ab224be
HA		 2329 * @sk: socket
2330 * @size: memory size to allocate
f8c3bf00 2331 * @amt: pages to allocate
3ab224be
HA		 2332 * @kind: allocation type
2333 *
f8c3bf00 2334 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
3ab224be 2335 */
f8c3bf00 2336int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
3ab224be
HA		 2337{
2338 struct proto *prot = sk->sk_prot;
f8c3bf00 2339 long allocated = sk_memory_allocated_add(sk, amt);
d6f19938 2340 bool charged = true;
e805605c 2341
baac50bb 2342 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
d6f19938 2343 !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt)))
e805605c 2344 goto suppress_allocation;
3ab224be
HA		 2345
2346 /* Under limit. */
e805605c 2347 if (allocated <= sk_prot_mem_limits(sk, 0)) {
180d8cd9 2348 sk_leave_memory_pressure(sk);
3ab224be
HA		 2349 return 1;
2350 }
2351
e805605c
JW		 2352 /* Under pressure. */
2353 if (allocated > sk_prot_mem_limits(sk, 1))
180d8cd9 2354 sk_enter_memory_pressure(sk);
3ab224be 2355
e805605c
JW		 2356 /* Over hard limit. */
2357 if (allocated > sk_prot_mem_limits(sk, 2))
3ab224be
HA		 2358 goto suppress_allocation;
2359
2360 /* guarantee minimum buffer size under pressure */
2361 if (kind == SK_MEM_RECV) {
a3dcaf17 2362 if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
3ab224be 2363 return 1;
180d8cd9 2364
3ab224be 2365 } else { /* SK_MEM_SEND */
a3dcaf17
ED		 2366 int wmem0 = sk_get_wmem0(sk, prot);
2367
3ab224be 2368 if (sk->sk_type == SOCK_STREAM) {
a3dcaf17 2369 if (sk->sk_wmem_queued < wmem0)
3ab224be 2370 return 1;
a3dcaf17 2371 } else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
3ab224be 2372 return 1;
a3dcaf17 2373 }
3ab224be
HA		 2374 }
2375
180d8cd9 2376 if (sk_has_memory_pressure(sk)) {
1748376b
ED		 2377 int alloc;
2378
180d8cd9 2379 if (!sk_under_memory_pressure(sk))
1748376b 2380 return 1;
180d8cd9
GC		 2381 alloc = sk_sockets_allocated_read_positive(sk);
2382 if (sk_prot_mem_limits(sk, 2) > alloc *
3ab224be
HA		 2383 sk_mem_pages(sk->sk_wmem_queued +
2384 atomic_read(&sk->sk_rmem_alloc) +
2385 sk->sk_forward_alloc))
2386 return 1;
2387 }
2388
2389suppress_allocation:
2390
2391 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2392 sk_stream_moderate_sndbuf(sk);
2393
2394 /* Fail only if socket is _under_ its sndbuf.
2395 * In this case we cannot block, so that we have to fail.
2396 */
2397 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2398 return 1;
2399 }
2400
d6f19938
YS		 2401 if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged))
2402 trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
3847ce32 2403
0e90b31f 2404 sk_memory_allocated_sub(sk, amt);
180d8cd9 2405
baac50bb
JW		 2406 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2407 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
e805605c 2408
3ab224be
HA		 2409 return 0;
2410}
f8c3bf00
PA		 2411EXPORT_SYMBOL(__sk_mem_raise_allocated);
2412
2413/**
2414 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2415 * @sk: socket
2416 * @size: memory size to allocate
2417 * @kind: allocation type
2418 *
2419 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2420 * rmem allocation. This function assumes that protocols which have
2421 * memory_pressure use sk_wmem_queued as write buffer accounting.
2422 */
2423int __sk_mem_schedule(struct sock *sk, int size, int kind)
2424{
2425 int ret, amt = sk_mem_pages(size);
2426
2427 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
2428 ret = __sk_mem_raise_allocated(sk, size, amt, kind);
2429 if (!ret)
2430 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
2431 return ret;
2432}
3ab224be
HA		 2433EXPORT_SYMBOL(__sk_mem_schedule);
2434
2435/**
f8c3bf00 2436 * __sk_mem_reduce_allocated - reclaim memory_allocated
3ab224be 2437 * @sk: socket
f8c3bf00
PA		 2438 * @amount: number of quanta
2439 *
2440 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3ab224be 2441 */
f8c3bf00 2442void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3ab224be 2443{
1a24e04e 2444 sk_memory_allocated_sub(sk, amount);
3ab224be 2445
baac50bb
JW		 2446 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2447 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
e805605c 2448
180d8cd9
GC		 2449 if (sk_under_memory_pressure(sk) &&
2450 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2451 sk_leave_memory_pressure(sk);
3ab224be 2452}
f8c3bf00
PA		 2453EXPORT_SYMBOL(__sk_mem_reduce_allocated);
2454
2455/**
2456 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
2457 * @sk: socket
2458 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2459 */
2460void __sk_mem_reclaim(struct sock *sk, int amount)
2461{
2462 amount >>= SK_MEM_QUANTUM_SHIFT;
2463 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2464 __sk_mem_reduce_allocated(sk, amount);
2465}
3ab224be
HA		 2466EXPORT_SYMBOL(__sk_mem_reclaim);
2467
627d2d6b 2468int sk_set_peek_off(struct sock *sk, int val)
2469{
627d2d6b 2470 sk->sk_peek_off = val;
2471 return 0;
2472}
2473EXPORT_SYMBOL_GPL(sk_set_peek_off);
3ab224be 2474
1da177e4
LT		 2475/*
2476 * Set of default routines for initialising struct proto_ops when
2477 * the protocol does not support a particular function. In certain
2478 * cases where it makes no sense for a protocol to have a "do nothing"
2479 * function, some default processing is provided.
2480 */
2481
2482int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2483{
2484 return -EOPNOTSUPP;
2485}
2a91525c 2486EXPORT_SYMBOL(sock_no_bind);
1da177e4 2487
4ec93edb 2488int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
1da177e4
LT		 2489 int len, int flags)
2490{
2491 return -EOPNOTSUPP;
2492}
2a91525c 2493EXPORT_SYMBOL(sock_no_connect);
1da177e4
LT		 2494
2495int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2496{
2497 return -EOPNOTSUPP;
2498}
2a91525c 2499EXPORT_SYMBOL(sock_no_socketpair);
1da177e4 2500
cdfbabfb
DH		 2501int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
2502 bool kern)
1da177e4
LT		 2503{
2504 return -EOPNOTSUPP;
2505}
2a91525c 2506EXPORT_SYMBOL(sock_no_accept);
1da177e4 2507
4ec93edb 2508int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
9b2c45d4 2509 int peer)
1da177e4
LT		 2510{
2511 return -EOPNOTSUPP;
2512}
2a91525c 2513EXPORT_SYMBOL(sock_no_getname);
1da177e4 2514
1da177e4
LT		 2515int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2516{
2517 return -EOPNOTSUPP;
2518}
2a91525c 2519EXPORT_SYMBOL(sock_no_ioctl);
1da177e4
LT		 2520
2521int sock_no_listen(struct socket *sock, int backlog)
2522{
2523 return -EOPNOTSUPP;
2524}
2a91525c 2525EXPORT_SYMBOL(sock_no_listen);
1da177e4
LT		 2526
2527int sock_no_shutdown(struct socket *sock, int how)
2528{
2529 return -EOPNOTSUPP;
2530}
2a91525c 2531EXPORT_SYMBOL(sock_no_shutdown);
1da177e4
LT		 2532
2533int sock_no_setsockopt(struct socket *sock, int level, int optname,
b7058842 2534 char __user *optval, unsigned int optlen)
1da177e4
LT		 2535{
2536 return -EOPNOTSUPP;
2537}
2a91525c 2538EXPORT_SYMBOL(sock_no_setsockopt);
1da177e4
LT		 2539
2540int sock_no_getsockopt(struct socket *sock, int level, int optname,
2541 char __user *optval, int __user *optlen)
2542{
2543 return -EOPNOTSUPP;
2544}
2a91525c 2545EXPORT_SYMBOL(sock_no_getsockopt);
1da177e4 2546
1b784140 2547int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
1da177e4
LT		 2548{
2549 return -EOPNOTSUPP;
2550}
2a91525c 2551EXPORT_SYMBOL(sock_no_sendmsg);
1da177e4 2552
306b13eb
TH		 2553int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
2554{
2555 return -EOPNOTSUPP;
2556}
2557EXPORT_SYMBOL(sock_no_sendmsg_locked);
2558
1b784140
YX		 2559int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2560 int flags)
1da177e4
LT		 2561{
2562 return -EOPNOTSUPP;
2563}
2a91525c 2564EXPORT_SYMBOL(sock_no_recvmsg);
1da177e4
LT		 2565
2566int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2567{
2568 /* Mirror missing mmap method error code */
2569 return -ENODEV;
2570}
2a91525c 2571EXPORT_SYMBOL(sock_no_mmap);
1da177e4
LT		 2572
2573ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2574{
2575 ssize_t res;
2576 struct msghdr msg = {.msg_flags = flags};
2577 struct kvec iov;
2578 char *kaddr = kmap(page);
2579 iov.iov_base = kaddr + offset;
2580 iov.iov_len = size;
2581 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2582 kunmap(page);
2583 return res;
2584}
2a91525c 2585EXPORT_SYMBOL(sock_no_sendpage);
1da177e4 2586
306b13eb
TH		 2587ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
2588 int offset, size_t size, int flags)
2589{
2590 ssize_t res;
2591 struct msghdr msg = {.msg_flags = flags};
2592 struct kvec iov;
2593 char *kaddr = kmap(page);
2594
2595 iov.iov_base = kaddr + offset;
2596 iov.iov_len = size;
2597 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
2598 kunmap(page);
2599 return res;
2600}
2601EXPORT_SYMBOL(sock_no_sendpage_locked);
2602
1da177e4
LT		 2603/*
2604 * Default Socket Callbacks
2605 */
2606
2607static void sock_def_wakeup(struct sock *sk)
2608{
43815482
ED		 2609 struct socket_wq *wq;
2610
2611 rcu_read_lock();
2612 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2613 if (skwq_has_sleeper(wq))
43815482
ED		 2614 wake_up_interruptible_all(&wq->wait);
2615 rcu_read_unlock();
1da177e4
LT		 2616}
2617
2618static void sock_def_error_report(struct sock *sk)
2619{
43815482
ED		 2620 struct socket_wq *wq;
2621
2622 rcu_read_lock();
2623 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2624 if (skwq_has_sleeper(wq))
a9a08845 2625 wake_up_interruptible_poll(&wq->wait, EPOLLERR);
8d8ad9d7 2626 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
43815482 2627 rcu_read_unlock();
1da177e4
LT		 2628}
2629
676d2369 2630static void sock_def_readable(struct sock *sk)
1da177e4 2631{
43815482
ED		 2632 struct socket_wq *wq;
2633
2634 rcu_read_lock();
2635 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2636 if (skwq_has_sleeper(wq))
a9a08845
LT		 2637 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
2638 EPOLLRDNORM | EPOLLRDBAND);
8d8ad9d7 2639 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
43815482 2640 rcu_read_unlock();
1da177e4
LT		 2641}
2642
2643static void sock_def_write_space(struct sock *sk)
2644{
43815482
ED		 2645 struct socket_wq *wq;
2646
2647 rcu_read_lock();
1da177e4
LT		 2648
2649 /* Do not wake up a writer until he can make "significant"
2650 * progress. --DaveM
2651 */
14afee4b 2652 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
43815482 2653 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2654 if (skwq_has_sleeper(wq))
a9a08845
LT		 2655 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
2656 EPOLLWRNORM | EPOLLWRBAND);
1da177e4
LT		 2657
2658 /* Should agree with poll, otherwise some programs break */
2659 if (sock_writeable(sk))
8d8ad9d7 2660 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
1da177e4
LT		 2661 }
2662
43815482 2663 rcu_read_unlock();
1da177e4
LT		 2664}
2665
2666static void sock_def_destruct(struct sock *sk)
2667{
1da177e4
LT		 2668}
2669
2670void sk_send_sigurg(struct sock *sk)
2671{
2672 if (sk->sk_socket && sk->sk_socket->file)
2673 if (send_sigurg(&sk->sk_socket->file->f_owner))
8d8ad9d7 2674 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
1da177e4 2675}
2a91525c 2676EXPORT_SYMBOL(sk_send_sigurg);
1da177e4
LT		 2677
2678void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2679 unsigned long expires)
2680{
2681 if (!mod_timer(timer, expires))
2682 sock_hold(sk);
2683}
1da177e4
LT		 2684EXPORT_SYMBOL(sk_reset_timer);
2685
2686void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2687{
25cc4ae9 2688 if (del_timer(timer))
1da177e4
LT		 2689 __sock_put(sk);
2690}
1da177e4
LT		 2691EXPORT_SYMBOL(sk_stop_timer);
2692
2693void sock_init_data(struct socket *sock, struct sock *sk)
2694{
581319c5 2695 sk_init_common(sk);
1da177e4
LT		 2696 sk->sk_send_head = NULL;
2697
99767f27 2698 timer_setup(&sk->sk_timer, NULL, 0);
4ec93edb 2699
1da177e4
LT		 2700 sk->sk_allocation = GFP_KERNEL;
2701 sk->sk_rcvbuf = sysctl_rmem_default;
2702 sk->sk_sndbuf = sysctl_wmem_default;
2703 sk->sk_state = TCP_CLOSE;
972692e0 2704 sk_set_socket(sk, sock);
1da177e4
LT		 2705
2706 sock_set_flag(sk, SOCK_ZAPPED);
2707
e71a4783 2708 if (sock) {
1da177e4 2709 sk->sk_type = sock->type;
43815482 2710 sk->sk_wq = sock->wq;
1da177e4 2711 sock->sk = sk;
86741ec2
LC		 2712 sk->sk_uid = SOCK_INODE(sock)->i_uid;
2713 } else {
43815482 2714 sk->sk_wq = NULL;
86741ec2
LC		 2715 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
2716 }
1da177e4 2717
1da177e4 2718 rwlock_init(&sk->sk_callback_lock);
cdfbabfb
DH		 2719 if (sk->sk_kern_sock)
2720 lockdep_set_class_and_name(
2721 &sk->sk_callback_lock,
2722 af_kern_callback_keys + sk->sk_family,
2723 af_family_kern_clock_key_strings[sk->sk_family]);
2724 else
2725 lockdep_set_class_and_name(
2726 &sk->sk_callback_lock,
443aef0e
PZ		 2727 af_callback_keys + sk->sk_family,
2728 af_family_clock_key_strings[sk->sk_family]);
1da177e4
LT		 2729
2730 sk->sk_state_change = sock_def_wakeup;
2731 sk->sk_data_ready = sock_def_readable;
2732 sk->sk_write_space = sock_def_write_space;
2733 sk->sk_error_report = sock_def_error_report;
2734 sk->sk_destruct = sock_def_destruct;
2735
5640f768
ED		 2736 sk->sk_frag.page = NULL;
2737 sk->sk_frag.offset = 0;
ef64a54f 2738 sk->sk_peek_off = -1;
1da177e4 2739
109f6e39
EB		 2740 sk->sk_peer_pid = NULL;
2741 sk->sk_peer_cred = NULL;
1da177e4
LT		 2742 sk->sk_write_pending = 0;
2743 sk->sk_rcvlowat = 1;
2744 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
2745 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
2746
6c7c98ba 2747 sk->sk_stamp = SK_DEFAULT_STAMP;
52267790 2748 atomic_set(&sk->sk_zckey, 0);
1da177e4 2749
e0d1095a 2750#ifdef CONFIG_NET_RX_BUSY_POLL
06021292 2751 sk->sk_napi_id = 0;
64b0dc51 2752 sk->sk_ll_usec = sysctl_net_busy_read;
06021292
ET		 2753#endif
2754
76a9ebe8
ED		 2755 sk->sk_max_pacing_rate = ~0UL;
2756 sk->sk_pacing_rate = ~0UL;
3a9b76fd 2757 sk->sk_pacing_shift = 10;
70da268b 2758 sk->sk_incoming_cpu = -1;
c6345ce7
AN		 2759
2760 sk_rx_queue_clear(sk);
4dc6dc71
ED		 2761 /*
2762 * Before updating sk_refcnt, we must commit prior changes to memory
2763 * (Documentation/RCU/rculist_nulls.txt for details)
2764 */
2765 smp_wmb();
41c6d650 2766 refcount_set(&sk->sk_refcnt, 1);
33c732c3 2767 atomic_set(&sk->sk_drops, 0);
1da177e4 2768}
2a91525c 2769EXPORT_SYMBOL(sock_init_data);
1da177e4 2770
b5606c2d 2771void lock_sock_nested(struct sock *sk, int subclass)
1da177e4
LT		 2772{
2773 might_sleep();
a5b5bb9a 2774 spin_lock_bh(&sk->sk_lock.slock);
d2e9117c 2775 if (sk->sk_lock.owned)
1da177e4 2776 __lock_sock(sk);
d2e9117c 2777 sk->sk_lock.owned = 1;
a5b5bb9a
IM		 2778 spin_unlock(&sk->sk_lock.slock);
2779 /*
2780 * The sk_lock has mutex_lock() semantics here:
2781 */
fcc70d5f 2782 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
a5b5bb9a 2783 local_bh_enable();
1da177e4 2784}
fcc70d5f 2785EXPORT_SYMBOL(lock_sock_nested);
1da177e4 2786
b5606c2d 2787void release_sock(struct sock *sk)
1da177e4 2788{
a5b5bb9a 2789 spin_lock_bh(&sk->sk_lock.slock);
1da177e4
LT		 2790 if (sk->sk_backlog.tail)
2791 __release_sock(sk);
46d3ceab 2792
c3f9b018
ED		 2793 /* Warning : release_cb() might need to release sk ownership,
2794 * ie call sock_release_ownership(sk) before us.
2795 */
46d3ceab
ED		 2796 if (sk->sk_prot->release_cb)
2797 sk->sk_prot->release_cb(sk);
2798
c3f9b018 2799 sock_release_ownership(sk);
a5b5bb9a
IM		 2800 if (waitqueue_active(&sk->sk_lock.wq))
2801 wake_up(&sk->sk_lock.wq);
2802 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4
LT		 2803}
2804EXPORT_SYMBOL(release_sock);
2805
8a74ad60
ED		 2806/**
2807 * lock_sock_fast - fast version of lock_sock
2808 * @sk: socket
2809 *
2810 * This version should be used for very small section, where process wont block
d651983d
MCC		 2811 * return false if fast path is taken:
2812 *
8a74ad60 2813 * sk_lock.slock locked, owned = 0, BH disabled
d651983d
MCC		 2814 *
2815 * return true if slow path is taken:
2816 *
8a74ad60
ED		 2817 * sk_lock.slock unlocked, owned = 1, BH enabled
2818 */
2819bool lock_sock_fast(struct sock *sk)
2820{
2821 might_sleep();
2822 spin_lock_bh(&sk->sk_lock.slock);
2823
2824 if (!sk->sk_lock.owned)
2825 /*
2826 * Note : We must disable BH
2827 */
2828 return false;
2829
2830 __lock_sock(sk);
2831 sk->sk_lock.owned = 1;
2832 spin_unlock(&sk->sk_lock.slock);
2833 /*
2834 * The sk_lock has mutex_lock() semantics here:
2835 */
2836 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
2837 local_bh_enable();
2838 return true;
2839}
2840EXPORT_SYMBOL(lock_sock_fast);
2841
1da177e4 2842int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
4ec93edb 2843{
b7aa0bf7 2844 struct timeval tv;
9dae3497
YS		 2845
2846 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
b7aa0bf7
ED		 2847 tv = ktime_to_timeval(sk->sk_stamp);
2848 if (tv.tv_sec == -1)
1da177e4 2849 return -ENOENT;
b7aa0bf7
ED		 2850 if (tv.tv_sec == 0) {
2851 sk->sk_stamp = ktime_get_real();
2852 tv = ktime_to_timeval(sk->sk_stamp);
2853 }
2854 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0;
4ec93edb 2855}
1da177e4
LT		 2856EXPORT_SYMBOL(sock_get_timestamp);
2857
ae40eb1e
ED		 2858int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
2859{
2860 struct timespec ts;
9dae3497
YS		 2861
2862 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
ae40eb1e
ED		 2863 ts = ktime_to_timespec(sk->sk_stamp);
2864 if (ts.tv_sec == -1)
2865 return -ENOENT;
2866 if (ts.tv_sec == 0) {
2867 sk->sk_stamp = ktime_get_real();
2868 ts = ktime_to_timespec(sk->sk_stamp);
2869 }
2870 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0;
2871}
2872EXPORT_SYMBOL(sock_get_timestampns);
2873
20d49473 2874void sock_enable_timestamp(struct sock *sk, int flag)
4ec93edb 2875{
20d49473 2876 if (!sock_flag(sk, flag)) {
08e29af3
ED		 2877 unsigned long previous_flags = sk->sk_flags;
2878
20d49473
PO		 2879 sock_set_flag(sk, flag);
2880 /*
2881 * we just set one of the two flags which require net
2882 * time stamping, but time stamping might have been on
2883 * already because of the other one
2884 */
080a270f
HFS		 2885 if (sock_needs_netstamp(sk) &&
2886 !(previous_flags & SK_FLAGS_TIMESTAMP))
20d49473 2887 net_enable_timestamp();
1da177e4
LT		 2888 }
2889}
1da177e4 2890
cb820f8e
RC		 2891int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
2892 int level, int type)
2893{
2894 struct sock_exterr_skb *serr;
364a9e93 2895 struct sk_buff *skb;
cb820f8e
RC		 2896 int copied, err;
2897
2898 err = -EAGAIN;
364a9e93 2899 skb = sock_dequeue_err_skb(sk);
cb820f8e
RC		 2900 if (skb == NULL)
2901 goto out;
2902
2903 copied = skb->len;
2904 if (copied > len) {
2905 msg->msg_flags |= MSG_TRUNC;
2906 copied = len;
2907 }
51f3d02b 2908 err = skb_copy_datagram_msg(skb, 0, msg, copied);
cb820f8e
RC		 2909 if (err)
2910 goto out_free_skb;
2911
2912 sock_recv_timestamp(msg, sk, skb);
2913
2914 serr = SKB_EXT_ERR(skb);
2915 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
2916
2917 msg->msg_flags |= MSG_ERRQUEUE;
2918 err = copied;
2919
cb820f8e
RC		 2920out_free_skb:
2921 kfree_skb(skb);
2922out:
2923 return err;
2924}
2925EXPORT_SYMBOL(sock_recv_errqueue);
2926
1da177e4
LT		 2927/*
2928 * Get a socket option on an socket.
2929 *
2930 * FIX: POSIX 1003.1g is very ambiguous here. It states that
2931 * asynchronous errors should be reported by getsockopt. We assume
2932 * this means if you specify SO_ERROR (otherwise whats the point of it).
2933 */
2934int sock_common_getsockopt(struct socket *sock, int level, int optname,
2935 char __user *optval, int __user *optlen)
2936{
2937 struct sock *sk = sock->sk;
2938
2939 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2940}
1da177e4
LT		 2941EXPORT_SYMBOL(sock_common_getsockopt);
2942
3fdadf7d 2943#ifdef CONFIG_COMPAT
543d9cfe
ACM		 2944int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
2945 char __user *optval, int __user *optlen)
3fdadf7d
DM		 2946{
2947 struct sock *sk = sock->sk;
2948
1e51f951 2949 if (sk->sk_prot->compat_getsockopt != NULL)
543d9cfe
ACM		 2950 return sk->sk_prot->compat_getsockopt(sk, level, optname,
2951 optval, optlen);
3fdadf7d
DM		 2952 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2953}
2954EXPORT_SYMBOL(compat_sock_common_getsockopt);
2955#endif
2956
1b784140
YX		 2957int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
2958 int flags)
1da177e4
LT		 2959{
2960 struct sock *sk = sock->sk;
2961 int addr_len = 0;
2962 int err;
2963
1b784140 2964 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
1da177e4
LT		 2965 flags & ~MSG_DONTWAIT, &addr_len);
2966 if (err >= 0)
2967 msg->msg_namelen = addr_len;
2968 return err;
2969}
1da177e4
LT		 2970EXPORT_SYMBOL(sock_common_recvmsg);
2971
2972/*
2973 * Set socket options on an inet socket.
2974 */
2975int sock_common_setsockopt(struct socket *sock, int level, int optname,
b7058842 2976 char __user *optval, unsigned int optlen)
1da177e4
LT		 2977{
2978 struct sock *sk = sock->sk;
2979
2980 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2981}
1da177e4
LT		 2982EXPORT_SYMBOL(sock_common_setsockopt);
2983
3fdadf7d 2984#ifdef CONFIG_COMPAT
543d9cfe 2985int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
b7058842 2986 char __user *optval, unsigned int optlen)
3fdadf7d
DM		 2987{
2988 struct sock *sk = sock->sk;
2989
543d9cfe
ACM		 2990 if (sk->sk_prot->compat_setsockopt != NULL)
2991 return sk->sk_prot->compat_setsockopt(sk, level, optname,
2992 optval, optlen);
3fdadf7d
DM		 2993 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2994}
2995EXPORT_SYMBOL(compat_sock_common_setsockopt);
2996#endif
2997
1da177e4
LT		 2998void sk_common_release(struct sock *sk)
2999{
3000 if (sk->sk_prot->destroy)
3001 sk->sk_prot->destroy(sk);
3002
3003 /*
3004 * Observation: when sock_common_release is called, processes have
3005 * no access to socket. But net still has.
3006 * Step one, detach it from networking:
3007 *
3008 * A. Remove from hash tables.
3009 */
3010
3011 sk->sk_prot->unhash(sk);
3012
3013 /*
3014 * In this point socket cannot receive new packets, but it is possible
3015 * that some packets are in flight because some CPU runs receiver and
3016 * did hash table lookup before we unhashed socket. They will achieve
3017 * receive queue and will be purged by socket destructor.
3018 *
3019 * Also we still have packets pending on receive queue and probably,
3020 * our own packets waiting in device queues. sock_destroy will drain
3021 * receive queue, but transmitted packets will delay socket destruction
3022 * until the last reference will be released.
3023 */
3024
3025 sock_orphan(sk);
3026
3027 xfrm_sk_free_policy(sk);
3028
e6848976 3029 sk_refcnt_debug_release(sk);
5640f768 3030
1da177e4
LT		 3031 sock_put(sk);
3032}
1da177e4
LT		 3033EXPORT_SYMBOL(sk_common_release);
3034
a2d133b1
JH		 3035void sk_get_meminfo(const struct sock *sk, u32 *mem)
3036{
3037 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
3038
3039 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
3040 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf;
3041 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
3042 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf;
3043 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
3044 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued;
3045 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
3046 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len;
3047 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
3048}
3049
13ff3d6f
PE		 3050#ifdef CONFIG_PROC_FS
3051#define PROTO_INUSE_NR 64 /* should be enough for the first time */
1338d466
PE		 3052struct prot_inuse {
3053 int val[PROTO_INUSE_NR];
3054};
13ff3d6f
PE		 3055
3056static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
70ee1159 3057
70ee1159
PE		 3058void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
3059{
08fc7f81 3060 __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val);
70ee1159
PE		 3061}
3062EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
3063
3064int sock_prot_inuse_get(struct net *net, struct proto *prot)
3065{
3066 int cpu, idx = prot->inuse_idx;
3067 int res = 0;
3068
3069 for_each_possible_cpu(cpu)
08fc7f81 3070 res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
70ee1159
PE		 3071
3072 return res >= 0 ? res : 0;
3073}
3074EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
3075
648845ab
TZ		 3076static void sock_inuse_add(struct net *net, int val)
3077{
3078 this_cpu_add(*net->core.sock_inuse, val);
3079}
3080
3081int sock_inuse_get(struct net *net)
3082{
3083 int cpu, res = 0;
3084
3085 for_each_possible_cpu(cpu)
3086 res += *per_cpu_ptr(net->core.sock_inuse, cpu);
3087
3088 return res;
3089}
3090
3091EXPORT_SYMBOL_GPL(sock_inuse_get);
3092
2c8c1e72 3093static int __net_init sock_inuse_init_net(struct net *net)
70ee1159 3094{
08fc7f81 3095 net->core.prot_inuse = alloc_percpu(struct prot_inuse);
648845ab
TZ		 3096 if (net->core.prot_inuse == NULL)
3097 return -ENOMEM;
3098
3099 net->core.sock_inuse = alloc_percpu(int);
3100 if (net->core.sock_inuse == NULL)
3101 goto out;
3102
3103 return 0;
3104
3105out:
3106 free_percpu(net->core.prot_inuse);
3107 return -ENOMEM;
70ee1159
PE		 3108}
3109
2c8c1e72 3110static void __net_exit sock_inuse_exit_net(struct net *net)
70ee1159 3111{
08fc7f81 3112 free_percpu(net->core.prot_inuse);
648845ab 3113 free_percpu(net->core.sock_inuse);
70ee1159
PE		 3114}
3115
3116static struct pernet_operations net_inuse_ops = {
3117 .init = sock_inuse_init_net,
3118 .exit = sock_inuse_exit_net,
3119};
3120
3121static __init int net_inuse_init(void)
3122{
3123 if (register_pernet_subsys(&net_inuse_ops))
3124 panic("Cannot initialize net inuse counters");
3125
3126 return 0;
3127}
3128
3129core_initcall(net_inuse_init);
13ff3d6f
PE		 3130
3131static void assign_proto_idx(struct proto *prot)
3132{
3133 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3134
3135 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
e005d193 3136 pr_err("PROTO_INUSE_NR exhausted\n");
13ff3d6f
PE		 3137 return;
3138 }
3139
3140 set_bit(prot->inuse_idx, proto_inuse_idx);
3141}
3142
3143static void release_proto_idx(struct proto *prot)
3144{
3145 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3146 clear_bit(prot->inuse_idx, proto_inuse_idx);
3147}
3148#else
3149static inline void assign_proto_idx(struct proto *prot)
3150{
3151}
3152
3153static inline void release_proto_idx(struct proto *prot)
3154{
3155}
648845ab
TZ		 3156
3157static void sock_inuse_add(struct net *net, int val)
3158{
3159}
13ff3d6f
PE		 3160#endif
3161
0159dfd3
ED		 3162static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3163{
3164 if (!rsk_prot)
3165 return;
3166 kfree(rsk_prot->slab_name);
3167 rsk_prot->slab_name = NULL;
adf78eda
JL		 3168 kmem_cache_destroy(rsk_prot->slab);
3169 rsk_prot->slab = NULL;
0159dfd3
ED		 3170}
3171
3172static int req_prot_init(const struct proto *prot)
3173{
3174 struct request_sock_ops *rsk_prot = prot->rsk_prot;
3175
3176 if (!rsk_prot)
3177 return 0;
3178
3179 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3180 prot->name);
3181 if (!rsk_prot->slab_name)
3182 return -ENOMEM;
3183
3184 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3185 rsk_prot->obj_size, 0,
e699e2c6
SB		 3186 SLAB_ACCOUNT | prot->slab_flags,
3187 NULL);
0159dfd3
ED		 3188
3189 if (!rsk_prot->slab) {
3190 pr_crit("%s: Can't create request sock SLAB cache!\n",
3191 prot->name);
3192 return -ENOMEM;
3193 }
3194 return 0;
3195}
3196
b733c007
PE		 3197int proto_register(struct proto *prot, int alloc_slab)
3198{
1da177e4 3199 if (alloc_slab) {
30c2c9f1
DW		 3200 prot->slab = kmem_cache_create_usercopy(prot->name,
3201 prot->obj_size, 0,
e699e2c6
SB		 3202 SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
3203 prot->slab_flags,
289a4860 3204 prot->useroffset, prot->usersize,
271b72c7 3205 NULL);
1da177e4
LT		 3206
3207 if (prot->slab == NULL) {
e005d193
JP		 3208 pr_crit("%s: Can't create sock SLAB cache!\n",
3209 prot->name);
60e7663d 3210 goto out;
1da177e4 3211 }
2e6599cb 3212
0159dfd3
ED		 3213 if (req_prot_init(prot))
3214 goto out_free_request_sock_slab;
8feaf0c0 3215
6d6ee43e 3216 if (prot->twsk_prot != NULL) {
faf23422 3217 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
8feaf0c0 3218
7e56b5d6 3219 if (prot->twsk_prot->twsk_slab_name == NULL)
8feaf0c0
ACM		 3220 goto out_free_request_sock_slab;
3221
6d6ee43e 3222 prot->twsk_prot->twsk_slab =
7e56b5d6 3223 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
6d6ee43e 3224 prot->twsk_prot->twsk_obj_size,
3ab5aee7 3225 0,
e699e2c6 3226 SLAB_ACCOUNT |
52db70dc 3227 prot->slab_flags,
20c2df83 3228 NULL);
6d6ee43e 3229 if (prot->twsk_prot->twsk_slab == NULL)
8feaf0c0
ACM		 3230 goto out_free_timewait_sock_slab_name;
3231 }
1da177e4
LT		 3232 }
3233
36b77a52 3234 mutex_lock(&proto_list_mutex);
1da177e4 3235 list_add(&prot->node, &proto_list);
13ff3d6f 3236 assign_proto_idx(prot);
36b77a52 3237 mutex_unlock(&proto_list_mutex);
b733c007
PE		 3238 return 0;
3239
8feaf0c0 3240out_free_timewait_sock_slab_name:
7e56b5d6 3241 kfree(prot->twsk_prot->twsk_slab_name);
8feaf0c0 3242out_free_request_sock_slab:
0159dfd3
ED		 3243 req_prot_cleanup(prot->rsk_prot);
3244
2e6599cb
ACM		 3245 kmem_cache_destroy(prot->slab);
3246 prot->slab = NULL;
b733c007
PE		 3247out:
3248 return -ENOBUFS;
1da177e4 3249}
1da177e4
LT		 3250EXPORT_SYMBOL(proto_register);
3251
3252void proto_unregister(struct proto *prot)
3253{
36b77a52 3254 mutex_lock(&proto_list_mutex);
13ff3d6f 3255 release_proto_idx(prot);
0a3f4358 3256 list_del(&prot->node);
36b77a52 3257 mutex_unlock(&proto_list_mutex);
1da177e4 3258
adf78eda
JL		 3259 kmem_cache_destroy(prot->slab);
3260 prot->slab = NULL;
1da177e4 3261
0159dfd3 3262 req_prot_cleanup(prot->rsk_prot);
2e6599cb 3263
6d6ee43e 3264 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
6d6ee43e 3265 kmem_cache_destroy(prot->twsk_prot->twsk_slab);
7e56b5d6 3266 kfree(prot->twsk_prot->twsk_slab_name);
6d6ee43e 3267 prot->twsk_prot->twsk_slab = NULL;
8feaf0c0 3268 }
1da177e4 3269}
1da177e4
LT		 3270EXPORT_SYMBOL(proto_unregister);
3271
bf2ae2e4
XL		 3272int sock_load_diag_module(int family, int protocol)
3273{
3274 if (!protocol) {
3275 if (!sock_is_registered(family))
3276 return -ENOENT;
3277
3278 return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
3279 NETLINK_SOCK_DIAG, family);
3280 }
3281
3282#ifdef CONFIG_INET
3283 if (family == AF_INET &&
c34c1287 3284 protocol != IPPROTO_RAW &&
bf2ae2e4
XL		 3285 !rcu_access_pointer(inet_protos[protocol]))
3286 return -ENOENT;
3287#endif
3288
3289 return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
3290 NETLINK_SOCK_DIAG, family, protocol);
3291}
3292EXPORT_SYMBOL(sock_load_diag_module);
3293
1da177e4 3294#ifdef CONFIG_PROC_FS
1da177e4 3295static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
36b77a52 3296 __acquires(proto_list_mutex)
1da177e4 3297{
36b77a52 3298 mutex_lock(&proto_list_mutex);
60f0438a 3299 return seq_list_start_head(&proto_list, *pos);
1da177e4
LT		 3300}
3301
3302static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3303{
60f0438a 3304 return seq_list_next(v, &proto_list, pos);
1da177e4
LT		 3305}
3306
3307static void proto_seq_stop(struct seq_file *seq, void *v)
36b77a52 3308 __releases(proto_list_mutex)
1da177e4 3309{
36b77a52 3310 mutex_unlock(&proto_list_mutex);
1da177e4
LT		 3311}
3312
3313static char proto_method_implemented(const void *method)
3314{
3315 return method == NULL ? 'n' : 'y';
3316}
180d8cd9
GC		 3317static long sock_prot_memory_allocated(struct proto *proto)
3318{
cb75a36c 3319 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
180d8cd9
GC		 3320}
3321
3322static char *sock_prot_memory_pressure(struct proto *proto)
3323{
3324 return proto->memory_pressure != NULL ?
3325 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3326}
1da177e4
LT		 3327
3328static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3329{
180d8cd9 3330
8d987e5c 3331 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
1da177e4
LT		 3332 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3333 proto->name,
3334 proto->obj_size,
14e943db 3335 sock_prot_inuse_get(seq_file_net(seq), proto),
180d8cd9
GC		 3336 sock_prot_memory_allocated(proto),
3337 sock_prot_memory_pressure(proto),
1da177e4
LT		 3338 proto->max_header,
3339 proto->slab == NULL ? "no" : "yes",
3340 module_name(proto->owner),
3341 proto_method_implemented(proto->close),
3342 proto_method_implemented(proto->connect),
3343 proto_method_implemented(proto->disconnect),
3344 proto_method_implemented(proto->accept),
3345 proto_method_implemented(proto->ioctl),
3346 proto_method_implemented(proto->init),
3347 proto_method_implemented(proto->destroy),
3348 proto_method_implemented(proto->shutdown),
3349 proto_method_implemented(proto->setsockopt),
3350 proto_method_implemented(proto->getsockopt),
3351 proto_method_implemented(proto->sendmsg),
3352 proto_method_implemented(proto->recvmsg),
3353 proto_method_implemented(proto->sendpage),
3354 proto_method_implemented(proto->bind),
3355 proto_method_implemented(proto->backlog_rcv),
3356 proto_method_implemented(proto->hash),
3357 proto_method_implemented(proto->unhash),
3358 proto_method_implemented(proto->get_port),
3359 proto_method_implemented(proto->enter_memory_pressure));
3360}
3361
3362static int proto_seq_show(struct seq_file *seq, void *v)
3363{
60f0438a 3364 if (v == &proto_list)
1da177e4
LT		 3365 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3366 "protocol",
3367 "size",
3368 "sockets",
3369 "memory",
3370 "press",
3371 "maxhdr",
3372 "slab",
3373 "module",
3374 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3375 else
60f0438a 3376 proto_seq_printf(seq, list_entry(v, struct proto, node));
1da177e4
LT		 3377 return 0;
3378}
3379
f690808e 3380static const struct seq_operations proto_seq_ops = {
1da177e4
LT		 3381 .start = proto_seq_start,
3382 .next = proto_seq_next,
3383 .stop = proto_seq_stop,
3384 .show = proto_seq_show,
3385};
3386
14e943db
ED		 3387static __net_init int proto_init_net(struct net *net)
3388{
c3506372
CH		 3389 if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
3390 sizeof(struct seq_net_private)))
14e943db
ED		 3391 return -ENOMEM;
3392
3393 return 0;
3394}
3395
3396static __net_exit void proto_exit_net(struct net *net)
3397{
ece31ffd 3398 remove_proc_entry("protocols", net->proc_net);
14e943db
ED		 3399}
3400
3401
3402static __net_initdata struct pernet_operations proto_net_ops = {
3403 .init = proto_init_net,
3404 .exit = proto_exit_net,
1da177e4
LT		 3405};
3406
3407static int __init proto_init(void)
3408{
14e943db 3409 return register_pernet_subsys(&proto_net_ops);
1da177e4
LT		 3410}
3411
3412subsys_initcall(proto_init);
3413
3414#endif /* PROC_FS */
7db6b048
SS		 3415
3416#ifdef CONFIG_NET_RX_BUSY_POLL
3417bool sk_busy_loop_end(void *p, unsigned long start_time)
3418{
3419 struct sock *sk = p;
3420
3421 return !skb_queue_empty(&sk->sk_receive_queue) ||
3422 sk_busy_loop_timeout(sk, start_time);
3423}
3424EXPORT_SYMBOL(sk_busy_loop_end);
3425#endif /* CONFIG_NET_RX_BUSY_POLL */
Linux 6.x block layer and io_uring tree(s)