projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'mlxsw-gre-fixes'
[linux-2.6-block.git] / net / core / sock.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic socket support routines. Memory allocators, socket lock/release
7 * handler for protocols to use and generic option handler.
8 *
9 *
02c30a84 10 * Authors: Ross Biro
1da177e4
LT		 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
4ec93edb 35 * code. The ACK stuff can wait and needs major
1da177e4
LT		 36 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
84 *
85 *
86 * This program is free software; you can redistribute it and/or
87 * modify it under the terms of the GNU General Public License
88 * as published by the Free Software Foundation; either version
89 * 2 of the License, or (at your option) any later version.
90 */
91
e005d193
JP		 92#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
93
4fc268d2 94#include <linux/capability.h>
1da177e4 95#include <linux/errno.h>
cb820f8e 96#include <linux/errqueue.h>
1da177e4
LT		 97#include <linux/types.h>
98#include <linux/socket.h>
99#include <linux/in.h>
100#include <linux/kernel.h>
1da177e4
LT		 101#include <linux/module.h>
102#include <linux/proc_fs.h>
103#include <linux/seq_file.h>
104#include <linux/sched.h>
f1083048 105#include <linux/sched/mm.h>
1da177e4
LT		 106#include <linux/timer.h>
107#include <linux/string.h>
108#include <linux/sockios.h>
109#include <linux/net.h>
110#include <linux/mm.h>
111#include <linux/slab.h>
112#include <linux/interrupt.h>
113#include <linux/poll.h>
114#include <linux/tcp.h>
115#include <linux/init.h>
a1f8e7f7 116#include <linux/highmem.h>
3f551f94 117#include <linux/user_namespace.h>
c5905afb 118#include <linux/static_key.h>
3969eb38 119#include <linux/memcontrol.h>
8c1ae10d 120#include <linux/prefetch.h>
1da177e4 121
7c0f6ba6 122#include <linux/uaccess.h>
1da177e4
LT		 123
124#include <linux/netdevice.h>
125#include <net/protocol.h>
126#include <linux/skbuff.h>
457c4cbc 127#include <net/net_namespace.h>
2e6599cb 128#include <net/request_sock.h>
1da177e4 129#include <net/sock.h>
20d49473 130#include <linux/net_tstamp.h>
1da177e4
LT		 131#include <net/xfrm.h>
132#include <linux/ipsec.h>
f8451725 133#include <net/cls_cgroup.h>
5bc1421e 134#include <net/netprio_cgroup.h>
eb4cb008 135#include <linux/sock_diag.h>
1da177e4
LT		 136
137#include <linux/filter.h>
538950a1 138#include <net/sock_reuseport.h>
1da177e4 139
3847ce32
SM		 140#include <trace/events/sock.h>
141
1da177e4 142#include <net/tcp.h>
076bb0c8 143#include <net/busy_poll.h>
06021292 144
36b77a52 145static DEFINE_MUTEX(proto_list_mutex);
d1a4c0b3
GC		 146static LIST_HEAD(proto_list);
147
a3b299da
EB		 148/**
149 * sk_ns_capable - General socket capability test
150 * @sk: Socket to use a capability on or through
151 * @user_ns: The user namespace of the capability to use
152 * @cap: The capability to use
153 *
154 * Test to see if the opener of the socket had when the socket was
155 * created and the current process has the capability @cap in the user
156 * namespace @user_ns.
157 */
158bool sk_ns_capable(const struct sock *sk,
159 struct user_namespace *user_ns, int cap)
160{
161 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
162 ns_capable(user_ns, cap);
163}
164EXPORT_SYMBOL(sk_ns_capable);
165
166/**
167 * sk_capable - Socket global capability test
168 * @sk: Socket to use a capability on or through
e793c0f7 169 * @cap: The global capability to use
a3b299da
EB		 170 *
171 * Test to see if the opener of the socket had when the socket was
172 * created and the current process has the capability @cap in all user
173 * namespaces.
174 */
175bool sk_capable(const struct sock *sk, int cap)
176{
177 return sk_ns_capable(sk, &init_user_ns, cap);
178}
179EXPORT_SYMBOL(sk_capable);
180
181/**
182 * sk_net_capable - Network namespace socket capability test
183 * @sk: Socket to use a capability on or through
184 * @cap: The capability to use
185 *
e793c0f7 186 * Test to see if the opener of the socket had when the socket was created
a3b299da
EB		 187 * and the current process has the capability @cap over the network namespace
188 * the socket is a member of.
189 */
190bool sk_net_capable(const struct sock *sk, int cap)
191{
192 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
193}
194EXPORT_SYMBOL(sk_net_capable);
195
da21f24d
IM		 196/*
197 * Each address family might have different locking rules, so we have
cdfbabfb
DH		 198 * one slock key per address family and separate keys for internal and
199 * userspace sockets.
da21f24d 200 */
a5b5bb9a 201static struct lock_class_key af_family_keys[AF_MAX];
cdfbabfb 202static struct lock_class_key af_family_kern_keys[AF_MAX];
a5b5bb9a 203static struct lock_class_key af_family_slock_keys[AF_MAX];
cdfbabfb 204static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
a5b5bb9a 205
a5b5bb9a
IM		 206/*
207 * Make lock validator output more readable. (we pre-construct these
208 * strings build-time, so that runtime initialization of socket
209 * locks is fast):
210 */
cdfbabfb
DH		 211
212#define _sock_locks(x) \
213 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \
214 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \
215 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \
216 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \
217 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \
218 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \
219 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \
220 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \
221 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \
222 x "27" , x "28" , x "AF_CAN" , \
223 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \
224 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
225 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
226 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
227 x "AF_QIPCRTR", x "AF_SMC" , x "AF_MAX"
228
36cbd3dc 229static const char *const af_family_key_strings[AF_MAX+1] = {
cdfbabfb 230 _sock_locks("sk_lock-")
a5b5bb9a 231};
36cbd3dc 232static const char *const af_family_slock_key_strings[AF_MAX+1] = {
cdfbabfb 233 _sock_locks("slock-")
a5b5bb9a 234};
36cbd3dc 235static const char *const af_family_clock_key_strings[AF_MAX+1] = {
cdfbabfb
DH		 236 _sock_locks("clock-")
237};
238
239static const char *const af_family_kern_key_strings[AF_MAX+1] = {
240 _sock_locks("k-sk_lock-")
241};
242static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
243 _sock_locks("k-slock-")
244};
245static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
246 _sock_locks("k-clock-")
443aef0e 247};
581319c5
PA		 248static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
249 "rlock-AF_UNSPEC", "rlock-AF_UNIX" , "rlock-AF_INET" ,
250 "rlock-AF_AX25" , "rlock-AF_IPX" , "rlock-AF_APPLETALK",
251 "rlock-AF_NETROM", "rlock-AF_BRIDGE" , "rlock-AF_ATMPVC" ,
252 "rlock-AF_X25" , "rlock-AF_INET6" , "rlock-AF_ROSE" ,
253 "rlock-AF_DECnet", "rlock-AF_NETBEUI" , "rlock-AF_SECURITY" ,
254 "rlock-AF_KEY" , "rlock-AF_NETLINK" , "rlock-AF_PACKET" ,
255 "rlock-AF_ASH" , "rlock-AF_ECONET" , "rlock-AF_ATMSVC" ,
256 "rlock-AF_RDS" , "rlock-AF_SNA" , "rlock-AF_IRDA" ,
257 "rlock-AF_PPPOX" , "rlock-AF_WANPIPE" , "rlock-AF_LLC" ,
258 "rlock-27" , "rlock-28" , "rlock-AF_CAN" ,
259 "rlock-AF_TIPC" , "rlock-AF_BLUETOOTH", "rlock-AF_IUCV" ,
260 "rlock-AF_RXRPC" , "rlock-AF_ISDN" , "rlock-AF_PHONET" ,
261 "rlock-AF_IEEE802154", "rlock-AF_CAIF" , "rlock-AF_ALG" ,
262 "rlock-AF_NFC" , "rlock-AF_VSOCK" , "rlock-AF_KCM" ,
263 "rlock-AF_QIPCRTR", "rlock-AF_SMC" , "rlock-AF_MAX"
264};
265static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
266 "wlock-AF_UNSPEC", "wlock-AF_UNIX" , "wlock-AF_INET" ,
267 "wlock-AF_AX25" , "wlock-AF_IPX" , "wlock-AF_APPLETALK",
268 "wlock-AF_NETROM", "wlock-AF_BRIDGE" , "wlock-AF_ATMPVC" ,
269 "wlock-AF_X25" , "wlock-AF_INET6" , "wlock-AF_ROSE" ,
270 "wlock-AF_DECnet", "wlock-AF_NETBEUI" , "wlock-AF_SECURITY" ,
271 "wlock-AF_KEY" , "wlock-AF_NETLINK" , "wlock-AF_PACKET" ,
272 "wlock-AF_ASH" , "wlock-AF_ECONET" , "wlock-AF_ATMSVC" ,
273 "wlock-AF_RDS" , "wlock-AF_SNA" , "wlock-AF_IRDA" ,
274 "wlock-AF_PPPOX" , "wlock-AF_WANPIPE" , "wlock-AF_LLC" ,
275 "wlock-27" , "wlock-28" , "wlock-AF_CAN" ,
276 "wlock-AF_TIPC" , "wlock-AF_BLUETOOTH", "wlock-AF_IUCV" ,
277 "wlock-AF_RXRPC" , "wlock-AF_ISDN" , "wlock-AF_PHONET" ,
278 "wlock-AF_IEEE802154", "wlock-AF_CAIF" , "wlock-AF_ALG" ,
279 "wlock-AF_NFC" , "wlock-AF_VSOCK" , "wlock-AF_KCM" ,
280 "wlock-AF_QIPCRTR", "wlock-AF_SMC" , "wlock-AF_MAX"
281};
282static const char *const af_family_elock_key_strings[AF_MAX+1] = {
283 "elock-AF_UNSPEC", "elock-AF_UNIX" , "elock-AF_INET" ,
284 "elock-AF_AX25" , "elock-AF_IPX" , "elock-AF_APPLETALK",
285 "elock-AF_NETROM", "elock-AF_BRIDGE" , "elock-AF_ATMPVC" ,
286 "elock-AF_X25" , "elock-AF_INET6" , "elock-AF_ROSE" ,
287 "elock-AF_DECnet", "elock-AF_NETBEUI" , "elock-AF_SECURITY" ,
288 "elock-AF_KEY" , "elock-AF_NETLINK" , "elock-AF_PACKET" ,
289 "elock-AF_ASH" , "elock-AF_ECONET" , "elock-AF_ATMSVC" ,
290 "elock-AF_RDS" , "elock-AF_SNA" , "elock-AF_IRDA" ,
291 "elock-AF_PPPOX" , "elock-AF_WANPIPE" , "elock-AF_LLC" ,
292 "elock-27" , "elock-28" , "elock-AF_CAN" ,
293 "elock-AF_TIPC" , "elock-AF_BLUETOOTH", "elock-AF_IUCV" ,
294 "elock-AF_RXRPC" , "elock-AF_ISDN" , "elock-AF_PHONET" ,
295 "elock-AF_IEEE802154", "elock-AF_CAIF" , "elock-AF_ALG" ,
296 "elock-AF_NFC" , "elock-AF_VSOCK" , "elock-AF_KCM" ,
297 "elock-AF_QIPCRTR", "elock-AF_SMC" , "elock-AF_MAX"
298};
da21f24d
IM		 299
300/*
581319c5 301 * sk_callback_lock and sk queues locking rules are per-address-family,
da21f24d
IM		 302 * so split the lock classes by using a per-AF key:
303 */
304static struct lock_class_key af_callback_keys[AF_MAX];
581319c5
PA		 305static struct lock_class_key af_rlock_keys[AF_MAX];
306static struct lock_class_key af_wlock_keys[AF_MAX];
307static struct lock_class_key af_elock_keys[AF_MAX];
cdfbabfb 308static struct lock_class_key af_kern_callback_keys[AF_MAX];
da21f24d 309
1da177e4 310/* Run time adjustable parameters. */
ab32ea5d 311__u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
6d8ebc8a 312EXPORT_SYMBOL(sysctl_wmem_max);
ab32ea5d 313__u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
6d8ebc8a 314EXPORT_SYMBOL(sysctl_rmem_max);
ab32ea5d
BH		 315__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
316__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
1da177e4 317
25985edc 318/* Maximal space eaten by iovec or ancillary data plus some space */
ab32ea5d 319int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
2a91525c 320EXPORT_SYMBOL(sysctl_optmem_max);
1da177e4 321
b245be1f
WB		 322int sysctl_tstamp_allow_data __read_mostly = 1;
323
c93bdd0e
MG		 324struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE;
325EXPORT_SYMBOL_GPL(memalloc_socks);
326
7cb02404
MG		 327/**
328 * sk_set_memalloc - sets %SOCK_MEMALLOC
329 * @sk: socket to set it on
330 *
331 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
332 * It's the responsibility of the admin to adjust min_free_kbytes
333 * to meet the requirements
334 */
335void sk_set_memalloc(struct sock *sk)
336{
337 sock_set_flag(sk, SOCK_MEMALLOC);
338 sk->sk_allocation |= __GFP_MEMALLOC;
c93bdd0e 339 static_key_slow_inc(&memalloc_socks);
7cb02404
MG		 340}
341EXPORT_SYMBOL_GPL(sk_set_memalloc);
342
343void sk_clear_memalloc(struct sock *sk)
344{
345 sock_reset_flag(sk, SOCK_MEMALLOC);
346 sk->sk_allocation &= ~__GFP_MEMALLOC;
c93bdd0e 347 static_key_slow_dec(&memalloc_socks);
c76562b6
MG		 348
349 /*
350 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
5d753610
MG		 351 * progress of swapping. SOCK_MEMALLOC may be cleared while
352 * it has rmem allocations due to the last swapfile being deactivated
353 * but there is a risk that the socket is unusable due to exceeding
354 * the rmem limits. Reclaim the reserves and obey rmem limits again.
c76562b6 355 */
5d753610 356 sk_mem_reclaim(sk);
7cb02404
MG		 357}
358EXPORT_SYMBOL_GPL(sk_clear_memalloc);
359
b4b9e355
MG		 360int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
361{
362 int ret;
f1083048 363 unsigned int noreclaim_flag;
b4b9e355
MG		 364
365 /* these should have been dropped before queueing */
366 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
367
f1083048 368 noreclaim_flag = memalloc_noreclaim_save();
b4b9e355 369 ret = sk->sk_backlog_rcv(sk, skb);
f1083048 370 memalloc_noreclaim_restore(noreclaim_flag);
b4b9e355
MG		 371
372 return ret;
373}
374EXPORT_SYMBOL(__sk_backlog_rcv);
375
1da177e4
LT		 376static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
377{
378 struct timeval tv;
379
380 if (optlen < sizeof(tv))
381 return -EINVAL;
382 if (copy_from_user(&tv, optval, sizeof(tv)))
383 return -EFAULT;
ba78073e
VA		 384 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
385 return -EDOM;
1da177e4 386
ba78073e 387 if (tv.tv_sec < 0) {
6f11df83
AM		 388 static int warned __read_mostly;
389
ba78073e 390 *timeo_p = 0;
50aab54f 391 if (warned < 10 && net_ratelimit()) {
ba78073e 392 warned++;
e005d193
JP		 393 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
394 __func__, current->comm, task_pid_nr(current));
50aab54f 395 }
ba78073e
VA		 396 return 0;
397 }
1da177e4
LT		 398 *timeo_p = MAX_SCHEDULE_TIMEOUT;
399 if (tv.tv_sec == 0 && tv.tv_usec == 0)
400 return 0;
401 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1))
8ccde4c5 402 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ);
1da177e4
LT		 403 return 0;
404}
405
406static void sock_warn_obsolete_bsdism(const char *name)
407{
408 static int warned;
409 static char warncomm[TASK_COMM_LEN];
4ec93edb
YH		 410 if (strcmp(warncomm, current->comm) && warned < 5) {
411 strcpy(warncomm, current->comm);
e005d193
JP		 412 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
413 warncomm, name);
1da177e4
LT		 414 warned++;
415 }
416}
417
080a270f
HFS		 418static bool sock_needs_netstamp(const struct sock *sk)
419{
420 switch (sk->sk_family) {
421 case AF_UNSPEC:
422 case AF_UNIX:
423 return false;
424 default:
425 return true;
426 }
427}
428
08e29af3 429static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
4ec93edb 430{
08e29af3
ED		 431 if (sk->sk_flags & flags) {
432 sk->sk_flags &= ~flags;
080a270f
HFS		 433 if (sock_needs_netstamp(sk) &&
434 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
20d49473 435 net_disable_timestamp();
1da177e4
LT		 436 }
437}
438
439
e6afc8ac 440int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
f0088a50 441{
3b885787
NH		 442 unsigned long flags;
443 struct sk_buff_head *list = &sk->sk_receive_queue;
f0088a50 444
0fd7bac6 445 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
766e9037 446 atomic_inc(&sk->sk_drops);
3847ce32 447 trace_sock_rcvqueue_full(sk, skb);
766e9037 448 return -ENOMEM;
f0088a50
DV		 449 }
450
c76562b6 451 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
766e9037
ED		 452 atomic_inc(&sk->sk_drops);
453 return -ENOBUFS;
3ab224be
HA		 454 }
455
f0088a50
DV		 456 skb->dev = NULL;
457 skb_set_owner_r(skb, sk);
49ad9599 458
7fee226a
ED		 459 /* we escape from rcu protected region, make sure we dont leak
460 * a norefcounted dst
461 */
462 skb_dst_force(skb);
463
3b885787 464 spin_lock_irqsave(&list->lock, flags);
3bc3b96f 465 sock_skb_set_dropcount(sk, skb);
3b885787
NH		 466 __skb_queue_tail(list, skb);
467 spin_unlock_irqrestore(&list->lock, flags);
f0088a50
DV		 468
469 if (!sock_flag(sk, SOCK_DEAD))
676d2369 470 sk->sk_data_ready(sk);
766e9037 471 return 0;
f0088a50 472}
e6afc8ac 473EXPORT_SYMBOL(__sock_queue_rcv_skb);
474
475int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
476{
477 int err;
478
479 err = sk_filter(sk, skb);
480 if (err)
481 return err;
482
483 return __sock_queue_rcv_skb(sk, skb);
484}
f0088a50
DV		 485EXPORT_SYMBOL(sock_queue_rcv_skb);
486
4f0c40d9 487int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
c3f24cfb 488 const int nested, unsigned int trim_cap, bool refcounted)
f0088a50
DV		 489{
490 int rc = NET_RX_SUCCESS;
491
4f0c40d9 492 if (sk_filter_trim_cap(sk, skb, trim_cap))
f0088a50
DV		 493 goto discard_and_relse;
494
495 skb->dev = NULL;
496
274f482d 497 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
c377411f
ED		 498 atomic_inc(&sk->sk_drops);
499 goto discard_and_relse;
500 }
58a5a7b9
ACM		 501 if (nested)
502 bh_lock_sock_nested(sk);
503 else
504 bh_lock_sock(sk);
a5b5bb9a
IM		 505 if (!sock_owned_by_user(sk)) {
506 /*
507 * trylock + unlock semantics:
508 */
509 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
510
c57943a1 511 rc = sk_backlog_rcv(sk, skb);
a5b5bb9a
IM		 512
513 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
f545a38f 514 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
8eae939f
ZY		 515 bh_unlock_sock(sk);
516 atomic_inc(&sk->sk_drops);
517 goto discard_and_relse;
518 }
519
f0088a50
DV		 520 bh_unlock_sock(sk);
521out:
c3f24cfb
ED		 522 if (refcounted)
523 sock_put(sk);
f0088a50
DV		 524 return rc;
525discard_and_relse:
526 kfree_skb(skb);
527 goto out;
528}
4f0c40d9 529EXPORT_SYMBOL(__sk_receive_skb);
f0088a50
DV		 530
531struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
532{
b6c6712a 533 struct dst_entry *dst = __sk_dst_get(sk);
f0088a50
DV		 534
535 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
e022f0b4 536 sk_tx_queue_clear(sk);
9b8805a3 537 sk->sk_dst_pending_confirm = 0;
a9b3cd7f 538 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
f0088a50
DV		 539 dst_release(dst);
540 return NULL;
541 }
542
543 return dst;
544}
545EXPORT_SYMBOL(__sk_dst_check);
546
547struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
548{
549 struct dst_entry *dst = sk_dst_get(sk);
550
551 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
552 sk_dst_reset(sk);
553 dst_release(dst);
554 return NULL;
555 }
556
557 return dst;
558}
559EXPORT_SYMBOL(sk_dst_check);
560
c91f6df2
BH		 561static int sock_setbindtodevice(struct sock *sk, char __user *optval,
562 int optlen)
4878809f
DM		 563{
564 int ret = -ENOPROTOOPT;
565#ifdef CONFIG_NETDEVICES
3b1e0a65 566 struct net *net = sock_net(sk);
4878809f
DM		 567 char devname[IFNAMSIZ];
568 int index;
569
570 /* Sorry... */
571 ret = -EPERM;
5e1fccc0 572 if (!ns_capable(net->user_ns, CAP_NET_RAW))
4878809f
DM		 573 goto out;
574
575 ret = -EINVAL;
576 if (optlen < 0)
577 goto out;
578
579 /* Bind this socket to a particular device like "eth0",
580 * as specified in the passed interface name. If the
581 * name is "" or the option length is zero the socket
582 * is not bound.
583 */
584 if (optlen > IFNAMSIZ - 1)
585 optlen = IFNAMSIZ - 1;
586 memset(devname, 0, sizeof(devname));
587
588 ret = -EFAULT;
589 if (copy_from_user(devname, optval, optlen))
590 goto out;
591
000ba2e4
DM		 592 index = 0;
593 if (devname[0] != '\0') {
bf8e56bf 594 struct net_device *dev;
4878809f 595
bf8e56bf
ED		 596 rcu_read_lock();
597 dev = dev_get_by_name_rcu(net, devname);
598 if (dev)
599 index = dev->ifindex;
600 rcu_read_unlock();
4878809f
DM		 601 ret = -ENODEV;
602 if (!dev)
603 goto out;
4878809f
DM		 604 }
605
606 lock_sock(sk);
607 sk->sk_bound_dev_if = index;
608 sk_dst_reset(sk);
609 release_sock(sk);
610
611 ret = 0;
612
613out:
614#endif
615
616 return ret;
617}
618
c91f6df2
BH		 619static int sock_getbindtodevice(struct sock *sk, char __user *optval,
620 int __user *optlen, int len)
621{
622 int ret = -ENOPROTOOPT;
623#ifdef CONFIG_NETDEVICES
624 struct net *net = sock_net(sk);
c91f6df2 625 char devname[IFNAMSIZ];
c91f6df2
BH		 626
627 if (sk->sk_bound_dev_if == 0) {
628 len = 0;
629 goto zero;
630 }
631
632 ret = -EINVAL;
633 if (len < IFNAMSIZ)
634 goto out;
635
5dbe7c17
NS		 636 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
637 if (ret)
c91f6df2 638 goto out;
c91f6df2
BH		 639
640 len = strlen(devname) + 1;
641
642 ret = -EFAULT;
643 if (copy_to_user(optval, devname, len))
644 goto out;
645
646zero:
647 ret = -EFAULT;
648 if (put_user(len, optlen))
649 goto out;
650
651 ret = 0;
652
653out:
654#endif
655
656 return ret;
657}
658
c0ef877b
PE		 659static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool)
660{
661 if (valbool)
662 sock_set_flag(sk, bit);
663 else
664 sock_reset_flag(sk, bit);
665}
666
f60e5990 667bool sk_mc_loop(struct sock *sk)
668{
669 if (dev_recursion_level())
670 return false;
671 if (!sk)
672 return true;
673 switch (sk->sk_family) {
674 case AF_INET:
675 return inet_sk(sk)->mc_loop;
676#if IS_ENABLED(CONFIG_IPV6)
677 case AF_INET6:
678 return inet6_sk(sk)->mc_loop;
679#endif
680 }
681 WARN_ON(1);
682 return true;
683}
684EXPORT_SYMBOL(sk_mc_loop);
685
1da177e4
LT		 686/*
687 * This is meant for all protocols to use and covers goings on
688 * at the socket level. Everything here is generic.
689 */
690
691int sock_setsockopt(struct socket *sock, int level, int optname,
b7058842 692 char __user *optval, unsigned int optlen)
1da177e4 693{
2a91525c 694 struct sock *sk = sock->sk;
1da177e4
LT		 695 int val;
696 int valbool;
697 struct linger ling;
698 int ret = 0;
4ec93edb 699
1da177e4
LT		 700 /*
701 * Options without arguments
702 */
703
4878809f 704 if (optname == SO_BINDTODEVICE)
c91f6df2 705 return sock_setbindtodevice(sk, optval, optlen);
4878809f 706
e71a4783
SH		 707 if (optlen < sizeof(int))
708 return -EINVAL;
4ec93edb 709
1da177e4
LT		 710 if (get_user(val, (int __user *)optval))
711 return -EFAULT;
4ec93edb 712
2a91525c 713 valbool = val ? 1 : 0;
1da177e4
LT		 714
715 lock_sock(sk);
716
2a91525c 717 switch (optname) {
e71a4783 718 case SO_DEBUG:
2a91525c 719 if (val && !capable(CAP_NET_ADMIN))
e71a4783 720 ret = -EACCES;
2a91525c 721 else
c0ef877b 722 sock_valbool_flag(sk, SOCK_DBG, valbool);
e71a4783
SH		 723 break;
724 case SO_REUSEADDR:
4a17fd52 725 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
e71a4783 726 break;
055dc21a
TH		 727 case SO_REUSEPORT:
728 sk->sk_reuseport = valbool;
729 break;
e71a4783 730 case SO_TYPE:
49c794e9 731 case SO_PROTOCOL:
0d6038ee 732 case SO_DOMAIN:
e71a4783
SH		 733 case SO_ERROR:
734 ret = -ENOPROTOOPT;
735 break;
736 case SO_DONTROUTE:
c0ef877b 737 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
e71a4783
SH		 738 break;
739 case SO_BROADCAST:
740 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
741 break;
742 case SO_SNDBUF:
743 /* Don't error on this BSD doesn't and if you think
82981930
ED		 744 * about it this is right. Otherwise apps have to
745 * play 'guess the biggest size' games. RCVBUF/SNDBUF
746 * are treated in BSD as hints
747 */
748 val = min_t(u32, val, sysctl_wmem_max);
b0573dea 749set_sndbuf:
e71a4783 750 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
b98b0bc8 751 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);
82981930 752 /* Wake up sending tasks if we upped the value. */
e71a4783
SH		 753 sk->sk_write_space(sk);
754 break;
1da177e4 755
e71a4783
SH		 756 case SO_SNDBUFFORCE:
757 if (!capable(CAP_NET_ADMIN)) {
758 ret = -EPERM;
759 break;
760 }
761 goto set_sndbuf;
b0573dea 762
e71a4783
SH		 763 case SO_RCVBUF:
764 /* Don't error on this BSD doesn't and if you think
82981930
ED		 765 * about it this is right. Otherwise apps have to
766 * play 'guess the biggest size' games. RCVBUF/SNDBUF
767 * are treated in BSD as hints
768 */
769 val = min_t(u32, val, sysctl_rmem_max);
b0573dea 770set_rcvbuf:
e71a4783
SH		 771 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
772 /*
773 * We double it on the way in to account for
774 * "struct sk_buff" etc. overhead. Applications
775 * assume that the SO_RCVBUF setting they make will
776 * allow that much actual data to be received on that
777 * socket.
778 *
779 * Applications are unaware that "struct sk_buff" and
780 * other overheads allocate from the receive buffer
781 * during socket buffer allocation.
782 *
783 * And after considering the possible alternatives,
784 * returning the value we actually used in getsockopt
785 * is the most desirable behavior.
786 */
b98b0bc8 787 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF);
e71a4783
SH		 788 break;
789
790 case SO_RCVBUFFORCE:
791 if (!capable(CAP_NET_ADMIN)) {
792 ret = -EPERM;
1da177e4 793 break;
e71a4783
SH		 794 }
795 goto set_rcvbuf;
1da177e4 796
e71a4783 797 case SO_KEEPALIVE:
4b9d07a4
UB		 798 if (sk->sk_prot->keepalive)
799 sk->sk_prot->keepalive(sk, valbool);
e71a4783
SH		 800 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
801 break;
802
803 case SO_OOBINLINE:
804 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
805 break;
806
807 case SO_NO_CHECK:
28448b80 808 sk->sk_no_check_tx = valbool;
e71a4783
SH		 809 break;
810
811 case SO_PRIORITY:
5e1fccc0
EB		 812 if ((val >= 0 && val <= 6) ||
813 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
e71a4783
SH		 814 sk->sk_priority = val;
815 else
816 ret = -EPERM;
817 break;
818
819 case SO_LINGER:
820 if (optlen < sizeof(ling)) {
821 ret = -EINVAL; /* 1003.1g */
1da177e4 822 break;
e71a4783 823 }
2a91525c 824 if (copy_from_user(&ling, optval, sizeof(ling))) {
e71a4783 825 ret = -EFAULT;
1da177e4 826 break;
e71a4783
SH		 827 }
828 if (!ling.l_onoff)
829 sock_reset_flag(sk, SOCK_LINGER);
830 else {
1da177e4 831#if (BITS_PER_LONG == 32)
e71a4783
SH		 832 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
833 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
1da177e4 834 else
e71a4783
SH		 835#endif
836 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
837 sock_set_flag(sk, SOCK_LINGER);
838 }
839 break;
840
841 case SO_BSDCOMPAT:
842 sock_warn_obsolete_bsdism("setsockopt");
843 break;
844
845 case SO_PASSCRED:
846 if (valbool)
847 set_bit(SOCK_PASSCRED, &sock->flags);
848 else
849 clear_bit(SOCK_PASSCRED, &sock->flags);
850 break;
851
852 case SO_TIMESTAMP:
92f37fd2 853 case SO_TIMESTAMPNS:
e71a4783 854 if (valbool) {
92f37fd2
ED		 855 if (optname == SO_TIMESTAMP)
856 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
857 else
858 sock_set_flag(sk, SOCK_RCVTSTAMPNS);
e71a4783 859 sock_set_flag(sk, SOCK_RCVTSTAMP);
20d49473 860 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
92f37fd2 861 } else {
e71a4783 862 sock_reset_flag(sk, SOCK_RCVTSTAMP);
92f37fd2
ED		 863 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
864 }
e71a4783
SH		 865 break;
866
20d49473
PO		 867 case SO_TIMESTAMPING:
868 if (val & ~SOF_TIMESTAMPING_MASK) {
f249fb78 869 ret = -EINVAL;
20d49473
PO		 870 break;
871 }
b245be1f 872
09c2d251 873 if (val & SOF_TIMESTAMPING_OPT_ID &&
4ed2d765 874 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
ac5cc977
WC		 875 if (sk->sk_protocol == IPPROTO_TCP &&
876 sk->sk_type == SOCK_STREAM) {
6db8b963
SHY		 877 if ((1 << sk->sk_state) &
878 (TCPF_CLOSE | TCPF_LISTEN)) {
4ed2d765
WB		 879 ret = -EINVAL;
880 break;
881 }
882 sk->sk_tskey = tcp_sk(sk)->snd_una;
883 } else {
884 sk->sk_tskey = 0;
885 }
886 }
1c885808
FY		 887
888 if (val & SOF_TIMESTAMPING_OPT_STATS &&
889 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
890 ret = -EINVAL;
891 break;
892 }
893
b9f40e21 894 sk->sk_tsflags = val;
20d49473
PO		 895 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
896 sock_enable_timestamp(sk,
897 SOCK_TIMESTAMPING_RX_SOFTWARE);
898 else
899 sock_disable_timestamp(sk,
08e29af3 900 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
20d49473
PO		 901 break;
902
e71a4783
SH		 903 case SO_RCVLOWAT:
904 if (val < 0)
905 val = INT_MAX;
906 sk->sk_rcvlowat = val ? : 1;
907 break;
908
909 case SO_RCVTIMEO:
910 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen);
911 break;
912
913 case SO_SNDTIMEO:
914 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen);
915 break;
1da177e4 916
e71a4783
SH		 917 case SO_ATTACH_FILTER:
918 ret = -EINVAL;
919 if (optlen == sizeof(struct sock_fprog)) {
920 struct sock_fprog fprog;
1da177e4 921
e71a4783
SH		 922 ret = -EFAULT;
923 if (copy_from_user(&fprog, optval, sizeof(fprog)))
1da177e4 924 break;
e71a4783
SH		 925
926 ret = sk_attach_filter(&fprog, sk);
927 }
928 break;
929
89aa0758
AS		 930 case SO_ATTACH_BPF:
931 ret = -EINVAL;
932 if (optlen == sizeof(u32)) {
933 u32 ufd;
934
935 ret = -EFAULT;
936 if (copy_from_user(&ufd, optval, sizeof(ufd)))
937 break;
938
939 ret = sk_attach_bpf(ufd, sk);
940 }
941 break;
942
538950a1
CG		 943 case SO_ATTACH_REUSEPORT_CBPF:
944 ret = -EINVAL;
945 if (optlen == sizeof(struct sock_fprog)) {
946 struct sock_fprog fprog;
947
948 ret = -EFAULT;
949 if (copy_from_user(&fprog, optval, sizeof(fprog)))
950 break;
951
952 ret = sk_reuseport_attach_filter(&fprog, sk);
953 }
954 break;
955
956 case SO_ATTACH_REUSEPORT_EBPF:
957 ret = -EINVAL;
958 if (optlen == sizeof(u32)) {
959 u32 ufd;
960
961 ret = -EFAULT;
962 if (copy_from_user(&ufd, optval, sizeof(ufd)))
963 break;
964
965 ret = sk_reuseport_attach_bpf(ufd, sk);
966 }
967 break;
968
e71a4783 969 case SO_DETACH_FILTER:
55b33325 970 ret = sk_detach_filter(sk);
e71a4783 971 break;
1da177e4 972
d59577b6
VB		 973 case SO_LOCK_FILTER:
974 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
975 ret = -EPERM;
976 else
977 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
978 break;
979
e71a4783
SH		 980 case SO_PASSSEC:
981 if (valbool)
982 set_bit(SOCK_PASSSEC, &sock->flags);
983 else
984 clear_bit(SOCK_PASSSEC, &sock->flags);
985 break;
4a19ec58 986 case SO_MARK:
5e1fccc0 987 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
4a19ec58 988 ret = -EPERM;
2a91525c 989 else
4a19ec58 990 sk->sk_mark = val;
4a19ec58 991 break;
877ce7c1 992
3b885787 993 case SO_RXQ_OVFL:
8083f0fc 994 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
3b885787 995 break;
6e3e939f
JB		 996
997 case SO_WIFI_STATUS:
998 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
999 break;
1000
ef64a54f
PE		 1001 case SO_PEEK_OFF:
1002 if (sock->ops->set_peek_off)
12663bfc 1003 ret = sock->ops->set_peek_off(sk, val);
ef64a54f
PE		 1004 else
1005 ret = -EOPNOTSUPP;
1006 break;
3bdc0eba
BG		 1007
1008 case SO_NOFCS:
1009 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1010 break;
1011
7d4c04fc
KJ		 1012 case SO_SELECT_ERR_QUEUE:
1013 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1014 break;
1015
e0d1095a 1016#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1017 case SO_BUSY_POLL:
dafcc438
ET		 1018 /* allow unprivileged users to decrease the value */
1019 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
1020 ret = -EPERM;
1021 else {
1022 if (val < 0)
1023 ret = -EINVAL;
1024 else
1025 sk->sk_ll_usec = val;
1026 }
1027 break;
1028#endif
62748f32
ED		 1029
1030 case SO_MAX_PACING_RATE:
218af599
ED		 1031 if (val != ~0U)
1032 cmpxchg(&sk->sk_pacing_status,
1033 SK_PACING_NONE,
1034 SK_PACING_NEEDED);
62748f32
ED		 1035 sk->sk_max_pacing_rate = val;
1036 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
1037 sk->sk_max_pacing_rate);
1038 break;
1039
70da268b
ED		 1040 case SO_INCOMING_CPU:
1041 sk->sk_incoming_cpu = val;
1042 break;
1043
a87cb3e4
TH		 1044 case SO_CNX_ADVICE:
1045 if (val == 1)
1046 dst_negative_advice(sk);
1047 break;
76851d12
WB		 1048
1049 case SO_ZEROCOPY:
1050 if (sk->sk_family != PF_INET && sk->sk_family != PF_INET6)
1051 ret = -ENOTSUPP;
1052 else if (sk->sk_protocol != IPPROTO_TCP)
1053 ret = -ENOTSUPP;
1054 else if (sk->sk_state != TCP_CLOSE)
1055 ret = -EBUSY;
1056 else if (val < 0 || val > 1)
1057 ret = -EINVAL;
1058 else
1059 sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
1060 break;
1061
e71a4783
SH		 1062 default:
1063 ret = -ENOPROTOOPT;
1064 break;
4ec93edb 1065 }
1da177e4
LT		 1066 release_sock(sk);
1067 return ret;
1068}
2a91525c 1069EXPORT_SYMBOL(sock_setsockopt);
1da177e4
LT		 1070
1071
8f09898b 1072static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1073 struct ucred *ucred)
3f551f94
EB		 1074{
1075 ucred->pid = pid_vnr(pid);
1076 ucred->uid = ucred->gid = -1;
1077 if (cred) {
1078 struct user_namespace *current_ns = current_user_ns();
1079
b2e4f544
EB		 1080 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1081 ucred->gid = from_kgid_munged(current_ns, cred->egid);
3f551f94
EB		 1082 }
1083}
1084
28b5ba2a
DH		 1085static int groups_to_user(gid_t __user *dst, const struct group_info *src)
1086{
1087 struct user_namespace *user_ns = current_user_ns();
1088 int i;
1089
1090 for (i = 0; i < src->ngroups; i++)
1091 if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
1092 return -EFAULT;
1093
1094 return 0;
1095}
1096
1da177e4
LT		 1097int sock_getsockopt(struct socket *sock, int level, int optname,
1098 char __user *optval, int __user *optlen)
1099{
1100 struct sock *sk = sock->sk;
4ec93edb 1101
e71a4783 1102 union {
4ec93edb 1103 int val;
5daab9db 1104 u64 val64;
4ec93edb 1105 struct linger ling;
1da177e4
LT		 1106 struct timeval tm;
1107 } v;
4ec93edb 1108
4d0392be 1109 int lv = sizeof(int);
1da177e4 1110 int len;
4ec93edb 1111
e71a4783 1112 if (get_user(len, optlen))
4ec93edb 1113 return -EFAULT;
e71a4783 1114 if (len < 0)
1da177e4 1115 return -EINVAL;
4ec93edb 1116
50fee1de 1117 memset(&v, 0, sizeof(v));
df0bca04 1118
2a91525c 1119 switch (optname) {
e71a4783
SH		 1120 case SO_DEBUG:
1121 v.val = sock_flag(sk, SOCK_DBG);
1122 break;
1123
1124 case SO_DONTROUTE:
1125 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1126 break;
1127
1128 case SO_BROADCAST:
1b23a5df 1129 v.val = sock_flag(sk, SOCK_BROADCAST);
e71a4783
SH		 1130 break;
1131
1132 case SO_SNDBUF:
1133 v.val = sk->sk_sndbuf;
1134 break;
1135
1136 case SO_RCVBUF:
1137 v.val = sk->sk_rcvbuf;
1138 break;
1139
1140 case SO_REUSEADDR:
1141 v.val = sk->sk_reuse;
1142 break;
1143
055dc21a
TH		 1144 case SO_REUSEPORT:
1145 v.val = sk->sk_reuseport;
1146 break;
1147
e71a4783 1148 case SO_KEEPALIVE:
1b23a5df 1149 v.val = sock_flag(sk, SOCK_KEEPOPEN);
e71a4783
SH		 1150 break;
1151
1152 case SO_TYPE:
1153 v.val = sk->sk_type;
1154 break;
1155
49c794e9
JE		 1156 case SO_PROTOCOL:
1157 v.val = sk->sk_protocol;
1158 break;
1159
0d6038ee
JE		 1160 case SO_DOMAIN:
1161 v.val = sk->sk_family;
1162 break;
1163
e71a4783
SH		 1164 case SO_ERROR:
1165 v.val = -sock_error(sk);
2a91525c 1166 if (v.val == 0)
e71a4783
SH		 1167 v.val = xchg(&sk->sk_err_soft, 0);
1168 break;
1169
1170 case SO_OOBINLINE:
1b23a5df 1171 v.val = sock_flag(sk, SOCK_URGINLINE);
e71a4783
SH		 1172 break;
1173
1174 case SO_NO_CHECK:
28448b80 1175 v.val = sk->sk_no_check_tx;
e71a4783
SH		 1176 break;
1177
1178 case SO_PRIORITY:
1179 v.val = sk->sk_priority;
1180 break;
1181
1182 case SO_LINGER:
1183 lv = sizeof(v.ling);
1b23a5df 1184 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
e71a4783
SH		 1185 v.ling.l_linger = sk->sk_lingertime / HZ;
1186 break;
1187
1188 case SO_BSDCOMPAT:
1189 sock_warn_obsolete_bsdism("getsockopt");
1190 break;
1191
1192 case SO_TIMESTAMP:
92f37fd2
ED		 1193 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1194 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1195 break;
1196
1197 case SO_TIMESTAMPNS:
1198 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
e71a4783
SH		 1199 break;
1200
20d49473 1201 case SO_TIMESTAMPING:
b9f40e21 1202 v.val = sk->sk_tsflags;
20d49473
PO		 1203 break;
1204
e71a4783 1205 case SO_RCVTIMEO:
2a91525c 1206 lv = sizeof(struct timeval);
e71a4783
SH		 1207 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
1208 v.tm.tv_sec = 0;
1209 v.tm.tv_usec = 0;
1210 } else {
1211 v.tm.tv_sec = sk->sk_rcvtimeo / HZ;
8ccde4c5 1212 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ;
e71a4783
SH		 1213 }
1214 break;
1215
1216 case SO_SNDTIMEO:
2a91525c 1217 lv = sizeof(struct timeval);
e71a4783
SH		 1218 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) {
1219 v.tm.tv_sec = 0;
1220 v.tm.tv_usec = 0;
1221 } else {
1222 v.tm.tv_sec = sk->sk_sndtimeo / HZ;
8ccde4c5 1223 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ;
e71a4783
SH		 1224 }
1225 break;
1da177e4 1226
e71a4783
SH		 1227 case SO_RCVLOWAT:
1228 v.val = sk->sk_rcvlowat;
1229 break;
1da177e4 1230
e71a4783 1231 case SO_SNDLOWAT:
2a91525c 1232 v.val = 1;
e71a4783 1233 break;
1da177e4 1234
e71a4783 1235 case SO_PASSCRED:
82981930 1236 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
e71a4783 1237 break;
1da177e4 1238
e71a4783 1239 case SO_PEERCRED:
109f6e39
EB		 1240 {
1241 struct ucred peercred;
1242 if (len > sizeof(peercred))
1243 len = sizeof(peercred);
1244 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1245 if (copy_to_user(optval, &peercred, len))
e71a4783
SH		 1246 return -EFAULT;
1247 goto lenout;
109f6e39 1248 }
1da177e4 1249
28b5ba2a
DH		 1250 case SO_PEERGROUPS:
1251 {
1252 int ret, n;
1253
1254 if (!sk->sk_peer_cred)
1255 return -ENODATA;
1256
1257 n = sk->sk_peer_cred->group_info->ngroups;
1258 if (len < n * sizeof(gid_t)) {
1259 len = n * sizeof(gid_t);
1260 return put_user(len, optlen) ? -EFAULT : -ERANGE;
1261 }
1262 len = n * sizeof(gid_t);
1263
1264 ret = groups_to_user((gid_t __user *)optval,
1265 sk->sk_peer_cred->group_info);
1266 if (ret)
1267 return ret;
1268 goto lenout;
1269 }
1270
e71a4783
SH		 1271 case SO_PEERNAME:
1272 {
1273 char address[128];
1274
1275 if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2))
1276 return -ENOTCONN;
1277 if (lv < len)
1278 return -EINVAL;
1279 if (copy_to_user(optval, address, len))
1280 return -EFAULT;
1281 goto lenout;
1282 }
1da177e4 1283
e71a4783
SH		 1284 /* Dubious BSD thing... Probably nobody even uses it, but
1285 * the UNIX standard wants it for whatever reason... -DaveM
1286 */
1287 case SO_ACCEPTCONN:
1288 v.val = sk->sk_state == TCP_LISTEN;
1289 break;
1da177e4 1290
e71a4783 1291 case SO_PASSSEC:
82981930 1292 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
e71a4783 1293 break;
877ce7c1 1294
e71a4783
SH		 1295 case SO_PEERSEC:
1296 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1da177e4 1297
4a19ec58
LAT		 1298 case SO_MARK:
1299 v.val = sk->sk_mark;
1300 break;
1301
3b885787 1302 case SO_RXQ_OVFL:
1b23a5df 1303 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
3b885787
NH		 1304 break;
1305
6e3e939f 1306 case SO_WIFI_STATUS:
1b23a5df 1307 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
6e3e939f
JB		 1308 break;
1309
ef64a54f
PE		 1310 case SO_PEEK_OFF:
1311 if (!sock->ops->set_peek_off)
1312 return -EOPNOTSUPP;
1313
1314 v.val = sk->sk_peek_off;
1315 break;
bc2f7996 1316 case SO_NOFCS:
1b23a5df 1317 v.val = sock_flag(sk, SOCK_NOFCS);
bc2f7996 1318 break;
c91f6df2 1319
f7b86bfe 1320 case SO_BINDTODEVICE:
c91f6df2
BH		 1321 return sock_getbindtodevice(sk, optval, optlen, len);
1322
a8fc9277
PE		 1323 case SO_GET_FILTER:
1324 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1325 if (len < 0)
1326 return len;
1327
1328 goto lenout;
c91f6df2 1329
d59577b6
VB		 1330 case SO_LOCK_FILTER:
1331 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1332 break;
1333
ea02f941
MS		 1334 case SO_BPF_EXTENSIONS:
1335 v.val = bpf_tell_extensions();
1336 break;
1337
7d4c04fc
KJ		 1338 case SO_SELECT_ERR_QUEUE:
1339 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1340 break;
1341
e0d1095a 1342#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1343 case SO_BUSY_POLL:
dafcc438
ET		 1344 v.val = sk->sk_ll_usec;
1345 break;
1346#endif
1347
62748f32
ED		 1348 case SO_MAX_PACING_RATE:
1349 v.val = sk->sk_max_pacing_rate;
1350 break;
1351
2c8c56e1
ED		 1352 case SO_INCOMING_CPU:
1353 v.val = sk->sk_incoming_cpu;
1354 break;
1355
a2d133b1
JH		 1356 case SO_MEMINFO:
1357 {
1358 u32 meminfo[SK_MEMINFO_VARS];
1359
1360 if (get_user(len, optlen))
1361 return -EFAULT;
1362
1363 sk_get_meminfo(sk, meminfo);
1364
1365 len = min_t(unsigned int, len, sizeof(meminfo));
1366 if (copy_to_user(optval, &meminfo, len))
1367 return -EFAULT;
1368
1369 goto lenout;
1370 }
6d433902
SS		 1371
1372#ifdef CONFIG_NET_RX_BUSY_POLL
1373 case SO_INCOMING_NAPI_ID:
1374 v.val = READ_ONCE(sk->sk_napi_id);
1375
1376 /* aggregate non-NAPI IDs down to 0 */
1377 if (v.val < MIN_NAPI_ID)
1378 v.val = 0;
1379
1380 break;
1381#endif
1382
5daab9db
CF		 1383 case SO_COOKIE:
1384 lv = sizeof(u64);
1385 if (len < lv)
1386 return -EINVAL;
1387 v.val64 = sock_gen_cookie(sk);
1388 break;
1389
76851d12
WB		 1390 case SO_ZEROCOPY:
1391 v.val = sock_flag(sk, SOCK_ZEROCOPY);
1392 break;
1393
e71a4783 1394 default:
443b5991
YH		 1395 /* We implement the SO_SNDLOWAT etc to not be settable
1396 * (1003.1g 7).
1397 */
e71a4783 1398 return -ENOPROTOOPT;
1da177e4 1399 }
e71a4783 1400
1da177e4
LT		 1401 if (len > lv)
1402 len = lv;
1403 if (copy_to_user(optval, &v, len))
1404 return -EFAULT;
1405lenout:
4ec93edb
YH		 1406 if (put_user(len, optlen))
1407 return -EFAULT;
1408 return 0;
1da177e4
LT		 1409}
1410
a5b5bb9a
IM		 1411/*
1412 * Initialize an sk_lock.
1413 *
1414 * (We also register the sk_lock with the lock validator.)
1415 */
b6f99a21 1416static inline void sock_lock_init(struct sock *sk)
a5b5bb9a 1417{
cdfbabfb
DH		 1418 if (sk->sk_kern_sock)
1419 sock_lock_init_class_and_name(
1420 sk,
1421 af_family_kern_slock_key_strings[sk->sk_family],
1422 af_family_kern_slock_keys + sk->sk_family,
1423 af_family_kern_key_strings[sk->sk_family],
1424 af_family_kern_keys + sk->sk_family);
1425 else
1426 sock_lock_init_class_and_name(
1427 sk,
ed07536e
PZ		 1428 af_family_slock_key_strings[sk->sk_family],
1429 af_family_slock_keys + sk->sk_family,
1430 af_family_key_strings[sk->sk_family],
1431 af_family_keys + sk->sk_family);
a5b5bb9a
IM		 1432}
1433
4dc6dc71
ED		 1434/*
1435 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1436 * even temporarly, because of RCU lookups. sk_node should also be left as is.
68835aba 1437 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
4dc6dc71 1438 */
f1a6c4da
PE		 1439static void sock_copy(struct sock *nsk, const struct sock *osk)
1440{
1441#ifdef CONFIG_SECURITY_NETWORK
1442 void *sptr = nsk->sk_security;
1443#endif
68835aba
ED		 1444 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1445
1446 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1447 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1448
f1a6c4da
PE		 1449#ifdef CONFIG_SECURITY_NETWORK
1450 nsk->sk_security = sptr;
1451 security_sk_clone(osk, nsk);
1452#endif
1453}
1454
2e4afe7b
PE		 1455static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1456 int family)
c308c1b2
PE		 1457{
1458 struct sock *sk;
1459 struct kmem_cache *slab;
1460
1461 slab = prot->slab;
e912b114
ED		 1462 if (slab != NULL) {
1463 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1464 if (!sk)
1465 return sk;
ba2489b0
ED		 1466 if (priority & __GFP_ZERO)
1467 sk_prot_clear_nulls(sk, prot->obj_size);
fcbdf09d 1468 } else
c308c1b2
PE		 1469 sk = kmalloc(prot->obj_size, priority);
1470
2e4afe7b 1471 if (sk != NULL) {
a98b65a3
VN		 1472 kmemcheck_annotate_bitfield(sk, flags);
1473
2e4afe7b
PE		 1474 if (security_sk_alloc(sk, family, priority))
1475 goto out_free;
1476
1477 if (!try_module_get(prot->owner))
1478 goto out_free_sec;
e022f0b4 1479 sk_tx_queue_clear(sk);
2e4afe7b
PE		 1480 }
1481
c308c1b2 1482 return sk;
2e4afe7b
PE		 1483
1484out_free_sec:
1485 security_sk_free(sk);
1486out_free:
1487 if (slab != NULL)
1488 kmem_cache_free(slab, sk);
1489 else
1490 kfree(sk);
1491 return NULL;
c308c1b2
PE		 1492}
1493
1494static void sk_prot_free(struct proto *prot, struct sock *sk)
1495{
1496 struct kmem_cache *slab;
2e4afe7b 1497 struct module *owner;
c308c1b2 1498
2e4afe7b 1499 owner = prot->owner;
c308c1b2 1500 slab = prot->slab;
2e4afe7b 1501
bd1060a1 1502 cgroup_sk_free(&sk->sk_cgrp_data);
2d758073 1503 mem_cgroup_sk_free(sk);
2e4afe7b 1504 security_sk_free(sk);
c308c1b2
PE		 1505 if (slab != NULL)
1506 kmem_cache_free(slab, sk);
1507 else
1508 kfree(sk);
2e4afe7b 1509 module_put(owner);
c308c1b2
PE		 1510}
1511
1da177e4
LT		 1512/**
1513 * sk_alloc - All socket objects are allocated here
c4ea43c5 1514 * @net: the applicable net namespace
4dc3b16b
PP		 1515 * @family: protocol family
1516 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1517 * @prot: struct proto associated with this new sock instance
11aa9c28 1518 * @kern: is this to be a kernel socket?
1da177e4 1519 */
1b8d7ae4 1520struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
11aa9c28 1521 struct proto *prot, int kern)
1da177e4 1522{
c308c1b2 1523 struct sock *sk;
1da177e4 1524
154adbc8 1525 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1da177e4 1526 if (sk) {
154adbc8
PE		 1527 sk->sk_family = family;
1528 /*
1529 * See comment in struct sock definition to understand
1530 * why we need sk_prot_creator -acme
1531 */
1532 sk->sk_prot = sk->sk_prot_creator = prot;
cdfbabfb 1533 sk->sk_kern_sock = kern;
154adbc8 1534 sock_lock_init(sk);
26abe143
EB		 1535 sk->sk_net_refcnt = kern ? 0 : 1;
1536 if (likely(sk->sk_net_refcnt))
1537 get_net(net);
1538 sock_net_set(sk, net);
14afee4b 1539 refcount_set(&sk->sk_wmem_alloc, 1);
f8451725 1540
2d758073 1541 mem_cgroup_sk_alloc(sk);
d979a39d 1542 cgroup_sk_alloc(&sk->sk_cgrp_data);
2a56a1fe
TH		 1543 sock_update_classid(&sk->sk_cgrp_data);
1544 sock_update_netprioidx(&sk->sk_cgrp_data);
1da177e4 1545 }
a79af59e 1546
2e4afe7b 1547 return sk;
1da177e4 1548}
2a91525c 1549EXPORT_SYMBOL(sk_alloc);
1da177e4 1550
a4298e45
ED		 1551/* Sockets having SOCK_RCU_FREE will call this function after one RCU
1552 * grace period. This is the case for UDP sockets and TCP listeners.
1553 */
1554static void __sk_destruct(struct rcu_head *head)
1da177e4 1555{
a4298e45 1556 struct sock *sk = container_of(head, struct sock, sk_rcu);
1da177e4 1557 struct sk_filter *filter;
1da177e4
LT		 1558
1559 if (sk->sk_destruct)
1560 sk->sk_destruct(sk);
1561
a898def2 1562 filter = rcu_dereference_check(sk->sk_filter,
14afee4b 1563 refcount_read(&sk->sk_wmem_alloc) == 0);
1da177e4 1564 if (filter) {
309dd5fc 1565 sk_filter_uncharge(sk, filter);
a9b3cd7f 1566 RCU_INIT_POINTER(sk->sk_filter, NULL);
1da177e4 1567 }
538950a1
CG		 1568 if (rcu_access_pointer(sk->sk_reuseport_cb))
1569 reuseport_detach_sock(sk);
1da177e4 1570
08e29af3 1571 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1da177e4
LT		 1572
1573 if (atomic_read(&sk->sk_omem_alloc))
e005d193
JP		 1574 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1575 __func__, atomic_read(&sk->sk_omem_alloc));
1da177e4 1576
22a0e18e
ED		 1577 if (sk->sk_frag.page) {
1578 put_page(sk->sk_frag.page);
1579 sk->sk_frag.page = NULL;
1580 }
1581
109f6e39
EB		 1582 if (sk->sk_peer_cred)
1583 put_cred(sk->sk_peer_cred);
1584 put_pid(sk->sk_peer_pid);
26abe143
EB		 1585 if (likely(sk->sk_net_refcnt))
1586 put_net(sock_net(sk));
c308c1b2 1587 sk_prot_free(sk->sk_prot_creator, sk);
1da177e4 1588}
2b85a34e 1589
a4298e45
ED		 1590void sk_destruct(struct sock *sk)
1591{
1592 if (sock_flag(sk, SOCK_RCU_FREE))
1593 call_rcu(&sk->sk_rcu, __sk_destruct);
1594 else
1595 __sk_destruct(&sk->sk_rcu);
1596}
1597
eb4cb008
CG		 1598static void __sk_free(struct sock *sk)
1599{
b922622e 1600 if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt))
eb4cb008
CG		 1601 sock_diag_broadcast_destroy(sk);
1602 else
1603 sk_destruct(sk);
1604}
1605
2b85a34e
ED		 1606void sk_free(struct sock *sk)
1607{
1608 /*
25985edc 1609 * We subtract one from sk_wmem_alloc and can know if
2b85a34e
ED		 1610 * some packets are still in some tx queue.
1611 * If not null, sock_wfree() will call __sk_free(sk) later
1612 */
14afee4b 1613 if (refcount_dec_and_test(&sk->sk_wmem_alloc))
2b85a34e
ED		 1614 __sk_free(sk);
1615}
2a91525c 1616EXPORT_SYMBOL(sk_free);
1da177e4 1617
581319c5
PA		 1618static void sk_init_common(struct sock *sk)
1619{
1620 skb_queue_head_init(&sk->sk_receive_queue);
1621 skb_queue_head_init(&sk->sk_write_queue);
1622 skb_queue_head_init(&sk->sk_error_queue);
1623
1624 rwlock_init(&sk->sk_callback_lock);
1625 lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
1626 af_rlock_keys + sk->sk_family,
1627 af_family_rlock_key_strings[sk->sk_family]);
1628 lockdep_set_class_and_name(&sk->sk_write_queue.lock,
1629 af_wlock_keys + sk->sk_family,
1630 af_family_wlock_key_strings[sk->sk_family]);
1631 lockdep_set_class_and_name(&sk->sk_error_queue.lock,
1632 af_elock_keys + sk->sk_family,
1633 af_family_elock_key_strings[sk->sk_family]);
1634 lockdep_set_class_and_name(&sk->sk_callback_lock,
1635 af_callback_keys + sk->sk_family,
1636 af_family_clock_key_strings[sk->sk_family]);
1637}
1638
e56c57d0
ED		 1639/**
1640 * sk_clone_lock - clone a socket, and lock its clone
1641 * @sk: the socket to clone
1642 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1643 *
1644 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1645 */
1646struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
87d11ceb 1647{
8fd1d178 1648 struct sock *newsk;
278571ba 1649 bool is_charged = true;
87d11ceb 1650
8fd1d178 1651 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
87d11ceb
ACM		 1652 if (newsk != NULL) {
1653 struct sk_filter *filter;
1654
892c141e 1655 sock_copy(newsk, sk);
87d11ceb 1656
9d538fa6
CP		 1657 newsk->sk_prot_creator = sk->sk_prot;
1658
87d11ceb 1659 /* SANITY */
8a681736
SV		 1660 if (likely(newsk->sk_net_refcnt))
1661 get_net(sock_net(newsk));
87d11ceb
ACM		 1662 sk_node_init(&newsk->sk_node);
1663 sock_lock_init(newsk);
1664 bh_lock_sock(newsk);
fa438ccf 1665 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
8eae939f 1666 newsk->sk_backlog.len = 0;
87d11ceb
ACM		 1667
1668 atomic_set(&newsk->sk_rmem_alloc, 0);
2b85a34e
ED		 1669 /*
1670 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1671 */
14afee4b 1672 refcount_set(&newsk->sk_wmem_alloc, 1);
87d11ceb 1673 atomic_set(&newsk->sk_omem_alloc, 0);
581319c5 1674 sk_init_common(newsk);
87d11ceb
ACM		 1675
1676 newsk->sk_dst_cache = NULL;
9b8805a3 1677 newsk->sk_dst_pending_confirm = 0;
87d11ceb
ACM		 1678 newsk->sk_wmem_queued = 0;
1679 newsk->sk_forward_alloc = 0;
9caad864 1680 atomic_set(&newsk->sk_drops, 0);
87d11ceb 1681 newsk->sk_send_head = NULL;
87d11ceb 1682 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
52267790 1683 atomic_set(&newsk->sk_zckey, 0);
87d11ceb
ACM		 1684
1685 sock_reset_flag(newsk, SOCK_DONE);
87d11ceb 1686
0d7da9dd 1687 filter = rcu_dereference_protected(newsk->sk_filter, 1);
87d11ceb 1688 if (filter != NULL)
278571ba
AS		 1689 /* though it's an empty new sock, the charging may fail
1690 * if sysctl_optmem_max was changed between creation of
1691 * original socket and cloning
1692 */
1693 is_charged = sk_filter_charge(newsk, filter);
87d11ceb 1694
d188ba86 1695 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
a97e50cc
DB		 1696 /* We need to make sure that we don't uncharge the new
1697 * socket if we couldn't charge it in the first place
1698 * as otherwise we uncharge the parent's filter.
1699 */
1700 if (!is_charged)
1701 RCU_INIT_POINTER(newsk->sk_filter, NULL);
94352d45 1702 sk_free_unlock_clone(newsk);
87d11ceb
ACM		 1703 newsk = NULL;
1704 goto out;
1705 }
fa463497 1706 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
87d11ceb
ACM		 1707
1708 newsk->sk_err = 0;
e551c32d 1709 newsk->sk_err_soft = 0;
87d11ceb 1710 newsk->sk_priority = 0;
2c8c56e1 1711 newsk->sk_incoming_cpu = raw_smp_processor_id();
33cf7c90 1712 atomic64_set(&newsk->sk_cookie, 0);
d979a39d 1713
2d758073 1714 mem_cgroup_sk_alloc(newsk);
d979a39d
JW		 1715 cgroup_sk_alloc(&newsk->sk_cgrp_data);
1716
4dc6dc71
ED		 1717 /*
1718 * Before updating sk_refcnt, we must commit prior changes to memory
1719 * (Documentation/RCU/rculist_nulls.txt for details)
1720 */
1721 smp_wmb();
41c6d650 1722 refcount_set(&newsk->sk_refcnt, 2);
87d11ceb
ACM		 1723
1724 /*
1725 * Increment the counter in the same struct proto as the master
1726 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1727 * is the same as sk->sk_prot->socks, as this field was copied
1728 * with memcpy).
1729 *
1730 * This _changes_ the previous behaviour, where
1731 * tcp_create_openreq_child always was incrementing the
1732 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1733 * to be taken into account in all callers. -acme
1734 */
1735 sk_refcnt_debug_inc(newsk);
972692e0 1736 sk_set_socket(newsk, NULL);
43815482 1737 newsk->sk_wq = NULL;
87d11ceb
ACM		 1738
1739 if (newsk->sk_prot->sockets_allocated)
180d8cd9 1740 sk_sockets_allocated_inc(newsk);
704da560 1741
080a270f
HFS		 1742 if (sock_needs_netstamp(sk) &&
1743 newsk->sk_flags & SK_FLAGS_TIMESTAMP)
704da560 1744 net_enable_timestamp();
87d11ceb
ACM		 1745 }
1746out:
1747 return newsk;
1748}
e56c57d0 1749EXPORT_SYMBOL_GPL(sk_clone_lock);
87d11ceb 1750
94352d45
ACM		 1751void sk_free_unlock_clone(struct sock *sk)
1752{
1753 /* It is still raw copy of parent, so invalidate
1754 * destructor and make plain sk_free() */
1755 sk->sk_destruct = NULL;
1756 bh_unlock_sock(sk);
1757 sk_free(sk);
1758}
1759EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
1760
9958089a
AK		 1761void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1762{
d6a4e26a
ED		 1763 u32 max_segs = 1;
1764
6bd4f355 1765 sk_dst_set(sk, dst);
9958089a
AK		 1766 sk->sk_route_caps = dst->dev->features;
1767 if (sk->sk_route_caps & NETIF_F_GSO)
4fcd6b99 1768 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
a465419b 1769 sk->sk_route_caps &= ~sk->sk_route_nocaps;
9958089a 1770 if (sk_can_gso(sk)) {
f70f250a 1771 if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
9958089a 1772 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
82cc1a7a 1773 } else {
9958089a 1774 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
82cc1a7a 1775 sk->sk_gso_max_size = dst->dev->gso_max_size;
d6a4e26a 1776 max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
82cc1a7a 1777 }
9958089a 1778 }
d6a4e26a 1779 sk->sk_gso_max_segs = max_segs;
9958089a
AK		 1780}
1781EXPORT_SYMBOL_GPL(sk_setup_caps);
1782
1da177e4
LT		 1783/*
1784 * Simple resource managers for sockets.
1785 */
1786
1787
4ec93edb
YH		 1788/*
1789 * Write buffer destructor automatically called from kfree_skb.
1da177e4
LT		 1790 */
1791void sock_wfree(struct sk_buff *skb)
1792{
1793 struct sock *sk = skb->sk;
d99927f4 1794 unsigned int len = skb->truesize;
1da177e4 1795
d99927f4
ED		 1796 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
1797 /*
1798 * Keep a reference on sk_wmem_alloc, this will be released
1799 * after sk_write_space() call
1800 */
14afee4b 1801 WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
1da177e4 1802 sk->sk_write_space(sk);
d99927f4
ED		 1803 len = 1;
1804 }
2b85a34e 1805 /*
d99927f4
ED		 1806 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
1807 * could not do because of in-flight packets
2b85a34e 1808 */
14afee4b 1809 if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2b85a34e 1810 __sk_free(sk);
1da177e4 1811}
2a91525c 1812EXPORT_SYMBOL(sock_wfree);
1da177e4 1813
1d2077ac
ED		 1814/* This variant of sock_wfree() is used by TCP,
1815 * since it sets SOCK_USE_WRITE_QUEUE.
1816 */
1817void __sock_wfree(struct sk_buff *skb)
1818{
1819 struct sock *sk = skb->sk;
1820
14afee4b 1821 if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1d2077ac
ED		 1822 __sk_free(sk);
1823}
1824
9e17f8a4
ED		 1825void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1826{
1827 skb_orphan(skb);
1828 skb->sk = sk;
1829#ifdef CONFIG_INET
1830 if (unlikely(!sk_fullsock(sk))) {
1831 skb->destructor = sock_edemux;
1832 sock_hold(sk);
1833 return;
1834 }
1835#endif
1836 skb->destructor = sock_wfree;
1837 skb_set_hash_from_sk(skb, sk);
1838 /*
1839 * We used to take a refcount on sk, but following operation
1840 * is enough to guarantee sk_free() wont free this sock until
1841 * all in-flight packets are completed
1842 */
14afee4b 1843 refcount_add(skb->truesize, &sk->sk_wmem_alloc);
9e17f8a4
ED		 1844}
1845EXPORT_SYMBOL(skb_set_owner_w);
1846
1d2077ac
ED		 1847/* This helper is used by netem, as it can hold packets in its
1848 * delay queue. We want to allow the owner socket to send more
1849 * packets, as if they were already TX completed by a typical driver.
1850 * But we also want to keep skb->sk set because some packet schedulers
f6ba8d33 1851 * rely on it (sch_fq for example).
1d2077ac 1852 */
f2f872f9
ED		 1853void skb_orphan_partial(struct sk_buff *skb)
1854{
f6ba8d33 1855 if (skb_is_tcp_pure_ack(skb))
1d2077ac
ED		 1856 return;
1857
f2f872f9
ED		 1858 if (skb->destructor == sock_wfree
1859#ifdef CONFIG_INET
1860 || skb->destructor == tcp_wfree
1861#endif
1862 ) {
f6ba8d33
ED		 1863 struct sock *sk = skb->sk;
1864
41c6d650 1865 if (refcount_inc_not_zero(&sk->sk_refcnt)) {
14afee4b 1866 WARN_ON(refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc));
f6ba8d33
ED		 1867 skb->destructor = sock_efree;
1868 }
f2f872f9
ED		 1869 } else {
1870 skb_orphan(skb);
1871 }
1872}
1873EXPORT_SYMBOL(skb_orphan_partial);
1874
4ec93edb
YH		 1875/*
1876 * Read buffer destructor automatically called from kfree_skb.
1da177e4
LT		 1877 */
1878void sock_rfree(struct sk_buff *skb)
1879{
1880 struct sock *sk = skb->sk;
d361fd59 1881 unsigned int len = skb->truesize;
1da177e4 1882
d361fd59
ED		 1883 atomic_sub(len, &sk->sk_rmem_alloc);
1884 sk_mem_uncharge(sk, len);
1da177e4 1885}
2a91525c 1886EXPORT_SYMBOL(sock_rfree);
1da177e4 1887
7768eed8
OH		 1888/*
1889 * Buffer destructor for skbs that are not used directly in read or write
1890 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
1891 */
62bccb8c
AD		 1892void sock_efree(struct sk_buff *skb)
1893{
1894 sock_put(skb->sk);
1895}
1896EXPORT_SYMBOL(sock_efree);
1897
976d0201 1898kuid_t sock_i_uid(struct sock *sk)
1da177e4 1899{
976d0201 1900 kuid_t uid;
1da177e4 1901
f064af1e 1902 read_lock_bh(&sk->sk_callback_lock);
976d0201 1903 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
f064af1e 1904 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 1905 return uid;
1906}
2a91525c 1907EXPORT_SYMBOL(sock_i_uid);
1da177e4
LT		 1908
1909unsigned long sock_i_ino(struct sock *sk)
1910{
1911 unsigned long ino;
1912
f064af1e 1913 read_lock_bh(&sk->sk_callback_lock);
1da177e4 1914 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
f064af1e 1915 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 1916 return ino;
1917}
2a91525c 1918EXPORT_SYMBOL(sock_i_ino);
1da177e4
LT		 1919
1920/*
1921 * Allocate a skb from the socket's send buffer.
1922 */
86a76caf 1923struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
dd0fc66f 1924 gfp_t priority)
1da177e4 1925{
14afee4b 1926 if (force || refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) {
2a91525c 1927 struct sk_buff *skb = alloc_skb(size, priority);
1da177e4
LT		 1928 if (skb) {
1929 skb_set_owner_w(skb, sk);
1930 return skb;
1931 }
1932 }
1933 return NULL;
1934}
2a91525c 1935EXPORT_SYMBOL(sock_wmalloc);
1da177e4 1936
98ba0bd5
WB		 1937static void sock_ofree(struct sk_buff *skb)
1938{
1939 struct sock *sk = skb->sk;
1940
1941 atomic_sub(skb->truesize, &sk->sk_omem_alloc);
1942}
1943
1944struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1945 gfp_t priority)
1946{
1947 struct sk_buff *skb;
1948
1949 /* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
1950 if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
1951 sysctl_optmem_max)
1952 return NULL;
1953
1954 skb = alloc_skb(size, priority);
1955 if (!skb)
1956 return NULL;
1957
1958 atomic_add(skb->truesize, &sk->sk_omem_alloc);
1959 skb->sk = sk;
1960 skb->destructor = sock_ofree;
1961 return skb;
1962}
1963
4ec93edb 1964/*
1da177e4 1965 * Allocate a memory block from the socket's option memory buffer.
4ec93edb 1966 */
dd0fc66f 1967void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
1da177e4 1968{
95c96174 1969 if ((unsigned int)size <= sysctl_optmem_max &&
1da177e4
LT		 1970 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
1971 void *mem;
1972 /* First do the add, to avoid the race if kmalloc
4ec93edb 1973 * might sleep.
1da177e4
LT		 1974 */
1975 atomic_add(size, &sk->sk_omem_alloc);
1976 mem = kmalloc(size, priority);
1977 if (mem)
1978 return mem;
1979 atomic_sub(size, &sk->sk_omem_alloc);
1980 }
1981 return NULL;
1982}
2a91525c 1983EXPORT_SYMBOL(sock_kmalloc);
1da177e4 1984
79e88659
DB		 1985/* Free an option memory block. Note, we actually want the inline
1986 * here as this allows gcc to detect the nullify and fold away the
1987 * condition entirely.
1da177e4 1988 */
79e88659
DB		 1989static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
1990 const bool nullify)
1da177e4 1991{
e53da5fb
DM		 1992 if (WARN_ON_ONCE(!mem))
1993 return;
79e88659
DB		 1994 if (nullify)
1995 kzfree(mem);
1996 else
1997 kfree(mem);
1da177e4
LT		 1998 atomic_sub(size, &sk->sk_omem_alloc);
1999}
79e88659
DB		 2000
2001void sock_kfree_s(struct sock *sk, void *mem, int size)
2002{
2003 __sock_kfree_s(sk, mem, size, false);
2004}
2a91525c 2005EXPORT_SYMBOL(sock_kfree_s);
1da177e4 2006
79e88659
DB		 2007void sock_kzfree_s(struct sock *sk, void *mem, int size)
2008{
2009 __sock_kfree_s(sk, mem, size, true);
2010}
2011EXPORT_SYMBOL(sock_kzfree_s);
2012
1da177e4
LT		 2013/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
2014 I think, these locks should be removed for datagram sockets.
2015 */
2a91525c 2016static long sock_wait_for_wmem(struct sock *sk, long timeo)
1da177e4
LT		 2017{
2018 DEFINE_WAIT(wait);
2019
9cd3e072 2020 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4
LT		 2021 for (;;) {
2022 if (!timeo)
2023 break;
2024 if (signal_pending(current))
2025 break;
2026 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
aa395145 2027 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
14afee4b 2028 if (refcount_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf)
1da177e4
LT		 2029 break;
2030 if (sk->sk_shutdown & SEND_SHUTDOWN)
2031 break;
2032 if (sk->sk_err)
2033 break;
2034 timeo = schedule_timeout(timeo);
2035 }
aa395145 2036 finish_wait(sk_sleep(sk), &wait);
1da177e4
LT		 2037 return timeo;
2038}
2039
2040
2041/*
2042 * Generic send/receive buffer handlers
2043 */
2044
4cc7f68d
HX		 2045struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
2046 unsigned long data_len, int noblock,
28d64271 2047 int *errcode, int max_page_order)
1da177e4 2048{
2e4e4410 2049 struct sk_buff *skb;
1da177e4
LT		 2050 long timeo;
2051 int err;
2052
1da177e4 2053 timeo = sock_sndtimeo(sk, noblock);
2e4e4410 2054 for (;;) {
1da177e4
LT		 2055 err = sock_error(sk);
2056 if (err != 0)
2057 goto failure;
2058
2059 err = -EPIPE;
2060 if (sk->sk_shutdown & SEND_SHUTDOWN)
2061 goto failure;
2062
2e4e4410
ED		 2063 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf)
2064 break;
28d64271 2065
9cd3e072 2066 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2e4e4410
ED		 2067 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2068 err = -EAGAIN;
2069 if (!timeo)
1da177e4 2070 goto failure;
2e4e4410
ED		 2071 if (signal_pending(current))
2072 goto interrupted;
2073 timeo = sock_wait_for_wmem(sk, timeo);
1da177e4 2074 }
2e4e4410
ED		 2075 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2076 errcode, sk->sk_allocation);
2077 if (skb)
2078 skb_set_owner_w(skb, sk);
1da177e4
LT		 2079 return skb;
2080
2081interrupted:
2082 err = sock_intr_errno(timeo);
2083failure:
2084 *errcode = err;
2085 return NULL;
2086}
4cc7f68d 2087EXPORT_SYMBOL(sock_alloc_send_pskb);
1da177e4 2088
4ec93edb 2089struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1da177e4
LT		 2090 int noblock, int *errcode)
2091{
28d64271 2092 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
1da177e4 2093}
2a91525c 2094EXPORT_SYMBOL(sock_alloc_send_skb);
1da177e4 2095
39771b12
WB		 2096int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2097 struct sockcm_cookie *sockc)
2098{
3dd17e63
SHY		 2099 u32 tsflags;
2100
39771b12
WB		 2101 switch (cmsg->cmsg_type) {
2102 case SO_MARK:
2103 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
2104 return -EPERM;
2105 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2106 return -EINVAL;
2107 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2108 break;
3dd17e63
SHY		 2109 case SO_TIMESTAMPING:
2110 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2111 return -EINVAL;
2112
2113 tsflags = *(u32 *)CMSG_DATA(cmsg);
2114 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2115 return -EINVAL;
2116
2117 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2118 sockc->tsflags |= tsflags;
2119 break;
779f1ede
SHY		 2120 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2121 case SCM_RIGHTS:
2122 case SCM_CREDENTIALS:
2123 break;
39771b12
WB		 2124 default:
2125 return -EINVAL;
2126 }
2127 return 0;
2128}
2129EXPORT_SYMBOL(__sock_cmsg_send);
2130
f28ea365
EJ		 2131int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2132 struct sockcm_cookie *sockc)
2133{
2134 struct cmsghdr *cmsg;
39771b12 2135 int ret;
f28ea365
EJ		 2136
2137 for_each_cmsghdr(cmsg, msg) {
2138 if (!CMSG_OK(msg, cmsg))
2139 return -EINVAL;
2140 if (cmsg->cmsg_level != SOL_SOCKET)
2141 continue;
39771b12
WB		 2142 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2143 if (ret)
2144 return ret;
f28ea365
EJ		 2145 }
2146 return 0;
2147}
2148EXPORT_SYMBOL(sock_cmsg_send);
2149
06044751
ED		 2150static void sk_enter_memory_pressure(struct sock *sk)
2151{
2152 if (!sk->sk_prot->enter_memory_pressure)
2153 return;
2154
2155 sk->sk_prot->enter_memory_pressure(sk);
2156}
2157
2158static void sk_leave_memory_pressure(struct sock *sk)
2159{
2160 if (sk->sk_prot->leave_memory_pressure) {
2161 sk->sk_prot->leave_memory_pressure(sk);
2162 } else {
2163 unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2164
2165 if (memory_pressure && *memory_pressure)
2166 *memory_pressure = 0;
2167 }
2168}
2169
5640f768
ED		 2170/* On 32bit arches, an skb frag is limited to 2^15 */
2171#define SKB_FRAG_PAGE_ORDER get_order(32768)
2172
400dfd3a
ED		 2173/**
2174 * skb_page_frag_refill - check that a page_frag contains enough room
2175 * @sz: minimum size of the fragment we want to get
2176 * @pfrag: pointer to page_frag
82d5e2b8 2177 * @gfp: priority for memory allocation
400dfd3a
ED		 2178 *
2179 * Note: While this allocator tries to use high order pages, there is
2180 * no guarantee that allocations succeed. Therefore, @sz MUST be
2181 * less or equal than PAGE_SIZE.
2182 */
d9b2938a 2183bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
5640f768 2184{
5640f768 2185 if (pfrag->page) {
fe896d18 2186 if (page_ref_count(pfrag->page) == 1) {
5640f768
ED		 2187 pfrag->offset = 0;
2188 return true;
2189 }
400dfd3a 2190 if (pfrag->offset + sz <= pfrag->size)
5640f768
ED		 2191 return true;
2192 put_page(pfrag->page);
2193 }
2194
d9b2938a
ED		 2195 pfrag->offset = 0;
2196 if (SKB_FRAG_PAGE_ORDER) {
d0164adc
MG		 2197 /* Avoid direct reclaim but allow kswapd to wake */
2198 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2199 __GFP_COMP | __GFP_NOWARN |
2200 __GFP_NORETRY,
d9b2938a 2201 SKB_FRAG_PAGE_ORDER);
5640f768 2202 if (likely(pfrag->page)) {
d9b2938a 2203 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
5640f768
ED		 2204 return true;
2205 }
d9b2938a
ED		 2206 }
2207 pfrag->page = alloc_page(gfp);
2208 if (likely(pfrag->page)) {
2209 pfrag->size = PAGE_SIZE;
2210 return true;
2211 }
400dfd3a
ED		 2212 return false;
2213}
2214EXPORT_SYMBOL(skb_page_frag_refill);
2215
2216bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2217{
2218 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2219 return true;
2220
5640f768
ED		 2221 sk_enter_memory_pressure(sk);
2222 sk_stream_moderate_sndbuf(sk);
2223 return false;
2224}
2225EXPORT_SYMBOL(sk_page_frag_refill);
2226
1da177e4 2227static void __lock_sock(struct sock *sk)
f39234d6
NK		 2228 __releases(&sk->sk_lock.slock)
2229 __acquires(&sk->sk_lock.slock)
1da177e4
LT		 2230{
2231 DEFINE_WAIT(wait);
2232
e71a4783 2233 for (;;) {
1da177e4
LT		 2234 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2235 TASK_UNINTERRUPTIBLE);
2236 spin_unlock_bh(&sk->sk_lock.slock);
2237 schedule();
2238 spin_lock_bh(&sk->sk_lock.slock);
e71a4783 2239 if (!sock_owned_by_user(sk))
1da177e4
LT		 2240 break;
2241 }
2242 finish_wait(&sk->sk_lock.wq, &wait);
2243}
2244
2245static void __release_sock(struct sock *sk)
f39234d6
NK		 2246 __releases(&sk->sk_lock.slock)
2247 __acquires(&sk->sk_lock.slock)
1da177e4 2248{
5413d1ba 2249 struct sk_buff *skb, *next;
1da177e4 2250
5413d1ba 2251 while ((skb = sk->sk_backlog.head) != NULL) {
1da177e4 2252 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
1da177e4 2253
5413d1ba 2254 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4 2255
5413d1ba
ED		 2256 do {
2257 next = skb->next;
e4cbb02a 2258 prefetch(next);
7fee226a 2259 WARN_ON_ONCE(skb_dst_is_noref(skb));
1da177e4 2260 skb->next = NULL;
c57943a1 2261 sk_backlog_rcv(sk, skb);
1da177e4 2262
5413d1ba 2263 cond_resched();
1da177e4
LT		 2264
2265 skb = next;
2266 } while (skb != NULL);
2267
5413d1ba
ED		 2268 spin_lock_bh(&sk->sk_lock.slock);
2269 }
8eae939f
ZY		 2270
2271 /*
2272 * Doing the zeroing here guarantee we can not loop forever
2273 * while a wild producer attempts to flood us.
2274 */
2275 sk->sk_backlog.len = 0;
1da177e4
LT		 2276}
2277
d41a69f1
ED		 2278void __sk_flush_backlog(struct sock *sk)
2279{
2280 spin_lock_bh(&sk->sk_lock.slock);
2281 __release_sock(sk);
2282 spin_unlock_bh(&sk->sk_lock.slock);
2283}
2284
1da177e4
LT		 2285/**
2286 * sk_wait_data - wait for data to arrive at sk_receive_queue
4dc3b16b
PP		 2287 * @sk: sock to wait on
2288 * @timeo: for how long
dfbafc99 2289 * @skb: last skb seen on sk_receive_queue
1da177e4
LT		 2290 *
2291 * Now socket state including sk->sk_err is changed only under lock,
2292 * hence we may omit checks after joining wait queue.
2293 * We check receive queue before schedule() only as optimization;
2294 * it is very likely that release_sock() added new data.
2295 */
dfbafc99 2296int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
1da177e4 2297{
d9dc8b0f 2298 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1da177e4 2299 int rc;
1da177e4 2300
d9dc8b0f 2301 add_wait_queue(sk_sleep(sk), &wait);
9cd3e072 2302 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2303 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
9cd3e072 2304 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2305 remove_wait_queue(sk_sleep(sk), &wait);
1da177e4
LT		 2306 return rc;
2307}
1da177e4
LT		 2308EXPORT_SYMBOL(sk_wait_data);
2309
3ab224be 2310/**
f8c3bf00 2311 * __sk_mem_raise_allocated - increase memory_allocated
3ab224be
HA		 2312 * @sk: socket
2313 * @size: memory size to allocate
f8c3bf00 2314 * @amt: pages to allocate
3ab224be
HA		 2315 * @kind: allocation type
2316 *
f8c3bf00 2317 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
3ab224be 2318 */
f8c3bf00 2319int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
3ab224be
HA		 2320{
2321 struct proto *prot = sk->sk_prot;
f8c3bf00 2322 long allocated = sk_memory_allocated_add(sk, amt);
e805605c 2323
baac50bb
JW		 2324 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
2325 !mem_cgroup_charge_skmem(sk->sk_memcg, amt))
e805605c 2326 goto suppress_allocation;
3ab224be
HA		 2327
2328 /* Under limit. */
e805605c 2329 if (allocated <= sk_prot_mem_limits(sk, 0)) {
180d8cd9 2330 sk_leave_memory_pressure(sk);
3ab224be
HA		 2331 return 1;
2332 }
2333
e805605c
JW		 2334 /* Under pressure. */
2335 if (allocated > sk_prot_mem_limits(sk, 1))
180d8cd9 2336 sk_enter_memory_pressure(sk);
3ab224be 2337
e805605c
JW		 2338 /* Over hard limit. */
2339 if (allocated > sk_prot_mem_limits(sk, 2))
3ab224be
HA		 2340 goto suppress_allocation;
2341
2342 /* guarantee minimum buffer size under pressure */
2343 if (kind == SK_MEM_RECV) {
2344 if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0])
2345 return 1;
180d8cd9 2346
3ab224be
HA		 2347 } else { /* SK_MEM_SEND */
2348 if (sk->sk_type == SOCK_STREAM) {
2349 if (sk->sk_wmem_queued < prot->sysctl_wmem[0])
2350 return 1;
14afee4b 2351 } else if (refcount_read(&sk->sk_wmem_alloc) <
3ab224be
HA		 2352 prot->sysctl_wmem[0])
2353 return 1;
2354 }
2355
180d8cd9 2356 if (sk_has_memory_pressure(sk)) {
1748376b
ED		 2357 int alloc;
2358
180d8cd9 2359 if (!sk_under_memory_pressure(sk))
1748376b 2360 return 1;
180d8cd9
GC		 2361 alloc = sk_sockets_allocated_read_positive(sk);
2362 if (sk_prot_mem_limits(sk, 2) > alloc *
3ab224be
HA		 2363 sk_mem_pages(sk->sk_wmem_queued +
2364 atomic_read(&sk->sk_rmem_alloc) +
2365 sk->sk_forward_alloc))
2366 return 1;
2367 }
2368
2369suppress_allocation:
2370
2371 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2372 sk_stream_moderate_sndbuf(sk);
2373
2374 /* Fail only if socket is _under_ its sndbuf.
2375 * In this case we cannot block, so that we have to fail.
2376 */
2377 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2378 return 1;
2379 }
2380
3847ce32
SM		 2381 trace_sock_exceed_buf_limit(sk, prot, allocated);
2382
0e90b31f 2383 sk_memory_allocated_sub(sk, amt);
180d8cd9 2384
baac50bb
JW		 2385 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2386 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
e805605c 2387
3ab224be
HA		 2388 return 0;
2389}
f8c3bf00
PA		 2390EXPORT_SYMBOL(__sk_mem_raise_allocated);
2391
2392/**
2393 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2394 * @sk: socket
2395 * @size: memory size to allocate
2396 * @kind: allocation type
2397 *
2398 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2399 * rmem allocation. This function assumes that protocols which have
2400 * memory_pressure use sk_wmem_queued as write buffer accounting.
2401 */
2402int __sk_mem_schedule(struct sock *sk, int size, int kind)
2403{
2404 int ret, amt = sk_mem_pages(size);
2405
2406 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
2407 ret = __sk_mem_raise_allocated(sk, size, amt, kind);
2408 if (!ret)
2409 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
2410 return ret;
2411}
3ab224be
HA		 2412EXPORT_SYMBOL(__sk_mem_schedule);
2413
2414/**
f8c3bf00 2415 * __sk_mem_reduce_allocated - reclaim memory_allocated
3ab224be 2416 * @sk: socket
f8c3bf00
PA		 2417 * @amount: number of quanta
2418 *
2419 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3ab224be 2420 */
f8c3bf00 2421void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3ab224be 2422{
1a24e04e 2423 sk_memory_allocated_sub(sk, amount);
3ab224be 2424
baac50bb
JW		 2425 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2426 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
e805605c 2427
180d8cd9
GC		 2428 if (sk_under_memory_pressure(sk) &&
2429 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2430 sk_leave_memory_pressure(sk);
3ab224be 2431}
f8c3bf00
PA		 2432EXPORT_SYMBOL(__sk_mem_reduce_allocated);
2433
2434/**
2435 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
2436 * @sk: socket
2437 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2438 */
2439void __sk_mem_reclaim(struct sock *sk, int amount)
2440{
2441 amount >>= SK_MEM_QUANTUM_SHIFT;
2442 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2443 __sk_mem_reduce_allocated(sk, amount);
2444}
3ab224be
HA		 2445EXPORT_SYMBOL(__sk_mem_reclaim);
2446
627d2d6b 2447int sk_set_peek_off(struct sock *sk, int val)
2448{
627d2d6b 2449 sk->sk_peek_off = val;
2450 return 0;
2451}
2452EXPORT_SYMBOL_GPL(sk_set_peek_off);
3ab224be 2453
1da177e4
LT		 2454/*
2455 * Set of default routines for initialising struct proto_ops when
2456 * the protocol does not support a particular function. In certain
2457 * cases where it makes no sense for a protocol to have a "do nothing"
2458 * function, some default processing is provided.
2459 */
2460
2461int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2462{
2463 return -EOPNOTSUPP;
2464}
2a91525c 2465EXPORT_SYMBOL(sock_no_bind);
1da177e4 2466
4ec93edb 2467int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
1da177e4
LT		 2468 int len, int flags)
2469{
2470 return -EOPNOTSUPP;
2471}
2a91525c 2472EXPORT_SYMBOL(sock_no_connect);
1da177e4
LT		 2473
2474int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2475{
2476 return -EOPNOTSUPP;
2477}
2a91525c 2478EXPORT_SYMBOL(sock_no_socketpair);
1da177e4 2479
cdfbabfb
DH		 2480int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
2481 bool kern)
1da177e4
LT		 2482{
2483 return -EOPNOTSUPP;
2484}
2a91525c 2485EXPORT_SYMBOL(sock_no_accept);
1da177e4 2486
4ec93edb 2487int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
1da177e4
LT		 2488 int *len, int peer)
2489{
2490 return -EOPNOTSUPP;
2491}
2a91525c 2492EXPORT_SYMBOL(sock_no_getname);
1da177e4 2493
2a91525c 2494unsigned int sock_no_poll(struct file *file, struct socket *sock, poll_table *pt)
1da177e4
LT		 2495{
2496 return 0;
2497}
2a91525c 2498EXPORT_SYMBOL(sock_no_poll);
1da177e4
LT		 2499
2500int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2501{
2502 return -EOPNOTSUPP;
2503}
2a91525c 2504EXPORT_SYMBOL(sock_no_ioctl);
1da177e4
LT		 2505
2506int sock_no_listen(struct socket *sock, int backlog)
2507{
2508 return -EOPNOTSUPP;
2509}
2a91525c 2510EXPORT_SYMBOL(sock_no_listen);
1da177e4
LT		 2511
2512int sock_no_shutdown(struct socket *sock, int how)
2513{
2514 return -EOPNOTSUPP;
2515}
2a91525c 2516EXPORT_SYMBOL(sock_no_shutdown);
1da177e4
LT		 2517
2518int sock_no_setsockopt(struct socket *sock, int level, int optname,
b7058842 2519 char __user *optval, unsigned int optlen)
1da177e4
LT		 2520{
2521 return -EOPNOTSUPP;
2522}
2a91525c 2523EXPORT_SYMBOL(sock_no_setsockopt);
1da177e4
LT		 2524
2525int sock_no_getsockopt(struct socket *sock, int level, int optname,
2526 char __user *optval, int __user *optlen)
2527{
2528 return -EOPNOTSUPP;
2529}
2a91525c 2530EXPORT_SYMBOL(sock_no_getsockopt);
1da177e4 2531
1b784140 2532int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
1da177e4
LT		 2533{
2534 return -EOPNOTSUPP;
2535}
2a91525c 2536EXPORT_SYMBOL(sock_no_sendmsg);
1da177e4 2537
306b13eb
TH		 2538int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
2539{
2540 return -EOPNOTSUPP;
2541}
2542EXPORT_SYMBOL(sock_no_sendmsg_locked);
2543
1b784140
YX		 2544int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2545 int flags)
1da177e4
LT		 2546{
2547 return -EOPNOTSUPP;
2548}
2a91525c 2549EXPORT_SYMBOL(sock_no_recvmsg);
1da177e4
LT		 2550
2551int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2552{
2553 /* Mirror missing mmap method error code */
2554 return -ENODEV;
2555}
2a91525c 2556EXPORT_SYMBOL(sock_no_mmap);
1da177e4
LT		 2557
2558ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2559{
2560 ssize_t res;
2561 struct msghdr msg = {.msg_flags = flags};
2562 struct kvec iov;
2563 char *kaddr = kmap(page);
2564 iov.iov_base = kaddr + offset;
2565 iov.iov_len = size;
2566 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2567 kunmap(page);
2568 return res;
2569}
2a91525c 2570EXPORT_SYMBOL(sock_no_sendpage);
1da177e4 2571
306b13eb
TH		 2572ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
2573 int offset, size_t size, int flags)
2574{
2575 ssize_t res;
2576 struct msghdr msg = {.msg_flags = flags};
2577 struct kvec iov;
2578 char *kaddr = kmap(page);
2579
2580 iov.iov_base = kaddr + offset;
2581 iov.iov_len = size;
2582 res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
2583 kunmap(page);
2584 return res;
2585}
2586EXPORT_SYMBOL(sock_no_sendpage_locked);
2587
1da177e4
LT		 2588/*
2589 * Default Socket Callbacks
2590 */
2591
2592static void sock_def_wakeup(struct sock *sk)
2593{
43815482
ED		 2594 struct socket_wq *wq;
2595
2596 rcu_read_lock();
2597 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2598 if (skwq_has_sleeper(wq))
43815482
ED		 2599 wake_up_interruptible_all(&wq->wait);
2600 rcu_read_unlock();
1da177e4
LT		 2601}
2602
2603static void sock_def_error_report(struct sock *sk)
2604{
43815482
ED		 2605 struct socket_wq *wq;
2606
2607 rcu_read_lock();
2608 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2609 if (skwq_has_sleeper(wq))
43815482 2610 wake_up_interruptible_poll(&wq->wait, POLLERR);
8d8ad9d7 2611 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
43815482 2612 rcu_read_unlock();
1da177e4
LT		 2613}
2614
676d2369 2615static void sock_def_readable(struct sock *sk)
1da177e4 2616{
43815482
ED		 2617 struct socket_wq *wq;
2618
2619 rcu_read_lock();
2620 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2621 if (skwq_has_sleeper(wq))
2c6607c6 2622 wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI |
37e5540b 2623 POLLRDNORM | POLLRDBAND);
8d8ad9d7 2624 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
43815482 2625 rcu_read_unlock();
1da177e4
LT		 2626}
2627
2628static void sock_def_write_space(struct sock *sk)
2629{
43815482
ED		 2630 struct socket_wq *wq;
2631
2632 rcu_read_lock();
1da177e4
LT		 2633
2634 /* Do not wake up a writer until he can make "significant"
2635 * progress. --DaveM
2636 */
14afee4b 2637 if ((refcount_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
43815482 2638 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2639 if (skwq_has_sleeper(wq))
43815482 2640 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
37e5540b 2641 POLLWRNORM | POLLWRBAND);
1da177e4
LT		 2642
2643 /* Should agree with poll, otherwise some programs break */
2644 if (sock_writeable(sk))
8d8ad9d7 2645 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
1da177e4
LT		 2646 }
2647
43815482 2648 rcu_read_unlock();
1da177e4
LT		 2649}
2650
2651static void sock_def_destruct(struct sock *sk)
2652{
1da177e4
LT		 2653}
2654
2655void sk_send_sigurg(struct sock *sk)
2656{
2657 if (sk->sk_socket && sk->sk_socket->file)
2658 if (send_sigurg(&sk->sk_socket->file->f_owner))
8d8ad9d7 2659 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
1da177e4 2660}
2a91525c 2661EXPORT_SYMBOL(sk_send_sigurg);
1da177e4
LT		 2662
2663void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2664 unsigned long expires)
2665{
2666 if (!mod_timer(timer, expires))
2667 sock_hold(sk);
2668}
1da177e4
LT		 2669EXPORT_SYMBOL(sk_reset_timer);
2670
2671void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2672{
25cc4ae9 2673 if (del_timer(timer))
1da177e4
LT		 2674 __sock_put(sk);
2675}
1da177e4
LT		 2676EXPORT_SYMBOL(sk_stop_timer);
2677
2678void sock_init_data(struct socket *sock, struct sock *sk)
2679{
581319c5 2680 sk_init_common(sk);
1da177e4
LT		 2681 sk->sk_send_head = NULL;
2682
2683 init_timer(&sk->sk_timer);
4ec93edb 2684
1da177e4
LT		 2685 sk->sk_allocation = GFP_KERNEL;
2686 sk->sk_rcvbuf = sysctl_rmem_default;
2687 sk->sk_sndbuf = sysctl_wmem_default;
2688 sk->sk_state = TCP_CLOSE;
972692e0 2689 sk_set_socket(sk, sock);
1da177e4
LT		 2690
2691 sock_set_flag(sk, SOCK_ZAPPED);
2692
e71a4783 2693 if (sock) {
1da177e4 2694 sk->sk_type = sock->type;
43815482 2695 sk->sk_wq = sock->wq;
1da177e4 2696 sock->sk = sk;
86741ec2
LC		 2697 sk->sk_uid = SOCK_INODE(sock)->i_uid;
2698 } else {
43815482 2699 sk->sk_wq = NULL;
86741ec2
LC		 2700 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
2701 }
1da177e4 2702
1da177e4 2703 rwlock_init(&sk->sk_callback_lock);
cdfbabfb
DH		 2704 if (sk->sk_kern_sock)
2705 lockdep_set_class_and_name(
2706 &sk->sk_callback_lock,
2707 af_kern_callback_keys + sk->sk_family,
2708 af_family_kern_clock_key_strings[sk->sk_family]);
2709 else
2710 lockdep_set_class_and_name(
2711 &sk->sk_callback_lock,
443aef0e
PZ		 2712 af_callback_keys + sk->sk_family,
2713 af_family_clock_key_strings[sk->sk_family]);
1da177e4
LT		 2714
2715 sk->sk_state_change = sock_def_wakeup;
2716 sk->sk_data_ready = sock_def_readable;
2717 sk->sk_write_space = sock_def_write_space;
2718 sk->sk_error_report = sock_def_error_report;
2719 sk->sk_destruct = sock_def_destruct;
2720
5640f768
ED		 2721 sk->sk_frag.page = NULL;
2722 sk->sk_frag.offset = 0;
ef64a54f 2723 sk->sk_peek_off = -1;
1da177e4 2724
109f6e39
EB		 2725 sk->sk_peer_pid = NULL;
2726 sk->sk_peer_cred = NULL;
1da177e4
LT		 2727 sk->sk_write_pending = 0;
2728 sk->sk_rcvlowat = 1;
2729 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
2730 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
2731
6c7c98ba 2732 sk->sk_stamp = SK_DEFAULT_STAMP;
52267790 2733 atomic_set(&sk->sk_zckey, 0);
1da177e4 2734
e0d1095a 2735#ifdef CONFIG_NET_RX_BUSY_POLL
06021292 2736 sk->sk_napi_id = 0;
64b0dc51 2737 sk->sk_ll_usec = sysctl_net_busy_read;
06021292
ET		 2738#endif
2739
62748f32 2740 sk->sk_max_pacing_rate = ~0U;
7eec4174 2741 sk->sk_pacing_rate = ~0U;
70da268b 2742 sk->sk_incoming_cpu = -1;
4dc6dc71
ED		 2743 /*
2744 * Before updating sk_refcnt, we must commit prior changes to memory
2745 * (Documentation/RCU/rculist_nulls.txt for details)
2746 */
2747 smp_wmb();
41c6d650 2748 refcount_set(&sk->sk_refcnt, 1);
33c732c3 2749 atomic_set(&sk->sk_drops, 0);
1da177e4 2750}
2a91525c 2751EXPORT_SYMBOL(sock_init_data);
1da177e4 2752
b5606c2d 2753void lock_sock_nested(struct sock *sk, int subclass)
1da177e4
LT		 2754{
2755 might_sleep();
a5b5bb9a 2756 spin_lock_bh(&sk->sk_lock.slock);
d2e9117c 2757 if (sk->sk_lock.owned)
1da177e4 2758 __lock_sock(sk);
d2e9117c 2759 sk->sk_lock.owned = 1;
a5b5bb9a
IM		 2760 spin_unlock(&sk->sk_lock.slock);
2761 /*
2762 * The sk_lock has mutex_lock() semantics here:
2763 */
fcc70d5f 2764 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
a5b5bb9a 2765 local_bh_enable();
1da177e4 2766}
fcc70d5f 2767EXPORT_SYMBOL(lock_sock_nested);
1da177e4 2768
b5606c2d 2769void release_sock(struct sock *sk)
1da177e4 2770{
a5b5bb9a 2771 spin_lock_bh(&sk->sk_lock.slock);
1da177e4
LT		 2772 if (sk->sk_backlog.tail)
2773 __release_sock(sk);
46d3ceab 2774
c3f9b018
ED		 2775 /* Warning : release_cb() might need to release sk ownership,
2776 * ie call sock_release_ownership(sk) before us.
2777 */
46d3ceab
ED		 2778 if (sk->sk_prot->release_cb)
2779 sk->sk_prot->release_cb(sk);
2780
c3f9b018 2781 sock_release_ownership(sk);
a5b5bb9a
IM		 2782 if (waitqueue_active(&sk->sk_lock.wq))
2783 wake_up(&sk->sk_lock.wq);
2784 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4
LT		 2785}
2786EXPORT_SYMBOL(release_sock);
2787
8a74ad60
ED		 2788/**
2789 * lock_sock_fast - fast version of lock_sock
2790 * @sk: socket
2791 *
2792 * This version should be used for very small section, where process wont block
d651983d
MCC		 2793 * return false if fast path is taken:
2794 *
8a74ad60 2795 * sk_lock.slock locked, owned = 0, BH disabled
d651983d
MCC		 2796 *
2797 * return true if slow path is taken:
2798 *
8a74ad60
ED		 2799 * sk_lock.slock unlocked, owned = 1, BH enabled
2800 */
2801bool lock_sock_fast(struct sock *sk)
2802{
2803 might_sleep();
2804 spin_lock_bh(&sk->sk_lock.slock);
2805
2806 if (!sk->sk_lock.owned)
2807 /*
2808 * Note : We must disable BH
2809 */
2810 return false;
2811
2812 __lock_sock(sk);
2813 sk->sk_lock.owned = 1;
2814 spin_unlock(&sk->sk_lock.slock);
2815 /*
2816 * The sk_lock has mutex_lock() semantics here:
2817 */
2818 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
2819 local_bh_enable();
2820 return true;
2821}
2822EXPORT_SYMBOL(lock_sock_fast);
2823
1da177e4 2824int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
4ec93edb 2825{
b7aa0bf7 2826 struct timeval tv;
1da177e4 2827 if (!sock_flag(sk, SOCK_TIMESTAMP))
20d49473 2828 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
b7aa0bf7
ED		 2829 tv = ktime_to_timeval(sk->sk_stamp);
2830 if (tv.tv_sec == -1)
1da177e4 2831 return -ENOENT;
b7aa0bf7
ED		 2832 if (tv.tv_sec == 0) {
2833 sk->sk_stamp = ktime_get_real();
2834 tv = ktime_to_timeval(sk->sk_stamp);
2835 }
2836 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0;
4ec93edb 2837}
1da177e4
LT		 2838EXPORT_SYMBOL(sock_get_timestamp);
2839
ae40eb1e
ED		 2840int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
2841{
2842 struct timespec ts;
2843 if (!sock_flag(sk, SOCK_TIMESTAMP))
20d49473 2844 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
ae40eb1e
ED		 2845 ts = ktime_to_timespec(sk->sk_stamp);
2846 if (ts.tv_sec == -1)
2847 return -ENOENT;
2848 if (ts.tv_sec == 0) {
2849 sk->sk_stamp = ktime_get_real();
2850 ts = ktime_to_timespec(sk->sk_stamp);
2851 }
2852 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0;
2853}
2854EXPORT_SYMBOL(sock_get_timestampns);
2855
20d49473 2856void sock_enable_timestamp(struct sock *sk, int flag)
4ec93edb 2857{
20d49473 2858 if (!sock_flag(sk, flag)) {
08e29af3
ED		 2859 unsigned long previous_flags = sk->sk_flags;
2860
20d49473
PO		 2861 sock_set_flag(sk, flag);
2862 /*
2863 * we just set one of the two flags which require net
2864 * time stamping, but time stamping might have been on
2865 * already because of the other one
2866 */
080a270f
HFS		 2867 if (sock_needs_netstamp(sk) &&
2868 !(previous_flags & SK_FLAGS_TIMESTAMP))
20d49473 2869 net_enable_timestamp();
1da177e4
LT		 2870 }
2871}
1da177e4 2872
cb820f8e
RC		 2873int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
2874 int level, int type)
2875{
2876 struct sock_exterr_skb *serr;
364a9e93 2877 struct sk_buff *skb;
cb820f8e
RC		 2878 int copied, err;
2879
2880 err = -EAGAIN;
364a9e93 2881 skb = sock_dequeue_err_skb(sk);
cb820f8e
RC		 2882 if (skb == NULL)
2883 goto out;
2884
2885 copied = skb->len;
2886 if (copied > len) {
2887 msg->msg_flags |= MSG_TRUNC;
2888 copied = len;
2889 }
51f3d02b 2890 err = skb_copy_datagram_msg(skb, 0, msg, copied);
cb820f8e
RC		 2891 if (err)
2892 goto out_free_skb;
2893
2894 sock_recv_timestamp(msg, sk, skb);
2895
2896 serr = SKB_EXT_ERR(skb);
2897 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
2898
2899 msg->msg_flags |= MSG_ERRQUEUE;
2900 err = copied;
2901
cb820f8e
RC		 2902out_free_skb:
2903 kfree_skb(skb);
2904out:
2905 return err;
2906}
2907EXPORT_SYMBOL(sock_recv_errqueue);
2908
1da177e4
LT		 2909/*
2910 * Get a socket option on an socket.
2911 *
2912 * FIX: POSIX 1003.1g is very ambiguous here. It states that
2913 * asynchronous errors should be reported by getsockopt. We assume
2914 * this means if you specify SO_ERROR (otherwise whats the point of it).
2915 */
2916int sock_common_getsockopt(struct socket *sock, int level, int optname,
2917 char __user *optval, int __user *optlen)
2918{
2919 struct sock *sk = sock->sk;
2920
2921 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2922}
1da177e4
LT		 2923EXPORT_SYMBOL(sock_common_getsockopt);
2924
3fdadf7d 2925#ifdef CONFIG_COMPAT
543d9cfe
ACM		 2926int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
2927 char __user *optval, int __user *optlen)
3fdadf7d
DM		 2928{
2929 struct sock *sk = sock->sk;
2930
1e51f951 2931 if (sk->sk_prot->compat_getsockopt != NULL)
543d9cfe
ACM		 2932 return sk->sk_prot->compat_getsockopt(sk, level, optname,
2933 optval, optlen);
3fdadf7d
DM		 2934 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2935}
2936EXPORT_SYMBOL(compat_sock_common_getsockopt);
2937#endif
2938
1b784140
YX		 2939int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
2940 int flags)
1da177e4
LT		 2941{
2942 struct sock *sk = sock->sk;
2943 int addr_len = 0;
2944 int err;
2945
1b784140 2946 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
1da177e4
LT		 2947 flags & ~MSG_DONTWAIT, &addr_len);
2948 if (err >= 0)
2949 msg->msg_namelen = addr_len;
2950 return err;
2951}
1da177e4
LT		 2952EXPORT_SYMBOL(sock_common_recvmsg);
2953
2954/*
2955 * Set socket options on an inet socket.
2956 */
2957int sock_common_setsockopt(struct socket *sock, int level, int optname,
b7058842 2958 char __user *optval, unsigned int optlen)
1da177e4
LT		 2959{
2960 struct sock *sk = sock->sk;
2961
2962 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2963}
1da177e4
LT		 2964EXPORT_SYMBOL(sock_common_setsockopt);
2965
3fdadf7d 2966#ifdef CONFIG_COMPAT
543d9cfe 2967int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
b7058842 2968 char __user *optval, unsigned int optlen)
3fdadf7d
DM		 2969{
2970 struct sock *sk = sock->sk;
2971
543d9cfe
ACM		 2972 if (sk->sk_prot->compat_setsockopt != NULL)
2973 return sk->sk_prot->compat_setsockopt(sk, level, optname,
2974 optval, optlen);
3fdadf7d
DM		 2975 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2976}
2977EXPORT_SYMBOL(compat_sock_common_setsockopt);
2978#endif
2979
1da177e4
LT		 2980void sk_common_release(struct sock *sk)
2981{
2982 if (sk->sk_prot->destroy)
2983 sk->sk_prot->destroy(sk);
2984
2985 /*
2986 * Observation: when sock_common_release is called, processes have
2987 * no access to socket. But net still has.
2988 * Step one, detach it from networking:
2989 *
2990 * A. Remove from hash tables.
2991 */
2992
2993 sk->sk_prot->unhash(sk);
2994
2995 /*
2996 * In this point socket cannot receive new packets, but it is possible
2997 * that some packets are in flight because some CPU runs receiver and
2998 * did hash table lookup before we unhashed socket. They will achieve
2999 * receive queue and will be purged by socket destructor.
3000 *
3001 * Also we still have packets pending on receive queue and probably,
3002 * our own packets waiting in device queues. sock_destroy will drain
3003 * receive queue, but transmitted packets will delay socket destruction
3004 * until the last reference will be released.
3005 */
3006
3007 sock_orphan(sk);
3008
3009 xfrm_sk_free_policy(sk);
3010
e6848976 3011 sk_refcnt_debug_release(sk);
5640f768 3012
1da177e4
LT		 3013 sock_put(sk);
3014}
1da177e4
LT		 3015EXPORT_SYMBOL(sk_common_release);
3016
a2d133b1
JH		 3017void sk_get_meminfo(const struct sock *sk, u32 *mem)
3018{
3019 memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
3020
3021 mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
3022 mem[SK_MEMINFO_RCVBUF] = sk->sk_rcvbuf;
3023 mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
3024 mem[SK_MEMINFO_SNDBUF] = sk->sk_sndbuf;
3025 mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
3026 mem[SK_MEMINFO_WMEM_QUEUED] = sk->sk_wmem_queued;
3027 mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
3028 mem[SK_MEMINFO_BACKLOG] = sk->sk_backlog.len;
3029 mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
3030}
3031
13ff3d6f
PE		 3032#ifdef CONFIG_PROC_FS
3033#define PROTO_INUSE_NR 64 /* should be enough for the first time */
1338d466
PE		 3034struct prot_inuse {
3035 int val[PROTO_INUSE_NR];
3036};
13ff3d6f
PE		 3037
3038static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
70ee1159
PE		 3039
3040#ifdef CONFIG_NET_NS
3041void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
3042{
d6d9ca0f 3043 __this_cpu_add(net->core.inuse->val[prot->inuse_idx], val);
70ee1159
PE		 3044}
3045EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
3046
3047int sock_prot_inuse_get(struct net *net, struct proto *prot)
3048{
3049 int cpu, idx = prot->inuse_idx;
3050 int res = 0;
3051
3052 for_each_possible_cpu(cpu)
3053 res += per_cpu_ptr(net->core.inuse, cpu)->val[idx];
3054
3055 return res >= 0 ? res : 0;
3056}
3057EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
3058
2c8c1e72 3059static int __net_init sock_inuse_init_net(struct net *net)
70ee1159
PE		 3060{
3061 net->core.inuse = alloc_percpu(struct prot_inuse);
3062 return net->core.inuse ? 0 : -ENOMEM;
3063}
3064
2c8c1e72 3065static void __net_exit sock_inuse_exit_net(struct net *net)
70ee1159
PE		 3066{
3067 free_percpu(net->core.inuse);
3068}
3069
3070static struct pernet_operations net_inuse_ops = {
3071 .init = sock_inuse_init_net,
3072 .exit = sock_inuse_exit_net,
3073};
3074
3075static __init int net_inuse_init(void)
3076{
3077 if (register_pernet_subsys(&net_inuse_ops))
3078 panic("Cannot initialize net inuse counters");
3079
3080 return 0;
3081}
3082
3083core_initcall(net_inuse_init);
3084#else
1338d466
PE		 3085static DEFINE_PER_CPU(struct prot_inuse, prot_inuse);
3086
c29a0bc4 3087void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
1338d466 3088{
d6d9ca0f 3089 __this_cpu_add(prot_inuse.val[prot->inuse_idx], val);
1338d466
PE		 3090}
3091EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
3092
c29a0bc4 3093int sock_prot_inuse_get(struct net *net, struct proto *prot)
1338d466
PE		 3094{
3095 int cpu, idx = prot->inuse_idx;
3096 int res = 0;
3097
3098 for_each_possible_cpu(cpu)
3099 res += per_cpu(prot_inuse, cpu).val[idx];
3100
3101 return res >= 0 ? res : 0;
3102}
3103EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
70ee1159 3104#endif
13ff3d6f
PE		 3105
3106static void assign_proto_idx(struct proto *prot)
3107{
3108 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3109
3110 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
e005d193 3111 pr_err("PROTO_INUSE_NR exhausted\n");
13ff3d6f
PE		 3112 return;
3113 }
3114
3115 set_bit(prot->inuse_idx, proto_inuse_idx);
3116}
3117
3118static void release_proto_idx(struct proto *prot)
3119{
3120 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3121 clear_bit(prot->inuse_idx, proto_inuse_idx);
3122}
3123#else
3124static inline void assign_proto_idx(struct proto *prot)
3125{
3126}
3127
3128static inline void release_proto_idx(struct proto *prot)
3129{
3130}
3131#endif
3132
0159dfd3
ED		 3133static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3134{
3135 if (!rsk_prot)
3136 return;
3137 kfree(rsk_prot->slab_name);
3138 rsk_prot->slab_name = NULL;
adf78eda
JL		 3139 kmem_cache_destroy(rsk_prot->slab);
3140 rsk_prot->slab = NULL;
0159dfd3
ED		 3141}
3142
3143static int req_prot_init(const struct proto *prot)
3144{
3145 struct request_sock_ops *rsk_prot = prot->rsk_prot;
3146
3147 if (!rsk_prot)
3148 return 0;
3149
3150 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3151 prot->name);
3152 if (!rsk_prot->slab_name)
3153 return -ENOMEM;
3154
3155 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3156 rsk_prot->obj_size, 0,
e96f78ab 3157 prot->slab_flags, NULL);
0159dfd3
ED		 3158
3159 if (!rsk_prot->slab) {
3160 pr_crit("%s: Can't create request sock SLAB cache!\n",
3161 prot->name);
3162 return -ENOMEM;
3163 }
3164 return 0;
3165}
3166
b733c007
PE		 3167int proto_register(struct proto *prot, int alloc_slab)
3168{
1da177e4
LT		 3169 if (alloc_slab) {
3170 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
271b72c7
ED		 3171 SLAB_HWCACHE_ALIGN | prot->slab_flags,
3172 NULL);
1da177e4
LT		 3173
3174 if (prot->slab == NULL) {
e005d193
JP		 3175 pr_crit("%s: Can't create sock SLAB cache!\n",
3176 prot->name);
60e7663d 3177 goto out;
1da177e4 3178 }
2e6599cb 3179
0159dfd3
ED		 3180 if (req_prot_init(prot))
3181 goto out_free_request_sock_slab;
8feaf0c0 3182
6d6ee43e 3183 if (prot->twsk_prot != NULL) {
faf23422 3184 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
8feaf0c0 3185
7e56b5d6 3186 if (prot->twsk_prot->twsk_slab_name == NULL)
8feaf0c0
ACM		 3187 goto out_free_request_sock_slab;
3188
6d6ee43e 3189 prot->twsk_prot->twsk_slab =
7e56b5d6 3190 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
6d6ee43e 3191 prot->twsk_prot->twsk_obj_size,
3ab5aee7 3192 0,
52db70dc 3193 prot->slab_flags,
20c2df83 3194 NULL);
6d6ee43e 3195 if (prot->twsk_prot->twsk_slab == NULL)
8feaf0c0
ACM		 3196 goto out_free_timewait_sock_slab_name;
3197 }
1da177e4
LT		 3198 }
3199
36b77a52 3200 mutex_lock(&proto_list_mutex);
1da177e4 3201 list_add(&prot->node, &proto_list);
13ff3d6f 3202 assign_proto_idx(prot);
36b77a52 3203 mutex_unlock(&proto_list_mutex);
b733c007
PE		 3204 return 0;
3205
8feaf0c0 3206out_free_timewait_sock_slab_name:
7e56b5d6 3207 kfree(prot->twsk_prot->twsk_slab_name);
8feaf0c0 3208out_free_request_sock_slab:
0159dfd3
ED		 3209 req_prot_cleanup(prot->rsk_prot);
3210
2e6599cb
ACM		 3211 kmem_cache_destroy(prot->slab);
3212 prot->slab = NULL;
b733c007
PE		 3213out:
3214 return -ENOBUFS;
1da177e4 3215}
1da177e4
LT		 3216EXPORT_SYMBOL(proto_register);
3217
3218void proto_unregister(struct proto *prot)
3219{
36b77a52 3220 mutex_lock(&proto_list_mutex);
13ff3d6f 3221 release_proto_idx(prot);
0a3f4358 3222 list_del(&prot->node);
36b77a52 3223 mutex_unlock(&proto_list_mutex);
1da177e4 3224
adf78eda
JL		 3225 kmem_cache_destroy(prot->slab);
3226 prot->slab = NULL;
1da177e4 3227
0159dfd3 3228 req_prot_cleanup(prot->rsk_prot);
2e6599cb 3229
6d6ee43e 3230 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
6d6ee43e 3231 kmem_cache_destroy(prot->twsk_prot->twsk_slab);
7e56b5d6 3232 kfree(prot->twsk_prot->twsk_slab_name);
6d6ee43e 3233 prot->twsk_prot->twsk_slab = NULL;
8feaf0c0 3234 }
1da177e4 3235}
1da177e4
LT		 3236EXPORT_SYMBOL(proto_unregister);
3237
3238#ifdef CONFIG_PROC_FS
1da177e4 3239static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
36b77a52 3240 __acquires(proto_list_mutex)
1da177e4 3241{
36b77a52 3242 mutex_lock(&proto_list_mutex);
60f0438a 3243 return seq_list_start_head(&proto_list, *pos);
1da177e4
LT		 3244}
3245
3246static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3247{
60f0438a 3248 return seq_list_next(v, &proto_list, pos);
1da177e4
LT		 3249}
3250
3251static void proto_seq_stop(struct seq_file *seq, void *v)
36b77a52 3252 __releases(proto_list_mutex)
1da177e4 3253{
36b77a52 3254 mutex_unlock(&proto_list_mutex);
1da177e4
LT		 3255}
3256
3257static char proto_method_implemented(const void *method)
3258{
3259 return method == NULL ? 'n' : 'y';
3260}
180d8cd9
GC		 3261static long sock_prot_memory_allocated(struct proto *proto)
3262{
cb75a36c 3263 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
180d8cd9
GC		 3264}
3265
3266static char *sock_prot_memory_pressure(struct proto *proto)
3267{
3268 return proto->memory_pressure != NULL ?
3269 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3270}
1da177e4
LT		 3271
3272static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3273{
180d8cd9 3274
8d987e5c 3275 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
1da177e4
LT		 3276 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3277 proto->name,
3278 proto->obj_size,
14e943db 3279 sock_prot_inuse_get(seq_file_net(seq), proto),
180d8cd9
GC		 3280 sock_prot_memory_allocated(proto),
3281 sock_prot_memory_pressure(proto),
1da177e4
LT		 3282 proto->max_header,
3283 proto->slab == NULL ? "no" : "yes",
3284 module_name(proto->owner),
3285 proto_method_implemented(proto->close),
3286 proto_method_implemented(proto->connect),
3287 proto_method_implemented(proto->disconnect),
3288 proto_method_implemented(proto->accept),
3289 proto_method_implemented(proto->ioctl),
3290 proto_method_implemented(proto->init),
3291 proto_method_implemented(proto->destroy),
3292 proto_method_implemented(proto->shutdown),
3293 proto_method_implemented(proto->setsockopt),
3294 proto_method_implemented(proto->getsockopt),
3295 proto_method_implemented(proto->sendmsg),
3296 proto_method_implemented(proto->recvmsg),
3297 proto_method_implemented(proto->sendpage),
3298 proto_method_implemented(proto->bind),
3299 proto_method_implemented(proto->backlog_rcv),
3300 proto_method_implemented(proto->hash),
3301 proto_method_implemented(proto->unhash),
3302 proto_method_implemented(proto->get_port),
3303 proto_method_implemented(proto->enter_memory_pressure));
3304}
3305
3306static int proto_seq_show(struct seq_file *seq, void *v)
3307{
60f0438a 3308 if (v == &proto_list)
1da177e4
LT		 3309 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3310 "protocol",
3311 "size",
3312 "sockets",
3313 "memory",
3314 "press",
3315 "maxhdr",
3316 "slab",
3317 "module",
3318 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3319 else
60f0438a 3320 proto_seq_printf(seq, list_entry(v, struct proto, node));
1da177e4
LT		 3321 return 0;
3322}
3323
f690808e 3324static const struct seq_operations proto_seq_ops = {
1da177e4
LT		 3325 .start = proto_seq_start,
3326 .next = proto_seq_next,
3327 .stop = proto_seq_stop,
3328 .show = proto_seq_show,
3329};
3330
3331static int proto_seq_open(struct inode *inode, struct file *file)
3332{
14e943db
ED		 3333 return seq_open_net(inode, file, &proto_seq_ops,
3334 sizeof(struct seq_net_private));
1da177e4
LT		 3335}
3336
9a32144e 3337static const struct file_operations proto_seq_fops = {
1da177e4
LT		 3338 .owner = THIS_MODULE,
3339 .open = proto_seq_open,
3340 .read = seq_read,
3341 .llseek = seq_lseek,
14e943db
ED		 3342 .release = seq_release_net,
3343};
3344
3345static __net_init int proto_init_net(struct net *net)
3346{
d4beaa66 3347 if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops))
14e943db
ED		 3348 return -ENOMEM;
3349
3350 return 0;
3351}
3352
3353static __net_exit void proto_exit_net(struct net *net)
3354{
ece31ffd 3355 remove_proc_entry("protocols", net->proc_net);
14e943db
ED		 3356}
3357
3358
3359static __net_initdata struct pernet_operations proto_net_ops = {
3360 .init = proto_init_net,
3361 .exit = proto_exit_net,
1da177e4
LT		 3362};
3363
3364static int __init proto_init(void)
3365{
14e943db 3366 return register_pernet_subsys(&proto_net_ops);
1da177e4
LT		 3367}
3368
3369subsys_initcall(proto_init);
3370
3371#endif /* PROC_FS */
7db6b048
SS		 3372
3373#ifdef CONFIG_NET_RX_BUSY_POLL
3374bool sk_busy_loop_end(void *p, unsigned long start_time)
3375{
3376 struct sock *sk = p;
3377
3378 return !skb_queue_empty(&sk->sk_receive_queue) ||
3379 sk_busy_loop_timeout(sk, start_time);
3380}
3381EXPORT_SYMBOL(sk_busy_loop_end);
3382#endif /* CONFIG_NET_RX_BUSY_POLL */
Linux 6.x block layer and io_uring tree(s)