projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mlxsw: spectrum: fix swapped order of arguments packets and bytes
[linux-2.6-block.git] / net / core / sock.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * Generic socket support routines. Memory allocators, socket lock/release
7 * handler for protocols to use and generic option handler.
8 *
9 *
02c30a84 10 * Authors: Ross Biro
1da177e4
LT		 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12 * Florian La Roche, <flla@stud.uni-sb.de>
13 * Alan Cox, <A.Cox@swansea.ac.uk>
14 *
15 * Fixes:
16 * Alan Cox : Numerous verify_area() problems
17 * Alan Cox : Connecting on a connecting socket
18 * now returns an error for tcp.
19 * Alan Cox : sock->protocol is set correctly.
20 * and is not sometimes left as 0.
21 * Alan Cox : connect handles icmp errors on a
22 * connect properly. Unfortunately there
23 * is a restart syscall nasty there. I
24 * can't match BSD without hacking the C
25 * library. Ideas urgently sought!
26 * Alan Cox : Disallow bind() to addresses that are
27 * not ours - especially broadcast ones!!
28 * Alan Cox : Socket 1024 _IS_ ok for users. (fencepost)
29 * Alan Cox : sock_wfree/sock_rfree don't destroy sockets,
30 * instead they leave that for the DESTROY timer.
31 * Alan Cox : Clean up error flag in accept
32 * Alan Cox : TCP ack handling is buggy, the DESTROY timer
33 * was buggy. Put a remove_sock() in the handler
34 * for memory when we hit 0. Also altered the timer
4ec93edb 35 * code. The ACK stuff can wait and needs major
1da177e4
LT		 36 * TCP layer surgery.
37 * Alan Cox : Fixed TCP ack bug, removed remove sock
38 * and fixed timer/inet_bh race.
39 * Alan Cox : Added zapped flag for TCP
40 * Alan Cox : Move kfree_skb into skbuff.c and tidied up surplus code
41 * Alan Cox : for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42 * Alan Cox : kfree_s calls now are kfree_skbmem so we can track skb resources
43 * Alan Cox : Supports socket option broadcast now as does udp. Packet and raw need fixing.
44 * Alan Cox : Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45 * Rick Sladkey : Relaxed UDP rules for matching packets.
46 * C.E.Hawkins : IFF_PROMISC/SIOCGHWADDR support
47 * Pauline Middelink : identd support
48 * Alan Cox : Fixed connect() taking signals I think.
49 * Alan Cox : SO_LINGER supported
50 * Alan Cox : Error reporting fixes
51 * Anonymous : inet_create tidied up (sk->reuse setting)
52 * Alan Cox : inet sockets don't set sk->type!
53 * Alan Cox : Split socket option code
54 * Alan Cox : Callbacks
55 * Alan Cox : Nagle flag for Charles & Johannes stuff
56 * Alex : Removed restriction on inet fioctl
57 * Alan Cox : Splitting INET from NET core
58 * Alan Cox : Fixed bogus SO_TYPE handling in getsockopt()
59 * Adam Caldwell : Missing return in SO_DONTROUTE/SO_DEBUG code
60 * Alan Cox : Split IP from generic code
61 * Alan Cox : New kfree_skbmem()
62 * Alan Cox : Make SO_DEBUG superuser only.
63 * Alan Cox : Allow anyone to clear SO_DEBUG
64 * (compatibility fix)
65 * Alan Cox : Added optimistic memory grabbing for AF_UNIX throughput.
66 * Alan Cox : Allocator for a socket is settable.
67 * Alan Cox : SO_ERROR includes soft errors.
68 * Alan Cox : Allow NULL arguments on some SO_ opts
69 * Alan Cox : Generic socket allocation to make hooks
70 * easier (suggested by Craig Metz).
71 * Michael Pall : SO_ERROR returns positive errno again
72 * Steve Whitehouse: Added default destructor to free
73 * protocol private data.
74 * Steve Whitehouse: Added various other default routines
75 * common to several socket families.
76 * Chris Evans : Call suser() check last on F_SETOWN
77 * Jay Schulist : Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78 * Andi Kleen : Add sock_kmalloc()/sock_kfree_s()
79 * Andi Kleen : Fix write_space callback
80 * Chris Evans : Security fixes - signedness again
81 * Arnaldo C. Melo : cleanups, use skb_queue_purge
82 *
83 * To Fix:
84 *
85 *
86 * This program is free software; you can redistribute it and/or
87 * modify it under the terms of the GNU General Public License
88 * as published by the Free Software Foundation; either version
89 * 2 of the License, or (at your option) any later version.
90 */
91
e005d193
JP		 92#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
93
4fc268d2 94#include <linux/capability.h>
1da177e4 95#include <linux/errno.h>
cb820f8e 96#include <linux/errqueue.h>
1da177e4
LT		 97#include <linux/types.h>
98#include <linux/socket.h>
99#include <linux/in.h>
100#include <linux/kernel.h>
1da177e4
LT		 101#include <linux/module.h>
102#include <linux/proc_fs.h>
103#include <linux/seq_file.h>
104#include <linux/sched.h>
105#include <linux/timer.h>
106#include <linux/string.h>
107#include <linux/sockios.h>
108#include <linux/net.h>
109#include <linux/mm.h>
110#include <linux/slab.h>
111#include <linux/interrupt.h>
112#include <linux/poll.h>
113#include <linux/tcp.h>
114#include <linux/init.h>
a1f8e7f7 115#include <linux/highmem.h>
3f551f94 116#include <linux/user_namespace.h>
c5905afb 117#include <linux/static_key.h>
3969eb38 118#include <linux/memcontrol.h>
8c1ae10d 119#include <linux/prefetch.h>
1da177e4 120
7c0f6ba6 121#include <linux/uaccess.h>
1da177e4
LT		 122
123#include <linux/netdevice.h>
124#include <net/protocol.h>
125#include <linux/skbuff.h>
457c4cbc 126#include <net/net_namespace.h>
2e6599cb 127#include <net/request_sock.h>
1da177e4 128#include <net/sock.h>
20d49473 129#include <linux/net_tstamp.h>
1da177e4
LT		 130#include <net/xfrm.h>
131#include <linux/ipsec.h>
f8451725 132#include <net/cls_cgroup.h>
5bc1421e 133#include <net/netprio_cgroup.h>
eb4cb008 134#include <linux/sock_diag.h>
1da177e4
LT		 135
136#include <linux/filter.h>
538950a1 137#include <net/sock_reuseport.h>
1da177e4 138
3847ce32
SM		 139#include <trace/events/sock.h>
140
1da177e4
LT		 141#ifdef CONFIG_INET
142#include <net/tcp.h>
143#endif
144
076bb0c8 145#include <net/busy_poll.h>
06021292 146
36b77a52 147static DEFINE_MUTEX(proto_list_mutex);
d1a4c0b3
GC		 148static LIST_HEAD(proto_list);
149
a3b299da
EB		 150/**
151 * sk_ns_capable - General socket capability test
152 * @sk: Socket to use a capability on or through
153 * @user_ns: The user namespace of the capability to use
154 * @cap: The capability to use
155 *
156 * Test to see if the opener of the socket had when the socket was
157 * created and the current process has the capability @cap in the user
158 * namespace @user_ns.
159 */
160bool sk_ns_capable(const struct sock *sk,
161 struct user_namespace *user_ns, int cap)
162{
163 return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
164 ns_capable(user_ns, cap);
165}
166EXPORT_SYMBOL(sk_ns_capable);
167
168/**
169 * sk_capable - Socket global capability test
170 * @sk: Socket to use a capability on or through
e793c0f7 171 * @cap: The global capability to use
a3b299da
EB		 172 *
173 * Test to see if the opener of the socket had when the socket was
174 * created and the current process has the capability @cap in all user
175 * namespaces.
176 */
177bool sk_capable(const struct sock *sk, int cap)
178{
179 return sk_ns_capable(sk, &init_user_ns, cap);
180}
181EXPORT_SYMBOL(sk_capable);
182
183/**
184 * sk_net_capable - Network namespace socket capability test
185 * @sk: Socket to use a capability on or through
186 * @cap: The capability to use
187 *
e793c0f7 188 * Test to see if the opener of the socket had when the socket was created
a3b299da
EB		 189 * and the current process has the capability @cap over the network namespace
190 * the socket is a member of.
191 */
192bool sk_net_capable(const struct sock *sk, int cap)
193{
194 return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
195}
196EXPORT_SYMBOL(sk_net_capable);
197
da21f24d
IM		 198/*
199 * Each address family might have different locking rules, so we have
cdfbabfb
DH		 200 * one slock key per address family and separate keys for internal and
201 * userspace sockets.
da21f24d 202 */
a5b5bb9a 203static struct lock_class_key af_family_keys[AF_MAX];
cdfbabfb 204static struct lock_class_key af_family_kern_keys[AF_MAX];
a5b5bb9a 205static struct lock_class_key af_family_slock_keys[AF_MAX];
cdfbabfb 206static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
a5b5bb9a 207
a5b5bb9a
IM		 208/*
209 * Make lock validator output more readable. (we pre-construct these
210 * strings build-time, so that runtime initialization of socket
211 * locks is fast):
212 */
cdfbabfb
DH		 213
214#define _sock_locks(x) \
215 x "AF_UNSPEC", x "AF_UNIX" , x "AF_INET" , \
216 x "AF_AX25" , x "AF_IPX" , x "AF_APPLETALK", \
217 x "AF_NETROM", x "AF_BRIDGE" , x "AF_ATMPVC" , \
218 x "AF_X25" , x "AF_INET6" , x "AF_ROSE" , \
219 x "AF_DECnet", x "AF_NETBEUI" , x "AF_SECURITY" , \
220 x "AF_KEY" , x "AF_NETLINK" , x "AF_PACKET" , \
221 x "AF_ASH" , x "AF_ECONET" , x "AF_ATMSVC" , \
222 x "AF_RDS" , x "AF_SNA" , x "AF_IRDA" , \
223 x "AF_PPPOX" , x "AF_WANPIPE" , x "AF_LLC" , \
224 x "27" , x "28" , x "AF_CAN" , \
225 x "AF_TIPC" , x "AF_BLUETOOTH", x "IUCV" , \
226 x "AF_RXRPC" , x "AF_ISDN" , x "AF_PHONET" , \
227 x "AF_IEEE802154", x "AF_CAIF" , x "AF_ALG" , \
228 x "AF_NFC" , x "AF_VSOCK" , x "AF_KCM" , \
229 x "AF_QIPCRTR", x "AF_SMC" , x "AF_MAX"
230
36cbd3dc 231static const char *const af_family_key_strings[AF_MAX+1] = {
cdfbabfb 232 _sock_locks("sk_lock-")
a5b5bb9a 233};
36cbd3dc 234static const char *const af_family_slock_key_strings[AF_MAX+1] = {
cdfbabfb 235 _sock_locks("slock-")
a5b5bb9a 236};
36cbd3dc 237static const char *const af_family_clock_key_strings[AF_MAX+1] = {
cdfbabfb
DH		 238 _sock_locks("clock-")
239};
240
241static const char *const af_family_kern_key_strings[AF_MAX+1] = {
242 _sock_locks("k-sk_lock-")
243};
244static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
245 _sock_locks("k-slock-")
246};
247static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
248 _sock_locks("k-clock-")
443aef0e 249};
581319c5
PA		 250static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
251 "rlock-AF_UNSPEC", "rlock-AF_UNIX" , "rlock-AF_INET" ,
252 "rlock-AF_AX25" , "rlock-AF_IPX" , "rlock-AF_APPLETALK",
253 "rlock-AF_NETROM", "rlock-AF_BRIDGE" , "rlock-AF_ATMPVC" ,
254 "rlock-AF_X25" , "rlock-AF_INET6" , "rlock-AF_ROSE" ,
255 "rlock-AF_DECnet", "rlock-AF_NETBEUI" , "rlock-AF_SECURITY" ,
256 "rlock-AF_KEY" , "rlock-AF_NETLINK" , "rlock-AF_PACKET" ,
257 "rlock-AF_ASH" , "rlock-AF_ECONET" , "rlock-AF_ATMSVC" ,
258 "rlock-AF_RDS" , "rlock-AF_SNA" , "rlock-AF_IRDA" ,
259 "rlock-AF_PPPOX" , "rlock-AF_WANPIPE" , "rlock-AF_LLC" ,
260 "rlock-27" , "rlock-28" , "rlock-AF_CAN" ,
261 "rlock-AF_TIPC" , "rlock-AF_BLUETOOTH", "rlock-AF_IUCV" ,
262 "rlock-AF_RXRPC" , "rlock-AF_ISDN" , "rlock-AF_PHONET" ,
263 "rlock-AF_IEEE802154", "rlock-AF_CAIF" , "rlock-AF_ALG" ,
264 "rlock-AF_NFC" , "rlock-AF_VSOCK" , "rlock-AF_KCM" ,
265 "rlock-AF_QIPCRTR", "rlock-AF_SMC" , "rlock-AF_MAX"
266};
267static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
268 "wlock-AF_UNSPEC", "wlock-AF_UNIX" , "wlock-AF_INET" ,
269 "wlock-AF_AX25" , "wlock-AF_IPX" , "wlock-AF_APPLETALK",
270 "wlock-AF_NETROM", "wlock-AF_BRIDGE" , "wlock-AF_ATMPVC" ,
271 "wlock-AF_X25" , "wlock-AF_INET6" , "wlock-AF_ROSE" ,
272 "wlock-AF_DECnet", "wlock-AF_NETBEUI" , "wlock-AF_SECURITY" ,
273 "wlock-AF_KEY" , "wlock-AF_NETLINK" , "wlock-AF_PACKET" ,
274 "wlock-AF_ASH" , "wlock-AF_ECONET" , "wlock-AF_ATMSVC" ,
275 "wlock-AF_RDS" , "wlock-AF_SNA" , "wlock-AF_IRDA" ,
276 "wlock-AF_PPPOX" , "wlock-AF_WANPIPE" , "wlock-AF_LLC" ,
277 "wlock-27" , "wlock-28" , "wlock-AF_CAN" ,
278 "wlock-AF_TIPC" , "wlock-AF_BLUETOOTH", "wlock-AF_IUCV" ,
279 "wlock-AF_RXRPC" , "wlock-AF_ISDN" , "wlock-AF_PHONET" ,
280 "wlock-AF_IEEE802154", "wlock-AF_CAIF" , "wlock-AF_ALG" ,
281 "wlock-AF_NFC" , "wlock-AF_VSOCK" , "wlock-AF_KCM" ,
282 "wlock-AF_QIPCRTR", "wlock-AF_SMC" , "wlock-AF_MAX"
283};
284static const char *const af_family_elock_key_strings[AF_MAX+1] = {
285 "elock-AF_UNSPEC", "elock-AF_UNIX" , "elock-AF_INET" ,
286 "elock-AF_AX25" , "elock-AF_IPX" , "elock-AF_APPLETALK",
287 "elock-AF_NETROM", "elock-AF_BRIDGE" , "elock-AF_ATMPVC" ,
288 "elock-AF_X25" , "elock-AF_INET6" , "elock-AF_ROSE" ,
289 "elock-AF_DECnet", "elock-AF_NETBEUI" , "elock-AF_SECURITY" ,
290 "elock-AF_KEY" , "elock-AF_NETLINK" , "elock-AF_PACKET" ,
291 "elock-AF_ASH" , "elock-AF_ECONET" , "elock-AF_ATMSVC" ,
292 "elock-AF_RDS" , "elock-AF_SNA" , "elock-AF_IRDA" ,
293 "elock-AF_PPPOX" , "elock-AF_WANPIPE" , "elock-AF_LLC" ,
294 "elock-27" , "elock-28" , "elock-AF_CAN" ,
295 "elock-AF_TIPC" , "elock-AF_BLUETOOTH", "elock-AF_IUCV" ,
296 "elock-AF_RXRPC" , "elock-AF_ISDN" , "elock-AF_PHONET" ,
297 "elock-AF_IEEE802154", "elock-AF_CAIF" , "elock-AF_ALG" ,
298 "elock-AF_NFC" , "elock-AF_VSOCK" , "elock-AF_KCM" ,
299 "elock-AF_QIPCRTR", "elock-AF_SMC" , "elock-AF_MAX"
300};
da21f24d
IM		 301
302/*
581319c5 303 * sk_callback_lock and sk queues locking rules are per-address-family,
da21f24d
IM		 304 * so split the lock classes by using a per-AF key:
305 */
306static struct lock_class_key af_callback_keys[AF_MAX];
581319c5
PA		 307static struct lock_class_key af_rlock_keys[AF_MAX];
308static struct lock_class_key af_wlock_keys[AF_MAX];
309static struct lock_class_key af_elock_keys[AF_MAX];
cdfbabfb 310static struct lock_class_key af_kern_callback_keys[AF_MAX];
da21f24d 311
1da177e4
LT		 312/* Take into consideration the size of the struct sk_buff overhead in the
313 * determination of these values, since that is non-constant across
314 * platforms. This makes socket queueing behavior and performance
315 * not depend upon such differences.
316 */
317#define _SK_MEM_PACKETS 256
87fb4b7b 318#define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
1da177e4
LT		 319#define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
320#define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
321
322/* Run time adjustable parameters. */
ab32ea5d 323__u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
6d8ebc8a 324EXPORT_SYMBOL(sysctl_wmem_max);
ab32ea5d 325__u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
6d8ebc8a 326EXPORT_SYMBOL(sysctl_rmem_max);
ab32ea5d
BH		 327__u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
328__u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
1da177e4 329
25985edc 330/* Maximal space eaten by iovec or ancillary data plus some space */
ab32ea5d 331int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
2a91525c 332EXPORT_SYMBOL(sysctl_optmem_max);
1da177e4 333
b245be1f
WB		 334int sysctl_tstamp_allow_data __read_mostly = 1;
335
c93bdd0e
MG		 336struct static_key memalloc_socks = STATIC_KEY_INIT_FALSE;
337EXPORT_SYMBOL_GPL(memalloc_socks);
338
7cb02404
MG		 339/**
340 * sk_set_memalloc - sets %SOCK_MEMALLOC
341 * @sk: socket to set it on
342 *
343 * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
344 * It's the responsibility of the admin to adjust min_free_kbytes
345 * to meet the requirements
346 */
347void sk_set_memalloc(struct sock *sk)
348{
349 sock_set_flag(sk, SOCK_MEMALLOC);
350 sk->sk_allocation |= __GFP_MEMALLOC;
c93bdd0e 351 static_key_slow_inc(&memalloc_socks);
7cb02404
MG		 352}
353EXPORT_SYMBOL_GPL(sk_set_memalloc);
354
355void sk_clear_memalloc(struct sock *sk)
356{
357 sock_reset_flag(sk, SOCK_MEMALLOC);
358 sk->sk_allocation &= ~__GFP_MEMALLOC;
c93bdd0e 359 static_key_slow_dec(&memalloc_socks);
c76562b6
MG		 360
361 /*
362 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
5d753610
MG		 363 * progress of swapping. SOCK_MEMALLOC may be cleared while
364 * it has rmem allocations due to the last swapfile being deactivated
365 * but there is a risk that the socket is unusable due to exceeding
366 * the rmem limits. Reclaim the reserves and obey rmem limits again.
c76562b6 367 */
5d753610 368 sk_mem_reclaim(sk);
7cb02404
MG		 369}
370EXPORT_SYMBOL_GPL(sk_clear_memalloc);
371
b4b9e355
MG		 372int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
373{
374 int ret;
375 unsigned long pflags = current->flags;
376
377 /* these should have been dropped before queueing */
378 BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
379
380 current->flags |= PF_MEMALLOC;
381 ret = sk->sk_backlog_rcv(sk, skb);
382 tsk_restore_flags(current, pflags, PF_MEMALLOC);
383
384 return ret;
385}
386EXPORT_SYMBOL(__sk_backlog_rcv);
387
1da177e4
LT		 388static int sock_set_timeout(long *timeo_p, char __user *optval, int optlen)
389{
390 struct timeval tv;
391
392 if (optlen < sizeof(tv))
393 return -EINVAL;
394 if (copy_from_user(&tv, optval, sizeof(tv)))
395 return -EFAULT;
ba78073e
VA		 396 if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
397 return -EDOM;
1da177e4 398
ba78073e 399 if (tv.tv_sec < 0) {
6f11df83
AM		 400 static int warned __read_mostly;
401
ba78073e 402 *timeo_p = 0;
50aab54f 403 if (warned < 10 && net_ratelimit()) {
ba78073e 404 warned++;
e005d193
JP		 405 pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
406 __func__, current->comm, task_pid_nr(current));
50aab54f 407 }
ba78073e
VA		 408 return 0;
409 }
1da177e4
LT		 410 *timeo_p = MAX_SCHEDULE_TIMEOUT;
411 if (tv.tv_sec == 0 && tv.tv_usec == 0)
412 return 0;
413 if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT/HZ - 1))
8ccde4c5 414 *timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP(tv.tv_usec, USEC_PER_SEC / HZ);
1da177e4
LT		 415 return 0;
416}
417
418static void sock_warn_obsolete_bsdism(const char *name)
419{
420 static int warned;
421 static char warncomm[TASK_COMM_LEN];
4ec93edb
YH		 422 if (strcmp(warncomm, current->comm) && warned < 5) {
423 strcpy(warncomm, current->comm);
e005d193
JP		 424 pr_warn("process `%s' is using obsolete %s SO_BSDCOMPAT\n",
425 warncomm, name);
1da177e4
LT		 426 warned++;
427 }
428}
429
080a270f
HFS		 430static bool sock_needs_netstamp(const struct sock *sk)
431{
432 switch (sk->sk_family) {
433 case AF_UNSPEC:
434 case AF_UNIX:
435 return false;
436 default:
437 return true;
438 }
439}
440
08e29af3 441static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
4ec93edb 442{
08e29af3
ED		 443 if (sk->sk_flags & flags) {
444 sk->sk_flags &= ~flags;
080a270f
HFS		 445 if (sock_needs_netstamp(sk) &&
446 !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
20d49473 447 net_disable_timestamp();
1da177e4
LT		 448 }
449}
450
451
e6afc8ac 452int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
f0088a50 453{
3b885787
NH		 454 unsigned long flags;
455 struct sk_buff_head *list = &sk->sk_receive_queue;
f0088a50 456
0fd7bac6 457 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
766e9037 458 atomic_inc(&sk->sk_drops);
3847ce32 459 trace_sock_rcvqueue_full(sk, skb);
766e9037 460 return -ENOMEM;
f0088a50
DV		 461 }
462
c76562b6 463 if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
766e9037
ED		 464 atomic_inc(&sk->sk_drops);
465 return -ENOBUFS;
3ab224be
HA		 466 }
467
f0088a50
DV		 468 skb->dev = NULL;
469 skb_set_owner_r(skb, sk);
49ad9599 470
7fee226a
ED		 471 /* we escape from rcu protected region, make sure we dont leak
472 * a norefcounted dst
473 */
474 skb_dst_force(skb);
475
3b885787 476 spin_lock_irqsave(&list->lock, flags);
3bc3b96f 477 sock_skb_set_dropcount(sk, skb);
3b885787
NH		 478 __skb_queue_tail(list, skb);
479 spin_unlock_irqrestore(&list->lock, flags);
f0088a50
DV		 480
481 if (!sock_flag(sk, SOCK_DEAD))
676d2369 482 sk->sk_data_ready(sk);
766e9037 483 return 0;
f0088a50 484}
e6afc8ac 485EXPORT_SYMBOL(__sock_queue_rcv_skb);
486
487int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
488{
489 int err;
490
491 err = sk_filter(sk, skb);
492 if (err)
493 return err;
494
495 return __sock_queue_rcv_skb(sk, skb);
496}
f0088a50
DV		 497EXPORT_SYMBOL(sock_queue_rcv_skb);
498
4f0c40d9 499int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
c3f24cfb 500 const int nested, unsigned int trim_cap, bool refcounted)
f0088a50
DV		 501{
502 int rc = NET_RX_SUCCESS;
503
4f0c40d9 504 if (sk_filter_trim_cap(sk, skb, trim_cap))
f0088a50
DV		 505 goto discard_and_relse;
506
507 skb->dev = NULL;
508
274f482d 509 if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
c377411f
ED		 510 atomic_inc(&sk->sk_drops);
511 goto discard_and_relse;
512 }
58a5a7b9
ACM		 513 if (nested)
514 bh_lock_sock_nested(sk);
515 else
516 bh_lock_sock(sk);
a5b5bb9a
IM		 517 if (!sock_owned_by_user(sk)) {
518 /*
519 * trylock + unlock semantics:
520 */
521 mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
522
c57943a1 523 rc = sk_backlog_rcv(sk, skb);
a5b5bb9a
IM		 524
525 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
f545a38f 526 } else if (sk_add_backlog(sk, skb, sk->sk_rcvbuf)) {
8eae939f
ZY		 527 bh_unlock_sock(sk);
528 atomic_inc(&sk->sk_drops);
529 goto discard_and_relse;
530 }
531
f0088a50
DV		 532 bh_unlock_sock(sk);
533out:
c3f24cfb
ED		 534 if (refcounted)
535 sock_put(sk);
f0088a50
DV		 536 return rc;
537discard_and_relse:
538 kfree_skb(skb);
539 goto out;
540}
4f0c40d9 541EXPORT_SYMBOL(__sk_receive_skb);
f0088a50
DV		 542
543struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
544{
b6c6712a 545 struct dst_entry *dst = __sk_dst_get(sk);
f0088a50
DV		 546
547 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
e022f0b4 548 sk_tx_queue_clear(sk);
9b8805a3 549 sk->sk_dst_pending_confirm = 0;
a9b3cd7f 550 RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
f0088a50
DV		 551 dst_release(dst);
552 return NULL;
553 }
554
555 return dst;
556}
557EXPORT_SYMBOL(__sk_dst_check);
558
559struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
560{
561 struct dst_entry *dst = sk_dst_get(sk);
562
563 if (dst && dst->obsolete && dst->ops->check(dst, cookie) == NULL) {
564 sk_dst_reset(sk);
565 dst_release(dst);
566 return NULL;
567 }
568
569 return dst;
570}
571EXPORT_SYMBOL(sk_dst_check);
572
c91f6df2
BH		 573static int sock_setbindtodevice(struct sock *sk, char __user *optval,
574 int optlen)
4878809f
DM		 575{
576 int ret = -ENOPROTOOPT;
577#ifdef CONFIG_NETDEVICES
3b1e0a65 578 struct net *net = sock_net(sk);
4878809f
DM		 579 char devname[IFNAMSIZ];
580 int index;
581
582 /* Sorry... */
583 ret = -EPERM;
5e1fccc0 584 if (!ns_capable(net->user_ns, CAP_NET_RAW))
4878809f
DM		 585 goto out;
586
587 ret = -EINVAL;
588 if (optlen < 0)
589 goto out;
590
591 /* Bind this socket to a particular device like "eth0",
592 * as specified in the passed interface name. If the
593 * name is "" or the option length is zero the socket
594 * is not bound.
595 */
596 if (optlen > IFNAMSIZ - 1)
597 optlen = IFNAMSIZ - 1;
598 memset(devname, 0, sizeof(devname));
599
600 ret = -EFAULT;
601 if (copy_from_user(devname, optval, optlen))
602 goto out;
603
000ba2e4
DM		 604 index = 0;
605 if (devname[0] != '\0') {
bf8e56bf 606 struct net_device *dev;
4878809f 607
bf8e56bf
ED		 608 rcu_read_lock();
609 dev = dev_get_by_name_rcu(net, devname);
610 if (dev)
611 index = dev->ifindex;
612 rcu_read_unlock();
4878809f
DM		 613 ret = -ENODEV;
614 if (!dev)
615 goto out;
4878809f
DM		 616 }
617
618 lock_sock(sk);
619 sk->sk_bound_dev_if = index;
620 sk_dst_reset(sk);
621 release_sock(sk);
622
623 ret = 0;
624
625out:
626#endif
627
628 return ret;
629}
630
c91f6df2
BH		 631static int sock_getbindtodevice(struct sock *sk, char __user *optval,
632 int __user *optlen, int len)
633{
634 int ret = -ENOPROTOOPT;
635#ifdef CONFIG_NETDEVICES
636 struct net *net = sock_net(sk);
c91f6df2 637 char devname[IFNAMSIZ];
c91f6df2
BH		 638
639 if (sk->sk_bound_dev_if == 0) {
640 len = 0;
641 goto zero;
642 }
643
644 ret = -EINVAL;
645 if (len < IFNAMSIZ)
646 goto out;
647
5dbe7c17
NS		 648 ret = netdev_get_name(net, devname, sk->sk_bound_dev_if);
649 if (ret)
c91f6df2 650 goto out;
c91f6df2
BH		 651
652 len = strlen(devname) + 1;
653
654 ret = -EFAULT;
655 if (copy_to_user(optval, devname, len))
656 goto out;
657
658zero:
659 ret = -EFAULT;
660 if (put_user(len, optlen))
661 goto out;
662
663 ret = 0;
664
665out:
666#endif
667
668 return ret;
669}
670
c0ef877b
PE		 671static inline void sock_valbool_flag(struct sock *sk, int bit, int valbool)
672{
673 if (valbool)
674 sock_set_flag(sk, bit);
675 else
676 sock_reset_flag(sk, bit);
677}
678
f60e5990 679bool sk_mc_loop(struct sock *sk)
680{
681 if (dev_recursion_level())
682 return false;
683 if (!sk)
684 return true;
685 switch (sk->sk_family) {
686 case AF_INET:
687 return inet_sk(sk)->mc_loop;
688#if IS_ENABLED(CONFIG_IPV6)
689 case AF_INET6:
690 return inet6_sk(sk)->mc_loop;
691#endif
692 }
693 WARN_ON(1);
694 return true;
695}
696EXPORT_SYMBOL(sk_mc_loop);
697
1da177e4
LT		 698/*
699 * This is meant for all protocols to use and covers goings on
700 * at the socket level. Everything here is generic.
701 */
702
703int sock_setsockopt(struct socket *sock, int level, int optname,
b7058842 704 char __user *optval, unsigned int optlen)
1da177e4 705{
2a91525c 706 struct sock *sk = sock->sk;
1da177e4
LT		 707 int val;
708 int valbool;
709 struct linger ling;
710 int ret = 0;
4ec93edb 711
1da177e4
LT		 712 /*
713 * Options without arguments
714 */
715
4878809f 716 if (optname == SO_BINDTODEVICE)
c91f6df2 717 return sock_setbindtodevice(sk, optval, optlen);
4878809f 718
e71a4783
SH		 719 if (optlen < sizeof(int))
720 return -EINVAL;
4ec93edb 721
1da177e4
LT		 722 if (get_user(val, (int __user *)optval))
723 return -EFAULT;
4ec93edb 724
2a91525c 725 valbool = val ? 1 : 0;
1da177e4
LT		 726
727 lock_sock(sk);
728
2a91525c 729 switch (optname) {
e71a4783 730 case SO_DEBUG:
2a91525c 731 if (val && !capable(CAP_NET_ADMIN))
e71a4783 732 ret = -EACCES;
2a91525c 733 else
c0ef877b 734 sock_valbool_flag(sk, SOCK_DBG, valbool);
e71a4783
SH		 735 break;
736 case SO_REUSEADDR:
4a17fd52 737 sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
e71a4783 738 break;
055dc21a
TH		 739 case SO_REUSEPORT:
740 sk->sk_reuseport = valbool;
741 break;
e71a4783 742 case SO_TYPE:
49c794e9 743 case SO_PROTOCOL:
0d6038ee 744 case SO_DOMAIN:
e71a4783
SH		 745 case SO_ERROR:
746 ret = -ENOPROTOOPT;
747 break;
748 case SO_DONTROUTE:
c0ef877b 749 sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
e71a4783
SH		 750 break;
751 case SO_BROADCAST:
752 sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
753 break;
754 case SO_SNDBUF:
755 /* Don't error on this BSD doesn't and if you think
82981930
ED		 756 * about it this is right. Otherwise apps have to
757 * play 'guess the biggest size' games. RCVBUF/SNDBUF
758 * are treated in BSD as hints
759 */
760 val = min_t(u32, val, sysctl_wmem_max);
b0573dea 761set_sndbuf:
e71a4783 762 sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
b98b0bc8 763 sk->sk_sndbuf = max_t(int, val * 2, SOCK_MIN_SNDBUF);
82981930 764 /* Wake up sending tasks if we upped the value. */
e71a4783
SH		 765 sk->sk_write_space(sk);
766 break;
1da177e4 767
e71a4783
SH		 768 case SO_SNDBUFFORCE:
769 if (!capable(CAP_NET_ADMIN)) {
770 ret = -EPERM;
771 break;
772 }
773 goto set_sndbuf;
b0573dea 774
e71a4783
SH		 775 case SO_RCVBUF:
776 /* Don't error on this BSD doesn't and if you think
82981930
ED		 777 * about it this is right. Otherwise apps have to
778 * play 'guess the biggest size' games. RCVBUF/SNDBUF
779 * are treated in BSD as hints
780 */
781 val = min_t(u32, val, sysctl_rmem_max);
b0573dea 782set_rcvbuf:
e71a4783
SH		 783 sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
784 /*
785 * We double it on the way in to account for
786 * "struct sk_buff" etc. overhead. Applications
787 * assume that the SO_RCVBUF setting they make will
788 * allow that much actual data to be received on that
789 * socket.
790 *
791 * Applications are unaware that "struct sk_buff" and
792 * other overheads allocate from the receive buffer
793 * during socket buffer allocation.
794 *
795 * And after considering the possible alternatives,
796 * returning the value we actually used in getsockopt
797 * is the most desirable behavior.
798 */
b98b0bc8 799 sk->sk_rcvbuf = max_t(int, val * 2, SOCK_MIN_RCVBUF);
e71a4783
SH		 800 break;
801
802 case SO_RCVBUFFORCE:
803 if (!capable(CAP_NET_ADMIN)) {
804 ret = -EPERM;
1da177e4 805 break;
e71a4783
SH		 806 }
807 goto set_rcvbuf;
1da177e4 808
e71a4783 809 case SO_KEEPALIVE:
4b9d07a4
UB		 810 if (sk->sk_prot->keepalive)
811 sk->sk_prot->keepalive(sk, valbool);
e71a4783
SH		 812 sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
813 break;
814
815 case SO_OOBINLINE:
816 sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
817 break;
818
819 case SO_NO_CHECK:
28448b80 820 sk->sk_no_check_tx = valbool;
e71a4783
SH		 821 break;
822
823 case SO_PRIORITY:
5e1fccc0
EB		 824 if ((val >= 0 && val <= 6) ||
825 ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
e71a4783
SH		 826 sk->sk_priority = val;
827 else
828 ret = -EPERM;
829 break;
830
831 case SO_LINGER:
832 if (optlen < sizeof(ling)) {
833 ret = -EINVAL; /* 1003.1g */
1da177e4 834 break;
e71a4783 835 }
2a91525c 836 if (copy_from_user(&ling, optval, sizeof(ling))) {
e71a4783 837 ret = -EFAULT;
1da177e4 838 break;
e71a4783
SH		 839 }
840 if (!ling.l_onoff)
841 sock_reset_flag(sk, SOCK_LINGER);
842 else {
1da177e4 843#if (BITS_PER_LONG == 32)
e71a4783
SH		 844 if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
845 sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
1da177e4 846 else
e71a4783
SH		 847#endif
848 sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
849 sock_set_flag(sk, SOCK_LINGER);
850 }
851 break;
852
853 case SO_BSDCOMPAT:
854 sock_warn_obsolete_bsdism("setsockopt");
855 break;
856
857 case SO_PASSCRED:
858 if (valbool)
859 set_bit(SOCK_PASSCRED, &sock->flags);
860 else
861 clear_bit(SOCK_PASSCRED, &sock->flags);
862 break;
863
864 case SO_TIMESTAMP:
92f37fd2 865 case SO_TIMESTAMPNS:
e71a4783 866 if (valbool) {
92f37fd2
ED		 867 if (optname == SO_TIMESTAMP)
868 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
869 else
870 sock_set_flag(sk, SOCK_RCVTSTAMPNS);
e71a4783 871 sock_set_flag(sk, SOCK_RCVTSTAMP);
20d49473 872 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
92f37fd2 873 } else {
e71a4783 874 sock_reset_flag(sk, SOCK_RCVTSTAMP);
92f37fd2
ED		 875 sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
876 }
e71a4783
SH		 877 break;
878
20d49473
PO		 879 case SO_TIMESTAMPING:
880 if (val & ~SOF_TIMESTAMPING_MASK) {
f249fb78 881 ret = -EINVAL;
20d49473
PO		 882 break;
883 }
b245be1f 884
09c2d251 885 if (val & SOF_TIMESTAMPING_OPT_ID &&
4ed2d765 886 !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
ac5cc977
WC		 887 if (sk->sk_protocol == IPPROTO_TCP &&
888 sk->sk_type == SOCK_STREAM) {
6db8b963
SHY		 889 if ((1 << sk->sk_state) &
890 (TCPF_CLOSE | TCPF_LISTEN)) {
4ed2d765
WB		 891 ret = -EINVAL;
892 break;
893 }
894 sk->sk_tskey = tcp_sk(sk)->snd_una;
895 } else {
896 sk->sk_tskey = 0;
897 }
898 }
1c885808
FY		 899
900 if (val & SOF_TIMESTAMPING_OPT_STATS &&
901 !(val & SOF_TIMESTAMPING_OPT_TSONLY)) {
902 ret = -EINVAL;
903 break;
904 }
905
b9f40e21 906 sk->sk_tsflags = val;
20d49473
PO		 907 if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
908 sock_enable_timestamp(sk,
909 SOCK_TIMESTAMPING_RX_SOFTWARE);
910 else
911 sock_disable_timestamp(sk,
08e29af3 912 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
20d49473
PO		 913 break;
914
e71a4783
SH		 915 case SO_RCVLOWAT:
916 if (val < 0)
917 val = INT_MAX;
918 sk->sk_rcvlowat = val ? : 1;
919 break;
920
921 case SO_RCVTIMEO:
922 ret = sock_set_timeout(&sk->sk_rcvtimeo, optval, optlen);
923 break;
924
925 case SO_SNDTIMEO:
926 ret = sock_set_timeout(&sk->sk_sndtimeo, optval, optlen);
927 break;
1da177e4 928
e71a4783
SH		 929 case SO_ATTACH_FILTER:
930 ret = -EINVAL;
931 if (optlen == sizeof(struct sock_fprog)) {
932 struct sock_fprog fprog;
1da177e4 933
e71a4783
SH		 934 ret = -EFAULT;
935 if (copy_from_user(&fprog, optval, sizeof(fprog)))
1da177e4 936 break;
e71a4783
SH		 937
938 ret = sk_attach_filter(&fprog, sk);
939 }
940 break;
941
89aa0758
AS		 942 case SO_ATTACH_BPF:
943 ret = -EINVAL;
944 if (optlen == sizeof(u32)) {
945 u32 ufd;
946
947 ret = -EFAULT;
948 if (copy_from_user(&ufd, optval, sizeof(ufd)))
949 break;
950
951 ret = sk_attach_bpf(ufd, sk);
952 }
953 break;
954
538950a1
CG		 955 case SO_ATTACH_REUSEPORT_CBPF:
956 ret = -EINVAL;
957 if (optlen == sizeof(struct sock_fprog)) {
958 struct sock_fprog fprog;
959
960 ret = -EFAULT;
961 if (copy_from_user(&fprog, optval, sizeof(fprog)))
962 break;
963
964 ret = sk_reuseport_attach_filter(&fprog, sk);
965 }
966 break;
967
968 case SO_ATTACH_REUSEPORT_EBPF:
969 ret = -EINVAL;
970 if (optlen == sizeof(u32)) {
971 u32 ufd;
972
973 ret = -EFAULT;
974 if (copy_from_user(&ufd, optval, sizeof(ufd)))
975 break;
976
977 ret = sk_reuseport_attach_bpf(ufd, sk);
978 }
979 break;
980
e71a4783 981 case SO_DETACH_FILTER:
55b33325 982 ret = sk_detach_filter(sk);
e71a4783 983 break;
1da177e4 984
d59577b6
VB		 985 case SO_LOCK_FILTER:
986 if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
987 ret = -EPERM;
988 else
989 sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
990 break;
991
e71a4783
SH		 992 case SO_PASSSEC:
993 if (valbool)
994 set_bit(SOCK_PASSSEC, &sock->flags);
995 else
996 clear_bit(SOCK_PASSSEC, &sock->flags);
997 break;
4a19ec58 998 case SO_MARK:
5e1fccc0 999 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
4a19ec58 1000 ret = -EPERM;
2a91525c 1001 else
4a19ec58 1002 sk->sk_mark = val;
4a19ec58 1003 break;
877ce7c1 1004
3b885787 1005 case SO_RXQ_OVFL:
8083f0fc 1006 sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
3b885787 1007 break;
6e3e939f
JB		 1008
1009 case SO_WIFI_STATUS:
1010 sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1011 break;
1012
ef64a54f
PE		 1013 case SO_PEEK_OFF:
1014 if (sock->ops->set_peek_off)
12663bfc 1015 ret = sock->ops->set_peek_off(sk, val);
ef64a54f
PE		 1016 else
1017 ret = -EOPNOTSUPP;
1018 break;
3bdc0eba
BG		 1019
1020 case SO_NOFCS:
1021 sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1022 break;
1023
7d4c04fc
KJ		 1024 case SO_SELECT_ERR_QUEUE:
1025 sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1026 break;
1027
e0d1095a 1028#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1029 case SO_BUSY_POLL:
dafcc438
ET		 1030 /* allow unprivileged users to decrease the value */
1031 if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
1032 ret = -EPERM;
1033 else {
1034 if (val < 0)
1035 ret = -EINVAL;
1036 else
1037 sk->sk_ll_usec = val;
1038 }
1039 break;
1040#endif
62748f32
ED		 1041
1042 case SO_MAX_PACING_RATE:
1043 sk->sk_max_pacing_rate = val;
1044 sk->sk_pacing_rate = min(sk->sk_pacing_rate,
1045 sk->sk_max_pacing_rate);
1046 break;
1047
70da268b
ED		 1048 case SO_INCOMING_CPU:
1049 sk->sk_incoming_cpu = val;
1050 break;
1051
a87cb3e4
TH		 1052 case SO_CNX_ADVICE:
1053 if (val == 1)
1054 dst_negative_advice(sk);
1055 break;
e71a4783
SH		 1056 default:
1057 ret = -ENOPROTOOPT;
1058 break;
4ec93edb 1059 }
1da177e4
LT		 1060 release_sock(sk);
1061 return ret;
1062}
2a91525c 1063EXPORT_SYMBOL(sock_setsockopt);
1da177e4
LT		 1064
1065
8f09898b 1066static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1067 struct ucred *ucred)
3f551f94
EB		 1068{
1069 ucred->pid = pid_vnr(pid);
1070 ucred->uid = ucred->gid = -1;
1071 if (cred) {
1072 struct user_namespace *current_ns = current_user_ns();
1073
b2e4f544
EB		 1074 ucred->uid = from_kuid_munged(current_ns, cred->euid);
1075 ucred->gid = from_kgid_munged(current_ns, cred->egid);
3f551f94
EB		 1076 }
1077}
1078
1da177e4
LT		 1079int sock_getsockopt(struct socket *sock, int level, int optname,
1080 char __user *optval, int __user *optlen)
1081{
1082 struct sock *sk = sock->sk;
4ec93edb 1083
e71a4783 1084 union {
4ec93edb
YH		 1085 int val;
1086 struct linger ling;
1da177e4
LT		 1087 struct timeval tm;
1088 } v;
4ec93edb 1089
4d0392be 1090 int lv = sizeof(int);
1da177e4 1091 int len;
4ec93edb 1092
e71a4783 1093 if (get_user(len, optlen))
4ec93edb 1094 return -EFAULT;
e71a4783 1095 if (len < 0)
1da177e4 1096 return -EINVAL;
4ec93edb 1097
50fee1de 1098 memset(&v, 0, sizeof(v));
df0bca04 1099
2a91525c 1100 switch (optname) {
e71a4783
SH		 1101 case SO_DEBUG:
1102 v.val = sock_flag(sk, SOCK_DBG);
1103 break;
1104
1105 case SO_DONTROUTE:
1106 v.val = sock_flag(sk, SOCK_LOCALROUTE);
1107 break;
1108
1109 case SO_BROADCAST:
1b23a5df 1110 v.val = sock_flag(sk, SOCK_BROADCAST);
e71a4783
SH		 1111 break;
1112
1113 case SO_SNDBUF:
1114 v.val = sk->sk_sndbuf;
1115 break;
1116
1117 case SO_RCVBUF:
1118 v.val = sk->sk_rcvbuf;
1119 break;
1120
1121 case SO_REUSEADDR:
1122 v.val = sk->sk_reuse;
1123 break;
1124
055dc21a
TH		 1125 case SO_REUSEPORT:
1126 v.val = sk->sk_reuseport;
1127 break;
1128
e71a4783 1129 case SO_KEEPALIVE:
1b23a5df 1130 v.val = sock_flag(sk, SOCK_KEEPOPEN);
e71a4783
SH		 1131 break;
1132
1133 case SO_TYPE:
1134 v.val = sk->sk_type;
1135 break;
1136
49c794e9
JE		 1137 case SO_PROTOCOL:
1138 v.val = sk->sk_protocol;
1139 break;
1140
0d6038ee
JE		 1141 case SO_DOMAIN:
1142 v.val = sk->sk_family;
1143 break;
1144
e71a4783
SH		 1145 case SO_ERROR:
1146 v.val = -sock_error(sk);
2a91525c 1147 if (v.val == 0)
e71a4783
SH		 1148 v.val = xchg(&sk->sk_err_soft, 0);
1149 break;
1150
1151 case SO_OOBINLINE:
1b23a5df 1152 v.val = sock_flag(sk, SOCK_URGINLINE);
e71a4783
SH		 1153 break;
1154
1155 case SO_NO_CHECK:
28448b80 1156 v.val = sk->sk_no_check_tx;
e71a4783
SH		 1157 break;
1158
1159 case SO_PRIORITY:
1160 v.val = sk->sk_priority;
1161 break;
1162
1163 case SO_LINGER:
1164 lv = sizeof(v.ling);
1b23a5df 1165 v.ling.l_onoff = sock_flag(sk, SOCK_LINGER);
e71a4783
SH		 1166 v.ling.l_linger = sk->sk_lingertime / HZ;
1167 break;
1168
1169 case SO_BSDCOMPAT:
1170 sock_warn_obsolete_bsdism("getsockopt");
1171 break;
1172
1173 case SO_TIMESTAMP:
92f37fd2
ED		 1174 v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1175 !sock_flag(sk, SOCK_RCVTSTAMPNS);
1176 break;
1177
1178 case SO_TIMESTAMPNS:
1179 v.val = sock_flag(sk, SOCK_RCVTSTAMPNS);
e71a4783
SH		 1180 break;
1181
20d49473 1182 case SO_TIMESTAMPING:
b9f40e21 1183 v.val = sk->sk_tsflags;
20d49473
PO		 1184 break;
1185
e71a4783 1186 case SO_RCVTIMEO:
2a91525c 1187 lv = sizeof(struct timeval);
e71a4783
SH		 1188 if (sk->sk_rcvtimeo == MAX_SCHEDULE_TIMEOUT) {
1189 v.tm.tv_sec = 0;
1190 v.tm.tv_usec = 0;
1191 } else {
1192 v.tm.tv_sec = sk->sk_rcvtimeo / HZ;
8ccde4c5 1193 v.tm.tv_usec = ((sk->sk_rcvtimeo % HZ) * USEC_PER_SEC) / HZ;
e71a4783
SH		 1194 }
1195 break;
1196
1197 case SO_SNDTIMEO:
2a91525c 1198 lv = sizeof(struct timeval);
e71a4783
SH		 1199 if (sk->sk_sndtimeo == MAX_SCHEDULE_TIMEOUT) {
1200 v.tm.tv_sec = 0;
1201 v.tm.tv_usec = 0;
1202 } else {
1203 v.tm.tv_sec = sk->sk_sndtimeo / HZ;
8ccde4c5 1204 v.tm.tv_usec = ((sk->sk_sndtimeo % HZ) * USEC_PER_SEC) / HZ;
e71a4783
SH		 1205 }
1206 break;
1da177e4 1207
e71a4783
SH		 1208 case SO_RCVLOWAT:
1209 v.val = sk->sk_rcvlowat;
1210 break;
1da177e4 1211
e71a4783 1212 case SO_SNDLOWAT:
2a91525c 1213 v.val = 1;
e71a4783 1214 break;
1da177e4 1215
e71a4783 1216 case SO_PASSCRED:
82981930 1217 v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
e71a4783 1218 break;
1da177e4 1219
e71a4783 1220 case SO_PEERCRED:
109f6e39
EB		 1221 {
1222 struct ucred peercred;
1223 if (len > sizeof(peercred))
1224 len = sizeof(peercred);
1225 cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1226 if (copy_to_user(optval, &peercred, len))
e71a4783
SH		 1227 return -EFAULT;
1228 goto lenout;
109f6e39 1229 }
1da177e4 1230
e71a4783
SH		 1231 case SO_PEERNAME:
1232 {
1233 char address[128];
1234
1235 if (sock->ops->getname(sock, (struct sockaddr *)address, &lv, 2))
1236 return -ENOTCONN;
1237 if (lv < len)
1238 return -EINVAL;
1239 if (copy_to_user(optval, address, len))
1240 return -EFAULT;
1241 goto lenout;
1242 }
1da177e4 1243
e71a4783
SH		 1244 /* Dubious BSD thing... Probably nobody even uses it, but
1245 * the UNIX standard wants it for whatever reason... -DaveM
1246 */
1247 case SO_ACCEPTCONN:
1248 v.val = sk->sk_state == TCP_LISTEN;
1249 break;
1da177e4 1250
e71a4783 1251 case SO_PASSSEC:
82981930 1252 v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
e71a4783 1253 break;
877ce7c1 1254
e71a4783
SH		 1255 case SO_PEERSEC:
1256 return security_socket_getpeersec_stream(sock, optval, optlen, len);
1da177e4 1257
4a19ec58
LAT		 1258 case SO_MARK:
1259 v.val = sk->sk_mark;
1260 break;
1261
3b885787 1262 case SO_RXQ_OVFL:
1b23a5df 1263 v.val = sock_flag(sk, SOCK_RXQ_OVFL);
3b885787
NH		 1264 break;
1265
6e3e939f 1266 case SO_WIFI_STATUS:
1b23a5df 1267 v.val = sock_flag(sk, SOCK_WIFI_STATUS);
6e3e939f
JB		 1268 break;
1269
ef64a54f
PE		 1270 case SO_PEEK_OFF:
1271 if (!sock->ops->set_peek_off)
1272 return -EOPNOTSUPP;
1273
1274 v.val = sk->sk_peek_off;
1275 break;
bc2f7996 1276 case SO_NOFCS:
1b23a5df 1277 v.val = sock_flag(sk, SOCK_NOFCS);
bc2f7996 1278 break;
c91f6df2 1279
f7b86bfe 1280 case SO_BINDTODEVICE:
c91f6df2
BH		 1281 return sock_getbindtodevice(sk, optval, optlen, len);
1282
a8fc9277
PE		 1283 case SO_GET_FILTER:
1284 len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
1285 if (len < 0)
1286 return len;
1287
1288 goto lenout;
c91f6df2 1289
d59577b6
VB		 1290 case SO_LOCK_FILTER:
1291 v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1292 break;
1293
ea02f941
MS		 1294 case SO_BPF_EXTENSIONS:
1295 v.val = bpf_tell_extensions();
1296 break;
1297
7d4c04fc
KJ		 1298 case SO_SELECT_ERR_QUEUE:
1299 v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1300 break;
1301
e0d1095a 1302#ifdef CONFIG_NET_RX_BUSY_POLL
64b0dc51 1303 case SO_BUSY_POLL:
dafcc438
ET		 1304 v.val = sk->sk_ll_usec;
1305 break;
1306#endif
1307
62748f32
ED		 1308 case SO_MAX_PACING_RATE:
1309 v.val = sk->sk_max_pacing_rate;
1310 break;
1311
2c8c56e1
ED		 1312 case SO_INCOMING_CPU:
1313 v.val = sk->sk_incoming_cpu;
1314 break;
1315
e71a4783 1316 default:
443b5991
YH		 1317 /* We implement the SO_SNDLOWAT etc to not be settable
1318 * (1003.1g 7).
1319 */
e71a4783 1320 return -ENOPROTOOPT;
1da177e4 1321 }
e71a4783 1322
1da177e4
LT		 1323 if (len > lv)
1324 len = lv;
1325 if (copy_to_user(optval, &v, len))
1326 return -EFAULT;
1327lenout:
4ec93edb
YH		 1328 if (put_user(len, optlen))
1329 return -EFAULT;
1330 return 0;
1da177e4
LT		 1331}
1332
a5b5bb9a
IM		 1333/*
1334 * Initialize an sk_lock.
1335 *
1336 * (We also register the sk_lock with the lock validator.)
1337 */
b6f99a21 1338static inline void sock_lock_init(struct sock *sk)
a5b5bb9a 1339{
cdfbabfb
DH		 1340 if (sk->sk_kern_sock)
1341 sock_lock_init_class_and_name(
1342 sk,
1343 af_family_kern_slock_key_strings[sk->sk_family],
1344 af_family_kern_slock_keys + sk->sk_family,
1345 af_family_kern_key_strings[sk->sk_family],
1346 af_family_kern_keys + sk->sk_family);
1347 else
1348 sock_lock_init_class_and_name(
1349 sk,
ed07536e
PZ		 1350 af_family_slock_key_strings[sk->sk_family],
1351 af_family_slock_keys + sk->sk_family,
1352 af_family_key_strings[sk->sk_family],
1353 af_family_keys + sk->sk_family);
a5b5bb9a
IM		 1354}
1355
4dc6dc71
ED		 1356/*
1357 * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1358 * even temporarly, because of RCU lookups. sk_node should also be left as is.
68835aba 1359 * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
4dc6dc71 1360 */
f1a6c4da
PE		 1361static void sock_copy(struct sock *nsk, const struct sock *osk)
1362{
1363#ifdef CONFIG_SECURITY_NETWORK
1364 void *sptr = nsk->sk_security;
1365#endif
68835aba
ED		 1366 memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
1367
1368 memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
1369 osk->sk_prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
1370
f1a6c4da
PE		 1371#ifdef CONFIG_SECURITY_NETWORK
1372 nsk->sk_security = sptr;
1373 security_sk_clone(osk, nsk);
1374#endif
1375}
1376
2e4afe7b
PE		 1377static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
1378 int family)
c308c1b2
PE		 1379{
1380 struct sock *sk;
1381 struct kmem_cache *slab;
1382
1383 slab = prot->slab;
e912b114
ED		 1384 if (slab != NULL) {
1385 sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
1386 if (!sk)
1387 return sk;
ba2489b0
ED		 1388 if (priority & __GFP_ZERO)
1389 sk_prot_clear_nulls(sk, prot->obj_size);
fcbdf09d 1390 } else
c308c1b2
PE		 1391 sk = kmalloc(prot->obj_size, priority);
1392
2e4afe7b 1393 if (sk != NULL) {
a98b65a3
VN		 1394 kmemcheck_annotate_bitfield(sk, flags);
1395
2e4afe7b
PE		 1396 if (security_sk_alloc(sk, family, priority))
1397 goto out_free;
1398
1399 if (!try_module_get(prot->owner))
1400 goto out_free_sec;
e022f0b4 1401 sk_tx_queue_clear(sk);
2e4afe7b
PE		 1402 }
1403
c308c1b2 1404 return sk;
2e4afe7b
PE		 1405
1406out_free_sec:
1407 security_sk_free(sk);
1408out_free:
1409 if (slab != NULL)
1410 kmem_cache_free(slab, sk);
1411 else
1412 kfree(sk);
1413 return NULL;
c308c1b2
PE		 1414}
1415
1416static void sk_prot_free(struct proto *prot, struct sock *sk)
1417{
1418 struct kmem_cache *slab;
2e4afe7b 1419 struct module *owner;
c308c1b2 1420
2e4afe7b 1421 owner = prot->owner;
c308c1b2 1422 slab = prot->slab;
2e4afe7b 1423
bd1060a1 1424 cgroup_sk_free(&sk->sk_cgrp_data);
2d758073 1425 mem_cgroup_sk_free(sk);
2e4afe7b 1426 security_sk_free(sk);
c308c1b2
PE		 1427 if (slab != NULL)
1428 kmem_cache_free(slab, sk);
1429 else
1430 kfree(sk);
2e4afe7b 1431 module_put(owner);
c308c1b2
PE		 1432}
1433
1da177e4
LT		 1434/**
1435 * sk_alloc - All socket objects are allocated here
c4ea43c5 1436 * @net: the applicable net namespace
4dc3b16b
PP		 1437 * @family: protocol family
1438 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1439 * @prot: struct proto associated with this new sock instance
11aa9c28 1440 * @kern: is this to be a kernel socket?
1da177e4 1441 */
1b8d7ae4 1442struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
11aa9c28 1443 struct proto *prot, int kern)
1da177e4 1444{
c308c1b2 1445 struct sock *sk;
1da177e4 1446
154adbc8 1447 sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
1da177e4 1448 if (sk) {
154adbc8
PE		 1449 sk->sk_family = family;
1450 /*
1451 * See comment in struct sock definition to understand
1452 * why we need sk_prot_creator -acme
1453 */
1454 sk->sk_prot = sk->sk_prot_creator = prot;
cdfbabfb 1455 sk->sk_kern_sock = kern;
154adbc8 1456 sock_lock_init(sk);
26abe143
EB		 1457 sk->sk_net_refcnt = kern ? 0 : 1;
1458 if (likely(sk->sk_net_refcnt))
1459 get_net(net);
1460 sock_net_set(sk, net);
d66ee058 1461 atomic_set(&sk->sk_wmem_alloc, 1);
f8451725 1462
2d758073 1463 mem_cgroup_sk_alloc(sk);
d979a39d 1464 cgroup_sk_alloc(&sk->sk_cgrp_data);
2a56a1fe
TH		 1465 sock_update_classid(&sk->sk_cgrp_data);
1466 sock_update_netprioidx(&sk->sk_cgrp_data);
1da177e4 1467 }
a79af59e 1468
2e4afe7b 1469 return sk;
1da177e4 1470}
2a91525c 1471EXPORT_SYMBOL(sk_alloc);
1da177e4 1472
a4298e45
ED		 1473/* Sockets having SOCK_RCU_FREE will call this function after one RCU
1474 * grace period. This is the case for UDP sockets and TCP listeners.
1475 */
1476static void __sk_destruct(struct rcu_head *head)
1da177e4 1477{
a4298e45 1478 struct sock *sk = container_of(head, struct sock, sk_rcu);
1da177e4 1479 struct sk_filter *filter;
1da177e4
LT		 1480
1481 if (sk->sk_destruct)
1482 sk->sk_destruct(sk);
1483
a898def2
PM		 1484 filter = rcu_dereference_check(sk->sk_filter,
1485 atomic_read(&sk->sk_wmem_alloc) == 0);
1da177e4 1486 if (filter) {
309dd5fc 1487 sk_filter_uncharge(sk, filter);
a9b3cd7f 1488 RCU_INIT_POINTER(sk->sk_filter, NULL);
1da177e4 1489 }
538950a1
CG		 1490 if (rcu_access_pointer(sk->sk_reuseport_cb))
1491 reuseport_detach_sock(sk);
1da177e4 1492
08e29af3 1493 sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
1da177e4
LT		 1494
1495 if (atomic_read(&sk->sk_omem_alloc))
e005d193
JP		 1496 pr_debug("%s: optmem leakage (%d bytes) detected\n",
1497 __func__, atomic_read(&sk->sk_omem_alloc));
1da177e4 1498
109f6e39
EB		 1499 if (sk->sk_peer_cred)
1500 put_cred(sk->sk_peer_cred);
1501 put_pid(sk->sk_peer_pid);
26abe143
EB		 1502 if (likely(sk->sk_net_refcnt))
1503 put_net(sock_net(sk));
c308c1b2 1504 sk_prot_free(sk->sk_prot_creator, sk);
1da177e4 1505}
2b85a34e 1506
a4298e45
ED		 1507void sk_destruct(struct sock *sk)
1508{
1509 if (sock_flag(sk, SOCK_RCU_FREE))
1510 call_rcu(&sk->sk_rcu, __sk_destruct);
1511 else
1512 __sk_destruct(&sk->sk_rcu);
1513}
1514
eb4cb008
CG		 1515static void __sk_free(struct sock *sk)
1516{
b922622e 1517 if (unlikely(sock_diag_has_destroy_listeners(sk) && sk->sk_net_refcnt))
eb4cb008
CG		 1518 sock_diag_broadcast_destroy(sk);
1519 else
1520 sk_destruct(sk);
1521}
1522
2b85a34e
ED		 1523void sk_free(struct sock *sk)
1524{
1525 /*
25985edc 1526 * We subtract one from sk_wmem_alloc and can know if
2b85a34e
ED		 1527 * some packets are still in some tx queue.
1528 * If not null, sock_wfree() will call __sk_free(sk) later
1529 */
1530 if (atomic_dec_and_test(&sk->sk_wmem_alloc))
1531 __sk_free(sk);
1532}
2a91525c 1533EXPORT_SYMBOL(sk_free);
1da177e4 1534
581319c5
PA		 1535static void sk_init_common(struct sock *sk)
1536{
1537 skb_queue_head_init(&sk->sk_receive_queue);
1538 skb_queue_head_init(&sk->sk_write_queue);
1539 skb_queue_head_init(&sk->sk_error_queue);
1540
1541 rwlock_init(&sk->sk_callback_lock);
1542 lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
1543 af_rlock_keys + sk->sk_family,
1544 af_family_rlock_key_strings[sk->sk_family]);
1545 lockdep_set_class_and_name(&sk->sk_write_queue.lock,
1546 af_wlock_keys + sk->sk_family,
1547 af_family_wlock_key_strings[sk->sk_family]);
1548 lockdep_set_class_and_name(&sk->sk_error_queue.lock,
1549 af_elock_keys + sk->sk_family,
1550 af_family_elock_key_strings[sk->sk_family]);
1551 lockdep_set_class_and_name(&sk->sk_callback_lock,
1552 af_callback_keys + sk->sk_family,
1553 af_family_clock_key_strings[sk->sk_family]);
1554}
1555
e56c57d0
ED		 1556/**
1557 * sk_clone_lock - clone a socket, and lock its clone
1558 * @sk: the socket to clone
1559 * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
1560 *
1561 * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
1562 */
1563struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
87d11ceb 1564{
8fd1d178 1565 struct sock *newsk;
278571ba 1566 bool is_charged = true;
87d11ceb 1567
8fd1d178 1568 newsk = sk_prot_alloc(sk->sk_prot, priority, sk->sk_family);
87d11ceb
ACM		 1569 if (newsk != NULL) {
1570 struct sk_filter *filter;
1571
892c141e 1572 sock_copy(newsk, sk);
87d11ceb
ACM		 1573
1574 /* SANITY */
8a681736
SV		 1575 if (likely(newsk->sk_net_refcnt))
1576 get_net(sock_net(newsk));
87d11ceb
ACM		 1577 sk_node_init(&newsk->sk_node);
1578 sock_lock_init(newsk);
1579 bh_lock_sock(newsk);
fa438ccf 1580 newsk->sk_backlog.head = newsk->sk_backlog.tail = NULL;
8eae939f 1581 newsk->sk_backlog.len = 0;
87d11ceb
ACM		 1582
1583 atomic_set(&newsk->sk_rmem_alloc, 0);
2b85a34e
ED		 1584 /*
1585 * sk_wmem_alloc set to one (see sk_free() and sock_wfree())
1586 */
1587 atomic_set(&newsk->sk_wmem_alloc, 1);
87d11ceb 1588 atomic_set(&newsk->sk_omem_alloc, 0);
581319c5 1589 sk_init_common(newsk);
87d11ceb
ACM		 1590
1591 newsk->sk_dst_cache = NULL;
9b8805a3 1592 newsk->sk_dst_pending_confirm = 0;
87d11ceb
ACM		 1593 newsk->sk_wmem_queued = 0;
1594 newsk->sk_forward_alloc = 0;
9caad864 1595 atomic_set(&newsk->sk_drops, 0);
87d11ceb 1596 newsk->sk_send_head = NULL;
87d11ceb
ACM		 1597 newsk->sk_userlocks = sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
1598
1599 sock_reset_flag(newsk, SOCK_DONE);
87d11ceb 1600
0d7da9dd 1601 filter = rcu_dereference_protected(newsk->sk_filter, 1);
87d11ceb 1602 if (filter != NULL)
278571ba
AS		 1603 /* though it's an empty new sock, the charging may fail
1604 * if sysctl_optmem_max was changed between creation of
1605 * original socket and cloning
1606 */
1607 is_charged = sk_filter_charge(newsk, filter);
87d11ceb 1608
d188ba86 1609 if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
94352d45 1610 sk_free_unlock_clone(newsk);
87d11ceb
ACM		 1611 newsk = NULL;
1612 goto out;
1613 }
fa463497 1614 RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
87d11ceb
ACM		 1615
1616 newsk->sk_err = 0;
e551c32d 1617 newsk->sk_err_soft = 0;
87d11ceb 1618 newsk->sk_priority = 0;
2c8c56e1 1619 newsk->sk_incoming_cpu = raw_smp_processor_id();
33cf7c90 1620 atomic64_set(&newsk->sk_cookie, 0);
d979a39d 1621
2d758073 1622 mem_cgroup_sk_alloc(newsk);
d979a39d
JW		 1623 cgroup_sk_alloc(&newsk->sk_cgrp_data);
1624
4dc6dc71
ED		 1625 /*
1626 * Before updating sk_refcnt, we must commit prior changes to memory
1627 * (Documentation/RCU/rculist_nulls.txt for details)
1628 */
1629 smp_wmb();
87d11ceb
ACM		 1630 atomic_set(&newsk->sk_refcnt, 2);
1631
1632 /*
1633 * Increment the counter in the same struct proto as the master
1634 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
1635 * is the same as sk->sk_prot->socks, as this field was copied
1636 * with memcpy).
1637 *
1638 * This _changes_ the previous behaviour, where
1639 * tcp_create_openreq_child always was incrementing the
1640 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
1641 * to be taken into account in all callers. -acme
1642 */
1643 sk_refcnt_debug_inc(newsk);
972692e0 1644 sk_set_socket(newsk, NULL);
43815482 1645 newsk->sk_wq = NULL;
87d11ceb
ACM		 1646
1647 if (newsk->sk_prot->sockets_allocated)
180d8cd9 1648 sk_sockets_allocated_inc(newsk);
704da560 1649
080a270f
HFS		 1650 if (sock_needs_netstamp(sk) &&
1651 newsk->sk_flags & SK_FLAGS_TIMESTAMP)
704da560 1652 net_enable_timestamp();
87d11ceb
ACM		 1653 }
1654out:
1655 return newsk;
1656}
e56c57d0 1657EXPORT_SYMBOL_GPL(sk_clone_lock);
87d11ceb 1658
94352d45
ACM		 1659void sk_free_unlock_clone(struct sock *sk)
1660{
1661 /* It is still raw copy of parent, so invalidate
1662 * destructor and make plain sk_free() */
1663 sk->sk_destruct = NULL;
1664 bh_unlock_sock(sk);
1665 sk_free(sk);
1666}
1667EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
1668
9958089a
AK		 1669void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
1670{
d6a4e26a
ED		 1671 u32 max_segs = 1;
1672
6bd4f355 1673 sk_dst_set(sk, dst);
9958089a
AK		 1674 sk->sk_route_caps = dst->dev->features;
1675 if (sk->sk_route_caps & NETIF_F_GSO)
4fcd6b99 1676 sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
a465419b 1677 sk->sk_route_caps &= ~sk->sk_route_nocaps;
9958089a 1678 if (sk_can_gso(sk)) {
82cc1a7a 1679 if (dst->header_len) {
9958089a 1680 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
82cc1a7a 1681 } else {
9958089a 1682 sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
82cc1a7a 1683 sk->sk_gso_max_size = dst->dev->gso_max_size;
d6a4e26a 1684 max_segs = max_t(u32, dst->dev->gso_max_segs, 1);
82cc1a7a 1685 }
9958089a 1686 }
d6a4e26a 1687 sk->sk_gso_max_segs = max_segs;
9958089a
AK		 1688}
1689EXPORT_SYMBOL_GPL(sk_setup_caps);
1690
1da177e4
LT		 1691/*
1692 * Simple resource managers for sockets.
1693 */
1694
1695
4ec93edb
YH		 1696/*
1697 * Write buffer destructor automatically called from kfree_skb.
1da177e4
LT		 1698 */
1699void sock_wfree(struct sk_buff *skb)
1700{
1701 struct sock *sk = skb->sk;
d99927f4 1702 unsigned int len = skb->truesize;
1da177e4 1703
d99927f4
ED		 1704 if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
1705 /*
1706 * Keep a reference on sk_wmem_alloc, this will be released
1707 * after sk_write_space() call
1708 */
1709 atomic_sub(len - 1, &sk->sk_wmem_alloc);
1da177e4 1710 sk->sk_write_space(sk);
d99927f4
ED		 1711 len = 1;
1712 }
2b85a34e 1713 /*
d99927f4
ED		 1714 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
1715 * could not do because of in-flight packets
2b85a34e 1716 */
d99927f4 1717 if (atomic_sub_and_test(len, &sk->sk_wmem_alloc))
2b85a34e 1718 __sk_free(sk);
1da177e4 1719}
2a91525c 1720EXPORT_SYMBOL(sock_wfree);
1da177e4 1721
1d2077ac
ED		 1722/* This variant of sock_wfree() is used by TCP,
1723 * since it sets SOCK_USE_WRITE_QUEUE.
1724 */
1725void __sock_wfree(struct sk_buff *skb)
1726{
1727 struct sock *sk = skb->sk;
1728
1729 if (atomic_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
1730 __sk_free(sk);
1731}
1732
9e17f8a4
ED		 1733void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1734{
1735 skb_orphan(skb);
1736 skb->sk = sk;
1737#ifdef CONFIG_INET
1738 if (unlikely(!sk_fullsock(sk))) {
1739 skb->destructor = sock_edemux;
1740 sock_hold(sk);
1741 return;
1742 }
1743#endif
1744 skb->destructor = sock_wfree;
1745 skb_set_hash_from_sk(skb, sk);
1746 /*
1747 * We used to take a refcount on sk, but following operation
1748 * is enough to guarantee sk_free() wont free this sock until
1749 * all in-flight packets are completed
1750 */
1751 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
1752}
1753EXPORT_SYMBOL(skb_set_owner_w);
1754
1d2077ac
ED		 1755/* This helper is used by netem, as it can hold packets in its
1756 * delay queue. We want to allow the owner socket to send more
1757 * packets, as if they were already TX completed by a typical driver.
1758 * But we also want to keep skb->sk set because some packet schedulers
1759 * rely on it (sch_fq for example). So we set skb->truesize to a small
1760 * amount (1) and decrease sk_wmem_alloc accordingly.
1761 */
f2f872f9
ED		 1762void skb_orphan_partial(struct sk_buff *skb)
1763{
1d2077ac
ED		 1764 /* If this skb is a TCP pure ACK or already went here,
1765 * we have nothing to do. 2 is already a very small truesize.
1766 */
1767 if (skb->truesize <= 2)
1768 return;
1769
f2f872f9
ED		 1770 /* TCP stack sets skb->ooo_okay based on sk_wmem_alloc,
1771 * so we do not completely orphan skb, but transfert all
1772 * accounted bytes but one, to avoid unexpected reorders.
1773 */
1774 if (skb->destructor == sock_wfree
1775#ifdef CONFIG_INET
1776 || skb->destructor == tcp_wfree
1777#endif
1778 ) {
1779 atomic_sub(skb->truesize - 1, &skb->sk->sk_wmem_alloc);
1780 skb->truesize = 1;
1781 } else {
1782 skb_orphan(skb);
1783 }
1784}
1785EXPORT_SYMBOL(skb_orphan_partial);
1786
4ec93edb
YH		 1787/*
1788 * Read buffer destructor automatically called from kfree_skb.
1da177e4
LT		 1789 */
1790void sock_rfree(struct sk_buff *skb)
1791{
1792 struct sock *sk = skb->sk;
d361fd59 1793 unsigned int len = skb->truesize;
1da177e4 1794
d361fd59
ED		 1795 atomic_sub(len, &sk->sk_rmem_alloc);
1796 sk_mem_uncharge(sk, len);
1da177e4 1797}
2a91525c 1798EXPORT_SYMBOL(sock_rfree);
1da177e4 1799
7768eed8
OH		 1800/*
1801 * Buffer destructor for skbs that are not used directly in read or write
1802 * path, e.g. for error handler skbs. Automatically called from kfree_skb.
1803 */
62bccb8c
AD		 1804void sock_efree(struct sk_buff *skb)
1805{
1806 sock_put(skb->sk);
1807}
1808EXPORT_SYMBOL(sock_efree);
1809
976d0201 1810kuid_t sock_i_uid(struct sock *sk)
1da177e4 1811{
976d0201 1812 kuid_t uid;
1da177e4 1813
f064af1e 1814 read_lock_bh(&sk->sk_callback_lock);
976d0201 1815 uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
f064af1e 1816 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 1817 return uid;
1818}
2a91525c 1819EXPORT_SYMBOL(sock_i_uid);
1da177e4
LT		 1820
1821unsigned long sock_i_ino(struct sock *sk)
1822{
1823 unsigned long ino;
1824
f064af1e 1825 read_lock_bh(&sk->sk_callback_lock);
1da177e4 1826 ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
f064af1e 1827 read_unlock_bh(&sk->sk_callback_lock);
1da177e4
LT		 1828 return ino;
1829}
2a91525c 1830EXPORT_SYMBOL(sock_i_ino);
1da177e4
LT		 1831
1832/*
1833 * Allocate a skb from the socket's send buffer.
1834 */
86a76caf 1835struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
dd0fc66f 1836 gfp_t priority)
1da177e4
LT		 1837{
1838 if (force || atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf) {
2a91525c 1839 struct sk_buff *skb = alloc_skb(size, priority);
1da177e4
LT		 1840 if (skb) {
1841 skb_set_owner_w(skb, sk);
1842 return skb;
1843 }
1844 }
1845 return NULL;
1846}
2a91525c 1847EXPORT_SYMBOL(sock_wmalloc);
1da177e4 1848
4ec93edb 1849/*
1da177e4 1850 * Allocate a memory block from the socket's option memory buffer.
4ec93edb 1851 */
dd0fc66f 1852void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
1da177e4 1853{
95c96174 1854 if ((unsigned int)size <= sysctl_optmem_max &&
1da177e4
LT		 1855 atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
1856 void *mem;
1857 /* First do the add, to avoid the race if kmalloc
4ec93edb 1858 * might sleep.
1da177e4
LT		 1859 */
1860 atomic_add(size, &sk->sk_omem_alloc);
1861 mem = kmalloc(size, priority);
1862 if (mem)
1863 return mem;
1864 atomic_sub(size, &sk->sk_omem_alloc);
1865 }
1866 return NULL;
1867}
2a91525c 1868EXPORT_SYMBOL(sock_kmalloc);
1da177e4 1869
79e88659
DB		 1870/* Free an option memory block. Note, we actually want the inline
1871 * here as this allows gcc to detect the nullify and fold away the
1872 * condition entirely.
1da177e4 1873 */
79e88659
DB		 1874static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
1875 const bool nullify)
1da177e4 1876{
e53da5fb
DM		 1877 if (WARN_ON_ONCE(!mem))
1878 return;
79e88659
DB		 1879 if (nullify)
1880 kzfree(mem);
1881 else
1882 kfree(mem);
1da177e4
LT		 1883 atomic_sub(size, &sk->sk_omem_alloc);
1884}
79e88659
DB		 1885
1886void sock_kfree_s(struct sock *sk, void *mem, int size)
1887{
1888 __sock_kfree_s(sk, mem, size, false);
1889}
2a91525c 1890EXPORT_SYMBOL(sock_kfree_s);
1da177e4 1891
79e88659
DB		 1892void sock_kzfree_s(struct sock *sk, void *mem, int size)
1893{
1894 __sock_kfree_s(sk, mem, size, true);
1895}
1896EXPORT_SYMBOL(sock_kzfree_s);
1897
1da177e4
LT		 1898/* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
1899 I think, these locks should be removed for datagram sockets.
1900 */
2a91525c 1901static long sock_wait_for_wmem(struct sock *sk, long timeo)
1da177e4
LT		 1902{
1903 DEFINE_WAIT(wait);
1904
9cd3e072 1905 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1da177e4
LT		 1906 for (;;) {
1907 if (!timeo)
1908 break;
1909 if (signal_pending(current))
1910 break;
1911 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
aa395145 1912 prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1da177e4
LT		 1913 if (atomic_read(&sk->sk_wmem_alloc) < sk->sk_sndbuf)
1914 break;
1915 if (sk->sk_shutdown & SEND_SHUTDOWN)
1916 break;
1917 if (sk->sk_err)
1918 break;
1919 timeo = schedule_timeout(timeo);
1920 }
aa395145 1921 finish_wait(sk_sleep(sk), &wait);
1da177e4
LT		 1922 return timeo;
1923}
1924
1925
1926/*
1927 * Generic send/receive buffer handlers
1928 */
1929
4cc7f68d
HX		 1930struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1931 unsigned long data_len, int noblock,
28d64271 1932 int *errcode, int max_page_order)
1da177e4 1933{
2e4e4410 1934 struct sk_buff *skb;
1da177e4
LT		 1935 long timeo;
1936 int err;
1937
1da177e4 1938 timeo = sock_sndtimeo(sk, noblock);
2e4e4410 1939 for (;;) {
1da177e4
LT		 1940 err = sock_error(sk);
1941 if (err != 0)
1942 goto failure;
1943
1944 err = -EPIPE;
1945 if (sk->sk_shutdown & SEND_SHUTDOWN)
1946 goto failure;
1947
2e4e4410
ED		 1948 if (sk_wmem_alloc_get(sk) < sk->sk_sndbuf)
1949 break;
28d64271 1950
9cd3e072 1951 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2e4e4410
ED		 1952 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1953 err = -EAGAIN;
1954 if (!timeo)
1da177e4 1955 goto failure;
2e4e4410
ED		 1956 if (signal_pending(current))
1957 goto interrupted;
1958 timeo = sock_wait_for_wmem(sk, timeo);
1da177e4 1959 }
2e4e4410
ED		 1960 skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
1961 errcode, sk->sk_allocation);
1962 if (skb)
1963 skb_set_owner_w(skb, sk);
1da177e4
LT		 1964 return skb;
1965
1966interrupted:
1967 err = sock_intr_errno(timeo);
1968failure:
1969 *errcode = err;
1970 return NULL;
1971}
4cc7f68d 1972EXPORT_SYMBOL(sock_alloc_send_pskb);
1da177e4 1973
4ec93edb 1974struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1da177e4
LT		 1975 int noblock, int *errcode)
1976{
28d64271 1977 return sock_alloc_send_pskb(sk, size, 0, noblock, errcode, 0);
1da177e4 1978}
2a91525c 1979EXPORT_SYMBOL(sock_alloc_send_skb);
1da177e4 1980
39771b12
WB		 1981int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
1982 struct sockcm_cookie *sockc)
1983{
3dd17e63
SHY		 1984 u32 tsflags;
1985
39771b12
WB		 1986 switch (cmsg->cmsg_type) {
1987 case SO_MARK:
1988 if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
1989 return -EPERM;
1990 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
1991 return -EINVAL;
1992 sockc->mark = *(u32 *)CMSG_DATA(cmsg);
1993 break;
3dd17e63
SHY		 1994 case SO_TIMESTAMPING:
1995 if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
1996 return -EINVAL;
1997
1998 tsflags = *(u32 *)CMSG_DATA(cmsg);
1999 if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2000 return -EINVAL;
2001
2002 sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2003 sockc->tsflags |= tsflags;
2004 break;
779f1ede
SHY		 2005 /* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2006 case SCM_RIGHTS:
2007 case SCM_CREDENTIALS:
2008 break;
39771b12
WB		 2009 default:
2010 return -EINVAL;
2011 }
2012 return 0;
2013}
2014EXPORT_SYMBOL(__sock_cmsg_send);
2015
f28ea365
EJ		 2016int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2017 struct sockcm_cookie *sockc)
2018{
2019 struct cmsghdr *cmsg;
39771b12 2020 int ret;
f28ea365
EJ		 2021
2022 for_each_cmsghdr(cmsg, msg) {
2023 if (!CMSG_OK(msg, cmsg))
2024 return -EINVAL;
2025 if (cmsg->cmsg_level != SOL_SOCKET)
2026 continue;
39771b12
WB		 2027 ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2028 if (ret)
2029 return ret;
f28ea365
EJ		 2030 }
2031 return 0;
2032}
2033EXPORT_SYMBOL(sock_cmsg_send);
2034
5640f768
ED		 2035/* On 32bit arches, an skb frag is limited to 2^15 */
2036#define SKB_FRAG_PAGE_ORDER get_order(32768)
2037
400dfd3a
ED		 2038/**
2039 * skb_page_frag_refill - check that a page_frag contains enough room
2040 * @sz: minimum size of the fragment we want to get
2041 * @pfrag: pointer to page_frag
82d5e2b8 2042 * @gfp: priority for memory allocation
400dfd3a
ED		 2043 *
2044 * Note: While this allocator tries to use high order pages, there is
2045 * no guarantee that allocations succeed. Therefore, @sz MUST be
2046 * less or equal than PAGE_SIZE.
2047 */
d9b2938a 2048bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
5640f768 2049{
5640f768 2050 if (pfrag->page) {
fe896d18 2051 if (page_ref_count(pfrag->page) == 1) {
5640f768
ED		 2052 pfrag->offset = 0;
2053 return true;
2054 }
400dfd3a 2055 if (pfrag->offset + sz <= pfrag->size)
5640f768
ED		 2056 return true;
2057 put_page(pfrag->page);
2058 }
2059
d9b2938a
ED		 2060 pfrag->offset = 0;
2061 if (SKB_FRAG_PAGE_ORDER) {
d0164adc
MG		 2062 /* Avoid direct reclaim but allow kswapd to wake */
2063 pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2064 __GFP_COMP | __GFP_NOWARN |
2065 __GFP_NORETRY,
d9b2938a 2066 SKB_FRAG_PAGE_ORDER);
5640f768 2067 if (likely(pfrag->page)) {
d9b2938a 2068 pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
5640f768
ED		 2069 return true;
2070 }
d9b2938a
ED		 2071 }
2072 pfrag->page = alloc_page(gfp);
2073 if (likely(pfrag->page)) {
2074 pfrag->size = PAGE_SIZE;
2075 return true;
2076 }
400dfd3a
ED		 2077 return false;
2078}
2079EXPORT_SYMBOL(skb_page_frag_refill);
2080
2081bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2082{
2083 if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2084 return true;
2085
5640f768
ED		 2086 sk_enter_memory_pressure(sk);
2087 sk_stream_moderate_sndbuf(sk);
2088 return false;
2089}
2090EXPORT_SYMBOL(sk_page_frag_refill);
2091
1da177e4 2092static void __lock_sock(struct sock *sk)
f39234d6
NK		 2093 __releases(&sk->sk_lock.slock)
2094 __acquires(&sk->sk_lock.slock)
1da177e4
LT		 2095{
2096 DEFINE_WAIT(wait);
2097
e71a4783 2098 for (;;) {
1da177e4
LT		 2099 prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2100 TASK_UNINTERRUPTIBLE);
2101 spin_unlock_bh(&sk->sk_lock.slock);
2102 schedule();
2103 spin_lock_bh(&sk->sk_lock.slock);
e71a4783 2104 if (!sock_owned_by_user(sk))
1da177e4
LT		 2105 break;
2106 }
2107 finish_wait(&sk->sk_lock.wq, &wait);
2108}
2109
2110static void __release_sock(struct sock *sk)
f39234d6
NK		 2111 __releases(&sk->sk_lock.slock)
2112 __acquires(&sk->sk_lock.slock)
1da177e4 2113{
5413d1ba 2114 struct sk_buff *skb, *next;
1da177e4 2115
5413d1ba 2116 while ((skb = sk->sk_backlog.head) != NULL) {
1da177e4 2117 sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
1da177e4 2118
5413d1ba 2119 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4 2120
5413d1ba
ED		 2121 do {
2122 next = skb->next;
e4cbb02a 2123 prefetch(next);
7fee226a 2124 WARN_ON_ONCE(skb_dst_is_noref(skb));
1da177e4 2125 skb->next = NULL;
c57943a1 2126 sk_backlog_rcv(sk, skb);
1da177e4 2127
5413d1ba 2128 cond_resched();
1da177e4
LT		 2129
2130 skb = next;
2131 } while (skb != NULL);
2132
5413d1ba
ED		 2133 spin_lock_bh(&sk->sk_lock.slock);
2134 }
8eae939f
ZY		 2135
2136 /*
2137 * Doing the zeroing here guarantee we can not loop forever
2138 * while a wild producer attempts to flood us.
2139 */
2140 sk->sk_backlog.len = 0;
1da177e4
LT		 2141}
2142
d41a69f1
ED		 2143void __sk_flush_backlog(struct sock *sk)
2144{
2145 spin_lock_bh(&sk->sk_lock.slock);
2146 __release_sock(sk);
2147 spin_unlock_bh(&sk->sk_lock.slock);
2148}
2149
1da177e4
LT		 2150/**
2151 * sk_wait_data - wait for data to arrive at sk_receive_queue
4dc3b16b
PP		 2152 * @sk: sock to wait on
2153 * @timeo: for how long
dfbafc99 2154 * @skb: last skb seen on sk_receive_queue
1da177e4
LT		 2155 *
2156 * Now socket state including sk->sk_err is changed only under lock,
2157 * hence we may omit checks after joining wait queue.
2158 * We check receive queue before schedule() only as optimization;
2159 * it is very likely that release_sock() added new data.
2160 */
dfbafc99 2161int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
1da177e4 2162{
d9dc8b0f 2163 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1da177e4 2164 int rc;
1da177e4 2165
d9dc8b0f 2166 add_wait_queue(sk_sleep(sk), &wait);
9cd3e072 2167 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2168 rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
9cd3e072 2169 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
d9dc8b0f 2170 remove_wait_queue(sk_sleep(sk), &wait);
1da177e4
LT		 2171 return rc;
2172}
1da177e4
LT		 2173EXPORT_SYMBOL(sk_wait_data);
2174
3ab224be 2175/**
f8c3bf00 2176 * __sk_mem_raise_allocated - increase memory_allocated
3ab224be
HA		 2177 * @sk: socket
2178 * @size: memory size to allocate
f8c3bf00 2179 * @amt: pages to allocate
3ab224be
HA		 2180 * @kind: allocation type
2181 *
f8c3bf00 2182 * Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
3ab224be 2183 */
f8c3bf00 2184int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
3ab224be
HA		 2185{
2186 struct proto *prot = sk->sk_prot;
f8c3bf00 2187 long allocated = sk_memory_allocated_add(sk, amt);
e805605c 2188
baac50bb
JW		 2189 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
2190 !mem_cgroup_charge_skmem(sk->sk_memcg, amt))
e805605c 2191 goto suppress_allocation;
3ab224be
HA		 2192
2193 /* Under limit. */
e805605c 2194 if (allocated <= sk_prot_mem_limits(sk, 0)) {
180d8cd9 2195 sk_leave_memory_pressure(sk);
3ab224be
HA		 2196 return 1;
2197 }
2198
e805605c
JW		 2199 /* Under pressure. */
2200 if (allocated > sk_prot_mem_limits(sk, 1))
180d8cd9 2201 sk_enter_memory_pressure(sk);
3ab224be 2202
e805605c
JW		 2203 /* Over hard limit. */
2204 if (allocated > sk_prot_mem_limits(sk, 2))
3ab224be
HA		 2205 goto suppress_allocation;
2206
2207 /* guarantee minimum buffer size under pressure */
2208 if (kind == SK_MEM_RECV) {
2209 if (atomic_read(&sk->sk_rmem_alloc) < prot->sysctl_rmem[0])
2210 return 1;
180d8cd9 2211
3ab224be
HA		 2212 } else { /* SK_MEM_SEND */
2213 if (sk->sk_type == SOCK_STREAM) {
2214 if (sk->sk_wmem_queued < prot->sysctl_wmem[0])
2215 return 1;
2216 } else if (atomic_read(&sk->sk_wmem_alloc) <
2217 prot->sysctl_wmem[0])
2218 return 1;
2219 }
2220
180d8cd9 2221 if (sk_has_memory_pressure(sk)) {
1748376b
ED		 2222 int alloc;
2223
180d8cd9 2224 if (!sk_under_memory_pressure(sk))
1748376b 2225 return 1;
180d8cd9
GC		 2226 alloc = sk_sockets_allocated_read_positive(sk);
2227 if (sk_prot_mem_limits(sk, 2) > alloc *
3ab224be
HA		 2228 sk_mem_pages(sk->sk_wmem_queued +
2229 atomic_read(&sk->sk_rmem_alloc) +
2230 sk->sk_forward_alloc))
2231 return 1;
2232 }
2233
2234suppress_allocation:
2235
2236 if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
2237 sk_stream_moderate_sndbuf(sk);
2238
2239 /* Fail only if socket is _under_ its sndbuf.
2240 * In this case we cannot block, so that we have to fail.
2241 */
2242 if (sk->sk_wmem_queued + size >= sk->sk_sndbuf)
2243 return 1;
2244 }
2245
3847ce32
SM		 2246 trace_sock_exceed_buf_limit(sk, prot, allocated);
2247
0e90b31f 2248 sk_memory_allocated_sub(sk, amt);
180d8cd9 2249
baac50bb
JW		 2250 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2251 mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
e805605c 2252
3ab224be
HA		 2253 return 0;
2254}
f8c3bf00
PA		 2255EXPORT_SYMBOL(__sk_mem_raise_allocated);
2256
2257/**
2258 * __sk_mem_schedule - increase sk_forward_alloc and memory_allocated
2259 * @sk: socket
2260 * @size: memory size to allocate
2261 * @kind: allocation type
2262 *
2263 * If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
2264 * rmem allocation. This function assumes that protocols which have
2265 * memory_pressure use sk_wmem_queued as write buffer accounting.
2266 */
2267int __sk_mem_schedule(struct sock *sk, int size, int kind)
2268{
2269 int ret, amt = sk_mem_pages(size);
2270
2271 sk->sk_forward_alloc += amt << SK_MEM_QUANTUM_SHIFT;
2272 ret = __sk_mem_raise_allocated(sk, size, amt, kind);
2273 if (!ret)
2274 sk->sk_forward_alloc -= amt << SK_MEM_QUANTUM_SHIFT;
2275 return ret;
2276}
3ab224be
HA		 2277EXPORT_SYMBOL(__sk_mem_schedule);
2278
2279/**
f8c3bf00 2280 * __sk_mem_reduce_allocated - reclaim memory_allocated
3ab224be 2281 * @sk: socket
f8c3bf00
PA		 2282 * @amount: number of quanta
2283 *
2284 * Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3ab224be 2285 */
f8c3bf00 2286void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3ab224be 2287{
1a24e04e 2288 sk_memory_allocated_sub(sk, amount);
3ab224be 2289
baac50bb
JW		 2290 if (mem_cgroup_sockets_enabled && sk->sk_memcg)
2291 mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
e805605c 2292
180d8cd9
GC		 2293 if (sk_under_memory_pressure(sk) &&
2294 (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
2295 sk_leave_memory_pressure(sk);
3ab224be 2296}
f8c3bf00
PA		 2297EXPORT_SYMBOL(__sk_mem_reduce_allocated);
2298
2299/**
2300 * __sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
2301 * @sk: socket
2302 * @amount: number of bytes (rounded down to a SK_MEM_QUANTUM multiple)
2303 */
2304void __sk_mem_reclaim(struct sock *sk, int amount)
2305{
2306 amount >>= SK_MEM_QUANTUM_SHIFT;
2307 sk->sk_forward_alloc -= amount << SK_MEM_QUANTUM_SHIFT;
2308 __sk_mem_reduce_allocated(sk, amount);
2309}
3ab224be
HA		 2310EXPORT_SYMBOL(__sk_mem_reclaim);
2311
627d2d6b 2312int sk_set_peek_off(struct sock *sk, int val)
2313{
2314 if (val < 0)
2315 return -EINVAL;
2316
2317 sk->sk_peek_off = val;
2318 return 0;
2319}
2320EXPORT_SYMBOL_GPL(sk_set_peek_off);
3ab224be 2321
1da177e4
LT		 2322/*
2323 * Set of default routines for initialising struct proto_ops when
2324 * the protocol does not support a particular function. In certain
2325 * cases where it makes no sense for a protocol to have a "do nothing"
2326 * function, some default processing is provided.
2327 */
2328
2329int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
2330{
2331 return -EOPNOTSUPP;
2332}
2a91525c 2333EXPORT_SYMBOL(sock_no_bind);
1da177e4 2334
4ec93edb 2335int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
1da177e4
LT		 2336 int len, int flags)
2337{
2338 return -EOPNOTSUPP;
2339}
2a91525c 2340EXPORT_SYMBOL(sock_no_connect);
1da177e4
LT		 2341
2342int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
2343{
2344 return -EOPNOTSUPP;
2345}
2a91525c 2346EXPORT_SYMBOL(sock_no_socketpair);
1da177e4 2347
cdfbabfb
DH		 2348int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
2349 bool kern)
1da177e4
LT		 2350{
2351 return -EOPNOTSUPP;
2352}
2a91525c 2353EXPORT_SYMBOL(sock_no_accept);
1da177e4 2354
4ec93edb 2355int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
1da177e4
LT		 2356 int *len, int peer)
2357{
2358 return -EOPNOTSUPP;
2359}
2a91525c 2360EXPORT_SYMBOL(sock_no_getname);
1da177e4 2361
2a91525c 2362unsigned int sock_no_poll(struct file *file, struct socket *sock, poll_table *pt)
1da177e4
LT		 2363{
2364 return 0;
2365}
2a91525c 2366EXPORT_SYMBOL(sock_no_poll);
1da177e4
LT		 2367
2368int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
2369{
2370 return -EOPNOTSUPP;
2371}
2a91525c 2372EXPORT_SYMBOL(sock_no_ioctl);
1da177e4
LT		 2373
2374int sock_no_listen(struct socket *sock, int backlog)
2375{
2376 return -EOPNOTSUPP;
2377}
2a91525c 2378EXPORT_SYMBOL(sock_no_listen);
1da177e4
LT		 2379
2380int sock_no_shutdown(struct socket *sock, int how)
2381{
2382 return -EOPNOTSUPP;
2383}
2a91525c 2384EXPORT_SYMBOL(sock_no_shutdown);
1da177e4
LT		 2385
2386int sock_no_setsockopt(struct socket *sock, int level, int optname,
b7058842 2387 char __user *optval, unsigned int optlen)
1da177e4
LT		 2388{
2389 return -EOPNOTSUPP;
2390}
2a91525c 2391EXPORT_SYMBOL(sock_no_setsockopt);
1da177e4
LT		 2392
2393int sock_no_getsockopt(struct socket *sock, int level, int optname,
2394 char __user *optval, int __user *optlen)
2395{
2396 return -EOPNOTSUPP;
2397}
2a91525c 2398EXPORT_SYMBOL(sock_no_getsockopt);
1da177e4 2399
1b784140 2400int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
1da177e4
LT		 2401{
2402 return -EOPNOTSUPP;
2403}
2a91525c 2404EXPORT_SYMBOL(sock_no_sendmsg);
1da177e4 2405
1b784140
YX		 2406int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
2407 int flags)
1da177e4
LT		 2408{
2409 return -EOPNOTSUPP;
2410}
2a91525c 2411EXPORT_SYMBOL(sock_no_recvmsg);
1da177e4
LT		 2412
2413int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
2414{
2415 /* Mirror missing mmap method error code */
2416 return -ENODEV;
2417}
2a91525c 2418EXPORT_SYMBOL(sock_no_mmap);
1da177e4
LT		 2419
2420ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
2421{
2422 ssize_t res;
2423 struct msghdr msg = {.msg_flags = flags};
2424 struct kvec iov;
2425 char *kaddr = kmap(page);
2426 iov.iov_base = kaddr + offset;
2427 iov.iov_len = size;
2428 res = kernel_sendmsg(sock, &msg, &iov, 1, size);
2429 kunmap(page);
2430 return res;
2431}
2a91525c 2432EXPORT_SYMBOL(sock_no_sendpage);
1da177e4
LT		 2433
2434/*
2435 * Default Socket Callbacks
2436 */
2437
2438static void sock_def_wakeup(struct sock *sk)
2439{
43815482
ED		 2440 struct socket_wq *wq;
2441
2442 rcu_read_lock();
2443 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2444 if (skwq_has_sleeper(wq))
43815482
ED		 2445 wake_up_interruptible_all(&wq->wait);
2446 rcu_read_unlock();
1da177e4
LT		 2447}
2448
2449static void sock_def_error_report(struct sock *sk)
2450{
43815482
ED		 2451 struct socket_wq *wq;
2452
2453 rcu_read_lock();
2454 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2455 if (skwq_has_sleeper(wq))
43815482 2456 wake_up_interruptible_poll(&wq->wait, POLLERR);
8d8ad9d7 2457 sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
43815482 2458 rcu_read_unlock();
1da177e4
LT		 2459}
2460
676d2369 2461static void sock_def_readable(struct sock *sk)
1da177e4 2462{
43815482
ED		 2463 struct socket_wq *wq;
2464
2465 rcu_read_lock();
2466 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2467 if (skwq_has_sleeper(wq))
2c6607c6 2468 wake_up_interruptible_sync_poll(&wq->wait, POLLIN | POLLPRI |
37e5540b 2469 POLLRDNORM | POLLRDBAND);
8d8ad9d7 2470 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
43815482 2471 rcu_read_unlock();
1da177e4
LT		 2472}
2473
2474static void sock_def_write_space(struct sock *sk)
2475{
43815482
ED		 2476 struct socket_wq *wq;
2477
2478 rcu_read_lock();
1da177e4
LT		 2479
2480 /* Do not wake up a writer until he can make "significant"
2481 * progress. --DaveM
2482 */
e71a4783 2483 if ((atomic_read(&sk->sk_wmem_alloc) << 1) <= sk->sk_sndbuf) {
43815482 2484 wq = rcu_dereference(sk->sk_wq);
1ce0bf50 2485 if (skwq_has_sleeper(wq))
43815482 2486 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
37e5540b 2487 POLLWRNORM | POLLWRBAND);
1da177e4
LT		 2488
2489 /* Should agree with poll, otherwise some programs break */
2490 if (sock_writeable(sk))
8d8ad9d7 2491 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
1da177e4
LT		 2492 }
2493
43815482 2494 rcu_read_unlock();
1da177e4
LT		 2495}
2496
2497static void sock_def_destruct(struct sock *sk)
2498{
1da177e4
LT		 2499}
2500
2501void sk_send_sigurg(struct sock *sk)
2502{
2503 if (sk->sk_socket && sk->sk_socket->file)
2504 if (send_sigurg(&sk->sk_socket->file->f_owner))
8d8ad9d7 2505 sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
1da177e4 2506}
2a91525c 2507EXPORT_SYMBOL(sk_send_sigurg);
1da177e4
LT		 2508
2509void sk_reset_timer(struct sock *sk, struct timer_list* timer,
2510 unsigned long expires)
2511{
2512 if (!mod_timer(timer, expires))
2513 sock_hold(sk);
2514}
1da177e4
LT		 2515EXPORT_SYMBOL(sk_reset_timer);
2516
2517void sk_stop_timer(struct sock *sk, struct timer_list* timer)
2518{
25cc4ae9 2519 if (del_timer(timer))
1da177e4
LT		 2520 __sock_put(sk);
2521}
1da177e4
LT		 2522EXPORT_SYMBOL(sk_stop_timer);
2523
2524void sock_init_data(struct socket *sock, struct sock *sk)
2525{
581319c5 2526 sk_init_common(sk);
1da177e4
LT		 2527 sk->sk_send_head = NULL;
2528
2529 init_timer(&sk->sk_timer);
4ec93edb 2530
1da177e4
LT		 2531 sk->sk_allocation = GFP_KERNEL;
2532 sk->sk_rcvbuf = sysctl_rmem_default;
2533 sk->sk_sndbuf = sysctl_wmem_default;
2534 sk->sk_state = TCP_CLOSE;
972692e0 2535 sk_set_socket(sk, sock);
1da177e4
LT		 2536
2537 sock_set_flag(sk, SOCK_ZAPPED);
2538
e71a4783 2539 if (sock) {
1da177e4 2540 sk->sk_type = sock->type;
43815482 2541 sk->sk_wq = sock->wq;
1da177e4 2542 sock->sk = sk;
86741ec2
LC		 2543 sk->sk_uid = SOCK_INODE(sock)->i_uid;
2544 } else {
43815482 2545 sk->sk_wq = NULL;
86741ec2
LC		 2546 sk->sk_uid = make_kuid(sock_net(sk)->user_ns, 0);
2547 }
1da177e4 2548
1da177e4 2549 rwlock_init(&sk->sk_callback_lock);
cdfbabfb
DH		 2550 if (sk->sk_kern_sock)
2551 lockdep_set_class_and_name(
2552 &sk->sk_callback_lock,
2553 af_kern_callback_keys + sk->sk_family,
2554 af_family_kern_clock_key_strings[sk->sk_family]);
2555 else
2556 lockdep_set_class_and_name(
2557 &sk->sk_callback_lock,
443aef0e
PZ		 2558 af_callback_keys + sk->sk_family,
2559 af_family_clock_key_strings[sk->sk_family]);
1da177e4
LT		 2560
2561 sk->sk_state_change = sock_def_wakeup;
2562 sk->sk_data_ready = sock_def_readable;
2563 sk->sk_write_space = sock_def_write_space;
2564 sk->sk_error_report = sock_def_error_report;
2565 sk->sk_destruct = sock_def_destruct;
2566
5640f768
ED		 2567 sk->sk_frag.page = NULL;
2568 sk->sk_frag.offset = 0;
ef64a54f 2569 sk->sk_peek_off = -1;
1da177e4 2570
109f6e39
EB		 2571 sk->sk_peer_pid = NULL;
2572 sk->sk_peer_cred = NULL;
1da177e4
LT		 2573 sk->sk_write_pending = 0;
2574 sk->sk_rcvlowat = 1;
2575 sk->sk_rcvtimeo = MAX_SCHEDULE_TIMEOUT;
2576 sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
2577
f37f0afb 2578 sk->sk_stamp = ktime_set(-1L, 0);
1da177e4 2579
e0d1095a 2580#ifdef CONFIG_NET_RX_BUSY_POLL
06021292 2581 sk->sk_napi_id = 0;
64b0dc51 2582 sk->sk_ll_usec = sysctl_net_busy_read;
06021292
ET		 2583#endif
2584
62748f32 2585 sk->sk_max_pacing_rate = ~0U;
7eec4174 2586 sk->sk_pacing_rate = ~0U;
70da268b 2587 sk->sk_incoming_cpu = -1;
4dc6dc71
ED		 2588 /*
2589 * Before updating sk_refcnt, we must commit prior changes to memory
2590 * (Documentation/RCU/rculist_nulls.txt for details)
2591 */
2592 smp_wmb();
1da177e4 2593 atomic_set(&sk->sk_refcnt, 1);
33c732c3 2594 atomic_set(&sk->sk_drops, 0);
1da177e4 2595}
2a91525c 2596EXPORT_SYMBOL(sock_init_data);
1da177e4 2597
b5606c2d 2598void lock_sock_nested(struct sock *sk, int subclass)
1da177e4
LT		 2599{
2600 might_sleep();
a5b5bb9a 2601 spin_lock_bh(&sk->sk_lock.slock);
d2e9117c 2602 if (sk->sk_lock.owned)
1da177e4 2603 __lock_sock(sk);
d2e9117c 2604 sk->sk_lock.owned = 1;
a5b5bb9a
IM		 2605 spin_unlock(&sk->sk_lock.slock);
2606 /*
2607 * The sk_lock has mutex_lock() semantics here:
2608 */
fcc70d5f 2609 mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
a5b5bb9a 2610 local_bh_enable();
1da177e4 2611}
fcc70d5f 2612EXPORT_SYMBOL(lock_sock_nested);
1da177e4 2613
b5606c2d 2614void release_sock(struct sock *sk)
1da177e4 2615{
a5b5bb9a 2616 spin_lock_bh(&sk->sk_lock.slock);
1da177e4
LT		 2617 if (sk->sk_backlog.tail)
2618 __release_sock(sk);
46d3ceab 2619
c3f9b018
ED		 2620 /* Warning : release_cb() might need to release sk ownership,
2621 * ie call sock_release_ownership(sk) before us.
2622 */
46d3ceab
ED		 2623 if (sk->sk_prot->release_cb)
2624 sk->sk_prot->release_cb(sk);
2625
c3f9b018 2626 sock_release_ownership(sk);
a5b5bb9a
IM		 2627 if (waitqueue_active(&sk->sk_lock.wq))
2628 wake_up(&sk->sk_lock.wq);
2629 spin_unlock_bh(&sk->sk_lock.slock);
1da177e4
LT		 2630}
2631EXPORT_SYMBOL(release_sock);
2632
8a74ad60
ED		 2633/**
2634 * lock_sock_fast - fast version of lock_sock
2635 * @sk: socket
2636 *
2637 * This version should be used for very small section, where process wont block
2638 * return false if fast path is taken
2639 * sk_lock.slock locked, owned = 0, BH disabled
2640 * return true if slow path is taken
2641 * sk_lock.slock unlocked, owned = 1, BH enabled
2642 */
2643bool lock_sock_fast(struct sock *sk)
2644{
2645 might_sleep();
2646 spin_lock_bh(&sk->sk_lock.slock);
2647
2648 if (!sk->sk_lock.owned)
2649 /*
2650 * Note : We must disable BH
2651 */
2652 return false;
2653
2654 __lock_sock(sk);
2655 sk->sk_lock.owned = 1;
2656 spin_unlock(&sk->sk_lock.slock);
2657 /*
2658 * The sk_lock has mutex_lock() semantics here:
2659 */
2660 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
2661 local_bh_enable();
2662 return true;
2663}
2664EXPORT_SYMBOL(lock_sock_fast);
2665
1da177e4 2666int sock_get_timestamp(struct sock *sk, struct timeval __user *userstamp)
4ec93edb 2667{
b7aa0bf7 2668 struct timeval tv;
1da177e4 2669 if (!sock_flag(sk, SOCK_TIMESTAMP))
20d49473 2670 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
b7aa0bf7
ED		 2671 tv = ktime_to_timeval(sk->sk_stamp);
2672 if (tv.tv_sec == -1)
1da177e4 2673 return -ENOENT;
b7aa0bf7
ED		 2674 if (tv.tv_sec == 0) {
2675 sk->sk_stamp = ktime_get_real();
2676 tv = ktime_to_timeval(sk->sk_stamp);
2677 }
2678 return copy_to_user(userstamp, &tv, sizeof(tv)) ? -EFAULT : 0;
4ec93edb 2679}
1da177e4
LT		 2680EXPORT_SYMBOL(sock_get_timestamp);
2681
ae40eb1e
ED		 2682int sock_get_timestampns(struct sock *sk, struct timespec __user *userstamp)
2683{
2684 struct timespec ts;
2685 if (!sock_flag(sk, SOCK_TIMESTAMP))
20d49473 2686 sock_enable_timestamp(sk, SOCK_TIMESTAMP);
ae40eb1e
ED		 2687 ts = ktime_to_timespec(sk->sk_stamp);
2688 if (ts.tv_sec == -1)
2689 return -ENOENT;
2690 if (ts.tv_sec == 0) {
2691 sk->sk_stamp = ktime_get_real();
2692 ts = ktime_to_timespec(sk->sk_stamp);
2693 }
2694 return copy_to_user(userstamp, &ts, sizeof(ts)) ? -EFAULT : 0;
2695}
2696EXPORT_SYMBOL(sock_get_timestampns);
2697
20d49473 2698void sock_enable_timestamp(struct sock *sk, int flag)
4ec93edb 2699{
20d49473 2700 if (!sock_flag(sk, flag)) {
08e29af3
ED		 2701 unsigned long previous_flags = sk->sk_flags;
2702
20d49473
PO		 2703 sock_set_flag(sk, flag);
2704 /*
2705 * we just set one of the two flags which require net
2706 * time stamping, but time stamping might have been on
2707 * already because of the other one
2708 */
080a270f
HFS		 2709 if (sock_needs_netstamp(sk) &&
2710 !(previous_flags & SK_FLAGS_TIMESTAMP))
20d49473 2711 net_enable_timestamp();
1da177e4
LT		 2712 }
2713}
1da177e4 2714
cb820f8e
RC		 2715int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
2716 int level, int type)
2717{
2718 struct sock_exterr_skb *serr;
364a9e93 2719 struct sk_buff *skb;
cb820f8e
RC		 2720 int copied, err;
2721
2722 err = -EAGAIN;
364a9e93 2723 skb = sock_dequeue_err_skb(sk);
cb820f8e
RC		 2724 if (skb == NULL)
2725 goto out;
2726
2727 copied = skb->len;
2728 if (copied > len) {
2729 msg->msg_flags |= MSG_TRUNC;
2730 copied = len;
2731 }
51f3d02b 2732 err = skb_copy_datagram_msg(skb, 0, msg, copied);
cb820f8e
RC		 2733 if (err)
2734 goto out_free_skb;
2735
2736 sock_recv_timestamp(msg, sk, skb);
2737
2738 serr = SKB_EXT_ERR(skb);
2739 put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
2740
2741 msg->msg_flags |= MSG_ERRQUEUE;
2742 err = copied;
2743
cb820f8e
RC		 2744out_free_skb:
2745 kfree_skb(skb);
2746out:
2747 return err;
2748}
2749EXPORT_SYMBOL(sock_recv_errqueue);
2750
1da177e4
LT		 2751/*
2752 * Get a socket option on an socket.
2753 *
2754 * FIX: POSIX 1003.1g is very ambiguous here. It states that
2755 * asynchronous errors should be reported by getsockopt. We assume
2756 * this means if you specify SO_ERROR (otherwise whats the point of it).
2757 */
2758int sock_common_getsockopt(struct socket *sock, int level, int optname,
2759 char __user *optval, int __user *optlen)
2760{
2761 struct sock *sk = sock->sk;
2762
2763 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2764}
1da177e4
LT		 2765EXPORT_SYMBOL(sock_common_getsockopt);
2766
3fdadf7d 2767#ifdef CONFIG_COMPAT
543d9cfe
ACM		 2768int compat_sock_common_getsockopt(struct socket *sock, int level, int optname,
2769 char __user *optval, int __user *optlen)
3fdadf7d
DM		 2770{
2771 struct sock *sk = sock->sk;
2772
1e51f951 2773 if (sk->sk_prot->compat_getsockopt != NULL)
543d9cfe
ACM		 2774 return sk->sk_prot->compat_getsockopt(sk, level, optname,
2775 optval, optlen);
3fdadf7d
DM		 2776 return sk->sk_prot->getsockopt(sk, level, optname, optval, optlen);
2777}
2778EXPORT_SYMBOL(compat_sock_common_getsockopt);
2779#endif
2780
1b784140
YX		 2781int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
2782 int flags)
1da177e4
LT		 2783{
2784 struct sock *sk = sock->sk;
2785 int addr_len = 0;
2786 int err;
2787
1b784140 2788 err = sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
1da177e4
LT		 2789 flags & ~MSG_DONTWAIT, &addr_len);
2790 if (err >= 0)
2791 msg->msg_namelen = addr_len;
2792 return err;
2793}
1da177e4
LT		 2794EXPORT_SYMBOL(sock_common_recvmsg);
2795
2796/*
2797 * Set socket options on an inet socket.
2798 */
2799int sock_common_setsockopt(struct socket *sock, int level, int optname,
b7058842 2800 char __user *optval, unsigned int optlen)
1da177e4
LT		 2801{
2802 struct sock *sk = sock->sk;
2803
2804 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2805}
1da177e4
LT		 2806EXPORT_SYMBOL(sock_common_setsockopt);
2807
3fdadf7d 2808#ifdef CONFIG_COMPAT
543d9cfe 2809int compat_sock_common_setsockopt(struct socket *sock, int level, int optname,
b7058842 2810 char __user *optval, unsigned int optlen)
3fdadf7d
DM		 2811{
2812 struct sock *sk = sock->sk;
2813
543d9cfe
ACM		 2814 if (sk->sk_prot->compat_setsockopt != NULL)
2815 return sk->sk_prot->compat_setsockopt(sk, level, optname,
2816 optval, optlen);
3fdadf7d
DM		 2817 return sk->sk_prot->setsockopt(sk, level, optname, optval, optlen);
2818}
2819EXPORT_SYMBOL(compat_sock_common_setsockopt);
2820#endif
2821
1da177e4
LT		 2822void sk_common_release(struct sock *sk)
2823{
2824 if (sk->sk_prot->destroy)
2825 sk->sk_prot->destroy(sk);
2826
2827 /*
2828 * Observation: when sock_common_release is called, processes have
2829 * no access to socket. But net still has.
2830 * Step one, detach it from networking:
2831 *
2832 * A. Remove from hash tables.
2833 */
2834
2835 sk->sk_prot->unhash(sk);
2836
2837 /*
2838 * In this point socket cannot receive new packets, but it is possible
2839 * that some packets are in flight because some CPU runs receiver and
2840 * did hash table lookup before we unhashed socket. They will achieve
2841 * receive queue and will be purged by socket destructor.
2842 *
2843 * Also we still have packets pending on receive queue and probably,
2844 * our own packets waiting in device queues. sock_destroy will drain
2845 * receive queue, but transmitted packets will delay socket destruction
2846 * until the last reference will be released.
2847 */
2848
2849 sock_orphan(sk);
2850
2851 xfrm_sk_free_policy(sk);
2852
e6848976 2853 sk_refcnt_debug_release(sk);
5640f768
ED		 2854
2855 if (sk->sk_frag.page) {
2856 put_page(sk->sk_frag.page);
2857 sk->sk_frag.page = NULL;
2858 }
2859
1da177e4
LT		 2860 sock_put(sk);
2861}
1da177e4
LT		 2862EXPORT_SYMBOL(sk_common_release);
2863
13ff3d6f
PE		 2864#ifdef CONFIG_PROC_FS
2865#define PROTO_INUSE_NR 64 /* should be enough for the first time */
1338d466
PE		 2866struct prot_inuse {
2867 int val[PROTO_INUSE_NR];
2868};
13ff3d6f
PE		 2869
2870static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
70ee1159
PE		 2871
2872#ifdef CONFIG_NET_NS
2873void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
2874{
d6d9ca0f 2875 __this_cpu_add(net->core.inuse->val[prot->inuse_idx], val);
70ee1159
PE		 2876}
2877EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2878
2879int sock_prot_inuse_get(struct net *net, struct proto *prot)
2880{
2881 int cpu, idx = prot->inuse_idx;
2882 int res = 0;
2883
2884 for_each_possible_cpu(cpu)
2885 res += per_cpu_ptr(net->core.inuse, cpu)->val[idx];
2886
2887 return res >= 0 ? res : 0;
2888}
2889EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
2890
2c8c1e72 2891static int __net_init sock_inuse_init_net(struct net *net)
70ee1159
PE		 2892{
2893 net->core.inuse = alloc_percpu(struct prot_inuse);
2894 return net->core.inuse ? 0 : -ENOMEM;
2895}
2896
2c8c1e72 2897static void __net_exit sock_inuse_exit_net(struct net *net)
70ee1159
PE		 2898{
2899 free_percpu(net->core.inuse);
2900}
2901
2902static struct pernet_operations net_inuse_ops = {
2903 .init = sock_inuse_init_net,
2904 .exit = sock_inuse_exit_net,
2905};
2906
2907static __init int net_inuse_init(void)
2908{
2909 if (register_pernet_subsys(&net_inuse_ops))
2910 panic("Cannot initialize net inuse counters");
2911
2912 return 0;
2913}
2914
2915core_initcall(net_inuse_init);
2916#else
1338d466
PE		 2917static DEFINE_PER_CPU(struct prot_inuse, prot_inuse);
2918
c29a0bc4 2919void sock_prot_inuse_add(struct net *net, struct proto *prot, int val)
1338d466 2920{
d6d9ca0f 2921 __this_cpu_add(prot_inuse.val[prot->inuse_idx], val);
1338d466
PE		 2922}
2923EXPORT_SYMBOL_GPL(sock_prot_inuse_add);
2924
c29a0bc4 2925int sock_prot_inuse_get(struct net *net, struct proto *prot)
1338d466
PE		 2926{
2927 int cpu, idx = prot->inuse_idx;
2928 int res = 0;
2929
2930 for_each_possible_cpu(cpu)
2931 res += per_cpu(prot_inuse, cpu).val[idx];
2932
2933 return res >= 0 ? res : 0;
2934}
2935EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
70ee1159 2936#endif
13ff3d6f
PE		 2937
2938static void assign_proto_idx(struct proto *prot)
2939{
2940 prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
2941
2942 if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
e005d193 2943 pr_err("PROTO_INUSE_NR exhausted\n");
13ff3d6f
PE		 2944 return;
2945 }
2946
2947 set_bit(prot->inuse_idx, proto_inuse_idx);
2948}
2949
2950static void release_proto_idx(struct proto *prot)
2951{
2952 if (prot->inuse_idx != PROTO_INUSE_NR - 1)
2953 clear_bit(prot->inuse_idx, proto_inuse_idx);
2954}
2955#else
2956static inline void assign_proto_idx(struct proto *prot)
2957{
2958}
2959
2960static inline void release_proto_idx(struct proto *prot)
2961{
2962}
2963#endif
2964
0159dfd3
ED		 2965static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
2966{
2967 if (!rsk_prot)
2968 return;
2969 kfree(rsk_prot->slab_name);
2970 rsk_prot->slab_name = NULL;
adf78eda
JL		 2971 kmem_cache_destroy(rsk_prot->slab);
2972 rsk_prot->slab = NULL;
0159dfd3
ED		 2973}
2974
2975static int req_prot_init(const struct proto *prot)
2976{
2977 struct request_sock_ops *rsk_prot = prot->rsk_prot;
2978
2979 if (!rsk_prot)
2980 return 0;
2981
2982 rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
2983 prot->name);
2984 if (!rsk_prot->slab_name)
2985 return -ENOMEM;
2986
2987 rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
2988 rsk_prot->obj_size, 0,
e96f78ab 2989 prot->slab_flags, NULL);
0159dfd3
ED		 2990
2991 if (!rsk_prot->slab) {
2992 pr_crit("%s: Can't create request sock SLAB cache!\n",
2993 prot->name);
2994 return -ENOMEM;
2995 }
2996 return 0;
2997}
2998
b733c007
PE		 2999int proto_register(struct proto *prot, int alloc_slab)
3000{
1da177e4
LT		 3001 if (alloc_slab) {
3002 prot->slab = kmem_cache_create(prot->name, prot->obj_size, 0,
271b72c7
ED		 3003 SLAB_HWCACHE_ALIGN | prot->slab_flags,
3004 NULL);
1da177e4
LT		 3005
3006 if (prot->slab == NULL) {
e005d193
JP		 3007 pr_crit("%s: Can't create sock SLAB cache!\n",
3008 prot->name);
60e7663d 3009 goto out;
1da177e4 3010 }
2e6599cb 3011
0159dfd3
ED		 3012 if (req_prot_init(prot))
3013 goto out_free_request_sock_slab;
8feaf0c0 3014
6d6ee43e 3015 if (prot->twsk_prot != NULL) {
faf23422 3016 prot->twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s", prot->name);
8feaf0c0 3017
7e56b5d6 3018 if (prot->twsk_prot->twsk_slab_name == NULL)
8feaf0c0
ACM		 3019 goto out_free_request_sock_slab;
3020
6d6ee43e 3021 prot->twsk_prot->twsk_slab =
7e56b5d6 3022 kmem_cache_create(prot->twsk_prot->twsk_slab_name,
6d6ee43e 3023 prot->twsk_prot->twsk_obj_size,
3ab5aee7 3024 0,
52db70dc 3025 prot->slab_flags,
20c2df83 3026 NULL);
6d6ee43e 3027 if (prot->twsk_prot->twsk_slab == NULL)
8feaf0c0
ACM		 3028 goto out_free_timewait_sock_slab_name;
3029 }
1da177e4
LT		 3030 }
3031
36b77a52 3032 mutex_lock(&proto_list_mutex);
1da177e4 3033 list_add(&prot->node, &proto_list);
13ff3d6f 3034 assign_proto_idx(prot);
36b77a52 3035 mutex_unlock(&proto_list_mutex);
b733c007
PE		 3036 return 0;
3037
8feaf0c0 3038out_free_timewait_sock_slab_name:
7e56b5d6 3039 kfree(prot->twsk_prot->twsk_slab_name);
8feaf0c0 3040out_free_request_sock_slab:
0159dfd3
ED		 3041 req_prot_cleanup(prot->rsk_prot);
3042
2e6599cb
ACM		 3043 kmem_cache_destroy(prot->slab);
3044 prot->slab = NULL;
b733c007
PE		 3045out:
3046 return -ENOBUFS;
1da177e4 3047}
1da177e4
LT		 3048EXPORT_SYMBOL(proto_register);
3049
3050void proto_unregister(struct proto *prot)
3051{
36b77a52 3052 mutex_lock(&proto_list_mutex);
13ff3d6f 3053 release_proto_idx(prot);
0a3f4358 3054 list_del(&prot->node);
36b77a52 3055 mutex_unlock(&proto_list_mutex);
1da177e4 3056
adf78eda
JL		 3057 kmem_cache_destroy(prot->slab);
3058 prot->slab = NULL;
1da177e4 3059
0159dfd3 3060 req_prot_cleanup(prot->rsk_prot);
2e6599cb 3061
6d6ee43e 3062 if (prot->twsk_prot != NULL && prot->twsk_prot->twsk_slab != NULL) {
6d6ee43e 3063 kmem_cache_destroy(prot->twsk_prot->twsk_slab);
7e56b5d6 3064 kfree(prot->twsk_prot->twsk_slab_name);
6d6ee43e 3065 prot->twsk_prot->twsk_slab = NULL;
8feaf0c0 3066 }
1da177e4 3067}
1da177e4
LT		 3068EXPORT_SYMBOL(proto_unregister);
3069
3070#ifdef CONFIG_PROC_FS
1da177e4 3071static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
36b77a52 3072 __acquires(proto_list_mutex)
1da177e4 3073{
36b77a52 3074 mutex_lock(&proto_list_mutex);
60f0438a 3075 return seq_list_start_head(&proto_list, *pos);
1da177e4
LT		 3076}
3077
3078static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3079{
60f0438a 3080 return seq_list_next(v, &proto_list, pos);
1da177e4
LT		 3081}
3082
3083static void proto_seq_stop(struct seq_file *seq, void *v)
36b77a52 3084 __releases(proto_list_mutex)
1da177e4 3085{
36b77a52 3086 mutex_unlock(&proto_list_mutex);
1da177e4
LT		 3087}
3088
3089static char proto_method_implemented(const void *method)
3090{
3091 return method == NULL ? 'n' : 'y';
3092}
180d8cd9
GC		 3093static long sock_prot_memory_allocated(struct proto *proto)
3094{
cb75a36c 3095 return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
180d8cd9
GC		 3096}
3097
3098static char *sock_prot_memory_pressure(struct proto *proto)
3099{
3100 return proto->memory_pressure != NULL ?
3101 proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3102}
1da177e4
LT		 3103
3104static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3105{
180d8cd9 3106
8d987e5c 3107 seq_printf(seq, "%-9s %4u %6d %6ld %-3s %6u %-3s %-10s "
1da177e4
LT		 3108 "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3109 proto->name,
3110 proto->obj_size,
14e943db 3111 sock_prot_inuse_get(seq_file_net(seq), proto),
180d8cd9
GC		 3112 sock_prot_memory_allocated(proto),
3113 sock_prot_memory_pressure(proto),
1da177e4
LT		 3114 proto->max_header,
3115 proto->slab == NULL ? "no" : "yes",
3116 module_name(proto->owner),
3117 proto_method_implemented(proto->close),
3118 proto_method_implemented(proto->connect),
3119 proto_method_implemented(proto->disconnect),
3120 proto_method_implemented(proto->accept),
3121 proto_method_implemented(proto->ioctl),
3122 proto_method_implemented(proto->init),
3123 proto_method_implemented(proto->destroy),
3124 proto_method_implemented(proto->shutdown),
3125 proto_method_implemented(proto->setsockopt),
3126 proto_method_implemented(proto->getsockopt),
3127 proto_method_implemented(proto->sendmsg),
3128 proto_method_implemented(proto->recvmsg),
3129 proto_method_implemented(proto->sendpage),
3130 proto_method_implemented(proto->bind),
3131 proto_method_implemented(proto->backlog_rcv),
3132 proto_method_implemented(proto->hash),
3133 proto_method_implemented(proto->unhash),
3134 proto_method_implemented(proto->get_port),
3135 proto_method_implemented(proto->enter_memory_pressure));
3136}
3137
3138static int proto_seq_show(struct seq_file *seq, void *v)
3139{
60f0438a 3140 if (v == &proto_list)
1da177e4
LT		 3141 seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
3142 "protocol",
3143 "size",
3144 "sockets",
3145 "memory",
3146 "press",
3147 "maxhdr",
3148 "slab",
3149 "module",
3150 "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
3151 else
60f0438a 3152 proto_seq_printf(seq, list_entry(v, struct proto, node));
1da177e4
LT		 3153 return 0;
3154}
3155
f690808e 3156static const struct seq_operations proto_seq_ops = {
1da177e4
LT		 3157 .start = proto_seq_start,
3158 .next = proto_seq_next,
3159 .stop = proto_seq_stop,
3160 .show = proto_seq_show,
3161};
3162
3163static int proto_seq_open(struct inode *inode, struct file *file)
3164{
14e943db
ED		 3165 return seq_open_net(inode, file, &proto_seq_ops,
3166 sizeof(struct seq_net_private));
1da177e4
LT		 3167}
3168
9a32144e 3169static const struct file_operations proto_seq_fops = {
1da177e4
LT		 3170 .owner = THIS_MODULE,
3171 .open = proto_seq_open,
3172 .read = seq_read,
3173 .llseek = seq_lseek,
14e943db
ED		 3174 .release = seq_release_net,
3175};
3176
3177static __net_init int proto_init_net(struct net *net)
3178{
d4beaa66 3179 if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops))
14e943db
ED		 3180 return -ENOMEM;
3181
3182 return 0;
3183}
3184
3185static __net_exit void proto_exit_net(struct net *net)
3186{
ece31ffd 3187 remove_proc_entry("protocols", net->proc_net);
14e943db
ED		 3188}
3189
3190
3191static __net_initdata struct pernet_operations proto_net_ops = {
3192 .init = proto_init_net,
3193 .exit = proto_exit_net,
1da177e4
LT		 3194};
3195
3196static int __init proto_init(void)
3197{
14e943db 3198 return register_pernet_subsys(&proto_net_ops);
1da177e4
LT		 3199}
3200
3201subsys_initcall(proto_init);
3202
3203#endif /* PROC_FS */
Linux 6.x block layer and io_uring tree(s)