projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
etf: Drop all expired packets
[linux-2.6-block.git] / net / ipv4 / udp.c
CommitLineData
1da177e4
LT		 1/*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * The User Datagram Protocol (UDP).
7 *
02c30a84 8 * Authors: Ross Biro
1da177e4
LT		 9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
113aa838 11 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
1da177e4
LT		 12 * Hirokazu Takahashi, <taka@valinux.co.jp>
13 *
14 * Fixes:
15 * Alan Cox : verify_area() calls
16 * Alan Cox : stopped close while in use off icmp
17 * messages. Not a fix but a botch that
18 * for udp at least is 'valid'.
19 * Alan Cox : Fixed icmp handling properly
20 * Alan Cox : Correct error for oversized datagrams
e905a9ed
YH		 21 * Alan Cox : Tidied select() semantics.
22 * Alan Cox : udp_err() fixed properly, also now
1da177e4
LT		 23 * select and read wake correctly on errors
24 * Alan Cox : udp_send verify_area moved to avoid mem leak
25 * Alan Cox : UDP can count its memory
26 * Alan Cox : send to an unknown connection causes
27 * an ECONNREFUSED off the icmp, but
28 * does NOT close.
29 * Alan Cox : Switched to new sk_buff handlers. No more backlog!
30 * Alan Cox : Using generic datagram code. Even smaller and the PEEK
31 * bug no longer crashes it.
32 * Fred Van Kempen : Net2e support for sk->broadcast.
33 * Alan Cox : Uses skb_free_datagram
34 * Alan Cox : Added get/set sockopt support.
35 * Alan Cox : Broadcasting without option set returns EACCES.
36 * Alan Cox : No wakeup calls. Instead we now use the callbacks.
37 * Alan Cox : Use ip_tos and ip_ttl
38 * Alan Cox : SNMP Mibs
39 * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
40 * Matt Dillon : UDP length checks.
41 * Alan Cox : Smarter af_inet used properly.
42 * Alan Cox : Use new kernel side addressing.
43 * Alan Cox : Incorrect return on truncated datagram receive.
44 * Arnt Gulbrandsen : New udp_send and stuff
45 * Alan Cox : Cache last socket
46 * Alan Cox : Route cache
47 * Jon Peatfield : Minor efficiency fix to sendto().
48 * Mike Shaver : RFC1122 checks.
49 * Alan Cox : Nonblocking error fix.
50 * Willy Konynenberg : Transparent proxying support.
51 * Mike McLagan : Routing by source
52 * David S. Miller : New socket lookup architecture.
53 * Last socket cache retained as it
54 * does have a high hit rate.
55 * Olaf Kirch : Don't linearise iovec on sendmsg.
56 * Andi Kleen : Some cleanups, cache destination entry
e905a9ed 57 * for connect.
1da177e4
LT		 58 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
59 * Melvin Smith : Check msg_name not msg_namelen in sendto(),
60 * return ENOTCONN for unconnected sockets (POSIX)
61 * Janos Farkas : don't deliver multi/broadcasts to a different
62 * bound-to-device socket
63 * Hirokazu Takahashi : HW checksumming for outgoing UDP
64 * datagrams.
65 * Hirokazu Takahashi : sendfile() on UDP works now.
66 * Arnaldo C. Melo : convert /proc/net/udp to seq_file
67 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
68 * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
69 * a single port at the same time.
70 * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
342f0234 71 * James Chapman : Add L2TP encapsulation type.
1da177e4
LT		 72 *
73 *
74 * This program is free software; you can redistribute it and/or
75 * modify it under the terms of the GNU General Public License
76 * as published by the Free Software Foundation; either version
77 * 2 of the License, or (at your option) any later version.
78 */
e905a9ed 79
afd46503
JP		 80#define pr_fmt(fmt) "UDP: " fmt
81
7c0f6ba6 82#include <linux/uaccess.h>
1da177e4 83#include <asm/ioctls.h>
57c8a661 84#include <linux/memblock.h>
8203efb3
ED		 85#include <linux/highmem.h>
86#include <linux/swap.h>
1da177e4
LT		 87#include <linux/types.h>
88#include <linux/fcntl.h>
89#include <linux/module.h>
90#include <linux/socket.h>
91#include <linux/sockios.h>
14c85021 92#include <linux/igmp.h>
6e540309 93#include <linux/inetdevice.h>
1da177e4
LT		 94#include <linux/in.h>
95#include <linux/errno.h>
96#include <linux/timer.h>
97#include <linux/mm.h>
1da177e4 98#include <linux/inet.h>
1da177e4 99#include <linux/netdevice.h>
5a0e3ad6 100#include <linux/slab.h>
c752f073 101#include <net/tcp_states.h>
1da177e4
LT		 102#include <linux/skbuff.h>
103#include <linux/proc_fs.h>
104#include <linux/seq_file.h>
457c4cbc 105#include <net/net_namespace.h>
1da177e4 106#include <net/icmp.h>
421b3885 107#include <net/inet_hashtables.h>
e7cc0824 108#include <net/ip_tunnels.h>
1da177e4 109#include <net/route.h>
1da177e4
LT		 110#include <net/checksum.h>
111#include <net/xfrm.h>
296f7ea7 112#include <trace/events/udp.h>
447167bf 113#include <linux/static_key.h>
22911fc5 114#include <trace/events/skb.h>
076bb0c8 115#include <net/busy_poll.h>
ba4e58ec 116#include "udp_impl.h"
e32ea7e7 117#include <net/sock_reuseport.h>
217375a0 118#include <net/addrconf.h>
60fb9567 119#include <net/udp_tunnel.h>
1da177e4 120
f86dcc5a 121struct udp_table udp_table __read_mostly;
645ca708 122EXPORT_SYMBOL(udp_table);
1da177e4 123
8d987e5c 124long sysctl_udp_mem[3] __read_mostly;
95766fff 125EXPORT_SYMBOL(sysctl_udp_mem);
c482c568 126
8d987e5c 127atomic_long_t udp_memory_allocated;
95766fff
HA		 128EXPORT_SYMBOL(udp_memory_allocated);
129
f86dcc5a
ED		 130#define MAX_UDP_PORTS 65536
131#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
98322f22 132
63a6fff3
RS		 133/* IPCB reference means this can not be used from early demux */
134static bool udp_lib_exact_dif_match(struct net *net, struct sk_buff *skb)
135{
136#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV)
137 if (!net->ipv4.sysctl_udp_l3mdev_accept &&
138 skb && ipv4_l3mdev_skb(IPCB(skb)->flags))
139 return true;
140#endif
141 return false;
142}
143
f24d43c0 144static int udp_lib_lport_inuse(struct net *net, __u16 num,
645ca708 145 const struct udp_hslot *hslot,
98322f22 146 unsigned long *bitmap,
fe38d2a1 147 struct sock *sk, unsigned int log)
1da177e4 148{
f24d43c0 149 struct sock *sk2;
ba418fa3 150 kuid_t uid = sock_i_uid(sk);
25030a7f 151
ca065d0c 152 sk_for_each(sk2, &hslot->head) {
9d4fb27d
JP		 153 if (net_eq(sock_net(sk2), net) &&
154 sk2 != sk &&
d4cada4a 155 (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
9d4fb27d
JP		 156 (!sk2->sk_reuse || !sk->sk_reuse) &&
157 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
158 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
fe38d2a1 159 inet_rcv_saddr_equal(sk, sk2, true)) {
df560056
EG		 160 if (sk2->sk_reuseport && sk->sk_reuseport &&
161 !rcu_access_pointer(sk->sk_reuseport_cb) &&
162 uid_eq(uid, sock_i_uid(sk2))) {
163 if (!bitmap)
164 return 0;
165 } else {
166 if (!bitmap)
167 return 1;
168 __set_bit(udp_sk(sk2)->udp_port_hash >> log,
169 bitmap);
170 }
98322f22 171 }
4243cdc2 172 }
25030a7f
GR		 173 return 0;
174}
175
30fff923
ED		 176/*
177 * Note: we still hold spinlock of primary hash chain, so no other writer
178 * can insert/delete a socket with local_port == num
179 */
180static int udp_lib_lport_inuse2(struct net *net, __u16 num,
4243cdc2 181 struct udp_hslot *hslot2,
fe38d2a1 182 struct sock *sk)
30fff923
ED		 183{
184 struct sock *sk2;
ba418fa3 185 kuid_t uid = sock_i_uid(sk);
30fff923
ED		 186 int res = 0;
187
188 spin_lock(&hslot2->lock);
ca065d0c 189 udp_portaddr_for_each_entry(sk2, &hslot2->head) {
9d4fb27d
JP		 190 if (net_eq(sock_net(sk2), net) &&
191 sk2 != sk &&
192 (udp_sk(sk2)->udp_port_hash == num) &&
193 (!sk2->sk_reuse || !sk->sk_reuse) &&
194 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
195 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
fe38d2a1 196 inet_rcv_saddr_equal(sk, sk2, true)) {
df560056
EG		 197 if (sk2->sk_reuseport && sk->sk_reuseport &&
198 !rcu_access_pointer(sk->sk_reuseport_cb) &&
199 uid_eq(uid, sock_i_uid(sk2))) {
200 res = 0;
201 } else {
202 res = 1;
203 }
30fff923
ED		 204 break;
205 }
4243cdc2 206 }
30fff923
ED		 207 spin_unlock(&hslot2->lock);
208 return res;
209}
210
fe38d2a1 211static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
e32ea7e7
CG		 212{
213 struct net *net = sock_net(sk);
e32ea7e7
CG		 214 kuid_t uid = sock_i_uid(sk);
215 struct sock *sk2;
216
ca065d0c 217 sk_for_each(sk2, &hslot->head) {
e32ea7e7
CG		 218 if (net_eq(sock_net(sk2), net) &&
219 sk2 != sk &&
220 sk2->sk_family == sk->sk_family &&
221 ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
222 (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
223 (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
224 sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
fe38d2a1 225 inet_rcv_saddr_equal(sk, sk2, false)) {
2dbb9b9e
MKL		 226 return reuseport_add_sock(sk, sk2,
227 inet_rcv_saddr_any(sk));
e32ea7e7
CG		 228 }
229 }
230
2dbb9b9e 231 return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
e32ea7e7
CG		 232}
233
25030a7f 234/**
6ba5a3c5 235 * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
25030a7f
GR		 236 *
237 * @sk: socket struct in question
238 * @snum: port number to look up
25985edc 239 * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
30fff923 240 * with NULL address
25030a7f 241 */
6ba5a3c5 242int udp_lib_get_port(struct sock *sk, unsigned short snum,
30fff923 243 unsigned int hash2_nulladdr)
25030a7f 244{
512615b6 245 struct udp_hslot *hslot, *hslot2;
645ca708 246 struct udp_table *udptable = sk->sk_prot->h.udp_table;
25030a7f 247 int error = 1;
3b1e0a65 248 struct net *net = sock_net(sk);
1da177e4 249
32c1da70 250 if (!snum) {
9088c560 251 int low, high, remaining;
95c96174 252 unsigned int rand;
98322f22
ED		 253 unsigned short first, last;
254 DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
32c1da70 255
0bbf87d8 256 inet_get_local_port_range(net, &low, &high);
a25de534 257 remaining = (high - low) + 1;
227b60f5 258
63862b5b 259 rand = prandom_u32();
8fc54f68 260 first = reciprocal_scale(rand, remaining) + low;
98322f22
ED		 261 /*
262 * force rand to be an odd multiple of UDP_HTABLE_SIZE
263 */
f86dcc5a 264 rand = (rand | 1) * (udptable->mask + 1);
5781b235
ED		 265 last = first + udptable->mask + 1;
266 do {
f86dcc5a 267 hslot = udp_hashslot(udptable, net, first);
98322f22 268 bitmap_zero(bitmap, PORTS_PER_CHAIN);
645ca708 269 spin_lock_bh(&hslot->lock);
98322f22 270 udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
fe38d2a1 271 udptable->log);
98322f22
ED		 272
273 snum = first;
274 /*
275 * Iterate on all possible values of snum for this hash.
276 * Using steps of an odd multiple of UDP_HTABLE_SIZE
277 * give us randomization and full range coverage.
278 */
9088c560 279 do {
98322f22 280 if (low <= snum && snum <= high &&
e3826f1e 281 !test_bit(snum >> udptable->log, bitmap) &&
122ff243 282 !inet_is_local_reserved_port(net, snum))
98322f22
ED		 283 goto found;
284 snum += rand;
285 } while (snum != first);
286 spin_unlock_bh(&hslot->lock);
df560056 287 cond_resched();
5781b235 288 } while (++first != last);
98322f22 289 goto fail;
645ca708 290 } else {
f86dcc5a 291 hslot = udp_hashslot(udptable, net, snum);
645ca708 292 spin_lock_bh(&hslot->lock);
30fff923
ED		 293 if (hslot->count > 10) {
294 int exist;
295 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
296
297 slot2 &= udptable->mask;
298 hash2_nulladdr &= udptable->mask;
299
300 hslot2 = udp_hashslot2(udptable, slot2);
301 if (hslot->count < hslot2->count)
302 goto scan_primary_hash;
303
fe38d2a1 304 exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
30fff923
ED		 305 if (!exist && (hash2_nulladdr != slot2)) {
306 hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
307 exist = udp_lib_lport_inuse2(net, snum, hslot2,
fe38d2a1 308 sk);
30fff923
ED		 309 }
310 if (exist)
311 goto fail_unlock;
312 else
313 goto found;
314 }
315scan_primary_hash:
fe38d2a1 316 if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
645ca708
ED		 317 goto fail_unlock;
318 }
98322f22 319found:
c720c7e8 320 inet_sk(sk)->inet_num = snum;
d4cada4a
ED		 321 udp_sk(sk)->udp_port_hash = snum;
322 udp_sk(sk)->udp_portaddr_hash ^= snum;
1da177e4 323 if (sk_unhashed(sk)) {
e32ea7e7 324 if (sk->sk_reuseport &&
fe38d2a1 325 udp_reuseport_add_sock(sk, hslot)) {
e32ea7e7
CG		 326 inet_sk(sk)->inet_num = 0;
327 udp_sk(sk)->udp_port_hash = 0;
328 udp_sk(sk)->udp_portaddr_hash ^= snum;
329 goto fail_unlock;
330 }
331
ca065d0c 332 sk_add_node_rcu(sk, &hslot->head);
fdcc8aa9 333 hslot->count++;
c29a0bc4 334 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
512615b6
ED		 335
336 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
337 spin_lock(&hslot2->lock);
d894ba18 338 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
1602f49b
DM		 339 sk->sk_family == AF_INET6)
340 hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
341 &hslot2->head);
d894ba18 342 else
1602f49b
DM		 343 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
344 &hslot2->head);
512615b6
ED		 345 hslot2->count++;
346 spin_unlock(&hslot2->lock);
1da177e4 347 }
ca065d0c 348 sock_set_flag(sk, SOCK_RCU_FREE);
25030a7f 349 error = 0;
645ca708
ED		 350fail_unlock:
351 spin_unlock_bh(&hslot->lock);
1da177e4 352fail:
25030a7f
GR		 353 return error;
354}
c482c568 355EXPORT_SYMBOL(udp_lib_get_port);
25030a7f 356
6ba5a3c5 357int udp_v4_get_port(struct sock *sk, unsigned short snum)
db8dac20 358{
30fff923 359 unsigned int hash2_nulladdr =
f0b1e64c 360 ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
30fff923 361 unsigned int hash2_partial =
f0b1e64c 362 ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
30fff923 363
d4cada4a 364 /* precompute partial secondary hash */
30fff923 365 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
fe38d2a1 366 return udp_lib_get_port(sk, snum, hash2_nulladdr);
db8dac20
DM		 367}
368
d1e37288
SX		 369static int compute_score(struct sock *sk, struct net *net,
370 __be32 saddr, __be16 sport,
fb74c277
DA		 371 __be32 daddr, unsigned short hnum,
372 int dif, int sdif, bool exact_dif)
645ca708 373{
60c04aec
JP		 374 int score;
375 struct inet_sock *inet;
6da5b0f0 376 bool dev_match;
645ca708 377
60c04aec
JP		 378 if (!net_eq(sock_net(sk), net) ||
379 udp_sk(sk)->udp_port_hash != hnum ||
380 ipv6_only_sock(sk))
381 return -1;
645ca708 382
60c04aec
JP		 383 score = (sk->sk_family == PF_INET) ? 2 : 1;
384 inet = inet_sk(sk);
385
386 if (inet->inet_rcv_saddr) {
387 if (inet->inet_rcv_saddr != daddr)
388 return -1;
389 score += 4;
645ca708 390 }
60c04aec
JP		 391
392 if (inet->inet_daddr) {
393 if (inet->inet_daddr != saddr)
394 return -1;
395 score += 4;
396 }
397
398 if (inet->inet_dport) {
399 if (inet->inet_dport != sport)
400 return -1;
401 score += 4;
402 }
403
6da5b0f0
MM		 404 dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
405 dif, sdif);
406 if (!dev_match)
407 return -1;
408 score += 4;
fb74c277 409
70da268b
ED		 410 if (sk->sk_incoming_cpu == raw_smp_processor_id())
411 score++;
645ca708
ED		 412 return score;
413}
414
6eada011
ED		 415static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
416 const __u16 lport, const __be32 faddr,
417 const __be16 fport)
65cd8033 418{
1bbdceef
HFS		 419 static u32 udp_ehash_secret __read_mostly;
420
421 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
422
65cd8033 423 return __inet_ehashfn(laddr, lport, faddr, fport,
1bbdceef 424 udp_ehash_secret + net_hash_mix(net));
65cd8033
HFS		 425}
426
d1e37288 427/* called with rcu_read_lock() */
5051ebd2 428static struct sock *udp4_lib_lookup2(struct net *net,
fb74c277
DA		 429 __be32 saddr, __be16 sport,
430 __be32 daddr, unsigned int hnum,
431 int dif, int sdif, bool exact_dif,
432 struct udp_hslot *hslot2,
433 struct sk_buff *skb)
5051ebd2
ED		 434{
435 struct sock *sk, *result;
e94a62f5 436 int score, badness;
ba418fa3 437 u32 hash = 0;
5051ebd2 438
5051ebd2 439 result = NULL;
ba418fa3 440 badness = 0;
ca065d0c 441 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
d1e37288 442 score = compute_score(sk, net, saddr, sport,
fb74c277 443 daddr, hnum, dif, sdif, exact_dif);
5051ebd2 444 if (score > badness) {
e94a62f5 445 if (sk->sk_reuseport) {
65cd8033
HFS		 446 hash = udp_ehashfn(net, daddr, hnum,
447 saddr, sport);
ca065d0c 448 result = reuseport_select_sock(sk, hash, skb,
ed0dfffd 449 sizeof(struct udphdr));
ca065d0c
ED		 450 if (result)
451 return result;
ba418fa3 452 }
ca065d0c
ED		 453 badness = score;
454 result = sk;
5051ebd2
ED		 455 }
456 }
5051ebd2
ED		 457 return result;
458}
459
db8dac20
DM		 460/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
461 * harder than this. -DaveM
462 */
fce82338 463struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
fb74c277
DA		 464 __be16 sport, __be32 daddr, __be16 dport, int dif,
465 int sdif, struct udp_table *udptable, struct sk_buff *skb)
db8dac20 466{
271b72c7 467 struct sock *sk, *result;
db8dac20 468 unsigned short hnum = ntohs(dport);
5051ebd2
ED		 469 unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask);
470 struct udp_hslot *hslot2, *hslot = &udptable->hash[slot];
63a6fff3 471 bool exact_dif = udp_lib_exact_dif_match(net, skb);
e94a62f5 472 int score, badness;
ba418fa3 473 u32 hash = 0;
645ca708 474
5051ebd2 475 if (hslot->count > 10) {
f0b1e64c 476 hash2 = ipv4_portaddr_hash(net, daddr, hnum);
5051ebd2
ED		 477 slot2 = hash2 & udptable->mask;
478 hslot2 = &udptable->hash2[slot2];
479 if (hslot->count < hslot2->count)
480 goto begin;
481
482 result = udp4_lib_lookup2(net, saddr, sport,
fb74c277 483 daddr, hnum, dif, sdif,
63a6fff3 484 exact_dif, hslot2, skb);
5051ebd2 485 if (!result) {
d1e37288 486 unsigned int old_slot2 = slot2;
f0b1e64c 487 hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
5051ebd2 488 slot2 = hash2 & udptable->mask;
d1e37288
SX		 489 /* avoid searching the same slot again. */
490 if (unlikely(slot2 == old_slot2))
491 return result;
492
5051ebd2
ED		 493 hslot2 = &udptable->hash2[slot2];
494 if (hslot->count < hslot2->count)
495 goto begin;
496
1223c67c 497 result = udp4_lib_lookup2(net, saddr, sport,
fb74c277 498 daddr, hnum, dif, sdif,
63a6fff3 499 exact_dif, hslot2, skb);
5051ebd2 500 }
8217ca65
MKL		 501 if (unlikely(IS_ERR(result)))
502 return NULL;
5051ebd2
ED		 503 return result;
504 }
271b72c7
ED		 505begin:
506 result = NULL;
ba418fa3 507 badness = 0;
ca065d0c 508 sk_for_each_rcu(sk, &hslot->head) {
d1e37288 509 score = compute_score(sk, net, saddr, sport,
fb74c277 510 daddr, hnum, dif, sdif, exact_dif);
645ca708 511 if (score > badness) {
e94a62f5 512 if (sk->sk_reuseport) {
65cd8033
HFS		 513 hash = udp_ehashfn(net, daddr, hnum,
514 saddr, sport);
ca065d0c 515 result = reuseport_select_sock(sk, hash, skb,
538950a1 516 sizeof(struct udphdr));
8217ca65
MKL		 517 if (unlikely(IS_ERR(result)))
518 return NULL;
ca065d0c
ED		 519 if (result)
520 return result;
ba418fa3 521 }
ca065d0c
ED		 522 result = sk;
523 badness = score;
db8dac20
DM		 524 }
525 }
db8dac20
DM		 526 return result;
527}
fce82338 528EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
db8dac20 529
607c4aaf
KK		 530static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
531 __be16 sport, __be16 dport,
645ca708 532 struct udp_table *udptable)
607c4aaf
KK		 533{
534 const struct iphdr *iph = ip_hdr(skb);
535
ed7cbbce 536 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
8afdd99a 537 iph->daddr, dport, inet_iif(skb),
fb74c277 538 inet_sdif(skb), udptable, skb);
607c4aaf
KK		 539}
540
63058308
TH		 541struct sock *udp4_lib_lookup_skb(struct sk_buff *skb,
542 __be16 sport, __be16 dport)
543{
ed7cbbce 544 return __udp4_lib_lookup_skb(skb, sport, dport, &udp_table);
63058308
TH		 545}
546EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb);
547
ca065d0c
ED		 548/* Must be called under rcu_read_lock().
549 * Does increment socket refcount.
550 */
6e86000c 551#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
bcd41303
KK		 552struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
553 __be32 daddr, __be16 dport, int dif)
554{
ca065d0c
ED		 555 struct sock *sk;
556
557 sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
fb74c277 558 dif, 0, &udp_table, NULL);
41c6d650 559 if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
ca065d0c
ED		 560 sk = NULL;
561 return sk;
bcd41303
KK		 562}
563EXPORT_SYMBOL_GPL(udp4_lib_lookup);
ca065d0c 564#endif
bcd41303 565
421b3885
SB		 566static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
567 __be16 loc_port, __be32 loc_addr,
568 __be16 rmt_port, __be32 rmt_addr,
fb74c277 569 int dif, int sdif, unsigned short hnum)
421b3885
SB		 570{
571 struct inet_sock *inet = inet_sk(sk);
572
573 if (!net_eq(sock_net(sk), net) ||
574 udp_sk(sk)->udp_port_hash != hnum ||
575 (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
576 (inet->inet_dport != rmt_port && inet->inet_dport) ||
577 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
578 ipv6_only_sock(sk) ||
fb74c277
DA		 579 (sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif &&
580 sk->sk_bound_dev_if != sdif))
421b3885 581 return false;
60d9b031 582 if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
421b3885
SB		 583 return false;
584 return true;
585}
586
a36e185e
SB		 587DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
588void udp_encap_enable(void)
589{
590 static_branch_enable(&udp_encap_needed_key);
591}
592EXPORT_SYMBOL(udp_encap_enable);
593
e7cc0824
SB		 594/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
595 * through error handlers in encapsulations looking for a match.
596 */
597static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
598{
599 int i;
600
601 for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
602 int (*handler)(struct sk_buff *skb, u32 info);
603
604 if (!iptun_encaps[i])
605 continue;
606 handler = rcu_dereference(iptun_encaps[i]->err_handler);
607 if (handler && !handler(skb, info))
608 return 0;
609 }
610
611 return -ENOENT;
612}
613
a36e185e
SB		 614/* Try to match ICMP errors to UDP tunnels by looking up a socket without
615 * reversing source and destination port: this will match tunnels that force the
616 * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
617 * lwtunnels might actually break this assumption by being configured with
618 * different destination ports on endpoints, in this case we won't be able to
619 * trace ICMP messages back to them.
620 *
e7cc0824
SB		 621 * If this doesn't match any socket, probe tunnels with arbitrary destination
622 * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
623 * we've sent packets to won't necessarily match the local destination port.
624 *
a36e185e
SB		 625 * Then ask the tunnel implementation to match the error against a valid
626 * association.
627 *
e7cc0824
SB		 628 * Return an error if we can't find a match, the socket if we need further
629 * processing, zero otherwise.
a36e185e
SB		 630 */
631static struct sock *__udp4_lib_err_encap(struct net *net,
632 const struct iphdr *iph,
633 struct udphdr *uh,
634 struct udp_table *udptable,
e7cc0824 635 struct sk_buff *skb, u32 info)
a36e185e 636{
a36e185e 637 int network_offset, transport_offset;
a36e185e
SB		 638 struct sock *sk;
639
a36e185e
SB		 640 network_offset = skb_network_offset(skb);
641 transport_offset = skb_transport_offset(skb);
642
643 /* Network header needs to point to the outer IPv4 header inside ICMP */
644 skb_reset_network_header(skb);
645
646 /* Transport header needs to point to the UDP header */
647 skb_set_transport_header(skb, iph->ihl << 2);
648
e7cc0824
SB		 649 sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
650 iph->saddr, uh->dest, skb->dev->ifindex, 0,
651 udptable, NULL);
652 if (sk) {
653 int (*lookup)(struct sock *sk, struct sk_buff *skb);
654 struct udp_sock *up = udp_sk(sk);
655
656 lookup = READ_ONCE(up->encap_err_lookup);
657 if (!lookup || lookup(sk, skb))
658 sk = NULL;
659 }
660
661 if (!sk)
662 sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
a36e185e
SB		 663
664 skb_set_transport_header(skb, transport_offset);
665 skb_set_network_header(skb, network_offset);
666
667 return sk;
668}
669
db8dac20
DM		 670/*
671 * This routine is called by the ICMP module when it gets some
672 * sort of error condition. If err < 0 then the socket should
673 * be closed and the error returned to the user. If err > 0
674 * it's just the icmp type << 8 | icmp code.
675 * Header points to the ip header of the error packet. We move
676 * on past this. Then (as it used to claim before adjustment)
677 * header points to the first 8 bytes of the udp header. We need
678 * to find the appropriate port.
679 */
680
32bbd879 681int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
db8dac20
DM		 682{
683 struct inet_sock *inet;
b71d1d42 684 const struct iphdr *iph = (const struct iphdr *)skb->data;
c482c568 685 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
db8dac20
DM		 686 const int type = icmp_hdr(skb)->type;
687 const int code = icmp_hdr(skb)->code;
a36e185e 688 bool tunnel = false;
db8dac20
DM		 689 struct sock *sk;
690 int harderr;
691 int err;
fd54d716 692 struct net *net = dev_net(skb->dev);
db8dac20 693
fd54d716 694 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
f64bf6b8
MM		 695 iph->saddr, uh->source, skb->dev->ifindex,
696 inet_sdif(skb), udptable, NULL);
51456b29 697 if (!sk) {
a36e185e 698 /* No socket for error: try tunnels before discarding */
e7cc0824
SB		 699 sk = ERR_PTR(-ENOENT);
700 if (static_branch_unlikely(&udp_encap_needed_key)) {
701 sk = __udp4_lib_err_encap(net, iph, uh, udptable, skb,
702 info);
703 if (!sk)
704 return 0;
705 }
a36e185e 706
e7cc0824 707 if (IS_ERR(sk)) {
a36e185e 708 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
e7cc0824 709 return PTR_ERR(sk);
a36e185e 710 }
e7cc0824 711
a36e185e 712 tunnel = true;
db8dac20
DM		 713 }
714
715 err = 0;
716 harderr = 0;
717 inet = inet_sk(sk);
718
719 switch (type) {
720 default:
721 case ICMP_TIME_EXCEEDED:
722 err = EHOSTUNREACH;
723 break;
724 case ICMP_SOURCE_QUENCH:
725 goto out;
726 case ICMP_PARAMETERPROB:
727 err = EPROTO;
728 harderr = 1;
729 break;
730 case ICMP_DEST_UNREACH:
731 if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
36393395 732 ipv4_sk_update_pmtu(skb, sk, info);
db8dac20
DM		 733 if (inet->pmtudisc != IP_PMTUDISC_DONT) {
734 err = EMSGSIZE;
735 harderr = 1;
736 break;
737 }
738 goto out;
739 }
740 err = EHOSTUNREACH;
741 if (code <= NR_ICMP_UNREACH) {
742 harderr = icmp_err_convert[code].fatal;
743 err = icmp_err_convert[code].errno;
744 }
745 break;
55be7a9c
DM		 746 case ICMP_REDIRECT:
747 ipv4_sk_redirect(skb, sk);
1a462d18 748 goto out;
db8dac20
DM		 749 }
750
751 /*
752 * RFC1122: OK. Passes ICMP errors back to application, as per
753 * 4.1.3.3.
754 */
a36e185e
SB		 755 if (tunnel) {
756 /* ...not for tunnels though: we don't have a sending socket */
757 goto out;
758 }
db8dac20
DM		 759 if (!inet->recverr) {
760 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
761 goto out;
b1faf566 762 } else
c482c568 763 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
b1faf566 764
db8dac20
DM		 765 sk->sk_err = err;
766 sk->sk_error_report(sk);
767out:
32bbd879 768 return 0;
db8dac20
DM		 769}
770
32bbd879 771int udp_err(struct sk_buff *skb, u32 info)
db8dac20 772{
32bbd879 773 return __udp4_lib_err(skb, info, &udp_table);
db8dac20
DM		 774}
775
776/*
777 * Throw away all pending data and cancel the corking. Socket is locked.
778 */
36d926b9 779void udp_flush_pending_frames(struct sock *sk)
db8dac20
DM		 780{
781 struct udp_sock *up = udp_sk(sk);
782
783 if (up->pending) {
784 up->len = 0;
785 up->pending = 0;
786 ip_flush_pending_frames(sk);
787 }
788}
36d926b9 789EXPORT_SYMBOL(udp_flush_pending_frames);
db8dac20
DM		 790
791/**
f6b9664f 792 * udp4_hwcsum - handle outgoing HW checksumming
db8dac20
DM		 793 * @skb: sk_buff containing the filled-in UDP header
794 * (checksum field must be zeroed out)
f6b9664f
HX		 795 * @src: source IP address
796 * @dst: destination IP address
db8dac20 797 */
c26bf4a5 798void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
db8dac20 799{
db8dac20 800 struct udphdr *uh = udp_hdr(skb);
f6b9664f
HX		 801 int offset = skb_transport_offset(skb);
802 int len = skb->len - offset;
803 int hlen = len;
db8dac20
DM		 804 __wsum csum = 0;
805
ebbe495f 806 if (!skb_has_frag_list(skb)) {
db8dac20
DM		 807 /*
808 * Only one fragment on the socket.
809 */
810 skb->csum_start = skb_transport_header(skb) - skb->head;
811 skb->csum_offset = offsetof(struct udphdr, check);
f6b9664f
HX		 812 uh->check = ~csum_tcpudp_magic(src, dst, len,
813 IPPROTO_UDP, 0);
db8dac20 814 } else {
ebbe495f
WC		 815 struct sk_buff *frags;
816
db8dac20
DM		 817 /*
818 * HW-checksum won't work as there are two or more
819 * fragments on the socket so that all csums of sk_buffs
820 * should be together
821 */
ebbe495f 822 skb_walk_frags(skb, frags) {
f6b9664f
HX		 823 csum = csum_add(csum, frags->csum);
824 hlen -= frags->len;
ebbe495f 825 }
db8dac20 826
f6b9664f 827 csum = skb_checksum(skb, offset, hlen, csum);
db8dac20
DM		 828 skb->ip_summed = CHECKSUM_NONE;
829
db8dac20
DM		 830 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
831 if (uh->check == 0)
832 uh->check = CSUM_MANGLED_0;
833 }
834}
c26bf4a5 835EXPORT_SYMBOL_GPL(udp4_hwcsum);
db8dac20 836
af5fcba7
TH		 837/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
838 * for the simple case like when setting the checksum for a UDP tunnel.
839 */
840void udp_set_csum(bool nocheck, struct sk_buff *skb,
841 __be32 saddr, __be32 daddr, int len)
842{
843 struct udphdr *uh = udp_hdr(skb);
844
179bc67f 845 if (nocheck) {
af5fcba7 846 uh->check = 0;
179bc67f 847 } else if (skb_is_gso(skb)) {
af5fcba7 848 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
179bc67f
EC		 849 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
850 uh->check = 0;
851 uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
852 if (uh->check == 0)
853 uh->check = CSUM_MANGLED_0;
d75f1306 854 } else {
af5fcba7
TH		 855 skb->ip_summed = CHECKSUM_PARTIAL;
856 skb->csum_start = skb_transport_header(skb) - skb->head;
857 skb->csum_offset = offsetof(struct udphdr, check);
858 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
af5fcba7
TH		 859 }
860}
861EXPORT_SYMBOL(udp_set_csum);
862
bec1f6f6
WB		 863static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
864 struct inet_cork *cork)
db8dac20 865{
f6b9664f 866 struct sock *sk = skb->sk;
db8dac20 867 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM		 868 struct udphdr *uh;
869 int err = 0;
870 int is_udplite = IS_UDPLITE(sk);
f6b9664f
HX		 871 int offset = skb_transport_offset(skb);
872 int len = skb->len - offset;
db8dac20
DM		 873 __wsum csum = 0;
874
db8dac20
DM		 875 /*
876 * Create a UDP header
877 */
878 uh = udp_hdr(skb);
f6b9664f 879 uh->source = inet->inet_sport;
79ab0531 880 uh->dest = fl4->fl4_dport;
f6b9664f 881 uh->len = htons(len);
db8dac20
DM		 882 uh->check = 0;
883
bec1f6f6
WB		 884 if (cork->gso_size) {
885 const int hlen = skb_network_header_len(skb) +
886 sizeof(struct udphdr);
887
888 if (hlen + cork->gso_size > cork->fragsize)
889 return -EINVAL;
890 if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS)
891 return -EINVAL;
a8c744a8
WB		 892 if (sk->sk_no_check_tx)
893 return -EINVAL;
ff06342c
WB		 894 if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
895 dst_xfrm(skb_dst(skb)))
bec1f6f6
WB		 896 return -EIO;
897
898 skb_shinfo(skb)->gso_size = cork->gso_size;
899 skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
dfec0ee2
AD		 900 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(len - sizeof(uh),
901 cork->gso_size);
a8c744a8 902 goto csum_partial;
bec1f6f6
WB		 903 }
904
db8dac20 905 if (is_udplite) /* UDP-Lite */
f6b9664f 906 csum = udplite_csum(skb);
db8dac20 907
ab2fb7e3 908 else if (sk->sk_no_check_tx) { /* UDP csum off */
db8dac20
DM		 909
910 skb->ip_summed = CHECKSUM_NONE;
911 goto send;
912
913 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
a8c744a8 914csum_partial:
db8dac20 915
79ab0531 916 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
db8dac20
DM		 917 goto send;
918
f6b9664f
HX		 919 } else
920 csum = udp_csum(skb);
db8dac20
DM		 921
922 /* add protocol-dependent pseudo-header */
79ab0531 923 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
c482c568 924 sk->sk_protocol, csum);
db8dac20
DM		 925 if (uh->check == 0)
926 uh->check = CSUM_MANGLED_0;
927
928send:
b5ec8eea 929 err = ip_send_skb(sock_net(sk), skb);
6ce9e7b5
ED		 930 if (err) {
931 if (err == -ENOBUFS && !inet->recverr) {
6aef70a8
ED		 932 UDP_INC_STATS(sock_net(sk),
933 UDP_MIB_SNDBUFERRORS, is_udplite);
6ce9e7b5
ED		 934 err = 0;
935 }
936 } else
6aef70a8
ED		 937 UDP_INC_STATS(sock_net(sk),
938 UDP_MIB_OUTDATAGRAMS, is_udplite);
f6b9664f
HX		 939 return err;
940}
941
942/*
943 * Push out all pending data as one UDP datagram. Socket is locked.
944 */
8822b64a 945int udp_push_pending_frames(struct sock *sk)
f6b9664f
HX		 946{
947 struct udp_sock *up = udp_sk(sk);
948 struct inet_sock *inet = inet_sk(sk);
b6f21b26 949 struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
f6b9664f
HX		 950 struct sk_buff *skb;
951 int err = 0;
952
77968b78 953 skb = ip_finish_skb(sk, fl4);
f6b9664f
HX		 954 if (!skb)
955 goto out;
956
bec1f6f6 957 err = udp_send_skb(skb, fl4, &inet->cork.base);
f6b9664f 958
db8dac20
DM		 959out:
960 up->len = 0;
961 up->pending = 0;
db8dac20
DM		 962 return err;
963}
8822b64a 964EXPORT_SYMBOL(udp_push_pending_frames);
db8dac20 965
2e8de857
WB		 966static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
967{
968 switch (cmsg->cmsg_type) {
969 case UDP_SEGMENT:
970 if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
971 return -EINVAL;
972 *gso_size = *(__u16 *)CMSG_DATA(cmsg);
973 return 0;
974 default:
975 return -EINVAL;
976 }
977}
978
979int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
980{
981 struct cmsghdr *cmsg;
982 bool need_ip = false;
983 int err;
984
985 for_each_cmsghdr(cmsg, msg) {
986 if (!CMSG_OK(msg, cmsg))
987 return -EINVAL;
988
989 if (cmsg->cmsg_level != SOL_UDP) {
990 need_ip = true;
991 continue;
992 }
993
994 err = __udp_cmsg_send(cmsg, gso_size);
995 if (err)
996 return err;
997 }
998
999 return need_ip;
1000}
1001EXPORT_SYMBOL_GPL(udp_cmsg_send);
1002
1b784140 1003int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
db8dac20
DM		 1004{
1005 struct inet_sock *inet = inet_sk(sk);
1006 struct udp_sock *up = udp_sk(sk);
1cedee13 1007 DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
e474995f 1008 struct flowi4 fl4_stack;
b6f21b26 1009 struct flowi4 *fl4;
db8dac20
DM		 1010 int ulen = len;
1011 struct ipcm_cookie ipc;
1012 struct rtable *rt = NULL;
1013 int free = 0;
1014 int connected = 0;
1015 __be32 daddr, faddr, saddr;
1016 __be16 dport;
1017 u8 tos;
1018 int err, is_udplite = IS_UDPLITE(sk);
1019 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
1020 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
903ab86d 1021 struct sk_buff *skb;
f6d8bd05 1022 struct ip_options_data opt_copy;
db8dac20
DM		 1023
1024 if (len > 0xFFFF)
1025 return -EMSGSIZE;
1026
1027 /*
1028 * Check the flags.
1029 */
1030
c482c568 1031 if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
db8dac20
DM		 1032 return -EOPNOTSUPP;
1033
903ab86d
HX		 1034 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
1035
f5fca608 1036 fl4 = &inet->cork.fl.u.ip4;
db8dac20
DM		 1037 if (up->pending) {
1038 /*
1039 * There are pending frames.
1040 * The socket lock must be held while it's corked.
1041 */
1042 lock_sock(sk);
1043 if (likely(up->pending)) {
1044 if (unlikely(up->pending != AF_INET)) {
1045 release_sock(sk);
1046 return -EINVAL;
1047 }
1048 goto do_append_data;
1049 }
1050 release_sock(sk);
1051 }
1052 ulen += sizeof(struct udphdr);
1053
1054 /*
1055 * Get and verify the address.
1056 */
1cedee13 1057 if (usin) {
db8dac20
DM		 1058 if (msg->msg_namelen < sizeof(*usin))
1059 return -EINVAL;
1060 if (usin->sin_family != AF_INET) {
1061 if (usin->sin_family != AF_UNSPEC)
1062 return -EAFNOSUPPORT;
1063 }
1064
1065 daddr = usin->sin_addr.s_addr;
1066 dport = usin->sin_port;
1067 if (dport == 0)
1068 return -EINVAL;
1069 } else {
1070 if (sk->sk_state != TCP_ESTABLISHED)
1071 return -EDESTADDRREQ;
c720c7e8
ED		 1072 daddr = inet->inet_daddr;
1073 dport = inet->inet_dport;
db8dac20
DM		 1074 /* Open fast path for connected socket.
1075 Route will not be used, if at least one option is set.
1076 */
1077 connected = 1;
1078 }
db8dac20 1079
35178206 1080 ipcm_init_sk(&ipc, inet);
bec1f6f6 1081 ipc.gso_size = up->gso_size;
bf84a010 1082
db8dac20 1083 if (msg->msg_controllen) {
2e8de857
WB		 1084 err = udp_cmsg_send(sk, msg, &ipc.gso_size);
1085 if (err > 0)
1086 err = ip_cmsg_send(sk, msg, &ipc,
1087 sk->sk_family == AF_INET6);
1088 if (unlikely(err < 0)) {
91948309 1089 kfree(ipc.opt);
db8dac20 1090 return err;
91948309 1091 }
db8dac20
DM		 1092 if (ipc.opt)
1093 free = 1;
1094 connected = 0;
1095 }
f6d8bd05
ED		 1096 if (!ipc.opt) {
1097 struct ip_options_rcu *inet_opt;
1098
1099 rcu_read_lock();
1100 inet_opt = rcu_dereference(inet->inet_opt);
1101 if (inet_opt) {
1102 memcpy(&opt_copy, inet_opt,
1103 sizeof(*inet_opt) + inet_opt->opt.optlen);
1104 ipc.opt = &opt_copy.opt;
1105 }
1106 rcu_read_unlock();
1107 }
db8dac20 1108
1cedee13
AI		 1109 if (cgroup_bpf_enabled && !connected) {
1110 err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
1111 (struct sockaddr *)usin, &ipc.addr);
1112 if (err)
1113 goto out_free;
1114 if (usin) {
1115 if (usin->sin_port == 0) {
1116 /* BPF program set invalid port. Reject it. */
1117 err = -EINVAL;
1118 goto out_free;
1119 }
1120 daddr = usin->sin_addr.s_addr;
1121 dport = usin->sin_port;
1122 }
1123 }
1124
db8dac20
DM		 1125 saddr = ipc.addr;
1126 ipc.addr = faddr = daddr;
1127
f6d8bd05 1128 if (ipc.opt && ipc.opt->opt.srr) {
1b97013b
AI		 1129 if (!daddr) {
1130 err = -EINVAL;
1131 goto out_free;
1132 }
f6d8bd05 1133 faddr = ipc.opt->opt.faddr;
db8dac20
DM		 1134 connected = 0;
1135 }
aa661581 1136 tos = get_rttos(&ipc, inet);
db8dac20
DM		 1137 if (sock_flag(sk, SOCK_LOCALROUTE) ||
1138 (msg->msg_flags & MSG_DONTROUTE) ||
f6d8bd05 1139 (ipc.opt && ipc.opt->opt.is_strictroute)) {
db8dac20
DM		 1140 tos |= RTO_ONLINK;
1141 connected = 0;
1142 }
1143
1144 if (ipv4_is_multicast(daddr)) {
854da991 1145 if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
db8dac20
DM		 1146 ipc.oif = inet->mc_index;
1147 if (!saddr)
1148 saddr = inet->mc_addr;
1149 connected = 0;
9515a2e0 1150 } else if (!ipc.oif) {
76e21053 1151 ipc.oif = inet->uc_index;
9515a2e0
DA		 1152 } else if (ipv4_is_lbcast(daddr) && inet->uc_index) {
1153 /* oif is set, packet is to local broadcast and
1154 * and uc_index is set. oif is most likely set
1155 * by sk_bound_dev_if. If uc_index != oif check if the
1156 * oif is an L3 master and uc_index is an L3 slave.
1157 * If so, we want to allow the send using the uc_index.
1158 */
1159 if (ipc.oif != inet->uc_index &&
1160 ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
1161 inet->uc_index)) {
1162 ipc.oif = inet->uc_index;
1163 }
1164 }
db8dac20
DM		 1165
1166 if (connected)
c482c568 1167 rt = (struct rtable *)sk_dst_check(sk, 0);
db8dac20 1168
51456b29 1169 if (!rt) {
84a3aa00 1170 struct net *net = sock_net(sk);
9a24abfa 1171 __u8 flow_flags = inet_sk_flowi_flags(sk);
84a3aa00 1172
e474995f 1173 fl4 = &fl4_stack;
9a24abfa 1174
e474995f 1175 flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos,
c0951cbc 1176 RT_SCOPE_UNIVERSE, sk->sk_protocol,
9a24abfa 1177 flow_flags,
e2d118a1
LC		 1178 faddr, saddr, dport, inet->inet_sport,
1179 sk->sk_uid);
c0951cbc 1180
e474995f
DM		 1181 security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
1182 rt = ip_route_output_flow(net, fl4, sk);
b23dd4fe
DM		 1183 if (IS_ERR(rt)) {
1184 err = PTR_ERR(rt);
06dc94b1 1185 rt = NULL;
db8dac20 1186 if (err == -ENETUNREACH)
f1d8cba6 1187 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
db8dac20
DM		 1188 goto out;
1189 }
1190
1191 err = -EACCES;
1192 if ((rt->rt_flags & RTCF_BROADCAST) &&
1193 !sock_flag(sk, SOCK_BROADCAST))
1194 goto out;
1195 if (connected)
d8d1f30b 1196 sk_dst_set(sk, dst_clone(&rt->dst));
db8dac20
DM		 1197 }
1198
1199 if (msg->msg_flags&MSG_CONFIRM)
1200 goto do_confirm;
1201back_from_confirm:
1202
e474995f 1203 saddr = fl4->saddr;
db8dac20 1204 if (!ipc.addr)
e474995f 1205 daddr = ipc.addr = fl4->daddr;
db8dac20 1206
903ab86d
HX		 1207 /* Lockless fast path for the non-corking case. */
1208 if (!corkreq) {
1cd7884d
WB		 1209 struct inet_cork cork;
1210
f69e6d13 1211 skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
903ab86d 1212 sizeof(struct udphdr), &ipc, &rt,
1cd7884d 1213 &cork, msg->msg_flags);
903ab86d 1214 err = PTR_ERR(skb);
50c3a487 1215 if (!IS_ERR_OR_NULL(skb))
bec1f6f6 1216 err = udp_send_skb(skb, fl4, &cork);
903ab86d
HX		 1217 goto out;
1218 }
1219
db8dac20
DM		 1220 lock_sock(sk);
1221 if (unlikely(up->pending)) {
1222 /* The socket is already corked while preparing it. */
1223 /* ... which is an evident application bug. --ANK */
1224 release_sock(sk);
1225
197df02c 1226 net_dbg_ratelimited("socket already corked\n");
db8dac20
DM		 1227 err = -EINVAL;
1228 goto out;
1229 }
1230 /*
1231 * Now cork the socket to pend data.
1232 */
b6f21b26
DM		 1233 fl4 = &inet->cork.fl.u.ip4;
1234 fl4->daddr = daddr;
1235 fl4->saddr = saddr;
9cce96df
DM		 1236 fl4->fl4_dport = dport;
1237 fl4->fl4_sport = inet->inet_sport;
db8dac20
DM		 1238 up->pending = AF_INET;
1239
1240do_append_data:
1241 up->len += ulen;
f69e6d13 1242 err = ip_append_data(sk, fl4, getfrag, msg, ulen,
f5fca608
DM		 1243 sizeof(struct udphdr), &ipc, &rt,
1244 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
db8dac20
DM		 1245 if (err)
1246 udp_flush_pending_frames(sk);
1247 else if (!corkreq)
1248 err = udp_push_pending_frames(sk);
1249 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1250 up->pending = 0;
1251 release_sock(sk);
1252
1253out:
1254 ip_rt_put(rt);
1b97013b 1255out_free:
db8dac20
DM		 1256 if (free)
1257 kfree(ipc.opt);
1258 if (!err)
1259 return len;
1260 /*
1261 * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
1262 * ENOBUFS might not be good (it's not tunable per se), but otherwise
1263 * we don't have a good statistic (IpOutDiscards but it can be too many
1264 * things). We could add another new stat but at least for now that
1265 * seems like overkill.
1266 */
1267 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
6aef70a8
ED		 1268 UDP_INC_STATS(sock_net(sk),
1269 UDP_MIB_SNDBUFERRORS, is_udplite);
db8dac20
DM		 1270 }
1271 return err;
1272
1273do_confirm:
0dec879f
JA		 1274 if (msg->msg_flags & MSG_PROBE)
1275 dst_confirm_neigh(&rt->dst, &fl4->daddr);
db8dac20
DM		 1276 if (!(msg->msg_flags&MSG_PROBE) || len)
1277 goto back_from_confirm;
1278 err = 0;
1279 goto out;
1280}
c482c568 1281EXPORT_SYMBOL(udp_sendmsg);
db8dac20
DM		 1282
1283int udp_sendpage(struct sock *sk, struct page *page, int offset,
1284 size_t size, int flags)
1285{
f5fca608 1286 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM		 1287 struct udp_sock *up = udp_sk(sk);
1288 int ret;
1289
d3f7d56a
SL		 1290 if (flags & MSG_SENDPAGE_NOTLAST)
1291 flags |= MSG_MORE;
1292
db8dac20
DM		 1293 if (!up->pending) {
1294 struct msghdr msg = { .msg_flags = flags|MSG_MORE };
1295
1296 /* Call udp_sendmsg to specify destination address which
1297 * sendpage interface can't pass.
1298 * This will succeed only when the socket is connected.
1299 */
1b784140 1300 ret = udp_sendmsg(sk, &msg, 0);
db8dac20
DM		 1301 if (ret < 0)
1302 return ret;
1303 }
1304
1305 lock_sock(sk);
1306
1307 if (unlikely(!up->pending)) {
1308 release_sock(sk);
1309
197df02c 1310 net_dbg_ratelimited("cork failed\n");
db8dac20
DM		 1311 return -EINVAL;
1312 }
1313
f5fca608
DM		 1314 ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
1315 page, offset, size, flags);
db8dac20
DM		 1316 if (ret == -EOPNOTSUPP) {
1317 release_sock(sk);
1318 return sock_no_sendpage(sk->sk_socket, page, offset,
1319 size, flags);
1320 }
1321 if (ret < 0) {
1322 udp_flush_pending_frames(sk);
1323 goto out;
1324 }
1325
1326 up->len += size;
1327 if (!(up->corkflag || (flags&MSG_MORE)))
1328 ret = udp_push_pending_frames(sk);
1329 if (!ret)
1330 ret = size;
1331out:
1332 release_sock(sk);
1333 return ret;
1334}
1335
dce4551c
PA		 1336#define UDP_SKB_IS_STATELESS 0x80000000
1337
b65ac446
PA		 1338static void udp_set_dev_scratch(struct sk_buff *skb)
1339{
dce4551c 1340 struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
b65ac446
PA		 1341
1342 BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
dce4551c
PA		 1343 scratch->_tsize_state = skb->truesize;
1344#if BITS_PER_LONG == 64
b65ac446
PA		 1345 scratch->len = skb->len;
1346 scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1347 scratch->is_linear = !skb_is_nonlinear(skb);
dce4551c 1348#endif
3bdefdf9
PA		 1349 /* all head states execept sp (dst, sk, nf) are always cleared by
1350 * udp_rcv() and we need to preserve secpath, if present, to eventually
1351 * process IP_CMSG_PASSSEC at recvmsg() time
1352 */
1353 if (likely(!skb_sec_path(skb)))
dce4551c 1354 scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
b65ac446
PA		 1355}
1356
1357static int udp_skb_truesize(struct sk_buff *skb)
1358{
dce4551c 1359 return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
b65ac446
PA		 1360}
1361
dce4551c 1362static bool udp_skb_has_head_state(struct sk_buff *skb)
b65ac446 1363{
dce4551c 1364 return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
b65ac446 1365}
b65ac446 1366
7c13f97f 1367/* fully reclaim rmem/fwd memory allocated for skb */
6dfb4367
PA		 1368static void udp_rmem_release(struct sock *sk, int size, int partial,
1369 bool rx_queue_lock_held)
f970bd9e 1370{
6b229cf7 1371 struct udp_sock *up = udp_sk(sk);
2276f58a 1372 struct sk_buff_head *sk_queue;
f970bd9e
PA		 1373 int amt;
1374
6b229cf7
ED		 1375 if (likely(partial)) {
1376 up->forward_deficit += size;
1377 size = up->forward_deficit;
0d4a6608 1378 if (size < (sk->sk_rcvbuf >> 2))
6b229cf7
ED		 1379 return;
1380 } else {
1381 size += up->forward_deficit;
1382 }
1383 up->forward_deficit = 0;
1384
6dfb4367
PA		 1385 /* acquire the sk_receive_queue for fwd allocated memory scheduling,
1386 * if the called don't held it already
1387 */
2276f58a 1388 sk_queue = &sk->sk_receive_queue;
6dfb4367
PA		 1389 if (!rx_queue_lock_held)
1390 spin_lock(&sk_queue->lock);
1391
2276f58a 1392
f970bd9e
PA		 1393 sk->sk_forward_alloc += size;
1394 amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1);
1395 sk->sk_forward_alloc -= amt;
f970bd9e
PA		 1396
1397 if (amt)
1398 __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT);
02ab0d13
ED		 1399
1400 atomic_sub(size, &sk->sk_rmem_alloc);
2276f58a
PA		 1401
1402 /* this can save us from acquiring the rx queue lock on next receive */
1403 skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
1404
6dfb4367
PA		 1405 if (!rx_queue_lock_held)
1406 spin_unlock(&sk_queue->lock);
f970bd9e
PA		 1407}
1408
2276f58a 1409/* Note: called with reader_queue.lock held.
c84d9490
ED		 1410 * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
1411 * This avoids a cache line miss while receive_queue lock is held.
1412 * Look at __udp_enqueue_schedule_skb() to find where this copy is done.
1413 */
7c13f97f 1414void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
f970bd9e 1415{
b65ac446
PA		 1416 prefetch(&skb->data);
1417 udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
f970bd9e 1418}
7c13f97f 1419EXPORT_SYMBOL(udp_skb_destructor);
f970bd9e 1420
6dfb4367 1421/* as above, but the caller held the rx queue lock, too */
64f5102d 1422static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
6dfb4367 1423{
b65ac446
PA		 1424 prefetch(&skb->data);
1425 udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
6dfb4367
PA		 1426}
1427
4b272750
ED		 1428/* Idea of busylocks is to let producers grab an extra spinlock
1429 * to relieve pressure on the receive_queue spinlock shared by consumer.
1430 * Under flood, this means that only one producer can be in line
1431 * trying to acquire the receive_queue spinlock.
1432 * These busylock can be allocated on a per cpu manner, instead of a
1433 * per socket one (that would consume a cache line per socket)
1434 */
1435static int udp_busylocks_log __read_mostly;
1436static spinlock_t *udp_busylocks __read_mostly;
1437
1438static spinlock_t *busylock_acquire(void *ptr)
1439{
1440 spinlock_t *busy;
1441
1442 busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log);
1443 spin_lock(busy);
1444 return busy;
1445}
1446
1447static void busylock_release(spinlock_t *busy)
1448{
1449 if (busy)
1450 spin_unlock(busy);
1451}
1452
f970bd9e
PA		 1453int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
1454{
1455 struct sk_buff_head *list = &sk->sk_receive_queue;
1456 int rmem, delta, amt, err = -ENOMEM;
4b272750 1457 spinlock_t *busy = NULL;
c8c8b127 1458 int size;
f970bd9e
PA		 1459
1460 /* try to avoid the costly atomic add/sub pair when the receive
1461 * queue is full; always allow at least a packet
1462 */
1463 rmem = atomic_read(&sk->sk_rmem_alloc);
363dc73a 1464 if (rmem > sk->sk_rcvbuf)
f970bd9e
PA		 1465 goto drop;
1466
c8c8b127
ED		 1467 /* Under mem pressure, it might be helpful to help udp_recvmsg()
1468 * having linear skbs :
1469 * - Reduce memory overhead and thus increase receive queue capacity
1470 * - Less cache line misses at copyout() time
1471 * - Less work at consume_skb() (less alien page frag freeing)
1472 */
4b272750 1473 if (rmem > (sk->sk_rcvbuf >> 1)) {
c8c8b127 1474 skb_condense(skb);
4b272750
ED		 1475
1476 busy = busylock_acquire(sk);
1477 }
c8c8b127 1478 size = skb->truesize;
b65ac446 1479 udp_set_dev_scratch(skb);
c8c8b127 1480
f970bd9e
PA		 1481 /* we drop only if the receive buf is full and the receive
1482 * queue contains some other skb
1483 */
1484 rmem = atomic_add_return(size, &sk->sk_rmem_alloc);
363dc73a 1485 if (rmem > (size + sk->sk_rcvbuf))
f970bd9e
PA		 1486 goto uncharge_drop;
1487
1488 spin_lock(&list->lock);
1489 if (size >= sk->sk_forward_alloc) {
1490 amt = sk_mem_pages(size);
1491 delta = amt << SK_MEM_QUANTUM_SHIFT;
1492 if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) {
1493 err = -ENOBUFS;
1494 spin_unlock(&list->lock);
1495 goto uncharge_drop;
1496 }
1497
1498 sk->sk_forward_alloc += delta;
1499 }
1500
1501 sk->sk_forward_alloc -= size;
1502
7c13f97f
PA		 1503 /* no need to setup a destructor, we will explicitly release the
1504 * forward allocated memory on dequeue
1505 */
f970bd9e
PA		 1506 sock_skb_set_dropcount(sk, skb);
1507
1508 __skb_queue_tail(list, skb);
1509 spin_unlock(&list->lock);
1510
1511 if (!sock_flag(sk, SOCK_DEAD))
1512 sk->sk_data_ready(sk);
1513
4b272750 1514 busylock_release(busy);
f970bd9e
PA		 1515 return 0;
1516
1517uncharge_drop:
1518 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
1519
1520drop:
1521 atomic_inc(&sk->sk_drops);
4b272750 1522 busylock_release(busy);
f970bd9e
PA		 1523 return err;
1524}
1525EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
1526
c915fe13 1527void udp_destruct_sock(struct sock *sk)
f970bd9e
PA		 1528{
1529 /* reclaim completely the forward allocated memory */
2276f58a 1530 struct udp_sock *up = udp_sk(sk);
7c13f97f
PA		 1531 unsigned int total = 0;
1532 struct sk_buff *skb;
1533
2276f58a
PA		 1534 skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
1535 while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
7c13f97f
PA		 1536 total += skb->truesize;
1537 kfree_skb(skb);
1538 }
6dfb4367 1539 udp_rmem_release(sk, total, 0, true);
7c13f97f 1540
f970bd9e
PA		 1541 inet_sock_destruct(sk);
1542}
c915fe13 1543EXPORT_SYMBOL_GPL(udp_destruct_sock);
f970bd9e
PA		 1544
1545int udp_init_sock(struct sock *sk)
1546{
2276f58a 1547 skb_queue_head_init(&udp_sk(sk)->reader_queue);
f970bd9e
PA		 1548 sk->sk_destruct = udp_destruct_sock;
1549 return 0;
1550}
1551EXPORT_SYMBOL_GPL(udp_init_sock);
1552
1553void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
1554{
1555 if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
1556 bool slow = lock_sock_fast(sk);
1557
1558 sk_peek_offset_bwd(sk, len);
1559 unlock_sock_fast(sk, slow);
1560 }
0a463c78 1561
ca2c1418
PA		 1562 if (!skb_unref(skb))
1563 return;
1564
dce4551c
PA		 1565 /* In the more common cases we cleared the head states previously,
1566 * see __udp_queue_rcv_skb().
0ddf3fb2 1567 */
dce4551c 1568 if (unlikely(udp_skb_has_head_state(skb)))
0ddf3fb2 1569 skb_release_head_state(skb);
ca2c1418 1570 __consume_stateless_skb(skb);
f970bd9e
PA		 1571}
1572EXPORT_SYMBOL_GPL(skb_consume_udp);
1573
2276f58a
PA		 1574static struct sk_buff *__first_packet_length(struct sock *sk,
1575 struct sk_buff_head *rcvq,
1576 int *total)
1577{
1578 struct sk_buff *skb;
1579
9bd780f5
PA		 1580 while ((skb = skb_peek(rcvq)) != NULL) {
1581 if (udp_lib_checksum_complete(skb)) {
1582 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
1583 IS_UDPLITE(sk));
1584 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
1585 IS_UDPLITE(sk));
1586 atomic_inc(&sk->sk_drops);
1587 __skb_unlink(skb, rcvq);
1588 *total += skb->truesize;
1589 kfree_skb(skb);
1590 } else {
1591 /* the csum related bits could be changed, refresh
1592 * the scratch area
1593 */
1594 udp_set_dev_scratch(skb);
1595 break;
1596 }
2276f58a
PA		 1597 }
1598 return skb;
1599}
1600
85584672
ED		 1601/**
1602 * first_packet_length - return length of first packet in receive queue
1603 * @sk: socket
1604 *
1605 * Drops all bad checksum frames, until a valid one is found.
e83c6744 1606 * Returns the length of found skb, or -1 if none is found.
85584672 1607 */
e83c6744 1608static int first_packet_length(struct sock *sk)
85584672 1609{
2276f58a
PA		 1610 struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
1611 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
85584672 1612 struct sk_buff *skb;
7c13f97f 1613 int total = 0;
e83c6744 1614 int res;
85584672 1615
85584672 1616 spin_lock_bh(&rcvq->lock);
2276f58a
PA		 1617 skb = __first_packet_length(sk, rcvq, &total);
1618 if (!skb && !skb_queue_empty(sk_queue)) {
1619 spin_lock(&sk_queue->lock);
1620 skb_queue_splice_tail_init(sk_queue, rcvq);
1621 spin_unlock(&sk_queue->lock);
1622
1623 skb = __first_packet_length(sk, rcvq, &total);
85584672 1624 }
e83c6744 1625 res = skb ? skb->len : -1;
7c13f97f 1626 if (total)
6dfb4367 1627 udp_rmem_release(sk, total, 1, false);
85584672 1628 spin_unlock_bh(&rcvq->lock);
85584672
ED		 1629 return res;
1630}
1631
1da177e4
LT		 1632/*
1633 * IOCTL requests applicable to the UDP protocol
1634 */
e905a9ed 1635
1da177e4
LT		 1636int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
1637{
6516c655
SH		 1638 switch (cmd) {
1639 case SIOCOUTQ:
1da177e4 1640 {
31e6d363
ED		 1641 int amount = sk_wmem_alloc_get(sk);
1642
6516c655
SH		 1643 return put_user(amount, (int __user *)arg);
1644 }
1da177e4 1645
6516c655
SH		 1646 case SIOCINQ:
1647 {
e83c6744 1648 int amount = max_t(int, 0, first_packet_length(sk));
6516c655 1649
6516c655
SH		 1650 return put_user(amount, (int __user *)arg);
1651 }
1da177e4 1652
6516c655
SH		 1653 default:
1654 return -ENOIOCTLCMD;
1da177e4 1655 }
6516c655
SH		 1656
1657 return 0;
1da177e4 1658}
c482c568 1659EXPORT_SYMBOL(udp_ioctl);
1da177e4 1660
2276f58a
PA		 1661struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
1662 int noblock, int *peeked, int *off, int *err)
1663{
1664 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1665 struct sk_buff_head *queue;
1666 struct sk_buff *last;
1667 long timeo;
1668 int error;
1669
1670 queue = &udp_sk(sk)->reader_queue;
1671 flags |= noblock ? MSG_DONTWAIT : 0;
1672 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1673 do {
1674 struct sk_buff *skb;
1675
1676 error = sock_error(sk);
1677 if (error)
1678 break;
1679
1680 error = -EAGAIN;
1681 *peeked = 0;
1682 do {
2276f58a
PA		 1683 spin_lock_bh(&queue->lock);
1684 skb = __skb_try_recv_from_queue(sk, queue, flags,
1685 udp_skb_destructor,
de321ed3 1686 peeked, off, err,
2276f58a
PA		 1687 &last);
1688 if (skb) {
1689 spin_unlock_bh(&queue->lock);
2276f58a
PA		 1690 return skb;
1691 }
1692
1693 if (skb_queue_empty(sk_queue)) {
1694 spin_unlock_bh(&queue->lock);
1695 goto busy_check;
1696 }
1697
6dfb4367
PA		 1698 /* refill the reader queue and walk it again
1699 * keep both queues locked to avoid re-acquiring
1700 * the sk_receive_queue lock if fwd memory scheduling
1701 * is needed.
1702 */
2276f58a
PA		 1703 spin_lock(&sk_queue->lock);
1704 skb_queue_splice_tail_init(sk_queue, queue);
2276f58a
PA		 1705
1706 skb = __skb_try_recv_from_queue(sk, queue, flags,
6dfb4367 1707 udp_skb_dtor_locked,
de321ed3 1708 peeked, off, err,
2276f58a 1709 &last);
6dfb4367 1710 spin_unlock(&sk_queue->lock);
2276f58a 1711 spin_unlock_bh(&queue->lock);
de321ed3 1712 if (skb)
2276f58a 1713 return skb;
2276f58a
PA		 1714
1715busy_check:
1716 if (!sk_can_busy_loop(sk))
1717 break;
1718
1719 sk_busy_loop(sk, flags & MSG_DONTWAIT);
1720 } while (!skb_queue_empty(sk_queue));
1721
1722 /* sk_queue is empty, reader_queue may contain peeked packets */
1723 } while (timeo &&
1724 !__skb_wait_for_more_packets(sk, &error, &timeo,
1725 (struct sk_buff *)sk_queue));
1726
1727 *err = error;
1728 return NULL;
1729}
7e823644 1730EXPORT_SYMBOL(__skb_recv_udp);
2276f58a 1731
db8dac20
DM		 1732/*
1733 * This should be easy, if there is something there we
1734 * return it, otherwise we block.
1735 */
1736
1b784140
YX		 1737int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,
1738 int flags, int *addr_len)
db8dac20
DM		 1739{
1740 struct inet_sock *inet = inet_sk(sk);
342dfc30 1741 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
db8dac20 1742 struct sk_buff *skb;
59c2cdae 1743 unsigned int ulen, copied;
627d2d6b 1744 int peeked, peeking, off;
db8dac20
DM		 1745 int err;
1746 int is_udplite = IS_UDPLITE(sk);
197c949e 1747 bool checksum_valid = false;
db8dac20 1748
db8dac20 1749 if (flags & MSG_ERRQUEUE)
85fbaa75 1750 return ip_recv_error(sk, msg, len, addr_len);
db8dac20
DM		 1751
1752try_again:
a0917e0b
MD		 1753 peeking = flags & MSG_PEEK;
1754 off = sk_peek_offset(sk, flags);
7c13f97f 1755 skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err);
db8dac20 1756 if (!skb)
627d2d6b 1757 return err;
db8dac20 1758
b65ac446 1759 ulen = udp_skb_len(skb);
59c2cdae 1760 copied = len;
627d2d6b 1761 if (copied > ulen - off)
1762 copied = ulen - off;
59c2cdae 1763 else if (copied < ulen)
db8dac20
DM		 1764 msg->msg_flags |= MSG_TRUNC;
1765
1766 /*
1767 * If checksum is needed at all, try to do it while copying the
1768 * data. If the data is truncated, or if we only want a partial
1769 * coverage checksum (UDP-Lite), do it before the copy.
1770 */
1771
d21dbdfe
ED		 1772 if (copied < ulen || peeking ||
1773 (is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
b65ac446
PA		 1774 checksum_valid = udp_skb_csum_unnecessary(skb) ||
1775 !__udp_lib_checksum_complete(skb);
197c949e 1776 if (!checksum_valid)
db8dac20
DM		 1777 goto csum_copy_err;
1778 }
1779
b65ac446
PA		 1780 if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
1781 if (udp_skb_is_linear(skb))
1782 err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
1783 else
1784 err = skb_copy_datagram_msg(skb, off, msg, copied);
1785 } else {
627d2d6b 1786 err = skb_copy_and_csum_datagram_msg(skb, off, msg);
db8dac20
DM		 1787
1788 if (err == -EINVAL)
1789 goto csum_copy_err;
1790 }
1791
22911fc5 1792 if (unlikely(err)) {
979402b1
ED		 1793 if (!peeked) {
1794 atomic_inc(&sk->sk_drops);
6aef70a8
ED		 1795 UDP_INC_STATS(sock_net(sk),
1796 UDP_MIB_INERRORS, is_udplite);
979402b1 1797 }
850cbadd 1798 kfree_skb(skb);
627d2d6b 1799 return err;
22911fc5 1800 }
db8dac20
DM		 1801
1802 if (!peeked)
6aef70a8
ED		 1803 UDP_INC_STATS(sock_net(sk),
1804 UDP_MIB_INDATAGRAMS, is_udplite);
db8dac20 1805
3b885787 1806 sock_recv_ts_and_drops(msg, sk, skb);
db8dac20
DM		 1807
1808 /* Copy the address. */
c482c568 1809 if (sin) {
db8dac20
DM		 1810 sin->sin_family = AF_INET;
1811 sin->sin_port = udp_hdr(skb)->source;
1812 sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1813 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
bceaa902 1814 *addr_len = sizeof(*sin);
db8dac20 1815 }
bcd1665e
PA		 1816
1817 if (udp_sk(sk)->gro_enabled)
1818 udp_cmsg_recv(msg, sk, skb);
1819
db8dac20 1820 if (inet->cmsg_flags)
ad959036 1821 ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
db8dac20 1822
59c2cdae 1823 err = copied;
db8dac20
DM		 1824 if (flags & MSG_TRUNC)
1825 err = ulen;
1826
850cbadd 1827 skb_consume_udp(sk, skb, peeking ? -err : err);
db8dac20
DM		 1828 return err;
1829
1830csum_copy_err:
2276f58a
PA		 1831 if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
1832 udp_skb_destructor)) {
6aef70a8
ED		 1833 UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1834 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
6a5dc9e5 1835 }
850cbadd 1836 kfree_skb(skb);
db8dac20 1837
beb39db5
ED		 1838 /* starting over for a new packet, but check if we need to yield */
1839 cond_resched();
9cfaa8de 1840 msg->msg_flags &= ~MSG_TRUNC;
db8dac20
DM		 1841 goto try_again;
1842}
1843
d74bad4e
AI		 1844int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
1845{
1846 /* This check is replicated from __ip4_datagram_connect() and
1847 * intended to prevent BPF program called below from accessing bytes
1848 * that are out of the bound specified by user in addr_len.
1849 */
1850 if (addr_len < sizeof(struct sockaddr_in))
1851 return -EINVAL;
1852
1853 return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr);
1854}
1855EXPORT_SYMBOL(udp_pre_connect);
1856
286c72de 1857int __udp_disconnect(struct sock *sk, int flags)
1da177e4
LT		 1858{
1859 struct inet_sock *inet = inet_sk(sk);
1860 /*
1861 * 1003.1g - break association.
1862 */
e905a9ed 1863
1da177e4 1864 sk->sk_state = TCP_CLOSE;
c720c7e8
ED		 1865 inet->inet_daddr = 0;
1866 inet->inet_dport = 0;
bdeab991 1867 sock_rps_reset_rxhash(sk);
1da177e4
LT		 1868 sk->sk_bound_dev_if = 0;
1869 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1870 inet_reset_saddr(sk);
1871
1872 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1873 sk->sk_prot->unhash(sk);
c720c7e8 1874 inet->inet_sport = 0;
1da177e4
LT		 1875 }
1876 sk_dst_reset(sk);
1877 return 0;
1878}
286c72de
ED		 1879EXPORT_SYMBOL(__udp_disconnect);
1880
1881int udp_disconnect(struct sock *sk, int flags)
1882{
1883 lock_sock(sk);
1884 __udp_disconnect(sk, flags);
1885 release_sock(sk);
1886 return 0;
1887}
c482c568 1888EXPORT_SYMBOL(udp_disconnect);
1da177e4 1889
645ca708
ED		 1890void udp_lib_unhash(struct sock *sk)
1891{
723b4610
ED		 1892 if (sk_hashed(sk)) {
1893 struct udp_table *udptable = sk->sk_prot->h.udp_table;
512615b6
ED		 1894 struct udp_hslot *hslot, *hslot2;
1895
1896 hslot = udp_hashslot(udptable, sock_net(sk),
1897 udp_sk(sk)->udp_port_hash);
1898 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
645ca708 1899
723b4610 1900 spin_lock_bh(&hslot->lock);
e32ea7e7
CG		 1901 if (rcu_access_pointer(sk->sk_reuseport_cb))
1902 reuseport_detach_sock(sk);
ca065d0c 1903 if (sk_del_node_init_rcu(sk)) {
fdcc8aa9 1904 hslot->count--;
c720c7e8 1905 inet_sk(sk)->inet_num = 0;
723b4610 1906 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
512615b6
ED		 1907
1908 spin_lock(&hslot2->lock);
ca065d0c 1909 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
512615b6
ED		 1910 hslot2->count--;
1911 spin_unlock(&hslot2->lock);
723b4610
ED		 1912 }
1913 spin_unlock_bh(&hslot->lock);
645ca708 1914 }
645ca708
ED		 1915}
1916EXPORT_SYMBOL(udp_lib_unhash);
1917
719f8358
ED		 1918/*
1919 * inet_rcv_saddr was changed, we must rehash secondary hash
1920 */
1921void udp_lib_rehash(struct sock *sk, u16 newhash)
1922{
1923 if (sk_hashed(sk)) {
1924 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1925 struct udp_hslot *hslot, *hslot2, *nhslot2;
1926
1927 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1928 nhslot2 = udp_hashslot2(udptable, newhash);
1929 udp_sk(sk)->udp_portaddr_hash = newhash;
e32ea7e7
CG		 1930
1931 if (hslot2 != nhslot2 ||
1932 rcu_access_pointer(sk->sk_reuseport_cb)) {
719f8358
ED		 1933 hslot = udp_hashslot(udptable, sock_net(sk),
1934 udp_sk(sk)->udp_port_hash);
1935 /* we must lock primary chain too */
1936 spin_lock_bh(&hslot->lock);
e32ea7e7
CG		 1937 if (rcu_access_pointer(sk->sk_reuseport_cb))
1938 reuseport_detach_sock(sk);
1939
1940 if (hslot2 != nhslot2) {
1941 spin_lock(&hslot2->lock);
ca065d0c 1942 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
e32ea7e7
CG		 1943 hslot2->count--;
1944 spin_unlock(&hslot2->lock);
1945
1946 spin_lock(&nhslot2->lock);
ca065d0c 1947 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
e32ea7e7
CG		 1948 &nhslot2->head);
1949 nhslot2->count++;
1950 spin_unlock(&nhslot2->lock);
1951 }
719f8358
ED		 1952
1953 spin_unlock_bh(&hslot->lock);
1954 }
1955 }
1956}
1957EXPORT_SYMBOL(udp_lib_rehash);
1958
1959static void udp_v4_rehash(struct sock *sk)
1960{
f0b1e64c 1961 u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
719f8358
ED		 1962 inet_sk(sk)->inet_rcv_saddr,
1963 inet_sk(sk)->inet_num);
1964 udp_lib_rehash(sk, new_hash);
1965}
1966
a3f96c47 1967static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
93821778 1968{
fec5e652 1969 int rc;
766e9037 1970
005ec974 1971 if (inet_sk(sk)->inet_daddr) {
bdeab991 1972 sock_rps_save_rxhash(sk, skb);
005ec974 1973 sk_mark_napi_id(sk, skb);
2c8c56e1 1974 sk_incoming_cpu_update(sk);
e68b6e50
ED		 1975 } else {
1976 sk_mark_napi_id_once(sk, skb);
005ec974 1977 }
fec5e652 1978
850cbadd 1979 rc = __udp_enqueue_schedule_skb(sk, skb);
766e9037
ED		 1980 if (rc < 0) {
1981 int is_udplite = IS_UDPLITE(sk);
93821778 1982
93821778 1983 /* Note that an ENOMEM error is charged twice */
766e9037 1984 if (rc == -ENOMEM)
e61da9e2 1985 UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
02c22347 1986 is_udplite);
e61da9e2 1987 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
766e9037 1988 kfree_skb(skb);
296f7ea7 1989 trace_udp_fail_queue_rcv_skb(rc, sk);
766e9037 1990 return -1;
93821778
HX		 1991 }
1992
1993 return 0;
93821778
HX		 1994}
1995
db8dac20
DM		 1996/* returns:
1997 * -1: error
1998 * 0: success
1999 * >0: "udp encap" protocol resubmission
2000 *
2001 * Note that in the success and error cases, the skb is assumed to
2002 * have either been requeued or freed.
2003 */
cf329aa4 2004static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
db8dac20
DM		 2005{
2006 struct udp_sock *up = udp_sk(sk);
db8dac20
DM		 2007 int is_udplite = IS_UDPLITE(sk);
2008
2009 /*
2010 * Charge it to the socket, dropping if the queue is full.
2011 */
2012 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
2013 goto drop;
2014 nf_reset(skb);
2015
88ab3108 2016 if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
0ad92ad0
ED		 2017 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
2018
db8dac20
DM		 2019 /*
2020 * This is an encapsulation socket so pass the skb to
2021 * the socket's udp_encap_rcv() hook. Otherwise, just
2022 * fall through and pass this up the UDP socket.
2023 * up->encap_rcv() returns the following value:
2024 * =0 if skb was successfully passed to the encap
2025 * handler or was discarded by it.
2026 * >0 if skb should be passed on to UDP.
2027 * <0 if skb should be resubmitted as proto -N
2028 */
2029
2030 /* if we're overly short, let UDP handle it */
6aa7de05 2031 encap_rcv = READ_ONCE(up->encap_rcv);
e5aed006 2032 if (encap_rcv) {
db8dac20
DM		 2033 int ret;
2034
0a80966b
TH		 2035 /* Verify checksum before giving to encap */
2036 if (udp_lib_checksum_complete(skb))
2037 goto csum_error;
2038
0ad92ad0 2039 ret = encap_rcv(sk, skb);
db8dac20 2040 if (ret <= 0) {
02c22347
ED		 2041 __UDP_INC_STATS(sock_net(sk),
2042 UDP_MIB_INDATAGRAMS,
2043 is_udplite);
db8dac20
DM		 2044 return -ret;
2045 }
2046 }
2047
2048 /* FALLTHROUGH -- it's a UDP Packet */
2049 }
2050
2051 /*
2052 * UDP-Lite specific tests, ignored on UDP sockets
2053 */
2054 if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
2055
2056 /*
2057 * MIB statistics other than incrementing the error count are
2058 * disabled for the following two types of errors: these depend
2059 * on the application settings, not on the functioning of the
2060 * protocol stack as such.
2061 *
2062 * RFC 3828 here recommends (sec 3.3): "There should also be a
2063 * way ... to ... at least let the receiving application block
2064 * delivery of packets with coverage values less than a value
2065 * provided by the application."
2066 */
2067 if (up->pcrlen == 0) { /* full coverage was set */
ba7a46f1
JP		 2068 net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
2069 UDP_SKB_CB(skb)->cscov, skb->len);
db8dac20
DM		 2070 goto drop;
2071 }
2072 /* The next case involves violating the min. coverage requested
2073 * by the receiver. This is subtle: if receiver wants x and x is
2074 * greater than the buffersize/MTU then receiver will complain
2075 * that it wants x while sender emits packets of smaller size y.
2076 * Therefore the above ...()->partial_cov statement is essential.
2077 */
2078 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
ba7a46f1
JP		 2079 net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
2080 UDP_SKB_CB(skb)->cscov, up->pcrlen);
db8dac20
DM		 2081 goto drop;
2082 }
2083 }
2084
dd99e425 2085 prefetch(&sk->sk_rmem_alloc);
ce25d66a
ED		 2086 if (rcu_access_pointer(sk->sk_filter) &&
2087 udp_lib_checksum_complete(skb))
e6afc8ac 2088 goto csum_error;
ce25d66a 2089
ba66bbe5 2090 if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr)))
a6127697 2091 goto drop;
db8dac20 2092
e6afc8ac 2093 udp_csum_pull_header(skb);
db8dac20 2094
fbf8866d 2095 ipv4_pktinfo_prepare(sk, skb);
850cbadd 2096 return __udp_queue_rcv_skb(sk, skb);
db8dac20 2097
6a5dc9e5 2098csum_error:
02c22347 2099 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
db8dac20 2100drop:
02c22347 2101 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
8edf19c2 2102 atomic_inc(&sk->sk_drops);
db8dac20
DM		 2103 kfree_skb(skb);
2104 return -1;
2105}
2106
cf329aa4
PA		 2107static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2108{
2109 struct sk_buff *next, *segs;
2110 int ret;
2111
2112 if (likely(!udp_unexpected_gso(sk, skb)))
2113 return udp_queue_rcv_one_skb(sk, skb);
2114
2115 BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_SGO_CB_OFFSET);
2116 __skb_push(skb, -skb_mac_offset(skb));
2117 segs = udp_rcv_segment(sk, skb, true);
2118 for (skb = segs; skb; skb = next) {
2119 next = skb->next;
2120 __skb_pull(skb, skb_transport_offset(skb));
2121 ret = udp_queue_rcv_one_skb(sk, skb);
2122 if (ret > 0)
2123 ip_protocol_deliver_rcu(dev_net(skb->dev), skb, -ret);
2124 }
2125 return 0;
2126}
2127
97502231 2128/* For TCP sockets, sk_rx_dst is protected by socket lock
e47eb5df 2129 * For UDP, we use xchg() to guard against concurrent changes.
97502231 2130 */
64f0f5d1 2131bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
421b3885 2132{
97502231
ED		 2133 struct dst_entry *old;
2134
d24406c8
WW		 2135 if (dst_hold_safe(dst)) {
2136 old = xchg(&sk->sk_rx_dst, dst);
2137 dst_release(old);
64f0f5d1 2138 return old != dst;
d24406c8 2139 }
64f0f5d1 2140 return false;
421b3885 2141}
c9f2c1ae 2142EXPORT_SYMBOL(udp_sk_rx_dst_set);
421b3885 2143
db8dac20
DM		 2144/*
2145 * Multicasts and broadcasts go to each listener.
2146 *
1240d137 2147 * Note: called only from the BH handler context.
db8dac20 2148 */
e3163493 2149static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
db8dac20
DM		 2150 struct udphdr *uh,
2151 __be32 saddr, __be32 daddr,
36cbb245
RJ		 2152 struct udp_table *udptable,
2153 int proto)
db8dac20 2154{
ca065d0c 2155 struct sock *sk, *first = NULL;
5cf3d461
DH		 2156 unsigned short hnum = ntohs(uh->dest);
2157 struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
2dc41cff 2158 unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
ca065d0c
ED		 2159 unsigned int offset = offsetof(typeof(*sk), sk_node);
2160 int dif = skb->dev->ifindex;
fb74c277 2161 int sdif = inet_sdif(skb);
ca065d0c
ED		 2162 struct hlist_node *node;
2163 struct sk_buff *nskb;
2dc41cff
DH		 2164
2165 if (use_hash2) {
f0b1e64c 2166 hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
73e2d5e3 2167 udptable->mask;
f0b1e64c 2168 hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
2dc41cff 2169start_lookup:
73e2d5e3 2170 hslot = &udptable->hash2[hash2];
2dc41cff
DH		 2171 offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
2172 }
db8dac20 2173
ca065d0c
ED		 2174 sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
2175 if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
fb74c277 2176 uh->source, saddr, dif, sdif, hnum))
ca065d0c
ED		 2177 continue;
2178
2179 if (!first) {
2180 first = sk;
2181 continue;
1240d137 2182 }
ca065d0c 2183 nskb = skb_clone(skb, GFP_ATOMIC);
1240d137 2184
ca065d0c
ED		 2185 if (unlikely(!nskb)) {
2186 atomic_inc(&sk->sk_drops);
02c22347
ED		 2187 __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
2188 IS_UDPLITE(sk));
2189 __UDP_INC_STATS(net, UDP_MIB_INERRORS,
2190 IS_UDPLITE(sk));
ca065d0c
ED		 2191 continue;
2192 }
2193 if (udp_queue_rcv_skb(sk, nskb) > 0)
2194 consume_skb(nskb);
2195 }
1240d137 2196
2dc41cff
DH		 2197 /* Also lookup *:port if we are using hash2 and haven't done so yet. */
2198 if (use_hash2 && hash2 != hash2_any) {
2199 hash2 = hash2_any;
2200 goto start_lookup;
2201 }
2202
ca065d0c
ED		 2203 if (first) {
2204 if (udp_queue_rcv_skb(first, skb) > 0)
2205 consume_skb(skb);
1240d137 2206 } else {
ca065d0c 2207 kfree_skb(skb);
02c22347
ED		 2208 __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
2209 proto == IPPROTO_UDPLITE);
1240d137 2210 }
db8dac20
DM		 2211 return 0;
2212}
2213
2214/* Initialize UDP checksum. If exited with zero value (success),
2215 * CHECKSUM_UNNECESSARY means, that no more checks are required.
2216 * Otherwise, csum completion requires chacksumming packet body,
2217 * including udp header and folding it to skb->csum.
2218 */
2219static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
2220 int proto)
2221{
db8dac20
DM		 2222 int err;
2223
2224 UDP_SKB_CB(skb)->partial_cov = 0;
2225 UDP_SKB_CB(skb)->cscov = skb->len;
2226
2227 if (proto == IPPROTO_UDPLITE) {
2228 err = udplite_checksum_init(skb, uh);
2229 if (err)
2230 return err;
15f35d49
AK		 2231
2232 if (UDP_SKB_CB(skb)->partial_cov) {
2233 skb->csum = inet_compute_pseudo(skb, proto);
2234 return 0;
2235 }
db8dac20
DM		 2236 }
2237
b46d9f62
HFS		 2238 /* Note, we are only interested in != 0 or == 0, thus the
2239 * force to int.
2240 */
db4f1be3
ST		 2241 err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check,
2242 inet_compute_pseudo);
2243 if (err)
2244 return err;
2245
2246 if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
2247 /* If SW calculated the value, we know it's bad */
2248 if (skb->csum_complete_sw)
2249 return 1;
2250
2251 /* HW says the value is bad. Let's validate that.
2252 * skb->csum is no longer the full packet checksum,
2253 * so don't treat it as such.
2254 */
2255 skb_checksum_complete_unset(skb);
2256 }
2257
2258 return 0;
db8dac20
DM		 2259}
2260
2b5a9217
PA		 2261/* wrapper for udp_queue_rcv_skb tacking care of csum conversion and
2262 * return code conversion for ip layer consumption
2263 */
2264static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
2265 struct udphdr *uh)
2266{
2267 int ret;
2268
2269 if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
2270 skb_checksum_try_convert(skb, IPPROTO_UDP, uh->check,
2271 inet_compute_pseudo);
2272
2273 ret = udp_queue_rcv_skb(sk, skb);
2274
2275 /* a return value > 0 means to resubmit the input, but
2276 * it wants the return to be -protocol, or 0
2277 */
2278 if (ret > 0)
2279 return -ret;
2280 return 0;
2281}
2282
db8dac20
DM		 2283/*
2284 * All we need to do is get the socket, and then do a checksum.
2285 */
2286
645ca708 2287int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
db8dac20
DM		 2288 int proto)
2289{
2290 struct sock *sk;
7b5e56f9 2291 struct udphdr *uh;
db8dac20 2292 unsigned short ulen;
adf30907 2293 struct rtable *rt = skb_rtable(skb);
2783ef23 2294 __be32 saddr, daddr;
0283328e 2295 struct net *net = dev_net(skb->dev);
db8dac20
DM		 2296
2297 /*
2298 * Validate the packet.
2299 */
2300 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
2301 goto drop; /* No space for header. */
2302
7b5e56f9 2303 uh = udp_hdr(skb);
db8dac20 2304 ulen = ntohs(uh->len);
ccc2d97c
BM		 2305 saddr = ip_hdr(skb)->saddr;
2306 daddr = ip_hdr(skb)->daddr;
2307
db8dac20
DM		 2308 if (ulen > skb->len)
2309 goto short_packet;
2310
2311 if (proto == IPPROTO_UDP) {
2312 /* UDP validates ulen. */
2313 if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
2314 goto short_packet;
2315 uh = udp_hdr(skb);
2316 }
2317
2318 if (udp4_csum_init(skb, uh, proto))
2319 goto csum_error;
2320
8afdd99a
ED		 2321 sk = skb_steal_sock(skb);
2322 if (sk) {
97502231 2323 struct dst_entry *dst = skb_dst(skb);
421b3885 2324 int ret;
421b3885 2325
97502231
ED		 2326 if (unlikely(sk->sk_rx_dst != dst))
2327 udp_sk_rx_dst_set(sk, dst);
db8dac20 2328
2b5a9217 2329 ret = udp_unicast_rcv_skb(sk, skb, uh);
8afdd99a 2330 sock_put(sk);
2b5a9217 2331 return ret;
421b3885 2332 }
db8dac20 2333
c18450a5
FF		 2334 if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
2335 return __udp4_lib_mcast_deliver(net, skb, uh,
36cbb245 2336 saddr, daddr, udptable, proto);
c18450a5
FF		 2337
2338 sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
2b5a9217
PA		 2339 if (sk)
2340 return udp_unicast_rcv_skb(sk, skb, uh);
db8dac20
DM		 2341
2342 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2343 goto drop;
2344 nf_reset(skb);
2345
2346 /* No socket. Drop packet silently, if checksum is wrong */
2347 if (udp_lib_checksum_complete(skb))
2348 goto csum_error;
2349
02c22347 2350 __UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
db8dac20
DM		 2351 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
2352
2353 /*
2354 * Hmm. We got an UDP packet to a port to which we
2355 * don't wanna listen. Ignore it.
2356 */
2357 kfree_skb(skb);
2358 return 0;
2359
2360short_packet:
ba7a46f1
JP		 2361 net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
2362 proto == IPPROTO_UDPLITE ? "Lite" : "",
2363 &saddr, ntohs(uh->source),
2364 ulen, skb->len,
2365 &daddr, ntohs(uh->dest));
db8dac20
DM		 2366 goto drop;
2367
2368csum_error:
2369 /*
2370 * RFC1122: OK. Discards the bad packet silently (as far as
2371 * the network is concerned, anyway) as per 4.1.3.4 (MUST).
2372 */
ba7a46f1
JP		 2373 net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
2374 proto == IPPROTO_UDPLITE ? "Lite" : "",
2375 &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
2376 ulen);
02c22347 2377 __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
db8dac20 2378drop:
02c22347 2379 __UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
db8dac20
DM		 2380 kfree_skb(skb);
2381 return 0;
2382}
2383
421b3885
SB		 2384/* We can only early demux multicast if there is a single matching socket.
2385 * If more than one socket found returns NULL
2386 */
2387static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
2388 __be16 loc_port, __be32 loc_addr,
2389 __be16 rmt_port, __be32 rmt_addr,
fb74c277 2390 int dif, int sdif)
421b3885
SB		 2391{
2392 struct sock *sk, *result;
421b3885 2393 unsigned short hnum = ntohs(loc_port);
ca065d0c 2394 unsigned int slot = udp_hashfn(net, hnum, udp_table.mask);
421b3885
SB		 2395 struct udp_hslot *hslot = &udp_table.hash[slot];
2396
63c6f81c
ED		 2397 /* Do not bother scanning a too big list */
2398 if (hslot->count > 10)
2399 return NULL;
2400
421b3885 2401 result = NULL;
ca065d0c
ED		 2402 sk_for_each_rcu(sk, &hslot->head) {
2403 if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
fb74c277 2404 rmt_port, rmt_addr, dif, sdif, hnum)) {
ca065d0c
ED		 2405 if (result)
2406 return NULL;
421b3885 2407 result = sk;
421b3885
SB		 2408 }
2409 }
ca065d0c 2410
421b3885
SB		 2411 return result;
2412}
2413
2414/* For unicast we should only early demux connected sockets or we can
2415 * break forwarding setups. The chains here can be long so only check
2416 * if the first socket is an exact match and if not move on.
2417 */
2418static struct sock *__udp4_lib_demux_lookup(struct net *net,
2419 __be16 loc_port, __be32 loc_addr,
2420 __be16 rmt_port, __be32 rmt_addr,
3fa6f616 2421 int dif, int sdif)
421b3885 2422{
421b3885 2423 unsigned short hnum = ntohs(loc_port);
f0b1e64c 2424 unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
421b3885
SB		 2425 unsigned int slot2 = hash2 & udp_table.mask;
2426 struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
c7228317 2427 INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
421b3885 2428 const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
ca065d0c 2429 struct sock *sk;
421b3885 2430
ca065d0c
ED		 2431 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
2432 if (INET_MATCH(sk, net, acookie, rmt_addr,
3fa6f616 2433 loc_addr, ports, dif, sdif))
ca065d0c 2434 return sk;
421b3885
SB		 2435 /* Only check first socket in chain */
2436 break;
2437 }
ca065d0c 2438 return NULL;
421b3885
SB		 2439}
2440
7487449c 2441int udp_v4_early_demux(struct sk_buff *skb)
421b3885 2442{
610438b7 2443 struct net *net = dev_net(skb->dev);
bc044e8d 2444 struct in_device *in_dev = NULL;
610438b7
ED		 2445 const struct iphdr *iph;
2446 const struct udphdr *uh;
ca065d0c 2447 struct sock *sk = NULL;
421b3885 2448 struct dst_entry *dst;
421b3885 2449 int dif = skb->dev->ifindex;
fb74c277 2450 int sdif = inet_sdif(skb);
6e540309 2451 int ours;
421b3885
SB		 2452
2453 /* validate the packet */
2454 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
7487449c 2455 return 0;
421b3885 2456
610438b7
ED		 2457 iph = ip_hdr(skb);
2458 uh = udp_hdr(skb);
2459
996b44fc 2460 if (skb->pkt_type == PACKET_MULTICAST) {
bc044e8d 2461 in_dev = __in_dev_get_rcu(skb->dev);
6e540309
SB		 2462
2463 if (!in_dev)
7487449c 2464 return 0;
6e540309 2465
996b44fc
PA		 2466 ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
2467 iph->protocol);
2468 if (!ours)
2469 return 0;
ad0ea198 2470
421b3885 2471 sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
fb74c277
DA		 2472 uh->source, iph->saddr,
2473 dif, sdif);
6e540309 2474 } else if (skb->pkt_type == PACKET_HOST) {
421b3885 2475 sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
3fa6f616 2476 uh->source, iph->saddr, dif, sdif);
6e540309 2477 }
421b3885 2478
41c6d650 2479 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
7487449c 2480 return 0;
421b3885
SB		 2481
2482 skb->sk = sk;
82eabd9e 2483 skb->destructor = sock_efree;
10e2eb87 2484 dst = READ_ONCE(sk->sk_rx_dst);
421b3885
SB		 2485
2486 if (dst)
2487 dst = dst_check(dst, 0);
10e2eb87 2488 if (dst) {
bc044e8d
PA		 2489 u32 itag = 0;
2490
d24406c8
WW		 2491 /* set noref for now.
2492 * any place which wants to hold dst has to call
2493 * dst_hold_safe()
2494 */
2495 skb_dst_set_noref(skb, dst);
bc044e8d
PA		 2496
2497 /* for unconnected multicast sockets we need to validate
2498 * the source on each packet
2499 */
2500 if (!inet_sk(sk)->inet_daddr && in_dev)
2501 return ip_mc_validate_source(skb, iph->daddr,
2502 iph->saddr, iph->tos,
2503 skb->dev, in_dev, &itag);
10e2eb87 2504 }
7487449c 2505 return 0;
421b3885
SB		 2506}
2507
db8dac20
DM		 2508int udp_rcv(struct sk_buff *skb)
2509{
645ca708 2510 return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP);
db8dac20
DM		 2511}
2512
7d06b2e0 2513void udp_destroy_sock(struct sock *sk)
db8dac20 2514{
44046a59 2515 struct udp_sock *up = udp_sk(sk);
8a74ad60 2516 bool slow = lock_sock_fast(sk);
db8dac20 2517 udp_flush_pending_frames(sk);
8a74ad60 2518 unlock_sock_fast(sk, slow);
60fb9567
PA		 2519 if (static_branch_unlikely(&udp_encap_needed_key)) {
2520 if (up->encap_type) {
2521 void (*encap_destroy)(struct sock *sk);
2522 encap_destroy = READ_ONCE(up->encap_destroy);
2523 if (encap_destroy)
2524 encap_destroy(sk);
2525 }
2526 if (up->encap_enabled)
2527 static_branch_disable(&udp_encap_needed_key);
44046a59 2528 }
db8dac20
DM		 2529}
2530
1da177e4
LT		 2531/*
2532 * Socket option code for UDP
2533 */
4c0a6cb0 2534int udp_lib_setsockopt(struct sock *sk, int level, int optname,
b7058842 2535 char __user *optval, unsigned int optlen,
4c0a6cb0 2536 int (*push_pending_frames)(struct sock *))
1da177e4
LT		 2537{
2538 struct udp_sock *up = udp_sk(sk);
1c19448c 2539 int val, valbool;
1da177e4 2540 int err = 0;
b2bf1e26 2541 int is_udplite = IS_UDPLITE(sk);
1da177e4 2542
c482c568 2543 if (optlen < sizeof(int))
1da177e4
LT		 2544 return -EINVAL;
2545
2546 if (get_user(val, (int __user *)optval))
2547 return -EFAULT;
2548
1c19448c
TH		 2549 valbool = val ? 1 : 0;
2550
6516c655 2551 switch (optname) {
1da177e4
LT		 2552 case UDP_CORK:
2553 if (val != 0) {
2554 up->corkflag = 1;
2555 } else {
2556 up->corkflag = 0;
2557 lock_sock(sk);
4243cdc2 2558 push_pending_frames(sk);
1da177e4
LT		 2559 release_sock(sk);
2560 }
2561 break;
e905a9ed 2562
1da177e4
LT		 2563 case UDP_ENCAP:
2564 switch (val) {
2565 case 0:
2566 case UDP_ENCAP_ESPINUDP:
2567 case UDP_ENCAP_ESPINUDP_NON_IKE:
067b207b
JC		 2568 up->encap_rcv = xfrm4_udp_encap_rcv;
2569 /* FALLTHROUGH */
342f0234 2570 case UDP_ENCAP_L2TPINUDP:
1da177e4 2571 up->encap_type = val;
60fb9567
PA		 2572 lock_sock(sk);
2573 udp_tunnel_encap_enable(sk->sk_socket);
2574 release_sock(sk);
1da177e4
LT		 2575 break;
2576 default:
2577 err = -ENOPROTOOPT;
2578 break;
2579 }
2580 break;
2581
1c19448c
TH		 2582 case UDP_NO_CHECK6_TX:
2583 up->no_check6_tx = valbool;
2584 break;
2585
2586 case UDP_NO_CHECK6_RX:
2587 up->no_check6_rx = valbool;
2588 break;
2589
bec1f6f6
WB		 2590 case UDP_SEGMENT:
2591 if (val < 0 || val > USHRT_MAX)
2592 return -EINVAL;
2593 up->gso_size = val;
2594 break;
2595
e20cf8d3
PA		 2596 case UDP_GRO:
2597 lock_sock(sk);
2598 if (valbool)
2599 udp_tunnel_encap_enable(sk->sk_socket);
2600 up->gro_enabled = valbool;
2601 release_sock(sk);
2602 break;
2603
ba4e58ec
GR		 2604 /*
2605 * UDP-Lite's partial checksum coverage (RFC 3828).
2606 */
2607 /* The sender sets actual checksum coverage length via this option.
2608 * The case coverage > packet length is handled by send module. */
2609 case UDPLITE_SEND_CSCOV:
b2bf1e26 2610 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR		 2611 return -ENOPROTOOPT;
2612 if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
2613 val = 8;
4be929be
AD		 2614 else if (val > USHRT_MAX)
2615 val = USHRT_MAX;
ba4e58ec
GR		 2616 up->pcslen = val;
2617 up->pcflag |= UDPLITE_SEND_CC;
2618 break;
2619
e905a9ed
YH		 2620 /* The receiver specifies a minimum checksum coverage value. To make
2621 * sense, this should be set to at least 8 (as done below). If zero is
ba4e58ec
GR		 2622 * used, this again means full checksum coverage. */
2623 case UDPLITE_RECV_CSCOV:
b2bf1e26 2624 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR		 2625 return -ENOPROTOOPT;
2626 if (val != 0 && val < 8) /* Avoid silly minimal values. */
2627 val = 8;
4be929be
AD		 2628 else if (val > USHRT_MAX)
2629 val = USHRT_MAX;
ba4e58ec
GR		 2630 up->pcrlen = val;
2631 up->pcflag |= UDPLITE_RECV_CC;
2632 break;
2633
1da177e4
LT		 2634 default:
2635 err = -ENOPROTOOPT;
2636 break;
6516c655 2637 }
1da177e4
LT		 2638
2639 return err;
2640}
c482c568 2641EXPORT_SYMBOL(udp_lib_setsockopt);
1da177e4 2642
db8dac20 2643int udp_setsockopt(struct sock *sk, int level, int optname,
b7058842 2644 char __user *optval, unsigned int optlen)
db8dac20
DM		 2645{
2646 if (level == SOL_UDP || level == SOL_UDPLITE)
2647 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2648 udp_push_pending_frames);
2649 return ip_setsockopt(sk, level, optname, optval, optlen);
2650}
2651
2652#ifdef CONFIG_COMPAT
2653int compat_udp_setsockopt(struct sock *sk, int level, int optname,
b7058842 2654 char __user *optval, unsigned int optlen)
db8dac20
DM		 2655{
2656 if (level == SOL_UDP || level == SOL_UDPLITE)
2657 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2658 udp_push_pending_frames);
2659 return compat_ip_setsockopt(sk, level, optname, optval, optlen);
2660}
2661#endif
2662
4c0a6cb0
GR		 2663int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2664 char __user *optval, int __user *optlen)
1da177e4
LT		 2665{
2666 struct udp_sock *up = udp_sk(sk);
2667 int val, len;
2668
c482c568 2669 if (get_user(len, optlen))
1da177e4
LT		 2670 return -EFAULT;
2671
2672 len = min_t(unsigned int, len, sizeof(int));
e905a9ed 2673
6516c655 2674 if (len < 0)
1da177e4
LT		 2675 return -EINVAL;
2676
6516c655 2677 switch (optname) {
1da177e4
LT		 2678 case UDP_CORK:
2679 val = up->corkflag;
2680 break;
2681
2682 case UDP_ENCAP:
2683 val = up->encap_type;
2684 break;
2685
1c19448c
TH		 2686 case UDP_NO_CHECK6_TX:
2687 val = up->no_check6_tx;
2688 break;
2689
2690 case UDP_NO_CHECK6_RX:
2691 val = up->no_check6_rx;
2692 break;
2693
bec1f6f6
WB		 2694 case UDP_SEGMENT:
2695 val = up->gso_size;
2696 break;
2697
ba4e58ec
GR		 2698 /* The following two cannot be changed on UDP sockets, the return is
2699 * always 0 (which corresponds to the full checksum coverage of UDP). */
2700 case UDPLITE_SEND_CSCOV:
2701 val = up->pcslen;
2702 break;
2703
2704 case UDPLITE_RECV_CSCOV:
2705 val = up->pcrlen;
2706 break;
2707
1da177e4
LT		 2708 default:
2709 return -ENOPROTOOPT;
6516c655 2710 }
1da177e4 2711
6516c655 2712 if (put_user(len, optlen))
e905a9ed 2713 return -EFAULT;
c482c568 2714 if (copy_to_user(optval, &val, len))
1da177e4 2715 return -EFAULT;
e905a9ed 2716 return 0;
1da177e4 2717}
c482c568 2718EXPORT_SYMBOL(udp_lib_getsockopt);
1da177e4 2719
db8dac20
DM		 2720int udp_getsockopt(struct sock *sk, int level, int optname,
2721 char __user *optval, int __user *optlen)
2722{
2723 if (level == SOL_UDP || level == SOL_UDPLITE)
2724 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2725 return ip_getsockopt(sk, level, optname, optval, optlen);
2726}
2727
2728#ifdef CONFIG_COMPAT
2729int compat_udp_getsockopt(struct sock *sk, int level, int optname,
2730 char __user *optval, int __user *optlen)
2731{
2732 if (level == SOL_UDP || level == SOL_UDPLITE)
2733 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2734 return compat_ip_getsockopt(sk, level, optname, optval, optlen);
2735}
2736#endif
1da177e4
LT		 2737/**
2738 * udp_poll - wait for a UDP event.
2739 * @file - file struct
2740 * @sock - socket
a11e1d43 2741 * @wait - poll table
1da177e4 2742 *
e905a9ed 2743 * This is same as datagram poll, except for the special case of
1da177e4
LT		 2744 * blocking sockets. If application is using a blocking fd
2745 * and a packet with checksum error is in the queue;
2746 * then it could get return from select indicating data available
2747 * but then block when reading it. Add special case code
2748 * to work around these arguably broken applications.
2749 */
a11e1d43 2750__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
1da177e4 2751{
a11e1d43 2752 __poll_t mask = datagram_poll(file, sock, wait);
1da177e4 2753 struct sock *sk = sock->sk;
ba4e58ec 2754
2276f58a 2755 if (!skb_queue_empty(&udp_sk(sk)->reader_queue))
a9a08845 2756 mask |= EPOLLIN | EPOLLRDNORM;
2276f58a 2757
1da177e4 2758 /* Check for false positives due to checksum errors */
a11e1d43 2759 if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
e83c6744 2760 !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
a9a08845 2761 mask &= ~(EPOLLIN | EPOLLRDNORM);
1da177e4
LT		 2762
2763 return mask;
e905a9ed 2764
1da177e4 2765}
a11e1d43 2766EXPORT_SYMBOL(udp_poll);
1da177e4 2767
5d77dca8
DA		 2768int udp_abort(struct sock *sk, int err)
2769{
2770 lock_sock(sk);
2771
2772 sk->sk_err = err;
2773 sk->sk_error_report(sk);
286c72de 2774 __udp_disconnect(sk, 0);
5d77dca8
DA		 2775
2776 release_sock(sk);
2777
2778 return 0;
2779}
2780EXPORT_SYMBOL_GPL(udp_abort);
2781
db8dac20 2782struct proto udp_prot = {
1e802951
TZ		 2783 .name = "UDP",
2784 .owner = THIS_MODULE,
2785 .close = udp_lib_close,
d74bad4e 2786 .pre_connect = udp_pre_connect,
1e802951
TZ		 2787 .connect = ip4_datagram_connect,
2788 .disconnect = udp_disconnect,
2789 .ioctl = udp_ioctl,
2790 .init = udp_init_sock,
2791 .destroy = udp_destroy_sock,
2792 .setsockopt = udp_setsockopt,
2793 .getsockopt = udp_getsockopt,
2794 .sendmsg = udp_sendmsg,
2795 .recvmsg = udp_recvmsg,
2796 .sendpage = udp_sendpage,
2797 .release_cb = ip4_datagram_release_cb,
2798 .hash = udp_lib_hash,
2799 .unhash = udp_lib_unhash,
2800 .rehash = udp_v4_rehash,
2801 .get_port = udp_v4_get_port,
2802 .memory_allocated = &udp_memory_allocated,
2803 .sysctl_mem = sysctl_udp_mem,
2804 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
2805 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
2806 .obj_size = sizeof(struct udp_sock),
2807 .h.udp_table = &udp_table,
db8dac20 2808#ifdef CONFIG_COMPAT
1e802951
TZ		 2809 .compat_setsockopt = compat_udp_setsockopt,
2810 .compat_getsockopt = compat_udp_getsockopt,
db8dac20 2811#endif
1e802951 2812 .diag_destroy = udp_abort,
db8dac20 2813};
c482c568 2814EXPORT_SYMBOL(udp_prot);
1da177e4
LT		 2815
2816/* ------------------------------------------------------------------------ */
2817#ifdef CONFIG_PROC_FS
2818
645ca708 2819static struct sock *udp_get_first(struct seq_file *seq, int start)
1da177e4
LT		 2820{
2821 struct sock *sk;
a3d2599b 2822 struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
1da177e4 2823 struct udp_iter_state *state = seq->private;
6f191efe 2824 struct net *net = seq_file_net(seq);
1da177e4 2825
a3d2599b 2826 for (state->bucket = start; state->bucket <= afinfo->udp_table->mask;
f86dcc5a 2827 ++state->bucket) {
a3d2599b 2828 struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket];
f86dcc5a 2829
ca065d0c 2830 if (hlist_empty(&hslot->head))
f86dcc5a
ED		 2831 continue;
2832
645ca708 2833 spin_lock_bh(&hslot->lock);
ca065d0c 2834 sk_for_each(sk, &hslot->head) {
878628fb 2835 if (!net_eq(sock_net(sk), net))
a91275ef 2836 continue;
a3d2599b 2837 if (sk->sk_family == afinfo->family)
1da177e4
LT		 2838 goto found;
2839 }
645ca708 2840 spin_unlock_bh(&hslot->lock);
1da177e4
LT		 2841 }
2842 sk = NULL;
2843found:
2844 return sk;
2845}
2846
2847static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
2848{
a3d2599b 2849 struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
1da177e4 2850 struct udp_iter_state *state = seq->private;
6f191efe 2851 struct net *net = seq_file_net(seq);
1da177e4
LT		 2852
2853 do {
ca065d0c 2854 sk = sk_next(sk);
a3d2599b 2855 } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != afinfo->family));
1da177e4 2856
645ca708 2857 if (!sk) {
a3d2599b
CH		 2858 if (state->bucket <= afinfo->udp_table->mask)
2859 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
645ca708 2860 return udp_get_first(seq, state->bucket + 1);
1da177e4
LT		 2861 }
2862 return sk;
2863}
2864
2865static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
2866{
645ca708 2867 struct sock *sk = udp_get_first(seq, 0);
1da177e4
LT		 2868
2869 if (sk)
6516c655 2870 while (pos && (sk = udp_get_next(seq, sk)) != NULL)
1da177e4
LT		 2871 --pos;
2872 return pos ? NULL : sk;
2873}
2874
a3d2599b 2875void *udp_seq_start(struct seq_file *seq, loff_t *pos)
1da177e4 2876{
30842f29 2877 struct udp_iter_state *state = seq->private;
f86dcc5a 2878 state->bucket = MAX_UDP_PORTS;
30842f29 2879
b50660f1 2880 return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
1da177e4 2881}
a3d2599b 2882EXPORT_SYMBOL(udp_seq_start);
1da177e4 2883
a3d2599b 2884void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1da177e4
LT		 2885{
2886 struct sock *sk;
2887
b50660f1 2888 if (v == SEQ_START_TOKEN)
1da177e4
LT		 2889 sk = udp_get_idx(seq, 0);
2890 else
2891 sk = udp_get_next(seq, v);
2892
2893 ++*pos;
2894 return sk;
2895}
a3d2599b 2896EXPORT_SYMBOL(udp_seq_next);
1da177e4 2897
a3d2599b 2898void udp_seq_stop(struct seq_file *seq, void *v)
1da177e4 2899{
a3d2599b 2900 struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
645ca708
ED		 2901 struct udp_iter_state *state = seq->private;
2902
a3d2599b
CH		 2903 if (state->bucket <= afinfo->udp_table->mask)
2904 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
1da177e4 2905}
a3d2599b 2906EXPORT_SYMBOL(udp_seq_stop);
db8dac20
DM		 2907
2908/* ------------------------------------------------------------------------ */
5e659e4c 2909static void udp4_format_sock(struct sock *sp, struct seq_file *f,
652586df 2910 int bucket)
db8dac20
DM		 2911{
2912 struct inet_sock *inet = inet_sk(sp);
c720c7e8
ED		 2913 __be32 dest = inet->inet_daddr;
2914 __be32 src = inet->inet_rcv_saddr;
2915 __u16 destp = ntohs(inet->inet_dport);
2916 __u16 srcp = ntohs(inet->inet_sport);
db8dac20 2917
f86dcc5a 2918 seq_printf(f, "%5d: %08X:%04X %08X:%04X"
652586df 2919 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d",
db8dac20 2920 bucket, src, srcp, dest, destp, sp->sk_state,
31e6d363 2921 sk_wmem_alloc_get(sp),
6c206b20 2922 udp_rqueue_get(sp),
a7cb5a49
EB		 2923 0, 0L, 0,
2924 from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
2925 0, sock_i_ino(sp),
41c6d650 2926 refcount_read(&sp->sk_refcnt), sp,
652586df 2927 atomic_read(&sp->sk_drops));
db8dac20
DM		 2928}
2929
2930int udp4_seq_show(struct seq_file *seq, void *v)
2931{
652586df 2932 seq_setwidth(seq, 127);
db8dac20 2933 if (v == SEQ_START_TOKEN)
652586df 2934 seq_puts(seq, " sl local_address rem_address st tx_queue "
db8dac20 2935 "rx_queue tr tm->when retrnsmt uid timeout "
cb61cb9b 2936 "inode ref pointer drops");
db8dac20 2937 else {
db8dac20
DM		 2938 struct udp_iter_state *state = seq->private;
2939
652586df 2940 udp4_format_sock(v, seq, state->bucket);
db8dac20 2941 }
652586df 2942 seq_pad(seq, '\n');
db8dac20
DM		 2943 return 0;
2944}
2945
c3506372 2946const struct seq_operations udp_seq_ops = {
a3d2599b
CH		 2947 .start = udp_seq_start,
2948 .next = udp_seq_next,
2949 .stop = udp_seq_stop,
2950 .show = udp4_seq_show,
2951};
c3506372 2952EXPORT_SYMBOL(udp_seq_ops);
73cb88ec 2953
db8dac20 2954static struct udp_seq_afinfo udp4_seq_afinfo = {
db8dac20 2955 .family = AF_INET,
645ca708 2956 .udp_table = &udp_table,
db8dac20
DM		 2957};
2958
2c8c1e72 2959static int __net_init udp4_proc_init_net(struct net *net)
15439feb 2960{
c3506372
CH		 2961 if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops,
2962 sizeof(struct udp_iter_state), &udp4_seq_afinfo))
a3d2599b
CH		 2963 return -ENOMEM;
2964 return 0;
15439feb
PE		 2965}
2966
2c8c1e72 2967static void __net_exit udp4_proc_exit_net(struct net *net)
15439feb 2968{
a3d2599b 2969 remove_proc_entry("udp", net->proc_net);
15439feb
PE		 2970}
2971
2972static struct pernet_operations udp4_net_ops = {
2973 .init = udp4_proc_init_net,
2974 .exit = udp4_proc_exit_net,
2975};
2976
db8dac20
DM		 2977int __init udp4_proc_init(void)
2978{
15439feb 2979 return register_pernet_subsys(&udp4_net_ops);
db8dac20
DM		 2980}
2981
2982void udp4_proc_exit(void)
2983{
15439feb 2984 unregister_pernet_subsys(&udp4_net_ops);
db8dac20 2985}
1da177e4
LT		 2986#endif /* CONFIG_PROC_FS */
2987
f86dcc5a
ED		 2988static __initdata unsigned long uhash_entries;
2989static int __init set_uhash_entries(char *str)
645ca708 2990{
413c27d8
EZ		 2991 ssize_t ret;
2992
f86dcc5a
ED		 2993 if (!str)
2994 return 0;
413c27d8
EZ		 2995
2996 ret = kstrtoul(str, 0, &uhash_entries);
2997 if (ret)
2998 return 0;
2999
f86dcc5a
ED		 3000 if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
3001 uhash_entries = UDP_HTABLE_SIZE_MIN;
3002 return 1;
3003}
3004__setup("uhash_entries=", set_uhash_entries);
645ca708 3005
f86dcc5a
ED		 3006void __init udp_table_init(struct udp_table *table, const char *name)
3007{
3008 unsigned int i;
3009
31fe62b9
TB		 3010 table->hash = alloc_large_system_hash(name,
3011 2 * sizeof(struct udp_hslot),
3012 uhash_entries,
3013 21, /* one slot per 2 MB */
3014 0,
3015 &table->log,
3016 &table->mask,
3017 UDP_HTABLE_SIZE_MIN,
3018 64 * 1024);
3019
512615b6 3020 table->hash2 = table->hash + (table->mask + 1);
f86dcc5a 3021 for (i = 0; i <= table->mask; i++) {
ca065d0c 3022 INIT_HLIST_HEAD(&table->hash[i].head);
fdcc8aa9 3023 table->hash[i].count = 0;
645ca708
ED		 3024 spin_lock_init(&table->hash[i].lock);
3025 }
512615b6 3026 for (i = 0; i <= table->mask; i++) {
ca065d0c 3027 INIT_HLIST_HEAD(&table->hash2[i].head);
512615b6
ED		 3028 table->hash2[i].count = 0;
3029 spin_lock_init(&table->hash2[i].lock);
3030 }
645ca708
ED		 3031}
3032
723b8e46
TH		 3033u32 udp_flow_hashrnd(void)
3034{
3035 static u32 hashrnd __read_mostly;
3036
3037 net_get_random_once(&hashrnd, sizeof(hashrnd));
3038
3039 return hashrnd;
3040}
3041EXPORT_SYMBOL(udp_flow_hashrnd);
3042
1e802951
TZ		 3043static void __udp_sysctl_init(struct net *net)
3044{
3045 net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM;
3046 net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM;
3047
3048#ifdef CONFIG_NET_L3_MASTER_DEV
3049 net->ipv4.sysctl_udp_l3mdev_accept = 0;
3050#endif
3051}
3052
3053static int __net_init udp_sysctl_init(struct net *net)
3054{
3055 __udp_sysctl_init(net);
3056 return 0;
3057}
3058
3059static struct pernet_operations __net_initdata udp_sysctl_ops = {
fc18999e 3060 .init = udp_sysctl_init,
1e802951
TZ		 3061};
3062
95766fff
HA		 3063void __init udp_init(void)
3064{
f03d78db 3065 unsigned long limit;
4b272750 3066 unsigned int i;
95766fff 3067
f86dcc5a 3068 udp_table_init(&udp_table, "UDP");
f03d78db 3069 limit = nr_free_buffer_pages() / 8;
95766fff
HA		 3070 limit = max(limit, 128UL);
3071 sysctl_udp_mem[0] = limit / 4 * 3;
3072 sysctl_udp_mem[1] = limit;
3073 sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
3074
1e802951 3075 __udp_sysctl_init(&init_net);
4b272750
ED		 3076
3077 /* 16 spinlocks per cpu */
3078 udp_busylocks_log = ilog2(nr_cpu_ids) + 4;
3079 udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log,
3080 GFP_KERNEL);
3081 if (!udp_busylocks)
3082 panic("UDP: failed to alloc udp_busylocks\n");
3083 for (i = 0; i < (1U << udp_busylocks_log); i++)
3084 spin_lock_init(udp_busylocks + i);
1e802951
TZ		 3085
3086 if (register_pernet_subsys(&udp_sysctl_ops))
3087 panic("UDP: failed to init sysctl parameters.\n");
95766fff 3088}
Linux 6.x block layer and io_uring tree(s)