projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libbpf: Make destructors more robust by handling ERR_PTR(err) cases
[linux-2.6-block.git] / net / ipv4 / udp.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * The User Datagram Protocol (UDP).
8 *
02c30a84 9 * Authors: Ross Biro
1da177e4
LT		 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
113aa838 12 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
1da177e4
LT		 13 * Hirokazu Takahashi, <taka@valinux.co.jp>
14 *
15 * Fixes:
16 * Alan Cox : verify_area() calls
17 * Alan Cox : stopped close while in use off icmp
18 * messages. Not a fix but a botch that
19 * for udp at least is 'valid'.
20 * Alan Cox : Fixed icmp handling properly
21 * Alan Cox : Correct error for oversized datagrams
e905a9ed
YH		 22 * Alan Cox : Tidied select() semantics.
23 * Alan Cox : udp_err() fixed properly, also now
1da177e4
LT		 24 * select and read wake correctly on errors
25 * Alan Cox : udp_send verify_area moved to avoid mem leak
26 * Alan Cox : UDP can count its memory
27 * Alan Cox : send to an unknown connection causes
28 * an ECONNREFUSED off the icmp, but
29 * does NOT close.
30 * Alan Cox : Switched to new sk_buff handlers. No more backlog!
31 * Alan Cox : Using generic datagram code. Even smaller and the PEEK
32 * bug no longer crashes it.
33 * Fred Van Kempen : Net2e support for sk->broadcast.
34 * Alan Cox : Uses skb_free_datagram
35 * Alan Cox : Added get/set sockopt support.
36 * Alan Cox : Broadcasting without option set returns EACCES.
37 * Alan Cox : No wakeup calls. Instead we now use the callbacks.
38 * Alan Cox : Use ip_tos and ip_ttl
39 * Alan Cox : SNMP Mibs
40 * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
41 * Matt Dillon : UDP length checks.
42 * Alan Cox : Smarter af_inet used properly.
43 * Alan Cox : Use new kernel side addressing.
44 * Alan Cox : Incorrect return on truncated datagram receive.
45 * Arnt Gulbrandsen : New udp_send and stuff
46 * Alan Cox : Cache last socket
47 * Alan Cox : Route cache
48 * Jon Peatfield : Minor efficiency fix to sendto().
49 * Mike Shaver : RFC1122 checks.
50 * Alan Cox : Nonblocking error fix.
51 * Willy Konynenberg : Transparent proxying support.
52 * Mike McLagan : Routing by source
53 * David S. Miller : New socket lookup architecture.
54 * Last socket cache retained as it
55 * does have a high hit rate.
56 * Olaf Kirch : Don't linearise iovec on sendmsg.
57 * Andi Kleen : Some cleanups, cache destination entry
e905a9ed 58 * for connect.
1da177e4
LT		 59 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
60 * Melvin Smith : Check msg_name not msg_namelen in sendto(),
61 * return ENOTCONN for unconnected sockets (POSIX)
62 * Janos Farkas : don't deliver multi/broadcasts to a different
63 * bound-to-device socket
64 * Hirokazu Takahashi : HW checksumming for outgoing UDP
65 * datagrams.
66 * Hirokazu Takahashi : sendfile() on UDP works now.
67 * Arnaldo C. Melo : convert /proc/net/udp to seq_file
68 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
69 * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
70 * a single port at the same time.
71 * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
342f0234 72 * James Chapman : Add L2TP encapsulation type.
1da177e4 73 */
e905a9ed 74
afd46503
JP		 75#define pr_fmt(fmt) "UDP: " fmt
76
7c0f6ba6 77#include <linux/uaccess.h>
1da177e4 78#include <asm/ioctls.h>
57c8a661 79#include <linux/memblock.h>
8203efb3
ED		 80#include <linux/highmem.h>
81#include <linux/swap.h>
1da177e4
LT		 82#include <linux/types.h>
83#include <linux/fcntl.h>
84#include <linux/module.h>
85#include <linux/socket.h>
86#include <linux/sockios.h>
14c85021 87#include <linux/igmp.h>
6e540309 88#include <linux/inetdevice.h>
1da177e4
LT		 89#include <linux/in.h>
90#include <linux/errno.h>
91#include <linux/timer.h>
92#include <linux/mm.h>
1da177e4 93#include <linux/inet.h>
1da177e4 94#include <linux/netdevice.h>
5a0e3ad6 95#include <linux/slab.h>
c752f073 96#include <net/tcp_states.h>
1da177e4
LT		 97#include <linux/skbuff.h>
98#include <linux/proc_fs.h>
99#include <linux/seq_file.h>
457c4cbc 100#include <net/net_namespace.h>
1da177e4 101#include <net/icmp.h>
421b3885 102#include <net/inet_hashtables.h>
e7cc0824 103#include <net/ip_tunnels.h>
1da177e4 104#include <net/route.h>
1da177e4
LT		 105#include <net/checksum.h>
106#include <net/xfrm.h>
296f7ea7 107#include <trace/events/udp.h>
447167bf 108#include <linux/static_key.h>
951cf368 109#include <linux/btf_ids.h>
22911fc5 110#include <trace/events/skb.h>
076bb0c8 111#include <net/busy_poll.h>
ba4e58ec 112#include "udp_impl.h"
e32ea7e7 113#include <net/sock_reuseport.h>
217375a0 114#include <net/addrconf.h>
60fb9567 115#include <net/udp_tunnel.h>
0146dca7
SD		 116#if IS_ENABLED(CONFIG_IPV6)
117#include <net/ipv6_stubs.h>
118#endif
1da177e4 119
f86dcc5a 120struct udp_table udp_table __read_mostly;
645ca708 121EXPORT_SYMBOL(udp_table);
1da177e4 122
8d987e5c 123long sysctl_udp_mem[3] __read_mostly;
95766fff 124EXPORT_SYMBOL(sysctl_udp_mem);
c482c568 125
8d987e5c 126atomic_long_t udp_memory_allocated;
95766fff
HA		 127EXPORT_SYMBOL(udp_memory_allocated);
128
f86dcc5a
ED		 129#define MAX_UDP_PORTS 65536
130#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
98322f22 131
f24d43c0 132static int udp_lib_lport_inuse(struct net *net, __u16 num,
645ca708 133 const struct udp_hslot *hslot,
98322f22 134 unsigned long *bitmap,
fe38d2a1 135 struct sock *sk, unsigned int log)
1da177e4 136{
f24d43c0 137 struct sock *sk2;
ba418fa3 138 kuid_t uid = sock_i_uid(sk);
25030a7f 139
ca065d0c 140 sk_for_each(sk2, &hslot->head) {
9d4fb27d
JP		 141 if (net_eq(sock_net(sk2), net) &&
142 sk2 != sk &&
d4cada4a 143 (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
9d4fb27d
JP		 144 (!sk2->sk_reuse || !sk->sk_reuse) &&
145 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
146 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
fe38d2a1 147 inet_rcv_saddr_equal(sk, sk2, true)) {
df560056
EG		 148 if (sk2->sk_reuseport && sk->sk_reuseport &&
149 !rcu_access_pointer(sk->sk_reuseport_cb) &&
150 uid_eq(uid, sock_i_uid(sk2))) {
151 if (!bitmap)
152 return 0;
153 } else {
154 if (!bitmap)
155 return 1;
156 __set_bit(udp_sk(sk2)->udp_port_hash >> log,
157 bitmap);
158 }
98322f22 159 }
4243cdc2 160 }
25030a7f
GR		 161 return 0;
162}
163
30fff923
ED		 164/*
165 * Note: we still hold spinlock of primary hash chain, so no other writer
166 * can insert/delete a socket with local_port == num
167 */
168static int udp_lib_lport_inuse2(struct net *net, __u16 num,
4243cdc2 169 struct udp_hslot *hslot2,
fe38d2a1 170 struct sock *sk)
30fff923
ED		 171{
172 struct sock *sk2;
ba418fa3 173 kuid_t uid = sock_i_uid(sk);
30fff923
ED		 174 int res = 0;
175
176 spin_lock(&hslot2->lock);
ca065d0c 177 udp_portaddr_for_each_entry(sk2, &hslot2->head) {
9d4fb27d
JP		 178 if (net_eq(sock_net(sk2), net) &&
179 sk2 != sk &&
180 (udp_sk(sk2)->udp_port_hash == num) &&
181 (!sk2->sk_reuse || !sk->sk_reuse) &&
182 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
183 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
fe38d2a1 184 inet_rcv_saddr_equal(sk, sk2, true)) {
df560056
EG		 185 if (sk2->sk_reuseport && sk->sk_reuseport &&
186 !rcu_access_pointer(sk->sk_reuseport_cb) &&
187 uid_eq(uid, sock_i_uid(sk2))) {
188 res = 0;
189 } else {
190 res = 1;
191 }
30fff923
ED		 192 break;
193 }
4243cdc2 194 }
30fff923
ED		 195 spin_unlock(&hslot2->lock);
196 return res;
197}
198
fe38d2a1 199static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
e32ea7e7
CG		 200{
201 struct net *net = sock_net(sk);
e32ea7e7
CG		 202 kuid_t uid = sock_i_uid(sk);
203 struct sock *sk2;
204
ca065d0c 205 sk_for_each(sk2, &hslot->head) {
e32ea7e7
CG		 206 if (net_eq(sock_net(sk2), net) &&
207 sk2 != sk &&
208 sk2->sk_family == sk->sk_family &&
209 ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
210 (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
211 (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
212 sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
fe38d2a1 213 inet_rcv_saddr_equal(sk, sk2, false)) {
2dbb9b9e
MKL		 214 return reuseport_add_sock(sk, sk2,
215 inet_rcv_saddr_any(sk));
e32ea7e7
CG		 216 }
217 }
218
2dbb9b9e 219 return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
e32ea7e7
CG		 220}
221
25030a7f 222/**
6ba5a3c5 223 * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
25030a7f
GR		 224 *
225 * @sk: socket struct in question
226 * @snum: port number to look up
25985edc 227 * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
30fff923 228 * with NULL address
25030a7f 229 */
6ba5a3c5 230int udp_lib_get_port(struct sock *sk, unsigned short snum,
30fff923 231 unsigned int hash2_nulladdr)
25030a7f 232{
512615b6 233 struct udp_hslot *hslot, *hslot2;
645ca708 234 struct udp_table *udptable = sk->sk_prot->h.udp_table;
25030a7f 235 int error = 1;
3b1e0a65 236 struct net *net = sock_net(sk);
1da177e4 237
32c1da70 238 if (!snum) {
9088c560 239 int low, high, remaining;
95c96174 240 unsigned int rand;
98322f22
ED		 241 unsigned short first, last;
242 DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
32c1da70 243
0bbf87d8 244 inet_get_local_port_range(net, &low, &high);
a25de534 245 remaining = (high - low) + 1;
227b60f5 246
63862b5b 247 rand = prandom_u32();
8fc54f68 248 first = reciprocal_scale(rand, remaining) + low;
98322f22
ED		 249 /*
250 * force rand to be an odd multiple of UDP_HTABLE_SIZE
251 */
f86dcc5a 252 rand = (rand | 1) * (udptable->mask + 1);
5781b235
ED		 253 last = first + udptable->mask + 1;
254 do {
f86dcc5a 255 hslot = udp_hashslot(udptable, net, first);
98322f22 256 bitmap_zero(bitmap, PORTS_PER_CHAIN);
645ca708 257 spin_lock_bh(&hslot->lock);
98322f22 258 udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
fe38d2a1 259 udptable->log);
98322f22
ED		 260
261 snum = first;
262 /*
263 * Iterate on all possible values of snum for this hash.
264 * Using steps of an odd multiple of UDP_HTABLE_SIZE
265 * give us randomization and full range coverage.
266 */
9088c560 267 do {
98322f22 268 if (low <= snum && snum <= high &&
e3826f1e 269 !test_bit(snum >> udptable->log, bitmap) &&
122ff243 270 !inet_is_local_reserved_port(net, snum))
98322f22
ED		 271 goto found;
272 snum += rand;
273 } while (snum != first);
274 spin_unlock_bh(&hslot->lock);
df560056 275 cond_resched();
5781b235 276 } while (++first != last);
98322f22 277 goto fail;
645ca708 278 } else {
f86dcc5a 279 hslot = udp_hashslot(udptable, net, snum);
645ca708 280 spin_lock_bh(&hslot->lock);
30fff923
ED		 281 if (hslot->count > 10) {
282 int exist;
283 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
284
285 slot2 &= udptable->mask;
286 hash2_nulladdr &= udptable->mask;
287
288 hslot2 = udp_hashslot2(udptable, slot2);
289 if (hslot->count < hslot2->count)
290 goto scan_primary_hash;
291
fe38d2a1 292 exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
30fff923
ED		 293 if (!exist && (hash2_nulladdr != slot2)) {
294 hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
295 exist = udp_lib_lport_inuse2(net, snum, hslot2,
fe38d2a1 296 sk);
30fff923
ED		 297 }
298 if (exist)
299 goto fail_unlock;
300 else
301 goto found;
302 }
303scan_primary_hash:
fe38d2a1 304 if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
645ca708
ED		 305 goto fail_unlock;
306 }
98322f22 307found:
c720c7e8 308 inet_sk(sk)->inet_num = snum;
d4cada4a
ED		 309 udp_sk(sk)->udp_port_hash = snum;
310 udp_sk(sk)->udp_portaddr_hash ^= snum;
1da177e4 311 if (sk_unhashed(sk)) {
e32ea7e7 312 if (sk->sk_reuseport &&
fe38d2a1 313 udp_reuseport_add_sock(sk, hslot)) {
e32ea7e7
CG		 314 inet_sk(sk)->inet_num = 0;
315 udp_sk(sk)->udp_port_hash = 0;
316 udp_sk(sk)->udp_portaddr_hash ^= snum;
317 goto fail_unlock;
318 }
319
ca065d0c 320 sk_add_node_rcu(sk, &hslot->head);
fdcc8aa9 321 hslot->count++;
c29a0bc4 322 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
512615b6
ED		 323
324 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
325 spin_lock(&hslot2->lock);
d894ba18 326 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
1602f49b
DM		 327 sk->sk_family == AF_INET6)
328 hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
329 &hslot2->head);
d894ba18 330 else
1602f49b
DM		 331 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
332 &hslot2->head);
512615b6
ED		 333 hslot2->count++;
334 spin_unlock(&hslot2->lock);
1da177e4 335 }
ca065d0c 336 sock_set_flag(sk, SOCK_RCU_FREE);
25030a7f 337 error = 0;
645ca708
ED		 338fail_unlock:
339 spin_unlock_bh(&hslot->lock);
1da177e4 340fail:
25030a7f
GR		 341 return error;
342}
c482c568 343EXPORT_SYMBOL(udp_lib_get_port);
25030a7f 344
6ba5a3c5 345int udp_v4_get_port(struct sock *sk, unsigned short snum)
db8dac20 346{
30fff923 347 unsigned int hash2_nulladdr =
f0b1e64c 348 ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
30fff923 349 unsigned int hash2_partial =
f0b1e64c 350 ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
30fff923 351
d4cada4a 352 /* precompute partial secondary hash */
30fff923 353 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
fe38d2a1 354 return udp_lib_get_port(sk, snum, hash2_nulladdr);
db8dac20
DM		 355}
356
d1e37288
SX		 357static int compute_score(struct sock *sk, struct net *net,
358 __be32 saddr, __be16 sport,
fb74c277 359 __be32 daddr, unsigned short hnum,
73545373 360 int dif, int sdif)
645ca708 361{
60c04aec
JP		 362 int score;
363 struct inet_sock *inet;
6da5b0f0 364 bool dev_match;
645ca708 365
60c04aec
JP		 366 if (!net_eq(sock_net(sk), net) ||
367 udp_sk(sk)->udp_port_hash != hnum ||
368 ipv6_only_sock(sk))
369 return -1;
645ca708 370
4cdeeee9
PO		 371 if (sk->sk_rcv_saddr != daddr)
372 return -1;
60c04aec 373
4cdeeee9 374 score = (sk->sk_family == PF_INET) ? 2 : 1;
60c04aec 375
4cdeeee9 376 inet = inet_sk(sk);
60c04aec
JP		 377 if (inet->inet_daddr) {
378 if (inet->inet_daddr != saddr)
379 return -1;
380 score += 4;
381 }
382
383 if (inet->inet_dport) {
384 if (inet->inet_dport != sport)
385 return -1;
386 score += 4;
387 }
388
6da5b0f0
MM		 389 dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
390 dif, sdif);
391 if (!dev_match)
392 return -1;
393 score += 4;
fb74c277 394
7170a977 395 if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
70da268b 396 score++;
645ca708
ED		 397 return score;
398}
399
6eada011
ED		 400static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
401 const __u16 lport, const __be32 faddr,
402 const __be16 fport)
65cd8033 403{
1bbdceef
HFS		 404 static u32 udp_ehash_secret __read_mostly;
405
406 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
407
65cd8033 408 return __inet_ehashfn(laddr, lport, faddr, fport,
1bbdceef 409 udp_ehash_secret + net_hash_mix(net));
65cd8033
HFS		 410}
411
a57066b1
DM		 412static struct sock *lookup_reuseport(struct net *net, struct sock *sk,
413 struct sk_buff *skb,
414 __be32 saddr, __be16 sport,
415 __be32 daddr, unsigned short hnum)
7629c73a
JS		 416{
417 struct sock *reuse_sk = NULL;
418 u32 hash;
419
420 if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) {
421 hash = udp_ehashfn(net, daddr, hnum, saddr, sport);
422 reuse_sk = reuseport_select_sock(sk, hash, skb,
423 sizeof(struct udphdr));
7629c73a
JS		 424 }
425 return reuse_sk;
426}
427
d1e37288 428/* called with rcu_read_lock() */
5051ebd2 429static struct sock *udp4_lib_lookup2(struct net *net,
fb74c277
DA		 430 __be32 saddr, __be16 sport,
431 __be32 daddr, unsigned int hnum,
73545373 432 int dif, int sdif,
fb74c277
DA		 433 struct udp_hslot *hslot2,
434 struct sk_buff *skb)
5051ebd2
ED		 435{
436 struct sock *sk, *result;
e94a62f5 437 int score, badness;
5051ebd2 438
5051ebd2 439 result = NULL;
ba418fa3 440 badness = 0;
ca065d0c 441 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
d1e37288 442 score = compute_score(sk, net, saddr, sport,
73545373 443 daddr, hnum, dif, sdif);
5051ebd2 444 if (score > badness) {
7629c73a
JS		 445 result = lookup_reuseport(net, sk, skb,
446 saddr, sport, daddr, hnum);
a57066b1
DM		 447 /* Fall back to scoring if group has connections */
448 if (result && !reuseport_has_conns(sk, false))
7629c73a
JS		 449 return result;
450
a57066b1 451 result = result ? : sk;
ca065d0c 452 badness = score;
5051ebd2
ED		 453 }
454 }
5051ebd2
ED		 455 return result;
456}
457
a57066b1
DM		 458static struct sock *udp4_lookup_run_bpf(struct net *net,
459 struct udp_table *udptable,
460 struct sk_buff *skb,
461 __be32 saddr, __be16 sport,
462 __be32 daddr, u16 hnum)
72f7e944
JS		 463{
464 struct sock *sk, *reuse_sk;
465 bool no_reuseport;
466
467 if (udptable != &udp_table)
468 return NULL; /* only UDP is supported */
469
470 no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_UDP,
471 saddr, sport, daddr, hnum, &sk);
472 if (no_reuseport || IS_ERR_OR_NULL(sk))
473 return sk;
474
475 reuse_sk = lookup_reuseport(net, sk, skb, saddr, sport, daddr, hnum);
a57066b1 476 if (reuse_sk && !reuseport_has_conns(sk, false))
72f7e944
JS		 477 sk = reuse_sk;
478 return sk;
479}
480
db8dac20
DM		 481/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
482 * harder than this. -DaveM
483 */
fce82338 484struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
fb74c277
DA		 485 __be16 sport, __be32 daddr, __be16 dport, int dif,
486 int sdif, struct udp_table *udptable, struct sk_buff *skb)
db8dac20 487{
db8dac20 488 unsigned short hnum = ntohs(dport);
4cdeeee9
PO		 489 unsigned int hash2, slot2;
490 struct udp_hslot *hslot2;
72f7e944 491 struct sock *result, *sk;
645ca708 492
4cdeeee9
PO		 493 hash2 = ipv4_portaddr_hash(net, daddr, hnum);
494 slot2 = hash2 & udptable->mask;
495 hslot2 = &udptable->hash2[slot2];
496
72f7e944 497 /* Lookup connected or non-wildcard socket */
4cdeeee9
PO		 498 result = udp4_lib_lookup2(net, saddr, sport,
499 daddr, hnum, dif, sdif,
73545373 500 hslot2, skb);
72f7e944
JS		 501 if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
502 goto done;
503
504 /* Lookup redirect from BPF */
505 if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
506 sk = udp4_lookup_run_bpf(net, udptable, skb,
507 saddr, sport, daddr, hnum);
508 if (sk) {
509 result = sk;
510 goto done;
511 }
db8dac20 512 }
72f7e944
JS		 513
514 /* Got non-wildcard socket or error on first lookup */
515 if (result)
516 goto done;
517
518 /* Lookup wildcard sockets */
519 hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
520 slot2 = hash2 & udptable->mask;
521 hslot2 = &udptable->hash2[slot2];
522
523 result = udp4_lib_lookup2(net, saddr, sport,
524 htonl(INADDR_ANY), hnum, dif, sdif,
525 hslot2, skb);
526done:
88e235b8 527 if (IS_ERR(result))
4cdeeee9 528 return NULL;
db8dac20
DM		 529 return result;
530}
fce82338 531EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
db8dac20 532
607c4aaf
KK		 533static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
534 __be16 sport, __be16 dport,
645ca708 535 struct udp_table *udptable)
607c4aaf
KK		 536{
537 const struct iphdr *iph = ip_hdr(skb);
538
ed7cbbce 539 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
8afdd99a 540 iph->daddr, dport, inet_iif(skb),
fb74c277 541 inet_sdif(skb), udptable, skb);
607c4aaf
KK		 542}
543
63058308
TH		 544struct sock *udp4_lib_lookup_skb(struct sk_buff *skb,
545 __be16 sport, __be16 dport)
546{
257a525f
MKL		 547 const struct iphdr *iph = ip_hdr(skb);
548
549 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
550 iph->daddr, dport, inet_iif(skb),
551 inet_sdif(skb), &udp_table, NULL);
63058308
TH		 552}
553EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb);
554
ca065d0c
ED		 555/* Must be called under rcu_read_lock().
556 * Does increment socket refcount.
557 */
6e86000c 558#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
bcd41303
KK		 559struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
560 __be32 daddr, __be16 dport, int dif)
561{
ca065d0c
ED		 562 struct sock *sk;
563
564 sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
fb74c277 565 dif, 0, &udp_table, NULL);
41c6d650 566 if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
ca065d0c
ED		 567 sk = NULL;
568 return sk;
bcd41303
KK		 569}
570EXPORT_SYMBOL_GPL(udp4_lib_lookup);
ca065d0c 571#endif
bcd41303 572
421b3885
SB		 573static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
574 __be16 loc_port, __be32 loc_addr,
575 __be16 rmt_port, __be32 rmt_addr,
fb74c277 576 int dif, int sdif, unsigned short hnum)
421b3885
SB		 577{
578 struct inet_sock *inet = inet_sk(sk);
579
580 if (!net_eq(sock_net(sk), net) ||
581 udp_sk(sk)->udp_port_hash != hnum ||
582 (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
583 (inet->inet_dport != rmt_port && inet->inet_dport) ||
584 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
585 ipv6_only_sock(sk) ||
82ba25c6 586 !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
421b3885 587 return false;
60d9b031 588 if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
421b3885
SB		 589 return false;
590 return true;
591}
592
a36e185e
SB		 593DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
594void udp_encap_enable(void)
595{
9c480601 596 static_branch_inc(&udp_encap_needed_key);
a36e185e
SB		 597}
598EXPORT_SYMBOL(udp_encap_enable);
599
e7cc0824
SB		 600/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
601 * through error handlers in encapsulations looking for a match.
602 */
603static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
604{
605 int i;
606
607 for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
608 int (*handler)(struct sk_buff *skb, u32 info);
92b95364 609 const struct ip_tunnel_encap_ops *encap;
e7cc0824 610
92b95364
PA		 611 encap = rcu_dereference(iptun_encaps[i]);
612 if (!encap)
e7cc0824 613 continue;
92b95364 614 handler = encap->err_handler;
e7cc0824
SB		 615 if (handler && !handler(skb, info))
616 return 0;
617 }
618
619 return -ENOENT;
620}
621
a36e185e
SB		 622/* Try to match ICMP errors to UDP tunnels by looking up a socket without
623 * reversing source and destination port: this will match tunnels that force the
624 * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
625 * lwtunnels might actually break this assumption by being configured with
626 * different destination ports on endpoints, in this case we won't be able to
627 * trace ICMP messages back to them.
628 *
e7cc0824
SB		 629 * If this doesn't match any socket, probe tunnels with arbitrary destination
630 * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
631 * we've sent packets to won't necessarily match the local destination port.
632 *
a36e185e
SB		 633 * Then ask the tunnel implementation to match the error against a valid
634 * association.
635 *
e7cc0824
SB		 636 * Return an error if we can't find a match, the socket if we need further
637 * processing, zero otherwise.
a36e185e
SB		 638 */
639static struct sock *__udp4_lib_err_encap(struct net *net,
640 const struct iphdr *iph,
641 struct udphdr *uh,
642 struct udp_table *udptable,
e7cc0824 643 struct sk_buff *skb, u32 info)
a36e185e 644{
a36e185e 645 int network_offset, transport_offset;
a36e185e
SB		 646 struct sock *sk;
647
a36e185e
SB		 648 network_offset = skb_network_offset(skb);
649 transport_offset = skb_transport_offset(skb);
650
651 /* Network header needs to point to the outer IPv4 header inside ICMP */
652 skb_reset_network_header(skb);
653
654 /* Transport header needs to point to the UDP header */
655 skb_set_transport_header(skb, iph->ihl << 2);
656
e7cc0824
SB		 657 sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
658 iph->saddr, uh->dest, skb->dev->ifindex, 0,
659 udptable, NULL);
660 if (sk) {
661 int (*lookup)(struct sock *sk, struct sk_buff *skb);
662 struct udp_sock *up = udp_sk(sk);
663
664 lookup = READ_ONCE(up->encap_err_lookup);
665 if (!lookup || lookup(sk, skb))
666 sk = NULL;
667 }
668
669 if (!sk)
670 sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
a36e185e
SB		 671
672 skb_set_transport_header(skb, transport_offset);
673 skb_set_network_header(skb, network_offset);
674
675 return sk;
676}
677
db8dac20
DM		 678/*
679 * This routine is called by the ICMP module when it gets some
680 * sort of error condition. If err < 0 then the socket should
681 * be closed and the error returned to the user. If err > 0
682 * it's just the icmp type << 8 | icmp code.
683 * Header points to the ip header of the error packet. We move
684 * on past this. Then (as it used to claim before adjustment)
685 * header points to the first 8 bytes of the udp header. We need
686 * to find the appropriate port.
687 */
688
32bbd879 689int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
db8dac20
DM		 690{
691 struct inet_sock *inet;
b71d1d42 692 const struct iphdr *iph = (const struct iphdr *)skb->data;
c482c568 693 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
db8dac20
DM		 694 const int type = icmp_hdr(skb)->type;
695 const int code = icmp_hdr(skb)->code;
a36e185e 696 bool tunnel = false;
db8dac20
DM		 697 struct sock *sk;
698 int harderr;
699 int err;
fd54d716 700 struct net *net = dev_net(skb->dev);
db8dac20 701
fd54d716 702 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
f64bf6b8
MM		 703 iph->saddr, uh->source, skb->dev->ifindex,
704 inet_sdif(skb), udptable, NULL);
51456b29 705 if (!sk) {
a36e185e 706 /* No socket for error: try tunnels before discarding */
e7cc0824
SB		 707 sk = ERR_PTR(-ENOENT);
708 if (static_branch_unlikely(&udp_encap_needed_key)) {
709 sk = __udp4_lib_err_encap(net, iph, uh, udptable, skb,
710 info);
711 if (!sk)
712 return 0;
713 }
a36e185e 714
e7cc0824 715 if (IS_ERR(sk)) {
a36e185e 716 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
e7cc0824 717 return PTR_ERR(sk);
a36e185e 718 }
e7cc0824 719
a36e185e 720 tunnel = true;
db8dac20
DM		 721 }
722
723 err = 0;
724 harderr = 0;
725 inet = inet_sk(sk);
726
727 switch (type) {
728 default:
729 case ICMP_TIME_EXCEEDED:
730 err = EHOSTUNREACH;
731 break;
732 case ICMP_SOURCE_QUENCH:
733 goto out;
734 case ICMP_PARAMETERPROB:
735 err = EPROTO;
736 harderr = 1;
737 break;
738 case ICMP_DEST_UNREACH:
739 if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
36393395 740 ipv4_sk_update_pmtu(skb, sk, info);
db8dac20
DM		 741 if (inet->pmtudisc != IP_PMTUDISC_DONT) {
742 err = EMSGSIZE;
743 harderr = 1;
744 break;
745 }
746 goto out;
747 }
748 err = EHOSTUNREACH;
749 if (code <= NR_ICMP_UNREACH) {
750 harderr = icmp_err_convert[code].fatal;
751 err = icmp_err_convert[code].errno;
752 }
753 break;
55be7a9c
DM		 754 case ICMP_REDIRECT:
755 ipv4_sk_redirect(skb, sk);
1a462d18 756 goto out;
db8dac20
DM		 757 }
758
759 /*
760 * RFC1122: OK. Passes ICMP errors back to application, as per
761 * 4.1.3.3.
762 */
a36e185e
SB		 763 if (tunnel) {
764 /* ...not for tunnels though: we don't have a sending socket */
765 goto out;
766 }
db8dac20
DM		 767 if (!inet->recverr) {
768 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
769 goto out;
b1faf566 770 } else
c482c568 771 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
b1faf566 772
db8dac20
DM		 773 sk->sk_err = err;
774 sk->sk_error_report(sk);
775out:
32bbd879 776 return 0;
db8dac20
DM		 777}
778
32bbd879 779int udp_err(struct sk_buff *skb, u32 info)
db8dac20 780{
32bbd879 781 return __udp4_lib_err(skb, info, &udp_table);
db8dac20
DM		 782}
783
784/*
785 * Throw away all pending data and cancel the corking. Socket is locked.
786 */
36d926b9 787void udp_flush_pending_frames(struct sock *sk)
db8dac20
DM		 788{
789 struct udp_sock *up = udp_sk(sk);
790
791 if (up->pending) {
792 up->len = 0;
793 up->pending = 0;
794 ip_flush_pending_frames(sk);
795 }
796}
36d926b9 797EXPORT_SYMBOL(udp_flush_pending_frames);
db8dac20
DM		 798
799/**
f6b9664f 800 * udp4_hwcsum - handle outgoing HW checksumming
db8dac20
DM		 801 * @skb: sk_buff containing the filled-in UDP header
802 * (checksum field must be zeroed out)
f6b9664f
HX		 803 * @src: source IP address
804 * @dst: destination IP address
db8dac20 805 */
c26bf4a5 806void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
db8dac20 807{
db8dac20 808 struct udphdr *uh = udp_hdr(skb);
f6b9664f
HX		 809 int offset = skb_transport_offset(skb);
810 int len = skb->len - offset;
811 int hlen = len;
db8dac20
DM		 812 __wsum csum = 0;
813
ebbe495f 814 if (!skb_has_frag_list(skb)) {
db8dac20
DM		 815 /*
816 * Only one fragment on the socket.
817 */
818 skb->csum_start = skb_transport_header(skb) - skb->head;
819 skb->csum_offset = offsetof(struct udphdr, check);
f6b9664f
HX		 820 uh->check = ~csum_tcpudp_magic(src, dst, len,
821 IPPROTO_UDP, 0);
db8dac20 822 } else {
ebbe495f
WC		 823 struct sk_buff *frags;
824
db8dac20
DM		 825 /*
826 * HW-checksum won't work as there are two or more
827 * fragments on the socket so that all csums of sk_buffs
828 * should be together
829 */
ebbe495f 830 skb_walk_frags(skb, frags) {
f6b9664f
HX		 831 csum = csum_add(csum, frags->csum);
832 hlen -= frags->len;
ebbe495f 833 }
db8dac20 834
f6b9664f 835 csum = skb_checksum(skb, offset, hlen, csum);
db8dac20
DM		 836 skb->ip_summed = CHECKSUM_NONE;
837
db8dac20
DM		 838 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
839 if (uh->check == 0)
840 uh->check = CSUM_MANGLED_0;
841 }
842}
c26bf4a5 843EXPORT_SYMBOL_GPL(udp4_hwcsum);
db8dac20 844
af5fcba7
TH		 845/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
846 * for the simple case like when setting the checksum for a UDP tunnel.
847 */
848void udp_set_csum(bool nocheck, struct sk_buff *skb,
849 __be32 saddr, __be32 daddr, int len)
850{
851 struct udphdr *uh = udp_hdr(skb);
852
179bc67f 853 if (nocheck) {
af5fcba7 854 uh->check = 0;
179bc67f 855 } else if (skb_is_gso(skb)) {
af5fcba7 856 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
179bc67f
EC		 857 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
858 uh->check = 0;
859 uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
860 if (uh->check == 0)
861 uh->check = CSUM_MANGLED_0;
d75f1306 862 } else {
af5fcba7
TH		 863 skb->ip_summed = CHECKSUM_PARTIAL;
864 skb->csum_start = skb_transport_header(skb) - skb->head;
865 skb->csum_offset = offsetof(struct udphdr, check);
866 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
af5fcba7
TH		 867 }
868}
869EXPORT_SYMBOL(udp_set_csum);
870
bec1f6f6
WB		 871static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
872 struct inet_cork *cork)
db8dac20 873{
f6b9664f 874 struct sock *sk = skb->sk;
db8dac20 875 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM		 876 struct udphdr *uh;
877 int err = 0;
878 int is_udplite = IS_UDPLITE(sk);
f6b9664f
HX		 879 int offset = skb_transport_offset(skb);
880 int len = skb->len - offset;
4094871d 881 int datalen = len - sizeof(*uh);
db8dac20
DM		 882 __wsum csum = 0;
883
db8dac20
DM		 884 /*
885 * Create a UDP header
886 */
887 uh = udp_hdr(skb);
f6b9664f 888 uh->source = inet->inet_sport;
79ab0531 889 uh->dest = fl4->fl4_dport;
f6b9664f 890 uh->len = htons(len);
db8dac20
DM		 891 uh->check = 0;
892
bec1f6f6
WB		 893 if (cork->gso_size) {
894 const int hlen = skb_network_header_len(skb) +
895 sizeof(struct udphdr);
896
0f149c9f
WB		 897 if (hlen + cork->gso_size > cork->fragsize) {
898 kfree_skb(skb);
bec1f6f6 899 return -EINVAL;
0f149c9f
WB		 900 }
901 if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) {
902 kfree_skb(skb);
bec1f6f6 903 return -EINVAL;
0f149c9f
WB		 904 }
905 if (sk->sk_no_check_tx) {
906 kfree_skb(skb);
a8c744a8 907 return -EINVAL;
0f149c9f 908 }
ff06342c 909 if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
0f149c9f
WB		 910 dst_xfrm(skb_dst(skb))) {
911 kfree_skb(skb);
bec1f6f6 912 return -EIO;
0f149c9f 913 }
bec1f6f6 914
4094871d
JH		 915 if (datalen > cork->gso_size) {
916 skb_shinfo(skb)->gso_size = cork->gso_size;
917 skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
918 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
919 cork->gso_size);
920 }
a8c744a8 921 goto csum_partial;
bec1f6f6
WB		 922 }
923
db8dac20 924 if (is_udplite) /* UDP-Lite */
f6b9664f 925 csum = udplite_csum(skb);
db8dac20 926
ab2fb7e3 927 else if (sk->sk_no_check_tx) { /* UDP csum off */
db8dac20
DM		 928
929 skb->ip_summed = CHECKSUM_NONE;
930 goto send;
931
932 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
a8c744a8 933csum_partial:
db8dac20 934
79ab0531 935 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
db8dac20
DM		 936 goto send;
937
f6b9664f
HX		 938 } else
939 csum = udp_csum(skb);
db8dac20
DM		 940
941 /* add protocol-dependent pseudo-header */
79ab0531 942 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
c482c568 943 sk->sk_protocol, csum);
db8dac20
DM		 944 if (uh->check == 0)
945 uh->check = CSUM_MANGLED_0;
946
947send:
b5ec8eea 948 err = ip_send_skb(sock_net(sk), skb);
6ce9e7b5
ED		 949 if (err) {
950 if (err == -ENOBUFS && !inet->recverr) {
6aef70a8
ED		 951 UDP_INC_STATS(sock_net(sk),
952 UDP_MIB_SNDBUFERRORS, is_udplite);
6ce9e7b5
ED		 953 err = 0;
954 }
955 } else
6aef70a8
ED		 956 UDP_INC_STATS(sock_net(sk),
957 UDP_MIB_OUTDATAGRAMS, is_udplite);
f6b9664f
HX		 958 return err;
959}
960
961/*
962 * Push out all pending data as one UDP datagram. Socket is locked.
963 */
8822b64a 964int udp_push_pending_frames(struct sock *sk)
f6b9664f
HX		 965{
966 struct udp_sock *up = udp_sk(sk);
967 struct inet_sock *inet = inet_sk(sk);
b6f21b26 968 struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
f6b9664f
HX		 969 struct sk_buff *skb;
970 int err = 0;
971
77968b78 972 skb = ip_finish_skb(sk, fl4);
f6b9664f
HX		 973 if (!skb)
974 goto out;
975
bec1f6f6 976 err = udp_send_skb(skb, fl4, &inet->cork.base);
f6b9664f 977
db8dac20
DM		 978out:
979 up->len = 0;
980 up->pending = 0;
db8dac20
DM		 981 return err;
982}
8822b64a 983EXPORT_SYMBOL(udp_push_pending_frames);
db8dac20 984
2e8de857
WB		 985static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
986{
987 switch (cmsg->cmsg_type) {
988 case UDP_SEGMENT:
989 if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
990 return -EINVAL;
991 *gso_size = *(__u16 *)CMSG_DATA(cmsg);
992 return 0;
993 default:
994 return -EINVAL;
995 }
996}
997
998int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
999{
1000 struct cmsghdr *cmsg;
1001 bool need_ip = false;
1002 int err;
1003
1004 for_each_cmsghdr(cmsg, msg) {
1005 if (!CMSG_OK(msg, cmsg))
1006 return -EINVAL;
1007
1008 if (cmsg->cmsg_level != SOL_UDP) {
1009 need_ip = true;
1010 continue;
1011 }
1012
1013 err = __udp_cmsg_send(cmsg, gso_size);
1014 if (err)
1015 return err;
1016 }
1017
1018 return need_ip;
1019}
1020EXPORT_SYMBOL_GPL(udp_cmsg_send);
1021
1b784140 1022int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
db8dac20
DM		 1023{
1024 struct inet_sock *inet = inet_sk(sk);
1025 struct udp_sock *up = udp_sk(sk);
1cedee13 1026 DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
e474995f 1027 struct flowi4 fl4_stack;
b6f21b26 1028 struct flowi4 *fl4;
db8dac20
DM		 1029 int ulen = len;
1030 struct ipcm_cookie ipc;
1031 struct rtable *rt = NULL;
1032 int free = 0;
1033 int connected = 0;
1034 __be32 daddr, faddr, saddr;
1035 __be16 dport;
1036 u8 tos;
1037 int err, is_udplite = IS_UDPLITE(sk);
1038 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
1039 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
903ab86d 1040 struct sk_buff *skb;
f6d8bd05 1041 struct ip_options_data opt_copy;
db8dac20
DM		 1042
1043 if (len > 0xFFFF)
1044 return -EMSGSIZE;
1045
1046 /*
1047 * Check the flags.
1048 */
1049
c482c568 1050 if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
db8dac20
DM		 1051 return -EOPNOTSUPP;
1052
903ab86d
HX		 1053 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
1054
f5fca608 1055 fl4 = &inet->cork.fl.u.ip4;
db8dac20
DM		 1056 if (up->pending) {
1057 /*
1058 * There are pending frames.
1059 * The socket lock must be held while it's corked.
1060 */
1061 lock_sock(sk);
1062 if (likely(up->pending)) {
1063 if (unlikely(up->pending != AF_INET)) {
1064 release_sock(sk);
1065 return -EINVAL;
1066 }
1067 goto do_append_data;
1068 }
1069 release_sock(sk);
1070 }
1071 ulen += sizeof(struct udphdr);
1072
1073 /*
1074 * Get and verify the address.
1075 */
1cedee13 1076 if (usin) {
db8dac20
DM		 1077 if (msg->msg_namelen < sizeof(*usin))
1078 return -EINVAL;
1079 if (usin->sin_family != AF_INET) {
1080 if (usin->sin_family != AF_UNSPEC)
1081 return -EAFNOSUPPORT;
1082 }
1083
1084 daddr = usin->sin_addr.s_addr;
1085 dport = usin->sin_port;
1086 if (dport == 0)
1087 return -EINVAL;
1088 } else {
1089 if (sk->sk_state != TCP_ESTABLISHED)
1090 return -EDESTADDRREQ;
c720c7e8
ED		 1091 daddr = inet->inet_daddr;
1092 dport = inet->inet_dport;
db8dac20
DM		 1093 /* Open fast path for connected socket.
1094 Route will not be used, if at least one option is set.
1095 */
1096 connected = 1;
1097 }
db8dac20 1098
35178206 1099 ipcm_init_sk(&ipc, inet);
bec1f6f6 1100 ipc.gso_size = up->gso_size;
bf84a010 1101
db8dac20 1102 if (msg->msg_controllen) {
2e8de857
WB		 1103 err = udp_cmsg_send(sk, msg, &ipc.gso_size);
1104 if (err > 0)
1105 err = ip_cmsg_send(sk, msg, &ipc,
1106 sk->sk_family == AF_INET6);
1107 if (unlikely(err < 0)) {
91948309 1108 kfree(ipc.opt);
db8dac20 1109 return err;
91948309 1110 }
db8dac20
DM		 1111 if (ipc.opt)
1112 free = 1;
1113 connected = 0;
1114 }
f6d8bd05
ED		 1115 if (!ipc.opt) {
1116 struct ip_options_rcu *inet_opt;
1117
1118 rcu_read_lock();
1119 inet_opt = rcu_dereference(inet->inet_opt);
1120 if (inet_opt) {
1121 memcpy(&opt_copy, inet_opt,
1122 sizeof(*inet_opt) + inet_opt->opt.optlen);
1123 ipc.opt = &opt_copy.opt;
1124 }
1125 rcu_read_unlock();
1126 }
db8dac20 1127
1cedee13
AI		 1128 if (cgroup_bpf_enabled && !connected) {
1129 err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
1130 (struct sockaddr *)usin, &ipc.addr);
1131 if (err)
1132 goto out_free;
1133 if (usin) {
1134 if (usin->sin_port == 0) {
1135 /* BPF program set invalid port. Reject it. */
1136 err = -EINVAL;
1137 goto out_free;
1138 }
1139 daddr = usin->sin_addr.s_addr;
1140 dport = usin->sin_port;
1141 }
1142 }
1143
db8dac20
DM		 1144 saddr = ipc.addr;
1145 ipc.addr = faddr = daddr;
1146
f6d8bd05 1147 if (ipc.opt && ipc.opt->opt.srr) {
1b97013b
AI		 1148 if (!daddr) {
1149 err = -EINVAL;
1150 goto out_free;
1151 }
f6d8bd05 1152 faddr = ipc.opt->opt.faddr;
db8dac20
DM		 1153 connected = 0;
1154 }
aa661581 1155 tos = get_rttos(&ipc, inet);
db8dac20
DM		 1156 if (sock_flag(sk, SOCK_LOCALROUTE) ||
1157 (msg->msg_flags & MSG_DONTROUTE) ||
f6d8bd05 1158 (ipc.opt && ipc.opt->opt.is_strictroute)) {
db8dac20
DM		 1159 tos |= RTO_ONLINK;
1160 connected = 0;
1161 }
1162
1163 if (ipv4_is_multicast(daddr)) {
854da991 1164 if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
db8dac20
DM		 1165 ipc.oif = inet->mc_index;
1166 if (!saddr)
1167 saddr = inet->mc_addr;
1168 connected = 0;
9515a2e0 1169 } else if (!ipc.oif) {
76e21053 1170 ipc.oif = inet->uc_index;
9515a2e0
DA		 1171 } else if (ipv4_is_lbcast(daddr) && inet->uc_index) {
1172 /* oif is set, packet is to local broadcast and
1173 * and uc_index is set. oif is most likely set
1174 * by sk_bound_dev_if. If uc_index != oif check if the
1175 * oif is an L3 master and uc_index is an L3 slave.
1176 * If so, we want to allow the send using the uc_index.
1177 */
1178 if (ipc.oif != inet->uc_index &&
1179 ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
1180 inet->uc_index)) {
1181 ipc.oif = inet->uc_index;
1182 }
1183 }
db8dac20
DM		 1184
1185 if (connected)
c482c568 1186 rt = (struct rtable *)sk_dst_check(sk, 0);
db8dac20 1187
51456b29 1188 if (!rt) {
84a3aa00 1189 struct net *net = sock_net(sk);
9a24abfa 1190 __u8 flow_flags = inet_sk_flowi_flags(sk);
84a3aa00 1191
e474995f 1192 fl4 = &fl4_stack;
9a24abfa 1193
c6af0c22 1194 flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos,
c0951cbc 1195 RT_SCOPE_UNIVERSE, sk->sk_protocol,
9a24abfa 1196 flow_flags,
e2d118a1
LC		 1197 faddr, saddr, dport, inet->inet_sport,
1198 sk->sk_uid);
c0951cbc 1199
e474995f
DM		 1200 security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
1201 rt = ip_route_output_flow(net, fl4, sk);
b23dd4fe
DM		 1202 if (IS_ERR(rt)) {
1203 err = PTR_ERR(rt);
06dc94b1 1204 rt = NULL;
db8dac20 1205 if (err == -ENETUNREACH)
f1d8cba6 1206 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
db8dac20
DM		 1207 goto out;
1208 }
1209
1210 err = -EACCES;
1211 if ((rt->rt_flags & RTCF_BROADCAST) &&
1212 !sock_flag(sk, SOCK_BROADCAST))
1213 goto out;
1214 if (connected)
d8d1f30b 1215 sk_dst_set(sk, dst_clone(&rt->dst));
db8dac20
DM		 1216 }
1217
1218 if (msg->msg_flags&MSG_CONFIRM)
1219 goto do_confirm;
1220back_from_confirm:
1221
e474995f 1222 saddr = fl4->saddr;
db8dac20 1223 if (!ipc.addr)
e474995f 1224 daddr = ipc.addr = fl4->daddr;
db8dac20 1225
903ab86d
HX		 1226 /* Lockless fast path for the non-corking case. */
1227 if (!corkreq) {
1cd7884d
WB		 1228 struct inet_cork cork;
1229
f69e6d13 1230 skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
903ab86d 1231 sizeof(struct udphdr), &ipc, &rt,
1cd7884d 1232 &cork, msg->msg_flags);
903ab86d 1233 err = PTR_ERR(skb);
50c3a487 1234 if (!IS_ERR_OR_NULL(skb))
bec1f6f6 1235 err = udp_send_skb(skb, fl4, &cork);
903ab86d
HX		 1236 goto out;
1237 }
1238
db8dac20
DM		 1239 lock_sock(sk);
1240 if (unlikely(up->pending)) {
1241 /* The socket is already corked while preparing it. */
1242 /* ... which is an evident application bug. --ANK */
1243 release_sock(sk);
1244
197df02c 1245 net_dbg_ratelimited("socket already corked\n");
db8dac20
DM		 1246 err = -EINVAL;
1247 goto out;
1248 }
1249 /*
1250 * Now cork the socket to pend data.
1251 */
b6f21b26
DM		 1252 fl4 = &inet->cork.fl.u.ip4;
1253 fl4->daddr = daddr;
1254 fl4->saddr = saddr;
9cce96df
DM		 1255 fl4->fl4_dport = dport;
1256 fl4->fl4_sport = inet->inet_sport;
db8dac20
DM		 1257 up->pending = AF_INET;
1258
1259do_append_data:
1260 up->len += ulen;
f69e6d13 1261 err = ip_append_data(sk, fl4, getfrag, msg, ulen,
f5fca608
DM		 1262 sizeof(struct udphdr), &ipc, &rt,
1263 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
db8dac20
DM		 1264 if (err)
1265 udp_flush_pending_frames(sk);
1266 else if (!corkreq)
1267 err = udp_push_pending_frames(sk);
1268 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1269 up->pending = 0;
1270 release_sock(sk);
1271
1272out:
1273 ip_rt_put(rt);
1b97013b 1274out_free:
db8dac20
DM		 1275 if (free)
1276 kfree(ipc.opt);
1277 if (!err)
1278 return len;
1279 /*
1280 * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
1281 * ENOBUFS might not be good (it's not tunable per se), but otherwise
1282 * we don't have a good statistic (IpOutDiscards but it can be too many
1283 * things). We could add another new stat but at least for now that
1284 * seems like overkill.
1285 */
1286 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
6aef70a8
ED		 1287 UDP_INC_STATS(sock_net(sk),
1288 UDP_MIB_SNDBUFERRORS, is_udplite);
db8dac20
DM		 1289 }
1290 return err;
1291
1292do_confirm:
0dec879f
JA		 1293 if (msg->msg_flags & MSG_PROBE)
1294 dst_confirm_neigh(&rt->dst, &fl4->daddr);
db8dac20
DM		 1295 if (!(msg->msg_flags&MSG_PROBE) || len)
1296 goto back_from_confirm;
1297 err = 0;
1298 goto out;
1299}
c482c568 1300EXPORT_SYMBOL(udp_sendmsg);
db8dac20
DM		 1301
1302int udp_sendpage(struct sock *sk, struct page *page, int offset,
1303 size_t size, int flags)
1304{
f5fca608 1305 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM		 1306 struct udp_sock *up = udp_sk(sk);
1307 int ret;
1308
d3f7d56a
SL		 1309 if (flags & MSG_SENDPAGE_NOTLAST)
1310 flags |= MSG_MORE;
1311
db8dac20
DM		 1312 if (!up->pending) {
1313 struct msghdr msg = { .msg_flags = flags|MSG_MORE };
1314
1315 /* Call udp_sendmsg to specify destination address which
1316 * sendpage interface can't pass.
1317 * This will succeed only when the socket is connected.
1318 */
1b784140 1319 ret = udp_sendmsg(sk, &msg, 0);
db8dac20
DM		 1320 if (ret < 0)
1321 return ret;
1322 }
1323
1324 lock_sock(sk);
1325
1326 if (unlikely(!up->pending)) {
1327 release_sock(sk);
1328
197df02c 1329 net_dbg_ratelimited("cork failed\n");
db8dac20
DM		 1330 return -EINVAL;
1331 }
1332
f5fca608
DM		 1333 ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
1334 page, offset, size, flags);
db8dac20
DM		 1335 if (ret == -EOPNOTSUPP) {
1336 release_sock(sk);
1337 return sock_no_sendpage(sk->sk_socket, page, offset,
1338 size, flags);
1339 }
1340 if (ret < 0) {
1341 udp_flush_pending_frames(sk);
1342 goto out;
1343 }
1344
1345 up->len += size;
1346 if (!(up->corkflag || (flags&MSG_MORE)))
1347 ret = udp_push_pending_frames(sk);
1348 if (!ret)
1349 ret = size;
1350out:
1351 release_sock(sk);
1352 return ret;
1353}
1354
dce4551c
PA		 1355#define UDP_SKB_IS_STATELESS 0x80000000
1356
677bf08c
FW		 1357/* all head states (dst, sk, nf conntrack) except skb extensions are
1358 * cleared by udp_rcv().
1359 *
1360 * We need to preserve secpath, if present, to eventually process
1361 * IP_CMSG_PASSSEC at recvmsg() time.
1362 *
1363 * Other extensions can be cleared.
1364 */
1365static bool udp_try_make_stateless(struct sk_buff *skb)
1366{
1367 if (!skb_has_extensions(skb))
1368 return true;
1369
1370 if (!secpath_exists(skb)) {
1371 skb_ext_reset(skb);
1372 return true;
1373 }
1374
1375 return false;
1376}
1377
b65ac446
PA		 1378static void udp_set_dev_scratch(struct sk_buff *skb)
1379{
dce4551c 1380 struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
b65ac446
PA		 1381
1382 BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
dce4551c
PA		 1383 scratch->_tsize_state = skb->truesize;
1384#if BITS_PER_LONG == 64
b65ac446
PA		 1385 scratch->len = skb->len;
1386 scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1387 scratch->is_linear = !skb_is_nonlinear(skb);
dce4551c 1388#endif
677bf08c 1389 if (udp_try_make_stateless(skb))
dce4551c 1390 scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
b65ac446
PA		 1391}
1392
a793183c
ED		 1393static void udp_skb_csum_unnecessary_set(struct sk_buff *skb)
1394{
1395 /* We come here after udp_lib_checksum_complete() returned 0.
1396 * This means that __skb_checksum_complete() might have
1397 * set skb->csum_valid to 1.
1398 * On 64bit platforms, we can set csum_unnecessary
1399 * to true, but only if the skb is not shared.
1400 */
1401#if BITS_PER_LONG == 64
1402 if (!skb_shared(skb))
1403 udp_skb_scratch(skb)->csum_unnecessary = true;
1404#endif
1405}
1406
b65ac446
PA		 1407static int udp_skb_truesize(struct sk_buff *skb)
1408{
dce4551c 1409 return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
b65ac446
PA		 1410}
1411
dce4551c 1412static bool udp_skb_has_head_state(struct sk_buff *skb)
b65ac446 1413{
dce4551c 1414 return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
b65ac446 1415}
b65ac446 1416
7c13f97f 1417/* fully reclaim rmem/fwd memory allocated for skb */
6dfb4367
PA		 1418static void udp_rmem_release(struct sock *sk, int size, int partial,
1419 bool rx_queue_lock_held)
f970bd9e 1420{
6b229cf7 1421 struct udp_sock *up = udp_sk(sk);
2276f58a 1422 struct sk_buff_head *sk_queue;
f970bd9e
PA		 1423 int amt;
1424
6b229cf7
ED		 1425 if (likely(partial)) {
1426 up->forward_deficit += size;
1427 size = up->forward_deficit;
d39ca259
PA		 1428 if (size < (sk->sk_rcvbuf >> 2) &&
1429 !skb_queue_empty(&up->reader_queue))
6b229cf7
ED		 1430 return;
1431 } else {
1432 size += up->forward_deficit;
1433 }
1434 up->forward_deficit = 0;
1435
6dfb4367
PA		 1436 /* acquire the sk_receive_queue for fwd allocated memory scheduling,
1437 * if the called don't held it already
1438 */
2276f58a 1439 sk_queue = &sk->sk_receive_queue;
6dfb4367
PA		 1440 if (!rx_queue_lock_held)
1441 spin_lock(&sk_queue->lock);
1442
2276f58a 1443
f970bd9e
PA		 1444 sk->sk_forward_alloc += size;
1445 amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1);
1446 sk->sk_forward_alloc -= amt;
f970bd9e
PA		 1447
1448 if (amt)
1449 __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT);
02ab0d13
ED		 1450
1451 atomic_sub(size, &sk->sk_rmem_alloc);
2276f58a
PA		 1452
1453 /* this can save us from acquiring the rx queue lock on next receive */
1454 skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
1455
6dfb4367
PA		 1456 if (!rx_queue_lock_held)
1457 spin_unlock(&sk_queue->lock);
f970bd9e
PA		 1458}
1459
2276f58a 1460/* Note: called with reader_queue.lock held.
c84d9490
ED		 1461 * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
1462 * This avoids a cache line miss while receive_queue lock is held.
1463 * Look at __udp_enqueue_schedule_skb() to find where this copy is done.
1464 */
7c13f97f 1465void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
f970bd9e 1466{
b65ac446
PA		 1467 prefetch(&skb->data);
1468 udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
f970bd9e 1469}
7c13f97f 1470EXPORT_SYMBOL(udp_skb_destructor);
f970bd9e 1471
6dfb4367 1472/* as above, but the caller held the rx queue lock, too */
64f5102d 1473static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
6dfb4367 1474{
b65ac446
PA		 1475 prefetch(&skb->data);
1476 udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
6dfb4367
PA		 1477}
1478
4b272750
ED		 1479/* Idea of busylocks is to let producers grab an extra spinlock
1480 * to relieve pressure on the receive_queue spinlock shared by consumer.
1481 * Under flood, this means that only one producer can be in line
1482 * trying to acquire the receive_queue spinlock.
1483 * These busylock can be allocated on a per cpu manner, instead of a
1484 * per socket one (that would consume a cache line per socket)
1485 */
1486static int udp_busylocks_log __read_mostly;
1487static spinlock_t *udp_busylocks __read_mostly;
1488
1489static spinlock_t *busylock_acquire(void *ptr)
1490{
1491 spinlock_t *busy;
1492
1493 busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log);
1494 spin_lock(busy);
1495 return busy;
1496}
1497
1498static void busylock_release(spinlock_t *busy)
1499{
1500 if (busy)
1501 spin_unlock(busy);
1502}
1503
f970bd9e
PA		 1504int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
1505{
1506 struct sk_buff_head *list = &sk->sk_receive_queue;
1507 int rmem, delta, amt, err = -ENOMEM;
4b272750 1508 spinlock_t *busy = NULL;
c8c8b127 1509 int size;
f970bd9e
PA		 1510
1511 /* try to avoid the costly atomic add/sub pair when the receive
1512 * queue is full; always allow at least a packet
1513 */
1514 rmem = atomic_read(&sk->sk_rmem_alloc);
363dc73a 1515 if (rmem > sk->sk_rcvbuf)
f970bd9e
PA		 1516 goto drop;
1517
c8c8b127
ED		 1518 /* Under mem pressure, it might be helpful to help udp_recvmsg()
1519 * having linear skbs :
1520 * - Reduce memory overhead and thus increase receive queue capacity
1521 * - Less cache line misses at copyout() time
1522 * - Less work at consume_skb() (less alien page frag freeing)
1523 */
4b272750 1524 if (rmem > (sk->sk_rcvbuf >> 1)) {
c8c8b127 1525 skb_condense(skb);
4b272750
ED		 1526
1527 busy = busylock_acquire(sk);
1528 }
c8c8b127 1529 size = skb->truesize;
b65ac446 1530 udp_set_dev_scratch(skb);
c8c8b127 1531
f970bd9e
PA		 1532 /* we drop only if the receive buf is full and the receive
1533 * queue contains some other skb
1534 */
1535 rmem = atomic_add_return(size, &sk->sk_rmem_alloc);
feed8a4f 1536 if (rmem > (size + (unsigned int)sk->sk_rcvbuf))
f970bd9e
PA		 1537 goto uncharge_drop;
1538
1539 spin_lock(&list->lock);
1540 if (size >= sk->sk_forward_alloc) {
1541 amt = sk_mem_pages(size);
1542 delta = amt << SK_MEM_QUANTUM_SHIFT;
1543 if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) {
1544 err = -ENOBUFS;
1545 spin_unlock(&list->lock);
1546 goto uncharge_drop;
1547 }
1548
1549 sk->sk_forward_alloc += delta;
1550 }
1551
1552 sk->sk_forward_alloc -= size;
1553
7c13f97f
PA		 1554 /* no need to setup a destructor, we will explicitly release the
1555 * forward allocated memory on dequeue
1556 */
f970bd9e
PA		 1557 sock_skb_set_dropcount(sk, skb);
1558
1559 __skb_queue_tail(list, skb);
1560 spin_unlock(&list->lock);
1561
1562 if (!sock_flag(sk, SOCK_DEAD))
1563 sk->sk_data_ready(sk);
1564
4b272750 1565 busylock_release(busy);
f970bd9e
PA		 1566 return 0;
1567
1568uncharge_drop:
1569 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
1570
1571drop:
1572 atomic_inc(&sk->sk_drops);
4b272750 1573 busylock_release(busy);
f970bd9e
PA		 1574 return err;
1575}
1576EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
1577
c915fe13 1578void udp_destruct_sock(struct sock *sk)
f970bd9e
PA		 1579{
1580 /* reclaim completely the forward allocated memory */
2276f58a 1581 struct udp_sock *up = udp_sk(sk);
7c13f97f
PA		 1582 unsigned int total = 0;
1583 struct sk_buff *skb;
1584
2276f58a
PA		 1585 skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
1586 while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
7c13f97f
PA		 1587 total += skb->truesize;
1588 kfree_skb(skb);
1589 }
6dfb4367 1590 udp_rmem_release(sk, total, 0, true);
7c13f97f 1591
f970bd9e
PA		 1592 inet_sock_destruct(sk);
1593}
c915fe13 1594EXPORT_SYMBOL_GPL(udp_destruct_sock);
f970bd9e
PA		 1595
1596int udp_init_sock(struct sock *sk)
1597{
2276f58a 1598 skb_queue_head_init(&udp_sk(sk)->reader_queue);
f970bd9e
PA		 1599 sk->sk_destruct = udp_destruct_sock;
1600 return 0;
1601}
1602EXPORT_SYMBOL_GPL(udp_init_sock);
1603
1604void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
1605{
1606 if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
1607 bool slow = lock_sock_fast(sk);
1608
1609 sk_peek_offset_bwd(sk, len);
1610 unlock_sock_fast(sk, slow);
1611 }
0a463c78 1612
ca2c1418
PA		 1613 if (!skb_unref(skb))
1614 return;
1615
dce4551c
PA		 1616 /* In the more common cases we cleared the head states previously,
1617 * see __udp_queue_rcv_skb().
0ddf3fb2 1618 */
dce4551c 1619 if (unlikely(udp_skb_has_head_state(skb)))
0ddf3fb2 1620 skb_release_head_state(skb);
ca2c1418 1621 __consume_stateless_skb(skb);
f970bd9e
PA		 1622}
1623EXPORT_SYMBOL_GPL(skb_consume_udp);
1624
2276f58a
PA		 1625static struct sk_buff *__first_packet_length(struct sock *sk,
1626 struct sk_buff_head *rcvq,
1627 int *total)
1628{
1629 struct sk_buff *skb;
1630
9bd780f5
PA		 1631 while ((skb = skb_peek(rcvq)) != NULL) {
1632 if (udp_lib_checksum_complete(skb)) {
1633 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
1634 IS_UDPLITE(sk));
1635 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
1636 IS_UDPLITE(sk));
1637 atomic_inc(&sk->sk_drops);
1638 __skb_unlink(skb, rcvq);
1639 *total += skb->truesize;
1640 kfree_skb(skb);
1641 } else {
a793183c 1642 udp_skb_csum_unnecessary_set(skb);
9bd780f5
PA		 1643 break;
1644 }
2276f58a
PA		 1645 }
1646 return skb;
1647}
1648
85584672
ED		 1649/**
1650 * first_packet_length - return length of first packet in receive queue
1651 * @sk: socket
1652 *
1653 * Drops all bad checksum frames, until a valid one is found.
e83c6744 1654 * Returns the length of found skb, or -1 if none is found.
85584672 1655 */
e83c6744 1656static int first_packet_length(struct sock *sk)
85584672 1657{
2276f58a
PA		 1658 struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
1659 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
85584672 1660 struct sk_buff *skb;
7c13f97f 1661 int total = 0;
e83c6744 1662 int res;
85584672 1663
85584672 1664 spin_lock_bh(&rcvq->lock);
2276f58a 1665 skb = __first_packet_length(sk, rcvq, &total);
137a0dbe 1666 if (!skb && !skb_queue_empty_lockless(sk_queue)) {
2276f58a
PA		 1667 spin_lock(&sk_queue->lock);
1668 skb_queue_splice_tail_init(sk_queue, rcvq);
1669 spin_unlock(&sk_queue->lock);
1670
1671 skb = __first_packet_length(sk, rcvq, &total);
85584672 1672 }
e83c6744 1673 res = skb ? skb->len : -1;
7c13f97f 1674 if (total)
6dfb4367 1675 udp_rmem_release(sk, total, 1, false);
85584672 1676 spin_unlock_bh(&rcvq->lock);
85584672
ED		 1677 return res;
1678}
1679
1da177e4
LT		 1680/*
1681 * IOCTL requests applicable to the UDP protocol
1682 */
e905a9ed 1683
1da177e4
LT		 1684int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
1685{
6516c655
SH		 1686 switch (cmd) {
1687 case SIOCOUTQ:
1da177e4 1688 {
31e6d363
ED		 1689 int amount = sk_wmem_alloc_get(sk);
1690
6516c655
SH		 1691 return put_user(amount, (int __user *)arg);
1692 }
1da177e4 1693
6516c655
SH		 1694 case SIOCINQ:
1695 {
e83c6744 1696 int amount = max_t(int, 0, first_packet_length(sk));
6516c655 1697
6516c655
SH		 1698 return put_user(amount, (int __user *)arg);
1699 }
1da177e4 1700
6516c655
SH		 1701 default:
1702 return -ENOIOCTLCMD;
1da177e4 1703 }
6516c655
SH		 1704
1705 return 0;
1da177e4 1706}
c482c568 1707EXPORT_SYMBOL(udp_ioctl);
1da177e4 1708
2276f58a 1709struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
fd69c399 1710 int noblock, int *off, int *err)
2276f58a
PA		 1711{
1712 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1713 struct sk_buff_head *queue;
1714 struct sk_buff *last;
1715 long timeo;
1716 int error;
1717
1718 queue = &udp_sk(sk)->reader_queue;
1719 flags |= noblock ? MSG_DONTWAIT : 0;
1720 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1721 do {
1722 struct sk_buff *skb;
1723
1724 error = sock_error(sk);
1725 if (error)
1726 break;
1727
1728 error = -EAGAIN;
2276f58a 1729 do {
2276f58a 1730 spin_lock_bh(&queue->lock);
e427cad6
PA		 1731 skb = __skb_try_recv_from_queue(sk, queue, flags, off,
1732 err, &last);
2276f58a 1733 if (skb) {
e427cad6
PA		 1734 if (!(flags & MSG_PEEK))
1735 udp_skb_destructor(sk, skb);
2276f58a 1736 spin_unlock_bh(&queue->lock);
2276f58a
PA		 1737 return skb;
1738 }
1739
137a0dbe 1740 if (skb_queue_empty_lockless(sk_queue)) {
2276f58a
PA		 1741 spin_unlock_bh(&queue->lock);
1742 goto busy_check;
1743 }
1744
6dfb4367
PA		 1745 /* refill the reader queue and walk it again
1746 * keep both queues locked to avoid re-acquiring
1747 * the sk_receive_queue lock if fwd memory scheduling
1748 * is needed.
1749 */
2276f58a
PA		 1750 spin_lock(&sk_queue->lock);
1751 skb_queue_splice_tail_init(sk_queue, queue);
2276f58a 1752
e427cad6
PA		 1753 skb = __skb_try_recv_from_queue(sk, queue, flags, off,
1754 err, &last);
1755 if (skb && !(flags & MSG_PEEK))
1756 udp_skb_dtor_locked(sk, skb);
6dfb4367 1757 spin_unlock(&sk_queue->lock);
2276f58a 1758 spin_unlock_bh(&queue->lock);
de321ed3 1759 if (skb)
2276f58a 1760 return skb;
2276f58a
PA		 1761
1762busy_check:
1763 if (!sk_can_busy_loop(sk))
1764 break;
1765
1766 sk_busy_loop(sk, flags & MSG_DONTWAIT);
137a0dbe 1767 } while (!skb_queue_empty_lockless(sk_queue));
2276f58a
PA		 1768
1769 /* sk_queue is empty, reader_queue may contain peeked packets */
1770 } while (timeo &&
b50b0580
SD		 1771 !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
1772 &error, &timeo,
2276f58a
PA		 1773 (struct sk_buff *)sk_queue));
1774
1775 *err = error;
1776 return NULL;
1777}
7e823644 1778EXPORT_SYMBOL(__skb_recv_udp);
2276f58a 1779
db8dac20
DM		 1780/*
1781 * This should be easy, if there is something there we
1782 * return it, otherwise we block.
1783 */
1784
1b784140
YX		 1785int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,
1786 int flags, int *addr_len)
db8dac20
DM		 1787{
1788 struct inet_sock *inet = inet_sk(sk);
342dfc30 1789 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
db8dac20 1790 struct sk_buff *skb;
59c2cdae 1791 unsigned int ulen, copied;
fd69c399 1792 int off, err, peeking = flags & MSG_PEEK;
db8dac20 1793 int is_udplite = IS_UDPLITE(sk);
197c949e 1794 bool checksum_valid = false;
db8dac20 1795
db8dac20 1796 if (flags & MSG_ERRQUEUE)
85fbaa75 1797 return ip_recv_error(sk, msg, len, addr_len);
db8dac20
DM		 1798
1799try_again:
a0917e0b 1800 off = sk_peek_offset(sk, flags);
fd69c399 1801 skb = __skb_recv_udp(sk, flags, noblock, &off, &err);
db8dac20 1802 if (!skb)
627d2d6b 1803 return err;
db8dac20 1804
b65ac446 1805 ulen = udp_skb_len(skb);
59c2cdae 1806 copied = len;
627d2d6b 1807 if (copied > ulen - off)
1808 copied = ulen - off;
59c2cdae 1809 else if (copied < ulen)
db8dac20
DM		 1810 msg->msg_flags |= MSG_TRUNC;
1811
1812 /*
1813 * If checksum is needed at all, try to do it while copying the
1814 * data. If the data is truncated, or if we only want a partial
1815 * coverage checksum (UDP-Lite), do it before the copy.
1816 */
1817
d21dbdfe
ED		 1818 if (copied < ulen || peeking ||
1819 (is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
b65ac446
PA		 1820 checksum_valid = udp_skb_csum_unnecessary(skb) ||
1821 !__udp_lib_checksum_complete(skb);
197c949e 1822 if (!checksum_valid)
db8dac20
DM		 1823 goto csum_copy_err;
1824 }
1825
b65ac446
PA		 1826 if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
1827 if (udp_skb_is_linear(skb))
1828 err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
1829 else
1830 err = skb_copy_datagram_msg(skb, off, msg, copied);
1831 } else {
627d2d6b 1832 err = skb_copy_and_csum_datagram_msg(skb, off, msg);
db8dac20
DM		 1833
1834 if (err == -EINVAL)
1835 goto csum_copy_err;
1836 }
1837
22911fc5 1838 if (unlikely(err)) {
fd69c399 1839 if (!peeking) {
979402b1 1840 atomic_inc(&sk->sk_drops);
6aef70a8
ED		 1841 UDP_INC_STATS(sock_net(sk),
1842 UDP_MIB_INERRORS, is_udplite);
979402b1 1843 }
850cbadd 1844 kfree_skb(skb);
627d2d6b 1845 return err;
22911fc5 1846 }
db8dac20 1847
fd69c399 1848 if (!peeking)
6aef70a8
ED		 1849 UDP_INC_STATS(sock_net(sk),
1850 UDP_MIB_INDATAGRAMS, is_udplite);
db8dac20 1851
3b885787 1852 sock_recv_ts_and_drops(msg, sk, skb);
db8dac20
DM		 1853
1854 /* Copy the address. */
c482c568 1855 if (sin) {
db8dac20
DM		 1856 sin->sin_family = AF_INET;
1857 sin->sin_port = udp_hdr(skb)->source;
1858 sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1859 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
bceaa902 1860 *addr_len = sizeof(*sin);
983695fa
DB		 1861
1862 if (cgroup_bpf_enabled)
1863 BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
1864 (struct sockaddr *)sin);
db8dac20 1865 }
bcd1665e
PA		 1866
1867 if (udp_sk(sk)->gro_enabled)
1868 udp_cmsg_recv(msg, sk, skb);
1869
db8dac20 1870 if (inet->cmsg_flags)
ad959036 1871 ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
db8dac20 1872
59c2cdae 1873 err = copied;
db8dac20
DM		 1874 if (flags & MSG_TRUNC)
1875 err = ulen;
1876
850cbadd 1877 skb_consume_udp(sk, skb, peeking ? -err : err);
db8dac20
DM		 1878 return err;
1879
1880csum_copy_err:
2276f58a
PA		 1881 if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
1882 udp_skb_destructor)) {
6aef70a8
ED		 1883 UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1884 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
6a5dc9e5 1885 }
850cbadd 1886 kfree_skb(skb);
db8dac20 1887
beb39db5
ED		 1888 /* starting over for a new packet, but check if we need to yield */
1889 cond_resched();
9cfaa8de 1890 msg->msg_flags &= ~MSG_TRUNC;
db8dac20
DM		 1891 goto try_again;
1892}
1893
d74bad4e
AI		 1894int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
1895{
1896 /* This check is replicated from __ip4_datagram_connect() and
1897 * intended to prevent BPF program called below from accessing bytes
1898 * that are out of the bound specified by user in addr_len.
1899 */
1900 if (addr_len < sizeof(struct sockaddr_in))
1901 return -EINVAL;
1902
1903 return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr);
1904}
1905EXPORT_SYMBOL(udp_pre_connect);
1906
286c72de 1907int __udp_disconnect(struct sock *sk, int flags)
1da177e4
LT		 1908{
1909 struct inet_sock *inet = inet_sk(sk);
1910 /*
1911 * 1003.1g - break association.
1912 */
e905a9ed 1913
1da177e4 1914 sk->sk_state = TCP_CLOSE;
c720c7e8
ED		 1915 inet->inet_daddr = 0;
1916 inet->inet_dport = 0;
bdeab991 1917 sock_rps_reset_rxhash(sk);
1da177e4 1918 sk->sk_bound_dev_if = 0;
303d0403 1919 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) {
1da177e4 1920 inet_reset_saddr(sk);
303d0403
WB		 1921 if (sk->sk_prot->rehash &&
1922 (sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1923 sk->sk_prot->rehash(sk);
1924 }
1da177e4
LT		 1925
1926 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1927 sk->sk_prot->unhash(sk);
c720c7e8 1928 inet->inet_sport = 0;
1da177e4
LT		 1929 }
1930 sk_dst_reset(sk);
1931 return 0;
1932}
286c72de
ED		 1933EXPORT_SYMBOL(__udp_disconnect);
1934
1935int udp_disconnect(struct sock *sk, int flags)
1936{
1937 lock_sock(sk);
1938 __udp_disconnect(sk, flags);
1939 release_sock(sk);
1940 return 0;
1941}
c482c568 1942EXPORT_SYMBOL(udp_disconnect);
1da177e4 1943
645ca708
ED		 1944void udp_lib_unhash(struct sock *sk)
1945{
723b4610
ED		 1946 if (sk_hashed(sk)) {
1947 struct udp_table *udptable = sk->sk_prot->h.udp_table;
512615b6
ED		 1948 struct udp_hslot *hslot, *hslot2;
1949
1950 hslot = udp_hashslot(udptable, sock_net(sk),
1951 udp_sk(sk)->udp_port_hash);
1952 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
645ca708 1953
723b4610 1954 spin_lock_bh(&hslot->lock);
e32ea7e7
CG		 1955 if (rcu_access_pointer(sk->sk_reuseport_cb))
1956 reuseport_detach_sock(sk);
ca065d0c 1957 if (sk_del_node_init_rcu(sk)) {
fdcc8aa9 1958 hslot->count--;
c720c7e8 1959 inet_sk(sk)->inet_num = 0;
723b4610 1960 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
512615b6
ED		 1961
1962 spin_lock(&hslot2->lock);
ca065d0c 1963 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
512615b6
ED		 1964 hslot2->count--;
1965 spin_unlock(&hslot2->lock);
723b4610
ED		 1966 }
1967 spin_unlock_bh(&hslot->lock);
645ca708 1968 }
645ca708
ED		 1969}
1970EXPORT_SYMBOL(udp_lib_unhash);
1971
719f8358
ED		 1972/*
1973 * inet_rcv_saddr was changed, we must rehash secondary hash
1974 */
1975void udp_lib_rehash(struct sock *sk, u16 newhash)
1976{
1977 if (sk_hashed(sk)) {
1978 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1979 struct udp_hslot *hslot, *hslot2, *nhslot2;
1980
1981 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1982 nhslot2 = udp_hashslot2(udptable, newhash);
1983 udp_sk(sk)->udp_portaddr_hash = newhash;
e32ea7e7
CG		 1984
1985 if (hslot2 != nhslot2 ||
1986 rcu_access_pointer(sk->sk_reuseport_cb)) {
719f8358
ED		 1987 hslot = udp_hashslot(udptable, sock_net(sk),
1988 udp_sk(sk)->udp_port_hash);
1989 /* we must lock primary chain too */
1990 spin_lock_bh(&hslot->lock);
e32ea7e7
CG		 1991 if (rcu_access_pointer(sk->sk_reuseport_cb))
1992 reuseport_detach_sock(sk);
1993
1994 if (hslot2 != nhslot2) {
1995 spin_lock(&hslot2->lock);
ca065d0c 1996 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
e32ea7e7
CG		 1997 hslot2->count--;
1998 spin_unlock(&hslot2->lock);
1999
2000 spin_lock(&nhslot2->lock);
ca065d0c 2001 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
e32ea7e7
CG		 2002 &nhslot2->head);
2003 nhslot2->count++;
2004 spin_unlock(&nhslot2->lock);
2005 }
719f8358
ED		 2006
2007 spin_unlock_bh(&hslot->lock);
2008 }
2009 }
2010}
2011EXPORT_SYMBOL(udp_lib_rehash);
2012
8f6b5392 2013void udp_v4_rehash(struct sock *sk)
719f8358 2014{
f0b1e64c 2015 u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
719f8358
ED		 2016 inet_sk(sk)->inet_rcv_saddr,
2017 inet_sk(sk)->inet_num);
2018 udp_lib_rehash(sk, new_hash);
2019}
2020
a3f96c47 2021static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
93821778 2022{
fec5e652 2023 int rc;
766e9037 2024
005ec974 2025 if (inet_sk(sk)->inet_daddr) {
bdeab991 2026 sock_rps_save_rxhash(sk, skb);
005ec974 2027 sk_mark_napi_id(sk, skb);
2c8c56e1 2028 sk_incoming_cpu_update(sk);
e68b6e50
ED		 2029 } else {
2030 sk_mark_napi_id_once(sk, skb);
005ec974 2031 }
fec5e652 2032
850cbadd 2033 rc = __udp_enqueue_schedule_skb(sk, skb);
766e9037
ED		 2034 if (rc < 0) {
2035 int is_udplite = IS_UDPLITE(sk);
93821778 2036
93821778 2037 /* Note that an ENOMEM error is charged twice */
766e9037 2038 if (rc == -ENOMEM)
e61da9e2 2039 UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
02c22347 2040 is_udplite);
e61da9e2 2041 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
766e9037 2042 kfree_skb(skb);
296f7ea7 2043 trace_udp_fail_queue_rcv_skb(rc, sk);
766e9037 2044 return -1;
93821778
HX		 2045 }
2046
2047 return 0;
93821778
HX		 2048}
2049
db8dac20
DM		 2050/* returns:
2051 * -1: error
2052 * 0: success
2053 * >0: "udp encap" protocol resubmission
2054 *
2055 * Note that in the success and error cases, the skb is assumed to
2056 * have either been requeued or freed.
2057 */
cf329aa4 2058static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
db8dac20
DM		 2059{
2060 struct udp_sock *up = udp_sk(sk);
db8dac20
DM		 2061 int is_udplite = IS_UDPLITE(sk);
2062
2063 /*
2064 * Charge it to the socket, dropping if the queue is full.
2065 */
2066 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
2067 goto drop;
895b5c9f 2068 nf_reset_ct(skb);
db8dac20 2069
88ab3108 2070 if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
0ad92ad0
ED		 2071 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
2072
db8dac20
DM		 2073 /*
2074 * This is an encapsulation socket so pass the skb to
2075 * the socket's udp_encap_rcv() hook. Otherwise, just
2076 * fall through and pass this up the UDP socket.
2077 * up->encap_rcv() returns the following value:
2078 * =0 if skb was successfully passed to the encap
2079 * handler or was discarded by it.
2080 * >0 if skb should be passed on to UDP.
2081 * <0 if skb should be resubmitted as proto -N
2082 */
2083
2084 /* if we're overly short, let UDP handle it */
6aa7de05 2085 encap_rcv = READ_ONCE(up->encap_rcv);
e5aed006 2086 if (encap_rcv) {
db8dac20
DM		 2087 int ret;
2088
0a80966b
TH		 2089 /* Verify checksum before giving to encap */
2090 if (udp_lib_checksum_complete(skb))
2091 goto csum_error;
2092
0ad92ad0 2093 ret = encap_rcv(sk, skb);
db8dac20 2094 if (ret <= 0) {
02c22347
ED		 2095 __UDP_INC_STATS(sock_net(sk),
2096 UDP_MIB_INDATAGRAMS,
2097 is_udplite);
db8dac20
DM		 2098 return -ret;
2099 }
2100 }
2101
2102 /* FALLTHROUGH -- it's a UDP Packet */
2103 }
2104
2105 /*
2106 * UDP-Lite specific tests, ignored on UDP sockets
2107 */
b0a42277 2108 if ((up->pcflag & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
db8dac20
DM		 2109
2110 /*
2111 * MIB statistics other than incrementing the error count are
2112 * disabled for the following two types of errors: these depend
2113 * on the application settings, not on the functioning of the
2114 * protocol stack as such.
2115 *
2116 * RFC 3828 here recommends (sec 3.3): "There should also be a
2117 * way ... to ... at least let the receiving application block
2118 * delivery of packets with coverage values less than a value
2119 * provided by the application."
2120 */
2121 if (up->pcrlen == 0) { /* full coverage was set */
ba7a46f1
JP		 2122 net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
2123 UDP_SKB_CB(skb)->cscov, skb->len);
db8dac20
DM		 2124 goto drop;
2125 }
2126 /* The next case involves violating the min. coverage requested
2127 * by the receiver. This is subtle: if receiver wants x and x is
2128 * greater than the buffersize/MTU then receiver will complain
2129 * that it wants x while sender emits packets of smaller size y.
2130 * Therefore the above ...()->partial_cov statement is essential.
2131 */
2132 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
ba7a46f1
JP		 2133 net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
2134 UDP_SKB_CB(skb)->cscov, up->pcrlen);
db8dac20
DM		 2135 goto drop;
2136 }
2137 }
2138
dd99e425 2139 prefetch(&sk->sk_rmem_alloc);
ce25d66a
ED		 2140 if (rcu_access_pointer(sk->sk_filter) &&
2141 udp_lib_checksum_complete(skb))
e6afc8ac 2142 goto csum_error;
ce25d66a 2143
ba66bbe5 2144 if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr)))
a6127697 2145 goto drop;
db8dac20 2146
e6afc8ac 2147 udp_csum_pull_header(skb);
db8dac20 2148
fbf8866d 2149 ipv4_pktinfo_prepare(sk, skb);
850cbadd 2150 return __udp_queue_rcv_skb(sk, skb);
db8dac20 2151
6a5dc9e5 2152csum_error:
02c22347 2153 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
db8dac20 2154drop:
02c22347 2155 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
8edf19c2 2156 atomic_inc(&sk->sk_drops);
db8dac20
DM		 2157 kfree_skb(skb);
2158 return -1;
2159}
2160
cf329aa4
PA		 2161static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2162{
2163 struct sk_buff *next, *segs;
2164 int ret;
2165
2166 if (likely(!udp_unexpected_gso(sk, skb)))
2167 return udp_queue_rcv_one_skb(sk, skb);
2168
a08e7fd9 2169 BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
cf329aa4
PA		 2170 __skb_push(skb, -skb_mac_offset(skb));
2171 segs = udp_rcv_segment(sk, skb, true);
1a186c14 2172 skb_list_walk_safe(segs, skb, next) {
cf329aa4
PA		 2173 __skb_pull(skb, skb_transport_offset(skb));
2174 ret = udp_queue_rcv_one_skb(sk, skb);
2175 if (ret > 0)
2176 ip_protocol_deliver_rcu(dev_net(skb->dev), skb, -ret);
2177 }
2178 return 0;
2179}
2180
97502231 2181/* For TCP sockets, sk_rx_dst is protected by socket lock
e47eb5df 2182 * For UDP, we use xchg() to guard against concurrent changes.
97502231 2183 */
64f0f5d1 2184bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
421b3885 2185{
97502231
ED		 2186 struct dst_entry *old;
2187
d24406c8
WW		 2188 if (dst_hold_safe(dst)) {
2189 old = xchg(&sk->sk_rx_dst, dst);
2190 dst_release(old);
64f0f5d1 2191 return old != dst;
d24406c8 2192 }
64f0f5d1 2193 return false;
421b3885 2194}
c9f2c1ae 2195EXPORT_SYMBOL(udp_sk_rx_dst_set);
421b3885 2196
db8dac20
DM		 2197/*
2198 * Multicasts and broadcasts go to each listener.
2199 *
1240d137 2200 * Note: called only from the BH handler context.
db8dac20 2201 */
e3163493 2202static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
db8dac20
DM		 2203 struct udphdr *uh,
2204 __be32 saddr, __be32 daddr,
36cbb245
RJ		 2205 struct udp_table *udptable,
2206 int proto)
db8dac20 2207{
ca065d0c 2208 struct sock *sk, *first = NULL;
5cf3d461
DH		 2209 unsigned short hnum = ntohs(uh->dest);
2210 struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
2dc41cff 2211 unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
ca065d0c
ED		 2212 unsigned int offset = offsetof(typeof(*sk), sk_node);
2213 int dif = skb->dev->ifindex;
fb74c277 2214 int sdif = inet_sdif(skb);
ca065d0c
ED		 2215 struct hlist_node *node;
2216 struct sk_buff *nskb;
2dc41cff
DH		 2217
2218 if (use_hash2) {
f0b1e64c 2219 hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
73e2d5e3 2220 udptable->mask;
f0b1e64c 2221 hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
2dc41cff 2222start_lookup:
73e2d5e3 2223 hslot = &udptable->hash2[hash2];
2dc41cff
DH		 2224 offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
2225 }
db8dac20 2226
ca065d0c
ED		 2227 sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
2228 if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
fb74c277 2229 uh->source, saddr, dif, sdif, hnum))
ca065d0c
ED		 2230 continue;
2231
2232 if (!first) {
2233 first = sk;
2234 continue;
1240d137 2235 }
ca065d0c 2236 nskb = skb_clone(skb, GFP_ATOMIC);
1240d137 2237
ca065d0c
ED		 2238 if (unlikely(!nskb)) {
2239 atomic_inc(&sk->sk_drops);
02c22347
ED		 2240 __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
2241 IS_UDPLITE(sk));
2242 __UDP_INC_STATS(net, UDP_MIB_INERRORS,
2243 IS_UDPLITE(sk));
ca065d0c
ED		 2244 continue;
2245 }
2246 if (udp_queue_rcv_skb(sk, nskb) > 0)
2247 consume_skb(nskb);
2248 }
1240d137 2249
2dc41cff
DH		 2250 /* Also lookup *:port if we are using hash2 and haven't done so yet. */
2251 if (use_hash2 && hash2 != hash2_any) {
2252 hash2 = hash2_any;
2253 goto start_lookup;
2254 }
2255
ca065d0c
ED		 2256 if (first) {
2257 if (udp_queue_rcv_skb(first, skb) > 0)
2258 consume_skb(skb);
1240d137 2259 } else {
ca065d0c 2260 kfree_skb(skb);
02c22347
ED		 2261 __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
2262 proto == IPPROTO_UDPLITE);
1240d137 2263 }
db8dac20
DM		 2264 return 0;
2265}
2266
2267/* Initialize UDP checksum. If exited with zero value (success),
2268 * CHECKSUM_UNNECESSARY means, that no more checks are required.
666a3d6e 2269 * Otherwise, csum completion requires checksumming packet body,
db8dac20
DM		 2270 * including udp header and folding it to skb->csum.
2271 */
2272static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
2273 int proto)
2274{
db8dac20
DM		 2275 int err;
2276
2277 UDP_SKB_CB(skb)->partial_cov = 0;
2278 UDP_SKB_CB(skb)->cscov = skb->len;
2279
2280 if (proto == IPPROTO_UDPLITE) {
2281 err = udplite_checksum_init(skb, uh);
2282 if (err)
2283 return err;
15f35d49
AK		 2284
2285 if (UDP_SKB_CB(skb)->partial_cov) {
2286 skb->csum = inet_compute_pseudo(skb, proto);
2287 return 0;
2288 }
db8dac20
DM		 2289 }
2290
b46d9f62
HFS		 2291 /* Note, we are only interested in != 0 or == 0, thus the
2292 * force to int.
2293 */
db4f1be3
ST		 2294 err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check,
2295 inet_compute_pseudo);
2296 if (err)
2297 return err;
2298
2299 if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
2300 /* If SW calculated the value, we know it's bad */
2301 if (skb->csum_complete_sw)
2302 return 1;
2303
2304 /* HW says the value is bad. Let's validate that.
2305 * skb->csum is no longer the full packet checksum,
2306 * so don't treat it as such.
2307 */
2308 skb_checksum_complete_unset(skb);
2309 }
2310
2311 return 0;
db8dac20
DM		 2312}
2313
2b5a9217
PA		 2314/* wrapper for udp_queue_rcv_skb tacking care of csum conversion and
2315 * return code conversion for ip layer consumption
2316 */
2317static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
2318 struct udphdr *uh)
2319{
2320 int ret;
2321
2322 if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
e4aa33ad 2323 skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
2b5a9217
PA		 2324
2325 ret = udp_queue_rcv_skb(sk, skb);
2326
2327 /* a return value > 0 means to resubmit the input, but
2328 * it wants the return to be -protocol, or 0
2329 */
2330 if (ret > 0)
2331 return -ret;
2332 return 0;
2333}
2334
db8dac20
DM		 2335/*
2336 * All we need to do is get the socket, and then do a checksum.
2337 */
2338
645ca708 2339int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
db8dac20
DM		 2340 int proto)
2341{
2342 struct sock *sk;
7b5e56f9 2343 struct udphdr *uh;
db8dac20 2344 unsigned short ulen;
adf30907 2345 struct rtable *rt = skb_rtable(skb);
2783ef23 2346 __be32 saddr, daddr;
0283328e 2347 struct net *net = dev_net(skb->dev);
71489e21 2348 bool refcounted;
db8dac20
DM		 2349
2350 /*
2351 * Validate the packet.
2352 */
2353 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
2354 goto drop; /* No space for header. */
2355
7b5e56f9 2356 uh = udp_hdr(skb);
db8dac20 2357 ulen = ntohs(uh->len);
ccc2d97c
BM		 2358 saddr = ip_hdr(skb)->saddr;
2359 daddr = ip_hdr(skb)->daddr;
2360
db8dac20
DM		 2361 if (ulen > skb->len)
2362 goto short_packet;
2363
2364 if (proto == IPPROTO_UDP) {
2365 /* UDP validates ulen. */
2366 if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
2367 goto short_packet;
2368 uh = udp_hdr(skb);
2369 }
2370
2371 if (udp4_csum_init(skb, uh, proto))
2372 goto csum_error;
2373
71489e21 2374 sk = skb_steal_sock(skb, &refcounted);
8afdd99a 2375 if (sk) {
97502231 2376 struct dst_entry *dst = skb_dst(skb);
421b3885 2377 int ret;
421b3885 2378
97502231
ED		 2379 if (unlikely(sk->sk_rx_dst != dst))
2380 udp_sk_rx_dst_set(sk, dst);
db8dac20 2381
2b5a9217 2382 ret = udp_unicast_rcv_skb(sk, skb, uh);
71489e21
JS		 2383 if (refcounted)
2384 sock_put(sk);
2b5a9217 2385 return ret;
421b3885 2386 }
db8dac20 2387
c18450a5
FF		 2388 if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
2389 return __udp4_lib_mcast_deliver(net, skb, uh,
36cbb245 2390 saddr, daddr, udptable, proto);
c18450a5
FF		 2391
2392 sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
2b5a9217
PA		 2393 if (sk)
2394 return udp_unicast_rcv_skb(sk, skb, uh);
db8dac20
DM		 2395
2396 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2397 goto drop;
895b5c9f 2398 nf_reset_ct(skb);
db8dac20
DM		 2399
2400 /* No socket. Drop packet silently, if checksum is wrong */
2401 if (udp_lib_checksum_complete(skb))
2402 goto csum_error;
2403
02c22347 2404 __UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
db8dac20
DM		 2405 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
2406
2407 /*
2408 * Hmm. We got an UDP packet to a port to which we
2409 * don't wanna listen. Ignore it.
2410 */
2411 kfree_skb(skb);
2412 return 0;
2413
2414short_packet:
ba7a46f1
JP		 2415 net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
2416 proto == IPPROTO_UDPLITE ? "Lite" : "",
2417 &saddr, ntohs(uh->source),
2418 ulen, skb->len,
2419 &daddr, ntohs(uh->dest));
db8dac20
DM		 2420 goto drop;
2421
2422csum_error:
2423 /*
2424 * RFC1122: OK. Discards the bad packet silently (as far as
2425 * the network is concerned, anyway) as per 4.1.3.4 (MUST).
2426 */
ba7a46f1
JP		 2427 net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
2428 proto == IPPROTO_UDPLITE ? "Lite" : "",
2429 &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
2430 ulen);
02c22347 2431 __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
db8dac20 2432drop:
02c22347 2433 __UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
db8dac20
DM		 2434 kfree_skb(skb);
2435 return 0;
2436}
2437
421b3885
SB		 2438/* We can only early demux multicast if there is a single matching socket.
2439 * If more than one socket found returns NULL
2440 */
2441static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
2442 __be16 loc_port, __be32 loc_addr,
2443 __be16 rmt_port, __be32 rmt_addr,
fb74c277 2444 int dif, int sdif)
421b3885
SB		 2445{
2446 struct sock *sk, *result;
421b3885 2447 unsigned short hnum = ntohs(loc_port);
ca065d0c 2448 unsigned int slot = udp_hashfn(net, hnum, udp_table.mask);
421b3885
SB		 2449 struct udp_hslot *hslot = &udp_table.hash[slot];
2450
63c6f81c
ED		 2451 /* Do not bother scanning a too big list */
2452 if (hslot->count > 10)
2453 return NULL;
2454
421b3885 2455 result = NULL;
ca065d0c
ED		 2456 sk_for_each_rcu(sk, &hslot->head) {
2457 if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
fb74c277 2458 rmt_port, rmt_addr, dif, sdif, hnum)) {
ca065d0c
ED		 2459 if (result)
2460 return NULL;
421b3885 2461 result = sk;
421b3885
SB		 2462 }
2463 }
ca065d0c 2464
421b3885
SB		 2465 return result;
2466}
2467
2468/* For unicast we should only early demux connected sockets or we can
2469 * break forwarding setups. The chains here can be long so only check
2470 * if the first socket is an exact match and if not move on.
2471 */
2472static struct sock *__udp4_lib_demux_lookup(struct net *net,
2473 __be16 loc_port, __be32 loc_addr,
2474 __be16 rmt_port, __be32 rmt_addr,
3fa6f616 2475 int dif, int sdif)
421b3885 2476{
421b3885 2477 unsigned short hnum = ntohs(loc_port);
f0b1e64c 2478 unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
421b3885
SB		 2479 unsigned int slot2 = hash2 & udp_table.mask;
2480 struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
c7228317 2481 INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
421b3885 2482 const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
ca065d0c 2483 struct sock *sk;
421b3885 2484
ca065d0c
ED		 2485 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
2486 if (INET_MATCH(sk, net, acookie, rmt_addr,
3fa6f616 2487 loc_addr, ports, dif, sdif))
ca065d0c 2488 return sk;
421b3885
SB		 2489 /* Only check first socket in chain */
2490 break;
2491 }
ca065d0c 2492 return NULL;
421b3885
SB		 2493}
2494
7487449c 2495int udp_v4_early_demux(struct sk_buff *skb)
421b3885 2496{
610438b7 2497 struct net *net = dev_net(skb->dev);
bc044e8d 2498 struct in_device *in_dev = NULL;
610438b7
ED		 2499 const struct iphdr *iph;
2500 const struct udphdr *uh;
ca065d0c 2501 struct sock *sk = NULL;
421b3885 2502 struct dst_entry *dst;
421b3885 2503 int dif = skb->dev->ifindex;
fb74c277 2504 int sdif = inet_sdif(skb);
6e540309 2505 int ours;
421b3885
SB		 2506
2507 /* validate the packet */
2508 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
7487449c 2509 return 0;
421b3885 2510
610438b7
ED		 2511 iph = ip_hdr(skb);
2512 uh = udp_hdr(skb);
2513
996b44fc 2514 if (skb->pkt_type == PACKET_MULTICAST) {
bc044e8d 2515 in_dev = __in_dev_get_rcu(skb->dev);
6e540309
SB		 2516
2517 if (!in_dev)
7487449c 2518 return 0;
6e540309 2519
996b44fc
PA		 2520 ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
2521 iph->protocol);
2522 if (!ours)
2523 return 0;
ad0ea198 2524
421b3885 2525 sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
fb74c277
DA		 2526 uh->source, iph->saddr,
2527 dif, sdif);
6e540309 2528 } else if (skb->pkt_type == PACKET_HOST) {
421b3885 2529 sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
3fa6f616 2530 uh->source, iph->saddr, dif, sdif);
6e540309 2531 }
421b3885 2532
41c6d650 2533 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
7487449c 2534 return 0;
421b3885
SB		 2535
2536 skb->sk = sk;
82eabd9e 2537 skb->destructor = sock_efree;
10e2eb87 2538 dst = READ_ONCE(sk->sk_rx_dst);
421b3885
SB		 2539
2540 if (dst)
2541 dst = dst_check(dst, 0);
10e2eb87 2542 if (dst) {
bc044e8d
PA		 2543 u32 itag = 0;
2544
d24406c8
WW		 2545 /* set noref for now.
2546 * any place which wants to hold dst has to call
2547 * dst_hold_safe()
2548 */
2549 skb_dst_set_noref(skb, dst);
bc044e8d
PA		 2550
2551 /* for unconnected multicast sockets we need to validate
2552 * the source on each packet
2553 */
2554 if (!inet_sk(sk)->inet_daddr && in_dev)
2555 return ip_mc_validate_source(skb, iph->daddr,
2556 iph->saddr, iph->tos,
2557 skb->dev, in_dev, &itag);
10e2eb87 2558 }
7487449c 2559 return 0;
421b3885
SB		 2560}
2561
db8dac20
DM		 2562int udp_rcv(struct sk_buff *skb)
2563{
645ca708 2564 return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP);
db8dac20
DM		 2565}
2566
7d06b2e0 2567void udp_destroy_sock(struct sock *sk)
db8dac20 2568{
44046a59 2569 struct udp_sock *up = udp_sk(sk);
8a74ad60 2570 bool slow = lock_sock_fast(sk);
db8dac20 2571 udp_flush_pending_frames(sk);
8a74ad60 2572 unlock_sock_fast(sk, slow);
60fb9567
PA		 2573 if (static_branch_unlikely(&udp_encap_needed_key)) {
2574 if (up->encap_type) {
2575 void (*encap_destroy)(struct sock *sk);
2576 encap_destroy = READ_ONCE(up->encap_destroy);
2577 if (encap_destroy)
2578 encap_destroy(sk);
2579 }
2580 if (up->encap_enabled)
9c480601 2581 static_branch_dec(&udp_encap_needed_key);
44046a59 2582 }
db8dac20
DM		 2583}
2584
1da177e4
LT		 2585/*
2586 * Socket option code for UDP
2587 */
4c0a6cb0 2588int udp_lib_setsockopt(struct sock *sk, int level, int optname,
91ac1cca 2589 sockptr_t optval, unsigned int optlen,
4c0a6cb0 2590 int (*push_pending_frames)(struct sock *))
1da177e4
LT		 2591{
2592 struct udp_sock *up = udp_sk(sk);
1c19448c 2593 int val, valbool;
1da177e4 2594 int err = 0;
b2bf1e26 2595 int is_udplite = IS_UDPLITE(sk);
1da177e4 2596
c482c568 2597 if (optlen < sizeof(int))
1da177e4
LT		 2598 return -EINVAL;
2599
91ac1cca 2600 if (copy_from_sockptr(&val, optval, sizeof(val)))
1da177e4
LT		 2601 return -EFAULT;
2602
1c19448c
TH		 2603 valbool = val ? 1 : 0;
2604
6516c655 2605 switch (optname) {
1da177e4
LT		 2606 case UDP_CORK:
2607 if (val != 0) {
2608 up->corkflag = 1;
2609 } else {
2610 up->corkflag = 0;
2611 lock_sock(sk);
4243cdc2 2612 push_pending_frames(sk);
1da177e4
LT		 2613 release_sock(sk);
2614 }
2615 break;
e905a9ed 2616
1da177e4
LT		 2617 case UDP_ENCAP:
2618 switch (val) {
2619 case 0:
fd1ac07f 2620#ifdef CONFIG_XFRM
1da177e4
LT		 2621 case UDP_ENCAP_ESPINUDP:
2622 case UDP_ENCAP_ESPINUDP_NON_IKE:
0146dca7
SD		 2623#if IS_ENABLED(CONFIG_IPV6)
2624 if (sk->sk_family == AF_INET6)
2625 up->encap_rcv = ipv6_stub->xfrm6_udp_encap_rcv;
2626 else
2627#endif
2628 up->encap_rcv = xfrm4_udp_encap_rcv;
fd1ac07f 2629#endif
a8eceea8 2630 fallthrough;
342f0234 2631 case UDP_ENCAP_L2TPINUDP:
1da177e4 2632 up->encap_type = val;
60fb9567
PA		 2633 lock_sock(sk);
2634 udp_tunnel_encap_enable(sk->sk_socket);
2635 release_sock(sk);
1da177e4
LT		 2636 break;
2637 default:
2638 err = -ENOPROTOOPT;
2639 break;
2640 }
2641 break;
2642
1c19448c
TH		 2643 case UDP_NO_CHECK6_TX:
2644 up->no_check6_tx = valbool;
2645 break;
2646
2647 case UDP_NO_CHECK6_RX:
2648 up->no_check6_rx = valbool;
2649 break;
2650
bec1f6f6
WB		 2651 case UDP_SEGMENT:
2652 if (val < 0 || val > USHRT_MAX)
2653 return -EINVAL;
2654 up->gso_size = val;
2655 break;
2656
e20cf8d3
PA		 2657 case UDP_GRO:
2658 lock_sock(sk);
2659 if (valbool)
2660 udp_tunnel_encap_enable(sk->sk_socket);
2661 up->gro_enabled = valbool;
2662 release_sock(sk);
2663 break;
2664
ba4e58ec
GR		 2665 /*
2666 * UDP-Lite's partial checksum coverage (RFC 3828).
2667 */
2668 /* The sender sets actual checksum coverage length via this option.
2669 * The case coverage > packet length is handled by send module. */
2670 case UDPLITE_SEND_CSCOV:
b2bf1e26 2671 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR		 2672 return -ENOPROTOOPT;
2673 if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
2674 val = 8;
4be929be
AD		 2675 else if (val > USHRT_MAX)
2676 val = USHRT_MAX;
ba4e58ec
GR		 2677 up->pcslen = val;
2678 up->pcflag |= UDPLITE_SEND_CC;
2679 break;
2680
e905a9ed
YH		 2681 /* The receiver specifies a minimum checksum coverage value. To make
2682 * sense, this should be set to at least 8 (as done below). If zero is
ba4e58ec
GR		 2683 * used, this again means full checksum coverage. */
2684 case UDPLITE_RECV_CSCOV:
b2bf1e26 2685 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR		 2686 return -ENOPROTOOPT;
2687 if (val != 0 && val < 8) /* Avoid silly minimal values. */
2688 val = 8;
4be929be
AD		 2689 else if (val > USHRT_MAX)
2690 val = USHRT_MAX;
ba4e58ec
GR		 2691 up->pcrlen = val;
2692 up->pcflag |= UDPLITE_RECV_CC;
2693 break;
2694
1da177e4
LT		 2695 default:
2696 err = -ENOPROTOOPT;
2697 break;
6516c655 2698 }
1da177e4
LT		 2699
2700 return err;
2701}
c482c568 2702EXPORT_SYMBOL(udp_lib_setsockopt);
1da177e4 2703
a7b75c5a
CH		 2704int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
2705 unsigned int optlen)
db8dac20
DM		 2706{
2707 if (level == SOL_UDP || level == SOL_UDPLITE)
91ac1cca 2708 return udp_lib_setsockopt(sk, level, optname,
a7b75c5a 2709 optval, optlen,
db8dac20
DM		 2710 udp_push_pending_frames);
2711 return ip_setsockopt(sk, level, optname, optval, optlen);
2712}
2713
4c0a6cb0
GR		 2714int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2715 char __user *optval, int __user *optlen)
1da177e4
LT		 2716{
2717 struct udp_sock *up = udp_sk(sk);
2718 int val, len;
2719
c482c568 2720 if (get_user(len, optlen))
1da177e4
LT		 2721 return -EFAULT;
2722
2723 len = min_t(unsigned int, len, sizeof(int));
e905a9ed 2724
6516c655 2725 if (len < 0)
1da177e4
LT		 2726 return -EINVAL;
2727
6516c655 2728 switch (optname) {
1da177e4
LT		 2729 case UDP_CORK:
2730 val = up->corkflag;
2731 break;
2732
2733 case UDP_ENCAP:
2734 val = up->encap_type;
2735 break;
2736
1c19448c
TH		 2737 case UDP_NO_CHECK6_TX:
2738 val = up->no_check6_tx;
2739 break;
2740
2741 case UDP_NO_CHECK6_RX:
2742 val = up->no_check6_rx;
2743 break;
2744
bec1f6f6
WB		 2745 case UDP_SEGMENT:
2746 val = up->gso_size;
2747 break;
2748
ba4e58ec
GR		 2749 /* The following two cannot be changed on UDP sockets, the return is
2750 * always 0 (which corresponds to the full checksum coverage of UDP). */
2751 case UDPLITE_SEND_CSCOV:
2752 val = up->pcslen;
2753 break;
2754
2755 case UDPLITE_RECV_CSCOV:
2756 val = up->pcrlen;
2757 break;
2758
1da177e4
LT		 2759 default:
2760 return -ENOPROTOOPT;
6516c655 2761 }
1da177e4 2762
6516c655 2763 if (put_user(len, optlen))
e905a9ed 2764 return -EFAULT;
c482c568 2765 if (copy_to_user(optval, &val, len))
1da177e4 2766 return -EFAULT;
e905a9ed 2767 return 0;
1da177e4 2768}
c482c568 2769EXPORT_SYMBOL(udp_lib_getsockopt);
1da177e4 2770
db8dac20
DM		 2771int udp_getsockopt(struct sock *sk, int level, int optname,
2772 char __user *optval, int __user *optlen)
2773{
2774 if (level == SOL_UDP || level == SOL_UDPLITE)
2775 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2776 return ip_getsockopt(sk, level, optname, optval, optlen);
2777}
2778
1da177e4
LT		 2779/**
2780 * udp_poll - wait for a UDP event.
3628e3cb
AL		 2781 * @file: - file struct
2782 * @sock: - socket
2783 * @wait: - poll table
1da177e4 2784 *
e905a9ed 2785 * This is same as datagram poll, except for the special case of
1da177e4
LT		 2786 * blocking sockets. If application is using a blocking fd
2787 * and a packet with checksum error is in the queue;
2788 * then it could get return from select indicating data available
2789 * but then block when reading it. Add special case code
2790 * to work around these arguably broken applications.
2791 */
a11e1d43 2792__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
1da177e4 2793{
a11e1d43 2794 __poll_t mask = datagram_poll(file, sock, wait);
1da177e4 2795 struct sock *sk = sock->sk;
ba4e58ec 2796
3ef7cf57 2797 if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue))
a9a08845 2798 mask |= EPOLLIN | EPOLLRDNORM;
2276f58a 2799
1da177e4 2800 /* Check for false positives due to checksum errors */
a11e1d43 2801 if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
e83c6744 2802 !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
a9a08845 2803 mask &= ~(EPOLLIN | EPOLLRDNORM);
1da177e4
LT		 2804
2805 return mask;
e905a9ed 2806
1da177e4 2807}
a11e1d43 2808EXPORT_SYMBOL(udp_poll);
1da177e4 2809
5d77dca8
DA		 2810int udp_abort(struct sock *sk, int err)
2811{
2812 lock_sock(sk);
2813
2814 sk->sk_err = err;
2815 sk->sk_error_report(sk);
286c72de 2816 __udp_disconnect(sk, 0);
5d77dca8
DA		 2817
2818 release_sock(sk);
2819
2820 return 0;
2821}
2822EXPORT_SYMBOL_GPL(udp_abort);
2823
db8dac20 2824struct proto udp_prot = {
1e802951
TZ		 2825 .name = "UDP",
2826 .owner = THIS_MODULE,
2827 .close = udp_lib_close,
d74bad4e 2828 .pre_connect = udp_pre_connect,
1e802951
TZ		 2829 .connect = ip4_datagram_connect,
2830 .disconnect = udp_disconnect,
2831 .ioctl = udp_ioctl,
2832 .init = udp_init_sock,
2833 .destroy = udp_destroy_sock,
2834 .setsockopt = udp_setsockopt,
2835 .getsockopt = udp_getsockopt,
2836 .sendmsg = udp_sendmsg,
2837 .recvmsg = udp_recvmsg,
2838 .sendpage = udp_sendpage,
2839 .release_cb = ip4_datagram_release_cb,
2840 .hash = udp_lib_hash,
2841 .unhash = udp_lib_unhash,
2842 .rehash = udp_v4_rehash,
2843 .get_port = udp_v4_get_port,
2844 .memory_allocated = &udp_memory_allocated,
2845 .sysctl_mem = sysctl_udp_mem,
2846 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
2847 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
2848 .obj_size = sizeof(struct udp_sock),
2849 .h.udp_table = &udp_table,
1e802951 2850 .diag_destroy = udp_abort,
db8dac20 2851};
c482c568 2852EXPORT_SYMBOL(udp_prot);
1da177e4
LT		 2853
2854/* ------------------------------------------------------------------------ */
2855#ifdef CONFIG_PROC_FS
2856
645ca708 2857static struct sock *udp_get_first(struct seq_file *seq, int start)
1da177e4
LT		 2858{
2859 struct sock *sk;
9e8ca27a 2860 struct udp_seq_afinfo *afinfo;
1da177e4 2861 struct udp_iter_state *state = seq->private;
6f191efe 2862 struct net *net = seq_file_net(seq);
1da177e4 2863
9e8ca27a
YS		 2864 if (state->bpf_seq_afinfo)
2865 afinfo = state->bpf_seq_afinfo;
2866 else
2867 afinfo = PDE_DATA(file_inode(seq->file));
2868
a3d2599b 2869 for (state->bucket = start; state->bucket <= afinfo->udp_table->mask;
f86dcc5a 2870 ++state->bucket) {
a3d2599b 2871 struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket];
f86dcc5a 2872
ca065d0c 2873 if (hlist_empty(&hslot->head))
f86dcc5a
ED		 2874 continue;
2875
645ca708 2876 spin_lock_bh(&hslot->lock);
ca065d0c 2877 sk_for_each(sk, &hslot->head) {
878628fb 2878 if (!net_eq(sock_net(sk), net))
a91275ef 2879 continue;
9e8ca27a
YS		 2880 if (afinfo->family == AF_UNSPEC ||
2881 sk->sk_family == afinfo->family)
1da177e4
LT		 2882 goto found;
2883 }
645ca708 2884 spin_unlock_bh(&hslot->lock);
1da177e4
LT		 2885 }
2886 sk = NULL;
2887found:
2888 return sk;
2889}
2890
2891static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
2892{
9e8ca27a 2893 struct udp_seq_afinfo *afinfo;
1da177e4 2894 struct udp_iter_state *state = seq->private;
6f191efe 2895 struct net *net = seq_file_net(seq);
1da177e4 2896
9e8ca27a
YS		 2897 if (state->bpf_seq_afinfo)
2898 afinfo = state->bpf_seq_afinfo;
2899 else
2900 afinfo = PDE_DATA(file_inode(seq->file));
2901
1da177e4 2902 do {
ca065d0c 2903 sk = sk_next(sk);
9e8ca27a
YS		 2904 } while (sk && (!net_eq(sock_net(sk), net) ||
2905 (afinfo->family != AF_UNSPEC &&
2906 sk->sk_family != afinfo->family)));
1da177e4 2907
645ca708 2908 if (!sk) {
a3d2599b
CH		 2909 if (state->bucket <= afinfo->udp_table->mask)
2910 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
645ca708 2911 return udp_get_first(seq, state->bucket + 1);
1da177e4
LT		 2912 }
2913 return sk;
2914}
2915
2916static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
2917{
645ca708 2918 struct sock *sk = udp_get_first(seq, 0);
1da177e4
LT		 2919
2920 if (sk)
6516c655 2921 while (pos && (sk = udp_get_next(seq, sk)) != NULL)
1da177e4
LT		 2922 --pos;
2923 return pos ? NULL : sk;
2924}
2925
a3d2599b 2926void *udp_seq_start(struct seq_file *seq, loff_t *pos)
1da177e4 2927{
30842f29 2928 struct udp_iter_state *state = seq->private;
f86dcc5a 2929 state->bucket = MAX_UDP_PORTS;
30842f29 2930
b50660f1 2931 return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
1da177e4 2932}
a3d2599b 2933EXPORT_SYMBOL(udp_seq_start);
1da177e4 2934
a3d2599b 2935void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1da177e4
LT		 2936{
2937 struct sock *sk;
2938
b50660f1 2939 if (v == SEQ_START_TOKEN)
1da177e4
LT		 2940 sk = udp_get_idx(seq, 0);
2941 else
2942 sk = udp_get_next(seq, v);
2943
2944 ++*pos;
2945 return sk;
2946}
a3d2599b 2947EXPORT_SYMBOL(udp_seq_next);
1da177e4 2948
a3d2599b 2949void udp_seq_stop(struct seq_file *seq, void *v)
1da177e4 2950{
9e8ca27a 2951 struct udp_seq_afinfo *afinfo;
645ca708
ED		 2952 struct udp_iter_state *state = seq->private;
2953
9e8ca27a
YS		 2954 if (state->bpf_seq_afinfo)
2955 afinfo = state->bpf_seq_afinfo;
2956 else
2957 afinfo = PDE_DATA(file_inode(seq->file));
2958
a3d2599b
CH		 2959 if (state->bucket <= afinfo->udp_table->mask)
2960 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
1da177e4 2961}
a3d2599b 2962EXPORT_SYMBOL(udp_seq_stop);
db8dac20
DM		 2963
2964/* ------------------------------------------------------------------------ */
5e659e4c 2965static void udp4_format_sock(struct sock *sp, struct seq_file *f,
652586df 2966 int bucket)
db8dac20
DM		 2967{
2968 struct inet_sock *inet = inet_sk(sp);
c720c7e8
ED		 2969 __be32 dest = inet->inet_daddr;
2970 __be32 src = inet->inet_rcv_saddr;
2971 __u16 destp = ntohs(inet->inet_dport);
2972 __u16 srcp = ntohs(inet->inet_sport);
db8dac20 2973
f86dcc5a 2974 seq_printf(f, "%5d: %08X:%04X %08X:%04X"
ea9a0379 2975 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u",
db8dac20 2976 bucket, src, srcp, dest, destp, sp->sk_state,
31e6d363 2977 sk_wmem_alloc_get(sp),
6c206b20 2978 udp_rqueue_get(sp),
a7cb5a49
EB		 2979 0, 0L, 0,
2980 from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
2981 0, sock_i_ino(sp),
41c6d650 2982 refcount_read(&sp->sk_refcnt), sp,
652586df 2983 atomic_read(&sp->sk_drops));
db8dac20
DM		 2984}
2985
2986int udp4_seq_show(struct seq_file *seq, void *v)
2987{
652586df 2988 seq_setwidth(seq, 127);
db8dac20 2989 if (v == SEQ_START_TOKEN)
652586df 2990 seq_puts(seq, " sl local_address rem_address st tx_queue "
db8dac20 2991 "rx_queue tr tm->when retrnsmt uid timeout "
cb61cb9b 2992 "inode ref pointer drops");
db8dac20 2993 else {
db8dac20
DM		 2994 struct udp_iter_state *state = seq->private;
2995
652586df 2996 udp4_format_sock(v, seq, state->bucket);
db8dac20 2997 }
652586df 2998 seq_pad(seq, '\n');
db8dac20
DM		 2999 return 0;
3000}
3001
5788b3a0
YS		 3002#ifdef CONFIG_BPF_SYSCALL
3003struct bpf_iter__udp {
3004 __bpf_md_ptr(struct bpf_iter_meta *, meta);
3005 __bpf_md_ptr(struct udp_sock *, udp_sk);
3006 uid_t uid __aligned(8);
3007 int bucket __aligned(8);
3008};
3009
3010static int udp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
3011 struct udp_sock *udp_sk, uid_t uid, int bucket)
3012{
3013 struct bpf_iter__udp ctx;
3014
3015 meta->seq_num--; /* skip SEQ_START_TOKEN */
3016 ctx.meta = meta;
3017 ctx.udp_sk = udp_sk;
3018 ctx.uid = uid;
3019 ctx.bucket = bucket;
3020 return bpf_iter_run_prog(prog, &ctx);
3021}
3022
3023static int bpf_iter_udp_seq_show(struct seq_file *seq, void *v)
3024{
3025 struct udp_iter_state *state = seq->private;
3026 struct bpf_iter_meta meta;
3027 struct bpf_prog *prog;
3028 struct sock *sk = v;
3029 uid_t uid;
3030
3031 if (v == SEQ_START_TOKEN)
3032 return 0;
3033
3034 uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
3035 meta.seq = seq;
3036 prog = bpf_iter_get_info(&meta, false);
3037 return udp_prog_seq_show(prog, &meta, v, uid, state->bucket);
3038}
3039
3040static void bpf_iter_udp_seq_stop(struct seq_file *seq, void *v)
3041{
3042 struct bpf_iter_meta meta;
3043 struct bpf_prog *prog;
3044
3045 if (!v) {
3046 meta.seq = seq;
3047 prog = bpf_iter_get_info(&meta, true);
3048 if (prog)
3049 (void)udp_prog_seq_show(prog, &meta, v, 0, 0);
3050 }
3051
3052 udp_seq_stop(seq, v);
3053}
3054
3055static const struct seq_operations bpf_iter_udp_seq_ops = {
3056 .start = udp_seq_start,
3057 .next = udp_seq_next,
3058 .stop = bpf_iter_udp_seq_stop,
3059 .show = bpf_iter_udp_seq_show,
3060};
3061#endif
3062
c3506372 3063const struct seq_operations udp_seq_ops = {
a3d2599b
CH		 3064 .start = udp_seq_start,
3065 .next = udp_seq_next,
3066 .stop = udp_seq_stop,
3067 .show = udp4_seq_show,
3068};
c3506372 3069EXPORT_SYMBOL(udp_seq_ops);
73cb88ec 3070
db8dac20 3071static struct udp_seq_afinfo udp4_seq_afinfo = {
db8dac20 3072 .family = AF_INET,
645ca708 3073 .udp_table = &udp_table,
db8dac20
DM		 3074};
3075
2c8c1e72 3076static int __net_init udp4_proc_init_net(struct net *net)
15439feb 3077{
c3506372
CH		 3078 if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops,
3079 sizeof(struct udp_iter_state), &udp4_seq_afinfo))
a3d2599b
CH		 3080 return -ENOMEM;
3081 return 0;
15439feb
PE		 3082}
3083
2c8c1e72 3084static void __net_exit udp4_proc_exit_net(struct net *net)
15439feb 3085{
a3d2599b 3086 remove_proc_entry("udp", net->proc_net);
15439feb
PE		 3087}
3088
3089static struct pernet_operations udp4_net_ops = {
3090 .init = udp4_proc_init_net,
3091 .exit = udp4_proc_exit_net,
3092};
3093
db8dac20
DM		 3094int __init udp4_proc_init(void)
3095{
15439feb 3096 return register_pernet_subsys(&udp4_net_ops);
db8dac20
DM		 3097}
3098
3099void udp4_proc_exit(void)
3100{
15439feb 3101 unregister_pernet_subsys(&udp4_net_ops);
db8dac20 3102}
1da177e4
LT		 3103#endif /* CONFIG_PROC_FS */
3104
f86dcc5a
ED		 3105static __initdata unsigned long uhash_entries;
3106static int __init set_uhash_entries(char *str)
645ca708 3107{
413c27d8
EZ		 3108 ssize_t ret;
3109
f86dcc5a
ED		 3110 if (!str)
3111 return 0;
413c27d8
EZ		 3112
3113 ret = kstrtoul(str, 0, &uhash_entries);
3114 if (ret)
3115 return 0;
3116
f86dcc5a
ED		 3117 if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
3118 uhash_entries = UDP_HTABLE_SIZE_MIN;
3119 return 1;
3120}
3121__setup("uhash_entries=", set_uhash_entries);
645ca708 3122
f86dcc5a
ED		 3123void __init udp_table_init(struct udp_table *table, const char *name)
3124{
3125 unsigned int i;
3126
31fe62b9
TB		 3127 table->hash = alloc_large_system_hash(name,
3128 2 * sizeof(struct udp_hslot),
3129 uhash_entries,
3130 21, /* one slot per 2 MB */
3131 0,
3132 &table->log,
3133 &table->mask,
3134 UDP_HTABLE_SIZE_MIN,
3135 64 * 1024);
3136
512615b6 3137 table->hash2 = table->hash + (table->mask + 1);
f86dcc5a 3138 for (i = 0; i <= table->mask; i++) {
ca065d0c 3139 INIT_HLIST_HEAD(&table->hash[i].head);
fdcc8aa9 3140 table->hash[i].count = 0;
645ca708
ED		 3141 spin_lock_init(&table->hash[i].lock);
3142 }
512615b6 3143 for (i = 0; i <= table->mask; i++) {
ca065d0c 3144 INIT_HLIST_HEAD(&table->hash2[i].head);
512615b6
ED		 3145 table->hash2[i].count = 0;
3146 spin_lock_init(&table->hash2[i].lock);
3147 }
645ca708
ED		 3148}
3149
723b8e46
TH		 3150u32 udp_flow_hashrnd(void)
3151{
3152 static u32 hashrnd __read_mostly;
3153
3154 net_get_random_once(&hashrnd, sizeof(hashrnd));
3155
3156 return hashrnd;
3157}
3158EXPORT_SYMBOL(udp_flow_hashrnd);
3159
1e802951
TZ		 3160static void __udp_sysctl_init(struct net *net)
3161{
3162 net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM;
3163 net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM;
3164
3165#ifdef CONFIG_NET_L3_MASTER_DEV
3166 net->ipv4.sysctl_udp_l3mdev_accept = 0;
3167#endif
3168}
3169
3170static int __net_init udp_sysctl_init(struct net *net)
3171{
3172 __udp_sysctl_init(net);
3173 return 0;
3174}
3175
3176static struct pernet_operations __net_initdata udp_sysctl_ops = {
fc18999e 3177 .init = udp_sysctl_init,
1e802951
TZ		 3178};
3179
5788b3a0
YS		 3180#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3181DEFINE_BPF_ITER_FUNC(udp, struct bpf_iter_meta *meta,
3182 struct udp_sock *udp_sk, uid_t uid, int bucket)
3183
f9c79272 3184static int bpf_iter_init_udp(void *priv_data, struct bpf_iter_aux_info *aux)
5788b3a0
YS		 3185{
3186 struct udp_iter_state *st = priv_data;
3187 struct udp_seq_afinfo *afinfo;
3188 int ret;
3189
3190 afinfo = kmalloc(sizeof(*afinfo), GFP_USER | __GFP_NOWARN);
3191 if (!afinfo)
3192 return -ENOMEM;
3193
3194 afinfo->family = AF_UNSPEC;
3195 afinfo->udp_table = &udp_table;
3196 st->bpf_seq_afinfo = afinfo;
f9c79272 3197 ret = bpf_iter_init_seq_net(priv_data, aux);
5788b3a0
YS		 3198 if (ret)
3199 kfree(afinfo);
3200 return ret;
3201}
3202
3203static void bpf_iter_fini_udp(void *priv_data)
3204{
3205 struct udp_iter_state *st = priv_data;
3206
3207 kfree(st->bpf_seq_afinfo);
3208 bpf_iter_fini_seq_net(priv_data);
3209}
3210
14fc6bd6 3211static const struct bpf_iter_seq_info udp_seq_info = {
5788b3a0
YS		 3212 .seq_ops = &bpf_iter_udp_seq_ops,
3213 .init_seq_private = bpf_iter_init_udp,
3214 .fini_seq_private = bpf_iter_fini_udp,
3215 .seq_priv_size = sizeof(struct udp_iter_state),
14fc6bd6
YS		 3216};
3217
3218static struct bpf_iter_reg udp_reg_info = {
3219 .target = "udp",
5788b3a0
YS		 3220 .ctx_arg_info_size = 1,
3221 .ctx_arg_info = {
3222 { offsetof(struct bpf_iter__udp, udp_sk),
3223 PTR_TO_BTF_ID_OR_NULL },
3224 },
14fc6bd6 3225 .seq_info = &udp_seq_info,
5788b3a0
YS		 3226};
3227
3228static void __init bpf_iter_register(void)
3229{
951cf368 3230 udp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UDP];
5788b3a0
YS		 3231 if (bpf_iter_reg_target(&udp_reg_info))
3232 pr_warn("Warning: could not register bpf iterator udp\n");
3233}
3234#endif
3235
95766fff
HA		 3236void __init udp_init(void)
3237{
f03d78db 3238 unsigned long limit;
4b272750 3239 unsigned int i;
95766fff 3240
f86dcc5a 3241 udp_table_init(&udp_table, "UDP");
f03d78db 3242 limit = nr_free_buffer_pages() / 8;
95766fff
HA		 3243 limit = max(limit, 128UL);
3244 sysctl_udp_mem[0] = limit / 4 * 3;
3245 sysctl_udp_mem[1] = limit;
3246 sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
3247
1e802951 3248 __udp_sysctl_init(&init_net);
4b272750
ED		 3249
3250 /* 16 spinlocks per cpu */
3251 udp_busylocks_log = ilog2(nr_cpu_ids) + 4;
3252 udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log,
3253 GFP_KERNEL);
3254 if (!udp_busylocks)
3255 panic("UDP: failed to alloc udp_busylocks\n");
3256 for (i = 0; i < (1U << udp_busylocks_log); i++)
3257 spin_lock_init(udp_busylocks + i);
1e802951
TZ		 3258
3259 if (register_pernet_subsys(&udp_sysctl_ops))
3260 panic("UDP: failed to init sysctl parameters.\n");
5788b3a0
YS		 3261
3262#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3263 bpf_iter_register();
3264#endif
95766fff 3265}
Linux 6.x block layer and io_uring tree(s)