projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
icmp: remove duplicate code
[linux-2.6-block.git] / net / ipv4 / udp.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * The User Datagram Protocol (UDP).
8 *
02c30a84 9 * Authors: Ross Biro
1da177e4
LT		 10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
113aa838 12 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
1da177e4
LT		 13 * Hirokazu Takahashi, <taka@valinux.co.jp>
14 *
15 * Fixes:
16 * Alan Cox : verify_area() calls
17 * Alan Cox : stopped close while in use off icmp
18 * messages. Not a fix but a botch that
19 * for udp at least is 'valid'.
20 * Alan Cox : Fixed icmp handling properly
21 * Alan Cox : Correct error for oversized datagrams
e905a9ed
YH		 22 * Alan Cox : Tidied select() semantics.
23 * Alan Cox : udp_err() fixed properly, also now
1da177e4
LT		 24 * select and read wake correctly on errors
25 * Alan Cox : udp_send verify_area moved to avoid mem leak
26 * Alan Cox : UDP can count its memory
27 * Alan Cox : send to an unknown connection causes
28 * an ECONNREFUSED off the icmp, but
29 * does NOT close.
30 * Alan Cox : Switched to new sk_buff handlers. No more backlog!
31 * Alan Cox : Using generic datagram code. Even smaller and the PEEK
32 * bug no longer crashes it.
33 * Fred Van Kempen : Net2e support for sk->broadcast.
34 * Alan Cox : Uses skb_free_datagram
35 * Alan Cox : Added get/set sockopt support.
36 * Alan Cox : Broadcasting without option set returns EACCES.
37 * Alan Cox : No wakeup calls. Instead we now use the callbacks.
38 * Alan Cox : Use ip_tos and ip_ttl
39 * Alan Cox : SNMP Mibs
40 * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support.
41 * Matt Dillon : UDP length checks.
42 * Alan Cox : Smarter af_inet used properly.
43 * Alan Cox : Use new kernel side addressing.
44 * Alan Cox : Incorrect return on truncated datagram receive.
45 * Arnt Gulbrandsen : New udp_send and stuff
46 * Alan Cox : Cache last socket
47 * Alan Cox : Route cache
48 * Jon Peatfield : Minor efficiency fix to sendto().
49 * Mike Shaver : RFC1122 checks.
50 * Alan Cox : Nonblocking error fix.
51 * Willy Konynenberg : Transparent proxying support.
52 * Mike McLagan : Routing by source
53 * David S. Miller : New socket lookup architecture.
54 * Last socket cache retained as it
55 * does have a high hit rate.
56 * Olaf Kirch : Don't linearise iovec on sendmsg.
57 * Andi Kleen : Some cleanups, cache destination entry
e905a9ed 58 * for connect.
1da177e4
LT		 59 * Vitaly E. Lavrov : Transparent proxy revived after year coma.
60 * Melvin Smith : Check msg_name not msg_namelen in sendto(),
61 * return ENOTCONN for unconnected sockets (POSIX)
62 * Janos Farkas : don't deliver multi/broadcasts to a different
63 * bound-to-device socket
64 * Hirokazu Takahashi : HW checksumming for outgoing UDP
65 * datagrams.
66 * Hirokazu Takahashi : sendfile() on UDP works now.
67 * Arnaldo C. Melo : convert /proc/net/udp to seq_file
68 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
69 * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind
70 * a single port at the same time.
71 * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support
342f0234 72 * James Chapman : Add L2TP encapsulation type.
1da177e4 73 */
e905a9ed 74
afd46503
JP		 75#define pr_fmt(fmt) "UDP: " fmt
76
7c0f6ba6 77#include <linux/uaccess.h>
1da177e4 78#include <asm/ioctls.h>
57c8a661 79#include <linux/memblock.h>
8203efb3
ED		 80#include <linux/highmem.h>
81#include <linux/swap.h>
1da177e4
LT		 82#include <linux/types.h>
83#include <linux/fcntl.h>
84#include <linux/module.h>
85#include <linux/socket.h>
86#include <linux/sockios.h>
14c85021 87#include <linux/igmp.h>
6e540309 88#include <linux/inetdevice.h>
1da177e4
LT		 89#include <linux/in.h>
90#include <linux/errno.h>
91#include <linux/timer.h>
92#include <linux/mm.h>
1da177e4 93#include <linux/inet.h>
1da177e4 94#include <linux/netdevice.h>
5a0e3ad6 95#include <linux/slab.h>
c752f073 96#include <net/tcp_states.h>
1da177e4
LT		 97#include <linux/skbuff.h>
98#include <linux/proc_fs.h>
99#include <linux/seq_file.h>
457c4cbc 100#include <net/net_namespace.h>
1da177e4 101#include <net/icmp.h>
421b3885 102#include <net/inet_hashtables.h>
e7cc0824 103#include <net/ip_tunnels.h>
1da177e4 104#include <net/route.h>
1da177e4
LT		 105#include <net/checksum.h>
106#include <net/xfrm.h>
296f7ea7 107#include <trace/events/udp.h>
447167bf 108#include <linux/static_key.h>
22911fc5 109#include <trace/events/skb.h>
076bb0c8 110#include <net/busy_poll.h>
ba4e58ec 111#include "udp_impl.h"
e32ea7e7 112#include <net/sock_reuseport.h>
217375a0 113#include <net/addrconf.h>
60fb9567 114#include <net/udp_tunnel.h>
1da177e4 115
f86dcc5a 116struct udp_table udp_table __read_mostly;
645ca708 117EXPORT_SYMBOL(udp_table);
1da177e4 118
8d987e5c 119long sysctl_udp_mem[3] __read_mostly;
95766fff 120EXPORT_SYMBOL(sysctl_udp_mem);
c482c568 121
8d987e5c 122atomic_long_t udp_memory_allocated;
95766fff
HA		 123EXPORT_SYMBOL(udp_memory_allocated);
124
f86dcc5a
ED		 125#define MAX_UDP_PORTS 65536
126#define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN)
98322f22 127
f24d43c0 128static int udp_lib_lport_inuse(struct net *net, __u16 num,
645ca708 129 const struct udp_hslot *hslot,
98322f22 130 unsigned long *bitmap,
fe38d2a1 131 struct sock *sk, unsigned int log)
1da177e4 132{
f24d43c0 133 struct sock *sk2;
ba418fa3 134 kuid_t uid = sock_i_uid(sk);
25030a7f 135
ca065d0c 136 sk_for_each(sk2, &hslot->head) {
9d4fb27d
JP		 137 if (net_eq(sock_net(sk2), net) &&
138 sk2 != sk &&
d4cada4a 139 (bitmap || udp_sk(sk2)->udp_port_hash == num) &&
9d4fb27d
JP		 140 (!sk2->sk_reuse || !sk->sk_reuse) &&
141 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
142 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
fe38d2a1 143 inet_rcv_saddr_equal(sk, sk2, true)) {
df560056
EG		 144 if (sk2->sk_reuseport && sk->sk_reuseport &&
145 !rcu_access_pointer(sk->sk_reuseport_cb) &&
146 uid_eq(uid, sock_i_uid(sk2))) {
147 if (!bitmap)
148 return 0;
149 } else {
150 if (!bitmap)
151 return 1;
152 __set_bit(udp_sk(sk2)->udp_port_hash >> log,
153 bitmap);
154 }
98322f22 155 }
4243cdc2 156 }
25030a7f
GR		 157 return 0;
158}
159
30fff923
ED		 160/*
161 * Note: we still hold spinlock of primary hash chain, so no other writer
162 * can insert/delete a socket with local_port == num
163 */
164static int udp_lib_lport_inuse2(struct net *net, __u16 num,
4243cdc2 165 struct udp_hslot *hslot2,
fe38d2a1 166 struct sock *sk)
30fff923
ED		 167{
168 struct sock *sk2;
ba418fa3 169 kuid_t uid = sock_i_uid(sk);
30fff923
ED		 170 int res = 0;
171
172 spin_lock(&hslot2->lock);
ca065d0c 173 udp_portaddr_for_each_entry(sk2, &hslot2->head) {
9d4fb27d
JP		 174 if (net_eq(sock_net(sk2), net) &&
175 sk2 != sk &&
176 (udp_sk(sk2)->udp_port_hash == num) &&
177 (!sk2->sk_reuse || !sk->sk_reuse) &&
178 (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if ||
179 sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
fe38d2a1 180 inet_rcv_saddr_equal(sk, sk2, true)) {
df560056
EG		 181 if (sk2->sk_reuseport && sk->sk_reuseport &&
182 !rcu_access_pointer(sk->sk_reuseport_cb) &&
183 uid_eq(uid, sock_i_uid(sk2))) {
184 res = 0;
185 } else {
186 res = 1;
187 }
30fff923
ED		 188 break;
189 }
4243cdc2 190 }
30fff923
ED		 191 spin_unlock(&hslot2->lock);
192 return res;
193}
194
fe38d2a1 195static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot)
e32ea7e7
CG		 196{
197 struct net *net = sock_net(sk);
e32ea7e7
CG		 198 kuid_t uid = sock_i_uid(sk);
199 struct sock *sk2;
200
ca065d0c 201 sk_for_each(sk2, &hslot->head) {
e32ea7e7
CG		 202 if (net_eq(sock_net(sk2), net) &&
203 sk2 != sk &&
204 sk2->sk_family == sk->sk_family &&
205 ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
206 (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) &&
207 (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) &&
208 sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
fe38d2a1 209 inet_rcv_saddr_equal(sk, sk2, false)) {
2dbb9b9e
MKL		 210 return reuseport_add_sock(sk, sk2,
211 inet_rcv_saddr_any(sk));
e32ea7e7
CG		 212 }
213 }
214
2dbb9b9e 215 return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
e32ea7e7
CG		 216}
217
25030a7f 218/**
6ba5a3c5 219 * udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6
25030a7f
GR		 220 *
221 * @sk: socket struct in question
222 * @snum: port number to look up
25985edc 223 * @hash2_nulladdr: AF-dependent hash value in secondary hash chains,
30fff923 224 * with NULL address
25030a7f 225 */
6ba5a3c5 226int udp_lib_get_port(struct sock *sk, unsigned short snum,
30fff923 227 unsigned int hash2_nulladdr)
25030a7f 228{
512615b6 229 struct udp_hslot *hslot, *hslot2;
645ca708 230 struct udp_table *udptable = sk->sk_prot->h.udp_table;
25030a7f 231 int error = 1;
3b1e0a65 232 struct net *net = sock_net(sk);
1da177e4 233
32c1da70 234 if (!snum) {
9088c560 235 int low, high, remaining;
95c96174 236 unsigned int rand;
98322f22
ED		 237 unsigned short first, last;
238 DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN);
32c1da70 239
0bbf87d8 240 inet_get_local_port_range(net, &low, &high);
a25de534 241 remaining = (high - low) + 1;
227b60f5 242
63862b5b 243 rand = prandom_u32();
8fc54f68 244 first = reciprocal_scale(rand, remaining) + low;
98322f22
ED		 245 /*
246 * force rand to be an odd multiple of UDP_HTABLE_SIZE
247 */
f86dcc5a 248 rand = (rand | 1) * (udptable->mask + 1);
5781b235
ED		 249 last = first + udptable->mask + 1;
250 do {
f86dcc5a 251 hslot = udp_hashslot(udptable, net, first);
98322f22 252 bitmap_zero(bitmap, PORTS_PER_CHAIN);
645ca708 253 spin_lock_bh(&hslot->lock);
98322f22 254 udp_lib_lport_inuse(net, snum, hslot, bitmap, sk,
fe38d2a1 255 udptable->log);
98322f22
ED		 256
257 snum = first;
258 /*
259 * Iterate on all possible values of snum for this hash.
260 * Using steps of an odd multiple of UDP_HTABLE_SIZE
261 * give us randomization and full range coverage.
262 */
9088c560 263 do {
98322f22 264 if (low <= snum && snum <= high &&
e3826f1e 265 !test_bit(snum >> udptable->log, bitmap) &&
122ff243 266 !inet_is_local_reserved_port(net, snum))
98322f22
ED		 267 goto found;
268 snum += rand;
269 } while (snum != first);
270 spin_unlock_bh(&hslot->lock);
df560056 271 cond_resched();
5781b235 272 } while (++first != last);
98322f22 273 goto fail;
645ca708 274 } else {
f86dcc5a 275 hslot = udp_hashslot(udptable, net, snum);
645ca708 276 spin_lock_bh(&hslot->lock);
30fff923
ED		 277 if (hslot->count > 10) {
278 int exist;
279 unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum;
280
281 slot2 &= udptable->mask;
282 hash2_nulladdr &= udptable->mask;
283
284 hslot2 = udp_hashslot2(udptable, slot2);
285 if (hslot->count < hslot2->count)
286 goto scan_primary_hash;
287
fe38d2a1 288 exist = udp_lib_lport_inuse2(net, snum, hslot2, sk);
30fff923
ED		 289 if (!exist && (hash2_nulladdr != slot2)) {
290 hslot2 = udp_hashslot2(udptable, hash2_nulladdr);
291 exist = udp_lib_lport_inuse2(net, snum, hslot2,
fe38d2a1 292 sk);
30fff923
ED		 293 }
294 if (exist)
295 goto fail_unlock;
296 else
297 goto found;
298 }
299scan_primary_hash:
fe38d2a1 300 if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0))
645ca708
ED		 301 goto fail_unlock;
302 }
98322f22 303found:
c720c7e8 304 inet_sk(sk)->inet_num = snum;
d4cada4a
ED		 305 udp_sk(sk)->udp_port_hash = snum;
306 udp_sk(sk)->udp_portaddr_hash ^= snum;
1da177e4 307 if (sk_unhashed(sk)) {
e32ea7e7 308 if (sk->sk_reuseport &&
fe38d2a1 309 udp_reuseport_add_sock(sk, hslot)) {
e32ea7e7
CG		 310 inet_sk(sk)->inet_num = 0;
311 udp_sk(sk)->udp_port_hash = 0;
312 udp_sk(sk)->udp_portaddr_hash ^= snum;
313 goto fail_unlock;
314 }
315
ca065d0c 316 sk_add_node_rcu(sk, &hslot->head);
fdcc8aa9 317 hslot->count++;
c29a0bc4 318 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
512615b6
ED		 319
320 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
321 spin_lock(&hslot2->lock);
d894ba18 322 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
1602f49b
DM		 323 sk->sk_family == AF_INET6)
324 hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node,
325 &hslot2->head);
d894ba18 326 else
1602f49b
DM		 327 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
328 &hslot2->head);
512615b6
ED		 329 hslot2->count++;
330 spin_unlock(&hslot2->lock);
1da177e4 331 }
ca065d0c 332 sock_set_flag(sk, SOCK_RCU_FREE);
25030a7f 333 error = 0;
645ca708
ED		 334fail_unlock:
335 spin_unlock_bh(&hslot->lock);
1da177e4 336fail:
25030a7f
GR		 337 return error;
338}
c482c568 339EXPORT_SYMBOL(udp_lib_get_port);
25030a7f 340
6ba5a3c5 341int udp_v4_get_port(struct sock *sk, unsigned short snum)
db8dac20 342{
30fff923 343 unsigned int hash2_nulladdr =
f0b1e64c 344 ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
30fff923 345 unsigned int hash2_partial =
f0b1e64c 346 ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
30fff923 347
d4cada4a 348 /* precompute partial secondary hash */
30fff923 349 udp_sk(sk)->udp_portaddr_hash = hash2_partial;
fe38d2a1 350 return udp_lib_get_port(sk, snum, hash2_nulladdr);
db8dac20
DM		 351}
352
d1e37288
SX		 353static int compute_score(struct sock *sk, struct net *net,
354 __be32 saddr, __be16 sport,
fb74c277 355 __be32 daddr, unsigned short hnum,
73545373 356 int dif, int sdif)
645ca708 357{
60c04aec
JP		 358 int score;
359 struct inet_sock *inet;
6da5b0f0 360 bool dev_match;
645ca708 361
60c04aec
JP		 362 if (!net_eq(sock_net(sk), net) ||
363 udp_sk(sk)->udp_port_hash != hnum ||
364 ipv6_only_sock(sk))
365 return -1;
645ca708 366
4cdeeee9
PO		 367 if (sk->sk_rcv_saddr != daddr)
368 return -1;
60c04aec 369
4cdeeee9 370 score = (sk->sk_family == PF_INET) ? 2 : 1;
60c04aec 371
4cdeeee9 372 inet = inet_sk(sk);
60c04aec
JP		 373 if (inet->inet_daddr) {
374 if (inet->inet_daddr != saddr)
375 return -1;
376 score += 4;
377 }
378
379 if (inet->inet_dport) {
380 if (inet->inet_dport != sport)
381 return -1;
382 score += 4;
383 }
384
6da5b0f0
MM		 385 dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
386 dif, sdif);
387 if (!dev_match)
388 return -1;
389 score += 4;
fb74c277 390
7170a977 391 if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
70da268b 392 score++;
645ca708
ED		 393 return score;
394}
395
6eada011
ED		 396static u32 udp_ehashfn(const struct net *net, const __be32 laddr,
397 const __u16 lport, const __be32 faddr,
398 const __be16 fport)
65cd8033 399{
1bbdceef
HFS		 400 static u32 udp_ehash_secret __read_mostly;
401
402 net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret));
403
65cd8033 404 return __inet_ehashfn(laddr, lport, faddr, fport,
1bbdceef 405 udp_ehash_secret + net_hash_mix(net));
65cd8033
HFS		 406}
407
d1e37288 408/* called with rcu_read_lock() */
5051ebd2 409static struct sock *udp4_lib_lookup2(struct net *net,
fb74c277
DA		 410 __be32 saddr, __be16 sport,
411 __be32 daddr, unsigned int hnum,
73545373 412 int dif, int sdif,
fb74c277
DA		 413 struct udp_hslot *hslot2,
414 struct sk_buff *skb)
5051ebd2
ED		 415{
416 struct sock *sk, *result;
e94a62f5 417 int score, badness;
ba418fa3 418 u32 hash = 0;
5051ebd2 419
5051ebd2 420 result = NULL;
ba418fa3 421 badness = 0;
ca065d0c 422 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
d1e37288 423 score = compute_score(sk, net, saddr, sport,
73545373 424 daddr, hnum, dif, sdif);
5051ebd2 425 if (score > badness) {
acdcecc6
WB		 426 if (sk->sk_reuseport &&
427 sk->sk_state != TCP_ESTABLISHED) {
65cd8033
HFS		 428 hash = udp_ehashfn(net, daddr, hnum,
429 saddr, sport);
ca065d0c 430 result = reuseport_select_sock(sk, hash, skb,
ed0dfffd 431 sizeof(struct udphdr));
acdcecc6 432 if (result && !reuseport_has_conns(sk, false))
ca065d0c 433 return result;
ba418fa3 434 }
ca065d0c
ED		 435 badness = score;
436 result = sk;
5051ebd2
ED		 437 }
438 }
5051ebd2
ED		 439 return result;
440}
441
db8dac20
DM		 442/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
443 * harder than this. -DaveM
444 */
fce82338 445struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr,
fb74c277
DA		 446 __be16 sport, __be32 daddr, __be16 dport, int dif,
447 int sdif, struct udp_table *udptable, struct sk_buff *skb)
db8dac20 448{
4cdeeee9 449 struct sock *result;
db8dac20 450 unsigned short hnum = ntohs(dport);
4cdeeee9
PO		 451 unsigned int hash2, slot2;
452 struct udp_hslot *hslot2;
645ca708 453
4cdeeee9
PO		 454 hash2 = ipv4_portaddr_hash(net, daddr, hnum);
455 slot2 = hash2 & udptable->mask;
456 hslot2 = &udptable->hash2[slot2];
457
458 result = udp4_lib_lookup2(net, saddr, sport,
459 daddr, hnum, dif, sdif,
73545373 460 hslot2, skb);
4cdeeee9
PO		 461 if (!result) {
462 hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
5051ebd2
ED		 463 slot2 = hash2 & udptable->mask;
464 hslot2 = &udptable->hash2[slot2];
5051ebd2
ED		 465
466 result = udp4_lib_lookup2(net, saddr, sport,
4cdeeee9 467 htonl(INADDR_ANY), hnum, dif, sdif,
73545373 468 hslot2, skb);
db8dac20 469 }
88e235b8 470 if (IS_ERR(result))
4cdeeee9 471 return NULL;
db8dac20
DM		 472 return result;
473}
fce82338 474EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
db8dac20 475
607c4aaf
KK		 476static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
477 __be16 sport, __be16 dport,
645ca708 478 struct udp_table *udptable)
607c4aaf
KK		 479{
480 const struct iphdr *iph = ip_hdr(skb);
481
ed7cbbce 482 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
8afdd99a 483 iph->daddr, dport, inet_iif(skb),
fb74c277 484 inet_sdif(skb), udptable, skb);
607c4aaf
KK		 485}
486
63058308
TH		 487struct sock *udp4_lib_lookup_skb(struct sk_buff *skb,
488 __be16 sport, __be16 dport)
489{
257a525f
MKL		 490 const struct iphdr *iph = ip_hdr(skb);
491
492 return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
493 iph->daddr, dport, inet_iif(skb),
494 inet_sdif(skb), &udp_table, NULL);
63058308
TH		 495}
496EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb);
497
ca065d0c
ED		 498/* Must be called under rcu_read_lock().
499 * Does increment socket refcount.
500 */
6e86000c 501#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
bcd41303
KK		 502struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport,
503 __be32 daddr, __be16 dport, int dif)
504{
ca065d0c
ED		 505 struct sock *sk;
506
507 sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport,
fb74c277 508 dif, 0, &udp_table, NULL);
41c6d650 509 if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
ca065d0c
ED		 510 sk = NULL;
511 return sk;
bcd41303
KK		 512}
513EXPORT_SYMBOL_GPL(udp4_lib_lookup);
ca065d0c 514#endif
bcd41303 515
421b3885
SB		 516static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk,
517 __be16 loc_port, __be32 loc_addr,
518 __be16 rmt_port, __be32 rmt_addr,
fb74c277 519 int dif, int sdif, unsigned short hnum)
421b3885
SB		 520{
521 struct inet_sock *inet = inet_sk(sk);
522
523 if (!net_eq(sock_net(sk), net) ||
524 udp_sk(sk)->udp_port_hash != hnum ||
525 (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
526 (inet->inet_dport != rmt_port && inet->inet_dport) ||
527 (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
528 ipv6_only_sock(sk) ||
82ba25c6 529 !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
421b3885 530 return false;
60d9b031 531 if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
421b3885
SB		 532 return false;
533 return true;
534}
535
a36e185e
SB		 536DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
537void udp_encap_enable(void)
538{
9c480601 539 static_branch_inc(&udp_encap_needed_key);
a36e185e
SB		 540}
541EXPORT_SYMBOL(udp_encap_enable);
542
e7cc0824
SB		 543/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
544 * through error handlers in encapsulations looking for a match.
545 */
546static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
547{
548 int i;
549
550 for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
551 int (*handler)(struct sk_buff *skb, u32 info);
92b95364 552 const struct ip_tunnel_encap_ops *encap;
e7cc0824 553
92b95364
PA		 554 encap = rcu_dereference(iptun_encaps[i]);
555 if (!encap)
e7cc0824 556 continue;
92b95364 557 handler = encap->err_handler;
e7cc0824
SB		 558 if (handler && !handler(skb, info))
559 return 0;
560 }
561
562 return -ENOENT;
563}
564
a36e185e
SB		 565/* Try to match ICMP errors to UDP tunnels by looking up a socket without
566 * reversing source and destination port: this will match tunnels that force the
567 * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
568 * lwtunnels might actually break this assumption by being configured with
569 * different destination ports on endpoints, in this case we won't be able to
570 * trace ICMP messages back to them.
571 *
e7cc0824
SB		 572 * If this doesn't match any socket, probe tunnels with arbitrary destination
573 * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
574 * we've sent packets to won't necessarily match the local destination port.
575 *
a36e185e
SB		 576 * Then ask the tunnel implementation to match the error against a valid
577 * association.
578 *
e7cc0824
SB		 579 * Return an error if we can't find a match, the socket if we need further
580 * processing, zero otherwise.
a36e185e
SB		 581 */
582static struct sock *__udp4_lib_err_encap(struct net *net,
583 const struct iphdr *iph,
584 struct udphdr *uh,
585 struct udp_table *udptable,
e7cc0824 586 struct sk_buff *skb, u32 info)
a36e185e 587{
a36e185e 588 int network_offset, transport_offset;
a36e185e
SB		 589 struct sock *sk;
590
a36e185e
SB		 591 network_offset = skb_network_offset(skb);
592 transport_offset = skb_transport_offset(skb);
593
594 /* Network header needs to point to the outer IPv4 header inside ICMP */
595 skb_reset_network_header(skb);
596
597 /* Transport header needs to point to the UDP header */
598 skb_set_transport_header(skb, iph->ihl << 2);
599
e7cc0824
SB		 600 sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
601 iph->saddr, uh->dest, skb->dev->ifindex, 0,
602 udptable, NULL);
603 if (sk) {
604 int (*lookup)(struct sock *sk, struct sk_buff *skb);
605 struct udp_sock *up = udp_sk(sk);
606
607 lookup = READ_ONCE(up->encap_err_lookup);
608 if (!lookup || lookup(sk, skb))
609 sk = NULL;
610 }
611
612 if (!sk)
613 sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
a36e185e
SB		 614
615 skb_set_transport_header(skb, transport_offset);
616 skb_set_network_header(skb, network_offset);
617
618 return sk;
619}
620
db8dac20
DM		 621/*
622 * This routine is called by the ICMP module when it gets some
623 * sort of error condition. If err < 0 then the socket should
624 * be closed and the error returned to the user. If err > 0
625 * it's just the icmp type << 8 | icmp code.
626 * Header points to the ip header of the error packet. We move
627 * on past this. Then (as it used to claim before adjustment)
628 * header points to the first 8 bytes of the udp header. We need
629 * to find the appropriate port.
630 */
631
32bbd879 632int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable)
db8dac20
DM		 633{
634 struct inet_sock *inet;
b71d1d42 635 const struct iphdr *iph = (const struct iphdr *)skb->data;
c482c568 636 struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2));
db8dac20
DM		 637 const int type = icmp_hdr(skb)->type;
638 const int code = icmp_hdr(skb)->code;
a36e185e 639 bool tunnel = false;
db8dac20
DM		 640 struct sock *sk;
641 int harderr;
642 int err;
fd54d716 643 struct net *net = dev_net(skb->dev);
db8dac20 644
fd54d716 645 sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
f64bf6b8
MM		 646 iph->saddr, uh->source, skb->dev->ifindex,
647 inet_sdif(skb), udptable, NULL);
51456b29 648 if (!sk) {
a36e185e 649 /* No socket for error: try tunnels before discarding */
e7cc0824
SB		 650 sk = ERR_PTR(-ENOENT);
651 if (static_branch_unlikely(&udp_encap_needed_key)) {
652 sk = __udp4_lib_err_encap(net, iph, uh, udptable, skb,
653 info);
654 if (!sk)
655 return 0;
656 }
a36e185e 657
e7cc0824 658 if (IS_ERR(sk)) {
a36e185e 659 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
e7cc0824 660 return PTR_ERR(sk);
a36e185e 661 }
e7cc0824 662
a36e185e 663 tunnel = true;
db8dac20
DM		 664 }
665
666 err = 0;
667 harderr = 0;
668 inet = inet_sk(sk);
669
670 switch (type) {
671 default:
672 case ICMP_TIME_EXCEEDED:
673 err = EHOSTUNREACH;
674 break;
675 case ICMP_SOURCE_QUENCH:
676 goto out;
677 case ICMP_PARAMETERPROB:
678 err = EPROTO;
679 harderr = 1;
680 break;
681 case ICMP_DEST_UNREACH:
682 if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
36393395 683 ipv4_sk_update_pmtu(skb, sk, info);
db8dac20
DM		 684 if (inet->pmtudisc != IP_PMTUDISC_DONT) {
685 err = EMSGSIZE;
686 harderr = 1;
687 break;
688 }
689 goto out;
690 }
691 err = EHOSTUNREACH;
692 if (code <= NR_ICMP_UNREACH) {
693 harderr = icmp_err_convert[code].fatal;
694 err = icmp_err_convert[code].errno;
695 }
696 break;
55be7a9c
DM		 697 case ICMP_REDIRECT:
698 ipv4_sk_redirect(skb, sk);
1a462d18 699 goto out;
db8dac20
DM		 700 }
701
702 /*
703 * RFC1122: OK. Passes ICMP errors back to application, as per
704 * 4.1.3.3.
705 */
a36e185e
SB		 706 if (tunnel) {
707 /* ...not for tunnels though: we don't have a sending socket */
708 goto out;
709 }
db8dac20
DM		 710 if (!inet->recverr) {
711 if (!harderr || sk->sk_state != TCP_ESTABLISHED)
712 goto out;
b1faf566 713 } else
c482c568 714 ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
b1faf566 715
db8dac20
DM		 716 sk->sk_err = err;
717 sk->sk_error_report(sk);
718out:
32bbd879 719 return 0;
db8dac20
DM		 720}
721
32bbd879 722int udp_err(struct sk_buff *skb, u32 info)
db8dac20 723{
32bbd879 724 return __udp4_lib_err(skb, info, &udp_table);
db8dac20
DM		 725}
726
727/*
728 * Throw away all pending data and cancel the corking. Socket is locked.
729 */
36d926b9 730void udp_flush_pending_frames(struct sock *sk)
db8dac20
DM		 731{
732 struct udp_sock *up = udp_sk(sk);
733
734 if (up->pending) {
735 up->len = 0;
736 up->pending = 0;
737 ip_flush_pending_frames(sk);
738 }
739}
36d926b9 740EXPORT_SYMBOL(udp_flush_pending_frames);
db8dac20
DM		 741
742/**
f6b9664f 743 * udp4_hwcsum - handle outgoing HW checksumming
db8dac20
DM		 744 * @skb: sk_buff containing the filled-in UDP header
745 * (checksum field must be zeroed out)
f6b9664f
HX		 746 * @src: source IP address
747 * @dst: destination IP address
db8dac20 748 */
c26bf4a5 749void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
db8dac20 750{
db8dac20 751 struct udphdr *uh = udp_hdr(skb);
f6b9664f
HX		 752 int offset = skb_transport_offset(skb);
753 int len = skb->len - offset;
754 int hlen = len;
db8dac20
DM		 755 __wsum csum = 0;
756
ebbe495f 757 if (!skb_has_frag_list(skb)) {
db8dac20
DM		 758 /*
759 * Only one fragment on the socket.
760 */
761 skb->csum_start = skb_transport_header(skb) - skb->head;
762 skb->csum_offset = offsetof(struct udphdr, check);
f6b9664f
HX		 763 uh->check = ~csum_tcpudp_magic(src, dst, len,
764 IPPROTO_UDP, 0);
db8dac20 765 } else {
ebbe495f
WC		 766 struct sk_buff *frags;
767
db8dac20
DM		 768 /*
769 * HW-checksum won't work as there are two or more
770 * fragments on the socket so that all csums of sk_buffs
771 * should be together
772 */
ebbe495f 773 skb_walk_frags(skb, frags) {
f6b9664f
HX		 774 csum = csum_add(csum, frags->csum);
775 hlen -= frags->len;
ebbe495f 776 }
db8dac20 777
f6b9664f 778 csum = skb_checksum(skb, offset, hlen, csum);
db8dac20
DM		 779 skb->ip_summed = CHECKSUM_NONE;
780
db8dac20
DM		 781 uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
782 if (uh->check == 0)
783 uh->check = CSUM_MANGLED_0;
784 }
785}
c26bf4a5 786EXPORT_SYMBOL_GPL(udp4_hwcsum);
db8dac20 787
af5fcba7
TH		 788/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
789 * for the simple case like when setting the checksum for a UDP tunnel.
790 */
791void udp_set_csum(bool nocheck, struct sk_buff *skb,
792 __be32 saddr, __be32 daddr, int len)
793{
794 struct udphdr *uh = udp_hdr(skb);
795
179bc67f 796 if (nocheck) {
af5fcba7 797 uh->check = 0;
179bc67f 798 } else if (skb_is_gso(skb)) {
af5fcba7 799 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
179bc67f
EC		 800 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
801 uh->check = 0;
802 uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb));
803 if (uh->check == 0)
804 uh->check = CSUM_MANGLED_0;
d75f1306 805 } else {
af5fcba7
TH		 806 skb->ip_summed = CHECKSUM_PARTIAL;
807 skb->csum_start = skb_transport_header(skb) - skb->head;
808 skb->csum_offset = offsetof(struct udphdr, check);
809 uh->check = ~udp_v4_check(len, saddr, daddr, 0);
af5fcba7
TH		 810 }
811}
812EXPORT_SYMBOL(udp_set_csum);
813
bec1f6f6
WB		 814static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
815 struct inet_cork *cork)
db8dac20 816{
f6b9664f 817 struct sock *sk = skb->sk;
db8dac20 818 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM		 819 struct udphdr *uh;
820 int err = 0;
821 int is_udplite = IS_UDPLITE(sk);
f6b9664f
HX		 822 int offset = skb_transport_offset(skb);
823 int len = skb->len - offset;
4094871d 824 int datalen = len - sizeof(*uh);
db8dac20
DM		 825 __wsum csum = 0;
826
db8dac20
DM		 827 /*
828 * Create a UDP header
829 */
830 uh = udp_hdr(skb);
f6b9664f 831 uh->source = inet->inet_sport;
79ab0531 832 uh->dest = fl4->fl4_dport;
f6b9664f 833 uh->len = htons(len);
db8dac20
DM		 834 uh->check = 0;
835
bec1f6f6
WB		 836 if (cork->gso_size) {
837 const int hlen = skb_network_header_len(skb) +
838 sizeof(struct udphdr);
839
0f149c9f
WB		 840 if (hlen + cork->gso_size > cork->fragsize) {
841 kfree_skb(skb);
bec1f6f6 842 return -EINVAL;
0f149c9f
WB		 843 }
844 if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) {
845 kfree_skb(skb);
bec1f6f6 846 return -EINVAL;
0f149c9f
WB		 847 }
848 if (sk->sk_no_check_tx) {
849 kfree_skb(skb);
a8c744a8 850 return -EINVAL;
0f149c9f 851 }
ff06342c 852 if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite ||
0f149c9f
WB		 853 dst_xfrm(skb_dst(skb))) {
854 kfree_skb(skb);
bec1f6f6 855 return -EIO;
0f149c9f 856 }
bec1f6f6 857
4094871d
JH		 858 if (datalen > cork->gso_size) {
859 skb_shinfo(skb)->gso_size = cork->gso_size;
860 skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
861 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
862 cork->gso_size);
863 }
a8c744a8 864 goto csum_partial;
bec1f6f6
WB		 865 }
866
db8dac20 867 if (is_udplite) /* UDP-Lite */
f6b9664f 868 csum = udplite_csum(skb);
db8dac20 869
ab2fb7e3 870 else if (sk->sk_no_check_tx) { /* UDP csum off */
db8dac20
DM		 871
872 skb->ip_summed = CHECKSUM_NONE;
873 goto send;
874
875 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
a8c744a8 876csum_partial:
db8dac20 877
79ab0531 878 udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
db8dac20
DM		 879 goto send;
880
f6b9664f
HX		 881 } else
882 csum = udp_csum(skb);
db8dac20
DM		 883
884 /* add protocol-dependent pseudo-header */
79ab0531 885 uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
c482c568 886 sk->sk_protocol, csum);
db8dac20
DM		 887 if (uh->check == 0)
888 uh->check = CSUM_MANGLED_0;
889
890send:
b5ec8eea 891 err = ip_send_skb(sock_net(sk), skb);
6ce9e7b5
ED		 892 if (err) {
893 if (err == -ENOBUFS && !inet->recverr) {
6aef70a8
ED		 894 UDP_INC_STATS(sock_net(sk),
895 UDP_MIB_SNDBUFERRORS, is_udplite);
6ce9e7b5
ED		 896 err = 0;
897 }
898 } else
6aef70a8
ED		 899 UDP_INC_STATS(sock_net(sk),
900 UDP_MIB_OUTDATAGRAMS, is_udplite);
f6b9664f
HX		 901 return err;
902}
903
904/*
905 * Push out all pending data as one UDP datagram. Socket is locked.
906 */
8822b64a 907int udp_push_pending_frames(struct sock *sk)
f6b9664f
HX		 908{
909 struct udp_sock *up = udp_sk(sk);
910 struct inet_sock *inet = inet_sk(sk);
b6f21b26 911 struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
f6b9664f
HX		 912 struct sk_buff *skb;
913 int err = 0;
914
77968b78 915 skb = ip_finish_skb(sk, fl4);
f6b9664f
HX		 916 if (!skb)
917 goto out;
918
bec1f6f6 919 err = udp_send_skb(skb, fl4, &inet->cork.base);
f6b9664f 920
db8dac20
DM		 921out:
922 up->len = 0;
923 up->pending = 0;
db8dac20
DM		 924 return err;
925}
8822b64a 926EXPORT_SYMBOL(udp_push_pending_frames);
db8dac20 927
2e8de857
WB		 928static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
929{
930 switch (cmsg->cmsg_type) {
931 case UDP_SEGMENT:
932 if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16)))
933 return -EINVAL;
934 *gso_size = *(__u16 *)CMSG_DATA(cmsg);
935 return 0;
936 default:
937 return -EINVAL;
938 }
939}
940
941int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
942{
943 struct cmsghdr *cmsg;
944 bool need_ip = false;
945 int err;
946
947 for_each_cmsghdr(cmsg, msg) {
948 if (!CMSG_OK(msg, cmsg))
949 return -EINVAL;
950
951 if (cmsg->cmsg_level != SOL_UDP) {
952 need_ip = true;
953 continue;
954 }
955
956 err = __udp_cmsg_send(cmsg, gso_size);
957 if (err)
958 return err;
959 }
960
961 return need_ip;
962}
963EXPORT_SYMBOL_GPL(udp_cmsg_send);
964
1b784140 965int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
db8dac20
DM		 966{
967 struct inet_sock *inet = inet_sk(sk);
968 struct udp_sock *up = udp_sk(sk);
1cedee13 969 DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
e474995f 970 struct flowi4 fl4_stack;
b6f21b26 971 struct flowi4 *fl4;
db8dac20
DM		 972 int ulen = len;
973 struct ipcm_cookie ipc;
974 struct rtable *rt = NULL;
975 int free = 0;
976 int connected = 0;
977 __be32 daddr, faddr, saddr;
978 __be16 dport;
979 u8 tos;
980 int err, is_udplite = IS_UDPLITE(sk);
981 int corkreq = up->corkflag || msg->msg_flags&MSG_MORE;
982 int (*getfrag)(void *, char *, int, int, int, struct sk_buff *);
903ab86d 983 struct sk_buff *skb;
f6d8bd05 984 struct ip_options_data opt_copy;
db8dac20
DM		 985
986 if (len > 0xFFFF)
987 return -EMSGSIZE;
988
989 /*
990 * Check the flags.
991 */
992
c482c568 993 if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
db8dac20
DM		 994 return -EOPNOTSUPP;
995
903ab86d
HX		 996 getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag;
997
f5fca608 998 fl4 = &inet->cork.fl.u.ip4;
db8dac20
DM		 999 if (up->pending) {
1000 /*
1001 * There are pending frames.
1002 * The socket lock must be held while it's corked.
1003 */
1004 lock_sock(sk);
1005 if (likely(up->pending)) {
1006 if (unlikely(up->pending != AF_INET)) {
1007 release_sock(sk);
1008 return -EINVAL;
1009 }
1010 goto do_append_data;
1011 }
1012 release_sock(sk);
1013 }
1014 ulen += sizeof(struct udphdr);
1015
1016 /*
1017 * Get and verify the address.
1018 */
1cedee13 1019 if (usin) {
db8dac20
DM		 1020 if (msg->msg_namelen < sizeof(*usin))
1021 return -EINVAL;
1022 if (usin->sin_family != AF_INET) {
1023 if (usin->sin_family != AF_UNSPEC)
1024 return -EAFNOSUPPORT;
1025 }
1026
1027 daddr = usin->sin_addr.s_addr;
1028 dport = usin->sin_port;
1029 if (dport == 0)
1030 return -EINVAL;
1031 } else {
1032 if (sk->sk_state != TCP_ESTABLISHED)
1033 return -EDESTADDRREQ;
c720c7e8
ED		 1034 daddr = inet->inet_daddr;
1035 dport = inet->inet_dport;
db8dac20
DM		 1036 /* Open fast path for connected socket.
1037 Route will not be used, if at least one option is set.
1038 */
1039 connected = 1;
1040 }
db8dac20 1041
35178206 1042 ipcm_init_sk(&ipc, inet);
bec1f6f6 1043 ipc.gso_size = up->gso_size;
bf84a010 1044
db8dac20 1045 if (msg->msg_controllen) {
2e8de857
WB		 1046 err = udp_cmsg_send(sk, msg, &ipc.gso_size);
1047 if (err > 0)
1048 err = ip_cmsg_send(sk, msg, &ipc,
1049 sk->sk_family == AF_INET6);
1050 if (unlikely(err < 0)) {
91948309 1051 kfree(ipc.opt);
db8dac20 1052 return err;
91948309 1053 }
db8dac20
DM		 1054 if (ipc.opt)
1055 free = 1;
1056 connected = 0;
1057 }
f6d8bd05
ED		 1058 if (!ipc.opt) {
1059 struct ip_options_rcu *inet_opt;
1060
1061 rcu_read_lock();
1062 inet_opt = rcu_dereference(inet->inet_opt);
1063 if (inet_opt) {
1064 memcpy(&opt_copy, inet_opt,
1065 sizeof(*inet_opt) + inet_opt->opt.optlen);
1066 ipc.opt = &opt_copy.opt;
1067 }
1068 rcu_read_unlock();
1069 }
db8dac20 1070
1cedee13
AI		 1071 if (cgroup_bpf_enabled && !connected) {
1072 err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
1073 (struct sockaddr *)usin, &ipc.addr);
1074 if (err)
1075 goto out_free;
1076 if (usin) {
1077 if (usin->sin_port == 0) {
1078 /* BPF program set invalid port. Reject it. */
1079 err = -EINVAL;
1080 goto out_free;
1081 }
1082 daddr = usin->sin_addr.s_addr;
1083 dport = usin->sin_port;
1084 }
1085 }
1086
db8dac20
DM		 1087 saddr = ipc.addr;
1088 ipc.addr = faddr = daddr;
1089
f6d8bd05 1090 if (ipc.opt && ipc.opt->opt.srr) {
1b97013b
AI		 1091 if (!daddr) {
1092 err = -EINVAL;
1093 goto out_free;
1094 }
f6d8bd05 1095 faddr = ipc.opt->opt.faddr;
db8dac20
DM		 1096 connected = 0;
1097 }
aa661581 1098 tos = get_rttos(&ipc, inet);
db8dac20
DM		 1099 if (sock_flag(sk, SOCK_LOCALROUTE) ||
1100 (msg->msg_flags & MSG_DONTROUTE) ||
f6d8bd05 1101 (ipc.opt && ipc.opt->opt.is_strictroute)) {
db8dac20
DM		 1102 tos |= RTO_ONLINK;
1103 connected = 0;
1104 }
1105
1106 if (ipv4_is_multicast(daddr)) {
854da991 1107 if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
db8dac20
DM		 1108 ipc.oif = inet->mc_index;
1109 if (!saddr)
1110 saddr = inet->mc_addr;
1111 connected = 0;
9515a2e0 1112 } else if (!ipc.oif) {
76e21053 1113 ipc.oif = inet->uc_index;
9515a2e0
DA		 1114 } else if (ipv4_is_lbcast(daddr) && inet->uc_index) {
1115 /* oif is set, packet is to local broadcast and
1116 * and uc_index is set. oif is most likely set
1117 * by sk_bound_dev_if. If uc_index != oif check if the
1118 * oif is an L3 master and uc_index is an L3 slave.
1119 * If so, we want to allow the send using the uc_index.
1120 */
1121 if (ipc.oif != inet->uc_index &&
1122 ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
1123 inet->uc_index)) {
1124 ipc.oif = inet->uc_index;
1125 }
1126 }
db8dac20
DM		 1127
1128 if (connected)
c482c568 1129 rt = (struct rtable *)sk_dst_check(sk, 0);
db8dac20 1130
51456b29 1131 if (!rt) {
84a3aa00 1132 struct net *net = sock_net(sk);
9a24abfa 1133 __u8 flow_flags = inet_sk_flowi_flags(sk);
84a3aa00 1134
e474995f 1135 fl4 = &fl4_stack;
9a24abfa 1136
c6af0c22 1137 flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos,
c0951cbc 1138 RT_SCOPE_UNIVERSE, sk->sk_protocol,
9a24abfa 1139 flow_flags,
e2d118a1
LC		 1140 faddr, saddr, dport, inet->inet_sport,
1141 sk->sk_uid);
c0951cbc 1142
e474995f
DM		 1143 security_sk_classify_flow(sk, flowi4_to_flowi(fl4));
1144 rt = ip_route_output_flow(net, fl4, sk);
b23dd4fe
DM		 1145 if (IS_ERR(rt)) {
1146 err = PTR_ERR(rt);
06dc94b1 1147 rt = NULL;
db8dac20 1148 if (err == -ENETUNREACH)
f1d8cba6 1149 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
db8dac20
DM		 1150 goto out;
1151 }
1152
1153 err = -EACCES;
1154 if ((rt->rt_flags & RTCF_BROADCAST) &&
1155 !sock_flag(sk, SOCK_BROADCAST))
1156 goto out;
1157 if (connected)
d8d1f30b 1158 sk_dst_set(sk, dst_clone(&rt->dst));
db8dac20
DM		 1159 }
1160
1161 if (msg->msg_flags&MSG_CONFIRM)
1162 goto do_confirm;
1163back_from_confirm:
1164
e474995f 1165 saddr = fl4->saddr;
db8dac20 1166 if (!ipc.addr)
e474995f 1167 daddr = ipc.addr = fl4->daddr;
db8dac20 1168
903ab86d
HX		 1169 /* Lockless fast path for the non-corking case. */
1170 if (!corkreq) {
1cd7884d
WB		 1171 struct inet_cork cork;
1172
f69e6d13 1173 skb = ip_make_skb(sk, fl4, getfrag, msg, ulen,
903ab86d 1174 sizeof(struct udphdr), &ipc, &rt,
1cd7884d 1175 &cork, msg->msg_flags);
903ab86d 1176 err = PTR_ERR(skb);
50c3a487 1177 if (!IS_ERR_OR_NULL(skb))
bec1f6f6 1178 err = udp_send_skb(skb, fl4, &cork);
903ab86d
HX		 1179 goto out;
1180 }
1181
db8dac20
DM		 1182 lock_sock(sk);
1183 if (unlikely(up->pending)) {
1184 /* The socket is already corked while preparing it. */
1185 /* ... which is an evident application bug. --ANK */
1186 release_sock(sk);
1187
197df02c 1188 net_dbg_ratelimited("socket already corked\n");
db8dac20
DM		 1189 err = -EINVAL;
1190 goto out;
1191 }
1192 /*
1193 * Now cork the socket to pend data.
1194 */
b6f21b26
DM		 1195 fl4 = &inet->cork.fl.u.ip4;
1196 fl4->daddr = daddr;
1197 fl4->saddr = saddr;
9cce96df
DM		 1198 fl4->fl4_dport = dport;
1199 fl4->fl4_sport = inet->inet_sport;
db8dac20
DM		 1200 up->pending = AF_INET;
1201
1202do_append_data:
1203 up->len += ulen;
f69e6d13 1204 err = ip_append_data(sk, fl4, getfrag, msg, ulen,
f5fca608
DM		 1205 sizeof(struct udphdr), &ipc, &rt,
1206 corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
db8dac20
DM		 1207 if (err)
1208 udp_flush_pending_frames(sk);
1209 else if (!corkreq)
1210 err = udp_push_pending_frames(sk);
1211 else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
1212 up->pending = 0;
1213 release_sock(sk);
1214
1215out:
1216 ip_rt_put(rt);
1b97013b 1217out_free:
db8dac20
DM		 1218 if (free)
1219 kfree(ipc.opt);
1220 if (!err)
1221 return len;
1222 /*
1223 * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
1224 * ENOBUFS might not be good (it's not tunable per se), but otherwise
1225 * we don't have a good statistic (IpOutDiscards but it can be too many
1226 * things). We could add another new stat but at least for now that
1227 * seems like overkill.
1228 */
1229 if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
6aef70a8
ED		 1230 UDP_INC_STATS(sock_net(sk),
1231 UDP_MIB_SNDBUFERRORS, is_udplite);
db8dac20
DM		 1232 }
1233 return err;
1234
1235do_confirm:
0dec879f
JA		 1236 if (msg->msg_flags & MSG_PROBE)
1237 dst_confirm_neigh(&rt->dst, &fl4->daddr);
db8dac20
DM		 1238 if (!(msg->msg_flags&MSG_PROBE) || len)
1239 goto back_from_confirm;
1240 err = 0;
1241 goto out;
1242}
c482c568 1243EXPORT_SYMBOL(udp_sendmsg);
db8dac20
DM		 1244
1245int udp_sendpage(struct sock *sk, struct page *page, int offset,
1246 size_t size, int flags)
1247{
f5fca608 1248 struct inet_sock *inet = inet_sk(sk);
db8dac20
DM		 1249 struct udp_sock *up = udp_sk(sk);
1250 int ret;
1251
d3f7d56a
SL		 1252 if (flags & MSG_SENDPAGE_NOTLAST)
1253 flags |= MSG_MORE;
1254
db8dac20
DM		 1255 if (!up->pending) {
1256 struct msghdr msg = { .msg_flags = flags|MSG_MORE };
1257
1258 /* Call udp_sendmsg to specify destination address which
1259 * sendpage interface can't pass.
1260 * This will succeed only when the socket is connected.
1261 */
1b784140 1262 ret = udp_sendmsg(sk, &msg, 0);
db8dac20
DM		 1263 if (ret < 0)
1264 return ret;
1265 }
1266
1267 lock_sock(sk);
1268
1269 if (unlikely(!up->pending)) {
1270 release_sock(sk);
1271
197df02c 1272 net_dbg_ratelimited("cork failed\n");
db8dac20
DM		 1273 return -EINVAL;
1274 }
1275
f5fca608
DM		 1276 ret = ip_append_page(sk, &inet->cork.fl.u.ip4,
1277 page, offset, size, flags);
db8dac20
DM		 1278 if (ret == -EOPNOTSUPP) {
1279 release_sock(sk);
1280 return sock_no_sendpage(sk->sk_socket, page, offset,
1281 size, flags);
1282 }
1283 if (ret < 0) {
1284 udp_flush_pending_frames(sk);
1285 goto out;
1286 }
1287
1288 up->len += size;
1289 if (!(up->corkflag || (flags&MSG_MORE)))
1290 ret = udp_push_pending_frames(sk);
1291 if (!ret)
1292 ret = size;
1293out:
1294 release_sock(sk);
1295 return ret;
1296}
1297
dce4551c
PA		 1298#define UDP_SKB_IS_STATELESS 0x80000000
1299
b65ac446
PA		 1300static void udp_set_dev_scratch(struct sk_buff *skb)
1301{
dce4551c 1302 struct udp_dev_scratch *scratch = udp_skb_scratch(skb);
b65ac446
PA		 1303
1304 BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long));
dce4551c
PA		 1305 scratch->_tsize_state = skb->truesize;
1306#if BITS_PER_LONG == 64
b65ac446
PA		 1307 scratch->len = skb->len;
1308 scratch->csum_unnecessary = !!skb_csum_unnecessary(skb);
1309 scratch->is_linear = !skb_is_nonlinear(skb);
dce4551c 1310#endif
3bdefdf9
PA		 1311 /* all head states execept sp (dst, sk, nf) are always cleared by
1312 * udp_rcv() and we need to preserve secpath, if present, to eventually
1313 * process IP_CMSG_PASSSEC at recvmsg() time
1314 */
1315 if (likely(!skb_sec_path(skb)))
dce4551c 1316 scratch->_tsize_state |= UDP_SKB_IS_STATELESS;
b65ac446
PA		 1317}
1318
a793183c
ED		 1319static void udp_skb_csum_unnecessary_set(struct sk_buff *skb)
1320{
1321 /* We come here after udp_lib_checksum_complete() returned 0.
1322 * This means that __skb_checksum_complete() might have
1323 * set skb->csum_valid to 1.
1324 * On 64bit platforms, we can set csum_unnecessary
1325 * to true, but only if the skb is not shared.
1326 */
1327#if BITS_PER_LONG == 64
1328 if (!skb_shared(skb))
1329 udp_skb_scratch(skb)->csum_unnecessary = true;
1330#endif
1331}
1332
b65ac446
PA		 1333static int udp_skb_truesize(struct sk_buff *skb)
1334{
dce4551c 1335 return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS;
b65ac446
PA		 1336}
1337
dce4551c 1338static bool udp_skb_has_head_state(struct sk_buff *skb)
b65ac446 1339{
dce4551c 1340 return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS);
b65ac446 1341}
b65ac446 1342
7c13f97f 1343/* fully reclaim rmem/fwd memory allocated for skb */
6dfb4367
PA		 1344static void udp_rmem_release(struct sock *sk, int size, int partial,
1345 bool rx_queue_lock_held)
f970bd9e 1346{
6b229cf7 1347 struct udp_sock *up = udp_sk(sk);
2276f58a 1348 struct sk_buff_head *sk_queue;
f970bd9e
PA		 1349 int amt;
1350
6b229cf7
ED		 1351 if (likely(partial)) {
1352 up->forward_deficit += size;
1353 size = up->forward_deficit;
0d4a6608 1354 if (size < (sk->sk_rcvbuf >> 2))
6b229cf7
ED		 1355 return;
1356 } else {
1357 size += up->forward_deficit;
1358 }
1359 up->forward_deficit = 0;
1360
6dfb4367
PA		 1361 /* acquire the sk_receive_queue for fwd allocated memory scheduling,
1362 * if the called don't held it already
1363 */
2276f58a 1364 sk_queue = &sk->sk_receive_queue;
6dfb4367
PA		 1365 if (!rx_queue_lock_held)
1366 spin_lock(&sk_queue->lock);
1367
2276f58a 1368
f970bd9e
PA		 1369 sk->sk_forward_alloc += size;
1370 amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1);
1371 sk->sk_forward_alloc -= amt;
f970bd9e
PA		 1372
1373 if (amt)
1374 __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT);
02ab0d13
ED		 1375
1376 atomic_sub(size, &sk->sk_rmem_alloc);
2276f58a
PA		 1377
1378 /* this can save us from acquiring the rx queue lock on next receive */
1379 skb_queue_splice_tail_init(sk_queue, &up->reader_queue);
1380
6dfb4367
PA		 1381 if (!rx_queue_lock_held)
1382 spin_unlock(&sk_queue->lock);
f970bd9e
PA		 1383}
1384
2276f58a 1385/* Note: called with reader_queue.lock held.
c84d9490
ED		 1386 * Instead of using skb->truesize here, find a copy of it in skb->dev_scratch
1387 * This avoids a cache line miss while receive_queue lock is held.
1388 * Look at __udp_enqueue_schedule_skb() to find where this copy is done.
1389 */
7c13f97f 1390void udp_skb_destructor(struct sock *sk, struct sk_buff *skb)
f970bd9e 1391{
b65ac446
PA		 1392 prefetch(&skb->data);
1393 udp_rmem_release(sk, udp_skb_truesize(skb), 1, false);
f970bd9e 1394}
7c13f97f 1395EXPORT_SYMBOL(udp_skb_destructor);
f970bd9e 1396
6dfb4367 1397/* as above, but the caller held the rx queue lock, too */
64f5102d 1398static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb)
6dfb4367 1399{
b65ac446
PA		 1400 prefetch(&skb->data);
1401 udp_rmem_release(sk, udp_skb_truesize(skb), 1, true);
6dfb4367
PA		 1402}
1403
4b272750
ED		 1404/* Idea of busylocks is to let producers grab an extra spinlock
1405 * to relieve pressure on the receive_queue spinlock shared by consumer.
1406 * Under flood, this means that only one producer can be in line
1407 * trying to acquire the receive_queue spinlock.
1408 * These busylock can be allocated on a per cpu manner, instead of a
1409 * per socket one (that would consume a cache line per socket)
1410 */
1411static int udp_busylocks_log __read_mostly;
1412static spinlock_t *udp_busylocks __read_mostly;
1413
1414static spinlock_t *busylock_acquire(void *ptr)
1415{
1416 spinlock_t *busy;
1417
1418 busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log);
1419 spin_lock(busy);
1420 return busy;
1421}
1422
1423static void busylock_release(spinlock_t *busy)
1424{
1425 if (busy)
1426 spin_unlock(busy);
1427}
1428
f970bd9e
PA		 1429int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
1430{
1431 struct sk_buff_head *list = &sk->sk_receive_queue;
1432 int rmem, delta, amt, err = -ENOMEM;
4b272750 1433 spinlock_t *busy = NULL;
c8c8b127 1434 int size;
f970bd9e
PA		 1435
1436 /* try to avoid the costly atomic add/sub pair when the receive
1437 * queue is full; always allow at least a packet
1438 */
1439 rmem = atomic_read(&sk->sk_rmem_alloc);
363dc73a 1440 if (rmem > sk->sk_rcvbuf)
f970bd9e
PA		 1441 goto drop;
1442
c8c8b127
ED		 1443 /* Under mem pressure, it might be helpful to help udp_recvmsg()
1444 * having linear skbs :
1445 * - Reduce memory overhead and thus increase receive queue capacity
1446 * - Less cache line misses at copyout() time
1447 * - Less work at consume_skb() (less alien page frag freeing)
1448 */
4b272750 1449 if (rmem > (sk->sk_rcvbuf >> 1)) {
c8c8b127 1450 skb_condense(skb);
4b272750
ED		 1451
1452 busy = busylock_acquire(sk);
1453 }
c8c8b127 1454 size = skb->truesize;
b65ac446 1455 udp_set_dev_scratch(skb);
c8c8b127 1456
f970bd9e
PA		 1457 /* we drop only if the receive buf is full and the receive
1458 * queue contains some other skb
1459 */
1460 rmem = atomic_add_return(size, &sk->sk_rmem_alloc);
363dc73a 1461 if (rmem > (size + sk->sk_rcvbuf))
f970bd9e
PA		 1462 goto uncharge_drop;
1463
1464 spin_lock(&list->lock);
1465 if (size >= sk->sk_forward_alloc) {
1466 amt = sk_mem_pages(size);
1467 delta = amt << SK_MEM_QUANTUM_SHIFT;
1468 if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) {
1469 err = -ENOBUFS;
1470 spin_unlock(&list->lock);
1471 goto uncharge_drop;
1472 }
1473
1474 sk->sk_forward_alloc += delta;
1475 }
1476
1477 sk->sk_forward_alloc -= size;
1478
7c13f97f
PA		 1479 /* no need to setup a destructor, we will explicitly release the
1480 * forward allocated memory on dequeue
1481 */
f970bd9e
PA		 1482 sock_skb_set_dropcount(sk, skb);
1483
1484 __skb_queue_tail(list, skb);
1485 spin_unlock(&list->lock);
1486
1487 if (!sock_flag(sk, SOCK_DEAD))
1488 sk->sk_data_ready(sk);
1489
4b272750 1490 busylock_release(busy);
f970bd9e
PA		 1491 return 0;
1492
1493uncharge_drop:
1494 atomic_sub(skb->truesize, &sk->sk_rmem_alloc);
1495
1496drop:
1497 atomic_inc(&sk->sk_drops);
4b272750 1498 busylock_release(busy);
f970bd9e
PA		 1499 return err;
1500}
1501EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb);
1502
c915fe13 1503void udp_destruct_sock(struct sock *sk)
f970bd9e
PA		 1504{
1505 /* reclaim completely the forward allocated memory */
2276f58a 1506 struct udp_sock *up = udp_sk(sk);
7c13f97f
PA		 1507 unsigned int total = 0;
1508 struct sk_buff *skb;
1509
2276f58a
PA		 1510 skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue);
1511 while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) {
7c13f97f
PA		 1512 total += skb->truesize;
1513 kfree_skb(skb);
1514 }
6dfb4367 1515 udp_rmem_release(sk, total, 0, true);
7c13f97f 1516
f970bd9e
PA		 1517 inet_sock_destruct(sk);
1518}
c915fe13 1519EXPORT_SYMBOL_GPL(udp_destruct_sock);
f970bd9e
PA		 1520
1521int udp_init_sock(struct sock *sk)
1522{
2276f58a 1523 skb_queue_head_init(&udp_sk(sk)->reader_queue);
f970bd9e
PA		 1524 sk->sk_destruct = udp_destruct_sock;
1525 return 0;
1526}
1527EXPORT_SYMBOL_GPL(udp_init_sock);
1528
1529void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
1530{
1531 if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
1532 bool slow = lock_sock_fast(sk);
1533
1534 sk_peek_offset_bwd(sk, len);
1535 unlock_sock_fast(sk, slow);
1536 }
0a463c78 1537
ca2c1418
PA		 1538 if (!skb_unref(skb))
1539 return;
1540
dce4551c
PA		 1541 /* In the more common cases we cleared the head states previously,
1542 * see __udp_queue_rcv_skb().
0ddf3fb2 1543 */
dce4551c 1544 if (unlikely(udp_skb_has_head_state(skb)))
0ddf3fb2 1545 skb_release_head_state(skb);
ca2c1418 1546 __consume_stateless_skb(skb);
f970bd9e
PA		 1547}
1548EXPORT_SYMBOL_GPL(skb_consume_udp);
1549
2276f58a
PA		 1550static struct sk_buff *__first_packet_length(struct sock *sk,
1551 struct sk_buff_head *rcvq,
1552 int *total)
1553{
1554 struct sk_buff *skb;
1555
9bd780f5
PA		 1556 while ((skb = skb_peek(rcvq)) != NULL) {
1557 if (udp_lib_checksum_complete(skb)) {
1558 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
1559 IS_UDPLITE(sk));
1560 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
1561 IS_UDPLITE(sk));
1562 atomic_inc(&sk->sk_drops);
1563 __skb_unlink(skb, rcvq);
1564 *total += skb->truesize;
1565 kfree_skb(skb);
1566 } else {
a793183c 1567 udp_skb_csum_unnecessary_set(skb);
9bd780f5
PA		 1568 break;
1569 }
2276f58a
PA		 1570 }
1571 return skb;
1572}
1573
85584672
ED		 1574/**
1575 * first_packet_length - return length of first packet in receive queue
1576 * @sk: socket
1577 *
1578 * Drops all bad checksum frames, until a valid one is found.
e83c6744 1579 * Returns the length of found skb, or -1 if none is found.
85584672 1580 */
e83c6744 1581static int first_packet_length(struct sock *sk)
85584672 1582{
2276f58a
PA		 1583 struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue;
1584 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
85584672 1585 struct sk_buff *skb;
7c13f97f 1586 int total = 0;
e83c6744 1587 int res;
85584672 1588
85584672 1589 spin_lock_bh(&rcvq->lock);
2276f58a 1590 skb = __first_packet_length(sk, rcvq, &total);
137a0dbe 1591 if (!skb && !skb_queue_empty_lockless(sk_queue)) {
2276f58a
PA		 1592 spin_lock(&sk_queue->lock);
1593 skb_queue_splice_tail_init(sk_queue, rcvq);
1594 spin_unlock(&sk_queue->lock);
1595
1596 skb = __first_packet_length(sk, rcvq, &total);
85584672 1597 }
e83c6744 1598 res = skb ? skb->len : -1;
7c13f97f 1599 if (total)
6dfb4367 1600 udp_rmem_release(sk, total, 1, false);
85584672 1601 spin_unlock_bh(&rcvq->lock);
85584672
ED		 1602 return res;
1603}
1604
1da177e4
LT		 1605/*
1606 * IOCTL requests applicable to the UDP protocol
1607 */
e905a9ed 1608
1da177e4
LT		 1609int udp_ioctl(struct sock *sk, int cmd, unsigned long arg)
1610{
6516c655
SH		 1611 switch (cmd) {
1612 case SIOCOUTQ:
1da177e4 1613 {
31e6d363
ED		 1614 int amount = sk_wmem_alloc_get(sk);
1615
6516c655
SH		 1616 return put_user(amount, (int __user *)arg);
1617 }
1da177e4 1618
6516c655
SH		 1619 case SIOCINQ:
1620 {
e83c6744 1621 int amount = max_t(int, 0, first_packet_length(sk));
6516c655 1622
6516c655
SH		 1623 return put_user(amount, (int __user *)arg);
1624 }
1da177e4 1625
6516c655
SH		 1626 default:
1627 return -ENOIOCTLCMD;
1da177e4 1628 }
6516c655
SH		 1629
1630 return 0;
1da177e4 1631}
c482c568 1632EXPORT_SYMBOL(udp_ioctl);
1da177e4 1633
2276f58a 1634struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
fd69c399 1635 int noblock, int *off, int *err)
2276f58a
PA		 1636{
1637 struct sk_buff_head *sk_queue = &sk->sk_receive_queue;
1638 struct sk_buff_head *queue;
1639 struct sk_buff *last;
1640 long timeo;
1641 int error;
1642
1643 queue = &udp_sk(sk)->reader_queue;
1644 flags |= noblock ? MSG_DONTWAIT : 0;
1645 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1646 do {
1647 struct sk_buff *skb;
1648
1649 error = sock_error(sk);
1650 if (error)
1651 break;
1652
1653 error = -EAGAIN;
2276f58a 1654 do {
2276f58a
PA		 1655 spin_lock_bh(&queue->lock);
1656 skb = __skb_try_recv_from_queue(sk, queue, flags,
1657 udp_skb_destructor,
fd69c399 1658 off, err, &last);
2276f58a
PA		 1659 if (skb) {
1660 spin_unlock_bh(&queue->lock);
2276f58a
PA		 1661 return skb;
1662 }
1663
137a0dbe 1664 if (skb_queue_empty_lockless(sk_queue)) {
2276f58a
PA		 1665 spin_unlock_bh(&queue->lock);
1666 goto busy_check;
1667 }
1668
6dfb4367
PA		 1669 /* refill the reader queue and walk it again
1670 * keep both queues locked to avoid re-acquiring
1671 * the sk_receive_queue lock if fwd memory scheduling
1672 * is needed.
1673 */
2276f58a
PA		 1674 spin_lock(&sk_queue->lock);
1675 skb_queue_splice_tail_init(sk_queue, queue);
2276f58a
PA		 1676
1677 skb = __skb_try_recv_from_queue(sk, queue, flags,
6dfb4367 1678 udp_skb_dtor_locked,
fd69c399 1679 off, err, &last);
6dfb4367 1680 spin_unlock(&sk_queue->lock);
2276f58a 1681 spin_unlock_bh(&queue->lock);
de321ed3 1682 if (skb)
2276f58a 1683 return skb;
2276f58a
PA		 1684
1685busy_check:
1686 if (!sk_can_busy_loop(sk))
1687 break;
1688
1689 sk_busy_loop(sk, flags & MSG_DONTWAIT);
137a0dbe 1690 } while (!skb_queue_empty_lockless(sk_queue));
2276f58a
PA		 1691
1692 /* sk_queue is empty, reader_queue may contain peeked packets */
1693 } while (timeo &&
1694 !__skb_wait_for_more_packets(sk, &error, &timeo,
1695 (struct sk_buff *)sk_queue));
1696
1697 *err = error;
1698 return NULL;
1699}
7e823644 1700EXPORT_SYMBOL(__skb_recv_udp);
2276f58a 1701
db8dac20
DM		 1702/*
1703 * This should be easy, if there is something there we
1704 * return it, otherwise we block.
1705 */
1706
1b784140
YX		 1707int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock,
1708 int flags, int *addr_len)
db8dac20
DM		 1709{
1710 struct inet_sock *inet = inet_sk(sk);
342dfc30 1711 DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
db8dac20 1712 struct sk_buff *skb;
59c2cdae 1713 unsigned int ulen, copied;
fd69c399 1714 int off, err, peeking = flags & MSG_PEEK;
db8dac20 1715 int is_udplite = IS_UDPLITE(sk);
197c949e 1716 bool checksum_valid = false;
db8dac20 1717
db8dac20 1718 if (flags & MSG_ERRQUEUE)
85fbaa75 1719 return ip_recv_error(sk, msg, len, addr_len);
db8dac20
DM		 1720
1721try_again:
a0917e0b 1722 off = sk_peek_offset(sk, flags);
fd69c399 1723 skb = __skb_recv_udp(sk, flags, noblock, &off, &err);
db8dac20 1724 if (!skb)
627d2d6b 1725 return err;
db8dac20 1726
b65ac446 1727 ulen = udp_skb_len(skb);
59c2cdae 1728 copied = len;
627d2d6b 1729 if (copied > ulen - off)
1730 copied = ulen - off;
59c2cdae 1731 else if (copied < ulen)
db8dac20
DM		 1732 msg->msg_flags |= MSG_TRUNC;
1733
1734 /*
1735 * If checksum is needed at all, try to do it while copying the
1736 * data. If the data is truncated, or if we only want a partial
1737 * coverage checksum (UDP-Lite), do it before the copy.
1738 */
1739
d21dbdfe
ED		 1740 if (copied < ulen || peeking ||
1741 (is_udplite && UDP_SKB_CB(skb)->partial_cov)) {
b65ac446
PA		 1742 checksum_valid = udp_skb_csum_unnecessary(skb) ||
1743 !__udp_lib_checksum_complete(skb);
197c949e 1744 if (!checksum_valid)
db8dac20
DM		 1745 goto csum_copy_err;
1746 }
1747
b65ac446
PA		 1748 if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
1749 if (udp_skb_is_linear(skb))
1750 err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
1751 else
1752 err = skb_copy_datagram_msg(skb, off, msg, copied);
1753 } else {
627d2d6b 1754 err = skb_copy_and_csum_datagram_msg(skb, off, msg);
db8dac20
DM		 1755
1756 if (err == -EINVAL)
1757 goto csum_copy_err;
1758 }
1759
22911fc5 1760 if (unlikely(err)) {
fd69c399 1761 if (!peeking) {
979402b1 1762 atomic_inc(&sk->sk_drops);
6aef70a8
ED		 1763 UDP_INC_STATS(sock_net(sk),
1764 UDP_MIB_INERRORS, is_udplite);
979402b1 1765 }
850cbadd 1766 kfree_skb(skb);
627d2d6b 1767 return err;
22911fc5 1768 }
db8dac20 1769
fd69c399 1770 if (!peeking)
6aef70a8
ED		 1771 UDP_INC_STATS(sock_net(sk),
1772 UDP_MIB_INDATAGRAMS, is_udplite);
db8dac20 1773
3b885787 1774 sock_recv_ts_and_drops(msg, sk, skb);
db8dac20
DM		 1775
1776 /* Copy the address. */
c482c568 1777 if (sin) {
db8dac20
DM		 1778 sin->sin_family = AF_INET;
1779 sin->sin_port = udp_hdr(skb)->source;
1780 sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
1781 memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
bceaa902 1782 *addr_len = sizeof(*sin);
983695fa
DB		 1783
1784 if (cgroup_bpf_enabled)
1785 BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
1786 (struct sockaddr *)sin);
db8dac20 1787 }
bcd1665e
PA		 1788
1789 if (udp_sk(sk)->gro_enabled)
1790 udp_cmsg_recv(msg, sk, skb);
1791
db8dac20 1792 if (inet->cmsg_flags)
ad959036 1793 ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
db8dac20 1794
59c2cdae 1795 err = copied;
db8dac20
DM		 1796 if (flags & MSG_TRUNC)
1797 err = ulen;
1798
850cbadd 1799 skb_consume_udp(sk, skb, peeking ? -err : err);
db8dac20
DM		 1800 return err;
1801
1802csum_copy_err:
2276f58a
PA		 1803 if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
1804 udp_skb_destructor)) {
6aef70a8
ED		 1805 UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
1806 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
6a5dc9e5 1807 }
850cbadd 1808 kfree_skb(skb);
db8dac20 1809
beb39db5
ED		 1810 /* starting over for a new packet, but check if we need to yield */
1811 cond_resched();
9cfaa8de 1812 msg->msg_flags &= ~MSG_TRUNC;
db8dac20
DM		 1813 goto try_again;
1814}
1815
d74bad4e
AI		 1816int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
1817{
1818 /* This check is replicated from __ip4_datagram_connect() and
1819 * intended to prevent BPF program called below from accessing bytes
1820 * that are out of the bound specified by user in addr_len.
1821 */
1822 if (addr_len < sizeof(struct sockaddr_in))
1823 return -EINVAL;
1824
1825 return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr);
1826}
1827EXPORT_SYMBOL(udp_pre_connect);
1828
286c72de 1829int __udp_disconnect(struct sock *sk, int flags)
1da177e4
LT		 1830{
1831 struct inet_sock *inet = inet_sk(sk);
1832 /*
1833 * 1003.1g - break association.
1834 */
e905a9ed 1835
1da177e4 1836 sk->sk_state = TCP_CLOSE;
c720c7e8
ED		 1837 inet->inet_daddr = 0;
1838 inet->inet_dport = 0;
bdeab991 1839 sock_rps_reset_rxhash(sk);
1da177e4
LT		 1840 sk->sk_bound_dev_if = 0;
1841 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1842 inet_reset_saddr(sk);
1843
1844 if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) {
1845 sk->sk_prot->unhash(sk);
c720c7e8 1846 inet->inet_sport = 0;
1da177e4
LT		 1847 }
1848 sk_dst_reset(sk);
1849 return 0;
1850}
286c72de
ED		 1851EXPORT_SYMBOL(__udp_disconnect);
1852
1853int udp_disconnect(struct sock *sk, int flags)
1854{
1855 lock_sock(sk);
1856 __udp_disconnect(sk, flags);
1857 release_sock(sk);
1858 return 0;
1859}
c482c568 1860EXPORT_SYMBOL(udp_disconnect);
1da177e4 1861
645ca708
ED		 1862void udp_lib_unhash(struct sock *sk)
1863{
723b4610
ED		 1864 if (sk_hashed(sk)) {
1865 struct udp_table *udptable = sk->sk_prot->h.udp_table;
512615b6
ED		 1866 struct udp_hslot *hslot, *hslot2;
1867
1868 hslot = udp_hashslot(udptable, sock_net(sk),
1869 udp_sk(sk)->udp_port_hash);
1870 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
645ca708 1871
723b4610 1872 spin_lock_bh(&hslot->lock);
e32ea7e7
CG		 1873 if (rcu_access_pointer(sk->sk_reuseport_cb))
1874 reuseport_detach_sock(sk);
ca065d0c 1875 if (sk_del_node_init_rcu(sk)) {
fdcc8aa9 1876 hslot->count--;
c720c7e8 1877 inet_sk(sk)->inet_num = 0;
723b4610 1878 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
512615b6
ED		 1879
1880 spin_lock(&hslot2->lock);
ca065d0c 1881 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
512615b6
ED		 1882 hslot2->count--;
1883 spin_unlock(&hslot2->lock);
723b4610
ED		 1884 }
1885 spin_unlock_bh(&hslot->lock);
645ca708 1886 }
645ca708
ED		 1887}
1888EXPORT_SYMBOL(udp_lib_unhash);
1889
719f8358
ED		 1890/*
1891 * inet_rcv_saddr was changed, we must rehash secondary hash
1892 */
1893void udp_lib_rehash(struct sock *sk, u16 newhash)
1894{
1895 if (sk_hashed(sk)) {
1896 struct udp_table *udptable = sk->sk_prot->h.udp_table;
1897 struct udp_hslot *hslot, *hslot2, *nhslot2;
1898
1899 hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash);
1900 nhslot2 = udp_hashslot2(udptable, newhash);
1901 udp_sk(sk)->udp_portaddr_hash = newhash;
e32ea7e7
CG		 1902
1903 if (hslot2 != nhslot2 ||
1904 rcu_access_pointer(sk->sk_reuseport_cb)) {
719f8358
ED		 1905 hslot = udp_hashslot(udptable, sock_net(sk),
1906 udp_sk(sk)->udp_port_hash);
1907 /* we must lock primary chain too */
1908 spin_lock_bh(&hslot->lock);
e32ea7e7
CG		 1909 if (rcu_access_pointer(sk->sk_reuseport_cb))
1910 reuseport_detach_sock(sk);
1911
1912 if (hslot2 != nhslot2) {
1913 spin_lock(&hslot2->lock);
ca065d0c 1914 hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node);
e32ea7e7
CG		 1915 hslot2->count--;
1916 spin_unlock(&hslot2->lock);
1917
1918 spin_lock(&nhslot2->lock);
ca065d0c 1919 hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node,
e32ea7e7
CG		 1920 &nhslot2->head);
1921 nhslot2->count++;
1922 spin_unlock(&nhslot2->lock);
1923 }
719f8358
ED		 1924
1925 spin_unlock_bh(&hslot->lock);
1926 }
1927 }
1928}
1929EXPORT_SYMBOL(udp_lib_rehash);
1930
8f6b5392 1931void udp_v4_rehash(struct sock *sk)
719f8358 1932{
f0b1e64c 1933 u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
719f8358
ED		 1934 inet_sk(sk)->inet_rcv_saddr,
1935 inet_sk(sk)->inet_num);
1936 udp_lib_rehash(sk, new_hash);
1937}
1938
a3f96c47 1939static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
93821778 1940{
fec5e652 1941 int rc;
766e9037 1942
005ec974 1943 if (inet_sk(sk)->inet_daddr) {
bdeab991 1944 sock_rps_save_rxhash(sk, skb);
005ec974 1945 sk_mark_napi_id(sk, skb);
2c8c56e1 1946 sk_incoming_cpu_update(sk);
e68b6e50
ED		 1947 } else {
1948 sk_mark_napi_id_once(sk, skb);
005ec974 1949 }
fec5e652 1950
850cbadd 1951 rc = __udp_enqueue_schedule_skb(sk, skb);
766e9037
ED		 1952 if (rc < 0) {
1953 int is_udplite = IS_UDPLITE(sk);
93821778 1954
93821778 1955 /* Note that an ENOMEM error is charged twice */
766e9037 1956 if (rc == -ENOMEM)
e61da9e2 1957 UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS,
02c22347 1958 is_udplite);
e61da9e2 1959 UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
766e9037 1960 kfree_skb(skb);
296f7ea7 1961 trace_udp_fail_queue_rcv_skb(rc, sk);
766e9037 1962 return -1;
93821778
HX		 1963 }
1964
1965 return 0;
93821778
HX		 1966}
1967
db8dac20
DM		 1968/* returns:
1969 * -1: error
1970 * 0: success
1971 * >0: "udp encap" protocol resubmission
1972 *
1973 * Note that in the success and error cases, the skb is assumed to
1974 * have either been requeued or freed.
1975 */
cf329aa4 1976static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
db8dac20
DM		 1977{
1978 struct udp_sock *up = udp_sk(sk);
db8dac20
DM		 1979 int is_udplite = IS_UDPLITE(sk);
1980
1981 /*
1982 * Charge it to the socket, dropping if the queue is full.
1983 */
1984 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1985 goto drop;
895b5c9f 1986 nf_reset_ct(skb);
db8dac20 1987
88ab3108 1988 if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
0ad92ad0
ED		 1989 int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
1990
db8dac20
DM		 1991 /*
1992 * This is an encapsulation socket so pass the skb to
1993 * the socket's udp_encap_rcv() hook. Otherwise, just
1994 * fall through and pass this up the UDP socket.
1995 * up->encap_rcv() returns the following value:
1996 * =0 if skb was successfully passed to the encap
1997 * handler or was discarded by it.
1998 * >0 if skb should be passed on to UDP.
1999 * <0 if skb should be resubmitted as proto -N
2000 */
2001
2002 /* if we're overly short, let UDP handle it */
6aa7de05 2003 encap_rcv = READ_ONCE(up->encap_rcv);
e5aed006 2004 if (encap_rcv) {
db8dac20
DM		 2005 int ret;
2006
0a80966b
TH		 2007 /* Verify checksum before giving to encap */
2008 if (udp_lib_checksum_complete(skb))
2009 goto csum_error;
2010
0ad92ad0 2011 ret = encap_rcv(sk, skb);
db8dac20 2012 if (ret <= 0) {
02c22347
ED		 2013 __UDP_INC_STATS(sock_net(sk),
2014 UDP_MIB_INDATAGRAMS,
2015 is_udplite);
db8dac20
DM		 2016 return -ret;
2017 }
2018 }
2019
2020 /* FALLTHROUGH -- it's a UDP Packet */
2021 }
2022
2023 /*
2024 * UDP-Lite specific tests, ignored on UDP sockets
2025 */
2026 if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) {
2027
2028 /*
2029 * MIB statistics other than incrementing the error count are
2030 * disabled for the following two types of errors: these depend
2031 * on the application settings, not on the functioning of the
2032 * protocol stack as such.
2033 *
2034 * RFC 3828 here recommends (sec 3.3): "There should also be a
2035 * way ... to ... at least let the receiving application block
2036 * delivery of packets with coverage values less than a value
2037 * provided by the application."
2038 */
2039 if (up->pcrlen == 0) { /* full coverage was set */
ba7a46f1
JP		 2040 net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested\n",
2041 UDP_SKB_CB(skb)->cscov, skb->len);
db8dac20
DM		 2042 goto drop;
2043 }
2044 /* The next case involves violating the min. coverage requested
2045 * by the receiver. This is subtle: if receiver wants x and x is
2046 * greater than the buffersize/MTU then receiver will complain
2047 * that it wants x while sender emits packets of smaller size y.
2048 * Therefore the above ...()->partial_cov statement is essential.
2049 */
2050 if (UDP_SKB_CB(skb)->cscov < up->pcrlen) {
ba7a46f1
JP		 2051 net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d\n",
2052 UDP_SKB_CB(skb)->cscov, up->pcrlen);
db8dac20
DM		 2053 goto drop;
2054 }
2055 }
2056
dd99e425 2057 prefetch(&sk->sk_rmem_alloc);
ce25d66a
ED		 2058 if (rcu_access_pointer(sk->sk_filter) &&
2059 udp_lib_checksum_complete(skb))
e6afc8ac 2060 goto csum_error;
ce25d66a 2061
ba66bbe5 2062 if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr)))
a6127697 2063 goto drop;
db8dac20 2064
e6afc8ac 2065 udp_csum_pull_header(skb);
db8dac20 2066
fbf8866d 2067 ipv4_pktinfo_prepare(sk, skb);
850cbadd 2068 return __udp_queue_rcv_skb(sk, skb);
db8dac20 2069
6a5dc9e5 2070csum_error:
02c22347 2071 __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
db8dac20 2072drop:
02c22347 2073 __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
8edf19c2 2074 atomic_inc(&sk->sk_drops);
db8dac20
DM		 2075 kfree_skb(skb);
2076 return -1;
2077}
2078
cf329aa4
PA		 2079static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2080{
2081 struct sk_buff *next, *segs;
2082 int ret;
2083
2084 if (likely(!udp_unexpected_gso(sk, skb)))
2085 return udp_queue_rcv_one_skb(sk, skb);
2086
2087 BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_SGO_CB_OFFSET);
2088 __skb_push(skb, -skb_mac_offset(skb));
2089 segs = udp_rcv_segment(sk, skb, true);
2090 for (skb = segs; skb; skb = next) {
2091 next = skb->next;
2092 __skb_pull(skb, skb_transport_offset(skb));
2093 ret = udp_queue_rcv_one_skb(sk, skb);
2094 if (ret > 0)
2095 ip_protocol_deliver_rcu(dev_net(skb->dev), skb, -ret);
2096 }
2097 return 0;
2098}
2099
97502231 2100/* For TCP sockets, sk_rx_dst is protected by socket lock
e47eb5df 2101 * For UDP, we use xchg() to guard against concurrent changes.
97502231 2102 */
64f0f5d1 2103bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
421b3885 2104{
97502231
ED		 2105 struct dst_entry *old;
2106
d24406c8
WW		 2107 if (dst_hold_safe(dst)) {
2108 old = xchg(&sk->sk_rx_dst, dst);
2109 dst_release(old);
64f0f5d1 2110 return old != dst;
d24406c8 2111 }
64f0f5d1 2112 return false;
421b3885 2113}
c9f2c1ae 2114EXPORT_SYMBOL(udp_sk_rx_dst_set);
421b3885 2115
db8dac20
DM		 2116/*
2117 * Multicasts and broadcasts go to each listener.
2118 *
1240d137 2119 * Note: called only from the BH handler context.
db8dac20 2120 */
e3163493 2121static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
db8dac20
DM		 2122 struct udphdr *uh,
2123 __be32 saddr, __be32 daddr,
36cbb245
RJ		 2124 struct udp_table *udptable,
2125 int proto)
db8dac20 2126{
ca065d0c 2127 struct sock *sk, *first = NULL;
5cf3d461
DH		 2128 unsigned short hnum = ntohs(uh->dest);
2129 struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum);
2dc41cff 2130 unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10);
ca065d0c
ED		 2131 unsigned int offset = offsetof(typeof(*sk), sk_node);
2132 int dif = skb->dev->ifindex;
fb74c277 2133 int sdif = inet_sdif(skb);
ca065d0c
ED		 2134 struct hlist_node *node;
2135 struct sk_buff *nskb;
2dc41cff
DH		 2136
2137 if (use_hash2) {
f0b1e64c 2138 hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
73e2d5e3 2139 udptable->mask;
f0b1e64c 2140 hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
2dc41cff 2141start_lookup:
73e2d5e3 2142 hslot = &udptable->hash2[hash2];
2dc41cff
DH		 2143 offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
2144 }
db8dac20 2145
ca065d0c
ED		 2146 sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
2147 if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
fb74c277 2148 uh->source, saddr, dif, sdif, hnum))
ca065d0c
ED		 2149 continue;
2150
2151 if (!first) {
2152 first = sk;
2153 continue;
1240d137 2154 }
ca065d0c 2155 nskb = skb_clone(skb, GFP_ATOMIC);
1240d137 2156
ca065d0c
ED		 2157 if (unlikely(!nskb)) {
2158 atomic_inc(&sk->sk_drops);
02c22347
ED		 2159 __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS,
2160 IS_UDPLITE(sk));
2161 __UDP_INC_STATS(net, UDP_MIB_INERRORS,
2162 IS_UDPLITE(sk));
ca065d0c
ED		 2163 continue;
2164 }
2165 if (udp_queue_rcv_skb(sk, nskb) > 0)
2166 consume_skb(nskb);
2167 }
1240d137 2168
2dc41cff
DH		 2169 /* Also lookup *:port if we are using hash2 and haven't done so yet. */
2170 if (use_hash2 && hash2 != hash2_any) {
2171 hash2 = hash2_any;
2172 goto start_lookup;
2173 }
2174
ca065d0c
ED		 2175 if (first) {
2176 if (udp_queue_rcv_skb(first, skb) > 0)
2177 consume_skb(skb);
1240d137 2178 } else {
ca065d0c 2179 kfree_skb(skb);
02c22347
ED		 2180 __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI,
2181 proto == IPPROTO_UDPLITE);
1240d137 2182 }
db8dac20
DM		 2183 return 0;
2184}
2185
2186/* Initialize UDP checksum. If exited with zero value (success),
2187 * CHECKSUM_UNNECESSARY means, that no more checks are required.
666a3d6e 2188 * Otherwise, csum completion requires checksumming packet body,
db8dac20
DM		 2189 * including udp header and folding it to skb->csum.
2190 */
2191static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh,
2192 int proto)
2193{
db8dac20
DM		 2194 int err;
2195
2196 UDP_SKB_CB(skb)->partial_cov = 0;
2197 UDP_SKB_CB(skb)->cscov = skb->len;
2198
2199 if (proto == IPPROTO_UDPLITE) {
2200 err = udplite_checksum_init(skb, uh);
2201 if (err)
2202 return err;
15f35d49
AK		 2203
2204 if (UDP_SKB_CB(skb)->partial_cov) {
2205 skb->csum = inet_compute_pseudo(skb, proto);
2206 return 0;
2207 }
db8dac20
DM		 2208 }
2209
b46d9f62
HFS		 2210 /* Note, we are only interested in != 0 or == 0, thus the
2211 * force to int.
2212 */
db4f1be3
ST		 2213 err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check,
2214 inet_compute_pseudo);
2215 if (err)
2216 return err;
2217
2218 if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
2219 /* If SW calculated the value, we know it's bad */
2220 if (skb->csum_complete_sw)
2221 return 1;
2222
2223 /* HW says the value is bad. Let's validate that.
2224 * skb->csum is no longer the full packet checksum,
2225 * so don't treat it as such.
2226 */
2227 skb_checksum_complete_unset(skb);
2228 }
2229
2230 return 0;
db8dac20
DM		 2231}
2232
2b5a9217
PA		 2233/* wrapper for udp_queue_rcv_skb tacking care of csum conversion and
2234 * return code conversion for ip layer consumption
2235 */
2236static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
2237 struct udphdr *uh)
2238{
2239 int ret;
2240
2241 if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk))
e4aa33ad 2242 skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
2b5a9217
PA		 2243
2244 ret = udp_queue_rcv_skb(sk, skb);
2245
2246 /* a return value > 0 means to resubmit the input, but
2247 * it wants the return to be -protocol, or 0
2248 */
2249 if (ret > 0)
2250 return -ret;
2251 return 0;
2252}
2253
db8dac20
DM		 2254/*
2255 * All we need to do is get the socket, and then do a checksum.
2256 */
2257
645ca708 2258int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable,
db8dac20
DM		 2259 int proto)
2260{
2261 struct sock *sk;
7b5e56f9 2262 struct udphdr *uh;
db8dac20 2263 unsigned short ulen;
adf30907 2264 struct rtable *rt = skb_rtable(skb);
2783ef23 2265 __be32 saddr, daddr;
0283328e 2266 struct net *net = dev_net(skb->dev);
db8dac20
DM		 2267
2268 /*
2269 * Validate the packet.
2270 */
2271 if (!pskb_may_pull(skb, sizeof(struct udphdr)))
2272 goto drop; /* No space for header. */
2273
7b5e56f9 2274 uh = udp_hdr(skb);
db8dac20 2275 ulen = ntohs(uh->len);
ccc2d97c
BM		 2276 saddr = ip_hdr(skb)->saddr;
2277 daddr = ip_hdr(skb)->daddr;
2278
db8dac20
DM		 2279 if (ulen > skb->len)
2280 goto short_packet;
2281
2282 if (proto == IPPROTO_UDP) {
2283 /* UDP validates ulen. */
2284 if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen))
2285 goto short_packet;
2286 uh = udp_hdr(skb);
2287 }
2288
2289 if (udp4_csum_init(skb, uh, proto))
2290 goto csum_error;
2291
8afdd99a
ED		 2292 sk = skb_steal_sock(skb);
2293 if (sk) {
97502231 2294 struct dst_entry *dst = skb_dst(skb);
421b3885 2295 int ret;
421b3885 2296
97502231
ED		 2297 if (unlikely(sk->sk_rx_dst != dst))
2298 udp_sk_rx_dst_set(sk, dst);
db8dac20 2299
2b5a9217 2300 ret = udp_unicast_rcv_skb(sk, skb, uh);
8afdd99a 2301 sock_put(sk);
2b5a9217 2302 return ret;
421b3885 2303 }
db8dac20 2304
c18450a5
FF		 2305 if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
2306 return __udp4_lib_mcast_deliver(net, skb, uh,
36cbb245 2307 saddr, daddr, udptable, proto);
c18450a5
FF		 2308
2309 sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable);
2b5a9217
PA		 2310 if (sk)
2311 return udp_unicast_rcv_skb(sk, skb, uh);
db8dac20
DM		 2312
2313 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2314 goto drop;
895b5c9f 2315 nf_reset_ct(skb);
db8dac20
DM		 2316
2317 /* No socket. Drop packet silently, if checksum is wrong */
2318 if (udp_lib_checksum_complete(skb))
2319 goto csum_error;
2320
02c22347 2321 __UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE);
db8dac20
DM		 2322 icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
2323
2324 /*
2325 * Hmm. We got an UDP packet to a port to which we
2326 * don't wanna listen. Ignore it.
2327 */
2328 kfree_skb(skb);
2329 return 0;
2330
2331short_packet:
ba7a46f1
JP		 2332 net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
2333 proto == IPPROTO_UDPLITE ? "Lite" : "",
2334 &saddr, ntohs(uh->source),
2335 ulen, skb->len,
2336 &daddr, ntohs(uh->dest));
db8dac20
DM		 2337 goto drop;
2338
2339csum_error:
2340 /*
2341 * RFC1122: OK. Discards the bad packet silently (as far as
2342 * the network is concerned, anyway) as per 4.1.3.4 (MUST).
2343 */
ba7a46f1
JP		 2344 net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
2345 proto == IPPROTO_UDPLITE ? "Lite" : "",
2346 &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
2347 ulen);
02c22347 2348 __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE);
db8dac20 2349drop:
02c22347 2350 __UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE);
db8dac20
DM		 2351 kfree_skb(skb);
2352 return 0;
2353}
2354
421b3885
SB		 2355/* We can only early demux multicast if there is a single matching socket.
2356 * If more than one socket found returns NULL
2357 */
2358static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
2359 __be16 loc_port, __be32 loc_addr,
2360 __be16 rmt_port, __be32 rmt_addr,
fb74c277 2361 int dif, int sdif)
421b3885
SB		 2362{
2363 struct sock *sk, *result;
421b3885 2364 unsigned short hnum = ntohs(loc_port);
ca065d0c 2365 unsigned int slot = udp_hashfn(net, hnum, udp_table.mask);
421b3885
SB		 2366 struct udp_hslot *hslot = &udp_table.hash[slot];
2367
63c6f81c
ED		 2368 /* Do not bother scanning a too big list */
2369 if (hslot->count > 10)
2370 return NULL;
2371
421b3885 2372 result = NULL;
ca065d0c
ED		 2373 sk_for_each_rcu(sk, &hslot->head) {
2374 if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
fb74c277 2375 rmt_port, rmt_addr, dif, sdif, hnum)) {
ca065d0c
ED		 2376 if (result)
2377 return NULL;
421b3885 2378 result = sk;
421b3885
SB		 2379 }
2380 }
ca065d0c 2381
421b3885
SB		 2382 return result;
2383}
2384
2385/* For unicast we should only early demux connected sockets or we can
2386 * break forwarding setups. The chains here can be long so only check
2387 * if the first socket is an exact match and if not move on.
2388 */
2389static struct sock *__udp4_lib_demux_lookup(struct net *net,
2390 __be16 loc_port, __be32 loc_addr,
2391 __be16 rmt_port, __be32 rmt_addr,
3fa6f616 2392 int dif, int sdif)
421b3885 2393{
421b3885 2394 unsigned short hnum = ntohs(loc_port);
f0b1e64c 2395 unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
421b3885
SB		 2396 unsigned int slot2 = hash2 & udp_table.mask;
2397 struct udp_hslot *hslot2 = &udp_table.hash2[slot2];
c7228317 2398 INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
421b3885 2399 const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum);
ca065d0c 2400 struct sock *sk;
421b3885 2401
ca065d0c
ED		 2402 udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
2403 if (INET_MATCH(sk, net, acookie, rmt_addr,
3fa6f616 2404 loc_addr, ports, dif, sdif))
ca065d0c 2405 return sk;
421b3885
SB		 2406 /* Only check first socket in chain */
2407 break;
2408 }
ca065d0c 2409 return NULL;
421b3885
SB		 2410}
2411
7487449c 2412int udp_v4_early_demux(struct sk_buff *skb)
421b3885 2413{
610438b7 2414 struct net *net = dev_net(skb->dev);
bc044e8d 2415 struct in_device *in_dev = NULL;
610438b7
ED		 2416 const struct iphdr *iph;
2417 const struct udphdr *uh;
ca065d0c 2418 struct sock *sk = NULL;
421b3885 2419 struct dst_entry *dst;
421b3885 2420 int dif = skb->dev->ifindex;
fb74c277 2421 int sdif = inet_sdif(skb);
6e540309 2422 int ours;
421b3885
SB		 2423
2424 /* validate the packet */
2425 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
7487449c 2426 return 0;
421b3885 2427
610438b7
ED		 2428 iph = ip_hdr(skb);
2429 uh = udp_hdr(skb);
2430
996b44fc 2431 if (skb->pkt_type == PACKET_MULTICAST) {
bc044e8d 2432 in_dev = __in_dev_get_rcu(skb->dev);
6e540309
SB		 2433
2434 if (!in_dev)
7487449c 2435 return 0;
6e540309 2436
996b44fc
PA		 2437 ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
2438 iph->protocol);
2439 if (!ours)
2440 return 0;
ad0ea198 2441
421b3885 2442 sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
fb74c277
DA		 2443 uh->source, iph->saddr,
2444 dif, sdif);
6e540309 2445 } else if (skb->pkt_type == PACKET_HOST) {
421b3885 2446 sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
3fa6f616 2447 uh->source, iph->saddr, dif, sdif);
6e540309 2448 }
421b3885 2449
41c6d650 2450 if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt))
7487449c 2451 return 0;
421b3885
SB		 2452
2453 skb->sk = sk;
82eabd9e 2454 skb->destructor = sock_efree;
10e2eb87 2455 dst = READ_ONCE(sk->sk_rx_dst);
421b3885
SB		 2456
2457 if (dst)
2458 dst = dst_check(dst, 0);
10e2eb87 2459 if (dst) {
bc044e8d
PA		 2460 u32 itag = 0;
2461
d24406c8
WW		 2462 /* set noref for now.
2463 * any place which wants to hold dst has to call
2464 * dst_hold_safe()
2465 */
2466 skb_dst_set_noref(skb, dst);
bc044e8d
PA		 2467
2468 /* for unconnected multicast sockets we need to validate
2469 * the source on each packet
2470 */
2471 if (!inet_sk(sk)->inet_daddr && in_dev)
2472 return ip_mc_validate_source(skb, iph->daddr,
2473 iph->saddr, iph->tos,
2474 skb->dev, in_dev, &itag);
10e2eb87 2475 }
7487449c 2476 return 0;
421b3885
SB		 2477}
2478
db8dac20
DM		 2479int udp_rcv(struct sk_buff *skb)
2480{
645ca708 2481 return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP);
db8dac20
DM		 2482}
2483
7d06b2e0 2484void udp_destroy_sock(struct sock *sk)
db8dac20 2485{
44046a59 2486 struct udp_sock *up = udp_sk(sk);
8a74ad60 2487 bool slow = lock_sock_fast(sk);
db8dac20 2488 udp_flush_pending_frames(sk);
8a74ad60 2489 unlock_sock_fast(sk, slow);
60fb9567
PA		 2490 if (static_branch_unlikely(&udp_encap_needed_key)) {
2491 if (up->encap_type) {
2492 void (*encap_destroy)(struct sock *sk);
2493 encap_destroy = READ_ONCE(up->encap_destroy);
2494 if (encap_destroy)
2495 encap_destroy(sk);
2496 }
2497 if (up->encap_enabled)
9c480601 2498 static_branch_dec(&udp_encap_needed_key);
44046a59 2499 }
db8dac20
DM		 2500}
2501
1da177e4
LT		 2502/*
2503 * Socket option code for UDP
2504 */
4c0a6cb0 2505int udp_lib_setsockopt(struct sock *sk, int level, int optname,
b7058842 2506 char __user *optval, unsigned int optlen,
4c0a6cb0 2507 int (*push_pending_frames)(struct sock *))
1da177e4
LT		 2508{
2509 struct udp_sock *up = udp_sk(sk);
1c19448c 2510 int val, valbool;
1da177e4 2511 int err = 0;
b2bf1e26 2512 int is_udplite = IS_UDPLITE(sk);
1da177e4 2513
c482c568 2514 if (optlen < sizeof(int))
1da177e4
LT		 2515 return -EINVAL;
2516
2517 if (get_user(val, (int __user *)optval))
2518 return -EFAULT;
2519
1c19448c
TH		 2520 valbool = val ? 1 : 0;
2521
6516c655 2522 switch (optname) {
1da177e4
LT		 2523 case UDP_CORK:
2524 if (val != 0) {
2525 up->corkflag = 1;
2526 } else {
2527 up->corkflag = 0;
2528 lock_sock(sk);
4243cdc2 2529 push_pending_frames(sk);
1da177e4
LT		 2530 release_sock(sk);
2531 }
2532 break;
e905a9ed 2533
1da177e4
LT		 2534 case UDP_ENCAP:
2535 switch (val) {
2536 case 0:
2537 case UDP_ENCAP_ESPINUDP:
2538 case UDP_ENCAP_ESPINUDP_NON_IKE:
067b207b
JC		 2539 up->encap_rcv = xfrm4_udp_encap_rcv;
2540 /* FALLTHROUGH */
342f0234 2541 case UDP_ENCAP_L2TPINUDP:
1da177e4 2542 up->encap_type = val;
60fb9567
PA		 2543 lock_sock(sk);
2544 udp_tunnel_encap_enable(sk->sk_socket);
2545 release_sock(sk);
1da177e4
LT		 2546 break;
2547 default:
2548 err = -ENOPROTOOPT;
2549 break;
2550 }
2551 break;
2552
1c19448c
TH		 2553 case UDP_NO_CHECK6_TX:
2554 up->no_check6_tx = valbool;
2555 break;
2556
2557 case UDP_NO_CHECK6_RX:
2558 up->no_check6_rx = valbool;
2559 break;
2560
bec1f6f6
WB		 2561 case UDP_SEGMENT:
2562 if (val < 0 || val > USHRT_MAX)
2563 return -EINVAL;
2564 up->gso_size = val;
2565 break;
2566
e20cf8d3
PA		 2567 case UDP_GRO:
2568 lock_sock(sk);
2569 if (valbool)
2570 udp_tunnel_encap_enable(sk->sk_socket);
2571 up->gro_enabled = valbool;
2572 release_sock(sk);
2573 break;
2574
ba4e58ec
GR		 2575 /*
2576 * UDP-Lite's partial checksum coverage (RFC 3828).
2577 */
2578 /* The sender sets actual checksum coverage length via this option.
2579 * The case coverage > packet length is handled by send module. */
2580 case UDPLITE_SEND_CSCOV:
b2bf1e26 2581 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR		 2582 return -ENOPROTOOPT;
2583 if (val != 0 && val < 8) /* Illegal coverage: use default (8) */
2584 val = 8;
4be929be
AD		 2585 else if (val > USHRT_MAX)
2586 val = USHRT_MAX;
ba4e58ec
GR		 2587 up->pcslen = val;
2588 up->pcflag |= UDPLITE_SEND_CC;
2589 break;
2590
e905a9ed
YH		 2591 /* The receiver specifies a minimum checksum coverage value. To make
2592 * sense, this should be set to at least 8 (as done below). If zero is
ba4e58ec
GR		 2593 * used, this again means full checksum coverage. */
2594 case UDPLITE_RECV_CSCOV:
b2bf1e26 2595 if (!is_udplite) /* Disable the option on UDP sockets */
ba4e58ec
GR		 2596 return -ENOPROTOOPT;
2597 if (val != 0 && val < 8) /* Avoid silly minimal values. */
2598 val = 8;
4be929be
AD		 2599 else if (val > USHRT_MAX)
2600 val = USHRT_MAX;
ba4e58ec
GR		 2601 up->pcrlen = val;
2602 up->pcflag |= UDPLITE_RECV_CC;
2603 break;
2604
1da177e4
LT		 2605 default:
2606 err = -ENOPROTOOPT;
2607 break;
6516c655 2608 }
1da177e4
LT		 2609
2610 return err;
2611}
c482c568 2612EXPORT_SYMBOL(udp_lib_setsockopt);
1da177e4 2613
db8dac20 2614int udp_setsockopt(struct sock *sk, int level, int optname,
b7058842 2615 char __user *optval, unsigned int optlen)
db8dac20
DM		 2616{
2617 if (level == SOL_UDP || level == SOL_UDPLITE)
2618 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2619 udp_push_pending_frames);
2620 return ip_setsockopt(sk, level, optname, optval, optlen);
2621}
2622
2623#ifdef CONFIG_COMPAT
2624int compat_udp_setsockopt(struct sock *sk, int level, int optname,
b7058842 2625 char __user *optval, unsigned int optlen)
db8dac20
DM		 2626{
2627 if (level == SOL_UDP || level == SOL_UDPLITE)
2628 return udp_lib_setsockopt(sk, level, optname, optval, optlen,
2629 udp_push_pending_frames);
2630 return compat_ip_setsockopt(sk, level, optname, optval, optlen);
2631}
2632#endif
2633
4c0a6cb0
GR		 2634int udp_lib_getsockopt(struct sock *sk, int level, int optname,
2635 char __user *optval, int __user *optlen)
1da177e4
LT		 2636{
2637 struct udp_sock *up = udp_sk(sk);
2638 int val, len;
2639
c482c568 2640 if (get_user(len, optlen))
1da177e4
LT		 2641 return -EFAULT;
2642
2643 len = min_t(unsigned int, len, sizeof(int));
e905a9ed 2644
6516c655 2645 if (len < 0)
1da177e4
LT		 2646 return -EINVAL;
2647
6516c655 2648 switch (optname) {
1da177e4
LT		 2649 case UDP_CORK:
2650 val = up->corkflag;
2651 break;
2652
2653 case UDP_ENCAP:
2654 val = up->encap_type;
2655 break;
2656
1c19448c
TH		 2657 case UDP_NO_CHECK6_TX:
2658 val = up->no_check6_tx;
2659 break;
2660
2661 case UDP_NO_CHECK6_RX:
2662 val = up->no_check6_rx;
2663 break;
2664
bec1f6f6
WB		 2665 case UDP_SEGMENT:
2666 val = up->gso_size;
2667 break;
2668
ba4e58ec
GR		 2669 /* The following two cannot be changed on UDP sockets, the return is
2670 * always 0 (which corresponds to the full checksum coverage of UDP). */
2671 case UDPLITE_SEND_CSCOV:
2672 val = up->pcslen;
2673 break;
2674
2675 case UDPLITE_RECV_CSCOV:
2676 val = up->pcrlen;
2677 break;
2678
1da177e4
LT		 2679 default:
2680 return -ENOPROTOOPT;
6516c655 2681 }
1da177e4 2682
6516c655 2683 if (put_user(len, optlen))
e905a9ed 2684 return -EFAULT;
c482c568 2685 if (copy_to_user(optval, &val, len))
1da177e4 2686 return -EFAULT;
e905a9ed 2687 return 0;
1da177e4 2688}
c482c568 2689EXPORT_SYMBOL(udp_lib_getsockopt);
1da177e4 2690
db8dac20
DM		 2691int udp_getsockopt(struct sock *sk, int level, int optname,
2692 char __user *optval, int __user *optlen)
2693{
2694 if (level == SOL_UDP || level == SOL_UDPLITE)
2695 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2696 return ip_getsockopt(sk, level, optname, optval, optlen);
2697}
2698
2699#ifdef CONFIG_COMPAT
2700int compat_udp_getsockopt(struct sock *sk, int level, int optname,
2701 char __user *optval, int __user *optlen)
2702{
2703 if (level == SOL_UDP || level == SOL_UDPLITE)
2704 return udp_lib_getsockopt(sk, level, optname, optval, optlen);
2705 return compat_ip_getsockopt(sk, level, optname, optval, optlen);
2706}
2707#endif
1da177e4
LT		 2708/**
2709 * udp_poll - wait for a UDP event.
2710 * @file - file struct
2711 * @sock - socket
a11e1d43 2712 * @wait - poll table
1da177e4 2713 *
e905a9ed 2714 * This is same as datagram poll, except for the special case of
1da177e4
LT		 2715 * blocking sockets. If application is using a blocking fd
2716 * and a packet with checksum error is in the queue;
2717 * then it could get return from select indicating data available
2718 * but then block when reading it. Add special case code
2719 * to work around these arguably broken applications.
2720 */
a11e1d43 2721__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait)
1da177e4 2722{
a11e1d43 2723 __poll_t mask = datagram_poll(file, sock, wait);
1da177e4 2724 struct sock *sk = sock->sk;
ba4e58ec 2725
3ef7cf57 2726 if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue))
a9a08845 2727 mask |= EPOLLIN | EPOLLRDNORM;
2276f58a 2728
1da177e4 2729 /* Check for false positives due to checksum errors */
a11e1d43 2730 if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) &&
e83c6744 2731 !(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1)
a9a08845 2732 mask &= ~(EPOLLIN | EPOLLRDNORM);
1da177e4
LT		 2733
2734 return mask;
e905a9ed 2735
1da177e4 2736}
a11e1d43 2737EXPORT_SYMBOL(udp_poll);
1da177e4 2738
5d77dca8
DA		 2739int udp_abort(struct sock *sk, int err)
2740{
2741 lock_sock(sk);
2742
2743 sk->sk_err = err;
2744 sk->sk_error_report(sk);
286c72de 2745 __udp_disconnect(sk, 0);
5d77dca8
DA		 2746
2747 release_sock(sk);
2748
2749 return 0;
2750}
2751EXPORT_SYMBOL_GPL(udp_abort);
2752
db8dac20 2753struct proto udp_prot = {
1e802951
TZ		 2754 .name = "UDP",
2755 .owner = THIS_MODULE,
2756 .close = udp_lib_close,
d74bad4e 2757 .pre_connect = udp_pre_connect,
1e802951
TZ		 2758 .connect = ip4_datagram_connect,
2759 .disconnect = udp_disconnect,
2760 .ioctl = udp_ioctl,
2761 .init = udp_init_sock,
2762 .destroy = udp_destroy_sock,
2763 .setsockopt = udp_setsockopt,
2764 .getsockopt = udp_getsockopt,
2765 .sendmsg = udp_sendmsg,
2766 .recvmsg = udp_recvmsg,
2767 .sendpage = udp_sendpage,
2768 .release_cb = ip4_datagram_release_cb,
2769 .hash = udp_lib_hash,
2770 .unhash = udp_lib_unhash,
2771 .rehash = udp_v4_rehash,
2772 .get_port = udp_v4_get_port,
2773 .memory_allocated = &udp_memory_allocated,
2774 .sysctl_mem = sysctl_udp_mem,
2775 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
2776 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
2777 .obj_size = sizeof(struct udp_sock),
2778 .h.udp_table = &udp_table,
db8dac20 2779#ifdef CONFIG_COMPAT
1e802951
TZ		 2780 .compat_setsockopt = compat_udp_setsockopt,
2781 .compat_getsockopt = compat_udp_getsockopt,
db8dac20 2782#endif
1e802951 2783 .diag_destroy = udp_abort,
db8dac20 2784};
c482c568 2785EXPORT_SYMBOL(udp_prot);
1da177e4
LT		 2786
2787/* ------------------------------------------------------------------------ */
2788#ifdef CONFIG_PROC_FS
2789
645ca708 2790static struct sock *udp_get_first(struct seq_file *seq, int start)
1da177e4
LT		 2791{
2792 struct sock *sk;
a3d2599b 2793 struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
1da177e4 2794 struct udp_iter_state *state = seq->private;
6f191efe 2795 struct net *net = seq_file_net(seq);
1da177e4 2796
a3d2599b 2797 for (state->bucket = start; state->bucket <= afinfo->udp_table->mask;
f86dcc5a 2798 ++state->bucket) {
a3d2599b 2799 struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket];
f86dcc5a 2800
ca065d0c 2801 if (hlist_empty(&hslot->head))
f86dcc5a
ED		 2802 continue;
2803
645ca708 2804 spin_lock_bh(&hslot->lock);
ca065d0c 2805 sk_for_each(sk, &hslot->head) {
878628fb 2806 if (!net_eq(sock_net(sk), net))
a91275ef 2807 continue;
a3d2599b 2808 if (sk->sk_family == afinfo->family)
1da177e4
LT		 2809 goto found;
2810 }
645ca708 2811 spin_unlock_bh(&hslot->lock);
1da177e4
LT		 2812 }
2813 sk = NULL;
2814found:
2815 return sk;
2816}
2817
2818static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk)
2819{
a3d2599b 2820 struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
1da177e4 2821 struct udp_iter_state *state = seq->private;
6f191efe 2822 struct net *net = seq_file_net(seq);
1da177e4
LT		 2823
2824 do {
ca065d0c 2825 sk = sk_next(sk);
a3d2599b 2826 } while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != afinfo->family));
1da177e4 2827
645ca708 2828 if (!sk) {
a3d2599b
CH		 2829 if (state->bucket <= afinfo->udp_table->mask)
2830 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
645ca708 2831 return udp_get_first(seq, state->bucket + 1);
1da177e4
LT		 2832 }
2833 return sk;
2834}
2835
2836static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos)
2837{
645ca708 2838 struct sock *sk = udp_get_first(seq, 0);
1da177e4
LT		 2839
2840 if (sk)
6516c655 2841 while (pos && (sk = udp_get_next(seq, sk)) != NULL)
1da177e4
LT		 2842 --pos;
2843 return pos ? NULL : sk;
2844}
2845
a3d2599b 2846void *udp_seq_start(struct seq_file *seq, loff_t *pos)
1da177e4 2847{
30842f29 2848 struct udp_iter_state *state = seq->private;
f86dcc5a 2849 state->bucket = MAX_UDP_PORTS;
30842f29 2850
b50660f1 2851 return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN;
1da177e4 2852}
a3d2599b 2853EXPORT_SYMBOL(udp_seq_start);
1da177e4 2854
a3d2599b 2855void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1da177e4
LT		 2856{
2857 struct sock *sk;
2858
b50660f1 2859 if (v == SEQ_START_TOKEN)
1da177e4
LT		 2860 sk = udp_get_idx(seq, 0);
2861 else
2862 sk = udp_get_next(seq, v);
2863
2864 ++*pos;
2865 return sk;
2866}
a3d2599b 2867EXPORT_SYMBOL(udp_seq_next);
1da177e4 2868
a3d2599b 2869void udp_seq_stop(struct seq_file *seq, void *v)
1da177e4 2870{
a3d2599b 2871 struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file));
645ca708
ED		 2872 struct udp_iter_state *state = seq->private;
2873
a3d2599b
CH		 2874 if (state->bucket <= afinfo->udp_table->mask)
2875 spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock);
1da177e4 2876}
a3d2599b 2877EXPORT_SYMBOL(udp_seq_stop);
db8dac20
DM		 2878
2879/* ------------------------------------------------------------------------ */
5e659e4c 2880static void udp4_format_sock(struct sock *sp, struct seq_file *f,
652586df 2881 int bucket)
db8dac20
DM		 2882{
2883 struct inet_sock *inet = inet_sk(sp);
c720c7e8
ED		 2884 __be32 dest = inet->inet_daddr;
2885 __be32 src = inet->inet_rcv_saddr;
2886 __u16 destp = ntohs(inet->inet_dport);
2887 __u16 srcp = ntohs(inet->inet_sport);
db8dac20 2888
f86dcc5a 2889 seq_printf(f, "%5d: %08X:%04X %08X:%04X"
ea9a0379 2890 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u",
db8dac20 2891 bucket, src, srcp, dest, destp, sp->sk_state,
31e6d363 2892 sk_wmem_alloc_get(sp),
6c206b20 2893 udp_rqueue_get(sp),
a7cb5a49
EB		 2894 0, 0L, 0,
2895 from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)),
2896 0, sock_i_ino(sp),
41c6d650 2897 refcount_read(&sp->sk_refcnt), sp,
652586df 2898 atomic_read(&sp->sk_drops));
db8dac20
DM		 2899}
2900
2901int udp4_seq_show(struct seq_file *seq, void *v)
2902{
652586df 2903 seq_setwidth(seq, 127);
db8dac20 2904 if (v == SEQ_START_TOKEN)
652586df 2905 seq_puts(seq, " sl local_address rem_address st tx_queue "
db8dac20 2906 "rx_queue tr tm->when retrnsmt uid timeout "
cb61cb9b 2907 "inode ref pointer drops");
db8dac20 2908 else {
db8dac20
DM		 2909 struct udp_iter_state *state = seq->private;
2910
652586df 2911 udp4_format_sock(v, seq, state->bucket);
db8dac20 2912 }
652586df 2913 seq_pad(seq, '\n');
db8dac20
DM		 2914 return 0;
2915}
2916
c3506372 2917const struct seq_operations udp_seq_ops = {
a3d2599b
CH		 2918 .start = udp_seq_start,
2919 .next = udp_seq_next,
2920 .stop = udp_seq_stop,
2921 .show = udp4_seq_show,
2922};
c3506372 2923EXPORT_SYMBOL(udp_seq_ops);
73cb88ec 2924
db8dac20 2925static struct udp_seq_afinfo udp4_seq_afinfo = {
db8dac20 2926 .family = AF_INET,
645ca708 2927 .udp_table = &udp_table,
db8dac20
DM		 2928};
2929
2c8c1e72 2930static int __net_init udp4_proc_init_net(struct net *net)
15439feb 2931{
c3506372
CH		 2932 if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops,
2933 sizeof(struct udp_iter_state), &udp4_seq_afinfo))
a3d2599b
CH		 2934 return -ENOMEM;
2935 return 0;
15439feb
PE		 2936}
2937
2c8c1e72 2938static void __net_exit udp4_proc_exit_net(struct net *net)
15439feb 2939{
a3d2599b 2940 remove_proc_entry("udp", net->proc_net);
15439feb
PE		 2941}
2942
2943static struct pernet_operations udp4_net_ops = {
2944 .init = udp4_proc_init_net,
2945 .exit = udp4_proc_exit_net,
2946};
2947
db8dac20
DM		 2948int __init udp4_proc_init(void)
2949{
15439feb 2950 return register_pernet_subsys(&udp4_net_ops);
db8dac20
DM		 2951}
2952
2953void udp4_proc_exit(void)
2954{
15439feb 2955 unregister_pernet_subsys(&udp4_net_ops);
db8dac20 2956}
1da177e4
LT		 2957#endif /* CONFIG_PROC_FS */
2958
f86dcc5a
ED		 2959static __initdata unsigned long uhash_entries;
2960static int __init set_uhash_entries(char *str)
645ca708 2961{
413c27d8
EZ		 2962 ssize_t ret;
2963
f86dcc5a
ED		 2964 if (!str)
2965 return 0;
413c27d8
EZ		 2966
2967 ret = kstrtoul(str, 0, &uhash_entries);
2968 if (ret)
2969 return 0;
2970
f86dcc5a
ED		 2971 if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN)
2972 uhash_entries = UDP_HTABLE_SIZE_MIN;
2973 return 1;
2974}
2975__setup("uhash_entries=", set_uhash_entries);
645ca708 2976
f86dcc5a
ED		 2977void __init udp_table_init(struct udp_table *table, const char *name)
2978{
2979 unsigned int i;
2980
31fe62b9
TB		 2981 table->hash = alloc_large_system_hash(name,
2982 2 * sizeof(struct udp_hslot),
2983 uhash_entries,
2984 21, /* one slot per 2 MB */
2985 0,
2986 &table->log,
2987 &table->mask,
2988 UDP_HTABLE_SIZE_MIN,
2989 64 * 1024);
2990
512615b6 2991 table->hash2 = table->hash + (table->mask + 1);
f86dcc5a 2992 for (i = 0; i <= table->mask; i++) {
ca065d0c 2993 INIT_HLIST_HEAD(&table->hash[i].head);
fdcc8aa9 2994 table->hash[i].count = 0;
645ca708
ED		 2995 spin_lock_init(&table->hash[i].lock);
2996 }
512615b6 2997 for (i = 0; i <= table->mask; i++) {
ca065d0c 2998 INIT_HLIST_HEAD(&table->hash2[i].head);
512615b6
ED		 2999 table->hash2[i].count = 0;
3000 spin_lock_init(&table->hash2[i].lock);
3001 }
645ca708
ED		 3002}
3003
723b8e46
TH		 3004u32 udp_flow_hashrnd(void)
3005{
3006 static u32 hashrnd __read_mostly;
3007
3008 net_get_random_once(&hashrnd, sizeof(hashrnd));
3009
3010 return hashrnd;
3011}
3012EXPORT_SYMBOL(udp_flow_hashrnd);
3013
1e802951
TZ		 3014static void __udp_sysctl_init(struct net *net)
3015{
3016 net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM;
3017 net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM;
3018
3019#ifdef CONFIG_NET_L3_MASTER_DEV
3020 net->ipv4.sysctl_udp_l3mdev_accept = 0;
3021#endif
3022}
3023
3024static int __net_init udp_sysctl_init(struct net *net)
3025{
3026 __udp_sysctl_init(net);
3027 return 0;
3028}
3029
3030static struct pernet_operations __net_initdata udp_sysctl_ops = {
fc18999e 3031 .init = udp_sysctl_init,
1e802951
TZ		 3032};
3033
95766fff
HA		 3034void __init udp_init(void)
3035{
f03d78db 3036 unsigned long limit;
4b272750 3037 unsigned int i;
95766fff 3038
f86dcc5a 3039 udp_table_init(&udp_table, "UDP");
f03d78db 3040 limit = nr_free_buffer_pages() / 8;
95766fff
HA		 3041 limit = max(limit, 128UL);
3042 sysctl_udp_mem[0] = limit / 4 * 3;
3043 sysctl_udp_mem[1] = limit;
3044 sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2;
3045
1e802951 3046 __udp_sysctl_init(&init_net);
4b272750
ED		 3047
3048 /* 16 spinlocks per cpu */
3049 udp_busylocks_log = ilog2(nr_cpu_ids) + 4;
3050 udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log,
3051 GFP_KERNEL);
3052 if (!udp_busylocks)
3053 panic("UDP: failed to alloc udp_busylocks\n");
3054 for (i = 0; i < (1U << udp_busylocks_log); i++)
3055 spin_lock_init(udp_busylocks + i);
1e802951
TZ		 3056
3057 if (register_pernet_subsys(&udp_sysctl_ops))
3058 panic("UDP: failed to init sysctl parameters.\n");
95766fff 3059}
Linux 6.x block layer and io_uring tree(s)