net: optimize GRO for the common case.
[linux-block.git] / net / core / dev.c
CommitLineData
2874c5fd 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4 2/*
722c9a0c 3 * NET3 Protocol independent device support routines.
1da177e4 4 *
1da177e4 5 * Derived from the non IP parts of dev.c 1.0.19
722c9a0c 6 * Authors: Ross Biro
1da177e4
LT
7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8 * Mark Evans, <evansmp@uhura.aston.ac.uk>
9 *
10 * Additional Authors:
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
12 * Alan Cox <gw4pts@gw4pts.ampr.org>
13 * David Hinds <dahinds@users.sourceforge.net>
14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15 * Adam Sulmicki <adam@cfar.umd.edu>
16 * Pekka Riikonen <priikone@poesidon.pspt.fi>
17 *
18 * Changes:
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
722c9a0c 20 * to 2 if register_netdev gets called
21 * before net_dev_init & also removed a
22 * few lines of code in the process.
1da177e4
LT
23 * Alan Cox : device private ioctl copies fields back.
24 * Alan Cox : Transmit queue code does relevant
25 * stunts to keep the queue safe.
26 * Alan Cox : Fixed double lock.
27 * Alan Cox : Fixed promisc NULL pointer trap
28 * ???????? : Support the full private ioctl range
29 * Alan Cox : Moved ioctl permission check into
30 * drivers
31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI
32 * Alan Cox : 100 backlog just doesn't cut it when
33 * you start doing multicast video 8)
34 * Alan Cox : Rewrote net_bh and list manager.
722c9a0c 35 * Alan Cox : Fix ETH_P_ALL echoback lengths.
1da177e4
LT
36 * Alan Cox : Took out transmit every packet pass
37 * Saved a few bytes in the ioctl handler
38 * Alan Cox : Network driver sets packet type before
39 * calling netif_rx. Saves a function
40 * call a packet.
41 * Alan Cox : Hashed net_bh()
42 * Richard Kooijman: Timestamp fixes.
43 * Alan Cox : Wrong field in SIOCGIFDSTADDR
44 * Alan Cox : Device lock protection.
722c9a0c 45 * Alan Cox : Fixed nasty side effect of device close
1da177e4
LT
46 * changes.
47 * Rudi Cilibrasi : Pass the right thing to
48 * set_mac_address()
49 * Dave Miller : 32bit quantity for the device lock to
50 * make it work out on a Sparc.
51 * Bjorn Ekwall : Added KERNELD hack.
52 * Alan Cox : Cleaned up the backlog initialise.
53 * Craig Metz : SIOCGIFCONF fix if space for under
54 * 1 device.
55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there
56 * is no device open function.
57 * Andi Kleen : Fix error reporting for SIOCGIFCONF
58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF
59 * Cyrus Durgin : Cleaned for KMOD
60 * Adam Sulmicki : Bug Fix : Network Device Unload
61 * A network device unload needs to purge
62 * the backlog queue.
63 * Paul Rusty Russell : SIOCSIFNAME
64 * Pekka Riikonen : Netdev boot-time settings code
65 * Andrew Morton : Make unregister_netdevice wait
722c9a0c 66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
1da177e4
LT
68 * - netif_rx() feedback
69 */
70
7c0f6ba6 71#include <linux/uaccess.h>
1da177e4 72#include <linux/bitops.h>
4fc268d2 73#include <linux/capability.h>
1da177e4
LT
74#include <linux/cpu.h>
75#include <linux/types.h>
76#include <linux/kernel.h>
08e9897d 77#include <linux/hash.h>
5a0e3ad6 78#include <linux/slab.h>
1da177e4 79#include <linux/sched.h>
f1083048 80#include <linux/sched/mm.h>
4a3e2f71 81#include <linux/mutex.h>
11d6011c 82#include <linux/rwsem.h>
1da177e4
LT
83#include <linux/string.h>
84#include <linux/mm.h>
85#include <linux/socket.h>
86#include <linux/sockios.h>
87#include <linux/errno.h>
88#include <linux/interrupt.h>
89#include <linux/if_ether.h>
90#include <linux/netdevice.h>
91#include <linux/etherdevice.h>
0187bdfb 92#include <linux/ethtool.h>
1da177e4 93#include <linux/skbuff.h>
29863d41 94#include <linux/kthread.h>
a7862b45 95#include <linux/bpf.h>
b5cdae32 96#include <linux/bpf_trace.h>
457c4cbc 97#include <net/net_namespace.h>
1da177e4 98#include <net/sock.h>
02d62e86 99#include <net/busy_poll.h>
1da177e4 100#include <linux/rtnetlink.h>
1da177e4 101#include <linux/stat.h>
b14a9fc4 102#include <net/dsa.h>
1da177e4 103#include <net/dst.h>
fc4099f1 104#include <net/dst_metadata.h>
04f00ab2 105#include <net/gro.h>
1da177e4 106#include <net/pkt_sched.h>
87d83093 107#include <net/pkt_cls.h>
1da177e4 108#include <net/checksum.h>
44540960 109#include <net/xfrm.h>
1da177e4
LT
110#include <linux/highmem.h>
111#include <linux/init.h>
1da177e4 112#include <linux/module.h>
1da177e4
LT
113#include <linux/netpoll.h>
114#include <linux/rcupdate.h>
115#include <linux/delay.h>
1da177e4 116#include <net/iw_handler.h>
1da177e4 117#include <asm/current.h>
5bdb9886 118#include <linux/audit.h>
db217334 119#include <linux/dmaengine.h>
f6a78bfc 120#include <linux/err.h>
c7fa9d18 121#include <linux/ctype.h>
723e98b7 122#include <linux/if_arp.h>
6de329e2 123#include <linux/if_vlan.h>
8f0f2223 124#include <linux/ip.h>
ad55dcaf 125#include <net/ip.h>
25cd9ba0 126#include <net/mpls.h>
8f0f2223
DM
127#include <linux/ipv6.h>
128#include <linux/in.h>
b6b2fed1
DM
129#include <linux/jhash.h>
130#include <linux/random.h>
9cbc1cb8 131#include <trace/events/napi.h>
cf66ba58 132#include <trace/events/net.h>
07dc22e7 133#include <trace/events/skb.h>
70713ddd 134#include <trace/events/qdisc.h>
caeda9b9 135#include <linux/inetdevice.h>
c445477d 136#include <linux/cpu_rmap.h>
c5905afb 137#include <linux/static_key.h>
af12fa6e 138#include <linux/hashtable.h>
60877a32 139#include <linux/vmalloc.h>
529d0489 140#include <linux/if_macvlan.h>
e7fd2885 141#include <linux/errqueue.h>
3b47d303 142#include <linux/hrtimer.h>
357b6cc5 143#include <linux/netfilter_ingress.h>
40e4e713 144#include <linux/crash_dump.h>
b72b5bf6 145#include <linux/sctp.h>
ae847f40 146#include <net/udp_tunnel.h>
6621dd29 147#include <linux/net_namespace.h>
aaa5d90b 148#include <linux/indirect_call_wrapper.h>
af3836df 149#include <net/devlink.h>
bd869245 150#include <linux/pm_runtime.h>
3744741a 151#include <linux/prandom.h>
127d7355 152#include <linux/once_lite.h>
1da177e4 153
342709ef
PE
154#include "net-sysfs.h"
155
d565b0a1
HX
156#define MAX_GRO_SKBS 8
157
5d38a079
HX
158/* This should be increased if a protocol with a bigger head is added. */
159#define GRO_MAX_HEAD (MAX_HEADER + 128)
160
1da177e4 161static DEFINE_SPINLOCK(ptype_lock);
62532da9 162static DEFINE_SPINLOCK(offload_lock);
900ff8c6
CW
163struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
164struct list_head ptype_all __read_mostly; /* Taps */
62532da9 165static struct list_head offload_base __read_mostly;
1da177e4 166
ae78dbfa 167static int netif_rx_internal(struct sk_buff *skb);
54951194 168static int call_netdevice_notifiers_info(unsigned long val,
54951194 169 struct netdev_notifier_info *info);
26372605
PM
170static int call_netdevice_notifiers_extack(unsigned long val,
171 struct net_device *dev,
172 struct netlink_ext_ack *extack);
90b602f8 173static struct napi_struct *napi_by_id(unsigned int napi_id);
ae78dbfa 174
1da177e4 175/*
7562f876 176 * The @dev_base_head list is protected by @dev_base_lock and the rtnl
1da177e4
LT
177 * semaphore.
178 *
c6d14c84 179 * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
1da177e4
LT
180 *
181 * Writers must hold the rtnl semaphore while they loop through the
7562f876 182 * dev_base_head list, and hold dev_base_lock for writing when they do the
1da177e4
LT
183 * actual updates. This allows pure readers to access the list even
184 * while a writer is preparing to update it.
185 *
186 * To put it another way, dev_base_lock is held for writing only to
187 * protect against pure readers; the rtnl semaphore provides the
188 * protection against other writers.
189 *
190 * See, for example usages, register_netdevice() and
191 * unregister_netdevice(), which must be called with the rtnl
192 * semaphore held.
193 */
1da177e4 194DEFINE_RWLOCK(dev_base_lock);
1da177e4
LT
195EXPORT_SYMBOL(dev_base_lock);
196
6c557001
FW
197static DEFINE_MUTEX(ifalias_mutex);
198
af12fa6e
ET
199/* protects napi_hash addition/deletion and napi_gen_id */
200static DEFINE_SPINLOCK(napi_hash_lock);
201
52bd2d62 202static unsigned int napi_gen_id = NR_CPUS;
6180d9de 203static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
af12fa6e 204
11d6011c 205static DECLARE_RWSEM(devnet_rename_sem);
c91f6df2 206
4e985ada
TG
207static inline void dev_base_seq_inc(struct net *net)
208{
643aa9cb 209 while (++net->dev_base_seq == 0)
210 ;
4e985ada
TG
211}
212
881d966b 213static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
1da177e4 214{
8387ff25 215 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
95c96174 216
08e9897d 217 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
1da177e4
LT
218}
219
881d966b 220static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
1da177e4 221{
7c28bd0b 222 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
1da177e4
LT
223}
224
e36fa2f7 225static inline void rps_lock(struct softnet_data *sd)
152102c7
CG
226{
227#ifdef CONFIG_RPS
e36fa2f7 228 spin_lock(&sd->input_pkt_queue.lock);
152102c7
CG
229#endif
230}
231
e36fa2f7 232static inline void rps_unlock(struct softnet_data *sd)
152102c7
CG
233{
234#ifdef CONFIG_RPS
e36fa2f7 235 spin_unlock(&sd->input_pkt_queue.lock);
152102c7
CG
236#endif
237}
238
ff927412
JP
239static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
240 const char *name)
241{
242 struct netdev_name_node *name_node;
243
244 name_node = kmalloc(sizeof(*name_node), GFP_KERNEL);
245 if (!name_node)
246 return NULL;
247 INIT_HLIST_NODE(&name_node->hlist);
248 name_node->dev = dev;
249 name_node->name = name;
250 return name_node;
251}
252
253static struct netdev_name_node *
254netdev_name_node_head_alloc(struct net_device *dev)
255{
36fbf1e5
JP
256 struct netdev_name_node *name_node;
257
258 name_node = netdev_name_node_alloc(dev, dev->name);
259 if (!name_node)
260 return NULL;
261 INIT_LIST_HEAD(&name_node->list);
262 return name_node;
ff927412
JP
263}
264
265static void netdev_name_node_free(struct netdev_name_node *name_node)
266{
267 kfree(name_node);
268}
269
270static void netdev_name_node_add(struct net *net,
271 struct netdev_name_node *name_node)
272{
273 hlist_add_head_rcu(&name_node->hlist,
274 dev_name_hash(net, name_node->name));
275}
276
277static void netdev_name_node_del(struct netdev_name_node *name_node)
278{
279 hlist_del_rcu(&name_node->hlist);
280}
281
282static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
283 const char *name)
284{
285 struct hlist_head *head = dev_name_hash(net, name);
286 struct netdev_name_node *name_node;
287
288 hlist_for_each_entry(name_node, head, hlist)
289 if (!strcmp(name_node->name, name))
290 return name_node;
291 return NULL;
292}
293
294static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
295 const char *name)
296{
297 struct hlist_head *head = dev_name_hash(net, name);
298 struct netdev_name_node *name_node;
299
300 hlist_for_each_entry_rcu(name_node, head, hlist)
301 if (!strcmp(name_node->name, name))
302 return name_node;
303 return NULL;
304}
305
36fbf1e5
JP
306int netdev_name_node_alt_create(struct net_device *dev, const char *name)
307{
308 struct netdev_name_node *name_node;
309 struct net *net = dev_net(dev);
310
311 name_node = netdev_name_node_lookup(net, name);
312 if (name_node)
313 return -EEXIST;
314 name_node = netdev_name_node_alloc(dev, name);
315 if (!name_node)
316 return -ENOMEM;
317 netdev_name_node_add(net, name_node);
318 /* The node that holds dev->name acts as a head of per-device list. */
319 list_add_tail(&name_node->list, &dev->name_node->list);
320
321 return 0;
322}
323EXPORT_SYMBOL(netdev_name_node_alt_create);
324
325static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
326{
327 list_del(&name_node->list);
328 netdev_name_node_del(name_node);
329 kfree(name_node->name);
330 netdev_name_node_free(name_node);
331}
332
333int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
334{
335 struct netdev_name_node *name_node;
336 struct net *net = dev_net(dev);
337
338 name_node = netdev_name_node_lookup(net, name);
339 if (!name_node)
340 return -ENOENT;
e08ad805
ED
341 /* lookup might have found our primary name or a name belonging
342 * to another device.
343 */
344 if (name_node == dev->name_node || name_node->dev != dev)
345 return -EINVAL;
346
36fbf1e5
JP
347 __netdev_name_node_alt_destroy(name_node);
348
349 return 0;
350}
351EXPORT_SYMBOL(netdev_name_node_alt_destroy);
352
353static void netdev_name_node_alt_flush(struct net_device *dev)
354{
355 struct netdev_name_node *name_node, *tmp;
356
357 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list)
358 __netdev_name_node_alt_destroy(name_node);
359}
360
ce286d32 361/* Device list insertion */
53759be9 362static void list_netdevice(struct net_device *dev)
ce286d32 363{
c346dca1 364 struct net *net = dev_net(dev);
ce286d32
EB
365
366 ASSERT_RTNL();
367
368 write_lock_bh(&dev_base_lock);
c6d14c84 369 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
ff927412 370 netdev_name_node_add(net, dev->name_node);
fb699dfd
ED
371 hlist_add_head_rcu(&dev->index_hlist,
372 dev_index_hash(net, dev->ifindex));
ce286d32 373 write_unlock_bh(&dev_base_lock);
4e985ada
TG
374
375 dev_base_seq_inc(net);
ce286d32
EB
376}
377
fb699dfd
ED
378/* Device list removal
379 * caller must respect a RCU grace period before freeing/reusing dev
380 */
ce286d32
EB
381static void unlist_netdevice(struct net_device *dev)
382{
383 ASSERT_RTNL();
384
385 /* Unlink dev from the device chain */
386 write_lock_bh(&dev_base_lock);
c6d14c84 387 list_del_rcu(&dev->dev_list);
ff927412 388 netdev_name_node_del(dev->name_node);
fb699dfd 389 hlist_del_rcu(&dev->index_hlist);
ce286d32 390 write_unlock_bh(&dev_base_lock);
4e985ada
TG
391
392 dev_base_seq_inc(dev_net(dev));
ce286d32
EB
393}
394
1da177e4
LT
395/*
396 * Our notifier list
397 */
398
f07d5b94 399static RAW_NOTIFIER_HEAD(netdev_chain);
1da177e4
LT
400
401/*
402 * Device drivers call our routines to queue packets here. We empty the
403 * queue in the local softnet handler.
404 */
bea3348e 405
9958da05 406DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
d1b19dff 407EXPORT_PER_CPU_SYMBOL(softnet_data);
1da177e4 408
1a33e10e
CW
409#ifdef CONFIG_LOCKDEP
410/*
411 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
412 * according to dev->type
413 */
414static const unsigned short netdev_lock_type[] = {
415 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
416 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
417 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
418 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
419 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
420 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
421 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
422 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
423 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
424 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
425 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
426 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
427 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
428 ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
429 ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
430
431static const char *const netdev_lock_name[] = {
432 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
433 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
434 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
435 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
436 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
437 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
438 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
439 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
440 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
441 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
442 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
443 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
444 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
445 "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
446 "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
447
448static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
845e0ebb 449static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
1a33e10e
CW
450
451static inline unsigned short netdev_lock_pos(unsigned short dev_type)
452{
453 int i;
454
455 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
456 if (netdev_lock_type[i] == dev_type)
457 return i;
458 /* the last key is used by default */
459 return ARRAY_SIZE(netdev_lock_type) - 1;
460}
461
462static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
463 unsigned short dev_type)
464{
465 int i;
466
467 i = netdev_lock_pos(dev_type);
468 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
469 netdev_lock_name[i]);
470}
845e0ebb
CW
471
472static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
473{
474 int i;
475
476 i = netdev_lock_pos(dev->type);
477 lockdep_set_class_and_name(&dev->addr_list_lock,
478 &netdev_addr_lock_key[i],
479 netdev_lock_name[i]);
480}
1a33e10e
CW
481#else
482static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
483 unsigned short dev_type)
484{
485}
845e0ebb
CW
486
487static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
488{
489}
1a33e10e
CW
490#endif
491
1da177e4 492/*******************************************************************************
eb13da1a 493 *
494 * Protocol management and registration routines
495 *
496 *******************************************************************************/
1da177e4 497
1da177e4 498
1da177e4
LT
499/*
500 * Add a protocol ID to the list. Now that the input handler is
501 * smarter we can dispense with all the messy stuff that used to be
502 * here.
503 *
504 * BEWARE!!! Protocol handlers, mangling input packets,
505 * MUST BE last in hash buckets and checking protocol handlers
506 * MUST start from promiscuous ptype_all chain in net_bh.
507 * It is true now, do not change it.
508 * Explanation follows: if protocol handler, mangling packet, will
509 * be the first on list, it is not able to sense, that packet
510 * is cloned and should be copied-on-write, so that it will
511 * change it and subsequent readers will get broken packet.
512 * --ANK (980803)
513 */
514
c07b68e8
ED
515static inline struct list_head *ptype_head(const struct packet_type *pt)
516{
517 if (pt->type == htons(ETH_P_ALL))
7866a621 518 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
c07b68e8 519 else
7866a621
SN
520 return pt->dev ? &pt->dev->ptype_specific :
521 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
c07b68e8
ED
522}
523
1da177e4
LT
524/**
525 * dev_add_pack - add packet handler
526 * @pt: packet type declaration
527 *
528 * Add a protocol handler to the networking stack. The passed &packet_type
529 * is linked into kernel lists and may not be freed until it has been
530 * removed from the kernel lists.
531 *
4ec93edb 532 * This call does not sleep therefore it can not
1da177e4
LT
533 * guarantee all CPU's that are in middle of receiving packets
534 * will see the new packet type (until the next received packet).
535 */
536
537void dev_add_pack(struct packet_type *pt)
538{
c07b68e8 539 struct list_head *head = ptype_head(pt);
1da177e4 540
c07b68e8
ED
541 spin_lock(&ptype_lock);
542 list_add_rcu(&pt->list, head);
543 spin_unlock(&ptype_lock);
1da177e4 544}
d1b19dff 545EXPORT_SYMBOL(dev_add_pack);
1da177e4 546
1da177e4
LT
547/**
548 * __dev_remove_pack - remove packet handler
549 * @pt: packet type declaration
550 *
551 * Remove a protocol handler that was previously added to the kernel
552 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
553 * from the kernel lists and can be freed or reused once this function
4ec93edb 554 * returns.
1da177e4
LT
555 *
556 * The packet type might still be in use by receivers
557 * and must not be freed until after all the CPU's have gone
558 * through a quiescent state.
559 */
560void __dev_remove_pack(struct packet_type *pt)
561{
c07b68e8 562 struct list_head *head = ptype_head(pt);
1da177e4
LT
563 struct packet_type *pt1;
564
c07b68e8 565 spin_lock(&ptype_lock);
1da177e4
LT
566
567 list_for_each_entry(pt1, head, list) {
568 if (pt == pt1) {
569 list_del_rcu(&pt->list);
570 goto out;
571 }
572 }
573
7b6cd1ce 574 pr_warn("dev_remove_pack: %p not found\n", pt);
1da177e4 575out:
c07b68e8 576 spin_unlock(&ptype_lock);
1da177e4 577}
d1b19dff
ED
578EXPORT_SYMBOL(__dev_remove_pack);
579
1da177e4
LT
580/**
581 * dev_remove_pack - remove packet handler
582 * @pt: packet type declaration
583 *
584 * Remove a protocol handler that was previously added to the kernel
585 * protocol handlers by dev_add_pack(). The passed &packet_type is removed
586 * from the kernel lists and can be freed or reused once this function
587 * returns.
588 *
589 * This call sleeps to guarantee that no CPU is looking at the packet
590 * type after return.
591 */
592void dev_remove_pack(struct packet_type *pt)
593{
594 __dev_remove_pack(pt);
4ec93edb 595
1da177e4
LT
596 synchronize_net();
597}
d1b19dff 598EXPORT_SYMBOL(dev_remove_pack);
1da177e4 599
62532da9
VY
600
601/**
602 * dev_add_offload - register offload handlers
603 * @po: protocol offload declaration
604 *
605 * Add protocol offload handlers to the networking stack. The passed
606 * &proto_offload is linked into kernel lists and may not be freed until
607 * it has been removed from the kernel lists.
608 *
609 * This call does not sleep therefore it can not
610 * guarantee all CPU's that are in middle of receiving packets
611 * will see the new offload handlers (until the next received packet).
612 */
613void dev_add_offload(struct packet_offload *po)
614{
bdef7de4 615 struct packet_offload *elem;
62532da9
VY
616
617 spin_lock(&offload_lock);
bdef7de4
DM
618 list_for_each_entry(elem, &offload_base, list) {
619 if (po->priority < elem->priority)
620 break;
621 }
622 list_add_rcu(&po->list, elem->list.prev);
62532da9
VY
623 spin_unlock(&offload_lock);
624}
625EXPORT_SYMBOL(dev_add_offload);
626
627/**
628 * __dev_remove_offload - remove offload handler
629 * @po: packet offload declaration
630 *
631 * Remove a protocol offload handler that was previously added to the
632 * kernel offload handlers by dev_add_offload(). The passed &offload_type
633 * is removed from the kernel lists and can be freed or reused once this
634 * function returns.
635 *
636 * The packet type might still be in use by receivers
637 * and must not be freed until after all the CPU's have gone
638 * through a quiescent state.
639 */
1d143d9f 640static void __dev_remove_offload(struct packet_offload *po)
62532da9
VY
641{
642 struct list_head *head = &offload_base;
643 struct packet_offload *po1;
644
c53aa505 645 spin_lock(&offload_lock);
62532da9
VY
646
647 list_for_each_entry(po1, head, list) {
648 if (po == po1) {
649 list_del_rcu(&po->list);
650 goto out;
651 }
652 }
653
654 pr_warn("dev_remove_offload: %p not found\n", po);
655out:
c53aa505 656 spin_unlock(&offload_lock);
62532da9 657}
62532da9
VY
658
659/**
660 * dev_remove_offload - remove packet offload handler
661 * @po: packet offload declaration
662 *
663 * Remove a packet offload handler that was previously added to the kernel
664 * offload handlers by dev_add_offload(). The passed &offload_type is
665 * removed from the kernel lists and can be freed or reused once this
666 * function returns.
667 *
668 * This call sleeps to guarantee that no CPU is looking at the packet
669 * type after return.
670 */
671void dev_remove_offload(struct packet_offload *po)
672{
673 __dev_remove_offload(po);
674
675 synchronize_net();
676}
677EXPORT_SYMBOL(dev_remove_offload);
678
1da177e4 679/******************************************************************************
eb13da1a 680 *
681 * Device Boot-time Settings Routines
682 *
683 ******************************************************************************/
1da177e4
LT
684
685/* Boot time configuration table */
686static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
687
688/**
689 * netdev_boot_setup_add - add new setup entry
690 * @name: name of the device
691 * @map: configured settings for the device
692 *
693 * Adds new setup entry to the dev_boot_setup list. The function
694 * returns 0 on error and 1 on success. This is a generic routine to
695 * all netdevices.
696 */
697static int netdev_boot_setup_add(char *name, struct ifmap *map)
698{
699 struct netdev_boot_setup *s;
700 int i;
701
702 s = dev_boot_setup;
703 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
704 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
705 memset(s[i].name, 0, sizeof(s[i].name));
93b3cff9 706 strlcpy(s[i].name, name, IFNAMSIZ);
1da177e4
LT
707 memcpy(&s[i].map, map, sizeof(s[i].map));
708 break;
709 }
710 }
711
712 return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
713}
714
715/**
722c9a0c 716 * netdev_boot_setup_check - check boot time settings
717 * @dev: the netdevice
1da177e4 718 *
722c9a0c 719 * Check boot time settings for the device.
720 * The found settings are set for the device to be used
721 * later in the device probing.
722 * Returns 0 if no settings found, 1 if they are.
1da177e4
LT
723 */
724int netdev_boot_setup_check(struct net_device *dev)
725{
726 struct netdev_boot_setup *s = dev_boot_setup;
727 int i;
728
729 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
730 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
93b3cff9 731 !strcmp(dev->name, s[i].name)) {
722c9a0c 732 dev->irq = s[i].map.irq;
733 dev->base_addr = s[i].map.base_addr;
734 dev->mem_start = s[i].map.mem_start;
735 dev->mem_end = s[i].map.mem_end;
1da177e4
LT
736 return 1;
737 }
738 }
739 return 0;
740}
d1b19dff 741EXPORT_SYMBOL(netdev_boot_setup_check);
1da177e4
LT
742
743
744/**
722c9a0c 745 * netdev_boot_base - get address from boot time settings
746 * @prefix: prefix for network device
747 * @unit: id for network device
748 *
749 * Check boot time settings for the base address of device.
750 * The found settings are set for the device to be used
751 * later in the device probing.
752 * Returns 0 if no settings found.
1da177e4
LT
753 */
754unsigned long netdev_boot_base(const char *prefix, int unit)
755{
756 const struct netdev_boot_setup *s = dev_boot_setup;
757 char name[IFNAMSIZ];
758 int i;
759
760 sprintf(name, "%s%d", prefix, unit);
761
762 /*
763 * If device already registered then return base of 1
764 * to indicate not to probe for this interface
765 */
881d966b 766 if (__dev_get_by_name(&init_net, name))
1da177e4
LT
767 return 1;
768
769 for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
770 if (!strcmp(name, s[i].name))
771 return s[i].map.base_addr;
772 return 0;
773}
774
775/*
776 * Saves at boot time configured settings for any netdevice.
777 */
778int __init netdev_boot_setup(char *str)
779{
780 int ints[5];
781 struct ifmap map;
782
783 str = get_options(str, ARRAY_SIZE(ints), ints);
784 if (!str || !*str)
785 return 0;
786
787 /* Save settings */
788 memset(&map, 0, sizeof(map));
789 if (ints[0] > 0)
790 map.irq = ints[1];
791 if (ints[0] > 1)
792 map.base_addr = ints[2];
793 if (ints[0] > 2)
794 map.mem_start = ints[3];
795 if (ints[0] > 3)
796 map.mem_end = ints[4];
797
798 /* Add new entry to the list */
799 return netdev_boot_setup_add(str, &map);
800}
801
802__setup("netdev=", netdev_boot_setup);
803
804/*******************************************************************************
eb13da1a 805 *
806 * Device Interface Subroutines
807 *
808 *******************************************************************************/
1da177e4 809
a54acb3a
ND
810/**
811 * dev_get_iflink - get 'iflink' value of a interface
812 * @dev: targeted interface
813 *
814 * Indicates the ifindex the interface is linked to.
815 * Physical interfaces have the same 'ifindex' and 'iflink' values.
816 */
817
818int dev_get_iflink(const struct net_device *dev)
819{
820 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
821 return dev->netdev_ops->ndo_get_iflink(dev);
822
7a66bbc9 823 return dev->ifindex;
a54acb3a
ND
824}
825EXPORT_SYMBOL(dev_get_iflink);
826
fc4099f1
PS
827/**
828 * dev_fill_metadata_dst - Retrieve tunnel egress information.
829 * @dev: targeted interface
830 * @skb: The packet.
831 *
832 * For better visibility of tunnel traffic OVS needs to retrieve
833 * egress tunnel information for a packet. Following API allows
834 * user to get this info.
835 */
836int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
837{
838 struct ip_tunnel_info *info;
839
840 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst)
841 return -EINVAL;
842
843 info = skb_tunnel_info_unclone(skb);
844 if (!info)
845 return -ENOMEM;
846 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
847 return -EINVAL;
848
849 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
850}
851EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
852
ddb94eaf
PNA
853static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
854{
855 int k = stack->num_paths++;
856
857 if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
858 return NULL;
859
860 return &stack->path[k];
861}
862
863int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
864 struct net_device_path_stack *stack)
865{
866 const struct net_device *last_dev;
867 struct net_device_path_ctx ctx = {
868 .dev = dev,
869 .daddr = daddr,
870 };
871 struct net_device_path *path;
872 int ret = 0;
873
874 stack->num_paths = 0;
875 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
876 last_dev = ctx.dev;
877 path = dev_fwd_path(stack);
878 if (!path)
879 return -1;
880
881 memset(path, 0, sizeof(struct net_device_path));
882 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
883 if (ret < 0)
884 return -1;
885
886 if (WARN_ON_ONCE(last_dev == ctx.dev))
887 return -1;
888 }
889 path = dev_fwd_path(stack);
890 if (!path)
891 return -1;
892 path->type = DEV_PATH_ETHERNET;
893 path->dev = ctx.dev;
894
895 return ret;
896}
897EXPORT_SYMBOL_GPL(dev_fill_forward_path);
898
1da177e4
LT
899/**
900 * __dev_get_by_name - find a device by its name
c4ea43c5 901 * @net: the applicable net namespace
1da177e4
LT
902 * @name: name to find
903 *
904 * Find an interface by name. Must be called under RTNL semaphore
905 * or @dev_base_lock. If the name is found a pointer to the device
906 * is returned. If the name is not found then %NULL is returned. The
907 * reference counters are not incremented so the caller must be
908 * careful with locks.
909 */
910
881d966b 911struct net_device *__dev_get_by_name(struct net *net, const char *name)
1da177e4 912{
ff927412 913 struct netdev_name_node *node_name;
1da177e4 914
ff927412
JP
915 node_name = netdev_name_node_lookup(net, name);
916 return node_name ? node_name->dev : NULL;
1da177e4 917}
d1b19dff 918EXPORT_SYMBOL(__dev_get_by_name);
1da177e4 919
72c9528b 920/**
722c9a0c 921 * dev_get_by_name_rcu - find a device by its name
922 * @net: the applicable net namespace
923 * @name: name to find
924 *
925 * Find an interface by name.
926 * If the name is found a pointer to the device is returned.
927 * If the name is not found then %NULL is returned.
928 * The reference counters are not incremented so the caller must be
929 * careful with locks. The caller must hold RCU lock.
72c9528b
ED
930 */
931
932struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
933{
ff927412 934 struct netdev_name_node *node_name;
72c9528b 935
ff927412
JP
936 node_name = netdev_name_node_lookup_rcu(net, name);
937 return node_name ? node_name->dev : NULL;
72c9528b
ED
938}
939EXPORT_SYMBOL(dev_get_by_name_rcu);
940
1da177e4
LT
941/**
942 * dev_get_by_name - find a device by its name
c4ea43c5 943 * @net: the applicable net namespace
1da177e4
LT
944 * @name: name to find
945 *
946 * Find an interface by name. This can be called from any
947 * context and does its own locking. The returned handle has
948 * the usage count incremented and the caller must use dev_put() to
949 * release it when it is no longer needed. %NULL is returned if no
950 * matching device is found.
951 */
952
881d966b 953struct net_device *dev_get_by_name(struct net *net, const char *name)
1da177e4
LT
954{
955 struct net_device *dev;
956
72c9528b
ED
957 rcu_read_lock();
958 dev = dev_get_by_name_rcu(net, name);
1da177e4
LT
959 if (dev)
960 dev_hold(dev);
72c9528b 961 rcu_read_unlock();
1da177e4
LT
962 return dev;
963}
d1b19dff 964EXPORT_SYMBOL(dev_get_by_name);
1da177e4
LT
965
966/**
967 * __dev_get_by_index - find a device by its ifindex
c4ea43c5 968 * @net: the applicable net namespace
1da177e4
LT
969 * @ifindex: index of device
970 *
971 * Search for an interface by index. Returns %NULL if the device
972 * is not found or a pointer to the device. The device has not
973 * had its reference counter increased so the caller must be careful
974 * about locking. The caller must hold either the RTNL semaphore
975 * or @dev_base_lock.
976 */
977
881d966b 978struct net_device *__dev_get_by_index(struct net *net, int ifindex)
1da177e4 979{
0bd8d536
ED
980 struct net_device *dev;
981 struct hlist_head *head = dev_index_hash(net, ifindex);
1da177e4 982
b67bfe0d 983 hlist_for_each_entry(dev, head, index_hlist)
1da177e4
LT
984 if (dev->ifindex == ifindex)
985 return dev;
0bd8d536 986
1da177e4
LT
987 return NULL;
988}
d1b19dff 989EXPORT_SYMBOL(__dev_get_by_index);
1da177e4 990
fb699dfd
ED
991/**
992 * dev_get_by_index_rcu - find a device by its ifindex
993 * @net: the applicable net namespace
994 * @ifindex: index of device
995 *
996 * Search for an interface by index. Returns %NULL if the device
997 * is not found or a pointer to the device. The device has not
998 * had its reference counter increased so the caller must be careful
999 * about locking. The caller must hold RCU lock.
1000 */
1001
1002struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
1003{
fb699dfd
ED
1004 struct net_device *dev;
1005 struct hlist_head *head = dev_index_hash(net, ifindex);
1006
b67bfe0d 1007 hlist_for_each_entry_rcu(dev, head, index_hlist)
fb699dfd
ED
1008 if (dev->ifindex == ifindex)
1009 return dev;
1010
1011 return NULL;
1012}
1013EXPORT_SYMBOL(dev_get_by_index_rcu);
1014
1da177e4
LT
1015
1016/**
1017 * dev_get_by_index - find a device by its ifindex
c4ea43c5 1018 * @net: the applicable net namespace
1da177e4
LT
1019 * @ifindex: index of device
1020 *
1021 * Search for an interface by index. Returns NULL if the device
1022 * is not found or a pointer to the device. The device returned has
1023 * had a reference added and the pointer is safe until the user calls
1024 * dev_put to indicate they have finished with it.
1025 */
1026
881d966b 1027struct net_device *dev_get_by_index(struct net *net, int ifindex)
1da177e4
LT
1028{
1029 struct net_device *dev;
1030
fb699dfd
ED
1031 rcu_read_lock();
1032 dev = dev_get_by_index_rcu(net, ifindex);
1da177e4
LT
1033 if (dev)
1034 dev_hold(dev);
fb699dfd 1035 rcu_read_unlock();
1da177e4
LT
1036 return dev;
1037}
d1b19dff 1038EXPORT_SYMBOL(dev_get_by_index);
1da177e4 1039
90b602f8
ML
1040/**
1041 * dev_get_by_napi_id - find a device by napi_id
1042 * @napi_id: ID of the NAPI struct
1043 *
1044 * Search for an interface by NAPI ID. Returns %NULL if the device
1045 * is not found or a pointer to the device. The device has not had
1046 * its reference counter increased so the caller must be careful
1047 * about locking. The caller must hold RCU lock.
1048 */
1049
1050struct net_device *dev_get_by_napi_id(unsigned int napi_id)
1051{
1052 struct napi_struct *napi;
1053
1054 WARN_ON_ONCE(!rcu_read_lock_held());
1055
1056 if (napi_id < MIN_NAPI_ID)
1057 return NULL;
1058
1059 napi = napi_by_id(napi_id);
1060
1061 return napi ? napi->dev : NULL;
1062}
1063EXPORT_SYMBOL(dev_get_by_napi_id);
1064
5dbe7c17
NS
1065/**
1066 * netdev_get_name - get a netdevice name, knowing its ifindex.
1067 * @net: network namespace
1068 * @name: a pointer to the buffer where the name will be stored.
1069 * @ifindex: the ifindex of the interface to get the name from.
5dbe7c17
NS
1070 */
1071int netdev_get_name(struct net *net, char *name, int ifindex)
1072{
1073 struct net_device *dev;
11d6011c 1074 int ret;
5dbe7c17 1075
11d6011c 1076 down_read(&devnet_rename_sem);
5dbe7c17 1077 rcu_read_lock();
11d6011c 1078
5dbe7c17
NS
1079 dev = dev_get_by_index_rcu(net, ifindex);
1080 if (!dev) {
11d6011c
AD
1081 ret = -ENODEV;
1082 goto out;
5dbe7c17
NS
1083 }
1084
1085 strcpy(name, dev->name);
5dbe7c17 1086
11d6011c
AD
1087 ret = 0;
1088out:
1089 rcu_read_unlock();
1090 up_read(&devnet_rename_sem);
1091 return ret;
5dbe7c17
NS
1092}
1093
1da177e4 1094/**
941666c2 1095 * dev_getbyhwaddr_rcu - find a device by its hardware address
c4ea43c5 1096 * @net: the applicable net namespace
1da177e4
LT
1097 * @type: media type of device
1098 * @ha: hardware address
1099 *
1100 * Search for an interface by MAC address. Returns NULL if the device
c506653d
ED
1101 * is not found or a pointer to the device.
1102 * The caller must hold RCU or RTNL.
941666c2 1103 * The returned device has not had its ref count increased
1da177e4
LT
1104 * and the caller must therefore be careful about locking
1105 *
1da177e4
LT
1106 */
1107
941666c2
ED
1108struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
1109 const char *ha)
1da177e4
LT
1110{
1111 struct net_device *dev;
1112
941666c2 1113 for_each_netdev_rcu(net, dev)
1da177e4
LT
1114 if (dev->type == type &&
1115 !memcmp(dev->dev_addr, ha, dev->addr_len))
7562f876
PE
1116 return dev;
1117
1118 return NULL;
1da177e4 1119}
941666c2 1120EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
cf309e3f 1121
881d966b 1122struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
4e9cac2b 1123{
99fe3c39 1124 struct net_device *dev, *ret = NULL;
4e9cac2b 1125
99fe3c39
ED
1126 rcu_read_lock();
1127 for_each_netdev_rcu(net, dev)
1128 if (dev->type == type) {
1129 dev_hold(dev);
1130 ret = dev;
1131 break;
1132 }
1133 rcu_read_unlock();
1134 return ret;
1da177e4 1135}
1da177e4
LT
1136EXPORT_SYMBOL(dev_getfirstbyhwtype);
1137
1138/**
6c555490 1139 * __dev_get_by_flags - find any device with given flags
c4ea43c5 1140 * @net: the applicable net namespace
1da177e4
LT
1141 * @if_flags: IFF_* values
1142 * @mask: bitmask of bits in if_flags to check
1143 *
1144 * Search for any interface with the given flags. Returns NULL if a device
bb69ae04 1145 * is not found or a pointer to the device. Must be called inside
6c555490 1146 * rtnl_lock(), and result refcount is unchanged.
1da177e4
LT
1147 */
1148
6c555490
WC
1149struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
1150 unsigned short mask)
1da177e4 1151{
7562f876 1152 struct net_device *dev, *ret;
1da177e4 1153
6c555490
WC
1154 ASSERT_RTNL();
1155
7562f876 1156 ret = NULL;
6c555490 1157 for_each_netdev(net, dev) {
1da177e4 1158 if (((dev->flags ^ if_flags) & mask) == 0) {
7562f876 1159 ret = dev;
1da177e4
LT
1160 break;
1161 }
1162 }
7562f876 1163 return ret;
1da177e4 1164}
6c555490 1165EXPORT_SYMBOL(__dev_get_by_flags);
1da177e4
LT
1166
1167/**
1168 * dev_valid_name - check if name is okay for network device
1169 * @name: name string
1170 *
1171 * Network device names need to be valid file names to
4250b75b 1172 * allow sysfs to work. We also disallow any kind of
c7fa9d18 1173 * whitespace.
1da177e4 1174 */
95f050bf 1175bool dev_valid_name(const char *name)
1da177e4 1176{
c7fa9d18 1177 if (*name == '\0')
95f050bf 1178 return false;
a9d48205 1179 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
95f050bf 1180 return false;
c7fa9d18 1181 if (!strcmp(name, ".") || !strcmp(name, ".."))
95f050bf 1182 return false;
c7fa9d18
DM
1183
1184 while (*name) {
a4176a93 1185 if (*name == '/' || *name == ':' || isspace(*name))
95f050bf 1186 return false;
c7fa9d18
DM
1187 name++;
1188 }
95f050bf 1189 return true;
1da177e4 1190}
d1b19dff 1191EXPORT_SYMBOL(dev_valid_name);
1da177e4
LT
1192
1193/**
b267b179
EB
1194 * __dev_alloc_name - allocate a name for a device
1195 * @net: network namespace to allocate the device name in
1da177e4 1196 * @name: name format string
b267b179 1197 * @buf: scratch buffer and result name string
1da177e4
LT
1198 *
1199 * Passed a format string - eg "lt%d" it will try and find a suitable
3041a069
SH
1200 * id. It scans list of devices to build up a free map, then chooses
1201 * the first empty slot. The caller must hold the dev_base or rtnl lock
1202 * while allocating the name and adding the device in order to avoid
1203 * duplicates.
1204 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1205 * Returns the number of the unit assigned or a negative errno code.
1da177e4
LT
1206 */
1207
b267b179 1208static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1da177e4
LT
1209{
1210 int i = 0;
1da177e4
LT
1211 const char *p;
1212 const int max_netdevices = 8*PAGE_SIZE;
cfcabdcc 1213 unsigned long *inuse;
1da177e4
LT
1214 struct net_device *d;
1215
93809105
RV
1216 if (!dev_valid_name(name))
1217 return -EINVAL;
1218
51f299dd 1219 p = strchr(name, '%');
1da177e4
LT
1220 if (p) {
1221 /*
1222 * Verify the string as this thing may have come from
1223 * the user. There must be either one "%d" and no other "%"
1224 * characters.
1225 */
1226 if (p[1] != 'd' || strchr(p + 2, '%'))
1227 return -EINVAL;
1228
1229 /* Use one page as a bit array of possible slots */
cfcabdcc 1230 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1da177e4
LT
1231 if (!inuse)
1232 return -ENOMEM;
1233
881d966b 1234 for_each_netdev(net, d) {
6c015a22
JB
1235 struct netdev_name_node *name_node;
1236 list_for_each_entry(name_node, &d->name_node->list, list) {
1237 if (!sscanf(name_node->name, name, &i))
1238 continue;
1239 if (i < 0 || i >= max_netdevices)
1240 continue;
1241
1242 /* avoid cases where sscanf is not exact inverse of printf */
1243 snprintf(buf, IFNAMSIZ, name, i);
1244 if (!strncmp(buf, name_node->name, IFNAMSIZ))
1245 set_bit(i, inuse);
1246 }
1da177e4
LT
1247 if (!sscanf(d->name, name, &i))
1248 continue;
1249 if (i < 0 || i >= max_netdevices)
1250 continue;
1251
1252 /* avoid cases where sscanf is not exact inverse of printf */
b267b179 1253 snprintf(buf, IFNAMSIZ, name, i);
1da177e4
LT
1254 if (!strncmp(buf, d->name, IFNAMSIZ))
1255 set_bit(i, inuse);
1256 }
1257
1258 i = find_first_zero_bit(inuse, max_netdevices);
1259 free_page((unsigned long) inuse);
1260 }
1261
6224abda 1262 snprintf(buf, IFNAMSIZ, name, i);
b267b179 1263 if (!__dev_get_by_name(net, buf))
1da177e4 1264 return i;
1da177e4
LT
1265
1266 /* It is possible to run out of possible slots
1267 * when the name is long and there isn't enough space left
1268 * for the digits, or if all bits are used.
1269 */
029b6d14 1270 return -ENFILE;
1da177e4
LT
1271}
1272
2c88b855
RV
1273static int dev_alloc_name_ns(struct net *net,
1274 struct net_device *dev,
1275 const char *name)
1276{
1277 char buf[IFNAMSIZ];
1278 int ret;
1279
c46d7642 1280 BUG_ON(!net);
2c88b855
RV
1281 ret = __dev_alloc_name(net, name, buf);
1282 if (ret >= 0)
1283 strlcpy(dev->name, buf, IFNAMSIZ);
1284 return ret;
1da177e4
LT
1285}
1286
b267b179
EB
1287/**
1288 * dev_alloc_name - allocate a name for a device
1289 * @dev: device
1290 * @name: name format string
1291 *
1292 * Passed a format string - eg "lt%d" it will try and find a suitable
1293 * id. It scans list of devices to build up a free map, then chooses
1294 * the first empty slot. The caller must hold the dev_base or rtnl lock
1295 * while allocating the name and adding the device in order to avoid
1296 * duplicates.
1297 * Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1298 * Returns the number of the unit assigned or a negative errno code.
1299 */
1300
1301int dev_alloc_name(struct net_device *dev, const char *name)
1302{
c46d7642 1303 return dev_alloc_name_ns(dev_net(dev), dev, name);
b267b179 1304}
d1b19dff 1305EXPORT_SYMBOL(dev_alloc_name);
b267b179 1306
bacb7e18
ED
1307static int dev_get_valid_name(struct net *net, struct net_device *dev,
1308 const char *name)
828de4f6 1309{
55a5ec9b
DM
1310 BUG_ON(!net);
1311
1312 if (!dev_valid_name(name))
1313 return -EINVAL;
1314
1315 if (strchr(name, '%'))
1316 return dev_alloc_name_ns(net, dev, name);
1317 else if (__dev_get_by_name(net, name))
1318 return -EEXIST;
1319 else if (dev->name != name)
1320 strlcpy(dev->name, name, IFNAMSIZ);
1321
1322 return 0;
d9031024 1323}
1da177e4
LT
1324
1325/**
1326 * dev_change_name - change name of a device
1327 * @dev: device
1328 * @newname: name (or format string) must be at least IFNAMSIZ
1329 *
1330 * Change name of a device, can pass format strings "eth%d".
1331 * for wildcarding.
1332 */
cf04a4c7 1333int dev_change_name(struct net_device *dev, const char *newname)
1da177e4 1334{
238fa362 1335 unsigned char old_assign_type;
fcc5a03a 1336 char oldname[IFNAMSIZ];
1da177e4 1337 int err = 0;
fcc5a03a 1338 int ret;
881d966b 1339 struct net *net;
1da177e4
LT
1340
1341 ASSERT_RTNL();
c346dca1 1342 BUG_ON(!dev_net(dev));
1da177e4 1343
c346dca1 1344 net = dev_net(dev);
8065a779
SWL
1345
1346 /* Some auto-enslaved devices e.g. failover slaves are
1347 * special, as userspace might rename the device after
1348 * the interface had been brought up and running since
1349 * the point kernel initiated auto-enslavement. Allow
1350 * live name change even when these slave devices are
1351 * up and running.
1352 *
1353 * Typically, users of these auto-enslaving devices
1354 * don't actually care about slave name change, as
1355 * they are supposed to operate on master interface
1356 * directly.
1357 */
1358 if (dev->flags & IFF_UP &&
1359 likely(!(dev->priv_flags & IFF_LIVE_RENAME_OK)))
1da177e4
LT
1360 return -EBUSY;
1361
11d6011c 1362 down_write(&devnet_rename_sem);
c91f6df2
BH
1363
1364 if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
11d6011c 1365 up_write(&devnet_rename_sem);
c8d90dca 1366 return 0;
c91f6df2 1367 }
c8d90dca 1368
fcc5a03a
HX
1369 memcpy(oldname, dev->name, IFNAMSIZ);
1370
828de4f6 1371 err = dev_get_valid_name(net, dev, newname);
c91f6df2 1372 if (err < 0) {
11d6011c 1373 up_write(&devnet_rename_sem);
d9031024 1374 return err;
c91f6df2 1375 }
1da177e4 1376
6fe82a39
VF
1377 if (oldname[0] && !strchr(oldname, '%'))
1378 netdev_info(dev, "renamed from %s\n", oldname);
1379
238fa362
TG
1380 old_assign_type = dev->name_assign_type;
1381 dev->name_assign_type = NET_NAME_RENAMED;
1382
fcc5a03a 1383rollback:
a1b3f594
EB
1384 ret = device_rename(&dev->dev, dev->name);
1385 if (ret) {
1386 memcpy(dev->name, oldname, IFNAMSIZ);
238fa362 1387 dev->name_assign_type = old_assign_type;
11d6011c 1388 up_write(&devnet_rename_sem);
a1b3f594 1389 return ret;
dcc99773 1390 }
7f988eab 1391
11d6011c 1392 up_write(&devnet_rename_sem);
c91f6df2 1393
5bb025fa
VF
1394 netdev_adjacent_rename_links(dev, oldname);
1395
7f988eab 1396 write_lock_bh(&dev_base_lock);
ff927412 1397 netdev_name_node_del(dev->name_node);
72c9528b
ED
1398 write_unlock_bh(&dev_base_lock);
1399
1400 synchronize_rcu();
1401
1402 write_lock_bh(&dev_base_lock);
ff927412 1403 netdev_name_node_add(net, dev->name_node);
7f988eab
HX
1404 write_unlock_bh(&dev_base_lock);
1405
056925ab 1406 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
fcc5a03a
HX
1407 ret = notifier_to_errno(ret);
1408
1409 if (ret) {
91e9c07b
ED
1410 /* err >= 0 after dev_alloc_name() or stores the first errno */
1411 if (err >= 0) {
fcc5a03a 1412 err = ret;
11d6011c 1413 down_write(&devnet_rename_sem);
fcc5a03a 1414 memcpy(dev->name, oldname, IFNAMSIZ);
5bb025fa 1415 memcpy(oldname, newname, IFNAMSIZ);
238fa362
TG
1416 dev->name_assign_type = old_assign_type;
1417 old_assign_type = NET_NAME_RENAMED;
fcc5a03a 1418 goto rollback;
91e9c07b 1419 } else {
7b6cd1ce 1420 pr_err("%s: name change rollback failed: %d\n",
91e9c07b 1421 dev->name, ret);
fcc5a03a
HX
1422 }
1423 }
1da177e4
LT
1424
1425 return err;
1426}
1427
0b815a1a
SH
1428/**
1429 * dev_set_alias - change ifalias of a device
1430 * @dev: device
1431 * @alias: name up to IFALIASZ
f0db275a 1432 * @len: limit of bytes to copy from info
0b815a1a
SH
1433 *
1434 * Set ifalias for a device,
1435 */
1436int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1437{
6c557001 1438 struct dev_ifalias *new_alias = NULL;
0b815a1a
SH
1439
1440 if (len >= IFALIASZ)
1441 return -EINVAL;
1442
6c557001
FW
1443 if (len) {
1444 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1445 if (!new_alias)
1446 return -ENOMEM;
1447
1448 memcpy(new_alias->ifalias, alias, len);
1449 new_alias->ifalias[len] = 0;
96ca4a2c
OH
1450 }
1451
6c557001 1452 mutex_lock(&ifalias_mutex);
e3f0d761
PM
1453 new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1454 mutex_is_locked(&ifalias_mutex));
6c557001
FW
1455 mutex_unlock(&ifalias_mutex);
1456
1457 if (new_alias)
1458 kfree_rcu(new_alias, rcuhead);
0b815a1a 1459
0b815a1a
SH
1460 return len;
1461}
0fe554a4 1462EXPORT_SYMBOL(dev_set_alias);
0b815a1a 1463
6c557001
FW
1464/**
1465 * dev_get_alias - get ifalias of a device
1466 * @dev: device
20e88320 1467 * @name: buffer to store name of ifalias
6c557001
FW
1468 * @len: size of buffer
1469 *
1470 * get ifalias for a device. Caller must make sure dev cannot go
1471 * away, e.g. rcu read lock or own a reference count to device.
1472 */
1473int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1474{
1475 const struct dev_ifalias *alias;
1476 int ret = 0;
1477
1478 rcu_read_lock();
1479 alias = rcu_dereference(dev->ifalias);
1480 if (alias)
1481 ret = snprintf(name, len, "%s", alias->ifalias);
1482 rcu_read_unlock();
1483
1484 return ret;
1485}
0b815a1a 1486
d8a33ac4 1487/**
3041a069 1488 * netdev_features_change - device changes features
d8a33ac4
SH
1489 * @dev: device to cause notification
1490 *
1491 * Called to indicate a device has changed features.
1492 */
1493void netdev_features_change(struct net_device *dev)
1494{
056925ab 1495 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
d8a33ac4
SH
1496}
1497EXPORT_SYMBOL(netdev_features_change);
1498
1da177e4
LT
1499/**
1500 * netdev_state_change - device changes state
1501 * @dev: device to cause notification
1502 *
1503 * Called to indicate a device has changed state. This function calls
1504 * the notifier chains for netdev_chain and sends a NEWLINK message
1505 * to the routing socket.
1506 */
1507void netdev_state_change(struct net_device *dev)
1508{
1509 if (dev->flags & IFF_UP) {
51d0c047
DA
1510 struct netdev_notifier_change_info change_info = {
1511 .info.dev = dev,
1512 };
54951194 1513
51d0c047 1514 call_netdevice_notifiers_info(NETDEV_CHANGE,
54951194 1515 &change_info.info);
7f294054 1516 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1da177e4
LT
1517 }
1518}
d1b19dff 1519EXPORT_SYMBOL(netdev_state_change);
1da177e4 1520
7061eb8c
LP
1521/**
1522 * __netdev_notify_peers - notify network peers about existence of @dev,
1523 * to be called when rtnl lock is already held.
1524 * @dev: network device
1525 *
1526 * Generate traffic such that interested network peers are aware of
1527 * @dev, such as by generating a gratuitous ARP. This may be used when
1528 * a device wants to inform the rest of the network about some sort of
1529 * reconfiguration such as a failover event or virtual machine
1530 * migration.
1531 */
1532void __netdev_notify_peers(struct net_device *dev)
1533{
1534 ASSERT_RTNL();
1535 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1536 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1537}
1538EXPORT_SYMBOL(__netdev_notify_peers);
1539
ee89bab1 1540/**
722c9a0c 1541 * netdev_notify_peers - notify network peers about existence of @dev
1542 * @dev: network device
ee89bab1
AW
1543 *
1544 * Generate traffic such that interested network peers are aware of
1545 * @dev, such as by generating a gratuitous ARP. This may be used when
1546 * a device wants to inform the rest of the network about some sort of
1547 * reconfiguration such as a failover event or virtual machine
1548 * migration.
1549 */
1550void netdev_notify_peers(struct net_device *dev)
c1da4ac7 1551{
ee89bab1 1552 rtnl_lock();
7061eb8c 1553 __netdev_notify_peers(dev);
ee89bab1 1554 rtnl_unlock();
c1da4ac7 1555}
ee89bab1 1556EXPORT_SYMBOL(netdev_notify_peers);
c1da4ac7 1557
29863d41
WW
1558static int napi_threaded_poll(void *data);
1559
1560static int napi_kthread_create(struct napi_struct *n)
1561{
1562 int err = 0;
1563
1564 /* Create and wake up the kthread once to put it in
1565 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1566 * warning and work with loadavg.
1567 */
1568 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1569 n->dev->name, n->napi_id);
1570 if (IS_ERR(n->thread)) {
1571 err = PTR_ERR(n->thread);
1572 pr_err("kthread_run failed with err %d\n", err);
1573 n->thread = NULL;
1574 }
1575
1576 return err;
1577}
1578
40c900aa 1579static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1da177e4 1580{
d314774c 1581 const struct net_device_ops *ops = dev->netdev_ops;
3b8bcfd5 1582 int ret;
1da177e4 1583
e46b66bc
BH
1584 ASSERT_RTNL();
1585
bd869245
HK
1586 if (!netif_device_present(dev)) {
1587 /* may be detached because parent is runtime-suspended */
1588 if (dev->dev.parent)
1589 pm_runtime_resume(dev->dev.parent);
1590 if (!netif_device_present(dev))
1591 return -ENODEV;
1592 }
1da177e4 1593
ca99ca14
NH
1594 /* Block netpoll from trying to do any rx path servicing.
1595 * If we don't do this there is a chance ndo_poll_controller
1596 * or ndo_poll may be running while we open the device
1597 */
66b5552f 1598 netpoll_poll_disable(dev);
ca99ca14 1599
40c900aa 1600 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
3b8bcfd5
JB
1601 ret = notifier_to_errno(ret);
1602 if (ret)
1603 return ret;
1604
1da177e4 1605 set_bit(__LINK_STATE_START, &dev->state);
bada339b 1606
d314774c
SH
1607 if (ops->ndo_validate_addr)
1608 ret = ops->ndo_validate_addr(dev);
bada339b 1609
d314774c
SH
1610 if (!ret && ops->ndo_open)
1611 ret = ops->ndo_open(dev);
1da177e4 1612
66b5552f 1613 netpoll_poll_enable(dev);
ca99ca14 1614
bada339b
JG
1615 if (ret)
1616 clear_bit(__LINK_STATE_START, &dev->state);
1617 else {
1da177e4 1618 dev->flags |= IFF_UP;
4417da66 1619 dev_set_rx_mode(dev);
1da177e4 1620 dev_activate(dev);
7bf23575 1621 add_device_randomness(dev->dev_addr, dev->addr_len);
1da177e4 1622 }
bada339b 1623
1da177e4
LT
1624 return ret;
1625}
1626
1627/**
bd380811 1628 * dev_open - prepare an interface for use.
00f54e68
PM
1629 * @dev: device to open
1630 * @extack: netlink extended ack
1da177e4 1631 *
bd380811
PM
1632 * Takes a device from down to up state. The device's private open
1633 * function is invoked and then the multicast lists are loaded. Finally
1634 * the device is moved into the up state and a %NETDEV_UP message is
1635 * sent to the netdev notifier chain.
1636 *
1637 * Calling this function on an active interface is a nop. On a failure
1638 * a negative errno code is returned.
1da177e4 1639 */
00f54e68 1640int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
bd380811
PM
1641{
1642 int ret;
1643
bd380811
PM
1644 if (dev->flags & IFF_UP)
1645 return 0;
1646
40c900aa 1647 ret = __dev_open(dev, extack);
bd380811
PM
1648 if (ret < 0)
1649 return ret;
1650
7f294054 1651 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
bd380811
PM
1652 call_netdevice_notifiers(NETDEV_UP, dev);
1653
1654 return ret;
1655}
1656EXPORT_SYMBOL(dev_open);
1657
7051b88a 1658static void __dev_close_many(struct list_head *head)
1da177e4 1659{
44345724 1660 struct net_device *dev;
e46b66bc 1661
bd380811 1662 ASSERT_RTNL();
9d5010db
DM
1663 might_sleep();
1664
5cde2829 1665 list_for_each_entry(dev, head, close_list) {
3f4df206 1666 /* Temporarily disable netpoll until the interface is down */
66b5552f 1667 netpoll_poll_disable(dev);
3f4df206 1668
44345724 1669 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1da177e4 1670
44345724 1671 clear_bit(__LINK_STATE_START, &dev->state);
1da177e4 1672
44345724
OP
1673 /* Synchronize to scheduled poll. We cannot touch poll list, it
1674 * can be even on different cpu. So just clear netif_running().
1675 *
1676 * dev->stop() will invoke napi_disable() on all of it's
1677 * napi_struct instances on this device.
1678 */
4e857c58 1679 smp_mb__after_atomic(); /* Commit netif_running(). */
44345724 1680 }
1da177e4 1681
44345724 1682 dev_deactivate_many(head);
d8b2a4d2 1683
5cde2829 1684 list_for_each_entry(dev, head, close_list) {
44345724 1685 const struct net_device_ops *ops = dev->netdev_ops;
1da177e4 1686
44345724
OP
1687 /*
1688 * Call the device specific close. This cannot fail.
1689 * Only if device is UP
1690 *
1691 * We allow it to be called even after a DETACH hot-plug
1692 * event.
1693 */
1694 if (ops->ndo_stop)
1695 ops->ndo_stop(dev);
1696
44345724 1697 dev->flags &= ~IFF_UP;
66b5552f 1698 netpoll_poll_enable(dev);
44345724 1699 }
44345724
OP
1700}
1701
7051b88a 1702static void __dev_close(struct net_device *dev)
44345724
OP
1703{
1704 LIST_HEAD(single);
1705
5cde2829 1706 list_add(&dev->close_list, &single);
7051b88a 1707 __dev_close_many(&single);
f87e6f47 1708 list_del(&single);
44345724
OP
1709}
1710
7051b88a 1711void dev_close_many(struct list_head *head, bool unlink)
44345724
OP
1712{
1713 struct net_device *dev, *tmp;
1da177e4 1714
5cde2829
EB
1715 /* Remove the devices that don't need to be closed */
1716 list_for_each_entry_safe(dev, tmp, head, close_list)
44345724 1717 if (!(dev->flags & IFF_UP))
5cde2829 1718 list_del_init(&dev->close_list);
44345724
OP
1719
1720 __dev_close_many(head);
1da177e4 1721
5cde2829 1722 list_for_each_entry_safe(dev, tmp, head, close_list) {
7f294054 1723 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
44345724 1724 call_netdevice_notifiers(NETDEV_DOWN, dev);
99c4a26a
DM
1725 if (unlink)
1726 list_del_init(&dev->close_list);
44345724 1727 }
bd380811 1728}
99c4a26a 1729EXPORT_SYMBOL(dev_close_many);
bd380811
PM
1730
1731/**
1732 * dev_close - shutdown an interface.
1733 * @dev: device to shutdown
1734 *
1735 * This function moves an active device into down state. A
1736 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1737 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1738 * chain.
1739 */
7051b88a 1740void dev_close(struct net_device *dev)
bd380811 1741{
e14a5993
ED
1742 if (dev->flags & IFF_UP) {
1743 LIST_HEAD(single);
1da177e4 1744
5cde2829 1745 list_add(&dev->close_list, &single);
99c4a26a 1746 dev_close_many(&single, true);
e14a5993
ED
1747 list_del(&single);
1748 }
1da177e4 1749}
d1b19dff 1750EXPORT_SYMBOL(dev_close);
1da177e4
LT
1751
1752
0187bdfb
BH
1753/**
1754 * dev_disable_lro - disable Large Receive Offload on a device
1755 * @dev: device
1756 *
1757 * Disable Large Receive Offload (LRO) on a net device. Must be
1758 * called under RTNL. This is needed if received packets may be
1759 * forwarded to another interface.
1760 */
1761void dev_disable_lro(struct net_device *dev)
1762{
fbe168ba
MK
1763 struct net_device *lower_dev;
1764 struct list_head *iter;
529d0489 1765
bc5787c6
MM
1766 dev->wanted_features &= ~NETIF_F_LRO;
1767 netdev_update_features(dev);
27660515 1768
22d5969f
MM
1769 if (unlikely(dev->features & NETIF_F_LRO))
1770 netdev_WARN(dev, "failed to disable LRO!\n");
fbe168ba
MK
1771
1772 netdev_for_each_lower_dev(dev, lower_dev, iter)
1773 dev_disable_lro(lower_dev);
0187bdfb
BH
1774}
1775EXPORT_SYMBOL(dev_disable_lro);
1776
56f5aa77
MC
1777/**
1778 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1779 * @dev: device
1780 *
1781 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be
1782 * called under RTNL. This is needed if Generic XDP is installed on
1783 * the device.
1784 */
1785static void dev_disable_gro_hw(struct net_device *dev)
1786{
1787 dev->wanted_features &= ~NETIF_F_GRO_HW;
1788 netdev_update_features(dev);
1789
1790 if (unlikely(dev->features & NETIF_F_GRO_HW))
1791 netdev_WARN(dev, "failed to disable GRO_HW!\n");
1792}
1793
ede2762d
KT
1794const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1795{
1796#define N(val) \
1797 case NETDEV_##val: \
1798 return "NETDEV_" __stringify(val);
1799 switch (cmd) {
1800 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1801 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1802 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1803 N(POST_INIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) N(CHANGEUPPER)
1804 N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) N(BONDING_INFO)
1805 N(PRECHANGEUPPER) N(CHANGELOWERSTATE) N(UDP_TUNNEL_PUSH_INFO)
1806 N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
9daae9bd
GP
1807 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1808 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1570415f 1809 N(PRE_CHANGEADDR)
3f5ecd8a 1810 }
ede2762d
KT
1811#undef N
1812 return "UNKNOWN_NETDEV_EVENT";
1813}
1814EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1815
351638e7
JP
1816static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1817 struct net_device *dev)
1818{
51d0c047
DA
1819 struct netdev_notifier_info info = {
1820 .dev = dev,
1821 };
351638e7 1822
351638e7
JP
1823 return nb->notifier_call(nb, val, &info);
1824}
0187bdfb 1825
afa0df59
JP
1826static int call_netdevice_register_notifiers(struct notifier_block *nb,
1827 struct net_device *dev)
1828{
1829 int err;
1830
1831 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1832 err = notifier_to_errno(err);
1833 if (err)
1834 return err;
1835
1836 if (!(dev->flags & IFF_UP))
1837 return 0;
1838
1839 call_netdevice_notifier(nb, NETDEV_UP, dev);
1840 return 0;
1841}
1842
1843static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1844 struct net_device *dev)
1845{
1846 if (dev->flags & IFF_UP) {
1847 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1848 dev);
1849 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1850 }
1851 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1852}
1853
1854static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1855 struct net *net)
1856{
1857 struct net_device *dev;
1858 int err;
1859
1860 for_each_netdev(net, dev) {
1861 err = call_netdevice_register_notifiers(nb, dev);
1862 if (err)
1863 goto rollback;
1864 }
1865 return 0;
1866
1867rollback:
1868 for_each_netdev_continue_reverse(net, dev)
1869 call_netdevice_unregister_notifiers(nb, dev);
1870 return err;
1871}
1872
1873static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
1874 struct net *net)
1875{
1876 struct net_device *dev;
1877
1878 for_each_netdev(net, dev)
1879 call_netdevice_unregister_notifiers(nb, dev);
1880}
1881
881d966b
EB
1882static int dev_boot_phase = 1;
1883
1da177e4 1884/**
722c9a0c 1885 * register_netdevice_notifier - register a network notifier block
1886 * @nb: notifier
1da177e4 1887 *
722c9a0c 1888 * Register a notifier to be called when network device events occur.
1889 * The notifier passed is linked into the kernel structures and must
1890 * not be reused until it has been unregistered. A negative errno code
1891 * is returned on a failure.
1da177e4 1892 *
722c9a0c 1893 * When registered all registration and up events are replayed
1894 * to the new notifier to allow device to have a race free
1895 * view of the network device list.
1da177e4
LT
1896 */
1897
1898int register_netdevice_notifier(struct notifier_block *nb)
1899{
881d966b 1900 struct net *net;
1da177e4
LT
1901 int err;
1902
328fbe74
KT
1903 /* Close race with setup_net() and cleanup_net() */
1904 down_write(&pernet_ops_rwsem);
1da177e4 1905 rtnl_lock();
f07d5b94 1906 err = raw_notifier_chain_register(&netdev_chain, nb);
fcc5a03a
HX
1907 if (err)
1908 goto unlock;
881d966b
EB
1909 if (dev_boot_phase)
1910 goto unlock;
1911 for_each_net(net) {
afa0df59
JP
1912 err = call_netdevice_register_net_notifiers(nb, net);
1913 if (err)
1914 goto rollback;
1da177e4 1915 }
fcc5a03a
HX
1916
1917unlock:
1da177e4 1918 rtnl_unlock();
328fbe74 1919 up_write(&pernet_ops_rwsem);
1da177e4 1920 return err;
fcc5a03a
HX
1921
1922rollback:
afa0df59
JP
1923 for_each_net_continue_reverse(net)
1924 call_netdevice_unregister_net_notifiers(nb, net);
c67625a1
PE
1925
1926 raw_notifier_chain_unregister(&netdev_chain, nb);
fcc5a03a 1927 goto unlock;
1da177e4 1928}
d1b19dff 1929EXPORT_SYMBOL(register_netdevice_notifier);
1da177e4
LT
1930
1931/**
722c9a0c 1932 * unregister_netdevice_notifier - unregister a network notifier block
1933 * @nb: notifier
1da177e4 1934 *
722c9a0c 1935 * Unregister a notifier previously registered by
1936 * register_netdevice_notifier(). The notifier is unlinked into the
1937 * kernel structures and may then be reused. A negative errno code
1938 * is returned on a failure.
7d3d43da 1939 *
722c9a0c 1940 * After unregistering unregister and down device events are synthesized
1941 * for all devices on the device list to the removed notifier to remove
1942 * the need for special case cleanup code.
1da177e4
LT
1943 */
1944
1945int unregister_netdevice_notifier(struct notifier_block *nb)
1946{
7d3d43da 1947 struct net *net;
9f514950
HX
1948 int err;
1949
328fbe74
KT
1950 /* Close race with setup_net() and cleanup_net() */
1951 down_write(&pernet_ops_rwsem);
9f514950 1952 rtnl_lock();
f07d5b94 1953 err = raw_notifier_chain_unregister(&netdev_chain, nb);
7d3d43da
EB
1954 if (err)
1955 goto unlock;
1956
48b3a137
JP
1957 for_each_net(net)
1958 call_netdevice_unregister_net_notifiers(nb, net);
1959
7d3d43da 1960unlock:
9f514950 1961 rtnl_unlock();
328fbe74 1962 up_write(&pernet_ops_rwsem);
9f514950 1963 return err;
1da177e4 1964}
d1b19dff 1965EXPORT_SYMBOL(unregister_netdevice_notifier);
1da177e4 1966
1f637703
JP
1967static int __register_netdevice_notifier_net(struct net *net,
1968 struct notifier_block *nb,
1969 bool ignore_call_fail)
1970{
1971 int err;
1972
1973 err = raw_notifier_chain_register(&net->netdev_chain, nb);
1974 if (err)
1975 return err;
1976 if (dev_boot_phase)
1977 return 0;
1978
1979 err = call_netdevice_register_net_notifiers(nb, net);
1980 if (err && !ignore_call_fail)
1981 goto chain_unregister;
1982
1983 return 0;
1984
1985chain_unregister:
1986 raw_notifier_chain_unregister(&net->netdev_chain, nb);
1987 return err;
1988}
1989
1990static int __unregister_netdevice_notifier_net(struct net *net,
1991 struct notifier_block *nb)
1992{
1993 int err;
1994
1995 err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
1996 if (err)
1997 return err;
1998
1999 call_netdevice_unregister_net_notifiers(nb, net);
2000 return 0;
2001}
2002
a30c7b42
JP
2003/**
2004 * register_netdevice_notifier_net - register a per-netns network notifier block
2005 * @net: network namespace
2006 * @nb: notifier
2007 *
2008 * Register a notifier to be called when network device events occur.
2009 * The notifier passed is linked into the kernel structures and must
2010 * not be reused until it has been unregistered. A negative errno code
2011 * is returned on a failure.
2012 *
2013 * When registered all registration and up events are replayed
2014 * to the new notifier to allow device to have a race free
2015 * view of the network device list.
2016 */
2017
2018int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
2019{
2020 int err;
2021
2022 rtnl_lock();
1f637703 2023 err = __register_netdevice_notifier_net(net, nb, false);
a30c7b42
JP
2024 rtnl_unlock();
2025 return err;
a30c7b42
JP
2026}
2027EXPORT_SYMBOL(register_netdevice_notifier_net);
2028
2029/**
2030 * unregister_netdevice_notifier_net - unregister a per-netns
2031 * network notifier block
2032 * @net: network namespace
2033 * @nb: notifier
2034 *
2035 * Unregister a notifier previously registered by
2036 * register_netdevice_notifier(). The notifier is unlinked into the
2037 * kernel structures and may then be reused. A negative errno code
2038 * is returned on a failure.
2039 *
2040 * After unregistering unregister and down device events are synthesized
2041 * for all devices on the device list to the removed notifier to remove
2042 * the need for special case cleanup code.
2043 */
2044
2045int unregister_netdevice_notifier_net(struct net *net,
2046 struct notifier_block *nb)
2047{
2048 int err;
2049
2050 rtnl_lock();
1f637703 2051 err = __unregister_netdevice_notifier_net(net, nb);
a30c7b42
JP
2052 rtnl_unlock();
2053 return err;
2054}
2055EXPORT_SYMBOL(unregister_netdevice_notifier_net);
a30c7b42 2056
93642e14
JP
2057int register_netdevice_notifier_dev_net(struct net_device *dev,
2058 struct notifier_block *nb,
2059 struct netdev_net_notifier *nn)
2060{
2061 int err;
a30c7b42 2062
93642e14
JP
2063 rtnl_lock();
2064 err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
2065 if (!err) {
2066 nn->nb = nb;
2067 list_add(&nn->list, &dev->net_notifier_list);
2068 }
a30c7b42
JP
2069 rtnl_unlock();
2070 return err;
2071}
93642e14
JP
2072EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
2073
2074int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2075 struct notifier_block *nb,
2076 struct netdev_net_notifier *nn)
2077{
2078 int err;
2079
2080 rtnl_lock();
2081 list_del(&nn->list);
2082 err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
2083 rtnl_unlock();
2084 return err;
2085}
2086EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
2087
2088static void move_netdevice_notifiers_dev_net(struct net_device *dev,
2089 struct net *net)
2090{
2091 struct netdev_net_notifier *nn;
2092
2093 list_for_each_entry(nn, &dev->net_notifier_list, list) {
2094 __unregister_netdevice_notifier_net(dev_net(dev), nn->nb);
2095 __register_netdevice_notifier_net(net, nn->nb, true);
2096 }
2097}
a30c7b42 2098
351638e7
JP
2099/**
2100 * call_netdevice_notifiers_info - call all network notifier blocks
2101 * @val: value passed unmodified to notifier function
351638e7
JP
2102 * @info: notifier information data
2103 *
2104 * Call all network notifier blocks. Parameters and return value
2105 * are as for raw_notifier_call_chain().
2106 */
2107
1d143d9f 2108static int call_netdevice_notifiers_info(unsigned long val,
1d143d9f 2109 struct netdev_notifier_info *info)
351638e7 2110{
a30c7b42
JP
2111 struct net *net = dev_net(info->dev);
2112 int ret;
2113
351638e7 2114 ASSERT_RTNL();
a30c7b42
JP
2115
2116 /* Run per-netns notifier block chain first, then run the global one.
2117 * Hopefully, one day, the global one is going to be removed after
2118 * all notifier block registrators get converted to be per-netns.
2119 */
2120 ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
2121 if (ret & NOTIFY_STOP_MASK)
2122 return ret;
351638e7
JP
2123 return raw_notifier_call_chain(&netdev_chain, val, info);
2124}
351638e7 2125
26372605
PM
2126static int call_netdevice_notifiers_extack(unsigned long val,
2127 struct net_device *dev,
2128 struct netlink_ext_ack *extack)
2129{
2130 struct netdev_notifier_info info = {
2131 .dev = dev,
2132 .extack = extack,
2133 };
2134
2135 return call_netdevice_notifiers_info(val, &info);
2136}
2137
1da177e4
LT
2138/**
2139 * call_netdevice_notifiers - call all network notifier blocks
2140 * @val: value passed unmodified to notifier function
c4ea43c5 2141 * @dev: net_device pointer passed unmodified to notifier function
1da177e4
LT
2142 *
2143 * Call all network notifier blocks. Parameters and return value
f07d5b94 2144 * are as for raw_notifier_call_chain().
1da177e4
LT
2145 */
2146
ad7379d4 2147int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1da177e4 2148{
26372605 2149 return call_netdevice_notifiers_extack(val, dev, NULL);
1da177e4 2150}
edf947f1 2151EXPORT_SYMBOL(call_netdevice_notifiers);
1da177e4 2152
af7d6cce
SD
2153/**
2154 * call_netdevice_notifiers_mtu - call all network notifier blocks
2155 * @val: value passed unmodified to notifier function
2156 * @dev: net_device pointer passed unmodified to notifier function
2157 * @arg: additional u32 argument passed to the notifier function
2158 *
2159 * Call all network notifier blocks. Parameters and return value
2160 * are as for raw_notifier_call_chain().
2161 */
2162static int call_netdevice_notifiers_mtu(unsigned long val,
2163 struct net_device *dev, u32 arg)
2164{
2165 struct netdev_notifier_info_ext info = {
2166 .info.dev = dev,
2167 .ext.mtu = arg,
2168 };
2169
2170 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2171
2172 return call_netdevice_notifiers_info(val, &info.info);
2173}
2174
1cf51900 2175#ifdef CONFIG_NET_INGRESS
aabf6772 2176static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
4577139b
DB
2177
2178void net_inc_ingress_queue(void)
2179{
aabf6772 2180 static_branch_inc(&ingress_needed_key);
4577139b
DB
2181}
2182EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2183
2184void net_dec_ingress_queue(void)
2185{
aabf6772 2186 static_branch_dec(&ingress_needed_key);
4577139b
DB
2187}
2188EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2189#endif
2190
1f211a1b 2191#ifdef CONFIG_NET_EGRESS
aabf6772 2192static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
1f211a1b
DB
2193
2194void net_inc_egress_queue(void)
2195{
aabf6772 2196 static_branch_inc(&egress_needed_key);
1f211a1b
DB
2197}
2198EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2199
2200void net_dec_egress_queue(void)
2201{
aabf6772 2202 static_branch_dec(&egress_needed_key);
1f211a1b
DB
2203}
2204EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2205#endif
2206
39e83922 2207static DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
e9666d10 2208#ifdef CONFIG_JUMP_LABEL
b90e5794 2209static atomic_t netstamp_needed_deferred;
13baa00a 2210static atomic_t netstamp_wanted;
5fa8bbda 2211static void netstamp_clear(struct work_struct *work)
1da177e4 2212{
b90e5794 2213 int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
13baa00a 2214 int wanted;
b90e5794 2215
13baa00a
ED
2216 wanted = atomic_add_return(deferred, &netstamp_wanted);
2217 if (wanted > 0)
39e83922 2218 static_branch_enable(&netstamp_needed_key);
13baa00a 2219 else
39e83922 2220 static_branch_disable(&netstamp_needed_key);
5fa8bbda
ED
2221}
2222static DECLARE_WORK(netstamp_work, netstamp_clear);
b90e5794 2223#endif
5fa8bbda
ED
2224
2225void net_enable_timestamp(void)
2226{
e9666d10 2227#ifdef CONFIG_JUMP_LABEL
13baa00a
ED
2228 int wanted;
2229
2230 while (1) {
2231 wanted = atomic_read(&netstamp_wanted);
2232 if (wanted <= 0)
2233 break;
2234 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted + 1) == wanted)
2235 return;
2236 }
2237 atomic_inc(&netstamp_needed_deferred);
2238 schedule_work(&netstamp_work);
2239#else
39e83922 2240 static_branch_inc(&netstamp_needed_key);
13baa00a 2241#endif
1da177e4 2242}
d1b19dff 2243EXPORT_SYMBOL(net_enable_timestamp);
1da177e4
LT
2244
2245void net_disable_timestamp(void)
2246{
e9666d10 2247#ifdef CONFIG_JUMP_LABEL
13baa00a
ED
2248 int wanted;
2249
2250 while (1) {
2251 wanted = atomic_read(&netstamp_wanted);
2252 if (wanted <= 1)
2253 break;
2254 if (atomic_cmpxchg(&netstamp_wanted, wanted, wanted - 1) == wanted)
2255 return;
2256 }
2257 atomic_dec(&netstamp_needed_deferred);
5fa8bbda
ED
2258 schedule_work(&netstamp_work);
2259#else
39e83922 2260 static_branch_dec(&netstamp_needed_key);
5fa8bbda 2261#endif
1da177e4 2262}
d1b19dff 2263EXPORT_SYMBOL(net_disable_timestamp);
1da177e4 2264
3b098e2d 2265static inline void net_timestamp_set(struct sk_buff *skb)
1da177e4 2266{
2456e855 2267 skb->tstamp = 0;
39e83922 2268 if (static_branch_unlikely(&netstamp_needed_key))
a61bbcf2 2269 __net_timestamp(skb);
1da177e4
LT
2270}
2271
39e83922
DB
2272#define net_timestamp_check(COND, SKB) \
2273 if (static_branch_unlikely(&netstamp_needed_key)) { \
2274 if ((COND) && !(SKB)->tstamp) \
2275 __net_timestamp(SKB); \
2276 } \
3b098e2d 2277
f4b05d27 2278bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
79b569f0 2279{
5f7d5728 2280 return __is_skb_forwardable(dev, skb, true);
79b569f0 2281}
1ee481fb 2282EXPORT_SYMBOL_GPL(is_skb_forwardable);
79b569f0 2283
5f7d5728
JDB
2284static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2285 bool check_mtu)
a0265d28 2286{
5f7d5728 2287 int ret = ____dev_forward_skb(dev, skb, check_mtu);
a0265d28 2288
4e3264d2
MKL
2289 if (likely(!ret)) {
2290 skb->protocol = eth_type_trans(skb, dev);
2291 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2292 }
a0265d28 2293
4e3264d2 2294 return ret;
a0265d28 2295}
5f7d5728
JDB
2296
2297int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2298{
2299 return __dev_forward_skb2(dev, skb, true);
2300}
a0265d28
HX
2301EXPORT_SYMBOL_GPL(__dev_forward_skb);
2302
44540960
AB
2303/**
2304 * dev_forward_skb - loopback an skb to another netif
2305 *
2306 * @dev: destination network device
2307 * @skb: buffer to forward
2308 *
2309 * return values:
2310 * NET_RX_SUCCESS (no congestion)
6ec82562 2311 * NET_RX_DROP (packet was dropped, but freed)
44540960
AB
2312 *
2313 * dev_forward_skb can be used for injecting an skb from the
2314 * start_xmit function of one device into the receive queue
2315 * of another device.
2316 *
2317 * The receiving device may be in another namespace, so
2318 * we have to clear all information in the skb that could
2319 * impact namespace isolation.
2320 */
2321int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2322{
a0265d28 2323 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
44540960
AB
2324}
2325EXPORT_SYMBOL_GPL(dev_forward_skb);
2326
5f7d5728
JDB
2327int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2328{
2329 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2330}
2331
71d9dec2
CG
2332static inline int deliver_skb(struct sk_buff *skb,
2333 struct packet_type *pt_prev,
2334 struct net_device *orig_dev)
2335{
1f8b977a 2336 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
1080e512 2337 return -ENOMEM;
63354797 2338 refcount_inc(&skb->users);
71d9dec2
CG
2339 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2340}
2341
7866a621
SN
2342static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2343 struct packet_type **pt,
fbcb2170
JP
2344 struct net_device *orig_dev,
2345 __be16 type,
7866a621
SN
2346 struct list_head *ptype_list)
2347{
2348 struct packet_type *ptype, *pt_prev = *pt;
2349
2350 list_for_each_entry_rcu(ptype, ptype_list, list) {
2351 if (ptype->type != type)
2352 continue;
2353 if (pt_prev)
fbcb2170 2354 deliver_skb(skb, pt_prev, orig_dev);
7866a621
SN
2355 pt_prev = ptype;
2356 }
2357 *pt = pt_prev;
2358}
2359
c0de08d0
EL
2360static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2361{
a3d744e9 2362 if (!ptype->af_packet_priv || !skb->sk)
c0de08d0
EL
2363 return false;
2364
2365 if (ptype->id_match)
2366 return ptype->id_match(ptype, skb->sk);
2367 else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2368 return true;
2369
2370 return false;
2371}
2372
9f9a742d
MR
2373/**
2374 * dev_nit_active - return true if any network interface taps are in use
2375 *
2376 * @dev: network device to check for the presence of taps
2377 */
2378bool dev_nit_active(struct net_device *dev)
2379{
2380 return !list_empty(&ptype_all) || !list_empty(&dev->ptype_all);
2381}
2382EXPORT_SYMBOL_GPL(dev_nit_active);
2383
1da177e4
LT
2384/*
2385 * Support routine. Sends outgoing frames to any network
2386 * taps currently in use.
2387 */
2388
74b20582 2389void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1da177e4
LT
2390{
2391 struct packet_type *ptype;
71d9dec2
CG
2392 struct sk_buff *skb2 = NULL;
2393 struct packet_type *pt_prev = NULL;
7866a621 2394 struct list_head *ptype_list = &ptype_all;
a61bbcf2 2395
1da177e4 2396 rcu_read_lock();
7866a621
SN
2397again:
2398 list_for_each_entry_rcu(ptype, ptype_list, list) {
fa788d98
VW
2399 if (ptype->ignore_outgoing)
2400 continue;
2401
1da177e4
LT
2402 /* Never send packets back to the socket
2403 * they originated from - MvS (miquels@drinkel.ow.org)
2404 */
7866a621
SN
2405 if (skb_loop_sk(ptype, skb))
2406 continue;
71d9dec2 2407
7866a621
SN
2408 if (pt_prev) {
2409 deliver_skb(skb2, pt_prev, skb->dev);
2410 pt_prev = ptype;
2411 continue;
2412 }
1da177e4 2413
7866a621
SN
2414 /* need to clone skb, done only once */
2415 skb2 = skb_clone(skb, GFP_ATOMIC);
2416 if (!skb2)
2417 goto out_unlock;
70978182 2418
7866a621 2419 net_timestamp_set(skb2);
1da177e4 2420
7866a621
SN
2421 /* skb->nh should be correctly
2422 * set by sender, so that the second statement is
2423 * just protection against buggy protocols.
2424 */
2425 skb_reset_mac_header(skb2);
2426
2427 if (skb_network_header(skb2) < skb2->data ||
2428 skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2429 net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2430 ntohs(skb2->protocol),
2431 dev->name);
2432 skb_reset_network_header(skb2);
1da177e4 2433 }
7866a621
SN
2434
2435 skb2->transport_header = skb2->network_header;
2436 skb2->pkt_type = PACKET_OUTGOING;
2437 pt_prev = ptype;
2438 }
2439
2440 if (ptype_list == &ptype_all) {
2441 ptype_list = &dev->ptype_all;
2442 goto again;
1da177e4 2443 }
7866a621 2444out_unlock:
581fe0ea
WB
2445 if (pt_prev) {
2446 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2447 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2448 else
2449 kfree_skb(skb2);
2450 }
1da177e4
LT
2451 rcu_read_unlock();
2452}
74b20582 2453EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
1da177e4 2454
2c53040f
BH
2455/**
2456 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
4f57c087
JF
2457 * @dev: Network device
2458 * @txq: number of queues available
2459 *
2460 * If real_num_tx_queues is changed the tc mappings may no longer be
2461 * valid. To resolve this verify the tc mapping remains valid and if
2462 * not NULL the mapping. With no priorities mapping to this
2463 * offset/count pair it will no longer be used. In the worst case TC0
2464 * is invalid nothing can be done so disable priority mappings. If is
2465 * expected that drivers will fix this mapping if they can before
2466 * calling netif_set_real_num_tx_queues.
2467 */
bb134d22 2468static void netif_setup_tc(struct net_device *dev, unsigned int txq)
4f57c087
JF
2469{
2470 int i;
2471 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2472
2473 /* If TC0 is invalidated disable TC mapping */
2474 if (tc->offset + tc->count > txq) {
7b6cd1ce 2475 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
4f57c087
JF
2476 dev->num_tc = 0;
2477 return;
2478 }
2479
2480 /* Invalidated prio to tc mappings set to TC0 */
2481 for (i = 1; i < TC_BITMASK + 1; i++) {
2482 int q = netdev_get_prio_tc_map(dev, i);
2483
2484 tc = &dev->tc_to_txq[q];
2485 if (tc->offset + tc->count > txq) {
7b6cd1ce
JP
2486 pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2487 i, q);
4f57c087
JF
2488 netdev_set_prio_tc_map(dev, i, 0);
2489 }
2490 }
2491}
2492
8d059b0f
AD
2493int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2494{
2495 if (dev->num_tc) {
2496 struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2497 int i;
2498
ffcfe25b 2499 /* walk through the TCs and see if it falls into any of them */
8d059b0f
AD
2500 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2501 if ((txq - tc->offset) < tc->count)
2502 return i;
2503 }
2504
ffcfe25b 2505 /* didn't find it, just return -1 to indicate no match */
8d059b0f
AD
2506 return -1;
2507 }
2508
2509 return 0;
2510}
8a5f2166 2511EXPORT_SYMBOL(netdev_txq_to_tc);
8d059b0f 2512
537c00de 2513#ifdef CONFIG_XPS
5da9ace3
VO
2514static struct static_key xps_needed __read_mostly;
2515static struct static_key xps_rxqs_needed __read_mostly;
537c00de
AD
2516static DEFINE_MUTEX(xps_map_mutex);
2517#define xmap_dereference(P) \
2518 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2519
6234f874 2520static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2d05bf01 2521 struct xps_dev_maps *old_maps, int tci, u16 index)
537c00de 2522{
10cdc3f3
AD
2523 struct xps_map *map = NULL;
2524 int pos;
537c00de 2525
10cdc3f3 2526 if (dev_maps)
80d19669 2527 map = xmap_dereference(dev_maps->attr_map[tci]);
6234f874
AD
2528 if (!map)
2529 return false;
537c00de 2530
6234f874
AD
2531 for (pos = map->len; pos--;) {
2532 if (map->queues[pos] != index)
2533 continue;
2534
2535 if (map->len > 1) {
2536 map->queues[pos] = map->queues[--map->len];
10cdc3f3 2537 break;
537c00de 2538 }
6234f874 2539
2d05bf01
AT
2540 if (old_maps)
2541 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
80d19669 2542 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
6234f874
AD
2543 kfree_rcu(map, rcu);
2544 return false;
537c00de
AD
2545 }
2546
6234f874 2547 return true;
10cdc3f3
AD
2548}
2549
6234f874
AD
2550static bool remove_xps_queue_cpu(struct net_device *dev,
2551 struct xps_dev_maps *dev_maps,
2552 int cpu, u16 offset, u16 count)
2553{
255c04a8 2554 int num_tc = dev_maps->num_tc;
184c449f
AD
2555 bool active = false;
2556 int tci;
6234f874 2557
184c449f
AD
2558 for (tci = cpu * num_tc; num_tc--; tci++) {
2559 int i, j;
2560
2561 for (i = count, j = offset; i--; j++) {
2d05bf01 2562 if (!remove_xps_queue(dev_maps, NULL, tci, j))
184c449f
AD
2563 break;
2564 }
2565
2566 active |= i < 0;
6234f874
AD
2567 }
2568
184c449f 2569 return active;
6234f874
AD
2570}
2571
867d0ad4
SD
2572static void reset_xps_maps(struct net_device *dev,
2573 struct xps_dev_maps *dev_maps,
044ab86d 2574 enum xps_map_type type)
867d0ad4 2575{
867d0ad4 2576 static_key_slow_dec_cpuslocked(&xps_needed);
044ab86d
AT
2577 if (type == XPS_RXQS)
2578 static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2579
2580 RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2581
867d0ad4
SD
2582 kfree_rcu(dev_maps, rcu);
2583}
2584
044ab86d
AT
2585static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2586 u16 offset, u16 count)
80d19669 2587{
044ab86d 2588 struct xps_dev_maps *dev_maps;
80d19669
AN
2589 bool active = false;
2590 int i, j;
2591
044ab86d
AT
2592 dev_maps = xmap_dereference(dev->xps_maps[type]);
2593 if (!dev_maps)
2594 return;
2595
6f36158e
AT
2596 for (j = 0; j < dev_maps->nr_ids; j++)
2597 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
867d0ad4 2598 if (!active)
044ab86d 2599 reset_xps_maps(dev, dev_maps, type);
80d19669 2600
044ab86d 2601 if (type == XPS_CPUS) {
6f36158e 2602 for (i = offset + (count - 1); count--; i--)
f28c020f 2603 netdev_queue_numa_node_write(
6f36158e 2604 netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
80d19669
AN
2605 }
2606}
2607
6234f874
AD
2608static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2609 u16 count)
10cdc3f3 2610{
04157469
AN
2611 if (!static_key_false(&xps_needed))
2612 return;
10cdc3f3 2613
4d99f660 2614 cpus_read_lock();
04157469 2615 mutex_lock(&xps_map_mutex);
10cdc3f3 2616
044ab86d
AT
2617 if (static_key_false(&xps_rxqs_needed))
2618 clean_xps_maps(dev, XPS_RXQS, offset, count);
80d19669 2619
044ab86d 2620 clean_xps_maps(dev, XPS_CPUS, offset, count);
024e9679 2621
537c00de 2622 mutex_unlock(&xps_map_mutex);
4d99f660 2623 cpus_read_unlock();
537c00de
AD
2624}
2625
6234f874
AD
2626static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2627{
2628 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2629}
2630
80d19669
AN
2631static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2632 u16 index, bool is_rxqs_map)
01c5f864
AD
2633{
2634 struct xps_map *new_map;
2635 int alloc_len = XPS_MIN_MAP_ALLOC;
2636 int i, pos;
2637
2638 for (pos = 0; map && pos < map->len; pos++) {
2639 if (map->queues[pos] != index)
2640 continue;
2641 return map;
2642 }
2643
80d19669 2644 /* Need to add tx-queue to this CPU's/rx-queue's existing map */
01c5f864
AD
2645 if (map) {
2646 if (pos < map->alloc_len)
2647 return map;
2648
2649 alloc_len = map->alloc_len * 2;
2650 }
2651
80d19669
AN
2652 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2653 * map
2654 */
2655 if (is_rxqs_map)
2656 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2657 else
2658 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2659 cpu_to_node(attr_index));
01c5f864
AD
2660 if (!new_map)
2661 return NULL;
2662
2663 for (i = 0; i < pos; i++)
2664 new_map->queues[i] = map->queues[i];
2665 new_map->alloc_len = alloc_len;
2666 new_map->len = pos;
2667
2668 return new_map;
2669}
2670
402fbb99
AT
2671/* Copy xps maps at a given index */
2672static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2673 struct xps_dev_maps *new_dev_maps, int index,
2674 int tc, bool skip_tc)
2675{
2676 int i, tci = index * dev_maps->num_tc;
2677 struct xps_map *map;
2678
2679 /* copy maps belonging to foreign traffic classes */
2680 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2681 if (i == tc && skip_tc)
2682 continue;
2683
2684 /* fill in the new device map from the old device map */
2685 map = xmap_dereference(dev_maps->attr_map[tci]);
2686 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2687 }
2688}
2689
4d99f660 2690/* Must be called under cpus_read_lock */
80d19669 2691int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
044ab86d 2692 u16 index, enum xps_map_type type)
537c00de 2693{
2d05bf01 2694 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
6f36158e 2695 const unsigned long *online_mask = NULL;
255c04a8 2696 bool active = false, copy = false;
80d19669 2697 int i, j, tci, numa_node_id = -2;
184c449f 2698 int maps_sz, num_tc = 1, tc = 0;
537c00de 2699 struct xps_map *map, *new_map;
80d19669 2700 unsigned int nr_ids;
537c00de 2701
184c449f 2702 if (dev->num_tc) {
ffcfe25b 2703 /* Do not allow XPS on subordinate device directly */
184c449f 2704 num_tc = dev->num_tc;
ffcfe25b
AD
2705 if (num_tc < 0)
2706 return -EINVAL;
2707
2708 /* If queue belongs to subordinate dev use its map */
2709 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2710
184c449f
AD
2711 tc = netdev_txq_to_tc(dev, index);
2712 if (tc < 0)
2713 return -EINVAL;
2714 }
2715
537c00de 2716 mutex_lock(&xps_map_mutex);
044ab86d
AT
2717
2718 dev_maps = xmap_dereference(dev->xps_maps[type]);
2719 if (type == XPS_RXQS) {
80d19669 2720 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
80d19669
AN
2721 nr_ids = dev->num_rx_queues;
2722 } else {
2723 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
6f36158e 2724 if (num_possible_cpus() > 1)
80d19669 2725 online_mask = cpumask_bits(cpu_online_mask);
80d19669
AN
2726 nr_ids = nr_cpu_ids;
2727 }
537c00de 2728
80d19669
AN
2729 if (maps_sz < L1_CACHE_BYTES)
2730 maps_sz = L1_CACHE_BYTES;
537c00de 2731
255c04a8 2732 /* The old dev_maps could be larger or smaller than the one we're
5478fcd0
AT
2733 * setting up now, as dev->num_tc or nr_ids could have been updated in
2734 * between. We could try to be smart, but let's be safe instead and only
2735 * copy foreign traffic classes if the two map sizes match.
255c04a8 2736 */
5478fcd0
AT
2737 if (dev_maps &&
2738 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
255c04a8
AT
2739 copy = true;
2740
01c5f864 2741 /* allocate memory for queue storage */
80d19669
AN
2742 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2743 j < nr_ids;) {
2bb60cb9 2744 if (!new_dev_maps) {
255c04a8
AT
2745 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2746 if (!new_dev_maps) {
2747 mutex_unlock(&xps_map_mutex);
2748 return -ENOMEM;
2749 }
2750
5478fcd0 2751 new_dev_maps->nr_ids = nr_ids;
255c04a8 2752 new_dev_maps->num_tc = num_tc;
2bb60cb9 2753 }
01c5f864 2754
80d19669 2755 tci = j * num_tc + tc;
255c04a8 2756 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
01c5f864 2757
044ab86d 2758 map = expand_xps_map(map, j, index, type == XPS_RXQS);
01c5f864
AD
2759 if (!map)
2760 goto error;
2761
80d19669 2762 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
01c5f864
AD
2763 }
2764
2765 if (!new_dev_maps)
2766 goto out_no_new_maps;
2767
867d0ad4
SD
2768 if (!dev_maps) {
2769 /* Increment static keys at most once per type */
2770 static_key_slow_inc_cpuslocked(&xps_needed);
044ab86d 2771 if (type == XPS_RXQS)
867d0ad4
SD
2772 static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
2773 }
04157469 2774
6f36158e 2775 for (j = 0; j < nr_ids; j++) {
402fbb99 2776 bool skip_tc = false;
184c449f 2777
80d19669 2778 tci = j * num_tc + tc;
80d19669
AN
2779 if (netif_attr_test_mask(j, mask, nr_ids) &&
2780 netif_attr_test_online(j, online_mask, nr_ids)) {
2781 /* add tx-queue to CPU/rx-queue maps */
01c5f864
AD
2782 int pos = 0;
2783
402fbb99
AT
2784 skip_tc = true;
2785
80d19669 2786 map = xmap_dereference(new_dev_maps->attr_map[tci]);
01c5f864
AD
2787 while ((pos < map->len) && (map->queues[pos] != index))
2788 pos++;
2789
2790 if (pos == map->len)
2791 map->queues[map->len++] = index;
537c00de 2792#ifdef CONFIG_NUMA
044ab86d 2793 if (type == XPS_CPUS) {
80d19669
AN
2794 if (numa_node_id == -2)
2795 numa_node_id = cpu_to_node(j);
2796 else if (numa_node_id != cpu_to_node(j))
2797 numa_node_id = -1;
2798 }
537c00de 2799#endif
537c00de 2800 }
01c5f864 2801
402fbb99
AT
2802 if (copy)
2803 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
2804 skip_tc);
537c00de
AD
2805 }
2806
044ab86d 2807 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
01c5f864 2808
537c00de 2809 /* Cleanup old maps */
184c449f
AD
2810 if (!dev_maps)
2811 goto out_no_old_maps;
2812
6f36158e 2813 for (j = 0; j < dev_maps->nr_ids; j++) {
255c04a8 2814 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
80d19669 2815 map = xmap_dereference(dev_maps->attr_map[tci]);
255c04a8
AT
2816 if (!map)
2817 continue;
2818
2819 if (copy) {
2820 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
2821 if (map == new_map)
2822 continue;
2823 }
2824
75b2758a 2825 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
255c04a8 2826 kfree_rcu(map, rcu);
01c5f864 2827 }
537c00de
AD
2828 }
2829
2d05bf01 2830 old_dev_maps = dev_maps;
184c449f
AD
2831
2832out_no_old_maps:
01c5f864
AD
2833 dev_maps = new_dev_maps;
2834 active = true;
537c00de 2835
01c5f864 2836out_no_new_maps:
044ab86d 2837 if (type == XPS_CPUS)
80d19669
AN
2838 /* update Tx queue numa node */
2839 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2840 (numa_node_id >= 0) ?
2841 numa_node_id : NUMA_NO_NODE);
537c00de 2842
01c5f864
AD
2843 if (!dev_maps)
2844 goto out_no_maps;
2845
80d19669 2846 /* removes tx-queue from unused CPUs/rx-queues */
6f36158e 2847 for (j = 0; j < dev_maps->nr_ids; j++) {
132f743b
AT
2848 tci = j * dev_maps->num_tc;
2849
2850 for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2851 if (i == tc &&
2852 netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
2853 netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
2854 continue;
2855
2d05bf01
AT
2856 active |= remove_xps_queue(dev_maps,
2857 copy ? old_dev_maps : NULL,
2858 tci, index);
132f743b 2859 }
01c5f864
AD
2860 }
2861
2d05bf01
AT
2862 if (old_dev_maps)
2863 kfree_rcu(old_dev_maps, rcu);
2864
01c5f864 2865 /* free map if not active */
867d0ad4 2866 if (!active)
044ab86d 2867 reset_xps_maps(dev, dev_maps, type);
01c5f864
AD
2868
2869out_no_maps:
537c00de
AD
2870 mutex_unlock(&xps_map_mutex);
2871
2872 return 0;
2873error:
01c5f864 2874 /* remove any maps that we added */
6f36158e 2875 for (j = 0; j < nr_ids; j++) {
80d19669
AN
2876 for (i = num_tc, tci = j * num_tc; i--; tci++) {
2877 new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
255c04a8 2878 map = copy ?
80d19669 2879 xmap_dereference(dev_maps->attr_map[tci]) :
184c449f
AD
2880 NULL;
2881 if (new_map && new_map != map)
2882 kfree(new_map);
2883 }
01c5f864
AD
2884 }
2885
537c00de
AD
2886 mutex_unlock(&xps_map_mutex);
2887
537c00de
AD
2888 kfree(new_dev_maps);
2889 return -ENOMEM;
2890}
4d99f660 2891EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
80d19669
AN
2892
2893int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2894 u16 index)
2895{
4d99f660
AV
2896 int ret;
2897
2898 cpus_read_lock();
044ab86d 2899 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
4d99f660
AV
2900 cpus_read_unlock();
2901
2902 return ret;
80d19669 2903}
537c00de
AD
2904EXPORT_SYMBOL(netif_set_xps_queue);
2905
2906#endif
ffcfe25b
AD
2907static void netdev_unbind_all_sb_channels(struct net_device *dev)
2908{
2909 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2910
2911 /* Unbind any subordinate channels */
2912 while (txq-- != &dev->_tx[0]) {
2913 if (txq->sb_dev)
2914 netdev_unbind_sb_channel(dev, txq->sb_dev);
2915 }
2916}
2917
9cf1f6a8
AD
2918void netdev_reset_tc(struct net_device *dev)
2919{
6234f874
AD
2920#ifdef CONFIG_XPS
2921 netif_reset_xps_queues_gt(dev, 0);
2922#endif
ffcfe25b
AD
2923 netdev_unbind_all_sb_channels(dev);
2924
2925 /* Reset TC configuration of device */
9cf1f6a8
AD
2926 dev->num_tc = 0;
2927 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq));
2928 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map));
2929}
2930EXPORT_SYMBOL(netdev_reset_tc);
2931
2932int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
2933{
2934 if (tc >= dev->num_tc)
2935 return -EINVAL;
2936
6234f874
AD
2937#ifdef CONFIG_XPS
2938 netif_reset_xps_queues(dev, offset, count);
2939#endif
9cf1f6a8
AD
2940 dev->tc_to_txq[tc].count = count;
2941 dev->tc_to_txq[tc].offset = offset;
2942 return 0;
2943}
2944EXPORT_SYMBOL(netdev_set_tc_queue);
2945
2946int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
2947{
2948 if (num_tc > TC_MAX_QUEUE)
2949 return -EINVAL;
2950
6234f874
AD
2951#ifdef CONFIG_XPS
2952 netif_reset_xps_queues_gt(dev, 0);
2953#endif
ffcfe25b
AD
2954 netdev_unbind_all_sb_channels(dev);
2955
9cf1f6a8
AD
2956 dev->num_tc = num_tc;
2957 return 0;
2958}
2959EXPORT_SYMBOL(netdev_set_num_tc);
2960
ffcfe25b
AD
2961void netdev_unbind_sb_channel(struct net_device *dev,
2962 struct net_device *sb_dev)
2963{
2964 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
2965
2966#ifdef CONFIG_XPS
2967 netif_reset_xps_queues_gt(sb_dev, 0);
2968#endif
2969 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq));
2970 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map));
2971
2972 while (txq-- != &dev->_tx[0]) {
2973 if (txq->sb_dev == sb_dev)
2974 txq->sb_dev = NULL;
2975 }
2976}
2977EXPORT_SYMBOL(netdev_unbind_sb_channel);
2978
2979int netdev_bind_sb_channel_queue(struct net_device *dev,
2980 struct net_device *sb_dev,
2981 u8 tc, u16 count, u16 offset)
2982{
2983 /* Make certain the sb_dev and dev are already configured */
2984 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc)
2985 return -EINVAL;
2986
2987 /* We cannot hand out queues we don't have */
2988 if ((offset + count) > dev->real_num_tx_queues)
2989 return -EINVAL;
2990
2991 /* Record the mapping */
2992 sb_dev->tc_to_txq[tc].count = count;
2993 sb_dev->tc_to_txq[tc].offset = offset;
2994
2995 /* Provide a way for Tx queue to find the tc_to_txq map or
2996 * XPS map for itself.
2997 */
2998 while (count--)
2999 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
3000
3001 return 0;
3002}
3003EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
3004
3005int netdev_set_sb_channel(struct net_device *dev, u16 channel)
3006{
3007 /* Do not use a multiqueue device to represent a subordinate channel */
3008 if (netif_is_multiqueue(dev))
3009 return -ENODEV;
3010
3011 /* We allow channels 1 - 32767 to be used for subordinate channels.
3012 * Channel 0 is meant to be "native" mode and used only to represent
3013 * the main root device. We allow writing 0 to reset the device back
3014 * to normal mode after being used as a subordinate channel.
3015 */
3016 if (channel > S16_MAX)
3017 return -EINVAL;
3018
3019 dev->num_tc = -channel;
3020
3021 return 0;
3022}
3023EXPORT_SYMBOL(netdev_set_sb_channel);
3024
f0796d5c
JF
3025/*
3026 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
3a053b1a 3027 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
f0796d5c 3028 */
e6484930 3029int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
f0796d5c 3030{
ac5b7019 3031 bool disabling;
1d24eb48
TH
3032 int rc;
3033
ac5b7019
JK
3034 disabling = txq < dev->real_num_tx_queues;
3035
e6484930
TH
3036 if (txq < 1 || txq > dev->num_tx_queues)
3037 return -EINVAL;
f0796d5c 3038
5c56580b
BH
3039 if (dev->reg_state == NETREG_REGISTERED ||
3040 dev->reg_state == NETREG_UNREGISTERING) {
e6484930
TH
3041 ASSERT_RTNL();
3042
1d24eb48
TH
3043 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
3044 txq);
bf264145
TH
3045 if (rc)
3046 return rc;
3047
4f57c087
JF
3048 if (dev->num_tc)
3049 netif_setup_tc(dev, txq);
3050
ac5b7019
JK
3051 dev->real_num_tx_queues = txq;
3052
3053 if (disabling) {
3054 synchronize_net();
e6484930 3055 qdisc_reset_all_tx_gt(dev, txq);
024e9679
AD
3056#ifdef CONFIG_XPS
3057 netif_reset_xps_queues_gt(dev, txq);
3058#endif
3059 }
ac5b7019
JK
3060 } else {
3061 dev->real_num_tx_queues = txq;
f0796d5c 3062 }
e6484930 3063
e6484930 3064 return 0;
f0796d5c
JF
3065}
3066EXPORT_SYMBOL(netif_set_real_num_tx_queues);
56079431 3067
a953be53 3068#ifdef CONFIG_SYSFS
62fe0b40
BH
3069/**
3070 * netif_set_real_num_rx_queues - set actual number of RX queues used
3071 * @dev: Network device
3072 * @rxq: Actual number of RX queues
3073 *
3074 * This must be called either with the rtnl_lock held or before
3075 * registration of the net device. Returns 0 on success, or a
4e7f7951
BH
3076 * negative error code. If called before registration, it always
3077 * succeeds.
62fe0b40
BH
3078 */
3079int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
3080{
3081 int rc;
3082
bd25fa7b
TH
3083 if (rxq < 1 || rxq > dev->num_rx_queues)
3084 return -EINVAL;
3085
62fe0b40
BH
3086 if (dev->reg_state == NETREG_REGISTERED) {
3087 ASSERT_RTNL();
3088
62fe0b40
BH
3089 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
3090 rxq);
3091 if (rc)
3092 return rc;
62fe0b40
BH
3093 }
3094
3095 dev->real_num_rx_queues = rxq;
3096 return 0;
3097}
3098EXPORT_SYMBOL(netif_set_real_num_rx_queues);
3099#endif
3100
2c53040f
BH
3101/**
3102 * netif_get_num_default_rss_queues - default number of RSS queues
16917b87
YM
3103 *
3104 * This routine should set an upper limit on the number of RSS queues
3105 * used by default by multiqueue devices.
3106 */
a55b138b 3107int netif_get_num_default_rss_queues(void)
16917b87 3108{
40e4e713
HS
3109 return is_kdump_kernel() ?
3110 1 : min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
16917b87
YM
3111}
3112EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3113
3bcb846c 3114static void __netif_reschedule(struct Qdisc *q)
56079431 3115{
def82a1d
JP
3116 struct softnet_data *sd;
3117 unsigned long flags;
56079431 3118
def82a1d 3119 local_irq_save(flags);
903ceff7 3120 sd = this_cpu_ptr(&softnet_data);
a9cbd588
CG
3121 q->next_sched = NULL;
3122 *sd->output_queue_tailp = q;
3123 sd->output_queue_tailp = &q->next_sched;
def82a1d
JP
3124 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3125 local_irq_restore(flags);
3126}
3127
3128void __netif_schedule(struct Qdisc *q)
3129{
3130 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3131 __netif_reschedule(q);
56079431
DV
3132}
3133EXPORT_SYMBOL(__netif_schedule);
3134
e6247027
ED
3135struct dev_kfree_skb_cb {
3136 enum skb_free_reason reason;
3137};
3138
3139static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
56079431 3140{
e6247027
ED
3141 return (struct dev_kfree_skb_cb *)skb->cb;
3142}
3143
46e5da40
JF
3144void netif_schedule_queue(struct netdev_queue *txq)
3145{
3146 rcu_read_lock();
5be5515a 3147 if (!netif_xmit_stopped(txq)) {
46e5da40
JF
3148 struct Qdisc *q = rcu_dereference(txq->qdisc);
3149
3150 __netif_schedule(q);
3151 }
3152 rcu_read_unlock();
3153}
3154EXPORT_SYMBOL(netif_schedule_queue);
3155
46e5da40
JF
3156void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3157{
3158 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3159 struct Qdisc *q;
3160
3161 rcu_read_lock();
3162 q = rcu_dereference(dev_queue->qdisc);
3163 __netif_schedule(q);
3164 rcu_read_unlock();
3165 }
3166}
3167EXPORT_SYMBOL(netif_tx_wake_queue);
3168
e6247027 3169void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
56079431 3170{
e6247027 3171 unsigned long flags;
56079431 3172
9899886d
MJ
3173 if (unlikely(!skb))
3174 return;
3175
63354797 3176 if (likely(refcount_read(&skb->users) == 1)) {
e6247027 3177 smp_rmb();
63354797
RE
3178 refcount_set(&skb->users, 0);
3179 } else if (likely(!refcount_dec_and_test(&skb->users))) {
e6247027 3180 return;
bea3348e 3181 }
e6247027
ED
3182 get_kfree_skb_cb(skb)->reason = reason;
3183 local_irq_save(flags);
3184 skb->next = __this_cpu_read(softnet_data.completion_queue);
3185 __this_cpu_write(softnet_data.completion_queue, skb);
3186 raise_softirq_irqoff(NET_TX_SOFTIRQ);
3187 local_irq_restore(flags);
56079431 3188}
e6247027 3189EXPORT_SYMBOL(__dev_kfree_skb_irq);
56079431 3190
e6247027 3191void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
56079431
DV
3192{
3193 if (in_irq() || irqs_disabled())
e6247027 3194 __dev_kfree_skb_irq(skb, reason);
56079431
DV
3195 else
3196 dev_kfree_skb(skb);
3197}
e6247027 3198EXPORT_SYMBOL(__dev_kfree_skb_any);
56079431
DV
3199
3200
bea3348e
SH
3201/**
3202 * netif_device_detach - mark device as removed
3203 * @dev: network device
3204 *
3205 * Mark device as removed from system and therefore no longer available.
3206 */
56079431
DV
3207void netif_device_detach(struct net_device *dev)
3208{
3209 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3210 netif_running(dev)) {
d543103a 3211 netif_tx_stop_all_queues(dev);
56079431
DV
3212 }
3213}
3214EXPORT_SYMBOL(netif_device_detach);
3215
bea3348e
SH
3216/**
3217 * netif_device_attach - mark device as attached
3218 * @dev: network device
3219 *
3220 * Mark device as attached from system and restart if needed.
3221 */
56079431
DV
3222void netif_device_attach(struct net_device *dev)
3223{
3224 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3225 netif_running(dev)) {
d543103a 3226 netif_tx_wake_all_queues(dev);
4ec93edb 3227 __netdev_watchdog_up(dev);
56079431
DV
3228 }
3229}
3230EXPORT_SYMBOL(netif_device_attach);
3231
5605c762
JP
3232/*
3233 * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3234 * to be used as a distribution range.
3235 */
eadec877
AD
3236static u16 skb_tx_hash(const struct net_device *dev,
3237 const struct net_device *sb_dev,
3238 struct sk_buff *skb)
5605c762
JP
3239{
3240 u32 hash;
3241 u16 qoffset = 0;
1b837d48 3242 u16 qcount = dev->real_num_tx_queues;
5605c762 3243
eadec877
AD
3244 if (dev->num_tc) {
3245 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3246
3247 qoffset = sb_dev->tc_to_txq[tc].offset;
3248 qcount = sb_dev->tc_to_txq[tc].count;
3249 }
3250
5605c762
JP
3251 if (skb_rx_queue_recorded(skb)) {
3252 hash = skb_get_rx_queue(skb);
6e11d157
AN
3253 if (hash >= qoffset)
3254 hash -= qoffset;
1b837d48
AD
3255 while (unlikely(hash >= qcount))
3256 hash -= qcount;
eadec877 3257 return hash + qoffset;
5605c762
JP
3258 }
3259
3260 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3261}
5605c762 3262
36c92474
BH
3263static void skb_warn_bad_offload(const struct sk_buff *skb)
3264{
84d15ae5 3265 static const netdev_features_t null_features;
36c92474 3266 struct net_device *dev = skb->dev;
88ad4175 3267 const char *name = "";
36c92474 3268
c846ad9b
BG
3269 if (!net_ratelimit())
3270 return;
3271
88ad4175
BM
3272 if (dev) {
3273 if (dev->dev.parent)
3274 name = dev_driver_string(dev->dev.parent);
3275 else
3276 name = netdev_name(dev);
3277 }
6413139d
WB
3278 skb_dump(KERN_WARNING, skb, false);
3279 WARN(1, "%s: caps=(%pNF, %pNF)\n",
88ad4175 3280 name, dev ? &dev->features : &null_features,
6413139d 3281 skb->sk ? &skb->sk->sk_route_caps : &null_features);
36c92474
BH
3282}
3283
1da177e4
LT
3284/*
3285 * Invalidate hardware checksum when packet is to be mangled, and
3286 * complete checksum manually on outgoing path.
3287 */
84fa7933 3288int skb_checksum_help(struct sk_buff *skb)
1da177e4 3289{
d3bc23e7 3290 __wsum csum;
663ead3b 3291 int ret = 0, offset;
1da177e4 3292
84fa7933 3293 if (skb->ip_summed == CHECKSUM_COMPLETE)
a430a43d
HX
3294 goto out_set_summed;
3295
3aefd7d6 3296 if (unlikely(skb_is_gso(skb))) {
36c92474
BH
3297 skb_warn_bad_offload(skb);
3298 return -EINVAL;
1da177e4
LT
3299 }
3300
cef401de
ED
3301 /* Before computing a checksum, we should make sure no frag could
3302 * be modified by an external entity : checksum could be wrong.
3303 */
3304 if (skb_has_shared_frag(skb)) {
3305 ret = __skb_linearize(skb);
3306 if (ret)
3307 goto out;
3308 }
3309
55508d60 3310 offset = skb_checksum_start_offset(skb);
a030847e
HX
3311 BUG_ON(offset >= skb_headlen(skb));
3312 csum = skb_checksum(skb, offset, skb->len - offset, 0);
3313
3314 offset += skb->csum_offset;
3315 BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
3316
8211fbfa
HK
3317 ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3318 if (ret)
3319 goto out;
1da177e4 3320
4f2e4ad5 3321 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
a430a43d 3322out_set_summed:
1da177e4 3323 skb->ip_summed = CHECKSUM_NONE;
4ec93edb 3324out:
1da177e4
LT
3325 return ret;
3326}
d1b19dff 3327EXPORT_SYMBOL(skb_checksum_help);
1da177e4 3328
b72b5bf6
DC
3329int skb_crc32c_csum_help(struct sk_buff *skb)
3330{
3331 __le32 crc32c_csum;
3332 int ret = 0, offset, start;
3333
3334 if (skb->ip_summed != CHECKSUM_PARTIAL)
3335 goto out;
3336
3337 if (unlikely(skb_is_gso(skb)))
3338 goto out;
3339
3340 /* Before computing a checksum, we should make sure no frag could
3341 * be modified by an external entity : checksum could be wrong.
3342 */
3343 if (unlikely(skb_has_shared_frag(skb))) {
3344 ret = __skb_linearize(skb);
3345 if (ret)
3346 goto out;
3347 }
3348 start = skb_checksum_start_offset(skb);
3349 offset = start + offsetof(struct sctphdr, checksum);
3350 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3351 ret = -EINVAL;
3352 goto out;
3353 }
8211fbfa
HK
3354
3355 ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3356 if (ret)
3357 goto out;
3358
b72b5bf6
DC
3359 crc32c_csum = cpu_to_le32(~__skb_checksum(skb, start,
3360 skb->len - start, ~(__u32)0,
3361 crc32c_csum_stub));
3362 *(__le32 *)(skb->data + offset) = crc32c_csum;
3363 skb->ip_summed = CHECKSUM_NONE;
dba00306 3364 skb->csum_not_inet = 0;
b72b5bf6
DC
3365out:
3366 return ret;
3367}
3368
53d6471c 3369__be16 skb_network_protocol(struct sk_buff *skb, int *depth)
f6a78bfc 3370{
252e3346 3371 __be16 type = skb->protocol;
f6a78bfc 3372
19acc327
PS
3373 /* Tunnel gso handlers can set protocol to ethernet. */
3374 if (type == htons(ETH_P_TEB)) {
3375 struct ethhdr *eth;
3376
3377 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3378 return 0;
3379
1dfe82eb 3380 eth = (struct ethhdr *)skb->data;
19acc327
PS
3381 type = eth->h_proto;
3382 }
3383
d4bcef3f 3384 return __vlan_get_protocol(skb, type, depth);
ec5f0615
PS
3385}
3386
3387/**
3388 * skb_mac_gso_segment - mac layer segmentation handler.
3389 * @skb: buffer to segment
3390 * @features: features for the output path (see dev->features)
3391 */
3392struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
3393 netdev_features_t features)
3394{
3395 struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
3396 struct packet_offload *ptype;
53d6471c
VY
3397 int vlan_depth = skb->mac_len;
3398 __be16 type = skb_network_protocol(skb, &vlan_depth);
ec5f0615
PS
3399
3400 if (unlikely(!type))
3401 return ERR_PTR(-EINVAL);
3402
53d6471c 3403 __skb_pull(skb, vlan_depth);
f6a78bfc
HX
3404
3405 rcu_read_lock();
22061d80 3406 list_for_each_entry_rcu(ptype, &offload_base, list) {
f191a1d1 3407 if (ptype->type == type && ptype->callbacks.gso_segment) {
f191a1d1 3408 segs = ptype->callbacks.gso_segment(skb, features);
f6a78bfc
HX
3409 break;
3410 }
3411 }
3412 rcu_read_unlock();
3413
98e399f8 3414 __skb_push(skb, skb->data - skb_mac_header(skb));
576a30eb 3415
f6a78bfc
HX
3416 return segs;
3417}
05e8ef4a
PS
3418EXPORT_SYMBOL(skb_mac_gso_segment);
3419
3420
3421/* openvswitch calls this on rx path, so we need a different check.
3422 */
3423static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
3424{
3425 if (tx_path)
0c19f846
WB
3426 return skb->ip_summed != CHECKSUM_PARTIAL &&
3427 skb->ip_summed != CHECKSUM_UNNECESSARY;
6e7bc478
ED
3428
3429 return skb->ip_summed == CHECKSUM_NONE;
05e8ef4a
PS
3430}
3431
3432/**
3433 * __skb_gso_segment - Perform segmentation on skb.
3434 * @skb: buffer to segment
3435 * @features: features for the output path (see dev->features)
3436 * @tx_path: whether it is called in TX path
3437 *
3438 * This function segments the given skb and returns a list of segments.
3439 *
3440 * It may return NULL if the skb requires no segmentation. This is
3441 * only possible when GSO is used for verifying header integrity.
9207f9d4 3442 *
a08e7fd9 3443 * Segmentation preserves SKB_GSO_CB_OFFSET bytes of previous skb cb.
05e8ef4a
PS
3444 */
3445struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
3446 netdev_features_t features, bool tx_path)
3447{
b2504a5d
ED
3448 struct sk_buff *segs;
3449
05e8ef4a
PS
3450 if (unlikely(skb_needs_check(skb, tx_path))) {
3451 int err;
3452
b2504a5d 3453 /* We're going to init ->check field in TCP or UDP header */
a40e0a66 3454 err = skb_cow_head(skb, 0);
3455 if (err < 0)
05e8ef4a
PS
3456 return ERR_PTR(err);
3457 }
3458
802ab55a
AD
3459 /* Only report GSO partial support if it will enable us to
3460 * support segmentation on this frame without needing additional
3461 * work.
3462 */
3463 if (features & NETIF_F_GSO_PARTIAL) {
3464 netdev_features_t partial_features = NETIF_F_GSO_ROBUST;
3465 struct net_device *dev = skb->dev;
3466
3467 partial_features |= dev->features & dev->gso_partial_features;
3468 if (!skb_gso_ok(skb, features | partial_features))
3469 features &= ~NETIF_F_GSO_PARTIAL;
3470 }
3471
a08e7fd9 3472 BUILD_BUG_ON(SKB_GSO_CB_OFFSET +
9207f9d4
KK
3473 sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
3474
68c33163 3475 SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
3347c960
ED
3476 SKB_GSO_CB(skb)->encap_level = 0;
3477
05e8ef4a
PS
3478 skb_reset_mac_header(skb);
3479 skb_reset_mac_len(skb);
3480
b2504a5d
ED
3481 segs = skb_mac_gso_segment(skb, features);
3482
3a1296a3 3483 if (segs != skb && unlikely(skb_needs_check(skb, tx_path) && !IS_ERR(segs)))
b2504a5d
ED
3484 skb_warn_bad_offload(skb);
3485
3486 return segs;
05e8ef4a 3487}
12b0004d 3488EXPORT_SYMBOL(__skb_gso_segment);
f6a78bfc 3489
fb286bb2
HX
3490/* Take action when hardware reception checksum errors are detected. */
3491#ifdef CONFIG_BUG
127d7355
TL
3492static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3493{
3494 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
3495 skb_dump(KERN_ERR, skb, true);
3496 dump_stack();
3497}
3498
7fe50ac8 3499void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
fb286bb2 3500{
127d7355 3501 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
fb286bb2
HX
3502}
3503EXPORT_SYMBOL(netdev_rx_csum_fault);
3504#endif
3505
ab74cfeb 3506/* XXX: check that highmem exists at all on the given machine. */
c1e756bf 3507static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
1da177e4 3508{
3d3a8533 3509#ifdef CONFIG_HIGHMEM
1da177e4 3510 int i;
f4563a75 3511
5acbbd42 3512 if (!(dev->features & NETIF_F_HIGHDMA)) {
ea2ab693
IC
3513 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3514 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
f4563a75 3515
ea2ab693 3516 if (PageHighMem(skb_frag_page(frag)))
5acbbd42 3517 return 1;
ea2ab693 3518 }
5acbbd42 3519 }
3d3a8533 3520#endif
1da177e4
LT
3521 return 0;
3522}
1da177e4 3523
3b392ddb
SH
3524/* If MPLS offload request, verify we are testing hardware MPLS features
3525 * instead of standard features for the netdev.
3526 */
d0edc7bf 3527#if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3b392ddb
SH
3528static netdev_features_t net_mpls_features(struct sk_buff *skb,
3529 netdev_features_t features,
3530 __be16 type)
3531{
25cd9ba0 3532 if (eth_p_mpls(type))
3b392ddb
SH
3533 features &= skb->dev->mpls_features;
3534
3535 return features;
3536}
3537#else
3538static netdev_features_t net_mpls_features(struct sk_buff *skb,
3539 netdev_features_t features,
3540 __be16 type)
3541{
3542 return features;
3543}
3544#endif
3545
c8f44aff 3546static netdev_features_t harmonize_features(struct sk_buff *skb,
c1e756bf 3547 netdev_features_t features)
f01a5236 3548{
3b392ddb
SH
3549 __be16 type;
3550
9fc95f50 3551 type = skb_network_protocol(skb, NULL);
3b392ddb 3552 features = net_mpls_features(skb, features, type);
53d6471c 3553
c0d680e5 3554 if (skb->ip_summed != CHECKSUM_NONE &&
3b392ddb 3555 !can_checksum_protocol(features, type)) {
996e8021 3556 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
f01a5236 3557 }
7be2c82c
ED
3558 if (illegal_highdma(skb->dev, skb))
3559 features &= ~NETIF_F_SG;
f01a5236
JG
3560
3561 return features;
3562}
3563
e38f3025
TM
3564netdev_features_t passthru_features_check(struct sk_buff *skb,
3565 struct net_device *dev,
3566 netdev_features_t features)
3567{
3568 return features;
3569}
3570EXPORT_SYMBOL(passthru_features_check);
3571
7ce23672 3572static netdev_features_t dflt_features_check(struct sk_buff *skb,
8cb65d00
TM
3573 struct net_device *dev,
3574 netdev_features_t features)
3575{
3576 return vlan_features_check(skb, features);
3577}
3578
cbc53e08
AD
3579static netdev_features_t gso_features_check(const struct sk_buff *skb,
3580 struct net_device *dev,
3581 netdev_features_t features)
3582{
3583 u16 gso_segs = skb_shinfo(skb)->gso_segs;
3584
3585 if (gso_segs > dev->gso_max_segs)
3586 return features & ~NETIF_F_GSO_MASK;
3587
1d155dfd
HK
3588 if (!skb_shinfo(skb)->gso_type) {
3589 skb_warn_bad_offload(skb);
3590 return features & ~NETIF_F_GSO_MASK;
3591 }
3592
802ab55a
AD
3593 /* Support for GSO partial features requires software
3594 * intervention before we can actually process the packets
3595 * so we need to strip support for any partial features now
3596 * and we can pull them back in after we have partially
3597 * segmented the frame.
3598 */
3599 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3600 features &= ~dev->gso_partial_features;
3601
3602 /* Make sure to clear the IPv4 ID mangling feature if the
3603 * IPv4 header has the potential to be fragmented.
cbc53e08
AD
3604 */
3605 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3606 struct iphdr *iph = skb->encapsulation ?
3607 inner_ip_hdr(skb) : ip_hdr(skb);
3608
3609 if (!(iph->frag_off & htons(IP_DF)))
3610 features &= ~NETIF_F_TSO_MANGLEID;
3611 }
3612
3613 return features;
3614}
3615
c1e756bf 3616netdev_features_t netif_skb_features(struct sk_buff *skb)
58e998c6 3617{
5f35227e 3618 struct net_device *dev = skb->dev;
fcbeb976 3619 netdev_features_t features = dev->features;
58e998c6 3620
cbc53e08
AD
3621 if (skb_is_gso(skb))
3622 features = gso_features_check(skb, dev, features);
30b678d8 3623
5f35227e
JG
3624 /* If encapsulation offload request, verify we are testing
3625 * hardware encapsulation features instead of standard
3626 * features for the netdev
3627 */
3628 if (skb->encapsulation)
3629 features &= dev->hw_enc_features;
3630
f5a7fb88
TM
3631 if (skb_vlan_tagged(skb))
3632 features = netdev_intersect_features(features,
3633 dev->vlan_features |
3634 NETIF_F_HW_VLAN_CTAG_TX |
3635 NETIF_F_HW_VLAN_STAG_TX);
f01a5236 3636
5f35227e
JG
3637 if (dev->netdev_ops->ndo_features_check)
3638 features &= dev->netdev_ops->ndo_features_check(skb, dev,
3639 features);
8cb65d00
TM
3640 else
3641 features &= dflt_features_check(skb, dev, features);
5f35227e 3642
c1e756bf 3643 return harmonize_features(skb, features);
58e998c6 3644}
c1e756bf 3645EXPORT_SYMBOL(netif_skb_features);
58e998c6 3646
2ea25513 3647static int xmit_one(struct sk_buff *skb, struct net_device *dev,
95f6b3dd 3648 struct netdev_queue *txq, bool more)
f6a78bfc 3649{
2ea25513
DM
3650 unsigned int len;
3651 int rc;
00829823 3652
9f9a742d 3653 if (dev_nit_active(dev))
2ea25513 3654 dev_queue_xmit_nit(skb, dev);
fc741216 3655
2ea25513 3656 len = skb->len;
3744741a 3657 PRANDOM_ADD_NOISE(skb, dev, txq, len + jiffies);
2ea25513 3658 trace_net_dev_start_xmit(skb, dev);
95f6b3dd 3659 rc = netdev_start_xmit(skb, dev, txq, more);
2ea25513 3660 trace_net_dev_xmit(skb, rc, dev, len);
adf30907 3661
2ea25513
DM
3662 return rc;
3663}
7b9c6090 3664
8dcda22a
DM
3665struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
3666 struct netdev_queue *txq, int *ret)
7f2e870f
DM
3667{
3668 struct sk_buff *skb = first;
3669 int rc = NETDEV_TX_OK;
7b9c6090 3670
7f2e870f
DM
3671 while (skb) {
3672 struct sk_buff *next = skb->next;
fc70fb64 3673
a8305bff 3674 skb_mark_not_on_list(skb);
95f6b3dd 3675 rc = xmit_one(skb, dev, txq, next != NULL);
7f2e870f
DM
3676 if (unlikely(!dev_xmit_complete(rc))) {
3677 skb->next = next;
3678 goto out;
3679 }
6afff0ca 3680
7f2e870f 3681 skb = next;
fe60faa5 3682 if (netif_tx_queue_stopped(txq) && skb) {
7f2e870f
DM
3683 rc = NETDEV_TX_BUSY;
3684 break;
9ccb8975 3685 }
7f2e870f 3686 }
9ccb8975 3687
7f2e870f
DM
3688out:
3689 *ret = rc;
3690 return skb;
3691}
b40863c6 3692
1ff0dc94
ED
3693static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
3694 netdev_features_t features)
f6a78bfc 3695{
df8a39de 3696 if (skb_vlan_tag_present(skb) &&
5968250c
JP
3697 !vlan_hw_offload_capable(features, skb->vlan_proto))
3698 skb = __vlan_hwaccel_push_inside(skb);
eae3f88e
DM
3699 return skb;
3700}
f6a78bfc 3701
43c26a1a
DC
3702int skb_csum_hwoffload_help(struct sk_buff *skb,
3703 const netdev_features_t features)
3704{
fa821170 3705 if (unlikely(skb_csum_is_sctp(skb)))
43c26a1a
DC
3706 return !!(features & NETIF_F_SCTP_CRC) ? 0 :
3707 skb_crc32c_csum_help(skb);
3708
62fafcd6
XL
3709 if (features & NETIF_F_HW_CSUM)
3710 return 0;
3711
3712 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
3713 switch (skb->csum_offset) {
3714 case offsetof(struct tcphdr, check):
3715 case offsetof(struct udphdr, check):
3716 return 0;
3717 }
3718 }
3719
3720 return skb_checksum_help(skb);
43c26a1a
DC
3721}
3722EXPORT_SYMBOL(skb_csum_hwoffload_help);
3723
f53c7239 3724static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
eae3f88e
DM
3725{
3726 netdev_features_t features;
f6a78bfc 3727
eae3f88e
DM
3728 features = netif_skb_features(skb);
3729 skb = validate_xmit_vlan(skb, features);
3730 if (unlikely(!skb))
3731 goto out_null;
7b9c6090 3732
ebf4e808
IL
3733 skb = sk_validate_xmit_skb(skb, dev);
3734 if (unlikely(!skb))
3735 goto out_null;
3736
8b86a61d 3737 if (netif_needs_gso(skb, features)) {
ce93718f
DM
3738 struct sk_buff *segs;
3739
3740 segs = skb_gso_segment(skb, features);
cecda693 3741 if (IS_ERR(segs)) {
af6dabc9 3742 goto out_kfree_skb;
cecda693
JW
3743 } else if (segs) {
3744 consume_skb(skb);
3745 skb = segs;
f6a78bfc 3746 }
eae3f88e
DM
3747 } else {
3748 if (skb_needs_linearize(skb, features) &&
3749 __skb_linearize(skb))
3750 goto out_kfree_skb;
4ec93edb 3751
eae3f88e
DM
3752 /* If packet is not checksummed and device does not
3753 * support checksumming for this protocol, complete
3754 * checksumming here.
3755 */
3756 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3757 if (skb->encapsulation)
3758 skb_set_inner_transport_header(skb,
3759 skb_checksum_start_offset(skb));
3760 else
3761 skb_set_transport_header(skb,
3762 skb_checksum_start_offset(skb));
43c26a1a 3763 if (skb_csum_hwoffload_help(skb, features))
eae3f88e 3764 goto out_kfree_skb;
7b9c6090 3765 }
0c772159 3766 }
7b9c6090 3767
f53c7239 3768 skb = validate_xmit_xfrm(skb, features, again);
3dca3f38 3769
eae3f88e 3770 return skb;
fc70fb64 3771
f6a78bfc
HX
3772out_kfree_skb:
3773 kfree_skb(skb);
eae3f88e 3774out_null:
d21fd63e 3775 atomic_long_inc(&dev->tx_dropped);
eae3f88e
DM
3776 return NULL;
3777}
6afff0ca 3778
f53c7239 3779struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
55a93b3e
ED
3780{
3781 struct sk_buff *next, *head = NULL, *tail;
3782
bec3cfdc 3783 for (; skb != NULL; skb = next) {
55a93b3e 3784 next = skb->next;
a8305bff 3785 skb_mark_not_on_list(skb);
bec3cfdc
ED
3786
3787 /* in case skb wont be segmented, point to itself */
3788 skb->prev = skb;
3789
f53c7239 3790 skb = validate_xmit_skb(skb, dev, again);
bec3cfdc
ED
3791 if (!skb)
3792 continue;
55a93b3e 3793
bec3cfdc
ED
3794 if (!head)
3795 head = skb;
3796 else
3797 tail->next = skb;
3798 /* If skb was segmented, skb->prev points to
3799 * the last segment. If not, it still contains skb.
3800 */
3801 tail = skb->prev;
55a93b3e
ED
3802 }
3803 return head;
f6a78bfc 3804}
104ba78c 3805EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
f6a78bfc 3806
1def9238
ED
3807static void qdisc_pkt_len_init(struct sk_buff *skb)
3808{
3809 const struct skb_shared_info *shinfo = skb_shinfo(skb);
3810
3811 qdisc_skb_cb(skb)->pkt_len = skb->len;
3812
3813 /* To get more precise estimation of bytes sent on wire,
3814 * we add to pkt_len the headers size of all segments
3815 */
a0dce875 3816 if (shinfo->gso_size && skb_transport_header_was_set(skb)) {
757b8b1d 3817 unsigned int hdr_len;
15e5a030 3818 u16 gso_segs = shinfo->gso_segs;
1def9238 3819
757b8b1d
ED
3820 /* mac layer + network layer */
3821 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3822
3823 /* + transport layer */
7c68d1a6
ED
3824 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
3825 const struct tcphdr *th;
3826 struct tcphdr _tcphdr;
3827
3828 th = skb_header_pointer(skb, skb_transport_offset(skb),
3829 sizeof(_tcphdr), &_tcphdr);
3830 if (likely(th))
3831 hdr_len += __tcp_hdrlen(th);
3832 } else {
3833 struct udphdr _udphdr;
3834
3835 if (skb_header_pointer(skb, skb_transport_offset(skb),
3836 sizeof(_udphdr), &_udphdr))
3837 hdr_len += sizeof(struct udphdr);
3838 }
15e5a030
JW
3839
3840 if (shinfo->gso_type & SKB_GSO_DODGY)
3841 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3842 shinfo->gso_size);
3843
3844 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
1def9238
ED
3845 }
3846}
3847
70713ddd
QX
3848static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
3849 struct sk_buff **to_free,
3850 struct netdev_queue *txq)
3851{
3852 int rc;
3853
3854 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
3855 if (rc == NET_XMIT_SUCCESS)
3856 trace_qdisc_enqueue(q, txq, skb);
3857 return rc;
3858}
3859
bbd8a0d3
KK
3860static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3861 struct net_device *dev,
3862 struct netdev_queue *txq)
3863{
3864 spinlock_t *root_lock = qdisc_lock(q);
520ac30f 3865 struct sk_buff *to_free = NULL;
a2da570d 3866 bool contended;
bbd8a0d3
KK
3867 int rc;
3868
a2da570d 3869 qdisc_calculate_pkt_len(skb, q);
6b3ba914
JF
3870
3871 if (q->flags & TCQ_F_NOLOCK) {
c4fef01b
YL
3872 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
3873 qdisc_run_begin(q)) {
3874 /* Retest nolock_qdisc_is_empty() within the protection
3875 * of q->seqlock to protect from racing with requeuing.
3876 */
3877 if (unlikely(!nolock_qdisc_is_empty(q))) {
70713ddd 3878 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
c4fef01b
YL
3879 __qdisc_run(q);
3880 qdisc_run_end(q);
3881
3882 goto no_lock_out;
3883 }
3884
3885 qdisc_bstats_cpu_update(q, skb);
3886 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
3887 !nolock_qdisc_is_empty(q))
3888 __qdisc_run(q);
3889
3890 qdisc_run_end(q);
3891 return NET_XMIT_SUCCESS;
3892 }
3893
70713ddd 3894 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
c4fef01b 3895 qdisc_run(q);
6b3ba914 3896
c4fef01b 3897no_lock_out:
6b3ba914
JF
3898 if (unlikely(to_free))
3899 kfree_skb_list(to_free);
3900 return rc;
3901 }
3902
79640a4c
ED
3903 /*
3904 * Heuristic to force contended enqueues to serialize on a
3905 * separate lock before trying to get qdisc main lock.
f9eb8aea 3906 * This permits qdisc->running owner to get the lock more
9bf2b8c2 3907 * often and dequeue packets faster.
79640a4c 3908 */
a2da570d 3909 contended = qdisc_is_running(q);
79640a4c
ED
3910 if (unlikely(contended))
3911 spin_lock(&q->busylock);
3912
bbd8a0d3
KK
3913 spin_lock(root_lock);
3914 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
520ac30f 3915 __qdisc_drop(skb, &to_free);
bbd8a0d3
KK
3916 rc = NET_XMIT_DROP;
3917 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
bc135b23 3918 qdisc_run_begin(q)) {
bbd8a0d3
KK
3919 /*
3920 * This is a work-conserving queue; there are no old skbs
3921 * waiting to be sent out; and the qdisc is not running -
3922 * xmit the skb directly.
3923 */
bfe0d029 3924
bfe0d029
ED
3925 qdisc_bstats_update(q, skb);
3926
55a93b3e 3927 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
79640a4c
ED
3928 if (unlikely(contended)) {
3929 spin_unlock(&q->busylock);
3930 contended = false;
3931 }
bbd8a0d3 3932 __qdisc_run(q);
6c148184 3933 }
bbd8a0d3 3934
6c148184 3935 qdisc_run_end(q);
bbd8a0d3
KK
3936 rc = NET_XMIT_SUCCESS;
3937 } else {
70713ddd 3938 rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
79640a4c
ED
3939 if (qdisc_run_begin(q)) {
3940 if (unlikely(contended)) {
3941 spin_unlock(&q->busylock);
3942 contended = false;
3943 }
3944 __qdisc_run(q);
6c148184 3945 qdisc_run_end(q);
79640a4c 3946 }
bbd8a0d3
KK
3947 }
3948 spin_unlock(root_lock);
520ac30f
ED
3949 if (unlikely(to_free))
3950 kfree_skb_list(to_free);
79640a4c
ED
3951 if (unlikely(contended))
3952 spin_unlock(&q->busylock);
bbd8a0d3
KK
3953 return rc;
3954}
3955
86f8515f 3956#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
5bc1421e
NH
3957static void skb_update_prio(struct sk_buff *skb)
3958{
4dcb31d4
ED
3959 const struct netprio_map *map;
3960 const struct sock *sk;
3961 unsigned int prioidx;
5bc1421e 3962
4dcb31d4
ED
3963 if (skb->priority)
3964 return;
3965 map = rcu_dereference_bh(skb->dev->priomap);
3966 if (!map)
3967 return;
3968 sk = skb_to_full_sk(skb);
3969 if (!sk)
3970 return;
91c68ce2 3971
4dcb31d4
ED
3972 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
3973
3974 if (prioidx < map->priomap_len)
3975 skb->priority = map->priomap[prioidx];
5bc1421e
NH
3976}
3977#else
3978#define skb_update_prio(skb)
3979#endif
3980
95603e22
MM
3981/**
3982 * dev_loopback_xmit - loop back @skb
0c4b51f0
EB
3983 * @net: network namespace this loopback is happening in
3984 * @sk: sk needed to be a netfilter okfn
95603e22
MM
3985 * @skb: buffer to transmit
3986 */
0c4b51f0 3987int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
95603e22
MM
3988{
3989 skb_reset_mac_header(skb);
3990 __skb_pull(skb, skb_network_offset(skb));
3991 skb->pkt_type = PACKET_LOOPBACK;
3992 skb->ip_summed = CHECKSUM_UNNECESSARY;
3993 WARN_ON(!skb_dst(skb));
3994 skb_dst_force(skb);
3995 netif_rx_ni(skb);
3996 return 0;
3997}
3998EXPORT_SYMBOL(dev_loopback_xmit);
3999
1f211a1b
DB
4000#ifdef CONFIG_NET_EGRESS
4001static struct sk_buff *
4002sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4003{
46209401 4004 struct mini_Qdisc *miniq = rcu_dereference_bh(dev->miniq_egress);
1f211a1b
DB
4005 struct tcf_result cl_res;
4006
46209401 4007 if (!miniq)
1f211a1b
DB
4008 return skb;
4009
8dc07fdb 4010 /* qdisc_skb_cb(skb)->pkt_len was already set by the caller. */
aadaca9e 4011 qdisc_skb_cb(skb)->mru = 0;
7baf2429 4012 qdisc_skb_cb(skb)->post_ct = false;
46209401 4013 mini_qdisc_bstats_cpu_update(miniq, skb);
1f211a1b 4014
46209401 4015 switch (tcf_classify(skb, miniq->filter_list, &cl_res, false)) {
1f211a1b
DB
4016 case TC_ACT_OK:
4017 case TC_ACT_RECLASSIFY:
4018 skb->tc_index = TC_H_MIN(cl_res.classid);
4019 break;
4020 case TC_ACT_SHOT:
46209401 4021 mini_qdisc_qstats_cpu_drop(miniq);
1f211a1b 4022 *ret = NET_XMIT_DROP;
7e2c3aea
DB
4023 kfree_skb(skb);
4024 return NULL;
1f211a1b
DB
4025 case TC_ACT_STOLEN:
4026 case TC_ACT_QUEUED:
e25ea21f 4027 case TC_ACT_TRAP:
1f211a1b 4028 *ret = NET_XMIT_SUCCESS;
7e2c3aea 4029 consume_skb(skb);
1f211a1b
DB
4030 return NULL;
4031 case TC_ACT_REDIRECT:
4032 /* No need to push/pop skb's mac_header here on egress! */
4033 skb_do_redirect(skb);
4034 *ret = NET_XMIT_SUCCESS;
4035 return NULL;
4036 default:
4037 break;
4038 }
357b6cc5 4039
1f211a1b
DB
4040 return skb;
4041}
4042#endif /* CONFIG_NET_EGRESS */
4043
fc9bab24
AN
4044#ifdef CONFIG_XPS
4045static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4046 struct xps_dev_maps *dev_maps, unsigned int tci)
4047{
255c04a8 4048 int tc = netdev_get_prio_tc_map(dev, skb->priority);
fc9bab24
AN
4049 struct xps_map *map;
4050 int queue_index = -1;
4051
5478fcd0 4052 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
255c04a8
AT
4053 return queue_index;
4054
4055 tci *= dev_maps->num_tc;
4056 tci += tc;
fc9bab24
AN
4057
4058 map = rcu_dereference(dev_maps->attr_map[tci]);
4059 if (map) {
4060 if (map->len == 1)
4061 queue_index = map->queues[0];
4062 else
4063 queue_index = map->queues[reciprocal_scale(
4064 skb_get_hash(skb), map->len)];
4065 if (unlikely(queue_index >= dev->real_num_tx_queues))
4066 queue_index = -1;
4067 }
4068 return queue_index;
4069}
4070#endif
4071
eadec877
AD
4072static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4073 struct sk_buff *skb)
638b2a69
JP
4074{
4075#ifdef CONFIG_XPS
4076 struct xps_dev_maps *dev_maps;
fc9bab24 4077 struct sock *sk = skb->sk;
638b2a69
JP
4078 int queue_index = -1;
4079
04157469
AN
4080 if (!static_key_false(&xps_needed))
4081 return -1;
4082
638b2a69 4083 rcu_read_lock();
fc9bab24
AN
4084 if (!static_key_false(&xps_rxqs_needed))
4085 goto get_cpus_map;
4086
044ab86d 4087 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
638b2a69 4088 if (dev_maps) {
fc9bab24 4089 int tci = sk_rx_queue_get(sk);
184c449f 4090
5478fcd0 4091 if (tci >= 0)
fc9bab24
AN
4092 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4093 tci);
4094 }
184c449f 4095
fc9bab24
AN
4096get_cpus_map:
4097 if (queue_index < 0) {
044ab86d 4098 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
fc9bab24
AN
4099 if (dev_maps) {
4100 unsigned int tci = skb->sender_cpu - 1;
4101
4102 queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4103 tci);
638b2a69
JP
4104 }
4105 }
4106 rcu_read_unlock();
4107
4108 return queue_index;
4109#else
4110 return -1;
4111#endif
4112}
4113
a4ea8a3d 4114u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
a350ecce 4115 struct net_device *sb_dev)
a4ea8a3d
AD
4116{
4117 return 0;
4118}
4119EXPORT_SYMBOL(dev_pick_tx_zero);
4120
4121u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
a350ecce 4122 struct net_device *sb_dev)
a4ea8a3d
AD
4123{
4124 return (u16)raw_smp_processor_id() % dev->real_num_tx_queues;
4125}
4126EXPORT_SYMBOL(dev_pick_tx_cpu_id);
4127
b71b5837
PA
4128u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4129 struct net_device *sb_dev)
638b2a69
JP
4130{
4131 struct sock *sk = skb->sk;
4132 int queue_index = sk_tx_queue_get(sk);
4133
eadec877
AD
4134 sb_dev = sb_dev ? : dev;
4135
638b2a69
JP
4136 if (queue_index < 0 || skb->ooo_okay ||
4137 queue_index >= dev->real_num_tx_queues) {
eadec877 4138 int new_index = get_xps_queue(dev, sb_dev, skb);
f4563a75 4139
638b2a69 4140 if (new_index < 0)
eadec877 4141 new_index = skb_tx_hash(dev, sb_dev, skb);
638b2a69
JP
4142
4143 if (queue_index != new_index && sk &&
004a5d01 4144 sk_fullsock(sk) &&
638b2a69
JP
4145 rcu_access_pointer(sk->sk_dst_cache))
4146 sk_tx_queue_set(sk, new_index);
4147
4148 queue_index = new_index;
4149 }
4150
4151 return queue_index;
4152}
b71b5837 4153EXPORT_SYMBOL(netdev_pick_tx);
638b2a69 4154
4bd97d51
PA
4155struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4156 struct sk_buff *skb,
4157 struct net_device *sb_dev)
638b2a69
JP
4158{
4159 int queue_index = 0;
4160
4161#ifdef CONFIG_XPS
52bd2d62
ED
4162 u32 sender_cpu = skb->sender_cpu - 1;
4163
4164 if (sender_cpu >= (u32)NR_CPUS)
638b2a69
JP
4165 skb->sender_cpu = raw_smp_processor_id() + 1;
4166#endif
4167
4168 if (dev->real_num_tx_queues != 1) {
4169 const struct net_device_ops *ops = dev->netdev_ops;
f4563a75 4170
638b2a69 4171 if (ops->ndo_select_queue)
a350ecce 4172 queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
638b2a69 4173 else
4bd97d51 4174 queue_index = netdev_pick_tx(dev, skb, sb_dev);
638b2a69 4175
d584527c 4176 queue_index = netdev_cap_txqueue(dev, queue_index);
638b2a69
JP
4177 }
4178
4179 skb_set_queue_mapping(skb, queue_index);
4180 return netdev_get_tx_queue(dev, queue_index);
4181}
4182
d29f749e 4183/**
9d08dd3d 4184 * __dev_queue_xmit - transmit a buffer
d29f749e 4185 * @skb: buffer to transmit
eadec877 4186 * @sb_dev: suboordinate device used for L2 forwarding offload
d29f749e
DJ
4187 *
4188 * Queue a buffer for transmission to a network device. The caller must
4189 * have set the device and priority and built the buffer before calling
4190 * this function. The function can be called from an interrupt.
4191 *
4192 * A negative errno code is returned on a failure. A success does not
4193 * guarantee the frame will be transmitted as it may be dropped due
4194 * to congestion or traffic shaping.
4195 *
4196 * -----------------------------------------------------------------------------------
4197 * I notice this method can also return errors from the queue disciplines,
4198 * including NET_XMIT_DROP, which is a positive value. So, errors can also
4199 * be positive.
4200 *
4201 * Regardless of the return value, the skb is consumed, so it is currently
4202 * difficult to retry a send to this method. (You can bump the ref count
4203 * before sending to hold a reference for retry if you are careful.)
4204 *
4205 * When calling this method, interrupts MUST be enabled. This is because
4206 * the BH enable code must have IRQs enabled so that it will not deadlock.
4207 * --BLG
4208 */
eadec877 4209static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
1da177e4
LT
4210{
4211 struct net_device *dev = skb->dev;
dc2b4847 4212 struct netdev_queue *txq;
1da177e4
LT
4213 struct Qdisc *q;
4214 int rc = -ENOMEM;
f53c7239 4215 bool again = false;
1da177e4 4216
6d1ccff6
ED
4217 skb_reset_mac_header(skb);
4218
e7fd2885 4219 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
e7ed11ee 4220 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
e7fd2885 4221
4ec93edb
YH
4222 /* Disable soft irqs for various locks below. Also
4223 * stops preemption for RCU.
1da177e4 4224 */
4ec93edb 4225 rcu_read_lock_bh();
1da177e4 4226
5bc1421e
NH
4227 skb_update_prio(skb);
4228
1f211a1b
DB
4229 qdisc_pkt_len_init(skb);
4230#ifdef CONFIG_NET_CLS_ACT
8dc07fdb 4231 skb->tc_at_ingress = 0;
357b6cc5 4232# ifdef CONFIG_NET_EGRESS
aabf6772 4233 if (static_branch_unlikely(&egress_needed_key)) {
1f211a1b
DB
4234 skb = sch_handle_egress(skb, &rc, dev);
4235 if (!skb)
4236 goto out;
4237 }
357b6cc5 4238# endif
1f211a1b 4239#endif
02875878
ED
4240 /* If device/qdisc don't need skb->dst, release it right now while
4241 * its hot in this cpu cache.
4242 */
4243 if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
4244 skb_dst_drop(skb);
4245 else
4246 skb_dst_force(skb);
4247
4bd97d51 4248 txq = netdev_core_pick_tx(dev, skb, sb_dev);
a898def2 4249 q = rcu_dereference_bh(txq->qdisc);
37437bb2 4250
cf66ba58 4251 trace_net_dev_queue(skb);
1da177e4 4252 if (q->enqueue) {
bbd8a0d3 4253 rc = __dev_xmit_skb(skb, q, dev, txq);
37437bb2 4254 goto out;
1da177e4
LT
4255 }
4256
4257 /* The device has no queue. Common case for software devices:
eb13da1a 4258 * loopback, all the sorts of tunnels...
1da177e4 4259
eb13da1a 4260 * Really, it is unlikely that netif_tx_lock protection is necessary
4261 * here. (f.e. loopback and IP tunnels are clean ignoring statistics
4262 * counters.)
4263 * However, it is possible, that they rely on protection
4264 * made by us here.
1da177e4 4265
eb13da1a 4266 * Check this and shot the lock. It is not prone from deadlocks.
4267 *Either shot noqueue qdisc, it is even simpler 8)
1da177e4
LT
4268 */
4269 if (dev->flags & IFF_UP) {
4270 int cpu = smp_processor_id(); /* ok because BHs are off */
4271
c773e847 4272 if (txq->xmit_lock_owner != cpu) {
97cdcf37 4273 if (dev_xmit_recursion())
745e20f1
ED
4274 goto recursion_alert;
4275
f53c7239 4276 skb = validate_xmit_skb(skb, dev, &again);
1f59533f 4277 if (!skb)
d21fd63e 4278 goto out;
1f59533f 4279
3744741a 4280 PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
c773e847 4281 HARD_TX_LOCK(dev, txq, cpu);
1da177e4 4282
73466498 4283 if (!netif_xmit_stopped(txq)) {
97cdcf37 4284 dev_xmit_recursion_inc();
ce93718f 4285 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
97cdcf37 4286 dev_xmit_recursion_dec();
572a9d7b 4287 if (dev_xmit_complete(rc)) {
c773e847 4288 HARD_TX_UNLOCK(dev, txq);
1da177e4
LT
4289 goto out;
4290 }
4291 }
c773e847 4292 HARD_TX_UNLOCK(dev, txq);
e87cc472
JP
4293 net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
4294 dev->name);
1da177e4
LT
4295 } else {
4296 /* Recursion is detected! It is possible,
745e20f1
ED
4297 * unfortunately
4298 */
4299recursion_alert:
e87cc472
JP
4300 net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
4301 dev->name);
1da177e4
LT
4302 }
4303 }
4304
4305 rc = -ENETDOWN;
d4828d85 4306 rcu_read_unlock_bh();
1da177e4 4307
015f0688 4308 atomic_long_inc(&dev->tx_dropped);
1f59533f 4309 kfree_skb_list(skb);
1da177e4
LT
4310 return rc;
4311out:
d4828d85 4312 rcu_read_unlock_bh();
1da177e4
LT
4313 return rc;
4314}
f663dd9a 4315
2b4aa3ce 4316int dev_queue_xmit(struct sk_buff *skb)
f663dd9a
JW
4317{
4318 return __dev_queue_xmit(skb, NULL);
4319}
2b4aa3ce 4320EXPORT_SYMBOL(dev_queue_xmit);
1da177e4 4321
eadec877 4322int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev)
f663dd9a 4323{
eadec877 4324 return __dev_queue_xmit(skb, sb_dev);
f663dd9a
JW
4325}
4326EXPORT_SYMBOL(dev_queue_xmit_accel);
4327
36ccdf85 4328int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
865b03f2
MK
4329{
4330 struct net_device *dev = skb->dev;
4331 struct sk_buff *orig_skb = skb;
4332 struct netdev_queue *txq;
4333 int ret = NETDEV_TX_BUSY;
4334 bool again = false;
4335
4336 if (unlikely(!netif_running(dev) ||
4337 !netif_carrier_ok(dev)))
4338 goto drop;
4339
4340 skb = validate_xmit_skb_list(skb, dev, &again);
4341 if (skb != orig_skb)
4342 goto drop;
4343
4344 skb_set_queue_mapping(skb, queue_id);
4345 txq = skb_get_tx_queue(dev, skb);
3744741a 4346 PRANDOM_ADD_NOISE(skb, dev, txq, jiffies);
865b03f2
MK
4347
4348 local_bh_disable();
4349
0ad6f6e7 4350 dev_xmit_recursion_inc();
865b03f2
MK
4351 HARD_TX_LOCK(dev, txq, smp_processor_id());
4352 if (!netif_xmit_frozen_or_drv_stopped(txq))
4353 ret = netdev_start_xmit(skb, dev, txq, false);
4354 HARD_TX_UNLOCK(dev, txq);
0ad6f6e7 4355 dev_xmit_recursion_dec();
865b03f2
MK
4356
4357 local_bh_enable();
865b03f2
MK
4358 return ret;
4359drop:
4360 atomic_long_inc(&dev->tx_dropped);
4361 kfree_skb_list(skb);
4362 return NET_XMIT_DROP;
4363}
36ccdf85 4364EXPORT_SYMBOL(__dev_direct_xmit);
1da177e4 4365
eb13da1a 4366/*************************************************************************
4367 * Receiver routines
4368 *************************************************************************/
1da177e4 4369
6b2bedc3 4370int netdev_max_backlog __read_mostly = 1000;
c9e6bc64
ED
4371EXPORT_SYMBOL(netdev_max_backlog);
4372
3b098e2d 4373int netdev_tstamp_prequeue __read_mostly = 1;
6b2bedc3 4374int netdev_budget __read_mostly = 300;
a4837980
KK
4375/* Must be at least 2 jiffes to guarantee 1 jiffy timeout */
4376unsigned int __read_mostly netdev_budget_usecs = 2 * USEC_PER_SEC / HZ;
3d48b53f
MT
4377int weight_p __read_mostly = 64; /* old backlog weight */
4378int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */
4379int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */
4380int dev_rx_weight __read_mostly = 64;
4381int dev_tx_weight __read_mostly = 64;
323ebb61
EC
4382/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */
4383int gro_normal_batch __read_mostly = 8;
1da177e4 4384
eecfd7c4
ED
4385/* Called with irq disabled */
4386static inline void ____napi_schedule(struct softnet_data *sd,
4387 struct napi_struct *napi)
4388{
29863d41
WW
4389 struct task_struct *thread;
4390
4391 if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
4392 /* Paired with smp_mb__before_atomic() in
5fdd2f0e
WW
4393 * napi_enable()/dev_set_threaded().
4394 * Use READ_ONCE() to guarantee a complete
4395 * read on napi->thread. Only call
29863d41
WW
4396 * wake_up_process() when it's not NULL.
4397 */
4398 thread = READ_ONCE(napi->thread);
4399 if (thread) {
cb038357
WW
4400 /* Avoid doing set_bit() if the thread is in
4401 * INTERRUPTIBLE state, cause napi_thread_wait()
4402 * makes sure to proceed with napi polling
4403 * if the thread is explicitly woken from here.
4404 */
2f064a59 4405 if (READ_ONCE(thread->__state) != TASK_INTERRUPTIBLE)
cb038357 4406 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
29863d41
WW
4407 wake_up_process(thread);
4408 return;
4409 }
4410 }
4411
eecfd7c4
ED
4412 list_add_tail(&napi->poll_list, &sd->poll_list);
4413 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4414}
4415
bfb564e7
KK
4416#ifdef CONFIG_RPS
4417
4418/* One global table that all flow-based protocols share. */
6e3f7faf 4419struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
bfb564e7 4420EXPORT_SYMBOL(rps_sock_flow_table);
567e4b79
ED
4421u32 rps_cpu_mask __read_mostly;
4422EXPORT_SYMBOL(rps_cpu_mask);
bfb564e7 4423
dc05360f 4424struct static_key_false rps_needed __read_mostly;
3df97ba8 4425EXPORT_SYMBOL(rps_needed);
dc05360f 4426struct static_key_false rfs_needed __read_mostly;
13bfff25 4427EXPORT_SYMBOL(rfs_needed);
adc9300e 4428
c445477d
BH
4429static struct rps_dev_flow *
4430set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4431 struct rps_dev_flow *rflow, u16 next_cpu)
4432{
a31196b0 4433 if (next_cpu < nr_cpu_ids) {
c445477d
BH
4434#ifdef CONFIG_RFS_ACCEL
4435 struct netdev_rx_queue *rxqueue;
4436 struct rps_dev_flow_table *flow_table;
4437 struct rps_dev_flow *old_rflow;
4438 u32 flow_id;
4439 u16 rxq_index;
4440 int rc;
4441
4442 /* Should we steer this flow to a different hardware queue? */
69a19ee6
BH
4443 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
4444 !(dev->features & NETIF_F_NTUPLE))
c445477d
BH
4445 goto out;
4446 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
4447 if (rxq_index == skb_get_rx_queue(skb))
4448 goto out;
4449
4450 rxqueue = dev->_rx + rxq_index;
4451 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4452 if (!flow_table)
4453 goto out;
61b905da 4454 flow_id = skb_get_hash(skb) & flow_table->mask;
c445477d
BH
4455 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
4456 rxq_index, flow_id);
4457 if (rc < 0)
4458 goto out;
4459 old_rflow = rflow;
4460 rflow = &flow_table->flows[flow_id];
c445477d
BH
4461 rflow->filter = rc;
4462 if (old_rflow->filter == rflow->filter)
4463 old_rflow->filter = RPS_NO_FILTER;
4464 out:
4465#endif
4466 rflow->last_qtail =
09994d1b 4467 per_cpu(softnet_data, next_cpu).input_queue_head;
c445477d
BH
4468 }
4469
09994d1b 4470 rflow->cpu = next_cpu;
c445477d
BH
4471 return rflow;
4472}
4473
bfb564e7
KK
4474/*
4475 * get_rps_cpu is called from netif_receive_skb and returns the target
4476 * CPU from the RPS map of the receiving queue for a given skb.
4477 * rcu_read_lock must be held on entry.
4478 */
4479static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
4480 struct rps_dev_flow **rflowp)
4481{
567e4b79
ED
4482 const struct rps_sock_flow_table *sock_flow_table;
4483 struct netdev_rx_queue *rxqueue = dev->_rx;
bfb564e7 4484 struct rps_dev_flow_table *flow_table;
567e4b79 4485 struct rps_map *map;
bfb564e7 4486 int cpu = -1;
567e4b79 4487 u32 tcpu;
61b905da 4488 u32 hash;
bfb564e7
KK
4489
4490 if (skb_rx_queue_recorded(skb)) {
4491 u16 index = skb_get_rx_queue(skb);
567e4b79 4492
62fe0b40
BH
4493 if (unlikely(index >= dev->real_num_rx_queues)) {
4494 WARN_ONCE(dev->real_num_rx_queues > 1,
4495 "%s received packet on queue %u, but number "
4496 "of RX queues is %u\n",
4497 dev->name, index, dev->real_num_rx_queues);
bfb564e7
KK
4498 goto done;
4499 }
567e4b79
ED
4500 rxqueue += index;
4501 }
bfb564e7 4502
567e4b79
ED
4503 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
4504
4505 flow_table = rcu_dereference(rxqueue->rps_flow_table);
6e3f7faf 4506 map = rcu_dereference(rxqueue->rps_map);
567e4b79 4507 if (!flow_table && !map)
bfb564e7
KK
4508 goto done;
4509
2d47b459 4510 skb_reset_network_header(skb);
61b905da
TH
4511 hash = skb_get_hash(skb);
4512 if (!hash)
bfb564e7
KK
4513 goto done;
4514
fec5e652
TH
4515 sock_flow_table = rcu_dereference(rps_sock_flow_table);
4516 if (flow_table && sock_flow_table) {
fec5e652 4517 struct rps_dev_flow *rflow;
567e4b79
ED
4518 u32 next_cpu;
4519 u32 ident;
4520
4521 /* First check into global flow table if there is a match */
4522 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
4523 if ((ident ^ hash) & ~rps_cpu_mask)
4524 goto try_rps;
fec5e652 4525
567e4b79
ED
4526 next_cpu = ident & rps_cpu_mask;
4527
4528 /* OK, now we know there is a match,
4529 * we can look at the local (per receive queue) flow table
4530 */
61b905da 4531 rflow = &flow_table->flows[hash & flow_table->mask];
fec5e652
TH
4532 tcpu = rflow->cpu;
4533
fec5e652
TH
4534 /*
4535 * If the desired CPU (where last recvmsg was done) is
4536 * different from current CPU (one in the rx-queue flow
4537 * table entry), switch if one of the following holds:
a31196b0 4538 * - Current CPU is unset (>= nr_cpu_ids).
fec5e652
TH
4539 * - Current CPU is offline.
4540 * - The current CPU's queue tail has advanced beyond the
4541 * last packet that was enqueued using this table entry.
4542 * This guarantees that all previous packets for the flow
4543 * have been dequeued, thus preserving in order delivery.
4544 */
4545 if (unlikely(tcpu != next_cpu) &&
a31196b0 4546 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
fec5e652 4547 ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
baefa31d
TH
4548 rflow->last_qtail)) >= 0)) {
4549 tcpu = next_cpu;
c445477d 4550 rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
baefa31d 4551 }
c445477d 4552
a31196b0 4553 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
fec5e652
TH
4554 *rflowp = rflow;
4555 cpu = tcpu;
4556 goto done;
4557 }
4558 }
4559
567e4b79
ED
4560try_rps:
4561
0a9627f2 4562 if (map) {
8fc54f68 4563 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
0a9627f2
TH
4564 if (cpu_online(tcpu)) {
4565 cpu = tcpu;
4566 goto done;
4567 }
4568 }
4569
4570done:
0a9627f2
TH
4571 return cpu;
4572}
4573
c445477d
BH
4574#ifdef CONFIG_RFS_ACCEL
4575
4576/**
4577 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4578 * @dev: Device on which the filter was set
4579 * @rxq_index: RX queue index
4580 * @flow_id: Flow ID passed to ndo_rx_flow_steer()
4581 * @filter_id: Filter ID returned by ndo_rx_flow_steer()
4582 *
4583 * Drivers that implement ndo_rx_flow_steer() should periodically call
4584 * this function for each installed filter and remove the filters for
4585 * which it returns %true.
4586 */
4587bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
4588 u32 flow_id, u16 filter_id)
4589{
4590 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
4591 struct rps_dev_flow_table *flow_table;
4592 struct rps_dev_flow *rflow;
4593 bool expire = true;
a31196b0 4594 unsigned int cpu;
c445477d
BH
4595
4596 rcu_read_lock();
4597 flow_table = rcu_dereference(rxqueue->rps_flow_table);
4598 if (flow_table && flow_id <= flow_table->mask) {
4599 rflow = &flow_table->flows[flow_id];
6aa7de05 4600 cpu = READ_ONCE(rflow->cpu);
a31196b0 4601 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
c445477d
BH
4602 ((int)(per_cpu(softnet_data, cpu).input_queue_head -
4603 rflow->last_qtail) <
4604 (int)(10 * flow_table->mask)))
4605 expire = false;
4606 }
4607 rcu_read_unlock();
4608 return expire;
4609}
4610EXPORT_SYMBOL(rps_may_expire_flow);
4611
4612#endif /* CONFIG_RFS_ACCEL */
4613
0a9627f2 4614/* Called from hardirq (IPI) context */
e36fa2f7 4615static void rps_trigger_softirq(void *data)
0a9627f2 4616{
e36fa2f7
ED
4617 struct softnet_data *sd = data;
4618
eecfd7c4 4619 ____napi_schedule(sd, &sd->backlog);
dee42870 4620 sd->received_rps++;
0a9627f2 4621}
e36fa2f7 4622
fec5e652 4623#endif /* CONFIG_RPS */
0a9627f2 4624
e36fa2f7
ED
4625/*
4626 * Check if this softnet_data structure is another cpu one
4627 * If yes, queue it to our IPI list and return 1
4628 * If no, return 0
4629 */
4630static int rps_ipi_queued(struct softnet_data *sd)
4631{
4632#ifdef CONFIG_RPS
903ceff7 4633 struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
e36fa2f7
ED
4634
4635 if (sd != mysd) {
4636 sd->rps_ipi_next = mysd->rps_ipi_list;
4637 mysd->rps_ipi_list = sd;
4638
4639 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
4640 return 1;
4641 }
4642#endif /* CONFIG_RPS */
4643 return 0;
4644}
4645
99bbc707
WB
4646#ifdef CONFIG_NET_FLOW_LIMIT
4647int netdev_flow_limit_table_len __read_mostly = (1 << 12);
4648#endif
4649
4650static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
4651{
4652#ifdef CONFIG_NET_FLOW_LIMIT
4653 struct sd_flow_limit *fl;
4654 struct softnet_data *sd;
4655 unsigned int old_flow, new_flow;
4656
4657 if (qlen < (netdev_max_backlog >> 1))
4658 return false;
4659
903ceff7 4660 sd = this_cpu_ptr(&softnet_data);
99bbc707
WB
4661
4662 rcu_read_lock();
4663 fl = rcu_dereference(sd->flow_limit);
4664 if (fl) {
3958afa1 4665 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
99bbc707
WB
4666 old_flow = fl->history[fl->history_head];
4667 fl->history[fl->history_head] = new_flow;
4668
4669 fl->history_head++;
4670 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
4671
4672 if (likely(fl->buckets[old_flow]))
4673 fl->buckets[old_flow]--;
4674
4675 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
4676 fl->count++;
4677 rcu_read_unlock();
4678 return true;
4679 }
4680 }
4681 rcu_read_unlock();
4682#endif
4683 return false;
4684}
4685
0a9627f2
TH
4686/*
4687 * enqueue_to_backlog is called to queue an skb to a per CPU backlog
4688 * queue (may be a remote CPU queue).
4689 */
fec5e652
TH
4690static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
4691 unsigned int *qtail)
0a9627f2 4692{
e36fa2f7 4693 struct softnet_data *sd;
0a9627f2 4694 unsigned long flags;
99bbc707 4695 unsigned int qlen;
0a9627f2 4696
e36fa2f7 4697 sd = &per_cpu(softnet_data, cpu);
0a9627f2
TH
4698
4699 local_irq_save(flags);
0a9627f2 4700
e36fa2f7 4701 rps_lock(sd);
e9e4dd32
JA
4702 if (!netif_running(skb->dev))
4703 goto drop;
99bbc707
WB
4704 qlen = skb_queue_len(&sd->input_pkt_queue);
4705 if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
e008f3f0 4706 if (qlen) {
0a9627f2 4707enqueue:
e36fa2f7 4708 __skb_queue_tail(&sd->input_pkt_queue, skb);
76cc8b13 4709 input_queue_tail_incr_save(sd, qtail);
e36fa2f7 4710 rps_unlock(sd);
152102c7 4711 local_irq_restore(flags);
0a9627f2
TH
4712 return NET_RX_SUCCESS;
4713 }
4714
ebda37c2
ED
4715 /* Schedule NAPI for backlog device
4716 * We can use non atomic operation since we own the queue lock
4717 */
4718 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
e36fa2f7 4719 if (!rps_ipi_queued(sd))
eecfd7c4 4720 ____napi_schedule(sd, &sd->backlog);
0a9627f2
TH
4721 }
4722 goto enqueue;
4723 }
4724
e9e4dd32 4725drop:
dee42870 4726 sd->dropped++;
e36fa2f7 4727 rps_unlock(sd);
0a9627f2 4728
0a9627f2
TH
4729 local_irq_restore(flags);
4730
caf586e5 4731 atomic_long_inc(&skb->dev->rx_dropped);
0a9627f2
TH
4732 kfree_skb(skb);
4733 return NET_RX_DROP;
4734}
1da177e4 4735
e817f856
JDB
4736static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
4737{
4738 struct net_device *dev = skb->dev;
4739 struct netdev_rx_queue *rxqueue;
4740
4741 rxqueue = dev->_rx;
4742
4743 if (skb_rx_queue_recorded(skb)) {
4744 u16 index = skb_get_rx_queue(skb);
4745
4746 if (unlikely(index >= dev->real_num_rx_queues)) {
4747 WARN_ONCE(dev->real_num_rx_queues > 1,
4748 "%s received packet on queue %u, but number "
4749 "of RX queues is %u\n",
4750 dev->name, index, dev->real_num_rx_queues);
4751
4752 return rxqueue; /* Return first rxqueue */
4753 }
4754 rxqueue += index;
4755 }
4756 return rxqueue;
4757}
4758
fe21cb91
KKD
4759u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4760 struct bpf_prog *xdp_prog)
d4455169 4761{
be9df4af 4762 void *orig_data, *orig_data_end, *hard_start;
e817f856 4763 struct netdev_rx_queue *rxqueue;
22b60343 4764 bool orig_bcast, orig_host;
43b5169d 4765 u32 mac_len, frame_sz;
29724956
JDB
4766 __be16 orig_eth_type;
4767 struct ethhdr *eth;
fe21cb91 4768 u32 metalen, act;
be9df4af 4769 int off;
d4455169 4770
d4455169
JF
4771 /* The XDP program wants to see the packet starting at the MAC
4772 * header.
4773 */
4774 mac_len = skb->data - skb_mac_header(skb);
be9df4af 4775 hard_start = skb->data - skb_headroom(skb);
a075767b
JDB
4776
4777 /* SKB "head" area always have tailroom for skb_shared_info */
be9df4af 4778 frame_sz = (void *)skb_end_pointer(skb) - hard_start;
43b5169d 4779 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
a075767b 4780
be9df4af
LB
4781 rxqueue = netif_get_rxqueue(skb);
4782 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
4783 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
4784 skb_headlen(skb) + mac_len, true);
a075767b 4785
02671e23
BT
4786 orig_data_end = xdp->data_end;
4787 orig_data = xdp->data;
29724956 4788 eth = (struct ethhdr *)xdp->data;
22b60343 4789 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
29724956
JDB
4790 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
4791 orig_eth_type = eth->h_proto;
d4455169 4792
02671e23 4793 act = bpf_prog_run_xdp(xdp_prog, xdp);
d4455169 4794
065af355 4795 /* check if bpf_xdp_adjust_head was used */
02671e23 4796 off = xdp->data - orig_data;
065af355
JDB
4797 if (off) {
4798 if (off > 0)
4799 __skb_pull(skb, off);
4800 else if (off < 0)
4801 __skb_push(skb, -off);
4802
4803 skb->mac_header += off;
4804 skb_reset_network_header(skb);
4805 }
d4455169 4806
a075767b
JDB
4807 /* check if bpf_xdp_adjust_tail was used */
4808 off = xdp->data_end - orig_data_end;
f7613120 4809 if (off != 0) {
02671e23 4810 skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
a075767b 4811 skb->len += off; /* positive on grow, negative on shrink */
f7613120 4812 }
198d83bb 4813
29724956
JDB
4814 /* check if XDP changed eth hdr such SKB needs update */
4815 eth = (struct ethhdr *)xdp->data;
4816 if ((orig_eth_type != eth->h_proto) ||
22b60343
MW
4817 (orig_host != ether_addr_equal_64bits(eth->h_dest,
4818 skb->dev->dev_addr)) ||
29724956
JDB
4819 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
4820 __skb_push(skb, ETH_HLEN);
22b60343 4821 skb->pkt_type = PACKET_HOST;
29724956
JDB
4822 skb->protocol = eth_type_trans(skb, skb->dev);
4823 }
4824
fe21cb91
KKD
4825 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
4826 * before calling us again on redirect path. We do not call do_redirect
4827 * as we leave that up to the caller.
4828 *
4829 * Caller is responsible for managing lifetime of skb (i.e. calling
4830 * kfree_skb in response to actions it cannot handle/XDP_DROP).
4831 */
d4455169 4832 switch (act) {
6103aa96 4833 case XDP_REDIRECT:
d4455169
JF
4834 case XDP_TX:
4835 __skb_push(skb, mac_len);
de8f3a83 4836 break;
d4455169 4837 case XDP_PASS:
02671e23 4838 metalen = xdp->data - xdp->data_meta;
de8f3a83
DB
4839 if (metalen)
4840 skb_metadata_set(skb, metalen);
d4455169 4841 break;
fe21cb91
KKD
4842 }
4843
4844 return act;
4845}
4846
4847static u32 netif_receive_generic_xdp(struct sk_buff *skb,
4848 struct xdp_buff *xdp,
4849 struct bpf_prog *xdp_prog)
4850{
4851 u32 act = XDP_DROP;
4852
4853 /* Reinjected packets coming from act_mirred or similar should
4854 * not get XDP generic processing.
4855 */
4856 if (skb_is_redirected(skb))
4857 return XDP_PASS;
4858
4859 /* XDP packets must be linear and must have sufficient headroom
4860 * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
4861 * native XDP provides, thus we need to do it here as well.
4862 */
4863 if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
4864 skb_headroom(skb) < XDP_PACKET_HEADROOM) {
4865 int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
4866 int troom = skb->tail + skb->data_len - skb->end;
4867
4868 /* In case we have to go down the path and also linearize,
4869 * then lets do the pskb_expand_head() work just once here.
4870 */
4871 if (pskb_expand_head(skb,
4872 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
4873 troom > 0 ? troom + 128 : 0, GFP_ATOMIC))
4874 goto do_drop;
4875 if (skb_linearize(skb))
4876 goto do_drop;
4877 }
4878
4879 act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
4880 switch (act) {
4881 case XDP_REDIRECT:
4882 case XDP_TX:
4883 case XDP_PASS:
4884 break;
d4455169
JF
4885 default:
4886 bpf_warn_invalid_xdp_action(act);
df561f66 4887 fallthrough;
d4455169
JF
4888 case XDP_ABORTED:
4889 trace_xdp_exception(skb->dev, xdp_prog, act);
df561f66 4890 fallthrough;
d4455169
JF
4891 case XDP_DROP:
4892 do_drop:
4893 kfree_skb(skb);
4894 break;
4895 }
4896
4897 return act;
4898}
4899
4900/* When doing generic XDP we have to bypass the qdisc layer and the
4901 * network taps in order to match in-driver-XDP behavior.
4902 */
7c497478 4903void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
d4455169
JF
4904{
4905 struct net_device *dev = skb->dev;
4906 struct netdev_queue *txq;
4907 bool free_skb = true;
4908 int cpu, rc;
4909
4bd97d51 4910 txq = netdev_core_pick_tx(dev, skb, NULL);
d4455169
JF
4911 cpu = smp_processor_id();
4912 HARD_TX_LOCK(dev, txq, cpu);
4913 if (!netif_xmit_stopped(txq)) {
4914 rc = netdev_start_xmit(skb, dev, txq, 0);
4915 if (dev_xmit_complete(rc))
4916 free_skb = false;
4917 }
4918 HARD_TX_UNLOCK(dev, txq);
4919 if (free_skb) {
4920 trace_xdp_exception(dev, xdp_prog, XDP_TX);
4921 kfree_skb(skb);
4922 }
4923}
4924
02786475 4925static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
d4455169 4926
7c497478 4927int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
d4455169 4928{
d4455169 4929 if (xdp_prog) {
02671e23
BT
4930 struct xdp_buff xdp;
4931 u32 act;
6103aa96 4932 int err;
d4455169 4933
02671e23 4934 act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
d4455169 4935 if (act != XDP_PASS) {
6103aa96
JF
4936 switch (act) {
4937 case XDP_REDIRECT:
2facaad6 4938 err = xdp_do_generic_redirect(skb->dev, skb,
02671e23 4939 &xdp, xdp_prog);
6103aa96
JF
4940 if (err)
4941 goto out_redir;
02671e23 4942 break;
6103aa96 4943 case XDP_TX:
d4455169 4944 generic_xdp_tx(skb, xdp_prog);
6103aa96
JF
4945 break;
4946 }
d4455169
JF
4947 return XDP_DROP;
4948 }
4949 }
4950 return XDP_PASS;
6103aa96 4951out_redir:
6103aa96
JF
4952 kfree_skb(skb);
4953 return XDP_DROP;
d4455169 4954}
7c497478 4955EXPORT_SYMBOL_GPL(do_xdp_generic);
d4455169 4956
ae78dbfa 4957static int netif_rx_internal(struct sk_buff *skb)
1da177e4 4958{
b0e28f1e 4959 int ret;
1da177e4 4960
588f0330 4961 net_timestamp_check(netdev_tstamp_prequeue, skb);
1da177e4 4962
cf66ba58 4963 trace_netif_rx(skb);
d4455169 4964
df334545 4965#ifdef CONFIG_RPS
dc05360f 4966 if (static_branch_unlikely(&rps_needed)) {
fec5e652 4967 struct rps_dev_flow voidflow, *rflow = &voidflow;
b0e28f1e
ED
4968 int cpu;
4969
cece1945 4970 preempt_disable();
b0e28f1e 4971 rcu_read_lock();
fec5e652
TH
4972
4973 cpu = get_rps_cpu(skb->dev, skb, &rflow);
b0e28f1e
ED
4974 if (cpu < 0)
4975 cpu = smp_processor_id();
fec5e652
TH
4976
4977 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4978
b0e28f1e 4979 rcu_read_unlock();
cece1945 4980 preempt_enable();
adc9300e
ED
4981 } else
4982#endif
fec5e652
TH
4983 {
4984 unsigned int qtail;
f4563a75 4985
fec5e652
TH
4986 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
4987 put_cpu();
4988 }
b0e28f1e 4989 return ret;
1da177e4 4990}
ae78dbfa
BH
4991
4992/**
4993 * netif_rx - post buffer to the network code
4994 * @skb: buffer to post
4995 *
4996 * This function receives a packet from a device driver and queues it for
4997 * the upper (protocol) levels to process. It always succeeds. The buffer
4998 * may be dropped during processing for congestion control or by the
4999 * protocol layers.
5000 *
5001 * return values:
5002 * NET_RX_SUCCESS (no congestion)
5003 * NET_RX_DROP (packet was dropped)
5004 *
5005 */
5006
5007int netif_rx(struct sk_buff *skb)
5008{
b0e3f1bd
GB
5009 int ret;
5010
ae78dbfa
BH
5011 trace_netif_rx_entry(skb);
5012
b0e3f1bd
GB
5013 ret = netif_rx_internal(skb);
5014 trace_netif_rx_exit(ret);
5015
5016 return ret;
ae78dbfa 5017}
d1b19dff 5018EXPORT_SYMBOL(netif_rx);
1da177e4
LT
5019
5020int netif_rx_ni(struct sk_buff *skb)
5021{
5022 int err;
5023
ae78dbfa
BH
5024 trace_netif_rx_ni_entry(skb);
5025
1da177e4 5026 preempt_disable();
ae78dbfa 5027 err = netif_rx_internal(skb);
1da177e4
LT
5028 if (local_softirq_pending())
5029 do_softirq();
5030 preempt_enable();
b0e3f1bd 5031 trace_netif_rx_ni_exit(err);
1da177e4
LT
5032
5033 return err;
5034}
1da177e4
LT
5035EXPORT_SYMBOL(netif_rx_ni);
5036
c11171a4
SAS
5037int netif_rx_any_context(struct sk_buff *skb)
5038{
5039 /*
5040 * If invoked from contexts which do not invoke bottom half
5041 * processing either at return from interrupt or when softrqs are
5042 * reenabled, use netif_rx_ni() which invokes bottomhalf processing
5043 * directly.
5044 */
5045 if (in_interrupt())
5046 return netif_rx(skb);
5047 else
5048 return netif_rx_ni(skb);
5049}
5050EXPORT_SYMBOL(netif_rx_any_context);
5051
0766f788 5052static __latent_entropy void net_tx_action(struct softirq_action *h)
1da177e4 5053{
903ceff7 5054 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
1da177e4
LT
5055
5056 if (sd->completion_queue) {
5057 struct sk_buff *clist;
5058
5059 local_irq_disable();
5060 clist = sd->completion_queue;
5061 sd->completion_queue = NULL;
5062 local_irq_enable();
5063
5064 while (clist) {
5065 struct sk_buff *skb = clist;
f4563a75 5066
1da177e4
LT
5067 clist = clist->next;
5068
63354797 5069 WARN_ON(refcount_read(&skb->users));
e6247027
ED
5070 if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
5071 trace_consume_skb(skb);
5072 else
5073 trace_kfree_skb(skb, net_tx_action);
15fad714
JDB
5074
5075 if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
5076 __kfree_skb(skb);
5077 else
5078 __kfree_skb_defer(skb);
1da177e4
LT
5079 }
5080 }
5081
5082 if (sd->output_queue) {
37437bb2 5083 struct Qdisc *head;
1da177e4
LT
5084
5085 local_irq_disable();
5086 head = sd->output_queue;
5087 sd->output_queue = NULL;
a9cbd588 5088 sd->output_queue_tailp = &sd->output_queue;
1da177e4
LT
5089 local_irq_enable();
5090
102b55ee
YL
5091 rcu_read_lock();
5092
1da177e4 5093 while (head) {
37437bb2 5094 struct Qdisc *q = head;
6b3ba914 5095 spinlock_t *root_lock = NULL;
37437bb2 5096
1da177e4
LT
5097 head = head->next_sched;
5098
3bcb846c
ED
5099 /* We need to make sure head->next_sched is read
5100 * before clearing __QDISC_STATE_SCHED
5101 */
5102 smp_mb__before_atomic();
102b55ee
YL
5103
5104 if (!(q->flags & TCQ_F_NOLOCK)) {
5105 root_lock = qdisc_lock(q);
5106 spin_lock(root_lock);
5107 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
5108 &q->state))) {
5109 /* There is a synchronize_net() between
5110 * STATE_DEACTIVATED flag being set and
5111 * qdisc_reset()/some_qdisc_is_busy() in
5112 * dev_deactivate(), so we can safely bail out
5113 * early here to avoid data race between
5114 * qdisc_deactivate() and some_qdisc_is_busy()
5115 * for lockless qdisc.
5116 */
5117 clear_bit(__QDISC_STATE_SCHED, &q->state);
5118 continue;
5119 }
5120
3bcb846c
ED
5121 clear_bit(__QDISC_STATE_SCHED, &q->state);
5122 qdisc_run(q);
6b3ba914
JF
5123 if (root_lock)
5124 spin_unlock(root_lock);
1da177e4 5125 }
102b55ee
YL
5126
5127 rcu_read_unlock();
1da177e4 5128 }
f53c7239
SK
5129
5130 xfrm_dev_backlog(sd);
1da177e4
LT
5131}
5132
181402a5 5133#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_ATM_LANE)
da678292
MM
5134/* This hook is defined here for ATM LANE */
5135int (*br_fdb_test_addr_hook)(struct net_device *dev,
5136 unsigned char *addr) __read_mostly;
4fb019a0 5137EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
da678292 5138#endif
1da177e4 5139
1f211a1b
DB
5140static inline struct sk_buff *
5141sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
9aa1206e 5142 struct net_device *orig_dev, bool *another)
f697c3e8 5143{
e7582bab 5144#ifdef CONFIG_NET_CLS_ACT
46209401 5145 struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
d2788d34 5146 struct tcf_result cl_res;
24824a09 5147
c9e99fd0
DB
5148 /* If there's at least one ingress present somewhere (so
5149 * we get here via enabled static key), remaining devices
5150 * that are not configured with an ingress qdisc will bail
d2788d34 5151 * out here.
c9e99fd0 5152 */
46209401 5153 if (!miniq)
4577139b 5154 return skb;
46209401 5155
f697c3e8
HX
5156 if (*pt_prev) {
5157 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5158 *pt_prev = NULL;
1da177e4
LT
5159 }
5160
3365495c 5161 qdisc_skb_cb(skb)->pkt_len = skb->len;
aadaca9e 5162 qdisc_skb_cb(skb)->mru = 0;
7baf2429 5163 qdisc_skb_cb(skb)->post_ct = false;
8dc07fdb 5164 skb->tc_at_ingress = 1;
46209401 5165 mini_qdisc_bstats_cpu_update(miniq, skb);
c9e99fd0 5166
7d17c544
PB
5167 switch (tcf_classify_ingress(skb, miniq->block, miniq->filter_list,
5168 &cl_res, false)) {
d2788d34
DB
5169 case TC_ACT_OK:
5170 case TC_ACT_RECLASSIFY:
5171 skb->tc_index = TC_H_MIN(cl_res.classid);
5172 break;
5173 case TC_ACT_SHOT:
46209401 5174 mini_qdisc_qstats_cpu_drop(miniq);
8a3a4c6e
ED
5175 kfree_skb(skb);
5176 return NULL;
d2788d34
DB
5177 case TC_ACT_STOLEN:
5178 case TC_ACT_QUEUED:
e25ea21f 5179 case TC_ACT_TRAP:
8a3a4c6e 5180 consume_skb(skb);
d2788d34 5181 return NULL;
27b29f63
AS
5182 case TC_ACT_REDIRECT:
5183 /* skb_mac_header check was done by cls/act_bpf, so
5184 * we can safely push the L2 header back before
5185 * redirecting to another netdev
5186 */
5187 __skb_push(skb, skb->mac_len);
9aa1206e
DB
5188 if (skb_do_redirect(skb) == -EAGAIN) {
5189 __skb_pull(skb, skb->mac_len);
5190 *another = true;
5191 break;
5192 }
27b29f63 5193 return NULL;
720f22fe 5194 case TC_ACT_CONSUMED:
cd11b164 5195 return NULL;
d2788d34
DB
5196 default:
5197 break;
f697c3e8 5198 }
e7582bab 5199#endif /* CONFIG_NET_CLS_ACT */
e687ad60
PN
5200 return skb;
5201}
1da177e4 5202
24b27fc4
MB
5203/**
5204 * netdev_is_rx_handler_busy - check if receive handler is registered
5205 * @dev: device to check
5206 *
5207 * Check if a receive handler is already registered for a given device.
5208 * Return true if there one.
5209 *
5210 * The caller must hold the rtnl_mutex.
5211 */
5212bool netdev_is_rx_handler_busy(struct net_device *dev)
5213{
5214 ASSERT_RTNL();
5215 return dev && rtnl_dereference(dev->rx_handler);
5216}
5217EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
5218
ab95bfe0
JP
5219/**
5220 * netdev_rx_handler_register - register receive handler
5221 * @dev: device to register a handler for
5222 * @rx_handler: receive handler to register
93e2c32b 5223 * @rx_handler_data: data pointer that is used by rx handler
ab95bfe0 5224 *
e227867f 5225 * Register a receive handler for a device. This handler will then be
ab95bfe0
JP
5226 * called from __netif_receive_skb. A negative errno code is returned
5227 * on a failure.
5228 *
5229 * The caller must hold the rtnl_mutex.
8a4eb573
JP
5230 *
5231 * For a general description of rx_handler, see enum rx_handler_result.
ab95bfe0
JP
5232 */
5233int netdev_rx_handler_register(struct net_device *dev,
93e2c32b
JP
5234 rx_handler_func_t *rx_handler,
5235 void *rx_handler_data)
ab95bfe0 5236{
1b7cd004 5237 if (netdev_is_rx_handler_busy(dev))
ab95bfe0
JP
5238 return -EBUSY;
5239
f5426250
PA
5240 if (dev->priv_flags & IFF_NO_RX_HANDLER)
5241 return -EINVAL;
5242
00cfec37 5243 /* Note: rx_handler_data must be set before rx_handler */
93e2c32b 5244 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
ab95bfe0
JP
5245 rcu_assign_pointer(dev->rx_handler, rx_handler);
5246
5247 return 0;
5248}
5249EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
5250
5251/**
5252 * netdev_rx_handler_unregister - unregister receive handler
5253 * @dev: device to unregister a handler from
5254 *
166ec369 5255 * Unregister a receive handler from a device.
ab95bfe0
JP
5256 *
5257 * The caller must hold the rtnl_mutex.
5258 */
5259void netdev_rx_handler_unregister(struct net_device *dev)
5260{
5261
5262 ASSERT_RTNL();
a9b3cd7f 5263 RCU_INIT_POINTER(dev->rx_handler, NULL);
00cfec37
ED
5264 /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
5265 * section has a guarantee to see a non NULL rx_handler_data
5266 * as well.
5267 */
5268 synchronize_net();
a9b3cd7f 5269 RCU_INIT_POINTER(dev->rx_handler_data, NULL);
ab95bfe0
JP
5270}
5271EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
5272
b4b9e355
MG
5273/*
5274 * Limit the use of PFMEMALLOC reserves to those protocols that implement
5275 * the special handling of PFMEMALLOC skbs.
5276 */
5277static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
5278{
5279 switch (skb->protocol) {
2b8837ae
JP
5280 case htons(ETH_P_ARP):
5281 case htons(ETH_P_IP):
5282 case htons(ETH_P_IPV6):
5283 case htons(ETH_P_8021Q):
5284 case htons(ETH_P_8021AD):
b4b9e355
MG
5285 return true;
5286 default:
5287 return false;
5288 }
5289}
5290
e687ad60
PN
5291static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
5292 int *ret, struct net_device *orig_dev)
5293{
5294 if (nf_hook_ingress_active(skb)) {
2c1e2703
AC
5295 int ingress_retval;
5296
e687ad60
PN
5297 if (*pt_prev) {
5298 *ret = deliver_skb(skb, *pt_prev, orig_dev);
5299 *pt_prev = NULL;
5300 }
5301
2c1e2703
AC
5302 rcu_read_lock();
5303 ingress_retval = nf_hook_ingress(skb);
5304 rcu_read_unlock();
5305 return ingress_retval;
e687ad60
PN
5306 }
5307 return 0;
5308}
e687ad60 5309
c0bbbdc3 5310static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
88eb1944 5311 struct packet_type **ppt_prev)
1da177e4
LT
5312{
5313 struct packet_type *ptype, *pt_prev;
ab95bfe0 5314 rx_handler_func_t *rx_handler;
c0bbbdc3 5315 struct sk_buff *skb = *pskb;
f2ccd8fa 5316 struct net_device *orig_dev;
8a4eb573 5317 bool deliver_exact = false;
1da177e4 5318 int ret = NET_RX_DROP;
252e3346 5319 __be16 type;
1da177e4 5320
588f0330 5321 net_timestamp_check(!netdev_tstamp_prequeue, skb);
81bbb3d4 5322
cf66ba58 5323 trace_netif_receive_skb(skb);
9b22ea56 5324
cc9bd5ce 5325 orig_dev = skb->dev;
8f903c70 5326
c1d2bbe1 5327 skb_reset_network_header(skb);
fda55eca
ED
5328 if (!skb_transport_header_was_set(skb))
5329 skb_reset_transport_header(skb);
0b5c9db1 5330 skb_reset_mac_len(skb);
1da177e4
LT
5331
5332 pt_prev = NULL;
5333
63d8ea7f 5334another_round:
b6858177 5335 skb->skb_iif = skb->dev->ifindex;
63d8ea7f
DM
5336
5337 __this_cpu_inc(softnet_data.processed);
5338
458bf2f2
SH
5339 if (static_branch_unlikely(&generic_xdp_needed_key)) {
5340 int ret2;
5341
2b4cd14f 5342 migrate_disable();
458bf2f2 5343 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
2b4cd14f 5344 migrate_enable();
458bf2f2 5345
c0bbbdc3
BS
5346 if (ret2 != XDP_PASS) {
5347 ret = NET_RX_DROP;
5348 goto out;
5349 }
458bf2f2
SH
5350 }
5351
324cefaf 5352 if (eth_type_vlan(skb->protocol)) {
0d5501c1 5353 skb = skb_vlan_untag(skb);
bcc6d479 5354 if (unlikely(!skb))
2c17d27c 5355 goto out;
bcc6d479
JP
5356 }
5357
e7246e12
WB
5358 if (skb_skip_tc_classify(skb))
5359 goto skip_classify;
1da177e4 5360
9754e293 5361 if (pfmemalloc)
b4b9e355
MG
5362 goto skip_taps;
5363
1da177e4 5364 list_for_each_entry_rcu(ptype, &ptype_all, list) {
7866a621
SN
5365 if (pt_prev)
5366 ret = deliver_skb(skb, pt_prev, orig_dev);
5367 pt_prev = ptype;
5368 }
5369
5370 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
5371 if (pt_prev)
5372 ret = deliver_skb(skb, pt_prev, orig_dev);
5373 pt_prev = ptype;
1da177e4
LT
5374 }
5375
b4b9e355 5376skip_taps:
1cf51900 5377#ifdef CONFIG_NET_INGRESS
aabf6772 5378 if (static_branch_unlikely(&ingress_needed_key)) {
9aa1206e
DB
5379 bool another = false;
5380
5381 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
5382 &another);
5383 if (another)
5384 goto another_round;
4577139b 5385 if (!skb)
2c17d27c 5386 goto out;
e687ad60
PN
5387
5388 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
2c17d27c 5389 goto out;
4577139b 5390 }
1cf51900 5391#endif
2c64605b 5392 skb_reset_redirect(skb);
e7246e12 5393skip_classify:
9754e293 5394 if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
b4b9e355
MG
5395 goto drop;
5396
df8a39de 5397 if (skb_vlan_tag_present(skb)) {
2425717b
JF
5398 if (pt_prev) {
5399 ret = deliver_skb(skb, pt_prev, orig_dev);
5400 pt_prev = NULL;
5401 }
48cc32d3 5402 if (vlan_do_receive(&skb))
2425717b
JF
5403 goto another_round;
5404 else if (unlikely(!skb))
2c17d27c 5405 goto out;
2425717b
JF
5406 }
5407
48cc32d3 5408 rx_handler = rcu_dereference(skb->dev->rx_handler);
ab95bfe0
JP
5409 if (rx_handler) {
5410 if (pt_prev) {
5411 ret = deliver_skb(skb, pt_prev, orig_dev);
5412 pt_prev = NULL;
5413 }
8a4eb573
JP
5414 switch (rx_handler(&skb)) {
5415 case RX_HANDLER_CONSUMED:
3bc1b1ad 5416 ret = NET_RX_SUCCESS;
2c17d27c 5417 goto out;
8a4eb573 5418 case RX_HANDLER_ANOTHER:
63d8ea7f 5419 goto another_round;
8a4eb573
JP
5420 case RX_HANDLER_EXACT:
5421 deliver_exact = true;
b1866bff 5422 break;
8a4eb573
JP
5423 case RX_HANDLER_PASS:
5424 break;
5425 default:
5426 BUG();
5427 }
ab95bfe0 5428 }
1da177e4 5429
b14a9fc4 5430 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
36b2f61a
GV
5431check_vlan_id:
5432 if (skb_vlan_tag_get_id(skb)) {
5433 /* Vlan id is non 0 and vlan_do_receive() above couldn't
5434 * find vlan device.
5435 */
d4b812de 5436 skb->pkt_type = PACKET_OTHERHOST;
324cefaf 5437 } else if (eth_type_vlan(skb->protocol)) {
36b2f61a
GV
5438 /* Outer header is 802.1P with vlan 0, inner header is
5439 * 802.1Q or 802.1AD and vlan_do_receive() above could
5440 * not find vlan dev for vlan id 0.
5441 */
5442 __vlan_hwaccel_clear_tag(skb);
5443 skb = skb_vlan_untag(skb);
5444 if (unlikely(!skb))
5445 goto out;
5446 if (vlan_do_receive(&skb))
5447 /* After stripping off 802.1P header with vlan 0
5448 * vlan dev is found for inner header.
5449 */
5450 goto another_round;
5451 else if (unlikely(!skb))
5452 goto out;
5453 else
5454 /* We have stripped outer 802.1P vlan 0 header.
5455 * But could not find vlan dev.
5456 * check again for vlan id to set OTHERHOST.
5457 */
5458 goto check_vlan_id;
5459 }
d4b812de
ED
5460 /* Note: we might in the future use prio bits
5461 * and set skb->priority like in vlan_do_receive()
5462 * For the time being, just ignore Priority Code Point
5463 */
b1817524 5464 __vlan_hwaccel_clear_tag(skb);
d4b812de 5465 }
48cc32d3 5466
7866a621
SN
5467 type = skb->protocol;
5468
63d8ea7f 5469 /* deliver only exact match when indicated */
7866a621
SN
5470 if (likely(!deliver_exact)) {
5471 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5472 &ptype_base[ntohs(type) &
5473 PTYPE_HASH_MASK]);
5474 }
1f3c8804 5475
7866a621
SN
5476 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5477 &orig_dev->ptype_specific);
5478
5479 if (unlikely(skb->dev != orig_dev)) {
5480 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
5481 &skb->dev->ptype_specific);
1da177e4
LT
5482 }
5483
5484 if (pt_prev) {
1f8b977a 5485 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
0e698bf6 5486 goto drop;
88eb1944 5487 *ppt_prev = pt_prev;
1da177e4 5488 } else {
b4b9e355 5489drop:
6e7333d3
JW
5490 if (!deliver_exact)
5491 atomic_long_inc(&skb->dev->rx_dropped);
5492 else
5493 atomic_long_inc(&skb->dev->rx_nohandler);
1da177e4
LT
5494 kfree_skb(skb);
5495 /* Jamal, now you will not able to escape explaining
5496 * me how you were going to use this. :-)
5497 */
5498 ret = NET_RX_DROP;
5499 }
5500
2c17d27c 5501out:
c0bbbdc3
BS
5502 /* The invariant here is that if *ppt_prev is not NULL
5503 * then skb should also be non-NULL.
5504 *
5505 * Apparently *ppt_prev assignment above holds this invariant due to
5506 * skb dereferencing near it.
5507 */
5508 *pskb = skb;
9754e293
DM
5509 return ret;
5510}
5511
88eb1944
EC
5512static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
5513{
5514 struct net_device *orig_dev = skb->dev;
5515 struct packet_type *pt_prev = NULL;
5516 int ret;
5517
c0bbbdc3 5518 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
88eb1944 5519 if (pt_prev)
f5737cba
PA
5520 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
5521 skb->dev, pt_prev, orig_dev);
88eb1944
EC
5522 return ret;
5523}
5524
1c601d82
JDB
5525/**
5526 * netif_receive_skb_core - special purpose version of netif_receive_skb
5527 * @skb: buffer to process
5528 *
5529 * More direct receive version of netif_receive_skb(). It should
5530 * only be used by callers that have a need to skip RPS and Generic XDP.
2de9780f 5531 * Caller must also take care of handling if ``(page_is_)pfmemalloc``.
1c601d82
JDB
5532 *
5533 * This function may only be called from softirq context and interrupts
5534 * should be enabled.
5535 *
5536 * Return values (usually ignored):
5537 * NET_RX_SUCCESS: no congestion
5538 * NET_RX_DROP: packet was dropped
5539 */
5540int netif_receive_skb_core(struct sk_buff *skb)
5541{
5542 int ret;
5543
5544 rcu_read_lock();
88eb1944 5545 ret = __netif_receive_skb_one_core(skb, false);
1c601d82
JDB
5546 rcu_read_unlock();
5547
5548 return ret;
5549}
5550EXPORT_SYMBOL(netif_receive_skb_core);
5551
88eb1944
EC
5552static inline void __netif_receive_skb_list_ptype(struct list_head *head,
5553 struct packet_type *pt_prev,
5554 struct net_device *orig_dev)
4ce0017a
EC
5555{
5556 struct sk_buff *skb, *next;
5557
88eb1944
EC
5558 if (!pt_prev)
5559 return;
5560 if (list_empty(head))
5561 return;
17266ee9 5562 if (pt_prev->list_func != NULL)
fdf71426
PA
5563 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
5564 ip_list_rcv, head, pt_prev, orig_dev);
17266ee9 5565 else
9a5a90d1
AL
5566 list_for_each_entry_safe(skb, next, head, list) {
5567 skb_list_del_init(skb);
fdf71426 5568 pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
9a5a90d1 5569 }
88eb1944
EC
5570}
5571
5572static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
5573{
5574 /* Fast-path assumptions:
5575 * - There is no RX handler.
5576 * - Only one packet_type matches.
5577 * If either of these fails, we will end up doing some per-packet
5578 * processing in-line, then handling the 'last ptype' for the whole
5579 * sublist. This can't cause out-of-order delivery to any single ptype,
5580 * because the 'last ptype' must be constant across the sublist, and all
5581 * other ptypes are handled per-packet.
5582 */
5583 /* Current (common) ptype of sublist */
5584 struct packet_type *pt_curr = NULL;
5585 /* Current (common) orig_dev of sublist */
5586 struct net_device *od_curr = NULL;
5587 struct list_head sublist;
5588 struct sk_buff *skb, *next;
5589
9af86f93 5590 INIT_LIST_HEAD(&sublist);
88eb1944
EC
5591 list_for_each_entry_safe(skb, next, head, list) {
5592 struct net_device *orig_dev = skb->dev;
5593 struct packet_type *pt_prev = NULL;
5594
22f6bbb7 5595 skb_list_del_init(skb);
c0bbbdc3 5596 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
9af86f93
EC
5597 if (!pt_prev)
5598 continue;
88eb1944
EC
5599 if (pt_curr != pt_prev || od_curr != orig_dev) {
5600 /* dispatch old sublist */
88eb1944
EC
5601 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
5602 /* start new sublist */
9af86f93 5603 INIT_LIST_HEAD(&sublist);
88eb1944
EC
5604 pt_curr = pt_prev;
5605 od_curr = orig_dev;
5606 }
9af86f93 5607 list_add_tail(&skb->list, &sublist);
88eb1944
EC
5608 }
5609
5610 /* dispatch final sublist */
9af86f93 5611 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
4ce0017a
EC
5612}
5613
9754e293
DM
5614static int __netif_receive_skb(struct sk_buff *skb)
5615{
5616 int ret;
5617
5618 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
f1083048 5619 unsigned int noreclaim_flag;
9754e293
DM
5620
5621 /*
5622 * PFMEMALLOC skbs are special, they should
5623 * - be delivered to SOCK_MEMALLOC sockets only
5624 * - stay away from userspace
5625 * - have bounded memory usage
5626 *
5627 * Use PF_MEMALLOC as this saves us from propagating the allocation
5628 * context down to all allocation sites.
5629 */
f1083048 5630 noreclaim_flag = memalloc_noreclaim_save();
88eb1944 5631 ret = __netif_receive_skb_one_core(skb, true);
f1083048 5632 memalloc_noreclaim_restore(noreclaim_flag);
9754e293 5633 } else
88eb1944 5634 ret = __netif_receive_skb_one_core(skb, false);
9754e293 5635
1da177e4
LT
5636 return ret;
5637}
0a9627f2 5638
4ce0017a
EC
5639static void __netif_receive_skb_list(struct list_head *head)
5640{
5641 unsigned long noreclaim_flag = 0;
5642 struct sk_buff *skb, *next;
5643 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
5644
5645 list_for_each_entry_safe(skb, next, head, list) {
5646 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
5647 struct list_head sublist;
5648
5649 /* Handle the previous sublist */
5650 list_cut_before(&sublist, head, &skb->list);
b9f463d6
EC
5651 if (!list_empty(&sublist))
5652 __netif_receive_skb_list_core(&sublist, pfmemalloc);
4ce0017a
EC
5653 pfmemalloc = !pfmemalloc;
5654 /* See comments in __netif_receive_skb */
5655 if (pfmemalloc)
5656 noreclaim_flag = memalloc_noreclaim_save();
5657 else
5658 memalloc_noreclaim_restore(noreclaim_flag);
5659 }
5660 }
5661 /* Handle the remaining sublist */
b9f463d6
EC
5662 if (!list_empty(head))
5663 __netif_receive_skb_list_core(head, pfmemalloc);
4ce0017a
EC
5664 /* Restore pflags */
5665 if (pfmemalloc)
5666 memalloc_noreclaim_restore(noreclaim_flag);
5667}
5668
f4e63525 5669static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
b5cdae32 5670{
58038695 5671 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
b5cdae32
DM
5672 struct bpf_prog *new = xdp->prog;
5673 int ret = 0;
5674
5675 switch (xdp->command) {
58038695 5676 case XDP_SETUP_PROG:
b5cdae32
DM
5677 rcu_assign_pointer(dev->xdp_prog, new);
5678 if (old)
5679 bpf_prog_put(old);
5680
5681 if (old && !new) {
02786475 5682 static_branch_dec(&generic_xdp_needed_key);
b5cdae32 5683 } else if (new && !old) {
02786475 5684 static_branch_inc(&generic_xdp_needed_key);
b5cdae32 5685 dev_disable_lro(dev);
56f5aa77 5686 dev_disable_gro_hw(dev);
b5cdae32
DM
5687 }
5688 break;
b5cdae32 5689
b5cdae32
DM
5690 default:
5691 ret = -EINVAL;
5692 break;
5693 }
5694
5695 return ret;
5696}
5697
ae78dbfa 5698static int netif_receive_skb_internal(struct sk_buff *skb)
0a9627f2 5699{
2c17d27c
JA
5700 int ret;
5701
588f0330 5702 net_timestamp_check(netdev_tstamp_prequeue, skb);
3b098e2d 5703
c1f19b51
RC
5704 if (skb_defer_rx_timestamp(skb))
5705 return NET_RX_SUCCESS;
5706
bbbe211c 5707 rcu_read_lock();
df334545 5708#ifdef CONFIG_RPS
dc05360f 5709 if (static_branch_unlikely(&rps_needed)) {
3b098e2d 5710 struct rps_dev_flow voidflow, *rflow = &voidflow;
2c17d27c 5711 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
0a9627f2 5712
3b098e2d
ED
5713 if (cpu >= 0) {
5714 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5715 rcu_read_unlock();
adc9300e 5716 return ret;
3b098e2d 5717 }
fec5e652 5718 }
1e94d72f 5719#endif
2c17d27c
JA
5720 ret = __netif_receive_skb(skb);
5721 rcu_read_unlock();
5722 return ret;
0a9627f2 5723}
ae78dbfa 5724
7da517a3
EC
5725static void netif_receive_skb_list_internal(struct list_head *head)
5726{
7da517a3 5727 struct sk_buff *skb, *next;
8c057efa 5728 struct list_head sublist;
7da517a3 5729
8c057efa 5730 INIT_LIST_HEAD(&sublist);
7da517a3
EC
5731 list_for_each_entry_safe(skb, next, head, list) {
5732 net_timestamp_check(netdev_tstamp_prequeue, skb);
22f6bbb7 5733 skb_list_del_init(skb);
8c057efa
EC
5734 if (!skb_defer_rx_timestamp(skb))
5735 list_add_tail(&skb->list, &sublist);
7da517a3 5736 }
8c057efa 5737 list_splice_init(&sublist, head);
7da517a3 5738
7da517a3
EC
5739 rcu_read_lock();
5740#ifdef CONFIG_RPS
dc05360f 5741 if (static_branch_unlikely(&rps_needed)) {
7da517a3
EC
5742 list_for_each_entry_safe(skb, next, head, list) {
5743 struct rps_dev_flow voidflow, *rflow = &voidflow;
5744 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
5745
5746 if (cpu >= 0) {
8c057efa 5747 /* Will be handled, remove from list */
22f6bbb7 5748 skb_list_del_init(skb);
8c057efa 5749 enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
7da517a3
EC
5750 }
5751 }
5752 }
5753#endif
5754 __netif_receive_skb_list(head);
5755 rcu_read_unlock();
5756}
5757
ae78dbfa
BH
5758/**
5759 * netif_receive_skb - process receive buffer from network
5760 * @skb: buffer to process
5761 *
5762 * netif_receive_skb() is the main receive data processing function.
5763 * It always succeeds. The buffer may be dropped during processing
5764 * for congestion control or by the protocol layers.
5765 *
5766 * This function may only be called from softirq context and interrupts
5767 * should be enabled.
5768 *
5769 * Return values (usually ignored):
5770 * NET_RX_SUCCESS: no congestion
5771 * NET_RX_DROP: packet was dropped
5772 */
04eb4489 5773int netif_receive_skb(struct sk_buff *skb)
ae78dbfa 5774{
b0e3f1bd
GB
5775 int ret;
5776
ae78dbfa
BH
5777 trace_netif_receive_skb_entry(skb);
5778
b0e3f1bd
GB
5779 ret = netif_receive_skb_internal(skb);
5780 trace_netif_receive_skb_exit(ret);
5781
5782 return ret;
ae78dbfa 5783}
04eb4489 5784EXPORT_SYMBOL(netif_receive_skb);
1da177e4 5785
f6ad8c1b
EC
5786/**
5787 * netif_receive_skb_list - process many receive buffers from network
5788 * @head: list of skbs to process.
5789 *
7da517a3
EC
5790 * Since return value of netif_receive_skb() is normally ignored, and
5791 * wouldn't be meaningful for a list, this function returns void.
f6ad8c1b
EC
5792 *
5793 * This function may only be called from softirq context and interrupts
5794 * should be enabled.
5795 */
5796void netif_receive_skb_list(struct list_head *head)
5797{
7da517a3 5798 struct sk_buff *skb;
f6ad8c1b 5799
b9f463d6
EC
5800 if (list_empty(head))
5801 return;
b0e3f1bd
GB
5802 if (trace_netif_receive_skb_list_entry_enabled()) {
5803 list_for_each_entry(skb, head, list)
5804 trace_netif_receive_skb_list_entry(skb);
5805 }
7da517a3 5806 netif_receive_skb_list_internal(head);
b0e3f1bd 5807 trace_netif_receive_skb_list_exit(0);
f6ad8c1b
EC
5808}
5809EXPORT_SYMBOL(netif_receive_skb_list);
5810
ce1e2a77 5811static DEFINE_PER_CPU(struct work_struct, flush_works);
145dd5f9
PA
5812
5813/* Network device is going away, flush any packets still pending */
5814static void flush_backlog(struct work_struct *work)
6e583ce5 5815{
6e583ce5 5816 struct sk_buff *skb, *tmp;
145dd5f9
PA
5817 struct softnet_data *sd;
5818
5819 local_bh_disable();
5820 sd = this_cpu_ptr(&softnet_data);
6e583ce5 5821
145dd5f9 5822 local_irq_disable();
e36fa2f7 5823 rps_lock(sd);
6e7676c1 5824 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
41852497 5825 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
e36fa2f7 5826 __skb_unlink(skb, &sd->input_pkt_queue);
7df5cb75 5827 dev_kfree_skb_irq(skb);
76cc8b13 5828 input_queue_head_incr(sd);
6e583ce5 5829 }
6e7676c1 5830 }
e36fa2f7 5831 rps_unlock(sd);
145dd5f9 5832 local_irq_enable();
6e7676c1
CG
5833
5834 skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
41852497 5835 if (skb->dev->reg_state == NETREG_UNREGISTERING) {
6e7676c1
CG
5836 __skb_unlink(skb, &sd->process_queue);
5837 kfree_skb(skb);
76cc8b13 5838 input_queue_head_incr(sd);
6e7676c1
CG
5839 }
5840 }
145dd5f9
PA
5841 local_bh_enable();
5842}
5843
2de79ee2
PA
5844static bool flush_required(int cpu)
5845{
5846#if IS_ENABLED(CONFIG_RPS)
5847 struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5848 bool do_flush;
5849
5850 local_irq_disable();
5851 rps_lock(sd);
5852
5853 /* as insertion into process_queue happens with the rps lock held,
5854 * process_queue access may race only with dequeue
5855 */
5856 do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
5857 !skb_queue_empty_lockless(&sd->process_queue);
5858 rps_unlock(sd);
5859 local_irq_enable();
5860
5861 return do_flush;
5862#endif
5863 /* without RPS we can't safely check input_pkt_queue: during a
5864 * concurrent remote skb_queue_splice() we can detect as empty both
5865 * input_pkt_queue and process_queue even if the latter could end-up
5866 * containing a lot of packets.
5867 */
5868 return true;
5869}
5870
41852497 5871static void flush_all_backlogs(void)
145dd5f9 5872{
2de79ee2 5873 static cpumask_t flush_cpus;
145dd5f9
PA
5874 unsigned int cpu;
5875
2de79ee2
PA
5876 /* since we are under rtnl lock protection we can use static data
5877 * for the cpumask and avoid allocating on stack the possibly
5878 * large mask
5879 */
5880 ASSERT_RTNL();
5881
145dd5f9
PA
5882 get_online_cpus();
5883
2de79ee2
PA
5884 cpumask_clear(&flush_cpus);
5885 for_each_online_cpu(cpu) {
5886 if (flush_required(cpu)) {
5887 queue_work_on(cpu, system_highpri_wq,
5888 per_cpu_ptr(&flush_works, cpu));
5889 cpumask_set_cpu(cpu, &flush_cpus);
5890 }
5891 }
145dd5f9 5892
2de79ee2 5893 /* we can have in flight packet[s] on the cpus we are not flushing,
0cbe1e57 5894 * synchronize_net() in unregister_netdevice_many() will take care of
2de79ee2
PA
5895 * them
5896 */
5897 for_each_cpu(cpu, &flush_cpus)
41852497 5898 flush_work(per_cpu_ptr(&flush_works, cpu));
145dd5f9
PA
5899
5900 put_online_cpus();
6e583ce5
SH
5901}
5902
c8079432
MM
5903/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
5904static void gro_normal_list(struct napi_struct *napi)
5905{
5906 if (!napi->rx_count)
5907 return;
5908 netif_receive_skb_list_internal(&napi->rx_list);
5909 INIT_LIST_HEAD(&napi->rx_list);
5910 napi->rx_count = 0;
5911}
5912
5913/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
5914 * pass the whole batch up to the stack.
5915 */
8dc1c444 5916static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs)
c8079432
MM
5917{
5918 list_add_tail(&skb->list, &napi->rx_list);
8dc1c444
ED
5919 napi->rx_count += segs;
5920 if (napi->rx_count >= gro_normal_batch)
c8079432
MM
5921 gro_normal_list(napi);
5922}
5923
c8079432 5924static int napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
d565b0a1 5925{
22061d80 5926 struct packet_offload *ptype;
d565b0a1 5927 __be16 type = skb->protocol;
22061d80 5928 struct list_head *head = &offload_base;
d565b0a1
HX
5929 int err = -ENOENT;
5930
c3c7c254
ED
5931 BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
5932
fc59f9a3
HX
5933 if (NAPI_GRO_CB(skb)->count == 1) {
5934 skb_shinfo(skb)->gso_size = 0;
d565b0a1 5935 goto out;
fc59f9a3 5936 }
d565b0a1
HX
5937
5938 rcu_read_lock();
5939 list_for_each_entry_rcu(ptype, head, list) {
f191a1d1 5940 if (ptype->type != type || !ptype->callbacks.gro_complete)
d565b0a1
HX
5941 continue;
5942
aaa5d90b
PA
5943 err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
5944 ipv6_gro_complete, inet_gro_complete,
5945 skb, 0);
d565b0a1
HX
5946 break;
5947 }
5948 rcu_read_unlock();
5949
5950 if (err) {
5951 WARN_ON(&ptype->list == head);
5952 kfree_skb(skb);
5953 return NET_RX_SUCCESS;
5954 }
5955
5956out:
8dc1c444 5957 gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count);
c8079432 5958 return NET_RX_SUCCESS;
d565b0a1
HX
5959}
5960
6312fe77 5961static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index,
07d78363 5962 bool flush_old)
d565b0a1 5963{
6312fe77 5964 struct list_head *head = &napi->gro_hash[index].list;
d4546c25 5965 struct sk_buff *skb, *p;
2e71a6f8 5966
07d78363 5967 list_for_each_entry_safe_reverse(skb, p, head, list) {
2e71a6f8
ED
5968 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
5969 return;
992cba7e 5970 skb_list_del_init(skb);
c8079432 5971 napi_gro_complete(napi, skb);
6312fe77 5972 napi->gro_hash[index].count--;
d565b0a1 5973 }
d9f37d01
LR
5974
5975 if (!napi->gro_hash[index].count)
5976 __clear_bit(index, &napi->gro_bitmask);
d565b0a1 5977}
07d78363 5978
6312fe77 5979/* napi->gro_hash[].list contains packets ordered by age.
07d78363
DM
5980 * youngest packets at the head of it.
5981 * Complete skbs in reverse order to reduce latencies.
5982 */
5983void napi_gro_flush(struct napi_struct *napi, bool flush_old)
5984{
42519ede
ED
5985 unsigned long bitmask = napi->gro_bitmask;
5986 unsigned int i, base = ~0U;
07d78363 5987
42519ede
ED
5988 while ((i = ffs(bitmask)) != 0) {
5989 bitmask >>= i;
5990 base += i;
5991 __napi_gro_flush_chain(napi, base, flush_old);
d9f37d01 5992 }
07d78363 5993}
86cac58b 5994EXPORT_SYMBOL(napi_gro_flush);
d565b0a1 5995
0ccf4d50
AL
5996static void gro_list_prepare(const struct list_head *head,
5997 const struct sk_buff *skb)
89c5fa33 5998{
89c5fa33 5999 unsigned int maclen = skb->dev->hard_header_len;
0b4cec8c 6000 u32 hash = skb_get_hash_raw(skb);
d4546c25 6001 struct sk_buff *p;
89c5fa33 6002
07d78363 6003 list_for_each_entry(p, head, list) {
89c5fa33
ED
6004 unsigned long diffs;
6005
0b4cec8c
TH
6006 NAPI_GRO_CB(p)->flush = 0;
6007
6008 if (hash != skb_get_hash_raw(p)) {
6009 NAPI_GRO_CB(p)->same_flow = 0;
6010 continue;
6011 }
6012
89c5fa33 6013 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
b1817524
MM
6014 diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);
6015 if (skb_vlan_tag_present(p))
fc5141cb 6016 diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb);
de8f3a83 6017 diffs |= skb_metadata_differs(p, skb);
89c5fa33
ED
6018 if (maclen == ETH_HLEN)
6019 diffs |= compare_ether_header(skb_mac_header(p),
a50e233c 6020 skb_mac_header(skb));
89c5fa33
ED
6021 else if (!diffs)
6022 diffs = memcmp(skb_mac_header(p),
a50e233c 6023 skb_mac_header(skb),
89c5fa33 6024 maclen);
8550ff8d 6025
9efb4b5b
PA
6026 /* in most common scenarions _state is 0
6027 * otherwise we are already on some slower paths
6028 * either skip all the infrequent tests altogether or
6029 * avoid trying too hard to skip each of them individually
6030 */
6031 if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) {
6032#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
6033 struct tc_skb_ext *skb_ext;
6034 struct tc_skb_ext *p_ext;
6035#endif
6036
6037 diffs |= skb_metadata_dst_cmp(p, skb);
6038 diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb);
6039
9615fe36 6040#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
9efb4b5b
PA
6041 skb_ext = skb_ext_find(skb, TC_SKB_EXT);
6042 p_ext = skb_ext_find(p, TC_SKB_EXT);
8550ff8d
PB
6043
6044 diffs |= (!!p_ext) ^ (!!skb_ext);
6045 if (!diffs && unlikely(skb_ext))
6046 diffs |= p_ext->chain ^ skb_ext->chain;
9615fe36 6047#endif
9efb4b5b 6048 }
8550ff8d 6049
89c5fa33 6050 NAPI_GRO_CB(p)->same_flow = !diffs;
89c5fa33
ED
6051 }
6052}
6053
7ad18ff6 6054static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
299603e8
JC
6055{
6056 const struct skb_shared_info *pinfo = skb_shinfo(skb);
6057 const skb_frag_t *frag0 = &pinfo->frags[0];
6058
6059 NAPI_GRO_CB(skb)->data_offset = 0;
6060 NAPI_GRO_CB(skb)->frag0 = NULL;
6061 NAPI_GRO_CB(skb)->frag0_len = 0;
6062
8aef998d 6063 if (!skb_headlen(skb) && pinfo->nr_frags &&
38ec4944 6064 !PageHighMem(skb_frag_page(frag0)) &&
7ad18ff6 6065 (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
299603e8 6066 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
7cfd5fd5
ED
6067 NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
6068 skb_frag_size(frag0),
6069 skb->end - skb->tail);
89c5fa33
ED
6070 }
6071}
6072
a50e233c
ED
6073static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
6074{
6075 struct skb_shared_info *pinfo = skb_shinfo(skb);
6076
6077 BUG_ON(skb->end - skb->tail < grow);
6078
6079 memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
6080
6081 skb->data_len -= grow;
6082 skb->tail += grow;
6083
b54c9d5b 6084 skb_frag_off_add(&pinfo->frags[0], grow);
a50e233c
ED
6085 skb_frag_size_sub(&pinfo->frags[0], grow);
6086
6087 if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
6088 skb_frag_unref(skb, 0);
6089 memmove(pinfo->frags, pinfo->frags + 1,
6090 --pinfo->nr_frags * sizeof(pinfo->frags[0]));
6091 }
6092}
6093
c8079432 6094static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head)
07d78363 6095{
6312fe77 6096 struct sk_buff *oldest;
07d78363 6097
6312fe77 6098 oldest = list_last_entry(head, struct sk_buff, list);
07d78363 6099
6312fe77 6100 /* We are called with head length >= MAX_GRO_SKBS, so this is
07d78363
DM
6101 * impossible.
6102 */
6103 if (WARN_ON_ONCE(!oldest))
6104 return;
6105
d9f37d01
LR
6106 /* Do not adjust napi->gro_hash[].count, caller is adding a new
6107 * SKB to the chain.
07d78363 6108 */
ece23711 6109 skb_list_del_init(oldest);
c8079432 6110 napi_gro_complete(napi, oldest);
07d78363
DM
6111}
6112
bb728820 6113static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
d565b0a1 6114{
d0eed5c3
AL
6115 u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1);
6116 struct gro_list *gro_list = &napi->gro_hash[bucket];
d4546c25 6117 struct list_head *head = &offload_base;
22061d80 6118 struct packet_offload *ptype;
d565b0a1 6119 __be16 type = skb->protocol;
d4546c25 6120 struct sk_buff *pp = NULL;
5b252f0c 6121 enum gro_result ret;
d4546c25 6122 int same_flow;
a50e233c 6123 int grow;
d565b0a1 6124
b5cdae32 6125 if (netif_elide_gro(skb->dev))
d565b0a1
HX
6126 goto normal;
6127
9dc2c313 6128 gro_list_prepare(&gro_list->list, skb);
89c5fa33 6129
d565b0a1
HX
6130 rcu_read_lock();
6131 list_for_each_entry_rcu(ptype, head, list) {
f191a1d1 6132 if (ptype->type != type || !ptype->callbacks.gro_receive)
d565b0a1
HX
6133 continue;
6134
86911732 6135 skb_set_network_header(skb, skb_gro_offset(skb));
efd9450e 6136 skb_reset_mac_len(skb);
d565b0a1 6137 NAPI_GRO_CB(skb)->same_flow = 0;
d61d072e 6138 NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
5d38a079 6139 NAPI_GRO_CB(skb)->free = 0;
fac8e0f5 6140 NAPI_GRO_CB(skb)->encap_mark = 0;
fcd91dd4 6141 NAPI_GRO_CB(skb)->recursion_counter = 0;
a0ca153f 6142 NAPI_GRO_CB(skb)->is_fou = 0;
1530545e 6143 NAPI_GRO_CB(skb)->is_atomic = 1;
15e2396d 6144 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
d565b0a1 6145
662880f4
TH
6146 /* Setup for GRO checksum validation */
6147 switch (skb->ip_summed) {
6148 case CHECKSUM_COMPLETE:
6149 NAPI_GRO_CB(skb)->csum = skb->csum;
6150 NAPI_GRO_CB(skb)->csum_valid = 1;
6151 NAPI_GRO_CB(skb)->csum_cnt = 0;
6152 break;
6153 case CHECKSUM_UNNECESSARY:
6154 NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
6155 NAPI_GRO_CB(skb)->csum_valid = 0;
6156 break;
6157 default:
6158 NAPI_GRO_CB(skb)->csum_cnt = 0;
6159 NAPI_GRO_CB(skb)->csum_valid = 0;
6160 }
d565b0a1 6161
aaa5d90b
PA
6162 pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,
6163 ipv6_gro_receive, inet_gro_receive,
9dc2c313 6164 &gro_list->list, skb);
d565b0a1
HX
6165 break;
6166 }
6167 rcu_read_unlock();
6168
6169 if (&ptype->list == head)
6170 goto normal;
6171
45586c70 6172 if (PTR_ERR(pp) == -EINPROGRESS) {
25393d3f
SK
6173 ret = GRO_CONSUMED;
6174 goto ok;
6175 }
6176
0da2afd5 6177 same_flow = NAPI_GRO_CB(skb)->same_flow;
5d0d9be8 6178 ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
0da2afd5 6179
d565b0a1 6180 if (pp) {
992cba7e 6181 skb_list_del_init(pp);
c8079432 6182 napi_gro_complete(napi, pp);
9dc2c313 6183 gro_list->count--;
d565b0a1
HX
6184 }
6185
0da2afd5 6186 if (same_flow)
d565b0a1
HX
6187 goto ok;
6188
600adc18 6189 if (NAPI_GRO_CB(skb)->flush)
d565b0a1 6190 goto normal;
d565b0a1 6191
9dc2c313
AL
6192 if (unlikely(gro_list->count >= MAX_GRO_SKBS))
6193 gro_flush_oldest(napi, &gro_list->list);
6194 else
6195 gro_list->count++;
6196
d565b0a1 6197 NAPI_GRO_CB(skb)->count = 1;
2e71a6f8 6198 NAPI_GRO_CB(skb)->age = jiffies;
29e98242 6199 NAPI_GRO_CB(skb)->last = skb;
86911732 6200 skb_shinfo(skb)->gso_size = skb_gro_len(skb);
9dc2c313 6201 list_add(&skb->list, &gro_list->list);
5d0d9be8 6202 ret = GRO_HELD;
d565b0a1 6203
ad0f9904 6204pull:
a50e233c
ED
6205 grow = skb_gro_offset(skb) - skb_headlen(skb);
6206 if (grow > 0)
6207 gro_pull_from_frag0(skb, grow);
d565b0a1 6208ok:
9dc2c313 6209 if (gro_list->count) {
d0eed5c3
AL
6210 if (!test_bit(bucket, &napi->gro_bitmask))
6211 __set_bit(bucket, &napi->gro_bitmask);
6212 } else if (test_bit(bucket, &napi->gro_bitmask)) {
6213 __clear_bit(bucket, &napi->gro_bitmask);
d9f37d01
LR
6214 }
6215
5d0d9be8 6216 return ret;
d565b0a1
HX
6217
6218normal:
ad0f9904
HX
6219 ret = GRO_NORMAL;
6220 goto pull;
5d38a079 6221}
96e93eab 6222
bf5a755f
JC
6223struct packet_offload *gro_find_receive_by_type(__be16 type)
6224{
6225 struct list_head *offload_head = &offload_base;
6226 struct packet_offload *ptype;
6227
6228 list_for_each_entry_rcu(ptype, offload_head, list) {
6229 if (ptype->type != type || !ptype->callbacks.gro_receive)
6230 continue;
6231 return ptype;
6232 }
6233 return NULL;
6234}
e27a2f83 6235EXPORT_SYMBOL(gro_find_receive_by_type);
bf5a755f
JC
6236
6237struct packet_offload *gro_find_complete_by_type(__be16 type)
6238{
6239 struct list_head *offload_head = &offload_base;
6240 struct packet_offload *ptype;
6241
6242 list_for_each_entry_rcu(ptype, offload_head, list) {
6243 if (ptype->type != type || !ptype->callbacks.gro_complete)
6244 continue;
6245 return ptype;
6246 }
6247 return NULL;
6248}
e27a2f83 6249EXPORT_SYMBOL(gro_find_complete_by_type);
5d38a079 6250
6570bc79
AL
6251static gro_result_t napi_skb_finish(struct napi_struct *napi,
6252 struct sk_buff *skb,
6253 gro_result_t ret)
5d38a079 6254{
5d0d9be8
HX
6255 switch (ret) {
6256 case GRO_NORMAL:
8dc1c444 6257 gro_normal_one(napi, skb, 1);
c7c4b3b6 6258 break;
5d38a079 6259
daa86548 6260 case GRO_MERGED_FREE:
e44699d2
MK
6261 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
6262 napi_skb_free_stolen_head(skb);
28b34f01
AT
6263 else if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
6264 __kfree_skb(skb);
e44699d2 6265 else
9243adfc 6266 __kfree_skb_defer(skb);
daa86548
ED
6267 break;
6268
5b252f0c
BH
6269 case GRO_HELD:
6270 case GRO_MERGED:
25393d3f 6271 case GRO_CONSUMED:
5b252f0c 6272 break;
5d38a079
HX
6273 }
6274
c7c4b3b6 6275 return ret;
5d0d9be8 6276}
5d0d9be8 6277
c7c4b3b6 6278gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
5d0d9be8 6279{
b0e3f1bd
GB
6280 gro_result_t ret;
6281
93f93a44 6282 skb_mark_napi_id(skb, napi);
ae78dbfa 6283 trace_napi_gro_receive_entry(skb);
86911732 6284
7ad18ff6 6285 skb_gro_reset_offset(skb, 0);
a50e233c 6286
6570bc79 6287 ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb));
b0e3f1bd
GB
6288 trace_napi_gro_receive_exit(ret);
6289
6290 return ret;
d565b0a1
HX
6291}
6292EXPORT_SYMBOL(napi_gro_receive);
6293
d0c2b0d2 6294static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
96e93eab 6295{
93a35f59
ED
6296 if (unlikely(skb->pfmemalloc)) {
6297 consume_skb(skb);
6298 return;
6299 }
96e93eab 6300 __skb_pull(skb, skb_headlen(skb));
2a2a459e
ED
6301 /* restore the reserve we had after netdev_alloc_skb_ip_align() */
6302 skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
b1817524 6303 __vlan_hwaccel_clear_tag(skb);
66c46d74 6304 skb->dev = napi->dev;
6d152e23 6305 skb->skb_iif = 0;
33d9a2c7
ED
6306
6307 /* eth_type_trans() assumes pkt_type is PACKET_HOST */
6308 skb->pkt_type = PACKET_HOST;
6309
c3caf119
JC
6310 skb->encapsulation = 0;
6311 skb_shinfo(skb)->gso_type = 0;
e33d0ba8 6312 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
9efb4b5b
PA
6313 if (unlikely(skb->slow_gro)) {
6314 skb_ext_reset(skb);
6315 nf_reset_ct(skb);
6316 skb->slow_gro = 0;
6317 }
96e93eab
HX
6318
6319 napi->skb = skb;
6320}
96e93eab 6321
76620aaf 6322struct sk_buff *napi_get_frags(struct napi_struct *napi)
5d38a079 6323{
5d38a079 6324 struct sk_buff *skb = napi->skb;
5d38a079
HX
6325
6326 if (!skb) {
fd11a83d 6327 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
e2f9dc3b
ED
6328 if (skb) {
6329 napi->skb = skb;
6330 skb_mark_napi_id(skb, napi);
6331 }
80595d59 6332 }
96e93eab
HX
6333 return skb;
6334}
76620aaf 6335EXPORT_SYMBOL(napi_get_frags);
96e93eab 6336
a50e233c
ED
6337static gro_result_t napi_frags_finish(struct napi_struct *napi,
6338 struct sk_buff *skb,
6339 gro_result_t ret)
96e93eab 6340{
5d0d9be8
HX
6341 switch (ret) {
6342 case GRO_NORMAL:
a50e233c
ED
6343 case GRO_HELD:
6344 __skb_push(skb, ETH_HLEN);
6345 skb->protocol = eth_type_trans(skb, skb->dev);
323ebb61 6346 if (ret == GRO_NORMAL)
8dc1c444 6347 gro_normal_one(napi, skb, 1);
86911732 6348 break;
5d38a079 6349
e44699d2
MK
6350 case GRO_MERGED_FREE:
6351 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD)
6352 napi_skb_free_stolen_head(skb);
6353 else
6354 napi_reuse_skb(napi, skb);
6355 break;
6356
5b252f0c 6357 case GRO_MERGED:
25393d3f 6358 case GRO_CONSUMED:
5b252f0c 6359 break;
5d0d9be8 6360 }
5d38a079 6361
c7c4b3b6 6362 return ret;
5d38a079 6363}
5d0d9be8 6364
a50e233c
ED
6365/* Upper GRO stack assumes network header starts at gro_offset=0
6366 * Drivers could call both napi_gro_frags() and napi_gro_receive()
6367 * We copy ethernet header into skb->data to have a common layout.
6368 */
4adb9c4a 6369static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
76620aaf
HX
6370{
6371 struct sk_buff *skb = napi->skb;
a50e233c
ED
6372 const struct ethhdr *eth;
6373 unsigned int hlen = sizeof(*eth);
76620aaf
HX
6374
6375 napi->skb = NULL;
6376
a50e233c 6377 skb_reset_mac_header(skb);
7ad18ff6 6378 skb_gro_reset_offset(skb, hlen);
a50e233c 6379
a50e233c
ED
6380 if (unlikely(skb_gro_header_hard(skb, hlen))) {
6381 eth = skb_gro_header_slow(skb, hlen, 0);
6382 if (unlikely(!eth)) {
4da46ceb
AC
6383 net_warn_ratelimited("%s: dropping impossible skb from %s\n",
6384 __func__, napi->dev->name);
a50e233c
ED
6385 napi_reuse_skb(napi, skb);
6386 return NULL;
6387 }
6388 } else {
a4270d67 6389 eth = (const struct ethhdr *)skb->data;
a50e233c
ED
6390 gro_pull_from_frag0(skb, hlen);
6391 NAPI_GRO_CB(skb)->frag0 += hlen;
6392 NAPI_GRO_CB(skb)->frag0_len -= hlen;
76620aaf 6393 }
a50e233c
ED
6394 __skb_pull(skb, hlen);
6395
6396 /*
6397 * This works because the only protocols we care about don't require
6398 * special handling.
6399 * We'll fix it up properly in napi_frags_finish()
6400 */
6401 skb->protocol = eth->h_proto;
76620aaf 6402
76620aaf
HX
6403 return skb;
6404}
76620aaf 6405
c7c4b3b6 6406gro_result_t napi_gro_frags(struct napi_struct *napi)
5d0d9be8 6407{
b0e3f1bd 6408 gro_result_t ret;
76620aaf 6409 struct sk_buff *skb = napi_frags_skb(napi);
5d0d9be8 6410
ae78dbfa
BH
6411 trace_napi_gro_frags_entry(skb);
6412
b0e3f1bd
GB
6413 ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
6414 trace_napi_gro_frags_exit(ret);
6415
6416 return ret;
5d0d9be8 6417}
5d38a079
HX
6418EXPORT_SYMBOL(napi_gro_frags);
6419
573e8fca
TH
6420/* Compute the checksum from gro_offset and return the folded value
6421 * after adding in any pseudo checksum.
6422 */
6423__sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
6424{
6425 __wsum wsum;
6426 __sum16 sum;
6427
6428 wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
6429
6430 /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
6431 sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
14641931 6432 /* See comments in __skb_checksum_complete(). */
573e8fca
TH
6433 if (likely(!sum)) {
6434 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
6435 !skb->csum_complete_sw)
7fe50ac8 6436 netdev_rx_csum_fault(skb->dev, skb);
573e8fca
TH
6437 }
6438
6439 NAPI_GRO_CB(skb)->csum = wsum;
6440 NAPI_GRO_CB(skb)->csum_valid = 1;
6441
6442 return sum;
6443}
6444EXPORT_SYMBOL(__skb_gro_checksum_complete);
6445
773fc8f6 6446static void net_rps_send_ipi(struct softnet_data *remsd)
6447{
6448#ifdef CONFIG_RPS
6449 while (remsd) {
6450 struct softnet_data *next = remsd->rps_ipi_next;
6451
6452 if (cpu_online(remsd->cpu))
6453 smp_call_function_single_async(remsd->cpu, &remsd->csd);
6454 remsd = next;
6455 }
6456#endif
6457}
6458
e326bed2 6459/*
855abcf0 6460 * net_rps_action_and_irq_enable sends any pending IPI's for rps.
e326bed2
ED
6461 * Note: called with local irq disabled, but exits with local irq enabled.
6462 */
6463static void net_rps_action_and_irq_enable(struct softnet_data *sd)
6464{
6465#ifdef CONFIG_RPS
6466 struct softnet_data *remsd = sd->rps_ipi_list;
6467
6468 if (remsd) {
6469 sd->rps_ipi_list = NULL;
6470
6471 local_irq_enable();
6472
6473 /* Send pending IPI's to kick RPS processing on remote cpus. */
773fc8f6 6474 net_rps_send_ipi(remsd);
e326bed2
ED
6475 } else
6476#endif
6477 local_irq_enable();
6478}
6479
d75b1ade
ED
6480static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
6481{
6482#ifdef CONFIG_RPS
6483 return sd->rps_ipi_list != NULL;
6484#else
6485 return false;
6486#endif
6487}
6488
bea3348e 6489static int process_backlog(struct napi_struct *napi, int quota)
1da177e4 6490{
eecfd7c4 6491 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
145dd5f9
PA
6492 bool again = true;
6493 int work = 0;
1da177e4 6494
e326bed2
ED
6495 /* Check if we have pending ipi, its better to send them now,
6496 * not waiting net_rx_action() end.
6497 */
d75b1ade 6498 if (sd_has_rps_ipi_waiting(sd)) {
e326bed2
ED
6499 local_irq_disable();
6500 net_rps_action_and_irq_enable(sd);
6501 }
d75b1ade 6502
3d48b53f 6503 napi->weight = dev_rx_weight;
145dd5f9 6504 while (again) {
1da177e4 6505 struct sk_buff *skb;
6e7676c1
CG
6506
6507 while ((skb = __skb_dequeue(&sd->process_queue))) {
2c17d27c 6508 rcu_read_lock();
6e7676c1 6509 __netif_receive_skb(skb);
2c17d27c 6510 rcu_read_unlock();
76cc8b13 6511 input_queue_head_incr(sd);
145dd5f9 6512 if (++work >= quota)
76cc8b13 6513 return work;
145dd5f9 6514
6e7676c1 6515 }
1da177e4 6516
145dd5f9 6517 local_irq_disable();
e36fa2f7 6518 rps_lock(sd);
11ef7a89 6519 if (skb_queue_empty(&sd->input_pkt_queue)) {
eecfd7c4
ED
6520 /*
6521 * Inline a custom version of __napi_complete().
6522 * only current cpu owns and manipulates this napi,
11ef7a89
TH
6523 * and NAPI_STATE_SCHED is the only possible flag set
6524 * on backlog.
6525 * We can use a plain write instead of clear_bit(),
eecfd7c4
ED
6526 * and we dont need an smp_mb() memory barrier.
6527 */
eecfd7c4 6528 napi->state = 0;
145dd5f9
PA
6529 again = false;
6530 } else {
6531 skb_queue_splice_tail_init(&sd->input_pkt_queue,
6532 &sd->process_queue);
bea3348e 6533 }
e36fa2f7 6534 rps_unlock(sd);
145dd5f9 6535 local_irq_enable();
6e7676c1 6536 }
1da177e4 6537
bea3348e
SH
6538 return work;
6539}
1da177e4 6540
bea3348e
SH
6541/**
6542 * __napi_schedule - schedule for receive
c4ea43c5 6543 * @n: entry to schedule
bea3348e 6544 *
bc9ad166
ED
6545 * The entry's receive function will be scheduled to run.
6546 * Consider using __napi_schedule_irqoff() if hard irqs are masked.
bea3348e 6547 */
b5606c2d 6548void __napi_schedule(struct napi_struct *n)
bea3348e
SH
6549{
6550 unsigned long flags;
1da177e4 6551
bea3348e 6552 local_irq_save(flags);
903ceff7 6553 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
bea3348e 6554 local_irq_restore(flags);
1da177e4 6555}
bea3348e
SH
6556EXPORT_SYMBOL(__napi_schedule);
6557
39e6c820
ED
6558/**
6559 * napi_schedule_prep - check if napi can be scheduled
6560 * @n: napi context
6561 *
6562 * Test if NAPI routine is already running, and if not mark
ee1a4c84 6563 * it as running. This is used as a condition variable to
39e6c820
ED
6564 * insure only one NAPI poll instance runs. We also make
6565 * sure there is no pending NAPI disable.
6566 */
6567bool napi_schedule_prep(struct napi_struct *n)
6568{
6569 unsigned long val, new;
6570
6571 do {
6572 val = READ_ONCE(n->state);
6573 if (unlikely(val & NAPIF_STATE_DISABLE))
6574 return false;
6575 new = val | NAPIF_STATE_SCHED;
6576
6577 /* Sets STATE_MISSED bit if STATE_SCHED was already set
6578 * This was suggested by Alexander Duyck, as compiler
6579 * emits better code than :
6580 * if (val & NAPIF_STATE_SCHED)
6581 * new |= NAPIF_STATE_MISSED;
6582 */
6583 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6584 NAPIF_STATE_MISSED;
6585 } while (cmpxchg(&n->state, val, new) != val);
6586
6587 return !(val & NAPIF_STATE_SCHED);
6588}
6589EXPORT_SYMBOL(napi_schedule_prep);
6590
bc9ad166
ED
6591/**
6592 * __napi_schedule_irqoff - schedule for receive
6593 * @n: entry to schedule
6594 *
8380c81d
SAS
6595 * Variant of __napi_schedule() assuming hard irqs are masked.
6596 *
6597 * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
6598 * because the interrupt disabled assumption might not be true
6599 * due to force-threaded interrupts and spinlock substitution.
bc9ad166
ED
6600 */
6601void __napi_schedule_irqoff(struct napi_struct *n)
6602{
8380c81d
SAS
6603 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
6604 ____napi_schedule(this_cpu_ptr(&softnet_data), n);
6605 else
6606 __napi_schedule(n);
bc9ad166
ED
6607}
6608EXPORT_SYMBOL(__napi_schedule_irqoff);
6609
364b6055 6610bool napi_complete_done(struct napi_struct *n, int work_done)
d565b0a1 6611{
6f8b12d6
ED
6612 unsigned long flags, val, new, timeout = 0;
6613 bool ret = true;
d565b0a1
HX
6614
6615 /*
217f6974
ED
6616 * 1) Don't let napi dequeue from the cpu poll list
6617 * just in case its running on a different cpu.
6618 * 2) If we are busy polling, do nothing here, we have
6619 * the guarantee we will be called later.
d565b0a1 6620 */
217f6974
ED
6621 if (unlikely(n->state & (NAPIF_STATE_NPSVC |
6622 NAPIF_STATE_IN_BUSY_POLL)))
364b6055 6623 return false;
d565b0a1 6624
6f8b12d6
ED
6625 if (work_done) {
6626 if (n->gro_bitmask)
7e417a66
ED
6627 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6628 n->defer_hard_irqs_count = READ_ONCE(n->dev->napi_defer_hard_irqs);
6f8b12d6
ED
6629 }
6630 if (n->defer_hard_irqs_count > 0) {
6631 n->defer_hard_irqs_count--;
7e417a66 6632 timeout = READ_ONCE(n->dev->gro_flush_timeout);
6f8b12d6
ED
6633 if (timeout)
6634 ret = false;
6635 }
6636 if (n->gro_bitmask) {
605108ac
PA
6637 /* When the NAPI instance uses a timeout and keeps postponing
6638 * it, we need to bound somehow the time packets are kept in
6639 * the GRO layer
6640 */
6641 napi_gro_flush(n, !!timeout);
3b47d303 6642 }
c8079432
MM
6643
6644 gro_normal_list(n);
6645
02c1602e 6646 if (unlikely(!list_empty(&n->poll_list))) {
d75b1ade
ED
6647 /* If n->poll_list is not empty, we need to mask irqs */
6648 local_irq_save(flags);
02c1602e 6649 list_del_init(&n->poll_list);
d75b1ade
ED
6650 local_irq_restore(flags);
6651 }
39e6c820
ED
6652
6653 do {
6654 val = READ_ONCE(n->state);
6655
6656 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
6657
7fd3253a 6658 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
cb038357 6659 NAPIF_STATE_SCHED_THREADED |
7fd3253a 6660 NAPIF_STATE_PREFER_BUSY_POLL);
39e6c820
ED
6661
6662 /* If STATE_MISSED was set, leave STATE_SCHED set,
6663 * because we will call napi->poll() one more time.
6664 * This C code was suggested by Alexander Duyck to help gcc.
6665 */
6666 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
6667 NAPIF_STATE_SCHED;
6668 } while (cmpxchg(&n->state, val, new) != val);
6669
6670 if (unlikely(val & NAPIF_STATE_MISSED)) {
6671 __napi_schedule(n);
6672 return false;
6673 }
6674
6f8b12d6
ED
6675 if (timeout)
6676 hrtimer_start(&n->timer, ns_to_ktime(timeout),
6677 HRTIMER_MODE_REL_PINNED);
6678 return ret;
d565b0a1 6679}
3b47d303 6680EXPORT_SYMBOL(napi_complete_done);
d565b0a1 6681
af12fa6e 6682/* must be called under rcu_read_lock(), as we dont take a reference */
02d62e86 6683static struct napi_struct *napi_by_id(unsigned int napi_id)
af12fa6e
ET
6684{
6685 unsigned int hash = napi_id % HASH_SIZE(napi_hash);
6686 struct napi_struct *napi;
6687
6688 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
6689 if (napi->napi_id == napi_id)
6690 return napi;
6691
6692 return NULL;
6693}
02d62e86
ED
6694
6695#if defined(CONFIG_NET_RX_BUSY_POLL)
217f6974 6696
7fd3253a 6697static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule)
217f6974 6698{
7fd3253a
BT
6699 if (!skip_schedule) {
6700 gro_normal_list(napi);
6701 __napi_schedule(napi);
6702 return;
6703 }
217f6974 6704
7fd3253a
BT
6705 if (napi->gro_bitmask) {
6706 /* flush too old packets
6707 * If HZ < 1000, flush all packets.
6708 */
6709 napi_gro_flush(napi, HZ >= 1000);
6710 }
217f6974 6711
7fd3253a
BT
6712 gro_normal_list(napi);
6713 clear_bit(NAPI_STATE_SCHED, &napi->state);
6714}
6715
7c951caf
BT
6716static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, bool prefer_busy_poll,
6717 u16 budget)
217f6974 6718{
7fd3253a
BT
6719 bool skip_schedule = false;
6720 unsigned long timeout;
217f6974
ED
6721 int rc;
6722
39e6c820
ED
6723 /* Busy polling means there is a high chance device driver hard irq
6724 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
6725 * set in napi_schedule_prep().
6726 * Since we are about to call napi->poll() once more, we can safely
6727 * clear NAPI_STATE_MISSED.
6728 *
6729 * Note: x86 could use a single "lock and ..." instruction
6730 * to perform these two clear_bit()
6731 */
6732 clear_bit(NAPI_STATE_MISSED, &napi->state);
217f6974
ED
6733 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
6734
6735 local_bh_disable();
6736
7fd3253a
BT
6737 if (prefer_busy_poll) {
6738 napi->defer_hard_irqs_count = READ_ONCE(napi->dev->napi_defer_hard_irqs);
6739 timeout = READ_ONCE(napi->dev->gro_flush_timeout);
6740 if (napi->defer_hard_irqs_count && timeout) {
6741 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED);
6742 skip_schedule = true;
6743 }
6744 }
6745
217f6974
ED
6746 /* All we really want here is to re-enable device interrupts.
6747 * Ideally, a new ndo_busy_poll_stop() could avoid another round.
6748 */
7c951caf 6749 rc = napi->poll(napi, budget);
323ebb61
EC
6750 /* We can't gro_normal_list() here, because napi->poll() might have
6751 * rearmed the napi (napi_complete_done()) in which case it could
6752 * already be running on another CPU.
6753 */
7c951caf 6754 trace_napi_poll(napi, rc, budget);
217f6974 6755 netpoll_poll_unlock(have_poll_lock);
7c951caf 6756 if (rc == budget)
7fd3253a 6757 __busy_poll_stop(napi, skip_schedule);
217f6974 6758 local_bh_enable();
217f6974
ED
6759}
6760
7db6b048
SS
6761void napi_busy_loop(unsigned int napi_id,
6762 bool (*loop_end)(void *, unsigned long),
7c951caf 6763 void *loop_end_arg, bool prefer_busy_poll, u16 budget)
02d62e86 6764{
7db6b048 6765 unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
217f6974 6766 int (*napi_poll)(struct napi_struct *napi, int budget);
217f6974 6767 void *have_poll_lock = NULL;
02d62e86 6768 struct napi_struct *napi;
217f6974
ED
6769
6770restart:
217f6974 6771 napi_poll = NULL;
02d62e86 6772
2a028ecb 6773 rcu_read_lock();
02d62e86 6774
545cd5e5 6775 napi = napi_by_id(napi_id);
02d62e86
ED
6776 if (!napi)
6777 goto out;
6778
217f6974
ED
6779 preempt_disable();
6780 for (;;) {
2b5cd0df
AD
6781 int work = 0;
6782
2a028ecb 6783 local_bh_disable();
217f6974
ED
6784 if (!napi_poll) {
6785 unsigned long val = READ_ONCE(napi->state);
6786
6787 /* If multiple threads are competing for this napi,
6788 * we avoid dirtying napi->state as much as we can.
6789 */
6790 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
7fd3253a
BT
6791 NAPIF_STATE_IN_BUSY_POLL)) {
6792 if (prefer_busy_poll)
6793 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
217f6974 6794 goto count;
7fd3253a 6795 }
217f6974
ED
6796 if (cmpxchg(&napi->state, val,
6797 val | NAPIF_STATE_IN_BUSY_POLL |
7fd3253a
BT
6798 NAPIF_STATE_SCHED) != val) {
6799 if (prefer_busy_poll)
6800 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
217f6974 6801 goto count;
7fd3253a 6802 }
217f6974
ED
6803 have_poll_lock = netpoll_poll_lock(napi);
6804 napi_poll = napi->poll;
6805 }
7c951caf
BT
6806 work = napi_poll(napi, budget);
6807 trace_napi_poll(napi, work, budget);
323ebb61 6808 gro_normal_list(napi);
217f6974 6809count:
2b5cd0df 6810 if (work > 0)
7db6b048 6811 __NET_ADD_STATS(dev_net(napi->dev),
2b5cd0df 6812 LINUX_MIB_BUSYPOLLRXPACKETS, work);
2a028ecb 6813 local_bh_enable();
02d62e86 6814
7db6b048 6815 if (!loop_end || loop_end(loop_end_arg, start_time))
217f6974 6816 break;
02d62e86 6817
217f6974
ED
6818 if (unlikely(need_resched())) {
6819 if (napi_poll)
7c951caf 6820 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
217f6974
ED
6821 preempt_enable();
6822 rcu_read_unlock();
6823 cond_resched();
7db6b048 6824 if (loop_end(loop_end_arg, start_time))
2b5cd0df 6825 return;
217f6974
ED
6826 goto restart;
6827 }
6cdf89b1 6828 cpu_relax();
217f6974
ED
6829 }
6830 if (napi_poll)
7c951caf 6831 busy_poll_stop(napi, have_poll_lock, prefer_busy_poll, budget);
217f6974 6832 preempt_enable();
02d62e86 6833out:
2a028ecb 6834 rcu_read_unlock();
02d62e86 6835}
7db6b048 6836EXPORT_SYMBOL(napi_busy_loop);
02d62e86
ED
6837
6838#endif /* CONFIG_NET_RX_BUSY_POLL */
af12fa6e 6839
149d6ad8 6840static void napi_hash_add(struct napi_struct *napi)
af12fa6e 6841{
4d092dd2 6842 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
52bd2d62 6843 return;
af12fa6e 6844
52bd2d62 6845 spin_lock(&napi_hash_lock);
af12fa6e 6846
545cd5e5 6847 /* 0..NR_CPUS range is reserved for sender_cpu use */
52bd2d62 6848 do {
545cd5e5
AD
6849 if (unlikely(++napi_gen_id < MIN_NAPI_ID))
6850 napi_gen_id = MIN_NAPI_ID;
52bd2d62
ED
6851 } while (napi_by_id(napi_gen_id));
6852 napi->napi_id = napi_gen_id;
af12fa6e 6853
52bd2d62
ED
6854 hlist_add_head_rcu(&napi->napi_hash_node,
6855 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
af12fa6e 6856
52bd2d62 6857 spin_unlock(&napi_hash_lock);
af12fa6e 6858}
af12fa6e
ET
6859
6860/* Warning : caller is responsible to make sure rcu grace period
6861 * is respected before freeing memory containing @napi
6862 */
5198d545 6863static void napi_hash_del(struct napi_struct *napi)
af12fa6e
ET
6864{
6865 spin_lock(&napi_hash_lock);
6866
4d092dd2 6867 hlist_del_init_rcu(&napi->napi_hash_node);
5198d545 6868
af12fa6e
ET
6869 spin_unlock(&napi_hash_lock);
6870}
af12fa6e 6871
3b47d303
ED
6872static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
6873{
6874 struct napi_struct *napi;
6875
6876 napi = container_of(timer, struct napi_struct, timer);
39e6c820
ED
6877
6878 /* Note : we use a relaxed variant of napi_schedule_prep() not setting
6879 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
6880 */
6f8b12d6 6881 if (!napi_disable_pending(napi) &&
7fd3253a
BT
6882 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
6883 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
39e6c820 6884 __napi_schedule_irqoff(napi);
7fd3253a 6885 }
3b47d303
ED
6886
6887 return HRTIMER_NORESTART;
6888}
6889
7c4ec749 6890static void init_gro_hash(struct napi_struct *napi)
d565b0a1 6891{
07d78363
DM
6892 int i;
6893
6312fe77
LR
6894 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
6895 INIT_LIST_HEAD(&napi->gro_hash[i].list);
6896 napi->gro_hash[i].count = 0;
6897 }
7c4ec749
DM
6898 napi->gro_bitmask = 0;
6899}
6900
5fdd2f0e
WW
6901int dev_set_threaded(struct net_device *dev, bool threaded)
6902{
6903 struct napi_struct *napi;
6904 int err = 0;
6905
6906 if (dev->threaded == threaded)
6907 return 0;
6908
6909 if (threaded) {
6910 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6911 if (!napi->thread) {
6912 err = napi_kthread_create(napi);
6913 if (err) {
6914 threaded = false;
6915 break;
6916 }
6917 }
6918 }
6919 }
6920
6921 dev->threaded = threaded;
6922
6923 /* Make sure kthread is created before THREADED bit
6924 * is set.
6925 */
6926 smp_mb__before_atomic();
6927
6928 /* Setting/unsetting threaded mode on a napi might not immediately
6929 * take effect, if the current napi instance is actively being
6930 * polled. In this case, the switch between threaded mode and
6931 * softirq mode will happen in the next round of napi_schedule().
6932 * This should not cause hiccups/stalls to the live traffic.
6933 */
6934 list_for_each_entry(napi, &dev->napi_list, dev_list) {
6935 if (threaded)
6936 set_bit(NAPI_STATE_THREADED, &napi->state);
6937 else
6938 clear_bit(NAPI_STATE_THREADED, &napi->state);
6939 }
6940
6941 return err;
6942}
8f64860f 6943EXPORT_SYMBOL(dev_set_threaded);
5fdd2f0e 6944
7c4ec749
DM
6945void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
6946 int (*poll)(struct napi_struct *, int), int weight)
6947{
4d092dd2
JK
6948 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
6949 return;
6950
7c4ec749 6951 INIT_LIST_HEAD(&napi->poll_list);
4d092dd2 6952 INIT_HLIST_NODE(&napi->napi_hash_node);
7c4ec749
DM
6953 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
6954 napi->timer.function = napi_watchdog;
6955 init_gro_hash(napi);
5d38a079 6956 napi->skb = NULL;
323ebb61
EC
6957 INIT_LIST_HEAD(&napi->rx_list);
6958 napi->rx_count = 0;
d565b0a1 6959 napi->poll = poll;
82dc3c63 6960 if (weight > NAPI_POLL_WEIGHT)
bf29e9e9
QC
6961 netdev_err_once(dev, "%s() called with weight %d\n", __func__,
6962 weight);
d565b0a1 6963 napi->weight = weight;
d565b0a1 6964 napi->dev = dev;
5d38a079 6965#ifdef CONFIG_NETPOLL
d565b0a1
HX
6966 napi->poll_owner = -1;
6967#endif
6968 set_bit(NAPI_STATE_SCHED, &napi->state);
96e97bc0
JK
6969 set_bit(NAPI_STATE_NPSVC, &napi->state);
6970 list_add_rcu(&napi->dev_list, &dev->napi_list);
93d05d4a 6971 napi_hash_add(napi);
29863d41
WW
6972 /* Create kthread for this napi if dev->threaded is set.
6973 * Clear dev->threaded if kthread creation failed so that
6974 * threaded mode will not be enabled in napi_enable().
6975 */
6976 if (dev->threaded && napi_kthread_create(napi))
6977 dev->threaded = 0;
d565b0a1
HX
6978}
6979EXPORT_SYMBOL(netif_napi_add);
6980
3b47d303
ED
6981void napi_disable(struct napi_struct *n)
6982{
6983 might_sleep();
6984 set_bit(NAPI_STATE_DISABLE, &n->state);
6985
6986 while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
6987 msleep(1);
2d8bff12
NH
6988 while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
6989 msleep(1);
3b47d303
ED
6990
6991 hrtimer_cancel(&n->timer);
6992
7fd3253a 6993 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
3b47d303 6994 clear_bit(NAPI_STATE_DISABLE, &n->state);
29863d41 6995 clear_bit(NAPI_STATE_THREADED, &n->state);
3b47d303
ED
6996}
6997EXPORT_SYMBOL(napi_disable);
6998
29863d41
WW
6999/**
7000 * napi_enable - enable NAPI scheduling
7001 * @n: NAPI context
7002 *
7003 * Resume NAPI from being scheduled on this context.
7004 * Must be paired with napi_disable.
7005 */
7006void napi_enable(struct napi_struct *n)
7007{
7008 BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
7009 smp_mb__before_atomic();
7010 clear_bit(NAPI_STATE_SCHED, &n->state);
7011 clear_bit(NAPI_STATE_NPSVC, &n->state);
7012 if (n->dev->threaded && n->thread)
7013 set_bit(NAPI_STATE_THREADED, &n->state);
7014}
7015EXPORT_SYMBOL(napi_enable);
7016
07d78363 7017static void flush_gro_hash(struct napi_struct *napi)
d4546c25 7018{
07d78363 7019 int i;
d4546c25 7020
07d78363
DM
7021 for (i = 0; i < GRO_HASH_BUCKETS; i++) {
7022 struct sk_buff *skb, *n;
7023
6312fe77 7024 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list)
07d78363 7025 kfree_skb(skb);
6312fe77 7026 napi->gro_hash[i].count = 0;
07d78363 7027 }
d4546c25
DM
7028}
7029
93d05d4a 7030/* Must be called in process context */
5198d545 7031void __netif_napi_del(struct napi_struct *napi)
d565b0a1 7032{
4d092dd2
JK
7033 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
7034 return;
7035
5198d545 7036 napi_hash_del(napi);
5251ef82 7037 list_del_rcu(&napi->dev_list);
76620aaf 7038 napi_free_frags(napi);
d565b0a1 7039
07d78363 7040 flush_gro_hash(napi);
d9f37d01 7041 napi->gro_bitmask = 0;
29863d41
WW
7042
7043 if (napi->thread) {
7044 kthread_stop(napi->thread);
7045 napi->thread = NULL;
7046 }
d565b0a1 7047}
5198d545 7048EXPORT_SYMBOL(__netif_napi_del);
d565b0a1 7049
898f8015 7050static int __napi_poll(struct napi_struct *n, bool *repoll)
726ce70e 7051{
726ce70e
HX
7052 int work, weight;
7053
726ce70e
HX
7054 weight = n->weight;
7055
7056 /* This NAPI_STATE_SCHED test is for avoiding a race
7057 * with netpoll's poll_napi(). Only the entity which
7058 * obtains the lock and sees NAPI_STATE_SCHED set will
7059 * actually make the ->poll() call. Therefore we avoid
7060 * accidentally calling ->poll() when NAPI is not scheduled.
7061 */
7062 work = 0;
7063 if (test_bit(NAPI_STATE_SCHED, &n->state)) {
7064 work = n->poll(n, weight);
1db19db7 7065 trace_napi_poll(n, work, weight);
726ce70e
HX
7066 }
7067
427d5838
ED
7068 if (unlikely(work > weight))
7069 pr_err_once("NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
7070 n->poll, work, weight);
726ce70e
HX
7071
7072 if (likely(work < weight))
898f8015 7073 return work;
726ce70e
HX
7074
7075 /* Drivers must not modify the NAPI state if they
7076 * consume the entire weight. In such cases this code
7077 * still "owns" the NAPI instance and therefore can
7078 * move the instance around on the list at-will.
7079 */
7080 if (unlikely(napi_disable_pending(n))) {
7081 napi_complete(n);
898f8015 7082 return work;
726ce70e
HX
7083 }
7084
7fd3253a
BT
7085 /* The NAPI context has more processing work, but busy-polling
7086 * is preferred. Exit early.
7087 */
7088 if (napi_prefer_busy_poll(n)) {
7089 if (napi_complete_done(n, work)) {
7090 /* If timeout is not set, we need to make sure
7091 * that the NAPI is re-scheduled.
7092 */
7093 napi_schedule(n);
7094 }
898f8015 7095 return work;
7fd3253a
BT
7096 }
7097
d9f37d01 7098 if (n->gro_bitmask) {
726ce70e
HX
7099 /* flush too old packets
7100 * If HZ < 1000, flush all packets.
7101 */
7102 napi_gro_flush(n, HZ >= 1000);
7103 }
7104
c8079432
MM
7105 gro_normal_list(n);
7106
001ce546
HX
7107 /* Some drivers may have called napi_schedule
7108 * prior to exhausting their budget.
7109 */
7110 if (unlikely(!list_empty(&n->poll_list))) {
7111 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
7112 n->dev ? n->dev->name : "backlog");
898f8015 7113 return work;
001ce546
HX
7114 }
7115
898f8015
FF
7116 *repoll = true;
7117
7118 return work;
7119}
7120
7121static int napi_poll(struct napi_struct *n, struct list_head *repoll)
7122{
7123 bool do_repoll = false;
7124 void *have;
7125 int work;
7126
7127 list_del_init(&n->poll_list);
7128
7129 have = netpoll_poll_lock(n);
7130
7131 work = __napi_poll(n, &do_repoll);
7132
7133 if (do_repoll)
7134 list_add_tail(&n->poll_list, repoll);
726ce70e 7135
726ce70e
HX
7136 netpoll_poll_unlock(have);
7137
7138 return work;
7139}
7140
29863d41
WW
7141static int napi_thread_wait(struct napi_struct *napi)
7142{
cb038357
WW
7143 bool woken = false;
7144
29863d41
WW
7145 set_current_state(TASK_INTERRUPTIBLE);
7146
27f0ad71 7147 while (!kthread_should_stop()) {
cb038357
WW
7148 /* Testing SCHED_THREADED bit here to make sure the current
7149 * kthread owns this napi and could poll on this napi.
7150 * Testing SCHED bit is not enough because SCHED bit might be
7151 * set by some other busy poll thread or by napi_disable().
7152 */
7153 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state) || woken) {
29863d41
WW
7154 WARN_ON(!list_empty(&napi->poll_list));
7155 __set_current_state(TASK_RUNNING);
7156 return 0;
7157 }
7158
7159 schedule();
cb038357
WW
7160 /* woken being true indicates this thread owns this napi. */
7161 woken = true;
29863d41
WW
7162 set_current_state(TASK_INTERRUPTIBLE);
7163 }
7164 __set_current_state(TASK_RUNNING);
27f0ad71 7165
29863d41
WW
7166 return -1;
7167}
7168
7169static int napi_threaded_poll(void *data)
7170{
7171 struct napi_struct *napi = data;
7172 void *have;
7173
7174 while (!napi_thread_wait(napi)) {
7175 for (;;) {
7176 bool repoll = false;
7177
7178 local_bh_disable();
7179
7180 have = netpoll_poll_lock(napi);
7181 __napi_poll(napi, &repoll);
7182 netpoll_poll_unlock(have);
7183
29863d41
WW
7184 local_bh_enable();
7185
7186 if (!repoll)
7187 break;
7188
7189 cond_resched();
7190 }
7191 }
7192 return 0;
7193}
7194
0766f788 7195static __latent_entropy void net_rx_action(struct softirq_action *h)
1da177e4 7196{
903ceff7 7197 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
7acf8a1e
MW
7198 unsigned long time_limit = jiffies +
7199 usecs_to_jiffies(netdev_budget_usecs);
51b0bded 7200 int budget = netdev_budget;
d75b1ade
ED
7201 LIST_HEAD(list);
7202 LIST_HEAD(repoll);
53fb95d3 7203
1da177e4 7204 local_irq_disable();
d75b1ade
ED
7205 list_splice_init(&sd->poll_list, &list);
7206 local_irq_enable();
1da177e4 7207
ceb8d5bf 7208 for (;;) {
bea3348e 7209 struct napi_struct *n;
1da177e4 7210
ceb8d5bf
HX
7211 if (list_empty(&list)) {
7212 if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
fec6e49b 7213 return;
ceb8d5bf
HX
7214 break;
7215 }
7216
6bd373eb
HX
7217 n = list_first_entry(&list, struct napi_struct, poll_list);
7218 budget -= napi_poll(n, &repoll);
7219
d75b1ade 7220 /* If softirq window is exhausted then punt.
24f8b238
SH
7221 * Allow this to run for 2 jiffies since which will allow
7222 * an average latency of 1.5/HZ.
bea3348e 7223 */
ceb8d5bf
HX
7224 if (unlikely(budget <= 0 ||
7225 time_after_eq(jiffies, time_limit))) {
7226 sd->time_squeeze++;
7227 break;
7228 }
1da177e4 7229 }
d75b1ade 7230
d75b1ade
ED
7231 local_irq_disable();
7232
7233 list_splice_tail_init(&sd->poll_list, &list);
7234 list_splice_tail(&repoll, &list);
7235 list_splice(&list, &sd->poll_list);
7236 if (!list_empty(&sd->poll_list))
7237 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
7238
e326bed2 7239 net_rps_action_and_irq_enable(sd);
1da177e4
LT
7240}
7241
aa9d8560 7242struct netdev_adjacent {
9ff162a8 7243 struct net_device *dev;
5d261913
VF
7244
7245 /* upper master flag, there can only be one master device per list */
9ff162a8 7246 bool master;
5d261913 7247
32b6d34f
TY
7248 /* lookup ignore flag */
7249 bool ignore;
7250
5d261913
VF
7251 /* counter for the number of times this device was added to us */
7252 u16 ref_nr;
7253
402dae96
VF
7254 /* private field for the users */
7255 void *private;
7256
9ff162a8
JP
7257 struct list_head list;
7258 struct rcu_head rcu;
9ff162a8
JP
7259};
7260
6ea29da1 7261static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
2f268f12 7262 struct list_head *adj_list)
9ff162a8 7263{
5d261913 7264 struct netdev_adjacent *adj;
5d261913 7265
2f268f12 7266 list_for_each_entry(adj, adj_list, list) {
5d261913
VF
7267 if (adj->dev == adj_dev)
7268 return adj;
9ff162a8
JP
7269 }
7270 return NULL;
7271}
7272
eff74233
TY
7273static int ____netdev_has_upper_dev(struct net_device *upper_dev,
7274 struct netdev_nested_priv *priv)
f1170fd4 7275{
eff74233 7276 struct net_device *dev = (struct net_device *)priv->data;
f1170fd4
DA
7277
7278 return upper_dev == dev;
7279}
7280
9ff162a8
JP
7281/**
7282 * netdev_has_upper_dev - Check if device is linked to an upper device
7283 * @dev: device
7284 * @upper_dev: upper device to check
7285 *
7286 * Find out if a device is linked to specified upper device and return true
7287 * in case it is. Note that this checks only immediate upper device,
7288 * not through a complete stack of devices. The caller must hold the RTNL lock.
7289 */
7290bool netdev_has_upper_dev(struct net_device *dev,
7291 struct net_device *upper_dev)
7292{
eff74233
TY
7293 struct netdev_nested_priv priv = {
7294 .data = (void *)upper_dev,
7295 };
7296
9ff162a8
JP
7297 ASSERT_RTNL();
7298
32b6d34f 7299 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
eff74233 7300 &priv);
9ff162a8
JP
7301}
7302EXPORT_SYMBOL(netdev_has_upper_dev);
7303
1a3f060c 7304/**
c1639be9 7305 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
1a3f060c
DA
7306 * @dev: device
7307 * @upper_dev: upper device to check
7308 *
7309 * Find out if a device is linked to specified upper device and return true
7310 * in case it is. Note that this checks the entire upper device chain.
7311 * The caller must hold rcu lock.
7312 */
7313
1a3f060c
DA
7314bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
7315 struct net_device *upper_dev)
7316{
eff74233
TY
7317 struct netdev_nested_priv priv = {
7318 .data = (void *)upper_dev,
7319 };
7320
32b6d34f 7321 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
eff74233 7322 &priv);
1a3f060c
DA
7323}
7324EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
7325
9ff162a8
JP
7326/**
7327 * netdev_has_any_upper_dev - Check if device is linked to some device
7328 * @dev: device
7329 *
7330 * Find out if a device is linked to an upper device and return true in case
7331 * it is. The caller must hold the RTNL lock.
7332 */
25cc72a3 7333bool netdev_has_any_upper_dev(struct net_device *dev)
9ff162a8
JP
7334{
7335 ASSERT_RTNL();
7336
f1170fd4 7337 return !list_empty(&dev->adj_list.upper);
9ff162a8 7338}
25cc72a3 7339EXPORT_SYMBOL(netdev_has_any_upper_dev);
9ff162a8
JP
7340
7341/**
7342 * netdev_master_upper_dev_get - Get master upper device
7343 * @dev: device
7344 *
7345 * Find a master upper device and return pointer to it or NULL in case
7346 * it's not there. The caller must hold the RTNL lock.
7347 */
7348struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
7349{
aa9d8560 7350 struct netdev_adjacent *upper;
9ff162a8
JP
7351
7352 ASSERT_RTNL();
7353
2f268f12 7354 if (list_empty(&dev->adj_list.upper))
9ff162a8
JP
7355 return NULL;
7356
2f268f12 7357 upper = list_first_entry(&dev->adj_list.upper,
aa9d8560 7358 struct netdev_adjacent, list);
9ff162a8
JP
7359 if (likely(upper->master))
7360 return upper->dev;
7361 return NULL;
7362}
7363EXPORT_SYMBOL(netdev_master_upper_dev_get);
7364
32b6d34f
TY
7365static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
7366{
7367 struct netdev_adjacent *upper;
7368
7369 ASSERT_RTNL();
7370
7371 if (list_empty(&dev->adj_list.upper))
7372 return NULL;
7373
7374 upper = list_first_entry(&dev->adj_list.upper,
7375 struct netdev_adjacent, list);
7376 if (likely(upper->master) && !upper->ignore)
7377 return upper->dev;
7378 return NULL;
7379}
7380
0f524a80
DA
7381/**
7382 * netdev_has_any_lower_dev - Check if device is linked to some device
7383 * @dev: device
7384 *
7385 * Find out if a device is linked to a lower device and return true in case
7386 * it is. The caller must hold the RTNL lock.
7387 */
7388static bool netdev_has_any_lower_dev(struct net_device *dev)
7389{
7390 ASSERT_RTNL();
7391
7392 return !list_empty(&dev->adj_list.lower);
7393}
7394
b6ccba4c
VF
7395void *netdev_adjacent_get_private(struct list_head *adj_list)
7396{
7397 struct netdev_adjacent *adj;
7398
7399 adj = list_entry(adj_list, struct netdev_adjacent, list);
7400
7401 return adj->private;
7402}
7403EXPORT_SYMBOL(netdev_adjacent_get_private);
7404
44a40855
VY
7405/**
7406 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7407 * @dev: device
7408 * @iter: list_head ** of the current position
7409 *
7410 * Gets the next device from the dev's upper list, starting from iter
7411 * position. The caller must hold RCU read lock.
7412 */
7413struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
7414 struct list_head **iter)
7415{
7416 struct netdev_adjacent *upper;
7417
7418 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7419
7420 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7421
7422 if (&upper->list == &dev->adj_list.upper)
7423 return NULL;
7424
7425 *iter = &upper->list;
7426
7427 return upper->dev;
7428}
7429EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
7430
32b6d34f
TY
7431static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
7432 struct list_head **iter,
7433 bool *ignore)
5343da4c
TY
7434{
7435 struct netdev_adjacent *upper;
7436
7437 upper = list_entry((*iter)->next, struct netdev_adjacent, list);
7438
7439 if (&upper->list == &dev->adj_list.upper)
7440 return NULL;
7441
7442 *iter = &upper->list;
32b6d34f 7443 *ignore = upper->ignore;
5343da4c
TY
7444
7445 return upper->dev;
7446}
7447
1a3f060c
DA
7448static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
7449 struct list_head **iter)
7450{
7451 struct netdev_adjacent *upper;
7452
7453 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
7454
7455 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7456
7457 if (&upper->list == &dev->adj_list.upper)
7458 return NULL;
7459
7460 *iter = &upper->list;
7461
7462 return upper->dev;
7463}
7464
32b6d34f
TY
7465static int __netdev_walk_all_upper_dev(struct net_device *dev,
7466 int (*fn)(struct net_device *dev,
eff74233
TY
7467 struct netdev_nested_priv *priv),
7468 struct netdev_nested_priv *priv)
5343da4c
TY
7469{
7470 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7471 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7472 int ret, cur = 0;
32b6d34f 7473 bool ignore;
5343da4c
TY
7474
7475 now = dev;
7476 iter = &dev->adj_list.upper;
7477
7478 while (1) {
7479 if (now != dev) {
eff74233 7480 ret = fn(now, priv);
5343da4c
TY
7481 if (ret)
7482 return ret;
7483 }
7484
7485 next = NULL;
7486 while (1) {
32b6d34f 7487 udev = __netdev_next_upper_dev(now, &iter, &ignore);
5343da4c
TY
7488 if (!udev)
7489 break;
32b6d34f
TY
7490 if (ignore)
7491 continue;
5343da4c
TY
7492
7493 next = udev;
7494 niter = &udev->adj_list.upper;
7495 dev_stack[cur] = now;
7496 iter_stack[cur++] = iter;
7497 break;
7498 }
7499
7500 if (!next) {
7501 if (!cur)
7502 return 0;
7503 next = dev_stack[--cur];
7504 niter = iter_stack[cur];
7505 }
7506
7507 now = next;
7508 iter = niter;
7509 }
7510
7511 return 0;
7512}
7513
1a3f060c
DA
7514int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
7515 int (*fn)(struct net_device *dev,
eff74233
TY
7516 struct netdev_nested_priv *priv),
7517 struct netdev_nested_priv *priv)
1a3f060c 7518{
5343da4c
TY
7519 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7520 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7521 int ret, cur = 0;
1a3f060c 7522
5343da4c
TY
7523 now = dev;
7524 iter = &dev->adj_list.upper;
1a3f060c 7525
5343da4c
TY
7526 while (1) {
7527 if (now != dev) {
eff74233 7528 ret = fn(now, priv);
5343da4c
TY
7529 if (ret)
7530 return ret;
7531 }
7532
7533 next = NULL;
7534 while (1) {
7535 udev = netdev_next_upper_dev_rcu(now, &iter);
7536 if (!udev)
7537 break;
7538
7539 next = udev;
7540 niter = &udev->adj_list.upper;
7541 dev_stack[cur] = now;
7542 iter_stack[cur++] = iter;
7543 break;
7544 }
7545
7546 if (!next) {
7547 if (!cur)
7548 return 0;
7549 next = dev_stack[--cur];
7550 niter = iter_stack[cur];
7551 }
7552
7553 now = next;
7554 iter = niter;
1a3f060c
DA
7555 }
7556
7557 return 0;
7558}
7559EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
7560
32b6d34f
TY
7561static bool __netdev_has_upper_dev(struct net_device *dev,
7562 struct net_device *upper_dev)
7563{
eff74233 7564 struct netdev_nested_priv priv = {
1fc70edb 7565 .flags = 0,
eff74233
TY
7566 .data = (void *)upper_dev,
7567 };
7568
32b6d34f
TY
7569 ASSERT_RTNL();
7570
7571 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
eff74233 7572 &priv);
32b6d34f
TY
7573}
7574
31088a11
VF
7575/**
7576 * netdev_lower_get_next_private - Get the next ->private from the
7577 * lower neighbour list
7578 * @dev: device
7579 * @iter: list_head ** of the current position
7580 *
7581 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7582 * list, starting from iter position. The caller must hold either hold the
7583 * RTNL lock or its own locking that guarantees that the neighbour lower
b469139e 7584 * list will remain unchanged.
31088a11
VF
7585 */
7586void *netdev_lower_get_next_private(struct net_device *dev,
7587 struct list_head **iter)
7588{
7589 struct netdev_adjacent *lower;
7590
7591 lower = list_entry(*iter, struct netdev_adjacent, list);
7592
7593 if (&lower->list == &dev->adj_list.lower)
7594 return NULL;
7595
6859e7df 7596 *iter = lower->list.next;
31088a11
VF
7597
7598 return lower->private;
7599}
7600EXPORT_SYMBOL(netdev_lower_get_next_private);
7601
7602/**
7603 * netdev_lower_get_next_private_rcu - Get the next ->private from the
7604 * lower neighbour list, RCU
7605 * variant
7606 * @dev: device
7607 * @iter: list_head ** of the current position
7608 *
7609 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7610 * list, starting from iter position. The caller must hold RCU read lock.
7611 */
7612void *netdev_lower_get_next_private_rcu(struct net_device *dev,
7613 struct list_head **iter)
7614{
7615 struct netdev_adjacent *lower;
7616
7617 WARN_ON_ONCE(!rcu_read_lock_held());
7618
7619 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7620
7621 if (&lower->list == &dev->adj_list.lower)
7622 return NULL;
7623
6859e7df 7624 *iter = &lower->list;
31088a11
VF
7625
7626 return lower->private;
7627}
7628EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
7629
4085ebe8
VY
7630/**
7631 * netdev_lower_get_next - Get the next device from the lower neighbour
7632 * list
7633 * @dev: device
7634 * @iter: list_head ** of the current position
7635 *
7636 * Gets the next netdev_adjacent from the dev's lower neighbour
7637 * list, starting from iter position. The caller must hold RTNL lock or
7638 * its own locking that guarantees that the neighbour lower
b469139e 7639 * list will remain unchanged.
4085ebe8
VY
7640 */
7641void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
7642{
7643 struct netdev_adjacent *lower;
7644
cfdd28be 7645 lower = list_entry(*iter, struct netdev_adjacent, list);
4085ebe8
VY
7646
7647 if (&lower->list == &dev->adj_list.lower)
7648 return NULL;
7649
cfdd28be 7650 *iter = lower->list.next;
4085ebe8
VY
7651
7652 return lower->dev;
7653}
7654EXPORT_SYMBOL(netdev_lower_get_next);
7655
1a3f060c
DA
7656static struct net_device *netdev_next_lower_dev(struct net_device *dev,
7657 struct list_head **iter)
7658{
7659 struct netdev_adjacent *lower;
7660
46b5ab1a 7661 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
1a3f060c
DA
7662
7663 if (&lower->list == &dev->adj_list.lower)
7664 return NULL;
7665
46b5ab1a 7666 *iter = &lower->list;
1a3f060c
DA
7667
7668 return lower->dev;
7669}
7670
32b6d34f
TY
7671static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
7672 struct list_head **iter,
7673 bool *ignore)
7674{
7675 struct netdev_adjacent *lower;
7676
7677 lower = list_entry((*iter)->next, struct netdev_adjacent, list);
7678
7679 if (&lower->list == &dev->adj_list.lower)
7680 return NULL;
7681
7682 *iter = &lower->list;
7683 *ignore = lower->ignore;
7684
7685 return lower->dev;
7686}
7687
1a3f060c
DA
7688int netdev_walk_all_lower_dev(struct net_device *dev,
7689 int (*fn)(struct net_device *dev,
eff74233
TY
7690 struct netdev_nested_priv *priv),
7691 struct netdev_nested_priv *priv)
1a3f060c 7692{
5343da4c
TY
7693 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7694 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7695 int ret, cur = 0;
1a3f060c 7696
5343da4c
TY
7697 now = dev;
7698 iter = &dev->adj_list.lower;
1a3f060c 7699
5343da4c
TY
7700 while (1) {
7701 if (now != dev) {
eff74233 7702 ret = fn(now, priv);
5343da4c
TY
7703 if (ret)
7704 return ret;
7705 }
7706
7707 next = NULL;
7708 while (1) {
7709 ldev = netdev_next_lower_dev(now, &iter);
7710 if (!ldev)
7711 break;
7712
7713 next = ldev;
7714 niter = &ldev->adj_list.lower;
7715 dev_stack[cur] = now;
7716 iter_stack[cur++] = iter;
7717 break;
7718 }
7719
7720 if (!next) {
7721 if (!cur)
7722 return 0;
7723 next = dev_stack[--cur];
7724 niter = iter_stack[cur];
7725 }
7726
7727 now = next;
7728 iter = niter;
1a3f060c
DA
7729 }
7730
7731 return 0;
7732}
7733EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
7734
32b6d34f
TY
7735static int __netdev_walk_all_lower_dev(struct net_device *dev,
7736 int (*fn)(struct net_device *dev,
eff74233
TY
7737 struct netdev_nested_priv *priv),
7738 struct netdev_nested_priv *priv)
32b6d34f
TY
7739{
7740 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7741 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7742 int ret, cur = 0;
7743 bool ignore;
7744
7745 now = dev;
7746 iter = &dev->adj_list.lower;
7747
7748 while (1) {
7749 if (now != dev) {
eff74233 7750 ret = fn(now, priv);
32b6d34f
TY
7751 if (ret)
7752 return ret;
7753 }
7754
7755 next = NULL;
7756 while (1) {
7757 ldev = __netdev_next_lower_dev(now, &iter, &ignore);
7758 if (!ldev)
7759 break;
7760 if (ignore)
7761 continue;
7762
7763 next = ldev;
7764 niter = &ldev->adj_list.lower;
7765 dev_stack[cur] = now;
7766 iter_stack[cur++] = iter;
7767 break;
7768 }
7769
7770 if (!next) {
7771 if (!cur)
7772 return 0;
7773 next = dev_stack[--cur];
7774 niter = iter_stack[cur];
7775 }
7776
7777 now = next;
7778 iter = niter;
7779 }
7780
7781 return 0;
7782}
7783
7151affe
TY
7784struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
7785 struct list_head **iter)
1a3f060c
DA
7786{
7787 struct netdev_adjacent *lower;
7788
7789 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
7790 if (&lower->list == &dev->adj_list.lower)
7791 return NULL;
7792
7793 *iter = &lower->list;
7794
7795 return lower->dev;
7796}
7151affe 7797EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
1a3f060c 7798
5343da4c
TY
7799static u8 __netdev_upper_depth(struct net_device *dev)
7800{
7801 struct net_device *udev;
7802 struct list_head *iter;
7803 u8 max_depth = 0;
32b6d34f 7804 bool ignore;
5343da4c
TY
7805
7806 for (iter = &dev->adj_list.upper,
32b6d34f 7807 udev = __netdev_next_upper_dev(dev, &iter, &ignore);
5343da4c 7808 udev;
32b6d34f
TY
7809 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
7810 if (ignore)
7811 continue;
5343da4c
TY
7812 if (max_depth < udev->upper_level)
7813 max_depth = udev->upper_level;
7814 }
7815
7816 return max_depth;
7817}
7818
7819static u8 __netdev_lower_depth(struct net_device *dev)
1a3f060c
DA
7820{
7821 struct net_device *ldev;
7822 struct list_head *iter;
5343da4c 7823 u8 max_depth = 0;
32b6d34f 7824 bool ignore;
1a3f060c
DA
7825
7826 for (iter = &dev->adj_list.lower,
32b6d34f 7827 ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
1a3f060c 7828 ldev;
32b6d34f
TY
7829 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
7830 if (ignore)
7831 continue;
5343da4c
TY
7832 if (max_depth < ldev->lower_level)
7833 max_depth = ldev->lower_level;
7834 }
1a3f060c 7835
5343da4c
TY
7836 return max_depth;
7837}
7838
eff74233
TY
7839static int __netdev_update_upper_level(struct net_device *dev,
7840 struct netdev_nested_priv *__unused)
5343da4c
TY
7841{
7842 dev->upper_level = __netdev_upper_depth(dev) + 1;
7843 return 0;
7844}
7845
eff74233 7846static int __netdev_update_lower_level(struct net_device *dev,
1fc70edb 7847 struct netdev_nested_priv *priv)
5343da4c
TY
7848{
7849 dev->lower_level = __netdev_lower_depth(dev) + 1;
1fc70edb
TY
7850
7851#ifdef CONFIG_LOCKDEP
7852 if (!priv)
7853 return 0;
7854
7855 if (priv->flags & NESTED_SYNC_IMM)
7856 dev->nested_level = dev->lower_level - 1;
7857 if (priv->flags & NESTED_SYNC_TODO)
7858 net_unlink_todo(dev);
7859#endif
5343da4c
TY
7860 return 0;
7861}
7862
7863int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
7864 int (*fn)(struct net_device *dev,
eff74233
TY
7865 struct netdev_nested_priv *priv),
7866 struct netdev_nested_priv *priv)
5343da4c
TY
7867{
7868 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
7869 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
7870 int ret, cur = 0;
7871
7872 now = dev;
7873 iter = &dev->adj_list.lower;
7874
7875 while (1) {
7876 if (now != dev) {
eff74233 7877 ret = fn(now, priv);
5343da4c
TY
7878 if (ret)
7879 return ret;
7880 }
7881
7882 next = NULL;
7883 while (1) {
7884 ldev = netdev_next_lower_dev_rcu(now, &iter);
7885 if (!ldev)
7886 break;
7887
7888 next = ldev;
7889 niter = &ldev->adj_list.lower;
7890 dev_stack[cur] = now;
7891 iter_stack[cur++] = iter;
7892 break;
7893 }
7894
7895 if (!next) {
7896 if (!cur)
7897 return 0;
7898 next = dev_stack[--cur];
7899 niter = iter_stack[cur];
7900 }
7901
7902 now = next;
7903 iter = niter;
1a3f060c
DA
7904 }
7905
7906 return 0;
7907}
7908EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
7909
e001bfad 7910/**
7911 * netdev_lower_get_first_private_rcu - Get the first ->private from the
7912 * lower neighbour list, RCU
7913 * variant
7914 * @dev: device
7915 *
7916 * Gets the first netdev_adjacent->private from the dev's lower neighbour
7917 * list. The caller must hold RCU read lock.
7918 */
7919void *netdev_lower_get_first_private_rcu(struct net_device *dev)
7920{
7921 struct netdev_adjacent *lower;
7922
7923 lower = list_first_or_null_rcu(&dev->adj_list.lower,
7924 struct netdev_adjacent, list);
7925 if (lower)
7926 return lower->private;
7927 return NULL;
7928}
7929EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
7930
9ff162a8
JP
7931/**
7932 * netdev_master_upper_dev_get_rcu - Get master upper device
7933 * @dev: device
7934 *
7935 * Find a master upper device and return pointer to it or NULL in case
7936 * it's not there. The caller must hold the RCU read lock.
7937 */
7938struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
7939{
aa9d8560 7940 struct netdev_adjacent *upper;
9ff162a8 7941
2f268f12 7942 upper = list_first_or_null_rcu(&dev->adj_list.upper,
aa9d8560 7943 struct netdev_adjacent, list);
9ff162a8
JP
7944 if (upper && likely(upper->master))
7945 return upper->dev;
7946 return NULL;
7947}
7948EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
7949
0a59f3a9 7950static int netdev_adjacent_sysfs_add(struct net_device *dev,
3ee32707
VF
7951 struct net_device *adj_dev,
7952 struct list_head *dev_list)
7953{
7954 char linkname[IFNAMSIZ+7];
f4563a75 7955
3ee32707
VF
7956 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7957 "upper_%s" : "lower_%s", adj_dev->name);
7958 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
7959 linkname);
7960}
0a59f3a9 7961static void netdev_adjacent_sysfs_del(struct net_device *dev,
3ee32707
VF
7962 char *name,
7963 struct list_head *dev_list)
7964{
7965 char linkname[IFNAMSIZ+7];
f4563a75 7966
3ee32707
VF
7967 sprintf(linkname, dev_list == &dev->adj_list.upper ?
7968 "upper_%s" : "lower_%s", name);
7969 sysfs_remove_link(&(dev->dev.kobj), linkname);
7970}
7971
7ce64c79
AF
7972static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
7973 struct net_device *adj_dev,
7974 struct list_head *dev_list)
7975{
7976 return (dev_list == &dev->adj_list.upper ||
7977 dev_list == &dev->adj_list.lower) &&
7978 net_eq(dev_net(dev), dev_net(adj_dev));
7979}
3ee32707 7980
5d261913
VF
7981static int __netdev_adjacent_dev_insert(struct net_device *dev,
7982 struct net_device *adj_dev,
7863c054 7983 struct list_head *dev_list,
402dae96 7984 void *private, bool master)
5d261913
VF
7985{
7986 struct netdev_adjacent *adj;
842d67a7 7987 int ret;
5d261913 7988
6ea29da1 7989 adj = __netdev_find_adj(adj_dev, dev_list);
5d261913
VF
7990
7991 if (adj) {
790510d9 7992 adj->ref_nr += 1;
67b62f98
DA
7993 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
7994 dev->name, adj_dev->name, adj->ref_nr);
7995
5d261913
VF
7996 return 0;
7997 }
7998
7999 adj = kmalloc(sizeof(*adj), GFP_KERNEL);
8000 if (!adj)
8001 return -ENOMEM;
8002
8003 adj->dev = adj_dev;
8004 adj->master = master;
790510d9 8005 adj->ref_nr = 1;
402dae96 8006 adj->private = private;
32b6d34f 8007 adj->ignore = false;
5d261913 8008 dev_hold(adj_dev);
2f268f12 8009
67b62f98
DA
8010 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
8011 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
5d261913 8012
7ce64c79 8013 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
3ee32707 8014 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5831d66e
VF
8015 if (ret)
8016 goto free_adj;
8017 }
8018
7863c054 8019 /* Ensure that master link is always the first item in list. */
842d67a7
VF
8020 if (master) {
8021 ret = sysfs_create_link(&(dev->dev.kobj),
8022 &(adj_dev->dev.kobj), "master");
8023 if (ret)
5831d66e 8024 goto remove_symlinks;
842d67a7 8025
7863c054 8026 list_add_rcu(&adj->list, dev_list);
842d67a7 8027 } else {
7863c054 8028 list_add_tail_rcu(&adj->list, dev_list);
842d67a7 8029 }
5d261913
VF
8030
8031 return 0;
842d67a7 8032
5831d66e 8033remove_symlinks:
7ce64c79 8034 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
3ee32707 8035 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
842d67a7
VF
8036free_adj:
8037 kfree(adj);
974daef7 8038 dev_put(adj_dev);
842d67a7
VF
8039
8040 return ret;
5d261913
VF
8041}
8042
1d143d9f 8043static void __netdev_adjacent_dev_remove(struct net_device *dev,
8044 struct net_device *adj_dev,
93409033 8045 u16 ref_nr,
1d143d9f 8046 struct list_head *dev_list)
5d261913
VF
8047{
8048 struct netdev_adjacent *adj;
8049
67b62f98
DA
8050 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
8051 dev->name, adj_dev->name, ref_nr);
8052
6ea29da1 8053 adj = __netdev_find_adj(adj_dev, dev_list);
5d261913 8054
2f268f12 8055 if (!adj) {
67b62f98 8056 pr_err("Adjacency does not exist for device %s from %s\n",
2f268f12 8057 dev->name, adj_dev->name);
67b62f98
DA
8058 WARN_ON(1);
8059 return;
2f268f12 8060 }
5d261913 8061
93409033 8062 if (adj->ref_nr > ref_nr) {
67b62f98
DA
8063 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
8064 dev->name, adj_dev->name, ref_nr,
8065 adj->ref_nr - ref_nr);
93409033 8066 adj->ref_nr -= ref_nr;
5d261913
VF
8067 return;
8068 }
8069
842d67a7
VF
8070 if (adj->master)
8071 sysfs_remove_link(&(dev->dev.kobj), "master");
8072
7ce64c79 8073 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
3ee32707 8074 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5831d66e 8075
5d261913 8076 list_del_rcu(&adj->list);
67b62f98 8077 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
2f268f12 8078 adj_dev->name, dev->name, adj_dev->name);
5d261913
VF
8079 dev_put(adj_dev);
8080 kfree_rcu(adj, rcu);
8081}
8082
1d143d9f 8083static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
8084 struct net_device *upper_dev,
8085 struct list_head *up_list,
8086 struct list_head *down_list,
8087 void *private, bool master)
5d261913
VF
8088{
8089 int ret;
8090
790510d9 8091 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
93409033 8092 private, master);
5d261913
VF
8093 if (ret)
8094 return ret;
8095
790510d9 8096 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
93409033 8097 private, false);
5d261913 8098 if (ret) {
790510d9 8099 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
5d261913
VF
8100 return ret;
8101 }
8102
8103 return 0;
8104}
8105
1d143d9f 8106static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
8107 struct net_device *upper_dev,
93409033 8108 u16 ref_nr,
1d143d9f 8109 struct list_head *up_list,
8110 struct list_head *down_list)
5d261913 8111{
93409033
AC
8112 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
8113 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
5d261913
VF
8114}
8115
1d143d9f 8116static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
8117 struct net_device *upper_dev,
8118 void *private, bool master)
2f268f12 8119{
f1170fd4
DA
8120 return __netdev_adjacent_dev_link_lists(dev, upper_dev,
8121 &dev->adj_list.upper,
8122 &upper_dev->adj_list.lower,
8123 private, master);
5d261913
VF
8124}
8125
1d143d9f 8126static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
8127 struct net_device *upper_dev)
2f268f12 8128{
93409033 8129 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
2f268f12
VF
8130 &dev->adj_list.upper,
8131 &upper_dev->adj_list.lower);
8132}
5d261913 8133
9ff162a8 8134static int __netdev_upper_dev_link(struct net_device *dev,
402dae96 8135 struct net_device *upper_dev, bool master,
42ab19ee 8136 void *upper_priv, void *upper_info,
1fc70edb 8137 struct netdev_nested_priv *priv,
42ab19ee 8138 struct netlink_ext_ack *extack)
9ff162a8 8139{
51d0c047
DA
8140 struct netdev_notifier_changeupper_info changeupper_info = {
8141 .info = {
8142 .dev = dev,
42ab19ee 8143 .extack = extack,
51d0c047
DA
8144 },
8145 .upper_dev = upper_dev,
8146 .master = master,
8147 .linking = true,
8148 .upper_info = upper_info,
8149 };
50d629e7 8150 struct net_device *master_dev;
5d261913 8151 int ret = 0;
9ff162a8
JP
8152
8153 ASSERT_RTNL();
8154
8155 if (dev == upper_dev)
8156 return -EBUSY;
8157
8158 /* To prevent loops, check if dev is not upper device to upper_dev. */
32b6d34f 8159 if (__netdev_has_upper_dev(upper_dev, dev))
9ff162a8
JP
8160 return -EBUSY;
8161
5343da4c
TY
8162 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
8163 return -EMLINK;
8164
50d629e7 8165 if (!master) {
32b6d34f 8166 if (__netdev_has_upper_dev(dev, upper_dev))
50d629e7
MM
8167 return -EEXIST;
8168 } else {
32b6d34f 8169 master_dev = __netdev_master_upper_dev_get(dev);
50d629e7
MM
8170 if (master_dev)
8171 return master_dev == upper_dev ? -EEXIST : -EBUSY;
8172 }
9ff162a8 8173
51d0c047 8174 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
573c7ba0
JP
8175 &changeupper_info.info);
8176 ret = notifier_to_errno(ret);
8177 if (ret)
8178 return ret;
8179
6dffb044 8180 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
402dae96 8181 master);
5d261913
VF
8182 if (ret)
8183 return ret;
9ff162a8 8184
51d0c047 8185 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
b03804e7
IS
8186 &changeupper_info.info);
8187 ret = notifier_to_errno(ret);
8188 if (ret)
f1170fd4 8189 goto rollback;
b03804e7 8190
5343da4c 8191 __netdev_update_upper_level(dev, NULL);
32b6d34f 8192 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
5343da4c 8193
1fc70edb 8194 __netdev_update_lower_level(upper_dev, priv);
32b6d34f 8195 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
1fc70edb 8196 priv);
5343da4c 8197
9ff162a8 8198 return 0;
5d261913 8199
f1170fd4 8200rollback:
2f268f12 8201 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5d261913
VF
8202
8203 return ret;
9ff162a8
JP
8204}
8205
8206/**
8207 * netdev_upper_dev_link - Add a link to the upper device
8208 * @dev: device
8209 * @upper_dev: new upper device
7a006d59 8210 * @extack: netlink extended ack
9ff162a8
JP
8211 *
8212 * Adds a link to device which is upper to this one. The caller must hold
8213 * the RTNL lock. On a failure a negative errno code is returned.
8214 * On success the reference counts are adjusted and the function
8215 * returns zero.
8216 */
8217int netdev_upper_dev_link(struct net_device *dev,
42ab19ee
DA
8218 struct net_device *upper_dev,
8219 struct netlink_ext_ack *extack)
9ff162a8 8220{
1fc70edb
TY
8221 struct netdev_nested_priv priv = {
8222 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8223 .data = NULL,
8224 };
8225
42ab19ee 8226 return __netdev_upper_dev_link(dev, upper_dev, false,
1fc70edb 8227 NULL, NULL, &priv, extack);
9ff162a8
JP
8228}
8229EXPORT_SYMBOL(netdev_upper_dev_link);
8230
8231/**
8232 * netdev_master_upper_dev_link - Add a master link to the upper device
8233 * @dev: device
8234 * @upper_dev: new upper device
6dffb044 8235 * @upper_priv: upper device private
29bf24af 8236 * @upper_info: upper info to be passed down via notifier
7a006d59 8237 * @extack: netlink extended ack
9ff162a8
JP
8238 *
8239 * Adds a link to device which is upper to this one. In this case, only
8240 * one master upper device can be linked, although other non-master devices
8241 * might be linked as well. The caller must hold the RTNL lock.
8242 * On a failure a negative errno code is returned. On success the reference
8243 * counts are adjusted and the function returns zero.
8244 */
8245int netdev_master_upper_dev_link(struct net_device *dev,
6dffb044 8246 struct net_device *upper_dev,
42ab19ee
DA
8247 void *upper_priv, void *upper_info,
8248 struct netlink_ext_ack *extack)
9ff162a8 8249{
1fc70edb
TY
8250 struct netdev_nested_priv priv = {
8251 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8252 .data = NULL,
8253 };
8254
29bf24af 8255 return __netdev_upper_dev_link(dev, upper_dev, true,
1fc70edb 8256 upper_priv, upper_info, &priv, extack);
9ff162a8
JP
8257}
8258EXPORT_SYMBOL(netdev_master_upper_dev_link);
8259
fe8300fd 8260static void __netdev_upper_dev_unlink(struct net_device *dev,
1fc70edb
TY
8261 struct net_device *upper_dev,
8262 struct netdev_nested_priv *priv)
9ff162a8 8263{
51d0c047
DA
8264 struct netdev_notifier_changeupper_info changeupper_info = {
8265 .info = {
8266 .dev = dev,
8267 },
8268 .upper_dev = upper_dev,
8269 .linking = false,
8270 };
f4563a75 8271
9ff162a8
JP
8272 ASSERT_RTNL();
8273
0e4ead9d 8274 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
0e4ead9d 8275
51d0c047 8276 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
573c7ba0
JP
8277 &changeupper_info.info);
8278
2f268f12 8279 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5d261913 8280
51d0c047 8281 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
0e4ead9d 8282 &changeupper_info.info);
5343da4c
TY
8283
8284 __netdev_update_upper_level(dev, NULL);
32b6d34f 8285 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
5343da4c 8286
1fc70edb 8287 __netdev_update_lower_level(upper_dev, priv);
32b6d34f 8288 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
1fc70edb 8289 priv);
9ff162a8 8290}
fe8300fd
TY
8291
8292/**
8293 * netdev_upper_dev_unlink - Removes a link to upper device
8294 * @dev: device
8295 * @upper_dev: new upper device
8296 *
8297 * Removes a link to device which is upper to this one. The caller must hold
8298 * the RTNL lock.
8299 */
8300void netdev_upper_dev_unlink(struct net_device *dev,
8301 struct net_device *upper_dev)
8302{
1fc70edb
TY
8303 struct netdev_nested_priv priv = {
8304 .flags = NESTED_SYNC_TODO,
8305 .data = NULL,
8306 };
8307
8308 __netdev_upper_dev_unlink(dev, upper_dev, &priv);
9ff162a8
JP
8309}
8310EXPORT_SYMBOL(netdev_upper_dev_unlink);
8311
32b6d34f
TY
8312static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
8313 struct net_device *lower_dev,
8314 bool val)
8315{
8316 struct netdev_adjacent *adj;
8317
8318 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
8319 if (adj)
8320 adj->ignore = val;
8321
8322 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
8323 if (adj)
8324 adj->ignore = val;
8325}
8326
8327static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
8328 struct net_device *lower_dev)
8329{
8330 __netdev_adjacent_dev_set(upper_dev, lower_dev, true);
8331}
8332
8333static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
8334 struct net_device *lower_dev)
8335{
8336 __netdev_adjacent_dev_set(upper_dev, lower_dev, false);
8337}
8338
8339int netdev_adjacent_change_prepare(struct net_device *old_dev,
8340 struct net_device *new_dev,
8341 struct net_device *dev,
8342 struct netlink_ext_ack *extack)
8343{
1fc70edb
TY
8344 struct netdev_nested_priv priv = {
8345 .flags = 0,
8346 .data = NULL,
8347 };
32b6d34f
TY
8348 int err;
8349
8350 if (!new_dev)
8351 return 0;
8352
8353 if (old_dev && new_dev != old_dev)
8354 netdev_adjacent_dev_disable(dev, old_dev);
1fc70edb
TY
8355 err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
8356 extack);
32b6d34f
TY
8357 if (err) {
8358 if (old_dev && new_dev != old_dev)
8359 netdev_adjacent_dev_enable(dev, old_dev);
8360 return err;
8361 }
8362
8363 return 0;
8364}
8365EXPORT_SYMBOL(netdev_adjacent_change_prepare);
8366
8367void netdev_adjacent_change_commit(struct net_device *old_dev,
8368 struct net_device *new_dev,
8369 struct net_device *dev)
8370{
1fc70edb
TY
8371 struct netdev_nested_priv priv = {
8372 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
8373 .data = NULL,
8374 };
8375
32b6d34f
TY
8376 if (!new_dev || !old_dev)
8377 return;
8378
8379 if (new_dev == old_dev)
8380 return;
8381
8382 netdev_adjacent_dev_enable(dev, old_dev);
1fc70edb 8383 __netdev_upper_dev_unlink(old_dev, dev, &priv);
32b6d34f
TY
8384}
8385EXPORT_SYMBOL(netdev_adjacent_change_commit);
8386
8387void netdev_adjacent_change_abort(struct net_device *old_dev,
8388 struct net_device *new_dev,
8389 struct net_device *dev)
8390{
1fc70edb
TY
8391 struct netdev_nested_priv priv = {
8392 .flags = 0,
8393 .data = NULL,
8394 };
8395
32b6d34f
TY
8396 if (!new_dev)
8397 return;
8398
8399 if (old_dev && new_dev != old_dev)
8400 netdev_adjacent_dev_enable(dev, old_dev);
8401
1fc70edb 8402 __netdev_upper_dev_unlink(new_dev, dev, &priv);
32b6d34f
TY
8403}
8404EXPORT_SYMBOL(netdev_adjacent_change_abort);
8405
61bd3857
MS
8406/**
8407 * netdev_bonding_info_change - Dispatch event about slave change
8408 * @dev: device
4a26e453 8409 * @bonding_info: info to dispatch
61bd3857
MS
8410 *
8411 * Send NETDEV_BONDING_INFO to netdev notifiers with info.
8412 * The caller must hold the RTNL lock.
8413 */
8414void netdev_bonding_info_change(struct net_device *dev,
8415 struct netdev_bonding_info *bonding_info)
8416{
51d0c047
DA
8417 struct netdev_notifier_bonding_info info = {
8418 .info.dev = dev,
8419 };
61bd3857
MS
8420
8421 memcpy(&info.bonding_info, bonding_info,
8422 sizeof(struct netdev_bonding_info));
51d0c047 8423 call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
61bd3857
MS
8424 &info.info);
8425}
8426EXPORT_SYMBOL(netdev_bonding_info_change);
8427
cff9f12b
MG
8428/**
8429 * netdev_get_xmit_slave - Get the xmit slave of master device
8842500d 8430 * @dev: device
cff9f12b
MG
8431 * @skb: The packet
8432 * @all_slaves: assume all the slaves are active
8433 *
8434 * The reference counters are not incremented so the caller must be
8435 * careful with locks. The caller must hold RCU lock.
8436 * %NULL is returned if no slave is found.
8437 */
8438
8439struct net_device *netdev_get_xmit_slave(struct net_device *dev,
8440 struct sk_buff *skb,
8441 bool all_slaves)
8442{
8443 const struct net_device_ops *ops = dev->netdev_ops;
8444
8445 if (!ops->ndo_get_xmit_slave)
8446 return NULL;
8447 return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
8448}
8449EXPORT_SYMBOL(netdev_get_xmit_slave);
8450
719a402c
TT
8451static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
8452 struct sock *sk)
8453{
8454 const struct net_device_ops *ops = dev->netdev_ops;
8455
8456 if (!ops->ndo_sk_get_lower_dev)
8457 return NULL;
8458 return ops->ndo_sk_get_lower_dev(dev, sk);
8459}
8460
8461/**
8462 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8463 * @dev: device
8464 * @sk: the socket
8465 *
8466 * %NULL is returned if no lower device is found.
8467 */
8468
8469struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
8470 struct sock *sk)
8471{
8472 struct net_device *lower;
8473
8474 lower = netdev_sk_get_lower_dev(dev, sk);
8475 while (lower) {
8476 dev = lower;
8477 lower = netdev_sk_get_lower_dev(dev, sk);
8478 }
8479
8480 return dev;
8481}
8482EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
8483
2ce1ee17 8484static void netdev_adjacent_add_links(struct net_device *dev)
4c75431a
AF
8485{
8486 struct netdev_adjacent *iter;
8487
8488 struct net *net = dev_net(dev);
8489
8490 list_for_each_entry(iter, &dev->adj_list.upper, list) {
be4da0e3 8491 if (!net_eq(net, dev_net(iter->dev)))
4c75431a
AF
8492 continue;
8493 netdev_adjacent_sysfs_add(iter->dev, dev,
8494 &iter->dev->adj_list.lower);
8495 netdev_adjacent_sysfs_add(dev, iter->dev,
8496 &dev->adj_list.upper);
8497 }
8498
8499 list_for_each_entry(iter, &dev->adj_list.lower, list) {
be4da0e3 8500 if (!net_eq(net, dev_net(iter->dev)))
4c75431a
AF
8501 continue;
8502 netdev_adjacent_sysfs_add(iter->dev, dev,
8503 &iter->dev->adj_list.upper);
8504 netdev_adjacent_sysfs_add(dev, iter->dev,
8505 &dev->adj_list.lower);
8506 }
8507}
8508
2ce1ee17 8509static void netdev_adjacent_del_links(struct net_device *dev)
4c75431a
AF
8510{
8511 struct netdev_adjacent *iter;
8512
8513 struct net *net = dev_net(dev);
8514
8515 list_for_each_entry(iter, &dev->adj_list.upper, list) {
be4da0e3 8516 if (!net_eq(net, dev_net(iter->dev)))
4c75431a
AF
8517 continue;
8518 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8519 &iter->dev->adj_list.lower);
8520 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8521 &dev->adj_list.upper);
8522 }
8523
8524 list_for_each_entry(iter, &dev->adj_list.lower, list) {
be4da0e3 8525 if (!net_eq(net, dev_net(iter->dev)))
4c75431a
AF
8526 continue;
8527 netdev_adjacent_sysfs_del(iter->dev, dev->name,
8528 &iter->dev->adj_list.upper);
8529 netdev_adjacent_sysfs_del(dev, iter->dev->name,
8530 &dev->adj_list.lower);
8531 }
8532}
8533
5bb025fa 8534void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
402dae96 8535{
5bb025fa 8536 struct netdev_adjacent *iter;
402dae96 8537
4c75431a
AF
8538 struct net *net = dev_net(dev);
8539
5bb025fa 8540 list_for_each_entry(iter, &dev->adj_list.upper, list) {
be4da0e3 8541 if (!net_eq(net, dev_net(iter->dev)))
4c75431a 8542 continue;
5bb025fa
VF
8543 netdev_adjacent_sysfs_del(iter->dev, oldname,
8544 &iter->dev->adj_list.lower);
8545 netdev_adjacent_sysfs_add(iter->dev, dev,
8546 &iter->dev->adj_list.lower);
8547 }
402dae96 8548
5bb025fa 8549 list_for_each_entry(iter, &dev->adj_list.lower, list) {
be4da0e3 8550 if (!net_eq(net, dev_net(iter->dev)))
4c75431a 8551 continue;
5bb025fa
VF
8552 netdev_adjacent_sysfs_del(iter->dev, oldname,
8553 &iter->dev->adj_list.upper);
8554 netdev_adjacent_sysfs_add(iter->dev, dev,
8555 &iter->dev->adj_list.upper);
8556 }
402dae96 8557}
402dae96
VF
8558
8559void *netdev_lower_dev_get_private(struct net_device *dev,
8560 struct net_device *lower_dev)
8561{
8562 struct netdev_adjacent *lower;
8563
8564 if (!lower_dev)
8565 return NULL;
6ea29da1 8566 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
402dae96
VF
8567 if (!lower)
8568 return NULL;
8569
8570 return lower->private;
8571}
8572EXPORT_SYMBOL(netdev_lower_dev_get_private);
8573
4085ebe8 8574
04d48266 8575/**
c1639be9 8576 * netdev_lower_state_changed - Dispatch event about lower device state change
04d48266
JP
8577 * @lower_dev: device
8578 * @lower_state_info: state to dispatch
8579 *
8580 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
8581 * The caller must hold the RTNL lock.
8582 */
8583void netdev_lower_state_changed(struct net_device *lower_dev,
8584 void *lower_state_info)
8585{
51d0c047
DA
8586 struct netdev_notifier_changelowerstate_info changelowerstate_info = {
8587 .info.dev = lower_dev,
8588 };
04d48266
JP
8589
8590 ASSERT_RTNL();
8591 changelowerstate_info.lower_state_info = lower_state_info;
51d0c047 8592 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
04d48266
JP
8593 &changelowerstate_info.info);
8594}
8595EXPORT_SYMBOL(netdev_lower_state_changed);
8596
b6c40d68
PM
8597static void dev_change_rx_flags(struct net_device *dev, int flags)
8598{
d314774c
SH
8599 const struct net_device_ops *ops = dev->netdev_ops;
8600
d2615bf4 8601 if (ops->ndo_change_rx_flags)
d314774c 8602 ops->ndo_change_rx_flags(dev, flags);
b6c40d68
PM
8603}
8604
991fb3f7 8605static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
1da177e4 8606{
b536db93 8607 unsigned int old_flags = dev->flags;
d04a48b0
EB
8608 kuid_t uid;
8609 kgid_t gid;
1da177e4 8610
24023451
PM
8611 ASSERT_RTNL();
8612
dad9b335
WC
8613 dev->flags |= IFF_PROMISC;
8614 dev->promiscuity += inc;
8615 if (dev->promiscuity == 0) {
8616 /*
8617 * Avoid overflow.
8618 * If inc causes overflow, untouch promisc and return error.
8619 */
8620 if (inc < 0)
8621 dev->flags &= ~IFF_PROMISC;
8622 else {
8623 dev->promiscuity -= inc;
7b6cd1ce
JP
8624 pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
8625 dev->name);
dad9b335
WC
8626 return -EOVERFLOW;
8627 }
8628 }
52609c0b 8629 if (dev->flags != old_flags) {
7b6cd1ce
JP
8630 pr_info("device %s %s promiscuous mode\n",
8631 dev->name,
8632 dev->flags & IFF_PROMISC ? "entered" : "left");
8192b0c4
DH
8633 if (audit_enabled) {
8634 current_uid_gid(&uid, &gid);
cdfb6b34
RGB
8635 audit_log(audit_context(), GFP_ATOMIC,
8636 AUDIT_ANOM_PROMISCUOUS,
8637 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
8638 dev->name, (dev->flags & IFF_PROMISC),
8639 (old_flags & IFF_PROMISC),
8640 from_kuid(&init_user_ns, audit_get_loginuid(current)),
8641 from_kuid(&init_user_ns, uid),
8642 from_kgid(&init_user_ns, gid),
8643 audit_get_sessionid(current));
8192b0c4 8644 }
24023451 8645
b6c40d68 8646 dev_change_rx_flags(dev, IFF_PROMISC);
1da177e4 8647 }
991fb3f7
ND
8648 if (notify)
8649 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
dad9b335 8650 return 0;
1da177e4
LT
8651}
8652
4417da66
PM
8653/**
8654 * dev_set_promiscuity - update promiscuity count on a device
8655 * @dev: device
8656 * @inc: modifier
8657 *
8658 * Add or remove promiscuity from a device. While the count in the device
8659 * remains above zero the interface remains promiscuous. Once it hits zero
8660 * the device reverts back to normal filtering operation. A negative inc
8661 * value is used to drop promiscuity on the device.
dad9b335 8662 * Return 0 if successful or a negative errno code on error.
4417da66 8663 */
dad9b335 8664int dev_set_promiscuity(struct net_device *dev, int inc)
4417da66 8665{
b536db93 8666 unsigned int old_flags = dev->flags;
dad9b335 8667 int err;
4417da66 8668
991fb3f7 8669 err = __dev_set_promiscuity(dev, inc, true);
4b5a698e 8670 if (err < 0)
dad9b335 8671 return err;
4417da66
PM
8672 if (dev->flags != old_flags)
8673 dev_set_rx_mode(dev);
dad9b335 8674 return err;
4417da66 8675}
d1b19dff 8676EXPORT_SYMBOL(dev_set_promiscuity);
4417da66 8677
991fb3f7 8678static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
1da177e4 8679{
991fb3f7 8680 unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
1da177e4 8681
24023451
PM
8682 ASSERT_RTNL();
8683
1da177e4 8684 dev->flags |= IFF_ALLMULTI;
dad9b335
WC
8685 dev->allmulti += inc;
8686 if (dev->allmulti == 0) {
8687 /*
8688 * Avoid overflow.
8689 * If inc causes overflow, untouch allmulti and return error.
8690 */
8691 if (inc < 0)
8692 dev->flags &= ~IFF_ALLMULTI;
8693 else {
8694 dev->allmulti -= inc;
7b6cd1ce
JP
8695 pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
8696 dev->name);
dad9b335
WC
8697 return -EOVERFLOW;
8698 }
8699 }
24023451 8700 if (dev->flags ^ old_flags) {
b6c40d68 8701 dev_change_rx_flags(dev, IFF_ALLMULTI);
4417da66 8702 dev_set_rx_mode(dev);
991fb3f7
ND
8703 if (notify)
8704 __dev_notify_flags(dev, old_flags,
8705 dev->gflags ^ old_gflags);
24023451 8706 }
dad9b335 8707 return 0;
4417da66 8708}
991fb3f7
ND
8709
8710/**
8711 * dev_set_allmulti - update allmulti count on a device
8712 * @dev: device
8713 * @inc: modifier
8714 *
8715 * Add or remove reception of all multicast frames to a device. While the
8716 * count in the device remains above zero the interface remains listening
8717 * to all interfaces. Once it hits zero the device reverts back to normal
8718 * filtering operation. A negative @inc value is used to drop the counter
8719 * when releasing a resource needing all multicasts.
8720 * Return 0 if successful or a negative errno code on error.
8721 */
8722
8723int dev_set_allmulti(struct net_device *dev, int inc)
8724{
8725 return __dev_set_allmulti(dev, inc, true);
8726}
d1b19dff 8727EXPORT_SYMBOL(dev_set_allmulti);
4417da66
PM
8728
8729/*
8730 * Upload unicast and multicast address lists to device and
8731 * configure RX filtering. When the device doesn't support unicast
53ccaae1 8732 * filtering it is put in promiscuous mode while unicast addresses
4417da66
PM
8733 * are present.
8734 */
8735void __dev_set_rx_mode(struct net_device *dev)
8736{
d314774c
SH
8737 const struct net_device_ops *ops = dev->netdev_ops;
8738
4417da66
PM
8739 /* dev_open will call this function so the list will stay sane. */
8740 if (!(dev->flags&IFF_UP))
8741 return;
8742
8743 if (!netif_device_present(dev))
40b77c94 8744 return;
4417da66 8745
01789349 8746 if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
4417da66
PM
8747 /* Unicast addresses changes may only happen under the rtnl,
8748 * therefore calling __dev_set_promiscuity here is safe.
8749 */
32e7bfc4 8750 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
991fb3f7 8751 __dev_set_promiscuity(dev, 1, false);
2d348d1f 8752 dev->uc_promisc = true;
32e7bfc4 8753 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
991fb3f7 8754 __dev_set_promiscuity(dev, -1, false);
2d348d1f 8755 dev->uc_promisc = false;
4417da66 8756 }
4417da66 8757 }
01789349
JP
8758
8759 if (ops->ndo_set_rx_mode)
8760 ops->ndo_set_rx_mode(dev);
4417da66
PM
8761}
8762
8763void dev_set_rx_mode(struct net_device *dev)
8764{
b9e40857 8765 netif_addr_lock_bh(dev);
4417da66 8766 __dev_set_rx_mode(dev);
b9e40857 8767 netif_addr_unlock_bh(dev);
1da177e4
LT
8768}
8769
f0db275a
SH
8770/**
8771 * dev_get_flags - get flags reported to userspace
8772 * @dev: device
8773 *
8774 * Get the combination of flag bits exported through APIs to userspace.
8775 */
95c96174 8776unsigned int dev_get_flags(const struct net_device *dev)
1da177e4 8777{
95c96174 8778 unsigned int flags;
1da177e4
LT
8779
8780 flags = (dev->flags & ~(IFF_PROMISC |
8781 IFF_ALLMULTI |
b00055aa
SR
8782 IFF_RUNNING |
8783 IFF_LOWER_UP |
8784 IFF_DORMANT)) |
1da177e4
LT
8785 (dev->gflags & (IFF_PROMISC |
8786 IFF_ALLMULTI));
8787
b00055aa
SR
8788 if (netif_running(dev)) {
8789 if (netif_oper_up(dev))
8790 flags |= IFF_RUNNING;
8791 if (netif_carrier_ok(dev))
8792 flags |= IFF_LOWER_UP;
8793 if (netif_dormant(dev))
8794 flags |= IFF_DORMANT;
8795 }
1da177e4
LT
8796
8797 return flags;
8798}
d1b19dff 8799EXPORT_SYMBOL(dev_get_flags);
1da177e4 8800
6d040321
PM
8801int __dev_change_flags(struct net_device *dev, unsigned int flags,
8802 struct netlink_ext_ack *extack)
1da177e4 8803{
b536db93 8804 unsigned int old_flags = dev->flags;
bd380811 8805 int ret;
1da177e4 8806
24023451
PM
8807 ASSERT_RTNL();
8808
1da177e4
LT
8809 /*
8810 * Set the flags on our device.
8811 */
8812
8813 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
8814 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
8815 IFF_AUTOMEDIA)) |
8816 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
8817 IFF_ALLMULTI));
8818
8819 /*
8820 * Load in the correct multicast list now the flags have changed.
8821 */
8822
b6c40d68
PM
8823 if ((old_flags ^ flags) & IFF_MULTICAST)
8824 dev_change_rx_flags(dev, IFF_MULTICAST);
24023451 8825
4417da66 8826 dev_set_rx_mode(dev);
1da177e4
LT
8827
8828 /*
8829 * Have we downed the interface. We handle IFF_UP ourselves
8830 * according to user attempts to set it, rather than blindly
8831 * setting it.
8832 */
8833
8834 ret = 0;
7051b88a 8835 if ((old_flags ^ flags) & IFF_UP) {
8836 if (old_flags & IFF_UP)
8837 __dev_close(dev);
8838 else
40c900aa 8839 ret = __dev_open(dev, extack);
7051b88a 8840 }
1da177e4 8841
1da177e4 8842 if ((flags ^ dev->gflags) & IFF_PROMISC) {
d1b19dff 8843 int inc = (flags & IFF_PROMISC) ? 1 : -1;
991fb3f7 8844 unsigned int old_flags = dev->flags;
d1b19dff 8845
1da177e4 8846 dev->gflags ^= IFF_PROMISC;
991fb3f7
ND
8847
8848 if (__dev_set_promiscuity(dev, inc, false) >= 0)
8849 if (dev->flags != old_flags)
8850 dev_set_rx_mode(dev);
1da177e4
LT
8851 }
8852
8853 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
eb13da1a 8854 * is important. Some (broken) drivers set IFF_PROMISC, when
8855 * IFF_ALLMULTI is requested not asking us and not reporting.
1da177e4
LT
8856 */
8857 if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
d1b19dff
ED
8858 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
8859
1da177e4 8860 dev->gflags ^= IFF_ALLMULTI;
991fb3f7 8861 __dev_set_allmulti(dev, inc, false);
1da177e4
LT
8862 }
8863
bd380811
PM
8864 return ret;
8865}
8866
a528c219
ND
8867void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
8868 unsigned int gchanges)
bd380811
PM
8869{
8870 unsigned int changes = dev->flags ^ old_flags;
8871
a528c219 8872 if (gchanges)
7f294054 8873 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
a528c219 8874
bd380811
PM
8875 if (changes & IFF_UP) {
8876 if (dev->flags & IFF_UP)
8877 call_netdevice_notifiers(NETDEV_UP, dev);
8878 else
8879 call_netdevice_notifiers(NETDEV_DOWN, dev);
8880 }
8881
8882 if (dev->flags & IFF_UP &&
be9efd36 8883 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
51d0c047
DA
8884 struct netdev_notifier_change_info change_info = {
8885 .info = {
8886 .dev = dev,
8887 },
8888 .flags_changed = changes,
8889 };
be9efd36 8890
51d0c047 8891 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
be9efd36 8892 }
bd380811
PM
8893}
8894
8895/**
8896 * dev_change_flags - change device settings
8897 * @dev: device
8898 * @flags: device state flags
567c5e13 8899 * @extack: netlink extended ack
bd380811
PM
8900 *
8901 * Change settings on device based state flags. The flags are
8902 * in the userspace exported format.
8903 */
567c5e13
PM
8904int dev_change_flags(struct net_device *dev, unsigned int flags,
8905 struct netlink_ext_ack *extack)
bd380811 8906{
b536db93 8907 int ret;
991fb3f7 8908 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
bd380811 8909
6d040321 8910 ret = __dev_change_flags(dev, flags, extack);
bd380811
PM
8911 if (ret < 0)
8912 return ret;
8913
991fb3f7 8914 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
a528c219 8915 __dev_notify_flags(dev, old_flags, changes);
1da177e4
LT
8916 return ret;
8917}
d1b19dff 8918EXPORT_SYMBOL(dev_change_flags);
1da177e4 8919
f51048c3 8920int __dev_set_mtu(struct net_device *dev, int new_mtu)
2315dc91
VF
8921{
8922 const struct net_device_ops *ops = dev->netdev_ops;
8923
8924 if (ops->ndo_change_mtu)
8925 return ops->ndo_change_mtu(dev, new_mtu);
8926
501a90c9
ED
8927 /* Pairs with all the lockless reads of dev->mtu in the stack */
8928 WRITE_ONCE(dev->mtu, new_mtu);
2315dc91
VF
8929 return 0;
8930}
f51048c3 8931EXPORT_SYMBOL(__dev_set_mtu);
2315dc91 8932
d836f5c6
ED
8933int dev_validate_mtu(struct net_device *dev, int new_mtu,
8934 struct netlink_ext_ack *extack)
8935{
8936 /* MTU must be positive, and in range */
8937 if (new_mtu < 0 || new_mtu < dev->min_mtu) {
8938 NL_SET_ERR_MSG(extack, "mtu less than device minimum");
8939 return -EINVAL;
8940 }
8941
8942 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
8943 NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
8944 return -EINVAL;
8945 }
8946 return 0;
8947}
8948
f0db275a 8949/**
7a4c53be 8950 * dev_set_mtu_ext - Change maximum transfer unit
f0db275a
SH
8951 * @dev: device
8952 * @new_mtu: new transfer unit
7a4c53be 8953 * @extack: netlink extended ack
f0db275a
SH
8954 *
8955 * Change the maximum transfer size of the network device.
8956 */
7a4c53be
SH
8957int dev_set_mtu_ext(struct net_device *dev, int new_mtu,
8958 struct netlink_ext_ack *extack)
1da177e4 8959{
2315dc91 8960 int err, orig_mtu;
1da177e4
LT
8961
8962 if (new_mtu == dev->mtu)
8963 return 0;
8964
d836f5c6
ED
8965 err = dev_validate_mtu(dev, new_mtu, extack);
8966 if (err)
8967 return err;
1da177e4
LT
8968
8969 if (!netif_device_present(dev))
8970 return -ENODEV;
8971
1d486bfb
VF
8972 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
8973 err = notifier_to_errno(err);
8974 if (err)
8975 return err;
d314774c 8976
2315dc91
VF
8977 orig_mtu = dev->mtu;
8978 err = __dev_set_mtu(dev, new_mtu);
d314774c 8979
2315dc91 8980 if (!err) {
af7d6cce
SD
8981 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8982 orig_mtu);
2315dc91
VF
8983 err = notifier_to_errno(err);
8984 if (err) {
8985 /* setting mtu back and notifying everyone again,
8986 * so that they have a chance to revert changes.
8987 */
8988 __dev_set_mtu(dev, orig_mtu);
af7d6cce
SD
8989 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
8990 new_mtu);
2315dc91
VF
8991 }
8992 }
1da177e4
LT
8993 return err;
8994}
7a4c53be
SH
8995
8996int dev_set_mtu(struct net_device *dev, int new_mtu)
8997{
8998 struct netlink_ext_ack extack;
8999 int err;
9000
a6bcfc89 9001 memset(&extack, 0, sizeof(extack));
7a4c53be 9002 err = dev_set_mtu_ext(dev, new_mtu, &extack);
a6bcfc89 9003 if (err && extack._msg)
7a4c53be
SH
9004 net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
9005 return err;
9006}
d1b19dff 9007EXPORT_SYMBOL(dev_set_mtu);
1da177e4 9008
6a643ddb
CW
9009/**
9010 * dev_change_tx_queue_len - Change TX queue length of a netdevice
9011 * @dev: device
9012 * @new_len: new tx queue length
9013 */
9014int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
9015{
9016 unsigned int orig_len = dev->tx_queue_len;
9017 int res;
9018
9019 if (new_len != (unsigned int)new_len)
9020 return -ERANGE;
9021
9022 if (new_len != orig_len) {
9023 dev->tx_queue_len = new_len;
9024 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
9025 res = notifier_to_errno(res);
7effaf06
TT
9026 if (res)
9027 goto err_rollback;
9028 res = dev_qdisc_change_tx_queue_len(dev);
9029 if (res)
9030 goto err_rollback;
6a643ddb
CW
9031 }
9032
9033 return 0;
7effaf06
TT
9034
9035err_rollback:
9036 netdev_err(dev, "refused to change device tx_queue_len\n");
9037 dev->tx_queue_len = orig_len;
9038 return res;
6a643ddb
CW
9039}
9040
cbda10fa
VD
9041/**
9042 * dev_set_group - Change group this device belongs to
9043 * @dev: device
9044 * @new_group: group this device should belong to
9045 */
9046void dev_set_group(struct net_device *dev, int new_group)
9047{
9048 dev->group = new_group;
9049}
9050EXPORT_SYMBOL(dev_set_group);
9051
d59cdf94
PM
9052/**
9053 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
9054 * @dev: device
9055 * @addr: new address
9056 * @extack: netlink extended ack
9057 */
9058int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,
9059 struct netlink_ext_ack *extack)
9060{
9061 struct netdev_notifier_pre_changeaddr_info info = {
9062 .info.dev = dev,
9063 .info.extack = extack,
9064 .dev_addr = addr,
9065 };
9066 int rc;
9067
9068 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
9069 return notifier_to_errno(rc);
9070}
9071EXPORT_SYMBOL(dev_pre_changeaddr_notify);
9072
f0db275a
SH
9073/**
9074 * dev_set_mac_address - Change Media Access Control Address
9075 * @dev: device
9076 * @sa: new address
3a37a963 9077 * @extack: netlink extended ack
f0db275a
SH
9078 *
9079 * Change the hardware (MAC) address of the device
9080 */
3a37a963
PM
9081int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,
9082 struct netlink_ext_ack *extack)
1da177e4 9083{
d314774c 9084 const struct net_device_ops *ops = dev->netdev_ops;
1da177e4
LT
9085 int err;
9086
d314774c 9087 if (!ops->ndo_set_mac_address)
1da177e4
LT
9088 return -EOPNOTSUPP;
9089 if (sa->sa_family != dev->type)
9090 return -EINVAL;
9091 if (!netif_device_present(dev))
9092 return -ENODEV;
d59cdf94
PM
9093 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);
9094 if (err)
9095 return err;
d314774c 9096 err = ops->ndo_set_mac_address(dev, sa);
f6521516
JP
9097 if (err)
9098 return err;
fbdeca2d 9099 dev->addr_assign_type = NET_ADDR_SET;
f6521516 9100 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
7bf23575 9101 add_device_randomness(dev->dev_addr, dev->addr_len);
f6521516 9102 return 0;
1da177e4 9103}
d1b19dff 9104EXPORT_SYMBOL(dev_set_mac_address);
1da177e4 9105
3b23a32a
CW
9106static DECLARE_RWSEM(dev_addr_sem);
9107
9108int dev_set_mac_address_user(struct net_device *dev, struct sockaddr *sa,
9109 struct netlink_ext_ack *extack)
9110{
9111 int ret;
9112
9113 down_write(&dev_addr_sem);
9114 ret = dev_set_mac_address(dev, sa, extack);
9115 up_write(&dev_addr_sem);
9116 return ret;
9117}
9118EXPORT_SYMBOL(dev_set_mac_address_user);
9119
9120int dev_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
9121{
9122 size_t size = sizeof(sa->sa_data);
9123 struct net_device *dev;
9124 int ret = 0;
9125
9126 down_read(&dev_addr_sem);
9127 rcu_read_lock();
9128
9129 dev = dev_get_by_name_rcu(net, dev_name);
9130 if (!dev) {
9131 ret = -ENODEV;
9132 goto unlock;
9133 }
9134 if (!dev->addr_len)
9135 memset(sa->sa_data, 0, size);
9136 else
9137 memcpy(sa->sa_data, dev->dev_addr,
9138 min_t(size_t, size, dev->addr_len));
9139 sa->sa_family = dev->type;
9140
9141unlock:
9142 rcu_read_unlock();
9143 up_read(&dev_addr_sem);
9144 return ret;
9145}
9146EXPORT_SYMBOL(dev_get_mac_address);
9147
4bf84c35
JP
9148/**
9149 * dev_change_carrier - Change device carrier
9150 * @dev: device
691b3b7e 9151 * @new_carrier: new value
4bf84c35
JP
9152 *
9153 * Change device carrier
9154 */
9155int dev_change_carrier(struct net_device *dev, bool new_carrier)
9156{
9157 const struct net_device_ops *ops = dev->netdev_ops;
9158
9159 if (!ops->ndo_change_carrier)
9160 return -EOPNOTSUPP;
9161 if (!netif_device_present(dev))
9162 return -ENODEV;
9163 return ops->ndo_change_carrier(dev, new_carrier);
9164}
9165EXPORT_SYMBOL(dev_change_carrier);
9166
66b52b0d
JP
9167/**
9168 * dev_get_phys_port_id - Get device physical port ID
9169 * @dev: device
9170 * @ppid: port ID
9171 *
9172 * Get device physical port ID
9173 */
9174int dev_get_phys_port_id(struct net_device *dev,
02637fce 9175 struct netdev_phys_item_id *ppid)
66b52b0d
JP
9176{
9177 const struct net_device_ops *ops = dev->netdev_ops;
9178
9179 if (!ops->ndo_get_phys_port_id)
9180 return -EOPNOTSUPP;
9181 return ops->ndo_get_phys_port_id(dev, ppid);
9182}
9183EXPORT_SYMBOL(dev_get_phys_port_id);
9184
db24a904
DA
9185/**
9186 * dev_get_phys_port_name - Get device physical port name
9187 * @dev: device
9188 * @name: port name
ed49e650 9189 * @len: limit of bytes to copy to name
db24a904
DA
9190 *
9191 * Get device physical port name
9192 */
9193int dev_get_phys_port_name(struct net_device *dev,
9194 char *name, size_t len)
9195{
9196 const struct net_device_ops *ops = dev->netdev_ops;
af3836df 9197 int err;
db24a904 9198
af3836df
JP
9199 if (ops->ndo_get_phys_port_name) {
9200 err = ops->ndo_get_phys_port_name(dev, name, len);
9201 if (err != -EOPNOTSUPP)
9202 return err;
9203 }
9204 return devlink_compat_phys_port_name_get(dev, name, len);
db24a904
DA
9205}
9206EXPORT_SYMBOL(dev_get_phys_port_name);
9207
d6abc596
FF
9208/**
9209 * dev_get_port_parent_id - Get the device's port parent identifier
9210 * @dev: network device
9211 * @ppid: pointer to a storage for the port's parent identifier
9212 * @recurse: allow/disallow recursion to lower devices
9213 *
9214 * Get the devices's port parent identifier
9215 */
9216int dev_get_port_parent_id(struct net_device *dev,
9217 struct netdev_phys_item_id *ppid,
9218 bool recurse)
9219{
9220 const struct net_device_ops *ops = dev->netdev_ops;
9221 struct netdev_phys_item_id first = { };
9222 struct net_device *lower_dev;
9223 struct list_head *iter;
7e1146e8
JP
9224 int err;
9225
9226 if (ops->ndo_get_port_parent_id) {
9227 err = ops->ndo_get_port_parent_id(dev, ppid);
9228 if (err != -EOPNOTSUPP)
9229 return err;
9230 }
d6abc596 9231
7e1146e8
JP
9232 err = devlink_compat_switch_id_get(dev, ppid);
9233 if (!err || err != -EOPNOTSUPP)
9234 return err;
d6abc596
FF
9235
9236 if (!recurse)
7e1146e8 9237 return -EOPNOTSUPP;
d6abc596
FF
9238
9239 netdev_for_each_lower_dev(dev, lower_dev, iter) {
9240 err = dev_get_port_parent_id(lower_dev, ppid, recurse);
9241 if (err)
9242 break;
9243 if (!first.id_len)
9244 first = *ppid;
9245 else if (memcmp(&first, ppid, sizeof(*ppid)))
e1b9efe6 9246 return -EOPNOTSUPP;
d6abc596
FF
9247 }
9248
9249 return err;
9250}
9251EXPORT_SYMBOL(dev_get_port_parent_id);
9252
9253/**
9254 * netdev_port_same_parent_id - Indicate if two network devices have
9255 * the same port parent identifier
9256 * @a: first network device
9257 * @b: second network device
9258 */
9259bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
9260{
9261 struct netdev_phys_item_id a_id = { };
9262 struct netdev_phys_item_id b_id = { };
9263
9264 if (dev_get_port_parent_id(a, &a_id, true) ||
9265 dev_get_port_parent_id(b, &b_id, true))
9266 return false;
9267
9268 return netdev_phys_item_id_same(&a_id, &b_id);
9269}
9270EXPORT_SYMBOL(netdev_port_same_parent_id);
9271
d746d707
AK
9272/**
9273 * dev_change_proto_down - update protocol port state information
9274 * @dev: device
9275 * @proto_down: new value
9276 *
9277 * This info can be used by switch drivers to set the phys state of the
9278 * port.
9279 */
9280int dev_change_proto_down(struct net_device *dev, bool proto_down)
9281{
9282 const struct net_device_ops *ops = dev->netdev_ops;
9283
9284 if (!ops->ndo_change_proto_down)
9285 return -EOPNOTSUPP;
9286 if (!netif_device_present(dev))
9287 return -ENODEV;
9288 return ops->ndo_change_proto_down(dev, proto_down);
9289}
9290EXPORT_SYMBOL(dev_change_proto_down);
9291
b5899679
AR
9292/**
9293 * dev_change_proto_down_generic - generic implementation for
9294 * ndo_change_proto_down that sets carrier according to
9295 * proto_down.
9296 *
9297 * @dev: device
9298 * @proto_down: new value
9299 */
9300int dev_change_proto_down_generic(struct net_device *dev, bool proto_down)
9301{
9302 if (proto_down)
9303 netif_carrier_off(dev);
9304 else
9305 netif_carrier_on(dev);
9306 dev->proto_down = proto_down;
9307 return 0;
9308}
9309EXPORT_SYMBOL(dev_change_proto_down_generic);
9310
829eb208
RP
9311/**
9312 * dev_change_proto_down_reason - proto down reason
9313 *
9314 * @dev: device
9315 * @mask: proto down mask
9316 * @value: proto down value
9317 */
9318void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask,
9319 u32 value)
9320{
9321 int b;
9322
9323 if (!mask) {
9324 dev->proto_down_reason = value;
9325 } else {
9326 for_each_set_bit(b, &mask, 32) {
9327 if (value & (1 << b))
9328 dev->proto_down_reason |= BIT(b);
9329 else
9330 dev->proto_down_reason &= ~BIT(b);
9331 }
9332 }
9333}
9334EXPORT_SYMBOL(dev_change_proto_down_reason);
9335
aa8d3a71
AN
9336struct bpf_xdp_link {
9337 struct bpf_link link;
9338 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
9339 int flags;
9340};
9341
c8a36f19 9342static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
d67b9cd2 9343{
7f0a8382
AN
9344 if (flags & XDP_FLAGS_HW_MODE)
9345 return XDP_MODE_HW;
9346 if (flags & XDP_FLAGS_DRV_MODE)
9347 return XDP_MODE_DRV;
c8a36f19
AN
9348 if (flags & XDP_FLAGS_SKB_MODE)
9349 return XDP_MODE_SKB;
9350 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
7f0a8382 9351}
d67b9cd2 9352
7f0a8382
AN
9353static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
9354{
9355 switch (mode) {
9356 case XDP_MODE_SKB:
9357 return generic_xdp_install;
9358 case XDP_MODE_DRV:
9359 case XDP_MODE_HW:
9360 return dev->netdev_ops->ndo_bpf;
9361 default:
9362 return NULL;
5d867245 9363 }
7f0a8382 9364}
118b4aa2 9365
aa8d3a71
AN
9366static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
9367 enum bpf_xdp_mode mode)
9368{
9369 return dev->xdp_state[mode].link;
9370}
9371
7f0a8382
AN
9372static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
9373 enum bpf_xdp_mode mode)
9374{
aa8d3a71
AN
9375 struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
9376
9377 if (link)
9378 return link->link.prog;
7f0a8382
AN
9379 return dev->xdp_state[mode].prog;
9380}
9381
998f1729
THJ
9382static u8 dev_xdp_prog_count(struct net_device *dev)
9383{
9384 u8 count = 0;
9385 int i;
9386
9387 for (i = 0; i < __MAX_XDP_MODE; i++)
9388 if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
9389 count++;
9390 return count;
9391}
9392
7f0a8382
AN
9393u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
9394{
9395 struct bpf_prog *prog = dev_xdp_prog(dev, mode);
118b4aa2 9396
7f0a8382
AN
9397 return prog ? prog->aux->id : 0;
9398}
58038695 9399
aa8d3a71
AN
9400static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
9401 struct bpf_xdp_link *link)
9402{
9403 dev->xdp_state[mode].link = link;
9404 dev->xdp_state[mode].prog = NULL;
d67b9cd2
DB
9405}
9406
7f0a8382
AN
9407static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
9408 struct bpf_prog *prog)
9409{
aa8d3a71 9410 dev->xdp_state[mode].link = NULL;
7f0a8382 9411 dev->xdp_state[mode].prog = prog;
d67b9cd2
DB
9412}
9413
7f0a8382
AN
9414static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
9415 bpf_op_t bpf_op, struct netlink_ext_ack *extack,
9416 u32 flags, struct bpf_prog *prog)
d67b9cd2 9417{
f4e63525 9418 struct netdev_bpf xdp;
7e6897f9
BT
9419 int err;
9420
d67b9cd2 9421 memset(&xdp, 0, sizeof(xdp));
7f0a8382 9422 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
d67b9cd2 9423 xdp.extack = extack;
32d60277 9424 xdp.flags = flags;
d67b9cd2
DB
9425 xdp.prog = prog;
9426
7f0a8382
AN
9427 /* Drivers assume refcnt is already incremented (i.e, prog pointer is
9428 * "moved" into driver), so they don't increment it on their own, but
9429 * they do decrement refcnt when program is detached or replaced.
9430 * Given net_device also owns link/prog, we need to bump refcnt here
9431 * to prevent drivers from underflowing it.
9432 */
9433 if (prog)
9434 bpf_prog_inc(prog);
7e6897f9 9435 err = bpf_op(dev, &xdp);
7f0a8382
AN
9436 if (err) {
9437 if (prog)
9438 bpf_prog_put(prog);
9439 return err;
9440 }
7e6897f9 9441
7f0a8382
AN
9442 if (mode != XDP_MODE_HW)
9443 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
7e6897f9 9444
7f0a8382 9445 return 0;
d67b9cd2
DB
9446}
9447
bd0b2e7f
JK
9448static void dev_xdp_uninstall(struct net_device *dev)
9449{
aa8d3a71 9450 struct bpf_xdp_link *link;
7f0a8382
AN
9451 struct bpf_prog *prog;
9452 enum bpf_xdp_mode mode;
9453 bpf_op_t bpf_op;
bd0b2e7f 9454
7f0a8382 9455 ASSERT_RTNL();
bd0b2e7f 9456
7f0a8382
AN
9457 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
9458 prog = dev_xdp_prog(dev, mode);
9459 if (!prog)
9460 continue;
bd0b2e7f 9461
7f0a8382
AN
9462 bpf_op = dev_xdp_bpf_op(dev, mode);
9463 if (!bpf_op)
9464 continue;
bd0b2e7f 9465
7f0a8382
AN
9466 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9467
aa8d3a71
AN
9468 /* auto-detach link from net device */
9469 link = dev_xdp_link(dev, mode);
9470 if (link)
9471 link->dev = NULL;
9472 else
9473 bpf_prog_put(prog);
9474
9475 dev_xdp_set_link(dev, mode, NULL);
7f0a8382 9476 }
bd0b2e7f
JK
9477}
9478
d4baa936 9479static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
aa8d3a71
AN
9480 struct bpf_xdp_link *link, struct bpf_prog *new_prog,
9481 struct bpf_prog *old_prog, u32 flags)
a7862b45 9482{
998f1729 9483 unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
d4baa936
AN
9484 struct bpf_prog *cur_prog;
9485 enum bpf_xdp_mode mode;
7f0a8382 9486 bpf_op_t bpf_op;
a7862b45
BB
9487 int err;
9488
85de8576
DB
9489 ASSERT_RTNL();
9490
aa8d3a71
AN
9491 /* either link or prog attachment, never both */
9492 if (link && (new_prog || old_prog))
9493 return -EINVAL;
9494 /* link supports only XDP mode flags */
9495 if (link && (flags & ~XDP_FLAGS_MODES)) {
9496 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
9497 return -EINVAL;
9498 }
998f1729
THJ
9499 /* just one XDP mode bit should be set, zero defaults to drv/skb mode */
9500 if (num_modes > 1) {
d4baa936
AN
9501 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
9502 return -EINVAL;
9503 }
998f1729
THJ
9504 /* avoid ambiguity if offload + drv/skb mode progs are both loaded */
9505 if (!num_modes && dev_xdp_prog_count(dev) > 1) {
9506 NL_SET_ERR_MSG(extack,
9507 "More than one program loaded, unset mode is ambiguous");
9508 return -EINVAL;
9509 }
d4baa936
AN
9510 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
9511 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
9512 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
9513 return -EINVAL;
01dde20c 9514 }
a25717d2 9515
c8a36f19 9516 mode = dev_xdp_mode(dev, flags);
aa8d3a71
AN
9517 /* can't replace attached link */
9518 if (dev_xdp_link(dev, mode)) {
9519 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
9520 return -EBUSY;
01dde20c 9521 }
c14a9f63 9522
d4baa936 9523 cur_prog = dev_xdp_prog(dev, mode);
aa8d3a71
AN
9524 /* can't replace attached prog with link */
9525 if (link && cur_prog) {
9526 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
9527 return -EBUSY;
9528 }
d4baa936
AN
9529 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
9530 NL_SET_ERR_MSG(extack, "Active program does not match expected");
9531 return -EEXIST;
92234c8f 9532 }
c14a9f63 9533
aa8d3a71
AN
9534 /* put effective new program into new_prog */
9535 if (link)
9536 new_prog = link->link.prog;
85de8576 9537
d4baa936
AN
9538 if (new_prog) {
9539 bool offload = mode == XDP_MODE_HW;
7f0a8382
AN
9540 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
9541 ? XDP_MODE_DRV : XDP_MODE_SKB;
441a3303 9542
068d9d1e
AN
9543 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
9544 NL_SET_ERR_MSG(extack, "XDP program already attached");
9545 return -EBUSY;
9546 }
d4baa936 9547 if (!offload && dev_xdp_prog(dev, other_mode)) {
7f0a8382 9548 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
d67b9cd2 9549 return -EEXIST;
01dde20c 9550 }
d4baa936 9551 if (!offload && bpf_prog_is_dev_bound(new_prog->aux)) {
7f0a8382 9552 NL_SET_ERR_MSG(extack, "Using device-bound program without HW_MODE flag is not supported");
441a3303
JK
9553 return -EINVAL;
9554 }
d4baa936 9555 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) {
fbee97fe 9556 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
fbee97fe
DA
9557 return -EINVAL;
9558 }
d4baa936
AN
9559 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) {
9560 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
92164774
LB
9561 return -EINVAL;
9562 }
d4baa936 9563 }
92164774 9564
d4baa936
AN
9565 /* don't call drivers if the effective program didn't change */
9566 if (new_prog != cur_prog) {
9567 bpf_op = dev_xdp_bpf_op(dev, mode);
9568 if (!bpf_op) {
9569 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
9570 return -EOPNOTSUPP;
c14a9f63 9571 }
a7862b45 9572
d4baa936
AN
9573 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
9574 if (err)
9575 return err;
7f0a8382 9576 }
d4baa936 9577
aa8d3a71
AN
9578 if (link)
9579 dev_xdp_set_link(dev, mode, link);
9580 else
9581 dev_xdp_set_prog(dev, mode, new_prog);
d4baa936
AN
9582 if (cur_prog)
9583 bpf_prog_put(cur_prog);
a7862b45 9584
7f0a8382 9585 return 0;
a7862b45 9586}
a7862b45 9587
aa8d3a71
AN
9588static int dev_xdp_attach_link(struct net_device *dev,
9589 struct netlink_ext_ack *extack,
9590 struct bpf_xdp_link *link)
9591{
9592 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
9593}
9594
9595static int dev_xdp_detach_link(struct net_device *dev,
9596 struct netlink_ext_ack *extack,
9597 struct bpf_xdp_link *link)
9598{
9599 enum bpf_xdp_mode mode;
9600 bpf_op_t bpf_op;
9601
9602 ASSERT_RTNL();
9603
c8a36f19 9604 mode = dev_xdp_mode(dev, link->flags);
aa8d3a71
AN
9605 if (dev_xdp_link(dev, mode) != link)
9606 return -EINVAL;
9607
9608 bpf_op = dev_xdp_bpf_op(dev, mode);
9609 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
9610 dev_xdp_set_link(dev, mode, NULL);
9611 return 0;
9612}
9613
9614static void bpf_xdp_link_release(struct bpf_link *link)
9615{
9616 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9617
9618 rtnl_lock();
9619
9620 /* if racing with net_device's tear down, xdp_link->dev might be
9621 * already NULL, in which case link was already auto-detached
9622 */
73b11c2a 9623 if (xdp_link->dev) {
aa8d3a71 9624 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
73b11c2a
AN
9625 xdp_link->dev = NULL;
9626 }
aa8d3a71
AN
9627
9628 rtnl_unlock();
9629}
9630
73b11c2a
AN
9631static int bpf_xdp_link_detach(struct bpf_link *link)
9632{
9633 bpf_xdp_link_release(link);
9634 return 0;
9635}
9636
aa8d3a71
AN
9637static void bpf_xdp_link_dealloc(struct bpf_link *link)
9638{
9639 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9640
9641 kfree(xdp_link);
9642}
9643
c1931c97
AN
9644static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
9645 struct seq_file *seq)
9646{
9647 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9648 u32 ifindex = 0;
9649
9650 rtnl_lock();
9651 if (xdp_link->dev)
9652 ifindex = xdp_link->dev->ifindex;
9653 rtnl_unlock();
9654
9655 seq_printf(seq, "ifindex:\t%u\n", ifindex);
9656}
9657
9658static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
9659 struct bpf_link_info *info)
9660{
9661 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9662 u32 ifindex = 0;
9663
9664 rtnl_lock();
9665 if (xdp_link->dev)
9666 ifindex = xdp_link->dev->ifindex;
9667 rtnl_unlock();
9668
9669 info->xdp.ifindex = ifindex;
9670 return 0;
9671}
9672
026a4c28
AN
9673static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
9674 struct bpf_prog *old_prog)
9675{
9676 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
9677 enum bpf_xdp_mode mode;
9678 bpf_op_t bpf_op;
9679 int err = 0;
9680
9681 rtnl_lock();
9682
9683 /* link might have been auto-released already, so fail */
9684 if (!xdp_link->dev) {
9685 err = -ENOLINK;
9686 goto out_unlock;
9687 }
9688
9689 if (old_prog && link->prog != old_prog) {
9690 err = -EPERM;
9691 goto out_unlock;
9692 }
9693 old_prog = link->prog;
9694 if (old_prog == new_prog) {
9695 /* no-op, don't disturb drivers */
9696 bpf_prog_put(new_prog);
9697 goto out_unlock;
9698 }
9699
c8a36f19 9700 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
026a4c28
AN
9701 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
9702 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
9703 xdp_link->flags, new_prog);
9704 if (err)
9705 goto out_unlock;
9706
9707 old_prog = xchg(&link->prog, new_prog);
9708 bpf_prog_put(old_prog);
9709
9710out_unlock:
9711 rtnl_unlock();
9712 return err;
9713}
9714
aa8d3a71
AN
9715static const struct bpf_link_ops bpf_xdp_link_lops = {
9716 .release = bpf_xdp_link_release,
9717 .dealloc = bpf_xdp_link_dealloc,
73b11c2a 9718 .detach = bpf_xdp_link_detach,
c1931c97
AN
9719 .show_fdinfo = bpf_xdp_link_show_fdinfo,
9720 .fill_link_info = bpf_xdp_link_fill_link_info,
026a4c28 9721 .update_prog = bpf_xdp_link_update,
aa8d3a71
AN
9722};
9723
9724int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
9725{
9726 struct net *net = current->nsproxy->net_ns;
9727 struct bpf_link_primer link_primer;
9728 struct bpf_xdp_link *link;
9729 struct net_device *dev;
9730 int err, fd;
9731
5acc7d3e 9732 rtnl_lock();
aa8d3a71 9733 dev = dev_get_by_index(net, attr->link_create.target_ifindex);
5acc7d3e
XZ
9734 if (!dev) {
9735 rtnl_unlock();
aa8d3a71 9736 return -EINVAL;
5acc7d3e 9737 }
aa8d3a71
AN
9738
9739 link = kzalloc(sizeof(*link), GFP_USER);
9740 if (!link) {
9741 err = -ENOMEM;
5acc7d3e 9742 goto unlock;
aa8d3a71
AN
9743 }
9744
9745 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog);
9746 link->dev = dev;
9747 link->flags = attr->link_create.flags;
9748
9749 err = bpf_link_prime(&link->link, &link_primer);
9750 if (err) {
9751 kfree(link);
5acc7d3e 9752 goto unlock;
aa8d3a71
AN
9753 }
9754
aa8d3a71
AN
9755 err = dev_xdp_attach_link(dev, NULL, link);
9756 rtnl_unlock();
9757
9758 if (err) {
5acc7d3e 9759 link->dev = NULL;
aa8d3a71
AN
9760 bpf_link_cleanup(&link_primer);
9761 goto out_put_dev;
9762 }
9763
9764 fd = bpf_link_settle(&link_primer);
9765 /* link itself doesn't hold dev's refcnt to not complicate shutdown */
9766 dev_put(dev);
9767 return fd;
9768
5acc7d3e
XZ
9769unlock:
9770 rtnl_unlock();
9771
aa8d3a71
AN
9772out_put_dev:
9773 dev_put(dev);
9774 return err;
9775}
9776
d4baa936
AN
9777/**
9778 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
9779 * @dev: device
9780 * @extack: netlink extended ack
9781 * @fd: new program fd or negative value to clear
9782 * @expected_fd: old program fd that userspace expects to replace or clear
9783 * @flags: xdp-related flags
9784 *
9785 * Set or clear a bpf program for a device
9786 */
9787int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
9788 int fd, int expected_fd, u32 flags)
9789{
c8a36f19 9790 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
d4baa936
AN
9791 struct bpf_prog *new_prog = NULL, *old_prog = NULL;
9792 int err;
9793
9794 ASSERT_RTNL();
9795
9796 if (fd >= 0) {
9797 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
9798 mode != XDP_MODE_SKB);
9799 if (IS_ERR(new_prog))
9800 return PTR_ERR(new_prog);
9801 }
9802
9803 if (expected_fd >= 0) {
9804 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
9805 mode != XDP_MODE_SKB);
9806 if (IS_ERR(old_prog)) {
9807 err = PTR_ERR(old_prog);
9808 old_prog = NULL;
9809 goto err_out;
c14a9f63 9810 }
a7862b45
BB
9811 }
9812
aa8d3a71 9813 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
a7862b45 9814
d4baa936
AN
9815err_out:
9816 if (err && new_prog)
9817 bpf_prog_put(new_prog);
9818 if (old_prog)
9819 bpf_prog_put(old_prog);
a7862b45
BB
9820 return err;
9821}
a7862b45 9822
1da177e4
LT
9823/**
9824 * dev_new_index - allocate an ifindex
c4ea43c5 9825 * @net: the applicable net namespace
1da177e4
LT
9826 *
9827 * Returns a suitable unique value for a new device interface
9828 * number. The caller must hold the rtnl semaphore or the
9829 * dev_base_lock to be sure it remains unique.
9830 */
881d966b 9831static int dev_new_index(struct net *net)
1da177e4 9832{
aa79e66e 9833 int ifindex = net->ifindex;
f4563a75 9834
1da177e4
LT
9835 for (;;) {
9836 if (++ifindex <= 0)
9837 ifindex = 1;
881d966b 9838 if (!__dev_get_by_index(net, ifindex))
aa79e66e 9839 return net->ifindex = ifindex;
1da177e4
LT
9840 }
9841}
9842
1da177e4 9843/* Delayed registration/unregisteration */
3b5b34fd 9844static LIST_HEAD(net_todo_list);
200b916f 9845DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
1da177e4 9846
6f05f629 9847static void net_set_todo(struct net_device *dev)
1da177e4 9848{
1da177e4 9849 list_add_tail(&dev->todo_list, &net_todo_list);
50624c93 9850 dev_net(dev)->dev_unreg_count++;
1da177e4
LT
9851}
9852
fd867d51
JW
9853static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
9854 struct net_device *upper, netdev_features_t features)
9855{
9856 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9857 netdev_features_t feature;
5ba3f7d6 9858 int feature_bit;
fd867d51 9859
3b89ea9c 9860 for_each_netdev_feature(upper_disables, feature_bit) {
5ba3f7d6 9861 feature = __NETIF_F_BIT(feature_bit);
fd867d51
JW
9862 if (!(upper->wanted_features & feature)
9863 && (features & feature)) {
9864 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
9865 &feature, upper->name);
9866 features &= ~feature;
9867 }
9868 }
9869
9870 return features;
9871}
9872
9873static void netdev_sync_lower_features(struct net_device *upper,
9874 struct net_device *lower, netdev_features_t features)
9875{
9876 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
9877 netdev_features_t feature;
5ba3f7d6 9878 int feature_bit;
fd867d51 9879
3b89ea9c 9880 for_each_netdev_feature(upper_disables, feature_bit) {
5ba3f7d6 9881 feature = __NETIF_F_BIT(feature_bit);
fd867d51
JW
9882 if (!(features & feature) && (lower->features & feature)) {
9883 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
9884 &feature, lower->name);
9885 lower->wanted_features &= ~feature;
dd912306 9886 __netdev_update_features(lower);
fd867d51
JW
9887
9888 if (unlikely(lower->features & feature))
9889 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
9890 &feature, lower->name);
dd912306
CW
9891 else
9892 netdev_features_change(lower);
fd867d51
JW
9893 }
9894 }
9895}
9896
c8f44aff
MM
9897static netdev_features_t netdev_fix_features(struct net_device *dev,
9898 netdev_features_t features)
b63365a2 9899{
57422dc5
MM
9900 /* Fix illegal checksum combinations */
9901 if ((features & NETIF_F_HW_CSUM) &&
9902 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6f404e44 9903 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
57422dc5
MM
9904 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
9905 }
9906
b63365a2 9907 /* TSO requires that SG is present as well. */
ea2d3688 9908 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6f404e44 9909 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
ea2d3688 9910 features &= ~NETIF_F_ALL_TSO;
b63365a2
HX
9911 }
9912
ec5f0615
PS
9913 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
9914 !(features & NETIF_F_IP_CSUM)) {
9915 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
9916 features &= ~NETIF_F_TSO;
9917 features &= ~NETIF_F_TSO_ECN;
9918 }
9919
9920 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
9921 !(features & NETIF_F_IPV6_CSUM)) {
9922 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
9923 features &= ~NETIF_F_TSO6;
9924 }
9925
b1dc497b
AD
9926 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
9927 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
9928 features &= ~NETIF_F_TSO_MANGLEID;
9929
31d8b9e0
BH
9930 /* TSO ECN requires that TSO is present as well. */
9931 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
9932 features &= ~NETIF_F_TSO_ECN;
9933
212b573f
MM
9934 /* Software GSO depends on SG. */
9935 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6f404e44 9936 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
212b573f
MM
9937 features &= ~NETIF_F_GSO;
9938 }
9939
802ab55a
AD
9940 /* GSO partial features require GSO partial be set */
9941 if ((features & dev->gso_partial_features) &&
9942 !(features & NETIF_F_GSO_PARTIAL)) {
9943 netdev_dbg(dev,
9944 "Dropping partially supported GSO features since no GSO partial.\n");
9945 features &= ~dev->gso_partial_features;
9946 }
9947
fb1f5f79
MC
9948 if (!(features & NETIF_F_RXCSUM)) {
9949 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet
9950 * successfully merged by hardware must also have the
9951 * checksum verified by hardware. If the user does not
9952 * want to enable RXCSUM, logically, we should disable GRO_HW.
9953 */
9954 if (features & NETIF_F_GRO_HW) {
9955 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
9956 features &= ~NETIF_F_GRO_HW;
9957 }
9958 }
9959
de8d5ab2
GP
9960 /* LRO/HW-GRO features cannot be combined with RX-FCS */
9961 if (features & NETIF_F_RXFCS) {
9962 if (features & NETIF_F_LRO) {
9963 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
9964 features &= ~NETIF_F_LRO;
9965 }
9966
9967 if (features & NETIF_F_GRO_HW) {
9968 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
9969 features &= ~NETIF_F_GRO_HW;
9970 }
e6c6a929
GP
9971 }
9972
25537d71
TT
9973 if (features & NETIF_F_HW_TLS_TX) {
9974 bool ip_csum = (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) ==
9975 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
9976 bool hw_csum = features & NETIF_F_HW_CSUM;
9977
9978 if (!ip_csum && !hw_csum) {
9979 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
9980 features &= ~NETIF_F_HW_TLS_TX;
9981 }
ae0b04b2
TT
9982 }
9983
a3eb4e9d
TT
9984 if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
9985 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
9986 features &= ~NETIF_F_HW_TLS_RX;
9987 }
9988
b63365a2
HX
9989 return features;
9990}
b63365a2 9991
6cb6a27c 9992int __netdev_update_features(struct net_device *dev)
5455c699 9993{
fd867d51 9994 struct net_device *upper, *lower;
c8f44aff 9995 netdev_features_t features;
fd867d51 9996 struct list_head *iter;
e7868a85 9997 int err = -1;
5455c699 9998
87267485
MM
9999 ASSERT_RTNL();
10000
5455c699
MM
10001 features = netdev_get_wanted_features(dev);
10002
10003 if (dev->netdev_ops->ndo_fix_features)
10004 features = dev->netdev_ops->ndo_fix_features(dev, features);
10005
10006 /* driver might be less strict about feature dependencies */
10007 features = netdev_fix_features(dev, features);
10008
4250b75b 10009 /* some features can't be enabled if they're off on an upper device */
fd867d51
JW
10010 netdev_for_each_upper_dev_rcu(dev, upper, iter)
10011 features = netdev_sync_upper_features(dev, upper, features);
10012
5455c699 10013 if (dev->features == features)
e7868a85 10014 goto sync_lower;
5455c699 10015
c8f44aff
MM
10016 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
10017 &dev->features, &features);
5455c699
MM
10018
10019 if (dev->netdev_ops->ndo_set_features)
10020 err = dev->netdev_ops->ndo_set_features(dev, features);
5f8dc33e
NA
10021 else
10022 err = 0;
5455c699 10023
6cb6a27c 10024 if (unlikely(err < 0)) {
5455c699 10025 netdev_err(dev,
c8f44aff
MM
10026 "set_features() failed (%d); wanted %pNF, left %pNF\n",
10027 err, &features, &dev->features);
17b85d29
NA
10028 /* return non-0 since some features might have changed and
10029 * it's better to fire a spurious notification than miss it
10030 */
10031 return -1;
6cb6a27c
MM
10032 }
10033
e7868a85 10034sync_lower:
fd867d51
JW
10035 /* some features must be disabled on lower devices when disabled
10036 * on an upper device (think: bonding master or bridge)
10037 */
10038 netdev_for_each_lower_dev(dev, lower, iter)
10039 netdev_sync_lower_features(dev, lower, features);
10040
ae847f40
SD
10041 if (!err) {
10042 netdev_features_t diff = features ^ dev->features;
10043
10044 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
10045 /* udp_tunnel_{get,drop}_rx_info both need
10046 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
10047 * device, or they won't do anything.
10048 * Thus we need to update dev->features
10049 * *before* calling udp_tunnel_get_rx_info,
10050 * but *after* calling udp_tunnel_drop_rx_info.
10051 */
10052 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
10053 dev->features = features;
10054 udp_tunnel_get_rx_info(dev);
10055 } else {
10056 udp_tunnel_drop_rx_info(dev);
10057 }
10058 }
10059
9daae9bd
GP
10060 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
10061 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
10062 dev->features = features;
10063 err |= vlan_get_rx_ctag_filter_info(dev);
10064 } else {
10065 vlan_drop_rx_ctag_filter_info(dev);
10066 }
10067 }
10068
10069 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
10070 if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
10071 dev->features = features;
10072 err |= vlan_get_rx_stag_filter_info(dev);
10073 } else {
10074 vlan_drop_rx_stag_filter_info(dev);
10075 }
10076 }
10077
6cb6a27c 10078 dev->features = features;
ae847f40 10079 }
6cb6a27c 10080
e7868a85 10081 return err < 0 ? 0 : 1;
6cb6a27c
MM
10082}
10083
afe12cc8
MM
10084/**
10085 * netdev_update_features - recalculate device features
10086 * @dev: the device to check
10087 *
10088 * Recalculate dev->features set and send notifications if it
10089 * has changed. Should be called after driver or hardware dependent
10090 * conditions might have changed that influence the features.
10091 */
6cb6a27c
MM
10092void netdev_update_features(struct net_device *dev)
10093{
10094 if (__netdev_update_features(dev))
10095 netdev_features_change(dev);
5455c699
MM
10096}
10097EXPORT_SYMBOL(netdev_update_features);
10098
afe12cc8
MM
10099/**
10100 * netdev_change_features - recalculate device features
10101 * @dev: the device to check
10102 *
10103 * Recalculate dev->features set and send notifications even
10104 * if they have not changed. Should be called instead of
10105 * netdev_update_features() if also dev->vlan_features might
10106 * have changed to allow the changes to be propagated to stacked
10107 * VLAN devices.
10108 */
10109void netdev_change_features(struct net_device *dev)
10110{
10111 __netdev_update_features(dev);
10112 netdev_features_change(dev);
10113}
10114EXPORT_SYMBOL(netdev_change_features);
10115
fc4a7489
PM
10116/**
10117 * netif_stacked_transfer_operstate - transfer operstate
10118 * @rootdev: the root or lower level device to transfer state from
10119 * @dev: the device to transfer operstate to
10120 *
10121 * Transfer operational state from root to device. This is normally
10122 * called when a stacking relationship exists between the root
10123 * device and the device(a leaf device).
10124 */
10125void netif_stacked_transfer_operstate(const struct net_device *rootdev,
10126 struct net_device *dev)
10127{
10128 if (rootdev->operstate == IF_OPER_DORMANT)
10129 netif_dormant_on(dev);
10130 else
10131 netif_dormant_off(dev);
10132
eec517cd
AL
10133 if (rootdev->operstate == IF_OPER_TESTING)
10134 netif_testing_on(dev);
10135 else
10136 netif_testing_off(dev);
10137
0575c86b
ZS
10138 if (netif_carrier_ok(rootdev))
10139 netif_carrier_on(dev);
10140 else
10141 netif_carrier_off(dev);
fc4a7489
PM
10142}
10143EXPORT_SYMBOL(netif_stacked_transfer_operstate);
10144
1b4bf461
ED
10145static int netif_alloc_rx_queues(struct net_device *dev)
10146{
1b4bf461 10147 unsigned int i, count = dev->num_rx_queues;
bd25fa7b 10148 struct netdev_rx_queue *rx;
10595902 10149 size_t sz = count * sizeof(*rx);
e817f856 10150 int err = 0;
1b4bf461 10151
bd25fa7b 10152 BUG_ON(count < 1);
1b4bf461 10153
c948f51c 10154 rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
da6bc57a
MH
10155 if (!rx)
10156 return -ENOMEM;
10157
bd25fa7b
TH
10158 dev->_rx = rx;
10159
e817f856 10160 for (i = 0; i < count; i++) {
fe822240 10161 rx[i].dev = dev;
e817f856
JDB
10162
10163 /* XDP RX-queue setup */
b02e5a0e 10164 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
e817f856
JDB
10165 if (err < 0)
10166 goto err_rxq_info;
10167 }
1b4bf461 10168 return 0;
e817f856
JDB
10169
10170err_rxq_info:
10171 /* Rollback successful reg's and free other resources */
10172 while (i--)
10173 xdp_rxq_info_unreg(&rx[i].xdp_rxq);
141b52a9 10174 kvfree(dev->_rx);
e817f856
JDB
10175 dev->_rx = NULL;
10176 return err;
10177}
10178
10179static void netif_free_rx_queues(struct net_device *dev)
10180{
10181 unsigned int i, count = dev->num_rx_queues;
e817f856
JDB
10182
10183 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
10184 if (!dev->_rx)
10185 return;
10186
e817f856 10187 for (i = 0; i < count; i++)
82aaff2f
JK
10188 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
10189
10190 kvfree(dev->_rx);
1b4bf461
ED
10191}
10192
aa942104
CG
10193static void netdev_init_one_queue(struct net_device *dev,
10194 struct netdev_queue *queue, void *_unused)
10195{
10196 /* Initialize queue lock */
10197 spin_lock_init(&queue->_xmit_lock);
1a33e10e 10198 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
aa942104 10199 queue->xmit_lock_owner = -1;
b236da69 10200 netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
aa942104 10201 queue->dev = dev;
114cf580
TH
10202#ifdef CONFIG_BQL
10203 dql_init(&queue->dql, HZ);
10204#endif
aa942104
CG
10205}
10206
60877a32
ED
10207static void netif_free_tx_queues(struct net_device *dev)
10208{
4cb28970 10209 kvfree(dev->_tx);
60877a32
ED
10210}
10211
e6484930
TH
10212static int netif_alloc_netdev_queues(struct net_device *dev)
10213{
10214 unsigned int count = dev->num_tx_queues;
10215 struct netdev_queue *tx;
60877a32 10216 size_t sz = count * sizeof(*tx);
e6484930 10217
d339727c
ED
10218 if (count < 1 || count > 0xffff)
10219 return -EINVAL;
62b5942a 10220
c948f51c 10221 tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
da6bc57a
MH
10222 if (!tx)
10223 return -ENOMEM;
10224
e6484930 10225 dev->_tx = tx;
1d24eb48 10226
e6484930
TH
10227 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
10228 spin_lock_init(&dev->tx_global_lock);
aa942104
CG
10229
10230 return 0;
e6484930
TH
10231}
10232
a2029240
DV
10233void netif_tx_stop_all_queues(struct net_device *dev)
10234{
10235 unsigned int i;
10236
10237 for (i = 0; i < dev->num_tx_queues; i++) {
10238 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
f4563a75 10239
a2029240
DV
10240 netif_tx_stop_queue(txq);
10241 }
10242}
10243EXPORT_SYMBOL(netif_tx_stop_all_queues);
10244
1da177e4
LT
10245/**
10246 * register_netdevice - register a network device
10247 * @dev: device to register
10248 *
10249 * Take a completed network device structure and add it to the kernel
10250 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10251 * chain. 0 is returned on success. A negative errno code is returned
10252 * on a failure to set up the device, or if the name is a duplicate.
10253 *
10254 * Callers must hold the rtnl semaphore. You may want
10255 * register_netdev() instead of this.
10256 *
10257 * BUGS:
10258 * The locking appears insufficient to guarantee two parallel registers
10259 * will not get the same name.
10260 */
10261
10262int register_netdevice(struct net_device *dev)
10263{
1da177e4 10264 int ret;
d314774c 10265 struct net *net = dev_net(dev);
1da177e4 10266
e283de3a
FF
10267 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
10268 NETDEV_FEATURE_COUNT);
1da177e4
LT
10269 BUG_ON(dev_boot_phase);
10270 ASSERT_RTNL();
10271
b17a7c17
SH
10272 might_sleep();
10273
1da177e4
LT
10274 /* When net_device's are persistent, this will be fatal. */
10275 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
d314774c 10276 BUG_ON(!net);
1da177e4 10277
9000edb7
JK
10278 ret = ethtool_check_ops(dev->ethtool_ops);
10279 if (ret)
10280 return ret;
10281
f1f28aa3 10282 spin_lock_init(&dev->addr_list_lock);
845e0ebb 10283 netdev_set_addr_lockdep_class(dev);
1da177e4 10284
828de4f6 10285 ret = dev_get_valid_name(net, dev, dev->name);
0696c3a8
PP
10286 if (ret < 0)
10287 goto out;
10288
9077f052 10289 ret = -ENOMEM;
ff927412
JP
10290 dev->name_node = netdev_name_node_head_alloc(dev);
10291 if (!dev->name_node)
10292 goto out;
10293
1da177e4 10294 /* Init, if this function is available */
d314774c
SH
10295 if (dev->netdev_ops->ndo_init) {
10296 ret = dev->netdev_ops->ndo_init(dev);
1da177e4
LT
10297 if (ret) {
10298 if (ret > 0)
10299 ret = -EIO;
42c17fa6 10300 goto err_free_name;
1da177e4
LT
10301 }
10302 }
4ec93edb 10303
f646968f
PM
10304 if (((dev->hw_features | dev->features) &
10305 NETIF_F_HW_VLAN_CTAG_FILTER) &&
d2ed273d
MM
10306 (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
10307 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
10308 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
10309 ret = -EINVAL;
10310 goto err_uninit;
10311 }
10312
9c7dafbf
PE
10313 ret = -EBUSY;
10314 if (!dev->ifindex)
10315 dev->ifindex = dev_new_index(net);
10316 else if (__dev_get_by_index(net, dev->ifindex))
10317 goto err_uninit;
10318
5455c699
MM
10319 /* Transfer changeable features to wanted_features and enable
10320 * software offloads (GSO and GRO).
10321 */
1a3c998f 10322 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
14d1232f 10323 dev->features |= NETIF_F_SOFT_FEATURES;
d764a122 10324
876c4384 10325 if (dev->udp_tunnel_nic_info) {
d764a122
SD
10326 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10327 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
10328 }
10329
14d1232f 10330 dev->wanted_features = dev->features & dev->hw_features;
1da177e4 10331
cbc53e08 10332 if (!(dev->flags & IFF_LOOPBACK))
34324dc2 10333 dev->hw_features |= NETIF_F_NOCACHE_COPY;
cbc53e08 10334
7f348a60
AD
10335 /* If IPv4 TCP segmentation offload is supported we should also
10336 * allow the device to enable segmenting the frame with the option
10337 * of ignoring a static IP ID value. This doesn't enable the
10338 * feature itself but allows the user to enable it later.
10339 */
cbc53e08
AD
10340 if (dev->hw_features & NETIF_F_TSO)
10341 dev->hw_features |= NETIF_F_TSO_MANGLEID;
7f348a60
AD
10342 if (dev->vlan_features & NETIF_F_TSO)
10343 dev->vlan_features |= NETIF_F_TSO_MANGLEID;
10344 if (dev->mpls_features & NETIF_F_TSO)
10345 dev->mpls_features |= NETIF_F_TSO_MANGLEID;
10346 if (dev->hw_enc_features & NETIF_F_TSO)
10347 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
c6e1a0d1 10348
1180e7d6 10349 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
16c3ea78 10350 */
1180e7d6 10351 dev->vlan_features |= NETIF_F_HIGHDMA;
16c3ea78 10352
ee579677
PS
10353 /* Make NETIF_F_SG inheritable to tunnel devices.
10354 */
802ab55a 10355 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
ee579677 10356
0d89d203
SH
10357 /* Make NETIF_F_SG inheritable to MPLS.
10358 */
10359 dev->mpls_features |= NETIF_F_SG;
10360
7ffbe3fd
JB
10361 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
10362 ret = notifier_to_errno(ret);
10363 if (ret)
10364 goto err_uninit;
10365
8b41d188 10366 ret = netdev_register_kobject(dev);
cb626bf5
JH
10367 if (ret) {
10368 dev->reg_state = NETREG_UNREGISTERED;
7ce1b0ed 10369 goto err_uninit;
cb626bf5 10370 }
b17a7c17
SH
10371 dev->reg_state = NETREG_REGISTERED;
10372
6cb6a27c 10373 __netdev_update_features(dev);
8e9b59b2 10374
1da177e4
LT
10375 /*
10376 * Default initial state at registry is that the
10377 * device is present.
10378 */
10379
10380 set_bit(__LINK_STATE_PRESENT, &dev->state);
10381
8f4cccbb
BH
10382 linkwatch_init_dev(dev);
10383
1da177e4 10384 dev_init_scheduler(dev);
1da177e4 10385 dev_hold(dev);
ce286d32 10386 list_netdevice(dev);
7bf23575 10387 add_device_randomness(dev->dev_addr, dev->addr_len);
1da177e4 10388
948b337e
JP
10389 /* If the device has permanent device address, driver should
10390 * set dev_addr and also addr_assign_type should be set to
10391 * NET_ADDR_PERM (default value).
10392 */
10393 if (dev->addr_assign_type == NET_ADDR_PERM)
10394 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
10395
1da177e4 10396 /* Notify protocols, that a new device appeared. */
056925ab 10397 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
fcc5a03a 10398 ret = notifier_to_errno(ret);
93ee31f1 10399 if (ret) {
766b0515
JK
10400 /* Expect explicit free_netdev() on failure */
10401 dev->needs_free_netdev = false;
037e56bd 10402 unregister_netdevice_queue(dev, NULL);
766b0515 10403 goto out;
93ee31f1 10404 }
d90a909e
EB
10405 /*
10406 * Prevent userspace races by waiting until the network
10407 * device is fully setup before sending notifications.
10408 */
a2835763
PM
10409 if (!dev->rtnl_link_ops ||
10410 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7f294054 10411 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
1da177e4
LT
10412
10413out:
10414 return ret;
7ce1b0ed
HX
10415
10416err_uninit:
d314774c
SH
10417 if (dev->netdev_ops->ndo_uninit)
10418 dev->netdev_ops->ndo_uninit(dev);
cf124db5
DM
10419 if (dev->priv_destructor)
10420 dev->priv_destructor(dev);
42c17fa6
DC
10421err_free_name:
10422 netdev_name_node_free(dev->name_node);
7ce1b0ed 10423 goto out;
1da177e4 10424}
d1b19dff 10425EXPORT_SYMBOL(register_netdevice);
1da177e4 10426
937f1ba5
BH
10427/**
10428 * init_dummy_netdev - init a dummy network device for NAPI
10429 * @dev: device to init
10430 *
10431 * This takes a network device structure and initialize the minimum
10432 * amount of fields so it can be used to schedule NAPI polls without
10433 * registering a full blown interface. This is to be used by drivers
10434 * that need to tie several hardware interfaces to a single NAPI
10435 * poll scheduler due to HW limitations.
10436 */
10437int init_dummy_netdev(struct net_device *dev)
10438{
10439 /* Clear everything. Note we don't initialize spinlocks
10440 * are they aren't supposed to be taken by any of the
10441 * NAPI code and this dummy netdev is supposed to be
10442 * only ever used for NAPI polls
10443 */
10444 memset(dev, 0, sizeof(struct net_device));
10445
10446 /* make sure we BUG if trying to hit standard
10447 * register/unregister code path
10448 */
10449 dev->reg_state = NETREG_DUMMY;
10450
937f1ba5
BH
10451 /* NAPI wants this */
10452 INIT_LIST_HEAD(&dev->napi_list);
10453
10454 /* a dummy interface is started by default */
10455 set_bit(__LINK_STATE_PRESENT, &dev->state);
10456 set_bit(__LINK_STATE_START, &dev->state);
10457
35edfdc7
JE
10458 /* napi_busy_loop stats accounting wants this */
10459 dev_net_set(dev, &init_net);
10460
29b4433d
ED
10461 /* Note : We dont allocate pcpu_refcnt for dummy devices,
10462 * because users of this 'device' dont need to change
10463 * its refcount.
10464 */
10465
937f1ba5
BH
10466 return 0;
10467}
10468EXPORT_SYMBOL_GPL(init_dummy_netdev);
10469
10470
1da177e4
LT
10471/**
10472 * register_netdev - register a network device
10473 * @dev: device to register
10474 *
10475 * Take a completed network device structure and add it to the kernel
10476 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
10477 * chain. 0 is returned on success. A negative errno code is returned
10478 * on a failure to set up the device, or if the name is a duplicate.
10479 *
38b4da38 10480 * This is a wrapper around register_netdevice that takes the rtnl semaphore
1da177e4
LT
10481 * and expands the device name if you passed a format string to
10482 * alloc_netdev.
10483 */
10484int register_netdev(struct net_device *dev)
10485{
10486 int err;
10487
b0f3debc
KT
10488 if (rtnl_lock_killable())
10489 return -EINTR;
1da177e4 10490 err = register_netdevice(dev);
1da177e4
LT
10491 rtnl_unlock();
10492 return err;
10493}
10494EXPORT_SYMBOL(register_netdev);
10495
29b4433d
ED
10496int netdev_refcnt_read(const struct net_device *dev)
10497{
919067cc 10498#ifdef CONFIG_PCPU_DEV_REFCNT
29b4433d
ED
10499 int i, refcnt = 0;
10500
10501 for_each_possible_cpu(i)
10502 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
10503 return refcnt;
919067cc
ED
10504#else
10505 return refcount_read(&dev->dev_refcnt);
10506#endif
29b4433d
ED
10507}
10508EXPORT_SYMBOL(netdev_refcnt_read);
10509
5aa3afe1
DV
10510int netdev_unregister_timeout_secs __read_mostly = 10;
10511
de2b541b
MCC
10512#define WAIT_REFS_MIN_MSECS 1
10513#define WAIT_REFS_MAX_MSECS 250
2c53040f 10514/**
1da177e4 10515 * netdev_wait_allrefs - wait until all references are gone.
3de7a37b 10516 * @dev: target net_device
1da177e4
LT
10517 *
10518 * This is called when unregistering network devices.
10519 *
10520 * Any protocol or device that holds a reference should register
10521 * for netdevice notification, and cleanup and put back the
10522 * reference if they receive an UNREGISTER event.
10523 * We can get stuck here if buggy protocols don't correctly
4ec93edb 10524 * call dev_put.
1da177e4
LT
10525 */
10526static void netdev_wait_allrefs(struct net_device *dev)
10527{
10528 unsigned long rebroadcast_time, warning_time;
0e4be9e5 10529 int wait = 0, refcnt;
1da177e4 10530
e014debe
ED
10531 linkwatch_forget_dev(dev);
10532
1da177e4 10533 rebroadcast_time = warning_time = jiffies;
29b4433d
ED
10534 refcnt = netdev_refcnt_read(dev);
10535
add2d736 10536 while (refcnt != 1) {
1da177e4 10537 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
6756ae4b 10538 rtnl_lock();
1da177e4
LT
10539
10540 /* Rebroadcast unregister notification */
056925ab 10541 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
1da177e4 10542
748e2d93 10543 __rtnl_unlock();
0115e8e3 10544 rcu_barrier();
748e2d93
ED
10545 rtnl_lock();
10546
1da177e4
LT
10547 if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
10548 &dev->state)) {
10549 /* We must not have linkwatch events
10550 * pending on unregister. If this
10551 * happens, we simply run the queue
10552 * unscheduled, resulting in a noop
10553 * for this device.
10554 */
10555 linkwatch_run_queue();
10556 }
10557
6756ae4b 10558 __rtnl_unlock();
1da177e4
LT
10559
10560 rebroadcast_time = jiffies;
10561 }
10562
0e4be9e5
FR
10563 if (!wait) {
10564 rcu_barrier();
10565 wait = WAIT_REFS_MIN_MSECS;
10566 } else {
10567 msleep(wait);
10568 wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
10569 }
1da177e4 10570
29b4433d
ED
10571 refcnt = netdev_refcnt_read(dev);
10572
6c996e19 10573 if (refcnt != 1 &&
5aa3afe1
DV
10574 time_after(jiffies, warning_time +
10575 netdev_unregister_timeout_secs * HZ)) {
7b6cd1ce
JP
10576 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
10577 dev->name, refcnt);
1da177e4
LT
10578 warning_time = jiffies;
10579 }
10580 }
10581}
10582
10583/* The sequence is:
10584 *
10585 * rtnl_lock();
10586 * ...
10587 * register_netdevice(x1);
10588 * register_netdevice(x2);
10589 * ...
10590 * unregister_netdevice(y1);
10591 * unregister_netdevice(y2);
10592 * ...
10593 * rtnl_unlock();
10594 * free_netdev(y1);
10595 * free_netdev(y2);
10596 *
58ec3b4d 10597 * We are invoked by rtnl_unlock().
1da177e4 10598 * This allows us to deal with problems:
b17a7c17 10599 * 1) We can delete sysfs objects which invoke hotplug
1da177e4
LT
10600 * without deadlocking with linkwatch via keventd.
10601 * 2) Since we run with the RTNL semaphore not held, we can sleep
10602 * safely in order to wait for the netdev refcnt to drop to zero.
58ec3b4d
HX
10603 *
10604 * We must not return until all unregister events added during
10605 * the interval the lock was held have been completed.
1da177e4 10606 */
1da177e4
LT
10607void netdev_run_todo(void)
10608{
626ab0e6 10609 struct list_head list;
1fc70edb
TY
10610#ifdef CONFIG_LOCKDEP
10611 struct list_head unlink_list;
10612
10613 list_replace_init(&net_unlink_list, &unlink_list);
10614
10615 while (!list_empty(&unlink_list)) {
10616 struct net_device *dev = list_first_entry(&unlink_list,
10617 struct net_device,
10618 unlink_list);
0e8b8d6a 10619 list_del_init(&dev->unlink_list);
1fc70edb
TY
10620 dev->nested_level = dev->lower_level - 1;
10621 }
10622#endif
1da177e4 10623
1da177e4 10624 /* Snapshot list, allow later requests */
626ab0e6 10625 list_replace_init(&net_todo_list, &list);
58ec3b4d
HX
10626
10627 __rtnl_unlock();
626ab0e6 10628
0115e8e3
ED
10629
10630 /* Wait for rcu callbacks to finish before next phase */
850a545b
EB
10631 if (!list_empty(&list))
10632 rcu_barrier();
10633
1da177e4
LT
10634 while (!list_empty(&list)) {
10635 struct net_device *dev
e5e26d75 10636 = list_first_entry(&list, struct net_device, todo_list);
1da177e4
LT
10637 list_del(&dev->todo_list);
10638
b17a7c17 10639 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7b6cd1ce 10640 pr_err("network todo '%s' but state %d\n",
b17a7c17
SH
10641 dev->name, dev->reg_state);
10642 dump_stack();
10643 continue;
10644 }
1da177e4 10645
b17a7c17 10646 dev->reg_state = NETREG_UNREGISTERED;
1da177e4 10647
b17a7c17 10648 netdev_wait_allrefs(dev);
1da177e4 10649
b17a7c17 10650 /* paranoia */
add2d736 10651 BUG_ON(netdev_refcnt_read(dev) != 1);
7866a621
SN
10652 BUG_ON(!list_empty(&dev->ptype_all));
10653 BUG_ON(!list_empty(&dev->ptype_specific));
33d480ce
ED
10654 WARN_ON(rcu_access_pointer(dev->ip_ptr));
10655 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
330c7272 10656#if IS_ENABLED(CONFIG_DECNET)
547b792c 10657 WARN_ON(dev->dn_ptr);
330c7272 10658#endif
cf124db5
DM
10659 if (dev->priv_destructor)
10660 dev->priv_destructor(dev);
10661 if (dev->needs_free_netdev)
10662 free_netdev(dev);
9093bbb2 10663
50624c93
EB
10664 /* Report a network device has been unregistered */
10665 rtnl_lock();
10666 dev_net(dev)->dev_unreg_count--;
10667 __rtnl_unlock();
10668 wake_up(&netdev_unregistering_wq);
10669
9093bbb2
SH
10670 /* Free network device */
10671 kobject_put(&dev->dev.kobj);
1da177e4 10672 }
1da177e4
LT
10673}
10674
9256645a
JW
10675/* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
10676 * all the same fields in the same order as net_device_stats, with only
10677 * the type differing, but rtnl_link_stats64 may have additional fields
10678 * at the end for newer counters.
3cfde79c 10679 */
77a1abf5
ED
10680void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
10681 const struct net_device_stats *netdev_stats)
3cfde79c
BH
10682{
10683#if BITS_PER_LONG == 64
9256645a 10684 BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
9af9959e 10685 memcpy(stats64, netdev_stats, sizeof(*netdev_stats));
9256645a
JW
10686 /* zero out counters that only exist in rtnl_link_stats64 */
10687 memset((char *)stats64 + sizeof(*netdev_stats), 0,
10688 sizeof(*stats64) - sizeof(*netdev_stats));
3cfde79c 10689#else
9256645a 10690 size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
3cfde79c
BH
10691 const unsigned long *src = (const unsigned long *)netdev_stats;
10692 u64 *dst = (u64 *)stats64;
10693
9256645a 10694 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
3cfde79c
BH
10695 for (i = 0; i < n; i++)
10696 dst[i] = src[i];
9256645a
JW
10697 /* zero out counters that only exist in rtnl_link_stats64 */
10698 memset((char *)stats64 + n * sizeof(u64), 0,
10699 sizeof(*stats64) - n * sizeof(u64));
3cfde79c
BH
10700#endif
10701}
77a1abf5 10702EXPORT_SYMBOL(netdev_stats_to_stats64);
3cfde79c 10703
eeda3fd6
SH
10704/**
10705 * dev_get_stats - get network device statistics
10706 * @dev: device to get statistics from
28172739 10707 * @storage: place to store stats
eeda3fd6 10708 *
d7753516
BH
10709 * Get network statistics from device. Return @storage.
10710 * The device driver may provide its own method by setting
10711 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
10712 * otherwise the internal statistics structure is used.
eeda3fd6 10713 */
d7753516
BH
10714struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
10715 struct rtnl_link_stats64 *storage)
7004bf25 10716{
eeda3fd6
SH
10717 const struct net_device_ops *ops = dev->netdev_ops;
10718
28172739
ED
10719 if (ops->ndo_get_stats64) {
10720 memset(storage, 0, sizeof(*storage));
caf586e5
ED
10721 ops->ndo_get_stats64(dev, storage);
10722 } else if (ops->ndo_get_stats) {
3cfde79c 10723 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
caf586e5
ED
10724 } else {
10725 netdev_stats_to_stats64(storage, &dev->stats);
28172739 10726 }
6f64ec74
ED
10727 storage->rx_dropped += (unsigned long)atomic_long_read(&dev->rx_dropped);
10728 storage->tx_dropped += (unsigned long)atomic_long_read(&dev->tx_dropped);
10729 storage->rx_nohandler += (unsigned long)atomic_long_read(&dev->rx_nohandler);
28172739 10730 return storage;
c45d286e 10731}
eeda3fd6 10732EXPORT_SYMBOL(dev_get_stats);
c45d286e 10733
44fa32f0
HK
10734/**
10735 * dev_fetch_sw_netstats - get per-cpu network device statistics
10736 * @s: place to store stats
10737 * @netstats: per-cpu network stats to read from
10738 *
10739 * Read per-cpu network statistics and populate the related fields in @s.
10740 */
10741void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
10742 const struct pcpu_sw_netstats __percpu *netstats)
10743{
10744 int cpu;
10745
10746 for_each_possible_cpu(cpu) {
10747 const struct pcpu_sw_netstats *stats;
10748 struct pcpu_sw_netstats tmp;
10749 unsigned int start;
10750
10751 stats = per_cpu_ptr(netstats, cpu);
10752 do {
10753 start = u64_stats_fetch_begin_irq(&stats->syncp);
10754 tmp.rx_packets = stats->rx_packets;
10755 tmp.rx_bytes = stats->rx_bytes;
10756 tmp.tx_packets = stats->tx_packets;
10757 tmp.tx_bytes = stats->tx_bytes;
10758 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
10759
10760 s->rx_packets += tmp.rx_packets;
10761 s->rx_bytes += tmp.rx_bytes;
10762 s->tx_packets += tmp.tx_packets;
10763 s->tx_bytes += tmp.tx_bytes;
10764 }
10765}
10766EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
10767
a1839426
HK
10768/**
10769 * dev_get_tstats64 - ndo_get_stats64 implementation
10770 * @dev: device to get statistics from
10771 * @s: place to store stats
10772 *
10773 * Populate @s from dev->stats and dev->tstats. Can be used as
10774 * ndo_get_stats64() callback.
10775 */
10776void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
10777{
10778 netdev_stats_to_stats64(s, &dev->stats);
10779 dev_fetch_sw_netstats(s, dev->tstats);
10780}
10781EXPORT_SYMBOL_GPL(dev_get_tstats64);
10782
24824a09 10783struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
dc2b4847 10784{
24824a09 10785 struct netdev_queue *queue = dev_ingress_queue(dev);
dc2b4847 10786
24824a09
ED
10787#ifdef CONFIG_NET_CLS_ACT
10788 if (queue)
10789 return queue;
10790 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
10791 if (!queue)
10792 return NULL;
10793 netdev_init_one_queue(dev, queue, NULL);
2ce1ee17 10794 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
24824a09
ED
10795 queue->qdisc_sleeping = &noop_qdisc;
10796 rcu_assign_pointer(dev->ingress_queue, queue);
10797#endif
10798 return queue;
bb949fbd
DM
10799}
10800
2c60db03
ED
10801static const struct ethtool_ops default_ethtool_ops;
10802
d07d7507
SG
10803void netdev_set_default_ethtool_ops(struct net_device *dev,
10804 const struct ethtool_ops *ops)
10805{
10806 if (dev->ethtool_ops == &default_ethtool_ops)
10807 dev->ethtool_ops = ops;
10808}
10809EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
10810
74d332c1
ED
10811void netdev_freemem(struct net_device *dev)
10812{
10813 char *addr = (char *)dev - dev->padded;
10814
4cb28970 10815 kvfree(addr);
74d332c1
ED
10816}
10817
1da177e4 10818/**
722c9a0c 10819 * alloc_netdev_mqs - allocate network device
10820 * @sizeof_priv: size of private data to allocate space for
10821 * @name: device name format string
10822 * @name_assign_type: origin of device name
10823 * @setup: callback to initialize device
10824 * @txqs: the number of TX subqueues to allocate
10825 * @rxqs: the number of RX subqueues to allocate
10826 *
10827 * Allocates a struct net_device with private data area for driver use
10828 * and performs basic initialization. Also allocates subqueue structs
10829 * for each queue on the device.
1da177e4 10830 */
36909ea4 10831struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
c835a677 10832 unsigned char name_assign_type,
36909ea4
TH
10833 void (*setup)(struct net_device *),
10834 unsigned int txqs, unsigned int rxqs)
1da177e4 10835{
1da177e4 10836 struct net_device *dev;
52a59bd5 10837 unsigned int alloc_size;
1ce8e7b5 10838 struct net_device *p;
1da177e4 10839
b6fe17d6
SH
10840 BUG_ON(strlen(name) >= sizeof(dev->name));
10841
36909ea4 10842 if (txqs < 1) {
7b6cd1ce 10843 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
55513fb4
TH
10844 return NULL;
10845 }
10846
36909ea4 10847 if (rxqs < 1) {
7b6cd1ce 10848 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
36909ea4
TH
10849 return NULL;
10850 }
36909ea4 10851
fd2ea0a7 10852 alloc_size = sizeof(struct net_device);
d1643d24
AD
10853 if (sizeof_priv) {
10854 /* ensure 32-byte alignment of private area */
1ce8e7b5 10855 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
d1643d24
AD
10856 alloc_size += sizeof_priv;
10857 }
10858 /* ensure 32-byte alignment of whole construct */
1ce8e7b5 10859 alloc_size += NETDEV_ALIGN - 1;
1da177e4 10860
c948f51c 10861 p = kvzalloc(alloc_size, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
62b5942a 10862 if (!p)
1da177e4 10863 return NULL;
1da177e4 10864
1ce8e7b5 10865 dev = PTR_ALIGN(p, NETDEV_ALIGN);
1da177e4 10866 dev->padded = (char *)dev - (char *)p;
ab9c73cc 10867
919067cc 10868#ifdef CONFIG_PCPU_DEV_REFCNT
29b4433d
ED
10869 dev->pcpu_refcnt = alloc_percpu(int);
10870 if (!dev->pcpu_refcnt)
74d332c1 10871 goto free_dev;
add2d736
ED
10872 dev_hold(dev);
10873#else
10874 refcount_set(&dev->dev_refcnt, 1);
919067cc 10875#endif
ab9c73cc 10876
ab9c73cc 10877 if (dev_addr_init(dev))
29b4433d 10878 goto free_pcpu;
ab9c73cc 10879
22bedad3 10880 dev_mc_init(dev);
a748ee24 10881 dev_uc_init(dev);
ccffad25 10882
c346dca1 10883 dev_net_set(dev, &init_net);
1da177e4 10884
8d3bdbd5 10885 dev->gso_max_size = GSO_MAX_SIZE;
30b678d8 10886 dev->gso_max_segs = GSO_MAX_SEGS;
5343da4c
TY
10887 dev->upper_level = 1;
10888 dev->lower_level = 1;
1fc70edb
TY
10889#ifdef CONFIG_LOCKDEP
10890 dev->nested_level = 0;
10891 INIT_LIST_HEAD(&dev->unlink_list);
10892#endif
8d3bdbd5 10893
8d3bdbd5
DM
10894 INIT_LIST_HEAD(&dev->napi_list);
10895 INIT_LIST_HEAD(&dev->unreg_list);
5cde2829 10896 INIT_LIST_HEAD(&dev->close_list);
8d3bdbd5 10897 INIT_LIST_HEAD(&dev->link_watch_list);
2f268f12
VF
10898 INIT_LIST_HEAD(&dev->adj_list.upper);
10899 INIT_LIST_HEAD(&dev->adj_list.lower);
7866a621
SN
10900 INIT_LIST_HEAD(&dev->ptype_all);
10901 INIT_LIST_HEAD(&dev->ptype_specific);
93642e14 10902 INIT_LIST_HEAD(&dev->net_notifier_list);
59cc1f61
JK
10903#ifdef CONFIG_NET_SCHED
10904 hash_init(dev->qdisc_hash);
10905#endif
02875878 10906 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
8d3bdbd5
DM
10907 setup(dev);
10908
a813104d 10909 if (!dev->tx_queue_len) {
f84bb1ea 10910 dev->priv_flags |= IFF_NO_QUEUE;
11597084 10911 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
a813104d 10912 }
906470c1 10913
36909ea4
TH
10914 dev->num_tx_queues = txqs;
10915 dev->real_num_tx_queues = txqs;
ed9af2e8 10916 if (netif_alloc_netdev_queues(dev))
8d3bdbd5 10917 goto free_all;
e8a0464c 10918
36909ea4
TH
10919 dev->num_rx_queues = rxqs;
10920 dev->real_num_rx_queues = rxqs;
fe822240 10921 if (netif_alloc_rx_queues(dev))
8d3bdbd5 10922 goto free_all;
0a9627f2 10923
1da177e4 10924 strcpy(dev->name, name);
c835a677 10925 dev->name_assign_type = name_assign_type;
cbda10fa 10926 dev->group = INIT_NETDEV_GROUP;
2c60db03
ED
10927 if (!dev->ethtool_ops)
10928 dev->ethtool_ops = &default_ethtool_ops;
e687ad60 10929
357b6cc5 10930 nf_hook_ingress_init(dev);
e687ad60 10931
1da177e4 10932 return dev;
ab9c73cc 10933
8d3bdbd5
DM
10934free_all:
10935 free_netdev(dev);
10936 return NULL;
10937
29b4433d 10938free_pcpu:
919067cc 10939#ifdef CONFIG_PCPU_DEV_REFCNT
29b4433d 10940 free_percpu(dev->pcpu_refcnt);
74d332c1 10941free_dev:
919067cc 10942#endif
74d332c1 10943 netdev_freemem(dev);
ab9c73cc 10944 return NULL;
1da177e4 10945}
36909ea4 10946EXPORT_SYMBOL(alloc_netdev_mqs);
1da177e4
LT
10947
10948/**
722c9a0c 10949 * free_netdev - free network device
10950 * @dev: device
1da177e4 10951 *
722c9a0c 10952 * This function does the last stage of destroying an allocated device
10953 * interface. The reference to the device object is released. If this
10954 * is the last reference then it will be freed.Must be called in process
10955 * context.
1da177e4
LT
10956 */
10957void free_netdev(struct net_device *dev)
10958{
d565b0a1
HX
10959 struct napi_struct *p, *n;
10960
93d05d4a 10961 might_sleep();
c269a24c
JK
10962
10963 /* When called immediately after register_netdevice() failed the unwind
10964 * handling may still be dismantling the device. Handle that case by
10965 * deferring the free.
10966 */
10967 if (dev->reg_state == NETREG_UNREGISTERING) {
10968 ASSERT_RTNL();
10969 dev->needs_free_netdev = true;
10970 return;
10971 }
10972
60877a32 10973 netif_free_tx_queues(dev);
e817f856 10974 netif_free_rx_queues(dev);
e8a0464c 10975
33d480ce 10976 kfree(rcu_dereference_protected(dev->ingress_queue, 1));
24824a09 10977
f001fde5
JP
10978 /* Flush device addresses */
10979 dev_addr_flush(dev);
10980
d565b0a1
HX
10981 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
10982 netif_napi_del(p);
10983
919067cc 10984#ifdef CONFIG_PCPU_DEV_REFCNT
29b4433d
ED
10985 free_percpu(dev->pcpu_refcnt);
10986 dev->pcpu_refcnt = NULL;
919067cc 10987#endif
75ccae62
THJ
10988 free_percpu(dev->xdp_bulkq);
10989 dev->xdp_bulkq = NULL;
29b4433d 10990
3041a069 10991 /* Compatibility with error handling in drivers */
1da177e4 10992 if (dev->reg_state == NETREG_UNINITIALIZED) {
74d332c1 10993 netdev_freemem(dev);
1da177e4
LT
10994 return;
10995 }
10996
10997 BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
10998 dev->reg_state = NETREG_RELEASED;
10999
43cb76d9
GKH
11000 /* will free via device release */
11001 put_device(&dev->dev);
1da177e4 11002}
d1b19dff 11003EXPORT_SYMBOL(free_netdev);
4ec93edb 11004
f0db275a
SH
11005/**
11006 * synchronize_net - Synchronize with packet receive processing
11007 *
11008 * Wait for packets currently being received to be done.
11009 * Does not block later packets from starting.
11010 */
4ec93edb 11011void synchronize_net(void)
1da177e4
LT
11012{
11013 might_sleep();
be3fc413
ED
11014 if (rtnl_is_locked())
11015 synchronize_rcu_expedited();
11016 else
11017 synchronize_rcu();
1da177e4 11018}
d1b19dff 11019EXPORT_SYMBOL(synchronize_net);
1da177e4
LT
11020
11021/**
44a0873d 11022 * unregister_netdevice_queue - remove device from the kernel
1da177e4 11023 * @dev: device
44a0873d 11024 * @head: list
6ebfbc06 11025 *
1da177e4 11026 * This function shuts down a device interface and removes it
d59b54b1 11027 * from the kernel tables.
44a0873d 11028 * If head not NULL, device is queued to be unregistered later.
1da177e4
LT
11029 *
11030 * Callers must hold the rtnl semaphore. You may want
11031 * unregister_netdev() instead of this.
11032 */
11033
44a0873d 11034void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
1da177e4 11035{
a6620712
HX
11036 ASSERT_RTNL();
11037
44a0873d 11038 if (head) {
9fdce099 11039 list_move_tail(&dev->unreg_list, head);
44a0873d 11040 } else {
037e56bd
JK
11041 LIST_HEAD(single);
11042
11043 list_add(&dev->unreg_list, &single);
0cbe1e57 11044 unregister_netdevice_many(&single);
44a0873d 11045 }
1da177e4 11046}
44a0873d 11047EXPORT_SYMBOL(unregister_netdevice_queue);
1da177e4 11048
9b5e383c
ED
11049/**
11050 * unregister_netdevice_many - unregister many devices
11051 * @head: list of devices
87757a91
ED
11052 *
11053 * Note: As most callers use a stack allocated list_head,
11054 * we force a list_del() to make sure stack wont be corrupted later.
9b5e383c
ED
11055 */
11056void unregister_netdevice_many(struct list_head *head)
bcfe2f1a
JK
11057{
11058 struct net_device *dev, *tmp;
11059 LIST_HEAD(close_head);
11060
11061 BUG_ON(dev_boot_phase);
11062 ASSERT_RTNL();
11063
0cbe1e57
JK
11064 if (list_empty(head))
11065 return;
11066
bcfe2f1a
JK
11067 list_for_each_entry_safe(dev, tmp, head, unreg_list) {
11068 /* Some devices call without registering
11069 * for initialization unwind. Remove those
11070 * devices and proceed with the remaining.
11071 */
11072 if (dev->reg_state == NETREG_UNINITIALIZED) {
11073 pr_debug("unregister_netdevice: device %s/%p never was registered\n",
11074 dev->name, dev);
11075
11076 WARN_ON(1);
11077 list_del(&dev->unreg_list);
11078 continue;
11079 }
11080 dev->dismantle = true;
11081 BUG_ON(dev->reg_state != NETREG_REGISTERED);
11082 }
11083
11084 /* If device is running, close it first. */
11085 list_for_each_entry(dev, head, unreg_list)
11086 list_add_tail(&dev->close_list, &close_head);
11087 dev_close_many(&close_head, true);
11088
11089 list_for_each_entry(dev, head, unreg_list) {
11090 /* And unlink it from device chain. */
11091 unlist_netdevice(dev);
11092
11093 dev->reg_state = NETREG_UNREGISTERING;
11094 }
11095 flush_all_backlogs();
11096
11097 synchronize_net();
11098
11099 list_for_each_entry(dev, head, unreg_list) {
11100 struct sk_buff *skb = NULL;
11101
11102 /* Shutdown queueing discipline. */
11103 dev_shutdown(dev);
11104
11105 dev_xdp_uninstall(dev);
11106
11107 /* Notify protocols, that we are about to destroy
11108 * this device. They should clean all the things.
11109 */
11110 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11111
11112 if (!dev->rtnl_link_ops ||
11113 dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
11114 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
11115 GFP_KERNEL, NULL, 0);
11116
11117 /*
11118 * Flush the unicast and multicast chains
11119 */
11120 dev_uc_flush(dev);
11121 dev_mc_flush(dev);
11122
11123 netdev_name_node_alt_flush(dev);
11124 netdev_name_node_free(dev->name_node);
11125
11126 if (dev->netdev_ops->ndo_uninit)
11127 dev->netdev_ops->ndo_uninit(dev);
11128
11129 if (skb)
11130 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
11131
11132 /* Notifier chain MUST detach us all upper devices. */
11133 WARN_ON(netdev_has_any_upper_dev(dev));
11134 WARN_ON(netdev_has_any_lower_dev(dev));
11135
11136 /* Remove entries from kobject tree */
11137 netdev_unregister_kobject(dev);
11138#ifdef CONFIG_XPS
11139 /* Remove XPS queueing entries */
11140 netif_reset_xps_queues_gt(dev, 0);
11141#endif
11142 }
11143
11144 synchronize_net();
11145
11146 list_for_each_entry(dev, head, unreg_list) {
11147 dev_put(dev);
11148 net_set_todo(dev);
11149 }
0cbe1e57
JK
11150
11151 list_del(head);
bcfe2f1a 11152}
0cbe1e57 11153EXPORT_SYMBOL(unregister_netdevice_many);
bcfe2f1a 11154
1da177e4
LT
11155/**
11156 * unregister_netdev - remove device from the kernel
11157 * @dev: device
11158 *
11159 * This function shuts down a device interface and removes it
d59b54b1 11160 * from the kernel tables.
1da177e4
LT
11161 *
11162 * This is just a wrapper for unregister_netdevice that takes
11163 * the rtnl semaphore. In general you want to use this and not
11164 * unregister_netdevice.
11165 */
11166void unregister_netdev(struct net_device *dev)
11167{
11168 rtnl_lock();
11169 unregister_netdevice(dev);
11170 rtnl_unlock();
11171}
1da177e4
LT
11172EXPORT_SYMBOL(unregister_netdev);
11173
ce286d32 11174/**
0854fa82 11175 * __dev_change_net_namespace - move device to different nethost namespace
ce286d32
EB
11176 * @dev: device
11177 * @net: network namespace
11178 * @pat: If not NULL name pattern to try if the current device name
11179 * is already taken in the destination network namespace.
eeb85a14
AV
11180 * @new_ifindex: If not zero, specifies device index in the target
11181 * namespace.
ce286d32
EB
11182 *
11183 * This function shuts down a device interface and moves it
11184 * to a new network namespace. On success 0 is returned, on
11185 * a failure a netagive errno code is returned.
11186 *
11187 * Callers must hold the rtnl semaphore.
11188 */
11189
0854fa82
AV
11190int __dev_change_net_namespace(struct net_device *dev, struct net *net,
11191 const char *pat, int new_ifindex)
ce286d32 11192{
ef6a4c88 11193 struct net *net_old = dev_net(dev);
eeb85a14 11194 int err, new_nsid;
ce286d32
EB
11195
11196 ASSERT_RTNL();
11197
11198 /* Don't allow namespace local devices to be moved. */
11199 err = -EINVAL;
11200 if (dev->features & NETIF_F_NETNS_LOCAL)
11201 goto out;
11202
11203 /* Ensure the device has been registrered */
ce286d32
EB
11204 if (dev->reg_state != NETREG_REGISTERED)
11205 goto out;
11206
11207 /* Get out if there is nothing todo */
11208 err = 0;
ef6a4c88 11209 if (net_eq(net_old, net))
ce286d32
EB
11210 goto out;
11211
11212 /* Pick the destination device name, and ensure
11213 * we can use it in the destination network namespace.
11214 */
11215 err = -EEXIST;
d9031024 11216 if (__dev_get_by_name(net, dev->name)) {
ce286d32
EB
11217 /* We get here if we can't use the current device name */
11218 if (!pat)
11219 goto out;
7892bd08
LR
11220 err = dev_get_valid_name(net, dev, pat);
11221 if (err < 0)
ce286d32
EB
11222 goto out;
11223 }
11224
eeb85a14
AV
11225 /* Check that new_ifindex isn't used yet. */
11226 err = -EBUSY;
11227 if (new_ifindex && __dev_get_by_index(net, new_ifindex))
11228 goto out;
11229
ce286d32
EB
11230 /*
11231 * And now a mini version of register_netdevice unregister_netdevice.
11232 */
11233
11234 /* If device is running close it first. */
9b772652 11235 dev_close(dev);
ce286d32
EB
11236
11237 /* And unlink it from device chain */
ce286d32
EB
11238 unlist_netdevice(dev);
11239
11240 synchronize_net();
11241
11242 /* Shutdown queueing discipline. */
11243 dev_shutdown(dev);
11244
11245 /* Notify protocols, that we are about to destroy
eb13da1a 11246 * this device. They should clean all the things.
11247 *
11248 * Note that dev->reg_state stays at NETREG_REGISTERED.
11249 * This is wanted because this way 8021q and macvlan know
11250 * the device is just moving and can keep their slaves up.
11251 */
ce286d32 11252 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6549dd43 11253 rcu_barrier();
38e01b30 11254
d4e4fdf9 11255 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
38e01b30 11256 /* If there is an ifindex conflict assign a new one */
eeb85a14
AV
11257 if (!new_ifindex) {
11258 if (__dev_get_by_index(net, dev->ifindex))
11259 new_ifindex = dev_new_index(net);
11260 else
11261 new_ifindex = dev->ifindex;
11262 }
38e01b30
ND
11263
11264 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
11265 new_ifindex);
ce286d32
EB
11266
11267 /*
11268 * Flush the unicast and multicast chains
11269 */
a748ee24 11270 dev_uc_flush(dev);
22bedad3 11271 dev_mc_flush(dev);
ce286d32 11272
4e66ae2e
SH
11273 /* Send a netdev-removed uevent to the old namespace */
11274 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
4c75431a 11275 netdev_adjacent_del_links(dev);
4e66ae2e 11276
93642e14
JP
11277 /* Move per-net netdevice notifiers that are following the netdevice */
11278 move_netdevice_notifiers_dev_net(dev, net);
11279
ce286d32 11280 /* Actually switch the network namespace */
c346dca1 11281 dev_net_set(dev, net);
38e01b30 11282 dev->ifindex = new_ifindex;
ce286d32 11283
4e66ae2e
SH
11284 /* Send a netdev-add uevent to the new namespace */
11285 kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
4c75431a 11286 netdev_adjacent_add_links(dev);
4e66ae2e 11287
8b41d188 11288 /* Fixup kobjects */
a1b3f594 11289 err = device_rename(&dev->dev, dev->name);
8b41d188 11290 WARN_ON(err);
ce286d32 11291
ef6a4c88
CB
11292 /* Adapt owner in case owning user namespace of target network
11293 * namespace is different from the original one.
11294 */
11295 err = netdev_change_owner(dev, net_old, net);
11296 WARN_ON(err);
11297
ce286d32
EB
11298 /* Add the device back in the hashes */
11299 list_netdevice(dev);
11300
11301 /* Notify protocols, that a new device appeared. */
11302 call_netdevice_notifiers(NETDEV_REGISTER, dev);
11303
d90a909e
EB
11304 /*
11305 * Prevent userspace races by waiting until the network
11306 * device is fully setup before sending notifications.
11307 */
7f294054 11308 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
d90a909e 11309
ce286d32
EB
11310 synchronize_net();
11311 err = 0;
11312out:
11313 return err;
11314}
0854fa82 11315EXPORT_SYMBOL_GPL(__dev_change_net_namespace);
ce286d32 11316
f0bf90de 11317static int dev_cpu_dead(unsigned int oldcpu)
1da177e4
LT
11318{
11319 struct sk_buff **list_skb;
1da177e4 11320 struct sk_buff *skb;
f0bf90de 11321 unsigned int cpu;
97d8b6e3 11322 struct softnet_data *sd, *oldsd, *remsd = NULL;
1da177e4 11323
1da177e4
LT
11324 local_irq_disable();
11325 cpu = smp_processor_id();
11326 sd = &per_cpu(softnet_data, cpu);
11327 oldsd = &per_cpu(softnet_data, oldcpu);
11328
11329 /* Find end of our completion_queue. */
11330 list_skb = &sd->completion_queue;
11331 while (*list_skb)
11332 list_skb = &(*list_skb)->next;
11333 /* Append completion queue from offline CPU. */
11334 *list_skb = oldsd->completion_queue;
11335 oldsd->completion_queue = NULL;
11336
1da177e4 11337 /* Append output queue from offline CPU. */
a9cbd588
CG
11338 if (oldsd->output_queue) {
11339 *sd->output_queue_tailp = oldsd->output_queue;
11340 sd->output_queue_tailp = oldsd->output_queue_tailp;
11341 oldsd->output_queue = NULL;
11342 oldsd->output_queue_tailp = &oldsd->output_queue;
11343 }
ac64da0b
ED
11344 /* Append NAPI poll list from offline CPU, with one exception :
11345 * process_backlog() must be called by cpu owning percpu backlog.
11346 * We properly handle process_queue & input_pkt_queue later.
11347 */
11348 while (!list_empty(&oldsd->poll_list)) {
11349 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
11350 struct napi_struct,
11351 poll_list);
11352
11353 list_del_init(&napi->poll_list);
11354 if (napi->poll == process_backlog)
11355 napi->state = 0;
11356 else
11357 ____napi_schedule(sd, napi);
264524d5 11358 }
1da177e4
LT
11359
11360 raise_softirq_irqoff(NET_TX_SOFTIRQ);
11361 local_irq_enable();
11362
773fc8f6 11363#ifdef CONFIG_RPS
11364 remsd = oldsd->rps_ipi_list;
11365 oldsd->rps_ipi_list = NULL;
11366#endif
11367 /* send out pending IPI's on offline CPU */
11368 net_rps_send_ipi(remsd);
11369
1da177e4 11370 /* Process offline CPU's input_pkt_queue */
76cc8b13 11371 while ((skb = __skb_dequeue(&oldsd->process_queue))) {
91e83133 11372 netif_rx_ni(skb);
76cc8b13 11373 input_queue_head_incr(oldsd);
fec5e652 11374 }
ac64da0b 11375 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
91e83133 11376 netif_rx_ni(skb);
76cc8b13
TH
11377 input_queue_head_incr(oldsd);
11378 }
1da177e4 11379
f0bf90de 11380 return 0;
1da177e4 11381}
1da177e4 11382
7f353bf2 11383/**
b63365a2
HX
11384 * netdev_increment_features - increment feature set by one
11385 * @all: current feature set
11386 * @one: new feature set
11387 * @mask: mask feature set
7f353bf2
HX
11388 *
11389 * Computes a new feature set after adding a device with feature set
b63365a2
HX
11390 * @one to the master device with current feature set @all. Will not
11391 * enable anything that is off in @mask. Returns the new feature set.
7f353bf2 11392 */
c8f44aff
MM
11393netdev_features_t netdev_increment_features(netdev_features_t all,
11394 netdev_features_t one, netdev_features_t mask)
b63365a2 11395{
c8cd0989 11396 if (mask & NETIF_F_HW_CSUM)
a188222b 11397 mask |= NETIF_F_CSUM_MASK;
1742f183 11398 mask |= NETIF_F_VLAN_CHALLENGED;
7f353bf2 11399
a188222b 11400 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
1742f183 11401 all &= one | ~NETIF_F_ALL_FOR_ALL;
c6e1a0d1 11402
1742f183 11403 /* If one device supports hw checksumming, set for all. */
c8cd0989
TH
11404 if (all & NETIF_F_HW_CSUM)
11405 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
7f353bf2
HX
11406
11407 return all;
11408}
b63365a2 11409EXPORT_SYMBOL(netdev_increment_features);
7f353bf2 11410
430f03cd 11411static struct hlist_head * __net_init netdev_create_hash(void)
30d97d35
PE
11412{
11413 int i;
11414 struct hlist_head *hash;
11415
6da2ec56 11416 hash = kmalloc_array(NETDEV_HASHENTRIES, sizeof(*hash), GFP_KERNEL);
30d97d35
PE
11417 if (hash != NULL)
11418 for (i = 0; i < NETDEV_HASHENTRIES; i++)
11419 INIT_HLIST_HEAD(&hash[i]);
11420
11421 return hash;
11422}
11423
881d966b 11424/* Initialize per network namespace state */
4665079c 11425static int __net_init netdev_init(struct net *net)
881d966b 11426{
d9f37d01 11427 BUILD_BUG_ON(GRO_HASH_BUCKETS >
c593642c 11428 8 * sizeof_field(struct napi_struct, gro_bitmask));
d9f37d01 11429
734b6541
RM
11430 if (net != &init_net)
11431 INIT_LIST_HEAD(&net->dev_base_head);
881d966b 11432
30d97d35
PE
11433 net->dev_name_head = netdev_create_hash();
11434 if (net->dev_name_head == NULL)
11435 goto err_name;
881d966b 11436
30d97d35
PE
11437 net->dev_index_head = netdev_create_hash();
11438 if (net->dev_index_head == NULL)
11439 goto err_idx;
881d966b 11440
a30c7b42
JP
11441 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
11442
881d966b 11443 return 0;
30d97d35
PE
11444
11445err_idx:
11446 kfree(net->dev_name_head);
11447err_name:
11448 return -ENOMEM;
881d966b
EB
11449}
11450
f0db275a
SH
11451/**
11452 * netdev_drivername - network driver for the device
11453 * @dev: network device
f0db275a
SH
11454 *
11455 * Determine network driver for device.
11456 */
3019de12 11457const char *netdev_drivername(const struct net_device *dev)
6579e57b 11458{
cf04a4c7
SH
11459 const struct device_driver *driver;
11460 const struct device *parent;
3019de12 11461 const char *empty = "";
6579e57b
AV
11462
11463 parent = dev->dev.parent;
6579e57b 11464 if (!parent)
3019de12 11465 return empty;
6579e57b
AV
11466
11467 driver = parent->driver;
11468 if (driver && driver->name)
3019de12
DM
11469 return driver->name;
11470 return empty;
6579e57b
AV
11471}
11472
6ea754eb
JP
11473static void __netdev_printk(const char *level, const struct net_device *dev,
11474 struct va_format *vaf)
256df2f3 11475{
b004ff49 11476 if (dev && dev->dev.parent) {
6ea754eb
JP
11477 dev_printk_emit(level[1] - '0',
11478 dev->dev.parent,
11479 "%s %s %s%s: %pV",
11480 dev_driver_string(dev->dev.parent),
11481 dev_name(dev->dev.parent),
11482 netdev_name(dev), netdev_reg_state(dev),
11483 vaf);
b004ff49 11484 } else if (dev) {
6ea754eb
JP
11485 printk("%s%s%s: %pV",
11486 level, netdev_name(dev), netdev_reg_state(dev), vaf);
b004ff49 11487 } else {
6ea754eb 11488 printk("%s(NULL net_device): %pV", level, vaf);
b004ff49 11489 }
256df2f3
JP
11490}
11491
6ea754eb
JP
11492void netdev_printk(const char *level, const struct net_device *dev,
11493 const char *format, ...)
256df2f3
JP
11494{
11495 struct va_format vaf;
11496 va_list args;
256df2f3
JP
11497
11498 va_start(args, format);
11499
11500 vaf.fmt = format;
11501 vaf.va = &args;
11502
6ea754eb 11503 __netdev_printk(level, dev, &vaf);
b004ff49 11504
256df2f3 11505 va_end(args);
256df2f3
JP
11506}
11507EXPORT_SYMBOL(netdev_printk);
11508
11509#define define_netdev_printk_level(func, level) \
6ea754eb 11510void func(const struct net_device *dev, const char *fmt, ...) \
256df2f3 11511{ \
256df2f3
JP
11512 struct va_format vaf; \
11513 va_list args; \
11514 \
11515 va_start(args, fmt); \
11516 \
11517 vaf.fmt = fmt; \
11518 vaf.va = &args; \
11519 \
6ea754eb 11520 __netdev_printk(level, dev, &vaf); \
b004ff49 11521 \
256df2f3 11522 va_end(args); \
256df2f3
JP
11523} \
11524EXPORT_SYMBOL(func);
11525
11526define_netdev_printk_level(netdev_emerg, KERN_EMERG);
11527define_netdev_printk_level(netdev_alert, KERN_ALERT);
11528define_netdev_printk_level(netdev_crit, KERN_CRIT);
11529define_netdev_printk_level(netdev_err, KERN_ERR);
11530define_netdev_printk_level(netdev_warn, KERN_WARNING);
11531define_netdev_printk_level(netdev_notice, KERN_NOTICE);
11532define_netdev_printk_level(netdev_info, KERN_INFO);
11533
4665079c 11534static void __net_exit netdev_exit(struct net *net)
881d966b
EB
11535{
11536 kfree(net->dev_name_head);
11537 kfree(net->dev_index_head);
ee21b18b
VA
11538 if (net != &init_net)
11539 WARN_ON_ONCE(!list_empty(&net->dev_base_head));
881d966b
EB
11540}
11541
022cbae6 11542static struct pernet_operations __net_initdata netdev_net_ops = {
881d966b
EB
11543 .init = netdev_init,
11544 .exit = netdev_exit,
11545};
11546
4665079c 11547static void __net_exit default_device_exit(struct net *net)
ce286d32 11548{
e008b5fc 11549 struct net_device *dev, *aux;
ce286d32 11550 /*
e008b5fc 11551 * Push all migratable network devices back to the
ce286d32
EB
11552 * initial network namespace
11553 */
11554 rtnl_lock();
e008b5fc 11555 for_each_netdev_safe(net, dev, aux) {
ce286d32 11556 int err;
aca51397 11557 char fb_name[IFNAMSIZ];
ce286d32
EB
11558
11559 /* Ignore unmoveable devices (i.e. loopback) */
11560 if (dev->features & NETIF_F_NETNS_LOCAL)
11561 continue;
11562
e008b5fc 11563 /* Leave virtual devices for the generic cleanup */
3a5ca857 11564 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
e008b5fc 11565 continue;
d0c082ce 11566
25985edc 11567 /* Push remaining network devices to init_net */
aca51397 11568 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
55b40dbf
JP
11569 if (__dev_get_by_name(&init_net, fb_name))
11570 snprintf(fb_name, IFNAMSIZ, "dev%%d");
0854fa82 11571 err = dev_change_net_namespace(dev, &init_net, fb_name);
ce286d32 11572 if (err) {
7b6cd1ce
JP
11573 pr_emerg("%s: failed to move %s to init_net: %d\n",
11574 __func__, dev->name, err);
aca51397 11575 BUG();
ce286d32
EB
11576 }
11577 }
11578 rtnl_unlock();
11579}
11580
50624c93
EB
11581static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
11582{
11583 /* Return with the rtnl_lock held when there are no network
11584 * devices unregistering in any network namespace in net_list.
11585 */
11586 struct net *net;
11587 bool unregistering;
ff960a73 11588 DEFINE_WAIT_FUNC(wait, woken_wake_function);
50624c93 11589
ff960a73 11590 add_wait_queue(&netdev_unregistering_wq, &wait);
50624c93 11591 for (;;) {
50624c93
EB
11592 unregistering = false;
11593 rtnl_lock();
11594 list_for_each_entry(net, net_list, exit_list) {
11595 if (net->dev_unreg_count > 0) {
11596 unregistering = true;
11597 break;
11598 }
11599 }
11600 if (!unregistering)
11601 break;
11602 __rtnl_unlock();
ff960a73
PZ
11603
11604 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
50624c93 11605 }
ff960a73 11606 remove_wait_queue(&netdev_unregistering_wq, &wait);
50624c93
EB
11607}
11608
04dc7f6b
EB
11609static void __net_exit default_device_exit_batch(struct list_head *net_list)
11610{
11611 /* At exit all network devices most be removed from a network
b595076a 11612 * namespace. Do this in the reverse order of registration.
04dc7f6b
EB
11613 * Do this across as many network namespaces as possible to
11614 * improve batching efficiency.
11615 */
11616 struct net_device *dev;
11617 struct net *net;
11618 LIST_HEAD(dev_kill_list);
11619
50624c93
EB
11620 /* To prevent network device cleanup code from dereferencing
11621 * loopback devices or network devices that have been freed
11622 * wait here for all pending unregistrations to complete,
11623 * before unregistring the loopback device and allowing the
11624 * network namespace be freed.
11625 *
11626 * The netdev todo list containing all network devices
11627 * unregistrations that happen in default_device_exit_batch
11628 * will run in the rtnl_unlock() at the end of
11629 * default_device_exit_batch.
11630 */
11631 rtnl_lock_unregistering(net_list);
04dc7f6b
EB
11632 list_for_each_entry(net, net_list, exit_list) {
11633 for_each_netdev_reverse(net, dev) {
b0ab2fab 11634 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
04dc7f6b
EB
11635 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
11636 else
11637 unregister_netdevice_queue(dev, &dev_kill_list);
11638 }
11639 }
11640 unregister_netdevice_many(&dev_kill_list);
11641 rtnl_unlock();
11642}
11643
022cbae6 11644static struct pernet_operations __net_initdata default_device_ops = {
ce286d32 11645 .exit = default_device_exit,
04dc7f6b 11646 .exit_batch = default_device_exit_batch,
ce286d32
EB
11647};
11648
1da177e4
LT
11649/*
11650 * Initialize the DEV module. At boot time this walks the device list and
11651 * unhooks any devices that fail to initialise (normally hardware not
11652 * present) and leaves us with a valid list of present and active devices.
11653 *
11654 */
11655
11656/*
11657 * This is called single threaded during boot, so no need
11658 * to take the rtnl semaphore.
11659 */
11660static int __init net_dev_init(void)
11661{
11662 int i, rc = -ENOMEM;
11663
11664 BUG_ON(!dev_boot_phase);
11665
1da177e4
LT
11666 if (dev_proc_init())
11667 goto out;
11668
8b41d188 11669 if (netdev_kobject_init())
1da177e4
LT
11670 goto out;
11671
11672 INIT_LIST_HEAD(&ptype_all);
82d8a867 11673 for (i = 0; i < PTYPE_HASH_SIZE; i++)
1da177e4
LT
11674 INIT_LIST_HEAD(&ptype_base[i]);
11675
62532da9
VY
11676 INIT_LIST_HEAD(&offload_base);
11677
881d966b
EB
11678 if (register_pernet_subsys(&netdev_net_ops))
11679 goto out;
1da177e4
LT
11680
11681 /*
11682 * Initialise the packet receive queues.
11683 */
11684
6f912042 11685 for_each_possible_cpu(i) {
41852497 11686 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
e36fa2f7 11687 struct softnet_data *sd = &per_cpu(softnet_data, i);
1da177e4 11688
41852497
ED
11689 INIT_WORK(flush, flush_backlog);
11690
e36fa2f7 11691 skb_queue_head_init(&sd->input_pkt_queue);
6e7676c1 11692 skb_queue_head_init(&sd->process_queue);
f53c7239
SK
11693#ifdef CONFIG_XFRM_OFFLOAD
11694 skb_queue_head_init(&sd->xfrm_backlog);
11695#endif
e36fa2f7 11696 INIT_LIST_HEAD(&sd->poll_list);
a9cbd588 11697 sd->output_queue_tailp = &sd->output_queue;
df334545 11698#ifdef CONFIG_RPS
545b8c8d 11699 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
e36fa2f7 11700 sd->cpu = i;
1e94d72f 11701#endif
0a9627f2 11702
7c4ec749 11703 init_gro_hash(&sd->backlog);
e36fa2f7
ED
11704 sd->backlog.poll = process_backlog;
11705 sd->backlog.weight = weight_p;
1da177e4
LT
11706 }
11707
1da177e4
LT
11708 dev_boot_phase = 0;
11709
505d4f73
EB
11710 /* The loopback device is special if any other network devices
11711 * is present in a network namespace the loopback device must
11712 * be present. Since we now dynamically allocate and free the
11713 * loopback device ensure this invariant is maintained by
11714 * keeping the loopback device as the first device on the
11715 * list of network devices. Ensuring the loopback devices
11716 * is the first device that appears and the last network device
11717 * that disappears.
11718 */
11719 if (register_pernet_device(&loopback_net_ops))
11720 goto out;
11721
11722 if (register_pernet_device(&default_device_ops))
11723 goto out;
11724
962cf36c
CM
11725 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11726 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
1da177e4 11727
f0bf90de
SAS
11728 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11729 NULL, dev_cpu_dead);
11730 WARN_ON(rc < 0);
1da177e4
LT
11731 rc = 0;
11732out:
11733 return rc;
11734}
11735
11736subsys_initcall(net_dev_init);