Merge branch 'cleanups-4.7' into for-chris-4.7-20160525
[linux-2.6-block.git] / net / core / dev.c
1 /*
2  *      NET3    Protocol independent device support routines.
3  *
4  *              This program is free software; you can redistribute it and/or
5  *              modify it under the terms of the GNU General Public License
6  *              as published by the Free Software Foundation; either version
7  *              2 of the License, or (at your option) any later version.
8  *
9  *      Derived from the non IP parts of dev.c 1.0.19
10  *              Authors:        Ross Biro
11  *                              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *                              Mark Evans, <evansmp@uhura.aston.ac.uk>
13  *
14  *      Additional Authors:
15  *              Florian la Roche <rzsfl@rz.uni-sb.de>
16  *              Alan Cox <gw4pts@gw4pts.ampr.org>
17  *              David Hinds <dahinds@users.sourceforge.net>
18  *              Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
19  *              Adam Sulmicki <adam@cfar.umd.edu>
20  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
21  *
22  *      Changes:
23  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
24  *                                      to 2 if register_netdev gets called
25  *                                      before net_dev_init & also removed a
26  *                                      few lines of code in the process.
27  *              Alan Cox        :       device private ioctl copies fields back.
28  *              Alan Cox        :       Transmit queue code does relevant
29  *                                      stunts to keep the queue safe.
30  *              Alan Cox        :       Fixed double lock.
31  *              Alan Cox        :       Fixed promisc NULL pointer trap
32  *              ????????        :       Support the full private ioctl range
33  *              Alan Cox        :       Moved ioctl permission check into
34  *                                      drivers
35  *              Tim Kordas      :       SIOCADDMULTI/SIOCDELMULTI
36  *              Alan Cox        :       100 backlog just doesn't cut it when
37  *                                      you start doing multicast video 8)
38  *              Alan Cox        :       Rewrote net_bh and list manager.
39  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
40  *              Alan Cox        :       Took out transmit every packet pass
41  *                                      Saved a few bytes in the ioctl handler
42  *              Alan Cox        :       Network driver sets packet type before
43  *                                      calling netif_rx. Saves a function
44  *                                      call a packet.
45  *              Alan Cox        :       Hashed net_bh()
46  *              Richard Kooijman:       Timestamp fixes.
47  *              Alan Cox        :       Wrong field in SIOCGIFDSTADDR
48  *              Alan Cox        :       Device lock protection.
49  *              Alan Cox        :       Fixed nasty side effect of device close
50  *                                      changes.
51  *              Rudi Cilibrasi  :       Pass the right thing to
52  *                                      set_mac_address()
53  *              Dave Miller     :       32bit quantity for the device lock to
54  *                                      make it work out on a Sparc.
55  *              Bjorn Ekwall    :       Added KERNELD hack.
56  *              Alan Cox        :       Cleaned up the backlog initialise.
57  *              Craig Metz      :       SIOCGIFCONF fix if space for under
58  *                                      1 device.
59  *          Thomas Bogendoerfer :       Return ENODEV for dev_open, if there
60  *                                      is no device open function.
61  *              Andi Kleen      :       Fix error reporting for SIOCGIFCONF
62  *          Michael Chastain    :       Fix signed/unsigned for SIOCGIFCONF
63  *              Cyrus Durgin    :       Cleaned for KMOD
64  *              Adam Sulmicki   :       Bug Fix : Network Device Unload
65  *                                      A network device unload needs to purge
66  *                                      the backlog queue.
67  *      Paul Rusty Russell      :       SIOCSIFNAME
68  *              Pekka Riikonen  :       Netdev boot-time settings code
69  *              Andrew Morton   :       Make unregister_netdevice wait
70  *                                      indefinitely on dev->refcnt
71  *              J Hadi Salim    :       - Backlog queue sampling
72  *                                      - netif_rx() feedback
73  */
74
75 #include <asm/uaccess.h>
76 #include <linux/bitops.h>
77 #include <linux/capability.h>
78 #include <linux/cpu.h>
79 #include <linux/types.h>
80 #include <linux/kernel.h>
81 #include <linux/hash.h>
82 #include <linux/slab.h>
83 #include <linux/sched.h>
84 #include <linux/mutex.h>
85 #include <linux/string.h>
86 #include <linux/mm.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/notifier.h>
96 #include <linux/skbuff.h>
97 #include <net/net_namespace.h>
98 #include <net/sock.h>
99 #include <net/busy_poll.h>
100 #include <linux/rtnetlink.h>
101 #include <linux/stat.h>
102 #include <net/dst.h>
103 #include <net/dst_metadata.h>
104 #include <net/pkt_sched.h>
105 #include <net/checksum.h>
106 #include <net/xfrm.h>
107 #include <linux/highmem.h>
108 #include <linux/init.h>
109 #include <linux/module.h>
110 #include <linux/netpoll.h>
111 #include <linux/rcupdate.h>
112 #include <linux/delay.h>
113 #include <net/iw_handler.h>
114 #include <asm/current.h>
115 #include <linux/audit.h>
116 #include <linux/dmaengine.h>
117 #include <linux/err.h>
118 #include <linux/ctype.h>
119 #include <linux/if_arp.h>
120 #include <linux/if_vlan.h>
121 #include <linux/ip.h>
122 #include <net/ip.h>
123 #include <net/mpls.h>
124 #include <linux/ipv6.h>
125 #include <linux/in.h>
126 #include <linux/jhash.h>
127 #include <linux/random.h>
128 #include <trace/events/napi.h>
129 #include <trace/events/net.h>
130 #include <trace/events/skb.h>
131 #include <linux/pci.h>
132 #include <linux/inetdevice.h>
133 #include <linux/cpu_rmap.h>
134 #include <linux/static_key.h>
135 #include <linux/hashtable.h>
136 #include <linux/vmalloc.h>
137 #include <linux/if_macvlan.h>
138 #include <linux/errqueue.h>
139 #include <linux/hrtimer.h>
140 #include <linux/netfilter_ingress.h>
141 #include <linux/sctp.h>
142
143 #include "net-sysfs.h"
144
145 /* Instead of increasing this, you should create a hash table. */
146 #define MAX_GRO_SKBS 8
147
148 /* This should be increased if a protocol with a bigger head is added. */
149 #define GRO_MAX_HEAD (MAX_HEADER + 128)
150
151 static DEFINE_SPINLOCK(ptype_lock);
152 static DEFINE_SPINLOCK(offload_lock);
153 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
154 struct list_head ptype_all __read_mostly;       /* Taps */
155 static struct list_head offload_base __read_mostly;
156
157 static int netif_rx_internal(struct sk_buff *skb);
158 static int call_netdevice_notifiers_info(unsigned long val,
159                                          struct net_device *dev,
160                                          struct netdev_notifier_info *info);
161
162 /*
163  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
164  * semaphore.
165  *
166  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
167  *
168  * Writers must hold the rtnl semaphore while they loop through the
169  * dev_base_head list, and hold dev_base_lock for writing when they do the
170  * actual updates.  This allows pure readers to access the list even
171  * while a writer is preparing to update it.
172  *
173  * To put it another way, dev_base_lock is held for writing only to
174  * protect against pure readers; the rtnl semaphore provides the
175  * protection against other writers.
176  *
177  * See, for example usages, register_netdevice() and
178  * unregister_netdevice(), which must be called with the rtnl
179  * semaphore held.
180  */
181 DEFINE_RWLOCK(dev_base_lock);
182 EXPORT_SYMBOL(dev_base_lock);
183
184 /* protects napi_hash addition/deletion and napi_gen_id */
185 static DEFINE_SPINLOCK(napi_hash_lock);
186
187 static unsigned int napi_gen_id = NR_CPUS;
188 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
189
190 static seqcount_t devnet_rename_seq;
191
192 static inline void dev_base_seq_inc(struct net *net)
193 {
194         while (++net->dev_base_seq == 0);
195 }
196
197 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
198 {
199         unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
200
201         return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
202 }
203
204 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
205 {
206         return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
207 }
208
209 static inline void rps_lock(struct softnet_data *sd)
210 {
211 #ifdef CONFIG_RPS
212         spin_lock(&sd->input_pkt_queue.lock);
213 #endif
214 }
215
216 static inline void rps_unlock(struct softnet_data *sd)
217 {
218 #ifdef CONFIG_RPS
219         spin_unlock(&sd->input_pkt_queue.lock);
220 #endif
221 }
222
223 /* Device list insertion */
224 static void list_netdevice(struct net_device *dev)
225 {
226         struct net *net = dev_net(dev);
227
228         ASSERT_RTNL();
229
230         write_lock_bh(&dev_base_lock);
231         list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
232         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
233         hlist_add_head_rcu(&dev->index_hlist,
234                            dev_index_hash(net, dev->ifindex));
235         write_unlock_bh(&dev_base_lock);
236
237         dev_base_seq_inc(net);
238 }
239
240 /* Device list removal
241  * caller must respect a RCU grace period before freeing/reusing dev
242  */
243 static void unlist_netdevice(struct net_device *dev)
244 {
245         ASSERT_RTNL();
246
247         /* Unlink dev from the device chain */
248         write_lock_bh(&dev_base_lock);
249         list_del_rcu(&dev->dev_list);
250         hlist_del_rcu(&dev->name_hlist);
251         hlist_del_rcu(&dev->index_hlist);
252         write_unlock_bh(&dev_base_lock);
253
254         dev_base_seq_inc(dev_net(dev));
255 }
256
257 /*
258  *      Our notifier list
259  */
260
261 static RAW_NOTIFIER_HEAD(netdev_chain);
262
263 /*
264  *      Device drivers call our routines to queue packets here. We empty the
265  *      queue in the local softnet handler.
266  */
267
268 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
269 EXPORT_PER_CPU_SYMBOL(softnet_data);
270
271 #ifdef CONFIG_LOCKDEP
272 /*
273  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
274  * according to dev->type
275  */
276 static const unsigned short netdev_lock_type[] =
277         {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
278          ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
279          ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
280          ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
281          ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
282          ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
283          ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
284          ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
285          ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
286          ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
287          ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
288          ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
289          ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
290          ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
291          ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE};
292
293 static const char *const netdev_lock_name[] =
294         {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
295          "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
296          "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
297          "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
298          "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
299          "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
300          "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
301          "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
302          "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
303          "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
304          "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
305          "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
306          "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
307          "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE",
308          "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"};
309
310 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
311 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
312
313 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
314 {
315         int i;
316
317         for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
318                 if (netdev_lock_type[i] == dev_type)
319                         return i;
320         /* the last key is used by default */
321         return ARRAY_SIZE(netdev_lock_type) - 1;
322 }
323
324 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
325                                                  unsigned short dev_type)
326 {
327         int i;
328
329         i = netdev_lock_pos(dev_type);
330         lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
331                                    netdev_lock_name[i]);
332 }
333
334 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
335 {
336         int i;
337
338         i = netdev_lock_pos(dev->type);
339         lockdep_set_class_and_name(&dev->addr_list_lock,
340                                    &netdev_addr_lock_key[i],
341                                    netdev_lock_name[i]);
342 }
343 #else
344 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
345                                                  unsigned short dev_type)
346 {
347 }
348 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
349 {
350 }
351 #endif
352
353 /*******************************************************************************
354
355                 Protocol management and registration routines
356
357 *******************************************************************************/
358
359 /*
360  *      Add a protocol ID to the list. Now that the input handler is
361  *      smarter we can dispense with all the messy stuff that used to be
362  *      here.
363  *
364  *      BEWARE!!! Protocol handlers, mangling input packets,
365  *      MUST BE last in hash buckets and checking protocol handlers
366  *      MUST start from promiscuous ptype_all chain in net_bh.
367  *      It is true now, do not change it.
368  *      Explanation follows: if protocol handler, mangling packet, will
369  *      be the first on list, it is not able to sense, that packet
370  *      is cloned and should be copied-on-write, so that it will
371  *      change it and subsequent readers will get broken packet.
372  *                                                      --ANK (980803)
373  */
374
375 static inline struct list_head *ptype_head(const struct packet_type *pt)
376 {
377         if (pt->type == htons(ETH_P_ALL))
378                 return pt->dev ? &pt->dev->ptype_all : &ptype_all;
379         else
380                 return pt->dev ? &pt->dev->ptype_specific :
381                                  &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
382 }
383
384 /**
385  *      dev_add_pack - add packet handler
386  *      @pt: packet type declaration
387  *
388  *      Add a protocol handler to the networking stack. The passed &packet_type
389  *      is linked into kernel lists and may not be freed until it has been
390  *      removed from the kernel lists.
391  *
392  *      This call does not sleep therefore it can not
393  *      guarantee all CPU's that are in middle of receiving packets
394  *      will see the new packet type (until the next received packet).
395  */
396
397 void dev_add_pack(struct packet_type *pt)
398 {
399         struct list_head *head = ptype_head(pt);
400
401         spin_lock(&ptype_lock);
402         list_add_rcu(&pt->list, head);
403         spin_unlock(&ptype_lock);
404 }
405 EXPORT_SYMBOL(dev_add_pack);
406
407 /**
408  *      __dev_remove_pack        - remove packet handler
409  *      @pt: packet type declaration
410  *
411  *      Remove a protocol handler that was previously added to the kernel
412  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
413  *      from the kernel lists and can be freed or reused once this function
414  *      returns.
415  *
416  *      The packet type might still be in use by receivers
417  *      and must not be freed until after all the CPU's have gone
418  *      through a quiescent state.
419  */
420 void __dev_remove_pack(struct packet_type *pt)
421 {
422         struct list_head *head = ptype_head(pt);
423         struct packet_type *pt1;
424
425         spin_lock(&ptype_lock);
426
427         list_for_each_entry(pt1, head, list) {
428                 if (pt == pt1) {
429                         list_del_rcu(&pt->list);
430                         goto out;
431                 }
432         }
433
434         pr_warn("dev_remove_pack: %p not found\n", pt);
435 out:
436         spin_unlock(&ptype_lock);
437 }
438 EXPORT_SYMBOL(__dev_remove_pack);
439
440 /**
441  *      dev_remove_pack  - remove packet handler
442  *      @pt: packet type declaration
443  *
444  *      Remove a protocol handler that was previously added to the kernel
445  *      protocol handlers by dev_add_pack(). The passed &packet_type is removed
446  *      from the kernel lists and can be freed or reused once this function
447  *      returns.
448  *
449  *      This call sleeps to guarantee that no CPU is looking at the packet
450  *      type after return.
451  */
452 void dev_remove_pack(struct packet_type *pt)
453 {
454         __dev_remove_pack(pt);
455
456         synchronize_net();
457 }
458 EXPORT_SYMBOL(dev_remove_pack);
459
460
461 /**
462  *      dev_add_offload - register offload handlers
463  *      @po: protocol offload declaration
464  *
465  *      Add protocol offload handlers to the networking stack. The passed
466  *      &proto_offload is linked into kernel lists and may not be freed until
467  *      it has been removed from the kernel lists.
468  *
469  *      This call does not sleep therefore it can not
470  *      guarantee all CPU's that are in middle of receiving packets
471  *      will see the new offload handlers (until the next received packet).
472  */
473 void dev_add_offload(struct packet_offload *po)
474 {
475         struct packet_offload *elem;
476
477         spin_lock(&offload_lock);
478         list_for_each_entry(elem, &offload_base, list) {
479                 if (po->priority < elem->priority)
480                         break;
481         }
482         list_add_rcu(&po->list, elem->list.prev);
483         spin_unlock(&offload_lock);
484 }
485 EXPORT_SYMBOL(dev_add_offload);
486
487 /**
488  *      __dev_remove_offload     - remove offload handler
489  *      @po: packet offload declaration
490  *
491  *      Remove a protocol offload handler that was previously added to the
492  *      kernel offload handlers by dev_add_offload(). The passed &offload_type
493  *      is removed from the kernel lists and can be freed or reused once this
494  *      function returns.
495  *
496  *      The packet type might still be in use by receivers
497  *      and must not be freed until after all the CPU's have gone
498  *      through a quiescent state.
499  */
500 static void __dev_remove_offload(struct packet_offload *po)
501 {
502         struct list_head *head = &offload_base;
503         struct packet_offload *po1;
504
505         spin_lock(&offload_lock);
506
507         list_for_each_entry(po1, head, list) {
508                 if (po == po1) {
509                         list_del_rcu(&po->list);
510                         goto out;
511                 }
512         }
513
514         pr_warn("dev_remove_offload: %p not found\n", po);
515 out:
516         spin_unlock(&offload_lock);
517 }
518
519 /**
520  *      dev_remove_offload       - remove packet offload handler
521  *      @po: packet offload declaration
522  *
523  *      Remove a packet offload handler that was previously added to the kernel
524  *      offload handlers by dev_add_offload(). The passed &offload_type is
525  *      removed from the kernel lists and can be freed or reused once this
526  *      function returns.
527  *
528  *      This call sleeps to guarantee that no CPU is looking at the packet
529  *      type after return.
530  */
531 void dev_remove_offload(struct packet_offload *po)
532 {
533         __dev_remove_offload(po);
534
535         synchronize_net();
536 }
537 EXPORT_SYMBOL(dev_remove_offload);
538
539 /******************************************************************************
540
541                       Device Boot-time Settings Routines
542
543 *******************************************************************************/
544
545 /* Boot time configuration table */
546 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
547
548 /**
549  *      netdev_boot_setup_add   - add new setup entry
550  *      @name: name of the device
551  *      @map: configured settings for the device
552  *
553  *      Adds new setup entry to the dev_boot_setup list.  The function
554  *      returns 0 on error and 1 on success.  This is a generic routine to
555  *      all netdevices.
556  */
557 static int netdev_boot_setup_add(char *name, struct ifmap *map)
558 {
559         struct netdev_boot_setup *s;
560         int i;
561
562         s = dev_boot_setup;
563         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
564                 if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
565                         memset(s[i].name, 0, sizeof(s[i].name));
566                         strlcpy(s[i].name, name, IFNAMSIZ);
567                         memcpy(&s[i].map, map, sizeof(s[i].map));
568                         break;
569                 }
570         }
571
572         return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
573 }
574
575 /**
576  *      netdev_boot_setup_check - check boot time settings
577  *      @dev: the netdevice
578  *
579  *      Check boot time settings for the device.
580  *      The found settings are set for the device to be used
581  *      later in the device probing.
582  *      Returns 0 if no settings found, 1 if they are.
583  */
584 int netdev_boot_setup_check(struct net_device *dev)
585 {
586         struct netdev_boot_setup *s = dev_boot_setup;
587         int i;
588
589         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
590                 if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
591                     !strcmp(dev->name, s[i].name)) {
592                         dev->irq        = s[i].map.irq;
593                         dev->base_addr  = s[i].map.base_addr;
594                         dev->mem_start  = s[i].map.mem_start;
595                         dev->mem_end    = s[i].map.mem_end;
596                         return 1;
597                 }
598         }
599         return 0;
600 }
601 EXPORT_SYMBOL(netdev_boot_setup_check);
602
603
604 /**
605  *      netdev_boot_base        - get address from boot time settings
606  *      @prefix: prefix for network device
607  *      @unit: id for network device
608  *
609  *      Check boot time settings for the base address of device.
610  *      The found settings are set for the device to be used
611  *      later in the device probing.
612  *      Returns 0 if no settings found.
613  */
614 unsigned long netdev_boot_base(const char *prefix, int unit)
615 {
616         const struct netdev_boot_setup *s = dev_boot_setup;
617         char name[IFNAMSIZ];
618         int i;
619
620         sprintf(name, "%s%d", prefix, unit);
621
622         /*
623          * If device already registered then return base of 1
624          * to indicate not to probe for this interface
625          */
626         if (__dev_get_by_name(&init_net, name))
627                 return 1;
628
629         for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
630                 if (!strcmp(name, s[i].name))
631                         return s[i].map.base_addr;
632         return 0;
633 }
634
635 /*
636  * Saves at boot time configured settings for any netdevice.
637  */
638 int __init netdev_boot_setup(char *str)
639 {
640         int ints[5];
641         struct ifmap map;
642
643         str = get_options(str, ARRAY_SIZE(ints), ints);
644         if (!str || !*str)
645                 return 0;
646
647         /* Save settings */
648         memset(&map, 0, sizeof(map));
649         if (ints[0] > 0)
650                 map.irq = ints[1];
651         if (ints[0] > 1)
652                 map.base_addr = ints[2];
653         if (ints[0] > 2)
654                 map.mem_start = ints[3];
655         if (ints[0] > 3)
656                 map.mem_end = ints[4];
657
658         /* Add new entry to the list */
659         return netdev_boot_setup_add(str, &map);
660 }
661
662 __setup("netdev=", netdev_boot_setup);
663
664 /*******************************************************************************
665
666                             Device Interface Subroutines
667
668 *******************************************************************************/
669
670 /**
671  *      dev_get_iflink  - get 'iflink' value of a interface
672  *      @dev: targeted interface
673  *
674  *      Indicates the ifindex the interface is linked to.
675  *      Physical interfaces have the same 'ifindex' and 'iflink' values.
676  */
677
678 int dev_get_iflink(const struct net_device *dev)
679 {
680         if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
681                 return dev->netdev_ops->ndo_get_iflink(dev);
682
683         return dev->ifindex;
684 }
685 EXPORT_SYMBOL(dev_get_iflink);
686
687 /**
688  *      dev_fill_metadata_dst - Retrieve tunnel egress information.
689  *      @dev: targeted interface
690  *      @skb: The packet.
691  *
692  *      For better visibility of tunnel traffic OVS needs to retrieve
693  *      egress tunnel information for a packet. Following API allows
694  *      user to get this info.
695  */
696 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
697 {
698         struct ip_tunnel_info *info;
699
700         if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
701                 return -EINVAL;
702
703         info = skb_tunnel_info_unclone(skb);
704         if (!info)
705                 return -ENOMEM;
706         if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
707                 return -EINVAL;
708
709         return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
710 }
711 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
712
713 /**
714  *      __dev_get_by_name       - find a device by its name
715  *      @net: the applicable net namespace
716  *      @name: name to find
717  *
718  *      Find an interface by name. Must be called under RTNL semaphore
719  *      or @dev_base_lock. If the name is found a pointer to the device
720  *      is returned. If the name is not found then %NULL is returned. The
721  *      reference counters are not incremented so the caller must be
722  *      careful with locks.
723  */
724
725 struct net_device *__dev_get_by_name(struct net *net, const char *name)
726 {
727         struct net_device *dev;
728         struct hlist_head *head = dev_name_hash(net, name);
729
730         hlist_for_each_entry(dev, head, name_hlist)
731                 if (!strncmp(dev->name, name, IFNAMSIZ))
732                         return dev;
733
734         return NULL;
735 }
736 EXPORT_SYMBOL(__dev_get_by_name);
737
738 /**
739  *      dev_get_by_name_rcu     - find a device by its name
740  *      @net: the applicable net namespace
741  *      @name: name to find
742  *
743  *      Find an interface by name.
744  *      If the name is found a pointer to the device is returned.
745  *      If the name is not found then %NULL is returned.
746  *      The reference counters are not incremented so the caller must be
747  *      careful with locks. The caller must hold RCU lock.
748  */
749
750 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
751 {
752         struct net_device *dev;
753         struct hlist_head *head = dev_name_hash(net, name);
754
755         hlist_for_each_entry_rcu(dev, head, name_hlist)
756                 if (!strncmp(dev->name, name, IFNAMSIZ))
757                         return dev;
758
759         return NULL;
760 }
761 EXPORT_SYMBOL(dev_get_by_name_rcu);
762
763 /**
764  *      dev_get_by_name         - find a device by its name
765  *      @net: the applicable net namespace
766  *      @name: name to find
767  *
768  *      Find an interface by name. This can be called from any
769  *      context and does its own locking. The returned handle has
770  *      the usage count incremented and the caller must use dev_put() to
771  *      release it when it is no longer needed. %NULL is returned if no
772  *      matching device is found.
773  */
774
775 struct net_device *dev_get_by_name(struct net *net, const char *name)
776 {
777         struct net_device *dev;
778
779         rcu_read_lock();
780         dev = dev_get_by_name_rcu(net, name);
781         if (dev)
782                 dev_hold(dev);
783         rcu_read_unlock();
784         return dev;
785 }
786 EXPORT_SYMBOL(dev_get_by_name);
787
788 /**
789  *      __dev_get_by_index - find a device by its ifindex
790  *      @net: the applicable net namespace
791  *      @ifindex: index of device
792  *
793  *      Search for an interface by index. Returns %NULL if the device
794  *      is not found or a pointer to the device. The device has not
795  *      had its reference counter increased so the caller must be careful
796  *      about locking. The caller must hold either the RTNL semaphore
797  *      or @dev_base_lock.
798  */
799
800 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
801 {
802         struct net_device *dev;
803         struct hlist_head *head = dev_index_hash(net, ifindex);
804
805         hlist_for_each_entry(dev, head, index_hlist)
806                 if (dev->ifindex == ifindex)
807                         return dev;
808
809         return NULL;
810 }
811 EXPORT_SYMBOL(__dev_get_by_index);
812
813 /**
814  *      dev_get_by_index_rcu - find a device by its ifindex
815  *      @net: the applicable net namespace
816  *      @ifindex: index of device
817  *
818  *      Search for an interface by index. Returns %NULL if the device
819  *      is not found or a pointer to the device. The device has not
820  *      had its reference counter increased so the caller must be careful
821  *      about locking. The caller must hold RCU lock.
822  */
823
824 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
825 {
826         struct net_device *dev;
827         struct hlist_head *head = dev_index_hash(net, ifindex);
828
829         hlist_for_each_entry_rcu(dev, head, index_hlist)
830                 if (dev->ifindex == ifindex)
831                         return dev;
832
833         return NULL;
834 }
835 EXPORT_SYMBOL(dev_get_by_index_rcu);
836
837
838 /**
839  *      dev_get_by_index - find a device by its ifindex
840  *      @net: the applicable net namespace
841  *      @ifindex: index of device
842  *
843  *      Search for an interface by index. Returns NULL if the device
844  *      is not found or a pointer to the device. The device returned has
845  *      had a reference added and the pointer is safe until the user calls
846  *      dev_put to indicate they have finished with it.
847  */
848
849 struct net_device *dev_get_by_index(struct net *net, int ifindex)
850 {
851         struct net_device *dev;
852
853         rcu_read_lock();
854         dev = dev_get_by_index_rcu(net, ifindex);
855         if (dev)
856                 dev_hold(dev);
857         rcu_read_unlock();
858         return dev;
859 }
860 EXPORT_SYMBOL(dev_get_by_index);
861
862 /**
863  *      netdev_get_name - get a netdevice name, knowing its ifindex.
864  *      @net: network namespace
865  *      @name: a pointer to the buffer where the name will be stored.
866  *      @ifindex: the ifindex of the interface to get the name from.
867  *
868  *      The use of raw_seqcount_begin() and cond_resched() before
869  *      retrying is required as we want to give the writers a chance
870  *      to complete when CONFIG_PREEMPT is not set.
871  */
872 int netdev_get_name(struct net *net, char *name, int ifindex)
873 {
874         struct net_device *dev;
875         unsigned int seq;
876
877 retry:
878         seq = raw_seqcount_begin(&devnet_rename_seq);
879         rcu_read_lock();
880         dev = dev_get_by_index_rcu(net, ifindex);
881         if (!dev) {
882                 rcu_read_unlock();
883                 return -ENODEV;
884         }
885
886         strcpy(name, dev->name);
887         rcu_read_unlock();
888         if (read_seqcount_retry(&devnet_rename_seq, seq)) {
889                 cond_resched();
890                 goto retry;
891         }
892
893         return 0;
894 }
895
896 /**
897  *      dev_getbyhwaddr_rcu - find a device by its hardware address
898  *      @net: the applicable net namespace
899  *      @type: media type of device
900  *      @ha: hardware address
901  *
902  *      Search for an interface by MAC address. Returns NULL if the device
903  *      is not found or a pointer to the device.
904  *      The caller must hold RCU or RTNL.
905  *      The returned device has not had its ref count increased
906  *      and the caller must therefore be careful about locking
907  *
908  */
909
910 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
911                                        const char *ha)
912 {
913         struct net_device *dev;
914
915         for_each_netdev_rcu(net, dev)
916                 if (dev->type == type &&
917                     !memcmp(dev->dev_addr, ha, dev->addr_len))
918                         return dev;
919
920         return NULL;
921 }
922 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
923
924 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
925 {
926         struct net_device *dev;
927
928         ASSERT_RTNL();
929         for_each_netdev(net, dev)
930                 if (dev->type == type)
931                         return dev;
932
933         return NULL;
934 }
935 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
936
937 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
938 {
939         struct net_device *dev, *ret = NULL;
940
941         rcu_read_lock();
942         for_each_netdev_rcu(net, dev)
943                 if (dev->type == type) {
944                         dev_hold(dev);
945                         ret = dev;
946                         break;
947                 }
948         rcu_read_unlock();
949         return ret;
950 }
951 EXPORT_SYMBOL(dev_getfirstbyhwtype);
952
953 /**
954  *      __dev_get_by_flags - find any device with given flags
955  *      @net: the applicable net namespace
956  *      @if_flags: IFF_* values
957  *      @mask: bitmask of bits in if_flags to check
958  *
959  *      Search for any interface with the given flags. Returns NULL if a device
960  *      is not found or a pointer to the device. Must be called inside
961  *      rtnl_lock(), and result refcount is unchanged.
962  */
963
964 struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags,
965                                       unsigned short mask)
966 {
967         struct net_device *dev, *ret;
968
969         ASSERT_RTNL();
970
971         ret = NULL;
972         for_each_netdev(net, dev) {
973                 if (((dev->flags ^ if_flags) & mask) == 0) {
974                         ret = dev;
975                         break;
976                 }
977         }
978         return ret;
979 }
980 EXPORT_SYMBOL(__dev_get_by_flags);
981
982 /**
983  *      dev_valid_name - check if name is okay for network device
984  *      @name: name string
985  *
986  *      Network device names need to be valid file names to
987  *      to allow sysfs to work.  We also disallow any kind of
988  *      whitespace.
989  */
990 bool dev_valid_name(const char *name)
991 {
992         if (*name == '\0')
993                 return false;
994         if (strlen(name) >= IFNAMSIZ)
995                 return false;
996         if (!strcmp(name, ".") || !strcmp(name, ".."))
997                 return false;
998
999         while (*name) {
1000                 if (*name == '/' || *name == ':' || isspace(*name))
1001                         return false;
1002                 name++;
1003         }
1004         return true;
1005 }
1006 EXPORT_SYMBOL(dev_valid_name);
1007
1008 /**
1009  *      __dev_alloc_name - allocate a name for a device
1010  *      @net: network namespace to allocate the device name in
1011  *      @name: name format string
1012  *      @buf:  scratch buffer and result name string
1013  *
1014  *      Passed a format string - eg "lt%d" it will try and find a suitable
1015  *      id. It scans list of devices to build up a free map, then chooses
1016  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1017  *      while allocating the name and adding the device in order to avoid
1018  *      duplicates.
1019  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1020  *      Returns the number of the unit assigned or a negative errno code.
1021  */
1022
1023 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
1024 {
1025         int i = 0;
1026         const char *p;
1027         const int max_netdevices = 8*PAGE_SIZE;
1028         unsigned long *inuse;
1029         struct net_device *d;
1030
1031         p = strnchr(name, IFNAMSIZ-1, '%');
1032         if (p) {
1033                 /*
1034                  * Verify the string as this thing may have come from
1035                  * the user.  There must be either one "%d" and no other "%"
1036                  * characters.
1037                  */
1038                 if (p[1] != 'd' || strchr(p + 2, '%'))
1039                         return -EINVAL;
1040
1041                 /* Use one page as a bit array of possible slots */
1042                 inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
1043                 if (!inuse)
1044                         return -ENOMEM;
1045
1046                 for_each_netdev(net, d) {
1047                         if (!sscanf(d->name, name, &i))
1048                                 continue;
1049                         if (i < 0 || i >= max_netdevices)
1050                                 continue;
1051
1052                         /*  avoid cases where sscanf is not exact inverse of printf */
1053                         snprintf(buf, IFNAMSIZ, name, i);
1054                         if (!strncmp(buf, d->name, IFNAMSIZ))
1055                                 set_bit(i, inuse);
1056                 }
1057
1058                 i = find_first_zero_bit(inuse, max_netdevices);
1059                 free_page((unsigned long) inuse);
1060         }
1061
1062         if (buf != name)
1063                 snprintf(buf, IFNAMSIZ, name, i);
1064         if (!__dev_get_by_name(net, buf))
1065                 return i;
1066
1067         /* It is possible to run out of possible slots
1068          * when the name is long and there isn't enough space left
1069          * for the digits, or if all bits are used.
1070          */
1071         return -ENFILE;
1072 }
1073
1074 /**
1075  *      dev_alloc_name - allocate a name for a device
1076  *      @dev: device
1077  *      @name: name format string
1078  *
1079  *      Passed a format string - eg "lt%d" it will try and find a suitable
1080  *      id. It scans list of devices to build up a free map, then chooses
1081  *      the first empty slot. The caller must hold the dev_base or rtnl lock
1082  *      while allocating the name and adding the device in order to avoid
1083  *      duplicates.
1084  *      Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1085  *      Returns the number of the unit assigned or a negative errno code.
1086  */
1087
1088 int dev_alloc_name(struct net_device *dev, const char *name)
1089 {
1090         char buf[IFNAMSIZ];
1091         struct net *net;
1092         int ret;
1093
1094         BUG_ON(!dev_net(dev));
1095         net = dev_net(dev);
1096         ret = __dev_alloc_name(net, name, buf);
1097         if (ret >= 0)
1098                 strlcpy(dev->name, buf, IFNAMSIZ);
1099         return ret;
1100 }
1101 EXPORT_SYMBOL(dev_alloc_name);
1102
1103 static int dev_alloc_name_ns(struct net *net,
1104                              struct net_device *dev,
1105                              const char *name)
1106 {
1107         char buf[IFNAMSIZ];
1108         int ret;
1109
1110         ret = __dev_alloc_name(net, name, buf);
1111         if (ret >= 0)
1112                 strlcpy(dev->name, buf, IFNAMSIZ);
1113         return ret;
1114 }
1115
1116 static int dev_get_valid_name(struct net *net,
1117                               struct net_device *dev,
1118                               const char *name)
1119 {
1120         BUG_ON(!net);
1121
1122         if (!dev_valid_name(name))
1123                 return -EINVAL;
1124
1125         if (strchr(name, '%'))
1126                 return dev_alloc_name_ns(net, dev, name);
1127         else if (__dev_get_by_name(net, name))
1128                 return -EEXIST;
1129         else if (dev->name != name)
1130                 strlcpy(dev->name, name, IFNAMSIZ);
1131
1132         return 0;
1133 }
1134
1135 /**
1136  *      dev_change_name - change name of a device
1137  *      @dev: device
1138  *      @newname: name (or format string) must be at least IFNAMSIZ
1139  *
1140  *      Change name of a device, can pass format strings "eth%d".
1141  *      for wildcarding.
1142  */
1143 int dev_change_name(struct net_device *dev, const char *newname)
1144 {
1145         unsigned char old_assign_type;
1146         char oldname[IFNAMSIZ];
1147         int err = 0;
1148         int ret;
1149         struct net *net;
1150
1151         ASSERT_RTNL();
1152         BUG_ON(!dev_net(dev));
1153
1154         net = dev_net(dev);
1155         if (dev->flags & IFF_UP)
1156                 return -EBUSY;
1157
1158         write_seqcount_begin(&devnet_rename_seq);
1159
1160         if (strncmp(newname, dev->name, IFNAMSIZ) == 0) {
1161                 write_seqcount_end(&devnet_rename_seq);
1162                 return 0;
1163         }
1164
1165         memcpy(oldname, dev->name, IFNAMSIZ);
1166
1167         err = dev_get_valid_name(net, dev, newname);
1168         if (err < 0) {
1169                 write_seqcount_end(&devnet_rename_seq);
1170                 return err;
1171         }
1172
1173         if (oldname[0] && !strchr(oldname, '%'))
1174                 netdev_info(dev, "renamed from %s\n", oldname);
1175
1176         old_assign_type = dev->name_assign_type;
1177         dev->name_assign_type = NET_NAME_RENAMED;
1178
1179 rollback:
1180         ret = device_rename(&dev->dev, dev->name);
1181         if (ret) {
1182                 memcpy(dev->name, oldname, IFNAMSIZ);
1183                 dev->name_assign_type = old_assign_type;
1184                 write_seqcount_end(&devnet_rename_seq);
1185                 return ret;
1186         }
1187
1188         write_seqcount_end(&devnet_rename_seq);
1189
1190         netdev_adjacent_rename_links(dev, oldname);
1191
1192         write_lock_bh(&dev_base_lock);
1193         hlist_del_rcu(&dev->name_hlist);
1194         write_unlock_bh(&dev_base_lock);
1195
1196         synchronize_rcu();
1197
1198         write_lock_bh(&dev_base_lock);
1199         hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
1200         write_unlock_bh(&dev_base_lock);
1201
1202         ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1203         ret = notifier_to_errno(ret);
1204
1205         if (ret) {
1206                 /* err >= 0 after dev_alloc_name() or stores the first errno */
1207                 if (err >= 0) {
1208                         err = ret;
1209                         write_seqcount_begin(&devnet_rename_seq);
1210                         memcpy(dev->name, oldname, IFNAMSIZ);
1211                         memcpy(oldname, newname, IFNAMSIZ);
1212                         dev->name_assign_type = old_assign_type;
1213                         old_assign_type = NET_NAME_RENAMED;
1214                         goto rollback;
1215                 } else {
1216                         pr_err("%s: name change rollback failed: %d\n",
1217                                dev->name, ret);
1218                 }
1219         }
1220
1221         return err;
1222 }
1223
1224 /**
1225  *      dev_set_alias - change ifalias of a device
1226  *      @dev: device
1227  *      @alias: name up to IFALIASZ
1228  *      @len: limit of bytes to copy from info
1229  *
1230  *      Set ifalias for a device,
1231  */
1232 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
1233 {
1234         char *new_ifalias;
1235
1236         ASSERT_RTNL();
1237
1238         if (len >= IFALIASZ)
1239                 return -EINVAL;
1240
1241         if (!len) {
1242                 kfree(dev->ifalias);
1243                 dev->ifalias = NULL;
1244                 return 0;
1245         }
1246
1247         new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
1248         if (!new_ifalias)
1249                 return -ENOMEM;
1250         dev->ifalias = new_ifalias;
1251
1252         strlcpy(dev->ifalias, alias, len+1);
1253         return len;
1254 }
1255
1256
1257 /**
1258  *      netdev_features_change - device changes features
1259  *      @dev: device to cause notification
1260  *
1261  *      Called to indicate a device has changed features.
1262  */
1263 void netdev_features_change(struct net_device *dev)
1264 {
1265         call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1266 }
1267 EXPORT_SYMBOL(netdev_features_change);
1268
1269 /**
1270  *      netdev_state_change - device changes state
1271  *      @dev: device to cause notification
1272  *
1273  *      Called to indicate a device has changed state. This function calls
1274  *      the notifier chains for netdev_chain and sends a NEWLINK message
1275  *      to the routing socket.
1276  */
1277 void netdev_state_change(struct net_device *dev)
1278 {
1279         if (dev->flags & IFF_UP) {
1280                 struct netdev_notifier_change_info change_info;
1281
1282                 change_info.flags_changed = 0;
1283                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
1284                                               &change_info.info);
1285                 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL);
1286         }
1287 }
1288 EXPORT_SYMBOL(netdev_state_change);
1289
1290 /**
1291  *      netdev_notify_peers - notify network peers about existence of @dev
1292  *      @dev: network device
1293  *
1294  * Generate traffic such that interested network peers are aware of
1295  * @dev, such as by generating a gratuitous ARP. This may be used when
1296  * a device wants to inform the rest of the network about some sort of
1297  * reconfiguration such as a failover event or virtual machine
1298  * migration.
1299  */
1300 void netdev_notify_peers(struct net_device *dev)
1301 {
1302         rtnl_lock();
1303         call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1304         rtnl_unlock();
1305 }
1306 EXPORT_SYMBOL(netdev_notify_peers);
1307
1308 static int __dev_open(struct net_device *dev)
1309 {
1310         const struct net_device_ops *ops = dev->netdev_ops;
1311         int ret;
1312
1313         ASSERT_RTNL();
1314
1315         if (!netif_device_present(dev))
1316                 return -ENODEV;
1317
1318         /* Block netpoll from trying to do any rx path servicing.
1319          * If we don't do this there is a chance ndo_poll_controller
1320          * or ndo_poll may be running while we open the device
1321          */
1322         netpoll_poll_disable(dev);
1323
1324         ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
1325         ret = notifier_to_errno(ret);
1326         if (ret)
1327                 return ret;
1328
1329         set_bit(__LINK_STATE_START, &dev->state);
1330
1331         if (ops->ndo_validate_addr)
1332                 ret = ops->ndo_validate_addr(dev);
1333
1334         if (!ret && ops->ndo_open)
1335                 ret = ops->ndo_open(dev);
1336
1337         netpoll_poll_enable(dev);
1338
1339         if (ret)
1340                 clear_bit(__LINK_STATE_START, &dev->state);
1341         else {
1342                 dev->flags |= IFF_UP;
1343                 dev_set_rx_mode(dev);
1344                 dev_activate(dev);
1345                 add_device_randomness(dev->dev_addr, dev->addr_len);
1346         }
1347
1348         return ret;
1349 }
1350
1351 /**
1352  *      dev_open        - prepare an interface for use.
1353  *      @dev:   device to open
1354  *
1355  *      Takes a device from down to up state. The device's private open
1356  *      function is invoked and then the multicast lists are loaded. Finally
1357  *      the device is moved into the up state and a %NETDEV_UP message is
1358  *      sent to the netdev notifier chain.
1359  *
1360  *      Calling this function on an active interface is a nop. On a failure
1361  *      a negative errno code is returned.
1362  */
1363 int dev_open(struct net_device *dev)
1364 {
1365         int ret;
1366
1367         if (dev->flags & IFF_UP)
1368                 return 0;
1369
1370         ret = __dev_open(dev);
1371         if (ret < 0)
1372                 return ret;
1373
1374         rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1375         call_netdevice_notifiers(NETDEV_UP, dev);
1376
1377         return ret;
1378 }
1379 EXPORT_SYMBOL(dev_open);
1380
1381 static int __dev_close_many(struct list_head *head)
1382 {
1383         struct net_device *dev;
1384
1385         ASSERT_RTNL();
1386         might_sleep();
1387
1388         list_for_each_entry(dev, head, close_list) {
1389                 /* Temporarily disable netpoll until the interface is down */
1390                 netpoll_poll_disable(dev);
1391
1392                 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1393
1394                 clear_bit(__LINK_STATE_START, &dev->state);
1395
1396                 /* Synchronize to scheduled poll. We cannot touch poll list, it
1397                  * can be even on different cpu. So just clear netif_running().
1398                  *
1399                  * dev->stop() will invoke napi_disable() on all of it's
1400                  * napi_struct instances on this device.
1401                  */
1402                 smp_mb__after_atomic(); /* Commit netif_running(). */
1403         }
1404
1405         dev_deactivate_many(head);
1406
1407         list_for_each_entry(dev, head, close_list) {
1408                 const struct net_device_ops *ops = dev->netdev_ops;
1409
1410                 /*
1411                  *      Call the device specific close. This cannot fail.
1412                  *      Only if device is UP
1413                  *
1414                  *      We allow it to be called even after a DETACH hot-plug
1415                  *      event.
1416                  */
1417                 if (ops->ndo_stop)
1418                         ops->ndo_stop(dev);
1419
1420                 dev->flags &= ~IFF_UP;
1421                 netpoll_poll_enable(dev);
1422         }
1423
1424         return 0;
1425 }
1426
1427 static int __dev_close(struct net_device *dev)
1428 {
1429         int retval;
1430         LIST_HEAD(single);
1431
1432         list_add(&dev->close_list, &single);
1433         retval = __dev_close_many(&single);
1434         list_del(&single);
1435
1436         return retval;
1437 }
1438
1439 int dev_close_many(struct list_head *head, bool unlink)
1440 {
1441         struct net_device *dev, *tmp;
1442
1443         /* Remove the devices that don't need to be closed */
1444         list_for_each_entry_safe(dev, tmp, head, close_list)
1445                 if (!(dev->flags & IFF_UP))
1446                         list_del_init(&dev->close_list);
1447
1448         __dev_close_many(head);
1449
1450         list_for_each_entry_safe(dev, tmp, head, close_list) {
1451                 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL);
1452                 call_netdevice_notifiers(NETDEV_DOWN, dev);
1453                 if (unlink)
1454                         list_del_init(&dev->close_list);
1455         }
1456
1457         return 0;
1458 }
1459 EXPORT_SYMBOL(dev_close_many);
1460
1461 /**
1462  *      dev_close - shutdown an interface.
1463  *      @dev: device to shutdown
1464  *
1465  *      This function moves an active device into down state. A
1466  *      %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
1467  *      is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
1468  *      chain.
1469  */
1470 int dev_close(struct net_device *dev)
1471 {
1472         if (dev->flags & IFF_UP) {
1473                 LIST_HEAD(single);
1474
1475                 list_add(&dev->close_list, &single);
1476                 dev_close_many(&single, true);
1477                 list_del(&single);
1478         }
1479         return 0;
1480 }
1481 EXPORT_SYMBOL(dev_close);
1482
1483
1484 /**
1485  *      dev_disable_lro - disable Large Receive Offload on a device
1486  *      @dev: device
1487  *
1488  *      Disable Large Receive Offload (LRO) on a net device.  Must be
1489  *      called under RTNL.  This is needed if received packets may be
1490  *      forwarded to another interface.
1491  */
1492 void dev_disable_lro(struct net_device *dev)
1493 {
1494         struct net_device *lower_dev;
1495         struct list_head *iter;
1496
1497         dev->wanted_features &= ~NETIF_F_LRO;
1498         netdev_update_features(dev);
1499
1500         if (unlikely(dev->features & NETIF_F_LRO))
1501                 netdev_WARN(dev, "failed to disable LRO!\n");
1502
1503         netdev_for_each_lower_dev(dev, lower_dev, iter)
1504                 dev_disable_lro(lower_dev);
1505 }
1506 EXPORT_SYMBOL(dev_disable_lro);
1507
1508 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1509                                    struct net_device *dev)
1510 {
1511         struct netdev_notifier_info info;
1512
1513         netdev_notifier_info_init(&info, dev);
1514         return nb->notifier_call(nb, val, &info);
1515 }
1516
1517 static int dev_boot_phase = 1;
1518
1519 /**
1520  *      register_netdevice_notifier - register a network notifier block
1521  *      @nb: notifier
1522  *
1523  *      Register a notifier to be called when network device events occur.
1524  *      The notifier passed is linked into the kernel structures and must
1525  *      not be reused until it has been unregistered. A negative errno code
1526  *      is returned on a failure.
1527  *
1528  *      When registered all registration and up events are replayed
1529  *      to the new notifier to allow device to have a race free
1530  *      view of the network device list.
1531  */
1532
1533 int register_netdevice_notifier(struct notifier_block *nb)
1534 {
1535         struct net_device *dev;
1536         struct net_device *last;
1537         struct net *net;
1538         int err;
1539
1540         rtnl_lock();
1541         err = raw_notifier_chain_register(&netdev_chain, nb);
1542         if (err)
1543                 goto unlock;
1544         if (dev_boot_phase)
1545                 goto unlock;
1546         for_each_net(net) {
1547                 for_each_netdev(net, dev) {
1548                         err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1549                         err = notifier_to_errno(err);
1550                         if (err)
1551                                 goto rollback;
1552
1553                         if (!(dev->flags & IFF_UP))
1554                                 continue;
1555
1556                         call_netdevice_notifier(nb, NETDEV_UP, dev);
1557                 }
1558         }
1559
1560 unlock:
1561         rtnl_unlock();
1562         return err;
1563
1564 rollback:
1565         last = dev;
1566         for_each_net(net) {
1567                 for_each_netdev(net, dev) {
1568                         if (dev == last)
1569                                 goto outroll;
1570
1571                         if (dev->flags & IFF_UP) {
1572                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1573                                                         dev);
1574                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1575                         }
1576                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1577                 }
1578         }
1579
1580 outroll:
1581         raw_notifier_chain_unregister(&netdev_chain, nb);
1582         goto unlock;
1583 }
1584 EXPORT_SYMBOL(register_netdevice_notifier);
1585
1586 /**
1587  *      unregister_netdevice_notifier - unregister a network notifier block
1588  *      @nb: notifier
1589  *
1590  *      Unregister a notifier previously registered by
1591  *      register_netdevice_notifier(). The notifier is unlinked into the
1592  *      kernel structures and may then be reused. A negative errno code
1593  *      is returned on a failure.
1594  *
1595  *      After unregistering unregister and down device events are synthesized
1596  *      for all devices on the device list to the removed notifier to remove
1597  *      the need for special case cleanup code.
1598  */
1599
1600 int unregister_netdevice_notifier(struct notifier_block *nb)
1601 {
1602         struct net_device *dev;
1603         struct net *net;
1604         int err;
1605
1606         rtnl_lock();
1607         err = raw_notifier_chain_unregister(&netdev_chain, nb);
1608         if (err)
1609                 goto unlock;
1610
1611         for_each_net(net) {
1612                 for_each_netdev(net, dev) {
1613                         if (dev->flags & IFF_UP) {
1614                                 call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1615                                                         dev);
1616                                 call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1617                         }
1618                         call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1619                 }
1620         }
1621 unlock:
1622         rtnl_unlock();
1623         return err;
1624 }
1625 EXPORT_SYMBOL(unregister_netdevice_notifier);
1626
1627 /**
1628  *      call_netdevice_notifiers_info - call all network notifier blocks
1629  *      @val: value passed unmodified to notifier function
1630  *      @dev: net_device pointer passed unmodified to notifier function
1631  *      @info: notifier information data
1632  *
1633  *      Call all network notifier blocks.  Parameters and return value
1634  *      are as for raw_notifier_call_chain().
1635  */
1636
1637 static int call_netdevice_notifiers_info(unsigned long val,
1638                                          struct net_device *dev,
1639                                          struct netdev_notifier_info *info)
1640 {
1641         ASSERT_RTNL();
1642         netdev_notifier_info_init(info, dev);
1643         return raw_notifier_call_chain(&netdev_chain, val, info);
1644 }
1645
1646 /**
1647  *      call_netdevice_notifiers - call all network notifier blocks
1648  *      @val: value passed unmodified to notifier function
1649  *      @dev: net_device pointer passed unmodified to notifier function
1650  *
1651  *      Call all network notifier blocks.  Parameters and return value
1652  *      are as for raw_notifier_call_chain().
1653  */
1654
1655 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
1656 {
1657         struct netdev_notifier_info info;
1658
1659         return call_netdevice_notifiers_info(val, dev, &info);
1660 }
1661 EXPORT_SYMBOL(call_netdevice_notifiers);
1662
1663 #ifdef CONFIG_NET_INGRESS
1664 static struct static_key ingress_needed __read_mostly;
1665
1666 void net_inc_ingress_queue(void)
1667 {
1668         static_key_slow_inc(&ingress_needed);
1669 }
1670 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
1671
1672 void net_dec_ingress_queue(void)
1673 {
1674         static_key_slow_dec(&ingress_needed);
1675 }
1676 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
1677 #endif
1678
1679 #ifdef CONFIG_NET_EGRESS
1680 static struct static_key egress_needed __read_mostly;
1681
1682 void net_inc_egress_queue(void)
1683 {
1684         static_key_slow_inc(&egress_needed);
1685 }
1686 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
1687
1688 void net_dec_egress_queue(void)
1689 {
1690         static_key_slow_dec(&egress_needed);
1691 }
1692 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
1693 #endif
1694
1695 static struct static_key netstamp_needed __read_mostly;
1696 #ifdef HAVE_JUMP_LABEL
1697 /* We are not allowed to call static_key_slow_dec() from irq context
1698  * If net_disable_timestamp() is called from irq context, defer the
1699  * static_key_slow_dec() calls.
1700  */
1701 static atomic_t netstamp_needed_deferred;
1702 #endif
1703
1704 void net_enable_timestamp(void)
1705 {
1706 #ifdef HAVE_JUMP_LABEL
1707         int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
1708
1709         if (deferred) {
1710                 while (--deferred)
1711                         static_key_slow_dec(&netstamp_needed);
1712                 return;
1713         }
1714 #endif
1715         static_key_slow_inc(&netstamp_needed);
1716 }
1717 EXPORT_SYMBOL(net_enable_timestamp);
1718
1719 void net_disable_timestamp(void)
1720 {
1721 #ifdef HAVE_JUMP_LABEL
1722         if (in_interrupt()) {
1723                 atomic_inc(&netstamp_needed_deferred);
1724                 return;
1725         }
1726 #endif
1727         static_key_slow_dec(&netstamp_needed);
1728 }
1729 EXPORT_SYMBOL(net_disable_timestamp);
1730
1731 static inline void net_timestamp_set(struct sk_buff *skb)
1732 {
1733         skb->tstamp.tv64 = 0;
1734         if (static_key_false(&netstamp_needed))
1735                 __net_timestamp(skb);
1736 }
1737
1738 #define net_timestamp_check(COND, SKB)                  \
1739         if (static_key_false(&netstamp_needed)) {               \
1740                 if ((COND) && !(SKB)->tstamp.tv64)      \
1741                         __net_timestamp(SKB);           \
1742         }                                               \
1743
1744 bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb)
1745 {
1746         unsigned int len;
1747
1748         if (!(dev->flags & IFF_UP))
1749                 return false;
1750
1751         len = dev->mtu + dev->hard_header_len + VLAN_HLEN;
1752         if (skb->len <= len)
1753                 return true;
1754
1755         /* if TSO is enabled, we don't care about the length as the packet
1756          * could be forwarded without being segmented before
1757          */
1758         if (skb_is_gso(skb))
1759                 return true;
1760
1761         return false;
1762 }
1763 EXPORT_SYMBOL_GPL(is_skb_forwardable);
1764
1765 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1766 {
1767         if (skb_orphan_frags(skb, GFP_ATOMIC) ||
1768             unlikely(!is_skb_forwardable(dev, skb))) {
1769                 atomic_long_inc(&dev->rx_dropped);
1770                 kfree_skb(skb);
1771                 return NET_RX_DROP;
1772         }
1773
1774         skb_scrub_packet(skb, true);
1775         skb->priority = 0;
1776         skb->protocol = eth_type_trans(skb, dev);
1777         skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
1778
1779         return 0;
1780 }
1781 EXPORT_SYMBOL_GPL(__dev_forward_skb);
1782
1783 /**
1784  * dev_forward_skb - loopback an skb to another netif
1785  *
1786  * @dev: destination network device
1787  * @skb: buffer to forward
1788  *
1789  * return values:
1790  *      NET_RX_SUCCESS  (no congestion)
1791  *      NET_RX_DROP     (packet was dropped, but freed)
1792  *
1793  * dev_forward_skb can be used for injecting an skb from the
1794  * start_xmit function of one device into the receive queue
1795  * of another device.
1796  *
1797  * The receiving device may be in another namespace, so
1798  * we have to clear all information in the skb that could
1799  * impact namespace isolation.
1800  */
1801 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
1802 {
1803         return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
1804 }
1805 EXPORT_SYMBOL_GPL(dev_forward_skb);
1806
1807 static inline int deliver_skb(struct sk_buff *skb,
1808                               struct packet_type *pt_prev,
1809                               struct net_device *orig_dev)
1810 {
1811         if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
1812                 return -ENOMEM;
1813         atomic_inc(&skb->users);
1814         return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
1815 }
1816
1817 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
1818                                           struct packet_type **pt,
1819                                           struct net_device *orig_dev,
1820                                           __be16 type,
1821                                           struct list_head *ptype_list)
1822 {
1823         struct packet_type *ptype, *pt_prev = *pt;
1824
1825         list_for_each_entry_rcu(ptype, ptype_list, list) {
1826                 if (ptype->type != type)
1827                         continue;
1828                 if (pt_prev)
1829                         deliver_skb(skb, pt_prev, orig_dev);
1830                 pt_prev = ptype;
1831         }
1832         *pt = pt_prev;
1833 }
1834
1835 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
1836 {
1837         if (!ptype->af_packet_priv || !skb->sk)
1838                 return false;
1839
1840         if (ptype->id_match)
1841                 return ptype->id_match(ptype, skb->sk);
1842         else if ((struct sock *)ptype->af_packet_priv == skb->sk)
1843                 return true;
1844
1845         return false;
1846 }
1847
1848 /*
1849  *      Support routine. Sends outgoing frames to any network
1850  *      taps currently in use.
1851  */
1852
1853 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
1854 {
1855         struct packet_type *ptype;
1856         struct sk_buff *skb2 = NULL;
1857         struct packet_type *pt_prev = NULL;
1858         struct list_head *ptype_list = &ptype_all;
1859
1860         rcu_read_lock();
1861 again:
1862         list_for_each_entry_rcu(ptype, ptype_list, list) {
1863                 /* Never send packets back to the socket
1864                  * they originated from - MvS (miquels@drinkel.ow.org)
1865                  */
1866                 if (skb_loop_sk(ptype, skb))
1867                         continue;
1868
1869                 if (pt_prev) {
1870                         deliver_skb(skb2, pt_prev, skb->dev);
1871                         pt_prev = ptype;
1872                         continue;
1873                 }
1874
1875                 /* need to clone skb, done only once */
1876                 skb2 = skb_clone(skb, GFP_ATOMIC);
1877                 if (!skb2)
1878                         goto out_unlock;
1879
1880                 net_timestamp_set(skb2);
1881
1882                 /* skb->nh should be correctly
1883                  * set by sender, so that the second statement is
1884                  * just protection against buggy protocols.
1885                  */
1886                 skb_reset_mac_header(skb2);
1887
1888                 if (skb_network_header(skb2) < skb2->data ||
1889                     skb_network_header(skb2) > skb_tail_pointer(skb2)) {
1890                         net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
1891                                              ntohs(skb2->protocol),
1892                                              dev->name);
1893                         skb_reset_network_header(skb2);
1894                 }
1895
1896                 skb2->transport_header = skb2->network_header;
1897                 skb2->pkt_type = PACKET_OUTGOING;
1898                 pt_prev = ptype;
1899         }
1900
1901         if (ptype_list == &ptype_all) {
1902                 ptype_list = &dev->ptype_all;
1903                 goto again;
1904         }
1905 out_unlock:
1906         if (pt_prev)
1907                 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
1908         rcu_read_unlock();
1909 }
1910
1911 /**
1912  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
1913  * @dev: Network device
1914  * @txq: number of queues available
1915  *
1916  * If real_num_tx_queues is changed the tc mappings may no longer be
1917  * valid. To resolve this verify the tc mapping remains valid and if
1918  * not NULL the mapping. With no priorities mapping to this
1919  * offset/count pair it will no longer be used. In the worst case TC0
1920  * is invalid nothing can be done so disable priority mappings. If is
1921  * expected that drivers will fix this mapping if they can before
1922  * calling netif_set_real_num_tx_queues.
1923  */
1924 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
1925 {
1926         int i;
1927         struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
1928
1929         /* If TC0 is invalidated disable TC mapping */
1930         if (tc->offset + tc->count > txq) {
1931                 pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
1932                 dev->num_tc = 0;
1933                 return;
1934         }
1935
1936         /* Invalidated prio to tc mappings set to TC0 */
1937         for (i = 1; i < TC_BITMASK + 1; i++) {
1938                 int q = netdev_get_prio_tc_map(dev, i);
1939
1940                 tc = &dev->tc_to_txq[q];
1941                 if (tc->offset + tc->count > txq) {
1942                         pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
1943                                 i, q);
1944                         netdev_set_prio_tc_map(dev, i, 0);
1945                 }
1946         }
1947 }
1948
1949 #ifdef CONFIG_XPS
1950 static DEFINE_MUTEX(xps_map_mutex);
1951 #define xmap_dereference(P)             \
1952         rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
1953
1954 static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps,
1955                                         int cpu, u16 index)
1956 {
1957         struct xps_map *map = NULL;
1958         int pos;
1959
1960         if (dev_maps)
1961                 map = xmap_dereference(dev_maps->cpu_map[cpu]);
1962
1963         for (pos = 0; map && pos < map->len; pos++) {
1964                 if (map->queues[pos] == index) {
1965                         if (map->len > 1) {
1966                                 map->queues[pos] = map->queues[--map->len];
1967                         } else {
1968                                 RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL);
1969                                 kfree_rcu(map, rcu);
1970                                 map = NULL;
1971                         }
1972                         break;
1973                 }
1974         }
1975
1976         return map;
1977 }
1978
1979 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
1980 {
1981         struct xps_dev_maps *dev_maps;
1982         int cpu, i;
1983         bool active = false;
1984
1985         mutex_lock(&xps_map_mutex);
1986         dev_maps = xmap_dereference(dev->xps_maps);
1987
1988         if (!dev_maps)
1989                 goto out_no_maps;
1990
1991         for_each_possible_cpu(cpu) {
1992                 for (i = index; i < dev->num_tx_queues; i++) {
1993                         if (!remove_xps_queue(dev_maps, cpu, i))
1994                                 break;
1995                 }
1996                 if (i == dev->num_tx_queues)
1997                         active = true;
1998         }
1999
2000         if (!active) {
2001                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2002                 kfree_rcu(dev_maps, rcu);
2003         }
2004
2005         for (i = index; i < dev->num_tx_queues; i++)
2006                 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
2007                                              NUMA_NO_NODE);
2008
2009 out_no_maps:
2010         mutex_unlock(&xps_map_mutex);
2011 }
2012
2013 static struct xps_map *expand_xps_map(struct xps_map *map,
2014                                       int cpu, u16 index)
2015 {
2016         struct xps_map *new_map;
2017         int alloc_len = XPS_MIN_MAP_ALLOC;
2018         int i, pos;
2019
2020         for (pos = 0; map && pos < map->len; pos++) {
2021                 if (map->queues[pos] != index)
2022                         continue;
2023                 return map;
2024         }
2025
2026         /* Need to add queue to this CPU's existing map */
2027         if (map) {
2028                 if (pos < map->alloc_len)
2029                         return map;
2030
2031                 alloc_len = map->alloc_len * 2;
2032         }
2033
2034         /* Need to allocate new map to store queue on this CPU's map */
2035         new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2036                                cpu_to_node(cpu));
2037         if (!new_map)
2038                 return NULL;
2039
2040         for (i = 0; i < pos; i++)
2041                 new_map->queues[i] = map->queues[i];
2042         new_map->alloc_len = alloc_len;
2043         new_map->len = pos;
2044
2045         return new_map;
2046 }
2047
2048 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
2049                         u16 index)
2050 {
2051         struct xps_dev_maps *dev_maps, *new_dev_maps = NULL;
2052         struct xps_map *map, *new_map;
2053         int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES);
2054         int cpu, numa_node_id = -2;
2055         bool active = false;
2056
2057         mutex_lock(&xps_map_mutex);
2058
2059         dev_maps = xmap_dereference(dev->xps_maps);
2060
2061         /* allocate memory for queue storage */
2062         for_each_online_cpu(cpu) {
2063                 if (!cpumask_test_cpu(cpu, mask))
2064                         continue;
2065
2066                 if (!new_dev_maps)
2067                         new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2068                 if (!new_dev_maps) {
2069                         mutex_unlock(&xps_map_mutex);
2070                         return -ENOMEM;
2071                 }
2072
2073                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2074                                  NULL;
2075
2076                 map = expand_xps_map(map, cpu, index);
2077                 if (!map)
2078                         goto error;
2079
2080                 RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2081         }
2082
2083         if (!new_dev_maps)
2084                 goto out_no_new_maps;
2085
2086         for_each_possible_cpu(cpu) {
2087                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) {
2088                         /* add queue to CPU maps */
2089                         int pos = 0;
2090
2091                         map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2092                         while ((pos < map->len) && (map->queues[pos] != index))
2093                                 pos++;
2094
2095                         if (pos == map->len)
2096                                 map->queues[map->len++] = index;
2097 #ifdef CONFIG_NUMA
2098                         if (numa_node_id == -2)
2099                                 numa_node_id = cpu_to_node(cpu);
2100                         else if (numa_node_id != cpu_to_node(cpu))
2101                                 numa_node_id = -1;
2102 #endif
2103                 } else if (dev_maps) {
2104                         /* fill in the new device map from the old device map */
2105                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2106                         RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map);
2107                 }
2108
2109         }
2110
2111         rcu_assign_pointer(dev->xps_maps, new_dev_maps);
2112
2113         /* Cleanup old maps */
2114         if (dev_maps) {
2115                 for_each_possible_cpu(cpu) {
2116                         new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2117                         map = xmap_dereference(dev_maps->cpu_map[cpu]);
2118                         if (map && map != new_map)
2119                                 kfree_rcu(map, rcu);
2120                 }
2121
2122                 kfree_rcu(dev_maps, rcu);
2123         }
2124
2125         dev_maps = new_dev_maps;
2126         active = true;
2127
2128 out_no_new_maps:
2129         /* update Tx queue numa node */
2130         netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
2131                                      (numa_node_id >= 0) ? numa_node_id :
2132                                      NUMA_NO_NODE);
2133
2134         if (!dev_maps)
2135                 goto out_no_maps;
2136
2137         /* removes queue from unused CPUs */
2138         for_each_possible_cpu(cpu) {
2139                 if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu))
2140                         continue;
2141
2142                 if (remove_xps_queue(dev_maps, cpu, index))
2143                         active = true;
2144         }
2145
2146         /* free map if not active */
2147         if (!active) {
2148                 RCU_INIT_POINTER(dev->xps_maps, NULL);
2149                 kfree_rcu(dev_maps, rcu);
2150         }
2151
2152 out_no_maps:
2153         mutex_unlock(&xps_map_mutex);
2154
2155         return 0;
2156 error:
2157         /* remove any maps that we added */
2158         for_each_possible_cpu(cpu) {
2159                 new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]);
2160                 map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) :
2161                                  NULL;
2162                 if (new_map && new_map != map)
2163                         kfree(new_map);
2164         }
2165
2166         mutex_unlock(&xps_map_mutex);
2167
2168         kfree(new_dev_maps);
2169         return -ENOMEM;
2170 }
2171 EXPORT_SYMBOL(netif_set_xps_queue);
2172
2173 #endif
2174 /*
2175  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
2176  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
2177  */
2178 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
2179 {
2180         int rc;
2181
2182         if (txq < 1 || txq > dev->num_tx_queues)
2183                 return -EINVAL;
2184
2185         if (dev->reg_state == NETREG_REGISTERED ||
2186             dev->reg_state == NETREG_UNREGISTERING) {
2187                 ASSERT_RTNL();
2188
2189                 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
2190                                                   txq);
2191                 if (rc)
2192                         return rc;
2193
2194                 if (dev->num_tc)
2195                         netif_setup_tc(dev, txq);
2196
2197                 if (txq < dev->real_num_tx_queues) {
2198                         qdisc_reset_all_tx_gt(dev, txq);
2199 #ifdef CONFIG_XPS
2200                         netif_reset_xps_queues_gt(dev, txq);
2201 #endif
2202                 }
2203         }
2204
2205         dev->real_num_tx_queues = txq;
2206         return 0;
2207 }
2208 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
2209
2210 #ifdef CONFIG_SYSFS
2211 /**
2212  *      netif_set_real_num_rx_queues - set actual number of RX queues used
2213  *      @dev: Network device
2214  *      @rxq: Actual number of RX queues
2215  *
2216  *      This must be called either with the rtnl_lock held or before
2217  *      registration of the net device.  Returns 0 on success, or a
2218  *      negative error code.  If called before registration, it always
2219  *      succeeds.
2220  */
2221 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
2222 {
2223         int rc;
2224
2225         if (rxq < 1 || rxq > dev->num_rx_queues)
2226                 return -EINVAL;
2227
2228         if (dev->reg_state == NETREG_REGISTERED) {
2229                 ASSERT_RTNL();
2230
2231                 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
2232                                                   rxq);
2233                 if (rc)
2234                         return rc;
2235         }
2236
2237         dev->real_num_rx_queues = rxq;
2238         return 0;
2239 }
2240 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
2241 #endif
2242
2243 /**
2244  * netif_get_num_default_rss_queues - default number of RSS queues
2245  *
2246  * This routine should set an upper limit on the number of RSS queues
2247  * used by default by multiqueue devices.
2248  */
2249 int netif_get_num_default_rss_queues(void)
2250 {
2251         return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus());
2252 }
2253 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
2254
2255 static inline void __netif_reschedule(struct Qdisc *q)
2256 {
2257         struct softnet_data *sd;
2258         unsigned long flags;
2259
2260         local_irq_save(flags);
2261         sd = this_cpu_ptr(&softnet_data);
2262         q->next_sched = NULL;
2263         *sd->output_queue_tailp = q;
2264         sd->output_queue_tailp = &q->next_sched;
2265         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2266         local_irq_restore(flags);
2267 }
2268
2269 void __netif_schedule(struct Qdisc *q)
2270 {
2271         if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
2272                 __netif_reschedule(q);
2273 }
2274 EXPORT_SYMBOL(__netif_schedule);
2275
2276 struct dev_kfree_skb_cb {
2277         enum skb_free_reason reason;
2278 };
2279
2280 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
2281 {
2282         return (struct dev_kfree_skb_cb *)skb->cb;
2283 }
2284
2285 void netif_schedule_queue(struct netdev_queue *txq)
2286 {
2287         rcu_read_lock();
2288         if (!(txq->state & QUEUE_STATE_ANY_XOFF)) {
2289                 struct Qdisc *q = rcu_dereference(txq->qdisc);
2290
2291                 __netif_schedule(q);
2292         }
2293         rcu_read_unlock();
2294 }
2295 EXPORT_SYMBOL(netif_schedule_queue);
2296
2297 /**
2298  *      netif_wake_subqueue - allow sending packets on subqueue
2299  *      @dev: network device
2300  *      @queue_index: sub queue index
2301  *
2302  * Resume individual transmit queue of a device with multiple transmit queues.
2303  */
2304 void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
2305 {
2306         struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
2307
2308         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) {
2309                 struct Qdisc *q;
2310
2311                 rcu_read_lock();
2312                 q = rcu_dereference(txq->qdisc);
2313                 __netif_schedule(q);
2314                 rcu_read_unlock();
2315         }
2316 }
2317 EXPORT_SYMBOL(netif_wake_subqueue);
2318
2319 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
2320 {
2321         if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
2322                 struct Qdisc *q;
2323
2324                 rcu_read_lock();
2325                 q = rcu_dereference(dev_queue->qdisc);
2326                 __netif_schedule(q);
2327                 rcu_read_unlock();
2328         }
2329 }
2330 EXPORT_SYMBOL(netif_tx_wake_queue);
2331
2332 void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason)
2333 {
2334         unsigned long flags;
2335
2336         if (likely(atomic_read(&skb->users) == 1)) {
2337                 smp_rmb();
2338                 atomic_set(&skb->users, 0);
2339         } else if (likely(!atomic_dec_and_test(&skb->users))) {
2340                 return;
2341         }
2342         get_kfree_skb_cb(skb)->reason = reason;
2343         local_irq_save(flags);
2344         skb->next = __this_cpu_read(softnet_data.completion_queue);
2345         __this_cpu_write(softnet_data.completion_queue, skb);
2346         raise_softirq_irqoff(NET_TX_SOFTIRQ);
2347         local_irq_restore(flags);
2348 }
2349 EXPORT_SYMBOL(__dev_kfree_skb_irq);
2350
2351 void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason)
2352 {
2353         if (in_irq() || irqs_disabled())
2354                 __dev_kfree_skb_irq(skb, reason);
2355         else
2356                 dev_kfree_skb(skb);
2357 }
2358 EXPORT_SYMBOL(__dev_kfree_skb_any);
2359
2360
2361 /**
2362  * netif_device_detach - mark device as removed
2363  * @dev: network device
2364  *
2365  * Mark device as removed from system and therefore no longer available.
2366  */
2367 void netif_device_detach(struct net_device *dev)
2368 {
2369         if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
2370             netif_running(dev)) {
2371                 netif_tx_stop_all_queues(dev);
2372         }
2373 }
2374 EXPORT_SYMBOL(netif_device_detach);
2375
2376 /**
2377  * netif_device_attach - mark device as attached
2378  * @dev: network device
2379  *
2380  * Mark device as attached from system and restart if needed.
2381  */
2382 void netif_device_attach(struct net_device *dev)
2383 {
2384         if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
2385             netif_running(dev)) {
2386                 netif_tx_wake_all_queues(dev);
2387                 __netdev_watchdog_up(dev);
2388         }
2389 }
2390 EXPORT_SYMBOL(netif_device_attach);
2391
2392 /*
2393  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
2394  * to be used as a distribution range.
2395  */
2396 u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
2397                   unsigned int num_tx_queues)
2398 {
2399         u32 hash;
2400         u16 qoffset = 0;
2401         u16 qcount = num_tx_queues;
2402
2403         if (skb_rx_queue_recorded(skb)) {
2404                 hash = skb_get_rx_queue(skb);
2405                 while (unlikely(hash >= num_tx_queues))
2406                         hash -= num_tx_queues;
2407                 return hash;
2408         }
2409
2410         if (dev->num_tc) {
2411                 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
2412                 qoffset = dev->tc_to_txq[tc].offset;
2413                 qcount = dev->tc_to_txq[tc].count;
2414         }
2415
2416         return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
2417 }
2418 EXPORT_SYMBOL(__skb_tx_hash);
2419
2420 static void skb_warn_bad_offload(const struct sk_buff *skb)
2421 {
2422         static const netdev_features_t null_features = 0;
2423         struct net_device *dev = skb->dev;
2424         const char *name = "";
2425
2426         if (!net_ratelimit())
2427                 return;
2428
2429         if (dev) {
2430                 if (dev->dev.parent)
2431                         name = dev_driver_string(dev->dev.parent);
2432                 else
2433                         name = netdev_name(dev);
2434         }
2435         WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d "
2436              "gso_type=%d ip_summed=%d\n",
2437              name, dev ? &dev->features : &null_features,
2438              skb->sk ? &skb->sk->sk_route_caps : &null_features,
2439              skb->len, skb->data_len, skb_shinfo(skb)->gso_size,
2440              skb_shinfo(skb)->gso_type, skb->ip_summed);
2441 }
2442
2443 /*
2444  * Invalidate hardware checksum when packet is to be mangled, and
2445  * complete checksum manually on outgoing path.
2446  */
2447 int skb_checksum_help(struct sk_buff *skb)
2448 {
2449         __wsum csum;
2450         int ret = 0, offset;
2451
2452         if (skb->ip_summed == CHECKSUM_COMPLETE)
2453                 goto out_set_summed;
2454
2455         if (unlikely(skb_shinfo(skb)->gso_size)) {
2456                 skb_warn_bad_offload(skb);
2457                 return -EINVAL;
2458         }
2459
2460         /* Before computing a checksum, we should make sure no frag could
2461          * be modified by an external entity : checksum could be wrong.
2462          */
2463         if (skb_has_shared_frag(skb)) {
2464                 ret = __skb_linearize(skb);
2465                 if (ret)
2466                         goto out;
2467         }
2468
2469         offset = skb_checksum_start_offset(skb);
2470         BUG_ON(offset >= skb_headlen(skb));
2471         csum = skb_checksum(skb, offset, skb->len - offset, 0);
2472
2473         offset += skb->csum_offset;
2474         BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
2475
2476         if (skb_cloned(skb) &&
2477             !skb_clone_writable(skb, offset + sizeof(__sum16))) {
2478                 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2479                 if (ret)
2480                         goto out;
2481         }
2482
2483         *(__sum16 *)(skb->data + offset) = csum_fold(csum);
2484 out_set_summed:
2485         skb->ip_summed = CHECKSUM_NONE;
2486 out:
2487         return ret;
2488 }
2489 EXPORT_SYMBOL(skb_checksum_help);
2490
2491 /* skb_csum_offload_check - Driver helper function to determine if a device
2492  * with limited checksum offload capabilities is able to offload the checksum
2493  * for a given packet.
2494  *
2495  * Arguments:
2496  *   skb - sk_buff for the packet in question
2497  *   spec - contains the description of what device can offload
2498  *   csum_encapped - returns true if the checksum being offloaded is
2499  *            encpasulated. That is it is checksum for the transport header
2500  *            in the inner headers.
2501  *   checksum_help - when set indicates that helper function should
2502  *            call skb_checksum_help if offload checks fail
2503  *
2504  * Returns:
2505  *   true: Packet has passed the checksum checks and should be offloadable to
2506  *         the device (a driver may still need to check for additional
2507  *         restrictions of its device)
2508  *   false: Checksum is not offloadable. If checksum_help was set then
2509  *         skb_checksum_help was called to resolve checksum for non-GSO
2510  *         packets and when IP protocol is not SCTP
2511  */
2512 bool __skb_csum_offload_chk(struct sk_buff *skb,
2513                             const struct skb_csum_offl_spec *spec,
2514                             bool *csum_encapped,
2515                             bool csum_help)
2516 {
2517         struct iphdr *iph;
2518         struct ipv6hdr *ipv6;
2519         void *nhdr;
2520         int protocol;
2521         u8 ip_proto;
2522
2523         if (skb->protocol == htons(ETH_P_8021Q) ||
2524             skb->protocol == htons(ETH_P_8021AD)) {
2525                 if (!spec->vlan_okay)
2526                         goto need_help;
2527         }
2528
2529         /* We check whether the checksum refers to a transport layer checksum in
2530          * the outermost header or an encapsulated transport layer checksum that
2531          * corresponds to the inner headers of the skb. If the checksum is for
2532          * something else in the packet we need help.
2533          */
2534         if (skb_checksum_start_offset(skb) == skb_transport_offset(skb)) {
2535                 /* Non-encapsulated checksum */
2536                 protocol = eproto_to_ipproto(vlan_get_protocol(skb));
2537                 nhdr = skb_network_header(skb);
2538                 *csum_encapped = false;
2539                 if (spec->no_not_encapped)
2540                         goto need_help;
2541         } else if (skb->encapsulation && spec->encap_okay &&
2542                    skb_checksum_start_offset(skb) ==
2543                    skb_inner_transport_offset(skb)) {
2544                 /* Encapsulated checksum */
2545                 *csum_encapped = true;
2546                 switch (skb->inner_protocol_type) {
2547                 case ENCAP_TYPE_ETHER:
2548                         protocol = eproto_to_ipproto(skb->inner_protocol);
2549                         break;
2550                 case ENCAP_TYPE_IPPROTO:
2551                         protocol = skb->inner_protocol;
2552                         break;
2553                 }
2554                 nhdr = skb_inner_network_header(skb);
2555         } else {
2556                 goto need_help;
2557         }
2558
2559         switch (protocol) {
2560         case IPPROTO_IP:
2561                 if (!spec->ipv4_okay)
2562                         goto need_help;
2563                 iph = nhdr;
2564                 ip_proto = iph->protocol;
2565                 if (iph->ihl != 5 && !spec->ip_options_okay)
2566                         goto need_help;
2567                 break;
2568         case IPPROTO_IPV6:
2569                 if (!spec->ipv6_okay)
2570                         goto need_help;
2571                 if (spec->no_encapped_ipv6 && *csum_encapped)
2572                         goto need_help;
2573                 ipv6 = nhdr;
2574                 nhdr += sizeof(*ipv6);
2575                 ip_proto = ipv6->nexthdr;
2576                 break;
2577         default:
2578                 goto need_help;
2579         }
2580
2581 ip_proto_again:
2582         switch (ip_proto) {
2583         case IPPROTO_TCP:
2584                 if (!spec->tcp_okay ||
2585                     skb->csum_offset != offsetof(struct tcphdr, check))
2586                         goto need_help;
2587                 break;
2588         case IPPROTO_UDP:
2589                 if (!spec->udp_okay ||
2590                     skb->csum_offset != offsetof(struct udphdr, check))
2591                         goto need_help;
2592                 break;
2593         case IPPROTO_SCTP:
2594                 if (!spec->sctp_okay ||
2595                     skb->csum_offset != offsetof(struct sctphdr, checksum))
2596                         goto cant_help;
2597                 break;
2598         case NEXTHDR_HOP:
2599         case NEXTHDR_ROUTING:
2600         case NEXTHDR_DEST: {
2601                 u8 *opthdr = nhdr;
2602
2603                 if (protocol != IPPROTO_IPV6 || !spec->ext_hdrs_okay)
2604                         goto need_help;
2605
2606                 ip_proto = opthdr[0];
2607                 nhdr += (opthdr[1] + 1) << 3;
2608
2609                 goto ip_proto_again;
2610         }
2611         default:
2612                 goto need_help;
2613         }
2614
2615         /* Passed the tests for offloading checksum */
2616         return true;
2617
2618 need_help:
2619         if (csum_help && !skb_shinfo(skb)->gso_size)
2620                 skb_checksum_help(skb);
2621 cant_help:
2622         return false;
2623 }
2624 EXPORT_SYMBOL(__skb_csum_offload_chk);
2625
2626 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
2627 {
2628         __be16 type = skb->protocol;
2629
2630         /* Tunnel gso handlers can set protocol to ethernet. */
2631         if (type == htons(ETH_P_TEB)) {
2632                 struct ethhdr *eth;
2633
2634                 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
2635                         return 0;
2636
2637                 eth = (struct ethhdr *)skb_mac_header(skb);
2638                 type = eth->h_proto;
2639         }
2640
2641         return __vlan_get_protocol(skb, type, depth);
2642 }
2643
2644 /**
2645  *      skb_mac_gso_segment - mac layer segmentation handler.
2646  *      @skb: buffer to segment
2647  *      @features: features for the output path (see dev->features)
2648  */
2649 struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb,
2650                                     netdev_features_t features)
2651 {
2652         struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
2653         struct packet_offload *ptype;
2654         int vlan_depth = skb->mac_len;
2655         __be16 type = skb_network_protocol(skb, &vlan_depth);
2656
2657         if (unlikely(!type))
2658                 return ERR_PTR(-EINVAL);
2659
2660         __skb_pull(skb, vlan_depth);
2661
2662         rcu_read_lock();
2663         list_for_each_entry_rcu(ptype, &offload_base, list) {
2664                 if (ptype->type == type && ptype->callbacks.gso_segment) {
2665                         segs = ptype->callbacks.gso_segment(skb, features);
2666                         break;
2667                 }
2668         }
2669         rcu_read_unlock();
2670
2671         __skb_push(skb, skb->data - skb_mac_header(skb));
2672
2673         return segs;
2674 }
2675 EXPORT_SYMBOL(skb_mac_gso_segment);
2676
2677
2678 /* openvswitch calls this on rx path, so we need a different check.
2679  */
2680 static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path)
2681 {
2682         if (tx_path)
2683                 return skb->ip_summed != CHECKSUM_PARTIAL;
2684         else
2685                 return skb->ip_summed == CHECKSUM_NONE;
2686 }
2687
2688 /**
2689  *      __skb_gso_segment - Perform segmentation on skb.
2690  *      @skb: buffer to segment
2691  *      @features: features for the output path (see dev->features)
2692  *      @tx_path: whether it is called in TX path
2693  *
2694  *      This function segments the given skb and returns a list of segments.
2695  *
2696  *      It may return NULL if the skb requires no segmentation.  This is
2697  *      only possible when GSO is used for verifying header integrity.
2698  *
2699  *      Segmentation preserves SKB_SGO_CB_OFFSET bytes of previous skb cb.
2700  */
2701 struct sk_buff *__skb_gso_segment(struct sk_buff *skb,
2702                                   netdev_features_t features, bool tx_path)
2703 {
2704         if (unlikely(skb_needs_check(skb, tx_path))) {
2705                 int err;
2706
2707                 skb_warn_bad_offload(skb);
2708
2709                 err = skb_cow_head(skb, 0);
2710                 if (err < 0)
2711                         return ERR_PTR(err);
2712         }
2713
2714         BUILD_BUG_ON(SKB_SGO_CB_OFFSET +
2715                      sizeof(*SKB_GSO_CB(skb)) > sizeof(skb->cb));
2716
2717         SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb);
2718         SKB_GSO_CB(skb)->encap_level = 0;
2719
2720         skb_reset_mac_header(skb);
2721         skb_reset_mac_len(skb);
2722
2723         return skb_mac_gso_segment(skb, features);
2724 }
2725 EXPORT_SYMBOL(__skb_gso_segment);
2726
2727 /* Take action when hardware reception checksum errors are detected. */
2728 #ifdef CONFIG_BUG
2729 void netdev_rx_csum_fault(struct net_device *dev)
2730 {
2731         if (net_ratelimit()) {
2732                 pr_err("%s: hw csum failure\n", dev ? dev->name : "<unknown>");
2733                 dump_stack();
2734         }
2735 }
2736 EXPORT_SYMBOL(netdev_rx_csum_fault);
2737 #endif
2738
2739 /* Actually, we should eliminate this check as soon as we know, that:
2740  * 1. IOMMU is present and allows to map all the memory.
2741  * 2. No high memory really exists on this machine.
2742  */
2743
2744 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
2745 {
2746 #ifdef CONFIG_HIGHMEM
2747         int i;
2748         if (!(dev->features & NETIF_F_HIGHDMA)) {
2749                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2750                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2751                         if (PageHighMem(skb_frag_page(frag)))
2752                                 return 1;
2753                 }
2754         }
2755
2756         if (PCI_DMA_BUS_IS_PHYS) {
2757                 struct device *pdev = dev->dev.parent;
2758
2759                 if (!pdev)
2760                         return 0;
2761                 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2762                         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2763                         dma_addr_t addr = page_to_phys(skb_frag_page(frag));
2764                         if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
2765                                 return 1;
2766                 }
2767         }
2768 #endif
2769         return 0;
2770 }
2771
2772 /* If MPLS offload request, verify we are testing hardware MPLS features
2773  * instead of standard features for the netdev.
2774  */
2775 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
2776 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2777                                            netdev_features_t features,
2778                                            __be16 type)
2779 {
2780         if (eth_p_mpls(type))
2781                 features &= skb->dev->mpls_features;
2782
2783         return features;
2784 }
2785 #else
2786 static netdev_features_t net_mpls_features(struct sk_buff *skb,
2787                                            netdev_features_t features,
2788                                            __be16 type)
2789 {
2790         return features;
2791 }
2792 #endif
2793
2794 static netdev_features_t harmonize_features(struct sk_buff *skb,
2795         netdev_features_t features)
2796 {
2797         int tmp;
2798         __be16 type;
2799
2800         type = skb_network_protocol(skb, &tmp);
2801         features = net_mpls_features(skb, features, type);
2802
2803         if (skb->ip_summed != CHECKSUM_NONE &&
2804             !can_checksum_protocol(features, type)) {
2805                 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
2806         } else if (illegal_highdma(skb->dev, skb)) {
2807                 features &= ~NETIF_F_SG;
2808         }
2809
2810         return features;
2811 }
2812
2813 netdev_features_t passthru_features_check(struct sk_buff *skb,
2814                                           struct net_device *dev,
2815                                           netdev_features_t features)
2816 {
2817         return features;
2818 }
2819 EXPORT_SYMBOL(passthru_features_check);
2820
2821 static netdev_features_t dflt_features_check(const struct sk_buff *skb,
2822                                              struct net_device *dev,
2823                                              netdev_features_t features)
2824 {
2825         return vlan_features_check(skb, features);
2826 }
2827
2828 netdev_features_t netif_skb_features(struct sk_buff *skb)
2829 {
2830         struct net_device *dev = skb->dev;
2831         netdev_features_t features = dev->features;
2832         u16 gso_segs = skb_shinfo(skb)->gso_segs;
2833
2834         if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs)
2835                 features &= ~NETIF_F_GSO_MASK;
2836
2837         /* If encapsulation offload request, verify we are testing
2838          * hardware encapsulation features instead of standard
2839          * features for the netdev
2840          */
2841         if (skb->encapsulation)
2842                 features &= dev->hw_enc_features;
2843
2844         if (skb_vlan_tagged(skb))
2845                 features = netdev_intersect_features(features,
2846                                                      dev->vlan_features |
2847                                                      NETIF_F_HW_VLAN_CTAG_TX |
2848                                                      NETIF_F_HW_VLAN_STAG_TX);
2849
2850         if (dev->netdev_ops->ndo_features_check)
2851                 features &= dev->netdev_ops->ndo_features_check(skb, dev,
2852                                                                 features);
2853         else
2854                 features &= dflt_features_check(skb, dev, features);
2855
2856         return harmonize_features(skb, features);
2857 }
2858 EXPORT_SYMBOL(netif_skb_features);
2859
2860 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
2861                     struct netdev_queue *txq, bool more)
2862 {
2863         unsigned int len;
2864         int rc;
2865
2866         if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all))
2867                 dev_queue_xmit_nit(skb, dev);
2868
2869         len = skb->len;
2870         trace_net_dev_start_xmit(skb, dev);
2871         rc = netdev_start_xmit(skb, dev, txq, more);
2872         trace_net_dev_xmit(skb, rc, dev, len);
2873
2874         return rc;
2875 }
2876
2877 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
2878                                     struct netdev_queue *txq, int *ret)
2879 {
2880         struct sk_buff *skb = first;
2881         int rc = NETDEV_TX_OK;
2882
2883         while (skb) {
2884                 struct sk_buff *next = skb->next;
2885
2886                 skb->next = NULL;
2887                 rc = xmit_one(skb, dev, txq, next != NULL);
2888                 if (unlikely(!dev_xmit_complete(rc))) {
2889                         skb->next = next;
2890                         goto out;
2891                 }
2892
2893                 skb = next;
2894                 if (netif_xmit_stopped(txq) && skb) {
2895                         rc = NETDEV_TX_BUSY;
2896                         break;
2897                 }
2898         }
2899
2900 out:
2901         *ret = rc;
2902         return skb;
2903 }
2904
2905 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
2906                                           netdev_features_t features)
2907 {
2908         if (skb_vlan_tag_present(skb) &&
2909             !vlan_hw_offload_capable(features, skb->vlan_proto))
2910                 skb = __vlan_hwaccel_push_inside(skb);
2911         return skb;
2912 }
2913
2914 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev)
2915 {
2916         netdev_features_t features;
2917
2918         if (skb->next)
2919                 return skb;
2920
2921         features = netif_skb_features(skb);
2922         skb = validate_xmit_vlan(skb, features);
2923         if (unlikely(!skb))
2924                 goto out_null;
2925
2926         if (netif_needs_gso(skb, features)) {
2927                 struct sk_buff *segs;
2928
2929                 segs = skb_gso_segment(skb, features);
2930                 if (IS_ERR(segs)) {
2931                         goto out_kfree_skb;
2932                 } else if (segs) {
2933                         consume_skb(skb);
2934                         skb = segs;
2935                 }
2936         } else {
2937                 if (skb_needs_linearize(skb, features) &&
2938                     __skb_linearize(skb))
2939                         goto out_kfree_skb;
2940
2941                 /* If packet is not checksummed and device does not
2942                  * support checksumming for this protocol, complete
2943                  * checksumming here.
2944                  */
2945                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2946                         if (skb->encapsulation)
2947                                 skb_set_inner_transport_header(skb,
2948                                                                skb_checksum_start_offset(skb));
2949                         else
2950                                 skb_set_transport_header(skb,
2951                                                          skb_checksum_start_offset(skb));
2952                         if (!(features & NETIF_F_CSUM_MASK) &&
2953                             skb_checksum_help(skb))
2954                                 goto out_kfree_skb;
2955                 }
2956         }
2957
2958         return skb;
2959
2960 out_kfree_skb:
2961         kfree_skb(skb);
2962 out_null:
2963         return NULL;
2964 }
2965
2966 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev)
2967 {
2968         struct sk_buff *next, *head = NULL, *tail;
2969
2970         for (; skb != NULL; skb = next) {
2971                 next = skb->next;
2972                 skb->next = NULL;
2973
2974                 /* in case skb wont be segmented, point to itself */
2975                 skb->prev = skb;
2976
2977                 skb = validate_xmit_skb(skb, dev);
2978                 if (!skb)
2979                         continue;
2980
2981                 if (!head)
2982                         head = skb;
2983                 else
2984                         tail->next = skb;
2985                 /* If skb was segmented, skb->prev points to
2986                  * the last segment. If not, it still contains skb.
2987                  */
2988                 tail = skb->prev;
2989         }
2990         return head;
2991 }
2992
2993 static void qdisc_pkt_len_init(struct sk_buff *skb)
2994 {
2995         const struct skb_shared_info *shinfo = skb_shinfo(skb);
2996
2997         qdisc_skb_cb(skb)->pkt_len = skb->len;
2998
2999         /* To get more precise estimation of bytes sent on wire,
3000          * we add to pkt_len the headers size of all segments
3001          */
3002         if (shinfo->gso_size)  {
3003                 unsigned int hdr_len;
3004                 u16 gso_segs = shinfo->gso_segs;
3005
3006                 /* mac layer + network layer */
3007                 hdr_len = skb_transport_header(skb) - skb_mac_header(skb);
3008
3009                 /* + transport layer */
3010                 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
3011                         hdr_len += tcp_hdrlen(skb);
3012                 else
3013                         hdr_len += sizeof(struct udphdr);
3014
3015                 if (shinfo->gso_type & SKB_GSO_DODGY)
3016                         gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
3017                                                 shinfo->gso_size);
3018
3019                 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
3020         }
3021 }
3022
3023 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
3024                                  struct net_device *dev,
3025                                  struct netdev_queue *txq)
3026 {
3027         spinlock_t *root_lock = qdisc_lock(q);
3028         bool contended;
3029         int rc;
3030
3031         qdisc_calculate_pkt_len(skb, q);
3032         /*
3033          * Heuristic to force contended enqueues to serialize on a
3034          * separate lock before trying to get qdisc main lock.
3035          * This permits __QDISC___STATE_RUNNING owner to get the lock more
3036          * often and dequeue packets faster.
3037          */
3038         contended = qdisc_is_running(q);
3039         if (unlikely(contended))
3040                 spin_lock(&q->busylock);
3041
3042         spin_lock(root_lock);
3043         if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
3044                 kfree_skb(skb);
3045                 rc = NET_XMIT_DROP;
3046         } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
3047                    qdisc_run_begin(q)) {
3048                 /*
3049                  * This is a work-conserving queue; there are no old skbs
3050                  * waiting to be sent out; and the qdisc is not running -
3051                  * xmit the skb directly.
3052                  */
3053
3054                 qdisc_bstats_update(q, skb);
3055
3056                 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) {
3057                         if (unlikely(contended)) {
3058                                 spin_unlock(&q->busylock);
3059                                 contended = false;
3060                         }
3061                         __qdisc_run(q);
3062                 } else
3063                         qdisc_run_end(q);
3064
3065                 rc = NET_XMIT_SUCCESS;
3066         } else {
3067                 rc = q->enqueue(skb, q) & NET_XMIT_MASK;
3068                 if (qdisc_run_begin(q)) {
3069                         if (unlikely(contended)) {
3070                                 spin_unlock(&q->busylock);
3071                                 contended = false;
3072                         }
3073                         __qdisc_run(q);
3074                 }
3075         }
3076         spin_unlock(root_lock);
3077         if (unlikely(contended))
3078                 spin_unlock(&q->busylock);
3079         return rc;
3080 }
3081
3082 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
3083 static void skb_update_prio(struct sk_buff *skb)
3084 {
3085         struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap);
3086
3087         if (!skb->priority && skb->sk && map) {
3088                 unsigned int prioidx =
3089                         sock_cgroup_prioidx(&skb->sk->sk_cgrp_data);
3090
3091                 if (prioidx < map->priomap_len)
3092                         skb->priority = map->priomap[prioidx];
3093         }
3094 }
3095 #else
3096 #define skb_update_prio(skb)
3097 #endif
3098
3099 DEFINE_PER_CPU(int, xmit_recursion);
3100 EXPORT_SYMBOL(xmit_recursion);
3101
3102 #define RECURSION_LIMIT 10
3103
3104 /**
3105  *      dev_loopback_xmit - loop back @skb
3106  *      @net: network namespace this loopback is happening in
3107  *      @sk:  sk needed to be a netfilter okfn
3108  *      @skb: buffer to transmit
3109  */
3110 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
3111 {
3112         skb_reset_mac_header(skb);
3113         __skb_pull(skb, skb_network_offset(skb));
3114         skb->pkt_type = PACKET_LOOPBACK;
3115         skb->ip_summed = CHECKSUM_UNNECESSARY;
3116         WARN_ON(!skb_dst(skb));
3117         skb_dst_force(skb);
3118         netif_rx_ni(skb);
3119         return 0;
3120 }
3121 EXPORT_SYMBOL(dev_loopback_xmit);
3122
3123 #ifdef CONFIG_NET_EGRESS
3124 static struct sk_buff *
3125 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
3126 {
3127         struct tcf_proto *cl = rcu_dereference_bh(dev->egress_cl_list);
3128         struct tcf_result cl_res;
3129
3130         if (!cl)
3131                 return skb;
3132
3133         /* skb->tc_verd and qdisc_skb_cb(skb)->pkt_len were already set
3134          * earlier by the caller.
3135          */
3136         qdisc_bstats_cpu_update(cl->q, skb);
3137
3138         switch (tc_classify(skb, cl, &cl_res, false)) {
3139         case TC_ACT_OK:
3140         case TC_ACT_RECLASSIFY:
3141                 skb->tc_index = TC_H_MIN(cl_res.classid);
3142                 break;
3143         case TC_ACT_SHOT:
3144                 qdisc_qstats_cpu_drop(cl->q);
3145                 *ret = NET_XMIT_DROP;
3146                 goto drop;
3147         case TC_ACT_STOLEN:
3148         case TC_ACT_QUEUED:
3149                 *ret = NET_XMIT_SUCCESS;
3150 drop:
3151                 kfree_skb(skb);
3152                 return NULL;
3153         case TC_ACT_REDIRECT:
3154                 /* No need to push/pop skb's mac_header here on egress! */
3155                 skb_do_redirect(skb);
3156                 *ret = NET_XMIT_SUCCESS;
3157                 return NULL;
3158         default:
3159                 break;
3160         }
3161
3162         return skb;
3163 }
3164 #endif /* CONFIG_NET_EGRESS */
3165
3166 static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
3167 {
3168 #ifdef CONFIG_XPS
3169         struct xps_dev_maps *dev_maps;
3170         struct xps_map *map;
3171         int queue_index = -1;
3172
3173         rcu_read_lock();
3174         dev_maps = rcu_dereference(dev->xps_maps);
3175         if (dev_maps) {
3176                 map = rcu_dereference(
3177                     dev_maps->cpu_map[skb->sender_cpu - 1]);
3178                 if (map) {
3179                         if (map->len == 1)
3180                                 queue_index = map->queues[0];
3181                         else
3182                                 queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
3183                                                                            map->len)];
3184                         if (unlikely(queue_index >= dev->real_num_tx_queues))
3185                                 queue_index = -1;
3186                 }
3187         }
3188         rcu_read_unlock();
3189
3190         return queue_index;
3191 #else
3192         return -1;
3193 #endif
3194 }
3195
3196 static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
3197 {
3198         struct sock *sk = skb->sk;
3199         int queue_index = sk_tx_queue_get(sk);
3200
3201         if (queue_index < 0 || skb->ooo_okay ||
3202             queue_index >= dev->real_num_tx_queues) {
3203                 int new_index = get_xps_queue(dev, skb);
3204                 if (new_index < 0)
3205                         new_index = skb_tx_hash(dev, skb);
3206
3207                 if (queue_index != new_index && sk &&
3208                     sk_fullsock(sk) &&
3209                     rcu_access_pointer(sk->sk_dst_cache))
3210                         sk_tx_queue_set(sk, new_index);
3211
3212                 queue_index = new_index;
3213         }
3214
3215         return queue_index;
3216 }
3217
3218 struct netdev_queue *netdev_pick_tx(struct net_device *dev,
3219                                     struct sk_buff *skb,
3220                                     void *accel_priv)
3221 {
3222         int queue_index = 0;
3223
3224 #ifdef CONFIG_XPS
3225         u32 sender_cpu = skb->sender_cpu - 1;
3226
3227         if (sender_cpu >= (u32)NR_CPUS)
3228                 skb->sender_cpu = raw_smp_processor_id() + 1;
3229 #endif
3230
3231         if (dev->real_num_tx_queues != 1) {
3232                 const struct net_device_ops *ops = dev->netdev_ops;
3233                 if (ops->ndo_select_queue)
3234                         queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
3235                                                             __netdev_pick_tx);
3236                 else
3237                         queue_index = __netdev_pick_tx(dev, skb);
3238
3239                 if (!accel_priv)
3240                         queue_index = netdev_cap_txqueue(dev, queue_index);
3241         }
3242
3243         skb_set_queue_mapping(skb, queue_index);
3244         return netdev_get_tx_queue(dev, queue_index);
3245 }
3246
3247 /**
3248  *      __dev_queue_xmit - transmit a buffer
3249  *      @skb: buffer to transmit
3250  *      @accel_priv: private data used for L2 forwarding offload
3251  *
3252  *      Queue a buffer for transmission to a network device. The caller must
3253  *      have set the device and priority and built the buffer before calling
3254  *      this function. The function can be called from an interrupt.
3255  *
3256  *      A negative errno code is returned on a failure. A success does not
3257  *      guarantee the frame will be transmitted as it may be dropped due
3258  *      to congestion or traffic shaping.
3259  *
3260  * -----------------------------------------------------------------------------------
3261  *      I notice this method can also return errors from the queue disciplines,
3262  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
3263  *      be positive.
3264  *
3265  *      Regardless of the return value, the skb is consumed, so it is currently
3266  *      difficult to retry a send to this method.  (You can bump the ref count
3267  *      before sending to hold a reference for retry if you are careful.)
3268  *
3269  *      When calling this method, interrupts MUST be enabled.  This is because
3270  *      the BH enable code must have IRQs enabled so that it will not deadlock.
3271  *          --BLG
3272  */
3273 static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
3274 {
3275         struct net_device *dev = skb->dev;
3276         struct netdev_queue *txq;
3277         struct Qdisc *q;
3278         int rc = -ENOMEM;
3279
3280         skb_reset_mac_header(skb);
3281
3282         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
3283                 __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
3284
3285         /* Disable soft irqs for various locks below. Also
3286          * stops preemption for RCU.
3287          */
3288         rcu_read_lock_bh();
3289
3290         skb_update_prio(skb);
3291
3292         qdisc_pkt_len_init(skb);
3293 #ifdef CONFIG_NET_CLS_ACT
3294         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
3295 # ifdef CONFIG_NET_EGRESS
3296         if (static_key_false(&egress_needed)) {
3297                 skb = sch_handle_egress(skb, &rc, dev);
3298                 if (!skb)
3299                         goto out;
3300         }
3301 # endif
3302 #endif
3303         /* If device/qdisc don't need skb->dst, release it right now while
3304          * its hot in this cpu cache.
3305          */
3306         if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
3307                 skb_dst_drop(skb);
3308         else
3309                 skb_dst_force(skb);
3310
3311 #ifdef CONFIG_NET_SWITCHDEV
3312         /* Don't forward if offload device already forwarded */
3313         if (skb->offload_fwd_mark &&
3314             skb->offload_fwd_mark == dev->offload_fwd_mark) {
3315                 consume_skb(skb);
3316                 rc = NET_XMIT_SUCCESS;
3317                 goto out;
3318         }
3319 #endif
3320
3321         txq = netdev_pick_tx(dev, skb, accel_priv);
3322         q = rcu_dereference_bh(txq->qdisc);
3323
3324         trace_net_dev_queue(skb);
3325         if (q->enqueue) {
3326                 rc = __dev_xmit_skb(skb, q, dev, txq);
3327                 goto out;
3328         }
3329
3330         /* The device has no queue. Common case for software devices:
3331            loopback, all the sorts of tunnels...
3332
3333            Really, it is unlikely that netif_tx_lock protection is necessary
3334            here.  (f.e. loopback and IP tunnels are clean ignoring statistics
3335            counters.)
3336            However, it is possible, that they rely on protection
3337            made by us here.
3338
3339            Check this and shot the lock. It is not prone from deadlocks.
3340            Either shot noqueue qdisc, it is even simpler 8)
3341          */
3342         if (dev->flags & IFF_UP) {
3343                 int cpu = smp_processor_id(); /* ok because BHs are off */
3344
3345                 if (txq->xmit_lock_owner != cpu) {
3346
3347                         if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
3348                                 goto recursion_alert;
3349
3350                         skb = validate_xmit_skb(skb, dev);
3351                         if (!skb)
3352                                 goto drop;
3353
3354                         HARD_TX_LOCK(dev, txq, cpu);
3355
3356                         if (!netif_xmit_stopped(txq)) {
3357                                 __this_cpu_inc(xmit_recursion);
3358                                 skb = dev_hard_start_xmit(skb, dev, txq, &rc);
3359                                 __this_cpu_dec(xmit_recursion);
3360                                 if (dev_xmit_complete(rc)) {
3361                                         HARD_TX_UNLOCK(dev, txq);
3362                                         goto out;
3363                                 }
3364                         }
3365                         HARD_TX_UNLOCK(dev, txq);
3366                         net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
3367                                              dev->name);
3368                 } else {
3369                         /* Recursion is detected! It is possible,
3370                          * unfortunately
3371                          */
3372 recursion_alert:
3373                         net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n",
3374                                              dev->name);
3375                 }
3376         }
3377
3378         rc = -ENETDOWN;
3379 drop:
3380         rcu_read_unlock_bh();
3381
3382         atomic_long_inc(&dev->tx_dropped);
3383         kfree_skb_list(skb);
3384         return rc;
3385 out:
3386         rcu_read_unlock_bh();
3387         return rc;
3388 }
3389
3390 int dev_queue_xmit(struct sk_buff *skb)
3391 {
3392         return __dev_queue_xmit(skb, NULL);
3393 }
3394 EXPORT_SYMBOL(dev_queue_xmit);
3395
3396 int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv)
3397 {
3398         return __dev_queue_xmit(skb, accel_priv);
3399 }
3400 EXPORT_SYMBOL(dev_queue_xmit_accel);
3401
3402
3403 /*=======================================================================
3404                         Receiver routines
3405   =======================================================================*/
3406
3407 int netdev_max_backlog __read_mostly = 1000;
3408 EXPORT_SYMBOL(netdev_max_backlog);
3409
3410 int netdev_tstamp_prequeue __read_mostly = 1;
3411 int netdev_budget __read_mostly = 300;
3412 int weight_p __read_mostly = 64;            /* old backlog weight */
3413
3414 /* Called with irq disabled */
3415 static inline void ____napi_schedule(struct softnet_data *sd,
3416                                      struct napi_struct *napi)
3417 {
3418         list_add_tail(&napi->poll_list, &sd->poll_list);
3419         __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3420 }
3421
3422 #ifdef CONFIG_RPS
3423
3424 /* One global table that all flow-based protocols share. */
3425 struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly;
3426 EXPORT_SYMBOL(rps_sock_flow_table);
3427 u32 rps_cpu_mask __read_mostly;
3428 EXPORT_SYMBOL(rps_cpu_mask);
3429
3430 struct static_key rps_needed __read_mostly;
3431
3432 static struct rps_dev_flow *
3433 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3434             struct rps_dev_flow *rflow, u16 next_cpu)
3435 {
3436         if (next_cpu < nr_cpu_ids) {
3437 #ifdef CONFIG_RFS_ACCEL
3438                 struct netdev_rx_queue *rxqueue;
3439                 struct rps_dev_flow_table *flow_table;
3440                 struct rps_dev_flow *old_rflow;
3441                 u32 flow_id;
3442                 u16 rxq_index;
3443                 int rc;
3444
3445                 /* Should we steer this flow to a different hardware queue? */
3446                 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
3447                     !(dev->features & NETIF_F_NTUPLE))
3448                         goto out;
3449                 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
3450                 if (rxq_index == skb_get_rx_queue(skb))
3451                         goto out;
3452
3453                 rxqueue = dev->_rx + rxq_index;
3454                 flow_table = rcu_dereference(rxqueue->rps_flow_table);
3455                 if (!flow_table)
3456                         goto out;
3457                 flow_id = skb_get_hash(skb) & flow_table->mask;
3458                 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
3459                                                         rxq_index, flow_id);
3460                 if (rc < 0)
3461                         goto out;
3462                 old_rflow = rflow;
3463                 rflow = &flow_table->flows[flow_id];
3464                 rflow->filter = rc;
3465                 if (old_rflow->filter == rflow->filter)
3466                         old_rflow->filter = RPS_NO_FILTER;
3467         out:
3468 #endif
3469                 rflow->last_qtail =
3470                         per_cpu(softnet_data, next_cpu).input_queue_head;
3471         }
3472
3473         rflow->cpu = next_cpu;
3474         return rflow;
3475 }
3476
3477 /*
3478  * get_rps_cpu is called from netif_receive_skb and returns the target
3479  * CPU from the RPS map of the receiving queue for a given skb.
3480  * rcu_read_lock must be held on entry.
3481  */
3482 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
3483                        struct rps_dev_flow **rflowp)
3484 {
3485         const struct rps_sock_flow_table *sock_flow_table;
3486         struct netdev_rx_queue *rxqueue = dev->_rx;
3487         struct rps_dev_flow_table *flow_table;
3488         struct rps_map *map;
3489         int cpu = -1;
3490         u32 tcpu;
3491         u32 hash;
3492
3493         if (skb_rx_queue_recorded(skb)) {
3494                 u16 index = skb_get_rx_queue(skb);
3495
3496                 if (unlikely(index >= dev->real_num_rx_queues)) {
3497                         WARN_ONCE(dev->real_num_rx_queues > 1,
3498                                   "%s received packet on queue %u, but number "
3499                                   "of RX queues is %u\n",
3500                                   dev->name, index, dev->real_num_rx_queues);
3501                         goto done;
3502                 }
3503                 rxqueue += index;
3504         }
3505
3506         /* Avoid computing hash if RFS/RPS is not active for this rxqueue */
3507
3508         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3509         map = rcu_dereference(rxqueue->rps_map);
3510         if (!flow_table && !map)
3511                 goto done;
3512
3513         skb_reset_network_header(skb);
3514         hash = skb_get_hash(skb);
3515         if (!hash)
3516                 goto done;
3517
3518         sock_flow_table = rcu_dereference(rps_sock_flow_table);
3519         if (flow_table && sock_flow_table) {
3520                 struct rps_dev_flow *rflow;
3521                 u32 next_cpu;
3522                 u32 ident;
3523
3524                 /* First check into global flow table if there is a match */
3525                 ident = sock_flow_table->ents[hash & sock_flow_table->mask];
3526                 if ((ident ^ hash) & ~rps_cpu_mask)
3527                         goto try_rps;
3528
3529                 next_cpu = ident & rps_cpu_mask;
3530
3531                 /* OK, now we know there is a match,
3532                  * we can look at the local (per receive queue) flow table
3533                  */
3534                 rflow = &flow_table->flows[hash & flow_table->mask];
3535                 tcpu = rflow->cpu;
3536
3537                 /*
3538                  * If the desired CPU (where last recvmsg was done) is
3539                  * different from current CPU (one in the rx-queue flow
3540                  * table entry), switch if one of the following holds:
3541                  *   - Current CPU is unset (>= nr_cpu_ids).
3542                  *   - Current CPU is offline.
3543                  *   - The current CPU's queue tail has advanced beyond the
3544                  *     last packet that was enqueued using this table entry.
3545                  *     This guarantees that all previous packets for the flow
3546                  *     have been dequeued, thus preserving in order delivery.
3547                  */
3548                 if (unlikely(tcpu != next_cpu) &&
3549                     (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
3550                      ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
3551                       rflow->last_qtail)) >= 0)) {
3552                         tcpu = next_cpu;
3553                         rflow = set_rps_cpu(dev, skb, rflow, next_cpu);
3554                 }
3555
3556                 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
3557                         *rflowp = rflow;
3558                         cpu = tcpu;
3559                         goto done;
3560                 }
3561         }
3562
3563 try_rps:
3564
3565         if (map) {
3566                 tcpu = map->cpus[reciprocal_scale(hash, map->len)];
3567                 if (cpu_online(tcpu)) {
3568                         cpu = tcpu;
3569                         goto done;
3570                 }
3571         }
3572
3573 done:
3574         return cpu;
3575 }
3576
3577 #ifdef CONFIG_RFS_ACCEL
3578
3579 /**
3580  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
3581  * @dev: Device on which the filter was set
3582  * @rxq_index: RX queue index
3583  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
3584  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
3585  *
3586  * Drivers that implement ndo_rx_flow_steer() should periodically call
3587  * this function for each installed filter and remove the filters for
3588  * which it returns %true.
3589  */
3590 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
3591                          u32 flow_id, u16 filter_id)
3592 {
3593         struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
3594         struct rps_dev_flow_table *flow_table;
3595         struct rps_dev_flow *rflow;
3596         bool expire = true;
3597         unsigned int cpu;
3598
3599         rcu_read_lock();
3600         flow_table = rcu_dereference(rxqueue->rps_flow_table);
3601         if (flow_table && flow_id <= flow_table->mask) {
3602                 rflow = &flow_table->flows[flow_id];
3603                 cpu = ACCESS_ONCE(rflow->cpu);
3604                 if (rflow->filter == filter_id && cpu < nr_cpu_ids &&
3605                     ((int)(per_cpu(softnet_data, cpu).input_queue_head -
3606                            rflow->last_qtail) <
3607                      (int)(10 * flow_table->mask)))
3608                         expire = false;
3609         }
3610         rcu_read_unlock();
3611         return expire;
3612 }
3613 EXPORT_SYMBOL(rps_may_expire_flow);
3614
3615 #endif /* CONFIG_RFS_ACCEL */
3616
3617 /* Called from hardirq (IPI) context */
3618 static void rps_trigger_softirq(void *data)
3619 {
3620         struct softnet_data *sd = data;
3621
3622         ____napi_schedule(sd, &sd->backlog);
3623         sd->received_rps++;
3624 }
3625
3626 #endif /* CONFIG_RPS */
3627
3628 /*
3629  * Check if this softnet_data structure is another cpu one
3630  * If yes, queue it to our IPI list and return 1
3631  * If no, return 0
3632  */
3633 static int rps_ipi_queued(struct softnet_data *sd)
3634 {
3635 #ifdef CONFIG_RPS
3636         struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
3637
3638         if (sd != mysd) {
3639                 sd->rps_ipi_next = mysd->rps_ipi_list;
3640                 mysd->rps_ipi_list = sd;
3641
3642                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
3643                 return 1;
3644         }
3645 #endif /* CONFIG_RPS */
3646         return 0;
3647 }
3648
3649 #ifdef CONFIG_NET_FLOW_LIMIT
3650 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
3651 #endif
3652
3653 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen)
3654 {
3655 #ifdef CONFIG_NET_FLOW_LIMIT
3656         struct sd_flow_limit *fl;
3657         struct softnet_data *sd;
3658         unsigned int old_flow, new_flow;
3659
3660         if (qlen < (netdev_max_backlog >> 1))
3661                 return false;
3662
3663         sd = this_cpu_ptr(&softnet_data);
3664
3665         rcu_read_lock();
3666         fl = rcu_dereference(sd->flow_limit);
3667         if (fl) {
3668                 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1);
3669                 old_flow = fl->history[fl->history_head];
3670                 fl->history[fl->history_head] = new_flow;
3671
3672                 fl->history_head++;
3673                 fl->history_head &= FLOW_LIMIT_HISTORY - 1;
3674
3675                 if (likely(fl->buckets[old_flow]))
3676                         fl->buckets[old_flow]--;
3677
3678                 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
3679                         fl->count++;
3680                         rcu_read_unlock();
3681                         return true;
3682                 }
3683         }
3684         rcu_read_unlock();
3685 #endif
3686         return false;
3687 }
3688
3689 /*
3690  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
3691  * queue (may be a remote CPU queue).
3692  */
3693 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
3694                               unsigned int *qtail)
3695 {
3696         struct softnet_data *sd;
3697         unsigned long flags;
3698         unsigned int qlen;
3699
3700         sd = &per_cpu(softnet_data, cpu);
3701
3702         local_irq_save(flags);
3703
3704         rps_lock(sd);
3705         if (!netif_running(skb->dev))
3706                 goto drop;
3707         qlen = skb_queue_len(&sd->input_pkt_queue);
3708         if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) {
3709                 if (qlen) {
3710 enqueue:
3711                         __skb_queue_tail(&sd->input_pkt_queue, skb);
3712                         input_queue_tail_incr_save(sd, qtail);
3713                         rps_unlock(sd);
3714                         local_irq_restore(flags);
3715                         return NET_RX_SUCCESS;
3716                 }
3717
3718                 /* Schedule NAPI for backlog device
3719                  * We can use non atomic operation since we own the queue lock
3720                  */
3721                 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
3722                         if (!rps_ipi_queued(sd))
3723                                 ____napi_schedule(sd, &sd->backlog);
3724                 }
3725                 goto enqueue;
3726         }
3727
3728 drop:
3729         sd->dropped++;
3730         rps_unlock(sd);
3731
3732         local_irq_restore(flags);
3733
3734         atomic_long_inc(&skb->dev->rx_dropped);
3735         kfree_skb(skb);
3736         return NET_RX_DROP;
3737 }
3738
3739 static int netif_rx_internal(struct sk_buff *skb)
3740 {
3741         int ret;
3742
3743         net_timestamp_check(netdev_tstamp_prequeue, skb);
3744
3745         trace_netif_rx(skb);
3746 #ifdef CONFIG_RPS
3747         if (static_key_false(&rps_needed)) {
3748                 struct rps_dev_flow voidflow, *rflow = &voidflow;
3749                 int cpu;
3750
3751                 preempt_disable();
3752                 rcu_read_lock();
3753
3754                 cpu = get_rps_cpu(skb->dev, skb, &rflow);
3755                 if (cpu < 0)
3756                         cpu = smp_processor_id();
3757
3758                 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
3759
3760                 rcu_read_unlock();
3761                 preempt_enable();
3762         } else
3763 #endif
3764         {
3765                 unsigned int qtail;
3766                 ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
3767                 put_cpu();
3768         }
3769         return ret;
3770 }
3771
3772 /**
3773  *      netif_rx        -       post buffer to the network code
3774  *      @skb: buffer to post
3775  *
3776  *      This function receives a packet from a device driver and queues it for
3777  *      the upper (protocol) levels to process.  It always succeeds. The buffer
3778  *      may be dropped during processing for congestion control or by the
3779  *      protocol layers.
3780  *
3781  *      return values:
3782  *      NET_RX_SUCCESS  (no congestion)
3783  *      NET_RX_DROP     (packet was dropped)
3784  *
3785  */
3786
3787 int netif_rx(struct sk_buff *skb)
3788 {
3789         trace_netif_rx_entry(skb);
3790
3791         return netif_rx_internal(skb);
3792 }
3793 EXPORT_SYMBOL(netif_rx);
3794
3795 int netif_rx_ni(struct sk_buff *skb)
3796 {
3797         int err;
3798
3799         trace_netif_rx_ni_entry(skb);
3800
3801         preempt_disable();
3802         err = netif_rx_internal(skb);
3803         if (local_softirq_pending())
3804                 do_softirq();
3805         preempt_enable();
3806
3807         return err;
3808 }
3809 EXPORT_SYMBOL(netif_rx_ni);
3810
3811 static void net_tx_action(struct softirq_action *h)
3812 {
3813         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
3814
3815         if (sd->completion_queue) {
3816                 struct sk_buff *clist;
3817
3818                 local_irq_disable();
3819                 clist = sd->completion_queue;
3820                 sd->completion_queue = NULL;
3821                 local_irq_enable();
3822
3823                 while (clist) {
3824                         struct sk_buff *skb = clist;
3825                         clist = clist->next;
3826
3827                         WARN_ON(atomic_read(&skb->users));
3828                         if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED))
3829                                 trace_consume_skb(skb);
3830                         else
3831                                 trace_kfree_skb(skb, net_tx_action);
3832
3833                         if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
3834                                 __kfree_skb(skb);
3835                         else
3836                                 __kfree_skb_defer(skb);
3837                 }
3838
3839                 __kfree_skb_flush();
3840         }
3841
3842         if (sd->output_queue) {
3843                 struct Qdisc *head;
3844
3845                 local_irq_disable();
3846                 head = sd->output_queue;
3847                 sd->output_queue = NULL;
3848                 sd->output_queue_tailp = &sd->output_queue;
3849                 local_irq_enable();
3850
3851                 while (head) {
3852                         struct Qdisc *q = head;
3853                         spinlock_t *root_lock;
3854
3855                         head = head->next_sched;
3856
3857                         root_lock = qdisc_lock(q);
3858                         if (spin_trylock(root_lock)) {
3859                                 smp_mb__before_atomic();
3860                                 clear_bit(__QDISC_STATE_SCHED,
3861                                           &q->state);
3862                                 qdisc_run(q);
3863                                 spin_unlock(root_lock);
3864                         } else {
3865                                 if (!test_bit(__QDISC_STATE_DEACTIVATED,
3866                                               &q->state)) {
3867                                         __netif_reschedule(q);
3868                                 } else {
3869                                         smp_mb__before_atomic();
3870                                         clear_bit(__QDISC_STATE_SCHED,
3871                                                   &q->state);
3872                                 }
3873                         }
3874                 }
3875         }
3876 }
3877
3878 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
3879     (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
3880 /* This hook is defined here for ATM LANE */
3881 int (*br_fdb_test_addr_hook)(struct net_device *dev,
3882                              unsigned char *addr) __read_mostly;
3883 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
3884 #endif
3885
3886 static inline struct sk_buff *
3887 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
3888                    struct net_device *orig_dev)
3889 {
3890 #ifdef CONFIG_NET_CLS_ACT
3891         struct tcf_proto *cl = rcu_dereference_bh(skb->dev->ingress_cl_list);
3892         struct tcf_result cl_res;
3893
3894         /* If there's at least one ingress present somewhere (so
3895          * we get here via enabled static key), remaining devices
3896          * that are not configured with an ingress qdisc will bail
3897          * out here.
3898          */
3899         if (!cl)
3900                 return skb;
3901         if (*pt_prev) {
3902                 *ret = deliver_skb(skb, *pt_prev, orig_dev);
3903                 *pt_prev = NULL;
3904         }
3905
3906         qdisc_skb_cb(skb)->pkt_len = skb->len;
3907         skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
3908         qdisc_bstats_cpu_update(cl->q, skb);
3909
3910         switch (tc_classify(skb, cl, &cl_res, false)) {
3911         case TC_ACT_OK:
3912         case TC_ACT_RECLASSIFY:
3913                 skb->tc_index = TC_H_MIN(cl_res.classid);
3914                 break;
3915         case TC_ACT_SHOT:
3916                 qdisc_qstats_cpu_drop(cl->q);
3917         case TC_ACT_STOLEN:
3918         case TC_ACT_QUEUED:
3919                 kfree_skb(skb);
3920                 return NULL;
3921         case TC_ACT_REDIRECT:
3922                 /* skb_mac_header check was done by cls/act_bpf, so
3923                  * we can safely push the L2 header back before
3924                  * redirecting to another netdev
3925                  */
3926                 __skb_push(skb, skb->mac_len);
3927                 skb_do_redirect(skb);
3928                 return NULL;
3929         default:
3930                 break;
3931         }
3932 #endif /* CONFIG_NET_CLS_ACT */
3933         return skb;
3934 }
3935
3936 /**
3937  *      netdev_rx_handler_register - register receive handler
3938  *      @dev: device to register a handler for
3939  *      @rx_handler: receive handler to register
3940  *      @rx_handler_data: data pointer that is used by rx handler
3941  *
3942  *      Register a receive handler for a device. This handler will then be
3943  *      called from __netif_receive_skb. A negative errno code is returned
3944  *      on a failure.
3945  *
3946  *      The caller must hold the rtnl_mutex.
3947  *
3948  *      For a general description of rx_handler, see enum rx_handler_result.
3949  */
3950 int netdev_rx_handler_register(struct net_device *dev,
3951                                rx_handler_func_t *rx_handler,
3952                                void *rx_handler_data)
3953 {
3954         ASSERT_RTNL();
3955
3956         if (dev->rx_handler)
3957                 return -EBUSY;
3958
3959         /* Note: rx_handler_data must be set before rx_handler */
3960         rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
3961         rcu_assign_pointer(dev->rx_handler, rx_handler);
3962
3963         return 0;
3964 }
3965 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
3966
3967 /**
3968  *      netdev_rx_handler_unregister - unregister receive handler
3969  *      @dev: device to unregister a handler from
3970  *
3971  *      Unregister a receive handler from a device.
3972  *
3973  *      The caller must hold the rtnl_mutex.
3974  */
3975 void netdev_rx_handler_unregister(struct net_device *dev)
3976 {
3977
3978         ASSERT_RTNL();
3979         RCU_INIT_POINTER(dev->rx_handler, NULL);
3980         /* a reader seeing a non NULL rx_handler in a rcu_read_lock()
3981          * section has a guarantee to see a non NULL rx_handler_data
3982          * as well.
3983          */
3984         synchronize_net();
3985         RCU_INIT_POINTER(dev->rx_handler_data, NULL);
3986 }
3987 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
3988
3989 /*
3990  * Limit the use of PFMEMALLOC reserves to those protocols that implement
3991  * the special handling of PFMEMALLOC skbs.
3992  */
3993 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
3994 {
3995         switch (skb->protocol) {
3996         case htons(ETH_P_ARP):
3997         case htons(ETH_P_IP):
3998         case htons(ETH_P_IPV6):
3999         case htons(ETH_P_8021Q):
4000         case htons(ETH_P_8021AD):
4001                 return true;
4002         default:
4003                 return false;
4004         }
4005 }
4006
4007 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
4008                              int *ret, struct net_device *orig_dev)
4009 {
4010 #ifdef CONFIG_NETFILTER_INGRESS
4011         if (nf_hook_ingress_active(skb)) {
4012                 if (*pt_prev) {
4013                         *ret = deliver_skb(skb, *pt_prev, orig_dev);
4014                         *pt_prev = NULL;
4015                 }
4016
4017                 return nf_hook_ingress(skb);
4018         }
4019 #endif /* CONFIG_NETFILTER_INGRESS */
4020         return 0;
4021 }
4022
4023 static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
4024 {
4025         struct packet_type *ptype, *pt_prev;
4026         rx_handler_func_t *rx_handler;
4027         struct net_device *orig_dev;
4028         bool deliver_exact = false;
4029         int ret = NET_RX_DROP;
4030         __be16 type;
4031
4032         net_timestamp_check(!netdev_tstamp_prequeue, skb);
4033
4034         trace_netif_receive_skb(skb);
4035
4036         orig_dev = skb->dev;
4037
4038         skb_reset_network_header(skb);
4039         if (!skb_transport_header_was_set(skb))
4040                 skb_reset_transport_header(skb);
4041         skb_reset_mac_len(skb);
4042
4043         pt_prev = NULL;
4044
4045 another_round:
4046         skb->skb_iif = skb->dev->ifindex;
4047
4048         __this_cpu_inc(softnet_data.processed);
4049
4050         if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
4051             skb->protocol == cpu_to_be16(ETH_P_8021AD)) {
4052                 skb = skb_vlan_untag(skb);
4053                 if (unlikely(!skb))
4054                         goto out;
4055         }
4056
4057 #ifdef CONFIG_NET_CLS_ACT
4058         if (skb->tc_verd & TC_NCLS) {
4059                 skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
4060                 goto ncls;
4061         }
4062 #endif
4063
4064         if (pfmemalloc)
4065                 goto skip_taps;
4066
4067         list_for_each_entry_rcu(ptype, &ptype_all, list) {
4068                 if (pt_prev)
4069                         ret = deliver_skb(skb, pt_prev, orig_dev);
4070                 pt_prev = ptype;
4071         }
4072
4073         list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
4074                 if (pt_prev)
4075                         ret = deliver_skb(skb, pt_prev, orig_dev);
4076                 pt_prev = ptype;
4077         }
4078
4079 skip_taps:
4080 #ifdef CONFIG_NET_INGRESS
4081         if (static_key_false(&ingress_needed)) {
4082                 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
4083                 if (!skb)
4084                         goto out;
4085
4086                 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
4087                         goto out;
4088         }
4089 #endif
4090 #ifdef CONFIG_NET_CLS_ACT
4091         skb->tc_verd = 0;
4092 ncls:
4093 #endif
4094         if (pfmemalloc && !skb_pfmemalloc_protocol(skb))
4095                 goto drop;
4096
4097         if (skb_vlan_tag_present(skb)) {
4098                 if (pt_prev) {
4099                         ret = deliver_skb(skb, pt_prev, orig_dev);
4100                         pt_prev = NULL;
4101                 }
4102                 if (vlan_do_receive(&skb))
4103                         goto another_round;
4104                 else if (unlikely(!skb))
4105                         goto out;
4106         }
4107
4108         rx_handler = rcu_dereference(skb->dev->rx_handler);
4109         if (rx_handler) {
4110                 if (pt_prev) {
4111                         ret = deliver_skb(skb, pt_prev, orig_dev);
4112                         pt_prev = NULL;
4113                 }
4114                 switch (rx_handler(&skb)) {
4115                 case RX_HANDLER_CONSUMED:
4116                         ret = NET_RX_SUCCESS;
4117                         goto out;
4118                 case RX_HANDLER_ANOTHER:
4119                         goto another_round;
4120                 case RX_HANDLER_EXACT:
4121                         deliver_exact = true;
4122                 case RX_HANDLER_PASS:
4123                         break;
4124                 default:
4125                         BUG();
4126                 }
4127         }
4128
4129         if (unlikely(skb_vlan_tag_present(skb))) {
4130                 if (skb_vlan_tag_get_id(skb))
4131                         skb->pkt_type = PACKET_OTHERHOST;
4132                 /* Note: we might in the future use prio bits
4133                  * and set skb->priority like in vlan_do_receive()
4134                  * For the time being, just ignore Priority Code Point
4135                  */
4136                 skb->vlan_tci = 0;
4137         }
4138
4139         type = skb->protocol;
4140
4141         /* deliver only exact match when indicated */
4142         if (likely(!deliver_exact)) {
4143                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4144                                        &ptype_base[ntohs(type) &
4145                                                    PTYPE_HASH_MASK]);
4146         }
4147
4148         deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4149                                &orig_dev->ptype_specific);
4150
4151         if (unlikely(skb->dev != orig_dev)) {
4152                 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
4153                                        &skb->dev->ptype_specific);
4154         }
4155
4156         if (pt_prev) {
4157                 if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
4158                         goto drop;
4159                 else
4160                         ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
4161         } else {
4162 drop:
4163                 if (!deliver_exact)
4164                         atomic_long_inc(&skb->dev->rx_dropped);
4165                 else
4166                         atomic_long_inc(&skb->dev->rx_nohandler);
4167                 kfree_skb(skb);
4168                 /* Jamal, now you will not able to escape explaining
4169                  * me how you were going to use this. :-)
4170                  */
4171                 ret = NET_RX_DROP;
4172         }
4173
4174 out:
4175         return ret;
4176 }
4177
4178 static int __netif_receive_skb(struct sk_buff *skb)
4179 {
4180         int ret;
4181
4182         if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
4183                 unsigned long pflags = current->flags;
4184
4185                 /*
4186                  * PFMEMALLOC skbs are special, they should
4187                  * - be delivered to SOCK_MEMALLOC sockets only
4188                  * - stay away from userspace
4189                  * - have bounded memory usage
4190                  *
4191                  * Use PF_MEMALLOC as this saves us from propagating the allocation
4192                  * context down to all allocation sites.
4193                  */
4194                 current->flags |= PF_MEMALLOC;
4195                 ret = __netif_receive_skb_core(skb, true);
4196                 tsk_restore_flags(current, pflags, PF_MEMALLOC);
4197         } else
4198                 ret = __netif_receive_skb_core(skb, false);
4199
4200         return ret;
4201 }
4202
4203 static int netif_receive_skb_internal(struct sk_buff *skb)
4204 {
4205         int ret;
4206
4207         net_timestamp_check(netdev_tstamp_prequeue, skb);
4208
4209         if (skb_defer_rx_timestamp(skb))
4210                 return NET_RX_SUCCESS;
4211
4212         rcu_read_lock();
4213
4214 #ifdef CONFIG_RPS
4215         if (static_key_false(&rps_needed)) {
4216                 struct rps_dev_flow voidflow, *rflow = &voidflow;
4217                 int cpu = get_rps_cpu(skb->dev, skb, &rflow);
4218
4219                 if (cpu >= 0) {
4220                         ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
4221                         rcu_read_unlock();
4222                         return ret;
4223                 }
4224         }
4225 #endif
4226         ret = __netif_receive_skb(skb);
4227         rcu_read_unlock();
4228         return ret;
4229 }
4230
4231 /**
4232  *      netif_receive_skb - process receive buffer from network
4233  *      @skb: buffer to process
4234  *
4235  *      netif_receive_skb() is the main receive data processing function.
4236  *      It always succeeds. The buffer may be dropped during processing
4237  *      for congestion control or by the protocol layers.
4238  *
4239  *      This function may only be called from softirq context and interrupts
4240  *      should be enabled.
4241  *
4242  *      Return values (usually ignored):
4243  *      NET_RX_SUCCESS: no congestion
4244  *      NET_RX_DROP: packet was dropped
4245  */
4246 int netif_receive_skb(struct sk_buff *skb)
4247 {
4248         trace_netif_receive_skb_entry(skb);
4249
4250         return netif_receive_skb_internal(skb);
4251 }
4252 EXPORT_SYMBOL(netif_receive_skb);
4253
4254 /* Network device is going away, flush any packets still pending
4255  * Called with irqs disabled.
4256  */
4257 static void flush_backlog(void *arg)
4258 {
4259         struct net_device *dev = arg;
4260         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4261         struct sk_buff *skb, *tmp;
4262
4263         rps_lock(sd);
4264         skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
4265                 if (skb->dev == dev) {
4266                         __skb_unlink(skb, &sd->input_pkt_queue);
4267                         kfree_skb(skb);
4268                         input_queue_head_incr(sd);
4269                 }
4270         }
4271         rps_unlock(sd);
4272
4273         skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
4274                 if (skb->dev == dev) {
4275                         __skb_unlink(skb, &sd->process_queue);
4276                         kfree_skb(skb);
4277                         input_queue_head_incr(sd);
4278                 }
4279         }
4280 }
4281
4282 static int napi_gro_complete(struct sk_buff *skb)
4283 {
4284         struct packet_offload *ptype;
4285         __be16 type = skb->protocol;
4286         struct list_head *head = &offload_base;
4287         int err = -ENOENT;
4288
4289         BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb));
4290
4291         if (NAPI_GRO_CB(skb)->count == 1) {
4292                 skb_shinfo(skb)->gso_size = 0;
4293                 goto out;
4294         }
4295
4296         rcu_read_lock();
4297         list_for_each_entry_rcu(ptype, head, list) {
4298                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4299                         continue;
4300
4301                 err = ptype->callbacks.gro_complete(skb, 0);
4302                 break;
4303         }
4304         rcu_read_unlock();
4305
4306         if (err) {
4307                 WARN_ON(&ptype->list == head);
4308                 kfree_skb(skb);
4309                 return NET_RX_SUCCESS;
4310         }
4311
4312 out:
4313         return netif_receive_skb_internal(skb);
4314 }
4315
4316 /* napi->gro_list contains packets ordered by age.
4317  * youngest packets at the head of it.
4318  * Complete skbs in reverse order to reduce latencies.
4319  */
4320 void napi_gro_flush(struct napi_struct *napi, bool flush_old)
4321 {
4322         struct sk_buff *skb, *prev = NULL;
4323
4324         /* scan list and build reverse chain */
4325         for (skb = napi->gro_list; skb != NULL; skb = skb->next) {
4326                 skb->prev = prev;
4327                 prev = skb;
4328         }
4329
4330         for (skb = prev; skb; skb = prev) {
4331                 skb->next = NULL;
4332
4333                 if (flush_old && NAPI_GRO_CB(skb)->age == jiffies)
4334                         return;
4335
4336                 prev = skb->prev;
4337                 napi_gro_complete(skb);
4338                 napi->gro_count--;
4339         }
4340
4341         napi->gro_list = NULL;
4342 }
4343 EXPORT_SYMBOL(napi_gro_flush);
4344
4345 static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb)
4346 {
4347         struct sk_buff *p;
4348         unsigned int maclen = skb->dev->hard_header_len;
4349         u32 hash = skb_get_hash_raw(skb);
4350
4351         for (p = napi->gro_list; p; p = p->next) {
4352                 unsigned long diffs;
4353
4354                 NAPI_GRO_CB(p)->flush = 0;
4355
4356                 if (hash != skb_get_hash_raw(p)) {
4357                         NAPI_GRO_CB(p)->same_flow = 0;
4358                         continue;
4359                 }
4360
4361                 diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
4362                 diffs |= p->vlan_tci ^ skb->vlan_tci;
4363                 diffs |= skb_metadata_dst_cmp(p, skb);
4364                 if (maclen == ETH_HLEN)
4365                         diffs |= compare_ether_header(skb_mac_header(p),
4366                                                       skb_mac_header(skb));
4367                 else if (!diffs)
4368                         diffs = memcmp(skb_mac_header(p),
4369                                        skb_mac_header(skb),
4370                                        maclen);
4371                 NAPI_GRO_CB(p)->same_flow = !diffs;
4372         }
4373 }
4374
4375 static void skb_gro_reset_offset(struct sk_buff *skb)
4376 {
4377         const struct skb_shared_info *pinfo = skb_shinfo(skb);
4378         const skb_frag_t *frag0 = &pinfo->frags[0];
4379
4380         NAPI_GRO_CB(skb)->data_offset = 0;
4381         NAPI_GRO_CB(skb)->frag0 = NULL;
4382         NAPI_GRO_CB(skb)->frag0_len = 0;
4383
4384         if (skb_mac_header(skb) == skb_tail_pointer(skb) &&
4385             pinfo->nr_frags &&
4386             !PageHighMem(skb_frag_page(frag0))) {
4387                 NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
4388                 NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0);
4389         }
4390 }
4391
4392 static void gro_pull_from_frag0(struct sk_buff *skb, int grow)
4393 {
4394         struct skb_shared_info *pinfo = skb_shinfo(skb);
4395
4396         BUG_ON(skb->end - skb->tail < grow);
4397
4398         memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
4399
4400         skb->data_len -= grow;
4401         skb->tail += grow;
4402
4403         pinfo->frags[0].page_offset += grow;
4404         skb_frag_size_sub(&pinfo->frags[0], grow);
4405
4406         if (unlikely(!skb_frag_size(&pinfo->frags[0]))) {
4407                 skb_frag_unref(skb, 0);
4408                 memmove(pinfo->frags, pinfo->frags + 1,
4409                         --pinfo->nr_frags * sizeof(pinfo->frags[0]));
4410         }
4411 }
4412
4413 static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4414 {
4415         struct sk_buff **pp = NULL;
4416         struct packet_offload *ptype;
4417         __be16 type = skb->protocol;
4418         struct list_head *head = &offload_base;
4419         int same_flow;
4420         enum gro_result ret;
4421         int grow;
4422
4423         if (!(skb->dev->features & NETIF_F_GRO))
4424                 goto normal;
4425
4426         if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad)
4427                 goto normal;
4428
4429         gro_list_prepare(napi, skb);
4430
4431         rcu_read_lock();
4432         list_for_each_entry_rcu(ptype, head, list) {
4433                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4434                         continue;
4435
4436                 skb_set_network_header(skb, skb_gro_offset(skb));
4437                 skb_reset_mac_len(skb);
4438                 NAPI_GRO_CB(skb)->same_flow = 0;
4439                 NAPI_GRO_CB(skb)->flush = 0;
4440                 NAPI_GRO_CB(skb)->free = 0;
4441                 NAPI_GRO_CB(skb)->encap_mark = 0;
4442                 NAPI_GRO_CB(skb)->is_fou = 0;
4443                 NAPI_GRO_CB(skb)->gro_remcsum_start = 0;
4444
4445                 /* Setup for GRO checksum validation */
4446                 switch (skb->ip_summed) {
4447                 case CHECKSUM_COMPLETE:
4448                         NAPI_GRO_CB(skb)->csum = skb->csum;
4449                         NAPI_GRO_CB(skb)->csum_valid = 1;
4450                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4451                         break;
4452                 case CHECKSUM_UNNECESSARY:
4453                         NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1;
4454                         NAPI_GRO_CB(skb)->csum_valid = 0;
4455                         break;
4456                 default:
4457                         NAPI_GRO_CB(skb)->csum_cnt = 0;
4458                         NAPI_GRO_CB(skb)->csum_valid = 0;
4459                 }
4460
4461                 pp = ptype->callbacks.gro_receive(&napi->gro_list, skb);
4462                 break;
4463         }
4464         rcu_read_unlock();
4465
4466         if (&ptype->list == head)
4467                 goto normal;
4468
4469         same_flow = NAPI_GRO_CB(skb)->same_flow;
4470         ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
4471
4472         if (pp) {
4473                 struct sk_buff *nskb = *pp;
4474
4475                 *pp = nskb->next;
4476                 nskb->next = NULL;
4477                 napi_gro_complete(nskb);
4478                 napi->gro_count--;
4479         }
4480
4481         if (same_flow)
4482                 goto ok;
4483
4484         if (NAPI_GRO_CB(skb)->flush)
4485                 goto normal;
4486
4487         if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) {
4488                 struct sk_buff *nskb = napi->gro_list;
4489
4490                 /* locate the end of the list to select the 'oldest' flow */
4491                 while (nskb->next) {
4492                         pp = &nskb->next;
4493                         nskb = *pp;
4494                 }
4495                 *pp = NULL;
4496                 nskb->next = NULL;
4497                 napi_gro_complete(nskb);
4498         } else {
4499                 napi->gro_count++;
4500         }
4501         NAPI_GRO_CB(skb)->count = 1;
4502         NAPI_GRO_CB(skb)->age = jiffies;
4503         NAPI_GRO_CB(skb)->last = skb;
4504         skb_shinfo(skb)->gso_size = skb_gro_len(skb);
4505         skb->next = napi->gro_list;
4506         napi->gro_list = skb;
4507         ret = GRO_HELD;
4508
4509 pull:
4510         grow = skb_gro_offset(skb) - skb_headlen(skb);
4511         if (grow > 0)
4512                 gro_pull_from_frag0(skb, grow);
4513 ok:
4514         return ret;
4515
4516 normal:
4517         ret = GRO_NORMAL;
4518         goto pull;
4519 }
4520
4521 struct packet_offload *gro_find_receive_by_type(__be16 type)
4522 {
4523         struct list_head *offload_head = &offload_base;
4524         struct packet_offload *ptype;
4525
4526         list_for_each_entry_rcu(ptype, offload_head, list) {
4527                 if (ptype->type != type || !ptype->callbacks.gro_receive)
4528                         continue;
4529                 return ptype;
4530         }
4531         return NULL;
4532 }
4533 EXPORT_SYMBOL(gro_find_receive_by_type);
4534
4535 struct packet_offload *gro_find_complete_by_type(__be16 type)
4536 {
4537         struct list_head *offload_head = &offload_base;
4538         struct packet_offload *ptype;
4539
4540         list_for_each_entry_rcu(ptype, offload_head, list) {
4541                 if (ptype->type != type || !ptype->callbacks.gro_complete)
4542                         continue;
4543                 return ptype;
4544         }
4545         return NULL;
4546 }
4547 EXPORT_SYMBOL(gro_find_complete_by_type);
4548
4549 static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
4550 {
4551         switch (ret) {
4552         case GRO_NORMAL:
4553                 if (netif_receive_skb_internal(skb))
4554                         ret = GRO_DROP;
4555                 break;
4556
4557         case GRO_DROP:
4558                 kfree_skb(skb);
4559                 break;
4560
4561         case GRO_MERGED_FREE:
4562                 if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) {
4563                         skb_dst_drop(skb);
4564                         kmem_cache_free(skbuff_head_cache, skb);
4565                 } else {
4566                         __kfree_skb(skb);
4567                 }
4568                 break;
4569
4570         case GRO_HELD:
4571         case GRO_MERGED:
4572                 break;
4573         }
4574
4575         return ret;
4576 }
4577
4578 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
4579 {
4580         skb_mark_napi_id(skb, napi);
4581         trace_napi_gro_receive_entry(skb);
4582
4583         skb_gro_reset_offset(skb);
4584
4585         return napi_skb_finish(dev_gro_receive(napi, skb), skb);
4586 }
4587 EXPORT_SYMBOL(napi_gro_receive);
4588
4589 static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
4590 {
4591         if (unlikely(skb->pfmemalloc)) {
4592                 consume_skb(skb);
4593                 return;
4594         }
4595         __skb_pull(skb, skb_headlen(skb));
4596         /* restore the reserve we had after netdev_alloc_skb_ip_align() */
4597         skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb));
4598         skb->vlan_tci = 0;
4599         skb->dev = napi->dev;
4600         skb->skb_iif = 0;
4601         skb->encapsulation = 0;
4602         skb_shinfo(skb)->gso_type = 0;
4603         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
4604
4605         napi->skb = skb;
4606 }
4607
4608 struct sk_buff *napi_get_frags(struct napi_struct *napi)
4609 {
4610         struct sk_buff *skb = napi->skb;
4611
4612         if (!skb) {
4613                 skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
4614                 if (skb) {
4615                         napi->skb = skb;
4616                         skb_mark_napi_id(skb, napi);
4617                 }
4618         }
4619         return skb;
4620 }
4621 EXPORT_SYMBOL(napi_get_frags);
4622
4623 static gro_result_t napi_frags_finish(struct napi_struct *napi,
4624                                       struct sk_buff *skb,
4625                                       gro_result_t ret)
4626 {
4627         switch (ret) {
4628         case GRO_NORMAL:
4629         case GRO_HELD:
4630                 __skb_push(skb, ETH_HLEN);
4631                 skb->protocol = eth_type_trans(skb, skb->dev);
4632                 if (ret == GRO_NORMAL && netif_receive_skb_internal(skb))
4633                         ret = GRO_DROP;
4634                 break;
4635
4636         case GRO_DROP:
4637         case GRO_MERGED_FREE:
4638                 napi_reuse_skb(napi, skb);
4639                 break;
4640
4641         case GRO_MERGED:
4642                 break;
4643         }
4644
4645         return ret;
4646 }
4647
4648 /* Upper GRO stack assumes network header starts at gro_offset=0
4649  * Drivers could call both napi_gro_frags() and napi_gro_receive()
4650  * We copy ethernet header into skb->data to have a common layout.
4651  */
4652 static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
4653 {
4654         struct sk_buff *skb = napi->skb;
4655         const struct ethhdr *eth;
4656         unsigned int hlen = sizeof(*eth);
4657
4658         napi->skb = NULL;
4659
4660         skb_reset_mac_header(skb);
4661         skb_gro_reset_offset(skb);
4662
4663         eth = skb_gro_header_fast(skb, 0);
4664         if (unlikely(skb_gro_header_hard(skb, hlen))) {
4665                 eth = skb_gro_header_slow(skb, hlen, 0);
4666                 if (unlikely(!eth)) {
4667                         napi_reuse_skb(napi, skb);
4668                         return NULL;
4669                 }
4670         } else {
4671                 gro_pull_from_frag0(skb, hlen);
4672                 NAPI_GRO_CB(skb)->frag0 += hlen;
4673                 NAPI_GRO_CB(skb)->frag0_len -= hlen;
4674         }
4675         __skb_pull(skb, hlen);
4676
4677         /*
4678          * This works because the only protocols we care about don't require
4679          * special handling.
4680          * We'll fix it up properly in napi_frags_finish()
4681          */
4682         skb->protocol = eth->h_proto;
4683
4684         return skb;
4685 }
4686
4687 gro_result_t napi_gro_frags(struct napi_struct *napi)
4688 {
4689         struct sk_buff *skb = napi_frags_skb(napi);
4690
4691         if (!skb)
4692                 return GRO_DROP;
4693
4694         trace_napi_gro_frags_entry(skb);
4695
4696         return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));
4697 }
4698 EXPORT_SYMBOL(napi_gro_frags);
4699
4700 /* Compute the checksum from gro_offset and return the folded value
4701  * after adding in any pseudo checksum.
4702  */
4703 __sum16 __skb_gro_checksum_complete(struct sk_buff *skb)
4704 {
4705         __wsum wsum;
4706         __sum16 sum;
4707
4708         wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0);
4709
4710         /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */
4711         sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum));
4712         if (likely(!sum)) {
4713                 if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&
4714                     !skb->csum_complete_sw)
4715                         netdev_rx_csum_fault(skb->dev);
4716         }
4717
4718         NAPI_GRO_CB(skb)->csum = wsum;
4719         NAPI_GRO_CB(skb)->csum_valid = 1;
4720
4721         return sum;
4722 }
4723 EXPORT_SYMBOL(__skb_gro_checksum_complete);
4724
4725 /*
4726  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
4727  * Note: called with local irq disabled, but exits with local irq enabled.
4728  */
4729 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4730 {
4731 #ifdef CONFIG_RPS
4732         struct softnet_data *remsd = sd->rps_ipi_list;
4733
4734         if (remsd) {
4735                 sd->rps_ipi_list = NULL;
4736
4737                 local_irq_enable();
4738
4739                 /* Send pending IPI's to kick RPS processing on remote cpus. */
4740                 while (remsd) {
4741                         struct softnet_data *next = remsd->rps_ipi_next;
4742
4743                         if (cpu_online(remsd->cpu))
4744                                 smp_call_function_single_async(remsd->cpu,
4745                                                            &remsd->csd);
4746                         remsd = next;
4747                 }
4748         } else
4749 #endif
4750                 local_irq_enable();
4751 }
4752
4753 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
4754 {
4755 #ifdef CONFIG_RPS
4756         return sd->rps_ipi_list != NULL;
4757 #else
4758         return false;
4759 #endif
4760 }
4761
4762 static int process_backlog(struct napi_struct *napi, int quota)
4763 {
4764         int work = 0;
4765         struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
4766
4767         /* Check if we have pending ipi, its better to send them now,
4768          * not waiting net_rx_action() end.
4769          */
4770         if (sd_has_rps_ipi_waiting(sd)) {
4771                 local_irq_disable();
4772                 net_rps_action_and_irq_enable(sd);
4773         }
4774
4775         napi->weight = weight_p;
4776         local_irq_disable();
4777         while (1) {
4778                 struct sk_buff *skb;
4779
4780                 while ((skb = __skb_dequeue(&sd->process_queue))) {
4781                         rcu_read_lock();
4782                         local_irq_enable();
4783                         __netif_receive_skb(skb);
4784                         rcu_read_unlock();
4785                         local_irq_disable();
4786                         input_queue_head_incr(sd);
4787                         if (++work >= quota) {
4788                                 local_irq_enable();
4789                                 return work;
4790                         }
4791                 }
4792
4793                 rps_lock(sd);
4794                 if (skb_queue_empty(&sd->input_pkt_queue)) {
4795                         /*
4796                          * Inline a custom version of __napi_complete().
4797                          * only current cpu owns and manipulates this napi,
4798                          * and NAPI_STATE_SCHED is the only possible flag set
4799                          * on backlog.
4800                          * We can use a plain write instead of clear_bit(),
4801                          * and we dont need an smp_mb() memory barrier.
4802                          */
4803                         napi->state = 0;
4804                         rps_unlock(sd);
4805
4806                         break;
4807                 }
4808
4809                 skb_queue_splice_tail_init(&sd->input_pkt_queue,
4810                                            &sd->process_queue);
4811                 rps_unlock(sd);
4812         }
4813         local_irq_enable();
4814
4815         return work;
4816 }
4817
4818 /**
4819  * __napi_schedule - schedule for receive
4820  * @n: entry to schedule
4821  *
4822  * The entry's receive function will be scheduled to run.
4823  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
4824  */
4825 void __napi_schedule(struct napi_struct *n)
4826 {
4827         unsigned long flags;
4828
4829         local_irq_save(flags);
4830         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4831         local_irq_restore(flags);
4832 }
4833 EXPORT_SYMBOL(__napi_schedule);
4834
4835 /**
4836  * __napi_schedule_irqoff - schedule for receive
4837  * @n: entry to schedule
4838  *
4839  * Variant of __napi_schedule() assuming hard irqs are masked
4840  */
4841 void __napi_schedule_irqoff(struct napi_struct *n)
4842 {
4843         ____napi_schedule(this_cpu_ptr(&softnet_data), n);
4844 }
4845 EXPORT_SYMBOL(__napi_schedule_irqoff);
4846
4847 void __napi_complete(struct napi_struct *n)
4848 {
4849         BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
4850
4851         list_del_init(&n->poll_list);
4852         smp_mb__before_atomic();
4853         clear_bit(NAPI_STATE_SCHED, &n->state);
4854 }
4855 EXPORT_SYMBOL(__napi_complete);
4856
4857 void napi_complete_done(struct napi_struct *n, int work_done)
4858 {
4859         unsigned long flags;
4860
4861         /*
4862          * don't let napi dequeue from the cpu poll list
4863          * just in case its running on a different cpu
4864          */
4865         if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
4866                 return;
4867
4868         if (n->gro_list) {
4869                 unsigned long timeout = 0;
4870
4871                 if (work_done)
4872                         timeout = n->dev->gro_flush_timeout;
4873
4874                 if (timeout)
4875                         hrtimer_start(&n->timer, ns_to_ktime(timeout),
4876                                       HRTIMER_MODE_REL_PINNED);
4877                 else
4878                         napi_gro_flush(n, false);
4879         }
4880         if (likely(list_empty(&n->poll_list))) {
4881                 WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state));
4882         } else {
4883                 /* If n->poll_list is not empty, we need to mask irqs */
4884                 local_irq_save(flags);
4885                 __napi_complete(n);
4886                 local_irq_restore(flags);
4887         }
4888 }
4889 EXPORT_SYMBOL(napi_complete_done);
4890
4891 /* must be called under rcu_read_lock(), as we dont take a reference */
4892 static struct napi_struct *napi_by_id(unsigned int napi_id)
4893 {
4894         unsigned int hash = napi_id % HASH_SIZE(napi_hash);
4895         struct napi_struct *napi;
4896
4897         hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
4898                 if (napi->napi_id == napi_id)
4899                         return napi;
4900
4901         return NULL;
4902 }
4903
4904 #if defined(CONFIG_NET_RX_BUSY_POLL)
4905 #define BUSY_POLL_BUDGET 8
4906 bool sk_busy_loop(struct sock *sk, int nonblock)
4907 {
4908         unsigned long end_time = !nonblock ? sk_busy_loop_end_time(sk) : 0;
4909         int (*busy_poll)(struct napi_struct *dev);
4910         struct napi_struct *napi;
4911         int rc = false;
4912
4913         rcu_read_lock();
4914
4915         napi = napi_by_id(sk->sk_napi_id);
4916         if (!napi)
4917                 goto out;
4918
4919         /* Note: ndo_busy_poll method is optional in linux-4.5 */
4920         busy_poll = napi->dev->netdev_ops->ndo_busy_poll;
4921
4922         do {
4923                 rc = 0;
4924                 local_bh_disable();
4925                 if (busy_poll) {
4926                         rc = busy_poll(napi);
4927                 } else if (napi_schedule_prep(napi)) {
4928                         void *have = netpoll_poll_lock(napi);
4929
4930                         if (test_bit(NAPI_STATE_SCHED, &napi->state)) {
4931                                 rc = napi->poll(napi, BUSY_POLL_BUDGET);
4932                                 trace_napi_poll(napi);
4933                                 if (rc == BUSY_POLL_BUDGET) {
4934                                         napi_complete_done(napi, rc);
4935                                         napi_schedule(napi);
4936                                 }
4937                         }
4938                         netpoll_poll_unlock(have);
4939                 }
4940                 if (rc > 0)
4941                         NET_ADD_STATS_BH(sock_net(sk),
4942                                          LINUX_MIB_BUSYPOLLRXPACKETS, rc);
4943                 local_bh_enable();
4944
4945                 if (rc == LL_FLUSH_FAILED)
4946                         break; /* permanent failure */
4947
4948                 cpu_relax();
4949         } while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) &&
4950                  !need_resched() && !busy_loop_timeout(end_time));
4951
4952         rc = !skb_queue_empty(&sk->sk_receive_queue);
4953 out:
4954         rcu_read_unlock();
4955         return rc;
4956 }
4957 EXPORT_SYMBOL(sk_busy_loop);
4958
4959 #endif /* CONFIG_NET_RX_BUSY_POLL */
4960
4961 void napi_hash_add(struct napi_struct *napi)
4962 {
4963         if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state) ||
4964             test_and_set_bit(NAPI_STATE_HASHED, &napi->state))
4965                 return;
4966
4967         spin_lock(&napi_hash_lock);
4968
4969         /* 0..NR_CPUS+1 range is reserved for sender_cpu use */
4970         do {
4971                 if (unlikely(++napi_gen_id < NR_CPUS + 1))
4972                         napi_gen_id = NR_CPUS + 1;
4973         } while (napi_by_id(napi_gen_id));
4974         napi->napi_id = napi_gen_id;
4975
4976         hlist_add_head_rcu(&napi->napi_hash_node,
4977                            &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
4978
4979         spin_unlock(&napi_hash_lock);
4980 }
4981 EXPORT_SYMBOL_GPL(napi_hash_add);
4982
4983 /* Warning : caller is responsible to make sure rcu grace period
4984  * is respected before freeing memory containing @napi
4985  */
4986 bool napi_hash_del(struct napi_struct *napi)
4987 {
4988         bool rcu_sync_needed = false;
4989
4990         spin_lock(&napi_hash_lock);
4991
4992         if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) {
4993                 rcu_sync_needed = true;
4994                 hlist_del_rcu(&napi->napi_hash_node);
4995         }
4996         spin_unlock(&napi_hash_lock);
4997         return rcu_sync_needed;
4998 }
4999 EXPORT_SYMBOL_GPL(napi_hash_del);
5000
5001 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
5002 {
5003         struct napi_struct *napi;
5004
5005         napi = container_of(timer, struct napi_struct, timer);
5006         if (napi->gro_list)
5007                 napi_schedule(napi);
5008
5009         return HRTIMER_NORESTART;
5010 }
5011
5012 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
5013                     int (*poll)(struct napi_struct *, int), int weight)
5014 {
5015         INIT_LIST_HEAD(&napi->poll_list);
5016         hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
5017         napi->timer.function = napi_watchdog;
5018         napi->gro_count = 0;
5019         napi->gro_list = NULL;
5020         napi->skb = NULL;
5021         napi->poll = poll;
5022         if (weight > NAPI_POLL_WEIGHT)
5023                 pr_err_once("netif_napi_add() called with weight %d on device %s\n",
5024                             weight, dev->name);
5025         napi->weight = weight;
5026         list_add(&napi->dev_list, &dev->napi_list);
5027         napi->dev = dev;
5028 #ifdef CONFIG_NETPOLL
5029         spin_lock_init(&napi->poll_lock);
5030         napi->poll_owner = -1;
5031 #endif
5032         set_bit(NAPI_STATE_SCHED, &napi->state);
5033         napi_hash_add(napi);
5034 }
5035 EXPORT_SYMBOL(netif_napi_add);
5036
5037 void napi_disable(struct napi_struct *n)
5038 {
5039         might_sleep();
5040         set_bit(NAPI_STATE_DISABLE, &n->state);
5041
5042         while (test_and_set_bit(NAPI_STATE_SCHED, &n->state))
5043                 msleep(1);
5044         while (test_and_set_bit(NAPI_STATE_NPSVC, &n->state))
5045                 msleep(1);
5046
5047         hrtimer_cancel(&n->timer);
5048
5049         clear_bit(NAPI_STATE_DISABLE, &n->state);
5050 }
5051 EXPORT_SYMBOL(napi_disable);
5052
5053 /* Must be called in process context */
5054 void netif_napi_del(struct napi_struct *napi)
5055 {
5056         might_sleep();
5057         if (napi_hash_del(napi))
5058                 synchronize_net();
5059         list_del_init(&napi->dev_list);
5060         napi_free_frags(napi);
5061
5062         kfree_skb_list(napi->gro_list);
5063         napi->gro_list = NULL;
5064         napi->gro_count = 0;
5065 }
5066 EXPORT_SYMBOL(netif_napi_del);
5067
5068 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
5069 {
5070         void *have;
5071         int work, weight;
5072
5073         list_del_init(&n->poll_list);
5074
5075         have = netpoll_poll_lock(n);
5076
5077         weight = n->weight;
5078
5079         /* This NAPI_STATE_SCHED test is for avoiding a race
5080          * with netpoll's poll_napi().  Only the entity which
5081          * obtains the lock and sees NAPI_STATE_SCHED set will
5082          * actually make the ->poll() call.  Therefore we avoid
5083          * accidentally calling ->poll() when NAPI is not scheduled.
5084          */
5085         work = 0;
5086         if (test_bit(NAPI_STATE_SCHED, &n->state)) {
5087                 work = n->poll(n, weight);
5088                 trace_napi_poll(n);
5089         }
5090
5091         WARN_ON_ONCE(work > weight);
5092
5093         if (likely(work < weight))
5094                 goto out_unlock;
5095
5096         /* Drivers must not modify the NAPI state if they
5097          * consume the entire weight.  In such cases this code
5098          * still "owns" the NAPI instance and therefore can
5099          * move the instance around on the list at-will.
5100          */
5101         if (unlikely(napi_disable_pending(n))) {
5102                 napi_complete(n);
5103                 goto out_unlock;
5104         }
5105
5106         if (n->gro_list) {
5107                 /* flush too old packets
5108                  * If HZ < 1000, flush all packets.
5109                  */
5110                 napi_gro_flush(n, HZ >= 1000);
5111         }
5112
5113         /* Some drivers may have called napi_schedule
5114          * prior to exhausting their budget.
5115          */
5116         if (unlikely(!list_empty(&n->poll_list))) {
5117                 pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
5118                              n->dev ? n->dev->name : "backlog");
5119                 goto out_unlock;
5120         }
5121
5122         list_add_tail(&n->poll_list, repoll);
5123
5124 out_unlock:
5125         netpoll_poll_unlock(have);
5126
5127         return work;
5128 }
5129
5130 static void net_rx_action(struct softirq_action *h)
5131 {
5132         struct softnet_data *sd = this_cpu_ptr(&softnet_data);
5133         unsigned long time_limit = jiffies + 2;
5134         int budget = netdev_budget;
5135         LIST_HEAD(list);
5136         LIST_HEAD(repoll);
5137
5138         local_irq_disable();
5139         list_splice_init(&sd->poll_list, &list);
5140         local_irq_enable();
5141
5142         for (;;) {
5143                 struct napi_struct *n;
5144
5145                 if (list_empty(&list)) {
5146                         if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll))
5147                                 return;
5148                         break;
5149                 }
5150
5151                 n = list_first_entry(&list, struct napi_struct, poll_list);
5152                 budget -= napi_poll(n, &repoll);
5153
5154                 /* If softirq window is exhausted then punt.
5155                  * Allow this to run for 2 jiffies since which will allow
5156                  * an average latency of 1.5/HZ.
5157                  */
5158                 if (unlikely(budget <= 0 ||
5159                              time_after_eq(jiffies, time_limit))) {
5160                         sd->time_squeeze++;
5161                         break;
5162                 }
5163         }
5164
5165         __kfree_skb_flush();
5166         local_irq_disable();
5167
5168         list_splice_tail_init(&sd->poll_list, &list);
5169         list_splice_tail(&repoll, &list);
5170         list_splice(&list, &sd->poll_list);
5171         if (!list_empty(&sd->poll_list))
5172                 __raise_softirq_irqoff(NET_RX_SOFTIRQ);
5173
5174         net_rps_action_and_irq_enable(sd);
5175 }
5176
5177 struct netdev_adjacent {
5178         struct net_device *dev;
5179
5180         /* upper master flag, there can only be one master device per list */
5181         bool master;
5182
5183         /* counter for the number of times this device was added to us */
5184         u16 ref_nr;
5185
5186         /* private field for the users */
5187         void *private;
5188
5189         struct list_head list;
5190         struct rcu_head rcu;
5191 };
5192
5193 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
5194                                                  struct list_head *adj_list)
5195 {
5196         struct netdev_adjacent *adj;
5197
5198         list_for_each_entry(adj, adj_list, list) {
5199                 if (adj->dev == adj_dev)
5200                         return adj;
5201         }
5202         return NULL;
5203 }
5204
5205 /**
5206  * netdev_has_upper_dev - Check if device is linked to an upper device
5207  * @dev: device
5208  * @upper_dev: upper device to check
5209  *
5210  * Find out if a device is linked to specified upper device and return true
5211  * in case it is. Note that this checks only immediate upper device,
5212  * not through a complete stack of devices. The caller must hold the RTNL lock.
5213  */
5214 bool netdev_has_upper_dev(struct net_device *dev,
5215                           struct net_device *upper_dev)
5216 {
5217         ASSERT_RTNL();
5218
5219         return __netdev_find_adj(upper_dev, &dev->all_adj_list.upper);
5220 }
5221 EXPORT_SYMBOL(netdev_has_upper_dev);
5222
5223 /**
5224  * netdev_has_any_upper_dev - Check if device is linked to some device
5225  * @dev: device
5226  *
5227  * Find out if a device is linked to an upper device and return true in case
5228  * it is. The caller must hold the RTNL lock.
5229  */
5230 static bool netdev_has_any_upper_dev(struct net_device *dev)
5231 {
5232         ASSERT_RTNL();
5233
5234         return !list_empty(&dev->all_adj_list.upper);
5235 }
5236
5237 /**
5238  * netdev_master_upper_dev_get - Get master upper device
5239  * @dev: device
5240  *
5241  * Find a master upper device and return pointer to it or NULL in case
5242  * it's not there. The caller must hold the RTNL lock.
5243  */
5244 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
5245 {
5246         struct netdev_adjacent *upper;
5247
5248         ASSERT_RTNL();
5249
5250         if (list_empty(&dev->adj_list.upper))
5251                 return NULL;
5252
5253         upper = list_first_entry(&dev->adj_list.upper,
5254                                  struct netdev_adjacent, list);
5255         if (likely(upper->master))
5256                 return upper->dev;
5257         return NULL;
5258 }
5259 EXPORT_SYMBOL(netdev_master_upper_dev_get);
5260
5261 void *netdev_adjacent_get_private(struct list_head *adj_list)
5262 {
5263         struct netdev_adjacent *adj;
5264
5265         adj = list_entry(adj_list, struct netdev_adjacent, list);
5266
5267         return adj->private;
5268 }
5269 EXPORT_SYMBOL(netdev_adjacent_get_private);
5270
5271 /**
5272  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
5273  * @dev: device
5274  * @iter: list_head ** of the current position
5275  *
5276  * Gets the next device from the dev's upper list, starting from iter
5277  * position. The caller must hold RCU read lock.
5278  */
5279 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5280                                                  struct list_head **iter)
5281 {
5282         struct netdev_adjacent *upper;
5283
5284         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5285
5286         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5287
5288         if (&upper->list == &dev->adj_list.upper)
5289                 return NULL;
5290
5291         *iter = &upper->list;
5292
5293         return upper->dev;
5294 }
5295 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
5296
5297 /**
5298  * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list
5299  * @dev: device
5300  * @iter: list_head ** of the current position
5301  *
5302  * Gets the next device from the dev's upper list, starting from iter
5303  * position. The caller must hold RCU read lock.
5304  */
5305 struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev,
5306                                                      struct list_head **iter)
5307 {
5308         struct netdev_adjacent *upper;
5309
5310         WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
5311
5312         upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5313
5314         if (&upper->list == &dev->all_adj_list.upper)
5315                 return NULL;
5316
5317         *iter = &upper->list;
5318
5319         return upper->dev;
5320 }
5321 EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu);
5322
5323 /**
5324  * netdev_lower_get_next_private - Get the next ->private from the
5325  *                                 lower neighbour list
5326  * @dev: device
5327  * @iter: list_head ** of the current position
5328  *
5329  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5330  * list, starting from iter position. The caller must hold either hold the
5331  * RTNL lock or its own locking that guarantees that the neighbour lower
5332  * list will remain unchanged.
5333  */
5334 void *netdev_lower_get_next_private(struct net_device *dev,
5335                                     struct list_head **iter)
5336 {
5337         struct netdev_adjacent *lower;
5338
5339         lower = list_entry(*iter, struct netdev_adjacent, list);
5340
5341         if (&lower->list == &dev->adj_list.lower)
5342                 return NULL;
5343
5344         *iter = lower->list.next;
5345
5346         return lower->private;
5347 }
5348 EXPORT_SYMBOL(netdev_lower_get_next_private);
5349
5350 /**
5351  * netdev_lower_get_next_private_rcu - Get the next ->private from the
5352  *                                     lower neighbour list, RCU
5353  *                                     variant
5354  * @dev: device
5355  * @iter: list_head ** of the current position
5356  *
5357  * Gets the next netdev_adjacent->private from the dev's lower neighbour
5358  * list, starting from iter position. The caller must hold RCU read lock.
5359  */
5360 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5361                                         struct list_head **iter)
5362 {
5363         struct netdev_adjacent *lower;
5364
5365         WARN_ON_ONCE(!rcu_read_lock_held());
5366
5367         lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
5368
5369         if (&lower->list == &dev->adj_list.lower)
5370                 return NULL;
5371
5372         *iter = &lower->list;
5373
5374         return lower->private;
5375 }
5376 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
5377
5378 /**
5379  * netdev_lower_get_next - Get the next device from the lower neighbour
5380  *                         list
5381  * @dev: device
5382  * @iter: list_head ** of the current position
5383  *
5384  * Gets the next netdev_adjacent from the dev's lower neighbour
5385  * list, starting from iter position. The caller must hold RTNL lock or
5386  * its own locking that guarantees that the neighbour lower
5387  * list will remain unchanged.
5388  */
5389 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
5390 {
5391         struct netdev_adjacent *lower;
5392
5393         lower = list_entry(*iter, struct netdev_adjacent, list);
5394
5395         if (&lower->list == &dev->adj_list.lower)
5396                 return NULL;
5397
5398         *iter = lower->list.next;
5399
5400         return lower->dev;
5401 }
5402 EXPORT_SYMBOL(netdev_lower_get_next);
5403
5404 /**
5405  * netdev_lower_get_first_private_rcu - Get the first ->private from the
5406  *                                     lower neighbour list, RCU
5407  *                                     variant
5408  * @dev: device
5409  *
5410  * Gets the first netdev_adjacent->private from the dev's lower neighbour
5411  * list. The caller must hold RCU read lock.
5412  */
5413 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
5414 {
5415         struct netdev_adjacent *lower;
5416
5417         lower = list_first_or_null_rcu(&dev->adj_list.lower,
5418                         struct netdev_adjacent, list);
5419         if (lower)
5420                 return lower->private;
5421         return NULL;
5422 }
5423 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
5424
5425 /**
5426  * netdev_master_upper_dev_get_rcu - Get master upper device
5427  * @dev: device
5428  *
5429  * Find a master upper device and return pointer to it or NULL in case
5430  * it's not there. The caller must hold the RCU read lock.
5431  */
5432 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
5433 {
5434         struct netdev_adjacent *upper;
5435
5436         upper = list_first_or_null_rcu(&dev->adj_list.upper,
5437                                        struct netdev_adjacent, list);
5438         if (upper && likely(upper->master))
5439                 return upper->dev;
5440         return NULL;
5441 }
5442 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
5443
5444 static int netdev_adjacent_sysfs_add(struct net_device *dev,
5445                               struct net_device *adj_dev,
5446                               struct list_head *dev_list)
5447 {
5448         char linkname[IFNAMSIZ+7];
5449         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5450                 "upper_%s" : "lower_%s", adj_dev->name);
5451         return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
5452                                  linkname);
5453 }
5454 static void netdev_adjacent_sysfs_del(struct net_device *dev,
5455                                char *name,
5456                                struct list_head *dev_list)
5457 {
5458         char linkname[IFNAMSIZ+7];
5459         sprintf(linkname, dev_list == &dev->adj_list.upper ?
5460                 "upper_%s" : "lower_%s", name);
5461         sysfs_remove_link(&(dev->dev.kobj), linkname);
5462 }
5463
5464 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
5465                                                  struct net_device *adj_dev,
5466                                                  struct list_head *dev_list)
5467 {
5468         return (dev_list == &dev->adj_list.upper ||
5469                 dev_list == &dev->adj_list.lower) &&
5470                 net_eq(dev_net(dev), dev_net(adj_dev));
5471 }
5472
5473 static int __netdev_adjacent_dev_insert(struct net_device *dev,
5474                                         struct net_device *adj_dev,
5475                                         struct list_head *dev_list,
5476                                         void *private, bool master)
5477 {
5478         struct netdev_adjacent *adj;
5479         int ret;
5480
5481         adj = __netdev_find_adj(adj_dev, dev_list);
5482
5483         if (adj) {
5484                 adj->ref_nr++;
5485                 return 0;
5486         }
5487
5488         adj = kmalloc(sizeof(*adj), GFP_KERNEL);
5489         if (!adj)
5490                 return -ENOMEM;
5491
5492         adj->dev = adj_dev;
5493         adj->master = master;
5494         adj->ref_nr = 1;
5495         adj->private = private;
5496         dev_hold(adj_dev);
5497
5498         pr_debug("dev_hold for %s, because of link added from %s to %s\n",
5499                  adj_dev->name, dev->name, adj_dev->name);
5500
5501         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
5502                 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
5503                 if (ret)
5504                         goto free_adj;
5505         }
5506
5507         /* Ensure that master link is always the first item in list. */
5508         if (master) {
5509                 ret = sysfs_create_link(&(dev->dev.kobj),
5510                                         &(adj_dev->dev.kobj), "master");
5511                 if (ret)
5512                         goto remove_symlinks;
5513
5514                 list_add_rcu(&adj->list, dev_list);
5515         } else {
5516                 list_add_tail_rcu(&adj->list, dev_list);
5517         }
5518
5519         return 0;
5520
5521 remove_symlinks:
5522         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5523                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5524 free_adj:
5525         kfree(adj);
5526         dev_put(adj_dev);
5527
5528         return ret;
5529 }
5530
5531 static void __netdev_adjacent_dev_remove(struct net_device *dev,
5532                                          struct net_device *adj_dev,
5533                                          struct list_head *dev_list)
5534 {
5535         struct netdev_adjacent *adj;
5536
5537         adj = __netdev_find_adj(adj_dev, dev_list);
5538
5539         if (!adj) {
5540                 pr_err("tried to remove device %s from %s\n",
5541                        dev->name, adj_dev->name);
5542                 BUG();
5543         }
5544
5545         if (adj->ref_nr > 1) {
5546                 pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name,
5547                          adj->ref_nr-1);
5548                 adj->ref_nr--;
5549                 return;
5550         }
5551
5552         if (adj->master)
5553                 sysfs_remove_link(&(dev->dev.kobj), "master");
5554
5555         if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
5556                 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
5557
5558         list_del_rcu(&adj->list);
5559         pr_debug("dev_put for %s, because link removed from %s to %s\n",
5560                  adj_dev->name, dev->name, adj_dev->name);
5561         dev_put(adj_dev);
5562         kfree_rcu(adj, rcu);
5563 }
5564
5565 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
5566                                             struct net_device *upper_dev,
5567                                             struct list_head *up_list,
5568                                             struct list_head *down_list,
5569                                             void *private, bool master)
5570 {
5571         int ret;
5572
5573         ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private,
5574                                            master);
5575         if (ret)
5576                 return ret;
5577
5578         ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private,
5579                                            false);
5580         if (ret) {
5581                 __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5582                 return ret;
5583         }
5584
5585         return 0;
5586 }
5587
5588 static int __netdev_adjacent_dev_link(struct net_device *dev,
5589                                       struct net_device *upper_dev)
5590 {
5591         return __netdev_adjacent_dev_link_lists(dev, upper_dev,
5592                                                 &dev->all_adj_list.upper,
5593                                                 &upper_dev->all_adj_list.lower,
5594                                                 NULL, false);
5595 }
5596
5597 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
5598                                                struct net_device *upper_dev,
5599                                                struct list_head *up_list,
5600                                                struct list_head *down_list)
5601 {
5602         __netdev_adjacent_dev_remove(dev, upper_dev, up_list);
5603         __netdev_adjacent_dev_remove(upper_dev, dev, down_list);
5604 }
5605
5606 static void __netdev_adjacent_dev_unlink(struct net_device *dev,
5607                                          struct net_device *upper_dev)
5608 {
5609         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5610                                            &dev->all_adj_list.upper,
5611                                            &upper_dev->all_adj_list.lower);
5612 }
5613
5614 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
5615                                                 struct net_device *upper_dev,
5616                                                 void *private, bool master)
5617 {
5618         int ret = __netdev_adjacent_dev_link(dev, upper_dev);
5619
5620         if (ret)
5621                 return ret;
5622
5623         ret = __netdev_adjacent_dev_link_lists(dev, upper_dev,
5624                                                &dev->adj_list.upper,
5625                                                &upper_dev->adj_list.lower,
5626                                                private, master);
5627         if (ret) {
5628                 __netdev_adjacent_dev_unlink(dev, upper_dev);
5629                 return ret;
5630         }
5631
5632         return 0;
5633 }
5634
5635 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
5636                                                    struct net_device *upper_dev)
5637 {
5638         __netdev_adjacent_dev_unlink(dev, upper_dev);
5639         __netdev_adjacent_dev_unlink_lists(dev, upper_dev,
5640                                            &dev->adj_list.upper,
5641                                            &upper_dev->adj_list.lower);
5642 }
5643
5644 static int __netdev_upper_dev_link(struct net_device *dev,
5645                                    struct net_device *upper_dev, bool master,
5646                                    void *upper_priv, void *upper_info)
5647 {
5648         struct netdev_notifier_changeupper_info changeupper_info;
5649         struct netdev_adjacent *i, *j, *to_i, *to_j;
5650         int ret = 0;
5651
5652         ASSERT_RTNL();
5653
5654         if (dev == upper_dev)
5655                 return -EBUSY;
5656
5657         /* To prevent loops, check if dev is not upper device to upper_dev. */
5658         if (__netdev_find_adj(dev, &upper_dev->all_adj_list.upper))
5659                 return -EBUSY;
5660
5661         if (__netdev_find_adj(upper_dev, &dev->adj_list.upper))
5662                 return -EEXIST;
5663
5664         if (master && netdev_master_upper_dev_get(dev))
5665                 return -EBUSY;
5666
5667         changeupper_info.upper_dev = upper_dev;
5668         changeupper_info.master = master;
5669         changeupper_info.linking = true;
5670         changeupper_info.upper_info = upper_info;
5671
5672         ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5673                                             &changeupper_info.info);
5674         ret = notifier_to_errno(ret);
5675         if (ret)
5676                 return ret;
5677
5678         ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
5679                                                    master);
5680         if (ret)
5681                 return ret;
5682
5683         /* Now that we linked these devs, make all the upper_dev's
5684          * all_adj_list.upper visible to every dev's all_adj_list.lower an
5685          * versa, and don't forget the devices itself. All of these
5686          * links are non-neighbours.
5687          */
5688         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5689                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5690                         pr_debug("Interlinking %s with %s, non-neighbour\n",
5691                                  i->dev->name, j->dev->name);
5692                         ret = __netdev_adjacent_dev_link(i->dev, j->dev);
5693                         if (ret)
5694                                 goto rollback_mesh;
5695                 }
5696         }
5697
5698         /* add dev to every upper_dev's upper device */
5699         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5700                 pr_debug("linking %s's upper device %s with %s\n",
5701                          upper_dev->name, i->dev->name, dev->name);
5702                 ret = __netdev_adjacent_dev_link(dev, i->dev);
5703                 if (ret)
5704                         goto rollback_upper_mesh;
5705         }
5706
5707         /* add upper_dev to every dev's lower device */
5708         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5709                 pr_debug("linking %s's lower device %s with %s\n", dev->name,
5710                          i->dev->name, upper_dev->name);
5711                 ret = __netdev_adjacent_dev_link(i->dev, upper_dev);
5712                 if (ret)
5713                         goto rollback_lower_mesh;
5714         }
5715
5716         ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5717                                             &changeupper_info.info);
5718         ret = notifier_to_errno(ret);
5719         if (ret)
5720                 goto rollback_lower_mesh;
5721
5722         return 0;
5723
5724 rollback_lower_mesh:
5725         to_i = i;
5726         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5727                 if (i == to_i)
5728                         break;
5729                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5730         }
5731
5732         i = NULL;
5733
5734 rollback_upper_mesh:
5735         to_i = i;
5736         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) {
5737                 if (i == to_i)
5738                         break;
5739                 __netdev_adjacent_dev_unlink(dev, i->dev);
5740         }
5741
5742         i = j = NULL;
5743
5744 rollback_mesh:
5745         to_i = i;
5746         to_j = j;
5747         list_for_each_entry(i, &dev->all_adj_list.lower, list) {
5748                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) {
5749                         if (i == to_i && j == to_j)
5750                                 break;
5751                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5752                 }
5753                 if (i == to_i)
5754                         break;
5755         }
5756
5757         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5758
5759         return ret;
5760 }
5761
5762 /**
5763  * netdev_upper_dev_link - Add a link to the upper device
5764  * @dev: device
5765  * @upper_dev: new upper device
5766  *
5767  * Adds a link to device which is upper to this one. The caller must hold
5768  * the RTNL lock. On a failure a negative errno code is returned.
5769  * On success the reference counts are adjusted and the function
5770  * returns zero.
5771  */
5772 int netdev_upper_dev_link(struct net_device *dev,
5773                           struct net_device *upper_dev)
5774 {
5775         return __netdev_upper_dev_link(dev, upper_dev, false, NULL, NULL);
5776 }
5777 EXPORT_SYMBOL(netdev_upper_dev_link);
5778
5779 /**
5780  * netdev_master_upper_dev_link - Add a master link to the upper device
5781  * @dev: device
5782  * @upper_dev: new upper device
5783  * @upper_priv: upper device private
5784  * @upper_info: upper info to be passed down via notifier
5785  *
5786  * Adds a link to device which is upper to this one. In this case, only
5787  * one master upper device can be linked, although other non-master devices
5788  * might be linked as well. The caller must hold the RTNL lock.
5789  * On a failure a negative errno code is returned. On success the reference
5790  * counts are adjusted and the function returns zero.
5791  */
5792 int netdev_master_upper_dev_link(struct net_device *dev,
5793                                  struct net_device *upper_dev,
5794                                  void *upper_priv, void *upper_info)
5795 {
5796         return __netdev_upper_dev_link(dev, upper_dev, true,
5797                                        upper_priv, upper_info);
5798 }
5799 EXPORT_SYMBOL(netdev_master_upper_dev_link);
5800
5801 /**
5802  * netdev_upper_dev_unlink - Removes a link to upper device
5803  * @dev: device
5804  * @upper_dev: new upper device
5805  *
5806  * Removes a link to device which is upper to this one. The caller must hold
5807  * the RTNL lock.
5808  */
5809 void netdev_upper_dev_unlink(struct net_device *dev,
5810                              struct net_device *upper_dev)
5811 {
5812         struct netdev_notifier_changeupper_info changeupper_info;
5813         struct netdev_adjacent *i, *j;
5814         ASSERT_RTNL();
5815
5816         changeupper_info.upper_dev = upper_dev;
5817         changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
5818         changeupper_info.linking = false;
5819
5820         call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, dev,
5821                                       &changeupper_info.info);
5822
5823         __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
5824
5825         /* Here is the tricky part. We must remove all dev's lower
5826          * devices from all upper_dev's upper devices and vice
5827          * versa, to maintain the graph relationship.
5828          */
5829         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5830                 list_for_each_entry(j, &upper_dev->all_adj_list.upper, list)
5831                         __netdev_adjacent_dev_unlink(i->dev, j->dev);
5832
5833         /* remove also the devices itself from lower/upper device
5834          * list
5835          */
5836         list_for_each_entry(i, &dev->all_adj_list.lower, list)
5837                 __netdev_adjacent_dev_unlink(i->dev, upper_dev);
5838
5839         list_for_each_entry(i, &upper_dev->all_adj_list.upper, list)
5840                 __netdev_adjacent_dev_unlink(dev, i->dev);
5841
5842         call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, dev,
5843                                       &changeupper_info.info);
5844 }
5845 EXPORT_SYMBOL(netdev_upper_dev_unlink);
5846
5847 /**
5848  * netdev_bonding_info_change - Dispatch event about slave change
5849  * @dev: device
5850  * @bonding_info: info to dispatch
5851  *
5852  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
5853  * The caller must hold the RTNL lock.
5854  */
5855 void netdev_bonding_info_change(struct net_device *dev,
5856                                 struct netdev_bonding_info *bonding_info)
5857 {
5858         struct netdev_notifier_bonding_info     info;
5859
5860         memcpy(&info.bonding_info, bonding_info,
5861                sizeof(struct netdev_bonding_info));
5862         call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev,
5863                                       &info.info);
5864 }
5865 EXPORT_SYMBOL(netdev_bonding_info_change);
5866
5867 static void netdev_adjacent_add_links(struct net_device *dev)
5868 {
5869         struct netdev_adjacent *iter;
5870
5871         struct net *net = dev_net(dev);
5872
5873         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5874                 if (!net_eq(net,dev_net(iter->dev)))
5875                         continue;
5876                 netdev_adjacent_sysfs_add(iter->dev, dev,
5877                                           &iter->dev->adj_list.lower);
5878                 netdev_adjacent_sysfs_add(dev, iter->dev,
5879                                           &dev->adj_list.upper);
5880         }
5881
5882         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5883                 if (!net_eq(net,dev_net(iter->dev)))
5884                         continue;
5885                 netdev_adjacent_sysfs_add(iter->dev, dev,
5886                                           &iter->dev->adj_list.upper);
5887                 netdev_adjacent_sysfs_add(dev, iter->dev,
5888                                           &dev->adj_list.lower);
5889         }
5890 }
5891
5892 static void netdev_adjacent_del_links(struct net_device *dev)
5893 {
5894         struct netdev_adjacent *iter;
5895
5896         struct net *net = dev_net(dev);
5897
5898         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5899                 if (!net_eq(net,dev_net(iter->dev)))
5900                         continue;
5901                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5902                                           &iter->dev->adj_list.lower);
5903                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5904                                           &dev->adj_list.upper);
5905         }
5906
5907         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5908                 if (!net_eq(net,dev_net(iter->dev)))
5909                         continue;
5910                 netdev_adjacent_sysfs_del(iter->dev, dev->name,
5911                                           &iter->dev->adj_list.upper);
5912                 netdev_adjacent_sysfs_del(dev, iter->dev->name,
5913                                           &dev->adj_list.lower);
5914         }
5915 }
5916
5917 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
5918 {
5919         struct netdev_adjacent *iter;
5920
5921         struct net *net = dev_net(dev);
5922
5923         list_for_each_entry(iter, &dev->adj_list.upper, list) {
5924                 if (!net_eq(net,dev_net(iter->dev)))
5925                         continue;
5926                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5927                                           &iter->dev->adj_list.lower);
5928                 netdev_adjacent_sysfs_add(iter->dev, dev,
5929                                           &iter->dev->adj_list.lower);
5930         }
5931
5932         list_for_each_entry(iter, &dev->adj_list.lower, list) {
5933                 if (!net_eq(net,dev_net(iter->dev)))
5934                         continue;
5935                 netdev_adjacent_sysfs_del(iter->dev, oldname,
5936                                           &iter->dev->adj_list.upper);
5937                 netdev_adjacent_sysfs_add(iter->dev, dev,
5938                                           &iter->dev->adj_list.upper);
5939         }
5940 }
5941
5942 void *netdev_lower_dev_get_private(struct net_device *dev,
5943                                    struct net_device *lower_dev)
5944 {
5945         struct netdev_adjacent *lower;
5946
5947         if (!lower_dev)
5948                 return NULL;
5949         lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
5950         if (!lower)
5951                 return NULL;
5952
5953         return lower->private;
5954 }
5955 EXPORT_SYMBOL(netdev_lower_dev_get_private);
5956
5957
5958 int dev_get_nest_level(struct net_device *dev,
5959                        bool (*type_check)(const struct net_device *dev))
5960 {
5961         struct net_device *lower = NULL;
5962         struct list_head *iter;
5963         int max_nest = -1;
5964         int nest;
5965
5966         ASSERT_RTNL();
5967
5968         netdev_for_each_lower_dev(dev, lower, iter) {
5969                 nest = dev_get_nest_level(lower, type_check);
5970                 if (max_nest < nest)
5971                         max_nest = nest;
5972         }
5973
5974         if (type_check(dev))
5975                 max_nest++;
5976
5977         return max_nest;
5978 }
5979 EXPORT_SYMBOL(dev_get_nest_level);
5980
5981 /**
5982  * netdev_lower_change - Dispatch event about lower device state change
5983  * @lower_dev: device
5984  * @lower_state_info: state to dispatch
5985  *
5986  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
5987  * The caller must hold the RTNL lock.
5988  */
5989 void netdev_lower_state_changed(struct net_device *lower_dev,
5990                                 void *lower_state_info)
5991 {
5992         struct netdev_notifier_changelowerstate_info changelowerstate_info;
5993
5994         ASSERT_RTNL();
5995         changelowerstate_info.lower_state_info = lower_state_info;
5996         call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, lower_dev,
5997                                       &changelowerstate_info.info);
5998 }
5999 EXPORT_SYMBOL(netdev_lower_state_changed);
6000
6001 static void dev_change_rx_flags(struct net_device *dev, int flags)
6002 {
6003         const struct net_device_ops *ops = dev->netdev_ops;
6004
6005         if (ops->ndo_change_rx_flags)
6006                 ops->ndo_change_rx_flags(dev, flags);
6007 }
6008
6009 static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
6010 {
6011         unsigned int old_flags = dev->flags;
6012         kuid_t uid;
6013         kgid_t gid;
6014
6015         ASSERT_RTNL();
6016
6017         dev->flags |= IFF_PROMISC;
6018         dev->promiscuity += inc;
6019         if (dev->promiscuity == 0) {
6020                 /*
6021                  * Avoid overflow.
6022                  * If inc causes overflow, untouch promisc and return error.
6023                  */
6024                 if (inc < 0)
6025                         dev->flags &= ~IFF_PROMISC;
6026                 else {
6027                         dev->promiscuity -= inc;
6028                         pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n",
6029                                 dev->name);
6030                         return -EOVERFLOW;
6031                 }
6032         }
6033         if (dev->flags != old_flags) {
6034                 pr_info("device %s %s promiscuous mode\n",
6035                         dev->name,
6036                         dev->flags & IFF_PROMISC ? "entered" : "left");
6037                 if (audit_enabled) {
6038                         current_uid_gid(&uid, &gid);
6039                         audit_log(current->audit_context, GFP_ATOMIC,
6040                                 AUDIT_ANOM_PROMISCUOUS,
6041                                 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
6042                                 dev->name, (dev->flags & IFF_PROMISC),
6043                                 (old_flags & IFF_PROMISC),
6044                                 from_kuid(&init_user_ns, audit_get_loginuid(current)),
6045                                 from_kuid(&init_user_ns, uid),
6046                                 from_kgid(&init_user_ns, gid),
6047                                 audit_get_sessionid(current));
6048                 }
6049
6050                 dev_change_rx_flags(dev, IFF_PROMISC);
6051         }
6052         if (notify)
6053                 __dev_notify_flags(dev, old_flags, IFF_PROMISC);
6054         return 0;
6055 }
6056
6057 /**
6058  *      dev_set_promiscuity     - update promiscuity count on a device
6059  *      @dev: device
6060  *      @inc: modifier
6061  *
6062  *      Add or remove promiscuity from a device. While the count in the device
6063  *      remains above zero the interface remains promiscuous. Once it hits zero
6064  *      the device reverts back to normal filtering operation. A negative inc
6065  *      value is used to drop promiscuity on the device.
6066  *      Return 0 if successful or a negative errno code on error.
6067  */
6068 int dev_set_promiscuity(struct net_device *dev, int inc)
6069 {
6070         unsigned int old_flags = dev->flags;
6071         int err;
6072
6073         err = __dev_set_promiscuity(dev, inc, true);
6074         if (err < 0)
6075                 return err;
6076         if (dev->flags != old_flags)
6077                 dev_set_rx_mode(dev);
6078         return err;
6079 }
6080 EXPORT_SYMBOL(dev_set_promiscuity);
6081
6082 static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify)
6083 {
6084         unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
6085
6086         ASSERT_RTNL();
6087
6088         dev->flags |= IFF_ALLMULTI;
6089         dev->allmulti += inc;
6090         if (dev->allmulti == 0) {
6091                 /*
6092                  * Avoid overflow.
6093                  * If inc causes overflow, untouch allmulti and return error.
6094                  */
6095                 if (inc < 0)
6096                         dev->flags &= ~IFF_ALLMULTI;
6097                 else {
6098                         dev->allmulti -= inc;
6099                         pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n",
6100                                 dev->name);
6101                         return -EOVERFLOW;
6102                 }
6103         }
6104         if (dev->flags ^ old_flags) {
6105                 dev_change_rx_flags(dev, IFF_ALLMULTI);
6106                 dev_set_rx_mode(dev);
6107                 if (notify)
6108                         __dev_notify_flags(dev, old_flags,
6109                                            dev->gflags ^ old_gflags);
6110         }
6111         return 0;
6112 }
6113
6114 /**
6115  *      dev_set_allmulti        - update allmulti count on a device
6116  *      @dev: device
6117  *      @inc: modifier
6118  *
6119  *      Add or remove reception of all multicast frames to a device. While the
6120  *      count in the device remains above zero the interface remains listening
6121  *      to all interfaces. Once it hits zero the device reverts back to normal
6122  *      filtering operation. A negative @inc value is used to drop the counter
6123  *      when releasing a resource needing all multicasts.
6124  *      Return 0 if successful or a negative errno code on error.
6125  */
6126
6127 int dev_set_allmulti(struct net_device *dev, int inc)
6128 {
6129         return __dev_set_allmulti(dev, inc, true);
6130 }
6131 EXPORT_SYMBOL(dev_set_allmulti);
6132
6133 /*
6134  *      Upload unicast and multicast address lists to device and
6135  *      configure RX filtering. When the device doesn't support unicast
6136  *      filtering it is put in promiscuous mode while unicast addresses
6137  *      are present.
6138  */
6139 void __dev_set_rx_mode(struct net_device *dev)
6140 {
6141         const struct net_device_ops *ops = dev->netdev_ops;
6142
6143         /* dev_open will call this function so the list will stay sane. */
6144         if (!(dev->flags&IFF_UP))
6145                 return;
6146
6147         if (!netif_device_present(dev))
6148                 return;
6149
6150         if (!(dev->priv_flags & IFF_UNICAST_FLT)) {
6151                 /* Unicast addresses changes may only happen under the rtnl,
6152                  * therefore calling __dev_set_promiscuity here is safe.
6153                  */
6154                 if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
6155                         __dev_set_promiscuity(dev, 1, false);
6156                         dev->uc_promisc = true;
6157                 } else if (netdev_uc_empty(dev) && dev->uc_promisc) {
6158                         __dev_set_promiscuity(dev, -1, false);
6159                         dev->uc_promisc = false;
6160                 }
6161         }
6162
6163         if (ops->ndo_set_rx_mode)
6164                 ops->ndo_set_rx_mode(dev);
6165 }
6166
6167 void dev_set_rx_mode(struct net_device *dev)
6168 {
6169         netif_addr_lock_bh(dev);
6170         __dev_set_rx_mode(dev);
6171         netif_addr_unlock_bh(dev);
6172 }
6173
6174 /**
6175  *      dev_get_flags - get flags reported to userspace
6176  *      @dev: device
6177  *
6178  *      Get the combination of flag bits exported through APIs to userspace.
6179  */
6180 unsigned int dev_get_flags(const struct net_device *dev)
6181 {
6182         unsigned int flags;
6183
6184         flags = (dev->flags & ~(IFF_PROMISC |
6185                                 IFF_ALLMULTI |
6186                                 IFF_RUNNING |
6187                                 IFF_LOWER_UP |
6188                                 IFF_DORMANT)) |
6189                 (dev->gflags & (IFF_PROMISC |
6190                                 IFF_ALLMULTI));
6191
6192         if (netif_running(dev)) {
6193                 if (netif_oper_up(dev))
6194                         flags |= IFF_RUNNING;
6195                 if (netif_carrier_ok(dev))
6196                         flags |= IFF_LOWER_UP;
6197                 if (netif_dormant(dev))
6198                         flags |= IFF_DORMANT;
6199         }
6200
6201         return flags;
6202 }
6203 EXPORT_SYMBOL(dev_get_flags);
6204
6205 int __dev_change_flags(struct net_device *dev, unsigned int flags)
6206 {
6207         unsigned int old_flags = dev->flags;
6208         int ret;
6209
6210         ASSERT_RTNL();
6211
6212         /*
6213          *      Set the flags on our device.
6214          */
6215
6216         dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
6217                                IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
6218                                IFF_AUTOMEDIA)) |
6219                      (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
6220                                     IFF_ALLMULTI));
6221
6222         /*
6223          *      Load in the correct multicast list now the flags have changed.
6224          */
6225
6226         if ((old_flags ^ flags) & IFF_MULTICAST)
6227                 dev_change_rx_flags(dev, IFF_MULTICAST);
6228
6229         dev_set_rx_mode(dev);
6230
6231         /*
6232          *      Have we downed the interface. We handle IFF_UP ourselves
6233          *      according to user attempts to set it, rather than blindly
6234          *      setting it.
6235          */
6236
6237         ret = 0;
6238         if ((old_flags ^ flags) & IFF_UP)
6239                 ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
6240
6241         if ((flags ^ dev->gflags) & IFF_PROMISC) {
6242                 int inc = (flags & IFF_PROMISC) ? 1 : -1;
6243                 unsigned int old_flags = dev->flags;
6244
6245                 dev->gflags ^= IFF_PROMISC;
6246
6247                 if (__dev_set_promiscuity(dev, inc, false) >= 0)
6248                         if (dev->flags != old_flags)
6249                                 dev_set_rx_mode(dev);
6250         }
6251
6252         /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
6253            is important. Some (broken) drivers set IFF_PROMISC, when
6254            IFF_ALLMULTI is requested not asking us and not reporting.
6255          */
6256         if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
6257                 int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
6258
6259                 dev->gflags ^= IFF_ALLMULTI;
6260                 __dev_set_allmulti(dev, inc, false);
6261         }
6262
6263         return ret;
6264 }
6265
6266 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
6267                         unsigned int gchanges)
6268 {
6269         unsigned int changes = dev->flags ^ old_flags;
6270
6271         if (gchanges)
6272                 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC);
6273
6274         if (changes & IFF_UP) {
6275                 if (dev->flags & IFF_UP)
6276                         call_netdevice_notifiers(NETDEV_UP, dev);
6277                 else
6278                         call_netdevice_notifiers(NETDEV_DOWN, dev);
6279         }
6280
6281         if (dev->flags & IFF_UP &&
6282             (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
6283                 struct netdev_notifier_change_info change_info;
6284
6285                 change_info.flags_changed = changes;
6286                 call_netdevice_notifiers_info(NETDEV_CHANGE, dev,
6287                                               &change_info.info);
6288         }
6289 }
6290
6291 /**
6292  *      dev_change_flags - change device settings
6293  *      @dev: device
6294  *      @flags: device state flags
6295  *
6296  *      Change settings on device based state flags. The flags are
6297  *      in the userspace exported format.
6298  */
6299 int dev_change_flags(struct net_device *dev, unsigned int flags)
6300 {
6301         int ret;
6302         unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
6303
6304         ret = __dev_change_flags(dev, flags);
6305         if (ret < 0)
6306                 return ret;
6307
6308         changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
6309         __dev_notify_flags(dev, old_flags, changes);
6310         return ret;
6311 }
6312 EXPORT_SYMBOL(dev_change_flags);
6313
6314 static int __dev_set_mtu(struct net_device *dev, int new_mtu)
6315 {
6316         const struct net_device_ops *ops = dev->netdev_ops;
6317
6318         if (ops->ndo_change_mtu)
6319                 return ops->ndo_change_mtu(dev, new_mtu);
6320
6321         dev->mtu = new_mtu;
6322         return 0;
6323 }
6324
6325 /**
6326  *      dev_set_mtu - Change maximum transfer unit
6327  *      @dev: device
6328  *      @new_mtu: new transfer unit
6329  *
6330  *      Change the maximum transfer size of the network device.
6331  */
6332 int dev_set_mtu(struct net_device *dev, int new_mtu)
6333 {
6334         int err, orig_mtu;
6335
6336         if (new_mtu == dev->mtu)
6337                 return 0;
6338
6339         /*      MTU must be positive.    */
6340         if (new_mtu < 0)
6341                 return -EINVAL;
6342
6343         if (!netif_device_present(dev))
6344                 return -ENODEV;
6345
6346         err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
6347         err = notifier_to_errno(err);
6348         if (err)
6349                 return err;
6350
6351         orig_mtu = dev->mtu;
6352         err = __dev_set_mtu(dev, new_mtu);
6353
6354         if (!err) {
6355                 err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6356                 err = notifier_to_errno(err);
6357                 if (err) {
6358                         /* setting mtu back and notifying everyone again,
6359                          * so that they have a chance to revert changes.
6360                          */
6361                         __dev_set_mtu(dev, orig_mtu);
6362                         call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
6363                 }
6364         }
6365         return err;
6366 }
6367 EXPORT_SYMBOL(dev_set_mtu);
6368
6369 /**
6370  *      dev_set_group - Change group this device belongs to
6371  *      @dev: device
6372  *      @new_group: group this device should belong to
6373  */
6374 void dev_set_group(struct net_device *dev, int new_group)
6375 {
6376         dev->group = new_group;
6377 }
6378 EXPORT_SYMBOL(dev_set_group);
6379
6380 /**
6381  *      dev_set_mac_address - Change Media Access Control Address
6382  *      @dev: device
6383  *      @sa: new address
6384  *
6385  *      Change the hardware (MAC) address of the device
6386  */
6387 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
6388 {
6389         const struct net_device_ops *ops = dev->netdev_ops;
6390         int err;
6391
6392         if (!ops->ndo_set_mac_address)
6393                 return -EOPNOTSUPP;
6394         if (sa->sa_family != dev->type)
6395                 return -EINVAL;
6396         if (!netif_device_present(dev))
6397                 return -ENODEV;
6398         err = ops->ndo_set_mac_address(dev, sa);
6399         if (err)
6400                 return err;
6401         dev->addr_assign_type = NET_ADDR_SET;
6402         call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
6403         add_device_randomness(dev->dev_addr, dev->addr_len);
6404         return 0;
6405 }
6406 EXPORT_SYMBOL(dev_set_mac_address);
6407
6408 /**
6409  *      dev_change_carrier - Change device carrier
6410  *      @dev: device
6411  *      @new_carrier: new value
6412  *
6413  *      Change device carrier
6414  */
6415 int dev_change_carrier(struct net_device *dev, bool new_carrier)
6416 {
6417         const struct net_device_ops *ops = dev->netdev_ops;
6418
6419         if (!ops->ndo_change_carrier)
6420                 return -EOPNOTSUPP;
6421         if (!netif_device_present(dev))
6422                 return -ENODEV;
6423         return ops->ndo_change_carrier(dev, new_carrier);
6424 }
6425 EXPORT_SYMBOL(dev_change_carrier);
6426
6427 /**
6428  *      dev_get_phys_port_id - Get device physical port ID
6429  *      @dev: device
6430  *      @ppid: port ID
6431  *
6432  *      Get device physical port ID
6433  */
6434 int dev_get_phys_port_id(struct net_device *dev,
6435                          struct netdev_phys_item_id *ppid)
6436 {
6437         const struct net_device_ops *ops = dev->netdev_ops;
6438
6439         if (!ops->ndo_get_phys_port_id)
6440                 return -EOPNOTSUPP;
6441         return ops->ndo_get_phys_port_id(dev, ppid);
6442 }
6443 EXPORT_SYMBOL(dev_get_phys_port_id);
6444
6445 /**
6446  *      dev_get_phys_port_name - Get device physical port name
6447  *      @dev: device
6448  *      @name: port name
6449  *      @len: limit of bytes to copy to name
6450  *
6451  *      Get device physical port name
6452  */
6453 int dev_get_phys_port_name(struct net_device *dev,
6454                            char *name, size_t len)
6455 {
6456         const struct net_device_ops *ops = dev->netdev_ops;
6457
6458         if (!ops->ndo_get_phys_port_name)
6459                 return -EOPNOTSUPP;
6460         return ops->ndo_get_phys_port_name(dev, name, len);
6461 }
6462 EXPORT_SYMBOL(dev_get_phys_port_name);
6463
6464 /**
6465  *      dev_change_proto_down - update protocol port state information
6466  *      @dev: device
6467  *      @proto_down: new value
6468  *
6469  *      This info can be used by switch drivers to set the phys state of the
6470  *      port.
6471  */
6472 int dev_change_proto_down(struct net_device *dev, bool proto_down)
6473 {
6474         const struct net_device_ops *ops = dev->netdev_ops;
6475
6476         if (!ops->ndo_change_proto_down)
6477                 return -EOPNOTSUPP;
6478         if (!netif_device_present(dev))
6479                 return -ENODEV;
6480         return ops->ndo_change_proto_down(dev, proto_down);
6481 }
6482 EXPORT_SYMBOL(dev_change_proto_down);
6483
6484 /**
6485  *      dev_new_index   -       allocate an ifindex
6486  *      @net: the applicable net namespace
6487  *
6488  *      Returns a suitable unique value for a new device interface
6489  *      number.  The caller must hold the rtnl semaphore or the
6490  *      dev_base_lock to be sure it remains unique.
6491  */
6492 static int dev_new_index(struct net *net)
6493 {
6494         int ifindex = net->ifindex;
6495         for (;;) {
6496                 if (++ifindex <= 0)
6497                         ifindex = 1;
6498                 if (!__dev_get_by_index(net, ifindex))
6499                         return net->ifindex = ifindex;
6500         }
6501 }
6502
6503 /* Delayed registration/unregisteration */
6504 static LIST_HEAD(net_todo_list);
6505 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
6506
6507 static void net_set_todo(struct net_device *dev)
6508 {
6509         list_add_tail(&dev->todo_list, &net_todo_list);
6510         dev_net(dev)->dev_unreg_count++;
6511 }
6512
6513 static void rollback_registered_many(struct list_head *head)
6514 {
6515         struct net_device *dev, *tmp;
6516         LIST_HEAD(close_head);
6517
6518         BUG_ON(dev_boot_phase);
6519         ASSERT_RTNL();
6520
6521         list_for_each_entry_safe(dev, tmp, head, unreg_list) {
6522                 /* Some devices call without registering
6523                  * for initialization unwind. Remove those
6524                  * devices and proceed with the remaining.
6525                  */
6526                 if (dev->reg_state == NETREG_UNINITIALIZED) {
6527                         pr_debug("unregister_netdevice: device %s/%p never was registered\n",
6528                                  dev->name, dev);
6529
6530                         WARN_ON(1);
6531                         list_del(&dev->unreg_list);
6532                         continue;
6533                 }
6534                 dev->dismantle = true;
6535                 BUG_ON(dev->reg_state != NETREG_REGISTERED);
6536         }
6537
6538         /* If device is running, close it first. */
6539         list_for_each_entry(dev, head, unreg_list)
6540                 list_add_tail(&dev->close_list, &close_head);
6541         dev_close_many(&close_head, true);
6542
6543         list_for_each_entry(dev, head, unreg_list) {
6544                 /* And unlink it from device chain. */
6545                 unlist_netdevice(dev);
6546
6547                 dev->reg_state = NETREG_UNREGISTERING;
6548                 on_each_cpu(flush_backlog, dev, 1);
6549         }
6550
6551         synchronize_net();
6552
6553         list_for_each_entry(dev, head, unreg_list) {
6554                 struct sk_buff *skb = NULL;
6555
6556                 /* Shutdown queueing discipline. */
6557                 dev_shutdown(dev);
6558
6559
6560                 /* Notify protocols, that we are about to destroy
6561                    this device. They should clean all the things.
6562                 */
6563                 call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
6564
6565                 if (!dev->rtnl_link_ops ||
6566                     dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
6567                         skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U,
6568                                                      GFP_KERNEL);
6569
6570                 /*
6571                  *      Flush the unicast and multicast chains
6572                  */
6573                 dev_uc_flush(dev);
6574                 dev_mc_flush(dev);
6575
6576                 if (dev->netdev_ops->ndo_uninit)
6577                         dev->netdev_ops->ndo_uninit(dev);
6578
6579                 if (skb)
6580                         rtmsg_ifinfo_send(skb, dev, GFP_KERNEL);
6581
6582                 /* Notifier chain MUST detach us all upper devices. */
6583                 WARN_ON(netdev_has_any_upper_dev(dev));
6584
6585                 /* Remove entries from kobject tree */
6586                 netdev_unregister_kobject(dev);
6587 #ifdef CONFIG_XPS
6588                 /* Remove XPS queueing entries */
6589                 netif_reset_xps_queues_gt(dev, 0);
6590 #endif
6591         }
6592
6593         synchronize_net();
6594
6595         list_for_each_entry(dev, head, unreg_list)
6596                 dev_put(dev);
6597 }
6598
6599 static void rollback_registered(struct net_device *dev)
6600 {
6601         LIST_HEAD(single);
6602
6603         list_add(&dev->unreg_list, &single);
6604         rollback_registered_many(&single);
6605         list_del(&single);
6606 }
6607
6608 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
6609         struct net_device *upper, netdev_features_t features)
6610 {
6611         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6612         netdev_features_t feature;
6613         int feature_bit;
6614
6615         for_each_netdev_feature(&upper_disables, feature_bit) {
6616                 feature = __NETIF_F_BIT(feature_bit);
6617                 if (!(upper->wanted_features & feature)
6618                     && (features & feature)) {
6619                         netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
6620                                    &feature, upper->name);
6621                         features &= ~feature;
6622                 }
6623         }
6624
6625         return features;
6626 }
6627
6628 static void netdev_sync_lower_features(struct net_device *upper,
6629         struct net_device *lower, netdev_features_t features)
6630 {
6631         netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
6632         netdev_features_t feature;
6633         int feature_bit;
6634
6635         for_each_netdev_feature(&upper_disables, feature_bit) {
6636                 feature = __NETIF_F_BIT(feature_bit);
6637                 if (!(features & feature) && (lower->features & feature)) {
6638                         netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
6639                                    &feature, lower->name);
6640                         lower->wanted_features &= ~feature;
6641                         netdev_update_features(lower);
6642
6643                         if (unlikely(lower->features & feature))
6644                                 netdev_WARN(upper, "failed to disable %pNF on %s!\n",
6645                                             &feature, lower->name);
6646                 }
6647         }
6648 }
6649
6650 static netdev_features_t netdev_fix_features(struct net_device *dev,
6651         netdev_features_t features)
6652 {
6653         /* Fix illegal checksum combinations */
6654         if ((features & NETIF_F_HW_CSUM) &&
6655             (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
6656                 netdev_warn(dev, "mixed HW and IP checksum settings.\n");
6657                 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
6658         }
6659
6660         /* TSO requires that SG is present as well. */
6661         if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
6662                 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
6663                 features &= ~NETIF_F_ALL_TSO;
6664         }
6665
6666         if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
6667                                         !(features & NETIF_F_IP_CSUM)) {
6668                 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
6669                 features &= ~NETIF_F_TSO;
6670                 features &= ~NETIF_F_TSO_ECN;
6671         }
6672
6673         if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
6674                                          !(features & NETIF_F_IPV6_CSUM)) {
6675                 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
6676                 features &= ~NETIF_F_TSO6;
6677         }
6678
6679         /* TSO ECN requires that TSO is present as well. */
6680         if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
6681                 features &= ~NETIF_F_TSO_ECN;
6682
6683         /* Software GSO depends on SG. */
6684         if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
6685                 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
6686                 features &= ~NETIF_F_GSO;
6687         }
6688
6689         /* UFO needs SG and checksumming */
6690         if (features & NETIF_F_UFO) {
6691                 /* maybe split UFO into V4 and V6? */
6692                 if (!(features & NETIF_F_HW_CSUM) &&
6693                     ((features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) !=
6694                      (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))) {
6695                         netdev_dbg(dev,
6696                                 "Dropping NETIF_F_UFO since no checksum offload features.\n");
6697                         features &= ~NETIF_F_UFO;
6698                 }
6699
6700                 if (!(features & NETIF_F_SG)) {
6701                         netdev_dbg(dev,
6702                                 "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n");
6703                         features &= ~NETIF_F_UFO;
6704                 }
6705         }
6706
6707 #ifdef CONFIG_NET_RX_BUSY_POLL
6708         if (dev->netdev_ops->ndo_busy_poll)
6709                 features |= NETIF_F_BUSY_POLL;
6710         else
6711 #endif
6712                 features &= ~NETIF_F_BUSY_POLL;
6713
6714         return features;
6715 }
6716
6717 int __netdev_update_features(struct net_device *dev)
6718 {
6719         struct net_device *upper, *lower;
6720         netdev_features_t features;
6721         struct list_head *iter;
6722         int err = -1;
6723
6724         ASSERT_RTNL();
6725
6726         features = netdev_get_wanted_features(dev);
6727
6728         if (dev->netdev_ops->ndo_fix_features)
6729                 features = dev->netdev_ops->ndo_fix_features(dev, features);
6730
6731         /* driver might be less strict about feature dependencies */
6732         features = netdev_fix_features(dev, features);
6733
6734         /* some features can't be enabled if they're off an an upper device */
6735         netdev_for_each_upper_dev_rcu(dev, upper, iter)
6736                 features = netdev_sync_upper_features(dev, upper, features);
6737
6738         if (dev->features == features)
6739                 goto sync_lower;
6740
6741         netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
6742                 &dev->features, &features);
6743
6744         if (dev->netdev_ops->ndo_set_features)
6745                 err = dev->netdev_ops->ndo_set_features(dev, features);
6746         else
6747                 err = 0;
6748
6749         if (unlikely(err < 0)) {
6750                 netdev_err(dev,
6751                         "set_features() failed (%d); wanted %pNF, left %pNF\n",
6752                         err, &features, &dev->features);
6753                 /* return non-0 since some features might have changed and
6754                  * it's better to fire a spurious notification than miss it
6755                  */
6756                 return -1;
6757         }
6758
6759 sync_lower:
6760         /* some features must be disabled on lower devices when disabled
6761          * on an upper device (think: bonding master or bridge)
6762          */
6763         netdev_for_each_lower_dev(dev, lower, iter)
6764                 netdev_sync_lower_features(dev, lower, features);
6765
6766         if (!err)
6767                 dev->features = features;
6768
6769         return err < 0 ? 0 : 1;
6770 }
6771
6772 /**
6773  *      netdev_update_features - recalculate device features
6774  *      @dev: the device to check
6775  *
6776  *      Recalculate dev->features set and send notifications if it
6777  *      has changed. Should be called after driver or hardware dependent
6778  *      conditions might have changed that influence the features.
6779  */
6780 void netdev_update_features(struct net_device *dev)
6781 {
6782         if (__netdev_update_features(dev))
6783                 netdev_features_change(dev);
6784 }
6785 EXPORT_SYMBOL(netdev_update_features);
6786
6787 /**
6788  *      netdev_change_features - recalculate device features
6789  *      @dev: the device to check
6790  *
6791  *      Recalculate dev->features set and send notifications even
6792  *      if they have not changed. Should be called instead of
6793  *      netdev_update_features() if also dev->vlan_features might
6794  *      have changed to allow the changes to be propagated to stacked
6795  *      VLAN devices.
6796  */
6797 void netdev_change_features(struct net_device *dev)
6798 {
6799         __netdev_update_features(dev);
6800         netdev_features_change(dev);
6801 }
6802 EXPORT_SYMBOL(netdev_change_features);
6803
6804 /**
6805  *      netif_stacked_transfer_operstate -      transfer operstate
6806  *      @rootdev: the root or lower level device to transfer state from
6807  *      @dev: the device to transfer operstate to
6808  *
6809  *      Transfer operational state from root to device. This is normally
6810  *      called when a stacking relationship exists between the root
6811  *      device and the device(a leaf device).
6812  */
6813 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
6814                                         struct net_device *dev)
6815 {
6816         if (rootdev->operstate == IF_OPER_DORMANT)
6817                 netif_dormant_on(dev);
6818         else
6819                 netif_dormant_off(dev);
6820
6821         if (netif_carrier_ok(rootdev)) {
6822                 if (!netif_carrier_ok(dev))
6823                         netif_carrier_on(dev);
6824         } else {
6825                 if (netif_carrier_ok(dev))
6826                         netif_carrier_off(dev);
6827         }
6828 }
6829 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
6830
6831 #ifdef CONFIG_SYSFS
6832 static int netif_alloc_rx_queues(struct net_device *dev)
6833 {
6834         unsigned int i, count = dev->num_rx_queues;
6835         struct netdev_rx_queue *rx;
6836         size_t sz = count * sizeof(*rx);
6837
6838         BUG_ON(count < 1);
6839
6840         rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6841         if (!rx) {
6842                 rx = vzalloc(sz);
6843                 if (!rx)
6844                         return -ENOMEM;
6845         }
6846         dev->_rx = rx;
6847
6848         for (i = 0; i < count; i++)
6849                 rx[i].dev = dev;
6850         return 0;
6851 }
6852 #endif
6853
6854 static void netdev_init_one_queue(struct net_device *dev,
6855                                   struct netdev_queue *queue, void *_unused)
6856 {
6857         /* Initialize queue lock */
6858         spin_lock_init(&queue->_xmit_lock);
6859         netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
6860         queue->xmit_lock_owner = -1;
6861         netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
6862         queue->dev = dev;
6863 #ifdef CONFIG_BQL
6864         dql_init(&queue->dql, HZ);
6865 #endif
6866 }
6867
6868 static void netif_free_tx_queues(struct net_device *dev)
6869 {
6870         kvfree(dev->_tx);
6871 }
6872
6873 static int netif_alloc_netdev_queues(struct net_device *dev)
6874 {
6875         unsigned int count = dev->num_tx_queues;
6876         struct netdev_queue *tx;
6877         size_t sz = count * sizeof(*tx);
6878
6879         if (count < 1 || count > 0xffff)
6880                 return -EINVAL;
6881
6882         tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
6883         if (!tx) {
6884                 tx = vzalloc(sz);
6885                 if (!tx)
6886                         return -ENOMEM;
6887         }
6888         dev->_tx = tx;
6889
6890         netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
6891         spin_lock_init(&dev->tx_global_lock);
6892
6893         return 0;
6894 }
6895
6896 void netif_tx_stop_all_queues(struct net_device *dev)
6897 {
6898         unsigned int i;
6899
6900         for (i = 0; i < dev->num_tx_queues; i++) {
6901                 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
6902                 netif_tx_stop_queue(txq);
6903         }
6904 }
6905 EXPORT_SYMBOL(netif_tx_stop_all_queues);
6906
6907 /**
6908  *      register_netdevice      - register a network device
6909  *      @dev: device to register
6910  *
6911  *      Take a completed network device structure and add it to the kernel
6912  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
6913  *      chain. 0 is returned on success. A negative errno code is returned
6914  *      on a failure to set up the device, or if the name is a duplicate.
6915  *
6916  *      Callers must hold the rtnl semaphore. You may want
6917  *      register_netdev() instead of this.
6918  *
6919  *      BUGS:
6920  *      The locking appears insufficient to guarantee two parallel registers
6921  *      will not get the same name.
6922  */
6923
6924 int register_netdevice(struct net_device *dev)
6925 {
6926         int ret;
6927         struct net *net = dev_net(dev);
6928
6929         BUG_ON(dev_boot_phase);
6930         ASSERT_RTNL();
6931
6932         might_sleep();
6933
6934         /* When net_device's are persistent, this will be fatal. */
6935         BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
6936         BUG_ON(!net);
6937
6938         spin_lock_init(&dev->addr_list_lock);
6939         netdev_set_addr_lockdep_class(dev);
6940
6941         ret = dev_get_valid_name(net, dev, dev->name);
6942         if (ret < 0)
6943                 goto out;
6944
6945         /* Init, if this function is available */
6946         if (dev->netdev_ops->ndo_init) {
6947                 ret = dev->netdev_ops->ndo_init(dev);
6948                 if (ret) {
6949                         if (ret > 0)
6950                                 ret = -EIO;
6951                         goto out;
6952                 }
6953         }
6954
6955         if (((dev->hw_features | dev->features) &
6956              NETIF_F_HW_VLAN_CTAG_FILTER) &&
6957             (!dev->netdev_ops->ndo_vlan_rx_add_vid ||
6958              !dev->netdev_ops->ndo_vlan_rx_kill_vid)) {
6959                 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
6960                 ret = -EINVAL;
6961                 goto err_uninit;
6962         }
6963
6964         ret = -EBUSY;
6965         if (!dev->ifindex)
6966                 dev->ifindex = dev_new_index(net);
6967         else if (__dev_get_by_index(net, dev->ifindex))
6968                 goto err_uninit;
6969
6970         /* Transfer changeable features to wanted_features and enable
6971          * software offloads (GSO and GRO).
6972          */
6973         dev->hw_features |= NETIF_F_SOFT_FEATURES;
6974         dev->features |= NETIF_F_SOFT_FEATURES;
6975         dev->wanted_features = dev->features & dev->hw_features;
6976
6977         if (!(dev->flags & IFF_LOOPBACK)) {
6978                 dev->hw_features |= NETIF_F_NOCACHE_COPY;
6979         }
6980
6981         /* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
6982          */
6983         dev->vlan_features |= NETIF_F_HIGHDMA;
6984
6985         /* Make NETIF_F_SG inheritable to tunnel devices.
6986          */
6987         dev->hw_enc_features |= NETIF_F_SG;
6988
6989         /* Make NETIF_F_SG inheritable to MPLS.
6990          */
6991         dev->mpls_features |= NETIF_F_SG;
6992
6993         ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
6994         ret = notifier_to_errno(ret);
6995         if (ret)
6996                 goto err_uninit;
6997
6998         ret = netdev_register_kobject(dev);
6999         if (ret)
7000                 goto err_uninit;
7001         dev->reg_state = NETREG_REGISTERED;
7002
7003         __netdev_update_features(dev);
7004
7005         /*
7006          *      Default initial state at registry is that the
7007          *      device is present.
7008          */
7009
7010         set_bit(__LINK_STATE_PRESENT, &dev->state);
7011
7012         linkwatch_init_dev(dev);
7013
7014         dev_init_scheduler(dev);
7015         dev_hold(dev);
7016         list_netdevice(dev);
7017         add_device_randomness(dev->dev_addr, dev->addr_len);
7018
7019         /* If the device has permanent device address, driver should
7020          * set dev_addr and also addr_assign_type should be set to
7021          * NET_ADDR_PERM (default value).
7022          */
7023         if (dev->addr_assign_type == NET_ADDR_PERM)
7024                 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
7025
7026         /* Notify protocols, that a new device appeared. */
7027         ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
7028         ret = notifier_to_errno(ret);
7029         if (ret) {
7030                 rollback_registered(dev);
7031                 dev->reg_state = NETREG_UNREGISTERED;
7032         }
7033         /*
7034          *      Prevent userspace races by waiting until the network
7035          *      device is fully setup before sending notifications.
7036          */
7037         if (!dev->rtnl_link_ops ||
7038             dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
7039                 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7040
7041 out:
7042         return ret;
7043
7044 err_uninit:
7045         if (dev->netdev_ops->ndo_uninit)
7046                 dev->netdev_ops->ndo_uninit(dev);
7047         goto out;
7048 }
7049 EXPORT_SYMBOL(register_netdevice);
7050
7051 /**
7052  *      init_dummy_netdev       - init a dummy network device for NAPI
7053  *      @dev: device to init
7054  *
7055  *      This takes a network device structure and initialize the minimum
7056  *      amount of fields so it can be used to schedule NAPI polls without
7057  *      registering a full blown interface. This is to be used by drivers
7058  *      that need to tie several hardware interfaces to a single NAPI
7059  *      poll scheduler due to HW limitations.
7060  */
7061 int init_dummy_netdev(struct net_device *dev)
7062 {
7063         /* Clear everything. Note we don't initialize spinlocks
7064          * are they aren't supposed to be taken by any of the
7065          * NAPI code and this dummy netdev is supposed to be
7066          * only ever used for NAPI polls
7067          */
7068         memset(dev, 0, sizeof(struct net_device));
7069
7070         /* make sure we BUG if trying to hit standard
7071          * register/unregister code path
7072          */
7073         dev->reg_state = NETREG_DUMMY;
7074
7075         /* NAPI wants this */
7076         INIT_LIST_HEAD(&dev->napi_list);
7077
7078         /* a dummy interface is started by default */
7079         set_bit(__LINK_STATE_PRESENT, &dev->state);
7080         set_bit(__LINK_STATE_START, &dev->state);
7081
7082         /* Note : We dont allocate pcpu_refcnt for dummy devices,
7083          * because users of this 'device' dont need to change
7084          * its refcount.
7085          */
7086
7087         return 0;
7088 }
7089 EXPORT_SYMBOL_GPL(init_dummy_netdev);
7090
7091
7092 /**
7093  *      register_netdev - register a network device
7094  *      @dev: device to register
7095  *
7096  *      Take a completed network device structure and add it to the kernel
7097  *      interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
7098  *      chain. 0 is returned on success. A negative errno code is returned
7099  *      on a failure to set up the device, or if the name is a duplicate.
7100  *
7101  *      This is a wrapper around register_netdevice that takes the rtnl semaphore
7102  *      and expands the device name if you passed a format string to
7103  *      alloc_netdev.
7104  */
7105 int register_netdev(struct net_device *dev)
7106 {
7107         int err;
7108
7109         rtnl_lock();
7110         err = register_netdevice(dev);
7111         rtnl_unlock();
7112         return err;
7113 }
7114 EXPORT_SYMBOL(register_netdev);
7115
7116 int netdev_refcnt_read(const struct net_device *dev)
7117 {
7118         int i, refcnt = 0;
7119
7120         for_each_possible_cpu(i)
7121                 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
7122         return refcnt;
7123 }
7124 EXPORT_SYMBOL(netdev_refcnt_read);
7125
7126 /**
7127  * netdev_wait_allrefs - wait until all references are gone.
7128  * @dev: target net_device
7129  *
7130  * This is called when unregistering network devices.
7131  *
7132  * Any protocol or device that holds a reference should register
7133  * for netdevice notification, and cleanup and put back the
7134  * reference if they receive an UNREGISTER event.
7135  * We can get stuck here if buggy protocols don't correctly
7136  * call dev_put.
7137  */
7138 static void netdev_wait_allrefs(struct net_device *dev)
7139 {
7140         unsigned long rebroadcast_time, warning_time;
7141         int refcnt;
7142
7143         linkwatch_forget_dev(dev);
7144
7145         rebroadcast_time = warning_time = jiffies;
7146         refcnt = netdev_refcnt_read(dev);
7147
7148         while (refcnt != 0) {
7149                 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
7150                         rtnl_lock();
7151
7152                         /* Rebroadcast unregister notification */
7153                         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7154
7155                         __rtnl_unlock();
7156                         rcu_barrier();
7157                         rtnl_lock();
7158
7159                         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7160                         if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
7161                                      &dev->state)) {
7162                                 /* We must not have linkwatch events
7163                                  * pending on unregister. If this
7164                                  * happens, we simply run the queue
7165                                  * unscheduled, resulting in a noop
7166                                  * for this device.
7167                                  */
7168                                 linkwatch_run_queue();
7169                         }
7170
7171                         __rtnl_unlock();
7172
7173                         rebroadcast_time = jiffies;
7174                 }
7175
7176                 msleep(250);
7177
7178                 refcnt = netdev_refcnt_read(dev);
7179
7180                 if (time_after(jiffies, warning_time + 10 * HZ)) {
7181                         pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
7182                                  dev->name, refcnt);
7183                         warning_time = jiffies;
7184                 }
7185         }
7186 }
7187
7188 /* The sequence is:
7189  *
7190  *      rtnl_lock();
7191  *      ...
7192  *      register_netdevice(x1);
7193  *      register_netdevice(x2);
7194  *      ...
7195  *      unregister_netdevice(y1);
7196  *      unregister_netdevice(y2);
7197  *      ...
7198  *      rtnl_unlock();
7199  *      free_netdev(y1);
7200  *      free_netdev(y2);
7201  *
7202  * We are invoked by rtnl_unlock().
7203  * This allows us to deal with problems:
7204  * 1) We can delete sysfs objects which invoke hotplug
7205  *    without deadlocking with linkwatch via keventd.
7206  * 2) Since we run with the RTNL semaphore not held, we can sleep
7207  *    safely in order to wait for the netdev refcnt to drop to zero.
7208  *
7209  * We must not return until all unregister events added during
7210  * the interval the lock was held have been completed.
7211  */
7212 void netdev_run_todo(void)
7213 {
7214         struct list_head list;
7215
7216         /* Snapshot list, allow later requests */
7217         list_replace_init(&net_todo_list, &list);
7218
7219         __rtnl_unlock();
7220
7221
7222         /* Wait for rcu callbacks to finish before next phase */
7223         if (!list_empty(&list))
7224                 rcu_barrier();
7225
7226         while (!list_empty(&list)) {
7227                 struct net_device *dev
7228                         = list_first_entry(&list, struct net_device, todo_list);
7229                 list_del(&dev->todo_list);
7230
7231                 rtnl_lock();
7232                 call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7233                 __rtnl_unlock();
7234
7235                 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
7236                         pr_err("network todo '%s' but state %d\n",
7237                                dev->name, dev->reg_state);
7238                         dump_stack();
7239                         continue;
7240                 }
7241
7242                 dev->reg_state = NETREG_UNREGISTERED;
7243
7244                 netdev_wait_allrefs(dev);
7245
7246                 /* paranoia */
7247                 BUG_ON(netdev_refcnt_read(dev));
7248                 BUG_ON(!list_empty(&dev->ptype_all));
7249                 BUG_ON(!list_empty(&dev->ptype_specific));
7250                 WARN_ON(rcu_access_pointer(dev->ip_ptr));
7251                 WARN_ON(rcu_access_pointer(dev->ip6_ptr));
7252                 WARN_ON(dev->dn_ptr);
7253
7254                 if (dev->destructor)
7255                         dev->destructor(dev);
7256
7257                 /* Report a network device has been unregistered */
7258                 rtnl_lock();
7259                 dev_net(dev)->dev_unreg_count--;
7260                 __rtnl_unlock();
7261                 wake_up(&netdev_unregistering_wq);
7262
7263                 /* Free network device */
7264                 kobject_put(&dev->dev.kobj);
7265         }
7266 }
7267
7268 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
7269  * all the same fields in the same order as net_device_stats, with only
7270  * the type differing, but rtnl_link_stats64 may have additional fields
7271  * at the end for newer counters.
7272  */
7273 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
7274                              const struct net_device_stats *netdev_stats)
7275 {
7276 #if BITS_PER_LONG == 64
7277         BUILD_BUG_ON(sizeof(*stats64) < sizeof(*netdev_stats));
7278         memcpy(stats64, netdev_stats, sizeof(*stats64));
7279         /* zero out counters that only exist in rtnl_link_stats64 */
7280         memset((char *)stats64 + sizeof(*netdev_stats), 0,
7281                sizeof(*stats64) - sizeof(*netdev_stats));
7282 #else
7283         size_t i, n = sizeof(*netdev_stats) / sizeof(unsigned long);
7284         const unsigned long *src = (const unsigned long *)netdev_stats;
7285         u64 *dst = (u64 *)stats64;
7286
7287         BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
7288         for (i = 0; i < n; i++)
7289                 dst[i] = src[i];
7290         /* zero out counters that only exist in rtnl_link_stats64 */
7291         memset((char *)stats64 + n * sizeof(u64), 0,
7292                sizeof(*stats64) - n * sizeof(u64));
7293 #endif
7294 }
7295 EXPORT_SYMBOL(netdev_stats_to_stats64);
7296
7297 /**
7298  *      dev_get_stats   - get network device statistics
7299  *      @dev: device to get statistics from
7300  *      @storage: place to store stats
7301  *
7302  *      Get network statistics from device. Return @storage.
7303  *      The device driver may provide its own method by setting
7304  *      dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
7305  *      otherwise the internal statistics structure is used.
7306  */
7307 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
7308                                         struct rtnl_link_stats64 *storage)
7309 {
7310         const struct net_device_ops *ops = dev->netdev_ops;
7311
7312         if (ops->ndo_get_stats64) {
7313                 memset(storage, 0, sizeof(*storage));
7314                 ops->ndo_get_stats64(dev, storage);
7315         } else if (ops->ndo_get_stats) {
7316                 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
7317         } else {
7318                 netdev_stats_to_stats64(storage, &dev->stats);
7319         }
7320         storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
7321         storage->tx_dropped += atomic_long_read(&dev->tx_dropped);
7322         storage->rx_nohandler += atomic_long_read(&dev->rx_nohandler);
7323         return storage;
7324 }
7325 EXPORT_SYMBOL(dev_get_stats);
7326
7327 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
7328 {
7329         struct netdev_queue *queue = dev_ingress_queue(dev);
7330
7331 #ifdef CONFIG_NET_CLS_ACT
7332         if (queue)
7333                 return queue;
7334         queue = kzalloc(sizeof(*queue), GFP_KERNEL);
7335         if (!queue)
7336                 return NULL;
7337         netdev_init_one_queue(dev, queue, NULL);
7338         RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
7339         queue->qdisc_sleeping = &noop_qdisc;
7340         rcu_assign_pointer(dev->ingress_queue, queue);
7341 #endif
7342         return queue;
7343 }
7344
7345 static const struct ethtool_ops default_ethtool_ops;
7346
7347 void netdev_set_default_ethtool_ops(struct net_device *dev,
7348                                     const struct ethtool_ops *ops)
7349 {
7350         if (dev->ethtool_ops == &default_ethtool_ops)
7351                 dev->ethtool_ops = ops;
7352 }
7353 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
7354
7355 void netdev_freemem(struct net_device *dev)
7356 {
7357         char *addr = (char *)dev - dev->padded;
7358
7359         kvfree(addr);
7360 }
7361
7362 /**
7363  *      alloc_netdev_mqs - allocate network device
7364  *      @sizeof_priv:           size of private data to allocate space for
7365  *      @name:                  device name format string
7366  *      @name_assign_type:      origin of device name
7367  *      @setup:                 callback to initialize device
7368  *      @txqs:                  the number of TX subqueues to allocate
7369  *      @rxqs:                  the number of RX subqueues to allocate
7370  *
7371  *      Allocates a struct net_device with private data area for driver use
7372  *      and performs basic initialization.  Also allocates subqueue structs
7373  *      for each queue on the device.
7374  */
7375 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
7376                 unsigned char name_assign_type,
7377                 void (*setup)(struct net_device *),
7378                 unsigned int txqs, unsigned int rxqs)
7379 {
7380         struct net_device *dev;
7381         size_t alloc_size;
7382         struct net_device *p;
7383
7384         BUG_ON(strlen(name) >= sizeof(dev->name));
7385
7386         if (txqs < 1) {
7387                 pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
7388                 return NULL;
7389         }
7390
7391 #ifdef CONFIG_SYSFS
7392         if (rxqs < 1) {
7393                 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
7394                 return NULL;
7395         }
7396 #endif
7397
7398         alloc_size = sizeof(struct net_device);
7399         if (sizeof_priv) {
7400                 /* ensure 32-byte alignment of private area */
7401                 alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
7402                 alloc_size += sizeof_priv;
7403         }
7404         /* ensure 32-byte alignment of whole construct */
7405         alloc_size += NETDEV_ALIGN - 1;
7406
7407         p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
7408         if (!p)
7409                 p = vzalloc(alloc_size);
7410         if (!p)
7411                 return NULL;
7412
7413         dev = PTR_ALIGN(p, NETDEV_ALIGN);
7414         dev->padded = (char *)dev - (char *)p;
7415
7416         dev->pcpu_refcnt = alloc_percpu(int);
7417         if (!dev->pcpu_refcnt)
7418                 goto free_dev;
7419
7420         if (dev_addr_init(dev))
7421                 goto free_pcpu;
7422
7423         dev_mc_init(dev);
7424         dev_uc_init(dev);
7425
7426         dev_net_set(dev, &init_net);
7427
7428         dev->gso_max_size = GSO_MAX_SIZE;
7429         dev->gso_max_segs = GSO_MAX_SEGS;
7430         dev->gso_min_segs = 0;
7431
7432         INIT_LIST_HEAD(&dev->napi_list);
7433         INIT_LIST_HEAD(&dev->unreg_list);
7434         INIT_LIST_HEAD(&dev->close_list);
7435         INIT_LIST_HEAD(&dev->link_watch_list);
7436         INIT_LIST_HEAD(&dev->adj_list.upper);
7437         INIT_LIST_HEAD(&dev->adj_list.lower);
7438         INIT_LIST_HEAD(&dev->all_adj_list.upper);
7439         INIT_LIST_HEAD(&dev->all_adj_list.lower);
7440         INIT_LIST_HEAD(&dev->ptype_all);
7441         INIT_LIST_HEAD(&dev->ptype_specific);
7442         dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
7443         setup(dev);
7444
7445         if (!dev->tx_queue_len) {
7446                 dev->priv_flags |= IFF_NO_QUEUE;
7447                 dev->tx_queue_len = 1;
7448         }
7449
7450         dev->num_tx_queues = txqs;
7451         dev->real_num_tx_queues = txqs;
7452         if (netif_alloc_netdev_queues(dev))
7453                 goto free_all;
7454
7455 #ifdef CONFIG_SYSFS
7456         dev->num_rx_queues = rxqs;
7457         dev->real_num_rx_queues = rxqs;
7458         if (netif_alloc_rx_queues(dev))
7459                 goto free_all;
7460 #endif
7461
7462         strcpy(dev->name, name);
7463         dev->name_assign_type = name_assign_type;
7464         dev->group = INIT_NETDEV_GROUP;
7465         if (!dev->ethtool_ops)
7466                 dev->ethtool_ops = &default_ethtool_ops;
7467
7468         nf_hook_ingress_init(dev);
7469
7470         return dev;
7471
7472 free_all:
7473         free_netdev(dev);
7474         return NULL;
7475
7476 free_pcpu:
7477         free_percpu(dev->pcpu_refcnt);
7478 free_dev:
7479         netdev_freemem(dev);
7480         return NULL;
7481 }
7482 EXPORT_SYMBOL(alloc_netdev_mqs);
7483
7484 /**
7485  *      free_netdev - free network device
7486  *      @dev: device
7487  *
7488  *      This function does the last stage of destroying an allocated device
7489  *      interface. The reference to the device object is released.
7490  *      If this is the last reference then it will be freed.
7491  *      Must be called in process context.
7492  */
7493 void free_netdev(struct net_device *dev)
7494 {
7495         struct napi_struct *p, *n;
7496
7497         might_sleep();
7498         netif_free_tx_queues(dev);
7499 #ifdef CONFIG_SYSFS
7500         kvfree(dev->_rx);
7501 #endif
7502
7503         kfree(rcu_dereference_protected(dev->ingress_queue, 1));
7504
7505         /* Flush device addresses */
7506         dev_addr_flush(dev);
7507
7508         list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
7509                 netif_napi_del(p);
7510
7511         free_percpu(dev->pcpu_refcnt);
7512         dev->pcpu_refcnt = NULL;
7513
7514         /*  Compatibility with error handling in drivers */
7515         if (dev->reg_state == NETREG_UNINITIALIZED) {
7516                 netdev_freemem(dev);
7517                 return;
7518         }
7519
7520         BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
7521         dev->reg_state = NETREG_RELEASED;
7522
7523         /* will free via device release */
7524         put_device(&dev->dev);
7525 }
7526 EXPORT_SYMBOL(free_netdev);
7527
7528 /**
7529  *      synchronize_net -  Synchronize with packet receive processing
7530  *
7531  *      Wait for packets currently being received to be done.
7532  *      Does not block later packets from starting.
7533  */
7534 void synchronize_net(void)
7535 {
7536         might_sleep();
7537         if (rtnl_is_locked())
7538                 synchronize_rcu_expedited();
7539         else
7540                 synchronize_rcu();
7541 }
7542 EXPORT_SYMBOL(synchronize_net);
7543
7544 /**
7545  *      unregister_netdevice_queue - remove device from the kernel
7546  *      @dev: device
7547  *      @head: list
7548  *
7549  *      This function shuts down a device interface and removes it
7550  *      from the kernel tables.
7551  *      If head not NULL, device is queued to be unregistered later.
7552  *
7553  *      Callers must hold the rtnl semaphore.  You may want
7554  *      unregister_netdev() instead of this.
7555  */
7556
7557 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
7558 {
7559         ASSERT_RTNL();
7560
7561         if (head) {
7562                 list_move_tail(&dev->unreg_list, head);
7563         } else {
7564                 rollback_registered(dev);
7565                 /* Finish processing unregister after unlock */
7566                 net_set_todo(dev);
7567         }
7568 }
7569 EXPORT_SYMBOL(unregister_netdevice_queue);
7570
7571 /**
7572  *      unregister_netdevice_many - unregister many devices
7573  *      @head: list of devices
7574  *
7575  *  Note: As most callers use a stack allocated list_head,
7576  *  we force a list_del() to make sure stack wont be corrupted later.
7577  */
7578 void unregister_netdevice_many(struct list_head *head)
7579 {
7580         struct net_device *dev;
7581
7582         if (!list_empty(head)) {
7583                 rollback_registered_many(head);
7584                 list_for_each_entry(dev, head, unreg_list)
7585                         net_set_todo(dev);
7586                 list_del(head);
7587         }
7588 }
7589 EXPORT_SYMBOL(unregister_netdevice_many);
7590
7591 /**
7592  *      unregister_netdev - remove device from the kernel
7593  *      @dev: device
7594  *
7595  *      This function shuts down a device interface and removes it
7596  *      from the kernel tables.
7597  *
7598  *      This is just a wrapper for unregister_netdevice that takes
7599  *      the rtnl semaphore.  In general you want to use this and not
7600  *      unregister_netdevice.
7601  */
7602 void unregister_netdev(struct net_device *dev)
7603 {
7604         rtnl_lock();
7605         unregister_netdevice(dev);
7606         rtnl_unlock();
7607 }
7608 EXPORT_SYMBOL(unregister_netdev);
7609
7610 /**
7611  *      dev_change_net_namespace - move device to different nethost namespace
7612  *      @dev: device
7613  *      @net: network namespace
7614  *      @pat: If not NULL name pattern to try if the current device name
7615  *            is already taken in the destination network namespace.
7616  *
7617  *      This function shuts down a device interface and moves it
7618  *      to a new network namespace. On success 0 is returned, on
7619  *      a failure a netagive errno code is returned.
7620  *
7621  *      Callers must hold the rtnl semaphore.
7622  */
7623
7624 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
7625 {
7626         int err;
7627
7628         ASSERT_RTNL();
7629
7630         /* Don't allow namespace local devices to be moved. */
7631         err = -EINVAL;
7632         if (dev->features & NETIF_F_NETNS_LOCAL)
7633                 goto out;
7634
7635         /* Ensure the device has been registrered */
7636         if (dev->reg_state != NETREG_REGISTERED)
7637                 goto out;
7638
7639         /* Get out if there is nothing todo */
7640         err = 0;
7641         if (net_eq(dev_net(dev), net))
7642                 goto out;
7643
7644         /* Pick the destination device name, and ensure
7645          * we can use it in the destination network namespace.
7646          */
7647         err = -EEXIST;
7648         if (__dev_get_by_name(net, dev->name)) {
7649                 /* We get here if we can't use the current device name */
7650                 if (!pat)
7651                         goto out;
7652                 if (dev_get_valid_name(net, dev, pat) < 0)
7653                         goto out;
7654         }
7655
7656         /*
7657          * And now a mini version of register_netdevice unregister_netdevice.
7658          */
7659
7660         /* If device is running close it first. */
7661         dev_close(dev);
7662
7663         /* And unlink it from device chain */
7664         err = -ENODEV;
7665         unlist_netdevice(dev);
7666
7667         synchronize_net();
7668
7669         /* Shutdown queueing discipline. */
7670         dev_shutdown(dev);
7671
7672         /* Notify protocols, that we are about to destroy
7673            this device. They should clean all the things.
7674
7675            Note that dev->reg_state stays at NETREG_REGISTERED.
7676            This is wanted because this way 8021q and macvlan know
7677            the device is just moving and can keep their slaves up.
7678         */
7679         call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
7680         rcu_barrier();
7681         call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev);
7682         rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL);
7683
7684         /*
7685          *      Flush the unicast and multicast chains
7686          */
7687         dev_uc_flush(dev);
7688         dev_mc_flush(dev);
7689
7690         /* Send a netdev-removed uevent to the old namespace */
7691         kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
7692         netdev_adjacent_del_links(dev);
7693
7694         /* Actually switch the network namespace */
7695         dev_net_set(dev, net);
7696
7697         /* If there is an ifindex conflict assign a new one */
7698         if (__dev_get_by_index(net, dev->ifindex))
7699                 dev->ifindex = dev_new_index(net);
7700
7701         /* Send a netdev-add uevent to the new namespace */
7702         kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
7703         netdev_adjacent_add_links(dev);
7704
7705         /* Fixup kobjects */
7706         err = device_rename(&dev->dev, dev->name);
7707         WARN_ON(err);
7708
7709         /* Add the device back in the hashes */
7710         list_netdevice(dev);
7711
7712         /* Notify protocols, that a new device appeared. */
7713         call_netdevice_notifiers(NETDEV_REGISTER, dev);
7714
7715         /*
7716          *      Prevent userspace races by waiting until the network
7717          *      device is fully setup before sending notifications.
7718          */
7719         rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL);
7720
7721         synchronize_net();
7722         err = 0;
7723 out:
7724         return err;
7725 }
7726 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
7727
7728 static int dev_cpu_callback(struct notifier_block *nfb,
7729                             unsigned long action,
7730                             void *ocpu)
7731 {
7732         struct sk_buff **list_skb;
7733         struct sk_buff *skb;
7734         unsigned int cpu, oldcpu = (unsigned long)ocpu;
7735         struct softnet_data *sd, *oldsd;
7736
7737         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
7738                 return NOTIFY_OK;
7739
7740         local_irq_disable();
7741         cpu = smp_processor_id();
7742         sd = &per_cpu(softnet_data, cpu);
7743         oldsd = &per_cpu(softnet_data, oldcpu);
7744
7745         /* Find end of our completion_queue. */
7746         list_skb = &sd->completion_queue;
7747         while (*list_skb)
7748                 list_skb = &(*list_skb)->next;
7749         /* Append completion queue from offline CPU. */
7750         *list_skb = oldsd->completion_queue;
7751         oldsd->completion_queue = NULL;
7752
7753         /* Append output queue from offline CPU. */
7754         if (oldsd->output_queue) {
7755                 *sd->output_queue_tailp = oldsd->output_queue;
7756                 sd->output_queue_tailp = oldsd->output_queue_tailp;
7757                 oldsd->output_queue = NULL;
7758                 oldsd->output_queue_tailp = &oldsd->output_queue;
7759         }
7760         /* Append NAPI poll list from offline CPU, with one exception :
7761          * process_backlog() must be called by cpu owning percpu backlog.
7762          * We properly handle process_queue & input_pkt_queue later.
7763          */
7764         while (!list_empty(&oldsd->poll_list)) {
7765                 struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
7766                                                             struct napi_struct,
7767                                                             poll_list);
7768
7769                 list_del_init(&napi->poll_list);
7770                 if (napi->poll == process_backlog)
7771                         napi->state = 0;
7772                 else
7773                         ____napi_schedule(sd, napi);
7774         }
7775
7776         raise_softirq_irqoff(NET_TX_SOFTIRQ);
7777         local_irq_enable();
7778
7779         /* Process offline CPU's input_pkt_queue */
7780         while ((skb = __skb_dequeue(&oldsd->process_queue))) {
7781                 netif_rx_ni(skb);
7782                 input_queue_head_incr(oldsd);
7783         }
7784         while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
7785                 netif_rx_ni(skb);
7786                 input_queue_head_incr(oldsd);
7787         }
7788
7789         return NOTIFY_OK;
7790 }
7791
7792
7793 /**
7794  *      netdev_increment_features - increment feature set by one
7795  *      @all: current feature set
7796  *      @one: new feature set
7797  *      @mask: mask feature set
7798  *
7799  *      Computes a new feature set after adding a device with feature set
7800  *      @one to the master device with current feature set @all.  Will not
7801  *      enable anything that is off in @mask. Returns the new feature set.
7802  */
7803 netdev_features_t netdev_increment_features(netdev_features_t all,
7804         netdev_features_t one, netdev_features_t mask)
7805 {
7806         if (mask & NETIF_F_HW_CSUM)
7807                 mask |= NETIF_F_CSUM_MASK;
7808         mask |= NETIF_F_VLAN_CHALLENGED;
7809
7810         all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
7811         all &= one | ~NETIF_F_ALL_FOR_ALL;
7812
7813         /* If one device supports hw checksumming, set for all. */
7814         if (all & NETIF_F_HW_CSUM)
7815                 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
7816
7817         return all;
7818 }
7819 EXPORT_SYMBOL(netdev_increment_features);
7820
7821 static struct hlist_head * __net_init netdev_create_hash(void)
7822 {
7823         int i;
7824         struct hlist_head *hash;
7825
7826         hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
7827         if (hash != NULL)
7828                 for (i = 0; i < NETDEV_HASHENTRIES; i++)
7829                         INIT_HLIST_HEAD(&hash[i]);
7830
7831         return hash;
7832 }
7833
7834 /* Initialize per network namespace state */
7835 static int __net_init netdev_init(struct net *net)
7836 {
7837         if (net != &init_net)
7838                 INIT_LIST_HEAD(&net->dev_base_head);
7839
7840         net->dev_name_head = netdev_create_hash();
7841         if (net->dev_name_head == NULL)
7842                 goto err_name;
7843
7844         net->dev_index_head = netdev_create_hash();
7845         if (net->dev_index_head == NULL)
7846                 goto err_idx;
7847
7848         return 0;
7849
7850 err_idx:
7851         kfree(net->dev_name_head);
7852 err_name:
7853         return -ENOMEM;
7854 }
7855
7856 /**
7857  *      netdev_drivername - network driver for the device
7858  *      @dev: network device
7859  *
7860  *      Determine network driver for device.
7861  */
7862 const char *netdev_drivername(const struct net_device *dev)
7863 {
7864         const struct device_driver *driver;
7865         const struct device *parent;
7866         const char *empty = "";
7867
7868         parent = dev->dev.parent;
7869         if (!parent)
7870                 return empty;
7871
7872         driver = parent->driver;
7873         if (driver && driver->name)
7874                 return driver->name;
7875         return empty;
7876 }
7877
7878 static void __netdev_printk(const char *level, const struct net_device *dev,
7879                             struct va_format *vaf)
7880 {
7881         if (dev && dev->dev.parent) {
7882                 dev_printk_emit(level[1] - '0',
7883                                 dev->dev.parent,
7884                                 "%s %s %s%s: %pV",
7885                                 dev_driver_string(dev->dev.parent),
7886                                 dev_name(dev->dev.parent),
7887                                 netdev_name(dev), netdev_reg_state(dev),
7888                                 vaf);
7889         } else if (dev) {
7890                 printk("%s%s%s: %pV",
7891                        level, netdev_name(dev), netdev_reg_state(dev), vaf);
7892         } else {
7893                 printk("%s(NULL net_device): %pV", level, vaf);
7894         }
7895 }
7896
7897 void netdev_printk(const char *level, const struct net_device *dev,
7898                    const char *format, ...)
7899 {
7900         struct va_format vaf;
7901         va_list args;
7902
7903         va_start(args, format);
7904
7905         vaf.fmt = format;
7906         vaf.va = &args;
7907
7908         __netdev_printk(level, dev, &vaf);
7909
7910         va_end(args);
7911 }
7912 EXPORT_SYMBOL(netdev_printk);
7913
7914 #define define_netdev_printk_level(func, level)                 \
7915 void func(const struct net_device *dev, const char *fmt, ...)   \
7916 {                                                               \
7917         struct va_format vaf;                                   \
7918         va_list args;                                           \
7919                                                                 \
7920         va_start(args, fmt);                                    \
7921                                                                 \
7922         vaf.fmt = fmt;                                          \
7923         vaf.va = &args;                                         \
7924                                                                 \
7925         __netdev_printk(level, dev, &vaf);                      \
7926                                                                 \
7927         va_end(args);                                           \
7928 }                                                               \
7929 EXPORT_SYMBOL(func);
7930
7931 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
7932 define_netdev_printk_level(netdev_alert, KERN_ALERT);
7933 define_netdev_printk_level(netdev_crit, KERN_CRIT);
7934 define_netdev_printk_level(netdev_err, KERN_ERR);
7935 define_netdev_printk_level(netdev_warn, KERN_WARNING);
7936 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
7937 define_netdev_printk_level(netdev_info, KERN_INFO);
7938
7939 static void __net_exit netdev_exit(struct net *net)
7940 {
7941         kfree(net->dev_name_head);
7942         kfree(net->dev_index_head);
7943 }
7944
7945 static struct pernet_operations __net_initdata netdev_net_ops = {
7946         .init = netdev_init,
7947         .exit = netdev_exit,
7948 };
7949
7950 static void __net_exit default_device_exit(struct net *net)
7951 {
7952         struct net_device *dev, *aux;
7953         /*
7954          * Push all migratable network devices back to the
7955          * initial network namespace
7956          */
7957         rtnl_lock();
7958         for_each_netdev_safe(net, dev, aux) {
7959                 int err;
7960                 char fb_name[IFNAMSIZ];
7961
7962                 /* Ignore unmoveable devices (i.e. loopback) */
7963                 if (dev->features & NETIF_F_NETNS_LOCAL)
7964                         continue;
7965
7966                 /* Leave virtual devices for the generic cleanup */
7967                 if (dev->rtnl_link_ops)
7968                         continue;
7969
7970                 /* Push remaining network devices to init_net */
7971                 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
7972                 err = dev_change_net_namespace(dev, &init_net, fb_name);
7973                 if (err) {
7974                         pr_emerg("%s: failed to move %s to init_net: %d\n",
7975                                  __func__, dev->name, err);
7976                         BUG();
7977                 }
7978         }
7979         rtnl_unlock();
7980 }
7981
7982 static void __net_exit rtnl_lock_unregistering(struct list_head *net_list)
7983 {
7984         /* Return with the rtnl_lock held when there are no network
7985          * devices unregistering in any network namespace in net_list.
7986          */
7987         struct net *net;
7988         bool unregistering;
7989         DEFINE_WAIT_FUNC(wait, woken_wake_function);
7990
7991         add_wait_queue(&netdev_unregistering_wq, &wait);
7992         for (;;) {
7993                 unregistering = false;
7994                 rtnl_lock();
7995                 list_for_each_entry(net, net_list, exit_list) {
7996                         if (net->dev_unreg_count > 0) {
7997                                 unregistering = true;
7998                                 break;
7999                         }
8000                 }
8001                 if (!unregistering)
8002                         break;
8003                 __rtnl_unlock();
8004
8005                 wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
8006         }
8007         remove_wait_queue(&netdev_unregistering_wq, &wait);
8008 }
8009
8010 static void __net_exit default_device_exit_batch(struct list_head *net_list)
8011 {
8012         /* At exit all network devices most be removed from a network
8013          * namespace.  Do this in the reverse order of registration.
8014          * Do this across as many network namespaces as possible to
8015          * improve batching efficiency.
8016          */
8017         struct net_device *dev;
8018         struct net *net;
8019         LIST_HEAD(dev_kill_list);
8020
8021         /* To prevent network device cleanup code from dereferencing
8022          * loopback devices or network devices that have been freed
8023          * wait here for all pending unregistrations to complete,
8024          * before unregistring the loopback device and allowing the
8025          * network namespace be freed.
8026          *
8027          * The netdev todo list containing all network devices
8028          * unregistrations that happen in default_device_exit_batch
8029          * will run in the rtnl_unlock() at the end of
8030          * default_device_exit_batch.
8031          */
8032         rtnl_lock_unregistering(net_list);
8033         list_for_each_entry(net, net_list, exit_list) {
8034                 for_each_netdev_reverse(net, dev) {
8035                         if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
8036                                 dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
8037                         else
8038                                 unregister_netdevice_queue(dev, &dev_kill_list);
8039                 }
8040         }
8041         unregister_netdevice_many(&dev_kill_list);
8042         rtnl_unlock();
8043 }
8044
8045 static struct pernet_operations __net_initdata default_device_ops = {
8046         .exit = default_device_exit,
8047         .exit_batch = default_device_exit_batch,
8048 };
8049
8050 /*
8051  *      Initialize the DEV module. At boot time this walks the device list and
8052  *      unhooks any devices that fail to initialise (normally hardware not
8053  *      present) and leaves us with a valid list of present and active devices.
8054  *
8055  */
8056
8057 /*
8058  *       This is called single threaded during boot, so no need
8059  *       to take the rtnl semaphore.
8060  */
8061 static int __init net_dev_init(void)
8062 {
8063         int i, rc = -ENOMEM;
8064
8065         BUG_ON(!dev_boot_phase);
8066
8067         if (dev_proc_init())
8068                 goto out;
8069
8070         if (netdev_kobject_init())
8071                 goto out;
8072
8073         INIT_LIST_HEAD(&ptype_all);
8074         for (i = 0; i < PTYPE_HASH_SIZE; i++)
8075                 INIT_LIST_HEAD(&ptype_base[i]);
8076
8077         INIT_LIST_HEAD(&offload_base);
8078
8079         if (register_pernet_subsys(&netdev_net_ops))
8080                 goto out;
8081
8082         /*
8083          *      Initialise the packet receive queues.
8084          */
8085
8086         for_each_possible_cpu(i) {
8087                 struct softnet_data *sd = &per_cpu(softnet_data, i);
8088
8089                 skb_queue_head_init(&sd->input_pkt_queue);
8090                 skb_queue_head_init(&sd->process_queue);
8091                 INIT_LIST_HEAD(&sd->poll_list);
8092                 sd->output_queue_tailp = &sd->output_queue;
8093 #ifdef CONFIG_RPS
8094                 sd->csd.func = rps_trigger_softirq;
8095                 sd->csd.info = sd;
8096                 sd->cpu = i;
8097 #endif
8098
8099                 sd->backlog.poll = process_backlog;
8100                 sd->backlog.weight = weight_p;
8101         }
8102
8103         dev_boot_phase = 0;
8104
8105         /* The loopback device is special if any other network devices
8106          * is present in a network namespace the loopback device must
8107          * be present. Since we now dynamically allocate and free the
8108          * loopback device ensure this invariant is maintained by
8109          * keeping the loopback device as the first device on the
8110          * list of network devices.  Ensuring the loopback devices
8111          * is the first device that appears and the last network device
8112          * that disappears.
8113          */
8114         if (register_pernet_device(&loopback_net_ops))
8115                 goto out;
8116
8117         if (register_pernet_device(&default_device_ops))
8118                 goto out;
8119
8120         open_softirq(NET_TX_SOFTIRQ, net_tx_action);
8121         open_softirq(NET_RX_SOFTIRQ, net_rx_action);
8122
8123         hotcpu_notifier(dev_cpu_callback, 0);
8124         dst_subsys_init();
8125         rc = 0;
8126 out:
8127         return rc;
8128 }
8129
8130 subsys_initcall(net_dev_init);