net: place xmit recursion in softnet data

This fills a hole in softnet data, so no change in structure size.

Also prepares for xmit_more placement in the same spot;
skb->xmit_more will be removed in followup patch.

Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 78f5ec4ebf64496886e34a078f563291c47fea05..2b25824642fab6545f51c3d2895c08619659cccf 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2659,14 +2659,6 @@ void netdev_freemem(struct net_device *dev);
 void synchronize_net(void);
 int init_dummy_netdev(struct net_device *dev);
 
-DECLARE_PER_CPU(int, xmit_recursion);
-#define XMIT_RECURSION_LIMIT   10
-
-static inline int dev_recursion_level(void)
-{
-       return this_cpu_read(xmit_recursion);
-}
-
 struct net_device *dev_get_by_index(struct net *net, int ifindex);
 struct net_device *__dev_get_by_index(struct net *net, int ifindex);
 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
@@ -3015,6 +3007,11 @@ struct softnet_data {
 #ifdef CONFIG_XFRM_OFFLOAD
        struct sk_buff_head     xfrm_backlog;
 #endif
+       /* written and read only by owning cpu: */
+       struct {
+               u16 recursion;
+               u8  more;
+       } xmit;
 #ifdef CONFIG_RPS
        /* input_queue_head should be written by cpu owning this struct,
         * and only read by other cpus. Worth using a cache line.
@@ -3050,6 +3047,28 @@ static inline void input_queue_tail_incr_save(struct softnet_data *sd,
 
 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 
+static inline int dev_recursion_level(void)
+{
+       return __this_cpu_read(softnet_data.xmit.recursion);
+}
+
+#define XMIT_RECURSION_LIMIT   10
+static inline bool dev_xmit_recursion(void)
+{
+       return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
+                       XMIT_RECURSION_LIMIT);
+}
+
+static inline void dev_xmit_recursion_inc(void)
+{
+       __this_cpu_inc(softnet_data.xmit.recursion);
+}
+
+static inline void dev_xmit_recursion_dec(void)
+{
+       __this_cpu_dec(softnet_data.xmit.recursion);
+}
+
 void __netif_schedule(struct Qdisc *q);
 void netif_schedule_queue(struct netdev_queue *txq);
 
@@ -4409,6 +4428,11 @@ static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
        return ops->ndo_start_xmit(skb, dev);
 }
 
+static inline bool netdev_xmit_more(void)
+{
+       return __this_cpu_read(softnet_data.xmit.more);
+}
+
 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
                                            struct netdev_queue *txq, bool more)
 {

diff --git a/net/core/dev.c b/net/core/dev.c
index 9823b7713f79dec2f8927b5ebb380d3dddd8fbb4..d5b1315218d3c2da4bde7e1ab5478ca543db48fa 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3566,9 +3566,6 @@ static void skb_update_prio(struct sk_buff *skb)
 #define skb_update_prio(skb)
 #endif
 
-DEFINE_PER_CPU(int, xmit_recursion);
-EXPORT_SYMBOL(xmit_recursion);
-
 /**
  *     dev_loopback_xmit - loop back @skb
  *     @net: network namespace this loopback is happening in
@@ -3857,8 +3854,7 @@ static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
                int cpu = smp_processor_id(); /* ok because BHs are off */
 
                if (txq->xmit_lock_owner != cpu) {
-                       if (unlikely(__this_cpu_read(xmit_recursion) >
-                                    XMIT_RECURSION_LIMIT))
+                       if (dev_xmit_recursion())
                                goto recursion_alert;
 
                        skb = validate_xmit_skb(skb, dev, &again);
@@ -3868,9 +3864,9 @@ static int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
                        HARD_TX_LOCK(dev, txq, cpu);
 
                        if (!netif_xmit_stopped(txq)) {
-                               __this_cpu_inc(xmit_recursion);
+                               dev_xmit_recursion_inc();
                                skb = dev_hard_start_xmit(skb, dev, txq, &rc);
-                               __this_cpu_dec(xmit_recursion);
+                               dev_xmit_recursion_dec();
                                if (dev_xmit_complete(rc)) {
                                        HARD_TX_UNLOCK(dev, txq);
                                        goto out;

diff --git a/net/core/filter.c b/net/core/filter.c
index 4a8455757507c49965c234bd4b00e1bfe7f00beb..cdaafa3322dbe6c217f61ed3b2c9f37fab90ead8 100644 (file)
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -2016,7 +2016,7 @@ static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
 {
        int ret;
 
-       if (unlikely(__this_cpu_read(xmit_recursion) > XMIT_RECURSION_LIMIT)) {
+       if (dev_xmit_recursion()) {
                net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
                kfree_skb(skb);
                return -ENETDOWN;
@@ -2024,9 +2024,9 @@ static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
 
        skb->dev = dev;
 
-       __this_cpu_inc(xmit_recursion);
+       dev_xmit_recursion_inc();
        ret = dev_queue_xmit(skb);
-       __this_cpu_dec(xmit_recursion);
+       dev_xmit_recursion_dec();
 
        return ret;
 }