Merge tag 'for-linus-6.4-1' of https://github.com/cminyard/linux-ipmi

[linux-block.git] / include / net / dst.h

diff --git a/include/net/dst.h b/include/net/dst.h
index d67fda89cd0fabb887c8f95222f531b7fb470a59..78884429deed82cc4872ff30404af69aee11a333 100644 (file)
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -16,6 +16,7 @@
 #include <linux/bug.h>
 #include <linux/jiffies.h>
 #include <linux/refcount.h>
+#include <linux/rcuref.h>
 #include <net/neighbour.h>
 #include <asm/processor.h>
 #include <linux/indirect_call_wrapper.h>
@@ -61,23 +62,36 @@ struct dst_entry {
        unsigned short          trailer_len;    /* space to reserve at tail */
 
        /*
-        * __refcnt wants to be on a different cache line from
+        * __rcuref wants to be on a different cache line from
         * input/output/ops or performance tanks badly
         */
 #ifdef CONFIG_64BIT
-       atomic_t                __refcnt;       /* 64-bit offset 64 */
+       rcuref_t                __rcuref;       /* 64-bit offset 64 */
 #endif
        int                     __use;
        unsigned long           lastuse;
-       struct lwtunnel_state   *lwtstate;
        struct rcu_head         rcu_head;
        short                   error;
        short                   __pad;
        __u32                   tclassid;
 #ifndef CONFIG_64BIT
-       atomic_t                __refcnt;       /* 32-bit offset 64 */
+       struct lwtunnel_state   *lwtstate;
+       rcuref_t                __rcuref;       /* 32-bit offset 64 */
 #endif
        netdevice_tracker       dev_tracker;
+
+       /*
+        * Used by rtable and rt6_info. Moves lwtstate into the next cache
+        * line on 64bit so that lwtstate does not cause false sharing with
+        * __rcuref under contention of __rcuref. This also puts the
+        * frequently accessed members of rtable and rt6_info out of the
+        * __rcuref cache line.
+        */
+       struct list_head        rt_uncached;
+       struct uncached_list    *rt_uncached_list;
+#ifdef CONFIG_64BIT
+       struct lwtunnel_state   *lwtstate;
+#endif
 };
 
 struct dst_metrics {
@@ -225,10 +239,10 @@ static inline void dst_hold(struct dst_entry *dst)
 {
        /*
         * If your kernel compilation stops here, please check
-        * the placement of __refcnt in struct dst_entry
+        * the placement of __rcuref in struct dst_entry
         */
-       BUILD_BUG_ON(offsetof(struct dst_entry, __refcnt) & 63);
-       WARN_ON(atomic_inc_not_zero(&dst->__refcnt) == 0);
+       BUILD_BUG_ON(offsetof(struct dst_entry, __rcuref) & 63);
+       WARN_ON(!rcuref_get(&dst->__rcuref));
 }
 
 static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
@@ -292,7 +306,7 @@ static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb
  */
 static inline bool dst_hold_safe(struct dst_entry *dst)
 {
-       return atomic_inc_not_zero(&dst->__refcnt);
+       return rcuref_get(&dst->__rcuref);
 }
 
 /**