net/dst: use a smaller percpu_counter batch for dst entries accounting

percpu_counter_add() uses a default batch size which is quite big
on platforms with 256 cpus. (2*256 -> 512)

This means dst_entries_get_fast() can be off by +/- 2*(nr_cpus^2)
(131072 on servers with 256 cpus)

Reduce the batch size to something more reasonable, and
add logic to ip6_dst_gc() to call dst_entries_get_slow()
before calling the _very_ expensive fib6_run_gc() function.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/dst_ops.h b/include/net/dst_ops.h
index 443863c7b8da362476c15fd290ac2a32a8aa86e3..88ff7bb2bb9bd950cc54fd5e0ae4573d4c66873d 100644 (file)
--- a/include/net/dst_ops.h
+++ b/include/net/dst_ops.h
@@ -53,9 +53,11 @@ static inline int dst_entries_get_slow(struct dst_ops *dst)
        return percpu_counter_sum_positive(&dst->pcpuc_entries);
 }
 
+#define DST_PERCPU_COUNTER_BATCH 32
 static inline void dst_entries_add(struct dst_ops *dst, int val)
 {
-       percpu_counter_add(&dst->pcpuc_entries, val);
+       percpu_counter_add_batch(&dst->pcpuc_entries, val,
+                                DST_PERCPU_COUNTER_BATCH);
 }
 
 static inline int dst_entries_init(struct dst_ops *dst)

diff --git a/net/core/dst.c b/net/core/dst.c
index 193af526e908afa4b868cf128470f0fbc3850698..d6b6ced0d451a39c0ccb88ae39dba225ea9f5705 100644 (file)
--- a/net/core/dst.c
+++ b/net/core/dst.c
@@ -81,11 +81,11 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
 {
        struct dst_entry *dst;
 
-       if (ops->gc && dst_entries_get_fast(ops) > ops->gc_thresh) {
+       if (ops->gc &&
+           !(flags & DST_NOCOUNT) &&
+           dst_entries_get_fast(ops) > ops->gc_thresh) {
                if (ops->gc(ops)) {
-                       printk_ratelimited(KERN_NOTICE "Route cache is full: "
-                                          "consider increasing sysctl "
-                                          "net.ipv[4|6].route.max_size.\n");
+                       pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
                        return NULL;
                }
        }

diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index 1ff142393c768f85c495474a1d05e1ae1642301c..a9072dba00f4fb0b61bce1fc0f44a3a81ba702fa 100644 (file)
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -3195,6 +3195,9 @@ static int ip6_dst_gc(struct dst_ops *ops)
        int entries;
 
        entries = dst_entries_get_fast(ops);
+       if (entries > rt_max_size)
+               entries = dst_entries_get_slow(ops);
+
        if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
            entries <= rt_max_size)
                goto out;