neighbour: Add hlist_node to struct neighbour

Add a doubly-linked node to neighbours, so that they
can be deleted without iterating the entire bucket they're in.

Signed-off-by: Gilad Naaman <gnaaman@drivenets.com>
Reviewed-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20241107160444.2913124-2-gnaaman@drivenets.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 3887ed9e5026d21a0fee19ae19d08c6ea8a009de..0402447854c74dae5613f6d7dbf96c5b77af7fdb 100644 (file)
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -136,6 +136,7 @@ struct neigh_statistics {
 
 struct neighbour {
        struct neighbour __rcu  *next;
+       struct hlist_node       hash;
        struct neigh_table      *tbl;
        struct neigh_parms      *parms;
        unsigned long           confirmed;
@@ -191,6 +192,7 @@ struct pneigh_entry {
 
 struct neigh_hash_table {
        struct neighbour __rcu  **hash_buckets;
+       struct hlist_head       *hash_heads;
        unsigned int            hash_shift;
        __u32                   hash_rnd[NEIGH_NUM_HASH_RND];
        struct rcu_head         rcu;

diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 4b871cecd2cee9ccc88e5d29d090c49978cbae9f..5552e6b05c82ee749d152e0631a2697bc2e4fc72 100644 (file)
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -216,6 +216,7 @@ static bool neigh_del(struct neighbour *n, struct neighbour __rcu **np,
                neigh = rcu_dereference_protected(n->next,
                                                  lockdep_is_held(&tbl->lock));
                rcu_assign_pointer(*np, neigh);
+               hlist_del_rcu(&n->hash);
                neigh_mark_dead(n);
                retval = true;
        }
@@ -402,6 +403,7 @@ static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev,
                        rcu_assign_pointer(*np,
                                   rcu_dereference_protected(n->next,
                                                lockdep_is_held(&tbl->lock)));
+                       hlist_del_rcu(&n->hash);
                        write_lock(&n->lock);
                        neigh_del_timer(n);
                        neigh_mark_dead(n);
@@ -529,20 +531,30 @@ static void neigh_get_hash_rnd(u32 *x)
 
 static struct neigh_hash_table *neigh_hash_alloc(unsigned int shift)
 {
+       size_t hash_heads_size = (1 << shift) * sizeof(struct hlist_head);
        size_t size = (1 << shift) * sizeof(struct neighbour *);
-       struct neigh_hash_table *ret;
        struct neighbour __rcu **buckets;
+       struct hlist_head *hash_heads;
+       struct neigh_hash_table *ret;
        int i;
 
        ret = kmalloc(sizeof(*ret), GFP_ATOMIC);
        if (!ret)
                return NULL;
+
        buckets = kvzalloc(size, GFP_ATOMIC);
        if (!buckets) {
                kfree(ret);
                return NULL;
        }
+       hash_heads = kvzalloc(hash_heads_size, GFP_ATOMIC);
+       if (!hash_heads) {
+               kvfree(buckets);
+               kfree(ret);
+               return NULL;
+       }
        ret->hash_buckets = buckets;
+       ret->hash_heads = hash_heads;
        ret->hash_shift = shift;
        for (i = 0; i < NEIGH_NUM_HASH_RND; i++)
                neigh_get_hash_rnd(&ret->hash_rnd[i]);
@@ -556,6 +568,7 @@ static void neigh_hash_free_rcu(struct rcu_head *head)
                                                    rcu);
 
        kvfree(nht->hash_buckets);
+       kvfree(nht->hash_heads);
        kfree(nht);
 }
 
@@ -592,6 +605,8 @@ static struct neigh_hash_table *neigh_hash_grow(struct neigh_table *tbl,
                                                new_nht->hash_buckets[hash],
                                                lockdep_is_held(&tbl->lock)));
                        rcu_assign_pointer(new_nht->hash_buckets[hash], n);
+                       hlist_del_rcu(&n->hash);
+                       hlist_add_head_rcu(&n->hash, &new_nht->hash_heads[hash]);
                }
        }
 
@@ -702,6 +717,7 @@ ___neigh_create(struct neigh_table *tbl, const void *pkey,
                           rcu_dereference_protected(nht->hash_buckets[hash_val],
                                                     lockdep_is_held(&tbl->lock)));
        rcu_assign_pointer(nht->hash_buckets[hash_val], n);
+       hlist_add_head_rcu(&n->hash, &nht->hash_heads[hash_val]);
        write_unlock_bh(&tbl->lock);
        neigh_dbg(2, "neigh %p is created\n", n);
        rc = n;
@@ -987,6 +1003,7 @@ static void neigh_periodic_work(struct work_struct *work)
                                rcu_assign_pointer(*np,
                                        rcu_dereference_protected(n->next,
                                                lockdep_is_held(&tbl->lock)));
+                               hlist_del_rcu(&n->hash);
                                neigh_mark_dead(n);
                                write_unlock(&n->lock);
                                neigh_cleanup_and_release(n);
@@ -3116,6 +3133,7 @@ void __neigh_for_each_release(struct neigh_table *tbl,
                                rcu_assign_pointer(*np,
                                        rcu_dereference_protected(n->next,
                                                lockdep_is_held(&tbl->lock)));
+                               hlist_del_rcu(&n->hash);
                                neigh_mark_dead(n);
                        } else
                                np = &n->next;