rhashtable: plumb through alloc tag

This gives better memory allocation profiling results; rhashtable
allocations will be accounted to the code that initialized the rhashtable.

[surenb@google.com: undo _noprof additions in the documentation]
Link: https://lkml.kernel.org/r/20240326231453.1206227-1-surenb@google.com
Link: https://lkml.kernel.org/r/20240321163705.3067592-32-surenb@google.com
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Tested-by: Kees Cook <keescook@chromium.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andreas Hindborg <a.hindborg@samsung.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Gary Guo <gary@garyguo.net>
Cc: Miguel Ojeda <ojeda@kernel.org>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
index 2aea7afa1fab9f8a2456753529f0d41022534a5f..0567766e50bdbbb940ff5a0d3ac579668a09af73 100644 (file)
--- a/include/linux/alloc_tag.h
+++ b/include/linux/alloc_tag.h
@@ -152,6 +152,8 @@ static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
        ref->ct = NULL;
 }
 
+#define alloc_tag_record(p)    ((p) = current->alloc_tag)
+
 #else /* CONFIG_MEM_ALLOC_PROFILING */
 
 #define DEFINE_ALLOC_TAG(_alloc_tag)
@@ -159,6 +161,7 @@ static inline bool mem_alloc_profiling_enabled(void) { return false; }
 static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
                                 size_t bytes) {}
 static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
+#define alloc_tag_record(p)    do {} while (0)
 
 #endif /* CONFIG_MEM_ALLOC_PROFILING */
 

diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h
index b6f3797277ff83145797c885111431da4d7fcbd2..015c8298bebc45c20ccc120e061978f2c5f9e25a 100644 (file)
--- a/include/linux/rhashtable-types.h
+++ b/include/linux/rhashtable-types.h
@@ -9,6 +9,7 @@
 #ifndef _LINUX_RHASHTABLE_TYPES_H
 #define _LINUX_RHASHTABLE_TYPES_H
 
+#include <linux/alloc_tag.h>
 #include <linux/atomic.h>
 #include <linux/compiler.h>
 #include <linux/mutex.h>
@@ -88,6 +89,9 @@ struct rhashtable {
        struct mutex                    mutex;
        spinlock_t                      lock;
        atomic_t                        nelems;
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+       struct alloc_tag                *alloc_tag;
+#endif
 };
 
 /**
@@ -127,9 +131,12 @@ struct rhashtable_iter {
        bool end_of_table;
 };
 
-int rhashtable_init(struct rhashtable *ht,
+int rhashtable_init_noprof(struct rhashtable *ht,
                    const struct rhashtable_params *params);
-int rhltable_init(struct rhltable *hlt,
+#define rhashtable_init(...)   alloc_hooks(rhashtable_init_noprof(__VA_ARGS__))
+
+int rhltable_init_noprof(struct rhltable *hlt,
                  const struct rhashtable_params *params);
+#define rhltable_init(...)     alloc_hooks(rhltable_init_noprof(__VA_ARGS__))
 
 #endif /* _LINUX_RHASHTABLE_TYPES_H */

diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 6ae2ba8e06a2138d77e6b0d5f51c67e17658b1a7..dbbed19f8fff99f4fb58889a76a40eaac330f51a 100644 (file)
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -130,7 +130,8 @@ static union nested_table *nested_table_alloc(struct rhashtable *ht,
        if (ntbl)
                return ntbl;
 
-       ntbl = kzalloc(PAGE_SIZE, GFP_ATOMIC);
+       ntbl = alloc_hooks_tag(ht->alloc_tag,
+                       kmalloc_noprof(PAGE_SIZE, GFP_ATOMIC|__GFP_ZERO));
 
        if (ntbl && leaf) {
                for (i = 0; i < PAGE_SIZE / sizeof(ntbl[0]); i++)
@@ -157,7 +158,8 @@ static struct bucket_table *nested_bucket_table_alloc(struct rhashtable *ht,
 
        size = sizeof(*tbl) + sizeof(tbl->buckets[0]);
 
-       tbl = kzalloc(size, gfp);
+       tbl = alloc_hooks_tag(ht->alloc_tag,
+                       kmalloc_noprof(size, gfp|__GFP_ZERO));
        if (!tbl)
                return NULL;
 
@@ -181,7 +183,9 @@ static struct bucket_table *bucket_table_alloc(struct rhashtable *ht,
        int i;
        static struct lock_class_key __key;
 
-       tbl = kvzalloc(struct_size(tbl, buckets, nbuckets), gfp);
+       tbl = alloc_hooks_tag(ht->alloc_tag,
+                       kvmalloc_node_noprof(struct_size(tbl, buckets, nbuckets),
+                                            gfp|__GFP_ZERO, NUMA_NO_NODE));
 
        size = nbuckets;
 
@@ -1016,7 +1020,7 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed)
  *     .obj_hashfn = my_hash_fn,
  * };
  */
-int rhashtable_init(struct rhashtable *ht,
+int rhashtable_init_noprof(struct rhashtable *ht,
                    const struct rhashtable_params *params)
 {
        struct bucket_table *tbl;
@@ -1031,6 +1035,8 @@ int rhashtable_init(struct rhashtable *ht,
        spin_lock_init(&ht->lock);
        memcpy(&ht->p, params, sizeof(*params));
 
+       alloc_tag_record(ht->alloc_tag);
+
        if (params->min_size)
                ht->p.min_size = roundup_pow_of_two(params->min_size);
 
@@ -1076,7 +1082,7 @@ int rhashtable_init(struct rhashtable *ht,
 
        return 0;
 }
-EXPORT_SYMBOL_GPL(rhashtable_init);
+EXPORT_SYMBOL_GPL(rhashtable_init_noprof);
 
 /**
  * rhltable_init - initialize a new hash list table
@@ -1087,15 +1093,15 @@ EXPORT_SYMBOL_GPL(rhashtable_init);
  *
  * See documentation for rhashtable_init.
  */
-int rhltable_init(struct rhltable *hlt, const struct rhashtable_params *params)
+int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params)
 {
        int err;
 
-       err = rhashtable_init(&hlt->ht, params);
+       err = rhashtable_init_noprof(&hlt->ht, params);
        hlt->ht.rhlist = true;
        return err;
 }
-EXPORT_SYMBOL_GPL(rhltable_init);
+EXPORT_SYMBOL_GPL(rhltable_init_noprof);
 
 static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj,
                                void (*free_fn)(void *ptr, void *arg),