bpf: Use pcpu_alloc_size() in bpf_mem_free{_rcu}()

For bpf_global_percpu_ma, the pointer passed to bpf_mem_free_rcu() is
allocated by kmalloc() and its size is fixed (16-bytes on x86-64). So
no matter which cache allocates the dynamic per-cpu area, on x86-64
cache[2] will always be used to free the per-cpu area.

Fix the unbalance by checking whether the bpf memory allocator is
per-cpu or not and use pcpu_alloc_size() instead of ksize() to
find the correct cache for per-cpu free.

Signed-off-by: Hou Tao <houtao1@huawei.com>
Link: https://lore.kernel.org/r/20231020133202.4043247-5-houtao@huaweicloud.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h
index d644bbb298af4aa3b2613cf51c426382e971f051..bb1223b2130877d6fee6a2d58926647c354f6cb7 100644 (file)
--- a/include/linux/bpf_mem_alloc.h
+++ b/include/linux/bpf_mem_alloc.h
@@ -11,6 +11,7 @@ struct bpf_mem_caches;
 struct bpf_mem_alloc {
        struct bpf_mem_caches __percpu *caches;
        struct bpf_mem_cache __percpu *cache;
+       bool percpu;
        struct work_struct work;
 };
 

diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
index 776bdf5ffd80bc4790f36cdc9a22d8c75672de87..5308e386380af1098a5e5bbe4fefeaef6eb0593d 100644 (file)
--- a/kernel/bpf/memalloc.c
+++ b/kernel/bpf/memalloc.c
@@ -525,6 +525,7 @@ int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
        /* room for llist_node and per-cpu pointer */
        if (percpu)
                percpu_size = LLIST_NODE_SZ + sizeof(void *);
+       ma->percpu = percpu;
 
        if (size) {
                pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
@@ -874,6 +875,17 @@ void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)
        return !ret ? NULL : ret + LLIST_NODE_SZ;
 }
 
+static notrace int bpf_mem_free_idx(void *ptr, bool percpu)
+{
+       size_t size;
+
+       if (percpu)
+               size = pcpu_alloc_size(*((void **)ptr));
+       else
+               size = ksize(ptr - LLIST_NODE_SZ);
+       return bpf_mem_cache_idx(size);
+}
+
 void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
 {
        int idx;
@@ -881,7 +893,7 @@ void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
        if (!ptr)
                return;
 
-       idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));
+       idx = bpf_mem_free_idx(ptr, ma->percpu);
        if (idx < 0)
                return;
 
@@ -895,7 +907,7 @@ void notrace bpf_mem_free_rcu(struct bpf_mem_alloc *ma, void *ptr)
        if (!ptr)
                return;
 
-       idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));
+       idx = bpf_mem_free_idx(ptr, ma->percpu);
        if (idx < 0)
                return;