bpf: extend bpf_prog_array to store pointers to the cgroup storage

This patch converts bpf_prog_array from an array of prog pointers
to the array of struct bpf_prog_array_item elements.

This allows to save a cgroup storage pointer for each bpf program
efficiently attached to a cgroup.

Signed-off-by: Roman Gushchin <guro@fb.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c
index fcfab6635f9c6649a64e0b144f55df672d621389..8c26df9b96c1af95b67b41cb3cd9b2f375ee151f 100644 (file)
--- a/drivers/media/rc/bpf-lirc.c
+++ b/drivers/media/rc/bpf-lirc.c
@@ -195,14 +195,16 @@ void lirc_bpf_run(struct rc_dev *rcdev, u32 sample)
  */
 void lirc_bpf_free(struct rc_dev *rcdev)
 {
-       struct bpf_prog **progs;
+       struct bpf_prog_array_item *item;
 
        if (!rcdev->raw->progs)
                return;
 
-       progs = rcu_dereference(rcdev->raw->progs)->progs;
-       while (*progs)
-               bpf_prog_put(*progs++);
+       item = rcu_dereference(rcdev->raw->progs)->items;
+       while (item->prog) {
+               bpf_prog_put(item->prog);
+               item++;
+       }
 
        bpf_prog_array_free(rcdev->raw->progs);
 }

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 9d1e4727495e028e7dccdade36dc6373bfdb9fcd..16be67888c3084a650a8bd927fbbd6e16e177329 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -349,9 +349,14 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
  * The 'struct bpf_prog_array *' should only be replaced with xchg()
  * since other cpus are walking the array of pointers in parallel.
  */
+struct bpf_prog_array_item {
+       struct bpf_prog *prog;
+       struct bpf_cgroup_storage *cgroup_storage;
+};
+
 struct bpf_prog_array {
        struct rcu_head rcu;
-       struct bpf_prog *progs[0];
+       struct bpf_prog_array_item items[0];
 };
 
 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
@@ -372,7 +377,8 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
 
 #define __BPF_PROG_RUN_ARRAY(array, ctx, func, check_non_null) \
        ({                                              \
-               struct bpf_prog **_prog, *__prog;       \
+               struct bpf_prog_array_item *_item;      \
+               struct bpf_prog *_prog;                 \
                struct bpf_prog_array *_array;          \
                u32 _ret = 1;                           \
                preempt_disable();                      \
@@ -380,10 +386,11 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
                _array = rcu_dereference(array);        \
                if (unlikely(check_non_null && !_array))\
                        goto _out;                      \
-               _prog = _array->progs;                  \
-               while ((__prog = READ_ONCE(*_prog))) {  \
-                       _ret &= func(__prog, ctx);      \
-                       _prog++;                        \
+               _item = &_array->items[0];              \
+               while ((_prog = READ_ONCE(_item->prog))) {              \
+                       bpf_cgroup_storage_set(_item->cgroup_storage);  \
+                       _ret &= func(_prog, ctx);       \
+                       _item++;                        \
                }                                       \
 _out:                                                  \
                rcu_read_unlock();                      \

diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 935274c86bfedaf56d2b0682277be8ab408aafd9..ddfa6cc13e57a344f727b7d9bc825db031ed57b5 100644 (file)
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -117,15 +117,18 @@ static int compute_effective_progs(struct cgroup *cgrp,
        cnt = 0;
        p = cgrp;
        do {
-               if (cnt == 0 || (p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
-                       list_for_each_entry(pl,
-                                           &p->bpf.progs[type], node) {
-                               if (!pl->prog)
-                                       continue;
-                               progs->progs[cnt++] = pl->prog;
-                       }
-               p = cgroup_parent(p);
-       } while (p);
+               if (cnt > 0 && !(p->bpf.flags[type] & BPF_F_ALLOW_MULTI))
+                       continue;
+
+               list_for_each_entry(pl, &p->bpf.progs[type], node) {
+                       if (!pl->prog)
+                               continue;
+
+                       progs->items[cnt].prog = pl->prog;
+                       progs->items[cnt].cgroup_storage = pl->storage;
+                       cnt++;
+               }
+       } while ((p = cgroup_parent(p)));
 
        rcu_assign_pointer(*array, progs);
        return 0;

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 253aa8e79c7b4e22bfb5f626c35e5a11a7f035e0..9abcf25ebf9f70c71b0cc4a79a0c8e228690a129 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1542,7 +1542,8 @@ struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
 {
        if (prog_cnt)
                return kzalloc(sizeof(struct bpf_prog_array) +
-                              sizeof(struct bpf_prog *) * (prog_cnt + 1),
+                              sizeof(struct bpf_prog_array_item) *
+                              (prog_cnt + 1),
                               flags);
 
        return &empty_prog_array.hdr;
@@ -1556,43 +1557,45 @@ void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
        kfree_rcu(progs, rcu);
 }
 
-int bpf_prog_array_length(struct bpf_prog_array __rcu *progs)
+int bpf_prog_array_length(struct bpf_prog_array __rcu *array)
 {
-       struct bpf_prog **prog;
+       struct bpf_prog_array_item *item;
        u32 cnt = 0;
 
        rcu_read_lock();
-       prog = rcu_dereference(progs)->progs;
-       for (; *prog; prog++)
-               if (*prog != &dummy_bpf_prog.prog)
+       item = rcu_dereference(array)->items;
+       for (; item->prog; item++)
+               if (item->prog != &dummy_bpf_prog.prog)
                        cnt++;
        rcu_read_unlock();
        return cnt;
 }
 
-static bool bpf_prog_array_copy_core(struct bpf_prog **prog,
+
+static bool bpf_prog_array_copy_core(struct bpf_prog_array __rcu *array,
                                     u32 *prog_ids,
                                     u32 request_cnt)
 {
+       struct bpf_prog_array_item *item;
        int i = 0;
 
-       for (; *prog; prog++) {
-               if (*prog == &dummy_bpf_prog.prog)
+       item = rcu_dereference(array)->items;
+       for (; item->prog; item++) {
+               if (item->prog == &dummy_bpf_prog.prog)
                        continue;
-               prog_ids[i] = (*prog)->aux->id;
+               prog_ids[i] = item->prog->aux->id;
                if (++i == request_cnt) {
-                       prog++;
+                       item++;
                        break;
                }
        }
 
-       return !!(*prog);
+       return !!(item->prog);
 }
 
-int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
+int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *array,
                                __u32 __user *prog_ids, u32 cnt)
 {
-       struct bpf_prog **prog;
        unsigned long err = 0;
        bool nospc;
        u32 *ids;
@@ -1611,8 +1614,7 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
        if (!ids)
                return -ENOMEM;
        rcu_read_lock();
-       prog = rcu_dereference(progs)->progs;
-       nospc = bpf_prog_array_copy_core(prog, ids, cnt);
+       nospc = bpf_prog_array_copy_core(array, ids, cnt);
        rcu_read_unlock();
        err = copy_to_user(prog_ids, ids, cnt * sizeof(u32));
        kfree(ids);
@@ -1623,14 +1625,14 @@ int bpf_prog_array_copy_to_user(struct bpf_prog_array __rcu *progs,
        return 0;
 }
 
-void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *progs,
+void bpf_prog_array_delete_safe(struct bpf_prog_array __rcu *array,
                                struct bpf_prog *old_prog)
 {
-       struct bpf_prog **prog = progs->progs;
+       struct bpf_prog_array_item *item = array->items;
 
-       for (; *prog; prog++)
-               if (*prog == old_prog) {
-                       WRITE_ONCE(*prog, &dummy_bpf_prog.prog);
+       for (; item->prog; item++)
+               if (item->prog == old_prog) {
+                       WRITE_ONCE(item->prog, &dummy_bpf_prog.prog);
                        break;
                }
 }
@@ -1641,7 +1643,7 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
                        struct bpf_prog_array **new_array)
 {
        int new_prog_cnt, carry_prog_cnt = 0;
-       struct bpf_prog **existing_prog;
+       struct bpf_prog_array_item *existing;
        struct bpf_prog_array *array;
        bool found_exclude = false;
        int new_prog_idx = 0;
@@ -1650,15 +1652,15 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
         * the new array.
         */
        if (old_array) {
-               existing_prog = old_array->progs;
-               for (; *existing_prog; existing_prog++) {
-                       if (*existing_prog == exclude_prog) {
+               existing = old_array->items;
+               for (; existing->prog; existing++) {
+                       if (existing->prog == exclude_prog) {
                                found_exclude = true;
                                continue;
                        }
-                       if (*existing_prog != &dummy_bpf_prog.prog)
+                       if (existing->prog != &dummy_bpf_prog.prog)
                                carry_prog_cnt++;
-                       if (*existing_prog == include_prog)
+                       if (existing->prog == include_prog)
                                return -EEXIST;
                }
        }
@@ -1684,15 +1686,17 @@ int bpf_prog_array_copy(struct bpf_prog_array __rcu *old_array,
 
        /* Fill in the new prog array */
        if (carry_prog_cnt) {
-               existing_prog = old_array->progs;
-               for (; *existing_prog; existing_prog++)
-                       if (*existing_prog != exclude_prog &&
-                           *existing_prog != &dummy_bpf_prog.prog)
-                               array->progs[new_prog_idx++] = *existing_prog;
+               existing = old_array->items;
+               for (; existing->prog; existing++)
+                       if (existing->prog != exclude_prog &&
+                           existing->prog != &dummy_bpf_prog.prog) {
+                               array->items[new_prog_idx++].prog =
+                                       existing->prog;
+                       }
        }
        if (include_prog)
-               array->progs[new_prog_idx++] = include_prog;
-       array->progs[new_prog_idx] = NULL;
+               array->items[new_prog_idx++].prog = include_prog;
+       array->items[new_prog_idx].prog = NULL;
        *new_array = array;
        return 0;
 }
@@ -1701,7 +1705,6 @@ int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
                             u32 *prog_ids, u32 request_cnt,
                             u32 *prog_cnt)
 {
-       struct bpf_prog **prog;
        u32 cnt = 0;
 
        if (array)
@@ -1714,8 +1717,7 @@ int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
                return 0;
 
        /* this function is called under trace/bpf_trace.c: bpf_event_mutex */
-       prog = rcu_dereference_check(array, 1)->progs;
-       return bpf_prog_array_copy_core(prog, prog_ids, request_cnt) ? -ENOSPC
+       return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC
                                                                     : 0;
 }