bpf: Add support for __prog argument suffix to pass in prog->aux

Instead of hardcoding the list of kfuncs that need prog->aux passed to
them with a combination of fixup_kfunc_call adjustment + __ign suffix,
combine both in __prog suffix, which ignores the argument passed in, and
fixes it up to the prog->aux. This allows kfuncs to have the prog->aux
passed into them without having to touch the verifier.

Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Link: https://lore.kernel.org/r/20250513142812.1021591-1-memxor@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst
index a8f5782bd83318cc9b394b45b09eaa1ff94f6a3d..ae468b781d31181ba37b132722812e233bce00ea 100644 (file)
--- a/Documentation/bpf/kfuncs.rst
+++ b/Documentation/bpf/kfuncs.rst
@@ -160,6 +160,23 @@ Or::
                 ...
         }
 
+2.2.6 __prog Annotation
+---------------------------
+This annotation is used to indicate that the argument needs to be fixed up to
+the bpf_prog_aux of the caller BPF program. Any value passed into this argument
+is ignored, and rewritten by the verifier.
+
+An example is given below::
+
+        __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,
+                                                 int (callback_fn)(void *map, int *key, void *value),
+                                                 unsigned int flags,
+                                                 void *aux__prog)
+         {
+                struct bpf_prog_aux *aux = aux__prog;
+                ...
+         }
+
 .. _BPF_kfunc_nodef:
 
 2.3 Using an existing kernel function

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 9734544b6957cfffe10a4fee3f5d652166538347..cedd66867ecf28a3934f53e69ba090245551e75e 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -591,6 +591,7 @@ struct bpf_insn_aux_data {
         * bpf_fastcall pattern.
         */
        u8 fastcall_spills_num:3;
+       u8 arg_prog:4;
 
        /* below fields are initialized once */
        unsigned int orig_idx; /* original instruction index */

diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index ebb604e39eb1ed85947ad89726b90635d24b3f13..c1113b74e1e2c8d714adc1439addbcb51eb0770a 100644 (file)
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -3002,9 +3002,9 @@ __bpf_kfunc int bpf_wq_start(struct bpf_wq *wq, unsigned int flags)
 __bpf_kfunc int bpf_wq_set_callback_impl(struct bpf_wq *wq,
                                         int (callback_fn)(void *map, int *key, void *value),
                                         unsigned int flags,
-                                        void *aux__ign)
+                                        void *aux__prog)
 {
-       struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__ign;
+       struct bpf_prog_aux *aux = (struct bpf_prog_aux *)aux__prog;
        struct bpf_async_kern *async = (struct bpf_async_kern *)wq;
 
        if (flags)

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 28f5a7899bd6763cf02cccd995671c31aadf1280..f6d3655b3a7a7d11987cd9e6911ad4d5b2cbaafc 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -322,6 +322,7 @@ struct bpf_kfunc_call_arg_meta {
        struct btf *arg_btf;
        u32 arg_btf_id;
        bool arg_owning_ref;
+       bool arg_prog;
 
        struct {
                struct btf_field *field;
@@ -11897,6 +11898,11 @@ static bool is_kfunc_arg_irq_flag(const struct btf *btf, const struct btf_param
        return btf_param_match_suffix(btf, arg, "__irq_flag");
 }
 
+static bool is_kfunc_arg_prog(const struct btf *btf, const struct btf_param *arg)
+{
+       return btf_param_match_suffix(btf, arg, "__prog");
+}
+
 static bool is_kfunc_arg_scalar_with_name(const struct btf *btf,
                                          const struct btf_param *arg,
                                          const char *name)
@@ -12938,6 +12944,17 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
                if (is_kfunc_arg_ignore(btf, &args[i]))
                        continue;
 
+               if (is_kfunc_arg_prog(btf, &args[i])) {
+                       /* Used to reject repeated use of __prog. */
+                       if (meta->arg_prog) {
+                               verbose(env, "Only 1 prog->aux argument supported per-kfunc\n");
+                               return -EFAULT;
+                       }
+                       meta->arg_prog = true;
+                       cur_aux(env)->arg_prog = regno;
+                       continue;
+               }
+
                if (btf_type_is_scalar(t)) {
                        if (reg->type != SCALAR_VALUE) {
                                verbose(env, "R%d is not a scalar\n", regno);
@@ -21517,13 +21534,17 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
                   desc->func_id == special_kfunc_list[KF_bpf_rdonly_cast]) {
                insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_1);
                *cnt = 1;
-       } else if (is_bpf_wq_set_callback_impl_kfunc(desc->func_id)) {
-               struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(BPF_REG_4, (long)env->prog->aux) };
+       }
 
-               insn_buf[0] = ld_addrs[0];
-               insn_buf[1] = ld_addrs[1];
-               insn_buf[2] = *insn;
-               *cnt = 3;
+       if (env->insn_aux_data[insn_idx].arg_prog) {
+               u32 regno = env->insn_aux_data[insn_idx].arg_prog;
+               struct bpf_insn ld_addrs[2] = { BPF_LD_IMM64(regno, (long)env->prog->aux) };
+               int idx = *cnt;
+
+               insn_buf[idx++] = ld_addrs[0];
+               insn_buf[idx++] = ld_addrs[1];
+               insn_buf[idx++] = *insn;
+               *cnt = idx;
        }
        return 0;
 }