bpf: Prevent tailcall infinite loop caused by freplace

There is a potential infinite loop issue that can occur when using a
combination of tail calls and freplace.

In an upcoming selftest, the attach target for entry_freplace of
tailcall_freplace.c is subprog_tc of tc_bpf2bpf.c, while the tail call in
entry_freplace leads to entry_tc. This results in an infinite loop:

entry_tc -> subprog_tc -> entry_freplace --tailcall-> entry_tc.

The problem arises because the tail_call_cnt in entry_freplace resets to
zero each time entry_freplace is executed, causing the tail call mechanism
to never terminate, eventually leading to a kernel panic.

To fix this issue, the solution is twofold:

1. Prevent updating a program extended by an freplace program to a
   prog_array map.
2. Prevent extending a program that is already part of a prog_array map
   with an freplace program.

This ensures that:

* If a program or its subprogram has been extended by an freplace program,
  it can no longer be updated to a prog_array map.
* If a program has been added to a prog_array map, neither it nor its
  subprograms can be extended by an freplace program.

Moreover, an extension program should not be tailcalled. As such, return
-EINVAL if the program has a type of BPF_PROG_TYPE_EXT when adding it to a
prog_array map.

Additionally, fix a minor code style issue by replacing eight spaces with a
tab for proper formatting.

Reviewed-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Leon Hwang <leon.hwang@linux.dev>
Link: https://lore.kernel.org/r/20241015150207.70264-2-leon.hwang@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 19d8ca8ac960f75ae8ca37c302d9e1055ef92af4..0c216e71cec76913f1c67579acffc45926f7a035 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1292,8 +1292,12 @@ void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
 bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
 
 #ifdef CONFIG_BPF_JIT
-int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
+                            struct bpf_trampoline *tr,
+                            struct bpf_prog *tgt_prog);
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
+                              struct bpf_trampoline *tr,
+                              struct bpf_prog *tgt_prog);
 struct bpf_trampoline *bpf_trampoline_get(u64 key,
                                          struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -1374,12 +1378,14 @@ void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
 #else
 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
-                                          struct bpf_trampoline *tr)
+                                          struct bpf_trampoline *tr,
+                                          struct bpf_prog *tgt_prog)
 {
        return -ENOTSUPP;
 }
 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
-                                            struct bpf_trampoline *tr)
+                                            struct bpf_trampoline *tr,
+                                            struct bpf_prog *tgt_prog)
 {
        return -ENOTSUPP;
 }
@@ -1483,6 +1489,9 @@ struct bpf_prog_aux {
        bool xdp_has_frags;
        bool exception_cb;
        bool exception_boundary;
+       bool is_extended; /* true if extended by freplace program */
+       u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
+       struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
        struct bpf_arena *arena;
        /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
        const struct btf_type *attach_func_proto;

diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 79660e3fca4c1b4c829f67210b9bfe22eb72eca7..6cdbb4c33d31d5d56dea634b1f062d6fe3caf0df 100644 (file)
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -947,22 +947,44 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map,
                                   struct file *map_file, int fd)
 {
        struct bpf_prog *prog = bpf_prog_get(fd);
+       bool is_extended;
 
        if (IS_ERR(prog))
                return prog;
 
-       if (!bpf_prog_map_compatible(map, prog)) {
+       if (prog->type == BPF_PROG_TYPE_EXT ||
+           !bpf_prog_map_compatible(map, prog)) {
                bpf_prog_put(prog);
                return ERR_PTR(-EINVAL);
        }
 
+       mutex_lock(&prog->aux->ext_mutex);
+       is_extended = prog->aux->is_extended;
+       if (!is_extended)
+               prog->aux->prog_array_member_cnt++;
+       mutex_unlock(&prog->aux->ext_mutex);
+       if (is_extended) {
+               /* Extended prog can not be tail callee. It's to prevent a
+                * potential infinite loop like:
+                * tail callee prog entry -> tail callee prog subprog ->
+                * freplace prog entry --tailcall-> tail callee prog entry.
+                */
+               bpf_prog_put(prog);
+               return ERR_PTR(-EBUSY);
+       }
+
        return prog;
 }
 
 static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
 {
+       struct bpf_prog *prog = ptr;
+
+       mutex_lock(&prog->aux->ext_mutex);
+       prog->aux->prog_array_member_cnt--;
+       mutex_unlock(&prog->aux->ext_mutex);
        /* bpf_prog is freed after one RCU or tasks trace grace period */
-       bpf_prog_put(ptr);
+       bpf_prog_put(prog);
 }
 
 static u32 prog_fd_array_sys_lookup_elem(void *ptr)

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 5e77c58e06010e48f0e247d86ec9cf0c773d69f9..233ea78f8f1bd9bc7ec9c8690bf8fdc526187eb9 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -131,6 +131,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag
        INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode);
 #endif
        mutex_init(&fp->aux->used_maps_mutex);
+       mutex_init(&fp->aux->ext_mutex);
        mutex_init(&fp->aux->dst_mutex);
 
        return fp;

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a8f1808a1ca54371d5b1fbc85c0290055e62bd4a..4d04d4d9c1f30df8dd1bff63df6e5b50534b12c0 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -3214,7 +3214,8 @@ static void bpf_tracing_link_release(struct bpf_link *link)
                container_of(link, struct bpf_tracing_link, link.link);
 
        WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
-                                               tr_link->trampoline));
+                                               tr_link->trampoline,
+                                               tr_link->tgt_prog));
 
        bpf_trampoline_put(tr_link->trampoline);
 
@@ -3354,7 +3355,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
         *   in prog->aux
         *
         * - if prog->aux->dst_trampoline is NULL, the program has already been
-         *   attached to a target and its initial target was cleared (below)
+        *   attached to a target and its initial target was cleared (below)
         *
         * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
         *   target_btf_id using the link_create API.
@@ -3429,7 +3430,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
        if (err)
                goto out_unlock;
 
-       err = bpf_trampoline_link_prog(&link->link, tr);
+       err = bpf_trampoline_link_prog(&link->link, tr, tgt_prog);
        if (err) {
                bpf_link_cleanup(&link_primer);
                link = NULL;

diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index f8302a5ca400dac6749288f5ea1af317cc9baa7f..9f36c049f4c288a690725c7c03f024a1354d598d 100644 (file)
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -523,7 +523,27 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)
        }
 }
 
-static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
+static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
+{
+       struct bpf_prog_aux *aux = tgt_prog->aux;
+
+       guard(mutex)(&aux->ext_mutex);
+       if (aux->prog_array_member_cnt)
+               /* Program extensions can not extend target prog when the target
+                * prog has been updated to any prog_array map as tail callee.
+                * It's to prevent a potential infinite loop like:
+                * tgt prog entry -> tgt prog subprog -> freplace prog entry
+                * --tailcall-> tgt prog entry.
+                */
+               return -EBUSY;
+
+       aux->is_extended = true;
+       return 0;
+}
+
+static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
+                                     struct bpf_trampoline *tr,
+                                     struct bpf_prog *tgt_prog)
 {
        enum bpf_tramp_prog_type kind;
        struct bpf_tramp_link *link_exiting;
@@ -544,6 +564,9 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr
                /* Cannot attach extension if fentry/fexit are in use. */
                if (cnt)
                        return -EBUSY;
+               err = bpf_freplace_check_tgt_prog(tgt_prog);
+               if (err)
+                       return err;
                tr->extension_prog = link->link.prog;
                return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,
                                          link->link.prog->bpf_func);
@@ -570,17 +593,21 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr
        return err;
 }
 
-int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
+int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
+                            struct bpf_trampoline *tr,
+                            struct bpf_prog *tgt_prog)
 {
        int err;
 
        mutex_lock(&tr->mutex);
-       err = __bpf_trampoline_link_prog(link, tr);
+       err = __bpf_trampoline_link_prog(link, tr, tgt_prog);
        mutex_unlock(&tr->mutex);
        return err;
 }
 
-static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
+static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
+                                       struct bpf_trampoline *tr,
+                                       struct bpf_prog *tgt_prog)
 {
        enum bpf_tramp_prog_type kind;
        int err;
@@ -591,6 +618,8 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_
                err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
                                         tr->extension_prog->bpf_func, NULL);
                tr->extension_prog = NULL;
+               guard(mutex)(&tgt_prog->aux->ext_mutex);
+               tgt_prog->aux->is_extended = false;
                return err;
        }
        hlist_del_init(&link->tramp_hlist);
@@ -599,12 +628,14 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_
 }
 
 /* bpf_trampoline_unlink_prog() should never fail. */
-int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)
+int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
+                              struct bpf_trampoline *tr,
+                              struct bpf_prog *tgt_prog)
 {
        int err;
 
        mutex_lock(&tr->mutex);
-       err = __bpf_trampoline_unlink_prog(link, tr);
+       err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);
        mutex_unlock(&tr->mutex);
        return err;
 }
@@ -619,7 +650,7 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link)
        if (!shim_link->trampoline)
                return;
 
-       WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline));
+       WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline, NULL));
        bpf_trampoline_put(shim_link->trampoline);
 }
 
@@ -733,7 +764,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
                goto err;
        }
 
-       err = __bpf_trampoline_link_prog(&shim_link->link, tr);
+       err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL);
        if (err)
                goto err;