Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

[linux-2.6-block.git] / kernel / bpf / verifier.c

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 97216f799ba8965c07f6a8a86c281eb8e890a0b6..a36ef33ff611e172ca4e3e55f2bf37b2a5ef8423 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -12413,33 +12413,19 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                        goto out_free;
                func[i]->is_func = 1;
                func[i]->aux->func_idx = i;
-               /* the btf and func_info will be freed only at prog->aux */
+               /* Below members will be freed only at prog->aux */
                func[i]->aux->btf = prog->aux->btf;
                func[i]->aux->func_info = prog->aux->func_info;
+               func[i]->aux->poke_tab = prog->aux->poke_tab;
+               func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
 
                for (j = 0; j < prog->aux->size_poke_tab; j++) {
-                       u32 insn_idx = prog->aux->poke_tab[j].insn_idx;
-                       int ret;
+                       struct bpf_jit_poke_descriptor *poke;
 
-                       if (!(insn_idx >= subprog_start &&
-                             insn_idx <= subprog_end))
-                               continue;
-
-                       ret = bpf_jit_add_poke_descriptor(func[i],
-                                                         &prog->aux->poke_tab[j]);
-                       if (ret < 0) {
-                               verbose(env, "adding tail call poke descriptor failed\n");
-                               goto out_free;
-                       }
-
-                       func[i]->insnsi[insn_idx - subprog_start].imm = ret + 1;
-
-                       map_ptr = func[i]->aux->poke_tab[ret].tail_call.map;
-                       ret = map_ptr->ops->map_poke_track(map_ptr, func[i]->aux);
-                       if (ret < 0) {
-                               verbose(env, "tracking tail call prog failed\n");
-                               goto out_free;
-                       }
+                       poke = &prog->aux->poke_tab[j];
+                       if (poke->insn_idx < subprog_end &&
+                           poke->insn_idx >= subprog_start)
+                               poke->aux = func[i]->aux;
                }
 
                /* Use bpf_prog_F_tag to indicate functions in stack traces.
@@ -12470,18 +12456,6 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                cond_resched();
        }
 
-       /* Untrack main program's aux structs so that during map_poke_run()
-        * we will not stumble upon the unfilled poke descriptors; each
-        * of the main program's poke descs got distributed across subprogs
-        * and got tracked onto map, so we are sure that none of them will
-        * be missed after the operation below
-        */
-       for (i = 0; i < prog->aux->size_poke_tab; i++) {
-               map_ptr = prog->aux->poke_tab[i].tail_call.map;
-
-               map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
-       }
-
        /* at this point all bpf functions were successfully JITed
         * now populate all bpf_calls with correct addresses and
         * run last pass of JIT
@@ -12559,14 +12533,22 @@ static int jit_subprogs(struct bpf_verifier_env *env)
        bpf_prog_jit_attempt_done(prog);
        return 0;
 out_free:
+       /* We failed JIT'ing, so at this point we need to unregister poke
+        * descriptors from subprogs, so that kernel is not attempting to
+        * patch it anymore as we're freeing the subprog JIT memory.
+        */
+       for (i = 0; i < prog->aux->size_poke_tab; i++) {
+               map_ptr = prog->aux->poke_tab[i].tail_call.map;
+               map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
+       }
+       /* At this point we're guaranteed that poke descriptors are not
+        * live anymore. We can just unlink its descriptor table as it's
+        * released with the main prog.
+        */
        for (i = 0; i < env->subprog_cnt; i++) {
                if (!func[i])
                        continue;
-
-               for (j = 0; j < func[i]->aux->size_poke_tab; j++) {
-                       map_ptr = func[i]->aux->poke_tab[j].tail_call.map;
-                       map_ptr->ops->map_poke_untrack(map_ptr, func[i]->aux);
-               }
+               func[i]->aux->poke_tab = NULL;
                bpf_jit_free(func[i]);
        }
        kfree(func);