powerpc/bpf: Handle large branch ranges with BPF_EXIT

In some scenarios, it is possible that the program epilogue is outside
the branch range for a BPF_EXIT instruction. Instead of rejecting such
programs, emit epilogue as an alternate exit point from the program.
Track the location of the same so that subsequent exits can take either
of the two paths.

Reported-by: Jordan Niethe <jniethe5@gmail.com>
Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/33aa2e92645a92712be23b18035a2c6dcb92ff8d.1644834730.git.naveen.n.rao@linux.vnet.ibm.com

diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h
index 25a7190bcee95df0763a057b38934714072b6ab5..e58cf29bb0cf6736ce3533a49ec928d1f3bdca6a 100644 (file)
--- a/arch/powerpc/net/bpf_jit.h
+++ b/arch/powerpc/net/bpf_jit.h
@@ -148,6 +148,7 @@ struct codegen_context {
        unsigned int stack_size;
        int b2p[ARRAY_SIZE(b2p)];
        unsigned int exentry_idx;
+       unsigned int alt_exit_addr;
 };
 
 #ifdef CONFIG_PPC32
@@ -183,6 +184,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
 void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx);
 void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx);
 void bpf_jit_realloc_regs(struct codegen_context *ctx);
+int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr);
 
 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
                          int insn_idx, int jmp_off, int dst_reg);

diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 56dd1f4e3e4447ba79ce39802ad076caf8855a96..141e64585b6458754d705cf8702c37a3a90445db 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -89,6 +89,22 @@ static int bpf_jit_fixup_addresses(struct bpf_prog *fp, u32 *image,
        return 0;
 }
 
+int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
+{
+       if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
+               PPC_JMP(exit_addr);
+       } else if (ctx->alt_exit_addr) {
+               if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4))))
+                       return -1;
+               PPC_JMP(ctx->alt_exit_addr);
+       } else {
+               ctx->alt_exit_addr = ctx->idx * 4;
+               bpf_jit_build_epilogue(image, ctx);
+       }
+
+       return 0;
+}
+
 struct powerpc64_jit_data {
        struct bpf_binary_header *header;
        u32 *addrs;
@@ -177,8 +193,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
         * If we have seen a tail call, we need a second pass.
         * This is because bpf_jit_emit_common_epilogue() is called
         * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen.
+        * We also need a second pass if we ended up with too large
+        * a program so as to ensure BPF_EXIT branches are in range.
         */
-       if (cgctx.seen & SEEN_TAILCALL) {
+       if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
                cgctx.idx = 0;
                if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
                        fp = org_fp;
@@ -193,6 +211,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
         * calculate total size from idx.
         */
        bpf_jit_build_prologue(0, &cgctx);
+       addrs[fp->len] = cgctx.idx * 4;
        bpf_jit_build_epilogue(0, &cgctx);
 
        fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
@@ -233,6 +252,7 @@ skip_init_ctx:
        for (pass = 1; pass < 3; pass++) {
                /* Now build the prologue, body code & epilogue for real. */
                cgctx.idx = 0;
+               cgctx.alt_exit_addr = 0;
                bpf_jit_build_prologue(code_base, &cgctx);
                if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {
                        bpf_jit_binary_free(bpf_hdr);

diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c
index 5ba5340a63871f6410d0ed803931378537c3b4d4..8e743b7bf8f51aed07759e77145a0ee2ed394f84 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp32.c
+++ b/arch/powerpc/net/bpf_jit_comp32.c
@@ -937,8 +937,11 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
                         * the epilogue. If we _are_ the last instruction,
                         * we'll just fall through to the epilogue.
                         */
-                       if (i != flen - 1)
-                               PPC_JMP(exit_addr);
+                       if (i != flen - 1) {
+                               ret = bpf_jit_emit_exit_insn(image, ctx, _R0, exit_addr);
+                               if (ret)
+                                       return ret;
+                       }
                        /* else fall through to the epilogue */
                        break;
 

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index b1ed8611091d2bf05809c8e349a040b5cafb6a71..371bd5a16859c7d7589e69b8b89d725fdaccf0c6 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -871,8 +871,11 @@ emit_clear:
                         * the epilogue. If we _are_ the last instruction,
                         * we'll just fall through to the epilogue.
                         */
-                       if (i != flen - 1)
-                               PPC_JMP(exit_addr);
+                       if (i != flen - 1) {
+                               ret = bpf_jit_emit_exit_insn(image, ctx, b2p[TMP_REG_1], exit_addr);
+                               if (ret)
+                                       return ret;
+                       }
                        /* else fall through to the epilogue */
                        break;