powerpc/bpf: enable kfunc call

Currently, bpf jit code on powerpc assumes all the bpf functions and
helpers to be part of core kernel text. This is false for kfunc case,
as function addresses may not be part of core kernel text area. So,
add support for addresses that are not within core kernel text area
too, to enable kfunc support. Emit instructions based on whether the
function address is within core kernel text address or not, to retain
optimized instruction sequence where possible.

In case of PCREL, as a bpf function that is not within core kernel
text area is likely to go out of range with relative addressing on
kernel base, use PC relative addressing. If that goes out of range,
load the full address with PPC_LI64().

With addresses that are not within core kernel text area supported,
override bpf_jit_supports_kfunc_call() to enable kfunc support. Also,
override bpf_jit_supports_far_kfunc_call() to enable 64-bit pointers,
as an address offset can be more than 32-bit long on PPC64.

Signed-off-by: Hari Bathini <hbathini@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://msgid.link/20240502173205.142794-2-hbathini@linux.ibm.com

diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 0f9a2178332961ed70d699efd6447c33f4ca941c..984655419da52fd0fb9b1faebc16e101d0554059 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -359,3 +359,13 @@ void bpf_jit_free(struct bpf_prog *fp)
 
        bpf_prog_unlock_free(fp);
 }
+
+bool bpf_jit_supports_kfunc_call(void)
+{
+       return true;
+}
+
+bool bpf_jit_supports_far_kfunc_call(void)
+{
+       return IS_ENABLED(CONFIG_PPC64);
+}

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 4de08e35e2848e6d586769b959349cd839baef09..8afc14a4a1258ebab01c802efdaa1c55cb73fd1b 100644 (file)
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -208,17 +208,13 @@ bpf_jit_emit_func_call_hlp(u32 *image, u32 *fimage, struct codegen_context *ctx,
        unsigned long func_addr = func ? ppc_function_entry((void *)func) : 0;
        long reladdr;
 
-       if (WARN_ON_ONCE(!core_kernel_text(func_addr)))
+       if (WARN_ON_ONCE(!kernel_text_address(func_addr)))
                return -EINVAL;
 
-       if (IS_ENABLED(CONFIG_PPC_KERNEL_PCREL)) {
-               reladdr = func_addr - local_paca->kernelbase;
+#ifdef CONFIG_PPC_KERNEL_PCREL
+       reladdr = func_addr - local_paca->kernelbase;
 
-               if (reladdr >= (long)SZ_8G || reladdr < -(long)SZ_8G) {
-                       pr_err("eBPF: address of %ps out of range of 34-bit relative address.\n",
-                              (void *)func);
-                       return -ERANGE;
-               }
+       if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) {
                EMIT(PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase)));
                /* Align for subsequent prefix instruction */
                if (!IS_ALIGNED((unsigned long)fimage + CTX_NIA(ctx), 8))
@@ -227,6 +223,26 @@ bpf_jit_emit_func_call_hlp(u32 *image, u32 *fimage, struct codegen_context *ctx,
                EMIT(PPC_PREFIX_MLS | __PPC_PRFX_R(0) | IMM_H18(reladdr));
                EMIT(PPC_INST_PADDI | ___PPC_RT(_R12) | ___PPC_RA(_R12) | IMM_L(reladdr));
        } else {
+               unsigned long pc = (unsigned long)fimage + CTX_NIA(ctx);
+               bool alignment_needed = !IS_ALIGNED(pc, 8);
+
+               reladdr = func_addr - (alignment_needed ? pc + 4 :  pc);
+
+               if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) {
+                       if (alignment_needed)
+                               EMIT(PPC_RAW_NOP());
+                       /* pla r12,addr */
+                       EMIT(PPC_PREFIX_MLS | __PPC_PRFX_R(1) | IMM_H18(reladdr));
+                       EMIT(PPC_INST_PADDI | ___PPC_RT(_R12) | IMM_L(reladdr));
+               } else {
+                       /* We can clobber r12 */
+                       PPC_LI64(_R12, func);
+               }
+       }
+       EMIT(PPC_RAW_MTCTR(_R12));
+       EMIT(PPC_RAW_BCTRL());
+#else
+       if (core_kernel_text(func_addr)) {
                reladdr = func_addr - kernel_toc_addr();
                if (reladdr > 0x7FFFFFFF || reladdr < -(0x80000000L)) {
                        pr_err("eBPF: address of %ps out of range of kernel_toc.\n", (void *)func);
@@ -235,9 +251,34 @@ bpf_jit_emit_func_call_hlp(u32 *image, u32 *fimage, struct codegen_context *ctx,
 
                EMIT(PPC_RAW_ADDIS(_R12, _R2, PPC_HA(reladdr)));
                EMIT(PPC_RAW_ADDI(_R12, _R12, PPC_LO(reladdr)));
+               EMIT(PPC_RAW_MTCTR(_R12));
+               EMIT(PPC_RAW_BCTRL());
+       } else {
+               if (IS_ENABLED(CONFIG_PPC64_ELF_ABI_V1)) {
+                       /* func points to the function descriptor */
+                       PPC_LI64(bpf_to_ppc(TMP_REG_2), func);
+                       /* Load actual entry point from function descriptor */
+                       EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), 0));
+                       /* ... and move it to CTR */
+                       EMIT(PPC_RAW_MTCTR(bpf_to_ppc(TMP_REG_1)));
+                       /*
+                        * Load TOC from function descriptor at offset 8.
+                        * We can clobber r2 since we get called through a
+                        * function pointer (so caller will save/restore r2).
+                        */
+                       EMIT(PPC_RAW_LD(_R2, bpf_to_ppc(TMP_REG_2), 8));
+               } else {
+                       PPC_LI64(_R12, func);
+                       EMIT(PPC_RAW_MTCTR(_R12));
+               }
+               EMIT(PPC_RAW_BCTRL());
+               /*
+                * Load r2 with kernel TOC as kernel TOC is used if function address falls
+                * within core kernel text.
+                */
+               EMIT(PPC_RAW_LD(_R2, _R13, offsetof(struct paca_struct, kernel_toc)));
        }
-       EMIT(PPC_RAW_MTCTR(_R12));
-       EMIT(PPC_RAW_BCTRL());
+#endif
 
        return 0;
 }