bpf: JIT helpers for fmod_ret progs

* Split the invoke_bpf program to prepare for special handling of
  fmod_ret programs introduced in a subsequent patch.
* Move the definition of emit_cond_near_jump and emit_nops as they are
  needed for fmod_ret.
* Refactor branch target alignment into its own generic helper function
  i.e. emit_align.

Signed-off-by: KP Singh <kpsingh@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Andrii Nakryiko <andriin@fb.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20200304191853.1529-3-kpsingh@chromium.org

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 15c7d28bc05c97f3cc6381dc16911ebe9327679b..d6349e930b06bacededbfedc45e65949afb53c47 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1361,35 +1361,95 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
                         -(stack_size - i * 8));
 }
 
+static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
+                          struct bpf_prog *p, int stack_size)
+{
+       u8 *prog = *pprog;
+       int cnt = 0;
+
+       if (emit_call(&prog, __bpf_prog_enter, prog))
+               return -EINVAL;
+       /* remember prog start time returned by __bpf_prog_enter */
+       emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
+
+       /* arg1: lea rdi, [rbp - stack_size] */
+       EMIT4(0x48, 0x8D, 0x7D, -stack_size);
+       /* arg2: progs[i]->insnsi for interpreter */
+       if (!p->jited)
+               emit_mov_imm64(&prog, BPF_REG_2,
+                              (long) p->insnsi >> 32,
+                              (u32) (long) p->insnsi);
+       /* call JITed bpf program or interpreter */
+       if (emit_call(&prog, p->bpf_func, prog))
+               return -EINVAL;
+
+       /* arg1: mov rdi, progs[i] */
+       emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32,
+                      (u32) (long) p);
+       /* arg2: mov rsi, rbx <- start time in nsec */
+       emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
+       if (emit_call(&prog, __bpf_prog_exit, prog))
+               return -EINVAL;
+
+       *pprog = prog;
+       return 0;
+}
+
+static void emit_nops(u8 **pprog, unsigned int len)
+{
+       unsigned int i, noplen;
+       u8 *prog = *pprog;
+       int cnt = 0;
+
+       while (len > 0) {
+               noplen = len;
+
+               if (noplen > ASM_NOP_MAX)
+                       noplen = ASM_NOP_MAX;
+
+               for (i = 0; i < noplen; i++)
+                       EMIT1(ideal_nops[noplen][i]);
+               len -= noplen;
+       }
+
+       *pprog = prog;
+}
+
+static void emit_align(u8 **pprog, u32 align)
+{
+       u8 *target, *prog = *pprog;
+
+       target = PTR_ALIGN(prog, align);
+       if (target != prog)
+               emit_nops(&prog, target - prog);
+
+       *pprog = prog;
+}
+
+static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
+{
+       u8 *prog = *pprog;
+       int cnt = 0;
+       s64 offset;
+
+       offset = func - (ip + 2 + 4);
+       if (!is_simm32(offset)) {
+               pr_err("Target %p is out of range\n", func);
+               return -EINVAL;
+       }
+       EMIT2_off32(0x0F, jmp_cond + 0x10, offset);
+       *pprog = prog;
+       return 0;
+}
+
 static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
                      struct bpf_tramp_progs *tp, int stack_size)
 {
+       int i;
        u8 *prog = *pprog;
-       int cnt = 0, i;
 
        for (i = 0; i < tp->nr_progs; i++) {
-               if (emit_call(&prog, __bpf_prog_enter, prog))
-                       return -EINVAL;
-               /* remember prog start time returned by __bpf_prog_enter */
-               emit_mov_reg(&prog, true, BPF_REG_6, BPF_REG_0);
-
-               /* arg1: lea rdi, [rbp - stack_size] */
-               EMIT4(0x48, 0x8D, 0x7D, -stack_size);
-               /* arg2: progs[i]->insnsi for interpreter */
-               if (!tp->progs[i]->jited)
-                       emit_mov_imm64(&prog, BPF_REG_2,
-                                      (long) tp->progs[i]->insnsi >> 32,
-                                      (u32) (long) tp->progs[i]->insnsi);
-               /* call JITed bpf program or interpreter */
-               if (emit_call(&prog, tp->progs[i]->bpf_func, prog))
-                       return -EINVAL;
-
-               /* arg1: mov rdi, progs[i] */
-               emit_mov_imm64(&prog, BPF_REG_1, (long) tp->progs[i] >> 32,
-                              (u32) (long) tp->progs[i]);
-               /* arg2: mov rsi, rbx <- start time in nsec */
-               emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
-               if (emit_call(&prog, __bpf_prog_exit, prog))
+               if (invoke_bpf_prog(m, &prog, tp->progs[i], stack_size))
                        return -EINVAL;
        }
        *pprog = prog;
@@ -1531,42 +1591,6 @@ int arch_prepare_bpf_trampoline(void *image, void *image_end,
        return prog - (u8 *)image;
 }
 
-static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
-{
-       u8 *prog = *pprog;
-       int cnt = 0;
-       s64 offset;
-
-       offset = func - (ip + 2 + 4);
-       if (!is_simm32(offset)) {
-               pr_err("Target %p is out of range\n", func);
-               return -EINVAL;
-       }
-       EMIT2_off32(0x0F, jmp_cond + 0x10, offset);
-       *pprog = prog;
-       return 0;
-}
-
-static void emit_nops(u8 **pprog, unsigned int len)
-{
-       unsigned int i, noplen;
-       u8 *prog = *pprog;
-       int cnt = 0;
-
-       while (len > 0) {
-               noplen = len;
-
-               if (noplen > ASM_NOP_MAX)
-                       noplen = ASM_NOP_MAX;
-
-               for (i = 0; i < noplen; i++)
-                       EMIT1(ideal_nops[noplen][i]);
-               len -= noplen;
-       }
-
-       *pprog = prog;
-}
-
 static int emit_fallback_jump(u8 **pprog)
 {
        u8 *prog = *pprog;
@@ -1589,7 +1613,7 @@ static int emit_fallback_jump(u8 **pprog)
 
 static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)
 {
-       u8 *jg_reloc, *jg_target, *prog = *pprog;
+       u8 *jg_reloc, *prog = *pprog;
        int pivot, err, jg_bytes = 1, cnt = 0;
        s64 jg_offset;
 
@@ -1644,9 +1668,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)
         * Coding Rule 11: All branch targets should be 16-byte
         * aligned.
         */
-       jg_target = PTR_ALIGN(prog, 16);
-       if (jg_target != prog)
-               emit_nops(&prog, jg_target - prog);
+       emit_align(&prog, 16);
        jg_offset = prog - jg_reloc;
        emit_code(jg_reloc - jg_bytes, jg_offset, jg_bytes);