bpf: Support new signed div/mod instructions.

Add interpreter/jit support for new signed div/mod insns.
The new signed div/mod instructions are encoded with
unsigned div/mod instructions plus insn->off == 1.
Also add basic verifier support to ensure new insns get
accepted.

Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20230728011219.3714605-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 4942a4c188b98652f0b9435cc50787daa0af9c15..a89b62eb2b400467ec47433baeb056533f8dff62 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1194,15 +1194,26 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
                                /* mov rax, dst_reg */
                                emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg);
 
-                       /*
-                        * xor edx, edx
-                        * equivalent to 'xor rdx, rdx', but one byte less
-                        */
-                       EMIT2(0x31, 0xd2);
+                       if (insn->off == 0) {
+                               /*
+                                * xor edx, edx
+                                * equivalent to 'xor rdx, rdx', but one byte less
+                                */
+                               EMIT2(0x31, 0xd2);
 
-                       /* div src_reg */
-                       maybe_emit_1mod(&prog, src_reg, is64);
-                       EMIT2(0xF7, add_1reg(0xF0, src_reg));
+                               /* div src_reg */
+                               maybe_emit_1mod(&prog, src_reg, is64);
+                               EMIT2(0xF7, add_1reg(0xF0, src_reg));
+                       } else {
+                               if (BPF_CLASS(insn->code) == BPF_ALU)
+                                       EMIT1(0x99); /* cdq */
+                               else
+                                       EMIT2(0x48, 0x99); /* cqo */
+
+                               /* idiv src_reg */
+                               maybe_emit_1mod(&prog, src_reg, is64);
+                               EMIT2(0xF7, add_1reg(0xF8, src_reg));
+                       }
 
                        if (BPF_OP(insn->code) == BPF_MOD &&
                            dst_reg != BPF_REG_3)

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index ad58697cec4be1431a79bd678df960552ea5f273..3fe895199f6e2f3f060216f6048c9e9b0b3e92cb 100644 (file)
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1792,36 +1792,114 @@ select_insn:
                (*(s64 *) &DST) >>= IMM;
                CONT;
        ALU64_MOD_X:
-               div64_u64_rem(DST, SRC, &AX);
-               DST = AX;
+               switch (OFF) {
+               case 0:
+                       div64_u64_rem(DST, SRC, &AX);
+                       DST = AX;
+                       break;
+               case 1:
+                       AX = div64_s64(DST, SRC);
+                       DST = DST - AX * SRC;
+                       break;
+               }
                CONT;
        ALU_MOD_X:
-               AX = (u32) DST;
-               DST = do_div(AX, (u32) SRC);
+               switch (OFF) {
+               case 0:
+                       AX = (u32) DST;
+                       DST = do_div(AX, (u32) SRC);
+                       break;
+               case 1:
+                       AX = abs((s32)DST);
+                       AX = do_div(AX, abs((s32)SRC));
+                       if ((s32)DST < 0)
+                               DST = (u32)-AX;
+                       else
+                               DST = (u32)AX;
+                       break;
+               }
                CONT;
        ALU64_MOD_K:
-               div64_u64_rem(DST, IMM, &AX);
-               DST = AX;
+               switch (OFF) {
+               case 0:
+                       div64_u64_rem(DST, IMM, &AX);
+                       DST = AX;
+                       break;
+               case 1:
+                       AX = div64_s64(DST, IMM);
+                       DST = DST - AX * IMM;
+                       break;
+               }
                CONT;
        ALU_MOD_K:
-               AX = (u32) DST;
-               DST = do_div(AX, (u32) IMM);
+               switch (OFF) {
+               case 0:
+                       AX = (u32) DST;
+                       DST = do_div(AX, (u32) IMM);
+                       break;
+               case 1:
+                       AX = abs((s32)DST);
+                       AX = do_div(AX, abs((s32)IMM));
+                       if ((s32)DST < 0)
+                               DST = (u32)-AX;
+                       else
+                               DST = (u32)AX;
+                       break;
+               }
                CONT;
        ALU64_DIV_X:
-               DST = div64_u64(DST, SRC);
+               switch (OFF) {
+               case 0:
+                       DST = div64_u64(DST, SRC);
+                       break;
+               case 1:
+                       DST = div64_s64(DST, SRC);
+                       break;
+               }
                CONT;
        ALU_DIV_X:
-               AX = (u32) DST;
-               do_div(AX, (u32) SRC);
-               DST = (u32) AX;
+               switch (OFF) {
+               case 0:
+                       AX = (u32) DST;
+                       do_div(AX, (u32) SRC);
+                       DST = (u32) AX;
+                       break;
+               case 1:
+                       AX = abs((s32)DST);
+                       do_div(AX, abs((s32)SRC));
+                       if ((s32)DST < 0 == (s32)SRC < 0)
+                               DST = (u32)AX;
+                       else
+                               DST = (u32)-AX;
+                       break;
+               }
                CONT;
        ALU64_DIV_K:
-               DST = div64_u64(DST, IMM);
+               switch (OFF) {
+               case 0:
+                       DST = div64_u64(DST, IMM);
+                       break;
+               case 1:
+                       DST = div64_s64(DST, IMM);
+                       break;
+               }
                CONT;
        ALU_DIV_K:
-               AX = (u32) DST;
-               do_div(AX, (u32) IMM);
-               DST = (u32) AX;
+               switch (OFF) {
+               case 0:
+                       AX = (u32) DST;
+                       do_div(AX, (u32) IMM);
+                       DST = (u32) AX;
+                       break;
+               case 1:
+                       AX = abs((s32)DST);
+                       do_div(AX, abs((s32)IMM));
+                       if ((s32)DST < 0 == (s32)IMM < 0)
+                               DST = (u32)AX;
+                       else
+                               DST = (u32)-AX;
+                       break;
+               }
                CONT;
        ALU_END_TO_BE:
                switch (IMM) {

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index a3dcaeed821758d5676aedf0423f86b6df8c213b..c0aceedfcb9cb0fd63b7803e7a400dca1b639093 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -13237,7 +13237,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
        } else {        /* all other ALU ops: and, sub, xor, add, ... */
 
                if (BPF_SRC(insn->code) == BPF_X) {
-                       if (insn->imm != 0 || insn->off != 0) {
+                       if (insn->imm != 0 || insn->off > 1 ||
+                           (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
                                verbose(env, "BPF_ALU uses reserved fields\n");
                                return -EINVAL;
                        }
@@ -13246,7 +13247,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
                        if (err)
                                return err;
                } else {
-                       if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
+                       if (insn->src_reg != BPF_REG_0 || insn->off > 1 ||
+                           (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
                                verbose(env, "BPF_ALU uses reserved fields\n");
                                return -EINVAL;
                        }