bpf: Recognize addr_space_cast instruction in the verifier.

rY = addr_space_cast(rX, 0, 1) tells the verifier that rY->type = PTR_TO_ARENA.
Any further operations on PTR_TO_ARENA register have to be in 32-bit domain.

The verifier will mark load/store through PTR_TO_ARENA with PROBE_MEM32.
JIT will generate them as kern_vm_start + 32bit_addr memory accesses.

rY = addr_space_cast(rX, 1, 0) tells the verifier that rY->type = unknown scalar.
If arena->map_flags has BPF_F_NO_USER_CONV set then convert cast_user to mov32 as well.
Otherwise JIT will convert it to:
  rY = (u32)rX;
  if (rY)
     rY |= arena->user_vm_start & ~(u64)~0U;

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
Link: https://lore.kernel.org/bpf/20240308010812.89848-6-alexei.starovoitov@gmail.com

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 8904d16061257ff4780481544193a669ac9ada73..d0c836ba009dd9f978beefb49f57e8854f78dddf 100644 (file)
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -883,6 +883,7 @@ enum bpf_reg_type {
         * an explicit null check is required for this struct.
         */
        PTR_TO_MEM,              /* reg points to valid memory region */
+       PTR_TO_ARENA,
        PTR_TO_BUF,              /* reg points to a read/write buffer */
        PTR_TO_FUNC,             /* reg points to a bpf program function */
        CONST_PTR_TO_DYNPTR,     /* reg points to a const struct bpf_dynptr */

diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 4b0f6600e499194cb7d30f59ed19c0bbe9172109..7cb1b75eee381979c0c56d38f871c93c48d352ed 100644 (file)
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -548,6 +548,7 @@ struct bpf_insn_aux_data {
        u32 seen; /* this insn was processed by the verifier at env->pass_cnt */
        bool sanitize_stack_spill; /* subject to Spectre v4 sanitation */
        bool zext_dst; /* this insn zero extends dst reg */
+       bool needs_zext; /* alu op needs to clear upper bits */
        bool storage_get_func_atomic; /* bpf_*_storage_get() with atomic memory alloc */
        bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */
        bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */

diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 63c34e7b07155488784d2e738ea1f7d203241d4f..2a243cf37c60b1dbca97817dcde61c68af9f5cd5 100644 (file)
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -458,6 +458,7 @@ const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type)
                [PTR_TO_XDP_SOCK]       = "xdp_sock",
                [PTR_TO_BTF_ID]         = "ptr_",
                [PTR_TO_MEM]            = "mem",
+               [PTR_TO_ARENA]          = "arena",
                [PTR_TO_BUF]            = "buf",
                [PTR_TO_FUNC]           = "func",
                [PTR_TO_MAP_KEY]        = "map_key",
@@ -693,6 +694,8 @@ static void print_reg_state(struct bpf_verifier_env *env,
        }
 
        verbose(env, "%s", reg_type_str(env, t));
+       if (t == PTR_TO_ARENA)
+               return;
        if (t == PTR_TO_STACK) {
                if (state->frameno != reg->frameno)
                        verbose(env, "[%d]", reg->frameno);

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 67923e41a07edf6c2b3b0e81ef30b42d8d6a8fa6..07f2a4db4511c8f9ffeeec4798e5ed3532eff54b 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -4453,6 +4453,12 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
                        continue;
                }
 
+               if ((BPF_CLASS(code) == BPF_LDX || BPF_CLASS(code) == BPF_STX ||
+                    BPF_CLASS(code) == BPF_ST) && BPF_MODE(code) == BPF_PROBE_MEM32) {
+                       insns[i].code = BPF_CLASS(code) | BPF_SIZE(code) | BPF_MEM;
+                       continue;
+               }
+
                if (code != (BPF_LD | BPF_IMM | BPF_DW))
                        continue;
 

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index fbcf2e5e635af5021960aac74d9496b8fc3be474..1358e20d315adf03e1d9230cf70384f804155f46 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -4386,6 +4386,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type)
        case PTR_TO_MEM:
        case PTR_TO_FUNC:
        case PTR_TO_MAP_KEY:
+       case PTR_TO_ARENA:
                return true;
        default:
                return false;
@@ -5828,6 +5829,8 @@ static int check_ptr_alignment(struct bpf_verifier_env *env,
        case PTR_TO_XDP_SOCK:
                pointer_desc = "xdp_sock ";
                break;
+       case PTR_TO_ARENA:
+               return 0;
        default:
                break;
        }
@@ -6937,6 +6940,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 
                if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
                        mark_reg_unknown(env, regs, value_regno);
+       } else if (reg->type == PTR_TO_ARENA) {
+               if (t == BPF_READ && value_regno >= 0)
+                       mark_reg_unknown(env, regs, value_regno);
        } else {
                verbose(env, "R%d invalid mem access '%s'\n", regno,
                        reg_type_str(env, reg->type));
@@ -8408,6 +8414,7 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env,
        case PTR_TO_MEM | MEM_RINGBUF:
        case PTR_TO_BUF:
        case PTR_TO_BUF | MEM_RDONLY:
+       case PTR_TO_ARENA:
        case SCALAR_VALUE:
                return 0;
        /* All the rest must be rejected, except PTR_TO_BTF_ID which allows
@@ -13852,6 +13859,21 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
 
        dst_reg = &regs[insn->dst_reg];
        src_reg = NULL;
+
+       if (dst_reg->type == PTR_TO_ARENA) {
+               struct bpf_insn_aux_data *aux = cur_aux(env);
+
+               if (BPF_CLASS(insn->code) == BPF_ALU64)
+                       /*
+                        * 32-bit operations zero upper bits automatically.
+                        * 64-bit operations need to be converted to 32.
+                        */
+                       aux->needs_zext = true;
+
+               /* Any arithmetic operations are allowed on arena pointers */
+               return 0;
+       }
+
        if (dst_reg->type != SCALAR_VALUE)
                ptr_reg = dst_reg;
        else
@@ -13969,19 +13991,20 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
        } else if (opcode == BPF_MOV) {
 
                if (BPF_SRC(insn->code) == BPF_X) {
-                       if (insn->imm != 0) {
-                               verbose(env, "BPF_MOV uses reserved fields\n");
-                               return -EINVAL;
-                       }
-
                        if (BPF_CLASS(insn->code) == BPF_ALU) {
-                               if (insn->off != 0 && insn->off != 8 && insn->off != 16) {
+                               if ((insn->off != 0 && insn->off != 8 && insn->off != 16) ||
+                                   insn->imm) {
                                        verbose(env, "BPF_MOV uses reserved fields\n");
                                        return -EINVAL;
                                }
+                       } else if (insn->off == BPF_ADDR_SPACE_CAST) {
+                               if (insn->imm != 1 && insn->imm != 1u << 16) {
+                                       verbose(env, "addr_space_cast insn can only convert between address space 1 and 0\n");
+                                       return -EINVAL;
+                               }
                        } else {
-                               if (insn->off != 0 && insn->off != 8 && insn->off != 16 &&
-                                   insn->off != 32) {
+                               if ((insn->off != 0 && insn->off != 8 && insn->off != 16 &&
+                                    insn->off != 32) || insn->imm) {
                                        verbose(env, "BPF_MOV uses reserved fields\n");
                                        return -EINVAL;
                                }
@@ -14008,7 +14031,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
                        struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
 
                        if (BPF_CLASS(insn->code) == BPF_ALU64) {
-                               if (insn->off == 0) {
+                               if (insn->imm) {
+                                       /* off == BPF_ADDR_SPACE_CAST */
+                                       mark_reg_unknown(env, regs, insn->dst_reg);
+                                       if (insn->imm == 1) /* cast from as(1) to as(0) */
+                                               dst_reg->type = PTR_TO_ARENA;
+                               } else if (insn->off == 0) {
                                        /* case: R1 = R2
                                         * copy register state to dest reg
                                         */
@@ -15182,6 +15210,10 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
 
        if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
            insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
+               if (map->map_type == BPF_MAP_TYPE_ARENA) {
+                       __mark_reg_unknown(env, dst_reg);
+                       return 0;
+               }
                dst_reg->type = PTR_TO_MAP_VALUE;
                dst_reg->off = aux->map_off;
                WARN_ON_ONCE(map->max_entries != 1);
@@ -16568,6 +16600,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
                 * the same stack frame, since fp-8 in foo != fp-8 in bar
                 */
                return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno;
+       case PTR_TO_ARENA:
+               return true;
        default:
                return regs_exact(rold, rcur, idmap);
        }
@@ -17443,6 +17477,7 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type)
        case PTR_TO_TCP_SOCK:
        case PTR_TO_XDP_SOCK:
        case PTR_TO_BTF_ID:
+       case PTR_TO_ARENA:
                return false;
        default:
                return true;
@@ -18296,6 +18331,31 @@ static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
                                fdput(f);
                                return -EBUSY;
                        }
+                       if (map->map_type == BPF_MAP_TYPE_ARENA) {
+                               if (env->prog->aux->arena) {
+                                       verbose(env, "Only one arena per program\n");
+                                       fdput(f);
+                                       return -EBUSY;
+                               }
+                               if (!env->allow_ptr_leaks || !env->bpf_capable) {
+                                       verbose(env, "CAP_BPF and CAP_PERFMON are required to use arena\n");
+                                       fdput(f);
+                                       return -EPERM;
+                               }
+                               if (!env->prog->jit_requested) {
+                                       verbose(env, "JIT is required to use arena\n");
+                                       return -EOPNOTSUPP;
+                               }
+                               if (!bpf_jit_supports_arena()) {
+                                       verbose(env, "JIT doesn't support arena\n");
+                                       return -EOPNOTSUPP;
+                               }
+                               env->prog->aux->arena = (void *)map;
+                               if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {
+                                       verbose(env, "arena's user address must be set via map_extra or mmap()\n");
+                                       return -EINVAL;
+                               }
+                       }
 
                        fdput(f);
 next_insn:
@@ -18917,6 +18977,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
                                env->prog->aux->num_exentries++;
                        }
                        continue;
+               case PTR_TO_ARENA:
+                       if (BPF_MODE(insn->code) == BPF_MEMSX) {
+                               verbose(env, "sign extending loads from arena are not supported yet\n");
+                               return -EOPNOTSUPP;
+                       }
+                       insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
+                       env->prog->aux->num_exentries++;
+                       continue;
                default:
                        continue;
                }
@@ -19102,13 +19170,19 @@ static int jit_subprogs(struct bpf_verifier_env *env)
                func[i]->aux->nr_linfo = prog->aux->nr_linfo;
                func[i]->aux->jited_linfo = prog->aux->jited_linfo;
                func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
+               func[i]->aux->arena = prog->aux->arena;
                num_exentries = 0;
                insn = func[i]->insnsi;
                for (j = 0; j < func[i]->len; j++, insn++) {
                        if (BPF_CLASS(insn->code) == BPF_LDX &&
                            (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
+                            BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
                             BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
                                num_exentries++;
+                       if ((BPF_CLASS(insn->code) == BPF_STX ||
+                            BPF_CLASS(insn->code) == BPF_ST) &&
+                            BPF_MODE(insn->code) == BPF_PROBE_MEM32)
+                               num_exentries++;
                }
                func[i]->aux->num_exentries = num_exentries;
                func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
@@ -19507,6 +19581,21 @@ static int do_misc_fixups(struct bpf_verifier_env *env)
        }
 
        for (i = 0; i < insn_cnt;) {
+               if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) {
+                       if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) ||
+                           (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {
+                               /* convert to 32-bit mov that clears upper 32-bit */
+                               insn->code = BPF_ALU | BPF_MOV | BPF_X;
+                               /* clear off, so it's a normal 'wX = wY' from JIT pov */
+                               insn->off = 0;
+                       } /* cast from as(0) to as(1) should be handled by JIT */
+                       goto next_insn;
+               }
+
+               if (env->insn_aux_data[i + delta].needs_zext)
+                       /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */
+                       insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code);
+
                /* Make divide-by-zero exceptions impossible. */
                if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
                    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||