| 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 2 | /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com |
| 3 | * Copyright (c) 2016 Facebook |
| 4 | * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io |
| 5 | */ |
| 6 | #include <uapi/linux/btf.h> |
| 7 | #include <linux/kernel.h> |
| 8 | #include <linux/types.h> |
| 9 | #include <linux/slab.h> |
| 10 | #include <linux/bpf.h> |
| 11 | #include <linux/btf.h> |
| 12 | #include <linux/bpf_verifier.h> |
| 13 | #include <linux/filter.h> |
| 14 | #include <net/netlink.h> |
| 15 | #include <linux/file.h> |
| 16 | #include <linux/vmalloc.h> |
| 17 | #include <linux/stringify.h> |
| 18 | #include <linux/bsearch.h> |
| 19 | #include <linux/sort.h> |
| 20 | #include <linux/perf_event.h> |
| 21 | #include <linux/ctype.h> |
| 22 | #include <linux/error-injection.h> |
| 23 | #include <linux/bpf_lsm.h> |
| 24 | #include <linux/btf_ids.h> |
| 25 | |
| 26 | #include "disasm.h" |
| 27 | |
| 28 | static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { |
| 29 | #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ |
| 30 | [_id] = & _name ## _verifier_ops, |
| 31 | #define BPF_MAP_TYPE(_id, _ops) |
| 32 | #define BPF_LINK_TYPE(_id, _name) |
| 33 | #include <linux/bpf_types.h> |
| 34 | #undef BPF_PROG_TYPE |
| 35 | #undef BPF_MAP_TYPE |
| 36 | #undef BPF_LINK_TYPE |
| 37 | }; |
| 38 | |
| 39 | /* bpf_check() is a static code analyzer that walks eBPF program |
| 40 | * instruction by instruction and updates register/stack state. |
| 41 | * All paths of conditional branches are analyzed until 'bpf_exit' insn. |
| 42 | * |
| 43 | * The first pass is depth-first-search to check that the program is a DAG. |
| 44 | * It rejects the following programs: |
| 45 | * - larger than BPF_MAXINSNS insns |
| 46 | * - if loop is present (detected via back-edge) |
| 47 | * - unreachable insns exist (shouldn't be a forest. program = one function) |
| 48 | * - out of bounds or malformed jumps |
| 49 | * The second pass is all possible path descent from the 1st insn. |
| 50 | * Since it's analyzing all pathes through the program, the length of the |
| 51 | * analysis is limited to 64k insn, which may be hit even if total number of |
| 52 | * insn is less then 4K, but there are too many branches that change stack/regs. |
| 53 | * Number of 'branches to be analyzed' is limited to 1k |
| 54 | * |
| 55 | * On entry to each instruction, each register has a type, and the instruction |
| 56 | * changes the types of the registers depending on instruction semantics. |
| 57 | * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is |
| 58 | * copied to R1. |
| 59 | * |
| 60 | * All registers are 64-bit. |
| 61 | * R0 - return register |
| 62 | * R1-R5 argument passing registers |
| 63 | * R6-R9 callee saved registers |
| 64 | * R10 - frame pointer read-only |
| 65 | * |
| 66 | * At the start of BPF program the register R1 contains a pointer to bpf_context |
| 67 | * and has type PTR_TO_CTX. |
| 68 | * |
| 69 | * Verifier tracks arithmetic operations on pointers in case: |
| 70 | * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), |
| 71 | * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), |
| 72 | * 1st insn copies R10 (which has FRAME_PTR) type into R1 |
| 73 | * and 2nd arithmetic instruction is pattern matched to recognize |
| 74 | * that it wants to construct a pointer to some element within stack. |
| 75 | * So after 2nd insn, the register R1 has type PTR_TO_STACK |
| 76 | * (and -20 constant is saved for further stack bounds checking). |
| 77 | * Meaning that this reg is a pointer to stack plus known immediate constant. |
| 78 | * |
| 79 | * Most of the time the registers have SCALAR_VALUE type, which |
| 80 | * means the register has some value, but it's not a valid pointer. |
| 81 | * (like pointer plus pointer becomes SCALAR_VALUE type) |
| 82 | * |
| 83 | * When verifier sees load or store instructions the type of base register |
| 84 | * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are |
| 85 | * four pointer types recognized by check_mem_access() function. |
| 86 | * |
| 87 | * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' |
| 88 | * and the range of [ptr, ptr + map's value_size) is accessible. |
| 89 | * |
| 90 | * registers used to pass values to function calls are checked against |
| 91 | * function argument constraints. |
| 92 | * |
| 93 | * ARG_PTR_TO_MAP_KEY is one of such argument constraints. |
| 94 | * It means that the register type passed to this function must be |
| 95 | * PTR_TO_STACK and it will be used inside the function as |
| 96 | * 'pointer to map element key' |
| 97 | * |
| 98 | * For example the argument constraints for bpf_map_lookup_elem(): |
| 99 | * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, |
| 100 | * .arg1_type = ARG_CONST_MAP_PTR, |
| 101 | * .arg2_type = ARG_PTR_TO_MAP_KEY, |
| 102 | * |
| 103 | * ret_type says that this function returns 'pointer to map elem value or null' |
| 104 | * function expects 1st argument to be a const pointer to 'struct bpf_map' and |
| 105 | * 2nd argument should be a pointer to stack, which will be used inside |
| 106 | * the helper function as a pointer to map element key. |
| 107 | * |
| 108 | * On the kernel side the helper function looks like: |
| 109 | * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) |
| 110 | * { |
| 111 | * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; |
| 112 | * void *key = (void *) (unsigned long) r2; |
| 113 | * void *value; |
| 114 | * |
| 115 | * here kernel can access 'key' and 'map' pointers safely, knowing that |
| 116 | * [key, key + map->key_size) bytes are valid and were initialized on |
| 117 | * the stack of eBPF program. |
| 118 | * } |
| 119 | * |
| 120 | * Corresponding eBPF program may look like: |
| 121 | * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR |
| 122 | * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK |
| 123 | * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP |
| 124 | * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), |
| 125 | * here verifier looks at prototype of map_lookup_elem() and sees: |
| 126 | * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, |
| 127 | * Now verifier knows that this map has key of R1->map_ptr->key_size bytes |
| 128 | * |
| 129 | * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, |
| 130 | * Now verifier checks that [R2, R2 + map's key_size) are within stack limits |
| 131 | * and were initialized prior to this call. |
| 132 | * If it's ok, then verifier allows this BPF_CALL insn and looks at |
| 133 | * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets |
| 134 | * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function |
| 135 | * returns ether pointer to map value or NULL. |
| 136 | * |
| 137 | * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' |
| 138 | * insn, the register holding that pointer in the true branch changes state to |
| 139 | * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false |
| 140 | * branch. See check_cond_jmp_op(). |
| 141 | * |
| 142 | * After the call R0 is set to return type of the function and registers R1-R5 |
| 143 | * are set to NOT_INIT to indicate that they are no longer readable. |
| 144 | * |
| 145 | * The following reference types represent a potential reference to a kernel |
| 146 | * resource which, after first being allocated, must be checked and freed by |
| 147 | * the BPF program: |
| 148 | * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET |
| 149 | * |
| 150 | * When the verifier sees a helper call return a reference type, it allocates a |
| 151 | * pointer id for the reference and stores it in the current function state. |
| 152 | * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into |
| 153 | * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type |
| 154 | * passes through a NULL-check conditional. For the branch wherein the state is |
| 155 | * changed to CONST_IMM, the verifier releases the reference. |
| 156 | * |
| 157 | * For each helper function that allocates a reference, such as |
| 158 | * bpf_sk_lookup_tcp(), there is a corresponding release function, such as |
| 159 | * bpf_sk_release(). When a reference type passes into the release function, |
| 160 | * the verifier also releases the reference. If any unchecked or unreleased |
| 161 | * reference remains at the end of the program, the verifier rejects it. |
| 162 | */ |
| 163 | |
| 164 | /* verifier_state + insn_idx are pushed to stack when branch is encountered */ |
| 165 | struct bpf_verifier_stack_elem { |
| 166 | /* verifer state is 'st' |
| 167 | * before processing instruction 'insn_idx' |
| 168 | * and after processing instruction 'prev_insn_idx' |
| 169 | */ |
| 170 | struct bpf_verifier_state st; |
| 171 | int insn_idx; |
| 172 | int prev_insn_idx; |
| 173 | struct bpf_verifier_stack_elem *next; |
| 174 | /* length of verifier log at the time this state was pushed on stack */ |
| 175 | u32 log_pos; |
| 176 | }; |
| 177 | |
| 178 | #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 |
| 179 | #define BPF_COMPLEXITY_LIMIT_STATES 64 |
| 180 | |
| 181 | #define BPF_MAP_KEY_POISON (1ULL << 63) |
| 182 | #define BPF_MAP_KEY_SEEN (1ULL << 62) |
| 183 | |
| 184 | #define BPF_MAP_PTR_UNPRIV 1UL |
| 185 | #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ |
| 186 | POISON_POINTER_DELTA)) |
| 187 | #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) |
| 188 | |
| 189 | static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) |
| 190 | { |
| 191 | return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON; |
| 192 | } |
| 193 | |
| 194 | static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) |
| 195 | { |
| 196 | return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV; |
| 197 | } |
| 198 | |
| 199 | static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, |
| 200 | const struct bpf_map *map, bool unpriv) |
| 201 | { |
| 202 | BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); |
| 203 | unpriv |= bpf_map_ptr_unpriv(aux); |
| 204 | aux->map_ptr_state = (unsigned long)map | |
| 205 | (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); |
| 206 | } |
| 207 | |
| 208 | static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) |
| 209 | { |
| 210 | return aux->map_key_state & BPF_MAP_KEY_POISON; |
| 211 | } |
| 212 | |
| 213 | static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) |
| 214 | { |
| 215 | return !(aux->map_key_state & BPF_MAP_KEY_SEEN); |
| 216 | } |
| 217 | |
| 218 | static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) |
| 219 | { |
| 220 | return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); |
| 221 | } |
| 222 | |
| 223 | static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state) |
| 224 | { |
| 225 | bool poisoned = bpf_map_key_poisoned(aux); |
| 226 | |
| 227 | aux->map_key_state = state | BPF_MAP_KEY_SEEN | |
| 228 | (poisoned ? BPF_MAP_KEY_POISON : 0ULL); |
| 229 | } |
| 230 | |
| 231 | static bool bpf_pseudo_call(const struct bpf_insn *insn) |
| 232 | { |
| 233 | return insn->code == (BPF_JMP | BPF_CALL) && |
| 234 | insn->src_reg == BPF_PSEUDO_CALL; |
| 235 | } |
| 236 | |
| 237 | struct bpf_call_arg_meta { |
| 238 | struct bpf_map *map_ptr; |
| 239 | bool raw_mode; |
| 240 | bool pkt_access; |
| 241 | int regno; |
| 242 | int access_size; |
| 243 | int mem_size; |
| 244 | u64 msize_max_value; |
| 245 | int ref_obj_id; |
| 246 | int func_id; |
| 247 | struct btf *btf; |
| 248 | u32 btf_id; |
| 249 | struct btf *ret_btf; |
| 250 | u32 ret_btf_id; |
| 251 | }; |
| 252 | |
| 253 | struct btf *btf_vmlinux; |
| 254 | |
| 255 | static DEFINE_MUTEX(bpf_verifier_lock); |
| 256 | |
| 257 | static const struct bpf_line_info * |
| 258 | find_linfo(const struct bpf_verifier_env *env, u32 insn_off) |
| 259 | { |
| 260 | const struct bpf_line_info *linfo; |
| 261 | const struct bpf_prog *prog; |
| 262 | u32 i, nr_linfo; |
| 263 | |
| 264 | prog = env->prog; |
| 265 | nr_linfo = prog->aux->nr_linfo; |
| 266 | |
| 267 | if (!nr_linfo || insn_off >= prog->len) |
| 268 | return NULL; |
| 269 | |
| 270 | linfo = prog->aux->linfo; |
| 271 | for (i = 1; i < nr_linfo; i++) |
| 272 | if (insn_off < linfo[i].insn_off) |
| 273 | break; |
| 274 | |
| 275 | return &linfo[i - 1]; |
| 276 | } |
| 277 | |
| 278 | void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, |
| 279 | va_list args) |
| 280 | { |
| 281 | unsigned int n; |
| 282 | |
| 283 | n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); |
| 284 | |
| 285 | WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, |
| 286 | "verifier log line truncated - local buffer too short\n"); |
| 287 | |
| 288 | n = min(log->len_total - log->len_used - 1, n); |
| 289 | log->kbuf[n] = '\0'; |
| 290 | |
| 291 | if (log->level == BPF_LOG_KERNEL) { |
| 292 | pr_err("BPF:%s\n", log->kbuf); |
| 293 | return; |
| 294 | } |
| 295 | if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) |
| 296 | log->len_used += n; |
| 297 | else |
| 298 | log->ubuf = NULL; |
| 299 | } |
| 300 | |
| 301 | static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos) |
| 302 | { |
| 303 | char zero = 0; |
| 304 | |
| 305 | if (!bpf_verifier_log_needed(log)) |
| 306 | return; |
| 307 | |
| 308 | log->len_used = new_pos; |
| 309 | if (put_user(zero, log->ubuf + new_pos)) |
| 310 | log->ubuf = NULL; |
| 311 | } |
| 312 | |
| 313 | /* log_level controls verbosity level of eBPF verifier. |
| 314 | * bpf_verifier_log_write() is used to dump the verification trace to the log, |
| 315 | * so the user can figure out what's wrong with the program |
| 316 | */ |
| 317 | __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, |
| 318 | const char *fmt, ...) |
| 319 | { |
| 320 | va_list args; |
| 321 | |
| 322 | if (!bpf_verifier_log_needed(&env->log)) |
| 323 | return; |
| 324 | |
| 325 | va_start(args, fmt); |
| 326 | bpf_verifier_vlog(&env->log, fmt, args); |
| 327 | va_end(args); |
| 328 | } |
| 329 | EXPORT_SYMBOL_GPL(bpf_verifier_log_write); |
| 330 | |
| 331 | __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) |
| 332 | { |
| 333 | struct bpf_verifier_env *env = private_data; |
| 334 | va_list args; |
| 335 | |
| 336 | if (!bpf_verifier_log_needed(&env->log)) |
| 337 | return; |
| 338 | |
| 339 | va_start(args, fmt); |
| 340 | bpf_verifier_vlog(&env->log, fmt, args); |
| 341 | va_end(args); |
| 342 | } |
| 343 | |
| 344 | __printf(2, 3) void bpf_log(struct bpf_verifier_log *log, |
| 345 | const char *fmt, ...) |
| 346 | { |
| 347 | va_list args; |
| 348 | |
| 349 | if (!bpf_verifier_log_needed(log)) |
| 350 | return; |
| 351 | |
| 352 | va_start(args, fmt); |
| 353 | bpf_verifier_vlog(log, fmt, args); |
| 354 | va_end(args); |
| 355 | } |
| 356 | |
| 357 | static const char *ltrim(const char *s) |
| 358 | { |
| 359 | while (isspace(*s)) |
| 360 | s++; |
| 361 | |
| 362 | return s; |
| 363 | } |
| 364 | |
| 365 | __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, |
| 366 | u32 insn_off, |
| 367 | const char *prefix_fmt, ...) |
| 368 | { |
| 369 | const struct bpf_line_info *linfo; |
| 370 | |
| 371 | if (!bpf_verifier_log_needed(&env->log)) |
| 372 | return; |
| 373 | |
| 374 | linfo = find_linfo(env, insn_off); |
| 375 | if (!linfo || linfo == env->prev_linfo) |
| 376 | return; |
| 377 | |
| 378 | if (prefix_fmt) { |
| 379 | va_list args; |
| 380 | |
| 381 | va_start(args, prefix_fmt); |
| 382 | bpf_verifier_vlog(&env->log, prefix_fmt, args); |
| 383 | va_end(args); |
| 384 | } |
| 385 | |
| 386 | verbose(env, "%s\n", |
| 387 | ltrim(btf_name_by_offset(env->prog->aux->btf, |
| 388 | linfo->line_off))); |
| 389 | |
| 390 | env->prev_linfo = linfo; |
| 391 | } |
| 392 | |
| 393 | static bool type_is_pkt_pointer(enum bpf_reg_type type) |
| 394 | { |
| 395 | return type == PTR_TO_PACKET || |
| 396 | type == PTR_TO_PACKET_META; |
| 397 | } |
| 398 | |
| 399 | static bool type_is_sk_pointer(enum bpf_reg_type type) |
| 400 | { |
| 401 | return type == PTR_TO_SOCKET || |
| 402 | type == PTR_TO_SOCK_COMMON || |
| 403 | type == PTR_TO_TCP_SOCK || |
| 404 | type == PTR_TO_XDP_SOCK; |
| 405 | } |
| 406 | |
| 407 | static bool reg_type_not_null(enum bpf_reg_type type) |
| 408 | { |
| 409 | return type == PTR_TO_SOCKET || |
| 410 | type == PTR_TO_TCP_SOCK || |
| 411 | type == PTR_TO_MAP_VALUE || |
| 412 | type == PTR_TO_SOCK_COMMON; |
| 413 | } |
| 414 | |
| 415 | static bool reg_type_may_be_null(enum bpf_reg_type type) |
| 416 | { |
| 417 | return type == PTR_TO_MAP_VALUE_OR_NULL || |
| 418 | type == PTR_TO_SOCKET_OR_NULL || |
| 419 | type == PTR_TO_SOCK_COMMON_OR_NULL || |
| 420 | type == PTR_TO_TCP_SOCK_OR_NULL || |
| 421 | type == PTR_TO_BTF_ID_OR_NULL || |
| 422 | type == PTR_TO_MEM_OR_NULL || |
| 423 | type == PTR_TO_RDONLY_BUF_OR_NULL || |
| 424 | type == PTR_TO_RDWR_BUF_OR_NULL; |
| 425 | } |
| 426 | |
| 427 | static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) |
| 428 | { |
| 429 | return reg->type == PTR_TO_MAP_VALUE && |
| 430 | map_value_has_spin_lock(reg->map_ptr); |
| 431 | } |
| 432 | |
| 433 | static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) |
| 434 | { |
| 435 | return type == PTR_TO_SOCKET || |
| 436 | type == PTR_TO_SOCKET_OR_NULL || |
| 437 | type == PTR_TO_TCP_SOCK || |
| 438 | type == PTR_TO_TCP_SOCK_OR_NULL || |
| 439 | type == PTR_TO_MEM || |
| 440 | type == PTR_TO_MEM_OR_NULL; |
| 441 | } |
| 442 | |
| 443 | static bool arg_type_may_be_refcounted(enum bpf_arg_type type) |
| 444 | { |
| 445 | return type == ARG_PTR_TO_SOCK_COMMON; |
| 446 | } |
| 447 | |
| 448 | static bool arg_type_may_be_null(enum bpf_arg_type type) |
| 449 | { |
| 450 | return type == ARG_PTR_TO_MAP_VALUE_OR_NULL || |
| 451 | type == ARG_PTR_TO_MEM_OR_NULL || |
| 452 | type == ARG_PTR_TO_CTX_OR_NULL || |
| 453 | type == ARG_PTR_TO_SOCKET_OR_NULL || |
| 454 | type == ARG_PTR_TO_ALLOC_MEM_OR_NULL; |
| 455 | } |
| 456 | |
| 457 | /* Determine whether the function releases some resources allocated by another |
| 458 | * function call. The first reference type argument will be assumed to be |
| 459 | * released by release_reference(). |
| 460 | */ |
| 461 | static bool is_release_function(enum bpf_func_id func_id) |
| 462 | { |
| 463 | return func_id == BPF_FUNC_sk_release || |
| 464 | func_id == BPF_FUNC_ringbuf_submit || |
| 465 | func_id == BPF_FUNC_ringbuf_discard; |
| 466 | } |
| 467 | |
| 468 | static bool may_be_acquire_function(enum bpf_func_id func_id) |
| 469 | { |
| 470 | return func_id == BPF_FUNC_sk_lookup_tcp || |
| 471 | func_id == BPF_FUNC_sk_lookup_udp || |
| 472 | func_id == BPF_FUNC_skc_lookup_tcp || |
| 473 | func_id == BPF_FUNC_map_lookup_elem || |
| 474 | func_id == BPF_FUNC_ringbuf_reserve; |
| 475 | } |
| 476 | |
| 477 | static bool is_acquire_function(enum bpf_func_id func_id, |
| 478 | const struct bpf_map *map) |
| 479 | { |
| 480 | enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC; |
| 481 | |
| 482 | if (func_id == BPF_FUNC_sk_lookup_tcp || |
| 483 | func_id == BPF_FUNC_sk_lookup_udp || |
| 484 | func_id == BPF_FUNC_skc_lookup_tcp || |
| 485 | func_id == BPF_FUNC_ringbuf_reserve) |
| 486 | return true; |
| 487 | |
| 488 | if (func_id == BPF_FUNC_map_lookup_elem && |
| 489 | (map_type == BPF_MAP_TYPE_SOCKMAP || |
| 490 | map_type == BPF_MAP_TYPE_SOCKHASH)) |
| 491 | return true; |
| 492 | |
| 493 | return false; |
| 494 | } |
| 495 | |
| 496 | static bool is_ptr_cast_function(enum bpf_func_id func_id) |
| 497 | { |
| 498 | return func_id == BPF_FUNC_tcp_sock || |
| 499 | func_id == BPF_FUNC_sk_fullsock || |
| 500 | func_id == BPF_FUNC_skc_to_tcp_sock || |
| 501 | func_id == BPF_FUNC_skc_to_tcp6_sock || |
| 502 | func_id == BPF_FUNC_skc_to_udp6_sock || |
| 503 | func_id == BPF_FUNC_skc_to_tcp_timewait_sock || |
| 504 | func_id == BPF_FUNC_skc_to_tcp_request_sock; |
| 505 | } |
| 506 | |
| 507 | static bool is_cmpxchg_insn(const struct bpf_insn *insn) |
| 508 | { |
| 509 | return BPF_CLASS(insn->code) == BPF_STX && |
| 510 | BPF_MODE(insn->code) == BPF_ATOMIC && |
| 511 | insn->imm == BPF_CMPXCHG; |
| 512 | } |
| 513 | |
| 514 | /* string representation of 'enum bpf_reg_type' */ |
| 515 | static const char * const reg_type_str[] = { |
| 516 | [NOT_INIT] = "?", |
| 517 | [SCALAR_VALUE] = "inv", |
| 518 | [PTR_TO_CTX] = "ctx", |
| 519 | [CONST_PTR_TO_MAP] = "map_ptr", |
| 520 | [PTR_TO_MAP_VALUE] = "map_value", |
| 521 | [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", |
| 522 | [PTR_TO_STACK] = "fp", |
| 523 | [PTR_TO_PACKET] = "pkt", |
| 524 | [PTR_TO_PACKET_META] = "pkt_meta", |
| 525 | [PTR_TO_PACKET_END] = "pkt_end", |
| 526 | [PTR_TO_FLOW_KEYS] = "flow_keys", |
| 527 | [PTR_TO_SOCKET] = "sock", |
| 528 | [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", |
| 529 | [PTR_TO_SOCK_COMMON] = "sock_common", |
| 530 | [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", |
| 531 | [PTR_TO_TCP_SOCK] = "tcp_sock", |
| 532 | [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", |
| 533 | [PTR_TO_TP_BUFFER] = "tp_buffer", |
| 534 | [PTR_TO_XDP_SOCK] = "xdp_sock", |
| 535 | [PTR_TO_BTF_ID] = "ptr_", |
| 536 | [PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_", |
| 537 | [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_", |
| 538 | [PTR_TO_MEM] = "mem", |
| 539 | [PTR_TO_MEM_OR_NULL] = "mem_or_null", |
| 540 | [PTR_TO_RDONLY_BUF] = "rdonly_buf", |
| 541 | [PTR_TO_RDONLY_BUF_OR_NULL] = "rdonly_buf_or_null", |
| 542 | [PTR_TO_RDWR_BUF] = "rdwr_buf", |
| 543 | [PTR_TO_RDWR_BUF_OR_NULL] = "rdwr_buf_or_null", |
| 544 | }; |
| 545 | |
| 546 | static char slot_type_char[] = { |
| 547 | [STACK_INVALID] = '?', |
| 548 | [STACK_SPILL] = 'r', |
| 549 | [STACK_MISC] = 'm', |
| 550 | [STACK_ZERO] = '0', |
| 551 | }; |
| 552 | |
| 553 | static void print_liveness(struct bpf_verifier_env *env, |
| 554 | enum bpf_reg_liveness live) |
| 555 | { |
| 556 | if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) |
| 557 | verbose(env, "_"); |
| 558 | if (live & REG_LIVE_READ) |
| 559 | verbose(env, "r"); |
| 560 | if (live & REG_LIVE_WRITTEN) |
| 561 | verbose(env, "w"); |
| 562 | if (live & REG_LIVE_DONE) |
| 563 | verbose(env, "D"); |
| 564 | } |
| 565 | |
| 566 | static struct bpf_func_state *func(struct bpf_verifier_env *env, |
| 567 | const struct bpf_reg_state *reg) |
| 568 | { |
| 569 | struct bpf_verifier_state *cur = env->cur_state; |
| 570 | |
| 571 | return cur->frame[reg->frameno]; |
| 572 | } |
| 573 | |
| 574 | static const char *kernel_type_name(const struct btf* btf, u32 id) |
| 575 | { |
| 576 | return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off); |
| 577 | } |
| 578 | |
| 579 | static void print_verifier_state(struct bpf_verifier_env *env, |
| 580 | const struct bpf_func_state *state) |
| 581 | { |
| 582 | const struct bpf_reg_state *reg; |
| 583 | enum bpf_reg_type t; |
| 584 | int i; |
| 585 | |
| 586 | if (state->frameno) |
| 587 | verbose(env, " frame%d:", state->frameno); |
| 588 | for (i = 0; i < MAX_BPF_REG; i++) { |
| 589 | reg = &state->regs[i]; |
| 590 | t = reg->type; |
| 591 | if (t == NOT_INIT) |
| 592 | continue; |
| 593 | verbose(env, " R%d", i); |
| 594 | print_liveness(env, reg->live); |
| 595 | verbose(env, "=%s", reg_type_str[t]); |
| 596 | if (t == SCALAR_VALUE && reg->precise) |
| 597 | verbose(env, "P"); |
| 598 | if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && |
| 599 | tnum_is_const(reg->var_off)) { |
| 600 | /* reg->off should be 0 for SCALAR_VALUE */ |
| 601 | verbose(env, "%lld", reg->var_off.value + reg->off); |
| 602 | } else { |
| 603 | if (t == PTR_TO_BTF_ID || |
| 604 | t == PTR_TO_BTF_ID_OR_NULL || |
| 605 | t == PTR_TO_PERCPU_BTF_ID) |
| 606 | verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id)); |
| 607 | verbose(env, "(id=%d", reg->id); |
| 608 | if (reg_type_may_be_refcounted_or_null(t)) |
| 609 | verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); |
| 610 | if (t != SCALAR_VALUE) |
| 611 | verbose(env, ",off=%d", reg->off); |
| 612 | if (type_is_pkt_pointer(t)) |
| 613 | verbose(env, ",r=%d", reg->range); |
| 614 | else if (t == CONST_PTR_TO_MAP || |
| 615 | t == PTR_TO_MAP_VALUE || |
| 616 | t == PTR_TO_MAP_VALUE_OR_NULL) |
| 617 | verbose(env, ",ks=%d,vs=%d", |
| 618 | reg->map_ptr->key_size, |
| 619 | reg->map_ptr->value_size); |
| 620 | if (tnum_is_const(reg->var_off)) { |
| 621 | /* Typically an immediate SCALAR_VALUE, but |
| 622 | * could be a pointer whose offset is too big |
| 623 | * for reg->off |
| 624 | */ |
| 625 | verbose(env, ",imm=%llx", reg->var_off.value); |
| 626 | } else { |
| 627 | if (reg->smin_value != reg->umin_value && |
| 628 | reg->smin_value != S64_MIN) |
| 629 | verbose(env, ",smin_value=%lld", |
| 630 | (long long)reg->smin_value); |
| 631 | if (reg->smax_value != reg->umax_value && |
| 632 | reg->smax_value != S64_MAX) |
| 633 | verbose(env, ",smax_value=%lld", |
| 634 | (long long)reg->smax_value); |
| 635 | if (reg->umin_value != 0) |
| 636 | verbose(env, ",umin_value=%llu", |
| 637 | (unsigned long long)reg->umin_value); |
| 638 | if (reg->umax_value != U64_MAX) |
| 639 | verbose(env, ",umax_value=%llu", |
| 640 | (unsigned long long)reg->umax_value); |
| 641 | if (!tnum_is_unknown(reg->var_off)) { |
| 642 | char tn_buf[48]; |
| 643 | |
| 644 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 645 | verbose(env, ",var_off=%s", tn_buf); |
| 646 | } |
| 647 | if (reg->s32_min_value != reg->smin_value && |
| 648 | reg->s32_min_value != S32_MIN) |
| 649 | verbose(env, ",s32_min_value=%d", |
| 650 | (int)(reg->s32_min_value)); |
| 651 | if (reg->s32_max_value != reg->smax_value && |
| 652 | reg->s32_max_value != S32_MAX) |
| 653 | verbose(env, ",s32_max_value=%d", |
| 654 | (int)(reg->s32_max_value)); |
| 655 | if (reg->u32_min_value != reg->umin_value && |
| 656 | reg->u32_min_value != U32_MIN) |
| 657 | verbose(env, ",u32_min_value=%d", |
| 658 | (int)(reg->u32_min_value)); |
| 659 | if (reg->u32_max_value != reg->umax_value && |
| 660 | reg->u32_max_value != U32_MAX) |
| 661 | verbose(env, ",u32_max_value=%d", |
| 662 | (int)(reg->u32_max_value)); |
| 663 | } |
| 664 | verbose(env, ")"); |
| 665 | } |
| 666 | } |
| 667 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { |
| 668 | char types_buf[BPF_REG_SIZE + 1]; |
| 669 | bool valid = false; |
| 670 | int j; |
| 671 | |
| 672 | for (j = 0; j < BPF_REG_SIZE; j++) { |
| 673 | if (state->stack[i].slot_type[j] != STACK_INVALID) |
| 674 | valid = true; |
| 675 | types_buf[j] = slot_type_char[ |
| 676 | state->stack[i].slot_type[j]]; |
| 677 | } |
| 678 | types_buf[BPF_REG_SIZE] = 0; |
| 679 | if (!valid) |
| 680 | continue; |
| 681 | verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); |
| 682 | print_liveness(env, state->stack[i].spilled_ptr.live); |
| 683 | if (state->stack[i].slot_type[0] == STACK_SPILL) { |
| 684 | reg = &state->stack[i].spilled_ptr; |
| 685 | t = reg->type; |
| 686 | verbose(env, "=%s", reg_type_str[t]); |
| 687 | if (t == SCALAR_VALUE && reg->precise) |
| 688 | verbose(env, "P"); |
| 689 | if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) |
| 690 | verbose(env, "%lld", reg->var_off.value + reg->off); |
| 691 | } else { |
| 692 | verbose(env, "=%s", types_buf); |
| 693 | } |
| 694 | } |
| 695 | if (state->acquired_refs && state->refs[0].id) { |
| 696 | verbose(env, " refs=%d", state->refs[0].id); |
| 697 | for (i = 1; i < state->acquired_refs; i++) |
| 698 | if (state->refs[i].id) |
| 699 | verbose(env, ",%d", state->refs[i].id); |
| 700 | } |
| 701 | verbose(env, "\n"); |
| 702 | } |
| 703 | |
| 704 | #define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \ |
| 705 | static int copy_##NAME##_state(struct bpf_func_state *dst, \ |
| 706 | const struct bpf_func_state *src) \ |
| 707 | { \ |
| 708 | if (!src->FIELD) \ |
| 709 | return 0; \ |
| 710 | if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \ |
| 711 | /* internal bug, make state invalid to reject the program */ \ |
| 712 | memset(dst, 0, sizeof(*dst)); \ |
| 713 | return -EFAULT; \ |
| 714 | } \ |
| 715 | memcpy(dst->FIELD, src->FIELD, \ |
| 716 | sizeof(*src->FIELD) * (src->COUNT / SIZE)); \ |
| 717 | return 0; \ |
| 718 | } |
| 719 | /* copy_reference_state() */ |
| 720 | COPY_STATE_FN(reference, acquired_refs, refs, 1) |
| 721 | /* copy_stack_state() */ |
| 722 | COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) |
| 723 | #undef COPY_STATE_FN |
| 724 | |
| 725 | #define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \ |
| 726 | static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \ |
| 727 | bool copy_old) \ |
| 728 | { \ |
| 729 | u32 old_size = state->COUNT; \ |
| 730 | struct bpf_##NAME##_state *new_##FIELD; \ |
| 731 | int slot = size / SIZE; \ |
| 732 | \ |
| 733 | if (size <= old_size || !size) { \ |
| 734 | if (copy_old) \ |
| 735 | return 0; \ |
| 736 | state->COUNT = slot * SIZE; \ |
| 737 | if (!size && old_size) { \ |
| 738 | kfree(state->FIELD); \ |
| 739 | state->FIELD = NULL; \ |
| 740 | } \ |
| 741 | return 0; \ |
| 742 | } \ |
| 743 | new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \ |
| 744 | GFP_KERNEL); \ |
| 745 | if (!new_##FIELD) \ |
| 746 | return -ENOMEM; \ |
| 747 | if (copy_old) { \ |
| 748 | if (state->FIELD) \ |
| 749 | memcpy(new_##FIELD, state->FIELD, \ |
| 750 | sizeof(*new_##FIELD) * (old_size / SIZE)); \ |
| 751 | memset(new_##FIELD + old_size / SIZE, 0, \ |
| 752 | sizeof(*new_##FIELD) * (size - old_size) / SIZE); \ |
| 753 | } \ |
| 754 | state->COUNT = slot * SIZE; \ |
| 755 | kfree(state->FIELD); \ |
| 756 | state->FIELD = new_##FIELD; \ |
| 757 | return 0; \ |
| 758 | } |
| 759 | /* realloc_reference_state() */ |
| 760 | REALLOC_STATE_FN(reference, acquired_refs, refs, 1) |
| 761 | /* realloc_stack_state() */ |
| 762 | REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) |
| 763 | #undef REALLOC_STATE_FN |
| 764 | |
| 765 | /* do_check() starts with zero-sized stack in struct bpf_verifier_state to |
| 766 | * make it consume minimal amount of memory. check_stack_write() access from |
| 767 | * the program calls into realloc_func_state() to grow the stack size. |
| 768 | * Note there is a non-zero 'parent' pointer inside bpf_verifier_state |
| 769 | * which realloc_stack_state() copies over. It points to previous |
| 770 | * bpf_verifier_state which is never reallocated. |
| 771 | */ |
| 772 | static int realloc_func_state(struct bpf_func_state *state, int stack_size, |
| 773 | int refs_size, bool copy_old) |
| 774 | { |
| 775 | int err = realloc_reference_state(state, refs_size, copy_old); |
| 776 | if (err) |
| 777 | return err; |
| 778 | return realloc_stack_state(state, stack_size, copy_old); |
| 779 | } |
| 780 | |
| 781 | /* Acquire a pointer id from the env and update the state->refs to include |
| 782 | * this new pointer reference. |
| 783 | * On success, returns a valid pointer id to associate with the register |
| 784 | * On failure, returns a negative errno. |
| 785 | */ |
| 786 | static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) |
| 787 | { |
| 788 | struct bpf_func_state *state = cur_func(env); |
| 789 | int new_ofs = state->acquired_refs; |
| 790 | int id, err; |
| 791 | |
| 792 | err = realloc_reference_state(state, state->acquired_refs + 1, true); |
| 793 | if (err) |
| 794 | return err; |
| 795 | id = ++env->id_gen; |
| 796 | state->refs[new_ofs].id = id; |
| 797 | state->refs[new_ofs].insn_idx = insn_idx; |
| 798 | |
| 799 | return id; |
| 800 | } |
| 801 | |
| 802 | /* release function corresponding to acquire_reference_state(). Idempotent. */ |
| 803 | static int release_reference_state(struct bpf_func_state *state, int ptr_id) |
| 804 | { |
| 805 | int i, last_idx; |
| 806 | |
| 807 | last_idx = state->acquired_refs - 1; |
| 808 | for (i = 0; i < state->acquired_refs; i++) { |
| 809 | if (state->refs[i].id == ptr_id) { |
| 810 | if (last_idx && i != last_idx) |
| 811 | memcpy(&state->refs[i], &state->refs[last_idx], |
| 812 | sizeof(*state->refs)); |
| 813 | memset(&state->refs[last_idx], 0, sizeof(*state->refs)); |
| 814 | state->acquired_refs--; |
| 815 | return 0; |
| 816 | } |
| 817 | } |
| 818 | return -EINVAL; |
| 819 | } |
| 820 | |
| 821 | static int transfer_reference_state(struct bpf_func_state *dst, |
| 822 | struct bpf_func_state *src) |
| 823 | { |
| 824 | int err = realloc_reference_state(dst, src->acquired_refs, false); |
| 825 | if (err) |
| 826 | return err; |
| 827 | err = copy_reference_state(dst, src); |
| 828 | if (err) |
| 829 | return err; |
| 830 | return 0; |
| 831 | } |
| 832 | |
| 833 | static void free_func_state(struct bpf_func_state *state) |
| 834 | { |
| 835 | if (!state) |
| 836 | return; |
| 837 | kfree(state->refs); |
| 838 | kfree(state->stack); |
| 839 | kfree(state); |
| 840 | } |
| 841 | |
| 842 | static void clear_jmp_history(struct bpf_verifier_state *state) |
| 843 | { |
| 844 | kfree(state->jmp_history); |
| 845 | state->jmp_history = NULL; |
| 846 | state->jmp_history_cnt = 0; |
| 847 | } |
| 848 | |
| 849 | static void free_verifier_state(struct bpf_verifier_state *state, |
| 850 | bool free_self) |
| 851 | { |
| 852 | int i; |
| 853 | |
| 854 | for (i = 0; i <= state->curframe; i++) { |
| 855 | free_func_state(state->frame[i]); |
| 856 | state->frame[i] = NULL; |
| 857 | } |
| 858 | clear_jmp_history(state); |
| 859 | if (free_self) |
| 860 | kfree(state); |
| 861 | } |
| 862 | |
| 863 | /* copy verifier state from src to dst growing dst stack space |
| 864 | * when necessary to accommodate larger src stack |
| 865 | */ |
| 866 | static int copy_func_state(struct bpf_func_state *dst, |
| 867 | const struct bpf_func_state *src) |
| 868 | { |
| 869 | int err; |
| 870 | |
| 871 | err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs, |
| 872 | false); |
| 873 | if (err) |
| 874 | return err; |
| 875 | memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); |
| 876 | err = copy_reference_state(dst, src); |
| 877 | if (err) |
| 878 | return err; |
| 879 | return copy_stack_state(dst, src); |
| 880 | } |
| 881 | |
| 882 | static int copy_verifier_state(struct bpf_verifier_state *dst_state, |
| 883 | const struct bpf_verifier_state *src) |
| 884 | { |
| 885 | struct bpf_func_state *dst; |
| 886 | u32 jmp_sz = sizeof(struct bpf_idx_pair) * src->jmp_history_cnt; |
| 887 | int i, err; |
| 888 | |
| 889 | if (dst_state->jmp_history_cnt < src->jmp_history_cnt) { |
| 890 | kfree(dst_state->jmp_history); |
| 891 | dst_state->jmp_history = kmalloc(jmp_sz, GFP_USER); |
| 892 | if (!dst_state->jmp_history) |
| 893 | return -ENOMEM; |
| 894 | } |
| 895 | memcpy(dst_state->jmp_history, src->jmp_history, jmp_sz); |
| 896 | dst_state->jmp_history_cnt = src->jmp_history_cnt; |
| 897 | |
| 898 | /* if dst has more stack frames then src frame, free them */ |
| 899 | for (i = src->curframe + 1; i <= dst_state->curframe; i++) { |
| 900 | free_func_state(dst_state->frame[i]); |
| 901 | dst_state->frame[i] = NULL; |
| 902 | } |
| 903 | dst_state->speculative = src->speculative; |
| 904 | dst_state->curframe = src->curframe; |
| 905 | dst_state->active_spin_lock = src->active_spin_lock; |
| 906 | dst_state->branches = src->branches; |
| 907 | dst_state->parent = src->parent; |
| 908 | dst_state->first_insn_idx = src->first_insn_idx; |
| 909 | dst_state->last_insn_idx = src->last_insn_idx; |
| 910 | for (i = 0; i <= src->curframe; i++) { |
| 911 | dst = dst_state->frame[i]; |
| 912 | if (!dst) { |
| 913 | dst = kzalloc(sizeof(*dst), GFP_KERNEL); |
| 914 | if (!dst) |
| 915 | return -ENOMEM; |
| 916 | dst_state->frame[i] = dst; |
| 917 | } |
| 918 | err = copy_func_state(dst, src->frame[i]); |
| 919 | if (err) |
| 920 | return err; |
| 921 | } |
| 922 | return 0; |
| 923 | } |
| 924 | |
| 925 | static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st) |
| 926 | { |
| 927 | while (st) { |
| 928 | u32 br = --st->branches; |
| 929 | |
| 930 | /* WARN_ON(br > 1) technically makes sense here, |
| 931 | * but see comment in push_stack(), hence: |
| 932 | */ |
| 933 | WARN_ONCE((int)br < 0, |
| 934 | "BUG update_branch_counts:branches_to_explore=%d\n", |
| 935 | br); |
| 936 | if (br) |
| 937 | break; |
| 938 | st = st->parent; |
| 939 | } |
| 940 | } |
| 941 | |
| 942 | static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, |
| 943 | int *insn_idx, bool pop_log) |
| 944 | { |
| 945 | struct bpf_verifier_state *cur = env->cur_state; |
| 946 | struct bpf_verifier_stack_elem *elem, *head = env->head; |
| 947 | int err; |
| 948 | |
| 949 | if (env->head == NULL) |
| 950 | return -ENOENT; |
| 951 | |
| 952 | if (cur) { |
| 953 | err = copy_verifier_state(cur, &head->st); |
| 954 | if (err) |
| 955 | return err; |
| 956 | } |
| 957 | if (pop_log) |
| 958 | bpf_vlog_reset(&env->log, head->log_pos); |
| 959 | if (insn_idx) |
| 960 | *insn_idx = head->insn_idx; |
| 961 | if (prev_insn_idx) |
| 962 | *prev_insn_idx = head->prev_insn_idx; |
| 963 | elem = head->next; |
| 964 | free_verifier_state(&head->st, false); |
| 965 | kfree(head); |
| 966 | env->head = elem; |
| 967 | env->stack_size--; |
| 968 | return 0; |
| 969 | } |
| 970 | |
| 971 | static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, |
| 972 | int insn_idx, int prev_insn_idx, |
| 973 | bool speculative) |
| 974 | { |
| 975 | struct bpf_verifier_state *cur = env->cur_state; |
| 976 | struct bpf_verifier_stack_elem *elem; |
| 977 | int err; |
| 978 | |
| 979 | elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); |
| 980 | if (!elem) |
| 981 | goto err; |
| 982 | |
| 983 | elem->insn_idx = insn_idx; |
| 984 | elem->prev_insn_idx = prev_insn_idx; |
| 985 | elem->next = env->head; |
| 986 | elem->log_pos = env->log.len_used; |
| 987 | env->head = elem; |
| 988 | env->stack_size++; |
| 989 | err = copy_verifier_state(&elem->st, cur); |
| 990 | if (err) |
| 991 | goto err; |
| 992 | elem->st.speculative |= speculative; |
| 993 | if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { |
| 994 | verbose(env, "The sequence of %d jumps is too complex.\n", |
| 995 | env->stack_size); |
| 996 | goto err; |
| 997 | } |
| 998 | if (elem->st.parent) { |
| 999 | ++elem->st.parent->branches; |
| 1000 | /* WARN_ON(branches > 2) technically makes sense here, |
| 1001 | * but |
| 1002 | * 1. speculative states will bump 'branches' for non-branch |
| 1003 | * instructions |
| 1004 | * 2. is_state_visited() heuristics may decide not to create |
| 1005 | * a new state for a sequence of branches and all such current |
| 1006 | * and cloned states will be pointing to a single parent state |
| 1007 | * which might have large 'branches' count. |
| 1008 | */ |
| 1009 | } |
| 1010 | return &elem->st; |
| 1011 | err: |
| 1012 | free_verifier_state(env->cur_state, true); |
| 1013 | env->cur_state = NULL; |
| 1014 | /* pop all elements and return */ |
| 1015 | while (!pop_stack(env, NULL, NULL, false)); |
| 1016 | return NULL; |
| 1017 | } |
| 1018 | |
| 1019 | #define CALLER_SAVED_REGS 6 |
| 1020 | static const int caller_saved[CALLER_SAVED_REGS] = { |
| 1021 | BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 |
| 1022 | }; |
| 1023 | |
| 1024 | static void __mark_reg_not_init(const struct bpf_verifier_env *env, |
| 1025 | struct bpf_reg_state *reg); |
| 1026 | |
| 1027 | /* This helper doesn't clear reg->id */ |
| 1028 | static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) |
| 1029 | { |
| 1030 | reg->var_off = tnum_const(imm); |
| 1031 | reg->smin_value = (s64)imm; |
| 1032 | reg->smax_value = (s64)imm; |
| 1033 | reg->umin_value = imm; |
| 1034 | reg->umax_value = imm; |
| 1035 | |
| 1036 | reg->s32_min_value = (s32)imm; |
| 1037 | reg->s32_max_value = (s32)imm; |
| 1038 | reg->u32_min_value = (u32)imm; |
| 1039 | reg->u32_max_value = (u32)imm; |
| 1040 | } |
| 1041 | |
| 1042 | /* Mark the unknown part of a register (variable offset or scalar value) as |
| 1043 | * known to have the value @imm. |
| 1044 | */ |
| 1045 | static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) |
| 1046 | { |
| 1047 | /* Clear id, off, and union(map_ptr, range) */ |
| 1048 | memset(((u8 *)reg) + sizeof(reg->type), 0, |
| 1049 | offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); |
| 1050 | ___mark_reg_known(reg, imm); |
| 1051 | } |
| 1052 | |
| 1053 | static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm) |
| 1054 | { |
| 1055 | reg->var_off = tnum_const_subreg(reg->var_off, imm); |
| 1056 | reg->s32_min_value = (s32)imm; |
| 1057 | reg->s32_max_value = (s32)imm; |
| 1058 | reg->u32_min_value = (u32)imm; |
| 1059 | reg->u32_max_value = (u32)imm; |
| 1060 | } |
| 1061 | |
| 1062 | /* Mark the 'variable offset' part of a register as zero. This should be |
| 1063 | * used only on registers holding a pointer type. |
| 1064 | */ |
| 1065 | static void __mark_reg_known_zero(struct bpf_reg_state *reg) |
| 1066 | { |
| 1067 | __mark_reg_known(reg, 0); |
| 1068 | } |
| 1069 | |
| 1070 | static void __mark_reg_const_zero(struct bpf_reg_state *reg) |
| 1071 | { |
| 1072 | __mark_reg_known(reg, 0); |
| 1073 | reg->type = SCALAR_VALUE; |
| 1074 | } |
| 1075 | |
| 1076 | static void mark_reg_known_zero(struct bpf_verifier_env *env, |
| 1077 | struct bpf_reg_state *regs, u32 regno) |
| 1078 | { |
| 1079 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
| 1080 | verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); |
| 1081 | /* Something bad happened, let's kill all regs */ |
| 1082 | for (regno = 0; regno < MAX_BPF_REG; regno++) |
| 1083 | __mark_reg_not_init(env, regs + regno); |
| 1084 | return; |
| 1085 | } |
| 1086 | __mark_reg_known_zero(regs + regno); |
| 1087 | } |
| 1088 | |
| 1089 | static void mark_ptr_not_null_reg(struct bpf_reg_state *reg) |
| 1090 | { |
| 1091 | switch (reg->type) { |
| 1092 | case PTR_TO_MAP_VALUE_OR_NULL: { |
| 1093 | const struct bpf_map *map = reg->map_ptr; |
| 1094 | |
| 1095 | if (map->inner_map_meta) { |
| 1096 | reg->type = CONST_PTR_TO_MAP; |
| 1097 | reg->map_ptr = map->inner_map_meta; |
| 1098 | } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { |
| 1099 | reg->type = PTR_TO_XDP_SOCK; |
| 1100 | } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || |
| 1101 | map->map_type == BPF_MAP_TYPE_SOCKHASH) { |
| 1102 | reg->type = PTR_TO_SOCKET; |
| 1103 | } else { |
| 1104 | reg->type = PTR_TO_MAP_VALUE; |
| 1105 | } |
| 1106 | break; |
| 1107 | } |
| 1108 | case PTR_TO_SOCKET_OR_NULL: |
| 1109 | reg->type = PTR_TO_SOCKET; |
| 1110 | break; |
| 1111 | case PTR_TO_SOCK_COMMON_OR_NULL: |
| 1112 | reg->type = PTR_TO_SOCK_COMMON; |
| 1113 | break; |
| 1114 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 1115 | reg->type = PTR_TO_TCP_SOCK; |
| 1116 | break; |
| 1117 | case PTR_TO_BTF_ID_OR_NULL: |
| 1118 | reg->type = PTR_TO_BTF_ID; |
| 1119 | break; |
| 1120 | case PTR_TO_MEM_OR_NULL: |
| 1121 | reg->type = PTR_TO_MEM; |
| 1122 | break; |
| 1123 | case PTR_TO_RDONLY_BUF_OR_NULL: |
| 1124 | reg->type = PTR_TO_RDONLY_BUF; |
| 1125 | break; |
| 1126 | case PTR_TO_RDWR_BUF_OR_NULL: |
| 1127 | reg->type = PTR_TO_RDWR_BUF; |
| 1128 | break; |
| 1129 | default: |
| 1130 | WARN_ONCE(1, "unknown nullable register type"); |
| 1131 | } |
| 1132 | } |
| 1133 | |
| 1134 | static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) |
| 1135 | { |
| 1136 | return type_is_pkt_pointer(reg->type); |
| 1137 | } |
| 1138 | |
| 1139 | static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) |
| 1140 | { |
| 1141 | return reg_is_pkt_pointer(reg) || |
| 1142 | reg->type == PTR_TO_PACKET_END; |
| 1143 | } |
| 1144 | |
| 1145 | /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ |
| 1146 | static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, |
| 1147 | enum bpf_reg_type which) |
| 1148 | { |
| 1149 | /* The register can already have a range from prior markings. |
| 1150 | * This is fine as long as it hasn't been advanced from its |
| 1151 | * origin. |
| 1152 | */ |
| 1153 | return reg->type == which && |
| 1154 | reg->id == 0 && |
| 1155 | reg->off == 0 && |
| 1156 | tnum_equals_const(reg->var_off, 0); |
| 1157 | } |
| 1158 | |
| 1159 | /* Reset the min/max bounds of a register */ |
| 1160 | static void __mark_reg_unbounded(struct bpf_reg_state *reg) |
| 1161 | { |
| 1162 | reg->smin_value = S64_MIN; |
| 1163 | reg->smax_value = S64_MAX; |
| 1164 | reg->umin_value = 0; |
| 1165 | reg->umax_value = U64_MAX; |
| 1166 | |
| 1167 | reg->s32_min_value = S32_MIN; |
| 1168 | reg->s32_max_value = S32_MAX; |
| 1169 | reg->u32_min_value = 0; |
| 1170 | reg->u32_max_value = U32_MAX; |
| 1171 | } |
| 1172 | |
| 1173 | static void __mark_reg64_unbounded(struct bpf_reg_state *reg) |
| 1174 | { |
| 1175 | reg->smin_value = S64_MIN; |
| 1176 | reg->smax_value = S64_MAX; |
| 1177 | reg->umin_value = 0; |
| 1178 | reg->umax_value = U64_MAX; |
| 1179 | } |
| 1180 | |
| 1181 | static void __mark_reg32_unbounded(struct bpf_reg_state *reg) |
| 1182 | { |
| 1183 | reg->s32_min_value = S32_MIN; |
| 1184 | reg->s32_max_value = S32_MAX; |
| 1185 | reg->u32_min_value = 0; |
| 1186 | reg->u32_max_value = U32_MAX; |
| 1187 | } |
| 1188 | |
| 1189 | static void __update_reg32_bounds(struct bpf_reg_state *reg) |
| 1190 | { |
| 1191 | struct tnum var32_off = tnum_subreg(reg->var_off); |
| 1192 | |
| 1193 | /* min signed is max(sign bit) | min(other bits) */ |
| 1194 | reg->s32_min_value = max_t(s32, reg->s32_min_value, |
| 1195 | var32_off.value | (var32_off.mask & S32_MIN)); |
| 1196 | /* max signed is min(sign bit) | max(other bits) */ |
| 1197 | reg->s32_max_value = min_t(s32, reg->s32_max_value, |
| 1198 | var32_off.value | (var32_off.mask & S32_MAX)); |
| 1199 | reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value); |
| 1200 | reg->u32_max_value = min(reg->u32_max_value, |
| 1201 | (u32)(var32_off.value | var32_off.mask)); |
| 1202 | } |
| 1203 | |
| 1204 | static void __update_reg64_bounds(struct bpf_reg_state *reg) |
| 1205 | { |
| 1206 | /* min signed is max(sign bit) | min(other bits) */ |
| 1207 | reg->smin_value = max_t(s64, reg->smin_value, |
| 1208 | reg->var_off.value | (reg->var_off.mask & S64_MIN)); |
| 1209 | /* max signed is min(sign bit) | max(other bits) */ |
| 1210 | reg->smax_value = min_t(s64, reg->smax_value, |
| 1211 | reg->var_off.value | (reg->var_off.mask & S64_MAX)); |
| 1212 | reg->umin_value = max(reg->umin_value, reg->var_off.value); |
| 1213 | reg->umax_value = min(reg->umax_value, |
| 1214 | reg->var_off.value | reg->var_off.mask); |
| 1215 | } |
| 1216 | |
| 1217 | static void __update_reg_bounds(struct bpf_reg_state *reg) |
| 1218 | { |
| 1219 | __update_reg32_bounds(reg); |
| 1220 | __update_reg64_bounds(reg); |
| 1221 | } |
| 1222 | |
| 1223 | /* Uses signed min/max values to inform unsigned, and vice-versa */ |
| 1224 | static void __reg32_deduce_bounds(struct bpf_reg_state *reg) |
| 1225 | { |
| 1226 | /* Learn sign from signed bounds. |
| 1227 | * If we cannot cross the sign boundary, then signed and unsigned bounds |
| 1228 | * are the same, so combine. This works even in the negative case, e.g. |
| 1229 | * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. |
| 1230 | */ |
| 1231 | if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) { |
| 1232 | reg->s32_min_value = reg->u32_min_value = |
| 1233 | max_t(u32, reg->s32_min_value, reg->u32_min_value); |
| 1234 | reg->s32_max_value = reg->u32_max_value = |
| 1235 | min_t(u32, reg->s32_max_value, reg->u32_max_value); |
| 1236 | return; |
| 1237 | } |
| 1238 | /* Learn sign from unsigned bounds. Signed bounds cross the sign |
| 1239 | * boundary, so we must be careful. |
| 1240 | */ |
| 1241 | if ((s32)reg->u32_max_value >= 0) { |
| 1242 | /* Positive. We can't learn anything from the smin, but smax |
| 1243 | * is positive, hence safe. |
| 1244 | */ |
| 1245 | reg->s32_min_value = reg->u32_min_value; |
| 1246 | reg->s32_max_value = reg->u32_max_value = |
| 1247 | min_t(u32, reg->s32_max_value, reg->u32_max_value); |
| 1248 | } else if ((s32)reg->u32_min_value < 0) { |
| 1249 | /* Negative. We can't learn anything from the smax, but smin |
| 1250 | * is negative, hence safe. |
| 1251 | */ |
| 1252 | reg->s32_min_value = reg->u32_min_value = |
| 1253 | max_t(u32, reg->s32_min_value, reg->u32_min_value); |
| 1254 | reg->s32_max_value = reg->u32_max_value; |
| 1255 | } |
| 1256 | } |
| 1257 | |
| 1258 | static void __reg64_deduce_bounds(struct bpf_reg_state *reg) |
| 1259 | { |
| 1260 | /* Learn sign from signed bounds. |
| 1261 | * If we cannot cross the sign boundary, then signed and unsigned bounds |
| 1262 | * are the same, so combine. This works even in the negative case, e.g. |
| 1263 | * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. |
| 1264 | */ |
| 1265 | if (reg->smin_value >= 0 || reg->smax_value < 0) { |
| 1266 | reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, |
| 1267 | reg->umin_value); |
| 1268 | reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, |
| 1269 | reg->umax_value); |
| 1270 | return; |
| 1271 | } |
| 1272 | /* Learn sign from unsigned bounds. Signed bounds cross the sign |
| 1273 | * boundary, so we must be careful. |
| 1274 | */ |
| 1275 | if ((s64)reg->umax_value >= 0) { |
| 1276 | /* Positive. We can't learn anything from the smin, but smax |
| 1277 | * is positive, hence safe. |
| 1278 | */ |
| 1279 | reg->smin_value = reg->umin_value; |
| 1280 | reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, |
| 1281 | reg->umax_value); |
| 1282 | } else if ((s64)reg->umin_value < 0) { |
| 1283 | /* Negative. We can't learn anything from the smax, but smin |
| 1284 | * is negative, hence safe. |
| 1285 | */ |
| 1286 | reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, |
| 1287 | reg->umin_value); |
| 1288 | reg->smax_value = reg->umax_value; |
| 1289 | } |
| 1290 | } |
| 1291 | |
| 1292 | static void __reg_deduce_bounds(struct bpf_reg_state *reg) |
| 1293 | { |
| 1294 | __reg32_deduce_bounds(reg); |
| 1295 | __reg64_deduce_bounds(reg); |
| 1296 | } |
| 1297 | |
| 1298 | /* Attempts to improve var_off based on unsigned min/max information */ |
| 1299 | static void __reg_bound_offset(struct bpf_reg_state *reg) |
| 1300 | { |
| 1301 | struct tnum var64_off = tnum_intersect(reg->var_off, |
| 1302 | tnum_range(reg->umin_value, |
| 1303 | reg->umax_value)); |
| 1304 | struct tnum var32_off = tnum_intersect(tnum_subreg(reg->var_off), |
| 1305 | tnum_range(reg->u32_min_value, |
| 1306 | reg->u32_max_value)); |
| 1307 | |
| 1308 | reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off); |
| 1309 | } |
| 1310 | |
| 1311 | static void __reg_assign_32_into_64(struct bpf_reg_state *reg) |
| 1312 | { |
| 1313 | reg->umin_value = reg->u32_min_value; |
| 1314 | reg->umax_value = reg->u32_max_value; |
| 1315 | /* Attempt to pull 32-bit signed bounds into 64-bit bounds |
| 1316 | * but must be positive otherwise set to worse case bounds |
| 1317 | * and refine later from tnum. |
| 1318 | */ |
| 1319 | if (reg->s32_min_value >= 0 && reg->s32_max_value >= 0) |
| 1320 | reg->smax_value = reg->s32_max_value; |
| 1321 | else |
| 1322 | reg->smax_value = U32_MAX; |
| 1323 | if (reg->s32_min_value >= 0) |
| 1324 | reg->smin_value = reg->s32_min_value; |
| 1325 | else |
| 1326 | reg->smin_value = 0; |
| 1327 | } |
| 1328 | |
| 1329 | static void __reg_combine_32_into_64(struct bpf_reg_state *reg) |
| 1330 | { |
| 1331 | /* special case when 64-bit register has upper 32-bit register |
| 1332 | * zeroed. Typically happens after zext or <<32, >>32 sequence |
| 1333 | * allowing us to use 32-bit bounds directly, |
| 1334 | */ |
| 1335 | if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) { |
| 1336 | __reg_assign_32_into_64(reg); |
| 1337 | } else { |
| 1338 | /* Otherwise the best we can do is push lower 32bit known and |
| 1339 | * unknown bits into register (var_off set from jmp logic) |
| 1340 | * then learn as much as possible from the 64-bit tnum |
| 1341 | * known and unknown bits. The previous smin/smax bounds are |
| 1342 | * invalid here because of jmp32 compare so mark them unknown |
| 1343 | * so they do not impact tnum bounds calculation. |
| 1344 | */ |
| 1345 | __mark_reg64_unbounded(reg); |
| 1346 | __update_reg_bounds(reg); |
| 1347 | } |
| 1348 | |
| 1349 | /* Intersecting with the old var_off might have improved our bounds |
| 1350 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 1351 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 1352 | */ |
| 1353 | __reg_deduce_bounds(reg); |
| 1354 | __reg_bound_offset(reg); |
| 1355 | __update_reg_bounds(reg); |
| 1356 | } |
| 1357 | |
| 1358 | static bool __reg64_bound_s32(s64 a) |
| 1359 | { |
| 1360 | return a > S32_MIN && a < S32_MAX; |
| 1361 | } |
| 1362 | |
| 1363 | static bool __reg64_bound_u32(u64 a) |
| 1364 | { |
| 1365 | if (a > U32_MIN && a < U32_MAX) |
| 1366 | return true; |
| 1367 | return false; |
| 1368 | } |
| 1369 | |
| 1370 | static void __reg_combine_64_into_32(struct bpf_reg_state *reg) |
| 1371 | { |
| 1372 | __mark_reg32_unbounded(reg); |
| 1373 | |
| 1374 | if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) { |
| 1375 | reg->s32_min_value = (s32)reg->smin_value; |
| 1376 | reg->s32_max_value = (s32)reg->smax_value; |
| 1377 | } |
| 1378 | if (__reg64_bound_u32(reg->umin_value)) |
| 1379 | reg->u32_min_value = (u32)reg->umin_value; |
| 1380 | if (__reg64_bound_u32(reg->umax_value)) |
| 1381 | reg->u32_max_value = (u32)reg->umax_value; |
| 1382 | |
| 1383 | /* Intersecting with the old var_off might have improved our bounds |
| 1384 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 1385 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 1386 | */ |
| 1387 | __reg_deduce_bounds(reg); |
| 1388 | __reg_bound_offset(reg); |
| 1389 | __update_reg_bounds(reg); |
| 1390 | } |
| 1391 | |
| 1392 | /* Mark a register as having a completely unknown (scalar) value. */ |
| 1393 | static void __mark_reg_unknown(const struct bpf_verifier_env *env, |
| 1394 | struct bpf_reg_state *reg) |
| 1395 | { |
| 1396 | /* |
| 1397 | * Clear type, id, off, and union(map_ptr, range) and |
| 1398 | * padding between 'type' and union |
| 1399 | */ |
| 1400 | memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); |
| 1401 | reg->type = SCALAR_VALUE; |
| 1402 | reg->var_off = tnum_unknown; |
| 1403 | reg->frameno = 0; |
| 1404 | reg->precise = env->subprog_cnt > 1 || !env->bpf_capable; |
| 1405 | __mark_reg_unbounded(reg); |
| 1406 | } |
| 1407 | |
| 1408 | static void mark_reg_unknown(struct bpf_verifier_env *env, |
| 1409 | struct bpf_reg_state *regs, u32 regno) |
| 1410 | { |
| 1411 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
| 1412 | verbose(env, "mark_reg_unknown(regs, %u)\n", regno); |
| 1413 | /* Something bad happened, let's kill all regs except FP */ |
| 1414 | for (regno = 0; regno < BPF_REG_FP; regno++) |
| 1415 | __mark_reg_not_init(env, regs + regno); |
| 1416 | return; |
| 1417 | } |
| 1418 | __mark_reg_unknown(env, regs + regno); |
| 1419 | } |
| 1420 | |
| 1421 | static void __mark_reg_not_init(const struct bpf_verifier_env *env, |
| 1422 | struct bpf_reg_state *reg) |
| 1423 | { |
| 1424 | __mark_reg_unknown(env, reg); |
| 1425 | reg->type = NOT_INIT; |
| 1426 | } |
| 1427 | |
| 1428 | static void mark_reg_not_init(struct bpf_verifier_env *env, |
| 1429 | struct bpf_reg_state *regs, u32 regno) |
| 1430 | { |
| 1431 | if (WARN_ON(regno >= MAX_BPF_REG)) { |
| 1432 | verbose(env, "mark_reg_not_init(regs, %u)\n", regno); |
| 1433 | /* Something bad happened, let's kill all regs except FP */ |
| 1434 | for (regno = 0; regno < BPF_REG_FP; regno++) |
| 1435 | __mark_reg_not_init(env, regs + regno); |
| 1436 | return; |
| 1437 | } |
| 1438 | __mark_reg_not_init(env, regs + regno); |
| 1439 | } |
| 1440 | |
| 1441 | static void mark_btf_ld_reg(struct bpf_verifier_env *env, |
| 1442 | struct bpf_reg_state *regs, u32 regno, |
| 1443 | enum bpf_reg_type reg_type, |
| 1444 | struct btf *btf, u32 btf_id) |
| 1445 | { |
| 1446 | if (reg_type == SCALAR_VALUE) { |
| 1447 | mark_reg_unknown(env, regs, regno); |
| 1448 | return; |
| 1449 | } |
| 1450 | mark_reg_known_zero(env, regs, regno); |
| 1451 | regs[regno].type = PTR_TO_BTF_ID; |
| 1452 | regs[regno].btf = btf; |
| 1453 | regs[regno].btf_id = btf_id; |
| 1454 | } |
| 1455 | |
| 1456 | #define DEF_NOT_SUBREG (0) |
| 1457 | static void init_reg_state(struct bpf_verifier_env *env, |
| 1458 | struct bpf_func_state *state) |
| 1459 | { |
| 1460 | struct bpf_reg_state *regs = state->regs; |
| 1461 | int i; |
| 1462 | |
| 1463 | for (i = 0; i < MAX_BPF_REG; i++) { |
| 1464 | mark_reg_not_init(env, regs, i); |
| 1465 | regs[i].live = REG_LIVE_NONE; |
| 1466 | regs[i].parent = NULL; |
| 1467 | regs[i].subreg_def = DEF_NOT_SUBREG; |
| 1468 | } |
| 1469 | |
| 1470 | /* frame pointer */ |
| 1471 | regs[BPF_REG_FP].type = PTR_TO_STACK; |
| 1472 | mark_reg_known_zero(env, regs, BPF_REG_FP); |
| 1473 | regs[BPF_REG_FP].frameno = state->frameno; |
| 1474 | } |
| 1475 | |
| 1476 | #define BPF_MAIN_FUNC (-1) |
| 1477 | static void init_func_state(struct bpf_verifier_env *env, |
| 1478 | struct bpf_func_state *state, |
| 1479 | int callsite, int frameno, int subprogno) |
| 1480 | { |
| 1481 | state->callsite = callsite; |
| 1482 | state->frameno = frameno; |
| 1483 | state->subprogno = subprogno; |
| 1484 | init_reg_state(env, state); |
| 1485 | } |
| 1486 | |
| 1487 | enum reg_arg_type { |
| 1488 | SRC_OP, /* register is used as source operand */ |
| 1489 | DST_OP, /* register is used as destination operand */ |
| 1490 | DST_OP_NO_MARK /* same as above, check only, don't mark */ |
| 1491 | }; |
| 1492 | |
| 1493 | static int cmp_subprogs(const void *a, const void *b) |
| 1494 | { |
| 1495 | return ((struct bpf_subprog_info *)a)->start - |
| 1496 | ((struct bpf_subprog_info *)b)->start; |
| 1497 | } |
| 1498 | |
| 1499 | static int find_subprog(struct bpf_verifier_env *env, int off) |
| 1500 | { |
| 1501 | struct bpf_subprog_info *p; |
| 1502 | |
| 1503 | p = bsearch(&off, env->subprog_info, env->subprog_cnt, |
| 1504 | sizeof(env->subprog_info[0]), cmp_subprogs); |
| 1505 | if (!p) |
| 1506 | return -ENOENT; |
| 1507 | return p - env->subprog_info; |
| 1508 | |
| 1509 | } |
| 1510 | |
| 1511 | static int add_subprog(struct bpf_verifier_env *env, int off) |
| 1512 | { |
| 1513 | int insn_cnt = env->prog->len; |
| 1514 | int ret; |
| 1515 | |
| 1516 | if (off >= insn_cnt || off < 0) { |
| 1517 | verbose(env, "call to invalid destination\n"); |
| 1518 | return -EINVAL; |
| 1519 | } |
| 1520 | ret = find_subprog(env, off); |
| 1521 | if (ret >= 0) |
| 1522 | return 0; |
| 1523 | if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { |
| 1524 | verbose(env, "too many subprograms\n"); |
| 1525 | return -E2BIG; |
| 1526 | } |
| 1527 | env->subprog_info[env->subprog_cnt++].start = off; |
| 1528 | sort(env->subprog_info, env->subprog_cnt, |
| 1529 | sizeof(env->subprog_info[0]), cmp_subprogs, NULL); |
| 1530 | return 0; |
| 1531 | } |
| 1532 | |
| 1533 | static int check_subprogs(struct bpf_verifier_env *env) |
| 1534 | { |
| 1535 | int i, ret, subprog_start, subprog_end, off, cur_subprog = 0; |
| 1536 | struct bpf_subprog_info *subprog = env->subprog_info; |
| 1537 | struct bpf_insn *insn = env->prog->insnsi; |
| 1538 | int insn_cnt = env->prog->len; |
| 1539 | |
| 1540 | /* Add entry function. */ |
| 1541 | ret = add_subprog(env, 0); |
| 1542 | if (ret < 0) |
| 1543 | return ret; |
| 1544 | |
| 1545 | /* determine subprog starts. The end is one before the next starts */ |
| 1546 | for (i = 0; i < insn_cnt; i++) { |
| 1547 | if (!bpf_pseudo_call(insn + i)) |
| 1548 | continue; |
| 1549 | if (!env->bpf_capable) { |
| 1550 | verbose(env, |
| 1551 | "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); |
| 1552 | return -EPERM; |
| 1553 | } |
| 1554 | ret = add_subprog(env, i + insn[i].imm + 1); |
| 1555 | if (ret < 0) |
| 1556 | return ret; |
| 1557 | } |
| 1558 | |
| 1559 | /* Add a fake 'exit' subprog which could simplify subprog iteration |
| 1560 | * logic. 'subprog_cnt' should not be increased. |
| 1561 | */ |
| 1562 | subprog[env->subprog_cnt].start = insn_cnt; |
| 1563 | |
| 1564 | if (env->log.level & BPF_LOG_LEVEL2) |
| 1565 | for (i = 0; i < env->subprog_cnt; i++) |
| 1566 | verbose(env, "func#%d @%d\n", i, subprog[i].start); |
| 1567 | |
| 1568 | /* now check that all jumps are within the same subprog */ |
| 1569 | subprog_start = subprog[cur_subprog].start; |
| 1570 | subprog_end = subprog[cur_subprog + 1].start; |
| 1571 | for (i = 0; i < insn_cnt; i++) { |
| 1572 | u8 code = insn[i].code; |
| 1573 | |
| 1574 | if (code == (BPF_JMP | BPF_CALL) && |
| 1575 | insn[i].imm == BPF_FUNC_tail_call && |
| 1576 | insn[i].src_reg != BPF_PSEUDO_CALL) |
| 1577 | subprog[cur_subprog].has_tail_call = true; |
| 1578 | if (BPF_CLASS(code) == BPF_LD && |
| 1579 | (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND)) |
| 1580 | subprog[cur_subprog].has_ld_abs = true; |
| 1581 | if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) |
| 1582 | goto next; |
| 1583 | if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) |
| 1584 | goto next; |
| 1585 | off = i + insn[i].off + 1; |
| 1586 | if (off < subprog_start || off >= subprog_end) { |
| 1587 | verbose(env, "jump out of range from insn %d to %d\n", i, off); |
| 1588 | return -EINVAL; |
| 1589 | } |
| 1590 | next: |
| 1591 | if (i == subprog_end - 1) { |
| 1592 | /* to avoid fall-through from one subprog into another |
| 1593 | * the last insn of the subprog should be either exit |
| 1594 | * or unconditional jump back |
| 1595 | */ |
| 1596 | if (code != (BPF_JMP | BPF_EXIT) && |
| 1597 | code != (BPF_JMP | BPF_JA)) { |
| 1598 | verbose(env, "last insn is not an exit or jmp\n"); |
| 1599 | return -EINVAL; |
| 1600 | } |
| 1601 | subprog_start = subprog_end; |
| 1602 | cur_subprog++; |
| 1603 | if (cur_subprog < env->subprog_cnt) |
| 1604 | subprog_end = subprog[cur_subprog + 1].start; |
| 1605 | } |
| 1606 | } |
| 1607 | return 0; |
| 1608 | } |
| 1609 | |
| 1610 | /* Parentage chain of this register (or stack slot) should take care of all |
| 1611 | * issues like callee-saved registers, stack slot allocation time, etc. |
| 1612 | */ |
| 1613 | static int mark_reg_read(struct bpf_verifier_env *env, |
| 1614 | const struct bpf_reg_state *state, |
| 1615 | struct bpf_reg_state *parent, u8 flag) |
| 1616 | { |
| 1617 | bool writes = parent == state->parent; /* Observe write marks */ |
| 1618 | int cnt = 0; |
| 1619 | |
| 1620 | while (parent) { |
| 1621 | /* if read wasn't screened by an earlier write ... */ |
| 1622 | if (writes && state->live & REG_LIVE_WRITTEN) |
| 1623 | break; |
| 1624 | if (parent->live & REG_LIVE_DONE) { |
| 1625 | verbose(env, "verifier BUG type %s var_off %lld off %d\n", |
| 1626 | reg_type_str[parent->type], |
| 1627 | parent->var_off.value, parent->off); |
| 1628 | return -EFAULT; |
| 1629 | } |
| 1630 | /* The first condition is more likely to be true than the |
| 1631 | * second, checked it first. |
| 1632 | */ |
| 1633 | if ((parent->live & REG_LIVE_READ) == flag || |
| 1634 | parent->live & REG_LIVE_READ64) |
| 1635 | /* The parentage chain never changes and |
| 1636 | * this parent was already marked as LIVE_READ. |
| 1637 | * There is no need to keep walking the chain again and |
| 1638 | * keep re-marking all parents as LIVE_READ. |
| 1639 | * This case happens when the same register is read |
| 1640 | * multiple times without writes into it in-between. |
| 1641 | * Also, if parent has the stronger REG_LIVE_READ64 set, |
| 1642 | * then no need to set the weak REG_LIVE_READ32. |
| 1643 | */ |
| 1644 | break; |
| 1645 | /* ... then we depend on parent's value */ |
| 1646 | parent->live |= flag; |
| 1647 | /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */ |
| 1648 | if (flag == REG_LIVE_READ64) |
| 1649 | parent->live &= ~REG_LIVE_READ32; |
| 1650 | state = parent; |
| 1651 | parent = state->parent; |
| 1652 | writes = true; |
| 1653 | cnt++; |
| 1654 | } |
| 1655 | |
| 1656 | if (env->longest_mark_read_walk < cnt) |
| 1657 | env->longest_mark_read_walk = cnt; |
| 1658 | return 0; |
| 1659 | } |
| 1660 | |
| 1661 | /* This function is supposed to be used by the following 32-bit optimization |
| 1662 | * code only. It returns TRUE if the source or destination register operates |
| 1663 | * on 64-bit, otherwise return FALSE. |
| 1664 | */ |
| 1665 | static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 1666 | u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t) |
| 1667 | { |
| 1668 | u8 code, class, op; |
| 1669 | |
| 1670 | code = insn->code; |
| 1671 | class = BPF_CLASS(code); |
| 1672 | op = BPF_OP(code); |
| 1673 | if (class == BPF_JMP) { |
| 1674 | /* BPF_EXIT for "main" will reach here. Return TRUE |
| 1675 | * conservatively. |
| 1676 | */ |
| 1677 | if (op == BPF_EXIT) |
| 1678 | return true; |
| 1679 | if (op == BPF_CALL) { |
| 1680 | /* BPF to BPF call will reach here because of marking |
| 1681 | * caller saved clobber with DST_OP_NO_MARK for which we |
| 1682 | * don't care the register def because they are anyway |
| 1683 | * marked as NOT_INIT already. |
| 1684 | */ |
| 1685 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 1686 | return false; |
| 1687 | /* Helper call will reach here because of arg type |
| 1688 | * check, conservatively return TRUE. |
| 1689 | */ |
| 1690 | if (t == SRC_OP) |
| 1691 | return true; |
| 1692 | |
| 1693 | return false; |
| 1694 | } |
| 1695 | } |
| 1696 | |
| 1697 | if (class == BPF_ALU64 || class == BPF_JMP || |
| 1698 | /* BPF_END always use BPF_ALU class. */ |
| 1699 | (class == BPF_ALU && op == BPF_END && insn->imm == 64)) |
| 1700 | return true; |
| 1701 | |
| 1702 | if (class == BPF_ALU || class == BPF_JMP32) |
| 1703 | return false; |
| 1704 | |
| 1705 | if (class == BPF_LDX) { |
| 1706 | if (t != SRC_OP) |
| 1707 | return BPF_SIZE(code) == BPF_DW; |
| 1708 | /* LDX source must be ptr. */ |
| 1709 | return true; |
| 1710 | } |
| 1711 | |
| 1712 | if (class == BPF_STX) { |
| 1713 | /* BPF_STX (including atomic variants) has multiple source |
| 1714 | * operands, one of which is a ptr. Check whether the caller is |
| 1715 | * asking about it. |
| 1716 | */ |
| 1717 | if (t == SRC_OP && reg->type != SCALAR_VALUE) |
| 1718 | return true; |
| 1719 | return BPF_SIZE(code) == BPF_DW; |
| 1720 | } |
| 1721 | |
| 1722 | if (class == BPF_LD) { |
| 1723 | u8 mode = BPF_MODE(code); |
| 1724 | |
| 1725 | /* LD_IMM64 */ |
| 1726 | if (mode == BPF_IMM) |
| 1727 | return true; |
| 1728 | |
| 1729 | /* Both LD_IND and LD_ABS return 32-bit data. */ |
| 1730 | if (t != SRC_OP) |
| 1731 | return false; |
| 1732 | |
| 1733 | /* Implicit ctx ptr. */ |
| 1734 | if (regno == BPF_REG_6) |
| 1735 | return true; |
| 1736 | |
| 1737 | /* Explicit source could be any width. */ |
| 1738 | return true; |
| 1739 | } |
| 1740 | |
| 1741 | if (class == BPF_ST) |
| 1742 | /* The only source register for BPF_ST is a ptr. */ |
| 1743 | return true; |
| 1744 | |
| 1745 | /* Conservatively return true at default. */ |
| 1746 | return true; |
| 1747 | } |
| 1748 | |
| 1749 | /* Return the regno defined by the insn, or -1. */ |
| 1750 | static int insn_def_regno(const struct bpf_insn *insn) |
| 1751 | { |
| 1752 | switch (BPF_CLASS(insn->code)) { |
| 1753 | case BPF_JMP: |
| 1754 | case BPF_JMP32: |
| 1755 | case BPF_ST: |
| 1756 | return -1; |
| 1757 | case BPF_STX: |
| 1758 | if (BPF_MODE(insn->code) == BPF_ATOMIC && |
| 1759 | (insn->imm & BPF_FETCH)) { |
| 1760 | if (insn->imm == BPF_CMPXCHG) |
| 1761 | return BPF_REG_0; |
| 1762 | else |
| 1763 | return insn->src_reg; |
| 1764 | } else { |
| 1765 | return -1; |
| 1766 | } |
| 1767 | default: |
| 1768 | return insn->dst_reg; |
| 1769 | } |
| 1770 | } |
| 1771 | |
| 1772 | /* Return TRUE if INSN has defined any 32-bit value explicitly. */ |
| 1773 | static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn) |
| 1774 | { |
| 1775 | int dst_reg = insn_def_regno(insn); |
| 1776 | |
| 1777 | if (dst_reg == -1) |
| 1778 | return false; |
| 1779 | |
| 1780 | return !is_reg64(env, insn, dst_reg, NULL, DST_OP); |
| 1781 | } |
| 1782 | |
| 1783 | static void mark_insn_zext(struct bpf_verifier_env *env, |
| 1784 | struct bpf_reg_state *reg) |
| 1785 | { |
| 1786 | s32 def_idx = reg->subreg_def; |
| 1787 | |
| 1788 | if (def_idx == DEF_NOT_SUBREG) |
| 1789 | return; |
| 1790 | |
| 1791 | env->insn_aux_data[def_idx - 1].zext_dst = true; |
| 1792 | /* The dst will be zero extended, so won't be sub-register anymore. */ |
| 1793 | reg->subreg_def = DEF_NOT_SUBREG; |
| 1794 | } |
| 1795 | |
| 1796 | static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, |
| 1797 | enum reg_arg_type t) |
| 1798 | { |
| 1799 | struct bpf_verifier_state *vstate = env->cur_state; |
| 1800 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 1801 | struct bpf_insn *insn = env->prog->insnsi + env->insn_idx; |
| 1802 | struct bpf_reg_state *reg, *regs = state->regs; |
| 1803 | bool rw64; |
| 1804 | |
| 1805 | if (regno >= MAX_BPF_REG) { |
| 1806 | verbose(env, "R%d is invalid\n", regno); |
| 1807 | return -EINVAL; |
| 1808 | } |
| 1809 | |
| 1810 | reg = ®s[regno]; |
| 1811 | rw64 = is_reg64(env, insn, regno, reg, t); |
| 1812 | if (t == SRC_OP) { |
| 1813 | /* check whether register used as source operand can be read */ |
| 1814 | if (reg->type == NOT_INIT) { |
| 1815 | verbose(env, "R%d !read_ok\n", regno); |
| 1816 | return -EACCES; |
| 1817 | } |
| 1818 | /* We don't need to worry about FP liveness because it's read-only */ |
| 1819 | if (regno == BPF_REG_FP) |
| 1820 | return 0; |
| 1821 | |
| 1822 | if (rw64) |
| 1823 | mark_insn_zext(env, reg); |
| 1824 | |
| 1825 | return mark_reg_read(env, reg, reg->parent, |
| 1826 | rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32); |
| 1827 | } else { |
| 1828 | /* check whether register used as dest operand can be written to */ |
| 1829 | if (regno == BPF_REG_FP) { |
| 1830 | verbose(env, "frame pointer is read only\n"); |
| 1831 | return -EACCES; |
| 1832 | } |
| 1833 | reg->live |= REG_LIVE_WRITTEN; |
| 1834 | reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1; |
| 1835 | if (t == DST_OP) |
| 1836 | mark_reg_unknown(env, regs, regno); |
| 1837 | } |
| 1838 | return 0; |
| 1839 | } |
| 1840 | |
| 1841 | /* for any branch, call, exit record the history of jmps in the given state */ |
| 1842 | static int push_jmp_history(struct bpf_verifier_env *env, |
| 1843 | struct bpf_verifier_state *cur) |
| 1844 | { |
| 1845 | u32 cnt = cur->jmp_history_cnt; |
| 1846 | struct bpf_idx_pair *p; |
| 1847 | |
| 1848 | cnt++; |
| 1849 | p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER); |
| 1850 | if (!p) |
| 1851 | return -ENOMEM; |
| 1852 | p[cnt - 1].idx = env->insn_idx; |
| 1853 | p[cnt - 1].prev_idx = env->prev_insn_idx; |
| 1854 | cur->jmp_history = p; |
| 1855 | cur->jmp_history_cnt = cnt; |
| 1856 | return 0; |
| 1857 | } |
| 1858 | |
| 1859 | /* Backtrack one insn at a time. If idx is not at the top of recorded |
| 1860 | * history then previous instruction came from straight line execution. |
| 1861 | */ |
| 1862 | static int get_prev_insn_idx(struct bpf_verifier_state *st, int i, |
| 1863 | u32 *history) |
| 1864 | { |
| 1865 | u32 cnt = *history; |
| 1866 | |
| 1867 | if (cnt && st->jmp_history[cnt - 1].idx == i) { |
| 1868 | i = st->jmp_history[cnt - 1].prev_idx; |
| 1869 | (*history)--; |
| 1870 | } else { |
| 1871 | i--; |
| 1872 | } |
| 1873 | return i; |
| 1874 | } |
| 1875 | |
| 1876 | /* For given verifier state backtrack_insn() is called from the last insn to |
| 1877 | * the first insn. Its purpose is to compute a bitmask of registers and |
| 1878 | * stack slots that needs precision in the parent verifier state. |
| 1879 | */ |
| 1880 | static int backtrack_insn(struct bpf_verifier_env *env, int idx, |
| 1881 | u32 *reg_mask, u64 *stack_mask) |
| 1882 | { |
| 1883 | const struct bpf_insn_cbs cbs = { |
| 1884 | .cb_print = verbose, |
| 1885 | .private_data = env, |
| 1886 | }; |
| 1887 | struct bpf_insn *insn = env->prog->insnsi + idx; |
| 1888 | u8 class = BPF_CLASS(insn->code); |
| 1889 | u8 opcode = BPF_OP(insn->code); |
| 1890 | u8 mode = BPF_MODE(insn->code); |
| 1891 | u32 dreg = 1u << insn->dst_reg; |
| 1892 | u32 sreg = 1u << insn->src_reg; |
| 1893 | u32 spi; |
| 1894 | |
| 1895 | if (insn->code == 0) |
| 1896 | return 0; |
| 1897 | if (env->log.level & BPF_LOG_LEVEL) { |
| 1898 | verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask); |
| 1899 | verbose(env, "%d: ", idx); |
| 1900 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); |
| 1901 | } |
| 1902 | |
| 1903 | if (class == BPF_ALU || class == BPF_ALU64) { |
| 1904 | if (!(*reg_mask & dreg)) |
| 1905 | return 0; |
| 1906 | if (opcode == BPF_MOV) { |
| 1907 | if (BPF_SRC(insn->code) == BPF_X) { |
| 1908 | /* dreg = sreg |
| 1909 | * dreg needs precision after this insn |
| 1910 | * sreg needs precision before this insn |
| 1911 | */ |
| 1912 | *reg_mask &= ~dreg; |
| 1913 | *reg_mask |= sreg; |
| 1914 | } else { |
| 1915 | /* dreg = K |
| 1916 | * dreg needs precision after this insn. |
| 1917 | * Corresponding register is already marked |
| 1918 | * as precise=true in this verifier state. |
| 1919 | * No further markings in parent are necessary |
| 1920 | */ |
| 1921 | *reg_mask &= ~dreg; |
| 1922 | } |
| 1923 | } else { |
| 1924 | if (BPF_SRC(insn->code) == BPF_X) { |
| 1925 | /* dreg += sreg |
| 1926 | * both dreg and sreg need precision |
| 1927 | * before this insn |
| 1928 | */ |
| 1929 | *reg_mask |= sreg; |
| 1930 | } /* else dreg += K |
| 1931 | * dreg still needs precision before this insn |
| 1932 | */ |
| 1933 | } |
| 1934 | } else if (class == BPF_LDX) { |
| 1935 | if (!(*reg_mask & dreg)) |
| 1936 | return 0; |
| 1937 | *reg_mask &= ~dreg; |
| 1938 | |
| 1939 | /* scalars can only be spilled into stack w/o losing precision. |
| 1940 | * Load from any other memory can be zero extended. |
| 1941 | * The desire to keep that precision is already indicated |
| 1942 | * by 'precise' mark in corresponding register of this state. |
| 1943 | * No further tracking necessary. |
| 1944 | */ |
| 1945 | if (insn->src_reg != BPF_REG_FP) |
| 1946 | return 0; |
| 1947 | if (BPF_SIZE(insn->code) != BPF_DW) |
| 1948 | return 0; |
| 1949 | |
| 1950 | /* dreg = *(u64 *)[fp - off] was a fill from the stack. |
| 1951 | * that [fp - off] slot contains scalar that needs to be |
| 1952 | * tracked with precision |
| 1953 | */ |
| 1954 | spi = (-insn->off - 1) / BPF_REG_SIZE; |
| 1955 | if (spi >= 64) { |
| 1956 | verbose(env, "BUG spi %d\n", spi); |
| 1957 | WARN_ONCE(1, "verifier backtracking bug"); |
| 1958 | return -EFAULT; |
| 1959 | } |
| 1960 | *stack_mask |= 1ull << spi; |
| 1961 | } else if (class == BPF_STX || class == BPF_ST) { |
| 1962 | if (*reg_mask & dreg) |
| 1963 | /* stx & st shouldn't be using _scalar_ dst_reg |
| 1964 | * to access memory. It means backtracking |
| 1965 | * encountered a case of pointer subtraction. |
| 1966 | */ |
| 1967 | return -ENOTSUPP; |
| 1968 | /* scalars can only be spilled into stack */ |
| 1969 | if (insn->dst_reg != BPF_REG_FP) |
| 1970 | return 0; |
| 1971 | if (BPF_SIZE(insn->code) != BPF_DW) |
| 1972 | return 0; |
| 1973 | spi = (-insn->off - 1) / BPF_REG_SIZE; |
| 1974 | if (spi >= 64) { |
| 1975 | verbose(env, "BUG spi %d\n", spi); |
| 1976 | WARN_ONCE(1, "verifier backtracking bug"); |
| 1977 | return -EFAULT; |
| 1978 | } |
| 1979 | if (!(*stack_mask & (1ull << spi))) |
| 1980 | return 0; |
| 1981 | *stack_mask &= ~(1ull << spi); |
| 1982 | if (class == BPF_STX) |
| 1983 | *reg_mask |= sreg; |
| 1984 | } else if (class == BPF_JMP || class == BPF_JMP32) { |
| 1985 | if (opcode == BPF_CALL) { |
| 1986 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 1987 | return -ENOTSUPP; |
| 1988 | /* regular helper call sets R0 */ |
| 1989 | *reg_mask &= ~1; |
| 1990 | if (*reg_mask & 0x3f) { |
| 1991 | /* if backtracing was looking for registers R1-R5 |
| 1992 | * they should have been found already. |
| 1993 | */ |
| 1994 | verbose(env, "BUG regs %x\n", *reg_mask); |
| 1995 | WARN_ONCE(1, "verifier backtracking bug"); |
| 1996 | return -EFAULT; |
| 1997 | } |
| 1998 | } else if (opcode == BPF_EXIT) { |
| 1999 | return -ENOTSUPP; |
| 2000 | } |
| 2001 | } else if (class == BPF_LD) { |
| 2002 | if (!(*reg_mask & dreg)) |
| 2003 | return 0; |
| 2004 | *reg_mask &= ~dreg; |
| 2005 | /* It's ld_imm64 or ld_abs or ld_ind. |
| 2006 | * For ld_imm64 no further tracking of precision |
| 2007 | * into parent is necessary |
| 2008 | */ |
| 2009 | if (mode == BPF_IND || mode == BPF_ABS) |
| 2010 | /* to be analyzed */ |
| 2011 | return -ENOTSUPP; |
| 2012 | } |
| 2013 | return 0; |
| 2014 | } |
| 2015 | |
| 2016 | /* the scalar precision tracking algorithm: |
| 2017 | * . at the start all registers have precise=false. |
| 2018 | * . scalar ranges are tracked as normal through alu and jmp insns. |
| 2019 | * . once precise value of the scalar register is used in: |
| 2020 | * . ptr + scalar alu |
| 2021 | * . if (scalar cond K|scalar) |
| 2022 | * . helper_call(.., scalar, ...) where ARG_CONST is expected |
| 2023 | * backtrack through the verifier states and mark all registers and |
| 2024 | * stack slots with spilled constants that these scalar regisers |
| 2025 | * should be precise. |
| 2026 | * . during state pruning two registers (or spilled stack slots) |
| 2027 | * are equivalent if both are not precise. |
| 2028 | * |
| 2029 | * Note the verifier cannot simply walk register parentage chain, |
| 2030 | * since many different registers and stack slots could have been |
| 2031 | * used to compute single precise scalar. |
| 2032 | * |
| 2033 | * The approach of starting with precise=true for all registers and then |
| 2034 | * backtrack to mark a register as not precise when the verifier detects |
| 2035 | * that program doesn't care about specific value (e.g., when helper |
| 2036 | * takes register as ARG_ANYTHING parameter) is not safe. |
| 2037 | * |
| 2038 | * It's ok to walk single parentage chain of the verifier states. |
| 2039 | * It's possible that this backtracking will go all the way till 1st insn. |
| 2040 | * All other branches will be explored for needing precision later. |
| 2041 | * |
| 2042 | * The backtracking needs to deal with cases like: |
| 2043 | * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0) |
| 2044 | * r9 -= r8 |
| 2045 | * r5 = r9 |
| 2046 | * if r5 > 0x79f goto pc+7 |
| 2047 | * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff)) |
| 2048 | * r5 += 1 |
| 2049 | * ... |
| 2050 | * call bpf_perf_event_output#25 |
| 2051 | * where .arg5_type = ARG_CONST_SIZE_OR_ZERO |
| 2052 | * |
| 2053 | * and this case: |
| 2054 | * r6 = 1 |
| 2055 | * call foo // uses callee's r6 inside to compute r0 |
| 2056 | * r0 += r6 |
| 2057 | * if r0 == 0 goto |
| 2058 | * |
| 2059 | * to track above reg_mask/stack_mask needs to be independent for each frame. |
| 2060 | * |
| 2061 | * Also if parent's curframe > frame where backtracking started, |
| 2062 | * the verifier need to mark registers in both frames, otherwise callees |
| 2063 | * may incorrectly prune callers. This is similar to |
| 2064 | * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences") |
| 2065 | * |
| 2066 | * For now backtracking falls back into conservative marking. |
| 2067 | */ |
| 2068 | static void mark_all_scalars_precise(struct bpf_verifier_env *env, |
| 2069 | struct bpf_verifier_state *st) |
| 2070 | { |
| 2071 | struct bpf_func_state *func; |
| 2072 | struct bpf_reg_state *reg; |
| 2073 | int i, j; |
| 2074 | |
| 2075 | /* big hammer: mark all scalars precise in this path. |
| 2076 | * pop_stack may still get !precise scalars. |
| 2077 | */ |
| 2078 | for (; st; st = st->parent) |
| 2079 | for (i = 0; i <= st->curframe; i++) { |
| 2080 | func = st->frame[i]; |
| 2081 | for (j = 0; j < BPF_REG_FP; j++) { |
| 2082 | reg = &func->regs[j]; |
| 2083 | if (reg->type != SCALAR_VALUE) |
| 2084 | continue; |
| 2085 | reg->precise = true; |
| 2086 | } |
| 2087 | for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) { |
| 2088 | if (func->stack[j].slot_type[0] != STACK_SPILL) |
| 2089 | continue; |
| 2090 | reg = &func->stack[j].spilled_ptr; |
| 2091 | if (reg->type != SCALAR_VALUE) |
| 2092 | continue; |
| 2093 | reg->precise = true; |
| 2094 | } |
| 2095 | } |
| 2096 | } |
| 2097 | |
| 2098 | static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, |
| 2099 | int spi) |
| 2100 | { |
| 2101 | struct bpf_verifier_state *st = env->cur_state; |
| 2102 | int first_idx = st->first_insn_idx; |
| 2103 | int last_idx = env->insn_idx; |
| 2104 | struct bpf_func_state *func; |
| 2105 | struct bpf_reg_state *reg; |
| 2106 | u32 reg_mask = regno >= 0 ? 1u << regno : 0; |
| 2107 | u64 stack_mask = spi >= 0 ? 1ull << spi : 0; |
| 2108 | bool skip_first = true; |
| 2109 | bool new_marks = false; |
| 2110 | int i, err; |
| 2111 | |
| 2112 | if (!env->bpf_capable) |
| 2113 | return 0; |
| 2114 | |
| 2115 | func = st->frame[st->curframe]; |
| 2116 | if (regno >= 0) { |
| 2117 | reg = &func->regs[regno]; |
| 2118 | if (reg->type != SCALAR_VALUE) { |
| 2119 | WARN_ONCE(1, "backtracing misuse"); |
| 2120 | return -EFAULT; |
| 2121 | } |
| 2122 | if (!reg->precise) |
| 2123 | new_marks = true; |
| 2124 | else |
| 2125 | reg_mask = 0; |
| 2126 | reg->precise = true; |
| 2127 | } |
| 2128 | |
| 2129 | while (spi >= 0) { |
| 2130 | if (func->stack[spi].slot_type[0] != STACK_SPILL) { |
| 2131 | stack_mask = 0; |
| 2132 | break; |
| 2133 | } |
| 2134 | reg = &func->stack[spi].spilled_ptr; |
| 2135 | if (reg->type != SCALAR_VALUE) { |
| 2136 | stack_mask = 0; |
| 2137 | break; |
| 2138 | } |
| 2139 | if (!reg->precise) |
| 2140 | new_marks = true; |
| 2141 | else |
| 2142 | stack_mask = 0; |
| 2143 | reg->precise = true; |
| 2144 | break; |
| 2145 | } |
| 2146 | |
| 2147 | if (!new_marks) |
| 2148 | return 0; |
| 2149 | if (!reg_mask && !stack_mask) |
| 2150 | return 0; |
| 2151 | for (;;) { |
| 2152 | DECLARE_BITMAP(mask, 64); |
| 2153 | u32 history = st->jmp_history_cnt; |
| 2154 | |
| 2155 | if (env->log.level & BPF_LOG_LEVEL) |
| 2156 | verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx); |
| 2157 | for (i = last_idx;;) { |
| 2158 | if (skip_first) { |
| 2159 | err = 0; |
| 2160 | skip_first = false; |
| 2161 | } else { |
| 2162 | err = backtrack_insn(env, i, ®_mask, &stack_mask); |
| 2163 | } |
| 2164 | if (err == -ENOTSUPP) { |
| 2165 | mark_all_scalars_precise(env, st); |
| 2166 | return 0; |
| 2167 | } else if (err) { |
| 2168 | return err; |
| 2169 | } |
| 2170 | if (!reg_mask && !stack_mask) |
| 2171 | /* Found assignment(s) into tracked register in this state. |
| 2172 | * Since this state is already marked, just return. |
| 2173 | * Nothing to be tracked further in the parent state. |
| 2174 | */ |
| 2175 | return 0; |
| 2176 | if (i == first_idx) |
| 2177 | break; |
| 2178 | i = get_prev_insn_idx(st, i, &history); |
| 2179 | if (i >= env->prog->len) { |
| 2180 | /* This can happen if backtracking reached insn 0 |
| 2181 | * and there are still reg_mask or stack_mask |
| 2182 | * to backtrack. |
| 2183 | * It means the backtracking missed the spot where |
| 2184 | * particular register was initialized with a constant. |
| 2185 | */ |
| 2186 | verbose(env, "BUG backtracking idx %d\n", i); |
| 2187 | WARN_ONCE(1, "verifier backtracking bug"); |
| 2188 | return -EFAULT; |
| 2189 | } |
| 2190 | } |
| 2191 | st = st->parent; |
| 2192 | if (!st) |
| 2193 | break; |
| 2194 | |
| 2195 | new_marks = false; |
| 2196 | func = st->frame[st->curframe]; |
| 2197 | bitmap_from_u64(mask, reg_mask); |
| 2198 | for_each_set_bit(i, mask, 32) { |
| 2199 | reg = &func->regs[i]; |
| 2200 | if (reg->type != SCALAR_VALUE) { |
| 2201 | reg_mask &= ~(1u << i); |
| 2202 | continue; |
| 2203 | } |
| 2204 | if (!reg->precise) |
| 2205 | new_marks = true; |
| 2206 | reg->precise = true; |
| 2207 | } |
| 2208 | |
| 2209 | bitmap_from_u64(mask, stack_mask); |
| 2210 | for_each_set_bit(i, mask, 64) { |
| 2211 | if (i >= func->allocated_stack / BPF_REG_SIZE) { |
| 2212 | /* the sequence of instructions: |
| 2213 | * 2: (bf) r3 = r10 |
| 2214 | * 3: (7b) *(u64 *)(r3 -8) = r0 |
| 2215 | * 4: (79) r4 = *(u64 *)(r10 -8) |
| 2216 | * doesn't contain jmps. It's backtracked |
| 2217 | * as a single block. |
| 2218 | * During backtracking insn 3 is not recognized as |
| 2219 | * stack access, so at the end of backtracking |
| 2220 | * stack slot fp-8 is still marked in stack_mask. |
| 2221 | * However the parent state may not have accessed |
| 2222 | * fp-8 and it's "unallocated" stack space. |
| 2223 | * In such case fallback to conservative. |
| 2224 | */ |
| 2225 | mark_all_scalars_precise(env, st); |
| 2226 | return 0; |
| 2227 | } |
| 2228 | |
| 2229 | if (func->stack[i].slot_type[0] != STACK_SPILL) { |
| 2230 | stack_mask &= ~(1ull << i); |
| 2231 | continue; |
| 2232 | } |
| 2233 | reg = &func->stack[i].spilled_ptr; |
| 2234 | if (reg->type != SCALAR_VALUE) { |
| 2235 | stack_mask &= ~(1ull << i); |
| 2236 | continue; |
| 2237 | } |
| 2238 | if (!reg->precise) |
| 2239 | new_marks = true; |
| 2240 | reg->precise = true; |
| 2241 | } |
| 2242 | if (env->log.level & BPF_LOG_LEVEL) { |
| 2243 | print_verifier_state(env, func); |
| 2244 | verbose(env, "parent %s regs=%x stack=%llx marks\n", |
| 2245 | new_marks ? "didn't have" : "already had", |
| 2246 | reg_mask, stack_mask); |
| 2247 | } |
| 2248 | |
| 2249 | if (!reg_mask && !stack_mask) |
| 2250 | break; |
| 2251 | if (!new_marks) |
| 2252 | break; |
| 2253 | |
| 2254 | last_idx = st->last_insn_idx; |
| 2255 | first_idx = st->first_insn_idx; |
| 2256 | } |
| 2257 | return 0; |
| 2258 | } |
| 2259 | |
| 2260 | static int mark_chain_precision(struct bpf_verifier_env *env, int regno) |
| 2261 | { |
| 2262 | return __mark_chain_precision(env, regno, -1); |
| 2263 | } |
| 2264 | |
| 2265 | static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi) |
| 2266 | { |
| 2267 | return __mark_chain_precision(env, -1, spi); |
| 2268 | } |
| 2269 | |
| 2270 | static bool is_spillable_regtype(enum bpf_reg_type type) |
| 2271 | { |
| 2272 | switch (type) { |
| 2273 | case PTR_TO_MAP_VALUE: |
| 2274 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 2275 | case PTR_TO_STACK: |
| 2276 | case PTR_TO_CTX: |
| 2277 | case PTR_TO_PACKET: |
| 2278 | case PTR_TO_PACKET_META: |
| 2279 | case PTR_TO_PACKET_END: |
| 2280 | case PTR_TO_FLOW_KEYS: |
| 2281 | case CONST_PTR_TO_MAP: |
| 2282 | case PTR_TO_SOCKET: |
| 2283 | case PTR_TO_SOCKET_OR_NULL: |
| 2284 | case PTR_TO_SOCK_COMMON: |
| 2285 | case PTR_TO_SOCK_COMMON_OR_NULL: |
| 2286 | case PTR_TO_TCP_SOCK: |
| 2287 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 2288 | case PTR_TO_XDP_SOCK: |
| 2289 | case PTR_TO_BTF_ID: |
| 2290 | case PTR_TO_BTF_ID_OR_NULL: |
| 2291 | case PTR_TO_RDONLY_BUF: |
| 2292 | case PTR_TO_RDONLY_BUF_OR_NULL: |
| 2293 | case PTR_TO_RDWR_BUF: |
| 2294 | case PTR_TO_RDWR_BUF_OR_NULL: |
| 2295 | case PTR_TO_PERCPU_BTF_ID: |
| 2296 | case PTR_TO_MEM: |
| 2297 | case PTR_TO_MEM_OR_NULL: |
| 2298 | return true; |
| 2299 | default: |
| 2300 | return false; |
| 2301 | } |
| 2302 | } |
| 2303 | |
| 2304 | /* Does this register contain a constant zero? */ |
| 2305 | static bool register_is_null(struct bpf_reg_state *reg) |
| 2306 | { |
| 2307 | return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); |
| 2308 | } |
| 2309 | |
| 2310 | static bool register_is_const(struct bpf_reg_state *reg) |
| 2311 | { |
| 2312 | return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off); |
| 2313 | } |
| 2314 | |
| 2315 | static bool __is_scalar_unbounded(struct bpf_reg_state *reg) |
| 2316 | { |
| 2317 | return tnum_is_unknown(reg->var_off) && |
| 2318 | reg->smin_value == S64_MIN && reg->smax_value == S64_MAX && |
| 2319 | reg->umin_value == 0 && reg->umax_value == U64_MAX && |
| 2320 | reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX && |
| 2321 | reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX; |
| 2322 | } |
| 2323 | |
| 2324 | static bool register_is_bounded(struct bpf_reg_state *reg) |
| 2325 | { |
| 2326 | return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg); |
| 2327 | } |
| 2328 | |
| 2329 | static bool __is_pointer_value(bool allow_ptr_leaks, |
| 2330 | const struct bpf_reg_state *reg) |
| 2331 | { |
| 2332 | if (allow_ptr_leaks) |
| 2333 | return false; |
| 2334 | |
| 2335 | return reg->type != SCALAR_VALUE; |
| 2336 | } |
| 2337 | |
| 2338 | static void save_register_state(struct bpf_func_state *state, |
| 2339 | int spi, struct bpf_reg_state *reg) |
| 2340 | { |
| 2341 | int i; |
| 2342 | |
| 2343 | state->stack[spi].spilled_ptr = *reg; |
| 2344 | state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; |
| 2345 | |
| 2346 | for (i = 0; i < BPF_REG_SIZE; i++) |
| 2347 | state->stack[spi].slot_type[i] = STACK_SPILL; |
| 2348 | } |
| 2349 | |
| 2350 | /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, |
| 2351 | * stack boundary and alignment are checked in check_mem_access() |
| 2352 | */ |
| 2353 | static int check_stack_write_fixed_off(struct bpf_verifier_env *env, |
| 2354 | /* stack frame we're writing to */ |
| 2355 | struct bpf_func_state *state, |
| 2356 | int off, int size, int value_regno, |
| 2357 | int insn_idx) |
| 2358 | { |
| 2359 | struct bpf_func_state *cur; /* state of the current function */ |
| 2360 | int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; |
| 2361 | u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; |
| 2362 | struct bpf_reg_state *reg = NULL; |
| 2363 | |
| 2364 | err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE), |
| 2365 | state->acquired_refs, true); |
| 2366 | if (err) |
| 2367 | return err; |
| 2368 | /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, |
| 2369 | * so it's aligned access and [off, off + size) are within stack limits |
| 2370 | */ |
| 2371 | if (!env->allow_ptr_leaks && |
| 2372 | state->stack[spi].slot_type[0] == STACK_SPILL && |
| 2373 | size != BPF_REG_SIZE) { |
| 2374 | verbose(env, "attempt to corrupt spilled pointer on stack\n"); |
| 2375 | return -EACCES; |
| 2376 | } |
| 2377 | |
| 2378 | cur = env->cur_state->frame[env->cur_state->curframe]; |
| 2379 | if (value_regno >= 0) |
| 2380 | reg = &cur->regs[value_regno]; |
| 2381 | |
| 2382 | if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) && |
| 2383 | !register_is_null(reg) && env->bpf_capable) { |
| 2384 | if (dst_reg != BPF_REG_FP) { |
| 2385 | /* The backtracking logic can only recognize explicit |
| 2386 | * stack slot address like [fp - 8]. Other spill of |
| 2387 | * scalar via different register has to be conervative. |
| 2388 | * Backtrack from here and mark all registers as precise |
| 2389 | * that contributed into 'reg' being a constant. |
| 2390 | */ |
| 2391 | err = mark_chain_precision(env, value_regno); |
| 2392 | if (err) |
| 2393 | return err; |
| 2394 | } |
| 2395 | save_register_state(state, spi, reg); |
| 2396 | } else if (reg && is_spillable_regtype(reg->type)) { |
| 2397 | /* register containing pointer is being spilled into stack */ |
| 2398 | if (size != BPF_REG_SIZE) { |
| 2399 | verbose_linfo(env, insn_idx, "; "); |
| 2400 | verbose(env, "invalid size of register spill\n"); |
| 2401 | return -EACCES; |
| 2402 | } |
| 2403 | |
| 2404 | if (state != cur && reg->type == PTR_TO_STACK) { |
| 2405 | verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); |
| 2406 | return -EINVAL; |
| 2407 | } |
| 2408 | |
| 2409 | if (!env->bypass_spec_v4) { |
| 2410 | bool sanitize = false; |
| 2411 | |
| 2412 | if (state->stack[spi].slot_type[0] == STACK_SPILL && |
| 2413 | register_is_const(&state->stack[spi].spilled_ptr)) |
| 2414 | sanitize = true; |
| 2415 | for (i = 0; i < BPF_REG_SIZE; i++) |
| 2416 | if (state->stack[spi].slot_type[i] == STACK_MISC) { |
| 2417 | sanitize = true; |
| 2418 | break; |
| 2419 | } |
| 2420 | if (sanitize) { |
| 2421 | int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; |
| 2422 | int soff = (-spi - 1) * BPF_REG_SIZE; |
| 2423 | |
| 2424 | /* detected reuse of integer stack slot with a pointer |
| 2425 | * which means either llvm is reusing stack slot or |
| 2426 | * an attacker is trying to exploit CVE-2018-3639 |
| 2427 | * (speculative store bypass) |
| 2428 | * Have to sanitize that slot with preemptive |
| 2429 | * store of zero. |
| 2430 | */ |
| 2431 | if (*poff && *poff != soff) { |
| 2432 | /* disallow programs where single insn stores |
| 2433 | * into two different stack slots, since verifier |
| 2434 | * cannot sanitize them |
| 2435 | */ |
| 2436 | verbose(env, |
| 2437 | "insn %d cannot access two stack slots fp%d and fp%d", |
| 2438 | insn_idx, *poff, soff); |
| 2439 | return -EINVAL; |
| 2440 | } |
| 2441 | *poff = soff; |
| 2442 | } |
| 2443 | } |
| 2444 | save_register_state(state, spi, reg); |
| 2445 | } else { |
| 2446 | u8 type = STACK_MISC; |
| 2447 | |
| 2448 | /* regular write of data into stack destroys any spilled ptr */ |
| 2449 | state->stack[spi].spilled_ptr.type = NOT_INIT; |
| 2450 | /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ |
| 2451 | if (state->stack[spi].slot_type[0] == STACK_SPILL) |
| 2452 | for (i = 0; i < BPF_REG_SIZE; i++) |
| 2453 | state->stack[spi].slot_type[i] = STACK_MISC; |
| 2454 | |
| 2455 | /* only mark the slot as written if all 8 bytes were written |
| 2456 | * otherwise read propagation may incorrectly stop too soon |
| 2457 | * when stack slots are partially written. |
| 2458 | * This heuristic means that read propagation will be |
| 2459 | * conservative, since it will add reg_live_read marks |
| 2460 | * to stack slots all the way to first state when programs |
| 2461 | * writes+reads less than 8 bytes |
| 2462 | */ |
| 2463 | if (size == BPF_REG_SIZE) |
| 2464 | state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; |
| 2465 | |
| 2466 | /* when we zero initialize stack slots mark them as such */ |
| 2467 | if (reg && register_is_null(reg)) { |
| 2468 | /* backtracking doesn't work for STACK_ZERO yet. */ |
| 2469 | err = mark_chain_precision(env, value_regno); |
| 2470 | if (err) |
| 2471 | return err; |
| 2472 | type = STACK_ZERO; |
| 2473 | } |
| 2474 | |
| 2475 | /* Mark slots affected by this stack write. */ |
| 2476 | for (i = 0; i < size; i++) |
| 2477 | state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = |
| 2478 | type; |
| 2479 | } |
| 2480 | return 0; |
| 2481 | } |
| 2482 | |
| 2483 | /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is |
| 2484 | * known to contain a variable offset. |
| 2485 | * This function checks whether the write is permitted and conservatively |
| 2486 | * tracks the effects of the write, considering that each stack slot in the |
| 2487 | * dynamic range is potentially written to. |
| 2488 | * |
| 2489 | * 'off' includes 'regno->off'. |
| 2490 | * 'value_regno' can be -1, meaning that an unknown value is being written to |
| 2491 | * the stack. |
| 2492 | * |
| 2493 | * Spilled pointers in range are not marked as written because we don't know |
| 2494 | * what's going to be actually written. This means that read propagation for |
| 2495 | * future reads cannot be terminated by this write. |
| 2496 | * |
| 2497 | * For privileged programs, uninitialized stack slots are considered |
| 2498 | * initialized by this write (even though we don't know exactly what offsets |
| 2499 | * are going to be written to). The idea is that we don't want the verifier to |
| 2500 | * reject future reads that access slots written to through variable offsets. |
| 2501 | */ |
| 2502 | static int check_stack_write_var_off(struct bpf_verifier_env *env, |
| 2503 | /* func where register points to */ |
| 2504 | struct bpf_func_state *state, |
| 2505 | int ptr_regno, int off, int size, |
| 2506 | int value_regno, int insn_idx) |
| 2507 | { |
| 2508 | struct bpf_func_state *cur; /* state of the current function */ |
| 2509 | int min_off, max_off; |
| 2510 | int i, err; |
| 2511 | struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL; |
| 2512 | bool writing_zero = false; |
| 2513 | /* set if the fact that we're writing a zero is used to let any |
| 2514 | * stack slots remain STACK_ZERO |
| 2515 | */ |
| 2516 | bool zero_used = false; |
| 2517 | |
| 2518 | cur = env->cur_state->frame[env->cur_state->curframe]; |
| 2519 | ptr_reg = &cur->regs[ptr_regno]; |
| 2520 | min_off = ptr_reg->smin_value + off; |
| 2521 | max_off = ptr_reg->smax_value + off + size; |
| 2522 | if (value_regno >= 0) |
| 2523 | value_reg = &cur->regs[value_regno]; |
| 2524 | if (value_reg && register_is_null(value_reg)) |
| 2525 | writing_zero = true; |
| 2526 | |
| 2527 | err = realloc_func_state(state, round_up(-min_off, BPF_REG_SIZE), |
| 2528 | state->acquired_refs, true); |
| 2529 | if (err) |
| 2530 | return err; |
| 2531 | |
| 2532 | |
| 2533 | /* Variable offset writes destroy any spilled pointers in range. */ |
| 2534 | for (i = min_off; i < max_off; i++) { |
| 2535 | u8 new_type, *stype; |
| 2536 | int slot, spi; |
| 2537 | |
| 2538 | slot = -i - 1; |
| 2539 | spi = slot / BPF_REG_SIZE; |
| 2540 | stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; |
| 2541 | |
| 2542 | if (!env->allow_ptr_leaks |
| 2543 | && *stype != NOT_INIT |
| 2544 | && *stype != SCALAR_VALUE) { |
| 2545 | /* Reject the write if there's are spilled pointers in |
| 2546 | * range. If we didn't reject here, the ptr status |
| 2547 | * would be erased below (even though not all slots are |
| 2548 | * actually overwritten), possibly opening the door to |
| 2549 | * leaks. |
| 2550 | */ |
| 2551 | verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", |
| 2552 | insn_idx, i); |
| 2553 | return -EINVAL; |
| 2554 | } |
| 2555 | |
| 2556 | /* Erase all spilled pointers. */ |
| 2557 | state->stack[spi].spilled_ptr.type = NOT_INIT; |
| 2558 | |
| 2559 | /* Update the slot type. */ |
| 2560 | new_type = STACK_MISC; |
| 2561 | if (writing_zero && *stype == STACK_ZERO) { |
| 2562 | new_type = STACK_ZERO; |
| 2563 | zero_used = true; |
| 2564 | } |
| 2565 | /* If the slot is STACK_INVALID, we check whether it's OK to |
| 2566 | * pretend that it will be initialized by this write. The slot |
| 2567 | * might not actually be written to, and so if we mark it as |
| 2568 | * initialized future reads might leak uninitialized memory. |
| 2569 | * For privileged programs, we will accept such reads to slots |
| 2570 | * that may or may not be written because, if we're reject |
| 2571 | * them, the error would be too confusing. |
| 2572 | */ |
| 2573 | if (*stype == STACK_INVALID && !env->allow_uninit_stack) { |
| 2574 | verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", |
| 2575 | insn_idx, i); |
| 2576 | return -EINVAL; |
| 2577 | } |
| 2578 | *stype = new_type; |
| 2579 | } |
| 2580 | if (zero_used) { |
| 2581 | /* backtracking doesn't work for STACK_ZERO yet. */ |
| 2582 | err = mark_chain_precision(env, value_regno); |
| 2583 | if (err) |
| 2584 | return err; |
| 2585 | } |
| 2586 | return 0; |
| 2587 | } |
| 2588 | |
| 2589 | /* When register 'dst_regno' is assigned some values from stack[min_off, |
| 2590 | * max_off), we set the register's type according to the types of the |
| 2591 | * respective stack slots. If all the stack values are known to be zeros, then |
| 2592 | * so is the destination reg. Otherwise, the register is considered to be |
| 2593 | * SCALAR. This function does not deal with register filling; the caller must |
| 2594 | * ensure that all spilled registers in the stack range have been marked as |
| 2595 | * read. |
| 2596 | */ |
| 2597 | static void mark_reg_stack_read(struct bpf_verifier_env *env, |
| 2598 | /* func where src register points to */ |
| 2599 | struct bpf_func_state *ptr_state, |
| 2600 | int min_off, int max_off, int dst_regno) |
| 2601 | { |
| 2602 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2603 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 2604 | int i, slot, spi; |
| 2605 | u8 *stype; |
| 2606 | int zeros = 0; |
| 2607 | |
| 2608 | for (i = min_off; i < max_off; i++) { |
| 2609 | slot = -i - 1; |
| 2610 | spi = slot / BPF_REG_SIZE; |
| 2611 | stype = ptr_state->stack[spi].slot_type; |
| 2612 | if (stype[slot % BPF_REG_SIZE] != STACK_ZERO) |
| 2613 | break; |
| 2614 | zeros++; |
| 2615 | } |
| 2616 | if (zeros == max_off - min_off) { |
| 2617 | /* any access_size read into register is zero extended, |
| 2618 | * so the whole register == const_zero |
| 2619 | */ |
| 2620 | __mark_reg_const_zero(&state->regs[dst_regno]); |
| 2621 | /* backtracking doesn't support STACK_ZERO yet, |
| 2622 | * so mark it precise here, so that later |
| 2623 | * backtracking can stop here. |
| 2624 | * Backtracking may not need this if this register |
| 2625 | * doesn't participate in pointer adjustment. |
| 2626 | * Forward propagation of precise flag is not |
| 2627 | * necessary either. This mark is only to stop |
| 2628 | * backtracking. Any register that contributed |
| 2629 | * to const 0 was marked precise before spill. |
| 2630 | */ |
| 2631 | state->regs[dst_regno].precise = true; |
| 2632 | } else { |
| 2633 | /* have read misc data from the stack */ |
| 2634 | mark_reg_unknown(env, state->regs, dst_regno); |
| 2635 | } |
| 2636 | state->regs[dst_regno].live |= REG_LIVE_WRITTEN; |
| 2637 | } |
| 2638 | |
| 2639 | /* Read the stack at 'off' and put the results into the register indicated by |
| 2640 | * 'dst_regno'. It handles reg filling if the addressed stack slot is a |
| 2641 | * spilled reg. |
| 2642 | * |
| 2643 | * 'dst_regno' can be -1, meaning that the read value is not going to a |
| 2644 | * register. |
| 2645 | * |
| 2646 | * The access is assumed to be within the current stack bounds. |
| 2647 | */ |
| 2648 | static int check_stack_read_fixed_off(struct bpf_verifier_env *env, |
| 2649 | /* func where src register points to */ |
| 2650 | struct bpf_func_state *reg_state, |
| 2651 | int off, int size, int dst_regno) |
| 2652 | { |
| 2653 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2654 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 2655 | int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; |
| 2656 | struct bpf_reg_state *reg; |
| 2657 | u8 *stype; |
| 2658 | |
| 2659 | stype = reg_state->stack[spi].slot_type; |
| 2660 | reg = ®_state->stack[spi].spilled_ptr; |
| 2661 | |
| 2662 | if (stype[0] == STACK_SPILL) { |
| 2663 | if (size != BPF_REG_SIZE) { |
| 2664 | if (reg->type != SCALAR_VALUE) { |
| 2665 | verbose_linfo(env, env->insn_idx, "; "); |
| 2666 | verbose(env, "invalid size of register fill\n"); |
| 2667 | return -EACCES; |
| 2668 | } |
| 2669 | if (dst_regno >= 0) { |
| 2670 | mark_reg_unknown(env, state->regs, dst_regno); |
| 2671 | state->regs[dst_regno].live |= REG_LIVE_WRITTEN; |
| 2672 | } |
| 2673 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); |
| 2674 | return 0; |
| 2675 | } |
| 2676 | for (i = 1; i < BPF_REG_SIZE; i++) { |
| 2677 | if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { |
| 2678 | verbose(env, "corrupted spill memory\n"); |
| 2679 | return -EACCES; |
| 2680 | } |
| 2681 | } |
| 2682 | |
| 2683 | if (dst_regno >= 0) { |
| 2684 | /* restore register state from stack */ |
| 2685 | state->regs[dst_regno] = *reg; |
| 2686 | /* mark reg as written since spilled pointer state likely |
| 2687 | * has its liveness marks cleared by is_state_visited() |
| 2688 | * which resets stack/reg liveness for state transitions |
| 2689 | */ |
| 2690 | state->regs[dst_regno].live |= REG_LIVE_WRITTEN; |
| 2691 | } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) { |
| 2692 | /* If dst_regno==-1, the caller is asking us whether |
| 2693 | * it is acceptable to use this value as a SCALAR_VALUE |
| 2694 | * (e.g. for XADD). |
| 2695 | * We must not allow unprivileged callers to do that |
| 2696 | * with spilled pointers. |
| 2697 | */ |
| 2698 | verbose(env, "leaking pointer from stack off %d\n", |
| 2699 | off); |
| 2700 | return -EACCES; |
| 2701 | } |
| 2702 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); |
| 2703 | } else { |
| 2704 | u8 type; |
| 2705 | |
| 2706 | for (i = 0; i < size; i++) { |
| 2707 | type = stype[(slot - i) % BPF_REG_SIZE]; |
| 2708 | if (type == STACK_MISC) |
| 2709 | continue; |
| 2710 | if (type == STACK_ZERO) |
| 2711 | continue; |
| 2712 | verbose(env, "invalid read from stack off %d+%d size %d\n", |
| 2713 | off, i, size); |
| 2714 | return -EACCES; |
| 2715 | } |
| 2716 | mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64); |
| 2717 | if (dst_regno >= 0) |
| 2718 | mark_reg_stack_read(env, reg_state, off, off + size, dst_regno); |
| 2719 | } |
| 2720 | return 0; |
| 2721 | } |
| 2722 | |
| 2723 | enum stack_access_src { |
| 2724 | ACCESS_DIRECT = 1, /* the access is performed by an instruction */ |
| 2725 | ACCESS_HELPER = 2, /* the access is performed by a helper */ |
| 2726 | }; |
| 2727 | |
| 2728 | static int check_stack_range_initialized(struct bpf_verifier_env *env, |
| 2729 | int regno, int off, int access_size, |
| 2730 | bool zero_size_allowed, |
| 2731 | enum stack_access_src type, |
| 2732 | struct bpf_call_arg_meta *meta); |
| 2733 | |
| 2734 | static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) |
| 2735 | { |
| 2736 | return cur_regs(env) + regno; |
| 2737 | } |
| 2738 | |
| 2739 | /* Read the stack at 'ptr_regno + off' and put the result into the register |
| 2740 | * 'dst_regno'. |
| 2741 | * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'), |
| 2742 | * but not its variable offset. |
| 2743 | * 'size' is assumed to be <= reg size and the access is assumed to be aligned. |
| 2744 | * |
| 2745 | * As opposed to check_stack_read_fixed_off, this function doesn't deal with |
| 2746 | * filling registers (i.e. reads of spilled register cannot be detected when |
| 2747 | * the offset is not fixed). We conservatively mark 'dst_regno' as containing |
| 2748 | * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable |
| 2749 | * offset; for a fixed offset check_stack_read_fixed_off should be used |
| 2750 | * instead. |
| 2751 | */ |
| 2752 | static int check_stack_read_var_off(struct bpf_verifier_env *env, |
| 2753 | int ptr_regno, int off, int size, int dst_regno) |
| 2754 | { |
| 2755 | /* The state of the source register. */ |
| 2756 | struct bpf_reg_state *reg = reg_state(env, ptr_regno); |
| 2757 | struct bpf_func_state *ptr_state = func(env, reg); |
| 2758 | int err; |
| 2759 | int min_off, max_off; |
| 2760 | |
| 2761 | /* Note that we pass a NULL meta, so raw access will not be permitted. |
| 2762 | */ |
| 2763 | err = check_stack_range_initialized(env, ptr_regno, off, size, |
| 2764 | false, ACCESS_DIRECT, NULL); |
| 2765 | if (err) |
| 2766 | return err; |
| 2767 | |
| 2768 | min_off = reg->smin_value + off; |
| 2769 | max_off = reg->smax_value + off; |
| 2770 | mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno); |
| 2771 | return 0; |
| 2772 | } |
| 2773 | |
| 2774 | /* check_stack_read dispatches to check_stack_read_fixed_off or |
| 2775 | * check_stack_read_var_off. |
| 2776 | * |
| 2777 | * The caller must ensure that the offset falls within the allocated stack |
| 2778 | * bounds. |
| 2779 | * |
| 2780 | * 'dst_regno' is a register which will receive the value from the stack. It |
| 2781 | * can be -1, meaning that the read value is not going to a register. |
| 2782 | */ |
| 2783 | static int check_stack_read(struct bpf_verifier_env *env, |
| 2784 | int ptr_regno, int off, int size, |
| 2785 | int dst_regno) |
| 2786 | { |
| 2787 | struct bpf_reg_state *reg = reg_state(env, ptr_regno); |
| 2788 | struct bpf_func_state *state = func(env, reg); |
| 2789 | int err; |
| 2790 | /* Some accesses are only permitted with a static offset. */ |
| 2791 | bool var_off = !tnum_is_const(reg->var_off); |
| 2792 | |
| 2793 | /* The offset is required to be static when reads don't go to a |
| 2794 | * register, in order to not leak pointers (see |
| 2795 | * check_stack_read_fixed_off). |
| 2796 | */ |
| 2797 | if (dst_regno < 0 && var_off) { |
| 2798 | char tn_buf[48]; |
| 2799 | |
| 2800 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 2801 | verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", |
| 2802 | tn_buf, off, size); |
| 2803 | return -EACCES; |
| 2804 | } |
| 2805 | /* Variable offset is prohibited for unprivileged mode for simplicity |
| 2806 | * since it requires corresponding support in Spectre masking for stack |
| 2807 | * ALU. See also retrieve_ptr_limit(). |
| 2808 | */ |
| 2809 | if (!env->bypass_spec_v1 && var_off) { |
| 2810 | char tn_buf[48]; |
| 2811 | |
| 2812 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 2813 | verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", |
| 2814 | ptr_regno, tn_buf); |
| 2815 | return -EACCES; |
| 2816 | } |
| 2817 | |
| 2818 | if (!var_off) { |
| 2819 | off += reg->var_off.value; |
| 2820 | err = check_stack_read_fixed_off(env, state, off, size, |
| 2821 | dst_regno); |
| 2822 | } else { |
| 2823 | /* Variable offset stack reads need more conservative handling |
| 2824 | * than fixed offset ones. Note that dst_regno >= 0 on this |
| 2825 | * branch. |
| 2826 | */ |
| 2827 | err = check_stack_read_var_off(env, ptr_regno, off, size, |
| 2828 | dst_regno); |
| 2829 | } |
| 2830 | return err; |
| 2831 | } |
| 2832 | |
| 2833 | |
| 2834 | /* check_stack_write dispatches to check_stack_write_fixed_off or |
| 2835 | * check_stack_write_var_off. |
| 2836 | * |
| 2837 | * 'ptr_regno' is the register used as a pointer into the stack. |
| 2838 | * 'off' includes 'ptr_regno->off', but not its variable offset (if any). |
| 2839 | * 'value_regno' is the register whose value we're writing to the stack. It can |
| 2840 | * be -1, meaning that we're not writing from a register. |
| 2841 | * |
| 2842 | * The caller must ensure that the offset falls within the maximum stack size. |
| 2843 | */ |
| 2844 | static int check_stack_write(struct bpf_verifier_env *env, |
| 2845 | int ptr_regno, int off, int size, |
| 2846 | int value_regno, int insn_idx) |
| 2847 | { |
| 2848 | struct bpf_reg_state *reg = reg_state(env, ptr_regno); |
| 2849 | struct bpf_func_state *state = func(env, reg); |
| 2850 | int err; |
| 2851 | |
| 2852 | if (tnum_is_const(reg->var_off)) { |
| 2853 | off += reg->var_off.value; |
| 2854 | err = check_stack_write_fixed_off(env, state, off, size, |
| 2855 | value_regno, insn_idx); |
| 2856 | } else { |
| 2857 | /* Variable offset stack reads need more conservative handling |
| 2858 | * than fixed offset ones. |
| 2859 | */ |
| 2860 | err = check_stack_write_var_off(env, state, |
| 2861 | ptr_regno, off, size, |
| 2862 | value_regno, insn_idx); |
| 2863 | } |
| 2864 | return err; |
| 2865 | } |
| 2866 | |
| 2867 | static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, |
| 2868 | int off, int size, enum bpf_access_type type) |
| 2869 | { |
| 2870 | struct bpf_reg_state *regs = cur_regs(env); |
| 2871 | struct bpf_map *map = regs[regno].map_ptr; |
| 2872 | u32 cap = bpf_map_flags_to_cap(map); |
| 2873 | |
| 2874 | if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { |
| 2875 | verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", |
| 2876 | map->value_size, off, size); |
| 2877 | return -EACCES; |
| 2878 | } |
| 2879 | |
| 2880 | if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { |
| 2881 | verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", |
| 2882 | map->value_size, off, size); |
| 2883 | return -EACCES; |
| 2884 | } |
| 2885 | |
| 2886 | return 0; |
| 2887 | } |
| 2888 | |
| 2889 | /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */ |
| 2890 | static int __check_mem_access(struct bpf_verifier_env *env, int regno, |
| 2891 | int off, int size, u32 mem_size, |
| 2892 | bool zero_size_allowed) |
| 2893 | { |
| 2894 | bool size_ok = size > 0 || (size == 0 && zero_size_allowed); |
| 2895 | struct bpf_reg_state *reg; |
| 2896 | |
| 2897 | if (off >= 0 && size_ok && (u64)off + size <= mem_size) |
| 2898 | return 0; |
| 2899 | |
| 2900 | reg = &cur_regs(env)[regno]; |
| 2901 | switch (reg->type) { |
| 2902 | case PTR_TO_MAP_VALUE: |
| 2903 | verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", |
| 2904 | mem_size, off, size); |
| 2905 | break; |
| 2906 | case PTR_TO_PACKET: |
| 2907 | case PTR_TO_PACKET_META: |
| 2908 | case PTR_TO_PACKET_END: |
| 2909 | verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", |
| 2910 | off, size, regno, reg->id, off, mem_size); |
| 2911 | break; |
| 2912 | case PTR_TO_MEM: |
| 2913 | default: |
| 2914 | verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", |
| 2915 | mem_size, off, size); |
| 2916 | } |
| 2917 | |
| 2918 | return -EACCES; |
| 2919 | } |
| 2920 | |
| 2921 | /* check read/write into a memory region with possible variable offset */ |
| 2922 | static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, |
| 2923 | int off, int size, u32 mem_size, |
| 2924 | bool zero_size_allowed) |
| 2925 | { |
| 2926 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2927 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 2928 | struct bpf_reg_state *reg = &state->regs[regno]; |
| 2929 | int err; |
| 2930 | |
| 2931 | /* We may have adjusted the register pointing to memory region, so we |
| 2932 | * need to try adding each of min_value and max_value to off |
| 2933 | * to make sure our theoretical access will be safe. |
| 2934 | */ |
| 2935 | if (env->log.level & BPF_LOG_LEVEL) |
| 2936 | print_verifier_state(env, state); |
| 2937 | |
| 2938 | /* The minimum value is only important with signed |
| 2939 | * comparisons where we can't assume the floor of a |
| 2940 | * value is 0. If we are using signed variables for our |
| 2941 | * index'es we need to make sure that whatever we use |
| 2942 | * will have a set floor within our range. |
| 2943 | */ |
| 2944 | if (reg->smin_value < 0 && |
| 2945 | (reg->smin_value == S64_MIN || |
| 2946 | (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || |
| 2947 | reg->smin_value + off < 0)) { |
| 2948 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
| 2949 | regno); |
| 2950 | return -EACCES; |
| 2951 | } |
| 2952 | err = __check_mem_access(env, regno, reg->smin_value + off, size, |
| 2953 | mem_size, zero_size_allowed); |
| 2954 | if (err) { |
| 2955 | verbose(env, "R%d min value is outside of the allowed memory range\n", |
| 2956 | regno); |
| 2957 | return err; |
| 2958 | } |
| 2959 | |
| 2960 | /* If we haven't set a max value then we need to bail since we can't be |
| 2961 | * sure we won't do bad things. |
| 2962 | * If reg->umax_value + off could overflow, treat that as unbounded too. |
| 2963 | */ |
| 2964 | if (reg->umax_value >= BPF_MAX_VAR_OFF) { |
| 2965 | verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", |
| 2966 | regno); |
| 2967 | return -EACCES; |
| 2968 | } |
| 2969 | err = __check_mem_access(env, regno, reg->umax_value + off, size, |
| 2970 | mem_size, zero_size_allowed); |
| 2971 | if (err) { |
| 2972 | verbose(env, "R%d max value is outside of the allowed memory range\n", |
| 2973 | regno); |
| 2974 | return err; |
| 2975 | } |
| 2976 | |
| 2977 | return 0; |
| 2978 | } |
| 2979 | |
| 2980 | /* check read/write into a map element with possible variable offset */ |
| 2981 | static int check_map_access(struct bpf_verifier_env *env, u32 regno, |
| 2982 | int off, int size, bool zero_size_allowed) |
| 2983 | { |
| 2984 | struct bpf_verifier_state *vstate = env->cur_state; |
| 2985 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 2986 | struct bpf_reg_state *reg = &state->regs[regno]; |
| 2987 | struct bpf_map *map = reg->map_ptr; |
| 2988 | int err; |
| 2989 | |
| 2990 | err = check_mem_region_access(env, regno, off, size, map->value_size, |
| 2991 | zero_size_allowed); |
| 2992 | if (err) |
| 2993 | return err; |
| 2994 | |
| 2995 | if (map_value_has_spin_lock(map)) { |
| 2996 | u32 lock = map->spin_lock_off; |
| 2997 | |
| 2998 | /* if any part of struct bpf_spin_lock can be touched by |
| 2999 | * load/store reject this program. |
| 3000 | * To check that [x1, x2) overlaps with [y1, y2) |
| 3001 | * it is sufficient to check x1 < y2 && y1 < x2. |
| 3002 | */ |
| 3003 | if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && |
| 3004 | lock < reg->umax_value + off + size) { |
| 3005 | verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); |
| 3006 | return -EACCES; |
| 3007 | } |
| 3008 | } |
| 3009 | return err; |
| 3010 | } |
| 3011 | |
| 3012 | #define MAX_PACKET_OFF 0xffff |
| 3013 | |
| 3014 | static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog) |
| 3015 | { |
| 3016 | return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type; |
| 3017 | } |
| 3018 | |
| 3019 | static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, |
| 3020 | const struct bpf_call_arg_meta *meta, |
| 3021 | enum bpf_access_type t) |
| 3022 | { |
| 3023 | enum bpf_prog_type prog_type = resolve_prog_type(env->prog); |
| 3024 | |
| 3025 | switch (prog_type) { |
| 3026 | /* Program types only with direct read access go here! */ |
| 3027 | case BPF_PROG_TYPE_LWT_IN: |
| 3028 | case BPF_PROG_TYPE_LWT_OUT: |
| 3029 | case BPF_PROG_TYPE_LWT_SEG6LOCAL: |
| 3030 | case BPF_PROG_TYPE_SK_REUSEPORT: |
| 3031 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
| 3032 | case BPF_PROG_TYPE_CGROUP_SKB: |
| 3033 | if (t == BPF_WRITE) |
| 3034 | return false; |
| 3035 | fallthrough; |
| 3036 | |
| 3037 | /* Program types with direct read + write access go here! */ |
| 3038 | case BPF_PROG_TYPE_SCHED_CLS: |
| 3039 | case BPF_PROG_TYPE_SCHED_ACT: |
| 3040 | case BPF_PROG_TYPE_XDP: |
| 3041 | case BPF_PROG_TYPE_LWT_XMIT: |
| 3042 | case BPF_PROG_TYPE_SK_SKB: |
| 3043 | case BPF_PROG_TYPE_SK_MSG: |
| 3044 | if (meta) |
| 3045 | return meta->pkt_access; |
| 3046 | |
| 3047 | env->seen_direct_write = true; |
| 3048 | return true; |
| 3049 | |
| 3050 | case BPF_PROG_TYPE_CGROUP_SOCKOPT: |
| 3051 | if (t == BPF_WRITE) |
| 3052 | env->seen_direct_write = true; |
| 3053 | |
| 3054 | return true; |
| 3055 | |
| 3056 | default: |
| 3057 | return false; |
| 3058 | } |
| 3059 | } |
| 3060 | |
| 3061 | static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, |
| 3062 | int size, bool zero_size_allowed) |
| 3063 | { |
| 3064 | struct bpf_reg_state *regs = cur_regs(env); |
| 3065 | struct bpf_reg_state *reg = ®s[regno]; |
| 3066 | int err; |
| 3067 | |
| 3068 | /* We may have added a variable offset to the packet pointer; but any |
| 3069 | * reg->range we have comes after that. We are only checking the fixed |
| 3070 | * offset. |
| 3071 | */ |
| 3072 | |
| 3073 | /* We don't allow negative numbers, because we aren't tracking enough |
| 3074 | * detail to prove they're safe. |
| 3075 | */ |
| 3076 | if (reg->smin_value < 0) { |
| 3077 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
| 3078 | regno); |
| 3079 | return -EACCES; |
| 3080 | } |
| 3081 | |
| 3082 | err = reg->range < 0 ? -EINVAL : |
| 3083 | __check_mem_access(env, regno, off, size, reg->range, |
| 3084 | zero_size_allowed); |
| 3085 | if (err) { |
| 3086 | verbose(env, "R%d offset is outside of the packet\n", regno); |
| 3087 | return err; |
| 3088 | } |
| 3089 | |
| 3090 | /* __check_mem_access has made sure "off + size - 1" is within u16. |
| 3091 | * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, |
| 3092 | * otherwise find_good_pkt_pointers would have refused to set range info |
| 3093 | * that __check_mem_access would have rejected this pkt access. |
| 3094 | * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. |
| 3095 | */ |
| 3096 | env->prog->aux->max_pkt_offset = |
| 3097 | max_t(u32, env->prog->aux->max_pkt_offset, |
| 3098 | off + reg->umax_value + size - 1); |
| 3099 | |
| 3100 | return err; |
| 3101 | } |
| 3102 | |
| 3103 | /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ |
| 3104 | static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, |
| 3105 | enum bpf_access_type t, enum bpf_reg_type *reg_type, |
| 3106 | struct btf **btf, u32 *btf_id) |
| 3107 | { |
| 3108 | struct bpf_insn_access_aux info = { |
| 3109 | .reg_type = *reg_type, |
| 3110 | .log = &env->log, |
| 3111 | }; |
| 3112 | |
| 3113 | if (env->ops->is_valid_access && |
| 3114 | env->ops->is_valid_access(off, size, t, env->prog, &info)) { |
| 3115 | /* A non zero info.ctx_field_size indicates that this field is a |
| 3116 | * candidate for later verifier transformation to load the whole |
| 3117 | * field and then apply a mask when accessed with a narrower |
| 3118 | * access than actual ctx access size. A zero info.ctx_field_size |
| 3119 | * will only allow for whole field access and rejects any other |
| 3120 | * type of narrower access. |
| 3121 | */ |
| 3122 | *reg_type = info.reg_type; |
| 3123 | |
| 3124 | if (*reg_type == PTR_TO_BTF_ID || *reg_type == PTR_TO_BTF_ID_OR_NULL) { |
| 3125 | *btf = info.btf; |
| 3126 | *btf_id = info.btf_id; |
| 3127 | } else { |
| 3128 | env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; |
| 3129 | } |
| 3130 | /* remember the offset of last byte accessed in ctx */ |
| 3131 | if (env->prog->aux->max_ctx_offset < off + size) |
| 3132 | env->prog->aux->max_ctx_offset = off + size; |
| 3133 | return 0; |
| 3134 | } |
| 3135 | |
| 3136 | verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); |
| 3137 | return -EACCES; |
| 3138 | } |
| 3139 | |
| 3140 | static int check_flow_keys_access(struct bpf_verifier_env *env, int off, |
| 3141 | int size) |
| 3142 | { |
| 3143 | if (size < 0 || off < 0 || |
| 3144 | (u64)off + size > sizeof(struct bpf_flow_keys)) { |
| 3145 | verbose(env, "invalid access to flow keys off=%d size=%d\n", |
| 3146 | off, size); |
| 3147 | return -EACCES; |
| 3148 | } |
| 3149 | return 0; |
| 3150 | } |
| 3151 | |
| 3152 | static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, |
| 3153 | u32 regno, int off, int size, |
| 3154 | enum bpf_access_type t) |
| 3155 | { |
| 3156 | struct bpf_reg_state *regs = cur_regs(env); |
| 3157 | struct bpf_reg_state *reg = ®s[regno]; |
| 3158 | struct bpf_insn_access_aux info = {}; |
| 3159 | bool valid; |
| 3160 | |
| 3161 | if (reg->smin_value < 0) { |
| 3162 | verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", |
| 3163 | regno); |
| 3164 | return -EACCES; |
| 3165 | } |
| 3166 | |
| 3167 | switch (reg->type) { |
| 3168 | case PTR_TO_SOCK_COMMON: |
| 3169 | valid = bpf_sock_common_is_valid_access(off, size, t, &info); |
| 3170 | break; |
| 3171 | case PTR_TO_SOCKET: |
| 3172 | valid = bpf_sock_is_valid_access(off, size, t, &info); |
| 3173 | break; |
| 3174 | case PTR_TO_TCP_SOCK: |
| 3175 | valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); |
| 3176 | break; |
| 3177 | case PTR_TO_XDP_SOCK: |
| 3178 | valid = bpf_xdp_sock_is_valid_access(off, size, t, &info); |
| 3179 | break; |
| 3180 | default: |
| 3181 | valid = false; |
| 3182 | } |
| 3183 | |
| 3184 | |
| 3185 | if (valid) { |
| 3186 | env->insn_aux_data[insn_idx].ctx_field_size = |
| 3187 | info.ctx_field_size; |
| 3188 | return 0; |
| 3189 | } |
| 3190 | |
| 3191 | verbose(env, "R%d invalid %s access off=%d size=%d\n", |
| 3192 | regno, reg_type_str[reg->type], off, size); |
| 3193 | |
| 3194 | return -EACCES; |
| 3195 | } |
| 3196 | |
| 3197 | static bool is_pointer_value(struct bpf_verifier_env *env, int regno) |
| 3198 | { |
| 3199 | return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); |
| 3200 | } |
| 3201 | |
| 3202 | static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) |
| 3203 | { |
| 3204 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 3205 | |
| 3206 | return reg->type == PTR_TO_CTX; |
| 3207 | } |
| 3208 | |
| 3209 | static bool is_sk_reg(struct bpf_verifier_env *env, int regno) |
| 3210 | { |
| 3211 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 3212 | |
| 3213 | return type_is_sk_pointer(reg->type); |
| 3214 | } |
| 3215 | |
| 3216 | static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) |
| 3217 | { |
| 3218 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 3219 | |
| 3220 | return type_is_pkt_pointer(reg->type); |
| 3221 | } |
| 3222 | |
| 3223 | static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) |
| 3224 | { |
| 3225 | const struct bpf_reg_state *reg = reg_state(env, regno); |
| 3226 | |
| 3227 | /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ |
| 3228 | return reg->type == PTR_TO_FLOW_KEYS; |
| 3229 | } |
| 3230 | |
| 3231 | static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, |
| 3232 | const struct bpf_reg_state *reg, |
| 3233 | int off, int size, bool strict) |
| 3234 | { |
| 3235 | struct tnum reg_off; |
| 3236 | int ip_align; |
| 3237 | |
| 3238 | /* Byte size accesses are always allowed. */ |
| 3239 | if (!strict || size == 1) |
| 3240 | return 0; |
| 3241 | |
| 3242 | /* For platforms that do not have a Kconfig enabling |
| 3243 | * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of |
| 3244 | * NET_IP_ALIGN is universally set to '2'. And on platforms |
| 3245 | * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get |
| 3246 | * to this code only in strict mode where we want to emulate |
| 3247 | * the NET_IP_ALIGN==2 checking. Therefore use an |
| 3248 | * unconditional IP align value of '2'. |
| 3249 | */ |
| 3250 | ip_align = 2; |
| 3251 | |
| 3252 | reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); |
| 3253 | if (!tnum_is_aligned(reg_off, size)) { |
| 3254 | char tn_buf[48]; |
| 3255 | |
| 3256 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3257 | verbose(env, |
| 3258 | "misaligned packet access off %d+%s+%d+%d size %d\n", |
| 3259 | ip_align, tn_buf, reg->off, off, size); |
| 3260 | return -EACCES; |
| 3261 | } |
| 3262 | |
| 3263 | return 0; |
| 3264 | } |
| 3265 | |
| 3266 | static int check_generic_ptr_alignment(struct bpf_verifier_env *env, |
| 3267 | const struct bpf_reg_state *reg, |
| 3268 | const char *pointer_desc, |
| 3269 | int off, int size, bool strict) |
| 3270 | { |
| 3271 | struct tnum reg_off; |
| 3272 | |
| 3273 | /* Byte size accesses are always allowed. */ |
| 3274 | if (!strict || size == 1) |
| 3275 | return 0; |
| 3276 | |
| 3277 | reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); |
| 3278 | if (!tnum_is_aligned(reg_off, size)) { |
| 3279 | char tn_buf[48]; |
| 3280 | |
| 3281 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3282 | verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", |
| 3283 | pointer_desc, tn_buf, reg->off, off, size); |
| 3284 | return -EACCES; |
| 3285 | } |
| 3286 | |
| 3287 | return 0; |
| 3288 | } |
| 3289 | |
| 3290 | static int check_ptr_alignment(struct bpf_verifier_env *env, |
| 3291 | const struct bpf_reg_state *reg, int off, |
| 3292 | int size, bool strict_alignment_once) |
| 3293 | { |
| 3294 | bool strict = env->strict_alignment || strict_alignment_once; |
| 3295 | const char *pointer_desc = ""; |
| 3296 | |
| 3297 | switch (reg->type) { |
| 3298 | case PTR_TO_PACKET: |
| 3299 | case PTR_TO_PACKET_META: |
| 3300 | /* Special case, because of NET_IP_ALIGN. Given metadata sits |
| 3301 | * right in front, treat it the very same way. |
| 3302 | */ |
| 3303 | return check_pkt_ptr_alignment(env, reg, off, size, strict); |
| 3304 | case PTR_TO_FLOW_KEYS: |
| 3305 | pointer_desc = "flow keys "; |
| 3306 | break; |
| 3307 | case PTR_TO_MAP_VALUE: |
| 3308 | pointer_desc = "value "; |
| 3309 | break; |
| 3310 | case PTR_TO_CTX: |
| 3311 | pointer_desc = "context "; |
| 3312 | break; |
| 3313 | case PTR_TO_STACK: |
| 3314 | pointer_desc = "stack "; |
| 3315 | /* The stack spill tracking logic in check_stack_write_fixed_off() |
| 3316 | * and check_stack_read_fixed_off() relies on stack accesses being |
| 3317 | * aligned. |
| 3318 | */ |
| 3319 | strict = true; |
| 3320 | break; |
| 3321 | case PTR_TO_SOCKET: |
| 3322 | pointer_desc = "sock "; |
| 3323 | break; |
| 3324 | case PTR_TO_SOCK_COMMON: |
| 3325 | pointer_desc = "sock_common "; |
| 3326 | break; |
| 3327 | case PTR_TO_TCP_SOCK: |
| 3328 | pointer_desc = "tcp_sock "; |
| 3329 | break; |
| 3330 | case PTR_TO_XDP_SOCK: |
| 3331 | pointer_desc = "xdp_sock "; |
| 3332 | break; |
| 3333 | default: |
| 3334 | break; |
| 3335 | } |
| 3336 | return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, |
| 3337 | strict); |
| 3338 | } |
| 3339 | |
| 3340 | static int update_stack_depth(struct bpf_verifier_env *env, |
| 3341 | const struct bpf_func_state *func, |
| 3342 | int off) |
| 3343 | { |
| 3344 | u16 stack = env->subprog_info[func->subprogno].stack_depth; |
| 3345 | |
| 3346 | if (stack >= -off) |
| 3347 | return 0; |
| 3348 | |
| 3349 | /* update known max for given subprogram */ |
| 3350 | env->subprog_info[func->subprogno].stack_depth = -off; |
| 3351 | return 0; |
| 3352 | } |
| 3353 | |
| 3354 | /* starting from main bpf function walk all instructions of the function |
| 3355 | * and recursively walk all callees that given function can call. |
| 3356 | * Ignore jump and exit insns. |
| 3357 | * Since recursion is prevented by check_cfg() this algorithm |
| 3358 | * only needs a local stack of MAX_CALL_FRAMES to remember callsites |
| 3359 | */ |
| 3360 | static int check_max_stack_depth(struct bpf_verifier_env *env) |
| 3361 | { |
| 3362 | int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; |
| 3363 | struct bpf_subprog_info *subprog = env->subprog_info; |
| 3364 | struct bpf_insn *insn = env->prog->insnsi; |
| 3365 | bool tail_call_reachable = false; |
| 3366 | int ret_insn[MAX_CALL_FRAMES]; |
| 3367 | int ret_prog[MAX_CALL_FRAMES]; |
| 3368 | int j; |
| 3369 | |
| 3370 | process_func: |
| 3371 | /* protect against potential stack overflow that might happen when |
| 3372 | * bpf2bpf calls get combined with tailcalls. Limit the caller's stack |
| 3373 | * depth for such case down to 256 so that the worst case scenario |
| 3374 | * would result in 8k stack size (32 which is tailcall limit * 256 = |
| 3375 | * 8k). |
| 3376 | * |
| 3377 | * To get the idea what might happen, see an example: |
| 3378 | * func1 -> sub rsp, 128 |
| 3379 | * subfunc1 -> sub rsp, 256 |
| 3380 | * tailcall1 -> add rsp, 256 |
| 3381 | * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320) |
| 3382 | * subfunc2 -> sub rsp, 64 |
| 3383 | * subfunc22 -> sub rsp, 128 |
| 3384 | * tailcall2 -> add rsp, 128 |
| 3385 | * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416) |
| 3386 | * |
| 3387 | * tailcall will unwind the current stack frame but it will not get rid |
| 3388 | * of caller's stack as shown on the example above. |
| 3389 | */ |
| 3390 | if (idx && subprog[idx].has_tail_call && depth >= 256) { |
| 3391 | verbose(env, |
| 3392 | "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n", |
| 3393 | depth); |
| 3394 | return -EACCES; |
| 3395 | } |
| 3396 | /* round up to 32-bytes, since this is granularity |
| 3397 | * of interpreter stack size |
| 3398 | */ |
| 3399 | depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); |
| 3400 | if (depth > MAX_BPF_STACK) { |
| 3401 | verbose(env, "combined stack size of %d calls is %d. Too large\n", |
| 3402 | frame + 1, depth); |
| 3403 | return -EACCES; |
| 3404 | } |
| 3405 | continue_func: |
| 3406 | subprog_end = subprog[idx + 1].start; |
| 3407 | for (; i < subprog_end; i++) { |
| 3408 | if (!bpf_pseudo_call(insn + i)) |
| 3409 | continue; |
| 3410 | /* remember insn and function to return to */ |
| 3411 | ret_insn[frame] = i + 1; |
| 3412 | ret_prog[frame] = idx; |
| 3413 | |
| 3414 | /* find the callee */ |
| 3415 | i = i + insn[i].imm + 1; |
| 3416 | idx = find_subprog(env, i); |
| 3417 | if (idx < 0) { |
| 3418 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 3419 | i); |
| 3420 | return -EFAULT; |
| 3421 | } |
| 3422 | |
| 3423 | if (subprog[idx].has_tail_call) |
| 3424 | tail_call_reachable = true; |
| 3425 | |
| 3426 | frame++; |
| 3427 | if (frame >= MAX_CALL_FRAMES) { |
| 3428 | verbose(env, "the call stack of %d frames is too deep !\n", |
| 3429 | frame); |
| 3430 | return -E2BIG; |
| 3431 | } |
| 3432 | goto process_func; |
| 3433 | } |
| 3434 | /* if tail call got detected across bpf2bpf calls then mark each of the |
| 3435 | * currently present subprog frames as tail call reachable subprogs; |
| 3436 | * this info will be utilized by JIT so that we will be preserving the |
| 3437 | * tail call counter throughout bpf2bpf calls combined with tailcalls |
| 3438 | */ |
| 3439 | if (tail_call_reachable) |
| 3440 | for (j = 0; j < frame; j++) |
| 3441 | subprog[ret_prog[j]].tail_call_reachable = true; |
| 3442 | |
| 3443 | /* end of for() loop means the last insn of the 'subprog' |
| 3444 | * was reached. Doesn't matter whether it was JA or EXIT |
| 3445 | */ |
| 3446 | if (frame == 0) |
| 3447 | return 0; |
| 3448 | depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); |
| 3449 | frame--; |
| 3450 | i = ret_insn[frame]; |
| 3451 | idx = ret_prog[frame]; |
| 3452 | goto continue_func; |
| 3453 | } |
| 3454 | |
| 3455 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
| 3456 | static int get_callee_stack_depth(struct bpf_verifier_env *env, |
| 3457 | const struct bpf_insn *insn, int idx) |
| 3458 | { |
| 3459 | int start = idx + insn->imm + 1, subprog; |
| 3460 | |
| 3461 | subprog = find_subprog(env, start); |
| 3462 | if (subprog < 0) { |
| 3463 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 3464 | start); |
| 3465 | return -EFAULT; |
| 3466 | } |
| 3467 | return env->subprog_info[subprog].stack_depth; |
| 3468 | } |
| 3469 | #endif |
| 3470 | |
| 3471 | int check_ctx_reg(struct bpf_verifier_env *env, |
| 3472 | const struct bpf_reg_state *reg, int regno) |
| 3473 | { |
| 3474 | /* Access to ctx or passing it to a helper is only allowed in |
| 3475 | * its original, unmodified form. |
| 3476 | */ |
| 3477 | |
| 3478 | if (reg->off) { |
| 3479 | verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", |
| 3480 | regno, reg->off); |
| 3481 | return -EACCES; |
| 3482 | } |
| 3483 | |
| 3484 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 3485 | char tn_buf[48]; |
| 3486 | |
| 3487 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3488 | verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); |
| 3489 | return -EACCES; |
| 3490 | } |
| 3491 | |
| 3492 | return 0; |
| 3493 | } |
| 3494 | |
| 3495 | static int __check_buffer_access(struct bpf_verifier_env *env, |
| 3496 | const char *buf_info, |
| 3497 | const struct bpf_reg_state *reg, |
| 3498 | int regno, int off, int size) |
| 3499 | { |
| 3500 | if (off < 0) { |
| 3501 | verbose(env, |
| 3502 | "R%d invalid %s buffer access: off=%d, size=%d\n", |
| 3503 | regno, buf_info, off, size); |
| 3504 | return -EACCES; |
| 3505 | } |
| 3506 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 3507 | char tn_buf[48]; |
| 3508 | |
| 3509 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3510 | verbose(env, |
| 3511 | "R%d invalid variable buffer offset: off=%d, var_off=%s\n", |
| 3512 | regno, off, tn_buf); |
| 3513 | return -EACCES; |
| 3514 | } |
| 3515 | |
| 3516 | return 0; |
| 3517 | } |
| 3518 | |
| 3519 | static int check_tp_buffer_access(struct bpf_verifier_env *env, |
| 3520 | const struct bpf_reg_state *reg, |
| 3521 | int regno, int off, int size) |
| 3522 | { |
| 3523 | int err; |
| 3524 | |
| 3525 | err = __check_buffer_access(env, "tracepoint", reg, regno, off, size); |
| 3526 | if (err) |
| 3527 | return err; |
| 3528 | |
| 3529 | if (off + size > env->prog->aux->max_tp_access) |
| 3530 | env->prog->aux->max_tp_access = off + size; |
| 3531 | |
| 3532 | return 0; |
| 3533 | } |
| 3534 | |
| 3535 | static int check_buffer_access(struct bpf_verifier_env *env, |
| 3536 | const struct bpf_reg_state *reg, |
| 3537 | int regno, int off, int size, |
| 3538 | bool zero_size_allowed, |
| 3539 | const char *buf_info, |
| 3540 | u32 *max_access) |
| 3541 | { |
| 3542 | int err; |
| 3543 | |
| 3544 | err = __check_buffer_access(env, buf_info, reg, regno, off, size); |
| 3545 | if (err) |
| 3546 | return err; |
| 3547 | |
| 3548 | if (off + size > *max_access) |
| 3549 | *max_access = off + size; |
| 3550 | |
| 3551 | return 0; |
| 3552 | } |
| 3553 | |
| 3554 | /* BPF architecture zero extends alu32 ops into 64-bit registesr */ |
| 3555 | static void zext_32_to_64(struct bpf_reg_state *reg) |
| 3556 | { |
| 3557 | reg->var_off = tnum_subreg(reg->var_off); |
| 3558 | __reg_assign_32_into_64(reg); |
| 3559 | } |
| 3560 | |
| 3561 | /* truncate register to smaller size (in bytes) |
| 3562 | * must be called with size < BPF_REG_SIZE |
| 3563 | */ |
| 3564 | static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) |
| 3565 | { |
| 3566 | u64 mask; |
| 3567 | |
| 3568 | /* clear high bits in bit representation */ |
| 3569 | reg->var_off = tnum_cast(reg->var_off, size); |
| 3570 | |
| 3571 | /* fix arithmetic bounds */ |
| 3572 | mask = ((u64)1 << (size * 8)) - 1; |
| 3573 | if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { |
| 3574 | reg->umin_value &= mask; |
| 3575 | reg->umax_value &= mask; |
| 3576 | } else { |
| 3577 | reg->umin_value = 0; |
| 3578 | reg->umax_value = mask; |
| 3579 | } |
| 3580 | reg->smin_value = reg->umin_value; |
| 3581 | reg->smax_value = reg->umax_value; |
| 3582 | |
| 3583 | /* If size is smaller than 32bit register the 32bit register |
| 3584 | * values are also truncated so we push 64-bit bounds into |
| 3585 | * 32-bit bounds. Above were truncated < 32-bits already. |
| 3586 | */ |
| 3587 | if (size >= 4) |
| 3588 | return; |
| 3589 | __reg_combine_64_into_32(reg); |
| 3590 | } |
| 3591 | |
| 3592 | static bool bpf_map_is_rdonly(const struct bpf_map *map) |
| 3593 | { |
| 3594 | return (map->map_flags & BPF_F_RDONLY_PROG) && map->frozen; |
| 3595 | } |
| 3596 | |
| 3597 | static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val) |
| 3598 | { |
| 3599 | void *ptr; |
| 3600 | u64 addr; |
| 3601 | int err; |
| 3602 | |
| 3603 | err = map->ops->map_direct_value_addr(map, &addr, off); |
| 3604 | if (err) |
| 3605 | return err; |
| 3606 | ptr = (void *)(long)addr + off; |
| 3607 | |
| 3608 | switch (size) { |
| 3609 | case sizeof(u8): |
| 3610 | *val = (u64)*(u8 *)ptr; |
| 3611 | break; |
| 3612 | case sizeof(u16): |
| 3613 | *val = (u64)*(u16 *)ptr; |
| 3614 | break; |
| 3615 | case sizeof(u32): |
| 3616 | *val = (u64)*(u32 *)ptr; |
| 3617 | break; |
| 3618 | case sizeof(u64): |
| 3619 | *val = *(u64 *)ptr; |
| 3620 | break; |
| 3621 | default: |
| 3622 | return -EINVAL; |
| 3623 | } |
| 3624 | return 0; |
| 3625 | } |
| 3626 | |
| 3627 | static int check_ptr_to_btf_access(struct bpf_verifier_env *env, |
| 3628 | struct bpf_reg_state *regs, |
| 3629 | int regno, int off, int size, |
| 3630 | enum bpf_access_type atype, |
| 3631 | int value_regno) |
| 3632 | { |
| 3633 | struct bpf_reg_state *reg = regs + regno; |
| 3634 | const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id); |
| 3635 | const char *tname = btf_name_by_offset(reg->btf, t->name_off); |
| 3636 | u32 btf_id; |
| 3637 | int ret; |
| 3638 | |
| 3639 | if (off < 0) { |
| 3640 | verbose(env, |
| 3641 | "R%d is ptr_%s invalid negative access: off=%d\n", |
| 3642 | regno, tname, off); |
| 3643 | return -EACCES; |
| 3644 | } |
| 3645 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 3646 | char tn_buf[48]; |
| 3647 | |
| 3648 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3649 | verbose(env, |
| 3650 | "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", |
| 3651 | regno, tname, off, tn_buf); |
| 3652 | return -EACCES; |
| 3653 | } |
| 3654 | |
| 3655 | if (env->ops->btf_struct_access) { |
| 3656 | ret = env->ops->btf_struct_access(&env->log, reg->btf, t, |
| 3657 | off, size, atype, &btf_id); |
| 3658 | } else { |
| 3659 | if (atype != BPF_READ) { |
| 3660 | verbose(env, "only read is supported\n"); |
| 3661 | return -EACCES; |
| 3662 | } |
| 3663 | |
| 3664 | ret = btf_struct_access(&env->log, reg->btf, t, off, size, |
| 3665 | atype, &btf_id); |
| 3666 | } |
| 3667 | |
| 3668 | if (ret < 0) |
| 3669 | return ret; |
| 3670 | |
| 3671 | if (atype == BPF_READ && value_regno >= 0) |
| 3672 | mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id); |
| 3673 | |
| 3674 | return 0; |
| 3675 | } |
| 3676 | |
| 3677 | static int check_ptr_to_map_access(struct bpf_verifier_env *env, |
| 3678 | struct bpf_reg_state *regs, |
| 3679 | int regno, int off, int size, |
| 3680 | enum bpf_access_type atype, |
| 3681 | int value_regno) |
| 3682 | { |
| 3683 | struct bpf_reg_state *reg = regs + regno; |
| 3684 | struct bpf_map *map = reg->map_ptr; |
| 3685 | const struct btf_type *t; |
| 3686 | const char *tname; |
| 3687 | u32 btf_id; |
| 3688 | int ret; |
| 3689 | |
| 3690 | if (!btf_vmlinux) { |
| 3691 | verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n"); |
| 3692 | return -ENOTSUPP; |
| 3693 | } |
| 3694 | |
| 3695 | if (!map->ops->map_btf_id || !*map->ops->map_btf_id) { |
| 3696 | verbose(env, "map_ptr access not supported for map type %d\n", |
| 3697 | map->map_type); |
| 3698 | return -ENOTSUPP; |
| 3699 | } |
| 3700 | |
| 3701 | t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id); |
| 3702 | tname = btf_name_by_offset(btf_vmlinux, t->name_off); |
| 3703 | |
| 3704 | if (!env->allow_ptr_to_map_access) { |
| 3705 | verbose(env, |
| 3706 | "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", |
| 3707 | tname); |
| 3708 | return -EPERM; |
| 3709 | } |
| 3710 | |
| 3711 | if (off < 0) { |
| 3712 | verbose(env, "R%d is %s invalid negative access: off=%d\n", |
| 3713 | regno, tname, off); |
| 3714 | return -EACCES; |
| 3715 | } |
| 3716 | |
| 3717 | if (atype != BPF_READ) { |
| 3718 | verbose(env, "only read from %s is supported\n", tname); |
| 3719 | return -EACCES; |
| 3720 | } |
| 3721 | |
| 3722 | ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id); |
| 3723 | if (ret < 0) |
| 3724 | return ret; |
| 3725 | |
| 3726 | if (value_regno >= 0) |
| 3727 | mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id); |
| 3728 | |
| 3729 | return 0; |
| 3730 | } |
| 3731 | |
| 3732 | /* Check that the stack access at the given offset is within bounds. The |
| 3733 | * maximum valid offset is -1. |
| 3734 | * |
| 3735 | * The minimum valid offset is -MAX_BPF_STACK for writes, and |
| 3736 | * -state->allocated_stack for reads. |
| 3737 | */ |
| 3738 | static int check_stack_slot_within_bounds(int off, |
| 3739 | struct bpf_func_state *state, |
| 3740 | enum bpf_access_type t) |
| 3741 | { |
| 3742 | int min_valid_off; |
| 3743 | |
| 3744 | if (t == BPF_WRITE) |
| 3745 | min_valid_off = -MAX_BPF_STACK; |
| 3746 | else |
| 3747 | min_valid_off = -state->allocated_stack; |
| 3748 | |
| 3749 | if (off < min_valid_off || off > -1) |
| 3750 | return -EACCES; |
| 3751 | return 0; |
| 3752 | } |
| 3753 | |
| 3754 | /* Check that the stack access at 'regno + off' falls within the maximum stack |
| 3755 | * bounds. |
| 3756 | * |
| 3757 | * 'off' includes `regno->offset`, but not its dynamic part (if any). |
| 3758 | */ |
| 3759 | static int check_stack_access_within_bounds( |
| 3760 | struct bpf_verifier_env *env, |
| 3761 | int regno, int off, int access_size, |
| 3762 | enum stack_access_src src, enum bpf_access_type type) |
| 3763 | { |
| 3764 | struct bpf_reg_state *regs = cur_regs(env); |
| 3765 | struct bpf_reg_state *reg = regs + regno; |
| 3766 | struct bpf_func_state *state = func(env, reg); |
| 3767 | int min_off, max_off; |
| 3768 | int err; |
| 3769 | char *err_extra; |
| 3770 | |
| 3771 | if (src == ACCESS_HELPER) |
| 3772 | /* We don't know if helpers are reading or writing (or both). */ |
| 3773 | err_extra = " indirect access to"; |
| 3774 | else if (type == BPF_READ) |
| 3775 | err_extra = " read from"; |
| 3776 | else |
| 3777 | err_extra = " write to"; |
| 3778 | |
| 3779 | if (tnum_is_const(reg->var_off)) { |
| 3780 | min_off = reg->var_off.value + off; |
| 3781 | if (access_size > 0) |
| 3782 | max_off = min_off + access_size - 1; |
| 3783 | else |
| 3784 | max_off = min_off; |
| 3785 | } else { |
| 3786 | if (reg->smax_value >= BPF_MAX_VAR_OFF || |
| 3787 | reg->smin_value <= -BPF_MAX_VAR_OFF) { |
| 3788 | verbose(env, "invalid unbounded variable-offset%s stack R%d\n", |
| 3789 | err_extra, regno); |
| 3790 | return -EACCES; |
| 3791 | } |
| 3792 | min_off = reg->smin_value + off; |
| 3793 | if (access_size > 0) |
| 3794 | max_off = reg->smax_value + off + access_size - 1; |
| 3795 | else |
| 3796 | max_off = min_off; |
| 3797 | } |
| 3798 | |
| 3799 | err = check_stack_slot_within_bounds(min_off, state, type); |
| 3800 | if (!err) |
| 3801 | err = check_stack_slot_within_bounds(max_off, state, type); |
| 3802 | |
| 3803 | if (err) { |
| 3804 | if (tnum_is_const(reg->var_off)) { |
| 3805 | verbose(env, "invalid%s stack R%d off=%d size=%d\n", |
| 3806 | err_extra, regno, off, access_size); |
| 3807 | } else { |
| 3808 | char tn_buf[48]; |
| 3809 | |
| 3810 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 3811 | verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", |
| 3812 | err_extra, regno, tn_buf, access_size); |
| 3813 | } |
| 3814 | } |
| 3815 | return err; |
| 3816 | } |
| 3817 | |
| 3818 | /* check whether memory at (regno + off) is accessible for t = (read | write) |
| 3819 | * if t==write, value_regno is a register which value is stored into memory |
| 3820 | * if t==read, value_regno is a register which will receive the value from memory |
| 3821 | * if t==write && value_regno==-1, some unknown value is stored into memory |
| 3822 | * if t==read && value_regno==-1, don't care what we read from memory |
| 3823 | */ |
| 3824 | static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, |
| 3825 | int off, int bpf_size, enum bpf_access_type t, |
| 3826 | int value_regno, bool strict_alignment_once) |
| 3827 | { |
| 3828 | struct bpf_reg_state *regs = cur_regs(env); |
| 3829 | struct bpf_reg_state *reg = regs + regno; |
| 3830 | struct bpf_func_state *state; |
| 3831 | int size, err = 0; |
| 3832 | |
| 3833 | size = bpf_size_to_bytes(bpf_size); |
| 3834 | if (size < 0) |
| 3835 | return size; |
| 3836 | |
| 3837 | /* alignment checks will add in reg->off themselves */ |
| 3838 | err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); |
| 3839 | if (err) |
| 3840 | return err; |
| 3841 | |
| 3842 | /* for access checks, reg->off is just part of off */ |
| 3843 | off += reg->off; |
| 3844 | |
| 3845 | if (reg->type == PTR_TO_MAP_VALUE) { |
| 3846 | if (t == BPF_WRITE && value_regno >= 0 && |
| 3847 | is_pointer_value(env, value_regno)) { |
| 3848 | verbose(env, "R%d leaks addr into map\n", value_regno); |
| 3849 | return -EACCES; |
| 3850 | } |
| 3851 | err = check_map_access_type(env, regno, off, size, t); |
| 3852 | if (err) |
| 3853 | return err; |
| 3854 | err = check_map_access(env, regno, off, size, false); |
| 3855 | if (!err && t == BPF_READ && value_regno >= 0) { |
| 3856 | struct bpf_map *map = reg->map_ptr; |
| 3857 | |
| 3858 | /* if map is read-only, track its contents as scalars */ |
| 3859 | if (tnum_is_const(reg->var_off) && |
| 3860 | bpf_map_is_rdonly(map) && |
| 3861 | map->ops->map_direct_value_addr) { |
| 3862 | int map_off = off + reg->var_off.value; |
| 3863 | u64 val = 0; |
| 3864 | |
| 3865 | err = bpf_map_direct_read(map, map_off, size, |
| 3866 | &val); |
| 3867 | if (err) |
| 3868 | return err; |
| 3869 | |
| 3870 | regs[value_regno].type = SCALAR_VALUE; |
| 3871 | __mark_reg_known(®s[value_regno], val); |
| 3872 | } else { |
| 3873 | mark_reg_unknown(env, regs, value_regno); |
| 3874 | } |
| 3875 | } |
| 3876 | } else if (reg->type == PTR_TO_MEM) { |
| 3877 | if (t == BPF_WRITE && value_regno >= 0 && |
| 3878 | is_pointer_value(env, value_regno)) { |
| 3879 | verbose(env, "R%d leaks addr into mem\n", value_regno); |
| 3880 | return -EACCES; |
| 3881 | } |
| 3882 | err = check_mem_region_access(env, regno, off, size, |
| 3883 | reg->mem_size, false); |
| 3884 | if (!err && t == BPF_READ && value_regno >= 0) |
| 3885 | mark_reg_unknown(env, regs, value_regno); |
| 3886 | } else if (reg->type == PTR_TO_CTX) { |
| 3887 | enum bpf_reg_type reg_type = SCALAR_VALUE; |
| 3888 | struct btf *btf = NULL; |
| 3889 | u32 btf_id = 0; |
| 3890 | |
| 3891 | if (t == BPF_WRITE && value_regno >= 0 && |
| 3892 | is_pointer_value(env, value_regno)) { |
| 3893 | verbose(env, "R%d leaks addr into ctx\n", value_regno); |
| 3894 | return -EACCES; |
| 3895 | } |
| 3896 | |
| 3897 | err = check_ctx_reg(env, reg, regno); |
| 3898 | if (err < 0) |
| 3899 | return err; |
| 3900 | |
| 3901 | err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf, &btf_id); |
| 3902 | if (err) |
| 3903 | verbose_linfo(env, insn_idx, "; "); |
| 3904 | if (!err && t == BPF_READ && value_regno >= 0) { |
| 3905 | /* ctx access returns either a scalar, or a |
| 3906 | * PTR_TO_PACKET[_META,_END]. In the latter |
| 3907 | * case, we know the offset is zero. |
| 3908 | */ |
| 3909 | if (reg_type == SCALAR_VALUE) { |
| 3910 | mark_reg_unknown(env, regs, value_regno); |
| 3911 | } else { |
| 3912 | mark_reg_known_zero(env, regs, |
| 3913 | value_regno); |
| 3914 | if (reg_type_may_be_null(reg_type)) |
| 3915 | regs[value_regno].id = ++env->id_gen; |
| 3916 | /* A load of ctx field could have different |
| 3917 | * actual load size with the one encoded in the |
| 3918 | * insn. When the dst is PTR, it is for sure not |
| 3919 | * a sub-register. |
| 3920 | */ |
| 3921 | regs[value_regno].subreg_def = DEF_NOT_SUBREG; |
| 3922 | if (reg_type == PTR_TO_BTF_ID || |
| 3923 | reg_type == PTR_TO_BTF_ID_OR_NULL) { |
| 3924 | regs[value_regno].btf = btf; |
| 3925 | regs[value_regno].btf_id = btf_id; |
| 3926 | } |
| 3927 | } |
| 3928 | regs[value_regno].type = reg_type; |
| 3929 | } |
| 3930 | |
| 3931 | } else if (reg->type == PTR_TO_STACK) { |
| 3932 | /* Basic bounds checks. */ |
| 3933 | err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t); |
| 3934 | if (err) |
| 3935 | return err; |
| 3936 | |
| 3937 | state = func(env, reg); |
| 3938 | err = update_stack_depth(env, state, off); |
| 3939 | if (err) |
| 3940 | return err; |
| 3941 | |
| 3942 | if (t == BPF_READ) |
| 3943 | err = check_stack_read(env, regno, off, size, |
| 3944 | value_regno); |
| 3945 | else |
| 3946 | err = check_stack_write(env, regno, off, size, |
| 3947 | value_regno, insn_idx); |
| 3948 | } else if (reg_is_pkt_pointer(reg)) { |
| 3949 | if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { |
| 3950 | verbose(env, "cannot write into packet\n"); |
| 3951 | return -EACCES; |
| 3952 | } |
| 3953 | if (t == BPF_WRITE && value_regno >= 0 && |
| 3954 | is_pointer_value(env, value_regno)) { |
| 3955 | verbose(env, "R%d leaks addr into packet\n", |
| 3956 | value_regno); |
| 3957 | return -EACCES; |
| 3958 | } |
| 3959 | err = check_packet_access(env, regno, off, size, false); |
| 3960 | if (!err && t == BPF_READ && value_regno >= 0) |
| 3961 | mark_reg_unknown(env, regs, value_regno); |
| 3962 | } else if (reg->type == PTR_TO_FLOW_KEYS) { |
| 3963 | if (t == BPF_WRITE && value_regno >= 0 && |
| 3964 | is_pointer_value(env, value_regno)) { |
| 3965 | verbose(env, "R%d leaks addr into flow keys\n", |
| 3966 | value_regno); |
| 3967 | return -EACCES; |
| 3968 | } |
| 3969 | |
| 3970 | err = check_flow_keys_access(env, off, size); |
| 3971 | if (!err && t == BPF_READ && value_regno >= 0) |
| 3972 | mark_reg_unknown(env, regs, value_regno); |
| 3973 | } else if (type_is_sk_pointer(reg->type)) { |
| 3974 | if (t == BPF_WRITE) { |
| 3975 | verbose(env, "R%d cannot write into %s\n", |
| 3976 | regno, reg_type_str[reg->type]); |
| 3977 | return -EACCES; |
| 3978 | } |
| 3979 | err = check_sock_access(env, insn_idx, regno, off, size, t); |
| 3980 | if (!err && value_regno >= 0) |
| 3981 | mark_reg_unknown(env, regs, value_regno); |
| 3982 | } else if (reg->type == PTR_TO_TP_BUFFER) { |
| 3983 | err = check_tp_buffer_access(env, reg, regno, off, size); |
| 3984 | if (!err && t == BPF_READ && value_regno >= 0) |
| 3985 | mark_reg_unknown(env, regs, value_regno); |
| 3986 | } else if (reg->type == PTR_TO_BTF_ID) { |
| 3987 | err = check_ptr_to_btf_access(env, regs, regno, off, size, t, |
| 3988 | value_regno); |
| 3989 | } else if (reg->type == CONST_PTR_TO_MAP) { |
| 3990 | err = check_ptr_to_map_access(env, regs, regno, off, size, t, |
| 3991 | value_regno); |
| 3992 | } else if (reg->type == PTR_TO_RDONLY_BUF) { |
| 3993 | if (t == BPF_WRITE) { |
| 3994 | verbose(env, "R%d cannot write into %s\n", |
| 3995 | regno, reg_type_str[reg->type]); |
| 3996 | return -EACCES; |
| 3997 | } |
| 3998 | err = check_buffer_access(env, reg, regno, off, size, false, |
| 3999 | "rdonly", |
| 4000 | &env->prog->aux->max_rdonly_access); |
| 4001 | if (!err && value_regno >= 0) |
| 4002 | mark_reg_unknown(env, regs, value_regno); |
| 4003 | } else if (reg->type == PTR_TO_RDWR_BUF) { |
| 4004 | err = check_buffer_access(env, reg, regno, off, size, false, |
| 4005 | "rdwr", |
| 4006 | &env->prog->aux->max_rdwr_access); |
| 4007 | if (!err && t == BPF_READ && value_regno >= 0) |
| 4008 | mark_reg_unknown(env, regs, value_regno); |
| 4009 | } else { |
| 4010 | verbose(env, "R%d invalid mem access '%s'\n", regno, |
| 4011 | reg_type_str[reg->type]); |
| 4012 | return -EACCES; |
| 4013 | } |
| 4014 | |
| 4015 | if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && |
| 4016 | regs[value_regno].type == SCALAR_VALUE) { |
| 4017 | /* b/h/w load zero-extends, mark upper bits as known 0 */ |
| 4018 | coerce_reg_to_size(®s[value_regno], size); |
| 4019 | } |
| 4020 | return err; |
| 4021 | } |
| 4022 | |
| 4023 | static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) |
| 4024 | { |
| 4025 | int load_reg; |
| 4026 | int err; |
| 4027 | |
| 4028 | switch (insn->imm) { |
| 4029 | case BPF_ADD: |
| 4030 | case BPF_ADD | BPF_FETCH: |
| 4031 | case BPF_AND: |
| 4032 | case BPF_AND | BPF_FETCH: |
| 4033 | case BPF_OR: |
| 4034 | case BPF_OR | BPF_FETCH: |
| 4035 | case BPF_XOR: |
| 4036 | case BPF_XOR | BPF_FETCH: |
| 4037 | case BPF_XCHG: |
| 4038 | case BPF_CMPXCHG: |
| 4039 | break; |
| 4040 | default: |
| 4041 | verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm); |
| 4042 | return -EINVAL; |
| 4043 | } |
| 4044 | |
| 4045 | if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) { |
| 4046 | verbose(env, "invalid atomic operand size\n"); |
| 4047 | return -EINVAL; |
| 4048 | } |
| 4049 | |
| 4050 | /* check src1 operand */ |
| 4051 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
| 4052 | if (err) |
| 4053 | return err; |
| 4054 | |
| 4055 | /* check src2 operand */ |
| 4056 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
| 4057 | if (err) |
| 4058 | return err; |
| 4059 | |
| 4060 | if (insn->imm == BPF_CMPXCHG) { |
| 4061 | /* Check comparison of R0 with memory location */ |
| 4062 | err = check_reg_arg(env, BPF_REG_0, SRC_OP); |
| 4063 | if (err) |
| 4064 | return err; |
| 4065 | } |
| 4066 | |
| 4067 | if (is_pointer_value(env, insn->src_reg)) { |
| 4068 | verbose(env, "R%d leaks addr into mem\n", insn->src_reg); |
| 4069 | return -EACCES; |
| 4070 | } |
| 4071 | |
| 4072 | if (is_ctx_reg(env, insn->dst_reg) || |
| 4073 | is_pkt_reg(env, insn->dst_reg) || |
| 4074 | is_flow_key_reg(env, insn->dst_reg) || |
| 4075 | is_sk_reg(env, insn->dst_reg)) { |
| 4076 | verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n", |
| 4077 | insn->dst_reg, |
| 4078 | reg_type_str[reg_state(env, insn->dst_reg)->type]); |
| 4079 | return -EACCES; |
| 4080 | } |
| 4081 | |
| 4082 | if (insn->imm & BPF_FETCH) { |
| 4083 | if (insn->imm == BPF_CMPXCHG) |
| 4084 | load_reg = BPF_REG_0; |
| 4085 | else |
| 4086 | load_reg = insn->src_reg; |
| 4087 | |
| 4088 | /* check and record load of old value */ |
| 4089 | err = check_reg_arg(env, load_reg, DST_OP); |
| 4090 | if (err) |
| 4091 | return err; |
| 4092 | } else { |
| 4093 | /* This instruction accesses a memory location but doesn't |
| 4094 | * actually load it into a register. |
| 4095 | */ |
| 4096 | load_reg = -1; |
| 4097 | } |
| 4098 | |
| 4099 | /* check whether we can read the memory */ |
| 4100 | err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, |
| 4101 | BPF_SIZE(insn->code), BPF_READ, load_reg, true); |
| 4102 | if (err) |
| 4103 | return err; |
| 4104 | |
| 4105 | /* check whether we can write into the same memory */ |
| 4106 | err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, |
| 4107 | BPF_SIZE(insn->code), BPF_WRITE, -1, true); |
| 4108 | if (err) |
| 4109 | return err; |
| 4110 | |
| 4111 | return 0; |
| 4112 | } |
| 4113 | |
| 4114 | /* When register 'regno' is used to read the stack (either directly or through |
| 4115 | * a helper function) make sure that it's within stack boundary and, depending |
| 4116 | * on the access type, that all elements of the stack are initialized. |
| 4117 | * |
| 4118 | * 'off' includes 'regno->off', but not its dynamic part (if any). |
| 4119 | * |
| 4120 | * All registers that have been spilled on the stack in the slots within the |
| 4121 | * read offsets are marked as read. |
| 4122 | */ |
| 4123 | static int check_stack_range_initialized( |
| 4124 | struct bpf_verifier_env *env, int regno, int off, |
| 4125 | int access_size, bool zero_size_allowed, |
| 4126 | enum stack_access_src type, struct bpf_call_arg_meta *meta) |
| 4127 | { |
| 4128 | struct bpf_reg_state *reg = reg_state(env, regno); |
| 4129 | struct bpf_func_state *state = func(env, reg); |
| 4130 | int err, min_off, max_off, i, j, slot, spi; |
| 4131 | char *err_extra = type == ACCESS_HELPER ? " indirect" : ""; |
| 4132 | enum bpf_access_type bounds_check_type; |
| 4133 | /* Some accesses can write anything into the stack, others are |
| 4134 | * read-only. |
| 4135 | */ |
| 4136 | bool clobber = false; |
| 4137 | |
| 4138 | if (access_size == 0 && !zero_size_allowed) { |
| 4139 | verbose(env, "invalid zero-sized read\n"); |
| 4140 | return -EACCES; |
| 4141 | } |
| 4142 | |
| 4143 | if (type == ACCESS_HELPER) { |
| 4144 | /* The bounds checks for writes are more permissive than for |
| 4145 | * reads. However, if raw_mode is not set, we'll do extra |
| 4146 | * checks below. |
| 4147 | */ |
| 4148 | bounds_check_type = BPF_WRITE; |
| 4149 | clobber = true; |
| 4150 | } else { |
| 4151 | bounds_check_type = BPF_READ; |
| 4152 | } |
| 4153 | err = check_stack_access_within_bounds(env, regno, off, access_size, |
| 4154 | type, bounds_check_type); |
| 4155 | if (err) |
| 4156 | return err; |
| 4157 | |
| 4158 | |
| 4159 | if (tnum_is_const(reg->var_off)) { |
| 4160 | min_off = max_off = reg->var_off.value + off; |
| 4161 | } else { |
| 4162 | /* Variable offset is prohibited for unprivileged mode for |
| 4163 | * simplicity since it requires corresponding support in |
| 4164 | * Spectre masking for stack ALU. |
| 4165 | * See also retrieve_ptr_limit(). |
| 4166 | */ |
| 4167 | if (!env->bypass_spec_v1) { |
| 4168 | char tn_buf[48]; |
| 4169 | |
| 4170 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 4171 | verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", |
| 4172 | regno, err_extra, tn_buf); |
| 4173 | return -EACCES; |
| 4174 | } |
| 4175 | /* Only initialized buffer on stack is allowed to be accessed |
| 4176 | * with variable offset. With uninitialized buffer it's hard to |
| 4177 | * guarantee that whole memory is marked as initialized on |
| 4178 | * helper return since specific bounds are unknown what may |
| 4179 | * cause uninitialized stack leaking. |
| 4180 | */ |
| 4181 | if (meta && meta->raw_mode) |
| 4182 | meta = NULL; |
| 4183 | |
| 4184 | min_off = reg->smin_value + off; |
| 4185 | max_off = reg->smax_value + off; |
| 4186 | } |
| 4187 | |
| 4188 | if (meta && meta->raw_mode) { |
| 4189 | meta->access_size = access_size; |
| 4190 | meta->regno = regno; |
| 4191 | return 0; |
| 4192 | } |
| 4193 | |
| 4194 | for (i = min_off; i < max_off + access_size; i++) { |
| 4195 | u8 *stype; |
| 4196 | |
| 4197 | slot = -i - 1; |
| 4198 | spi = slot / BPF_REG_SIZE; |
| 4199 | if (state->allocated_stack <= slot) |
| 4200 | goto err; |
| 4201 | stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; |
| 4202 | if (*stype == STACK_MISC) |
| 4203 | goto mark; |
| 4204 | if (*stype == STACK_ZERO) { |
| 4205 | if (clobber) { |
| 4206 | /* helper can write anything into the stack */ |
| 4207 | *stype = STACK_MISC; |
| 4208 | } |
| 4209 | goto mark; |
| 4210 | } |
| 4211 | |
| 4212 | if (state->stack[spi].slot_type[0] == STACK_SPILL && |
| 4213 | state->stack[spi].spilled_ptr.type == PTR_TO_BTF_ID) |
| 4214 | goto mark; |
| 4215 | |
| 4216 | if (state->stack[spi].slot_type[0] == STACK_SPILL && |
| 4217 | (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || |
| 4218 | env->allow_ptr_leaks)) { |
| 4219 | if (clobber) { |
| 4220 | __mark_reg_unknown(env, &state->stack[spi].spilled_ptr); |
| 4221 | for (j = 0; j < BPF_REG_SIZE; j++) |
| 4222 | state->stack[spi].slot_type[j] = STACK_MISC; |
| 4223 | } |
| 4224 | goto mark; |
| 4225 | } |
| 4226 | |
| 4227 | err: |
| 4228 | if (tnum_is_const(reg->var_off)) { |
| 4229 | verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", |
| 4230 | err_extra, regno, min_off, i - min_off, access_size); |
| 4231 | } else { |
| 4232 | char tn_buf[48]; |
| 4233 | |
| 4234 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 4235 | verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", |
| 4236 | err_extra, regno, tn_buf, i - min_off, access_size); |
| 4237 | } |
| 4238 | return -EACCES; |
| 4239 | mark: |
| 4240 | /* reading any byte out of 8-byte 'spill_slot' will cause |
| 4241 | * the whole slot to be marked as 'read' |
| 4242 | */ |
| 4243 | mark_reg_read(env, &state->stack[spi].spilled_ptr, |
| 4244 | state->stack[spi].spilled_ptr.parent, |
| 4245 | REG_LIVE_READ64); |
| 4246 | } |
| 4247 | return update_stack_depth(env, state, min_off); |
| 4248 | } |
| 4249 | |
| 4250 | static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, |
| 4251 | int access_size, bool zero_size_allowed, |
| 4252 | struct bpf_call_arg_meta *meta) |
| 4253 | { |
| 4254 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 4255 | |
| 4256 | switch (reg->type) { |
| 4257 | case PTR_TO_PACKET: |
| 4258 | case PTR_TO_PACKET_META: |
| 4259 | return check_packet_access(env, regno, reg->off, access_size, |
| 4260 | zero_size_allowed); |
| 4261 | case PTR_TO_MAP_VALUE: |
| 4262 | if (check_map_access_type(env, regno, reg->off, access_size, |
| 4263 | meta && meta->raw_mode ? BPF_WRITE : |
| 4264 | BPF_READ)) |
| 4265 | return -EACCES; |
| 4266 | return check_map_access(env, regno, reg->off, access_size, |
| 4267 | zero_size_allowed); |
| 4268 | case PTR_TO_MEM: |
| 4269 | return check_mem_region_access(env, regno, reg->off, |
| 4270 | access_size, reg->mem_size, |
| 4271 | zero_size_allowed); |
| 4272 | case PTR_TO_RDONLY_BUF: |
| 4273 | if (meta && meta->raw_mode) |
| 4274 | return -EACCES; |
| 4275 | return check_buffer_access(env, reg, regno, reg->off, |
| 4276 | access_size, zero_size_allowed, |
| 4277 | "rdonly", |
| 4278 | &env->prog->aux->max_rdonly_access); |
| 4279 | case PTR_TO_RDWR_BUF: |
| 4280 | return check_buffer_access(env, reg, regno, reg->off, |
| 4281 | access_size, zero_size_allowed, |
| 4282 | "rdwr", |
| 4283 | &env->prog->aux->max_rdwr_access); |
| 4284 | case PTR_TO_STACK: |
| 4285 | return check_stack_range_initialized( |
| 4286 | env, |
| 4287 | regno, reg->off, access_size, |
| 4288 | zero_size_allowed, ACCESS_HELPER, meta); |
| 4289 | default: /* scalar_value or invalid ptr */ |
| 4290 | /* Allow zero-byte read from NULL, regardless of pointer type */ |
| 4291 | if (zero_size_allowed && access_size == 0 && |
| 4292 | register_is_null(reg)) |
| 4293 | return 0; |
| 4294 | |
| 4295 | verbose(env, "R%d type=%s expected=%s\n", regno, |
| 4296 | reg_type_str[reg->type], |
| 4297 | reg_type_str[PTR_TO_STACK]); |
| 4298 | return -EACCES; |
| 4299 | } |
| 4300 | } |
| 4301 | |
| 4302 | int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, |
| 4303 | u32 regno, u32 mem_size) |
| 4304 | { |
| 4305 | if (register_is_null(reg)) |
| 4306 | return 0; |
| 4307 | |
| 4308 | if (reg_type_may_be_null(reg->type)) { |
| 4309 | /* Assuming that the register contains a value check if the memory |
| 4310 | * access is safe. Temporarily save and restore the register's state as |
| 4311 | * the conversion shouldn't be visible to a caller. |
| 4312 | */ |
| 4313 | const struct bpf_reg_state saved_reg = *reg; |
| 4314 | int rv; |
| 4315 | |
| 4316 | mark_ptr_not_null_reg(reg); |
| 4317 | rv = check_helper_mem_access(env, regno, mem_size, true, NULL); |
| 4318 | *reg = saved_reg; |
| 4319 | return rv; |
| 4320 | } |
| 4321 | |
| 4322 | return check_helper_mem_access(env, regno, mem_size, true, NULL); |
| 4323 | } |
| 4324 | |
| 4325 | /* Implementation details: |
| 4326 | * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL |
| 4327 | * Two bpf_map_lookups (even with the same key) will have different reg->id. |
| 4328 | * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after |
| 4329 | * value_or_null->value transition, since the verifier only cares about |
| 4330 | * the range of access to valid map value pointer and doesn't care about actual |
| 4331 | * address of the map element. |
| 4332 | * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps |
| 4333 | * reg->id > 0 after value_or_null->value transition. By doing so |
| 4334 | * two bpf_map_lookups will be considered two different pointers that |
| 4335 | * point to different bpf_spin_locks. |
| 4336 | * The verifier allows taking only one bpf_spin_lock at a time to avoid |
| 4337 | * dead-locks. |
| 4338 | * Since only one bpf_spin_lock is allowed the checks are simpler than |
| 4339 | * reg_is_refcounted() logic. The verifier needs to remember only |
| 4340 | * one spin_lock instead of array of acquired_refs. |
| 4341 | * cur_state->active_spin_lock remembers which map value element got locked |
| 4342 | * and clears it after bpf_spin_unlock. |
| 4343 | */ |
| 4344 | static int process_spin_lock(struct bpf_verifier_env *env, int regno, |
| 4345 | bool is_lock) |
| 4346 | { |
| 4347 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 4348 | struct bpf_verifier_state *cur = env->cur_state; |
| 4349 | bool is_const = tnum_is_const(reg->var_off); |
| 4350 | struct bpf_map *map = reg->map_ptr; |
| 4351 | u64 val = reg->var_off.value; |
| 4352 | |
| 4353 | if (!is_const) { |
| 4354 | verbose(env, |
| 4355 | "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", |
| 4356 | regno); |
| 4357 | return -EINVAL; |
| 4358 | } |
| 4359 | if (!map->btf) { |
| 4360 | verbose(env, |
| 4361 | "map '%s' has to have BTF in order to use bpf_spin_lock\n", |
| 4362 | map->name); |
| 4363 | return -EINVAL; |
| 4364 | } |
| 4365 | if (!map_value_has_spin_lock(map)) { |
| 4366 | if (map->spin_lock_off == -E2BIG) |
| 4367 | verbose(env, |
| 4368 | "map '%s' has more than one 'struct bpf_spin_lock'\n", |
| 4369 | map->name); |
| 4370 | else if (map->spin_lock_off == -ENOENT) |
| 4371 | verbose(env, |
| 4372 | "map '%s' doesn't have 'struct bpf_spin_lock'\n", |
| 4373 | map->name); |
| 4374 | else |
| 4375 | verbose(env, |
| 4376 | "map '%s' is not a struct type or bpf_spin_lock is mangled\n", |
| 4377 | map->name); |
| 4378 | return -EINVAL; |
| 4379 | } |
| 4380 | if (map->spin_lock_off != val + reg->off) { |
| 4381 | verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", |
| 4382 | val + reg->off); |
| 4383 | return -EINVAL; |
| 4384 | } |
| 4385 | if (is_lock) { |
| 4386 | if (cur->active_spin_lock) { |
| 4387 | verbose(env, |
| 4388 | "Locking two bpf_spin_locks are not allowed\n"); |
| 4389 | return -EINVAL; |
| 4390 | } |
| 4391 | cur->active_spin_lock = reg->id; |
| 4392 | } else { |
| 4393 | if (!cur->active_spin_lock) { |
| 4394 | verbose(env, "bpf_spin_unlock without taking a lock\n"); |
| 4395 | return -EINVAL; |
| 4396 | } |
| 4397 | if (cur->active_spin_lock != reg->id) { |
| 4398 | verbose(env, "bpf_spin_unlock of different lock\n"); |
| 4399 | return -EINVAL; |
| 4400 | } |
| 4401 | cur->active_spin_lock = 0; |
| 4402 | } |
| 4403 | return 0; |
| 4404 | } |
| 4405 | |
| 4406 | static bool arg_type_is_mem_ptr(enum bpf_arg_type type) |
| 4407 | { |
| 4408 | return type == ARG_PTR_TO_MEM || |
| 4409 | type == ARG_PTR_TO_MEM_OR_NULL || |
| 4410 | type == ARG_PTR_TO_UNINIT_MEM; |
| 4411 | } |
| 4412 | |
| 4413 | static bool arg_type_is_mem_size(enum bpf_arg_type type) |
| 4414 | { |
| 4415 | return type == ARG_CONST_SIZE || |
| 4416 | type == ARG_CONST_SIZE_OR_ZERO; |
| 4417 | } |
| 4418 | |
| 4419 | static bool arg_type_is_alloc_size(enum bpf_arg_type type) |
| 4420 | { |
| 4421 | return type == ARG_CONST_ALLOC_SIZE_OR_ZERO; |
| 4422 | } |
| 4423 | |
| 4424 | static bool arg_type_is_int_ptr(enum bpf_arg_type type) |
| 4425 | { |
| 4426 | return type == ARG_PTR_TO_INT || |
| 4427 | type == ARG_PTR_TO_LONG; |
| 4428 | } |
| 4429 | |
| 4430 | static int int_ptr_type_to_size(enum bpf_arg_type type) |
| 4431 | { |
| 4432 | if (type == ARG_PTR_TO_INT) |
| 4433 | return sizeof(u32); |
| 4434 | else if (type == ARG_PTR_TO_LONG) |
| 4435 | return sizeof(u64); |
| 4436 | |
| 4437 | return -EINVAL; |
| 4438 | } |
| 4439 | |
| 4440 | static int resolve_map_arg_type(struct bpf_verifier_env *env, |
| 4441 | const struct bpf_call_arg_meta *meta, |
| 4442 | enum bpf_arg_type *arg_type) |
| 4443 | { |
| 4444 | if (!meta->map_ptr) { |
| 4445 | /* kernel subsystem misconfigured verifier */ |
| 4446 | verbose(env, "invalid map_ptr to access map->type\n"); |
| 4447 | return -EACCES; |
| 4448 | } |
| 4449 | |
| 4450 | switch (meta->map_ptr->map_type) { |
| 4451 | case BPF_MAP_TYPE_SOCKMAP: |
| 4452 | case BPF_MAP_TYPE_SOCKHASH: |
| 4453 | if (*arg_type == ARG_PTR_TO_MAP_VALUE) { |
| 4454 | *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON; |
| 4455 | } else { |
| 4456 | verbose(env, "invalid arg_type for sockmap/sockhash\n"); |
| 4457 | return -EINVAL; |
| 4458 | } |
| 4459 | break; |
| 4460 | |
| 4461 | default: |
| 4462 | break; |
| 4463 | } |
| 4464 | return 0; |
| 4465 | } |
| 4466 | |
| 4467 | struct bpf_reg_types { |
| 4468 | const enum bpf_reg_type types[10]; |
| 4469 | u32 *btf_id; |
| 4470 | }; |
| 4471 | |
| 4472 | static const struct bpf_reg_types map_key_value_types = { |
| 4473 | .types = { |
| 4474 | PTR_TO_STACK, |
| 4475 | PTR_TO_PACKET, |
| 4476 | PTR_TO_PACKET_META, |
| 4477 | PTR_TO_MAP_VALUE, |
| 4478 | }, |
| 4479 | }; |
| 4480 | |
| 4481 | static const struct bpf_reg_types sock_types = { |
| 4482 | .types = { |
| 4483 | PTR_TO_SOCK_COMMON, |
| 4484 | PTR_TO_SOCKET, |
| 4485 | PTR_TO_TCP_SOCK, |
| 4486 | PTR_TO_XDP_SOCK, |
| 4487 | }, |
| 4488 | }; |
| 4489 | |
| 4490 | #ifdef CONFIG_NET |
| 4491 | static const struct bpf_reg_types btf_id_sock_common_types = { |
| 4492 | .types = { |
| 4493 | PTR_TO_SOCK_COMMON, |
| 4494 | PTR_TO_SOCKET, |
| 4495 | PTR_TO_TCP_SOCK, |
| 4496 | PTR_TO_XDP_SOCK, |
| 4497 | PTR_TO_BTF_ID, |
| 4498 | }, |
| 4499 | .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON], |
| 4500 | }; |
| 4501 | #endif |
| 4502 | |
| 4503 | static const struct bpf_reg_types mem_types = { |
| 4504 | .types = { |
| 4505 | PTR_TO_STACK, |
| 4506 | PTR_TO_PACKET, |
| 4507 | PTR_TO_PACKET_META, |
| 4508 | PTR_TO_MAP_VALUE, |
| 4509 | PTR_TO_MEM, |
| 4510 | PTR_TO_RDONLY_BUF, |
| 4511 | PTR_TO_RDWR_BUF, |
| 4512 | }, |
| 4513 | }; |
| 4514 | |
| 4515 | static const struct bpf_reg_types int_ptr_types = { |
| 4516 | .types = { |
| 4517 | PTR_TO_STACK, |
| 4518 | PTR_TO_PACKET, |
| 4519 | PTR_TO_PACKET_META, |
| 4520 | PTR_TO_MAP_VALUE, |
| 4521 | }, |
| 4522 | }; |
| 4523 | |
| 4524 | static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } }; |
| 4525 | static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } }; |
| 4526 | static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } }; |
| 4527 | static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; |
| 4528 | static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } }; |
| 4529 | static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } }; |
| 4530 | static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } }; |
| 4531 | static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } }; |
| 4532 | |
| 4533 | static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { |
| 4534 | [ARG_PTR_TO_MAP_KEY] = &map_key_value_types, |
| 4535 | [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types, |
| 4536 | [ARG_PTR_TO_UNINIT_MAP_VALUE] = &map_key_value_types, |
| 4537 | [ARG_PTR_TO_MAP_VALUE_OR_NULL] = &map_key_value_types, |
| 4538 | [ARG_CONST_SIZE] = &scalar_types, |
| 4539 | [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, |
| 4540 | [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, |
| 4541 | [ARG_CONST_MAP_PTR] = &const_map_ptr_types, |
| 4542 | [ARG_PTR_TO_CTX] = &context_types, |
| 4543 | [ARG_PTR_TO_CTX_OR_NULL] = &context_types, |
| 4544 | [ARG_PTR_TO_SOCK_COMMON] = &sock_types, |
| 4545 | #ifdef CONFIG_NET |
| 4546 | [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types, |
| 4547 | #endif |
| 4548 | [ARG_PTR_TO_SOCKET] = &fullsock_types, |
| 4549 | [ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types, |
| 4550 | [ARG_PTR_TO_BTF_ID] = &btf_ptr_types, |
| 4551 | [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types, |
| 4552 | [ARG_PTR_TO_MEM] = &mem_types, |
| 4553 | [ARG_PTR_TO_MEM_OR_NULL] = &mem_types, |
| 4554 | [ARG_PTR_TO_UNINIT_MEM] = &mem_types, |
| 4555 | [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types, |
| 4556 | [ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types, |
| 4557 | [ARG_PTR_TO_INT] = &int_ptr_types, |
| 4558 | [ARG_PTR_TO_LONG] = &int_ptr_types, |
| 4559 | [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types, |
| 4560 | }; |
| 4561 | |
| 4562 | static int check_reg_type(struct bpf_verifier_env *env, u32 regno, |
| 4563 | enum bpf_arg_type arg_type, |
| 4564 | const u32 *arg_btf_id) |
| 4565 | { |
| 4566 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 4567 | enum bpf_reg_type expected, type = reg->type; |
| 4568 | const struct bpf_reg_types *compatible; |
| 4569 | int i, j; |
| 4570 | |
| 4571 | compatible = compatible_reg_types[arg_type]; |
| 4572 | if (!compatible) { |
| 4573 | verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type); |
| 4574 | return -EFAULT; |
| 4575 | } |
| 4576 | |
| 4577 | for (i = 0; i < ARRAY_SIZE(compatible->types); i++) { |
| 4578 | expected = compatible->types[i]; |
| 4579 | if (expected == NOT_INIT) |
| 4580 | break; |
| 4581 | |
| 4582 | if (type == expected) |
| 4583 | goto found; |
| 4584 | } |
| 4585 | |
| 4586 | verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]); |
| 4587 | for (j = 0; j + 1 < i; j++) |
| 4588 | verbose(env, "%s, ", reg_type_str[compatible->types[j]]); |
| 4589 | verbose(env, "%s\n", reg_type_str[compatible->types[j]]); |
| 4590 | return -EACCES; |
| 4591 | |
| 4592 | found: |
| 4593 | if (type == PTR_TO_BTF_ID) { |
| 4594 | if (!arg_btf_id) { |
| 4595 | if (!compatible->btf_id) { |
| 4596 | verbose(env, "verifier internal error: missing arg compatible BTF ID\n"); |
| 4597 | return -EFAULT; |
| 4598 | } |
| 4599 | arg_btf_id = compatible->btf_id; |
| 4600 | } |
| 4601 | |
| 4602 | if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off, |
| 4603 | btf_vmlinux, *arg_btf_id)) { |
| 4604 | verbose(env, "R%d is of type %s but %s is expected\n", |
| 4605 | regno, kernel_type_name(reg->btf, reg->btf_id), |
| 4606 | kernel_type_name(btf_vmlinux, *arg_btf_id)); |
| 4607 | return -EACCES; |
| 4608 | } |
| 4609 | |
| 4610 | if (!tnum_is_const(reg->var_off) || reg->var_off.value) { |
| 4611 | verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n", |
| 4612 | regno); |
| 4613 | return -EACCES; |
| 4614 | } |
| 4615 | } |
| 4616 | |
| 4617 | return 0; |
| 4618 | } |
| 4619 | |
| 4620 | static int check_func_arg(struct bpf_verifier_env *env, u32 arg, |
| 4621 | struct bpf_call_arg_meta *meta, |
| 4622 | const struct bpf_func_proto *fn) |
| 4623 | { |
| 4624 | u32 regno = BPF_REG_1 + arg; |
| 4625 | struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; |
| 4626 | enum bpf_arg_type arg_type = fn->arg_type[arg]; |
| 4627 | enum bpf_reg_type type = reg->type; |
| 4628 | int err = 0; |
| 4629 | |
| 4630 | if (arg_type == ARG_DONTCARE) |
| 4631 | return 0; |
| 4632 | |
| 4633 | err = check_reg_arg(env, regno, SRC_OP); |
| 4634 | if (err) |
| 4635 | return err; |
| 4636 | |
| 4637 | if (arg_type == ARG_ANYTHING) { |
| 4638 | if (is_pointer_value(env, regno)) { |
| 4639 | verbose(env, "R%d leaks addr into helper function\n", |
| 4640 | regno); |
| 4641 | return -EACCES; |
| 4642 | } |
| 4643 | return 0; |
| 4644 | } |
| 4645 | |
| 4646 | if (type_is_pkt_pointer(type) && |
| 4647 | !may_access_direct_pkt_data(env, meta, BPF_READ)) { |
| 4648 | verbose(env, "helper access to the packet is not allowed\n"); |
| 4649 | return -EACCES; |
| 4650 | } |
| 4651 | |
| 4652 | if (arg_type == ARG_PTR_TO_MAP_VALUE || |
| 4653 | arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || |
| 4654 | arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { |
| 4655 | err = resolve_map_arg_type(env, meta, &arg_type); |
| 4656 | if (err) |
| 4657 | return err; |
| 4658 | } |
| 4659 | |
| 4660 | if (register_is_null(reg) && arg_type_may_be_null(arg_type)) |
| 4661 | /* A NULL register has a SCALAR_VALUE type, so skip |
| 4662 | * type checking. |
| 4663 | */ |
| 4664 | goto skip_type_check; |
| 4665 | |
| 4666 | err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]); |
| 4667 | if (err) |
| 4668 | return err; |
| 4669 | |
| 4670 | if (type == PTR_TO_CTX) { |
| 4671 | err = check_ctx_reg(env, reg, regno); |
| 4672 | if (err < 0) |
| 4673 | return err; |
| 4674 | } |
| 4675 | |
| 4676 | skip_type_check: |
| 4677 | if (reg->ref_obj_id) { |
| 4678 | if (meta->ref_obj_id) { |
| 4679 | verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", |
| 4680 | regno, reg->ref_obj_id, |
| 4681 | meta->ref_obj_id); |
| 4682 | return -EFAULT; |
| 4683 | } |
| 4684 | meta->ref_obj_id = reg->ref_obj_id; |
| 4685 | } |
| 4686 | |
| 4687 | if (arg_type == ARG_CONST_MAP_PTR) { |
| 4688 | /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ |
| 4689 | meta->map_ptr = reg->map_ptr; |
| 4690 | } else if (arg_type == ARG_PTR_TO_MAP_KEY) { |
| 4691 | /* bpf_map_xxx(..., map_ptr, ..., key) call: |
| 4692 | * check that [key, key + map->key_size) are within |
| 4693 | * stack limits and initialized |
| 4694 | */ |
| 4695 | if (!meta->map_ptr) { |
| 4696 | /* in function declaration map_ptr must come before |
| 4697 | * map_key, so that it's verified and known before |
| 4698 | * we have to check map_key here. Otherwise it means |
| 4699 | * that kernel subsystem misconfigured verifier |
| 4700 | */ |
| 4701 | verbose(env, "invalid map_ptr to access map->key\n"); |
| 4702 | return -EACCES; |
| 4703 | } |
| 4704 | err = check_helper_mem_access(env, regno, |
| 4705 | meta->map_ptr->key_size, false, |
| 4706 | NULL); |
| 4707 | } else if (arg_type == ARG_PTR_TO_MAP_VALUE || |
| 4708 | (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && |
| 4709 | !register_is_null(reg)) || |
| 4710 | arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { |
| 4711 | /* bpf_map_xxx(..., map_ptr, ..., value) call: |
| 4712 | * check [value, value + map->value_size) validity |
| 4713 | */ |
| 4714 | if (!meta->map_ptr) { |
| 4715 | /* kernel subsystem misconfigured verifier */ |
| 4716 | verbose(env, "invalid map_ptr to access map->value\n"); |
| 4717 | return -EACCES; |
| 4718 | } |
| 4719 | meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); |
| 4720 | err = check_helper_mem_access(env, regno, |
| 4721 | meta->map_ptr->value_size, false, |
| 4722 | meta); |
| 4723 | } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) { |
| 4724 | if (!reg->btf_id) { |
| 4725 | verbose(env, "Helper has invalid btf_id in R%d\n", regno); |
| 4726 | return -EACCES; |
| 4727 | } |
| 4728 | meta->ret_btf = reg->btf; |
| 4729 | meta->ret_btf_id = reg->btf_id; |
| 4730 | } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { |
| 4731 | if (meta->func_id == BPF_FUNC_spin_lock) { |
| 4732 | if (process_spin_lock(env, regno, true)) |
| 4733 | return -EACCES; |
| 4734 | } else if (meta->func_id == BPF_FUNC_spin_unlock) { |
| 4735 | if (process_spin_lock(env, regno, false)) |
| 4736 | return -EACCES; |
| 4737 | } else { |
| 4738 | verbose(env, "verifier internal error\n"); |
| 4739 | return -EFAULT; |
| 4740 | } |
| 4741 | } else if (arg_type_is_mem_ptr(arg_type)) { |
| 4742 | /* The access to this pointer is only checked when we hit the |
| 4743 | * next is_mem_size argument below. |
| 4744 | */ |
| 4745 | meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MEM); |
| 4746 | } else if (arg_type_is_mem_size(arg_type)) { |
| 4747 | bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); |
| 4748 | |
| 4749 | /* This is used to refine r0 return value bounds for helpers |
| 4750 | * that enforce this value as an upper bound on return values. |
| 4751 | * See do_refine_retval_range() for helpers that can refine |
| 4752 | * the return value. C type of helper is u32 so we pull register |
| 4753 | * bound from umax_value however, if negative verifier errors |
| 4754 | * out. Only upper bounds can be learned because retval is an |
| 4755 | * int type and negative retvals are allowed. |
| 4756 | */ |
| 4757 | meta->msize_max_value = reg->umax_value; |
| 4758 | |
| 4759 | /* The register is SCALAR_VALUE; the access check |
| 4760 | * happens using its boundaries. |
| 4761 | */ |
| 4762 | if (!tnum_is_const(reg->var_off)) |
| 4763 | /* For unprivileged variable accesses, disable raw |
| 4764 | * mode so that the program is required to |
| 4765 | * initialize all the memory that the helper could |
| 4766 | * just partially fill up. |
| 4767 | */ |
| 4768 | meta = NULL; |
| 4769 | |
| 4770 | if (reg->smin_value < 0) { |
| 4771 | verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", |
| 4772 | regno); |
| 4773 | return -EACCES; |
| 4774 | } |
| 4775 | |
| 4776 | if (reg->umin_value == 0) { |
| 4777 | err = check_helper_mem_access(env, regno - 1, 0, |
| 4778 | zero_size_allowed, |
| 4779 | meta); |
| 4780 | if (err) |
| 4781 | return err; |
| 4782 | } |
| 4783 | |
| 4784 | if (reg->umax_value >= BPF_MAX_VAR_SIZ) { |
| 4785 | verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", |
| 4786 | regno); |
| 4787 | return -EACCES; |
| 4788 | } |
| 4789 | err = check_helper_mem_access(env, regno - 1, |
| 4790 | reg->umax_value, |
| 4791 | zero_size_allowed, meta); |
| 4792 | if (!err) |
| 4793 | err = mark_chain_precision(env, regno); |
| 4794 | } else if (arg_type_is_alloc_size(arg_type)) { |
| 4795 | if (!tnum_is_const(reg->var_off)) { |
| 4796 | verbose(env, "R%d is not a known constant'\n", |
| 4797 | regno); |
| 4798 | return -EACCES; |
| 4799 | } |
| 4800 | meta->mem_size = reg->var_off.value; |
| 4801 | } else if (arg_type_is_int_ptr(arg_type)) { |
| 4802 | int size = int_ptr_type_to_size(arg_type); |
| 4803 | |
| 4804 | err = check_helper_mem_access(env, regno, size, false, meta); |
| 4805 | if (err) |
| 4806 | return err; |
| 4807 | err = check_ptr_alignment(env, reg, 0, size, true); |
| 4808 | } |
| 4809 | |
| 4810 | return err; |
| 4811 | } |
| 4812 | |
| 4813 | static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id) |
| 4814 | { |
| 4815 | enum bpf_attach_type eatype = env->prog->expected_attach_type; |
| 4816 | enum bpf_prog_type type = resolve_prog_type(env->prog); |
| 4817 | |
| 4818 | if (func_id != BPF_FUNC_map_update_elem) |
| 4819 | return false; |
| 4820 | |
| 4821 | /* It's not possible to get access to a locked struct sock in these |
| 4822 | * contexts, so updating is safe. |
| 4823 | */ |
| 4824 | switch (type) { |
| 4825 | case BPF_PROG_TYPE_TRACING: |
| 4826 | if (eatype == BPF_TRACE_ITER) |
| 4827 | return true; |
| 4828 | break; |
| 4829 | case BPF_PROG_TYPE_SOCKET_FILTER: |
| 4830 | case BPF_PROG_TYPE_SCHED_CLS: |
| 4831 | case BPF_PROG_TYPE_SCHED_ACT: |
| 4832 | case BPF_PROG_TYPE_XDP: |
| 4833 | case BPF_PROG_TYPE_SK_REUSEPORT: |
| 4834 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
| 4835 | case BPF_PROG_TYPE_SK_LOOKUP: |
| 4836 | return true; |
| 4837 | default: |
| 4838 | break; |
| 4839 | } |
| 4840 | |
| 4841 | verbose(env, "cannot update sockmap in this context\n"); |
| 4842 | return false; |
| 4843 | } |
| 4844 | |
| 4845 | static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env) |
| 4846 | { |
| 4847 | return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64); |
| 4848 | } |
| 4849 | |
| 4850 | static int check_map_func_compatibility(struct bpf_verifier_env *env, |
| 4851 | struct bpf_map *map, int func_id) |
| 4852 | { |
| 4853 | if (!map) |
| 4854 | return 0; |
| 4855 | |
| 4856 | /* We need a two way check, first is from map perspective ... */ |
| 4857 | switch (map->map_type) { |
| 4858 | case BPF_MAP_TYPE_PROG_ARRAY: |
| 4859 | if (func_id != BPF_FUNC_tail_call) |
| 4860 | goto error; |
| 4861 | break; |
| 4862 | case BPF_MAP_TYPE_PERF_EVENT_ARRAY: |
| 4863 | if (func_id != BPF_FUNC_perf_event_read && |
| 4864 | func_id != BPF_FUNC_perf_event_output && |
| 4865 | func_id != BPF_FUNC_skb_output && |
| 4866 | func_id != BPF_FUNC_perf_event_read_value && |
| 4867 | func_id != BPF_FUNC_xdp_output) |
| 4868 | goto error; |
| 4869 | break; |
| 4870 | case BPF_MAP_TYPE_RINGBUF: |
| 4871 | if (func_id != BPF_FUNC_ringbuf_output && |
| 4872 | func_id != BPF_FUNC_ringbuf_reserve && |
| 4873 | func_id != BPF_FUNC_ringbuf_submit && |
| 4874 | func_id != BPF_FUNC_ringbuf_discard && |
| 4875 | func_id != BPF_FUNC_ringbuf_query) |
| 4876 | goto error; |
| 4877 | break; |
| 4878 | case BPF_MAP_TYPE_STACK_TRACE: |
| 4879 | if (func_id != BPF_FUNC_get_stackid) |
| 4880 | goto error; |
| 4881 | break; |
| 4882 | case BPF_MAP_TYPE_CGROUP_ARRAY: |
| 4883 | if (func_id != BPF_FUNC_skb_under_cgroup && |
| 4884 | func_id != BPF_FUNC_current_task_under_cgroup) |
| 4885 | goto error; |
| 4886 | break; |
| 4887 | case BPF_MAP_TYPE_CGROUP_STORAGE: |
| 4888 | case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: |
| 4889 | if (func_id != BPF_FUNC_get_local_storage) |
| 4890 | goto error; |
| 4891 | break; |
| 4892 | case BPF_MAP_TYPE_DEVMAP: |
| 4893 | case BPF_MAP_TYPE_DEVMAP_HASH: |
| 4894 | if (func_id != BPF_FUNC_redirect_map && |
| 4895 | func_id != BPF_FUNC_map_lookup_elem) |
| 4896 | goto error; |
| 4897 | break; |
| 4898 | /* Restrict bpf side of cpumap and xskmap, open when use-cases |
| 4899 | * appear. |
| 4900 | */ |
| 4901 | case BPF_MAP_TYPE_CPUMAP: |
| 4902 | if (func_id != BPF_FUNC_redirect_map) |
| 4903 | goto error; |
| 4904 | break; |
| 4905 | case BPF_MAP_TYPE_XSKMAP: |
| 4906 | if (func_id != BPF_FUNC_redirect_map && |
| 4907 | func_id != BPF_FUNC_map_lookup_elem) |
| 4908 | goto error; |
| 4909 | break; |
| 4910 | case BPF_MAP_TYPE_ARRAY_OF_MAPS: |
| 4911 | case BPF_MAP_TYPE_HASH_OF_MAPS: |
| 4912 | if (func_id != BPF_FUNC_map_lookup_elem) |
| 4913 | goto error; |
| 4914 | break; |
| 4915 | case BPF_MAP_TYPE_SOCKMAP: |
| 4916 | if (func_id != BPF_FUNC_sk_redirect_map && |
| 4917 | func_id != BPF_FUNC_sock_map_update && |
| 4918 | func_id != BPF_FUNC_map_delete_elem && |
| 4919 | func_id != BPF_FUNC_msg_redirect_map && |
| 4920 | func_id != BPF_FUNC_sk_select_reuseport && |
| 4921 | func_id != BPF_FUNC_map_lookup_elem && |
| 4922 | !may_update_sockmap(env, func_id)) |
| 4923 | goto error; |
| 4924 | break; |
| 4925 | case BPF_MAP_TYPE_SOCKHASH: |
| 4926 | if (func_id != BPF_FUNC_sk_redirect_hash && |
| 4927 | func_id != BPF_FUNC_sock_hash_update && |
| 4928 | func_id != BPF_FUNC_map_delete_elem && |
| 4929 | func_id != BPF_FUNC_msg_redirect_hash && |
| 4930 | func_id != BPF_FUNC_sk_select_reuseport && |
| 4931 | func_id != BPF_FUNC_map_lookup_elem && |
| 4932 | !may_update_sockmap(env, func_id)) |
| 4933 | goto error; |
| 4934 | break; |
| 4935 | case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: |
| 4936 | if (func_id != BPF_FUNC_sk_select_reuseport) |
| 4937 | goto error; |
| 4938 | break; |
| 4939 | case BPF_MAP_TYPE_QUEUE: |
| 4940 | case BPF_MAP_TYPE_STACK: |
| 4941 | if (func_id != BPF_FUNC_map_peek_elem && |
| 4942 | func_id != BPF_FUNC_map_pop_elem && |
| 4943 | func_id != BPF_FUNC_map_push_elem) |
| 4944 | goto error; |
| 4945 | break; |
| 4946 | case BPF_MAP_TYPE_SK_STORAGE: |
| 4947 | if (func_id != BPF_FUNC_sk_storage_get && |
| 4948 | func_id != BPF_FUNC_sk_storage_delete) |
| 4949 | goto error; |
| 4950 | break; |
| 4951 | case BPF_MAP_TYPE_INODE_STORAGE: |
| 4952 | if (func_id != BPF_FUNC_inode_storage_get && |
| 4953 | func_id != BPF_FUNC_inode_storage_delete) |
| 4954 | goto error; |
| 4955 | break; |
| 4956 | case BPF_MAP_TYPE_TASK_STORAGE: |
| 4957 | if (func_id != BPF_FUNC_task_storage_get && |
| 4958 | func_id != BPF_FUNC_task_storage_delete) |
| 4959 | goto error; |
| 4960 | break; |
| 4961 | default: |
| 4962 | break; |
| 4963 | } |
| 4964 | |
| 4965 | /* ... and second from the function itself. */ |
| 4966 | switch (func_id) { |
| 4967 | case BPF_FUNC_tail_call: |
| 4968 | if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) |
| 4969 | goto error; |
| 4970 | if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) { |
| 4971 | verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); |
| 4972 | return -EINVAL; |
| 4973 | } |
| 4974 | break; |
| 4975 | case BPF_FUNC_perf_event_read: |
| 4976 | case BPF_FUNC_perf_event_output: |
| 4977 | case BPF_FUNC_perf_event_read_value: |
| 4978 | case BPF_FUNC_skb_output: |
| 4979 | case BPF_FUNC_xdp_output: |
| 4980 | if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) |
| 4981 | goto error; |
| 4982 | break; |
| 4983 | case BPF_FUNC_get_stackid: |
| 4984 | if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) |
| 4985 | goto error; |
| 4986 | break; |
| 4987 | case BPF_FUNC_current_task_under_cgroup: |
| 4988 | case BPF_FUNC_skb_under_cgroup: |
| 4989 | if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) |
| 4990 | goto error; |
| 4991 | break; |
| 4992 | case BPF_FUNC_redirect_map: |
| 4993 | if (map->map_type != BPF_MAP_TYPE_DEVMAP && |
| 4994 | map->map_type != BPF_MAP_TYPE_DEVMAP_HASH && |
| 4995 | map->map_type != BPF_MAP_TYPE_CPUMAP && |
| 4996 | map->map_type != BPF_MAP_TYPE_XSKMAP) |
| 4997 | goto error; |
| 4998 | break; |
| 4999 | case BPF_FUNC_sk_redirect_map: |
| 5000 | case BPF_FUNC_msg_redirect_map: |
| 5001 | case BPF_FUNC_sock_map_update: |
| 5002 | if (map->map_type != BPF_MAP_TYPE_SOCKMAP) |
| 5003 | goto error; |
| 5004 | break; |
| 5005 | case BPF_FUNC_sk_redirect_hash: |
| 5006 | case BPF_FUNC_msg_redirect_hash: |
| 5007 | case BPF_FUNC_sock_hash_update: |
| 5008 | if (map->map_type != BPF_MAP_TYPE_SOCKHASH) |
| 5009 | goto error; |
| 5010 | break; |
| 5011 | case BPF_FUNC_get_local_storage: |
| 5012 | if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && |
| 5013 | map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) |
| 5014 | goto error; |
| 5015 | break; |
| 5016 | case BPF_FUNC_sk_select_reuseport: |
| 5017 | if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && |
| 5018 | map->map_type != BPF_MAP_TYPE_SOCKMAP && |
| 5019 | map->map_type != BPF_MAP_TYPE_SOCKHASH) |
| 5020 | goto error; |
| 5021 | break; |
| 5022 | case BPF_FUNC_map_peek_elem: |
| 5023 | case BPF_FUNC_map_pop_elem: |
| 5024 | case BPF_FUNC_map_push_elem: |
| 5025 | if (map->map_type != BPF_MAP_TYPE_QUEUE && |
| 5026 | map->map_type != BPF_MAP_TYPE_STACK) |
| 5027 | goto error; |
| 5028 | break; |
| 5029 | case BPF_FUNC_sk_storage_get: |
| 5030 | case BPF_FUNC_sk_storage_delete: |
| 5031 | if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) |
| 5032 | goto error; |
| 5033 | break; |
| 5034 | case BPF_FUNC_inode_storage_get: |
| 5035 | case BPF_FUNC_inode_storage_delete: |
| 5036 | if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE) |
| 5037 | goto error; |
| 5038 | break; |
| 5039 | case BPF_FUNC_task_storage_get: |
| 5040 | case BPF_FUNC_task_storage_delete: |
| 5041 | if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) |
| 5042 | goto error; |
| 5043 | break; |
| 5044 | default: |
| 5045 | break; |
| 5046 | } |
| 5047 | |
| 5048 | return 0; |
| 5049 | error: |
| 5050 | verbose(env, "cannot pass map_type %d into func %s#%d\n", |
| 5051 | map->map_type, func_id_name(func_id), func_id); |
| 5052 | return -EINVAL; |
| 5053 | } |
| 5054 | |
| 5055 | static bool check_raw_mode_ok(const struct bpf_func_proto *fn) |
| 5056 | { |
| 5057 | int count = 0; |
| 5058 | |
| 5059 | if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) |
| 5060 | count++; |
| 5061 | if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) |
| 5062 | count++; |
| 5063 | if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) |
| 5064 | count++; |
| 5065 | if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) |
| 5066 | count++; |
| 5067 | if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) |
| 5068 | count++; |
| 5069 | |
| 5070 | /* We only support one arg being in raw mode at the moment, |
| 5071 | * which is sufficient for the helper functions we have |
| 5072 | * right now. |
| 5073 | */ |
| 5074 | return count <= 1; |
| 5075 | } |
| 5076 | |
| 5077 | static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, |
| 5078 | enum bpf_arg_type arg_next) |
| 5079 | { |
| 5080 | return (arg_type_is_mem_ptr(arg_curr) && |
| 5081 | !arg_type_is_mem_size(arg_next)) || |
| 5082 | (!arg_type_is_mem_ptr(arg_curr) && |
| 5083 | arg_type_is_mem_size(arg_next)); |
| 5084 | } |
| 5085 | |
| 5086 | static bool check_arg_pair_ok(const struct bpf_func_proto *fn) |
| 5087 | { |
| 5088 | /* bpf_xxx(..., buf, len) call will access 'len' |
| 5089 | * bytes from memory 'buf'. Both arg types need |
| 5090 | * to be paired, so make sure there's no buggy |
| 5091 | * helper function specification. |
| 5092 | */ |
| 5093 | if (arg_type_is_mem_size(fn->arg1_type) || |
| 5094 | arg_type_is_mem_ptr(fn->arg5_type) || |
| 5095 | check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || |
| 5096 | check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || |
| 5097 | check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || |
| 5098 | check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) |
| 5099 | return false; |
| 5100 | |
| 5101 | return true; |
| 5102 | } |
| 5103 | |
| 5104 | static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) |
| 5105 | { |
| 5106 | int count = 0; |
| 5107 | |
| 5108 | if (arg_type_may_be_refcounted(fn->arg1_type)) |
| 5109 | count++; |
| 5110 | if (arg_type_may_be_refcounted(fn->arg2_type)) |
| 5111 | count++; |
| 5112 | if (arg_type_may_be_refcounted(fn->arg3_type)) |
| 5113 | count++; |
| 5114 | if (arg_type_may_be_refcounted(fn->arg4_type)) |
| 5115 | count++; |
| 5116 | if (arg_type_may_be_refcounted(fn->arg5_type)) |
| 5117 | count++; |
| 5118 | |
| 5119 | /* A reference acquiring function cannot acquire |
| 5120 | * another refcounted ptr. |
| 5121 | */ |
| 5122 | if (may_be_acquire_function(func_id) && count) |
| 5123 | return false; |
| 5124 | |
| 5125 | /* We only support one arg being unreferenced at the moment, |
| 5126 | * which is sufficient for the helper functions we have right now. |
| 5127 | */ |
| 5128 | return count <= 1; |
| 5129 | } |
| 5130 | |
| 5131 | static bool check_btf_id_ok(const struct bpf_func_proto *fn) |
| 5132 | { |
| 5133 | int i; |
| 5134 | |
| 5135 | for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) { |
| 5136 | if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i]) |
| 5137 | return false; |
| 5138 | |
| 5139 | if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i]) |
| 5140 | return false; |
| 5141 | } |
| 5142 | |
| 5143 | return true; |
| 5144 | } |
| 5145 | |
| 5146 | static int check_func_proto(const struct bpf_func_proto *fn, int func_id) |
| 5147 | { |
| 5148 | return check_raw_mode_ok(fn) && |
| 5149 | check_arg_pair_ok(fn) && |
| 5150 | check_btf_id_ok(fn) && |
| 5151 | check_refcount_ok(fn, func_id) ? 0 : -EINVAL; |
| 5152 | } |
| 5153 | |
| 5154 | /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] |
| 5155 | * are now invalid, so turn them into unknown SCALAR_VALUE. |
| 5156 | */ |
| 5157 | static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, |
| 5158 | struct bpf_func_state *state) |
| 5159 | { |
| 5160 | struct bpf_reg_state *regs = state->regs, *reg; |
| 5161 | int i; |
| 5162 | |
| 5163 | for (i = 0; i < MAX_BPF_REG; i++) |
| 5164 | if (reg_is_pkt_pointer_any(®s[i])) |
| 5165 | mark_reg_unknown(env, regs, i); |
| 5166 | |
| 5167 | bpf_for_each_spilled_reg(i, state, reg) { |
| 5168 | if (!reg) |
| 5169 | continue; |
| 5170 | if (reg_is_pkt_pointer_any(reg)) |
| 5171 | __mark_reg_unknown(env, reg); |
| 5172 | } |
| 5173 | } |
| 5174 | |
| 5175 | static void clear_all_pkt_pointers(struct bpf_verifier_env *env) |
| 5176 | { |
| 5177 | struct bpf_verifier_state *vstate = env->cur_state; |
| 5178 | int i; |
| 5179 | |
| 5180 | for (i = 0; i <= vstate->curframe; i++) |
| 5181 | __clear_all_pkt_pointers(env, vstate->frame[i]); |
| 5182 | } |
| 5183 | |
| 5184 | enum { |
| 5185 | AT_PKT_END = -1, |
| 5186 | BEYOND_PKT_END = -2, |
| 5187 | }; |
| 5188 | |
| 5189 | static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open) |
| 5190 | { |
| 5191 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 5192 | struct bpf_reg_state *reg = &state->regs[regn]; |
| 5193 | |
| 5194 | if (reg->type != PTR_TO_PACKET) |
| 5195 | /* PTR_TO_PACKET_META is not supported yet */ |
| 5196 | return; |
| 5197 | |
| 5198 | /* The 'reg' is pkt > pkt_end or pkt >= pkt_end. |
| 5199 | * How far beyond pkt_end it goes is unknown. |
| 5200 | * if (!range_open) it's the case of pkt >= pkt_end |
| 5201 | * if (range_open) it's the case of pkt > pkt_end |
| 5202 | * hence this pointer is at least 1 byte bigger than pkt_end |
| 5203 | */ |
| 5204 | if (range_open) |
| 5205 | reg->range = BEYOND_PKT_END; |
| 5206 | else |
| 5207 | reg->range = AT_PKT_END; |
| 5208 | } |
| 5209 | |
| 5210 | static void release_reg_references(struct bpf_verifier_env *env, |
| 5211 | struct bpf_func_state *state, |
| 5212 | int ref_obj_id) |
| 5213 | { |
| 5214 | struct bpf_reg_state *regs = state->regs, *reg; |
| 5215 | int i; |
| 5216 | |
| 5217 | for (i = 0; i < MAX_BPF_REG; i++) |
| 5218 | if (regs[i].ref_obj_id == ref_obj_id) |
| 5219 | mark_reg_unknown(env, regs, i); |
| 5220 | |
| 5221 | bpf_for_each_spilled_reg(i, state, reg) { |
| 5222 | if (!reg) |
| 5223 | continue; |
| 5224 | if (reg->ref_obj_id == ref_obj_id) |
| 5225 | __mark_reg_unknown(env, reg); |
| 5226 | } |
| 5227 | } |
| 5228 | |
| 5229 | /* The pointer with the specified id has released its reference to kernel |
| 5230 | * resources. Identify all copies of the same pointer and clear the reference. |
| 5231 | */ |
| 5232 | static int release_reference(struct bpf_verifier_env *env, |
| 5233 | int ref_obj_id) |
| 5234 | { |
| 5235 | struct bpf_verifier_state *vstate = env->cur_state; |
| 5236 | int err; |
| 5237 | int i; |
| 5238 | |
| 5239 | err = release_reference_state(cur_func(env), ref_obj_id); |
| 5240 | if (err) |
| 5241 | return err; |
| 5242 | |
| 5243 | for (i = 0; i <= vstate->curframe; i++) |
| 5244 | release_reg_references(env, vstate->frame[i], ref_obj_id); |
| 5245 | |
| 5246 | return 0; |
| 5247 | } |
| 5248 | |
| 5249 | static void clear_caller_saved_regs(struct bpf_verifier_env *env, |
| 5250 | struct bpf_reg_state *regs) |
| 5251 | { |
| 5252 | int i; |
| 5253 | |
| 5254 | /* after the call registers r0 - r5 were scratched */ |
| 5255 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
| 5256 | mark_reg_not_init(env, regs, caller_saved[i]); |
| 5257 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 5258 | } |
| 5259 | } |
| 5260 | |
| 5261 | static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, |
| 5262 | int *insn_idx) |
| 5263 | { |
| 5264 | struct bpf_verifier_state *state = env->cur_state; |
| 5265 | struct bpf_func_info_aux *func_info_aux; |
| 5266 | struct bpf_func_state *caller, *callee; |
| 5267 | int i, err, subprog, target_insn; |
| 5268 | bool is_global = false; |
| 5269 | |
| 5270 | if (state->curframe + 1 >= MAX_CALL_FRAMES) { |
| 5271 | verbose(env, "the call stack of %d frames is too deep\n", |
| 5272 | state->curframe + 2); |
| 5273 | return -E2BIG; |
| 5274 | } |
| 5275 | |
| 5276 | target_insn = *insn_idx + insn->imm; |
| 5277 | subprog = find_subprog(env, target_insn + 1); |
| 5278 | if (subprog < 0) { |
| 5279 | verbose(env, "verifier bug. No program starts at insn %d\n", |
| 5280 | target_insn + 1); |
| 5281 | return -EFAULT; |
| 5282 | } |
| 5283 | |
| 5284 | caller = state->frame[state->curframe]; |
| 5285 | if (state->frame[state->curframe + 1]) { |
| 5286 | verbose(env, "verifier bug. Frame %d already allocated\n", |
| 5287 | state->curframe + 1); |
| 5288 | return -EFAULT; |
| 5289 | } |
| 5290 | |
| 5291 | func_info_aux = env->prog->aux->func_info_aux; |
| 5292 | if (func_info_aux) |
| 5293 | is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL; |
| 5294 | err = btf_check_func_arg_match(env, subprog, caller->regs); |
| 5295 | if (err == -EFAULT) |
| 5296 | return err; |
| 5297 | if (is_global) { |
| 5298 | if (err) { |
| 5299 | verbose(env, "Caller passes invalid args into func#%d\n", |
| 5300 | subprog); |
| 5301 | return err; |
| 5302 | } else { |
| 5303 | if (env->log.level & BPF_LOG_LEVEL) |
| 5304 | verbose(env, |
| 5305 | "Func#%d is global and valid. Skipping.\n", |
| 5306 | subprog); |
| 5307 | clear_caller_saved_regs(env, caller->regs); |
| 5308 | |
| 5309 | /* All global functions return a 64-bit SCALAR_VALUE */ |
| 5310 | mark_reg_unknown(env, caller->regs, BPF_REG_0); |
| 5311 | caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
| 5312 | |
| 5313 | /* continue with next insn after call */ |
| 5314 | return 0; |
| 5315 | } |
| 5316 | } |
| 5317 | |
| 5318 | callee = kzalloc(sizeof(*callee), GFP_KERNEL); |
| 5319 | if (!callee) |
| 5320 | return -ENOMEM; |
| 5321 | state->frame[state->curframe + 1] = callee; |
| 5322 | |
| 5323 | /* callee cannot access r0, r6 - r9 for reading and has to write |
| 5324 | * into its own stack before reading from it. |
| 5325 | * callee can read/write into caller's stack |
| 5326 | */ |
| 5327 | init_func_state(env, callee, |
| 5328 | /* remember the callsite, it will be used by bpf_exit */ |
| 5329 | *insn_idx /* callsite */, |
| 5330 | state->curframe + 1 /* frameno within this callchain */, |
| 5331 | subprog /* subprog number within this prog */); |
| 5332 | |
| 5333 | /* Transfer references to the callee */ |
| 5334 | err = transfer_reference_state(callee, caller); |
| 5335 | if (err) |
| 5336 | return err; |
| 5337 | |
| 5338 | /* copy r1 - r5 args that callee can access. The copy includes parent |
| 5339 | * pointers, which connects us up to the liveness chain |
| 5340 | */ |
| 5341 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) |
| 5342 | callee->regs[i] = caller->regs[i]; |
| 5343 | |
| 5344 | clear_caller_saved_regs(env, caller->regs); |
| 5345 | |
| 5346 | /* only increment it after check_reg_arg() finished */ |
| 5347 | state->curframe++; |
| 5348 | |
| 5349 | /* and go analyze first insn of the callee */ |
| 5350 | *insn_idx = target_insn; |
| 5351 | |
| 5352 | if (env->log.level & BPF_LOG_LEVEL) { |
| 5353 | verbose(env, "caller:\n"); |
| 5354 | print_verifier_state(env, caller); |
| 5355 | verbose(env, "callee:\n"); |
| 5356 | print_verifier_state(env, callee); |
| 5357 | } |
| 5358 | return 0; |
| 5359 | } |
| 5360 | |
| 5361 | static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) |
| 5362 | { |
| 5363 | struct bpf_verifier_state *state = env->cur_state; |
| 5364 | struct bpf_func_state *caller, *callee; |
| 5365 | struct bpf_reg_state *r0; |
| 5366 | int err; |
| 5367 | |
| 5368 | callee = state->frame[state->curframe]; |
| 5369 | r0 = &callee->regs[BPF_REG_0]; |
| 5370 | if (r0->type == PTR_TO_STACK) { |
| 5371 | /* technically it's ok to return caller's stack pointer |
| 5372 | * (or caller's caller's pointer) back to the caller, |
| 5373 | * since these pointers are valid. Only current stack |
| 5374 | * pointer will be invalid as soon as function exits, |
| 5375 | * but let's be conservative |
| 5376 | */ |
| 5377 | verbose(env, "cannot return stack pointer to the caller\n"); |
| 5378 | return -EINVAL; |
| 5379 | } |
| 5380 | |
| 5381 | state->curframe--; |
| 5382 | caller = state->frame[state->curframe]; |
| 5383 | /* return to the caller whatever r0 had in the callee */ |
| 5384 | caller->regs[BPF_REG_0] = *r0; |
| 5385 | |
| 5386 | /* Transfer references to the caller */ |
| 5387 | err = transfer_reference_state(caller, callee); |
| 5388 | if (err) |
| 5389 | return err; |
| 5390 | |
| 5391 | *insn_idx = callee->callsite + 1; |
| 5392 | if (env->log.level & BPF_LOG_LEVEL) { |
| 5393 | verbose(env, "returning from callee:\n"); |
| 5394 | print_verifier_state(env, callee); |
| 5395 | verbose(env, "to caller at %d:\n", *insn_idx); |
| 5396 | print_verifier_state(env, caller); |
| 5397 | } |
| 5398 | /* clear everything in the callee */ |
| 5399 | free_func_state(callee); |
| 5400 | state->frame[state->curframe + 1] = NULL; |
| 5401 | return 0; |
| 5402 | } |
| 5403 | |
| 5404 | static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, |
| 5405 | int func_id, |
| 5406 | struct bpf_call_arg_meta *meta) |
| 5407 | { |
| 5408 | struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; |
| 5409 | |
| 5410 | if (ret_type != RET_INTEGER || |
| 5411 | (func_id != BPF_FUNC_get_stack && |
| 5412 | func_id != BPF_FUNC_probe_read_str && |
| 5413 | func_id != BPF_FUNC_probe_read_kernel_str && |
| 5414 | func_id != BPF_FUNC_probe_read_user_str)) |
| 5415 | return; |
| 5416 | |
| 5417 | ret_reg->smax_value = meta->msize_max_value; |
| 5418 | ret_reg->s32_max_value = meta->msize_max_value; |
| 5419 | ret_reg->smin_value = -MAX_ERRNO; |
| 5420 | ret_reg->s32_min_value = -MAX_ERRNO; |
| 5421 | __reg_deduce_bounds(ret_reg); |
| 5422 | __reg_bound_offset(ret_reg); |
| 5423 | __update_reg_bounds(ret_reg); |
| 5424 | } |
| 5425 | |
| 5426 | static int |
| 5427 | record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, |
| 5428 | int func_id, int insn_idx) |
| 5429 | { |
| 5430 | struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; |
| 5431 | struct bpf_map *map = meta->map_ptr; |
| 5432 | |
| 5433 | if (func_id != BPF_FUNC_tail_call && |
| 5434 | func_id != BPF_FUNC_map_lookup_elem && |
| 5435 | func_id != BPF_FUNC_map_update_elem && |
| 5436 | func_id != BPF_FUNC_map_delete_elem && |
| 5437 | func_id != BPF_FUNC_map_push_elem && |
| 5438 | func_id != BPF_FUNC_map_pop_elem && |
| 5439 | func_id != BPF_FUNC_map_peek_elem) |
| 5440 | return 0; |
| 5441 | |
| 5442 | if (map == NULL) { |
| 5443 | verbose(env, "kernel subsystem misconfigured verifier\n"); |
| 5444 | return -EINVAL; |
| 5445 | } |
| 5446 | |
| 5447 | /* In case of read-only, some additional restrictions |
| 5448 | * need to be applied in order to prevent altering the |
| 5449 | * state of the map from program side. |
| 5450 | */ |
| 5451 | if ((map->map_flags & BPF_F_RDONLY_PROG) && |
| 5452 | (func_id == BPF_FUNC_map_delete_elem || |
| 5453 | func_id == BPF_FUNC_map_update_elem || |
| 5454 | func_id == BPF_FUNC_map_push_elem || |
| 5455 | func_id == BPF_FUNC_map_pop_elem)) { |
| 5456 | verbose(env, "write into map forbidden\n"); |
| 5457 | return -EACCES; |
| 5458 | } |
| 5459 | |
| 5460 | if (!BPF_MAP_PTR(aux->map_ptr_state)) |
| 5461 | bpf_map_ptr_store(aux, meta->map_ptr, |
| 5462 | !meta->map_ptr->bypass_spec_v1); |
| 5463 | else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) |
| 5464 | bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, |
| 5465 | !meta->map_ptr->bypass_spec_v1); |
| 5466 | return 0; |
| 5467 | } |
| 5468 | |
| 5469 | static int |
| 5470 | record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, |
| 5471 | int func_id, int insn_idx) |
| 5472 | { |
| 5473 | struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; |
| 5474 | struct bpf_reg_state *regs = cur_regs(env), *reg; |
| 5475 | struct bpf_map *map = meta->map_ptr; |
| 5476 | struct tnum range; |
| 5477 | u64 val; |
| 5478 | int err; |
| 5479 | |
| 5480 | if (func_id != BPF_FUNC_tail_call) |
| 5481 | return 0; |
| 5482 | if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) { |
| 5483 | verbose(env, "kernel subsystem misconfigured verifier\n"); |
| 5484 | return -EINVAL; |
| 5485 | } |
| 5486 | |
| 5487 | range = tnum_range(0, map->max_entries - 1); |
| 5488 | reg = ®s[BPF_REG_3]; |
| 5489 | |
| 5490 | if (!register_is_const(reg) || !tnum_in(range, reg->var_off)) { |
| 5491 | bpf_map_key_store(aux, BPF_MAP_KEY_POISON); |
| 5492 | return 0; |
| 5493 | } |
| 5494 | |
| 5495 | err = mark_chain_precision(env, BPF_REG_3); |
| 5496 | if (err) |
| 5497 | return err; |
| 5498 | |
| 5499 | val = reg->var_off.value; |
| 5500 | if (bpf_map_key_unseen(aux)) |
| 5501 | bpf_map_key_store(aux, val); |
| 5502 | else if (!bpf_map_key_poisoned(aux) && |
| 5503 | bpf_map_key_immediate(aux) != val) |
| 5504 | bpf_map_key_store(aux, BPF_MAP_KEY_POISON); |
| 5505 | return 0; |
| 5506 | } |
| 5507 | |
| 5508 | static int check_reference_leak(struct bpf_verifier_env *env) |
| 5509 | { |
| 5510 | struct bpf_func_state *state = cur_func(env); |
| 5511 | int i; |
| 5512 | |
| 5513 | for (i = 0; i < state->acquired_refs; i++) { |
| 5514 | verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", |
| 5515 | state->refs[i].id, state->refs[i].insn_idx); |
| 5516 | } |
| 5517 | return state->acquired_refs ? -EINVAL : 0; |
| 5518 | } |
| 5519 | |
| 5520 | static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) |
| 5521 | { |
| 5522 | const struct bpf_func_proto *fn = NULL; |
| 5523 | struct bpf_reg_state *regs; |
| 5524 | struct bpf_call_arg_meta meta; |
| 5525 | bool changes_data; |
| 5526 | int i, err; |
| 5527 | |
| 5528 | /* find function prototype */ |
| 5529 | if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { |
| 5530 | verbose(env, "invalid func %s#%d\n", func_id_name(func_id), |
| 5531 | func_id); |
| 5532 | return -EINVAL; |
| 5533 | } |
| 5534 | |
| 5535 | if (env->ops->get_func_proto) |
| 5536 | fn = env->ops->get_func_proto(func_id, env->prog); |
| 5537 | if (!fn) { |
| 5538 | verbose(env, "unknown func %s#%d\n", func_id_name(func_id), |
| 5539 | func_id); |
| 5540 | return -EINVAL; |
| 5541 | } |
| 5542 | |
| 5543 | /* eBPF programs must be GPL compatible to use GPL-ed functions */ |
| 5544 | if (!env->prog->gpl_compatible && fn->gpl_only) { |
| 5545 | verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); |
| 5546 | return -EINVAL; |
| 5547 | } |
| 5548 | |
| 5549 | if (fn->allowed && !fn->allowed(env->prog)) { |
| 5550 | verbose(env, "helper call is not allowed in probe\n"); |
| 5551 | return -EINVAL; |
| 5552 | } |
| 5553 | |
| 5554 | /* With LD_ABS/IND some JITs save/restore skb from r1. */ |
| 5555 | changes_data = bpf_helper_changes_pkt_data(fn->func); |
| 5556 | if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { |
| 5557 | verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", |
| 5558 | func_id_name(func_id), func_id); |
| 5559 | return -EINVAL; |
| 5560 | } |
| 5561 | |
| 5562 | memset(&meta, 0, sizeof(meta)); |
| 5563 | meta.pkt_access = fn->pkt_access; |
| 5564 | |
| 5565 | err = check_func_proto(fn, func_id); |
| 5566 | if (err) { |
| 5567 | verbose(env, "kernel subsystem misconfigured func %s#%d\n", |
| 5568 | func_id_name(func_id), func_id); |
| 5569 | return err; |
| 5570 | } |
| 5571 | |
| 5572 | meta.func_id = func_id; |
| 5573 | /* check args */ |
| 5574 | for (i = 0; i < 5; i++) { |
| 5575 | err = check_func_arg(env, i, &meta, fn); |
| 5576 | if (err) |
| 5577 | return err; |
| 5578 | } |
| 5579 | |
| 5580 | err = record_func_map(env, &meta, func_id, insn_idx); |
| 5581 | if (err) |
| 5582 | return err; |
| 5583 | |
| 5584 | err = record_func_key(env, &meta, func_id, insn_idx); |
| 5585 | if (err) |
| 5586 | return err; |
| 5587 | |
| 5588 | /* Mark slots with STACK_MISC in case of raw mode, stack offset |
| 5589 | * is inferred from register state. |
| 5590 | */ |
| 5591 | for (i = 0; i < meta.access_size; i++) { |
| 5592 | err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, |
| 5593 | BPF_WRITE, -1, false); |
| 5594 | if (err) |
| 5595 | return err; |
| 5596 | } |
| 5597 | |
| 5598 | if (func_id == BPF_FUNC_tail_call) { |
| 5599 | err = check_reference_leak(env); |
| 5600 | if (err) { |
| 5601 | verbose(env, "tail_call would lead to reference leak\n"); |
| 5602 | return err; |
| 5603 | } |
| 5604 | } else if (is_release_function(func_id)) { |
| 5605 | err = release_reference(env, meta.ref_obj_id); |
| 5606 | if (err) { |
| 5607 | verbose(env, "func %s#%d reference has not been acquired before\n", |
| 5608 | func_id_name(func_id), func_id); |
| 5609 | return err; |
| 5610 | } |
| 5611 | } |
| 5612 | |
| 5613 | regs = cur_regs(env); |
| 5614 | |
| 5615 | /* check that flags argument in get_local_storage(map, flags) is 0, |
| 5616 | * this is required because get_local_storage() can't return an error. |
| 5617 | */ |
| 5618 | if (func_id == BPF_FUNC_get_local_storage && |
| 5619 | !register_is_null(®s[BPF_REG_2])) { |
| 5620 | verbose(env, "get_local_storage() doesn't support non-zero flags\n"); |
| 5621 | return -EINVAL; |
| 5622 | } |
| 5623 | |
| 5624 | /* reset caller saved regs */ |
| 5625 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
| 5626 | mark_reg_not_init(env, regs, caller_saved[i]); |
| 5627 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 5628 | } |
| 5629 | |
| 5630 | /* helper call returns 64-bit value. */ |
| 5631 | regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG; |
| 5632 | |
| 5633 | /* update return register (already marked as written above) */ |
| 5634 | if (fn->ret_type == RET_INTEGER) { |
| 5635 | /* sets type to SCALAR_VALUE */ |
| 5636 | mark_reg_unknown(env, regs, BPF_REG_0); |
| 5637 | } else if (fn->ret_type == RET_VOID) { |
| 5638 | regs[BPF_REG_0].type = NOT_INIT; |
| 5639 | } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || |
| 5640 | fn->ret_type == RET_PTR_TO_MAP_VALUE) { |
| 5641 | /* There is no offset yet applied, variable or fixed */ |
| 5642 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 5643 | /* remember map_ptr, so that check_map_access() |
| 5644 | * can check 'value_size' boundary of memory access |
| 5645 | * to map element returned from bpf_map_lookup_elem() |
| 5646 | */ |
| 5647 | if (meta.map_ptr == NULL) { |
| 5648 | verbose(env, |
| 5649 | "kernel subsystem misconfigured verifier\n"); |
| 5650 | return -EINVAL; |
| 5651 | } |
| 5652 | regs[BPF_REG_0].map_ptr = meta.map_ptr; |
| 5653 | if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { |
| 5654 | regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; |
| 5655 | if (map_value_has_spin_lock(meta.map_ptr)) |
| 5656 | regs[BPF_REG_0].id = ++env->id_gen; |
| 5657 | } else { |
| 5658 | regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; |
| 5659 | } |
| 5660 | } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { |
| 5661 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 5662 | regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; |
| 5663 | } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { |
| 5664 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 5665 | regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; |
| 5666 | } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { |
| 5667 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 5668 | regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; |
| 5669 | } else if (fn->ret_type == RET_PTR_TO_ALLOC_MEM_OR_NULL) { |
| 5670 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 5671 | regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL; |
| 5672 | regs[BPF_REG_0].mem_size = meta.mem_size; |
| 5673 | } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL || |
| 5674 | fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) { |
| 5675 | const struct btf_type *t; |
| 5676 | |
| 5677 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 5678 | t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL); |
| 5679 | if (!btf_type_is_struct(t)) { |
| 5680 | u32 tsize; |
| 5681 | const struct btf_type *ret; |
| 5682 | const char *tname; |
| 5683 | |
| 5684 | /* resolve the type size of ksym. */ |
| 5685 | ret = btf_resolve_size(meta.ret_btf, t, &tsize); |
| 5686 | if (IS_ERR(ret)) { |
| 5687 | tname = btf_name_by_offset(meta.ret_btf, t->name_off); |
| 5688 | verbose(env, "unable to resolve the size of type '%s': %ld\n", |
| 5689 | tname, PTR_ERR(ret)); |
| 5690 | return -EINVAL; |
| 5691 | } |
| 5692 | regs[BPF_REG_0].type = |
| 5693 | fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? |
| 5694 | PTR_TO_MEM : PTR_TO_MEM_OR_NULL; |
| 5695 | regs[BPF_REG_0].mem_size = tsize; |
| 5696 | } else { |
| 5697 | regs[BPF_REG_0].type = |
| 5698 | fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ? |
| 5699 | PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL; |
| 5700 | regs[BPF_REG_0].btf = meta.ret_btf; |
| 5701 | regs[BPF_REG_0].btf_id = meta.ret_btf_id; |
| 5702 | } |
| 5703 | } else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL || |
| 5704 | fn->ret_type == RET_PTR_TO_BTF_ID) { |
| 5705 | int ret_btf_id; |
| 5706 | |
| 5707 | mark_reg_known_zero(env, regs, BPF_REG_0); |
| 5708 | regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ? |
| 5709 | PTR_TO_BTF_ID : |
| 5710 | PTR_TO_BTF_ID_OR_NULL; |
| 5711 | ret_btf_id = *fn->ret_btf_id; |
| 5712 | if (ret_btf_id == 0) { |
| 5713 | verbose(env, "invalid return type %d of func %s#%d\n", |
| 5714 | fn->ret_type, func_id_name(func_id), func_id); |
| 5715 | return -EINVAL; |
| 5716 | } |
| 5717 | /* current BPF helper definitions are only coming from |
| 5718 | * built-in code with type IDs from vmlinux BTF |
| 5719 | */ |
| 5720 | regs[BPF_REG_0].btf = btf_vmlinux; |
| 5721 | regs[BPF_REG_0].btf_id = ret_btf_id; |
| 5722 | } else { |
| 5723 | verbose(env, "unknown return type %d of func %s#%d\n", |
| 5724 | fn->ret_type, func_id_name(func_id), func_id); |
| 5725 | return -EINVAL; |
| 5726 | } |
| 5727 | |
| 5728 | if (reg_type_may_be_null(regs[BPF_REG_0].type)) |
| 5729 | regs[BPF_REG_0].id = ++env->id_gen; |
| 5730 | |
| 5731 | if (is_ptr_cast_function(func_id)) { |
| 5732 | /* For release_reference() */ |
| 5733 | regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; |
| 5734 | } else if (is_acquire_function(func_id, meta.map_ptr)) { |
| 5735 | int id = acquire_reference_state(env, insn_idx); |
| 5736 | |
| 5737 | if (id < 0) |
| 5738 | return id; |
| 5739 | /* For mark_ptr_or_null_reg() */ |
| 5740 | regs[BPF_REG_0].id = id; |
| 5741 | /* For release_reference() */ |
| 5742 | regs[BPF_REG_0].ref_obj_id = id; |
| 5743 | } |
| 5744 | |
| 5745 | do_refine_retval_range(regs, fn->ret_type, func_id, &meta); |
| 5746 | |
| 5747 | err = check_map_func_compatibility(env, meta.map_ptr, func_id); |
| 5748 | if (err) |
| 5749 | return err; |
| 5750 | |
| 5751 | if ((func_id == BPF_FUNC_get_stack || |
| 5752 | func_id == BPF_FUNC_get_task_stack) && |
| 5753 | !env->prog->has_callchain_buf) { |
| 5754 | const char *err_str; |
| 5755 | |
| 5756 | #ifdef CONFIG_PERF_EVENTS |
| 5757 | err = get_callchain_buffers(sysctl_perf_event_max_stack); |
| 5758 | err_str = "cannot get callchain buffer for func %s#%d\n"; |
| 5759 | #else |
| 5760 | err = -ENOTSUPP; |
| 5761 | err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; |
| 5762 | #endif |
| 5763 | if (err) { |
| 5764 | verbose(env, err_str, func_id_name(func_id), func_id); |
| 5765 | return err; |
| 5766 | } |
| 5767 | |
| 5768 | env->prog->has_callchain_buf = true; |
| 5769 | } |
| 5770 | |
| 5771 | if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack) |
| 5772 | env->prog->call_get_stack = true; |
| 5773 | |
| 5774 | if (changes_data) |
| 5775 | clear_all_pkt_pointers(env); |
| 5776 | return 0; |
| 5777 | } |
| 5778 | |
| 5779 | static bool signed_add_overflows(s64 a, s64 b) |
| 5780 | { |
| 5781 | /* Do the add in u64, where overflow is well-defined */ |
| 5782 | s64 res = (s64)((u64)a + (u64)b); |
| 5783 | |
| 5784 | if (b < 0) |
| 5785 | return res > a; |
| 5786 | return res < a; |
| 5787 | } |
| 5788 | |
| 5789 | static bool signed_add32_overflows(s32 a, s32 b) |
| 5790 | { |
| 5791 | /* Do the add in u32, where overflow is well-defined */ |
| 5792 | s32 res = (s32)((u32)a + (u32)b); |
| 5793 | |
| 5794 | if (b < 0) |
| 5795 | return res > a; |
| 5796 | return res < a; |
| 5797 | } |
| 5798 | |
| 5799 | static bool signed_sub_overflows(s64 a, s64 b) |
| 5800 | { |
| 5801 | /* Do the sub in u64, where overflow is well-defined */ |
| 5802 | s64 res = (s64)((u64)a - (u64)b); |
| 5803 | |
| 5804 | if (b < 0) |
| 5805 | return res < a; |
| 5806 | return res > a; |
| 5807 | } |
| 5808 | |
| 5809 | static bool signed_sub32_overflows(s32 a, s32 b) |
| 5810 | { |
| 5811 | /* Do the sub in u32, where overflow is well-defined */ |
| 5812 | s32 res = (s32)((u32)a - (u32)b); |
| 5813 | |
| 5814 | if (b < 0) |
| 5815 | return res < a; |
| 5816 | return res > a; |
| 5817 | } |
| 5818 | |
| 5819 | static bool check_reg_sane_offset(struct bpf_verifier_env *env, |
| 5820 | const struct bpf_reg_state *reg, |
| 5821 | enum bpf_reg_type type) |
| 5822 | { |
| 5823 | bool known = tnum_is_const(reg->var_off); |
| 5824 | s64 val = reg->var_off.value; |
| 5825 | s64 smin = reg->smin_value; |
| 5826 | |
| 5827 | if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { |
| 5828 | verbose(env, "math between %s pointer and %lld is not allowed\n", |
| 5829 | reg_type_str[type], val); |
| 5830 | return false; |
| 5831 | } |
| 5832 | |
| 5833 | if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { |
| 5834 | verbose(env, "%s pointer offset %d is not allowed\n", |
| 5835 | reg_type_str[type], reg->off); |
| 5836 | return false; |
| 5837 | } |
| 5838 | |
| 5839 | if (smin == S64_MIN) { |
| 5840 | verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", |
| 5841 | reg_type_str[type]); |
| 5842 | return false; |
| 5843 | } |
| 5844 | |
| 5845 | if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { |
| 5846 | verbose(env, "value %lld makes %s pointer be out of bounds\n", |
| 5847 | smin, reg_type_str[type]); |
| 5848 | return false; |
| 5849 | } |
| 5850 | |
| 5851 | return true; |
| 5852 | } |
| 5853 | |
| 5854 | static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) |
| 5855 | { |
| 5856 | return &env->insn_aux_data[env->insn_idx]; |
| 5857 | } |
| 5858 | |
| 5859 | static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, |
| 5860 | u32 *ptr_limit, u8 opcode, bool off_is_neg) |
| 5861 | { |
| 5862 | bool mask_to_left = (opcode == BPF_ADD && off_is_neg) || |
| 5863 | (opcode == BPF_SUB && !off_is_neg); |
| 5864 | u32 off, max; |
| 5865 | |
| 5866 | switch (ptr_reg->type) { |
| 5867 | case PTR_TO_STACK: |
| 5868 | /* Offset 0 is out-of-bounds, but acceptable start for the |
| 5869 | * left direction, see BPF_REG_FP. |
| 5870 | */ |
| 5871 | max = MAX_BPF_STACK + mask_to_left; |
| 5872 | /* Indirect variable offset stack access is prohibited in |
| 5873 | * unprivileged mode so it's not handled here. |
| 5874 | */ |
| 5875 | off = ptr_reg->off + ptr_reg->var_off.value; |
| 5876 | if (mask_to_left) |
| 5877 | *ptr_limit = MAX_BPF_STACK + off; |
| 5878 | else |
| 5879 | *ptr_limit = -off - 1; |
| 5880 | return *ptr_limit >= max ? -ERANGE : 0; |
| 5881 | case PTR_TO_MAP_VALUE: |
| 5882 | max = ptr_reg->map_ptr->value_size; |
| 5883 | if (mask_to_left) { |
| 5884 | *ptr_limit = ptr_reg->umax_value + ptr_reg->off; |
| 5885 | } else { |
| 5886 | off = ptr_reg->smin_value + ptr_reg->off; |
| 5887 | *ptr_limit = ptr_reg->map_ptr->value_size - off - 1; |
| 5888 | } |
| 5889 | return *ptr_limit >= max ? -ERANGE : 0; |
| 5890 | default: |
| 5891 | return -EINVAL; |
| 5892 | } |
| 5893 | } |
| 5894 | |
| 5895 | static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, |
| 5896 | const struct bpf_insn *insn) |
| 5897 | { |
| 5898 | return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K; |
| 5899 | } |
| 5900 | |
| 5901 | static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, |
| 5902 | u32 alu_state, u32 alu_limit) |
| 5903 | { |
| 5904 | /* If we arrived here from different branches with different |
| 5905 | * state or limits to sanitize, then this won't work. |
| 5906 | */ |
| 5907 | if (aux->alu_state && |
| 5908 | (aux->alu_state != alu_state || |
| 5909 | aux->alu_limit != alu_limit)) |
| 5910 | return -EACCES; |
| 5911 | |
| 5912 | /* Corresponding fixup done in fixup_bpf_calls(). */ |
| 5913 | aux->alu_state = alu_state; |
| 5914 | aux->alu_limit = alu_limit; |
| 5915 | return 0; |
| 5916 | } |
| 5917 | |
| 5918 | static int sanitize_val_alu(struct bpf_verifier_env *env, |
| 5919 | struct bpf_insn *insn) |
| 5920 | { |
| 5921 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 5922 | |
| 5923 | if (can_skip_alu_sanitation(env, insn)) |
| 5924 | return 0; |
| 5925 | |
| 5926 | return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); |
| 5927 | } |
| 5928 | |
| 5929 | static int sanitize_ptr_alu(struct bpf_verifier_env *env, |
| 5930 | struct bpf_insn *insn, |
| 5931 | const struct bpf_reg_state *ptr_reg, |
| 5932 | struct bpf_reg_state *dst_reg, |
| 5933 | bool off_is_neg) |
| 5934 | { |
| 5935 | struct bpf_verifier_state *vstate = env->cur_state; |
| 5936 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 5937 | bool ptr_is_dst_reg = ptr_reg == dst_reg; |
| 5938 | u8 opcode = BPF_OP(insn->code); |
| 5939 | u32 alu_state, alu_limit; |
| 5940 | struct bpf_reg_state tmp; |
| 5941 | bool ret; |
| 5942 | int err; |
| 5943 | |
| 5944 | if (can_skip_alu_sanitation(env, insn)) |
| 5945 | return 0; |
| 5946 | |
| 5947 | /* We already marked aux for masking from non-speculative |
| 5948 | * paths, thus we got here in the first place. We only care |
| 5949 | * to explore bad access from here. |
| 5950 | */ |
| 5951 | if (vstate->speculative) |
| 5952 | goto do_sim; |
| 5953 | |
| 5954 | alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; |
| 5955 | alu_state |= ptr_is_dst_reg ? |
| 5956 | BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; |
| 5957 | |
| 5958 | err = retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg); |
| 5959 | if (err < 0) |
| 5960 | return err; |
| 5961 | |
| 5962 | err = update_alu_sanitation_state(aux, alu_state, alu_limit); |
| 5963 | if (err < 0) |
| 5964 | return err; |
| 5965 | do_sim: |
| 5966 | /* Simulate and find potential out-of-bounds access under |
| 5967 | * speculative execution from truncation as a result of |
| 5968 | * masking when off was not within expected range. If off |
| 5969 | * sits in dst, then we temporarily need to move ptr there |
| 5970 | * to simulate dst (== 0) +/-= ptr. Needed, for example, |
| 5971 | * for cases where we use K-based arithmetic in one direction |
| 5972 | * and truncated reg-based in the other in order to explore |
| 5973 | * bad access. |
| 5974 | */ |
| 5975 | if (!ptr_is_dst_reg) { |
| 5976 | tmp = *dst_reg; |
| 5977 | *dst_reg = *ptr_reg; |
| 5978 | } |
| 5979 | ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true); |
| 5980 | if (!ptr_is_dst_reg && ret) |
| 5981 | *dst_reg = tmp; |
| 5982 | return !ret ? -EFAULT : 0; |
| 5983 | } |
| 5984 | |
| 5985 | /* check that stack access falls within stack limits and that 'reg' doesn't |
| 5986 | * have a variable offset. |
| 5987 | * |
| 5988 | * Variable offset is prohibited for unprivileged mode for simplicity since it |
| 5989 | * requires corresponding support in Spectre masking for stack ALU. See also |
| 5990 | * retrieve_ptr_limit(). |
| 5991 | * |
| 5992 | * |
| 5993 | * 'off' includes 'reg->off'. |
| 5994 | */ |
| 5995 | static int check_stack_access_for_ptr_arithmetic( |
| 5996 | struct bpf_verifier_env *env, |
| 5997 | int regno, |
| 5998 | const struct bpf_reg_state *reg, |
| 5999 | int off) |
| 6000 | { |
| 6001 | if (!tnum_is_const(reg->var_off)) { |
| 6002 | char tn_buf[48]; |
| 6003 | |
| 6004 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 6005 | verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", |
| 6006 | regno, tn_buf, off); |
| 6007 | return -EACCES; |
| 6008 | } |
| 6009 | |
| 6010 | if (off >= 0 || off < -MAX_BPF_STACK) { |
| 6011 | verbose(env, "R%d stack pointer arithmetic goes out of range, " |
| 6012 | "prohibited for !root; off=%d\n", regno, off); |
| 6013 | return -EACCES; |
| 6014 | } |
| 6015 | |
| 6016 | return 0; |
| 6017 | } |
| 6018 | |
| 6019 | |
| 6020 | /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. |
| 6021 | * Caller should also handle BPF_MOV case separately. |
| 6022 | * If we return -EACCES, caller may want to try again treating pointer as a |
| 6023 | * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. |
| 6024 | */ |
| 6025 | static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, |
| 6026 | struct bpf_insn *insn, |
| 6027 | const struct bpf_reg_state *ptr_reg, |
| 6028 | const struct bpf_reg_state *off_reg) |
| 6029 | { |
| 6030 | struct bpf_verifier_state *vstate = env->cur_state; |
| 6031 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 6032 | struct bpf_reg_state *regs = state->regs, *dst_reg; |
| 6033 | bool known = tnum_is_const(off_reg->var_off); |
| 6034 | s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, |
| 6035 | smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; |
| 6036 | u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, |
| 6037 | umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; |
| 6038 | u32 dst = insn->dst_reg, src = insn->src_reg; |
| 6039 | u8 opcode = BPF_OP(insn->code); |
| 6040 | int ret; |
| 6041 | |
| 6042 | dst_reg = ®s[dst]; |
| 6043 | |
| 6044 | if ((known && (smin_val != smax_val || umin_val != umax_val)) || |
| 6045 | smin_val > smax_val || umin_val > umax_val) { |
| 6046 | /* Taint dst register if offset had invalid bounds derived from |
| 6047 | * e.g. dead branches. |
| 6048 | */ |
| 6049 | __mark_reg_unknown(env, dst_reg); |
| 6050 | return 0; |
| 6051 | } |
| 6052 | |
| 6053 | if (BPF_CLASS(insn->code) != BPF_ALU64) { |
| 6054 | /* 32-bit ALU ops on pointers produce (meaningless) scalars */ |
| 6055 | if (opcode == BPF_SUB && env->allow_ptr_leaks) { |
| 6056 | __mark_reg_unknown(env, dst_reg); |
| 6057 | return 0; |
| 6058 | } |
| 6059 | |
| 6060 | verbose(env, |
| 6061 | "R%d 32-bit pointer arithmetic prohibited\n", |
| 6062 | dst); |
| 6063 | return -EACCES; |
| 6064 | } |
| 6065 | |
| 6066 | switch (ptr_reg->type) { |
| 6067 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 6068 | verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", |
| 6069 | dst, reg_type_str[ptr_reg->type]); |
| 6070 | return -EACCES; |
| 6071 | case CONST_PTR_TO_MAP: |
| 6072 | /* smin_val represents the known value */ |
| 6073 | if (known && smin_val == 0 && opcode == BPF_ADD) |
| 6074 | break; |
| 6075 | fallthrough; |
| 6076 | case PTR_TO_PACKET_END: |
| 6077 | case PTR_TO_SOCKET: |
| 6078 | case PTR_TO_SOCKET_OR_NULL: |
| 6079 | case PTR_TO_SOCK_COMMON: |
| 6080 | case PTR_TO_SOCK_COMMON_OR_NULL: |
| 6081 | case PTR_TO_TCP_SOCK: |
| 6082 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 6083 | case PTR_TO_XDP_SOCK: |
| 6084 | verbose(env, "R%d pointer arithmetic on %s prohibited\n", |
| 6085 | dst, reg_type_str[ptr_reg->type]); |
| 6086 | return -EACCES; |
| 6087 | case PTR_TO_MAP_VALUE: |
| 6088 | if (!env->env->bypass_spec_v1 && !known && (smin_val < 0) != (smax_val < 0)) { |
| 6089 | verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n", |
| 6090 | off_reg == dst_reg ? dst : src); |
| 6091 | return -EACCES; |
| 6092 | } |
| 6093 | fallthrough; |
| 6094 | default: |
| 6095 | break; |
| 6096 | } |
| 6097 | |
| 6098 | /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. |
| 6099 | * The id may be overwritten later if we create a new variable offset. |
| 6100 | */ |
| 6101 | dst_reg->type = ptr_reg->type; |
| 6102 | dst_reg->id = ptr_reg->id; |
| 6103 | |
| 6104 | if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || |
| 6105 | !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) |
| 6106 | return -EINVAL; |
| 6107 | |
| 6108 | /* pointer types do not carry 32-bit bounds at the moment. */ |
| 6109 | __mark_reg32_unbounded(dst_reg); |
| 6110 | |
| 6111 | switch (opcode) { |
| 6112 | case BPF_ADD: |
| 6113 | ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); |
| 6114 | if (ret < 0) { |
| 6115 | verbose(env, "R%d tried to add from different maps, paths, or prohibited types\n", dst); |
| 6116 | return ret; |
| 6117 | } |
| 6118 | /* We can take a fixed offset as long as it doesn't overflow |
| 6119 | * the s32 'off' field |
| 6120 | */ |
| 6121 | if (known && (ptr_reg->off + smin_val == |
| 6122 | (s64)(s32)(ptr_reg->off + smin_val))) { |
| 6123 | /* pointer += K. Accumulate it into fixed offset */ |
| 6124 | dst_reg->smin_value = smin_ptr; |
| 6125 | dst_reg->smax_value = smax_ptr; |
| 6126 | dst_reg->umin_value = umin_ptr; |
| 6127 | dst_reg->umax_value = umax_ptr; |
| 6128 | dst_reg->var_off = ptr_reg->var_off; |
| 6129 | dst_reg->off = ptr_reg->off + smin_val; |
| 6130 | dst_reg->raw = ptr_reg->raw; |
| 6131 | break; |
| 6132 | } |
| 6133 | /* A new variable offset is created. Note that off_reg->off |
| 6134 | * == 0, since it's a scalar. |
| 6135 | * dst_reg gets the pointer type and since some positive |
| 6136 | * integer value was added to the pointer, give it a new 'id' |
| 6137 | * if it's a PTR_TO_PACKET. |
| 6138 | * this creates a new 'base' pointer, off_reg (variable) gets |
| 6139 | * added into the variable offset, and we copy the fixed offset |
| 6140 | * from ptr_reg. |
| 6141 | */ |
| 6142 | if (signed_add_overflows(smin_ptr, smin_val) || |
| 6143 | signed_add_overflows(smax_ptr, smax_val)) { |
| 6144 | dst_reg->smin_value = S64_MIN; |
| 6145 | dst_reg->smax_value = S64_MAX; |
| 6146 | } else { |
| 6147 | dst_reg->smin_value = smin_ptr + smin_val; |
| 6148 | dst_reg->smax_value = smax_ptr + smax_val; |
| 6149 | } |
| 6150 | if (umin_ptr + umin_val < umin_ptr || |
| 6151 | umax_ptr + umax_val < umax_ptr) { |
| 6152 | dst_reg->umin_value = 0; |
| 6153 | dst_reg->umax_value = U64_MAX; |
| 6154 | } else { |
| 6155 | dst_reg->umin_value = umin_ptr + umin_val; |
| 6156 | dst_reg->umax_value = umax_ptr + umax_val; |
| 6157 | } |
| 6158 | dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); |
| 6159 | dst_reg->off = ptr_reg->off; |
| 6160 | dst_reg->raw = ptr_reg->raw; |
| 6161 | if (reg_is_pkt_pointer(ptr_reg)) { |
| 6162 | dst_reg->id = ++env->id_gen; |
| 6163 | /* something was added to pkt_ptr, set range to zero */ |
| 6164 | memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); |
| 6165 | } |
| 6166 | break; |
| 6167 | case BPF_SUB: |
| 6168 | ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); |
| 6169 | if (ret < 0) { |
| 6170 | verbose(env, "R%d tried to sub from different maps, paths, or prohibited types\n", dst); |
| 6171 | return ret; |
| 6172 | } |
| 6173 | if (dst_reg == off_reg) { |
| 6174 | /* scalar -= pointer. Creates an unknown scalar */ |
| 6175 | verbose(env, "R%d tried to subtract pointer from scalar\n", |
| 6176 | dst); |
| 6177 | return -EACCES; |
| 6178 | } |
| 6179 | /* We don't allow subtraction from FP, because (according to |
| 6180 | * test_verifier.c test "invalid fp arithmetic", JITs might not |
| 6181 | * be able to deal with it. |
| 6182 | */ |
| 6183 | if (ptr_reg->type == PTR_TO_STACK) { |
| 6184 | verbose(env, "R%d subtraction from stack pointer prohibited\n", |
| 6185 | dst); |
| 6186 | return -EACCES; |
| 6187 | } |
| 6188 | if (known && (ptr_reg->off - smin_val == |
| 6189 | (s64)(s32)(ptr_reg->off - smin_val))) { |
| 6190 | /* pointer -= K. Subtract it from fixed offset */ |
| 6191 | dst_reg->smin_value = smin_ptr; |
| 6192 | dst_reg->smax_value = smax_ptr; |
| 6193 | dst_reg->umin_value = umin_ptr; |
| 6194 | dst_reg->umax_value = umax_ptr; |
| 6195 | dst_reg->var_off = ptr_reg->var_off; |
| 6196 | dst_reg->id = ptr_reg->id; |
| 6197 | dst_reg->off = ptr_reg->off - smin_val; |
| 6198 | dst_reg->raw = ptr_reg->raw; |
| 6199 | break; |
| 6200 | } |
| 6201 | /* A new variable offset is created. If the subtrahend is known |
| 6202 | * nonnegative, then any reg->range we had before is still good. |
| 6203 | */ |
| 6204 | if (signed_sub_overflows(smin_ptr, smax_val) || |
| 6205 | signed_sub_overflows(smax_ptr, smin_val)) { |
| 6206 | /* Overflow possible, we know nothing */ |
| 6207 | dst_reg->smin_value = S64_MIN; |
| 6208 | dst_reg->smax_value = S64_MAX; |
| 6209 | } else { |
| 6210 | dst_reg->smin_value = smin_ptr - smax_val; |
| 6211 | dst_reg->smax_value = smax_ptr - smin_val; |
| 6212 | } |
| 6213 | if (umin_ptr < umax_val) { |
| 6214 | /* Overflow possible, we know nothing */ |
| 6215 | dst_reg->umin_value = 0; |
| 6216 | dst_reg->umax_value = U64_MAX; |
| 6217 | } else { |
| 6218 | /* Cannot overflow (as long as bounds are consistent) */ |
| 6219 | dst_reg->umin_value = umin_ptr - umax_val; |
| 6220 | dst_reg->umax_value = umax_ptr - umin_val; |
| 6221 | } |
| 6222 | dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); |
| 6223 | dst_reg->off = ptr_reg->off; |
| 6224 | dst_reg->raw = ptr_reg->raw; |
| 6225 | if (reg_is_pkt_pointer(ptr_reg)) { |
| 6226 | dst_reg->id = ++env->id_gen; |
| 6227 | /* something was added to pkt_ptr, set range to zero */ |
| 6228 | if (smin_val < 0) |
| 6229 | memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); |
| 6230 | } |
| 6231 | break; |
| 6232 | case BPF_AND: |
| 6233 | case BPF_OR: |
| 6234 | case BPF_XOR: |
| 6235 | /* bitwise ops on pointers are troublesome, prohibit. */ |
| 6236 | verbose(env, "R%d bitwise operator %s on pointer prohibited\n", |
| 6237 | dst, bpf_alu_string[opcode >> 4]); |
| 6238 | return -EACCES; |
| 6239 | default: |
| 6240 | /* other operators (e.g. MUL,LSH) produce non-pointer results */ |
| 6241 | verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", |
| 6242 | dst, bpf_alu_string[opcode >> 4]); |
| 6243 | return -EACCES; |
| 6244 | } |
| 6245 | |
| 6246 | if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) |
| 6247 | return -EINVAL; |
| 6248 | |
| 6249 | __update_reg_bounds(dst_reg); |
| 6250 | __reg_deduce_bounds(dst_reg); |
| 6251 | __reg_bound_offset(dst_reg); |
| 6252 | |
| 6253 | /* For unprivileged we require that resulting offset must be in bounds |
| 6254 | * in order to be able to sanitize access later on. |
| 6255 | */ |
| 6256 | if (!env->bypass_spec_v1) { |
| 6257 | if (dst_reg->type == PTR_TO_MAP_VALUE && |
| 6258 | check_map_access(env, dst, dst_reg->off, 1, false)) { |
| 6259 | verbose(env, "R%d pointer arithmetic of map value goes out of range, " |
| 6260 | "prohibited for !root\n", dst); |
| 6261 | return -EACCES; |
| 6262 | } else if (dst_reg->type == PTR_TO_STACK && |
| 6263 | check_stack_access_for_ptr_arithmetic( |
| 6264 | env, dst, dst_reg, dst_reg->off + |
| 6265 | dst_reg->var_off.value)) { |
| 6266 | return -EACCES; |
| 6267 | } |
| 6268 | } |
| 6269 | |
| 6270 | return 0; |
| 6271 | } |
| 6272 | |
| 6273 | static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, |
| 6274 | struct bpf_reg_state *src_reg) |
| 6275 | { |
| 6276 | s32 smin_val = src_reg->s32_min_value; |
| 6277 | s32 smax_val = src_reg->s32_max_value; |
| 6278 | u32 umin_val = src_reg->u32_min_value; |
| 6279 | u32 umax_val = src_reg->u32_max_value; |
| 6280 | |
| 6281 | if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || |
| 6282 | signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { |
| 6283 | dst_reg->s32_min_value = S32_MIN; |
| 6284 | dst_reg->s32_max_value = S32_MAX; |
| 6285 | } else { |
| 6286 | dst_reg->s32_min_value += smin_val; |
| 6287 | dst_reg->s32_max_value += smax_val; |
| 6288 | } |
| 6289 | if (dst_reg->u32_min_value + umin_val < umin_val || |
| 6290 | dst_reg->u32_max_value + umax_val < umax_val) { |
| 6291 | dst_reg->u32_min_value = 0; |
| 6292 | dst_reg->u32_max_value = U32_MAX; |
| 6293 | } else { |
| 6294 | dst_reg->u32_min_value += umin_val; |
| 6295 | dst_reg->u32_max_value += umax_val; |
| 6296 | } |
| 6297 | } |
| 6298 | |
| 6299 | static void scalar_min_max_add(struct bpf_reg_state *dst_reg, |
| 6300 | struct bpf_reg_state *src_reg) |
| 6301 | { |
| 6302 | s64 smin_val = src_reg->smin_value; |
| 6303 | s64 smax_val = src_reg->smax_value; |
| 6304 | u64 umin_val = src_reg->umin_value; |
| 6305 | u64 umax_val = src_reg->umax_value; |
| 6306 | |
| 6307 | if (signed_add_overflows(dst_reg->smin_value, smin_val) || |
| 6308 | signed_add_overflows(dst_reg->smax_value, smax_val)) { |
| 6309 | dst_reg->smin_value = S64_MIN; |
| 6310 | dst_reg->smax_value = S64_MAX; |
| 6311 | } else { |
| 6312 | dst_reg->smin_value += smin_val; |
| 6313 | dst_reg->smax_value += smax_val; |
| 6314 | } |
| 6315 | if (dst_reg->umin_value + umin_val < umin_val || |
| 6316 | dst_reg->umax_value + umax_val < umax_val) { |
| 6317 | dst_reg->umin_value = 0; |
| 6318 | dst_reg->umax_value = U64_MAX; |
| 6319 | } else { |
| 6320 | dst_reg->umin_value += umin_val; |
| 6321 | dst_reg->umax_value += umax_val; |
| 6322 | } |
| 6323 | } |
| 6324 | |
| 6325 | static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, |
| 6326 | struct bpf_reg_state *src_reg) |
| 6327 | { |
| 6328 | s32 smin_val = src_reg->s32_min_value; |
| 6329 | s32 smax_val = src_reg->s32_max_value; |
| 6330 | u32 umin_val = src_reg->u32_min_value; |
| 6331 | u32 umax_val = src_reg->u32_max_value; |
| 6332 | |
| 6333 | if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || |
| 6334 | signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { |
| 6335 | /* Overflow possible, we know nothing */ |
| 6336 | dst_reg->s32_min_value = S32_MIN; |
| 6337 | dst_reg->s32_max_value = S32_MAX; |
| 6338 | } else { |
| 6339 | dst_reg->s32_min_value -= smax_val; |
| 6340 | dst_reg->s32_max_value -= smin_val; |
| 6341 | } |
| 6342 | if (dst_reg->u32_min_value < umax_val) { |
| 6343 | /* Overflow possible, we know nothing */ |
| 6344 | dst_reg->u32_min_value = 0; |
| 6345 | dst_reg->u32_max_value = U32_MAX; |
| 6346 | } else { |
| 6347 | /* Cannot overflow (as long as bounds are consistent) */ |
| 6348 | dst_reg->u32_min_value -= umax_val; |
| 6349 | dst_reg->u32_max_value -= umin_val; |
| 6350 | } |
| 6351 | } |
| 6352 | |
| 6353 | static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, |
| 6354 | struct bpf_reg_state *src_reg) |
| 6355 | { |
| 6356 | s64 smin_val = src_reg->smin_value; |
| 6357 | s64 smax_val = src_reg->smax_value; |
| 6358 | u64 umin_val = src_reg->umin_value; |
| 6359 | u64 umax_val = src_reg->umax_value; |
| 6360 | |
| 6361 | if (signed_sub_overflows(dst_reg->smin_value, smax_val) || |
| 6362 | signed_sub_overflows(dst_reg->smax_value, smin_val)) { |
| 6363 | /* Overflow possible, we know nothing */ |
| 6364 | dst_reg->smin_value = S64_MIN; |
| 6365 | dst_reg->smax_value = S64_MAX; |
| 6366 | } else { |
| 6367 | dst_reg->smin_value -= smax_val; |
| 6368 | dst_reg->smax_value -= smin_val; |
| 6369 | } |
| 6370 | if (dst_reg->umin_value < umax_val) { |
| 6371 | /* Overflow possible, we know nothing */ |
| 6372 | dst_reg->umin_value = 0; |
| 6373 | dst_reg->umax_value = U64_MAX; |
| 6374 | } else { |
| 6375 | /* Cannot overflow (as long as bounds are consistent) */ |
| 6376 | dst_reg->umin_value -= umax_val; |
| 6377 | dst_reg->umax_value -= umin_val; |
| 6378 | } |
| 6379 | } |
| 6380 | |
| 6381 | static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, |
| 6382 | struct bpf_reg_state *src_reg) |
| 6383 | { |
| 6384 | s32 smin_val = src_reg->s32_min_value; |
| 6385 | u32 umin_val = src_reg->u32_min_value; |
| 6386 | u32 umax_val = src_reg->u32_max_value; |
| 6387 | |
| 6388 | if (smin_val < 0 || dst_reg->s32_min_value < 0) { |
| 6389 | /* Ain't nobody got time to multiply that sign */ |
| 6390 | __mark_reg32_unbounded(dst_reg); |
| 6391 | return; |
| 6392 | } |
| 6393 | /* Both values are positive, so we can work with unsigned and |
| 6394 | * copy the result to signed (unless it exceeds S32_MAX). |
| 6395 | */ |
| 6396 | if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { |
| 6397 | /* Potential overflow, we know nothing */ |
| 6398 | __mark_reg32_unbounded(dst_reg); |
| 6399 | return; |
| 6400 | } |
| 6401 | dst_reg->u32_min_value *= umin_val; |
| 6402 | dst_reg->u32_max_value *= umax_val; |
| 6403 | if (dst_reg->u32_max_value > S32_MAX) { |
| 6404 | /* Overflow possible, we know nothing */ |
| 6405 | dst_reg->s32_min_value = S32_MIN; |
| 6406 | dst_reg->s32_max_value = S32_MAX; |
| 6407 | } else { |
| 6408 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 6409 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
| 6410 | } |
| 6411 | } |
| 6412 | |
| 6413 | static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, |
| 6414 | struct bpf_reg_state *src_reg) |
| 6415 | { |
| 6416 | s64 smin_val = src_reg->smin_value; |
| 6417 | u64 umin_val = src_reg->umin_value; |
| 6418 | u64 umax_val = src_reg->umax_value; |
| 6419 | |
| 6420 | if (smin_val < 0 || dst_reg->smin_value < 0) { |
| 6421 | /* Ain't nobody got time to multiply that sign */ |
| 6422 | __mark_reg64_unbounded(dst_reg); |
| 6423 | return; |
| 6424 | } |
| 6425 | /* Both values are positive, so we can work with unsigned and |
| 6426 | * copy the result to signed (unless it exceeds S64_MAX). |
| 6427 | */ |
| 6428 | if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { |
| 6429 | /* Potential overflow, we know nothing */ |
| 6430 | __mark_reg64_unbounded(dst_reg); |
| 6431 | return; |
| 6432 | } |
| 6433 | dst_reg->umin_value *= umin_val; |
| 6434 | dst_reg->umax_value *= umax_val; |
| 6435 | if (dst_reg->umax_value > S64_MAX) { |
| 6436 | /* Overflow possible, we know nothing */ |
| 6437 | dst_reg->smin_value = S64_MIN; |
| 6438 | dst_reg->smax_value = S64_MAX; |
| 6439 | } else { |
| 6440 | dst_reg->smin_value = dst_reg->umin_value; |
| 6441 | dst_reg->smax_value = dst_reg->umax_value; |
| 6442 | } |
| 6443 | } |
| 6444 | |
| 6445 | static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, |
| 6446 | struct bpf_reg_state *src_reg) |
| 6447 | { |
| 6448 | bool src_known = tnum_subreg_is_const(src_reg->var_off); |
| 6449 | bool dst_known = tnum_subreg_is_const(dst_reg->var_off); |
| 6450 | struct tnum var32_off = tnum_subreg(dst_reg->var_off); |
| 6451 | s32 smin_val = src_reg->s32_min_value; |
| 6452 | u32 umax_val = src_reg->u32_max_value; |
| 6453 | |
| 6454 | /* Assuming scalar64_min_max_and will be called so its safe |
| 6455 | * to skip updating register for known 32-bit case. |
| 6456 | */ |
| 6457 | if (src_known && dst_known) |
| 6458 | return; |
| 6459 | |
| 6460 | /* We get our minimum from the var_off, since that's inherently |
| 6461 | * bitwise. Our maximum is the minimum of the operands' maxima. |
| 6462 | */ |
| 6463 | dst_reg->u32_min_value = var32_off.value; |
| 6464 | dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); |
| 6465 | if (dst_reg->s32_min_value < 0 || smin_val < 0) { |
| 6466 | /* Lose signed bounds when ANDing negative numbers, |
| 6467 | * ain't nobody got time for that. |
| 6468 | */ |
| 6469 | dst_reg->s32_min_value = S32_MIN; |
| 6470 | dst_reg->s32_max_value = S32_MAX; |
| 6471 | } else { |
| 6472 | /* ANDing two positives gives a positive, so safe to |
| 6473 | * cast result into s64. |
| 6474 | */ |
| 6475 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 6476 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
| 6477 | } |
| 6478 | |
| 6479 | } |
| 6480 | |
| 6481 | static void scalar_min_max_and(struct bpf_reg_state *dst_reg, |
| 6482 | struct bpf_reg_state *src_reg) |
| 6483 | { |
| 6484 | bool src_known = tnum_is_const(src_reg->var_off); |
| 6485 | bool dst_known = tnum_is_const(dst_reg->var_off); |
| 6486 | s64 smin_val = src_reg->smin_value; |
| 6487 | u64 umax_val = src_reg->umax_value; |
| 6488 | |
| 6489 | if (src_known && dst_known) { |
| 6490 | __mark_reg_known(dst_reg, dst_reg->var_off.value); |
| 6491 | return; |
| 6492 | } |
| 6493 | |
| 6494 | /* We get our minimum from the var_off, since that's inherently |
| 6495 | * bitwise. Our maximum is the minimum of the operands' maxima. |
| 6496 | */ |
| 6497 | dst_reg->umin_value = dst_reg->var_off.value; |
| 6498 | dst_reg->umax_value = min(dst_reg->umax_value, umax_val); |
| 6499 | if (dst_reg->smin_value < 0 || smin_val < 0) { |
| 6500 | /* Lose signed bounds when ANDing negative numbers, |
| 6501 | * ain't nobody got time for that. |
| 6502 | */ |
| 6503 | dst_reg->smin_value = S64_MIN; |
| 6504 | dst_reg->smax_value = S64_MAX; |
| 6505 | } else { |
| 6506 | /* ANDing two positives gives a positive, so safe to |
| 6507 | * cast result into s64. |
| 6508 | */ |
| 6509 | dst_reg->smin_value = dst_reg->umin_value; |
| 6510 | dst_reg->smax_value = dst_reg->umax_value; |
| 6511 | } |
| 6512 | /* We may learn something more from the var_off */ |
| 6513 | __update_reg_bounds(dst_reg); |
| 6514 | } |
| 6515 | |
| 6516 | static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, |
| 6517 | struct bpf_reg_state *src_reg) |
| 6518 | { |
| 6519 | bool src_known = tnum_subreg_is_const(src_reg->var_off); |
| 6520 | bool dst_known = tnum_subreg_is_const(dst_reg->var_off); |
| 6521 | struct tnum var32_off = tnum_subreg(dst_reg->var_off); |
| 6522 | s32 smin_val = src_reg->s32_min_value; |
| 6523 | u32 umin_val = src_reg->u32_min_value; |
| 6524 | |
| 6525 | /* Assuming scalar64_min_max_or will be called so it is safe |
| 6526 | * to skip updating register for known case. |
| 6527 | */ |
| 6528 | if (src_known && dst_known) |
| 6529 | return; |
| 6530 | |
| 6531 | /* We get our maximum from the var_off, and our minimum is the |
| 6532 | * maximum of the operands' minima |
| 6533 | */ |
| 6534 | dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); |
| 6535 | dst_reg->u32_max_value = var32_off.value | var32_off.mask; |
| 6536 | if (dst_reg->s32_min_value < 0 || smin_val < 0) { |
| 6537 | /* Lose signed bounds when ORing negative numbers, |
| 6538 | * ain't nobody got time for that. |
| 6539 | */ |
| 6540 | dst_reg->s32_min_value = S32_MIN; |
| 6541 | dst_reg->s32_max_value = S32_MAX; |
| 6542 | } else { |
| 6543 | /* ORing two positives gives a positive, so safe to |
| 6544 | * cast result into s64. |
| 6545 | */ |
| 6546 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 6547 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
| 6548 | } |
| 6549 | } |
| 6550 | |
| 6551 | static void scalar_min_max_or(struct bpf_reg_state *dst_reg, |
| 6552 | struct bpf_reg_state *src_reg) |
| 6553 | { |
| 6554 | bool src_known = tnum_is_const(src_reg->var_off); |
| 6555 | bool dst_known = tnum_is_const(dst_reg->var_off); |
| 6556 | s64 smin_val = src_reg->smin_value; |
| 6557 | u64 umin_val = src_reg->umin_value; |
| 6558 | |
| 6559 | if (src_known && dst_known) { |
| 6560 | __mark_reg_known(dst_reg, dst_reg->var_off.value); |
| 6561 | return; |
| 6562 | } |
| 6563 | |
| 6564 | /* We get our maximum from the var_off, and our minimum is the |
| 6565 | * maximum of the operands' minima |
| 6566 | */ |
| 6567 | dst_reg->umin_value = max(dst_reg->umin_value, umin_val); |
| 6568 | dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; |
| 6569 | if (dst_reg->smin_value < 0 || smin_val < 0) { |
| 6570 | /* Lose signed bounds when ORing negative numbers, |
| 6571 | * ain't nobody got time for that. |
| 6572 | */ |
| 6573 | dst_reg->smin_value = S64_MIN; |
| 6574 | dst_reg->smax_value = S64_MAX; |
| 6575 | } else { |
| 6576 | /* ORing two positives gives a positive, so safe to |
| 6577 | * cast result into s64. |
| 6578 | */ |
| 6579 | dst_reg->smin_value = dst_reg->umin_value; |
| 6580 | dst_reg->smax_value = dst_reg->umax_value; |
| 6581 | } |
| 6582 | /* We may learn something more from the var_off */ |
| 6583 | __update_reg_bounds(dst_reg); |
| 6584 | } |
| 6585 | |
| 6586 | static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, |
| 6587 | struct bpf_reg_state *src_reg) |
| 6588 | { |
| 6589 | bool src_known = tnum_subreg_is_const(src_reg->var_off); |
| 6590 | bool dst_known = tnum_subreg_is_const(dst_reg->var_off); |
| 6591 | struct tnum var32_off = tnum_subreg(dst_reg->var_off); |
| 6592 | s32 smin_val = src_reg->s32_min_value; |
| 6593 | |
| 6594 | /* Assuming scalar64_min_max_xor will be called so it is safe |
| 6595 | * to skip updating register for known case. |
| 6596 | */ |
| 6597 | if (src_known && dst_known) |
| 6598 | return; |
| 6599 | |
| 6600 | /* We get both minimum and maximum from the var32_off. */ |
| 6601 | dst_reg->u32_min_value = var32_off.value; |
| 6602 | dst_reg->u32_max_value = var32_off.value | var32_off.mask; |
| 6603 | |
| 6604 | if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { |
| 6605 | /* XORing two positive sign numbers gives a positive, |
| 6606 | * so safe to cast u32 result into s32. |
| 6607 | */ |
| 6608 | dst_reg->s32_min_value = dst_reg->u32_min_value; |
| 6609 | dst_reg->s32_max_value = dst_reg->u32_max_value; |
| 6610 | } else { |
| 6611 | dst_reg->s32_min_value = S32_MIN; |
| 6612 | dst_reg->s32_max_value = S32_MAX; |
| 6613 | } |
| 6614 | } |
| 6615 | |
| 6616 | static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, |
| 6617 | struct bpf_reg_state *src_reg) |
| 6618 | { |
| 6619 | bool src_known = tnum_is_const(src_reg->var_off); |
| 6620 | bool dst_known = tnum_is_const(dst_reg->var_off); |
| 6621 | s64 smin_val = src_reg->smin_value; |
| 6622 | |
| 6623 | if (src_known && dst_known) { |
| 6624 | /* dst_reg->var_off.value has been updated earlier */ |
| 6625 | __mark_reg_known(dst_reg, dst_reg->var_off.value); |
| 6626 | return; |
| 6627 | } |
| 6628 | |
| 6629 | /* We get both minimum and maximum from the var_off. */ |
| 6630 | dst_reg->umin_value = dst_reg->var_off.value; |
| 6631 | dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; |
| 6632 | |
| 6633 | if (dst_reg->smin_value >= 0 && smin_val >= 0) { |
| 6634 | /* XORing two positive sign numbers gives a positive, |
| 6635 | * so safe to cast u64 result into s64. |
| 6636 | */ |
| 6637 | dst_reg->smin_value = dst_reg->umin_value; |
| 6638 | dst_reg->smax_value = dst_reg->umax_value; |
| 6639 | } else { |
| 6640 | dst_reg->smin_value = S64_MIN; |
| 6641 | dst_reg->smax_value = S64_MAX; |
| 6642 | } |
| 6643 | |
| 6644 | __update_reg_bounds(dst_reg); |
| 6645 | } |
| 6646 | |
| 6647 | static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 6648 | u64 umin_val, u64 umax_val) |
| 6649 | { |
| 6650 | /* We lose all sign bit information (except what we can pick |
| 6651 | * up from var_off) |
| 6652 | */ |
| 6653 | dst_reg->s32_min_value = S32_MIN; |
| 6654 | dst_reg->s32_max_value = S32_MAX; |
| 6655 | /* If we might shift our top bit out, then we know nothing */ |
| 6656 | if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { |
| 6657 | dst_reg->u32_min_value = 0; |
| 6658 | dst_reg->u32_max_value = U32_MAX; |
| 6659 | } else { |
| 6660 | dst_reg->u32_min_value <<= umin_val; |
| 6661 | dst_reg->u32_max_value <<= umax_val; |
| 6662 | } |
| 6663 | } |
| 6664 | |
| 6665 | static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 6666 | struct bpf_reg_state *src_reg) |
| 6667 | { |
| 6668 | u32 umax_val = src_reg->u32_max_value; |
| 6669 | u32 umin_val = src_reg->u32_min_value; |
| 6670 | /* u32 alu operation will zext upper bits */ |
| 6671 | struct tnum subreg = tnum_subreg(dst_reg->var_off); |
| 6672 | |
| 6673 | __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); |
| 6674 | dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); |
| 6675 | /* Not required but being careful mark reg64 bounds as unknown so |
| 6676 | * that we are forced to pick them up from tnum and zext later and |
| 6677 | * if some path skips this step we are still safe. |
| 6678 | */ |
| 6679 | __mark_reg64_unbounded(dst_reg); |
| 6680 | __update_reg32_bounds(dst_reg); |
| 6681 | } |
| 6682 | |
| 6683 | static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 6684 | u64 umin_val, u64 umax_val) |
| 6685 | { |
| 6686 | /* Special case <<32 because it is a common compiler pattern to sign |
| 6687 | * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are |
| 6688 | * positive we know this shift will also be positive so we can track |
| 6689 | * bounds correctly. Otherwise we lose all sign bit information except |
| 6690 | * what we can pick up from var_off. Perhaps we can generalize this |
| 6691 | * later to shifts of any length. |
| 6692 | */ |
| 6693 | if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) |
| 6694 | dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; |
| 6695 | else |
| 6696 | dst_reg->smax_value = S64_MAX; |
| 6697 | |
| 6698 | if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) |
| 6699 | dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; |
| 6700 | else |
| 6701 | dst_reg->smin_value = S64_MIN; |
| 6702 | |
| 6703 | /* If we might shift our top bit out, then we know nothing */ |
| 6704 | if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { |
| 6705 | dst_reg->umin_value = 0; |
| 6706 | dst_reg->umax_value = U64_MAX; |
| 6707 | } else { |
| 6708 | dst_reg->umin_value <<= umin_val; |
| 6709 | dst_reg->umax_value <<= umax_val; |
| 6710 | } |
| 6711 | } |
| 6712 | |
| 6713 | static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, |
| 6714 | struct bpf_reg_state *src_reg) |
| 6715 | { |
| 6716 | u64 umax_val = src_reg->umax_value; |
| 6717 | u64 umin_val = src_reg->umin_value; |
| 6718 | |
| 6719 | /* scalar64 calc uses 32bit unshifted bounds so must be called first */ |
| 6720 | __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); |
| 6721 | __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); |
| 6722 | |
| 6723 | dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); |
| 6724 | /* We may learn something more from the var_off */ |
| 6725 | __update_reg_bounds(dst_reg); |
| 6726 | } |
| 6727 | |
| 6728 | static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, |
| 6729 | struct bpf_reg_state *src_reg) |
| 6730 | { |
| 6731 | struct tnum subreg = tnum_subreg(dst_reg->var_off); |
| 6732 | u32 umax_val = src_reg->u32_max_value; |
| 6733 | u32 umin_val = src_reg->u32_min_value; |
| 6734 | |
| 6735 | /* BPF_RSH is an unsigned shift. If the value in dst_reg might |
| 6736 | * be negative, then either: |
| 6737 | * 1) src_reg might be zero, so the sign bit of the result is |
| 6738 | * unknown, so we lose our signed bounds |
| 6739 | * 2) it's known negative, thus the unsigned bounds capture the |
| 6740 | * signed bounds |
| 6741 | * 3) the signed bounds cross zero, so they tell us nothing |
| 6742 | * about the result |
| 6743 | * If the value in dst_reg is known nonnegative, then again the |
| 6744 | * unsigned bounds capture the signed bounds. |
| 6745 | * Thus, in all cases it suffices to blow away our signed bounds |
| 6746 | * and rely on inferring new ones from the unsigned bounds and |
| 6747 | * var_off of the result. |
| 6748 | */ |
| 6749 | dst_reg->s32_min_value = S32_MIN; |
| 6750 | dst_reg->s32_max_value = S32_MAX; |
| 6751 | |
| 6752 | dst_reg->var_off = tnum_rshift(subreg, umin_val); |
| 6753 | dst_reg->u32_min_value >>= umax_val; |
| 6754 | dst_reg->u32_max_value >>= umin_val; |
| 6755 | |
| 6756 | __mark_reg64_unbounded(dst_reg); |
| 6757 | __update_reg32_bounds(dst_reg); |
| 6758 | } |
| 6759 | |
| 6760 | static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, |
| 6761 | struct bpf_reg_state *src_reg) |
| 6762 | { |
| 6763 | u64 umax_val = src_reg->umax_value; |
| 6764 | u64 umin_val = src_reg->umin_value; |
| 6765 | |
| 6766 | /* BPF_RSH is an unsigned shift. If the value in dst_reg might |
| 6767 | * be negative, then either: |
| 6768 | * 1) src_reg might be zero, so the sign bit of the result is |
| 6769 | * unknown, so we lose our signed bounds |
| 6770 | * 2) it's known negative, thus the unsigned bounds capture the |
| 6771 | * signed bounds |
| 6772 | * 3) the signed bounds cross zero, so they tell us nothing |
| 6773 | * about the result |
| 6774 | * If the value in dst_reg is known nonnegative, then again the |
| 6775 | * unsigned bounds capture the signed bounds. |
| 6776 | * Thus, in all cases it suffices to blow away our signed bounds |
| 6777 | * and rely on inferring new ones from the unsigned bounds and |
| 6778 | * var_off of the result. |
| 6779 | */ |
| 6780 | dst_reg->smin_value = S64_MIN; |
| 6781 | dst_reg->smax_value = S64_MAX; |
| 6782 | dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); |
| 6783 | dst_reg->umin_value >>= umax_val; |
| 6784 | dst_reg->umax_value >>= umin_val; |
| 6785 | |
| 6786 | /* Its not easy to operate on alu32 bounds here because it depends |
| 6787 | * on bits being shifted in. Take easy way out and mark unbounded |
| 6788 | * so we can recalculate later from tnum. |
| 6789 | */ |
| 6790 | __mark_reg32_unbounded(dst_reg); |
| 6791 | __update_reg_bounds(dst_reg); |
| 6792 | } |
| 6793 | |
| 6794 | static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, |
| 6795 | struct bpf_reg_state *src_reg) |
| 6796 | { |
| 6797 | u64 umin_val = src_reg->u32_min_value; |
| 6798 | |
| 6799 | /* Upon reaching here, src_known is true and |
| 6800 | * umax_val is equal to umin_val. |
| 6801 | */ |
| 6802 | dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); |
| 6803 | dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); |
| 6804 | |
| 6805 | dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); |
| 6806 | |
| 6807 | /* blow away the dst_reg umin_value/umax_value and rely on |
| 6808 | * dst_reg var_off to refine the result. |
| 6809 | */ |
| 6810 | dst_reg->u32_min_value = 0; |
| 6811 | dst_reg->u32_max_value = U32_MAX; |
| 6812 | |
| 6813 | __mark_reg64_unbounded(dst_reg); |
| 6814 | __update_reg32_bounds(dst_reg); |
| 6815 | } |
| 6816 | |
| 6817 | static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, |
| 6818 | struct bpf_reg_state *src_reg) |
| 6819 | { |
| 6820 | u64 umin_val = src_reg->umin_value; |
| 6821 | |
| 6822 | /* Upon reaching here, src_known is true and umax_val is equal |
| 6823 | * to umin_val. |
| 6824 | */ |
| 6825 | dst_reg->smin_value >>= umin_val; |
| 6826 | dst_reg->smax_value >>= umin_val; |
| 6827 | |
| 6828 | dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); |
| 6829 | |
| 6830 | /* blow away the dst_reg umin_value/umax_value and rely on |
| 6831 | * dst_reg var_off to refine the result. |
| 6832 | */ |
| 6833 | dst_reg->umin_value = 0; |
| 6834 | dst_reg->umax_value = U64_MAX; |
| 6835 | |
| 6836 | /* Its not easy to operate on alu32 bounds here because it depends |
| 6837 | * on bits being shifted in from upper 32-bits. Take easy way out |
| 6838 | * and mark unbounded so we can recalculate later from tnum. |
| 6839 | */ |
| 6840 | __mark_reg32_unbounded(dst_reg); |
| 6841 | __update_reg_bounds(dst_reg); |
| 6842 | } |
| 6843 | |
| 6844 | /* WARNING: This function does calculations on 64-bit values, but the actual |
| 6845 | * execution may occur on 32-bit values. Therefore, things like bitshifts |
| 6846 | * need extra checks in the 32-bit case. |
| 6847 | */ |
| 6848 | static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, |
| 6849 | struct bpf_insn *insn, |
| 6850 | struct bpf_reg_state *dst_reg, |
| 6851 | struct bpf_reg_state src_reg) |
| 6852 | { |
| 6853 | struct bpf_reg_state *regs = cur_regs(env); |
| 6854 | u8 opcode = BPF_OP(insn->code); |
| 6855 | bool src_known; |
| 6856 | s64 smin_val, smax_val; |
| 6857 | u64 umin_val, umax_val; |
| 6858 | s32 s32_min_val, s32_max_val; |
| 6859 | u32 u32_min_val, u32_max_val; |
| 6860 | u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; |
| 6861 | u32 dst = insn->dst_reg; |
| 6862 | int ret; |
| 6863 | bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); |
| 6864 | |
| 6865 | smin_val = src_reg.smin_value; |
| 6866 | smax_val = src_reg.smax_value; |
| 6867 | umin_val = src_reg.umin_value; |
| 6868 | umax_val = src_reg.umax_value; |
| 6869 | |
| 6870 | s32_min_val = src_reg.s32_min_value; |
| 6871 | s32_max_val = src_reg.s32_max_value; |
| 6872 | u32_min_val = src_reg.u32_min_value; |
| 6873 | u32_max_val = src_reg.u32_max_value; |
| 6874 | |
| 6875 | if (alu32) { |
| 6876 | src_known = tnum_subreg_is_const(src_reg.var_off); |
| 6877 | if ((src_known && |
| 6878 | (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) || |
| 6879 | s32_min_val > s32_max_val || u32_min_val > u32_max_val) { |
| 6880 | /* Taint dst register if offset had invalid bounds |
| 6881 | * derived from e.g. dead branches. |
| 6882 | */ |
| 6883 | __mark_reg_unknown(env, dst_reg); |
| 6884 | return 0; |
| 6885 | } |
| 6886 | } else { |
| 6887 | src_known = tnum_is_const(src_reg.var_off); |
| 6888 | if ((src_known && |
| 6889 | (smin_val != smax_val || umin_val != umax_val)) || |
| 6890 | smin_val > smax_val || umin_val > umax_val) { |
| 6891 | /* Taint dst register if offset had invalid bounds |
| 6892 | * derived from e.g. dead branches. |
| 6893 | */ |
| 6894 | __mark_reg_unknown(env, dst_reg); |
| 6895 | return 0; |
| 6896 | } |
| 6897 | } |
| 6898 | |
| 6899 | if (!src_known && |
| 6900 | opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { |
| 6901 | __mark_reg_unknown(env, dst_reg); |
| 6902 | return 0; |
| 6903 | } |
| 6904 | |
| 6905 | /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. |
| 6906 | * There are two classes of instructions: The first class we track both |
| 6907 | * alu32 and alu64 sign/unsigned bounds independently this provides the |
| 6908 | * greatest amount of precision when alu operations are mixed with jmp32 |
| 6909 | * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD, |
| 6910 | * and BPF_OR. This is possible because these ops have fairly easy to |
| 6911 | * understand and calculate behavior in both 32-bit and 64-bit alu ops. |
| 6912 | * See alu32 verifier tests for examples. The second class of |
| 6913 | * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy |
| 6914 | * with regards to tracking sign/unsigned bounds because the bits may |
| 6915 | * cross subreg boundaries in the alu64 case. When this happens we mark |
| 6916 | * the reg unbounded in the subreg bound space and use the resulting |
| 6917 | * tnum to calculate an approximation of the sign/unsigned bounds. |
| 6918 | */ |
| 6919 | switch (opcode) { |
| 6920 | case BPF_ADD: |
| 6921 | ret = sanitize_val_alu(env, insn); |
| 6922 | if (ret < 0) { |
| 6923 | verbose(env, "R%d tried to add from different pointers or scalars\n", dst); |
| 6924 | return ret; |
| 6925 | } |
| 6926 | scalar32_min_max_add(dst_reg, &src_reg); |
| 6927 | scalar_min_max_add(dst_reg, &src_reg); |
| 6928 | dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); |
| 6929 | break; |
| 6930 | case BPF_SUB: |
| 6931 | ret = sanitize_val_alu(env, insn); |
| 6932 | if (ret < 0) { |
| 6933 | verbose(env, "R%d tried to sub from different pointers or scalars\n", dst); |
| 6934 | return ret; |
| 6935 | } |
| 6936 | scalar32_min_max_sub(dst_reg, &src_reg); |
| 6937 | scalar_min_max_sub(dst_reg, &src_reg); |
| 6938 | dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); |
| 6939 | break; |
| 6940 | case BPF_MUL: |
| 6941 | dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); |
| 6942 | scalar32_min_max_mul(dst_reg, &src_reg); |
| 6943 | scalar_min_max_mul(dst_reg, &src_reg); |
| 6944 | break; |
| 6945 | case BPF_AND: |
| 6946 | dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); |
| 6947 | scalar32_min_max_and(dst_reg, &src_reg); |
| 6948 | scalar_min_max_and(dst_reg, &src_reg); |
| 6949 | break; |
| 6950 | case BPF_OR: |
| 6951 | dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); |
| 6952 | scalar32_min_max_or(dst_reg, &src_reg); |
| 6953 | scalar_min_max_or(dst_reg, &src_reg); |
| 6954 | break; |
| 6955 | case BPF_XOR: |
| 6956 | dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); |
| 6957 | scalar32_min_max_xor(dst_reg, &src_reg); |
| 6958 | scalar_min_max_xor(dst_reg, &src_reg); |
| 6959 | break; |
| 6960 | case BPF_LSH: |
| 6961 | if (umax_val >= insn_bitness) { |
| 6962 | /* Shifts greater than 31 or 63 are undefined. |
| 6963 | * This includes shifts by a negative number. |
| 6964 | */ |
| 6965 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 6966 | break; |
| 6967 | } |
| 6968 | if (alu32) |
| 6969 | scalar32_min_max_lsh(dst_reg, &src_reg); |
| 6970 | else |
| 6971 | scalar_min_max_lsh(dst_reg, &src_reg); |
| 6972 | break; |
| 6973 | case BPF_RSH: |
| 6974 | if (umax_val >= insn_bitness) { |
| 6975 | /* Shifts greater than 31 or 63 are undefined. |
| 6976 | * This includes shifts by a negative number. |
| 6977 | */ |
| 6978 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 6979 | break; |
| 6980 | } |
| 6981 | if (alu32) |
| 6982 | scalar32_min_max_rsh(dst_reg, &src_reg); |
| 6983 | else |
| 6984 | scalar_min_max_rsh(dst_reg, &src_reg); |
| 6985 | break; |
| 6986 | case BPF_ARSH: |
| 6987 | if (umax_val >= insn_bitness) { |
| 6988 | /* Shifts greater than 31 or 63 are undefined. |
| 6989 | * This includes shifts by a negative number. |
| 6990 | */ |
| 6991 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 6992 | break; |
| 6993 | } |
| 6994 | if (alu32) |
| 6995 | scalar32_min_max_arsh(dst_reg, &src_reg); |
| 6996 | else |
| 6997 | scalar_min_max_arsh(dst_reg, &src_reg); |
| 6998 | break; |
| 6999 | default: |
| 7000 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 7001 | break; |
| 7002 | } |
| 7003 | |
| 7004 | /* ALU32 ops are zero extended into 64bit register */ |
| 7005 | if (alu32) |
| 7006 | zext_32_to_64(dst_reg); |
| 7007 | |
| 7008 | __update_reg_bounds(dst_reg); |
| 7009 | __reg_deduce_bounds(dst_reg); |
| 7010 | __reg_bound_offset(dst_reg); |
| 7011 | return 0; |
| 7012 | } |
| 7013 | |
| 7014 | /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max |
| 7015 | * and var_off. |
| 7016 | */ |
| 7017 | static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, |
| 7018 | struct bpf_insn *insn) |
| 7019 | { |
| 7020 | struct bpf_verifier_state *vstate = env->cur_state; |
| 7021 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 7022 | struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; |
| 7023 | struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; |
| 7024 | u8 opcode = BPF_OP(insn->code); |
| 7025 | int err; |
| 7026 | |
| 7027 | dst_reg = ®s[insn->dst_reg]; |
| 7028 | src_reg = NULL; |
| 7029 | if (dst_reg->type != SCALAR_VALUE) |
| 7030 | ptr_reg = dst_reg; |
| 7031 | else |
| 7032 | /* Make sure ID is cleared otherwise dst_reg min/max could be |
| 7033 | * incorrectly propagated into other registers by find_equal_scalars() |
| 7034 | */ |
| 7035 | dst_reg->id = 0; |
| 7036 | if (BPF_SRC(insn->code) == BPF_X) { |
| 7037 | src_reg = ®s[insn->src_reg]; |
| 7038 | if (src_reg->type != SCALAR_VALUE) { |
| 7039 | if (dst_reg->type != SCALAR_VALUE) { |
| 7040 | /* Combining two pointers by any ALU op yields |
| 7041 | * an arbitrary scalar. Disallow all math except |
| 7042 | * pointer subtraction |
| 7043 | */ |
| 7044 | if (opcode == BPF_SUB && env->allow_ptr_leaks) { |
| 7045 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 7046 | return 0; |
| 7047 | } |
| 7048 | verbose(env, "R%d pointer %s pointer prohibited\n", |
| 7049 | insn->dst_reg, |
| 7050 | bpf_alu_string[opcode >> 4]); |
| 7051 | return -EACCES; |
| 7052 | } else { |
| 7053 | /* scalar += pointer |
| 7054 | * This is legal, but we have to reverse our |
| 7055 | * src/dest handling in computing the range |
| 7056 | */ |
| 7057 | err = mark_chain_precision(env, insn->dst_reg); |
| 7058 | if (err) |
| 7059 | return err; |
| 7060 | return adjust_ptr_min_max_vals(env, insn, |
| 7061 | src_reg, dst_reg); |
| 7062 | } |
| 7063 | } else if (ptr_reg) { |
| 7064 | /* pointer += scalar */ |
| 7065 | err = mark_chain_precision(env, insn->src_reg); |
| 7066 | if (err) |
| 7067 | return err; |
| 7068 | return adjust_ptr_min_max_vals(env, insn, |
| 7069 | dst_reg, src_reg); |
| 7070 | } |
| 7071 | } else { |
| 7072 | /* Pretend the src is a reg with a known value, since we only |
| 7073 | * need to be able to read from this state. |
| 7074 | */ |
| 7075 | off_reg.type = SCALAR_VALUE; |
| 7076 | __mark_reg_known(&off_reg, insn->imm); |
| 7077 | src_reg = &off_reg; |
| 7078 | if (ptr_reg) /* pointer += K */ |
| 7079 | return adjust_ptr_min_max_vals(env, insn, |
| 7080 | ptr_reg, src_reg); |
| 7081 | } |
| 7082 | |
| 7083 | /* Got here implies adding two SCALAR_VALUEs */ |
| 7084 | if (WARN_ON_ONCE(ptr_reg)) { |
| 7085 | print_verifier_state(env, state); |
| 7086 | verbose(env, "verifier internal error: unexpected ptr_reg\n"); |
| 7087 | return -EINVAL; |
| 7088 | } |
| 7089 | if (WARN_ON(!src_reg)) { |
| 7090 | print_verifier_state(env, state); |
| 7091 | verbose(env, "verifier internal error: no src_reg\n"); |
| 7092 | return -EINVAL; |
| 7093 | } |
| 7094 | return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); |
| 7095 | } |
| 7096 | |
| 7097 | /* check validity of 32-bit and 64-bit arithmetic operations */ |
| 7098 | static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) |
| 7099 | { |
| 7100 | struct bpf_reg_state *regs = cur_regs(env); |
| 7101 | u8 opcode = BPF_OP(insn->code); |
| 7102 | int err; |
| 7103 | |
| 7104 | if (opcode == BPF_END || opcode == BPF_NEG) { |
| 7105 | if (opcode == BPF_NEG) { |
| 7106 | if (BPF_SRC(insn->code) != 0 || |
| 7107 | insn->src_reg != BPF_REG_0 || |
| 7108 | insn->off != 0 || insn->imm != 0) { |
| 7109 | verbose(env, "BPF_NEG uses reserved fields\n"); |
| 7110 | return -EINVAL; |
| 7111 | } |
| 7112 | } else { |
| 7113 | if (insn->src_reg != BPF_REG_0 || insn->off != 0 || |
| 7114 | (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || |
| 7115 | BPF_CLASS(insn->code) == BPF_ALU64) { |
| 7116 | verbose(env, "BPF_END uses reserved fields\n"); |
| 7117 | return -EINVAL; |
| 7118 | } |
| 7119 | } |
| 7120 | |
| 7121 | /* check src operand */ |
| 7122 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
| 7123 | if (err) |
| 7124 | return err; |
| 7125 | |
| 7126 | if (is_pointer_value(env, insn->dst_reg)) { |
| 7127 | verbose(env, "R%d pointer arithmetic prohibited\n", |
| 7128 | insn->dst_reg); |
| 7129 | return -EACCES; |
| 7130 | } |
| 7131 | |
| 7132 | /* check dest operand */ |
| 7133 | err = check_reg_arg(env, insn->dst_reg, DST_OP); |
| 7134 | if (err) |
| 7135 | return err; |
| 7136 | |
| 7137 | } else if (opcode == BPF_MOV) { |
| 7138 | |
| 7139 | if (BPF_SRC(insn->code) == BPF_X) { |
| 7140 | if (insn->imm != 0 || insn->off != 0) { |
| 7141 | verbose(env, "BPF_MOV uses reserved fields\n"); |
| 7142 | return -EINVAL; |
| 7143 | } |
| 7144 | |
| 7145 | /* check src operand */ |
| 7146 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
| 7147 | if (err) |
| 7148 | return err; |
| 7149 | } else { |
| 7150 | if (insn->src_reg != BPF_REG_0 || insn->off != 0) { |
| 7151 | verbose(env, "BPF_MOV uses reserved fields\n"); |
| 7152 | return -EINVAL; |
| 7153 | } |
| 7154 | } |
| 7155 | |
| 7156 | /* check dest operand, mark as required later */ |
| 7157 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
| 7158 | if (err) |
| 7159 | return err; |
| 7160 | |
| 7161 | if (BPF_SRC(insn->code) == BPF_X) { |
| 7162 | struct bpf_reg_state *src_reg = regs + insn->src_reg; |
| 7163 | struct bpf_reg_state *dst_reg = regs + insn->dst_reg; |
| 7164 | |
| 7165 | if (BPF_CLASS(insn->code) == BPF_ALU64) { |
| 7166 | /* case: R1 = R2 |
| 7167 | * copy register state to dest reg |
| 7168 | */ |
| 7169 | if (src_reg->type == SCALAR_VALUE && !src_reg->id) |
| 7170 | /* Assign src and dst registers the same ID |
| 7171 | * that will be used by find_equal_scalars() |
| 7172 | * to propagate min/max range. |
| 7173 | */ |
| 7174 | src_reg->id = ++env->id_gen; |
| 7175 | *dst_reg = *src_reg; |
| 7176 | dst_reg->live |= REG_LIVE_WRITTEN; |
| 7177 | dst_reg->subreg_def = DEF_NOT_SUBREG; |
| 7178 | } else { |
| 7179 | /* R1 = (u32) R2 */ |
| 7180 | if (is_pointer_value(env, insn->src_reg)) { |
| 7181 | verbose(env, |
| 7182 | "R%d partial copy of pointer\n", |
| 7183 | insn->src_reg); |
| 7184 | return -EACCES; |
| 7185 | } else if (src_reg->type == SCALAR_VALUE) { |
| 7186 | *dst_reg = *src_reg; |
| 7187 | /* Make sure ID is cleared otherwise |
| 7188 | * dst_reg min/max could be incorrectly |
| 7189 | * propagated into src_reg by find_equal_scalars() |
| 7190 | */ |
| 7191 | dst_reg->id = 0; |
| 7192 | dst_reg->live |= REG_LIVE_WRITTEN; |
| 7193 | dst_reg->subreg_def = env->insn_idx + 1; |
| 7194 | } else { |
| 7195 | mark_reg_unknown(env, regs, |
| 7196 | insn->dst_reg); |
| 7197 | } |
| 7198 | zext_32_to_64(dst_reg); |
| 7199 | } |
| 7200 | } else { |
| 7201 | /* case: R = imm |
| 7202 | * remember the value we stored into this reg |
| 7203 | */ |
| 7204 | /* clear any state __mark_reg_known doesn't set */ |
| 7205 | mark_reg_unknown(env, regs, insn->dst_reg); |
| 7206 | regs[insn->dst_reg].type = SCALAR_VALUE; |
| 7207 | if (BPF_CLASS(insn->code) == BPF_ALU64) { |
| 7208 | __mark_reg_known(regs + insn->dst_reg, |
| 7209 | insn->imm); |
| 7210 | } else { |
| 7211 | __mark_reg_known(regs + insn->dst_reg, |
| 7212 | (u32)insn->imm); |
| 7213 | } |
| 7214 | } |
| 7215 | |
| 7216 | } else if (opcode > BPF_END) { |
| 7217 | verbose(env, "invalid BPF_ALU opcode %x\n", opcode); |
| 7218 | return -EINVAL; |
| 7219 | |
| 7220 | } else { /* all other ALU ops: and, sub, xor, add, ... */ |
| 7221 | |
| 7222 | if (BPF_SRC(insn->code) == BPF_X) { |
| 7223 | if (insn->imm != 0 || insn->off != 0) { |
| 7224 | verbose(env, "BPF_ALU uses reserved fields\n"); |
| 7225 | return -EINVAL; |
| 7226 | } |
| 7227 | /* check src1 operand */ |
| 7228 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
| 7229 | if (err) |
| 7230 | return err; |
| 7231 | } else { |
| 7232 | if (insn->src_reg != BPF_REG_0 || insn->off != 0) { |
| 7233 | verbose(env, "BPF_ALU uses reserved fields\n"); |
| 7234 | return -EINVAL; |
| 7235 | } |
| 7236 | } |
| 7237 | |
| 7238 | /* check src2 operand */ |
| 7239 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
| 7240 | if (err) |
| 7241 | return err; |
| 7242 | |
| 7243 | if ((opcode == BPF_MOD || opcode == BPF_DIV) && |
| 7244 | BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { |
| 7245 | verbose(env, "div by zero\n"); |
| 7246 | return -EINVAL; |
| 7247 | } |
| 7248 | |
| 7249 | if ((opcode == BPF_LSH || opcode == BPF_RSH || |
| 7250 | opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { |
| 7251 | int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; |
| 7252 | |
| 7253 | if (insn->imm < 0 || insn->imm >= size) { |
| 7254 | verbose(env, "invalid shift %d\n", insn->imm); |
| 7255 | return -EINVAL; |
| 7256 | } |
| 7257 | } |
| 7258 | |
| 7259 | /* check dest operand */ |
| 7260 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
| 7261 | if (err) |
| 7262 | return err; |
| 7263 | |
| 7264 | return adjust_reg_min_max_vals(env, insn); |
| 7265 | } |
| 7266 | |
| 7267 | return 0; |
| 7268 | } |
| 7269 | |
| 7270 | static void __find_good_pkt_pointers(struct bpf_func_state *state, |
| 7271 | struct bpf_reg_state *dst_reg, |
| 7272 | enum bpf_reg_type type, int new_range) |
| 7273 | { |
| 7274 | struct bpf_reg_state *reg; |
| 7275 | int i; |
| 7276 | |
| 7277 | for (i = 0; i < MAX_BPF_REG; i++) { |
| 7278 | reg = &state->regs[i]; |
| 7279 | if (reg->type == type && reg->id == dst_reg->id) |
| 7280 | /* keep the maximum range already checked */ |
| 7281 | reg->range = max(reg->range, new_range); |
| 7282 | } |
| 7283 | |
| 7284 | bpf_for_each_spilled_reg(i, state, reg) { |
| 7285 | if (!reg) |
| 7286 | continue; |
| 7287 | if (reg->type == type && reg->id == dst_reg->id) |
| 7288 | reg->range = max(reg->range, new_range); |
| 7289 | } |
| 7290 | } |
| 7291 | |
| 7292 | static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, |
| 7293 | struct bpf_reg_state *dst_reg, |
| 7294 | enum bpf_reg_type type, |
| 7295 | bool range_right_open) |
| 7296 | { |
| 7297 | int new_range, i; |
| 7298 | |
| 7299 | if (dst_reg->off < 0 || |
| 7300 | (dst_reg->off == 0 && range_right_open)) |
| 7301 | /* This doesn't give us any range */ |
| 7302 | return; |
| 7303 | |
| 7304 | if (dst_reg->umax_value > MAX_PACKET_OFF || |
| 7305 | dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) |
| 7306 | /* Risk of overflow. For instance, ptr + (1<<63) may be less |
| 7307 | * than pkt_end, but that's because it's also less than pkt. |
| 7308 | */ |
| 7309 | return; |
| 7310 | |
| 7311 | new_range = dst_reg->off; |
| 7312 | if (range_right_open) |
| 7313 | new_range--; |
| 7314 | |
| 7315 | /* Examples for register markings: |
| 7316 | * |
| 7317 | * pkt_data in dst register: |
| 7318 | * |
| 7319 | * r2 = r3; |
| 7320 | * r2 += 8; |
| 7321 | * if (r2 > pkt_end) goto <handle exception> |
| 7322 | * <access okay> |
| 7323 | * |
| 7324 | * r2 = r3; |
| 7325 | * r2 += 8; |
| 7326 | * if (r2 < pkt_end) goto <access okay> |
| 7327 | * <handle exception> |
| 7328 | * |
| 7329 | * Where: |
| 7330 | * r2 == dst_reg, pkt_end == src_reg |
| 7331 | * r2=pkt(id=n,off=8,r=0) |
| 7332 | * r3=pkt(id=n,off=0,r=0) |
| 7333 | * |
| 7334 | * pkt_data in src register: |
| 7335 | * |
| 7336 | * r2 = r3; |
| 7337 | * r2 += 8; |
| 7338 | * if (pkt_end >= r2) goto <access okay> |
| 7339 | * <handle exception> |
| 7340 | * |
| 7341 | * r2 = r3; |
| 7342 | * r2 += 8; |
| 7343 | * if (pkt_end <= r2) goto <handle exception> |
| 7344 | * <access okay> |
| 7345 | * |
| 7346 | * Where: |
| 7347 | * pkt_end == dst_reg, r2 == src_reg |
| 7348 | * r2=pkt(id=n,off=8,r=0) |
| 7349 | * r3=pkt(id=n,off=0,r=0) |
| 7350 | * |
| 7351 | * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) |
| 7352 | * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) |
| 7353 | * and [r3, r3 + 8-1) respectively is safe to access depending on |
| 7354 | * the check. |
| 7355 | */ |
| 7356 | |
| 7357 | /* If our ids match, then we must have the same max_value. And we |
| 7358 | * don't care about the other reg's fixed offset, since if it's too big |
| 7359 | * the range won't allow anything. |
| 7360 | * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. |
| 7361 | */ |
| 7362 | for (i = 0; i <= vstate->curframe; i++) |
| 7363 | __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, |
| 7364 | new_range); |
| 7365 | } |
| 7366 | |
| 7367 | static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode) |
| 7368 | { |
| 7369 | struct tnum subreg = tnum_subreg(reg->var_off); |
| 7370 | s32 sval = (s32)val; |
| 7371 | |
| 7372 | switch (opcode) { |
| 7373 | case BPF_JEQ: |
| 7374 | if (tnum_is_const(subreg)) |
| 7375 | return !!tnum_equals_const(subreg, val); |
| 7376 | break; |
| 7377 | case BPF_JNE: |
| 7378 | if (tnum_is_const(subreg)) |
| 7379 | return !tnum_equals_const(subreg, val); |
| 7380 | break; |
| 7381 | case BPF_JSET: |
| 7382 | if ((~subreg.mask & subreg.value) & val) |
| 7383 | return 1; |
| 7384 | if (!((subreg.mask | subreg.value) & val)) |
| 7385 | return 0; |
| 7386 | break; |
| 7387 | case BPF_JGT: |
| 7388 | if (reg->u32_min_value > val) |
| 7389 | return 1; |
| 7390 | else if (reg->u32_max_value <= val) |
| 7391 | return 0; |
| 7392 | break; |
| 7393 | case BPF_JSGT: |
| 7394 | if (reg->s32_min_value > sval) |
| 7395 | return 1; |
| 7396 | else if (reg->s32_max_value <= sval) |
| 7397 | return 0; |
| 7398 | break; |
| 7399 | case BPF_JLT: |
| 7400 | if (reg->u32_max_value < val) |
| 7401 | return 1; |
| 7402 | else if (reg->u32_min_value >= val) |
| 7403 | return 0; |
| 7404 | break; |
| 7405 | case BPF_JSLT: |
| 7406 | if (reg->s32_max_value < sval) |
| 7407 | return 1; |
| 7408 | else if (reg->s32_min_value >= sval) |
| 7409 | return 0; |
| 7410 | break; |
| 7411 | case BPF_JGE: |
| 7412 | if (reg->u32_min_value >= val) |
| 7413 | return 1; |
| 7414 | else if (reg->u32_max_value < val) |
| 7415 | return 0; |
| 7416 | break; |
| 7417 | case BPF_JSGE: |
| 7418 | if (reg->s32_min_value >= sval) |
| 7419 | return 1; |
| 7420 | else if (reg->s32_max_value < sval) |
| 7421 | return 0; |
| 7422 | break; |
| 7423 | case BPF_JLE: |
| 7424 | if (reg->u32_max_value <= val) |
| 7425 | return 1; |
| 7426 | else if (reg->u32_min_value > val) |
| 7427 | return 0; |
| 7428 | break; |
| 7429 | case BPF_JSLE: |
| 7430 | if (reg->s32_max_value <= sval) |
| 7431 | return 1; |
| 7432 | else if (reg->s32_min_value > sval) |
| 7433 | return 0; |
| 7434 | break; |
| 7435 | } |
| 7436 | |
| 7437 | return -1; |
| 7438 | } |
| 7439 | |
| 7440 | |
| 7441 | static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode) |
| 7442 | { |
| 7443 | s64 sval = (s64)val; |
| 7444 | |
| 7445 | switch (opcode) { |
| 7446 | case BPF_JEQ: |
| 7447 | if (tnum_is_const(reg->var_off)) |
| 7448 | return !!tnum_equals_const(reg->var_off, val); |
| 7449 | break; |
| 7450 | case BPF_JNE: |
| 7451 | if (tnum_is_const(reg->var_off)) |
| 7452 | return !tnum_equals_const(reg->var_off, val); |
| 7453 | break; |
| 7454 | case BPF_JSET: |
| 7455 | if ((~reg->var_off.mask & reg->var_off.value) & val) |
| 7456 | return 1; |
| 7457 | if (!((reg->var_off.mask | reg->var_off.value) & val)) |
| 7458 | return 0; |
| 7459 | break; |
| 7460 | case BPF_JGT: |
| 7461 | if (reg->umin_value > val) |
| 7462 | return 1; |
| 7463 | else if (reg->umax_value <= val) |
| 7464 | return 0; |
| 7465 | break; |
| 7466 | case BPF_JSGT: |
| 7467 | if (reg->smin_value > sval) |
| 7468 | return 1; |
| 7469 | else if (reg->smax_value <= sval) |
| 7470 | return 0; |
| 7471 | break; |
| 7472 | case BPF_JLT: |
| 7473 | if (reg->umax_value < val) |
| 7474 | return 1; |
| 7475 | else if (reg->umin_value >= val) |
| 7476 | return 0; |
| 7477 | break; |
| 7478 | case BPF_JSLT: |
| 7479 | if (reg->smax_value < sval) |
| 7480 | return 1; |
| 7481 | else if (reg->smin_value >= sval) |
| 7482 | return 0; |
| 7483 | break; |
| 7484 | case BPF_JGE: |
| 7485 | if (reg->umin_value >= val) |
| 7486 | return 1; |
| 7487 | else if (reg->umax_value < val) |
| 7488 | return 0; |
| 7489 | break; |
| 7490 | case BPF_JSGE: |
| 7491 | if (reg->smin_value >= sval) |
| 7492 | return 1; |
| 7493 | else if (reg->smax_value < sval) |
| 7494 | return 0; |
| 7495 | break; |
| 7496 | case BPF_JLE: |
| 7497 | if (reg->umax_value <= val) |
| 7498 | return 1; |
| 7499 | else if (reg->umin_value > val) |
| 7500 | return 0; |
| 7501 | break; |
| 7502 | case BPF_JSLE: |
| 7503 | if (reg->smax_value <= sval) |
| 7504 | return 1; |
| 7505 | else if (reg->smin_value > sval) |
| 7506 | return 0; |
| 7507 | break; |
| 7508 | } |
| 7509 | |
| 7510 | return -1; |
| 7511 | } |
| 7512 | |
| 7513 | /* compute branch direction of the expression "if (reg opcode val) goto target;" |
| 7514 | * and return: |
| 7515 | * 1 - branch will be taken and "goto target" will be executed |
| 7516 | * 0 - branch will not be taken and fall-through to next insn |
| 7517 | * -1 - unknown. Example: "if (reg < 5)" is unknown when register value |
| 7518 | * range [0,10] |
| 7519 | */ |
| 7520 | static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, |
| 7521 | bool is_jmp32) |
| 7522 | { |
| 7523 | if (__is_pointer_value(false, reg)) { |
| 7524 | if (!reg_type_not_null(reg->type)) |
| 7525 | return -1; |
| 7526 | |
| 7527 | /* If pointer is valid tests against zero will fail so we can |
| 7528 | * use this to direct branch taken. |
| 7529 | */ |
| 7530 | if (val != 0) |
| 7531 | return -1; |
| 7532 | |
| 7533 | switch (opcode) { |
| 7534 | case BPF_JEQ: |
| 7535 | return 0; |
| 7536 | case BPF_JNE: |
| 7537 | return 1; |
| 7538 | default: |
| 7539 | return -1; |
| 7540 | } |
| 7541 | } |
| 7542 | |
| 7543 | if (is_jmp32) |
| 7544 | return is_branch32_taken(reg, val, opcode); |
| 7545 | return is_branch64_taken(reg, val, opcode); |
| 7546 | } |
| 7547 | |
| 7548 | static int flip_opcode(u32 opcode) |
| 7549 | { |
| 7550 | /* How can we transform "a <op> b" into "b <op> a"? */ |
| 7551 | static const u8 opcode_flip[16] = { |
| 7552 | /* these stay the same */ |
| 7553 | [BPF_JEQ >> 4] = BPF_JEQ, |
| 7554 | [BPF_JNE >> 4] = BPF_JNE, |
| 7555 | [BPF_JSET >> 4] = BPF_JSET, |
| 7556 | /* these swap "lesser" and "greater" (L and G in the opcodes) */ |
| 7557 | [BPF_JGE >> 4] = BPF_JLE, |
| 7558 | [BPF_JGT >> 4] = BPF_JLT, |
| 7559 | [BPF_JLE >> 4] = BPF_JGE, |
| 7560 | [BPF_JLT >> 4] = BPF_JGT, |
| 7561 | [BPF_JSGE >> 4] = BPF_JSLE, |
| 7562 | [BPF_JSGT >> 4] = BPF_JSLT, |
| 7563 | [BPF_JSLE >> 4] = BPF_JSGE, |
| 7564 | [BPF_JSLT >> 4] = BPF_JSGT |
| 7565 | }; |
| 7566 | return opcode_flip[opcode >> 4]; |
| 7567 | } |
| 7568 | |
| 7569 | static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, |
| 7570 | struct bpf_reg_state *src_reg, |
| 7571 | u8 opcode) |
| 7572 | { |
| 7573 | struct bpf_reg_state *pkt; |
| 7574 | |
| 7575 | if (src_reg->type == PTR_TO_PACKET_END) { |
| 7576 | pkt = dst_reg; |
| 7577 | } else if (dst_reg->type == PTR_TO_PACKET_END) { |
| 7578 | pkt = src_reg; |
| 7579 | opcode = flip_opcode(opcode); |
| 7580 | } else { |
| 7581 | return -1; |
| 7582 | } |
| 7583 | |
| 7584 | if (pkt->range >= 0) |
| 7585 | return -1; |
| 7586 | |
| 7587 | switch (opcode) { |
| 7588 | case BPF_JLE: |
| 7589 | /* pkt <= pkt_end */ |
| 7590 | fallthrough; |
| 7591 | case BPF_JGT: |
| 7592 | /* pkt > pkt_end */ |
| 7593 | if (pkt->range == BEYOND_PKT_END) |
| 7594 | /* pkt has at last one extra byte beyond pkt_end */ |
| 7595 | return opcode == BPF_JGT; |
| 7596 | break; |
| 7597 | case BPF_JLT: |
| 7598 | /* pkt < pkt_end */ |
| 7599 | fallthrough; |
| 7600 | case BPF_JGE: |
| 7601 | /* pkt >= pkt_end */ |
| 7602 | if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END) |
| 7603 | return opcode == BPF_JGE; |
| 7604 | break; |
| 7605 | } |
| 7606 | return -1; |
| 7607 | } |
| 7608 | |
| 7609 | /* Adjusts the register min/max values in the case that the dst_reg is the |
| 7610 | * variable register that we are working on, and src_reg is a constant or we're |
| 7611 | * simply doing a BPF_K check. |
| 7612 | * In JEQ/JNE cases we also adjust the var_off values. |
| 7613 | */ |
| 7614 | static void reg_set_min_max(struct bpf_reg_state *true_reg, |
| 7615 | struct bpf_reg_state *false_reg, |
| 7616 | u64 val, u32 val32, |
| 7617 | u8 opcode, bool is_jmp32) |
| 7618 | { |
| 7619 | struct tnum false_32off = tnum_subreg(false_reg->var_off); |
| 7620 | struct tnum false_64off = false_reg->var_off; |
| 7621 | struct tnum true_32off = tnum_subreg(true_reg->var_off); |
| 7622 | struct tnum true_64off = true_reg->var_off; |
| 7623 | s64 sval = (s64)val; |
| 7624 | s32 sval32 = (s32)val32; |
| 7625 | |
| 7626 | /* If the dst_reg is a pointer, we can't learn anything about its |
| 7627 | * variable offset from the compare (unless src_reg were a pointer into |
| 7628 | * the same object, but we don't bother with that. |
| 7629 | * Since false_reg and true_reg have the same type by construction, we |
| 7630 | * only need to check one of them for pointerness. |
| 7631 | */ |
| 7632 | if (__is_pointer_value(false, false_reg)) |
| 7633 | return; |
| 7634 | |
| 7635 | switch (opcode) { |
| 7636 | case BPF_JEQ: |
| 7637 | case BPF_JNE: |
| 7638 | { |
| 7639 | struct bpf_reg_state *reg = |
| 7640 | opcode == BPF_JEQ ? true_reg : false_reg; |
| 7641 | |
| 7642 | /* JEQ/JNE comparison doesn't change the register equivalence. |
| 7643 | * r1 = r2; |
| 7644 | * if (r1 == 42) goto label; |
| 7645 | * ... |
| 7646 | * label: // here both r1 and r2 are known to be 42. |
| 7647 | * |
| 7648 | * Hence when marking register as known preserve it's ID. |
| 7649 | */ |
| 7650 | if (is_jmp32) |
| 7651 | __mark_reg32_known(reg, val32); |
| 7652 | else |
| 7653 | ___mark_reg_known(reg, val); |
| 7654 | break; |
| 7655 | } |
| 7656 | case BPF_JSET: |
| 7657 | if (is_jmp32) { |
| 7658 | false_32off = tnum_and(false_32off, tnum_const(~val32)); |
| 7659 | if (is_power_of_2(val32)) |
| 7660 | true_32off = tnum_or(true_32off, |
| 7661 | tnum_const(val32)); |
| 7662 | } else { |
| 7663 | false_64off = tnum_and(false_64off, tnum_const(~val)); |
| 7664 | if (is_power_of_2(val)) |
| 7665 | true_64off = tnum_or(true_64off, |
| 7666 | tnum_const(val)); |
| 7667 | } |
| 7668 | break; |
| 7669 | case BPF_JGE: |
| 7670 | case BPF_JGT: |
| 7671 | { |
| 7672 | if (is_jmp32) { |
| 7673 | u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1; |
| 7674 | u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32; |
| 7675 | |
| 7676 | false_reg->u32_max_value = min(false_reg->u32_max_value, |
| 7677 | false_umax); |
| 7678 | true_reg->u32_min_value = max(true_reg->u32_min_value, |
| 7679 | true_umin); |
| 7680 | } else { |
| 7681 | u64 false_umax = opcode == BPF_JGT ? val : val - 1; |
| 7682 | u64 true_umin = opcode == BPF_JGT ? val + 1 : val; |
| 7683 | |
| 7684 | false_reg->umax_value = min(false_reg->umax_value, false_umax); |
| 7685 | true_reg->umin_value = max(true_reg->umin_value, true_umin); |
| 7686 | } |
| 7687 | break; |
| 7688 | } |
| 7689 | case BPF_JSGE: |
| 7690 | case BPF_JSGT: |
| 7691 | { |
| 7692 | if (is_jmp32) { |
| 7693 | s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1; |
| 7694 | s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32; |
| 7695 | |
| 7696 | false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax); |
| 7697 | true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin); |
| 7698 | } else { |
| 7699 | s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; |
| 7700 | s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; |
| 7701 | |
| 7702 | false_reg->smax_value = min(false_reg->smax_value, false_smax); |
| 7703 | true_reg->smin_value = max(true_reg->smin_value, true_smin); |
| 7704 | } |
| 7705 | break; |
| 7706 | } |
| 7707 | case BPF_JLE: |
| 7708 | case BPF_JLT: |
| 7709 | { |
| 7710 | if (is_jmp32) { |
| 7711 | u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1; |
| 7712 | u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32; |
| 7713 | |
| 7714 | false_reg->u32_min_value = max(false_reg->u32_min_value, |
| 7715 | false_umin); |
| 7716 | true_reg->u32_max_value = min(true_reg->u32_max_value, |
| 7717 | true_umax); |
| 7718 | } else { |
| 7719 | u64 false_umin = opcode == BPF_JLT ? val : val + 1; |
| 7720 | u64 true_umax = opcode == BPF_JLT ? val - 1 : val; |
| 7721 | |
| 7722 | false_reg->umin_value = max(false_reg->umin_value, false_umin); |
| 7723 | true_reg->umax_value = min(true_reg->umax_value, true_umax); |
| 7724 | } |
| 7725 | break; |
| 7726 | } |
| 7727 | case BPF_JSLE: |
| 7728 | case BPF_JSLT: |
| 7729 | { |
| 7730 | if (is_jmp32) { |
| 7731 | s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1; |
| 7732 | s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32; |
| 7733 | |
| 7734 | false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin); |
| 7735 | true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax); |
| 7736 | } else { |
| 7737 | s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; |
| 7738 | s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; |
| 7739 | |
| 7740 | false_reg->smin_value = max(false_reg->smin_value, false_smin); |
| 7741 | true_reg->smax_value = min(true_reg->smax_value, true_smax); |
| 7742 | } |
| 7743 | break; |
| 7744 | } |
| 7745 | default: |
| 7746 | return; |
| 7747 | } |
| 7748 | |
| 7749 | if (is_jmp32) { |
| 7750 | false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off), |
| 7751 | tnum_subreg(false_32off)); |
| 7752 | true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off), |
| 7753 | tnum_subreg(true_32off)); |
| 7754 | __reg_combine_32_into_64(false_reg); |
| 7755 | __reg_combine_32_into_64(true_reg); |
| 7756 | } else { |
| 7757 | false_reg->var_off = false_64off; |
| 7758 | true_reg->var_off = true_64off; |
| 7759 | __reg_combine_64_into_32(false_reg); |
| 7760 | __reg_combine_64_into_32(true_reg); |
| 7761 | } |
| 7762 | } |
| 7763 | |
| 7764 | /* Same as above, but for the case that dst_reg holds a constant and src_reg is |
| 7765 | * the variable reg. |
| 7766 | */ |
| 7767 | static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, |
| 7768 | struct bpf_reg_state *false_reg, |
| 7769 | u64 val, u32 val32, |
| 7770 | u8 opcode, bool is_jmp32) |
| 7771 | { |
| 7772 | opcode = flip_opcode(opcode); |
| 7773 | /* This uses zero as "not present in table"; luckily the zero opcode, |
| 7774 | * BPF_JA, can't get here. |
| 7775 | */ |
| 7776 | if (opcode) |
| 7777 | reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32); |
| 7778 | } |
| 7779 | |
| 7780 | /* Regs are known to be equal, so intersect their min/max/var_off */ |
| 7781 | static void __reg_combine_min_max(struct bpf_reg_state *src_reg, |
| 7782 | struct bpf_reg_state *dst_reg) |
| 7783 | { |
| 7784 | src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, |
| 7785 | dst_reg->umin_value); |
| 7786 | src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, |
| 7787 | dst_reg->umax_value); |
| 7788 | src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, |
| 7789 | dst_reg->smin_value); |
| 7790 | src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, |
| 7791 | dst_reg->smax_value); |
| 7792 | src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, |
| 7793 | dst_reg->var_off); |
| 7794 | /* We might have learned new bounds from the var_off. */ |
| 7795 | __update_reg_bounds(src_reg); |
| 7796 | __update_reg_bounds(dst_reg); |
| 7797 | /* We might have learned something about the sign bit. */ |
| 7798 | __reg_deduce_bounds(src_reg); |
| 7799 | __reg_deduce_bounds(dst_reg); |
| 7800 | /* We might have learned some bits from the bounds. */ |
| 7801 | __reg_bound_offset(src_reg); |
| 7802 | __reg_bound_offset(dst_reg); |
| 7803 | /* Intersecting with the old var_off might have improved our bounds |
| 7804 | * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), |
| 7805 | * then new var_off is (0; 0x7f...fc) which improves our umax. |
| 7806 | */ |
| 7807 | __update_reg_bounds(src_reg); |
| 7808 | __update_reg_bounds(dst_reg); |
| 7809 | } |
| 7810 | |
| 7811 | static void reg_combine_min_max(struct bpf_reg_state *true_src, |
| 7812 | struct bpf_reg_state *true_dst, |
| 7813 | struct bpf_reg_state *false_src, |
| 7814 | struct bpf_reg_state *false_dst, |
| 7815 | u8 opcode) |
| 7816 | { |
| 7817 | switch (opcode) { |
| 7818 | case BPF_JEQ: |
| 7819 | __reg_combine_min_max(true_src, true_dst); |
| 7820 | break; |
| 7821 | case BPF_JNE: |
| 7822 | __reg_combine_min_max(false_src, false_dst); |
| 7823 | break; |
| 7824 | } |
| 7825 | } |
| 7826 | |
| 7827 | static void mark_ptr_or_null_reg(struct bpf_func_state *state, |
| 7828 | struct bpf_reg_state *reg, u32 id, |
| 7829 | bool is_null) |
| 7830 | { |
| 7831 | if (reg_type_may_be_null(reg->type) && reg->id == id && |
| 7832 | !WARN_ON_ONCE(!reg->id)) { |
| 7833 | /* Old offset (both fixed and variable parts) should |
| 7834 | * have been known-zero, because we don't allow pointer |
| 7835 | * arithmetic on pointers that might be NULL. |
| 7836 | */ |
| 7837 | if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || |
| 7838 | !tnum_equals_const(reg->var_off, 0) || |
| 7839 | reg->off)) { |
| 7840 | __mark_reg_known_zero(reg); |
| 7841 | reg->off = 0; |
| 7842 | } |
| 7843 | if (is_null) { |
| 7844 | reg->type = SCALAR_VALUE; |
| 7845 | /* We don't need id and ref_obj_id from this point |
| 7846 | * onwards anymore, thus we should better reset it, |
| 7847 | * so that state pruning has chances to take effect. |
| 7848 | */ |
| 7849 | reg->id = 0; |
| 7850 | reg->ref_obj_id = 0; |
| 7851 | |
| 7852 | return; |
| 7853 | } |
| 7854 | |
| 7855 | mark_ptr_not_null_reg(reg); |
| 7856 | |
| 7857 | if (!reg_may_point_to_spin_lock(reg)) { |
| 7858 | /* For not-NULL ptr, reg->ref_obj_id will be reset |
| 7859 | * in release_reg_references(). |
| 7860 | * |
| 7861 | * reg->id is still used by spin_lock ptr. Other |
| 7862 | * than spin_lock ptr type, reg->id can be reset. |
| 7863 | */ |
| 7864 | reg->id = 0; |
| 7865 | } |
| 7866 | } |
| 7867 | } |
| 7868 | |
| 7869 | static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, |
| 7870 | bool is_null) |
| 7871 | { |
| 7872 | struct bpf_reg_state *reg; |
| 7873 | int i; |
| 7874 | |
| 7875 | for (i = 0; i < MAX_BPF_REG; i++) |
| 7876 | mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); |
| 7877 | |
| 7878 | bpf_for_each_spilled_reg(i, state, reg) { |
| 7879 | if (!reg) |
| 7880 | continue; |
| 7881 | mark_ptr_or_null_reg(state, reg, id, is_null); |
| 7882 | } |
| 7883 | } |
| 7884 | |
| 7885 | /* The logic is similar to find_good_pkt_pointers(), both could eventually |
| 7886 | * be folded together at some point. |
| 7887 | */ |
| 7888 | static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, |
| 7889 | bool is_null) |
| 7890 | { |
| 7891 | struct bpf_func_state *state = vstate->frame[vstate->curframe]; |
| 7892 | struct bpf_reg_state *regs = state->regs; |
| 7893 | u32 ref_obj_id = regs[regno].ref_obj_id; |
| 7894 | u32 id = regs[regno].id; |
| 7895 | int i; |
| 7896 | |
| 7897 | if (ref_obj_id && ref_obj_id == id && is_null) |
| 7898 | /* regs[regno] is in the " == NULL" branch. |
| 7899 | * No one could have freed the reference state before |
| 7900 | * doing the NULL check. |
| 7901 | */ |
| 7902 | WARN_ON_ONCE(release_reference_state(state, id)); |
| 7903 | |
| 7904 | for (i = 0; i <= vstate->curframe; i++) |
| 7905 | __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); |
| 7906 | } |
| 7907 | |
| 7908 | static bool try_match_pkt_pointers(const struct bpf_insn *insn, |
| 7909 | struct bpf_reg_state *dst_reg, |
| 7910 | struct bpf_reg_state *src_reg, |
| 7911 | struct bpf_verifier_state *this_branch, |
| 7912 | struct bpf_verifier_state *other_branch) |
| 7913 | { |
| 7914 | if (BPF_SRC(insn->code) != BPF_X) |
| 7915 | return false; |
| 7916 | |
| 7917 | /* Pointers are always 64-bit. */ |
| 7918 | if (BPF_CLASS(insn->code) == BPF_JMP32) |
| 7919 | return false; |
| 7920 | |
| 7921 | switch (BPF_OP(insn->code)) { |
| 7922 | case BPF_JGT: |
| 7923 | if ((dst_reg->type == PTR_TO_PACKET && |
| 7924 | src_reg->type == PTR_TO_PACKET_END) || |
| 7925 | (dst_reg->type == PTR_TO_PACKET_META && |
| 7926 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 7927 | /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ |
| 7928 | find_good_pkt_pointers(this_branch, dst_reg, |
| 7929 | dst_reg->type, false); |
| 7930 | mark_pkt_end(other_branch, insn->dst_reg, true); |
| 7931 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 7932 | src_reg->type == PTR_TO_PACKET) || |
| 7933 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 7934 | src_reg->type == PTR_TO_PACKET_META)) { |
| 7935 | /* pkt_end > pkt_data', pkt_data > pkt_meta' */ |
| 7936 | find_good_pkt_pointers(other_branch, src_reg, |
| 7937 | src_reg->type, true); |
| 7938 | mark_pkt_end(this_branch, insn->src_reg, false); |
| 7939 | } else { |
| 7940 | return false; |
| 7941 | } |
| 7942 | break; |
| 7943 | case BPF_JLT: |
| 7944 | if ((dst_reg->type == PTR_TO_PACKET && |
| 7945 | src_reg->type == PTR_TO_PACKET_END) || |
| 7946 | (dst_reg->type == PTR_TO_PACKET_META && |
| 7947 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 7948 | /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ |
| 7949 | find_good_pkt_pointers(other_branch, dst_reg, |
| 7950 | dst_reg->type, true); |
| 7951 | mark_pkt_end(this_branch, insn->dst_reg, false); |
| 7952 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 7953 | src_reg->type == PTR_TO_PACKET) || |
| 7954 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 7955 | src_reg->type == PTR_TO_PACKET_META)) { |
| 7956 | /* pkt_end < pkt_data', pkt_data > pkt_meta' */ |
| 7957 | find_good_pkt_pointers(this_branch, src_reg, |
| 7958 | src_reg->type, false); |
| 7959 | mark_pkt_end(other_branch, insn->src_reg, true); |
| 7960 | } else { |
| 7961 | return false; |
| 7962 | } |
| 7963 | break; |
| 7964 | case BPF_JGE: |
| 7965 | if ((dst_reg->type == PTR_TO_PACKET && |
| 7966 | src_reg->type == PTR_TO_PACKET_END) || |
| 7967 | (dst_reg->type == PTR_TO_PACKET_META && |
| 7968 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 7969 | /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ |
| 7970 | find_good_pkt_pointers(this_branch, dst_reg, |
| 7971 | dst_reg->type, true); |
| 7972 | mark_pkt_end(other_branch, insn->dst_reg, false); |
| 7973 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 7974 | src_reg->type == PTR_TO_PACKET) || |
| 7975 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 7976 | src_reg->type == PTR_TO_PACKET_META)) { |
| 7977 | /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ |
| 7978 | find_good_pkt_pointers(other_branch, src_reg, |
| 7979 | src_reg->type, false); |
| 7980 | mark_pkt_end(this_branch, insn->src_reg, true); |
| 7981 | } else { |
| 7982 | return false; |
| 7983 | } |
| 7984 | break; |
| 7985 | case BPF_JLE: |
| 7986 | if ((dst_reg->type == PTR_TO_PACKET && |
| 7987 | src_reg->type == PTR_TO_PACKET_END) || |
| 7988 | (dst_reg->type == PTR_TO_PACKET_META && |
| 7989 | reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { |
| 7990 | /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ |
| 7991 | find_good_pkt_pointers(other_branch, dst_reg, |
| 7992 | dst_reg->type, false); |
| 7993 | mark_pkt_end(this_branch, insn->dst_reg, true); |
| 7994 | } else if ((dst_reg->type == PTR_TO_PACKET_END && |
| 7995 | src_reg->type == PTR_TO_PACKET) || |
| 7996 | (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && |
| 7997 | src_reg->type == PTR_TO_PACKET_META)) { |
| 7998 | /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ |
| 7999 | find_good_pkt_pointers(this_branch, src_reg, |
| 8000 | src_reg->type, true); |
| 8001 | mark_pkt_end(other_branch, insn->src_reg, false); |
| 8002 | } else { |
| 8003 | return false; |
| 8004 | } |
| 8005 | break; |
| 8006 | default: |
| 8007 | return false; |
| 8008 | } |
| 8009 | |
| 8010 | return true; |
| 8011 | } |
| 8012 | |
| 8013 | static void find_equal_scalars(struct bpf_verifier_state *vstate, |
| 8014 | struct bpf_reg_state *known_reg) |
| 8015 | { |
| 8016 | struct bpf_func_state *state; |
| 8017 | struct bpf_reg_state *reg; |
| 8018 | int i, j; |
| 8019 | |
| 8020 | for (i = 0; i <= vstate->curframe; i++) { |
| 8021 | state = vstate->frame[i]; |
| 8022 | for (j = 0; j < MAX_BPF_REG; j++) { |
| 8023 | reg = &state->regs[j]; |
| 8024 | if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) |
| 8025 | *reg = *known_reg; |
| 8026 | } |
| 8027 | |
| 8028 | bpf_for_each_spilled_reg(j, state, reg) { |
| 8029 | if (!reg) |
| 8030 | continue; |
| 8031 | if (reg->type == SCALAR_VALUE && reg->id == known_reg->id) |
| 8032 | *reg = *known_reg; |
| 8033 | } |
| 8034 | } |
| 8035 | } |
| 8036 | |
| 8037 | static int check_cond_jmp_op(struct bpf_verifier_env *env, |
| 8038 | struct bpf_insn *insn, int *insn_idx) |
| 8039 | { |
| 8040 | struct bpf_verifier_state *this_branch = env->cur_state; |
| 8041 | struct bpf_verifier_state *other_branch; |
| 8042 | struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; |
| 8043 | struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; |
| 8044 | u8 opcode = BPF_OP(insn->code); |
| 8045 | bool is_jmp32; |
| 8046 | int pred = -1; |
| 8047 | int err; |
| 8048 | |
| 8049 | /* Only conditional jumps are expected to reach here. */ |
| 8050 | if (opcode == BPF_JA || opcode > BPF_JSLE) { |
| 8051 | verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); |
| 8052 | return -EINVAL; |
| 8053 | } |
| 8054 | |
| 8055 | if (BPF_SRC(insn->code) == BPF_X) { |
| 8056 | if (insn->imm != 0) { |
| 8057 | verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); |
| 8058 | return -EINVAL; |
| 8059 | } |
| 8060 | |
| 8061 | /* check src1 operand */ |
| 8062 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
| 8063 | if (err) |
| 8064 | return err; |
| 8065 | |
| 8066 | if (is_pointer_value(env, insn->src_reg)) { |
| 8067 | verbose(env, "R%d pointer comparison prohibited\n", |
| 8068 | insn->src_reg); |
| 8069 | return -EACCES; |
| 8070 | } |
| 8071 | src_reg = ®s[insn->src_reg]; |
| 8072 | } else { |
| 8073 | if (insn->src_reg != BPF_REG_0) { |
| 8074 | verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); |
| 8075 | return -EINVAL; |
| 8076 | } |
| 8077 | } |
| 8078 | |
| 8079 | /* check src2 operand */ |
| 8080 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
| 8081 | if (err) |
| 8082 | return err; |
| 8083 | |
| 8084 | dst_reg = ®s[insn->dst_reg]; |
| 8085 | is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; |
| 8086 | |
| 8087 | if (BPF_SRC(insn->code) == BPF_K) { |
| 8088 | pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); |
| 8089 | } else if (src_reg->type == SCALAR_VALUE && |
| 8090 | is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) { |
| 8091 | pred = is_branch_taken(dst_reg, |
| 8092 | tnum_subreg(src_reg->var_off).value, |
| 8093 | opcode, |
| 8094 | is_jmp32); |
| 8095 | } else if (src_reg->type == SCALAR_VALUE && |
| 8096 | !is_jmp32 && tnum_is_const(src_reg->var_off)) { |
| 8097 | pred = is_branch_taken(dst_reg, |
| 8098 | src_reg->var_off.value, |
| 8099 | opcode, |
| 8100 | is_jmp32); |
| 8101 | } else if (reg_is_pkt_pointer_any(dst_reg) && |
| 8102 | reg_is_pkt_pointer_any(src_reg) && |
| 8103 | !is_jmp32) { |
| 8104 | pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); |
| 8105 | } |
| 8106 | |
| 8107 | if (pred >= 0) { |
| 8108 | /* If we get here with a dst_reg pointer type it is because |
| 8109 | * above is_branch_taken() special cased the 0 comparison. |
| 8110 | */ |
| 8111 | if (!__is_pointer_value(false, dst_reg)) |
| 8112 | err = mark_chain_precision(env, insn->dst_reg); |
| 8113 | if (BPF_SRC(insn->code) == BPF_X && !err && |
| 8114 | !__is_pointer_value(false, src_reg)) |
| 8115 | err = mark_chain_precision(env, insn->src_reg); |
| 8116 | if (err) |
| 8117 | return err; |
| 8118 | } |
| 8119 | if (pred == 1) { |
| 8120 | /* only follow the goto, ignore fall-through */ |
| 8121 | *insn_idx += insn->off; |
| 8122 | return 0; |
| 8123 | } else if (pred == 0) { |
| 8124 | /* only follow fall-through branch, since |
| 8125 | * that's where the program will go |
| 8126 | */ |
| 8127 | return 0; |
| 8128 | } |
| 8129 | |
| 8130 | other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, |
| 8131 | false); |
| 8132 | if (!other_branch) |
| 8133 | return -EFAULT; |
| 8134 | other_branch_regs = other_branch->frame[other_branch->curframe]->regs; |
| 8135 | |
| 8136 | /* detect if we are comparing against a constant value so we can adjust |
| 8137 | * our min/max values for our dst register. |
| 8138 | * this is only legit if both are scalars (or pointers to the same |
| 8139 | * object, I suppose, but we don't support that right now), because |
| 8140 | * otherwise the different base pointers mean the offsets aren't |
| 8141 | * comparable. |
| 8142 | */ |
| 8143 | if (BPF_SRC(insn->code) == BPF_X) { |
| 8144 | struct bpf_reg_state *src_reg = ®s[insn->src_reg]; |
| 8145 | |
| 8146 | if (dst_reg->type == SCALAR_VALUE && |
| 8147 | src_reg->type == SCALAR_VALUE) { |
| 8148 | if (tnum_is_const(src_reg->var_off) || |
| 8149 | (is_jmp32 && |
| 8150 | tnum_is_const(tnum_subreg(src_reg->var_off)))) |
| 8151 | reg_set_min_max(&other_branch_regs[insn->dst_reg], |
| 8152 | dst_reg, |
| 8153 | src_reg->var_off.value, |
| 8154 | tnum_subreg(src_reg->var_off).value, |
| 8155 | opcode, is_jmp32); |
| 8156 | else if (tnum_is_const(dst_reg->var_off) || |
| 8157 | (is_jmp32 && |
| 8158 | tnum_is_const(tnum_subreg(dst_reg->var_off)))) |
| 8159 | reg_set_min_max_inv(&other_branch_regs[insn->src_reg], |
| 8160 | src_reg, |
| 8161 | dst_reg->var_off.value, |
| 8162 | tnum_subreg(dst_reg->var_off).value, |
| 8163 | opcode, is_jmp32); |
| 8164 | else if (!is_jmp32 && |
| 8165 | (opcode == BPF_JEQ || opcode == BPF_JNE)) |
| 8166 | /* Comparing for equality, we can combine knowledge */ |
| 8167 | reg_combine_min_max(&other_branch_regs[insn->src_reg], |
| 8168 | &other_branch_regs[insn->dst_reg], |
| 8169 | src_reg, dst_reg, opcode); |
| 8170 | if (src_reg->id && |
| 8171 | !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) { |
| 8172 | find_equal_scalars(this_branch, src_reg); |
| 8173 | find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]); |
| 8174 | } |
| 8175 | |
| 8176 | } |
| 8177 | } else if (dst_reg->type == SCALAR_VALUE) { |
| 8178 | reg_set_min_max(&other_branch_regs[insn->dst_reg], |
| 8179 | dst_reg, insn->imm, (u32)insn->imm, |
| 8180 | opcode, is_jmp32); |
| 8181 | } |
| 8182 | |
| 8183 | if (dst_reg->type == SCALAR_VALUE && dst_reg->id && |
| 8184 | !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { |
| 8185 | find_equal_scalars(this_branch, dst_reg); |
| 8186 | find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); |
| 8187 | } |
| 8188 | |
| 8189 | /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). |
| 8190 | * NOTE: these optimizations below are related with pointer comparison |
| 8191 | * which will never be JMP32. |
| 8192 | */ |
| 8193 | if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && |
| 8194 | insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && |
| 8195 | reg_type_may_be_null(dst_reg->type)) { |
| 8196 | /* Mark all identical registers in each branch as either |
| 8197 | * safe or unknown depending R == 0 or R != 0 conditional. |
| 8198 | */ |
| 8199 | mark_ptr_or_null_regs(this_branch, insn->dst_reg, |
| 8200 | opcode == BPF_JNE); |
| 8201 | mark_ptr_or_null_regs(other_branch, insn->dst_reg, |
| 8202 | opcode == BPF_JEQ); |
| 8203 | } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], |
| 8204 | this_branch, other_branch) && |
| 8205 | is_pointer_value(env, insn->dst_reg)) { |
| 8206 | verbose(env, "R%d pointer comparison prohibited\n", |
| 8207 | insn->dst_reg); |
| 8208 | return -EACCES; |
| 8209 | } |
| 8210 | if (env->log.level & BPF_LOG_LEVEL) |
| 8211 | print_verifier_state(env, this_branch->frame[this_branch->curframe]); |
| 8212 | return 0; |
| 8213 | } |
| 8214 | |
| 8215 | /* verify BPF_LD_IMM64 instruction */ |
| 8216 | static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) |
| 8217 | { |
| 8218 | struct bpf_insn_aux_data *aux = cur_aux(env); |
| 8219 | struct bpf_reg_state *regs = cur_regs(env); |
| 8220 | struct bpf_reg_state *dst_reg; |
| 8221 | struct bpf_map *map; |
| 8222 | int err; |
| 8223 | |
| 8224 | if (BPF_SIZE(insn->code) != BPF_DW) { |
| 8225 | verbose(env, "invalid BPF_LD_IMM insn\n"); |
| 8226 | return -EINVAL; |
| 8227 | } |
| 8228 | if (insn->off != 0) { |
| 8229 | verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); |
| 8230 | return -EINVAL; |
| 8231 | } |
| 8232 | |
| 8233 | err = check_reg_arg(env, insn->dst_reg, DST_OP); |
| 8234 | if (err) |
| 8235 | return err; |
| 8236 | |
| 8237 | dst_reg = ®s[insn->dst_reg]; |
| 8238 | if (insn->src_reg == 0) { |
| 8239 | u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; |
| 8240 | |
| 8241 | dst_reg->type = SCALAR_VALUE; |
| 8242 | __mark_reg_known(®s[insn->dst_reg], imm); |
| 8243 | return 0; |
| 8244 | } |
| 8245 | |
| 8246 | if (insn->src_reg == BPF_PSEUDO_BTF_ID) { |
| 8247 | mark_reg_known_zero(env, regs, insn->dst_reg); |
| 8248 | |
| 8249 | dst_reg->type = aux->btf_var.reg_type; |
| 8250 | switch (dst_reg->type) { |
| 8251 | case PTR_TO_MEM: |
| 8252 | dst_reg->mem_size = aux->btf_var.mem_size; |
| 8253 | break; |
| 8254 | case PTR_TO_BTF_ID: |
| 8255 | case PTR_TO_PERCPU_BTF_ID: |
| 8256 | dst_reg->btf = aux->btf_var.btf; |
| 8257 | dst_reg->btf_id = aux->btf_var.btf_id; |
| 8258 | break; |
| 8259 | default: |
| 8260 | verbose(env, "bpf verifier is misconfigured\n"); |
| 8261 | return -EFAULT; |
| 8262 | } |
| 8263 | return 0; |
| 8264 | } |
| 8265 | |
| 8266 | map = env->used_maps[aux->map_index]; |
| 8267 | mark_reg_known_zero(env, regs, insn->dst_reg); |
| 8268 | dst_reg->map_ptr = map; |
| 8269 | |
| 8270 | if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { |
| 8271 | dst_reg->type = PTR_TO_MAP_VALUE; |
| 8272 | dst_reg->off = aux->map_off; |
| 8273 | if (map_value_has_spin_lock(map)) |
| 8274 | dst_reg->id = ++env->id_gen; |
| 8275 | } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { |
| 8276 | dst_reg->type = CONST_PTR_TO_MAP; |
| 8277 | } else { |
| 8278 | verbose(env, "bpf verifier is misconfigured\n"); |
| 8279 | return -EINVAL; |
| 8280 | } |
| 8281 | |
| 8282 | return 0; |
| 8283 | } |
| 8284 | |
| 8285 | static bool may_access_skb(enum bpf_prog_type type) |
| 8286 | { |
| 8287 | switch (type) { |
| 8288 | case BPF_PROG_TYPE_SOCKET_FILTER: |
| 8289 | case BPF_PROG_TYPE_SCHED_CLS: |
| 8290 | case BPF_PROG_TYPE_SCHED_ACT: |
| 8291 | return true; |
| 8292 | default: |
| 8293 | return false; |
| 8294 | } |
| 8295 | } |
| 8296 | |
| 8297 | /* verify safety of LD_ABS|LD_IND instructions: |
| 8298 | * - they can only appear in the programs where ctx == skb |
| 8299 | * - since they are wrappers of function calls, they scratch R1-R5 registers, |
| 8300 | * preserve R6-R9, and store return value into R0 |
| 8301 | * |
| 8302 | * Implicit input: |
| 8303 | * ctx == skb == R6 == CTX |
| 8304 | * |
| 8305 | * Explicit input: |
| 8306 | * SRC == any register |
| 8307 | * IMM == 32-bit immediate |
| 8308 | * |
| 8309 | * Output: |
| 8310 | * R0 - 8/16/32-bit skb data converted to cpu endianness |
| 8311 | */ |
| 8312 | static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) |
| 8313 | { |
| 8314 | struct bpf_reg_state *regs = cur_regs(env); |
| 8315 | static const int ctx_reg = BPF_REG_6; |
| 8316 | u8 mode = BPF_MODE(insn->code); |
| 8317 | int i, err; |
| 8318 | |
| 8319 | if (!may_access_skb(resolve_prog_type(env->prog))) { |
| 8320 | verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); |
| 8321 | return -EINVAL; |
| 8322 | } |
| 8323 | |
| 8324 | if (!env->ops->gen_ld_abs) { |
| 8325 | verbose(env, "bpf verifier is misconfigured\n"); |
| 8326 | return -EINVAL; |
| 8327 | } |
| 8328 | |
| 8329 | if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || |
| 8330 | BPF_SIZE(insn->code) == BPF_DW || |
| 8331 | (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { |
| 8332 | verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); |
| 8333 | return -EINVAL; |
| 8334 | } |
| 8335 | |
| 8336 | /* check whether implicit source operand (register R6) is readable */ |
| 8337 | err = check_reg_arg(env, ctx_reg, SRC_OP); |
| 8338 | if (err) |
| 8339 | return err; |
| 8340 | |
| 8341 | /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as |
| 8342 | * gen_ld_abs() may terminate the program at runtime, leading to |
| 8343 | * reference leak. |
| 8344 | */ |
| 8345 | err = check_reference_leak(env); |
| 8346 | if (err) { |
| 8347 | verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); |
| 8348 | return err; |
| 8349 | } |
| 8350 | |
| 8351 | if (env->cur_state->active_spin_lock) { |
| 8352 | verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); |
| 8353 | return -EINVAL; |
| 8354 | } |
| 8355 | |
| 8356 | if (regs[ctx_reg].type != PTR_TO_CTX) { |
| 8357 | verbose(env, |
| 8358 | "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); |
| 8359 | return -EINVAL; |
| 8360 | } |
| 8361 | |
| 8362 | if (mode == BPF_IND) { |
| 8363 | /* check explicit source operand */ |
| 8364 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
| 8365 | if (err) |
| 8366 | return err; |
| 8367 | } |
| 8368 | |
| 8369 | err = check_ctx_reg(env, ®s[ctx_reg], ctx_reg); |
| 8370 | if (err < 0) |
| 8371 | return err; |
| 8372 | |
| 8373 | /* reset caller saved regs to unreadable */ |
| 8374 | for (i = 0; i < CALLER_SAVED_REGS; i++) { |
| 8375 | mark_reg_not_init(env, regs, caller_saved[i]); |
| 8376 | check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); |
| 8377 | } |
| 8378 | |
| 8379 | /* mark destination R0 register as readable, since it contains |
| 8380 | * the value fetched from the packet. |
| 8381 | * Already marked as written above. |
| 8382 | */ |
| 8383 | mark_reg_unknown(env, regs, BPF_REG_0); |
| 8384 | /* ld_abs load up to 32-bit skb data. */ |
| 8385 | regs[BPF_REG_0].subreg_def = env->insn_idx + 1; |
| 8386 | return 0; |
| 8387 | } |
| 8388 | |
| 8389 | static int check_return_code(struct bpf_verifier_env *env) |
| 8390 | { |
| 8391 | struct tnum enforce_attach_type_range = tnum_unknown; |
| 8392 | const struct bpf_prog *prog = env->prog; |
| 8393 | struct bpf_reg_state *reg; |
| 8394 | struct tnum range = tnum_range(0, 1); |
| 8395 | enum bpf_prog_type prog_type = resolve_prog_type(env->prog); |
| 8396 | int err; |
| 8397 | const bool is_subprog = env->cur_state->frame[0]->subprogno; |
| 8398 | |
| 8399 | /* LSM and struct_ops func-ptr's return type could be "void" */ |
| 8400 | if (!is_subprog && |
| 8401 | (prog_type == BPF_PROG_TYPE_STRUCT_OPS || |
| 8402 | prog_type == BPF_PROG_TYPE_LSM) && |
| 8403 | !prog->aux->attach_func_proto->type) |
| 8404 | return 0; |
| 8405 | |
| 8406 | /* eBPF calling convetion is such that R0 is used |
| 8407 | * to return the value from eBPF program. |
| 8408 | * Make sure that it's readable at this time |
| 8409 | * of bpf_exit, which means that program wrote |
| 8410 | * something into it earlier |
| 8411 | */ |
| 8412 | err = check_reg_arg(env, BPF_REG_0, SRC_OP); |
| 8413 | if (err) |
| 8414 | return err; |
| 8415 | |
| 8416 | if (is_pointer_value(env, BPF_REG_0)) { |
| 8417 | verbose(env, "R0 leaks addr as return value\n"); |
| 8418 | return -EACCES; |
| 8419 | } |
| 8420 | |
| 8421 | reg = cur_regs(env) + BPF_REG_0; |
| 8422 | if (is_subprog) { |
| 8423 | if (reg->type != SCALAR_VALUE) { |
| 8424 | verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", |
| 8425 | reg_type_str[reg->type]); |
| 8426 | return -EINVAL; |
| 8427 | } |
| 8428 | return 0; |
| 8429 | } |
| 8430 | |
| 8431 | switch (prog_type) { |
| 8432 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: |
| 8433 | if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || |
| 8434 | env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG || |
| 8435 | env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME || |
| 8436 | env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME || |
| 8437 | env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME || |
| 8438 | env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) |
| 8439 | range = tnum_range(1, 1); |
| 8440 | if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND || |
| 8441 | env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND) |
| 8442 | range = tnum_range(0, 3); |
| 8443 | break; |
| 8444 | case BPF_PROG_TYPE_CGROUP_SKB: |
| 8445 | if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) { |
| 8446 | range = tnum_range(0, 3); |
| 8447 | enforce_attach_type_range = tnum_range(2, 3); |
| 8448 | } |
| 8449 | break; |
| 8450 | case BPF_PROG_TYPE_CGROUP_SOCK: |
| 8451 | case BPF_PROG_TYPE_SOCK_OPS: |
| 8452 | case BPF_PROG_TYPE_CGROUP_DEVICE: |
| 8453 | case BPF_PROG_TYPE_CGROUP_SYSCTL: |
| 8454 | case BPF_PROG_TYPE_CGROUP_SOCKOPT: |
| 8455 | break; |
| 8456 | case BPF_PROG_TYPE_RAW_TRACEPOINT: |
| 8457 | if (!env->prog->aux->attach_btf_id) |
| 8458 | return 0; |
| 8459 | range = tnum_const(0); |
| 8460 | break; |
| 8461 | case BPF_PROG_TYPE_TRACING: |
| 8462 | switch (env->prog->expected_attach_type) { |
| 8463 | case BPF_TRACE_FENTRY: |
| 8464 | case BPF_TRACE_FEXIT: |
| 8465 | range = tnum_const(0); |
| 8466 | break; |
| 8467 | case BPF_TRACE_RAW_TP: |
| 8468 | case BPF_MODIFY_RETURN: |
| 8469 | return 0; |
| 8470 | case BPF_TRACE_ITER: |
| 8471 | break; |
| 8472 | default: |
| 8473 | return -ENOTSUPP; |
| 8474 | } |
| 8475 | break; |
| 8476 | case BPF_PROG_TYPE_SK_LOOKUP: |
| 8477 | range = tnum_range(SK_DROP, SK_PASS); |
| 8478 | break; |
| 8479 | case BPF_PROG_TYPE_EXT: |
| 8480 | /* freplace program can return anything as its return value |
| 8481 | * depends on the to-be-replaced kernel func or bpf program. |
| 8482 | */ |
| 8483 | default: |
| 8484 | return 0; |
| 8485 | } |
| 8486 | |
| 8487 | if (reg->type != SCALAR_VALUE) { |
| 8488 | verbose(env, "At program exit the register R0 is not a known value (%s)\n", |
| 8489 | reg_type_str[reg->type]); |
| 8490 | return -EINVAL; |
| 8491 | } |
| 8492 | |
| 8493 | if (!tnum_in(range, reg->var_off)) { |
| 8494 | char tn_buf[48]; |
| 8495 | |
| 8496 | verbose(env, "At program exit the register R0 "); |
| 8497 | if (!tnum_is_unknown(reg->var_off)) { |
| 8498 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); |
| 8499 | verbose(env, "has value %s", tn_buf); |
| 8500 | } else { |
| 8501 | verbose(env, "has unknown scalar value"); |
| 8502 | } |
| 8503 | tnum_strn(tn_buf, sizeof(tn_buf), range); |
| 8504 | verbose(env, " should have been in %s\n", tn_buf); |
| 8505 | return -EINVAL; |
| 8506 | } |
| 8507 | |
| 8508 | if (!tnum_is_unknown(enforce_attach_type_range) && |
| 8509 | tnum_in(enforce_attach_type_range, reg->var_off)) |
| 8510 | env->prog->enforce_expected_attach_type = 1; |
| 8511 | return 0; |
| 8512 | } |
| 8513 | |
| 8514 | /* non-recursive DFS pseudo code |
| 8515 | * 1 procedure DFS-iterative(G,v): |
| 8516 | * 2 label v as discovered |
| 8517 | * 3 let S be a stack |
| 8518 | * 4 S.push(v) |
| 8519 | * 5 while S is not empty |
| 8520 | * 6 t <- S.pop() |
| 8521 | * 7 if t is what we're looking for: |
| 8522 | * 8 return t |
| 8523 | * 9 for all edges e in G.adjacentEdges(t) do |
| 8524 | * 10 if edge e is already labelled |
| 8525 | * 11 continue with the next edge |
| 8526 | * 12 w <- G.adjacentVertex(t,e) |
| 8527 | * 13 if vertex w is not discovered and not explored |
| 8528 | * 14 label e as tree-edge |
| 8529 | * 15 label w as discovered |
| 8530 | * 16 S.push(w) |
| 8531 | * 17 continue at 5 |
| 8532 | * 18 else if vertex w is discovered |
| 8533 | * 19 label e as back-edge |
| 8534 | * 20 else |
| 8535 | * 21 // vertex w is explored |
| 8536 | * 22 label e as forward- or cross-edge |
| 8537 | * 23 label t as explored |
| 8538 | * 24 S.pop() |
| 8539 | * |
| 8540 | * convention: |
| 8541 | * 0x10 - discovered |
| 8542 | * 0x11 - discovered and fall-through edge labelled |
| 8543 | * 0x12 - discovered and fall-through and branch edges labelled |
| 8544 | * 0x20 - explored |
| 8545 | */ |
| 8546 | |
| 8547 | enum { |
| 8548 | DISCOVERED = 0x10, |
| 8549 | EXPLORED = 0x20, |
| 8550 | FALLTHROUGH = 1, |
| 8551 | BRANCH = 2, |
| 8552 | }; |
| 8553 | |
| 8554 | static u32 state_htab_size(struct bpf_verifier_env *env) |
| 8555 | { |
| 8556 | return env->prog->len; |
| 8557 | } |
| 8558 | |
| 8559 | static struct bpf_verifier_state_list **explored_state( |
| 8560 | struct bpf_verifier_env *env, |
| 8561 | int idx) |
| 8562 | { |
| 8563 | struct bpf_verifier_state *cur = env->cur_state; |
| 8564 | struct bpf_func_state *state = cur->frame[cur->curframe]; |
| 8565 | |
| 8566 | return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)]; |
| 8567 | } |
| 8568 | |
| 8569 | static void init_explored_state(struct bpf_verifier_env *env, int idx) |
| 8570 | { |
| 8571 | env->insn_aux_data[idx].prune_point = true; |
| 8572 | } |
| 8573 | |
| 8574 | enum { |
| 8575 | DONE_EXPLORING = 0, |
| 8576 | KEEP_EXPLORING = 1, |
| 8577 | }; |
| 8578 | |
| 8579 | /* t, w, e - match pseudo-code above: |
| 8580 | * t - index of current instruction |
| 8581 | * w - next instruction |
| 8582 | * e - edge |
| 8583 | */ |
| 8584 | static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, |
| 8585 | bool loop_ok) |
| 8586 | { |
| 8587 | int *insn_stack = env->cfg.insn_stack; |
| 8588 | int *insn_state = env->cfg.insn_state; |
| 8589 | |
| 8590 | if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) |
| 8591 | return DONE_EXPLORING; |
| 8592 | |
| 8593 | if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) |
| 8594 | return DONE_EXPLORING; |
| 8595 | |
| 8596 | if (w < 0 || w >= env->prog->len) { |
| 8597 | verbose_linfo(env, t, "%d: ", t); |
| 8598 | verbose(env, "jump out of range from insn %d to %d\n", t, w); |
| 8599 | return -EINVAL; |
| 8600 | } |
| 8601 | |
| 8602 | if (e == BRANCH) |
| 8603 | /* mark branch target for state pruning */ |
| 8604 | init_explored_state(env, w); |
| 8605 | |
| 8606 | if (insn_state[w] == 0) { |
| 8607 | /* tree-edge */ |
| 8608 | insn_state[t] = DISCOVERED | e; |
| 8609 | insn_state[w] = DISCOVERED; |
| 8610 | if (env->cfg.cur_stack >= env->prog->len) |
| 8611 | return -E2BIG; |
| 8612 | insn_stack[env->cfg.cur_stack++] = w; |
| 8613 | return KEEP_EXPLORING; |
| 8614 | } else if ((insn_state[w] & 0xF0) == DISCOVERED) { |
| 8615 | if (loop_ok && env->bpf_capable) |
| 8616 | return DONE_EXPLORING; |
| 8617 | verbose_linfo(env, t, "%d: ", t); |
| 8618 | verbose_linfo(env, w, "%d: ", w); |
| 8619 | verbose(env, "back-edge from insn %d to %d\n", t, w); |
| 8620 | return -EINVAL; |
| 8621 | } else if (insn_state[w] == EXPLORED) { |
| 8622 | /* forward- or cross-edge */ |
| 8623 | insn_state[t] = DISCOVERED | e; |
| 8624 | } else { |
| 8625 | verbose(env, "insn state internal bug\n"); |
| 8626 | return -EFAULT; |
| 8627 | } |
| 8628 | return DONE_EXPLORING; |
| 8629 | } |
| 8630 | |
| 8631 | /* Visits the instruction at index t and returns one of the following: |
| 8632 | * < 0 - an error occurred |
| 8633 | * DONE_EXPLORING - the instruction was fully explored |
| 8634 | * KEEP_EXPLORING - there is still work to be done before it is fully explored |
| 8635 | */ |
| 8636 | static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env) |
| 8637 | { |
| 8638 | struct bpf_insn *insns = env->prog->insnsi; |
| 8639 | int ret; |
| 8640 | |
| 8641 | /* All non-branch instructions have a single fall-through edge. */ |
| 8642 | if (BPF_CLASS(insns[t].code) != BPF_JMP && |
| 8643 | BPF_CLASS(insns[t].code) != BPF_JMP32) |
| 8644 | return push_insn(t, t + 1, FALLTHROUGH, env, false); |
| 8645 | |
| 8646 | switch (BPF_OP(insns[t].code)) { |
| 8647 | case BPF_EXIT: |
| 8648 | return DONE_EXPLORING; |
| 8649 | |
| 8650 | case BPF_CALL: |
| 8651 | ret = push_insn(t, t + 1, FALLTHROUGH, env, false); |
| 8652 | if (ret) |
| 8653 | return ret; |
| 8654 | |
| 8655 | if (t + 1 < insn_cnt) |
| 8656 | init_explored_state(env, t + 1); |
| 8657 | if (insns[t].src_reg == BPF_PSEUDO_CALL) { |
| 8658 | init_explored_state(env, t); |
| 8659 | ret = push_insn(t, t + insns[t].imm + 1, BRANCH, |
| 8660 | env, false); |
| 8661 | } |
| 8662 | return ret; |
| 8663 | |
| 8664 | case BPF_JA: |
| 8665 | if (BPF_SRC(insns[t].code) != BPF_K) |
| 8666 | return -EINVAL; |
| 8667 | |
| 8668 | /* unconditional jump with single edge */ |
| 8669 | ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, |
| 8670 | true); |
| 8671 | if (ret) |
| 8672 | return ret; |
| 8673 | |
| 8674 | /* unconditional jmp is not a good pruning point, |
| 8675 | * but it's marked, since backtracking needs |
| 8676 | * to record jmp history in is_state_visited(). |
| 8677 | */ |
| 8678 | init_explored_state(env, t + insns[t].off + 1); |
| 8679 | /* tell verifier to check for equivalent states |
| 8680 | * after every call and jump |
| 8681 | */ |
| 8682 | if (t + 1 < insn_cnt) |
| 8683 | init_explored_state(env, t + 1); |
| 8684 | |
| 8685 | return ret; |
| 8686 | |
| 8687 | default: |
| 8688 | /* conditional jump with two edges */ |
| 8689 | init_explored_state(env, t); |
| 8690 | ret = push_insn(t, t + 1, FALLTHROUGH, env, true); |
| 8691 | if (ret) |
| 8692 | return ret; |
| 8693 | |
| 8694 | return push_insn(t, t + insns[t].off + 1, BRANCH, env, true); |
| 8695 | } |
| 8696 | } |
| 8697 | |
| 8698 | /* non-recursive depth-first-search to detect loops in BPF program |
| 8699 | * loop == back-edge in directed graph |
| 8700 | */ |
| 8701 | static int check_cfg(struct bpf_verifier_env *env) |
| 8702 | { |
| 8703 | int insn_cnt = env->prog->len; |
| 8704 | int *insn_stack, *insn_state; |
| 8705 | int ret = 0; |
| 8706 | int i; |
| 8707 | |
| 8708 | insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
| 8709 | if (!insn_state) |
| 8710 | return -ENOMEM; |
| 8711 | |
| 8712 | insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); |
| 8713 | if (!insn_stack) { |
| 8714 | kvfree(insn_state); |
| 8715 | return -ENOMEM; |
| 8716 | } |
| 8717 | |
| 8718 | insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ |
| 8719 | insn_stack[0] = 0; /* 0 is the first instruction */ |
| 8720 | env->cfg.cur_stack = 1; |
| 8721 | |
| 8722 | while (env->cfg.cur_stack > 0) { |
| 8723 | int t = insn_stack[env->cfg.cur_stack - 1]; |
| 8724 | |
| 8725 | ret = visit_insn(t, insn_cnt, env); |
| 8726 | switch (ret) { |
| 8727 | case DONE_EXPLORING: |
| 8728 | insn_state[t] = EXPLORED; |
| 8729 | env->cfg.cur_stack--; |
| 8730 | break; |
| 8731 | case KEEP_EXPLORING: |
| 8732 | break; |
| 8733 | default: |
| 8734 | if (ret > 0) { |
| 8735 | verbose(env, "visit_insn internal bug\n"); |
| 8736 | ret = -EFAULT; |
| 8737 | } |
| 8738 | goto err_free; |
| 8739 | } |
| 8740 | } |
| 8741 | |
| 8742 | if (env->cfg.cur_stack < 0) { |
| 8743 | verbose(env, "pop stack internal bug\n"); |
| 8744 | ret = -EFAULT; |
| 8745 | goto err_free; |
| 8746 | } |
| 8747 | |
| 8748 | for (i = 0; i < insn_cnt; i++) { |
| 8749 | if (insn_state[i] != EXPLORED) { |
| 8750 | verbose(env, "unreachable insn %d\n", i); |
| 8751 | ret = -EINVAL; |
| 8752 | goto err_free; |
| 8753 | } |
| 8754 | } |
| 8755 | ret = 0; /* cfg looks good */ |
| 8756 | |
| 8757 | err_free: |
| 8758 | kvfree(insn_state); |
| 8759 | kvfree(insn_stack); |
| 8760 | env->cfg.insn_state = env->cfg.insn_stack = NULL; |
| 8761 | return ret; |
| 8762 | } |
| 8763 | |
| 8764 | static int check_abnormal_return(struct bpf_verifier_env *env) |
| 8765 | { |
| 8766 | int i; |
| 8767 | |
| 8768 | for (i = 1; i < env->subprog_cnt; i++) { |
| 8769 | if (env->subprog_info[i].has_ld_abs) { |
| 8770 | verbose(env, "LD_ABS is not allowed in subprogs without BTF\n"); |
| 8771 | return -EINVAL; |
| 8772 | } |
| 8773 | if (env->subprog_info[i].has_tail_call) { |
| 8774 | verbose(env, "tail_call is not allowed in subprogs without BTF\n"); |
| 8775 | return -EINVAL; |
| 8776 | } |
| 8777 | } |
| 8778 | return 0; |
| 8779 | } |
| 8780 | |
| 8781 | /* The minimum supported BTF func info size */ |
| 8782 | #define MIN_BPF_FUNCINFO_SIZE 8 |
| 8783 | #define MAX_FUNCINFO_REC_SIZE 252 |
| 8784 | |
| 8785 | static int check_btf_func(struct bpf_verifier_env *env, |
| 8786 | const union bpf_attr *attr, |
| 8787 | union bpf_attr __user *uattr) |
| 8788 | { |
| 8789 | const struct btf_type *type, *func_proto, *ret_type; |
| 8790 | u32 i, nfuncs, urec_size, min_size; |
| 8791 | u32 krec_size = sizeof(struct bpf_func_info); |
| 8792 | struct bpf_func_info *krecord; |
| 8793 | struct bpf_func_info_aux *info_aux = NULL; |
| 8794 | struct bpf_prog *prog; |
| 8795 | const struct btf *btf; |
| 8796 | void __user *urecord; |
| 8797 | u32 prev_offset = 0; |
| 8798 | bool scalar_return; |
| 8799 | int ret = -ENOMEM; |
| 8800 | |
| 8801 | nfuncs = attr->func_info_cnt; |
| 8802 | if (!nfuncs) { |
| 8803 | if (check_abnormal_return(env)) |
| 8804 | return -EINVAL; |
| 8805 | return 0; |
| 8806 | } |
| 8807 | |
| 8808 | if (nfuncs != env->subprog_cnt) { |
| 8809 | verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); |
| 8810 | return -EINVAL; |
| 8811 | } |
| 8812 | |
| 8813 | urec_size = attr->func_info_rec_size; |
| 8814 | if (urec_size < MIN_BPF_FUNCINFO_SIZE || |
| 8815 | urec_size > MAX_FUNCINFO_REC_SIZE || |
| 8816 | urec_size % sizeof(u32)) { |
| 8817 | verbose(env, "invalid func info rec size %u\n", urec_size); |
| 8818 | return -EINVAL; |
| 8819 | } |
| 8820 | |
| 8821 | prog = env->prog; |
| 8822 | btf = prog->aux->btf; |
| 8823 | |
| 8824 | urecord = u64_to_user_ptr(attr->func_info); |
| 8825 | min_size = min_t(u32, krec_size, urec_size); |
| 8826 | |
| 8827 | krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); |
| 8828 | if (!krecord) |
| 8829 | return -ENOMEM; |
| 8830 | info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN); |
| 8831 | if (!info_aux) |
| 8832 | goto err_free; |
| 8833 | |
| 8834 | for (i = 0; i < nfuncs; i++) { |
| 8835 | ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); |
| 8836 | if (ret) { |
| 8837 | if (ret == -E2BIG) { |
| 8838 | verbose(env, "nonzero tailing record in func info"); |
| 8839 | /* set the size kernel expects so loader can zero |
| 8840 | * out the rest of the record. |
| 8841 | */ |
| 8842 | if (put_user(min_size, &uattr->func_info_rec_size)) |
| 8843 | ret = -EFAULT; |
| 8844 | } |
| 8845 | goto err_free; |
| 8846 | } |
| 8847 | |
| 8848 | if (copy_from_user(&krecord[i], urecord, min_size)) { |
| 8849 | ret = -EFAULT; |
| 8850 | goto err_free; |
| 8851 | } |
| 8852 | |
| 8853 | /* check insn_off */ |
| 8854 | ret = -EINVAL; |
| 8855 | if (i == 0) { |
| 8856 | if (krecord[i].insn_off) { |
| 8857 | verbose(env, |
| 8858 | "nonzero insn_off %u for the first func info record", |
| 8859 | krecord[i].insn_off); |
| 8860 | goto err_free; |
| 8861 | } |
| 8862 | } else if (krecord[i].insn_off <= prev_offset) { |
| 8863 | verbose(env, |
| 8864 | "same or smaller insn offset (%u) than previous func info record (%u)", |
| 8865 | krecord[i].insn_off, prev_offset); |
| 8866 | goto err_free; |
| 8867 | } |
| 8868 | |
| 8869 | if (env->subprog_info[i].start != krecord[i].insn_off) { |
| 8870 | verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); |
| 8871 | goto err_free; |
| 8872 | } |
| 8873 | |
| 8874 | /* check type_id */ |
| 8875 | type = btf_type_by_id(btf, krecord[i].type_id); |
| 8876 | if (!type || !btf_type_is_func(type)) { |
| 8877 | verbose(env, "invalid type id %d in func info", |
| 8878 | krecord[i].type_id); |
| 8879 | goto err_free; |
| 8880 | } |
| 8881 | info_aux[i].linkage = BTF_INFO_VLEN(type->info); |
| 8882 | |
| 8883 | func_proto = btf_type_by_id(btf, type->type); |
| 8884 | if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto))) |
| 8885 | /* btf_func_check() already verified it during BTF load */ |
| 8886 | goto err_free; |
| 8887 | ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL); |
| 8888 | scalar_return = |
| 8889 | btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type); |
| 8890 | if (i && !scalar_return && env->subprog_info[i].has_ld_abs) { |
| 8891 | verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n"); |
| 8892 | goto err_free; |
| 8893 | } |
| 8894 | if (i && !scalar_return && env->subprog_info[i].has_tail_call) { |
| 8895 | verbose(env, "tail_call is only allowed in functions that return 'int'.\n"); |
| 8896 | goto err_free; |
| 8897 | } |
| 8898 | |
| 8899 | prev_offset = krecord[i].insn_off; |
| 8900 | urecord += urec_size; |
| 8901 | } |
| 8902 | |
| 8903 | prog->aux->func_info = krecord; |
| 8904 | prog->aux->func_info_cnt = nfuncs; |
| 8905 | prog->aux->func_info_aux = info_aux; |
| 8906 | return 0; |
| 8907 | |
| 8908 | err_free: |
| 8909 | kvfree(krecord); |
| 8910 | kfree(info_aux); |
| 8911 | return ret; |
| 8912 | } |
| 8913 | |
| 8914 | static void adjust_btf_func(struct bpf_verifier_env *env) |
| 8915 | { |
| 8916 | struct bpf_prog_aux *aux = env->prog->aux; |
| 8917 | int i; |
| 8918 | |
| 8919 | if (!aux->func_info) |
| 8920 | return; |
| 8921 | |
| 8922 | for (i = 0; i < env->subprog_cnt; i++) |
| 8923 | aux->func_info[i].insn_off = env->subprog_info[i].start; |
| 8924 | } |
| 8925 | |
| 8926 | #define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ |
| 8927 | sizeof(((struct bpf_line_info *)(0))->line_col)) |
| 8928 | #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE |
| 8929 | |
| 8930 | static int check_btf_line(struct bpf_verifier_env *env, |
| 8931 | const union bpf_attr *attr, |
| 8932 | union bpf_attr __user *uattr) |
| 8933 | { |
| 8934 | u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; |
| 8935 | struct bpf_subprog_info *sub; |
| 8936 | struct bpf_line_info *linfo; |
| 8937 | struct bpf_prog *prog; |
| 8938 | const struct btf *btf; |
| 8939 | void __user *ulinfo; |
| 8940 | int err; |
| 8941 | |
| 8942 | nr_linfo = attr->line_info_cnt; |
| 8943 | if (!nr_linfo) |
| 8944 | return 0; |
| 8945 | |
| 8946 | rec_size = attr->line_info_rec_size; |
| 8947 | if (rec_size < MIN_BPF_LINEINFO_SIZE || |
| 8948 | rec_size > MAX_LINEINFO_REC_SIZE || |
| 8949 | rec_size & (sizeof(u32) - 1)) |
| 8950 | return -EINVAL; |
| 8951 | |
| 8952 | /* Need to zero it in case the userspace may |
| 8953 | * pass in a smaller bpf_line_info object. |
| 8954 | */ |
| 8955 | linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), |
| 8956 | GFP_KERNEL | __GFP_NOWARN); |
| 8957 | if (!linfo) |
| 8958 | return -ENOMEM; |
| 8959 | |
| 8960 | prog = env->prog; |
| 8961 | btf = prog->aux->btf; |
| 8962 | |
| 8963 | s = 0; |
| 8964 | sub = env->subprog_info; |
| 8965 | ulinfo = u64_to_user_ptr(attr->line_info); |
| 8966 | expected_size = sizeof(struct bpf_line_info); |
| 8967 | ncopy = min_t(u32, expected_size, rec_size); |
| 8968 | for (i = 0; i < nr_linfo; i++) { |
| 8969 | err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); |
| 8970 | if (err) { |
| 8971 | if (err == -E2BIG) { |
| 8972 | verbose(env, "nonzero tailing record in line_info"); |
| 8973 | if (put_user(expected_size, |
| 8974 | &uattr->line_info_rec_size)) |
| 8975 | err = -EFAULT; |
| 8976 | } |
| 8977 | goto err_free; |
| 8978 | } |
| 8979 | |
| 8980 | if (copy_from_user(&linfo[i], ulinfo, ncopy)) { |
| 8981 | err = -EFAULT; |
| 8982 | goto err_free; |
| 8983 | } |
| 8984 | |
| 8985 | /* |
| 8986 | * Check insn_off to ensure |
| 8987 | * 1) strictly increasing AND |
| 8988 | * 2) bounded by prog->len |
| 8989 | * |
| 8990 | * The linfo[0].insn_off == 0 check logically falls into |
| 8991 | * the later "missing bpf_line_info for func..." case |
| 8992 | * because the first linfo[0].insn_off must be the |
| 8993 | * first sub also and the first sub must have |
| 8994 | * subprog_info[0].start == 0. |
| 8995 | */ |
| 8996 | if ((i && linfo[i].insn_off <= prev_offset) || |
| 8997 | linfo[i].insn_off >= prog->len) { |
| 8998 | verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", |
| 8999 | i, linfo[i].insn_off, prev_offset, |
| 9000 | prog->len); |
| 9001 | err = -EINVAL; |
| 9002 | goto err_free; |
| 9003 | } |
| 9004 | |
| 9005 | if (!prog->insnsi[linfo[i].insn_off].code) { |
| 9006 | verbose(env, |
| 9007 | "Invalid insn code at line_info[%u].insn_off\n", |
| 9008 | i); |
| 9009 | err = -EINVAL; |
| 9010 | goto err_free; |
| 9011 | } |
| 9012 | |
| 9013 | if (!btf_name_by_offset(btf, linfo[i].line_off) || |
| 9014 | !btf_name_by_offset(btf, linfo[i].file_name_off)) { |
| 9015 | verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); |
| 9016 | err = -EINVAL; |
| 9017 | goto err_free; |
| 9018 | } |
| 9019 | |
| 9020 | if (s != env->subprog_cnt) { |
| 9021 | if (linfo[i].insn_off == sub[s].start) { |
| 9022 | sub[s].linfo_idx = i; |
| 9023 | s++; |
| 9024 | } else if (sub[s].start < linfo[i].insn_off) { |
| 9025 | verbose(env, "missing bpf_line_info for func#%u\n", s); |
| 9026 | err = -EINVAL; |
| 9027 | goto err_free; |
| 9028 | } |
| 9029 | } |
| 9030 | |
| 9031 | prev_offset = linfo[i].insn_off; |
| 9032 | ulinfo += rec_size; |
| 9033 | } |
| 9034 | |
| 9035 | if (s != env->subprog_cnt) { |
| 9036 | verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", |
| 9037 | env->subprog_cnt - s, s); |
| 9038 | err = -EINVAL; |
| 9039 | goto err_free; |
| 9040 | } |
| 9041 | |
| 9042 | prog->aux->linfo = linfo; |
| 9043 | prog->aux->nr_linfo = nr_linfo; |
| 9044 | |
| 9045 | return 0; |
| 9046 | |
| 9047 | err_free: |
| 9048 | kvfree(linfo); |
| 9049 | return err; |
| 9050 | } |
| 9051 | |
| 9052 | static int check_btf_info(struct bpf_verifier_env *env, |
| 9053 | const union bpf_attr *attr, |
| 9054 | union bpf_attr __user *uattr) |
| 9055 | { |
| 9056 | struct btf *btf; |
| 9057 | int err; |
| 9058 | |
| 9059 | if (!attr->func_info_cnt && !attr->line_info_cnt) { |
| 9060 | if (check_abnormal_return(env)) |
| 9061 | return -EINVAL; |
| 9062 | return 0; |
| 9063 | } |
| 9064 | |
| 9065 | btf = btf_get_by_fd(attr->prog_btf_fd); |
| 9066 | if (IS_ERR(btf)) |
| 9067 | return PTR_ERR(btf); |
| 9068 | if (btf_is_kernel(btf)) { |
| 9069 | btf_put(btf); |
| 9070 | return -EACCES; |
| 9071 | } |
| 9072 | env->prog->aux->btf = btf; |
| 9073 | |
| 9074 | err = check_btf_func(env, attr, uattr); |
| 9075 | if (err) |
| 9076 | return err; |
| 9077 | |
| 9078 | err = check_btf_line(env, attr, uattr); |
| 9079 | if (err) |
| 9080 | return err; |
| 9081 | |
| 9082 | return 0; |
| 9083 | } |
| 9084 | |
| 9085 | /* check %cur's range satisfies %old's */ |
| 9086 | static bool range_within(struct bpf_reg_state *old, |
| 9087 | struct bpf_reg_state *cur) |
| 9088 | { |
| 9089 | return old->umin_value <= cur->umin_value && |
| 9090 | old->umax_value >= cur->umax_value && |
| 9091 | old->smin_value <= cur->smin_value && |
| 9092 | old->smax_value >= cur->smax_value && |
| 9093 | old->u32_min_value <= cur->u32_min_value && |
| 9094 | old->u32_max_value >= cur->u32_max_value && |
| 9095 | old->s32_min_value <= cur->s32_min_value && |
| 9096 | old->s32_max_value >= cur->s32_max_value; |
| 9097 | } |
| 9098 | |
| 9099 | /* Maximum number of register states that can exist at once */ |
| 9100 | #define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) |
| 9101 | struct idpair { |
| 9102 | u32 old; |
| 9103 | u32 cur; |
| 9104 | }; |
| 9105 | |
| 9106 | /* If in the old state two registers had the same id, then they need to have |
| 9107 | * the same id in the new state as well. But that id could be different from |
| 9108 | * the old state, so we need to track the mapping from old to new ids. |
| 9109 | * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent |
| 9110 | * regs with old id 5 must also have new id 9 for the new state to be safe. But |
| 9111 | * regs with a different old id could still have new id 9, we don't care about |
| 9112 | * that. |
| 9113 | * So we look through our idmap to see if this old id has been seen before. If |
| 9114 | * so, we require the new id to match; otherwise, we add the id pair to the map. |
| 9115 | */ |
| 9116 | static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap) |
| 9117 | { |
| 9118 | unsigned int i; |
| 9119 | |
| 9120 | for (i = 0; i < ID_MAP_SIZE; i++) { |
| 9121 | if (!idmap[i].old) { |
| 9122 | /* Reached an empty slot; haven't seen this id before */ |
| 9123 | idmap[i].old = old_id; |
| 9124 | idmap[i].cur = cur_id; |
| 9125 | return true; |
| 9126 | } |
| 9127 | if (idmap[i].old == old_id) |
| 9128 | return idmap[i].cur == cur_id; |
| 9129 | } |
| 9130 | /* We ran out of idmap slots, which should be impossible */ |
| 9131 | WARN_ON_ONCE(1); |
| 9132 | return false; |
| 9133 | } |
| 9134 | |
| 9135 | static void clean_func_state(struct bpf_verifier_env *env, |
| 9136 | struct bpf_func_state *st) |
| 9137 | { |
| 9138 | enum bpf_reg_liveness live; |
| 9139 | int i, j; |
| 9140 | |
| 9141 | for (i = 0; i < BPF_REG_FP; i++) { |
| 9142 | live = st->regs[i].live; |
| 9143 | /* liveness must not touch this register anymore */ |
| 9144 | st->regs[i].live |= REG_LIVE_DONE; |
| 9145 | if (!(live & REG_LIVE_READ)) |
| 9146 | /* since the register is unused, clear its state |
| 9147 | * to make further comparison simpler |
| 9148 | */ |
| 9149 | __mark_reg_not_init(env, &st->regs[i]); |
| 9150 | } |
| 9151 | |
| 9152 | for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { |
| 9153 | live = st->stack[i].spilled_ptr.live; |
| 9154 | /* liveness must not touch this stack slot anymore */ |
| 9155 | st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; |
| 9156 | if (!(live & REG_LIVE_READ)) { |
| 9157 | __mark_reg_not_init(env, &st->stack[i].spilled_ptr); |
| 9158 | for (j = 0; j < BPF_REG_SIZE; j++) |
| 9159 | st->stack[i].slot_type[j] = STACK_INVALID; |
| 9160 | } |
| 9161 | } |
| 9162 | } |
| 9163 | |
| 9164 | static void clean_verifier_state(struct bpf_verifier_env *env, |
| 9165 | struct bpf_verifier_state *st) |
| 9166 | { |
| 9167 | int i; |
| 9168 | |
| 9169 | if (st->frame[0]->regs[0].live & REG_LIVE_DONE) |
| 9170 | /* all regs in this state in all frames were already marked */ |
| 9171 | return; |
| 9172 | |
| 9173 | for (i = 0; i <= st->curframe; i++) |
| 9174 | clean_func_state(env, st->frame[i]); |
| 9175 | } |
| 9176 | |
| 9177 | /* the parentage chains form a tree. |
| 9178 | * the verifier states are added to state lists at given insn and |
| 9179 | * pushed into state stack for future exploration. |
| 9180 | * when the verifier reaches bpf_exit insn some of the verifer states |
| 9181 | * stored in the state lists have their final liveness state already, |
| 9182 | * but a lot of states will get revised from liveness point of view when |
| 9183 | * the verifier explores other branches. |
| 9184 | * Example: |
| 9185 | * 1: r0 = 1 |
| 9186 | * 2: if r1 == 100 goto pc+1 |
| 9187 | * 3: r0 = 2 |
| 9188 | * 4: exit |
| 9189 | * when the verifier reaches exit insn the register r0 in the state list of |
| 9190 | * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch |
| 9191 | * of insn 2 and goes exploring further. At the insn 4 it will walk the |
| 9192 | * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. |
| 9193 | * |
| 9194 | * Since the verifier pushes the branch states as it sees them while exploring |
| 9195 | * the program the condition of walking the branch instruction for the second |
| 9196 | * time means that all states below this branch were already explored and |
| 9197 | * their final liveness markes are already propagated. |
| 9198 | * Hence when the verifier completes the search of state list in is_state_visited() |
| 9199 | * we can call this clean_live_states() function to mark all liveness states |
| 9200 | * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' |
| 9201 | * will not be used. |
| 9202 | * This function also clears the registers and stack for states that !READ |
| 9203 | * to simplify state merging. |
| 9204 | * |
| 9205 | * Important note here that walking the same branch instruction in the callee |
| 9206 | * doesn't meant that the states are DONE. The verifier has to compare |
| 9207 | * the callsites |
| 9208 | */ |
| 9209 | static void clean_live_states(struct bpf_verifier_env *env, int insn, |
| 9210 | struct bpf_verifier_state *cur) |
| 9211 | { |
| 9212 | struct bpf_verifier_state_list *sl; |
| 9213 | int i; |
| 9214 | |
| 9215 | sl = *explored_state(env, insn); |
| 9216 | while (sl) { |
| 9217 | if (sl->state.branches) |
| 9218 | goto next; |
| 9219 | if (sl->state.insn_idx != insn || |
| 9220 | sl->state.curframe != cur->curframe) |
| 9221 | goto next; |
| 9222 | for (i = 0; i <= cur->curframe; i++) |
| 9223 | if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) |
| 9224 | goto next; |
| 9225 | clean_verifier_state(env, &sl->state); |
| 9226 | next: |
| 9227 | sl = sl->next; |
| 9228 | } |
| 9229 | } |
| 9230 | |
| 9231 | /* Returns true if (rold safe implies rcur safe) */ |
| 9232 | static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, |
| 9233 | struct idpair *idmap) |
| 9234 | { |
| 9235 | bool equal; |
| 9236 | |
| 9237 | if (!(rold->live & REG_LIVE_READ)) |
| 9238 | /* explored state didn't use this */ |
| 9239 | return true; |
| 9240 | |
| 9241 | equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; |
| 9242 | |
| 9243 | if (rold->type == PTR_TO_STACK) |
| 9244 | /* two stack pointers are equal only if they're pointing to |
| 9245 | * the same stack frame, since fp-8 in foo != fp-8 in bar |
| 9246 | */ |
| 9247 | return equal && rold->frameno == rcur->frameno; |
| 9248 | |
| 9249 | if (equal) |
| 9250 | return true; |
| 9251 | |
| 9252 | if (rold->type == NOT_INIT) |
| 9253 | /* explored state can't have used this */ |
| 9254 | return true; |
| 9255 | if (rcur->type == NOT_INIT) |
| 9256 | return false; |
| 9257 | switch (rold->type) { |
| 9258 | case SCALAR_VALUE: |
| 9259 | if (rcur->type == SCALAR_VALUE) { |
| 9260 | if (!rold->precise && !rcur->precise) |
| 9261 | return true; |
| 9262 | /* new val must satisfy old val knowledge */ |
| 9263 | return range_within(rold, rcur) && |
| 9264 | tnum_in(rold->var_off, rcur->var_off); |
| 9265 | } else { |
| 9266 | /* We're trying to use a pointer in place of a scalar. |
| 9267 | * Even if the scalar was unbounded, this could lead to |
| 9268 | * pointer leaks because scalars are allowed to leak |
| 9269 | * while pointers are not. We could make this safe in |
| 9270 | * special cases if root is calling us, but it's |
| 9271 | * probably not worth the hassle. |
| 9272 | */ |
| 9273 | return false; |
| 9274 | } |
| 9275 | case PTR_TO_MAP_VALUE: |
| 9276 | /* If the new min/max/var_off satisfy the old ones and |
| 9277 | * everything else matches, we are OK. |
| 9278 | * 'id' is not compared, since it's only used for maps with |
| 9279 | * bpf_spin_lock inside map element and in such cases if |
| 9280 | * the rest of the prog is valid for one map element then |
| 9281 | * it's valid for all map elements regardless of the key |
| 9282 | * used in bpf_map_lookup() |
| 9283 | */ |
| 9284 | return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && |
| 9285 | range_within(rold, rcur) && |
| 9286 | tnum_in(rold->var_off, rcur->var_off); |
| 9287 | case PTR_TO_MAP_VALUE_OR_NULL: |
| 9288 | /* a PTR_TO_MAP_VALUE could be safe to use as a |
| 9289 | * PTR_TO_MAP_VALUE_OR_NULL into the same map. |
| 9290 | * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- |
| 9291 | * checked, doing so could have affected others with the same |
| 9292 | * id, and we can't check for that because we lost the id when |
| 9293 | * we converted to a PTR_TO_MAP_VALUE. |
| 9294 | */ |
| 9295 | if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) |
| 9296 | return false; |
| 9297 | if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) |
| 9298 | return false; |
| 9299 | /* Check our ids match any regs they're supposed to */ |
| 9300 | return check_ids(rold->id, rcur->id, idmap); |
| 9301 | case PTR_TO_PACKET_META: |
| 9302 | case PTR_TO_PACKET: |
| 9303 | if (rcur->type != rold->type) |
| 9304 | return false; |
| 9305 | /* We must have at least as much range as the old ptr |
| 9306 | * did, so that any accesses which were safe before are |
| 9307 | * still safe. This is true even if old range < old off, |
| 9308 | * since someone could have accessed through (ptr - k), or |
| 9309 | * even done ptr -= k in a register, to get a safe access. |
| 9310 | */ |
| 9311 | if (rold->range > rcur->range) |
| 9312 | return false; |
| 9313 | /* If the offsets don't match, we can't trust our alignment; |
| 9314 | * nor can we be sure that we won't fall out of range. |
| 9315 | */ |
| 9316 | if (rold->off != rcur->off) |
| 9317 | return false; |
| 9318 | /* id relations must be preserved */ |
| 9319 | if (rold->id && !check_ids(rold->id, rcur->id, idmap)) |
| 9320 | return false; |
| 9321 | /* new val must satisfy old val knowledge */ |
| 9322 | return range_within(rold, rcur) && |
| 9323 | tnum_in(rold->var_off, rcur->var_off); |
| 9324 | case PTR_TO_CTX: |
| 9325 | case CONST_PTR_TO_MAP: |
| 9326 | case PTR_TO_PACKET_END: |
| 9327 | case PTR_TO_FLOW_KEYS: |
| 9328 | case PTR_TO_SOCKET: |
| 9329 | case PTR_TO_SOCKET_OR_NULL: |
| 9330 | case PTR_TO_SOCK_COMMON: |
| 9331 | case PTR_TO_SOCK_COMMON_OR_NULL: |
| 9332 | case PTR_TO_TCP_SOCK: |
| 9333 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 9334 | case PTR_TO_XDP_SOCK: |
| 9335 | /* Only valid matches are exact, which memcmp() above |
| 9336 | * would have accepted |
| 9337 | */ |
| 9338 | default: |
| 9339 | /* Don't know what's going on, just say it's not safe */ |
| 9340 | return false; |
| 9341 | } |
| 9342 | |
| 9343 | /* Shouldn't get here; if we do, say it's not safe */ |
| 9344 | WARN_ON_ONCE(1); |
| 9345 | return false; |
| 9346 | } |
| 9347 | |
| 9348 | static bool stacksafe(struct bpf_func_state *old, |
| 9349 | struct bpf_func_state *cur, |
| 9350 | struct idpair *idmap) |
| 9351 | { |
| 9352 | int i, spi; |
| 9353 | |
| 9354 | /* walk slots of the explored stack and ignore any additional |
| 9355 | * slots in the current stack, since explored(safe) state |
| 9356 | * didn't use them |
| 9357 | */ |
| 9358 | for (i = 0; i < old->allocated_stack; i++) { |
| 9359 | spi = i / BPF_REG_SIZE; |
| 9360 | |
| 9361 | if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { |
| 9362 | i += BPF_REG_SIZE - 1; |
| 9363 | /* explored state didn't use this */ |
| 9364 | continue; |
| 9365 | } |
| 9366 | |
| 9367 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) |
| 9368 | continue; |
| 9369 | |
| 9370 | /* explored stack has more populated slots than current stack |
| 9371 | * and these slots were used |
| 9372 | */ |
| 9373 | if (i >= cur->allocated_stack) |
| 9374 | return false; |
| 9375 | |
| 9376 | /* if old state was safe with misc data in the stack |
| 9377 | * it will be safe with zero-initialized stack. |
| 9378 | * The opposite is not true |
| 9379 | */ |
| 9380 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && |
| 9381 | cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) |
| 9382 | continue; |
| 9383 | if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != |
| 9384 | cur->stack[spi].slot_type[i % BPF_REG_SIZE]) |
| 9385 | /* Ex: old explored (safe) state has STACK_SPILL in |
| 9386 | * this stack slot, but current has STACK_MISC -> |
| 9387 | * this verifier states are not equivalent, |
| 9388 | * return false to continue verification of this path |
| 9389 | */ |
| 9390 | return false; |
| 9391 | if (i % BPF_REG_SIZE) |
| 9392 | continue; |
| 9393 | if (old->stack[spi].slot_type[0] != STACK_SPILL) |
| 9394 | continue; |
| 9395 | if (!regsafe(&old->stack[spi].spilled_ptr, |
| 9396 | &cur->stack[spi].spilled_ptr, |
| 9397 | idmap)) |
| 9398 | /* when explored and current stack slot are both storing |
| 9399 | * spilled registers, check that stored pointers types |
| 9400 | * are the same as well. |
| 9401 | * Ex: explored safe path could have stored |
| 9402 | * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} |
| 9403 | * but current path has stored: |
| 9404 | * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} |
| 9405 | * such verifier states are not equivalent. |
| 9406 | * return false to continue verification of this path |
| 9407 | */ |
| 9408 | return false; |
| 9409 | } |
| 9410 | return true; |
| 9411 | } |
| 9412 | |
| 9413 | static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) |
| 9414 | { |
| 9415 | if (old->acquired_refs != cur->acquired_refs) |
| 9416 | return false; |
| 9417 | return !memcmp(old->refs, cur->refs, |
| 9418 | sizeof(*old->refs) * old->acquired_refs); |
| 9419 | } |
| 9420 | |
| 9421 | /* compare two verifier states |
| 9422 | * |
| 9423 | * all states stored in state_list are known to be valid, since |
| 9424 | * verifier reached 'bpf_exit' instruction through them |
| 9425 | * |
| 9426 | * this function is called when verifier exploring different branches of |
| 9427 | * execution popped from the state stack. If it sees an old state that has |
| 9428 | * more strict register state and more strict stack state then this execution |
| 9429 | * branch doesn't need to be explored further, since verifier already |
| 9430 | * concluded that more strict state leads to valid finish. |
| 9431 | * |
| 9432 | * Therefore two states are equivalent if register state is more conservative |
| 9433 | * and explored stack state is more conservative than the current one. |
| 9434 | * Example: |
| 9435 | * explored current |
| 9436 | * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) |
| 9437 | * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) |
| 9438 | * |
| 9439 | * In other words if current stack state (one being explored) has more |
| 9440 | * valid slots than old one that already passed validation, it means |
| 9441 | * the verifier can stop exploring and conclude that current state is valid too |
| 9442 | * |
| 9443 | * Similarly with registers. If explored state has register type as invalid |
| 9444 | * whereas register type in current state is meaningful, it means that |
| 9445 | * the current state will reach 'bpf_exit' instruction safely |
| 9446 | */ |
| 9447 | static bool func_states_equal(struct bpf_func_state *old, |
| 9448 | struct bpf_func_state *cur) |
| 9449 | { |
| 9450 | struct idpair *idmap; |
| 9451 | bool ret = false; |
| 9452 | int i; |
| 9453 | |
| 9454 | idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL); |
| 9455 | /* If we failed to allocate the idmap, just say it's not safe */ |
| 9456 | if (!idmap) |
| 9457 | return false; |
| 9458 | |
| 9459 | for (i = 0; i < MAX_BPF_REG; i++) { |
| 9460 | if (!regsafe(&old->regs[i], &cur->regs[i], idmap)) |
| 9461 | goto out_free; |
| 9462 | } |
| 9463 | |
| 9464 | if (!stacksafe(old, cur, idmap)) |
| 9465 | goto out_free; |
| 9466 | |
| 9467 | if (!refsafe(old, cur)) |
| 9468 | goto out_free; |
| 9469 | ret = true; |
| 9470 | out_free: |
| 9471 | kfree(idmap); |
| 9472 | return ret; |
| 9473 | } |
| 9474 | |
| 9475 | static bool states_equal(struct bpf_verifier_env *env, |
| 9476 | struct bpf_verifier_state *old, |
| 9477 | struct bpf_verifier_state *cur) |
| 9478 | { |
| 9479 | int i; |
| 9480 | |
| 9481 | if (old->curframe != cur->curframe) |
| 9482 | return false; |
| 9483 | |
| 9484 | /* Verification state from speculative execution simulation |
| 9485 | * must never prune a non-speculative execution one. |
| 9486 | */ |
| 9487 | if (old->speculative && !cur->speculative) |
| 9488 | return false; |
| 9489 | |
| 9490 | if (old->active_spin_lock != cur->active_spin_lock) |
| 9491 | return false; |
| 9492 | |
| 9493 | /* for states to be equal callsites have to be the same |
| 9494 | * and all frame states need to be equivalent |
| 9495 | */ |
| 9496 | for (i = 0; i <= old->curframe; i++) { |
| 9497 | if (old->frame[i]->callsite != cur->frame[i]->callsite) |
| 9498 | return false; |
| 9499 | if (!func_states_equal(old->frame[i], cur->frame[i])) |
| 9500 | return false; |
| 9501 | } |
| 9502 | return true; |
| 9503 | } |
| 9504 | |
| 9505 | /* Return 0 if no propagation happened. Return negative error code if error |
| 9506 | * happened. Otherwise, return the propagated bit. |
| 9507 | */ |
| 9508 | static int propagate_liveness_reg(struct bpf_verifier_env *env, |
| 9509 | struct bpf_reg_state *reg, |
| 9510 | struct bpf_reg_state *parent_reg) |
| 9511 | { |
| 9512 | u8 parent_flag = parent_reg->live & REG_LIVE_READ; |
| 9513 | u8 flag = reg->live & REG_LIVE_READ; |
| 9514 | int err; |
| 9515 | |
| 9516 | /* When comes here, read flags of PARENT_REG or REG could be any of |
| 9517 | * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need |
| 9518 | * of propagation if PARENT_REG has strongest REG_LIVE_READ64. |
| 9519 | */ |
| 9520 | if (parent_flag == REG_LIVE_READ64 || |
| 9521 | /* Or if there is no read flag from REG. */ |
| 9522 | !flag || |
| 9523 | /* Or if the read flag from REG is the same as PARENT_REG. */ |
| 9524 | parent_flag == flag) |
| 9525 | return 0; |
| 9526 | |
| 9527 | err = mark_reg_read(env, reg, parent_reg, flag); |
| 9528 | if (err) |
| 9529 | return err; |
| 9530 | |
| 9531 | return flag; |
| 9532 | } |
| 9533 | |
| 9534 | /* A write screens off any subsequent reads; but write marks come from the |
| 9535 | * straight-line code between a state and its parent. When we arrive at an |
| 9536 | * equivalent state (jump target or such) we didn't arrive by the straight-line |
| 9537 | * code, so read marks in the state must propagate to the parent regardless |
| 9538 | * of the state's write marks. That's what 'parent == state->parent' comparison |
| 9539 | * in mark_reg_read() is for. |
| 9540 | */ |
| 9541 | static int propagate_liveness(struct bpf_verifier_env *env, |
| 9542 | const struct bpf_verifier_state *vstate, |
| 9543 | struct bpf_verifier_state *vparent) |
| 9544 | { |
| 9545 | struct bpf_reg_state *state_reg, *parent_reg; |
| 9546 | struct bpf_func_state *state, *parent; |
| 9547 | int i, frame, err = 0; |
| 9548 | |
| 9549 | if (vparent->curframe != vstate->curframe) { |
| 9550 | WARN(1, "propagate_live: parent frame %d current frame %d\n", |
| 9551 | vparent->curframe, vstate->curframe); |
| 9552 | return -EFAULT; |
| 9553 | } |
| 9554 | /* Propagate read liveness of registers... */ |
| 9555 | BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); |
| 9556 | for (frame = 0; frame <= vstate->curframe; frame++) { |
| 9557 | parent = vparent->frame[frame]; |
| 9558 | state = vstate->frame[frame]; |
| 9559 | parent_reg = parent->regs; |
| 9560 | state_reg = state->regs; |
| 9561 | /* We don't need to worry about FP liveness, it's read-only */ |
| 9562 | for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { |
| 9563 | err = propagate_liveness_reg(env, &state_reg[i], |
| 9564 | &parent_reg[i]); |
| 9565 | if (err < 0) |
| 9566 | return err; |
| 9567 | if (err == REG_LIVE_READ64) |
| 9568 | mark_insn_zext(env, &parent_reg[i]); |
| 9569 | } |
| 9570 | |
| 9571 | /* Propagate stack slots. */ |
| 9572 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && |
| 9573 | i < parent->allocated_stack / BPF_REG_SIZE; i++) { |
| 9574 | parent_reg = &parent->stack[i].spilled_ptr; |
| 9575 | state_reg = &state->stack[i].spilled_ptr; |
| 9576 | err = propagate_liveness_reg(env, state_reg, |
| 9577 | parent_reg); |
| 9578 | if (err < 0) |
| 9579 | return err; |
| 9580 | } |
| 9581 | } |
| 9582 | return 0; |
| 9583 | } |
| 9584 | |
| 9585 | /* find precise scalars in the previous equivalent state and |
| 9586 | * propagate them into the current state |
| 9587 | */ |
| 9588 | static int propagate_precision(struct bpf_verifier_env *env, |
| 9589 | const struct bpf_verifier_state *old) |
| 9590 | { |
| 9591 | struct bpf_reg_state *state_reg; |
| 9592 | struct bpf_func_state *state; |
| 9593 | int i, err = 0; |
| 9594 | |
| 9595 | state = old->frame[old->curframe]; |
| 9596 | state_reg = state->regs; |
| 9597 | for (i = 0; i < BPF_REG_FP; i++, state_reg++) { |
| 9598 | if (state_reg->type != SCALAR_VALUE || |
| 9599 | !state_reg->precise) |
| 9600 | continue; |
| 9601 | if (env->log.level & BPF_LOG_LEVEL2) |
| 9602 | verbose(env, "propagating r%d\n", i); |
| 9603 | err = mark_chain_precision(env, i); |
| 9604 | if (err < 0) |
| 9605 | return err; |
| 9606 | } |
| 9607 | |
| 9608 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { |
| 9609 | if (state->stack[i].slot_type[0] != STACK_SPILL) |
| 9610 | continue; |
| 9611 | state_reg = &state->stack[i].spilled_ptr; |
| 9612 | if (state_reg->type != SCALAR_VALUE || |
| 9613 | !state_reg->precise) |
| 9614 | continue; |
| 9615 | if (env->log.level & BPF_LOG_LEVEL2) |
| 9616 | verbose(env, "propagating fp%d\n", |
| 9617 | (-i - 1) * BPF_REG_SIZE); |
| 9618 | err = mark_chain_precision_stack(env, i); |
| 9619 | if (err < 0) |
| 9620 | return err; |
| 9621 | } |
| 9622 | return 0; |
| 9623 | } |
| 9624 | |
| 9625 | static bool states_maybe_looping(struct bpf_verifier_state *old, |
| 9626 | struct bpf_verifier_state *cur) |
| 9627 | { |
| 9628 | struct bpf_func_state *fold, *fcur; |
| 9629 | int i, fr = cur->curframe; |
| 9630 | |
| 9631 | if (old->curframe != fr) |
| 9632 | return false; |
| 9633 | |
| 9634 | fold = old->frame[fr]; |
| 9635 | fcur = cur->frame[fr]; |
| 9636 | for (i = 0; i < MAX_BPF_REG; i++) |
| 9637 | if (memcmp(&fold->regs[i], &fcur->regs[i], |
| 9638 | offsetof(struct bpf_reg_state, parent))) |
| 9639 | return false; |
| 9640 | return true; |
| 9641 | } |
| 9642 | |
| 9643 | |
| 9644 | static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) |
| 9645 | { |
| 9646 | struct bpf_verifier_state_list *new_sl; |
| 9647 | struct bpf_verifier_state_list *sl, **pprev; |
| 9648 | struct bpf_verifier_state *cur = env->cur_state, *new; |
| 9649 | int i, j, err, states_cnt = 0; |
| 9650 | bool add_new_state = env->test_state_freq ? true : false; |
| 9651 | |
| 9652 | cur->last_insn_idx = env->prev_insn_idx; |
| 9653 | if (!env->insn_aux_data[insn_idx].prune_point) |
| 9654 | /* this 'insn_idx' instruction wasn't marked, so we will not |
| 9655 | * be doing state search here |
| 9656 | */ |
| 9657 | return 0; |
| 9658 | |
| 9659 | /* bpf progs typically have pruning point every 4 instructions |
| 9660 | * http://vger.kernel.org/bpfconf2019.html#session-1 |
| 9661 | * Do not add new state for future pruning if the verifier hasn't seen |
| 9662 | * at least 2 jumps and at least 8 instructions. |
| 9663 | * This heuristics helps decrease 'total_states' and 'peak_states' metric. |
| 9664 | * In tests that amounts to up to 50% reduction into total verifier |
| 9665 | * memory consumption and 20% verifier time speedup. |
| 9666 | */ |
| 9667 | if (env->jmps_processed - env->prev_jmps_processed >= 2 && |
| 9668 | env->insn_processed - env->prev_insn_processed >= 8) |
| 9669 | add_new_state = true; |
| 9670 | |
| 9671 | pprev = explored_state(env, insn_idx); |
| 9672 | sl = *pprev; |
| 9673 | |
| 9674 | clean_live_states(env, insn_idx, cur); |
| 9675 | |
| 9676 | while (sl) { |
| 9677 | states_cnt++; |
| 9678 | if (sl->state.insn_idx != insn_idx) |
| 9679 | goto next; |
| 9680 | if (sl->state.branches) { |
| 9681 | if (states_maybe_looping(&sl->state, cur) && |
| 9682 | states_equal(env, &sl->state, cur)) { |
| 9683 | verbose_linfo(env, insn_idx, "; "); |
| 9684 | verbose(env, "infinite loop detected at insn %d\n", insn_idx); |
| 9685 | return -EINVAL; |
| 9686 | } |
| 9687 | /* if the verifier is processing a loop, avoid adding new state |
| 9688 | * too often, since different loop iterations have distinct |
| 9689 | * states and may not help future pruning. |
| 9690 | * This threshold shouldn't be too low to make sure that |
| 9691 | * a loop with large bound will be rejected quickly. |
| 9692 | * The most abusive loop will be: |
| 9693 | * r1 += 1 |
| 9694 | * if r1 < 1000000 goto pc-2 |
| 9695 | * 1M insn_procssed limit / 100 == 10k peak states. |
| 9696 | * This threshold shouldn't be too high either, since states |
| 9697 | * at the end of the loop are likely to be useful in pruning. |
| 9698 | */ |
| 9699 | if (env->jmps_processed - env->prev_jmps_processed < 20 && |
| 9700 | env->insn_processed - env->prev_insn_processed < 100) |
| 9701 | add_new_state = false; |
| 9702 | goto miss; |
| 9703 | } |
| 9704 | if (states_equal(env, &sl->state, cur)) { |
| 9705 | sl->hit_cnt++; |
| 9706 | /* reached equivalent register/stack state, |
| 9707 | * prune the search. |
| 9708 | * Registers read by the continuation are read by us. |
| 9709 | * If we have any write marks in env->cur_state, they |
| 9710 | * will prevent corresponding reads in the continuation |
| 9711 | * from reaching our parent (an explored_state). Our |
| 9712 | * own state will get the read marks recorded, but |
| 9713 | * they'll be immediately forgotten as we're pruning |
| 9714 | * this state and will pop a new one. |
| 9715 | */ |
| 9716 | err = propagate_liveness(env, &sl->state, cur); |
| 9717 | |
| 9718 | /* if previous state reached the exit with precision and |
| 9719 | * current state is equivalent to it (except precsion marks) |
| 9720 | * the precision needs to be propagated back in |
| 9721 | * the current state. |
| 9722 | */ |
| 9723 | err = err ? : push_jmp_history(env, cur); |
| 9724 | err = err ? : propagate_precision(env, &sl->state); |
| 9725 | if (err) |
| 9726 | return err; |
| 9727 | return 1; |
| 9728 | } |
| 9729 | miss: |
| 9730 | /* when new state is not going to be added do not increase miss count. |
| 9731 | * Otherwise several loop iterations will remove the state |
| 9732 | * recorded earlier. The goal of these heuristics is to have |
| 9733 | * states from some iterations of the loop (some in the beginning |
| 9734 | * and some at the end) to help pruning. |
| 9735 | */ |
| 9736 | if (add_new_state) |
| 9737 | sl->miss_cnt++; |
| 9738 | /* heuristic to determine whether this state is beneficial |
| 9739 | * to keep checking from state equivalence point of view. |
| 9740 | * Higher numbers increase max_states_per_insn and verification time, |
| 9741 | * but do not meaningfully decrease insn_processed. |
| 9742 | */ |
| 9743 | if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { |
| 9744 | /* the state is unlikely to be useful. Remove it to |
| 9745 | * speed up verification |
| 9746 | */ |
| 9747 | *pprev = sl->next; |
| 9748 | if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { |
| 9749 | u32 br = sl->state.branches; |
| 9750 | |
| 9751 | WARN_ONCE(br, |
| 9752 | "BUG live_done but branches_to_explore %d\n", |
| 9753 | br); |
| 9754 | free_verifier_state(&sl->state, false); |
| 9755 | kfree(sl); |
| 9756 | env->peak_states--; |
| 9757 | } else { |
| 9758 | /* cannot free this state, since parentage chain may |
| 9759 | * walk it later. Add it for free_list instead to |
| 9760 | * be freed at the end of verification |
| 9761 | */ |
| 9762 | sl->next = env->free_list; |
| 9763 | env->free_list = sl; |
| 9764 | } |
| 9765 | sl = *pprev; |
| 9766 | continue; |
| 9767 | } |
| 9768 | next: |
| 9769 | pprev = &sl->next; |
| 9770 | sl = *pprev; |
| 9771 | } |
| 9772 | |
| 9773 | if (env->max_states_per_insn < states_cnt) |
| 9774 | env->max_states_per_insn = states_cnt; |
| 9775 | |
| 9776 | if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) |
| 9777 | return push_jmp_history(env, cur); |
| 9778 | |
| 9779 | if (!add_new_state) |
| 9780 | return push_jmp_history(env, cur); |
| 9781 | |
| 9782 | /* There were no equivalent states, remember the current one. |
| 9783 | * Technically the current state is not proven to be safe yet, |
| 9784 | * but it will either reach outer most bpf_exit (which means it's safe) |
| 9785 | * or it will be rejected. When there are no loops the verifier won't be |
| 9786 | * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) |
| 9787 | * again on the way to bpf_exit. |
| 9788 | * When looping the sl->state.branches will be > 0 and this state |
| 9789 | * will not be considered for equivalence until branches == 0. |
| 9790 | */ |
| 9791 | new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); |
| 9792 | if (!new_sl) |
| 9793 | return -ENOMEM; |
| 9794 | env->total_states++; |
| 9795 | env->peak_states++; |
| 9796 | env->prev_jmps_processed = env->jmps_processed; |
| 9797 | env->prev_insn_processed = env->insn_processed; |
| 9798 | |
| 9799 | /* add new state to the head of linked list */ |
| 9800 | new = &new_sl->state; |
| 9801 | err = copy_verifier_state(new, cur); |
| 9802 | if (err) { |
| 9803 | free_verifier_state(new, false); |
| 9804 | kfree(new_sl); |
| 9805 | return err; |
| 9806 | } |
| 9807 | new->insn_idx = insn_idx; |
| 9808 | WARN_ONCE(new->branches != 1, |
| 9809 | "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx); |
| 9810 | |
| 9811 | cur->parent = new; |
| 9812 | cur->first_insn_idx = insn_idx; |
| 9813 | clear_jmp_history(cur); |
| 9814 | new_sl->next = *explored_state(env, insn_idx); |
| 9815 | *explored_state(env, insn_idx) = new_sl; |
| 9816 | /* connect new state to parentage chain. Current frame needs all |
| 9817 | * registers connected. Only r6 - r9 of the callers are alive (pushed |
| 9818 | * to the stack implicitly by JITs) so in callers' frames connect just |
| 9819 | * r6 - r9 as an optimization. Callers will have r1 - r5 connected to |
| 9820 | * the state of the call instruction (with WRITTEN set), and r0 comes |
| 9821 | * from callee with its full parentage chain, anyway. |
| 9822 | */ |
| 9823 | /* clear write marks in current state: the writes we did are not writes |
| 9824 | * our child did, so they don't screen off its reads from us. |
| 9825 | * (There are no read marks in current state, because reads always mark |
| 9826 | * their parent and current state never has children yet. Only |
| 9827 | * explored_states can get read marks.) |
| 9828 | */ |
| 9829 | for (j = 0; j <= cur->curframe; j++) { |
| 9830 | for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) |
| 9831 | cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; |
| 9832 | for (i = 0; i < BPF_REG_FP; i++) |
| 9833 | cur->frame[j]->regs[i].live = REG_LIVE_NONE; |
| 9834 | } |
| 9835 | |
| 9836 | /* all stack frames are accessible from callee, clear them all */ |
| 9837 | for (j = 0; j <= cur->curframe; j++) { |
| 9838 | struct bpf_func_state *frame = cur->frame[j]; |
| 9839 | struct bpf_func_state *newframe = new->frame[j]; |
| 9840 | |
| 9841 | for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { |
| 9842 | frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; |
| 9843 | frame->stack[i].spilled_ptr.parent = |
| 9844 | &newframe->stack[i].spilled_ptr; |
| 9845 | } |
| 9846 | } |
| 9847 | return 0; |
| 9848 | } |
| 9849 | |
| 9850 | /* Return true if it's OK to have the same insn return a different type. */ |
| 9851 | static bool reg_type_mismatch_ok(enum bpf_reg_type type) |
| 9852 | { |
| 9853 | switch (type) { |
| 9854 | case PTR_TO_CTX: |
| 9855 | case PTR_TO_SOCKET: |
| 9856 | case PTR_TO_SOCKET_OR_NULL: |
| 9857 | case PTR_TO_SOCK_COMMON: |
| 9858 | case PTR_TO_SOCK_COMMON_OR_NULL: |
| 9859 | case PTR_TO_TCP_SOCK: |
| 9860 | case PTR_TO_TCP_SOCK_OR_NULL: |
| 9861 | case PTR_TO_XDP_SOCK: |
| 9862 | case PTR_TO_BTF_ID: |
| 9863 | case PTR_TO_BTF_ID_OR_NULL: |
| 9864 | return false; |
| 9865 | default: |
| 9866 | return true; |
| 9867 | } |
| 9868 | } |
| 9869 | |
| 9870 | /* If an instruction was previously used with particular pointer types, then we |
| 9871 | * need to be careful to avoid cases such as the below, where it may be ok |
| 9872 | * for one branch accessing the pointer, but not ok for the other branch: |
| 9873 | * |
| 9874 | * R1 = sock_ptr |
| 9875 | * goto X; |
| 9876 | * ... |
| 9877 | * R1 = some_other_valid_ptr; |
| 9878 | * goto X; |
| 9879 | * ... |
| 9880 | * R2 = *(u32 *)(R1 + 0); |
| 9881 | */ |
| 9882 | static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) |
| 9883 | { |
| 9884 | return src != prev && (!reg_type_mismatch_ok(src) || |
| 9885 | !reg_type_mismatch_ok(prev)); |
| 9886 | } |
| 9887 | |
| 9888 | static int do_check(struct bpf_verifier_env *env) |
| 9889 | { |
| 9890 | bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); |
| 9891 | struct bpf_verifier_state *state = env->cur_state; |
| 9892 | struct bpf_insn *insns = env->prog->insnsi; |
| 9893 | struct bpf_reg_state *regs; |
| 9894 | int insn_cnt = env->prog->len; |
| 9895 | bool do_print_state = false; |
| 9896 | int prev_insn_idx = -1; |
| 9897 | |
| 9898 | for (;;) { |
| 9899 | struct bpf_insn *insn; |
| 9900 | u8 class; |
| 9901 | int err; |
| 9902 | |
| 9903 | env->prev_insn_idx = prev_insn_idx; |
| 9904 | if (env->insn_idx >= insn_cnt) { |
| 9905 | verbose(env, "invalid insn idx %d insn_cnt %d\n", |
| 9906 | env->insn_idx, insn_cnt); |
| 9907 | return -EFAULT; |
| 9908 | } |
| 9909 | |
| 9910 | insn = &insns[env->insn_idx]; |
| 9911 | class = BPF_CLASS(insn->code); |
| 9912 | |
| 9913 | if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { |
| 9914 | verbose(env, |
| 9915 | "BPF program is too large. Processed %d insn\n", |
| 9916 | env->insn_processed); |
| 9917 | return -E2BIG; |
| 9918 | } |
| 9919 | |
| 9920 | err = is_state_visited(env, env->insn_idx); |
| 9921 | if (err < 0) |
| 9922 | return err; |
| 9923 | if (err == 1) { |
| 9924 | /* found equivalent state, can prune the search */ |
| 9925 | if (env->log.level & BPF_LOG_LEVEL) { |
| 9926 | if (do_print_state) |
| 9927 | verbose(env, "\nfrom %d to %d%s: safe\n", |
| 9928 | env->prev_insn_idx, env->insn_idx, |
| 9929 | env->cur_state->speculative ? |
| 9930 | " (speculative execution)" : ""); |
| 9931 | else |
| 9932 | verbose(env, "%d: safe\n", env->insn_idx); |
| 9933 | } |
| 9934 | goto process_bpf_exit; |
| 9935 | } |
| 9936 | |
| 9937 | if (signal_pending(current)) |
| 9938 | return -EAGAIN; |
| 9939 | |
| 9940 | if (need_resched()) |
| 9941 | cond_resched(); |
| 9942 | |
| 9943 | if (env->log.level & BPF_LOG_LEVEL2 || |
| 9944 | (env->log.level & BPF_LOG_LEVEL && do_print_state)) { |
| 9945 | if (env->log.level & BPF_LOG_LEVEL2) |
| 9946 | verbose(env, "%d:", env->insn_idx); |
| 9947 | else |
| 9948 | verbose(env, "\nfrom %d to %d%s:", |
| 9949 | env->prev_insn_idx, env->insn_idx, |
| 9950 | env->cur_state->speculative ? |
| 9951 | " (speculative execution)" : ""); |
| 9952 | print_verifier_state(env, state->frame[state->curframe]); |
| 9953 | do_print_state = false; |
| 9954 | } |
| 9955 | |
| 9956 | if (env->log.level & BPF_LOG_LEVEL) { |
| 9957 | const struct bpf_insn_cbs cbs = { |
| 9958 | .cb_print = verbose, |
| 9959 | .private_data = env, |
| 9960 | }; |
| 9961 | |
| 9962 | verbose_linfo(env, env->insn_idx, "; "); |
| 9963 | verbose(env, "%d: ", env->insn_idx); |
| 9964 | print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); |
| 9965 | } |
| 9966 | |
| 9967 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
| 9968 | err = bpf_prog_offload_verify_insn(env, env->insn_idx, |
| 9969 | env->prev_insn_idx); |
| 9970 | if (err) |
| 9971 | return err; |
| 9972 | } |
| 9973 | |
| 9974 | regs = cur_regs(env); |
| 9975 | env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; |
| 9976 | prev_insn_idx = env->insn_idx; |
| 9977 | |
| 9978 | if (class == BPF_ALU || class == BPF_ALU64) { |
| 9979 | err = check_alu_op(env, insn); |
| 9980 | if (err) |
| 9981 | return err; |
| 9982 | |
| 9983 | } else if (class == BPF_LDX) { |
| 9984 | enum bpf_reg_type *prev_src_type, src_reg_type; |
| 9985 | |
| 9986 | /* check for reserved fields is already done */ |
| 9987 | |
| 9988 | /* check src operand */ |
| 9989 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
| 9990 | if (err) |
| 9991 | return err; |
| 9992 | |
| 9993 | err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); |
| 9994 | if (err) |
| 9995 | return err; |
| 9996 | |
| 9997 | src_reg_type = regs[insn->src_reg].type; |
| 9998 | |
| 9999 | /* check that memory (src_reg + off) is readable, |
| 10000 | * the state of dst_reg will be updated by this func |
| 10001 | */ |
| 10002 | err = check_mem_access(env, env->insn_idx, insn->src_reg, |
| 10003 | insn->off, BPF_SIZE(insn->code), |
| 10004 | BPF_READ, insn->dst_reg, false); |
| 10005 | if (err) |
| 10006 | return err; |
| 10007 | |
| 10008 | prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; |
| 10009 | |
| 10010 | if (*prev_src_type == NOT_INIT) { |
| 10011 | /* saw a valid insn |
| 10012 | * dst_reg = *(u32 *)(src_reg + off) |
| 10013 | * save type to validate intersecting paths |
| 10014 | */ |
| 10015 | *prev_src_type = src_reg_type; |
| 10016 | |
| 10017 | } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { |
| 10018 | /* ABuser program is trying to use the same insn |
| 10019 | * dst_reg = *(u32*) (src_reg + off) |
| 10020 | * with different pointer types: |
| 10021 | * src_reg == ctx in one branch and |
| 10022 | * src_reg == stack|map in some other branch. |
| 10023 | * Reject it. |
| 10024 | */ |
| 10025 | verbose(env, "same insn cannot be used with different pointers\n"); |
| 10026 | return -EINVAL; |
| 10027 | } |
| 10028 | |
| 10029 | } else if (class == BPF_STX) { |
| 10030 | enum bpf_reg_type *prev_dst_type, dst_reg_type; |
| 10031 | |
| 10032 | if (BPF_MODE(insn->code) == BPF_ATOMIC) { |
| 10033 | err = check_atomic(env, env->insn_idx, insn); |
| 10034 | if (err) |
| 10035 | return err; |
| 10036 | env->insn_idx++; |
| 10037 | continue; |
| 10038 | } |
| 10039 | |
| 10040 | if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) { |
| 10041 | verbose(env, "BPF_STX uses reserved fields\n"); |
| 10042 | return -EINVAL; |
| 10043 | } |
| 10044 | |
| 10045 | /* check src1 operand */ |
| 10046 | err = check_reg_arg(env, insn->src_reg, SRC_OP); |
| 10047 | if (err) |
| 10048 | return err; |
| 10049 | /* check src2 operand */ |
| 10050 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
| 10051 | if (err) |
| 10052 | return err; |
| 10053 | |
| 10054 | dst_reg_type = regs[insn->dst_reg].type; |
| 10055 | |
| 10056 | /* check that memory (dst_reg + off) is writeable */ |
| 10057 | err = check_mem_access(env, env->insn_idx, insn->dst_reg, |
| 10058 | insn->off, BPF_SIZE(insn->code), |
| 10059 | BPF_WRITE, insn->src_reg, false); |
| 10060 | if (err) |
| 10061 | return err; |
| 10062 | |
| 10063 | prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; |
| 10064 | |
| 10065 | if (*prev_dst_type == NOT_INIT) { |
| 10066 | *prev_dst_type = dst_reg_type; |
| 10067 | } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { |
| 10068 | verbose(env, "same insn cannot be used with different pointers\n"); |
| 10069 | return -EINVAL; |
| 10070 | } |
| 10071 | |
| 10072 | } else if (class == BPF_ST) { |
| 10073 | if (BPF_MODE(insn->code) != BPF_MEM || |
| 10074 | insn->src_reg != BPF_REG_0) { |
| 10075 | verbose(env, "BPF_ST uses reserved fields\n"); |
| 10076 | return -EINVAL; |
| 10077 | } |
| 10078 | /* check src operand */ |
| 10079 | err = check_reg_arg(env, insn->dst_reg, SRC_OP); |
| 10080 | if (err) |
| 10081 | return err; |
| 10082 | |
| 10083 | if (is_ctx_reg(env, insn->dst_reg)) { |
| 10084 | verbose(env, "BPF_ST stores into R%d %s is not allowed\n", |
| 10085 | insn->dst_reg, |
| 10086 | reg_type_str[reg_state(env, insn->dst_reg)->type]); |
| 10087 | return -EACCES; |
| 10088 | } |
| 10089 | |
| 10090 | /* check that memory (dst_reg + off) is writeable */ |
| 10091 | err = check_mem_access(env, env->insn_idx, insn->dst_reg, |
| 10092 | insn->off, BPF_SIZE(insn->code), |
| 10093 | BPF_WRITE, -1, false); |
| 10094 | if (err) |
| 10095 | return err; |
| 10096 | |
| 10097 | } else if (class == BPF_JMP || class == BPF_JMP32) { |
| 10098 | u8 opcode = BPF_OP(insn->code); |
| 10099 | |
| 10100 | env->jmps_processed++; |
| 10101 | if (opcode == BPF_CALL) { |
| 10102 | if (BPF_SRC(insn->code) != BPF_K || |
| 10103 | insn->off != 0 || |
| 10104 | (insn->src_reg != BPF_REG_0 && |
| 10105 | insn->src_reg != BPF_PSEUDO_CALL) || |
| 10106 | insn->dst_reg != BPF_REG_0 || |
| 10107 | class == BPF_JMP32) { |
| 10108 | verbose(env, "BPF_CALL uses reserved fields\n"); |
| 10109 | return -EINVAL; |
| 10110 | } |
| 10111 | |
| 10112 | if (env->cur_state->active_spin_lock && |
| 10113 | (insn->src_reg == BPF_PSEUDO_CALL || |
| 10114 | insn->imm != BPF_FUNC_spin_unlock)) { |
| 10115 | verbose(env, "function calls are not allowed while holding a lock\n"); |
| 10116 | return -EINVAL; |
| 10117 | } |
| 10118 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 10119 | err = check_func_call(env, insn, &env->insn_idx); |
| 10120 | else |
| 10121 | err = check_helper_call(env, insn->imm, env->insn_idx); |
| 10122 | if (err) |
| 10123 | return err; |
| 10124 | |
| 10125 | } else if (opcode == BPF_JA) { |
| 10126 | if (BPF_SRC(insn->code) != BPF_K || |
| 10127 | insn->imm != 0 || |
| 10128 | insn->src_reg != BPF_REG_0 || |
| 10129 | insn->dst_reg != BPF_REG_0 || |
| 10130 | class == BPF_JMP32) { |
| 10131 | verbose(env, "BPF_JA uses reserved fields\n"); |
| 10132 | return -EINVAL; |
| 10133 | } |
| 10134 | |
| 10135 | env->insn_idx += insn->off + 1; |
| 10136 | continue; |
| 10137 | |
| 10138 | } else if (opcode == BPF_EXIT) { |
| 10139 | if (BPF_SRC(insn->code) != BPF_K || |
| 10140 | insn->imm != 0 || |
| 10141 | insn->src_reg != BPF_REG_0 || |
| 10142 | insn->dst_reg != BPF_REG_0 || |
| 10143 | class == BPF_JMP32) { |
| 10144 | verbose(env, "BPF_EXIT uses reserved fields\n"); |
| 10145 | return -EINVAL; |
| 10146 | } |
| 10147 | |
| 10148 | if (env->cur_state->active_spin_lock) { |
| 10149 | verbose(env, "bpf_spin_unlock is missing\n"); |
| 10150 | return -EINVAL; |
| 10151 | } |
| 10152 | |
| 10153 | if (state->curframe) { |
| 10154 | /* exit from nested function */ |
| 10155 | err = prepare_func_exit(env, &env->insn_idx); |
| 10156 | if (err) |
| 10157 | return err; |
| 10158 | do_print_state = true; |
| 10159 | continue; |
| 10160 | } |
| 10161 | |
| 10162 | err = check_reference_leak(env); |
| 10163 | if (err) |
| 10164 | return err; |
| 10165 | |
| 10166 | err = check_return_code(env); |
| 10167 | if (err) |
| 10168 | return err; |
| 10169 | process_bpf_exit: |
| 10170 | update_branch_counts(env, env->cur_state); |
| 10171 | err = pop_stack(env, &prev_insn_idx, |
| 10172 | &env->insn_idx, pop_log); |
| 10173 | if (err < 0) { |
| 10174 | if (err != -ENOENT) |
| 10175 | return err; |
| 10176 | break; |
| 10177 | } else { |
| 10178 | do_print_state = true; |
| 10179 | continue; |
| 10180 | } |
| 10181 | } else { |
| 10182 | err = check_cond_jmp_op(env, insn, &env->insn_idx); |
| 10183 | if (err) |
| 10184 | return err; |
| 10185 | } |
| 10186 | } else if (class == BPF_LD) { |
| 10187 | u8 mode = BPF_MODE(insn->code); |
| 10188 | |
| 10189 | if (mode == BPF_ABS || mode == BPF_IND) { |
| 10190 | err = check_ld_abs(env, insn); |
| 10191 | if (err) |
| 10192 | return err; |
| 10193 | |
| 10194 | } else if (mode == BPF_IMM) { |
| 10195 | err = check_ld_imm(env, insn); |
| 10196 | if (err) |
| 10197 | return err; |
| 10198 | |
| 10199 | env->insn_idx++; |
| 10200 | env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; |
| 10201 | } else { |
| 10202 | verbose(env, "invalid BPF_LD mode\n"); |
| 10203 | return -EINVAL; |
| 10204 | } |
| 10205 | } else { |
| 10206 | verbose(env, "unknown insn class %d\n", class); |
| 10207 | return -EINVAL; |
| 10208 | } |
| 10209 | |
| 10210 | env->insn_idx++; |
| 10211 | } |
| 10212 | |
| 10213 | return 0; |
| 10214 | } |
| 10215 | |
| 10216 | static int find_btf_percpu_datasec(struct btf *btf) |
| 10217 | { |
| 10218 | const struct btf_type *t; |
| 10219 | const char *tname; |
| 10220 | int i, n; |
| 10221 | |
| 10222 | /* |
| 10223 | * Both vmlinux and module each have their own ".data..percpu" |
| 10224 | * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF |
| 10225 | * types to look at only module's own BTF types. |
| 10226 | */ |
| 10227 | n = btf_nr_types(btf); |
| 10228 | if (btf_is_module(btf)) |
| 10229 | i = btf_nr_types(btf_vmlinux); |
| 10230 | else |
| 10231 | i = 1; |
| 10232 | |
| 10233 | for(; i < n; i++) { |
| 10234 | t = btf_type_by_id(btf, i); |
| 10235 | if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) |
| 10236 | continue; |
| 10237 | |
| 10238 | tname = btf_name_by_offset(btf, t->name_off); |
| 10239 | if (!strcmp(tname, ".data..percpu")) |
| 10240 | return i; |
| 10241 | } |
| 10242 | |
| 10243 | return -ENOENT; |
| 10244 | } |
| 10245 | |
| 10246 | /* replace pseudo btf_id with kernel symbol address */ |
| 10247 | static int check_pseudo_btf_id(struct bpf_verifier_env *env, |
| 10248 | struct bpf_insn *insn, |
| 10249 | struct bpf_insn_aux_data *aux) |
| 10250 | { |
| 10251 | const struct btf_var_secinfo *vsi; |
| 10252 | const struct btf_type *datasec; |
| 10253 | struct btf_mod_pair *btf_mod; |
| 10254 | const struct btf_type *t; |
| 10255 | const char *sym_name; |
| 10256 | bool percpu = false; |
| 10257 | u32 type, id = insn->imm; |
| 10258 | struct btf *btf; |
| 10259 | s32 datasec_id; |
| 10260 | u64 addr; |
| 10261 | int i, btf_fd, err; |
| 10262 | |
| 10263 | btf_fd = insn[1].imm; |
| 10264 | if (btf_fd) { |
| 10265 | btf = btf_get_by_fd(btf_fd); |
| 10266 | if (IS_ERR(btf)) { |
| 10267 | verbose(env, "invalid module BTF object FD specified.\n"); |
| 10268 | return -EINVAL; |
| 10269 | } |
| 10270 | } else { |
| 10271 | if (!btf_vmlinux) { |
| 10272 | verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n"); |
| 10273 | return -EINVAL; |
| 10274 | } |
| 10275 | btf = btf_vmlinux; |
| 10276 | btf_get(btf); |
| 10277 | } |
| 10278 | |
| 10279 | t = btf_type_by_id(btf, id); |
| 10280 | if (!t) { |
| 10281 | verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id); |
| 10282 | err = -ENOENT; |
| 10283 | goto err_put; |
| 10284 | } |
| 10285 | |
| 10286 | if (!btf_type_is_var(t)) { |
| 10287 | verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id); |
| 10288 | err = -EINVAL; |
| 10289 | goto err_put; |
| 10290 | } |
| 10291 | |
| 10292 | sym_name = btf_name_by_offset(btf, t->name_off); |
| 10293 | addr = kallsyms_lookup_name(sym_name); |
| 10294 | if (!addr) { |
| 10295 | verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", |
| 10296 | sym_name); |
| 10297 | err = -ENOENT; |
| 10298 | goto err_put; |
| 10299 | } |
| 10300 | |
| 10301 | datasec_id = find_btf_percpu_datasec(btf); |
| 10302 | if (datasec_id > 0) { |
| 10303 | datasec = btf_type_by_id(btf, datasec_id); |
| 10304 | for_each_vsi(i, datasec, vsi) { |
| 10305 | if (vsi->type == id) { |
| 10306 | percpu = true; |
| 10307 | break; |
| 10308 | } |
| 10309 | } |
| 10310 | } |
| 10311 | |
| 10312 | insn[0].imm = (u32)addr; |
| 10313 | insn[1].imm = addr >> 32; |
| 10314 | |
| 10315 | type = t->type; |
| 10316 | t = btf_type_skip_modifiers(btf, type, NULL); |
| 10317 | if (percpu) { |
| 10318 | aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID; |
| 10319 | aux->btf_var.btf = btf; |
| 10320 | aux->btf_var.btf_id = type; |
| 10321 | } else if (!btf_type_is_struct(t)) { |
| 10322 | const struct btf_type *ret; |
| 10323 | const char *tname; |
| 10324 | u32 tsize; |
| 10325 | |
| 10326 | /* resolve the type size of ksym. */ |
| 10327 | ret = btf_resolve_size(btf, t, &tsize); |
| 10328 | if (IS_ERR(ret)) { |
| 10329 | tname = btf_name_by_offset(btf, t->name_off); |
| 10330 | verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", |
| 10331 | tname, PTR_ERR(ret)); |
| 10332 | err = -EINVAL; |
| 10333 | goto err_put; |
| 10334 | } |
| 10335 | aux->btf_var.reg_type = PTR_TO_MEM; |
| 10336 | aux->btf_var.mem_size = tsize; |
| 10337 | } else { |
| 10338 | aux->btf_var.reg_type = PTR_TO_BTF_ID; |
| 10339 | aux->btf_var.btf = btf; |
| 10340 | aux->btf_var.btf_id = type; |
| 10341 | } |
| 10342 | |
| 10343 | /* check whether we recorded this BTF (and maybe module) already */ |
| 10344 | for (i = 0; i < env->used_btf_cnt; i++) { |
| 10345 | if (env->used_btfs[i].btf == btf) { |
| 10346 | btf_put(btf); |
| 10347 | return 0; |
| 10348 | } |
| 10349 | } |
| 10350 | |
| 10351 | if (env->used_btf_cnt >= MAX_USED_BTFS) { |
| 10352 | err = -E2BIG; |
| 10353 | goto err_put; |
| 10354 | } |
| 10355 | |
| 10356 | btf_mod = &env->used_btfs[env->used_btf_cnt]; |
| 10357 | btf_mod->btf = btf; |
| 10358 | btf_mod->module = NULL; |
| 10359 | |
| 10360 | /* if we reference variables from kernel module, bump its refcount */ |
| 10361 | if (btf_is_module(btf)) { |
| 10362 | btf_mod->module = btf_try_get_module(btf); |
| 10363 | if (!btf_mod->module) { |
| 10364 | err = -ENXIO; |
| 10365 | goto err_put; |
| 10366 | } |
| 10367 | } |
| 10368 | |
| 10369 | env->used_btf_cnt++; |
| 10370 | |
| 10371 | return 0; |
| 10372 | err_put: |
| 10373 | btf_put(btf); |
| 10374 | return err; |
| 10375 | } |
| 10376 | |
| 10377 | static int check_map_prealloc(struct bpf_map *map) |
| 10378 | { |
| 10379 | return (map->map_type != BPF_MAP_TYPE_HASH && |
| 10380 | map->map_type != BPF_MAP_TYPE_PERCPU_HASH && |
| 10381 | map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || |
| 10382 | !(map->map_flags & BPF_F_NO_PREALLOC); |
| 10383 | } |
| 10384 | |
| 10385 | static bool is_tracing_prog_type(enum bpf_prog_type type) |
| 10386 | { |
| 10387 | switch (type) { |
| 10388 | case BPF_PROG_TYPE_KPROBE: |
| 10389 | case BPF_PROG_TYPE_TRACEPOINT: |
| 10390 | case BPF_PROG_TYPE_PERF_EVENT: |
| 10391 | case BPF_PROG_TYPE_RAW_TRACEPOINT: |
| 10392 | return true; |
| 10393 | default: |
| 10394 | return false; |
| 10395 | } |
| 10396 | } |
| 10397 | |
| 10398 | static bool is_preallocated_map(struct bpf_map *map) |
| 10399 | { |
| 10400 | if (!check_map_prealloc(map)) |
| 10401 | return false; |
| 10402 | if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta)) |
| 10403 | return false; |
| 10404 | return true; |
| 10405 | } |
| 10406 | |
| 10407 | static int check_map_prog_compatibility(struct bpf_verifier_env *env, |
| 10408 | struct bpf_map *map, |
| 10409 | struct bpf_prog *prog) |
| 10410 | |
| 10411 | { |
| 10412 | enum bpf_prog_type prog_type = resolve_prog_type(prog); |
| 10413 | /* |
| 10414 | * Validate that trace type programs use preallocated hash maps. |
| 10415 | * |
| 10416 | * For programs attached to PERF events this is mandatory as the |
| 10417 | * perf NMI can hit any arbitrary code sequence. |
| 10418 | * |
| 10419 | * All other trace types using preallocated hash maps are unsafe as |
| 10420 | * well because tracepoint or kprobes can be inside locked regions |
| 10421 | * of the memory allocator or at a place where a recursion into the |
| 10422 | * memory allocator would see inconsistent state. |
| 10423 | * |
| 10424 | * On RT enabled kernels run-time allocation of all trace type |
| 10425 | * programs is strictly prohibited due to lock type constraints. On |
| 10426 | * !RT kernels it is allowed for backwards compatibility reasons for |
| 10427 | * now, but warnings are emitted so developers are made aware of |
| 10428 | * the unsafety and can fix their programs before this is enforced. |
| 10429 | */ |
| 10430 | if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) { |
| 10431 | if (prog_type == BPF_PROG_TYPE_PERF_EVENT) { |
| 10432 | verbose(env, "perf_event programs can only use preallocated hash map\n"); |
| 10433 | return -EINVAL; |
| 10434 | } |
| 10435 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) { |
| 10436 | verbose(env, "trace type programs can only use preallocated hash map\n"); |
| 10437 | return -EINVAL; |
| 10438 | } |
| 10439 | WARN_ONCE(1, "trace type BPF program uses run-time allocation\n"); |
| 10440 | verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n"); |
| 10441 | } |
| 10442 | |
| 10443 | if (map_value_has_spin_lock(map)) { |
| 10444 | if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) { |
| 10445 | verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n"); |
| 10446 | return -EINVAL; |
| 10447 | } |
| 10448 | |
| 10449 | if (is_tracing_prog_type(prog_type)) { |
| 10450 | verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); |
| 10451 | return -EINVAL; |
| 10452 | } |
| 10453 | |
| 10454 | if (prog->aux->sleepable) { |
| 10455 | verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n"); |
| 10456 | return -EINVAL; |
| 10457 | } |
| 10458 | } |
| 10459 | |
| 10460 | if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && |
| 10461 | !bpf_offload_prog_map_match(prog, map)) { |
| 10462 | verbose(env, "offload device mismatch between prog and map\n"); |
| 10463 | return -EINVAL; |
| 10464 | } |
| 10465 | |
| 10466 | if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) { |
| 10467 | verbose(env, "bpf_struct_ops map cannot be used in prog\n"); |
| 10468 | return -EINVAL; |
| 10469 | } |
| 10470 | |
| 10471 | if (prog->aux->sleepable) |
| 10472 | switch (map->map_type) { |
| 10473 | case BPF_MAP_TYPE_HASH: |
| 10474 | case BPF_MAP_TYPE_LRU_HASH: |
| 10475 | case BPF_MAP_TYPE_ARRAY: |
| 10476 | case BPF_MAP_TYPE_PERCPU_HASH: |
| 10477 | case BPF_MAP_TYPE_PERCPU_ARRAY: |
| 10478 | case BPF_MAP_TYPE_LRU_PERCPU_HASH: |
| 10479 | case BPF_MAP_TYPE_ARRAY_OF_MAPS: |
| 10480 | case BPF_MAP_TYPE_HASH_OF_MAPS: |
| 10481 | if (!is_preallocated_map(map)) { |
| 10482 | verbose(env, |
| 10483 | "Sleepable programs can only use preallocated maps\n"); |
| 10484 | return -EINVAL; |
| 10485 | } |
| 10486 | break; |
| 10487 | case BPF_MAP_TYPE_RINGBUF: |
| 10488 | break; |
| 10489 | default: |
| 10490 | verbose(env, |
| 10491 | "Sleepable programs can only use array, hash, and ringbuf maps\n"); |
| 10492 | return -EINVAL; |
| 10493 | } |
| 10494 | |
| 10495 | return 0; |
| 10496 | } |
| 10497 | |
| 10498 | static bool bpf_map_is_cgroup_storage(struct bpf_map *map) |
| 10499 | { |
| 10500 | return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || |
| 10501 | map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); |
| 10502 | } |
| 10503 | |
| 10504 | /* find and rewrite pseudo imm in ld_imm64 instructions: |
| 10505 | * |
| 10506 | * 1. if it accesses map FD, replace it with actual map pointer. |
| 10507 | * 2. if it accesses btf_id of a VAR, replace it with pointer to the var. |
| 10508 | * |
| 10509 | * NOTE: btf_vmlinux is required for converting pseudo btf_id. |
| 10510 | */ |
| 10511 | static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env) |
| 10512 | { |
| 10513 | struct bpf_insn *insn = env->prog->insnsi; |
| 10514 | int insn_cnt = env->prog->len; |
| 10515 | int i, j, err; |
| 10516 | |
| 10517 | err = bpf_prog_calc_tag(env->prog); |
| 10518 | if (err) |
| 10519 | return err; |
| 10520 | |
| 10521 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 10522 | if (BPF_CLASS(insn->code) == BPF_LDX && |
| 10523 | (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { |
| 10524 | verbose(env, "BPF_LDX uses reserved fields\n"); |
| 10525 | return -EINVAL; |
| 10526 | } |
| 10527 | |
| 10528 | if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { |
| 10529 | struct bpf_insn_aux_data *aux; |
| 10530 | struct bpf_map *map; |
| 10531 | struct fd f; |
| 10532 | u64 addr; |
| 10533 | |
| 10534 | if (i == insn_cnt - 1 || insn[1].code != 0 || |
| 10535 | insn[1].dst_reg != 0 || insn[1].src_reg != 0 || |
| 10536 | insn[1].off != 0) { |
| 10537 | verbose(env, "invalid bpf_ld_imm64 insn\n"); |
| 10538 | return -EINVAL; |
| 10539 | } |
| 10540 | |
| 10541 | if (insn[0].src_reg == 0) |
| 10542 | /* valid generic load 64-bit imm */ |
| 10543 | goto next_insn; |
| 10544 | |
| 10545 | if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) { |
| 10546 | aux = &env->insn_aux_data[i]; |
| 10547 | err = check_pseudo_btf_id(env, insn, aux); |
| 10548 | if (err) |
| 10549 | return err; |
| 10550 | goto next_insn; |
| 10551 | } |
| 10552 | |
| 10553 | /* In final convert_pseudo_ld_imm64() step, this is |
| 10554 | * converted into regular 64-bit imm load insn. |
| 10555 | */ |
| 10556 | if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD && |
| 10557 | insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) || |
| 10558 | (insn[0].src_reg == BPF_PSEUDO_MAP_FD && |
| 10559 | insn[1].imm != 0)) { |
| 10560 | verbose(env, |
| 10561 | "unrecognized bpf_ld_imm64 insn\n"); |
| 10562 | return -EINVAL; |
| 10563 | } |
| 10564 | |
| 10565 | f = fdget(insn[0].imm); |
| 10566 | map = __bpf_map_get(f); |
| 10567 | if (IS_ERR(map)) { |
| 10568 | verbose(env, "fd %d is not pointing to valid bpf_map\n", |
| 10569 | insn[0].imm); |
| 10570 | return PTR_ERR(map); |
| 10571 | } |
| 10572 | |
| 10573 | err = check_map_prog_compatibility(env, map, env->prog); |
| 10574 | if (err) { |
| 10575 | fdput(f); |
| 10576 | return err; |
| 10577 | } |
| 10578 | |
| 10579 | aux = &env->insn_aux_data[i]; |
| 10580 | if (insn->src_reg == BPF_PSEUDO_MAP_FD) { |
| 10581 | addr = (unsigned long)map; |
| 10582 | } else { |
| 10583 | u32 off = insn[1].imm; |
| 10584 | |
| 10585 | if (off >= BPF_MAX_VAR_OFF) { |
| 10586 | verbose(env, "direct value offset of %u is not allowed\n", off); |
| 10587 | fdput(f); |
| 10588 | return -EINVAL; |
| 10589 | } |
| 10590 | |
| 10591 | if (!map->ops->map_direct_value_addr) { |
| 10592 | verbose(env, "no direct value access support for this map type\n"); |
| 10593 | fdput(f); |
| 10594 | return -EINVAL; |
| 10595 | } |
| 10596 | |
| 10597 | err = map->ops->map_direct_value_addr(map, &addr, off); |
| 10598 | if (err) { |
| 10599 | verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", |
| 10600 | map->value_size, off); |
| 10601 | fdput(f); |
| 10602 | return err; |
| 10603 | } |
| 10604 | |
| 10605 | aux->map_off = off; |
| 10606 | addr += off; |
| 10607 | } |
| 10608 | |
| 10609 | insn[0].imm = (u32)addr; |
| 10610 | insn[1].imm = addr >> 32; |
| 10611 | |
| 10612 | /* check whether we recorded this map already */ |
| 10613 | for (j = 0; j < env->used_map_cnt; j++) { |
| 10614 | if (env->used_maps[j] == map) { |
| 10615 | aux->map_index = j; |
| 10616 | fdput(f); |
| 10617 | goto next_insn; |
| 10618 | } |
| 10619 | } |
| 10620 | |
| 10621 | if (env->used_map_cnt >= MAX_USED_MAPS) { |
| 10622 | fdput(f); |
| 10623 | return -E2BIG; |
| 10624 | } |
| 10625 | |
| 10626 | /* hold the map. If the program is rejected by verifier, |
| 10627 | * the map will be released by release_maps() or it |
| 10628 | * will be used by the valid program until it's unloaded |
| 10629 | * and all maps are released in free_used_maps() |
| 10630 | */ |
| 10631 | bpf_map_inc(map); |
| 10632 | |
| 10633 | aux->map_index = env->used_map_cnt; |
| 10634 | env->used_maps[env->used_map_cnt++] = map; |
| 10635 | |
| 10636 | if (bpf_map_is_cgroup_storage(map) && |
| 10637 | bpf_cgroup_storage_assign(env->prog->aux, map)) { |
| 10638 | verbose(env, "only one cgroup storage of each type is allowed\n"); |
| 10639 | fdput(f); |
| 10640 | return -EBUSY; |
| 10641 | } |
| 10642 | |
| 10643 | fdput(f); |
| 10644 | next_insn: |
| 10645 | insn++; |
| 10646 | i++; |
| 10647 | continue; |
| 10648 | } |
| 10649 | |
| 10650 | /* Basic sanity check before we invest more work here. */ |
| 10651 | if (!bpf_opcode_in_insntable(insn->code)) { |
| 10652 | verbose(env, "unknown opcode %02x\n", insn->code); |
| 10653 | return -EINVAL; |
| 10654 | } |
| 10655 | } |
| 10656 | |
| 10657 | /* now all pseudo BPF_LD_IMM64 instructions load valid |
| 10658 | * 'struct bpf_map *' into a register instead of user map_fd. |
| 10659 | * These pointers will be used later by verifier to validate map access. |
| 10660 | */ |
| 10661 | return 0; |
| 10662 | } |
| 10663 | |
| 10664 | /* drop refcnt of maps used by the rejected program */ |
| 10665 | static void release_maps(struct bpf_verifier_env *env) |
| 10666 | { |
| 10667 | __bpf_free_used_maps(env->prog->aux, env->used_maps, |
| 10668 | env->used_map_cnt); |
| 10669 | } |
| 10670 | |
| 10671 | /* drop refcnt of maps used by the rejected program */ |
| 10672 | static void release_btfs(struct bpf_verifier_env *env) |
| 10673 | { |
| 10674 | __bpf_free_used_btfs(env->prog->aux, env->used_btfs, |
| 10675 | env->used_btf_cnt); |
| 10676 | } |
| 10677 | |
| 10678 | /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ |
| 10679 | static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) |
| 10680 | { |
| 10681 | struct bpf_insn *insn = env->prog->insnsi; |
| 10682 | int insn_cnt = env->prog->len; |
| 10683 | int i; |
| 10684 | |
| 10685 | for (i = 0; i < insn_cnt; i++, insn++) |
| 10686 | if (insn->code == (BPF_LD | BPF_IMM | BPF_DW)) |
| 10687 | insn->src_reg = 0; |
| 10688 | } |
| 10689 | |
| 10690 | /* single env->prog->insni[off] instruction was replaced with the range |
| 10691 | * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying |
| 10692 | * [0, off) and [off, end) to new locations, so the patched range stays zero |
| 10693 | */ |
| 10694 | static int adjust_insn_aux_data(struct bpf_verifier_env *env, |
| 10695 | struct bpf_prog *new_prog, u32 off, u32 cnt) |
| 10696 | { |
| 10697 | struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; |
| 10698 | struct bpf_insn *insn = new_prog->insnsi; |
| 10699 | u32 prog_len; |
| 10700 | int i; |
| 10701 | |
| 10702 | /* aux info at OFF always needs adjustment, no matter fast path |
| 10703 | * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the |
| 10704 | * original insn at old prog. |
| 10705 | */ |
| 10706 | old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1); |
| 10707 | |
| 10708 | if (cnt == 1) |
| 10709 | return 0; |
| 10710 | prog_len = new_prog->len; |
| 10711 | new_data = vzalloc(array_size(prog_len, |
| 10712 | sizeof(struct bpf_insn_aux_data))); |
| 10713 | if (!new_data) |
| 10714 | return -ENOMEM; |
| 10715 | memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); |
| 10716 | memcpy(new_data + off + cnt - 1, old_data + off, |
| 10717 | sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); |
| 10718 | for (i = off; i < off + cnt - 1; i++) { |
| 10719 | new_data[i].seen = env->pass_cnt; |
| 10720 | new_data[i].zext_dst = insn_has_def32(env, insn + i); |
| 10721 | } |
| 10722 | env->insn_aux_data = new_data; |
| 10723 | vfree(old_data); |
| 10724 | return 0; |
| 10725 | } |
| 10726 | |
| 10727 | static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) |
| 10728 | { |
| 10729 | int i; |
| 10730 | |
| 10731 | if (len == 1) |
| 10732 | return; |
| 10733 | /* NOTE: fake 'exit' subprog should be updated as well. */ |
| 10734 | for (i = 0; i <= env->subprog_cnt; i++) { |
| 10735 | if (env->subprog_info[i].start <= off) |
| 10736 | continue; |
| 10737 | env->subprog_info[i].start += len - 1; |
| 10738 | } |
| 10739 | } |
| 10740 | |
| 10741 | static void adjust_poke_descs(struct bpf_prog *prog, u32 len) |
| 10742 | { |
| 10743 | struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; |
| 10744 | int i, sz = prog->aux->size_poke_tab; |
| 10745 | struct bpf_jit_poke_descriptor *desc; |
| 10746 | |
| 10747 | for (i = 0; i < sz; i++) { |
| 10748 | desc = &tab[i]; |
| 10749 | desc->insn_idx += len - 1; |
| 10750 | } |
| 10751 | } |
| 10752 | |
| 10753 | static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, |
| 10754 | const struct bpf_insn *patch, u32 len) |
| 10755 | { |
| 10756 | struct bpf_prog *new_prog; |
| 10757 | |
| 10758 | new_prog = bpf_patch_insn_single(env->prog, off, patch, len); |
| 10759 | if (IS_ERR(new_prog)) { |
| 10760 | if (PTR_ERR(new_prog) == -ERANGE) |
| 10761 | verbose(env, |
| 10762 | "insn %d cannot be patched due to 16-bit range\n", |
| 10763 | env->insn_aux_data[off].orig_idx); |
| 10764 | return NULL; |
| 10765 | } |
| 10766 | if (adjust_insn_aux_data(env, new_prog, off, len)) |
| 10767 | return NULL; |
| 10768 | adjust_subprog_starts(env, off, len); |
| 10769 | adjust_poke_descs(new_prog, len); |
| 10770 | return new_prog; |
| 10771 | } |
| 10772 | |
| 10773 | static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, |
| 10774 | u32 off, u32 cnt) |
| 10775 | { |
| 10776 | int i, j; |
| 10777 | |
| 10778 | /* find first prog starting at or after off (first to remove) */ |
| 10779 | for (i = 0; i < env->subprog_cnt; i++) |
| 10780 | if (env->subprog_info[i].start >= off) |
| 10781 | break; |
| 10782 | /* find first prog starting at or after off + cnt (first to stay) */ |
| 10783 | for (j = i; j < env->subprog_cnt; j++) |
| 10784 | if (env->subprog_info[j].start >= off + cnt) |
| 10785 | break; |
| 10786 | /* if j doesn't start exactly at off + cnt, we are just removing |
| 10787 | * the front of previous prog |
| 10788 | */ |
| 10789 | if (env->subprog_info[j].start != off + cnt) |
| 10790 | j--; |
| 10791 | |
| 10792 | if (j > i) { |
| 10793 | struct bpf_prog_aux *aux = env->prog->aux; |
| 10794 | int move; |
| 10795 | |
| 10796 | /* move fake 'exit' subprog as well */ |
| 10797 | move = env->subprog_cnt + 1 - j; |
| 10798 | |
| 10799 | memmove(env->subprog_info + i, |
| 10800 | env->subprog_info + j, |
| 10801 | sizeof(*env->subprog_info) * move); |
| 10802 | env->subprog_cnt -= j - i; |
| 10803 | |
| 10804 | /* remove func_info */ |
| 10805 | if (aux->func_info) { |
| 10806 | move = aux->func_info_cnt - j; |
| 10807 | |
| 10808 | memmove(aux->func_info + i, |
| 10809 | aux->func_info + j, |
| 10810 | sizeof(*aux->func_info) * move); |
| 10811 | aux->func_info_cnt -= j - i; |
| 10812 | /* func_info->insn_off is set after all code rewrites, |
| 10813 | * in adjust_btf_func() - no need to adjust |
| 10814 | */ |
| 10815 | } |
| 10816 | } else { |
| 10817 | /* convert i from "first prog to remove" to "first to adjust" */ |
| 10818 | if (env->subprog_info[i].start == off) |
| 10819 | i++; |
| 10820 | } |
| 10821 | |
| 10822 | /* update fake 'exit' subprog as well */ |
| 10823 | for (; i <= env->subprog_cnt; i++) |
| 10824 | env->subprog_info[i].start -= cnt; |
| 10825 | |
| 10826 | return 0; |
| 10827 | } |
| 10828 | |
| 10829 | static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, |
| 10830 | u32 cnt) |
| 10831 | { |
| 10832 | struct bpf_prog *prog = env->prog; |
| 10833 | u32 i, l_off, l_cnt, nr_linfo; |
| 10834 | struct bpf_line_info *linfo; |
| 10835 | |
| 10836 | nr_linfo = prog->aux->nr_linfo; |
| 10837 | if (!nr_linfo) |
| 10838 | return 0; |
| 10839 | |
| 10840 | linfo = prog->aux->linfo; |
| 10841 | |
| 10842 | /* find first line info to remove, count lines to be removed */ |
| 10843 | for (i = 0; i < nr_linfo; i++) |
| 10844 | if (linfo[i].insn_off >= off) |
| 10845 | break; |
| 10846 | |
| 10847 | l_off = i; |
| 10848 | l_cnt = 0; |
| 10849 | for (; i < nr_linfo; i++) |
| 10850 | if (linfo[i].insn_off < off + cnt) |
| 10851 | l_cnt++; |
| 10852 | else |
| 10853 | break; |
| 10854 | |
| 10855 | /* First live insn doesn't match first live linfo, it needs to "inherit" |
| 10856 | * last removed linfo. prog is already modified, so prog->len == off |
| 10857 | * means no live instructions after (tail of the program was removed). |
| 10858 | */ |
| 10859 | if (prog->len != off && l_cnt && |
| 10860 | (i == nr_linfo || linfo[i].insn_off != off + cnt)) { |
| 10861 | l_cnt--; |
| 10862 | linfo[--i].insn_off = off + cnt; |
| 10863 | } |
| 10864 | |
| 10865 | /* remove the line info which refer to the removed instructions */ |
| 10866 | if (l_cnt) { |
| 10867 | memmove(linfo + l_off, linfo + i, |
| 10868 | sizeof(*linfo) * (nr_linfo - i)); |
| 10869 | |
| 10870 | prog->aux->nr_linfo -= l_cnt; |
| 10871 | nr_linfo = prog->aux->nr_linfo; |
| 10872 | } |
| 10873 | |
| 10874 | /* pull all linfo[i].insn_off >= off + cnt in by cnt */ |
| 10875 | for (i = l_off; i < nr_linfo; i++) |
| 10876 | linfo[i].insn_off -= cnt; |
| 10877 | |
| 10878 | /* fix up all subprogs (incl. 'exit') which start >= off */ |
| 10879 | for (i = 0; i <= env->subprog_cnt; i++) |
| 10880 | if (env->subprog_info[i].linfo_idx > l_off) { |
| 10881 | /* program may have started in the removed region but |
| 10882 | * may not be fully removed |
| 10883 | */ |
| 10884 | if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) |
| 10885 | env->subprog_info[i].linfo_idx -= l_cnt; |
| 10886 | else |
| 10887 | env->subprog_info[i].linfo_idx = l_off; |
| 10888 | } |
| 10889 | |
| 10890 | return 0; |
| 10891 | } |
| 10892 | |
| 10893 | static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) |
| 10894 | { |
| 10895 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 10896 | unsigned int orig_prog_len = env->prog->len; |
| 10897 | int err; |
| 10898 | |
| 10899 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
| 10900 | bpf_prog_offload_remove_insns(env, off, cnt); |
| 10901 | |
| 10902 | err = bpf_remove_insns(env->prog, off, cnt); |
| 10903 | if (err) |
| 10904 | return err; |
| 10905 | |
| 10906 | err = adjust_subprog_starts_after_remove(env, off, cnt); |
| 10907 | if (err) |
| 10908 | return err; |
| 10909 | |
| 10910 | err = bpf_adj_linfo_after_remove(env, off, cnt); |
| 10911 | if (err) |
| 10912 | return err; |
| 10913 | |
| 10914 | memmove(aux_data + off, aux_data + off + cnt, |
| 10915 | sizeof(*aux_data) * (orig_prog_len - off - cnt)); |
| 10916 | |
| 10917 | return 0; |
| 10918 | } |
| 10919 | |
| 10920 | /* The verifier does more data flow analysis than llvm and will not |
| 10921 | * explore branches that are dead at run time. Malicious programs can |
| 10922 | * have dead code too. Therefore replace all dead at-run-time code |
| 10923 | * with 'ja -1'. |
| 10924 | * |
| 10925 | * Just nops are not optimal, e.g. if they would sit at the end of the |
| 10926 | * program and through another bug we would manage to jump there, then |
| 10927 | * we'd execute beyond program memory otherwise. Returning exception |
| 10928 | * code also wouldn't work since we can have subprogs where the dead |
| 10929 | * code could be located. |
| 10930 | */ |
| 10931 | static void sanitize_dead_code(struct bpf_verifier_env *env) |
| 10932 | { |
| 10933 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 10934 | struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); |
| 10935 | struct bpf_insn *insn = env->prog->insnsi; |
| 10936 | const int insn_cnt = env->prog->len; |
| 10937 | int i; |
| 10938 | |
| 10939 | for (i = 0; i < insn_cnt; i++) { |
| 10940 | if (aux_data[i].seen) |
| 10941 | continue; |
| 10942 | memcpy(insn + i, &trap, sizeof(trap)); |
| 10943 | } |
| 10944 | } |
| 10945 | |
| 10946 | static bool insn_is_cond_jump(u8 code) |
| 10947 | { |
| 10948 | u8 op; |
| 10949 | |
| 10950 | if (BPF_CLASS(code) == BPF_JMP32) |
| 10951 | return true; |
| 10952 | |
| 10953 | if (BPF_CLASS(code) != BPF_JMP) |
| 10954 | return false; |
| 10955 | |
| 10956 | op = BPF_OP(code); |
| 10957 | return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; |
| 10958 | } |
| 10959 | |
| 10960 | static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) |
| 10961 | { |
| 10962 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 10963 | struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); |
| 10964 | struct bpf_insn *insn = env->prog->insnsi; |
| 10965 | const int insn_cnt = env->prog->len; |
| 10966 | int i; |
| 10967 | |
| 10968 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 10969 | if (!insn_is_cond_jump(insn->code)) |
| 10970 | continue; |
| 10971 | |
| 10972 | if (!aux_data[i + 1].seen) |
| 10973 | ja.off = insn->off; |
| 10974 | else if (!aux_data[i + 1 + insn->off].seen) |
| 10975 | ja.off = 0; |
| 10976 | else |
| 10977 | continue; |
| 10978 | |
| 10979 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
| 10980 | bpf_prog_offload_replace_insn(env, i, &ja); |
| 10981 | |
| 10982 | memcpy(insn, &ja, sizeof(ja)); |
| 10983 | } |
| 10984 | } |
| 10985 | |
| 10986 | static int opt_remove_dead_code(struct bpf_verifier_env *env) |
| 10987 | { |
| 10988 | struct bpf_insn_aux_data *aux_data = env->insn_aux_data; |
| 10989 | int insn_cnt = env->prog->len; |
| 10990 | int i, err; |
| 10991 | |
| 10992 | for (i = 0; i < insn_cnt; i++) { |
| 10993 | int j; |
| 10994 | |
| 10995 | j = 0; |
| 10996 | while (i + j < insn_cnt && !aux_data[i + j].seen) |
| 10997 | j++; |
| 10998 | if (!j) |
| 10999 | continue; |
| 11000 | |
| 11001 | err = verifier_remove_insns(env, i, j); |
| 11002 | if (err) |
| 11003 | return err; |
| 11004 | insn_cnt = env->prog->len; |
| 11005 | } |
| 11006 | |
| 11007 | return 0; |
| 11008 | } |
| 11009 | |
| 11010 | static int opt_remove_nops(struct bpf_verifier_env *env) |
| 11011 | { |
| 11012 | const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); |
| 11013 | struct bpf_insn *insn = env->prog->insnsi; |
| 11014 | int insn_cnt = env->prog->len; |
| 11015 | int i, err; |
| 11016 | |
| 11017 | for (i = 0; i < insn_cnt; i++) { |
| 11018 | if (memcmp(&insn[i], &ja, sizeof(ja))) |
| 11019 | continue; |
| 11020 | |
| 11021 | err = verifier_remove_insns(env, i, 1); |
| 11022 | if (err) |
| 11023 | return err; |
| 11024 | insn_cnt--; |
| 11025 | i--; |
| 11026 | } |
| 11027 | |
| 11028 | return 0; |
| 11029 | } |
| 11030 | |
| 11031 | static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, |
| 11032 | const union bpf_attr *attr) |
| 11033 | { |
| 11034 | struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4]; |
| 11035 | struct bpf_insn_aux_data *aux = env->insn_aux_data; |
| 11036 | int i, patch_len, delta = 0, len = env->prog->len; |
| 11037 | struct bpf_insn *insns = env->prog->insnsi; |
| 11038 | struct bpf_prog *new_prog; |
| 11039 | bool rnd_hi32; |
| 11040 | |
| 11041 | rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32; |
| 11042 | zext_patch[1] = BPF_ZEXT_REG(0); |
| 11043 | rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0); |
| 11044 | rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); |
| 11045 | rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX); |
| 11046 | for (i = 0; i < len; i++) { |
| 11047 | int adj_idx = i + delta; |
| 11048 | struct bpf_insn insn; |
| 11049 | int load_reg; |
| 11050 | |
| 11051 | insn = insns[adj_idx]; |
| 11052 | load_reg = insn_def_regno(&insn); |
| 11053 | if (!aux[adj_idx].zext_dst) { |
| 11054 | u8 code, class; |
| 11055 | u32 imm_rnd; |
| 11056 | |
| 11057 | if (!rnd_hi32) |
| 11058 | continue; |
| 11059 | |
| 11060 | code = insn.code; |
| 11061 | class = BPF_CLASS(code); |
| 11062 | if (load_reg == -1) |
| 11063 | continue; |
| 11064 | |
| 11065 | /* NOTE: arg "reg" (the fourth one) is only used for |
| 11066 | * BPF_STX + SRC_OP, so it is safe to pass NULL |
| 11067 | * here. |
| 11068 | */ |
| 11069 | if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) { |
| 11070 | if (class == BPF_LD && |
| 11071 | BPF_MODE(code) == BPF_IMM) |
| 11072 | i++; |
| 11073 | continue; |
| 11074 | } |
| 11075 | |
| 11076 | /* ctx load could be transformed into wider load. */ |
| 11077 | if (class == BPF_LDX && |
| 11078 | aux[adj_idx].ptr_type == PTR_TO_CTX) |
| 11079 | continue; |
| 11080 | |
| 11081 | imm_rnd = get_random_int(); |
| 11082 | rnd_hi32_patch[0] = insn; |
| 11083 | rnd_hi32_patch[1].imm = imm_rnd; |
| 11084 | rnd_hi32_patch[3].dst_reg = load_reg; |
| 11085 | patch = rnd_hi32_patch; |
| 11086 | patch_len = 4; |
| 11087 | goto apply_patch_buffer; |
| 11088 | } |
| 11089 | |
| 11090 | /* Add in an zero-extend instruction if a) the JIT has requested |
| 11091 | * it or b) it's a CMPXCHG. |
| 11092 | * |
| 11093 | * The latter is because: BPF_CMPXCHG always loads a value into |
| 11094 | * R0, therefore always zero-extends. However some archs' |
| 11095 | * equivalent instruction only does this load when the |
| 11096 | * comparison is successful. This detail of CMPXCHG is |
| 11097 | * orthogonal to the general zero-extension behaviour of the |
| 11098 | * CPU, so it's treated independently of bpf_jit_needs_zext. |
| 11099 | */ |
| 11100 | if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn)) |
| 11101 | continue; |
| 11102 | |
| 11103 | if (WARN_ON(load_reg == -1)) { |
| 11104 | verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n"); |
| 11105 | return -EFAULT; |
| 11106 | } |
| 11107 | |
| 11108 | zext_patch[0] = insn; |
| 11109 | zext_patch[1].dst_reg = load_reg; |
| 11110 | zext_patch[1].src_reg = load_reg; |
| 11111 | patch = zext_patch; |
| 11112 | patch_len = 2; |
| 11113 | apply_patch_buffer: |
| 11114 | new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len); |
| 11115 | if (!new_prog) |
| 11116 | return -ENOMEM; |
| 11117 | env->prog = new_prog; |
| 11118 | insns = new_prog->insnsi; |
| 11119 | aux = env->insn_aux_data; |
| 11120 | delta += patch_len - 1; |
| 11121 | } |
| 11122 | |
| 11123 | return 0; |
| 11124 | } |
| 11125 | |
| 11126 | /* convert load instructions that access fields of a context type into a |
| 11127 | * sequence of instructions that access fields of the underlying structure: |
| 11128 | * struct __sk_buff -> struct sk_buff |
| 11129 | * struct bpf_sock_ops -> struct sock |
| 11130 | */ |
| 11131 | static int convert_ctx_accesses(struct bpf_verifier_env *env) |
| 11132 | { |
| 11133 | const struct bpf_verifier_ops *ops = env->ops; |
| 11134 | int i, cnt, size, ctx_field_size, delta = 0; |
| 11135 | const int insn_cnt = env->prog->len; |
| 11136 | struct bpf_insn insn_buf[16], *insn; |
| 11137 | u32 target_size, size_default, off; |
| 11138 | struct bpf_prog *new_prog; |
| 11139 | enum bpf_access_type type; |
| 11140 | bool is_narrower_load; |
| 11141 | |
| 11142 | if (ops->gen_prologue || env->seen_direct_write) { |
| 11143 | if (!ops->gen_prologue) { |
| 11144 | verbose(env, "bpf verifier is misconfigured\n"); |
| 11145 | return -EINVAL; |
| 11146 | } |
| 11147 | cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, |
| 11148 | env->prog); |
| 11149 | if (cnt >= ARRAY_SIZE(insn_buf)) { |
| 11150 | verbose(env, "bpf verifier is misconfigured\n"); |
| 11151 | return -EINVAL; |
| 11152 | } else if (cnt) { |
| 11153 | new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); |
| 11154 | if (!new_prog) |
| 11155 | return -ENOMEM; |
| 11156 | |
| 11157 | env->prog = new_prog; |
| 11158 | delta += cnt - 1; |
| 11159 | } |
| 11160 | } |
| 11161 | |
| 11162 | if (bpf_prog_is_dev_bound(env->prog->aux)) |
| 11163 | return 0; |
| 11164 | |
| 11165 | insn = env->prog->insnsi + delta; |
| 11166 | |
| 11167 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 11168 | bpf_convert_ctx_access_t convert_ctx_access; |
| 11169 | |
| 11170 | if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || |
| 11171 | insn->code == (BPF_LDX | BPF_MEM | BPF_H) || |
| 11172 | insn->code == (BPF_LDX | BPF_MEM | BPF_W) || |
| 11173 | insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) |
| 11174 | type = BPF_READ; |
| 11175 | else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || |
| 11176 | insn->code == (BPF_STX | BPF_MEM | BPF_H) || |
| 11177 | insn->code == (BPF_STX | BPF_MEM | BPF_W) || |
| 11178 | insn->code == (BPF_STX | BPF_MEM | BPF_DW)) |
| 11179 | type = BPF_WRITE; |
| 11180 | else |
| 11181 | continue; |
| 11182 | |
| 11183 | if (type == BPF_WRITE && |
| 11184 | env->insn_aux_data[i + delta].sanitize_stack_off) { |
| 11185 | struct bpf_insn patch[] = { |
| 11186 | /* Sanitize suspicious stack slot with zero. |
| 11187 | * There are no memory dependencies for this store, |
| 11188 | * since it's only using frame pointer and immediate |
| 11189 | * constant of zero |
| 11190 | */ |
| 11191 | BPF_ST_MEM(BPF_DW, BPF_REG_FP, |
| 11192 | env->insn_aux_data[i + delta].sanitize_stack_off, |
| 11193 | 0), |
| 11194 | /* the original STX instruction will immediately |
| 11195 | * overwrite the same stack slot with appropriate value |
| 11196 | */ |
| 11197 | *insn, |
| 11198 | }; |
| 11199 | |
| 11200 | cnt = ARRAY_SIZE(patch); |
| 11201 | new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); |
| 11202 | if (!new_prog) |
| 11203 | return -ENOMEM; |
| 11204 | |
| 11205 | delta += cnt - 1; |
| 11206 | env->prog = new_prog; |
| 11207 | insn = new_prog->insnsi + i + delta; |
| 11208 | continue; |
| 11209 | } |
| 11210 | |
| 11211 | switch (env->insn_aux_data[i + delta].ptr_type) { |
| 11212 | case PTR_TO_CTX: |
| 11213 | if (!ops->convert_ctx_access) |
| 11214 | continue; |
| 11215 | convert_ctx_access = ops->convert_ctx_access; |
| 11216 | break; |
| 11217 | case PTR_TO_SOCKET: |
| 11218 | case PTR_TO_SOCK_COMMON: |
| 11219 | convert_ctx_access = bpf_sock_convert_ctx_access; |
| 11220 | break; |
| 11221 | case PTR_TO_TCP_SOCK: |
| 11222 | convert_ctx_access = bpf_tcp_sock_convert_ctx_access; |
| 11223 | break; |
| 11224 | case PTR_TO_XDP_SOCK: |
| 11225 | convert_ctx_access = bpf_xdp_sock_convert_ctx_access; |
| 11226 | break; |
| 11227 | case PTR_TO_BTF_ID: |
| 11228 | if (type == BPF_READ) { |
| 11229 | insn->code = BPF_LDX | BPF_PROBE_MEM | |
| 11230 | BPF_SIZE((insn)->code); |
| 11231 | env->prog->aux->num_exentries++; |
| 11232 | } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) { |
| 11233 | verbose(env, "Writes through BTF pointers are not allowed\n"); |
| 11234 | return -EINVAL; |
| 11235 | } |
| 11236 | continue; |
| 11237 | default: |
| 11238 | continue; |
| 11239 | } |
| 11240 | |
| 11241 | ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; |
| 11242 | size = BPF_LDST_BYTES(insn); |
| 11243 | |
| 11244 | /* If the read access is a narrower load of the field, |
| 11245 | * convert to a 4/8-byte load, to minimum program type specific |
| 11246 | * convert_ctx_access changes. If conversion is successful, |
| 11247 | * we will apply proper mask to the result. |
| 11248 | */ |
| 11249 | is_narrower_load = size < ctx_field_size; |
| 11250 | size_default = bpf_ctx_off_adjust_machine(ctx_field_size); |
| 11251 | off = insn->off; |
| 11252 | if (is_narrower_load) { |
| 11253 | u8 size_code; |
| 11254 | |
| 11255 | if (type == BPF_WRITE) { |
| 11256 | verbose(env, "bpf verifier narrow ctx access misconfigured\n"); |
| 11257 | return -EINVAL; |
| 11258 | } |
| 11259 | |
| 11260 | size_code = BPF_H; |
| 11261 | if (ctx_field_size == 4) |
| 11262 | size_code = BPF_W; |
| 11263 | else if (ctx_field_size == 8) |
| 11264 | size_code = BPF_DW; |
| 11265 | |
| 11266 | insn->off = off & ~(size_default - 1); |
| 11267 | insn->code = BPF_LDX | BPF_MEM | size_code; |
| 11268 | } |
| 11269 | |
| 11270 | target_size = 0; |
| 11271 | cnt = convert_ctx_access(type, insn, insn_buf, env->prog, |
| 11272 | &target_size); |
| 11273 | if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || |
| 11274 | (ctx_field_size && !target_size)) { |
| 11275 | verbose(env, "bpf verifier is misconfigured\n"); |
| 11276 | return -EINVAL; |
| 11277 | } |
| 11278 | |
| 11279 | if (is_narrower_load && size < target_size) { |
| 11280 | u8 shift = bpf_ctx_narrow_access_offset( |
| 11281 | off, size, size_default) * 8; |
| 11282 | if (ctx_field_size <= 4) { |
| 11283 | if (shift) |
| 11284 | insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, |
| 11285 | insn->dst_reg, |
| 11286 | shift); |
| 11287 | insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, |
| 11288 | (1 << size * 8) - 1); |
| 11289 | } else { |
| 11290 | if (shift) |
| 11291 | insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, |
| 11292 | insn->dst_reg, |
| 11293 | shift); |
| 11294 | insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, |
| 11295 | (1ULL << size * 8) - 1); |
| 11296 | } |
| 11297 | } |
| 11298 | |
| 11299 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 11300 | if (!new_prog) |
| 11301 | return -ENOMEM; |
| 11302 | |
| 11303 | delta += cnt - 1; |
| 11304 | |
| 11305 | /* keep walking new program and skip insns we just inserted */ |
| 11306 | env->prog = new_prog; |
| 11307 | insn = new_prog->insnsi + i + delta; |
| 11308 | } |
| 11309 | |
| 11310 | return 0; |
| 11311 | } |
| 11312 | |
| 11313 | static int jit_subprogs(struct bpf_verifier_env *env) |
| 11314 | { |
| 11315 | struct bpf_prog *prog = env->prog, **func, *tmp; |
| 11316 | int i, j, subprog_start, subprog_end = 0, len, subprog; |
| 11317 | struct bpf_map *map_ptr; |
| 11318 | struct bpf_insn *insn; |
| 11319 | void *old_bpf_func; |
| 11320 | int err, num_exentries; |
| 11321 | |
| 11322 | if (env->subprog_cnt <= 1) |
| 11323 | return 0; |
| 11324 | |
| 11325 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
| 11326 | if (!bpf_pseudo_call(insn)) |
| 11327 | continue; |
| 11328 | /* Upon error here we cannot fall back to interpreter but |
| 11329 | * need a hard reject of the program. Thus -EFAULT is |
| 11330 | * propagated in any case. |
| 11331 | */ |
| 11332 | subprog = find_subprog(env, i + insn->imm + 1); |
| 11333 | if (subprog < 0) { |
| 11334 | WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", |
| 11335 | i + insn->imm + 1); |
| 11336 | return -EFAULT; |
| 11337 | } |
| 11338 | /* temporarily remember subprog id inside insn instead of |
| 11339 | * aux_data, since next loop will split up all insns into funcs |
| 11340 | */ |
| 11341 | insn->off = subprog; |
| 11342 | /* remember original imm in case JIT fails and fallback |
| 11343 | * to interpreter will be needed |
| 11344 | */ |
| 11345 | env->insn_aux_data[i].call_imm = insn->imm; |
| 11346 | /* point imm to __bpf_call_base+1 from JITs point of view */ |
| 11347 | insn->imm = 1; |
| 11348 | } |
| 11349 | |
| 11350 | err = bpf_prog_alloc_jited_linfo(prog); |
| 11351 | if (err) |
| 11352 | goto out_undo_insn; |
| 11353 | |
| 11354 | err = -ENOMEM; |
| 11355 | func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); |
| 11356 | if (!func) |
| 11357 | goto out_undo_insn; |
| 11358 | |
| 11359 | for (i = 0; i < env->subprog_cnt; i++) { |
| 11360 | subprog_start = subprog_end; |
| 11361 | subprog_end = env->subprog_info[i + 1].start; |
| 11362 | |
| 11363 | len = subprog_end - subprog_start; |
| 11364 | /* BPF_PROG_RUN doesn't call subprogs directly, |
| 11365 | * hence main prog stats include the runtime of subprogs. |
| 11366 | * subprogs don't have IDs and not reachable via prog_get_next_id |
| 11367 | * func[i]->stats will never be accessed and stays NULL |
| 11368 | */ |
| 11369 | func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); |
| 11370 | if (!func[i]) |
| 11371 | goto out_free; |
| 11372 | memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], |
| 11373 | len * sizeof(struct bpf_insn)); |
| 11374 | func[i]->type = prog->type; |
| 11375 | func[i]->len = len; |
| 11376 | if (bpf_prog_calc_tag(func[i])) |
| 11377 | goto out_free; |
| 11378 | func[i]->is_func = 1; |
| 11379 | func[i]->aux->func_idx = i; |
| 11380 | /* the btf and func_info will be freed only at prog->aux */ |
| 11381 | func[i]->aux->btf = prog->aux->btf; |
| 11382 | func[i]->aux->func_info = prog->aux->func_info; |
| 11383 | |
| 11384 | for (j = 0; j < prog->aux->size_poke_tab; j++) { |
| 11385 | u32 insn_idx = prog->aux->poke_tab[j].insn_idx; |
| 11386 | int ret; |
| 11387 | |
| 11388 | if (!(insn_idx >= subprog_start && |
| 11389 | insn_idx <= subprog_end)) |
| 11390 | continue; |
| 11391 | |
| 11392 | ret = bpf_jit_add_poke_descriptor(func[i], |
| 11393 | &prog->aux->poke_tab[j]); |
| 11394 | if (ret < 0) { |
| 11395 | verbose(env, "adding tail call poke descriptor failed\n"); |
| 11396 | goto out_free; |
| 11397 | } |
| 11398 | |
| 11399 | func[i]->insnsi[insn_idx - subprog_start].imm = ret + 1; |
| 11400 | |
| 11401 | map_ptr = func[i]->aux->poke_tab[ret].tail_call.map; |
| 11402 | ret = map_ptr->ops->map_poke_track(map_ptr, func[i]->aux); |
| 11403 | if (ret < 0) { |
| 11404 | verbose(env, "tracking tail call prog failed\n"); |
| 11405 | goto out_free; |
| 11406 | } |
| 11407 | } |
| 11408 | |
| 11409 | /* Use bpf_prog_F_tag to indicate functions in stack traces. |
| 11410 | * Long term would need debug info to populate names |
| 11411 | */ |
| 11412 | func[i]->aux->name[0] = 'F'; |
| 11413 | func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; |
| 11414 | func[i]->jit_requested = 1; |
| 11415 | func[i]->aux->linfo = prog->aux->linfo; |
| 11416 | func[i]->aux->nr_linfo = prog->aux->nr_linfo; |
| 11417 | func[i]->aux->jited_linfo = prog->aux->jited_linfo; |
| 11418 | func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; |
| 11419 | num_exentries = 0; |
| 11420 | insn = func[i]->insnsi; |
| 11421 | for (j = 0; j < func[i]->len; j++, insn++) { |
| 11422 | if (BPF_CLASS(insn->code) == BPF_LDX && |
| 11423 | BPF_MODE(insn->code) == BPF_PROBE_MEM) |
| 11424 | num_exentries++; |
| 11425 | } |
| 11426 | func[i]->aux->num_exentries = num_exentries; |
| 11427 | func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable; |
| 11428 | func[i] = bpf_int_jit_compile(func[i]); |
| 11429 | if (!func[i]->jited) { |
| 11430 | err = -ENOTSUPP; |
| 11431 | goto out_free; |
| 11432 | } |
| 11433 | cond_resched(); |
| 11434 | } |
| 11435 | |
| 11436 | /* Untrack main program's aux structs so that during map_poke_run() |
| 11437 | * we will not stumble upon the unfilled poke descriptors; each |
| 11438 | * of the main program's poke descs got distributed across subprogs |
| 11439 | * and got tracked onto map, so we are sure that none of them will |
| 11440 | * be missed after the operation below |
| 11441 | */ |
| 11442 | for (i = 0; i < prog->aux->size_poke_tab; i++) { |
| 11443 | map_ptr = prog->aux->poke_tab[i].tail_call.map; |
| 11444 | |
| 11445 | map_ptr->ops->map_poke_untrack(map_ptr, prog->aux); |
| 11446 | } |
| 11447 | |
| 11448 | /* at this point all bpf functions were successfully JITed |
| 11449 | * now populate all bpf_calls with correct addresses and |
| 11450 | * run last pass of JIT |
| 11451 | */ |
| 11452 | for (i = 0; i < env->subprog_cnt; i++) { |
| 11453 | insn = func[i]->insnsi; |
| 11454 | for (j = 0; j < func[i]->len; j++, insn++) { |
| 11455 | if (!bpf_pseudo_call(insn)) |
| 11456 | continue; |
| 11457 | subprog = insn->off; |
| 11458 | insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - |
| 11459 | __bpf_call_base; |
| 11460 | } |
| 11461 | |
| 11462 | /* we use the aux data to keep a list of the start addresses |
| 11463 | * of the JITed images for each function in the program |
| 11464 | * |
| 11465 | * for some architectures, such as powerpc64, the imm field |
| 11466 | * might not be large enough to hold the offset of the start |
| 11467 | * address of the callee's JITed image from __bpf_call_base |
| 11468 | * |
| 11469 | * in such cases, we can lookup the start address of a callee |
| 11470 | * by using its subprog id, available from the off field of |
| 11471 | * the call instruction, as an index for this list |
| 11472 | */ |
| 11473 | func[i]->aux->func = func; |
| 11474 | func[i]->aux->func_cnt = env->subprog_cnt; |
| 11475 | } |
| 11476 | for (i = 0; i < env->subprog_cnt; i++) { |
| 11477 | old_bpf_func = func[i]->bpf_func; |
| 11478 | tmp = bpf_int_jit_compile(func[i]); |
| 11479 | if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { |
| 11480 | verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); |
| 11481 | err = -ENOTSUPP; |
| 11482 | goto out_free; |
| 11483 | } |
| 11484 | cond_resched(); |
| 11485 | } |
| 11486 | |
| 11487 | /* finally lock prog and jit images for all functions and |
| 11488 | * populate kallsysm |
| 11489 | */ |
| 11490 | for (i = 0; i < env->subprog_cnt; i++) { |
| 11491 | bpf_prog_lock_ro(func[i]); |
| 11492 | bpf_prog_kallsyms_add(func[i]); |
| 11493 | } |
| 11494 | |
| 11495 | /* Last step: make now unused interpreter insns from main |
| 11496 | * prog consistent for later dump requests, so they can |
| 11497 | * later look the same as if they were interpreted only. |
| 11498 | */ |
| 11499 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
| 11500 | if (!bpf_pseudo_call(insn)) |
| 11501 | continue; |
| 11502 | insn->off = env->insn_aux_data[i].call_imm; |
| 11503 | subprog = find_subprog(env, i + insn->off + 1); |
| 11504 | insn->imm = subprog; |
| 11505 | } |
| 11506 | |
| 11507 | prog->jited = 1; |
| 11508 | prog->bpf_func = func[0]->bpf_func; |
| 11509 | prog->aux->func = func; |
| 11510 | prog->aux->func_cnt = env->subprog_cnt; |
| 11511 | bpf_prog_free_unused_jited_linfo(prog); |
| 11512 | return 0; |
| 11513 | out_free: |
| 11514 | for (i = 0; i < env->subprog_cnt; i++) { |
| 11515 | if (!func[i]) |
| 11516 | continue; |
| 11517 | |
| 11518 | for (j = 0; j < func[i]->aux->size_poke_tab; j++) { |
| 11519 | map_ptr = func[i]->aux->poke_tab[j].tail_call.map; |
| 11520 | map_ptr->ops->map_poke_untrack(map_ptr, func[i]->aux); |
| 11521 | } |
| 11522 | bpf_jit_free(func[i]); |
| 11523 | } |
| 11524 | kfree(func); |
| 11525 | out_undo_insn: |
| 11526 | /* cleanup main prog to be interpreted */ |
| 11527 | prog->jit_requested = 0; |
| 11528 | for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { |
| 11529 | if (!bpf_pseudo_call(insn)) |
| 11530 | continue; |
| 11531 | insn->off = 0; |
| 11532 | insn->imm = env->insn_aux_data[i].call_imm; |
| 11533 | } |
| 11534 | bpf_prog_free_jited_linfo(prog); |
| 11535 | return err; |
| 11536 | } |
| 11537 | |
| 11538 | static int fixup_call_args(struct bpf_verifier_env *env) |
| 11539 | { |
| 11540 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
| 11541 | struct bpf_prog *prog = env->prog; |
| 11542 | struct bpf_insn *insn = prog->insnsi; |
| 11543 | int i, depth; |
| 11544 | #endif |
| 11545 | int err = 0; |
| 11546 | |
| 11547 | if (env->prog->jit_requested && |
| 11548 | !bpf_prog_is_dev_bound(env->prog->aux)) { |
| 11549 | err = jit_subprogs(env); |
| 11550 | if (err == 0) |
| 11551 | return 0; |
| 11552 | if (err == -EFAULT) |
| 11553 | return err; |
| 11554 | } |
| 11555 | #ifndef CONFIG_BPF_JIT_ALWAYS_ON |
| 11556 | if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) { |
| 11557 | /* When JIT fails the progs with bpf2bpf calls and tail_calls |
| 11558 | * have to be rejected, since interpreter doesn't support them yet. |
| 11559 | */ |
| 11560 | verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n"); |
| 11561 | return -EINVAL; |
| 11562 | } |
| 11563 | for (i = 0; i < prog->len; i++, insn++) { |
| 11564 | if (!bpf_pseudo_call(insn)) |
| 11565 | continue; |
| 11566 | depth = get_callee_stack_depth(env, insn, i); |
| 11567 | if (depth < 0) |
| 11568 | return depth; |
| 11569 | bpf_patch_call_args(insn, depth); |
| 11570 | } |
| 11571 | err = 0; |
| 11572 | #endif |
| 11573 | return err; |
| 11574 | } |
| 11575 | |
| 11576 | /* fixup insn->imm field of bpf_call instructions |
| 11577 | * and inline eligible helpers as explicit sequence of BPF instructions |
| 11578 | * |
| 11579 | * this function is called after eBPF program passed verification |
| 11580 | */ |
| 11581 | static int fixup_bpf_calls(struct bpf_verifier_env *env) |
| 11582 | { |
| 11583 | struct bpf_prog *prog = env->prog; |
| 11584 | bool expect_blinding = bpf_jit_blinding_enabled(prog); |
| 11585 | struct bpf_insn *insn = prog->insnsi; |
| 11586 | const struct bpf_func_proto *fn; |
| 11587 | const int insn_cnt = prog->len; |
| 11588 | const struct bpf_map_ops *ops; |
| 11589 | struct bpf_insn_aux_data *aux; |
| 11590 | struct bpf_insn insn_buf[16]; |
| 11591 | struct bpf_prog *new_prog; |
| 11592 | struct bpf_map *map_ptr; |
| 11593 | int i, ret, cnt, delta = 0; |
| 11594 | |
| 11595 | for (i = 0; i < insn_cnt; i++, insn++) { |
| 11596 | if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || |
| 11597 | insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || |
| 11598 | insn->code == (BPF_ALU | BPF_MOD | BPF_X) || |
| 11599 | insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { |
| 11600 | bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; |
| 11601 | bool isdiv = BPF_OP(insn->code) == BPF_DIV; |
| 11602 | struct bpf_insn *patchlet; |
| 11603 | struct bpf_insn chk_and_div[] = { |
| 11604 | /* [R,W]x div 0 -> 0 */ |
| 11605 | BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | |
| 11606 | BPF_JNE | BPF_K, insn->src_reg, |
| 11607 | 0, 2, 0), |
| 11608 | BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), |
| 11609 | BPF_JMP_IMM(BPF_JA, 0, 0, 1), |
| 11610 | *insn, |
| 11611 | }; |
| 11612 | struct bpf_insn chk_and_mod[] = { |
| 11613 | /* [R,W]x mod 0 -> [R,W]x */ |
| 11614 | BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) | |
| 11615 | BPF_JEQ | BPF_K, insn->src_reg, |
| 11616 | 0, 1 + (is64 ? 0 : 1), 0), |
| 11617 | *insn, |
| 11618 | BPF_JMP_IMM(BPF_JA, 0, 0, 1), |
| 11619 | BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), |
| 11620 | }; |
| 11621 | |
| 11622 | patchlet = isdiv ? chk_and_div : chk_and_mod; |
| 11623 | cnt = isdiv ? ARRAY_SIZE(chk_and_div) : |
| 11624 | ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0); |
| 11625 | |
| 11626 | new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); |
| 11627 | if (!new_prog) |
| 11628 | return -ENOMEM; |
| 11629 | |
| 11630 | delta += cnt - 1; |
| 11631 | env->prog = prog = new_prog; |
| 11632 | insn = new_prog->insnsi + i + delta; |
| 11633 | continue; |
| 11634 | } |
| 11635 | |
| 11636 | if (BPF_CLASS(insn->code) == BPF_LD && |
| 11637 | (BPF_MODE(insn->code) == BPF_ABS || |
| 11638 | BPF_MODE(insn->code) == BPF_IND)) { |
| 11639 | cnt = env->ops->gen_ld_abs(insn, insn_buf); |
| 11640 | if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { |
| 11641 | verbose(env, "bpf verifier is misconfigured\n"); |
| 11642 | return -EINVAL; |
| 11643 | } |
| 11644 | |
| 11645 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 11646 | if (!new_prog) |
| 11647 | return -ENOMEM; |
| 11648 | |
| 11649 | delta += cnt - 1; |
| 11650 | env->prog = prog = new_prog; |
| 11651 | insn = new_prog->insnsi + i + delta; |
| 11652 | continue; |
| 11653 | } |
| 11654 | |
| 11655 | if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || |
| 11656 | insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { |
| 11657 | const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; |
| 11658 | const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; |
| 11659 | struct bpf_insn insn_buf[16]; |
| 11660 | struct bpf_insn *patch = &insn_buf[0]; |
| 11661 | bool issrc, isneg; |
| 11662 | u32 off_reg; |
| 11663 | |
| 11664 | aux = &env->insn_aux_data[i + delta]; |
| 11665 | if (!aux->alu_state || |
| 11666 | aux->alu_state == BPF_ALU_NON_POINTER) |
| 11667 | continue; |
| 11668 | |
| 11669 | isneg = aux->alu_state & BPF_ALU_NEG_VALUE; |
| 11670 | issrc = (aux->alu_state & BPF_ALU_SANITIZE) == |
| 11671 | BPF_ALU_SANITIZE_SRC; |
| 11672 | |
| 11673 | off_reg = issrc ? insn->src_reg : insn->dst_reg; |
| 11674 | if (isneg) |
| 11675 | *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); |
| 11676 | *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit); |
| 11677 | *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); |
| 11678 | *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); |
| 11679 | *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); |
| 11680 | *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); |
| 11681 | if (issrc) { |
| 11682 | *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, |
| 11683 | off_reg); |
| 11684 | insn->src_reg = BPF_REG_AX; |
| 11685 | } else { |
| 11686 | *patch++ = BPF_ALU64_REG(BPF_AND, off_reg, |
| 11687 | BPF_REG_AX); |
| 11688 | } |
| 11689 | if (isneg) |
| 11690 | insn->code = insn->code == code_add ? |
| 11691 | code_sub : code_add; |
| 11692 | *patch++ = *insn; |
| 11693 | if (issrc && isneg) |
| 11694 | *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); |
| 11695 | cnt = patch - insn_buf; |
| 11696 | |
| 11697 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 11698 | if (!new_prog) |
| 11699 | return -ENOMEM; |
| 11700 | |
| 11701 | delta += cnt - 1; |
| 11702 | env->prog = prog = new_prog; |
| 11703 | insn = new_prog->insnsi + i + delta; |
| 11704 | continue; |
| 11705 | } |
| 11706 | |
| 11707 | if (insn->code != (BPF_JMP | BPF_CALL)) |
| 11708 | continue; |
| 11709 | if (insn->src_reg == BPF_PSEUDO_CALL) |
| 11710 | continue; |
| 11711 | |
| 11712 | if (insn->imm == BPF_FUNC_get_route_realm) |
| 11713 | prog->dst_needed = 1; |
| 11714 | if (insn->imm == BPF_FUNC_get_prandom_u32) |
| 11715 | bpf_user_rnd_init_once(); |
| 11716 | if (insn->imm == BPF_FUNC_override_return) |
| 11717 | prog->kprobe_override = 1; |
| 11718 | if (insn->imm == BPF_FUNC_tail_call) { |
| 11719 | /* If we tail call into other programs, we |
| 11720 | * cannot make any assumptions since they can |
| 11721 | * be replaced dynamically during runtime in |
| 11722 | * the program array. |
| 11723 | */ |
| 11724 | prog->cb_access = 1; |
| 11725 | if (!allow_tail_call_in_subprogs(env)) |
| 11726 | prog->aux->stack_depth = MAX_BPF_STACK; |
| 11727 | prog->aux->max_pkt_offset = MAX_PACKET_OFF; |
| 11728 | |
| 11729 | /* mark bpf_tail_call as different opcode to avoid |
| 11730 | * conditional branch in the interpeter for every normal |
| 11731 | * call and to prevent accidental JITing by JIT compiler |
| 11732 | * that doesn't support bpf_tail_call yet |
| 11733 | */ |
| 11734 | insn->imm = 0; |
| 11735 | insn->code = BPF_JMP | BPF_TAIL_CALL; |
| 11736 | |
| 11737 | aux = &env->insn_aux_data[i + delta]; |
| 11738 | if (env->bpf_capable && !expect_blinding && |
| 11739 | prog->jit_requested && |
| 11740 | !bpf_map_key_poisoned(aux) && |
| 11741 | !bpf_map_ptr_poisoned(aux) && |
| 11742 | !bpf_map_ptr_unpriv(aux)) { |
| 11743 | struct bpf_jit_poke_descriptor desc = { |
| 11744 | .reason = BPF_POKE_REASON_TAIL_CALL, |
| 11745 | .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state), |
| 11746 | .tail_call.key = bpf_map_key_immediate(aux), |
| 11747 | .insn_idx = i + delta, |
| 11748 | }; |
| 11749 | |
| 11750 | ret = bpf_jit_add_poke_descriptor(prog, &desc); |
| 11751 | if (ret < 0) { |
| 11752 | verbose(env, "adding tail call poke descriptor failed\n"); |
| 11753 | return ret; |
| 11754 | } |
| 11755 | |
| 11756 | insn->imm = ret + 1; |
| 11757 | continue; |
| 11758 | } |
| 11759 | |
| 11760 | if (!bpf_map_ptr_unpriv(aux)) |
| 11761 | continue; |
| 11762 | |
| 11763 | /* instead of changing every JIT dealing with tail_call |
| 11764 | * emit two extra insns: |
| 11765 | * if (index >= max_entries) goto out; |
| 11766 | * index &= array->index_mask; |
| 11767 | * to avoid out-of-bounds cpu speculation |
| 11768 | */ |
| 11769 | if (bpf_map_ptr_poisoned(aux)) { |
| 11770 | verbose(env, "tail_call abusing map_ptr\n"); |
| 11771 | return -EINVAL; |
| 11772 | } |
| 11773 | |
| 11774 | map_ptr = BPF_MAP_PTR(aux->map_ptr_state); |
| 11775 | insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, |
| 11776 | map_ptr->max_entries, 2); |
| 11777 | insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, |
| 11778 | container_of(map_ptr, |
| 11779 | struct bpf_array, |
| 11780 | map)->index_mask); |
| 11781 | insn_buf[2] = *insn; |
| 11782 | cnt = 3; |
| 11783 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); |
| 11784 | if (!new_prog) |
| 11785 | return -ENOMEM; |
| 11786 | |
| 11787 | delta += cnt - 1; |
| 11788 | env->prog = prog = new_prog; |
| 11789 | insn = new_prog->insnsi + i + delta; |
| 11790 | continue; |
| 11791 | } |
| 11792 | |
| 11793 | /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup |
| 11794 | * and other inlining handlers are currently limited to 64 bit |
| 11795 | * only. |
| 11796 | */ |
| 11797 | if (prog->jit_requested && BITS_PER_LONG == 64 && |
| 11798 | (insn->imm == BPF_FUNC_map_lookup_elem || |
| 11799 | insn->imm == BPF_FUNC_map_update_elem || |
| 11800 | insn->imm == BPF_FUNC_map_delete_elem || |
| 11801 | insn->imm == BPF_FUNC_map_push_elem || |
| 11802 | insn->imm == BPF_FUNC_map_pop_elem || |
| 11803 | insn->imm == BPF_FUNC_map_peek_elem)) { |
| 11804 | aux = &env->insn_aux_data[i + delta]; |
| 11805 | if (bpf_map_ptr_poisoned(aux)) |
| 11806 | goto patch_call_imm; |
| 11807 | |
| 11808 | map_ptr = BPF_MAP_PTR(aux->map_ptr_state); |
| 11809 | ops = map_ptr->ops; |
| 11810 | if (insn->imm == BPF_FUNC_map_lookup_elem && |
| 11811 | ops->map_gen_lookup) { |
| 11812 | cnt = ops->map_gen_lookup(map_ptr, insn_buf); |
| 11813 | if (cnt == -EOPNOTSUPP) |
| 11814 | goto patch_map_ops_generic; |
| 11815 | if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) { |
| 11816 | verbose(env, "bpf verifier is misconfigured\n"); |
| 11817 | return -EINVAL; |
| 11818 | } |
| 11819 | |
| 11820 | new_prog = bpf_patch_insn_data(env, i + delta, |
| 11821 | insn_buf, cnt); |
| 11822 | if (!new_prog) |
| 11823 | return -ENOMEM; |
| 11824 | |
| 11825 | delta += cnt - 1; |
| 11826 | env->prog = prog = new_prog; |
| 11827 | insn = new_prog->insnsi + i + delta; |
| 11828 | continue; |
| 11829 | } |
| 11830 | |
| 11831 | BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, |
| 11832 | (void *(*)(struct bpf_map *map, void *key))NULL)); |
| 11833 | BUILD_BUG_ON(!__same_type(ops->map_delete_elem, |
| 11834 | (int (*)(struct bpf_map *map, void *key))NULL)); |
| 11835 | BUILD_BUG_ON(!__same_type(ops->map_update_elem, |
| 11836 | (int (*)(struct bpf_map *map, void *key, void *value, |
| 11837 | u64 flags))NULL)); |
| 11838 | BUILD_BUG_ON(!__same_type(ops->map_push_elem, |
| 11839 | (int (*)(struct bpf_map *map, void *value, |
| 11840 | u64 flags))NULL)); |
| 11841 | BUILD_BUG_ON(!__same_type(ops->map_pop_elem, |
| 11842 | (int (*)(struct bpf_map *map, void *value))NULL)); |
| 11843 | BUILD_BUG_ON(!__same_type(ops->map_peek_elem, |
| 11844 | (int (*)(struct bpf_map *map, void *value))NULL)); |
| 11845 | patch_map_ops_generic: |
| 11846 | switch (insn->imm) { |
| 11847 | case BPF_FUNC_map_lookup_elem: |
| 11848 | insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - |
| 11849 | __bpf_call_base; |
| 11850 | continue; |
| 11851 | case BPF_FUNC_map_update_elem: |
| 11852 | insn->imm = BPF_CAST_CALL(ops->map_update_elem) - |
| 11853 | __bpf_call_base; |
| 11854 | continue; |
| 11855 | case BPF_FUNC_map_delete_elem: |
| 11856 | insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - |
| 11857 | __bpf_call_base; |
| 11858 | continue; |
| 11859 | case BPF_FUNC_map_push_elem: |
| 11860 | insn->imm = BPF_CAST_CALL(ops->map_push_elem) - |
| 11861 | __bpf_call_base; |
| 11862 | continue; |
| 11863 | case BPF_FUNC_map_pop_elem: |
| 11864 | insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - |
| 11865 | __bpf_call_base; |
| 11866 | continue; |
| 11867 | case BPF_FUNC_map_peek_elem: |
| 11868 | insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - |
| 11869 | __bpf_call_base; |
| 11870 | continue; |
| 11871 | } |
| 11872 | |
| 11873 | goto patch_call_imm; |
| 11874 | } |
| 11875 | |
| 11876 | if (prog->jit_requested && BITS_PER_LONG == 64 && |
| 11877 | insn->imm == BPF_FUNC_jiffies64) { |
| 11878 | struct bpf_insn ld_jiffies_addr[2] = { |
| 11879 | BPF_LD_IMM64(BPF_REG_0, |
| 11880 | (unsigned long)&jiffies), |
| 11881 | }; |
| 11882 | |
| 11883 | insn_buf[0] = ld_jiffies_addr[0]; |
| 11884 | insn_buf[1] = ld_jiffies_addr[1]; |
| 11885 | insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, |
| 11886 | BPF_REG_0, 0); |
| 11887 | cnt = 3; |
| 11888 | |
| 11889 | new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, |
| 11890 | cnt); |
| 11891 | if (!new_prog) |
| 11892 | return -ENOMEM; |
| 11893 | |
| 11894 | delta += cnt - 1; |
| 11895 | env->prog = prog = new_prog; |
| 11896 | insn = new_prog->insnsi + i + delta; |
| 11897 | continue; |
| 11898 | } |
| 11899 | |
| 11900 | patch_call_imm: |
| 11901 | fn = env->ops->get_func_proto(insn->imm, env->prog); |
| 11902 | /* all functions that have prototype and verifier allowed |
| 11903 | * programs to call them, must be real in-kernel functions |
| 11904 | */ |
| 11905 | if (!fn->func) { |
| 11906 | verbose(env, |
| 11907 | "kernel subsystem misconfigured func %s#%d\n", |
| 11908 | func_id_name(insn->imm), insn->imm); |
| 11909 | return -EFAULT; |
| 11910 | } |
| 11911 | insn->imm = fn->func - __bpf_call_base; |
| 11912 | } |
| 11913 | |
| 11914 | /* Since poke tab is now finalized, publish aux to tracker. */ |
| 11915 | for (i = 0; i < prog->aux->size_poke_tab; i++) { |
| 11916 | map_ptr = prog->aux->poke_tab[i].tail_call.map; |
| 11917 | if (!map_ptr->ops->map_poke_track || |
| 11918 | !map_ptr->ops->map_poke_untrack || |
| 11919 | !map_ptr->ops->map_poke_run) { |
| 11920 | verbose(env, "bpf verifier is misconfigured\n"); |
| 11921 | return -EINVAL; |
| 11922 | } |
| 11923 | |
| 11924 | ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux); |
| 11925 | if (ret < 0) { |
| 11926 | verbose(env, "tracking tail call prog failed\n"); |
| 11927 | return ret; |
| 11928 | } |
| 11929 | } |
| 11930 | |
| 11931 | return 0; |
| 11932 | } |
| 11933 | |
| 11934 | static void free_states(struct bpf_verifier_env *env) |
| 11935 | { |
| 11936 | struct bpf_verifier_state_list *sl, *sln; |
| 11937 | int i; |
| 11938 | |
| 11939 | sl = env->free_list; |
| 11940 | while (sl) { |
| 11941 | sln = sl->next; |
| 11942 | free_verifier_state(&sl->state, false); |
| 11943 | kfree(sl); |
| 11944 | sl = sln; |
| 11945 | } |
| 11946 | env->free_list = NULL; |
| 11947 | |
| 11948 | if (!env->explored_states) |
| 11949 | return; |
| 11950 | |
| 11951 | for (i = 0; i < state_htab_size(env); i++) { |
| 11952 | sl = env->explored_states[i]; |
| 11953 | |
| 11954 | while (sl) { |
| 11955 | sln = sl->next; |
| 11956 | free_verifier_state(&sl->state, false); |
| 11957 | kfree(sl); |
| 11958 | sl = sln; |
| 11959 | } |
| 11960 | env->explored_states[i] = NULL; |
| 11961 | } |
| 11962 | } |
| 11963 | |
| 11964 | /* The verifier is using insn_aux_data[] to store temporary data during |
| 11965 | * verification and to store information for passes that run after the |
| 11966 | * verification like dead code sanitization. do_check_common() for subprogram N |
| 11967 | * may analyze many other subprograms. sanitize_insn_aux_data() clears all |
| 11968 | * temporary data after do_check_common() finds that subprogram N cannot be |
| 11969 | * verified independently. pass_cnt counts the number of times |
| 11970 | * do_check_common() was run and insn->aux->seen tells the pass number |
| 11971 | * insn_aux_data was touched. These variables are compared to clear temporary |
| 11972 | * data from failed pass. For testing and experiments do_check_common() can be |
| 11973 | * run multiple times even when prior attempt to verify is unsuccessful. |
| 11974 | */ |
| 11975 | static void sanitize_insn_aux_data(struct bpf_verifier_env *env) |
| 11976 | { |
| 11977 | struct bpf_insn *insn = env->prog->insnsi; |
| 11978 | struct bpf_insn_aux_data *aux; |
| 11979 | int i, class; |
| 11980 | |
| 11981 | for (i = 0; i < env->prog->len; i++) { |
| 11982 | class = BPF_CLASS(insn[i].code); |
| 11983 | if (class != BPF_LDX && class != BPF_STX) |
| 11984 | continue; |
| 11985 | aux = &env->insn_aux_data[i]; |
| 11986 | if (aux->seen != env->pass_cnt) |
| 11987 | continue; |
| 11988 | memset(aux, 0, offsetof(typeof(*aux), orig_idx)); |
| 11989 | } |
| 11990 | } |
| 11991 | |
| 11992 | static int do_check_common(struct bpf_verifier_env *env, int subprog) |
| 11993 | { |
| 11994 | bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); |
| 11995 | struct bpf_verifier_state *state; |
| 11996 | struct bpf_reg_state *regs; |
| 11997 | int ret, i; |
| 11998 | |
| 11999 | env->prev_linfo = NULL; |
| 12000 | env->pass_cnt++; |
| 12001 | |
| 12002 | state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); |
| 12003 | if (!state) |
| 12004 | return -ENOMEM; |
| 12005 | state->curframe = 0; |
| 12006 | state->speculative = false; |
| 12007 | state->branches = 1; |
| 12008 | state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); |
| 12009 | if (!state->frame[0]) { |
| 12010 | kfree(state); |
| 12011 | return -ENOMEM; |
| 12012 | } |
| 12013 | env->cur_state = state; |
| 12014 | init_func_state(env, state->frame[0], |
| 12015 | BPF_MAIN_FUNC /* callsite */, |
| 12016 | 0 /* frameno */, |
| 12017 | subprog); |
| 12018 | |
| 12019 | regs = state->frame[state->curframe]->regs; |
| 12020 | if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { |
| 12021 | ret = btf_prepare_func_args(env, subprog, regs); |
| 12022 | if (ret) |
| 12023 | goto out; |
| 12024 | for (i = BPF_REG_1; i <= BPF_REG_5; i++) { |
| 12025 | if (regs[i].type == PTR_TO_CTX) |
| 12026 | mark_reg_known_zero(env, regs, i); |
| 12027 | else if (regs[i].type == SCALAR_VALUE) |
| 12028 | mark_reg_unknown(env, regs, i); |
| 12029 | else if (regs[i].type == PTR_TO_MEM_OR_NULL) { |
| 12030 | const u32 mem_size = regs[i].mem_size; |
| 12031 | |
| 12032 | mark_reg_known_zero(env, regs, i); |
| 12033 | regs[i].mem_size = mem_size; |
| 12034 | regs[i].id = ++env->id_gen; |
| 12035 | } |
| 12036 | } |
| 12037 | } else { |
| 12038 | /* 1st arg to a function */ |
| 12039 | regs[BPF_REG_1].type = PTR_TO_CTX; |
| 12040 | mark_reg_known_zero(env, regs, BPF_REG_1); |
| 12041 | ret = btf_check_func_arg_match(env, subprog, regs); |
| 12042 | if (ret == -EFAULT) |
| 12043 | /* unlikely verifier bug. abort. |
| 12044 | * ret == 0 and ret < 0 are sadly acceptable for |
| 12045 | * main() function due to backward compatibility. |
| 12046 | * Like socket filter program may be written as: |
| 12047 | * int bpf_prog(struct pt_regs *ctx) |
| 12048 | * and never dereference that ctx in the program. |
| 12049 | * 'struct pt_regs' is a type mismatch for socket |
| 12050 | * filter that should be using 'struct __sk_buff'. |
| 12051 | */ |
| 12052 | goto out; |
| 12053 | } |
| 12054 | |
| 12055 | ret = do_check(env); |
| 12056 | out: |
| 12057 | /* check for NULL is necessary, since cur_state can be freed inside |
| 12058 | * do_check() under memory pressure. |
| 12059 | */ |
| 12060 | if (env->cur_state) { |
| 12061 | free_verifier_state(env->cur_state, true); |
| 12062 | env->cur_state = NULL; |
| 12063 | } |
| 12064 | while (!pop_stack(env, NULL, NULL, false)); |
| 12065 | if (!ret && pop_log) |
| 12066 | bpf_vlog_reset(&env->log, 0); |
| 12067 | free_states(env); |
| 12068 | if (ret) |
| 12069 | /* clean aux data in case subprog was rejected */ |
| 12070 | sanitize_insn_aux_data(env); |
| 12071 | return ret; |
| 12072 | } |
| 12073 | |
| 12074 | /* Verify all global functions in a BPF program one by one based on their BTF. |
| 12075 | * All global functions must pass verification. Otherwise the whole program is rejected. |
| 12076 | * Consider: |
| 12077 | * int bar(int); |
| 12078 | * int foo(int f) |
| 12079 | * { |
| 12080 | * return bar(f); |
| 12081 | * } |
| 12082 | * int bar(int b) |
| 12083 | * { |
| 12084 | * ... |
| 12085 | * } |
| 12086 | * foo() will be verified first for R1=any_scalar_value. During verification it |
| 12087 | * will be assumed that bar() already verified successfully and call to bar() |
| 12088 | * from foo() will be checked for type match only. Later bar() will be verified |
| 12089 | * independently to check that it's safe for R1=any_scalar_value. |
| 12090 | */ |
| 12091 | static int do_check_subprogs(struct bpf_verifier_env *env) |
| 12092 | { |
| 12093 | struct bpf_prog_aux *aux = env->prog->aux; |
| 12094 | int i, ret; |
| 12095 | |
| 12096 | if (!aux->func_info) |
| 12097 | return 0; |
| 12098 | |
| 12099 | for (i = 1; i < env->subprog_cnt; i++) { |
| 12100 | if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) |
| 12101 | continue; |
| 12102 | env->insn_idx = env->subprog_info[i].start; |
| 12103 | WARN_ON_ONCE(env->insn_idx == 0); |
| 12104 | ret = do_check_common(env, i); |
| 12105 | if (ret) { |
| 12106 | return ret; |
| 12107 | } else if (env->log.level & BPF_LOG_LEVEL) { |
| 12108 | verbose(env, |
| 12109 | "Func#%d is safe for any args that match its prototype\n", |
| 12110 | i); |
| 12111 | } |
| 12112 | } |
| 12113 | return 0; |
| 12114 | } |
| 12115 | |
| 12116 | static int do_check_main(struct bpf_verifier_env *env) |
| 12117 | { |
| 12118 | int ret; |
| 12119 | |
| 12120 | env->insn_idx = 0; |
| 12121 | ret = do_check_common(env, 0); |
| 12122 | if (!ret) |
| 12123 | env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; |
| 12124 | return ret; |
| 12125 | } |
| 12126 | |
| 12127 | |
| 12128 | static void print_verification_stats(struct bpf_verifier_env *env) |
| 12129 | { |
| 12130 | int i; |
| 12131 | |
| 12132 | if (env->log.level & BPF_LOG_STATS) { |
| 12133 | verbose(env, "verification time %lld usec\n", |
| 12134 | div_u64(env->verification_time, 1000)); |
| 12135 | verbose(env, "stack depth "); |
| 12136 | for (i = 0; i < env->subprog_cnt; i++) { |
| 12137 | u32 depth = env->subprog_info[i].stack_depth; |
| 12138 | |
| 12139 | verbose(env, "%d", depth); |
| 12140 | if (i + 1 < env->subprog_cnt) |
| 12141 | verbose(env, "+"); |
| 12142 | } |
| 12143 | verbose(env, "\n"); |
| 12144 | } |
| 12145 | verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " |
| 12146 | "total_states %d peak_states %d mark_read %d\n", |
| 12147 | env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, |
| 12148 | env->max_states_per_insn, env->total_states, |
| 12149 | env->peak_states, env->longest_mark_read_walk); |
| 12150 | } |
| 12151 | |
| 12152 | static int check_struct_ops_btf_id(struct bpf_verifier_env *env) |
| 12153 | { |
| 12154 | const struct btf_type *t, *func_proto; |
| 12155 | const struct bpf_struct_ops *st_ops; |
| 12156 | const struct btf_member *member; |
| 12157 | struct bpf_prog *prog = env->prog; |
| 12158 | u32 btf_id, member_idx; |
| 12159 | const char *mname; |
| 12160 | |
| 12161 | if (!prog->gpl_compatible) { |
| 12162 | verbose(env, "struct ops programs must have a GPL compatible license\n"); |
| 12163 | return -EINVAL; |
| 12164 | } |
| 12165 | |
| 12166 | btf_id = prog->aux->attach_btf_id; |
| 12167 | st_ops = bpf_struct_ops_find(btf_id); |
| 12168 | if (!st_ops) { |
| 12169 | verbose(env, "attach_btf_id %u is not a supported struct\n", |
| 12170 | btf_id); |
| 12171 | return -ENOTSUPP; |
| 12172 | } |
| 12173 | |
| 12174 | t = st_ops->type; |
| 12175 | member_idx = prog->expected_attach_type; |
| 12176 | if (member_idx >= btf_type_vlen(t)) { |
| 12177 | verbose(env, "attach to invalid member idx %u of struct %s\n", |
| 12178 | member_idx, st_ops->name); |
| 12179 | return -EINVAL; |
| 12180 | } |
| 12181 | |
| 12182 | member = &btf_type_member(t)[member_idx]; |
| 12183 | mname = btf_name_by_offset(btf_vmlinux, member->name_off); |
| 12184 | func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type, |
| 12185 | NULL); |
| 12186 | if (!func_proto) { |
| 12187 | verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", |
| 12188 | mname, member_idx, st_ops->name); |
| 12189 | return -EINVAL; |
| 12190 | } |
| 12191 | |
| 12192 | if (st_ops->check_member) { |
| 12193 | int err = st_ops->check_member(t, member); |
| 12194 | |
| 12195 | if (err) { |
| 12196 | verbose(env, "attach to unsupported member %s of struct %s\n", |
| 12197 | mname, st_ops->name); |
| 12198 | return err; |
| 12199 | } |
| 12200 | } |
| 12201 | |
| 12202 | prog->aux->attach_func_proto = func_proto; |
| 12203 | prog->aux->attach_func_name = mname; |
| 12204 | env->ops = st_ops->verifier_ops; |
| 12205 | |
| 12206 | return 0; |
| 12207 | } |
| 12208 | #define SECURITY_PREFIX "security_" |
| 12209 | |
| 12210 | static int check_attach_modify_return(unsigned long addr, const char *func_name) |
| 12211 | { |
| 12212 | if (within_error_injection_list(addr) || |
| 12213 | !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1)) |
| 12214 | return 0; |
| 12215 | |
| 12216 | return -EINVAL; |
| 12217 | } |
| 12218 | |
| 12219 | /* list of non-sleepable functions that are otherwise on |
| 12220 | * ALLOW_ERROR_INJECTION list |
| 12221 | */ |
| 12222 | BTF_SET_START(btf_non_sleepable_error_inject) |
| 12223 | /* Three functions below can be called from sleepable and non-sleepable context. |
| 12224 | * Assume non-sleepable from bpf safety point of view. |
| 12225 | */ |
| 12226 | BTF_ID(func, __add_to_page_cache_locked) |
| 12227 | BTF_ID(func, should_fail_alloc_page) |
| 12228 | BTF_ID(func, should_failslab) |
| 12229 | BTF_SET_END(btf_non_sleepable_error_inject) |
| 12230 | |
| 12231 | static int check_non_sleepable_error_inject(u32 btf_id) |
| 12232 | { |
| 12233 | return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id); |
| 12234 | } |
| 12235 | |
| 12236 | int bpf_check_attach_target(struct bpf_verifier_log *log, |
| 12237 | const struct bpf_prog *prog, |
| 12238 | const struct bpf_prog *tgt_prog, |
| 12239 | u32 btf_id, |
| 12240 | struct bpf_attach_target_info *tgt_info) |
| 12241 | { |
| 12242 | bool prog_extension = prog->type == BPF_PROG_TYPE_EXT; |
| 12243 | const char prefix[] = "btf_trace_"; |
| 12244 | int ret = 0, subprog = -1, i; |
| 12245 | const struct btf_type *t; |
| 12246 | bool conservative = true; |
| 12247 | const char *tname; |
| 12248 | struct btf *btf; |
| 12249 | long addr = 0; |
| 12250 | |
| 12251 | if (!btf_id) { |
| 12252 | bpf_log(log, "Tracing programs must provide btf_id\n"); |
| 12253 | return -EINVAL; |
| 12254 | } |
| 12255 | btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf; |
| 12256 | if (!btf) { |
| 12257 | bpf_log(log, |
| 12258 | "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n"); |
| 12259 | return -EINVAL; |
| 12260 | } |
| 12261 | t = btf_type_by_id(btf, btf_id); |
| 12262 | if (!t) { |
| 12263 | bpf_log(log, "attach_btf_id %u is invalid\n", btf_id); |
| 12264 | return -EINVAL; |
| 12265 | } |
| 12266 | tname = btf_name_by_offset(btf, t->name_off); |
| 12267 | if (!tname) { |
| 12268 | bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id); |
| 12269 | return -EINVAL; |
| 12270 | } |
| 12271 | if (tgt_prog) { |
| 12272 | struct bpf_prog_aux *aux = tgt_prog->aux; |
| 12273 | |
| 12274 | for (i = 0; i < aux->func_info_cnt; i++) |
| 12275 | if (aux->func_info[i].type_id == btf_id) { |
| 12276 | subprog = i; |
| 12277 | break; |
| 12278 | } |
| 12279 | if (subprog == -1) { |
| 12280 | bpf_log(log, "Subprog %s doesn't exist\n", tname); |
| 12281 | return -EINVAL; |
| 12282 | } |
| 12283 | conservative = aux->func_info_aux[subprog].unreliable; |
| 12284 | if (prog_extension) { |
| 12285 | if (conservative) { |
| 12286 | bpf_log(log, |
| 12287 | "Cannot replace static functions\n"); |
| 12288 | return -EINVAL; |
| 12289 | } |
| 12290 | if (!prog->jit_requested) { |
| 12291 | bpf_log(log, |
| 12292 | "Extension programs should be JITed\n"); |
| 12293 | return -EINVAL; |
| 12294 | } |
| 12295 | } |
| 12296 | if (!tgt_prog->jited) { |
| 12297 | bpf_log(log, "Can attach to only JITed progs\n"); |
| 12298 | return -EINVAL; |
| 12299 | } |
| 12300 | if (tgt_prog->type == prog->type) { |
| 12301 | /* Cannot fentry/fexit another fentry/fexit program. |
| 12302 | * Cannot attach program extension to another extension. |
| 12303 | * It's ok to attach fentry/fexit to extension program. |
| 12304 | */ |
| 12305 | bpf_log(log, "Cannot recursively attach\n"); |
| 12306 | return -EINVAL; |
| 12307 | } |
| 12308 | if (tgt_prog->type == BPF_PROG_TYPE_TRACING && |
| 12309 | prog_extension && |
| 12310 | (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY || |
| 12311 | tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) { |
| 12312 | /* Program extensions can extend all program types |
| 12313 | * except fentry/fexit. The reason is the following. |
| 12314 | * The fentry/fexit programs are used for performance |
| 12315 | * analysis, stats and can be attached to any program |
| 12316 | * type except themselves. When extension program is |
| 12317 | * replacing XDP function it is necessary to allow |
| 12318 | * performance analysis of all functions. Both original |
| 12319 | * XDP program and its program extension. Hence |
| 12320 | * attaching fentry/fexit to BPF_PROG_TYPE_EXT is |
| 12321 | * allowed. If extending of fentry/fexit was allowed it |
| 12322 | * would be possible to create long call chain |
| 12323 | * fentry->extension->fentry->extension beyond |
| 12324 | * reasonable stack size. Hence extending fentry is not |
| 12325 | * allowed. |
| 12326 | */ |
| 12327 | bpf_log(log, "Cannot extend fentry/fexit\n"); |
| 12328 | return -EINVAL; |
| 12329 | } |
| 12330 | } else { |
| 12331 | if (prog_extension) { |
| 12332 | bpf_log(log, "Cannot replace kernel functions\n"); |
| 12333 | return -EINVAL; |
| 12334 | } |
| 12335 | } |
| 12336 | |
| 12337 | switch (prog->expected_attach_type) { |
| 12338 | case BPF_TRACE_RAW_TP: |
| 12339 | if (tgt_prog) { |
| 12340 | bpf_log(log, |
| 12341 | "Only FENTRY/FEXIT progs are attachable to another BPF prog\n"); |
| 12342 | return -EINVAL; |
| 12343 | } |
| 12344 | if (!btf_type_is_typedef(t)) { |
| 12345 | bpf_log(log, "attach_btf_id %u is not a typedef\n", |
| 12346 | btf_id); |
| 12347 | return -EINVAL; |
| 12348 | } |
| 12349 | if (strncmp(prefix, tname, sizeof(prefix) - 1)) { |
| 12350 | bpf_log(log, "attach_btf_id %u points to wrong type name %s\n", |
| 12351 | btf_id, tname); |
| 12352 | return -EINVAL; |
| 12353 | } |
| 12354 | tname += sizeof(prefix) - 1; |
| 12355 | t = btf_type_by_id(btf, t->type); |
| 12356 | if (!btf_type_is_ptr(t)) |
| 12357 | /* should never happen in valid vmlinux build */ |
| 12358 | return -EINVAL; |
| 12359 | t = btf_type_by_id(btf, t->type); |
| 12360 | if (!btf_type_is_func_proto(t)) |
| 12361 | /* should never happen in valid vmlinux build */ |
| 12362 | return -EINVAL; |
| 12363 | |
| 12364 | break; |
| 12365 | case BPF_TRACE_ITER: |
| 12366 | if (!btf_type_is_func(t)) { |
| 12367 | bpf_log(log, "attach_btf_id %u is not a function\n", |
| 12368 | btf_id); |
| 12369 | return -EINVAL; |
| 12370 | } |
| 12371 | t = btf_type_by_id(btf, t->type); |
| 12372 | if (!btf_type_is_func_proto(t)) |
| 12373 | return -EINVAL; |
| 12374 | ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); |
| 12375 | if (ret) |
| 12376 | return ret; |
| 12377 | break; |
| 12378 | default: |
| 12379 | if (!prog_extension) |
| 12380 | return -EINVAL; |
| 12381 | fallthrough; |
| 12382 | case BPF_MODIFY_RETURN: |
| 12383 | case BPF_LSM_MAC: |
| 12384 | case BPF_TRACE_FENTRY: |
| 12385 | case BPF_TRACE_FEXIT: |
| 12386 | if (!btf_type_is_func(t)) { |
| 12387 | bpf_log(log, "attach_btf_id %u is not a function\n", |
| 12388 | btf_id); |
| 12389 | return -EINVAL; |
| 12390 | } |
| 12391 | if (prog_extension && |
| 12392 | btf_check_type_match(log, prog, btf, t)) |
| 12393 | return -EINVAL; |
| 12394 | t = btf_type_by_id(btf, t->type); |
| 12395 | if (!btf_type_is_func_proto(t)) |
| 12396 | return -EINVAL; |
| 12397 | |
| 12398 | if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) && |
| 12399 | (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type || |
| 12400 | prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) |
| 12401 | return -EINVAL; |
| 12402 | |
| 12403 | if (tgt_prog && conservative) |
| 12404 | t = NULL; |
| 12405 | |
| 12406 | ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); |
| 12407 | if (ret < 0) |
| 12408 | return ret; |
| 12409 | |
| 12410 | if (tgt_prog) { |
| 12411 | if (subprog == 0) |
| 12412 | addr = (long) tgt_prog->bpf_func; |
| 12413 | else |
| 12414 | addr = (long) tgt_prog->aux->func[subprog]->bpf_func; |
| 12415 | } else { |
| 12416 | addr = kallsyms_lookup_name(tname); |
| 12417 | if (!addr) { |
| 12418 | bpf_log(log, |
| 12419 | "The address of function %s cannot be found\n", |
| 12420 | tname); |
| 12421 | return -ENOENT; |
| 12422 | } |
| 12423 | } |
| 12424 | |
| 12425 | if (prog->aux->sleepable) { |
| 12426 | ret = -EINVAL; |
| 12427 | switch (prog->type) { |
| 12428 | case BPF_PROG_TYPE_TRACING: |
| 12429 | /* fentry/fexit/fmod_ret progs can be sleepable only if they are |
| 12430 | * attached to ALLOW_ERROR_INJECTION and are not in denylist. |
| 12431 | */ |
| 12432 | if (!check_non_sleepable_error_inject(btf_id) && |
| 12433 | within_error_injection_list(addr)) |
| 12434 | ret = 0; |
| 12435 | break; |
| 12436 | case BPF_PROG_TYPE_LSM: |
| 12437 | /* LSM progs check that they are attached to bpf_lsm_*() funcs. |
| 12438 | * Only some of them are sleepable. |
| 12439 | */ |
| 12440 | if (bpf_lsm_is_sleepable_hook(btf_id)) |
| 12441 | ret = 0; |
| 12442 | break; |
| 12443 | default: |
| 12444 | break; |
| 12445 | } |
| 12446 | if (ret) { |
| 12447 | bpf_log(log, "%s is not sleepable\n", tname); |
| 12448 | return ret; |
| 12449 | } |
| 12450 | } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) { |
| 12451 | if (tgt_prog) { |
| 12452 | bpf_log(log, "can't modify return codes of BPF programs\n"); |
| 12453 | return -EINVAL; |
| 12454 | } |
| 12455 | ret = check_attach_modify_return(addr, tname); |
| 12456 | if (ret) { |
| 12457 | bpf_log(log, "%s() is not modifiable\n", tname); |
| 12458 | return ret; |
| 12459 | } |
| 12460 | } |
| 12461 | |
| 12462 | break; |
| 12463 | } |
| 12464 | tgt_info->tgt_addr = addr; |
| 12465 | tgt_info->tgt_name = tname; |
| 12466 | tgt_info->tgt_type = t; |
| 12467 | return 0; |
| 12468 | } |
| 12469 | |
| 12470 | static int check_attach_btf_id(struct bpf_verifier_env *env) |
| 12471 | { |
| 12472 | struct bpf_prog *prog = env->prog; |
| 12473 | struct bpf_prog *tgt_prog = prog->aux->dst_prog; |
| 12474 | struct bpf_attach_target_info tgt_info = {}; |
| 12475 | u32 btf_id = prog->aux->attach_btf_id; |
| 12476 | struct bpf_trampoline *tr; |
| 12477 | int ret; |
| 12478 | u64 key; |
| 12479 | |
| 12480 | if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && |
| 12481 | prog->type != BPF_PROG_TYPE_LSM) { |
| 12482 | verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n"); |
| 12483 | return -EINVAL; |
| 12484 | } |
| 12485 | |
| 12486 | if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) |
| 12487 | return check_struct_ops_btf_id(env); |
| 12488 | |
| 12489 | if (prog->type != BPF_PROG_TYPE_TRACING && |
| 12490 | prog->type != BPF_PROG_TYPE_LSM && |
| 12491 | prog->type != BPF_PROG_TYPE_EXT) |
| 12492 | return 0; |
| 12493 | |
| 12494 | ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info); |
| 12495 | if (ret) |
| 12496 | return ret; |
| 12497 | |
| 12498 | if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) { |
| 12499 | /* to make freplace equivalent to their targets, they need to |
| 12500 | * inherit env->ops and expected_attach_type for the rest of the |
| 12501 | * verification |
| 12502 | */ |
| 12503 | env->ops = bpf_verifier_ops[tgt_prog->type]; |
| 12504 | prog->expected_attach_type = tgt_prog->expected_attach_type; |
| 12505 | } |
| 12506 | |
| 12507 | /* store info about the attachment target that will be used later */ |
| 12508 | prog->aux->attach_func_proto = tgt_info.tgt_type; |
| 12509 | prog->aux->attach_func_name = tgt_info.tgt_name; |
| 12510 | |
| 12511 | if (tgt_prog) { |
| 12512 | prog->aux->saved_dst_prog_type = tgt_prog->type; |
| 12513 | prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type; |
| 12514 | } |
| 12515 | |
| 12516 | if (prog->expected_attach_type == BPF_TRACE_RAW_TP) { |
| 12517 | prog->aux->attach_btf_trace = true; |
| 12518 | return 0; |
| 12519 | } else if (prog->expected_attach_type == BPF_TRACE_ITER) { |
| 12520 | if (!bpf_iter_prog_supported(prog)) |
| 12521 | return -EINVAL; |
| 12522 | return 0; |
| 12523 | } |
| 12524 | |
| 12525 | if (prog->type == BPF_PROG_TYPE_LSM) { |
| 12526 | ret = bpf_lsm_verify_prog(&env->log, prog); |
| 12527 | if (ret < 0) |
| 12528 | return ret; |
| 12529 | } |
| 12530 | |
| 12531 | key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id); |
| 12532 | tr = bpf_trampoline_get(key, &tgt_info); |
| 12533 | if (!tr) |
| 12534 | return -ENOMEM; |
| 12535 | |
| 12536 | prog->aux->dst_trampoline = tr; |
| 12537 | return 0; |
| 12538 | } |
| 12539 | |
| 12540 | struct btf *bpf_get_btf_vmlinux(void) |
| 12541 | { |
| 12542 | if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { |
| 12543 | mutex_lock(&bpf_verifier_lock); |
| 12544 | if (!btf_vmlinux) |
| 12545 | btf_vmlinux = btf_parse_vmlinux(); |
| 12546 | mutex_unlock(&bpf_verifier_lock); |
| 12547 | } |
| 12548 | return btf_vmlinux; |
| 12549 | } |
| 12550 | |
| 12551 | int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, |
| 12552 | union bpf_attr __user *uattr) |
| 12553 | { |
| 12554 | u64 start_time = ktime_get_ns(); |
| 12555 | struct bpf_verifier_env *env; |
| 12556 | struct bpf_verifier_log *log; |
| 12557 | int i, len, ret = -EINVAL; |
| 12558 | bool is_priv; |
| 12559 | |
| 12560 | /* no program is valid */ |
| 12561 | if (ARRAY_SIZE(bpf_verifier_ops) == 0) |
| 12562 | return -EINVAL; |
| 12563 | |
| 12564 | /* 'struct bpf_verifier_env' can be global, but since it's not small, |
| 12565 | * allocate/free it every time bpf_check() is called |
| 12566 | */ |
| 12567 | env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); |
| 12568 | if (!env) |
| 12569 | return -ENOMEM; |
| 12570 | log = &env->log; |
| 12571 | |
| 12572 | len = (*prog)->len; |
| 12573 | env->insn_aux_data = |
| 12574 | vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); |
| 12575 | ret = -ENOMEM; |
| 12576 | if (!env->insn_aux_data) |
| 12577 | goto err_free_env; |
| 12578 | for (i = 0; i < len; i++) |
| 12579 | env->insn_aux_data[i].orig_idx = i; |
| 12580 | env->prog = *prog; |
| 12581 | env->ops = bpf_verifier_ops[env->prog->type]; |
| 12582 | is_priv = bpf_capable(); |
| 12583 | |
| 12584 | bpf_get_btf_vmlinux(); |
| 12585 | |
| 12586 | /* grab the mutex to protect few globals used by verifier */ |
| 12587 | if (!is_priv) |
| 12588 | mutex_lock(&bpf_verifier_lock); |
| 12589 | |
| 12590 | if (attr->log_level || attr->log_buf || attr->log_size) { |
| 12591 | /* user requested verbose verifier output |
| 12592 | * and supplied buffer to store the verification trace |
| 12593 | */ |
| 12594 | log->level = attr->log_level; |
| 12595 | log->ubuf = (char __user *) (unsigned long) attr->log_buf; |
| 12596 | log->len_total = attr->log_size; |
| 12597 | |
| 12598 | ret = -EINVAL; |
| 12599 | /* log attributes have to be sane */ |
| 12600 | if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || |
| 12601 | !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) |
| 12602 | goto err_unlock; |
| 12603 | } |
| 12604 | |
| 12605 | if (IS_ERR(btf_vmlinux)) { |
| 12606 | /* Either gcc or pahole or kernel are broken. */ |
| 12607 | verbose(env, "in-kernel BTF is malformed\n"); |
| 12608 | ret = PTR_ERR(btf_vmlinux); |
| 12609 | goto skip_full_check; |
| 12610 | } |
| 12611 | |
| 12612 | env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); |
| 12613 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) |
| 12614 | env->strict_alignment = true; |
| 12615 | if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) |
| 12616 | env->strict_alignment = false; |
| 12617 | |
| 12618 | env->allow_ptr_leaks = bpf_allow_ptr_leaks(); |
| 12619 | env->allow_uninit_stack = bpf_allow_uninit_stack(); |
| 12620 | env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access(); |
| 12621 | env->bypass_spec_v1 = bpf_bypass_spec_v1(); |
| 12622 | env->bypass_spec_v4 = bpf_bypass_spec_v4(); |
| 12623 | env->bpf_capable = bpf_capable(); |
| 12624 | |
| 12625 | if (is_priv) |
| 12626 | env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; |
| 12627 | |
| 12628 | if (bpf_prog_is_dev_bound(env->prog->aux)) { |
| 12629 | ret = bpf_prog_offload_verifier_prep(env->prog); |
| 12630 | if (ret) |
| 12631 | goto skip_full_check; |
| 12632 | } |
| 12633 | |
| 12634 | env->explored_states = kvcalloc(state_htab_size(env), |
| 12635 | sizeof(struct bpf_verifier_state_list *), |
| 12636 | GFP_USER); |
| 12637 | ret = -ENOMEM; |
| 12638 | if (!env->explored_states) |
| 12639 | goto skip_full_check; |
| 12640 | |
| 12641 | ret = check_subprogs(env); |
| 12642 | if (ret < 0) |
| 12643 | goto skip_full_check; |
| 12644 | |
| 12645 | ret = check_btf_info(env, attr, uattr); |
| 12646 | if (ret < 0) |
| 12647 | goto skip_full_check; |
| 12648 | |
| 12649 | ret = check_attach_btf_id(env); |
| 12650 | if (ret) |
| 12651 | goto skip_full_check; |
| 12652 | |
| 12653 | ret = resolve_pseudo_ldimm64(env); |
| 12654 | if (ret < 0) |
| 12655 | goto skip_full_check; |
| 12656 | |
| 12657 | ret = check_cfg(env); |
| 12658 | if (ret < 0) |
| 12659 | goto skip_full_check; |
| 12660 | |
| 12661 | ret = do_check_subprogs(env); |
| 12662 | ret = ret ?: do_check_main(env); |
| 12663 | |
| 12664 | if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) |
| 12665 | ret = bpf_prog_offload_finalize(env); |
| 12666 | |
| 12667 | skip_full_check: |
| 12668 | kvfree(env->explored_states); |
| 12669 | |
| 12670 | if (ret == 0) |
| 12671 | ret = check_max_stack_depth(env); |
| 12672 | |
| 12673 | /* instruction rewrites happen after this point */ |
| 12674 | if (is_priv) { |
| 12675 | if (ret == 0) |
| 12676 | opt_hard_wire_dead_code_branches(env); |
| 12677 | if (ret == 0) |
| 12678 | ret = opt_remove_dead_code(env); |
| 12679 | if (ret == 0) |
| 12680 | ret = opt_remove_nops(env); |
| 12681 | } else { |
| 12682 | if (ret == 0) |
| 12683 | sanitize_dead_code(env); |
| 12684 | } |
| 12685 | |
| 12686 | if (ret == 0) |
| 12687 | /* program is valid, convert *(u32*)(ctx + off) accesses */ |
| 12688 | ret = convert_ctx_accesses(env); |
| 12689 | |
| 12690 | if (ret == 0) |
| 12691 | ret = fixup_bpf_calls(env); |
| 12692 | |
| 12693 | /* do 32-bit optimization after insn patching has done so those patched |
| 12694 | * insns could be handled correctly. |
| 12695 | */ |
| 12696 | if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) { |
| 12697 | ret = opt_subreg_zext_lo32_rnd_hi32(env, attr); |
| 12698 | env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret |
| 12699 | : false; |
| 12700 | } |
| 12701 | |
| 12702 | if (ret == 0) |
| 12703 | ret = fixup_call_args(env); |
| 12704 | |
| 12705 | env->verification_time = ktime_get_ns() - start_time; |
| 12706 | print_verification_stats(env); |
| 12707 | |
| 12708 | if (log->level && bpf_verifier_log_full(log)) |
| 12709 | ret = -ENOSPC; |
| 12710 | if (log->level && !log->ubuf) { |
| 12711 | ret = -EFAULT; |
| 12712 | goto err_release_maps; |
| 12713 | } |
| 12714 | |
| 12715 | if (ret) |
| 12716 | goto err_release_maps; |
| 12717 | |
| 12718 | if (env->used_map_cnt) { |
| 12719 | /* if program passed verifier, update used_maps in bpf_prog_info */ |
| 12720 | env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, |
| 12721 | sizeof(env->used_maps[0]), |
| 12722 | GFP_KERNEL); |
| 12723 | |
| 12724 | if (!env->prog->aux->used_maps) { |
| 12725 | ret = -ENOMEM; |
| 12726 | goto err_release_maps; |
| 12727 | } |
| 12728 | |
| 12729 | memcpy(env->prog->aux->used_maps, env->used_maps, |
| 12730 | sizeof(env->used_maps[0]) * env->used_map_cnt); |
| 12731 | env->prog->aux->used_map_cnt = env->used_map_cnt; |
| 12732 | } |
| 12733 | if (env->used_btf_cnt) { |
| 12734 | /* if program passed verifier, update used_btfs in bpf_prog_aux */ |
| 12735 | env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt, |
| 12736 | sizeof(env->used_btfs[0]), |
| 12737 | GFP_KERNEL); |
| 12738 | if (!env->prog->aux->used_btfs) { |
| 12739 | ret = -ENOMEM; |
| 12740 | goto err_release_maps; |
| 12741 | } |
| 12742 | |
| 12743 | memcpy(env->prog->aux->used_btfs, env->used_btfs, |
| 12744 | sizeof(env->used_btfs[0]) * env->used_btf_cnt); |
| 12745 | env->prog->aux->used_btf_cnt = env->used_btf_cnt; |
| 12746 | } |
| 12747 | if (env->used_map_cnt || env->used_btf_cnt) { |
| 12748 | /* program is valid. Convert pseudo bpf_ld_imm64 into generic |
| 12749 | * bpf_ld_imm64 instructions |
| 12750 | */ |
| 12751 | convert_pseudo_ld_imm64(env); |
| 12752 | } |
| 12753 | |
| 12754 | adjust_btf_func(env); |
| 12755 | |
| 12756 | err_release_maps: |
| 12757 | if (!env->prog->aux->used_maps) |
| 12758 | /* if we didn't copy map pointers into bpf_prog_info, release |
| 12759 | * them now. Otherwise free_used_maps() will release them. |
| 12760 | */ |
| 12761 | release_maps(env); |
| 12762 | if (!env->prog->aux->used_btfs) |
| 12763 | release_btfs(env); |
| 12764 | |
| 12765 | /* extension progs temporarily inherit the attach_type of their targets |
| 12766 | for verification purposes, so set it back to zero before returning |
| 12767 | */ |
| 12768 | if (env->prog->type == BPF_PROG_TYPE_EXT) |
| 12769 | env->prog->expected_attach_type = 0; |
| 12770 | |
| 12771 | *prog = env->prog; |
| 12772 | err_unlock: |
| 12773 | if (!is_priv) |
| 12774 | mutex_unlock(&bpf_verifier_lock); |
| 12775 | vfree(env->insn_aux_data); |
| 12776 | err_free_env: |
| 12777 | kfree(env); |
| 12778 | return ret; |
| 12779 | } |