1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
7 #include <uapi/linux/bpf.h>
8 #include <uapi/linux/filter.h>
10 #include <linux/workqueue.h>
11 #include <linux/file.h>
12 #include <linux/percpu.h>
13 #include <linux/err.h>
14 #include <linux/rbtree_latch.h>
15 #include <linux/numa.h>
16 #include <linux/mm_types.h>
17 #include <linux/wait.h>
18 #include <linux/refcount.h>
19 #include <linux/mutex.h>
20 #include <linux/module.h>
21 #include <linux/kallsyms.h>
22 #include <linux/capability.h>
23 #include <linux/sched/mm.h>
24 #include <linux/slab.h>
25 #include <linux/percpu-refcount.h>
26 #include <linux/stddef.h>
27 #include <linux/bpfptr.h>
28 #include <linux/btf.h>
29 #include <linux/rcupdate_trace.h>
30 #include <linux/static_call.h>
31 #include <linux/memcontrol.h>
32 #include <linux/cfi.h>
34 struct bpf_verifier_env;
35 struct bpf_verifier_log;
45 struct exception_table_entry;
46 struct seq_operations;
47 struct bpf_iter_aux_info;
48 struct bpf_local_storage;
49 struct bpf_local_storage_map;
53 struct bpf_func_state;
57 struct user_namespace;
61 extern struct idr btf_idr;
62 extern spinlock_t btf_idr_lock;
63 extern struct kobject *btf_kobj;
64 extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma;
65 extern bool bpf_global_ma_set;
67 typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64);
68 typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
69 struct bpf_iter_aux_info *aux);
70 typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data);
71 typedef unsigned int (*bpf_func_t)(const void *,
72 const struct bpf_insn *);
73 struct bpf_iter_seq_info {
74 const struct seq_operations *seq_ops;
75 bpf_iter_init_seq_priv_t init_seq_private;
76 bpf_iter_fini_seq_priv_t fini_seq_private;
80 /* map is generic key/value storage optionally accessible by eBPF programs */
82 /* funcs callable from userspace (via syscall) */
83 int (*map_alloc_check)(union bpf_attr *attr);
84 struct bpf_map *(*map_alloc)(union bpf_attr *attr);
85 void (*map_release)(struct bpf_map *map, struct file *map_file);
86 void (*map_free)(struct bpf_map *map);
87 int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key);
88 void (*map_release_uref)(struct bpf_map *map);
89 void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key);
90 int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr,
91 union bpf_attr __user *uattr);
92 int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key,
93 void *value, u64 flags);
94 int (*map_lookup_and_delete_batch)(struct bpf_map *map,
95 const union bpf_attr *attr,
96 union bpf_attr __user *uattr);
97 int (*map_update_batch)(struct bpf_map *map, struct file *map_file,
98 const union bpf_attr *attr,
99 union bpf_attr __user *uattr);
100 int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr,
101 union bpf_attr __user *uattr);
103 /* funcs callable from userspace and from eBPF programs */
104 void *(*map_lookup_elem)(struct bpf_map *map, void *key);
105 long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags);
106 long (*map_delete_elem)(struct bpf_map *map, void *key);
107 long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags);
108 long (*map_pop_elem)(struct bpf_map *map, void *value);
109 long (*map_peek_elem)(struct bpf_map *map, void *value);
110 void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu);
112 /* funcs called by prog_array and perf_event_array map */
113 void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
115 /* If need_defer is true, the implementation should guarantee that
116 * the to-be-put element is still alive before the bpf program, which
117 * may manipulate it, exists.
119 void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer);
120 int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
121 u32 (*map_fd_sys_lookup_elem)(void *ptr);
122 void (*map_seq_show_elem)(struct bpf_map *map, void *key,
124 int (*map_check_btf)(const struct bpf_map *map,
125 const struct btf *btf,
126 const struct btf_type *key_type,
127 const struct btf_type *value_type);
129 /* Prog poke tracking helpers. */
130 int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux);
131 void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux);
132 void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old,
133 struct bpf_prog *new);
135 /* Direct value access helpers. */
136 int (*map_direct_value_addr)(const struct bpf_map *map,
138 int (*map_direct_value_meta)(const struct bpf_map *map,
140 int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma);
141 __poll_t (*map_poll)(struct bpf_map *map, struct file *filp,
142 struct poll_table_struct *pts);
143 unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr,
144 unsigned long len, unsigned long pgoff,
145 unsigned long flags);
147 /* Functions called by bpf_local_storage maps */
148 int (*map_local_storage_charge)(struct bpf_local_storage_map *smap,
149 void *owner, u32 size);
150 void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap,
151 void *owner, u32 size);
152 struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner);
155 long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags);
157 /* map_meta_equal must be implemented for maps that can be
158 * used as an inner map. It is a runtime check to ensure
159 * an inner map can be inserted to an outer map.
161 * Some properties of the inner map has been used during the
162 * verification time. When inserting an inner map at the runtime,
163 * map_meta_equal has to ensure the inserting map has the same
164 * properties that the verifier has used earlier.
166 bool (*map_meta_equal)(const struct bpf_map *meta0,
167 const struct bpf_map *meta1);
170 int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env,
171 struct bpf_func_state *caller,
172 struct bpf_func_state *callee);
173 long (*map_for_each_callback)(struct bpf_map *map,
174 bpf_callback_t callback_fn,
175 void *callback_ctx, u64 flags);
177 u64 (*map_mem_usage)(const struct bpf_map *map);
179 /* BTF id of struct allocated by map_alloc */
182 /* bpf_iter info used to open a seq_file */
183 const struct bpf_iter_seq_info *iter_seq_info;
187 /* Support at most 10 fields in a BTF type */
191 enum btf_field_type {
192 BPF_SPIN_LOCK = (1 << 0),
193 BPF_TIMER = (1 << 1),
194 BPF_KPTR_UNREF = (1 << 2),
195 BPF_KPTR_REF = (1 << 3),
196 BPF_KPTR_PERCPU = (1 << 4),
197 BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU,
198 BPF_LIST_HEAD = (1 << 5),
199 BPF_LIST_NODE = (1 << 6),
200 BPF_RB_ROOT = (1 << 7),
201 BPF_RB_NODE = (1 << 8),
202 BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE,
203 BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
204 BPF_REFCOUNT = (1 << 9),
207 typedef void (*btf_dtor_kfunc_t)(void *);
209 struct btf_field_kptr {
211 struct module *module;
212 /* dtor used if btf_is_kernel(btf), otherwise the type is
213 * program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used
215 btf_dtor_kfunc_t dtor;
219 struct btf_field_graph_root {
223 struct btf_record *value_rec;
229 enum btf_field_type type;
231 struct btf_field_kptr kptr;
232 struct btf_field_graph_root graph_root;
242 struct btf_field fields[];
245 /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */
246 struct bpf_rb_node_kern {
247 struct rb_node rb_node;
249 } __attribute__((aligned(8)));
251 /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */
252 struct bpf_list_node_kern {
253 struct list_head list_head;
255 } __attribute__((aligned(8)));
258 const struct bpf_map_ops *ops;
259 struct bpf_map *inner_map_meta;
260 #ifdef CONFIG_SECURITY
263 enum bpf_map_type map_type;
267 u64 map_extra; /* any per-map-type extra fields */
270 struct btf_record *record;
273 u32 btf_value_type_id;
274 u32 btf_vmlinux_value_type_id;
276 #ifdef CONFIG_MEMCG_KMEM
277 struct obj_cgroup *objcg;
279 char name[BPF_OBJ_NAME_LEN];
280 struct mutex freeze_mutex;
283 /* rcu is used before freeing and work is only used during freeing */
285 struct work_struct work;
289 /* 'Ownership' of program-containing map is claimed by the first program
290 * that is going to use this map or by the first program which FD is
291 * stored in the map to make sure that all callers and callees have the
292 * same prog type, JITed flag and xdp_has_frags flag.
296 enum bpf_prog_type type;
301 bool frozen; /* write-once; write-protected by freeze_mutex */
302 bool free_after_mult_rcu_gp;
303 bool free_after_rcu_gp;
304 atomic64_t sleepable_refcnt;
305 s64 __percpu *elem_count;
308 static inline const char *btf_field_type_name(enum btf_field_type type)
312 return "bpf_spin_lock";
318 case BPF_KPTR_PERCPU:
319 return "percpu_kptr";
321 return "bpf_list_head";
323 return "bpf_list_node";
325 return "bpf_rb_root";
327 return "bpf_rb_node";
329 return "bpf_refcount";
336 static inline u32 btf_field_type_size(enum btf_field_type type)
340 return sizeof(struct bpf_spin_lock);
342 return sizeof(struct bpf_timer);
345 case BPF_KPTR_PERCPU:
348 return sizeof(struct bpf_list_head);
350 return sizeof(struct bpf_list_node);
352 return sizeof(struct bpf_rb_root);
354 return sizeof(struct bpf_rb_node);
356 return sizeof(struct bpf_refcount);
363 static inline u32 btf_field_type_align(enum btf_field_type type)
367 return __alignof__(struct bpf_spin_lock);
369 return __alignof__(struct bpf_timer);
372 case BPF_KPTR_PERCPU:
373 return __alignof__(u64);
375 return __alignof__(struct bpf_list_head);
377 return __alignof__(struct bpf_list_node);
379 return __alignof__(struct bpf_rb_root);
381 return __alignof__(struct bpf_rb_node);
383 return __alignof__(struct bpf_refcount);
390 static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
392 memset(addr, 0, field->size);
394 switch (field->type) {
396 refcount_set((refcount_t *)addr, 1);
399 RB_CLEAR_NODE((struct rb_node *)addr);
403 INIT_LIST_HEAD((struct list_head *)addr);
406 /* RB_ROOT_CACHED 0-inits, no need to do anything after memset */
411 case BPF_KPTR_PERCPU:
419 static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type)
421 if (IS_ERR_OR_NULL(rec))
423 return rec->field_mask & type;
426 static inline void bpf_obj_init(const struct btf_record *rec, void *obj)
430 if (IS_ERR_OR_NULL(rec))
432 for (i = 0; i < rec->cnt; i++)
433 bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset);
436 /* 'dst' must be a temporary buffer and should not point to memory that is being
437 * used in parallel by a bpf program or bpf syscall, otherwise the access from
438 * the bpf program or bpf syscall may be corrupted by the reinitialization,
439 * leading to weird problems. Even 'dst' is newly-allocated from bpf memory
440 * allocator, it is still possible for 'dst' to be used in parallel by a bpf
441 * program or bpf syscall.
443 static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
445 bpf_obj_init(map->record, dst);
448 /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
449 * forced to use 'long' read/writes to try to atomically copy long counters.
450 * Best-effort only. No barriers here, since it _will_ race with concurrent
451 * updates from BPF programs. Called from bpf syscall and mostly used with
452 * size 8 or 16 bytes, so ask compiler to inline it.
454 static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
456 const long *lsrc = src;
459 size /= sizeof(long);
461 data_race(*ldst++ = *lsrc++);
464 /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
465 static inline void bpf_obj_memcpy(struct btf_record *rec,
466 void *dst, void *src, u32 size,
472 if (IS_ERR_OR_NULL(rec)) {
474 bpf_long_memcpy(dst, src, round_up(size, 8));
476 memcpy(dst, src, size);
480 for (i = 0; i < rec->cnt; i++) {
481 u32 next_off = rec->fields[i].offset;
482 u32 sz = next_off - curr_off;
484 memcpy(dst + curr_off, src + curr_off, sz);
485 curr_off += rec->fields[i].size + sz;
487 memcpy(dst + curr_off, src + curr_off, size - curr_off);
490 static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
492 bpf_obj_memcpy(map->record, dst, src, map->value_size, false);
495 static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
497 bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
500 static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
505 if (IS_ERR_OR_NULL(rec)) {
506 memset(dst, 0, size);
510 for (i = 0; i < rec->cnt; i++) {
511 u32 next_off = rec->fields[i].offset;
512 u32 sz = next_off - curr_off;
514 memset(dst + curr_off, 0, sz);
515 curr_off += rec->fields[i].size + sz;
517 memset(dst + curr_off, 0, size - curr_off);
520 static inline void zero_map_value(struct bpf_map *map, void *dst)
522 bpf_obj_memzero(map->record, dst, map->value_size);
525 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
527 void bpf_timer_cancel_and_free(void *timer);
528 void bpf_list_head_free(const struct btf_field *field, void *list_head,
529 struct bpf_spin_lock *spin_lock);
530 void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
531 struct bpf_spin_lock *spin_lock);
532 u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena);
533 u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena);
534 int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size);
536 struct bpf_offload_dev;
537 struct bpf_offloaded_map;
539 struct bpf_map_dev_ops {
540 int (*map_get_next_key)(struct bpf_offloaded_map *map,
541 void *key, void *next_key);
542 int (*map_lookup_elem)(struct bpf_offloaded_map *map,
543 void *key, void *value);
544 int (*map_update_elem)(struct bpf_offloaded_map *map,
545 void *key, void *value, u64 flags);
546 int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key);
549 struct bpf_offloaded_map {
551 struct net_device *netdev;
552 const struct bpf_map_dev_ops *dev_ops;
554 struct list_head offloads;
557 static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map)
559 return container_of(map, struct bpf_offloaded_map, map);
562 static inline bool bpf_map_offload_neutral(const struct bpf_map *map)
564 return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
567 static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
569 return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) &&
570 map->ops->map_seq_show_elem;
573 int map_check_no_btf(const struct bpf_map *map,
574 const struct btf *btf,
575 const struct btf_type *key_type,
576 const struct btf_type *value_type);
578 bool bpf_map_meta_equal(const struct bpf_map *meta0,
579 const struct bpf_map *meta1);
581 extern const struct bpf_map_ops bpf_map_offload_ops;
583 /* bpf_type_flag contains a set of flags that are applicable to the values of
584 * arg_type, ret_type and reg_type. For example, a pointer value may be null,
585 * or a memory is read-only. We classify types into two categories: base types
586 * and extended types. Extended types are base types combined with a type flag.
588 * Currently there are no more than 32 base types in arg_type, ret_type and
591 #define BPF_BASE_TYPE_BITS 8
594 /* PTR may be NULL. */
595 PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS),
597 /* MEM is read-only. When applied on bpf_arg, it indicates the arg is
598 * compatible with both mutable and immutable memory.
600 MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS),
602 /* MEM points to BPF ring buffer reservation. */
603 MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS),
605 /* MEM is in user address space. */
606 MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS),
608 /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged
609 * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In
610 * order to drop this tag, it must be passed into bpf_per_cpu_ptr()
611 * or bpf_this_cpu_ptr(), which will return the pointer corresponding
612 * to the specified cpu.
614 MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS),
616 /* Indicates that the argument will be released. */
617 OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS),
619 /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark
620 * unreferenced and referenced kptr loaded from map value using a load
621 * instruction, so that they can only be dereferenced but not escape the
622 * BPF program into the kernel (i.e. cannot be passed as arguments to
623 * kfunc or bpf helpers).
625 PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS),
627 MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS),
629 /* DYNPTR points to memory local to the bpf program. */
630 DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS),
632 /* DYNPTR points to a kernel-produced ringbuf record. */
633 DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS),
635 /* Size is known at compile time. */
636 MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS),
638 /* MEM is of an allocated object of type in program BTF. This is used to
639 * tag PTR_TO_BTF_ID allocated using bpf_obj_new.
641 MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS),
643 /* PTR was passed from the kernel in a trusted context, and may be
644 * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions.
645 * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above.
646 * PTR_UNTRUSTED refers to a kptr that was read directly from a map
647 * without invoking bpf_kptr_xchg(). What we really need to know is
648 * whether a pointer is safe to pass to a kfunc or BPF helper function.
649 * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF
650 * helpers, they do not cover all possible instances of unsafe
651 * pointers. For example, a pointer that was obtained from walking a
652 * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the
653 * fact that it may be NULL, invalid, etc. This is due to backwards
654 * compatibility requirements, as this was the behavior that was first
655 * introduced when kptrs were added. The behavior is now considered
656 * deprecated, and PTR_UNTRUSTED will eventually be removed.
658 * PTR_TRUSTED, on the other hand, is a pointer that the kernel
659 * guarantees to be valid and safe to pass to kfuncs and BPF helpers.
660 * For example, pointers passed to tracepoint arguments are considered
661 * PTR_TRUSTED, as are pointers that are passed to struct_ops
662 * callbacks. As alluded to above, pointers that are obtained from
663 * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a
664 * struct task_struct *task is PTR_TRUSTED, then accessing
665 * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored
666 * in a BPF register. Similarly, pointers passed to certain programs
667 * types such as kretprobes are not guaranteed to be valid, as they may
668 * for example contain an object that was recently freed.
670 PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS),
672 /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */
673 MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS),
675 /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning.
676 * Currently only valid for linked-list and rbtree nodes. If the nodes
677 * have a bpf_refcount_field, they must be tagged MEM_RCU as well.
679 NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS),
681 /* DYNPTR points to sk_buff */
682 DYNPTR_TYPE_SKB = BIT(15 + BPF_BASE_TYPE_BITS),
684 /* DYNPTR points to xdp_buff */
685 DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS),
688 __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
691 #define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \
694 /* Max number of base types. */
695 #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS)
697 /* Max number of all types. */
698 #define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1))
700 /* function argument constraints */
702 ARG_DONTCARE = 0, /* unused argument in helper function */
704 /* the following constraints used to prototype
705 * bpf_map_lookup/update/delete_elem() functions
707 ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */
708 ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
709 ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
711 /* Used to prototype bpf_memcmp() and other functions that access data
712 * on eBPF program stack
714 ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */
717 ARG_CONST_SIZE, /* number of bytes accessed from memory */
718 ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */
720 ARG_PTR_TO_CTX, /* pointer to context */
721 ARG_ANYTHING, /* any (initialized) argument is ok */
722 ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
723 ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
724 ARG_PTR_TO_INT, /* pointer to int */
725 ARG_PTR_TO_LONG, /* pointer to long */
726 ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
727 ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */
728 ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */
729 ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
730 ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
731 ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
732 ARG_PTR_TO_FUNC, /* pointer to a bpf program function */
733 ARG_PTR_TO_STACK, /* pointer to stack */
734 ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */
735 ARG_PTR_TO_TIMER, /* pointer to bpf_timer */
736 ARG_PTR_TO_KPTR, /* pointer to referenced kptr */
737 ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */
740 /* Extended arg_types. */
741 ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE,
742 ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM,
743 ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX,
744 ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET,
745 ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK,
746 ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID,
747 /* pointer to memory does not need to be initialized, helper function must fill
748 * all bytes or clear them in error case.
750 ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | ARG_PTR_TO_MEM,
751 /* Pointer to valid memory of size known at compile time. */
752 ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM,
754 /* This must be the last entry. Its purpose is to ensure the enum is
755 * wide enough to hold the higher bits reserved for bpf_type_flag.
757 __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT,
759 static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
761 /* type of values returned from helper functions */
762 enum bpf_return_type {
763 RET_INTEGER, /* function returns integer */
764 RET_VOID, /* function doesn't return anything */
765 RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */
766 RET_PTR_TO_SOCKET, /* returns a pointer to a socket */
767 RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */
768 RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */
769 RET_PTR_TO_MEM, /* returns a pointer to memory */
770 RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
771 RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
774 /* Extended ret_types. */
775 RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE,
776 RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET,
777 RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK,
778 RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON,
779 RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM,
780 RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MEM,
781 RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID,
782 RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED | RET_PTR_TO_BTF_ID,
784 /* This must be the last entry. Its purpose is to ensure the enum is
785 * wide enough to hold the higher bits reserved for bpf_type_flag.
787 __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT,
789 static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
791 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
792 * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL
793 * instructions after verifying
795 struct bpf_func_proto {
796 u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
800 enum bpf_return_type ret_type;
803 enum bpf_arg_type arg1_type;
804 enum bpf_arg_type arg2_type;
805 enum bpf_arg_type arg3_type;
806 enum bpf_arg_type arg4_type;
807 enum bpf_arg_type arg5_type;
809 enum bpf_arg_type arg_type[5];
829 int *ret_btf_id; /* return value btf_id */
830 bool (*allowed)(const struct bpf_prog *prog);
833 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is
834 * the first argument to eBPF programs.
835 * For socket filters: 'struct bpf_context *' == 'struct sk_buff *'
839 enum bpf_access_type {
844 /* types of values stored in eBPF registers */
845 /* Pointer types represent:
848 * pointer + (u16) var
849 * pointer + (u16) var + imm
850 * if (range > 0) then [ptr, ptr + range - off) is safe to access
851 * if (id > 0) means that some 'var' was added
852 * if (off > 0) means that 'imm' was added
855 NOT_INIT = 0, /* nothing was written into register */
856 SCALAR_VALUE, /* reg doesn't contain a valid pointer */
857 PTR_TO_CTX, /* reg points to bpf_context */
858 CONST_PTR_TO_MAP, /* reg points to struct bpf_map */
859 PTR_TO_MAP_VALUE, /* reg points to map element value */
860 PTR_TO_MAP_KEY, /* reg points to a map element key */
861 PTR_TO_STACK, /* reg == frame_pointer + offset */
862 PTR_TO_PACKET_META, /* skb->data - meta_len */
863 PTR_TO_PACKET, /* reg points to skb->data */
864 PTR_TO_PACKET_END, /* skb->data + headlen */
865 PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */
866 PTR_TO_SOCKET, /* reg points to struct bpf_sock */
867 PTR_TO_SOCK_COMMON, /* reg points to sock_common */
868 PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
869 PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
870 PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
871 /* PTR_TO_BTF_ID points to a kernel struct that does not need
872 * to be null checked by the BPF program. This does not imply the
873 * pointer is _not_ null and in practice this can easily be a null
874 * pointer when reading pointer chains. The assumption is program
875 * context will handle null pointer dereference typically via fault
876 * handling. The verifier must keep this in mind and can make no
877 * assumptions about null or non-null when doing branch analysis.
878 * Further, when passed into helpers the helpers can not, without
879 * additional context, assume the value is non-null.
882 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
883 * been checked for null. Used primarily to inform the verifier
884 * an explicit null check is required for this struct.
886 PTR_TO_MEM, /* reg points to valid memory region */
888 PTR_TO_BUF, /* reg points to a read/write buffer */
889 PTR_TO_FUNC, /* reg points to a bpf program function */
890 CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */
893 /* Extended reg_types. */
894 PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE,
895 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
896 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
897 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
898 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
900 /* This must be the last entry. Its purpose is to ensure the enum is
901 * wide enough to hold the higher bits reserved for bpf_type_flag.
903 __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT,
905 static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT);
907 /* The information passed from prog-specific *_is_valid_access
908 * back to the verifier.
910 struct bpf_insn_access_aux {
911 enum bpf_reg_type reg_type;
919 struct bpf_verifier_log *log; /* for verbose logs */
923 bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size)
925 aux->ctx_field_size = size;
928 static bool bpf_is_ldimm64(const struct bpf_insn *insn)
930 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
933 static inline bool bpf_pseudo_func(const struct bpf_insn *insn)
935 return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
938 struct bpf_prog_ops {
939 int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
940 union bpf_attr __user *uattr);
943 struct bpf_reg_state;
944 struct bpf_verifier_ops {
945 /* return eBPF function prototype for verification */
946 const struct bpf_func_proto *
947 (*get_func_proto)(enum bpf_func_id func_id,
948 const struct bpf_prog *prog);
950 /* return true if 'size' wide access at offset 'off' within bpf_context
951 * with 'type' (read or write) is allowed
953 bool (*is_valid_access)(int off, int size, enum bpf_access_type type,
954 const struct bpf_prog *prog,
955 struct bpf_insn_access_aux *info);
956 int (*gen_prologue)(struct bpf_insn *insn, bool direct_write,
957 const struct bpf_prog *prog);
958 int (*gen_ld_abs)(const struct bpf_insn *orig,
959 struct bpf_insn *insn_buf);
960 u32 (*convert_ctx_access)(enum bpf_access_type type,
961 const struct bpf_insn *src,
962 struct bpf_insn *dst,
963 struct bpf_prog *prog, u32 *target_size);
964 int (*btf_struct_access)(struct bpf_verifier_log *log,
965 const struct bpf_reg_state *reg,
969 struct bpf_prog_offload_ops {
970 /* verifier basic callbacks */
971 int (*insn_hook)(struct bpf_verifier_env *env,
972 int insn_idx, int prev_insn_idx);
973 int (*finalize)(struct bpf_verifier_env *env);
974 /* verifier optimization callbacks (called after .finalize) */
975 int (*replace_insn)(struct bpf_verifier_env *env, u32 off,
976 struct bpf_insn *insn);
977 int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt);
978 /* program management callbacks */
979 int (*prepare)(struct bpf_prog *prog);
980 int (*translate)(struct bpf_prog *prog);
981 void (*destroy)(struct bpf_prog *prog);
984 struct bpf_prog_offload {
985 struct bpf_prog *prog;
986 struct net_device *netdev;
987 struct bpf_offload_dev *offdev;
989 struct list_head offloads;
996 enum bpf_cgroup_storage_type {
997 BPF_CGROUP_STORAGE_SHARED,
998 BPF_CGROUP_STORAGE_PERCPU,
999 __BPF_CGROUP_STORAGE_MAX
1002 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
1004 /* The longest tracepoint has 12 args.
1005 * See include/trace/bpf_probe.h
1007 #define MAX_BPF_FUNC_ARGS 12
1009 /* The maximum number of arguments passed through registers
1010 * a single function may have.
1012 #define MAX_BPF_FUNC_REG_ARGS 5
1014 /* The argument is a structure. */
1015 #define BTF_FMODEL_STRUCT_ARG BIT(0)
1017 /* The argument is signed. */
1018 #define BTF_FMODEL_SIGNED_ARG BIT(1)
1020 struct btf_func_model {
1024 u8 arg_size[MAX_BPF_FUNC_ARGS];
1025 u8 arg_flags[MAX_BPF_FUNC_ARGS];
1028 /* Restore arguments before returning from trampoline to let original function
1029 * continue executing. This flag is used for fentry progs when there are no
1032 #define BPF_TRAMP_F_RESTORE_REGS BIT(0)
1033 /* Call original function after fentry progs, but before fexit progs.
1034 * Makes sense for fentry/fexit, normal calls and indirect calls.
1036 #define BPF_TRAMP_F_CALL_ORIG BIT(1)
1037 /* Skip current frame and return to parent. Makes sense for fentry/fexit
1038 * programs only. Should not be used with normal calls and indirect calls.
1040 #define BPF_TRAMP_F_SKIP_FRAME BIT(2)
1041 /* Store IP address of the caller on the trampoline stack,
1042 * so it's available for trampoline's programs.
1044 #define BPF_TRAMP_F_IP_ARG BIT(3)
1045 /* Return the return value of fentry prog. Only used by bpf_struct_ops. */
1046 #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4)
1048 /* Get original function from stack instead of from provided direct address.
1049 * Makes sense for trampolines with fexit or fmod_ret programs.
1051 #define BPF_TRAMP_F_ORIG_STACK BIT(5)
1053 /* This trampoline is on a function with another ftrace_ops with IPMODIFY,
1054 * e.g., a live patch. This flag is set and cleared by ftrace call backs,
1056 #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6)
1058 /* Indicate that current trampoline is in a tail call context. Then, it has to
1059 * cache and restore tail_call_cnt to avoid infinite tail call loop.
1061 #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7)
1064 * Indicate the trampoline should be suitable to receive indirect calls;
1065 * without this indirectly calling the generated code can result in #UD/#CP,
1066 * depending on the CFI options.
1068 * Used by bpf_struct_ops.
1070 * Incompatible with FENTRY usage, overloads @func_addr argument.
1072 #define BPF_TRAMP_F_INDIRECT BIT(8)
1074 /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50
1078 #if defined(__s390x__)
1079 BPF_MAX_TRAMP_LINKS = 27,
1081 BPF_MAX_TRAMP_LINKS = 38,
1085 struct bpf_tramp_links {
1086 struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS];
1090 struct bpf_tramp_run_ctx;
1092 /* Different use cases for BPF trampoline:
1093 * 1. replace nop at the function entry (kprobe equivalent)
1094 * flags = BPF_TRAMP_F_RESTORE_REGS
1095 * fentry = a set of programs to run before returning from trampoline
1097 * 2. replace nop at the function entry (kprobe + kretprobe equivalent)
1098 * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME
1099 * orig_call = fentry_ip + MCOUNT_INSN_SIZE
1100 * fentry = a set of program to run before calling original function
1101 * fexit = a set of program to run after original function
1103 * 3. replace direct call instruction anywhere in the function body
1104 * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid)
1106 * fentry = a set of programs to run before returning from trampoline
1107 * With flags = BPF_TRAMP_F_CALL_ORIG
1108 * orig_call = original callback addr or direct function addr
1109 * fentry = a set of program to run before calling original function
1110 * fexit = a set of program to run after original function
1112 struct bpf_tramp_image;
1113 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
1114 const struct btf_func_model *m, u32 flags,
1115 struct bpf_tramp_links *tlinks,
1117 void *arch_alloc_bpf_trampoline(unsigned int size);
1118 void arch_free_bpf_trampoline(void *image, unsigned int size);
1119 void arch_protect_bpf_trampoline(void *image, unsigned int size);
1120 void arch_unprotect_bpf_trampoline(void *image, unsigned int size);
1121 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
1122 struct bpf_tramp_links *tlinks, void *func_addr);
1124 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
1125 struct bpf_tramp_run_ctx *run_ctx);
1126 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,
1127 struct bpf_tramp_run_ctx *run_ctx);
1128 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
1129 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
1130 typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog,
1131 struct bpf_tramp_run_ctx *run_ctx);
1132 typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start,
1133 struct bpf_tramp_run_ctx *run_ctx);
1134 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog);
1135 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog);
1138 unsigned long start;
1140 char name[KSYM_NAME_LEN];
1141 struct list_head lnode;
1142 struct latch_tree_node tnode;
1146 enum bpf_tramp_prog_type {
1149 BPF_TRAMP_MODIFY_RETURN,
1151 BPF_TRAMP_REPLACE, /* more than MAX */
1154 struct bpf_tramp_image {
1157 struct bpf_ksym ksym;
1158 struct percpu_ref pcref;
1159 void *ip_after_call;
1162 struct rcu_head rcu;
1163 struct work_struct work;
1167 struct bpf_trampoline {
1168 /* hlist for trampoline_table */
1169 struct hlist_node hlist;
1170 struct ftrace_ops *fops;
1171 /* serializes access to fields of this trampoline */
1177 struct btf_func_model model;
1179 bool ftrace_managed;
1181 /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF
1182 * program by replacing one of its functions. func.addr is the address
1183 * of the function it replaced.
1185 struct bpf_prog *extension_prog;
1186 /* list of BPF programs using this trampoline */
1187 struct hlist_head progs_hlist[BPF_TRAMP_MAX];
1188 /* Number of attached programs. A counter per kind. */
1189 int progs_cnt[BPF_TRAMP_MAX];
1190 /* Executable image of trampoline */
1191 struct bpf_tramp_image *cur_image;
1194 struct bpf_attach_target_info {
1195 struct btf_func_model fmodel;
1197 struct module *tgt_mod;
1198 const char *tgt_name;
1199 const struct btf_type *tgt_type;
1202 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
1204 struct bpf_dispatcher_prog {
1205 struct bpf_prog *prog;
1209 struct bpf_dispatcher {
1210 /* dispatcher mutex */
1213 struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
1218 struct bpf_ksym ksym;
1219 #ifdef CONFIG_HAVE_STATIC_CALL
1220 struct static_call_key *sc_key;
1226 #define __bpfcall __nocfi
1229 static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func(
1231 const struct bpf_insn *insnsi,
1232 bpf_func_t bpf_func)
1234 return bpf_func(ctx, insnsi);
1237 /* the implementation of the opaque uapi struct bpf_dynptr */
1238 struct bpf_dynptr_kern {
1240 /* Size represents the number of usable bytes of dynptr data.
1241 * If for example the offset is at 4 for a local dynptr whose data is
1242 * of type u64, the number of usable bytes is 4.
1244 * The upper 8 bits are reserved. It is as follows:
1245 * Bits 0 - 23 = size
1246 * Bits 24 - 30 = dynptr type
1247 * Bit 31 = whether dynptr is read-only
1253 enum bpf_dynptr_type {
1254 BPF_DYNPTR_TYPE_INVALID,
1255 /* Points to memory that is local to the bpf program */
1256 BPF_DYNPTR_TYPE_LOCAL,
1257 /* Underlying data is a ringbuf record */
1258 BPF_DYNPTR_TYPE_RINGBUF,
1259 /* Underlying data is a sk_buff */
1260 BPF_DYNPTR_TYPE_SKB,
1261 /* Underlying data is a xdp_buff */
1262 BPF_DYNPTR_TYPE_XDP,
1265 int bpf_dynptr_check_size(u32 size);
1266 u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr);
1267 const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len);
1268 void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len);
1270 #ifdef CONFIG_BPF_JIT
1271 int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1272 int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);
1273 struct bpf_trampoline *bpf_trampoline_get(u64 key,
1274 struct bpf_attach_target_info *tgt_info);
1275 void bpf_trampoline_put(struct bpf_trampoline *tr);
1276 int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
1279 * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn
1280 * indirection with a direct call to the bpf program. If the architecture does
1281 * not have STATIC_CALL, avoid a double-indirection.
1283 #ifdef CONFIG_HAVE_STATIC_CALL
1285 #define __BPF_DISPATCHER_SC_INIT(_name) \
1286 .sc_key = &STATIC_CALL_KEY(_name), \
1287 .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name),
1289 #define __BPF_DISPATCHER_SC(name) \
1290 DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func)
1292 #define __BPF_DISPATCHER_CALL(name) \
1293 static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func)
1295 #define __BPF_DISPATCHER_UPDATE(_d, _new) \
1296 __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new))
1299 #define __BPF_DISPATCHER_SC_INIT(name)
1300 #define __BPF_DISPATCHER_SC(name)
1301 #define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi)
1302 #define __BPF_DISPATCHER_UPDATE(_d, _new)
1305 #define BPF_DISPATCHER_INIT(_name) { \
1306 .mutex = __MUTEX_INITIALIZER(_name.mutex), \
1307 .func = &_name##_func, \
1314 .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \
1316 __BPF_DISPATCHER_SC_INIT(_name##_call) \
1319 #define DEFINE_BPF_DISPATCHER(name) \
1320 __BPF_DISPATCHER_SC(name); \
1321 noinline __bpfcall unsigned int bpf_dispatcher_##name##_func( \
1323 const struct bpf_insn *insnsi, \
1324 bpf_func_t bpf_func) \
1326 return __BPF_DISPATCHER_CALL(name); \
1328 EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \
1329 struct bpf_dispatcher bpf_dispatcher_##name = \
1330 BPF_DISPATCHER_INIT(bpf_dispatcher_##name);
1332 #define DECLARE_BPF_DISPATCHER(name) \
1333 unsigned int bpf_dispatcher_##name##_func( \
1335 const struct bpf_insn *insnsi, \
1336 bpf_func_t bpf_func); \
1337 extern struct bpf_dispatcher bpf_dispatcher_##name;
1339 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func
1340 #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name)
1341 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
1342 struct bpf_prog *to);
1343 /* Called only from JIT-enabled code, so there's no need for stubs. */
1344 void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);
1345 void bpf_image_ksym_del(struct bpf_ksym *ksym);
1346 void bpf_ksym_add(struct bpf_ksym *ksym);
1347 void bpf_ksym_del(struct bpf_ksym *ksym);
1348 int bpf_jit_charge_modmem(u32 size);
1349 void bpf_jit_uncharge_modmem(u32 size);
1350 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1352 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1353 struct bpf_trampoline *tr)
1357 static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1358 struct bpf_trampoline *tr)
1362 static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
1363 struct bpf_attach_target_info *tgt_info)
1367 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
1368 #define DEFINE_BPF_DISPATCHER(name)
1369 #define DECLARE_BPF_DISPATCHER(name)
1370 #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func
1371 #define BPF_DISPATCHER_PTR(name) NULL
1372 static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d,
1373 struct bpf_prog *from,
1374 struct bpf_prog *to) {}
1375 static inline bool is_bpf_image_address(unsigned long address)
1379 static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
1385 struct bpf_func_info_aux {
1392 enum bpf_jit_poke_reason {
1393 BPF_POKE_REASON_TAIL_CALL,
1396 /* Descriptor of pokes pointing /into/ the JITed image. */
1397 struct bpf_jit_poke_descriptor {
1398 void *tailcall_target;
1399 void *tailcall_bypass;
1404 struct bpf_map *map;
1408 bool tailcall_target_stable;
1414 /* reg_type info for ctx arguments */
1415 struct bpf_ctx_arg_aux {
1417 enum bpf_reg_type reg_type;
1422 struct btf_mod_pair {
1424 struct module *module;
1427 struct bpf_kfunc_desc_tab;
1429 struct bpf_prog_aux {
1438 u32 func_cnt; /* used by non-func prog as the number of func progs */
1439 u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
1440 u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
1441 u32 attach_btf_id; /* in-kernel BTF type id to attach to */
1442 u32 ctx_arg_info_size;
1443 u32 max_rdonly_access;
1444 u32 max_rdwr_access;
1445 struct btf *attach_btf;
1446 const struct bpf_ctx_arg_aux *ctx_arg_info;
1447 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
1448 struct bpf_prog *dst_prog;
1449 struct bpf_trampoline *dst_trampoline;
1450 enum bpf_prog_type saved_dst_prog_type;
1451 enum bpf_attach_type saved_dst_attach_type;
1452 bool verifier_zext; /* Zero extensions has been inserted by verifier. */
1453 bool dev_bound; /* Program is bound to the netdev. */
1454 bool offload_requested; /* Program is bound and offloaded to the netdev. */
1455 bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
1456 bool attach_tracing_prog; /* true if tracing another tracing program */
1457 bool func_proto_unreliable;
1458 bool tail_call_reachable;
1461 bool exception_boundary;
1462 struct bpf_arena *arena;
1463 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1464 const struct btf_type *attach_func_proto;
1465 /* function name for valid attach_btf_id */
1466 const char *attach_func_name;
1467 struct bpf_prog **func;
1468 void *jit_data; /* JIT specific data. arch dependent */
1469 struct bpf_jit_poke_descriptor *poke_tab;
1470 struct bpf_kfunc_desc_tab *kfunc_tab;
1471 struct bpf_kfunc_btf_tab *kfunc_btf_tab;
1473 #ifdef CONFIG_FINEIBT
1474 struct bpf_ksym ksym_prefix;
1476 struct bpf_ksym ksym;
1477 const struct bpf_prog_ops *ops;
1478 struct bpf_map **used_maps;
1479 struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
1480 struct btf_mod_pair *used_btfs;
1481 struct bpf_prog *prog;
1482 struct user_struct *user;
1483 u64 load_time; /* ns since boottime */
1485 int cgroup_atype; /* enum cgroup_bpf_attach_type */
1486 struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1487 char name[BPF_OBJ_NAME_LEN];
1488 u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
1489 #ifdef CONFIG_SECURITY
1492 struct bpf_token *token;
1493 struct bpf_prog_offload *offload;
1495 struct bpf_func_info *func_info;
1496 struct bpf_func_info_aux *func_info_aux;
1497 /* bpf_line_info loaded from userspace. linfo->insn_off
1498 * has the xlated insn offset.
1499 * Both the main and sub prog share the same linfo.
1500 * The subprog can access its first linfo by
1501 * using the linfo_idx.
1503 struct bpf_line_info *linfo;
1504 /* jited_linfo is the jited addr of the linfo. It has a
1505 * one to one mapping to linfo:
1506 * jited_linfo[i] is the jited addr for the linfo[i]->insn_off.
1507 * Both the main and sub prog share the same jited_linfo.
1508 * The subprog can access its first jited_linfo by
1509 * using the linfo_idx.
1514 /* subprog can use linfo_idx to access its first linfo and
1516 * main prog always has linfo_idx == 0
1521 struct exception_table_entry *extable;
1523 struct work_struct work;
1524 struct rcu_head rcu;
1529 u16 pages; /* Number of allocated pages */
1530 u16 jited:1, /* Is our filter JIT'ed? */
1531 jit_requested:1,/* archs need to JIT the prog */
1532 gpl_compatible:1, /* Is filter GPL compatible? */
1533 cb_access:1, /* Is control block accessed? */
1534 dst_needed:1, /* Do we need dst entry? */
1535 blinding_requested:1, /* needs constant blinding */
1536 blinded:1, /* Was blinded */
1537 is_func:1, /* program is a bpf function */
1538 kprobe_override:1, /* Do we override a kprobe? */
1539 has_callchain_buf:1, /* callchain buffer allocated? */
1540 enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
1541 call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
1542 call_get_func_ip:1, /* Do we call get_func_ip() */
1543 tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */
1544 sleepable:1; /* BPF program is sleepable */
1545 enum bpf_prog_type type; /* Type of BPF program */
1546 enum bpf_attach_type expected_attach_type; /* For some prog types */
1547 u32 len; /* Number of filter blocks */
1548 u32 jited_len; /* Size of jited insns in bytes */
1549 u8 tag[BPF_TAG_SIZE];
1550 struct bpf_prog_stats __percpu *stats;
1551 int __percpu *active;
1552 unsigned int (*bpf_func)(const void *ctx,
1553 const struct bpf_insn *insn);
1554 struct bpf_prog_aux *aux; /* Auxiliary fields */
1555 struct sock_fprog_kern *orig_prog; /* Original BPF program */
1556 /* Instructions for interpreter */
1558 DECLARE_FLEX_ARRAY(struct sock_filter, insns);
1559 DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
1563 struct bpf_array_aux {
1564 /* Programs with direct jumps into programs part of this array. */
1565 struct list_head poke_progs;
1566 struct bpf_map *map;
1567 struct mutex poke_mutex;
1568 struct work_struct work;
1574 enum bpf_link_type type;
1575 const struct bpf_link_ops *ops;
1576 struct bpf_prog *prog;
1577 /* rcu is used before freeing, work can be used to schedule that
1578 * RCU-based freeing before that, so they never overlap
1581 struct rcu_head rcu;
1582 struct work_struct work;
1586 struct bpf_link_ops {
1587 void (*release)(struct bpf_link *link);
1588 /* deallocate link resources callback, called without RCU grace period
1591 void (*dealloc)(struct bpf_link *link);
1592 /* deallocate link resources callback, called after RCU grace period;
1593 * if underlying BPF program is sleepable we go through tasks trace
1594 * RCU GP and then "classic" RCU GP
1596 void (*dealloc_deferred)(struct bpf_link *link);
1597 int (*detach)(struct bpf_link *link);
1598 int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog,
1599 struct bpf_prog *old_prog);
1600 void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq);
1601 int (*fill_link_info)(const struct bpf_link *link,
1602 struct bpf_link_info *info);
1603 int (*update_map)(struct bpf_link *link, struct bpf_map *new_map,
1604 struct bpf_map *old_map);
1607 struct bpf_tramp_link {
1608 struct bpf_link link;
1609 struct hlist_node tramp_hlist;
1613 struct bpf_shim_tramp_link {
1614 struct bpf_tramp_link link;
1615 struct bpf_trampoline *trampoline;
1618 struct bpf_tracing_link {
1619 struct bpf_tramp_link link;
1620 enum bpf_attach_type attach_type;
1621 struct bpf_trampoline *trampoline;
1622 struct bpf_prog *tgt_prog;
1625 struct bpf_link_primer {
1626 struct bpf_link *link;
1632 struct bpf_mount_opts {
1637 /* BPF token-related delegation options */
1641 u64 delegate_attachs;
1645 struct work_struct work;
1647 struct user_namespace *userns;
1651 u64 allowed_attachs;
1652 #ifdef CONFIG_SECURITY
1657 struct bpf_struct_ops_value;
1660 #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
1662 * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to
1663 * define a BPF_MAP_TYPE_STRUCT_OPS map type composed
1664 * of BPF_PROG_TYPE_STRUCT_OPS progs.
1665 * @verifier_ops: A structure of callbacks that are invoked by the verifier
1666 * when determining whether the struct_ops progs in the
1667 * struct_ops map are valid.
1668 * @init: A callback that is invoked a single time, and before any other
1669 * callback, to initialize the structure. A nonzero return value means
1670 * the subsystem could not be initialized.
1671 * @check_member: When defined, a callback invoked by the verifier to allow
1672 * the subsystem to determine if an entry in the struct_ops map
1673 * is valid. A nonzero return value means that the map is
1674 * invalid and should be rejected by the verifier.
1675 * @init_member: A callback that is invoked for each member of the struct_ops
1676 * map to allow the subsystem to initialize the member. A nonzero
1677 * value means the member could not be initialized. This callback
1678 * is exclusive with the @type, @type_id, @value_type, and
1680 * @reg: A callback that is invoked when the struct_ops map has been
1681 * initialized and is being attached to. Zero means the struct_ops map
1682 * has been successfully registered and is live. A nonzero return value
1683 * means the struct_ops map could not be registered.
1684 * @unreg: A callback that is invoked when the struct_ops map should be
1686 * @update: A callback that is invoked when the live struct_ops map is being
1687 * updated to contain new values. This callback is only invoked when
1688 * the struct_ops map is loaded with BPF_F_LINK. If not defined, the
1689 * it is assumed that the struct_ops map cannot be updated.
1690 * @validate: A callback that is invoked after all of the members have been
1691 * initialized. This callback should perform static checks on the
1692 * map, meaning that it should either fail or succeed
1693 * deterministically. A struct_ops map that has been validated may
1694 * not necessarily succeed in being registered if the call to @reg
1695 * fails. For example, a valid struct_ops map may be loaded, but
1696 * then fail to be registered due to there being another active
1697 * struct_ops map on the system in the subsystem already. For this
1698 * reason, if this callback is not defined, the check is skipped as
1699 * the struct_ops map will have final verification performed in
1702 * @value_type: Value type.
1703 * @name: The name of the struct bpf_struct_ops object.
1704 * @func_models: Func models
1705 * @type_id: BTF type id.
1706 * @value_id: BTF value id.
1708 struct bpf_struct_ops {
1709 const struct bpf_verifier_ops *verifier_ops;
1710 int (*init)(struct btf *btf);
1711 int (*check_member)(const struct btf_type *t,
1712 const struct btf_member *member,
1713 const struct bpf_prog *prog);
1714 int (*init_member)(const struct btf_type *t,
1715 const struct btf_member *member,
1716 void *kdata, const void *udata);
1717 int (*reg)(void *kdata);
1718 void (*unreg)(void *kdata);
1719 int (*update)(void *kdata, void *old_kdata);
1720 int (*validate)(void *kdata);
1722 struct module *owner;
1724 struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
1727 /* Every member of a struct_ops type has an instance even a member is not
1728 * an operator (function pointer). The "info" field will be assigned to
1729 * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the
1730 * argument information required by the verifier to verify the program.
1732 * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the
1733 * corresponding entry for an given argument.
1735 struct bpf_struct_ops_arg_info {
1736 struct bpf_ctx_arg_aux *info;
1740 struct bpf_struct_ops_desc {
1741 struct bpf_struct_ops *st_ops;
1743 const struct btf_type *type;
1744 const struct btf_type *value_type;
1748 /* Collection of argument information for each member */
1749 struct bpf_struct_ops_arg_info *arg_info;
1752 enum bpf_struct_ops_state {
1753 BPF_STRUCT_OPS_STATE_INIT,
1754 BPF_STRUCT_OPS_STATE_INUSE,
1755 BPF_STRUCT_OPS_STATE_TOBEFREE,
1756 BPF_STRUCT_OPS_STATE_READY,
1759 struct bpf_struct_ops_common_value {
1761 enum bpf_struct_ops_state state;
1764 #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
1765 /* This macro helps developer to register a struct_ops type and generate
1766 * type information correctly. Developers should use this macro to register
1767 * a struct_ops type instead of calling __register_bpf_struct_ops() directly.
1769 #define register_bpf_struct_ops(st_ops, type) \
1771 struct bpf_struct_ops_##type { \
1772 struct bpf_struct_ops_common_value common; \
1773 struct type data ____cacheline_aligned_in_smp; \
1775 BTF_TYPE_EMIT(struct bpf_struct_ops_##type); \
1776 __register_bpf_struct_ops(st_ops); \
1778 #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
1779 bool bpf_struct_ops_get(const void *kdata);
1780 void bpf_struct_ops_put(const void *kdata);
1781 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
1783 int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks,
1784 struct bpf_tramp_link *link,
1785 const struct btf_func_model *model,
1787 void **image, u32 *image_off,
1789 void bpf_struct_ops_image_free(void *image);
1790 static inline bool bpf_try_module_get(const void *data, struct module *owner)
1792 if (owner == BPF_MODULE_OWNER)
1793 return bpf_struct_ops_get(data);
1795 return try_module_get(owner);
1797 static inline void bpf_module_put(const void *data, struct module *owner)
1799 if (owner == BPF_MODULE_OWNER)
1800 bpf_struct_ops_put(data);
1804 int bpf_struct_ops_link_create(union bpf_attr *attr);
1807 /* Define it here to avoid the use of forward declaration */
1808 struct bpf_dummy_ops_state {
1812 struct bpf_dummy_ops {
1813 int (*test_1)(struct bpf_dummy_ops_state *cb);
1814 int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2,
1815 char a3, unsigned long a4);
1816 int (*test_sleepable)(struct bpf_dummy_ops_state *cb);
1819 int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr,
1820 union bpf_attr __user *uattr);
1822 int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
1824 struct bpf_verifier_log *log);
1825 void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map);
1826 void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc);
1828 #define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; })
1829 static inline bool bpf_try_module_get(const void *data, struct module *owner)
1831 return try_module_get(owner);
1833 static inline void bpf_module_put(const void *data, struct module *owner)
1837 static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
1843 static inline int bpf_struct_ops_link_create(union bpf_attr *attr)
1847 static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map)
1851 static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc)
1857 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
1858 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1860 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog);
1862 static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
1867 static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)
1876 struct bpf_array_aux *aux;
1878 DECLARE_FLEX_ARRAY(char, value) __aligned(8);
1879 DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8);
1880 DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8);
1884 #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */
1885 #define MAX_TAIL_CALL_CNT 33
1887 /* Maximum number of loops for bpf_loop and bpf_iter_num.
1888 * It's enum to expose it (and thus make it discoverable) through BTF.
1891 BPF_MAX_LOOPS = 8 * 1024 * 1024,
1894 #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \
1895 BPF_F_RDONLY_PROG | \
1899 #define BPF_MAP_CAN_READ BIT(0)
1900 #define BPF_MAP_CAN_WRITE BIT(1)
1902 /* Maximum number of user-producer ring buffer samples that can be drained in
1903 * a call to bpf_user_ringbuf_drain().
1905 #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
1907 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
1909 u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1911 /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is
1914 if (access_flags & BPF_F_RDONLY_PROG)
1915 return BPF_MAP_CAN_READ;
1916 else if (access_flags & BPF_F_WRONLY_PROG)
1917 return BPF_MAP_CAN_WRITE;
1919 return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE;
1922 static inline bool bpf_map_flags_access_ok(u32 access_flags)
1924 return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
1925 (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
1928 struct bpf_event_entry {
1929 struct perf_event *event;
1930 struct file *perf_file;
1931 struct file *map_file;
1932 struct rcu_head rcu;
1935 static inline bool map_type_contains_progs(struct bpf_map *map)
1937 return map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
1938 map->map_type == BPF_MAP_TYPE_DEVMAP ||
1939 map->map_type == BPF_MAP_TYPE_CPUMAP;
1942 bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp);
1943 int bpf_prog_calc_tag(struct bpf_prog *fp);
1945 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
1946 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
1948 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
1949 unsigned long off, unsigned long len);
1950 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
1951 const struct bpf_insn *src,
1952 struct bpf_insn *dst,
1953 struct bpf_prog *prog,
1956 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
1957 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy);
1959 /* an array of programs to be executed under rcu_lock.
1962 * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run);
1964 * the structure returned by bpf_prog_array_alloc() should be populated
1965 * with program pointers and the last pointer must be NULL.
1966 * The user has to keep refcnt on the program and make sure the program
1967 * is removed from the array before bpf_prog_put().
1968 * The 'struct bpf_prog_array *' should only be replaced with xchg()
1969 * since other cpus are walking the array of pointers in parallel.
1971 struct bpf_prog_array_item {
1972 struct bpf_prog *prog;
1974 struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
1979 struct bpf_prog_array {
1980 struct rcu_head rcu;
1981 struct bpf_prog_array_item items[];
1984 struct bpf_empty_prog_array {
1985 struct bpf_prog_array hdr;
1986 struct bpf_prog *null_prog;
1989 /* to avoid allocating empty bpf_prog_array for cgroups that
1990 * don't have bpf program attached use one global 'bpf_empty_prog_array'
1991 * It will not be modified the caller of bpf_prog_array_alloc()
1992 * (since caller requested prog_cnt == 0)
1993 * that pointer should be 'freed' by bpf_prog_array_free()
1995 extern struct bpf_empty_prog_array bpf_empty_prog_array;
1997 struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
1998 void bpf_prog_array_free(struct bpf_prog_array *progs);
1999 /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
2000 void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
2001 int bpf_prog_array_length(struct bpf_prog_array *progs);
2002 bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
2003 int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
2004 __u32 __user *prog_ids, u32 cnt);
2006 void bpf_prog_array_delete_safe(struct bpf_prog_array *progs,
2007 struct bpf_prog *old_prog);
2008 int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index);
2009 int bpf_prog_array_update_at(struct bpf_prog_array *array, int index,
2010 struct bpf_prog *prog);
2011 int bpf_prog_array_copy_info(struct bpf_prog_array *array,
2012 u32 *prog_ids, u32 request_cnt,
2014 int bpf_prog_array_copy(struct bpf_prog_array *old_array,
2015 struct bpf_prog *exclude_prog,
2016 struct bpf_prog *include_prog,
2018 struct bpf_prog_array **new_array);
2020 struct bpf_run_ctx {};
2022 struct bpf_cg_run_ctx {
2023 struct bpf_run_ctx run_ctx;
2024 const struct bpf_prog_array_item *prog_item;
2028 struct bpf_trace_run_ctx {
2029 struct bpf_run_ctx run_ctx;
2034 struct bpf_tramp_run_ctx {
2035 struct bpf_run_ctx run_ctx;
2037 struct bpf_run_ctx *saved_run_ctx;
2040 static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
2042 struct bpf_run_ctx *old_ctx = NULL;
2044 #ifdef CONFIG_BPF_SYSCALL
2045 old_ctx = current->bpf_ctx;
2046 current->bpf_ctx = new_ctx;
2051 static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx)
2053 #ifdef CONFIG_BPF_SYSCALL
2054 current->bpf_ctx = old_ctx;
2058 /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */
2059 #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0)
2060 /* BPF program asks to set CN on the packet. */
2061 #define BPF_RET_SET_CN (1 << 0)
2063 typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx);
2065 static __always_inline u32
2066 bpf_prog_run_array(const struct bpf_prog_array *array,
2067 const void *ctx, bpf_prog_run_fn run_prog)
2069 const struct bpf_prog_array_item *item;
2070 const struct bpf_prog *prog;
2071 struct bpf_run_ctx *old_run_ctx;
2072 struct bpf_trace_run_ctx run_ctx;
2075 RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held");
2077 if (unlikely(!array))
2080 run_ctx.is_uprobe = false;
2083 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2084 item = &array->items[0];
2085 while ((prog = READ_ONCE(item->prog))) {
2086 run_ctx.bpf_cookie = item->bpf_cookie;
2087 ret &= run_prog(prog, ctx);
2090 bpf_reset_run_ctx(old_run_ctx);
2095 /* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
2097 * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
2098 * overall. As a result, we must use the bpf_prog_array_free_sleepable
2099 * in order to use the tasks_trace rcu grace period.
2101 * When a non-sleepable program is inside the array, we take the rcu read
2102 * section and disable preemption for that program alone, so it can access
2103 * rcu-protected dynamically sized maps.
2105 static __always_inline u32
2106 bpf_prog_run_array_uprobe(const struct bpf_prog_array __rcu *array_rcu,
2107 const void *ctx, bpf_prog_run_fn run_prog)
2109 const struct bpf_prog_array_item *item;
2110 const struct bpf_prog *prog;
2111 const struct bpf_prog_array *array;
2112 struct bpf_run_ctx *old_run_ctx;
2113 struct bpf_trace_run_ctx run_ctx;
2118 rcu_read_lock_trace();
2121 run_ctx.is_uprobe = true;
2123 array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
2124 if (unlikely(!array))
2126 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
2127 item = &array->items[0];
2128 while ((prog = READ_ONCE(item->prog))) {
2129 if (!prog->sleepable)
2132 run_ctx.bpf_cookie = item->bpf_cookie;
2133 ret &= run_prog(prog, ctx);
2136 if (!prog->sleepable)
2139 bpf_reset_run_ctx(old_run_ctx);
2142 rcu_read_unlock_trace();
2146 #ifdef CONFIG_BPF_SYSCALL
2147 DECLARE_PER_CPU(int, bpf_prog_active);
2148 extern struct mutex bpf_stats_enabled_mutex;
2151 * Block execution of BPF programs attached to instrumentation (perf,
2152 * kprobes, tracepoints) to prevent deadlocks on map operations as any of
2153 * these events can happen inside a region which holds a map bucket lock
2154 * and can deadlock on it.
2156 static inline void bpf_disable_instrumentation(void)
2159 this_cpu_inc(bpf_prog_active);
2162 static inline void bpf_enable_instrumentation(void)
2164 this_cpu_dec(bpf_prog_active);
2168 extern const struct super_operations bpf_super_ops;
2169 extern const struct file_operations bpf_map_fops;
2170 extern const struct file_operations bpf_prog_fops;
2171 extern const struct file_operations bpf_iter_fops;
2173 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
2174 extern const struct bpf_prog_ops _name ## _prog_ops; \
2175 extern const struct bpf_verifier_ops _name ## _verifier_ops;
2176 #define BPF_MAP_TYPE(_id, _ops) \
2177 extern const struct bpf_map_ops _ops;
2178 #define BPF_LINK_TYPE(_id, _name)
2179 #include <linux/bpf_types.h>
2180 #undef BPF_PROG_TYPE
2182 #undef BPF_LINK_TYPE
2184 extern const struct bpf_prog_ops bpf_offload_prog_ops;
2185 extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops;
2186 extern const struct bpf_verifier_ops xdp_analyzer_ops;
2188 struct bpf_prog *bpf_prog_get(u32 ufd);
2189 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
2191 void bpf_prog_add(struct bpf_prog *prog, int i);
2192 void bpf_prog_sub(struct bpf_prog *prog, int i);
2193 void bpf_prog_inc(struct bpf_prog *prog);
2194 struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog);
2195 void bpf_prog_put(struct bpf_prog *prog);
2197 void bpf_prog_free_id(struct bpf_prog *prog);
2198 void bpf_map_free_id(struct bpf_map *map);
2200 struct btf_field *btf_record_find(const struct btf_record *rec,
2201 u32 offset, u32 field_mask);
2202 void btf_record_free(struct btf_record *rec);
2203 void bpf_map_free_record(struct bpf_map *map);
2204 struct btf_record *btf_record_dup(const struct btf_record *rec);
2205 bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b);
2206 void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
2207 void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
2208 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
2210 struct bpf_map *bpf_map_get(u32 ufd);
2211 struct bpf_map *bpf_map_get_with_uref(u32 ufd);
2212 struct bpf_map *__bpf_map_get(struct fd f);
2213 void bpf_map_inc(struct bpf_map *map);
2214 void bpf_map_inc_with_uref(struct bpf_map *map);
2215 struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref);
2216 struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map);
2217 void bpf_map_put_with_uref(struct bpf_map *map);
2218 void bpf_map_put(struct bpf_map *map);
2219 void *bpf_map_area_alloc(u64 size, int numa_node);
2220 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node);
2221 void bpf_map_area_free(void *base);
2222 bool bpf_map_write_active(const struct bpf_map *map);
2223 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
2224 int generic_map_lookup_batch(struct bpf_map *map,
2225 const union bpf_attr *attr,
2226 union bpf_attr __user *uattr);
2227 int generic_map_update_batch(struct bpf_map *map, struct file *map_file,
2228 const union bpf_attr *attr,
2229 union bpf_attr __user *uattr);
2230 int generic_map_delete_batch(struct bpf_map *map,
2231 const union bpf_attr *attr,
2232 union bpf_attr __user *uattr);
2233 struct bpf_map *bpf_map_get_curr_or_next(u32 *id);
2234 struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);
2236 int bpf_map_alloc_pages(const struct bpf_map *map, gfp_t gfp, int nid,
2237 unsigned long nr_pages, struct page **page_array);
2238 #ifdef CONFIG_MEMCG_KMEM
2239 void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2241 void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags);
2242 void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
2244 void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
2245 size_t align, gfp_t flags);
2247 static inline void *
2248 bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
2251 return kmalloc_node(size, flags, node);
2254 static inline void *
2255 bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
2257 return kzalloc(size, flags);
2260 static inline void *
2261 bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags)
2263 return kvcalloc(n, size, flags);
2266 static inline void __percpu *
2267 bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align,
2270 return __alloc_percpu_gfp(size, align, flags);
2275 bpf_map_init_elem_count(struct bpf_map *map)
2277 size_t size = sizeof(*map->elem_count), align = size;
2278 gfp_t flags = GFP_USER | __GFP_NOWARN;
2280 map->elem_count = bpf_map_alloc_percpu(map, size, align, flags);
2281 if (!map->elem_count)
2288 bpf_map_free_elem_count(struct bpf_map *map)
2290 free_percpu(map->elem_count);
2293 static inline void bpf_map_inc_elem_count(struct bpf_map *map)
2295 this_cpu_inc(*map->elem_count);
2298 static inline void bpf_map_dec_elem_count(struct bpf_map *map)
2300 this_cpu_dec(*map->elem_count);
2303 extern int sysctl_unprivileged_bpf_disabled;
2305 bool bpf_token_capable(const struct bpf_token *token, int cap);
2307 static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token)
2309 return bpf_token_capable(token, CAP_PERFMON);
2312 static inline bool bpf_allow_uninit_stack(const struct bpf_token *token)
2314 return bpf_token_capable(token, CAP_PERFMON);
2317 static inline bool bpf_bypass_spec_v1(const struct bpf_token *token)
2319 return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
2322 static inline bool bpf_bypass_spec_v4(const struct bpf_token *token)
2324 return cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON);
2327 int bpf_map_new_fd(struct bpf_map *map, int flags);
2328 int bpf_prog_new_fd(struct bpf_prog *prog);
2330 void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2331 const struct bpf_link_ops *ops, struct bpf_prog *prog);
2332 int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer);
2333 int bpf_link_settle(struct bpf_link_primer *primer);
2334 void bpf_link_cleanup(struct bpf_link_primer *primer);
2335 void bpf_link_inc(struct bpf_link *link);
2336 void bpf_link_put(struct bpf_link *link);
2337 int bpf_link_new_fd(struct bpf_link *link);
2338 struct bpf_link *bpf_link_get_from_fd(u32 ufd);
2339 struct bpf_link *bpf_link_get_curr_or_next(u32 *id);
2341 void bpf_token_inc(struct bpf_token *token);
2342 void bpf_token_put(struct bpf_token *token);
2343 int bpf_token_create(union bpf_attr *attr);
2344 struct bpf_token *bpf_token_get_from_fd(u32 ufd);
2346 bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd);
2347 bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type);
2348 bool bpf_token_allow_prog_type(const struct bpf_token *token,
2349 enum bpf_prog_type prog_type,
2350 enum bpf_attach_type attach_type);
2352 int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname);
2353 int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags);
2354 struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir,
2357 #define BPF_ITER_FUNC_PREFIX "bpf_iter_"
2358 #define DEFINE_BPF_ITER_FUNC(target, args...) \
2359 extern int bpf_iter_ ## target(args); \
2360 int __init bpf_iter_ ## target(args) { return 0; }
2363 * The task type of iterators.
2365 * For BPF task iterators, they can be parameterized with various
2366 * parameters to visit only some of tasks.
2368 * BPF_TASK_ITER_ALL (default)
2369 * Iterate over resources of every task.
2372 * Iterate over resources of a task/tid.
2374 * BPF_TASK_ITER_TGID
2375 * Iterate over resources of every task of a process / task group.
2377 enum bpf_iter_task_type {
2378 BPF_TASK_ITER_ALL = 0,
2383 struct bpf_iter_aux_info {
2384 /* for map_elem iter */
2385 struct bpf_map *map;
2387 /* for cgroup iter */
2389 struct cgroup *start; /* starting cgroup */
2390 enum bpf_cgroup_iter_order order;
2393 enum bpf_iter_task_type type;
2398 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
2399 union bpf_iter_link_info *linfo,
2400 struct bpf_iter_aux_info *aux);
2401 typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux);
2402 typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
2403 struct seq_file *seq);
2404 typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
2405 struct bpf_link_info *info);
2406 typedef const struct bpf_func_proto *
2407 (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id,
2408 const struct bpf_prog *prog);
2410 enum bpf_iter_feature {
2411 BPF_ITER_RESCHED = BIT(0),
2414 #define BPF_ITER_CTX_ARG_MAX 2
2415 struct bpf_iter_reg {
2417 bpf_iter_attach_target_t attach_target;
2418 bpf_iter_detach_target_t detach_target;
2419 bpf_iter_show_fdinfo_t show_fdinfo;
2420 bpf_iter_fill_link_info_t fill_link_info;
2421 bpf_iter_get_func_proto_t get_func_proto;
2422 u32 ctx_arg_info_size;
2424 struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
2425 const struct bpf_iter_seq_info *seq_info;
2428 struct bpf_iter_meta {
2429 __bpf_md_ptr(struct seq_file *, seq);
2434 struct bpf_iter__bpf_map_elem {
2435 __bpf_md_ptr(struct bpf_iter_meta *, meta);
2436 __bpf_md_ptr(struct bpf_map *, map);
2437 __bpf_md_ptr(void *, key);
2438 __bpf_md_ptr(void *, value);
2441 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
2442 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
2443 bool bpf_iter_prog_supported(struct bpf_prog *prog);
2444 const struct bpf_func_proto *
2445 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
2446 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);
2447 int bpf_iter_new_fd(struct bpf_link *link);
2448 bool bpf_link_is_iter(struct bpf_link *link);
2449 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop);
2450 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx);
2451 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux,
2452 struct seq_file *seq);
2453 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux,
2454 struct bpf_link_info *info);
2456 int map_set_for_each_callback_args(struct bpf_verifier_env *env,
2457 struct bpf_func_state *caller,
2458 struct bpf_func_state *callee);
2460 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value);
2461 int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value);
2462 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value,
2464 int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
2467 int bpf_stackmap_copy(struct bpf_map *map, void *key, void *value);
2469 int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file,
2470 void *key, void *value, u64 map_flags);
2471 int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2472 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file,
2473 void *key, void *value, u64 map_flags);
2474 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value);
2476 int bpf_get_file_flag(int flags);
2477 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
2478 size_t actual_size);
2480 /* verify correctness of eBPF program */
2481 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
2483 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
2484 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
2487 struct btf *bpf_get_btf_vmlinux(void);
2492 struct bpf_dtab_netdev;
2493 struct bpf_cpu_map_entry;
2495 void __dev_flush(void);
2496 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2497 struct net_device *dev_rx);
2498 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2499 struct net_device *dev_rx);
2500 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2501 struct bpf_map *map, bool exclude_ingress);
2502 int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb,
2503 struct bpf_prog *xdp_prog);
2504 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2505 struct bpf_prog *xdp_prog, struct bpf_map *map,
2506 bool exclude_ingress);
2508 void __cpu_map_flush(void);
2509 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf,
2510 struct net_device *dev_rx);
2511 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2512 struct sk_buff *skb);
2514 /* Return map's numa specified by userspace */
2515 static inline int bpf_map_attr_numa_node(const union bpf_attr *attr)
2517 return (attr->map_flags & BPF_F_NUMA_NODE) ?
2518 attr->numa_node : NUMA_NO_NODE;
2521 struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type);
2522 int array_map_alloc_check(union bpf_attr *attr);
2524 int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr,
2525 union bpf_attr __user *uattr);
2526 int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
2527 union bpf_attr __user *uattr);
2528 int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2529 const union bpf_attr *kattr,
2530 union bpf_attr __user *uattr);
2531 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2532 const union bpf_attr *kattr,
2533 union bpf_attr __user *uattr);
2534 int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
2535 const union bpf_attr *kattr,
2536 union bpf_attr __user *uattr);
2537 int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2538 const union bpf_attr *kattr,
2539 union bpf_attr __user *uattr);
2540 int bpf_prog_test_run_nf(struct bpf_prog *prog,
2541 const union bpf_attr *kattr,
2542 union bpf_attr __user *uattr);
2543 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
2544 const struct bpf_prog *prog,
2545 struct bpf_insn_access_aux *info);
2547 static inline bool bpf_tracing_ctx_access(int off, int size,
2548 enum bpf_access_type type)
2550 if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
2552 if (type != BPF_READ)
2554 if (off % size != 0)
2559 static inline bool bpf_tracing_btf_ctx_access(int off, int size,
2560 enum bpf_access_type type,
2561 const struct bpf_prog *prog,
2562 struct bpf_insn_access_aux *info)
2564 if (!bpf_tracing_ctx_access(off, size, type))
2566 return btf_ctx_access(off, size, type, prog, info);
2569 int btf_struct_access(struct bpf_verifier_log *log,
2570 const struct bpf_reg_state *reg,
2571 int off, int size, enum bpf_access_type atype,
2572 u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name);
2573 bool btf_struct_ids_match(struct bpf_verifier_log *log,
2574 const struct btf *btf, u32 id, int off,
2575 const struct btf *need_btf, u32 need_type_id,
2578 int btf_distill_func_proto(struct bpf_verifier_log *log,
2580 const struct btf_type *func_proto,
2581 const char *func_name,
2582 struct btf_func_model *m);
2584 struct bpf_reg_state;
2585 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog);
2586 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
2587 struct btf *btf, const struct btf_type *t);
2588 const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt,
2589 int comp_idx, const char *tag_key);
2590 int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt,
2591 int comp_idx, const char *tag_key, int last_id);
2593 struct bpf_prog *bpf_prog_by_id(u32 id);
2594 struct bpf_link *bpf_link_by_id(u32 id);
2596 const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id,
2597 const struct bpf_prog *prog);
2598 void bpf_task_storage_free(struct task_struct *task);
2599 void bpf_cgrp_storage_free(struct cgroup *cgroup);
2600 bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog);
2601 const struct btf_func_model *
2602 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2603 const struct bpf_insn *insn);
2604 int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2605 u16 btf_fd_idx, u8 **func_addr);
2607 struct bpf_core_ctx {
2608 struct bpf_verifier_log *log;
2609 const struct btf *btf;
2612 bool btf_nested_type_is_trusted(struct bpf_verifier_log *log,
2613 const struct bpf_reg_state *reg,
2614 const char *field_name, u32 btf_id, const char *suffix);
2616 bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log,
2617 const struct btf *reg_btf, u32 reg_id,
2618 const struct btf *arg_btf, u32 arg_id);
2620 int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo,
2621 int relo_idx, void *insn);
2623 static inline bool unprivileged_ebpf_enabled(void)
2625 return !sysctl_unprivileged_bpf_disabled;
2628 /* Not all bpf prog type has the bpf_ctx.
2629 * For the bpf prog type that has initialized the bpf_ctx,
2630 * this function can be used to decide if a kernel function
2631 * is called by a bpf program.
2633 static inline bool has_current_bpf_ctx(void)
2635 return !!current->bpf_ctx;
2638 void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
2640 void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
2641 enum bpf_dynptr_type type, u32 offset, u32 size);
2642 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
2643 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
2645 bool dev_check_flush(void);
2646 bool cpu_map_check_flush(void);
2647 #else /* !CONFIG_BPF_SYSCALL */
2648 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
2650 return ERR_PTR(-EOPNOTSUPP);
2653 static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd,
2654 enum bpf_prog_type type,
2657 return ERR_PTR(-EOPNOTSUPP);
2660 static inline void bpf_prog_add(struct bpf_prog *prog, int i)
2664 static inline void bpf_prog_sub(struct bpf_prog *prog, int i)
2668 static inline void bpf_prog_put(struct bpf_prog *prog)
2672 static inline void bpf_prog_inc(struct bpf_prog *prog)
2676 static inline struct bpf_prog *__must_check
2677 bpf_prog_inc_not_zero(struct bpf_prog *prog)
2679 return ERR_PTR(-EOPNOTSUPP);
2682 static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type,
2683 const struct bpf_link_ops *ops,
2684 struct bpf_prog *prog)
2688 static inline int bpf_link_prime(struct bpf_link *link,
2689 struct bpf_link_primer *primer)
2694 static inline int bpf_link_settle(struct bpf_link_primer *primer)
2699 static inline void bpf_link_cleanup(struct bpf_link_primer *primer)
2703 static inline void bpf_link_inc(struct bpf_link *link)
2707 static inline void bpf_link_put(struct bpf_link *link)
2711 static inline int bpf_obj_get_user(const char __user *pathname, int flags)
2716 static inline bool bpf_token_capable(const struct bpf_token *token, int cap)
2718 return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN));
2721 static inline void bpf_token_inc(struct bpf_token *token)
2725 static inline void bpf_token_put(struct bpf_token *token)
2729 static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd)
2731 return ERR_PTR(-EOPNOTSUPP);
2734 static inline void __dev_flush(void)
2739 struct bpf_dtab_netdev;
2740 struct bpf_cpu_map_entry;
2743 int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
2744 struct net_device *dev_rx)
2750 int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf,
2751 struct net_device *dev_rx)
2757 int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
2758 struct bpf_map *map, bool exclude_ingress)
2765 static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst,
2766 struct sk_buff *skb,
2767 struct bpf_prog *xdp_prog)
2773 int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
2774 struct bpf_prog *xdp_prog, struct bpf_map *map,
2775 bool exclude_ingress)
2780 static inline void __cpu_map_flush(void)
2784 static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu,
2785 struct xdp_frame *xdpf,
2786 struct net_device *dev_rx)
2791 static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
2792 struct sk_buff *skb)
2797 static inline struct bpf_prog *bpf_prog_get_type_path(const char *name,
2798 enum bpf_prog_type type)
2800 return ERR_PTR(-EOPNOTSUPP);
2803 static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog,
2804 const union bpf_attr *kattr,
2805 union bpf_attr __user *uattr)
2810 static inline int bpf_prog_test_run_skb(struct bpf_prog *prog,
2811 const union bpf_attr *kattr,
2812 union bpf_attr __user *uattr)
2817 static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog,
2818 const union bpf_attr *kattr,
2819 union bpf_attr __user *uattr)
2824 static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
2825 const union bpf_attr *kattr,
2826 union bpf_attr __user *uattr)
2831 static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog,
2832 const union bpf_attr *kattr,
2833 union bpf_attr __user *uattr)
2838 static inline void bpf_map_put(struct bpf_map *map)
2842 static inline struct bpf_prog *bpf_prog_by_id(u32 id)
2844 return ERR_PTR(-ENOTSUPP);
2847 static inline int btf_struct_access(struct bpf_verifier_log *log,
2848 const struct bpf_reg_state *reg,
2849 int off, int size, enum bpf_access_type atype,
2850 u32 *next_btf_id, enum bpf_type_flag *flag,
2851 const char **field_name)
2856 static inline const struct bpf_func_proto *
2857 bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2862 static inline void bpf_task_storage_free(struct task_struct *task)
2866 static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
2871 static inline const struct btf_func_model *
2872 bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
2873 const struct bpf_insn *insn)
2879 bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
2880 u16 btf_fd_idx, u8 **func_addr)
2885 static inline bool unprivileged_ebpf_enabled(void)
2890 static inline bool has_current_bpf_ctx(void)
2895 static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
2899 static inline void bpf_cgrp_storage_free(struct cgroup *cgroup)
2903 static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
2904 enum bpf_dynptr_type type, u32 offset, u32 size)
2908 static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr)
2912 static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
2915 #endif /* CONFIG_BPF_SYSCALL */
2917 static __always_inline int
2918 bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
2922 if (IS_ENABLED(CONFIG_BPF_EVENTS))
2923 ret = copy_from_kernel_nofault(dst, unsafe_ptr, size);
2924 if (unlikely(ret < 0))
2925 memset(dst, 0, size);
2929 void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
2930 struct btf_mod_pair *used_btfs, u32 len);
2932 static inline struct bpf_prog *bpf_prog_get_type(u32 ufd,
2933 enum bpf_prog_type type)
2935 return bpf_prog_get_type_dev(ufd, type, false);
2938 void __bpf_free_used_maps(struct bpf_prog_aux *aux,
2939 struct bpf_map **used_maps, u32 len);
2941 bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool);
2943 int bpf_prog_offload_compile(struct bpf_prog *prog);
2944 void bpf_prog_dev_bound_destroy(struct bpf_prog *prog);
2945 int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
2946 struct bpf_prog *prog);
2948 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map);
2950 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value);
2951 int bpf_map_offload_update_elem(struct bpf_map *map,
2952 void *key, void *value, u64 flags);
2953 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
2954 int bpf_map_offload_get_next_key(struct bpf_map *map,
2955 void *key, void *next_key);
2957 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
2959 struct bpf_offload_dev *
2960 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv);
2961 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
2962 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev);
2963 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
2964 struct net_device *netdev);
2965 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
2966 struct net_device *netdev);
2967 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
2969 void unpriv_ebpf_notify(int new_state);
2971 #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
2972 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
2973 struct bpf_prog_aux *prog_aux);
2974 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id);
2975 int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr);
2976 int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog);
2977 void bpf_dev_bound_netdev_unregister(struct net_device *dev);
2979 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
2981 return aux->dev_bound;
2984 static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux)
2986 return aux->offload_requested;
2989 bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs);
2991 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
2993 return unlikely(map->ops == &bpf_map_offload_ops);
2996 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr);
2997 void bpf_map_offload_map_free(struct bpf_map *map);
2998 u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map);
2999 int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3000 const union bpf_attr *kattr,
3001 union bpf_attr __user *uattr);
3003 int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog);
3004 int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype);
3005 int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags);
3006 int sock_map_bpf_prog_query(const union bpf_attr *attr,
3007 union bpf_attr __user *uattr);
3009 void sock_map_unhash(struct sock *sk);
3010 void sock_map_destroy(struct sock *sk);
3011 void sock_map_close(struct sock *sk, long timeout);
3013 static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
3014 struct bpf_prog_aux *prog_aux)
3019 static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog,
3025 static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog,
3026 union bpf_attr *attr)
3031 static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog,
3032 struct bpf_prog *old_prog)
3037 static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev)
3041 static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
3046 static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux)
3051 static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
3056 static inline bool bpf_map_is_offloaded(struct bpf_map *map)
3061 static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
3063 return ERR_PTR(-EOPNOTSUPP);
3066 static inline void bpf_map_offload_map_free(struct bpf_map *map)
3070 static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
3075 static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog,
3076 const union bpf_attr *kattr,
3077 union bpf_attr __user *uattr)
3082 #ifdef CONFIG_BPF_SYSCALL
3083 static inline int sock_map_get_from_fd(const union bpf_attr *attr,
3084 struct bpf_prog *prog)
3089 static inline int sock_map_prog_detach(const union bpf_attr *attr,
3090 enum bpf_prog_type ptype)
3095 static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value,
3101 static inline int sock_map_bpf_prog_query(const union bpf_attr *attr,
3102 union bpf_attr __user *uattr)
3106 #endif /* CONFIG_BPF_SYSCALL */
3107 #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */
3109 static __always_inline void
3110 bpf_prog_inc_misses_counters(const struct bpf_prog_array *array)
3112 const struct bpf_prog_array_item *item;
3113 struct bpf_prog *prog;
3115 if (unlikely(!array))
3118 item = &array->items[0];
3119 while ((prog = READ_ONCE(item->prog))) {
3120 bpf_prog_inc_misses_counter(prog);
3125 #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL)
3126 void bpf_sk_reuseport_detach(struct sock *sk);
3127 int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key,
3129 int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key,
3130 void *value, u64 map_flags);
3132 static inline void bpf_sk_reuseport_detach(struct sock *sk)
3136 #ifdef CONFIG_BPF_SYSCALL
3137 static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map,
3138 void *key, void *value)
3143 static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
3144 void *key, void *value,
3149 #endif /* CONFIG_BPF_SYSCALL */
3150 #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */
3152 /* verifier prototypes for helper functions called from eBPF programs */
3153 extern const struct bpf_func_proto bpf_map_lookup_elem_proto;
3154 extern const struct bpf_func_proto bpf_map_update_elem_proto;
3155 extern const struct bpf_func_proto bpf_map_delete_elem_proto;
3156 extern const struct bpf_func_proto bpf_map_push_elem_proto;
3157 extern const struct bpf_func_proto bpf_map_pop_elem_proto;
3158 extern const struct bpf_func_proto bpf_map_peek_elem_proto;
3159 extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto;
3161 extern const struct bpf_func_proto bpf_get_prandom_u32_proto;
3162 extern const struct bpf_func_proto bpf_get_smp_processor_id_proto;
3163 extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
3164 extern const struct bpf_func_proto bpf_tail_call_proto;
3165 extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
3166 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
3167 extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
3168 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
3169 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
3170 extern const struct bpf_func_proto bpf_get_current_comm_proto;
3171 extern const struct bpf_func_proto bpf_get_stackid_proto;
3172 extern const struct bpf_func_proto bpf_get_stack_proto;
3173 extern const struct bpf_func_proto bpf_get_task_stack_proto;
3174 extern const struct bpf_func_proto bpf_get_stackid_proto_pe;
3175 extern const struct bpf_func_proto bpf_get_stack_proto_pe;
3176 extern const struct bpf_func_proto bpf_sock_map_update_proto;
3177 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
3178 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
3179 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
3180 extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
3181 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
3182 extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
3183 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
3184 extern const struct bpf_func_proto bpf_sk_redirect_map_proto;
3185 extern const struct bpf_func_proto bpf_spin_lock_proto;
3186 extern const struct bpf_func_proto bpf_spin_unlock_proto;
3187 extern const struct bpf_func_proto bpf_get_local_storage_proto;
3188 extern const struct bpf_func_proto bpf_strtol_proto;
3189 extern const struct bpf_func_proto bpf_strtoul_proto;
3190 extern const struct bpf_func_proto bpf_tcp_sock_proto;
3191 extern const struct bpf_func_proto bpf_jiffies64_proto;
3192 extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto;
3193 extern const struct bpf_func_proto bpf_event_output_data_proto;
3194 extern const struct bpf_func_proto bpf_ringbuf_output_proto;
3195 extern const struct bpf_func_proto bpf_ringbuf_reserve_proto;
3196 extern const struct bpf_func_proto bpf_ringbuf_submit_proto;
3197 extern const struct bpf_func_proto bpf_ringbuf_discard_proto;
3198 extern const struct bpf_func_proto bpf_ringbuf_query_proto;
3199 extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto;
3200 extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto;
3201 extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto;
3202 extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto;
3203 extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto;
3204 extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto;
3205 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
3206 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
3207 extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto;
3208 extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto;
3209 extern const struct bpf_func_proto bpf_copy_from_user_proto;
3210 extern const struct bpf_func_proto bpf_snprintf_btf_proto;
3211 extern const struct bpf_func_proto bpf_snprintf_proto;
3212 extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
3213 extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
3214 extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto;
3215 extern const struct bpf_func_proto bpf_sock_from_file_proto;
3216 extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto;
3217 extern const struct bpf_func_proto bpf_task_storage_get_recur_proto;
3218 extern const struct bpf_func_proto bpf_task_storage_get_proto;
3219 extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto;
3220 extern const struct bpf_func_proto bpf_task_storage_delete_proto;
3221 extern const struct bpf_func_proto bpf_for_each_map_elem_proto;
3222 extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
3223 extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
3224 extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
3225 extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto;
3226 extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto;
3227 extern const struct bpf_func_proto bpf_find_vma_proto;
3228 extern const struct bpf_func_proto bpf_loop_proto;
3229 extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
3230 extern const struct bpf_func_proto bpf_set_retval_proto;
3231 extern const struct bpf_func_proto bpf_get_retval_proto;
3232 extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
3233 extern const struct bpf_func_proto bpf_cgrp_storage_get_proto;
3234 extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto;
3236 const struct bpf_func_proto *tracing_prog_func_proto(
3237 enum bpf_func_id func_id, const struct bpf_prog *prog);
3239 /* Shared helpers among cBPF and eBPF. */
3240 void bpf_user_rnd_init_once(void);
3241 u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3242 u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
3244 #if defined(CONFIG_NET)
3245 bool bpf_sock_common_is_valid_access(int off, int size,
3246 enum bpf_access_type type,
3247 struct bpf_insn_access_aux *info);
3248 bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3249 struct bpf_insn_access_aux *info);
3250 u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3251 const struct bpf_insn *si,
3252 struct bpf_insn *insn_buf,
3253 struct bpf_prog *prog,
3255 int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags,
3256 struct bpf_dynptr_kern *ptr);
3258 static inline bool bpf_sock_common_is_valid_access(int off, int size,
3259 enum bpf_access_type type,
3260 struct bpf_insn_access_aux *info)
3264 static inline bool bpf_sock_is_valid_access(int off, int size,
3265 enum bpf_access_type type,
3266 struct bpf_insn_access_aux *info)
3270 static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
3271 const struct bpf_insn *si,
3272 struct bpf_insn *insn_buf,
3273 struct bpf_prog *prog,
3278 static inline int bpf_dynptr_from_skb_rdonly(struct sk_buff *skb, u64 flags,
3279 struct bpf_dynptr_kern *ptr)
3286 struct sk_reuseport_kern {
3287 struct sk_buff *skb;
3289 struct sock *selected_sk;
3290 struct sock *migrating_sk;
3296 bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3297 struct bpf_insn_access_aux *info);
3299 u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3300 const struct bpf_insn *si,
3301 struct bpf_insn *insn_buf,
3302 struct bpf_prog *prog,
3305 bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
3306 struct bpf_insn_access_aux *info);
3308 u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3309 const struct bpf_insn *si,
3310 struct bpf_insn *insn_buf,
3311 struct bpf_prog *prog,
3314 static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
3315 enum bpf_access_type type,
3316 struct bpf_insn_access_aux *info)
3321 static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
3322 const struct bpf_insn *si,
3323 struct bpf_insn *insn_buf,
3324 struct bpf_prog *prog,
3329 static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
3330 enum bpf_access_type type,
3331 struct bpf_insn_access_aux *info)
3336 static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
3337 const struct bpf_insn *si,
3338 struct bpf_insn *insn_buf,
3339 struct bpf_prog *prog,
3344 #endif /* CONFIG_INET */
3346 enum bpf_text_poke_type {
3351 int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
3352 void *addr1, void *addr2);
3354 void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke,
3355 struct bpf_prog *new, struct bpf_prog *old);
3357 void *bpf_arch_text_copy(void *dst, void *src, size_t len);
3358 int bpf_arch_text_invalidate(void *dst, size_t len);
3361 bool btf_id_set_contains(const struct btf_id_set *set, u32 id);
3363 #define MAX_BPRINTF_VARARGS 12
3364 #define MAX_BPRINTF_BUF 1024
3366 struct bpf_bprintf_data {
3373 int bpf_bprintf_prepare(char *fmt, u32 fmt_size, const u64 *raw_args,
3374 u32 num_args, struct bpf_bprintf_data *data);
3375 void bpf_bprintf_cleanup(struct bpf_bprintf_data *data);
3377 #ifdef CONFIG_BPF_LSM
3378 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
3379 void bpf_cgroup_atype_put(int cgroup_atype);
3381 static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
3382 static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
3383 #endif /* CONFIG_BPF_LSM */
3392 #endif /* CONFIG_KEYS */
3394 static inline bool type_is_alloc(u32 type)
3396 return type & MEM_ALLOC;
3399 static inline gfp_t bpf_memcg_flags(gfp_t flags)
3401 if (memcg_bpf_enabled())
3402 return flags | __GFP_ACCOUNT;
3406 static inline bool bpf_is_subprog(const struct bpf_prog *prog)
3408 return prog->aux->func_idx != 0;
3411 #endif /* _LINUX_BPF_H */