[linux-block.git] / include / linux / bpf-cgroup.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BPF_CGROUP_H
#define _BPF_CGROUP_H

#include <linux/bpf.h>
#include <linux/errno.h>
#include <linux/jump_label.h>
#include <linux/percpu.h>
#include <linux/percpu-refcount.h>
#include <linux/rbtree.h>
#include <uapi/linux/bpf.h>

struct sock;
struct sockaddr;
struct cgroup;
struct sk_buff;
struct bpf_map;
struct bpf_prog;
struct bpf_sock_ops_kern;
struct bpf_cgroup_storage;
struct ctl_table;
struct ctl_table_header;

#ifdef CONFIG_CGROUP_BPF

extern struct static_key_false cgroup_bpf_enabled_key;
#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key)

DECLARE_PER_CPU(struct bpf_cgroup_storage*,
		bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);

#define for_each_cgroup_storage_type(stype) \
	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)

struct bpf_cgroup_storage_map;

struct bpf_storage_buffer {
	struct rcu_head rcu;
	char data[];
};

struct bpf_cgroup_storage {
	union {
		struct bpf_storage_buffer *buf;
		void __percpu *percpu_buf;
	};
	struct bpf_cgroup_storage_map *map;
	struct bpf_cgroup_storage_key key;
	struct list_head list;
	struct rb_node node;
	struct rcu_head rcu;
};

struct bpf_cgroup_link {
	struct bpf_link link;
	struct cgroup *cgroup;
	enum bpf_attach_type type;
};

extern const struct bpf_link_ops bpf_cgroup_link_lops;

struct bpf_prog_list {
	struct list_head node;
	struct bpf_prog *prog;
	struct bpf_cgroup_link *link;
	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
};

struct bpf_prog_array;

struct cgroup_bpf {
	/* array of effective progs in this cgroup */
	struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE];

	/* attached progs to this cgroup and attach flags
	 * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
	 * have either zero or one element
	 * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
	 */
	struct list_head progs[MAX_BPF_ATTACH_TYPE];
	u32 flags[MAX_BPF_ATTACH_TYPE];

	/* temp storage for effective prog array used by prog_attach/detach */
	struct bpf_prog_array *inactive;

	/* reference counter used to detach bpf programs after cgroup removal */
	struct percpu_ref refcnt;

	/* cgroup_bpf is released using a work queue */
	struct work_struct release_work;
};

int cgroup_bpf_inherit(struct cgroup *cgrp);
void cgroup_bpf_offline(struct cgroup *cgrp);

int __cgroup_bpf_attach(struct cgroup *cgrp,
			struct bpf_prog *prog, struct bpf_prog *replace_prog,
			struct bpf_cgroup_link *link,
			enum bpf_attach_type type, u32 flags);
int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
			struct bpf_cgroup_link *link,
			enum bpf_attach_type type);
int __cgroup_bpf_replace(struct cgroup *cgrp, struct bpf_cgroup_link *link,
			 struct bpf_prog *new_prog);
int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
		       union bpf_attr __user *uattr);

/* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */
int cgroup_bpf_attach(struct cgroup *cgrp,
		      struct bpf_prog *prog, struct bpf_prog *replace_prog,
		      struct bpf_cgroup_link *link, enum bpf_attach_type type,
		      u32 flags);
int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
		      enum bpf_attach_type type);
int cgroup_bpf_replace(struct bpf_link *link, struct bpf_prog *old_prog,
		       struct bpf_prog *new_prog);
int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
		     union bpf_attr __user *uattr);

int __cgroup_bpf_run_filter_skb(struct sock *sk,
				struct sk_buff *skb,
				enum bpf_attach_type type);

int __cgroup_bpf_run_filter_sk(struct sock *sk,
			       enum bpf_attach_type type);

int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
				      struct sockaddr *uaddr,
				      enum bpf_attach_type type,
				      void *t_ctx);

int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
				     struct bpf_sock_ops_kern *sock_ops,
				     enum bpf_attach_type type);

int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
				      short access, enum bpf_attach_type type);

int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
				   struct ctl_table *table, int write,
				   void **buf, size_t *pcount, loff_t *ppos,
				   enum bpf_attach_type type);

int __cgroup_bpf_run_filter_setsockopt(struct sock *sock, int *level,
				       int *optname, char __user *optval,
				       int *optlen, char **kernel_optval);
int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
				       int optname, char __user *optval,
				       int __user *optlen, int max_optlen,
				       int retval);

static inline enum bpf_cgroup_storage_type cgroup_storage_type(
	struct bpf_map *map)
{
	if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
		return BPF_CGROUP_STORAGE_PERCPU;

	return BPF_CGROUP_STORAGE_SHARED;
}

static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage
					  *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
{
	enum bpf_cgroup_storage_type stype;

	for_each_cgroup_storage_type(stype)
		this_cpu_write(bpf_cgroup_storage[stype], storage[stype]);
}

struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog,
					enum bpf_cgroup_storage_type stype);
void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage);
void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
			     struct cgroup *cgroup,
			     enum bpf_attach_type type);
void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);
int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux, struct bpf_map *map);
void bpf_cgroup_storage_release(struct bpf_prog_aux *aux, struct bpf_map *map);

int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value);
int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,
				     void *value, u64 flags);

/* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */
#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb)			      \
({									      \
	int __ret = 0;							      \
	if (cgroup_bpf_enabled)						      \
		__ret = __cgroup_bpf_run_filter_skb(sk, skb,		      \
						    BPF_CGROUP_INET_INGRESS); \
									      \
	__ret;								      \
})

#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb)			       \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled && sk && sk == skb->sk) {		       \
		typeof(sk) __sk = sk_to_full_sk(sk);			       \
		if (sk_fullsock(__sk))					       \
			__ret = __cgroup_bpf_run_filter_skb(__sk, skb,	       \
						      BPF_CGROUP_INET_EGRESS); \
	}								       \
	__ret;								       \
})

#define BPF_CGROUP_RUN_SK_PROG(sk, type)				       \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled) {					       \
		__ret = __cgroup_bpf_run_filter_sk(sk, type);		       \
	}								       \
	__ret;								       \
})

#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk)				       \
	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_CREATE)

#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk)				       \
	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET4_POST_BIND)

#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk)				       \
	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET6_POST_BIND)

#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type)				       \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled)						       \
		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type,     \
							  NULL);	       \
	__ret;								       \
})

#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx)		       \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled)	{					       \
		lock_sock(sk);						       \
		__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type,     \
							  t_ctx);	       \
		release_sock(sk);					       \
	}								       \
	__ret;								       \
})

#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr)			       \
	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_BIND)

#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr)			       \
	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_BIND)

#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \
					    sk->sk_prot->pre_connect)

#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr)			       \
	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)

#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr)			       \
	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_CONNECT)

#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr)		       \
	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_CONNECT, NULL)

#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr)		       \
	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_CONNECT, NULL)

#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_SENDMSG, t_ctx)

#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx)		       \
	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_SENDMSG, t_ctx)

#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr)			\
	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_RECVMSG, NULL)

#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr)			\
	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_RECVMSG, NULL)

#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops)				       \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled && (sock_ops)->sk) {	       \
		typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk);	       \
		if (__sk && sk_fullsock(__sk))				       \
			__ret = __cgroup_bpf_run_filter_sock_ops(__sk,	       \
								 sock_ops,     \
							 BPF_CGROUP_SOCK_OPS); \
	}								       \
	__ret;								       \
})

#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access)	      \
({									      \
	int __ret = 0;							      \
	if (cgroup_bpf_enabled)						      \
		__ret = __cgroup_bpf_check_dev_permission(type, major, minor, \
							  access,	      \
							  BPF_CGROUP_DEVICE); \
									      \
	__ret;								      \
})


#define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos)  \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled)						       \
		__ret = __cgroup_bpf_run_filter_sysctl(head, table, write,     \
						       buf, count, pos,        \
						       BPF_CGROUP_SYSCTL);     \
	__ret;								       \
})

#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen,   \
				       kernel_optval)			       \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled)						       \
		__ret = __cgroup_bpf_run_filter_setsockopt(sock, level,	       \
							   optname, optval,    \
							   optlen,	       \
							   kernel_optval);     \
	__ret;								       \
})

#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen)			       \
({									       \
	int __ret = 0;							       \
	if (cgroup_bpf_enabled)						       \
		get_user(__ret, optlen);				       \
	__ret;								       \
})

#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen,   \
				       max_optlen, retval)		       \
({									       \
	int __ret = retval;						       \
	if (cgroup_bpf_enabled)						       \
		__ret = __cgroup_bpf_run_filter_getsockopt(sock, level,	       \
							   optname, optval,    \
							   optlen, max_optlen, \
							   retval);	       \
	__ret;								       \
})

int cgroup_bpf_prog_attach(const union bpf_attr *attr,
			   enum bpf_prog_type ptype, struct bpf_prog *prog);
int cgroup_bpf_prog_detach(const union bpf_attr *attr,
			   enum bpf_prog_type ptype);
int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
int cgroup_bpf_prog_query(const union bpf_attr *attr,
			  union bpf_attr __user *uattr);
#else

struct bpf_prog;
struct bpf_link;
struct cgroup_bpf {};
static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}

static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr,
					 enum bpf_prog_type ptype,
					 struct bpf_prog *prog)
{
	return -EINVAL;
}

static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr,
					 enum bpf_prog_type ptype)
{
	return -EINVAL;
}

static inline int cgroup_bpf_link_attach(const union bpf_attr *attr,
					 struct bpf_prog *prog)
{
	return -EINVAL;
}

static inline int cgroup_bpf_replace(struct bpf_link *link,
				     struct bpf_prog *old_prog,
				     struct bpf_prog *new_prog)
{
	return -EINVAL;
}

static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,
					union bpf_attr __user *uattr)
{
	return -EINVAL;
}

static inline void bpf_cgroup_storage_set(
	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}
static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
					    struct bpf_map *map) { return 0; }
static inline void bpf_cgroup_storage_release(struct bpf_prog_aux *aux,
					      struct bpf_map *map) {}
static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
	struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; }
static inline void bpf_cgroup_storage_free(
	struct bpf_cgroup_storage *storage) {}
static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key,
						 void *value) {
	return 0;
}
static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
					void *key, void *value, u64 flags) {
	return 0;
}

#define cgroup_bpf_enabled (0)
#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; })
#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; })
#define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; })
#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
				       optlen, max_optlen, retval) ({ retval; })
#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
				       kernel_optval) ({ 0; })

#define for_each_cgroup_storage_type(stype) for (; false; )

#endif /* CONFIG_CGROUP_BPF */

#endif /* _BPF_CGROUP_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
30070984 DM	2	#ifndef _BPF_CGROUP_H
	3	#define _BPF_CGROUP_H
	4
8bad74f9	5	#include <linux/bpf.h>
f292b87d	6	#include <linux/errno.h>
30070984	7	#include <linux/jump_label.h>
aa0ad5b0	8	#include <linux/percpu.h>
4bfc0bb2	9	#include <linux/percpu-refcount.h>
de9cbbaa	10	#include <linux/rbtree.h>
30070984 DM	11	#include <uapi/linux/bpf.h>
	12
	13	struct sock;
4fbac77d	14	struct sockaddr;
30070984 DM	15	struct cgroup;
30070984 DM	16	struct sk_buff;
de9cbbaa RG	17	struct bpf_map;
de9cbbaa RG	18	struct bpf_prog;
40304b2a	19	struct bpf_sock_ops_kern;
de9cbbaa	20	struct bpf_cgroup_storage;
7b146ceb AI	21	struct ctl_table;
7b146ceb AI	22	struct ctl_table_header;
30070984 DM	23
	24	#ifdef CONFIG_CGROUP_BPF
	25
	26	extern struct static_key_false cgroup_bpf_enabled_key;
	27	#define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key)
	28
f294b37e RG	29	DECLARE_PER_CPU(struct bpf_cgroup_storage*,
f294b37e RG	30	bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
8bad74f9 RG	31
	32	#define for_each_cgroup_storage_type(stype) \
	33	for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
aa0ad5b0	34
de9cbbaa RG	35	struct bpf_cgroup_storage_map;
	36
	37	struct bpf_storage_buffer {
	38	struct rcu_head rcu;
d7f10df8	39	char data[];
de9cbbaa RG	40	};
	41
	42	struct bpf_cgroup_storage {
b741f163 RG	43	union {
	44	struct bpf_storage_buffer *buf;
	45	void __percpu *percpu_buf;
	46	};
de9cbbaa RG	47	struct bpf_cgroup_storage_map *map;
	48	struct bpf_cgroup_storage_key key;
	49	struct list_head list;
	50	struct rb_node node;
	51	struct rcu_head rcu;
	52	};
	53
af6eea57 AN	54	struct bpf_cgroup_link {
	55	struct bpf_link link;
	56	struct cgroup *cgroup;
	57	enum bpf_attach_type type;
	58	};
	59
	60	extern const struct bpf_link_ops bpf_cgroup_link_lops;
	61
324bda9e AS	62	struct bpf_prog_list {
	63	struct list_head node;
	64	struct bpf_prog *prog;
af6eea57	65	struct bpf_cgroup_link *link;
8bad74f9	66	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
324bda9e AS	67	};
	68
	69	struct bpf_prog_array;
	70
30070984	71	struct cgroup_bpf {
324bda9e AS	72	/* array of effective progs in this cgroup */
	73	struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE];
	74
	75	/* attached progs to this cgroup and attach flags
	76	* when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
	77	* have either zero or one element
	78	* when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
30070984	79	*/
324bda9e AS	80	struct list_head progs[MAX_BPF_ATTACH_TYPE];
	81	u32 flags[MAX_BPF_ATTACH_TYPE];
	82
	83	/* temp storage for effective prog array used by prog_attach/detach */
dbcc1ba2	84	struct bpf_prog_array *inactive;
4bfc0bb2 RG	85
	86	/* reference counter used to detach bpf programs after cgroup removal */
	87	struct percpu_ref refcnt;
	88
	89	/* cgroup_bpf is released using a work queue */
	90	struct work_struct release_work;
30070984 DM	91	};
30070984 DM	92
324bda9e	93	int cgroup_bpf_inherit(struct cgroup *cgrp);
4bfc0bb2	94	void cgroup_bpf_offline(struct cgroup *cgrp);
30070984	95
af6eea57 AN	96	int __cgroup_bpf_attach(struct cgroup *cgrp,
	97	struct bpf_prog prog, struct bpf_prog replace_prog,
	98	struct bpf_cgroup_link *link,
324bda9e AS	99	enum bpf_attach_type type, u32 flags);
324bda9e AS	100	int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog,
af6eea57	101	struct bpf_cgroup_link *link,
1832f4ef	102	enum bpf_attach_type type);
0c991ebc AN	103	int __cgroup_bpf_replace(struct cgroup cgrp, struct bpf_cgroup_link link,
0c991ebc AN	104	struct bpf_prog *new_prog);
468e2f64 AS	105	int __cgroup_bpf_query(struct cgroup cgrp, const union bpf_attr attr,
468e2f64 AS	106	union bpf_attr __user *uattr);
30070984	107
324bda9e	108	/* Wrapper for __cgroup_bpf_() protected by cgroup_mutex /
af6eea57 AN	109	int cgroup_bpf_attach(struct cgroup *cgrp,
	110	struct bpf_prog prog, struct bpf_prog replace_prog,
	111	struct bpf_cgroup_link *link, enum bpf_attach_type type,
7dd68b32	112	u32 flags);
324bda9e	113	int cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog,
af6eea57	114	enum bpf_attach_type type);
0c991ebc AN	115	int cgroup_bpf_replace(struct bpf_link link, struct bpf_prog old_prog,
0c991ebc AN	116	struct bpf_prog *new_prog);
468e2f64 AS	117	int cgroup_bpf_query(struct cgroup cgrp, const union bpf_attr attr,
468e2f64 AS	118	union bpf_attr __user *uattr);
30070984	119
b2cd1257 DA	120	int __cgroup_bpf_run_filter_skb(struct sock *sk,
	121	struct sk_buff *skb,
	122	enum bpf_attach_type type);
	123
61023658 DA	124	int __cgroup_bpf_run_filter_sk(struct sock *sk,
	125	enum bpf_attach_type type);
	126
4fbac77d AI	127	int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
4fbac77d AI	128	struct sockaddr *uaddr,
1cedee13 AI	129	enum bpf_attach_type type,
1cedee13 AI	130	void *t_ctx);
4fbac77d	131
40304b2a LB	132	int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
	133	struct bpf_sock_ops_kern *sock_ops,
	134	enum bpf_attach_type type);
	135
ebc614f6 RG	136	int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
	137	short access, enum bpf_attach_type type);
	138
7b146ceb AI	139	int __cgroup_bpf_run_filter_sysctl(struct ctl_table_header *head,
7b146ceb AI	140	struct ctl_table *table, int write,
32927393	141	void *buf, size_t pcount, loff_t *ppos,
e1550bfe	142	enum bpf_attach_type type);
7b146ceb	143
0d01da6a SF	144	int __cgroup_bpf_run_filter_setsockopt(struct sock sock, int level,
	145	int optname, char __user optval,
	146	int optlen, char *kernel_optval);
	147	int __cgroup_bpf_run_filter_getsockopt(struct sock *sk, int level,
	148	int optname, char __user *optval,
	149	int __user *optlen, int max_optlen,
	150	int retval);
	151
8bad74f9 RG	152	static inline enum bpf_cgroup_storage_type cgroup_storage_type(
8bad74f9 RG	153	struct bpf_map *map)
aa0ad5b0	154	{
b741f163 RG	155	if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
	156	return BPF_CGROUP_STORAGE_PERCPU;
	157
8bad74f9 RG	158	return BPF_CGROUP_STORAGE_SHARED;
	159	}
	160
	161	static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage
	162	*storage[MAX_BPF_CGROUP_STORAGE_TYPE])
	163	{
	164	enum bpf_cgroup_storage_type stype;
aa0ad5b0	165
f294b37e RG	166	for_each_cgroup_storage_type(stype)
f294b37e RG	167	this_cpu_write(bpf_cgroup_storage[stype], storage[stype]);
aa0ad5b0 RG	168	}
aa0ad5b0 RG	169
8bad74f9 RG	170	struct bpf_cgroup_storage bpf_cgroup_storage_alloc(struct bpf_prog prog,
8bad74f9 RG	171	enum bpf_cgroup_storage_type stype);
de9cbbaa RG	172	void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage);
	173	void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
	174	struct cgroup *cgroup,
	175	enum bpf_attach_type type);
	176	void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);
e4730423 DB	177	int bpf_cgroup_storage_assign(struct bpf_prog_aux aux, struct bpf_map map);
e4730423 DB	178	void bpf_cgroup_storage_release(struct bpf_prog_aux aux, struct bpf_map map);
de9cbbaa	179
b741f163 RG	180	int bpf_percpu_cgroup_storage_copy(struct bpf_map map, void key, void *value);
	181	int bpf_percpu_cgroup_storage_update(struct bpf_map map, void key,
	182	void *value, u64 flags);
	183
b2cd1257 DA	184	/* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */
	185	#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \
	186	({ \
	187	int __ret = 0; \
	188	if (cgroup_bpf_enabled) \
	189	__ret = __cgroup_bpf_run_filter_skb(sk, skb, \
	190	BPF_CGROUP_INET_INGRESS); \
	191	\
	192	__ret; \
30070984 DM	193	})
30070984 DM	194
b2cd1257 DA	195	#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \
	196	({ \
	197	int __ret = 0; \
	198	if (cgroup_bpf_enabled && sk && sk == skb->sk) { \
	199	typeof(sk) __sk = sk_to_full_sk(sk); \
	200	if (sk_fullsock(__sk)) \
	201	__ret = __cgroup_bpf_run_filter_skb(__sk, skb, \
	202	BPF_CGROUP_INET_EGRESS); \
	203	} \
	204	__ret; \
30070984 DM	205	})
30070984 DM	206
aac3fc32	207	#define BPF_CGROUP_RUN_SK_PROG(sk, type) \
61023658 DA	208	({ \
61023658 DA	209	int __ret = 0; \
ee07862f	210	if (cgroup_bpf_enabled) { \
aac3fc32	211	__ret = __cgroup_bpf_run_filter_sk(sk, type); \
61023658 DA	212	} \
	213	__ret; \
	214	})
	215
aac3fc32 AI	216	#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) \
	217	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_CREATE)
	218
	219	#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) \
	220	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET4_POST_BIND)
	221
	222	#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) \
	223	BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET6_POST_BIND)
	224
4fbac77d AI	225	#define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \
	226	({ \
	227	int __ret = 0; \
	228	if (cgroup_bpf_enabled) \
1cedee13 AI	229	__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
1cedee13 AI	230	NULL); \
4fbac77d AI	231	__ret; \
	232	})
	233
1cedee13	234	#define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \
d74bad4e AI	235	({ \
	236	int __ret = 0; \
	237	if (cgroup_bpf_enabled) { \
	238	lock_sock(sk); \
1cedee13 AI	239	__ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
1cedee13 AI	240	t_ctx); \
d74bad4e AI	241	release_sock(sk); \
	242	} \
	243	__ret; \
	244	})
	245
4fbac77d AI	246	#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) \
	247	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_BIND)
	248
	249	#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) \
	250	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_BIND)
	251
d74bad4e AI	252	#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \
	253	sk->sk_prot->pre_connect)
	254
	255	#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \
	256	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
	257
	258	#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) \
	259	BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_CONNECT)
	260
	261	#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) \
1cedee13	262	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_CONNECT, NULL)
d74bad4e AI	263
d74bad4e AI	264	#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) \
1cedee13 AI	265	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_CONNECT, NULL)
	266
	267	#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) \
	268	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_SENDMSG, t_ctx)
	269
	270	#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) \
	271	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_SENDMSG, t_ctx)
d74bad4e	272
983695fa DB	273	#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) \
	274	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_RECVMSG, NULL)
	275
	276	#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) \
	277	BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_RECVMSG, NULL)
	278
40304b2a LB	279	#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \
	280	({ \
	281	int __ret = 0; \
	282	if (cgroup_bpf_enabled && (sock_ops)->sk) { \
	283	typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \
df39a9f1	284	if (__sk && sk_fullsock(__sk)) \
40304b2a LB	285	__ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
	286	sock_ops, \
	287	BPF_CGROUP_SOCK_OPS); \
	288	} \
	289	__ret; \
	290	})
ebc614f6 RG	291
	292	#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \
	293	({ \
	294	int __ret = 0; \
	295	if (cgroup_bpf_enabled) \
	296	__ret = __cgroup_bpf_check_dev_permission(type, major, minor, \
	297	access, \
	298	BPF_CGROUP_DEVICE); \
	299	\
	300	__ret; \
	301	})
7b146ceb AI	302
7b146ceb AI	303
32927393	304	#define BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, buf, count, pos) \
7b146ceb AI	305	({ \
	306	int __ret = 0; \
	307	if (cgroup_bpf_enabled) \
	308	__ret = __cgroup_bpf_run_filter_sysctl(head, table, write, \
32927393	309	buf, count, pos, \
7b146ceb AI	310	BPF_CGROUP_SYSCTL); \
	311	__ret; \
	312	})
	313
0d01da6a SF	314	#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
	315	kernel_optval) \
	316	({ \
	317	int __ret = 0; \
	318	if (cgroup_bpf_enabled) \
	319	__ret = __cgroup_bpf_run_filter_setsockopt(sock, level, \
	320	optname, optval, \
	321	optlen, \
	322	kernel_optval); \
	323	__ret; \
	324	})
	325
	326	#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) \
	327	({ \
	328	int __ret = 0; \
	329	if (cgroup_bpf_enabled) \
	330	get_user(__ret, optlen); \
	331	__ret; \
	332	})
	333
	334	#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, optlen, \
	335	max_optlen, retval) \
	336	({ \
	337	int __ret = retval; \
	338	if (cgroup_bpf_enabled) \
	339	__ret = __cgroup_bpf_run_filter_getsockopt(sock, level, \
	340	optname, optval, \
	341	optlen, max_optlen, \
	342	retval); \
	343	__ret; \
	344	})
	345
fdb5c453 SY	346	int cgroup_bpf_prog_attach(const union bpf_attr *attr,
	347	enum bpf_prog_type ptype, struct bpf_prog *prog);
	348	int cgroup_bpf_prog_detach(const union bpf_attr *attr,
	349	enum bpf_prog_type ptype);
af6eea57	350	int cgroup_bpf_link_attach(const union bpf_attr attr, struct bpf_prog prog);
fdb5c453 SY	351	int cgroup_bpf_prog_query(const union bpf_attr *attr,
fdb5c453 SY	352	union bpf_attr __user *uattr);
30070984 DM	353	#else
30070984 DM	354
fdb5c453	355	struct bpf_prog;
0c991ebc	356	struct bpf_link;
30070984	357	struct cgroup_bpf {};
324bda9e	358	static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
4bfc0bb2	359	static inline void cgroup_bpf_offline(struct cgroup *cgrp) {}
30070984	360
fdb5c453 SY	361	static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr,
	362	enum bpf_prog_type ptype,
	363	struct bpf_prog *prog)
	364	{
	365	return -EINVAL;
	366	}
	367
	368	static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr,
	369	enum bpf_prog_type ptype)
	370	{
	371	return -EINVAL;
	372	}
	373
af6eea57 AN	374	static inline int cgroup_bpf_link_attach(const union bpf_attr *attr,
	375	struct bpf_prog *prog)
	376	{
	377	return -EINVAL;
	378	}
	379
0c991ebc AN	380	static inline int cgroup_bpf_replace(struct bpf_link *link,
	381	struct bpf_prog *old_prog,
	382	struct bpf_prog *new_prog)
	383	{
	384	return -EINVAL;
	385	}
	386
fdb5c453 SY	387	static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,
	388	union bpf_attr __user *uattr)
	389	{
	390	return -EINVAL;
	391	}
	392
8bad74f9 RG	393	static inline void bpf_cgroup_storage_set(
8bad74f9 RG	394	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}
e4730423	395	static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
de9cbbaa	396	struct bpf_map *map) { return 0; }
e4730423	397	static inline void bpf_cgroup_storage_release(struct bpf_prog_aux *aux,
de9cbbaa RG	398	struct bpf_map *map) {}
de9cbbaa RG	399	static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
71b91a50	400	struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return NULL; }
de9cbbaa RG	401	static inline void bpf_cgroup_storage_free(
de9cbbaa RG	402	struct bpf_cgroup_storage *storage) {}
b741f163 RG	403	static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map map, void key,
	404	void *value) {
	405	return 0;
	406	}
	407	static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
	408	void key, void value, u64 flags) {
	409	return 0;
	410	}
de9cbbaa	411
13193b0f	412	#define cgroup_bpf_enabled (0)
d74bad4e	413	#define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
30070984 DM	414	#define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
30070984 DM	415	#define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
61023658	416	#define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
4fbac77d AI	417	#define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) ({ 0; })
4fbac77d AI	418	#define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) ({ 0; })
aac3fc32 AI	419	#define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
aac3fc32 AI	420	#define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
d74bad4e AI	421	#define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
	422	#define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; })
	423	#define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; })
	424	#define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; })
1cedee13 AI	425	#define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
1cedee13 AI	426	#define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
983695fa DB	427	#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr) ({ 0; })
983695fa DB	428	#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr) ({ 0; })
40304b2a	429	#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
ebc614f6	430	#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; })
32927393	431	#define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; })
0d01da6a SF	432	#define BPF_CGROUP_GETSOCKOPT_MAX_OPTLEN(optlen) ({ 0; })
	433	#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \
	434	optlen, max_optlen, retval) ({ retval; })
	435	#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
	436	kernel_optval) ({ 0; })
30070984	437
8bad74f9 RG	438	#define for_each_cgroup_storage_type(stype) for (; false; )
8bad74f9 RG	439
30070984 DM	440	#endif /* CONFIG_CGROUP_BPF */
	441
	442	#endif /* _BPF_CGROUP_H */