[linux-2.6-block.git] / kernel / bpf / helpers.c

/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 */
#include <linux/bpf.h>
#include <linux/rcupdate.h>
#include <linux/random.h>
#include <linux/smp.h>
#include <linux/topology.h>
#include <linux/ktime.h>
#include <linux/sched.h>
#include <linux/uidgid.h>
#include <linux/filter.h>

/* If kernel subsystem is allowing eBPF programs to call this function,
 * inside its own verifier_ops->get_func_proto() callback it should return
 * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments
 *
 * Different map implementations will rely on rcu in map methods
 * lookup/update/delete, therefore eBPF programs must run under rcu lock
 * if program is allowed to access maps, so check rcu_read_lock_held in
 * all three functions.
 */
BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	return (unsigned long) map->ops->map_lookup_elem(map, key);
}

const struct bpf_func_proto bpf_map_lookup_elem_proto = {
	.func		= bpf_map_lookup_elem,
	.gpl_only	= false,
	.pkt_access	= true,
	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_MAP_KEY,
};

BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
	   void *, value, u64, flags)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	return map->ops->map_update_elem(map, key, value, flags);
}

const struct bpf_func_proto bpf_map_update_elem_proto = {
	.func		= bpf_map_update_elem,
	.gpl_only	= false,
	.pkt_access	= true,
	.ret_type	= RET_INTEGER,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_MAP_KEY,
	.arg3_type	= ARG_PTR_TO_MAP_VALUE,
	.arg4_type	= ARG_ANYTHING,
};

BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	return map->ops->map_delete_elem(map, key);
}

const struct bpf_func_proto bpf_map_delete_elem_proto = {
	.func		= bpf_map_delete_elem,
	.gpl_only	= false,
	.pkt_access	= true,
	.ret_type	= RET_INTEGER,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_MAP_KEY,
};

BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags)
{
	return map->ops->map_push_elem(map, value, flags);
}

const struct bpf_func_proto bpf_map_push_elem_proto = {
	.func		= bpf_map_push_elem,
	.gpl_only	= false,
	.pkt_access	= true,
	.ret_type	= RET_INTEGER,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_MAP_VALUE,
	.arg3_type	= ARG_ANYTHING,
};

BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
{
	return map->ops->map_pop_elem(map, value);
}

const struct bpf_func_proto bpf_map_pop_elem_proto = {
	.func		= bpf_map_pop_elem,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_UNINIT_MAP_VALUE,
};

BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
{
	return map->ops->map_peek_elem(map, value);
}

const struct bpf_func_proto bpf_map_peek_elem_proto = {
	.func		= bpf_map_pop_elem,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_PTR_TO_UNINIT_MAP_VALUE,
};

const struct bpf_func_proto bpf_get_prandom_u32_proto = {
	.func		= bpf_user_rnd_u32,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
};

BPF_CALL_0(bpf_get_smp_processor_id)
{
	return smp_processor_id();
}

const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
	.func		= bpf_get_smp_processor_id,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
};

BPF_CALL_0(bpf_get_numa_node_id)
{
	return numa_node_id();
}

const struct bpf_func_proto bpf_get_numa_node_id_proto = {
	.func		= bpf_get_numa_node_id,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
};

BPF_CALL_0(bpf_ktime_get_ns)
{
	/* NMI safe access to clock monotonic */
	return ktime_get_mono_fast_ns();
}

const struct bpf_func_proto bpf_ktime_get_ns_proto = {
	.func		= bpf_ktime_get_ns,
	.gpl_only	= true,
	.ret_type	= RET_INTEGER,
};

BPF_CALL_0(bpf_get_current_pid_tgid)
{
	struct task_struct *task = current;

	if (unlikely(!task))
		return -EINVAL;

	return (u64) task->tgid << 32 | task->pid;
}

const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {
	.func		= bpf_get_current_pid_tgid,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
};

BPF_CALL_0(bpf_get_current_uid_gid)
{
	struct task_struct *task = current;
	kuid_t uid;
	kgid_t gid;

	if (unlikely(!task))
		return -EINVAL;

	current_uid_gid(&uid, &gid);
	return (u64) from_kgid(&init_user_ns, gid) << 32 |
		     from_kuid(&init_user_ns, uid);
}

const struct bpf_func_proto bpf_get_current_uid_gid_proto = {
	.func		= bpf_get_current_uid_gid,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
};

BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)
{
	struct task_struct *task = current;

	if (unlikely(!task))
		goto err_clear;

	strncpy(buf, task->comm, size);

	/* Verifier guarantees that size > 0. For task->comm exceeding
	 * size, guarantee that buf is %NUL-terminated. Unconditionally
	 * done here to save the size test.
	 */
	buf[size - 1] = 0;
	return 0;
err_clear:
	memset(buf, 0, size);
	return -EINVAL;
}

const struct bpf_func_proto bpf_get_current_comm_proto = {
	.func		= bpf_get_current_comm,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
	.arg2_type	= ARG_CONST_SIZE,
};

#if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK)

static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
{
	arch_spinlock_t *l = (void *)lock;
	union {
		__u32 val;
		arch_spinlock_t lock;
	} u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED };

	compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");
	BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));
	BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));
	arch_spin_lock(l);
}

static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
{
	arch_spinlock_t *l = (void *)lock;

	arch_spin_unlock(l);
}

#else

static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
{
	atomic_t *l = (void *)lock;

	BUILD_BUG_ON(sizeof(*l) != sizeof(*lock));
	do {
		atomic_cond_read_relaxed(l, !VAL);
	} while (atomic_xchg(l, 1));
}

static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
{
	atomic_t *l = (void *)lock;

	atomic_set_release(l, 0);
}

#endif

static DEFINE_PER_CPU(unsigned long, irqsave_flags);

notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
{
	unsigned long flags;

	local_irq_save(flags);
	__bpf_spin_lock(lock);
	__this_cpu_write(irqsave_flags, flags);
	return 0;
}

const struct bpf_func_proto bpf_spin_lock_proto = {
	.func		= bpf_spin_lock,
	.gpl_only	= false,
	.ret_type	= RET_VOID,
	.arg1_type	= ARG_PTR_TO_SPIN_LOCK,
};

notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
{
	unsigned long flags;

	flags = __this_cpu_read(irqsave_flags);
	__bpf_spin_unlock(lock);
	local_irq_restore(flags);
	return 0;
}

const struct bpf_func_proto bpf_spin_unlock_proto = {
	.func		= bpf_spin_unlock,
	.gpl_only	= false,
	.ret_type	= RET_VOID,
	.arg1_type	= ARG_PTR_TO_SPIN_LOCK,
};

void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
			   bool lock_src)
{
	struct bpf_spin_lock *lock;

	if (lock_src)
		lock = src + map->spin_lock_off;
	else
		lock = dst + map->spin_lock_off;
	preempt_disable();
	____bpf_spin_lock(lock);
	copy_map_value(map, dst, src);
	____bpf_spin_unlock(lock);
	preempt_enable();
}

#ifdef CONFIG_CGROUPS
BPF_CALL_0(bpf_get_current_cgroup_id)
{
	struct cgroup *cgrp = task_dfl_cgroup(current);

	return cgrp->kn->id.id;
}

const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
	.func		= bpf_get_current_cgroup_id,
	.gpl_only	= false,
	.ret_type	= RET_INTEGER,
};

#ifdef CONFIG_CGROUP_BPF
DECLARE_PER_CPU(struct bpf_cgroup_storage*,
		bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);

BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
{
	/* flags argument is not used now,
	 * but provides an ability to extend the API.
	 * verifier checks that its value is correct.
	 */
	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
	struct bpf_cgroup_storage *storage;
	void *ptr;

	storage = this_cpu_read(bpf_cgroup_storage[stype]);

	if (stype == BPF_CGROUP_STORAGE_SHARED)
		ptr = &READ_ONCE(storage->buf)->data[0];
	else
		ptr = this_cpu_ptr(storage->percpu_buf);

	return (unsigned long)ptr;
}

const struct bpf_func_proto bpf_get_local_storage_proto = {
	.func		= bpf_get_local_storage,
	.gpl_only	= false,
	.ret_type	= RET_PTR_TO_MAP_VALUE,
	.arg1_type	= ARG_CONST_MAP_PTR,
	.arg2_type	= ARG_ANYTHING,
};
#endif
#endif
Commit	Line	Data
d0003ec0 AS	1	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
	2	*
	3	* This program is free software; you can redistribute it and/or
	4	* modify it under the terms of version 2 of the GNU General Public
	5	* License as published by the Free Software Foundation.
	6	*
	7	* This program is distributed in the hope that it will be useful, but
	8	* WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	10	* General Public License for more details.
	11	*/
	12	#include <linux/bpf.h>
	13	#include <linux/rcupdate.h>
03e69b50	14	#include <linux/random.h>
c04167ce	15	#include <linux/smp.h>
2d0e30c3	16	#include <linux/topology.h>
17ca8cbf	17	#include <linux/ktime.h>
ffeedafb AS	18	#include <linux/sched.h>
ffeedafb AS	19	#include <linux/uidgid.h>
f3694e00	20	#include <linux/filter.h>
d0003ec0 AS	21
	22	/* If kernel subsystem is allowing eBPF programs to call this function,
	23	* inside its own verifier_ops->get_func_proto() callback it should return
	24	* bpf_map_lookup_elem_proto, so that verifier can properly check the arguments
	25	*
	26	* Different map implementations will rely on rcu in map methods
	27	* lookup/update/delete, therefore eBPF programs must run under rcu lock
	28	* if program is allowed to access maps, so check rcu_read_lock_held in
	29	* all three functions.
	30	*/
f3694e00	31	BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map , map, void , key)
d0003ec0	32	{
d0003ec0	33	WARN_ON_ONCE(!rcu_read_lock_held());
f3694e00	34	return (unsigned long) map->ops->map_lookup_elem(map, key);
d0003ec0 AS	35	}
d0003ec0 AS	36
a2c83fff	37	const struct bpf_func_proto bpf_map_lookup_elem_proto = {
3324b584 DB	38	.func = bpf_map_lookup_elem,
3324b584 DB	39	.gpl_only = false,
36bbef52	40	.pkt_access = true,
3324b584 DB	41	.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
	42	.arg1_type = ARG_CONST_MAP_PTR,
	43	.arg2_type = ARG_PTR_TO_MAP_KEY,
d0003ec0 AS	44	};
d0003ec0 AS	45
f3694e00 DB	46	BPF_CALL_4(bpf_map_update_elem, struct bpf_map , map, void , key,
f3694e00 DB	47	void *, value, u64, flags)
d0003ec0	48	{
d0003ec0	49	WARN_ON_ONCE(!rcu_read_lock_held());
f3694e00	50	return map->ops->map_update_elem(map, key, value, flags);
d0003ec0 AS	51	}
d0003ec0 AS	52
a2c83fff	53	const struct bpf_func_proto bpf_map_update_elem_proto = {
3324b584 DB	54	.func = bpf_map_update_elem,
3324b584 DB	55	.gpl_only = false,
36bbef52	56	.pkt_access = true,
3324b584 DB	57	.ret_type = RET_INTEGER,
	58	.arg1_type = ARG_CONST_MAP_PTR,
	59	.arg2_type = ARG_PTR_TO_MAP_KEY,
	60	.arg3_type = ARG_PTR_TO_MAP_VALUE,
	61	.arg4_type = ARG_ANYTHING,
d0003ec0 AS	62	};
d0003ec0 AS	63
f3694e00	64	BPF_CALL_2(bpf_map_delete_elem, struct bpf_map , map, void , key)
d0003ec0	65	{
d0003ec0	66	WARN_ON_ONCE(!rcu_read_lock_held());
d0003ec0 AS	67	return map->ops->map_delete_elem(map, key);
	68	}
	69
a2c83fff	70	const struct bpf_func_proto bpf_map_delete_elem_proto = {
3324b584 DB	71	.func = bpf_map_delete_elem,
3324b584 DB	72	.gpl_only = false,
36bbef52	73	.pkt_access = true,
3324b584 DB	74	.ret_type = RET_INTEGER,
	75	.arg1_type = ARG_CONST_MAP_PTR,
	76	.arg2_type = ARG_PTR_TO_MAP_KEY,
d0003ec0	77	};
03e69b50	78
f1a2e44a MV	79	BPF_CALL_3(bpf_map_push_elem, struct bpf_map , map, void , value, u64, flags)
	80	{
	81	return map->ops->map_push_elem(map, value, flags);
	82	}
	83
	84	const struct bpf_func_proto bpf_map_push_elem_proto = {
	85	.func = bpf_map_push_elem,
	86	.gpl_only = false,
	87	.pkt_access = true,
	88	.ret_type = RET_INTEGER,
	89	.arg1_type = ARG_CONST_MAP_PTR,
	90	.arg2_type = ARG_PTR_TO_MAP_VALUE,
	91	.arg3_type = ARG_ANYTHING,
	92	};
	93
	94	BPF_CALL_2(bpf_map_pop_elem, struct bpf_map , map, void , value)
	95	{
	96	return map->ops->map_pop_elem(map, value);
	97	}
	98
	99	const struct bpf_func_proto bpf_map_pop_elem_proto = {
	100	.func = bpf_map_pop_elem,
	101	.gpl_only = false,
f1a2e44a MV	102	.ret_type = RET_INTEGER,
	103	.arg1_type = ARG_CONST_MAP_PTR,
	104	.arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE,
	105	};
	106
	107	BPF_CALL_2(bpf_map_peek_elem, struct bpf_map , map, void , value)
	108	{
	109	return map->ops->map_peek_elem(map, value);
	110	}
	111
	112	const struct bpf_func_proto bpf_map_peek_elem_proto = {
	113	.func = bpf_map_pop_elem,
	114	.gpl_only = false,
f1a2e44a MV	115	.ret_type = RET_INTEGER,
	116	.arg1_type = ARG_CONST_MAP_PTR,
	117	.arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE,
	118	};
	119
03e69b50	120	const struct bpf_func_proto bpf_get_prandom_u32_proto = {
3ad00405	121	.func = bpf_user_rnd_u32,
03e69b50 DB	122	.gpl_only = false,
	123	.ret_type = RET_INTEGER,
	124	};
c04167ce	125
f3694e00	126	BPF_CALL_0(bpf_get_smp_processor_id)
c04167ce	127	{
80b48c44	128	return smp_processor_id();
c04167ce DB	129	}
	130
	131	const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
	132	.func = bpf_get_smp_processor_id,
	133	.gpl_only = false,
	134	.ret_type = RET_INTEGER,
	135	};
17ca8cbf	136
2d0e30c3 DB	137	BPF_CALL_0(bpf_get_numa_node_id)
	138	{
	139	return numa_node_id();
	140	}
	141
	142	const struct bpf_func_proto bpf_get_numa_node_id_proto = {
	143	.func = bpf_get_numa_node_id,
	144	.gpl_only = false,
	145	.ret_type = RET_INTEGER,
	146	};
	147
f3694e00	148	BPF_CALL_0(bpf_ktime_get_ns)
17ca8cbf DB	149	{
	150	/* NMI safe access to clock monotonic */
	151	return ktime_get_mono_fast_ns();
	152	}
	153
	154	const struct bpf_func_proto bpf_ktime_get_ns_proto = {
	155	.func = bpf_ktime_get_ns,
	156	.gpl_only = true,
	157	.ret_type = RET_INTEGER,
	158	};
ffeedafb	159
f3694e00	160	BPF_CALL_0(bpf_get_current_pid_tgid)
ffeedafb AS	161	{
	162	struct task_struct *task = current;
	163
6088b582	164	if (unlikely(!task))
ffeedafb AS	165	return -EINVAL;
	166
	167	return (u64) task->tgid << 32 \| task->pid;
	168	}
	169
	170	const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {
	171	.func = bpf_get_current_pid_tgid,
	172	.gpl_only = false,
	173	.ret_type = RET_INTEGER,
	174	};
	175
f3694e00	176	BPF_CALL_0(bpf_get_current_uid_gid)
ffeedafb AS	177	{
	178	struct task_struct *task = current;
	179	kuid_t uid;
	180	kgid_t gid;
	181
6088b582	182	if (unlikely(!task))
ffeedafb AS	183	return -EINVAL;
	184
	185	current_uid_gid(&uid, &gid);
	186	return (u64) from_kgid(&init_user_ns, gid) << 32 \|
6088b582	187	from_kuid(&init_user_ns, uid);
ffeedafb AS	188	}
	189
	190	const struct bpf_func_proto bpf_get_current_uid_gid_proto = {
	191	.func = bpf_get_current_uid_gid,
	192	.gpl_only = false,
	193	.ret_type = RET_INTEGER,
	194	};
	195
f3694e00	196	BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)
ffeedafb AS	197	{
ffeedafb AS	198	struct task_struct *task = current;
ffeedafb	199
074f528e DB	200	if (unlikely(!task))
074f528e DB	201	goto err_clear;
ffeedafb	202
074f528e DB	203	strncpy(buf, task->comm, size);
	204
	205	/* Verifier guarantees that size > 0. For task->comm exceeding
	206	* size, guarantee that buf is %NUL-terminated. Unconditionally
	207	* done here to save the size test.
	208	*/
	209	buf[size - 1] = 0;
ffeedafb	210	return 0;
074f528e DB	211	err_clear:
	212	memset(buf, 0, size);
	213	return -EINVAL;
ffeedafb AS	214	}
	215
	216	const struct bpf_func_proto bpf_get_current_comm_proto = {
	217	.func = bpf_get_current_comm,
	218	.gpl_only = false,
	219	.ret_type = RET_INTEGER,
39f19ebb AS	220	.arg1_type = ARG_PTR_TO_UNINIT_MEM,
39f19ebb AS	221	.arg2_type = ARG_CONST_SIZE,
ffeedafb	222	};
bf6fa2c8	223
d83525ca AS	224	#if defined(CONFIG_QUEUED_SPINLOCKS) \|\| defined(CONFIG_BPF_ARCH_SPINLOCK)
	225
	226	static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
	227	{
	228	arch_spinlock_t l = (void )lock;
	229	union {
	230	__u32 val;
	231	arch_spinlock_t lock;
	232	} u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED };
	233
	234	compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");
	235	BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));
	236	BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));
	237	arch_spin_lock(l);
	238	}
	239
	240	static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
	241	{
	242	arch_spinlock_t l = (void )lock;
	243
	244	arch_spin_unlock(l);
	245	}
	246
	247	#else
	248
	249	static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
	250	{
	251	atomic_t l = (void )lock;
	252
	253	BUILD_BUG_ON(sizeof(l) != sizeof(lock));
	254	do {
	255	atomic_cond_read_relaxed(l, !VAL);
	256	} while (atomic_xchg(l, 1));
	257	}
	258
	259	static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
	260	{
	261	atomic_t l = (void )lock;
	262
	263	atomic_set_release(l, 0);
	264	}
	265
	266	#endif
	267
	268	static DEFINE_PER_CPU(unsigned long, irqsave_flags);
	269
	270	notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
	271	{
	272	unsigned long flags;
	273
	274	local_irq_save(flags);
	275	__bpf_spin_lock(lock);
	276	__this_cpu_write(irqsave_flags, flags);
	277	return 0;
	278	}
	279
	280	const struct bpf_func_proto bpf_spin_lock_proto = {
	281	.func = bpf_spin_lock,
	282	.gpl_only = false,
	283	.ret_type = RET_VOID,
	284	.arg1_type = ARG_PTR_TO_SPIN_LOCK,
	285	};
	286
	287	notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
288	{
289	unsigned long flags;
290
291	flags = __this_cpu_read(irqsave_flags);
292	__bpf_spin_unlock(lock);
293	local_irq_restore(flags);
294	return 0;
295	}
296
297	const struct bpf_func_proto bpf_spin_unlock_proto = {
298	.func = bpf_spin_unlock,
299	.gpl_only = false,
300	.ret_type = RET_VOID,
301	.arg1_type = ARG_PTR_TO_SPIN_LOCK,
302	};
303
96049f3a AS	304	void copy_map_value_locked(struct bpf_map map, void dst, void *src,
	305	bool lock_src)
	306	{
	307	struct bpf_spin_lock *lock;
	308
	309	if (lock_src)
	310	lock = src + map->spin_lock_off;
	311	else
	312	lock = dst + map->spin_lock_off;
	313	preempt_disable();
	314	____bpf_spin_lock(lock);
	315	copy_map_value(map, dst, src);
	316	____bpf_spin_unlock(lock);
	317	preempt_enable();
	318	}
	319
bf6fa2c8 YS	320	#ifdef CONFIG_CGROUPS
	321	BPF_CALL_0(bpf_get_current_cgroup_id)
	322	{
	323	struct cgroup *cgrp = task_dfl_cgroup(current);
	324
	325	return cgrp->kn->id.id;
	326	}
	327
	328	const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
	329	.func = bpf_get_current_cgroup_id,
	330	.gpl_only = false,
	331	.ret_type = RET_INTEGER,
	332	};
cd339431	333
8bad74f9	334	#ifdef CONFIG_CGROUP_BPF
f294b37e RG	335	DECLARE_PER_CPU(struct bpf_cgroup_storage*,
f294b37e RG	336	bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
cd339431 RG	337
	338	BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
	339	{
8bad74f9 RG	340	/* flags argument is not used now,
	341	* but provides an ability to extend the API.
	342	* verifier checks that its value is correct.
cd339431	343	*/
8bad74f9	344	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
f294b37e	345	struct bpf_cgroup_storage *storage;
b741f163	346	void *ptr;
8bad74f9	347
f294b37e RG	348	storage = this_cpu_read(bpf_cgroup_storage[stype]);
f294b37e RG	349
b741f163 RG	350	if (stype == BPF_CGROUP_STORAGE_SHARED)
	351	ptr = &READ_ONCE(storage->buf)->data[0];
	352	else
	353	ptr = this_cpu_ptr(storage->percpu_buf);
	354
	355	return (unsigned long)ptr;
cd339431 RG	356	}
	357
	358	const struct bpf_func_proto bpf_get_local_storage_proto = {
	359	.func = bpf_get_local_storage,
	360	.gpl_only = false,
	361	.ret_type = RET_PTR_TO_MAP_VALUE,
	362	.arg1_type = ARG_CONST_MAP_PTR,
	363	.arg2_type = ARG_ANYTHING,
	364	};
bf6fa2c8	365	#endif
8bad74f9	366	#endif