[linux-2.6-block.git] / tools / lib / bpf / usdt.bpf.h

/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
#ifndef __USDT_BPF_H__
#define __USDT_BPF_H__

#include <linux/errno.h>
#include "bpf_helpers.h"
#include "bpf_tracing.h"

/* Below types and maps are internal implementation details of libbpf's USDT
 * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
 * be considered an unstable API as well and might be adjusted based on user
 * feedback from using libbpf's USDT support in production.
 */

/* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
 * map that keeps track of USDT argument specifications. This might be
 * necessary if there are a lot of USDT attachments.
 */
#ifndef BPF_USDT_MAX_SPEC_CNT
#define BPF_USDT_MAX_SPEC_CNT 256
#endif
/* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
 * map that keeps track of IP (memory address) mapping to USDT argument
 * specification.
 * Note, if kernel supports BPF cookies, this map is not used and could be
 * resized all the way to 1 to save a bit of memory.
 */
#ifndef BPF_USDT_MAX_IP_CNT
#define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
#endif

enum __bpf_usdt_arg_type {
	BPF_USDT_ARG_CONST,
	BPF_USDT_ARG_REG,
	BPF_USDT_ARG_REG_DEREF,
};

struct __bpf_usdt_arg_spec {
	/* u64 scalar interpreted depending on arg_type, see below */
	__u64 val_off;
	/* arg location case, see bpf_udst_arg() for details */
	enum __bpf_usdt_arg_type arg_type;
	/* offset of referenced register within struct pt_regs */
	short reg_off;
	/* whether arg should be interpreted as signed value */
	bool arg_signed;
	/* number of bits that need to be cleared and, optionally,
	 * sign-extended to cast arguments that are 1, 2, or 4 bytes
	 * long into final 8-byte u64/s64 value returned to user
	 */
	char arg_bitshift;
};

/* should match USDT_MAX_ARG_CNT in usdt.c exactly */
#define BPF_USDT_MAX_ARG_CNT 12
struct __bpf_usdt_spec {
	struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
	__u64 usdt_cookie;
	short arg_cnt;
};

struct {
	__uint(type, BPF_MAP_TYPE_ARRAY);
	__uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
	__type(key, int);
	__type(value, struct __bpf_usdt_spec);
} __bpf_usdt_specs SEC(".maps") __weak;

struct {
	__uint(type, BPF_MAP_TYPE_HASH);
	__uint(max_entries, BPF_USDT_MAX_IP_CNT);
	__type(key, long);
	__type(value, __u32);
} __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;

extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;

static __always_inline
int __bpf_usdt_spec_id(struct pt_regs *ctx)
{
	if (!LINUX_HAS_BPF_COOKIE) {
		long ip = PT_REGS_IP(ctx);
		int *spec_id_ptr;

		spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
		return spec_id_ptr ? *spec_id_ptr : -ESRCH;
	}

	return bpf_get_attach_cookie(ctx);
}

/* Return number of USDT arguments defined for currently traced USDT. */
__weak __hidden
int bpf_usdt_arg_cnt(struct pt_regs *ctx)
{
	struct __bpf_usdt_spec *spec;
	int spec_id;

	spec_id = __bpf_usdt_spec_id(ctx);
	if (spec_id < 0)
		return -ESRCH;

	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
	if (!spec)
		return -ESRCH;

	return spec->arg_cnt;
}

/* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
 * Returns 0 on success; negative error, otherwise.
 * On error *res is guaranteed to be set to zero.
 */
__weak __hidden
int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)
{
	struct __bpf_usdt_spec *spec;
	struct __bpf_usdt_arg_spec *arg_spec;
	unsigned long val;
	int err, spec_id;

	*res = 0;

	spec_id = __bpf_usdt_spec_id(ctx);
	if (spec_id < 0)
		return -ESRCH;

	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
	if (!spec)
		return -ESRCH;

	if (arg_num >= BPF_USDT_MAX_ARG_CNT)
		return -ENOENT;
	barrier_var(arg_num);
	if (arg_num >= spec->arg_cnt)
		return -ENOENT;

	arg_spec = &spec->args[arg_num];
	switch (arg_spec->arg_type) {
	case BPF_USDT_ARG_CONST:
		/* Arg is just a constant ("-4@$-9" in USDT arg spec).
		 * value is recorded in arg_spec->val_off directly.
		 */
		val = arg_spec->val_off;
		break;
	case BPF_USDT_ARG_REG:
		/* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
		 * so we read the contents of that register directly from
		 * struct pt_regs. To keep things simple user-space parts
		 * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
		 */
		err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
		if (err)
			return err;
		break;
	case BPF_USDT_ARG_REG_DEREF:
		/* Arg is in memory addressed by register, plus some offset
		 * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
		 * identified like with BPF_USDT_ARG_REG case, and the offset
		 * is in arg_spec->val_off. We first fetch register contents
		 * from pt_regs, then do another user-space probe read to
		 * fetch argument value itself.
		 */
		err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
		if (err)
			return err;
		err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
		if (err)
			return err;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
		val >>= arg_spec->arg_bitshift;
#endif
		break;
	default:
		return -EINVAL;
	}

	/* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
	 * necessary upper arg_bitshift bits, with sign extension if argument
	 * is signed
	 */
	val <<= arg_spec->arg_bitshift;
	if (arg_spec->arg_signed)
		val = ((long)val) >> arg_spec->arg_bitshift;
	else
		val = val >> arg_spec->arg_bitshift;
	*res = val;
	return 0;
}

/* Retrieve user-specified cookie value provided during attach as
 * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
 * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
 * utilizing BPF cookies internally, so user can't use BPF cookie directly
 * for USDT programs and has to use bpf_usdt_cookie() API instead.
 */
__weak __hidden
long bpf_usdt_cookie(struct pt_regs *ctx)
{
	struct __bpf_usdt_spec *spec;
	int spec_id;

	spec_id = __bpf_usdt_spec_id(ctx);
	if (spec_id < 0)
		return 0;

	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
	if (!spec)
		return 0;

	return spec->usdt_cookie;
}

/* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
#define ___bpf_usdt_args0() ctx
#define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); _x; })
#define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); _x; })
#define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); _x; })
#define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); _x; })
#define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); _x; })
#define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); _x; })
#define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); _x; })
#define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); _x; })
#define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); _x; })
#define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); _x; })
#define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); _x; })
#define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); _x; })
#define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)

/*
 * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
 * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
 * Original struct pt_regs * context is preserved as 'ctx' argument.
 */
#define BPF_USDT(name, args...)						    \
name(struct pt_regs *ctx);						    \
static __always_inline typeof(name(0))					    \
____##name(struct pt_regs *ctx, ##args);				    \
typeof(name(0)) name(struct pt_regs *ctx)				    \
{									    \
        _Pragma("GCC diagnostic push")					    \
        _Pragma("GCC diagnostic ignored \"-Wint-conversion\"")		    \
        return ____##name(___bpf_usdt_args(args));			    \
        _Pragma("GCC diagnostic pop")					    \
}									    \
static __always_inline typeof(name(0))					    \
____##name(struct pt_regs *ctx, ##args)

#endif /* __USDT_BPF_H__ */
Commit	Line	Data
d72e2968 AN	1	/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
	2	/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
	3	#ifndef __USDT_BPF_H__
	4	#define __USDT_BPF_H__
	5
	6	#include <linux/errno.h>
dde3979b SK	7	#include "bpf_helpers.h"
dde3979b SK	8	#include "bpf_tracing.h"
d72e2968 AN	9
	10	/* Below types and maps are internal implementation details of libbpf's USDT
	11	* support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
	12	* be considered an unstable API as well and might be adjusted based on user
	13	* feedback from using libbpf's USDT support in production.
	14	*/
	15
	16	/* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
	17	* map that keeps track of USDT argument specifications. This might be
	18	* necessary if there are a lot of USDT attachments.
	19	*/
	20	#ifndef BPF_USDT_MAX_SPEC_CNT
	21	#define BPF_USDT_MAX_SPEC_CNT 256
	22	#endif
	23	/* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
	24	* map that keeps track of IP (memory address) mapping to USDT argument
	25	* specification.
	26	* Note, if kernel supports BPF cookies, this map is not used and could be
	27	* resized all the way to 1 to save a bit of memory.
	28	*/
	29	#ifndef BPF_USDT_MAX_IP_CNT
	30	#define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
	31	#endif
d72e2968 AN	32
	33	enum __bpf_usdt_arg_type {
	34	BPF_USDT_ARG_CONST,
	35	BPF_USDT_ARG_REG,
	36	BPF_USDT_ARG_REG_DEREF,
	37	};
	38
	39	struct __bpf_usdt_arg_spec {
	40	/* u64 scalar interpreted depending on arg_type, see below */
	41	__u64 val_off;
	42	/* arg location case, see bpf_udst_arg() for details */
	43	enum __bpf_usdt_arg_type arg_type;
	44	/* offset of referenced register within struct pt_regs */
	45	short reg_off;
	46	/* whether arg should be interpreted as signed value */
	47	bool arg_signed;
	48	/* number of bits that need to be cleared and, optionally,
	49	* sign-extended to cast arguments that are 1, 2, or 4 bytes
	50	* long into final 8-byte u64/s64 value returned to user
	51	*/
	52	char arg_bitshift;
	53	};
	54
	55	/* should match USDT_MAX_ARG_CNT in usdt.c exactly */
	56	#define BPF_USDT_MAX_ARG_CNT 12
	57	struct __bpf_usdt_spec {
	58	struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
	59	__u64 usdt_cookie;
	60	short arg_cnt;
	61	};
	62
	63	struct {
	64	__uint(type, BPF_MAP_TYPE_ARRAY);
	65	__uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
	66	__type(key, int);
	67	__type(value, struct __bpf_usdt_spec);
	68	} __bpf_usdt_specs SEC(".maps") __weak;
	69
	70	struct {
	71	__uint(type, BPF_MAP_TYPE_HASH);
	72	__uint(max_entries, BPF_USDT_MAX_IP_CNT);
	73	__type(key, long);
	74	__type(value, __u32);
	75	} __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
	76
55d00c37	77	extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;
d72e2968 AN	78
	79	static __always_inline
	80	int __bpf_usdt_spec_id(struct pt_regs *ctx)
	81	{
55d00c37	82	if (!LINUX_HAS_BPF_COOKIE) {
d72e2968 AN	83	long ip = PT_REGS_IP(ctx);
	84	int *spec_id_ptr;
	85
	86	spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
	87	return spec_id_ptr ? *spec_id_ptr : -ESRCH;
	88	}
	89
	90	return bpf_get_attach_cookie(ctx);
	91	}
	92
	93	/* Return number of USDT arguments defined for currently traced USDT. */
2fa5b0f2	94	__weak __hidden
d72e2968 AN	95	int bpf_usdt_arg_cnt(struct pt_regs *ctx)
	96	{
	97	struct __bpf_usdt_spec *spec;
	98	int spec_id;
	99
	100	spec_id = __bpf_usdt_spec_id(ctx);
	101	if (spec_id < 0)
	102	return -ESRCH;
	103
	104	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
	105	if (!spec)
	106	return -ESRCH;
	107
	108	return spec->arg_cnt;
	109	}
	110
	111	/* Fetch USDT argument #arg_num (zero-indexed) and put its value into *res.
	112	* Returns 0 on success; negative error, otherwise.
	113	* On error *res is guaranteed to be set to zero.
	114	*/
2fa5b0f2	115	__weak __hidden
d72e2968 AN	116	int bpf_usdt_arg(struct pt_regs ctx, __u64 arg_num, long res)
	117	{
	118	struct __bpf_usdt_spec *spec;
	119	struct __bpf_usdt_arg_spec *arg_spec;
	120	unsigned long val;
	121	int err, spec_id;
	122
	123	*res = 0;
	124
	125	spec_id = __bpf_usdt_spec_id(ctx);
	126	if (spec_id < 0)
	127	return -ESRCH;
	128
	129	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
	130	if (!spec)
	131	return -ESRCH;
	132
25c76ed4 IL	133	if (arg_num >= BPF_USDT_MAX_ARG_CNT)
	134	return -ENOENT;
	135	barrier_var(arg_num);
	136	if (arg_num >= spec->arg_cnt)
d72e2968 AN	137	return -ENOENT;
	138
	139	arg_spec = &spec->args[arg_num];
	140	switch (arg_spec->arg_type) {
	141	case BPF_USDT_ARG_CONST:
	142	/* Arg is just a constant ("-4@$-9" in USDT arg spec).
	143	* value is recorded in arg_spec->val_off directly.
	144	*/
	145	val = arg_spec->val_off;
	146	break;
	147	case BPF_USDT_ARG_REG:
	148	/* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
	149	* so we read the contents of that register directly from
	150	* struct pt_regs. To keep things simple user-space parts
	151	* record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
	152	*/
	153	err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
	154	if (err)
	155	return err;
	156	break;
	157	case BPF_USDT_ARG_REG_DEREF:
	158	/* Arg is in memory addressed by register, plus some offset
	159	* (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
e1b6df59	160	* identified like with BPF_USDT_ARG_REG case, and the offset
d72e2968 AN	161	* is in arg_spec->val_off. We first fetch register contents
	162	* from pt_regs, then do another user-space probe read to
	163	* fetch argument value itself.
	164	*/
	165	err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
	166	if (err)
	167	return err;
	168	err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
	169	if (err)
	170	return err;
6f403d9d IL	171	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	172	val >>= arg_spec->arg_bitshift;
	173	#endif
d72e2968 AN	174	break;
	175	default:
	176	return -EINVAL;
	177	}
	178
	179	/* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
	180	* necessary upper arg_bitshift bits, with sign extension if argument
	181	* is signed
	182	*/
	183	val <<= arg_spec->arg_bitshift;
	184	if (arg_spec->arg_signed)
	185	val = ((long)val) >> arg_spec->arg_bitshift;
	186	else
	187	val = val >> arg_spec->arg_bitshift;
	188	*res = val;
	189	return 0;
	190	}
	191
	192	/* Retrieve user-specified cookie value provided during attach as
	193	* bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
	194	* returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
e1b6df59	195	* utilizing BPF cookies internally, so user can't use BPF cookie directly
d72e2968 AN	196	* for USDT programs and has to use bpf_usdt_cookie() API instead.
d72e2968 AN	197	*/
2fa5b0f2	198	__weak __hidden
d72e2968 AN	199	long bpf_usdt_cookie(struct pt_regs *ctx)
	200	{
	201	struct __bpf_usdt_spec *spec;
	202	int spec_id;
	203
	204	spec_id = __bpf_usdt_spec_id(ctx);
	205	if (spec_id < 0)
	206	return 0;
	207
	208	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
	209	if (!spec)
	210	return 0;
	211
	212	return spec->usdt_cookie;
	213	}
	214
	215	/* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
	216	#define ___bpf_usdt_args0() ctx
a9e7715c JM	217	#define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); _x; })
	218	#define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); _x; })
	219	#define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); _x; })
	220	#define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); _x; })
	221	#define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); _x; })
	222	#define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); _x; })
	223	#define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); _x; })
	224	#define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); _x; })
	225	#define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); _x; })
	226	#define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); _x; })
	227	#define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); _x; })
	228	#define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); _x; })
d72e2968 AN	229	#define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)
	230
	231	/*
	232	* BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
	233	* tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
	234	* Original struct pt_regs * context is preserved as 'ctx' argument.
	235	*/
	236	#define BPF_USDT(name, args...) \
	237	name(struct pt_regs *ctx); \
d25f40ff	238	static __always_inline typeof(name(0)) \
d72e2968 AN	239	____##name(struct pt_regs *ctx, ##args); \
	240	typeof(name(0)) name(struct pt_regs *ctx) \
	241	{ \
	242	_Pragma("GCC diagnostic push") \
	243	_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
	244	return ____##name(___bpf_usdt_args(args)); \
	245	_Pragma("GCC diagnostic pop") \
	246	} \
d25f40ff	247	static __always_inline typeof(name(0)) \
d72e2968 AN	248	____##name(struct pt_regs *ctx, ##args)
	249
	250	#endif /* __USDT_BPF_H__ */