[linux-2.6-block.git] / samples / bpf / xdp_monitor_kern.c

/* SPDX-License-Identifier: GPL-2.0
 *  Copyright(c) 2017-2018 Jesper Dangaard Brouer, Red Hat Inc.
 *
 * XDP monitor tool, based on tracepoints
 */
#include <uapi/linux/bpf.h>
#include "bpf_helpers.h"

struct bpf_map_def SEC("maps") redirect_err_cnt = {
	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
	.key_size = sizeof(u32),
	.value_size = sizeof(u64),
	.max_entries = 2,
	/* TODO: have entries for all possible errno's */
};

#define XDP_UNKNOWN	XDP_REDIRECT + 1
struct bpf_map_def SEC("maps") exception_cnt = {
	.type		= BPF_MAP_TYPE_PERCPU_ARRAY,
	.key_size	= sizeof(u32),
	.value_size	= sizeof(u64),
	.max_entries	= XDP_UNKNOWN + 1,
};

/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_redirect/format
 * Code in:                kernel/include/trace/events/xdp.h
 */
struct xdp_redirect_ctx {
	u64 __pad;		// First 8 bytes are not accessible by bpf code
	int prog_id;		//	offset:8;  size:4; signed:1;
	u32 act;		//	offset:12  size:4; signed:0;
	int ifindex;		//	offset:16  size:4; signed:1;
	int err;		//	offset:20  size:4; signed:1;
	int to_ifindex;		//	offset:24  size:4; signed:1;
	u32 map_id;		//	offset:28  size:4; signed:0;
	int map_index;		//	offset:32  size:4; signed:1;
};				//	offset:36

enum {
	XDP_REDIRECT_SUCCESS = 0,
	XDP_REDIRECT_ERROR = 1
};

static __always_inline
int xdp_redirect_collect_stat(struct xdp_redirect_ctx *ctx)
{
	u32 key = XDP_REDIRECT_ERROR;
	int err = ctx->err;
	u64 *cnt;

	if (!err)
		key = XDP_REDIRECT_SUCCESS;

	cnt  = bpf_map_lookup_elem(&redirect_err_cnt, &key);
	if (!cnt)
		return 1;
	*cnt += 1;

	return 0; /* Indicate event was filtered (no further processing)*/
	/*
	 * Returning 1 here would allow e.g. a perf-record tracepoint
	 * to see and record these events, but it doesn't work well
	 * in-practice as stopping perf-record also unload this
	 * bpf_prog.  Plus, there is additional overhead of doing so.
	 */
}

SEC("tracepoint/xdp/xdp_redirect_err")
int trace_xdp_redirect_err(struct xdp_redirect_ctx *ctx)
{
	return xdp_redirect_collect_stat(ctx);
}


SEC("tracepoint/xdp/xdp_redirect_map_err")
int trace_xdp_redirect_map_err(struct xdp_redirect_ctx *ctx)
{
	return xdp_redirect_collect_stat(ctx);
}

/* Likely unloaded when prog starts */
SEC("tracepoint/xdp/xdp_redirect")
int trace_xdp_redirect(struct xdp_redirect_ctx *ctx)
{
	return xdp_redirect_collect_stat(ctx);
}

/* Likely unloaded when prog starts */
SEC("tracepoint/xdp/xdp_redirect_map")
int trace_xdp_redirect_map(struct xdp_redirect_ctx *ctx)
{
	return xdp_redirect_collect_stat(ctx);
}

/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_exception/format
 * Code in:                kernel/include/trace/events/xdp.h
 */
struct xdp_exception_ctx {
	u64 __pad;	// First 8 bytes are not accessible by bpf code
	int prog_id;	//	offset:8;  size:4; signed:1;
	u32 act;	//	offset:12; size:4; signed:0;
	int ifindex;	//	offset:16; size:4; signed:1;
};

SEC("tracepoint/xdp/xdp_exception")
int trace_xdp_exception(struct xdp_exception_ctx *ctx)
{
	u64 *cnt;
	u32 key;

	key = ctx->act;
	if (key > XDP_REDIRECT)
		key = XDP_UNKNOWN;

	cnt = bpf_map_lookup_elem(&exception_cnt, &key);
	if (!cnt)
		return 1;
	*cnt += 1;

	return 0;
}

/* Common stats data record shared with _user.c */
struct datarec {
	u64 processed;
	u64 dropped;
	u64 info;
	u64 err;
};
#define MAX_CPUS 64

struct bpf_map_def SEC("maps") cpumap_enqueue_cnt = {
	.type		= BPF_MAP_TYPE_PERCPU_ARRAY,
	.key_size	= sizeof(u32),
	.value_size	= sizeof(struct datarec),
	.max_entries	= MAX_CPUS,
};

struct bpf_map_def SEC("maps") cpumap_kthread_cnt = {
	.type		= BPF_MAP_TYPE_PERCPU_ARRAY,
	.key_size	= sizeof(u32),
	.value_size	= sizeof(struct datarec),
	.max_entries	= 1,
};

/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_enqueue/format
 * Code in:         kernel/include/trace/events/xdp.h
 */
struct cpumap_enqueue_ctx {
	u64 __pad;		// First 8 bytes are not accessible by bpf code
	int map_id;		//	offset:8;  size:4; signed:1;
	u32 act;		//	offset:12; size:4; signed:0;
	int cpu;		//	offset:16; size:4; signed:1;
	unsigned int drops;	//	offset:20; size:4; signed:0;
	unsigned int processed;	//	offset:24; size:4; signed:0;
	int to_cpu;		//	offset:28; size:4; signed:1;
};

SEC("tracepoint/xdp/xdp_cpumap_enqueue")
int trace_xdp_cpumap_enqueue(struct cpumap_enqueue_ctx *ctx)
{
	u32 to_cpu = ctx->to_cpu;
	struct datarec *rec;

	if (to_cpu >= MAX_CPUS)
		return 1;

	rec = bpf_map_lookup_elem(&cpumap_enqueue_cnt, &to_cpu);
	if (!rec)
		return 0;
	rec->processed += ctx->processed;
	rec->dropped   += ctx->drops;

	/* Record bulk events, then userspace can calc average bulk size */
	if (ctx->processed > 0)
		rec->info += 1;

	return 0;
}

/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_kthread/format
 * Code in:         kernel/include/trace/events/xdp.h
 */
struct cpumap_kthread_ctx {
	u64 __pad;		// First 8 bytes are not accessible by bpf code
	int map_id;		//	offset:8;  size:4; signed:1;
	u32 act;		//	offset:12; size:4; signed:0;
	int cpu;		//	offset:16; size:4; signed:1;
	unsigned int drops;	//	offset:20; size:4; signed:0;
	unsigned int processed;	//	offset:24; size:4; signed:0;
	int sched;		//	offset:28; size:4; signed:1;
};

SEC("tracepoint/xdp/xdp_cpumap_kthread")
int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx)
{
	struct datarec *rec;
	u32 key = 0;

	rec = bpf_map_lookup_elem(&cpumap_kthread_cnt, &key);
	if (!rec)
		return 0;
	rec->processed += ctx->processed;
	rec->dropped   += ctx->drops;

	/* Count times kthread yielded CPU via schedule call */
	if (ctx->sched)
		rec->info++;

	return 0;
}

struct bpf_map_def SEC("maps") devmap_xmit_cnt = {
	.type		= BPF_MAP_TYPE_PERCPU_ARRAY,
	.key_size	= sizeof(u32),
	.value_size	= sizeof(struct datarec),
	.max_entries	= 1,
};

/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_devmap_xmit/format
 * Code in:         kernel/include/trace/events/xdp.h
 */
struct devmap_xmit_ctx {
	u64 __pad;		// First 8 bytes are not accessible by bpf code
	int map_id;		//	offset:8;  size:4; signed:1;
	u32 act;		//	offset:12; size:4; signed:0;
	u32 map_index;		//	offset:16; size:4; signed:0;
	int drops;		//	offset:20; size:4; signed:1;
	int sent;		//	offset:24; size:4; signed:1;
	int from_ifindex;	//	offset:28; size:4; signed:1;
	int to_ifindex;		//	offset:32; size:4; signed:1;
	int err;		//	offset:36; size:4; signed:1;
};

SEC("tracepoint/xdp/xdp_devmap_xmit")
int trace_xdp_devmap_xmit(struct devmap_xmit_ctx *ctx)
{
	struct datarec *rec;
	u32 key = 0;

	rec = bpf_map_lookup_elem(&devmap_xmit_cnt, &key);
	if (!rec)
		return 0;
	rec->processed += ctx->sent;
	rec->dropped   += ctx->drops;

	/* Record bulk events, then userspace can calc average bulk size */
	rec->info += 1;

	/* Record error cases, where no frame were sent */
	if (ctx->err)
		rec->err++;

	/* Catch API error of drv ndo_xdp_xmit sent more than count */
	if (ctx->drops < 0)
		rec->err++;

	return 1;
}
Commit	Line	Data
417f1d9f JDB	1	/* SPDX-License-Identifier: GPL-2.0
417f1d9f JDB	2	* Copyright(c) 2017-2018 Jesper Dangaard Brouer, Red Hat Inc.
3ffab546	3	*
417f1d9f	4	* XDP monitor tool, based on tracepoints
3ffab546 JDB	5	*/
	6	#include <uapi/linux/bpf.h>
	7	#include "bpf_helpers.h"
	8
	9	struct bpf_map_def SEC("maps") redirect_err_cnt = {
	10	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
	11	.key_size = sizeof(u32),
	12	.value_size = sizeof(u64),
	13	.max_entries = 2,
	14	/* TODO: have entries for all possible errno's */
	15	};
	16
280b058d JDB	17	#define XDP_UNKNOWN XDP_REDIRECT + 1
	18	struct bpf_map_def SEC("maps") exception_cnt = {
	19	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
	20	.key_size = sizeof(u32),
	21	.value_size = sizeof(u64),
	22	.max_entries = XDP_UNKNOWN + 1,
	23	};
	24
3ffab546 JDB	25	/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_redirect/format
	26	* Code in: kernel/include/trace/events/xdp.h
	27	*/
	28	struct xdp_redirect_ctx {
f4ce0a01 JDB	29	u64 __pad; // First 8 bytes are not accessible by bpf code
	30	int prog_id; // offset:8; size:4; signed:1;
	31	u32 act; // offset:12 size:4; signed:0;
	32	int ifindex; // offset:16 size:4; signed:1;
	33	int err; // offset:20 size:4; signed:1;
	34	int to_ifindex; // offset:24 size:4; signed:1;
	35	u32 map_id; // offset:28 size:4; signed:0;
	36	int map_index; // offset:32 size:4; signed:1;
	37	}; // offset:36
3ffab546 JDB	38
	39	enum {
	40	XDP_REDIRECT_SUCCESS = 0,
	41	XDP_REDIRECT_ERROR = 1
	42	};
	43
	44	static __always_inline
	45	int xdp_redirect_collect_stat(struct xdp_redirect_ctx *ctx)
	46	{
	47	u32 key = XDP_REDIRECT_ERROR;
	48	int err = ctx->err;
	49	u64 *cnt;
	50
	51	if (!err)
	52	key = XDP_REDIRECT_SUCCESS;
	53
	54	cnt = bpf_map_lookup_elem(&redirect_err_cnt, &key);
	55	if (!cnt)
280b058d	56	return 1;
3ffab546 JDB	57	*cnt += 1;
	58
	59	return 0; /* Indicate event was filtered (no further processing)*/
	60	/*
	61	* Returning 1 here would allow e.g. a perf-record tracepoint
	62	* to see and record these events, but it doesn't work well
	63	* in-practice as stopping perf-record also unload this
	64	* bpf_prog. Plus, there is additional overhead of doing so.
	65	*/
	66	}
	67
	68	SEC("tracepoint/xdp/xdp_redirect_err")
	69	int trace_xdp_redirect_err(struct xdp_redirect_ctx *ctx)
	70	{
	71	return xdp_redirect_collect_stat(ctx);
	72	}
	73
	74
	75	SEC("tracepoint/xdp/xdp_redirect_map_err")
	76	int trace_xdp_redirect_map_err(struct xdp_redirect_ctx *ctx)
	77	{
	78	return xdp_redirect_collect_stat(ctx);
	79	}
	80
	81	/* Likely unloaded when prog starts */
	82	SEC("tracepoint/xdp/xdp_redirect")
	83	int trace_xdp_redirect(struct xdp_redirect_ctx *ctx)
	84	{
	85	return xdp_redirect_collect_stat(ctx);
	86	}
	87
	88	/* Likely unloaded when prog starts */
	89	SEC("tracepoint/xdp/xdp_redirect_map")
	90	int trace_xdp_redirect_map(struct xdp_redirect_ctx *ctx)
	91	{
	92	return xdp_redirect_collect_stat(ctx);
	93	}
280b058d JDB	94
	95	/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_exception/format
	96	* Code in: kernel/include/trace/events/xdp.h
	97	*/
	98	struct xdp_exception_ctx {
	99	u64 __pad; // First 8 bytes are not accessible by bpf code
	100	int prog_id; // offset:8; size:4; signed:1;
	101	u32 act; // offset:12; size:4; signed:0;
	102	int ifindex; // offset:16; size:4; signed:1;
	103	};
	104
	105	SEC("tracepoint/xdp/xdp_exception")
	106	int trace_xdp_exception(struct xdp_exception_ctx *ctx)
	107	{
4c38f74c	108	u64 *cnt;
280b058d JDB	109	u32 key;
	110
	111	key = ctx->act;
	112	if (key > XDP_REDIRECT)
	113	key = XDP_UNKNOWN;
	114
	115	cnt = bpf_map_lookup_elem(&exception_cnt, &key);
	116	if (!cnt)
	117	return 1;
	118	*cnt += 1;
	119
	120	return 0;
	121	}
417f1d9f JDB	122
	123	/* Common stats data record shared with _user.c */
	124	struct datarec {
	125	u64 processed;
	126	u64 dropped;
	127	u64 info;
a570e48f	128	u64 err;
417f1d9f JDB	129	};
	130	#define MAX_CPUS 64
	131
	132	struct bpf_map_def SEC("maps") cpumap_enqueue_cnt = {
	133	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
	134	.key_size = sizeof(u32),
	135	.value_size = sizeof(struct datarec),
	136	.max_entries = MAX_CPUS,
	137	};
	138
	139	struct bpf_map_def SEC("maps") cpumap_kthread_cnt = {
	140	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
	141	.key_size = sizeof(u32),
	142	.value_size = sizeof(struct datarec),
	143	.max_entries = 1,
	144	};
	145
	146	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_enqueue/format
	147	* Code in: kernel/include/trace/events/xdp.h
	148	*/
	149	struct cpumap_enqueue_ctx {
	150	u64 __pad; // First 8 bytes are not accessible by bpf code
	151	int map_id; // offset:8; size:4; signed:1;
	152	u32 act; // offset:12; size:4; signed:0;
	153	int cpu; // offset:16; size:4; signed:1;
	154	unsigned int drops; // offset:20; size:4; signed:0;
	155	unsigned int processed; // offset:24; size:4; signed:0;
	156	int to_cpu; // offset:28; size:4; signed:1;
	157	};
	158
	159	SEC("tracepoint/xdp/xdp_cpumap_enqueue")
	160	int trace_xdp_cpumap_enqueue(struct cpumap_enqueue_ctx *ctx)
	161	{
	162	u32 to_cpu = ctx->to_cpu;
	163	struct datarec *rec;
	164
	165	if (to_cpu >= MAX_CPUS)
	166	return 1;
	167
	168	rec = bpf_map_lookup_elem(&cpumap_enqueue_cnt, &to_cpu);
	169	if (!rec)
	170	return 0;
	171	rec->processed += ctx->processed;
	172	rec->dropped += ctx->drops;
	173
	174	/* Record bulk events, then userspace can calc average bulk size */
	175	if (ctx->processed > 0)
	176	rec->info += 1;
	177
	178	return 0;
	179	}
	180
	181	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_kthread/format
	182	* Code in: kernel/include/trace/events/xdp.h
	183	*/
	184	struct cpumap_kthread_ctx {
	185	u64 __pad; // First 8 bytes are not accessible by bpf code
	186	int map_id; // offset:8; size:4; signed:1;
	187	u32 act; // offset:12; size:4; signed:0;
	188	int cpu; // offset:16; size:4; signed:1;
	189	unsigned int drops; // offset:20; size:4; signed:0;
	190	unsigned int processed; // offset:24; size:4; signed:0;
	191	int sched; // offset:28; size:4; signed:1;
	192	};
193
194	SEC("tracepoint/xdp/xdp_cpumap_kthread")
195	int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx)
196	{
197	struct datarec *rec;
198	u32 key = 0;
199
200	rec = bpf_map_lookup_elem(&cpumap_kthread_cnt, &key);
201	if (!rec)
202	return 0;
203	rec->processed += ctx->processed;
204	rec->dropped += ctx->drops;
205
206	/* Count times kthread yielded CPU via schedule call */
207	if (ctx->sched)
208	rec->info++;
209
210	return 0;
211	}
9940fbf6 JDB	212
	213	struct bpf_map_def SEC("maps") devmap_xmit_cnt = {
	214	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
	215	.key_size = sizeof(u32),
	216	.value_size = sizeof(struct datarec),
	217	.max_entries = 1,
	218	};
	219
	220	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_devmap_xmit/format
	221	* Code in: kernel/include/trace/events/xdp.h
	222	*/
	223	struct devmap_xmit_ctx {
	224	u64 __pad; // First 8 bytes are not accessible by bpf code
	225	int map_id; // offset:8; size:4; signed:1;
	226	u32 act; // offset:12; size:4; signed:0;
	227	u32 map_index; // offset:16; size:4; signed:0;
	228	int drops; // offset:20; size:4; signed:1;
	229	int sent; // offset:24; size:4; signed:1;
	230	int from_ifindex; // offset:28; size:4; signed:1;
	231	int to_ifindex; // offset:32; size:4; signed:1;
a570e48f	232	int err; // offset:36; size:4; signed:1;
9940fbf6 JDB	233	};
	234
	235	SEC("tracepoint/xdp/xdp_devmap_xmit")
	236	int trace_xdp_devmap_xmit(struct devmap_xmit_ctx *ctx)
	237	{
	238	struct datarec *rec;
	239	u32 key = 0;
	240
	241	rec = bpf_map_lookup_elem(&devmap_xmit_cnt, &key);
	242	if (!rec)
	243	return 0;
	244	rec->processed += ctx->sent;
	245	rec->dropped += ctx->drops;
	246
	247	/* Record bulk events, then userspace can calc average bulk size */
	248	rec->info += 1;
	249
a570e48f JDB	250	/* Record error cases, where no frame were sent */
	251	if (ctx->err)
	252	rec->err++;
	253
	254	/* Catch API error of drv ndo_xdp_xmit sent more than count */
	255	if (ctx->drops < 0)
	256	rec->err++;
	257
9940fbf6 JDB	258	return 1;
9940fbf6 JDB	259	}