[linux-block.git] / kernel / bpf / log.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
 * Copyright (c) 2016 Facebook
 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
 */
#include <uapi/linux/btf.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/math64.h>

static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
{
	/* ubuf and len_total should both be specified (or not) together */
	if (!!log->ubuf != !!log->len_total)
		return false;
	/* log buf without log_level is meaningless */
	if (log->ubuf && log->level == 0)
		return false;
	if (log->level & ~BPF_LOG_MASK)
		return false;
	if (log->len_total > UINT_MAX >> 2)
		return false;
	return true;
}

int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
		  char __user *log_buf, u32 log_size)
{
	log->level = log_level;
	log->ubuf = log_buf;
	log->len_total = log_size;

	/* log attributes have to be sane */
	if (!bpf_verifier_log_attr_valid(log))
		return -EINVAL;

	return 0;
}

static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len)
{
	/* add_len includes terminal \0, so no need for +1. */
	u64 len = log->end_pos + add_len;

	/* log->len_max could be larger than our current len due to
	 * bpf_vlog_reset() calls, so we maintain the max of any length at any
	 * previous point
	 */
	if (len > UINT_MAX)
		log->len_max = UINT_MAX;
	else if (len > log->len_max)
		log->len_max = len;
}

void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
		       va_list args)
{
	u64 cur_pos;
	u32 new_n, n;

	n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);

	WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
		  "verifier log line truncated - local buffer too short\n");

	if (log->level == BPF_LOG_KERNEL) {
		bool newline = n > 0 && log->kbuf[n - 1] == '\n';

		pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");
		return;
	}

	n += 1; /* include terminating zero */
	bpf_vlog_update_len_max(log, n);

	if (log->level & BPF_LOG_FIXED) {
		/* check if we have at least something to put into user buf */
		new_n = 0;
		if (log->end_pos < log->len_total) {
			new_n = min_t(u32, log->len_total - log->end_pos, n);
			log->kbuf[new_n - 1] = '\0';
		}

		cur_pos = log->end_pos;
		log->end_pos += n - 1; /* don't count terminating '\0' */

		if (log->ubuf && new_n &&
		    copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n))
			goto fail;
	} else {
		u64 new_end, new_start;
		u32 buf_start, buf_end, new_n;

		new_end = log->end_pos + n;
		if (new_end - log->start_pos >= log->len_total)
			new_start = new_end - log->len_total;
		else
			new_start = log->start_pos;

		log->start_pos = new_start;
		log->end_pos = new_end - 1; /* don't count terminating '\0' */

		if (!log->ubuf)
			return;

		new_n = min(n, log->len_total);
		cur_pos = new_end - new_n;
		div_u64_rem(cur_pos, log->len_total, &buf_start);
		div_u64_rem(new_end, log->len_total, &buf_end);
		/* new_end and buf_end are exclusive indices, so if buf_end is
		 * exactly zero, then it actually points right to the end of
		 * ubuf and there is no wrap around
		 */
		if (buf_end == 0)
			buf_end = log->len_total;

		/* if buf_start > buf_end, we wrapped around;
		 * if buf_start == buf_end, then we fill ubuf completely; we
		 * can't have buf_start == buf_end to mean that there is
		 * nothing to write, because we always write at least
		 * something, even if terminal '\0'
		 */
		if (buf_start < buf_end) {
			/* message fits within contiguous chunk of ubuf */
			if (copy_to_user(log->ubuf + buf_start,
					 log->kbuf + n - new_n,
					 buf_end - buf_start))
				goto fail;
		} else {
			/* message wraps around the end of ubuf, copy in two chunks */
			if (copy_to_user(log->ubuf + buf_start,
					 log->kbuf + n - new_n,
					 log->len_total - buf_start))
				goto fail;
			if (copy_to_user(log->ubuf,
					 log->kbuf + n - buf_end,
					 buf_end))
				goto fail;
		}
	}

	return;
fail:
	log->ubuf = NULL;
}

void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)
{
	char zero = 0;
	u32 pos;

	if (WARN_ON_ONCE(new_pos > log->end_pos))
		return;

	if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL)
		return;

	/* if position to which we reset is beyond current log window,
	 * then we didn't preserve any useful content and should adjust
	 * start_pos to end up with an empty log (start_pos == end_pos)
	 */
	log->end_pos = new_pos;
	if (log->end_pos < log->start_pos)
		log->start_pos = log->end_pos;

	if (!log->ubuf)
		return;

	if (log->level & BPF_LOG_FIXED)
		pos = log->end_pos + 1;
	else
		div_u64_rem(new_pos, log->len_total, &pos);

	if (pos < log->len_total && put_user(zero, log->ubuf + pos))
		log->ubuf = NULL;
}

static void bpf_vlog_reverse_kbuf(char *buf, int len)
{
	int i, j;

	for (i = 0, j = len - 1; i < j; i++, j--)
		swap(buf[i], buf[j]);
}

static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end)
{
	/* we split log->kbuf into two equal parts for both ends of array */
	int n = sizeof(log->kbuf) / 2, nn;
	char *lbuf = log->kbuf, *rbuf = log->kbuf + n;

	/* Read ubuf's section [start, end) two chunks at a time, from left
	 * and right side; within each chunk, swap all the bytes; after that
	 * reverse the order of lbuf and rbuf and write result back to ubuf.
	 * This way we'll end up with swapped contents of specified
	 * [start, end) ubuf segment.
	 */
	while (end - start > 1) {
		nn = min(n, (end - start ) / 2);

		if (copy_from_user(lbuf, log->ubuf + start, nn))
			return -EFAULT;
		if (copy_from_user(rbuf, log->ubuf + end - nn, nn))
			return -EFAULT;

		bpf_vlog_reverse_kbuf(lbuf, nn);
		bpf_vlog_reverse_kbuf(rbuf, nn);

		/* we write lbuf to the right end of ubuf, while rbuf to the
		 * left one to end up with properly reversed overall ubuf
		 */
		if (copy_to_user(log->ubuf + start, rbuf, nn))
			return -EFAULT;
		if (copy_to_user(log->ubuf + end - nn, lbuf, nn))
			return -EFAULT;

		start += nn;
		end -= nn;
	}

	return 0;
}

int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual)
{
	u32 sublen;
	int err;

	*log_size_actual = 0;
	if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL)
		return 0;

	if (!log->ubuf)
		goto skip_log_rotate;
	/* If we never truncated log, there is nothing to move around. */
	if (log->start_pos == 0)
		goto skip_log_rotate;

	/* Otherwise we need to rotate log contents to make it start from the
	 * buffer beginning and be a continuous zero-terminated string. Note
	 * that if log->start_pos != 0 then we definitely filled up entire log
	 * buffer with no gaps, and we just need to shift buffer contents to
	 * the left by (log->start_pos % log->len_total) bytes.
	 *
	 * Unfortunately, user buffer could be huge and we don't want to
	 * allocate temporary kernel memory of the same size just to shift
	 * contents in a straightforward fashion. Instead, we'll be clever and
	 * do in-place array rotation. This is a leetcode-style problem, which
	 * could be solved by three rotations.
	 *
	 * Let's say we have log buffer that has to be shifted left by 7 bytes
	 * (spaces and vertical bar is just for demonstrative purposes):
	 *   E F G H I J K | A B C D
	 *
	 * First, we reverse entire array:
	 *   D C B A | K J I H G F E
	 *
	 * Then we rotate first 4 bytes (DCBA) and separately last 7 bytes
	 * (KJIHGFE), resulting in a properly rotated array:
	 *   A B C D | E F G H I J K
	 *
	 * We'll utilize log->kbuf to read user memory chunk by chunk, swap
	 * bytes, and write them back. Doing it byte-by-byte would be
	 * unnecessarily inefficient. Altogether we are going to read and
	 * write each byte twice, for total 4 memory copies between kernel and
	 * user space.
	 */

	/* length of the chopped off part that will be the beginning;
	 * len(ABCD) in the example above
	 */
	div_u64_rem(log->start_pos, log->len_total, &sublen);
	sublen = log->len_total - sublen;

	err = bpf_vlog_reverse_ubuf(log, 0, log->len_total);
	err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen);
	err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total);
	if (err)
		log->ubuf = NULL;

skip_log_rotate:
	*log_size_actual = log->len_max;

	/* properly initialized log has either both ubuf!=NULL and len_total>0
	 * or ubuf==NULL and len_total==0, so if this condition doesn't hold,
	 * we got a fault somewhere along the way, so report it back
	 */
	if (!!log->ubuf != !!log->len_total)
		return -EFAULT;

	/* did truncation actually happen? */
	if (log->ubuf && log->len_max > log->len_total)
		return -ENOSPC;

	return 0;
}

/* log_level controls verbosity level of eBPF verifier.
 * bpf_verifier_log_write() is used to dump the verification trace to the log,
 * so the user can figure out what's wrong with the program
 */
__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
					   const char *fmt, ...)
{
	va_list args;

	if (!bpf_verifier_log_needed(&env->log))
		return;

	va_start(args, fmt);
	bpf_verifier_vlog(&env->log, fmt, args);
	va_end(args);
}
EXPORT_SYMBOL_GPL(bpf_verifier_log_write);

__printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
			    const char *fmt, ...)
{
	va_list args;

	if (!bpf_verifier_log_needed(log))
		return;

	va_start(args, fmt);
	bpf_verifier_vlog(log, fmt, args);
	va_end(args);
}
EXPORT_SYMBOL_GPL(bpf_log);
Commit	Line	Data
4294a0a7 AN	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
	3	* Copyright (c) 2016 Facebook
	4	* Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
	5	*/
	6	#include <uapi/linux/btf.h>
	7	#include <linux/kernel.h>
	8	#include <linux/types.h>
	9	#include <linux/bpf.h>
	10	#include <linux/bpf_verifier.h>
12166409	11	#include <linux/math64.h>
4294a0a7	12
bdcab414	13	static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
4294a0a7	14	{
fac08d45 AN	15	/* ubuf and len_total should both be specified (or not) together */
	16	if (!!log->ubuf != !!log->len_total)
	17	return false;
	18	/* log buf without log_level is meaningless */
	19	if (log->ubuf && log->level == 0)
	20	return false;
	21	if (log->level & ~BPF_LOG_MASK)
	22	return false;
	23	if (log->len_total > UINT_MAX >> 2)
	24	return false;
	25	return true;
4294a0a7 AN	26	}
4294a0a7 AN	27
bdcab414 AN	28	int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
	29	char __user *log_buf, u32 log_size)
	30	{
	31	log->level = log_level;
	32	log->ubuf = log_buf;
	33	log->len_total = log_size;
	34
	35	/* log attributes have to be sane */
	36	if (!bpf_verifier_log_attr_valid(log))
	37	return -EINVAL;
	38
	39	return 0;
	40	}
	41
fa1c7d5c AN	42	static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len)
	43	{
	44	/* add_len includes terminal \0, so no need for +1. */
	45	u64 len = log->end_pos + add_len;
	46
	47	/* log->len_max could be larger than our current len due to
	48	* bpf_vlog_reset() calls, so we maintain the max of any length at any
	49	* previous point
	50	*/
	51	if (len > UINT_MAX)
	52	log->len_max = UINT_MAX;
	53	else if (len > log->len_max)
	54	log->len_max = len;
	55	}
	56
4294a0a7 AN	57	void bpf_verifier_vlog(struct bpf_verifier_log log, const char fmt,
	58	va_list args)
	59	{
fa1c7d5c AN	60	u64 cur_pos;
fa1c7d5c AN	61	u32 new_n, n;
4294a0a7 AN	62
	63	n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
	64
	65	WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
	66	"verifier log line truncated - local buffer too short\n");
	67
	68	if (log->level == BPF_LOG_KERNEL) {
	69	bool newline = n > 0 && log->kbuf[n - 1] == '\n';
	70
	71	pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");
	72	return;
	73	}
	74
fa1c7d5c AN	75	n += 1; /* include terminating zero */
fa1c7d5c AN	76	bpf_vlog_update_len_max(log, n);
12166409	77
fa1c7d5c AN	78	if (log->level & BPF_LOG_FIXED) {
	79	/* check if we have at least something to put into user buf */
	80	new_n = 0;
	81	if (log->end_pos < log->len_total) {
	82	new_n = min_t(u32, log->len_total - log->end_pos, n);
	83	log->kbuf[new_n - 1] = '\0';
	84	}
12166409	85
fa1c7d5c	86	cur_pos = log->end_pos;
12166409	87	log->end_pos += n - 1; /* don't count terminating '\0' */
fa1c7d5c AN	88
	89	if (log->ubuf && new_n &&
	90	copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n))
	91	goto fail;
12166409	92	} else {
fa1c7d5c	93	u64 new_end, new_start;
12166409 AN	94	u32 buf_start, buf_end, new_n;
12166409 AN	95
12166409 AN	96	new_end = log->end_pos + n;
	97	if (new_end - log->start_pos >= log->len_total)
	98	new_start = new_end - log->len_total;
	99	else
	100	new_start = log->start_pos;
fac08d45 AN	101
	102	log->start_pos = new_start;
	103	log->end_pos = new_end - 1; /* don't count terminating '\0' */
	104
	105	if (!log->ubuf)
	106	return;
	107
12166409 AN	108	new_n = min(n, log->len_total);
12166409 AN	109	cur_pos = new_end - new_n;
12166409 AN	110	div_u64_rem(cur_pos, log->len_total, &buf_start);
	111	div_u64_rem(new_end, log->len_total, &buf_end);
	112	/* new_end and buf_end are exclusive indices, so if buf_end is
	113	* exactly zero, then it actually points right to the end of
	114	* ubuf and there is no wrap around
	115	*/
	116	if (buf_end == 0)
	117	buf_end = log->len_total;
	118
	119	/* if buf_start > buf_end, we wrapped around;
	120	* if buf_start == buf_end, then we fill ubuf completely; we
	121	* can't have buf_start == buf_end to mean that there is
	122	* nothing to write, because we always write at least
	123	* something, even if terminal '\0'
	124	*/
	125	if (buf_start < buf_end) {
	126	/* message fits within contiguous chunk of ubuf */
	127	if (copy_to_user(log->ubuf + buf_start,
	128	log->kbuf + n - new_n,
	129	buf_end - buf_start))
	130	goto fail;
	131	} else {
	132	/* message wraps around the end of ubuf, copy in two chunks */
	133	if (copy_to_user(log->ubuf + buf_start,
	134	log->kbuf + n - new_n,
	135	log->len_total - buf_start))
	136	goto fail;
	137	if (copy_to_user(log->ubuf,
	138	log->kbuf + n - buf_end,
	139	buf_end))
	140	goto fail;
	141	}
12166409 AN	142	}
	143
	144	return;
	145	fail:
	146	log->ubuf = NULL;
4294a0a7 AN	147	}
4294a0a7 AN	148
12166409	149	void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)
4294a0a7 AN	150	{
4294a0a7 AN	151	char zero = 0;
12166409 AN	152	u32 pos;
	153
	154	if (WARN_ON_ONCE(new_pos > log->end_pos))
	155	return;
4294a0a7	156
24bc8088	157	if (!bpf_verifier_log_needed(log) \|\| log->level == BPF_LOG_KERNEL)
4294a0a7 AN	158	return;
4294a0a7 AN	159
12166409 AN	160	/* if position to which we reset is beyond current log window,
	161	* then we didn't preserve any useful content and should adjust
	162	* start_pos to end up with an empty log (start_pos == end_pos)
	163	*/
	164	log->end_pos = new_pos;
	165	if (log->end_pos < log->start_pos)
	166	log->start_pos = log->end_pos;
fa1c7d5c	167
fac08d45 AN	168	if (!log->ubuf)
	169	return;
	170
fa1c7d5c AN	171	if (log->level & BPF_LOG_FIXED)
	172	pos = log->end_pos + 1;
	173	else
	174	div_u64_rem(new_pos, log->len_total, &pos);
	175
fac08d45	176	if (pos < log->len_total && put_user(zero, log->ubuf + pos))
12166409 AN	177	log->ubuf = NULL;
	178	}
	179
	180	static void bpf_vlog_reverse_kbuf(char *buf, int len)
	181	{
	182	int i, j;
	183
	184	for (i = 0, j = len - 1; i < j; i++, j--)
	185	swap(buf[i], buf[j]);
	186	}
	187
	188	static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end)
	189	{
	190	/* we split log->kbuf into two equal parts for both ends of array */
	191	int n = sizeof(log->kbuf) / 2, nn;
	192	char lbuf = log->kbuf, rbuf = log->kbuf + n;
	193
	194	/* Read ubuf's section [start, end) two chunks at a time, from left
	195	* and right side; within each chunk, swap all the bytes; after that
	196	* reverse the order of lbuf and rbuf and write result back to ubuf.
	197	* This way we'll end up with swapped contents of specified
	198	* [start, end) ubuf segment.
	199	*/
	200	while (end - start > 1) {
	201	nn = min(n, (end - start ) / 2);
	202
	203	if (copy_from_user(lbuf, log->ubuf + start, nn))
	204	return -EFAULT;
	205	if (copy_from_user(rbuf, log->ubuf + end - nn, nn))
	206	return -EFAULT;
	207
	208	bpf_vlog_reverse_kbuf(lbuf, nn);
	209	bpf_vlog_reverse_kbuf(rbuf, nn);
	210
	211	/* we write lbuf to the right end of ubuf, while rbuf to the
	212	* left one to end up with properly reversed overall ubuf
	213	*/
	214	if (copy_to_user(log->ubuf + start, rbuf, nn))
	215	return -EFAULT;
	216	if (copy_to_user(log->ubuf + end - nn, lbuf, nn))
	217	return -EFAULT;
	218
	219	start += nn;
	220	end -= nn;
	221	}
	222
	223	return 0;
	224	}
	225
bdcab414	226	int bpf_vlog_finalize(struct bpf_verifier_log log, u32 log_size_actual)
12166409 AN	227	{
	228	u32 sublen;
	229	int err;
	230
bdcab414 AN	231	*log_size_actual = 0;
	232	if (!log \|\| log->level == 0 \|\| log->level == BPF_LOG_KERNEL)
	233	return 0;
12166409	234
bdcab414 AN	235	if (!log->ubuf)
bdcab414 AN	236	goto skip_log_rotate;
12166409	237	/* If we never truncated log, there is nothing to move around. */
fac08d45	238	if (log->start_pos == 0)
bdcab414	239	goto skip_log_rotate;
12166409 AN	240
	241	/* Otherwise we need to rotate log contents to make it start from the
	242	* buffer beginning and be a continuous zero-terminated string. Note
	243	* that if log->start_pos != 0 then we definitely filled up entire log
	244	* buffer with no gaps, and we just need to shift buffer contents to
	245	* the left by (log->start_pos % log->len_total) bytes.
	246	*
	247	* Unfortunately, user buffer could be huge and we don't want to
	248	* allocate temporary kernel memory of the same size just to shift
	249	* contents in a straightforward fashion. Instead, we'll be clever and
	250	* do in-place array rotation. This is a leetcode-style problem, which
	251	* could be solved by three rotations.
	252	*
	253	* Let's say we have log buffer that has to be shifted left by 7 bytes
	254	* (spaces and vertical bar is just for demonstrative purposes):
	255	* E F G H I J K \| A B C D
	256	*
	257	* First, we reverse entire array:
	258	* D C B A \| K J I H G F E
	259	*
	260	* Then we rotate first 4 bytes (DCBA) and separately last 7 bytes
	261	* (KJIHGFE), resulting in a properly rotated array:
	262	* A B C D \| E F G H I J K
	263	*
	264	* We'll utilize log->kbuf to read user memory chunk by chunk, swap
	265	* bytes, and write them back. Doing it byte-by-byte would be
	266	* unnecessarily inefficient. Altogether we are going to read and
	267	* write each byte twice, for total 4 memory copies between kernel and
	268	* user space.
	269	*/
	270
	271	/* length of the chopped off part that will be the beginning;
	272	* len(ABCD) in the example above
	273	*/
	274	div_u64_rem(log->start_pos, log->len_total, &sublen);
	275	sublen = log->len_total - sublen;
	276
	277	err = bpf_vlog_reverse_ubuf(log, 0, log->len_total);
	278	err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen);
	279	err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total);
	280	if (err)
4294a0a7	281	log->ubuf = NULL;
bdcab414 AN	282
	283	skip_log_rotate:
	284	*log_size_actual = log->len_max;
	285
	286	/* properly initialized log has either both ubuf!=NULL and len_total>0
	287	* or ubuf==NULL and len_total==0, so if this condition doesn't hold,
	288	* we got a fault somewhere along the way, so report it back
	289	*/
	290	if (!!log->ubuf != !!log->len_total)
	291	return -EFAULT;
	292
fac08d45 AN	293	/* did truncation actually happen? */
fac08d45 AN	294	if (log->ubuf && log->len_max > log->len_total)
bdcab414 AN	295	return -ENOSPC;
	296
	297	return 0;
4294a0a7 AN	298	}
	299
	300	/* log_level controls verbosity level of eBPF verifier.
	301	* bpf_verifier_log_write() is used to dump the verification trace to the log,
	302	* so the user can figure out what's wrong with the program
	303	*/
	304	__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
	305	const char *fmt, ...)
	306	{
	307	va_list args;
	308
	309	if (!bpf_verifier_log_needed(&env->log))
	310	return;
	311
	312	va_start(args, fmt);
	313	bpf_verifier_vlog(&env->log, fmt, args);
	314	va_end(args);
	315	}
	316	EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
	317
	318	__printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
	319	const char *fmt, ...)
	320	{
	321	va_list args;
	322
	323	if (!bpf_verifier_log_needed(log))
	324	return;
	325
	326	va_start(args, fmt);
	327	bpf_verifier_vlog(log, fmt, args);
	328	va_end(args);
	329	}
	330	EXPORT_SYMBOL_GPL(bpf_log);