1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * Copyright (c) 2016 Facebook
4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
6 #include <uapi/linux/btf.h>
7 #include <linux/kernel.h>
8 #include <linux/types.h>
10 #include <linux/bpf_verifier.h>
11 #include <linux/math64.h>
13 static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
15 /* ubuf and len_total should both be specified (or not) together */
16 if (!!log->ubuf != !!log->len_total)
18 /* log buf without log_level is meaningless */
19 if (log->ubuf && log->level == 0)
21 if (log->level & ~BPF_LOG_MASK)
23 if (log->len_total > UINT_MAX >> 2)
28 int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
29 char __user *log_buf, u32 log_size)
31 log->level = log_level;
33 log->len_total = log_size;
35 /* log attributes have to be sane */
36 if (!bpf_verifier_log_attr_valid(log))
42 static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len)
44 /* add_len includes terminal \0, so no need for +1. */
45 u64 len = log->end_pos + add_len;
47 /* log->len_max could be larger than our current len due to
48 * bpf_vlog_reset() calls, so we maintain the max of any length at any
52 log->len_max = UINT_MAX;
53 else if (len > log->len_max)
57 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
63 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
65 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
66 "verifier log line truncated - local buffer too short\n");
68 if (log->level == BPF_LOG_KERNEL) {
69 bool newline = n > 0 && log->kbuf[n - 1] == '\n';
71 pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");
75 n += 1; /* include terminating zero */
76 bpf_vlog_update_len_max(log, n);
78 if (log->level & BPF_LOG_FIXED) {
79 /* check if we have at least something to put into user buf */
81 if (log->end_pos < log->len_total) {
82 new_n = min_t(u32, log->len_total - log->end_pos, n);
83 log->kbuf[new_n - 1] = '\0';
86 cur_pos = log->end_pos;
87 log->end_pos += n - 1; /* don't count terminating '\0' */
89 if (log->ubuf && new_n &&
90 copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n))
93 u64 new_end, new_start;
94 u32 buf_start, buf_end, new_n;
96 new_end = log->end_pos + n;
97 if (new_end - log->start_pos >= log->len_total)
98 new_start = new_end - log->len_total;
100 new_start = log->start_pos;
102 log->start_pos = new_start;
103 log->end_pos = new_end - 1; /* don't count terminating '\0' */
108 new_n = min(n, log->len_total);
109 cur_pos = new_end - new_n;
110 div_u64_rem(cur_pos, log->len_total, &buf_start);
111 div_u64_rem(new_end, log->len_total, &buf_end);
112 /* new_end and buf_end are exclusive indices, so if buf_end is
113 * exactly zero, then it actually points right to the end of
114 * ubuf and there is no wrap around
117 buf_end = log->len_total;
119 /* if buf_start > buf_end, we wrapped around;
120 * if buf_start == buf_end, then we fill ubuf completely; we
121 * can't have buf_start == buf_end to mean that there is
122 * nothing to write, because we always write at least
123 * something, even if terminal '\0'
125 if (buf_start < buf_end) {
126 /* message fits within contiguous chunk of ubuf */
127 if (copy_to_user(log->ubuf + buf_start,
128 log->kbuf + n - new_n,
129 buf_end - buf_start))
132 /* message wraps around the end of ubuf, copy in two chunks */
133 if (copy_to_user(log->ubuf + buf_start,
134 log->kbuf + n - new_n,
135 log->len_total - buf_start))
137 if (copy_to_user(log->ubuf,
138 log->kbuf + n - buf_end,
149 void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)
154 if (WARN_ON_ONCE(new_pos > log->end_pos))
157 if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL)
160 /* if position to which we reset is beyond current log window,
161 * then we didn't preserve any useful content and should adjust
162 * start_pos to end up with an empty log (start_pos == end_pos)
164 log->end_pos = new_pos;
165 if (log->end_pos < log->start_pos)
166 log->start_pos = log->end_pos;
171 if (log->level & BPF_LOG_FIXED)
172 pos = log->end_pos + 1;
174 div_u64_rem(new_pos, log->len_total, &pos);
176 if (pos < log->len_total && put_user(zero, log->ubuf + pos))
180 static void bpf_vlog_reverse_kbuf(char *buf, int len)
184 for (i = 0, j = len - 1; i < j; i++, j--)
185 swap(buf[i], buf[j]);
188 static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end)
190 /* we split log->kbuf into two equal parts for both ends of array */
191 int n = sizeof(log->kbuf) / 2, nn;
192 char *lbuf = log->kbuf, *rbuf = log->kbuf + n;
194 /* Read ubuf's section [start, end) two chunks at a time, from left
195 * and right side; within each chunk, swap all the bytes; after that
196 * reverse the order of lbuf and rbuf and write result back to ubuf.
197 * This way we'll end up with swapped contents of specified
198 * [start, end) ubuf segment.
200 while (end - start > 1) {
201 nn = min(n, (end - start ) / 2);
203 if (copy_from_user(lbuf, log->ubuf + start, nn))
205 if (copy_from_user(rbuf, log->ubuf + end - nn, nn))
208 bpf_vlog_reverse_kbuf(lbuf, nn);
209 bpf_vlog_reverse_kbuf(rbuf, nn);
211 /* we write lbuf to the right end of ubuf, while rbuf to the
212 * left one to end up with properly reversed overall ubuf
214 if (copy_to_user(log->ubuf + start, rbuf, nn))
216 if (copy_to_user(log->ubuf + end - nn, lbuf, nn))
226 int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual)
231 *log_size_actual = 0;
232 if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL)
236 goto skip_log_rotate;
237 /* If we never truncated log, there is nothing to move around. */
238 if (log->start_pos == 0)
239 goto skip_log_rotate;
241 /* Otherwise we need to rotate log contents to make it start from the
242 * buffer beginning and be a continuous zero-terminated string. Note
243 * that if log->start_pos != 0 then we definitely filled up entire log
244 * buffer with no gaps, and we just need to shift buffer contents to
245 * the left by (log->start_pos % log->len_total) bytes.
247 * Unfortunately, user buffer could be huge and we don't want to
248 * allocate temporary kernel memory of the same size just to shift
249 * contents in a straightforward fashion. Instead, we'll be clever and
250 * do in-place array rotation. This is a leetcode-style problem, which
251 * could be solved by three rotations.
253 * Let's say we have log buffer that has to be shifted left by 7 bytes
254 * (spaces and vertical bar is just for demonstrative purposes):
255 * E F G H I J K | A B C D
257 * First, we reverse entire array:
258 * D C B A | K J I H G F E
260 * Then we rotate first 4 bytes (DCBA) and separately last 7 bytes
261 * (KJIHGFE), resulting in a properly rotated array:
262 * A B C D | E F G H I J K
264 * We'll utilize log->kbuf to read user memory chunk by chunk, swap
265 * bytes, and write them back. Doing it byte-by-byte would be
266 * unnecessarily inefficient. Altogether we are going to read and
267 * write each byte twice, for total 4 memory copies between kernel and
271 /* length of the chopped off part that will be the beginning;
272 * len(ABCD) in the example above
274 div_u64_rem(log->start_pos, log->len_total, &sublen);
275 sublen = log->len_total - sublen;
277 err = bpf_vlog_reverse_ubuf(log, 0, log->len_total);
278 err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen);
279 err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total);
284 *log_size_actual = log->len_max;
286 /* properly initialized log has either both ubuf!=NULL and len_total>0
287 * or ubuf==NULL and len_total==0, so if this condition doesn't hold,
288 * we got a fault somewhere along the way, so report it back
290 if (!!log->ubuf != !!log->len_total)
293 /* did truncation actually happen? */
294 if (log->ubuf && log->len_max > log->len_total)
300 /* log_level controls verbosity level of eBPF verifier.
301 * bpf_verifier_log_write() is used to dump the verification trace to the log,
302 * so the user can figure out what's wrong with the program
304 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
305 const char *fmt, ...)
309 if (!bpf_verifier_log_needed(&env->log))
313 bpf_verifier_vlog(&env->log, fmt, args);
316 EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
318 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
319 const char *fmt, ...)
323 if (!bpf_verifier_log_needed(log))
327 bpf_verifier_vlog(log, fmt, args);
330 EXPORT_SYMBOL_GPL(bpf_log);