Commit | Line | Data |
---|---|---|
b6cf8b3f JO |
1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | ||
3 | #ifndef _KERNEL_PRINTK_RINGBUFFER_H | |
4 | #define _KERNEL_PRINTK_RINGBUFFER_H | |
5 | ||
6 | #include <linux/atomic.h> | |
74caba7f | 7 | #include <linux/dev_printk.h> |
b6cf8b3f JO |
8 | |
9 | /* | |
10 | * Meta information about each stored message. | |
11 | * | |
f35efc78 JO |
12 | * All fields are set by the printk code except for @seq, which is |
13 | * set by the ringbuffer code. | |
b6cf8b3f JO |
14 | */ |
15 | struct printk_info { | |
16 | u64 seq; /* sequence number */ | |
17 | u64 ts_nsec; /* timestamp in nanoseconds */ | |
18 | u16 text_len; /* length of text message */ | |
b6cf8b3f JO |
19 | u8 facility; /* syslog facility */ |
20 | u8 flags:5; /* internal record flags */ | |
21 | u8 level:3; /* syslog level */ | |
22 | u32 caller_id; /* thread id or processor id */ | |
74caba7f JO |
23 | |
24 | struct dev_printk_info dev_info; | |
b6cf8b3f JO |
25 | }; |
26 | ||
27 | /* | |
28 | * A structure providing the buffers, used by writers and readers. | |
29 | * | |
30 | * Writers: | |
f35efc78 JO |
31 | * Using prb_rec_init_wr(), a writer sets @text_buf_size before calling |
32 | * prb_reserve(). On success, prb_reserve() sets @info and @text_buf to | |
33 | * buffers reserved for that writer. | |
b6cf8b3f JO |
34 | * |
35 | * Readers: | |
36 | * Using prb_rec_init_rd(), a reader sets all fields before calling | |
f35efc78 JO |
37 | * prb_read_valid(). Note that the reader provides the @info and @text_buf, |
38 | * buffers. On success, the struct pointed to by @info will be filled and | |
39 | * the char array pointed to by @text_buf will be filled with text data. | |
b6cf8b3f JO |
40 | */ |
41 | struct printk_record { | |
42 | struct printk_info *info; | |
43 | char *text_buf; | |
b6cf8b3f | 44 | unsigned int text_buf_size; |
b6cf8b3f JO |
45 | }; |
46 | ||
47 | /* Specifies the logical position and span of a data block. */ | |
48 | struct prb_data_blk_lpos { | |
49 | unsigned long begin; | |
50 | unsigned long next; | |
51 | }; | |
52 | ||
53 | /* | |
54 | * A descriptor: the complete meta-data for a record. | |
55 | * | |
56 | * @state_var: A bitwise combination of descriptor ID and descriptor state. | |
57 | */ | |
58 | struct prb_desc { | |
b6cf8b3f JO |
59 | atomic_long_t state_var; |
60 | struct prb_data_blk_lpos text_blk_lpos; | |
b6cf8b3f JO |
61 | }; |
62 | ||
63 | /* A ringbuffer of "ID + data" elements. */ | |
64 | struct prb_data_ring { | |
65 | unsigned int size_bits; | |
66 | char *data; | |
67 | atomic_long_t head_lpos; | |
68 | atomic_long_t tail_lpos; | |
69 | }; | |
70 | ||
71 | /* A ringbuffer of "struct prb_desc" elements. */ | |
72 | struct prb_desc_ring { | |
73 | unsigned int count_bits; | |
74 | struct prb_desc *descs; | |
cfe2790b | 75 | struct printk_info *infos; |
b6cf8b3f JO |
76 | atomic_long_t head_id; |
77 | atomic_long_t tail_id; | |
5f72e52b | 78 | atomic_long_t last_finalized_seq; |
b6cf8b3f JO |
79 | }; |
80 | ||
81 | /* | |
82 | * The high level structure representing the printk ringbuffer. | |
83 | * | |
84 | * @fail: Count of failed prb_reserve() calls where not even a data-less | |
85 | * record was created. | |
86 | */ | |
87 | struct printk_ringbuffer { | |
88 | struct prb_desc_ring desc_ring; | |
89 | struct prb_data_ring text_data_ring; | |
b6cf8b3f JO |
90 | atomic_long_t fail; |
91 | }; | |
92 | ||
93 | /* | |
94 | * Used by writers as a reserve/commit handle. | |
95 | * | |
96 | * @rb: Ringbuffer where the entry is reserved. | |
97 | * @irqflags: Saved irq flags to restore on entry commit. | |
98 | * @id: ID of the reserved descriptor. | |
99 | * @text_space: Total occupied buffer space in the text data ring, including | |
100 | * ID, alignment padding, and wrapping data blocks. | |
101 | * | |
102 | * This structure is an opaque handle for writers. Its contents are only | |
103 | * to be used by the ringbuffer implementation. | |
104 | */ | |
105 | struct prb_reserved_entry { | |
106 | struct printk_ringbuffer *rb; | |
107 | unsigned long irqflags; | |
108 | unsigned long id; | |
109 | unsigned int text_space; | |
110 | }; | |
111 | ||
10dcb06d JO |
112 | /* The possible responses of a descriptor state-query. */ |
113 | enum desc_state { | |
114 | desc_miss = -1, /* ID mismatch (pseudo state) */ | |
115 | desc_reserved = 0x0, /* reserved, in use by writer */ | |
4cfc7258 JO |
116 | desc_committed = 0x1, /* committed by writer, could get reopened */ |
117 | desc_finalized = 0x2, /* committed, no further modification allowed */ | |
10dcb06d JO |
118 | desc_reusable = 0x3, /* free, not yet used by any writer */ |
119 | }; | |
120 | ||
121 | #define _DATA_SIZE(sz_bits) (1UL << (sz_bits)) | |
122 | #define _DESCS_COUNT(ct_bits) (1U << (ct_bits)) | |
123 | #define DESC_SV_BITS (sizeof(unsigned long) * 8) | |
124 | #define DESC_FLAGS_SHIFT (DESC_SV_BITS - 2) | |
125 | #define DESC_FLAGS_MASK (3UL << DESC_FLAGS_SHIFT) | |
126 | #define DESC_STATE(sv) (3UL & (sv >> DESC_FLAGS_SHIFT)) | |
127 | #define DESC_SV(id, state) (((unsigned long)state << DESC_FLAGS_SHIFT) | id) | |
128 | #define DESC_ID_MASK (~DESC_FLAGS_MASK) | |
129 | #define DESC_ID(sv) ((sv) & DESC_ID_MASK) | |
130 | #define FAILED_LPOS 0x1 | |
131 | #define NO_LPOS 0x3 | |
b6cf8b3f | 132 | |
d397820f | 133 | #define FAILED_BLK_LPOS \ |
b6cf8b3f | 134 | { \ |
d397820f JO |
135 | .begin = FAILED_LPOS, \ |
136 | .next = FAILED_LPOS, \ | |
b6cf8b3f JO |
137 | } |
138 | ||
139 | /* | |
140 | * Descriptor Bootstrap | |
141 | * | |
142 | * The descriptor array is minimally initialized to allow immediate usage | |
143 | * by readers and writers. The requirements that the descriptor array | |
144 | * initialization must satisfy: | |
145 | * | |
146 | * Req1 | |
147 | * The tail must point to an existing (committed or reusable) descriptor. | |
148 | * This is required by the implementation of prb_first_seq(). | |
149 | * | |
150 | * Req2 | |
151 | * Readers must see that the ringbuffer is initially empty. | |
152 | * | |
153 | * Req3 | |
154 | * The first record reserved by a writer is assigned sequence number 0. | |
155 | * | |
156 | * To satisfy Req1, the tail initially points to a descriptor that is | |
157 | * minimally initialized (having no data block, i.e. data-less with the | |
d397820f | 158 | * data block's lpos @begin and @next values set to FAILED_LPOS). |
b6cf8b3f JO |
159 | * |
160 | * To satisfy Req2, the initial tail descriptor is initialized to the | |
161 | * reusable state. Readers recognize reusable descriptors as existing | |
162 | * records, but skip over them. | |
163 | * | |
164 | * To satisfy Req3, the last descriptor in the array is used as the initial | |
165 | * head (and tail) descriptor. This allows the first record reserved by a | |
166 | * writer (head + 1) to be the first descriptor in the array. (Only the first | |
167 | * descriptor in the array could have a valid sequence number of 0.) | |
168 | * | |
169 | * The first time a descriptor is reserved, it is assigned a sequence number | |
170 | * with the value of the array index. A "first time reserved" descriptor can | |
171 | * be recognized because it has a sequence number of 0 but does not have an | |
172 | * index of 0. (Only the first descriptor in the array could have a valid | |
173 | * sequence number of 0.) After the first reservation, all future reservations | |
174 | * (recycling) simply involve incrementing the sequence number by the array | |
175 | * count. | |
176 | * | |
177 | * Hack #1 | |
178 | * Only the first descriptor in the array is allowed to have the sequence | |
179 | * number 0. In this case it is not possible to recognize if it is being | |
180 | * reserved the first time (set to index value) or has been reserved | |
181 | * previously (increment by the array count). This is handled by _always_ | |
182 | * incrementing the sequence number by the array count when reserving the | |
183 | * first descriptor in the array. In order to satisfy Req3, the sequence | |
184 | * number of the first descriptor in the array is initialized to minus | |
185 | * the array count. Then, upon the first reservation, it is incremented | |
186 | * to 0, thus satisfying Req3. | |
187 | * | |
188 | * Hack #2 | |
189 | * prb_first_seq() can be called at any time by readers to retrieve the | |
190 | * sequence number of the tail descriptor. However, due to Req2 and Req3, | |
191 | * initially there are no records to report the sequence number of | |
192 | * (sequence numbers are u64 and there is nothing less than 0). To handle | |
193 | * this, the sequence number of the initial tail descriptor is initialized | |
194 | * to 0. Technically this is incorrect, because there is no record with | |
195 | * sequence number 0 (yet) and the tail descriptor is not the first | |
196 | * descriptor in the array. But it allows prb_read_valid() to correctly | |
197 | * report the existence of a record for _any_ given sequence number at all | |
198 | * times. Bootstrapping is complete when the tail is pushed the first | |
199 | * time, thus finally pointing to the first descriptor reserved by a | |
200 | * writer, which has the assigned sequence number 0. | |
201 | */ | |
202 | ||
203 | /* | |
204 | * Initiating Logical Value Overflows | |
205 | * | |
206 | * Both logical position (lpos) and ID values can be mapped to array indexes | |
207 | * but may experience overflows during the lifetime of the system. To ensure | |
208 | * that printk_ringbuffer can handle the overflows for these types, initial | |
209 | * values are chosen that map to the correct initial array indexes, but will | |
210 | * result in overflows soon. | |
211 | * | |
212 | * BLK0_LPOS | |
213 | * The initial @head_lpos and @tail_lpos for data rings. It is at index | |
214 | * 0 and the lpos value is such that it will overflow on the first wrap. | |
215 | * | |
216 | * DESC0_ID | |
217 | * The initial @head_id and @tail_id for the desc ring. It is at the last | |
218 | * index of the descriptor array (see Req3 above) and the ID value is such | |
219 | * that it will overflow on the second wrap. | |
220 | */ | |
221 | #define BLK0_LPOS(sz_bits) (-(_DATA_SIZE(sz_bits))) | |
222 | #define DESC0_ID(ct_bits) DESC_ID(-(_DESCS_COUNT(ct_bits) + 1)) | |
10dcb06d | 223 | #define DESC0_SV(ct_bits) DESC_SV(DESC0_ID(ct_bits), desc_reusable) |
b6cf8b3f JO |
224 | |
225 | /* | |
226 | * Define a ringbuffer with an external text data buffer. The same as | |
227 | * DEFINE_PRINTKRB() but requires specifying an external buffer for the | |
228 | * text data. | |
229 | * | |
230 | * Note: The specified external buffer must be of the size: | |
231 | * 2 ^ (descbits + avgtextbits) | |
232 | */ | |
f35efc78 | 233 | #define _DEFINE_PRINTKRB(name, descbits, avgtextbits, text_buf) \ |
b6cf8b3f | 234 | static struct prb_desc _##name##_descs[_DESCS_COUNT(descbits)] = { \ |
b6cf8b3f JO |
235 | /* the initial head and tail */ \ |
236 | [_DESCS_COUNT(descbits) - 1] = { \ | |
b6cf8b3f JO |
237 | /* reusable */ \ |
238 | .state_var = ATOMIC_INIT(DESC0_SV(descbits)), \ | |
239 | /* no associated data block */ \ | |
d397820f | 240 | .text_blk_lpos = FAILED_BLK_LPOS, \ |
b6cf8b3f JO |
241 | }, \ |
242 | }; \ | |
cfe2790b JO |
243 | static struct printk_info _##name##_infos[_DESCS_COUNT(descbits)] = { \ |
244 | /* this will be the first record reserved by a writer */ \ | |
245 | [0] = { \ | |
246 | /* will be incremented to 0 on the first reservation */ \ | |
247 | .seq = -(u64)_DESCS_COUNT(descbits), \ | |
248 | }, \ | |
249 | /* the initial head and tail */ \ | |
250 | [_DESCS_COUNT(descbits) - 1] = { \ | |
251 | /* reports the first seq value during the bootstrap phase */ \ | |
252 | .seq = 0, \ | |
253 | }, \ | |
254 | }; \ | |
b6cf8b3f JO |
255 | static struct printk_ringbuffer name = { \ |
256 | .desc_ring = { \ | |
257 | .count_bits = descbits, \ | |
258 | .descs = &_##name##_descs[0], \ | |
cfe2790b | 259 | .infos = &_##name##_infos[0], \ |
b6cf8b3f JO |
260 | .head_id = ATOMIC_INIT(DESC0_ID(descbits)), \ |
261 | .tail_id = ATOMIC_INIT(DESC0_ID(descbits)), \ | |
5f72e52b | 262 | .last_finalized_seq = ATOMIC_INIT(0), \ |
b6cf8b3f JO |
263 | }, \ |
264 | .text_data_ring = { \ | |
265 | .size_bits = (avgtextbits) + (descbits), \ | |
266 | .data = text_buf, \ | |
267 | .head_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \ | |
268 | .tail_lpos = ATOMIC_LONG_INIT(BLK0_LPOS((avgtextbits) + (descbits))), \ | |
269 | }, \ | |
b6cf8b3f JO |
270 | .fail = ATOMIC_LONG_INIT(0), \ |
271 | } | |
272 | ||
273 | /** | |
274 | * DEFINE_PRINTKRB() - Define a ringbuffer. | |
275 | * | |
276 | * @name: The name of the ringbuffer variable. | |
277 | * @descbits: The number of descriptors as a power-of-2 value. | |
278 | * @avgtextbits: The average text data size per record as a power-of-2 value. | |
b6cf8b3f JO |
279 | * |
280 | * This is a macro for defining a ringbuffer and all internal structures | |
281 | * such that it is ready for immediate use. See _DEFINE_PRINTKRB() for a | |
282 | * variant where the text data buffer can be specified externally. | |
283 | */ | |
f35efc78 | 284 | #define DEFINE_PRINTKRB(name, descbits, avgtextbits) \ |
b6cf8b3f JO |
285 | static char _##name##_text[1U << ((avgtextbits) + (descbits))] \ |
286 | __aligned(__alignof__(unsigned long)); \ | |
f35efc78 | 287 | _DEFINE_PRINTKRB(name, descbits, avgtextbits, &_##name##_text[0]) |
b6cf8b3f JO |
288 | |
289 | /* Writer Interface */ | |
290 | ||
291 | /** | |
9bc284ca | 292 | * prb_rec_init_wr() - Initialize a buffer for writing records. |
b6cf8b3f JO |
293 | * |
294 | * @r: The record to initialize. | |
295 | * @text_buf_size: The needed text buffer size. | |
b6cf8b3f JO |
296 | */ |
297 | static inline void prb_rec_init_wr(struct printk_record *r, | |
f35efc78 | 298 | unsigned int text_buf_size) |
b6cf8b3f JO |
299 | { |
300 | r->info = NULL; | |
301 | r->text_buf = NULL; | |
b6cf8b3f | 302 | r->text_buf_size = text_buf_size; |
b6cf8b3f JO |
303 | } |
304 | ||
305 | bool prb_reserve(struct prb_reserved_entry *e, struct printk_ringbuffer *rb, | |
306 | struct printk_record *r); | |
4cfc7258 | 307 | bool prb_reserve_in_last(struct prb_reserved_entry *e, struct printk_ringbuffer *rb, |
59f8bcca | 308 | struct printk_record *r, u32 caller_id, unsigned int max_size); |
b6cf8b3f | 309 | void prb_commit(struct prb_reserved_entry *e); |
4cfc7258 | 310 | void prb_final_commit(struct prb_reserved_entry *e); |
b6cf8b3f JO |
311 | |
312 | void prb_init(struct printk_ringbuffer *rb, | |
313 | char *text_buf, unsigned int text_buf_size, | |
cfe2790b JO |
314 | struct prb_desc *descs, unsigned int descs_count_bits, |
315 | struct printk_info *infos); | |
b6cf8b3f JO |
316 | unsigned int prb_record_text_space(struct prb_reserved_entry *e); |
317 | ||
318 | /* Reader Interface */ | |
319 | ||
320 | /** | |
321 | * prb_rec_init_rd() - Initialize a buffer for reading records. | |
322 | * | |
323 | * @r: The record to initialize. | |
324 | * @info: A buffer to store record meta-data. | |
325 | * @text_buf: A buffer to store text data. | |
326 | * @text_buf_size: The size of @text_buf. | |
b6cf8b3f JO |
327 | * |
328 | * Initialize all the fields that a reader is interested in. All arguments | |
329 | * (except @r) are optional. Only record data for arguments that are | |
330 | * non-NULL or non-zero will be read. | |
331 | */ | |
332 | static inline void prb_rec_init_rd(struct printk_record *r, | |
333 | struct printk_info *info, | |
f35efc78 | 334 | char *text_buf, unsigned int text_buf_size) |
b6cf8b3f JO |
335 | { |
336 | r->info = info; | |
337 | r->text_buf = text_buf; | |
b6cf8b3f | 338 | r->text_buf_size = text_buf_size; |
b6cf8b3f JO |
339 | } |
340 | ||
341 | /** | |
342 | * prb_for_each_record() - Iterate over the records of a ringbuffer. | |
343 | * | |
344 | * @from: The sequence number to begin with. | |
345 | * @rb: The ringbuffer to iterate over. | |
346 | * @s: A u64 to store the sequence number on each iteration. | |
347 | * @r: A printk_record to store the record on each iteration. | |
348 | * | |
349 | * This is a macro for conveniently iterating over a ringbuffer. | |
350 | * Note that @s may not be the sequence number of the record on each | |
351 | * iteration. For the sequence number, @r->info->seq should be checked. | |
352 | * | |
353 | * Context: Any context. | |
354 | */ | |
355 | #define prb_for_each_record(from, rb, s, r) \ | |
356 | for ((s) = from; prb_read_valid(rb, s, r); (s) = (r)->info->seq + 1) | |
357 | ||
358 | /** | |
359 | * prb_for_each_info() - Iterate over the meta data of a ringbuffer. | |
360 | * | |
361 | * @from: The sequence number to begin with. | |
362 | * @rb: The ringbuffer to iterate over. | |
363 | * @s: A u64 to store the sequence number on each iteration. | |
364 | * @i: A printk_info to store the record meta data on each iteration. | |
365 | * @lc: An unsigned int to store the text line count of each record. | |
366 | * | |
367 | * This is a macro for conveniently iterating over a ringbuffer. | |
368 | * Note that @s may not be the sequence number of the record on each | |
369 | * iteration. For the sequence number, @r->info->seq should be checked. | |
370 | * | |
371 | * Context: Any context. | |
372 | */ | |
373 | #define prb_for_each_info(from, rb, s, i, lc) \ | |
374 | for ((s) = from; prb_read_valid_info(rb, s, i, lc); (s) = (i)->seq + 1) | |
375 | ||
376 | bool prb_read_valid(struct printk_ringbuffer *rb, u64 seq, | |
377 | struct printk_record *r); | |
378 | bool prb_read_valid_info(struct printk_ringbuffer *rb, u64 seq, | |
379 | struct printk_info *info, unsigned int *line_count); | |
380 | ||
90ad525c | 381 | u64 prb_first_seq(struct printk_ringbuffer *rb); |
b6cf8b3f JO |
382 | u64 prb_first_valid_seq(struct printk_ringbuffer *rb); |
383 | u64 prb_next_seq(struct printk_ringbuffer *rb); | |
384 | ||
5b73e706 JO |
385 | #ifdef CONFIG_64BIT |
386 | ||
387 | #define __u64seq_to_ulseq(u64seq) (u64seq) | |
388 | #define __ulseq_to_u64seq(rb, ulseq) (ulseq) | |
389 | ||
390 | #else /* CONFIG_64BIT */ | |
391 | ||
392 | #define __u64seq_to_ulseq(u64seq) ((u32)u64seq) | |
393 | ||
394 | static inline u64 __ulseq_to_u64seq(struct printk_ringbuffer *rb, u32 ulseq) | |
395 | { | |
90ad525c | 396 | u64 rb_first_seq = prb_first_seq(rb); |
5b73e706 | 397 | u64 seq; |
5b73e706 JO |
398 | |
399 | /* | |
400 | * The provided sequence is only the lower 32 bits of the ringbuffer | |
90ad525c | 401 | * sequence. It needs to be expanded to 64bit. Get the first sequence |
5b73e706 JO |
402 | * number from the ringbuffer and fold it. |
403 | * | |
404 | * Having a 32bit representation in the console is sufficient. | |
405 | * If a console ever gets more than 2^31 records behind | |
406 | * the ringbuffer then this is the least of the problems. | |
407 | * | |
408 | * Also the access to the ring buffer is always safe. | |
409 | */ | |
90ad525c | 410 | seq = rb_first_seq - (s32)((u32)rb_first_seq - ulseq); |
5b73e706 JO |
411 | |
412 | return seq; | |
413 | } | |
414 | ||
415 | #endif /* CONFIG_64BIT */ | |
416 | ||
b6cf8b3f | 417 | #endif /* _KERNEL_PRINTK_RINGBUFFER_H */ |