Commit | Line | Data |
---|---|---|
36994e58 FW |
1 | /* |
2 | * Memory allocator tracing | |
3 | * | |
4 | * Copyright (C) 2008 Eduard - Gabriel Munteanu | |
5 | * Copyright (C) 2008 Pekka Enberg <penberg@cs.helsinki.fi> | |
6 | * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com> | |
7 | */ | |
8 | ||
c826e3cd IM |
9 | #include <linux/tracepoint.h> |
10 | #include <linux/seq_file.h> | |
36994e58 | 11 | #include <linux/debugfs.h> |
c826e3cd | 12 | #include <linux/dcache.h> |
36994e58 | 13 | #include <linux/fs.h> |
c826e3cd | 14 | |
02af61bb | 15 | #include <linux/kmemtrace.h> |
36994e58 | 16 | |
36994e58 | 17 | #include "trace_output.h" |
c826e3cd | 18 | #include "trace.h" |
36994e58 FW |
19 | |
20 | /* Select an alternative, minimalistic output than the original one */ | |
21 | #define TRACE_KMEM_OPT_MINIMAL 0x1 | |
22 | ||
23 | static struct tracer_opt kmem_opts[] = { | |
24 | /* Default disable the minimalistic output */ | |
25 | { TRACER_OPT(kmem_minimalistic, TRACE_KMEM_OPT_MINIMAL) }, | |
26 | { } | |
27 | }; | |
28 | ||
29 | static struct tracer_flags kmem_tracer_flags = { | |
c826e3cd IM |
30 | .val = 0, |
31 | .opts = kmem_opts | |
36994e58 FW |
32 | }; |
33 | ||
36994e58 FW |
34 | static struct trace_array *kmemtrace_array; |
35 | ||
ca2b84cb EGM |
36 | /* Trace allocations */ |
37 | static inline void kmemtrace_alloc(enum kmemtrace_type_id type_id, | |
38 | unsigned long call_site, | |
39 | const void *ptr, | |
40 | size_t bytes_req, | |
41 | size_t bytes_alloc, | |
42 | gfp_t gfp_flags, | |
43 | int node) | |
44 | { | |
e1112b4d | 45 | struct ftrace_event_call *call = &event_kmem_alloc; |
ca2b84cb | 46 | struct trace_array *tr = kmemtrace_array; |
c826e3cd IM |
47 | struct kmemtrace_alloc_entry *entry; |
48 | struct ring_buffer_event *event; | |
ca2b84cb EGM |
49 | |
50 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | |
51 | if (!event) | |
52 | return; | |
c826e3cd IM |
53 | |
54 | entry = ring_buffer_event_data(event); | |
ca2b84cb EGM |
55 | tracing_generic_entry_update(&entry->ent, 0, 0); |
56 | ||
c826e3cd IM |
57 | entry->ent.type = TRACE_KMEM_ALLOC; |
58 | entry->type_id = type_id; | |
59 | entry->call_site = call_site; | |
60 | entry->ptr = ptr; | |
61 | entry->bytes_req = bytes_req; | |
62 | entry->bytes_alloc = bytes_alloc; | |
63 | entry->gfp_flags = gfp_flags; | |
64 | entry->node = node; | |
ca2b84cb | 65 | |
eb02ce01 TZ |
66 | if (!filter_check_discard(call, entry, tr->buffer, event)) |
67 | ring_buffer_unlock_commit(tr->buffer, event); | |
ca2b84cb EGM |
68 | |
69 | trace_wake_up(); | |
70 | } | |
71 | ||
72 | static inline void kmemtrace_free(enum kmemtrace_type_id type_id, | |
73 | unsigned long call_site, | |
74 | const void *ptr) | |
75 | { | |
e1112b4d | 76 | struct ftrace_event_call *call = &event_kmem_free; |
ca2b84cb | 77 | struct trace_array *tr = kmemtrace_array; |
c826e3cd IM |
78 | struct kmemtrace_free_entry *entry; |
79 | struct ring_buffer_event *event; | |
ca2b84cb EGM |
80 | |
81 | event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry)); | |
82 | if (!event) | |
83 | return; | |
84 | entry = ring_buffer_event_data(event); | |
85 | tracing_generic_entry_update(&entry->ent, 0, 0); | |
86 | ||
c826e3cd IM |
87 | entry->ent.type = TRACE_KMEM_FREE; |
88 | entry->type_id = type_id; | |
89 | entry->call_site = call_site; | |
90 | entry->ptr = ptr; | |
ca2b84cb | 91 | |
eb02ce01 TZ |
92 | if (!filter_check_discard(call, entry, tr->buffer, event)) |
93 | ring_buffer_unlock_commit(tr->buffer, event); | |
ca2b84cb EGM |
94 | |
95 | trace_wake_up(); | |
96 | } | |
97 | ||
98 | static void kmemtrace_kmalloc(unsigned long call_site, | |
99 | const void *ptr, | |
100 | size_t bytes_req, | |
101 | size_t bytes_alloc, | |
102 | gfp_t gfp_flags) | |
103 | { | |
104 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | |
105 | bytes_req, bytes_alloc, gfp_flags, -1); | |
106 | } | |
107 | ||
108 | static void kmemtrace_kmem_cache_alloc(unsigned long call_site, | |
109 | const void *ptr, | |
110 | size_t bytes_req, | |
111 | size_t bytes_alloc, | |
112 | gfp_t gfp_flags) | |
113 | { | |
114 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | |
115 | bytes_req, bytes_alloc, gfp_flags, -1); | |
116 | } | |
117 | ||
118 | static void kmemtrace_kmalloc_node(unsigned long call_site, | |
119 | const void *ptr, | |
120 | size_t bytes_req, | |
121 | size_t bytes_alloc, | |
122 | gfp_t gfp_flags, | |
123 | int node) | |
124 | { | |
125 | kmemtrace_alloc(KMEMTRACE_TYPE_KMALLOC, call_site, ptr, | |
126 | bytes_req, bytes_alloc, gfp_flags, node); | |
127 | } | |
128 | ||
129 | static void kmemtrace_kmem_cache_alloc_node(unsigned long call_site, | |
130 | const void *ptr, | |
131 | size_t bytes_req, | |
132 | size_t bytes_alloc, | |
133 | gfp_t gfp_flags, | |
134 | int node) | |
135 | { | |
136 | kmemtrace_alloc(KMEMTRACE_TYPE_CACHE, call_site, ptr, | |
137 | bytes_req, bytes_alloc, gfp_flags, node); | |
138 | } | |
139 | ||
140 | static void kmemtrace_kfree(unsigned long call_site, const void *ptr) | |
141 | { | |
142 | kmemtrace_free(KMEMTRACE_TYPE_KMALLOC, call_site, ptr); | |
143 | } | |
144 | ||
145 | static void kmemtrace_kmem_cache_free(unsigned long call_site, const void *ptr) | |
146 | { | |
147 | kmemtrace_free(KMEMTRACE_TYPE_CACHE, call_site, ptr); | |
148 | } | |
149 | ||
150 | static int kmemtrace_start_probes(void) | |
151 | { | |
152 | int err; | |
153 | ||
154 | err = register_trace_kmalloc(kmemtrace_kmalloc); | |
155 | if (err) | |
156 | return err; | |
157 | err = register_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | |
158 | if (err) | |
159 | return err; | |
160 | err = register_trace_kmalloc_node(kmemtrace_kmalloc_node); | |
161 | if (err) | |
162 | return err; | |
163 | err = register_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | |
164 | if (err) | |
165 | return err; | |
166 | err = register_trace_kfree(kmemtrace_kfree); | |
167 | if (err) | |
168 | return err; | |
169 | err = register_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | |
170 | ||
171 | return err; | |
172 | } | |
173 | ||
174 | static void kmemtrace_stop_probes(void) | |
175 | { | |
176 | unregister_trace_kmalloc(kmemtrace_kmalloc); | |
177 | unregister_trace_kmem_cache_alloc(kmemtrace_kmem_cache_alloc); | |
178 | unregister_trace_kmalloc_node(kmemtrace_kmalloc_node); | |
179 | unregister_trace_kmem_cache_alloc_node(kmemtrace_kmem_cache_alloc_node); | |
180 | unregister_trace_kfree(kmemtrace_kfree); | |
181 | unregister_trace_kmem_cache_free(kmemtrace_kmem_cache_free); | |
182 | } | |
183 | ||
36994e58 FW |
184 | static int kmem_trace_init(struct trace_array *tr) |
185 | { | |
186 | int cpu; | |
187 | kmemtrace_array = tr; | |
188 | ||
3f237a79 | 189 | for_each_cpu(cpu, cpu_possible_mask) |
36994e58 FW |
190 | tracing_reset(tr, cpu); |
191 | ||
ca2b84cb | 192 | kmemtrace_start_probes(); |
36994e58 FW |
193 | |
194 | return 0; | |
195 | } | |
196 | ||
197 | static void kmem_trace_reset(struct trace_array *tr) | |
198 | { | |
ca2b84cb | 199 | kmemtrace_stop_probes(); |
36994e58 FW |
200 | } |
201 | ||
202 | static void kmemtrace_headers(struct seq_file *s) | |
203 | { | |
204 | /* Don't need headers for the original kmemtrace output */ | |
205 | if (!(kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL)) | |
206 | return; | |
207 | ||
208 | seq_printf(s, "#\n"); | |
209 | seq_printf(s, "# ALLOC TYPE REQ GIVEN FLAGS " | |
210 | " POINTER NODE CALLER\n"); | |
211 | seq_printf(s, "# FREE | | | | " | |
212 | " | | | |\n"); | |
213 | seq_printf(s, "# |\n\n"); | |
214 | } | |
215 | ||
216 | /* | |
42af9054 EGM |
217 | * The following functions give the original output from kmemtrace, |
218 | * plus the origin CPU, since reordering occurs in-kernel now. | |
36994e58 | 219 | */ |
42af9054 EGM |
220 | |
221 | #define KMEMTRACE_USER_ALLOC 0 | |
222 | #define KMEMTRACE_USER_FREE 1 | |
223 | ||
224 | struct kmemtrace_user_event { | |
c826e3cd IM |
225 | u8 event_id; |
226 | u8 type_id; | |
227 | u16 event_size; | |
228 | u32 cpu; | |
229 | u64 timestamp; | |
230 | unsigned long call_site; | |
231 | unsigned long ptr; | |
42af9054 EGM |
232 | }; |
233 | ||
234 | struct kmemtrace_user_event_alloc { | |
c826e3cd IM |
235 | size_t bytes_req; |
236 | size_t bytes_alloc; | |
237 | unsigned gfp_flags; | |
238 | int node; | |
42af9054 EGM |
239 | }; |
240 | ||
36994e58 | 241 | static enum print_line_t |
42af9054 EGM |
242 | kmemtrace_print_alloc_user(struct trace_iterator *iter, |
243 | struct kmemtrace_alloc_entry *entry) | |
36994e58 | 244 | { |
c826e3cd | 245 | struct kmemtrace_user_event_alloc *ev_alloc; |
36994e58 | 246 | struct trace_seq *s = &iter->seq; |
42af9054 | 247 | struct kmemtrace_user_event *ev; |
36994e58 | 248 | |
42af9054 EGM |
249 | ev = trace_seq_reserve(s, sizeof(*ev)); |
250 | if (!ev) | |
251 | return TRACE_TYPE_PARTIAL_LINE; | |
c826e3cd IM |
252 | |
253 | ev->event_id = KMEMTRACE_USER_ALLOC; | |
254 | ev->type_id = entry->type_id; | |
255 | ev->event_size = sizeof(*ev) + sizeof(*ev_alloc); | |
256 | ev->cpu = iter->cpu; | |
257 | ev->timestamp = iter->ts; | |
258 | ev->call_site = entry->call_site; | |
259 | ev->ptr = (unsigned long)entry->ptr; | |
42af9054 EGM |
260 | |
261 | ev_alloc = trace_seq_reserve(s, sizeof(*ev_alloc)); | |
262 | if (!ev_alloc) | |
36994e58 | 263 | return TRACE_TYPE_PARTIAL_LINE; |
c826e3cd IM |
264 | |
265 | ev_alloc->bytes_req = entry->bytes_req; | |
266 | ev_alloc->bytes_alloc = entry->bytes_alloc; | |
267 | ev_alloc->gfp_flags = entry->gfp_flags; | |
268 | ev_alloc->node = entry->node; | |
36994e58 FW |
269 | |
270 | return TRACE_TYPE_HANDLED; | |
271 | } | |
272 | ||
273 | static enum print_line_t | |
42af9054 EGM |
274 | kmemtrace_print_free_user(struct trace_iterator *iter, |
275 | struct kmemtrace_free_entry *entry) | |
36994e58 FW |
276 | { |
277 | struct trace_seq *s = &iter->seq; | |
42af9054 | 278 | struct kmemtrace_user_event *ev; |
36994e58 | 279 | |
42af9054 EGM |
280 | ev = trace_seq_reserve(s, sizeof(*ev)); |
281 | if (!ev) | |
36994e58 | 282 | return TRACE_TYPE_PARTIAL_LINE; |
c826e3cd IM |
283 | |
284 | ev->event_id = KMEMTRACE_USER_FREE; | |
285 | ev->type_id = entry->type_id; | |
286 | ev->event_size = sizeof(*ev); | |
287 | ev->cpu = iter->cpu; | |
288 | ev->timestamp = iter->ts; | |
289 | ev->call_site = entry->call_site; | |
290 | ev->ptr = (unsigned long)entry->ptr; | |
36994e58 FW |
291 | |
292 | return TRACE_TYPE_HANDLED; | |
293 | } | |
294 | ||
36994e58 FW |
295 | /* The two other following provide a more minimalistic output */ |
296 | static enum print_line_t | |
297 | kmemtrace_print_alloc_compress(struct trace_iterator *iter, | |
298 | struct kmemtrace_alloc_entry *entry) | |
299 | { | |
300 | struct trace_seq *s = &iter->seq; | |
301 | int ret; | |
302 | ||
303 | /* Alloc entry */ | |
304 | ret = trace_seq_printf(s, " + "); | |
305 | if (!ret) | |
306 | return TRACE_TYPE_PARTIAL_LINE; | |
307 | ||
308 | /* Type */ | |
309 | switch (entry->type_id) { | |
310 | case KMEMTRACE_TYPE_KMALLOC: | |
311 | ret = trace_seq_printf(s, "K "); | |
312 | break; | |
313 | case KMEMTRACE_TYPE_CACHE: | |
314 | ret = trace_seq_printf(s, "C "); | |
315 | break; | |
316 | case KMEMTRACE_TYPE_PAGES: | |
317 | ret = trace_seq_printf(s, "P "); | |
318 | break; | |
319 | default: | |
320 | ret = trace_seq_printf(s, "? "); | |
321 | } | |
322 | ||
323 | if (!ret) | |
324 | return TRACE_TYPE_PARTIAL_LINE; | |
325 | ||
326 | /* Requested */ | |
ecf441b5 | 327 | ret = trace_seq_printf(s, "%4zu ", entry->bytes_req); |
36994e58 FW |
328 | if (!ret) |
329 | return TRACE_TYPE_PARTIAL_LINE; | |
330 | ||
331 | /* Allocated */ | |
ecf441b5 | 332 | ret = trace_seq_printf(s, "%4zu ", entry->bytes_alloc); |
36994e58 FW |
333 | if (!ret) |
334 | return TRACE_TYPE_PARTIAL_LINE; | |
335 | ||
336 | /* Flags | |
337 | * TODO: would be better to see the name of the GFP flag names | |
338 | */ | |
339 | ret = trace_seq_printf(s, "%08x ", entry->gfp_flags); | |
340 | if (!ret) | |
341 | return TRACE_TYPE_PARTIAL_LINE; | |
342 | ||
343 | /* Pointer to allocated */ | |
344 | ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr); | |
345 | if (!ret) | |
346 | return TRACE_TYPE_PARTIAL_LINE; | |
347 | ||
348 | /* Node */ | |
349 | ret = trace_seq_printf(s, "%4d ", entry->node); | |
350 | if (!ret) | |
351 | return TRACE_TYPE_PARTIAL_LINE; | |
352 | ||
353 | /* Call site */ | |
354 | ret = seq_print_ip_sym(s, entry->call_site, 0); | |
355 | if (!ret) | |
356 | return TRACE_TYPE_PARTIAL_LINE; | |
357 | ||
358 | if (!trace_seq_printf(s, "\n")) | |
359 | return TRACE_TYPE_PARTIAL_LINE; | |
360 | ||
361 | return TRACE_TYPE_HANDLED; | |
362 | } | |
363 | ||
364 | static enum print_line_t | |
365 | kmemtrace_print_free_compress(struct trace_iterator *iter, | |
c826e3cd | 366 | struct kmemtrace_free_entry *entry) |
36994e58 FW |
367 | { |
368 | struct trace_seq *s = &iter->seq; | |
369 | int ret; | |
370 | ||
371 | /* Free entry */ | |
372 | ret = trace_seq_printf(s, " - "); | |
373 | if (!ret) | |
374 | return TRACE_TYPE_PARTIAL_LINE; | |
375 | ||
376 | /* Type */ | |
377 | switch (entry->type_id) { | |
378 | case KMEMTRACE_TYPE_KMALLOC: | |
379 | ret = trace_seq_printf(s, "K "); | |
380 | break; | |
381 | case KMEMTRACE_TYPE_CACHE: | |
382 | ret = trace_seq_printf(s, "C "); | |
383 | break; | |
384 | case KMEMTRACE_TYPE_PAGES: | |
385 | ret = trace_seq_printf(s, "P "); | |
386 | break; | |
387 | default: | |
388 | ret = trace_seq_printf(s, "? "); | |
389 | } | |
390 | ||
391 | if (!ret) | |
392 | return TRACE_TYPE_PARTIAL_LINE; | |
393 | ||
394 | /* Skip requested/allocated/flags */ | |
395 | ret = trace_seq_printf(s, " "); | |
396 | if (!ret) | |
397 | return TRACE_TYPE_PARTIAL_LINE; | |
398 | ||
399 | /* Pointer to allocated */ | |
400 | ret = trace_seq_printf(s, "0x%tx ", (ptrdiff_t)entry->ptr); | |
401 | if (!ret) | |
402 | return TRACE_TYPE_PARTIAL_LINE; | |
403 | ||
404 | /* Skip node */ | |
405 | ret = trace_seq_printf(s, " "); | |
406 | if (!ret) | |
407 | return TRACE_TYPE_PARTIAL_LINE; | |
408 | ||
409 | /* Call site */ | |
410 | ret = seq_print_ip_sym(s, entry->call_site, 0); | |
411 | if (!ret) | |
412 | return TRACE_TYPE_PARTIAL_LINE; | |
413 | ||
414 | if (!trace_seq_printf(s, "\n")) | |
415 | return TRACE_TYPE_PARTIAL_LINE; | |
416 | ||
417 | return TRACE_TYPE_HANDLED; | |
418 | } | |
419 | ||
420 | static enum print_line_t kmemtrace_print_line(struct trace_iterator *iter) | |
421 | { | |
422 | struct trace_entry *entry = iter->ent; | |
423 | ||
424 | switch (entry->type) { | |
425 | case TRACE_KMEM_ALLOC: { | |
426 | struct kmemtrace_alloc_entry *field; | |
c826e3cd | 427 | |
36994e58 FW |
428 | trace_assign_type(field, entry); |
429 | if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL) | |
430 | return kmemtrace_print_alloc_compress(iter, field); | |
431 | else | |
42af9054 | 432 | return kmemtrace_print_alloc_user(iter, field); |
36994e58 FW |
433 | } |
434 | ||
435 | case TRACE_KMEM_FREE: { | |
436 | struct kmemtrace_free_entry *field; | |
c826e3cd | 437 | |
36994e58 FW |
438 | trace_assign_type(field, entry); |
439 | if (kmem_tracer_flags.val & TRACE_KMEM_OPT_MINIMAL) | |
440 | return kmemtrace_print_free_compress(iter, field); | |
441 | else | |
42af9054 | 442 | return kmemtrace_print_free_user(iter, field); |
36994e58 FW |
443 | } |
444 | ||
445 | default: | |
446 | return TRACE_TYPE_UNHANDLED; | |
447 | } | |
448 | } | |
449 | ||
36994e58 | 450 | static struct tracer kmem_tracer __read_mostly = { |
c826e3cd IM |
451 | .name = "kmemtrace", |
452 | .init = kmem_trace_init, | |
453 | .reset = kmem_trace_reset, | |
454 | .print_line = kmemtrace_print_line, | |
455 | .print_header = kmemtrace_headers, | |
456 | .flags = &kmem_tracer_flags | |
36994e58 FW |
457 | }; |
458 | ||
3e806802 IM |
459 | void kmemtrace_init(void) |
460 | { | |
461 | /* earliest opportunity to start kmem tracing */ | |
462 | } | |
463 | ||
36994e58 FW |
464 | static int __init init_kmem_tracer(void) |
465 | { | |
466 | return register_tracer(&kmem_tracer); | |
467 | } | |
36994e58 | 468 | device_initcall(init_kmem_tracer); |