Commit | Line | Data |
---|---|---|
8e86e015 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9251f904 BP |
2 | /* |
3 | * Performance events callchain code, extracted from core.c: | |
4 | * | |
5 | * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> | |
6 | * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar | |
90eec103 | 7 | * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra |
3723c632 | 8 | * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
9251f904 BP |
9 | */ |
10 | ||
11 | #include <linux/perf_event.h> | |
12 | #include <linux/slab.h> | |
68db0cf1 IM |
13 | #include <linux/sched/task_stack.h> |
14 | ||
9251f904 BP |
15 | #include "internal.h" |
16 | ||
17 | struct callchain_cpus_entries { | |
18 | struct rcu_head rcu_head; | |
c50c75e9 | 19 | struct perf_callchain_entry *cpu_entries[]; |
9251f904 BP |
20 | }; |
21 | ||
c5dfd78e | 22 | int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH; |
c85b0334 | 23 | int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK; |
c5dfd78e ACM |
24 | |
25 | static inline size_t perf_callchain_entry__sizeof(void) | |
26 | { | |
27 | return (sizeof(struct perf_callchain_entry) + | |
c85b0334 ACM |
28 | sizeof(__u64) * (sysctl_perf_event_max_stack + |
29 | sysctl_perf_event_max_contexts_per_stack)); | |
c5dfd78e ACM |
30 | } |
31 | ||
9251f904 BP |
32 | static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]); |
33 | static atomic_t nr_callchain_events; | |
34 | static DEFINE_MUTEX(callchain_mutex); | |
35 | static struct callchain_cpus_entries *callchain_cpus_entries; | |
36 | ||
37 | ||
cfbcf468 | 38 | __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, |
9251f904 BP |
39 | struct pt_regs *regs) |
40 | { | |
41 | } | |
42 | ||
cfbcf468 | 43 | __weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry, |
9251f904 BP |
44 | struct pt_regs *regs) |
45 | { | |
46 | } | |
47 | ||
48 | static void release_callchain_buffers_rcu(struct rcu_head *head) | |
49 | { | |
50 | struct callchain_cpus_entries *entries; | |
51 | int cpu; | |
52 | ||
53 | entries = container_of(head, struct callchain_cpus_entries, rcu_head); | |
54 | ||
55 | for_each_possible_cpu(cpu) | |
56 | kfree(entries->cpu_entries[cpu]); | |
57 | ||
58 | kfree(entries); | |
59 | } | |
60 | ||
61 | static void release_callchain_buffers(void) | |
62 | { | |
63 | struct callchain_cpus_entries *entries; | |
64 | ||
65 | entries = callchain_cpus_entries; | |
e0455e19 | 66 | RCU_INIT_POINTER(callchain_cpus_entries, NULL); |
9251f904 BP |
67 | call_rcu(&entries->rcu_head, release_callchain_buffers_rcu); |
68 | } | |
69 | ||
70 | static int alloc_callchain_buffers(void) | |
71 | { | |
72 | int cpu; | |
73 | int size; | |
74 | struct callchain_cpus_entries *entries; | |
75 | ||
76 | /* | |
77 | * We can't use the percpu allocation API for data that can be | |
78 | * accessed from NMI. Use a temporary manual per cpu allocation | |
79 | * until that gets sorted out. | |
80 | */ | |
81 | size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]); | |
82 | ||
83 | entries = kzalloc(size, GFP_KERNEL); | |
84 | if (!entries) | |
85 | return -ENOMEM; | |
86 | ||
c5dfd78e | 87 | size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS; |
9251f904 BP |
88 | |
89 | for_each_possible_cpu(cpu) { | |
90 | entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL, | |
91 | cpu_to_node(cpu)); | |
92 | if (!entries->cpu_entries[cpu]) | |
93 | goto fail; | |
94 | } | |
95 | ||
96 | rcu_assign_pointer(callchain_cpus_entries, entries); | |
97 | ||
98 | return 0; | |
99 | ||
100 | fail: | |
101 | for_each_possible_cpu(cpu) | |
102 | kfree(entries->cpu_entries[cpu]); | |
103 | kfree(entries); | |
104 | ||
105 | return -ENOMEM; | |
106 | } | |
107 | ||
97c79a38 | 108 | int get_callchain_buffers(int event_max_stack) |
9251f904 BP |
109 | { |
110 | int err = 0; | |
111 | int count; | |
112 | ||
113 | mutex_lock(&callchain_mutex); | |
114 | ||
115 | count = atomic_inc_return(&nr_callchain_events); | |
116 | if (WARN_ON_ONCE(count < 1)) { | |
117 | err = -EINVAL; | |
118 | goto exit; | |
119 | } | |
120 | ||
5af44ca5 JO |
121 | /* |
122 | * If requesting per event more than the global cap, | |
123 | * return a different error to help userspace figure | |
124 | * this out. | |
125 | * | |
126 | * And also do it here so that we have &callchain_mutex held. | |
127 | */ | |
128 | if (event_max_stack > sysctl_perf_event_max_stack) { | |
129 | err = -EOVERFLOW; | |
130 | goto exit; | |
131 | } | |
132 | ||
bfb3d7b8 JO |
133 | if (count == 1) |
134 | err = alloc_callchain_buffers(); | |
9251f904 | 135 | exit: |
90983b16 FW |
136 | if (err) |
137 | atomic_dec(&nr_callchain_events); | |
9251f904 | 138 | |
fc3b86d6 FW |
139 | mutex_unlock(&callchain_mutex); |
140 | ||
9251f904 BP |
141 | return err; |
142 | } | |
143 | ||
144 | void put_callchain_buffers(void) | |
145 | { | |
146 | if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) { | |
147 | release_callchain_buffers(); | |
148 | mutex_unlock(&callchain_mutex); | |
149 | } | |
150 | } | |
151 | ||
d141b8bc | 152 | struct perf_callchain_entry *get_callchain_entry(int *rctx) |
9251f904 BP |
153 | { |
154 | int cpu; | |
155 | struct callchain_cpus_entries *entries; | |
156 | ||
4a32fea9 | 157 | *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion)); |
9251f904 BP |
158 | if (*rctx == -1) |
159 | return NULL; | |
160 | ||
161 | entries = rcu_dereference(callchain_cpus_entries); | |
d141b8bc SL |
162 | if (!entries) { |
163 | put_recursion_context(this_cpu_ptr(callchain_recursion), *rctx); | |
9251f904 | 164 | return NULL; |
d141b8bc | 165 | } |
9251f904 BP |
166 | |
167 | cpu = smp_processor_id(); | |
168 | ||
c5dfd78e ACM |
169 | return (((void *)entries->cpu_entries[cpu]) + |
170 | (*rctx * perf_callchain_entry__sizeof())); | |
9251f904 BP |
171 | } |
172 | ||
d141b8bc | 173 | void |
9251f904 BP |
174 | put_callchain_entry(int rctx) |
175 | { | |
4a32fea9 | 176 | put_recursion_context(this_cpu_ptr(callchain_recursion), rctx); |
9251f904 BP |
177 | } |
178 | ||
568b329a AS |
179 | struct perf_callchain_entry * |
180 | get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, | |
cfbcf468 | 181 | u32 max_stack, bool crosstask, bool add_mark) |
568b329a AS |
182 | { |
183 | struct perf_callchain_entry *entry; | |
cfbcf468 | 184 | struct perf_callchain_entry_ctx ctx; |
568b329a AS |
185 | int rctx; |
186 | ||
9251f904 | 187 | entry = get_callchain_entry(&rctx); |
9251f904 | 188 | if (!entry) |
d141b8bc | 189 | return NULL; |
9251f904 | 190 | |
cfbcf468 ACM |
191 | ctx.entry = entry; |
192 | ctx.max_stack = max_stack; | |
3b1fff08 | 193 | ctx.nr = entry->nr = init_nr; |
c85b0334 ACM |
194 | ctx.contexts = 0; |
195 | ctx.contexts_maxed = false; | |
9251f904 | 196 | |
d0775264 | 197 | if (kernel && !user_mode(regs)) { |
568b329a | 198 | if (add_mark) |
3e4de4ec | 199 | perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL); |
cfbcf468 | 200 | perf_callchain_kernel(&ctx, regs); |
9251f904 BP |
201 | } |
202 | ||
d0775264 FW |
203 | if (user) { |
204 | if (!user_mode(regs)) { | |
205 | if (current->mm) | |
206 | regs = task_pt_regs(current); | |
207 | else | |
208 | regs = NULL; | |
209 | } | |
210 | ||
211 | if (regs) { | |
568b329a | 212 | if (crosstask) |
d0775264 FW |
213 | goto exit_put; |
214 | ||
568b329a | 215 | if (add_mark) |
3e4de4ec | 216 | perf_callchain_store_context(&ctx, PERF_CONTEXT_USER); |
88b0193d | 217 | |
cfbcf468 | 218 | perf_callchain_user(&ctx, regs); |
d0775264 | 219 | } |
9251f904 BP |
220 | } |
221 | ||
222 | exit_put: | |
223 | put_callchain_entry(rctx); | |
224 | ||
225 | return entry; | |
226 | } | |
c5dfd78e | 227 | |
c85b0334 ACM |
228 | /* |
229 | * Used for sysctl_perf_event_max_stack and | |
230 | * sysctl_perf_event_max_contexts_per_stack. | |
231 | */ | |
c5dfd78e | 232 | int perf_event_max_stack_handler(struct ctl_table *table, int write, |
32927393 | 233 | void *buffer, size_t *lenp, loff_t *ppos) |
c5dfd78e | 234 | { |
a831100a ACM |
235 | int *value = table->data; |
236 | int new_value = *value, ret; | |
c5dfd78e ACM |
237 | struct ctl_table new_table = *table; |
238 | ||
239 | new_table.data = &new_value; | |
240 | ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos); | |
241 | if (ret || !write) | |
242 | return ret; | |
243 | ||
244 | mutex_lock(&callchain_mutex); | |
245 | if (atomic_read(&nr_callchain_events)) | |
246 | ret = -EBUSY; | |
247 | else | |
a831100a | 248 | *value = new_value; |
c5dfd78e ACM |
249 | |
250 | mutex_unlock(&callchain_mutex); | |
251 | ||
252 | return ret; | |
253 | } |