Commit | Line | Data |
---|---|---|
9251f904 BP |
1 | /* |
2 | * Performance events callchain code, extracted from core.c: | |
3 | * | |
4 | * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> | |
5 | * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar | |
90eec103 | 6 | * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra |
3723c632 | 7 | * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> |
9251f904 BP |
8 | * |
9 | * For licensing details see kernel-base/COPYING | |
10 | */ | |
11 | ||
12 | #include <linux/perf_event.h> | |
13 | #include <linux/slab.h> | |
68db0cf1 IM |
14 | #include <linux/sched/task_stack.h> |
15 | ||
9251f904 BP |
16 | #include "internal.h" |
17 | ||
18 | struct callchain_cpus_entries { | |
19 | struct rcu_head rcu_head; | |
20 | struct perf_callchain_entry *cpu_entries[0]; | |
21 | }; | |
22 | ||
c5dfd78e | 23 | int sysctl_perf_event_max_stack __read_mostly = PERF_MAX_STACK_DEPTH; |
c85b0334 | 24 | int sysctl_perf_event_max_contexts_per_stack __read_mostly = PERF_MAX_CONTEXTS_PER_STACK; |
c5dfd78e ACM |
25 | |
26 | static inline size_t perf_callchain_entry__sizeof(void) | |
27 | { | |
28 | return (sizeof(struct perf_callchain_entry) + | |
c85b0334 ACM |
29 | sizeof(__u64) * (sysctl_perf_event_max_stack + |
30 | sysctl_perf_event_max_contexts_per_stack)); | |
c5dfd78e ACM |
31 | } |
32 | ||
9251f904 BP |
33 | static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]); |
34 | static atomic_t nr_callchain_events; | |
35 | static DEFINE_MUTEX(callchain_mutex); | |
36 | static struct callchain_cpus_entries *callchain_cpus_entries; | |
37 | ||
38 | ||
cfbcf468 | 39 | __weak void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, |
9251f904 BP |
40 | struct pt_regs *regs) |
41 | { | |
42 | } | |
43 | ||
cfbcf468 | 44 | __weak void perf_callchain_user(struct perf_callchain_entry_ctx *entry, |
9251f904 BP |
45 | struct pt_regs *regs) |
46 | { | |
47 | } | |
48 | ||
49 | static void release_callchain_buffers_rcu(struct rcu_head *head) | |
50 | { | |
51 | struct callchain_cpus_entries *entries; | |
52 | int cpu; | |
53 | ||
54 | entries = container_of(head, struct callchain_cpus_entries, rcu_head); | |
55 | ||
56 | for_each_possible_cpu(cpu) | |
57 | kfree(entries->cpu_entries[cpu]); | |
58 | ||
59 | kfree(entries); | |
60 | } | |
61 | ||
62 | static void release_callchain_buffers(void) | |
63 | { | |
64 | struct callchain_cpus_entries *entries; | |
65 | ||
66 | entries = callchain_cpus_entries; | |
e0455e19 | 67 | RCU_INIT_POINTER(callchain_cpus_entries, NULL); |
9251f904 BP |
68 | call_rcu(&entries->rcu_head, release_callchain_buffers_rcu); |
69 | } | |
70 | ||
71 | static int alloc_callchain_buffers(void) | |
72 | { | |
73 | int cpu; | |
74 | int size; | |
75 | struct callchain_cpus_entries *entries; | |
76 | ||
77 | /* | |
78 | * We can't use the percpu allocation API for data that can be | |
79 | * accessed from NMI. Use a temporary manual per cpu allocation | |
80 | * until that gets sorted out. | |
81 | */ | |
82 | size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]); | |
83 | ||
84 | entries = kzalloc(size, GFP_KERNEL); | |
85 | if (!entries) | |
86 | return -ENOMEM; | |
87 | ||
c5dfd78e | 88 | size = perf_callchain_entry__sizeof() * PERF_NR_CONTEXTS; |
9251f904 BP |
89 | |
90 | for_each_possible_cpu(cpu) { | |
91 | entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL, | |
92 | cpu_to_node(cpu)); | |
93 | if (!entries->cpu_entries[cpu]) | |
94 | goto fail; | |
95 | } | |
96 | ||
97 | rcu_assign_pointer(callchain_cpus_entries, entries); | |
98 | ||
99 | return 0; | |
100 | ||
101 | fail: | |
102 | for_each_possible_cpu(cpu) | |
103 | kfree(entries->cpu_entries[cpu]); | |
104 | kfree(entries); | |
105 | ||
106 | return -ENOMEM; | |
107 | } | |
108 | ||
97c79a38 | 109 | int get_callchain_buffers(int event_max_stack) |
9251f904 BP |
110 | { |
111 | int err = 0; | |
112 | int count; | |
113 | ||
114 | mutex_lock(&callchain_mutex); | |
115 | ||
116 | count = atomic_inc_return(&nr_callchain_events); | |
117 | if (WARN_ON_ONCE(count < 1)) { | |
118 | err = -EINVAL; | |
119 | goto exit; | |
120 | } | |
121 | ||
5af44ca5 JO |
122 | /* |
123 | * If requesting per event more than the global cap, | |
124 | * return a different error to help userspace figure | |
125 | * this out. | |
126 | * | |
127 | * And also do it here so that we have &callchain_mutex held. | |
128 | */ | |
129 | if (event_max_stack > sysctl_perf_event_max_stack) { | |
130 | err = -EOVERFLOW; | |
131 | goto exit; | |
132 | } | |
133 | ||
bfb3d7b8 JO |
134 | if (count == 1) |
135 | err = alloc_callchain_buffers(); | |
9251f904 | 136 | exit: |
90983b16 FW |
137 | if (err) |
138 | atomic_dec(&nr_callchain_events); | |
9251f904 | 139 | |
fc3b86d6 FW |
140 | mutex_unlock(&callchain_mutex); |
141 | ||
9251f904 BP |
142 | return err; |
143 | } | |
144 | ||
145 | void put_callchain_buffers(void) | |
146 | { | |
147 | if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) { | |
148 | release_callchain_buffers(); | |
149 | mutex_unlock(&callchain_mutex); | |
150 | } | |
151 | } | |
152 | ||
153 | static struct perf_callchain_entry *get_callchain_entry(int *rctx) | |
154 | { | |
155 | int cpu; | |
156 | struct callchain_cpus_entries *entries; | |
157 | ||
4a32fea9 | 158 | *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion)); |
9251f904 BP |
159 | if (*rctx == -1) |
160 | return NULL; | |
161 | ||
162 | entries = rcu_dereference(callchain_cpus_entries); | |
163 | if (!entries) | |
164 | return NULL; | |
165 | ||
166 | cpu = smp_processor_id(); | |
167 | ||
c5dfd78e ACM |
168 | return (((void *)entries->cpu_entries[cpu]) + |
169 | (*rctx * perf_callchain_entry__sizeof())); | |
9251f904 BP |
170 | } |
171 | ||
172 | static void | |
173 | put_callchain_entry(int rctx) | |
174 | { | |
4a32fea9 | 175 | put_recursion_context(this_cpu_ptr(callchain_recursion), rctx); |
9251f904 BP |
176 | } |
177 | ||
568b329a AS |
178 | struct perf_callchain_entry * |
179 | get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user, | |
cfbcf468 | 180 | u32 max_stack, bool crosstask, bool add_mark) |
568b329a AS |
181 | { |
182 | struct perf_callchain_entry *entry; | |
cfbcf468 | 183 | struct perf_callchain_entry_ctx ctx; |
568b329a AS |
184 | int rctx; |
185 | ||
9251f904 BP |
186 | entry = get_callchain_entry(&rctx); |
187 | if (rctx == -1) | |
188 | return NULL; | |
189 | ||
190 | if (!entry) | |
191 | goto exit_put; | |
192 | ||
cfbcf468 ACM |
193 | ctx.entry = entry; |
194 | ctx.max_stack = max_stack; | |
3b1fff08 | 195 | ctx.nr = entry->nr = init_nr; |
c85b0334 ACM |
196 | ctx.contexts = 0; |
197 | ctx.contexts_maxed = false; | |
9251f904 | 198 | |
d0775264 | 199 | if (kernel && !user_mode(regs)) { |
568b329a | 200 | if (add_mark) |
3e4de4ec | 201 | perf_callchain_store_context(&ctx, PERF_CONTEXT_KERNEL); |
cfbcf468 | 202 | perf_callchain_kernel(&ctx, regs); |
9251f904 BP |
203 | } |
204 | ||
d0775264 FW |
205 | if (user) { |
206 | if (!user_mode(regs)) { | |
207 | if (current->mm) | |
208 | regs = task_pt_regs(current); | |
209 | else | |
210 | regs = NULL; | |
211 | } | |
212 | ||
213 | if (regs) { | |
88b0193d WD |
214 | mm_segment_t fs; |
215 | ||
568b329a | 216 | if (crosstask) |
d0775264 FW |
217 | goto exit_put; |
218 | ||
568b329a | 219 | if (add_mark) |
3e4de4ec | 220 | perf_callchain_store_context(&ctx, PERF_CONTEXT_USER); |
88b0193d WD |
221 | |
222 | fs = get_fs(); | |
223 | set_fs(USER_DS); | |
cfbcf468 | 224 | perf_callchain_user(&ctx, regs); |
88b0193d | 225 | set_fs(fs); |
d0775264 | 226 | } |
9251f904 BP |
227 | } |
228 | ||
229 | exit_put: | |
230 | put_callchain_entry(rctx); | |
231 | ||
232 | return entry; | |
233 | } | |
c5dfd78e | 234 | |
c85b0334 ACM |
235 | /* |
236 | * Used for sysctl_perf_event_max_stack and | |
237 | * sysctl_perf_event_max_contexts_per_stack. | |
238 | */ | |
c5dfd78e ACM |
239 | int perf_event_max_stack_handler(struct ctl_table *table, int write, |
240 | void __user *buffer, size_t *lenp, loff_t *ppos) | |
241 | { | |
a831100a ACM |
242 | int *value = table->data; |
243 | int new_value = *value, ret; | |
c5dfd78e ACM |
244 | struct ctl_table new_table = *table; |
245 | ||
246 | new_table.data = &new_value; | |
247 | ret = proc_dointvec_minmax(&new_table, write, buffer, lenp, ppos); | |
248 | if (ret || !write) | |
249 | return ret; | |
250 | ||
251 | mutex_lock(&callchain_mutex); | |
252 | if (atomic_read(&nr_callchain_events)) | |
253 | ret = -EBUSY; | |
254 | else | |
a831100a | 255 | *value = new_value; |
c5dfd78e ACM |
256 | |
257 | mutex_unlock(&callchain_mutex); | |
258 | ||
259 | return ret; | |
260 | } |