Commit | Line | Data |
---|---|---|
478fcb2c WD |
1 | /* |
2 | * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, | |
3 | * using the CPU's debug registers. | |
4 | * | |
5 | * Copyright (C) 2012 ARM Limited | |
6 | * Author: Will Deacon <will.deacon@arm.com> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify | |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | */ | |
20 | ||
21 | #define pr_fmt(fmt) "hw-breakpoint: " fmt | |
22 | ||
60fc6942 | 23 | #include <linux/cpu_pm.h> |
478fcb2c WD |
24 | #include <linux/errno.h> |
25 | #include <linux/hw_breakpoint.h> | |
26 | #include <linux/perf_event.h> | |
27 | #include <linux/ptrace.h> | |
28 | #include <linux/smp.h> | |
29 | ||
30 | #include <asm/compat.h> | |
31 | #include <asm/current.h> | |
32 | #include <asm/debug-monitors.h> | |
33 | #include <asm/hw_breakpoint.h> | |
34 | #include <asm/kdebug.h> | |
35 | #include <asm/traps.h> | |
36 | #include <asm/cputype.h> | |
37 | #include <asm/system_misc.h> | |
38 | ||
39 | /* Breakpoint currently in use for each BRP. */ | |
40 | static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); | |
41 | ||
42 | /* Watchpoint currently in use for each WRP. */ | |
43 | static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); | |
44 | ||
45 | /* Currently stepping a per-CPU kernel breakpoint. */ | |
46 | static DEFINE_PER_CPU(int, stepping_kernel_bp); | |
47 | ||
48 | /* Number of BRP/WRP registers on this CPU. */ | |
49 | static int core_num_brps; | |
50 | static int core_num_wrps; | |
51 | ||
52 | /* Determine number of BRP registers available. */ | |
53 | static int get_num_brps(void) | |
54 | { | |
55 | return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1; | |
56 | } | |
57 | ||
58 | /* Determine number of WRP registers available. */ | |
59 | static int get_num_wrps(void) | |
60 | { | |
61 | return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1; | |
62 | } | |
63 | ||
64 | int hw_breakpoint_slots(int type) | |
65 | { | |
66 | /* | |
67 | * We can be called early, so don't rely on | |
68 | * our static variables being initialised. | |
69 | */ | |
70 | switch (type) { | |
71 | case TYPE_INST: | |
72 | return get_num_brps(); | |
73 | case TYPE_DATA: | |
74 | return get_num_wrps(); | |
75 | default: | |
76 | pr_warning("unknown slot type: %d\n", type); | |
77 | return 0; | |
78 | } | |
79 | } | |
80 | ||
81 | #define READ_WB_REG_CASE(OFF, N, REG, VAL) \ | |
82 | case (OFF + N): \ | |
83 | AARCH64_DBG_READ(N, REG, VAL); \ | |
84 | break | |
85 | ||
86 | #define WRITE_WB_REG_CASE(OFF, N, REG, VAL) \ | |
87 | case (OFF + N): \ | |
88 | AARCH64_DBG_WRITE(N, REG, VAL); \ | |
89 | break | |
90 | ||
91 | #define GEN_READ_WB_REG_CASES(OFF, REG, VAL) \ | |
92 | READ_WB_REG_CASE(OFF, 0, REG, VAL); \ | |
93 | READ_WB_REG_CASE(OFF, 1, REG, VAL); \ | |
94 | READ_WB_REG_CASE(OFF, 2, REG, VAL); \ | |
95 | READ_WB_REG_CASE(OFF, 3, REG, VAL); \ | |
96 | READ_WB_REG_CASE(OFF, 4, REG, VAL); \ | |
97 | READ_WB_REG_CASE(OFF, 5, REG, VAL); \ | |
98 | READ_WB_REG_CASE(OFF, 6, REG, VAL); \ | |
99 | READ_WB_REG_CASE(OFF, 7, REG, VAL); \ | |
100 | READ_WB_REG_CASE(OFF, 8, REG, VAL); \ | |
101 | READ_WB_REG_CASE(OFF, 9, REG, VAL); \ | |
102 | READ_WB_REG_CASE(OFF, 10, REG, VAL); \ | |
103 | READ_WB_REG_CASE(OFF, 11, REG, VAL); \ | |
104 | READ_WB_REG_CASE(OFF, 12, REG, VAL); \ | |
105 | READ_WB_REG_CASE(OFF, 13, REG, VAL); \ | |
106 | READ_WB_REG_CASE(OFF, 14, REG, VAL); \ | |
107 | READ_WB_REG_CASE(OFF, 15, REG, VAL) | |
108 | ||
109 | #define GEN_WRITE_WB_REG_CASES(OFF, REG, VAL) \ | |
110 | WRITE_WB_REG_CASE(OFF, 0, REG, VAL); \ | |
111 | WRITE_WB_REG_CASE(OFF, 1, REG, VAL); \ | |
112 | WRITE_WB_REG_CASE(OFF, 2, REG, VAL); \ | |
113 | WRITE_WB_REG_CASE(OFF, 3, REG, VAL); \ | |
114 | WRITE_WB_REG_CASE(OFF, 4, REG, VAL); \ | |
115 | WRITE_WB_REG_CASE(OFF, 5, REG, VAL); \ | |
116 | WRITE_WB_REG_CASE(OFF, 6, REG, VAL); \ | |
117 | WRITE_WB_REG_CASE(OFF, 7, REG, VAL); \ | |
118 | WRITE_WB_REG_CASE(OFF, 8, REG, VAL); \ | |
119 | WRITE_WB_REG_CASE(OFF, 9, REG, VAL); \ | |
120 | WRITE_WB_REG_CASE(OFF, 10, REG, VAL); \ | |
121 | WRITE_WB_REG_CASE(OFF, 11, REG, VAL); \ | |
122 | WRITE_WB_REG_CASE(OFF, 12, REG, VAL); \ | |
123 | WRITE_WB_REG_CASE(OFF, 13, REG, VAL); \ | |
124 | WRITE_WB_REG_CASE(OFF, 14, REG, VAL); \ | |
125 | WRITE_WB_REG_CASE(OFF, 15, REG, VAL) | |
126 | ||
127 | static u64 read_wb_reg(int reg, int n) | |
128 | { | |
129 | u64 val = 0; | |
130 | ||
131 | switch (reg + n) { | |
132 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); | |
133 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); | |
134 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); | |
135 | GEN_READ_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); | |
136 | default: | |
137 | pr_warning("attempt to read from unknown breakpoint register %d\n", n); | |
138 | } | |
139 | ||
140 | return val; | |
141 | } | |
142 | ||
143 | static void write_wb_reg(int reg, int n, u64 val) | |
144 | { | |
145 | switch (reg + n) { | |
146 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BVR, AARCH64_DBG_REG_NAME_BVR, val); | |
147 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_BCR, AARCH64_DBG_REG_NAME_BCR, val); | |
148 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WVR, AARCH64_DBG_REG_NAME_WVR, val); | |
149 | GEN_WRITE_WB_REG_CASES(AARCH64_DBG_REG_WCR, AARCH64_DBG_REG_NAME_WCR, val); | |
150 | default: | |
151 | pr_warning("attempt to write to unknown breakpoint register %d\n", n); | |
152 | } | |
153 | isb(); | |
154 | } | |
155 | ||
156 | /* | |
157 | * Convert a breakpoint privilege level to the corresponding exception | |
158 | * level. | |
159 | */ | |
160 | static enum debug_el debug_exception_level(int privilege) | |
161 | { | |
162 | switch (privilege) { | |
163 | case AARCH64_BREAKPOINT_EL0: | |
164 | return DBG_ACTIVE_EL0; | |
165 | case AARCH64_BREAKPOINT_EL1: | |
166 | return DBG_ACTIVE_EL1; | |
167 | default: | |
168 | pr_warning("invalid breakpoint privilege level %d\n", privilege); | |
169 | return -EINVAL; | |
170 | } | |
171 | } | |
172 | ||
2f043045 LP |
173 | enum hw_breakpoint_ops { |
174 | HW_BREAKPOINT_INSTALL, | |
60fc6942 LP |
175 | HW_BREAKPOINT_UNINSTALL, |
176 | HW_BREAKPOINT_RESTORE | |
2f043045 LP |
177 | }; |
178 | ||
179 | /** | |
180 | * hw_breakpoint_slot_setup - Find and setup a perf slot according to | |
181 | * operations | |
182 | * | |
183 | * @slots: pointer to array of slots | |
184 | * @max_slots: max number of slots | |
185 | * @bp: perf_event to setup | |
186 | * @ops: operation to be carried out on the slot | |
187 | * | |
188 | * Return: | |
189 | * slot index on success | |
190 | * -ENOSPC if no slot is available/matches | |
191 | * -EINVAL on wrong operations parameter | |
478fcb2c | 192 | */ |
2f043045 LP |
193 | static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots, |
194 | struct perf_event *bp, | |
195 | enum hw_breakpoint_ops ops) | |
196 | { | |
197 | int i; | |
198 | struct perf_event **slot; | |
199 | ||
200 | for (i = 0; i < max_slots; ++i) { | |
201 | slot = &slots[i]; | |
202 | switch (ops) { | |
203 | case HW_BREAKPOINT_INSTALL: | |
204 | if (!*slot) { | |
205 | *slot = bp; | |
206 | return i; | |
207 | } | |
208 | break; | |
209 | case HW_BREAKPOINT_UNINSTALL: | |
210 | if (*slot == bp) { | |
211 | *slot = NULL; | |
212 | return i; | |
213 | } | |
214 | break; | |
60fc6942 LP |
215 | case HW_BREAKPOINT_RESTORE: |
216 | if (*slot == bp) | |
217 | return i; | |
218 | break; | |
2f043045 LP |
219 | default: |
220 | pr_warn_once("Unhandled hw breakpoint ops %d\n", ops); | |
221 | return -EINVAL; | |
222 | } | |
223 | } | |
224 | return -ENOSPC; | |
225 | } | |
226 | ||
227 | static int hw_breakpoint_control(struct perf_event *bp, | |
228 | enum hw_breakpoint_ops ops) | |
478fcb2c WD |
229 | { |
230 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
2f043045 | 231 | struct perf_event **slots; |
478fcb2c WD |
232 | struct debug_info *debug_info = ¤t->thread.debug; |
233 | int i, max_slots, ctrl_reg, val_reg, reg_enable; | |
2f043045 | 234 | enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege); |
478fcb2c WD |
235 | u32 ctrl; |
236 | ||
237 | if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { | |
238 | /* Breakpoint */ | |
239 | ctrl_reg = AARCH64_DBG_REG_BCR; | |
240 | val_reg = AARCH64_DBG_REG_BVR; | |
1436c1aa | 241 | slots = this_cpu_ptr(bp_on_reg); |
478fcb2c WD |
242 | max_slots = core_num_brps; |
243 | reg_enable = !debug_info->bps_disabled; | |
244 | } else { | |
245 | /* Watchpoint */ | |
246 | ctrl_reg = AARCH64_DBG_REG_WCR; | |
247 | val_reg = AARCH64_DBG_REG_WVR; | |
1436c1aa | 248 | slots = this_cpu_ptr(wp_on_reg); |
478fcb2c WD |
249 | max_slots = core_num_wrps; |
250 | reg_enable = !debug_info->wps_disabled; | |
251 | } | |
252 | ||
2f043045 | 253 | i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops); |
478fcb2c | 254 | |
2f043045 LP |
255 | if (WARN_ONCE(i < 0, "Can't find any breakpoint slot")) |
256 | return i; | |
478fcb2c | 257 | |
2f043045 LP |
258 | switch (ops) { |
259 | case HW_BREAKPOINT_INSTALL: | |
260 | /* | |
261 | * Ensure debug monitors are enabled at the correct exception | |
262 | * level. | |
263 | */ | |
264 | enable_debug_monitors(dbg_el); | |
60fc6942 LP |
265 | /* Fall through */ |
266 | case HW_BREAKPOINT_RESTORE: | |
2f043045 LP |
267 | /* Setup the address register. */ |
268 | write_wb_reg(val_reg, i, info->address); | |
478fcb2c | 269 | |
2f043045 LP |
270 | /* Setup the control register. */ |
271 | ctrl = encode_ctrl_reg(info->ctrl); | |
272 | write_wb_reg(ctrl_reg, i, | |
273 | reg_enable ? ctrl | 0x1 : ctrl & ~0x1); | |
274 | break; | |
275 | case HW_BREAKPOINT_UNINSTALL: | |
276 | /* Reset the control register. */ | |
277 | write_wb_reg(ctrl_reg, i, 0); | |
278 | ||
279 | /* | |
280 | * Release the debug monitors for the correct exception | |
281 | * level. | |
282 | */ | |
283 | disable_debug_monitors(dbg_el); | |
284 | break; | |
285 | } | |
478fcb2c WD |
286 | |
287 | return 0; | |
288 | } | |
289 | ||
2f043045 LP |
290 | /* |
291 | * Install a perf counter breakpoint. | |
292 | */ | |
293 | int arch_install_hw_breakpoint(struct perf_event *bp) | |
478fcb2c | 294 | { |
2f043045 LP |
295 | return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL); |
296 | } | |
478fcb2c | 297 | |
2f043045 LP |
298 | void arch_uninstall_hw_breakpoint(struct perf_event *bp) |
299 | { | |
300 | hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL); | |
478fcb2c WD |
301 | } |
302 | ||
303 | static int get_hbp_len(u8 hbp_len) | |
304 | { | |
305 | unsigned int len_in_bytes = 0; | |
306 | ||
307 | switch (hbp_len) { | |
308 | case ARM_BREAKPOINT_LEN_1: | |
309 | len_in_bytes = 1; | |
310 | break; | |
311 | case ARM_BREAKPOINT_LEN_2: | |
312 | len_in_bytes = 2; | |
313 | break; | |
314 | case ARM_BREAKPOINT_LEN_4: | |
315 | len_in_bytes = 4; | |
316 | break; | |
317 | case ARM_BREAKPOINT_LEN_8: | |
318 | len_in_bytes = 8; | |
319 | break; | |
320 | } | |
321 | ||
322 | return len_in_bytes; | |
323 | } | |
324 | ||
325 | /* | |
326 | * Check whether bp virtual address is in kernel space. | |
327 | */ | |
328 | int arch_check_bp_in_kernelspace(struct perf_event *bp) | |
329 | { | |
330 | unsigned int len; | |
331 | unsigned long va; | |
332 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
333 | ||
334 | va = info->address; | |
335 | len = get_hbp_len(info->ctrl.len); | |
336 | ||
337 | return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); | |
338 | } | |
339 | ||
340 | /* | |
341 | * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. | |
342 | * Hopefully this will disappear when ptrace can bypass the conversion | |
343 | * to generic breakpoint descriptions. | |
344 | */ | |
345 | int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, | |
346 | int *gen_len, int *gen_type) | |
347 | { | |
348 | /* Type */ | |
349 | switch (ctrl.type) { | |
350 | case ARM_BREAKPOINT_EXECUTE: | |
351 | *gen_type = HW_BREAKPOINT_X; | |
352 | break; | |
353 | case ARM_BREAKPOINT_LOAD: | |
354 | *gen_type = HW_BREAKPOINT_R; | |
355 | break; | |
356 | case ARM_BREAKPOINT_STORE: | |
357 | *gen_type = HW_BREAKPOINT_W; | |
358 | break; | |
359 | case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: | |
360 | *gen_type = HW_BREAKPOINT_RW; | |
361 | break; | |
362 | default: | |
363 | return -EINVAL; | |
364 | } | |
365 | ||
366 | /* Len */ | |
367 | switch (ctrl.len) { | |
368 | case ARM_BREAKPOINT_LEN_1: | |
369 | *gen_len = HW_BREAKPOINT_LEN_1; | |
370 | break; | |
371 | case ARM_BREAKPOINT_LEN_2: | |
372 | *gen_len = HW_BREAKPOINT_LEN_2; | |
373 | break; | |
374 | case ARM_BREAKPOINT_LEN_4: | |
375 | *gen_len = HW_BREAKPOINT_LEN_4; | |
376 | break; | |
377 | case ARM_BREAKPOINT_LEN_8: | |
378 | *gen_len = HW_BREAKPOINT_LEN_8; | |
379 | break; | |
380 | default: | |
381 | return -EINVAL; | |
382 | } | |
383 | ||
384 | return 0; | |
385 | } | |
386 | ||
387 | /* | |
388 | * Construct an arch_hw_breakpoint from a perf_event. | |
389 | */ | |
390 | static int arch_build_bp_info(struct perf_event *bp) | |
391 | { | |
392 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
393 | ||
394 | /* Type */ | |
395 | switch (bp->attr.bp_type) { | |
396 | case HW_BREAKPOINT_X: | |
397 | info->ctrl.type = ARM_BREAKPOINT_EXECUTE; | |
398 | break; | |
399 | case HW_BREAKPOINT_R: | |
400 | info->ctrl.type = ARM_BREAKPOINT_LOAD; | |
401 | break; | |
402 | case HW_BREAKPOINT_W: | |
403 | info->ctrl.type = ARM_BREAKPOINT_STORE; | |
404 | break; | |
405 | case HW_BREAKPOINT_RW: | |
406 | info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; | |
407 | break; | |
408 | default: | |
409 | return -EINVAL; | |
410 | } | |
411 | ||
412 | /* Len */ | |
413 | switch (bp->attr.bp_len) { | |
414 | case HW_BREAKPOINT_LEN_1: | |
415 | info->ctrl.len = ARM_BREAKPOINT_LEN_1; | |
416 | break; | |
417 | case HW_BREAKPOINT_LEN_2: | |
418 | info->ctrl.len = ARM_BREAKPOINT_LEN_2; | |
419 | break; | |
420 | case HW_BREAKPOINT_LEN_4: | |
421 | info->ctrl.len = ARM_BREAKPOINT_LEN_4; | |
422 | break; | |
423 | case HW_BREAKPOINT_LEN_8: | |
424 | info->ctrl.len = ARM_BREAKPOINT_LEN_8; | |
425 | break; | |
426 | default: | |
427 | return -EINVAL; | |
428 | } | |
429 | ||
430 | /* | |
431 | * On AArch64, we only permit breakpoints of length 4, whereas | |
432 | * AArch32 also requires breakpoints of length 2 for Thumb. | |
433 | * Watchpoints can be of length 1, 2, 4 or 8 bytes. | |
434 | */ | |
435 | if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { | |
436 | if (is_compat_task()) { | |
437 | if (info->ctrl.len != ARM_BREAKPOINT_LEN_2 && | |
438 | info->ctrl.len != ARM_BREAKPOINT_LEN_4) | |
439 | return -EINVAL; | |
440 | } else if (info->ctrl.len != ARM_BREAKPOINT_LEN_4) { | |
441 | /* | |
442 | * FIXME: Some tools (I'm looking at you perf) assume | |
443 | * that breakpoints should be sizeof(long). This | |
444 | * is nonsense. For now, we fix up the parameter | |
445 | * but we should probably return -EINVAL instead. | |
446 | */ | |
447 | info->ctrl.len = ARM_BREAKPOINT_LEN_4; | |
448 | } | |
449 | } | |
450 | ||
451 | /* Address */ | |
452 | info->address = bp->attr.bp_addr; | |
453 | ||
454 | /* | |
455 | * Privilege | |
456 | * Note that we disallow combined EL0/EL1 breakpoints because | |
457 | * that would complicate the stepping code. | |
458 | */ | |
459 | if (arch_check_bp_in_kernelspace(bp)) | |
460 | info->ctrl.privilege = AARCH64_BREAKPOINT_EL1; | |
461 | else | |
462 | info->ctrl.privilege = AARCH64_BREAKPOINT_EL0; | |
463 | ||
464 | /* Enabled? */ | |
465 | info->ctrl.enabled = !bp->attr.disabled; | |
466 | ||
467 | return 0; | |
468 | } | |
469 | ||
470 | /* | |
471 | * Validate the arch-specific HW Breakpoint register settings. | |
472 | */ | |
473 | int arch_validate_hwbkpt_settings(struct perf_event *bp) | |
474 | { | |
475 | struct arch_hw_breakpoint *info = counter_arch_bp(bp); | |
476 | int ret; | |
477 | u64 alignment_mask, offset; | |
478 | ||
479 | /* Build the arch_hw_breakpoint. */ | |
480 | ret = arch_build_bp_info(bp); | |
481 | if (ret) | |
482 | return ret; | |
483 | ||
484 | /* | |
485 | * Check address alignment. | |
486 | * We don't do any clever alignment correction for watchpoints | |
487 | * because using 64-bit unaligned addresses is deprecated for | |
488 | * AArch64. | |
489 | * | |
490 | * AArch32 tasks expect some simple alignment fixups, so emulate | |
491 | * that here. | |
492 | */ | |
493 | if (is_compat_task()) { | |
494 | if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) | |
495 | alignment_mask = 0x7; | |
496 | else | |
497 | alignment_mask = 0x3; | |
498 | offset = info->address & alignment_mask; | |
499 | switch (offset) { | |
500 | case 0: | |
501 | /* Aligned */ | |
502 | break; | |
503 | case 1: | |
504 | /* Allow single byte watchpoint. */ | |
505 | if (info->ctrl.len == ARM_BREAKPOINT_LEN_1) | |
506 | break; | |
507 | case 2: | |
508 | /* Allow halfword watchpoints and breakpoints. */ | |
509 | if (info->ctrl.len == ARM_BREAKPOINT_LEN_2) | |
510 | break; | |
511 | default: | |
512 | return -EINVAL; | |
513 | } | |
514 | ||
515 | info->address &= ~alignment_mask; | |
516 | info->ctrl.len <<= offset; | |
517 | } else { | |
518 | if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) | |
519 | alignment_mask = 0x3; | |
520 | else | |
521 | alignment_mask = 0x7; | |
522 | if (info->address & alignment_mask) | |
523 | return -EINVAL; | |
524 | } | |
525 | ||
526 | /* | |
527 | * Disallow per-task kernel breakpoints since these would | |
528 | * complicate the stepping code. | |
529 | */ | |
530 | if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.bp_target) | |
531 | return -EINVAL; | |
532 | ||
533 | return 0; | |
534 | } | |
535 | ||
536 | /* | |
537 | * Enable/disable all of the breakpoints active at the specified | |
538 | * exception level at the register level. | |
539 | * This is used when single-stepping after a breakpoint exception. | |
540 | */ | |
541 | static void toggle_bp_registers(int reg, enum debug_el el, int enable) | |
542 | { | |
543 | int i, max_slots, privilege; | |
544 | u32 ctrl; | |
545 | struct perf_event **slots; | |
546 | ||
547 | switch (reg) { | |
548 | case AARCH64_DBG_REG_BCR: | |
1436c1aa | 549 | slots = this_cpu_ptr(bp_on_reg); |
478fcb2c WD |
550 | max_slots = core_num_brps; |
551 | break; | |
552 | case AARCH64_DBG_REG_WCR: | |
1436c1aa | 553 | slots = this_cpu_ptr(wp_on_reg); |
478fcb2c WD |
554 | max_slots = core_num_wrps; |
555 | break; | |
556 | default: | |
557 | return; | |
558 | } | |
559 | ||
560 | for (i = 0; i < max_slots; ++i) { | |
561 | if (!slots[i]) | |
562 | continue; | |
563 | ||
564 | privilege = counter_arch_bp(slots[i])->ctrl.privilege; | |
565 | if (debug_exception_level(privilege) != el) | |
566 | continue; | |
567 | ||
568 | ctrl = read_wb_reg(reg, i); | |
569 | if (enable) | |
570 | ctrl |= 0x1; | |
571 | else | |
572 | ctrl &= ~0x1; | |
573 | write_wb_reg(reg, i, ctrl); | |
574 | } | |
575 | } | |
576 | ||
577 | /* | |
578 | * Debug exception handlers. | |
579 | */ | |
580 | static int breakpoint_handler(unsigned long unused, unsigned int esr, | |
581 | struct pt_regs *regs) | |
582 | { | |
583 | int i, step = 0, *kernel_step; | |
584 | u32 ctrl_reg; | |
585 | u64 addr, val; | |
586 | struct perf_event *bp, **slots; | |
587 | struct debug_info *debug_info; | |
588 | struct arch_hw_breakpoint_ctrl ctrl; | |
589 | ||
1436c1aa | 590 | slots = this_cpu_ptr(bp_on_reg); |
478fcb2c WD |
591 | addr = instruction_pointer(regs); |
592 | debug_info = ¤t->thread.debug; | |
593 | ||
594 | for (i = 0; i < core_num_brps; ++i) { | |
595 | rcu_read_lock(); | |
596 | ||
597 | bp = slots[i]; | |
598 | ||
599 | if (bp == NULL) | |
600 | goto unlock; | |
601 | ||
602 | /* Check if the breakpoint value matches. */ | |
603 | val = read_wb_reg(AARCH64_DBG_REG_BVR, i); | |
604 | if (val != (addr & ~0x3)) | |
605 | goto unlock; | |
606 | ||
607 | /* Possible match, check the byte address select to confirm. */ | |
608 | ctrl_reg = read_wb_reg(AARCH64_DBG_REG_BCR, i); | |
609 | decode_ctrl_reg(ctrl_reg, &ctrl); | |
610 | if (!((1 << (addr & 0x3)) & ctrl.len)) | |
611 | goto unlock; | |
612 | ||
613 | counter_arch_bp(bp)->trigger = addr; | |
614 | perf_bp_event(bp, regs); | |
615 | ||
616 | /* Do we need to handle the stepping? */ | |
617 | if (!bp->overflow_handler) | |
618 | step = 1; | |
619 | unlock: | |
620 | rcu_read_unlock(); | |
621 | } | |
622 | ||
623 | if (!step) | |
624 | return 0; | |
625 | ||
626 | if (user_mode(regs)) { | |
627 | debug_info->bps_disabled = 1; | |
628 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 0); | |
629 | ||
630 | /* If we're already stepping a watchpoint, just return. */ | |
631 | if (debug_info->wps_disabled) | |
632 | return 0; | |
633 | ||
634 | if (test_thread_flag(TIF_SINGLESTEP)) | |
635 | debug_info->suspended_step = 1; | |
636 | else | |
637 | user_enable_single_step(current); | |
638 | } else { | |
639 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 0); | |
1436c1aa | 640 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
478fcb2c WD |
641 | |
642 | if (*kernel_step != ARM_KERNEL_STEP_NONE) | |
643 | return 0; | |
644 | ||
645 | if (kernel_active_single_step()) { | |
646 | *kernel_step = ARM_KERNEL_STEP_SUSPEND; | |
647 | } else { | |
648 | *kernel_step = ARM_KERNEL_STEP_ACTIVE; | |
649 | kernel_enable_single_step(regs); | |
650 | } | |
651 | } | |
652 | ||
653 | return 0; | |
654 | } | |
655 | ||
656 | static int watchpoint_handler(unsigned long addr, unsigned int esr, | |
657 | struct pt_regs *regs) | |
658 | { | |
659 | int i, step = 0, *kernel_step, access; | |
660 | u32 ctrl_reg; | |
661 | u64 val, alignment_mask; | |
662 | struct perf_event *wp, **slots; | |
663 | struct debug_info *debug_info; | |
664 | struct arch_hw_breakpoint *info; | |
665 | struct arch_hw_breakpoint_ctrl ctrl; | |
666 | ||
1436c1aa | 667 | slots = this_cpu_ptr(wp_on_reg); |
478fcb2c WD |
668 | debug_info = ¤t->thread.debug; |
669 | ||
670 | for (i = 0; i < core_num_wrps; ++i) { | |
671 | rcu_read_lock(); | |
672 | ||
673 | wp = slots[i]; | |
674 | ||
675 | if (wp == NULL) | |
676 | goto unlock; | |
677 | ||
678 | info = counter_arch_bp(wp); | |
679 | /* AArch32 watchpoints are either 4 or 8 bytes aligned. */ | |
680 | if (is_compat_task()) { | |
681 | if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) | |
682 | alignment_mask = 0x7; | |
683 | else | |
684 | alignment_mask = 0x3; | |
685 | } else { | |
686 | alignment_mask = 0x7; | |
687 | } | |
688 | ||
689 | /* Check if the watchpoint value matches. */ | |
690 | val = read_wb_reg(AARCH64_DBG_REG_WVR, i); | |
691 | if (val != (addr & ~alignment_mask)) | |
692 | goto unlock; | |
693 | ||
694 | /* Possible match, check the byte address select to confirm. */ | |
695 | ctrl_reg = read_wb_reg(AARCH64_DBG_REG_WCR, i); | |
696 | decode_ctrl_reg(ctrl_reg, &ctrl); | |
697 | if (!((1 << (addr & alignment_mask)) & ctrl.len)) | |
698 | goto unlock; | |
699 | ||
700 | /* | |
701 | * Check that the access type matches. | |
702 | * 0 => load, otherwise => store | |
703 | */ | |
704 | access = (esr & AARCH64_ESR_ACCESS_MASK) ? HW_BREAKPOINT_W : | |
705 | HW_BREAKPOINT_R; | |
706 | if (!(access & hw_breakpoint_type(wp))) | |
707 | goto unlock; | |
708 | ||
709 | info->trigger = addr; | |
710 | perf_bp_event(wp, regs); | |
711 | ||
712 | /* Do we need to handle the stepping? */ | |
713 | if (!wp->overflow_handler) | |
714 | step = 1; | |
715 | ||
716 | unlock: | |
717 | rcu_read_unlock(); | |
718 | } | |
719 | ||
720 | if (!step) | |
721 | return 0; | |
722 | ||
723 | /* | |
724 | * We always disable EL0 watchpoints because the kernel can | |
725 | * cause these to fire via an unprivileged access. | |
726 | */ | |
727 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 0); | |
728 | ||
729 | if (user_mode(regs)) { | |
730 | debug_info->wps_disabled = 1; | |
731 | ||
732 | /* If we're already stepping a breakpoint, just return. */ | |
733 | if (debug_info->bps_disabled) | |
734 | return 0; | |
735 | ||
736 | if (test_thread_flag(TIF_SINGLESTEP)) | |
737 | debug_info->suspended_step = 1; | |
738 | else | |
739 | user_enable_single_step(current); | |
740 | } else { | |
741 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 0); | |
1436c1aa | 742 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
478fcb2c WD |
743 | |
744 | if (*kernel_step != ARM_KERNEL_STEP_NONE) | |
745 | return 0; | |
746 | ||
747 | if (kernel_active_single_step()) { | |
748 | *kernel_step = ARM_KERNEL_STEP_SUSPEND; | |
749 | } else { | |
750 | *kernel_step = ARM_KERNEL_STEP_ACTIVE; | |
751 | kernel_enable_single_step(regs); | |
752 | } | |
753 | } | |
754 | ||
755 | return 0; | |
756 | } | |
757 | ||
758 | /* | |
759 | * Handle single-step exception. | |
760 | */ | |
761 | int reinstall_suspended_bps(struct pt_regs *regs) | |
762 | { | |
763 | struct debug_info *debug_info = ¤t->thread.debug; | |
764 | int handled_exception = 0, *kernel_step; | |
765 | ||
1436c1aa | 766 | kernel_step = this_cpu_ptr(&stepping_kernel_bp); |
478fcb2c WD |
767 | |
768 | /* | |
769 | * Called from single-step exception handler. | |
770 | * Return 0 if execution can resume, 1 if a SIGTRAP should be | |
771 | * reported. | |
772 | */ | |
773 | if (user_mode(regs)) { | |
774 | if (debug_info->bps_disabled) { | |
775 | debug_info->bps_disabled = 0; | |
776 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL0, 1); | |
777 | handled_exception = 1; | |
778 | } | |
779 | ||
780 | if (debug_info->wps_disabled) { | |
781 | debug_info->wps_disabled = 0; | |
782 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); | |
783 | handled_exception = 1; | |
784 | } | |
785 | ||
786 | if (handled_exception) { | |
787 | if (debug_info->suspended_step) { | |
788 | debug_info->suspended_step = 0; | |
789 | /* Allow exception handling to fall-through. */ | |
790 | handled_exception = 0; | |
791 | } else { | |
792 | user_disable_single_step(current); | |
793 | } | |
794 | } | |
795 | } else if (*kernel_step != ARM_KERNEL_STEP_NONE) { | |
796 | toggle_bp_registers(AARCH64_DBG_REG_BCR, DBG_ACTIVE_EL1, 1); | |
797 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL1, 1); | |
798 | ||
799 | if (!debug_info->wps_disabled) | |
800 | toggle_bp_registers(AARCH64_DBG_REG_WCR, DBG_ACTIVE_EL0, 1); | |
801 | ||
802 | if (*kernel_step != ARM_KERNEL_STEP_SUSPEND) { | |
803 | kernel_disable_single_step(); | |
804 | handled_exception = 1; | |
805 | } else { | |
806 | handled_exception = 0; | |
807 | } | |
808 | ||
809 | *kernel_step = ARM_KERNEL_STEP_NONE; | |
810 | } | |
811 | ||
812 | return !handled_exception; | |
813 | } | |
814 | ||
815 | /* | |
816 | * Context-switcher for restoring suspended breakpoints. | |
817 | */ | |
818 | void hw_breakpoint_thread_switch(struct task_struct *next) | |
819 | { | |
820 | /* | |
821 | * current next | |
822 | * disabled: 0 0 => The usual case, NOTIFY_DONE | |
823 | * 0 1 => Disable the registers | |
824 | * 1 0 => Enable the registers | |
825 | * 1 1 => NOTIFY_DONE. per-task bps will | |
826 | * get taken care of by perf. | |
827 | */ | |
828 | ||
829 | struct debug_info *current_debug_info, *next_debug_info; | |
830 | ||
831 | current_debug_info = ¤t->thread.debug; | |
832 | next_debug_info = &next->thread.debug; | |
833 | ||
834 | /* Update breakpoints. */ | |
835 | if (current_debug_info->bps_disabled != next_debug_info->bps_disabled) | |
836 | toggle_bp_registers(AARCH64_DBG_REG_BCR, | |
837 | DBG_ACTIVE_EL0, | |
838 | !next_debug_info->bps_disabled); | |
839 | ||
840 | /* Update watchpoints. */ | |
841 | if (current_debug_info->wps_disabled != next_debug_info->wps_disabled) | |
842 | toggle_bp_registers(AARCH64_DBG_REG_WCR, | |
843 | DBG_ACTIVE_EL0, | |
844 | !next_debug_info->wps_disabled); | |
845 | } | |
846 | ||
847 | /* | |
848 | * CPU initialisation. | |
849 | */ | |
60fc6942 | 850 | static void hw_breakpoint_reset(void *unused) |
478fcb2c WD |
851 | { |
852 | int i; | |
60fc6942 LP |
853 | struct perf_event **slots; |
854 | /* | |
855 | * When a CPU goes through cold-boot, it does not have any installed | |
856 | * slot, so it is safe to share the same function for restoring and | |
857 | * resetting breakpoints; when a CPU is hotplugged in, it goes | |
858 | * through the slots, which are all empty, hence it just resets control | |
859 | * and value for debug registers. | |
860 | * When this function is triggered on warm-boot through a CPU PM | |
861 | * notifier some slots might be initialized; if so they are | |
862 | * reprogrammed according to the debug slots content. | |
863 | */ | |
864 | for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) { | |
865 | if (slots[i]) { | |
866 | hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); | |
867 | } else { | |
868 | write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL); | |
869 | write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL); | |
870 | } | |
478fcb2c WD |
871 | } |
872 | ||
60fc6942 LP |
873 | for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) { |
874 | if (slots[i]) { | |
875 | hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE); | |
876 | } else { | |
877 | write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL); | |
878 | write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL); | |
879 | } | |
478fcb2c WD |
880 | } |
881 | } | |
882 | ||
b8c6453a | 883 | static int hw_breakpoint_reset_notify(struct notifier_block *self, |
478fcb2c WD |
884 | unsigned long action, |
885 | void *hcpu) | |
886 | { | |
887 | int cpu = (long)hcpu; | |
888 | if (action == CPU_ONLINE) | |
60fc6942 | 889 | smp_call_function_single(cpu, hw_breakpoint_reset, NULL, 1); |
478fcb2c WD |
890 | return NOTIFY_OK; |
891 | } | |
892 | ||
b8c6453a | 893 | static struct notifier_block hw_breakpoint_reset_nb = { |
478fcb2c WD |
894 | .notifier_call = hw_breakpoint_reset_notify, |
895 | }; | |
896 | ||
60fc6942 LP |
897 | #ifdef CONFIG_CPU_PM |
898 | static int hw_breakpoint_cpu_pm_notify(struct notifier_block *self, | |
899 | unsigned long action, | |
900 | void *v) | |
901 | { | |
902 | if (action == CPU_PM_EXIT) { | |
903 | hw_breakpoint_reset(NULL); | |
904 | return NOTIFY_OK; | |
905 | } | |
906 | ||
907 | return NOTIFY_DONE; | |
908 | } | |
909 | ||
910 | static struct notifier_block hw_breakpoint_cpu_pm_nb = { | |
911 | .notifier_call = hw_breakpoint_cpu_pm_notify, | |
912 | }; | |
913 | ||
914 | static void __init hw_breakpoint_pm_init(void) | |
915 | { | |
916 | cpu_pm_register_notifier(&hw_breakpoint_cpu_pm_nb); | |
917 | } | |
918 | #else | |
919 | static inline void hw_breakpoint_pm_init(void) | |
920 | { | |
921 | } | |
922 | #endif | |
923 | ||
478fcb2c WD |
924 | /* |
925 | * One-time initialisation. | |
926 | */ | |
927 | static int __init arch_hw_breakpoint_init(void) | |
928 | { | |
929 | core_num_brps = get_num_brps(); | |
930 | core_num_wrps = get_num_wrps(); | |
931 | ||
932 | pr_info("found %d breakpoint and %d watchpoint registers.\n", | |
933 | core_num_brps, core_num_wrps); | |
934 | ||
935 | /* | |
936 | * Reset the breakpoint resources. We assume that a halting | |
937 | * debugger will leave the world in a nice state for us. | |
938 | */ | |
60fc6942 LP |
939 | smp_call_function(hw_breakpoint_reset, NULL, 1); |
940 | hw_breakpoint_reset(NULL); | |
478fcb2c WD |
941 | |
942 | /* Register debug fault handlers. */ | |
943 | hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP, | |
944 | TRAP_HWBKPT, "hw-breakpoint handler"); | |
945 | hook_debug_fault_code(DBG_ESR_EVT_HWWP, watchpoint_handler, SIGTRAP, | |
946 | TRAP_HWBKPT, "hw-watchpoint handler"); | |
947 | ||
948 | /* Register hotplug notifier. */ | |
949 | register_cpu_notifier(&hw_breakpoint_reset_nb); | |
60fc6942 | 950 | hw_breakpoint_pm_init(); |
478fcb2c WD |
951 | |
952 | return 0; | |
953 | } | |
954 | arch_initcall(arch_hw_breakpoint_init); | |
955 | ||
956 | void hw_breakpoint_pmu_read(struct perf_event *bp) | |
957 | { | |
958 | } | |
959 | ||
960 | /* | |
961 | * Dummy function to register with die_notifier. | |
962 | */ | |
963 | int hw_breakpoint_exceptions_notify(struct notifier_block *unused, | |
964 | unsigned long val, void *data) | |
965 | { | |
966 | return NOTIFY_DONE; | |
967 | } |