Commit | Line | Data |
---|---|---|
587064b6 PA |
1 | /* |
2 | * Copyright (C) 2014 ARM Limited | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | ||
c852f320 | 9 | #include <linux/cpu.h> |
587064b6 PA |
10 | #include <linux/init.h> |
11 | #include <linux/list.h> | |
bd35a4ad PA |
12 | #include <linux/perf_event.h> |
13 | #include <linux/sched.h> | |
587064b6 PA |
14 | #include <linux/slab.h> |
15 | #include <linux/sysctl.h> | |
16 | ||
bd35a4ad PA |
17 | #include <asm/insn.h> |
18 | #include <asm/opcodes.h> | |
19 | #include <asm/system_misc.h> | |
587064b6 | 20 | #include <asm/traps.h> |
bd35a4ad | 21 | #include <asm/uaccess.h> |
587064b6 | 22 | |
d784e298 PA |
23 | #define CREATE_TRACE_POINTS |
24 | #include "trace-events-emulation.h" | |
25 | ||
587064b6 PA |
26 | /* |
27 | * The runtime support for deprecated instruction support can be in one of | |
28 | * following three states - | |
29 | * | |
30 | * 0 = undef | |
31 | * 1 = emulate (software emulation) | |
32 | * 2 = hw (supported in hardware) | |
33 | */ | |
34 | enum insn_emulation_mode { | |
35 | INSN_UNDEF, | |
36 | INSN_EMULATE, | |
37 | INSN_HW, | |
38 | }; | |
39 | ||
40 | enum legacy_insn_status { | |
41 | INSN_DEPRECATED, | |
42 | INSN_OBSOLETE, | |
43 | }; | |
44 | ||
45 | struct insn_emulation_ops { | |
46 | const char *name; | |
47 | enum legacy_insn_status status; | |
48 | struct undef_hook *hooks; | |
49 | int (*set_hw_mode)(bool enable); | |
50 | }; | |
51 | ||
52 | struct insn_emulation { | |
53 | struct list_head node; | |
54 | struct insn_emulation_ops *ops; | |
55 | int current_mode; | |
56 | int min; | |
57 | int max; | |
58 | }; | |
59 | ||
60 | static LIST_HEAD(insn_emulation); | |
61 | static int nr_insn_emulated; | |
62 | static DEFINE_RAW_SPINLOCK(insn_emulation_lock); | |
63 | ||
64 | static void register_emulation_hooks(struct insn_emulation_ops *ops) | |
65 | { | |
66 | struct undef_hook *hook; | |
67 | ||
68 | BUG_ON(!ops->hooks); | |
69 | ||
70 | for (hook = ops->hooks; hook->instr_mask; hook++) | |
71 | register_undef_hook(hook); | |
72 | ||
73 | pr_notice("Registered %s emulation handler\n", ops->name); | |
74 | } | |
75 | ||
76 | static void remove_emulation_hooks(struct insn_emulation_ops *ops) | |
77 | { | |
78 | struct undef_hook *hook; | |
79 | ||
80 | BUG_ON(!ops->hooks); | |
81 | ||
82 | for (hook = ops->hooks; hook->instr_mask; hook++) | |
83 | unregister_undef_hook(hook); | |
84 | ||
85 | pr_notice("Removed %s emulation handler\n", ops->name); | |
86 | } | |
87 | ||
88 | static int update_insn_emulation_mode(struct insn_emulation *insn, | |
89 | enum insn_emulation_mode prev) | |
90 | { | |
91 | int ret = 0; | |
92 | ||
93 | switch (prev) { | |
94 | case INSN_UNDEF: /* Nothing to be done */ | |
95 | break; | |
96 | case INSN_EMULATE: | |
97 | remove_emulation_hooks(insn->ops); | |
98 | break; | |
99 | case INSN_HW: | |
100 | if (insn->ops->set_hw_mode) { | |
101 | insn->ops->set_hw_mode(false); | |
102 | pr_notice("Disabled %s support\n", insn->ops->name); | |
103 | } | |
104 | break; | |
105 | } | |
106 | ||
107 | switch (insn->current_mode) { | |
108 | case INSN_UNDEF: | |
109 | break; | |
110 | case INSN_EMULATE: | |
111 | register_emulation_hooks(insn->ops); | |
112 | break; | |
113 | case INSN_HW: | |
114 | if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true)) | |
115 | pr_notice("Enabled %s support\n", insn->ops->name); | |
116 | else | |
117 | ret = -EINVAL; | |
118 | break; | |
119 | } | |
120 | ||
121 | return ret; | |
122 | } | |
123 | ||
124 | static void register_insn_emulation(struct insn_emulation_ops *ops) | |
125 | { | |
126 | unsigned long flags; | |
127 | struct insn_emulation *insn; | |
128 | ||
129 | insn = kzalloc(sizeof(*insn), GFP_KERNEL); | |
130 | insn->ops = ops; | |
131 | insn->min = INSN_UNDEF; | |
132 | ||
133 | switch (ops->status) { | |
134 | case INSN_DEPRECATED: | |
135 | insn->current_mode = INSN_EMULATE; | |
136 | insn->max = INSN_HW; | |
137 | break; | |
138 | case INSN_OBSOLETE: | |
139 | insn->current_mode = INSN_UNDEF; | |
140 | insn->max = INSN_EMULATE; | |
141 | break; | |
142 | } | |
143 | ||
144 | raw_spin_lock_irqsave(&insn_emulation_lock, flags); | |
145 | list_add(&insn->node, &insn_emulation); | |
146 | nr_insn_emulated++; | |
147 | raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); | |
148 | ||
149 | /* Register any handlers if required */ | |
150 | update_insn_emulation_mode(insn, INSN_UNDEF); | |
151 | } | |
152 | ||
153 | static int emulation_proc_handler(struct ctl_table *table, int write, | |
154 | void __user *buffer, size_t *lenp, | |
155 | loff_t *ppos) | |
156 | { | |
157 | int ret = 0; | |
158 | struct insn_emulation *insn = (struct insn_emulation *) table->data; | |
159 | enum insn_emulation_mode prev_mode = insn->current_mode; | |
160 | ||
161 | table->data = &insn->current_mode; | |
162 | ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); | |
163 | ||
164 | if (ret || !write || prev_mode == insn->current_mode) | |
165 | goto ret; | |
166 | ||
167 | ret = update_insn_emulation_mode(insn, prev_mode); | |
90963395 | 168 | if (ret) { |
587064b6 PA |
169 | /* Mode change failed, revert to previous mode. */ |
170 | insn->current_mode = prev_mode; | |
171 | update_insn_emulation_mode(insn, INSN_UNDEF); | |
172 | } | |
173 | ret: | |
174 | table->data = insn; | |
175 | return ret; | |
176 | } | |
177 | ||
178 | static struct ctl_table ctl_abi[] = { | |
179 | { | |
180 | .procname = "abi", | |
181 | .mode = 0555, | |
182 | }, | |
183 | { } | |
184 | }; | |
185 | ||
186 | static void register_insn_emulation_sysctl(struct ctl_table *table) | |
187 | { | |
188 | unsigned long flags; | |
189 | int i = 0; | |
190 | struct insn_emulation *insn; | |
191 | struct ctl_table *insns_sysctl, *sysctl; | |
192 | ||
193 | insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1), | |
194 | GFP_KERNEL); | |
195 | ||
196 | raw_spin_lock_irqsave(&insn_emulation_lock, flags); | |
197 | list_for_each_entry(insn, &insn_emulation, node) { | |
198 | sysctl = &insns_sysctl[i]; | |
199 | ||
200 | sysctl->mode = 0644; | |
201 | sysctl->maxlen = sizeof(int); | |
202 | ||
203 | sysctl->procname = insn->ops->name; | |
204 | sysctl->data = insn; | |
205 | sysctl->extra1 = &insn->min; | |
206 | sysctl->extra2 = &insn->max; | |
207 | sysctl->proc_handler = emulation_proc_handler; | |
208 | i++; | |
209 | } | |
210 | raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); | |
211 | ||
212 | table->child = insns_sysctl; | |
213 | register_sysctl_table(table); | |
214 | } | |
215 | ||
bd35a4ad PA |
216 | /* |
217 | * Implement emulation of the SWP/SWPB instructions using load-exclusive and | |
218 | * store-exclusive. | |
219 | * | |
220 | * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] | |
221 | * Where: Rt = destination | |
222 | * Rt2 = source | |
223 | * Rn = address | |
224 | */ | |
225 | ||
226 | /* | |
227 | * Error-checking SWP macros implemented using ldxr{b}/stxr{b} | |
228 | */ | |
229 | #define __user_swpX_asm(data, addr, res, temp, B) \ | |
230 | __asm__ __volatile__( \ | |
231 | " mov %w2, %w1\n" \ | |
232 | "0: ldxr"B" %w1, [%3]\n" \ | |
233 | "1: stxr"B" %w0, %w2, [%3]\n" \ | |
234 | " cbz %w0, 2f\n" \ | |
235 | " mov %w0, %w4\n" \ | |
236 | "2:\n" \ | |
237 | " .pushsection .fixup,\"ax\"\n" \ | |
238 | " .align 2\n" \ | |
239 | "3: mov %w0, %w5\n" \ | |
240 | " b 2b\n" \ | |
241 | " .popsection" \ | |
242 | " .pushsection __ex_table,\"a\"\n" \ | |
243 | " .align 3\n" \ | |
244 | " .quad 0b, 3b\n" \ | |
245 | " .quad 1b, 3b\n" \ | |
246 | " .popsection" \ | |
247 | : "=&r" (res), "+r" (data), "=&r" (temp) \ | |
248 | : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ | |
249 | : "memory") | |
250 | ||
251 | #define __user_swp_asm(data, addr, res, temp) \ | |
252 | __user_swpX_asm(data, addr, res, temp, "") | |
253 | #define __user_swpb_asm(data, addr, res, temp) \ | |
254 | __user_swpX_asm(data, addr, res, temp, "b") | |
255 | ||
256 | /* | |
257 | * Bit 22 of the instruction encoding distinguishes between | |
258 | * the SWP and SWPB variants (bit set means SWPB). | |
259 | */ | |
260 | #define TYPE_SWPB (1 << 22) | |
261 | ||
262 | /* | |
263 | * Set up process info to signal segmentation fault - called on access error. | |
264 | */ | |
265 | static void set_segfault(struct pt_regs *regs, unsigned long addr) | |
266 | { | |
267 | siginfo_t info; | |
268 | ||
269 | down_read(¤t->mm->mmap_sem); | |
270 | if (find_vma(current->mm, addr) == NULL) | |
271 | info.si_code = SEGV_MAPERR; | |
272 | else | |
273 | info.si_code = SEGV_ACCERR; | |
274 | up_read(¤t->mm->mmap_sem); | |
275 | ||
276 | info.si_signo = SIGSEGV; | |
277 | info.si_errno = 0; | |
278 | info.si_addr = (void *) instruction_pointer(regs); | |
279 | ||
280 | pr_debug("SWP{B} emulation: access caused memory abort!\n"); | |
281 | arm64_notify_die("Illegal memory access", regs, &info, 0); | |
282 | } | |
283 | ||
284 | static int emulate_swpX(unsigned int address, unsigned int *data, | |
285 | unsigned int type) | |
286 | { | |
287 | unsigned int res = 0; | |
288 | ||
289 | if ((type != TYPE_SWPB) && (address & 0x3)) { | |
290 | /* SWP to unaligned address not permitted */ | |
291 | pr_debug("SWP instruction on unaligned pointer!\n"); | |
292 | return -EFAULT; | |
293 | } | |
294 | ||
295 | while (1) { | |
296 | unsigned long temp; | |
297 | ||
298 | if (type == TYPE_SWPB) | |
299 | __user_swpb_asm(*data, address, res, temp); | |
300 | else | |
301 | __user_swp_asm(*data, address, res, temp); | |
302 | ||
303 | if (likely(res != -EAGAIN) || signal_pending(current)) | |
304 | break; | |
305 | ||
306 | cond_resched(); | |
307 | } | |
308 | ||
309 | return res; | |
310 | } | |
311 | ||
312 | /* | |
313 | * swp_handler logs the id of calling process, dissects the instruction, sanity | |
314 | * checks the memory location, calls emulate_swpX for the actual operation and | |
315 | * deals with fixup/error handling before returning | |
316 | */ | |
317 | static int swp_handler(struct pt_regs *regs, u32 instr) | |
318 | { | |
319 | u32 destreg, data, type, address = 0; | |
320 | int rn, rt2, res = 0; | |
321 | ||
322 | perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); | |
323 | ||
324 | type = instr & TYPE_SWPB; | |
325 | ||
326 | switch (arm_check_condition(instr, regs->pstate)) { | |
327 | case ARM_OPCODE_CONDTEST_PASS: | |
328 | break; | |
329 | case ARM_OPCODE_CONDTEST_FAIL: | |
330 | /* Condition failed - return to next instruction */ | |
331 | goto ret; | |
332 | case ARM_OPCODE_CONDTEST_UNCOND: | |
333 | /* If unconditional encoding - not a SWP, undef */ | |
334 | return -EFAULT; | |
335 | default: | |
336 | return -EINVAL; | |
337 | } | |
338 | ||
339 | rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET); | |
340 | rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET); | |
341 | ||
342 | address = (u32)regs->user_regs.regs[rn]; | |
343 | data = (u32)regs->user_regs.regs[rt2]; | |
344 | destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET); | |
345 | ||
346 | pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", | |
347 | rn, address, destreg, | |
348 | aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data); | |
349 | ||
350 | /* Check access in reasonable access range for both SWP and SWPB */ | |
351 | if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { | |
352 | pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n", | |
353 | address); | |
354 | goto fault; | |
355 | } | |
356 | ||
357 | res = emulate_swpX(address, &data, type); | |
358 | if (res == -EFAULT) | |
359 | goto fault; | |
360 | else if (res == 0) | |
361 | regs->user_regs.regs[destreg] = data; | |
362 | ||
363 | ret: | |
d784e298 PA |
364 | if (type == TYPE_SWPB) |
365 | trace_instruction_emulation("swpb", regs->pc); | |
366 | else | |
367 | trace_instruction_emulation("swp", regs->pc); | |
368 | ||
bd35a4ad PA |
369 | pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n", |
370 | current->comm, (unsigned long)current->pid, regs->pc); | |
371 | ||
372 | regs->pc += 4; | |
373 | return 0; | |
374 | ||
375 | fault: | |
376 | set_segfault(regs, address); | |
377 | ||
378 | return 0; | |
379 | } | |
380 | ||
381 | /* | |
382 | * Only emulate SWP/SWPB executed in ARM state/User mode. | |
383 | * The kernel must be SWP free and SWP{B} does not exist in Thumb. | |
384 | */ | |
385 | static struct undef_hook swp_hooks[] = { | |
386 | { | |
387 | .instr_mask = 0x0fb00ff0, | |
388 | .instr_val = 0x01000090, | |
389 | .pstate_mask = COMPAT_PSR_MODE_MASK, | |
390 | .pstate_val = COMPAT_PSR_MODE_USR, | |
391 | .fn = swp_handler | |
392 | }, | |
393 | { } | |
394 | }; | |
395 | ||
396 | static struct insn_emulation_ops swp_ops = { | |
397 | .name = "swp", | |
398 | .status = INSN_OBSOLETE, | |
399 | .hooks = swp_hooks, | |
400 | .set_hw_mode = NULL, | |
401 | }; | |
402 | ||
c852f320 PA |
403 | static int cp15barrier_handler(struct pt_regs *regs, u32 instr) |
404 | { | |
405 | perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); | |
406 | ||
407 | switch (arm_check_condition(instr, regs->pstate)) { | |
408 | case ARM_OPCODE_CONDTEST_PASS: | |
409 | break; | |
410 | case ARM_OPCODE_CONDTEST_FAIL: | |
411 | /* Condition failed - return to next instruction */ | |
412 | goto ret; | |
413 | case ARM_OPCODE_CONDTEST_UNCOND: | |
414 | /* If unconditional encoding - not a barrier instruction */ | |
415 | return -EFAULT; | |
416 | default: | |
417 | return -EINVAL; | |
418 | } | |
419 | ||
420 | switch (aarch32_insn_mcr_extract_crm(instr)) { | |
421 | case 10: | |
422 | /* | |
423 | * dmb - mcr p15, 0, Rt, c7, c10, 5 | |
424 | * dsb - mcr p15, 0, Rt, c7, c10, 4 | |
425 | */ | |
d784e298 | 426 | if (aarch32_insn_mcr_extract_opc2(instr) == 5) { |
c852f320 | 427 | dmb(sy); |
d784e298 PA |
428 | trace_instruction_emulation( |
429 | "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc); | |
430 | } else { | |
c852f320 | 431 | dsb(sy); |
d784e298 PA |
432 | trace_instruction_emulation( |
433 | "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc); | |
434 | } | |
c852f320 PA |
435 | break; |
436 | case 5: | |
437 | /* | |
438 | * isb - mcr p15, 0, Rt, c7, c5, 4 | |
439 | * | |
440 | * Taking an exception or returning from one acts as an | |
441 | * instruction barrier. So no explicit barrier needed here. | |
442 | */ | |
d784e298 PA |
443 | trace_instruction_emulation( |
444 | "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc); | |
c852f320 PA |
445 | break; |
446 | } | |
447 | ||
448 | ret: | |
449 | pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n", | |
450 | current->comm, (unsigned long)current->pid, regs->pc); | |
451 | ||
452 | regs->pc += 4; | |
453 | return 0; | |
454 | } | |
455 | ||
456 | #define SCTLR_EL1_CP15BEN (1 << 5) | |
457 | ||
458 | static inline void config_sctlr_el1(u32 clear, u32 set) | |
459 | { | |
460 | u32 val; | |
461 | ||
462 | asm volatile("mrs %0, sctlr_el1" : "=r" (val)); | |
463 | val &= ~clear; | |
464 | val |= set; | |
465 | asm volatile("msr sctlr_el1, %0" : : "r" (val)); | |
466 | } | |
467 | ||
468 | static void enable_cp15_ben(void *info) | |
469 | { | |
470 | config_sctlr_el1(0, SCTLR_EL1_CP15BEN); | |
471 | } | |
472 | ||
473 | static void disable_cp15_ben(void *info) | |
474 | { | |
475 | config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); | |
476 | } | |
477 | ||
478 | static int cpu_hotplug_notify(struct notifier_block *b, | |
479 | unsigned long action, void *hcpu) | |
480 | { | |
481 | switch (action) { | |
482 | case CPU_STARTING: | |
483 | case CPU_STARTING_FROZEN: | |
484 | enable_cp15_ben(NULL); | |
485 | return NOTIFY_DONE; | |
486 | case CPU_DYING: | |
487 | case CPU_DYING_FROZEN: | |
488 | disable_cp15_ben(NULL); | |
489 | return NOTIFY_DONE; | |
490 | } | |
491 | ||
492 | return NOTIFY_OK; | |
493 | } | |
494 | ||
495 | static struct notifier_block cpu_hotplug_notifier = { | |
496 | .notifier_call = cpu_hotplug_notify, | |
497 | }; | |
498 | ||
499 | static int cp15_barrier_set_hw_mode(bool enable) | |
500 | { | |
501 | if (enable) { | |
502 | register_cpu_notifier(&cpu_hotplug_notifier); | |
503 | on_each_cpu(enable_cp15_ben, NULL, true); | |
504 | } else { | |
505 | unregister_cpu_notifier(&cpu_hotplug_notifier); | |
506 | on_each_cpu(disable_cp15_ben, NULL, true); | |
507 | } | |
508 | ||
509 | return true; | |
510 | } | |
511 | ||
512 | static struct undef_hook cp15_barrier_hooks[] = { | |
513 | { | |
514 | .instr_mask = 0x0fff0fdf, | |
515 | .instr_val = 0x0e070f9a, | |
516 | .pstate_mask = COMPAT_PSR_MODE_MASK, | |
517 | .pstate_val = COMPAT_PSR_MODE_USR, | |
518 | .fn = cp15barrier_handler, | |
519 | }, | |
520 | { | |
521 | .instr_mask = 0x0fff0fff, | |
522 | .instr_val = 0x0e070f95, | |
523 | .pstate_mask = COMPAT_PSR_MODE_MASK, | |
524 | .pstate_val = COMPAT_PSR_MODE_USR, | |
525 | .fn = cp15barrier_handler, | |
526 | }, | |
527 | { } | |
528 | }; | |
529 | ||
530 | static struct insn_emulation_ops cp15_barrier_ops = { | |
531 | .name = "cp15_barrier", | |
532 | .status = INSN_DEPRECATED, | |
533 | .hooks = cp15_barrier_hooks, | |
534 | .set_hw_mode = cp15_barrier_set_hw_mode, | |
535 | }; | |
536 | ||
587064b6 PA |
537 | /* |
538 | * Invoked as late_initcall, since not needed before init spawned. | |
539 | */ | |
540 | static int __init armv8_deprecated_init(void) | |
541 | { | |
bd35a4ad PA |
542 | if (IS_ENABLED(CONFIG_SWP_EMULATION)) |
543 | register_insn_emulation(&swp_ops); | |
544 | ||
c852f320 PA |
545 | if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) |
546 | register_insn_emulation(&cp15_barrier_ops); | |
547 | ||
587064b6 PA |
548 | register_insn_emulation_sysctl(ctl_abi); |
549 | ||
550 | return 0; | |
551 | } | |
552 | ||
553 | late_initcall(armv8_deprecated_init); |