Commit | Line | Data |
---|---|---|
3fc24ef3 AB |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | // based on arch/arm/mm/alignment.c | |
3 | ||
4 | #include <linux/compiler.h> | |
5 | #include <linux/errno.h> | |
6 | #include <linux/kernel.h> | |
7 | #include <linux/init.h> | |
8 | #include <linux/perf_event.h> | |
9 | #include <linux/uaccess.h> | |
10 | ||
11 | #include <asm/exception.h> | |
12 | #include <asm/ptrace.h> | |
13 | #include <asm/traps.h> | |
14 | ||
15 | /* | |
16 | * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998 | |
17 | * | |
18 | * Speed optimisations and better fault handling by Russell King. | |
19 | */ | |
20 | #define CODING_BITS(i) (i & 0x0e000000) | |
21 | ||
22 | #define LDST_P_BIT(i) (i & (1 << 24)) /* Preindex */ | |
23 | #define LDST_U_BIT(i) (i & (1 << 23)) /* Add offset */ | |
24 | #define LDST_W_BIT(i) (i & (1 << 21)) /* Writeback */ | |
25 | #define LDST_L_BIT(i) (i & (1 << 20)) /* Load */ | |
26 | ||
27 | #define LDST_P_EQ_U(i) ((((i) ^ ((i) >> 1)) & (1 << 23)) == 0) | |
28 | ||
29 | #define LDSTHD_I_BIT(i) (i & (1 << 22)) /* double/half-word immed */ | |
30 | ||
31 | #define RN_BITS(i) ((i >> 16) & 15) /* Rn */ | |
32 | #define RD_BITS(i) ((i >> 12) & 15) /* Rd */ | |
33 | #define RM_BITS(i) (i & 15) /* Rm */ | |
34 | ||
35 | #define REGMASK_BITS(i) (i & 0xffff) | |
36 | ||
37 | #define BAD_INSTR 0xdeadc0de | |
38 | ||
39 | /* Thumb-2 32 bit format per ARMv7 DDI0406A A6.3, either f800h,e800h,f800h */ | |
40 | #define IS_T32(hi16) \ | |
41 | (((hi16) & 0xe000) == 0xe000 && ((hi16) & 0x1800)) | |
42 | ||
43 | union offset_union { | |
44 | unsigned long un; | |
45 | signed long sn; | |
46 | }; | |
47 | ||
48 | #define TYPE_ERROR 0 | |
49 | #define TYPE_FAULT 1 | |
50 | #define TYPE_LDST 2 | |
51 | #define TYPE_DONE 3 | |
52 | ||
53 | static void | |
54 | do_alignment_finish_ldst(unsigned long addr, u32 instr, struct pt_regs *regs, | |
55 | union offset_union offset) | |
56 | { | |
57 | if (!LDST_U_BIT(instr)) | |
58 | offset.un = -offset.un; | |
59 | ||
60 | if (!LDST_P_BIT(instr)) | |
61 | addr += offset.un; | |
62 | ||
63 | if (!LDST_P_BIT(instr) || LDST_W_BIT(instr)) | |
64 | regs->regs[RN_BITS(instr)] = addr; | |
65 | } | |
66 | ||
67 | static int | |
68 | do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs) | |
69 | { | |
70 | unsigned int rd = RD_BITS(instr); | |
71 | unsigned int rd2; | |
72 | int load; | |
73 | ||
74 | if ((instr & 0xfe000000) == 0xe8000000) { | |
75 | /* ARMv7 Thumb-2 32-bit LDRD/STRD */ | |
76 | rd2 = (instr >> 8) & 0xf; | |
77 | load = !!(LDST_L_BIT(instr)); | |
78 | } else if (((rd & 1) == 1) || (rd == 14)) { | |
79 | return TYPE_ERROR; | |
80 | } else { | |
81 | load = ((instr & 0xf0) == 0xd0); | |
82 | rd2 = rd + 1; | |
83 | } | |
84 | ||
85 | if (load) { | |
86 | unsigned int val, val2; | |
87 | ||
88 | if (get_user(val, (u32 __user *)addr) || | |
89 | get_user(val2, (u32 __user *)(addr + 4))) | |
90 | return TYPE_FAULT; | |
91 | regs->regs[rd] = val; | |
92 | regs->regs[rd2] = val2; | |
93 | } else { | |
94 | if (put_user(regs->regs[rd], (u32 __user *)addr) || | |
95 | put_user(regs->regs[rd2], (u32 __user *)(addr + 4))) | |
96 | return TYPE_FAULT; | |
97 | } | |
98 | return TYPE_LDST; | |
99 | } | |
100 | ||
101 | /* | |
102 | * LDM/STM alignment handler. | |
103 | * | |
104 | * There are 4 variants of this instruction: | |
105 | * | |
106 | * B = rn pointer before instruction, A = rn pointer after instruction | |
107 | * ------ increasing address -----> | |
108 | * | | r0 | r1 | ... | rx | | | |
109 | * PU = 01 B A | |
110 | * PU = 11 B A | |
111 | * PU = 00 A B | |
112 | * PU = 10 A B | |
113 | */ | |
114 | static int | |
115 | do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs) | |
116 | { | |
117 | unsigned int rd, rn, nr_regs, regbits; | |
118 | unsigned long eaddr, newaddr; | |
119 | unsigned int val; | |
120 | ||
121 | /* count the number of registers in the mask to be transferred */ | |
122 | nr_regs = hweight16(REGMASK_BITS(instr)) * 4; | |
123 | ||
124 | rn = RN_BITS(instr); | |
125 | newaddr = eaddr = regs->regs[rn]; | |
126 | ||
127 | if (!LDST_U_BIT(instr)) | |
128 | nr_regs = -nr_regs; | |
129 | newaddr += nr_regs; | |
130 | if (!LDST_U_BIT(instr)) | |
131 | eaddr = newaddr; | |
132 | ||
133 | if (LDST_P_EQ_U(instr)) /* U = P */ | |
134 | eaddr += 4; | |
135 | ||
136 | for (regbits = REGMASK_BITS(instr), rd = 0; regbits; | |
137 | regbits >>= 1, rd += 1) | |
138 | if (regbits & 1) { | |
139 | if (LDST_L_BIT(instr)) { | |
140 | if (get_user(val, (u32 __user *)eaddr)) | |
141 | return TYPE_FAULT; | |
142 | if (rd < 15) | |
143 | regs->regs[rd] = val; | |
144 | else | |
145 | regs->pc = val; | |
146 | } else { | |
147 | /* | |
148 | * The PC register has a bias of +8 in ARM mode | |
149 | * and +4 in Thumb mode. This means that a read | |
150 | * of the value of PC should account for this. | |
151 | * Since Thumb does not permit STM instructions | |
152 | * to refer to PC, just add 8 here. | |
153 | */ | |
154 | val = (rd < 15) ? regs->regs[rd] : regs->pc + 8; | |
155 | if (put_user(val, (u32 __user *)eaddr)) | |
156 | return TYPE_FAULT; | |
157 | } | |
158 | eaddr += 4; | |
159 | } | |
160 | ||
161 | if (LDST_W_BIT(instr)) | |
162 | regs->regs[rn] = newaddr; | |
163 | ||
164 | return TYPE_DONE; | |
165 | } | |
166 | ||
167 | /* | |
168 | * Convert Thumb multi-word load/store instruction forms to equivalent ARM | |
169 | * instructions so we can reuse ARM userland alignment fault fixups for Thumb. | |
170 | * | |
171 | * This implementation was initially based on the algorithm found in | |
172 | * gdb/sim/arm/thumbemu.c. It is basically just a code reduction of same | |
173 | * to convert only Thumb ld/st instruction forms to equivalent ARM forms. | |
174 | * | |
175 | * NOTES: | |
176 | * 1. Comments below refer to ARM ARM DDI0100E Thumb Instruction sections. | |
177 | * 2. If for some reason we're passed an non-ld/st Thumb instruction to | |
178 | * decode, we return 0xdeadc0de. This should never happen under normal | |
179 | * circumstances but if it does, we've got other problems to deal with | |
180 | * elsewhere and we obviously can't fix those problems here. | |
181 | */ | |
182 | ||
183 | static unsigned long thumb2arm(u16 tinstr) | |
184 | { | |
185 | u32 L = (tinstr & (1<<11)) >> 11; | |
186 | ||
187 | switch ((tinstr & 0xf800) >> 11) { | |
188 | /* 6.6.1 Format 1: */ | |
189 | case 0xc000 >> 11: /* 7.1.51 STMIA */ | |
190 | case 0xc800 >> 11: /* 7.1.25 LDMIA */ | |
191 | { | |
192 | u32 Rn = (tinstr & (7<<8)) >> 8; | |
193 | u32 W = ((L<<Rn) & (tinstr&255)) ? 0 : 1<<21; | |
194 | ||
195 | return 0xe8800000 | W | (L<<20) | (Rn<<16) | | |
196 | (tinstr&255); | |
197 | } | |
198 | ||
199 | /* 6.6.1 Format 2: */ | |
200 | case 0xb000 >> 11: /* 7.1.48 PUSH */ | |
201 | case 0xb800 >> 11: /* 7.1.47 POP */ | |
202 | if ((tinstr & (3 << 9)) == 0x0400) { | |
203 | static const u32 subset[4] = { | |
204 | 0xe92d0000, /* STMDB sp!,{registers} */ | |
205 | 0xe92d4000, /* STMDB sp!,{registers,lr} */ | |
206 | 0xe8bd0000, /* LDMIA sp!,{registers} */ | |
207 | 0xe8bd8000 /* LDMIA sp!,{registers,pc} */ | |
208 | }; | |
209 | return subset[(L<<1) | ((tinstr & (1<<8)) >> 8)] | | |
210 | (tinstr & 255); /* register_list */ | |
211 | } | |
212 | fallthrough; /* for illegal instruction case */ | |
213 | ||
214 | default: | |
215 | return BAD_INSTR; | |
216 | } | |
217 | } | |
218 | ||
219 | /* | |
220 | * Convert Thumb-2 32 bit LDM, STM, LDRD, STRD to equivalent instruction | |
221 | * handlable by ARM alignment handler, also find the corresponding handler, | |
222 | * so that we can reuse ARM userland alignment fault fixups for Thumb. | |
223 | * | |
224 | * @pinstr: original Thumb-2 instruction; returns new handlable instruction | |
225 | * @regs: register context. | |
226 | * @poffset: return offset from faulted addr for later writeback | |
227 | * | |
228 | * NOTES: | |
229 | * 1. Comments below refer to ARMv7 DDI0406A Thumb Instruction sections. | |
230 | * 2. Register name Rt from ARMv7 is same as Rd from ARMv6 (Rd is Rt) | |
231 | */ | |
232 | static void * | |
233 | do_alignment_t32_to_handler(u32 *pinstr, struct pt_regs *regs, | |
234 | union offset_union *poffset) | |
235 | { | |
236 | u32 instr = *pinstr; | |
237 | u16 tinst1 = (instr >> 16) & 0xffff; | |
238 | u16 tinst2 = instr & 0xffff; | |
239 | ||
240 | switch (tinst1 & 0xffe0) { | |
241 | /* A6.3.5 Load/Store multiple */ | |
242 | case 0xe880: /* STM/STMIA/STMEA,LDM/LDMIA, PUSH/POP T2 */ | |
243 | case 0xe8a0: /* ...above writeback version */ | |
244 | case 0xe900: /* STMDB/STMFD, LDMDB/LDMEA */ | |
245 | case 0xe920: /* ...above writeback version */ | |
246 | /* no need offset decision since handler calculates it */ | |
247 | return do_alignment_ldmstm; | |
248 | ||
249 | case 0xf840: /* POP/PUSH T3 (single register) */ | |
250 | if (RN_BITS(instr) == 13 && (tinst2 & 0x09ff) == 0x0904) { | |
251 | u32 L = !!(LDST_L_BIT(instr)); | |
252 | const u32 subset[2] = { | |
253 | 0xe92d0000, /* STMDB sp!,{registers} */ | |
254 | 0xe8bd0000, /* LDMIA sp!,{registers} */ | |
255 | }; | |
256 | *pinstr = subset[L] | (1<<RD_BITS(instr)); | |
257 | return do_alignment_ldmstm; | |
258 | } | |
259 | /* Else fall through for illegal instruction case */ | |
260 | break; | |
261 | ||
262 | /* A6.3.6 Load/store double, STRD/LDRD(immed, lit, reg) */ | |
263 | case 0xe860: | |
264 | case 0xe960: | |
265 | case 0xe8e0: | |
266 | case 0xe9e0: | |
267 | poffset->un = (tinst2 & 0xff) << 2; | |
268 | fallthrough; | |
269 | ||
270 | case 0xe940: | |
271 | case 0xe9c0: | |
272 | return do_alignment_ldrdstrd; | |
273 | ||
274 | /* | |
275 | * No need to handle load/store instructions up to word size | |
276 | * since ARMv6 and later CPUs can perform unaligned accesses. | |
277 | */ | |
278 | default: | |
279 | break; | |
280 | } | |
281 | return NULL; | |
282 | } | |
283 | ||
284 | static int alignment_get_arm(struct pt_regs *regs, __le32 __user *ip, u32 *inst) | |
285 | { | |
286 | __le32 instr = 0; | |
287 | int fault; | |
288 | ||
289 | fault = get_user(instr, ip); | |
290 | if (fault) | |
291 | return fault; | |
292 | ||
293 | *inst = __le32_to_cpu(instr); | |
294 | return 0; | |
295 | } | |
296 | ||
297 | static int alignment_get_thumb(struct pt_regs *regs, __le16 __user *ip, u16 *inst) | |
298 | { | |
299 | __le16 instr = 0; | |
300 | int fault; | |
301 | ||
302 | fault = get_user(instr, ip); | |
303 | if (fault) | |
304 | return fault; | |
305 | ||
306 | *inst = __le16_to_cpu(instr); | |
307 | return 0; | |
308 | } | |
309 | ||
310 | int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs) | |
311 | { | |
312 | union offset_union offset; | |
313 | unsigned long instrptr; | |
314 | int (*handler)(unsigned long addr, u32 instr, struct pt_regs *regs); | |
315 | unsigned int type; | |
316 | u32 instr = 0; | |
3fc24ef3 AB |
317 | int isize = 4; |
318 | int thumb2_32b = 0; | |
3fc24ef3 AB |
319 | |
320 | instrptr = instruction_pointer(regs); | |
321 | ||
322 | if (compat_thumb_mode(regs)) { | |
323 | __le16 __user *ptr = (__le16 __user *)(instrptr & ~1); | |
32d85999 | 324 | u16 tinstr, tinst2; |
3fc24ef3 | 325 | |
32d85999 AB |
326 | if (alignment_get_thumb(regs, ptr, &tinstr)) |
327 | return 1; | |
328 | ||
329 | if (IS_T32(tinstr)) { /* Thumb-2 32-bit */ | |
330 | if (alignment_get_thumb(regs, ptr + 1, &tinst2)) | |
331 | return 1; | |
332 | instr = ((u32)tinstr << 16) | tinst2; | |
333 | thumb2_32b = 1; | |
334 | } else { | |
335 | isize = 2; | |
336 | instr = thumb2arm(tinstr); | |
3fc24ef3 AB |
337 | } |
338 | } else { | |
32d85999 AB |
339 | if (alignment_get_arm(regs, (__le32 __user *)instrptr, &instr)) |
340 | return 1; | |
3fc24ef3 AB |
341 | } |
342 | ||
3fc24ef3 AB |
343 | switch (CODING_BITS(instr)) { |
344 | case 0x00000000: /* 3.13.4 load/store instruction extensions */ | |
345 | if (LDSTHD_I_BIT(instr)) | |
346 | offset.un = (instr & 0xf00) >> 4 | (instr & 15); | |
347 | else | |
348 | offset.un = regs->regs[RM_BITS(instr)]; | |
349 | ||
350 | if ((instr & 0x001000f0) == 0x000000d0 || /* LDRD */ | |
351 | (instr & 0x001000f0) == 0x000000f0) /* STRD */ | |
352 | handler = do_alignment_ldrdstrd; | |
353 | else | |
354 | return 1; | |
355 | break; | |
356 | ||
357 | case 0x08000000: /* ldm or stm, or thumb-2 32bit instruction */ | |
358 | if (thumb2_32b) { | |
359 | offset.un = 0; | |
360 | handler = do_alignment_t32_to_handler(&instr, regs, &offset); | |
361 | } else { | |
362 | offset.un = 0; | |
363 | handler = do_alignment_ldmstm; | |
364 | } | |
365 | break; | |
366 | ||
367 | default: | |
368 | return 1; | |
369 | } | |
370 | ||
371 | type = handler(addr, instr, regs); | |
372 | ||
373 | if (type == TYPE_ERROR || type == TYPE_FAULT) | |
374 | return 1; | |
375 | ||
376 | if (type == TYPE_LDST) | |
377 | do_alignment_finish_ldst(addr, instr, regs, offset); | |
378 | ||
379 | perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->pc); | |
380 | arm64_skip_faulting_instruction(regs, isize); | |
381 | ||
382 | return 0; | |
383 | } |