projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma
[linux-2.6-block.git] / arch / mips / kernel / unaligned.c
CommitLineData
1da177e4
LT		 1/*
2 * Handle unaligned accesses by emulation.
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
9d8e5736 10 * Copyright (C) 2014 Imagination Technologies Ltd.
1da177e4
LT		 11 *
12 * This file contains exception handler for address error exception with the
13 * special capability to execute faulting instructions in software. The
14 * handler does not try to handle the case when the program counter points
15 * to an address not aligned to a word boundary.
16 *
17 * Putting data to unaligned addresses is a bad practice even on Intel where
18 * only the performance is affected. Much worse is that such code is non-
19 * portable. Due to several programs that die on MIPS due to alignment
20 * problems I decided to implement this handler anyway though I originally
21 * didn't intend to do this at all for user code.
22 *
23 * For now I enable fixing of address errors by default to make life easier.
24 * I however intend to disable this somewhen in the future when the alignment
70342287 25 * problems with user programs have been fixed. For programmers this is the
1da177e4
LT		 26 * right way to go.
27 *
28 * Fixing address errors is a per process option. The option is inherited
70342287 29 * across fork(2) and execve(2) calls. If you really want to use the
1da177e4
LT		 30 * option in your user programs - I discourage the use of the software
31 * emulation strongly - use the following code in your userland stuff:
32 *
33 * #include <sys/sysmips.h>
34 *
35 * ...
36 * sysmips(MIPS_FIXADE, x);
37 * ...
38 *
39 * The argument x is 0 for disabling software emulation, enabled otherwise.
40 *
41 * Below a little program to play around with this feature.
42 *
43 * #include <stdio.h>
44 * #include <sys/sysmips.h>
45 *
46 * struct foo {
70342287 47 * unsigned char bar[8];
1da177e4
LT		 48 * };
49 *
50 * main(int argc, char *argv[])
51 * {
70342287
RB		 52 * struct foo x = {0, 1, 2, 3, 4, 5, 6, 7};
53 * unsigned int *p = (unsigned int *) (x.bar + 3);
54 * int i;
1da177e4 55 *
70342287
RB		 56 * if (argc > 1)
57 * sysmips(MIPS_FIXADE, atoi(argv[1]));
1da177e4 58 *
70342287 59 * printf("*p = %08lx\n", *p);
1da177e4 60 *
70342287 61 * *p = 0xdeadface;
1da177e4 62 *
70342287
RB		 63 * for(i = 0; i <= 7; i++)
64 * printf("%02x ", x.bar[i]);
65 * printf("\n");
1da177e4
LT		 66 * }
67 *
68 * Coprocessor loads are not supported; I think this case is unimportant
69 * in the practice.
70 *
71 * TODO: Handle ndc (attempted store to doubleword in uncached memory)
70342287
RB		 72 * exception for the R6000.
73 * A store crossing a page boundary might be executed only partially.
74 * Undo the partial store in this case.
1da177e4 75 */
c3fc5cd5 76#include <linux/context_tracking.h>
1da177e4 77#include <linux/mm.h>
1da177e4
LT		 78#include <linux/signal.h>
79#include <linux/smp.h>
e8edc6e0 80#include <linux/sched.h>
6312e0ee 81#include <linux/debugfs.h>
7f788d2d
DCZ		 82#include <linux/perf_event.h>
83
1da177e4
LT		 84#include <asm/asm.h>
85#include <asm/branch.h>
86#include <asm/byteorder.h>
69f3a7de 87#include <asm/cop2.h>
102cedc3
LY		 88#include <asm/fpu.h>
89#include <asm/fpu_emulator.h>
1da177e4
LT		 90#include <asm/inst.h>
91#include <asm/uaccess.h>
1da177e4 92
70342287 93#define STR(x) __STR(x)
1da177e4
LT		 94#define __STR(x) #x
95
6312e0ee
AN		 96enum {
97 UNALIGNED_ACTION_QUIET,
98 UNALIGNED_ACTION_SIGNAL,
99 UNALIGNED_ACTION_SHOW,
100};
101#ifdef CONFIG_DEBUG_FS
102static u32 unaligned_instructions;
103static u32 unaligned_action;
104#else
105#define unaligned_action UNALIGNED_ACTION_QUIET
1da177e4 106#endif
6312e0ee 107extern void show_registers(struct pt_regs *regs);
1da177e4 108
34c2f668 109#ifdef __BIG_ENDIAN
eeb53895 110#define _LoadHW(addr, value, res, type) \
3563c32d 111do { \
34c2f668 112 __asm__ __volatile__ (".set\tnoat\n" \
eeb53895
MC		 113 "1:\t"type##_lb("%0", "0(%2)")"\n" \
114 "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
34c2f668
LY		 115 "sll\t%0, 0x8\n\t" \
116 "or\t%0, $1\n\t" \
117 "li\t%1, 0\n" \
118 "3:\t.set\tat\n\t" \
119 ".insn\n\t" \
120 ".section\t.fixup,\"ax\"\n\t" \
121 "4:\tli\t%1, %3\n\t" \
122 "j\t3b\n\t" \
123 ".previous\n\t" \
124 ".section\t__ex_table,\"a\"\n\t" \
125 STR(PTR)"\t1b, 4b\n\t" \
126 STR(PTR)"\t2b, 4b\n\t" \
127 ".previous" \
128 : "=&r" (value), "=r" (res) \
3563c32d
MC		 129 : "r" (addr), "i" (-EFAULT)); \
130} while(0)
34c2f668 131
0593a44c 132#ifndef CONFIG_CPU_MIPSR6
eeb53895 133#define _LoadW(addr, value, res, type) \
3563c32d 134do { \
34c2f668 135 __asm__ __volatile__ ( \
eeb53895
MC		 136 "1:\t"type##_lwl("%0", "(%2)")"\n" \
137 "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
34c2f668
LY		 138 "li\t%1, 0\n" \
139 "3:\n\t" \
140 ".insn\n\t" \
141 ".section\t.fixup,\"ax\"\n\t" \
142 "4:\tli\t%1, %3\n\t" \
143 "j\t3b\n\t" \
144 ".previous\n\t" \
145 ".section\t__ex_table,\"a\"\n\t" \
146 STR(PTR)"\t1b, 4b\n\t" \
147 STR(PTR)"\t2b, 4b\n\t" \
148 ".previous" \
149 : "=&r" (value), "=r" (res) \
3563c32d
MC		 150 : "r" (addr), "i" (-EFAULT)); \
151} while(0)
152
0593a44c
LY		 153#else
154/* MIPSR6 has no lwl instruction */
eeb53895 155#define _LoadW(addr, value, res, type) \
3563c32d 156do { \
0593a44c
LY		 157 __asm__ __volatile__ ( \
158 ".set\tpush\n" \
159 ".set\tnoat\n\t" \
eeb53895
MC		 160 "1:"type##_lb("%0", "0(%2)")"\n\t" \
161 "2:"type##_lbu("$1", "1(%2)")"\n\t" \
0593a44c
LY		 162 "sll\t%0, 0x8\n\t" \
163 "or\t%0, $1\n\t" \
eeb53895 164 "3:"type##_lbu("$1", "2(%2)")"\n\t" \
0593a44c
LY		 165 "sll\t%0, 0x8\n\t" \
166 "or\t%0, $1\n\t" \
eeb53895 167 "4:"type##_lbu("$1", "3(%2)")"\n\t" \
0593a44c
LY		 168 "sll\t%0, 0x8\n\t" \
169 "or\t%0, $1\n\t" \
170 "li\t%1, 0\n" \
171 ".set\tpop\n" \
172 "10:\n\t" \
173 ".insn\n\t" \
174 ".section\t.fixup,\"ax\"\n\t" \
175 "11:\tli\t%1, %3\n\t" \
176 "j\t10b\n\t" \
177 ".previous\n\t" \
178 ".section\t__ex_table,\"a\"\n\t" \
179 STR(PTR)"\t1b, 11b\n\t" \
180 STR(PTR)"\t2b, 11b\n\t" \
181 STR(PTR)"\t3b, 11b\n\t" \
182 STR(PTR)"\t4b, 11b\n\t" \
183 ".previous" \
184 : "=&r" (value), "=r" (res) \
3563c32d
MC		 185 : "r" (addr), "i" (-EFAULT)); \
186} while(0)
187
0593a44c 188#endif /* CONFIG_CPU_MIPSR6 */
34c2f668 189
eeb53895 190#define _LoadHWU(addr, value, res, type) \
3563c32d 191do { \
34c2f668
LY		 192 __asm__ __volatile__ ( \
193 ".set\tnoat\n" \
eeb53895
MC		 194 "1:\t"type##_lbu("%0", "0(%2)")"\n" \
195 "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\
34c2f668
LY		 196 "sll\t%0, 0x8\n\t" \
197 "or\t%0, $1\n\t" \
198 "li\t%1, 0\n" \
199 "3:\n\t" \
200 ".insn\n\t" \
201 ".set\tat\n\t" \
202 ".section\t.fixup,\"ax\"\n\t" \
203 "4:\tli\t%1, %3\n\t" \
204 "j\t3b\n\t" \
205 ".previous\n\t" \
206 ".section\t__ex_table,\"a\"\n\t" \
207 STR(PTR)"\t1b, 4b\n\t" \
208 STR(PTR)"\t2b, 4b\n\t" \
209 ".previous" \
210 : "=&r" (value), "=r" (res) \
3563c32d
MC		 211 : "r" (addr), "i" (-EFAULT)); \
212} while(0)
34c2f668 213
0593a44c 214#ifndef CONFIG_CPU_MIPSR6
eeb53895 215#define _LoadWU(addr, value, res, type) \
3563c32d 216do { \
34c2f668 217 __asm__ __volatile__ ( \
eeb53895
MC		 218 "1:\t"type##_lwl("%0", "(%2)")"\n" \
219 "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\
34c2f668
LY		 220 "dsll\t%0, %0, 32\n\t" \
221 "dsrl\t%0, %0, 32\n\t" \
222 "li\t%1, 0\n" \
223 "3:\n\t" \
224 ".insn\n\t" \
225 "\t.section\t.fixup,\"ax\"\n\t" \
226 "4:\tli\t%1, %3\n\t" \
227 "j\t3b\n\t" \
228 ".previous\n\t" \
229 ".section\t__ex_table,\"a\"\n\t" \
230 STR(PTR)"\t1b, 4b\n\t" \
231 STR(PTR)"\t2b, 4b\n\t" \
232 ".previous" \
233 : "=&r" (value), "=r" (res) \
3563c32d
MC		 234 : "r" (addr), "i" (-EFAULT)); \
235} while(0)
34c2f668 236
eeb53895 237#define _LoadDW(addr, value, res) \
3563c32d 238do { \
34c2f668
LY		 239 __asm__ __volatile__ ( \
240 "1:\tldl\t%0, (%2)\n" \
241 "2:\tldr\t%0, 7(%2)\n\t" \
242 "li\t%1, 0\n" \
243 "3:\n\t" \
244 ".insn\n\t" \
245 "\t.section\t.fixup,\"ax\"\n\t" \
246 "4:\tli\t%1, %3\n\t" \
247 "j\t3b\n\t" \
248 ".previous\n\t" \
249 ".section\t__ex_table,\"a\"\n\t" \
250 STR(PTR)"\t1b, 4b\n\t" \
251 STR(PTR)"\t2b, 4b\n\t" \
252 ".previous" \
253 : "=&r" (value), "=r" (res) \
3563c32d
MC		 254 : "r" (addr), "i" (-EFAULT)); \
255} while(0)
256
0593a44c
LY		 257#else
258/* MIPSR6 has not lwl and ldl instructions */
eeb53895 259#define _LoadWU(addr, value, res, type) \
3563c32d 260do { \
0593a44c
LY		 261 __asm__ __volatile__ ( \
262 ".set\tpush\n\t" \
263 ".set\tnoat\n\t" \
eeb53895
MC		 264 "1:"type##_lbu("%0", "0(%2)")"\n\t" \
265 "2:"type##_lbu("$1", "1(%2)")"\n\t" \
0593a44c
LY		 266 "sll\t%0, 0x8\n\t" \
267 "or\t%0, $1\n\t" \
eeb53895 268 "3:"type##_lbu("$1", "2(%2)")"\n\t" \
0593a44c
LY		 269 "sll\t%0, 0x8\n\t" \
270 "or\t%0, $1\n\t" \
eeb53895 271 "4:"type##_lbu("$1", "3(%2)")"\n\t" \
0593a44c
LY		 272 "sll\t%0, 0x8\n\t" \
273 "or\t%0, $1\n\t" \
274 "li\t%1, 0\n" \
275 ".set\tpop\n" \
276 "10:\n\t" \
277 ".insn\n\t" \
278 ".section\t.fixup,\"ax\"\n\t" \
279 "11:\tli\t%1, %3\n\t" \
280 "j\t10b\n\t" \
281 ".previous\n\t" \
282 ".section\t__ex_table,\"a\"\n\t" \
283 STR(PTR)"\t1b, 11b\n\t" \
284 STR(PTR)"\t2b, 11b\n\t" \
285 STR(PTR)"\t3b, 11b\n\t" \
286 STR(PTR)"\t4b, 11b\n\t" \
287 ".previous" \
288 : "=&r" (value), "=r" (res) \
3563c32d
MC		 289 : "r" (addr), "i" (-EFAULT)); \
290} while(0)
0593a44c 291
eeb53895 292#define _LoadDW(addr, value, res) \
3563c32d 293do { \
0593a44c
LY		 294 __asm__ __volatile__ ( \
295 ".set\tpush\n\t" \
296 ".set\tnoat\n\t" \
297 "1:lb\t%0, 0(%2)\n\t" \
298 "2:lbu\t $1, 1(%2)\n\t" \
299 "dsll\t%0, 0x8\n\t" \
300 "or\t%0, $1\n\t" \
301 "3:lbu\t$1, 2(%2)\n\t" \
302 "dsll\t%0, 0x8\n\t" \
303 "or\t%0, $1\n\t" \
304 "4:lbu\t$1, 3(%2)\n\t" \
305 "dsll\t%0, 0x8\n\t" \
306 "or\t%0, $1\n\t" \
307 "5:lbu\t$1, 4(%2)\n\t" \
308 "dsll\t%0, 0x8\n\t" \
309 "or\t%0, $1\n\t" \
310 "6:lbu\t$1, 5(%2)\n\t" \
311 "dsll\t%0, 0x8\n\t" \
312 "or\t%0, $1\n\t" \
313 "7:lbu\t$1, 6(%2)\n\t" \
314 "dsll\t%0, 0x8\n\t" \
315 "or\t%0, $1\n\t" \
316 "8:lbu\t$1, 7(%2)\n\t" \
317 "dsll\t%0, 0x8\n\t" \
318 "or\t%0, $1\n\t" \
319 "li\t%1, 0\n" \
320 ".set\tpop\n\t" \
321 "10:\n\t" \
322 ".insn\n\t" \
323 ".section\t.fixup,\"ax\"\n\t" \
324 "11:\tli\t%1, %3\n\t" \
325 "j\t10b\n\t" \
326 ".previous\n\t" \
327 ".section\t__ex_table,\"a\"\n\t" \
328 STR(PTR)"\t1b, 11b\n\t" \
329 STR(PTR)"\t2b, 11b\n\t" \
330 STR(PTR)"\t3b, 11b\n\t" \
331 STR(PTR)"\t4b, 11b\n\t" \
332 STR(PTR)"\t5b, 11b\n\t" \
333 STR(PTR)"\t6b, 11b\n\t" \
334 STR(PTR)"\t7b, 11b\n\t" \
335 STR(PTR)"\t8b, 11b\n\t" \
336 ".previous" \
337 : "=&r" (value), "=r" (res) \
3563c32d
MC		 338 : "r" (addr), "i" (-EFAULT)); \
339} while(0)
340
0593a44c
LY		 341#endif /* CONFIG_CPU_MIPSR6 */
342
34c2f668 343
eeb53895 344#define _StoreHW(addr, value, res, type) \
3563c32d 345do { \
34c2f668
LY		 346 __asm__ __volatile__ ( \
347 ".set\tnoat\n" \
eeb53895 348 "1:\t"type##_sb("%1", "1(%2)")"\n" \
34c2f668 349 "srl\t$1, %1, 0x8\n" \
eeb53895 350 "2:\t"type##_sb("$1", "0(%2)")"\n" \
34c2f668
LY		 351 ".set\tat\n\t" \
352 "li\t%0, 0\n" \
353 "3:\n\t" \
354 ".insn\n\t" \
355 ".section\t.fixup,\"ax\"\n\t" \
356 "4:\tli\t%0, %3\n\t" \
357 "j\t3b\n\t" \
358 ".previous\n\t" \
359 ".section\t__ex_table,\"a\"\n\t" \
360 STR(PTR)"\t1b, 4b\n\t" \
361 STR(PTR)"\t2b, 4b\n\t" \
362 ".previous" \
363 : "=r" (res) \
3563c32d
MC		 364 : "r" (value), "r" (addr), "i" (-EFAULT));\
365} while(0)
34c2f668 366
0593a44c 367#ifndef CONFIG_CPU_MIPSR6
eeb53895 368#define _StoreW(addr, value, res, type) \
3563c32d 369do { \
34c2f668 370 __asm__ __volatile__ ( \
eeb53895
MC		 371 "1:\t"type##_swl("%1", "(%2)")"\n" \
372 "2:\t"type##_swr("%1", "3(%2)")"\n\t"\
34c2f668
LY		 373 "li\t%0, 0\n" \
374 "3:\n\t" \
375 ".insn\n\t" \
376 ".section\t.fixup,\"ax\"\n\t" \
377 "4:\tli\t%0, %3\n\t" \
378 "j\t3b\n\t" \
379 ".previous\n\t" \
380 ".section\t__ex_table,\"a\"\n\t" \
381 STR(PTR)"\t1b, 4b\n\t" \
382 STR(PTR)"\t2b, 4b\n\t" \
383 ".previous" \
384 : "=r" (res) \
3563c32d
MC		 385 : "r" (value), "r" (addr), "i" (-EFAULT)); \
386} while(0)
34c2f668 387
eeb53895 388#define _StoreDW(addr, value, res) \
3563c32d 389do { \
34c2f668
LY		 390 __asm__ __volatile__ ( \
391 "1:\tsdl\t%1,(%2)\n" \
392 "2:\tsdr\t%1, 7(%2)\n\t" \
393 "li\t%0, 0\n" \
394 "3:\n\t" \
395 ".insn\n\t" \
396 ".section\t.fixup,\"ax\"\n\t" \
397 "4:\tli\t%0, %3\n\t" \
398 "j\t3b\n\t" \
399 ".previous\n\t" \
400 ".section\t__ex_table,\"a\"\n\t" \
401 STR(PTR)"\t1b, 4b\n\t" \
402 STR(PTR)"\t2b, 4b\n\t" \
403 ".previous" \
404 : "=r" (res) \
3563c32d
MC		 405 : "r" (value), "r" (addr), "i" (-EFAULT)); \
406} while(0)
407
0593a44c
LY		 408#else
409/* MIPSR6 has no swl and sdl instructions */
eeb53895 410#define _StoreW(addr, value, res, type) \
3563c32d 411do { \
0593a44c
LY		 412 __asm__ __volatile__ ( \
413 ".set\tpush\n\t" \
414 ".set\tnoat\n\t" \
eeb53895 415 "1:"type##_sb("%1", "3(%2)")"\n\t" \
0593a44c 416 "srl\t$1, %1, 0x8\n\t" \
eeb53895 417 "2:"type##_sb("$1", "2(%2)")"\n\t" \
0593a44c 418 "srl\t$1, $1, 0x8\n\t" \
eeb53895 419 "3:"type##_sb("$1", "1(%2)")"\n\t" \
0593a44c 420 "srl\t$1, $1, 0x8\n\t" \
eeb53895 421 "4:"type##_sb("$1", "0(%2)")"\n\t" \
0593a44c
LY		 422 ".set\tpop\n\t" \
423 "li\t%0, 0\n" \
424 "10:\n\t" \
425 ".insn\n\t" \
426 ".section\t.fixup,\"ax\"\n\t" \
427 "11:\tli\t%0, %3\n\t" \
428 "j\t10b\n\t" \
429 ".previous\n\t" \
430 ".section\t__ex_table,\"a\"\n\t" \
431 STR(PTR)"\t1b, 11b\n\t" \
432 STR(PTR)"\t2b, 11b\n\t" \
433 STR(PTR)"\t3b, 11b\n\t" \
434 STR(PTR)"\t4b, 11b\n\t" \
435 ".previous" \
436 : "=&r" (res) \
437 : "r" (value), "r" (addr), "i" (-EFAULT) \
3563c32d
MC		 438 : "memory"); \
439} while(0)
0593a44c 440
531a6d59 441#define _StoreDW(addr, value, res) \
3563c32d 442do { \
0593a44c
LY		 443 __asm__ __volatile__ ( \
444 ".set\tpush\n\t" \
445 ".set\tnoat\n\t" \
446 "1:sb\t%1, 7(%2)\n\t" \
447 "dsrl\t$1, %1, 0x8\n\t" \
448 "2:sb\t$1, 6(%2)\n\t" \
449 "dsrl\t$1, $1, 0x8\n\t" \
450 "3:sb\t$1, 5(%2)\n\t" \
451 "dsrl\t$1, $1, 0x8\n\t" \
452 "4:sb\t$1, 4(%2)\n\t" \
453 "dsrl\t$1, $1, 0x8\n\t" \
454 "5:sb\t$1, 3(%2)\n\t" \
455 "dsrl\t$1, $1, 0x8\n\t" \
456 "6:sb\t$1, 2(%2)\n\t" \
457 "dsrl\t$1, $1, 0x8\n\t" \
458 "7:sb\t$1, 1(%2)\n\t" \
459 "dsrl\t$1, $1, 0x8\n\t" \
460 "8:sb\t$1, 0(%2)\n\t" \
461 "dsrl\t$1, $1, 0x8\n\t" \
462 ".set\tpop\n\t" \
463 "li\t%0, 0\n" \
464 "10:\n\t" \
465 ".insn\n\t" \
466 ".section\t.fixup,\"ax\"\n\t" \
467 "11:\tli\t%0, %3\n\t" \
468 "j\t10b\n\t" \
469 ".previous\n\t" \
470 ".section\t__ex_table,\"a\"\n\t" \
471 STR(PTR)"\t1b, 11b\n\t" \
472 STR(PTR)"\t2b, 11b\n\t" \
473 STR(PTR)"\t3b, 11b\n\t" \
474 STR(PTR)"\t4b, 11b\n\t" \
475 STR(PTR)"\t5b, 11b\n\t" \
476 STR(PTR)"\t6b, 11b\n\t" \
477 STR(PTR)"\t7b, 11b\n\t" \
478 STR(PTR)"\t8b, 11b\n\t" \
479 ".previous" \
480 : "=&r" (res) \
481 : "r" (value), "r" (addr), "i" (-EFAULT) \
3563c32d
MC		 482 : "memory"); \
483} while(0)
484
0593a44c
LY		 485#endif /* CONFIG_CPU_MIPSR6 */
486
487#else /* __BIG_ENDIAN */
34c2f668 488
eeb53895 489#define _LoadHW(addr, value, res, type) \
3563c32d 490do { \
34c2f668 491 __asm__ __volatile__ (".set\tnoat\n" \
eeb53895
MC		 492 "1:\t"type##_lb("%0", "1(%2)")"\n" \
493 "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
34c2f668
LY		 494 "sll\t%0, 0x8\n\t" \
495 "or\t%0, $1\n\t" \
496 "li\t%1, 0\n" \
497 "3:\t.set\tat\n\t" \
498 ".insn\n\t" \
499 ".section\t.fixup,\"ax\"\n\t" \
500 "4:\tli\t%1, %3\n\t" \
501 "j\t3b\n\t" \
502 ".previous\n\t" \
503 ".section\t__ex_table,\"a\"\n\t" \
504 STR(PTR)"\t1b, 4b\n\t" \
505 STR(PTR)"\t2b, 4b\n\t" \
506 ".previous" \
507 : "=&r" (value), "=r" (res) \
3563c32d
MC		 508 : "r" (addr), "i" (-EFAULT)); \
509} while(0)
34c2f668 510
0593a44c 511#ifndef CONFIG_CPU_MIPSR6
eeb53895 512#define _LoadW(addr, value, res, type) \
3563c32d 513do { \
34c2f668 514 __asm__ __volatile__ ( \
eeb53895
MC		 515 "1:\t"type##_lwl("%0", "3(%2)")"\n" \
516 "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
34c2f668
LY		 517 "li\t%1, 0\n" \
518 "3:\n\t" \
519 ".insn\n\t" \
520 ".section\t.fixup,\"ax\"\n\t" \
521 "4:\tli\t%1, %3\n\t" \
522 "j\t3b\n\t" \
523 ".previous\n\t" \
524 ".section\t__ex_table,\"a\"\n\t" \
525 STR(PTR)"\t1b, 4b\n\t" \
526 STR(PTR)"\t2b, 4b\n\t" \
527 ".previous" \
528 : "=&r" (value), "=r" (res) \
3563c32d
MC		 529 : "r" (addr), "i" (-EFAULT)); \
530} while(0)
531
0593a44c
LY		 532#else
533/* MIPSR6 has no lwl instruction */
eeb53895 534#define _LoadW(addr, value, res, type) \
3563c32d 535do { \
0593a44c
LY		 536 __asm__ __volatile__ ( \
537 ".set\tpush\n" \
538 ".set\tnoat\n\t" \
eeb53895
MC		 539 "1:"type##_lb("%0", "3(%2)")"\n\t" \
540 "2:"type##_lbu("$1", "2(%2)")"\n\t" \
0593a44c
LY		 541 "sll\t%0, 0x8\n\t" \
542 "or\t%0, $1\n\t" \
eeb53895 543 "3:"type##_lbu("$1", "1(%2)")"\n\t" \
0593a44c
LY		 544 "sll\t%0, 0x8\n\t" \
545 "or\t%0, $1\n\t" \
eeb53895 546 "4:"type##_lbu("$1", "0(%2)")"\n\t" \
0593a44c
LY		 547 "sll\t%0, 0x8\n\t" \
548 "or\t%0, $1\n\t" \
549 "li\t%1, 0\n" \
550 ".set\tpop\n" \
551 "10:\n\t" \
552 ".insn\n\t" \
553 ".section\t.fixup,\"ax\"\n\t" \
554 "11:\tli\t%1, %3\n\t" \
555 "j\t10b\n\t" \
556 ".previous\n\t" \
557 ".section\t__ex_table,\"a\"\n\t" \
558 STR(PTR)"\t1b, 11b\n\t" \
559 STR(PTR)"\t2b, 11b\n\t" \
560 STR(PTR)"\t3b, 11b\n\t" \
561 STR(PTR)"\t4b, 11b\n\t" \
562 ".previous" \
563 : "=&r" (value), "=r" (res) \
3563c32d
MC		 564 : "r" (addr), "i" (-EFAULT)); \
565} while(0)
566
0593a44c
LY		 567#endif /* CONFIG_CPU_MIPSR6 */
568
34c2f668 569
eeb53895 570#define _LoadHWU(addr, value, res, type) \
3563c32d 571do { \
34c2f668
LY		 572 __asm__ __volatile__ ( \
573 ".set\tnoat\n" \
eeb53895
MC		 574 "1:\t"type##_lbu("%0", "1(%2)")"\n" \
575 "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\
34c2f668
LY		 576 "sll\t%0, 0x8\n\t" \
577 "or\t%0, $1\n\t" \
578 "li\t%1, 0\n" \
579 "3:\n\t" \
580 ".insn\n\t" \
581 ".set\tat\n\t" \
582 ".section\t.fixup,\"ax\"\n\t" \
583 "4:\tli\t%1, %3\n\t" \
584 "j\t3b\n\t" \
585 ".previous\n\t" \
586 ".section\t__ex_table,\"a\"\n\t" \
587 STR(PTR)"\t1b, 4b\n\t" \
588 STR(PTR)"\t2b, 4b\n\t" \
589 ".previous" \
590 : "=&r" (value), "=r" (res) \
3563c32d
MC		 591 : "r" (addr), "i" (-EFAULT)); \
592} while(0)
34c2f668 593
0593a44c 594#ifndef CONFIG_CPU_MIPSR6
eeb53895 595#define _LoadWU(addr, value, res, type) \
3563c32d 596do { \
34c2f668 597 __asm__ __volatile__ ( \
eeb53895
MC		 598 "1:\t"type##_lwl("%0", "3(%2)")"\n" \
599 "2:\t"type##_lwr("%0", "(%2)")"\n\t"\
34c2f668
LY		 600 "dsll\t%0, %0, 32\n\t" \
601 "dsrl\t%0, %0, 32\n\t" \
602 "li\t%1, 0\n" \
603 "3:\n\t" \
604 ".insn\n\t" \
605 "\t.section\t.fixup,\"ax\"\n\t" \
606 "4:\tli\t%1, %3\n\t" \
607 "j\t3b\n\t" \
608 ".previous\n\t" \
609 ".section\t__ex_table,\"a\"\n\t" \
610 STR(PTR)"\t1b, 4b\n\t" \
611 STR(PTR)"\t2b, 4b\n\t" \
612 ".previous" \
613 : "=&r" (value), "=r" (res) \
3563c32d
MC		 614 : "r" (addr), "i" (-EFAULT)); \
615} while(0)
34c2f668 616
eeb53895 617#define _LoadDW(addr, value, res) \
3563c32d 618do { \
34c2f668
LY		 619 __asm__ __volatile__ ( \
620 "1:\tldl\t%0, 7(%2)\n" \
621 "2:\tldr\t%0, (%2)\n\t" \
622 "li\t%1, 0\n" \
623 "3:\n\t" \
624 ".insn\n\t" \
625 "\t.section\t.fixup,\"ax\"\n\t" \
626 "4:\tli\t%1, %3\n\t" \
627 "j\t3b\n\t" \
628 ".previous\n\t" \
629 ".section\t__ex_table,\"a\"\n\t" \
630 STR(PTR)"\t1b, 4b\n\t" \
631 STR(PTR)"\t2b, 4b\n\t" \
632 ".previous" \
633 : "=&r" (value), "=r" (res) \
3563c32d
MC		 634 : "r" (addr), "i" (-EFAULT)); \
635} while(0)
636
0593a44c
LY		 637#else
638/* MIPSR6 has not lwl and ldl instructions */
eeb53895 639#define _LoadWU(addr, value, res, type) \
3563c32d 640do { \
0593a44c
LY		 641 __asm__ __volatile__ ( \
642 ".set\tpush\n\t" \
643 ".set\tnoat\n\t" \
eeb53895
MC		 644 "1:"type##_lbu("%0", "3(%2)")"\n\t" \
645 "2:"type##_lbu("$1", "2(%2)")"\n\t" \
0593a44c
LY		 646 "sll\t%0, 0x8\n\t" \
647 "or\t%0, $1\n\t" \
eeb53895 648 "3:"type##_lbu("$1", "1(%2)")"\n\t" \
0593a44c
LY		 649 "sll\t%0, 0x8\n\t" \
650 "or\t%0, $1\n\t" \
eeb53895 651 "4:"type##_lbu("$1", "0(%2)")"\n\t" \
0593a44c
LY		 652 "sll\t%0, 0x8\n\t" \
653 "or\t%0, $1\n\t" \
654 "li\t%1, 0\n" \
655 ".set\tpop\n" \
656 "10:\n\t" \
657 ".insn\n\t" \
658 ".section\t.fixup,\"ax\"\n\t" \
659 "11:\tli\t%1, %3\n\t" \
660 "j\t10b\n\t" \
661 ".previous\n\t" \
662 ".section\t__ex_table,\"a\"\n\t" \
663 STR(PTR)"\t1b, 11b\n\t" \
664 STR(PTR)"\t2b, 11b\n\t" \
665 STR(PTR)"\t3b, 11b\n\t" \
666 STR(PTR)"\t4b, 11b\n\t" \
667 ".previous" \
668 : "=&r" (value), "=r" (res) \
3563c32d
MC		 669 : "r" (addr), "i" (-EFAULT)); \
670} while(0)
0593a44c 671
eeb53895 672#define _LoadDW(addr, value, res) \
3563c32d 673do { \
0593a44c
LY		 674 __asm__ __volatile__ ( \
675 ".set\tpush\n\t" \
676 ".set\tnoat\n\t" \
677 "1:lb\t%0, 7(%2)\n\t" \
678 "2:lbu\t$1, 6(%2)\n\t" \
679 "dsll\t%0, 0x8\n\t" \
680 "or\t%0, $1\n\t" \
681 "3:lbu\t$1, 5(%2)\n\t" \
682 "dsll\t%0, 0x8\n\t" \
683 "or\t%0, $1\n\t" \
684 "4:lbu\t$1, 4(%2)\n\t" \
685 "dsll\t%0, 0x8\n\t" \
686 "or\t%0, $1\n\t" \
687 "5:lbu\t$1, 3(%2)\n\t" \
688 "dsll\t%0, 0x8\n\t" \
689 "or\t%0, $1\n\t" \
690 "6:lbu\t$1, 2(%2)\n\t" \
691 "dsll\t%0, 0x8\n\t" \
692 "or\t%0, $1\n\t" \
693 "7:lbu\t$1, 1(%2)\n\t" \
694 "dsll\t%0, 0x8\n\t" \
695 "or\t%0, $1\n\t" \
696 "8:lbu\t$1, 0(%2)\n\t" \
697 "dsll\t%0, 0x8\n\t" \
698 "or\t%0, $1\n\t" \
699 "li\t%1, 0\n" \
700 ".set\tpop\n\t" \
701 "10:\n\t" \
702 ".insn\n\t" \
703 ".section\t.fixup,\"ax\"\n\t" \
704 "11:\tli\t%1, %3\n\t" \
705 "j\t10b\n\t" \
706 ".previous\n\t" \
707 ".section\t__ex_table,\"a\"\n\t" \
708 STR(PTR)"\t1b, 11b\n\t" \
709 STR(PTR)"\t2b, 11b\n\t" \
710 STR(PTR)"\t3b, 11b\n\t" \
711 STR(PTR)"\t4b, 11b\n\t" \
712 STR(PTR)"\t5b, 11b\n\t" \
713 STR(PTR)"\t6b, 11b\n\t" \
714 STR(PTR)"\t7b, 11b\n\t" \
715 STR(PTR)"\t8b, 11b\n\t" \
716 ".previous" \
717 : "=&r" (value), "=r" (res) \
3563c32d
MC		 718 : "r" (addr), "i" (-EFAULT)); \
719} while(0)
0593a44c 720#endif /* CONFIG_CPU_MIPSR6 */
34c2f668 721
eeb53895 722#define _StoreHW(addr, value, res, type) \
3563c32d 723do { \
34c2f668
LY		 724 __asm__ __volatile__ ( \
725 ".set\tnoat\n" \
eeb53895 726 "1:\t"type##_sb("%1", "0(%2)")"\n" \
34c2f668 727 "srl\t$1,%1, 0x8\n" \
eeb53895 728 "2:\t"type##_sb("$1", "1(%2)")"\n" \
34c2f668
LY		 729 ".set\tat\n\t" \
730 "li\t%0, 0\n" \
731 "3:\n\t" \
732 ".insn\n\t" \
733 ".section\t.fixup,\"ax\"\n\t" \
734 "4:\tli\t%0, %3\n\t" \
735 "j\t3b\n\t" \
736 ".previous\n\t" \
737 ".section\t__ex_table,\"a\"\n\t" \
738 STR(PTR)"\t1b, 4b\n\t" \
739 STR(PTR)"\t2b, 4b\n\t" \
740 ".previous" \
741 : "=r" (res) \
3563c32d
MC		 742 : "r" (value), "r" (addr), "i" (-EFAULT));\
743} while(0)
744
0593a44c 745#ifndef CONFIG_CPU_MIPSR6
eeb53895 746#define _StoreW(addr, value, res, type) \
3563c32d 747do { \
34c2f668 748 __asm__ __volatile__ ( \
eeb53895
MC		 749 "1:\t"type##_swl("%1", "3(%2)")"\n" \
750 "2:\t"type##_swr("%1", "(%2)")"\n\t"\
34c2f668
LY		 751 "li\t%0, 0\n" \
752 "3:\n\t" \
753 ".insn\n\t" \
754 ".section\t.fixup,\"ax\"\n\t" \
755 "4:\tli\t%0, %3\n\t" \
756 "j\t3b\n\t" \
757 ".previous\n\t" \
758 ".section\t__ex_table,\"a\"\n\t" \
759 STR(PTR)"\t1b, 4b\n\t" \
760 STR(PTR)"\t2b, 4b\n\t" \
761 ".previous" \
762 : "=r" (res) \
3563c32d
MC		 763 : "r" (value), "r" (addr), "i" (-EFAULT)); \
764} while(0)
34c2f668 765
eeb53895 766#define _StoreDW(addr, value, res) \
3563c32d 767do { \
34c2f668
LY		 768 __asm__ __volatile__ ( \
769 "1:\tsdl\t%1, 7(%2)\n" \
770 "2:\tsdr\t%1, (%2)\n\t" \
771 "li\t%0, 0\n" \
772 "3:\n\t" \
773 ".insn\n\t" \
774 ".section\t.fixup,\"ax\"\n\t" \
775 "4:\tli\t%0, %3\n\t" \
776 "j\t3b\n\t" \
777 ".previous\n\t" \
778 ".section\t__ex_table,\"a\"\n\t" \
779 STR(PTR)"\t1b, 4b\n\t" \
780 STR(PTR)"\t2b, 4b\n\t" \
781 ".previous" \
782 : "=r" (res) \
3563c32d
MC		 783 : "r" (value), "r" (addr), "i" (-EFAULT)); \
784} while(0)
785
0593a44c
LY		 786#else
787/* MIPSR6 has no swl and sdl instructions */
eeb53895 788#define _StoreW(addr, value, res, type) \
3563c32d 789do { \
0593a44c
LY		 790 __asm__ __volatile__ ( \
791 ".set\tpush\n\t" \
792 ".set\tnoat\n\t" \
eeb53895 793 "1:"type##_sb("%1", "0(%2)")"\n\t" \
0593a44c 794 "srl\t$1, %1, 0x8\n\t" \
eeb53895 795 "2:"type##_sb("$1", "1(%2)")"\n\t" \
0593a44c 796 "srl\t$1, $1, 0x8\n\t" \
eeb53895 797 "3:"type##_sb("$1", "2(%2)")"\n\t" \
0593a44c 798 "srl\t$1, $1, 0x8\n\t" \
eeb53895 799 "4:"type##_sb("$1", "3(%2)")"\n\t" \
0593a44c
LY		 800 ".set\tpop\n\t" \
801 "li\t%0, 0\n" \
802 "10:\n\t" \
803 ".insn\n\t" \
804 ".section\t.fixup,\"ax\"\n\t" \
805 "11:\tli\t%0, %3\n\t" \
806 "j\t10b\n\t" \
807 ".previous\n\t" \
808 ".section\t__ex_table,\"a\"\n\t" \
809 STR(PTR)"\t1b, 11b\n\t" \
810 STR(PTR)"\t2b, 11b\n\t" \
811 STR(PTR)"\t3b, 11b\n\t" \
812 STR(PTR)"\t4b, 11b\n\t" \
813 ".previous" \
814 : "=&r" (res) \
815 : "r" (value), "r" (addr), "i" (-EFAULT) \
3563c32d
MC		 816 : "memory"); \
817} while(0)
0593a44c 818
eeb53895 819#define _StoreDW(addr, value, res) \
3563c32d 820do { \
0593a44c
LY		 821 __asm__ __volatile__ ( \
822 ".set\tpush\n\t" \
823 ".set\tnoat\n\t" \
824 "1:sb\t%1, 0(%2)\n\t" \
825 "dsrl\t$1, %1, 0x8\n\t" \
826 "2:sb\t$1, 1(%2)\n\t" \
827 "dsrl\t$1, $1, 0x8\n\t" \
828 "3:sb\t$1, 2(%2)\n\t" \
829 "dsrl\t$1, $1, 0x8\n\t" \
830 "4:sb\t$1, 3(%2)\n\t" \
831 "dsrl\t$1, $1, 0x8\n\t" \
832 "5:sb\t$1, 4(%2)\n\t" \
833 "dsrl\t$1, $1, 0x8\n\t" \
834 "6:sb\t$1, 5(%2)\n\t" \
835 "dsrl\t$1, $1, 0x8\n\t" \
836 "7:sb\t$1, 6(%2)\n\t" \
837 "dsrl\t$1, $1, 0x8\n\t" \
838 "8:sb\t$1, 7(%2)\n\t" \
839 "dsrl\t$1, $1, 0x8\n\t" \
840 ".set\tpop\n\t" \
841 "li\t%0, 0\n" \
842 "10:\n\t" \
843 ".insn\n\t" \
844 ".section\t.fixup,\"ax\"\n\t" \
845 "11:\tli\t%0, %3\n\t" \
846 "j\t10b\n\t" \
847 ".previous\n\t" \
848 ".section\t__ex_table,\"a\"\n\t" \
849 STR(PTR)"\t1b, 11b\n\t" \
850 STR(PTR)"\t2b, 11b\n\t" \
851 STR(PTR)"\t3b, 11b\n\t" \
852 STR(PTR)"\t4b, 11b\n\t" \
853 STR(PTR)"\t5b, 11b\n\t" \
854 STR(PTR)"\t6b, 11b\n\t" \
855 STR(PTR)"\t7b, 11b\n\t" \
856 STR(PTR)"\t8b, 11b\n\t" \
857 ".previous" \
858 : "=&r" (res) \
859 : "r" (value), "r" (addr), "i" (-EFAULT) \
3563c32d
MC		 860 : "memory"); \
861} while(0)
862
0593a44c 863#endif /* CONFIG_CPU_MIPSR6 */
34c2f668
LY		 864#endif
865
eeb53895
MC		 866#define LoadHWU(addr, value, res) _LoadHWU(addr, value, res, kernel)
867#define LoadHWUE(addr, value, res) _LoadHWU(addr, value, res, user)
868#define LoadWU(addr, value, res) _LoadWU(addr, value, res, kernel)
869#define LoadWUE(addr, value, res) _LoadWU(addr, value, res, user)
870#define LoadHW(addr, value, res) _LoadHW(addr, value, res, kernel)
871#define LoadHWE(addr, value, res) _LoadHW(addr, value, res, user)
872#define LoadW(addr, value, res) _LoadW(addr, value, res, kernel)
873#define LoadWE(addr, value, res) _LoadW(addr, value, res, user)
874#define LoadDW(addr, value, res) _LoadDW(addr, value, res)
875
876#define StoreHW(addr, value, res) _StoreHW(addr, value, res, kernel)
877#define StoreHWE(addr, value, res) _StoreHW(addr, value, res, user)
878#define StoreW(addr, value, res) _StoreW(addr, value, res, kernel)
879#define StoreWE(addr, value, res) _StoreW(addr, value, res, user)
880#define StoreDW(addr, value, res) _StoreDW(addr, value, res)
881
7f18f151
RB		 882static void emulate_load_store_insn(struct pt_regs *regs,
883 void __user *addr, unsigned int __user *pc)
1da177e4
LT		 884{
885 union mips_instruction insn;
886 unsigned long value;
887 unsigned int res;
34c2f668
LY		 888 unsigned long origpc;
889 unsigned long orig31;
102cedc3 890 void __user *fault_addr = NULL;
c1771216
LY		 891#ifdef CONFIG_EVA
892 mm_segment_t seg;
893#endif
34c2f668
LY		 894 origpc = (unsigned long)pc;
895 orig31 = regs->regs[31];
896
a8b0ca17 897 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0);
7f788d2d 898
1da177e4
LT		 899 /*
900 * This load never faults.
901 */
fe00f943 902 __get_user(insn.word, pc);
1da177e4
LT		 903
904 switch (insn.i_format.opcode) {
34c2f668
LY		 905 /*
906 * These are instructions that a compiler doesn't generate. We
907 * can assume therefore that the code is MIPS-aware and
908 * really buggy. Emulating these instructions would break the
909 * semantics anyway.
910 */
1da177e4
LT		 911 case ll_op:
912 case lld_op:
913 case sc_op:
914 case scd_op:
915
34c2f668
LY		 916 /*
917 * For these instructions the only way to create an address
918 * error is an attempted access to kernel/supervisor address
919 * space.
920 */
1da177e4
LT		 921 case ldl_op:
922 case ldr_op:
923 case lwl_op:
924 case lwr_op:
925 case sdl_op:
926 case sdr_op:
927 case swl_op:
928 case swr_op:
929 case lb_op:
930 case lbu_op:
931 case sb_op:
932 goto sigbus;
933
34c2f668
LY		 934 /*
935 * The remaining opcodes are the ones that are really of
936 * interest.
937 */
c1771216
LY		 938#ifdef CONFIG_EVA
939 case spec3_op:
940 /*
941 * we can land here only from kernel accessing user memory,
942 * so we need to "switch" the address limit to user space, so
943 * address check can work properly.
944 */
945 seg = get_fs();
946 set_fs(USER_DS);
947 switch (insn.spec3_format.func) {
948 case lhe_op:
949 if (!access_ok(VERIFY_READ, addr, 2)) {
950 set_fs(seg);
951 goto sigbus;
952 }
eeb53895 953 LoadHWE(addr, value, res);
c1771216
LY		 954 if (res) {
955 set_fs(seg);
956 goto fault;
957 }
958 compute_return_epc(regs);
959 regs->regs[insn.spec3_format.rt] = value;
960 break;
961 case lwe_op:
962 if (!access_ok(VERIFY_READ, addr, 4)) {
963 set_fs(seg);
964 goto sigbus;
965 }
eeb53895 966 LoadWE(addr, value, res);
c1771216
LY		 967 if (res) {
968 set_fs(seg);
969 goto fault;
970 }
971 compute_return_epc(regs);
972 regs->regs[insn.spec3_format.rt] = value;
973 break;
974 case lhue_op:
975 if (!access_ok(VERIFY_READ, addr, 2)) {
976 set_fs(seg);
977 goto sigbus;
978 }
eeb53895 979 LoadHWUE(addr, value, res);
c1771216
LY		 980 if (res) {
981 set_fs(seg);
982 goto fault;
983 }
984 compute_return_epc(regs);
985 regs->regs[insn.spec3_format.rt] = value;
986 break;
987 case she_op:
988 if (!access_ok(VERIFY_WRITE, addr, 2)) {
989 set_fs(seg);
990 goto sigbus;
991 }
992 compute_return_epc(regs);
993 value = regs->regs[insn.spec3_format.rt];
eeb53895 994 StoreHWE(addr, value, res);
c1771216
LY		 995 if (res) {
996 set_fs(seg);
997 goto fault;
998 }
999 break;
1000 case swe_op:
1001 if (!access_ok(VERIFY_WRITE, addr, 4)) {
1002 set_fs(seg);
1003 goto sigbus;
1004 }
1005 compute_return_epc(regs);
1006 value = regs->regs[insn.spec3_format.rt];
eeb53895 1007 StoreWE(addr, value, res);
c1771216
LY		 1008 if (res) {
1009 set_fs(seg);
1010 goto fault;
1011 }
1012 break;
1013 default:
1014 set_fs(seg);
1015 goto sigill;
1016 }
1017 set_fs(seg);
1018 break;
1019#endif
1da177e4
LT		 1020 case lh_op:
1021 if (!access_ok(VERIFY_READ, addr, 2))
1022 goto sigbus;
1023
6eae3548
MC		 1024 if (config_enabled(CONFIG_EVA)) {
1025 if (segment_eq(get_fs(), get_ds()))
1026 LoadHW(addr, value, res);
1027 else
1028 LoadHWE(addr, value, res);
1029 } else {
1030 LoadHW(addr, value, res);
1031 }
1032
1da177e4
LT		 1033 if (res)
1034 goto fault;
7f18f151
RB		 1035 compute_return_epc(regs);
1036 regs->regs[insn.i_format.rt] = value;
1da177e4
LT		 1037 break;
1038
1039 case lw_op:
1040 if (!access_ok(VERIFY_READ, addr, 4))
1041 goto sigbus;
1042
6eae3548
MC		 1043 if (config_enabled(CONFIG_EVA)) {
1044 if (segment_eq(get_fs(), get_ds()))
1045 LoadW(addr, value, res);
1046 else
1047 LoadWE(addr, value, res);
1048 } else {
1049 LoadW(addr, value, res);
1050 }
1051
1da177e4
LT		 1052 if (res)
1053 goto fault;
7f18f151
RB		 1054 compute_return_epc(regs);
1055 regs->regs[insn.i_format.rt] = value;
1da177e4
LT		 1056 break;
1057
1058 case lhu_op:
1059 if (!access_ok(VERIFY_READ, addr, 2))
1060 goto sigbus;
1061
6eae3548
MC		 1062 if (config_enabled(CONFIG_EVA)) {
1063 if (segment_eq(get_fs(), get_ds()))
1064 LoadHWU(addr, value, res);
1065 else
1066 LoadHWUE(addr, value, res);
1067 } else {
1068 LoadHWU(addr, value, res);
1069 }
1070
1da177e4
LT		 1071 if (res)
1072 goto fault;
7f18f151
RB		 1073 compute_return_epc(regs);
1074 regs->regs[insn.i_format.rt] = value;
1da177e4
LT		 1075 break;
1076
1077 case lwu_op:
875d43e7 1078#ifdef CONFIG_64BIT
1da177e4
LT		 1079 /*
1080 * A 32-bit kernel might be running on a 64-bit processor. But
1081 * if we're on a 32-bit processor and an i-cache incoherency
1082 * or race makes us see a 64-bit instruction here the sdl/sdr
1083 * would blow up, so for now we don't handle unaligned 64-bit
1084 * instructions on 32-bit kernels.
1085 */
1086 if (!access_ok(VERIFY_READ, addr, 4))
1087 goto sigbus;
1088
34c2f668 1089 LoadWU(addr, value, res);
1da177e4
LT		 1090 if (res)
1091 goto fault;
7f18f151
RB		 1092 compute_return_epc(regs);
1093 regs->regs[insn.i_format.rt] = value;
1da177e4 1094 break;
875d43e7 1095#endif /* CONFIG_64BIT */
1da177e4
LT		 1096
1097 /* Cannot handle 64-bit instructions in 32-bit kernel */
1098 goto sigill;
1099
1100 case ld_op:
875d43e7 1101#ifdef CONFIG_64BIT
1da177e4
LT		 1102 /*
1103 * A 32-bit kernel might be running on a 64-bit processor. But
1104 * if we're on a 32-bit processor and an i-cache incoherency
1105 * or race makes us see a 64-bit instruction here the sdl/sdr
1106 * would blow up, so for now we don't handle unaligned 64-bit
1107 * instructions on 32-bit kernels.
1108 */
1109 if (!access_ok(VERIFY_READ, addr, 8))
1110 goto sigbus;
1111
34c2f668 1112 LoadDW(addr, value, res);
1da177e4
LT		 1113 if (res)
1114 goto fault;
7f18f151
RB		 1115 compute_return_epc(regs);
1116 regs->regs[insn.i_format.rt] = value;
1da177e4 1117 break;
875d43e7 1118#endif /* CONFIG_64BIT */
1da177e4
LT		 1119
1120 /* Cannot handle 64-bit instructions in 32-bit kernel */
1121 goto sigill;
1122
1123 case sh_op:
1124 if (!access_ok(VERIFY_WRITE, addr, 2))
1125 goto sigbus;
1126
34c2f668 1127 compute_return_epc(regs);
1da177e4 1128 value = regs->regs[insn.i_format.rt];
6eae3548
MC		 1129
1130 if (config_enabled(CONFIG_EVA)) {
1131 if (segment_eq(get_fs(), get_ds()))
1132 StoreHW(addr, value, res);
1133 else
1134 StoreHWE(addr, value, res);
1135 } else {
1136 StoreHW(addr, value, res);
1137 }
1138
1da177e4
LT		 1139 if (res)
1140 goto fault;
1141 break;
1142
1143 case sw_op:
1144 if (!access_ok(VERIFY_WRITE, addr, 4))
1145 goto sigbus;
1146
34c2f668 1147 compute_return_epc(regs);
1da177e4 1148 value = regs->regs[insn.i_format.rt];
6eae3548
MC		 1149
1150 if (config_enabled(CONFIG_EVA)) {
1151 if (segment_eq(get_fs(), get_ds()))
1152 StoreW(addr, value, res);
1153 else
1154 StoreWE(addr, value, res);
1155 } else {
1156 StoreW(addr, value, res);
1157 }
1158
1da177e4
LT		 1159 if (res)
1160 goto fault;
1161 break;
1162
1163 case sd_op:
875d43e7 1164#ifdef CONFIG_64BIT
1da177e4
LT		 1165 /*
1166 * A 32-bit kernel might be running on a 64-bit processor. But
1167 * if we're on a 32-bit processor and an i-cache incoherency
1168 * or race makes us see a 64-bit instruction here the sdl/sdr
1169 * would blow up, so for now we don't handle unaligned 64-bit
1170 * instructions on 32-bit kernels.
1171 */
1172 if (!access_ok(VERIFY_WRITE, addr, 8))
1173 goto sigbus;
1174
34c2f668 1175 compute_return_epc(regs);
1da177e4 1176 value = regs->regs[insn.i_format.rt];
34c2f668 1177 StoreDW(addr, value, res);
1da177e4
LT		 1178 if (res)
1179 goto fault;
1180 break;
875d43e7 1181#endif /* CONFIG_64BIT */
1da177e4
LT		 1182
1183 /* Cannot handle 64-bit instructions in 32-bit kernel */
1184 goto sigill;
1185
1186 case lwc1_op:
1187 case ldc1_op:
1188 case swc1_op:
1189 case sdc1_op:
102cedc3
LY		 1190 die_if_kernel("Unaligned FP access in kernel code", regs);
1191 BUG_ON(!used_math());
102cedc3
LY		 1192
1193 lose_fpu(1); /* Save FPU state for the emulator. */
1194 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1195 &fault_addr);
1196 own_fpu(1); /* Restore FPU state. */
1197
1198 /* Signal if something went wrong. */
304acb71 1199 process_fpemu_return(res, fault_addr, 0);
102cedc3
LY		 1200
1201 if (res == 0)
1202 break;
1203 return;
1da177e4 1204
0593a44c 1205#ifndef CONFIG_CPU_MIPSR6
69f3a7de
RB		 1206 /*
1207 * COP2 is available to implementor for application specific use.
1208 * It's up to applications to register a notifier chain and do
1209 * whatever they have to do, including possible sending of signals.
0593a44c
LY		 1210 *
1211 * This instruction has been reallocated in Release 6
69f3a7de 1212 */
1da177e4 1213 case lwc2_op:
69f3a7de
RB		 1214 cu2_notifier_call_chain(CU2_LWC2_OP, regs);
1215 break;
1216
1da177e4 1217 case ldc2_op:
69f3a7de
RB		 1218 cu2_notifier_call_chain(CU2_LDC2_OP, regs);
1219 break;
1220
1da177e4 1221 case swc2_op:
69f3a7de
RB		 1222 cu2_notifier_call_chain(CU2_SWC2_OP, regs);
1223 break;
1224
1da177e4 1225 case sdc2_op:
69f3a7de
RB		 1226 cu2_notifier_call_chain(CU2_SDC2_OP, regs);
1227 break;
0593a44c 1228#endif
1da177e4
LT		 1229 default:
1230 /*
1231 * Pheeee... We encountered an yet unknown instruction or
1232 * cache coherence problem. Die sucker, die ...
1233 */
1234 goto sigill;
1235 }
1236
6312e0ee 1237#ifdef CONFIG_DEBUG_FS
1da177e4
LT		 1238 unaligned_instructions++;
1239#endif
1240
7f18f151 1241 return;
1da177e4
LT		 1242
1243fault:
34c2f668
LY		 1244 /* roll back jump/branch */
1245 regs->cp0_epc = origpc;
1246 regs->regs[31] = orig31;
1247 /* Did we have an exception handler installed? */
1248 if (fixup_exception(regs))
1249 return;
1250
1251 die_if_kernel("Unhandled kernel unaligned access", regs);
1252 force_sig(SIGSEGV, current);
1253
1254 return;
1255
1256sigbus:
1257 die_if_kernel("Unhandled kernel unaligned access", regs);
1258 force_sig(SIGBUS, current);
1259
1260 return;
1261
1262sigill:
1263 die_if_kernel
1264 ("Unhandled kernel unaligned access or invalid instruction", regs);
1265 force_sig(SIGILL, current);
1266}
1267
1268/* Recode table from 16-bit register notation to 32-bit GPR. */
1269const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 };
1270
1271/* Recode table from 16-bit STORE register notation to 32-bit GPR. */
1272const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 };
1273
74338805
DD		 1274static void emulate_load_store_microMIPS(struct pt_regs *regs,
1275 void __user *addr)
34c2f668
LY		 1276{
1277 unsigned long value;
1278 unsigned int res;
1279 int i;
1280 unsigned int reg = 0, rvar;
1281 unsigned long orig31;
1282 u16 __user *pc16;
1283 u16 halfword;
1284 unsigned int word;
1285 unsigned long origpc, contpc;
1286 union mips_instruction insn;
1287 struct mm_decoded_insn mminsn;
1288 void __user *fault_addr = NULL;
1289
1290 origpc = regs->cp0_epc;
1291 orig31 = regs->regs[31];
1292
1293 mminsn.micro_mips_mode = 1;
1294
1295 /*
1296 * This load never faults.
1297 */
1298 pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc);
1299 __get_user(halfword, pc16);
1300 pc16++;
1301 contpc = regs->cp0_epc + 2;
1302 word = ((unsigned int)halfword << 16);
1303 mminsn.pc_inc = 2;
1304
1305 if (!mm_insn_16bit(halfword)) {
1306 __get_user(halfword, pc16);
1307 pc16++;
1308 contpc = regs->cp0_epc + 4;
1309 mminsn.pc_inc = 4;
1310 word |= halfword;
1311 }
1312 mminsn.insn = word;
1313
1314 if (get_user(halfword, pc16))
1315 goto fault;
1316 mminsn.next_pc_inc = 2;
1317 word = ((unsigned int)halfword << 16);
1318
1319 if (!mm_insn_16bit(halfword)) {
1320 pc16++;
1321 if (get_user(halfword, pc16))
1322 goto fault;
1323 mminsn.next_pc_inc = 4;
1324 word |= halfword;
1325 }
1326 mminsn.next_insn = word;
1327
1328 insn = (union mips_instruction)(mminsn.insn);
1329 if (mm_isBranchInstr(regs, mminsn, &contpc))
1330 insn = (union mips_instruction)(mminsn.next_insn);
1331
1332 /* Parse instruction to find what to do */
1333
1334 switch (insn.mm_i_format.opcode) {
1335
1336 case mm_pool32a_op:
1337 switch (insn.mm_x_format.func) {
1338 case mm_lwxs_op:
1339 reg = insn.mm_x_format.rd;
1340 goto loadW;
1341 }
1342
1343 goto sigbus;
1344
1345 case mm_pool32b_op:
1346 switch (insn.mm_m_format.func) {
1347 case mm_lwp_func:
1348 reg = insn.mm_m_format.rd;
1349 if (reg == 31)
1350 goto sigbus;
1351
1352 if (!access_ok(VERIFY_READ, addr, 8))
1353 goto sigbus;
1354
1355 LoadW(addr, value, res);
1356 if (res)
1357 goto fault;
1358 regs->regs[reg] = value;
1359 addr += 4;
1360 LoadW(addr, value, res);
1361 if (res)
1362 goto fault;
1363 regs->regs[reg + 1] = value;
1364 goto success;
1365
1366 case mm_swp_func:
1367 reg = insn.mm_m_format.rd;
1368 if (reg == 31)
1369 goto sigbus;
1370
1371 if (!access_ok(VERIFY_WRITE, addr, 8))
1372 goto sigbus;
1373
1374 value = regs->regs[reg];
1375 StoreW(addr, value, res);
1376 if (res)
1377 goto fault;
1378 addr += 4;
1379 value = regs->regs[reg + 1];
1380 StoreW(addr, value, res);
1381 if (res)
1382 goto fault;
1383 goto success;
1384
1385 case mm_ldp_func:
1386#ifdef CONFIG_64BIT
1387 reg = insn.mm_m_format.rd;
1388 if (reg == 31)
1389 goto sigbus;
1390
1391 if (!access_ok(VERIFY_READ, addr, 16))
1392 goto sigbus;
1393
1394 LoadDW(addr, value, res);
1395 if (res)
1396 goto fault;
1397 regs->regs[reg] = value;
1398 addr += 8;
1399 LoadDW(addr, value, res);
1400 if (res)
1401 goto fault;
1402 regs->regs[reg + 1] = value;
1403 goto success;
1404#endif /* CONFIG_64BIT */
1405
1406 goto sigill;
1407
1408 case mm_sdp_func:
1409#ifdef CONFIG_64BIT
1410 reg = insn.mm_m_format.rd;
1411 if (reg == 31)
1412 goto sigbus;
1413
1414 if (!access_ok(VERIFY_WRITE, addr, 16))
1415 goto sigbus;
1416
1417 value = regs->regs[reg];
1418 StoreDW(addr, value, res);
1419 if (res)
1420 goto fault;
1421 addr += 8;
1422 value = regs->regs[reg + 1];
1423 StoreDW(addr, value, res);
1424 if (res)
1425 goto fault;
1426 goto success;
1427#endif /* CONFIG_64BIT */
1428
1429 goto sigill;
1430
1431 case mm_lwm32_func:
1432 reg = insn.mm_m_format.rd;
1433 rvar = reg & 0xf;
1434 if ((rvar > 9) || !reg)
1435 goto sigill;
1436 if (reg & 0x10) {
1437 if (!access_ok
1438 (VERIFY_READ, addr, 4 * (rvar + 1)))
1439 goto sigbus;
1440 } else {
1441 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1442 goto sigbus;
1443 }
1444 if (rvar == 9)
1445 rvar = 8;
1446 for (i = 16; rvar; rvar--, i++) {
1447 LoadW(addr, value, res);
1448 if (res)
1449 goto fault;
1450 addr += 4;
1451 regs->regs[i] = value;
1452 }
1453 if ((reg & 0xf) == 9) {
1454 LoadW(addr, value, res);
1455 if (res)
1456 goto fault;
1457 addr += 4;
1458 regs->regs[30] = value;
1459 }
1460 if (reg & 0x10) {
1461 LoadW(addr, value, res);
1462 if (res)
1463 goto fault;
1464 regs->regs[31] = value;
1465 }
1466 goto success;
1467
1468 case mm_swm32_func:
1469 reg = insn.mm_m_format.rd;
1470 rvar = reg & 0xf;
1471 if ((rvar > 9) || !reg)
1472 goto sigill;
1473 if (reg & 0x10) {
1474 if (!access_ok
1475 (VERIFY_WRITE, addr, 4 * (rvar + 1)))
1476 goto sigbus;
1477 } else {
1478 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1479 goto sigbus;
1480 }
1481 if (rvar == 9)
1482 rvar = 8;
1483 for (i = 16; rvar; rvar--, i++) {
1484 value = regs->regs[i];
1485 StoreW(addr, value, res);
1486 if (res)
1487 goto fault;
1488 addr += 4;
1489 }
1490 if ((reg & 0xf) == 9) {
1491 value = regs->regs[30];
1492 StoreW(addr, value, res);
1493 if (res)
1494 goto fault;
1495 addr += 4;
1496 }
1497 if (reg & 0x10) {
1498 value = regs->regs[31];
1499 StoreW(addr, value, res);
1500 if (res)
1501 goto fault;
1502 }
1503 goto success;
1504
1505 case mm_ldm_func:
1506#ifdef CONFIG_64BIT
1507 reg = insn.mm_m_format.rd;
1508 rvar = reg & 0xf;
1509 if ((rvar > 9) || !reg)
1510 goto sigill;
1511 if (reg & 0x10) {
1512 if (!access_ok
1513 (VERIFY_READ, addr, 8 * (rvar + 1)))
1514 goto sigbus;
1515 } else {
1516 if (!access_ok(VERIFY_READ, addr, 8 * rvar))
1517 goto sigbus;
1518 }
1519 if (rvar == 9)
1520 rvar = 8;
1521
1522 for (i = 16; rvar; rvar--, i++) {
1523 LoadDW(addr, value, res);
1524 if (res)
1525 goto fault;
1526 addr += 4;
1527 regs->regs[i] = value;
1528 }
1529 if ((reg & 0xf) == 9) {
1530 LoadDW(addr, value, res);
1531 if (res)
1532 goto fault;
1533 addr += 8;
1534 regs->regs[30] = value;
1535 }
1536 if (reg & 0x10) {
1537 LoadDW(addr, value, res);
1538 if (res)
1539 goto fault;
1540 regs->regs[31] = value;
1541 }
1542 goto success;
1543#endif /* CONFIG_64BIT */
1544
1545 goto sigill;
1546
1547 case mm_sdm_func:
1548#ifdef CONFIG_64BIT
1549 reg = insn.mm_m_format.rd;
1550 rvar = reg & 0xf;
1551 if ((rvar > 9) || !reg)
1552 goto sigill;
1553 if (reg & 0x10) {
1554 if (!access_ok
1555 (VERIFY_WRITE, addr, 8 * (rvar + 1)))
1556 goto sigbus;
1557 } else {
1558 if (!access_ok(VERIFY_WRITE, addr, 8 * rvar))
1559 goto sigbus;
1560 }
1561 if (rvar == 9)
1562 rvar = 8;
1563
1564 for (i = 16; rvar; rvar--, i++) {
1565 value = regs->regs[i];
1566 StoreDW(addr, value, res);
1567 if (res)
1568 goto fault;
1569 addr += 8;
1570 }
1571 if ((reg & 0xf) == 9) {
1572 value = regs->regs[30];
1573 StoreDW(addr, value, res);
1574 if (res)
1575 goto fault;
1576 addr += 8;
1577 }
1578 if (reg & 0x10) {
1579 value = regs->regs[31];
1580 StoreDW(addr, value, res);
1581 if (res)
1582 goto fault;
1583 }
1584 goto success;
1585#endif /* CONFIG_64BIT */
1586
1587 goto sigill;
1588
1589 /* LWC2, SWC2, LDC2, SDC2 are not serviced */
1590 }
1591
1592 goto sigbus;
1593
1594 case mm_pool32c_op:
1595 switch (insn.mm_m_format.func) {
1596 case mm_lwu_func:
1597 reg = insn.mm_m_format.rd;
1598 goto loadWU;
1599 }
1600
1601 /* LL,SC,LLD,SCD are not serviced */
1602 goto sigbus;
1603
1604 case mm_pool32f_op:
1605 switch (insn.mm_x_format.func) {
1606 case mm_lwxc1_func:
1607 case mm_swxc1_func:
1608 case mm_ldxc1_func:
1609 case mm_sdxc1_func:
1610 goto fpu_emul;
1611 }
1612
1613 goto sigbus;
1614
1615 case mm_ldc132_op:
1616 case mm_sdc132_op:
1617 case mm_lwc132_op:
1618 case mm_swc132_op:
1619fpu_emul:
1620 /* roll back jump/branch */
1621 regs->cp0_epc = origpc;
1622 regs->regs[31] = orig31;
1623
1624 die_if_kernel("Unaligned FP access in kernel code", regs);
1625 BUG_ON(!used_math());
1626 BUG_ON(!is_fpu_owner());
1627
1628 lose_fpu(1); /* save the FPU state for the emulator */
1629 res = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
1630 &fault_addr);
1631 own_fpu(1); /* restore FPU state */
1632
1633 /* If something went wrong, signal */
304acb71 1634 process_fpemu_return(res, fault_addr, 0);
34c2f668
LY		 1635
1636 if (res == 0)
1637 goto success;
1638 return;
1639
1640 case mm_lh32_op:
1641 reg = insn.mm_i_format.rt;
1642 goto loadHW;
1643
1644 case mm_lhu32_op:
1645 reg = insn.mm_i_format.rt;
1646 goto loadHWU;
1647
1648 case mm_lw32_op:
1649 reg = insn.mm_i_format.rt;
1650 goto loadW;
1651
1652 case mm_sh32_op:
1653 reg = insn.mm_i_format.rt;
1654 goto storeHW;
1655
1656 case mm_sw32_op:
1657 reg = insn.mm_i_format.rt;
1658 goto storeW;
1659
1660 case mm_ld32_op:
1661 reg = insn.mm_i_format.rt;
1662 goto loadDW;
1663
1664 case mm_sd32_op:
1665 reg = insn.mm_i_format.rt;
1666 goto storeDW;
1667
1668 case mm_pool16c_op:
1669 switch (insn.mm16_m_format.func) {
1670 case mm_lwm16_op:
1671 reg = insn.mm16_m_format.rlist;
1672 rvar = reg + 1;
1673 if (!access_ok(VERIFY_READ, addr, 4 * rvar))
1674 goto sigbus;
1675
1676 for (i = 16; rvar; rvar--, i++) {
1677 LoadW(addr, value, res);
1678 if (res)
1679 goto fault;
1680 addr += 4;
1681 regs->regs[i] = value;
1682 }
1683 LoadW(addr, value, res);
1684 if (res)
1685 goto fault;
1686 regs->regs[31] = value;
1687
1688 goto success;
1689
1690 case mm_swm16_op:
1691 reg = insn.mm16_m_format.rlist;
1692 rvar = reg + 1;
1693 if (!access_ok(VERIFY_WRITE, addr, 4 * rvar))
1694 goto sigbus;
1695
1696 for (i = 16; rvar; rvar--, i++) {
1697 value = regs->regs[i];
1698 StoreW(addr, value, res);
1699 if (res)
1700 goto fault;
1701 addr += 4;
1702 }
1703 value = regs->regs[31];
1704 StoreW(addr, value, res);
1705 if (res)
1706 goto fault;
1707
1708 goto success;
1709
1710 }
1711
1712 goto sigbus;
1713
1714 case mm_lhu16_op:
1715 reg = reg16to32[insn.mm16_rb_format.rt];
1716 goto loadHWU;
1717
1718 case mm_lw16_op:
1719 reg = reg16to32[insn.mm16_rb_format.rt];
1720 goto loadW;
1721
1722 case mm_sh16_op:
1723 reg = reg16to32st[insn.mm16_rb_format.rt];
1724 goto storeHW;
1725
1726 case mm_sw16_op:
1727 reg = reg16to32st[insn.mm16_rb_format.rt];
1728 goto storeW;
1729
1730 case mm_lwsp16_op:
1731 reg = insn.mm16_r5_format.rt;
1732 goto loadW;
1733
1734 case mm_swsp16_op:
1735 reg = insn.mm16_r5_format.rt;
1736 goto storeW;
1737
1738 case mm_lwgp16_op:
1739 reg = reg16to32[insn.mm16_r3_format.rt];
1740 goto loadW;
1741
1742 default:
1743 goto sigill;
1744 }
1745
1746loadHW:
1747 if (!access_ok(VERIFY_READ, addr, 2))
1748 goto sigbus;
1749
1750 LoadHW(addr, value, res);
1751 if (res)
1752 goto fault;
1753 regs->regs[reg] = value;
1754 goto success;
1755
1756loadHWU:
1757 if (!access_ok(VERIFY_READ, addr, 2))
1758 goto sigbus;
1759
1760 LoadHWU(addr, value, res);
1761 if (res)
1762 goto fault;
1763 regs->regs[reg] = value;
1764 goto success;
1765
1766loadW:
1767 if (!access_ok(VERIFY_READ, addr, 4))
1768 goto sigbus;
1769
1770 LoadW(addr, value, res);
1771 if (res)
1772 goto fault;
1773 regs->regs[reg] = value;
1774 goto success;
1775
1776loadWU:
1777#ifdef CONFIG_64BIT
1778 /*
1779 * A 32-bit kernel might be running on a 64-bit processor. But
1780 * if we're on a 32-bit processor and an i-cache incoherency
1781 * or race makes us see a 64-bit instruction here the sdl/sdr
1782 * would blow up, so for now we don't handle unaligned 64-bit
1783 * instructions on 32-bit kernels.
1784 */
1785 if (!access_ok(VERIFY_READ, addr, 4))
1786 goto sigbus;
1787
1788 LoadWU(addr, value, res);
1789 if (res)
1790 goto fault;
1791 regs->regs[reg] = value;
1792 goto success;
1793#endif /* CONFIG_64BIT */
1794
1795 /* Cannot handle 64-bit instructions in 32-bit kernel */
1796 goto sigill;
1797
1798loadDW:
1799#ifdef CONFIG_64BIT
1800 /*
1801 * A 32-bit kernel might be running on a 64-bit processor. But
1802 * if we're on a 32-bit processor and an i-cache incoherency
1803 * or race makes us see a 64-bit instruction here the sdl/sdr
1804 * would blow up, so for now we don't handle unaligned 64-bit
1805 * instructions on 32-bit kernels.
1806 */
1807 if (!access_ok(VERIFY_READ, addr, 8))
1808 goto sigbus;
1809
1810 LoadDW(addr, value, res);
1811 if (res)
1812 goto fault;
1813 regs->regs[reg] = value;
1814 goto success;
1815#endif /* CONFIG_64BIT */
1816
1817 /* Cannot handle 64-bit instructions in 32-bit kernel */
1818 goto sigill;
1819
1820storeHW:
1821 if (!access_ok(VERIFY_WRITE, addr, 2))
1822 goto sigbus;
1823
1824 value = regs->regs[reg];
1825 StoreHW(addr, value, res);
1826 if (res)
1827 goto fault;
1828 goto success;
1829
1830storeW:
1831 if (!access_ok(VERIFY_WRITE, addr, 4))
1832 goto sigbus;
1833
1834 value = regs->regs[reg];
1835 StoreW(addr, value, res);
1836 if (res)
1837 goto fault;
1838 goto success;
1839
1840storeDW:
1841#ifdef CONFIG_64BIT
1842 /*
1843 * A 32-bit kernel might be running on a 64-bit processor. But
1844 * if we're on a 32-bit processor and an i-cache incoherency
1845 * or race makes us see a 64-bit instruction here the sdl/sdr
1846 * would blow up, so for now we don't handle unaligned 64-bit
1847 * instructions on 32-bit kernels.
1848 */
1849 if (!access_ok(VERIFY_WRITE, addr, 8))
1850 goto sigbus;
1851
1852 value = regs->regs[reg];
1853 StoreDW(addr, value, res);
1854 if (res)
1855 goto fault;
1856 goto success;
1857#endif /* CONFIG_64BIT */
1858
1859 /* Cannot handle 64-bit instructions in 32-bit kernel */
1860 goto sigill;
1861
1862success:
1863 regs->cp0_epc = contpc; /* advance or branch */
1864
1865#ifdef CONFIG_DEBUG_FS
1866 unaligned_instructions++;
1867#endif
1868 return;
1869
1870fault:
1871 /* roll back jump/branch */
1872 regs->cp0_epc = origpc;
1873 regs->regs[31] = orig31;
1da177e4
LT		 1874 /* Did we have an exception handler installed? */
1875 if (fixup_exception(regs))
7f18f151 1876 return;
1da177e4 1877
49a89efb 1878 die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04 1879 force_sig(SIGSEGV, current);
1da177e4 1880
7f18f151 1881 return;
1da177e4
LT		 1882
1883sigbus:
1884 die_if_kernel("Unhandled kernel unaligned access", regs);
a6d5ff04 1885 force_sig(SIGBUS, current);
1da177e4 1886
7f18f151 1887 return;
1da177e4
LT		 1888
1889sigill:
34c2f668
LY		 1890 die_if_kernel
1891 ("Unhandled kernel unaligned access or invalid instruction", regs);
a6d5ff04 1892 force_sig(SIGILL, current);
1da177e4
LT		 1893}
1894
451b001b
SH		 1895static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr)
1896{
1897 unsigned long value;
1898 unsigned int res;
1899 int reg;
1900 unsigned long orig31;
1901 u16 __user *pc16;
1902 unsigned long origpc;
1903 union mips16e_instruction mips16inst, oldinst;
1904
1905 origpc = regs->cp0_epc;
1906 orig31 = regs->regs[31];
1907 pc16 = (unsigned short __user *)msk_isa16_mode(origpc);
1908 /*
1909 * This load never faults.
1910 */
1911 __get_user(mips16inst.full, pc16);
1912 oldinst = mips16inst;
1913
1914 /* skip EXTEND instruction */
1915 if (mips16inst.ri.opcode == MIPS16e_extend_op) {
1916 pc16++;
1917 __get_user(mips16inst.full, pc16);
1918 } else if (delay_slot(regs)) {
1919 /* skip jump instructions */
1920 /* JAL/JALX are 32 bits but have OPCODE in first short int */
1921 if (mips16inst.ri.opcode == MIPS16e_jal_op)
1922 pc16++;
1923 pc16++;
1924 if (get_user(mips16inst.full, pc16))
1925 goto sigbus;
1926 }
1927
1928 switch (mips16inst.ri.opcode) {
1929 case MIPS16e_i64_op: /* I64 or RI64 instruction */
1930 switch (mips16inst.i64.func) { /* I64/RI64 func field check */
1931 case MIPS16e_ldpc_func:
1932 case MIPS16e_ldsp_func:
1933 reg = reg16to32[mips16inst.ri64.ry];
1934 goto loadDW;
1935
1936 case MIPS16e_sdsp_func:
1937 reg = reg16to32[mips16inst.ri64.ry];
1938 goto writeDW;
1939
1940 case MIPS16e_sdrasp_func:
1941 reg = 29; /* GPRSP */
1942 goto writeDW;
1943 }
1944
1945 goto sigbus;
1946
1947 case MIPS16e_swsp_op:
1948 case MIPS16e_lwpc_op:
1949 case MIPS16e_lwsp_op:
1950 reg = reg16to32[mips16inst.ri.rx];
1951 break;
1952
1953 case MIPS16e_i8_op:
1954 if (mips16inst.i8.func != MIPS16e_swrasp_func)
1955 goto sigbus;
1956 reg = 29; /* GPRSP */
1957 break;
1958
1959 default:
1960 reg = reg16to32[mips16inst.rri.ry];
1961 break;
1962 }
1963
1964 switch (mips16inst.ri.opcode) {
1965
1966 case MIPS16e_lb_op:
1967 case MIPS16e_lbu_op:
1968 case MIPS16e_sb_op:
1969 goto sigbus;
1970
1971 case MIPS16e_lh_op:
1972 if (!access_ok(VERIFY_READ, addr, 2))
1973 goto sigbus;
1974
1975 LoadHW(addr, value, res);
1976 if (res)
1977 goto fault;
1978 MIPS16e_compute_return_epc(regs, &oldinst);
1979 regs->regs[reg] = value;
1980 break;
1981
1982 case MIPS16e_lhu_op:
1983 if (!access_ok(VERIFY_READ, addr, 2))
1984 goto sigbus;
1985
1986 LoadHWU(addr, value, res);
1987 if (res)
1988 goto fault;
1989 MIPS16e_compute_return_epc(regs, &oldinst);
1990 regs->regs[reg] = value;
1991 break;
1992
1993 case MIPS16e_lw_op:
1994 case MIPS16e_lwpc_op:
1995 case MIPS16e_lwsp_op:
1996 if (!access_ok(VERIFY_READ, addr, 4))
1997 goto sigbus;
1998
1999 LoadW(addr, value, res);
2000 if (res)
2001 goto fault;
2002 MIPS16e_compute_return_epc(regs, &oldinst);
2003 regs->regs[reg] = value;
2004 break;
2005
2006 case MIPS16e_lwu_op:
2007#ifdef CONFIG_64BIT
2008 /*
2009 * A 32-bit kernel might be running on a 64-bit processor. But
2010 * if we're on a 32-bit processor and an i-cache incoherency
2011 * or race makes us see a 64-bit instruction here the sdl/sdr
2012 * would blow up, so for now we don't handle unaligned 64-bit
2013 * instructions on 32-bit kernels.
2014 */
2015 if (!access_ok(VERIFY_READ, addr, 4))
2016 goto sigbus;
2017
2018 LoadWU(addr, value, res);
2019 if (res)
2020 goto fault;
2021 MIPS16e_compute_return_epc(regs, &oldinst);
2022 regs->regs[reg] = value;
2023 break;
2024#endif /* CONFIG_64BIT */
2025
2026 /* Cannot handle 64-bit instructions in 32-bit kernel */
2027 goto sigill;
2028
2029 case MIPS16e_ld_op:
2030loadDW:
2031#ifdef CONFIG_64BIT
2032 /*
2033 * A 32-bit kernel might be running on a 64-bit processor. But
2034 * if we're on a 32-bit processor and an i-cache incoherency
2035 * or race makes us see a 64-bit instruction here the sdl/sdr
2036 * would blow up, so for now we don't handle unaligned 64-bit
2037 * instructions on 32-bit kernels.
2038 */
2039 if (!access_ok(VERIFY_READ, addr, 8))
2040 goto sigbus;
2041
2042 LoadDW(addr, value, res);
2043 if (res)
2044 goto fault;
2045 MIPS16e_compute_return_epc(regs, &oldinst);
2046 regs->regs[reg] = value;
2047 break;
2048#endif /* CONFIG_64BIT */
2049
2050 /* Cannot handle 64-bit instructions in 32-bit kernel */
2051 goto sigill;
2052
2053 case MIPS16e_sh_op:
2054 if (!access_ok(VERIFY_WRITE, addr, 2))
2055 goto sigbus;
2056
2057 MIPS16e_compute_return_epc(regs, &oldinst);
2058 value = regs->regs[reg];
2059 StoreHW(addr, value, res);
2060 if (res)
2061 goto fault;
2062 break;
2063
2064 case MIPS16e_sw_op:
2065 case MIPS16e_swsp_op:
2066 case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */
2067 if (!access_ok(VERIFY_WRITE, addr, 4))
2068 goto sigbus;
2069
2070 MIPS16e_compute_return_epc(regs, &oldinst);
2071 value = regs->regs[reg];
2072 StoreW(addr, value, res);
2073 if (res)
2074 goto fault;
2075 break;
2076
2077 case MIPS16e_sd_op:
2078writeDW:
2079#ifdef CONFIG_64BIT
2080 /*
2081 * A 32-bit kernel might be running on a 64-bit processor. But
2082 * if we're on a 32-bit processor and an i-cache incoherency
2083 * or race makes us see a 64-bit instruction here the sdl/sdr
2084 * would blow up, so for now we don't handle unaligned 64-bit
2085 * instructions on 32-bit kernels.
2086 */
2087 if (!access_ok(VERIFY_WRITE, addr, 8))
2088 goto sigbus;
2089
2090 MIPS16e_compute_return_epc(regs, &oldinst);
2091 value = regs->regs[reg];
2092 StoreDW(addr, value, res);
2093 if (res)
2094 goto fault;
2095 break;
2096#endif /* CONFIG_64BIT */
2097
2098 /* Cannot handle 64-bit instructions in 32-bit kernel */
2099 goto sigill;
2100
2101 default:
2102 /*
2103 * Pheeee... We encountered an yet unknown instruction or
2104 * cache coherence problem. Die sucker, die ...
2105 */
2106 goto sigill;
2107 }
2108
2109#ifdef CONFIG_DEBUG_FS
2110 unaligned_instructions++;
2111#endif
2112
2113 return;
2114
2115fault:
2116 /* roll back jump/branch */
2117 regs->cp0_epc = origpc;
2118 regs->regs[31] = orig31;
2119 /* Did we have an exception handler installed? */
2120 if (fixup_exception(regs))
2121 return;
2122
2123 die_if_kernel("Unhandled kernel unaligned access", regs);
2124 force_sig(SIGSEGV, current);
2125
2126 return;
2127
2128sigbus:
2129 die_if_kernel("Unhandled kernel unaligned access", regs);
2130 force_sig(SIGBUS, current);
2131
2132 return;
2133
2134sigill:
2135 die_if_kernel
2136 ("Unhandled kernel unaligned access or invalid instruction", regs);
2137 force_sig(SIGILL, current);
2138}
fc192e50 2139
1da177e4
LT		 2140asmlinkage void do_ade(struct pt_regs *regs)
2141{
c3fc5cd5 2142 enum ctx_state prev_state;
fe00f943 2143 unsigned int __user *pc;
1da177e4 2144 mm_segment_t seg;
1da177e4 2145
c3fc5cd5 2146 prev_state = exception_enter();
7f788d2d 2147 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS,
a8b0ca17 2148 1, regs, regs->cp0_badvaddr);
1da177e4
LT		 2149 /*
2150 * Did we catch a fault trying to load an instruction?
1da177e4 2151 */
34c2f668 2152 if (regs->cp0_badvaddr == regs->cp0_epc)
1da177e4
LT		 2153 goto sigbus;
2154
293c5bd1 2155 if (user_mode(regs) && !test_thread_flag(TIF_FIXADE))
1da177e4 2156 goto sigbus;
6312e0ee
AN		 2157 if (unaligned_action == UNALIGNED_ACTION_SIGNAL)
2158 goto sigbus;
1da177e4
LT		 2159
2160 /*
2161 * Do branch emulation only if we didn't forward the exception.
2162 * This is all so but ugly ...
2163 */
34c2f668
LY		 2164
2165 /*
2166 * Are we running in microMIPS mode?
2167 */
2168 if (get_isa16_mode(regs->cp0_epc)) {
2169 /*
2170 * Did we catch a fault trying to load an instruction in
2171 * 16-bit mode?
2172 */
2173 if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc))
2174 goto sigbus;
2175 if (unaligned_action == UNALIGNED_ACTION_SHOW)
2176 show_registers(regs);
2177
2178 if (cpu_has_mmips) {
2179 seg = get_fs();
2180 if (!user_mode(regs))
2181 set_fs(KERNEL_DS);
2182 emulate_load_store_microMIPS(regs,
2183 (void __user *)regs->cp0_badvaddr);
2184 set_fs(seg);
2185
2186 return;
2187 }
2188
451b001b
SH		 2189 if (cpu_has_mips16) {
2190 seg = get_fs();
2191 if (!user_mode(regs))
2192 set_fs(KERNEL_DS);
2193 emulate_load_store_MIPS16e(regs,
2194 (void __user *)regs->cp0_badvaddr);
2195 set_fs(seg);
2196
2197 return;
2198 }
2199
34c2f668
LY		 2200 goto sigbus;
2201 }
2202
2203 if (unaligned_action == UNALIGNED_ACTION_SHOW)
2204 show_registers(regs);
2205 pc = (unsigned int __user *)exception_epc(regs);
2206
1da177e4
LT		 2207 seg = get_fs();
2208 if (!user_mode(regs))
2209 set_fs(KERNEL_DS);
7f18f151 2210 emulate_load_store_insn(regs, (void __user *)regs->cp0_badvaddr, pc);
1da177e4
LT		 2211 set_fs(seg);
2212
2213 return;
2214
2215sigbus:
2216 die_if_kernel("Kernel unaligned instruction access", regs);
2217 force_sig(SIGBUS, current);
2218
2219 /*
2220 * XXX On return from the signal handler we should advance the epc
2221 */
c3fc5cd5 2222 exception_exit(prev_state);
1da177e4 2223}
6312e0ee
AN		 2224
2225#ifdef CONFIG_DEBUG_FS
2226extern struct dentry *mips_debugfs_dir;
2227static int __init debugfs_unaligned(void)
2228{
2229 struct dentry *d;
2230
2231 if (!mips_debugfs_dir)
2232 return -ENODEV;
2233 d = debugfs_create_u32("unaligned_instructions", S_IRUGO,
2234 mips_debugfs_dir, &unaligned_instructions);
b517531c
Z		 2235 if (!d)
2236 return -ENOMEM;
6312e0ee
AN		 2237 d = debugfs_create_u32("unaligned_action", S_IRUGO | S_IWUSR,
2238 mips_debugfs_dir, &unaligned_action);
b517531c
Z		 2239 if (!d)
2240 return -ENOMEM;
6312e0ee
AN		 2241 return 0;
2242}
2243__initcall(debugfs_unaligned);
2244#endif
Linux 6.x block layer and io_uring tree(s)