projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[PATCH] Notifier chain update: API changes
[linux-2.6-block.git] / arch / sparc64 / kernel / traps.c
CommitLineData
1da177e4
LT		 1/* $Id: traps.c,v 1.85 2002/02/09 19:49:31 davem Exp $
2 * arch/sparc64/kernel/traps.c
3 *
4 * Copyright (C) 1995,1997 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
6 */
7
8/*
9 * I like traps on v9, :))))
10 */
11
12#include <linux/config.h>
13#include <linux/module.h>
14#include <linux/sched.h> /* for jiffies */
15#include <linux/kernel.h>
16#include <linux/kallsyms.h>
17#include <linux/signal.h>
18#include <linux/smp.h>
19#include <linux/smp_lock.h>
20#include <linux/mm.h>
21#include <linux/init.h>
22
23#include <asm/delay.h>
24#include <asm/system.h>
25#include <asm/ptrace.h>
26#include <asm/oplib.h>
27#include <asm/page.h>
28#include <asm/pgtable.h>
29#include <asm/unistd.h>
30#include <asm/uaccess.h>
31#include <asm/fpumacro.h>
32#include <asm/lsu.h>
33#include <asm/dcu.h>
34#include <asm/estate.h>
35#include <asm/chafsr.h>
6c52a96e 36#include <asm/sfafsr.h>
1da177e4
LT		 37#include <asm/psrcompat.h>
38#include <asm/processor.h>
39#include <asm/timer.h>
40#include <asm/kdebug.h>
92704a1c 41#include <asm/head.h>
1da177e4
LT		 42#ifdef CONFIG_KMOD
43#include <linux/kmod.h>
44#endif
45
e041c683 46ATOMIC_NOTIFIER_HEAD(sparc64die_chain);
1da177e4
LT		 47
48int register_die_notifier(struct notifier_block *nb)
49{
e041c683 50 return atomic_notifier_chain_register(&sparc64die_chain, nb);
1da177e4 51}
e041c683
AS		 52EXPORT_SYMBOL(register_die_notifier);
53
54int unregister_die_notifier(struct notifier_block *nb)
55{
56 return atomic_notifier_chain_unregister(&sparc64die_chain, nb);
57}
58EXPORT_SYMBOL(unregister_die_notifier);
1da177e4
LT		 59
60/* When an irrecoverable trap occurs at tl > 0, the trap entry
61 * code logs the trap state registers at every level in the trap
62 * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout
63 * is as follows:
64 */
65struct tl1_traplog {
66 struct {
67 unsigned long tstate;
68 unsigned long tpc;
69 unsigned long tnpc;
70 unsigned long tt;
71 } trapstack[4];
72 unsigned long tl;
73};
74
75static void dump_tl1_traplog(struct tl1_traplog *p)
76{
3d6395cb 77 int i, limit;
1da177e4 78
04d74758
DM		 79 printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
80 "dumping track stack.\n", p->tl);
3d6395cb
DM		 81
82 limit = (tlb_type == hypervisor) ? 2 : 4;
39334a4b 83 for (i = 0; i < limit; i++) {
04d74758 84 printk(KERN_EMERG
1da177e4
LT		 85 "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
86 "TNPC[%016lx] TT[%lx]\n",
87 i + 1,
88 p->trapstack[i].tstate, p->trapstack[i].tpc,
89 p->trapstack[i].tnpc, p->trapstack[i].tt);
90 }
91}
92
93void do_call_debug(struct pt_regs *regs)
94{
95 notify_die(DIE_CALL, "debug call", regs, 0, 255, SIGINT);
96}
97
98void bad_trap(struct pt_regs *regs, long lvl)
99{
100 char buffer[32];
101 siginfo_t info;
102
103 if (notify_die(DIE_TRAP, "bad trap", regs,
104 0, lvl, SIGTRAP) == NOTIFY_STOP)
105 return;
106
107 if (lvl < 0x100) {
108 sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
109 die_if_kernel(buffer, regs);
110 }
111
112 lvl -= 0x100;
113 if (regs->tstate & TSTATE_PRIV) {
114 sprintf(buffer, "Kernel bad sw trap %lx", lvl);
115 die_if_kernel(buffer, regs);
116 }
117 if (test_thread_flag(TIF_32BIT)) {
118 regs->tpc &= 0xffffffff;
119 regs->tnpc &= 0xffffffff;
120 }
121 info.si_signo = SIGILL;
122 info.si_errno = 0;
123 info.si_code = ILL_ILLTRP;
124 info.si_addr = (void __user *)regs->tpc;
125 info.si_trapno = lvl;
126 force_sig_info(SIGILL, &info, current);
127}
128
129void bad_trap_tl1(struct pt_regs *regs, long lvl)
130{
131 char buffer[32];
132
133 if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
134 0, lvl, SIGTRAP) == NOTIFY_STOP)
135 return;
136
137 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
138
139 sprintf (buffer, "Bad trap %lx at tl>0", lvl);
140 die_if_kernel (buffer, regs);
141}
142
143#ifdef CONFIG_DEBUG_BUGVERBOSE
144void do_BUG(const char *file, int line)
145{
146 bust_spinlocks(1);
147 printk("kernel BUG at %s:%d!\n", file, line);
148}
149#endif
150
6c52a96e 151void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
1da177e4
LT		 152{
153 siginfo_t info;
154
155 if (notify_die(DIE_TRAP, "instruction access exception", regs,
156 0, 0x8, SIGTRAP) == NOTIFY_STOP)
157 return;
158
159 if (regs->tstate & TSTATE_PRIV) {
6c52a96e
DM		 160 printk("spitfire_insn_access_exception: SFSR[%016lx] "
161 "SFAR[%016lx], going.\n", sfsr, sfar);
1da177e4
LT		 162 die_if_kernel("Iax", regs);
163 }
164 if (test_thread_flag(TIF_32BIT)) {
165 regs->tpc &= 0xffffffff;
166 regs->tnpc &= 0xffffffff;
167 }
168 info.si_signo = SIGSEGV;
169 info.si_errno = 0;
170 info.si_code = SEGV_MAPERR;
171 info.si_addr = (void __user *)regs->tpc;
172 info.si_trapno = 0;
173 force_sig_info(SIGSEGV, &info, current);
174}
175
6c52a96e 176void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
1da177e4
LT		 177{
178 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
179 0, 0x8, SIGTRAP) == NOTIFY_STOP)
180 return;
181
182 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
6c52a96e 183 spitfire_insn_access_exception(regs, sfsr, sfar);
1da177e4
LT		 184}
185
ed6b0b45
DM		 186void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
187{
188 unsigned short type = (type_ctx >> 16);
189 unsigned short ctx = (type_ctx & 0xffff);
190 siginfo_t info;
191
192 if (notify_die(DIE_TRAP, "instruction access exception", regs,
193 0, 0x8, SIGTRAP) == NOTIFY_STOP)
194 return;
195
196 if (regs->tstate & TSTATE_PRIV) {
197 printk("sun4v_insn_access_exception: ADDR[%016lx] "
198 "CTX[%04x] TYPE[%04x], going.\n",
199 addr, ctx, type);
200 die_if_kernel("Iax", regs);
201 }
202
203 if (test_thread_flag(TIF_32BIT)) {
204 regs->tpc &= 0xffffffff;
205 regs->tnpc &= 0xffffffff;
206 }
207 info.si_signo = SIGSEGV;
208 info.si_errno = 0;
209 info.si_code = SEGV_MAPERR;
210 info.si_addr = (void __user *) addr;
211 info.si_trapno = 0;
212 force_sig_info(SIGSEGV, &info, current);
213}
214
215void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
216{
217 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
218 0, 0x8, SIGTRAP) == NOTIFY_STOP)
219 return;
220
221 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
222 sun4v_insn_access_exception(regs, addr, type_ctx);
223}
224
6c52a96e 225void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
1da177e4
LT		 226{
227 siginfo_t info;
228
229 if (notify_die(DIE_TRAP, "data access exception", regs,
230 0, 0x30, SIGTRAP) == NOTIFY_STOP)
231 return;
232
233 if (regs->tstate & TSTATE_PRIV) {
234 /* Test if this comes from uaccess places. */
8cf14af0 235 const struct exception_table_entry *entry;
1da177e4 236
8cf14af0
DM		 237 entry = search_exception_tables(regs->tpc);
238 if (entry) {
239 /* Ouch, somebody is trying VM hole tricks on us... */
1da177e4
LT		 240#ifdef DEBUG_EXCEPTIONS
241 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
8cf14af0
DM		 242 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
243 regs->tpc, entry->fixup);
1da177e4 244#endif
8cf14af0 245 regs->tpc = entry->fixup;
1da177e4 246 regs->tnpc = regs->tpc + 4;
1da177e4
LT		 247 return;
248 }
249 /* Shit... */
6c52a96e
DM		 250 printk("spitfire_data_access_exception: SFSR[%016lx] "
251 "SFAR[%016lx], going.\n", sfsr, sfar);
1da177e4
LT		 252 die_if_kernel("Dax", regs);
253 }
254
255 info.si_signo = SIGSEGV;
256 info.si_errno = 0;
257 info.si_code = SEGV_MAPERR;
258 info.si_addr = (void __user *)sfar;
259 info.si_trapno = 0;
260 force_sig_info(SIGSEGV, &info, current);
261}
262
6c52a96e 263void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
bde4e4ee
DM		 264{
265 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
266 0, 0x30, SIGTRAP) == NOTIFY_STOP)
267 return;
268
269 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
6c52a96e 270 spitfire_data_access_exception(regs, sfsr, sfar);
bde4e4ee
DM		 271}
272
ed6b0b45
DM		 273void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
274{
275 unsigned short type = (type_ctx >> 16);
276 unsigned short ctx = (type_ctx & 0xffff);
277 siginfo_t info;
278
279 if (notify_die(DIE_TRAP, "data access exception", regs,
280 0, 0x8, SIGTRAP) == NOTIFY_STOP)
281 return;
282
283 if (regs->tstate & TSTATE_PRIV) {
284 printk("sun4v_data_access_exception: ADDR[%016lx] "
285 "CTX[%04x] TYPE[%04x], going.\n",
286 addr, ctx, type);
55555633 287 die_if_kernel("Dax", regs);
ed6b0b45
DM		 288 }
289
290 if (test_thread_flag(TIF_32BIT)) {
291 regs->tpc &= 0xffffffff;
292 regs->tnpc &= 0xffffffff;
293 }
294 info.si_signo = SIGSEGV;
295 info.si_errno = 0;
296 info.si_code = SEGV_MAPERR;
297 info.si_addr = (void __user *) addr;
298 info.si_trapno = 0;
299 force_sig_info(SIGSEGV, &info, current);
300}
301
302void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
303{
304 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
305 0, 0x8, SIGTRAP) == NOTIFY_STOP)
306 return;
307
308 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
309 sun4v_data_access_exception(regs, addr, type_ctx);
310}
311
1da177e4
LT		 312#ifdef CONFIG_PCI
313/* This is really pathetic... */
314extern volatile int pci_poke_in_progress;
315extern volatile int pci_poke_cpu;
316extern volatile int pci_poke_faulted;
317#endif
318
319/* When access exceptions happen, we must do this. */
320static void spitfire_clean_and_reenable_l1_caches(void)
321{
322 unsigned long va;
323
324 if (tlb_type != spitfire)
325 BUG();
326
327 /* Clean 'em. */
328 for (va = 0; va < (PAGE_SIZE << 1); va += 32) {
329 spitfire_put_icache_tag(va, 0x0);
330 spitfire_put_dcache_tag(va, 0x0);
331 }
332
333 /* Re-enable in LSU. */
334 __asm__ __volatile__("flush %%g6\n\t"
335 "membar #Sync\n\t"
336 "stxa %0, [%%g0] %1\n\t"
337 "membar #Sync"
338 : /* no outputs */
339 : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
340 LSU_CONTROL_IM | LSU_CONTROL_DM),
341 "i" (ASI_LSU_CONTROL)
342 : "memory");
343}
344
6c52a96e 345static void spitfire_enable_estate_errors(void)
1da177e4 346{
6c52a96e
DM		 347 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
348 "membar #Sync"
349 : /* no outputs */
350 : "r" (ESTATE_ERR_ALL),
351 "i" (ASI_ESTATE_ERROR_EN));
1da177e4
LT		 352}
353
354static char ecc_syndrome_table[] = {
355 0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
356 0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
357 0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
358 0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
359 0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
360 0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
361 0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
362 0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
363 0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
364 0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
365 0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
366 0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
367 0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
368 0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
369 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
370 0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
371 0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
372 0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
373 0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
374 0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
375 0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
376 0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
377 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
378 0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
379 0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
380 0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
381 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
382 0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
383 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
384 0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
385 0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
386 0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
387};
388
1da177e4
LT		 389static char *syndrome_unknown = "<Unknown>";
390
6c52a96e 391static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
1da177e4 392{
6c52a96e
DM		 393 unsigned short scode;
394 char memmod_str[64], *p;
1da177e4 395
6c52a96e
DM		 396 if (udbl & bit) {
397 scode = ecc_syndrome_table[udbl & 0xff];
1da177e4
LT		 398 if (prom_getunumber(scode, afar,
399 memmod_str, sizeof(memmod_str)) == -1)
400 p = syndrome_unknown;
401 else
402 p = memmod_str;
403 printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
404 "Memory Module \"%s\"\n",
405 smp_processor_id(), scode, p);
406 }
407
6c52a96e
DM		 408 if (udbh & bit) {
409 scode = ecc_syndrome_table[udbh & 0xff];
1da177e4
LT		 410 if (prom_getunumber(scode, afar,
411 memmod_str, sizeof(memmod_str)) == -1)
412 p = syndrome_unknown;
413 else
414 p = memmod_str;
415 printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
416 "Memory Module \"%s\"\n",
417 smp_processor_id(), scode, p);
418 }
6c52a96e
DM		 419
420}
421
422static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
423{
424
425 printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
426 "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
427 smp_processor_id(), afsr, afar, udbl, udbh, tl1);
428
429 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
430
431 /* We always log it, even if someone is listening for this
432 * trap.
433 */
434 notify_die(DIE_TRAP, "Correctable ECC Error", regs,
435 0, TRAP_TYPE_CEE, SIGTRAP);
436
437 /* The Correctable ECC Error trap does not disable I/D caches. So
438 * we only have to restore the ESTATE Error Enable register.
439 */
440 spitfire_enable_estate_errors();
441}
442
443static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
444{
445 siginfo_t info;
446
447 printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
448 "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
449 smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
450
451 /* XXX add more human friendly logging of the error status
452 * XXX as is implemented for cheetah
453 */
454
455 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
456
457 /* We always log it, even if someone is listening for this
458 * trap.
459 */
460 notify_die(DIE_TRAP, "Uncorrectable Error", regs,
461 0, tt, SIGTRAP);
462
463 if (regs->tstate & TSTATE_PRIV) {
464 if (tl1)
465 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
466 die_if_kernel("UE", regs);
467 }
468
469 /* XXX need more intelligent processing here, such as is implemented
470 * XXX for cheetah errors, in fact if the E-cache still holds the
471 * XXX line with bad parity this will loop
472 */
473
474 spitfire_clean_and_reenable_l1_caches();
475 spitfire_enable_estate_errors();
476
477 if (test_thread_flag(TIF_32BIT)) {
478 regs->tpc &= 0xffffffff;
479 regs->tnpc &= 0xffffffff;
480 }
481 info.si_signo = SIGBUS;
482 info.si_errno = 0;
483 info.si_code = BUS_OBJERR;
484 info.si_addr = (void *)0;
485 info.si_trapno = 0;
486 force_sig_info(SIGBUS, &info, current);
487}
488
489void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
490{
491 unsigned long afsr, tt, udbh, udbl;
492 int tl1;
493
494 afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
495 tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
496 tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
497 udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
498 udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
499
500#ifdef CONFIG_PCI
501 if (tt == TRAP_TYPE_DAE &&
502 pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
503 spitfire_clean_and_reenable_l1_caches();
504 spitfire_enable_estate_errors();
505
506 pci_poke_faulted = 1;
507 regs->tnpc = regs->tpc + 4;
508 return;
509 }
510#endif
511
512 if (afsr & SFAFSR_UE)
513 spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
514
515 if (tt == TRAP_TYPE_CEE) {
516 /* Handle the case where we took a CEE trap, but ACK'd
517 * only the UE state in the UDB error registers.
518 */
519 if (afsr & SFAFSR_UE) {
520 if (udbh & UDBE_CE) {
521 __asm__ __volatile__(
522 "stxa %0, [%1] %2\n\t"
523 "membar #Sync"
524 : /* no outputs */
525 : "r" (udbh & UDBE_CE),
526 "r" (0x0), "i" (ASI_UDB_ERROR_W));
527 }
528 if (udbl & UDBE_CE) {
529 __asm__ __volatile__(
530 "stxa %0, [%1] %2\n\t"
531 "membar #Sync"
532 : /* no outputs */
533 : "r" (udbl & UDBE_CE),
534 "r" (0x18), "i" (ASI_UDB_ERROR_W));
535 }
536 }
537
538 spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
539 }
1da177e4
LT		 540}
541
816242da
DM		 542int cheetah_pcache_forced_on;
543
544void cheetah_enable_pcache(void)
545{
546 unsigned long dcr;
547
548 printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
549 smp_processor_id());
550
551 __asm__ __volatile__("ldxa [%%g0] %1, %0"
552 : "=r" (dcr)
553 : "i" (ASI_DCU_CONTROL_REG));
554 dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
555 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
556 "membar #Sync"
557 : /* no outputs */
558 : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
559}
560
1da177e4
LT		 561/* Cheetah error trap handling. */
562static unsigned long ecache_flush_physbase;
563static unsigned long ecache_flush_linesize;
564static unsigned long ecache_flush_size;
565
566/* WARNING: The error trap handlers in assembly know the precise
567 * layout of the following structure.
568 *
569 * C-level handlers below use this information to log the error
570 * and then determine how to recover (if possible).
571 */
572struct cheetah_err_info {
573/*0x00*/u64 afsr;
574/*0x08*/u64 afar;
575
576 /* D-cache state */
577/*0x10*/u64 dcache_data[4]; /* The actual data */
578/*0x30*/u64 dcache_index; /* D-cache index */
579/*0x38*/u64 dcache_tag; /* D-cache tag/valid */
580/*0x40*/u64 dcache_utag; /* D-cache microtag */
581/*0x48*/u64 dcache_stag; /* D-cache snooptag */
582
583 /* I-cache state */
584/*0x50*/u64 icache_data[8]; /* The actual insns + predecode */
585/*0x90*/u64 icache_index; /* I-cache index */
586/*0x98*/u64 icache_tag; /* I-cache phys tag */
587/*0xa0*/u64 icache_utag; /* I-cache microtag */
588/*0xa8*/u64 icache_stag; /* I-cache snooptag */
589/*0xb0*/u64 icache_upper; /* I-cache upper-tag */
590/*0xb8*/u64 icache_lower; /* I-cache lower-tag */
591
592 /* E-cache state */
593/*0xc0*/u64 ecache_data[4]; /* 32 bytes from staging registers */
594/*0xe0*/u64 ecache_index; /* E-cache index */
595/*0xe8*/u64 ecache_tag; /* E-cache tag/state */
596
597/*0xf0*/u64 __pad[32 - 30];
598};
599#define CHAFSR_INVALID ((u64)-1L)
600
601/* This table is ordered in priority of errors and matches the
602 * AFAR overwrite policy as well.
603 */
604
605struct afsr_error_table {
606 unsigned long mask;
607 const char *name;
608};
609
610static const char CHAFSR_PERR_msg[] =
611 "System interface protocol error";
612static const char CHAFSR_IERR_msg[] =
613 "Internal processor error";
614static const char CHAFSR_ISAP_msg[] =
615 "System request parity error on incoming addresss";
616static const char CHAFSR_UCU_msg[] =
617 "Uncorrectable E-cache ECC error for ifetch/data";
618static const char CHAFSR_UCC_msg[] =
619 "SW Correctable E-cache ECC error for ifetch/data";
620static const char CHAFSR_UE_msg[] =
621 "Uncorrectable system bus data ECC error for read";
622static const char CHAFSR_EDU_msg[] =
623 "Uncorrectable E-cache ECC error for stmerge/blkld";
624static const char CHAFSR_EMU_msg[] =
625 "Uncorrectable system bus MTAG error";
626static const char CHAFSR_WDU_msg[] =
627 "Uncorrectable E-cache ECC error for writeback";
628static const char CHAFSR_CPU_msg[] =
629 "Uncorrectable ECC error for copyout";
630static const char CHAFSR_CE_msg[] =
631 "HW corrected system bus data ECC error for read";
632static const char CHAFSR_EDC_msg[] =
633 "HW corrected E-cache ECC error for stmerge/blkld";
634static const char CHAFSR_EMC_msg[] =
635 "HW corrected system bus MTAG ECC error";
636static const char CHAFSR_WDC_msg[] =
637 "HW corrected E-cache ECC error for writeback";
638static const char CHAFSR_CPC_msg[] =
639 "HW corrected ECC error for copyout";
640static const char CHAFSR_TO_msg[] =
641 "Unmapped error from system bus";
642static const char CHAFSR_BERR_msg[] =
643 "Bus error response from system bus";
644static const char CHAFSR_IVC_msg[] =
645 "HW corrected system bus data ECC error for ivec read";
646static const char CHAFSR_IVU_msg[] =
647 "Uncorrectable system bus data ECC error for ivec read";
648static struct afsr_error_table __cheetah_error_table[] = {
649 { CHAFSR_PERR, CHAFSR_PERR_msg },
650 { CHAFSR_IERR, CHAFSR_IERR_msg },
651 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
652 { CHAFSR_UCU, CHAFSR_UCU_msg },
653 { CHAFSR_UCC, CHAFSR_UCC_msg },
654 { CHAFSR_UE, CHAFSR_UE_msg },
655 { CHAFSR_EDU, CHAFSR_EDU_msg },
656 { CHAFSR_EMU, CHAFSR_EMU_msg },
657 { CHAFSR_WDU, CHAFSR_WDU_msg },
658 { CHAFSR_CPU, CHAFSR_CPU_msg },
659 { CHAFSR_CE, CHAFSR_CE_msg },
660 { CHAFSR_EDC, CHAFSR_EDC_msg },
661 { CHAFSR_EMC, CHAFSR_EMC_msg },
662 { CHAFSR_WDC, CHAFSR_WDC_msg },
663 { CHAFSR_CPC, CHAFSR_CPC_msg },
664 { CHAFSR_TO, CHAFSR_TO_msg },
665 { CHAFSR_BERR, CHAFSR_BERR_msg },
666 /* These two do not update the AFAR. */
667 { CHAFSR_IVC, CHAFSR_IVC_msg },
668 { CHAFSR_IVU, CHAFSR_IVU_msg },
669 { 0, NULL },
670};
671static const char CHPAFSR_DTO_msg[] =
672 "System bus unmapped error for prefetch/storequeue-read";
673static const char CHPAFSR_DBERR_msg[] =
674 "System bus error for prefetch/storequeue-read";
675static const char CHPAFSR_THCE_msg[] =
676 "Hardware corrected E-cache Tag ECC error";
677static const char CHPAFSR_TSCE_msg[] =
678 "SW handled correctable E-cache Tag ECC error";
679static const char CHPAFSR_TUE_msg[] =
680 "Uncorrectable E-cache Tag ECC error";
681static const char CHPAFSR_DUE_msg[] =
682 "System bus uncorrectable data ECC error due to prefetch/store-fill";
683static struct afsr_error_table __cheetah_plus_error_table[] = {
684 { CHAFSR_PERR, CHAFSR_PERR_msg },
685 { CHAFSR_IERR, CHAFSR_IERR_msg },
686 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
687 { CHAFSR_UCU, CHAFSR_UCU_msg },
688 { CHAFSR_UCC, CHAFSR_UCC_msg },
689 { CHAFSR_UE, CHAFSR_UE_msg },
690 { CHAFSR_EDU, CHAFSR_EDU_msg },
691 { CHAFSR_EMU, CHAFSR_EMU_msg },
692 { CHAFSR_WDU, CHAFSR_WDU_msg },
693 { CHAFSR_CPU, CHAFSR_CPU_msg },
694 { CHAFSR_CE, CHAFSR_CE_msg },
695 { CHAFSR_EDC, CHAFSR_EDC_msg },
696 { CHAFSR_EMC, CHAFSR_EMC_msg },
697 { CHAFSR_WDC, CHAFSR_WDC_msg },
698 { CHAFSR_CPC, CHAFSR_CPC_msg },
699 { CHAFSR_TO, CHAFSR_TO_msg },
700 { CHAFSR_BERR, CHAFSR_BERR_msg },
701 { CHPAFSR_DTO, CHPAFSR_DTO_msg },
702 { CHPAFSR_DBERR, CHPAFSR_DBERR_msg },
703 { CHPAFSR_THCE, CHPAFSR_THCE_msg },
704 { CHPAFSR_TSCE, CHPAFSR_TSCE_msg },
705 { CHPAFSR_TUE, CHPAFSR_TUE_msg },
706 { CHPAFSR_DUE, CHPAFSR_DUE_msg },
707 /* These two do not update the AFAR. */
708 { CHAFSR_IVC, CHAFSR_IVC_msg },
709 { CHAFSR_IVU, CHAFSR_IVU_msg },
710 { 0, NULL },
711};
712static const char JPAFSR_JETO_msg[] =
713 "System interface protocol error, hw timeout caused";
714static const char JPAFSR_SCE_msg[] =
715 "Parity error on system snoop results";
716static const char JPAFSR_JEIC_msg[] =
717 "System interface protocol error, illegal command detected";
718static const char JPAFSR_JEIT_msg[] =
719 "System interface protocol error, illegal ADTYPE detected";
720static const char JPAFSR_OM_msg[] =
721 "Out of range memory error has occurred";
722static const char JPAFSR_ETP_msg[] =
723 "Parity error on L2 cache tag SRAM";
724static const char JPAFSR_UMS_msg[] =
725 "Error due to unsupported store";
726static const char JPAFSR_RUE_msg[] =
727 "Uncorrectable ECC error from remote cache/memory";
728static const char JPAFSR_RCE_msg[] =
729 "Correctable ECC error from remote cache/memory";
730static const char JPAFSR_BP_msg[] =
731 "JBUS parity error on returned read data";
732static const char JPAFSR_WBP_msg[] =
733 "JBUS parity error on data for writeback or block store";
734static const char JPAFSR_FRC_msg[] =
735 "Foreign read to DRAM incurring correctable ECC error";
736static const char JPAFSR_FRU_msg[] =
737 "Foreign read to DRAM incurring uncorrectable ECC error";
738static struct afsr_error_table __jalapeno_error_table[] = {
739 { JPAFSR_JETO, JPAFSR_JETO_msg },
740 { JPAFSR_SCE, JPAFSR_SCE_msg },
741 { JPAFSR_JEIC, JPAFSR_JEIC_msg },
742 { JPAFSR_JEIT, JPAFSR_JEIT_msg },
743 { CHAFSR_PERR, CHAFSR_PERR_msg },
744 { CHAFSR_IERR, CHAFSR_IERR_msg },
745 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
746 { CHAFSR_UCU, CHAFSR_UCU_msg },
747 { CHAFSR_UCC, CHAFSR_UCC_msg },
748 { CHAFSR_UE, CHAFSR_UE_msg },
749 { CHAFSR_EDU, CHAFSR_EDU_msg },
750 { JPAFSR_OM, JPAFSR_OM_msg },
751 { CHAFSR_WDU, CHAFSR_WDU_msg },
752 { CHAFSR_CPU, CHAFSR_CPU_msg },
753 { CHAFSR_CE, CHAFSR_CE_msg },
754 { CHAFSR_EDC, CHAFSR_EDC_msg },
755 { JPAFSR_ETP, JPAFSR_ETP_msg },
756 { CHAFSR_WDC, CHAFSR_WDC_msg },
757 { CHAFSR_CPC, CHAFSR_CPC_msg },
758 { CHAFSR_TO, CHAFSR_TO_msg },
759 { CHAFSR_BERR, CHAFSR_BERR_msg },
760 { JPAFSR_UMS, JPAFSR_UMS_msg },
761 { JPAFSR_RUE, JPAFSR_RUE_msg },
762 { JPAFSR_RCE, JPAFSR_RCE_msg },
763 { JPAFSR_BP, JPAFSR_BP_msg },
764 { JPAFSR_WBP, JPAFSR_WBP_msg },
765 { JPAFSR_FRC, JPAFSR_FRC_msg },
766 { JPAFSR_FRU, JPAFSR_FRU_msg },
767 /* These two do not update the AFAR. */
768 { CHAFSR_IVU, CHAFSR_IVU_msg },
769 { 0, NULL },
770};
771static struct afsr_error_table *cheetah_error_table;
772static unsigned long cheetah_afsr_errors;
773
774/* This is allocated at boot time based upon the largest hardware
775 * cpu ID in the system. We allocate two entries per cpu, one for
776 * TL==0 logging and one for TL >= 1 logging.
777 */
778struct cheetah_err_info *cheetah_error_log;
779
780static __inline__ struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
781{
782 struct cheetah_err_info *p;
783 int cpu = smp_processor_id();
784
785 if (!cheetah_error_log)
786 return NULL;
787
788 p = cheetah_error_log + (cpu * 2);
789 if ((afsr & CHAFSR_TL1) != 0UL)
790 p++;
791
792 return p;
793}
794
795extern unsigned int tl0_icpe[], tl1_icpe[];
796extern unsigned int tl0_dcpe[], tl1_dcpe[];
797extern unsigned int tl0_fecc[], tl1_fecc[];
798extern unsigned int tl0_cee[], tl1_cee[];
799extern unsigned int tl0_iae[], tl1_iae[];
800extern unsigned int tl0_dae[], tl1_dae[];
801extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
802extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
803extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
804extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
805extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
806
807void __init cheetah_ecache_flush_init(void)
808{
809 unsigned long largest_size, smallest_linesize, order, ver;
810 int node, i, instance;
811
812 /* Scan all cpu device tree nodes, note two values:
813 * 1) largest E-cache size
814 * 2) smallest E-cache line size
815 */
816 largest_size = 0UL;
817 smallest_linesize = ~0UL;
818
819 instance = 0;
820 while (!cpu_find_by_instance(instance, &node, NULL)) {
821 unsigned long val;
822
823 val = prom_getintdefault(node, "ecache-size",
824 (2 * 1024 * 1024));
825 if (val > largest_size)
826 largest_size = val;
827 val = prom_getintdefault(node, "ecache-line-size", 64);
828 if (val < smallest_linesize)
829 smallest_linesize = val;
830 instance++;
831 }
832
833 if (largest_size == 0UL || smallest_linesize == ~0UL) {
834 prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
835 "parameters.\n");
836 prom_halt();
837 }
838
839 ecache_flush_size = (2 * largest_size);
840 ecache_flush_linesize = smallest_linesize;
841
10147570 842 ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
1da177e4 843
10147570 844 if (ecache_flush_physbase == ~0UL) {
1da177e4 845 prom_printf("cheetah_ecache_flush_init: Cannot find %d byte "
10147570
DM		 846 "contiguous physical memory.\n",
847 ecache_flush_size);
1da177e4
LT		 848 prom_halt();
849 }
850
851 /* Now allocate error trap reporting scoreboard. */
852 node = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
853 for (order = 0; order < MAX_ORDER; order++) {
854 if ((PAGE_SIZE << order) >= node)
855 break;
856 }
857 cheetah_error_log = (struct cheetah_err_info *)
858 __get_free_pages(GFP_KERNEL, order);
859 if (!cheetah_error_log) {
860 prom_printf("cheetah_ecache_flush_init: Failed to allocate "
861 "error logging scoreboard (%d bytes).\n", node);
862 prom_halt();
863 }
864 memset(cheetah_error_log, 0, PAGE_SIZE << order);
865
866 /* Mark all AFSRs as invalid so that the trap handler will
867 * log new new information there.
868 */
869 for (i = 0; i < 2 * NR_CPUS; i++)
870 cheetah_error_log[i].afsr = CHAFSR_INVALID;
871
872 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
92704a1c
DM		 873 if ((ver >> 32) == __JALAPENO_ID ||
874 (ver >> 32) == __SERRANO_ID) {
1da177e4
LT		 875 cheetah_error_table = &__jalapeno_error_table[0];
876 cheetah_afsr_errors = JPAFSR_ERRORS;
877 } else if ((ver >> 32) == 0x003e0015) {
878 cheetah_error_table = &__cheetah_plus_error_table[0];
879 cheetah_afsr_errors = CHPAFSR_ERRORS;
880 } else {
881 cheetah_error_table = &__cheetah_error_table[0];
882 cheetah_afsr_errors = CHAFSR_ERRORS;
883 }
884
885 /* Now patch trap tables. */
886 memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
887 memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
888 memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
889 memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
890 memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
891 memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
892 memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
893 memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
894 if (tlb_type == cheetah_plus) {
895 memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
896 memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
897 memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
898 memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
899 }
900 flushi(PAGE_OFFSET);
901}
902
903static void cheetah_flush_ecache(void)
904{
905 unsigned long flush_base = ecache_flush_physbase;
906 unsigned long flush_linesize = ecache_flush_linesize;
907 unsigned long flush_size = ecache_flush_size;
908
909 __asm__ __volatile__("1: subcc %0, %4, %0\n\t"
910 " bne,pt %%xcc, 1b\n\t"
911 " ldxa [%2 + %0] %3, %%g0\n\t"
912 : "=&r" (flush_size)
913 : "0" (flush_size), "r" (flush_base),
914 "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
915}
916
917static void cheetah_flush_ecache_line(unsigned long physaddr)
918{
919 unsigned long alias;
920
921 physaddr &= ~(8UL - 1UL);
922 physaddr = (ecache_flush_physbase +
923 (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
924 alias = physaddr + (ecache_flush_size >> 1UL);
925 __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
926 "ldxa [%1] %2, %%g0\n\t"
927 "membar #Sync"
928 : /* no outputs */
929 : "r" (physaddr), "r" (alias),
930 "i" (ASI_PHYS_USE_EC));
931}
932
933/* Unfortunately, the diagnostic access to the I-cache tags we need to
934 * use to clear the thing interferes with I-cache coherency transactions.
935 *
936 * So we must only flush the I-cache when it is disabled.
937 */
938static void __cheetah_flush_icache(void)
939{
80dc0d6b
DM		 940 unsigned int icache_size, icache_line_size;
941 unsigned long addr;
942
943 icache_size = local_cpu_data().icache_size;
944 icache_line_size = local_cpu_data().icache_line_size;
1da177e4
LT		 945
946 /* Clear the valid bits in all the tags. */
80dc0d6b 947 for (addr = 0; addr < icache_size; addr += icache_line_size) {
1da177e4
LT		 948 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
949 "membar #Sync"
950 : /* no outputs */
80dc0d6b
DM		 951 : "r" (addr | (2 << 3)),
952 "i" (ASI_IC_TAG));
1da177e4
LT		 953 }
954}
955
956static void cheetah_flush_icache(void)
957{
958 unsigned long dcu_save;
959
960 /* Save current DCU, disable I-cache. */
961 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
962 "or %0, %2, %%g1\n\t"
963 "stxa %%g1, [%%g0] %1\n\t"
964 "membar #Sync"
965 : "=r" (dcu_save)
966 : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
967 : "g1");
968
969 __cheetah_flush_icache();
970
971 /* Restore DCU register */
972 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
973 "membar #Sync"
974 : /* no outputs */
975 : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
976}
977
978static void cheetah_flush_dcache(void)
979{
80dc0d6b
DM		 980 unsigned int dcache_size, dcache_line_size;
981 unsigned long addr;
982
983 dcache_size = local_cpu_data().dcache_size;
984 dcache_line_size = local_cpu_data().dcache_line_size;
1da177e4 985
80dc0d6b 986 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1da177e4
LT		 987 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
988 "membar #Sync"
989 : /* no outputs */
80dc0d6b 990 : "r" (addr), "i" (ASI_DCACHE_TAG));
1da177e4
LT		 991 }
992}
993
994/* In order to make the even parity correct we must do two things.
995 * First, we clear DC_data_parity and set DC_utag to an appropriate value.
996 * Next, we clear out all 32-bytes of data for that line. Data of
997 * all-zero + tag parity value of zero == correct parity.
998 */
999static void cheetah_plus_zap_dcache_parity(void)
1000{
80dc0d6b
DM		 1001 unsigned int dcache_size, dcache_line_size;
1002 unsigned long addr;
1003
1004 dcache_size = local_cpu_data().dcache_size;
1005 dcache_line_size = local_cpu_data().dcache_line_size;
1da177e4 1006
80dc0d6b
DM		 1007 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1008 unsigned long tag = (addr >> 14);
1009 unsigned long line;
1da177e4
LT		 1010
1011 __asm__ __volatile__("membar #Sync\n\t"
1012 "stxa %0, [%1] %2\n\t"
1013 "membar #Sync"
1014 : /* no outputs */
80dc0d6b 1015 : "r" (tag), "r" (addr),
1da177e4 1016 "i" (ASI_DCACHE_UTAG));
80dc0d6b 1017 for (line = addr; line < addr + dcache_line_size; line += 8)
1da177e4
LT		 1018 __asm__ __volatile__("membar #Sync\n\t"
1019 "stxa %%g0, [%0] %1\n\t"
1020 "membar #Sync"
1021 : /* no outputs */
80dc0d6b
DM		 1022 : "r" (line),
1023 "i" (ASI_DCACHE_DATA));
1da177e4
LT		 1024 }
1025}
1026
1027/* Conversion tables used to frob Cheetah AFSR syndrome values into
1028 * something palatable to the memory controller driver get_unumber
1029 * routine.
1030 */
1031#define MT0 137
1032#define MT1 138
1033#define MT2 139
1034#define NONE 254
1035#define MTC0 140
1036#define MTC1 141
1037#define MTC2 142
1038#define MTC3 143
1039#define C0 128
1040#define C1 129
1041#define C2 130
1042#define C3 131
1043#define C4 132
1044#define C5 133
1045#define C6 134
1046#define C7 135
1047#define C8 136
1048#define M2 144
1049#define M3 145
1050#define M4 146
1051#define M 147
1052static unsigned char cheetah_ecc_syntab[] = {
1053/*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1054/*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1055/*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1056/*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1057/*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1058/*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1059/*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1060/*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1061/*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1062/*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1063/*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1064/*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1065/*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1066/*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1067/*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1068/*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1069/*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1070/*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1071/*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1072/*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1073/*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1074/*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1075/*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1076/*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1077/*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1078/*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1079/*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1080/*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1081/*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1082/*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1083/*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1084/*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1085};
1086static unsigned char cheetah_mtag_syntab[] = {
1087 NONE, MTC0,
1088 MTC1, NONE,
1089 MTC2, NONE,
1090 NONE, MT0,
1091 MTC3, NONE,
1092 NONE, MT1,
1093 NONE, MT2,
1094 NONE, NONE
1095};
1096
1097/* Return the highest priority error conditon mentioned. */
1098static __inline__ unsigned long cheetah_get_hipri(unsigned long afsr)
1099{
1100 unsigned long tmp = 0;
1101 int i;
1102
1103 for (i = 0; cheetah_error_table[i].mask; i++) {
1104 if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1105 return tmp;
1106 }
1107 return tmp;
1108}
1109
1110static const char *cheetah_get_string(unsigned long bit)
1111{
1112 int i;
1113
1114 for (i = 0; cheetah_error_table[i].mask; i++) {
1115 if ((bit & cheetah_error_table[i].mask) != 0UL)
1116 return cheetah_error_table[i].name;
1117 }
1118 return "???";
1119}
1120
1121extern int chmc_getunumber(int, unsigned long, char *, int);
1122
1123static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1124 unsigned long afsr, unsigned long afar, int recoverable)
1125{
1126 unsigned long hipri;
1127 char unum[256];
1128
1129 printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1130 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1131 afsr, afar,
1132 (afsr & CHAFSR_TL1) ? 1 : 0);
1133 printk("%s" "ERROR(%d): TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1134 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1135 regs->tpc, regs->tnpc, regs->tstate);
1136 printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n",
1137 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1138 (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1139 (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1140 (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1141 (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1142 hipri = cheetah_get_hipri(afsr);
1143 printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1144 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1145 hipri, cheetah_get_string(hipri));
1146
1147 /* Try to get unumber if relevant. */
1148#define ESYND_ERRORS (CHAFSR_IVC | CHAFSR_IVU | \
1149 CHAFSR_CPC | CHAFSR_CPU | \
1150 CHAFSR_UE | CHAFSR_CE | \
1151 CHAFSR_EDC | CHAFSR_EDU | \
1152 CHAFSR_UCC | CHAFSR_UCU | \
1153 CHAFSR_WDU | CHAFSR_WDC)
1154#define MSYND_ERRORS (CHAFSR_EMC | CHAFSR_EMU)
1155 if (afsr & ESYND_ERRORS) {
1156 int syndrome;
1157 int ret;
1158
1159 syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1160 syndrome = cheetah_ecc_syntab[syndrome];
1161 ret = chmc_getunumber(syndrome, afar, unum, sizeof(unum));
1162 if (ret != -1)
1163 printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1164 (recoverable ? KERN_WARNING : KERN_CRIT),
1165 smp_processor_id(), unum);
1166 } else if (afsr & MSYND_ERRORS) {
1167 int syndrome;
1168 int ret;
1169
1170 syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1171 syndrome = cheetah_mtag_syntab[syndrome];
1172 ret = chmc_getunumber(syndrome, afar, unum, sizeof(unum));
1173 if (ret != -1)
1174 printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1175 (recoverable ? KERN_WARNING : KERN_CRIT),
1176 smp_processor_id(), unum);
1177 }
1178
1179 /* Now dump the cache snapshots. */
1180 printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016lx] utag[%016lx] stag[%016lx]\n",
1181 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1182 (int) info->dcache_index,
1183 info->dcache_tag,
1184 info->dcache_utag,
1185 info->dcache_stag);
1186 printk("%s" "ERROR(%d): D-cache data0[%016lx] data1[%016lx] data2[%016lx] data3[%016lx]\n",
1187 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1188 info->dcache_data[0],
1189 info->dcache_data[1],
1190 info->dcache_data[2],
1191 info->dcache_data[3]);
1192 printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016lx] utag[%016lx] stag[%016lx] "
1193 "u[%016lx] l[%016lx]\n",
1194 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1195 (int) info->icache_index,
1196 info->icache_tag,
1197 info->icache_utag,
1198 info->icache_stag,
1199 info->icache_upper,
1200 info->icache_lower);
1201 printk("%s" "ERROR(%d): I-cache INSN0[%016lx] INSN1[%016lx] INSN2[%016lx] INSN3[%016lx]\n",
1202 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1203 info->icache_data[0],
1204 info->icache_data[1],
1205 info->icache_data[2],
1206 info->icache_data[3]);
1207 printk("%s" "ERROR(%d): I-cache INSN4[%016lx] INSN5[%016lx] INSN6[%016lx] INSN7[%016lx]\n",
1208 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1209 info->icache_data[4],
1210 info->icache_data[5],
1211 info->icache_data[6],
1212 info->icache_data[7]);
1213 printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016lx]\n",
1214 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1215 (int) info->ecache_index, info->ecache_tag);
1216 printk("%s" "ERROR(%d): E-cache data0[%016lx] data1[%016lx] data2[%016lx] data3[%016lx]\n",
1217 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1218 info->ecache_data[0],
1219 info->ecache_data[1],
1220 info->ecache_data[2],
1221 info->ecache_data[3]);
1222
1223 afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1224 while (afsr != 0UL) {
1225 unsigned long bit = cheetah_get_hipri(afsr);
1226
1227 printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1228 (recoverable ? KERN_WARNING : KERN_CRIT),
1229 bit, cheetah_get_string(bit));
1230
1231 afsr &= ~bit;
1232 }
1233
1234 if (!recoverable)
1235 printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1236}
1237
1238static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1239{
1240 unsigned long afsr, afar;
1241 int ret = 0;
1242
1243 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1244 : "=r" (afsr)
1245 : "i" (ASI_AFSR));
1246 if ((afsr & cheetah_afsr_errors) != 0) {
1247 if (logp != NULL) {
1248 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1249 : "=r" (afar)
1250 : "i" (ASI_AFAR));
1251 logp->afsr = afsr;
1252 logp->afar = afar;
1253 }
1254 ret = 1;
1255 }
1256 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1257 "membar #Sync\n\t"
1258 : : "r" (afsr), "i" (ASI_AFSR));
1259
1260 return ret;
1261}
1262
1263void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1264{
1265 struct cheetah_err_info local_snapshot, *p;
1266 int recoverable;
1267
1268 /* Flush E-cache */
1269 cheetah_flush_ecache();
1270
1271 p = cheetah_get_error_log(afsr);
1272 if (!p) {
1273 prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1274 afsr, afar);
1275 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1276 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1277 prom_halt();
1278 }
1279
1280 /* Grab snapshot of logged error. */
1281 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1282
1283 /* If the current trap snapshot does not match what the
1284 * trap handler passed along into our args, big trouble.
1285 * In such a case, mark the local copy as invalid.
1286 *
1287 * Else, it matches and we mark the afsr in the non-local
1288 * copy as invalid so we may log new error traps there.
1289 */
1290 if (p->afsr != afsr || p->afar != afar)
1291 local_snapshot.afsr = CHAFSR_INVALID;
1292 else
1293 p->afsr = CHAFSR_INVALID;
1294
1295 cheetah_flush_icache();
1296 cheetah_flush_dcache();
1297
1298 /* Re-enable I-cache/D-cache */
1299 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1300 "or %%g1, %1, %%g1\n\t"
1301 "stxa %%g1, [%%g0] %0\n\t"
1302 "membar #Sync"
1303 : /* no outputs */
1304 : "i" (ASI_DCU_CONTROL_REG),
1305 "i" (DCU_DC | DCU_IC)
1306 : "g1");
1307
1308 /* Re-enable error reporting */
1309 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1310 "or %%g1, %1, %%g1\n\t"
1311 "stxa %%g1, [%%g0] %0\n\t"
1312 "membar #Sync"
1313 : /* no outputs */
1314 : "i" (ASI_ESTATE_ERROR_EN),
1315 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1316 : "g1");
1317
1318 /* Decide if we can continue after handling this trap and
1319 * logging the error.
1320 */
1321 recoverable = 1;
1322 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1323 recoverable = 0;
1324
1325 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1326 * error was logged while we had error reporting traps disabled.
1327 */
1328 if (cheetah_recheck_errors(&local_snapshot)) {
1329 unsigned long new_afsr = local_snapshot.afsr;
1330
1331 /* If we got a new asynchronous error, die... */
1332 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1333 CHAFSR_WDU | CHAFSR_CPU |
1334 CHAFSR_IVU | CHAFSR_UE |
1335 CHAFSR_BERR | CHAFSR_TO))
1336 recoverable = 0;
1337 }
1338
1339 /* Log errors. */
1340 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1341
1342 if (!recoverable)
1343 panic("Irrecoverable Fast-ECC error trap.\n");
1344
1345 /* Flush E-cache to kick the error trap handlers out. */
1346 cheetah_flush_ecache();
1347}
1348
1349/* Try to fix a correctable error by pushing the line out from
1350 * the E-cache. Recheck error reporting registers to see if the
1351 * problem is intermittent.
1352 */
1353static int cheetah_fix_ce(unsigned long physaddr)
1354{
1355 unsigned long orig_estate;
1356 unsigned long alias1, alias2;
1357 int ret;
1358
1359 /* Make sure correctable error traps are disabled. */
1360 __asm__ __volatile__("ldxa [%%g0] %2, %0\n\t"
1361 "andn %0, %1, %%g1\n\t"
1362 "stxa %%g1, [%%g0] %2\n\t"
1363 "membar #Sync"
1364 : "=&r" (orig_estate)
1365 : "i" (ESTATE_ERROR_CEEN),
1366 "i" (ASI_ESTATE_ERROR_EN)
1367 : "g1");
1368
1369 /* We calculate alias addresses that will force the
1370 * cache line in question out of the E-cache. Then
1371 * we bring it back in with an atomic instruction so
1372 * that we get it in some modified/exclusive state,
1373 * then we displace it again to try and get proper ECC
1374 * pushed back into the system.
1375 */
1376 physaddr &= ~(8UL - 1UL);
1377 alias1 = (ecache_flush_physbase +
1378 (physaddr & ((ecache_flush_size >> 1) - 1)));
1379 alias2 = alias1 + (ecache_flush_size >> 1);
1380 __asm__ __volatile__("ldxa [%0] %3, %%g0\n\t"
1381 "ldxa [%1] %3, %%g0\n\t"
1382 "casxa [%2] %3, %%g0, %%g0\n\t"
1383 "membar #StoreLoad | #StoreStore\n\t"
1384 "ldxa [%0] %3, %%g0\n\t"
1385 "ldxa [%1] %3, %%g0\n\t"
1386 "membar #Sync"
1387 : /* no outputs */
1388 : "r" (alias1), "r" (alias2),
1389 "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1390
1391 /* Did that trigger another error? */
1392 if (cheetah_recheck_errors(NULL)) {
1393 /* Try one more time. */
1394 __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1395 "membar #Sync"
1396 : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1397 if (cheetah_recheck_errors(NULL))
1398 ret = 2;
1399 else
1400 ret = 1;
1401 } else {
1402 /* No new error, intermittent problem. */
1403 ret = 0;
1404 }
1405
1406 /* Restore error enables. */
1407 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1408 "membar #Sync"
1409 : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1410
1411 return ret;
1412}
1413
1414/* Return non-zero if PADDR is a valid physical memory address. */
1415static int cheetah_check_main_memory(unsigned long paddr)
1416{
10147570 1417 unsigned long vaddr = PAGE_OFFSET + paddr;
1da177e4 1418
13edad7a 1419 if (vaddr > (unsigned long) high_memory)
ed3ffaf7
DM		 1420 return 0;
1421
10147570 1422 return kern_addr_valid(vaddr);
1da177e4
LT		 1423}
1424
1425void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1426{
1427 struct cheetah_err_info local_snapshot, *p;
1428 int recoverable, is_memory;
1429
1430 p = cheetah_get_error_log(afsr);
1431 if (!p) {
1432 prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1433 afsr, afar);
1434 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1435 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1436 prom_halt();
1437 }
1438
1439 /* Grab snapshot of logged error. */
1440 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1441
1442 /* If the current trap snapshot does not match what the
1443 * trap handler passed along into our args, big trouble.
1444 * In such a case, mark the local copy as invalid.
1445 *
1446 * Else, it matches and we mark the afsr in the non-local
1447 * copy as invalid so we may log new error traps there.
1448 */
1449 if (p->afsr != afsr || p->afar != afar)
1450 local_snapshot.afsr = CHAFSR_INVALID;
1451 else
1452 p->afsr = CHAFSR_INVALID;
1453
1454 is_memory = cheetah_check_main_memory(afar);
1455
1456 if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1457 /* XXX Might want to log the results of this operation
1458 * XXX somewhere... -DaveM
1459 */
1460 cheetah_fix_ce(afar);
1461 }
1462
1463 {
1464 int flush_all, flush_line;
1465
1466 flush_all = flush_line = 0;
1467 if ((afsr & CHAFSR_EDC) != 0UL) {
1468 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1469 flush_line = 1;
1470 else
1471 flush_all = 1;
1472 } else if ((afsr & CHAFSR_CPC) != 0UL) {
1473 if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1474 flush_line = 1;
1475 else
1476 flush_all = 1;
1477 }
1478
1479 /* Trap handler only disabled I-cache, flush it. */
1480 cheetah_flush_icache();
1481
1482 /* Re-enable I-cache */
1483 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1484 "or %%g1, %1, %%g1\n\t"
1485 "stxa %%g1, [%%g0] %0\n\t"
1486 "membar #Sync"
1487 : /* no outputs */
1488 : "i" (ASI_DCU_CONTROL_REG),
1489 "i" (DCU_IC)
1490 : "g1");
1491
1492 if (flush_all)
1493 cheetah_flush_ecache();
1494 else if (flush_line)
1495 cheetah_flush_ecache_line(afar);
1496 }
1497
1498 /* Re-enable error reporting */
1499 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1500 "or %%g1, %1, %%g1\n\t"
1501 "stxa %%g1, [%%g0] %0\n\t"
1502 "membar #Sync"
1503 : /* no outputs */
1504 : "i" (ASI_ESTATE_ERROR_EN),
1505 "i" (ESTATE_ERROR_CEEN)
1506 : "g1");
1507
1508 /* Decide if we can continue after handling this trap and
1509 * logging the error.
1510 */
1511 recoverable = 1;
1512 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1513 recoverable = 0;
1514
1515 /* Re-check AFSR/AFAR */
1516 (void) cheetah_recheck_errors(&local_snapshot);
1517
1518 /* Log errors. */
1519 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1520
1521 if (!recoverable)
1522 panic("Irrecoverable Correctable-ECC error trap.\n");
1523}
1524
1525void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1526{
1527 struct cheetah_err_info local_snapshot, *p;
1528 int recoverable, is_memory;
1529
1530#ifdef CONFIG_PCI
1531 /* Check for the special PCI poke sequence. */
1532 if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1533 cheetah_flush_icache();
1534 cheetah_flush_dcache();
1535
1536 /* Re-enable I-cache/D-cache */
1537 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1538 "or %%g1, %1, %%g1\n\t"
1539 "stxa %%g1, [%%g0] %0\n\t"
1540 "membar #Sync"
1541 : /* no outputs */
1542 : "i" (ASI_DCU_CONTROL_REG),
1543 "i" (DCU_DC | DCU_IC)
1544 : "g1");
1545
1546 /* Re-enable error reporting */
1547 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1548 "or %%g1, %1, %%g1\n\t"
1549 "stxa %%g1, [%%g0] %0\n\t"
1550 "membar #Sync"
1551 : /* no outputs */
1552 : "i" (ASI_ESTATE_ERROR_EN),
1553 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1554 : "g1");
1555
1556 (void) cheetah_recheck_errors(NULL);
1557
1558 pci_poke_faulted = 1;
1559 regs->tpc += 4;
1560 regs->tnpc = regs->tpc + 4;
1561 return;
1562 }
1563#endif
1564
1565 p = cheetah_get_error_log(afsr);
1566 if (!p) {
1567 prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1568 afsr, afar);
1569 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1570 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1571 prom_halt();
1572 }
1573
1574 /* Grab snapshot of logged error. */
1575 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1576
1577 /* If the current trap snapshot does not match what the
1578 * trap handler passed along into our args, big trouble.
1579 * In such a case, mark the local copy as invalid.
1580 *
1581 * Else, it matches and we mark the afsr in the non-local
1582 * copy as invalid so we may log new error traps there.
1583 */
1584 if (p->afsr != afsr || p->afar != afar)
1585 local_snapshot.afsr = CHAFSR_INVALID;
1586 else
1587 p->afsr = CHAFSR_INVALID;
1588
1589 is_memory = cheetah_check_main_memory(afar);
1590
1591 {
1592 int flush_all, flush_line;
1593
1594 flush_all = flush_line = 0;
1595 if ((afsr & CHAFSR_EDU) != 0UL) {
1596 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1597 flush_line = 1;
1598 else
1599 flush_all = 1;
1600 } else if ((afsr & CHAFSR_BERR) != 0UL) {
1601 if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1602 flush_line = 1;
1603 else
1604 flush_all = 1;
1605 }
1606
1607 cheetah_flush_icache();
1608 cheetah_flush_dcache();
1609
1610 /* Re-enable I/D caches */
1611 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1612 "or %%g1, %1, %%g1\n\t"
1613 "stxa %%g1, [%%g0] %0\n\t"
1614 "membar #Sync"
1615 : /* no outputs */
1616 : "i" (ASI_DCU_CONTROL_REG),
1617 "i" (DCU_IC | DCU_DC)
1618 : "g1");
1619
1620 if (flush_all)
1621 cheetah_flush_ecache();
1622 else if (flush_line)
1623 cheetah_flush_ecache_line(afar);
1624 }
1625
1626 /* Re-enable error reporting */
1627 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1628 "or %%g1, %1, %%g1\n\t"
1629 "stxa %%g1, [%%g0] %0\n\t"
1630 "membar #Sync"
1631 : /* no outputs */
1632 : "i" (ASI_ESTATE_ERROR_EN),
1633 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1634 : "g1");
1635
1636 /* Decide if we can continue after handling this trap and
1637 * logging the error.
1638 */
1639 recoverable = 1;
1640 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1641 recoverable = 0;
1642
1643 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1644 * error was logged while we had error reporting traps disabled.
1645 */
1646 if (cheetah_recheck_errors(&local_snapshot)) {
1647 unsigned long new_afsr = local_snapshot.afsr;
1648
1649 /* If we got a new asynchronous error, die... */
1650 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1651 CHAFSR_WDU | CHAFSR_CPU |
1652 CHAFSR_IVU | CHAFSR_UE |
1653 CHAFSR_BERR | CHAFSR_TO))
1654 recoverable = 0;
1655 }
1656
1657 /* Log errors. */
1658 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1659
1660 /* "Recoverable" here means we try to yank the page from ever
1661 * being newly used again. This depends upon a few things:
1662 * 1) Must be main memory, and AFAR must be valid.
1663 * 2) If we trapped from user, OK.
1664 * 3) Else, if we trapped from kernel we must find exception
1665 * table entry (ie. we have to have been accessing user
1666 * space).
1667 *
1668 * If AFAR is not in main memory, or we trapped from kernel
1669 * and cannot find an exception table entry, it is unacceptable
1670 * to try and continue.
1671 */
1672 if (recoverable && is_memory) {
1673 if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1674 /* OK, usermode access. */
1675 recoverable = 1;
1676 } else {
8cf14af0 1677 const struct exception_table_entry *entry;
1da177e4 1678
8cf14af0
DM		 1679 entry = search_exception_tables(regs->tpc);
1680 if (entry) {
1da177e4
LT		 1681 /* OK, kernel access to userspace. */
1682 recoverable = 1;
1683
1684 } else {
1685 /* BAD, privileged state is corrupted. */
1686 recoverable = 0;
1687 }
1688
1689 if (recoverable) {
1690 if (pfn_valid(afar >> PAGE_SHIFT))
1691 get_page(pfn_to_page(afar >> PAGE_SHIFT));
1692 else
1693 recoverable = 0;
1694
1695 /* Only perform fixup if we still have a
1696 * recoverable condition.
1697 */
1698 if (recoverable) {
8cf14af0 1699 regs->tpc = entry->fixup;
1da177e4 1700 regs->tnpc = regs->tpc + 4;
1da177e4
LT		 1701 }
1702 }
1703 }
1704 } else {
1705 recoverable = 0;
1706 }
1707
1708 if (!recoverable)
1709 panic("Irrecoverable deferred error trap.\n");
1710}
1711
1712/* Handle a D/I cache parity error trap. TYPE is encoded as:
1713 *
1714 * Bit0: 0=dcache,1=icache
1715 * Bit1: 0=recoverable,1=unrecoverable
1716 *
1717 * The hardware has disabled both the I-cache and D-cache in
1718 * the %dcr register.
1719 */
1720void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1721{
1722 if (type & 0x1)
1723 __cheetah_flush_icache();
1724 else
1725 cheetah_plus_zap_dcache_parity();
1726 cheetah_flush_dcache();
1727
1728 /* Re-enable I-cache/D-cache */
1729 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1730 "or %%g1, %1, %%g1\n\t"
1731 "stxa %%g1, [%%g0] %0\n\t"
1732 "membar #Sync"
1733 : /* no outputs */
1734 : "i" (ASI_DCU_CONTROL_REG),
1735 "i" (DCU_DC | DCU_IC)
1736 : "g1");
1737
1738 if (type & 0x2) {
1739 printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1740 smp_processor_id(),
1741 (type & 0x1) ? 'I' : 'D',
1742 regs->tpc);
1743 panic("Irrecoverable Cheetah+ parity error.");
1744 }
1745
1746 printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1747 smp_processor_id(),
1748 (type & 0x1) ? 'I' : 'D',
1749 regs->tpc);
1750}
1751
5b0c0572
DM		 1752struct sun4v_error_entry {
1753 u64 err_handle;
1754 u64 err_stick;
1755
1756 u32 err_type;
1757#define SUN4V_ERR_TYPE_UNDEFINED 0
1758#define SUN4V_ERR_TYPE_UNCORRECTED_RES 1
1759#define SUN4V_ERR_TYPE_PRECISE_NONRES 2
1760#define SUN4V_ERR_TYPE_DEFERRED_NONRES 3
1761#define SUN4V_ERR_TYPE_WARNING_RES 4
1762
1763 u32 err_attrs;
1764#define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001
1765#define SUN4V_ERR_ATTRS_MEMORY 0x00000002
1766#define SUN4V_ERR_ATTRS_PIO 0x00000004
1767#define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008
1768#define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010
1769#define SUN4V_ERR_ATTRS_USER_MODE 0x01000000
1770#define SUN4V_ERR_ATTRS_PRIV_MODE 0x02000000
1771#define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000
1772
1773 u64 err_raddr;
1774 u32 err_size;
1775 u16 err_cpu;
1776 u16 err_pad;
1777};
1778
1779static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1780static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1781
1782static const char *sun4v_err_type_to_str(u32 type)
1783{
1784 switch (type) {
1785 case SUN4V_ERR_TYPE_UNDEFINED:
1786 return "undefined";
1787 case SUN4V_ERR_TYPE_UNCORRECTED_RES:
1788 return "uncorrected resumable";
1789 case SUN4V_ERR_TYPE_PRECISE_NONRES:
1790 return "precise nonresumable";
1791 case SUN4V_ERR_TYPE_DEFERRED_NONRES:
1792 return "deferred nonresumable";
1793 case SUN4V_ERR_TYPE_WARNING_RES:
1794 return "warning resumable";
1795 default:
1796 return "unknown";
1797 };
1798}
1799
1800static void sun4v_log_error(struct sun4v_error_entry *ent, int cpu, const char *pfx, atomic_t *ocnt)
1801{
1802 int cnt;
1803
1804 printk("%s: Reporting on cpu %d\n", pfx, cpu);
1805 printk("%s: err_handle[%lx] err_stick[%lx] err_type[%08x:%s]\n",
1806 pfx,
1807 ent->err_handle, ent->err_stick,
1808 ent->err_type,
1809 sun4v_err_type_to_str(ent->err_type));
1810 printk("%s: err_attrs[%08x:%s %s %s %s %s %s %s %s]\n",
1811 pfx,
1812 ent->err_attrs,
1813 ((ent->err_attrs & SUN4V_ERR_ATTRS_PROCESSOR) ?
1814 "processor" : ""),
1815 ((ent->err_attrs & SUN4V_ERR_ATTRS_MEMORY) ?
1816 "memory" : ""),
1817 ((ent->err_attrs & SUN4V_ERR_ATTRS_PIO) ?
1818 "pio" : ""),
1819 ((ent->err_attrs & SUN4V_ERR_ATTRS_INT_REGISTERS) ?
1820 "integer-regs" : ""),
1821 ((ent->err_attrs & SUN4V_ERR_ATTRS_FPU_REGISTERS) ?
1822 "fpu-regs" : ""),
1823 ((ent->err_attrs & SUN4V_ERR_ATTRS_USER_MODE) ?
1824 "user" : ""),
1825 ((ent->err_attrs & SUN4V_ERR_ATTRS_PRIV_MODE) ?
1826 "privileged" : ""),
1827 ((ent->err_attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL) ?
1828 "queue-full" : ""));
1829 printk("%s: err_raddr[%016lx] err_size[%u] err_cpu[%u]\n",
1830 pfx,
1831 ent->err_raddr, ent->err_size, ent->err_cpu);
1832
1833 if ((cnt = atomic_read(ocnt)) != 0) {
1834 atomic_set(ocnt, 0);
1835 wmb();
1836 printk("%s: Queue overflowed %d times.\n",
1837 pfx, cnt);
1838 }
1839}
1840
1841/* We run with %pil set to 15 and PSTATE_IE enabled in %pstate.
1842 * Log the event and clear the first word of the entry.
1843 */
1844void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
1845{
1846 struct sun4v_error_entry *ent, local_copy;
1847 struct trap_per_cpu *tb;
1848 unsigned long paddr;
1849 int cpu;
1850
1851 cpu = get_cpu();
1852
1853 tb = &trap_block[cpu];
1854 paddr = tb->resum_kernel_buf_pa + offset;
1855 ent = __va(paddr);
1856
1857 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
1858
1859 /* We have a local copy now, so release the entry. */
1860 ent->err_handle = 0;
1861 wmb();
1862
1863 put_cpu();
1864
1865 sun4v_log_error(&local_copy, cpu,
1866 KERN_ERR "RESUMABLE ERROR",
1867 &sun4v_resum_oflow_cnt);
1868}
1869
1870/* If we try to printk() we'll probably make matters worse, by trying
1871 * to retake locks this cpu already holds or causing more errors. So
1872 * just bump a counter, and we'll report these counter bumps above.
1873 */
1874void sun4v_resum_overflow(struct pt_regs *regs)
1875{
1876 atomic_inc(&sun4v_resum_oflow_cnt);
1877}
1878
1879/* We run with %pil set to 15 and PSTATE_IE enabled in %pstate.
1880 * Log the event, clear the first word of the entry, and die.
1881 */
1882void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
1883{
1884 struct sun4v_error_entry *ent, local_copy;
1885 struct trap_per_cpu *tb;
1886 unsigned long paddr;
1887 int cpu;
1888
1889 cpu = get_cpu();
1890
1891 tb = &trap_block[cpu];
1892 paddr = tb->nonresum_kernel_buf_pa + offset;
1893 ent = __va(paddr);
1894
1895 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
1896
1897 /* We have a local copy now, so release the entry. */
1898 ent->err_handle = 0;
1899 wmb();
1900
1901 put_cpu();
1902
1903#ifdef CONFIG_PCI
1904 /* Check for the special PCI poke sequence. */
1905 if (pci_poke_in_progress && pci_poke_cpu == cpu) {
1906 pci_poke_faulted = 1;
1907 regs->tpc += 4;
1908 regs->tnpc = regs->tpc + 4;
1909 return;
1910 }
1911#endif
1912
1913 sun4v_log_error(&local_copy, cpu,
1914 KERN_EMERG "NON-RESUMABLE ERROR",
1915 &sun4v_nonresum_oflow_cnt);
1916
1917 panic("Non-resumable error.");
1918}
1919
1920/* If we try to printk() we'll probably make matters worse, by trying
1921 * to retake locks this cpu already holds or causing more errors. So
1922 * just bump a counter, and we'll report these counter bumps above.
1923 */
1924void sun4v_nonresum_overflow(struct pt_regs *regs)
1925{
1926 /* XXX Actually even this can make not that much sense. Perhaps
1927 * XXX we should just pull the plug and panic directly from here?
1928 */
1929 atomic_inc(&sun4v_nonresum_oflow_cnt);
1930}
1931
6c8927c9
DM		 1932unsigned long sun4v_err_itlb_vaddr;
1933unsigned long sun4v_err_itlb_ctx;
1934unsigned long sun4v_err_itlb_pte;
1935unsigned long sun4v_err_itlb_error;
1936
1937void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
1938{
1939 if (tl > 1)
1940 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
1941
04d74758
DM		 1942 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
1943 regs->tpc, tl);
1944 printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
1945 "pte[%lx] error[%lx]\n",
6c8927c9
DM		 1946 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
1947 sun4v_err_itlb_pte, sun4v_err_itlb_error);
04d74758 1948
6c8927c9
DM		 1949 prom_halt();
1950}
1951
1952unsigned long sun4v_err_dtlb_vaddr;
1953unsigned long sun4v_err_dtlb_ctx;
1954unsigned long sun4v_err_dtlb_pte;
1955unsigned long sun4v_err_dtlb_error;
1956
1957void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
1958{
1959 if (tl > 1)
1960 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
1961
04d74758
DM		 1962 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
1963 regs->tpc, tl);
1964 printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
1965 "pte[%lx] error[%lx]\n",
6c8927c9
DM		 1966 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
1967 sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
04d74758 1968
6c8927c9
DM		 1969 prom_halt();
1970}
1971
2a3a5f5d
DM		 1972void hypervisor_tlbop_error(unsigned long err, unsigned long op)
1973{
1974 printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
1975 err, op);
1976}
1977
1978void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
1979{
1980 printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
1981 err, op);
1982}
1983
1da177e4
LT		 1984void do_fpe_common(struct pt_regs *regs)
1985{
1986 if (regs->tstate & TSTATE_PRIV) {
1987 regs->tpc = regs->tnpc;
1988 regs->tnpc += 4;
1989 } else {
1990 unsigned long fsr = current_thread_info()->xfsr[0];
1991 siginfo_t info;
1992
1993 if (test_thread_flag(TIF_32BIT)) {
1994 regs->tpc &= 0xffffffff;
1995 regs->tnpc &= 0xffffffff;
1996 }
1997 info.si_signo = SIGFPE;
1998 info.si_errno = 0;
1999 info.si_addr = (void __user *)regs->tpc;
2000 info.si_trapno = 0;
2001 info.si_code = __SI_FAULT;
2002 if ((fsr & 0x1c000) == (1 << 14)) {
2003 if (fsr & 0x10)
2004 info.si_code = FPE_FLTINV;
2005 else if (fsr & 0x08)
2006 info.si_code = FPE_FLTOVF;
2007 else if (fsr & 0x04)
2008 info.si_code = FPE_FLTUND;
2009 else if (fsr & 0x02)
2010 info.si_code = FPE_FLTDIV;
2011 else if (fsr & 0x01)
2012 info.si_code = FPE_FLTRES;
2013 }
2014 force_sig_info(SIGFPE, &info, current);
2015 }
2016}
2017
2018void do_fpieee(struct pt_regs *regs)
2019{
2020 if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2021 0, 0x24, SIGFPE) == NOTIFY_STOP)
2022 return;
2023
2024 do_fpe_common(regs);
2025}
2026
2027extern int do_mathemu(struct pt_regs *, struct fpustate *);
2028
2029void do_fpother(struct pt_regs *regs)
2030{
2031 struct fpustate *f = FPUSTATE;
2032 int ret = 0;
2033
2034 if (notify_die(DIE_TRAP, "fpu exception other", regs,
2035 0, 0x25, SIGFPE) == NOTIFY_STOP)
2036 return;
2037
2038 switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2039 case (2 << 14): /* unfinished_FPop */
2040 case (3 << 14): /* unimplemented_FPop */
2041 ret = do_mathemu(regs, f);
2042 break;
2043 }
2044 if (ret)
2045 return;
2046 do_fpe_common(regs);
2047}
2048
2049void do_tof(struct pt_regs *regs)
2050{
2051 siginfo_t info;
2052
2053 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2054 0, 0x26, SIGEMT) == NOTIFY_STOP)
2055 return;
2056
2057 if (regs->tstate & TSTATE_PRIV)
2058 die_if_kernel("Penguin overflow trap from kernel mode", regs);
2059 if (test_thread_flag(TIF_32BIT)) {
2060 regs->tpc &= 0xffffffff;
2061 regs->tnpc &= 0xffffffff;
2062 }
2063 info.si_signo = SIGEMT;
2064 info.si_errno = 0;
2065 info.si_code = EMT_TAGOVF;
2066 info.si_addr = (void __user *)regs->tpc;
2067 info.si_trapno = 0;
2068 force_sig_info(SIGEMT, &info, current);
2069}
2070
2071void do_div0(struct pt_regs *regs)
2072{
2073 siginfo_t info;
2074
2075 if (notify_die(DIE_TRAP, "integer division by zero", regs,
2076 0, 0x28, SIGFPE) == NOTIFY_STOP)
2077 return;
2078
2079 if (regs->tstate & TSTATE_PRIV)
2080 die_if_kernel("TL0: Kernel divide by zero.", regs);
2081 if (test_thread_flag(TIF_32BIT)) {
2082 regs->tpc &= 0xffffffff;
2083 regs->tnpc &= 0xffffffff;
2084 }
2085 info.si_signo = SIGFPE;
2086 info.si_errno = 0;
2087 info.si_code = FPE_INTDIV;
2088 info.si_addr = (void __user *)regs->tpc;
2089 info.si_trapno = 0;
2090 force_sig_info(SIGFPE, &info, current);
2091}
2092
2093void instruction_dump (unsigned int *pc)
2094{
2095 int i;
2096
2097 if ((((unsigned long) pc) & 3))
2098 return;
2099
2100 printk("Instruction DUMP:");
2101 for (i = -3; i < 6; i++)
2102 printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2103 printk("\n");
2104}
2105
2106static void user_instruction_dump (unsigned int __user *pc)
2107{
2108 int i;
2109 unsigned int buf[9];
2110
2111 if ((((unsigned long) pc) & 3))
2112 return;
2113
2114 if (copy_from_user(buf, pc - 3, sizeof(buf)))
2115 return;
2116
2117 printk("Instruction DUMP:");
2118 for (i = 0; i < 9; i++)
2119 printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2120 printk("\n");
2121}
2122
2123void show_stack(struct task_struct *tsk, unsigned long *_ksp)
2124{
2125 unsigned long pc, fp, thread_base, ksp;
ee3eea16 2126 void *tp = task_stack_page(tsk);
1da177e4
LT		 2127 struct reg_window *rw;
2128 int count = 0;
2129
2130 ksp = (unsigned long) _ksp;
2131
2132 if (tp == current_thread_info())
2133 flushw_all();
2134
2135 fp = ksp + STACK_BIAS;
2136 thread_base = (unsigned long) tp;
2137
2138 printk("Call Trace:");
2139#ifdef CONFIG_KALLSYMS
2140 printk("\n");
2141#endif
2142 do {
2143 /* Bogus frame pointer? */
2144 if (fp < (thread_base + sizeof(struct thread_info)) ||
2145 fp >= (thread_base + THREAD_SIZE))
2146 break;
2147 rw = (struct reg_window *)fp;
2148 pc = rw->ins[7];
2149 printk(" [%016lx] ", pc);
2150 print_symbol("%s\n", pc);
2151 fp = rw->ins[6] + STACK_BIAS;
2152 } while (++count < 16);
2153#ifndef CONFIG_KALLSYMS
2154 printk("\n");
2155#endif
2156}
2157
2158void dump_stack(void)
2159{
2160 unsigned long *ksp;
2161
2162 __asm__ __volatile__("mov %%fp, %0"
2163 : "=r" (ksp));
2164 show_stack(current, ksp);
2165}
2166
2167EXPORT_SYMBOL(dump_stack);
2168
2169static inline int is_kernel_stack(struct task_struct *task,
2170 struct reg_window *rw)
2171{
2172 unsigned long rw_addr = (unsigned long) rw;
2173 unsigned long thread_base, thread_end;
2174
2175 if (rw_addr < PAGE_OFFSET) {
2176 if (task != &init_task)
2177 return 0;
2178 }
2179
ee3eea16 2180 thread_base = (unsigned long) task_stack_page(task);
1da177e4
LT		 2181 thread_end = thread_base + sizeof(union thread_union);
2182 if (rw_addr >= thread_base &&
2183 rw_addr < thread_end &&
2184 !(rw_addr & 0x7UL))
2185 return 1;
2186
2187 return 0;
2188}
2189
2190static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2191{
2192 unsigned long fp = rw->ins[6];
2193
2194 if (!fp)
2195 return NULL;
2196
2197 return (struct reg_window *) (fp + STACK_BIAS);
2198}
2199
2200void die_if_kernel(char *str, struct pt_regs *regs)
2201{
2202 static int die_counter;
2203 extern void __show_regs(struct pt_regs * regs);
2204 extern void smp_report_regs(void);
2205 int count = 0;
2206
2207 /* Amuse the user. */
2208 printk(
2209" \\|/ ____ \\|/\n"
2210" \"@'/ .. \\`@\"\n"
2211" /_| \\__/ |_\\\n"
2212" \\__U_/\n");
2213
2214 printk("%s(%d): %s [#%d]\n", current->comm, current->pid, str, ++die_counter);
2215 notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2216 __asm__ __volatile__("flushw");
2217 __show_regs(regs);
2218 if (regs->tstate & TSTATE_PRIV) {
2219 struct reg_window *rw = (struct reg_window *)
2220 (regs->u_regs[UREG_FP] + STACK_BIAS);
2221
2222 /* Stop the back trace when we hit userland or we
2223 * find some badly aligned kernel stack.
2224 */
2225 while (rw &&
2226 count++ < 30&&
2227 is_kernel_stack(current, rw)) {
2228 printk("Caller[%016lx]", rw->ins[7]);
2229 print_symbol(": %s", rw->ins[7]);
2230 printk("\n");
2231
2232 rw = kernel_stack_up(rw);
2233 }
2234 instruction_dump ((unsigned int *) regs->tpc);
2235 } else {
2236 if (test_thread_flag(TIF_32BIT)) {
2237 regs->tpc &= 0xffffffff;
2238 regs->tnpc &= 0xffffffff;
2239 }
2240 user_instruction_dump ((unsigned int __user *) regs->tpc);
2241 }
37133c00 2242#if 0
1da177e4
LT		 2243#ifdef CONFIG_SMP
2244 smp_report_regs();
2245#endif
37133c00 2246#endif
1da177e4
LT		 2247 if (regs->tstate & TSTATE_PRIV)
2248 do_exit(SIGKILL);
2249 do_exit(SIGSEGV);
2250}
2251
2252extern int handle_popc(u32 insn, struct pt_regs *regs);
2253extern int handle_ldf_stq(u32 insn, struct pt_regs *regs);
2254
2255void do_illegal_instruction(struct pt_regs *regs)
2256{
2257 unsigned long pc = regs->tpc;
2258 unsigned long tstate = regs->tstate;
2259 u32 insn;
2260 siginfo_t info;
2261
2262 if (notify_die(DIE_TRAP, "illegal instruction", regs,
2263 0, 0x10, SIGILL) == NOTIFY_STOP)
2264 return;
2265
2266 if (tstate & TSTATE_PRIV)
2267 die_if_kernel("Kernel illegal instruction", regs);
2268 if (test_thread_flag(TIF_32BIT))
2269 pc = (u32)pc;
2270 if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2271 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2272 if (handle_popc(insn, regs))
2273 return;
2274 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2275 if (handle_ldf_stq(insn, regs))
2276 return;
0c51ed93
DM		 2277 } else if (tlb_type == hypervisor) {
2278 extern int vis_emul(struct pt_regs *, unsigned int);
2279
2280 if (!vis_emul(regs, insn))
2281 return;
1da177e4
LT		 2282 }
2283 }
2284 info.si_signo = SIGILL;
2285 info.si_errno = 0;
2286 info.si_code = ILL_ILLOPC;
2287 info.si_addr = (void __user *)pc;
2288 info.si_trapno = 0;
2289 force_sig_info(SIGILL, &info, current);
2290}
2291
ed6b0b45
DM		 2292extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn);
2293
1da177e4
LT		 2294void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2295{
2296 siginfo_t info;
2297
2298 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2299 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2300 return;
2301
2302 if (regs->tstate & TSTATE_PRIV) {
ed6b0b45 2303 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
1da177e4
LT		 2304 return;
2305 }
2306 info.si_signo = SIGBUS;
2307 info.si_errno = 0;
2308 info.si_code = BUS_ADRALN;
2309 info.si_addr = (void __user *)sfar;
2310 info.si_trapno = 0;
2311 force_sig_info(SIGBUS, &info, current);
2312}
2313
9f8a5b84 2314void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
ed6b0b45
DM		 2315{
2316 siginfo_t info;
2317
2318 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2319 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2320 return;
2321
2322 if (regs->tstate & TSTATE_PRIV) {
2323 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2324 return;
2325 }
2326 info.si_signo = SIGBUS;
2327 info.si_errno = 0;
2328 info.si_code = BUS_ADRALN;
2329 info.si_addr = (void __user *) addr;
2330 info.si_trapno = 0;
2331 force_sig_info(SIGBUS, &info, current);
2332}
2333
1da177e4
LT		 2334void do_privop(struct pt_regs *regs)
2335{
2336 siginfo_t info;
2337
2338 if (notify_die(DIE_TRAP, "privileged operation", regs,
2339 0, 0x11, SIGILL) == NOTIFY_STOP)
2340 return;
2341
2342 if (test_thread_flag(TIF_32BIT)) {
2343 regs->tpc &= 0xffffffff;
2344 regs->tnpc &= 0xffffffff;
2345 }
2346 info.si_signo = SIGILL;
2347 info.si_errno = 0;
2348 info.si_code = ILL_PRVOPC;
2349 info.si_addr = (void __user *)regs->tpc;
2350 info.si_trapno = 0;
2351 force_sig_info(SIGILL, &info, current);
2352}
2353
2354void do_privact(struct pt_regs *regs)
2355{
2356 do_privop(regs);
2357}
2358
2359/* Trap level 1 stuff or other traps we should never see... */
2360void do_cee(struct pt_regs *regs)
2361{
2362 die_if_kernel("TL0: Cache Error Exception", regs);
2363}
2364
2365void do_cee_tl1(struct pt_regs *regs)
2366{
2367 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2368 die_if_kernel("TL1: Cache Error Exception", regs);
2369}
2370
2371void do_dae_tl1(struct pt_regs *regs)
2372{
2373 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2374 die_if_kernel("TL1: Data Access Exception", regs);
2375}
2376
2377void do_iae_tl1(struct pt_regs *regs)
2378{
2379 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2380 die_if_kernel("TL1: Instruction Access Exception", regs);
2381}
2382
2383void do_div0_tl1(struct pt_regs *regs)
2384{
2385 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2386 die_if_kernel("TL1: DIV0 Exception", regs);
2387}
2388
2389void do_fpdis_tl1(struct pt_regs *regs)
2390{
2391 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2392 die_if_kernel("TL1: FPU Disabled", regs);
2393}
2394
2395void do_fpieee_tl1(struct pt_regs *regs)
2396{
2397 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2398 die_if_kernel("TL1: FPU IEEE Exception", regs);
2399}
2400
2401void do_fpother_tl1(struct pt_regs *regs)
2402{
2403 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2404 die_if_kernel("TL1: FPU Other Exception", regs);
2405}
2406
2407void do_ill_tl1(struct pt_regs *regs)
2408{
2409 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2410 die_if_kernel("TL1: Illegal Instruction Exception", regs);
2411}
2412
2413void do_irq_tl1(struct pt_regs *regs)
2414{
2415 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2416 die_if_kernel("TL1: IRQ Exception", regs);
2417}
2418
2419void do_lddfmna_tl1(struct pt_regs *regs)
2420{
2421 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2422 die_if_kernel("TL1: LDDF Exception", regs);
2423}
2424
2425void do_stdfmna_tl1(struct pt_regs *regs)
2426{
2427 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2428 die_if_kernel("TL1: STDF Exception", regs);
2429}
2430
2431void do_paw(struct pt_regs *regs)
2432{
2433 die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2434}
2435
2436void do_paw_tl1(struct pt_regs *regs)
2437{
2438 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2439 die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2440}
2441
2442void do_vaw(struct pt_regs *regs)
2443{
2444 die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2445}
2446
2447void do_vaw_tl1(struct pt_regs *regs)
2448{
2449 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2450 die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2451}
2452
2453void do_tof_tl1(struct pt_regs *regs)
2454{
2455 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2456 die_if_kernel("TL1: Tag Overflow Exception", regs);
2457}
2458
2459void do_getpsr(struct pt_regs *regs)
2460{
2461 regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2462 regs->tpc = regs->tnpc;
2463 regs->tnpc += 4;
2464 if (test_thread_flag(TIF_32BIT)) {
2465 regs->tpc &= 0xffffffff;
2466 regs->tnpc &= 0xffffffff;
2467 }
2468}
2469
56fb4df6
DM		 2470struct trap_per_cpu trap_block[NR_CPUS];
2471
2472/* This can get invoked before sched_init() so play it super safe
2473 * and use hard_smp_processor_id().
2474 */
72aff53f 2475void init_cur_cpu_trap(struct thread_info *t)
56fb4df6
DM		 2476{
2477 int cpu = hard_smp_processor_id();
2478 struct trap_per_cpu *p = &trap_block[cpu];
2479
72aff53f 2480 p->thread = t;
56fb4df6
DM		 2481 p->pgd_paddr = 0;
2482}
2483
1da177e4 2484extern void thread_info_offsets_are_bolixed_dave(void);
56fb4df6 2485extern void trap_per_cpu_offsets_are_bolixed_dave(void);
dcc1e8dd 2486extern void tsb_config_offsets_are_bolixed_dave(void);
1da177e4
LT		 2487
2488/* Only invoked on boot processor. */
2489void __init trap_init(void)
2490{
2491 /* Compile time sanity check. */
2492 if (TI_TASK != offsetof(struct thread_info, task) ||
2493 TI_FLAGS != offsetof(struct thread_info, flags) ||
2494 TI_CPU != offsetof(struct thread_info, cpu) ||
2495 TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2496 TI_KSP != offsetof(struct thread_info, ksp) ||
2497 TI_FAULT_ADDR != offsetof(struct thread_info, fault_address) ||
2498 TI_KREGS != offsetof(struct thread_info, kregs) ||
2499 TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2500 TI_EXEC_DOMAIN != offsetof(struct thread_info, exec_domain) ||
2501 TI_REG_WINDOW != offsetof(struct thread_info, reg_window) ||
2502 TI_RWIN_SPTRS != offsetof(struct thread_info, rwbuf_stkptrs) ||
2503 TI_GSR != offsetof(struct thread_info, gsr) ||
2504 TI_XFSR != offsetof(struct thread_info, xfsr) ||
2505 TI_USER_CNTD0 != offsetof(struct thread_info, user_cntd0) ||
2506 TI_USER_CNTD1 != offsetof(struct thread_info, user_cntd1) ||
2507 TI_KERN_CNTD0 != offsetof(struct thread_info, kernel_cntd0) ||
2508 TI_KERN_CNTD1 != offsetof(struct thread_info, kernel_cntd1) ||
2509 TI_PCR != offsetof(struct thread_info, pcr_reg) ||
1da177e4 2510 TI_PRE_COUNT != offsetof(struct thread_info, preempt_count) ||
db7d9a4e
DM		 2511 TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2512 TI_SYS_NOERROR != offsetof(struct thread_info, syscall_noerror) ||
a3f99858
DM		 2513 TI_RESTART_BLOCK != offsetof(struct thread_info, restart_block) ||
2514 TI_KUNA_REGS != offsetof(struct thread_info, kern_una_regs) ||
2515 TI_KUNA_INSN != offsetof(struct thread_info, kern_una_insn) ||
1da177e4
LT		 2516 TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2517 (TI_FPREGS & (64 - 1)))
2518 thread_info_offsets_are_bolixed_dave();
2519
56fb4df6 2520 if (TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu, thread) ||
e088ad7c
DM		 2521 (TRAP_PER_CPU_PGD_PADDR !=
2522 offsetof(struct trap_per_cpu, pgd_paddr)) ||
2523 (TRAP_PER_CPU_CPU_MONDO_PA !=
2524 offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2525 (TRAP_PER_CPU_DEV_MONDO_PA !=
2526 offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2527 (TRAP_PER_CPU_RESUM_MONDO_PA !=
2528 offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
5b0c0572
DM		 2529 (TRAP_PER_CPU_RESUM_KBUF_PA !=
2530 offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
e088ad7c
DM		 2531 (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2532 offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
5b0c0572
DM		 2533 (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2534 offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
e088ad7c 2535 (TRAP_PER_CPU_FAULT_INFO !=
1d2f1f90
DM		 2536 offsetof(struct trap_per_cpu, fault_info)) ||
2537 (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2538 offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2539 (TRAP_PER_CPU_CPU_LIST_PA !=
dcc1e8dd
DM		 2540 offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2541 (TRAP_PER_CPU_TSB_HUGE !=
2542 offsetof(struct trap_per_cpu, tsb_huge)) ||
2543 (TRAP_PER_CPU_TSB_HUGE_TEMP !=
2544 offsetof(struct trap_per_cpu, tsb_huge_temp)))
56fb4df6
DM		 2545 trap_per_cpu_offsets_are_bolixed_dave();
2546
dcc1e8dd
DM		 2547 if ((TSB_CONFIG_TSB !=
2548 offsetof(struct tsb_config, tsb)) ||
2549 (TSB_CONFIG_RSS_LIMIT !=
2550 offsetof(struct tsb_config, tsb_rss_limit)) ||
2551 (TSB_CONFIG_NENTRIES !=
2552 offsetof(struct tsb_config, tsb_nentries)) ||
2553 (TSB_CONFIG_REG_VAL !=
2554 offsetof(struct tsb_config, tsb_reg_val)) ||
2555 (TSB_CONFIG_MAP_VADDR !=
2556 offsetof(struct tsb_config, tsb_map_vaddr)) ||
2557 (TSB_CONFIG_MAP_PTE !=
2558 offsetof(struct tsb_config, tsb_map_pte)))
2559 tsb_config_offsets_are_bolixed_dave();
2560
1da177e4
LT		 2561 /* Attach to the address space of init_task. On SMP we
2562 * do this in smp.c:smp_callin for other cpus.
2563 */
2564 atomic_inc(&init_mm.mm_count);
2565 current->active_mm = &init_mm;
2566}
Linux 6.x block layer and io_uring tree(s)