[linux-block.git] / arch / mips / mm / cerr-sb1.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2001,2002,2003 Broadcom Corporation
 */
#include <linux/sched.h>
#include <asm/mipsregs.h>
#include <asm/sibyte/sb1250.h>
#include <asm/sibyte/sb1250_regs.h>

#if !defined(CONFIG_SIBYTE_BUS_WATCHER) || defined(CONFIG_SIBYTE_BW_TRACE)
#include <asm/io.h>
#include <asm/sibyte/sb1250_scd.h>
#endif

/*
 * We'd like to dump the L2_ECC_TAG register on errors, but errata make
 * that unsafe... So for now we don't.	(BCM1250/BCM112x erratum SOC-48.)
 */
#undef DUMP_L2_ECC_TAG_ON_ERROR

/* SB1 definitions */

/* XXX should come from config1 XXX */
#define SB1_CACHE_INDEX_MASK   0x1fe0

#define CP0_ERRCTL_RECOVERABLE (1 << 31)
#define CP0_ERRCTL_DCACHE      (1 << 30)
#define CP0_ERRCTL_ICACHE      (1 << 29)
#define CP0_ERRCTL_MULTIBUS    (1 << 23)
#define CP0_ERRCTL_MC_TLB      (1 << 15)
#define CP0_ERRCTL_MC_TIMEOUT  (1 << 14)

#define CP0_CERRI_TAG_PARITY   (1 << 29)
#define CP0_CERRI_DATA_PARITY  (1 << 28)
#define CP0_CERRI_EXTERNAL     (1 << 26)

#define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
#define CP0_CERRI_DATA	       (CP0_CERRI_DATA_PARITY)

#define CP0_CERRD_MULTIPLE     (1 << 31)
#define CP0_CERRD_TAG_STATE    (1 << 30)
#define CP0_CERRD_TAG_ADDRESS  (1 << 29)
#define CP0_CERRD_DATA_SBE     (1 << 28)
#define CP0_CERRD_DATA_DBE     (1 << 27)
#define CP0_CERRD_EXTERNAL     (1 << 26)
#define CP0_CERRD_LOAD	       (1 << 25)
#define CP0_CERRD_STORE	       (1 << 24)
#define CP0_CERRD_FILLWB       (1 << 23)
#define CP0_CERRD_COHERENCY    (1 << 22)
#define CP0_CERRD_DUPTAG       (1 << 21)

#define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
#define CP0_CERRD_IDX_VALID(c) \
   (((c) & (CP0_CERRD_LOAD | CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
#define CP0_CERRD_CAUSES \
   (CP0_CERRD_LOAD | CP0_CERRD_STORE | CP0_CERRD_FILLWB | CP0_CERRD_COHERENCY | CP0_CERRD_DUPTAG)
#define CP0_CERRD_TYPES \
   (CP0_CERRD_TAG_STATE | CP0_CERRD_TAG_ADDRESS | CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE | CP0_CERRD_EXTERNAL)
#define CP0_CERRD_DATA	       (CP0_CERRD_DATA_SBE | CP0_CERRD_DATA_DBE)

static uint32_t extract_ic(unsigned short addr, int data);
static uint32_t extract_dc(unsigned short addr, int data);

static inline void breakout_errctl(unsigned int val)
{
	if (val & CP0_ERRCTL_RECOVERABLE)
		printk(" recoverable");
	if (val & CP0_ERRCTL_DCACHE)
		printk(" dcache");
	if (val & CP0_ERRCTL_ICACHE)
		printk(" icache");
	if (val & CP0_ERRCTL_MULTIBUS)
		printk(" multiple-buserr");
	printk("\n");
}

static inline void breakout_cerri(unsigned int val)
{
	if (val & CP0_CERRI_TAG_PARITY)
		printk(" tag-parity");
	if (val & CP0_CERRI_DATA_PARITY)
		printk(" data-parity");
	if (val & CP0_CERRI_EXTERNAL)
		printk(" external");
	printk("\n");
}

static inline void breakout_cerrd(unsigned int val)
{
	switch (val & CP0_CERRD_CAUSES) {
	case CP0_CERRD_LOAD:
		printk(" load,");
		break;
	case CP0_CERRD_STORE:
		printk(" store,");
		break;
	case CP0_CERRD_FILLWB:
		printk(" fill/wb,");
		break;
	case CP0_CERRD_COHERENCY:
		printk(" coherency,");
		break;
	case CP0_CERRD_DUPTAG:
		printk(" duptags,");
		break;
	default:
		printk(" NO CAUSE,");
		break;
	}
	if (!(val & CP0_CERRD_TYPES))
		printk(" NO TYPE");
	else {
		if (val & CP0_CERRD_MULTIPLE)
			printk(" multi-err");
		if (val & CP0_CERRD_TAG_STATE)
			printk(" tag-state");
		if (val & CP0_CERRD_TAG_ADDRESS)
			printk(" tag-address");
		if (val & CP0_CERRD_DATA_SBE)
			printk(" data-SBE");
		if (val & CP0_CERRD_DATA_DBE)
			printk(" data-DBE");
		if (val & CP0_CERRD_EXTERNAL)
			printk(" external");
	}
	printk("\n");
}

#ifndef CONFIG_SIBYTE_BUS_WATCHER

static void check_bus_watcher(void)
{
	uint32_t status, l2_err, memio_err;
#ifdef DUMP_L2_ECC_TAG_ON_ERROR
	uint64_t l2_tag;
#endif

	/* Destructive read, clears register and interrupt */
	status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
	/* Bit 31 is always on, but there's no #define for that */
	if (status & ~(1UL << 31)) {
		l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
#ifdef DUMP_L2_ECC_TAG_ON_ERROR
		l2_tag = in64(IOADDR(A_L2_ECC_TAG));
#endif
		memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
		printk("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
		printk("\nLast recorded signature:\n");
		printk("Request %02x from %d, answered by %d with Dcode %d\n",
		       (unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
		       (int)(G_SCD_BERR_TID(status) >> 6),
		       (int)G_SCD_BERR_RID(status),
		       (int)G_SCD_BERR_DCODE(status));
#ifdef DUMP_L2_ECC_TAG_ON_ERROR
		printk("Last L2 tag w/ bad ECC: %016llx\n", l2_tag);
#endif
	} else {
		printk("Bus watcher indicates no error\n");
	}
}
#else
extern void check_bus_watcher(void);
#endif

asmlinkage void sb1_cache_error(void)
{
	uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
	unsigned long long cerr_dpa;

#ifdef CONFIG_SIBYTE_BW_TRACE
	/* Freeze the trace buffer now */
	csr_out32(M_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG));
	printk("Trace buffer frozen\n");
#endif

	printk("Cache error exception on CPU %x:\n",
	       (read_c0_prid() >> 25) & 0x7);

	__asm__ __volatile__ (
	"	.set	push\n\t"
	"	.set	mips64\n\t"
	"	.set	noat\n\t"
	"	mfc0	%0, $26\n\t"
	"	mfc0	%1, $27\n\t"
	"	mfc0	%2, $27, 1\n\t"
	"	dmfc0	$1, $27, 3\n\t"
	"	dsrl32	%3, $1, 0 \n\t"
	"	sll	%4, $1, 0 \n\t"
	"	mfc0	%5, $30\n\t"
	"	.set	pop"
	: "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
	  "=r" (dpahi), "=r" (dpalo), "=r" (eepc));

	cerr_dpa = (((uint64_t)dpahi) << 32) | dpalo;
	printk(" c0_errorepc ==	  %08x\n", eepc);
	printk(" c0_errctl   ==	  %08x", errctl);
	breakout_errctl(errctl);
	if (errctl & CP0_ERRCTL_ICACHE) {
		printk(" c0_cerr_i   ==	  %08x", cerr_i);
		breakout_cerri(cerr_i);
		if (CP0_CERRI_IDX_VALID(cerr_i)) {
			/* Check index of EPC, allowing for delay slot */
			if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
			    ((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
				printk(" cerr_i idx doesn't match eepc\n");
			else {
				res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
						 (cerr_i & CP0_CERRI_DATA) != 0);
				if (!(res & cerr_i))
					printk("...didn't see indicated icache problem\n");
			}
		}
	}
	if (errctl & CP0_ERRCTL_DCACHE) {
		printk(" c0_cerr_d   ==	  %08x", cerr_d);
		breakout_cerrd(cerr_d);
		if (CP0_CERRD_DPA_VALID(cerr_d)) {
			printk(" c0_cerr_dpa == %010llx\n", cerr_dpa);
			if (!CP0_CERRD_IDX_VALID(cerr_d)) {
				res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
						 (cerr_d & CP0_CERRD_DATA) != 0);
				if (!(res & cerr_d))
					printk("...didn't see indicated dcache problem\n");
			} else {
				if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
					printk(" cerr_d idx doesn't match cerr_dpa\n");
				else {
					res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
							 (cerr_d & CP0_CERRD_DATA) != 0);
					if (!(res & cerr_d))
						printk("...didn't see indicated problem\n");
				}
			}
		}
	}

	check_bus_watcher();

	/*
	 * Calling panic() when a fatal cache error occurs scrambles the
	 * state of the system (and the cache), making it difficult to
	 * investigate after the fact.	However, if you just stall the CPU,
	 * the other CPU may keep on running, which is typically very
	 * undesirable.
	 */
#ifdef CONFIG_SB1_CERR_STALL
	while (1)
		;
#else
	panic("unhandled cache error");
#endif
}


/* Parity lookup table. */
static const uint8_t parity[256] = {
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0
};

/* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
static const uint64_t mask_72_64[8] = {
	0x0738C808099264FFULL,
	0x38C808099264FF07ULL,
	0xC808099264FF0738ULL,
	0x08099264FF0738C8ULL,
	0x099264FF0738C808ULL,
	0x9264FF0738C80809ULL,
	0x64FF0738C8080992ULL,
	0xFF0738C808099264ULL
};

/* Calculate the parity on a range of bits */
static char range_parity(uint64_t dword, int max, int min)
{
	char parity = 0;
	int i;
	dword >>= min;
	for (i=max-min; i>=0; i--) {
		if (dword & 0x1)
			parity = !parity;
		dword >>= 1;
	}
	return parity;
}

/* Calculate the 4-bit even byte-parity for an instruction */
static unsigned char inst_parity(uint32_t word)
{
	int i, j;
	char parity = 0;
	for (j=0; j<4; j++) {
		char byte_parity = 0;
		for (i=0; i<8; i++) {
			if (word & 0x80000000)
				byte_parity = !byte_parity;
			word <<= 1;
		}
		parity <<= 1;
		parity |= byte_parity;
	}
	return parity;
}

static uint32_t extract_ic(unsigned short addr, int data)
{
	unsigned short way;
	int valid;
	uint32_t taghi, taglolo, taglohi;
	unsigned long long taglo, va;
	uint64_t tlo_tmp;
	uint8_t lru;
	int res = 0;

	printk("Icache index 0x%04x  ", addr);
	for (way = 0; way < 4; way++) {
		/* Index-load-tag-I */
		__asm__ __volatile__ (
		"	.set	push		\n\t"
		"	.set	noreorder	\n\t"
		"	.set	mips64		\n\t"
		"	.set	noat		\n\t"
		"	cache	4, 0(%3)	\n\t"
		"	mfc0	%0, $29		\n\t"
		"	dmfc0	$1, $28		\n\t"
		"	dsrl32	%1, $1, 0	\n\t"
		"	sll	%2, $1, 0	\n\t"
		"	.set	pop"
		: "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
		: "r" ((way << 13) | addr));

		taglo = ((unsigned long long)taglohi << 32) | taglolo;
		if (way == 0) {
			lru = (taghi >> 14) & 0xff;
			printk("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
				    ((addr >> 5) & 0x3), /* bank */
				    ((addr >> 7) & 0x3f), /* index */
				    (lru & 0x3),
				    ((lru >> 2) & 0x3),
				    ((lru >> 4) & 0x3),
				    ((lru >> 6) & 0x3));
		}
		va = (taglo & 0xC0000FFFFFFFE000ULL) | addr;
		if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
			va |= 0x3FFFF00000000000ULL;
		valid = ((taghi >> 29) & 1);
		if (valid) {
			tlo_tmp = taglo & 0xfff3ff;
			if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
				printk("   ** bad parity in VTag0/G/ASID\n");
				res |= CP0_CERRI_TAG_PARITY;
			}
			if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
				printk("   ** bad parity in R/VTag1\n");
				res |= CP0_CERRI_TAG_PARITY;
			}
		}
		if (valid ^ ((taghi >> 27) & 1)) {
			printk("   ** bad parity for valid bit\n");
			res |= CP0_CERRI_TAG_PARITY;
		}
		printk(" %d  [VA %016llx]  [Vld? %d]  raw tags: %08X-%016llX\n",
			    way, va, valid, taghi, taglo);

		if (data) {
			uint32_t datahi, insta, instb;
			uint8_t predecode;
			int offset;

			/* (hit all banks and ways) */
			for (offset = 0; offset < 4; offset++) {
				/* Index-load-data-I */
				__asm__ __volatile__ (
				"	.set	push\n\t"
				"	.set	noreorder\n\t"
				"	.set	mips64\n\t"
				"	.set	noat\n\t"
				"	cache	6, 0(%3)  \n\t"
				"	mfc0	%0, $29, 1\n\t"
				"	dmfc0  $1, $28, 1\n\t"
				"	dsrl32 %1, $1, 0 \n\t"
				"	sll    %2, $1, 0 \n\t"
				"	.set	pop	    \n"
				: "=r" (datahi), "=r" (insta), "=r" (instb)
				: "r" ((way << 13) | addr | (offset << 3)));
				predecode = (datahi >> 8) & 0xff;
				if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
					printk("   ** bad parity in predecode\n");
					res |= CP0_CERRI_DATA_PARITY;
				}
				/* XXXKW should/could check predecode bits themselves */
				if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
					printk("   ** bad parity in instruction a\n");
					res |= CP0_CERRI_DATA_PARITY;
				}
				if ((datahi & 0xf) ^ inst_parity(instb)) {
					printk("   ** bad parity in instruction b\n");
					res |= CP0_CERRI_DATA_PARITY;
				}
				printk("  %05X-%08X%08X", datahi, insta, instb);
			}
			printk("\n");
		}
	}
	return res;
}

/* Compute the ECC for a data doubleword */
static uint8_t dc_ecc(uint64_t dword)
{
	uint64_t t;
	uint32_t w;
	uint8_t	 p;
	int	 i;

	p = 0;
	for (i = 7; i >= 0; i--)
	{
		p <<= 1;
		t = dword & mask_72_64[i];
		w = (uint32_t)(t >> 32);
		p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
		      ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
		w = (uint32_t)(t & 0xFFFFFFFF);
		p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
		      ^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
	}
	return p;
}

struct dc_state {
	unsigned char val;
	char *name;
};

static struct dc_state dc_states[] = {
	{ 0x00, "INVALID" },
	{ 0x0f, "COH-SHD" },
	{ 0x13, "NCO-E-C" },
	{ 0x19, "NCO-E-D" },
	{ 0x16, "COH-E-C" },
	{ 0x1c, "COH-E-D" },
	{ 0xff, "*ERROR*" }
};

#define DC_TAG_VALID(state) \
    (((state) == 0x0) || ((state) == 0xf) || ((state) == 0x13) || \
     ((state) == 0x19) || ((state) == 0x16) || ((state) == 0x1c))

static char *dc_state_str(unsigned char state)
{
	struct dc_state *dsc = dc_states;
	while (dsc->val != 0xff) {
		if (dsc->val == state)
			break;
		dsc++;
	}
	return dsc->name;
}

static uint32_t extract_dc(unsigned short addr, int data)
{
	int valid, way;
	unsigned char state;
	uint32_t taghi, taglolo, taglohi;
	unsigned long long taglo, pa;
	uint8_t ecc, lru;
	int res = 0;

	printk("Dcache index 0x%04x  ", addr);
	for (way = 0; way < 4; way++) {
		__asm__ __volatile__ (
		"	.set	push\n\t"
		"	.set	noreorder\n\t"
		"	.set	mips64\n\t"
		"	.set	noat\n\t"
		"	cache	5, 0(%3)\n\t"	/* Index-load-tag-D */
		"	mfc0	%0, $29, 2\n\t"
		"	dmfc0	$1, $28, 2\n\t"
		"	dsrl32	%1, $1, 0\n\t"
		"	sll	%2, $1, 0\n\t"
		"	.set	pop"
		: "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
		: "r" ((way << 13) | addr));

		taglo = ((unsigned long long)taglohi << 32) | taglolo;
		pa = (taglo & 0xFFFFFFE000ULL) | addr;
		if (way == 0) {
			lru = (taghi >> 14) & 0xff;
			printk("[Bank %d Set 0x%02x]  LRU > %d %d %d %d > MRU\n",
				    ((addr >> 11) & 0x2) | ((addr >> 5) & 1), /* bank */
				    ((addr >> 6) & 0x3f), /* index */
				    (lru & 0x3),
				    ((lru >> 2) & 0x3),
				    ((lru >> 4) & 0x3),
				    ((lru >> 6) & 0x3));
		}
		state = (taghi >> 25) & 0x1f;
		valid = DC_TAG_VALID(state);
		printk(" %d  [PA %010llx]  [state %s (%02x)]  raw tags: %08X-%016llX\n",
			    way, pa, dc_state_str(state), state, taghi, taglo);
		if (valid) {
			if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
				printk("   ** bad parity in PTag1\n");
				res |= CP0_CERRD_TAG_ADDRESS;
			}
			if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
				printk("   ** bad parity in PTag0\n");
				res |= CP0_CERRD_TAG_ADDRESS;
			}
		} else {
			res |= CP0_CERRD_TAG_STATE;
		}

		if (data) {
			uint32_t datalohi, datalolo, datahi;
			unsigned long long datalo;
			int offset;
			char bad_ecc = 0;

			for (offset = 0; offset < 4; offset++) {
				/* Index-load-data-D */
				__asm__ __volatile__ (
				"	.set	push\n\t"
				"	.set	noreorder\n\t"
				"	.set	mips64\n\t"
				"	.set	noat\n\t"
				"	cache	7, 0(%3)\n\t" /* Index-load-data-D */
				"	mfc0	%0, $29, 3\n\t"
				"	dmfc0	$1, $28, 3\n\t"
				"	dsrl32	%1, $1, 0 \n\t"
				"	sll	%2, $1, 0 \n\t"
				"	.set	pop"
				: "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
				: "r" ((way << 13) | addr | (offset << 3)));
				datalo = ((unsigned long long)datalohi << 32) | datalolo;
				ecc = dc_ecc(datalo);
				if (ecc != datahi) {
					int bits;
					bad_ecc |= 1 << (3-offset);
					ecc ^= datahi;
					bits = hweight8(ecc);
					res |= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
				}
				printk("  %02X-%016llX", datahi, datalo);
			}
			printk("\n");
			if (bad_ecc)
				printk("  dwords w/ bad ECC: %d %d %d %d\n",
				       !!(bad_ecc & 8), !!(bad_ecc & 4),
				       !!(bad_ecc & 2), !!(bad_ecc & 1));
		}
	}
	return res;
}
Commit	Line	Data
1a59d1b8	1	// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4 LT	2	/*
1da177e4 LT	3	* Copyright (C) 2001,2002,2003 Broadcom Corporation
1da177e4	4	*/
1da177e4 LT	5	#include <linux/sched.h>
	6	#include <asm/mipsregs.h>
	7	#include <asm/sibyte/sb1250.h>
a4b5bd9a	8	#include <asm/sibyte/sb1250_regs.h>
1da177e4	9
a4b5bd9a	10	#if !defined(CONFIG_SIBYTE_BUS_WATCHER) \|\| defined(CONFIG_SIBYTE_BW_TRACE)
1da177e4	11	#include <asm/io.h>
1da177e4 LT	12	#include <asm/sibyte/sb1250_scd.h>
1da177e4 LT	13	#endif
42a3b4f2	14
a4b5bd9a AI	15	/*
a4b5bd9a AI	16	* We'd like to dump the L2_ECC_TAG register on errors, but errata make
70342287	17	* that unsafe... So for now we don't. (BCM1250/BCM112x erratum SOC-48.)
a4b5bd9a AI	18	*/
	19	#undef DUMP_L2_ECC_TAG_ON_ERROR
	20
1da177e4 LT	21	/* SB1 definitions */
	22
	23	/* XXX should come from config1 XXX */
	24	#define SB1_CACHE_INDEX_MASK 0x1fe0
	25
	26	#define CP0_ERRCTL_RECOVERABLE (1 << 31)
	27	#define CP0_ERRCTL_DCACHE (1 << 30)
	28	#define CP0_ERRCTL_ICACHE (1 << 29)
	29	#define CP0_ERRCTL_MULTIBUS (1 << 23)
	30	#define CP0_ERRCTL_MC_TLB (1 << 15)
	31	#define CP0_ERRCTL_MC_TIMEOUT (1 << 14)
	32
	33	#define CP0_CERRI_TAG_PARITY (1 << 29)
	34	#define CP0_CERRI_DATA_PARITY (1 << 28)
	35	#define CP0_CERRI_EXTERNAL (1 << 26)
	36
	37	#define CP0_CERRI_IDX_VALID(c) (!((c) & CP0_CERRI_EXTERNAL))
70342287	38	#define CP0_CERRI_DATA (CP0_CERRI_DATA_PARITY)
1da177e4 LT	39
	40	#define CP0_CERRD_MULTIPLE (1 << 31)
	41	#define CP0_CERRD_TAG_STATE (1 << 30)
	42	#define CP0_CERRD_TAG_ADDRESS (1 << 29)
	43	#define CP0_CERRD_DATA_SBE (1 << 28)
	44	#define CP0_CERRD_DATA_DBE (1 << 27)
	45	#define CP0_CERRD_EXTERNAL (1 << 26)
70342287 RB	46	#define CP0_CERRD_LOAD (1 << 25)
70342287 RB	47	#define CP0_CERRD_STORE (1 << 24)
1da177e4 LT	48	#define CP0_CERRD_FILLWB (1 << 23)
	49	#define CP0_CERRD_COHERENCY (1 << 22)
	50	#define CP0_CERRD_DUPTAG (1 << 21)
	51
	52	#define CP0_CERRD_DPA_VALID(c) (!((c) & CP0_CERRD_EXTERNAL))
	53	#define CP0_CERRD_IDX_VALID(c) \
	54	(((c) & (CP0_CERRD_LOAD \| CP0_CERRD_STORE)) ? (!((c) & CP0_CERRD_EXTERNAL)) : 0)
	55	#define CP0_CERRD_CAUSES \
	56	(CP0_CERRD_LOAD \| CP0_CERRD_STORE \| CP0_CERRD_FILLWB \| CP0_CERRD_COHERENCY \| CP0_CERRD_DUPTAG)
	57	#define CP0_CERRD_TYPES \
	58	(CP0_CERRD_TAG_STATE \| CP0_CERRD_TAG_ADDRESS \| CP0_CERRD_DATA_SBE \| CP0_CERRD_DATA_DBE \| CP0_CERRD_EXTERNAL)
70342287	59	#define CP0_CERRD_DATA (CP0_CERRD_DATA_SBE \| CP0_CERRD_DATA_DBE)
1da177e4	60
70342287 RB	61	static uint32_t extract_ic(unsigned short addr, int data);
70342287 RB	62	static uint32_t extract_dc(unsigned short addr, int data);
1da177e4 LT	63
	64	static inline void breakout_errctl(unsigned int val)
	65	{
	66	if (val & CP0_ERRCTL_RECOVERABLE)
36a88530	67	printk(" recoverable");
1da177e4	68	if (val & CP0_ERRCTL_DCACHE)
36a88530	69	printk(" dcache");
1da177e4	70	if (val & CP0_ERRCTL_ICACHE)
36a88530	71	printk(" icache");
1da177e4	72	if (val & CP0_ERRCTL_MULTIBUS)
36a88530 RB	73	printk(" multiple-buserr");
36a88530 RB	74	printk("\n");
1da177e4 LT	75	}
	76
	77	static inline void breakout_cerri(unsigned int val)
	78	{
	79	if (val & CP0_CERRI_TAG_PARITY)
36a88530	80	printk(" tag-parity");
1da177e4	81	if (val & CP0_CERRI_DATA_PARITY)
36a88530	82	printk(" data-parity");
1da177e4	83	if (val & CP0_CERRI_EXTERNAL)
36a88530 RB	84	printk(" external");
36a88530 RB	85	printk("\n");
1da177e4 LT	86	}
	87
	88	static inline void breakout_cerrd(unsigned int val)
	89	{
	90	switch (val & CP0_CERRD_CAUSES) {
	91	case CP0_CERRD_LOAD:
36a88530	92	printk(" load,");
1da177e4 LT	93	break;
1da177e4 LT	94	case CP0_CERRD_STORE:
36a88530	95	printk(" store,");
1da177e4 LT	96	break;
1da177e4 LT	97	case CP0_CERRD_FILLWB:
36a88530	98	printk(" fill/wb,");
1da177e4 LT	99	break;
1da177e4 LT	100	case CP0_CERRD_COHERENCY:
36a88530	101	printk(" coherency,");
1da177e4 LT	102	break;
1da177e4 LT	103	case CP0_CERRD_DUPTAG:
36a88530	104	printk(" duptags,");
1da177e4 LT	105	break;
1da177e4 LT	106	default:
36a88530	107	printk(" NO CAUSE,");
1da177e4 LT	108	break;
	109	}
	110	if (!(val & CP0_CERRD_TYPES))
36a88530	111	printk(" NO TYPE");
1da177e4 LT	112	else {
1da177e4 LT	113	if (val & CP0_CERRD_MULTIPLE)
36a88530	114	printk(" multi-err");
1da177e4	115	if (val & CP0_CERRD_TAG_STATE)
36a88530	116	printk(" tag-state");
1da177e4	117	if (val & CP0_CERRD_TAG_ADDRESS)
36a88530	118	printk(" tag-address");
1da177e4	119	if (val & CP0_CERRD_DATA_SBE)
36a88530	120	printk(" data-SBE");
1da177e4	121	if (val & CP0_CERRD_DATA_DBE)
36a88530	122	printk(" data-DBE");
1da177e4	123	if (val & CP0_CERRD_EXTERNAL)
36a88530	124	printk(" external");
1da177e4	125	}
36a88530	126	printk("\n");
1da177e4 LT	127	}
	128
	129	#ifndef CONFIG_SIBYTE_BUS_WATCHER
	130
42a3b4f2 RB	131	static void check_bus_watcher(void)
42a3b4f2 RB	132	{
1da177e4	133	uint32_t status, l2_err, memio_err;
a4b5bd9a AI	134	#ifdef DUMP_L2_ECC_TAG_ON_ERROR
	135	uint64_t l2_tag;
	136	#endif
1da177e4 LT	137
	138	/* Destructive read, clears register and interrupt */
	139	status = csr_in32(IOADDR(A_SCD_BUS_ERR_STATUS));
	140	/* Bit 31 is always on, but there's no #define for that */
42a3b4f2	141	if (status & ~(1UL << 31)) {
1da177e4	142	l2_err = csr_in32(IOADDR(A_BUS_L2_ERRORS));
a4b5bd9a	143	#ifdef DUMP_L2_ECC_TAG_ON_ERROR
33b75e5c	144	l2_tag = in64(IOADDR(A_L2_ECC_TAG));
a4b5bd9a	145	#endif
1da177e4	146	memio_err = csr_in32(IOADDR(A_BUS_MEM_IO_ERRORS));
36a88530 RB	147	printk("Bus watcher error counters: %08x %08x\n", l2_err, memio_err);
	148	printk("\nLast recorded signature:\n");
	149	printk("Request %02x from %d, answered by %d with Dcode %d\n",
1da177e4 LT	150	(unsigned int)(G_SCD_BERR_TID(status) & 0x3f),
	151	(int)(G_SCD_BERR_TID(status) >> 6),
	152	(int)G_SCD_BERR_RID(status),
	153	(int)G_SCD_BERR_DCODE(status));
a4b5bd9a	154	#ifdef DUMP_L2_ECC_TAG_ON_ERROR
36a88530	155	printk("Last L2 tag w/ bad ECC: %016llx\n", l2_tag);
a4b5bd9a	156	#endif
42a3b4f2	157	} else {
36a88530	158	printk("Bus watcher indicates no error\n");
42a3b4f2 RB	159	}
	160	}
	161	#else
	162	extern void check_bus_watcher(void);
	163	#endif
	164
1da177e4 LT	165	asmlinkage void sb1_cache_error(void)
1da177e4 LT	166	{
1da177e4	167	uint32_t errctl, cerr_i, cerr_d, dpalo, dpahi, eepc, res;
41a8198f	168	unsigned long long cerr_dpa;
1da177e4	169
a4b5bd9a AI	170	#ifdef CONFIG_SIBYTE_BW_TRACE
a4b5bd9a AI	171	/* Freeze the trace buffer now */
33b75e5c	172	csr_out32(M_SCD_TRACE_CFG_FREEZE, IOADDR(A_SCD_TRACE_CFG));
36a88530	173	printk("Trace buffer frozen\n");
a4b5bd9a AI	174	#endif
a4b5bd9a AI	175
36a88530 RB	176	printk("Cache error exception on CPU %x:\n",
36a88530 RB	177	(read_c0_prid() >> 25) & 0x7);
1da177e4 LT	178
	179	__asm__ __volatile__ (
	180	" .set push\n\t"
	181	" .set mips64\n\t"
	182	" .set noat\n\t"
	183	" mfc0 %0, $26\n\t"
	184	" mfc0 %1, $27\n\t"
	185	" mfc0 %2, $27, 1\n\t"
	186	" dmfc0 $1, $27, 3\n\t"
	187	" dsrl32 %3, $1, 0 \n\t"
	188	" sll %4, $1, 0 \n\t"
	189	" mfc0 %5, $30\n\t"
	190	" .set pop"
	191	: "=r" (errctl), "=r" (cerr_i), "=r" (cerr_d),
	192	"=r" (dpahi), "=r" (dpalo), "=r" (eepc));
	193
	194	cerr_dpa = (((uint64_t)dpahi) << 32) \| dpalo;
70342287 RB	195	printk(" c0_errorepc == %08x\n", eepc);
70342287 RB	196	printk(" c0_errctl == %08x", errctl);
1da177e4 LT	197	breakout_errctl(errctl);
1da177e4 LT	198	if (errctl & CP0_ERRCTL_ICACHE) {
70342287	199	printk(" c0_cerr_i == %08x", cerr_i);
1da177e4 LT	200	breakout_cerri(cerr_i);
	201	if (CP0_CERRI_IDX_VALID(cerr_i)) {
	202	/* Check index of EPC, allowing for delay slot */
	203	if (((eepc & SB1_CACHE_INDEX_MASK) != (cerr_i & SB1_CACHE_INDEX_MASK)) &&
	204	((eepc & SB1_CACHE_INDEX_MASK) != ((cerr_i & SB1_CACHE_INDEX_MASK) - 4)))
36a88530	205	printk(" cerr_i idx doesn't match eepc\n");
1da177e4 LT	206	else {
	207	res = extract_ic(cerr_i & SB1_CACHE_INDEX_MASK,
	208	(cerr_i & CP0_CERRI_DATA) != 0);
	209	if (!(res & cerr_i))
36a88530	210	printk("...didn't see indicated icache problem\n");
1da177e4 LT	211	}
	212	}
	213	}
	214	if (errctl & CP0_ERRCTL_DCACHE) {
70342287	215	printk(" c0_cerr_d == %08x", cerr_d);
1da177e4 LT	216	breakout_cerrd(cerr_d);
1da177e4 LT	217	if (CP0_CERRD_DPA_VALID(cerr_d)) {
36a88530	218	printk(" c0_cerr_dpa == %010llx\n", cerr_dpa);
1da177e4 LT	219	if (!CP0_CERRD_IDX_VALID(cerr_d)) {
	220	res = extract_dc(cerr_dpa & SB1_CACHE_INDEX_MASK,
	221	(cerr_d & CP0_CERRD_DATA) != 0);
	222	if (!(res & cerr_d))
36a88530	223	printk("...didn't see indicated dcache problem\n");
1da177e4 LT	224	} else {
1da177e4 LT	225	if ((cerr_dpa & SB1_CACHE_INDEX_MASK) != (cerr_d & SB1_CACHE_INDEX_MASK))
36a88530	226	printk(" cerr_d idx doesn't match cerr_dpa\n");
1da177e4 LT	227	else {
	228	res = extract_dc(cerr_d & SB1_CACHE_INDEX_MASK,
	229	(cerr_d & CP0_CERRD_DATA) != 0);
	230	if (!(res & cerr_d))
36a88530	231	printk("...didn't see indicated problem\n");
1da177e4 LT	232	}
	233	}
	234	}
	235	}
	236
	237	check_bus_watcher();
	238
1da177e4	239	/*
a4b5bd9a AI	240	* Calling panic() when a fatal cache error occurs scrambles the
a4b5bd9a AI	241	* state of the system (and the cache), making it difficult to
70342287	242	* investigate after the fact. However, if you just stall the CPU,
a4b5bd9a AI	243	* the other CPU may keep on running, which is typically very
a4b5bd9a AI	244	* undesirable.
1da177e4	245	*/
a4b5bd9a AI	246	#ifdef CONFIG_SB1_CERR_STALL
	247	while (1)
	248	;
	249	#else
	250	panic("unhandled cache error");
	251	#endif
1da177e4 LT	252	}
	253
	254
	255	/* Parity lookup table. */
	256	static const uint8_t parity[256] = {
21a151d8 RB	257	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	258	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	259	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	260	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	261	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	262	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	263	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	264	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	265	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	266	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	267	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	268	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	269	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
	270	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	271	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
	272	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0
1da177e4 LT	273	};
	274
	275	/* Masks to select bits for Hamming parity, mask_72_64[i] for bit[i] */
	276	static const uint64_t mask_72_64[8] = {
	277	0x0738C808099264FFULL,
	278	0x38C808099264FF07ULL,
	279	0xC808099264FF0738ULL,
	280	0x08099264FF0738C8ULL,
	281	0x099264FF0738C808ULL,
	282	0x9264FF0738C80809ULL,
	283	0x64FF0738C8080992ULL,
	284	0xFF0738C808099264ULL
	285	};
	286
	287	/* Calculate the parity on a range of bits */
	288	static char range_parity(uint64_t dword, int max, int min)
	289	{
	290	char parity = 0;
	291	int i;
	292	dword >>= min;
	293	for (i=max-min; i>=0; i--) {
	294	if (dword & 0x1)
	295	parity = !parity;
	296	dword >>= 1;
	297	}
	298	return parity;
	299	}
	300
	301	/* Calculate the 4-bit even byte-parity for an instruction */
	302	static unsigned char inst_parity(uint32_t word)
	303	{
	304	int i, j;
	305	char parity = 0;
	306	for (j=0; j<4; j++) {
	307	char byte_parity = 0;
	308	for (i=0; i<8; i++) {
	309	if (word & 0x80000000)
	310	byte_parity = !byte_parity;
	311	word <<= 1;
	312	}
	313	parity <<= 1;
	314	parity \|= byte_parity;
	315	}
	316	return parity;
	317	}
	318
	319	static uint32_t extract_ic(unsigned short addr, int data)
	320	{
	321	unsigned short way;
	322	int valid;
1da177e4	323	uint32_t taghi, taglolo, taglohi;
41a8198f RB	324	unsigned long long taglo, va;
41a8198f RB	325	uint64_t tlo_tmp;
1da177e4 LT	326	uint8_t lru;
	327	int res = 0;
	328
36a88530	329	printk("Icache index 0x%04x ", addr);
1da177e4 LT	330	for (way = 0; way < 4; way++) {
	331	/* Index-load-tag-I */
	332	__asm__ __volatile__ (
	333	" .set push \n\t"
	334	" .set noreorder \n\t"
	335	" .set mips64 \n\t"
	336	" .set noat \n\t"
	337	" cache 4, 0(%3) \n\t"
	338	" mfc0 %0, $29 \n\t"
	339	" dmfc0 $1, $28 \n\t"
	340	" dsrl32 %1, $1, 0 \n\t"
	341	" sll %2, $1, 0 \n\t"
	342	" .set pop"
	343	: "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
	344	: "r" ((way << 13) \| addr));
	345
	346	taglo = ((unsigned long long)taglohi << 32) \| taglolo;
	347	if (way == 0) {
	348	lru = (taghi >> 14) & 0xff;
36a88530	349	printk("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
1da177e4 LT	350	((addr >> 5) & 0x3), /* bank */
	351	((addr >> 7) & 0x3f), /* index */
	352	(lru & 0x3),
	353	((lru >> 2) & 0x3),
	354	((lru >> 4) & 0x3),
	355	((lru >> 6) & 0x3));
	356	}
	357	va = (taglo & 0xC0000FFFFFFFE000ULL) \| addr;
	358	if ((taglo & (1 << 31)) && (((taglo >> 62) & 0x3) == 3))
	359	va \|= 0x3FFFF00000000000ULL;
	360	valid = ((taghi >> 29) & 1);
	361	if (valid) {
	362	tlo_tmp = taglo & 0xfff3ff;
	363	if (((taglo >> 10) & 1) ^ range_parity(tlo_tmp, 23, 0)) {
36a88530	364	printk(" ** bad parity in VTag0/G/ASID\n");
1da177e4 LT	365	res \|= CP0_CERRI_TAG_PARITY;
	366	}
	367	if (((taglo >> 11) & 1) ^ range_parity(taglo, 63, 24)) {
36a88530	368	printk(" ** bad parity in R/VTag1\n");
1da177e4 LT	369	res \|= CP0_CERRI_TAG_PARITY;
	370	}
	371	}
	372	if (valid ^ ((taghi >> 27) & 1)) {
36a88530	373	printk(" ** bad parity for valid bit\n");
1da177e4 LT	374	res \|= CP0_CERRI_TAG_PARITY;
1da177e4 LT	375	}
36a88530	376	printk(" %d [VA %016llx] [Vld? %d] raw tags: %08X-%016llX\n",
1da177e4 LT	377	way, va, valid, taghi, taglo);
	378
	379	if (data) {
	380	uint32_t datahi, insta, instb;
	381	uint8_t predecode;
	382	int offset;
	383
	384	/* (hit all banks and ways) */
	385	for (offset = 0; offset < 4; offset++) {
	386	/* Index-load-data-I */
	387	__asm__ __volatile__ (
	388	" .set push\n\t"
	389	" .set noreorder\n\t"
	390	" .set mips64\n\t"
	391	" .set noat\n\t"
	392	" cache 6, 0(%3) \n\t"
	393	" mfc0 %0, $29, 1\n\t"
	394	" dmfc0 $1, $28, 1\n\t"
	395	" dsrl32 %1, $1, 0 \n\t"
	396	" sll %2, $1, 0 \n\t"
70342287	397	" .set pop \n"
1da177e4 LT	398	: "=r" (datahi), "=r" (insta), "=r" (instb)
	399	: "r" ((way << 13) \| addr \| (offset << 3)));
	400	predecode = (datahi >> 8) & 0xff;
	401	if (((datahi >> 16) & 1) != (uint32_t)range_parity(predecode, 7, 0)) {
36a88530	402	printk(" ** bad parity in predecode\n");
1da177e4 LT	403	res \|= CP0_CERRI_DATA_PARITY;
	404	}
	405	/* XXXKW should/could check predecode bits themselves */
	406	if (((datahi >> 4) & 0xf) ^ inst_parity(insta)) {
36a88530	407	printk(" ** bad parity in instruction a\n");
1da177e4 LT	408	res \|= CP0_CERRI_DATA_PARITY;
	409	}
	410	if ((datahi & 0xf) ^ inst_parity(instb)) {
36a88530	411	printk(" ** bad parity in instruction b\n");
1da177e4 LT	412	res \|= CP0_CERRI_DATA_PARITY;
1da177e4 LT	413	}
36a88530	414	printk(" %05X-%08X%08X", datahi, insta, instb);
1da177e4	415	}
36a88530	416	printk("\n");
1da177e4 LT	417	}
	418	}
	419	return res;
	420	}
	421
	422	/* Compute the ECC for a data doubleword */
	423	static uint8_t dc_ecc(uint64_t dword)
	424	{
	425	uint64_t t;
	426	uint32_t w;
70342287 RB	427	uint8_t p;
70342287 RB	428	int i;
1da177e4 LT	429
	430	p = 0;
	431	for (i = 7; i >= 0; i--)
	432	{
	433	p <<= 1;
	434	t = dword & mask_72_64[i];
	435	w = (uint32_t)(t >> 32);
	436	p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
	437	^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
	438	w = (uint32_t)(t & 0xFFFFFFFF);
	439	p ^= (parity[w>>24] ^ parity[(w>>16) & 0xFF]
	440	^ parity[(w>>8) & 0xFF] ^ parity[w & 0xFF]);
	441	}
	442	return p;
	443	}
	444
	445	struct dc_state {
	446	unsigned char val;
	447	char *name;
	448	};
	449
	450	static struct dc_state dc_states[] = {
	451	{ 0x00, "INVALID" },
	452	{ 0x0f, "COH-SHD" },
	453	{ 0x13, "NCO-E-C" },
	454	{ 0x19, "NCO-E-D" },
	455	{ 0x16, "COH-E-C" },
	456	{ 0x1c, "COH-E-D" },
	457	{ 0xff, "ERROR" }
	458	};
	459
	460	#define DC_TAG_VALID(state) \
a4b5bd9a AI	461	(((state) == 0x0) \|\| ((state) == 0xf) \|\| ((state) == 0x13) \|\| \
a4b5bd9a AI	462	((state) == 0x19) \|\| ((state) == 0x16) \|\| ((state) == 0x1c))
1da177e4 LT	463
	464	static char *dc_state_str(unsigned char state)
	465	{
	466	struct dc_state *dsc = dc_states;
	467	while (dsc->val != 0xff) {
	468	if (dsc->val == state)
	469	break;
	470	dsc++;
	471	}
	472	return dsc->name;
	473	}
	474
	475	static uint32_t extract_dc(unsigned short addr, int data)
	476	{
	477	int valid, way;
	478	unsigned char state;
1da177e4	479	uint32_t taghi, taglolo, taglohi;
41a8198f	480	unsigned long long taglo, pa;
1da177e4 LT	481	uint8_t ecc, lru;
	482	int res = 0;
	483
36a88530	484	printk("Dcache index 0x%04x ", addr);
1da177e4 LT	485	for (way = 0; way < 4; way++) {
	486	__asm__ __volatile__ (
	487	" .set push\n\t"
	488	" .set noreorder\n\t"
	489	" .set mips64\n\t"
	490	" .set noat\n\t"
	491	" cache 5, 0(%3)\n\t" /* Index-load-tag-D */
	492	" mfc0 %0, $29, 2\n\t"
	493	" dmfc0 $1, $28, 2\n\t"
	494	" dsrl32 %1, $1, 0\n\t"
	495	" sll %2, $1, 0\n\t"
	496	" .set pop"
	497	: "=r" (taghi), "=r" (taglohi), "=r" (taglolo)
	498	: "r" ((way << 13) \| addr));
	499
	500	taglo = ((unsigned long long)taglohi << 32) \| taglolo;
	501	pa = (taglo & 0xFFFFFFE000ULL) \| addr;
	502	if (way == 0) {
	503	lru = (taghi >> 14) & 0xff;
36a88530	504	printk("[Bank %d Set 0x%02x] LRU > %d %d %d %d > MRU\n",
1da177e4 LT	505	((addr >> 11) & 0x2) \| ((addr >> 5) & 1), /* bank */
	506	((addr >> 6) & 0x3f), /* index */
	507	(lru & 0x3),
	508	((lru >> 2) & 0x3),
	509	((lru >> 4) & 0x3),
	510	((lru >> 6) & 0x3));
	511	}
	512	state = (taghi >> 25) & 0x1f;
	513	valid = DC_TAG_VALID(state);
36a88530	514	printk(" %d [PA %010llx] [state %s (%02x)] raw tags: %08X-%016llX\n",
1da177e4 LT	515	way, pa, dc_state_str(state), state, taghi, taglo);
	516	if (valid) {
	517	if (((taglo >> 11) & 1) ^ range_parity(taglo, 39, 26)) {
36a88530	518	printk(" ** bad parity in PTag1\n");
1da177e4 LT	519	res \|= CP0_CERRD_TAG_ADDRESS;
	520	}
	521	if (((taglo >> 10) & 1) ^ range_parity(taglo, 25, 13)) {
36a88530	522	printk(" ** bad parity in PTag0\n");
1da177e4 LT	523	res \|= CP0_CERRD_TAG_ADDRESS;
	524	}
	525	} else {
	526	res \|= CP0_CERRD_TAG_STATE;
	527	}
	528
	529	if (data) {
1da177e4	530	uint32_t datalohi, datalolo, datahi;
41a8198f	531	unsigned long long datalo;
1da177e4	532	int offset;
a4b5bd9a	533	char bad_ecc = 0;
1da177e4 LT	534
	535	for (offset = 0; offset < 4; offset++) {
	536	/* Index-load-data-D */
	537	__asm__ __volatile__ (
	538	" .set push\n\t"
	539	" .set noreorder\n\t"
	540	" .set mips64\n\t"
	541	" .set noat\n\t"
	542	" cache 7, 0(%3)\n\t" /* Index-load-data-D */
	543	" mfc0 %0, $29, 3\n\t"
	544	" dmfc0 $1, $28, 3\n\t"
	545	" dsrl32 %1, $1, 0 \n\t"
	546	" sll %2, $1, 0 \n\t"
	547	" .set pop"
	548	: "=r" (datahi), "=r" (datalohi), "=r" (datalolo)
	549	: "r" ((way << 13) \| addr \| (offset << 3)));
	550	datalo = ((unsigned long long)datalohi << 32) \| datalolo;
	551	ecc = dc_ecc(datalo);
	552	if (ecc != datahi) {
13e79b46	553	int bits;
a4b5bd9a	554	bad_ecc \|= 1 << (3-offset);
1da177e4	555	ecc ^= datahi;
13e79b46	556	bits = hweight8(ecc);
1da177e4 LT	557	res \|= (bits == 1) ? CP0_CERRD_DATA_SBE : CP0_CERRD_DATA_DBE;
1da177e4 LT	558	}
36a88530	559	printk(" %02X-%016llX", datahi, datalo);
1da177e4	560	}
36a88530	561	printk("\n");
a4b5bd9a	562	if (bad_ecc)
36a88530 RB	563	printk(" dwords w/ bad ECC: %d %d %d %d\n",
	564	!!(bad_ecc & 8), !!(bad_ecc & 4),
	565	!!(bad_ecc & 2), !!(bad_ecc & 1));
1da177e4 LT	566	}
	567	}
	568	return res;
	569	}