[linux-2.6-block.git] / arch / microblaze / kernel / cpu / cache.c

/*
 * Cache control for MicroBlaze cache memories
 *
 * Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu>
 * Copyright (C) 2007-2009 PetaLogix
 * Copyright (C) 2007-2009 John Williams <john.williams@petalogix.com>
 *
 * This file is subject to the terms and conditions of the GNU General
 * Public License. See the file COPYING in the main directory of this
 * archive for more details.
 */

#include <asm/cacheflush.h>
#include <linux/cache.h>
#include <asm/cpuinfo.h>
#include <asm/pvr.h>

static inline void __invalidate_flush_icache(unsigned int addr)
{
	__asm__ __volatile__ ("wic	%0, r0;"	\
					: : "r" (addr));
}

static inline void __flush_dcache(unsigned int addr)
{
	__asm__ __volatile__ ("wdc.flush	%0, r0;"	\
					: : "r" (addr));
}

static inline void __invalidate_dcache(unsigned int baseaddr,
						unsigned int offset)
{
	__asm__ __volatile__ ("wdc.clear	%0, %1;"	\
					: : "r" (baseaddr), "r" (offset));
}

static inline void __enable_icache_msr(void)
{
	__asm__ __volatile__ ("	msrset	r0, %0;		\
				nop; "			\
			: : "i" (MSR_ICE) : "memory");
}

static inline void __disable_icache_msr(void)
{
	__asm__ __volatile__ ("	msrclr	r0, %0;		\
				nop; "			\
			: : "i" (MSR_ICE) : "memory");
}

static inline void __enable_dcache_msr(void)
{
	__asm__ __volatile__ ("	msrset	r0, %0;		\
				nop; "			\
				:			\
				: "i" (MSR_DCE)		\
				: "memory");
}

static inline void __disable_dcache_msr(void)
{
	__asm__ __volatile__ ("	msrclr	r0, %0;		\
				nop; "			\
				:			\
				: "i" (MSR_DCE)		\
				: "memory");
}

static inline void __enable_icache_nomsr(void)
{
	__asm__ __volatile__ ("	mfs	r12, rmsr;	\
				nop;			\
				ori	r12, r12, %0;	\
				mts	rmsr, r12;	\
				nop; "			\
				:			\
				: "i" (MSR_ICE)		\
				: "memory", "r12");
}

static inline void __disable_icache_nomsr(void)
{
	__asm__ __volatile__ ("	mfs	r12, rmsr;	\
				nop;			\
				andi	r12, r12, ~%0;	\
				mts	rmsr, r12;	\
				nop; "			\
				:			\
				: "i" (MSR_ICE)		\
				: "memory", "r12");
}

static inline void __enable_dcache_nomsr(void)
{
	__asm__ __volatile__ ("	mfs	r12, rmsr;	\
				nop;			\
				ori	r12, r12, %0;	\
				mts	rmsr, r12;	\
				nop; "			\
				:			\
				: "i" (MSR_DCE)		\
				: "memory", "r12");
}

static inline void __disable_dcache_nomsr(void)
{
	__asm__ __volatile__ ("	mfs	r12, rmsr;	\
				nop;			\
				andi	r12, r12, ~%0;	\
				mts	rmsr, r12;	\
				nop; "			\
				:			\
				: "i" (MSR_DCE)		\
				: "memory", "r12");
}


/* Helper macro for computing the limits of cache range loops */
#define CACHE_LOOP_LIMITS(start, end, cache_line_length, cache_size)	\
do {									\
	int align = ~(cache_line_length - 1);				\
	end = min(start + cache_size, end);				\
	start &= align;							\
	end = ((end & align) + cache_line_length);			\
} while (0);

/*
 * Helper macro to loop over the specified cache_size/line_length and
 * execute 'op' on that cacheline
 */
#define CACHE_ALL_LOOP(cache_size, line_length, op)			\
do {									\
	unsigned int len = cache_size;					\
	int step = -line_length;					\
	BUG_ON(step >= 0);						\
									\
	__asm__ __volatile__ (" 1:      " #op " %0, r0;			\
					bgtid   %0, 1b;			\
					addk    %0, %0, %1;		\
					" : : "r" (len), "r" (step)	\
					: "memory");			\
} while (0);


#define CACHE_ALL_LOOP2(cache_size, line_length, op)			\
do {									\
	unsigned int len = cache_size;					\
	int step = -line_length;					\
	BUG_ON(step >= 0);						\
									\
	__asm__ __volatile__ (" 1:      " #op " r0, %0;			\
					bgtid   %0, 1b;			\
					addk    %0, %0, %1;		\
					" : : "r" (len), "r" (step)	\
					: "memory");			\
} while (0);

/* for wdc.flush/clear */
#define CACHE_RANGE_LOOP_2(start, end, line_length, op)			\
do {									\
	int step = -line_length;					\
	int count = end - start;					\
	BUG_ON(count <= 0);						\
									\
	__asm__ __volatile__ (" 1:	" #op " %0, %1;			\
					bgtid   %1, 1b;			\
					addk    %1, %1, %2;		\
					" : : "r" (start), "r" (count),	\
					"r" (step) : "memory");		\
} while (0);

/* It is used only first parameter for OP - for wic, wdc */
#define CACHE_RANGE_LOOP_1(start, end, line_length, op)			\
do {									\
	int volatile temp;						\
	BUG_ON(end - start <= 0);					\
									\
	__asm__ __volatile__ (" 1:	" #op " %1, r0;			\
					cmpu	%0, %1, %2;		\
					bgtid	%0, 1b;			\
					addk	%1, %1, %3;		\
				" : : "r" (temp), "r" (start), "r" (end),\
					"r" (line_length) : "memory");	\
} while (0);

static void __flush_icache_range_msr_irq(unsigned long start, unsigned long end)
{
	unsigned long flags;

	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);

	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.icache_line_length, cpuinfo.icache_size);

	local_irq_save(flags);
	__disable_icache_msr();

	CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);

	__enable_icache_msr();
	local_irq_restore(flags);
}

static void __flush_icache_range_nomsr_irq(unsigned long start,
				unsigned long end)
{
	unsigned long flags;

	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);

	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.icache_line_length, cpuinfo.icache_size);

	local_irq_save(flags);
	__disable_icache_nomsr();

	CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);

	__enable_icache_nomsr();
	local_irq_restore(flags);
}

static void __flush_icache_range_noirq(unsigned long start,
				unsigned long end)
{
	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);

	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.icache_line_length, cpuinfo.icache_size);
	CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
}

static void __flush_icache_all_msr_irq(void)
{
	unsigned long flags;

	pr_debug("%s\n", __func__);

	local_irq_save(flags);
	__disable_icache_msr();

	CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);

	__enable_icache_msr();
	local_irq_restore(flags);
}

static void __flush_icache_all_nomsr_irq(void)
{
	unsigned long flags;

	pr_debug("%s\n", __func__);

	local_irq_save(flags);
	__disable_icache_nomsr();

	CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);

	__enable_icache_nomsr();
	local_irq_restore(flags);
}

static void __flush_icache_all_noirq(void)
{
	pr_debug("%s\n", __func__);
	CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
}

static void __invalidate_dcache_all_msr_irq(void)
{
	unsigned long flags;

	pr_debug("%s\n", __func__);

	local_irq_save(flags);
	__disable_dcache_msr();

	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);

	__enable_dcache_msr();
	local_irq_restore(flags);
}

static void __invalidate_dcache_all_nomsr_irq(void)
{
	unsigned long flags;

	pr_debug("%s\n", __func__);

	local_irq_save(flags);
	__disable_dcache_nomsr();

	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);

	__enable_dcache_nomsr();
	local_irq_restore(flags);
}

static void __invalidate_dcache_all_noirq_wt(void)
{
	pr_debug("%s\n", __func__);
	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc)
}

/* FIXME this is weird - should be only wdc but not work
 * MS: I am getting bus errors and other weird things */
static void __invalidate_dcache_all_wb(void)
{
	pr_debug("%s\n", __func__);
	CACHE_ALL_LOOP2(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
					wdc.clear)
}

static void __invalidate_dcache_range_wb(unsigned long start,
						unsigned long end)
{
	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);

	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.dcache_line_length, cpuinfo.dcache_size);
	CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.clear);
}

static void __invalidate_dcache_range_nomsr_wt(unsigned long start,
							unsigned long end)
{
	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);
	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.dcache_line_length, cpuinfo.dcache_size);

	CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
}

static void __invalidate_dcache_range_msr_irq_wt(unsigned long start,
							unsigned long end)
{
	unsigned long flags;

	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);
	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.dcache_line_length, cpuinfo.dcache_size);

	local_irq_save(flags);
	__disable_dcache_msr();

	CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);

	__enable_dcache_msr();
	local_irq_restore(flags);
}

static void __invalidate_dcache_range_nomsr_irq(unsigned long start,
							unsigned long end)
{
	unsigned long flags;

	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);

	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.dcache_line_length, cpuinfo.dcache_size);

	local_irq_save(flags);
	__disable_dcache_nomsr();

	CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);

	__enable_dcache_nomsr();
	local_irq_restore(flags);
}

static void __flush_dcache_all_wb(void)
{
	pr_debug("%s\n", __func__);
	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
				wdc.flush);
}

static void __flush_dcache_range_wb(unsigned long start, unsigned long end)
{
	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
				(unsigned int)start, (unsigned int) end);

	CACHE_LOOP_LIMITS(start, end,
			cpuinfo.dcache_line_length, cpuinfo.dcache_size);
	CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.flush);
}

/* struct for wb caches and for wt caches */
struct scache *mbc;

/* new wb cache model */
const struct scache wb_msr = {
	.ie = __enable_icache_msr,
	.id = __disable_icache_msr,
	.ifl = __flush_icache_all_noirq,
	.iflr = __flush_icache_range_noirq,
	.iin = __flush_icache_all_noirq,
	.iinr = __flush_icache_range_noirq,
	.de = __enable_dcache_msr,
	.dd = __disable_dcache_msr,
	.dfl = __flush_dcache_all_wb,
	.dflr = __flush_dcache_range_wb,
	.din = __invalidate_dcache_all_wb,
	.dinr = __invalidate_dcache_range_wb,
};

/* There is only difference in ie, id, de, dd functions */
const struct scache wb_nomsr = {
	.ie = __enable_icache_nomsr,
	.id = __disable_icache_nomsr,
	.ifl = __flush_icache_all_noirq,
	.iflr = __flush_icache_range_noirq,
	.iin = __flush_icache_all_noirq,
	.iinr = __flush_icache_range_noirq,
	.de = __enable_dcache_nomsr,
	.dd = __disable_dcache_nomsr,
	.dfl = __flush_dcache_all_wb,
	.dflr = __flush_dcache_range_wb,
	.din = __invalidate_dcache_all_wb,
	.dinr = __invalidate_dcache_range_wb,
};

/* Old wt cache model with disabling irq and turn off cache */
const struct scache wt_msr = {
	.ie = __enable_icache_msr,
	.id = __disable_icache_msr,
	.ifl = __flush_icache_all_msr_irq,
	.iflr = __flush_icache_range_msr_irq,
	.iin = __flush_icache_all_msr_irq,
	.iinr = __flush_icache_range_msr_irq,
	.de = __enable_dcache_msr,
	.dd = __disable_dcache_msr,
	.dfl = __invalidate_dcache_all_msr_irq,
	.dflr = __invalidate_dcache_range_msr_irq_wt,
	.din = __invalidate_dcache_all_msr_irq,
	.dinr = __invalidate_dcache_range_msr_irq_wt,
};

const struct scache wt_nomsr = {
	.ie = __enable_icache_nomsr,
	.id = __disable_icache_nomsr,
	.ifl = __flush_icache_all_nomsr_irq,
	.iflr = __flush_icache_range_nomsr_irq,
	.iin = __flush_icache_all_nomsr_irq,
	.iinr = __flush_icache_range_nomsr_irq,
	.de = __enable_dcache_nomsr,
	.dd = __disable_dcache_nomsr,
	.dfl = __invalidate_dcache_all_nomsr_irq,
	.dflr = __invalidate_dcache_range_nomsr_irq,
	.din = __invalidate_dcache_all_nomsr_irq,
	.dinr = __invalidate_dcache_range_nomsr_irq,
};

/* New wt cache model for newer Microblaze versions */
const struct scache wt_msr_noirq = {
	.ie = __enable_icache_msr,
	.id = __disable_icache_msr,
	.ifl = __flush_icache_all_noirq,
	.iflr = __flush_icache_range_noirq,
	.iin = __flush_icache_all_noirq,
	.iinr = __flush_icache_range_noirq,
	.de = __enable_dcache_msr,
	.dd = __disable_dcache_msr,
	.dfl = __invalidate_dcache_all_noirq_wt,
	.dflr = __invalidate_dcache_range_nomsr_wt,
	.din = __invalidate_dcache_all_noirq_wt,
	.dinr = __invalidate_dcache_range_nomsr_wt,
};

const struct scache wt_nomsr_noirq = {
	.ie = __enable_icache_nomsr,
	.id = __disable_icache_nomsr,
	.ifl = __flush_icache_all_noirq,
	.iflr = __flush_icache_range_noirq,
	.iin = __flush_icache_all_noirq,
	.iinr = __flush_icache_range_noirq,
	.de = __enable_dcache_nomsr,
	.dd = __disable_dcache_nomsr,
	.dfl = __invalidate_dcache_all_noirq_wt,
	.dflr = __invalidate_dcache_range_nomsr_wt,
	.din = __invalidate_dcache_all_noirq_wt,
	.dinr = __invalidate_dcache_range_nomsr_wt,
};

/* CPU version code for 7.20.c - see arch/microblaze/kernel/cpu/cpuinfo.c */
#define CPUVER_7_20_A	0x0c
#define CPUVER_7_20_D	0x0f

#define INFO(s)	printk(KERN_INFO "cache: " s " \n");

void microblaze_cache_init(void)
{
	if (cpuinfo.use_instr & PVR2_USE_MSR_INSTR) {
		if (cpuinfo.dcache_wb) {
			INFO("wb_msr");
			mbc = (struct scache *)&wb_msr;
			if (cpuinfo.ver_code < CPUVER_7_20_D) {
				/* MS: problem with signal handling - hw bug */
				INFO("WB won't work properly");
			}
		} else {
			if (cpuinfo.ver_code >= CPUVER_7_20_A) {
				INFO("wt_msr_noirq");
				mbc = (struct scache *)&wt_msr_noirq;
			} else {
				INFO("wt_msr");
				mbc = (struct scache *)&wt_msr;
			}
		}
	} else {
		if (cpuinfo.dcache_wb) {
			INFO("wb_nomsr");
			mbc = (struct scache *)&wb_nomsr;
			if (cpuinfo.ver_code < CPUVER_7_20_D) {
				/* MS: problem with signal handling - hw bug */
				INFO("WB won't work properly");
			}
		} else {
			if (cpuinfo.ver_code >= CPUVER_7_20_A) {
				INFO("wt_nomsr_noirq");
				mbc = (struct scache *)&wt_nomsr_noirq;
			} else {
				INFO("wt_nomsr");
				mbc = (struct scache *)&wt_nomsr;
			}
		}
	}
}
Commit	Line	Data
8beb8503 MS	1	/*
	2	* Cache control for MicroBlaze cache memories
	3	*
	4	* Copyright (C) 2007-2009 Michal Simek <monstr@monstr.eu>
	5	* Copyright (C) 2007-2009 PetaLogix
2ee2ff87	6	* Copyright (C) 2007-2009 John Williams <john.williams@petalogix.com>
8beb8503 MS	7	*
	8	* This file is subject to the terms and conditions of the GNU General
	9	* Public License. See the file COPYING in the main directory of this
	10	* archive for more details.
	11	*/
	12
	13	#include <asm/cacheflush.h>
	14	#include <linux/cache.h>
	15	#include <asm/cpuinfo.h>
2ee2ff87	16	#include <asm/pvr.h>
8beb8503	17
2ee2ff87 MS	18	static inline void __invalidate_flush_icache(unsigned int addr)
	19	{
	20	__asm__ __volatile__ ("wic %0, r0;" \
	21	: : "r" (addr));
	22	}
	23
	24	static inline void __flush_dcache(unsigned int addr)
	25	{
	26	__asm__ __volatile__ ("wdc.flush %0, r0;" \
	27	: : "r" (addr));
	28	}
	29
	30	static inline void __invalidate_dcache(unsigned int baseaddr,
	31	unsigned int offset)
	32	{
	33	__asm__ __volatile__ ("wdc.clear %0, %1;" \
	34	: : "r" (baseaddr), "r" (offset));
	35	}
8beb8503	36
2ee2ff87	37	static inline void __enable_icache_msr(void)
8beb8503	38	{
2ee2ff87 MS	39	__asm__ __volatile__ (" msrset r0, %0; \
	40	nop; " \
	41	: : "i" (MSR_ICE) : "memory");
	42	}
	43
	44	static inline void __disable_icache_msr(void)
	45	{
	46	__asm__ __volatile__ (" msrclr r0, %0; \
	47	nop; " \
	48	: : "i" (MSR_ICE) : "memory");
	49	}
	50
	51	static inline void __enable_dcache_msr(void)
	52	{
	53	__asm__ __volatile__ (" msrset r0, %0; \
	54	nop; " \
	55	: \
	56	: "i" (MSR_DCE) \
8beb8503	57	: "memory");
8beb8503 MS	58	}
8beb8503 MS	59
2ee2ff87	60	static inline void __disable_dcache_msr(void)
8beb8503	61	{
2ee2ff87 MS	62	__asm__ __volatile__ (" msrclr r0, %0; \
	63	nop; " \
	64	: \
	65	: "i" (MSR_DCE) \
8beb8503	66	: "memory");
2ee2ff87 MS	67	}
	68
	69	static inline void __enable_icache_nomsr(void)
	70	{
	71	__asm__ __volatile__ (" mfs r12, rmsr; \
	72	nop; \
	73	ori r12, r12, %0; \
	74	mts rmsr, r12; \
	75	nop; " \
	76	: \
	77	: "i" (MSR_ICE) \
8beb8503	78	: "memory", "r12");
8beb8503 MS	79	}
8beb8503 MS	80
2ee2ff87	81	static inline void __disable_icache_nomsr(void)
8beb8503	82	{
2ee2ff87 MS	83	__asm__ __volatile__ (" mfs r12, rmsr; \
	84	nop; \
	85	andi r12, r12, ~%0; \
	86	mts rmsr, r12; \
	87	nop; " \
	88	: \
	89	: "i" (MSR_ICE) \
	90	: "memory", "r12");
8beb8503 MS	91	}
8beb8503 MS	92
2ee2ff87	93	static inline void __enable_dcache_nomsr(void)
8beb8503	94	{
2ee2ff87 MS	95	__asm__ __volatile__ (" mfs r12, rmsr; \
	96	nop; \
	97	ori r12, r12, %0; \
	98	mts rmsr, r12; \
	99	nop; " \
	100	: \
	101	: "i" (MSR_DCE) \
8beb8503	102	: "memory", "r12");
8beb8503 MS	103	}
8beb8503 MS	104
2ee2ff87	105	static inline void __disable_dcache_nomsr(void)
8beb8503	106	{
2ee2ff87 MS	107	__asm__ __volatile__ (" mfs r12, rmsr; \
	108	nop; \
	109	andi r12, r12, ~%0; \
	110	mts rmsr, r12; \
	111	nop; " \
	112	: \
	113	: "i" (MSR_DCE) \
8beb8503	114	: "memory", "r12");
8beb8503 MS	115	}
8beb8503 MS	116
2ee2ff87 MS	117
	118	/* Helper macro for computing the limits of cache range loops */
	119	#define CACHE_LOOP_LIMITS(start, end, cache_line_length, cache_size) \
	120	do { \
	121	int align = ~(cache_line_length - 1); \
	122	end = min(start + cache_size, end); \
	123	start &= align; \
	124	end = ((end & align) + cache_line_length); \
	125	} while (0);
	126
	127	/*
	128	* Helper macro to loop over the specified cache_size/line_length and
	129	* execute 'op' on that cacheline
	130	*/
	131	#define CACHE_ALL_LOOP(cache_size, line_length, op) \
	132	do { \
	133	unsigned int len = cache_size; \
	134	int step = -line_length; \
	135	BUG_ON(step >= 0); \
	136	\
	137	__asm__ __volatile__ (" 1: " #op " %0, r0; \
	138	bgtid %0, 1b; \
	139	addk %0, %0, %1; \
	140	" : : "r" (len), "r" (step) \
	141	: "memory"); \
	142	} while (0);
	143
	144
	145	#define CACHE_ALL_LOOP2(cache_size, line_length, op) \
	146	do { \
	147	unsigned int len = cache_size; \
	148	int step = -line_length; \
	149	BUG_ON(step >= 0); \
	150	\
	151	__asm__ __volatile__ (" 1: " #op " r0, %0; \
	152	bgtid %0, 1b; \
	153	addk %0, %0, %1; \
	154	" : : "r" (len), "r" (step) \
	155	: "memory"); \
	156	} while (0);
	157
	158	/* for wdc.flush/clear */
	159	#define CACHE_RANGE_LOOP_2(start, end, line_length, op) \
	160	do { \
	161	int step = -line_length; \
	162	int count = end - start; \
	163	BUG_ON(count <= 0); \
	164	\
	165	__asm__ __volatile__ (" 1: " #op " %0, %1; \
	166	bgtid %1, 1b; \
	167	addk %1, %1, %2; \
	168	" : : "r" (start), "r" (count), \
	169	"r" (step) : "memory"); \
	170	} while (0);
	171
	172	/* It is used only first parameter for OP - for wic, wdc */
	173	#define CACHE_RANGE_LOOP_1(start, end, line_length, op) \
	174	do { \
0d670b24 MS	175	int volatile temp; \
0d670b24 MS	176	BUG_ON(end - start <= 0); \
2ee2ff87	177	\
0d670b24 MS	178	__asm__ __volatile__ (" 1: " #op " %1, r0; \
	179	cmpu %0, %1, %2; \
	180	bgtid %0, 1b; \
	181	addk %1, %1, %3; \
	182	" : : "r" (temp), "r" (start), "r" (end),\
	183	"r" (line_length) : "memory"); \
2ee2ff87 MS	184	} while (0);
	185
	186	static void __flush_icache_range_msr_irq(unsigned long start, unsigned long end)
8beb8503	187	{
2ee2ff87 MS	188	unsigned long flags;
	189
	190	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
	191	(unsigned int)start, (unsigned int) end);
	192
	193	CACHE_LOOP_LIMITS(start, end,
	194	cpuinfo.icache_line_length, cpuinfo.icache_size);
	195
	196	local_irq_save(flags);
	197	__disable_icache_msr();
	198
	199	CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
	200
	201	__enable_icache_msr();
	202	local_irq_restore(flags);
8beb8503 MS	203	}
8beb8503 MS	204
2ee2ff87 MS	205	static void __flush_icache_range_nomsr_irq(unsigned long start,
2ee2ff87 MS	206	unsigned long end)
8beb8503	207	{
2ee2ff87	208	unsigned long flags;
8beb8503	209
2ee2ff87 MS	210	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
2ee2ff87 MS	211	(unsigned int)start, (unsigned int) end);
8beb8503	212
2ee2ff87 MS	213	CACHE_LOOP_LIMITS(start, end,
2ee2ff87 MS	214	cpuinfo.icache_line_length, cpuinfo.icache_size);
8beb8503	215
2ee2ff87 MS	216	local_irq_save(flags);
	217	__disable_icache_nomsr();
	218
	219	CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
	220
	221	__enable_icache_nomsr();
	222	local_irq_restore(flags);
8beb8503 MS	223	}
8beb8503 MS	224
2ee2ff87 MS	225	static void __flush_icache_range_noirq(unsigned long start,
2ee2ff87 MS	226	unsigned long end)
8beb8503	227	{
2ee2ff87 MS	228	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
	229	(unsigned int)start, (unsigned int) end);
	230
	231	CACHE_LOOP_LIMITS(start, end,
	232	cpuinfo.icache_line_length, cpuinfo.icache_size);
	233	CACHE_RANGE_LOOP_1(start, end, cpuinfo.icache_line_length, wic);
	234	}
	235
	236	static void __flush_icache_all_msr_irq(void)
	237	{
	238	unsigned long flags;
	239
	240	pr_debug("%s\n", __func__);
	241
	242	local_irq_save(flags);
	243	__disable_icache_msr();
	244
	245	CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
	246
	247	__enable_icache_msr();
	248	local_irq_restore(flags);
	249	}
	250
	251	static void __flush_icache_all_nomsr_irq(void)
	252	{
	253	unsigned long flags;
	254
	255	pr_debug("%s\n", __func__);
	256
	257	local_irq_save(flags);
	258	__disable_icache_nomsr();
	259
	260	CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
	261
	262	__enable_icache_nomsr();
	263	local_irq_restore(flags);
8beb8503 MS	264	}
8beb8503 MS	265
2ee2ff87	266	static void __flush_icache_all_noirq(void)
8beb8503	267	{
2ee2ff87 MS	268	pr_debug("%s\n", __func__);
2ee2ff87 MS	269	CACHE_ALL_LOOP(cpuinfo.icache_size, cpuinfo.icache_line_length, wic);
8beb8503 MS	270	}
8beb8503 MS	271
2ee2ff87	272	static void __invalidate_dcache_all_msr_irq(void)
8beb8503	273	{
2ee2ff87 MS	274	unsigned long flags;
	275
	276	pr_debug("%s\n", __func__);
	277
	278	local_irq_save(flags);
	279	__disable_dcache_msr();
	280
	281	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
	282
	283	__enable_dcache_msr();
	284	local_irq_restore(flags);
8beb8503 MS	285	}
8beb8503 MS	286
2ee2ff87	287	static void __invalidate_dcache_all_nomsr_irq(void)
8beb8503	288	{
2ee2ff87 MS	289	unsigned long flags;
	290
	291	pr_debug("%s\n", __func__);
	292
	293	local_irq_save(flags);
	294	__disable_dcache_nomsr();
	295
	296	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc);
	297
	298	__enable_dcache_nomsr();
	299	local_irq_restore(flags);
8beb8503 MS	300	}
8beb8503 MS	301
2ee2ff87	302	static void __invalidate_dcache_all_noirq_wt(void)
8beb8503	303	{
2ee2ff87 MS	304	pr_debug("%s\n", __func__);
2ee2ff87 MS	305	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length, wdc)
8beb8503 MS	306	}
8beb8503 MS	307
2ee2ff87 MS	308	/* FIXME this is weird - should be only wdc but not work
	309	* MS: I am getting bus errors and other weird things */
	310	static void __invalidate_dcache_all_wb(void)
8beb8503	311	{
2ee2ff87 MS	312	pr_debug("%s\n", __func__);
	313	CACHE_ALL_LOOP2(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
	314	wdc.clear)
2ee2ff87 MS	315	}
	316
	317	static void __invalidate_dcache_range_wb(unsigned long start,
	318	unsigned long end)
	319	{
	320	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
	321	(unsigned int)start, (unsigned int) end);
	322
	323	CACHE_LOOP_LIMITS(start, end,
	324	cpuinfo.dcache_line_length, cpuinfo.dcache_size);
	325	CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.clear);
	326	}
	327
	328	static void __invalidate_dcache_range_nomsr_wt(unsigned long start,
	329	unsigned long end)
	330	{
	331	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
	332	(unsigned int)start, (unsigned int) end);
	333	CACHE_LOOP_LIMITS(start, end,
	334	cpuinfo.dcache_line_length, cpuinfo.dcache_size);
	335
	336	CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
8beb8503 MS	337	}
8beb8503 MS	338
2ee2ff87 MS	339	static void __invalidate_dcache_range_msr_irq_wt(unsigned long start,
2ee2ff87 MS	340	unsigned long end)
8beb8503	341	{
2ee2ff87 MS	342	unsigned long flags;
	343
	344	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
	345	(unsigned int)start, (unsigned int) end);
	346	CACHE_LOOP_LIMITS(start, end,
	347	cpuinfo.dcache_line_length, cpuinfo.dcache_size);
	348
	349	local_irq_save(flags);
	350	__disable_dcache_msr();
	351
	352	CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
	353
	354	__enable_dcache_msr();
	355	local_irq_restore(flags);
	356	}
	357
	358	static void __invalidate_dcache_range_nomsr_irq(unsigned long start,
	359	unsigned long end)
	360	{
	361	unsigned long flags;
	362
	363	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
	364	(unsigned int)start, (unsigned int) end);
	365
	366	CACHE_LOOP_LIMITS(start, end,
	367	cpuinfo.dcache_line_length, cpuinfo.dcache_size);
	368
	369	local_irq_save(flags);
	370	__disable_dcache_nomsr();
	371
	372	CACHE_RANGE_LOOP_1(start, end, cpuinfo.dcache_line_length, wdc);
	373
	374	__enable_dcache_nomsr();
	375	local_irq_restore(flags);
	376	}
	377
	378	static void __flush_dcache_all_wb(void)
	379	{
	380	pr_debug("%s\n", __func__);
	381	CACHE_ALL_LOOP(cpuinfo.dcache_size, cpuinfo.dcache_line_length,
	382	wdc.flush);
8beb8503 MS	383	}
8beb8503 MS	384
2ee2ff87	385	static void __flush_dcache_range_wb(unsigned long start, unsigned long end)
8beb8503	386	{
2ee2ff87 MS	387	pr_debug("%s: start 0x%x, end 0x%x\n", __func__,
	388	(unsigned int)start, (unsigned int) end);
	389
	390	CACHE_LOOP_LIMITS(start, end,
	391	cpuinfo.dcache_line_length, cpuinfo.dcache_size);
	392	CACHE_RANGE_LOOP_2(start, end, cpuinfo.dcache_line_length, wdc.flush);
	393	}
	394
	395	/* struct for wb caches and for wt caches */
	396	struct scache *mbc;
	397
	398	/* new wb cache model */
	399	const struct scache wb_msr = {
	400	.ie = __enable_icache_msr,
	401	.id = __disable_icache_msr,
	402	.ifl = __flush_icache_all_noirq,
	403	.iflr = __flush_icache_range_noirq,
	404	.iin = __flush_icache_all_noirq,
	405	.iinr = __flush_icache_range_noirq,
	406	.de = __enable_dcache_msr,
	407	.dd = __disable_dcache_msr,
	408	.dfl = __flush_dcache_all_wb,
	409	.dflr = __flush_dcache_range_wb,
	410	.din = __invalidate_dcache_all_wb,
	411	.dinr = __invalidate_dcache_range_wb,
	412	};
	413
	414	/* There is only difference in ie, id, de, dd functions */
	415	const struct scache wb_nomsr = {
	416	.ie = __enable_icache_nomsr,
	417	.id = __disable_icache_nomsr,
	418	.ifl = __flush_icache_all_noirq,
	419	.iflr = __flush_icache_range_noirq,
	420	.iin = __flush_icache_all_noirq,
	421	.iinr = __flush_icache_range_noirq,
	422	.de = __enable_dcache_nomsr,
	423	.dd = __disable_dcache_nomsr,
	424	.dfl = __flush_dcache_all_wb,
	425	.dflr = __flush_dcache_range_wb,
	426	.din = __invalidate_dcache_all_wb,
	427	.dinr = __invalidate_dcache_range_wb,
	428	};
	429
	430	/* Old wt cache model with disabling irq and turn off cache */
	431	const struct scache wt_msr = {
	432	.ie = __enable_icache_msr,
	433	.id = __disable_icache_msr,
	434	.ifl = __flush_icache_all_msr_irq,
	435	.iflr = __flush_icache_range_msr_irq,
	436	.iin = __flush_icache_all_msr_irq,
	437	.iinr = __flush_icache_range_msr_irq,
	438	.de = __enable_dcache_msr,
	439	.dd = __disable_dcache_msr,
	440	.dfl = __invalidate_dcache_all_msr_irq,
	441	.dflr = __invalidate_dcache_range_msr_irq_wt,
	442	.din = __invalidate_dcache_all_msr_irq,
	443	.dinr = __invalidate_dcache_range_msr_irq_wt,
	444	};
	445
	446	const struct scache wt_nomsr = {
	447	.ie = __enable_icache_nomsr,
	448	.id = __disable_icache_nomsr,
	449	.ifl = __flush_icache_all_nomsr_irq,
	450	.iflr = __flush_icache_range_nomsr_irq,
451	.iin = __flush_icache_all_nomsr_irq,
452	.iinr = __flush_icache_range_nomsr_irq,
453	.de = __enable_dcache_nomsr,
454	.dd = __disable_dcache_nomsr,
455	.dfl = __invalidate_dcache_all_nomsr_irq,
456	.dflr = __invalidate_dcache_range_nomsr_irq,
457	.din = __invalidate_dcache_all_nomsr_irq,
458	.dinr = __invalidate_dcache_range_nomsr_irq,
459	};
460
461	/* New wt cache model for newer Microblaze versions */
462	const struct scache wt_msr_noirq = {
463	.ie = __enable_icache_msr,
464	.id = __disable_icache_msr,
465	.ifl = __flush_icache_all_noirq,
466	.iflr = __flush_icache_range_noirq,
467	.iin = __flush_icache_all_noirq,
468	.iinr = __flush_icache_range_noirq,
469	.de = __enable_dcache_msr,
470	.dd = __disable_dcache_msr,
471	.dfl = __invalidate_dcache_all_noirq_wt,
472	.dflr = __invalidate_dcache_range_nomsr_wt,
473	.din = __invalidate_dcache_all_noirq_wt,
474	.dinr = __invalidate_dcache_range_nomsr_wt,
475	};
476
477	const struct scache wt_nomsr_noirq = {
478	.ie = __enable_icache_nomsr,
479	.id = __disable_icache_nomsr,
480	.ifl = __flush_icache_all_noirq,
481	.iflr = __flush_icache_range_noirq,
482	.iin = __flush_icache_all_noirq,
483	.iinr = __flush_icache_range_noirq,
484	.de = __enable_dcache_nomsr,
485	.dd = __disable_dcache_nomsr,
486	.dfl = __invalidate_dcache_all_noirq_wt,
487	.dflr = __invalidate_dcache_range_nomsr_wt,
488	.din = __invalidate_dcache_all_noirq_wt,
489	.dinr = __invalidate_dcache_range_nomsr_wt,
490	};
491
492	/* CPU version code for 7.20.c - see arch/microblaze/kernel/cpu/cpuinfo.c */
493	#define CPUVER_7_20_A 0x0c
494	#define CPUVER_7_20_D 0x0f
495
496	#define INFO(s) printk(KERN_INFO "cache: " s " \n");
497
498	void microblaze_cache_init(void)
499	{
500	if (cpuinfo.use_instr & PVR2_USE_MSR_INSTR) {
501	if (cpuinfo.dcache_wb) {
502	INFO("wb_msr");
503	mbc = (struct scache *)&wb_msr;
504	if (cpuinfo.ver_code < CPUVER_7_20_D) {
505	/* MS: problem with signal handling - hw bug */
506	INFO("WB won't work properly");
507	}
508	} else {
509	if (cpuinfo.ver_code >= CPUVER_7_20_A) {
510	INFO("wt_msr_noirq");
511	mbc = (struct scache *)&wt_msr_noirq;
512	} else {
513	INFO("wt_msr");
514	mbc = (struct scache *)&wt_msr;
515	}
516	}
517	} else {
518	if (cpuinfo.dcache_wb) {
519	INFO("wb_nomsr");
520	mbc = (struct scache *)&wb_nomsr;
521	if (cpuinfo.ver_code < CPUVER_7_20_D) {
522	/* MS: problem with signal handling - hw bug */
523	INFO("WB won't work properly");
524	}
525	} else {
526	if (cpuinfo.ver_code >= CPUVER_7_20_A) {
527	INFO("wt_nomsr_noirq");
528	mbc = (struct scache *)&wt_nomsr_noirq;
529	} else {
530	INFO("wt_nomsr");
531	mbc = (struct scache *)&wt_nomsr;
532	}
533	}
534	}
8beb8503	535	}