[linux-block.git] / arch / mips / mm / tlb-r3k.c

// SPDX-License-Identifier: GPL-2.0
/*
 * r2300.c: R2000 and R3000 specific mmu/cache code.
 *
 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
 *
 * with a lot of changes to make this thing work for R3000s
 * Tx39XX R4k style caches added. HK
 * Copyright (C) 1998, 1999, 2000 Harald Koerfgen
 * Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
 * Copyright (C) 2002  Ralf Baechle
 * Copyright (C) 2002  Maciej W. Rozycki
 */
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>

#include <asm/page.h>
#include <asm/mmu_context.h>
#include <asm/tlbmisc.h>
#include <asm/isadep.h>
#include <asm/io.h>
#include <asm/bootinfo.h>
#include <asm/cpu.h>
#include <asm/setup.h>
#include <asm/tlbex.h>

#undef DEBUG_TLB

/* CP0 hazard avoidance. */
#define BARRIER				\
	__asm__ __volatile__(		\
		".set	push\n\t"	\
		".set	noreorder\n\t"	\
		"nop\n\t"		\
		".set	pop\n\t")

/* TLB operations. */
static void local_flush_tlb_from(int entry)
{
	unsigned long old_ctx;

	old_ctx = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
	write_c0_entrylo0(0);
	while (entry < current_cpu_data.tlbsize) {
		write_c0_index(entry << 8);
		write_c0_entryhi((entry | 0x80000) << 12);
		entry++;				/* BARRIER */
		tlb_write_indexed();
	}
	write_c0_entryhi(old_ctx);
}

void local_flush_tlb_all(void)
{
	unsigned long flags;

#ifdef DEBUG_TLB
	printk("[tlball]");
#endif
	local_irq_save(flags);
	local_flush_tlb_from(8);
	local_irq_restore(flags);
}

void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
			   unsigned long end)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	struct mm_struct *mm = vma->vm_mm;
	int cpu = smp_processor_id();

	if (cpu_context(cpu, mm) != 0) {
		unsigned long size, flags;

#ifdef DEBUG_TLB
		printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
			cpu_context(cpu, mm) & asid_mask, start, end);
#endif
		local_irq_save(flags);
		size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
		if (size <= current_cpu_data.tlbsize) {
			int oldpid = read_c0_entryhi() & asid_mask;
			int newpid = cpu_context(cpu, mm) & asid_mask;

			start &= PAGE_MASK;
			end += PAGE_SIZE - 1;
			end &= PAGE_MASK;
			while (start < end) {
				int idx;

				write_c0_entryhi(start | newpid);
				start += PAGE_SIZE;	/* BARRIER */
				tlb_probe();
				idx = read_c0_index();
				write_c0_entrylo0(0);
				write_c0_entryhi(KSEG0);
				if (idx < 0)		/* BARRIER */
					continue;
				tlb_write_indexed();
			}
			write_c0_entryhi(oldpid);
		} else {
			drop_mmu_context(mm);
		}
		local_irq_restore(flags);
	}
}

void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	unsigned long size, flags;

#ifdef DEBUG_TLB
	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
#endif
	local_irq_save(flags);
	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	if (size <= current_cpu_data.tlbsize) {
		int pid = read_c0_entryhi();

		start &= PAGE_MASK;
		end += PAGE_SIZE - 1;
		end &= PAGE_MASK;

		while (start < end) {
			int idx;

			write_c0_entryhi(start);
			start += PAGE_SIZE;		/* BARRIER */
			tlb_probe();
			idx = read_c0_index();
			write_c0_entrylo0(0);
			write_c0_entryhi(KSEG0);
			if (idx < 0)			/* BARRIER */
				continue;
			tlb_write_indexed();
		}
		write_c0_entryhi(pid);
	} else {
		local_flush_tlb_all();
	}
	local_irq_restore(flags);
}

void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	int cpu = smp_processor_id();

	if (cpu_context(cpu, vma->vm_mm) != 0) {
		unsigned long flags;
		int oldpid, newpid, idx;

#ifdef DEBUG_TLB
		printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
#endif
		newpid = cpu_context(cpu, vma->vm_mm) & asid_mask;
		page &= PAGE_MASK;
		local_irq_save(flags);
		oldpid = read_c0_entryhi() & asid_mask;
		write_c0_entryhi(page | newpid);
		BARRIER;
		tlb_probe();
		idx = read_c0_index();
		write_c0_entrylo0(0);
		write_c0_entryhi(KSEG0);
		if (idx < 0)				/* BARRIER */
			goto finish;
		tlb_write_indexed();

finish:
		write_c0_entryhi(oldpid);
		local_irq_restore(flags);
	}
}

void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	unsigned long flags;
	int idx, pid;

	/*
	 * Handle debugger faulting in for debuggee.
	 */
	if (current->active_mm != vma->vm_mm)
		return;

	pid = read_c0_entryhi() & asid_mask;

#ifdef DEBUG_TLB
	if ((pid != (cpu_context(cpu, vma->vm_mm) & asid_mask)) || (cpu_context(cpu, vma->vm_mm) == 0)) {
		printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
		       (cpu_context(cpu, vma->vm_mm)), pid);
	}
#endif

	local_irq_save(flags);
	address &= PAGE_MASK;
	write_c0_entryhi(address | pid);
	BARRIER;
	tlb_probe();
	idx = read_c0_index();
	write_c0_entrylo0(pte_val(pte));
	write_c0_entryhi(address | pid);
	if (idx < 0) {					/* BARRIER */
		tlb_write_random();
	} else {
		tlb_write_indexed();
	}
	write_c0_entryhi(pid);
	local_irq_restore(flags);
}

void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
		     unsigned long entryhi, unsigned long pagemask)
{
	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
	unsigned long flags;
	unsigned long old_ctx;
	static unsigned long wired = 0;

	if (wired < 8) {
#ifdef DEBUG_TLB
		printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
		       entrylo0, entryhi);
#endif

		local_irq_save(flags);
		old_ctx = read_c0_entryhi() & asid_mask;
		write_c0_entrylo0(entrylo0);
		write_c0_entryhi(entryhi);
		write_c0_index(wired);
		wired++;				/* BARRIER */
		tlb_write_indexed();
		write_c0_entryhi(old_ctx);
		local_flush_tlb_all();
		local_irq_restore(flags);
	}
}

void tlb_init(void)
{
	local_flush_tlb_from(0);
	build_tlb_refill_handler();
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* r2300.c: R2000 and R3000 specific mmu/cache code.
	4	*
79add627	5	* Copyright (C) 1996 David S. Miller (davem@davemloft.net)
1da177e4 LT	6	*
	7	* with a lot of changes to make this thing work for R3000s
	8	* Tx39XX R4k style caches added. HK
	9	* Copyright (C) 1998, 1999, 2000 Harald Koerfgen
	10	* Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
	11	* Copyright (C) 2002 Ralf Baechle
	12	* Copyright (C) 2002 Maciej W. Rozycki
	13	*/
1da177e4 LT	14	#include <linux/kernel.h>
1da177e4 LT	15	#include <linux/sched.h>
631330f5	16	#include <linux/smp.h>
1da177e4 LT	17	#include <linux/mm.h>
	18
	19	#include <asm/page.h>
1da177e4	20	#include <asm/mmu_context.h>
3d18c983	21	#include <asm/tlbmisc.h>
1da177e4 LT	22	#include <asm/isadep.h>
	23	#include <asm/io.h>
	24	#include <asm/bootinfo.h>
	25	#include <asm/cpu.h>
ec47b986	26	#include <asm/setup.h>
6fb04df9	27	#include <asm/tlbex.h>
1da177e4 LT	28
	29	#undef DEBUG_TLB
	30
1da177e4 LT	31	/* CP0 hazard avoidance. */
	32	#define BARRIER \
	33	__asm__ __volatile__( \
	34	".set push\n\t" \
	35	".set noreorder\n\t" \
	36	"nop\n\t" \
	37	".set pop\n\t")
	38
1da177e4	39	/* TLB operations. */
dbfbf60f	40	static void local_flush_tlb_from(int entry)
1da177e4	41	{
1da177e4	42	unsigned long old_ctx;
1da177e4	43
4edf00a4	44	old_ctx = read_c0_entryhi() & cpu_asid_mask(&current_cpu_data);
1da177e4	45	write_c0_entrylo0(0);
9a20b092	46	while (entry < current_cpu_data.tlbsize) {
1da177e4 LT	47	write_c0_index(entry << 8);
1da177e4 LT	48	write_c0_entryhi((entry \| 0x80000) << 12);
9a20b092	49	entry++; /* BARRIER */
1da177e4 LT	50	tlb_write_indexed();
	51	}
	52	write_c0_entryhi(old_ctx);
dbfbf60f MR	53	}
	54
	55	void local_flush_tlb_all(void)
	56	{
	57	unsigned long flags;
	58
	59	#ifdef DEBUG_TLB
	60	printk("[tlball]");
	61	#endif
	62	local_irq_save(flags);
455481fc	63	local_flush_tlb_from(8);
1da177e4 LT	64	local_irq_restore(flags);
	65	}
	66
1da177e4 LT	67	void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	68	unsigned long end)
	69	{
4edf00a4	70	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	71	struct mm_struct *mm = vma->vm_mm;
	72	int cpu = smp_processor_id();
	73
	74	if (cpu_context(cpu, mm) != 0) {
a5e696e5	75	unsigned long size, flags;
1da177e4 LT	76
	77	#ifdef DEBUG_TLB
	78	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]",
4edf00a4	79	cpu_context(cpu, mm) & asid_mask, start, end);
1da177e4 LT	80	#endif
	81	local_irq_save(flags);
	82	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	83	if (size <= current_cpu_data.tlbsize) {
4edf00a4 PB	84	int oldpid = read_c0_entryhi() & asid_mask;
4edf00a4 PB	85	int newpid = cpu_context(cpu, mm) & asid_mask;
1da177e4 LT	86
	87	start &= PAGE_MASK;
	88	end += PAGE_SIZE - 1;
	89	end &= PAGE_MASK;
	90	while (start < end) {
	91	int idx;
	92
	93	write_c0_entryhi(start \| newpid);
	94	start += PAGE_SIZE; /* BARRIER */
	95	tlb_probe();
	96	idx = read_c0_index();
	97	write_c0_entrylo0(0);
	98	write_c0_entryhi(KSEG0);
	99	if (idx < 0) /* BARRIER */
	100	continue;
	101	tlb_write_indexed();
	102	}
	103	write_c0_entryhi(oldpid);
	104	} else {
9a27324f	105	drop_mmu_context(mm);
1da177e4 LT	106	}
	107	local_irq_restore(flags);
	108	}
	109	}
	110
	111	void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
	112	{
a5e696e5	113	unsigned long size, flags;
1da177e4 LT	114
	115	#ifdef DEBUG_TLB
	116	printk("[tlbrange<%lu,0x%08lx,0x%08lx>]", start, end);
	117	#endif
	118	local_irq_save(flags);
	119	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	120	if (size <= current_cpu_data.tlbsize) {
	121	int pid = read_c0_entryhi();
	122
	123	start &= PAGE_MASK;
	124	end += PAGE_SIZE - 1;
	125	end &= PAGE_MASK;
	126
	127	while (start < end) {
	128	int idx;
	129
	130	write_c0_entryhi(start);
	131	start += PAGE_SIZE; /* BARRIER */
	132	tlb_probe();
	133	idx = read_c0_index();
	134	write_c0_entrylo0(0);
	135	write_c0_entryhi(KSEG0);
	136	if (idx < 0) /* BARRIER */
	137	continue;
	138	tlb_write_indexed();
	139	}
	140	write_c0_entryhi(pid);
	141	} else {
	142	local_flush_tlb_all();
	143	}
	144	local_irq_restore(flags);
	145	}
	146
	147	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
	148	{
4edf00a4	149	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	150	int cpu = smp_processor_id();
1da177e4 LT	151
e5cd534a	152	if (cpu_context(cpu, vma->vm_mm) != 0) {
1da177e4 LT	153	unsigned long flags;
	154	int oldpid, newpid, idx;
	155
	156	#ifdef DEBUG_TLB
	157	printk("[tlbpage<%lu,0x%08lx>]", cpu_context(cpu, vma->vm_mm), page);
	158	#endif
4edf00a4	159	newpid = cpu_context(cpu, vma->vm_mm) & asid_mask;
1da177e4 LT	160	page &= PAGE_MASK;
1da177e4 LT	161	local_irq_save(flags);
4edf00a4	162	oldpid = read_c0_entryhi() & asid_mask;
1da177e4 LT	163	write_c0_entryhi(page \| newpid);
	164	BARRIER;
	165	tlb_probe();
	166	idx = read_c0_index();
	167	write_c0_entrylo0(0);
	168	write_c0_entryhi(KSEG0);
	169	if (idx < 0) /* BARRIER */
	170	goto finish;
	171	tlb_write_indexed();
	172
	173	finish:
	174	write_c0_entryhi(oldpid);
	175	local_irq_restore(flags);
	176	}
	177	}
	178
	179	void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
	180	{
4edf00a4	181	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	182	unsigned long flags;
	183	int idx, pid;
	184
	185	/*
2f9060b1	186	* Handle debugger faulting in for debuggee.
1da177e4 LT	187	*/
	188	if (current->active_mm != vma->vm_mm)
	189	return;
	190
4edf00a4	191	pid = read_c0_entryhi() & asid_mask;
1da177e4 LT	192
1da177e4 LT	193	#ifdef DEBUG_TLB
4edf00a4	194	if ((pid != (cpu_context(cpu, vma->vm_mm) & asid_mask)) \|\| (cpu_context(cpu, vma->vm_mm) == 0)) {
1da177e4 LT	195	printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%lu tlbpid=%d\n",
	196	(cpu_context(cpu, vma->vm_mm)), pid);
	197	}
	198	#endif
	199
	200	local_irq_save(flags);
	201	address &= PAGE_MASK;
	202	write_c0_entryhi(address \| pid);
	203	BARRIER;
	204	tlb_probe();
	205	idx = read_c0_index();
	206	write_c0_entrylo0(pte_val(pte));
	207	write_c0_entryhi(address \| pid);
	208	if (idx < 0) { /* BARRIER */
	209	tlb_write_random();
	210	} else {
	211	tlb_write_indexed();
	212	}
	213	write_c0_entryhi(pid);
	214	local_irq_restore(flags);
	215	}
	216
694b8c35 ML	217	void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
694b8c35 ML	218	unsigned long entryhi, unsigned long pagemask)
1da177e4	219	{
4edf00a4	220	unsigned long asid_mask = cpu_asid_mask(&current_cpu_data);
1da177e4 LT	221	unsigned long flags;
	222	unsigned long old_ctx;
	223	static unsigned long wired = 0;
	224
455481fc	225	if (wired < 8) {
1da177e4 LT	226	#ifdef DEBUG_TLB
	227	printk("[tlbwired<entry lo0 %8x, hi %8x\n>]\n",
	228	entrylo0, entryhi);
	229	#endif
	230
	231	local_irq_save(flags);
4edf00a4	232	old_ctx = read_c0_entryhi() & asid_mask;
1da177e4 LT	233	write_c0_entrylo0(entrylo0);
	234	write_c0_entryhi(entryhi);
	235	write_c0_index(wired);
	236	wired++; /* BARRIER */
	237	tlb_write_indexed();
	238	write_c0_entryhi(old_ctx);
	239	local_flush_tlb_all();
	240	local_irq_restore(flags);
	241	}
	242	}
	243
078a55fc	244	void tlb_init(void)
1da177e4	245	{
dbfbf60f	246	local_flush_tlb_from(0);
1da177e4 LT	247	build_tlb_refill_handler();
1da177e4 LT	248	}