[linux-block.git] / arch / loongarch / mm / tlb.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
 */
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/export.h>

#include <asm/cpu.h>
#include <asm/bootinfo.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
#include <asm/tlb.h>

void local_flush_tlb_all(void)
{
	invtlb_all(INVTLB_CURRENT_ALL, 0, 0);
}
EXPORT_SYMBOL(local_flush_tlb_all);

void local_flush_tlb_user(void)
{
	invtlb_all(INVTLB_CURRENT_GFALSE, 0, 0);
}
EXPORT_SYMBOL(local_flush_tlb_user);

void local_flush_tlb_kernel(void)
{
	invtlb_all(INVTLB_CURRENT_GTRUE, 0, 0);
}
EXPORT_SYMBOL(local_flush_tlb_kernel);

/*
 * All entries common to a mm share an asid. To effectively flush
 * these entries, we just bump the asid.
 */
void local_flush_tlb_mm(struct mm_struct *mm)
{
	int cpu;

	preempt_disable();

	cpu = smp_processor_id();

	if (asid_valid(mm, cpu))
		drop_mmu_context(mm, cpu);
	else
		cpumask_clear_cpu(cpu, mm_cpumask(mm));

	preempt_enable();
}

void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	unsigned long end)
{
	struct mm_struct *mm = vma->vm_mm;
	int cpu = smp_processor_id();

	if (asid_valid(mm, cpu)) {
		unsigned long size, flags;

		local_irq_save(flags);
		start = round_down(start, PAGE_SIZE << 1);
		end = round_up(end, PAGE_SIZE << 1);
		size = (end - start) >> (PAGE_SHIFT + 1);
		if (size <= (current_cpu_data.tlbsizestlbsets ?
			     current_cpu_data.tlbsize / 8 :
			     current_cpu_data.tlbsize / 2)) {
			int asid = cpu_asid(cpu, mm);

			while (start < end) {
				invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, start);
				start += (PAGE_SIZE << 1);
			}
		} else {
			drop_mmu_context(mm, cpu);
		}
		local_irq_restore(flags);
	} else {
		cpumask_clear_cpu(cpu, mm_cpumask(mm));
	}
}

void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	unsigned long size, flags;

	local_irq_save(flags);
	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
	size = (size + 1) >> 1;
	if (size <= (current_cpu_data.tlbsizestlbsets ?
		     current_cpu_data.tlbsize / 8 :
		     current_cpu_data.tlbsize / 2)) {

		start &= (PAGE_MASK << 1);
		end += ((PAGE_SIZE << 1) - 1);
		end &= (PAGE_MASK << 1);

		while (start < end) {
			invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, start);
			start += (PAGE_SIZE << 1);
		}
	} else {
		local_flush_tlb_kernel();
	}
	local_irq_restore(flags);
}

void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
	int cpu = smp_processor_id();

	if (asid_valid(vma->vm_mm, cpu)) {
		int newpid;

		newpid = cpu_asid(cpu, vma->vm_mm);
		page &= (PAGE_MASK << 1);
		invtlb(INVTLB_ADDR_GFALSE_AND_ASID, newpid, page);
	} else {
		cpumask_clear_cpu(cpu, mm_cpumask(vma->vm_mm));
	}
}

/*
 * This one is only used for pages with the global bit set so we don't care
 * much about the ASID.
 */
void local_flush_tlb_one(unsigned long page)
{
	page &= (PAGE_MASK << 1);
	invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, page);
}

static void __update_hugetlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
#ifdef CONFIG_HUGETLB_PAGE
	int idx;
	unsigned long lo;
	unsigned long flags;

	local_irq_save(flags);

	address &= (PAGE_MASK << 1);
	write_csr_entryhi(address);
	tlb_probe();
	idx = read_csr_tlbidx();
	write_csr_pagesize(PS_HUGE_SIZE);
	lo = pmd_to_entrylo(pte_val(*ptep));
	write_csr_entrylo0(lo);
	write_csr_entrylo1(lo + (HPAGE_SIZE >> 1));

	if (idx < 0)
		tlb_write_random();
	else
		tlb_write_indexed();
	write_csr_pagesize(PS_DEFAULT_SIZE);

	local_irq_restore(flags);
#endif
}

void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
	int idx;
	unsigned long flags;

	/*
	 * Handle debugger faulting in for debugee.
	 */
	if (current->active_mm != vma->vm_mm)
		return;

	if (pte_val(*ptep) & _PAGE_HUGE) {
		__update_hugetlb(vma, address, ptep);
		return;
	}

	local_irq_save(flags);

	if ((unsigned long)ptep & sizeof(pte_t))
		ptep--;

	address &= (PAGE_MASK << 1);
	write_csr_entryhi(address);
	tlb_probe();
	idx = read_csr_tlbidx();
	write_csr_pagesize(PS_DEFAULT_SIZE);
	write_csr_entrylo0(pte_val(*ptep++));
	write_csr_entrylo1(pte_val(*ptep));
	if (idx < 0)
		tlb_write_random();
	else
		tlb_write_indexed();

	local_irq_restore(flags);
}

static void setup_ptwalker(void)
{
	unsigned long pwctl0, pwctl1;
	unsigned long pgd_i = 0, pgd_w = 0;
	unsigned long pud_i = 0, pud_w = 0;
	unsigned long pmd_i = 0, pmd_w = 0;
	unsigned long pte_i = 0, pte_w = 0;

	pgd_i = PGDIR_SHIFT;
	pgd_w = PAGE_SHIFT - 3;
#if CONFIG_PGTABLE_LEVELS > 3
	pud_i = PUD_SHIFT;
	pud_w = PAGE_SHIFT - 3;
#endif
#if CONFIG_PGTABLE_LEVELS > 2
	pmd_i = PMD_SHIFT;
	pmd_w = PAGE_SHIFT - 3;
#endif
	pte_i = PAGE_SHIFT;
	pte_w = PAGE_SHIFT - 3;

	pwctl0 = pte_i | pte_w << 5 | pmd_i << 10 | pmd_w << 15 | pud_i << 20 | pud_w << 25;
	pwctl1 = pgd_i | pgd_w << 6;

	csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0);
	csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1);
	csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH);
	csr_write64((long)invalid_pg_dir, LOONGARCH_CSR_PGDL);
	csr_write64((long)smp_processor_id(), LOONGARCH_CSR_TMID);
}

static void output_pgtable_bits_defines(void)
{
#define pr_define(fmt, ...)					\
	pr_debug("#define " fmt, ##__VA_ARGS__)

	pr_debug("#include <asm/asm.h>\n");
	pr_debug("#include <asm/regdef.h>\n");
	pr_debug("\n");

	pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
	pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
	pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
	pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
	pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
	pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
	pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
	pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
	pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
	pr_debug("\n");
}

#ifdef CONFIG_NUMA
static unsigned long pcpu_handlers[NR_CPUS];
#endif
extern long exception_handlers[VECSIZE * 128 / sizeof(long)];

void setup_tlb_handler(int cpu)
{
	setup_ptwalker();
	local_flush_tlb_all();

	/* The tlb handlers are generated only once */
	if (cpu == 0) {
		memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
		local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
		set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
		set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
		set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
		set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
		set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
		set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
		set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
	}
#ifdef CONFIG_NUMA
	else {
		void *addr;
		struct page *page;
		const int vec_sz = sizeof(exception_handlers);

		if (pcpu_handlers[cpu])
			return;

		page = alloc_pages_node(cpu_to_node(cpu), GFP_ATOMIC, get_order(vec_sz));
		if (!page)
			return;

		addr = page_address(page);
		pcpu_handlers[cpu] = (unsigned long)addr;
		memcpy((void *)addr, (void *)eentry, vec_sz);
		local_flush_icache_range((unsigned long)addr, (unsigned long)addr + vec_sz);
		csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_EENTRY);
		csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_MERRENTRY);
		csr_write64(pcpu_handlers[cpu] + 80*VECSIZE, LOONGARCH_CSR_TLBRENTRY);
	}
#endif
}

void tlb_init(int cpu)
{
	write_csr_pagesize(PS_DEFAULT_SIZE);
	write_csr_stlbpgsize(PS_DEFAULT_SIZE);
	write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE);

	setup_tlb_handler(cpu);
	output_pgtable_bits_defines();
}
Commit	Line	Data
09cfefb7 HC	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
	4	*/
	5	#include <linux/init.h>
	6	#include <linux/sched.h>
	7	#include <linux/smp.h>
	8	#include <linux/mm.h>
	9	#include <linux/hugetlb.h>
	10	#include <linux/export.h>
	11
	12	#include <asm/cpu.h>
	13	#include <asm/bootinfo.h>
	14	#include <asm/mmu_context.h>
	15	#include <asm/pgtable.h>
	16	#include <asm/tlb.h>
	17
	18	void local_flush_tlb_all(void)
	19	{
	20	invtlb_all(INVTLB_CURRENT_ALL, 0, 0);
	21	}
	22	EXPORT_SYMBOL(local_flush_tlb_all);
	23
	24	void local_flush_tlb_user(void)
	25	{
	26	invtlb_all(INVTLB_CURRENT_GFALSE, 0, 0);
	27	}
	28	EXPORT_SYMBOL(local_flush_tlb_user);
	29
	30	void local_flush_tlb_kernel(void)
	31	{
	32	invtlb_all(INVTLB_CURRENT_GTRUE, 0, 0);
	33	}
	34	EXPORT_SYMBOL(local_flush_tlb_kernel);
	35
	36	/*
	37	* All entries common to a mm share an asid. To effectively flush
	38	* these entries, we just bump the asid.
	39	*/
	40	void local_flush_tlb_mm(struct mm_struct *mm)
	41	{
	42	int cpu;
	43
	44	preempt_disable();
	45
	46	cpu = smp_processor_id();
	47
	48	if (asid_valid(mm, cpu))
	49	drop_mmu_context(mm, cpu);
	50	else
	51	cpumask_clear_cpu(cpu, mm_cpumask(mm));
	52
	53	preempt_enable();
	54	}
	55
	56	void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	57	unsigned long end)
	58	{
	59	struct mm_struct *mm = vma->vm_mm;
	60	int cpu = smp_processor_id();
	61
	62	if (asid_valid(mm, cpu)) {
	63	unsigned long size, flags;
	64
65	local_irq_save(flags);
66	start = round_down(start, PAGE_SIZE << 1);
67	end = round_up(end, PAGE_SIZE << 1);
68	size = (end - start) >> (PAGE_SHIFT + 1);
69	if (size <= (current_cpu_data.tlbsizestlbsets ?
70	current_cpu_data.tlbsize / 8 :
71	current_cpu_data.tlbsize / 2)) {
72	int asid = cpu_asid(cpu, mm);
73
74	while (start < end) {
75	invtlb(INVTLB_ADDR_GFALSE_AND_ASID, asid, start);
76	start += (PAGE_SIZE << 1);
77	}
78	} else {
79	drop_mmu_context(mm, cpu);
80	}
81	local_irq_restore(flags);
82	} else {
83	cpumask_clear_cpu(cpu, mm_cpumask(mm));
84	}
85	}
86
87	void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
88	{
89	unsigned long size, flags;
90
91	local_irq_save(flags);
92	size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
93	size = (size + 1) >> 1;
94	if (size <= (current_cpu_data.tlbsizestlbsets ?
95	current_cpu_data.tlbsize / 8 :
96	current_cpu_data.tlbsize / 2)) {
97
98	start &= (PAGE_MASK << 1);
99	end += ((PAGE_SIZE << 1) - 1);
100	end &= (PAGE_MASK << 1);
101
102	while (start < end) {
103	invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, start);
104	start += (PAGE_SIZE << 1);
105	}
106	} else {
107	local_flush_tlb_kernel();
108	}
109	local_irq_restore(flags);
110	}
111
112	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
113	{
114	int cpu = smp_processor_id();
115
116	if (asid_valid(vma->vm_mm, cpu)) {
117	int newpid;
118
119	newpid = cpu_asid(cpu, vma->vm_mm);
120	page &= (PAGE_MASK << 1);
121	invtlb(INVTLB_ADDR_GFALSE_AND_ASID, newpid, page);
122	} else {
123	cpumask_clear_cpu(cpu, mm_cpumask(vma->vm_mm));
124	}
125	}
126
127	/*
128	* This one is only used for pages with the global bit set so we don't care
129	* much about the ASID.
130	*/
131	void local_flush_tlb_one(unsigned long page)
132	{
133	page &= (PAGE_MASK << 1);
134	invtlb_addr(INVTLB_ADDR_GTRUE_OR_ASID, 0, page);
135	}
136
137	static void __update_hugetlb(struct vm_area_struct vma, unsigned long address, pte_t ptep)
138	{
139	#ifdef CONFIG_HUGETLB_PAGE
140	int idx;
141	unsigned long lo;
142	unsigned long flags;
143
144	local_irq_save(flags);
145
146	address &= (PAGE_MASK << 1);
147	write_csr_entryhi(address);
148	tlb_probe();
149	idx = read_csr_tlbidx();
150	write_csr_pagesize(PS_HUGE_SIZE);
151	lo = pmd_to_entrylo(pte_val(*ptep));
152	write_csr_entrylo0(lo);
153	write_csr_entrylo1(lo + (HPAGE_SIZE >> 1));
154
155	if (idx < 0)
156	tlb_write_random();
157	else
158	tlb_write_indexed();
159	write_csr_pagesize(PS_DEFAULT_SIZE);
160
161	local_irq_restore(flags);
162	#endif
163	}
164
165	void __update_tlb(struct vm_area_struct vma, unsigned long address, pte_t ptep)
166	{
167	int idx;
168	unsigned long flags;
169
170	/*
171	* Handle debugger faulting in for debugee.
172	*/
173	if (current->active_mm != vma->vm_mm)
174	return;
175
176	if (pte_val(*ptep) & _PAGE_HUGE) {
177	__update_hugetlb(vma, address, ptep);
178	return;
179	}
180
181	local_irq_save(flags);
182
183	if ((unsigned long)ptep & sizeof(pte_t))
184	ptep--;
185
186	address &= (PAGE_MASK << 1);
187	write_csr_entryhi(address);
188	tlb_probe();
189	idx = read_csr_tlbidx();
190	write_csr_pagesize(PS_DEFAULT_SIZE);
191	write_csr_entrylo0(pte_val(*ptep++));
192	write_csr_entrylo1(pte_val(*ptep));
193	if (idx < 0)
194	tlb_write_random();
195	else
196	tlb_write_indexed();
197
198	local_irq_restore(flags);
199	}
200
201	static void setup_ptwalker(void)
202	{
203	unsigned long pwctl0, pwctl1;
204	unsigned long pgd_i = 0, pgd_w = 0;
205	unsigned long pud_i = 0, pud_w = 0;
206	unsigned long pmd_i = 0, pmd_w = 0;
207	unsigned long pte_i = 0, pte_w = 0;
208
209	pgd_i = PGDIR_SHIFT;
210	pgd_w = PAGE_SHIFT - 3;
211	#if CONFIG_PGTABLE_LEVELS > 3
212	pud_i = PUD_SHIFT;
213	pud_w = PAGE_SHIFT - 3;
214	#endif
215	#if CONFIG_PGTABLE_LEVELS > 2
216	pmd_i = PMD_SHIFT;
217	pmd_w = PAGE_SHIFT - 3;
218	#endif
219	pte_i = PAGE_SHIFT;
220	pte_w = PAGE_SHIFT - 3;
221
222	pwctl0 = pte_i \| pte_w << 5 \| pmd_i << 10 \| pmd_w << 15 \| pud_i << 20 \| pud_w << 25;
223	pwctl1 = pgd_i \| pgd_w << 6;
224
225	csr_write64(pwctl0, LOONGARCH_CSR_PWCTL0);
226	csr_write64(pwctl1, LOONGARCH_CSR_PWCTL1);
227	csr_write64((long)swapper_pg_dir, LOONGARCH_CSR_PGDH);
228	csr_write64((long)invalid_pg_dir, LOONGARCH_CSR_PGDL);
229	csr_write64((long)smp_processor_id(), LOONGARCH_CSR_TMID);
230	}
231
232	static void output_pgtable_bits_defines(void)
233	{
234	#define pr_define(fmt, ...) \
235	pr_debug("#define " fmt, ##__VA_ARGS__)
236
237	pr_debug("#include <asm/asm.h>\n");
238	pr_debug("#include <asm/regdef.h>\n");
239	pr_debug("\n");
240
241	pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
242	pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
243	pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
244	pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
245	pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
246	pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
247	pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
248	pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
249	pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
250	pr_debug("\n");
251	}
252
d4b6f156 HC	253	#ifdef CONFIG_NUMA
	254	static unsigned long pcpu_handlers[NR_CPUS];
	255	#endif
	256	extern long exception_handlers[VECSIZE * 128 / sizeof(long)];
09cfefb7	257
d4b6f156 HC	258	void setup_tlb_handler(int cpu)
d4b6f156 HC	259	{
09cfefb7	260	setup_ptwalker();
1299a129	261	local_flush_tlb_all();
09cfefb7 HC	262
09cfefb7 HC	263	/* The tlb handlers are generated only once */
d4b6f156	264	if (cpu == 0) {
09cfefb7 HC	265	memcpy((void *)tlbrentry, handle_tlb_refill, 0x80);
	266	local_flush_icache_range(tlbrentry, tlbrentry + 0x80);
	267	set_handler(EXCCODE_TLBI * VECSIZE, handle_tlb_load, VECSIZE);
	268	set_handler(EXCCODE_TLBL * VECSIZE, handle_tlb_load, VECSIZE);
	269	set_handler(EXCCODE_TLBS * VECSIZE, handle_tlb_store, VECSIZE);
	270	set_handler(EXCCODE_TLBM * VECSIZE, handle_tlb_modify, VECSIZE);
	271	set_handler(EXCCODE_TLBNR * VECSIZE, handle_tlb_protect, VECSIZE);
	272	set_handler(EXCCODE_TLBNX * VECSIZE, handle_tlb_protect, VECSIZE);
	273	set_handler(EXCCODE_TLBPE * VECSIZE, handle_tlb_protect, VECSIZE);
09cfefb7	274	}
d4b6f156 HC	275	#ifdef CONFIG_NUMA
	276	else {
	277	void *addr;
	278	struct page *page;
	279	const int vec_sz = sizeof(exception_handlers);
	280
	281	if (pcpu_handlers[cpu])
	282	return;
	283
bab1c299	284	page = alloc_pages_node(cpu_to_node(cpu), GFP_ATOMIC, get_order(vec_sz));
d4b6f156 HC	285	if (!page)
	286	return;
	287
	288	addr = page_address(page);
26808ceb	289	pcpu_handlers[cpu] = (unsigned long)addr;
d4b6f156 HC	290	memcpy((void )addr, (void )eentry, vec_sz);
d4b6f156 HC	291	local_flush_icache_range((unsigned long)addr, (unsigned long)addr + vec_sz);
26808ceb HC	292	csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_EENTRY);
26808ceb HC	293	csr_write64(pcpu_handlers[cpu], LOONGARCH_CSR_MERRENTRY);
d4b6f156 HC	294	csr_write64(pcpu_handlers[cpu] + 80*VECSIZE, LOONGARCH_CSR_TLBRENTRY);
	295	}
	296	#endif
09cfefb7 HC	297	}
09cfefb7 HC	298
d4b6f156	299	void tlb_init(int cpu)
09cfefb7 HC	300	{
	301	write_csr_pagesize(PS_DEFAULT_SIZE);
	302	write_csr_stlbpgsize(PS_DEFAULT_SIZE);
	303	write_csr_tlbrefill_pagesize(PS_DEFAULT_SIZE);
1299a129	304
d4b6f156	305	setup_tlb_handler(cpu);
1299a129	306	output_pgtable_bits_defines();
09cfefb7	307	}