[linux-block.git] / arch / loongarch / mm / tlbex.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
 */
#include <asm/asm.h>
#include <asm/export.h>
#include <asm/loongarch.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/regdef.h>
#include <asm/stackframe.h>

	.macro tlb_do_page_fault, write
	SYM_FUNC_START(tlb_do_page_fault_\write)
	SAVE_ALL
	csrrd	a2, LOONGARCH_CSR_BADV
	move	a0, sp
	REG_S	a2, sp, PT_BVADDR
	li.w	a1, \write
	la.abs	t0, do_page_fault
	jirl    ra, t0, 0
	RESTORE_ALL_AND_RET
	SYM_FUNC_END(tlb_do_page_fault_\write)
	.endm

	tlb_do_page_fault 0
	tlb_do_page_fault 1

SYM_FUNC_START(handle_tlb_protect)
	BACKUP_T0T1
	SAVE_ALL
	move	a0, sp
	move	a1, zero
	csrrd	a2, LOONGARCH_CSR_BADV
	REG_S	a2, sp, PT_BVADDR
	la.abs	t0, do_page_fault
	jirl    ra, t0, 0
	RESTORE_ALL_AND_RET
SYM_FUNC_END(handle_tlb_protect)

SYM_FUNC_START(handle_tlb_load)
	csrwr	t0, EXCEPTION_KS0
	csrwr	t1, EXCEPTION_KS1
	csrwr	ra, EXCEPTION_KS2

	/*
	 * The vmalloc handling is not in the hotpath.
	 */
	csrrd	t0, LOONGARCH_CSR_BADV
	blt	t0, zero, vmalloc_load
	csrrd	t1, LOONGARCH_CSR_PGDL

vmalloc_done_load:
	/* Get PGD offset in bytes */
	srli.d	t0, t0, PGDIR_SHIFT
	andi	t0, t0, (PTRS_PER_PGD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#if CONFIG_PGTABLE_LEVELS > 3
	csrrd	t0, LOONGARCH_CSR_BADV
	ld.d	t1, t1, 0
	srli.d	t0, t0, PUD_SHIFT
	andi	t0, t0, (PTRS_PER_PUD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#endif
#if CONFIG_PGTABLE_LEVELS > 2
	csrrd	t0, LOONGARCH_CSR_BADV
	ld.d	t1, t1, 0
	srli.d	t0, t0, PMD_SHIFT
	andi	t0, t0, (PTRS_PER_PMD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#endif
	ld.d	ra, t1, 0

	/*
	 * For huge tlb entries, pmde doesn't contain an address but
	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
	 * see if we need to jump to huge tlb processing.
	 */
	andi	t0, ra, _PAGE_HUGE
	bne	t0, zero, tlb_huge_update_load

	csrrd	t0, LOONGARCH_CSR_BADV
	srli.d	t0, t0, (PAGE_SHIFT + PTE_ORDER)
	andi	t0, t0, (PTRS_PER_PTE - 1)
	slli.d	t0, t0, _PTE_T_LOG2
	add.d	t1, ra, t0

#ifdef CONFIG_SMP
smp_pgtable_change_load:
#endif
#ifdef CONFIG_SMP
	ll.d	t0, t1, 0
#else
	ld.d	t0, t1, 0
#endif
	tlbsrch

	srli.d	ra, t0, _PAGE_PRESENT_SHIFT
	andi	ra, ra, 1
	beq	ra, zero, nopage_tlb_load

	ori	t0, t0, _PAGE_VALID
#ifdef CONFIG_SMP
	sc.d	t0, t1, 0
	beq	t0, zero, smp_pgtable_change_load
#else
	st.d	t0, t1, 0
#endif
	ori	t1, t1, 8
	xori	t1, t1, 8
	ld.d	t0, t1, 0
	ld.d	t1, t1, 8
	csrwr	t0, LOONGARCH_CSR_TLBELO0
	csrwr	t1, LOONGARCH_CSR_TLBELO1
	tlbwr
leave_load:
	csrrd	t0, EXCEPTION_KS0
	csrrd	t1, EXCEPTION_KS1
	csrrd	ra, EXCEPTION_KS2
	ertn
#ifdef CONFIG_64BIT
vmalloc_load:
	la.abs	t1, swapper_pg_dir
	b	vmalloc_done_load
#endif

	/*
	 * This is the entry point when build_tlbchange_handler_head
	 * spots a huge page.
	 */
tlb_huge_update_load:
#ifdef CONFIG_SMP
	ll.d	t0, t1, 0
#else
	ld.d	t0, t1, 0
#endif
	srli.d	ra, t0, _PAGE_PRESENT_SHIFT
	andi	ra, ra, 1
	beq	ra, zero, nopage_tlb_load
	tlbsrch

	ori	t0, t0, _PAGE_VALID
#ifdef CONFIG_SMP
	sc.d	t0, t1, 0
	beq	t0, zero, tlb_huge_update_load
	ld.d	t0, t1, 0
#else
	st.d	t0, t1, 0
#endif
	addu16i.d	t1, zero, -(CSR_TLBIDX_EHINV >> 16)
	addi.d	ra, t1, 0
	csrxchg	ra, t1, LOONGARCH_CSR_TLBIDX
	tlbwr

	csrxchg	zero, t1, LOONGARCH_CSR_TLBIDX

	/*
	 * A huge PTE describes an area the size of the
	 * configured huge page size. This is twice the
	 * of the large TLB entry size we intend to use.
	 * A TLB entry half the size of the configured
	 * huge page size is configured into entrylo0
	 * and entrylo1 to cover the contiguous huge PTE
	 * address space.
	 */
	/* Huge page: Move Global bit */
	xori	t0, t0, _PAGE_HUGE
	lu12i.w	t1, _PAGE_HGLOBAL >> 12
	and	t1, t0, t1
	srli.d	t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
	or	t0, t0, t1

	addi.d	ra, t0, 0
	csrwr	t0, LOONGARCH_CSR_TLBELO0
	addi.d	t0, ra, 0

	/* Convert to entrylo1 */
	addi.d	t1, zero, 1
	slli.d	t1, t1, (HPAGE_SHIFT - 1)
	add.d	t0, t0, t1
	csrwr	t0, LOONGARCH_CSR_TLBELO1

	/* Set huge page tlb entry size */
	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX

	tlbfill

	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX

nopage_tlb_load:
	dbar	0
	csrrd	ra, EXCEPTION_KS2
	la.abs	t0, tlb_do_page_fault_0
	jr	t0
SYM_FUNC_END(handle_tlb_load)

SYM_FUNC_START(handle_tlb_store)
	csrwr	t0, EXCEPTION_KS0
	csrwr	t1, EXCEPTION_KS1
	csrwr	ra, EXCEPTION_KS2

	/*
	 * The vmalloc handling is not in the hotpath.
	 */
	csrrd	t0, LOONGARCH_CSR_BADV
	blt	t0, zero, vmalloc_store
	csrrd	t1, LOONGARCH_CSR_PGDL

vmalloc_done_store:
	/* Get PGD offset in bytes */
	srli.d	t0, t0, PGDIR_SHIFT
	andi	t0, t0, (PTRS_PER_PGD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0

#if CONFIG_PGTABLE_LEVELS > 3
	csrrd	t0, LOONGARCH_CSR_BADV
	ld.d	t1, t1, 0
	srli.d	t0, t0, PUD_SHIFT
	andi	t0, t0, (PTRS_PER_PUD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#endif
#if CONFIG_PGTABLE_LEVELS > 2
	csrrd	t0, LOONGARCH_CSR_BADV
	ld.d	t1, t1, 0
	srli.d	t0, t0, PMD_SHIFT
	andi	t0, t0, (PTRS_PER_PMD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#endif
	ld.d	ra, t1, 0

	/*
	 * For huge tlb entries, pmde doesn't contain an address but
	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
	 * see if we need to jump to huge tlb processing.
	 */
	andi	t0, ra, _PAGE_HUGE
	bne	t0, zero, tlb_huge_update_store

	csrrd	t0, LOONGARCH_CSR_BADV
	srli.d	t0, t0, (PAGE_SHIFT + PTE_ORDER)
	andi	t0, t0, (PTRS_PER_PTE - 1)
	slli.d	t0, t0, _PTE_T_LOG2
	add.d	t1, ra, t0

#ifdef CONFIG_SMP
smp_pgtable_change_store:
#endif
#ifdef CONFIG_SMP
	ll.d	t0, t1, 0
#else
	ld.d	t0, t1, 0
#endif
	tlbsrch

	srli.d	ra, t0, _PAGE_PRESENT_SHIFT
	andi	ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
	xori	ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
	bne	ra, zero, nopage_tlb_store

	ori	t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#ifdef CONFIG_SMP
	sc.d	t0, t1, 0
	beq	t0, zero, smp_pgtable_change_store
#else
	st.d	t0, t1, 0
#endif

	ori	t1, t1, 8
	xori	t1, t1, 8
	ld.d	t0, t1, 0
	ld.d	t1, t1, 8
	csrwr	t0, LOONGARCH_CSR_TLBELO0
	csrwr	t1, LOONGARCH_CSR_TLBELO1
	tlbwr
leave_store:
	csrrd	t0, EXCEPTION_KS0
	csrrd	t1, EXCEPTION_KS1
	csrrd	ra, EXCEPTION_KS2
	ertn
#ifdef CONFIG_64BIT
vmalloc_store:
	la.abs	t1, swapper_pg_dir
	b	vmalloc_done_store
#endif

	/*
	 * This is the entry point when build_tlbchange_handler_head
	 * spots a huge page.
	 */
tlb_huge_update_store:
#ifdef CONFIG_SMP
	ll.d	t0, t1, 0
#else
	ld.d	t0, t1, 0
#endif
	srli.d	ra, t0, _PAGE_PRESENT_SHIFT
	andi	ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
	xori	ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
	bne	ra, zero, nopage_tlb_store

	tlbsrch
	ori	t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)

#ifdef CONFIG_SMP
	sc.d	t0, t1, 0
	beq	t0, zero, tlb_huge_update_store
	ld.d	t0, t1, 0
#else
	st.d	t0, t1, 0
#endif
	addu16i.d	t1, zero, -(CSR_TLBIDX_EHINV >> 16)
	addi.d	ra, t1, 0
	csrxchg	ra, t1, LOONGARCH_CSR_TLBIDX
	tlbwr

	csrxchg	zero, t1, LOONGARCH_CSR_TLBIDX
	/*
	 * A huge PTE describes an area the size of the
	 * configured huge page size. This is twice the
	 * of the large TLB entry size we intend to use.
	 * A TLB entry half the size of the configured
	 * huge page size is configured into entrylo0
	 * and entrylo1 to cover the contiguous huge PTE
	 * address space.
	 */
	/* Huge page: Move Global bit */
	xori	t0, t0, _PAGE_HUGE
	lu12i.w	t1, _PAGE_HGLOBAL >> 12
	and	t1, t0, t1
	srli.d	t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
	or	t0, t0, t1

	addi.d	ra, t0, 0
	csrwr	t0, LOONGARCH_CSR_TLBELO0
	addi.d	t0, ra, 0

	/* Convert to entrylo1 */
	addi.d	t1, zero, 1
	slli.d	t1, t1, (HPAGE_SHIFT - 1)
	add.d	t0, t0, t1
	csrwr	t0, LOONGARCH_CSR_TLBELO1

	/* Set huge page tlb entry size */
	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX

	tlbfill

	/* Reset default page size */
	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX

nopage_tlb_store:
	dbar	0
	csrrd	ra, EXCEPTION_KS2
	la.abs	t0, tlb_do_page_fault_1
	jr	t0
SYM_FUNC_END(handle_tlb_store)

SYM_FUNC_START(handle_tlb_modify)
	csrwr	t0, EXCEPTION_KS0
	csrwr	t1, EXCEPTION_KS1
	csrwr	ra, EXCEPTION_KS2

	/*
	 * The vmalloc handling is not in the hotpath.
	 */
	csrrd	t0, LOONGARCH_CSR_BADV
	blt	t0, zero, vmalloc_modify
	csrrd	t1, LOONGARCH_CSR_PGDL

vmalloc_done_modify:
	/* Get PGD offset in bytes */
	srli.d	t0, t0, PGDIR_SHIFT
	andi	t0, t0, (PTRS_PER_PGD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#if CONFIG_PGTABLE_LEVELS > 3
	csrrd	t0, LOONGARCH_CSR_BADV
	ld.d	t1, t1, 0
	srli.d	t0, t0, PUD_SHIFT
	andi	t0, t0, (PTRS_PER_PUD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#endif
#if CONFIG_PGTABLE_LEVELS > 2
	csrrd	t0, LOONGARCH_CSR_BADV
	ld.d	t1, t1, 0
	srli.d	t0, t0, PMD_SHIFT
	andi	t0, t0, (PTRS_PER_PMD - 1)
	slli.d	t0, t0, 3
	add.d	t1, t1, t0
#endif
	ld.d	ra, t1, 0

	/*
	 * For huge tlb entries, pmde doesn't contain an address but
	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
	 * see if we need to jump to huge tlb processing.
	 */
	andi	t0, ra, _PAGE_HUGE
	bne	t0, zero, tlb_huge_update_modify

	csrrd	t0, LOONGARCH_CSR_BADV
	srli.d	t0, t0, (PAGE_SHIFT + PTE_ORDER)
	andi	t0, t0, (PTRS_PER_PTE - 1)
	slli.d	t0, t0, _PTE_T_LOG2
	add.d	t1, ra, t0

#ifdef CONFIG_SMP
smp_pgtable_change_modify:
#endif
#ifdef CONFIG_SMP
	ll.d	t0, t1, 0
#else
	ld.d	t0, t1, 0
#endif
	tlbsrch

	srli.d	ra, t0, _PAGE_WRITE_SHIFT
	andi	ra, ra, 1
	beq	ra, zero, nopage_tlb_modify

	ori	t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#ifdef CONFIG_SMP
	sc.d	t0, t1, 0
	beq	t0, zero, smp_pgtable_change_modify
#else
	st.d	t0, t1, 0
#endif
	ori	t1, t1, 8
	xori	t1, t1, 8
	ld.d	t0, t1, 0
	ld.d	t1, t1, 8
	csrwr	t0, LOONGARCH_CSR_TLBELO0
	csrwr	t1, LOONGARCH_CSR_TLBELO1
	tlbwr
leave_modify:
	csrrd	t0, EXCEPTION_KS0
	csrrd	t1, EXCEPTION_KS1
	csrrd	ra, EXCEPTION_KS2
	ertn
#ifdef CONFIG_64BIT
vmalloc_modify:
	la.abs  t1, swapper_pg_dir
	b	vmalloc_done_modify
#endif

	/*
	 * This is the entry point when
	 * build_tlbchange_handler_head spots a huge page.
	 */
tlb_huge_update_modify:
#ifdef CONFIG_SMP
	ll.d	t0, t1, 0
#else
	ld.d	t0, t1, 0
#endif

	srli.d	ra, t0, _PAGE_WRITE_SHIFT
	andi	ra, ra, 1
	beq	ra, zero, nopage_tlb_modify

	tlbsrch
	ori	t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)

#ifdef CONFIG_SMP
	sc.d	t0, t1, 0
	beq	t0, zero, tlb_huge_update_modify
	ld.d	t0, t1, 0
#else
	st.d	t0, t1, 0
#endif
	/*
	 * A huge PTE describes an area the size of the
	 * configured huge page size. This is twice the
	 * of the large TLB entry size we intend to use.
	 * A TLB entry half the size of the configured
	 * huge page size is configured into entrylo0
	 * and entrylo1 to cover the contiguous huge PTE
	 * address space.
	 */
	/* Huge page: Move Global bit */
	xori	t0, t0, _PAGE_HUGE
	lu12i.w	t1, _PAGE_HGLOBAL >> 12
	and	t1, t0, t1
	srli.d	t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
	or	t0, t0, t1

	addi.d	ra, t0, 0
	csrwr	t0, LOONGARCH_CSR_TLBELO0
	addi.d	t0, ra, 0

	/* Convert to entrylo1 */
	addi.d	t1, zero, 1
	slli.d	t1, t1, (HPAGE_SHIFT - 1)
	add.d	t0, t0, t1
	csrwr	t0, LOONGARCH_CSR_TLBELO1

	/* Set huge page tlb entry size */
	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
	csrxchg	t1, t0, LOONGARCH_CSR_TLBIDX

	tlbwr

	/* Reset default page size */
	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
	csrxchg	t1, t0, LOONGARCH_CSR_TLBIDX

nopage_tlb_modify:
	dbar	0
	csrrd	ra, EXCEPTION_KS2
	la.abs	t0, tlb_do_page_fault_1
	jr	t0
SYM_FUNC_END(handle_tlb_modify)

SYM_FUNC_START(handle_tlb_refill)
	csrwr	t0, LOONGARCH_CSR_TLBRSAVE
	csrrd	t0, LOONGARCH_CSR_PGD
	lddir	t0, t0, 3
#if CONFIG_PGTABLE_LEVELS > 3
	lddir	t0, t0, 2
#endif
#if CONFIG_PGTABLE_LEVELS > 2
	lddir	t0, t0, 1
#endif
	ldpte	t0, 0
	ldpte	t0, 1
	tlbfill
	csrrd	t0, LOONGARCH_CSR_TLBRSAVE
	ertn
SYM_FUNC_END(handle_tlb_refill)
Commit	Line	Data
09cfefb7 HC	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/*
	3	* Copyright (C) 2020-2022 Loongson Technology Corporation Limited
	4	*/
	5	#include <asm/asm.h>
	6	#include <asm/export.h>
	7	#include <asm/loongarch.h>
	8	#include <asm/page.h>
	9	#include <asm/pgtable.h>
	10	#include <asm/regdef.h>
	11	#include <asm/stackframe.h>
	12
	13	.macro tlb_do_page_fault, write
	14	SYM_FUNC_START(tlb_do_page_fault_\write)
	15	SAVE_ALL
	16	csrrd a2, LOONGARCH_CSR_BADV
	17	move a0, sp
	18	REG_S a2, sp, PT_BVADDR
	19	li.w a1, \write
	20	la.abs t0, do_page_fault
	21	jirl ra, t0, 0
	22	RESTORE_ALL_AND_RET
	23	SYM_FUNC_END(tlb_do_page_fault_\write)
	24	.endm
	25
	26	tlb_do_page_fault 0
	27	tlb_do_page_fault 1
	28
	29	SYM_FUNC_START(handle_tlb_protect)
	30	BACKUP_T0T1
	31	SAVE_ALL
	32	move a0, sp
	33	move a1, zero
	34	csrrd a2, LOONGARCH_CSR_BADV
	35	REG_S a2, sp, PT_BVADDR
	36	la.abs t0, do_page_fault
	37	jirl ra, t0, 0
	38	RESTORE_ALL_AND_RET
	39	SYM_FUNC_END(handle_tlb_protect)
	40
	41	SYM_FUNC_START(handle_tlb_load)
	42	csrwr t0, EXCEPTION_KS0
	43	csrwr t1, EXCEPTION_KS1
	44	csrwr ra, EXCEPTION_KS2
	45
	46	/*
	47	* The vmalloc handling is not in the hotpath.
	48	*/
	49	csrrd t0, LOONGARCH_CSR_BADV
d8e7f201	50	blt t0, zero, vmalloc_load
09cfefb7 HC	51	csrrd t1, LOONGARCH_CSR_PGDL
	52
	53	vmalloc_done_load:
	54	/* Get PGD offset in bytes */
	55	srli.d t0, t0, PGDIR_SHIFT
	56	andi t0, t0, (PTRS_PER_PGD - 1)
	57	slli.d t0, t0, 3
	58	add.d t1, t1, t0
	59	#if CONFIG_PGTABLE_LEVELS > 3
	60	csrrd t0, LOONGARCH_CSR_BADV
	61	ld.d t1, t1, 0
	62	srli.d t0, t0, PUD_SHIFT
	63	andi t0, t0, (PTRS_PER_PUD - 1)
	64	slli.d t0, t0, 3
	65	add.d t1, t1, t0
	66	#endif
	67	#if CONFIG_PGTABLE_LEVELS > 2
	68	csrrd t0, LOONGARCH_CSR_BADV
	69	ld.d t1, t1, 0
	70	srli.d t0, t0, PMD_SHIFT
	71	andi t0, t0, (PTRS_PER_PMD - 1)
	72	slli.d t0, t0, 3
	73	add.d t1, t1, t0
	74	#endif
	75	ld.d ra, t1, 0
	76
	77	/*
	78	* For huge tlb entries, pmde doesn't contain an address but
	79	* instead contains the tlb pte. Check the PAGE_HUGE bit and
	80	* see if we need to jump to huge tlb processing.
	81	*/
	82	andi t0, ra, _PAGE_HUGE
d8e7f201	83	bne t0, zero, tlb_huge_update_load
09cfefb7 HC	84
	85	csrrd t0, LOONGARCH_CSR_BADV
	86	srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
	87	andi t0, t0, (PTRS_PER_PTE - 1)
	88	slli.d t0, t0, _PTE_T_LOG2
	89	add.d t1, ra, t0
	90
46859ac8 HC	91	#ifdef CONFIG_SMP
	92	smp_pgtable_change_load:
	93	#endif
	94	#ifdef CONFIG_SMP
	95	ll.d t0, t1, 0
	96	#else
09cfefb7	97	ld.d t0, t1, 0
46859ac8	98	#endif
09cfefb7 HC	99	tlbsrch
	100
	101	srli.d ra, t0, _PAGE_PRESENT_SHIFT
	102	andi ra, ra, 1
d8e7f201	103	beq ra, zero, nopage_tlb_load
09cfefb7 HC	104
09cfefb7 HC	105	ori t0, t0, _PAGE_VALID
46859ac8 HC	106	#ifdef CONFIG_SMP
46859ac8 HC	107	sc.d t0, t1, 0
d8e7f201	108	beq t0, zero, smp_pgtable_change_load
46859ac8	109	#else
09cfefb7	110	st.d t0, t1, 0
46859ac8	111	#endif
09cfefb7 HC	112	ori t1, t1, 8
	113	xori t1, t1, 8
	114	ld.d t0, t1, 0
	115	ld.d t1, t1, 8
	116	csrwr t0, LOONGARCH_CSR_TLBELO0
	117	csrwr t1, LOONGARCH_CSR_TLBELO1
	118	tlbwr
	119	leave_load:
	120	csrrd t0, EXCEPTION_KS0
	121	csrrd t1, EXCEPTION_KS1
	122	csrrd ra, EXCEPTION_KS2
	123	ertn
	124	#ifdef CONFIG_64BIT
	125	vmalloc_load:
	126	la.abs t1, swapper_pg_dir
	127	b vmalloc_done_load
	128	#endif
	129
	130	/*
	131	* This is the entry point when build_tlbchange_handler_head
	132	* spots a huge page.
	133	*/
	134	tlb_huge_update_load:
46859ac8 HC	135	#ifdef CONFIG_SMP
	136	ll.d t0, t1, 0
	137	#else
09cfefb7	138	ld.d t0, t1, 0
46859ac8	139	#endif
09cfefb7 HC	140	srli.d ra, t0, _PAGE_PRESENT_SHIFT
09cfefb7 HC	141	andi ra, ra, 1
d8e7f201	142	beq ra, zero, nopage_tlb_load
09cfefb7 HC	143	tlbsrch
	144
	145	ori t0, t0, _PAGE_VALID
46859ac8 HC	146	#ifdef CONFIG_SMP
46859ac8 HC	147	sc.d t0, t1, 0
d8e7f201	148	beq t0, zero, tlb_huge_update_load
46859ac8 HC	149	ld.d t0, t1, 0
46859ac8 HC	150	#else
09cfefb7	151	st.d t0, t1, 0
46859ac8	152	#endif
d8e7f201	153	addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16)
09cfefb7 HC	154	addi.d ra, t1, 0
	155	csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
	156	tlbwr
	157
d8e7f201	158	csrxchg zero, t1, LOONGARCH_CSR_TLBIDX
09cfefb7 HC	159
	160	/*
	161	* A huge PTE describes an area the size of the
	162	* configured huge page size. This is twice the
	163	* of the large TLB entry size we intend to use.
	164	* A TLB entry half the size of the configured
	165	* huge page size is configured into entrylo0
	166	* and entrylo1 to cover the contiguous huge PTE
	167	* address space.
	168	*/
	169	/* Huge page: Move Global bit */
	170	xori t0, t0, _PAGE_HUGE
	171	lu12i.w t1, _PAGE_HGLOBAL >> 12
	172	and t1, t0, t1
	173	srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
	174	or t0, t0, t1
	175
	176	addi.d ra, t0, 0
	177	csrwr t0, LOONGARCH_CSR_TLBELO0
	178	addi.d t0, ra, 0
	179
	180	/* Convert to entrylo1 */
d8e7f201	181	addi.d t1, zero, 1
09cfefb7 HC	182	slli.d t1, t1, (HPAGE_SHIFT - 1)
	183	add.d t0, t0, t1
	184	csrwr t0, LOONGARCH_CSR_TLBELO1
	185
	186	/* Set huge page tlb entry size */
d8e7f201 WX	187	addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
d8e7f201 WX	188	addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
09cfefb7 HC	189	csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
	190
	191	tlbfill
	192
d8e7f201 WX	193	addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
d8e7f201 WX	194	addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
09cfefb7 HC	195	csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
	196
	197	nopage_tlb_load:
46859ac8	198	dbar 0
09cfefb7 HC	199	csrrd ra, EXCEPTION_KS2
09cfefb7 HC	200	la.abs t0, tlb_do_page_fault_0
07b48069	201	jr t0
09cfefb7 HC	202	SYM_FUNC_END(handle_tlb_load)
	203
	204	SYM_FUNC_START(handle_tlb_store)
	205	csrwr t0, EXCEPTION_KS0
	206	csrwr t1, EXCEPTION_KS1
	207	csrwr ra, EXCEPTION_KS2
	208
	209	/*
	210	* The vmalloc handling is not in the hotpath.
	211	*/
	212	csrrd t0, LOONGARCH_CSR_BADV
d8e7f201	213	blt t0, zero, vmalloc_store
09cfefb7 HC	214	csrrd t1, LOONGARCH_CSR_PGDL
	215
	216	vmalloc_done_store:
	217	/* Get PGD offset in bytes */
	218	srli.d t0, t0, PGDIR_SHIFT
	219	andi t0, t0, (PTRS_PER_PGD - 1)
	220	slli.d t0, t0, 3
	221	add.d t1, t1, t0
	222
	223	#if CONFIG_PGTABLE_LEVELS > 3
	224	csrrd t0, LOONGARCH_CSR_BADV
	225	ld.d t1, t1, 0
	226	srli.d t0, t0, PUD_SHIFT
	227	andi t0, t0, (PTRS_PER_PUD - 1)
	228	slli.d t0, t0, 3
	229	add.d t1, t1, t0
	230	#endif
	231	#if CONFIG_PGTABLE_LEVELS > 2
	232	csrrd t0, LOONGARCH_CSR_BADV
	233	ld.d t1, t1, 0
	234	srli.d t0, t0, PMD_SHIFT
	235	andi t0, t0, (PTRS_PER_PMD - 1)
	236	slli.d t0, t0, 3
	237	add.d t1, t1, t0
	238	#endif
	239	ld.d ra, t1, 0
	240
	241	/*
	242	* For huge tlb entries, pmde doesn't contain an address but
	243	* instead contains the tlb pte. Check the PAGE_HUGE bit and
	244	* see if we need to jump to huge tlb processing.
	245	*/
	246	andi t0, ra, _PAGE_HUGE
d8e7f201	247	bne t0, zero, tlb_huge_update_store
09cfefb7 HC	248
	249	csrrd t0, LOONGARCH_CSR_BADV
	250	srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
	251	andi t0, t0, (PTRS_PER_PTE - 1)
	252	slli.d t0, t0, _PTE_T_LOG2
	253	add.d t1, ra, t0
	254
46859ac8 HC	255	#ifdef CONFIG_SMP
	256	smp_pgtable_change_store:
	257	#endif
	258	#ifdef CONFIG_SMP
	259	ll.d t0, t1, 0
	260	#else
09cfefb7	261	ld.d t0, t1, 0
46859ac8	262	#endif
09cfefb7 HC	263	tlbsrch
	264
	265	srli.d ra, t0, _PAGE_PRESENT_SHIFT
	266	andi ra, ra, ((_PAGE_PRESENT \| _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
	267	xori ra, ra, ((_PAGE_PRESENT \| _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
d8e7f201	268	bne ra, zero, nopage_tlb_store
09cfefb7 HC	269
09cfefb7 HC	270	ori t0, t0, (_PAGE_VALID \| _PAGE_DIRTY \| _PAGE_MODIFIED)
46859ac8 HC	271	#ifdef CONFIG_SMP
46859ac8 HC	272	sc.d t0, t1, 0
d8e7f201	273	beq t0, zero, smp_pgtable_change_store
46859ac8	274	#else
09cfefb7	275	st.d t0, t1, 0
46859ac8	276	#endif
09cfefb7 HC	277
	278	ori t1, t1, 8
	279	xori t1, t1, 8
	280	ld.d t0, t1, 0
	281	ld.d t1, t1, 8
	282	csrwr t0, LOONGARCH_CSR_TLBELO0
	283	csrwr t1, LOONGARCH_CSR_TLBELO1
	284	tlbwr
	285	leave_store:
	286	csrrd t0, EXCEPTION_KS0
	287	csrrd t1, EXCEPTION_KS1
	288	csrrd ra, EXCEPTION_KS2
	289	ertn
	290	#ifdef CONFIG_64BIT
	291	vmalloc_store:
	292	la.abs t1, swapper_pg_dir
	293	b vmalloc_done_store
	294	#endif
	295
	296	/*
	297	* This is the entry point when build_tlbchange_handler_head
	298	* spots a huge page.
	299	*/
	300	tlb_huge_update_store:
46859ac8 HC	301	#ifdef CONFIG_SMP
	302	ll.d t0, t1, 0
	303	#else
09cfefb7	304	ld.d t0, t1, 0
46859ac8	305	#endif
09cfefb7 HC	306	srli.d ra, t0, _PAGE_PRESENT_SHIFT
	307	andi ra, ra, ((_PAGE_PRESENT \| _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
	308	xori ra, ra, ((_PAGE_PRESENT \| _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
d8e7f201	309	bne ra, zero, nopage_tlb_store
09cfefb7 HC	310
	311	tlbsrch
	312	ori t0, t0, (_PAGE_VALID \| _PAGE_DIRTY \| _PAGE_MODIFIED)
	313
46859ac8 HC	314	#ifdef CONFIG_SMP
46859ac8 HC	315	sc.d t0, t1, 0
d8e7f201	316	beq t0, zero, tlb_huge_update_store
46859ac8 HC	317	ld.d t0, t1, 0
46859ac8 HC	318	#else
09cfefb7	319	st.d t0, t1, 0
46859ac8	320	#endif
d8e7f201	321	addu16i.d t1, zero, -(CSR_TLBIDX_EHINV >> 16)
09cfefb7 HC	322	addi.d ra, t1, 0
	323	csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
	324	tlbwr
	325
d8e7f201	326	csrxchg zero, t1, LOONGARCH_CSR_TLBIDX
09cfefb7 HC	327	/*
	328	* A huge PTE describes an area the size of the
	329	* configured huge page size. This is twice the
	330	* of the large TLB entry size we intend to use.
	331	* A TLB entry half the size of the configured
	332	* huge page size is configured into entrylo0
	333	* and entrylo1 to cover the contiguous huge PTE
	334	* address space.
	335	*/
	336	/* Huge page: Move Global bit */
	337	xori t0, t0, _PAGE_HUGE
	338	lu12i.w t1, _PAGE_HGLOBAL >> 12
	339	and t1, t0, t1
	340	srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
	341	or t0, t0, t1
	342
	343	addi.d ra, t0, 0
	344	csrwr t0, LOONGARCH_CSR_TLBELO0
	345	addi.d t0, ra, 0
	346
	347	/* Convert to entrylo1 */
d8e7f201	348	addi.d t1, zero, 1
09cfefb7 HC	349	slli.d t1, t1, (HPAGE_SHIFT - 1)
	350	add.d t0, t0, t1
	351	csrwr t0, LOONGARCH_CSR_TLBELO1
	352
	353	/* Set huge page tlb entry size */
d8e7f201 WX	354	addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
d8e7f201 WX	355	addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
09cfefb7 HC	356	csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
	357
	358	tlbfill
	359
	360	/* Reset default page size */
d8e7f201 WX	361	addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
d8e7f201 WX	362	addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
09cfefb7 HC	363	csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
	364
	365	nopage_tlb_store:
46859ac8	366	dbar 0
09cfefb7 HC	367	csrrd ra, EXCEPTION_KS2
09cfefb7 HC	368	la.abs t0, tlb_do_page_fault_1
07b48069	369	jr t0
09cfefb7 HC	370	SYM_FUNC_END(handle_tlb_store)
	371
	372	SYM_FUNC_START(handle_tlb_modify)
	373	csrwr t0, EXCEPTION_KS0
	374	csrwr t1, EXCEPTION_KS1
	375	csrwr ra, EXCEPTION_KS2
	376
	377	/*
	378	* The vmalloc handling is not in the hotpath.
	379	*/
	380	csrrd t0, LOONGARCH_CSR_BADV
d8e7f201	381	blt t0, zero, vmalloc_modify
09cfefb7 HC	382	csrrd t1, LOONGARCH_CSR_PGDL
	383
	384	vmalloc_done_modify:
	385	/* Get PGD offset in bytes */
	386	srli.d t0, t0, PGDIR_SHIFT
	387	andi t0, t0, (PTRS_PER_PGD - 1)
	388	slli.d t0, t0, 3
	389	add.d t1, t1, t0
	390	#if CONFIG_PGTABLE_LEVELS > 3
	391	csrrd t0, LOONGARCH_CSR_BADV
	392	ld.d t1, t1, 0
	393	srli.d t0, t0, PUD_SHIFT
	394	andi t0, t0, (PTRS_PER_PUD - 1)
	395	slli.d t0, t0, 3
	396	add.d t1, t1, t0
	397	#endif
	398	#if CONFIG_PGTABLE_LEVELS > 2
	399	csrrd t0, LOONGARCH_CSR_BADV
	400	ld.d t1, t1, 0
	401	srli.d t0, t0, PMD_SHIFT
	402	andi t0, t0, (PTRS_PER_PMD - 1)
	403	slli.d t0, t0, 3
	404	add.d t1, t1, t0
	405	#endif
	406	ld.d ra, t1, 0
	407
	408	/*
	409	* For huge tlb entries, pmde doesn't contain an address but
	410	* instead contains the tlb pte. Check the PAGE_HUGE bit and
	411	* see if we need to jump to huge tlb processing.
	412	*/
	413	andi t0, ra, _PAGE_HUGE
d8e7f201	414	bne t0, zero, tlb_huge_update_modify
09cfefb7 HC	415
	416	csrrd t0, LOONGARCH_CSR_BADV
	417	srli.d t0, t0, (PAGE_SHIFT + PTE_ORDER)
	418	andi t0, t0, (PTRS_PER_PTE - 1)
	419	slli.d t0, t0, _PTE_T_LOG2
	420	add.d t1, ra, t0
	421
46859ac8 HC	422	#ifdef CONFIG_SMP
	423	smp_pgtable_change_modify:
	424	#endif
	425	#ifdef CONFIG_SMP
	426	ll.d t0, t1, 0
	427	#else
09cfefb7	428	ld.d t0, t1, 0
46859ac8	429	#endif
09cfefb7 HC	430	tlbsrch
	431
	432	srli.d ra, t0, _PAGE_WRITE_SHIFT
	433	andi ra, ra, 1
d8e7f201	434	beq ra, zero, nopage_tlb_modify
09cfefb7 HC	435
09cfefb7 HC	436	ori t0, t0, (_PAGE_VALID \| _PAGE_DIRTY \| _PAGE_MODIFIED)
46859ac8 HC	437	#ifdef CONFIG_SMP
46859ac8 HC	438	sc.d t0, t1, 0
d8e7f201	439	beq t0, zero, smp_pgtable_change_modify
46859ac8	440	#else
09cfefb7	441	st.d t0, t1, 0
46859ac8	442	#endif
09cfefb7 HC	443	ori t1, t1, 8
	444	xori t1, t1, 8
	445	ld.d t0, t1, 0
	446	ld.d t1, t1, 8
	447	csrwr t0, LOONGARCH_CSR_TLBELO0
	448	csrwr t1, LOONGARCH_CSR_TLBELO1
	449	tlbwr
	450	leave_modify:
	451	csrrd t0, EXCEPTION_KS0
	452	csrrd t1, EXCEPTION_KS1
	453	csrrd ra, EXCEPTION_KS2
	454	ertn
	455	#ifdef CONFIG_64BIT
	456	vmalloc_modify:
	457	la.abs t1, swapper_pg_dir
	458	b vmalloc_done_modify
	459	#endif
	460
	461	/*
	462	* This is the entry point when
	463	* build_tlbchange_handler_head spots a huge page.
	464	*/
	465	tlb_huge_update_modify:
46859ac8 HC	466	#ifdef CONFIG_SMP
	467	ll.d t0, t1, 0
	468	#else
09cfefb7	469	ld.d t0, t1, 0
46859ac8	470	#endif
09cfefb7 HC	471
	472	srli.d ra, t0, _PAGE_WRITE_SHIFT
	473	andi ra, ra, 1
d8e7f201	474	beq ra, zero, nopage_tlb_modify
09cfefb7 HC	475
	476	tlbsrch
	477	ori t0, t0, (_PAGE_VALID \| _PAGE_DIRTY \| _PAGE_MODIFIED)
	478
46859ac8 HC	479	#ifdef CONFIG_SMP
46859ac8 HC	480	sc.d t0, t1, 0
d8e7f201	481	beq t0, zero, tlb_huge_update_modify
46859ac8 HC	482	ld.d t0, t1, 0
46859ac8 HC	483	#else
09cfefb7	484	st.d t0, t1, 0
46859ac8	485	#endif
09cfefb7 HC	486	/*
	487	* A huge PTE describes an area the size of the
	488	* configured huge page size. This is twice the
	489	* of the large TLB entry size we intend to use.
	490	* A TLB entry half the size of the configured
	491	* huge page size is configured into entrylo0
	492	* and entrylo1 to cover the contiguous huge PTE
	493	* address space.
	494	*/
	495	/* Huge page: Move Global bit */
	496	xori t0, t0, _PAGE_HUGE
	497	lu12i.w t1, _PAGE_HGLOBAL >> 12
	498	and t1, t0, t1
	499	srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
	500	or t0, t0, t1
	501
	502	addi.d ra, t0, 0
	503	csrwr t0, LOONGARCH_CSR_TLBELO0
	504	addi.d t0, ra, 0
	505
	506	/* Convert to entrylo1 */
d8e7f201	507	addi.d t1, zero, 1
09cfefb7 HC	508	slli.d t1, t1, (HPAGE_SHIFT - 1)
	509	add.d t0, t0, t1
	510	csrwr t0, LOONGARCH_CSR_TLBELO1
	511
	512	/* Set huge page tlb entry size */
d8e7f201 WX	513	addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
d8e7f201 WX	514	addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
09cfefb7 HC	515	csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
	516
	517	tlbwr
	518
	519	/* Reset default page size */
d8e7f201 WX	520	addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16)
d8e7f201 WX	521	addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
09cfefb7 HC	522	csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
	523
	524	nopage_tlb_modify:
46859ac8	525	dbar 0
09cfefb7 HC	526	csrrd ra, EXCEPTION_KS2
09cfefb7 HC	527	la.abs t0, tlb_do_page_fault_1
07b48069	528	jr t0
09cfefb7 HC	529	SYM_FUNC_END(handle_tlb_modify)
	530
	531	SYM_FUNC_START(handle_tlb_refill)
	532	csrwr t0, LOONGARCH_CSR_TLBRSAVE
	533	csrrd t0, LOONGARCH_CSR_PGD
	534	lddir t0, t0, 3
	535	#if CONFIG_PGTABLE_LEVELS > 3
	536	lddir t0, t0, 2
	537	#endif
	538	#if CONFIG_PGTABLE_LEVELS > 2
	539	lddir t0, t0, 1
	540	#endif
	541	ldpte t0, 0
	542	ldpte t0, 1
	543	tlbfill
	544	csrrd t0, LOONGARCH_CSR_TLBRSAVE
	545	ertn
	546	SYM_FUNC_END(handle_tlb_refill)