[linux-block.git] / arch / arm64 / mm / hugetlbpage.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * arch/arm64/mm/hugetlbpage.c
 *
 * Copyright (C) 2013 Linaro Ltd.
 *
 * Based on arch/x86/mm/hugetlbpage.c.
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <asm/mman.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>

/*
 * HugeTLB Support Matrix
 *
 * ---------------------------------------------------
 * | Page Size | CONT PTE |  PMD  | CONT PMD |  PUD  |
 * ---------------------------------------------------
 * |     4K    |   64K    |   2M  |    32M   |   1G  |
 * |    16K    |    2M    |  32M  |     1G   |       |
 * |    64K    |    2M    | 512M  |    16G   |       |
 * ---------------------------------------------------
 */

/*
 * Reserve CMA areas for the largest supported gigantic
 * huge page when requested. Any other smaller gigantic
 * huge pages could still be served from those areas.
 */
#ifdef CONFIG_CMA
void __init arm64_hugetlb_cma_reserve(void)
{
	int order;

	if (pud_sect_supported())
		order = PUD_SHIFT - PAGE_SHIFT;
	else
		order = CONT_PMD_SHIFT - PAGE_SHIFT;

	/*
	 * HugeTLB CMA reservation is required for gigantic
	 * huge pages which could not be allocated via the
	 * page allocator. Just warn if there is any change
	 * breaking this assumption.
	 */
	WARN_ON(order <= MAX_ORDER);
	hugetlb_cma_reserve(order);
}
#endif /* CONFIG_CMA */

static bool __hugetlb_valid_size(unsigned long size)
{
	switch (size) {
#ifndef __PAGETABLE_PMD_FOLDED
	case PUD_SIZE:
		return pud_sect_supported();
#endif
	case CONT_PMD_SIZE:
	case PMD_SIZE:
	case CONT_PTE_SIZE:
		return true;
	}

	return false;
}

#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
bool arch_hugetlb_migration_supported(struct hstate *h)
{
	size_t pagesize = huge_page_size(h);

	if (!__hugetlb_valid_size(pagesize)) {
		pr_warn("%s: unrecognized huge page size 0x%lx\n",
			__func__, pagesize);
		return false;
	}
	return true;
}
#endif

int pmd_huge(pmd_t pmd)
{
	return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
}

int pud_huge(pud_t pud)
{
#ifndef __PAGETABLE_PMD_FOLDED
	return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
#else
	return 0;
#endif
}

/*
 * Select all bits except the pfn
 */
static inline pgprot_t pte_pgprot(pte_t pte)
{
	unsigned long pfn = pte_pfn(pte);

	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
}

static int find_num_contig(struct mm_struct *mm, unsigned long addr,
			   pte_t *ptep, size_t *pgsize)
{
	pgd_t *pgdp = pgd_offset(mm, addr);
	p4d_t *p4dp;
	pud_t *pudp;
	pmd_t *pmdp;

	*pgsize = PAGE_SIZE;
	p4dp = p4d_offset(pgdp, addr);
	pudp = pud_offset(p4dp, addr);
	pmdp = pmd_offset(pudp, addr);
	if ((pte_t *)pmdp == ptep) {
		*pgsize = PMD_SIZE;
		return CONT_PMDS;
	}
	return CONT_PTES;
}

static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
{
	int contig_ptes = 0;

	*pgsize = size;

	switch (size) {
#ifndef __PAGETABLE_PMD_FOLDED
	case PUD_SIZE:
		if (pud_sect_supported())
			contig_ptes = 1;
		break;
#endif
	case PMD_SIZE:
		contig_ptes = 1;
		break;
	case CONT_PMD_SIZE:
		*pgsize = PMD_SIZE;
		contig_ptes = CONT_PMDS;
		break;
	case CONT_PTE_SIZE:
		*pgsize = PAGE_SIZE;
		contig_ptes = CONT_PTES;
		break;
	}

	return contig_ptes;
}

pte_t huge_ptep_get(pte_t *ptep)
{
	int ncontig, i;
	size_t pgsize;
	pte_t orig_pte = ptep_get(ptep);

	if (!pte_present(orig_pte) || !pte_cont(orig_pte))
		return orig_pte;

	ncontig = num_contig_ptes(page_size(pte_page(orig_pte)), &pgsize);
	for (i = 0; i < ncontig; i++, ptep++) {
		pte_t pte = ptep_get(ptep);

		if (pte_dirty(pte))
			orig_pte = pte_mkdirty(orig_pte);

		if (pte_young(pte))
			orig_pte = pte_mkyoung(orig_pte);
	}
	return orig_pte;
}

/*
 * Changing some bits of contiguous entries requires us to follow a
 * Break-Before-Make approach, breaking the whole contiguous set
 * before we can change any entries. See ARM DDI 0487A.k_iss10775,
 * "Misprogramming of the Contiguous bit", page D4-1762.
 *
 * This helper performs the break step.
 */
static pte_t get_clear_contig(struct mm_struct *mm,
			     unsigned long addr,
			     pte_t *ptep,
			     unsigned long pgsize,
			     unsigned long ncontig)
{
	pte_t orig_pte = ptep_get(ptep);
	unsigned long i;

	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
		pte_t pte = ptep_get_and_clear(mm, addr, ptep);

		/*
		 * If HW_AFDBM is enabled, then the HW could turn on
		 * the dirty or accessed bit for any page in the set,
		 * so check them all.
		 */
		if (pte_dirty(pte))
			orig_pte = pte_mkdirty(orig_pte);

		if (pte_young(pte))
			orig_pte = pte_mkyoung(orig_pte);
	}
	return orig_pte;
}

/*
 * Changing some bits of contiguous entries requires us to follow a
 * Break-Before-Make approach, breaking the whole contiguous set
 * before we can change any entries. See ARM DDI 0487A.k_iss10775,
 * "Misprogramming of the Contiguous bit", page D4-1762.
 *
 * This helper performs the break step for use cases where the
 * original pte is not needed.
 */
static void clear_flush(struct mm_struct *mm,
			     unsigned long addr,
			     pte_t *ptep,
			     unsigned long pgsize,
			     unsigned long ncontig)
{
	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
	unsigned long i, saddr = addr;

	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
		pte_clear(mm, addr, ptep);

	flush_tlb_range(&vma, saddr, addr);
}

void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
			    pte_t *ptep, pte_t pte)
{
	size_t pgsize;
	int i;
	int ncontig;
	unsigned long pfn, dpfn;
	pgprot_t hugeprot;

	/*
	 * Code needs to be expanded to handle huge swap and migration
	 * entries. Needed for HUGETLB and MEMORY_FAILURE.
	 */
	WARN_ON(!pte_present(pte));

	if (!pte_cont(pte)) {
		set_pte_at(mm, addr, ptep, pte);
		return;
	}

	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
	pfn = pte_pfn(pte);
	dpfn = pgsize >> PAGE_SHIFT;
	hugeprot = pte_pgprot(pte);

	clear_flush(mm, addr, ptep, pgsize, ncontig);

	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
		set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
}

void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
			  pte_t *ptep, pte_t pte, unsigned long sz)
{
	int i, ncontig;
	size_t pgsize;

	ncontig = num_contig_ptes(sz, &pgsize);

	for (i = 0; i < ncontig; i++, ptep++)
		set_pte(ptep, pte);
}

pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
		      unsigned long addr, unsigned long sz)
{
	pgd_t *pgdp;
	p4d_t *p4dp;
	pud_t *pudp;
	pmd_t *pmdp;
	pte_t *ptep = NULL;

	pgdp = pgd_offset(mm, addr);
	p4dp = p4d_offset(pgdp, addr);
	pudp = pud_alloc(mm, p4dp, addr);
	if (!pudp)
		return NULL;

	if (sz == PUD_SIZE) {
		ptep = (pte_t *)pudp;
	} else if (sz == (CONT_PTE_SIZE)) {
		pmdp = pmd_alloc(mm, pudp, addr);
		if (!pmdp)
			return NULL;

		WARN_ON(addr & (sz - 1));
		/*
		 * Note that if this code were ever ported to the
		 * 32-bit arm platform then it will cause trouble in
		 * the case where CONFIG_HIGHPTE is set, since there
		 * will be no pte_unmap() to correspond with this
		 * pte_alloc_map().
		 */
		ptep = pte_alloc_map(mm, pmdp, addr);
	} else if (sz == PMD_SIZE) {
		if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
			ptep = huge_pmd_share(mm, vma, addr, pudp);
		else
			ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
	} else if (sz == (CONT_PMD_SIZE)) {
		pmdp = pmd_alloc(mm, pudp, addr);
		WARN_ON(addr & (sz - 1));
		return (pte_t *)pmdp;
	}

	return ptep;
}

pte_t *huge_pte_offset(struct mm_struct *mm,
		       unsigned long addr, unsigned long sz)
{
	pgd_t *pgdp;
	p4d_t *p4dp;
	pud_t *pudp, pud;
	pmd_t *pmdp, pmd;

	pgdp = pgd_offset(mm, addr);
	if (!pgd_present(READ_ONCE(*pgdp)))
		return NULL;

	p4dp = p4d_offset(pgdp, addr);
	if (!p4d_present(READ_ONCE(*p4dp)))
		return NULL;

	pudp = pud_offset(p4dp, addr);
	pud = READ_ONCE(*pudp);
	if (sz != PUD_SIZE && pud_none(pud))
		return NULL;
	/* hugepage or swap? */
	if (pud_huge(pud) || !pud_present(pud))
		return (pte_t *)pudp;
	/* table; check the next level */

	if (sz == CONT_PMD_SIZE)
		addr &= CONT_PMD_MASK;

	pmdp = pmd_offset(pudp, addr);
	pmd = READ_ONCE(*pmdp);
	if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
	    pmd_none(pmd))
		return NULL;
	if (pmd_huge(pmd) || !pmd_present(pmd))
		return (pte_t *)pmdp;

	if (sz == CONT_PTE_SIZE)
		return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK));

	return NULL;
}

pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
{
	size_t pagesize = 1UL << shift;

	entry = pte_mkhuge(entry);
	if (pagesize == CONT_PTE_SIZE) {
		entry = pte_mkcont(entry);
	} else if (pagesize == CONT_PMD_SIZE) {
		entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
	} else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
		pr_warn("%s: unrecognized huge page size 0x%lx\n",
			__func__, pagesize);
	}
	return entry;
}

void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
		    pte_t *ptep, unsigned long sz)
{
	int i, ncontig;
	size_t pgsize;

	ncontig = num_contig_ptes(sz, &pgsize);

	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
		pte_clear(mm, addr, ptep);
}

pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
			      unsigned long addr, pte_t *ptep)
{
	int ncontig;
	size_t pgsize;
	pte_t orig_pte = ptep_get(ptep);

	if (!pte_cont(orig_pte))
		return ptep_get_and_clear(mm, addr, ptep);

	ncontig = find_num_contig(mm, addr, ptep, &pgsize);

	return get_clear_contig(mm, addr, ptep, pgsize, ncontig);
}

/*
 * huge_ptep_set_access_flags will update access flags (dirty, accesssed)
 * and write permission.
 *
 * For a contiguous huge pte range we need to check whether or not write
 * permission has to change only on the first pte in the set. Then for
 * all the contiguous ptes we need to check whether or not there is a
 * discrepancy between dirty or young.
 */
static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
{
	int i;

	if (pte_write(pte) != pte_write(ptep_get(ptep)))
		return 1;

	for (i = 0; i < ncontig; i++) {
		pte_t orig_pte = ptep_get(ptep + i);

		if (pte_dirty(pte) != pte_dirty(orig_pte))
			return 1;

		if (pte_young(pte) != pte_young(orig_pte))
			return 1;
	}

	return 0;
}

int huge_ptep_set_access_flags(struct vm_area_struct *vma,
			       unsigned long addr, pte_t *ptep,
			       pte_t pte, int dirty)
{
	int ncontig, i;
	size_t pgsize = 0;
	unsigned long pfn = pte_pfn(pte), dpfn;
	pgprot_t hugeprot;
	pte_t orig_pte;

	if (!pte_cont(pte))
		return ptep_set_access_flags(vma, addr, ptep, pte, dirty);

	ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
	dpfn = pgsize >> PAGE_SHIFT;

	if (!__cont_access_flags_changed(ptep, pte, ncontig))
		return 0;

	orig_pte = get_clear_contig(vma->vm_mm, addr, ptep, pgsize, ncontig);

	/* Make sure we don't lose the dirty or young state */
	if (pte_dirty(orig_pte))
		pte = pte_mkdirty(pte);

	if (pte_young(orig_pte))
		pte = pte_mkyoung(pte);

	hugeprot = pte_pgprot(pte);
	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
		set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));

	return 1;
}

void huge_ptep_set_wrprotect(struct mm_struct *mm,
			     unsigned long addr, pte_t *ptep)
{
	unsigned long pfn, dpfn;
	pgprot_t hugeprot;
	int ncontig, i;
	size_t pgsize;
	pte_t pte;

	if (!pte_cont(READ_ONCE(*ptep))) {
		ptep_set_wrprotect(mm, addr, ptep);
		return;
	}

	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
	dpfn = pgsize >> PAGE_SHIFT;

	pte = get_clear_contig(mm, addr, ptep, pgsize, ncontig);
	pte = pte_wrprotect(pte);

	hugeprot = pte_pgprot(pte);
	pfn = pte_pfn(pte);

	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
		set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
}

pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
			    unsigned long addr, pte_t *ptep)
{
	size_t pgsize;
	int ncontig;
	pte_t orig_pte;

	if (!pte_cont(READ_ONCE(*ptep)))
		return ptep_clear_flush(vma, addr, ptep);

	ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
	orig_pte = get_clear_contig(vma->vm_mm, addr, ptep, pgsize, ncontig);
	flush_tlb_range(vma, addr, addr + pgsize * ncontig);
	return orig_pte;
}

static int __init hugetlbpage_init(void)
{
	if (pud_sect_supported())
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);

	hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT);
	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT);

	return 0;
}
arch_initcall(hugetlbpage_init);

bool __init arch_hugetlb_valid_size(unsigned long size)
{
	return __hugetlb_valid_size(size);
}
Commit	Line	Data
1802d0be	1	// SPDX-License-Identifier: GPL-2.0-only
084bd298 SC	2	/*
	3	* arch/arm64/mm/hugetlbpage.c
	4	*
	5	* Copyright (C) 2013 Linaro Ltd.
	6	*
	7	* Based on arch/x86/mm/hugetlbpage.c.
084bd298 SC	8	*/
	9
	10	#include <linux/init.h>
	11	#include <linux/fs.h>
	12	#include <linux/mm.h>
	13	#include <linux/hugetlb.h>
	14	#include <linux/pagemap.h>
	15	#include <linux/err.h>
	16	#include <linux/sysctl.h>
	17	#include <asm/mman.h>
	18	#include <asm/tlb.h>
	19	#include <asm/tlbflush.h>
084bd298	20
abb7962a AK	21	/*
	22	* HugeTLB Support Matrix
	23	*
	24	* ---------------------------------------------------
	25	* \| Page Size \| CONT PTE \| PMD \| CONT PMD \| PUD \|
	26	* ---------------------------------------------------
	27	* \| 4K \| 64K \| 2M \| 32M \| 1G \|
	28	* \| 16K \| 2M \| 32M \| 1G \| \|
	29	* \| 64K \| 2M \| 512M \| 16G \| \|
	30	* ---------------------------------------------------
	31	*/
	32
	33	/*
	34	* Reserve CMA areas for the largest supported gigantic
	35	* huge page when requested. Any other smaller gigantic
	36	* huge pages could still be served from those areas.
	37	*/
	38	#ifdef CONFIG_CMA
	39	void __init arm64_hugetlb_cma_reserve(void)
	40	{
	41	int order;
	42
f8b46c4b AK	43	if (pud_sect_supported())
	44	order = PUD_SHIFT - PAGE_SHIFT;
	45	else
e6359798 WD	46	order = CONT_PMD_SHIFT - PAGE_SHIFT;
e6359798 WD	47
abb7962a AK	48	/*
	49	* HugeTLB CMA reservation is required for gigantic
	50	* huge pages which could not be allocated via the
	51	* page allocator. Just warn if there is any change
	52	* breaking this assumption.
	53	*/
	54	WARN_ON(order <= MAX_ORDER);
	55	hugetlb_cma_reserve(order);
	56	}
	57	#endif /* CONFIG_CMA */
	58
a8a733b2	59	static bool __hugetlb_valid_size(unsigned long size)
5480280d	60	{
a8a733b2	61	switch (size) {
f8b46c4b	62	#ifndef __PAGETABLE_PMD_FOLDED
5480280d	63	case PUD_SIZE:
f8b46c4b	64	return pud_sect_supported();
5480280d	65	#endif
5480280d	66	case CONT_PMD_SIZE:
a8a733b2	67	case PMD_SIZE:
5480280d AK	68	case CONT_PTE_SIZE:
	69	return true;
	70	}
a8a733b2	71
5480280d AK	72	return false;
5480280d AK	73	}
a8a733b2 AK	74
	75	#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
	76	bool arch_hugetlb_migration_supported(struct hstate *h)
	77	{
	78	size_t pagesize = huge_page_size(h);
	79
	80	if (!__hugetlb_valid_size(pagesize)) {
	81	pr_warn("%s: unrecognized huge page size 0x%lx\n",
	82	__func__, pagesize);
	83	return false;
	84	}
	85	return true;
	86	}
5480280d AK	87	#endif
5480280d AK	88
084bd298 SC	89	int pmd_huge(pmd_t pmd)
084bd298 SC	90	{
fd28f5d4	91	return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
084bd298 SC	92	}
	93
	94	int pud_huge(pud_t pud)
	95	{
4797ec2d	96	#ifndef __PAGETABLE_PMD_FOLDED
fd28f5d4	97	return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
4797ec2d MS	98	#else
	99	return 0;
	100	#endif
084bd298 SC	101	}
084bd298 SC	102
b5b0be86 SC	103	/*
	104	* Select all bits except the pfn
	105	*/
	106	static inline pgprot_t pte_pgprot(pte_t pte)
	107	{
	108	unsigned long pfn = pte_pfn(pte);
	109
	110	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
	111	}
	112
66b3923a	113	static int find_num_contig(struct mm_struct *mm, unsigned long addr,
bb9dd3df	114	pte_t ptep, size_t pgsize)
66b3923a	115	{
20a004e7	116	pgd_t *pgdp = pgd_offset(mm, addr);
e9f63768	117	p4d_t *p4dp;
20a004e7 WD	118	pud_t *pudp;
20a004e7 WD	119	pmd_t *pmdp;
66b3923a DW	120
66b3923a DW	121	*pgsize = PAGE_SIZE;
e9f63768 MR	122	p4dp = p4d_offset(pgdp, addr);
e9f63768 MR	123	pudp = pud_offset(p4dp, addr);
20a004e7 WD	124	pmdp = pmd_offset(pudp, addr);
20a004e7 WD	125	if ((pte_t *)pmdp == ptep) {
66b3923a DW	126	*pgsize = PMD_SIZE;
	127	return CONT_PMDS;
	128	}
	129	return CONT_PTES;
	130	}
	131
c3e4ed5c PA	132	static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
	133	{
	134	int contig_ptes = 0;
	135
	136	*pgsize = size;
	137
	138	switch (size) {
f8b46c4b	139	#ifndef __PAGETABLE_PMD_FOLDED
c3e4ed5c	140	case PUD_SIZE:
f8b46c4b AK	141	if (pud_sect_supported())
	142	contig_ptes = 1;
	143	break;
c3e4ed5c PA	144	#endif
	145	case PMD_SIZE:
	146	contig_ptes = 1;
	147	break;
	148	case CONT_PMD_SIZE:
	149	*pgsize = PMD_SIZE;
	150	contig_ptes = CONT_PMDS;
	151	break;
	152	case CONT_PTE_SIZE:
	153	*pgsize = PAGE_SIZE;
	154	contig_ptes = CONT_PTES;
	155	break;
	156	}
	157
	158	return contig_ptes;
	159	}
	160
bc5dfb4f BW	161	pte_t huge_ptep_get(pte_t *ptep)
	162	{
	163	int ncontig, i;
	164	size_t pgsize;
	165	pte_t orig_pte = ptep_get(ptep);
	166
	167	if (!pte_present(orig_pte) \|\| !pte_cont(orig_pte))
	168	return orig_pte;
	169
	170	ncontig = num_contig_ptes(page_size(pte_page(orig_pte)), &pgsize);
	171	for (i = 0; i < ncontig; i++, ptep++) {
	172	pte_t pte = ptep_get(ptep);
	173
	174	if (pte_dirty(pte))
	175	orig_pte = pte_mkdirty(orig_pte);
	176
	177	if (pte_young(pte))
	178	orig_pte = pte_mkyoung(orig_pte);
	179	}
	180	return orig_pte;
	181	}
	182
d8bdcff2 SC	183	/*
	184	* Changing some bits of contiguous entries requires us to follow a
	185	* Break-Before-Make approach, breaking the whole contiguous set
	186	* before we can change any entries. See ARM DDI 0487A.k_iss10775,
	187	* "Misprogramming of the Contiguous bit", page D4-1762.
	188	*
	189	* This helper performs the break step.
	190	*/
fb396bb4	191	static pte_t get_clear_contig(struct mm_struct *mm,
d8bdcff2 SC	192	unsigned long addr,
	193	pte_t *ptep,
	194	unsigned long pgsize,
	195	unsigned long ncontig)
	196	{
f0d9d79e	197	pte_t orig_pte = ptep_get(ptep);
fb396bb4	198	unsigned long i;
d8bdcff2 SC	199
	200	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
	201	pte_t pte = ptep_get_and_clear(mm, addr, ptep);
	202
	203	/*
	204	* If HW_AFDBM is enabled, then the HW could turn on
469ed9d8 SC	205	* the dirty or accessed bit for any page in the set,
469ed9d8 SC	206	* so check them all.
d8bdcff2 SC	207	*/
	208	if (pte_dirty(pte))
	209	orig_pte = pte_mkdirty(orig_pte);
469ed9d8 SC	210
	211	if (pte_young(pte))
	212	orig_pte = pte_mkyoung(orig_pte);
d8bdcff2	213	}
d8bdcff2 SC	214	return orig_pte;
	215	}
	216
	217	/*
	218	* Changing some bits of contiguous entries requires us to follow a
	219	* Break-Before-Make approach, breaking the whole contiguous set
	220	* before we can change any entries. See ARM DDI 0487A.k_iss10775,
	221	* "Misprogramming of the Contiguous bit", page D4-1762.
	222	*
	223	* This helper performs the break step for use cases where the
	224	* original pte is not needed.
	225	*/
	226	static void clear_flush(struct mm_struct *mm,
	227	unsigned long addr,
	228	pte_t *ptep,
	229	unsigned long pgsize,
	230	unsigned long ncontig)
	231	{
8b11ec1b	232	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
d8bdcff2 SC	233	unsigned long i, saddr = addr;
	234
	235	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
	236	pte_clear(mm, addr, ptep);
	237
	238	flush_tlb_range(&vma, saddr, addr);
	239	}
	240
66b3923a DW	241	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
	242	pte_t *ptep, pte_t pte)
	243	{
	244	size_t pgsize;
	245	int i;
bb9dd3df	246	int ncontig;
29a7287d	247	unsigned long pfn, dpfn;
66b3923a DW	248	pgprot_t hugeprot;
66b3923a DW	249
d3ea7952 SC	250	/*
	251	* Code needs to be expanded to handle huge swap and migration
	252	* entries. Needed for HUGETLB and MEMORY_FAILURE.
	253	*/
	254	WARN_ON(!pte_present(pte));
	255
bb9dd3df	256	if (!pte_cont(pte)) {
66b3923a DW	257	set_pte_at(mm, addr, ptep, pte);
	258	return;
	259	}
	260
bb9dd3df	261	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
66b3923a	262	pfn = pte_pfn(pte);
29a7287d	263	dpfn = pgsize >> PAGE_SHIFT;
b5b0be86	264	hugeprot = pte_pgprot(pte);
29a7287d	265
d8bdcff2 SC	266	clear_flush(mm, addr, ptep, pgsize, ncontig);
d8bdcff2 SC	267
20a004e7	268	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
66b3923a	269	set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
66b3923a DW	270	}
66b3923a DW	271
a8d623ee PA	272	void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr,
	273	pte_t *ptep, pte_t pte, unsigned long sz)
	274	{
	275	int i, ncontig;
	276	size_t pgsize;
	277
	278	ncontig = num_contig_ptes(sz, &pgsize);
	279
	280	for (i = 0; i < ncontig; i++, ptep++)
	281	set_pte(ptep, pte);
	282	}
	283
aec44e0f	284	pte_t huge_pte_alloc(struct mm_struct mm, struct vm_area_struct *vma,
66b3923a DW	285	unsigned long addr, unsigned long sz)
66b3923a DW	286	{
20a004e7	287	pgd_t *pgdp;
e9f63768	288	p4d_t *p4dp;
20a004e7 WD	289	pud_t *pudp;
	290	pmd_t *pmdp;
	291	pte_t *ptep = NULL;
	292
	293	pgdp = pgd_offset(mm, addr);
e9f63768 MR	294	p4dp = p4d_offset(pgdp, addr);
e9f63768 MR	295	pudp = pud_alloc(mm, p4dp, addr);
20a004e7	296	if (!pudp)
66b3923a DW	297	return NULL;
	298
	299	if (sz == PUD_SIZE) {
20a004e7	300	ptep = (pte_t *)pudp;
441a6278	301	} else if (sz == (CONT_PTE_SIZE)) {
20a004e7	302	pmdp = pmd_alloc(mm, pudp, addr);
027d0c71 MR	303	if (!pmdp)
027d0c71 MR	304	return NULL;
66b3923a DW	305
	306	WARN_ON(addr & (sz - 1));
	307	/*
	308	* Note that if this code were ever ported to the
	309	* 32-bit arm platform then it will cause trouble in
	310	* the case where CONFIG_HIGHPTE is set, since there
	311	* will be no pte_unmap() to correspond with this
	312	* pte_alloc_map().
	313	*/
20a004e7	314	ptep = pte_alloc_map(mm, pmdp, addr);
66b3923a	315	} else if (sz == PMD_SIZE) {
c1991e07	316	if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
aec44e0f	317	ptep = huge_pmd_share(mm, vma, addr, pudp);
66b3923a	318	else
20a004e7	319	ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
441a6278	320	} else if (sz == (CONT_PMD_SIZE)) {
20a004e7	321	pmdp = pmd_alloc(mm, pudp, addr);
66b3923a	322	WARN_ON(addr & (sz - 1));
20a004e7	323	return (pte_t *)pmdp;
66b3923a DW	324	}
66b3923a DW	325
20a004e7	326	return ptep;
66b3923a DW	327	}
66b3923a DW	328
7868a208 PA	329	pte_t huge_pte_offset(struct mm_struct mm,
7868a208 PA	330	unsigned long addr, unsigned long sz)
66b3923a	331	{
20a004e7	332	pgd_t *pgdp;
e9f63768	333	p4d_t *p4dp;
20a004e7 WD	334	pud_t *pudp, pud;
20a004e7 WD	335	pmd_t *pmdp, pmd;
66b3923a	336
20a004e7 WD	337	pgdp = pgd_offset(mm, addr);
20a004e7 WD	338	if (!pgd_present(READ_ONCE(*pgdp)))
66b3923a	339	return NULL;
f02ab08a	340
e9f63768 MR	341	p4dp = p4d_offset(pgdp, addr);
	342	if (!p4d_present(READ_ONCE(*p4dp)))
	343	return NULL;
	344
	345	pudp = pud_offset(p4dp, addr);
20a004e7 WD	346	pud = READ_ONCE(*pudp);
20a004e7 WD	347	if (sz != PUD_SIZE && pud_none(pud))
66b3923a	348	return NULL;
30f3ac00	349	/* hugepage or swap? */
20a004e7 WD	350	if (pud_huge(pud) \|\| !pud_present(pud))
20a004e7 WD	351	return (pte_t *)pudp;
f02ab08a PA	352	/* table; check the next level */
f02ab08a PA	353
30f3ac00 PA	354	if (sz == CONT_PMD_SIZE)
	355	addr &= CONT_PMD_MASK;
	356
20a004e7 WD	357	pmdp = pmd_offset(pudp, addr);
20a004e7 WD	358	pmd = READ_ONCE(*pmdp);
30f3ac00	359	if (!(sz == PMD_SIZE \|\| sz == CONT_PMD_SIZE) &&
20a004e7	360	pmd_none(pmd))
66b3923a	361	return NULL;
20a004e7 WD	362	if (pmd_huge(pmd) \|\| !pmd_present(pmd))
20a004e7 WD	363	return (pte_t *)pmdp;
f02ab08a	364
20a004e7 WD	365	if (sz == CONT_PTE_SIZE)
20a004e7 WD	366	return pte_offset_kernel(pmdp, (addr & CONT_PTE_MASK));
30f3ac00	367
66b3923a DW	368	return NULL;
	369	}
	370
79c1c594	371	pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
66b3923a	372	{
79c1c594	373	size_t pagesize = 1UL << shift;
66b3923a	374
16785bd7	375	entry = pte_mkhuge(entry);
66b3923a DW	376	if (pagesize == CONT_PTE_SIZE) {
	377	entry = pte_mkcont(entry);
	378	} else if (pagesize == CONT_PMD_SIZE) {
	379	entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
	380	} else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
	381	pr_warn("%s: unrecognized huge page size 0x%lx\n",
	382	__func__, pagesize);
	383	}
	384	return entry;
	385	}
	386
c3e4ed5c PA	387	void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
	388	pte_t *ptep, unsigned long sz)
	389	{
	390	int i, ncontig;
	391	size_t pgsize;
	392
	393	ncontig = num_contig_ptes(sz, &pgsize);
	394
	395	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
	396	pte_clear(mm, addr, ptep);
	397	}
	398
66b3923a DW	399	pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
	400	unsigned long addr, pte_t *ptep)
	401	{
d8bdcff2	402	int ncontig;
29a7287d	403	size_t pgsize;
f0d9d79e	404	pte_t orig_pte = ptep_get(ptep);
29a7287d SC	405
29a7287d SC	406	if (!pte_cont(orig_pte))
66b3923a	407	return ptep_get_and_clear(mm, addr, ptep);
29a7287d SC	408
29a7287d SC	409	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
29a7287d	410
fb396bb4	411	return get_clear_contig(mm, addr, ptep, pgsize, ncontig);
66b3923a DW	412	}
66b3923a DW	413
031e6e6b SC	414	/*
	415	* huge_ptep_set_access_flags will update access flags (dirty, accesssed)
	416	* and write permission.
	417	*
	418	* For a contiguous huge pte range we need to check whether or not write
	419	* permission has to change only on the first pte in the set. Then for
	420	* all the contiguous ptes we need to check whether or not there is a
	421	* discrepancy between dirty or young.
	422	*/
	423	static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
	424	{
	425	int i;
	426
f0d9d79e	427	if (pte_write(pte) != pte_write(ptep_get(ptep)))
031e6e6b SC	428	return 1;
	429
	430	for (i = 0; i < ncontig; i++) {
f0d9d79e	431	pte_t orig_pte = ptep_get(ptep + i);
031e6e6b SC	432
	433	if (pte_dirty(pte) != pte_dirty(orig_pte))
	434	return 1;
	435
	436	if (pte_young(pte) != pte_young(orig_pte))
	437	return 1;
	438	}
	439
	440	return 0;
	441	}
	442
66b3923a DW	443	int huge_ptep_set_access_flags(struct vm_area_struct *vma,
	444	unsigned long addr, pte_t *ptep,
	445	pte_t pte, int dirty)
	446	{
031e6e6b	447	int ncontig, i;
29a7287d SC	448	size_t pgsize = 0;
	449	unsigned long pfn = pte_pfn(pte), dpfn;
	450	pgprot_t hugeprot;
d8bdcff2	451	pte_t orig_pte;
29a7287d SC	452
29a7287d SC	453	if (!pte_cont(pte))
66b3923a	454	return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
29a7287d SC	455
	456	ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
	457	dpfn = pgsize >> PAGE_SHIFT;
29a7287d	458
031e6e6b SC	459	if (!__cont_access_flags_changed(ptep, pte, ncontig))
	460	return 0;
	461
fb396bb4	462	orig_pte = get_clear_contig(vma->vm_mm, addr, ptep, pgsize, ncontig);
d8bdcff2	463
469ed9d8	464	/* Make sure we don't lose the dirty or young state */
d8bdcff2 SC	465	if (pte_dirty(orig_pte))
	466	pte = pte_mkdirty(pte);
	467
469ed9d8 SC	468	if (pte_young(orig_pte))
	469	pte = pte_mkyoung(pte);
	470
d8bdcff2 SC	471	hugeprot = pte_pgprot(pte);
	472	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
	473	set_pte_at(vma->vm_mm, addr, ptep, pfn_pte(pfn, hugeprot));
29a7287d	474
031e6e6b	475	return 1;
66b3923a DW	476	}
	477
	478	void huge_ptep_set_wrprotect(struct mm_struct *mm,
	479	unsigned long addr, pte_t *ptep)
	480	{
d8bdcff2 SC	481	unsigned long pfn, dpfn;
d8bdcff2 SC	482	pgprot_t hugeprot;
29a7287d SC	483	int ncontig, i;
29a7287d SC	484	size_t pgsize;
d8bdcff2	485	pte_t pte;
66b3923a	486
20a004e7	487	if (!pte_cont(READ_ONCE(*ptep))) {
66b3923a	488	ptep_set_wrprotect(mm, addr, ptep);
29a7287d	489	return;
66b3923a	490	}
29a7287d SC	491
29a7287d SC	492	ncontig = find_num_contig(mm, addr, ptep, &pgsize);
d8bdcff2 SC	493	dpfn = pgsize >> PAGE_SHIFT;
d8bdcff2 SC	494
fb396bb4	495	pte = get_clear_contig(mm, addr, ptep, pgsize, ncontig);
d8bdcff2 SC	496	pte = pte_wrprotect(pte);
	497
	498	hugeprot = pte_pgprot(pte);
	499	pfn = pte_pfn(pte);
	500
	501	for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
	502	set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
66b3923a DW	503	}
66b3923a DW	504
ae075629 BW	505	pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
ae075629 BW	506	unsigned long addr, pte_t *ptep)
66b3923a	507	{
29a7287d	508	size_t pgsize;
d8bdcff2	509	int ncontig;
e68b823a	510	pte_t orig_pte;
29a7287d	511
ae075629 BW	512	if (!pte_cont(READ_ONCE(*ptep)))
ae075629 BW	513	return ptep_clear_flush(vma, addr, ptep);
29a7287d SC	514
29a7287d SC	515	ncontig = find_num_contig(vma->vm_mm, addr, ptep, &pgsize);
e68b823a BW	516	orig_pte = get_clear_contig(vma->vm_mm, addr, ptep, pgsize, ncontig);
	517	flush_tlb_range(vma, addr, addr + pgsize * ncontig);
	518	return orig_pte;
66b3923a DW	519	}
66b3923a DW	520
a21b0b78 AP	521	static int __init hugetlbpage_init(void)
a21b0b78 AP	522	{
f8b46c4b AK	523	if (pud_sect_supported())
	524	hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	525
a1634a54	526	hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT);
38237830	527	hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
a1634a54	528	hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT);
a21b0b78 AP	529
	530	return 0;
	531	}
	532	arch_initcall(hugetlbpage_init);
	533
ae94da89	534	bool __init arch_hugetlb_valid_size(unsigned long size)
084bd298	535	{
a8a733b2	536	return __hugetlb_valid_size(size);
ae94da89	537	}