[linux-block.git] / include / asm-generic / pgalloc.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_GENERIC_PGALLOC_H
#define __ASM_GENERIC_PGALLOC_H

#ifdef CONFIG_MMU

#define GFP_PGTABLE_KERNEL	(GFP_KERNEL | __GFP_ZERO)
#define GFP_PGTABLE_USER	(GFP_PGTABLE_KERNEL | __GFP_ACCOUNT)

/**
 * __pte_alloc_one_kernel - allocate a page for PTE-level kernel page table
 * @mm: the mm_struct of the current context
 *
 * This function is intended for architectures that need
 * anything beyond simple page allocation.
 *
 * Return: pointer to the allocated memory or %NULL on error
 */
static inline pte_t *__pte_alloc_one_kernel(struct mm_struct *mm)
{
	return (pte_t *)__get_free_page(GFP_PGTABLE_KERNEL);
}

#ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL
/**
 * pte_alloc_one_kernel - allocate a page for PTE-level kernel page table
 * @mm: the mm_struct of the current context
 *
 * Return: pointer to the allocated memory or %NULL on error
 */
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
{
	return __pte_alloc_one_kernel(mm);
}
#endif

/**
 * pte_free_kernel - free PTE-level kernel page table page
 * @mm: the mm_struct of the current context
 * @pte: pointer to the memory containing the page table
 */
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
	free_page((unsigned long)pte);
}

/**
 * __pte_alloc_one - allocate a page for PTE-level user page table
 * @mm: the mm_struct of the current context
 * @gfp: GFP flags to use for the allocation
 *
 * Allocates a page and runs the pgtable_pte_page_ctor().
 *
 * This function is intended for architectures that need
 * anything beyond simple page allocation or must have custom GFP flags.
 *
 * Return: `struct page` initialized as page table or %NULL on error
 */
static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp)
{
	struct page *pte;

	pte = alloc_page(gfp);
	if (!pte)
		return NULL;
	if (!pgtable_pte_page_ctor(pte)) {
		__free_page(pte);
		return NULL;
	}

	return pte;
}

#ifndef __HAVE_ARCH_PTE_ALLOC_ONE
/**
 * pte_alloc_one - allocate a page for PTE-level user page table
 * @mm: the mm_struct of the current context
 *
 * Allocates a page and runs the pgtable_pte_page_ctor().
 *
 * Return: `struct page` initialized as page table or %NULL on error
 */
static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
{
	return __pte_alloc_one(mm, GFP_PGTABLE_USER);
}
#endif

/*
 * Should really implement gc for free page table pages. This could be
 * done with a reference count in struct page.
 */

/**
 * pte_free - free PTE-level user page table page
 * @mm: the mm_struct of the current context
 * @pte_page: the `struct page` representing the page table
 */
static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
{
	pgtable_pte_page_dtor(pte_page);
	__free_page(pte_page);
}


#if CONFIG_PGTABLE_LEVELS > 2

#ifndef __HAVE_ARCH_PMD_ALLOC_ONE
/**
 * pmd_alloc_one - allocate a page for PMD-level page table
 * @mm: the mm_struct of the current context
 *
 * Allocates a page and runs the pgtable_pmd_page_ctor().
 * Allocations use %GFP_PGTABLE_USER in user context and
 * %GFP_PGTABLE_KERNEL in kernel context.
 *
 * Return: pointer to the allocated memory or %NULL on error
 */
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	struct page *page;
	gfp_t gfp = GFP_PGTABLE_USER;

	if (mm == &init_mm)
		gfp = GFP_PGTABLE_KERNEL;
	page = alloc_page(gfp);
	if (!page)
		return NULL;
	if (!pgtable_pmd_page_ctor(page)) {
		__free_page(page);
		return NULL;
	}
	return (pmd_t *)page_address(page);
}
#endif

#ifndef __HAVE_ARCH_PMD_FREE
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
	BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
	pgtable_pmd_page_dtor(virt_to_page(pmd));
	free_page((unsigned long)pmd);
}
#endif

#endif /* CONFIG_PGTABLE_LEVELS > 2 */

#if CONFIG_PGTABLE_LEVELS > 3

static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	gfp_t gfp = GFP_PGTABLE_USER;

	if (mm == &init_mm)
		gfp = GFP_PGTABLE_KERNEL;
	return (pud_t *)get_zeroed_page(gfp);
}

#ifndef __HAVE_ARCH_PUD_ALLOC_ONE
/**
 * pud_alloc_one - allocate a page for PUD-level page table
 * @mm: the mm_struct of the current context
 *
 * Allocates a page using %GFP_PGTABLE_USER for user context and
 * %GFP_PGTABLE_KERNEL for kernel context.
 *
 * Return: pointer to the allocated memory or %NULL on error
 */
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
	return __pud_alloc_one(mm, addr);
}
#endif

static inline void __pud_free(struct mm_struct *mm, pud_t *pud)
{
	BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
	free_page((unsigned long)pud);
}

#ifndef __HAVE_ARCH_PUD_FREE
static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
	__pud_free(mm, pud);
}
#endif

#endif /* CONFIG_PGTABLE_LEVELS > 3 */

#ifndef __HAVE_ARCH_PGD_FREE
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
	free_page((unsigned long)pgd);
}
#endif

#endif /* CONFIG_MMU */

#endif /* __ASM_GENERIC_PGALLOC_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
5c01b46b AB	2	#ifndef __ASM_GENERIC_PGALLOC_H
5c01b46b AB	3	#define __ASM_GENERIC_PGALLOC_H
5fba4af4	4
5c01b46b	5	#ifdef CONFIG_MMU
5fba4af4 MR	6
	7	#define GFP_PGTABLE_KERNEL (GFP_KERNEL \| __GFP_ZERO)
	8	#define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL \| __GFP_ACCOUNT)
	9
	10	/**
	11	* __pte_alloc_one_kernel - allocate a page for PTE-level kernel page table
	12	* @mm: the mm_struct of the current context
	13	*
	14	* This function is intended for architectures that need
	15	* anything beyond simple page allocation.
	16	*
	17	* Return: pointer to the allocated memory or %NULL on error
	18	*/
	19	static inline pte_t __pte_alloc_one_kernel(struct mm_struct mm)
	20	{
	21	return (pte_t *)__get_free_page(GFP_PGTABLE_KERNEL);
	22	}
	23
	24	#ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL
	25	/**
	26	* pte_alloc_one_kernel - allocate a page for PTE-level kernel page table
	27	* @mm: the mm_struct of the current context
	28	*
	29	* Return: pointer to the allocated memory or %NULL on error
	30	*/
	31	static inline pte_t pte_alloc_one_kernel(struct mm_struct mm)
	32	{
	33	return __pte_alloc_one_kernel(mm);
	34	}
	35	#endif
	36
	37	/**
	38	* pte_free_kernel - free PTE-level kernel page table page
	39	* @mm: the mm_struct of the current context
	40	* @pte: pointer to the memory containing the page table
	41	*/
	42	static inline void pte_free_kernel(struct mm_struct mm, pte_t pte)
	43	{
	44	free_page((unsigned long)pte);
	45	}
	46
	47	/**
	48	* __pte_alloc_one - allocate a page for PTE-level user page table
	49	* @mm: the mm_struct of the current context
	50	* @gfp: GFP flags to use for the allocation
	51	*
b4ed71f5	52	* Allocates a page and runs the pgtable_pte_page_ctor().
5fba4af4 MR	53	*
	54	* This function is intended for architectures that need
	55	* anything beyond simple page allocation or must have custom GFP flags.
	56	*
	57	* Return: `struct page` initialized as page table or %NULL on error
	58	*/
	59	static inline pgtable_t __pte_alloc_one(struct mm_struct *mm, gfp_t gfp)
	60	{
	61	struct page *pte;
	62
	63	pte = alloc_page(gfp);
	64	if (!pte)
	65	return NULL;
b4ed71f5	66	if (!pgtable_pte_page_ctor(pte)) {
5fba4af4 MR	67	__free_page(pte);
	68	return NULL;
	69	}
	70
	71	return pte;
	72	}
	73
	74	#ifndef __HAVE_ARCH_PTE_ALLOC_ONE
	75	/**
	76	* pte_alloc_one - allocate a page for PTE-level user page table
	77	* @mm: the mm_struct of the current context
	78	*
b4ed71f5	79	* Allocates a page and runs the pgtable_pte_page_ctor().
5fba4af4 MR	80	*
	81	* Return: `struct page` initialized as page table or %NULL on error
	82	*/
	83	static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
	84	{
	85	return __pte_alloc_one(mm, GFP_PGTABLE_USER);
	86	}
5c01b46b AB	87	#endif
5c01b46b AB	88
5fba4af4 MR	89	/*
	90	* Should really implement gc for free page table pages. This could be
	91	* done with a reference count in struct page.
	92	*/
	93
	94	/**
	95	* pte_free - free PTE-level user page table page
	96	* @mm: the mm_struct of the current context
	97	* @pte_page: the `struct page` representing the page table
	98	*/
	99	static inline void pte_free(struct mm_struct mm, struct page pte_page)
	100	{
b4ed71f5	101	pgtable_pte_page_dtor(pte_page);
5fba4af4 MR	102	__free_page(pte_page);
	103	}
	104
1355c31e MR	105
	106	#if CONFIG_PGTABLE_LEVELS > 2
	107
	108	#ifndef __HAVE_ARCH_PMD_ALLOC_ONE
	109	/**
	110	* pmd_alloc_one - allocate a page for PMD-level page table
	111	* @mm: the mm_struct of the current context
	112	*
	113	* Allocates a page and runs the pgtable_pmd_page_ctor().
	114	* Allocations use %GFP_PGTABLE_USER in user context and
	115	* %GFP_PGTABLE_KERNEL in kernel context.
	116	*
	117	* Return: pointer to the allocated memory or %NULL on error
	118	*/
	119	static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long addr)
	120	{
	121	struct page *page;
	122	gfp_t gfp = GFP_PGTABLE_USER;
	123
	124	if (mm == &init_mm)
	125	gfp = GFP_PGTABLE_KERNEL;
dcc1be11	126	page = alloc_page(gfp);
1355c31e MR	127	if (!page)
	128	return NULL;
	129	if (!pgtable_pmd_page_ctor(page)) {
dcc1be11	130	__free_page(page);
1355c31e MR	131	return NULL;
	132	}
	133	return (pmd_t *)page_address(page);
	134	}
	135	#endif
	136
	137	#ifndef __HAVE_ARCH_PMD_FREE
	138	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)
	139	{
	140	BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
	141	pgtable_pmd_page_dtor(virt_to_page(pmd));
	142	free_page((unsigned long)pmd);
	143	}
	144	#endif
	145
	146	#endif /* CONFIG_PGTABLE_LEVELS > 2 */
	147
d9e8b929 MR	148	#if CONFIG_PGTABLE_LEVELS > 3
d9e8b929 MR	149
60639f74 AG	150	static inline pud_t __pud_alloc_one(struct mm_struct mm, unsigned long addr)
	151	{
	152	gfp_t gfp = GFP_PGTABLE_USER;
	153
	154	if (mm == &init_mm)
	155	gfp = GFP_PGTABLE_KERNEL;
	156	return (pud_t *)get_zeroed_page(gfp);
	157	}
	158
9922c1de	159	#ifndef __HAVE_ARCH_PUD_ALLOC_ONE
d9e8b929 MR	160	/**
	161	* pud_alloc_one - allocate a page for PUD-level page table
	162	* @mm: the mm_struct of the current context
	163	*
	164	* Allocates a page using %GFP_PGTABLE_USER for user context and
	165	* %GFP_PGTABLE_KERNEL for kernel context.
	166	*
	167	* Return: pointer to the allocated memory or %NULL on error
	168	*/
	169	static inline pud_t pud_alloc_one(struct mm_struct mm, unsigned long addr)
	170	{
60639f74	171	return __pud_alloc_one(mm, addr);
d9e8b929 MR	172	}
	173	#endif
	174
60639f74	175	static inline void __pud_free(struct mm_struct mm, pud_t pud)
d9e8b929 MR	176	{
	177	BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
	178	free_page((unsigned long)pud);
	179	}
	180
60639f74 AG	181	#ifndef __HAVE_ARCH_PUD_FREE
	182	static inline void pud_free(struct mm_struct mm, pud_t pud)
	183	{
	184	__pud_free(mm, pud);
	185	}
	186	#endif
	187
d9e8b929 MR	188	#endif /* CONFIG_PGTABLE_LEVELS > 3 */
d9e8b929 MR	189
f9cb654c MR	190	#ifndef __HAVE_ARCH_PGD_FREE
	191	static inline void pgd_free(struct mm_struct mm, pgd_t pgd)
	192	{
	193	free_page((unsigned long)pgd);
	194	}
	195	#endif
	196
5fba4af4 MR	197	#endif /* CONFIG_MMU */
5fba4af4 MR	198
5c01b46b	199	#endif /* __ASM_GENERIC_PGALLOC_H */