[linux-block.git] / mm / sparse-vmemmap.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Virtual Memory Map support
 *
 * (C) 2007 sgi. Christoph Lameter.
 *
 * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
 * virt_to_page, page_address() to be implemented as a base offset
 * calculation without memory access.
 *
 * However, virtual mappings need a page table and TLBs. Many Linux
 * architectures already map their physical space using 1-1 mappings
 * via TLBs. For those arches the virtual memory map is essentially
 * for free if we use the same page size as the 1-1 mappings. In that
 * case the overhead consists of a few additional pages that are
 * allocated to create a view of memory for vmemmap.
 *
 * The architecture is expected to provide a vmemmap_populate() function
 * to instantiate the mapping.
 */
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/memblock.h>
#include <linux/memremap.h>
#include <linux/highmem.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/vmalloc.h>
#include <linux/sched.h>

#include <asm/dma.h>
#include <asm/pgalloc.h>

/*
 * Allocate a block of memory to be used to back the virtual memory map
 * or to back the page tables that are used to create the mapping.
 * Uses the main allocators if they are available, else bootmem.
 */

static void * __ref __earlyonly_bootmem_alloc(int node,
				unsigned long size,
				unsigned long align,
				unsigned long goal)
{
	return memblock_alloc_try_nid_raw(size, align, goal,
					       MEMBLOCK_ALLOC_ACCESSIBLE, node);
}

void * __meminit vmemmap_alloc_block(unsigned long size, int node)
{
	/* If the main allocator is up use that, fallback to bootmem. */
	if (slab_is_available()) {
		gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
		int order = get_order(size);
		static bool warned;
		struct page *page;

		page = alloc_pages_node(node, gfp_mask, order);
		if (page)
			return page_address(page);

		if (!warned) {
			warn_alloc(gfp_mask & ~__GFP_NOWARN, NULL,
				   "vmemmap alloc failure: order:%u", order);
			warned = true;
		}
		return NULL;
	} else
		return __earlyonly_bootmem_alloc(node, size, size,
				__pa(MAX_DMA_ADDRESS));
}

static void * __meminit altmap_alloc_block_buf(unsigned long size,
					       struct vmem_altmap *altmap);

/* need to make sure size is all the same during early stage */
void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node,
					 struct vmem_altmap *altmap)
{
	void *ptr;

	if (altmap)
		return altmap_alloc_block_buf(size, altmap);

	ptr = sparse_buffer_alloc(size);
	if (!ptr)
		ptr = vmemmap_alloc_block(size, node);
	return ptr;
}

static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap)
{
	return altmap->base_pfn + altmap->reserve + altmap->alloc
		+ altmap->align;
}

static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap)
{
	unsigned long allocated = altmap->alloc + altmap->align;

	if (altmap->free > allocated)
		return altmap->free - allocated;
	return 0;
}

static void * __meminit altmap_alloc_block_buf(unsigned long size,
					       struct vmem_altmap *altmap)
{
	unsigned long pfn, nr_pfns, nr_align;

	if (size & ~PAGE_MASK) {
		pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld)\n",
				__func__, size);
		return NULL;
	}

	pfn = vmem_altmap_next_pfn(altmap);
	nr_pfns = size >> PAGE_SHIFT;
	nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG);
	nr_align = ALIGN(pfn, nr_align) - pfn;
	if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap))
		return NULL;

	altmap->alloc += nr_pfns;
	altmap->align += nr_align;
	pfn += nr_align;

	pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx\n",
			__func__, pfn, altmap->alloc, altmap->align, nr_pfns);
	return __va(__pfn_to_phys(pfn));
}

void __meminit vmemmap_verify(pte_t *pte, int node,
				unsigned long start, unsigned long end)
{
	unsigned long pfn = pte_pfn(*pte);
	int actual_node = early_pfn_to_nid(pfn);

	if (node_distance(actual_node, node) > LOCAL_DISTANCE)
		pr_warn_once("[%lx-%lx] potential offnode page_structs\n",
			start, end - 1);
}

pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
				       struct vmem_altmap *altmap,
				       struct page *reuse)
{
	pte_t *pte = pte_offset_kernel(pmd, addr);
	if (pte_none(*pte)) {
		pte_t entry;
		void *p;

		if (!reuse) {
			p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
			if (!p)
				return NULL;
		} else {
			/*
			 * When a PTE/PMD entry is freed from the init_mm
			 * there's a free_pages() call to this page allocated
			 * above. Thus this get_page() is paired with the
			 * put_page_testzero() on the freeing path.
			 * This can only called by certain ZONE_DEVICE path,
			 * and through vmemmap_populate_compound_pages() when
			 * slab is available.
			 */
			get_page(reuse);
			p = page_to_virt(reuse);
		}
		entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
		set_pte_at(&init_mm, addr, pte, entry);
	}
	return pte;
}

static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
{
	void *p = vmemmap_alloc_block(size, node);

	if (!p)
		return NULL;
	memset(p, 0, size);

	return p;
}

pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
{
	pmd_t *pmd = pmd_offset(pud, addr);
	if (pmd_none(*pmd)) {
		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
		if (!p)
			return NULL;
		pmd_populate_kernel(&init_mm, pmd, p);
	}
	return pmd;
}

void __weak __meminit pmd_init(void *addr)
{
}

pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
{
	pud_t *pud = pud_offset(p4d, addr);
	if (pud_none(*pud)) {
		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
		if (!p)
			return NULL;
		pmd_init(p);
		pud_populate(&init_mm, pud, p);
	}
	return pud;
}

void __weak __meminit pud_init(void *addr)
{
}

p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
{
	p4d_t *p4d = p4d_offset(pgd, addr);
	if (p4d_none(*p4d)) {
		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
		if (!p)
			return NULL;
		pud_init(p);
		p4d_populate(&init_mm, p4d, p);
	}
	return p4d;
}

pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
{
	pgd_t *pgd = pgd_offset_k(addr);
	if (pgd_none(*pgd)) {
		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
		if (!p)
			return NULL;
		pgd_populate(&init_mm, pgd, p);
	}
	return pgd;
}

static pte_t * __meminit vmemmap_populate_address(unsigned long addr, int node,
					      struct vmem_altmap *altmap,
					      struct page *reuse)
{
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;

	pgd = vmemmap_pgd_populate(addr, node);
	if (!pgd)
		return NULL;
	p4d = vmemmap_p4d_populate(pgd, addr, node);
	if (!p4d)
		return NULL;
	pud = vmemmap_pud_populate(p4d, addr, node);
	if (!pud)
		return NULL;
	pmd = vmemmap_pmd_populate(pud, addr, node);
	if (!pmd)
		return NULL;
	pte = vmemmap_pte_populate(pmd, addr, node, altmap, reuse);
	if (!pte)
		return NULL;
	vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);

	return pte;
}

static int __meminit vmemmap_populate_range(unsigned long start,
					    unsigned long end, int node,
					    struct vmem_altmap *altmap,
					    struct page *reuse)
{
	unsigned long addr = start;
	pte_t *pte;

	for (; addr < end; addr += PAGE_SIZE) {
		pte = vmemmap_populate_address(addr, node, altmap, reuse);
		if (!pte)
			return -ENOMEM;
	}

	return 0;
}

int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
					 int node, struct vmem_altmap *altmap)
{
	return vmemmap_populate_range(start, end, node, altmap, NULL);
}

void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
				      unsigned long addr, unsigned long next)
{
}

int __weak __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
				       unsigned long addr, unsigned long next)
{
	return 0;
}

int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
					 int node, struct vmem_altmap *altmap)
{
	unsigned long addr;
	unsigned long next;
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;

	for (addr = start; addr < end; addr = next) {
		next = pmd_addr_end(addr, end);

		pgd = vmemmap_pgd_populate(addr, node);
		if (!pgd)
			return -ENOMEM;

		p4d = vmemmap_p4d_populate(pgd, addr, node);
		if (!p4d)
			return -ENOMEM;

		pud = vmemmap_pud_populate(p4d, addr, node);
		if (!pud)
			return -ENOMEM;

		pmd = pmd_offset(pud, addr);
		if (pmd_none(READ_ONCE(*pmd))) {
			void *p;

			p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
			if (p) {
				vmemmap_set_pmd(pmd, p, node, addr, next);
				continue;
			} else if (altmap) {
				/*
				 * No fallback: In any case we care about, the
				 * altmap should be reasonably sized and aligned
				 * such that vmemmap_alloc_block_buf() will always
				 * succeed. For consistency with the PTE case,
				 * return an error here as failure could indicate
				 * a configuration issue with the size of the altmap.
				 */
				return -ENOMEM;
			}
		} else if (vmemmap_check_pmd(pmd, node, addr, next))
			continue;
		if (vmemmap_populate_basepages(addr, next, node, altmap))
			return -ENOMEM;
	}
	return 0;
}

/*
 * For compound pages bigger than section size (e.g. x86 1G compound
 * pages with 2M subsection size) fill the rest of sections as tail
 * pages.
 *
 * Note that memremap_pages() resets @nr_range value and will increment
 * it after each range successful onlining. Thus the value or @nr_range
 * at section memmap populate corresponds to the in-progress range
 * being onlined here.
 */
static bool __meminit reuse_compound_section(unsigned long start_pfn,
					     struct dev_pagemap *pgmap)
{
	unsigned long nr_pages = pgmap_vmemmap_nr(pgmap);
	unsigned long offset = start_pfn -
		PHYS_PFN(pgmap->ranges[pgmap->nr_range].start);

	return !IS_ALIGNED(offset, nr_pages) && nr_pages > PAGES_PER_SUBSECTION;
}

static pte_t * __meminit compound_section_tail_page(unsigned long addr)
{
	pte_t *pte;

	addr -= PAGE_SIZE;

	/*
	 * Assuming sections are populated sequentially, the previous section's
	 * page data can be reused.
	 */
	pte = pte_offset_kernel(pmd_off_k(addr), addr);
	if (!pte)
		return NULL;

	return pte;
}

static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
						     unsigned long start,
						     unsigned long end, int node,
						     struct dev_pagemap *pgmap)
{
	unsigned long size, addr;
	pte_t *pte;
	int rc;

	if (reuse_compound_section(start_pfn, pgmap)) {
		pte = compound_section_tail_page(start);
		if (!pte)
			return -ENOMEM;

		/*
		 * Reuse the page that was populated in the prior iteration
		 * with just tail struct pages.
		 */
		return vmemmap_populate_range(start, end, node, NULL,
					      pte_page(*pte));
	}

	size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page));
	for (addr = start; addr < end; addr += size) {
		unsigned long next, last = addr + size;

		/* Populate the head page vmemmap page */
		pte = vmemmap_populate_address(addr, node, NULL, NULL);
		if (!pte)
			return -ENOMEM;

		/* Populate the tail pages vmemmap page */
		next = addr + PAGE_SIZE;
		pte = vmemmap_populate_address(next, node, NULL, NULL);
		if (!pte)
			return -ENOMEM;

		/*
		 * Reuse the previous page for the rest of tail pages
		 * See layout diagram in Documentation/mm/vmemmap_dedup.rst
		 */
		next += PAGE_SIZE;
		rc = vmemmap_populate_range(next, last, node, NULL,
					    pte_page(*pte));
		if (rc)
			return -ENOMEM;
	}

	return 0;
}

struct page * __meminit __populate_section_memmap(unsigned long pfn,
		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
		struct dev_pagemap *pgmap)
{
	unsigned long start = (unsigned long) pfn_to_page(pfn);
	unsigned long end = start + nr_pages * sizeof(struct page);
	int r;

	if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) ||
		!IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
		return NULL;

	if (vmemmap_can_optimize(altmap, pgmap))
		r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap);
	else
		r = vmemmap_populate(start, end, nid, altmap);

	if (r < 0)
		return NULL;

	return pfn_to_page(pfn);
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
8f6aac41 CL	2	/*
	3	* Virtual Memory Map support
	4	*
cde53535	5	* (C) 2007 sgi. Christoph Lameter.
8f6aac41 CL	6	*
	7	* Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
	8	* virt_to_page, page_address() to be implemented as a base offset
	9	* calculation without memory access.
	10	*
	11	* However, virtual mappings need a page table and TLBs. Many Linux
	12	* architectures already map their physical space using 1-1 mappings
b595076a	13	* via TLBs. For those arches the virtual memory map is essentially
8f6aac41 CL	14	* for free if we use the same page size as the 1-1 mappings. In that
	15	* case the overhead consists of a few additional pages that are
	16	* allocated to create a view of memory for vmemmap.
	17	*
29c71111 AW	18	* The architecture is expected to provide a vmemmap_populate() function
29c71111 AW	19	* to instantiate the mapping.
8f6aac41 CL	20	*/
	21	#include <linux/mm.h>
	22	#include <linux/mmzone.h>
97ad1087	23	#include <linux/memblock.h>
4b94ffdc	24	#include <linux/memremap.h>
8f6aac41	25	#include <linux/highmem.h>
5a0e3ad6	26	#include <linux/slab.h>
8f6aac41 CL	27	#include <linux/spinlock.h>
8f6aac41 CL	28	#include <linux/vmalloc.h>
8bca44bb	29	#include <linux/sched.h>
f41f2ed4	30
8f6aac41 CL	31	#include <asm/dma.h>
8f6aac41 CL	32	#include <asm/pgalloc.h>
ad2fa371	33
8f6aac41 CL	34	/*
	35	* Allocate a block of memory to be used to back the virtual memory map
	36	* or to back the page tables that are used to create the mapping.
	37	* Uses the main allocators if they are available, else bootmem.
	38	*/
e0dc3a53	39
bd721ea7	40	static void * __ref __earlyonly_bootmem_alloc(int node,
e0dc3a53 KH	41	unsigned long size,
	42	unsigned long align,
	43	unsigned long goal)
	44	{
eb31d559	45	return memblock_alloc_try_nid_raw(size, align, goal,
97ad1087	46	MEMBLOCK_ALLOC_ACCESSIBLE, node);
e0dc3a53 KH	47	}
e0dc3a53 KH	48
8f6aac41 CL	49	void * __meminit vmemmap_alloc_block(unsigned long size, int node)
	50	{
	51	/* If the main allocator is up use that, fallback to bootmem. */
	52	if (slab_is_available()) {
fcdaf842 MH	53	gfp_t gfp_mask = GFP_KERNEL\|__GFP_RETRY_MAYFAIL\|__GFP_NOWARN;
	54	int order = get_order(size);
	55	static bool warned;
f52407ce SL	56	struct page *page;
f52407ce SL	57
fcdaf842	58	page = alloc_pages_node(node, gfp_mask, order);
8f6aac41 CL	59	if (page)
8f6aac41 CL	60	return page_address(page);
fcdaf842 MH	61
	62	if (!warned) {
	63	warn_alloc(gfp_mask & ~__GFP_NOWARN, NULL,
	64	"vmemmap alloc failure: order:%u", order);
	65	warned = true;
	66	}
8f6aac41 CL	67	return NULL;
8f6aac41 CL	68	} else
e0dc3a53	69	return __earlyonly_bootmem_alloc(node, size, size,
8f6aac41 CL	70	__pa(MAX_DMA_ADDRESS));
	71	}
	72
56993b4e AK	73	static void * __meminit altmap_alloc_block_buf(unsigned long size,
	74	struct vmem_altmap *altmap);
	75
9bdac914	76	/* need to make sure size is all the same during early stage */
56993b4e AK	77	void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node,
56993b4e AK	78	struct vmem_altmap *altmap)
9bdac914	79	{
56993b4e AK	80	void *ptr;
	81
	82	if (altmap)
	83	return altmap_alloc_block_buf(size, altmap);
9bdac914	84
56993b4e	85	ptr = sparse_buffer_alloc(size);
35fd1eb1 PT	86	if (!ptr)
35fd1eb1 PT	87	ptr = vmemmap_alloc_block(size, node);
9bdac914 YL	88	return ptr;
	89	}
	90
4b94ffdc DW	91	static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap)
	92	{
	93	return altmap->base_pfn + altmap->reserve + altmap->alloc
	94	+ altmap->align;
	95	}
	96
	97	static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap)
	98	{
	99	unsigned long allocated = altmap->alloc + altmap->align;
	100
	101	if (altmap->free > allocated)
	102	return altmap->free - allocated;
	103	return 0;
	104	}
	105
56993b4e AK	106	static void * __meminit altmap_alloc_block_buf(unsigned long size,
56993b4e AK	107	struct vmem_altmap *altmap)
4b94ffdc	108	{
eb804533	109	unsigned long pfn, nr_pfns, nr_align;
4b94ffdc DW	110
	111	if (size & ~PAGE_MASK) {
	112	pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld)\n",
	113	__func__, size);
	114	return NULL;
	115	}
	116
eb804533	117	pfn = vmem_altmap_next_pfn(altmap);
4b94ffdc	118	nr_pfns = size >> PAGE_SHIFT;
eb804533 CH	119	nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG);
	120	nr_align = ALIGN(pfn, nr_align) - pfn;
	121	if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap))
	122	return NULL;
	123
	124	altmap->alloc += nr_pfns;
	125	altmap->align += nr_align;
	126	pfn += nr_align;
	127
4b94ffdc DW	128	pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx\n",
4b94ffdc DW	129	__func__, pfn, altmap->alloc, altmap->align, nr_pfns);
eb804533	130	return __va(__pfn_to_phys(pfn));
4b94ffdc DW	131	}
4b94ffdc DW	132
8f6aac41 CL	133	void __meminit vmemmap_verify(pte_t *pte, int node,
	134	unsigned long start, unsigned long end)
	135	{
	136	unsigned long pfn = pte_pfn(*pte);
	137	int actual_node = early_pfn_to_nid(pfn);
	138
b41ad14c	139	if (node_distance(actual_node, node) > LOCAL_DISTANCE)
abd62377	140	pr_warn_once("[%lx-%lx] potential offnode page_structs\n",
1170532b	141	start, end - 1);
8f6aac41 CL	142	}
8f6aac41 CL	143
1d9cfee7	144	pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
4917f55b JM	145	struct vmem_altmap *altmap,
4917f55b JM	146	struct page *reuse)
8f6aac41	147	{
29c71111 AW	148	pte_t *pte = pte_offset_kernel(pmd, addr);
	149	if (pte_none(*pte)) {
	150	pte_t entry;
1d9cfee7 AK	151	void *p;
1d9cfee7 AK	152
4917f55b JM	153	if (!reuse) {
	154	p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
	155	if (!p)
	156	return NULL;
	157	} else {
	158	/*
	159	* When a PTE/PMD entry is freed from the init_mm
f673bd7c	160	* there's a free_pages() call to this page allocated
4917f55b JM	161	* above. Thus this get_page() is paired with the
	162	* put_page_testzero() on the freeing path.
	163	* This can only called by certain ZONE_DEVICE path,
	164	* and through vmemmap_populate_compound_pages() when
	165	* slab is available.
	166	*/
	167	get_page(reuse);
	168	p = page_to_virt(reuse);
	169	}
29c71111 AW	170	entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
	171	set_pte_at(&init_mm, addr, pte, entry);
	172	}
	173	return pte;
8f6aac41 CL	174	}
8f6aac41 CL	175
f7f99100 PT	176	static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
	177	{
	178	void *p = vmemmap_alloc_block(size, node);
	179
	180	if (!p)
	181	return NULL;
	182	memset(p, 0, size);
	183
	184	return p;
	185	}
	186
29c71111	187	pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
8f6aac41	188	{
29c71111 AW	189	pmd_t *pmd = pmd_offset(pud, addr);
29c71111 AW	190	if (pmd_none(*pmd)) {
f7f99100	191	void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
29c71111	192	if (!p)
9dce07f1	193	return NULL;
29c71111	194	pmd_populate_kernel(&init_mm, pmd, p);
8f6aac41	195	}
29c71111	196	return pmd;
8f6aac41	197	}
8f6aac41	198
7b09f5af FC	199	void __weak __meminit pmd_init(void *addr)
	200	{
	201	}
	202
c2febafc	203	pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
8f6aac41	204	{
c2febafc	205	pud_t *pud = pud_offset(p4d, addr);
29c71111	206	if (pud_none(*pud)) {
f7f99100	207	void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
29c71111	208	if (!p)
9dce07f1	209	return NULL;
7b09f5af	210	pmd_init(p);
29c71111 AW	211	pud_populate(&init_mm, pud, p);
	212	}
	213	return pud;
	214	}
8f6aac41	215
7b09f5af FC	216	void __weak __meminit pud_init(void *addr)
	217	{
	218	}
	219
c2febafc KS	220	p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
	221	{
	222	p4d_t *p4d = p4d_offset(pgd, addr);
	223	if (p4d_none(*p4d)) {
f7f99100	224	void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
c2febafc KS	225	if (!p)
c2febafc KS	226	return NULL;
7b09f5af	227	pud_init(p);
c2febafc KS	228	p4d_populate(&init_mm, p4d, p);
	229	}
	230	return p4d;
	231	}
	232
29c71111 AW	233	pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
	234	{
	235	pgd_t *pgd = pgd_offset_k(addr);
	236	if (pgd_none(*pgd)) {
f7f99100	237	void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
29c71111	238	if (!p)
9dce07f1	239	return NULL;
29c71111	240	pgd_populate(&init_mm, pgd, p);
8f6aac41	241	}
29c71111	242	return pgd;
8f6aac41 CL	243	}
8f6aac41 CL	244
2beea70a	245	static pte_t * __meminit vmemmap_populate_address(unsigned long addr, int node,
4917f55b JM	246	struct vmem_altmap *altmap,
4917f55b JM	247	struct page *reuse)
8f6aac41	248	{
29c71111	249	pgd_t *pgd;
c2febafc	250	p4d_t *p4d;
29c71111 AW	251	pud_t *pud;
	252	pmd_t *pmd;
	253	pte_t *pte;
8f6aac41	254
2beea70a JM	255	pgd = vmemmap_pgd_populate(addr, node);
	256	if (!pgd)
	257	return NULL;
	258	p4d = vmemmap_p4d_populate(pgd, addr, node);
	259	if (!p4d)
	260	return NULL;
	261	pud = vmemmap_pud_populate(p4d, addr, node);
	262	if (!pud)
	263	return NULL;
	264	pmd = vmemmap_pmd_populate(pud, addr, node);
	265	if (!pmd)
	266	return NULL;
4917f55b	267	pte = vmemmap_pte_populate(pmd, addr, node, altmap, reuse);
2beea70a JM	268	if (!pte)
	269	return NULL;
	270	vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
	271
	272	return pte;
	273	}
	274
	275	static int __meminit vmemmap_populate_range(unsigned long start,
	276	unsigned long end, int node,
4917f55b JM	277	struct vmem_altmap *altmap,
4917f55b JM	278	struct page *reuse)
2beea70a JM	279	{
	280	unsigned long addr = start;
	281	pte_t *pte;
	282
29c71111	283	for (; addr < end; addr += PAGE_SIZE) {
4917f55b	284	pte = vmemmap_populate_address(addr, node, altmap, reuse);
29c71111 AW	285	if (!pte)
29c71111 AW	286	return -ENOMEM;
8f6aac41	287	}
29c71111 AW	288
29c71111 AW	289	return 0;
8f6aac41	290	}
8f6aac41	291
2beea70a JM	292	int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
	293	int node, struct vmem_altmap *altmap)
	294	{
4917f55b JM	295	return vmemmap_populate_range(start, end, node, altmap, NULL);
	296	}
	297
2045a3b8 FC	298	void __weak __meminit vmemmap_set_pmd(pmd_t pmd, void p, int node,
	299	unsigned long addr, unsigned long next)
	300	{
	301	}
	302
	303	int __weak __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
	304	unsigned long addr, unsigned long next)
	305	{
	306	return 0;
	307	}
	308
	309	int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
	310	int node, struct vmem_altmap *altmap)
	311	{
	312	unsigned long addr;
	313	unsigned long next;
	314	pgd_t *pgd;
	315	p4d_t *p4d;
	316	pud_t *pud;
	317	pmd_t *pmd;
	318
	319	for (addr = start; addr < end; addr = next) {
	320	next = pmd_addr_end(addr, end);
	321
	322	pgd = vmemmap_pgd_populate(addr, node);
	323	if (!pgd)
	324	return -ENOMEM;
	325
	326	p4d = vmemmap_p4d_populate(pgd, addr, node);
	327	if (!p4d)
	328	return -ENOMEM;
	329
	330	pud = vmemmap_pud_populate(p4d, addr, node);
	331	if (!pud)
	332	return -ENOMEM;
	333
	334	pmd = pmd_offset(pud, addr);
	335	if (pmd_none(READ_ONCE(*pmd))) {
	336	void *p;
	337
	338	p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
	339	if (p) {
	340	vmemmap_set_pmd(pmd, p, node, addr, next);
	341	continue;
	342	} else if (altmap) {
	343	/*
	344	* No fallback: In any case we care about, the
	345	* altmap should be reasonably sized and aligned
	346	* such that vmemmap_alloc_block_buf() will always
	347	* succeed. For consistency with the PTE case,
	348	* return an error here as failure could indicate
	349	* a configuration issue with the size of the altmap.
	350	*/
	351	return -ENOMEM;
	352	}
	353	} else if (vmemmap_check_pmd(pmd, node, addr, next))
	354	continue;
	355	if (vmemmap_populate_basepages(addr, next, node, altmap))
	356	return -ENOMEM;
	357	}
	358	return 0;
	359	}
	360
4917f55b JM	361	/*
	362	* For compound pages bigger than section size (e.g. x86 1G compound
	363	* pages with 2M subsection size) fill the rest of sections as tail
	364	* pages.
	365	*
	366	* Note that memremap_pages() resets @nr_range value and will increment
	367	* it after each range successful onlining. Thus the value or @nr_range
	368	* at section memmap populate corresponds to the in-progress range
	369	* being onlined here.
	370	*/
	371	static bool __meminit reuse_compound_section(unsigned long start_pfn,
	372	struct dev_pagemap *pgmap)
	373	{
	374	unsigned long nr_pages = pgmap_vmemmap_nr(pgmap);
	375	unsigned long offset = start_pfn -
	376	PHYS_PFN(pgmap->ranges[pgmap->nr_range].start);
	377
	378	return !IS_ALIGNED(offset, nr_pages) && nr_pages > PAGES_PER_SUBSECTION;
	379	}
	380
	381	static pte_t * __meminit compound_section_tail_page(unsigned long addr)
	382	{
	383	pte_t *pte;
	384
	385	addr -= PAGE_SIZE;
	386
	387	/*
	388	* Assuming sections are populated sequentially, the previous section's
	389	* page data can be reused.
	390	*/
	391	pte = pte_offset_kernel(pmd_off_k(addr), addr);
	392	if (!pte)
	393	return NULL;
	394
	395	return pte;
	396	}
	397
	398	static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
	399	unsigned long start,
	400	unsigned long end, int node,
	401	struct dev_pagemap *pgmap)
	402	{
	403	unsigned long size, addr;
	404	pte_t *pte;
	405	int rc;
	406
	407	if (reuse_compound_section(start_pfn, pgmap)) {
	408	pte = compound_section_tail_page(start);
	409	if (!pte)
	410	return -ENOMEM;
	411
	412	/*
	413	* Reuse the page that was populated in the prior iteration
	414	* with just tail struct pages.
	415	*/
	416	return vmemmap_populate_range(start, end, node, NULL,
	417	pte_page(*pte));
	418	}
	419
	420	size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page));
	421	for (addr = start; addr < end; addr += size) {
55896f93	422	unsigned long next, last = addr + size;
4917f55b JM	423
	424	/* Populate the head page vmemmap page */
	425	pte = vmemmap_populate_address(addr, node, NULL, NULL);
	426	if (!pte)
	427	return -ENOMEM;
	428
	429	/* Populate the tail pages vmemmap page */
	430	next = addr + PAGE_SIZE;
	431	pte = vmemmap_populate_address(next, node, NULL, NULL);
	432	if (!pte)
	433	return -ENOMEM;
	434
	435	/*
	436	* Reuse the previous page for the rest of tail pages
ee65728e	437	* See layout diagram in Documentation/mm/vmemmap_dedup.rst
4917f55b JM	438	*/
	439	next += PAGE_SIZE;
	440	rc = vmemmap_populate_range(next, last, node, NULL,
	441	pte_page(*pte));
	442	if (rc)
	443	return -ENOMEM;
	444	}
	445
	446	return 0;
2beea70a JM	447	}
2beea70a JM	448
e9c0a3f0	449	struct page * __meminit __populate_section_memmap(unsigned long pfn,
e3246d8f JM	450	unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
e3246d8f JM	451	struct dev_pagemap *pgmap)
8f6aac41	452	{
6cda7204 WY	453	unsigned long start = (unsigned long) pfn_to_page(pfn);
6cda7204 WY	454	unsigned long end = start + nr_pages * sizeof(struct page);
4917f55b	455	int r;
6cda7204 WY	456
	457	if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) \|\|
	458	!IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
	459	return NULL;
0aad818b	460
87a7ae75	461	if (vmemmap_can_optimize(altmap, pgmap))
4917f55b JM	462	r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap);
	463	else
	464	r = vmemmap_populate(start, end, nid, altmap);
	465
	466	if (r < 0)
8f6aac41 CL	467	return NULL;
8f6aac41 CL	468
e9c0a3f0	469	return pfn_to_page(pfn);
8f6aac41	470	}