1 // SPDX-License-Identifier: GPL-2.0-only
3 * mm_init.c - Memory initialisation verification and debugging
5 * Copyright 2008 IBM Corporation, 2008
6 * Author Mel Gorman <mel@csn.ul.ie>
9 #include <linux/kernel.h>
10 #include <linux/init.h>
11 #include <linux/kobject.h>
12 #include <linux/export.h>
13 #include <linux/memory.h>
14 #include <linux/notifier.h>
15 #include <linux/sched.h>
16 #include <linux/mman.h>
17 #include <linux/memblock.h>
18 #include <linux/page-isolation.h>
19 #include <linux/padata.h>
20 #include <linux/nmi.h>
21 #include <linux/buffer_head.h>
22 #include <linux/kmemleak.h>
23 #include <linux/kfence.h>
24 #include <linux/page_ext.h>
25 #include <linux/pti.h>
26 #include <linux/pgtable.h>
27 #include <linux/swap.h>
28 #include <linux/cma.h>
33 #include <asm/setup.h>
35 #ifdef CONFIG_DEBUG_MEMORY_INIT
36 int __meminitdata mminit_loglevel;
38 /* The zonelists are simply reported, validation is manual. */
39 void __init mminit_verify_zonelist(void)
43 if (mminit_loglevel < MMINIT_VERIFY)
46 for_each_online_node(nid) {
47 pg_data_t *pgdat = NODE_DATA(nid);
50 struct zonelist *zonelist;
51 int i, listid, zoneid;
53 BUILD_BUG_ON(MAX_ZONELISTS > 2);
54 for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) {
56 /* Identify the zone and nodelist */
57 zoneid = i % MAX_NR_ZONES;
58 listid = i / MAX_NR_ZONES;
59 zonelist = &pgdat->node_zonelists[listid];
60 zone = &pgdat->node_zones[zoneid];
61 if (!populated_zone(zone))
64 /* Print information about the zonelist */
65 printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ",
66 listid > 0 ? "thisnode" : "general", nid,
69 /* Iterate the zonelist */
70 for_each_zone_zonelist(zone, z, zonelist, zoneid)
71 pr_cont("%d:%s ", zone_to_nid(zone), zone->name);
77 void __init mminit_verify_pageflags_layout(void)
80 unsigned long or_mask, add_mask;
82 shift = BITS_PER_LONG;
83 width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH
84 - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;
85 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
86 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
95 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
96 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",
102 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts",
103 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",
104 (unsigned long)SECTIONS_PGSHIFT,
105 (unsigned long)NODES_PGSHIFT,
106 (unsigned long)ZONES_PGSHIFT,
107 (unsigned long)LAST_CPUPID_PGSHIFT,
108 (unsigned long)KASAN_TAG_PGSHIFT);
109 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid",
110 "Node/Zone ID: %lu -> %lu\n",
111 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT),
112 (unsigned long)ZONEID_PGOFF);
113 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage",
114 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n",
115 shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0);
116 #ifdef NODE_NOT_IN_PAGE_FLAGS
117 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
118 "Node not in page flags");
120 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
121 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
122 "Last cpupid not in page flags");
125 if (SECTIONS_WIDTH) {
126 shift -= SECTIONS_WIDTH;
127 BUG_ON(shift != SECTIONS_PGSHIFT);
130 shift -= NODES_WIDTH;
131 BUG_ON(shift != NODES_PGSHIFT);
134 shift -= ZONES_WIDTH;
135 BUG_ON(shift != ZONES_PGSHIFT);
138 /* Check for bitmask overlaps */
139 or_mask = (ZONES_MASK << ZONES_PGSHIFT) |
140 (NODES_MASK << NODES_PGSHIFT) |
141 (SECTIONS_MASK << SECTIONS_PGSHIFT);
142 add_mask = (ZONES_MASK << ZONES_PGSHIFT) +
143 (NODES_MASK << NODES_PGSHIFT) +
144 (SECTIONS_MASK << SECTIONS_PGSHIFT);
145 BUG_ON(or_mask != add_mask);
148 static __init int set_mminit_loglevel(char *str)
150 get_option(&str, &mminit_loglevel);
153 early_param("mminit_loglevel", set_mminit_loglevel);
154 #endif /* CONFIG_DEBUG_MEMORY_INIT */
156 struct kobject *mm_kobj;
159 s32 vm_committed_as_batch = 32;
161 void mm_compute_batch(int overcommit_policy)
164 s32 nr = num_present_cpus();
165 s32 batch = max_t(s32, nr*2, 32);
166 unsigned long ram_pages = totalram_pages();
169 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of
170 * (total memory/#cpus), and lift it to 25% for other policies
171 * to easy the possible lock contention for percpu_counter
172 * vm_committed_as, while the max limit is INT_MAX
174 if (overcommit_policy == OVERCOMMIT_NEVER)
175 memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX);
177 memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX);
179 vm_committed_as_batch = max_t(s32, memsized_batch, batch);
182 static int __meminit mm_compute_batch_notifier(struct notifier_block *self,
183 unsigned long action, void *arg)
188 mm_compute_batch(sysctl_overcommit_memory);
196 static int __init mm_compute_batch_init(void)
198 mm_compute_batch(sysctl_overcommit_memory);
199 hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI);
203 __initcall(mm_compute_batch_init);
207 static int __init mm_sysfs_init(void)
209 mm_kobj = kobject_create_and_add("mm", kernel_kobj);
215 postcore_initcall(mm_sysfs_init);
217 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
218 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
219 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
221 static unsigned long required_kernelcore __initdata;
222 static unsigned long required_kernelcore_percent __initdata;
223 static unsigned long required_movablecore __initdata;
224 static unsigned long required_movablecore_percent __initdata;
226 static unsigned long nr_kernel_pages __initdata;
227 static unsigned long nr_all_pages __initdata;
228 static unsigned long dma_reserve __initdata;
230 static bool deferred_struct_pages __meminitdata;
232 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
234 static int __init cmdline_parse_core(char *p, unsigned long *core,
235 unsigned long *percent)
237 unsigned long long coremem;
243 /* Value may be a percentage of total memory, otherwise bytes */
244 coremem = simple_strtoull(p, &endptr, 0);
245 if (*endptr == '%') {
246 /* Paranoid check for percent values greater than 100 */
247 WARN_ON(coremem > 100);
251 coremem = memparse(p, &p);
252 /* Paranoid check that UL is enough for the coremem value */
253 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
255 *core = coremem >> PAGE_SHIFT;
261 bool mirrored_kernelcore __initdata_memblock;
264 * kernelcore=size sets the amount of memory for use for allocations that
265 * cannot be reclaimed or migrated.
267 static int __init cmdline_parse_kernelcore(char *p)
269 /* parse kernelcore=mirror */
270 if (parse_option_str(p, "mirror")) {
271 mirrored_kernelcore = true;
275 return cmdline_parse_core(p, &required_kernelcore,
276 &required_kernelcore_percent);
278 early_param("kernelcore", cmdline_parse_kernelcore);
281 * movablecore=size sets the amount of memory for use for allocations that
282 * can be reclaimed or migrated.
284 static int __init cmdline_parse_movablecore(char *p)
286 return cmdline_parse_core(p, &required_movablecore,
287 &required_movablecore_percent);
289 early_param("movablecore", cmdline_parse_movablecore);
292 * early_calculate_totalpages()
293 * Sum pages in active regions for movable zone.
294 * Populate N_MEMORY for calculating usable_nodes.
296 static unsigned long __init early_calculate_totalpages(void)
298 unsigned long totalpages = 0;
299 unsigned long start_pfn, end_pfn;
302 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
303 unsigned long pages = end_pfn - start_pfn;
307 node_set_state(nid, N_MEMORY);
313 * This finds a zone that can be used for ZONE_MOVABLE pages. The
314 * assumption is made that zones within a node are ordered in monotonic
315 * increasing memory addresses so that the "highest" populated zone is used
317 static void __init find_usable_zone_for_movable(void)
320 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
321 if (zone_index == ZONE_MOVABLE)
324 if (arch_zone_highest_possible_pfn[zone_index] >
325 arch_zone_lowest_possible_pfn[zone_index])
329 VM_BUG_ON(zone_index == -1);
330 movable_zone = zone_index;
334 * Find the PFN the Movable zone begins in each node. Kernel memory
335 * is spread evenly between nodes as long as the nodes have enough
336 * memory. When they don't, some nodes will have more kernelcore than
339 static void __init find_zone_movable_pfns_for_nodes(void)
342 unsigned long usable_startpfn;
343 unsigned long kernelcore_node, kernelcore_remaining;
344 /* save the state before borrow the nodemask */
345 nodemask_t saved_node_state = node_states[N_MEMORY];
346 unsigned long totalpages = early_calculate_totalpages();
347 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
348 struct memblock_region *r;
350 /* Need to find movable_zone earlier when movable_node is specified. */
351 find_usable_zone_for_movable();
354 * If movable_node is specified, ignore kernelcore and movablecore
357 if (movable_node_is_enabled()) {
358 for_each_mem_region(r) {
359 if (!memblock_is_hotpluggable(r))
362 nid = memblock_get_region_node(r);
364 usable_startpfn = PFN_DOWN(r->base);
365 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
366 min(usable_startpfn, zone_movable_pfn[nid]) :
374 * If kernelcore=mirror is specified, ignore movablecore option
376 if (mirrored_kernelcore) {
377 bool mem_below_4gb_not_mirrored = false;
379 if (!memblock_has_mirror()) {
380 pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n");
384 for_each_mem_region(r) {
385 if (memblock_is_mirror(r))
388 nid = memblock_get_region_node(r);
390 usable_startpfn = memblock_region_memory_base_pfn(r);
392 if (usable_startpfn < PHYS_PFN(SZ_4G)) {
393 mem_below_4gb_not_mirrored = true;
397 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
398 min(usable_startpfn, zone_movable_pfn[nid]) :
402 if (mem_below_4gb_not_mirrored)
403 pr_warn("This configuration results in unmirrored kernel memory.\n");
409 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
410 * amount of necessary memory.
412 if (required_kernelcore_percent)
413 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
415 if (required_movablecore_percent)
416 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
420 * If movablecore= was specified, calculate what size of
421 * kernelcore that corresponds so that memory usable for
422 * any allocation type is evenly spread. If both kernelcore
423 * and movablecore are specified, then the value of kernelcore
424 * will be used for required_kernelcore if it's greater than
425 * what movablecore would have allowed.
427 if (required_movablecore) {
428 unsigned long corepages;
431 * Round-up so that ZONE_MOVABLE is at least as large as what
432 * was requested by the user
434 required_movablecore =
435 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
436 required_movablecore = min(totalpages, required_movablecore);
437 corepages = totalpages - required_movablecore;
439 required_kernelcore = max(required_kernelcore, corepages);
443 * If kernelcore was not specified or kernelcore size is larger
444 * than totalpages, there is no ZONE_MOVABLE.
446 if (!required_kernelcore || required_kernelcore >= totalpages)
449 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
450 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
453 /* Spread kernelcore memory as evenly as possible throughout nodes */
454 kernelcore_node = required_kernelcore / usable_nodes;
455 for_each_node_state(nid, N_MEMORY) {
456 unsigned long start_pfn, end_pfn;
459 * Recalculate kernelcore_node if the division per node
460 * now exceeds what is necessary to satisfy the requested
461 * amount of memory for the kernel
463 if (required_kernelcore < kernelcore_node)
464 kernelcore_node = required_kernelcore / usable_nodes;
467 * As the map is walked, we track how much memory is usable
468 * by the kernel using kernelcore_remaining. When it is
469 * 0, the rest of the node is usable by ZONE_MOVABLE
471 kernelcore_remaining = kernelcore_node;
473 /* Go through each range of PFNs within this node */
474 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
475 unsigned long size_pages;
477 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
478 if (start_pfn >= end_pfn)
481 /* Account for what is only usable for kernelcore */
482 if (start_pfn < usable_startpfn) {
483 unsigned long kernel_pages;
484 kernel_pages = min(end_pfn, usable_startpfn)
487 kernelcore_remaining -= min(kernel_pages,
488 kernelcore_remaining);
489 required_kernelcore -= min(kernel_pages,
490 required_kernelcore);
492 /* Continue if range is now fully accounted */
493 if (end_pfn <= usable_startpfn) {
496 * Push zone_movable_pfn to the end so
497 * that if we have to rebalance
498 * kernelcore across nodes, we will
499 * not double account here
501 zone_movable_pfn[nid] = end_pfn;
504 start_pfn = usable_startpfn;
508 * The usable PFN range for ZONE_MOVABLE is from
509 * start_pfn->end_pfn. Calculate size_pages as the
510 * number of pages used as kernelcore
512 size_pages = end_pfn - start_pfn;
513 if (size_pages > kernelcore_remaining)
514 size_pages = kernelcore_remaining;
515 zone_movable_pfn[nid] = start_pfn + size_pages;
518 * Some kernelcore has been met, update counts and
519 * break if the kernelcore for this node has been
522 required_kernelcore -= min(required_kernelcore,
524 kernelcore_remaining -= size_pages;
525 if (!kernelcore_remaining)
531 * If there is still required_kernelcore, we do another pass with one
532 * less node in the count. This will push zone_movable_pfn[nid] further
533 * along on the nodes that still have memory until kernelcore is
537 if (usable_nodes && required_kernelcore > usable_nodes)
541 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
542 for (nid = 0; nid < MAX_NUMNODES; nid++) {
543 unsigned long start_pfn, end_pfn;
545 zone_movable_pfn[nid] =
546 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
548 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
549 if (zone_movable_pfn[nid] >= end_pfn)
550 zone_movable_pfn[nid] = 0;
554 /* restore the node_state */
555 node_states[N_MEMORY] = saved_node_state;
558 void __meminit __init_single_page(struct page *page, unsigned long pfn,
559 unsigned long zone, int nid)
561 mm_zero_struct_page(page);
562 set_page_links(page, zone, nid, pfn);
563 init_page_count(page);
564 page_mapcount_reset(page);
565 page_cpupid_reset_last(page);
566 page_kasan_tag_reset(page);
568 INIT_LIST_HEAD(&page->lru);
569 #ifdef WANT_PAGE_VIRTUAL
570 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
571 if (!is_highmem_idx(zone))
572 set_page_address(page, __va(pfn << PAGE_SHIFT));
578 * During memory init memblocks map pfns to nids. The search is expensive and
579 * this caches recent lookups. The implementation of __early_pfn_to_nid
580 * treats start/end as pfns.
582 struct mminit_pfnnid_cache {
583 unsigned long last_start;
584 unsigned long last_end;
588 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
591 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
593 static int __meminit __early_pfn_to_nid(unsigned long pfn,
594 struct mminit_pfnnid_cache *state)
596 unsigned long start_pfn, end_pfn;
599 if (state->last_start <= pfn && pfn < state->last_end)
600 return state->last_nid;
602 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
603 if (nid != NUMA_NO_NODE) {
604 state->last_start = start_pfn;
605 state->last_end = end_pfn;
606 state->last_nid = nid;
612 int __meminit early_pfn_to_nid(unsigned long pfn)
614 static DEFINE_SPINLOCK(early_pfn_lock);
617 spin_lock(&early_pfn_lock);
618 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
620 nid = first_online_node;
621 spin_unlock(&early_pfn_lock);
626 int hashdist = HASHDIST_DEFAULT;
628 static int __init set_hashdist(char *str)
632 hashdist = simple_strtoul(str, &str, 0);
635 __setup("hashdist=", set_hashdist);
637 static inline void fixup_hashdist(void)
639 if (num_node_state(N_MEMORY) == 1)
643 static inline void fixup_hashdist(void) {}
644 #endif /* CONFIG_NUMA */
646 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
647 static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
649 pgdat->first_deferred_pfn = ULONG_MAX;
652 /* Returns true if the struct page for the pfn is initialised */
653 static inline bool __meminit early_page_initialised(unsigned long pfn, int nid)
655 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
662 * Returns true when the remaining initialisation should be deferred until
663 * later in the boot cycle when it can be parallelised.
665 static bool __meminit
666 defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
668 static unsigned long prev_end_pfn, nr_initialised;
670 if (early_page_ext_enabled())
673 * prev_end_pfn static that contains the end of previous zone
674 * No need to protect because called very early in boot before smp_init.
676 if (prev_end_pfn != end_pfn) {
677 prev_end_pfn = end_pfn;
681 /* Always populate low zones for address-constrained allocations */
682 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
685 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
688 * We start only with one section of pages, more pages are added as
689 * needed until the rest of deferred pages are initialized.
692 if ((nr_initialised > PAGES_PER_SECTION) &&
693 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
694 NODE_DATA(nid)->first_deferred_pfn = pfn;
700 static void __meminit init_reserved_page(unsigned long pfn, int nid)
705 if (early_page_initialised(pfn, nid))
708 pgdat = NODE_DATA(nid);
710 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
711 struct zone *zone = &pgdat->node_zones[zid];
713 if (zone_spans_pfn(zone, pfn))
716 __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
719 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
721 static inline bool early_page_initialised(unsigned long pfn, int nid)
726 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
731 static inline void init_reserved_page(unsigned long pfn, int nid)
734 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
737 * Initialised pages do not have PageReserved set. This function is
738 * called for each range allocated by the bootmem allocator and
739 * marks the pages PageReserved. The remaining valid pages are later
740 * sent to the buddy page allocator.
742 void __meminit reserve_bootmem_region(phys_addr_t start,
743 phys_addr_t end, int nid)
745 unsigned long start_pfn = PFN_DOWN(start);
746 unsigned long end_pfn = PFN_UP(end);
748 for (; start_pfn < end_pfn; start_pfn++) {
749 if (pfn_valid(start_pfn)) {
750 struct page *page = pfn_to_page(start_pfn);
752 init_reserved_page(start_pfn, nid);
754 /* Avoid false-positive PageTail() */
755 INIT_LIST_HEAD(&page->lru);
758 * no need for atomic set_bit because the struct
759 * page is not visible yet so nobody should
762 __SetPageReserved(page);
767 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
768 static bool __meminit
769 overlap_memmap_init(unsigned long zone, unsigned long *pfn)
771 static struct memblock_region *r;
773 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
774 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
775 for_each_mem_region(r) {
776 if (*pfn < memblock_region_memory_end_pfn(r))
780 if (*pfn >= memblock_region_memory_base_pfn(r) &&
781 memblock_is_mirror(r)) {
782 *pfn = memblock_region_memory_end_pfn(r);
790 * Only struct pages that correspond to ranges defined by memblock.memory
791 * are zeroed and initialized by going through __init_single_page() during
792 * memmap_init_zone_range().
794 * But, there could be struct pages that correspond to holes in
795 * memblock.memory. This can happen because of the following reasons:
796 * - physical memory bank size is not necessarily the exact multiple of the
797 * arbitrary section size
798 * - early reserved memory may not be listed in memblock.memory
799 * - non-memory regions covered by the contigious flatmem mapping
800 * - memory layouts defined with memmap= kernel parameter may not align
801 * nicely with memmap sections
803 * Explicitly initialize those struct pages so that:
804 * - PG_Reserved is set
805 * - zone and node links point to zone and node that span the page if the
806 * hole is in the middle of a zone
807 * - zone and node links point to adjacent zone/node if the hole falls on
808 * the zone boundary; the pages in such holes will be prepended to the
809 * zone/node above the hole except for the trailing pages in the last
810 * section that will be appended to the zone/node below.
812 static void __init init_unavailable_range(unsigned long spfn,
819 for (pfn = spfn; pfn < epfn; pfn++) {
820 if (!pfn_valid(pageblock_start_pfn(pfn))) {
821 pfn = pageblock_end_pfn(pfn) - 1;
824 __init_single_page(pfn_to_page(pfn), pfn, zone, node);
825 __SetPageReserved(pfn_to_page(pfn));
830 pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n",
831 node, zone_names[zone], pgcnt);
835 * Initially all pages are reserved - free ones are freed
836 * up by memblock_free_all() once the early boot process is
837 * done. Non-atomic initialization, single-pass.
839 * All aligned pageblocks are initialized to the specified migratetype
840 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
841 * zone stats (e.g., nr_isolate_pageblock) are touched.
843 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,
844 unsigned long start_pfn, unsigned long zone_end_pfn,
845 enum meminit_context context,
846 struct vmem_altmap *altmap, int migratetype)
848 unsigned long pfn, end_pfn = start_pfn + size;
851 if (highest_memmap_pfn < end_pfn - 1)
852 highest_memmap_pfn = end_pfn - 1;
854 #ifdef CONFIG_ZONE_DEVICE
856 * Honor reservation requested by the driver for this ZONE_DEVICE
857 * memory. We limit the total number of pages to initialize to just
858 * those that might contain the memory mapping. We will defer the
859 * ZONE_DEVICE page initialization until after we have released
862 if (zone == ZONE_DEVICE) {
866 if (start_pfn == altmap->base_pfn)
867 start_pfn += altmap->reserve;
868 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
872 for (pfn = start_pfn; pfn < end_pfn; ) {
874 * There can be holes in boot-time mem_map[]s handed to this
875 * function. They do not exist on hotplugged memory.
877 if (context == MEMINIT_EARLY) {
878 if (overlap_memmap_init(zone, &pfn))
880 if (defer_init(nid, pfn, zone_end_pfn)) {
881 deferred_struct_pages = true;
886 page = pfn_to_page(pfn);
887 __init_single_page(page, pfn, zone, nid);
888 if (context == MEMINIT_HOTPLUG)
889 __SetPageReserved(page);
892 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
893 * such that unmovable allocations won't be scattered all
894 * over the place during system boot.
896 if (pageblock_aligned(pfn)) {
897 set_pageblock_migratetype(page, migratetype);
904 static void __init memmap_init_zone_range(struct zone *zone,
905 unsigned long start_pfn,
906 unsigned long end_pfn,
907 unsigned long *hole_pfn)
909 unsigned long zone_start_pfn = zone->zone_start_pfn;
910 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
911 int nid = zone_to_nid(zone), zone_id = zone_idx(zone);
913 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);
914 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);
916 if (start_pfn >= end_pfn)
919 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
920 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
922 if (*hole_pfn < start_pfn)
923 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
928 static void __init memmap_init(void)
930 unsigned long start_pfn, end_pfn;
931 unsigned long hole_pfn = 0;
932 int i, j, zone_id = 0, nid;
934 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
935 struct pglist_data *node = NODE_DATA(nid);
937 for (j = 0; j < MAX_NR_ZONES; j++) {
938 struct zone *zone = node->node_zones + j;
940 if (!populated_zone(zone))
943 memmap_init_zone_range(zone, start_pfn, end_pfn,
949 #ifdef CONFIG_SPARSEMEM
951 * Initialize the memory map for hole in the range [memory_end,
953 * Append the pages in this hole to the highest zone in the last
955 * The call to init_unavailable_range() is outside the ifdef to
956 * silence the compiler warining about zone_id set but not used;
957 * for FLATMEM it is a nop anyway
959 end_pfn = round_up(end_pfn, PAGES_PER_SECTION);
960 if (hole_pfn < end_pfn)
962 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);
965 #ifdef CONFIG_ZONE_DEVICE
966 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
967 unsigned long zone_idx, int nid,
968 struct dev_pagemap *pgmap)
971 __init_single_page(page, pfn, zone_idx, nid);
974 * Mark page reserved as it will need to wait for onlining
975 * phase for it to be fully associated with a zone.
977 * We can use the non-atomic __set_bit operation for setting
978 * the flag as we are still initializing the pages.
980 __SetPageReserved(page);
983 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
984 * and zone_device_data. It is a bug if a ZONE_DEVICE page is
985 * ever freed or placed on a driver-private list.
988 page->zone_device_data = NULL;
991 * Mark the block movable so that blocks are reserved for
992 * movable at startup. This will force kernel allocations
993 * to reserve their blocks rather than leaking throughout
994 * the address space during boot when many long-lived
995 * kernel allocations are made.
997 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
998 * because this is done early in section_activate()
1000 if (pageblock_aligned(pfn)) {
1001 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1006 * ZONE_DEVICE pages are released directly to the driver page allocator
1007 * which will set the page count to 1 when allocating the page.
1009 if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
1010 pgmap->type == MEMORY_DEVICE_COHERENT)
1011 set_page_count(page, 0);
1015 * With compound page geometry and when struct pages are stored in ram most
1016 * tail pages are reused. Consequently, the amount of unique struct pages to
1017 * initialize is a lot smaller that the total amount of struct pages being
1018 * mapped. This is a paired / mild layering violation with explicit knowledge
1019 * of how the sparse_vmemmap internals handle compound pages in the lack
1020 * of an altmap. See vmemmap_populate_compound_pages().
1022 static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,
1023 struct dev_pagemap *pgmap)
1025 if (!vmemmap_can_optimize(altmap, pgmap))
1026 return pgmap_vmemmap_nr(pgmap);
1028 return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page));
1031 static void __ref memmap_init_compound(struct page *head,
1032 unsigned long head_pfn,
1033 unsigned long zone_idx, int nid,
1034 struct dev_pagemap *pgmap,
1035 unsigned long nr_pages)
1037 unsigned long pfn, end_pfn = head_pfn + nr_pages;
1038 unsigned int order = pgmap->vmemmap_shift;
1040 __SetPageHead(head);
1041 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) {
1042 struct page *page = pfn_to_page(pfn);
1044 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
1045 prep_compound_tail(head, pfn - head_pfn);
1046 set_page_count(page, 0);
1049 * The first tail page stores important compound page info.
1050 * Call prep_compound_head() after the first tail page has
1051 * been initialized, to not have the data overwritten.
1053 if (pfn == head_pfn + 1)
1054 prep_compound_head(head, order);
1058 void __ref memmap_init_zone_device(struct zone *zone,
1059 unsigned long start_pfn,
1060 unsigned long nr_pages,
1061 struct dev_pagemap *pgmap)
1063 unsigned long pfn, end_pfn = start_pfn + nr_pages;
1064 struct pglist_data *pgdat = zone->zone_pgdat;
1065 struct vmem_altmap *altmap = pgmap_altmap(pgmap);
1066 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap);
1067 unsigned long zone_idx = zone_idx(zone);
1068 unsigned long start = jiffies;
1069 int nid = pgdat->node_id;
1071 if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))
1075 * The call to memmap_init should have already taken care
1076 * of the pages reserved for the memmap, so we can just jump to
1077 * the end of that region and start processing the device pages.
1080 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
1081 nr_pages = end_pfn - start_pfn;
1084 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {
1085 struct page *page = pfn_to_page(pfn);
1087 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
1089 if (pfns_per_compound == 1)
1092 memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
1093 compound_nr_pages(altmap, pgmap));
1096 pr_debug("%s initialised %lu pages in %ums\n", __func__,
1097 nr_pages, jiffies_to_msecs(jiffies - start));
1102 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
1103 * because it is sized independent of architecture. Unlike the other zones,
1104 * the starting point for ZONE_MOVABLE is not fixed. It may be different
1105 * in each node depending on the size of each node and how evenly kernelcore
1106 * is distributed. This helper function adjusts the zone ranges
1107 * provided by the architecture for a given node by using the end of the
1108 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
1109 * zones within a node are in order of monotonic increases memory addresses
1111 static void __init adjust_zone_range_for_zone_movable(int nid,
1112 unsigned long zone_type,
1113 unsigned long node_end_pfn,
1114 unsigned long *zone_start_pfn,
1115 unsigned long *zone_end_pfn)
1117 /* Only adjust if ZONE_MOVABLE is on this node */
1118 if (zone_movable_pfn[nid]) {
1119 /* Size ZONE_MOVABLE */
1120 if (zone_type == ZONE_MOVABLE) {
1121 *zone_start_pfn = zone_movable_pfn[nid];
1122 *zone_end_pfn = min(node_end_pfn,
1123 arch_zone_highest_possible_pfn[movable_zone]);
1125 /* Adjust for ZONE_MOVABLE starting within this range */
1126 } else if (!mirrored_kernelcore &&
1127 *zone_start_pfn < zone_movable_pfn[nid] &&
1128 *zone_end_pfn > zone_movable_pfn[nid]) {
1129 *zone_end_pfn = zone_movable_pfn[nid];
1131 /* Check if this whole range is within ZONE_MOVABLE */
1132 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
1133 *zone_start_pfn = *zone_end_pfn;
1138 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
1139 * then all holes in the requested range will be accounted for.
1141 unsigned long __init __absent_pages_in_range(int nid,
1142 unsigned long range_start_pfn,
1143 unsigned long range_end_pfn)
1145 unsigned long nr_absent = range_end_pfn - range_start_pfn;
1146 unsigned long start_pfn, end_pfn;
1149 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
1150 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
1151 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
1152 nr_absent -= end_pfn - start_pfn;
1158 * absent_pages_in_range - Return number of page frames in holes within a range
1159 * @start_pfn: The start PFN to start searching for holes
1160 * @end_pfn: The end PFN to stop searching for holes
1162 * Return: the number of pages frames in memory holes within a range.
1164 unsigned long __init absent_pages_in_range(unsigned long start_pfn,
1165 unsigned long end_pfn)
1167 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
1170 /* Return the number of page frames in holes in a zone on a node */
1171 static unsigned long __init zone_absent_pages_in_node(int nid,
1172 unsigned long zone_type,
1173 unsigned long zone_start_pfn,
1174 unsigned long zone_end_pfn)
1176 unsigned long nr_absent;
1178 /* zone is empty, we don't have any absent pages */
1179 if (zone_start_pfn == zone_end_pfn)
1182 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
1185 * ZONE_MOVABLE handling.
1186 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
1189 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
1190 unsigned long start_pfn, end_pfn;
1191 struct memblock_region *r;
1193 for_each_mem_region(r) {
1194 start_pfn = clamp(memblock_region_memory_base_pfn(r),
1195 zone_start_pfn, zone_end_pfn);
1196 end_pfn = clamp(memblock_region_memory_end_pfn(r),
1197 zone_start_pfn, zone_end_pfn);
1199 if (zone_type == ZONE_MOVABLE &&
1200 memblock_is_mirror(r))
1201 nr_absent += end_pfn - start_pfn;
1203 if (zone_type == ZONE_NORMAL &&
1204 !memblock_is_mirror(r))
1205 nr_absent += end_pfn - start_pfn;
1213 * Return the number of pages a zone spans in a node, including holes
1214 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
1216 static unsigned long __init zone_spanned_pages_in_node(int nid,
1217 unsigned long zone_type,
1218 unsigned long node_start_pfn,
1219 unsigned long node_end_pfn,
1220 unsigned long *zone_start_pfn,
1221 unsigned long *zone_end_pfn)
1223 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
1224 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
1226 /* Get the start and end of the zone */
1227 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
1228 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
1229 adjust_zone_range_for_zone_movable(nid, zone_type, node_end_pfn,
1230 zone_start_pfn, zone_end_pfn);
1232 /* Check that this node has pages within the zone's required range */
1233 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
1236 /* Move the zone boundaries inside the node if necessary */
1237 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
1238 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
1240 /* Return the spanned pages */
1241 return *zone_end_pfn - *zone_start_pfn;
1244 static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat)
1248 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) {
1249 z->zone_start_pfn = 0;
1250 z->spanned_pages = 0;
1251 z->present_pages = 0;
1252 #if defined(CONFIG_MEMORY_HOTPLUG)
1253 z->present_early_pages = 0;
1257 pgdat->node_spanned_pages = 0;
1258 pgdat->node_present_pages = 0;
1259 pr_debug("On node %d totalpages: 0\n", pgdat->node_id);
1262 static void __init calculate_node_totalpages(struct pglist_data *pgdat,
1263 unsigned long node_start_pfn,
1264 unsigned long node_end_pfn)
1266 unsigned long realtotalpages = 0, totalpages = 0;
1269 for (i = 0; i < MAX_NR_ZONES; i++) {
1270 struct zone *zone = pgdat->node_zones + i;
1271 unsigned long zone_start_pfn, zone_end_pfn;
1272 unsigned long spanned, absent;
1273 unsigned long real_size;
1275 spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
1280 absent = zone_absent_pages_in_node(pgdat->node_id, i,
1284 real_size = spanned - absent;
1287 zone->zone_start_pfn = zone_start_pfn;
1289 zone->zone_start_pfn = 0;
1290 zone->spanned_pages = spanned;
1291 zone->present_pages = real_size;
1292 #if defined(CONFIG_MEMORY_HOTPLUG)
1293 zone->present_early_pages = real_size;
1296 totalpages += spanned;
1297 realtotalpages += real_size;
1300 pgdat->node_spanned_pages = totalpages;
1301 pgdat->node_present_pages = realtotalpages;
1302 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
1305 static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
1306 unsigned long present_pages)
1308 unsigned long pages = spanned_pages;
1311 * Provide a more accurate estimation if there are holes within
1312 * the zone and SPARSEMEM is in use. If there are holes within the
1313 * zone, each populated memory region may cost us one or two extra
1314 * memmap pages due to alignment because memmap pages for each
1315 * populated regions may not be naturally aligned on page boundary.
1316 * So the (present_pages >> 4) heuristic is a tradeoff for that.
1318 if (spanned_pages > present_pages + (present_pages >> 4) &&
1319 IS_ENABLED(CONFIG_SPARSEMEM))
1320 pages = present_pages;
1322 return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
1325 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1326 static void pgdat_init_split_queue(struct pglist_data *pgdat)
1328 struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
1330 spin_lock_init(&ds_queue->split_queue_lock);
1331 INIT_LIST_HEAD(&ds_queue->split_queue);
1332 ds_queue->split_queue_len = 0;
1335 static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
1338 #ifdef CONFIG_COMPACTION
1339 static void pgdat_init_kcompactd(struct pglist_data *pgdat)
1341 init_waitqueue_head(&pgdat->kcompactd_wait);
1344 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
1347 static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
1351 pgdat_resize_init(pgdat);
1352 pgdat_kswapd_lock_init(pgdat);
1354 pgdat_init_split_queue(pgdat);
1355 pgdat_init_kcompactd(pgdat);
1357 init_waitqueue_head(&pgdat->kswapd_wait);
1358 init_waitqueue_head(&pgdat->pfmemalloc_wait);
1360 for (i = 0; i < NR_VMSCAN_THROTTLE; i++)
1361 init_waitqueue_head(&pgdat->reclaim_wait[i]);
1363 pgdat_page_ext_init(pgdat);
1364 lruvec_init(&pgdat->__lruvec);
1367 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
1368 unsigned long remaining_pages)
1370 atomic_long_set(&zone->managed_pages, remaining_pages);
1371 zone_set_nid(zone, nid);
1372 zone->name = zone_names[idx];
1373 zone->zone_pgdat = NODE_DATA(nid);
1374 spin_lock_init(&zone->lock);
1375 zone_seqlock_init(zone);
1376 zone_pcp_init(zone);
1379 static void __meminit zone_init_free_lists(struct zone *zone)
1381 unsigned int order, t;
1382 for_each_migratetype_order(order, t) {
1383 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
1384 zone->free_area[order].nr_free = 0;
1387 #ifdef CONFIG_UNACCEPTED_MEMORY
1388 INIT_LIST_HEAD(&zone->unaccepted_pages);
1392 void __meminit init_currently_empty_zone(struct zone *zone,
1393 unsigned long zone_start_pfn,
1396 struct pglist_data *pgdat = zone->zone_pgdat;
1397 int zone_idx = zone_idx(zone) + 1;
1399 if (zone_idx > pgdat->nr_zones)
1400 pgdat->nr_zones = zone_idx;
1402 zone->zone_start_pfn = zone_start_pfn;
1404 mminit_dprintk(MMINIT_TRACE, "memmap_init",
1405 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
1407 (unsigned long)zone_idx(zone),
1408 zone_start_pfn, (zone_start_pfn + size));
1410 zone_init_free_lists(zone);
1411 zone->initialized = 1;
1414 #ifndef CONFIG_SPARSEMEM
1416 * Calculate the size of the zone->blockflags rounded to an unsigned long
1417 * Start by making sure zonesize is a multiple of pageblock_order by rounding
1418 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
1419 * round what is now in bits to nearest long in bits, then return it in
1422 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
1424 unsigned long usemapsize;
1426 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
1427 usemapsize = roundup(zonesize, pageblock_nr_pages);
1428 usemapsize = usemapsize >> pageblock_order;
1429 usemapsize *= NR_PAGEBLOCK_BITS;
1430 usemapsize = roundup(usemapsize, BITS_PER_LONG);
1432 return usemapsize / BITS_PER_BYTE;
1435 static void __ref setup_usemap(struct zone *zone)
1437 unsigned long usemapsize = usemap_size(zone->zone_start_pfn,
1438 zone->spanned_pages);
1439 zone->pageblock_flags = NULL;
1441 zone->pageblock_flags =
1442 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
1444 if (!zone->pageblock_flags)
1445 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
1446 usemapsize, zone->name, zone_to_nid(zone));
1450 static inline void setup_usemap(struct zone *zone) {}
1451 #endif /* CONFIG_SPARSEMEM */
1453 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
1455 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
1456 void __init set_pageblock_order(void)
1458 unsigned int order = MAX_PAGE_ORDER;
1460 /* Check that pageblock_nr_pages has not already been setup */
1461 if (pageblock_order)
1464 /* Don't let pageblocks exceed the maximum allocation granularity. */
1465 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order)
1466 order = HUGETLB_PAGE_ORDER;
1469 * Assume the largest contiguous order of interest is a huge page.
1470 * This value may be variable depending on boot parameters on powerpc.
1472 pageblock_order = order;
1474 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1477 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
1478 * is unused as pageblock_order is set at compile-time. See
1479 * include/linux/pageblock-flags.h for the values of pageblock_order based on
1482 void __init set_pageblock_order(void)
1486 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1489 * Set up the zone data structures
1490 * - init pgdat internals
1491 * - init all zones belonging to this node
1493 * NOTE: this function is only called during memory hotplug
1495 #ifdef CONFIG_MEMORY_HOTPLUG
1496 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
1498 int nid = pgdat->node_id;
1502 pgdat_init_internals(pgdat);
1504 if (pgdat->per_cpu_nodestats == &boot_nodestats)
1505 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);
1508 * Reset the nr_zones, order and highest_zoneidx before reuse.
1509 * Note that kswapd will init kswapd_highest_zoneidx properly
1510 * when it starts in the near future.
1512 pgdat->nr_zones = 0;
1513 pgdat->kswapd_order = 0;
1514 pgdat->kswapd_highest_zoneidx = 0;
1515 pgdat->node_start_pfn = 0;
1516 pgdat->node_present_pages = 0;
1518 for_each_online_cpu(cpu) {
1519 struct per_cpu_nodestat *p;
1521 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
1522 memset(p, 0, sizeof(*p));
1526 * When memory is hot-added, all the memory is in offline state. So
1527 * clear all zones' present_pages and managed_pages because they will
1528 * be updated in online_pages() and offline_pages().
1530 for (z = 0; z < MAX_NR_ZONES; z++) {
1531 struct zone *zone = pgdat->node_zones + z;
1533 zone->present_pages = 0;
1534 zone_init_internals(zone, z, nid, 0);
1540 * Set up the zone data structures:
1541 * - mark all pages reserved
1542 * - mark all memory queues empty
1543 * - clear the memory bitmaps
1545 * NOTE: pgdat should get zeroed by caller.
1546 * NOTE: this function is only called during early init.
1548 static void __init free_area_init_core(struct pglist_data *pgdat)
1551 int nid = pgdat->node_id;
1553 pgdat_init_internals(pgdat);
1554 pgdat->per_cpu_nodestats = &boot_nodestats;
1556 for (j = 0; j < MAX_NR_ZONES; j++) {
1557 struct zone *zone = pgdat->node_zones + j;
1558 unsigned long size, freesize, memmap_pages;
1560 size = zone->spanned_pages;
1561 freesize = zone->present_pages;
1564 * Adjust freesize so that it accounts for how much memory
1565 * is used by this zone for memmap. This affects the watermark
1566 * and per-cpu initialisations
1568 memmap_pages = calc_memmap_size(size, freesize);
1569 if (!is_highmem_idx(j)) {
1570 if (freesize >= memmap_pages) {
1571 freesize -= memmap_pages;
1573 pr_debug(" %s zone: %lu pages used for memmap\n",
1574 zone_names[j], memmap_pages);
1576 pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n",
1577 zone_names[j], memmap_pages, freesize);
1580 /* Account for reserved pages */
1581 if (j == 0 && freesize > dma_reserve) {
1582 freesize -= dma_reserve;
1583 pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve);
1586 if (!is_highmem_idx(j))
1587 nr_kernel_pages += freesize;
1588 /* Charge for highmem memmap if there are enough kernel pages */
1589 else if (nr_kernel_pages > memmap_pages * 2)
1590 nr_kernel_pages -= memmap_pages;
1591 nr_all_pages += freesize;
1594 * Set an approximate value for lowmem here, it will be adjusted
1595 * when the bootmem allocator frees pages into the buddy system.
1596 * And all highmem pages will be managed by the buddy system.
1598 zone_init_internals(zone, j, nid, freesize);
1604 init_currently_empty_zone(zone, zone->zone_start_pfn, size);
1608 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align,
1609 phys_addr_t min_addr, int nid, bool exact_nid)
1614 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr,
1615 MEMBLOCK_ALLOC_ACCESSIBLE,
1618 ptr = memblock_alloc_try_nid_raw(size, align, min_addr,
1619 MEMBLOCK_ALLOC_ACCESSIBLE,
1622 if (ptr && size > 0)
1623 page_init_poison(ptr, size);
1628 #ifdef CONFIG_FLATMEM
1629 static void __init alloc_node_mem_map(struct pglist_data *pgdat)
1631 unsigned long start, offset, size, end;
1634 /* Skip empty nodes */
1635 if (!pgdat->node_spanned_pages)
1638 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
1639 offset = pgdat->node_start_pfn - start;
1641 * The zone's endpoints aren't required to be MAX_PAGE_ORDER
1642 * aligned but the node_mem_map endpoints must be in order
1643 * for the buddy allocator to function correctly.
1645 end = ALIGN(pgdat_end_pfn(pgdat), MAX_ORDER_NR_PAGES);
1646 size = (end - start) * sizeof(struct page);
1647 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT,
1648 pgdat->node_id, false);
1650 panic("Failed to allocate %ld bytes for node %d memory map\n",
1651 size, pgdat->node_id);
1652 pgdat->node_mem_map = map + offset;
1653 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
1654 __func__, pgdat->node_id, (unsigned long)pgdat,
1655 (unsigned long)pgdat->node_mem_map);
1657 /* the global mem_map is just set as node 0's */
1658 if (pgdat == NODE_DATA(0)) {
1659 mem_map = NODE_DATA(0)->node_mem_map;
1660 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
1666 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { }
1667 #endif /* CONFIG_FLATMEM */
1670 * get_pfn_range_for_nid - Return the start and end page frames for a node
1671 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
1672 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
1673 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
1675 * It returns the start and end page frame of a node based on information
1676 * provided by memblock_set_node(). If called for a node
1677 * with no available memory, the start and end PFNs will be 0.
1679 void __init get_pfn_range_for_nid(unsigned int nid,
1680 unsigned long *start_pfn, unsigned long *end_pfn)
1682 unsigned long this_start_pfn, this_end_pfn;
1688 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
1689 *start_pfn = min(*start_pfn, this_start_pfn);
1690 *end_pfn = max(*end_pfn, this_end_pfn);
1693 if (*start_pfn == -1UL)
1697 static void __init free_area_init_node(int nid)
1699 pg_data_t *pgdat = NODE_DATA(nid);
1700 unsigned long start_pfn = 0;
1701 unsigned long end_pfn = 0;
1703 /* pg_data_t should be reset to zero when it's allocated */
1704 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
1706 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
1708 pgdat->node_id = nid;
1709 pgdat->node_start_pfn = start_pfn;
1710 pgdat->per_cpu_nodestats = NULL;
1712 if (start_pfn != end_pfn) {
1713 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
1714 (u64)start_pfn << PAGE_SHIFT,
1715 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
1717 calculate_node_totalpages(pgdat, start_pfn, end_pfn);
1719 pr_info("Initmem setup node %d as memoryless\n", nid);
1721 reset_memoryless_node_totalpages(pgdat);
1724 alloc_node_mem_map(pgdat);
1725 pgdat_set_deferred_range(pgdat);
1727 free_area_init_core(pgdat);
1728 lru_gen_init_pgdat(pgdat);
1731 /* Any regular or high memory on that node ? */
1732 static void __init check_for_memory(pg_data_t *pgdat)
1734 enum zone_type zone_type;
1736 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
1737 struct zone *zone = &pgdat->node_zones[zone_type];
1738 if (populated_zone(zone)) {
1739 if (IS_ENABLED(CONFIG_HIGHMEM))
1740 node_set_state(pgdat->node_id, N_HIGH_MEMORY);
1741 if (zone_type <= ZONE_NORMAL)
1742 node_set_state(pgdat->node_id, N_NORMAL_MEMORY);
1748 #if MAX_NUMNODES > 1
1750 * Figure out the number of possible node ids.
1752 void __init setup_nr_node_ids(void)
1754 unsigned int highest;
1756 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
1757 nr_node_ids = highest + 1;
1762 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
1763 * such cases we allow max_zone_pfn sorted in the descending order
1765 static bool arch_has_descending_max_zone_pfns(void)
1767 return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40);
1771 * free_area_init - Initialise all pg_data_t and zone data
1772 * @max_zone_pfn: an array of max PFNs for each zone
1774 * This will call free_area_init_node() for each active node in the system.
1775 * Using the page ranges provided by memblock_set_node(), the size of each
1776 * zone in each node and their holes is calculated. If the maximum PFN
1777 * between two adjacent zones match, it is assumed that the zone is empty.
1778 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
1779 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
1780 * starts where the previous one ended. For example, ZONE_DMA32 starts
1781 * at arch_max_dma_pfn.
1783 void __init free_area_init(unsigned long *max_zone_pfn)
1785 unsigned long start_pfn, end_pfn;
1789 /* Record where the zone boundaries are */
1790 memset(arch_zone_lowest_possible_pfn, 0,
1791 sizeof(arch_zone_lowest_possible_pfn));
1792 memset(arch_zone_highest_possible_pfn, 0,
1793 sizeof(arch_zone_highest_possible_pfn));
1795 start_pfn = PHYS_PFN(memblock_start_of_DRAM());
1796 descending = arch_has_descending_max_zone_pfns();
1798 for (i = 0; i < MAX_NR_ZONES; i++) {
1800 zone = MAX_NR_ZONES - i - 1;
1804 if (zone == ZONE_MOVABLE)
1807 end_pfn = max(max_zone_pfn[zone], start_pfn);
1808 arch_zone_lowest_possible_pfn[zone] = start_pfn;
1809 arch_zone_highest_possible_pfn[zone] = end_pfn;
1811 start_pfn = end_pfn;
1814 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
1815 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
1816 find_zone_movable_pfns_for_nodes();
1818 /* Print out the zone ranges */
1819 pr_info("Zone ranges:\n");
1820 for (i = 0; i < MAX_NR_ZONES; i++) {
1821 if (i == ZONE_MOVABLE)
1823 pr_info(" %-8s ", zone_names[i]);
1824 if (arch_zone_lowest_possible_pfn[i] ==
1825 arch_zone_highest_possible_pfn[i])
1828 pr_cont("[mem %#018Lx-%#018Lx]\n",
1829 (u64)arch_zone_lowest_possible_pfn[i]
1831 ((u64)arch_zone_highest_possible_pfn[i]
1832 << PAGE_SHIFT) - 1);
1835 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
1836 pr_info("Movable zone start for each node\n");
1837 for (i = 0; i < MAX_NUMNODES; i++) {
1838 if (zone_movable_pfn[i])
1839 pr_info(" Node %d: %#018Lx\n", i,
1840 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
1844 * Print out the early node map, and initialize the
1845 * subsection-map relative to active online memory ranges to
1846 * enable future "sub-section" extensions of the memory map.
1848 pr_info("Early memory node ranges\n");
1849 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
1850 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
1851 (u64)start_pfn << PAGE_SHIFT,
1852 ((u64)end_pfn << PAGE_SHIFT) - 1);
1853 subsection_map_init(start_pfn, end_pfn - start_pfn);
1856 /* Initialise every node */
1857 mminit_verify_pageflags_layout();
1858 setup_nr_node_ids();
1859 set_pageblock_order();
1861 for_each_node(nid) {
1864 if (!node_online(nid)) {
1865 /* Allocator not initialized yet */
1866 pgdat = arch_alloc_nodedata(nid);
1868 panic("Cannot allocate %zuB for node %d.\n",
1869 sizeof(*pgdat), nid);
1870 arch_refresh_nodedata(nid, pgdat);
1871 free_area_init_node(nid);
1874 * We do not want to confuse userspace by sysfs
1875 * files/directories for node without any memory
1876 * attached to it, so this node is not marked as
1877 * N_MEMORY and not marked online so that no sysfs
1878 * hierarchy will be created via register_one_node for
1879 * it. The pgdat will get fully initialized by
1880 * hotadd_init_pgdat() when memory is hotplugged into
1886 pgdat = NODE_DATA(nid);
1887 free_area_init_node(nid);
1889 /* Any memory on that node */
1890 if (pgdat->node_present_pages)
1891 node_set_state(nid, N_MEMORY);
1892 check_for_memory(pgdat);
1897 /* disable hash distribution for systems with a single node */
1902 * node_map_pfn_alignment - determine the maximum internode alignment
1904 * This function should be called after node map is populated and sorted.
1905 * It calculates the maximum power of two alignment which can distinguish
1908 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
1909 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
1910 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
1911 * shifted, 1GiB is enough and this function will indicate so.
1913 * This is used to test whether pfn -> nid mapping of the chosen memory
1914 * model has fine enough granularity to avoid incorrect mapping for the
1915 * populated node map.
1917 * Return: the determined alignment in pfn's. 0 if there is no alignment
1918 * requirement (single node).
1920 unsigned long __init node_map_pfn_alignment(void)
1922 unsigned long accl_mask = 0, last_end = 0;
1923 unsigned long start, end, mask;
1924 int last_nid = NUMA_NO_NODE;
1927 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
1928 if (!start || last_nid < 0 || last_nid == nid) {
1935 * Start with a mask granular enough to pin-point to the
1936 * start pfn and tick off bits one-by-one until it becomes
1937 * too coarse to separate the current node from the last.
1939 mask = ~((1 << __ffs(start)) - 1);
1940 while (mask && last_end <= (start & (mask << 1)))
1943 /* accumulate all internode masks */
1947 /* convert mask to number of pages */
1948 return ~accl_mask + 1;
1951 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1952 static void __init deferred_free_range(unsigned long pfn,
1953 unsigned long nr_pages)
1961 page = pfn_to_page(pfn);
1963 /* Free a large naturally-aligned chunk if possible */
1964 if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) {
1965 for (i = 0; i < nr_pages; i += pageblock_nr_pages)
1966 set_pageblock_migratetype(page + i, MIGRATE_MOVABLE);
1967 __free_pages_core(page, MAX_PAGE_ORDER);
1971 /* Accept chunks smaller than MAX_PAGE_ORDER upfront */
1972 accept_memory(PFN_PHYS(pfn), PFN_PHYS(pfn + nr_pages));
1974 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1975 if (pageblock_aligned(pfn))
1976 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1977 __free_pages_core(page, 0);
1981 /* Completion tracking for deferred_init_memmap() threads */
1982 static atomic_t pgdat_init_n_undone __initdata;
1983 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1985 static inline void __init pgdat_init_report_one_done(void)
1987 if (atomic_dec_and_test(&pgdat_init_n_undone))
1988 complete(&pgdat_init_all_done_comp);
1992 * Returns true if page needs to be initialized or freed to buddy allocator.
1994 * We check if a current MAX_PAGE_ORDER block is valid by only checking the
1995 * validity of the head pfn.
1997 static inline bool __init deferred_pfn_valid(unsigned long pfn)
1999 if (IS_MAX_ORDER_ALIGNED(pfn) && !pfn_valid(pfn))
2005 * Free pages to buddy allocator. Try to free aligned pages in
2006 * MAX_ORDER_NR_PAGES sizes.
2008 static void __init deferred_free_pages(unsigned long pfn,
2009 unsigned long end_pfn)
2011 unsigned long nr_free = 0;
2013 for (; pfn < end_pfn; pfn++) {
2014 if (!deferred_pfn_valid(pfn)) {
2015 deferred_free_range(pfn - nr_free, nr_free);
2017 } else if (IS_MAX_ORDER_ALIGNED(pfn)) {
2018 deferred_free_range(pfn - nr_free, nr_free);
2024 /* Free the last block of pages to allocator */
2025 deferred_free_range(pfn - nr_free, nr_free);
2029 * Initialize struct pages. We minimize pfn page lookups and scheduler checks
2030 * by performing it only once every MAX_ORDER_NR_PAGES.
2031 * Return number of pages initialized.
2033 static unsigned long __init deferred_init_pages(struct zone *zone,
2035 unsigned long end_pfn)
2037 int nid = zone_to_nid(zone);
2038 unsigned long nr_pages = 0;
2039 int zid = zone_idx(zone);
2040 struct page *page = NULL;
2042 for (; pfn < end_pfn; pfn++) {
2043 if (!deferred_pfn_valid(pfn)) {
2046 } else if (!page || IS_MAX_ORDER_ALIGNED(pfn)) {
2047 page = pfn_to_page(pfn);
2051 __init_single_page(page, pfn, zid, nid);
2058 * This function is meant to pre-load the iterator for the zone init.
2059 * Specifically it walks through the ranges until we are caught up to the
2060 * first_init_pfn value and exits there. If we never encounter the value we
2061 * return false indicating there are no valid ranges left.
2064 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
2065 unsigned long *spfn, unsigned long *epfn,
2066 unsigned long first_init_pfn)
2071 * Start out by walking through the ranges in this zone that have
2072 * already been initialized. We don't need to do anything with them
2073 * so we just need to flush them out of the system.
2075 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
2076 if (*epfn <= first_init_pfn)
2078 if (*spfn < first_init_pfn)
2079 *spfn = first_init_pfn;
2088 * Initialize and free pages. We do it in two loops: first we initialize
2089 * struct page, then free to buddy allocator, because while we are
2090 * freeing pages we can access pages that are ahead (computing buddy
2091 * page in __free_one_page()).
2093 * In order to try and keep some memory in the cache we have the loop
2094 * broken along max page order boundaries. This way we will not cause
2095 * any issues with the buddy page computation.
2097 static unsigned long __init
2098 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
2099 unsigned long *end_pfn)
2101 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
2102 unsigned long spfn = *start_pfn, epfn = *end_pfn;
2103 unsigned long nr_pages = 0;
2106 /* First we loop through and initialize the page values */
2107 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
2110 if (mo_pfn <= *start_pfn)
2113 t = min(mo_pfn, *end_pfn);
2114 nr_pages += deferred_init_pages(zone, *start_pfn, t);
2116 if (mo_pfn < *end_pfn) {
2117 *start_pfn = mo_pfn;
2122 /* Reset values and now loop through freeing pages as needed */
2125 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
2131 t = min(mo_pfn, epfn);
2132 deferred_free_pages(spfn, t);
2142 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2145 unsigned long spfn, epfn;
2146 struct zone *zone = arg;
2149 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
2152 * Initialize and free pages in MAX_PAGE_ORDER sized increments so that
2153 * we can avoid introducing any issues with the buddy allocator.
2155 while (spfn < end_pfn) {
2156 deferred_init_maxorder(&i, zone, &spfn, &epfn);
2161 /* An arch may override for more concurrency. */
2163 deferred_page_init_max_threads(const struct cpumask *node_cpumask)
2168 /* Initialise remaining memory on a node */
2169 static int __init deferred_init_memmap(void *data)
2171 pg_data_t *pgdat = data;
2172 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2173 unsigned long spfn = 0, epfn = 0;
2174 unsigned long first_init_pfn, flags;
2175 unsigned long start = jiffies;
2177 int zid, max_threads;
2180 /* Bind memory initialisation thread to a local node if possible */
2181 if (!cpumask_empty(cpumask))
2182 set_cpus_allowed_ptr(current, cpumask);
2184 pgdat_resize_lock(pgdat, &flags);
2185 first_init_pfn = pgdat->first_deferred_pfn;
2186 if (first_init_pfn == ULONG_MAX) {
2187 pgdat_resize_unlock(pgdat, &flags);
2188 pgdat_init_report_one_done();
2192 /* Sanity check boundaries */
2193 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
2194 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
2195 pgdat->first_deferred_pfn = ULONG_MAX;
2198 * Once we unlock here, the zone cannot be grown anymore, thus if an
2199 * interrupt thread must allocate this early in boot, zone must be
2200 * pre-grown prior to start of deferred page initialization.
2202 pgdat_resize_unlock(pgdat, &flags);
2204 /* Only the highest zone is deferred so find it */
2205 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2206 zone = pgdat->node_zones + zid;
2207 if (first_init_pfn < zone_end_pfn(zone))
2211 /* If the zone is empty somebody else may have cleared out the zone */
2212 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2216 max_threads = deferred_page_init_max_threads(cpumask);
2218 while (spfn < epfn) {
2219 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
2220 struct padata_mt_job job = {
2221 .thread_fn = deferred_init_memmap_chunk,
2224 .size = epfn_align - spfn,
2225 .align = PAGES_PER_SECTION,
2226 .min_chunk = PAGES_PER_SECTION,
2227 .max_threads = max_threads,
2230 padata_do_multithreaded(&job);
2231 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2235 /* Sanity check that the next zone really is unpopulated */
2236 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
2238 pr_info("node %d deferred pages initialised in %ums\n",
2239 pgdat->node_id, jiffies_to_msecs(jiffies - start));
2241 pgdat_init_report_one_done();
2246 * If this zone has deferred pages, try to grow it by initializing enough
2247 * deferred pages to satisfy the allocation specified by order, rounded up to
2248 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
2249 * of SECTION_SIZE bytes by initializing struct pages in increments of
2250 * PAGES_PER_SECTION * sizeof(struct page) bytes.
2252 * Return true when zone was grown, otherwise return false. We return true even
2253 * when we grow less than requested, to let the caller decide if there are
2254 * enough pages to satisfy the allocation.
2256 * Note: We use noinline because this function is needed only during boot, and
2257 * it is called from a __ref function _deferred_grow_zone. This way we are
2258 * making sure that it is not inlined into permanent text section.
2260 bool __init deferred_grow_zone(struct zone *zone, unsigned int order)
2262 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
2263 pg_data_t *pgdat = zone->zone_pgdat;
2264 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
2265 unsigned long spfn, epfn, flags;
2266 unsigned long nr_pages = 0;
2269 /* Only the last zone may have deferred pages */
2270 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2273 pgdat_resize_lock(pgdat, &flags);
2276 * If someone grew this zone while we were waiting for spinlock, return
2277 * true, as there might be enough pages already.
2279 if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2280 pgdat_resize_unlock(pgdat, &flags);
2284 /* If the zone is empty somebody else may have cleared out the zone */
2285 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2286 first_deferred_pfn)) {
2287 pgdat->first_deferred_pfn = ULONG_MAX;
2288 pgdat_resize_unlock(pgdat, &flags);
2289 /* Retry only once. */
2290 return first_deferred_pfn != ULONG_MAX;
2294 * Initialize and free pages in MAX_PAGE_ORDER sized increments so
2295 * that we can avoid introducing any issues with the buddy
2298 while (spfn < epfn) {
2299 /* update our first deferred PFN for this section */
2300 first_deferred_pfn = spfn;
2302 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2303 touch_nmi_watchdog();
2305 /* We should only stop along section boundaries */
2306 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2309 /* If our quota has been met we can stop here */
2310 if (nr_pages >= nr_pages_needed)
2314 pgdat->first_deferred_pfn = spfn;
2315 pgdat_resize_unlock(pgdat, &flags);
2317 return nr_pages > 0;
2320 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
2323 void __init init_cma_reserved_pageblock(struct page *page)
2325 unsigned i = pageblock_nr_pages;
2326 struct page *p = page;
2329 __ClearPageReserved(p);
2330 set_page_count(p, 0);
2333 set_pageblock_migratetype(page, MIGRATE_CMA);
2334 set_page_refcounted(page);
2335 __free_pages(page, pageblock_order);
2337 adjust_managed_page_count(page, pageblock_nr_pages);
2338 page_zone(page)->cma_pages += pageblock_nr_pages;
2342 void set_zone_contiguous(struct zone *zone)
2344 unsigned long block_start_pfn = zone->zone_start_pfn;
2345 unsigned long block_end_pfn;
2347 block_end_pfn = pageblock_end_pfn(block_start_pfn);
2348 for (; block_start_pfn < zone_end_pfn(zone);
2349 block_start_pfn = block_end_pfn,
2350 block_end_pfn += pageblock_nr_pages) {
2352 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
2354 if (!__pageblock_pfn_to_page(block_start_pfn,
2355 block_end_pfn, zone))
2360 /* We confirm that there is no hole */
2361 zone->contiguous = true;
2364 void __init page_alloc_init_late(void)
2369 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
2371 /* There will be num_node_state(N_MEMORY) threads */
2372 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
2373 for_each_node_state(nid, N_MEMORY) {
2374 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2377 /* Block until all are initialised */
2378 wait_for_completion(&pgdat_init_all_done_comp);
2381 * We initialized the rest of the deferred pages. Permanently disable
2382 * on-demand struct page initialization.
2384 static_branch_disable(&deferred_pages);
2386 /* Reinit limits that are based on free pages after the kernel is up */
2387 files_maxfiles_init();
2392 /* Discard memblock private memory */
2395 for_each_node_state(nid, N_MEMORY)
2396 shuffle_free_memory(NODE_DATA(nid));
2398 for_each_populated_zone(zone)
2399 set_zone_contiguous(zone);
2401 /* Initialize page ext after all struct pages are initialized. */
2402 if (deferred_struct_pages)
2405 page_alloc_sysctl_init();
2408 #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
2410 * Returns the number of pages that arch has reserved but
2411 * is not known to alloc_large_system_hash().
2413 static unsigned long __init arch_reserved_kernel_pages(void)
2420 * Adaptive scale is meant to reduce sizes of hash tables on large memory
2421 * machines. As memory size is increased the scale is also increased but at
2422 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
2423 * quadruples the scale is increased by one, which means the size of hash table
2424 * only doubles, instead of quadrupling as well.
2425 * Because 32-bit systems cannot have large physical memory, where this scaling
2426 * makes sense, it is disabled on such platforms.
2428 #if __BITS_PER_LONG > 32
2429 #define ADAPT_SCALE_BASE (64ul << 30)
2430 #define ADAPT_SCALE_SHIFT 2
2431 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
2435 * allocate a large system hash table from bootmem
2436 * - it is assumed that the hash table must contain an exact power-of-2
2437 * quantity of entries
2438 * - limit is the number of hash buckets, not the total allocation size
2440 void *__init alloc_large_system_hash(const char *tablename,
2441 unsigned long bucketsize,
2442 unsigned long numentries,
2445 unsigned int *_hash_shift,
2446 unsigned int *_hash_mask,
2447 unsigned long low_limit,
2448 unsigned long high_limit)
2450 unsigned long long max = high_limit;
2451 unsigned long log2qty, size;
2457 /* allow the kernel cmdline to have a say */
2459 /* round applicable memory size up to nearest megabyte */
2460 numentries = nr_kernel_pages;
2461 numentries -= arch_reserved_kernel_pages();
2463 /* It isn't necessary when PAGE_SIZE >= 1MB */
2464 if (PAGE_SIZE < SZ_1M)
2465 numentries = round_up(numentries, SZ_1M / PAGE_SIZE);
2467 #if __BITS_PER_LONG > 32
2469 unsigned long adapt;
2471 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
2472 adapt <<= ADAPT_SCALE_SHIFT)
2477 /* limit to 1 bucket per 2^scale bytes of low memory */
2478 if (scale > PAGE_SHIFT)
2479 numentries >>= (scale - PAGE_SHIFT);
2481 numentries <<= (PAGE_SHIFT - scale);
2483 if (unlikely((numentries * bucketsize) < PAGE_SIZE))
2484 numentries = PAGE_SIZE / bucketsize;
2486 numentries = roundup_pow_of_two(numentries);
2488 /* limit allocation size to 1/16 total memory by default */
2490 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
2491 do_div(max, bucketsize);
2493 max = min(max, 0x80000000ULL);
2495 if (numentries < low_limit)
2496 numentries = low_limit;
2497 if (numentries > max)
2500 log2qty = ilog2(numentries);
2502 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
2505 size = bucketsize << log2qty;
2506 if (flags & HASH_EARLY) {
2507 if (flags & HASH_ZERO)
2508 table = memblock_alloc(size, SMP_CACHE_BYTES);
2510 table = memblock_alloc_raw(size,
2512 } else if (get_order(size) > MAX_PAGE_ORDER || hashdist) {
2513 table = vmalloc_huge(size, gfp_flags);
2516 huge = is_vm_area_hugepages(table);
2519 * If bucketsize is not a power-of-two, we may free
2520 * some pages at the end of hash table which
2521 * alloc_pages_exact() automatically does
2523 table = alloc_pages_exact(size, gfp_flags);
2524 kmemleak_alloc(table, size, 1, gfp_flags);
2526 } while (!table && size > PAGE_SIZE && --log2qty);
2529 panic("Failed to allocate %s hash table\n", tablename);
2531 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
2532 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
2533 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");
2536 *_hash_shift = log2qty;
2538 *_hash_mask = (1 << log2qty) - 1;
2544 * set_dma_reserve - set the specified number of pages reserved in the first zone
2545 * @new_dma_reserve: The number of pages to mark reserved
2547 * The per-cpu batchsize and zone watermarks are determined by managed_pages.
2548 * In the DMA zone, a significant percentage may be consumed by kernel image
2549 * and other unfreeable allocations which can skew the watermarks badly. This
2550 * function may optionally be used to account for unfreeable pages in the
2551 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
2552 * smaller per-cpu batchsize.
2554 void __init set_dma_reserve(unsigned long new_dma_reserve)
2556 dma_reserve = new_dma_reserve;
2559 void __init memblock_free_pages(struct page *page, unsigned long pfn,
2563 if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) {
2564 int nid = early_pfn_to_nid(pfn);
2566 if (!early_page_initialised(pfn, nid))
2570 if (!kmsan_memblock_free_pages(page, order)) {
2571 /* KMSAN will take care of these pages. */
2574 __free_pages_core(page, order);
2577 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
2578 EXPORT_SYMBOL(init_on_alloc);
2580 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
2581 EXPORT_SYMBOL(init_on_free);
2583 static bool _init_on_alloc_enabled_early __read_mostly
2584 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
2585 static int __init early_init_on_alloc(char *buf)
2588 return kstrtobool(buf, &_init_on_alloc_enabled_early);
2590 early_param("init_on_alloc", early_init_on_alloc);
2592 static bool _init_on_free_enabled_early __read_mostly
2593 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
2594 static int __init early_init_on_free(char *buf)
2596 return kstrtobool(buf, &_init_on_free_enabled_early);
2598 early_param("init_on_free", early_init_on_free);
2600 DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
2603 * Enable static keys related to various memory debugging and hardening options.
2604 * Some override others, and depend on early params that are evaluated in the
2605 * order of appearance. So we need to first gather the full picture of what was
2606 * enabled, and then make decisions.
2608 static void __init mem_debugging_and_hardening_init(void)
2610 bool page_poisoning_requested = false;
2611 bool want_check_pages = false;
2613 #ifdef CONFIG_PAGE_POISONING
2615 * Page poisoning is debug page alloc for some arches. If
2616 * either of those options are enabled, enable poisoning.
2618 if (page_poisoning_enabled() ||
2619 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
2620 debug_pagealloc_enabled())) {
2621 static_branch_enable(&_page_poisoning_enabled);
2622 page_poisoning_requested = true;
2623 want_check_pages = true;
2627 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
2628 page_poisoning_requested) {
2629 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
2630 "will take precedence over init_on_alloc and init_on_free\n");
2631 _init_on_alloc_enabled_early = false;
2632 _init_on_free_enabled_early = false;
2635 if (_init_on_alloc_enabled_early) {
2636 want_check_pages = true;
2637 static_branch_enable(&init_on_alloc);
2639 static_branch_disable(&init_on_alloc);
2642 if (_init_on_free_enabled_early) {
2643 want_check_pages = true;
2644 static_branch_enable(&init_on_free);
2646 static_branch_disable(&init_on_free);
2649 if (IS_ENABLED(CONFIG_KMSAN) &&
2650 (_init_on_alloc_enabled_early || _init_on_free_enabled_early))
2651 pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n");
2653 #ifdef CONFIG_DEBUG_PAGEALLOC
2654 if (debug_pagealloc_enabled()) {
2655 want_check_pages = true;
2656 static_branch_enable(&_debug_pagealloc_enabled);
2658 if (debug_guardpage_minorder())
2659 static_branch_enable(&_debug_guardpage_enabled);
2664 * Any page debugging or hardening option also enables sanity checking
2665 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's
2668 if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages)
2669 static_branch_enable(&check_pages_enabled);
2672 /* Report memory auto-initialization states for this boot. */
2673 static void __init report_meminit(void)
2677 if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN))
2678 stack = "all(pattern)";
2679 else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO))
2680 stack = "all(zero)";
2681 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL))
2682 stack = "byref_all(zero)";
2683 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF))
2684 stack = "byref(zero)";
2685 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER))
2686 stack = "__user(zero)";
2690 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n",
2691 stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off",
2692 want_init_on_free() ? "on" : "off");
2693 if (want_init_on_free())
2694 pr_info("mem auto-init: clearing system memory may take some time...\n");
2697 static void __init mem_init_print_info(void)
2699 unsigned long physpages, codesize, datasize, rosize, bss_size;
2700 unsigned long init_code_size, init_data_size;
2702 physpages = get_num_physpages();
2703 codesize = _etext - _stext;
2704 datasize = _edata - _sdata;
2705 rosize = __end_rodata - __start_rodata;
2706 bss_size = __bss_stop - __bss_start;
2707 init_data_size = __init_end - __init_begin;
2708 init_code_size = _einittext - _sinittext;
2711 * Detect special cases and adjust section sizes accordingly:
2712 * 1) .init.* may be embedded into .data sections
2713 * 2) .init.text.* may be out of [__init_begin, __init_end],
2714 * please refer to arch/tile/kernel/vmlinux.lds.S.
2715 * 3) .rodata.* may be embedded into .text or .data sections.
2717 #define adj_init_size(start, end, size, pos, adj) \
2719 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
2723 adj_init_size(__init_begin, __init_end, init_data_size,
2724 _sinittext, init_code_size);
2725 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
2726 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
2727 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
2728 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
2730 #undef adj_init_size
2732 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
2733 #ifdef CONFIG_HIGHMEM
2737 K(nr_free_pages()), K(physpages),
2738 codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K,
2739 (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K,
2740 K(physpages - totalram_pages() - totalcma_pages),
2742 #ifdef CONFIG_HIGHMEM
2743 , K(totalhigh_pages())
2749 * Set up kernel memory allocators
2751 void __init mm_core_init(void)
2753 /* Initializations relying on SMP setup */
2754 build_all_zonelists(NULL);
2755 page_alloc_init_cpuhp();
2758 * page_ext requires contiguous pages,
2759 * bigger than MAX_PAGE_ORDER unless SPARSEMEM.
2761 page_ext_init_flatmem();
2762 mem_debugging_and_hardening_init();
2763 kfence_alloc_pool_and_metadata();
2765 kmsan_init_shadow();
2766 stack_depot_early_init();
2768 mem_init_print_info();
2771 * page_owner must be initialized after buddy is ready, and also after
2772 * slab is ready so that stack_depot_init() works properly
2774 page_ext_init_flatmem_late();
2776 ptlock_cache_init();
2777 pgtable_cache_init();
2778 debug_objects_mem_init();
2780 /* If no deferred init page_ext now, as vmap is fully initialized */
2781 if (!deferred_struct_pages)
2783 /* Should be run before the first non-init thread is created */
2785 /* Should be run after espfix64 is set up. */
2787 kmsan_init_runtime();