1 // SPDX-License-Identifier: GPL-2.0-only
3 * mm_init.c - Memory initialisation verification and debugging
5 * Copyright 2008 IBM Corporation, 2008
6 * Author Mel Gorman <mel@csn.ul.ie>
9 #include <linux/kernel.h>
10 #include <linux/init.h>
11 #include <linux/kobject.h>
12 #include <linux/export.h>
13 #include <linux/memory.h>
14 #include <linux/notifier.h>
15 #include <linux/sched.h>
16 #include <linux/mman.h>
17 #include <linux/memblock.h>
18 #include <linux/page-isolation.h>
19 #include <linux/padata.h>
20 #include <linux/nmi.h>
21 #include <linux/buffer_head.h>
22 #include <linux/kmemleak.h>
23 #include <linux/kfence.h>
24 #include <linux/page_ext.h>
25 #include <linux/pti.h>
26 #include <linux/pgtable.h>
27 #include <linux/stackdepot.h>
28 #include <linux/swap.h>
29 #include <linux/cma.h>
30 #include <linux/crash_dump.h>
35 #include <asm/setup.h>
37 #ifdef CONFIG_DEBUG_MEMORY_INIT
38 int __meminitdata mminit_loglevel;
40 /* The zonelists are simply reported, validation is manual. */
41 void __init mminit_verify_zonelist(void)
45 if (mminit_loglevel < MMINIT_VERIFY)
48 for_each_online_node(nid) {
49 pg_data_t *pgdat = NODE_DATA(nid);
52 struct zonelist *zonelist;
53 int i, listid, zoneid;
55 BUILD_BUG_ON(MAX_ZONELISTS > 2);
56 for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) {
58 /* Identify the zone and nodelist */
59 zoneid = i % MAX_NR_ZONES;
60 listid = i / MAX_NR_ZONES;
61 zonelist = &pgdat->node_zonelists[listid];
62 zone = &pgdat->node_zones[zoneid];
63 if (!populated_zone(zone))
66 /* Print information about the zonelist */
67 printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ",
68 listid > 0 ? "thisnode" : "general", nid,
71 /* Iterate the zonelist */
72 for_each_zone_zonelist(zone, z, zonelist, zoneid)
73 pr_cont("%d:%s ", zone_to_nid(zone), zone->name);
79 void __init mminit_verify_pageflags_layout(void)
82 unsigned long or_mask, add_mask;
84 shift = BITS_PER_LONG;
85 width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH
86 - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH;
87 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths",
88 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n",
97 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts",
98 "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n",
104 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts",
105 "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n",
106 (unsigned long)SECTIONS_PGSHIFT,
107 (unsigned long)NODES_PGSHIFT,
108 (unsigned long)ZONES_PGSHIFT,
109 (unsigned long)LAST_CPUPID_PGSHIFT,
110 (unsigned long)KASAN_TAG_PGSHIFT);
111 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid",
112 "Node/Zone ID: %lu -> %lu\n",
113 (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT),
114 (unsigned long)ZONEID_PGOFF);
115 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage",
116 "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n",
117 shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0);
118 #ifdef NODE_NOT_IN_PAGE_FLAGS
119 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
120 "Node not in page flags");
122 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
123 mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags",
124 "Last cpupid not in page flags");
127 if (SECTIONS_WIDTH) {
128 shift -= SECTIONS_WIDTH;
129 BUG_ON(shift != SECTIONS_PGSHIFT);
132 shift -= NODES_WIDTH;
133 BUG_ON(shift != NODES_PGSHIFT);
136 shift -= ZONES_WIDTH;
137 BUG_ON(shift != ZONES_PGSHIFT);
140 /* Check for bitmask overlaps */
141 or_mask = (ZONES_MASK << ZONES_PGSHIFT) |
142 (NODES_MASK << NODES_PGSHIFT) |
143 (SECTIONS_MASK << SECTIONS_PGSHIFT);
144 add_mask = (ZONES_MASK << ZONES_PGSHIFT) +
145 (NODES_MASK << NODES_PGSHIFT) +
146 (SECTIONS_MASK << SECTIONS_PGSHIFT);
147 BUG_ON(or_mask != add_mask);
150 static __init int set_mminit_loglevel(char *str)
152 get_option(&str, &mminit_loglevel);
155 early_param("mminit_loglevel", set_mminit_loglevel);
156 #endif /* CONFIG_DEBUG_MEMORY_INIT */
158 struct kobject *mm_kobj;
161 s32 vm_committed_as_batch = 32;
163 void mm_compute_batch(int overcommit_policy)
166 s32 nr = num_present_cpus();
167 s32 batch = max_t(s32, nr*2, 32);
168 unsigned long ram_pages = totalram_pages();
171 * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of
172 * (total memory/#cpus), and lift it to 25% for other policies
173 * to easy the possible lock contention for percpu_counter
174 * vm_committed_as, while the max limit is INT_MAX
176 if (overcommit_policy == OVERCOMMIT_NEVER)
177 memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX);
179 memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX);
181 vm_committed_as_batch = max_t(s32, memsized_batch, batch);
184 static int __meminit mm_compute_batch_notifier(struct notifier_block *self,
185 unsigned long action, void *arg)
190 mm_compute_batch(sysctl_overcommit_memory);
198 static int __init mm_compute_batch_init(void)
200 mm_compute_batch(sysctl_overcommit_memory);
201 hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI);
205 __initcall(mm_compute_batch_init);
209 static int __init mm_sysfs_init(void)
211 mm_kobj = kobject_create_and_add("mm", kernel_kobj);
217 postcore_initcall(mm_sysfs_init);
219 static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
220 static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
221 static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
223 static unsigned long required_kernelcore __initdata;
224 static unsigned long required_kernelcore_percent __initdata;
225 static unsigned long required_movablecore __initdata;
226 static unsigned long required_movablecore_percent __initdata;
228 static unsigned long nr_kernel_pages __initdata;
229 static unsigned long nr_all_pages __initdata;
231 static bool deferred_struct_pages __meminitdata;
233 static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
235 static int __init cmdline_parse_core(char *p, unsigned long *core,
236 unsigned long *percent)
238 unsigned long long coremem;
244 /* Value may be a percentage of total memory, otherwise bytes */
245 coremem = simple_strtoull(p, &endptr, 0);
246 if (*endptr == '%') {
247 /* Paranoid check for percent values greater than 100 */
248 WARN_ON(coremem > 100);
252 coremem = memparse(p, &p);
253 /* Paranoid check that UL is enough for the coremem value */
254 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
256 *core = coremem >> PAGE_SHIFT;
262 bool mirrored_kernelcore __initdata_memblock;
265 * kernelcore=size sets the amount of memory for use for allocations that
266 * cannot be reclaimed or migrated.
268 static int __init cmdline_parse_kernelcore(char *p)
270 /* parse kernelcore=mirror */
271 if (parse_option_str(p, "mirror")) {
272 mirrored_kernelcore = true;
276 return cmdline_parse_core(p, &required_kernelcore,
277 &required_kernelcore_percent);
279 early_param("kernelcore", cmdline_parse_kernelcore);
282 * movablecore=size sets the amount of memory for use for allocations that
283 * can be reclaimed or migrated.
285 static int __init cmdline_parse_movablecore(char *p)
287 return cmdline_parse_core(p, &required_movablecore,
288 &required_movablecore_percent);
290 early_param("movablecore", cmdline_parse_movablecore);
293 * early_calculate_totalpages()
294 * Sum pages in active regions for movable zone.
295 * Populate N_MEMORY for calculating usable_nodes.
297 static unsigned long __init early_calculate_totalpages(void)
299 unsigned long totalpages = 0;
300 unsigned long start_pfn, end_pfn;
303 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
304 unsigned long pages = end_pfn - start_pfn;
308 node_set_state(nid, N_MEMORY);
314 * This finds a zone that can be used for ZONE_MOVABLE pages. The
315 * assumption is made that zones within a node are ordered in monotonic
316 * increasing memory addresses so that the "highest" populated zone is used
318 static void __init find_usable_zone_for_movable(void)
321 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
322 if (zone_index == ZONE_MOVABLE)
325 if (arch_zone_highest_possible_pfn[zone_index] >
326 arch_zone_lowest_possible_pfn[zone_index])
330 VM_BUG_ON(zone_index == -1);
331 movable_zone = zone_index;
335 * Find the PFN the Movable zone begins in each node. Kernel memory
336 * is spread evenly between nodes as long as the nodes have enough
337 * memory. When they don't, some nodes will have more kernelcore than
340 static void __init find_zone_movable_pfns_for_nodes(void)
343 unsigned long usable_startpfn;
344 unsigned long kernelcore_node, kernelcore_remaining;
345 /* save the state before borrow the nodemask */
346 nodemask_t saved_node_state = node_states[N_MEMORY];
347 unsigned long totalpages = early_calculate_totalpages();
348 int usable_nodes = nodes_weight(node_states[N_MEMORY]);
349 struct memblock_region *r;
351 /* Need to find movable_zone earlier when movable_node is specified. */
352 find_usable_zone_for_movable();
355 * If movable_node is specified, ignore kernelcore and movablecore
358 if (movable_node_is_enabled()) {
359 for_each_mem_region(r) {
360 if (!memblock_is_hotpluggable(r))
363 nid = memblock_get_region_node(r);
365 usable_startpfn = PFN_DOWN(r->base);
366 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
367 min(usable_startpfn, zone_movable_pfn[nid]) :
375 * If kernelcore=mirror is specified, ignore movablecore option
377 if (mirrored_kernelcore) {
378 bool mem_below_4gb_not_mirrored = false;
380 if (!memblock_has_mirror()) {
381 pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n");
385 if (is_kdump_kernel()) {
386 pr_warn("The system is under kdump, ignore kernelcore=mirror.\n");
390 for_each_mem_region(r) {
391 if (memblock_is_mirror(r))
394 nid = memblock_get_region_node(r);
396 usable_startpfn = memblock_region_memory_base_pfn(r);
398 if (usable_startpfn < PHYS_PFN(SZ_4G)) {
399 mem_below_4gb_not_mirrored = true;
403 zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
404 min(usable_startpfn, zone_movable_pfn[nid]) :
408 if (mem_below_4gb_not_mirrored)
409 pr_warn("This configuration results in unmirrored kernel memory.\n");
415 * If kernelcore=nn% or movablecore=nn% was specified, calculate the
416 * amount of necessary memory.
418 if (required_kernelcore_percent)
419 required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
421 if (required_movablecore_percent)
422 required_movablecore = (totalpages * 100 * required_movablecore_percent) /
426 * If movablecore= was specified, calculate what size of
427 * kernelcore that corresponds so that memory usable for
428 * any allocation type is evenly spread. If both kernelcore
429 * and movablecore are specified, then the value of kernelcore
430 * will be used for required_kernelcore if it's greater than
431 * what movablecore would have allowed.
433 if (required_movablecore) {
434 unsigned long corepages;
437 * Round-up so that ZONE_MOVABLE is at least as large as what
438 * was requested by the user
440 required_movablecore =
441 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
442 required_movablecore = min(totalpages, required_movablecore);
443 corepages = totalpages - required_movablecore;
445 required_kernelcore = max(required_kernelcore, corepages);
449 * If kernelcore was not specified or kernelcore size is larger
450 * than totalpages, there is no ZONE_MOVABLE.
452 if (!required_kernelcore || required_kernelcore >= totalpages)
455 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
456 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
459 /* Spread kernelcore memory as evenly as possible throughout nodes */
460 kernelcore_node = required_kernelcore / usable_nodes;
461 for_each_node_state(nid, N_MEMORY) {
462 unsigned long start_pfn, end_pfn;
465 * Recalculate kernelcore_node if the division per node
466 * now exceeds what is necessary to satisfy the requested
467 * amount of memory for the kernel
469 if (required_kernelcore < kernelcore_node)
470 kernelcore_node = required_kernelcore / usable_nodes;
473 * As the map is walked, we track how much memory is usable
474 * by the kernel using kernelcore_remaining. When it is
475 * 0, the rest of the node is usable by ZONE_MOVABLE
477 kernelcore_remaining = kernelcore_node;
479 /* Go through each range of PFNs within this node */
480 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
481 unsigned long size_pages;
483 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
484 if (start_pfn >= end_pfn)
487 /* Account for what is only usable for kernelcore */
488 if (start_pfn < usable_startpfn) {
489 unsigned long kernel_pages;
490 kernel_pages = min(end_pfn, usable_startpfn)
493 kernelcore_remaining -= min(kernel_pages,
494 kernelcore_remaining);
495 required_kernelcore -= min(kernel_pages,
496 required_kernelcore);
498 /* Continue if range is now fully accounted */
499 if (end_pfn <= usable_startpfn) {
502 * Push zone_movable_pfn to the end so
503 * that if we have to rebalance
504 * kernelcore across nodes, we will
505 * not double account here
507 zone_movable_pfn[nid] = end_pfn;
510 start_pfn = usable_startpfn;
514 * The usable PFN range for ZONE_MOVABLE is from
515 * start_pfn->end_pfn. Calculate size_pages as the
516 * number of pages used as kernelcore
518 size_pages = end_pfn - start_pfn;
519 if (size_pages > kernelcore_remaining)
520 size_pages = kernelcore_remaining;
521 zone_movable_pfn[nid] = start_pfn + size_pages;
524 * Some kernelcore has been met, update counts and
525 * break if the kernelcore for this node has been
528 required_kernelcore -= min(required_kernelcore,
530 kernelcore_remaining -= size_pages;
531 if (!kernelcore_remaining)
537 * If there is still required_kernelcore, we do another pass with one
538 * less node in the count. This will push zone_movable_pfn[nid] further
539 * along on the nodes that still have memory until kernelcore is
543 if (usable_nodes && required_kernelcore > usable_nodes)
547 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
548 for (nid = 0; nid < MAX_NUMNODES; nid++) {
549 unsigned long start_pfn, end_pfn;
551 zone_movable_pfn[nid] =
552 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
554 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
555 if (zone_movable_pfn[nid] >= end_pfn)
556 zone_movable_pfn[nid] = 0;
560 /* restore the node_state */
561 node_states[N_MEMORY] = saved_node_state;
564 void __meminit __init_single_page(struct page *page, unsigned long pfn,
565 unsigned long zone, int nid)
567 mm_zero_struct_page(page);
568 set_page_links(page, zone, nid, pfn);
569 init_page_count(page);
570 page_mapcount_reset(page);
571 page_cpupid_reset_last(page);
572 page_kasan_tag_reset(page);
574 INIT_LIST_HEAD(&page->lru);
575 #ifdef WANT_PAGE_VIRTUAL
576 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
577 if (!is_highmem_idx(zone))
578 set_page_address(page, __va(pfn << PAGE_SHIFT));
584 * During memory init memblocks map pfns to nids. The search is expensive and
585 * this caches recent lookups. The implementation of __early_pfn_to_nid
586 * treats start/end as pfns.
588 struct mminit_pfnnid_cache {
589 unsigned long last_start;
590 unsigned long last_end;
594 static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
597 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
599 static int __meminit __early_pfn_to_nid(unsigned long pfn,
600 struct mminit_pfnnid_cache *state)
602 unsigned long start_pfn, end_pfn;
605 if (state->last_start <= pfn && pfn < state->last_end)
606 return state->last_nid;
608 nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
609 if (nid != NUMA_NO_NODE) {
610 state->last_start = start_pfn;
611 state->last_end = end_pfn;
612 state->last_nid = nid;
618 int __meminit early_pfn_to_nid(unsigned long pfn)
620 static DEFINE_SPINLOCK(early_pfn_lock);
623 spin_lock(&early_pfn_lock);
624 nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
626 nid = first_online_node;
627 spin_unlock(&early_pfn_lock);
632 int hashdist = HASHDIST_DEFAULT;
634 static int __init set_hashdist(char *str)
638 hashdist = simple_strtoul(str, &str, 0);
641 __setup("hashdist=", set_hashdist);
643 static inline void fixup_hashdist(void)
645 if (num_node_state(N_MEMORY) == 1)
649 static inline void fixup_hashdist(void) {}
650 #endif /* CONFIG_NUMA */
652 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
653 static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
655 pgdat->first_deferred_pfn = ULONG_MAX;
658 /* Returns true if the struct page for the pfn is initialised */
659 static inline bool __meminit early_page_initialised(unsigned long pfn, int nid)
661 if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
668 * Returns true when the remaining initialisation should be deferred until
669 * later in the boot cycle when it can be parallelised.
671 static bool __meminit
672 defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
674 static unsigned long prev_end_pfn, nr_initialised;
676 if (early_page_ext_enabled())
679 * prev_end_pfn static that contains the end of previous zone
680 * No need to protect because called very early in boot before smp_init.
682 if (prev_end_pfn != end_pfn) {
683 prev_end_pfn = end_pfn;
687 /* Always populate low zones for address-constrained allocations */
688 if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
691 if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
694 * We start only with one section of pages, more pages are added as
695 * needed until the rest of deferred pages are initialized.
698 if ((nr_initialised > PAGES_PER_SECTION) &&
699 (pfn & (PAGES_PER_SECTION - 1)) == 0) {
700 NODE_DATA(nid)->first_deferred_pfn = pfn;
706 static void __meminit init_reserved_page(unsigned long pfn, int nid)
711 if (early_page_initialised(pfn, nid))
714 pgdat = NODE_DATA(nid);
716 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
717 struct zone *zone = &pgdat->node_zones[zid];
719 if (zone_spans_pfn(zone, pfn))
722 __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
725 static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
727 static inline bool early_page_initialised(unsigned long pfn, int nid)
732 static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
737 static inline void init_reserved_page(unsigned long pfn, int nid)
740 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
743 * Initialised pages do not have PageReserved set. This function is
744 * called for each range allocated by the bootmem allocator and
745 * marks the pages PageReserved. The remaining valid pages are later
746 * sent to the buddy page allocator.
748 void __meminit reserve_bootmem_region(phys_addr_t start,
749 phys_addr_t end, int nid)
751 unsigned long start_pfn = PFN_DOWN(start);
752 unsigned long end_pfn = PFN_UP(end);
754 for (; start_pfn < end_pfn; start_pfn++) {
755 if (pfn_valid(start_pfn)) {
756 struct page *page = pfn_to_page(start_pfn);
758 init_reserved_page(start_pfn, nid);
760 /* Avoid false-positive PageTail() */
761 INIT_LIST_HEAD(&page->lru);
764 * no need for atomic set_bit because the struct
765 * page is not visible yet so nobody should
768 __SetPageReserved(page);
773 /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
774 static bool __meminit
775 overlap_memmap_init(unsigned long zone, unsigned long *pfn)
777 static struct memblock_region *r;
779 if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
780 if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
781 for_each_mem_region(r) {
782 if (*pfn < memblock_region_memory_end_pfn(r))
786 if (*pfn >= memblock_region_memory_base_pfn(r) &&
787 memblock_is_mirror(r)) {
788 *pfn = memblock_region_memory_end_pfn(r);
796 * Only struct pages that correspond to ranges defined by memblock.memory
797 * are zeroed and initialized by going through __init_single_page() during
798 * memmap_init_zone_range().
800 * But, there could be struct pages that correspond to holes in
801 * memblock.memory. This can happen because of the following reasons:
802 * - physical memory bank size is not necessarily the exact multiple of the
803 * arbitrary section size
804 * - early reserved memory may not be listed in memblock.memory
805 * - non-memory regions covered by the contigious flatmem mapping
806 * - memory layouts defined with memmap= kernel parameter may not align
807 * nicely with memmap sections
809 * Explicitly initialize those struct pages so that:
810 * - PG_Reserved is set
811 * - zone and node links point to zone and node that span the page if the
812 * hole is in the middle of a zone
813 * - zone and node links point to adjacent zone/node if the hole falls on
814 * the zone boundary; the pages in such holes will be prepended to the
815 * zone/node above the hole except for the trailing pages in the last
816 * section that will be appended to the zone/node below.
818 static void __init init_unavailable_range(unsigned long spfn,
825 for (pfn = spfn; pfn < epfn; pfn++) {
826 if (!pfn_valid(pageblock_start_pfn(pfn))) {
827 pfn = pageblock_end_pfn(pfn) - 1;
830 __init_single_page(pfn_to_page(pfn), pfn, zone, node);
831 __SetPageReserved(pfn_to_page(pfn));
836 pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n",
837 node, zone_names[zone], pgcnt);
841 * Initially all pages are reserved - free ones are freed
842 * up by memblock_free_all() once the early boot process is
843 * done. Non-atomic initialization, single-pass.
845 * All aligned pageblocks are initialized to the specified migratetype
846 * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
847 * zone stats (e.g., nr_isolate_pageblock) are touched.
849 void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,
850 unsigned long start_pfn, unsigned long zone_end_pfn,
851 enum meminit_context context,
852 struct vmem_altmap *altmap, int migratetype)
854 unsigned long pfn, end_pfn = start_pfn + size;
857 if (highest_memmap_pfn < end_pfn - 1)
858 highest_memmap_pfn = end_pfn - 1;
860 #ifdef CONFIG_ZONE_DEVICE
862 * Honor reservation requested by the driver for this ZONE_DEVICE
863 * memory. We limit the total number of pages to initialize to just
864 * those that might contain the memory mapping. We will defer the
865 * ZONE_DEVICE page initialization until after we have released
868 if (zone == ZONE_DEVICE) {
872 if (start_pfn == altmap->base_pfn)
873 start_pfn += altmap->reserve;
874 end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
878 for (pfn = start_pfn; pfn < end_pfn; ) {
880 * There can be holes in boot-time mem_map[]s handed to this
881 * function. They do not exist on hotplugged memory.
883 if (context == MEMINIT_EARLY) {
884 if (overlap_memmap_init(zone, &pfn))
886 if (defer_init(nid, pfn, zone_end_pfn)) {
887 deferred_struct_pages = true;
892 page = pfn_to_page(pfn);
893 __init_single_page(page, pfn, zone, nid);
894 if (context == MEMINIT_HOTPLUG)
895 __SetPageReserved(page);
898 * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
899 * such that unmovable allocations won't be scattered all
900 * over the place during system boot.
902 if (pageblock_aligned(pfn)) {
903 set_pageblock_migratetype(page, migratetype);
910 static void __init memmap_init_zone_range(struct zone *zone,
911 unsigned long start_pfn,
912 unsigned long end_pfn,
913 unsigned long *hole_pfn)
915 unsigned long zone_start_pfn = zone->zone_start_pfn;
916 unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
917 int nid = zone_to_nid(zone), zone_id = zone_idx(zone);
919 start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);
920 end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);
922 if (start_pfn >= end_pfn)
925 memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
926 zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
928 if (*hole_pfn < start_pfn)
929 init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
934 static void __init memmap_init(void)
936 unsigned long start_pfn, end_pfn;
937 unsigned long hole_pfn = 0;
938 int i, j, zone_id = 0, nid;
940 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
941 struct pglist_data *node = NODE_DATA(nid);
943 for (j = 0; j < MAX_NR_ZONES; j++) {
944 struct zone *zone = node->node_zones + j;
946 if (!populated_zone(zone))
949 memmap_init_zone_range(zone, start_pfn, end_pfn,
955 #ifdef CONFIG_SPARSEMEM
957 * Initialize the memory map for hole in the range [memory_end,
959 * Append the pages in this hole to the highest zone in the last
961 * The call to init_unavailable_range() is outside the ifdef to
962 * silence the compiler warining about zone_id set but not used;
963 * for FLATMEM it is a nop anyway
965 end_pfn = round_up(end_pfn, PAGES_PER_SECTION);
966 if (hole_pfn < end_pfn)
968 init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);
971 #ifdef CONFIG_ZONE_DEVICE
972 static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
973 unsigned long zone_idx, int nid,
974 struct dev_pagemap *pgmap)
977 __init_single_page(page, pfn, zone_idx, nid);
980 * Mark page reserved as it will need to wait for onlining
981 * phase for it to be fully associated with a zone.
983 * We can use the non-atomic __set_bit operation for setting
984 * the flag as we are still initializing the pages.
986 __SetPageReserved(page);
989 * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
990 * and zone_device_data. It is a bug if a ZONE_DEVICE page is
991 * ever freed or placed on a driver-private list.
994 page->zone_device_data = NULL;
997 * Mark the block movable so that blocks are reserved for
998 * movable at startup. This will force kernel allocations
999 * to reserve their blocks rather than leaking throughout
1000 * the address space during boot when many long-lived
1001 * kernel allocations are made.
1003 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
1004 * because this is done early in section_activate()
1006 if (pageblock_aligned(pfn)) {
1007 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1012 * ZONE_DEVICE pages are released directly to the driver page allocator
1013 * which will set the page count to 1 when allocating the page.
1015 if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
1016 pgmap->type == MEMORY_DEVICE_COHERENT)
1017 set_page_count(page, 0);
1021 * With compound page geometry and when struct pages are stored in ram most
1022 * tail pages are reused. Consequently, the amount of unique struct pages to
1023 * initialize is a lot smaller that the total amount of struct pages being
1024 * mapped. This is a paired / mild layering violation with explicit knowledge
1025 * of how the sparse_vmemmap internals handle compound pages in the lack
1026 * of an altmap. See vmemmap_populate_compound_pages().
1028 static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,
1029 struct dev_pagemap *pgmap)
1031 if (!vmemmap_can_optimize(altmap, pgmap))
1032 return pgmap_vmemmap_nr(pgmap);
1034 return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page));
1037 static void __ref memmap_init_compound(struct page *head,
1038 unsigned long head_pfn,
1039 unsigned long zone_idx, int nid,
1040 struct dev_pagemap *pgmap,
1041 unsigned long nr_pages)
1043 unsigned long pfn, end_pfn = head_pfn + nr_pages;
1044 unsigned int order = pgmap->vmemmap_shift;
1046 __SetPageHead(head);
1047 for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) {
1048 struct page *page = pfn_to_page(pfn);
1050 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
1051 prep_compound_tail(head, pfn - head_pfn);
1052 set_page_count(page, 0);
1055 * The first tail page stores important compound page info.
1056 * Call prep_compound_head() after the first tail page has
1057 * been initialized, to not have the data overwritten.
1059 if (pfn == head_pfn + 1)
1060 prep_compound_head(head, order);
1064 void __ref memmap_init_zone_device(struct zone *zone,
1065 unsigned long start_pfn,
1066 unsigned long nr_pages,
1067 struct dev_pagemap *pgmap)
1069 unsigned long pfn, end_pfn = start_pfn + nr_pages;
1070 struct pglist_data *pgdat = zone->zone_pgdat;
1071 struct vmem_altmap *altmap = pgmap_altmap(pgmap);
1072 unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap);
1073 unsigned long zone_idx = zone_idx(zone);
1074 unsigned long start = jiffies;
1075 int nid = pgdat->node_id;
1077 if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))
1081 * The call to memmap_init should have already taken care
1082 * of the pages reserved for the memmap, so we can just jump to
1083 * the end of that region and start processing the device pages.
1086 start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
1087 nr_pages = end_pfn - start_pfn;
1090 for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {
1091 struct page *page = pfn_to_page(pfn);
1093 __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
1095 if (pfns_per_compound == 1)
1098 memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
1099 compound_nr_pages(altmap, pgmap));
1102 pr_debug("%s initialised %lu pages in %ums\n", __func__,
1103 nr_pages, jiffies_to_msecs(jiffies - start));
1108 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
1109 * because it is sized independent of architecture. Unlike the other zones,
1110 * the starting point for ZONE_MOVABLE is not fixed. It may be different
1111 * in each node depending on the size of each node and how evenly kernelcore
1112 * is distributed. This helper function adjusts the zone ranges
1113 * provided by the architecture for a given node by using the end of the
1114 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
1115 * zones within a node are in order of monotonic increases memory addresses
1117 static void __init adjust_zone_range_for_zone_movable(int nid,
1118 unsigned long zone_type,
1119 unsigned long node_end_pfn,
1120 unsigned long *zone_start_pfn,
1121 unsigned long *zone_end_pfn)
1123 /* Only adjust if ZONE_MOVABLE is on this node */
1124 if (zone_movable_pfn[nid]) {
1125 /* Size ZONE_MOVABLE */
1126 if (zone_type == ZONE_MOVABLE) {
1127 *zone_start_pfn = zone_movable_pfn[nid];
1128 *zone_end_pfn = min(node_end_pfn,
1129 arch_zone_highest_possible_pfn[movable_zone]);
1131 /* Adjust for ZONE_MOVABLE starting within this range */
1132 } else if (!mirrored_kernelcore &&
1133 *zone_start_pfn < zone_movable_pfn[nid] &&
1134 *zone_end_pfn > zone_movable_pfn[nid]) {
1135 *zone_end_pfn = zone_movable_pfn[nid];
1137 /* Check if this whole range is within ZONE_MOVABLE */
1138 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
1139 *zone_start_pfn = *zone_end_pfn;
1144 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
1145 * then all holes in the requested range will be accounted for.
1147 static unsigned long __init __absent_pages_in_range(int nid,
1148 unsigned long range_start_pfn,
1149 unsigned long range_end_pfn)
1151 unsigned long nr_absent = range_end_pfn - range_start_pfn;
1152 unsigned long start_pfn, end_pfn;
1155 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
1156 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
1157 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
1158 nr_absent -= end_pfn - start_pfn;
1164 * absent_pages_in_range - Return number of page frames in holes within a range
1165 * @start_pfn: The start PFN to start searching for holes
1166 * @end_pfn: The end PFN to stop searching for holes
1168 * Return: the number of pages frames in memory holes within a range.
1170 unsigned long __init absent_pages_in_range(unsigned long start_pfn,
1171 unsigned long end_pfn)
1173 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
1176 /* Return the number of page frames in holes in a zone on a node */
1177 static unsigned long __init zone_absent_pages_in_node(int nid,
1178 unsigned long zone_type,
1179 unsigned long zone_start_pfn,
1180 unsigned long zone_end_pfn)
1182 unsigned long nr_absent;
1184 /* zone is empty, we don't have any absent pages */
1185 if (zone_start_pfn == zone_end_pfn)
1188 nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
1191 * ZONE_MOVABLE handling.
1192 * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
1195 if (mirrored_kernelcore && zone_movable_pfn[nid]) {
1196 unsigned long start_pfn, end_pfn;
1197 struct memblock_region *r;
1199 for_each_mem_region(r) {
1200 start_pfn = clamp(memblock_region_memory_base_pfn(r),
1201 zone_start_pfn, zone_end_pfn);
1202 end_pfn = clamp(memblock_region_memory_end_pfn(r),
1203 zone_start_pfn, zone_end_pfn);
1205 if (zone_type == ZONE_MOVABLE &&
1206 memblock_is_mirror(r))
1207 nr_absent += end_pfn - start_pfn;
1209 if (zone_type == ZONE_NORMAL &&
1210 !memblock_is_mirror(r))
1211 nr_absent += end_pfn - start_pfn;
1219 * Return the number of pages a zone spans in a node, including holes
1220 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
1222 static unsigned long __init zone_spanned_pages_in_node(int nid,
1223 unsigned long zone_type,
1224 unsigned long node_start_pfn,
1225 unsigned long node_end_pfn,
1226 unsigned long *zone_start_pfn,
1227 unsigned long *zone_end_pfn)
1229 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
1230 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
1232 /* Get the start and end of the zone */
1233 *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
1234 *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
1235 adjust_zone_range_for_zone_movable(nid, zone_type, node_end_pfn,
1236 zone_start_pfn, zone_end_pfn);
1238 /* Check that this node has pages within the zone's required range */
1239 if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
1242 /* Move the zone boundaries inside the node if necessary */
1243 *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
1244 *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
1246 /* Return the spanned pages */
1247 return *zone_end_pfn - *zone_start_pfn;
1250 static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat)
1254 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) {
1255 z->zone_start_pfn = 0;
1256 z->spanned_pages = 0;
1257 z->present_pages = 0;
1258 #if defined(CONFIG_MEMORY_HOTPLUG)
1259 z->present_early_pages = 0;
1263 pgdat->node_spanned_pages = 0;
1264 pgdat->node_present_pages = 0;
1265 pr_debug("On node %d totalpages: 0\n", pgdat->node_id);
1268 static void __init calc_nr_kernel_pages(void)
1270 unsigned long start_pfn, end_pfn;
1271 phys_addr_t start_addr, end_addr;
1273 #ifdef CONFIG_HIGHMEM
1274 unsigned long high_zone_low = arch_zone_lowest_possible_pfn[ZONE_HIGHMEM];
1277 for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) {
1278 start_pfn = PFN_UP(start_addr);
1279 end_pfn = PFN_DOWN(end_addr);
1281 if (start_pfn < end_pfn) {
1282 nr_all_pages += end_pfn - start_pfn;
1283 #ifdef CONFIG_HIGHMEM
1284 start_pfn = clamp(start_pfn, 0, high_zone_low);
1285 end_pfn = clamp(end_pfn, 0, high_zone_low);
1287 nr_kernel_pages += end_pfn - start_pfn;
1292 static void __init calculate_node_totalpages(struct pglist_data *pgdat,
1293 unsigned long node_start_pfn,
1294 unsigned long node_end_pfn)
1296 unsigned long realtotalpages = 0, totalpages = 0;
1299 for (i = 0; i < MAX_NR_ZONES; i++) {
1300 struct zone *zone = pgdat->node_zones + i;
1301 unsigned long zone_start_pfn, zone_end_pfn;
1302 unsigned long spanned, absent;
1303 unsigned long real_size;
1305 spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
1310 absent = zone_absent_pages_in_node(pgdat->node_id, i,
1314 real_size = spanned - absent;
1317 zone->zone_start_pfn = zone_start_pfn;
1319 zone->zone_start_pfn = 0;
1320 zone->spanned_pages = spanned;
1321 zone->present_pages = real_size;
1322 #if defined(CONFIG_MEMORY_HOTPLUG)
1323 zone->present_early_pages = real_size;
1326 totalpages += spanned;
1327 realtotalpages += real_size;
1330 pgdat->node_spanned_pages = totalpages;
1331 pgdat->node_present_pages = realtotalpages;
1332 pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
1335 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1336 static void pgdat_init_split_queue(struct pglist_data *pgdat)
1338 struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
1340 spin_lock_init(&ds_queue->split_queue_lock);
1341 INIT_LIST_HEAD(&ds_queue->split_queue);
1342 ds_queue->split_queue_len = 0;
1345 static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
1348 #ifdef CONFIG_COMPACTION
1349 static void pgdat_init_kcompactd(struct pglist_data *pgdat)
1351 init_waitqueue_head(&pgdat->kcompactd_wait);
1354 static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
1357 static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
1361 pgdat_resize_init(pgdat);
1362 pgdat_kswapd_lock_init(pgdat);
1364 pgdat_init_split_queue(pgdat);
1365 pgdat_init_kcompactd(pgdat);
1367 init_waitqueue_head(&pgdat->kswapd_wait);
1368 init_waitqueue_head(&pgdat->pfmemalloc_wait);
1370 for (i = 0; i < NR_VMSCAN_THROTTLE; i++)
1371 init_waitqueue_head(&pgdat->reclaim_wait[i]);
1373 pgdat_page_ext_init(pgdat);
1374 lruvec_init(&pgdat->__lruvec);
1377 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
1378 unsigned long remaining_pages)
1380 atomic_long_set(&zone->managed_pages, remaining_pages);
1381 zone_set_nid(zone, nid);
1382 zone->name = zone_names[idx];
1383 zone->zone_pgdat = NODE_DATA(nid);
1384 spin_lock_init(&zone->lock);
1385 zone_seqlock_init(zone);
1386 zone_pcp_init(zone);
1389 static void __meminit zone_init_free_lists(struct zone *zone)
1391 unsigned int order, t;
1392 for_each_migratetype_order(order, t) {
1393 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
1394 zone->free_area[order].nr_free = 0;
1397 #ifdef CONFIG_UNACCEPTED_MEMORY
1398 INIT_LIST_HEAD(&zone->unaccepted_pages);
1402 void __meminit init_currently_empty_zone(struct zone *zone,
1403 unsigned long zone_start_pfn,
1406 struct pglist_data *pgdat = zone->zone_pgdat;
1407 int zone_idx = zone_idx(zone) + 1;
1409 if (zone_idx > pgdat->nr_zones)
1410 pgdat->nr_zones = zone_idx;
1412 zone->zone_start_pfn = zone_start_pfn;
1414 mminit_dprintk(MMINIT_TRACE, "memmap_init",
1415 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
1417 (unsigned long)zone_idx(zone),
1418 zone_start_pfn, (zone_start_pfn + size));
1420 zone_init_free_lists(zone);
1421 zone->initialized = 1;
1424 #ifndef CONFIG_SPARSEMEM
1426 * Calculate the size of the zone->blockflags rounded to an unsigned long
1427 * Start by making sure zonesize is a multiple of pageblock_order by rounding
1428 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
1429 * round what is now in bits to nearest long in bits, then return it in
1432 static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
1434 unsigned long usemapsize;
1436 zonesize += zone_start_pfn & (pageblock_nr_pages-1);
1437 usemapsize = roundup(zonesize, pageblock_nr_pages);
1438 usemapsize = usemapsize >> pageblock_order;
1439 usemapsize *= NR_PAGEBLOCK_BITS;
1440 usemapsize = roundup(usemapsize, BITS_PER_LONG);
1442 return usemapsize / BITS_PER_BYTE;
1445 static void __ref setup_usemap(struct zone *zone)
1447 unsigned long usemapsize = usemap_size(zone->zone_start_pfn,
1448 zone->spanned_pages);
1449 zone->pageblock_flags = NULL;
1451 zone->pageblock_flags =
1452 memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
1454 if (!zone->pageblock_flags)
1455 panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
1456 usemapsize, zone->name, zone_to_nid(zone));
1460 static inline void setup_usemap(struct zone *zone) {}
1461 #endif /* CONFIG_SPARSEMEM */
1463 #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
1465 /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
1466 void __init set_pageblock_order(void)
1468 unsigned int order = MAX_PAGE_ORDER;
1470 /* Check that pageblock_nr_pages has not already been setup */
1471 if (pageblock_order)
1474 /* Don't let pageblocks exceed the maximum allocation granularity. */
1475 if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order)
1476 order = HUGETLB_PAGE_ORDER;
1479 * Assume the largest contiguous order of interest is a huge page.
1480 * This value may be variable depending on boot parameters on powerpc.
1482 pageblock_order = order;
1484 #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1487 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
1488 * is unused as pageblock_order is set at compile-time. See
1489 * include/linux/pageblock-flags.h for the values of pageblock_order based on
1492 void __init set_pageblock_order(void)
1496 #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
1499 * Set up the zone data structures
1500 * - init pgdat internals
1501 * - init all zones belonging to this node
1503 * NOTE: this function is only called during memory hotplug
1505 #ifdef CONFIG_MEMORY_HOTPLUG
1506 void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
1508 int nid = pgdat->node_id;
1512 pgdat_init_internals(pgdat);
1514 if (pgdat->per_cpu_nodestats == &boot_nodestats)
1515 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);
1518 * Reset the nr_zones, order and highest_zoneidx before reuse.
1519 * Note that kswapd will init kswapd_highest_zoneidx properly
1520 * when it starts in the near future.
1522 pgdat->nr_zones = 0;
1523 pgdat->kswapd_order = 0;
1524 pgdat->kswapd_highest_zoneidx = 0;
1525 pgdat->node_start_pfn = 0;
1526 pgdat->node_present_pages = 0;
1528 for_each_online_cpu(cpu) {
1529 struct per_cpu_nodestat *p;
1531 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
1532 memset(p, 0, sizeof(*p));
1536 * When memory is hot-added, all the memory is in offline state. So
1537 * clear all zones' present_pages and managed_pages because they will
1538 * be updated in online_pages() and offline_pages().
1540 for (z = 0; z < MAX_NR_ZONES; z++) {
1541 struct zone *zone = pgdat->node_zones + z;
1543 zone->present_pages = 0;
1544 zone_init_internals(zone, z, nid, 0);
1549 static void __init free_area_init_core(struct pglist_data *pgdat)
1552 int nid = pgdat->node_id;
1554 pgdat_init_internals(pgdat);
1555 pgdat->per_cpu_nodestats = &boot_nodestats;
1557 for (j = 0; j < MAX_NR_ZONES; j++) {
1558 struct zone *zone = pgdat->node_zones + j;
1559 unsigned long size = zone->spanned_pages;
1562 * Initialize zone->managed_pages as 0 , it will be reset
1563 * when memblock allocator frees pages into buddy system.
1565 zone_init_internals(zone, j, nid, zone->present_pages);
1571 init_currently_empty_zone(zone, zone->zone_start_pfn, size);
1575 void __init *memmap_alloc(phys_addr_t size, phys_addr_t align,
1576 phys_addr_t min_addr, int nid, bool exact_nid)
1581 ptr = memblock_alloc_exact_nid_raw(size, align, min_addr,
1582 MEMBLOCK_ALLOC_ACCESSIBLE,
1585 ptr = memblock_alloc_try_nid_raw(size, align, min_addr,
1586 MEMBLOCK_ALLOC_ACCESSIBLE,
1589 if (ptr && size > 0)
1590 page_init_poison(ptr, size);
1595 #ifdef CONFIG_FLATMEM
1596 static void __init alloc_node_mem_map(struct pglist_data *pgdat)
1598 unsigned long start, offset, size, end;
1601 /* Skip empty nodes */
1602 if (!pgdat->node_spanned_pages)
1605 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
1606 offset = pgdat->node_start_pfn - start;
1608 * The zone's endpoints aren't required to be MAX_PAGE_ORDER
1609 * aligned but the node_mem_map endpoints must be in order
1610 * for the buddy allocator to function correctly.
1612 end = ALIGN(pgdat_end_pfn(pgdat), MAX_ORDER_NR_PAGES);
1613 size = (end - start) * sizeof(struct page);
1614 map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT,
1615 pgdat->node_id, false);
1617 panic("Failed to allocate %ld bytes for node %d memory map\n",
1618 size, pgdat->node_id);
1619 pgdat->node_mem_map = map + offset;
1620 pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
1621 __func__, pgdat->node_id, (unsigned long)pgdat,
1622 (unsigned long)pgdat->node_mem_map);
1624 /* the global mem_map is just set as node 0's */
1625 if (pgdat == NODE_DATA(0)) {
1626 mem_map = NODE_DATA(0)->node_mem_map;
1627 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
1633 static inline void alloc_node_mem_map(struct pglist_data *pgdat) { }
1634 #endif /* CONFIG_FLATMEM */
1637 * get_pfn_range_for_nid - Return the start and end page frames for a node
1638 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
1639 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
1640 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
1642 * It returns the start and end page frame of a node based on information
1643 * provided by memblock_set_node(). If called for a node
1644 * with no available memory, the start and end PFNs will be 0.
1646 void __init get_pfn_range_for_nid(unsigned int nid,
1647 unsigned long *start_pfn, unsigned long *end_pfn)
1649 unsigned long this_start_pfn, this_end_pfn;
1655 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
1656 *start_pfn = min(*start_pfn, this_start_pfn);
1657 *end_pfn = max(*end_pfn, this_end_pfn);
1660 if (*start_pfn == -1UL)
1664 static void __init free_area_init_node(int nid)
1666 pg_data_t *pgdat = NODE_DATA(nid);
1667 unsigned long start_pfn = 0;
1668 unsigned long end_pfn = 0;
1670 /* pg_data_t should be reset to zero when it's allocated */
1671 WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
1673 get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
1675 pgdat->node_id = nid;
1676 pgdat->node_start_pfn = start_pfn;
1677 pgdat->per_cpu_nodestats = NULL;
1679 if (start_pfn != end_pfn) {
1680 pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
1681 (u64)start_pfn << PAGE_SHIFT,
1682 end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
1684 calculate_node_totalpages(pgdat, start_pfn, end_pfn);
1686 pr_info("Initmem setup node %d as memoryless\n", nid);
1688 reset_memoryless_node_totalpages(pgdat);
1691 alloc_node_mem_map(pgdat);
1692 pgdat_set_deferred_range(pgdat);
1694 free_area_init_core(pgdat);
1695 lru_gen_init_pgdat(pgdat);
1698 /* Any regular or high memory on that node ? */
1699 static void __init check_for_memory(pg_data_t *pgdat)
1701 enum zone_type zone_type;
1703 for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
1704 struct zone *zone = &pgdat->node_zones[zone_type];
1705 if (populated_zone(zone)) {
1706 if (IS_ENABLED(CONFIG_HIGHMEM))
1707 node_set_state(pgdat->node_id, N_HIGH_MEMORY);
1708 if (zone_type <= ZONE_NORMAL)
1709 node_set_state(pgdat->node_id, N_NORMAL_MEMORY);
1715 #if MAX_NUMNODES > 1
1717 * Figure out the number of possible node ids.
1719 void __init setup_nr_node_ids(void)
1721 unsigned int highest;
1723 highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
1724 nr_node_ids = highest + 1;
1729 * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
1730 * such cases we allow max_zone_pfn sorted in the descending order
1732 static bool arch_has_descending_max_zone_pfns(void)
1734 return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40);
1738 * free_area_init - Initialise all pg_data_t and zone data
1739 * @max_zone_pfn: an array of max PFNs for each zone
1741 * This will call free_area_init_node() for each active node in the system.
1742 * Using the page ranges provided by memblock_set_node(), the size of each
1743 * zone in each node and their holes is calculated. If the maximum PFN
1744 * between two adjacent zones match, it is assumed that the zone is empty.
1745 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
1746 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
1747 * starts where the previous one ended. For example, ZONE_DMA32 starts
1748 * at arch_max_dma_pfn.
1750 void __init free_area_init(unsigned long *max_zone_pfn)
1752 unsigned long start_pfn, end_pfn;
1756 /* Record where the zone boundaries are */
1757 memset(arch_zone_lowest_possible_pfn, 0,
1758 sizeof(arch_zone_lowest_possible_pfn));
1759 memset(arch_zone_highest_possible_pfn, 0,
1760 sizeof(arch_zone_highest_possible_pfn));
1762 start_pfn = PHYS_PFN(memblock_start_of_DRAM());
1763 descending = arch_has_descending_max_zone_pfns();
1765 for (i = 0; i < MAX_NR_ZONES; i++) {
1767 zone = MAX_NR_ZONES - i - 1;
1771 if (zone == ZONE_MOVABLE)
1774 end_pfn = max(max_zone_pfn[zone], start_pfn);
1775 arch_zone_lowest_possible_pfn[zone] = start_pfn;
1776 arch_zone_highest_possible_pfn[zone] = end_pfn;
1778 start_pfn = end_pfn;
1781 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
1782 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
1783 find_zone_movable_pfns_for_nodes();
1785 /* Print out the zone ranges */
1786 pr_info("Zone ranges:\n");
1787 for (i = 0; i < MAX_NR_ZONES; i++) {
1788 if (i == ZONE_MOVABLE)
1790 pr_info(" %-8s ", zone_names[i]);
1791 if (arch_zone_lowest_possible_pfn[i] ==
1792 arch_zone_highest_possible_pfn[i])
1795 pr_cont("[mem %#018Lx-%#018Lx]\n",
1796 (u64)arch_zone_lowest_possible_pfn[i]
1798 ((u64)arch_zone_highest_possible_pfn[i]
1799 << PAGE_SHIFT) - 1);
1802 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
1803 pr_info("Movable zone start for each node\n");
1804 for (i = 0; i < MAX_NUMNODES; i++) {
1805 if (zone_movable_pfn[i])
1806 pr_info(" Node %d: %#018Lx\n", i,
1807 (u64)zone_movable_pfn[i] << PAGE_SHIFT);
1811 * Print out the early node map, and initialize the
1812 * subsection-map relative to active online memory ranges to
1813 * enable future "sub-section" extensions of the memory map.
1815 pr_info("Early memory node ranges\n");
1816 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
1817 pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
1818 (u64)start_pfn << PAGE_SHIFT,
1819 ((u64)end_pfn << PAGE_SHIFT) - 1);
1820 subsection_map_init(start_pfn, end_pfn - start_pfn);
1823 /* Initialise every node */
1824 mminit_verify_pageflags_layout();
1825 setup_nr_node_ids();
1826 set_pageblock_order();
1828 for_each_node(nid) {
1831 if (!node_online(nid)) {
1832 /* Allocator not initialized yet */
1833 pgdat = arch_alloc_nodedata(nid);
1835 panic("Cannot allocate %zuB for node %d.\n",
1836 sizeof(*pgdat), nid);
1837 arch_refresh_nodedata(nid, pgdat);
1840 pgdat = NODE_DATA(nid);
1841 free_area_init_node(nid);
1844 * No sysfs hierarcy will be created via register_one_node()
1845 *for memory-less node because here it's not marked as N_MEMORY
1846 *and won't be set online later. The benefit is userspace
1847 *program won't be confused by sysfs files/directories of
1848 *memory-less node. The pgdat will get fully initialized by
1849 *hotadd_init_pgdat() when memory is hotplugged into this node.
1851 if (pgdat->node_present_pages) {
1852 node_set_state(nid, N_MEMORY);
1853 check_for_memory(pgdat);
1857 calc_nr_kernel_pages();
1860 /* disable hash distribution for systems with a single node */
1865 * node_map_pfn_alignment - determine the maximum internode alignment
1867 * This function should be called after node map is populated and sorted.
1868 * It calculates the maximum power of two alignment which can distinguish
1871 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
1872 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
1873 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
1874 * shifted, 1GiB is enough and this function will indicate so.
1876 * This is used to test whether pfn -> nid mapping of the chosen memory
1877 * model has fine enough granularity to avoid incorrect mapping for the
1878 * populated node map.
1880 * Return: the determined alignment in pfn's. 0 if there is no alignment
1881 * requirement (single node).
1883 unsigned long __init node_map_pfn_alignment(void)
1885 unsigned long accl_mask = 0, last_end = 0;
1886 unsigned long start, end, mask;
1887 int last_nid = NUMA_NO_NODE;
1890 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
1891 if (!start || last_nid < 0 || last_nid == nid) {
1898 * Start with a mask granular enough to pin-point to the
1899 * start pfn and tick off bits one-by-one until it becomes
1900 * too coarse to separate the current node from the last.
1902 mask = ~((1 << __ffs(start)) - 1);
1903 while (mask && last_end <= (start & (mask << 1)))
1906 /* accumulate all internode masks */
1910 /* convert mask to number of pages */
1911 return ~accl_mask + 1;
1914 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
1915 static void __init deferred_free_range(unsigned long pfn,
1916 unsigned long nr_pages)
1924 page = pfn_to_page(pfn);
1926 /* Free a large naturally-aligned chunk if possible */
1927 if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) {
1928 for (i = 0; i < nr_pages; i += pageblock_nr_pages)
1929 set_pageblock_migratetype(page + i, MIGRATE_MOVABLE);
1930 __free_pages_core(page, MAX_PAGE_ORDER);
1934 /* Accept chunks smaller than MAX_PAGE_ORDER upfront */
1935 accept_memory(PFN_PHYS(pfn), PFN_PHYS(pfn + nr_pages));
1937 for (i = 0; i < nr_pages; i++, page++, pfn++) {
1938 if (pageblock_aligned(pfn))
1939 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
1940 __free_pages_core(page, 0);
1944 /* Completion tracking for deferred_init_memmap() threads */
1945 static atomic_t pgdat_init_n_undone __initdata;
1946 static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
1948 static inline void __init pgdat_init_report_one_done(void)
1950 if (atomic_dec_and_test(&pgdat_init_n_undone))
1951 complete(&pgdat_init_all_done_comp);
1955 * Returns true if page needs to be initialized or freed to buddy allocator.
1957 * We check if a current MAX_PAGE_ORDER block is valid by only checking the
1958 * validity of the head pfn.
1960 static inline bool __init deferred_pfn_valid(unsigned long pfn)
1962 if (IS_MAX_ORDER_ALIGNED(pfn) && !pfn_valid(pfn))
1968 * Free pages to buddy allocator. Try to free aligned pages in
1969 * MAX_ORDER_NR_PAGES sizes.
1971 static void __init deferred_free_pages(unsigned long pfn,
1972 unsigned long end_pfn)
1974 unsigned long nr_free = 0;
1976 for (; pfn < end_pfn; pfn++) {
1977 if (!deferred_pfn_valid(pfn)) {
1978 deferred_free_range(pfn - nr_free, nr_free);
1980 } else if (IS_MAX_ORDER_ALIGNED(pfn)) {
1981 deferred_free_range(pfn - nr_free, nr_free);
1987 /* Free the last block of pages to allocator */
1988 deferred_free_range(pfn - nr_free, nr_free);
1992 * Initialize struct pages. We minimize pfn page lookups and scheduler checks
1993 * by performing it only once every MAX_ORDER_NR_PAGES.
1994 * Return number of pages initialized.
1996 static unsigned long __init deferred_init_pages(struct zone *zone,
1998 unsigned long end_pfn)
2000 int nid = zone_to_nid(zone);
2001 unsigned long nr_pages = 0;
2002 int zid = zone_idx(zone);
2003 struct page *page = NULL;
2005 for (; pfn < end_pfn; pfn++) {
2006 if (!deferred_pfn_valid(pfn)) {
2009 } else if (!page || IS_MAX_ORDER_ALIGNED(pfn)) {
2010 page = pfn_to_page(pfn);
2014 __init_single_page(page, pfn, zid, nid);
2021 * This function is meant to pre-load the iterator for the zone init.
2022 * Specifically it walks through the ranges until we are caught up to the
2023 * first_init_pfn value and exits there. If we never encounter the value we
2024 * return false indicating there are no valid ranges left.
2027 deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
2028 unsigned long *spfn, unsigned long *epfn,
2029 unsigned long first_init_pfn)
2034 * Start out by walking through the ranges in this zone that have
2035 * already been initialized. We don't need to do anything with them
2036 * so we just need to flush them out of the system.
2038 for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
2039 if (*epfn <= first_init_pfn)
2041 if (*spfn < first_init_pfn)
2042 *spfn = first_init_pfn;
2051 * Initialize and free pages. We do it in two loops: first we initialize
2052 * struct page, then free to buddy allocator, because while we are
2053 * freeing pages we can access pages that are ahead (computing buddy
2054 * page in __free_one_page()).
2056 * In order to try and keep some memory in the cache we have the loop
2057 * broken along max page order boundaries. This way we will not cause
2058 * any issues with the buddy page computation.
2060 static unsigned long __init
2061 deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
2062 unsigned long *end_pfn)
2064 unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
2065 unsigned long spfn = *start_pfn, epfn = *end_pfn;
2066 unsigned long nr_pages = 0;
2069 /* First we loop through and initialize the page values */
2070 for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
2073 if (mo_pfn <= *start_pfn)
2076 t = min(mo_pfn, *end_pfn);
2077 nr_pages += deferred_init_pages(zone, *start_pfn, t);
2079 if (mo_pfn < *end_pfn) {
2080 *start_pfn = mo_pfn;
2085 /* Reset values and now loop through freeing pages as needed */
2088 for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
2094 t = min(mo_pfn, epfn);
2095 deferred_free_pages(spfn, t);
2105 deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
2108 unsigned long spfn, epfn;
2109 struct zone *zone = arg;
2112 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
2115 * Initialize and free pages in MAX_PAGE_ORDER sized increments so that
2116 * we can avoid introducing any issues with the buddy allocator.
2118 while (spfn < end_pfn) {
2119 deferred_init_maxorder(&i, zone, &spfn, &epfn);
2124 /* An arch may override for more concurrency. */
2126 deferred_page_init_max_threads(const struct cpumask *node_cpumask)
2131 /* Initialise remaining memory on a node */
2132 static int __init deferred_init_memmap(void *data)
2134 pg_data_t *pgdat = data;
2135 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
2136 unsigned long spfn = 0, epfn = 0;
2137 unsigned long first_init_pfn, flags;
2138 unsigned long start = jiffies;
2140 int zid, max_threads;
2143 /* Bind memory initialisation thread to a local node if possible */
2144 if (!cpumask_empty(cpumask))
2145 set_cpus_allowed_ptr(current, cpumask);
2147 pgdat_resize_lock(pgdat, &flags);
2148 first_init_pfn = pgdat->first_deferred_pfn;
2149 if (first_init_pfn == ULONG_MAX) {
2150 pgdat_resize_unlock(pgdat, &flags);
2151 pgdat_init_report_one_done();
2155 /* Sanity check boundaries */
2156 BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
2157 BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
2158 pgdat->first_deferred_pfn = ULONG_MAX;
2161 * Once we unlock here, the zone cannot be grown anymore, thus if an
2162 * interrupt thread must allocate this early in boot, zone must be
2163 * pre-grown prior to start of deferred page initialization.
2165 pgdat_resize_unlock(pgdat, &flags);
2167 /* Only the highest zone is deferred so find it */
2168 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2169 zone = pgdat->node_zones + zid;
2170 if (first_init_pfn < zone_end_pfn(zone))
2174 /* If the zone is empty somebody else may have cleared out the zone */
2175 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2179 max_threads = deferred_page_init_max_threads(cpumask);
2181 while (spfn < epfn) {
2182 unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
2183 struct padata_mt_job job = {
2184 .thread_fn = deferred_init_memmap_chunk,
2187 .size = epfn_align - spfn,
2188 .align = PAGES_PER_SECTION,
2189 .min_chunk = PAGES_PER_SECTION,
2190 .max_threads = max_threads,
2191 .numa_aware = false,
2194 padata_do_multithreaded(&job);
2195 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2199 /* Sanity check that the next zone really is unpopulated */
2200 WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
2202 pr_info("node %d deferred pages initialised in %ums\n",
2203 pgdat->node_id, jiffies_to_msecs(jiffies - start));
2205 pgdat_init_report_one_done();
2210 * If this zone has deferred pages, try to grow it by initializing enough
2211 * deferred pages to satisfy the allocation specified by order, rounded up to
2212 * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
2213 * of SECTION_SIZE bytes by initializing struct pages in increments of
2214 * PAGES_PER_SECTION * sizeof(struct page) bytes.
2216 * Return true when zone was grown, otherwise return false. We return true even
2217 * when we grow less than requested, to let the caller decide if there are
2218 * enough pages to satisfy the allocation.
2220 bool __init deferred_grow_zone(struct zone *zone, unsigned int order)
2222 unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
2223 pg_data_t *pgdat = zone->zone_pgdat;
2224 unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
2225 unsigned long spfn, epfn, flags;
2226 unsigned long nr_pages = 0;
2229 /* Only the last zone may have deferred pages */
2230 if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
2233 pgdat_resize_lock(pgdat, &flags);
2236 * If someone grew this zone while we were waiting for spinlock, return
2237 * true, as there might be enough pages already.
2239 if (first_deferred_pfn != pgdat->first_deferred_pfn) {
2240 pgdat_resize_unlock(pgdat, &flags);
2244 /* If the zone is empty somebody else may have cleared out the zone */
2245 if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
2246 first_deferred_pfn)) {
2247 pgdat->first_deferred_pfn = ULONG_MAX;
2248 pgdat_resize_unlock(pgdat, &flags);
2249 /* Retry only once. */
2250 return first_deferred_pfn != ULONG_MAX;
2254 * Initialize and free pages in MAX_PAGE_ORDER sized increments so
2255 * that we can avoid introducing any issues with the buddy
2258 while (spfn < epfn) {
2259 /* update our first deferred PFN for this section */
2260 first_deferred_pfn = spfn;
2262 nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
2263 touch_nmi_watchdog();
2265 /* We should only stop along section boundaries */
2266 if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
2269 /* If our quota has been met we can stop here */
2270 if (nr_pages >= nr_pages_needed)
2274 pgdat->first_deferred_pfn = spfn;
2275 pgdat_resize_unlock(pgdat, &flags);
2277 return nr_pages > 0;
2280 #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
2283 void __init init_cma_reserved_pageblock(struct page *page)
2285 unsigned i = pageblock_nr_pages;
2286 struct page *p = page;
2289 __ClearPageReserved(p);
2290 set_page_count(p, 0);
2293 set_pageblock_migratetype(page, MIGRATE_CMA);
2294 set_page_refcounted(page);
2295 __free_pages(page, pageblock_order);
2297 adjust_managed_page_count(page, pageblock_nr_pages);
2298 page_zone(page)->cma_pages += pageblock_nr_pages;
2302 void set_zone_contiguous(struct zone *zone)
2304 unsigned long block_start_pfn = zone->zone_start_pfn;
2305 unsigned long block_end_pfn;
2307 block_end_pfn = pageblock_end_pfn(block_start_pfn);
2308 for (; block_start_pfn < zone_end_pfn(zone);
2309 block_start_pfn = block_end_pfn,
2310 block_end_pfn += pageblock_nr_pages) {
2312 block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
2314 if (!__pageblock_pfn_to_page(block_start_pfn,
2315 block_end_pfn, zone))
2320 /* We confirm that there is no hole */
2321 zone->contiguous = true;
2324 void __init page_alloc_init_late(void)
2329 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
2331 /* There will be num_node_state(N_MEMORY) threads */
2332 atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
2333 for_each_node_state(nid, N_MEMORY) {
2334 kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
2337 /* Block until all are initialised */
2338 wait_for_completion(&pgdat_init_all_done_comp);
2341 * We initialized the rest of the deferred pages. Permanently disable
2342 * on-demand struct page initialization.
2344 static_branch_disable(&deferred_pages);
2346 /* Reinit limits that are based on free pages after the kernel is up */
2347 files_maxfiles_init();
2352 /* Discard memblock private memory */
2355 for_each_node_state(nid, N_MEMORY)
2356 shuffle_free_memory(NODE_DATA(nid));
2358 for_each_populated_zone(zone)
2359 set_zone_contiguous(zone);
2361 /* Initialize page ext after all struct pages are initialized. */
2362 if (deferred_struct_pages)
2365 page_alloc_sysctl_init();
2369 * Adaptive scale is meant to reduce sizes of hash tables on large memory
2370 * machines. As memory size is increased the scale is also increased but at
2371 * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
2372 * quadruples the scale is increased by one, which means the size of hash table
2373 * only doubles, instead of quadrupling as well.
2374 * Because 32-bit systems cannot have large physical memory, where this scaling
2375 * makes sense, it is disabled on such platforms.
2377 #if __BITS_PER_LONG > 32
2378 #define ADAPT_SCALE_BASE (64ul << 30)
2379 #define ADAPT_SCALE_SHIFT 2
2380 #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
2384 * allocate a large system hash table from bootmem
2385 * - it is assumed that the hash table must contain an exact power-of-2
2386 * quantity of entries
2387 * - limit is the number of hash buckets, not the total allocation size
2389 void *__init alloc_large_system_hash(const char *tablename,
2390 unsigned long bucketsize,
2391 unsigned long numentries,
2394 unsigned int *_hash_shift,
2395 unsigned int *_hash_mask,
2396 unsigned long low_limit,
2397 unsigned long high_limit)
2399 unsigned long long max = high_limit;
2400 unsigned long log2qty, size;
2406 /* allow the kernel cmdline to have a say */
2408 /* round applicable memory size up to nearest megabyte */
2409 numentries = nr_kernel_pages;
2411 /* It isn't necessary when PAGE_SIZE >= 1MB */
2412 if (PAGE_SIZE < SZ_1M)
2413 numentries = round_up(numentries, SZ_1M / PAGE_SIZE);
2415 #if __BITS_PER_LONG > 32
2417 unsigned long adapt;
2419 for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
2420 adapt <<= ADAPT_SCALE_SHIFT)
2425 /* limit to 1 bucket per 2^scale bytes of low memory */
2426 if (scale > PAGE_SHIFT)
2427 numentries >>= (scale - PAGE_SHIFT);
2429 numentries <<= (PAGE_SHIFT - scale);
2431 if (unlikely((numentries * bucketsize) < PAGE_SIZE))
2432 numentries = PAGE_SIZE / bucketsize;
2434 numentries = roundup_pow_of_two(numentries);
2436 /* limit allocation size to 1/16 total memory by default */
2438 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
2439 do_div(max, bucketsize);
2441 max = min(max, 0x80000000ULL);
2443 if (numentries < low_limit)
2444 numentries = low_limit;
2445 if (numentries > max)
2448 log2qty = ilog2(numentries);
2450 gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
2453 size = bucketsize << log2qty;
2454 if (flags & HASH_EARLY) {
2455 if (flags & HASH_ZERO)
2456 table = memblock_alloc(size, SMP_CACHE_BYTES);
2458 table = memblock_alloc_raw(size,
2460 } else if (get_order(size) > MAX_PAGE_ORDER || hashdist) {
2461 table = vmalloc_huge(size, gfp_flags);
2464 huge = is_vm_area_hugepages(table);
2467 * If bucketsize is not a power-of-two, we may free
2468 * some pages at the end of hash table which
2469 * alloc_pages_exact() automatically does
2471 table = alloc_pages_exact(size, gfp_flags);
2472 kmemleak_alloc(table, size, 1, gfp_flags);
2474 } while (!table && size > PAGE_SIZE && --log2qty);
2477 panic("Failed to allocate %s hash table\n", tablename);
2479 pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
2480 tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
2481 virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");
2484 *_hash_shift = log2qty;
2486 *_hash_mask = (1 << log2qty) - 1;
2491 void __init memblock_free_pages(struct page *page, unsigned long pfn,
2494 if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) {
2495 int nid = early_pfn_to_nid(pfn);
2497 if (!early_page_initialised(pfn, nid))
2501 if (!kmsan_memblock_free_pages(page, order)) {
2502 /* KMSAN will take care of these pages. */
2506 /* pages were reserved and not allocated */
2507 if (mem_alloc_profiling_enabled()) {
2508 union codetag_ref *ref = get_page_tag_ref(page);
2511 set_codetag_empty(ref);
2512 put_page_tag_ref(ref);
2516 __free_pages_core(page, order);
2519 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
2520 EXPORT_SYMBOL(init_on_alloc);
2522 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
2523 EXPORT_SYMBOL(init_on_free);
2525 DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON, init_mlocked_on_free);
2526 EXPORT_SYMBOL(init_mlocked_on_free);
2528 static bool _init_on_alloc_enabled_early __read_mostly
2529 = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
2530 static int __init early_init_on_alloc(char *buf)
2533 return kstrtobool(buf, &_init_on_alloc_enabled_early);
2535 early_param("init_on_alloc", early_init_on_alloc);
2537 static bool _init_on_free_enabled_early __read_mostly
2538 = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
2539 static int __init early_init_on_free(char *buf)
2541 return kstrtobool(buf, &_init_on_free_enabled_early);
2543 early_param("init_on_free", early_init_on_free);
2545 static bool _init_mlocked_on_free_enabled_early __read_mostly
2546 = IS_ENABLED(CONFIG_INIT_MLOCKED_ON_FREE_DEFAULT_ON);
2547 static int __init early_init_mlocked_on_free(char *buf)
2549 return kstrtobool(buf, &_init_mlocked_on_free_enabled_early);
2551 early_param("init_mlocked_on_free", early_init_mlocked_on_free);
2553 DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);
2556 * Enable static keys related to various memory debugging and hardening options.
2557 * Some override others, and depend on early params that are evaluated in the
2558 * order of appearance. So we need to first gather the full picture of what was
2559 * enabled, and then make decisions.
2561 static void __init mem_debugging_and_hardening_init(void)
2563 bool page_poisoning_requested = false;
2564 bool want_check_pages = false;
2566 #ifdef CONFIG_PAGE_POISONING
2568 * Page poisoning is debug page alloc for some arches. If
2569 * either of those options are enabled, enable poisoning.
2571 if (page_poisoning_enabled() ||
2572 (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
2573 debug_pagealloc_enabled())) {
2574 static_branch_enable(&_page_poisoning_enabled);
2575 page_poisoning_requested = true;
2576 want_check_pages = true;
2580 if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early ||
2581 _init_mlocked_on_free_enabled_early) &&
2582 page_poisoning_requested) {
2583 pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
2584 "will take precedence over init_on_alloc, init_on_free "
2585 "and init_mlocked_on_free\n");
2586 _init_on_alloc_enabled_early = false;
2587 _init_on_free_enabled_early = false;
2588 _init_mlocked_on_free_enabled_early = false;
2591 if (_init_mlocked_on_free_enabled_early && _init_on_free_enabled_early) {
2592 pr_info("mem auto-init: init_on_free is on, "
2593 "will take precedence over init_mlocked_on_free\n");
2594 _init_mlocked_on_free_enabled_early = false;
2597 if (_init_on_alloc_enabled_early) {
2598 want_check_pages = true;
2599 static_branch_enable(&init_on_alloc);
2601 static_branch_disable(&init_on_alloc);
2604 if (_init_on_free_enabled_early) {
2605 want_check_pages = true;
2606 static_branch_enable(&init_on_free);
2608 static_branch_disable(&init_on_free);
2611 if (_init_mlocked_on_free_enabled_early) {
2612 want_check_pages = true;
2613 static_branch_enable(&init_mlocked_on_free);
2615 static_branch_disable(&init_mlocked_on_free);
2618 if (IS_ENABLED(CONFIG_KMSAN) && (_init_on_alloc_enabled_early ||
2619 _init_on_free_enabled_early || _init_mlocked_on_free_enabled_early))
2620 pr_info("mem auto-init: please make sure init_on_alloc, init_on_free and "
2621 "init_mlocked_on_free are disabled when running KMSAN\n");
2623 #ifdef CONFIG_DEBUG_PAGEALLOC
2624 if (debug_pagealloc_enabled()) {
2625 want_check_pages = true;
2626 static_branch_enable(&_debug_pagealloc_enabled);
2628 if (debug_guardpage_minorder())
2629 static_branch_enable(&_debug_guardpage_enabled);
2634 * Any page debugging or hardening option also enables sanity checking
2635 * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's
2638 if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages)
2639 static_branch_enable(&check_pages_enabled);
2642 /* Report memory auto-initialization states for this boot. */
2643 static void __init report_meminit(void)
2647 if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN))
2648 stack = "all(pattern)";
2649 else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO))
2650 stack = "all(zero)";
2651 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL))
2652 stack = "byref_all(zero)";
2653 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF))
2654 stack = "byref(zero)";
2655 else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER))
2656 stack = "__user(zero)";
2660 pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s, mlocked free:%s\n",
2661 stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off",
2662 want_init_on_free() ? "on" : "off",
2663 want_init_mlocked_on_free() ? "on" : "off");
2664 if (want_init_on_free())
2665 pr_info("mem auto-init: clearing system memory may take some time...\n");
2668 static void __init mem_init_print_info(void)
2670 unsigned long physpages, codesize, datasize, rosize, bss_size;
2671 unsigned long init_code_size, init_data_size;
2673 physpages = get_num_physpages();
2674 codesize = _etext - _stext;
2675 datasize = _edata - _sdata;
2676 rosize = __end_rodata - __start_rodata;
2677 bss_size = __bss_stop - __bss_start;
2678 init_data_size = __init_end - __init_begin;
2679 init_code_size = _einittext - _sinittext;
2682 * Detect special cases and adjust section sizes accordingly:
2683 * 1) .init.* may be embedded into .data sections
2684 * 2) .init.text.* may be out of [__init_begin, __init_end],
2685 * please refer to arch/tile/kernel/vmlinux.lds.S.
2686 * 3) .rodata.* may be embedded into .text or .data sections.
2688 #define adj_init_size(start, end, size, pos, adj) \
2690 if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
2694 adj_init_size(__init_begin, __init_end, init_data_size,
2695 _sinittext, init_code_size);
2696 adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
2697 adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
2698 adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
2699 adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
2701 #undef adj_init_size
2703 pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
2704 #ifdef CONFIG_HIGHMEM
2708 K(nr_free_pages()), K(physpages),
2709 codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K,
2710 (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K,
2711 K(physpages - totalram_pages() - totalcma_pages),
2713 #ifdef CONFIG_HIGHMEM
2714 , K(totalhigh_pages())
2720 * Set up kernel memory allocators
2722 void __init mm_core_init(void)
2724 /* Initializations relying on SMP setup */
2725 build_all_zonelists(NULL);
2726 page_alloc_init_cpuhp();
2729 * page_ext requires contiguous pages,
2730 * bigger than MAX_PAGE_ORDER unless SPARSEMEM.
2732 page_ext_init_flatmem();
2733 mem_debugging_and_hardening_init();
2734 kfence_alloc_pool_and_metadata();
2736 kmsan_init_shadow();
2737 stack_depot_early_init();
2739 mem_init_print_info();
2742 * page_owner must be initialized after buddy is ready, and also after
2743 * slab is ready so that stack_depot_init() works properly
2745 page_ext_init_flatmem_late();
2747 ptlock_cache_init();
2748 pgtable_cache_init();
2749 debug_objects_mem_init();
2751 /* If no deferred init page_ext now, as vmap is fully initialized */
2752 if (!deferred_struct_pages)
2754 /* Should be run before the first non-init thread is created */
2756 /* Should be run after espfix64 is set up. */
2758 kmsan_init_runtime();