| 1 | // SPDX-License-Identifier: GPL-2.0-only |
| 2 | /* |
| 3 | * mm_init.c - Memory initialisation verification and debugging |
| 4 | * |
| 5 | * Copyright 2008 IBM Corporation, 2008 |
| 6 | * Author Mel Gorman <mel@csn.ul.ie> |
| 7 | * |
| 8 | */ |
| 9 | #include <linux/kernel.h> |
| 10 | #include <linux/init.h> |
| 11 | #include <linux/kobject.h> |
| 12 | #include <linux/export.h> |
| 13 | #include <linux/memory.h> |
| 14 | #include <linux/notifier.h> |
| 15 | #include <linux/sched.h> |
| 16 | #include <linux/mman.h> |
| 17 | #include <linux/memblock.h> |
| 18 | #include <linux/page-isolation.h> |
| 19 | #include <linux/padata.h> |
| 20 | #include <linux/nmi.h> |
| 21 | #include <linux/buffer_head.h> |
| 22 | #include <linux/kmemleak.h> |
| 23 | #include <linux/kfence.h> |
| 24 | #include <linux/page_ext.h> |
| 25 | #include <linux/pti.h> |
| 26 | #include <linux/pgtable.h> |
| 27 | #include <linux/stackdepot.h> |
| 28 | #include <linux/swap.h> |
| 29 | #include <linux/cma.h> |
| 30 | #include <linux/crash_dump.h> |
| 31 | #include <linux/execmem.h> |
| 32 | #include <linux/vmstat.h> |
| 33 | #include "internal.h" |
| 34 | #include "slab.h" |
| 35 | #include "shuffle.h" |
| 36 | |
| 37 | #include <asm/setup.h> |
| 38 | |
| 39 | #ifdef CONFIG_DEBUG_MEMORY_INIT |
| 40 | int __meminitdata mminit_loglevel; |
| 41 | |
| 42 | /* The zonelists are simply reported, validation is manual. */ |
| 43 | void __init mminit_verify_zonelist(void) |
| 44 | { |
| 45 | int nid; |
| 46 | |
| 47 | if (mminit_loglevel < MMINIT_VERIFY) |
| 48 | return; |
| 49 | |
| 50 | for_each_online_node(nid) { |
| 51 | pg_data_t *pgdat = NODE_DATA(nid); |
| 52 | struct zone *zone; |
| 53 | struct zoneref *z; |
| 54 | struct zonelist *zonelist; |
| 55 | int i, listid, zoneid; |
| 56 | |
| 57 | for (i = 0; i < MAX_ZONELISTS * MAX_NR_ZONES; i++) { |
| 58 | |
| 59 | /* Identify the zone and nodelist */ |
| 60 | zoneid = i % MAX_NR_ZONES; |
| 61 | listid = i / MAX_NR_ZONES; |
| 62 | zonelist = &pgdat->node_zonelists[listid]; |
| 63 | zone = &pgdat->node_zones[zoneid]; |
| 64 | if (!populated_zone(zone)) |
| 65 | continue; |
| 66 | |
| 67 | /* Print information about the zonelist */ |
| 68 | printk(KERN_DEBUG "mminit::zonelist %s %d:%s = ", |
| 69 | listid > 0 ? "thisnode" : "general", nid, |
| 70 | zone->name); |
| 71 | |
| 72 | /* Iterate the zonelist */ |
| 73 | for_each_zone_zonelist(zone, z, zonelist, zoneid) |
| 74 | pr_cont("%d:%s ", zone_to_nid(zone), zone->name); |
| 75 | pr_cont("\n"); |
| 76 | } |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | void __init mminit_verify_pageflags_layout(void) |
| 81 | { |
| 82 | int shift, width; |
| 83 | unsigned long or_mask, add_mask; |
| 84 | |
| 85 | shift = BITS_PER_LONG; |
| 86 | width = shift - SECTIONS_WIDTH - NODES_WIDTH - ZONES_WIDTH |
| 87 | - LAST_CPUPID_SHIFT - KASAN_TAG_WIDTH - LRU_GEN_WIDTH - LRU_REFS_WIDTH; |
| 88 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_widths", |
| 89 | "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d Gen %d Tier %d Flags %d\n", |
| 90 | SECTIONS_WIDTH, |
| 91 | NODES_WIDTH, |
| 92 | ZONES_WIDTH, |
| 93 | LAST_CPUPID_WIDTH, |
| 94 | KASAN_TAG_WIDTH, |
| 95 | LRU_GEN_WIDTH, |
| 96 | LRU_REFS_WIDTH, |
| 97 | NR_PAGEFLAGS); |
| 98 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_shifts", |
| 99 | "Section %d Node %d Zone %d Lastcpupid %d Kasantag %d\n", |
| 100 | SECTIONS_SHIFT, |
| 101 | NODES_SHIFT, |
| 102 | ZONES_SHIFT, |
| 103 | LAST_CPUPID_SHIFT, |
| 104 | KASAN_TAG_WIDTH); |
| 105 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_pgshifts", |
| 106 | "Section %lu Node %lu Zone %lu Lastcpupid %lu Kasantag %lu\n", |
| 107 | (unsigned long)SECTIONS_PGSHIFT, |
| 108 | (unsigned long)NODES_PGSHIFT, |
| 109 | (unsigned long)ZONES_PGSHIFT, |
| 110 | (unsigned long)LAST_CPUPID_PGSHIFT, |
| 111 | (unsigned long)KASAN_TAG_PGSHIFT); |
| 112 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodezoneid", |
| 113 | "Node/Zone ID: %lu -> %lu\n", |
| 114 | (unsigned long)(ZONEID_PGOFF + ZONEID_SHIFT), |
| 115 | (unsigned long)ZONEID_PGOFF); |
| 116 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_usage", |
| 117 | "location: %d -> %d layout %d -> %d unused %d -> %d page-flags\n", |
| 118 | shift, width, width, NR_PAGEFLAGS, NR_PAGEFLAGS, 0); |
| 119 | #ifdef NODE_NOT_IN_PAGE_FLAGS |
| 120 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", |
| 121 | "Node not in page flags"); |
| 122 | #endif |
| 123 | #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS |
| 124 | mminit_dprintk(MMINIT_TRACE, "pageflags_layout_nodeflags", |
| 125 | "Last cpupid not in page flags"); |
| 126 | #endif |
| 127 | |
| 128 | if (SECTIONS_WIDTH) { |
| 129 | shift -= SECTIONS_WIDTH; |
| 130 | BUG_ON(shift != SECTIONS_PGSHIFT); |
| 131 | } |
| 132 | if (NODES_WIDTH) { |
| 133 | shift -= NODES_WIDTH; |
| 134 | BUG_ON(shift != NODES_PGSHIFT); |
| 135 | } |
| 136 | if (ZONES_WIDTH) { |
| 137 | shift -= ZONES_WIDTH; |
| 138 | BUG_ON(shift != ZONES_PGSHIFT); |
| 139 | } |
| 140 | |
| 141 | /* Check for bitmask overlaps */ |
| 142 | or_mask = (ZONES_MASK << ZONES_PGSHIFT) | |
| 143 | (NODES_MASK << NODES_PGSHIFT) | |
| 144 | (SECTIONS_MASK << SECTIONS_PGSHIFT); |
| 145 | add_mask = (ZONES_MASK << ZONES_PGSHIFT) + |
| 146 | (NODES_MASK << NODES_PGSHIFT) + |
| 147 | (SECTIONS_MASK << SECTIONS_PGSHIFT); |
| 148 | BUG_ON(or_mask != add_mask); |
| 149 | } |
| 150 | |
| 151 | static __init int set_mminit_loglevel(char *str) |
| 152 | { |
| 153 | get_option(&str, &mminit_loglevel); |
| 154 | return 0; |
| 155 | } |
| 156 | early_param("mminit_loglevel", set_mminit_loglevel); |
| 157 | #endif /* CONFIG_DEBUG_MEMORY_INIT */ |
| 158 | |
| 159 | struct kobject *mm_kobj; |
| 160 | |
| 161 | #ifdef CONFIG_SMP |
| 162 | s32 vm_committed_as_batch = 32; |
| 163 | |
| 164 | void mm_compute_batch(int overcommit_policy) |
| 165 | { |
| 166 | u64 memsized_batch; |
| 167 | s32 nr = num_present_cpus(); |
| 168 | s32 batch = max_t(s32, nr*2, 32); |
| 169 | unsigned long ram_pages = totalram_pages(); |
| 170 | |
| 171 | /* |
| 172 | * For policy OVERCOMMIT_NEVER, set batch size to 0.4% of |
| 173 | * (total memory/#cpus), and lift it to 25% for other policies |
| 174 | * to easy the possible lock contention for percpu_counter |
| 175 | * vm_committed_as, while the max limit is INT_MAX |
| 176 | */ |
| 177 | if (overcommit_policy == OVERCOMMIT_NEVER) |
| 178 | memsized_batch = min_t(u64, ram_pages/nr/256, INT_MAX); |
| 179 | else |
| 180 | memsized_batch = min_t(u64, ram_pages/nr/4, INT_MAX); |
| 181 | |
| 182 | vm_committed_as_batch = max_t(s32, memsized_batch, batch); |
| 183 | } |
| 184 | |
| 185 | static int __meminit mm_compute_batch_notifier(struct notifier_block *self, |
| 186 | unsigned long action, void *arg) |
| 187 | { |
| 188 | switch (action) { |
| 189 | case MEM_ONLINE: |
| 190 | case MEM_OFFLINE: |
| 191 | mm_compute_batch(sysctl_overcommit_memory); |
| 192 | break; |
| 193 | default: |
| 194 | break; |
| 195 | } |
| 196 | return NOTIFY_OK; |
| 197 | } |
| 198 | |
| 199 | static int __init mm_compute_batch_init(void) |
| 200 | { |
| 201 | mm_compute_batch(sysctl_overcommit_memory); |
| 202 | hotplug_memory_notifier(mm_compute_batch_notifier, MM_COMPUTE_BATCH_PRI); |
| 203 | return 0; |
| 204 | } |
| 205 | |
| 206 | __initcall(mm_compute_batch_init); |
| 207 | |
| 208 | #endif |
| 209 | |
| 210 | static int __init mm_sysfs_init(void) |
| 211 | { |
| 212 | mm_kobj = kobject_create_and_add("mm", kernel_kobj); |
| 213 | if (!mm_kobj) |
| 214 | return -ENOMEM; |
| 215 | |
| 216 | return 0; |
| 217 | } |
| 218 | postcore_initcall(mm_sysfs_init); |
| 219 | |
| 220 | static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata; |
| 221 | static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata; |
| 222 | static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata; |
| 223 | |
| 224 | static unsigned long required_kernelcore __initdata; |
| 225 | static unsigned long required_kernelcore_percent __initdata; |
| 226 | static unsigned long required_movablecore __initdata; |
| 227 | static unsigned long required_movablecore_percent __initdata; |
| 228 | |
| 229 | static unsigned long nr_kernel_pages __initdata; |
| 230 | static unsigned long nr_all_pages __initdata; |
| 231 | |
| 232 | static bool deferred_struct_pages __meminitdata; |
| 233 | |
| 234 | static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats); |
| 235 | |
| 236 | static int __init cmdline_parse_core(char *p, unsigned long *core, |
| 237 | unsigned long *percent) |
| 238 | { |
| 239 | unsigned long long coremem; |
| 240 | char *endptr; |
| 241 | |
| 242 | if (!p) |
| 243 | return -EINVAL; |
| 244 | |
| 245 | /* Value may be a percentage of total memory, otherwise bytes */ |
| 246 | coremem = simple_strtoull(p, &endptr, 0); |
| 247 | if (*endptr == '%') { |
| 248 | /* Paranoid check for percent values greater than 100 */ |
| 249 | WARN_ON(coremem > 100); |
| 250 | |
| 251 | *percent = coremem; |
| 252 | } else { |
| 253 | coremem = memparse(p, &p); |
| 254 | /* Paranoid check that UL is enough for the coremem value */ |
| 255 | WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); |
| 256 | |
| 257 | *core = coremem >> PAGE_SHIFT; |
| 258 | *percent = 0UL; |
| 259 | } |
| 260 | return 0; |
| 261 | } |
| 262 | |
| 263 | bool mirrored_kernelcore __initdata_memblock; |
| 264 | |
| 265 | /* |
| 266 | * kernelcore=size sets the amount of memory for use for allocations that |
| 267 | * cannot be reclaimed or migrated. |
| 268 | */ |
| 269 | static int __init cmdline_parse_kernelcore(char *p) |
| 270 | { |
| 271 | /* parse kernelcore=mirror */ |
| 272 | if (parse_option_str(p, "mirror")) { |
| 273 | mirrored_kernelcore = true; |
| 274 | return 0; |
| 275 | } |
| 276 | |
| 277 | return cmdline_parse_core(p, &required_kernelcore, |
| 278 | &required_kernelcore_percent); |
| 279 | } |
| 280 | early_param("kernelcore", cmdline_parse_kernelcore); |
| 281 | |
| 282 | /* |
| 283 | * movablecore=size sets the amount of memory for use for allocations that |
| 284 | * can be reclaimed or migrated. |
| 285 | */ |
| 286 | static int __init cmdline_parse_movablecore(char *p) |
| 287 | { |
| 288 | return cmdline_parse_core(p, &required_movablecore, |
| 289 | &required_movablecore_percent); |
| 290 | } |
| 291 | early_param("movablecore", cmdline_parse_movablecore); |
| 292 | |
| 293 | /* |
| 294 | * early_calculate_totalpages() |
| 295 | * Sum pages in active regions for movable zone. |
| 296 | * Populate N_MEMORY for calculating usable_nodes. |
| 297 | */ |
| 298 | static unsigned long __init early_calculate_totalpages(void) |
| 299 | { |
| 300 | unsigned long totalpages = 0; |
| 301 | unsigned long start_pfn, end_pfn; |
| 302 | int i, nid; |
| 303 | |
| 304 | for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { |
| 305 | unsigned long pages = end_pfn - start_pfn; |
| 306 | |
| 307 | totalpages += pages; |
| 308 | if (pages) |
| 309 | node_set_state(nid, N_MEMORY); |
| 310 | } |
| 311 | return totalpages; |
| 312 | } |
| 313 | |
| 314 | /* |
| 315 | * This finds a zone that can be used for ZONE_MOVABLE pages. The |
| 316 | * assumption is made that zones within a node are ordered in monotonic |
| 317 | * increasing memory addresses so that the "highest" populated zone is used |
| 318 | */ |
| 319 | static void __init find_usable_zone_for_movable(void) |
| 320 | { |
| 321 | int zone_index; |
| 322 | for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { |
| 323 | if (zone_index == ZONE_MOVABLE) |
| 324 | continue; |
| 325 | |
| 326 | if (arch_zone_highest_possible_pfn[zone_index] > |
| 327 | arch_zone_lowest_possible_pfn[zone_index]) |
| 328 | break; |
| 329 | } |
| 330 | |
| 331 | VM_BUG_ON(zone_index == -1); |
| 332 | movable_zone = zone_index; |
| 333 | } |
| 334 | |
| 335 | /* |
| 336 | * Find the PFN the Movable zone begins in each node. Kernel memory |
| 337 | * is spread evenly between nodes as long as the nodes have enough |
| 338 | * memory. When they don't, some nodes will have more kernelcore than |
| 339 | * others |
| 340 | */ |
| 341 | static void __init find_zone_movable_pfns_for_nodes(void) |
| 342 | { |
| 343 | int i, nid; |
| 344 | unsigned long usable_startpfn; |
| 345 | unsigned long kernelcore_node, kernelcore_remaining; |
| 346 | /* save the state before borrow the nodemask */ |
| 347 | nodemask_t saved_node_state = node_states[N_MEMORY]; |
| 348 | unsigned long totalpages = early_calculate_totalpages(); |
| 349 | int usable_nodes = nodes_weight(node_states[N_MEMORY]); |
| 350 | struct memblock_region *r; |
| 351 | |
| 352 | /* Need to find movable_zone earlier when movable_node is specified. */ |
| 353 | find_usable_zone_for_movable(); |
| 354 | |
| 355 | /* |
| 356 | * If movable_node is specified, ignore kernelcore and movablecore |
| 357 | * options. |
| 358 | */ |
| 359 | if (movable_node_is_enabled()) { |
| 360 | for_each_mem_region(r) { |
| 361 | if (!memblock_is_hotpluggable(r)) |
| 362 | continue; |
| 363 | |
| 364 | nid = memblock_get_region_node(r); |
| 365 | |
| 366 | usable_startpfn = memblock_region_memory_base_pfn(r); |
| 367 | zone_movable_pfn[nid] = zone_movable_pfn[nid] ? |
| 368 | min(usable_startpfn, zone_movable_pfn[nid]) : |
| 369 | usable_startpfn; |
| 370 | } |
| 371 | |
| 372 | goto out2; |
| 373 | } |
| 374 | |
| 375 | /* |
| 376 | * If kernelcore=mirror is specified, ignore movablecore option |
| 377 | */ |
| 378 | if (mirrored_kernelcore) { |
| 379 | bool mem_below_4gb_not_mirrored = false; |
| 380 | |
| 381 | if (!memblock_has_mirror()) { |
| 382 | pr_warn("The system has no mirror memory, ignore kernelcore=mirror.\n"); |
| 383 | goto out; |
| 384 | } |
| 385 | |
| 386 | if (is_kdump_kernel()) { |
| 387 | pr_warn("The system is under kdump, ignore kernelcore=mirror.\n"); |
| 388 | goto out; |
| 389 | } |
| 390 | |
| 391 | for_each_mem_region(r) { |
| 392 | if (memblock_is_mirror(r)) |
| 393 | continue; |
| 394 | |
| 395 | nid = memblock_get_region_node(r); |
| 396 | |
| 397 | usable_startpfn = memblock_region_memory_base_pfn(r); |
| 398 | |
| 399 | if (usable_startpfn < PHYS_PFN(SZ_4G)) { |
| 400 | mem_below_4gb_not_mirrored = true; |
| 401 | continue; |
| 402 | } |
| 403 | |
| 404 | zone_movable_pfn[nid] = zone_movable_pfn[nid] ? |
| 405 | min(usable_startpfn, zone_movable_pfn[nid]) : |
| 406 | usable_startpfn; |
| 407 | } |
| 408 | |
| 409 | if (mem_below_4gb_not_mirrored) |
| 410 | pr_warn("This configuration results in unmirrored kernel memory.\n"); |
| 411 | |
| 412 | goto out2; |
| 413 | } |
| 414 | |
| 415 | /* |
| 416 | * If kernelcore=nn% or movablecore=nn% was specified, calculate the |
| 417 | * amount of necessary memory. |
| 418 | */ |
| 419 | if (required_kernelcore_percent) |
| 420 | required_kernelcore = (totalpages * 100 * required_kernelcore_percent) / |
| 421 | 10000UL; |
| 422 | if (required_movablecore_percent) |
| 423 | required_movablecore = (totalpages * 100 * required_movablecore_percent) / |
| 424 | 10000UL; |
| 425 | |
| 426 | /* |
| 427 | * If movablecore= was specified, calculate what size of |
| 428 | * kernelcore that corresponds so that memory usable for |
| 429 | * any allocation type is evenly spread. If both kernelcore |
| 430 | * and movablecore are specified, then the value of kernelcore |
| 431 | * will be used for required_kernelcore if it's greater than |
| 432 | * what movablecore would have allowed. |
| 433 | */ |
| 434 | if (required_movablecore) { |
| 435 | unsigned long corepages; |
| 436 | |
| 437 | /* |
| 438 | * Round-up so that ZONE_MOVABLE is at least as large as what |
| 439 | * was requested by the user |
| 440 | */ |
| 441 | required_movablecore = |
| 442 | roundup(required_movablecore, MAX_ORDER_NR_PAGES); |
| 443 | required_movablecore = min(totalpages, required_movablecore); |
| 444 | corepages = totalpages - required_movablecore; |
| 445 | |
| 446 | required_kernelcore = max(required_kernelcore, corepages); |
| 447 | } |
| 448 | |
| 449 | /* |
| 450 | * If kernelcore was not specified or kernelcore size is larger |
| 451 | * than totalpages, there is no ZONE_MOVABLE. |
| 452 | */ |
| 453 | if (!required_kernelcore || required_kernelcore >= totalpages) |
| 454 | goto out; |
| 455 | |
| 456 | /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ |
| 457 | usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; |
| 458 | |
| 459 | restart: |
| 460 | /* Spread kernelcore memory as evenly as possible throughout nodes */ |
| 461 | kernelcore_node = required_kernelcore / usable_nodes; |
| 462 | for_each_node_state(nid, N_MEMORY) { |
| 463 | unsigned long start_pfn, end_pfn; |
| 464 | |
| 465 | /* |
| 466 | * Recalculate kernelcore_node if the division per node |
| 467 | * now exceeds what is necessary to satisfy the requested |
| 468 | * amount of memory for the kernel |
| 469 | */ |
| 470 | if (required_kernelcore < kernelcore_node) |
| 471 | kernelcore_node = required_kernelcore / usable_nodes; |
| 472 | |
| 473 | /* |
| 474 | * As the map is walked, we track how much memory is usable |
| 475 | * by the kernel using kernelcore_remaining. When it is |
| 476 | * 0, the rest of the node is usable by ZONE_MOVABLE |
| 477 | */ |
| 478 | kernelcore_remaining = kernelcore_node; |
| 479 | |
| 480 | /* Go through each range of PFNs within this node */ |
| 481 | for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { |
| 482 | unsigned long size_pages; |
| 483 | |
| 484 | start_pfn = max(start_pfn, zone_movable_pfn[nid]); |
| 485 | if (start_pfn >= end_pfn) |
| 486 | continue; |
| 487 | |
| 488 | /* Account for what is only usable for kernelcore */ |
| 489 | if (start_pfn < usable_startpfn) { |
| 490 | unsigned long kernel_pages; |
| 491 | kernel_pages = min(end_pfn, usable_startpfn) |
| 492 | - start_pfn; |
| 493 | |
| 494 | kernelcore_remaining -= min(kernel_pages, |
| 495 | kernelcore_remaining); |
| 496 | required_kernelcore -= min(kernel_pages, |
| 497 | required_kernelcore); |
| 498 | |
| 499 | /* Continue if range is now fully accounted */ |
| 500 | if (end_pfn <= usable_startpfn) { |
| 501 | |
| 502 | /* |
| 503 | * Push zone_movable_pfn to the end so |
| 504 | * that if we have to rebalance |
| 505 | * kernelcore across nodes, we will |
| 506 | * not double account here |
| 507 | */ |
| 508 | zone_movable_pfn[nid] = end_pfn; |
| 509 | continue; |
| 510 | } |
| 511 | start_pfn = usable_startpfn; |
| 512 | } |
| 513 | |
| 514 | /* |
| 515 | * The usable PFN range for ZONE_MOVABLE is from |
| 516 | * start_pfn->end_pfn. Calculate size_pages as the |
| 517 | * number of pages used as kernelcore |
| 518 | */ |
| 519 | size_pages = end_pfn - start_pfn; |
| 520 | if (size_pages > kernelcore_remaining) |
| 521 | size_pages = kernelcore_remaining; |
| 522 | zone_movable_pfn[nid] = start_pfn + size_pages; |
| 523 | |
| 524 | /* |
| 525 | * Some kernelcore has been met, update counts and |
| 526 | * break if the kernelcore for this node has been |
| 527 | * satisfied |
| 528 | */ |
| 529 | required_kernelcore -= min(required_kernelcore, |
| 530 | size_pages); |
| 531 | kernelcore_remaining -= size_pages; |
| 532 | if (!kernelcore_remaining) |
| 533 | break; |
| 534 | } |
| 535 | } |
| 536 | |
| 537 | /* |
| 538 | * If there is still required_kernelcore, we do another pass with one |
| 539 | * less node in the count. This will push zone_movable_pfn[nid] further |
| 540 | * along on the nodes that still have memory until kernelcore is |
| 541 | * satisfied |
| 542 | */ |
| 543 | usable_nodes--; |
| 544 | if (usable_nodes && required_kernelcore > usable_nodes) |
| 545 | goto restart; |
| 546 | |
| 547 | out2: |
| 548 | /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ |
| 549 | for (nid = 0; nid < MAX_NUMNODES; nid++) { |
| 550 | unsigned long start_pfn, end_pfn; |
| 551 | |
| 552 | zone_movable_pfn[nid] = |
| 553 | roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); |
| 554 | |
| 555 | get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); |
| 556 | if (zone_movable_pfn[nid] >= end_pfn) |
| 557 | zone_movable_pfn[nid] = 0; |
| 558 | } |
| 559 | |
| 560 | out: |
| 561 | /* restore the node_state */ |
| 562 | node_states[N_MEMORY] = saved_node_state; |
| 563 | } |
| 564 | |
| 565 | void __meminit __init_single_page(struct page *page, unsigned long pfn, |
| 566 | unsigned long zone, int nid) |
| 567 | { |
| 568 | mm_zero_struct_page(page); |
| 569 | set_page_links(page, zone, nid, pfn); |
| 570 | init_page_count(page); |
| 571 | atomic_set(&page->_mapcount, -1); |
| 572 | page_cpupid_reset_last(page); |
| 573 | page_kasan_tag_reset(page); |
| 574 | |
| 575 | INIT_LIST_HEAD(&page->lru); |
| 576 | #ifdef WANT_PAGE_VIRTUAL |
| 577 | /* The shift won't overflow because ZONE_NORMAL is below 4G. */ |
| 578 | if (!is_highmem_idx(zone)) |
| 579 | set_page_address(page, __va(pfn << PAGE_SHIFT)); |
| 580 | #endif |
| 581 | } |
| 582 | |
| 583 | #ifdef CONFIG_NUMA |
| 584 | /* |
| 585 | * During memory init memblocks map pfns to nids. The search is expensive and |
| 586 | * this caches recent lookups. The implementation of __early_pfn_to_nid |
| 587 | * treats start/end as pfns. |
| 588 | */ |
| 589 | struct mminit_pfnnid_cache { |
| 590 | unsigned long last_start; |
| 591 | unsigned long last_end; |
| 592 | int last_nid; |
| 593 | }; |
| 594 | |
| 595 | static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata; |
| 596 | |
| 597 | /* |
| 598 | * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. |
| 599 | */ |
| 600 | static int __meminit __early_pfn_to_nid(unsigned long pfn, |
| 601 | struct mminit_pfnnid_cache *state) |
| 602 | { |
| 603 | unsigned long start_pfn, end_pfn; |
| 604 | int nid; |
| 605 | |
| 606 | if (state->last_start <= pfn && pfn < state->last_end) |
| 607 | return state->last_nid; |
| 608 | |
| 609 | nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); |
| 610 | if (nid != NUMA_NO_NODE) { |
| 611 | state->last_start = start_pfn; |
| 612 | state->last_end = end_pfn; |
| 613 | state->last_nid = nid; |
| 614 | } |
| 615 | |
| 616 | return nid; |
| 617 | } |
| 618 | |
| 619 | int __meminit early_pfn_to_nid(unsigned long pfn) |
| 620 | { |
| 621 | static DEFINE_SPINLOCK(early_pfn_lock); |
| 622 | int nid; |
| 623 | |
| 624 | spin_lock(&early_pfn_lock); |
| 625 | nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache); |
| 626 | if (nid < 0) |
| 627 | nid = first_online_node; |
| 628 | spin_unlock(&early_pfn_lock); |
| 629 | |
| 630 | return nid; |
| 631 | } |
| 632 | |
| 633 | int hashdist = HASHDIST_DEFAULT; |
| 634 | |
| 635 | static int __init set_hashdist(char *str) |
| 636 | { |
| 637 | if (!str) |
| 638 | return 0; |
| 639 | hashdist = simple_strtoul(str, &str, 0); |
| 640 | return 1; |
| 641 | } |
| 642 | __setup("hashdist=", set_hashdist); |
| 643 | |
| 644 | static inline void fixup_hashdist(void) |
| 645 | { |
| 646 | if (num_node_state(N_MEMORY) == 1) |
| 647 | hashdist = 0; |
| 648 | } |
| 649 | #else |
| 650 | static inline void fixup_hashdist(void) {} |
| 651 | #endif /* CONFIG_NUMA */ |
| 652 | |
| 653 | #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT |
| 654 | static inline void pgdat_set_deferred_range(pg_data_t *pgdat) |
| 655 | { |
| 656 | pgdat->first_deferred_pfn = ULONG_MAX; |
| 657 | } |
| 658 | |
| 659 | /* Returns true if the struct page for the pfn is initialised */ |
| 660 | static inline bool __meminit early_page_initialised(unsigned long pfn, int nid) |
| 661 | { |
| 662 | if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) |
| 663 | return false; |
| 664 | |
| 665 | return true; |
| 666 | } |
| 667 | |
| 668 | /* |
| 669 | * Returns true when the remaining initialisation should be deferred until |
| 670 | * later in the boot cycle when it can be parallelised. |
| 671 | */ |
| 672 | static bool __meminit |
| 673 | defer_init(int nid, unsigned long pfn, unsigned long end_pfn) |
| 674 | { |
| 675 | static unsigned long prev_end_pfn, nr_initialised; |
| 676 | |
| 677 | if (early_page_ext_enabled()) |
| 678 | return false; |
| 679 | |
| 680 | /* Always populate low zones for address-constrained allocations */ |
| 681 | if (end_pfn < pgdat_end_pfn(NODE_DATA(nid))) |
| 682 | return false; |
| 683 | |
| 684 | if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX) |
| 685 | return true; |
| 686 | |
| 687 | /* |
| 688 | * prev_end_pfn static that contains the end of previous zone |
| 689 | * No need to protect because called very early in boot before smp_init. |
| 690 | */ |
| 691 | if (prev_end_pfn != end_pfn) { |
| 692 | prev_end_pfn = end_pfn; |
| 693 | nr_initialised = 0; |
| 694 | } |
| 695 | |
| 696 | /* |
| 697 | * We start only with one section of pages, more pages are added as |
| 698 | * needed until the rest of deferred pages are initialized. |
| 699 | */ |
| 700 | nr_initialised++; |
| 701 | if ((nr_initialised > PAGES_PER_SECTION) && |
| 702 | (pfn & (PAGES_PER_SECTION - 1)) == 0) { |
| 703 | NODE_DATA(nid)->first_deferred_pfn = pfn; |
| 704 | return true; |
| 705 | } |
| 706 | return false; |
| 707 | } |
| 708 | |
| 709 | static void __meminit init_reserved_page(unsigned long pfn, int nid) |
| 710 | { |
| 711 | pg_data_t *pgdat; |
| 712 | int zid; |
| 713 | |
| 714 | if (early_page_initialised(pfn, nid)) |
| 715 | return; |
| 716 | |
| 717 | pgdat = NODE_DATA(nid); |
| 718 | |
| 719 | for (zid = 0; zid < MAX_NR_ZONES; zid++) { |
| 720 | struct zone *zone = &pgdat->node_zones[zid]; |
| 721 | |
| 722 | if (zone_spans_pfn(zone, pfn)) |
| 723 | break; |
| 724 | } |
| 725 | __init_single_page(pfn_to_page(pfn), pfn, zid, nid); |
| 726 | } |
| 727 | #else |
| 728 | static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {} |
| 729 | |
| 730 | static inline bool early_page_initialised(unsigned long pfn, int nid) |
| 731 | { |
| 732 | return true; |
| 733 | } |
| 734 | |
| 735 | static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn) |
| 736 | { |
| 737 | return false; |
| 738 | } |
| 739 | |
| 740 | static inline void init_reserved_page(unsigned long pfn, int nid) |
| 741 | { |
| 742 | } |
| 743 | #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ |
| 744 | |
| 745 | /* |
| 746 | * Initialised pages do not have PageReserved set. This function is |
| 747 | * called for each range allocated by the bootmem allocator and |
| 748 | * marks the pages PageReserved. The remaining valid pages are later |
| 749 | * sent to the buddy page allocator. |
| 750 | */ |
| 751 | void __meminit reserve_bootmem_region(phys_addr_t start, |
| 752 | phys_addr_t end, int nid) |
| 753 | { |
| 754 | unsigned long start_pfn = PFN_DOWN(start); |
| 755 | unsigned long end_pfn = PFN_UP(end); |
| 756 | |
| 757 | for (; start_pfn < end_pfn; start_pfn++) { |
| 758 | if (pfn_valid(start_pfn)) { |
| 759 | struct page *page = pfn_to_page(start_pfn); |
| 760 | |
| 761 | init_reserved_page(start_pfn, nid); |
| 762 | |
| 763 | /* |
| 764 | * no need for atomic set_bit because the struct |
| 765 | * page is not visible yet so nobody should |
| 766 | * access it yet. |
| 767 | */ |
| 768 | __SetPageReserved(page); |
| 769 | } |
| 770 | } |
| 771 | } |
| 772 | |
| 773 | /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */ |
| 774 | static bool __meminit |
| 775 | overlap_memmap_init(unsigned long zone, unsigned long *pfn) |
| 776 | { |
| 777 | static struct memblock_region *r; |
| 778 | |
| 779 | if (mirrored_kernelcore && zone == ZONE_MOVABLE) { |
| 780 | if (!r || *pfn >= memblock_region_memory_end_pfn(r)) { |
| 781 | for_each_mem_region(r) { |
| 782 | if (*pfn < memblock_region_memory_end_pfn(r)) |
| 783 | break; |
| 784 | } |
| 785 | } |
| 786 | if (*pfn >= memblock_region_memory_base_pfn(r) && |
| 787 | memblock_is_mirror(r)) { |
| 788 | *pfn = memblock_region_memory_end_pfn(r); |
| 789 | return true; |
| 790 | } |
| 791 | } |
| 792 | return false; |
| 793 | } |
| 794 | |
| 795 | /* |
| 796 | * Only struct pages that correspond to ranges defined by memblock.memory |
| 797 | * are zeroed and initialized by going through __init_single_page() during |
| 798 | * memmap_init_zone_range(). |
| 799 | * |
| 800 | * But, there could be struct pages that correspond to holes in |
| 801 | * memblock.memory. This can happen because of the following reasons: |
| 802 | * - physical memory bank size is not necessarily the exact multiple of the |
| 803 | * arbitrary section size |
| 804 | * - early reserved memory may not be listed in memblock.memory |
| 805 | * - non-memory regions covered by the contigious flatmem mapping |
| 806 | * - memory layouts defined with memmap= kernel parameter may not align |
| 807 | * nicely with memmap sections |
| 808 | * |
| 809 | * Explicitly initialize those struct pages so that: |
| 810 | * - PG_Reserved is set |
| 811 | * - zone and node links point to zone and node that span the page if the |
| 812 | * hole is in the middle of a zone |
| 813 | * - zone and node links point to adjacent zone/node if the hole falls on |
| 814 | * the zone boundary; the pages in such holes will be prepended to the |
| 815 | * zone/node above the hole except for the trailing pages in the last |
| 816 | * section that will be appended to the zone/node below. |
| 817 | */ |
| 818 | static void __init init_unavailable_range(unsigned long spfn, |
| 819 | unsigned long epfn, |
| 820 | int zone, int node) |
| 821 | { |
| 822 | unsigned long pfn; |
| 823 | u64 pgcnt = 0; |
| 824 | |
| 825 | for (pfn = spfn; pfn < epfn; pfn++) { |
| 826 | if (!pfn_valid(pageblock_start_pfn(pfn))) { |
| 827 | pfn = pageblock_end_pfn(pfn) - 1; |
| 828 | continue; |
| 829 | } |
| 830 | __init_single_page(pfn_to_page(pfn), pfn, zone, node); |
| 831 | __SetPageReserved(pfn_to_page(pfn)); |
| 832 | pgcnt++; |
| 833 | } |
| 834 | |
| 835 | if (pgcnt) |
| 836 | pr_info("On node %d, zone %s: %lld pages in unavailable ranges\n", |
| 837 | node, zone_names[zone], pgcnt); |
| 838 | } |
| 839 | |
| 840 | /* |
| 841 | * Initially all pages are reserved - free ones are freed |
| 842 | * up by memblock_free_all() once the early boot process is |
| 843 | * done. Non-atomic initialization, single-pass. |
| 844 | * |
| 845 | * All aligned pageblocks are initialized to the specified migratetype |
| 846 | * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related |
| 847 | * zone stats (e.g., nr_isolate_pageblock) are touched. |
| 848 | */ |
| 849 | void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone, |
| 850 | unsigned long start_pfn, unsigned long zone_end_pfn, |
| 851 | enum meminit_context context, |
| 852 | struct vmem_altmap *altmap, int migratetype) |
| 853 | { |
| 854 | unsigned long pfn, end_pfn = start_pfn + size; |
| 855 | struct page *page; |
| 856 | |
| 857 | if (highest_memmap_pfn < end_pfn - 1) |
| 858 | highest_memmap_pfn = end_pfn - 1; |
| 859 | |
| 860 | #ifdef CONFIG_ZONE_DEVICE |
| 861 | /* |
| 862 | * Honor reservation requested by the driver for this ZONE_DEVICE |
| 863 | * memory. We limit the total number of pages to initialize to just |
| 864 | * those that might contain the memory mapping. We will defer the |
| 865 | * ZONE_DEVICE page initialization until after we have released |
| 866 | * the hotplug lock. |
| 867 | */ |
| 868 | if (zone == ZONE_DEVICE) { |
| 869 | if (!altmap) |
| 870 | return; |
| 871 | |
| 872 | if (start_pfn == altmap->base_pfn) |
| 873 | start_pfn += altmap->reserve; |
| 874 | end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); |
| 875 | } |
| 876 | #endif |
| 877 | |
| 878 | for (pfn = start_pfn; pfn < end_pfn; ) { |
| 879 | /* |
| 880 | * There can be holes in boot-time mem_map[]s handed to this |
| 881 | * function. They do not exist on hotplugged memory. |
| 882 | */ |
| 883 | if (context == MEMINIT_EARLY) { |
| 884 | if (overlap_memmap_init(zone, &pfn)) |
| 885 | continue; |
| 886 | if (defer_init(nid, pfn, zone_end_pfn)) { |
| 887 | deferred_struct_pages = true; |
| 888 | break; |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | page = pfn_to_page(pfn); |
| 893 | __init_single_page(page, pfn, zone, nid); |
| 894 | if (context == MEMINIT_HOTPLUG) { |
| 895 | #ifdef CONFIG_ZONE_DEVICE |
| 896 | if (zone == ZONE_DEVICE) |
| 897 | __SetPageReserved(page); |
| 898 | else |
| 899 | #endif |
| 900 | __SetPageOffline(page); |
| 901 | } |
| 902 | |
| 903 | /* |
| 904 | * Usually, we want to mark the pageblock MIGRATE_MOVABLE, |
| 905 | * such that unmovable allocations won't be scattered all |
| 906 | * over the place during system boot. |
| 907 | */ |
| 908 | if (pageblock_aligned(pfn)) { |
| 909 | set_pageblock_migratetype(page, migratetype); |
| 910 | cond_resched(); |
| 911 | } |
| 912 | pfn++; |
| 913 | } |
| 914 | } |
| 915 | |
| 916 | static void __init memmap_init_zone_range(struct zone *zone, |
| 917 | unsigned long start_pfn, |
| 918 | unsigned long end_pfn, |
| 919 | unsigned long *hole_pfn) |
| 920 | { |
| 921 | unsigned long zone_start_pfn = zone->zone_start_pfn; |
| 922 | unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages; |
| 923 | int nid = zone_to_nid(zone), zone_id = zone_idx(zone); |
| 924 | |
| 925 | start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn); |
| 926 | end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn); |
| 927 | |
| 928 | if (start_pfn >= end_pfn) |
| 929 | return; |
| 930 | |
| 931 | memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn, |
| 932 | zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); |
| 933 | |
| 934 | if (*hole_pfn < start_pfn) |
| 935 | init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid); |
| 936 | |
| 937 | *hole_pfn = end_pfn; |
| 938 | } |
| 939 | |
| 940 | static void __init memmap_init(void) |
| 941 | { |
| 942 | unsigned long start_pfn, end_pfn; |
| 943 | unsigned long hole_pfn = 0; |
| 944 | int i, j, zone_id = 0, nid; |
| 945 | |
| 946 | for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { |
| 947 | struct pglist_data *node = NODE_DATA(nid); |
| 948 | |
| 949 | for (j = 0; j < MAX_NR_ZONES; j++) { |
| 950 | struct zone *zone = node->node_zones + j; |
| 951 | |
| 952 | if (!populated_zone(zone)) |
| 953 | continue; |
| 954 | |
| 955 | memmap_init_zone_range(zone, start_pfn, end_pfn, |
| 956 | &hole_pfn); |
| 957 | zone_id = j; |
| 958 | } |
| 959 | } |
| 960 | |
| 961 | #ifdef CONFIG_SPARSEMEM |
| 962 | /* |
| 963 | * Initialize the memory map for hole in the range [memory_end, |
| 964 | * section_end]. |
| 965 | * Append the pages in this hole to the highest zone in the last |
| 966 | * node. |
| 967 | * The call to init_unavailable_range() is outside the ifdef to |
| 968 | * silence the compiler warining about zone_id set but not used; |
| 969 | * for FLATMEM it is a nop anyway |
| 970 | */ |
| 971 | end_pfn = round_up(end_pfn, PAGES_PER_SECTION); |
| 972 | if (hole_pfn < end_pfn) |
| 973 | #endif |
| 974 | init_unavailable_range(hole_pfn, end_pfn, zone_id, nid); |
| 975 | } |
| 976 | |
| 977 | #ifdef CONFIG_ZONE_DEVICE |
| 978 | static void __ref __init_zone_device_page(struct page *page, unsigned long pfn, |
| 979 | unsigned long zone_idx, int nid, |
| 980 | struct dev_pagemap *pgmap) |
| 981 | { |
| 982 | |
| 983 | __init_single_page(page, pfn, zone_idx, nid); |
| 984 | |
| 985 | /* |
| 986 | * Mark page reserved as it will need to wait for onlining |
| 987 | * phase for it to be fully associated with a zone. |
| 988 | * |
| 989 | * We can use the non-atomic __set_bit operation for setting |
| 990 | * the flag as we are still initializing the pages. |
| 991 | */ |
| 992 | __SetPageReserved(page); |
| 993 | |
| 994 | /* |
| 995 | * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer |
| 996 | * and zone_device_data. It is a bug if a ZONE_DEVICE page is |
| 997 | * ever freed or placed on a driver-private list. |
| 998 | */ |
| 999 | page->pgmap = pgmap; |
| 1000 | page->zone_device_data = NULL; |
| 1001 | |
| 1002 | /* |
| 1003 | * Mark the block movable so that blocks are reserved for |
| 1004 | * movable at startup. This will force kernel allocations |
| 1005 | * to reserve their blocks rather than leaking throughout |
| 1006 | * the address space during boot when many long-lived |
| 1007 | * kernel allocations are made. |
| 1008 | * |
| 1009 | * Please note that MEMINIT_HOTPLUG path doesn't clear memmap |
| 1010 | * because this is done early in section_activate() |
| 1011 | */ |
| 1012 | if (pageblock_aligned(pfn)) { |
| 1013 | set_pageblock_migratetype(page, MIGRATE_MOVABLE); |
| 1014 | cond_resched(); |
| 1015 | } |
| 1016 | |
| 1017 | /* |
| 1018 | * ZONE_DEVICE pages are released directly to the driver page allocator |
| 1019 | * which will set the page count to 1 when allocating the page. |
| 1020 | */ |
| 1021 | if (pgmap->type == MEMORY_DEVICE_PRIVATE || |
| 1022 | pgmap->type == MEMORY_DEVICE_COHERENT) |
| 1023 | set_page_count(page, 0); |
| 1024 | } |
| 1025 | |
| 1026 | /* |
| 1027 | * With compound page geometry and when struct pages are stored in ram most |
| 1028 | * tail pages are reused. Consequently, the amount of unique struct pages to |
| 1029 | * initialize is a lot smaller that the total amount of struct pages being |
| 1030 | * mapped. This is a paired / mild layering violation with explicit knowledge |
| 1031 | * of how the sparse_vmemmap internals handle compound pages in the lack |
| 1032 | * of an altmap. See vmemmap_populate_compound_pages(). |
| 1033 | */ |
| 1034 | static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap, |
| 1035 | struct dev_pagemap *pgmap) |
| 1036 | { |
| 1037 | if (!vmemmap_can_optimize(altmap, pgmap)) |
| 1038 | return pgmap_vmemmap_nr(pgmap); |
| 1039 | |
| 1040 | return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page)); |
| 1041 | } |
| 1042 | |
| 1043 | static void __ref memmap_init_compound(struct page *head, |
| 1044 | unsigned long head_pfn, |
| 1045 | unsigned long zone_idx, int nid, |
| 1046 | struct dev_pagemap *pgmap, |
| 1047 | unsigned long nr_pages) |
| 1048 | { |
| 1049 | unsigned long pfn, end_pfn = head_pfn + nr_pages; |
| 1050 | unsigned int order = pgmap->vmemmap_shift; |
| 1051 | |
| 1052 | __SetPageHead(head); |
| 1053 | for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) { |
| 1054 | struct page *page = pfn_to_page(pfn); |
| 1055 | |
| 1056 | __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); |
| 1057 | prep_compound_tail(head, pfn - head_pfn); |
| 1058 | set_page_count(page, 0); |
| 1059 | |
| 1060 | /* |
| 1061 | * The first tail page stores important compound page info. |
| 1062 | * Call prep_compound_head() after the first tail page has |
| 1063 | * been initialized, to not have the data overwritten. |
| 1064 | */ |
| 1065 | if (pfn == head_pfn + 1) |
| 1066 | prep_compound_head(head, order); |
| 1067 | } |
| 1068 | } |
| 1069 | |
| 1070 | void __ref memmap_init_zone_device(struct zone *zone, |
| 1071 | unsigned long start_pfn, |
| 1072 | unsigned long nr_pages, |
| 1073 | struct dev_pagemap *pgmap) |
| 1074 | { |
| 1075 | unsigned long pfn, end_pfn = start_pfn + nr_pages; |
| 1076 | struct pglist_data *pgdat = zone->zone_pgdat; |
| 1077 | struct vmem_altmap *altmap = pgmap_altmap(pgmap); |
| 1078 | unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap); |
| 1079 | unsigned long zone_idx = zone_idx(zone); |
| 1080 | unsigned long start = jiffies; |
| 1081 | int nid = pgdat->node_id; |
| 1082 | |
| 1083 | if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE)) |
| 1084 | return; |
| 1085 | |
| 1086 | /* |
| 1087 | * The call to memmap_init should have already taken care |
| 1088 | * of the pages reserved for the memmap, so we can just jump to |
| 1089 | * the end of that region and start processing the device pages. |
| 1090 | */ |
| 1091 | if (altmap) { |
| 1092 | start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap); |
| 1093 | nr_pages = end_pfn - start_pfn; |
| 1094 | } |
| 1095 | |
| 1096 | for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) { |
| 1097 | struct page *page = pfn_to_page(pfn); |
| 1098 | |
| 1099 | __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); |
| 1100 | |
| 1101 | if (pfns_per_compound == 1) |
| 1102 | continue; |
| 1103 | |
| 1104 | memmap_init_compound(page, pfn, zone_idx, nid, pgmap, |
| 1105 | compound_nr_pages(altmap, pgmap)); |
| 1106 | } |
| 1107 | |
| 1108 | pr_debug("%s initialised %lu pages in %ums\n", __func__, |
| 1109 | nr_pages, jiffies_to_msecs(jiffies - start)); |
| 1110 | } |
| 1111 | #endif |
| 1112 | |
| 1113 | /* |
| 1114 | * The zone ranges provided by the architecture do not include ZONE_MOVABLE |
| 1115 | * because it is sized independent of architecture. Unlike the other zones, |
| 1116 | * the starting point for ZONE_MOVABLE is not fixed. It may be different |
| 1117 | * in each node depending on the size of each node and how evenly kernelcore |
| 1118 | * is distributed. This helper function adjusts the zone ranges |
| 1119 | * provided by the architecture for a given node by using the end of the |
| 1120 | * highest usable zone for ZONE_MOVABLE. This preserves the assumption that |
| 1121 | * zones within a node are in order of monotonic increases memory addresses |
| 1122 | */ |
| 1123 | static void __init adjust_zone_range_for_zone_movable(int nid, |
| 1124 | unsigned long zone_type, |
| 1125 | unsigned long node_end_pfn, |
| 1126 | unsigned long *zone_start_pfn, |
| 1127 | unsigned long *zone_end_pfn) |
| 1128 | { |
| 1129 | /* Only adjust if ZONE_MOVABLE is on this node */ |
| 1130 | if (zone_movable_pfn[nid]) { |
| 1131 | /* Size ZONE_MOVABLE */ |
| 1132 | if (zone_type == ZONE_MOVABLE) { |
| 1133 | *zone_start_pfn = zone_movable_pfn[nid]; |
| 1134 | *zone_end_pfn = min(node_end_pfn, |
| 1135 | arch_zone_highest_possible_pfn[movable_zone]); |
| 1136 | |
| 1137 | /* Adjust for ZONE_MOVABLE starting within this range */ |
| 1138 | } else if (!mirrored_kernelcore && |
| 1139 | *zone_start_pfn < zone_movable_pfn[nid] && |
| 1140 | *zone_end_pfn > zone_movable_pfn[nid]) { |
| 1141 | *zone_end_pfn = zone_movable_pfn[nid]; |
| 1142 | |
| 1143 | /* Check if this whole range is within ZONE_MOVABLE */ |
| 1144 | } else if (*zone_start_pfn >= zone_movable_pfn[nid]) |
| 1145 | *zone_start_pfn = *zone_end_pfn; |
| 1146 | } |
| 1147 | } |
| 1148 | |
| 1149 | /* |
| 1150 | * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, |
| 1151 | * then all holes in the requested range will be accounted for. |
| 1152 | */ |
| 1153 | static unsigned long __init __absent_pages_in_range(int nid, |
| 1154 | unsigned long range_start_pfn, |
| 1155 | unsigned long range_end_pfn) |
| 1156 | { |
| 1157 | unsigned long nr_absent = range_end_pfn - range_start_pfn; |
| 1158 | unsigned long start_pfn, end_pfn; |
| 1159 | int i; |
| 1160 | |
| 1161 | for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { |
| 1162 | start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); |
| 1163 | end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); |
| 1164 | nr_absent -= end_pfn - start_pfn; |
| 1165 | } |
| 1166 | return nr_absent; |
| 1167 | } |
| 1168 | |
| 1169 | /** |
| 1170 | * absent_pages_in_range - Return number of page frames in holes within a range |
| 1171 | * @start_pfn: The start PFN to start searching for holes |
| 1172 | * @end_pfn: The end PFN to stop searching for holes |
| 1173 | * |
| 1174 | * Return: the number of pages frames in memory holes within a range. |
| 1175 | */ |
| 1176 | unsigned long __init absent_pages_in_range(unsigned long start_pfn, |
| 1177 | unsigned long end_pfn) |
| 1178 | { |
| 1179 | return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); |
| 1180 | } |
| 1181 | |
| 1182 | /* Return the number of page frames in holes in a zone on a node */ |
| 1183 | static unsigned long __init zone_absent_pages_in_node(int nid, |
| 1184 | unsigned long zone_type, |
| 1185 | unsigned long zone_start_pfn, |
| 1186 | unsigned long zone_end_pfn) |
| 1187 | { |
| 1188 | unsigned long nr_absent; |
| 1189 | |
| 1190 | /* zone is empty, we don't have any absent pages */ |
| 1191 | if (zone_start_pfn == zone_end_pfn) |
| 1192 | return 0; |
| 1193 | |
| 1194 | nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); |
| 1195 | |
| 1196 | /* |
| 1197 | * ZONE_MOVABLE handling. |
| 1198 | * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages |
| 1199 | * and vice versa. |
| 1200 | */ |
| 1201 | if (mirrored_kernelcore && zone_movable_pfn[nid]) { |
| 1202 | unsigned long start_pfn, end_pfn; |
| 1203 | struct memblock_region *r; |
| 1204 | |
| 1205 | for_each_mem_region(r) { |
| 1206 | start_pfn = clamp(memblock_region_memory_base_pfn(r), |
| 1207 | zone_start_pfn, zone_end_pfn); |
| 1208 | end_pfn = clamp(memblock_region_memory_end_pfn(r), |
| 1209 | zone_start_pfn, zone_end_pfn); |
| 1210 | |
| 1211 | if (zone_type == ZONE_MOVABLE && |
| 1212 | memblock_is_mirror(r)) |
| 1213 | nr_absent += end_pfn - start_pfn; |
| 1214 | |
| 1215 | if (zone_type == ZONE_NORMAL && |
| 1216 | !memblock_is_mirror(r)) |
| 1217 | nr_absent += end_pfn - start_pfn; |
| 1218 | } |
| 1219 | } |
| 1220 | |
| 1221 | return nr_absent; |
| 1222 | } |
| 1223 | |
| 1224 | /* |
| 1225 | * Return the number of pages a zone spans in a node, including holes |
| 1226 | * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() |
| 1227 | */ |
| 1228 | static unsigned long __init zone_spanned_pages_in_node(int nid, |
| 1229 | unsigned long zone_type, |
| 1230 | unsigned long node_start_pfn, |
| 1231 | unsigned long node_end_pfn, |
| 1232 | unsigned long *zone_start_pfn, |
| 1233 | unsigned long *zone_end_pfn) |
| 1234 | { |
| 1235 | unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; |
| 1236 | unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; |
| 1237 | |
| 1238 | /* Get the start and end of the zone */ |
| 1239 | *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); |
| 1240 | *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); |
| 1241 | adjust_zone_range_for_zone_movable(nid, zone_type, node_end_pfn, |
| 1242 | zone_start_pfn, zone_end_pfn); |
| 1243 | |
| 1244 | /* Check that this node has pages within the zone's required range */ |
| 1245 | if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn) |
| 1246 | return 0; |
| 1247 | |
| 1248 | /* Move the zone boundaries inside the node if necessary */ |
| 1249 | *zone_end_pfn = min(*zone_end_pfn, node_end_pfn); |
| 1250 | *zone_start_pfn = max(*zone_start_pfn, node_start_pfn); |
| 1251 | |
| 1252 | /* Return the spanned pages */ |
| 1253 | return *zone_end_pfn - *zone_start_pfn; |
| 1254 | } |
| 1255 | |
| 1256 | static void __init reset_memoryless_node_totalpages(struct pglist_data *pgdat) |
| 1257 | { |
| 1258 | struct zone *z; |
| 1259 | |
| 1260 | for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) { |
| 1261 | z->zone_start_pfn = 0; |
| 1262 | z->spanned_pages = 0; |
| 1263 | z->present_pages = 0; |
| 1264 | #if defined(CONFIG_MEMORY_HOTPLUG) |
| 1265 | z->present_early_pages = 0; |
| 1266 | #endif |
| 1267 | } |
| 1268 | |
| 1269 | pgdat->node_spanned_pages = 0; |
| 1270 | pgdat->node_present_pages = 0; |
| 1271 | pr_debug("On node %d totalpages: 0\n", pgdat->node_id); |
| 1272 | } |
| 1273 | |
| 1274 | static void __init calc_nr_kernel_pages(void) |
| 1275 | { |
| 1276 | unsigned long start_pfn, end_pfn; |
| 1277 | phys_addr_t start_addr, end_addr; |
| 1278 | u64 u; |
| 1279 | #ifdef CONFIG_HIGHMEM |
| 1280 | unsigned long high_zone_low = arch_zone_lowest_possible_pfn[ZONE_HIGHMEM]; |
| 1281 | #endif |
| 1282 | |
| 1283 | for_each_free_mem_range(u, NUMA_NO_NODE, MEMBLOCK_NONE, &start_addr, &end_addr, NULL) { |
| 1284 | start_pfn = PFN_UP(start_addr); |
| 1285 | end_pfn = PFN_DOWN(end_addr); |
| 1286 | |
| 1287 | if (start_pfn < end_pfn) { |
| 1288 | nr_all_pages += end_pfn - start_pfn; |
| 1289 | #ifdef CONFIG_HIGHMEM |
| 1290 | start_pfn = clamp(start_pfn, 0, high_zone_low); |
| 1291 | end_pfn = clamp(end_pfn, 0, high_zone_low); |
| 1292 | #endif |
| 1293 | nr_kernel_pages += end_pfn - start_pfn; |
| 1294 | } |
| 1295 | } |
| 1296 | } |
| 1297 | |
| 1298 | static void __init calculate_node_totalpages(struct pglist_data *pgdat, |
| 1299 | unsigned long node_start_pfn, |
| 1300 | unsigned long node_end_pfn) |
| 1301 | { |
| 1302 | unsigned long realtotalpages = 0, totalpages = 0; |
| 1303 | enum zone_type i; |
| 1304 | |
| 1305 | for (i = 0; i < MAX_NR_ZONES; i++) { |
| 1306 | struct zone *zone = pgdat->node_zones + i; |
| 1307 | unsigned long zone_start_pfn, zone_end_pfn; |
| 1308 | unsigned long spanned, absent; |
| 1309 | unsigned long real_size; |
| 1310 | |
| 1311 | spanned = zone_spanned_pages_in_node(pgdat->node_id, i, |
| 1312 | node_start_pfn, |
| 1313 | node_end_pfn, |
| 1314 | &zone_start_pfn, |
| 1315 | &zone_end_pfn); |
| 1316 | absent = zone_absent_pages_in_node(pgdat->node_id, i, |
| 1317 | zone_start_pfn, |
| 1318 | zone_end_pfn); |
| 1319 | |
| 1320 | real_size = spanned - absent; |
| 1321 | |
| 1322 | if (spanned) |
| 1323 | zone->zone_start_pfn = zone_start_pfn; |
| 1324 | else |
| 1325 | zone->zone_start_pfn = 0; |
| 1326 | zone->spanned_pages = spanned; |
| 1327 | zone->present_pages = real_size; |
| 1328 | #if defined(CONFIG_MEMORY_HOTPLUG) |
| 1329 | zone->present_early_pages = real_size; |
| 1330 | #endif |
| 1331 | |
| 1332 | totalpages += spanned; |
| 1333 | realtotalpages += real_size; |
| 1334 | } |
| 1335 | |
| 1336 | pgdat->node_spanned_pages = totalpages; |
| 1337 | pgdat->node_present_pages = realtotalpages; |
| 1338 | pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); |
| 1339 | } |
| 1340 | |
| 1341 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 1342 | static void pgdat_init_split_queue(struct pglist_data *pgdat) |
| 1343 | { |
| 1344 | struct deferred_split *ds_queue = &pgdat->deferred_split_queue; |
| 1345 | |
| 1346 | spin_lock_init(&ds_queue->split_queue_lock); |
| 1347 | INIT_LIST_HEAD(&ds_queue->split_queue); |
| 1348 | ds_queue->split_queue_len = 0; |
| 1349 | } |
| 1350 | #else |
| 1351 | static void pgdat_init_split_queue(struct pglist_data *pgdat) {} |
| 1352 | #endif |
| 1353 | |
| 1354 | #ifdef CONFIG_COMPACTION |
| 1355 | static void pgdat_init_kcompactd(struct pglist_data *pgdat) |
| 1356 | { |
| 1357 | init_waitqueue_head(&pgdat->kcompactd_wait); |
| 1358 | } |
| 1359 | #else |
| 1360 | static void pgdat_init_kcompactd(struct pglist_data *pgdat) {} |
| 1361 | #endif |
| 1362 | |
| 1363 | static void __meminit pgdat_init_internals(struct pglist_data *pgdat) |
| 1364 | { |
| 1365 | int i; |
| 1366 | |
| 1367 | pgdat_resize_init(pgdat); |
| 1368 | pgdat_kswapd_lock_init(pgdat); |
| 1369 | |
| 1370 | pgdat_init_split_queue(pgdat); |
| 1371 | pgdat_init_kcompactd(pgdat); |
| 1372 | |
| 1373 | init_waitqueue_head(&pgdat->kswapd_wait); |
| 1374 | init_waitqueue_head(&pgdat->pfmemalloc_wait); |
| 1375 | |
| 1376 | for (i = 0; i < NR_VMSCAN_THROTTLE; i++) |
| 1377 | init_waitqueue_head(&pgdat->reclaim_wait[i]); |
| 1378 | |
| 1379 | pgdat_page_ext_init(pgdat); |
| 1380 | lruvec_init(&pgdat->__lruvec); |
| 1381 | } |
| 1382 | |
| 1383 | static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid, |
| 1384 | unsigned long remaining_pages) |
| 1385 | { |
| 1386 | atomic_long_set(&zone->managed_pages, remaining_pages); |
| 1387 | zone_set_nid(zone, nid); |
| 1388 | zone->name = zone_names[idx]; |
| 1389 | zone->zone_pgdat = NODE_DATA(nid); |
| 1390 | spin_lock_init(&zone->lock); |
| 1391 | zone_seqlock_init(zone); |
| 1392 | zone_pcp_init(zone); |
| 1393 | } |
| 1394 | |
| 1395 | static void __meminit zone_init_free_lists(struct zone *zone) |
| 1396 | { |
| 1397 | unsigned int order, t; |
| 1398 | for_each_migratetype_order(order, t) { |
| 1399 | INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); |
| 1400 | zone->free_area[order].nr_free = 0; |
| 1401 | } |
| 1402 | |
| 1403 | #ifdef CONFIG_UNACCEPTED_MEMORY |
| 1404 | INIT_LIST_HEAD(&zone->unaccepted_pages); |
| 1405 | #endif |
| 1406 | } |
| 1407 | |
| 1408 | void __meminit init_currently_empty_zone(struct zone *zone, |
| 1409 | unsigned long zone_start_pfn, |
| 1410 | unsigned long size) |
| 1411 | { |
| 1412 | struct pglist_data *pgdat = zone->zone_pgdat; |
| 1413 | int zone_idx = zone_idx(zone) + 1; |
| 1414 | |
| 1415 | if (zone_idx > pgdat->nr_zones) |
| 1416 | pgdat->nr_zones = zone_idx; |
| 1417 | |
| 1418 | zone->zone_start_pfn = zone_start_pfn; |
| 1419 | |
| 1420 | mminit_dprintk(MMINIT_TRACE, "memmap_init", |
| 1421 | "Initialising map node %d zone %lu pfns %lu -> %lu\n", |
| 1422 | pgdat->node_id, |
| 1423 | (unsigned long)zone_idx(zone), |
| 1424 | zone_start_pfn, (zone_start_pfn + size)); |
| 1425 | |
| 1426 | zone_init_free_lists(zone); |
| 1427 | zone->initialized = 1; |
| 1428 | } |
| 1429 | |
| 1430 | #ifndef CONFIG_SPARSEMEM |
| 1431 | /* |
| 1432 | * Calculate the size of the zone->blockflags rounded to an unsigned long |
| 1433 | * Start by making sure zonesize is a multiple of pageblock_order by rounding |
| 1434 | * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally |
| 1435 | * round what is now in bits to nearest long in bits, then return it in |
| 1436 | * bytes. |
| 1437 | */ |
| 1438 | static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) |
| 1439 | { |
| 1440 | unsigned long usemapsize; |
| 1441 | |
| 1442 | zonesize += zone_start_pfn & (pageblock_nr_pages-1); |
| 1443 | usemapsize = roundup(zonesize, pageblock_nr_pages); |
| 1444 | usemapsize = usemapsize >> pageblock_order; |
| 1445 | usemapsize *= NR_PAGEBLOCK_BITS; |
| 1446 | usemapsize = roundup(usemapsize, BITS_PER_LONG); |
| 1447 | |
| 1448 | return usemapsize / BITS_PER_BYTE; |
| 1449 | } |
| 1450 | |
| 1451 | static void __ref setup_usemap(struct zone *zone) |
| 1452 | { |
| 1453 | unsigned long usemapsize = usemap_size(zone->zone_start_pfn, |
| 1454 | zone->spanned_pages); |
| 1455 | zone->pageblock_flags = NULL; |
| 1456 | if (usemapsize) { |
| 1457 | zone->pageblock_flags = |
| 1458 | memblock_alloc_node(usemapsize, SMP_CACHE_BYTES, |
| 1459 | zone_to_nid(zone)); |
| 1460 | if (!zone->pageblock_flags) |
| 1461 | panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n", |
| 1462 | usemapsize, zone->name, zone_to_nid(zone)); |
| 1463 | } |
| 1464 | } |
| 1465 | #else |
| 1466 | static inline void setup_usemap(struct zone *zone) {} |
| 1467 | #endif /* CONFIG_SPARSEMEM */ |
| 1468 | |
| 1469 | #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE |
| 1470 | |
| 1471 | /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ |
| 1472 | void __init set_pageblock_order(void) |
| 1473 | { |
| 1474 | unsigned int order = MAX_PAGE_ORDER; |
| 1475 | |
| 1476 | /* Check that pageblock_nr_pages has not already been setup */ |
| 1477 | if (pageblock_order) |
| 1478 | return; |
| 1479 | |
| 1480 | /* Don't let pageblocks exceed the maximum allocation granularity. */ |
| 1481 | if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order) |
| 1482 | order = HUGETLB_PAGE_ORDER; |
| 1483 | |
| 1484 | /* |
| 1485 | * Assume the largest contiguous order of interest is a huge page. |
| 1486 | * This value may be variable depending on boot parameters on powerpc. |
| 1487 | */ |
| 1488 | pageblock_order = order; |
| 1489 | } |
| 1490 | #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ |
| 1491 | |
| 1492 | /* |
| 1493 | * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() |
| 1494 | * is unused as pageblock_order is set at compile-time. See |
| 1495 | * include/linux/pageblock-flags.h for the values of pageblock_order based on |
| 1496 | * the kernel config |
| 1497 | */ |
| 1498 | void __init set_pageblock_order(void) |
| 1499 | { |
| 1500 | } |
| 1501 | |
| 1502 | #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ |
| 1503 | |
| 1504 | /* |
| 1505 | * Set up the zone data structures |
| 1506 | * - init pgdat internals |
| 1507 | * - init all zones belonging to this node |
| 1508 | * |
| 1509 | * NOTE: this function is only called during memory hotplug |
| 1510 | */ |
| 1511 | #ifdef CONFIG_MEMORY_HOTPLUG |
| 1512 | void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) |
| 1513 | { |
| 1514 | int nid = pgdat->node_id; |
| 1515 | enum zone_type z; |
| 1516 | int cpu; |
| 1517 | |
| 1518 | pgdat_init_internals(pgdat); |
| 1519 | |
| 1520 | if (pgdat->per_cpu_nodestats == &boot_nodestats) |
| 1521 | pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); |
| 1522 | |
| 1523 | /* |
| 1524 | * Reset the nr_zones, order and highest_zoneidx before reuse. |
| 1525 | * Note that kswapd will init kswapd_highest_zoneidx properly |
| 1526 | * when it starts in the near future. |
| 1527 | */ |
| 1528 | pgdat->nr_zones = 0; |
| 1529 | pgdat->kswapd_order = 0; |
| 1530 | pgdat->kswapd_highest_zoneidx = 0; |
| 1531 | pgdat->node_start_pfn = 0; |
| 1532 | pgdat->node_present_pages = 0; |
| 1533 | |
| 1534 | for_each_online_cpu(cpu) { |
| 1535 | struct per_cpu_nodestat *p; |
| 1536 | |
| 1537 | p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); |
| 1538 | memset(p, 0, sizeof(*p)); |
| 1539 | } |
| 1540 | |
| 1541 | /* |
| 1542 | * When memory is hot-added, all the memory is in offline state. So |
| 1543 | * clear all zones' present_pages and managed_pages because they will |
| 1544 | * be updated in online_pages() and offline_pages(). |
| 1545 | */ |
| 1546 | for (z = 0; z < MAX_NR_ZONES; z++) { |
| 1547 | struct zone *zone = pgdat->node_zones + z; |
| 1548 | |
| 1549 | zone->present_pages = 0; |
| 1550 | zone_init_internals(zone, z, nid, 0); |
| 1551 | } |
| 1552 | } |
| 1553 | #endif |
| 1554 | |
| 1555 | static void __init free_area_init_core(struct pglist_data *pgdat) |
| 1556 | { |
| 1557 | enum zone_type j; |
| 1558 | int nid = pgdat->node_id; |
| 1559 | |
| 1560 | pgdat_init_internals(pgdat); |
| 1561 | pgdat->per_cpu_nodestats = &boot_nodestats; |
| 1562 | |
| 1563 | for (j = 0; j < MAX_NR_ZONES; j++) { |
| 1564 | struct zone *zone = pgdat->node_zones + j; |
| 1565 | unsigned long size = zone->spanned_pages; |
| 1566 | |
| 1567 | /* |
| 1568 | * Initialize zone->managed_pages as 0 , it will be reset |
| 1569 | * when memblock allocator frees pages into buddy system. |
| 1570 | */ |
| 1571 | zone_init_internals(zone, j, nid, zone->present_pages); |
| 1572 | |
| 1573 | if (!size) |
| 1574 | continue; |
| 1575 | |
| 1576 | setup_usemap(zone); |
| 1577 | init_currently_empty_zone(zone, zone->zone_start_pfn, size); |
| 1578 | } |
| 1579 | } |
| 1580 | |
| 1581 | void __init *memmap_alloc(phys_addr_t size, phys_addr_t align, |
| 1582 | phys_addr_t min_addr, int nid, bool exact_nid) |
| 1583 | { |
| 1584 | void *ptr; |
| 1585 | |
| 1586 | if (exact_nid) |
| 1587 | ptr = memblock_alloc_exact_nid_raw(size, align, min_addr, |
| 1588 | MEMBLOCK_ALLOC_ACCESSIBLE, |
| 1589 | nid); |
| 1590 | else |
| 1591 | ptr = memblock_alloc_try_nid_raw(size, align, min_addr, |
| 1592 | MEMBLOCK_ALLOC_ACCESSIBLE, |
| 1593 | nid); |
| 1594 | |
| 1595 | if (ptr && size > 0) |
| 1596 | page_init_poison(ptr, size); |
| 1597 | |
| 1598 | return ptr; |
| 1599 | } |
| 1600 | |
| 1601 | #ifdef CONFIG_FLATMEM |
| 1602 | static void __init alloc_node_mem_map(struct pglist_data *pgdat) |
| 1603 | { |
| 1604 | unsigned long start, offset, size, end; |
| 1605 | struct page *map; |
| 1606 | |
| 1607 | /* Skip empty nodes */ |
| 1608 | if (!pgdat->node_spanned_pages) |
| 1609 | return; |
| 1610 | |
| 1611 | start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); |
| 1612 | offset = pgdat->node_start_pfn - start; |
| 1613 | /* |
| 1614 | * The zone's endpoints aren't required to be MAX_PAGE_ORDER |
| 1615 | * aligned but the node_mem_map endpoints must be in order |
| 1616 | * for the buddy allocator to function correctly. |
| 1617 | */ |
| 1618 | end = ALIGN(pgdat_end_pfn(pgdat), MAX_ORDER_NR_PAGES); |
| 1619 | size = (end - start) * sizeof(struct page); |
| 1620 | map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT, |
| 1621 | pgdat->node_id, false); |
| 1622 | if (!map) |
| 1623 | panic("Failed to allocate %ld bytes for node %d memory map\n", |
| 1624 | size, pgdat->node_id); |
| 1625 | pgdat->node_mem_map = map + offset; |
| 1626 | memmap_boot_pages_add(DIV_ROUND_UP(size, PAGE_SIZE)); |
| 1627 | pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n", |
| 1628 | __func__, pgdat->node_id, (unsigned long)pgdat, |
| 1629 | (unsigned long)pgdat->node_mem_map); |
| 1630 | #ifndef CONFIG_NUMA |
| 1631 | /* the global mem_map is just set as node 0's */ |
| 1632 | if (pgdat == NODE_DATA(0)) { |
| 1633 | mem_map = NODE_DATA(0)->node_mem_map; |
| 1634 | if (page_to_pfn(mem_map) != pgdat->node_start_pfn) |
| 1635 | mem_map -= offset; |
| 1636 | } |
| 1637 | #endif |
| 1638 | } |
| 1639 | #else |
| 1640 | static inline void alloc_node_mem_map(struct pglist_data *pgdat) { } |
| 1641 | #endif /* CONFIG_FLATMEM */ |
| 1642 | |
| 1643 | /** |
| 1644 | * get_pfn_range_for_nid - Return the start and end page frames for a node |
| 1645 | * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. |
| 1646 | * @start_pfn: Passed by reference. On return, it will have the node start_pfn. |
| 1647 | * @end_pfn: Passed by reference. On return, it will have the node end_pfn. |
| 1648 | * |
| 1649 | * It returns the start and end page frame of a node based on information |
| 1650 | * provided by memblock_set_node(). If called for a node |
| 1651 | * with no available memory, the start and end PFNs will be 0. |
| 1652 | */ |
| 1653 | void __init get_pfn_range_for_nid(unsigned int nid, |
| 1654 | unsigned long *start_pfn, unsigned long *end_pfn) |
| 1655 | { |
| 1656 | unsigned long this_start_pfn, this_end_pfn; |
| 1657 | int i; |
| 1658 | |
| 1659 | *start_pfn = -1UL; |
| 1660 | *end_pfn = 0; |
| 1661 | |
| 1662 | for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { |
| 1663 | *start_pfn = min(*start_pfn, this_start_pfn); |
| 1664 | *end_pfn = max(*end_pfn, this_end_pfn); |
| 1665 | } |
| 1666 | |
| 1667 | if (*start_pfn == -1UL) |
| 1668 | *start_pfn = 0; |
| 1669 | } |
| 1670 | |
| 1671 | static void __init free_area_init_node(int nid) |
| 1672 | { |
| 1673 | pg_data_t *pgdat = NODE_DATA(nid); |
| 1674 | unsigned long start_pfn = 0; |
| 1675 | unsigned long end_pfn = 0; |
| 1676 | |
| 1677 | /* pg_data_t should be reset to zero when it's allocated */ |
| 1678 | WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx); |
| 1679 | |
| 1680 | get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); |
| 1681 | |
| 1682 | pgdat->node_id = nid; |
| 1683 | pgdat->node_start_pfn = start_pfn; |
| 1684 | pgdat->per_cpu_nodestats = NULL; |
| 1685 | |
| 1686 | if (start_pfn != end_pfn) { |
| 1687 | pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, |
| 1688 | (u64)start_pfn << PAGE_SHIFT, |
| 1689 | end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0); |
| 1690 | |
| 1691 | calculate_node_totalpages(pgdat, start_pfn, end_pfn); |
| 1692 | } else { |
| 1693 | pr_info("Initmem setup node %d as memoryless\n", nid); |
| 1694 | |
| 1695 | reset_memoryless_node_totalpages(pgdat); |
| 1696 | } |
| 1697 | |
| 1698 | alloc_node_mem_map(pgdat); |
| 1699 | pgdat_set_deferred_range(pgdat); |
| 1700 | |
| 1701 | free_area_init_core(pgdat); |
| 1702 | lru_gen_init_pgdat(pgdat); |
| 1703 | } |
| 1704 | |
| 1705 | /* Any regular or high memory on that node ? */ |
| 1706 | static void __init check_for_memory(pg_data_t *pgdat) |
| 1707 | { |
| 1708 | enum zone_type zone_type; |
| 1709 | |
| 1710 | for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { |
| 1711 | struct zone *zone = &pgdat->node_zones[zone_type]; |
| 1712 | if (populated_zone(zone)) { |
| 1713 | if (IS_ENABLED(CONFIG_HIGHMEM)) |
| 1714 | node_set_state(pgdat->node_id, N_HIGH_MEMORY); |
| 1715 | if (zone_type <= ZONE_NORMAL) |
| 1716 | node_set_state(pgdat->node_id, N_NORMAL_MEMORY); |
| 1717 | break; |
| 1718 | } |
| 1719 | } |
| 1720 | } |
| 1721 | |
| 1722 | #if MAX_NUMNODES > 1 |
| 1723 | /* |
| 1724 | * Figure out the number of possible node ids. |
| 1725 | */ |
| 1726 | void __init setup_nr_node_ids(void) |
| 1727 | { |
| 1728 | unsigned int highest; |
| 1729 | |
| 1730 | highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES); |
| 1731 | nr_node_ids = highest + 1; |
| 1732 | } |
| 1733 | #endif |
| 1734 | |
| 1735 | /* |
| 1736 | * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For |
| 1737 | * such cases we allow max_zone_pfn sorted in the descending order |
| 1738 | */ |
| 1739 | static bool arch_has_descending_max_zone_pfns(void) |
| 1740 | { |
| 1741 | return IS_ENABLED(CONFIG_ARC) && !IS_ENABLED(CONFIG_ARC_HAS_PAE40); |
| 1742 | } |
| 1743 | |
| 1744 | /** |
| 1745 | * free_area_init - Initialise all pg_data_t and zone data |
| 1746 | * @max_zone_pfn: an array of max PFNs for each zone |
| 1747 | * |
| 1748 | * This will call free_area_init_node() for each active node in the system. |
| 1749 | * Using the page ranges provided by memblock_set_node(), the size of each |
| 1750 | * zone in each node and their holes is calculated. If the maximum PFN |
| 1751 | * between two adjacent zones match, it is assumed that the zone is empty. |
| 1752 | * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed |
| 1753 | * that arch_max_dma32_pfn has no pages. It is also assumed that a zone |
| 1754 | * starts where the previous one ended. For example, ZONE_DMA32 starts |
| 1755 | * at arch_max_dma_pfn. |
| 1756 | */ |
| 1757 | void __init free_area_init(unsigned long *max_zone_pfn) |
| 1758 | { |
| 1759 | unsigned long start_pfn, end_pfn; |
| 1760 | int i, nid, zone; |
| 1761 | bool descending; |
| 1762 | |
| 1763 | /* Record where the zone boundaries are */ |
| 1764 | memset(arch_zone_lowest_possible_pfn, 0, |
| 1765 | sizeof(arch_zone_lowest_possible_pfn)); |
| 1766 | memset(arch_zone_highest_possible_pfn, 0, |
| 1767 | sizeof(arch_zone_highest_possible_pfn)); |
| 1768 | |
| 1769 | start_pfn = PHYS_PFN(memblock_start_of_DRAM()); |
| 1770 | descending = arch_has_descending_max_zone_pfns(); |
| 1771 | |
| 1772 | for (i = 0; i < MAX_NR_ZONES; i++) { |
| 1773 | if (descending) |
| 1774 | zone = MAX_NR_ZONES - i - 1; |
| 1775 | else |
| 1776 | zone = i; |
| 1777 | |
| 1778 | if (zone == ZONE_MOVABLE) |
| 1779 | continue; |
| 1780 | |
| 1781 | end_pfn = max(max_zone_pfn[zone], start_pfn); |
| 1782 | arch_zone_lowest_possible_pfn[zone] = start_pfn; |
| 1783 | arch_zone_highest_possible_pfn[zone] = end_pfn; |
| 1784 | |
| 1785 | start_pfn = end_pfn; |
| 1786 | } |
| 1787 | |
| 1788 | /* Find the PFNs that ZONE_MOVABLE begins at in each node */ |
| 1789 | memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); |
| 1790 | find_zone_movable_pfns_for_nodes(); |
| 1791 | |
| 1792 | /* Print out the zone ranges */ |
| 1793 | pr_info("Zone ranges:\n"); |
| 1794 | for (i = 0; i < MAX_NR_ZONES; i++) { |
| 1795 | if (i == ZONE_MOVABLE) |
| 1796 | continue; |
| 1797 | pr_info(" %-8s ", zone_names[i]); |
| 1798 | if (arch_zone_lowest_possible_pfn[i] == |
| 1799 | arch_zone_highest_possible_pfn[i]) |
| 1800 | pr_cont("empty\n"); |
| 1801 | else |
| 1802 | pr_cont("[mem %#018Lx-%#018Lx]\n", |
| 1803 | (u64)arch_zone_lowest_possible_pfn[i] |
| 1804 | << PAGE_SHIFT, |
| 1805 | ((u64)arch_zone_highest_possible_pfn[i] |
| 1806 | << PAGE_SHIFT) - 1); |
| 1807 | } |
| 1808 | |
| 1809 | /* Print out the PFNs ZONE_MOVABLE begins at in each node */ |
| 1810 | pr_info("Movable zone start for each node\n"); |
| 1811 | for (i = 0; i < MAX_NUMNODES; i++) { |
| 1812 | if (zone_movable_pfn[i]) |
| 1813 | pr_info(" Node %d: %#018Lx\n", i, |
| 1814 | (u64)zone_movable_pfn[i] << PAGE_SHIFT); |
| 1815 | } |
| 1816 | |
| 1817 | /* |
| 1818 | * Print out the early node map, and initialize the |
| 1819 | * subsection-map relative to active online memory ranges to |
| 1820 | * enable future "sub-section" extensions of the memory map. |
| 1821 | */ |
| 1822 | pr_info("Early memory node ranges\n"); |
| 1823 | for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { |
| 1824 | pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, |
| 1825 | (u64)start_pfn << PAGE_SHIFT, |
| 1826 | ((u64)end_pfn << PAGE_SHIFT) - 1); |
| 1827 | subsection_map_init(start_pfn, end_pfn - start_pfn); |
| 1828 | } |
| 1829 | |
| 1830 | /* Initialise every node */ |
| 1831 | mminit_verify_pageflags_layout(); |
| 1832 | setup_nr_node_ids(); |
| 1833 | set_pageblock_order(); |
| 1834 | |
| 1835 | for_each_node(nid) { |
| 1836 | pg_data_t *pgdat; |
| 1837 | |
| 1838 | if (!node_online(nid)) |
| 1839 | alloc_offline_node_data(nid); |
| 1840 | |
| 1841 | pgdat = NODE_DATA(nid); |
| 1842 | free_area_init_node(nid); |
| 1843 | |
| 1844 | /* |
| 1845 | * No sysfs hierarcy will be created via register_one_node() |
| 1846 | *for memory-less node because here it's not marked as N_MEMORY |
| 1847 | *and won't be set online later. The benefit is userspace |
| 1848 | *program won't be confused by sysfs files/directories of |
| 1849 | *memory-less node. The pgdat will get fully initialized by |
| 1850 | *hotadd_init_pgdat() when memory is hotplugged into this node. |
| 1851 | */ |
| 1852 | if (pgdat->node_present_pages) { |
| 1853 | node_set_state(nid, N_MEMORY); |
| 1854 | check_for_memory(pgdat); |
| 1855 | } |
| 1856 | } |
| 1857 | |
| 1858 | calc_nr_kernel_pages(); |
| 1859 | memmap_init(); |
| 1860 | |
| 1861 | /* disable hash distribution for systems with a single node */ |
| 1862 | fixup_hashdist(); |
| 1863 | } |
| 1864 | |
| 1865 | /** |
| 1866 | * node_map_pfn_alignment - determine the maximum internode alignment |
| 1867 | * |
| 1868 | * This function should be called after node map is populated and sorted. |
| 1869 | * It calculates the maximum power of two alignment which can distinguish |
| 1870 | * all the nodes. |
| 1871 | * |
| 1872 | * For example, if all nodes are 1GiB and aligned to 1GiB, the return value |
| 1873 | * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the |
| 1874 | * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is |
| 1875 | * shifted, 1GiB is enough and this function will indicate so. |
| 1876 | * |
| 1877 | * This is used to test whether pfn -> nid mapping of the chosen memory |
| 1878 | * model has fine enough granularity to avoid incorrect mapping for the |
| 1879 | * populated node map. |
| 1880 | * |
| 1881 | * Return: the determined alignment in pfn's. 0 if there is no alignment |
| 1882 | * requirement (single node). |
| 1883 | */ |
| 1884 | unsigned long __init node_map_pfn_alignment(void) |
| 1885 | { |
| 1886 | unsigned long accl_mask = 0, last_end = 0; |
| 1887 | unsigned long start, end, mask; |
| 1888 | int last_nid = NUMA_NO_NODE; |
| 1889 | int i, nid; |
| 1890 | |
| 1891 | for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { |
| 1892 | if (!start || last_nid < 0 || last_nid == nid) { |
| 1893 | last_nid = nid; |
| 1894 | last_end = end; |
| 1895 | continue; |
| 1896 | } |
| 1897 | |
| 1898 | /* |
| 1899 | * Start with a mask granular enough to pin-point to the |
| 1900 | * start pfn and tick off bits one-by-one until it becomes |
| 1901 | * too coarse to separate the current node from the last. |
| 1902 | */ |
| 1903 | mask = ~((1 << __ffs(start)) - 1); |
| 1904 | while (mask && last_end <= (start & (mask << 1))) |
| 1905 | mask <<= 1; |
| 1906 | |
| 1907 | /* accumulate all internode masks */ |
| 1908 | accl_mask |= mask; |
| 1909 | } |
| 1910 | |
| 1911 | /* convert mask to number of pages */ |
| 1912 | return ~accl_mask + 1; |
| 1913 | } |
| 1914 | |
| 1915 | #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT |
| 1916 | static void __init deferred_free_pages(unsigned long pfn, |
| 1917 | unsigned long nr_pages) |
| 1918 | { |
| 1919 | struct page *page; |
| 1920 | unsigned long i; |
| 1921 | |
| 1922 | if (!nr_pages) |
| 1923 | return; |
| 1924 | |
| 1925 | page = pfn_to_page(pfn); |
| 1926 | |
| 1927 | /* Free a large naturally-aligned chunk if possible */ |
| 1928 | if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) { |
| 1929 | for (i = 0; i < nr_pages; i += pageblock_nr_pages) |
| 1930 | set_pageblock_migratetype(page + i, MIGRATE_MOVABLE); |
| 1931 | __free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY); |
| 1932 | return; |
| 1933 | } |
| 1934 | |
| 1935 | /* Accept chunks smaller than MAX_PAGE_ORDER upfront */ |
| 1936 | accept_memory(PFN_PHYS(pfn), nr_pages * PAGE_SIZE); |
| 1937 | |
| 1938 | for (i = 0; i < nr_pages; i++, page++, pfn++) { |
| 1939 | if (pageblock_aligned(pfn)) |
| 1940 | set_pageblock_migratetype(page, MIGRATE_MOVABLE); |
| 1941 | __free_pages_core(page, 0, MEMINIT_EARLY); |
| 1942 | } |
| 1943 | } |
| 1944 | |
| 1945 | /* Completion tracking for deferred_init_memmap() threads */ |
| 1946 | static atomic_t pgdat_init_n_undone __initdata; |
| 1947 | static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp); |
| 1948 | |
| 1949 | static inline void __init pgdat_init_report_one_done(void) |
| 1950 | { |
| 1951 | if (atomic_dec_and_test(&pgdat_init_n_undone)) |
| 1952 | complete(&pgdat_init_all_done_comp); |
| 1953 | } |
| 1954 | |
| 1955 | /* |
| 1956 | * Initialize struct pages. We minimize pfn page lookups and scheduler checks |
| 1957 | * by performing it only once every MAX_ORDER_NR_PAGES. |
| 1958 | * Return number of pages initialized. |
| 1959 | */ |
| 1960 | static unsigned long __init deferred_init_pages(struct zone *zone, |
| 1961 | unsigned long pfn, unsigned long end_pfn) |
| 1962 | { |
| 1963 | int nid = zone_to_nid(zone); |
| 1964 | unsigned long nr_pages = end_pfn - pfn; |
| 1965 | int zid = zone_idx(zone); |
| 1966 | struct page *page = pfn_to_page(pfn); |
| 1967 | |
| 1968 | for (; pfn < end_pfn; pfn++, page++) |
| 1969 | __init_single_page(page, pfn, zid, nid); |
| 1970 | return nr_pages; |
| 1971 | } |
| 1972 | |
| 1973 | /* |
| 1974 | * This function is meant to pre-load the iterator for the zone init from |
| 1975 | * a given point. |
| 1976 | * Specifically it walks through the ranges starting with initial index |
| 1977 | * passed to it until we are caught up to the first_init_pfn value and |
| 1978 | * exits there. If we never encounter the value we return false indicating |
| 1979 | * there are no valid ranges left. |
| 1980 | */ |
| 1981 | static bool __init |
| 1982 | deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, |
| 1983 | unsigned long *spfn, unsigned long *epfn, |
| 1984 | unsigned long first_init_pfn) |
| 1985 | { |
| 1986 | u64 j = *i; |
| 1987 | |
| 1988 | if (j == 0) |
| 1989 | __next_mem_pfn_range_in_zone(&j, zone, spfn, epfn); |
| 1990 | |
| 1991 | /* |
| 1992 | * Start out by walking through the ranges in this zone that have |
| 1993 | * already been initialized. We don't need to do anything with them |
| 1994 | * so we just need to flush them out of the system. |
| 1995 | */ |
| 1996 | for_each_free_mem_pfn_range_in_zone_from(j, zone, spfn, epfn) { |
| 1997 | if (*epfn <= first_init_pfn) |
| 1998 | continue; |
| 1999 | if (*spfn < first_init_pfn) |
| 2000 | *spfn = first_init_pfn; |
| 2001 | *i = j; |
| 2002 | return true; |
| 2003 | } |
| 2004 | |
| 2005 | return false; |
| 2006 | } |
| 2007 | |
| 2008 | /* |
| 2009 | * Initialize and free pages. We do it in two loops: first we initialize |
| 2010 | * struct page, then free to buddy allocator, because while we are |
| 2011 | * freeing pages we can access pages that are ahead (computing buddy |
| 2012 | * page in __free_one_page()). |
| 2013 | * |
| 2014 | * In order to try and keep some memory in the cache we have the loop |
| 2015 | * broken along max page order boundaries. This way we will not cause |
| 2016 | * any issues with the buddy page computation. |
| 2017 | */ |
| 2018 | static unsigned long __init |
| 2019 | deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, |
| 2020 | unsigned long *end_pfn) |
| 2021 | { |
| 2022 | unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); |
| 2023 | unsigned long spfn = *start_pfn, epfn = *end_pfn; |
| 2024 | unsigned long nr_pages = 0; |
| 2025 | u64 j = *i; |
| 2026 | |
| 2027 | /* First we loop through and initialize the page values */ |
| 2028 | for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { |
| 2029 | unsigned long t; |
| 2030 | |
| 2031 | if (mo_pfn <= *start_pfn) |
| 2032 | break; |
| 2033 | |
| 2034 | t = min(mo_pfn, *end_pfn); |
| 2035 | nr_pages += deferred_init_pages(zone, *start_pfn, t); |
| 2036 | |
| 2037 | if (mo_pfn < *end_pfn) { |
| 2038 | *start_pfn = mo_pfn; |
| 2039 | break; |
| 2040 | } |
| 2041 | } |
| 2042 | |
| 2043 | /* Reset values and now loop through freeing pages as needed */ |
| 2044 | swap(j, *i); |
| 2045 | |
| 2046 | for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { |
| 2047 | unsigned long t; |
| 2048 | |
| 2049 | if (mo_pfn <= spfn) |
| 2050 | break; |
| 2051 | |
| 2052 | t = min(mo_pfn, epfn); |
| 2053 | deferred_free_pages(spfn, t - spfn); |
| 2054 | |
| 2055 | if (mo_pfn <= epfn) |
| 2056 | break; |
| 2057 | } |
| 2058 | |
| 2059 | return nr_pages; |
| 2060 | } |
| 2061 | |
| 2062 | static void __init |
| 2063 | deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, |
| 2064 | void *arg) |
| 2065 | { |
| 2066 | unsigned long spfn, epfn; |
| 2067 | struct zone *zone = arg; |
| 2068 | u64 i = 0; |
| 2069 | |
| 2070 | deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); |
| 2071 | |
| 2072 | /* |
| 2073 | * Initialize and free pages in MAX_PAGE_ORDER sized increments so that |
| 2074 | * we can avoid introducing any issues with the buddy allocator. |
| 2075 | */ |
| 2076 | while (spfn < end_pfn) { |
| 2077 | deferred_init_maxorder(&i, zone, &spfn, &epfn); |
| 2078 | cond_resched(); |
| 2079 | } |
| 2080 | } |
| 2081 | |
| 2082 | static unsigned int __init |
| 2083 | deferred_page_init_max_threads(const struct cpumask *node_cpumask) |
| 2084 | { |
| 2085 | return max(cpumask_weight(node_cpumask), 1U); |
| 2086 | } |
| 2087 | |
| 2088 | /* Initialise remaining memory on a node */ |
| 2089 | static int __init deferred_init_memmap(void *data) |
| 2090 | { |
| 2091 | pg_data_t *pgdat = data; |
| 2092 | const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); |
| 2093 | unsigned long spfn = 0, epfn = 0; |
| 2094 | unsigned long first_init_pfn, flags; |
| 2095 | unsigned long start = jiffies; |
| 2096 | struct zone *zone; |
| 2097 | int max_threads; |
| 2098 | u64 i = 0; |
| 2099 | |
| 2100 | /* Bind memory initialisation thread to a local node if possible */ |
| 2101 | if (!cpumask_empty(cpumask)) |
| 2102 | set_cpus_allowed_ptr(current, cpumask); |
| 2103 | |
| 2104 | pgdat_resize_lock(pgdat, &flags); |
| 2105 | first_init_pfn = pgdat->first_deferred_pfn; |
| 2106 | if (first_init_pfn == ULONG_MAX) { |
| 2107 | pgdat_resize_unlock(pgdat, &flags); |
| 2108 | pgdat_init_report_one_done(); |
| 2109 | return 0; |
| 2110 | } |
| 2111 | |
| 2112 | /* Sanity check boundaries */ |
| 2113 | BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn); |
| 2114 | BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat)); |
| 2115 | pgdat->first_deferred_pfn = ULONG_MAX; |
| 2116 | |
| 2117 | /* |
| 2118 | * Once we unlock here, the zone cannot be grown anymore, thus if an |
| 2119 | * interrupt thread must allocate this early in boot, zone must be |
| 2120 | * pre-grown prior to start of deferred page initialization. |
| 2121 | */ |
| 2122 | pgdat_resize_unlock(pgdat, &flags); |
| 2123 | |
| 2124 | /* Only the highest zone is deferred */ |
| 2125 | zone = pgdat->node_zones + pgdat->nr_zones - 1; |
| 2126 | |
| 2127 | max_threads = deferred_page_init_max_threads(cpumask); |
| 2128 | |
| 2129 | while (deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, first_init_pfn)) { |
| 2130 | first_init_pfn = ALIGN(epfn, PAGES_PER_SECTION); |
| 2131 | struct padata_mt_job job = { |
| 2132 | .thread_fn = deferred_init_memmap_chunk, |
| 2133 | .fn_arg = zone, |
| 2134 | .start = spfn, |
| 2135 | .size = first_init_pfn - spfn, |
| 2136 | .align = PAGES_PER_SECTION, |
| 2137 | .min_chunk = PAGES_PER_SECTION, |
| 2138 | .max_threads = max_threads, |
| 2139 | .numa_aware = false, |
| 2140 | }; |
| 2141 | |
| 2142 | padata_do_multithreaded(&job); |
| 2143 | } |
| 2144 | |
| 2145 | /* Sanity check that the next zone really is unpopulated */ |
| 2146 | WARN_ON(pgdat->nr_zones < MAX_NR_ZONES && populated_zone(++zone)); |
| 2147 | |
| 2148 | pr_info("node %d deferred pages initialised in %ums\n", |
| 2149 | pgdat->node_id, jiffies_to_msecs(jiffies - start)); |
| 2150 | |
| 2151 | pgdat_init_report_one_done(); |
| 2152 | return 0; |
| 2153 | } |
| 2154 | |
| 2155 | /* |
| 2156 | * If this zone has deferred pages, try to grow it by initializing enough |
| 2157 | * deferred pages to satisfy the allocation specified by order, rounded up to |
| 2158 | * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments |
| 2159 | * of SECTION_SIZE bytes by initializing struct pages in increments of |
| 2160 | * PAGES_PER_SECTION * sizeof(struct page) bytes. |
| 2161 | * |
| 2162 | * Return true when zone was grown, otherwise return false. We return true even |
| 2163 | * when we grow less than requested, to let the caller decide if there are |
| 2164 | * enough pages to satisfy the allocation. |
| 2165 | */ |
| 2166 | bool __init deferred_grow_zone(struct zone *zone, unsigned int order) |
| 2167 | { |
| 2168 | unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION); |
| 2169 | pg_data_t *pgdat = zone->zone_pgdat; |
| 2170 | unsigned long first_deferred_pfn = pgdat->first_deferred_pfn; |
| 2171 | unsigned long spfn, epfn, flags; |
| 2172 | unsigned long nr_pages = 0; |
| 2173 | u64 i = 0; |
| 2174 | |
| 2175 | /* Only the last zone may have deferred pages */ |
| 2176 | if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat)) |
| 2177 | return false; |
| 2178 | |
| 2179 | pgdat_resize_lock(pgdat, &flags); |
| 2180 | |
| 2181 | /* |
| 2182 | * If someone grew this zone while we were waiting for spinlock, return |
| 2183 | * true, as there might be enough pages already. |
| 2184 | */ |
| 2185 | if (first_deferred_pfn != pgdat->first_deferred_pfn) { |
| 2186 | pgdat_resize_unlock(pgdat, &flags); |
| 2187 | return true; |
| 2188 | } |
| 2189 | |
| 2190 | /* If the zone is empty somebody else may have cleared out the zone */ |
| 2191 | if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, |
| 2192 | first_deferred_pfn)) { |
| 2193 | pgdat->first_deferred_pfn = ULONG_MAX; |
| 2194 | pgdat_resize_unlock(pgdat, &flags); |
| 2195 | /* Retry only once. */ |
| 2196 | return first_deferred_pfn != ULONG_MAX; |
| 2197 | } |
| 2198 | |
| 2199 | /* |
| 2200 | * Initialize and free pages in MAX_PAGE_ORDER sized increments so |
| 2201 | * that we can avoid introducing any issues with the buddy |
| 2202 | * allocator. |
| 2203 | */ |
| 2204 | while (spfn < epfn) { |
| 2205 | /* update our first deferred PFN for this section */ |
| 2206 | first_deferred_pfn = spfn; |
| 2207 | |
| 2208 | nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); |
| 2209 | touch_nmi_watchdog(); |
| 2210 | |
| 2211 | /* We should only stop along section boundaries */ |
| 2212 | if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) |
| 2213 | continue; |
| 2214 | |
| 2215 | /* If our quota has been met we can stop here */ |
| 2216 | if (nr_pages >= nr_pages_needed) |
| 2217 | break; |
| 2218 | } |
| 2219 | |
| 2220 | pgdat->first_deferred_pfn = spfn; |
| 2221 | pgdat_resize_unlock(pgdat, &flags); |
| 2222 | |
| 2223 | return nr_pages > 0; |
| 2224 | } |
| 2225 | |
| 2226 | #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ |
| 2227 | |
| 2228 | #ifdef CONFIG_CMA |
| 2229 | void __init init_cma_reserved_pageblock(struct page *page) |
| 2230 | { |
| 2231 | unsigned i = pageblock_nr_pages; |
| 2232 | struct page *p = page; |
| 2233 | |
| 2234 | do { |
| 2235 | __ClearPageReserved(p); |
| 2236 | set_page_count(p, 0); |
| 2237 | } while (++p, --i); |
| 2238 | |
| 2239 | set_pageblock_migratetype(page, MIGRATE_CMA); |
| 2240 | set_page_refcounted(page); |
| 2241 | /* pages were reserved and not allocated */ |
| 2242 | clear_page_tag_ref(page); |
| 2243 | __free_pages(page, pageblock_order); |
| 2244 | |
| 2245 | adjust_managed_page_count(page, pageblock_nr_pages); |
| 2246 | page_zone(page)->cma_pages += pageblock_nr_pages; |
| 2247 | } |
| 2248 | #endif |
| 2249 | |
| 2250 | void set_zone_contiguous(struct zone *zone) |
| 2251 | { |
| 2252 | unsigned long block_start_pfn = zone->zone_start_pfn; |
| 2253 | unsigned long block_end_pfn; |
| 2254 | |
| 2255 | block_end_pfn = pageblock_end_pfn(block_start_pfn); |
| 2256 | for (; block_start_pfn < zone_end_pfn(zone); |
| 2257 | block_start_pfn = block_end_pfn, |
| 2258 | block_end_pfn += pageblock_nr_pages) { |
| 2259 | |
| 2260 | block_end_pfn = min(block_end_pfn, zone_end_pfn(zone)); |
| 2261 | |
| 2262 | if (!__pageblock_pfn_to_page(block_start_pfn, |
| 2263 | block_end_pfn, zone)) |
| 2264 | return; |
| 2265 | cond_resched(); |
| 2266 | } |
| 2267 | |
| 2268 | /* We confirm that there is no hole */ |
| 2269 | zone->contiguous = true; |
| 2270 | } |
| 2271 | |
| 2272 | static void __init mem_init_print_info(void); |
| 2273 | void __init page_alloc_init_late(void) |
| 2274 | { |
| 2275 | struct zone *zone; |
| 2276 | int nid; |
| 2277 | |
| 2278 | #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT |
| 2279 | |
| 2280 | /* There will be num_node_state(N_MEMORY) threads */ |
| 2281 | atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY)); |
| 2282 | for_each_node_state(nid, N_MEMORY) { |
| 2283 | kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid); |
| 2284 | } |
| 2285 | |
| 2286 | /* Block until all are initialised */ |
| 2287 | wait_for_completion(&pgdat_init_all_done_comp); |
| 2288 | |
| 2289 | /* |
| 2290 | * We initialized the rest of the deferred pages. Permanently disable |
| 2291 | * on-demand struct page initialization. |
| 2292 | */ |
| 2293 | static_branch_disable(&deferred_pages); |
| 2294 | |
| 2295 | /* Reinit limits that are based on free pages after the kernel is up */ |
| 2296 | files_maxfiles_init(); |
| 2297 | #endif |
| 2298 | |
| 2299 | /* Accounting of total+free memory is stable at this point. */ |
| 2300 | mem_init_print_info(); |
| 2301 | buffer_init(); |
| 2302 | |
| 2303 | /* Discard memblock private memory */ |
| 2304 | memblock_discard(); |
| 2305 | |
| 2306 | for_each_node_state(nid, N_MEMORY) |
| 2307 | shuffle_free_memory(NODE_DATA(nid)); |
| 2308 | |
| 2309 | for_each_populated_zone(zone) |
| 2310 | set_zone_contiguous(zone); |
| 2311 | |
| 2312 | /* Initialize page ext after all struct pages are initialized. */ |
| 2313 | if (deferred_struct_pages) |
| 2314 | page_ext_init(); |
| 2315 | |
| 2316 | page_alloc_sysctl_init(); |
| 2317 | } |
| 2318 | |
| 2319 | /* |
| 2320 | * Adaptive scale is meant to reduce sizes of hash tables on large memory |
| 2321 | * machines. As memory size is increased the scale is also increased but at |
| 2322 | * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory |
| 2323 | * quadruples the scale is increased by one, which means the size of hash table |
| 2324 | * only doubles, instead of quadrupling as well. |
| 2325 | * Because 32-bit systems cannot have large physical memory, where this scaling |
| 2326 | * makes sense, it is disabled on such platforms. |
| 2327 | */ |
| 2328 | #if __BITS_PER_LONG > 32 |
| 2329 | #define ADAPT_SCALE_BASE (64ul << 30) |
| 2330 | #define ADAPT_SCALE_SHIFT 2 |
| 2331 | #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT) |
| 2332 | #endif |
| 2333 | |
| 2334 | /* |
| 2335 | * allocate a large system hash table from bootmem |
| 2336 | * - it is assumed that the hash table must contain an exact power-of-2 |
| 2337 | * quantity of entries |
| 2338 | * - limit is the number of hash buckets, not the total allocation size |
| 2339 | */ |
| 2340 | void *__init alloc_large_system_hash(const char *tablename, |
| 2341 | unsigned long bucketsize, |
| 2342 | unsigned long numentries, |
| 2343 | int scale, |
| 2344 | int flags, |
| 2345 | unsigned int *_hash_shift, |
| 2346 | unsigned int *_hash_mask, |
| 2347 | unsigned long low_limit, |
| 2348 | unsigned long high_limit) |
| 2349 | { |
| 2350 | unsigned long long max = high_limit; |
| 2351 | unsigned long log2qty, size; |
| 2352 | void *table; |
| 2353 | gfp_t gfp_flags; |
| 2354 | bool virt; |
| 2355 | bool huge; |
| 2356 | |
| 2357 | /* allow the kernel cmdline to have a say */ |
| 2358 | if (!numentries) { |
| 2359 | /* round applicable memory size up to nearest megabyte */ |
| 2360 | numentries = nr_kernel_pages; |
| 2361 | |
| 2362 | /* It isn't necessary when PAGE_SIZE >= 1MB */ |
| 2363 | if (PAGE_SIZE < SZ_1M) |
| 2364 | numentries = round_up(numentries, SZ_1M / PAGE_SIZE); |
| 2365 | |
| 2366 | #if __BITS_PER_LONG > 32 |
| 2367 | if (!high_limit) { |
| 2368 | unsigned long adapt; |
| 2369 | |
| 2370 | for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries; |
| 2371 | adapt <<= ADAPT_SCALE_SHIFT) |
| 2372 | scale++; |
| 2373 | } |
| 2374 | #endif |
| 2375 | |
| 2376 | /* limit to 1 bucket per 2^scale bytes of low memory */ |
| 2377 | if (scale > PAGE_SHIFT) |
| 2378 | numentries >>= (scale - PAGE_SHIFT); |
| 2379 | else |
| 2380 | numentries <<= (PAGE_SHIFT - scale); |
| 2381 | |
| 2382 | if (unlikely((numentries * bucketsize) < PAGE_SIZE)) |
| 2383 | numentries = PAGE_SIZE / bucketsize; |
| 2384 | } |
| 2385 | numentries = roundup_pow_of_two(numentries); |
| 2386 | |
| 2387 | /* limit allocation size to 1/16 total memory by default */ |
| 2388 | if (max == 0) { |
| 2389 | max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; |
| 2390 | do_div(max, bucketsize); |
| 2391 | } |
| 2392 | max = min(max, 0x80000000ULL); |
| 2393 | |
| 2394 | if (numentries < low_limit) |
| 2395 | numentries = low_limit; |
| 2396 | if (numentries > max) |
| 2397 | numentries = max; |
| 2398 | |
| 2399 | log2qty = ilog2(numentries); |
| 2400 | |
| 2401 | gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC; |
| 2402 | do { |
| 2403 | virt = false; |
| 2404 | size = bucketsize << log2qty; |
| 2405 | if (flags & HASH_EARLY) { |
| 2406 | if (flags & HASH_ZERO) |
| 2407 | table = memblock_alloc(size, SMP_CACHE_BYTES); |
| 2408 | else |
| 2409 | table = memblock_alloc_raw(size, |
| 2410 | SMP_CACHE_BYTES); |
| 2411 | } else if (get_order(size) > MAX_PAGE_ORDER || hashdist) { |
| 2412 | table = vmalloc_huge(size, gfp_flags); |
| 2413 | virt = true; |
| 2414 | if (table) |
| 2415 | huge = is_vm_area_hugepages(table); |
| 2416 | } else { |
| 2417 | /* |
| 2418 | * If bucketsize is not a power-of-two, we may free |
| 2419 | * some pages at the end of hash table which |
| 2420 | * alloc_pages_exact() automatically does |
| 2421 | */ |
| 2422 | table = alloc_pages_exact(size, gfp_flags); |
| 2423 | kmemleak_alloc(table, size, 1, gfp_flags); |
| 2424 | } |
| 2425 | } while (!table && size > PAGE_SIZE && --log2qty); |
| 2426 | |
| 2427 | if (!table) |
| 2428 | panic("Failed to allocate %s hash table\n", tablename); |
| 2429 | |
| 2430 | pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n", |
| 2431 | tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size, |
| 2432 | virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear"); |
| 2433 | |
| 2434 | if (_hash_shift) |
| 2435 | *_hash_shift = log2qty; |
| 2436 | if (_hash_mask) |
| 2437 | *_hash_mask = (1 << log2qty) - 1; |
| 2438 | |
| 2439 | return table; |
| 2440 | } |
| 2441 | |
| 2442 | void __init memblock_free_pages(struct page *page, unsigned long pfn, |
| 2443 | unsigned int order) |
| 2444 | { |
| 2445 | if (IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT)) { |
| 2446 | int nid = early_pfn_to_nid(pfn); |
| 2447 | |
| 2448 | if (!early_page_initialised(pfn, nid)) |
| 2449 | return; |
| 2450 | } |
| 2451 | |
| 2452 | if (!kmsan_memblock_free_pages(page, order)) { |
| 2453 | /* KMSAN will take care of these pages. */ |
| 2454 | return; |
| 2455 | } |
| 2456 | |
| 2457 | /* pages were reserved and not allocated */ |
| 2458 | clear_page_tag_ref(page); |
| 2459 | __free_pages_core(page, order, MEMINIT_EARLY); |
| 2460 | } |
| 2461 | |
| 2462 | DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc); |
| 2463 | EXPORT_SYMBOL(init_on_alloc); |
| 2464 | |
| 2465 | DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free); |
| 2466 | EXPORT_SYMBOL(init_on_free); |
| 2467 | |
| 2468 | static bool _init_on_alloc_enabled_early __read_mostly |
| 2469 | = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON); |
| 2470 | static int __init early_init_on_alloc(char *buf) |
| 2471 | { |
| 2472 | |
| 2473 | return kstrtobool(buf, &_init_on_alloc_enabled_early); |
| 2474 | } |
| 2475 | early_param("init_on_alloc", early_init_on_alloc); |
| 2476 | |
| 2477 | static bool _init_on_free_enabled_early __read_mostly |
| 2478 | = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON); |
| 2479 | static int __init early_init_on_free(char *buf) |
| 2480 | { |
| 2481 | return kstrtobool(buf, &_init_on_free_enabled_early); |
| 2482 | } |
| 2483 | early_param("init_on_free", early_init_on_free); |
| 2484 | |
| 2485 | DEFINE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled); |
| 2486 | |
| 2487 | /* |
| 2488 | * Enable static keys related to various memory debugging and hardening options. |
| 2489 | * Some override others, and depend on early params that are evaluated in the |
| 2490 | * order of appearance. So we need to first gather the full picture of what was |
| 2491 | * enabled, and then make decisions. |
| 2492 | */ |
| 2493 | static void __init mem_debugging_and_hardening_init(void) |
| 2494 | { |
| 2495 | bool page_poisoning_requested = false; |
| 2496 | bool want_check_pages = false; |
| 2497 | |
| 2498 | #ifdef CONFIG_PAGE_POISONING |
| 2499 | /* |
| 2500 | * Page poisoning is debug page alloc for some arches. If |
| 2501 | * either of those options are enabled, enable poisoning. |
| 2502 | */ |
| 2503 | if (page_poisoning_enabled() || |
| 2504 | (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) && |
| 2505 | debug_pagealloc_enabled())) { |
| 2506 | static_branch_enable(&_page_poisoning_enabled); |
| 2507 | page_poisoning_requested = true; |
| 2508 | want_check_pages = true; |
| 2509 | } |
| 2510 | #endif |
| 2511 | |
| 2512 | if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) && |
| 2513 | page_poisoning_requested) { |
| 2514 | pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, " |
| 2515 | "will take precedence over init_on_alloc and init_on_free\n"); |
| 2516 | _init_on_alloc_enabled_early = false; |
| 2517 | _init_on_free_enabled_early = false; |
| 2518 | } |
| 2519 | |
| 2520 | if (_init_on_alloc_enabled_early) { |
| 2521 | want_check_pages = true; |
| 2522 | static_branch_enable(&init_on_alloc); |
| 2523 | } else { |
| 2524 | static_branch_disable(&init_on_alloc); |
| 2525 | } |
| 2526 | |
| 2527 | if (_init_on_free_enabled_early) { |
| 2528 | want_check_pages = true; |
| 2529 | static_branch_enable(&init_on_free); |
| 2530 | } else { |
| 2531 | static_branch_disable(&init_on_free); |
| 2532 | } |
| 2533 | |
| 2534 | if (IS_ENABLED(CONFIG_KMSAN) && |
| 2535 | (_init_on_alloc_enabled_early || _init_on_free_enabled_early)) |
| 2536 | pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n"); |
| 2537 | |
| 2538 | #ifdef CONFIG_DEBUG_PAGEALLOC |
| 2539 | if (debug_pagealloc_enabled()) { |
| 2540 | want_check_pages = true; |
| 2541 | static_branch_enable(&_debug_pagealloc_enabled); |
| 2542 | |
| 2543 | if (debug_guardpage_minorder()) |
| 2544 | static_branch_enable(&_debug_guardpage_enabled); |
| 2545 | } |
| 2546 | #endif |
| 2547 | |
| 2548 | /* |
| 2549 | * Any page debugging or hardening option also enables sanity checking |
| 2550 | * of struct pages being allocated or freed. With CONFIG_DEBUG_VM it's |
| 2551 | * enabled already. |
| 2552 | */ |
| 2553 | if (!IS_ENABLED(CONFIG_DEBUG_VM) && want_check_pages) |
| 2554 | static_branch_enable(&check_pages_enabled); |
| 2555 | } |
| 2556 | |
| 2557 | /* Report memory auto-initialization states for this boot. */ |
| 2558 | static void __init report_meminit(void) |
| 2559 | { |
| 2560 | const char *stack; |
| 2561 | |
| 2562 | if (IS_ENABLED(CONFIG_INIT_STACK_ALL_PATTERN)) |
| 2563 | stack = "all(pattern)"; |
| 2564 | else if (IS_ENABLED(CONFIG_INIT_STACK_ALL_ZERO)) |
| 2565 | stack = "all(zero)"; |
| 2566 | else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL)) |
| 2567 | stack = "byref_all(zero)"; |
| 2568 | else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF)) |
| 2569 | stack = "byref(zero)"; |
| 2570 | else if (IS_ENABLED(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER)) |
| 2571 | stack = "__user(zero)"; |
| 2572 | else |
| 2573 | stack = "off"; |
| 2574 | |
| 2575 | pr_info("mem auto-init: stack:%s, heap alloc:%s, heap free:%s\n", |
| 2576 | stack, want_init_on_alloc(GFP_KERNEL) ? "on" : "off", |
| 2577 | want_init_on_free() ? "on" : "off"); |
| 2578 | if (want_init_on_free()) |
| 2579 | pr_info("mem auto-init: clearing system memory may take some time...\n"); |
| 2580 | } |
| 2581 | |
| 2582 | static void __init mem_init_print_info(void) |
| 2583 | { |
| 2584 | unsigned long physpages, codesize, datasize, rosize, bss_size; |
| 2585 | unsigned long init_code_size, init_data_size; |
| 2586 | |
| 2587 | physpages = get_num_physpages(); |
| 2588 | codesize = _etext - _stext; |
| 2589 | datasize = _edata - _sdata; |
| 2590 | rosize = __end_rodata - __start_rodata; |
| 2591 | bss_size = __bss_stop - __bss_start; |
| 2592 | init_data_size = __init_end - __init_begin; |
| 2593 | init_code_size = _einittext - _sinittext; |
| 2594 | |
| 2595 | /* |
| 2596 | * Detect special cases and adjust section sizes accordingly: |
| 2597 | * 1) .init.* may be embedded into .data sections |
| 2598 | * 2) .init.text.* may be out of [__init_begin, __init_end], |
| 2599 | * please refer to arch/tile/kernel/vmlinux.lds.S. |
| 2600 | * 3) .rodata.* may be embedded into .text or .data sections. |
| 2601 | */ |
| 2602 | #define adj_init_size(start, end, size, pos, adj) \ |
| 2603 | do { \ |
| 2604 | if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \ |
| 2605 | size -= adj; \ |
| 2606 | } while (0) |
| 2607 | |
| 2608 | adj_init_size(__init_begin, __init_end, init_data_size, |
| 2609 | _sinittext, init_code_size); |
| 2610 | adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); |
| 2611 | adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); |
| 2612 | adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); |
| 2613 | adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); |
| 2614 | |
| 2615 | #undef adj_init_size |
| 2616 | |
| 2617 | pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved" |
| 2618 | #ifdef CONFIG_HIGHMEM |
| 2619 | ", %luK highmem" |
| 2620 | #endif |
| 2621 | ")\n", |
| 2622 | K(nr_free_pages()), K(physpages), |
| 2623 | codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K, |
| 2624 | (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K, |
| 2625 | K(physpages - totalram_pages() - totalcma_pages), |
| 2626 | K(totalcma_pages) |
| 2627 | #ifdef CONFIG_HIGHMEM |
| 2628 | , K(totalhigh_pages()) |
| 2629 | #endif |
| 2630 | ); |
| 2631 | } |
| 2632 | |
| 2633 | /* |
| 2634 | * Set up kernel memory allocators |
| 2635 | */ |
| 2636 | void __init mm_core_init(void) |
| 2637 | { |
| 2638 | /* Initializations relying on SMP setup */ |
| 2639 | BUILD_BUG_ON(MAX_ZONELISTS > 2); |
| 2640 | build_all_zonelists(NULL); |
| 2641 | page_alloc_init_cpuhp(); |
| 2642 | |
| 2643 | /* |
| 2644 | * page_ext requires contiguous pages, |
| 2645 | * bigger than MAX_PAGE_ORDER unless SPARSEMEM. |
| 2646 | */ |
| 2647 | page_ext_init_flatmem(); |
| 2648 | mem_debugging_and_hardening_init(); |
| 2649 | kfence_alloc_pool_and_metadata(); |
| 2650 | report_meminit(); |
| 2651 | kmsan_init_shadow(); |
| 2652 | stack_depot_early_init(); |
| 2653 | mem_init(); |
| 2654 | kmem_cache_init(); |
| 2655 | /* |
| 2656 | * page_owner must be initialized after buddy is ready, and also after |
| 2657 | * slab is ready so that stack_depot_init() works properly |
| 2658 | */ |
| 2659 | page_ext_init_flatmem_late(); |
| 2660 | kmemleak_init(); |
| 2661 | ptlock_cache_init(); |
| 2662 | pgtable_cache_init(); |
| 2663 | debug_objects_mem_init(); |
| 2664 | vmalloc_init(); |
| 2665 | /* If no deferred init page_ext now, as vmap is fully initialized */ |
| 2666 | if (!deferred_struct_pages) |
| 2667 | page_ext_init(); |
| 2668 | /* Should be run before the first non-init thread is created */ |
| 2669 | init_espfix_bsp(); |
| 2670 | /* Should be run after espfix64 is set up. */ |
| 2671 | pti_init(); |
| 2672 | kmsan_init_runtime(); |
| 2673 | mm_cache_init(); |
| 2674 | execmem_init(); |
| 2675 | } |