Commit | Line | Data |
---|---|---|
f41f2ed4 MS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
5981611d | 3 | * Optimize vmemmap pages associated with HugeTLB |
f41f2ed4 MS |
4 | * |
5 | * Copyright (c) 2020, Bytedance. All rights reserved. | |
6 | * | |
7 | * Author: Muchun Song <songmuchun@bytedance.com> | |
8 | * | |
60a427db | 9 | * See Documentation/vm/vmemmap_dedup.rst |
f41f2ed4 | 10 | */ |
e9fdff87 MS |
11 | #define pr_fmt(fmt) "HugeTLB: " fmt |
12 | ||
f41f2ed4 MS |
13 | #include "hugetlb_vmemmap.h" |
14 | ||
15 | /* | |
16 | * There are a lot of struct page structures associated with each HugeTLB page. | |
17 | * For tail pages, the value of compound_head is the same. So we can reuse first | |
e7d32485 MS |
18 | * page of head page structures. We map the virtual addresses of all the pages |
19 | * of tail page structures to the head page struct, and then free these page | |
20 | * frames. Therefore, we need to reserve one pages as vmemmap areas. | |
f41f2ed4 | 21 | */ |
e7d32485 | 22 | #define RESERVE_VMEMMAP_NR 1U |
f41f2ed4 MS |
23 | #define RESERVE_VMEMMAP_SIZE (RESERVE_VMEMMAP_NR << PAGE_SHIFT) |
24 | ||
47010c04 | 25 | DEFINE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON, |
f10f1442 MS |
26 | hugetlb_optimize_vmemmap_key); |
27 | EXPORT_SYMBOL(hugetlb_optimize_vmemmap_key); | |
e9fdff87 | 28 | |
5981611d | 29 | static int __init hugetlb_vmemmap_early_param(char *buf) |
e9fdff87 | 30 | { |
9c54c522 MS |
31 | bool enable; |
32 | ||
33 | if (kstrtobool(buf, &enable)) | |
e9fdff87 MS |
34 | return -EINVAL; |
35 | ||
9c54c522 | 36 | if (enable) |
f10f1442 | 37 | static_branch_enable(&hugetlb_optimize_vmemmap_key); |
e6d41f12 | 38 | else |
9c54c522 | 39 | static_branch_disable(&hugetlb_optimize_vmemmap_key); |
e9fdff87 MS |
40 | |
41 | return 0; | |
42 | } | |
5981611d | 43 | early_param("hugetlb_free_vmemmap", hugetlb_vmemmap_early_param); |
f41f2ed4 | 44 | |
ad2fa371 MS |
45 | /* |
46 | * Previously discarded vmemmap pages will be allocated and remapping | |
47 | * after this function returns zero. | |
48 | */ | |
5981611d | 49 | int hugetlb_vmemmap_alloc(struct hstate *h, struct page *head) |
ad2fa371 MS |
50 | { |
51 | int ret; | |
52 | unsigned long vmemmap_addr = (unsigned long)head; | |
5981611d | 53 | unsigned long vmemmap_end, vmemmap_reuse, vmemmap_pages; |
ad2fa371 MS |
54 | |
55 | if (!HPageVmemmapOptimized(head)) | |
56 | return 0; | |
57 | ||
5981611d MS |
58 | vmemmap_addr += RESERVE_VMEMMAP_SIZE; |
59 | vmemmap_pages = hugetlb_optimize_vmemmap_pages(h); | |
60 | vmemmap_end = vmemmap_addr + (vmemmap_pages << PAGE_SHIFT); | |
61 | vmemmap_reuse = vmemmap_addr - PAGE_SIZE; | |
62 | ||
ad2fa371 MS |
63 | /* |
64 | * The pages which the vmemmap virtual address range [@vmemmap_addr, | |
65 | * @vmemmap_end) are mapped to are freed to the buddy allocator, and | |
66 | * the range is mapped to the page which @vmemmap_reuse is mapped to. | |
67 | * When a HugeTLB page is freed to the buddy allocator, previously | |
68 | * discarded vmemmap pages must be allocated and remapping. | |
69 | */ | |
70 | ret = vmemmap_remap_alloc(vmemmap_addr, vmemmap_end, vmemmap_reuse, | |
71 | GFP_KERNEL | __GFP_NORETRY | __GFP_THISNODE); | |
ad2fa371 MS |
72 | if (!ret) |
73 | ClearHPageVmemmapOptimized(head); | |
74 | ||
75 | return ret; | |
76 | } | |
77 | ||
5981611d | 78 | void hugetlb_vmemmap_free(struct hstate *h, struct page *head) |
f41f2ed4 MS |
79 | { |
80 | unsigned long vmemmap_addr = (unsigned long)head; | |
5981611d | 81 | unsigned long vmemmap_end, vmemmap_reuse, vmemmap_pages; |
f41f2ed4 | 82 | |
5981611d MS |
83 | vmemmap_pages = hugetlb_optimize_vmemmap_pages(h); |
84 | if (!vmemmap_pages) | |
f41f2ed4 MS |
85 | return; |
86 | ||
5981611d MS |
87 | vmemmap_addr += RESERVE_VMEMMAP_SIZE; |
88 | vmemmap_end = vmemmap_addr + (vmemmap_pages << PAGE_SHIFT); | |
89 | vmemmap_reuse = vmemmap_addr - PAGE_SIZE; | |
f41f2ed4 MS |
90 | |
91 | /* | |
92 | * Remap the vmemmap virtual address range [@vmemmap_addr, @vmemmap_end) | |
93 | * to the page which @vmemmap_reuse is mapped to, then free the pages | |
94 | * which the range [@vmemmap_addr, @vmemmap_end] is mapped to. | |
95 | */ | |
3bc2b6a7 MS |
96 | if (!vmemmap_remap_free(vmemmap_addr, vmemmap_end, vmemmap_reuse)) |
97 | SetHPageVmemmapOptimized(head); | |
f41f2ed4 | 98 | } |
77490587 MS |
99 | |
100 | void __init hugetlb_vmemmap_init(struct hstate *h) | |
101 | { | |
102 | unsigned int nr_pages = pages_per_huge_page(h); | |
103 | unsigned int vmemmap_pages; | |
104 | ||
105 | /* | |
106 | * There are only (RESERVE_VMEMMAP_SIZE / sizeof(struct page)) struct | |
47010c04 | 107 | * page structs that can be used when CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP, |
77490587 MS |
108 | * so add a BUILD_BUG_ON to catch invalid usage of the tail struct page. |
109 | */ | |
110 | BUILD_BUG_ON(__NR_USED_SUBPAGE >= | |
111 | RESERVE_VMEMMAP_SIZE / sizeof(struct page)); | |
112 | ||
f10f1442 | 113 | if (!hugetlb_optimize_vmemmap_enabled()) |
77490587 MS |
114 | return; |
115 | ||
0effdf46 MS |
116 | if (!is_power_of_2(sizeof(struct page))) { |
117 | pr_warn_once("cannot optimize vmemmap pages because \"struct page\" crosses page boundaries\n"); | |
118 | static_branch_disable(&hugetlb_optimize_vmemmap_key); | |
119 | return; | |
120 | } | |
121 | ||
77490587 MS |
122 | vmemmap_pages = (nr_pages * sizeof(struct page)) >> PAGE_SHIFT; |
123 | /* | |
e7d32485 MS |
124 | * The head page is not to be freed to buddy allocator, the other tail |
125 | * pages will map to the head page, so they can be freed. | |
77490587 MS |
126 | * |
127 | * Could RESERVE_VMEMMAP_NR be greater than @vmemmap_pages? It is true | |
128 | * on some architectures (e.g. aarch64). See Documentation/arm64/ | |
129 | * hugetlbpage.rst for more details. | |
130 | */ | |
131 | if (likely(vmemmap_pages > RESERVE_VMEMMAP_NR)) | |
5981611d | 132 | h->optimize_vmemmap_pages = vmemmap_pages - RESERVE_VMEMMAP_NR; |
77490587 | 133 | |
5981611d MS |
134 | pr_info("can optimize %d vmemmap pages for %s\n", |
135 | h->optimize_vmemmap_pages, h->name); | |
77490587 | 136 | } |