Commit | Line | Data |
---|---|---|
f41f2ed4 MS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
5981611d | 3 | * Optimize vmemmap pages associated with HugeTLB |
f41f2ed4 MS |
4 | * |
5 | * Copyright (c) 2020, Bytedance. All rights reserved. | |
6 | * | |
7 | * Author: Muchun Song <songmuchun@bytedance.com> | |
8 | * | |
60a427db | 9 | * See Documentation/vm/vmemmap_dedup.rst |
f41f2ed4 | 10 | */ |
e9fdff87 MS |
11 | #define pr_fmt(fmt) "HugeTLB: " fmt |
12 | ||
f41f2ed4 MS |
13 | #include "hugetlb_vmemmap.h" |
14 | ||
15 | /* | |
16 | * There are a lot of struct page structures associated with each HugeTLB page. | |
17 | * For tail pages, the value of compound_head is the same. So we can reuse first | |
e7d32485 MS |
18 | * page of head page structures. We map the virtual addresses of all the pages |
19 | * of tail page structures to the head page struct, and then free these page | |
20 | * frames. Therefore, we need to reserve one pages as vmemmap areas. | |
f41f2ed4 | 21 | */ |
e7d32485 | 22 | #define RESERVE_VMEMMAP_NR 1U |
f41f2ed4 MS |
23 | #define RESERVE_VMEMMAP_SIZE (RESERVE_VMEMMAP_NR << PAGE_SHIFT) |
24 | ||
47010c04 | 25 | DEFINE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON, |
f10f1442 MS |
26 | hugetlb_optimize_vmemmap_key); |
27 | EXPORT_SYMBOL(hugetlb_optimize_vmemmap_key); | |
e9fdff87 | 28 | |
5981611d | 29 | static int __init hugetlb_vmemmap_early_param(char *buf) |
e9fdff87 MS |
30 | { |
31 | /* We cannot optimize if a "struct page" crosses page boundaries. */ | |
e7d32485 | 32 | if (!is_power_of_2(sizeof(struct page))) { |
e9fdff87 MS |
33 | pr_warn("cannot free vmemmap pages because \"struct page\" crosses page boundaries\n"); |
34 | return 0; | |
35 | } | |
36 | ||
37 | if (!buf) | |
38 | return -EINVAL; | |
39 | ||
40 | if (!strcmp(buf, "on")) | |
f10f1442 | 41 | static_branch_enable(&hugetlb_optimize_vmemmap_key); |
e6d41f12 | 42 | else if (!strcmp(buf, "off")) |
f10f1442 | 43 | static_branch_disable(&hugetlb_optimize_vmemmap_key); |
e6d41f12 | 44 | else |
e9fdff87 MS |
45 | return -EINVAL; |
46 | ||
47 | return 0; | |
48 | } | |
5981611d | 49 | early_param("hugetlb_free_vmemmap", hugetlb_vmemmap_early_param); |
f41f2ed4 | 50 | |
ad2fa371 MS |
51 | /* |
52 | * Previously discarded vmemmap pages will be allocated and remapping | |
53 | * after this function returns zero. | |
54 | */ | |
5981611d | 55 | int hugetlb_vmemmap_alloc(struct hstate *h, struct page *head) |
ad2fa371 MS |
56 | { |
57 | int ret; | |
58 | unsigned long vmemmap_addr = (unsigned long)head; | |
5981611d | 59 | unsigned long vmemmap_end, vmemmap_reuse, vmemmap_pages; |
ad2fa371 MS |
60 | |
61 | if (!HPageVmemmapOptimized(head)) | |
62 | return 0; | |
63 | ||
5981611d MS |
64 | vmemmap_addr += RESERVE_VMEMMAP_SIZE; |
65 | vmemmap_pages = hugetlb_optimize_vmemmap_pages(h); | |
66 | vmemmap_end = vmemmap_addr + (vmemmap_pages << PAGE_SHIFT); | |
67 | vmemmap_reuse = vmemmap_addr - PAGE_SIZE; | |
68 | ||
ad2fa371 MS |
69 | /* |
70 | * The pages which the vmemmap virtual address range [@vmemmap_addr, | |
71 | * @vmemmap_end) are mapped to are freed to the buddy allocator, and | |
72 | * the range is mapped to the page which @vmemmap_reuse is mapped to. | |
73 | * When a HugeTLB page is freed to the buddy allocator, previously | |
74 | * discarded vmemmap pages must be allocated and remapping. | |
75 | */ | |
76 | ret = vmemmap_remap_alloc(vmemmap_addr, vmemmap_end, vmemmap_reuse, | |
77 | GFP_KERNEL | __GFP_NORETRY | __GFP_THISNODE); | |
ad2fa371 MS |
78 | if (!ret) |
79 | ClearHPageVmemmapOptimized(head); | |
80 | ||
81 | return ret; | |
82 | } | |
83 | ||
5981611d | 84 | void hugetlb_vmemmap_free(struct hstate *h, struct page *head) |
f41f2ed4 MS |
85 | { |
86 | unsigned long vmemmap_addr = (unsigned long)head; | |
5981611d | 87 | unsigned long vmemmap_end, vmemmap_reuse, vmemmap_pages; |
f41f2ed4 | 88 | |
5981611d MS |
89 | vmemmap_pages = hugetlb_optimize_vmemmap_pages(h); |
90 | if (!vmemmap_pages) | |
f41f2ed4 MS |
91 | return; |
92 | ||
5981611d MS |
93 | vmemmap_addr += RESERVE_VMEMMAP_SIZE; |
94 | vmemmap_end = vmemmap_addr + (vmemmap_pages << PAGE_SHIFT); | |
95 | vmemmap_reuse = vmemmap_addr - PAGE_SIZE; | |
f41f2ed4 MS |
96 | |
97 | /* | |
98 | * Remap the vmemmap virtual address range [@vmemmap_addr, @vmemmap_end) | |
99 | * to the page which @vmemmap_reuse is mapped to, then free the pages | |
100 | * which the range [@vmemmap_addr, @vmemmap_end] is mapped to. | |
101 | */ | |
3bc2b6a7 MS |
102 | if (!vmemmap_remap_free(vmemmap_addr, vmemmap_end, vmemmap_reuse)) |
103 | SetHPageVmemmapOptimized(head); | |
f41f2ed4 | 104 | } |
77490587 MS |
105 | |
106 | void __init hugetlb_vmemmap_init(struct hstate *h) | |
107 | { | |
108 | unsigned int nr_pages = pages_per_huge_page(h); | |
109 | unsigned int vmemmap_pages; | |
110 | ||
111 | /* | |
112 | * There are only (RESERVE_VMEMMAP_SIZE / sizeof(struct page)) struct | |
47010c04 | 113 | * page structs that can be used when CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP, |
77490587 MS |
114 | * so add a BUILD_BUG_ON to catch invalid usage of the tail struct page. |
115 | */ | |
116 | BUILD_BUG_ON(__NR_USED_SUBPAGE >= | |
117 | RESERVE_VMEMMAP_SIZE / sizeof(struct page)); | |
118 | ||
f10f1442 | 119 | if (!hugetlb_optimize_vmemmap_enabled()) |
77490587 MS |
120 | return; |
121 | ||
122 | vmemmap_pages = (nr_pages * sizeof(struct page)) >> PAGE_SHIFT; | |
123 | /* | |
e7d32485 MS |
124 | * The head page is not to be freed to buddy allocator, the other tail |
125 | * pages will map to the head page, so they can be freed. | |
77490587 MS |
126 | * |
127 | * Could RESERVE_VMEMMAP_NR be greater than @vmemmap_pages? It is true | |
128 | * on some architectures (e.g. aarch64). See Documentation/arm64/ | |
129 | * hugetlbpage.rst for more details. | |
130 | */ | |
131 | if (likely(vmemmap_pages > RESERVE_VMEMMAP_NR)) | |
5981611d | 132 | h->optimize_vmemmap_pages = vmemmap_pages - RESERVE_VMEMMAP_NR; |
77490587 | 133 | |
5981611d MS |
134 | pr_info("can optimize %d vmemmap pages for %s\n", |
135 | h->optimize_vmemmap_pages, h->name); | |
77490587 | 136 | } |