Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * IA-32 Huge TLB Page Support for Kernel. | |
4 | * | |
5 | * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com> | |
6 | */ | |
7 | ||
1da177e4 LT |
8 | #include <linux/init.h> |
9 | #include <linux/fs.h> | |
10 | #include <linux/mm.h> | |
01042607 | 11 | #include <linux/sched/mm.h> |
1da177e4 LT |
12 | #include <linux/hugetlb.h> |
13 | #include <linux/pagemap.h> | |
1da177e4 LT |
14 | #include <linux/err.h> |
15 | #include <linux/sysctl.h> | |
e13b73dd | 16 | #include <linux/compat.h> |
1da177e4 LT |
17 | #include <asm/mman.h> |
18 | #include <asm/tlb.h> | |
19 | #include <asm/tlbflush.h> | |
a5a19c63 | 20 | #include <asm/pgalloc.h> |
e13b73dd | 21 | #include <asm/elf.h> |
44b04912 | 22 | #include <asm/mpx.h> |
1da177e4 | 23 | |
1da177e4 LT |
24 | #if 0 /* This is just for testing */ |
25 | struct page * | |
26 | follow_huge_addr(struct mm_struct *mm, unsigned long address, int write) | |
27 | { | |
28 | unsigned long start = address; | |
29 | int length = 1; | |
30 | int nr; | |
31 | struct page *page; | |
32 | struct vm_area_struct *vma; | |
33 | ||
34 | vma = find_vma(mm, addr); | |
35 | if (!vma || !is_vm_hugetlb_page(vma)) | |
36 | return ERR_PTR(-EINVAL); | |
37 | ||
7868a208 | 38 | pte = huge_pte_offset(mm, address, vma_mmu_pagesize(vma)); |
1da177e4 LT |
39 | |
40 | /* hugetlb should be locked, and hence, prefaulted */ | |
41 | WARN_ON(!pte || pte_none(*pte)); | |
42 | ||
43 | page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)]; | |
44 | ||
25e59881 | 45 | WARN_ON(!PageHead(page)); |
1da177e4 LT |
46 | |
47 | return page; | |
48 | } | |
49 | ||
50 | int pmd_huge(pmd_t pmd) | |
51 | { | |
52 | return 0; | |
53 | } | |
54 | ||
ceb86879 AK |
55 | int pud_huge(pud_t pud) |
56 | { | |
57 | return 0; | |
58 | } | |
59 | ||
1da177e4 LT |
60 | #else |
61 | ||
cbef8478 NH |
62 | /* |
63 | * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal | |
64 | * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry. | |
65 | * Otherwise, returns 0. | |
66 | */ | |
1da177e4 LT |
67 | int pmd_huge(pmd_t pmd) |
68 | { | |
cbef8478 NH |
69 | return !pmd_none(pmd) && |
70 | (pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_PSE)) != _PAGE_PRESENT; | |
1da177e4 LT |
71 | } |
72 | ||
ceb86879 AK |
73 | int pud_huge(pud_t pud) |
74 | { | |
39c11e6c | 75 | return !!(pud_val(pud) & _PAGE_PSE); |
ceb86879 | 76 | } |
1da177e4 LT |
77 | #endif |
78 | ||
fd8526ad | 79 | #ifdef CONFIG_HUGETLB_PAGE |
1da177e4 LT |
80 | static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, |
81 | unsigned long addr, unsigned long len, | |
82 | unsigned long pgoff, unsigned long flags) | |
83 | { | |
39c11e6c | 84 | struct hstate *h = hstate_file(file); |
cdc17344 ML |
85 | struct vm_unmapped_area_info info; |
86 | ||
87 | info.flags = 0; | |
88 | info.length = len; | |
e13b73dd | 89 | info.low_limit = get_mmap_base(1); |
b569bab7 KS |
90 | |
91 | /* | |
92 | * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area | |
93 | * in the full address space. | |
94 | */ | |
a846446b | 95 | info.high_limit = in_32bit_syscall() ? |
b569bab7 KS |
96 | task_size_32bit() : task_size_64bit(addr > DEFAULT_MAP_WINDOW); |
97 | ||
cdc17344 ML |
98 | info.align_mask = PAGE_MASK & ~huge_page_mask(h); |
99 | info.align_offset = 0; | |
100 | return vm_unmapped_area(&info); | |
1da177e4 LT |
101 | } |
102 | ||
103 | static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, | |
b569bab7 | 104 | unsigned long addr, unsigned long len, |
1da177e4 LT |
105 | unsigned long pgoff, unsigned long flags) |
106 | { | |
39c11e6c | 107 | struct hstate *h = hstate_file(file); |
cdc17344 | 108 | struct vm_unmapped_area_info info; |
1da177e4 | 109 | |
cdc17344 ML |
110 | info.flags = VM_UNMAPPED_AREA_TOPDOWN; |
111 | info.length = len; | |
112 | info.low_limit = PAGE_SIZE; | |
e13b73dd | 113 | info.high_limit = get_mmap_base(0); |
b569bab7 KS |
114 | |
115 | /* | |
116 | * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area | |
117 | * in the full address space. | |
118 | */ | |
a846446b | 119 | if (addr > DEFAULT_MAP_WINDOW && !in_32bit_syscall()) |
b569bab7 KS |
120 | info.high_limit += TASK_SIZE_MAX - DEFAULT_MAP_WINDOW; |
121 | ||
cdc17344 ML |
122 | info.align_mask = PAGE_MASK & ~huge_page_mask(h); |
123 | info.align_offset = 0; | |
124 | addr = vm_unmapped_area(&info); | |
1da177e4 | 125 | |
1da177e4 LT |
126 | /* |
127 | * A failed mmap() very likely causes application failure, | |
128 | * so fall back to the bottom-up function here. This scenario | |
129 | * can happen with large stack limits and large mmap() | |
130 | * allocations. | |
131 | */ | |
cdc17344 ML |
132 | if (addr & ~PAGE_MASK) { |
133 | VM_BUG_ON(addr != -ENOMEM); | |
134 | info.flags = 0; | |
135 | info.low_limit = TASK_UNMAPPED_BASE; | |
b569bab7 | 136 | info.high_limit = TASK_SIZE_LOW; |
cdc17344 ML |
137 | addr = vm_unmapped_area(&info); |
138 | } | |
1da177e4 LT |
139 | |
140 | return addr; | |
141 | } | |
142 | ||
143 | unsigned long | |
144 | hugetlb_get_unmapped_area(struct file *file, unsigned long addr, | |
145 | unsigned long len, unsigned long pgoff, unsigned long flags) | |
146 | { | |
39c11e6c | 147 | struct hstate *h = hstate_file(file); |
1da177e4 LT |
148 | struct mm_struct *mm = current->mm; |
149 | struct vm_area_struct *vma; | |
150 | ||
39c11e6c | 151 | if (len & ~huge_page_mask(h)) |
1da177e4 | 152 | return -EINVAL; |
44b04912 KS |
153 | |
154 | addr = mpx_unmapped_area_check(addr, len, flags); | |
155 | if (IS_ERR_VALUE(addr)) | |
156 | return addr; | |
157 | ||
1da177e4 LT |
158 | if (len > TASK_SIZE) |
159 | return -ENOMEM; | |
160 | ||
1e0f25db | 161 | /* No address checking. See comment at mmap_address_hint_valid() */ |
5a8130f2 | 162 | if (flags & MAP_FIXED) { |
a5516438 | 163 | if (prepare_hugepage_range(file, addr, len)) |
5a8130f2 BH |
164 | return -EINVAL; |
165 | return addr; | |
166 | } | |
167 | ||
1da177e4 | 168 | if (addr) { |
1e0f25db KS |
169 | addr &= huge_page_mask(h); |
170 | if (!mmap_address_hint_valid(addr, len)) | |
171 | goto get_unmapped_area; | |
172 | ||
1da177e4 | 173 | vma = find_vma(mm, addr); |
1e0f25db | 174 | if (!vma || addr + len <= vm_start_gap(vma)) |
1da177e4 LT |
175 | return addr; |
176 | } | |
1e0f25db KS |
177 | |
178 | get_unmapped_area: | |
1da177e4 LT |
179 | if (mm->get_unmapped_area == arch_get_unmapped_area) |
180 | return hugetlb_get_unmapped_area_bottomup(file, addr, len, | |
181 | pgoff, flags); | |
182 | else | |
183 | return hugetlb_get_unmapped_area_topdown(file, addr, len, | |
184 | pgoff, flags); | |
185 | } | |
fd8526ad | 186 | #endif /* CONFIG_HUGETLB_PAGE */ |
1da177e4 | 187 | |
b4718e62 AK |
188 | #ifdef CONFIG_X86_64 |
189 | static __init int setup_hugepagesz(char *opt) | |
190 | { | |
191 | unsigned long ps = memparse(opt, &opt); | |
192 | if (ps == PMD_SIZE) { | |
193 | hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); | |
b8291adc | 194 | } else if (ps == PUD_SIZE && boot_cpu_has(X86_FEATURE_GBPAGES)) { |
b4718e62 AK |
195 | hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); |
196 | } else { | |
2b18e532 | 197 | hugetlb_bad_size(); |
b4718e62 AK |
198 | printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n", |
199 | ps >> 20); | |
200 | return 0; | |
201 | } | |
202 | return 1; | |
203 | } | |
204 | __setup("hugepagesz=", setup_hugepagesz); | |
ece84b39 | 205 | |
8df995f6 | 206 | #ifdef CONFIG_CONTIG_ALLOC |
ece84b39 KS |
207 | static __init int gigantic_pages_init(void) |
208 | { | |
080fe206 | 209 | /* With compaction or CMA we can allocate gigantic pages at runtime */ |
b8291adc | 210 | if (boot_cpu_has(X86_FEATURE_GBPAGES) && !size_to_hstate(1UL << PUD_SHIFT)) |
ece84b39 KS |
211 | hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); |
212 | return 0; | |
213 | } | |
214 | arch_initcall(gigantic_pages_init); | |
215 | #endif | |
b4718e62 | 216 | #endif |