Commit | Line | Data |
---|---|---|
867e359b CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * TILE Huge TLB Page Support for Kernel. | |
15 | * Taken from i386 hugetlb implementation: | |
16 | * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com> | |
17 | */ | |
18 | ||
19 | #include <linux/init.h> | |
20 | #include <linux/fs.h> | |
21 | #include <linux/mm.h> | |
22 | #include <linux/hugetlb.h> | |
23 | #include <linux/pagemap.h> | |
867e359b CM |
24 | #include <linux/slab.h> |
25 | #include <linux/err.h> | |
26 | #include <linux/sysctl.h> | |
27 | #include <linux/mman.h> | |
28 | #include <asm/tlb.h> | |
29 | #include <asm/tlbflush.h> | |
30 | ||
31 | pte_t *huge_pte_alloc(struct mm_struct *mm, | |
32 | unsigned long addr, unsigned long sz) | |
33 | { | |
34 | pgd_t *pgd; | |
35 | pud_t *pud; | |
36 | pte_t *pte = NULL; | |
37 | ||
38 | /* We do not yet support multiple huge page sizes. */ | |
39 | BUG_ON(sz != PMD_SIZE); | |
40 | ||
41 | pgd = pgd_offset(mm, addr); | |
42 | pud = pud_alloc(mm, pgd, addr); | |
43 | if (pud) | |
44 | pte = (pte_t *) pmd_alloc(mm, pud, addr); | |
45 | BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); | |
46 | ||
47 | return pte; | |
48 | } | |
49 | ||
50 | pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) | |
51 | { | |
52 | pgd_t *pgd; | |
53 | pud_t *pud; | |
54 | pmd_t *pmd = NULL; | |
55 | ||
56 | pgd = pgd_offset(mm, addr); | |
57 | if (pgd_present(*pgd)) { | |
58 | pud = pud_offset(pgd, addr); | |
59 | if (pud_present(*pud)) | |
60 | pmd = pmd_offset(pud, addr); | |
61 | } | |
62 | return (pte_t *) pmd; | |
63 | } | |
64 | ||
65 | #ifdef HUGETLB_TEST | |
66 | struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, | |
67 | int write) | |
68 | { | |
69 | unsigned long start = address; | |
70 | int length = 1; | |
71 | int nr; | |
72 | struct page *page; | |
73 | struct vm_area_struct *vma; | |
74 | ||
75 | vma = find_vma(mm, addr); | |
76 | if (!vma || !is_vm_hugetlb_page(vma)) | |
77 | return ERR_PTR(-EINVAL); | |
78 | ||
79 | pte = huge_pte_offset(mm, address); | |
80 | ||
81 | /* hugetlb should be locked, and hence, prefaulted */ | |
82 | WARN_ON(!pte || pte_none(*pte)); | |
83 | ||
84 | page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)]; | |
85 | ||
86 | WARN_ON(!PageHead(page)); | |
87 | ||
88 | return page; | |
89 | } | |
90 | ||
91 | int pmd_huge(pmd_t pmd) | |
92 | { | |
93 | return 0; | |
94 | } | |
95 | ||
96 | int pud_huge(pud_t pud) | |
97 | { | |
98 | return 0; | |
99 | } | |
100 | ||
101 | struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, | |
102 | pmd_t *pmd, int write) | |
103 | { | |
104 | return NULL; | |
105 | } | |
106 | ||
107 | #else | |
108 | ||
109 | struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, | |
110 | int write) | |
111 | { | |
112 | return ERR_PTR(-EINVAL); | |
113 | } | |
114 | ||
115 | int pmd_huge(pmd_t pmd) | |
116 | { | |
117 | return !!(pmd_val(pmd) & _PAGE_HUGE_PAGE); | |
118 | } | |
119 | ||
120 | int pud_huge(pud_t pud) | |
121 | { | |
122 | return !!(pud_val(pud) & _PAGE_HUGE_PAGE); | |
123 | } | |
124 | ||
125 | struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, | |
126 | pmd_t *pmd, int write) | |
127 | { | |
128 | struct page *page; | |
129 | ||
130 | page = pte_page(*(pte_t *)pmd); | |
131 | if (page) | |
132 | page += ((address & ~PMD_MASK) >> PAGE_SHIFT); | |
133 | return page; | |
134 | } | |
135 | ||
136 | struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address, | |
137 | pud_t *pud, int write) | |
138 | { | |
139 | struct page *page; | |
140 | ||
141 | page = pte_page(*(pte_t *)pud); | |
142 | if (page) | |
143 | page += ((address & ~PUD_MASK) >> PAGE_SHIFT); | |
144 | return page; | |
145 | } | |
146 | ||
147 | int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) | |
148 | { | |
149 | return 0; | |
150 | } | |
151 | ||
152 | #endif | |
153 | ||
154 | #ifdef HAVE_ARCH_HUGETLB_UNMAPPED_AREA | |
155 | static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, | |
156 | unsigned long addr, unsigned long len, | |
157 | unsigned long pgoff, unsigned long flags) | |
158 | { | |
159 | struct hstate *h = hstate_file(file); | |
160 | struct mm_struct *mm = current->mm; | |
161 | struct vm_area_struct *vma; | |
162 | unsigned long start_addr; | |
163 | ||
164 | if (len > mm->cached_hole_size) { | |
165 | start_addr = mm->free_area_cache; | |
166 | } else { | |
167 | start_addr = TASK_UNMAPPED_BASE; | |
168 | mm->cached_hole_size = 0; | |
169 | } | |
170 | ||
171 | full_search: | |
172 | addr = ALIGN(start_addr, huge_page_size(h)); | |
173 | ||
174 | for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { | |
175 | /* At this point: (!vma || addr < vma->vm_end). */ | |
176 | if (TASK_SIZE - len < addr) { | |
177 | /* | |
178 | * Start a new search - just in case we missed | |
179 | * some holes. | |
180 | */ | |
181 | if (start_addr != TASK_UNMAPPED_BASE) { | |
182 | start_addr = TASK_UNMAPPED_BASE; | |
183 | mm->cached_hole_size = 0; | |
184 | goto full_search; | |
185 | } | |
186 | return -ENOMEM; | |
187 | } | |
188 | if (!vma || addr + len <= vma->vm_start) { | |
189 | mm->free_area_cache = addr + len; | |
190 | return addr; | |
191 | } | |
192 | if (addr + mm->cached_hole_size < vma->vm_start) | |
193 | mm->cached_hole_size = vma->vm_start - addr; | |
194 | addr = ALIGN(vma->vm_end, huge_page_size(h)); | |
195 | } | |
196 | } | |
197 | ||
198 | static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, | |
199 | unsigned long addr0, unsigned long len, | |
200 | unsigned long pgoff, unsigned long flags) | |
201 | { | |
202 | struct hstate *h = hstate_file(file); | |
203 | struct mm_struct *mm = current->mm; | |
204 | struct vm_area_struct *vma, *prev_vma; | |
205 | unsigned long base = mm->mmap_base, addr = addr0; | |
206 | unsigned long largest_hole = mm->cached_hole_size; | |
207 | int first_time = 1; | |
208 | ||
209 | /* don't allow allocations above current base */ | |
210 | if (mm->free_area_cache > base) | |
211 | mm->free_area_cache = base; | |
212 | ||
213 | if (len <= largest_hole) { | |
214 | largest_hole = 0; | |
215 | mm->free_area_cache = base; | |
216 | } | |
217 | try_again: | |
218 | /* make sure it can fit in the remaining address space */ | |
219 | if (mm->free_area_cache < len) | |
220 | goto fail; | |
221 | ||
25985edc | 222 | /* either no address requested or can't fit in requested address hole */ |
867e359b CM |
223 | addr = (mm->free_area_cache - len) & huge_page_mask(h); |
224 | do { | |
225 | /* | |
226 | * Lookup failure means no vma is above this address, | |
227 | * i.e. return with success: | |
228 | */ | |
229 | vma = find_vma_prev(mm, addr, &prev_vma); | |
230 | if (!vma) { | |
231 | return addr; | |
232 | break; | |
233 | } | |
234 | ||
235 | /* | |
236 | * new region fits between prev_vma->vm_end and | |
237 | * vma->vm_start, use it: | |
238 | */ | |
239 | if (addr + len <= vma->vm_start && | |
240 | (!prev_vma || (addr >= prev_vma->vm_end))) { | |
241 | /* remember the address as a hint for next time */ | |
242 | mm->cached_hole_size = largest_hole; | |
243 | mm->free_area_cache = addr; | |
244 | return addr; | |
245 | } else { | |
246 | /* pull free_area_cache down to the first hole */ | |
247 | if (mm->free_area_cache == vma->vm_end) { | |
248 | mm->free_area_cache = vma->vm_start; | |
249 | mm->cached_hole_size = largest_hole; | |
250 | } | |
251 | } | |
252 | ||
253 | /* remember the largest hole we saw so far */ | |
254 | if (addr + largest_hole < vma->vm_start) | |
255 | largest_hole = vma->vm_start - addr; | |
256 | ||
257 | /* try just below the current vma->vm_start */ | |
258 | addr = (vma->vm_start - len) & huge_page_mask(h); | |
259 | ||
260 | } while (len <= vma->vm_start); | |
261 | ||
262 | fail: | |
263 | /* | |
264 | * if hint left us with no space for the requested | |
265 | * mapping then try again: | |
266 | */ | |
267 | if (first_time) { | |
268 | mm->free_area_cache = base; | |
269 | largest_hole = 0; | |
270 | first_time = 0; | |
271 | goto try_again; | |
272 | } | |
273 | /* | |
274 | * A failed mmap() very likely causes application failure, | |
275 | * so fall back to the bottom-up function here. This scenario | |
276 | * can happen with large stack limits and large mmap() | |
277 | * allocations. | |
278 | */ | |
279 | mm->free_area_cache = TASK_UNMAPPED_BASE; | |
280 | mm->cached_hole_size = ~0UL; | |
281 | addr = hugetlb_get_unmapped_area_bottomup(file, addr0, | |
282 | len, pgoff, flags); | |
283 | ||
284 | /* | |
285 | * Restore the topdown base: | |
286 | */ | |
287 | mm->free_area_cache = base; | |
288 | mm->cached_hole_size = ~0UL; | |
289 | ||
290 | return addr; | |
291 | } | |
292 | ||
293 | unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, | |
294 | unsigned long len, unsigned long pgoff, unsigned long flags) | |
295 | { | |
296 | struct hstate *h = hstate_file(file); | |
297 | struct mm_struct *mm = current->mm; | |
298 | struct vm_area_struct *vma; | |
299 | ||
300 | if (len & ~huge_page_mask(h)) | |
301 | return -EINVAL; | |
302 | if (len > TASK_SIZE) | |
303 | return -ENOMEM; | |
304 | ||
305 | if (flags & MAP_FIXED) { | |
306 | if (prepare_hugepage_range(file, addr, len)) | |
307 | return -EINVAL; | |
308 | return addr; | |
309 | } | |
310 | ||
311 | if (addr) { | |
312 | addr = ALIGN(addr, huge_page_size(h)); | |
313 | vma = find_vma(mm, addr); | |
314 | if (TASK_SIZE - len >= addr && | |
315 | (!vma || addr + len <= vma->vm_start)) | |
316 | return addr; | |
317 | } | |
318 | if (current->mm->get_unmapped_area == arch_get_unmapped_area) | |
319 | return hugetlb_get_unmapped_area_bottomup(file, addr, len, | |
320 | pgoff, flags); | |
321 | else | |
322 | return hugetlb_get_unmapped_area_topdown(file, addr, len, | |
323 | pgoff, flags); | |
324 | } | |
325 | ||
326 | static __init int setup_hugepagesz(char *opt) | |
327 | { | |
328 | unsigned long ps = memparse(opt, &opt); | |
329 | if (ps == PMD_SIZE) { | |
330 | hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT); | |
331 | } else if (ps == PUD_SIZE) { | |
332 | hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); | |
333 | } else { | |
0707ad30 | 334 | pr_err("hugepagesz: Unsupported page size %lu M\n", |
867e359b CM |
335 | ps >> 20); |
336 | return 0; | |
337 | } | |
338 | return 1; | |
339 | } | |
340 | __setup("hugepagesz=", setup_hugepagesz); | |
341 | ||
342 | #endif /*HAVE_ARCH_HUGETLB_UNMAPPED_AREA*/ |