Merge branch 'topic/omap' into for-linus
[linux-2.6-block.git] / mm / hugetlb.c
CommitLineData
1da177e4
LT
1/*
2 * Generic hugetlb support.
6d49e352 3 * (C) Nadia Yvette Chambers, April 2004
1da177e4 4 */
1da177e4
LT
5#include <linux/list.h>
6#include <linux/init.h>
7#include <linux/module.h>
8#include <linux/mm.h>
e1759c21 9#include <linux/seq_file.h>
1da177e4
LT
10#include <linux/sysctl.h>
11#include <linux/highmem.h>
cddb8a5c 12#include <linux/mmu_notifier.h>
1da177e4 13#include <linux/nodemask.h>
63551ae0 14#include <linux/pagemap.h>
5da7ca86 15#include <linux/mempolicy.h>
3b32123d 16#include <linux/compiler.h>
aea47ff3 17#include <linux/cpuset.h>
3935baa9 18#include <linux/mutex.h>
aa888a74 19#include <linux/bootmem.h>
a3437870 20#include <linux/sysfs.h>
5a0e3ad6 21#include <linux/slab.h>
0fe6e20b 22#include <linux/rmap.h>
fd6a03ed
NH
23#include <linux/swap.h>
24#include <linux/swapops.h>
c8721bbb 25#include <linux/page-isolation.h>
8382d914 26#include <linux/jhash.h>
d6606683 27
63551ae0
DG
28#include <asm/page.h>
29#include <asm/pgtable.h>
24669e58 30#include <asm/tlb.h>
63551ae0 31
24669e58 32#include <linux/io.h>
63551ae0 33#include <linux/hugetlb.h>
9dd540e2 34#include <linux/hugetlb_cgroup.h>
9a305230 35#include <linux/node.h>
7835e98b 36#include "internal.h"
1da177e4 37
753162cd 38int hugepages_treat_as_movable;
a5516438 39
c3f38a38 40int hugetlb_max_hstate __read_mostly;
e5ff2159
AK
41unsigned int default_hstate_idx;
42struct hstate hstates[HUGE_MAX_HSTATE];
43
53ba51d2
JT
44__initdata LIST_HEAD(huge_boot_pages);
45
e5ff2159
AK
46/* for command line parsing */
47static struct hstate * __initdata parsed_hstate;
48static unsigned long __initdata default_hstate_max_huge_pages;
e11bfbfc 49static unsigned long __initdata default_hstate_size;
e5ff2159 50
3935baa9 51/*
31caf665
NH
52 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
53 * free_huge_pages, and surplus_huge_pages.
3935baa9 54 */
c3f38a38 55DEFINE_SPINLOCK(hugetlb_lock);
0bd0f9fb 56
8382d914
DB
57/*
58 * Serializes faults on the same logical page. This is used to
59 * prevent spurious OOMs when the hugepage pool is fully utilized.
60 */
61static int num_fault_mutexes;
62static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;
63
7ca02d0a
MK
64/* Forward declaration */
65static int hugetlb_acct_memory(struct hstate *h, long delta);
66
90481622
DG
67static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
68{
69 bool free = (spool->count == 0) && (spool->used_hpages == 0);
70
71 spin_unlock(&spool->lock);
72
73 /* If no pages are used, and no other handles to the subpool
7ca02d0a
MK
74 * remain, give up any reservations mased on minimum size and
75 * free the subpool */
76 if (free) {
77 if (spool->min_hpages != -1)
78 hugetlb_acct_memory(spool->hstate,
79 -spool->min_hpages);
90481622 80 kfree(spool);
7ca02d0a 81 }
90481622
DG
82}
83
7ca02d0a
MK
84struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
85 long min_hpages)
90481622
DG
86{
87 struct hugepage_subpool *spool;
88
c6a91820 89 spool = kzalloc(sizeof(*spool), GFP_KERNEL);
90481622
DG
90 if (!spool)
91 return NULL;
92
93 spin_lock_init(&spool->lock);
94 spool->count = 1;
7ca02d0a
MK
95 spool->max_hpages = max_hpages;
96 spool->hstate = h;
97 spool->min_hpages = min_hpages;
98
99 if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
100 kfree(spool);
101 return NULL;
102 }
103 spool->rsv_hpages = min_hpages;
90481622
DG
104
105 return spool;
106}
107
108void hugepage_put_subpool(struct hugepage_subpool *spool)
109{
110 spin_lock(&spool->lock);
111 BUG_ON(!spool->count);
112 spool->count--;
113 unlock_or_release_subpool(spool);
114}
115
1c5ecae3
MK
116/*
117 * Subpool accounting for allocating and reserving pages.
118 * Return -ENOMEM if there are not enough resources to satisfy the
119 * the request. Otherwise, return the number of pages by which the
120 * global pools must be adjusted (upward). The returned value may
121 * only be different than the passed value (delta) in the case where
122 * a subpool minimum size must be manitained.
123 */
124static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
90481622
DG
125 long delta)
126{
1c5ecae3 127 long ret = delta;
90481622
DG
128
129 if (!spool)
1c5ecae3 130 return ret;
90481622
DG
131
132 spin_lock(&spool->lock);
1c5ecae3
MK
133
134 if (spool->max_hpages != -1) { /* maximum size accounting */
135 if ((spool->used_hpages + delta) <= spool->max_hpages)
136 spool->used_hpages += delta;
137 else {
138 ret = -ENOMEM;
139 goto unlock_ret;
140 }
90481622 141 }
90481622 142
1c5ecae3
MK
143 if (spool->min_hpages != -1) { /* minimum size accounting */
144 if (delta > spool->rsv_hpages) {
145 /*
146 * Asking for more reserves than those already taken on
147 * behalf of subpool. Return difference.
148 */
149 ret = delta - spool->rsv_hpages;
150 spool->rsv_hpages = 0;
151 } else {
152 ret = 0; /* reserves already accounted for */
153 spool->rsv_hpages -= delta;
154 }
155 }
156
157unlock_ret:
158 spin_unlock(&spool->lock);
90481622
DG
159 return ret;
160}
161
1c5ecae3
MK
162/*
163 * Subpool accounting for freeing and unreserving pages.
164 * Return the number of global page reservations that must be dropped.
165 * The return value may only be different than the passed value (delta)
166 * in the case where a subpool minimum size must be maintained.
167 */
168static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
90481622
DG
169 long delta)
170{
1c5ecae3
MK
171 long ret = delta;
172
90481622 173 if (!spool)
1c5ecae3 174 return delta;
90481622
DG
175
176 spin_lock(&spool->lock);
1c5ecae3
MK
177
178 if (spool->max_hpages != -1) /* maximum size accounting */
179 spool->used_hpages -= delta;
180
181 if (spool->min_hpages != -1) { /* minimum size accounting */
182 if (spool->rsv_hpages + delta <= spool->min_hpages)
183 ret = 0;
184 else
185 ret = spool->rsv_hpages + delta - spool->min_hpages;
186
187 spool->rsv_hpages += delta;
188 if (spool->rsv_hpages > spool->min_hpages)
189 spool->rsv_hpages = spool->min_hpages;
190 }
191
192 /*
193 * If hugetlbfs_put_super couldn't free spool due to an outstanding
194 * quota reference, free it now.
195 */
90481622 196 unlock_or_release_subpool(spool);
1c5ecae3
MK
197
198 return ret;
90481622
DG
199}
200
201static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
202{
203 return HUGETLBFS_SB(inode->i_sb)->spool;
204}
205
206static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
207{
496ad9aa 208 return subpool_inode(file_inode(vma->vm_file));
90481622
DG
209}
210
96822904
AW
211/*
212 * Region tracking -- allows tracking of reservations and instantiated pages
213 * across the pages in a mapping.
84afd99b 214 *
7b24d861
DB
215 * The region data structures are embedded into a resv_map and
216 * protected by a resv_map's lock
96822904
AW
217 */
218struct file_region {
219 struct list_head link;
220 long from;
221 long to;
222};
223
1406ec9b 224static long region_add(struct resv_map *resv, long f, long t)
96822904 225{
1406ec9b 226 struct list_head *head = &resv->regions;
96822904
AW
227 struct file_region *rg, *nrg, *trg;
228
7b24d861 229 spin_lock(&resv->lock);
96822904
AW
230 /* Locate the region we are either in or before. */
231 list_for_each_entry(rg, head, link)
232 if (f <= rg->to)
233 break;
234
235 /* Round our left edge to the current segment if it encloses us. */
236 if (f > rg->from)
237 f = rg->from;
238
239 /* Check for and consume any regions we now overlap with. */
240 nrg = rg;
241 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
242 if (&rg->link == head)
243 break;
244 if (rg->from > t)
245 break;
246
247 /* If this area reaches higher then extend our area to
248 * include it completely. If this is not the first area
249 * which we intend to reuse, free it. */
250 if (rg->to > t)
251 t = rg->to;
252 if (rg != nrg) {
253 list_del(&rg->link);
254 kfree(rg);
255 }
256 }
257 nrg->from = f;
258 nrg->to = t;
7b24d861 259 spin_unlock(&resv->lock);
96822904
AW
260 return 0;
261}
262
1406ec9b 263static long region_chg(struct resv_map *resv, long f, long t)
96822904 264{
1406ec9b 265 struct list_head *head = &resv->regions;
7b24d861 266 struct file_region *rg, *nrg = NULL;
96822904
AW
267 long chg = 0;
268
7b24d861
DB
269retry:
270 spin_lock(&resv->lock);
96822904
AW
271 /* Locate the region we are before or in. */
272 list_for_each_entry(rg, head, link)
273 if (f <= rg->to)
274 break;
275
276 /* If we are below the current region then a new region is required.
277 * Subtle, allocate a new region at the position but make it zero
278 * size such that we can guarantee to record the reservation. */
279 if (&rg->link == head || t < rg->from) {
7b24d861
DB
280 if (!nrg) {
281 spin_unlock(&resv->lock);
282 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
283 if (!nrg)
284 return -ENOMEM;
285
286 nrg->from = f;
287 nrg->to = f;
288 INIT_LIST_HEAD(&nrg->link);
289 goto retry;
290 }
96822904 291
7b24d861
DB
292 list_add(&nrg->link, rg->link.prev);
293 chg = t - f;
294 goto out_nrg;
96822904
AW
295 }
296
297 /* Round our left edge to the current segment if it encloses us. */
298 if (f > rg->from)
299 f = rg->from;
300 chg = t - f;
301
302 /* Check for and consume any regions we now overlap with. */
303 list_for_each_entry(rg, rg->link.prev, link) {
304 if (&rg->link == head)
305 break;
306 if (rg->from > t)
7b24d861 307 goto out;
96822904 308
25985edc 309 /* We overlap with this area, if it extends further than
96822904
AW
310 * us then we must extend ourselves. Account for its
311 * existing reservation. */
312 if (rg->to > t) {
313 chg += rg->to - t;
314 t = rg->to;
315 }
316 chg -= rg->to - rg->from;
317 }
7b24d861
DB
318
319out:
320 spin_unlock(&resv->lock);
321 /* We already know we raced and no longer need the new region */
322 kfree(nrg);
323 return chg;
324out_nrg:
325 spin_unlock(&resv->lock);
96822904
AW
326 return chg;
327}
328
1406ec9b 329static long region_truncate(struct resv_map *resv, long end)
96822904 330{
1406ec9b 331 struct list_head *head = &resv->regions;
96822904
AW
332 struct file_region *rg, *trg;
333 long chg = 0;
334
7b24d861 335 spin_lock(&resv->lock);
96822904
AW
336 /* Locate the region we are either in or before. */
337 list_for_each_entry(rg, head, link)
338 if (end <= rg->to)
339 break;
340 if (&rg->link == head)
7b24d861 341 goto out;
96822904
AW
342
343 /* If we are in the middle of a region then adjust it. */
344 if (end > rg->from) {
345 chg = rg->to - end;
346 rg->to = end;
347 rg = list_entry(rg->link.next, typeof(*rg), link);
348 }
349
350 /* Drop any remaining regions. */
351 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
352 if (&rg->link == head)
353 break;
354 chg += rg->to - rg->from;
355 list_del(&rg->link);
356 kfree(rg);
357 }
7b24d861
DB
358
359out:
360 spin_unlock(&resv->lock);
96822904
AW
361 return chg;
362}
363
1406ec9b 364static long region_count(struct resv_map *resv, long f, long t)
84afd99b 365{
1406ec9b 366 struct list_head *head = &resv->regions;
84afd99b
AW
367 struct file_region *rg;
368 long chg = 0;
369
7b24d861 370 spin_lock(&resv->lock);
84afd99b
AW
371 /* Locate each segment we overlap with, and count that overlap. */
372 list_for_each_entry(rg, head, link) {
f2135a4a
WSH
373 long seg_from;
374 long seg_to;
84afd99b
AW
375
376 if (rg->to <= f)
377 continue;
378 if (rg->from >= t)
379 break;
380
381 seg_from = max(rg->from, f);
382 seg_to = min(rg->to, t);
383
384 chg += seg_to - seg_from;
385 }
7b24d861 386 spin_unlock(&resv->lock);
84afd99b
AW
387
388 return chg;
389}
390
e7c4b0bf
AW
391/*
392 * Convert the address within this vma to the page offset within
393 * the mapping, in pagecache page units; huge pages here.
394 */
a5516438
AK
395static pgoff_t vma_hugecache_offset(struct hstate *h,
396 struct vm_area_struct *vma, unsigned long address)
e7c4b0bf 397{
a5516438
AK
398 return ((address - vma->vm_start) >> huge_page_shift(h)) +
399 (vma->vm_pgoff >> huge_page_order(h));
e7c4b0bf
AW
400}
401
0fe6e20b
NH
402pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
403 unsigned long address)
404{
405 return vma_hugecache_offset(hstate_vma(vma), vma, address);
406}
407
08fba699
MG
408/*
409 * Return the size of the pages allocated when backing a VMA. In the majority
410 * cases this will be same size as used by the page table entries.
411 */
412unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
413{
414 struct hstate *hstate;
415
416 if (!is_vm_hugetlb_page(vma))
417 return PAGE_SIZE;
418
419 hstate = hstate_vma(vma);
420
2415cf12 421 return 1UL << huge_page_shift(hstate);
08fba699 422}
f340ca0f 423EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
08fba699 424
3340289d
MG
425/*
426 * Return the page size being used by the MMU to back a VMA. In the majority
427 * of cases, the page size used by the kernel matches the MMU size. On
428 * architectures where it differs, an architecture-specific version of this
429 * function is required.
430 */
431#ifndef vma_mmu_pagesize
432unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
433{
434 return vma_kernel_pagesize(vma);
435}
436#endif
437
84afd99b
AW
438/*
439 * Flags for MAP_PRIVATE reservations. These are stored in the bottom
440 * bits of the reservation map pointer, which are always clear due to
441 * alignment.
442 */
443#define HPAGE_RESV_OWNER (1UL << 0)
444#define HPAGE_RESV_UNMAPPED (1UL << 1)
04f2cbe3 445#define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
84afd99b 446
a1e78772
MG
447/*
448 * These helpers are used to track how many pages are reserved for
449 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
450 * is guaranteed to have their future faults succeed.
451 *
452 * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
453 * the reserve counters are updated with the hugetlb_lock held. It is safe
454 * to reset the VMA at fork() time as it is not in use yet and there is no
455 * chance of the global counters getting corrupted as a result of the values.
84afd99b
AW
456 *
457 * The private mapping reservation is represented in a subtly different
458 * manner to a shared mapping. A shared mapping has a region map associated
459 * with the underlying file, this region map represents the backing file
460 * pages which have ever had a reservation assigned which this persists even
461 * after the page is instantiated. A private mapping has a region map
462 * associated with the original mmap which is attached to all VMAs which
463 * reference it, this region map represents those offsets which have consumed
464 * reservation ie. where pages have been instantiated.
a1e78772 465 */
e7c4b0bf
AW
466static unsigned long get_vma_private_data(struct vm_area_struct *vma)
467{
468 return (unsigned long)vma->vm_private_data;
469}
470
471static void set_vma_private_data(struct vm_area_struct *vma,
472 unsigned long value)
473{
474 vma->vm_private_data = (void *)value;
475}
476
9119a41e 477struct resv_map *resv_map_alloc(void)
84afd99b
AW
478{
479 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
480 if (!resv_map)
481 return NULL;
482
483 kref_init(&resv_map->refs);
7b24d861 484 spin_lock_init(&resv_map->lock);
84afd99b
AW
485 INIT_LIST_HEAD(&resv_map->regions);
486
487 return resv_map;
488}
489
9119a41e 490void resv_map_release(struct kref *ref)
84afd99b
AW
491{
492 struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
493
494 /* Clear out any active regions before we release the map. */
1406ec9b 495 region_truncate(resv_map, 0);
84afd99b
AW
496 kfree(resv_map);
497}
498
4e35f483
JK
499static inline struct resv_map *inode_resv_map(struct inode *inode)
500{
501 return inode->i_mapping->private_data;
502}
503
84afd99b 504static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
a1e78772 505{
81d1b09c 506 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
4e35f483
JK
507 if (vma->vm_flags & VM_MAYSHARE) {
508 struct address_space *mapping = vma->vm_file->f_mapping;
509 struct inode *inode = mapping->host;
510
511 return inode_resv_map(inode);
512
513 } else {
84afd99b
AW
514 return (struct resv_map *)(get_vma_private_data(vma) &
515 ~HPAGE_RESV_MASK);
4e35f483 516 }
a1e78772
MG
517}
518
84afd99b 519static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
a1e78772 520{
81d1b09c
SL
521 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
522 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
a1e78772 523
84afd99b
AW
524 set_vma_private_data(vma, (get_vma_private_data(vma) &
525 HPAGE_RESV_MASK) | (unsigned long)map);
04f2cbe3
MG
526}
527
528static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
529{
81d1b09c
SL
530 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
531 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
e7c4b0bf
AW
532
533 set_vma_private_data(vma, get_vma_private_data(vma) | flags);
04f2cbe3
MG
534}
535
536static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
537{
81d1b09c 538 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
e7c4b0bf
AW
539
540 return (get_vma_private_data(vma) & flag) != 0;
a1e78772
MG
541}
542
04f2cbe3 543/* Reset counters to 0 and clear all HPAGE_RESV_* flags */
a1e78772
MG
544void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
545{
81d1b09c 546 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
f83a275d 547 if (!(vma->vm_flags & VM_MAYSHARE))
a1e78772
MG
548 vma->vm_private_data = (void *)0;
549}
550
551/* Returns true if the VMA has associated reserve pages */
af0ed73e 552static int vma_has_reserves(struct vm_area_struct *vma, long chg)
a1e78772 553{
af0ed73e
JK
554 if (vma->vm_flags & VM_NORESERVE) {
555 /*
556 * This address is already reserved by other process(chg == 0),
557 * so, we should decrement reserved count. Without decrementing,
558 * reserve count remains after releasing inode, because this
559 * allocated page will go into page cache and is regarded as
560 * coming from reserved pool in releasing step. Currently, we
561 * don't have any other solution to deal with this situation
562 * properly, so add work-around here.
563 */
564 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
565 return 1;
566 else
567 return 0;
568 }
a63884e9
JK
569
570 /* Shared mappings always use reserves */
f83a275d 571 if (vma->vm_flags & VM_MAYSHARE)
7f09ca51 572 return 1;
a63884e9
JK
573
574 /*
575 * Only the process that called mmap() has reserves for
576 * private mappings.
577 */
7f09ca51
MG
578 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
579 return 1;
a63884e9 580
7f09ca51 581 return 0;
a1e78772
MG
582}
583
a5516438 584static void enqueue_huge_page(struct hstate *h, struct page *page)
1da177e4
LT
585{
586 int nid = page_to_nid(page);
0edaecfa 587 list_move(&page->lru, &h->hugepage_freelists[nid]);
a5516438
AK
588 h->free_huge_pages++;
589 h->free_huge_pages_node[nid]++;
1da177e4
LT
590}
591
bf50bab2
NH
592static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
593{
594 struct page *page;
595
c8721bbb
NH
596 list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
597 if (!is_migrate_isolate_page(page))
598 break;
599 /*
600 * if 'non-isolated free hugepage' not found on the list,
601 * the allocation fails.
602 */
603 if (&h->hugepage_freelists[nid] == &page->lru)
bf50bab2 604 return NULL;
0edaecfa 605 list_move(&page->lru, &h->hugepage_activelist);
a9869b83 606 set_page_refcounted(page);
bf50bab2
NH
607 h->free_huge_pages--;
608 h->free_huge_pages_node[nid]--;
609 return page;
610}
611
86cdb465
NH
612/* Movability of hugepages depends on migration support. */
613static inline gfp_t htlb_alloc_mask(struct hstate *h)
614{
100873d7 615 if (hugepages_treat_as_movable || hugepage_migration_supported(h))
86cdb465
NH
616 return GFP_HIGHUSER_MOVABLE;
617 else
618 return GFP_HIGHUSER;
619}
620
a5516438
AK
621static struct page *dequeue_huge_page_vma(struct hstate *h,
622 struct vm_area_struct *vma,
af0ed73e
JK
623 unsigned long address, int avoid_reserve,
624 long chg)
1da177e4 625{
b1c12cbc 626 struct page *page = NULL;
480eccf9 627 struct mempolicy *mpol;
19770b32 628 nodemask_t *nodemask;
c0ff7453 629 struct zonelist *zonelist;
dd1a239f
MG
630 struct zone *zone;
631 struct zoneref *z;
cc9a6c87 632 unsigned int cpuset_mems_cookie;
1da177e4 633
a1e78772
MG
634 /*
635 * A child process with MAP_PRIVATE mappings created by their parent
636 * have no page reserves. This check ensures that reservations are
637 * not "stolen". The child may still get SIGKILLed
638 */
af0ed73e 639 if (!vma_has_reserves(vma, chg) &&
a5516438 640 h->free_huge_pages - h->resv_huge_pages == 0)
c0ff7453 641 goto err;
a1e78772 642
04f2cbe3 643 /* If reserves cannot be used, ensure enough pages are in the pool */
a5516438 644 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
6eab04a8 645 goto err;
04f2cbe3 646
9966c4bb 647retry_cpuset:
d26914d1 648 cpuset_mems_cookie = read_mems_allowed_begin();
9966c4bb 649 zonelist = huge_zonelist(vma, address,
86cdb465 650 htlb_alloc_mask(h), &mpol, &nodemask);
9966c4bb 651
19770b32
MG
652 for_each_zone_zonelist_nodemask(zone, z, zonelist,
653 MAX_NR_ZONES - 1, nodemask) {
344736f2 654 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
bf50bab2
NH
655 page = dequeue_huge_page_node(h, zone_to_nid(zone));
656 if (page) {
af0ed73e
JK
657 if (avoid_reserve)
658 break;
659 if (!vma_has_reserves(vma, chg))
660 break;
661
07443a85 662 SetPagePrivate(page);
af0ed73e 663 h->resv_huge_pages--;
bf50bab2
NH
664 break;
665 }
3abf7afd 666 }
1da177e4 667 }
cc9a6c87 668
52cd3b07 669 mpol_cond_put(mpol);
d26914d1 670 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
cc9a6c87 671 goto retry_cpuset;
1da177e4 672 return page;
cc9a6c87
MG
673
674err:
cc9a6c87 675 return NULL;
1da177e4
LT
676}
677
1cac6f2c
LC
678/*
679 * common helper functions for hstate_next_node_to_{alloc|free}.
680 * We may have allocated or freed a huge page based on a different
681 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
682 * be outside of *nodes_allowed. Ensure that we use an allowed
683 * node for alloc or free.
684 */
685static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
686{
687 nid = next_node(nid, *nodes_allowed);
688 if (nid == MAX_NUMNODES)
689 nid = first_node(*nodes_allowed);
690 VM_BUG_ON(nid >= MAX_NUMNODES);
691
692 return nid;
693}
694
695static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
696{
697 if (!node_isset(nid, *nodes_allowed))
698 nid = next_node_allowed(nid, nodes_allowed);
699 return nid;
700}
701
702/*
703 * returns the previously saved node ["this node"] from which to
704 * allocate a persistent huge page for the pool and advance the
705 * next node from which to allocate, handling wrap at end of node
706 * mask.
707 */
708static int hstate_next_node_to_alloc(struct hstate *h,
709 nodemask_t *nodes_allowed)
710{
711 int nid;
712
713 VM_BUG_ON(!nodes_allowed);
714
715 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
716 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
717
718 return nid;
719}
720
721/*
722 * helper for free_pool_huge_page() - return the previously saved
723 * node ["this node"] from which to free a huge page. Advance the
724 * next node id whether or not we find a free huge page to free so
725 * that the next attempt to free addresses the next node.
726 */
727static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
728{
729 int nid;
730
731 VM_BUG_ON(!nodes_allowed);
732
733 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
734 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
735
736 return nid;
737}
738
739#define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \
740 for (nr_nodes = nodes_weight(*mask); \
741 nr_nodes > 0 && \
742 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \
743 nr_nodes--)
744
745#define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \
746 for (nr_nodes = nodes_weight(*mask); \
747 nr_nodes > 0 && \
748 ((node = hstate_next_node_to_free(hs, mask)) || 1); \
749 nr_nodes--)
750
944d9fec
LC
751#if defined(CONFIG_CMA) && defined(CONFIG_X86_64)
752static void destroy_compound_gigantic_page(struct page *page,
753 unsigned long order)
754{
755 int i;
756 int nr_pages = 1 << order;
757 struct page *p = page + 1;
758
759 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
760 __ClearPageTail(p);
761 set_page_refcounted(p);
762 p->first_page = NULL;
763 }
764
765 set_compound_order(page, 0);
766 __ClearPageHead(page);
767}
768
769static void free_gigantic_page(struct page *page, unsigned order)
770{
771 free_contig_range(page_to_pfn(page), 1 << order);
772}
773
774static int __alloc_gigantic_page(unsigned long start_pfn,
775 unsigned long nr_pages)
776{
777 unsigned long end_pfn = start_pfn + nr_pages;
778 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
779}
780
781static bool pfn_range_valid_gigantic(unsigned long start_pfn,
782 unsigned long nr_pages)
783{
784 unsigned long i, end_pfn = start_pfn + nr_pages;
785 struct page *page;
786
787 for (i = start_pfn; i < end_pfn; i++) {
788 if (!pfn_valid(i))
789 return false;
790
791 page = pfn_to_page(i);
792
793 if (PageReserved(page))
794 return false;
795
796 if (page_count(page) > 0)
797 return false;
798
799 if (PageHuge(page))
800 return false;
801 }
802
803 return true;
804}
805
806static bool zone_spans_last_pfn(const struct zone *zone,
807 unsigned long start_pfn, unsigned long nr_pages)
808{
809 unsigned long last_pfn = start_pfn + nr_pages - 1;
810 return zone_spans_pfn(zone, last_pfn);
811}
812
813static struct page *alloc_gigantic_page(int nid, unsigned order)
814{
815 unsigned long nr_pages = 1 << order;
816 unsigned long ret, pfn, flags;
817 struct zone *z;
818
819 z = NODE_DATA(nid)->node_zones;
820 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
821 spin_lock_irqsave(&z->lock, flags);
822
823 pfn = ALIGN(z->zone_start_pfn, nr_pages);
824 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
825 if (pfn_range_valid_gigantic(pfn, nr_pages)) {
826 /*
827 * We release the zone lock here because
828 * alloc_contig_range() will also lock the zone
829 * at some point. If there's an allocation
830 * spinning on this lock, it may win the race
831 * and cause alloc_contig_range() to fail...
832 */
833 spin_unlock_irqrestore(&z->lock, flags);
834 ret = __alloc_gigantic_page(pfn, nr_pages);
835 if (!ret)
836 return pfn_to_page(pfn);
837 spin_lock_irqsave(&z->lock, flags);
838 }
839 pfn += nr_pages;
840 }
841
842 spin_unlock_irqrestore(&z->lock, flags);
843 }
844
845 return NULL;
846}
847
848static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
849static void prep_compound_gigantic_page(struct page *page, unsigned long order);
850
851static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
852{
853 struct page *page;
854
855 page = alloc_gigantic_page(nid, huge_page_order(h));
856 if (page) {
857 prep_compound_gigantic_page(page, huge_page_order(h));
858 prep_new_huge_page(h, page, nid);
859 }
860
861 return page;
862}
863
864static int alloc_fresh_gigantic_page(struct hstate *h,
865 nodemask_t *nodes_allowed)
866{
867 struct page *page = NULL;
868 int nr_nodes, node;
869
870 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
871 page = alloc_fresh_gigantic_page_node(h, node);
872 if (page)
873 return 1;
874 }
875
876 return 0;
877}
878
879static inline bool gigantic_page_supported(void) { return true; }
880#else
881static inline bool gigantic_page_supported(void) { return false; }
882static inline void free_gigantic_page(struct page *page, unsigned order) { }
883static inline void destroy_compound_gigantic_page(struct page *page,
884 unsigned long order) { }
885static inline int alloc_fresh_gigantic_page(struct hstate *h,
886 nodemask_t *nodes_allowed) { return 0; }
887#endif
888
a5516438 889static void update_and_free_page(struct hstate *h, struct page *page)
6af2acb6
AL
890{
891 int i;
a5516438 892
944d9fec
LC
893 if (hstate_is_gigantic(h) && !gigantic_page_supported())
894 return;
18229df5 895
a5516438
AK
896 h->nr_huge_pages--;
897 h->nr_huge_pages_node[page_to_nid(page)]--;
898 for (i = 0; i < pages_per_huge_page(h); i++) {
32f84528
CF
899 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
900 1 << PG_referenced | 1 << PG_dirty |
a7407a27
LC
901 1 << PG_active | 1 << PG_private |
902 1 << PG_writeback);
6af2acb6 903 }
309381fe 904 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
6af2acb6
AL
905 set_compound_page_dtor(page, NULL);
906 set_page_refcounted(page);
944d9fec
LC
907 if (hstate_is_gigantic(h)) {
908 destroy_compound_gigantic_page(page, huge_page_order(h));
909 free_gigantic_page(page, huge_page_order(h));
910 } else {
911 arch_release_hugepage(page);
912 __free_pages(page, huge_page_order(h));
913 }
6af2acb6
AL
914}
915
e5ff2159
AK
916struct hstate *size_to_hstate(unsigned long size)
917{
918 struct hstate *h;
919
920 for_each_hstate(h) {
921 if (huge_page_size(h) == size)
922 return h;
923 }
924 return NULL;
925}
926
bcc54222
NH
927/*
928 * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
929 * to hstate->hugepage_activelist.)
930 *
931 * This function can be called for tail pages, but never returns true for them.
932 */
933bool page_huge_active(struct page *page)
934{
935 VM_BUG_ON_PAGE(!PageHuge(page), page);
936 return PageHead(page) && PagePrivate(&page[1]);
937}
938
939/* never called for tail page */
940static void set_page_huge_active(struct page *page)
941{
942 VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
943 SetPagePrivate(&page[1]);
944}
945
946static void clear_page_huge_active(struct page *page)
947{
948 VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
949 ClearPagePrivate(&page[1]);
950}
951
8f1d26d0 952void free_huge_page(struct page *page)
27a85ef1 953{
a5516438
AK
954 /*
955 * Can't pass hstate in here because it is called from the
956 * compound page destructor.
957 */
e5ff2159 958 struct hstate *h = page_hstate(page);
7893d1d5 959 int nid = page_to_nid(page);
90481622
DG
960 struct hugepage_subpool *spool =
961 (struct hugepage_subpool *)page_private(page);
07443a85 962 bool restore_reserve;
27a85ef1 963
e5df70ab 964 set_page_private(page, 0);
23be7468 965 page->mapping = NULL;
7893d1d5 966 BUG_ON(page_count(page));
0fe6e20b 967 BUG_ON(page_mapcount(page));
07443a85 968 restore_reserve = PagePrivate(page);
16c794b4 969 ClearPagePrivate(page);
27a85ef1 970
1c5ecae3
MK
971 /*
972 * A return code of zero implies that the subpool will be under its
973 * minimum size if the reservation is not restored after page is free.
974 * Therefore, force restore_reserve operation.
975 */
976 if (hugepage_subpool_put_pages(spool, 1) == 0)
977 restore_reserve = true;
978
27a85ef1 979 spin_lock(&hugetlb_lock);
bcc54222 980 clear_page_huge_active(page);
6d76dcf4
AK
981 hugetlb_cgroup_uncharge_page(hstate_index(h),
982 pages_per_huge_page(h), page);
07443a85
JK
983 if (restore_reserve)
984 h->resv_huge_pages++;
985
944d9fec 986 if (h->surplus_huge_pages_node[nid]) {
0edaecfa
AK
987 /* remove the page from active list */
988 list_del(&page->lru);
a5516438
AK
989 update_and_free_page(h, page);
990 h->surplus_huge_pages--;
991 h->surplus_huge_pages_node[nid]--;
7893d1d5 992 } else {
5d3a551c 993 arch_clear_hugepage_flags(page);
a5516438 994 enqueue_huge_page(h, page);
7893d1d5 995 }
27a85ef1
DG
996 spin_unlock(&hugetlb_lock);
997}
998
a5516438 999static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
b7ba30c6 1000{
0edaecfa 1001 INIT_LIST_HEAD(&page->lru);
b7ba30c6
AK
1002 set_compound_page_dtor(page, free_huge_page);
1003 spin_lock(&hugetlb_lock);
9dd540e2 1004 set_hugetlb_cgroup(page, NULL);
a5516438
AK
1005 h->nr_huge_pages++;
1006 h->nr_huge_pages_node[nid]++;
b7ba30c6
AK
1007 spin_unlock(&hugetlb_lock);
1008 put_page(page); /* free it into the hugepage allocator */
1009}
1010
2906dd52 1011static void prep_compound_gigantic_page(struct page *page, unsigned long order)
20a0307c
WF
1012{
1013 int i;
1014 int nr_pages = 1 << order;
1015 struct page *p = page + 1;
1016
1017 /* we rely on prep_new_huge_page to set the destructor */
1018 set_compound_order(page, order);
1019 __SetPageHead(page);
ef5a22be 1020 __ClearPageReserved(page);
20a0307c 1021 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
ef5a22be
AA
1022 /*
1023 * For gigantic hugepages allocated through bootmem at
1024 * boot, it's safer to be consistent with the not-gigantic
1025 * hugepages and clear the PG_reserved bit from all tail pages
1026 * too. Otherwse drivers using get_user_pages() to access tail
1027 * pages may get the reference counting wrong if they see
1028 * PG_reserved set on a tail page (despite the head page not
1029 * having PG_reserved set). Enforcing this consistency between
1030 * head and tail pages allows drivers to optimize away a check
1031 * on the head page when they need know if put_page() is needed
1032 * after get_user_pages().
1033 */
1034 __ClearPageReserved(p);
58a84aa9 1035 set_page_count(p, 0);
20a0307c 1036 p->first_page = page;
44fc8057
DR
1037 /* Make sure p->first_page is always valid for PageTail() */
1038 smp_wmb();
1039 __SetPageTail(p);
20a0307c
WF
1040 }
1041}
1042
7795912c
AM
1043/*
1044 * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1045 * transparent huge pages. See the PageTransHuge() documentation for more
1046 * details.
1047 */
20a0307c
WF
1048int PageHuge(struct page *page)
1049{
20a0307c
WF
1050 if (!PageCompound(page))
1051 return 0;
1052
1053 page = compound_head(page);
758f66a2 1054 return get_compound_page_dtor(page) == free_huge_page;
20a0307c 1055}
43131e14
NH
1056EXPORT_SYMBOL_GPL(PageHuge);
1057
27c73ae7
AA
1058/*
1059 * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1060 * normal or transparent huge pages.
1061 */
1062int PageHeadHuge(struct page *page_head)
1063{
27c73ae7
AA
1064 if (!PageHead(page_head))
1065 return 0;
1066
758f66a2 1067 return get_compound_page_dtor(page_head) == free_huge_page;
27c73ae7 1068}
27c73ae7 1069
13d60f4b
ZY
1070pgoff_t __basepage_index(struct page *page)
1071{
1072 struct page *page_head = compound_head(page);
1073 pgoff_t index = page_index(page_head);
1074 unsigned long compound_idx;
1075
1076 if (!PageHuge(page_head))
1077 return page_index(page);
1078
1079 if (compound_order(page_head) >= MAX_ORDER)
1080 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1081 else
1082 compound_idx = page - page_head;
1083
1084 return (index << compound_order(page_head)) + compound_idx;
1085}
1086
a5516438 1087static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1da177e4 1088{
1da177e4 1089 struct page *page;
f96efd58 1090
6484eb3e 1091 page = alloc_pages_exact_node(nid,
86cdb465 1092 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
551883ae 1093 __GFP_REPEAT|__GFP_NOWARN,
a5516438 1094 huge_page_order(h));
1da177e4 1095 if (page) {
7f2e9525 1096 if (arch_prepare_hugepage(page)) {
caff3a2c 1097 __free_pages(page, huge_page_order(h));
7b8ee84d 1098 return NULL;
7f2e9525 1099 }
a5516438 1100 prep_new_huge_page(h, page, nid);
1da177e4 1101 }
63b4613c
NA
1102
1103 return page;
1104}
1105
b2261026
JK
1106static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1107{
1108 struct page *page;
1109 int nr_nodes, node;
1110 int ret = 0;
1111
1112 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1113 page = alloc_fresh_huge_page_node(h, node);
1114 if (page) {
1115 ret = 1;
1116 break;
1117 }
1118 }
1119
1120 if (ret)
1121 count_vm_event(HTLB_BUDDY_PGALLOC);
1122 else
1123 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1124
1125 return ret;
1126}
1127
e8c5c824
LS
1128/*
1129 * Free huge page from pool from next node to free.
1130 * Attempt to keep persistent huge pages more or less
1131 * balanced over allowed nodes.
1132 * Called with hugetlb_lock locked.
1133 */
6ae11b27
LS
1134static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1135 bool acct_surplus)
e8c5c824 1136{
b2261026 1137 int nr_nodes, node;
e8c5c824
LS
1138 int ret = 0;
1139
b2261026 1140 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
685f3457
LS
1141 /*
1142 * If we're returning unused surplus pages, only examine
1143 * nodes with surplus pages.
1144 */
b2261026
JK
1145 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1146 !list_empty(&h->hugepage_freelists[node])) {
e8c5c824 1147 struct page *page =
b2261026 1148 list_entry(h->hugepage_freelists[node].next,
e8c5c824
LS
1149 struct page, lru);
1150 list_del(&page->lru);
1151 h->free_huge_pages--;
b2261026 1152 h->free_huge_pages_node[node]--;
685f3457
LS
1153 if (acct_surplus) {
1154 h->surplus_huge_pages--;
b2261026 1155 h->surplus_huge_pages_node[node]--;
685f3457 1156 }
e8c5c824
LS
1157 update_and_free_page(h, page);
1158 ret = 1;
9a76db09 1159 break;
e8c5c824 1160 }
b2261026 1161 }
e8c5c824
LS
1162
1163 return ret;
1164}
1165
c8721bbb
NH
1166/*
1167 * Dissolve a given free hugepage into free buddy pages. This function does
1168 * nothing for in-use (including surplus) hugepages.
1169 */
1170static void dissolve_free_huge_page(struct page *page)
1171{
1172 spin_lock(&hugetlb_lock);
1173 if (PageHuge(page) && !page_count(page)) {
1174 struct hstate *h = page_hstate(page);
1175 int nid = page_to_nid(page);
1176 list_del(&page->lru);
1177 h->free_huge_pages--;
1178 h->free_huge_pages_node[nid]--;
1179 update_and_free_page(h, page);
1180 }
1181 spin_unlock(&hugetlb_lock);
1182}
1183
1184/*
1185 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1186 * make specified memory blocks removable from the system.
1187 * Note that start_pfn should aligned with (minimum) hugepage size.
1188 */
1189void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1190{
1191 unsigned int order = 8 * sizeof(void *);
1192 unsigned long pfn;
1193 struct hstate *h;
1194
d0177639
LZ
1195 if (!hugepages_supported())
1196 return;
1197
c8721bbb
NH
1198 /* Set scan step to minimum hugepage size */
1199 for_each_hstate(h)
1200 if (order > huge_page_order(h))
1201 order = huge_page_order(h);
1202 VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
1203 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
1204 dissolve_free_huge_page(pfn_to_page(pfn));
1205}
1206
bf50bab2 1207static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
7893d1d5
AL
1208{
1209 struct page *page;
bf50bab2 1210 unsigned int r_nid;
7893d1d5 1211
bae7f4ae 1212 if (hstate_is_gigantic(h))
aa888a74
AK
1213 return NULL;
1214
d1c3fb1f
NA
1215 /*
1216 * Assume we will successfully allocate the surplus page to
1217 * prevent racing processes from causing the surplus to exceed
1218 * overcommit
1219 *
1220 * This however introduces a different race, where a process B
1221 * tries to grow the static hugepage pool while alloc_pages() is
1222 * called by process A. B will only examine the per-node
1223 * counters in determining if surplus huge pages can be
1224 * converted to normal huge pages in adjust_pool_surplus(). A
1225 * won't be able to increment the per-node counter, until the
1226 * lock is dropped by B, but B doesn't drop hugetlb_lock until
1227 * no more huge pages can be converted from surplus to normal
1228 * state (and doesn't try to convert again). Thus, we have a
1229 * case where a surplus huge page exists, the pool is grown, and
1230 * the surplus huge page still exists after, even though it
1231 * should just have been converted to a normal huge page. This
1232 * does not leak memory, though, as the hugepage will be freed
1233 * once it is out of use. It also does not allow the counters to
1234 * go out of whack in adjust_pool_surplus() as we don't modify
1235 * the node values until we've gotten the hugepage and only the
1236 * per-node value is checked there.
1237 */
1238 spin_lock(&hugetlb_lock);
a5516438 1239 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
d1c3fb1f
NA
1240 spin_unlock(&hugetlb_lock);
1241 return NULL;
1242 } else {
a5516438
AK
1243 h->nr_huge_pages++;
1244 h->surplus_huge_pages++;
d1c3fb1f
NA
1245 }
1246 spin_unlock(&hugetlb_lock);
1247
bf50bab2 1248 if (nid == NUMA_NO_NODE)
86cdb465 1249 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
bf50bab2
NH
1250 __GFP_REPEAT|__GFP_NOWARN,
1251 huge_page_order(h));
1252 else
1253 page = alloc_pages_exact_node(nid,
86cdb465 1254 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
bf50bab2 1255 __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
d1c3fb1f 1256
caff3a2c
GS
1257 if (page && arch_prepare_hugepage(page)) {
1258 __free_pages(page, huge_page_order(h));
ea5768c7 1259 page = NULL;
caff3a2c
GS
1260 }
1261
d1c3fb1f 1262 spin_lock(&hugetlb_lock);
7893d1d5 1263 if (page) {
0edaecfa 1264 INIT_LIST_HEAD(&page->lru);
bf50bab2 1265 r_nid = page_to_nid(page);
7893d1d5 1266 set_compound_page_dtor(page, free_huge_page);
9dd540e2 1267 set_hugetlb_cgroup(page, NULL);
d1c3fb1f
NA
1268 /*
1269 * We incremented the global counters already
1270 */
bf50bab2
NH
1271 h->nr_huge_pages_node[r_nid]++;
1272 h->surplus_huge_pages_node[r_nid]++;
3b116300 1273 __count_vm_event(HTLB_BUDDY_PGALLOC);
d1c3fb1f 1274 } else {
a5516438
AK
1275 h->nr_huge_pages--;
1276 h->surplus_huge_pages--;
3b116300 1277 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
7893d1d5 1278 }
d1c3fb1f 1279 spin_unlock(&hugetlb_lock);
7893d1d5
AL
1280
1281 return page;
1282}
1283
bf50bab2
NH
1284/*
1285 * This allocation function is useful in the context where vma is irrelevant.
1286 * E.g. soft-offlining uses this function because it only cares physical
1287 * address of error page.
1288 */
1289struct page *alloc_huge_page_node(struct hstate *h, int nid)
1290{
4ef91848 1291 struct page *page = NULL;
bf50bab2
NH
1292
1293 spin_lock(&hugetlb_lock);
4ef91848
JK
1294 if (h->free_huge_pages - h->resv_huge_pages > 0)
1295 page = dequeue_huge_page_node(h, nid);
bf50bab2
NH
1296 spin_unlock(&hugetlb_lock);
1297
94ae8ba7 1298 if (!page)
bf50bab2
NH
1299 page = alloc_buddy_huge_page(h, nid);
1300
1301 return page;
1302}
1303
e4e574b7 1304/*
25985edc 1305 * Increase the hugetlb pool such that it can accommodate a reservation
e4e574b7
AL
1306 * of size 'delta'.
1307 */
a5516438 1308static int gather_surplus_pages(struct hstate *h, int delta)
e4e574b7
AL
1309{
1310 struct list_head surplus_list;
1311 struct page *page, *tmp;
1312 int ret, i;
1313 int needed, allocated;
28073b02 1314 bool alloc_ok = true;
e4e574b7 1315
a5516438 1316 needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
ac09b3a1 1317 if (needed <= 0) {
a5516438 1318 h->resv_huge_pages += delta;
e4e574b7 1319 return 0;
ac09b3a1 1320 }
e4e574b7
AL
1321
1322 allocated = 0;
1323 INIT_LIST_HEAD(&surplus_list);
1324
1325 ret = -ENOMEM;
1326retry:
1327 spin_unlock(&hugetlb_lock);
1328 for (i = 0; i < needed; i++) {
bf50bab2 1329 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
28073b02
HD
1330 if (!page) {
1331 alloc_ok = false;
1332 break;
1333 }
e4e574b7
AL
1334 list_add(&page->lru, &surplus_list);
1335 }
28073b02 1336 allocated += i;
e4e574b7
AL
1337
1338 /*
1339 * After retaking hugetlb_lock, we need to recalculate 'needed'
1340 * because either resv_huge_pages or free_huge_pages may have changed.
1341 */
1342 spin_lock(&hugetlb_lock);
a5516438
AK
1343 needed = (h->resv_huge_pages + delta) -
1344 (h->free_huge_pages + allocated);
28073b02
HD
1345 if (needed > 0) {
1346 if (alloc_ok)
1347 goto retry;
1348 /*
1349 * We were not able to allocate enough pages to
1350 * satisfy the entire reservation so we free what
1351 * we've allocated so far.
1352 */
1353 goto free;
1354 }
e4e574b7
AL
1355 /*
1356 * The surplus_list now contains _at_least_ the number of extra pages
25985edc 1357 * needed to accommodate the reservation. Add the appropriate number
e4e574b7 1358 * of pages to the hugetlb pool and free the extras back to the buddy
ac09b3a1
AL
1359 * allocator. Commit the entire reservation here to prevent another
1360 * process from stealing the pages as they are added to the pool but
1361 * before they are reserved.
e4e574b7
AL
1362 */
1363 needed += allocated;
a5516438 1364 h->resv_huge_pages += delta;
e4e574b7 1365 ret = 0;
a9869b83 1366
19fc3f0a 1367 /* Free the needed pages to the hugetlb pool */
e4e574b7 1368 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
19fc3f0a
AL
1369 if ((--needed) < 0)
1370 break;
a9869b83
NH
1371 /*
1372 * This page is now managed by the hugetlb allocator and has
1373 * no users -- drop the buddy allocator's reference.
1374 */
1375 put_page_testzero(page);
309381fe 1376 VM_BUG_ON_PAGE(page_count(page), page);
a5516438 1377 enqueue_huge_page(h, page);
19fc3f0a 1378 }
28073b02 1379free:
b0365c8d 1380 spin_unlock(&hugetlb_lock);
19fc3f0a
AL
1381
1382 /* Free unnecessary surplus pages to the buddy allocator */
c0d934ba
JK
1383 list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1384 put_page(page);
a9869b83 1385 spin_lock(&hugetlb_lock);
e4e574b7
AL
1386
1387 return ret;
1388}
1389
1390/*
1391 * When releasing a hugetlb pool reservation, any surplus pages that were
1392 * allocated to satisfy the reservation must be explicitly freed if they were
1393 * never used.
685f3457 1394 * Called with hugetlb_lock held.
e4e574b7 1395 */
a5516438
AK
1396static void return_unused_surplus_pages(struct hstate *h,
1397 unsigned long unused_resv_pages)
e4e574b7 1398{
e4e574b7
AL
1399 unsigned long nr_pages;
1400
ac09b3a1 1401 /* Uncommit the reservation */
a5516438 1402 h->resv_huge_pages -= unused_resv_pages;
ac09b3a1 1403
aa888a74 1404 /* Cannot return gigantic pages currently */
bae7f4ae 1405 if (hstate_is_gigantic(h))
aa888a74
AK
1406 return;
1407
a5516438 1408 nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
e4e574b7 1409
685f3457
LS
1410 /*
1411 * We want to release as many surplus pages as possible, spread
9b5e5d0f
LS
1412 * evenly across all nodes with memory. Iterate across these nodes
1413 * until we can no longer free unreserved surplus pages. This occurs
1414 * when the nodes with surplus pages have no free pages.
1415 * free_pool_huge_page() will balance the the freed pages across the
1416 * on-line nodes with memory and will handle the hstate accounting.
685f3457
LS
1417 */
1418 while (nr_pages--) {
8cebfcd0 1419 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
685f3457 1420 break;
7848a4bf 1421 cond_resched_lock(&hugetlb_lock);
e4e574b7
AL
1422 }
1423}
1424
c37f9fb1
AW
1425/*
1426 * Determine if the huge page at addr within the vma has an associated
1427 * reservation. Where it does not we will need to logically increase
90481622
DG
1428 * reservation and actually increase subpool usage before an allocation
1429 * can occur. Where any new reservation would be required the
1430 * reservation change is prepared, but not committed. Once the page
1431 * has been allocated from the subpool and instantiated the change should
1432 * be committed via vma_commit_reservation. No action is required on
1433 * failure.
c37f9fb1 1434 */
e2f17d94 1435static long vma_needs_reservation(struct hstate *h,
a5516438 1436 struct vm_area_struct *vma, unsigned long addr)
c37f9fb1 1437{
4e35f483
JK
1438 struct resv_map *resv;
1439 pgoff_t idx;
1440 long chg;
c37f9fb1 1441
4e35f483
JK
1442 resv = vma_resv_map(vma);
1443 if (!resv)
84afd99b 1444 return 1;
c37f9fb1 1445
4e35f483
JK
1446 idx = vma_hugecache_offset(h, vma, addr);
1447 chg = region_chg(resv, idx, idx + 1);
84afd99b 1448
4e35f483
JK
1449 if (vma->vm_flags & VM_MAYSHARE)
1450 return chg;
1451 else
1452 return chg < 0 ? chg : 0;
c37f9fb1 1453}
a5516438
AK
1454static void vma_commit_reservation(struct hstate *h,
1455 struct vm_area_struct *vma, unsigned long addr)
c37f9fb1 1456{
4e35f483
JK
1457 struct resv_map *resv;
1458 pgoff_t idx;
84afd99b 1459
4e35f483
JK
1460 resv = vma_resv_map(vma);
1461 if (!resv)
1462 return;
84afd99b 1463
4e35f483
JK
1464 idx = vma_hugecache_offset(h, vma, addr);
1465 region_add(resv, idx, idx + 1);
c37f9fb1
AW
1466}
1467
a1e78772 1468static struct page *alloc_huge_page(struct vm_area_struct *vma,
04f2cbe3 1469 unsigned long addr, int avoid_reserve)
1da177e4 1470{
90481622 1471 struct hugepage_subpool *spool = subpool_vma(vma);
a5516438 1472 struct hstate *h = hstate_vma(vma);
348ea204 1473 struct page *page;
e2f17d94 1474 long chg;
6d76dcf4
AK
1475 int ret, idx;
1476 struct hugetlb_cgroup *h_cg;
a1e78772 1477
6d76dcf4 1478 idx = hstate_index(h);
a1e78772 1479 /*
90481622
DG
1480 * Processes that did not create the mapping will have no
1481 * reserves and will not have accounted against subpool
1482 * limit. Check that the subpool limit can be made before
1483 * satisfying the allocation MAP_NORESERVE mappings may also
1484 * need pages and subpool limit allocated allocated if no reserve
1485 * mapping overlaps.
a1e78772 1486 */
a5516438 1487 chg = vma_needs_reservation(h, vma, addr);
c37f9fb1 1488 if (chg < 0)
76dcee75 1489 return ERR_PTR(-ENOMEM);
8bb3f12e 1490 if (chg || avoid_reserve)
1c5ecae3 1491 if (hugepage_subpool_get_pages(spool, 1) < 0)
76dcee75 1492 return ERR_PTR(-ENOSPC);
1da177e4 1493
6d76dcf4 1494 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
8f34af6f
JZ
1495 if (ret)
1496 goto out_subpool_put;
1497
1da177e4 1498 spin_lock(&hugetlb_lock);
af0ed73e 1499 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
81a6fcae 1500 if (!page) {
94ae8ba7 1501 spin_unlock(&hugetlb_lock);
bf50bab2 1502 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
8f34af6f
JZ
1503 if (!page)
1504 goto out_uncharge_cgroup;
1505
79dbb236
AK
1506 spin_lock(&hugetlb_lock);
1507 list_move(&page->lru, &h->hugepage_activelist);
81a6fcae 1508 /* Fall through */
68842c9b 1509 }
81a6fcae
JK
1510 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1511 spin_unlock(&hugetlb_lock);
348ea204 1512
90481622 1513 set_page_private(page, (unsigned long)spool);
90d8b7e6 1514
a5516438 1515 vma_commit_reservation(h, vma, addr);
90d8b7e6 1516 return page;
8f34af6f
JZ
1517
1518out_uncharge_cgroup:
1519 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1520out_subpool_put:
1521 if (chg || avoid_reserve)
1522 hugepage_subpool_put_pages(spool, 1);
1523 return ERR_PTR(-ENOSPC);
b45b5bd6
DG
1524}
1525
74060e4d
NH
1526/*
1527 * alloc_huge_page()'s wrapper which simply returns the page if allocation
1528 * succeeds, otherwise NULL. This function is called from new_vma_page(),
1529 * where no ERR_VALUE is expected to be returned.
1530 */
1531struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1532 unsigned long addr, int avoid_reserve)
1533{
1534 struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1535 if (IS_ERR(page))
1536 page = NULL;
1537 return page;
1538}
1539
91f47662 1540int __weak alloc_bootmem_huge_page(struct hstate *h)
aa888a74
AK
1541{
1542 struct huge_bootmem_page *m;
b2261026 1543 int nr_nodes, node;
aa888a74 1544
b2261026 1545 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
aa888a74
AK
1546 void *addr;
1547
8b89a116
GS
1548 addr = memblock_virt_alloc_try_nid_nopanic(
1549 huge_page_size(h), huge_page_size(h),
1550 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
aa888a74
AK
1551 if (addr) {
1552 /*
1553 * Use the beginning of the huge page to store the
1554 * huge_bootmem_page struct (until gather_bootmem
1555 * puts them into the mem_map).
1556 */
1557 m = addr;
91f47662 1558 goto found;
aa888a74 1559 }
aa888a74
AK
1560 }
1561 return 0;
1562
1563found:
df994ead 1564 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
aa888a74
AK
1565 /* Put them into a private list first because mem_map is not up yet */
1566 list_add(&m->list, &huge_boot_pages);
1567 m->hstate = h;
1568 return 1;
1569}
1570
f412c97a 1571static void __init prep_compound_huge_page(struct page *page, int order)
18229df5
AW
1572{
1573 if (unlikely(order > (MAX_ORDER - 1)))
1574 prep_compound_gigantic_page(page, order);
1575 else
1576 prep_compound_page(page, order);
1577}
1578
aa888a74
AK
1579/* Put bootmem huge pages into the standard lists after mem_map is up */
1580static void __init gather_bootmem_prealloc(void)
1581{
1582 struct huge_bootmem_page *m;
1583
1584 list_for_each_entry(m, &huge_boot_pages, list) {
aa888a74 1585 struct hstate *h = m->hstate;
ee8f248d
BB
1586 struct page *page;
1587
1588#ifdef CONFIG_HIGHMEM
1589 page = pfn_to_page(m->phys >> PAGE_SHIFT);
8b89a116
GS
1590 memblock_free_late(__pa(m),
1591 sizeof(struct huge_bootmem_page));
ee8f248d
BB
1592#else
1593 page = virt_to_page(m);
1594#endif
aa888a74 1595 WARN_ON(page_count(page) != 1);
18229df5 1596 prep_compound_huge_page(page, h->order);
ef5a22be 1597 WARN_ON(PageReserved(page));
aa888a74 1598 prep_new_huge_page(h, page, page_to_nid(page));
b0320c7b
RA
1599 /*
1600 * If we had gigantic hugepages allocated at boot time, we need
1601 * to restore the 'stolen' pages to totalram_pages in order to
1602 * fix confusing memory reports from free(1) and another
1603 * side-effects, like CommitLimit going negative.
1604 */
bae7f4ae 1605 if (hstate_is_gigantic(h))
3dcc0571 1606 adjust_managed_page_count(page, 1 << h->order);
aa888a74
AK
1607 }
1608}
1609
8faa8b07 1610static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1da177e4
LT
1611{
1612 unsigned long i;
a5516438 1613
e5ff2159 1614 for (i = 0; i < h->max_huge_pages; ++i) {
bae7f4ae 1615 if (hstate_is_gigantic(h)) {
aa888a74
AK
1616 if (!alloc_bootmem_huge_page(h))
1617 break;
9b5e5d0f 1618 } else if (!alloc_fresh_huge_page(h,
8cebfcd0 1619 &node_states[N_MEMORY]))
1da177e4 1620 break;
1da177e4 1621 }
8faa8b07 1622 h->max_huge_pages = i;
e5ff2159
AK
1623}
1624
1625static void __init hugetlb_init_hstates(void)
1626{
1627 struct hstate *h;
1628
1629 for_each_hstate(h) {
8faa8b07 1630 /* oversize hugepages were init'ed in early boot */
bae7f4ae 1631 if (!hstate_is_gigantic(h))
8faa8b07 1632 hugetlb_hstate_alloc_pages(h);
e5ff2159
AK
1633 }
1634}
1635
4abd32db
AK
1636static char * __init memfmt(char *buf, unsigned long n)
1637{
1638 if (n >= (1UL << 30))
1639 sprintf(buf, "%lu GB", n >> 30);
1640 else if (n >= (1UL << 20))
1641 sprintf(buf, "%lu MB", n >> 20);
1642 else
1643 sprintf(buf, "%lu KB", n >> 10);
1644 return buf;
1645}
1646
e5ff2159
AK
1647static void __init report_hugepages(void)
1648{
1649 struct hstate *h;
1650
1651 for_each_hstate(h) {
4abd32db 1652 char buf[32];
ffb22af5 1653 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
4abd32db
AK
1654 memfmt(buf, huge_page_size(h)),
1655 h->free_huge_pages);
e5ff2159
AK
1656 }
1657}
1658
1da177e4 1659#ifdef CONFIG_HIGHMEM
6ae11b27
LS
1660static void try_to_free_low(struct hstate *h, unsigned long count,
1661 nodemask_t *nodes_allowed)
1da177e4 1662{
4415cc8d
CL
1663 int i;
1664
bae7f4ae 1665 if (hstate_is_gigantic(h))
aa888a74
AK
1666 return;
1667
6ae11b27 1668 for_each_node_mask(i, *nodes_allowed) {
1da177e4 1669 struct page *page, *next;
a5516438
AK
1670 struct list_head *freel = &h->hugepage_freelists[i];
1671 list_for_each_entry_safe(page, next, freel, lru) {
1672 if (count >= h->nr_huge_pages)
6b0c880d 1673 return;
1da177e4
LT
1674 if (PageHighMem(page))
1675 continue;
1676 list_del(&page->lru);
e5ff2159 1677 update_and_free_page(h, page);
a5516438
AK
1678 h->free_huge_pages--;
1679 h->free_huge_pages_node[page_to_nid(page)]--;
1da177e4
LT
1680 }
1681 }
1682}
1683#else
6ae11b27
LS
1684static inline void try_to_free_low(struct hstate *h, unsigned long count,
1685 nodemask_t *nodes_allowed)
1da177e4
LT
1686{
1687}
1688#endif
1689
20a0307c
WF
1690/*
1691 * Increment or decrement surplus_huge_pages. Keep node-specific counters
1692 * balanced by operating on them in a round-robin fashion.
1693 * Returns 1 if an adjustment was made.
1694 */
6ae11b27
LS
1695static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1696 int delta)
20a0307c 1697{
b2261026 1698 int nr_nodes, node;
20a0307c
WF
1699
1700 VM_BUG_ON(delta != -1 && delta != 1);
20a0307c 1701
b2261026
JK
1702 if (delta < 0) {
1703 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1704 if (h->surplus_huge_pages_node[node])
1705 goto found;
e8c5c824 1706 }
b2261026
JK
1707 } else {
1708 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1709 if (h->surplus_huge_pages_node[node] <
1710 h->nr_huge_pages_node[node])
1711 goto found;
e8c5c824 1712 }
b2261026
JK
1713 }
1714 return 0;
20a0307c 1715
b2261026
JK
1716found:
1717 h->surplus_huge_pages += delta;
1718 h->surplus_huge_pages_node[node] += delta;
1719 return 1;
20a0307c
WF
1720}
1721
a5516438 1722#define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
6ae11b27
LS
1723static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1724 nodemask_t *nodes_allowed)
1da177e4 1725{
7893d1d5 1726 unsigned long min_count, ret;
1da177e4 1727
944d9fec 1728 if (hstate_is_gigantic(h) && !gigantic_page_supported())
aa888a74
AK
1729 return h->max_huge_pages;
1730
7893d1d5
AL
1731 /*
1732 * Increase the pool size
1733 * First take pages out of surplus state. Then make up the
1734 * remaining difference by allocating fresh huge pages.
d1c3fb1f
NA
1735 *
1736 * We might race with alloc_buddy_huge_page() here and be unable
1737 * to convert a surplus huge page to a normal huge page. That is
1738 * not critical, though, it just means the overall size of the
1739 * pool might be one hugepage larger than it needs to be, but
1740 * within all the constraints specified by the sysctls.
7893d1d5 1741 */
1da177e4 1742 spin_lock(&hugetlb_lock);
a5516438 1743 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
6ae11b27 1744 if (!adjust_pool_surplus(h, nodes_allowed, -1))
7893d1d5
AL
1745 break;
1746 }
1747
a5516438 1748 while (count > persistent_huge_pages(h)) {
7893d1d5
AL
1749 /*
1750 * If this allocation races such that we no longer need the
1751 * page, free_huge_page will handle it by freeing the page
1752 * and reducing the surplus.
1753 */
1754 spin_unlock(&hugetlb_lock);
944d9fec
LC
1755 if (hstate_is_gigantic(h))
1756 ret = alloc_fresh_gigantic_page(h, nodes_allowed);
1757 else
1758 ret = alloc_fresh_huge_page(h, nodes_allowed);
7893d1d5
AL
1759 spin_lock(&hugetlb_lock);
1760 if (!ret)
1761 goto out;
1762
536240f2
MG
1763 /* Bail for signals. Probably ctrl-c from user */
1764 if (signal_pending(current))
1765 goto out;
7893d1d5 1766 }
7893d1d5
AL
1767
1768 /*
1769 * Decrease the pool size
1770 * First return free pages to the buddy allocator (being careful
1771 * to keep enough around to satisfy reservations). Then place
1772 * pages into surplus state as needed so the pool will shrink
1773 * to the desired size as pages become free.
d1c3fb1f
NA
1774 *
1775 * By placing pages into the surplus state independent of the
1776 * overcommit value, we are allowing the surplus pool size to
1777 * exceed overcommit. There are few sane options here. Since
1778 * alloc_buddy_huge_page() is checking the global counter,
1779 * though, we'll note that we're not allowed to exceed surplus
1780 * and won't grow the pool anywhere else. Not until one of the
1781 * sysctls are changed, or the surplus pages go out of use.
7893d1d5 1782 */
a5516438 1783 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
6b0c880d 1784 min_count = max(count, min_count);
6ae11b27 1785 try_to_free_low(h, min_count, nodes_allowed);
a5516438 1786 while (min_count < persistent_huge_pages(h)) {
6ae11b27 1787 if (!free_pool_huge_page(h, nodes_allowed, 0))
1da177e4 1788 break;
55f67141 1789 cond_resched_lock(&hugetlb_lock);
1da177e4 1790 }
a5516438 1791 while (count < persistent_huge_pages(h)) {
6ae11b27 1792 if (!adjust_pool_surplus(h, nodes_allowed, 1))
7893d1d5
AL
1793 break;
1794 }
1795out:
a5516438 1796 ret = persistent_huge_pages(h);
1da177e4 1797 spin_unlock(&hugetlb_lock);
7893d1d5 1798 return ret;
1da177e4
LT
1799}
1800
a3437870
NA
1801#define HSTATE_ATTR_RO(_name) \
1802 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1803
1804#define HSTATE_ATTR(_name) \
1805 static struct kobj_attribute _name##_attr = \
1806 __ATTR(_name, 0644, _name##_show, _name##_store)
1807
1808static struct kobject *hugepages_kobj;
1809static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1810
9a305230
LS
1811static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1812
1813static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
a3437870
NA
1814{
1815 int i;
9a305230 1816
a3437870 1817 for (i = 0; i < HUGE_MAX_HSTATE; i++)
9a305230
LS
1818 if (hstate_kobjs[i] == kobj) {
1819 if (nidp)
1820 *nidp = NUMA_NO_NODE;
a3437870 1821 return &hstates[i];
9a305230
LS
1822 }
1823
1824 return kobj_to_node_hstate(kobj, nidp);
a3437870
NA
1825}
1826
06808b08 1827static ssize_t nr_hugepages_show_common(struct kobject *kobj,
a3437870
NA
1828 struct kobj_attribute *attr, char *buf)
1829{
9a305230
LS
1830 struct hstate *h;
1831 unsigned long nr_huge_pages;
1832 int nid;
1833
1834 h = kobj_to_hstate(kobj, &nid);
1835 if (nid == NUMA_NO_NODE)
1836 nr_huge_pages = h->nr_huge_pages;
1837 else
1838 nr_huge_pages = h->nr_huge_pages_node[nid];
1839
1840 return sprintf(buf, "%lu\n", nr_huge_pages);
a3437870 1841}
adbe8726 1842
238d3c13
DR
1843static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
1844 struct hstate *h, int nid,
1845 unsigned long count, size_t len)
a3437870
NA
1846{
1847 int err;
bad44b5b 1848 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
a3437870 1849
944d9fec 1850 if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
adbe8726
EM
1851 err = -EINVAL;
1852 goto out;
1853 }
1854
9a305230
LS
1855 if (nid == NUMA_NO_NODE) {
1856 /*
1857 * global hstate attribute
1858 */
1859 if (!(obey_mempolicy &&
1860 init_nodemask_of_mempolicy(nodes_allowed))) {
1861 NODEMASK_FREE(nodes_allowed);
8cebfcd0 1862 nodes_allowed = &node_states[N_MEMORY];
9a305230
LS
1863 }
1864 } else if (nodes_allowed) {
1865 /*
1866 * per node hstate attribute: adjust count to global,
1867 * but restrict alloc/free to the specified node.
1868 */
1869 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1870 init_nodemask_of_node(nodes_allowed, nid);
1871 } else
8cebfcd0 1872 nodes_allowed = &node_states[N_MEMORY];
9a305230 1873
06808b08 1874 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
a3437870 1875
8cebfcd0 1876 if (nodes_allowed != &node_states[N_MEMORY])
06808b08
LS
1877 NODEMASK_FREE(nodes_allowed);
1878
1879 return len;
adbe8726
EM
1880out:
1881 NODEMASK_FREE(nodes_allowed);
1882 return err;
06808b08
LS
1883}
1884
238d3c13
DR
1885static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1886 struct kobject *kobj, const char *buf,
1887 size_t len)
1888{
1889 struct hstate *h;
1890 unsigned long count;
1891 int nid;
1892 int err;
1893
1894 err = kstrtoul(buf, 10, &count);
1895 if (err)
1896 return err;
1897
1898 h = kobj_to_hstate(kobj, &nid);
1899 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
1900}
1901
06808b08
LS
1902static ssize_t nr_hugepages_show(struct kobject *kobj,
1903 struct kobj_attribute *attr, char *buf)
1904{
1905 return nr_hugepages_show_common(kobj, attr, buf);
1906}
1907
1908static ssize_t nr_hugepages_store(struct kobject *kobj,
1909 struct kobj_attribute *attr, const char *buf, size_t len)
1910{
238d3c13 1911 return nr_hugepages_store_common(false, kobj, buf, len);
a3437870
NA
1912}
1913HSTATE_ATTR(nr_hugepages);
1914
06808b08
LS
1915#ifdef CONFIG_NUMA
1916
1917/*
1918 * hstate attribute for optionally mempolicy-based constraint on persistent
1919 * huge page alloc/free.
1920 */
1921static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1922 struct kobj_attribute *attr, char *buf)
1923{
1924 return nr_hugepages_show_common(kobj, attr, buf);
1925}
1926
1927static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1928 struct kobj_attribute *attr, const char *buf, size_t len)
1929{
238d3c13 1930 return nr_hugepages_store_common(true, kobj, buf, len);
06808b08
LS
1931}
1932HSTATE_ATTR(nr_hugepages_mempolicy);
1933#endif
1934
1935
a3437870
NA
1936static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1937 struct kobj_attribute *attr, char *buf)
1938{
9a305230 1939 struct hstate *h = kobj_to_hstate(kobj, NULL);
a3437870
NA
1940 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1941}
adbe8726 1942
a3437870
NA
1943static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1944 struct kobj_attribute *attr, const char *buf, size_t count)
1945{
1946 int err;
1947 unsigned long input;
9a305230 1948 struct hstate *h = kobj_to_hstate(kobj, NULL);
a3437870 1949
bae7f4ae 1950 if (hstate_is_gigantic(h))
adbe8726
EM
1951 return -EINVAL;
1952
3dbb95f7 1953 err = kstrtoul(buf, 10, &input);
a3437870 1954 if (err)
73ae31e5 1955 return err;
a3437870
NA
1956
1957 spin_lock(&hugetlb_lock);
1958 h->nr_overcommit_huge_pages = input;
1959 spin_unlock(&hugetlb_lock);
1960
1961 return count;
1962}
1963HSTATE_ATTR(nr_overcommit_hugepages);
1964
1965static ssize_t free_hugepages_show(struct kobject *kobj,
1966 struct kobj_attribute *attr, char *buf)
1967{
9a305230
LS
1968 struct hstate *h;
1969 unsigned long free_huge_pages;
1970 int nid;
1971
1972 h = kobj_to_hstate(kobj, &nid);
1973 if (nid == NUMA_NO_NODE)
1974 free_huge_pages = h->free_huge_pages;
1975 else
1976 free_huge_pages = h->free_huge_pages_node[nid];
1977
1978 return sprintf(buf, "%lu\n", free_huge_pages);
a3437870
NA
1979}
1980HSTATE_ATTR_RO(free_hugepages);
1981
1982static ssize_t resv_hugepages_show(struct kobject *kobj,
1983 struct kobj_attribute *attr, char *buf)
1984{
9a305230 1985 struct hstate *h = kobj_to_hstate(kobj, NULL);
a3437870
NA
1986 return sprintf(buf, "%lu\n", h->resv_huge_pages);
1987}
1988HSTATE_ATTR_RO(resv_hugepages);
1989
1990static ssize_t surplus_hugepages_show(struct kobject *kobj,
1991 struct kobj_attribute *attr, char *buf)
1992{
9a305230
LS
1993 struct hstate *h;
1994 unsigned long surplus_huge_pages;
1995 int nid;
1996
1997 h = kobj_to_hstate(kobj, &nid);
1998 if (nid == NUMA_NO_NODE)
1999 surplus_huge_pages = h->surplus_huge_pages;
2000 else
2001 surplus_huge_pages = h->surplus_huge_pages_node[nid];
2002
2003 return sprintf(buf, "%lu\n", surplus_huge_pages);
a3437870
NA
2004}
2005HSTATE_ATTR_RO(surplus_hugepages);
2006
2007static struct attribute *hstate_attrs[] = {
2008 &nr_hugepages_attr.attr,
2009 &nr_overcommit_hugepages_attr.attr,
2010 &free_hugepages_attr.attr,
2011 &resv_hugepages_attr.attr,
2012 &surplus_hugepages_attr.attr,
06808b08
LS
2013#ifdef CONFIG_NUMA
2014 &nr_hugepages_mempolicy_attr.attr,
2015#endif
a3437870
NA
2016 NULL,
2017};
2018
2019static struct attribute_group hstate_attr_group = {
2020 .attrs = hstate_attrs,
2021};
2022
094e9539
JM
2023static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2024 struct kobject **hstate_kobjs,
2025 struct attribute_group *hstate_attr_group)
a3437870
NA
2026{
2027 int retval;
972dc4de 2028 int hi = hstate_index(h);
a3437870 2029
9a305230
LS
2030 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2031 if (!hstate_kobjs[hi])
a3437870
NA
2032 return -ENOMEM;
2033
9a305230 2034 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
a3437870 2035 if (retval)
9a305230 2036 kobject_put(hstate_kobjs[hi]);
a3437870
NA
2037
2038 return retval;
2039}
2040
2041static void __init hugetlb_sysfs_init(void)
2042{
2043 struct hstate *h;
2044 int err;
2045
2046 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2047 if (!hugepages_kobj)
2048 return;
2049
2050 for_each_hstate(h) {
9a305230
LS
2051 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2052 hstate_kobjs, &hstate_attr_group);
a3437870 2053 if (err)
ffb22af5 2054 pr_err("Hugetlb: Unable to add hstate %s", h->name);
a3437870
NA
2055 }
2056}
2057
9a305230
LS
2058#ifdef CONFIG_NUMA
2059
2060/*
2061 * node_hstate/s - associate per node hstate attributes, via their kobjects,
10fbcf4c
KS
2062 * with node devices in node_devices[] using a parallel array. The array
2063 * index of a node device or _hstate == node id.
2064 * This is here to avoid any static dependency of the node device driver, in
9a305230
LS
2065 * the base kernel, on the hugetlb module.
2066 */
2067struct node_hstate {
2068 struct kobject *hugepages_kobj;
2069 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2070};
2071struct node_hstate node_hstates[MAX_NUMNODES];
2072
2073/*
10fbcf4c 2074 * A subset of global hstate attributes for node devices
9a305230
LS
2075 */
2076static struct attribute *per_node_hstate_attrs[] = {
2077 &nr_hugepages_attr.attr,
2078 &free_hugepages_attr.attr,
2079 &surplus_hugepages_attr.attr,
2080 NULL,
2081};
2082
2083static struct attribute_group per_node_hstate_attr_group = {
2084 .attrs = per_node_hstate_attrs,
2085};
2086
2087/*
10fbcf4c 2088 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
9a305230
LS
2089 * Returns node id via non-NULL nidp.
2090 */
2091static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2092{
2093 int nid;
2094
2095 for (nid = 0; nid < nr_node_ids; nid++) {
2096 struct node_hstate *nhs = &node_hstates[nid];
2097 int i;
2098 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2099 if (nhs->hstate_kobjs[i] == kobj) {
2100 if (nidp)
2101 *nidp = nid;
2102 return &hstates[i];
2103 }
2104 }
2105
2106 BUG();
2107 return NULL;
2108}
2109
2110/*
10fbcf4c 2111 * Unregister hstate attributes from a single node device.
9a305230
LS
2112 * No-op if no hstate attributes attached.
2113 */
3cd8b44f 2114static void hugetlb_unregister_node(struct node *node)
9a305230
LS
2115{
2116 struct hstate *h;
10fbcf4c 2117 struct node_hstate *nhs = &node_hstates[node->dev.id];
9a305230
LS
2118
2119 if (!nhs->hugepages_kobj)
9b5e5d0f 2120 return; /* no hstate attributes */
9a305230 2121
972dc4de
AK
2122 for_each_hstate(h) {
2123 int idx = hstate_index(h);
2124 if (nhs->hstate_kobjs[idx]) {
2125 kobject_put(nhs->hstate_kobjs[idx]);
2126 nhs->hstate_kobjs[idx] = NULL;
9a305230 2127 }
972dc4de 2128 }
9a305230
LS
2129
2130 kobject_put(nhs->hugepages_kobj);
2131 nhs->hugepages_kobj = NULL;
2132}
2133
2134/*
10fbcf4c 2135 * hugetlb module exit: unregister hstate attributes from node devices
9a305230
LS
2136 * that have them.
2137 */
2138static void hugetlb_unregister_all_nodes(void)
2139{
2140 int nid;
2141
2142 /*
10fbcf4c 2143 * disable node device registrations.
9a305230
LS
2144 */
2145 register_hugetlbfs_with_node(NULL, NULL);
2146
2147 /*
2148 * remove hstate attributes from any nodes that have them.
2149 */
2150 for (nid = 0; nid < nr_node_ids; nid++)
8732794b 2151 hugetlb_unregister_node(node_devices[nid]);
9a305230
LS
2152}
2153
2154/*
10fbcf4c 2155 * Register hstate attributes for a single node device.
9a305230
LS
2156 * No-op if attributes already registered.
2157 */
3cd8b44f 2158static void hugetlb_register_node(struct node *node)
9a305230
LS
2159{
2160 struct hstate *h;
10fbcf4c 2161 struct node_hstate *nhs = &node_hstates[node->dev.id];
9a305230
LS
2162 int err;
2163
2164 if (nhs->hugepages_kobj)
2165 return; /* already allocated */
2166
2167 nhs->hugepages_kobj = kobject_create_and_add("hugepages",
10fbcf4c 2168 &node->dev.kobj);
9a305230
LS
2169 if (!nhs->hugepages_kobj)
2170 return;
2171
2172 for_each_hstate(h) {
2173 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2174 nhs->hstate_kobjs,
2175 &per_node_hstate_attr_group);
2176 if (err) {
ffb22af5
AM
2177 pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2178 h->name, node->dev.id);
9a305230
LS
2179 hugetlb_unregister_node(node);
2180 break;
2181 }
2182 }
2183}
2184
2185/*
9b5e5d0f 2186 * hugetlb init time: register hstate attributes for all registered node
10fbcf4c
KS
2187 * devices of nodes that have memory. All on-line nodes should have
2188 * registered their associated device by this time.
9a305230 2189 */
7d9ca000 2190static void __init hugetlb_register_all_nodes(void)
9a305230
LS
2191{
2192 int nid;
2193
8cebfcd0 2194 for_each_node_state(nid, N_MEMORY) {
8732794b 2195 struct node *node = node_devices[nid];
10fbcf4c 2196 if (node->dev.id == nid)
9a305230
LS
2197 hugetlb_register_node(node);
2198 }
2199
2200 /*
10fbcf4c 2201 * Let the node device driver know we're here so it can
9a305230
LS
2202 * [un]register hstate attributes on node hotplug.
2203 */
2204 register_hugetlbfs_with_node(hugetlb_register_node,
2205 hugetlb_unregister_node);
2206}
2207#else /* !CONFIG_NUMA */
2208
2209static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2210{
2211 BUG();
2212 if (nidp)
2213 *nidp = -1;
2214 return NULL;
2215}
2216
2217static void hugetlb_unregister_all_nodes(void) { }
2218
2219static void hugetlb_register_all_nodes(void) { }
2220
2221#endif
2222
a3437870
NA
2223static void __exit hugetlb_exit(void)
2224{
2225 struct hstate *h;
2226
9a305230
LS
2227 hugetlb_unregister_all_nodes();
2228
a3437870 2229 for_each_hstate(h) {
972dc4de 2230 kobject_put(hstate_kobjs[hstate_index(h)]);
a3437870
NA
2231 }
2232
2233 kobject_put(hugepages_kobj);
8382d914 2234 kfree(htlb_fault_mutex_table);
a3437870
NA
2235}
2236module_exit(hugetlb_exit);
2237
2238static int __init hugetlb_init(void)
2239{
8382d914
DB
2240 int i;
2241
457c1b27 2242 if (!hugepages_supported())
0ef89d25 2243 return 0;
a3437870 2244
e11bfbfc
NP
2245 if (!size_to_hstate(default_hstate_size)) {
2246 default_hstate_size = HPAGE_SIZE;
2247 if (!size_to_hstate(default_hstate_size))
2248 hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
a3437870 2249 }
972dc4de 2250 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
e11bfbfc
NP
2251 if (default_hstate_max_huge_pages)
2252 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
a3437870
NA
2253
2254 hugetlb_init_hstates();
aa888a74 2255 gather_bootmem_prealloc();
a3437870
NA
2256 report_hugepages();
2257
2258 hugetlb_sysfs_init();
9a305230 2259 hugetlb_register_all_nodes();
7179e7bf 2260 hugetlb_cgroup_file_init();
9a305230 2261
8382d914
DB
2262#ifdef CONFIG_SMP
2263 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2264#else
2265 num_fault_mutexes = 1;
2266#endif
2267 htlb_fault_mutex_table =
2268 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2269 BUG_ON(!htlb_fault_mutex_table);
2270
2271 for (i = 0; i < num_fault_mutexes; i++)
2272 mutex_init(&htlb_fault_mutex_table[i]);
a3437870
NA
2273 return 0;
2274}
2275module_init(hugetlb_init);
2276
2277/* Should be called on processing a hugepagesz=... option */
2278void __init hugetlb_add_hstate(unsigned order)
2279{
2280 struct hstate *h;
8faa8b07
AK
2281 unsigned long i;
2282
a3437870 2283 if (size_to_hstate(PAGE_SIZE << order)) {
ffb22af5 2284 pr_warning("hugepagesz= specified twice, ignoring\n");
a3437870
NA
2285 return;
2286 }
47d38344 2287 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
a3437870 2288 BUG_ON(order == 0);
47d38344 2289 h = &hstates[hugetlb_max_hstate++];
a3437870
NA
2290 h->order = order;
2291 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
8faa8b07
AK
2292 h->nr_huge_pages = 0;
2293 h->free_huge_pages = 0;
2294 for (i = 0; i < MAX_NUMNODES; ++i)
2295 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
0edaecfa 2296 INIT_LIST_HEAD(&h->hugepage_activelist);
8cebfcd0
LJ
2297 h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2298 h->next_nid_to_free = first_node(node_states[N_MEMORY]);
a3437870
NA
2299 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2300 huge_page_size(h)/1024);
8faa8b07 2301
a3437870
NA
2302 parsed_hstate = h;
2303}
2304
e11bfbfc 2305static int __init hugetlb_nrpages_setup(char *s)
a3437870
NA
2306{
2307 unsigned long *mhp;
8faa8b07 2308 static unsigned long *last_mhp;
a3437870
NA
2309
2310 /*
47d38344 2311 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
a3437870
NA
2312 * so this hugepages= parameter goes to the "default hstate".
2313 */
47d38344 2314 if (!hugetlb_max_hstate)
a3437870
NA
2315 mhp = &default_hstate_max_huge_pages;
2316 else
2317 mhp = &parsed_hstate->max_huge_pages;
2318
8faa8b07 2319 if (mhp == last_mhp) {
ffb22af5
AM
2320 pr_warning("hugepages= specified twice without "
2321 "interleaving hugepagesz=, ignoring\n");
8faa8b07
AK
2322 return 1;
2323 }
2324
a3437870
NA
2325 if (sscanf(s, "%lu", mhp) <= 0)
2326 *mhp = 0;
2327
8faa8b07
AK
2328 /*
2329 * Global state is always initialized later in hugetlb_init.
2330 * But we need to allocate >= MAX_ORDER hstates here early to still
2331 * use the bootmem allocator.
2332 */
47d38344 2333 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
8faa8b07
AK
2334 hugetlb_hstate_alloc_pages(parsed_hstate);
2335
2336 last_mhp = mhp;
2337
a3437870
NA
2338 return 1;
2339}
e11bfbfc
NP
2340__setup("hugepages=", hugetlb_nrpages_setup);
2341
2342static int __init hugetlb_default_setup(char *s)
2343{
2344 default_hstate_size = memparse(s, &s);
2345 return 1;
2346}
2347__setup("default_hugepagesz=", hugetlb_default_setup);
a3437870 2348
8a213460
NA
2349static unsigned int cpuset_mems_nr(unsigned int *array)
2350{
2351 int node;
2352 unsigned int nr = 0;
2353
2354 for_each_node_mask(node, cpuset_current_mems_allowed)
2355 nr += array[node];
2356
2357 return nr;
2358}
2359
2360#ifdef CONFIG_SYSCTL
06808b08
LS
2361static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2362 struct ctl_table *table, int write,
2363 void __user *buffer, size_t *length, loff_t *ppos)
1da177e4 2364{
e5ff2159 2365 struct hstate *h = &default_hstate;
238d3c13 2366 unsigned long tmp = h->max_huge_pages;
08d4a246 2367 int ret;
e5ff2159 2368
457c1b27
NA
2369 if (!hugepages_supported())
2370 return -ENOTSUPP;
2371
e5ff2159
AK
2372 table->data = &tmp;
2373 table->maxlen = sizeof(unsigned long);
08d4a246
MH
2374 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2375 if (ret)
2376 goto out;
e5ff2159 2377
238d3c13
DR
2378 if (write)
2379 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2380 NUMA_NO_NODE, tmp, *length);
08d4a246
MH
2381out:
2382 return ret;
1da177e4 2383}
396faf03 2384
06808b08
LS
2385int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2386 void __user *buffer, size_t *length, loff_t *ppos)
2387{
2388
2389 return hugetlb_sysctl_handler_common(false, table, write,
2390 buffer, length, ppos);
2391}
2392
2393#ifdef CONFIG_NUMA
2394int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2395 void __user *buffer, size_t *length, loff_t *ppos)
2396{
2397 return hugetlb_sysctl_handler_common(true, table, write,
2398 buffer, length, ppos);
2399}
2400#endif /* CONFIG_NUMA */
2401
a3d0c6aa 2402int hugetlb_overcommit_handler(struct ctl_table *table, int write,
8d65af78 2403 void __user *buffer,
a3d0c6aa
NA
2404 size_t *length, loff_t *ppos)
2405{
a5516438 2406 struct hstate *h = &default_hstate;
e5ff2159 2407 unsigned long tmp;
08d4a246 2408 int ret;
e5ff2159 2409
457c1b27
NA
2410 if (!hugepages_supported())
2411 return -ENOTSUPP;
2412
c033a93c 2413 tmp = h->nr_overcommit_huge_pages;
e5ff2159 2414
bae7f4ae 2415 if (write && hstate_is_gigantic(h))
adbe8726
EM
2416 return -EINVAL;
2417
e5ff2159
AK
2418 table->data = &tmp;
2419 table->maxlen = sizeof(unsigned long);
08d4a246
MH
2420 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2421 if (ret)
2422 goto out;
e5ff2159
AK
2423
2424 if (write) {
2425 spin_lock(&hugetlb_lock);
2426 h->nr_overcommit_huge_pages = tmp;
2427 spin_unlock(&hugetlb_lock);
2428 }
08d4a246
MH
2429out:
2430 return ret;
a3d0c6aa
NA
2431}
2432
1da177e4
LT
2433#endif /* CONFIG_SYSCTL */
2434
e1759c21 2435void hugetlb_report_meminfo(struct seq_file *m)
1da177e4 2436{
a5516438 2437 struct hstate *h = &default_hstate;
457c1b27
NA
2438 if (!hugepages_supported())
2439 return;
e1759c21 2440 seq_printf(m,
4f98a2fe
RR
2441 "HugePages_Total: %5lu\n"
2442 "HugePages_Free: %5lu\n"
2443 "HugePages_Rsvd: %5lu\n"
2444 "HugePages_Surp: %5lu\n"
2445 "Hugepagesize: %8lu kB\n",
a5516438
AK
2446 h->nr_huge_pages,
2447 h->free_huge_pages,
2448 h->resv_huge_pages,
2449 h->surplus_huge_pages,
2450 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
1da177e4
LT
2451}
2452
2453int hugetlb_report_node_meminfo(int nid, char *buf)
2454{
a5516438 2455 struct hstate *h = &default_hstate;
457c1b27
NA
2456 if (!hugepages_supported())
2457 return 0;
1da177e4
LT
2458 return sprintf(buf,
2459 "Node %d HugePages_Total: %5u\n"
a1de0919
NA
2460 "Node %d HugePages_Free: %5u\n"
2461 "Node %d HugePages_Surp: %5u\n",
a5516438
AK
2462 nid, h->nr_huge_pages_node[nid],
2463 nid, h->free_huge_pages_node[nid],
2464 nid, h->surplus_huge_pages_node[nid]);
1da177e4
LT
2465}
2466
949f7ec5
DR
2467void hugetlb_show_meminfo(void)
2468{
2469 struct hstate *h;
2470 int nid;
2471
457c1b27
NA
2472 if (!hugepages_supported())
2473 return;
2474
949f7ec5
DR
2475 for_each_node_state(nid, N_MEMORY)
2476 for_each_hstate(h)
2477 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2478 nid,
2479 h->nr_huge_pages_node[nid],
2480 h->free_huge_pages_node[nid],
2481 h->surplus_huge_pages_node[nid],
2482 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2483}
2484
1da177e4
LT
2485/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2486unsigned long hugetlb_total_pages(void)
2487{
d0028588
WL
2488 struct hstate *h;
2489 unsigned long nr_total_pages = 0;
2490
2491 for_each_hstate(h)
2492 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2493 return nr_total_pages;
1da177e4 2494}
1da177e4 2495
a5516438 2496static int hugetlb_acct_memory(struct hstate *h, long delta)
fc1b8a73
MG
2497{
2498 int ret = -ENOMEM;
2499
2500 spin_lock(&hugetlb_lock);
2501 /*
2502 * When cpuset is configured, it breaks the strict hugetlb page
2503 * reservation as the accounting is done on a global variable. Such
2504 * reservation is completely rubbish in the presence of cpuset because
2505 * the reservation is not checked against page availability for the
2506 * current cpuset. Application can still potentially OOM'ed by kernel
2507 * with lack of free htlb page in cpuset that the task is in.
2508 * Attempt to enforce strict accounting with cpuset is almost
2509 * impossible (or too ugly) because cpuset is too fluid that
2510 * task or memory node can be dynamically moved between cpusets.
2511 *
2512 * The change of semantics for shared hugetlb mapping with cpuset is
2513 * undesirable. However, in order to preserve some of the semantics,
2514 * we fall back to check against current free page availability as
2515 * a best attempt and hopefully to minimize the impact of changing
2516 * semantics that cpuset has.
2517 */
2518 if (delta > 0) {
a5516438 2519 if (gather_surplus_pages(h, delta) < 0)
fc1b8a73
MG
2520 goto out;
2521
a5516438
AK
2522 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2523 return_unused_surplus_pages(h, delta);
fc1b8a73
MG
2524 goto out;
2525 }
2526 }
2527
2528 ret = 0;
2529 if (delta < 0)
a5516438 2530 return_unused_surplus_pages(h, (unsigned long) -delta);
fc1b8a73
MG
2531
2532out:
2533 spin_unlock(&hugetlb_lock);
2534 return ret;
2535}
2536
84afd99b
AW
2537static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2538{
f522c3ac 2539 struct resv_map *resv = vma_resv_map(vma);
84afd99b
AW
2540
2541 /*
2542 * This new VMA should share its siblings reservation map if present.
2543 * The VMA will only ever have a valid reservation map pointer where
2544 * it is being copied for another still existing VMA. As that VMA
25985edc 2545 * has a reference to the reservation map it cannot disappear until
84afd99b
AW
2546 * after this open call completes. It is therefore safe to take a
2547 * new reference here without additional locking.
2548 */
4e35f483 2549 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
f522c3ac 2550 kref_get(&resv->refs);
84afd99b
AW
2551}
2552
a1e78772
MG
2553static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2554{
a5516438 2555 struct hstate *h = hstate_vma(vma);
f522c3ac 2556 struct resv_map *resv = vma_resv_map(vma);
90481622 2557 struct hugepage_subpool *spool = subpool_vma(vma);
4e35f483 2558 unsigned long reserve, start, end;
1c5ecae3 2559 long gbl_reserve;
84afd99b 2560
4e35f483
JK
2561 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2562 return;
84afd99b 2563
4e35f483
JK
2564 start = vma_hugecache_offset(h, vma, vma->vm_start);
2565 end = vma_hugecache_offset(h, vma, vma->vm_end);
84afd99b 2566
4e35f483 2567 reserve = (end - start) - region_count(resv, start, end);
84afd99b 2568
4e35f483
JK
2569 kref_put(&resv->refs, resv_map_release);
2570
2571 if (reserve) {
1c5ecae3
MK
2572 /*
2573 * Decrement reserve counts. The global reserve count may be
2574 * adjusted if the subpool has a minimum size.
2575 */
2576 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
2577 hugetlb_acct_memory(h, -gbl_reserve);
84afd99b 2578 }
a1e78772
MG
2579}
2580
1da177e4
LT
2581/*
2582 * We cannot handle pagefaults against hugetlb pages at all. They cause
2583 * handle_mm_fault() to try to instantiate regular-sized pages in the
2584 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
2585 * this far.
2586 */
d0217ac0 2587static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1da177e4
LT
2588{
2589 BUG();
d0217ac0 2590 return 0;
1da177e4
LT
2591}
2592
f0f37e2f 2593const struct vm_operations_struct hugetlb_vm_ops = {
d0217ac0 2594 .fault = hugetlb_vm_op_fault,
84afd99b 2595 .open = hugetlb_vm_op_open,
a1e78772 2596 .close = hugetlb_vm_op_close,
1da177e4
LT
2597};
2598
1e8f889b
DG
2599static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2600 int writable)
63551ae0
DG
2601{
2602 pte_t entry;
2603
1e8f889b 2604 if (writable) {
106c992a
GS
2605 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2606 vma->vm_page_prot)));
63551ae0 2607 } else {
106c992a
GS
2608 entry = huge_pte_wrprotect(mk_huge_pte(page,
2609 vma->vm_page_prot));
63551ae0
DG
2610 }
2611 entry = pte_mkyoung(entry);
2612 entry = pte_mkhuge(entry);
d9ed9faa 2613 entry = arch_make_huge_pte(entry, vma, page, writable);
63551ae0
DG
2614
2615 return entry;
2616}
2617
1e8f889b
DG
2618static void set_huge_ptep_writable(struct vm_area_struct *vma,
2619 unsigned long address, pte_t *ptep)
2620{
2621 pte_t entry;
2622
106c992a 2623 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
32f84528 2624 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
4b3073e1 2625 update_mmu_cache(vma, address, ptep);
1e8f889b
DG
2626}
2627
4a705fef
NH
2628static int is_hugetlb_entry_migration(pte_t pte)
2629{
2630 swp_entry_t swp;
2631
2632 if (huge_pte_none(pte) || pte_present(pte))
2633 return 0;
2634 swp = pte_to_swp_entry(pte);
2635 if (non_swap_entry(swp) && is_migration_entry(swp))
2636 return 1;
2637 else
2638 return 0;
2639}
2640
2641static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2642{
2643 swp_entry_t swp;
2644
2645 if (huge_pte_none(pte) || pte_present(pte))
2646 return 0;
2647 swp = pte_to_swp_entry(pte);
2648 if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2649 return 1;
2650 else
2651 return 0;
2652}
1e8f889b 2653
63551ae0
DG
2654int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2655 struct vm_area_struct *vma)
2656{
2657 pte_t *src_pte, *dst_pte, entry;
2658 struct page *ptepage;
1c59827d 2659 unsigned long addr;
1e8f889b 2660 int cow;
a5516438
AK
2661 struct hstate *h = hstate_vma(vma);
2662 unsigned long sz = huge_page_size(h);
e8569dd2
AS
2663 unsigned long mmun_start; /* For mmu_notifiers */
2664 unsigned long mmun_end; /* For mmu_notifiers */
2665 int ret = 0;
1e8f889b
DG
2666
2667 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
63551ae0 2668
e8569dd2
AS
2669 mmun_start = vma->vm_start;
2670 mmun_end = vma->vm_end;
2671 if (cow)
2672 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
2673
a5516438 2674 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
cb900f41 2675 spinlock_t *src_ptl, *dst_ptl;
c74df32c
HD
2676 src_pte = huge_pte_offset(src, addr);
2677 if (!src_pte)
2678 continue;
a5516438 2679 dst_pte = huge_pte_alloc(dst, addr, sz);
e8569dd2
AS
2680 if (!dst_pte) {
2681 ret = -ENOMEM;
2682 break;
2683 }
c5c99429
LW
2684
2685 /* If the pagetables are shared don't copy or take references */
2686 if (dst_pte == src_pte)
2687 continue;
2688
cb900f41
KS
2689 dst_ptl = huge_pte_lock(h, dst, dst_pte);
2690 src_ptl = huge_pte_lockptr(h, src, src_pte);
2691 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4a705fef
NH
2692 entry = huge_ptep_get(src_pte);
2693 if (huge_pte_none(entry)) { /* skip none entry */
2694 ;
2695 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
2696 is_hugetlb_entry_hwpoisoned(entry))) {
2697 swp_entry_t swp_entry = pte_to_swp_entry(entry);
2698
2699 if (is_write_migration_entry(swp_entry) && cow) {
2700 /*
2701 * COW mappings require pages in both
2702 * parent and child to be set to read.
2703 */
2704 make_migration_entry_read(&swp_entry);
2705 entry = swp_entry_to_pte(swp_entry);
2706 set_huge_pte_at(src, addr, src_pte, entry);
2707 }
2708 set_huge_pte_at(dst, addr, dst_pte, entry);
2709 } else {
34ee645e 2710 if (cow) {
7f2e9525 2711 huge_ptep_set_wrprotect(src, addr, src_pte);
34ee645e
JR
2712 mmu_notifier_invalidate_range(src, mmun_start,
2713 mmun_end);
2714 }
0253d634 2715 entry = huge_ptep_get(src_pte);
1c59827d
HD
2716 ptepage = pte_page(entry);
2717 get_page(ptepage);
0fe6e20b 2718 page_dup_rmap(ptepage);
1c59827d
HD
2719 set_huge_pte_at(dst, addr, dst_pte, entry);
2720 }
cb900f41
KS
2721 spin_unlock(src_ptl);
2722 spin_unlock(dst_ptl);
63551ae0 2723 }
63551ae0 2724
e8569dd2
AS
2725 if (cow)
2726 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
2727
2728 return ret;
63551ae0
DG
2729}
2730
24669e58
AK
2731void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
2732 unsigned long start, unsigned long end,
2733 struct page *ref_page)
63551ae0 2734{
24669e58 2735 int force_flush = 0;
63551ae0
DG
2736 struct mm_struct *mm = vma->vm_mm;
2737 unsigned long address;
c7546f8f 2738 pte_t *ptep;
63551ae0 2739 pte_t pte;
cb900f41 2740 spinlock_t *ptl;
63551ae0 2741 struct page *page;
a5516438
AK
2742 struct hstate *h = hstate_vma(vma);
2743 unsigned long sz = huge_page_size(h);
2ec74c3e
SG
2744 const unsigned long mmun_start = start; /* For mmu_notifiers */
2745 const unsigned long mmun_end = end; /* For mmu_notifiers */
a5516438 2746
63551ae0 2747 WARN_ON(!is_vm_hugetlb_page(vma));
a5516438
AK
2748 BUG_ON(start & ~huge_page_mask(h));
2749 BUG_ON(end & ~huge_page_mask(h));
63551ae0 2750
24669e58 2751 tlb_start_vma(tlb, vma);
2ec74c3e 2752 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
569f48b8 2753 address = start;
24669e58 2754again:
569f48b8 2755 for (; address < end; address += sz) {
c7546f8f 2756 ptep = huge_pte_offset(mm, address);
4c887265 2757 if (!ptep)
c7546f8f
DG
2758 continue;
2759
cb900f41 2760 ptl = huge_pte_lock(h, mm, ptep);
39dde65c 2761 if (huge_pmd_unshare(mm, &address, ptep))
cb900f41 2762 goto unlock;
39dde65c 2763
6629326b
HD
2764 pte = huge_ptep_get(ptep);
2765 if (huge_pte_none(pte))
cb900f41 2766 goto unlock;
6629326b
HD
2767
2768 /*
9fbc1f63
NH
2769 * Migrating hugepage or HWPoisoned hugepage is already
2770 * unmapped and its refcount is dropped, so just clear pte here.
6629326b 2771 */
9fbc1f63 2772 if (unlikely(!pte_present(pte))) {
106c992a 2773 huge_pte_clear(mm, address, ptep);
cb900f41 2774 goto unlock;
8c4894c6 2775 }
6629326b
HD
2776
2777 page = pte_page(pte);
04f2cbe3
MG
2778 /*
2779 * If a reference page is supplied, it is because a specific
2780 * page is being unmapped, not a range. Ensure the page we
2781 * are about to unmap is the actual page of interest.
2782 */
2783 if (ref_page) {
04f2cbe3 2784 if (page != ref_page)
cb900f41 2785 goto unlock;
04f2cbe3
MG
2786
2787 /*
2788 * Mark the VMA as having unmapped its page so that
2789 * future faults in this VMA will fail rather than
2790 * looking like data was lost
2791 */
2792 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2793 }
2794
c7546f8f 2795 pte = huge_ptep_get_and_clear(mm, address, ptep);
24669e58 2796 tlb_remove_tlb_entry(tlb, ptep, address);
106c992a 2797 if (huge_pte_dirty(pte))
6649a386 2798 set_page_dirty(page);
9e81130b 2799
24669e58
AK
2800 page_remove_rmap(page);
2801 force_flush = !__tlb_remove_page(tlb, page);
cb900f41 2802 if (force_flush) {
569f48b8 2803 address += sz;
cb900f41 2804 spin_unlock(ptl);
24669e58 2805 break;
cb900f41 2806 }
9e81130b 2807 /* Bail out after unmapping reference page if supplied */
cb900f41
KS
2808 if (ref_page) {
2809 spin_unlock(ptl);
9e81130b 2810 break;
cb900f41
KS
2811 }
2812unlock:
2813 spin_unlock(ptl);
63551ae0 2814 }
24669e58
AK
2815 /*
2816 * mmu_gather ran out of room to batch pages, we break out of
2817 * the PTE lock to avoid doing the potential expensive TLB invalidate
2818 * and page-free while holding it.
2819 */
2820 if (force_flush) {
2821 force_flush = 0;
2822 tlb_flush_mmu(tlb);
2823 if (address < end && !ref_page)
2824 goto again;
fe1668ae 2825 }
2ec74c3e 2826 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
24669e58 2827 tlb_end_vma(tlb, vma);
1da177e4 2828}
63551ae0 2829
d833352a
MG
2830void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2831 struct vm_area_struct *vma, unsigned long start,
2832 unsigned long end, struct page *ref_page)
2833{
2834 __unmap_hugepage_range(tlb, vma, start, end, ref_page);
2835
2836 /*
2837 * Clear this flag so that x86's huge_pmd_share page_table_shareable
2838 * test will fail on a vma being torn down, and not grab a page table
2839 * on its way out. We're lucky that the flag has such an appropriate
2840 * name, and can in fact be safely cleared here. We could clear it
2841 * before the __unmap_hugepage_range above, but all that's necessary
c8c06efa 2842 * is to clear it before releasing the i_mmap_rwsem. This works
d833352a 2843 * because in the context this is called, the VMA is about to be
c8c06efa 2844 * destroyed and the i_mmap_rwsem is held.
d833352a
MG
2845 */
2846 vma->vm_flags &= ~VM_MAYSHARE;
2847}
2848
502717f4 2849void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
04f2cbe3 2850 unsigned long end, struct page *ref_page)
502717f4 2851{
24669e58
AK
2852 struct mm_struct *mm;
2853 struct mmu_gather tlb;
2854
2855 mm = vma->vm_mm;
2856
2b047252 2857 tlb_gather_mmu(&tlb, mm, start, end);
24669e58
AK
2858 __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2859 tlb_finish_mmu(&tlb, start, end);
502717f4
CK
2860}
2861
04f2cbe3
MG
2862/*
2863 * This is called when the original mapper is failing to COW a MAP_PRIVATE
2864 * mappping it owns the reserve page for. The intention is to unmap the page
2865 * from other VMAs and let the children be SIGKILLed if they are faulting the
2866 * same region.
2867 */
2f4612af
DB
2868static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2869 struct page *page, unsigned long address)
04f2cbe3 2870{
7526674d 2871 struct hstate *h = hstate_vma(vma);
04f2cbe3
MG
2872 struct vm_area_struct *iter_vma;
2873 struct address_space *mapping;
04f2cbe3
MG
2874 pgoff_t pgoff;
2875
2876 /*
2877 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2878 * from page cache lookup which is in HPAGE_SIZE units.
2879 */
7526674d 2880 address = address & huge_page_mask(h);
36e4f20a
MH
2881 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2882 vma->vm_pgoff;
496ad9aa 2883 mapping = file_inode(vma->vm_file)->i_mapping;
04f2cbe3 2884
4eb2b1dc
MG
2885 /*
2886 * Take the mapping lock for the duration of the table walk. As
2887 * this mapping should be shared between all the VMAs,
2888 * __unmap_hugepage_range() is called as the lock is already held
2889 */
83cde9e8 2890 i_mmap_lock_write(mapping);
6b2dbba8 2891 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
04f2cbe3
MG
2892 /* Do not unmap the current VMA */
2893 if (iter_vma == vma)
2894 continue;
2895
2896 /*
2897 * Unmap the page from other VMAs without their own reserves.
2898 * They get marked to be SIGKILLed if they fault in these
2899 * areas. This is because a future no-page fault on this VMA
2900 * could insert a zeroed page instead of the data existing
2901 * from the time of fork. This would look like data corruption
2902 */
2903 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
24669e58
AK
2904 unmap_hugepage_range(iter_vma, address,
2905 address + huge_page_size(h), page);
04f2cbe3 2906 }
83cde9e8 2907 i_mmap_unlock_write(mapping);
04f2cbe3
MG
2908}
2909
0fe6e20b
NH
2910/*
2911 * Hugetlb_cow() should be called with page lock of the original hugepage held.
ef009b25
MH
2912 * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2913 * cannot race with other handlers or page migration.
2914 * Keep the pte_same checks anyway to make transition from the mutex easier.
0fe6e20b 2915 */
1e8f889b 2916static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
04f2cbe3 2917 unsigned long address, pte_t *ptep, pte_t pte,
cb900f41 2918 struct page *pagecache_page, spinlock_t *ptl)
1e8f889b 2919{
a5516438 2920 struct hstate *h = hstate_vma(vma);
1e8f889b 2921 struct page *old_page, *new_page;
ad4404a2 2922 int ret = 0, outside_reserve = 0;
2ec74c3e
SG
2923 unsigned long mmun_start; /* For mmu_notifiers */
2924 unsigned long mmun_end; /* For mmu_notifiers */
1e8f889b
DG
2925
2926 old_page = pte_page(pte);
2927
04f2cbe3 2928retry_avoidcopy:
1e8f889b
DG
2929 /* If no-one else is actually using this page, avoid the copy
2930 * and just make the page writable */
37a2140d
JK
2931 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2932 page_move_anon_rmap(old_page, vma, address);
1e8f889b 2933 set_huge_ptep_writable(vma, address, ptep);
83c54070 2934 return 0;
1e8f889b
DG
2935 }
2936
04f2cbe3
MG
2937 /*
2938 * If the process that created a MAP_PRIVATE mapping is about to
2939 * perform a COW due to a shared page count, attempt to satisfy
2940 * the allocation without using the existing reserves. The pagecache
2941 * page is used to determine if the reserve at this address was
2942 * consumed or not. If reserves were used, a partial faulted mapping
2943 * at the time of fork() could consume its reserves on COW instead
2944 * of the full address range.
2945 */
5944d011 2946 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
04f2cbe3
MG
2947 old_page != pagecache_page)
2948 outside_reserve = 1;
2949
1e8f889b 2950 page_cache_get(old_page);
b76c8cfb 2951
ad4404a2
DB
2952 /*
2953 * Drop page table lock as buddy allocator may be called. It will
2954 * be acquired again before returning to the caller, as expected.
2955 */
cb900f41 2956 spin_unlock(ptl);
04f2cbe3 2957 new_page = alloc_huge_page(vma, address, outside_reserve);
1e8f889b 2958
2fc39cec 2959 if (IS_ERR(new_page)) {
04f2cbe3
MG
2960 /*
2961 * If a process owning a MAP_PRIVATE mapping fails to COW,
2962 * it is due to references held by a child and an insufficient
2963 * huge page pool. To guarantee the original mappers
2964 * reliability, unmap the page from child processes. The child
2965 * may get SIGKILLed if it later faults.
2966 */
2967 if (outside_reserve) {
ad4404a2 2968 page_cache_release(old_page);
04f2cbe3 2969 BUG_ON(huge_pte_none(pte));
2f4612af
DB
2970 unmap_ref_private(mm, vma, old_page, address);
2971 BUG_ON(huge_pte_none(pte));
2972 spin_lock(ptl);
2973 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2974 if (likely(ptep &&
2975 pte_same(huge_ptep_get(ptep), pte)))
2976 goto retry_avoidcopy;
2977 /*
2978 * race occurs while re-acquiring page table
2979 * lock, and our job is done.
2980 */
2981 return 0;
04f2cbe3
MG
2982 }
2983
ad4404a2
DB
2984 ret = (PTR_ERR(new_page) == -ENOMEM) ?
2985 VM_FAULT_OOM : VM_FAULT_SIGBUS;
2986 goto out_release_old;
1e8f889b
DG
2987 }
2988
0fe6e20b
NH
2989 /*
2990 * When the original hugepage is shared one, it does not have
2991 * anon_vma prepared.
2992 */
44e2aa93 2993 if (unlikely(anon_vma_prepare(vma))) {
ad4404a2
DB
2994 ret = VM_FAULT_OOM;
2995 goto out_release_all;
44e2aa93 2996 }
0fe6e20b 2997
47ad8475
AA
2998 copy_user_huge_page(new_page, old_page, address, vma,
2999 pages_per_huge_page(h));
0ed361de 3000 __SetPageUptodate(new_page);
bcc54222 3001 set_page_huge_active(new_page);
1e8f889b 3002
2ec74c3e
SG
3003 mmun_start = address & huge_page_mask(h);
3004 mmun_end = mmun_start + huge_page_size(h);
3005 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
ad4404a2 3006
b76c8cfb 3007 /*
cb900f41 3008 * Retake the page table lock to check for racing updates
b76c8cfb
LW
3009 * before the page tables are altered
3010 */
cb900f41 3011 spin_lock(ptl);
a5516438 3012 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
a9af0c5d 3013 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
07443a85
JK
3014 ClearPagePrivate(new_page);
3015
1e8f889b 3016 /* Break COW */
8fe627ec 3017 huge_ptep_clear_flush(vma, address, ptep);
34ee645e 3018 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
1e8f889b
DG
3019 set_huge_pte_at(mm, address, ptep,
3020 make_huge_pte(vma, new_page, 1));
0fe6e20b 3021 page_remove_rmap(old_page);
cd67f0d2 3022 hugepage_add_new_anon_rmap(new_page, vma, address);
1e8f889b
DG
3023 /* Make the old page be freed below */
3024 new_page = old_page;
3025 }
cb900f41 3026 spin_unlock(ptl);
2ec74c3e 3027 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
ad4404a2 3028out_release_all:
1e8f889b 3029 page_cache_release(new_page);
ad4404a2 3030out_release_old:
1e8f889b 3031 page_cache_release(old_page);
8312034f 3032
ad4404a2
DB
3033 spin_lock(ptl); /* Caller expects lock to be held */
3034 return ret;
1e8f889b
DG
3035}
3036
04f2cbe3 3037/* Return the pagecache page at a given address within a VMA */
a5516438
AK
3038static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3039 struct vm_area_struct *vma, unsigned long address)
04f2cbe3
MG
3040{
3041 struct address_space *mapping;
e7c4b0bf 3042 pgoff_t idx;
04f2cbe3
MG
3043
3044 mapping = vma->vm_file->f_mapping;
a5516438 3045 idx = vma_hugecache_offset(h, vma, address);
04f2cbe3
MG
3046
3047 return find_lock_page(mapping, idx);
3048}
3049
3ae77f43
HD
3050/*
3051 * Return whether there is a pagecache page to back given address within VMA.
3052 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3053 */
3054static bool hugetlbfs_pagecache_present(struct hstate *h,
2a15efc9
HD
3055 struct vm_area_struct *vma, unsigned long address)
3056{
3057 struct address_space *mapping;
3058 pgoff_t idx;
3059 struct page *page;
3060
3061 mapping = vma->vm_file->f_mapping;
3062 idx = vma_hugecache_offset(h, vma, address);
3063
3064 page = find_get_page(mapping, idx);
3065 if (page)
3066 put_page(page);
3067 return page != NULL;
3068}
3069
a1ed3dda 3070static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
8382d914
DB
3071 struct address_space *mapping, pgoff_t idx,
3072 unsigned long address, pte_t *ptep, unsigned int flags)
ac9b9c66 3073{
a5516438 3074 struct hstate *h = hstate_vma(vma);
ac9b9c66 3075 int ret = VM_FAULT_SIGBUS;
409eb8c2 3076 int anon_rmap = 0;
4c887265 3077 unsigned long size;
4c887265 3078 struct page *page;
1e8f889b 3079 pte_t new_pte;
cb900f41 3080 spinlock_t *ptl;
4c887265 3081
04f2cbe3
MG
3082 /*
3083 * Currently, we are forced to kill the process in the event the
3084 * original mapper has unmapped pages from the child due to a failed
25985edc 3085 * COW. Warn that such a situation has occurred as it may not be obvious
04f2cbe3
MG
3086 */
3087 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
ffb22af5
AM
3088 pr_warning("PID %d killed due to inadequate hugepage pool\n",
3089 current->pid);
04f2cbe3
MG
3090 return ret;
3091 }
3092
4c887265
AL
3093 /*
3094 * Use page lock to guard against racing truncation
3095 * before we get page_table_lock.
3096 */
6bda666a
CL
3097retry:
3098 page = find_lock_page(mapping, idx);
3099 if (!page) {
a5516438 3100 size = i_size_read(mapping->host) >> huge_page_shift(h);
ebed4bfc
HD
3101 if (idx >= size)
3102 goto out;
04f2cbe3 3103 page = alloc_huge_page(vma, address, 0);
2fc39cec 3104 if (IS_ERR(page)) {
76dcee75
AK
3105 ret = PTR_ERR(page);
3106 if (ret == -ENOMEM)
3107 ret = VM_FAULT_OOM;
3108 else
3109 ret = VM_FAULT_SIGBUS;
6bda666a
CL
3110 goto out;
3111 }
47ad8475 3112 clear_huge_page(page, address, pages_per_huge_page(h));
0ed361de 3113 __SetPageUptodate(page);
bcc54222 3114 set_page_huge_active(page);
ac9b9c66 3115
f83a275d 3116 if (vma->vm_flags & VM_MAYSHARE) {
6bda666a 3117 int err;
45c682a6 3118 struct inode *inode = mapping->host;
6bda666a
CL
3119
3120 err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3121 if (err) {
3122 put_page(page);
6bda666a
CL
3123 if (err == -EEXIST)
3124 goto retry;
3125 goto out;
3126 }
07443a85 3127 ClearPagePrivate(page);
45c682a6
KC
3128
3129 spin_lock(&inode->i_lock);
a5516438 3130 inode->i_blocks += blocks_per_huge_page(h);
45c682a6 3131 spin_unlock(&inode->i_lock);
23be7468 3132 } else {
6bda666a 3133 lock_page(page);
0fe6e20b
NH
3134 if (unlikely(anon_vma_prepare(vma))) {
3135 ret = VM_FAULT_OOM;
3136 goto backout_unlocked;
3137 }
409eb8c2 3138 anon_rmap = 1;
23be7468 3139 }
0fe6e20b 3140 } else {
998b4382
NH
3141 /*
3142 * If memory error occurs between mmap() and fault, some process
3143 * don't have hwpoisoned swap entry for errored virtual address.
3144 * So we need to block hugepage fault by PG_hwpoison bit check.
3145 */
3146 if (unlikely(PageHWPoison(page))) {
32f84528 3147 ret = VM_FAULT_HWPOISON |
972dc4de 3148 VM_FAULT_SET_HINDEX(hstate_index(h));
998b4382
NH
3149 goto backout_unlocked;
3150 }
6bda666a 3151 }
1e8f889b 3152
57303d80
AW
3153 /*
3154 * If we are going to COW a private mapping later, we examine the
3155 * pending reservations for this page now. This will ensure that
3156 * any allocations necessary to record that reservation occur outside
3157 * the spinlock.
3158 */
788c7df4 3159 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2b26736c
AW
3160 if (vma_needs_reservation(h, vma, address) < 0) {
3161 ret = VM_FAULT_OOM;
3162 goto backout_unlocked;
3163 }
57303d80 3164
cb900f41
KS
3165 ptl = huge_pte_lockptr(h, mm, ptep);
3166 spin_lock(ptl);
a5516438 3167 size = i_size_read(mapping->host) >> huge_page_shift(h);
4c887265
AL
3168 if (idx >= size)
3169 goto backout;
3170
83c54070 3171 ret = 0;
7f2e9525 3172 if (!huge_pte_none(huge_ptep_get(ptep)))
4c887265
AL
3173 goto backout;
3174
07443a85
JK
3175 if (anon_rmap) {
3176 ClearPagePrivate(page);
409eb8c2 3177 hugepage_add_new_anon_rmap(page, vma, address);
ac714904 3178 } else
409eb8c2 3179 page_dup_rmap(page);
1e8f889b
DG
3180 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3181 && (vma->vm_flags & VM_SHARED)));
3182 set_huge_pte_at(mm, address, ptep, new_pte);
3183
788c7df4 3184 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
1e8f889b 3185 /* Optimization, do the COW without a second fault */
cb900f41 3186 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
1e8f889b
DG
3187 }
3188
cb900f41 3189 spin_unlock(ptl);
4c887265
AL
3190 unlock_page(page);
3191out:
ac9b9c66 3192 return ret;
4c887265
AL
3193
3194backout:
cb900f41 3195 spin_unlock(ptl);
2b26736c 3196backout_unlocked:
4c887265
AL
3197 unlock_page(page);
3198 put_page(page);
3199 goto out;
ac9b9c66
HD
3200}
3201
8382d914
DB
3202#ifdef CONFIG_SMP
3203static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3204 struct vm_area_struct *vma,
3205 struct address_space *mapping,
3206 pgoff_t idx, unsigned long address)
3207{
3208 unsigned long key[2];
3209 u32 hash;
3210
3211 if (vma->vm_flags & VM_SHARED) {
3212 key[0] = (unsigned long) mapping;
3213 key[1] = idx;
3214 } else {
3215 key[0] = (unsigned long) mm;
3216 key[1] = address >> huge_page_shift(h);
3217 }
3218
3219 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3220
3221 return hash & (num_fault_mutexes - 1);
3222}
3223#else
3224/*
3225 * For uniprocesor systems we always use a single mutex, so just
3226 * return 0 and avoid the hashing overhead.
3227 */
3228static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3229 struct vm_area_struct *vma,
3230 struct address_space *mapping,
3231 pgoff_t idx, unsigned long address)
3232{
3233 return 0;
3234}
3235#endif
3236
86e5216f 3237int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
788c7df4 3238 unsigned long address, unsigned int flags)
86e5216f 3239{
8382d914 3240 pte_t *ptep, entry;
cb900f41 3241 spinlock_t *ptl;
1e8f889b 3242 int ret;
8382d914
DB
3243 u32 hash;
3244 pgoff_t idx;
0fe6e20b 3245 struct page *page = NULL;
57303d80 3246 struct page *pagecache_page = NULL;
a5516438 3247 struct hstate *h = hstate_vma(vma);
8382d914 3248 struct address_space *mapping;
0f792cf9 3249 int need_wait_lock = 0;
86e5216f 3250
1e16a539
KH
3251 address &= huge_page_mask(h);
3252
fd6a03ed
NH
3253 ptep = huge_pte_offset(mm, address);
3254 if (ptep) {
3255 entry = huge_ptep_get(ptep);
290408d4 3256 if (unlikely(is_hugetlb_entry_migration(entry))) {
cb900f41 3257 migration_entry_wait_huge(vma, mm, ptep);
290408d4
NH
3258 return 0;
3259 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
32f84528 3260 return VM_FAULT_HWPOISON_LARGE |
972dc4de 3261 VM_FAULT_SET_HINDEX(hstate_index(h));
fd6a03ed
NH
3262 }
3263
a5516438 3264 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
86e5216f
AL
3265 if (!ptep)
3266 return VM_FAULT_OOM;
3267
8382d914
DB
3268 mapping = vma->vm_file->f_mapping;
3269 idx = vma_hugecache_offset(h, vma, address);
3270
3935baa9
DG
3271 /*
3272 * Serialize hugepage allocation and instantiation, so that we don't
3273 * get spurious allocation failures if two CPUs race to instantiate
3274 * the same page in the page cache.
3275 */
8382d914
DB
3276 hash = fault_mutex_hash(h, mm, vma, mapping, idx, address);
3277 mutex_lock(&htlb_fault_mutex_table[hash]);
3278
7f2e9525
GS
3279 entry = huge_ptep_get(ptep);
3280 if (huge_pte_none(entry)) {
8382d914 3281 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
b4d1d99f 3282 goto out_mutex;
3935baa9 3283 }
86e5216f 3284
83c54070 3285 ret = 0;
1e8f889b 3286
0f792cf9
NH
3287 /*
3288 * entry could be a migration/hwpoison entry at this point, so this
3289 * check prevents the kernel from going below assuming that we have
3290 * a active hugepage in pagecache. This goto expects the 2nd page fault,
3291 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3292 * handle it.
3293 */
3294 if (!pte_present(entry))
3295 goto out_mutex;
3296
57303d80
AW
3297 /*
3298 * If we are going to COW the mapping later, we examine the pending
3299 * reservations for this page now. This will ensure that any
3300 * allocations necessary to record that reservation occur outside the
3301 * spinlock. For private mappings, we also lookup the pagecache
3302 * page now as it is used to determine if a reservation has been
3303 * consumed.
3304 */
106c992a 3305 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
2b26736c
AW
3306 if (vma_needs_reservation(h, vma, address) < 0) {
3307 ret = VM_FAULT_OOM;
b4d1d99f 3308 goto out_mutex;
2b26736c 3309 }
57303d80 3310
f83a275d 3311 if (!(vma->vm_flags & VM_MAYSHARE))
57303d80
AW
3312 pagecache_page = hugetlbfs_pagecache_page(h,
3313 vma, address);
3314 }
3315
0f792cf9
NH
3316 ptl = huge_pte_lock(h, mm, ptep);
3317
3318 /* Check for a racing update before calling hugetlb_cow */
3319 if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3320 goto out_ptl;
3321
56c9cfb1
NH
3322 /*
3323 * hugetlb_cow() requires page locks of pte_page(entry) and
3324 * pagecache_page, so here we need take the former one
3325 * when page != pagecache_page or !pagecache_page.
56c9cfb1
NH
3326 */
3327 page = pte_page(entry);
3328 if (page != pagecache_page)
0f792cf9
NH
3329 if (!trylock_page(page)) {
3330 need_wait_lock = 1;
3331 goto out_ptl;
3332 }
b4d1d99f 3333
0f792cf9 3334 get_page(page);
b4d1d99f 3335
788c7df4 3336 if (flags & FAULT_FLAG_WRITE) {
106c992a 3337 if (!huge_pte_write(entry)) {
57303d80 3338 ret = hugetlb_cow(mm, vma, address, ptep, entry,
cb900f41 3339 pagecache_page, ptl);
0f792cf9 3340 goto out_put_page;
b4d1d99f 3341 }
106c992a 3342 entry = huge_pte_mkdirty(entry);
b4d1d99f
DG
3343 }
3344 entry = pte_mkyoung(entry);
788c7df4
HD
3345 if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3346 flags & FAULT_FLAG_WRITE))
4b3073e1 3347 update_mmu_cache(vma, address, ptep);
0f792cf9
NH
3348out_put_page:
3349 if (page != pagecache_page)
3350 unlock_page(page);
3351 put_page(page);
cb900f41
KS
3352out_ptl:
3353 spin_unlock(ptl);
57303d80
AW
3354
3355 if (pagecache_page) {
3356 unlock_page(pagecache_page);
3357 put_page(pagecache_page);
3358 }
b4d1d99f 3359out_mutex:
8382d914 3360 mutex_unlock(&htlb_fault_mutex_table[hash]);
0f792cf9
NH
3361 /*
3362 * Generally it's safe to hold refcount during waiting page lock. But
3363 * here we just wait to defer the next page fault to avoid busy loop and
3364 * the page is not used after unlocked before returning from the current
3365 * page fault. So we are safe from accessing freed page, even if we wait
3366 * here without taking refcount.
3367 */
3368 if (need_wait_lock)
3369 wait_on_page_locked(page);
1e8f889b 3370 return ret;
86e5216f
AL
3371}
3372
28a35716
ML
3373long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3374 struct page **pages, struct vm_area_struct **vmas,
3375 unsigned long *position, unsigned long *nr_pages,
3376 long i, unsigned int flags)
63551ae0 3377{
d5d4b0aa
CK
3378 unsigned long pfn_offset;
3379 unsigned long vaddr = *position;
28a35716 3380 unsigned long remainder = *nr_pages;
a5516438 3381 struct hstate *h = hstate_vma(vma);
63551ae0 3382
63551ae0 3383 while (vaddr < vma->vm_end && remainder) {
4c887265 3384 pte_t *pte;
cb900f41 3385 spinlock_t *ptl = NULL;
2a15efc9 3386 int absent;
4c887265 3387 struct page *page;
63551ae0 3388
02057967
DR
3389 /*
3390 * If we have a pending SIGKILL, don't keep faulting pages and
3391 * potentially allocating memory.
3392 */
3393 if (unlikely(fatal_signal_pending(current))) {
3394 remainder = 0;
3395 break;
3396 }
3397
4c887265
AL
3398 /*
3399 * Some archs (sparc64, sh*) have multiple pte_ts to
2a15efc9 3400 * each hugepage. We have to make sure we get the
4c887265 3401 * first, for the page indexing below to work.
cb900f41
KS
3402 *
3403 * Note that page table lock is not held when pte is null.
4c887265 3404 */
a5516438 3405 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
cb900f41
KS
3406 if (pte)
3407 ptl = huge_pte_lock(h, mm, pte);
2a15efc9
HD
3408 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3409
3410 /*
3411 * When coredumping, it suits get_dump_page if we just return
3ae77f43
HD
3412 * an error where there's an empty slot with no huge pagecache
3413 * to back it. This way, we avoid allocating a hugepage, and
3414 * the sparse dumpfile avoids allocating disk blocks, but its
3415 * huge holes still show up with zeroes where they need to be.
2a15efc9 3416 */
3ae77f43
HD
3417 if (absent && (flags & FOLL_DUMP) &&
3418 !hugetlbfs_pagecache_present(h, vma, vaddr)) {
cb900f41
KS
3419 if (pte)
3420 spin_unlock(ptl);
2a15efc9
HD
3421 remainder = 0;
3422 break;
3423 }
63551ae0 3424
9cc3a5bd
NH
3425 /*
3426 * We need call hugetlb_fault for both hugepages under migration
3427 * (in which case hugetlb_fault waits for the migration,) and
3428 * hwpoisoned hugepages (in which case we need to prevent the
3429 * caller from accessing to them.) In order to do this, we use
3430 * here is_swap_pte instead of is_hugetlb_entry_migration and
3431 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3432 * both cases, and because we can't follow correct pages
3433 * directly from any kind of swap entries.
3434 */
3435 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
106c992a
GS
3436 ((flags & FOLL_WRITE) &&
3437 !huge_pte_write(huge_ptep_get(pte)))) {
4c887265 3438 int ret;
63551ae0 3439
cb900f41
KS
3440 if (pte)
3441 spin_unlock(ptl);
2a15efc9
HD
3442 ret = hugetlb_fault(mm, vma, vaddr,
3443 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
a89182c7 3444 if (!(ret & VM_FAULT_ERROR))
4c887265 3445 continue;
63551ae0 3446
4c887265 3447 remainder = 0;
4c887265
AL
3448 break;
3449 }
3450
a5516438 3451 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
7f2e9525 3452 page = pte_page(huge_ptep_get(pte));
d5d4b0aa 3453same_page:
d6692183 3454 if (pages) {
2a15efc9 3455 pages[i] = mem_map_offset(page, pfn_offset);
a0368d4e 3456 get_page_foll(pages[i]);
d6692183 3457 }
63551ae0
DG
3458
3459 if (vmas)
3460 vmas[i] = vma;
3461
3462 vaddr += PAGE_SIZE;
d5d4b0aa 3463 ++pfn_offset;
63551ae0
DG
3464 --remainder;
3465 ++i;
d5d4b0aa 3466 if (vaddr < vma->vm_end && remainder &&
a5516438 3467 pfn_offset < pages_per_huge_page(h)) {
d5d4b0aa
CK
3468 /*
3469 * We use pfn_offset to avoid touching the pageframes
3470 * of this compound page.
3471 */
3472 goto same_page;
3473 }
cb900f41 3474 spin_unlock(ptl);
63551ae0 3475 }
28a35716 3476 *nr_pages = remainder;
63551ae0
DG
3477 *position = vaddr;
3478
2a15efc9 3479 return i ? i : -EFAULT;
63551ae0 3480}
8f860591 3481
7da4d641 3482unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
8f860591
ZY
3483 unsigned long address, unsigned long end, pgprot_t newprot)
3484{
3485 struct mm_struct *mm = vma->vm_mm;
3486 unsigned long start = address;
3487 pte_t *ptep;
3488 pte_t pte;
a5516438 3489 struct hstate *h = hstate_vma(vma);
7da4d641 3490 unsigned long pages = 0;
8f860591
ZY
3491
3492 BUG_ON(address >= end);
3493 flush_cache_range(vma, address, end);
3494
a5338093 3495 mmu_notifier_invalidate_range_start(mm, start, end);
83cde9e8 3496 i_mmap_lock_write(vma->vm_file->f_mapping);
a5516438 3497 for (; address < end; address += huge_page_size(h)) {
cb900f41 3498 spinlock_t *ptl;
8f860591
ZY
3499 ptep = huge_pte_offset(mm, address);
3500 if (!ptep)
3501 continue;
cb900f41 3502 ptl = huge_pte_lock(h, mm, ptep);
7da4d641
PZ
3503 if (huge_pmd_unshare(mm, &address, ptep)) {
3504 pages++;
cb900f41 3505 spin_unlock(ptl);
39dde65c 3506 continue;
7da4d641 3507 }
a8bda28d
NH
3508 pte = huge_ptep_get(ptep);
3509 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
3510 spin_unlock(ptl);
3511 continue;
3512 }
3513 if (unlikely(is_hugetlb_entry_migration(pte))) {
3514 swp_entry_t entry = pte_to_swp_entry(pte);
3515
3516 if (is_write_migration_entry(entry)) {
3517 pte_t newpte;
3518
3519 make_migration_entry_read(&entry);
3520 newpte = swp_entry_to_pte(entry);
3521 set_huge_pte_at(mm, address, ptep, newpte);
3522 pages++;
3523 }
3524 spin_unlock(ptl);
3525 continue;
3526 }
3527 if (!huge_pte_none(pte)) {
8f860591 3528 pte = huge_ptep_get_and_clear(mm, address, ptep);
106c992a 3529 pte = pte_mkhuge(huge_pte_modify(pte, newprot));
be7517d6 3530 pte = arch_make_huge_pte(pte, vma, NULL, 0);
8f860591 3531 set_huge_pte_at(mm, address, ptep, pte);
7da4d641 3532 pages++;
8f860591 3533 }
cb900f41 3534 spin_unlock(ptl);
8f860591 3535 }
d833352a 3536 /*
c8c06efa 3537 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
d833352a 3538 * may have cleared our pud entry and done put_page on the page table:
c8c06efa 3539 * once we release i_mmap_rwsem, another task can do the final put_page
d833352a
MG
3540 * and that page table be reused and filled with junk.
3541 */
8f860591 3542 flush_tlb_range(vma, start, end);
34ee645e 3543 mmu_notifier_invalidate_range(mm, start, end);
83cde9e8 3544 i_mmap_unlock_write(vma->vm_file->f_mapping);
a5338093 3545 mmu_notifier_invalidate_range_end(mm, start, end);
7da4d641
PZ
3546
3547 return pages << h->order;
8f860591
ZY
3548}
3549
a1e78772
MG
3550int hugetlb_reserve_pages(struct inode *inode,
3551 long from, long to,
5a6fe125 3552 struct vm_area_struct *vma,
ca16d140 3553 vm_flags_t vm_flags)
e4e574b7 3554{
17c9d12e 3555 long ret, chg;
a5516438 3556 struct hstate *h = hstate_inode(inode);
90481622 3557 struct hugepage_subpool *spool = subpool_inode(inode);
9119a41e 3558 struct resv_map *resv_map;
1c5ecae3 3559 long gbl_reserve;
e4e574b7 3560
17c9d12e
MG
3561 /*
3562 * Only apply hugepage reservation if asked. At fault time, an
3563 * attempt will be made for VM_NORESERVE to allocate a page
90481622 3564 * without using reserves
17c9d12e 3565 */
ca16d140 3566 if (vm_flags & VM_NORESERVE)
17c9d12e
MG
3567 return 0;
3568
a1e78772
MG
3569 /*
3570 * Shared mappings base their reservation on the number of pages that
3571 * are already allocated on behalf of the file. Private mappings need
3572 * to reserve the full area even if read-only as mprotect() may be
3573 * called to make the mapping read-write. Assume !vma is a shm mapping
3574 */
9119a41e 3575 if (!vma || vma->vm_flags & VM_MAYSHARE) {
4e35f483 3576 resv_map = inode_resv_map(inode);
9119a41e 3577
1406ec9b 3578 chg = region_chg(resv_map, from, to);
9119a41e
JK
3579
3580 } else {
3581 resv_map = resv_map_alloc();
17c9d12e
MG
3582 if (!resv_map)
3583 return -ENOMEM;
3584
a1e78772 3585 chg = to - from;
84afd99b 3586
17c9d12e
MG
3587 set_vma_resv_map(vma, resv_map);
3588 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3589 }
3590
c50ac050
DH
3591 if (chg < 0) {
3592 ret = chg;
3593 goto out_err;
3594 }
8a630112 3595
1c5ecae3
MK
3596 /*
3597 * There must be enough pages in the subpool for the mapping. If
3598 * the subpool has a minimum size, there may be some global
3599 * reservations already in place (gbl_reserve).
3600 */
3601 gbl_reserve = hugepage_subpool_get_pages(spool, chg);
3602 if (gbl_reserve < 0) {
c50ac050
DH
3603 ret = -ENOSPC;
3604 goto out_err;
3605 }
5a6fe125
MG
3606
3607 /*
17c9d12e 3608 * Check enough hugepages are available for the reservation.
90481622 3609 * Hand the pages back to the subpool if there are not
5a6fe125 3610 */
1c5ecae3 3611 ret = hugetlb_acct_memory(h, gbl_reserve);
68842c9b 3612 if (ret < 0) {
1c5ecae3
MK
3613 /* put back original number of pages, chg */
3614 (void)hugepage_subpool_put_pages(spool, chg);
c50ac050 3615 goto out_err;
68842c9b 3616 }
17c9d12e
MG
3617
3618 /*
3619 * Account for the reservations made. Shared mappings record regions
3620 * that have reservations as they are shared by multiple VMAs.
3621 * When the last VMA disappears, the region map says how much
3622 * the reservation was and the page cache tells how much of
3623 * the reservation was consumed. Private mappings are per-VMA and
3624 * only the consumed reservations are tracked. When the VMA
3625 * disappears, the original reservation is the VMA size and the
3626 * consumed reservations are stored in the map. Hence, nothing
3627 * else has to be done for private mappings here
3628 */
f83a275d 3629 if (!vma || vma->vm_flags & VM_MAYSHARE)
1406ec9b 3630 region_add(resv_map, from, to);
a43a8c39 3631 return 0;
c50ac050 3632out_err:
f031dd27
JK
3633 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3634 kref_put(&resv_map->refs, resv_map_release);
c50ac050 3635 return ret;
a43a8c39
CK
3636}
3637
3638void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3639{
a5516438 3640 struct hstate *h = hstate_inode(inode);
4e35f483 3641 struct resv_map *resv_map = inode_resv_map(inode);
9119a41e 3642 long chg = 0;
90481622 3643 struct hugepage_subpool *spool = subpool_inode(inode);
1c5ecae3 3644 long gbl_reserve;
45c682a6 3645
9119a41e 3646 if (resv_map)
1406ec9b 3647 chg = region_truncate(resv_map, offset);
45c682a6 3648 spin_lock(&inode->i_lock);
e4c6f8be 3649 inode->i_blocks -= (blocks_per_huge_page(h) * freed);
45c682a6
KC
3650 spin_unlock(&inode->i_lock);
3651
1c5ecae3
MK
3652 /*
3653 * If the subpool has a minimum size, the number of global
3654 * reservations to be released may be adjusted.
3655 */
3656 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
3657 hugetlb_acct_memory(h, -gbl_reserve);
a43a8c39 3658}
93f70f90 3659
3212b535
SC
3660#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3661static unsigned long page_table_shareable(struct vm_area_struct *svma,
3662 struct vm_area_struct *vma,
3663 unsigned long addr, pgoff_t idx)
3664{
3665 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3666 svma->vm_start;
3667 unsigned long sbase = saddr & PUD_MASK;
3668 unsigned long s_end = sbase + PUD_SIZE;
3669
3670 /* Allow segments to share if only one is marked locked */
3671 unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3672 unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3673
3674 /*
3675 * match the virtual addresses, permission and the alignment of the
3676 * page table page.
3677 */
3678 if (pmd_index(addr) != pmd_index(saddr) ||
3679 vm_flags != svm_flags ||
3680 sbase < svma->vm_start || svma->vm_end < s_end)
3681 return 0;
3682
3683 return saddr;
3684}
3685
3686static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3687{
3688 unsigned long base = addr & PUD_MASK;
3689 unsigned long end = base + PUD_SIZE;
3690
3691 /*
3692 * check on proper vm_flags and page table alignment
3693 */
3694 if (vma->vm_flags & VM_MAYSHARE &&
3695 vma->vm_start <= base && end <= vma->vm_end)
3696 return 1;
3697 return 0;
3698}
3699
3700/*
3701 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3702 * and returns the corresponding pte. While this is not necessary for the
3703 * !shared pmd case because we can allocate the pmd later as well, it makes the
3704 * code much cleaner. pmd allocation is essential for the shared case because
c8c06efa 3705 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
3212b535
SC
3706 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
3707 * bad pmd for sharing.
3708 */
3709pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3710{
3711 struct vm_area_struct *vma = find_vma(mm, addr);
3712 struct address_space *mapping = vma->vm_file->f_mapping;
3713 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3714 vma->vm_pgoff;
3715 struct vm_area_struct *svma;
3716 unsigned long saddr;
3717 pte_t *spte = NULL;
3718 pte_t *pte;
cb900f41 3719 spinlock_t *ptl;
3212b535
SC
3720
3721 if (!vma_shareable(vma, addr))
3722 return (pte_t *)pmd_alloc(mm, pud, addr);
3723
83cde9e8 3724 i_mmap_lock_write(mapping);
3212b535
SC
3725 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3726 if (svma == vma)
3727 continue;
3728
3729 saddr = page_table_shareable(svma, vma, addr, idx);
3730 if (saddr) {
3731 spte = huge_pte_offset(svma->vm_mm, saddr);
3732 if (spte) {
dc6c9a35 3733 mm_inc_nr_pmds(mm);
3212b535
SC
3734 get_page(virt_to_page(spte));
3735 break;
3736 }
3737 }
3738 }
3739
3740 if (!spte)
3741 goto out;
3742
cb900f41
KS
3743 ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
3744 spin_lock(ptl);
dc6c9a35 3745 if (pud_none(*pud)) {
3212b535
SC
3746 pud_populate(mm, pud,
3747 (pmd_t *)((unsigned long)spte & PAGE_MASK));
dc6c9a35 3748 } else {
3212b535 3749 put_page(virt_to_page(spte));
dc6c9a35
KS
3750 mm_inc_nr_pmds(mm);
3751 }
cb900f41 3752 spin_unlock(ptl);
3212b535
SC
3753out:
3754 pte = (pte_t *)pmd_alloc(mm, pud, addr);
83cde9e8 3755 i_mmap_unlock_write(mapping);
3212b535
SC
3756 return pte;
3757}
3758
3759/*
3760 * unmap huge page backed by shared pte.
3761 *
3762 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared
3763 * indicated by page_count > 1, unmap is achieved by clearing pud and
3764 * decrementing the ref count. If count == 1, the pte page is not shared.
3765 *
cb900f41 3766 * called with page table lock held.
3212b535
SC
3767 *
3768 * returns: 1 successfully unmapped a shared pte page
3769 * 0 the underlying pte page is not shared, or it is the last user
3770 */
3771int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
3772{
3773 pgd_t *pgd = pgd_offset(mm, *addr);
3774 pud_t *pud = pud_offset(pgd, *addr);
3775
3776 BUG_ON(page_count(virt_to_page(ptep)) == 0);
3777 if (page_count(virt_to_page(ptep)) == 1)
3778 return 0;
3779
3780 pud_clear(pud);
3781 put_page(virt_to_page(ptep));
dc6c9a35 3782 mm_dec_nr_pmds(mm);
3212b535
SC
3783 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3784 return 1;
3785}
9e5fc74c
SC
3786#define want_pmd_share() (1)
3787#else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3788pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3789{
3790 return NULL;
3791}
3792#define want_pmd_share() (0)
3212b535
SC
3793#endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3794
9e5fc74c
SC
3795#ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3796pte_t *huge_pte_alloc(struct mm_struct *mm,
3797 unsigned long addr, unsigned long sz)
3798{
3799 pgd_t *pgd;
3800 pud_t *pud;
3801 pte_t *pte = NULL;
3802
3803 pgd = pgd_offset(mm, addr);
3804 pud = pud_alloc(mm, pgd, addr);
3805 if (pud) {
3806 if (sz == PUD_SIZE) {
3807 pte = (pte_t *)pud;
3808 } else {
3809 BUG_ON(sz != PMD_SIZE);
3810 if (want_pmd_share() && pud_none(*pud))
3811 pte = huge_pmd_share(mm, addr, pud);
3812 else
3813 pte = (pte_t *)pmd_alloc(mm, pud, addr);
3814 }
3815 }
3816 BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
3817
3818 return pte;
3819}
3820
3821pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
3822{
3823 pgd_t *pgd;
3824 pud_t *pud;
3825 pmd_t *pmd = NULL;
3826
3827 pgd = pgd_offset(mm, addr);
3828 if (pgd_present(*pgd)) {
3829 pud = pud_offset(pgd, addr);
3830 if (pud_present(*pud)) {
3831 if (pud_huge(*pud))
3832 return (pte_t *)pud;
3833 pmd = pmd_offset(pud, addr);
3834 }
3835 }
3836 return (pte_t *) pmd;
3837}
3838
61f77eda
NH
3839#endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3840
3841/*
3842 * These functions are overwritable if your architecture needs its own
3843 * behavior.
3844 */
3845struct page * __weak
3846follow_huge_addr(struct mm_struct *mm, unsigned long address,
3847 int write)
3848{
3849 return ERR_PTR(-EINVAL);
3850}
3851
3852struct page * __weak
9e5fc74c 3853follow_huge_pmd(struct mm_struct *mm, unsigned long address,
e66f17ff 3854 pmd_t *pmd, int flags)
9e5fc74c 3855{
e66f17ff
NH
3856 struct page *page = NULL;
3857 spinlock_t *ptl;
3858retry:
3859 ptl = pmd_lockptr(mm, pmd);
3860 spin_lock(ptl);
3861 /*
3862 * make sure that the address range covered by this pmd is not
3863 * unmapped from other threads.
3864 */
3865 if (!pmd_huge(*pmd))
3866 goto out;
3867 if (pmd_present(*pmd)) {
97534127 3868 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
e66f17ff
NH
3869 if (flags & FOLL_GET)
3870 get_page(page);
3871 } else {
3872 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
3873 spin_unlock(ptl);
3874 __migration_entry_wait(mm, (pte_t *)pmd, ptl);
3875 goto retry;
3876 }
3877 /*
3878 * hwpoisoned entry is treated as no_page_table in
3879 * follow_page_mask().
3880 */
3881 }
3882out:
3883 spin_unlock(ptl);
9e5fc74c
SC
3884 return page;
3885}
3886
61f77eda 3887struct page * __weak
9e5fc74c 3888follow_huge_pud(struct mm_struct *mm, unsigned long address,
e66f17ff 3889 pud_t *pud, int flags)
9e5fc74c 3890{
e66f17ff
NH
3891 if (flags & FOLL_GET)
3892 return NULL;
9e5fc74c 3893
e66f17ff 3894 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
9e5fc74c
SC
3895}
3896
d5bd9106
AK
3897#ifdef CONFIG_MEMORY_FAILURE
3898
93f70f90
NH
3899/*
3900 * This function is called from memory failure code.
3901 * Assume the caller holds page lock of the head page.
3902 */
6de2b1aa 3903int dequeue_hwpoisoned_huge_page(struct page *hpage)
93f70f90
NH
3904{
3905 struct hstate *h = page_hstate(hpage);
3906 int nid = page_to_nid(hpage);
6de2b1aa 3907 int ret = -EBUSY;
93f70f90
NH
3908
3909 spin_lock(&hugetlb_lock);
7e1f049e
NH
3910 /*
3911 * Just checking !page_huge_active is not enough, because that could be
3912 * an isolated/hwpoisoned hugepage (which have >0 refcount).
3913 */
3914 if (!page_huge_active(hpage) && !page_count(hpage)) {
56f2fb14
NH
3915 /*
3916 * Hwpoisoned hugepage isn't linked to activelist or freelist,
3917 * but dangling hpage->lru can trigger list-debug warnings
3918 * (this happens when we call unpoison_memory() on it),
3919 * so let it point to itself with list_del_init().
3920 */
3921 list_del_init(&hpage->lru);
8c6c2ecb 3922 set_page_refcounted(hpage);
6de2b1aa
NH
3923 h->free_huge_pages--;
3924 h->free_huge_pages_node[nid]--;
3925 ret = 0;
3926 }
93f70f90 3927 spin_unlock(&hugetlb_lock);
6de2b1aa 3928 return ret;
93f70f90 3929}
6de2b1aa 3930#endif
31caf665
NH
3931
3932bool isolate_huge_page(struct page *page, struct list_head *list)
3933{
bcc54222
NH
3934 bool ret = true;
3935
309381fe 3936 VM_BUG_ON_PAGE(!PageHead(page), page);
31caf665 3937 spin_lock(&hugetlb_lock);
bcc54222
NH
3938 if (!page_huge_active(page) || !get_page_unless_zero(page)) {
3939 ret = false;
3940 goto unlock;
3941 }
3942 clear_page_huge_active(page);
31caf665 3943 list_move_tail(&page->lru, list);
bcc54222 3944unlock:
31caf665 3945 spin_unlock(&hugetlb_lock);
bcc54222 3946 return ret;
31caf665
NH
3947}
3948
3949void putback_active_hugepage(struct page *page)
3950{
309381fe 3951 VM_BUG_ON_PAGE(!PageHead(page), page);
31caf665 3952 spin_lock(&hugetlb_lock);
bcc54222 3953 set_page_huge_active(page);
31caf665
NH
3954 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3955 spin_unlock(&hugetlb_lock);
3956 put_page(page);
3957}