2 * Generic hugetlb support.
3 * (C) Nadia Yvette Chambers, April 2004
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/bootmem.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/rmap.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/page-isolation.h>
26 #include <linux/jhash.h>
29 #include <asm/pgtable.h>
33 #include <linux/hugetlb.h>
34 #include <linux/hugetlb_cgroup.h>
35 #include <linux/node.h>
38 int hugepages_treat_as_movable;
40 int hugetlb_max_hstate __read_mostly;
41 unsigned int default_hstate_idx;
42 struct hstate hstates[HUGE_MAX_HSTATE];
44 __initdata LIST_HEAD(huge_boot_pages);
46 /* for command line parsing */
47 static struct hstate * __initdata parsed_hstate;
48 static unsigned long __initdata default_hstate_max_huge_pages;
49 static unsigned long __initdata default_hstate_size;
52 * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
53 * free_huge_pages, and surplus_huge_pages.
55 DEFINE_SPINLOCK(hugetlb_lock);
58 * Serializes faults on the same logical page. This is used to
59 * prevent spurious OOMs when the hugepage pool is fully utilized.
61 static int num_fault_mutexes;
62 static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;
64 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
66 bool free = (spool->count == 0) && (spool->used_hpages == 0);
68 spin_unlock(&spool->lock);
70 /* If no pages are used, and no other handles to the subpool
71 * remain, free the subpool the subpool remain */
76 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
78 struct hugepage_subpool *spool;
80 spool = kzalloc(sizeof(*spool), GFP_KERNEL);
84 spin_lock_init(&spool->lock);
86 spool->max_hpages = nr_blocks;
91 void hugepage_put_subpool(struct hugepage_subpool *spool)
93 spin_lock(&spool->lock);
94 BUG_ON(!spool->count);
96 unlock_or_release_subpool(spool);
100 * Subpool accounting for allocating and reserving pages.
101 * Return -ENOMEM if there are not enough resources to satisfy the
102 * the request. Otherwise, return the number of pages by which the
103 * global pools must be adjusted (upward). The returned value may
104 * only be different than the passed value (delta) in the case where
105 * a subpool minimum size must be manitained.
107 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
115 spin_lock(&spool->lock);
117 if (spool->max_hpages != -1) { /* maximum size accounting */
118 if ((spool->used_hpages + delta) <= spool->max_hpages)
119 spool->used_hpages += delta;
126 if (spool->min_hpages != -1) { /* minimum size accounting */
127 if (delta > spool->rsv_hpages) {
129 * Asking for more reserves than those already taken on
130 * behalf of subpool. Return difference.
132 ret = delta - spool->rsv_hpages;
133 spool->rsv_hpages = 0;
135 ret = 0; /* reserves already accounted for */
136 spool->rsv_hpages -= delta;
141 spin_unlock(&spool->lock);
146 * Subpool accounting for freeing and unreserving pages.
147 * Return the number of global page reservations that must be dropped.
148 * The return value may only be different than the passed value (delta)
149 * in the case where a subpool minimum size must be maintained.
151 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
159 spin_lock(&spool->lock);
161 if (spool->max_hpages != -1) /* maximum size accounting */
162 spool->used_hpages -= delta;
164 if (spool->min_hpages != -1) { /* minimum size accounting */
165 if (spool->rsv_hpages + delta <= spool->min_hpages)
168 ret = spool->rsv_hpages + delta - spool->min_hpages;
170 spool->rsv_hpages += delta;
171 if (spool->rsv_hpages > spool->min_hpages)
172 spool->rsv_hpages = spool->min_hpages;
176 * If hugetlbfs_put_super couldn't free spool due to an outstanding
177 * quota reference, free it now.
179 unlock_or_release_subpool(spool);
184 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
186 return HUGETLBFS_SB(inode->i_sb)->spool;
189 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
191 return subpool_inode(file_inode(vma->vm_file));
195 * Region tracking -- allows tracking of reservations and instantiated pages
196 * across the pages in a mapping.
198 * The region data structures are embedded into a resv_map and
199 * protected by a resv_map's lock
202 struct list_head link;
207 static long region_add(struct resv_map *resv, long f, long t)
209 struct list_head *head = &resv->regions;
210 struct file_region *rg, *nrg, *trg;
212 spin_lock(&resv->lock);
213 /* Locate the region we are either in or before. */
214 list_for_each_entry(rg, head, link)
218 /* Round our left edge to the current segment if it encloses us. */
222 /* Check for and consume any regions we now overlap with. */
224 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
225 if (&rg->link == head)
230 /* If this area reaches higher then extend our area to
231 * include it completely. If this is not the first area
232 * which we intend to reuse, free it. */
242 spin_unlock(&resv->lock);
246 static long region_chg(struct resv_map *resv, long f, long t)
248 struct list_head *head = &resv->regions;
249 struct file_region *rg, *nrg = NULL;
253 spin_lock(&resv->lock);
254 /* Locate the region we are before or in. */
255 list_for_each_entry(rg, head, link)
259 /* If we are below the current region then a new region is required.
260 * Subtle, allocate a new region at the position but make it zero
261 * size such that we can guarantee to record the reservation. */
262 if (&rg->link == head || t < rg->from) {
264 spin_unlock(&resv->lock);
265 nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
271 INIT_LIST_HEAD(&nrg->link);
275 list_add(&nrg->link, rg->link.prev);
280 /* Round our left edge to the current segment if it encloses us. */
285 /* Check for and consume any regions we now overlap with. */
286 list_for_each_entry(rg, rg->link.prev, link) {
287 if (&rg->link == head)
292 /* We overlap with this area, if it extends further than
293 * us then we must extend ourselves. Account for its
294 * existing reservation. */
299 chg -= rg->to - rg->from;
303 spin_unlock(&resv->lock);
304 /* We already know we raced and no longer need the new region */
308 spin_unlock(&resv->lock);
312 static long region_truncate(struct resv_map *resv, long end)
314 struct list_head *head = &resv->regions;
315 struct file_region *rg, *trg;
318 spin_lock(&resv->lock);
319 /* Locate the region we are either in or before. */
320 list_for_each_entry(rg, head, link)
323 if (&rg->link == head)
326 /* If we are in the middle of a region then adjust it. */
327 if (end > rg->from) {
330 rg = list_entry(rg->link.next, typeof(*rg), link);
333 /* Drop any remaining regions. */
334 list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
335 if (&rg->link == head)
337 chg += rg->to - rg->from;
343 spin_unlock(&resv->lock);
347 static long region_count(struct resv_map *resv, long f, long t)
349 struct list_head *head = &resv->regions;
350 struct file_region *rg;
353 spin_lock(&resv->lock);
354 /* Locate each segment we overlap with, and count that overlap. */
355 list_for_each_entry(rg, head, link) {
364 seg_from = max(rg->from, f);
365 seg_to = min(rg->to, t);
367 chg += seg_to - seg_from;
369 spin_unlock(&resv->lock);
375 * Convert the address within this vma to the page offset within
376 * the mapping, in pagecache page units; huge pages here.
378 static pgoff_t vma_hugecache_offset(struct hstate *h,
379 struct vm_area_struct *vma, unsigned long address)
381 return ((address - vma->vm_start) >> huge_page_shift(h)) +
382 (vma->vm_pgoff >> huge_page_order(h));
385 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
386 unsigned long address)
388 return vma_hugecache_offset(hstate_vma(vma), vma, address);
392 * Return the size of the pages allocated when backing a VMA. In the majority
393 * cases this will be same size as used by the page table entries.
395 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
397 struct hstate *hstate;
399 if (!is_vm_hugetlb_page(vma))
402 hstate = hstate_vma(vma);
404 return 1UL << huge_page_shift(hstate);
406 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
409 * Return the page size being used by the MMU to back a VMA. In the majority
410 * of cases, the page size used by the kernel matches the MMU size. On
411 * architectures where it differs, an architecture-specific version of this
412 * function is required.
414 #ifndef vma_mmu_pagesize
415 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
417 return vma_kernel_pagesize(vma);
422 * Flags for MAP_PRIVATE reservations. These are stored in the bottom
423 * bits of the reservation map pointer, which are always clear due to
426 #define HPAGE_RESV_OWNER (1UL << 0)
427 #define HPAGE_RESV_UNMAPPED (1UL << 1)
428 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
431 * These helpers are used to track how many pages are reserved for
432 * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
433 * is guaranteed to have their future faults succeed.
435 * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
436 * the reserve counters are updated with the hugetlb_lock held. It is safe
437 * to reset the VMA at fork() time as it is not in use yet and there is no
438 * chance of the global counters getting corrupted as a result of the values.
440 * The private mapping reservation is represented in a subtly different
441 * manner to a shared mapping. A shared mapping has a region map associated
442 * with the underlying file, this region map represents the backing file
443 * pages which have ever had a reservation assigned which this persists even
444 * after the page is instantiated. A private mapping has a region map
445 * associated with the original mmap which is attached to all VMAs which
446 * reference it, this region map represents those offsets which have consumed
447 * reservation ie. where pages have been instantiated.
449 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
451 return (unsigned long)vma->vm_private_data;
454 static void set_vma_private_data(struct vm_area_struct *vma,
457 vma->vm_private_data = (void *)value;
460 struct resv_map *resv_map_alloc(void)
462 struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
466 kref_init(&resv_map->refs);
467 spin_lock_init(&resv_map->lock);
468 INIT_LIST_HEAD(&resv_map->regions);
473 void resv_map_release(struct kref *ref)
475 struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
477 /* Clear out any active regions before we release the map. */
478 region_truncate(resv_map, 0);
482 static inline struct resv_map *inode_resv_map(struct inode *inode)
484 return inode->i_mapping->private_data;
487 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
489 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
490 if (vma->vm_flags & VM_MAYSHARE) {
491 struct address_space *mapping = vma->vm_file->f_mapping;
492 struct inode *inode = mapping->host;
494 return inode_resv_map(inode);
497 return (struct resv_map *)(get_vma_private_data(vma) &
502 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
504 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
505 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
507 set_vma_private_data(vma, (get_vma_private_data(vma) &
508 HPAGE_RESV_MASK) | (unsigned long)map);
511 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
513 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
514 VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
516 set_vma_private_data(vma, get_vma_private_data(vma) | flags);
519 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
521 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
523 return (get_vma_private_data(vma) & flag) != 0;
526 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
527 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
529 VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
530 if (!(vma->vm_flags & VM_MAYSHARE))
531 vma->vm_private_data = (void *)0;
534 /* Returns true if the VMA has associated reserve pages */
535 static int vma_has_reserves(struct vm_area_struct *vma, long chg)
537 if (vma->vm_flags & VM_NORESERVE) {
539 * This address is already reserved by other process(chg == 0),
540 * so, we should decrement reserved count. Without decrementing,
541 * reserve count remains after releasing inode, because this
542 * allocated page will go into page cache and is regarded as
543 * coming from reserved pool in releasing step. Currently, we
544 * don't have any other solution to deal with this situation
545 * properly, so add work-around here.
547 if (vma->vm_flags & VM_MAYSHARE && chg == 0)
553 /* Shared mappings always use reserves */
554 if (vma->vm_flags & VM_MAYSHARE)
558 * Only the process that called mmap() has reserves for
561 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
567 static void enqueue_huge_page(struct hstate *h, struct page *page)
569 int nid = page_to_nid(page);
570 list_move(&page->lru, &h->hugepage_freelists[nid]);
571 h->free_huge_pages++;
572 h->free_huge_pages_node[nid]++;
575 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
579 list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
580 if (!is_migrate_isolate_page(page))
583 * if 'non-isolated free hugepage' not found on the list,
584 * the allocation fails.
586 if (&h->hugepage_freelists[nid] == &page->lru)
588 list_move(&page->lru, &h->hugepage_activelist);
589 set_page_refcounted(page);
590 h->free_huge_pages--;
591 h->free_huge_pages_node[nid]--;
595 /* Movability of hugepages depends on migration support. */
596 static inline gfp_t htlb_alloc_mask(struct hstate *h)
598 if (hugepages_treat_as_movable || hugepage_migration_supported(h))
599 return GFP_HIGHUSER_MOVABLE;
604 static struct page *dequeue_huge_page_vma(struct hstate *h,
605 struct vm_area_struct *vma,
606 unsigned long address, int avoid_reserve,
609 struct page *page = NULL;
610 struct mempolicy *mpol;
611 nodemask_t *nodemask;
612 struct zonelist *zonelist;
615 unsigned int cpuset_mems_cookie;
618 * A child process with MAP_PRIVATE mappings created by their parent
619 * have no page reserves. This check ensures that reservations are
620 * not "stolen". The child may still get SIGKILLed
622 if (!vma_has_reserves(vma, chg) &&
623 h->free_huge_pages - h->resv_huge_pages == 0)
626 /* If reserves cannot be used, ensure enough pages are in the pool */
627 if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
631 cpuset_mems_cookie = read_mems_allowed_begin();
632 zonelist = huge_zonelist(vma, address,
633 htlb_alloc_mask(h), &mpol, &nodemask);
635 for_each_zone_zonelist_nodemask(zone, z, zonelist,
636 MAX_NR_ZONES - 1, nodemask) {
637 if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
638 page = dequeue_huge_page_node(h, zone_to_nid(zone));
642 if (!vma_has_reserves(vma, chg))
645 SetPagePrivate(page);
646 h->resv_huge_pages--;
653 if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
662 * common helper functions for hstate_next_node_to_{alloc|free}.
663 * We may have allocated or freed a huge page based on a different
664 * nodes_allowed previously, so h->next_node_to_{alloc|free} might
665 * be outside of *nodes_allowed. Ensure that we use an allowed
666 * node for alloc or free.
668 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
670 nid = next_node(nid, *nodes_allowed);
671 if (nid == MAX_NUMNODES)
672 nid = first_node(*nodes_allowed);
673 VM_BUG_ON(nid >= MAX_NUMNODES);
678 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
680 if (!node_isset(nid, *nodes_allowed))
681 nid = next_node_allowed(nid, nodes_allowed);
686 * returns the previously saved node ["this node"] from which to
687 * allocate a persistent huge page for the pool and advance the
688 * next node from which to allocate, handling wrap at end of node
691 static int hstate_next_node_to_alloc(struct hstate *h,
692 nodemask_t *nodes_allowed)
696 VM_BUG_ON(!nodes_allowed);
698 nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
699 h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
705 * helper for free_pool_huge_page() - return the previously saved
706 * node ["this node"] from which to free a huge page. Advance the
707 * next node id whether or not we find a free huge page to free so
708 * that the next attempt to free addresses the next node.
710 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
714 VM_BUG_ON(!nodes_allowed);
716 nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
717 h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
722 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask) \
723 for (nr_nodes = nodes_weight(*mask); \
725 ((node = hstate_next_node_to_alloc(hs, mask)) || 1); \
728 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask) \
729 for (nr_nodes = nodes_weight(*mask); \
731 ((node = hstate_next_node_to_free(hs, mask)) || 1); \
734 #if defined(CONFIG_CMA) && defined(CONFIG_X86_64)
735 static void destroy_compound_gigantic_page(struct page *page,
739 int nr_pages = 1 << order;
740 struct page *p = page + 1;
742 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
744 set_page_refcounted(p);
745 p->first_page = NULL;
748 set_compound_order(page, 0);
749 __ClearPageHead(page);
752 static void free_gigantic_page(struct page *page, unsigned order)
754 free_contig_range(page_to_pfn(page), 1 << order);
757 static int __alloc_gigantic_page(unsigned long start_pfn,
758 unsigned long nr_pages)
760 unsigned long end_pfn = start_pfn + nr_pages;
761 return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
764 static bool pfn_range_valid_gigantic(unsigned long start_pfn,
765 unsigned long nr_pages)
767 unsigned long i, end_pfn = start_pfn + nr_pages;
770 for (i = start_pfn; i < end_pfn; i++) {
774 page = pfn_to_page(i);
776 if (PageReserved(page))
779 if (page_count(page) > 0)
789 static bool zone_spans_last_pfn(const struct zone *zone,
790 unsigned long start_pfn, unsigned long nr_pages)
792 unsigned long last_pfn = start_pfn + nr_pages - 1;
793 return zone_spans_pfn(zone, last_pfn);
796 static struct page *alloc_gigantic_page(int nid, unsigned order)
798 unsigned long nr_pages = 1 << order;
799 unsigned long ret, pfn, flags;
802 z = NODE_DATA(nid)->node_zones;
803 for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
804 spin_lock_irqsave(&z->lock, flags);
806 pfn = ALIGN(z->zone_start_pfn, nr_pages);
807 while (zone_spans_last_pfn(z, pfn, nr_pages)) {
808 if (pfn_range_valid_gigantic(pfn, nr_pages)) {
810 * We release the zone lock here because
811 * alloc_contig_range() will also lock the zone
812 * at some point. If there's an allocation
813 * spinning on this lock, it may win the race
814 * and cause alloc_contig_range() to fail...
816 spin_unlock_irqrestore(&z->lock, flags);
817 ret = __alloc_gigantic_page(pfn, nr_pages);
819 return pfn_to_page(pfn);
820 spin_lock_irqsave(&z->lock, flags);
825 spin_unlock_irqrestore(&z->lock, flags);
831 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
832 static void prep_compound_gigantic_page(struct page *page, unsigned long order);
834 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
838 page = alloc_gigantic_page(nid, huge_page_order(h));
840 prep_compound_gigantic_page(page, huge_page_order(h));
841 prep_new_huge_page(h, page, nid);
847 static int alloc_fresh_gigantic_page(struct hstate *h,
848 nodemask_t *nodes_allowed)
850 struct page *page = NULL;
853 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
854 page = alloc_fresh_gigantic_page_node(h, node);
862 static inline bool gigantic_page_supported(void) { return true; }
864 static inline bool gigantic_page_supported(void) { return false; }
865 static inline void free_gigantic_page(struct page *page, unsigned order) { }
866 static inline void destroy_compound_gigantic_page(struct page *page,
867 unsigned long order) { }
868 static inline int alloc_fresh_gigantic_page(struct hstate *h,
869 nodemask_t *nodes_allowed) { return 0; }
872 static void update_and_free_page(struct hstate *h, struct page *page)
876 if (hstate_is_gigantic(h) && !gigantic_page_supported())
880 h->nr_huge_pages_node[page_to_nid(page)]--;
881 for (i = 0; i < pages_per_huge_page(h); i++) {
882 page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
883 1 << PG_referenced | 1 << PG_dirty |
884 1 << PG_active | 1 << PG_private |
887 VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
888 set_compound_page_dtor(page, NULL);
889 set_page_refcounted(page);
890 if (hstate_is_gigantic(h)) {
891 destroy_compound_gigantic_page(page, huge_page_order(h));
892 free_gigantic_page(page, huge_page_order(h));
894 arch_release_hugepage(page);
895 __free_pages(page, huge_page_order(h));
899 struct hstate *size_to_hstate(unsigned long size)
904 if (huge_page_size(h) == size)
910 void free_huge_page(struct page *page)
913 * Can't pass hstate in here because it is called from the
914 * compound page destructor.
916 struct hstate *h = page_hstate(page);
917 int nid = page_to_nid(page);
918 struct hugepage_subpool *spool =
919 (struct hugepage_subpool *)page_private(page);
920 bool restore_reserve;
922 set_page_private(page, 0);
923 page->mapping = NULL;
924 BUG_ON(page_count(page));
925 BUG_ON(page_mapcount(page));
926 restore_reserve = PagePrivate(page);
927 ClearPagePrivate(page);
930 * A return code of zero implies that the subpool will be under its
931 * minimum size if the reservation is not restored after page is free.
932 * Therefore, force restore_reserve operation.
934 if (hugepage_subpool_put_pages(spool, 1) == 0)
935 restore_reserve = true;
937 spin_lock(&hugetlb_lock);
938 hugetlb_cgroup_uncharge_page(hstate_index(h),
939 pages_per_huge_page(h), page);
941 h->resv_huge_pages++;
943 if (h->surplus_huge_pages_node[nid]) {
944 /* remove the page from active list */
945 list_del(&page->lru);
946 update_and_free_page(h, page);
947 h->surplus_huge_pages--;
948 h->surplus_huge_pages_node[nid]--;
950 arch_clear_hugepage_flags(page);
951 enqueue_huge_page(h, page);
953 spin_unlock(&hugetlb_lock);
956 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
958 INIT_LIST_HEAD(&page->lru);
959 set_compound_page_dtor(page, free_huge_page);
960 spin_lock(&hugetlb_lock);
961 set_hugetlb_cgroup(page, NULL);
963 h->nr_huge_pages_node[nid]++;
964 spin_unlock(&hugetlb_lock);
965 put_page(page); /* free it into the hugepage allocator */
968 static void prep_compound_gigantic_page(struct page *page, unsigned long order)
971 int nr_pages = 1 << order;
972 struct page *p = page + 1;
974 /* we rely on prep_new_huge_page to set the destructor */
975 set_compound_order(page, order);
977 __ClearPageReserved(page);
978 for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
980 * For gigantic hugepages allocated through bootmem at
981 * boot, it's safer to be consistent with the not-gigantic
982 * hugepages and clear the PG_reserved bit from all tail pages
983 * too. Otherwse drivers using get_user_pages() to access tail
984 * pages may get the reference counting wrong if they see
985 * PG_reserved set on a tail page (despite the head page not
986 * having PG_reserved set). Enforcing this consistency between
987 * head and tail pages allows drivers to optimize away a check
988 * on the head page when they need know if put_page() is needed
989 * after get_user_pages().
991 __ClearPageReserved(p);
992 set_page_count(p, 0);
993 p->first_page = page;
994 /* Make sure p->first_page is always valid for PageTail() */
1001 * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1002 * transparent huge pages. See the PageTransHuge() documentation for more
1005 int PageHuge(struct page *page)
1007 if (!PageCompound(page))
1010 page = compound_head(page);
1011 return get_compound_page_dtor(page) == free_huge_page;
1013 EXPORT_SYMBOL_GPL(PageHuge);
1016 * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1017 * normal or transparent huge pages.
1019 int PageHeadHuge(struct page *page_head)
1021 if (!PageHead(page_head))
1024 return get_compound_page_dtor(page_head) == free_huge_page;
1027 pgoff_t __basepage_index(struct page *page)
1029 struct page *page_head = compound_head(page);
1030 pgoff_t index = page_index(page_head);
1031 unsigned long compound_idx;
1033 if (!PageHuge(page_head))
1034 return page_index(page);
1036 if (compound_order(page_head) >= MAX_ORDER)
1037 compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1039 compound_idx = page - page_head;
1041 return (index << compound_order(page_head)) + compound_idx;
1044 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1048 page = alloc_pages_exact_node(nid,
1049 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1050 __GFP_REPEAT|__GFP_NOWARN,
1051 huge_page_order(h));
1053 if (arch_prepare_hugepage(page)) {
1054 __free_pages(page, huge_page_order(h));
1057 prep_new_huge_page(h, page, nid);
1063 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1069 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1070 page = alloc_fresh_huge_page_node(h, node);
1078 count_vm_event(HTLB_BUDDY_PGALLOC);
1080 count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1086 * Free huge page from pool from next node to free.
1087 * Attempt to keep persistent huge pages more or less
1088 * balanced over allowed nodes.
1089 * Called with hugetlb_lock locked.
1091 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1097 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1099 * If we're returning unused surplus pages, only examine
1100 * nodes with surplus pages.
1102 if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1103 !list_empty(&h->hugepage_freelists[node])) {
1105 list_entry(h->hugepage_freelists[node].next,
1107 list_del(&page->lru);
1108 h->free_huge_pages--;
1109 h->free_huge_pages_node[node]--;
1111 h->surplus_huge_pages--;
1112 h->surplus_huge_pages_node[node]--;
1114 update_and_free_page(h, page);
1124 * Dissolve a given free hugepage into free buddy pages. This function does
1125 * nothing for in-use (including surplus) hugepages.
1127 static void dissolve_free_huge_page(struct page *page)
1129 spin_lock(&hugetlb_lock);
1130 if (PageHuge(page) && !page_count(page)) {
1131 struct hstate *h = page_hstate(page);
1132 int nid = page_to_nid(page);
1133 list_del(&page->lru);
1134 h->free_huge_pages--;
1135 h->free_huge_pages_node[nid]--;
1136 update_and_free_page(h, page);
1138 spin_unlock(&hugetlb_lock);
1142 * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1143 * make specified memory blocks removable from the system.
1144 * Note that start_pfn should aligned with (minimum) hugepage size.
1146 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1148 unsigned int order = 8 * sizeof(void *);
1152 if (!hugepages_supported())
1155 /* Set scan step to minimum hugepage size */
1157 if (order > huge_page_order(h))
1158 order = huge_page_order(h);
1159 VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
1160 for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
1161 dissolve_free_huge_page(pfn_to_page(pfn));
1164 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
1169 if (hstate_is_gigantic(h))
1173 * Assume we will successfully allocate the surplus page to
1174 * prevent racing processes from causing the surplus to exceed
1177 * This however introduces a different race, where a process B
1178 * tries to grow the static hugepage pool while alloc_pages() is
1179 * called by process A. B will only examine the per-node
1180 * counters in determining if surplus huge pages can be
1181 * converted to normal huge pages in adjust_pool_surplus(). A
1182 * won't be able to increment the per-node counter, until the
1183 * lock is dropped by B, but B doesn't drop hugetlb_lock until
1184 * no more huge pages can be converted from surplus to normal
1185 * state (and doesn't try to convert again). Thus, we have a
1186 * case where a surplus huge page exists, the pool is grown, and
1187 * the surplus huge page still exists after, even though it
1188 * should just have been converted to a normal huge page. This
1189 * does not leak memory, though, as the hugepage will be freed
1190 * once it is out of use. It also does not allow the counters to
1191 * go out of whack in adjust_pool_surplus() as we don't modify
1192 * the node values until we've gotten the hugepage and only the
1193 * per-node value is checked there.
1195 spin_lock(&hugetlb_lock);
1196 if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1197 spin_unlock(&hugetlb_lock);
1201 h->surplus_huge_pages++;
1203 spin_unlock(&hugetlb_lock);
1205 if (nid == NUMA_NO_NODE)
1206 page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
1207 __GFP_REPEAT|__GFP_NOWARN,
1208 huge_page_order(h));
1210 page = alloc_pages_exact_node(nid,
1211 htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1212 __GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
1214 if (page && arch_prepare_hugepage(page)) {
1215 __free_pages(page, huge_page_order(h));
1219 spin_lock(&hugetlb_lock);
1221 INIT_LIST_HEAD(&page->lru);
1222 r_nid = page_to_nid(page);
1223 set_compound_page_dtor(page, free_huge_page);
1224 set_hugetlb_cgroup(page, NULL);
1226 * We incremented the global counters already
1228 h->nr_huge_pages_node[r_nid]++;
1229 h->surplus_huge_pages_node[r_nid]++;
1230 __count_vm_event(HTLB_BUDDY_PGALLOC);
1233 h->surplus_huge_pages--;
1234 __count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1236 spin_unlock(&hugetlb_lock);
1242 * This allocation function is useful in the context where vma is irrelevant.
1243 * E.g. soft-offlining uses this function because it only cares physical
1244 * address of error page.
1246 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1248 struct page *page = NULL;
1250 spin_lock(&hugetlb_lock);
1251 if (h->free_huge_pages - h->resv_huge_pages > 0)
1252 page = dequeue_huge_page_node(h, nid);
1253 spin_unlock(&hugetlb_lock);
1256 page = alloc_buddy_huge_page(h, nid);
1262 * Increase the hugetlb pool such that it can accommodate a reservation
1265 static int gather_surplus_pages(struct hstate *h, int delta)
1267 struct list_head surplus_list;
1268 struct page *page, *tmp;
1270 int needed, allocated;
1271 bool alloc_ok = true;
1273 needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1275 h->resv_huge_pages += delta;
1280 INIT_LIST_HEAD(&surplus_list);
1284 spin_unlock(&hugetlb_lock);
1285 for (i = 0; i < needed; i++) {
1286 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1291 list_add(&page->lru, &surplus_list);
1296 * After retaking hugetlb_lock, we need to recalculate 'needed'
1297 * because either resv_huge_pages or free_huge_pages may have changed.
1299 spin_lock(&hugetlb_lock);
1300 needed = (h->resv_huge_pages + delta) -
1301 (h->free_huge_pages + allocated);
1306 * We were not able to allocate enough pages to
1307 * satisfy the entire reservation so we free what
1308 * we've allocated so far.
1313 * The surplus_list now contains _at_least_ the number of extra pages
1314 * needed to accommodate the reservation. Add the appropriate number
1315 * of pages to the hugetlb pool and free the extras back to the buddy
1316 * allocator. Commit the entire reservation here to prevent another
1317 * process from stealing the pages as they are added to the pool but
1318 * before they are reserved.
1320 needed += allocated;
1321 h->resv_huge_pages += delta;
1324 /* Free the needed pages to the hugetlb pool */
1325 list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1329 * This page is now managed by the hugetlb allocator and has
1330 * no users -- drop the buddy allocator's reference.
1332 put_page_testzero(page);
1333 VM_BUG_ON_PAGE(page_count(page), page);
1334 enqueue_huge_page(h, page);
1337 spin_unlock(&hugetlb_lock);
1339 /* Free unnecessary surplus pages to the buddy allocator */
1340 list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1342 spin_lock(&hugetlb_lock);
1348 * When releasing a hugetlb pool reservation, any surplus pages that were
1349 * allocated to satisfy the reservation must be explicitly freed if they were
1351 * Called with hugetlb_lock held.
1353 static void return_unused_surplus_pages(struct hstate *h,
1354 unsigned long unused_resv_pages)
1356 unsigned long nr_pages;
1358 /* Uncommit the reservation */
1359 h->resv_huge_pages -= unused_resv_pages;
1361 /* Cannot return gigantic pages currently */
1362 if (hstate_is_gigantic(h))
1365 nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1368 * We want to release as many surplus pages as possible, spread
1369 * evenly across all nodes with memory. Iterate across these nodes
1370 * until we can no longer free unreserved surplus pages. This occurs
1371 * when the nodes with surplus pages have no free pages.
1372 * free_pool_huge_page() will balance the the freed pages across the
1373 * on-line nodes with memory and will handle the hstate accounting.
1375 while (nr_pages--) {
1376 if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1378 cond_resched_lock(&hugetlb_lock);
1383 * Determine if the huge page at addr within the vma has an associated
1384 * reservation. Where it does not we will need to logically increase
1385 * reservation and actually increase subpool usage before an allocation
1386 * can occur. Where any new reservation would be required the
1387 * reservation change is prepared, but not committed. Once the page
1388 * has been allocated from the subpool and instantiated the change should
1389 * be committed via vma_commit_reservation. No action is required on
1392 static long vma_needs_reservation(struct hstate *h,
1393 struct vm_area_struct *vma, unsigned long addr)
1395 struct resv_map *resv;
1399 resv = vma_resv_map(vma);
1403 idx = vma_hugecache_offset(h, vma, addr);
1404 chg = region_chg(resv, idx, idx + 1);
1406 if (vma->vm_flags & VM_MAYSHARE)
1409 return chg < 0 ? chg : 0;
1411 static void vma_commit_reservation(struct hstate *h,
1412 struct vm_area_struct *vma, unsigned long addr)
1414 struct resv_map *resv;
1417 resv = vma_resv_map(vma);
1421 idx = vma_hugecache_offset(h, vma, addr);
1422 region_add(resv, idx, idx + 1);
1425 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1426 unsigned long addr, int avoid_reserve)
1428 struct hugepage_subpool *spool = subpool_vma(vma);
1429 struct hstate *h = hstate_vma(vma);
1433 struct hugetlb_cgroup *h_cg;
1435 idx = hstate_index(h);
1437 * Processes that did not create the mapping will have no
1438 * reserves and will not have accounted against subpool
1439 * limit. Check that the subpool limit can be made before
1440 * satisfying the allocation MAP_NORESERVE mappings may also
1441 * need pages and subpool limit allocated allocated if no reserve
1444 chg = vma_needs_reservation(h, vma, addr);
1446 return ERR_PTR(-ENOMEM);
1447 if (chg || avoid_reserve)
1448 if (hugepage_subpool_get_pages(spool, 1) < 0)
1449 return ERR_PTR(-ENOSPC);
1451 ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1453 goto out_subpool_put;
1455 spin_lock(&hugetlb_lock);
1456 page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
1458 spin_unlock(&hugetlb_lock);
1459 page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1461 goto out_uncharge_cgroup;
1463 spin_lock(&hugetlb_lock);
1464 list_move(&page->lru, &h->hugepage_activelist);
1467 hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1468 spin_unlock(&hugetlb_lock);
1470 set_page_private(page, (unsigned long)spool);
1472 vma_commit_reservation(h, vma, addr);
1475 out_uncharge_cgroup:
1476 hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1478 if (chg || avoid_reserve)
1479 hugepage_subpool_put_pages(spool, 1);
1480 return ERR_PTR(-ENOSPC);
1484 * alloc_huge_page()'s wrapper which simply returns the page if allocation
1485 * succeeds, otherwise NULL. This function is called from new_vma_page(),
1486 * where no ERR_VALUE is expected to be returned.
1488 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1489 unsigned long addr, int avoid_reserve)
1491 struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1497 int __weak alloc_bootmem_huge_page(struct hstate *h)
1499 struct huge_bootmem_page *m;
1502 for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1505 addr = memblock_virt_alloc_try_nid_nopanic(
1506 huge_page_size(h), huge_page_size(h),
1507 0, BOOTMEM_ALLOC_ACCESSIBLE, node);
1510 * Use the beginning of the huge page to store the
1511 * huge_bootmem_page struct (until gather_bootmem
1512 * puts them into the mem_map).
1521 BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
1522 /* Put them into a private list first because mem_map is not up yet */
1523 list_add(&m->list, &huge_boot_pages);
1528 static void __init prep_compound_huge_page(struct page *page, int order)
1530 if (unlikely(order > (MAX_ORDER - 1)))
1531 prep_compound_gigantic_page(page, order);
1533 prep_compound_page(page, order);
1536 /* Put bootmem huge pages into the standard lists after mem_map is up */
1537 static void __init gather_bootmem_prealloc(void)
1539 struct huge_bootmem_page *m;
1541 list_for_each_entry(m, &huge_boot_pages, list) {
1542 struct hstate *h = m->hstate;
1545 #ifdef CONFIG_HIGHMEM
1546 page = pfn_to_page(m->phys >> PAGE_SHIFT);
1547 memblock_free_late(__pa(m),
1548 sizeof(struct huge_bootmem_page));
1550 page = virt_to_page(m);
1552 WARN_ON(page_count(page) != 1);
1553 prep_compound_huge_page(page, h->order);
1554 WARN_ON(PageReserved(page));
1555 prep_new_huge_page(h, page, page_to_nid(page));
1557 * If we had gigantic hugepages allocated at boot time, we need
1558 * to restore the 'stolen' pages to totalram_pages in order to
1559 * fix confusing memory reports from free(1) and another
1560 * side-effects, like CommitLimit going negative.
1562 if (hstate_is_gigantic(h))
1563 adjust_managed_page_count(page, 1 << h->order);
1567 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1571 for (i = 0; i < h->max_huge_pages; ++i) {
1572 if (hstate_is_gigantic(h)) {
1573 if (!alloc_bootmem_huge_page(h))
1575 } else if (!alloc_fresh_huge_page(h,
1576 &node_states[N_MEMORY]))
1579 h->max_huge_pages = i;
1582 static void __init hugetlb_init_hstates(void)
1586 for_each_hstate(h) {
1587 /* oversize hugepages were init'ed in early boot */
1588 if (!hstate_is_gigantic(h))
1589 hugetlb_hstate_alloc_pages(h);
1593 static char * __init memfmt(char *buf, unsigned long n)
1595 if (n >= (1UL << 30))
1596 sprintf(buf, "%lu GB", n >> 30);
1597 else if (n >= (1UL << 20))
1598 sprintf(buf, "%lu MB", n >> 20);
1600 sprintf(buf, "%lu KB", n >> 10);
1604 static void __init report_hugepages(void)
1608 for_each_hstate(h) {
1610 pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1611 memfmt(buf, huge_page_size(h)),
1612 h->free_huge_pages);
1616 #ifdef CONFIG_HIGHMEM
1617 static void try_to_free_low(struct hstate *h, unsigned long count,
1618 nodemask_t *nodes_allowed)
1622 if (hstate_is_gigantic(h))
1625 for_each_node_mask(i, *nodes_allowed) {
1626 struct page *page, *next;
1627 struct list_head *freel = &h->hugepage_freelists[i];
1628 list_for_each_entry_safe(page, next, freel, lru) {
1629 if (count >= h->nr_huge_pages)
1631 if (PageHighMem(page))
1633 list_del(&page->lru);
1634 update_and_free_page(h, page);
1635 h->free_huge_pages--;
1636 h->free_huge_pages_node[page_to_nid(page)]--;
1641 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1642 nodemask_t *nodes_allowed)
1648 * Increment or decrement surplus_huge_pages. Keep node-specific counters
1649 * balanced by operating on them in a round-robin fashion.
1650 * Returns 1 if an adjustment was made.
1652 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1657 VM_BUG_ON(delta != -1 && delta != 1);
1660 for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1661 if (h->surplus_huge_pages_node[node])
1665 for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1666 if (h->surplus_huge_pages_node[node] <
1667 h->nr_huge_pages_node[node])
1674 h->surplus_huge_pages += delta;
1675 h->surplus_huge_pages_node[node] += delta;
1679 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1680 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1681 nodemask_t *nodes_allowed)
1683 unsigned long min_count, ret;
1685 if (hstate_is_gigantic(h) && !gigantic_page_supported())
1686 return h->max_huge_pages;
1689 * Increase the pool size
1690 * First take pages out of surplus state. Then make up the
1691 * remaining difference by allocating fresh huge pages.
1693 * We might race with alloc_buddy_huge_page() here and be unable
1694 * to convert a surplus huge page to a normal huge page. That is
1695 * not critical, though, it just means the overall size of the
1696 * pool might be one hugepage larger than it needs to be, but
1697 * within all the constraints specified by the sysctls.
1699 spin_lock(&hugetlb_lock);
1700 while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1701 if (!adjust_pool_surplus(h, nodes_allowed, -1))
1705 while (count > persistent_huge_pages(h)) {
1707 * If this allocation races such that we no longer need the
1708 * page, free_huge_page will handle it by freeing the page
1709 * and reducing the surplus.
1711 spin_unlock(&hugetlb_lock);
1712 if (hstate_is_gigantic(h))
1713 ret = alloc_fresh_gigantic_page(h, nodes_allowed);
1715 ret = alloc_fresh_huge_page(h, nodes_allowed);
1716 spin_lock(&hugetlb_lock);
1720 /* Bail for signals. Probably ctrl-c from user */
1721 if (signal_pending(current))
1726 * Decrease the pool size
1727 * First return free pages to the buddy allocator (being careful
1728 * to keep enough around to satisfy reservations). Then place
1729 * pages into surplus state as needed so the pool will shrink
1730 * to the desired size as pages become free.
1732 * By placing pages into the surplus state independent of the
1733 * overcommit value, we are allowing the surplus pool size to
1734 * exceed overcommit. There are few sane options here. Since
1735 * alloc_buddy_huge_page() is checking the global counter,
1736 * though, we'll note that we're not allowed to exceed surplus
1737 * and won't grow the pool anywhere else. Not until one of the
1738 * sysctls are changed, or the surplus pages go out of use.
1740 min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1741 min_count = max(count, min_count);
1742 try_to_free_low(h, min_count, nodes_allowed);
1743 while (min_count < persistent_huge_pages(h)) {
1744 if (!free_pool_huge_page(h, nodes_allowed, 0))
1746 cond_resched_lock(&hugetlb_lock);
1748 while (count < persistent_huge_pages(h)) {
1749 if (!adjust_pool_surplus(h, nodes_allowed, 1))
1753 ret = persistent_huge_pages(h);
1754 spin_unlock(&hugetlb_lock);
1758 #define HSTATE_ATTR_RO(_name) \
1759 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1761 #define HSTATE_ATTR(_name) \
1762 static struct kobj_attribute _name##_attr = \
1763 __ATTR(_name, 0644, _name##_show, _name##_store)
1765 static struct kobject *hugepages_kobj;
1766 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1768 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1770 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1774 for (i = 0; i < HUGE_MAX_HSTATE; i++)
1775 if (hstate_kobjs[i] == kobj) {
1777 *nidp = NUMA_NO_NODE;
1781 return kobj_to_node_hstate(kobj, nidp);
1784 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1785 struct kobj_attribute *attr, char *buf)
1788 unsigned long nr_huge_pages;
1791 h = kobj_to_hstate(kobj, &nid);
1792 if (nid == NUMA_NO_NODE)
1793 nr_huge_pages = h->nr_huge_pages;
1795 nr_huge_pages = h->nr_huge_pages_node[nid];
1797 return sprintf(buf, "%lu\n", nr_huge_pages);
1800 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
1801 struct hstate *h, int nid,
1802 unsigned long count, size_t len)
1805 NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1807 if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
1812 if (nid == NUMA_NO_NODE) {
1814 * global hstate attribute
1816 if (!(obey_mempolicy &&
1817 init_nodemask_of_mempolicy(nodes_allowed))) {
1818 NODEMASK_FREE(nodes_allowed);
1819 nodes_allowed = &node_states[N_MEMORY];
1821 } else if (nodes_allowed) {
1823 * per node hstate attribute: adjust count to global,
1824 * but restrict alloc/free to the specified node.
1826 count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1827 init_nodemask_of_node(nodes_allowed, nid);
1829 nodes_allowed = &node_states[N_MEMORY];
1831 h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1833 if (nodes_allowed != &node_states[N_MEMORY])
1834 NODEMASK_FREE(nodes_allowed);
1838 NODEMASK_FREE(nodes_allowed);
1842 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1843 struct kobject *kobj, const char *buf,
1847 unsigned long count;
1851 err = kstrtoul(buf, 10, &count);
1855 h = kobj_to_hstate(kobj, &nid);
1856 return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
1859 static ssize_t nr_hugepages_show(struct kobject *kobj,
1860 struct kobj_attribute *attr, char *buf)
1862 return nr_hugepages_show_common(kobj, attr, buf);
1865 static ssize_t nr_hugepages_store(struct kobject *kobj,
1866 struct kobj_attribute *attr, const char *buf, size_t len)
1868 return nr_hugepages_store_common(false, kobj, buf, len);
1870 HSTATE_ATTR(nr_hugepages);
1875 * hstate attribute for optionally mempolicy-based constraint on persistent
1876 * huge page alloc/free.
1878 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1879 struct kobj_attribute *attr, char *buf)
1881 return nr_hugepages_show_common(kobj, attr, buf);
1884 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1885 struct kobj_attribute *attr, const char *buf, size_t len)
1887 return nr_hugepages_store_common(true, kobj, buf, len);
1889 HSTATE_ATTR(nr_hugepages_mempolicy);
1893 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1894 struct kobj_attribute *attr, char *buf)
1896 struct hstate *h = kobj_to_hstate(kobj, NULL);
1897 return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1900 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1901 struct kobj_attribute *attr, const char *buf, size_t count)
1904 unsigned long input;
1905 struct hstate *h = kobj_to_hstate(kobj, NULL);
1907 if (hstate_is_gigantic(h))
1910 err = kstrtoul(buf, 10, &input);
1914 spin_lock(&hugetlb_lock);
1915 h->nr_overcommit_huge_pages = input;
1916 spin_unlock(&hugetlb_lock);
1920 HSTATE_ATTR(nr_overcommit_hugepages);
1922 static ssize_t free_hugepages_show(struct kobject *kobj,
1923 struct kobj_attribute *attr, char *buf)
1926 unsigned long free_huge_pages;
1929 h = kobj_to_hstate(kobj, &nid);
1930 if (nid == NUMA_NO_NODE)
1931 free_huge_pages = h->free_huge_pages;
1933 free_huge_pages = h->free_huge_pages_node[nid];
1935 return sprintf(buf, "%lu\n", free_huge_pages);
1937 HSTATE_ATTR_RO(free_hugepages);
1939 static ssize_t resv_hugepages_show(struct kobject *kobj,
1940 struct kobj_attribute *attr, char *buf)
1942 struct hstate *h = kobj_to_hstate(kobj, NULL);
1943 return sprintf(buf, "%lu\n", h->resv_huge_pages);
1945 HSTATE_ATTR_RO(resv_hugepages);
1947 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1948 struct kobj_attribute *attr, char *buf)
1951 unsigned long surplus_huge_pages;
1954 h = kobj_to_hstate(kobj, &nid);
1955 if (nid == NUMA_NO_NODE)
1956 surplus_huge_pages = h->surplus_huge_pages;
1958 surplus_huge_pages = h->surplus_huge_pages_node[nid];
1960 return sprintf(buf, "%lu\n", surplus_huge_pages);
1962 HSTATE_ATTR_RO(surplus_hugepages);
1964 static struct attribute *hstate_attrs[] = {
1965 &nr_hugepages_attr.attr,
1966 &nr_overcommit_hugepages_attr.attr,
1967 &free_hugepages_attr.attr,
1968 &resv_hugepages_attr.attr,
1969 &surplus_hugepages_attr.attr,
1971 &nr_hugepages_mempolicy_attr.attr,
1976 static struct attribute_group hstate_attr_group = {
1977 .attrs = hstate_attrs,
1980 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1981 struct kobject **hstate_kobjs,
1982 struct attribute_group *hstate_attr_group)
1985 int hi = hstate_index(h);
1987 hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1988 if (!hstate_kobjs[hi])
1991 retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1993 kobject_put(hstate_kobjs[hi]);
1998 static void __init hugetlb_sysfs_init(void)
2003 hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2004 if (!hugepages_kobj)
2007 for_each_hstate(h) {
2008 err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2009 hstate_kobjs, &hstate_attr_group);
2011 pr_err("Hugetlb: Unable to add hstate %s", h->name);
2018 * node_hstate/s - associate per node hstate attributes, via their kobjects,
2019 * with node devices in node_devices[] using a parallel array. The array
2020 * index of a node device or _hstate == node id.
2021 * This is here to avoid any static dependency of the node device driver, in
2022 * the base kernel, on the hugetlb module.
2024 struct node_hstate {
2025 struct kobject *hugepages_kobj;
2026 struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2028 struct node_hstate node_hstates[MAX_NUMNODES];
2031 * A subset of global hstate attributes for node devices
2033 static struct attribute *per_node_hstate_attrs[] = {
2034 &nr_hugepages_attr.attr,
2035 &free_hugepages_attr.attr,
2036 &surplus_hugepages_attr.attr,
2040 static struct attribute_group per_node_hstate_attr_group = {
2041 .attrs = per_node_hstate_attrs,
2045 * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2046 * Returns node id via non-NULL nidp.
2048 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2052 for (nid = 0; nid < nr_node_ids; nid++) {
2053 struct node_hstate *nhs = &node_hstates[nid];
2055 for (i = 0; i < HUGE_MAX_HSTATE; i++)
2056 if (nhs->hstate_kobjs[i] == kobj) {
2068 * Unregister hstate attributes from a single node device.
2069 * No-op if no hstate attributes attached.
2071 static void hugetlb_unregister_node(struct node *node)
2074 struct node_hstate *nhs = &node_hstates[node->dev.id];
2076 if (!nhs->hugepages_kobj)
2077 return; /* no hstate attributes */
2079 for_each_hstate(h) {
2080 int idx = hstate_index(h);
2081 if (nhs->hstate_kobjs[idx]) {
2082 kobject_put(nhs->hstate_kobjs[idx]);
2083 nhs->hstate_kobjs[idx] = NULL;
2087 kobject_put(nhs->hugepages_kobj);
2088 nhs->hugepages_kobj = NULL;
2092 * hugetlb module exit: unregister hstate attributes from node devices
2095 static void hugetlb_unregister_all_nodes(void)
2100 * disable node device registrations.
2102 register_hugetlbfs_with_node(NULL, NULL);
2105 * remove hstate attributes from any nodes that have them.
2107 for (nid = 0; nid < nr_node_ids; nid++)
2108 hugetlb_unregister_node(node_devices[nid]);
2112 * Register hstate attributes for a single node device.
2113 * No-op if attributes already registered.
2115 static void hugetlb_register_node(struct node *node)
2118 struct node_hstate *nhs = &node_hstates[node->dev.id];
2121 if (nhs->hugepages_kobj)
2122 return; /* already allocated */
2124 nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2126 if (!nhs->hugepages_kobj)
2129 for_each_hstate(h) {
2130 err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2132 &per_node_hstate_attr_group);
2134 pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2135 h->name, node->dev.id);
2136 hugetlb_unregister_node(node);
2143 * hugetlb init time: register hstate attributes for all registered node
2144 * devices of nodes that have memory. All on-line nodes should have
2145 * registered their associated device by this time.
2147 static void __init hugetlb_register_all_nodes(void)
2151 for_each_node_state(nid, N_MEMORY) {
2152 struct node *node = node_devices[nid];
2153 if (node->dev.id == nid)
2154 hugetlb_register_node(node);
2158 * Let the node device driver know we're here so it can
2159 * [un]register hstate attributes on node hotplug.
2161 register_hugetlbfs_with_node(hugetlb_register_node,
2162 hugetlb_unregister_node);
2164 #else /* !CONFIG_NUMA */
2166 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2174 static void hugetlb_unregister_all_nodes(void) { }
2176 static void hugetlb_register_all_nodes(void) { }
2180 static void __exit hugetlb_exit(void)
2184 hugetlb_unregister_all_nodes();
2186 for_each_hstate(h) {
2187 kobject_put(hstate_kobjs[hstate_index(h)]);
2190 kobject_put(hugepages_kobj);
2191 kfree(htlb_fault_mutex_table);
2193 module_exit(hugetlb_exit);
2195 static int __init hugetlb_init(void)
2199 if (!hugepages_supported())
2202 if (!size_to_hstate(default_hstate_size)) {
2203 default_hstate_size = HPAGE_SIZE;
2204 if (!size_to_hstate(default_hstate_size))
2205 hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2207 default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2208 if (default_hstate_max_huge_pages)
2209 default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2211 hugetlb_init_hstates();
2212 gather_bootmem_prealloc();
2215 hugetlb_sysfs_init();
2216 hugetlb_register_all_nodes();
2217 hugetlb_cgroup_file_init();
2220 num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2222 num_fault_mutexes = 1;
2224 htlb_fault_mutex_table =
2225 kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2226 BUG_ON(!htlb_fault_mutex_table);
2228 for (i = 0; i < num_fault_mutexes; i++)
2229 mutex_init(&htlb_fault_mutex_table[i]);
2232 module_init(hugetlb_init);
2234 /* Should be called on processing a hugepagesz=... option */
2235 void __init hugetlb_add_hstate(unsigned order)
2240 if (size_to_hstate(PAGE_SIZE << order)) {
2241 pr_warning("hugepagesz= specified twice, ignoring\n");
2244 BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2246 h = &hstates[hugetlb_max_hstate++];
2248 h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2249 h->nr_huge_pages = 0;
2250 h->free_huge_pages = 0;
2251 for (i = 0; i < MAX_NUMNODES; ++i)
2252 INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2253 INIT_LIST_HEAD(&h->hugepage_activelist);
2254 h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2255 h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2256 snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2257 huge_page_size(h)/1024);
2262 static int __init hugetlb_nrpages_setup(char *s)
2265 static unsigned long *last_mhp;
2268 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2269 * so this hugepages= parameter goes to the "default hstate".
2271 if (!hugetlb_max_hstate)
2272 mhp = &default_hstate_max_huge_pages;
2274 mhp = &parsed_hstate->max_huge_pages;
2276 if (mhp == last_mhp) {
2277 pr_warning("hugepages= specified twice without "
2278 "interleaving hugepagesz=, ignoring\n");
2282 if (sscanf(s, "%lu", mhp) <= 0)
2286 * Global state is always initialized later in hugetlb_init.
2287 * But we need to allocate >= MAX_ORDER hstates here early to still
2288 * use the bootmem allocator.
2290 if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2291 hugetlb_hstate_alloc_pages(parsed_hstate);
2297 __setup("hugepages=", hugetlb_nrpages_setup);
2299 static int __init hugetlb_default_setup(char *s)
2301 default_hstate_size = memparse(s, &s);
2304 __setup("default_hugepagesz=", hugetlb_default_setup);
2306 static unsigned int cpuset_mems_nr(unsigned int *array)
2309 unsigned int nr = 0;
2311 for_each_node_mask(node, cpuset_current_mems_allowed)
2317 #ifdef CONFIG_SYSCTL
2318 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2319 struct ctl_table *table, int write,
2320 void __user *buffer, size_t *length, loff_t *ppos)
2322 struct hstate *h = &default_hstate;
2323 unsigned long tmp = h->max_huge_pages;
2326 if (!hugepages_supported())
2330 table->maxlen = sizeof(unsigned long);
2331 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2336 ret = __nr_hugepages_store_common(obey_mempolicy, h,
2337 NUMA_NO_NODE, tmp, *length);
2342 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2343 void __user *buffer, size_t *length, loff_t *ppos)
2346 return hugetlb_sysctl_handler_common(false, table, write,
2347 buffer, length, ppos);
2351 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2352 void __user *buffer, size_t *length, loff_t *ppos)
2354 return hugetlb_sysctl_handler_common(true, table, write,
2355 buffer, length, ppos);
2357 #endif /* CONFIG_NUMA */
2359 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2360 void __user *buffer,
2361 size_t *length, loff_t *ppos)
2363 struct hstate *h = &default_hstate;
2367 if (!hugepages_supported())
2370 tmp = h->nr_overcommit_huge_pages;
2372 if (write && hstate_is_gigantic(h))
2376 table->maxlen = sizeof(unsigned long);
2377 ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2382 spin_lock(&hugetlb_lock);
2383 h->nr_overcommit_huge_pages = tmp;
2384 spin_unlock(&hugetlb_lock);
2390 #endif /* CONFIG_SYSCTL */
2392 void hugetlb_report_meminfo(struct seq_file *m)
2394 struct hstate *h = &default_hstate;
2395 if (!hugepages_supported())
2398 "HugePages_Total: %5lu\n"
2399 "HugePages_Free: %5lu\n"
2400 "HugePages_Rsvd: %5lu\n"
2401 "HugePages_Surp: %5lu\n"
2402 "Hugepagesize: %8lu kB\n",
2406 h->surplus_huge_pages,
2407 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2410 int hugetlb_report_node_meminfo(int nid, char *buf)
2412 struct hstate *h = &default_hstate;
2413 if (!hugepages_supported())
2416 "Node %d HugePages_Total: %5u\n"
2417 "Node %d HugePages_Free: %5u\n"
2418 "Node %d HugePages_Surp: %5u\n",
2419 nid, h->nr_huge_pages_node[nid],
2420 nid, h->free_huge_pages_node[nid],
2421 nid, h->surplus_huge_pages_node[nid]);
2424 void hugetlb_show_meminfo(void)
2429 if (!hugepages_supported())
2432 for_each_node_state(nid, N_MEMORY)
2434 pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2436 h->nr_huge_pages_node[nid],
2437 h->free_huge_pages_node[nid],
2438 h->surplus_huge_pages_node[nid],
2439 1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2442 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2443 unsigned long hugetlb_total_pages(void)
2446 unsigned long nr_total_pages = 0;
2449 nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2450 return nr_total_pages;
2453 static int hugetlb_acct_memory(struct hstate *h, long delta)
2457 spin_lock(&hugetlb_lock);
2459 * When cpuset is configured, it breaks the strict hugetlb page
2460 * reservation as the accounting is done on a global variable. Such
2461 * reservation is completely rubbish in the presence of cpuset because
2462 * the reservation is not checked against page availability for the
2463 * current cpuset. Application can still potentially OOM'ed by kernel
2464 * with lack of free htlb page in cpuset that the task is in.
2465 * Attempt to enforce strict accounting with cpuset is almost
2466 * impossible (or too ugly) because cpuset is too fluid that
2467 * task or memory node can be dynamically moved between cpusets.
2469 * The change of semantics for shared hugetlb mapping with cpuset is
2470 * undesirable. However, in order to preserve some of the semantics,
2471 * we fall back to check against current free page availability as
2472 * a best attempt and hopefully to minimize the impact of changing
2473 * semantics that cpuset has.
2476 if (gather_surplus_pages(h, delta) < 0)
2479 if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2480 return_unused_surplus_pages(h, delta);
2487 return_unused_surplus_pages(h, (unsigned long) -delta);
2490 spin_unlock(&hugetlb_lock);
2494 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2496 struct resv_map *resv = vma_resv_map(vma);
2499 * This new VMA should share its siblings reservation map if present.
2500 * The VMA will only ever have a valid reservation map pointer where
2501 * it is being copied for another still existing VMA. As that VMA
2502 * has a reference to the reservation map it cannot disappear until
2503 * after this open call completes. It is therefore safe to take a
2504 * new reference here without additional locking.
2506 if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2507 kref_get(&resv->refs);
2510 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2512 struct hstate *h = hstate_vma(vma);
2513 struct resv_map *resv = vma_resv_map(vma);
2514 struct hugepage_subpool *spool = subpool_vma(vma);
2515 unsigned long reserve, start, end;
2518 if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2521 start = vma_hugecache_offset(h, vma, vma->vm_start);
2522 end = vma_hugecache_offset(h, vma, vma->vm_end);
2524 reserve = (end - start) - region_count(resv, start, end);
2526 kref_put(&resv->refs, resv_map_release);
2530 * Decrement reserve counts. The global reserve count may be
2531 * adjusted if the subpool has a minimum size.
2533 gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
2534 hugetlb_acct_memory(h, -gbl_reserve);
2539 * We cannot handle pagefaults against hugetlb pages at all. They cause
2540 * handle_mm_fault() to try to instantiate regular-sized pages in the
2541 * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
2544 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2550 const struct vm_operations_struct hugetlb_vm_ops = {
2551 .fault = hugetlb_vm_op_fault,
2552 .open = hugetlb_vm_op_open,
2553 .close = hugetlb_vm_op_close,
2556 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2562 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2563 vma->vm_page_prot)));
2565 entry = huge_pte_wrprotect(mk_huge_pte(page,
2566 vma->vm_page_prot));
2568 entry = pte_mkyoung(entry);
2569 entry = pte_mkhuge(entry);
2570 entry = arch_make_huge_pte(entry, vma, page, writable);
2575 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2576 unsigned long address, pte_t *ptep)
2580 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2581 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2582 update_mmu_cache(vma, address, ptep);
2585 static int is_hugetlb_entry_migration(pte_t pte)
2589 if (huge_pte_none(pte) || pte_present(pte))
2591 swp = pte_to_swp_entry(pte);
2592 if (non_swap_entry(swp) && is_migration_entry(swp))
2598 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2602 if (huge_pte_none(pte) || pte_present(pte))
2604 swp = pte_to_swp_entry(pte);
2605 if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2611 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2612 struct vm_area_struct *vma)
2614 pte_t *src_pte, *dst_pte, entry;
2615 struct page *ptepage;
2618 struct hstate *h = hstate_vma(vma);
2619 unsigned long sz = huge_page_size(h);
2620 unsigned long mmun_start; /* For mmu_notifiers */
2621 unsigned long mmun_end; /* For mmu_notifiers */
2624 cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2626 mmun_start = vma->vm_start;
2627 mmun_end = vma->vm_end;
2629 mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
2631 for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2632 spinlock_t *src_ptl, *dst_ptl;
2633 src_pte = huge_pte_offset(src, addr);
2636 dst_pte = huge_pte_alloc(dst, addr, sz);
2642 /* If the pagetables are shared don't copy or take references */
2643 if (dst_pte == src_pte)
2646 dst_ptl = huge_pte_lock(h, dst, dst_pte);
2647 src_ptl = huge_pte_lockptr(h, src, src_pte);
2648 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
2649 entry = huge_ptep_get(src_pte);
2650 if (huge_pte_none(entry)) { /* skip none entry */
2652 } else if (unlikely(is_hugetlb_entry_migration(entry) ||
2653 is_hugetlb_entry_hwpoisoned(entry))) {
2654 swp_entry_t swp_entry = pte_to_swp_entry(entry);
2656 if (is_write_migration_entry(swp_entry) && cow) {
2658 * COW mappings require pages in both
2659 * parent and child to be set to read.
2661 make_migration_entry_read(&swp_entry);
2662 entry = swp_entry_to_pte(swp_entry);
2663 set_huge_pte_at(src, addr, src_pte, entry);
2665 set_huge_pte_at(dst, addr, dst_pte, entry);
2668 huge_ptep_set_wrprotect(src, addr, src_pte);
2669 mmu_notifier_invalidate_range(src, mmun_start,
2672 entry = huge_ptep_get(src_pte);
2673 ptepage = pte_page(entry);
2675 page_dup_rmap(ptepage);
2676 set_huge_pte_at(dst, addr, dst_pte, entry);
2678 spin_unlock(src_ptl);
2679 spin_unlock(dst_ptl);
2683 mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
2688 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
2689 unsigned long start, unsigned long end,
2690 struct page *ref_page)
2692 int force_flush = 0;
2693 struct mm_struct *mm = vma->vm_mm;
2694 unsigned long address;
2699 struct hstate *h = hstate_vma(vma);
2700 unsigned long sz = huge_page_size(h);
2701 const unsigned long mmun_start = start; /* For mmu_notifiers */
2702 const unsigned long mmun_end = end; /* For mmu_notifiers */
2704 WARN_ON(!is_vm_hugetlb_page(vma));
2705 BUG_ON(start & ~huge_page_mask(h));
2706 BUG_ON(end & ~huge_page_mask(h));
2708 tlb_start_vma(tlb, vma);
2709 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2712 for (; address < end; address += sz) {
2713 ptep = huge_pte_offset(mm, address);
2717 ptl = huge_pte_lock(h, mm, ptep);
2718 if (huge_pmd_unshare(mm, &address, ptep))
2721 pte = huge_ptep_get(ptep);
2722 if (huge_pte_none(pte))
2726 * Migrating hugepage or HWPoisoned hugepage is already
2727 * unmapped and its refcount is dropped, so just clear pte here.
2729 if (unlikely(!pte_present(pte))) {
2730 huge_pte_clear(mm, address, ptep);
2734 page = pte_page(pte);
2736 * If a reference page is supplied, it is because a specific
2737 * page is being unmapped, not a range. Ensure the page we
2738 * are about to unmap is the actual page of interest.
2741 if (page != ref_page)
2745 * Mark the VMA as having unmapped its page so that
2746 * future faults in this VMA will fail rather than
2747 * looking like data was lost
2749 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2752 pte = huge_ptep_get_and_clear(mm, address, ptep);
2753 tlb_remove_tlb_entry(tlb, ptep, address);
2754 if (huge_pte_dirty(pte))
2755 set_page_dirty(page);
2757 page_remove_rmap(page);
2758 force_flush = !__tlb_remove_page(tlb, page);
2764 /* Bail out after unmapping reference page if supplied */
2773 * mmu_gather ran out of room to batch pages, we break out of
2774 * the PTE lock to avoid doing the potential expensive TLB invalidate
2775 * and page-free while holding it.
2780 if (address < end && !ref_page)
2783 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2784 tlb_end_vma(tlb, vma);
2787 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2788 struct vm_area_struct *vma, unsigned long start,
2789 unsigned long end, struct page *ref_page)
2791 __unmap_hugepage_range(tlb, vma, start, end, ref_page);
2794 * Clear this flag so that x86's huge_pmd_share page_table_shareable
2795 * test will fail on a vma being torn down, and not grab a page table
2796 * on its way out. We're lucky that the flag has such an appropriate
2797 * name, and can in fact be safely cleared here. We could clear it
2798 * before the __unmap_hugepage_range above, but all that's necessary
2799 * is to clear it before releasing the i_mmap_rwsem. This works
2800 * because in the context this is called, the VMA is about to be
2801 * destroyed and the i_mmap_rwsem is held.
2803 vma->vm_flags &= ~VM_MAYSHARE;
2806 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2807 unsigned long end, struct page *ref_page)
2809 struct mm_struct *mm;
2810 struct mmu_gather tlb;
2814 tlb_gather_mmu(&tlb, mm, start, end);
2815 __unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2816 tlb_finish_mmu(&tlb, start, end);
2820 * This is called when the original mapper is failing to COW a MAP_PRIVATE
2821 * mappping it owns the reserve page for. The intention is to unmap the page
2822 * from other VMAs and let the children be SIGKILLed if they are faulting the
2825 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2826 struct page *page, unsigned long address)
2828 struct hstate *h = hstate_vma(vma);
2829 struct vm_area_struct *iter_vma;
2830 struct address_space *mapping;
2834 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2835 * from page cache lookup which is in HPAGE_SIZE units.
2837 address = address & huge_page_mask(h);
2838 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2840 mapping = file_inode(vma->vm_file)->i_mapping;
2843 * Take the mapping lock for the duration of the table walk. As
2844 * this mapping should be shared between all the VMAs,
2845 * __unmap_hugepage_range() is called as the lock is already held
2847 i_mmap_lock_write(mapping);
2848 vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
2849 /* Do not unmap the current VMA */
2850 if (iter_vma == vma)
2854 * Unmap the page from other VMAs without their own reserves.
2855 * They get marked to be SIGKILLed if they fault in these
2856 * areas. This is because a future no-page fault on this VMA
2857 * could insert a zeroed page instead of the data existing
2858 * from the time of fork. This would look like data corruption
2860 if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2861 unmap_hugepage_range(iter_vma, address,
2862 address + huge_page_size(h), page);
2864 i_mmap_unlock_write(mapping);
2868 * Hugetlb_cow() should be called with page lock of the original hugepage held.
2869 * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2870 * cannot race with other handlers or page migration.
2871 * Keep the pte_same checks anyway to make transition from the mutex easier.
2873 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2874 unsigned long address, pte_t *ptep, pte_t pte,
2875 struct page *pagecache_page, spinlock_t *ptl)
2877 struct hstate *h = hstate_vma(vma);
2878 struct page *old_page, *new_page;
2879 int ret = 0, outside_reserve = 0;
2880 unsigned long mmun_start; /* For mmu_notifiers */
2881 unsigned long mmun_end; /* For mmu_notifiers */
2883 old_page = pte_page(pte);
2886 /* If no-one else is actually using this page, avoid the copy
2887 * and just make the page writable */
2888 if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2889 page_move_anon_rmap(old_page, vma, address);
2890 set_huge_ptep_writable(vma, address, ptep);
2895 * If the process that created a MAP_PRIVATE mapping is about to
2896 * perform a COW due to a shared page count, attempt to satisfy
2897 * the allocation without using the existing reserves. The pagecache
2898 * page is used to determine if the reserve at this address was
2899 * consumed or not. If reserves were used, a partial faulted mapping
2900 * at the time of fork() could consume its reserves on COW instead
2901 * of the full address range.
2903 if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2904 old_page != pagecache_page)
2905 outside_reserve = 1;
2907 page_cache_get(old_page);
2910 * Drop page table lock as buddy allocator may be called. It will
2911 * be acquired again before returning to the caller, as expected.
2914 new_page = alloc_huge_page(vma, address, outside_reserve);
2916 if (IS_ERR(new_page)) {
2918 * If a process owning a MAP_PRIVATE mapping fails to COW,
2919 * it is due to references held by a child and an insufficient
2920 * huge page pool. To guarantee the original mappers
2921 * reliability, unmap the page from child processes. The child
2922 * may get SIGKILLed if it later faults.
2924 if (outside_reserve) {
2925 page_cache_release(old_page);
2926 BUG_ON(huge_pte_none(pte));
2927 unmap_ref_private(mm, vma, old_page, address);
2928 BUG_ON(huge_pte_none(pte));
2930 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2932 pte_same(huge_ptep_get(ptep), pte)))
2933 goto retry_avoidcopy;
2935 * race occurs while re-acquiring page table
2936 * lock, and our job is done.
2941 ret = (PTR_ERR(new_page) == -ENOMEM) ?
2942 VM_FAULT_OOM : VM_FAULT_SIGBUS;
2943 goto out_release_old;
2947 * When the original hugepage is shared one, it does not have
2948 * anon_vma prepared.
2950 if (unlikely(anon_vma_prepare(vma))) {
2952 goto out_release_all;
2955 copy_user_huge_page(new_page, old_page, address, vma,
2956 pages_per_huge_page(h));
2957 __SetPageUptodate(new_page);
2959 mmun_start = address & huge_page_mask(h);
2960 mmun_end = mmun_start + huge_page_size(h);
2961 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2964 * Retake the page table lock to check for racing updates
2965 * before the page tables are altered
2968 ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2969 if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
2970 ClearPagePrivate(new_page);
2973 huge_ptep_clear_flush(vma, address, ptep);
2974 mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
2975 set_huge_pte_at(mm, address, ptep,
2976 make_huge_pte(vma, new_page, 1));
2977 page_remove_rmap(old_page);
2978 hugepage_add_new_anon_rmap(new_page, vma, address);
2979 /* Make the old page be freed below */
2980 new_page = old_page;
2983 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2985 page_cache_release(new_page);
2987 page_cache_release(old_page);
2989 spin_lock(ptl); /* Caller expects lock to be held */
2993 /* Return the pagecache page at a given address within a VMA */
2994 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2995 struct vm_area_struct *vma, unsigned long address)
2997 struct address_space *mapping;
3000 mapping = vma->vm_file->f_mapping;
3001 idx = vma_hugecache_offset(h, vma, address);
3003 return find_lock_page(mapping, idx);
3007 * Return whether there is a pagecache page to back given address within VMA.
3008 * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3010 static bool hugetlbfs_pagecache_present(struct hstate *h,
3011 struct vm_area_struct *vma, unsigned long address)
3013 struct address_space *mapping;
3017 mapping = vma->vm_file->f_mapping;
3018 idx = vma_hugecache_offset(h, vma, address);
3020 page = find_get_page(mapping, idx);
3023 return page != NULL;
3026 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3027 struct address_space *mapping, pgoff_t idx,
3028 unsigned long address, pte_t *ptep, unsigned int flags)
3030 struct hstate *h = hstate_vma(vma);
3031 int ret = VM_FAULT_SIGBUS;
3039 * Currently, we are forced to kill the process in the event the
3040 * original mapper has unmapped pages from the child due to a failed
3041 * COW. Warn that such a situation has occurred as it may not be obvious
3043 if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3044 pr_warning("PID %d killed due to inadequate hugepage pool\n",
3050 * Use page lock to guard against racing truncation
3051 * before we get page_table_lock.
3054 page = find_lock_page(mapping, idx);
3056 size = i_size_read(mapping->host) >> huge_page_shift(h);
3059 page = alloc_huge_page(vma, address, 0);
3061 ret = PTR_ERR(page);
3065 ret = VM_FAULT_SIGBUS;
3068 clear_huge_page(page, address, pages_per_huge_page(h));
3069 __SetPageUptodate(page);
3071 if (vma->vm_flags & VM_MAYSHARE) {
3073 struct inode *inode = mapping->host;
3075 err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3082 ClearPagePrivate(page);
3084 spin_lock(&inode->i_lock);
3085 inode->i_blocks += blocks_per_huge_page(h);
3086 spin_unlock(&inode->i_lock);
3089 if (unlikely(anon_vma_prepare(vma))) {
3091 goto backout_unlocked;
3097 * If memory error occurs between mmap() and fault, some process
3098 * don't have hwpoisoned swap entry for errored virtual address.
3099 * So we need to block hugepage fault by PG_hwpoison bit check.
3101 if (unlikely(PageHWPoison(page))) {
3102 ret = VM_FAULT_HWPOISON |
3103 VM_FAULT_SET_HINDEX(hstate_index(h));
3104 goto backout_unlocked;
3109 * If we are going to COW a private mapping later, we examine the
3110 * pending reservations for this page now. This will ensure that
3111 * any allocations necessary to record that reservation occur outside
3114 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
3115 if (vma_needs_reservation(h, vma, address) < 0) {
3117 goto backout_unlocked;
3120 ptl = huge_pte_lockptr(h, mm, ptep);
3122 size = i_size_read(mapping->host) >> huge_page_shift(h);
3127 if (!huge_pte_none(huge_ptep_get(ptep)))
3131 ClearPagePrivate(page);
3132 hugepage_add_new_anon_rmap(page, vma, address);
3134 page_dup_rmap(page);
3135 new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3136 && (vma->vm_flags & VM_SHARED)));
3137 set_huge_pte_at(mm, address, ptep, new_pte);
3139 if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3140 /* Optimization, do the COW without a second fault */
3141 ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3158 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3159 struct vm_area_struct *vma,
3160 struct address_space *mapping,
3161 pgoff_t idx, unsigned long address)
3163 unsigned long key[2];
3166 if (vma->vm_flags & VM_SHARED) {
3167 key[0] = (unsigned long) mapping;
3170 key[0] = (unsigned long) mm;
3171 key[1] = address >> huge_page_shift(h);
3174 hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3176 return hash & (num_fault_mutexes - 1);
3180 * For uniprocesor systems we always use a single mutex, so just
3181 * return 0 and avoid the hashing overhead.
3183 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3184 struct vm_area_struct *vma,
3185 struct address_space *mapping,
3186 pgoff_t idx, unsigned long address)
3192 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3193 unsigned long address, unsigned int flags)
3200 struct page *page = NULL;
3201 struct page *pagecache_page = NULL;
3202 struct hstate *h = hstate_vma(vma);
3203 struct address_space *mapping;
3204 int need_wait_lock = 0;
3206 address &= huge_page_mask(h);
3208 ptep = huge_pte_offset(mm, address);
3210 entry = huge_ptep_get(ptep);
3211 if (unlikely(is_hugetlb_entry_migration(entry))) {
3212 migration_entry_wait_huge(vma, mm, ptep);
3214 } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3215 return VM_FAULT_HWPOISON_LARGE |
3216 VM_FAULT_SET_HINDEX(hstate_index(h));
3219 ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3221 return VM_FAULT_OOM;
3223 mapping = vma->vm_file->f_mapping;
3224 idx = vma_hugecache_offset(h, vma, address);
3227 * Serialize hugepage allocation and instantiation, so that we don't
3228 * get spurious allocation failures if two CPUs race to instantiate
3229 * the same page in the page cache.
3231 hash = fault_mutex_hash(h, mm, vma, mapping, idx, address);
3232 mutex_lock(&htlb_fault_mutex_table[hash]);
3234 entry = huge_ptep_get(ptep);
3235 if (huge_pte_none(entry)) {
3236 ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3243 * entry could be a migration/hwpoison entry at this point, so this
3244 * check prevents the kernel from going below assuming that we have
3245 * a active hugepage in pagecache. This goto expects the 2nd page fault,
3246 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3249 if (!pte_present(entry))
3253 * If we are going to COW the mapping later, we examine the pending
3254 * reservations for this page now. This will ensure that any
3255 * allocations necessary to record that reservation occur outside the
3256 * spinlock. For private mappings, we also lookup the pagecache
3257 * page now as it is used to determine if a reservation has been
3260 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3261 if (vma_needs_reservation(h, vma, address) < 0) {
3266 if (!(vma->vm_flags & VM_MAYSHARE))
3267 pagecache_page = hugetlbfs_pagecache_page(h,
3271 ptl = huge_pte_lock(h, mm, ptep);
3273 /* Check for a racing update before calling hugetlb_cow */
3274 if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3278 * hugetlb_cow() requires page locks of pte_page(entry) and
3279 * pagecache_page, so here we need take the former one
3280 * when page != pagecache_page or !pagecache_page.
3282 page = pte_page(entry);
3283 if (page != pagecache_page)
3284 if (!trylock_page(page)) {
3291 if (flags & FAULT_FLAG_WRITE) {
3292 if (!huge_pte_write(entry)) {
3293 ret = hugetlb_cow(mm, vma, address, ptep, entry,
3294 pagecache_page, ptl);
3297 entry = huge_pte_mkdirty(entry);
3299 entry = pte_mkyoung(entry);
3300 if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3301 flags & FAULT_FLAG_WRITE))
3302 update_mmu_cache(vma, address, ptep);
3304 if (page != pagecache_page)
3310 if (pagecache_page) {
3311 unlock_page(pagecache_page);
3312 put_page(pagecache_page);
3315 mutex_unlock(&htlb_fault_mutex_table[hash]);
3317 * Generally it's safe to hold refcount during waiting page lock. But
3318 * here we just wait to defer the next page fault to avoid busy loop and
3319 * the page is not used after unlocked before returning from the current
3320 * page fault. So we are safe from accessing freed page, even if we wait
3321 * here without taking refcount.
3324 wait_on_page_locked(page);
3328 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3329 struct page **pages, struct vm_area_struct **vmas,
3330 unsigned long *position, unsigned long *nr_pages,
3331 long i, unsigned int flags)
3333 unsigned long pfn_offset;
3334 unsigned long vaddr = *position;
3335 unsigned long remainder = *nr_pages;
3336 struct hstate *h = hstate_vma(vma);
3338 while (vaddr < vma->vm_end && remainder) {
3340 spinlock_t *ptl = NULL;
3345 * If we have a pending SIGKILL, don't keep faulting pages and
3346 * potentially allocating memory.
3348 if (unlikely(fatal_signal_pending(current))) {
3354 * Some archs (sparc64, sh*) have multiple pte_ts to
3355 * each hugepage. We have to make sure we get the
3356 * first, for the page indexing below to work.
3358 * Note that page table lock is not held when pte is null.
3360 pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3362 ptl = huge_pte_lock(h, mm, pte);
3363 absent = !pte || huge_pte_none(huge_ptep_get(pte));
3366 * When coredumping, it suits get_dump_page if we just return
3367 * an error where there's an empty slot with no huge pagecache
3368 * to back it. This way, we avoid allocating a hugepage, and
3369 * the sparse dumpfile avoids allocating disk blocks, but its
3370 * huge holes still show up with zeroes where they need to be.
3372 if (absent && (flags & FOLL_DUMP) &&
3373 !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3381 * We need call hugetlb_fault for both hugepages under migration
3382 * (in which case hugetlb_fault waits for the migration,) and
3383 * hwpoisoned hugepages (in which case we need to prevent the
3384 * caller from accessing to them.) In order to do this, we use
3385 * here is_swap_pte instead of is_hugetlb_entry_migration and
3386 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3387 * both cases, and because we can't follow correct pages
3388 * directly from any kind of swap entries.
3390 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3391 ((flags & FOLL_WRITE) &&
3392 !huge_pte_write(huge_ptep_get(pte)))) {
3397 ret = hugetlb_fault(mm, vma, vaddr,
3398 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3399 if (!(ret & VM_FAULT_ERROR))
3406 pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3407 page = pte_page(huge_ptep_get(pte));
3410 pages[i] = mem_map_offset(page, pfn_offset);
3411 get_page_foll(pages[i]);
3421 if (vaddr < vma->vm_end && remainder &&
3422 pfn_offset < pages_per_huge_page(h)) {
3424 * We use pfn_offset to avoid touching the pageframes
3425 * of this compound page.
3431 *nr_pages = remainder;
3434 return i ? i : -EFAULT;
3437 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3438 unsigned long address, unsigned long end, pgprot_t newprot)
3440 struct mm_struct *mm = vma->vm_mm;
3441 unsigned long start = address;
3444 struct hstate *h = hstate_vma(vma);
3445 unsigned long pages = 0;
3447 BUG_ON(address >= end);
3448 flush_cache_range(vma, address, end);
3450 mmu_notifier_invalidate_range_start(mm, start, end);
3451 i_mmap_lock_write(vma->vm_file->f_mapping);
3452 for (; address < end; address += huge_page_size(h)) {
3454 ptep = huge_pte_offset(mm, address);
3457 ptl = huge_pte_lock(h, mm, ptep);
3458 if (huge_pmd_unshare(mm, &address, ptep)) {
3463 pte = huge_ptep_get(ptep);
3464 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
3468 if (unlikely(is_hugetlb_entry_migration(pte))) {
3469 swp_entry_t entry = pte_to_swp_entry(pte);
3471 if (is_write_migration_entry(entry)) {
3474 make_migration_entry_read(&entry);
3475 newpte = swp_entry_to_pte(entry);
3476 set_huge_pte_at(mm, address, ptep, newpte);
3482 if (!huge_pte_none(pte)) {
3483 pte = huge_ptep_get_and_clear(mm, address, ptep);
3484 pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3485 pte = arch_make_huge_pte(pte, vma, NULL, 0);
3486 set_huge_pte_at(mm, address, ptep, pte);
3492 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
3493 * may have cleared our pud entry and done put_page on the page table:
3494 * once we release i_mmap_rwsem, another task can do the final put_page
3495 * and that page table be reused and filled with junk.
3497 flush_tlb_range(vma, start, end);
3498 mmu_notifier_invalidate_range(mm, start, end);
3499 i_mmap_unlock_write(vma->vm_file->f_mapping);
3500 mmu_notifier_invalidate_range_end(mm, start, end);
3502 return pages << h->order;
3505 int hugetlb_reserve_pages(struct inode *inode,
3507 struct vm_area_struct *vma,
3508 vm_flags_t vm_flags)
3511 struct hstate *h = hstate_inode(inode);
3512 struct hugepage_subpool *spool = subpool_inode(inode);
3513 struct resv_map *resv_map;
3517 * Only apply hugepage reservation if asked. At fault time, an
3518 * attempt will be made for VM_NORESERVE to allocate a page
3519 * without using reserves
3521 if (vm_flags & VM_NORESERVE)
3525 * Shared mappings base their reservation on the number of pages that
3526 * are already allocated on behalf of the file. Private mappings need
3527 * to reserve the full area even if read-only as mprotect() may be
3528 * called to make the mapping read-write. Assume !vma is a shm mapping
3530 if (!vma || vma->vm_flags & VM_MAYSHARE) {
3531 resv_map = inode_resv_map(inode);
3533 chg = region_chg(resv_map, from, to);
3536 resv_map = resv_map_alloc();
3542 set_vma_resv_map(vma, resv_map);
3543 set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3552 * There must be enough pages in the subpool for the mapping. If
3553 * the subpool has a minimum size, there may be some global
3554 * reservations already in place (gbl_reserve).
3556 gbl_reserve = hugepage_subpool_get_pages(spool, chg);
3557 if (gbl_reserve < 0) {
3563 * Check enough hugepages are available for the reservation.
3564 * Hand the pages back to the subpool if there are not
3566 ret = hugetlb_acct_memory(h, gbl_reserve);
3568 /* put back original number of pages, chg */
3569 (void)hugepage_subpool_put_pages(spool, chg);
3574 * Account for the reservations made. Shared mappings record regions
3575 * that have reservations as they are shared by multiple VMAs.
3576 * When the last VMA disappears, the region map says how much
3577 * the reservation was and the page cache tells how much of
3578 * the reservation was consumed. Private mappings are per-VMA and
3579 * only the consumed reservations are tracked. When the VMA
3580 * disappears, the original reservation is the VMA size and the
3581 * consumed reservations are stored in the map. Hence, nothing
3582 * else has to be done for private mappings here
3584 if (!vma || vma->vm_flags & VM_MAYSHARE)
3585 region_add(resv_map, from, to);
3588 if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3589 kref_put(&resv_map->refs, resv_map_release);
3593 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3595 struct hstate *h = hstate_inode(inode);
3596 struct resv_map *resv_map = inode_resv_map(inode);
3598 struct hugepage_subpool *spool = subpool_inode(inode);
3602 chg = region_truncate(resv_map, offset);
3603 spin_lock(&inode->i_lock);
3604 inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3605 spin_unlock(&inode->i_lock);
3608 * If the subpool has a minimum size, the number of global
3609 * reservations to be released may be adjusted.
3611 gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
3612 hugetlb_acct_memory(h, -gbl_reserve);
3615 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3616 static unsigned long page_table_shareable(struct vm_area_struct *svma,
3617 struct vm_area_struct *vma,
3618 unsigned long addr, pgoff_t idx)
3620 unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3622 unsigned long sbase = saddr & PUD_MASK;
3623 unsigned long s_end = sbase + PUD_SIZE;
3625 /* Allow segments to share if only one is marked locked */
3626 unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3627 unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3630 * match the virtual addresses, permission and the alignment of the
3633 if (pmd_index(addr) != pmd_index(saddr) ||
3634 vm_flags != svm_flags ||
3635 sbase < svma->vm_start || svma->vm_end < s_end)
3641 static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3643 unsigned long base = addr & PUD_MASK;
3644 unsigned long end = base + PUD_SIZE;
3647 * check on proper vm_flags and page table alignment
3649 if (vma->vm_flags & VM_MAYSHARE &&
3650 vma->vm_start <= base && end <= vma->vm_end)
3656 * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3657 * and returns the corresponding pte. While this is not necessary for the
3658 * !shared pmd case because we can allocate the pmd later as well, it makes the
3659 * code much cleaner. pmd allocation is essential for the shared case because
3660 * pud has to be populated inside the same i_mmap_rwsem section - otherwise
3661 * racing tasks could either miss the sharing (see huge_pte_offset) or select a
3662 * bad pmd for sharing.
3664 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3666 struct vm_area_struct *vma = find_vma(mm, addr);
3667 struct address_space *mapping = vma->vm_file->f_mapping;
3668 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3670 struct vm_area_struct *svma;
3671 unsigned long saddr;
3676 if (!vma_shareable(vma, addr))
3677 return (pte_t *)pmd_alloc(mm, pud, addr);
3679 i_mmap_lock_write(mapping);
3680 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3684 saddr = page_table_shareable(svma, vma, addr, idx);
3686 spte = huge_pte_offset(svma->vm_mm, saddr);
3689 get_page(virt_to_page(spte));
3698 ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
3700 if (pud_none(*pud)) {
3701 pud_populate(mm, pud,
3702 (pmd_t *)((unsigned long)spte & PAGE_MASK));
3704 put_page(virt_to_page(spte));
3709 pte = (pte_t *)pmd_alloc(mm, pud, addr);
3710 i_mmap_unlock_write(mapping);
3715 * unmap huge page backed by shared pte.
3717 * Hugetlb pte page is ref counted at the time of mapping. If pte is shared
3718 * indicated by page_count > 1, unmap is achieved by clearing pud and
3719 * decrementing the ref count. If count == 1, the pte page is not shared.
3721 * called with page table lock held.
3723 * returns: 1 successfully unmapped a shared pte page
3724 * 0 the underlying pte page is not shared, or it is the last user
3726 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
3728 pgd_t *pgd = pgd_offset(mm, *addr);
3729 pud_t *pud = pud_offset(pgd, *addr);
3731 BUG_ON(page_count(virt_to_page(ptep)) == 0);
3732 if (page_count(virt_to_page(ptep)) == 1)
3736 put_page(virt_to_page(ptep));
3738 *addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3741 #define want_pmd_share() (1)
3742 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3743 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3747 #define want_pmd_share() (0)
3748 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3750 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3751 pte_t *huge_pte_alloc(struct mm_struct *mm,
3752 unsigned long addr, unsigned long sz)
3758 pgd = pgd_offset(mm, addr);
3759 pud = pud_alloc(mm, pgd, addr);
3761 if (sz == PUD_SIZE) {
3764 BUG_ON(sz != PMD_SIZE);
3765 if (want_pmd_share() && pud_none(*pud))
3766 pte = huge_pmd_share(mm, addr, pud);
3768 pte = (pte_t *)pmd_alloc(mm, pud, addr);
3771 BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
3776 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
3782 pgd = pgd_offset(mm, addr);
3783 if (pgd_present(*pgd)) {
3784 pud = pud_offset(pgd, addr);
3785 if (pud_present(*pud)) {
3787 return (pte_t *)pud;
3788 pmd = pmd_offset(pud, addr);
3791 return (pte_t *) pmd;
3794 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3797 * These functions are overwritable if your architecture needs its own
3800 struct page * __weak
3801 follow_huge_addr(struct mm_struct *mm, unsigned long address,
3804 return ERR_PTR(-EINVAL);
3807 struct page * __weak
3808 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
3809 pmd_t *pmd, int flags)
3811 struct page *page = NULL;
3814 ptl = pmd_lockptr(mm, pmd);
3817 * make sure that the address range covered by this pmd is not
3818 * unmapped from other threads.
3820 if (!pmd_huge(*pmd))
3822 if (pmd_present(*pmd)) {
3823 page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
3824 if (flags & FOLL_GET)
3827 if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
3829 __migration_entry_wait(mm, (pte_t *)pmd, ptl);
3833 * hwpoisoned entry is treated as no_page_table in
3834 * follow_page_mask().
3842 struct page * __weak
3843 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3844 pud_t *pud, int flags)
3846 if (flags & FOLL_GET)
3849 return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
3852 #ifdef CONFIG_MEMORY_FAILURE
3854 /* Should be called in hugetlb_lock */
3855 static int is_hugepage_on_freelist(struct page *hpage)
3859 struct hstate *h = page_hstate(hpage);
3860 int nid = page_to_nid(hpage);
3862 list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3869 * This function is called from memory failure code.
3870 * Assume the caller holds page lock of the head page.
3872 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3874 struct hstate *h = page_hstate(hpage);
3875 int nid = page_to_nid(hpage);
3878 spin_lock(&hugetlb_lock);
3879 if (is_hugepage_on_freelist(hpage)) {
3881 * Hwpoisoned hugepage isn't linked to activelist or freelist,
3882 * but dangling hpage->lru can trigger list-debug warnings
3883 * (this happens when we call unpoison_memory() on it),
3884 * so let it point to itself with list_del_init().
3886 list_del_init(&hpage->lru);
3887 set_page_refcounted(hpage);
3888 h->free_huge_pages--;
3889 h->free_huge_pages_node[nid]--;
3892 spin_unlock(&hugetlb_lock);
3897 bool isolate_huge_page(struct page *page, struct list_head *list)
3899 VM_BUG_ON_PAGE(!PageHead(page), page);
3900 if (!get_page_unless_zero(page))
3902 spin_lock(&hugetlb_lock);
3903 list_move_tail(&page->lru, list);
3904 spin_unlock(&hugetlb_lock);
3908 void putback_active_hugepage(struct page *page)
3910 VM_BUG_ON_PAGE(!PageHead(page), page);
3911 spin_lock(&hugetlb_lock);
3912 list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3913 spin_unlock(&hugetlb_lock);
3917 bool is_hugepage_active(struct page *page)
3919 VM_BUG_ON_PAGE(!PageHuge(page), page);
3921 * This function can be called for a tail page because the caller,
3922 * scan_movable_pages, scans through a given pfn-range which typically
3923 * covers one memory block. In systems using gigantic hugepage (1GB
3924 * for x86_64,) a hugepage is larger than a memory block, and we don't
3925 * support migrating such large hugepages for now, so return false
3926 * when called for tail pages.
3931 * Refcount of a hwpoisoned hugepages is 1, but they are not active,
3932 * so we should return false for them.
3934 if (unlikely(PageHWPoison(page)))
3936 return page_count(page) > 0;