2 * Copyright (C) 2009 Red Hat, Inc.
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
9 #include <linux/sched.h>
10 #include <linux/highmem.h>
11 #include <linux/hugetlb.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
16 #include <asm/pgalloc.h>
19 unsigned long transparent_hugepage_flags __read_mostly =
20 (1<<TRANSPARENT_HUGEPAGE_FLAG);
23 static ssize_t double_flag_show(struct kobject *kobj,
24 struct kobj_attribute *attr, char *buf,
25 enum transparent_hugepage_flag enabled,
26 enum transparent_hugepage_flag req_madv)
28 if (test_bit(enabled, &transparent_hugepage_flags)) {
29 VM_BUG_ON(test_bit(req_madv, &transparent_hugepage_flags));
30 return sprintf(buf, "[always] madvise never\n");
31 } else if (test_bit(req_madv, &transparent_hugepage_flags))
32 return sprintf(buf, "always [madvise] never\n");
34 return sprintf(buf, "always madvise [never]\n");
36 static ssize_t double_flag_store(struct kobject *kobj,
37 struct kobj_attribute *attr,
38 const char *buf, size_t count,
39 enum transparent_hugepage_flag enabled,
40 enum transparent_hugepage_flag req_madv)
42 if (!memcmp("always", buf,
43 min(sizeof("always")-1, count))) {
44 set_bit(enabled, &transparent_hugepage_flags);
45 clear_bit(req_madv, &transparent_hugepage_flags);
46 } else if (!memcmp("madvise", buf,
47 min(sizeof("madvise")-1, count))) {
48 clear_bit(enabled, &transparent_hugepage_flags);
49 set_bit(req_madv, &transparent_hugepage_flags);
50 } else if (!memcmp("never", buf,
51 min(sizeof("never")-1, count))) {
52 clear_bit(enabled, &transparent_hugepage_flags);
53 clear_bit(req_madv, &transparent_hugepage_flags);
60 static ssize_t enabled_show(struct kobject *kobj,
61 struct kobj_attribute *attr, char *buf)
63 return double_flag_show(kobj, attr, buf,
64 TRANSPARENT_HUGEPAGE_FLAG,
65 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
67 static ssize_t enabled_store(struct kobject *kobj,
68 struct kobj_attribute *attr,
69 const char *buf, size_t count)
71 return double_flag_store(kobj, attr, buf, count,
72 TRANSPARENT_HUGEPAGE_FLAG,
73 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
75 static struct kobj_attribute enabled_attr =
76 __ATTR(enabled, 0644, enabled_show, enabled_store);
78 static ssize_t single_flag_show(struct kobject *kobj,
79 struct kobj_attribute *attr, char *buf,
80 enum transparent_hugepage_flag flag)
82 if (test_bit(flag, &transparent_hugepage_flags))
83 return sprintf(buf, "[yes] no\n");
85 return sprintf(buf, "yes [no]\n");
87 static ssize_t single_flag_store(struct kobject *kobj,
88 struct kobj_attribute *attr,
89 const char *buf, size_t count,
90 enum transparent_hugepage_flag flag)
92 if (!memcmp("yes", buf,
93 min(sizeof("yes")-1, count))) {
94 set_bit(flag, &transparent_hugepage_flags);
95 } else if (!memcmp("no", buf,
96 min(sizeof("no")-1, count))) {
97 clear_bit(flag, &transparent_hugepage_flags);
105 * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
106 * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
107 * memory just to allocate one more hugepage.
109 static ssize_t defrag_show(struct kobject *kobj,
110 struct kobj_attribute *attr, char *buf)
112 return double_flag_show(kobj, attr, buf,
113 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
114 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
116 static ssize_t defrag_store(struct kobject *kobj,
117 struct kobj_attribute *attr,
118 const char *buf, size_t count)
120 return double_flag_store(kobj, attr, buf, count,
121 TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
122 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
124 static struct kobj_attribute defrag_attr =
125 __ATTR(defrag, 0644, defrag_show, defrag_store);
127 #ifdef CONFIG_DEBUG_VM
128 static ssize_t debug_cow_show(struct kobject *kobj,
129 struct kobj_attribute *attr, char *buf)
131 return single_flag_show(kobj, attr, buf,
132 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
134 static ssize_t debug_cow_store(struct kobject *kobj,
135 struct kobj_attribute *attr,
136 const char *buf, size_t count)
138 return single_flag_store(kobj, attr, buf, count,
139 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
141 static struct kobj_attribute debug_cow_attr =
142 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
143 #endif /* CONFIG_DEBUG_VM */
145 static struct attribute *hugepage_attr[] = {
148 #ifdef CONFIG_DEBUG_VM
149 &debug_cow_attr.attr,
154 static struct attribute_group hugepage_attr_group = {
155 .attrs = hugepage_attr,
156 .name = "transparent_hugepage",
158 #endif /* CONFIG_SYSFS */
160 static int __init hugepage_init(void)
165 err = sysfs_create_group(mm_kobj, &hugepage_attr_group);
167 printk(KERN_ERR "hugepage: register sysfs failed\n");
171 module_init(hugepage_init)
173 static int __init setup_transparent_hugepage(char *str)
178 if (!strcmp(str, "always")) {
179 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
180 &transparent_hugepage_flags);
181 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
182 &transparent_hugepage_flags);
184 } else if (!strcmp(str, "madvise")) {
185 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
186 &transparent_hugepage_flags);
187 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
188 &transparent_hugepage_flags);
190 } else if (!strcmp(str, "never")) {
191 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
192 &transparent_hugepage_flags);
193 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
194 &transparent_hugepage_flags);
200 "transparent_hugepage= cannot parse, ignored\n");
203 __setup("transparent_hugepage=", setup_transparent_hugepage);
205 static void prepare_pmd_huge_pte(pgtable_t pgtable,
206 struct mm_struct *mm)
208 assert_spin_locked(&mm->page_table_lock);
211 if (!mm->pmd_huge_pte)
212 INIT_LIST_HEAD(&pgtable->lru);
214 list_add(&pgtable->lru, &mm->pmd_huge_pte->lru);
215 mm->pmd_huge_pte = pgtable;
218 static inline pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
220 if (likely(vma->vm_flags & VM_WRITE))
221 pmd = pmd_mkwrite(pmd);
225 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
226 struct vm_area_struct *vma,
227 unsigned long haddr, pmd_t *pmd,
233 VM_BUG_ON(!PageCompound(page));
234 pgtable = pte_alloc_one(mm, haddr);
235 if (unlikely(!pgtable)) {
240 clear_huge_page(page, haddr, HPAGE_PMD_NR);
241 __SetPageUptodate(page);
243 spin_lock(&mm->page_table_lock);
244 if (unlikely(!pmd_none(*pmd))) {
245 spin_unlock(&mm->page_table_lock);
247 pte_free(mm, pgtable);
250 entry = mk_pmd(page, vma->vm_page_prot);
251 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
252 entry = pmd_mkhuge(entry);
254 * The spinlocking to take the lru_lock inside
255 * page_add_new_anon_rmap() acts as a full memory
256 * barrier to be sure clear_huge_page writes become
257 * visible after the set_pmd_at() write.
259 page_add_new_anon_rmap(page, vma, haddr);
260 set_pmd_at(mm, haddr, pmd, entry);
261 prepare_pmd_huge_pte(pgtable, mm);
262 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
263 spin_unlock(&mm->page_table_lock);
269 static inline struct page *alloc_hugepage(int defrag)
271 return alloc_pages(GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT),
275 int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
276 unsigned long address, pmd_t *pmd,
280 unsigned long haddr = address & HPAGE_PMD_MASK;
283 if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) {
284 if (unlikely(anon_vma_prepare(vma)))
286 page = alloc_hugepage(transparent_hugepage_defrag(vma));
290 return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page);
294 * Use __pte_alloc instead of pte_alloc_map, because we can't
295 * run pte_offset_map on the pmd, if an huge pmd could
296 * materialize from under us from a different thread.
298 if (unlikely(__pte_alloc(mm, vma, pmd, address)))
300 /* if an huge pmd materialized from under us just retry later */
301 if (unlikely(pmd_trans_huge(*pmd)))
304 * A regular pmd is established and it can't morph into a huge pmd
305 * from under us anymore at this point because we hold the mmap_sem
306 * read mode and khugepaged takes it in write mode. So now it's
307 * safe to run pte_offset_map().
309 pte = pte_offset_map(pmd, address);
310 return handle_pte_fault(mm, vma, address, pte, pmd, flags);
313 int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
314 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
315 struct vm_area_struct *vma)
317 struct page *src_page;
323 pgtable = pte_alloc_one(dst_mm, addr);
324 if (unlikely(!pgtable))
327 spin_lock(&dst_mm->page_table_lock);
328 spin_lock_nested(&src_mm->page_table_lock, SINGLE_DEPTH_NESTING);
332 if (unlikely(!pmd_trans_huge(pmd))) {
333 pte_free(dst_mm, pgtable);
336 if (unlikely(pmd_trans_splitting(pmd))) {
337 /* split huge page running from under us */
338 spin_unlock(&src_mm->page_table_lock);
339 spin_unlock(&dst_mm->page_table_lock);
340 pte_free(dst_mm, pgtable);
342 wait_split_huge_page(vma->anon_vma, src_pmd); /* src_vma */
345 src_page = pmd_page(pmd);
346 VM_BUG_ON(!PageHead(src_page));
348 page_dup_rmap(src_page);
349 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
351 pmdp_set_wrprotect(src_mm, addr, src_pmd);
352 pmd = pmd_mkold(pmd_wrprotect(pmd));
353 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
354 prepare_pmd_huge_pte(pgtable, dst_mm);
358 spin_unlock(&src_mm->page_table_lock);
359 spin_unlock(&dst_mm->page_table_lock);
364 /* no "address" argument so destroys page coloring of some arch */
365 pgtable_t get_pmd_huge_pte(struct mm_struct *mm)
369 assert_spin_locked(&mm->page_table_lock);
372 pgtable = mm->pmd_huge_pte;
373 if (list_empty(&pgtable->lru))
374 mm->pmd_huge_pte = NULL;
376 mm->pmd_huge_pte = list_entry(pgtable->lru.next,
378 list_del(&pgtable->lru);
383 static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
384 struct vm_area_struct *vma,
385 unsigned long address,
386 pmd_t *pmd, pmd_t orig_pmd,
395 pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
397 if (unlikely(!pages)) {
402 for (i = 0; i < HPAGE_PMD_NR; i++) {
403 pages[i] = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
405 if (unlikely(!pages[i])) {
414 for (i = 0; i < HPAGE_PMD_NR; i++) {
415 copy_user_highpage(pages[i], page + i,
416 haddr + PAGE_SHIFT*i, vma);
417 __SetPageUptodate(pages[i]);
421 spin_lock(&mm->page_table_lock);
422 if (unlikely(!pmd_same(*pmd, orig_pmd)))
424 VM_BUG_ON(!PageHead(page));
426 pmdp_clear_flush_notify(vma, haddr, pmd);
427 /* leave pmd empty until pte is filled */
429 pgtable = get_pmd_huge_pte(mm);
430 pmd_populate(mm, &_pmd, pgtable);
432 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
434 entry = mk_pte(pages[i], vma->vm_page_prot);
435 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
436 page_add_new_anon_rmap(pages[i], vma, haddr);
437 pte = pte_offset_map(&_pmd, haddr);
438 VM_BUG_ON(!pte_none(*pte));
439 set_pte_at(mm, haddr, pte, entry);
445 smp_wmb(); /* make pte visible before pmd */
446 pmd_populate(mm, pmd, pgtable);
447 page_remove_rmap(page);
448 spin_unlock(&mm->page_table_lock);
450 ret |= VM_FAULT_WRITE;
457 spin_unlock(&mm->page_table_lock);
458 for (i = 0; i < HPAGE_PMD_NR; i++)
464 int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
465 unsigned long address, pmd_t *pmd, pmd_t orig_pmd)
468 struct page *page, *new_page;
471 VM_BUG_ON(!vma->anon_vma);
472 spin_lock(&mm->page_table_lock);
473 if (unlikely(!pmd_same(*pmd, orig_pmd)))
476 page = pmd_page(orig_pmd);
477 VM_BUG_ON(!PageCompound(page) || !PageHead(page));
478 haddr = address & HPAGE_PMD_MASK;
479 if (page_mapcount(page) == 1) {
481 entry = pmd_mkyoung(orig_pmd);
482 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
483 if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1))
484 update_mmu_cache(vma, address, entry);
485 ret |= VM_FAULT_WRITE;
489 spin_unlock(&mm->page_table_lock);
491 if (transparent_hugepage_enabled(vma) &&
492 !transparent_hugepage_debug_cow())
493 new_page = alloc_hugepage(transparent_hugepage_defrag(vma));
497 if (unlikely(!new_page)) {
498 ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
499 pmd, orig_pmd, page, haddr);
504 copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
505 __SetPageUptodate(new_page);
507 spin_lock(&mm->page_table_lock);
509 if (unlikely(!pmd_same(*pmd, orig_pmd)))
513 VM_BUG_ON(!PageHead(page));
514 entry = mk_pmd(new_page, vma->vm_page_prot);
515 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
516 entry = pmd_mkhuge(entry);
517 pmdp_clear_flush_notify(vma, haddr, pmd);
518 page_add_new_anon_rmap(new_page, vma, haddr);
519 set_pmd_at(mm, haddr, pmd, entry);
520 update_mmu_cache(vma, address, entry);
521 page_remove_rmap(page);
523 ret |= VM_FAULT_WRITE;
526 spin_unlock(&mm->page_table_lock);
531 struct page *follow_trans_huge_pmd(struct mm_struct *mm,
536 struct page *page = NULL;
538 assert_spin_locked(&mm->page_table_lock);
540 if (flags & FOLL_WRITE && !pmd_write(*pmd))
543 page = pmd_page(*pmd);
544 VM_BUG_ON(!PageHead(page));
545 if (flags & FOLL_TOUCH) {
548 * We should set the dirty bit only for FOLL_WRITE but
549 * for now the dirty bit in the pmd is meaningless.
550 * And if the dirty bit will become meaningful and
551 * we'll only set it with FOLL_WRITE, an atomic
552 * set_bit will be required on the pmd to set the
553 * young bit, instead of the current set_pmd_at.
555 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
556 set_pmd_at(mm, addr & HPAGE_PMD_MASK, pmd, _pmd);
558 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
559 VM_BUG_ON(!PageCompound(page));
560 if (flags & FOLL_GET)
567 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
572 spin_lock(&tlb->mm->page_table_lock);
573 if (likely(pmd_trans_huge(*pmd))) {
574 if (unlikely(pmd_trans_splitting(*pmd))) {
575 spin_unlock(&tlb->mm->page_table_lock);
576 wait_split_huge_page(vma->anon_vma,
581 pgtable = get_pmd_huge_pte(tlb->mm);
582 page = pmd_page(*pmd);
584 page_remove_rmap(page);
585 VM_BUG_ON(page_mapcount(page) < 0);
586 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
587 VM_BUG_ON(!PageHead(page));
588 spin_unlock(&tlb->mm->page_table_lock);
589 tlb_remove_page(tlb, page);
590 pte_free(tlb->mm, pgtable);
594 spin_unlock(&tlb->mm->page_table_lock);
599 pmd_t *page_check_address_pmd(struct page *page,
600 struct mm_struct *mm,
601 unsigned long address,
602 enum page_check_address_pmd_flag flag)
606 pmd_t *pmd, *ret = NULL;
608 if (address & ~HPAGE_PMD_MASK)
611 pgd = pgd_offset(mm, address);
612 if (!pgd_present(*pgd))
615 pud = pud_offset(pgd, address);
616 if (!pud_present(*pud))
619 pmd = pmd_offset(pud, address);
622 if (pmd_page(*pmd) != page)
624 VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG &&
625 pmd_trans_splitting(*pmd));
626 if (pmd_trans_huge(*pmd)) {
627 VM_BUG_ON(flag == PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG &&
628 !pmd_trans_splitting(*pmd));
635 static int __split_huge_page_splitting(struct page *page,
636 struct vm_area_struct *vma,
637 unsigned long address)
639 struct mm_struct *mm = vma->vm_mm;
643 spin_lock(&mm->page_table_lock);
644 pmd = page_check_address_pmd(page, mm, address,
645 PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG);
648 * We can't temporarily set the pmd to null in order
649 * to split it, the pmd must remain marked huge at all
650 * times or the VM won't take the pmd_trans_huge paths
651 * and it won't wait on the anon_vma->root->lock to
652 * serialize against split_huge_page*.
654 pmdp_splitting_flush_notify(vma, address, pmd);
657 spin_unlock(&mm->page_table_lock);
662 static void __split_huge_page_refcount(struct page *page)
665 unsigned long head_index = page->index;
666 struct zone *zone = page_zone(page);
668 /* prevent PageLRU to go away from under us, and freeze lru stats */
669 spin_lock_irq(&zone->lru_lock);
672 for (i = 1; i < HPAGE_PMD_NR; i++) {
673 struct page *page_tail = page + i;
675 /* tail_page->_count cannot change */
676 atomic_sub(atomic_read(&page_tail->_count), &page->_count);
677 BUG_ON(page_count(page) <= 0);
678 atomic_add(page_mapcount(page) + 1, &page_tail->_count);
679 BUG_ON(atomic_read(&page_tail->_count) <= 0);
681 /* after clearing PageTail the gup refcount can be released */
684 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
685 page_tail->flags |= (page->flags &
686 ((1L << PG_referenced) |
687 (1L << PG_swapbacked) |
689 (1L << PG_uptodate)));
690 page_tail->flags |= (1L << PG_dirty);
693 * 1) clear PageTail before overwriting first_page
694 * 2) clear PageTail before clearing PageHead for VM_BUG_ON
699 * __split_huge_page_splitting() already set the
700 * splitting bit in all pmd that could map this
701 * hugepage, that will ensure no CPU can alter the
702 * mapcount on the head page. The mapcount is only
703 * accounted in the head page and it has to be
704 * transferred to all tail pages in the below code. So
705 * for this code to be safe, the split the mapcount
706 * can't change. But that doesn't mean userland can't
707 * keep changing and reading the page contents while
708 * we transfer the mapcount, so the pmd splitting
709 * status is achieved setting a reserved bit in the
710 * pmd, not by clearing the present bit.
712 BUG_ON(page_mapcount(page_tail));
713 page_tail->_mapcount = page->_mapcount;
715 BUG_ON(page_tail->mapping);
716 page_tail->mapping = page->mapping;
718 page_tail->index = ++head_index;
720 BUG_ON(!PageAnon(page_tail));
721 BUG_ON(!PageUptodate(page_tail));
722 BUG_ON(!PageDirty(page_tail));
723 BUG_ON(!PageSwapBacked(page_tail));
725 lru_add_page_tail(zone, page, page_tail);
728 ClearPageCompound(page);
729 compound_unlock(page);
730 spin_unlock_irq(&zone->lru_lock);
732 for (i = 1; i < HPAGE_PMD_NR; i++) {
733 struct page *page_tail = page + i;
734 BUG_ON(page_count(page_tail) <= 0);
736 * Tail pages may be freed if there wasn't any mapping
737 * like if add_to_swap() is running on a lru page that
738 * had its mapping zapped. And freeing these pages
739 * requires taking the lru_lock so we do the put_page
740 * of the tail pages after the split is complete.
746 * Only the head page (now become a regular page) is required
747 * to be pinned by the caller.
749 BUG_ON(page_count(page) <= 0);
752 static int __split_huge_page_map(struct page *page,
753 struct vm_area_struct *vma,
754 unsigned long address)
756 struct mm_struct *mm = vma->vm_mm;
762 spin_lock(&mm->page_table_lock);
763 pmd = page_check_address_pmd(page, mm, address,
764 PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG);
766 pgtable = get_pmd_huge_pte(mm);
767 pmd_populate(mm, &_pmd, pgtable);
769 for (i = 0, haddr = address; i < HPAGE_PMD_NR;
770 i++, haddr += PAGE_SIZE) {
772 BUG_ON(PageCompound(page+i));
773 entry = mk_pte(page + i, vma->vm_page_prot);
774 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
775 if (!pmd_write(*pmd))
776 entry = pte_wrprotect(entry);
778 BUG_ON(page_mapcount(page) != 1);
779 if (!pmd_young(*pmd))
780 entry = pte_mkold(entry);
781 pte = pte_offset_map(&_pmd, haddr);
782 BUG_ON(!pte_none(*pte));
783 set_pte_at(mm, haddr, pte, entry);
788 smp_wmb(); /* make pte visible before pmd */
790 * Up to this point the pmd is present and huge and
791 * userland has the whole access to the hugepage
792 * during the split (which happens in place). If we
793 * overwrite the pmd with the not-huge version
794 * pointing to the pte here (which of course we could
795 * if all CPUs were bug free), userland could trigger
796 * a small page size TLB miss on the small sized TLB
797 * while the hugepage TLB entry is still established
798 * in the huge TLB. Some CPU doesn't like that. See
799 * http://support.amd.com/us/Processor_TechDocs/41322.pdf,
800 * Erratum 383 on page 93. Intel should be safe but is
801 * also warns that it's only safe if the permission
802 * and cache attributes of the two entries loaded in
803 * the two TLB is identical (which should be the case
804 * here). But it is generally safer to never allow
805 * small and huge TLB entries for the same virtual
806 * address to be loaded simultaneously. So instead of
807 * doing "pmd_populate(); flush_tlb_range();" we first
808 * mark the current pmd notpresent (atomically because
809 * here the pmd_trans_huge and pmd_trans_splitting
810 * must remain set at all times on the pmd until the
811 * split is complete for this pmd), then we flush the
812 * SMP TLB and finally we write the non-huge version
813 * of the pmd entry with pmd_populate.
815 set_pmd_at(mm, address, pmd, pmd_mknotpresent(*pmd));
816 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
817 pmd_populate(mm, pmd, pgtable);
820 spin_unlock(&mm->page_table_lock);
825 /* must be called with anon_vma->root->lock hold */
826 static void __split_huge_page(struct page *page,
827 struct anon_vma *anon_vma)
829 int mapcount, mapcount2;
830 struct anon_vma_chain *avc;
832 BUG_ON(!PageHead(page));
833 BUG_ON(PageTail(page));
836 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
837 struct vm_area_struct *vma = avc->vma;
838 unsigned long addr = vma_address(page, vma);
839 BUG_ON(is_vma_temporary_stack(vma));
842 mapcount += __split_huge_page_splitting(page, vma, addr);
845 * It is critical that new vmas are added to the tail of the
846 * anon_vma list. This guarantes that if copy_huge_pmd() runs
847 * and establishes a child pmd before
848 * __split_huge_page_splitting() freezes the parent pmd (so if
849 * we fail to prevent copy_huge_pmd() from running until the
850 * whole __split_huge_page() is complete), we will still see
851 * the newly established pmd of the child later during the
852 * walk, to be able to set it as pmd_trans_splitting too.
854 if (mapcount != page_mapcount(page))
855 printk(KERN_ERR "mapcount %d page_mapcount %d\n",
856 mapcount, page_mapcount(page));
857 BUG_ON(mapcount != page_mapcount(page));
859 __split_huge_page_refcount(page);
862 list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
863 struct vm_area_struct *vma = avc->vma;
864 unsigned long addr = vma_address(page, vma);
865 BUG_ON(is_vma_temporary_stack(vma));
868 mapcount2 += __split_huge_page_map(page, vma, addr);
870 if (mapcount != mapcount2)
871 printk(KERN_ERR "mapcount %d mapcount2 %d page_mapcount %d\n",
872 mapcount, mapcount2, page_mapcount(page));
873 BUG_ON(mapcount != mapcount2);
876 int split_huge_page(struct page *page)
878 struct anon_vma *anon_vma;
881 BUG_ON(!PageAnon(page));
882 anon_vma = page_lock_anon_vma(page);
886 if (!PageCompound(page))
889 BUG_ON(!PageSwapBacked(page));
890 __split_huge_page(page, anon_vma);
892 BUG_ON(PageCompound(page));
894 page_unlock_anon_vma(anon_vma);
899 int hugepage_madvise(unsigned long *vm_flags)
902 * Be somewhat over-protective like KSM for now!
904 if (*vm_flags & (VM_HUGEPAGE | VM_SHARED | VM_MAYSHARE |
905 VM_PFNMAP | VM_IO | VM_DONTEXPAND |
906 VM_RESERVED | VM_HUGETLB | VM_INSERTPAGE |
907 VM_MIXEDMAP | VM_SAO))
910 *vm_flags |= VM_HUGEPAGE;
915 void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd)
919 spin_lock(&mm->page_table_lock);
920 if (unlikely(!pmd_trans_huge(*pmd))) {
921 spin_unlock(&mm->page_table_lock);
924 page = pmd_page(*pmd);
925 VM_BUG_ON(!page_count(page));
927 spin_unlock(&mm->page_table_lock);
929 split_huge_page(page);
932 BUG_ON(pmd_trans_huge(*pmd));