mm: mmu_gather: remove __tlb_reset_range() for force flush

[linux-2.6-block.git] / mm / mmu_gather.c

diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 99740e1dd27304b31097eb3deedc309a6c398490..8c943a6e1696c095d77ae65bb5d784cc09a85afd 100644 (file)
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -245,14 +245,28 @@ void tlb_finish_mmu(struct mmu_gather *tlb,
 {
        /*
         * If there are parallel threads are doing PTE changes on same range
-        * under non-exclusive lock(e.g., mmap_sem read-side) but defer TLB
-        * flush by batching, a thread has stable TLB entry can fail to flush
-        * the TLB by observing pte_none|!pte_dirty, for example so flush TLB
-        * forcefully if we detect parallel PTE batching threads.
+        * under non-exclusive lock (e.g., mmap_sem read-side) but defer TLB
+        * flush by batching, one thread may end up seeing inconsistent PTEs
+        * and result in having stale TLB entries.  So flush TLB forcefully
+        * if we detect parallel PTE batching threads.
+        *
+        * However, some syscalls, e.g. munmap(), may free page tables, this
+        * needs force flush everything in the given range. Otherwise this
+        * may result in having stale TLB entries for some architectures,
+        * e.g. aarch64, that could specify flush what level TLB.
         */
        if (mm_tlb_flush_nested(tlb->mm)) {
+               /*
+                * The aarch64 yields better performance with fullmm by
+                * avoiding multiple CPUs spamming TLBI messages at the
+                * same time.
+                *
+                * On x86 non-fullmm doesn't yield significant difference
+                * against fullmm.
+                */
+               tlb->fullmm = 1;
                __tlb_reset_range(tlb);
-               __tlb_adjust_range(tlb, start, end - start);
+               tlb->freed_tables = 1;
        }
 
        tlb_flush_mmu(tlb);