mm, vmacache: hash addresses based on pmd

[linux-2.6-block.git] / mm / vmacache.c

diff --git a/mm/vmacache.c b/mm/vmacache.c
index db7596eb6132e8118b73e9c851563cde4a6d376a..ea517bef7dc552a10ac6ed424ca06aa50670c303 100644 (file)
--- a/mm/vmacache.c
+++ b/mm/vmacache.c
@@ -6,6 +6,18 @@
 #include <linux/sched/task.h>
 #include <linux/mm.h>
 #include <linux/vmacache.h>
+#include <asm/pgtable.h>
+
+/*
+ * Hash based on the pmd of addr if configured with MMU, which provides a good
+ * hit rate for workloads with spatial locality.  Otherwise, use pages.
+ */
+#ifdef CONFIG_MMU
+#define VMACACHE_SHIFT PMD_SHIFT
+#else
+#define VMACACHE_SHIFT PAGE_SHIFT
+#endif
+#define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK)
 
 /*
  * Flush vma caches for threads that share a given mm.
@@ -87,6 +99,7 @@ static bool vmacache_valid(struct mm_struct *mm)
 
 struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
 {
+       int idx = VMACACHE_HASH(addr);
        int i;
 
        count_vm_vmacache_event(VMACACHE_FIND_CALLS);
@@ -95,16 +108,20 @@ struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)
                return NULL;
 
        for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[i];
+               struct vm_area_struct *vma = current->vmacache.vmas[idx];
 
-               if (!vma)
-                       continue;
-               if (WARN_ON_ONCE(vma->vm_mm != mm))
-                       break;
-               if (vma->vm_start <= addr && vma->vm_end > addr) {
-                       count_vm_vmacache_event(VMACACHE_FIND_HITS);
-                       return vma;
+               if (vma) {
+#ifdef CONFIG_DEBUG_VM_VMACACHE
+                       if (WARN_ON_ONCE(vma->vm_mm != mm))
+                               break;
+#endif
+                       if (vma->vm_start <= addr && vma->vm_end > addr) {
+                               count_vm_vmacache_event(VMACACHE_FIND_HITS);
+                               return vma;
+                       }
                }
+               if (++idx == VMACACHE_SIZE)
+                       idx = 0;
        }
 
        return NULL;
@@ -115,6 +132,7 @@ struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
                                           unsigned long start,
                                           unsigned long end)
 {
+       int idx = VMACACHE_HASH(start);
        int i;
 
        count_vm_vmacache_event(VMACACHE_FIND_CALLS);
@@ -123,12 +141,14 @@ struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,
                return NULL;
 
        for (i = 0; i < VMACACHE_SIZE; i++) {
-               struct vm_area_struct *vma = current->vmacache.vmas[i];
+               struct vm_area_struct *vma = current->vmacache.vmas[idx];
 
                if (vma && vma->vm_start == start && vma->vm_end == end) {
                        count_vm_vmacache_event(VMACACHE_FIND_HITS);
                        return vma;
                }
+               if (++idx == VMACACHE_SIZE)
+                       idx = 0;
        }
 
        return NULL;