lazy tlb: introduce lazy tlb mm refcount helper functions

[linux-block.git] / kernel / kthread.c

diff --git a/kernel/kthread.c b/kernel/kthread.c
index 1f1b60f1a7466d2eb5c5ed5c56d169dce702c964..470708c205e8982030966e41c5f3d4d8c4a9113b 100644 (file)
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -1415,6 +1415,11 @@ void kthread_use_mm(struct mm_struct *mm)
        WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD));
        WARN_ON_ONCE(tsk->mm);
 
+       /*
+        * It is possible for mm to be the same as tsk->active_mm, but
+        * we must still mmgrab(mm) and mmdrop_lazy_tlb(active_mm),
+        * because these references are not equivalent.
+        */
        mmgrab(mm);
 
        task_lock(tsk);
@@ -1438,9 +1443,9 @@ void kthread_use_mm(struct mm_struct *mm)
         * memory barrier after storing to tsk->mm, before accessing
         * user-space memory. A full memory barrier for membarrier
         * {PRIVATE,GLOBAL}_EXPEDITED is implicitly provided by
-        * mmdrop().
+        * mmdrop_lazy_tlb().
         */
-       mmdrop(active_mm);
+       mmdrop_lazy_tlb(active_mm);
 }
 EXPORT_SYMBOL_GPL(kthread_use_mm);
 
@@ -1468,10 +1473,13 @@ void kthread_unuse_mm(struct mm_struct *mm)
        local_irq_disable();
        tsk->mm = NULL;
        membarrier_update_current_mm(NULL);
+       mmgrab_lazy_tlb(mm);
        /* active_mm is still 'mm' */
        enter_lazy_tlb(mm, tsk);
        local_irq_enable();
        task_unlock(tsk);
+
+       mmdrop(mm);
 }
 EXPORT_SYMBOL_GPL(kthread_unuse_mm);