ext4: fix performance regression in ext4_writepages

[linux-2.6-block.git] / kernel / rcutree.c

diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 64eaafb6c8f70c9797841d81b40898598650f812..32618b3fe4e6aa375b3cd3bde1acd12bdd4237b3 100644 (file)
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -54,6 +54,7 @@
 #include <linux/stop_machine.h>
 #include <linux/random.h>
 #include <linux/ftrace_event.h>
+#include <linux/suspend.h>
 
 #include "rcutree.h"
 #include <trace/events/rcu.h>
@@ -1556,10 +1557,12 @@ rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp,
 
        /*
         * We can't do wakeups while holding the rnp->lock, as that
-        * could cause possible deadlocks with the rq->lock. Deter
-        * the wakeup to interrupt context.
+        * could cause possible deadlocks with the rq->lock. Defer
+        * the wakeup to interrupt context.  And don't bother waking
+        * up the running kthread.
         */
-       irq_work_queue(&rsp->wakeup_work);
+       if (current != rsp->gp_kthread)
+               irq_work_queue(&rsp->wakeup_work);
 }
 
 /*
@@ -2290,6 +2293,13 @@ static void __call_rcu_core(struct rcu_state *rsp, struct rcu_data *rdp,
        }
 }
 
+/*
+ * RCU callback function to leak a callback.
+ */
+static void rcu_leak_callback(struct rcu_head *rhp)
+{
+}
+
 /*
  * Helper function for call_rcu() and friends.  The cpu argument will
  * normally be -1, indicating "currently running CPU".  It may specify
@@ -2304,7 +2314,12 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
        struct rcu_data *rdp;
 
        WARN_ON_ONCE((unsigned long)head & 0x3); /* Misaligned rcu_head! */
-       debug_rcu_head_queue(head);
+       if (debug_rcu_head_queue(head)) {
+               /* Probable double call_rcu(), so leak the callback. */
+               ACCESS_ONCE(head->func) = rcu_leak_callback;
+               WARN_ONCE(1, "__call_rcu(): Leaked duplicate callback\n");
+               return;
+       }
        head->func = func;
        head->next = NULL;
 
@@ -2789,9 +2804,20 @@ static void _rcu_barrier(struct rcu_state *rsp)
         * transition.  The "if" expression below therefore rounds the old
         * value up to the next even number and adds two before comparing.
         */
-       snap_done = ACCESS_ONCE(rsp->n_barrier_done);
+       snap_done = rsp->n_barrier_done;
        _rcu_barrier_trace(rsp, "Check", -1, snap_done);
-       if (ULONG_CMP_GE(snap_done, ((snap + 1) & ~0x1) + 2)) {
+
+       /*
+        * If the value in snap is odd, we needed to wait for the current
+        * rcu_barrier() to complete, then wait for the next one, in other
+        * words, we need the value of snap_done to be three larger than
+        * the value of snap.  On the other hand, if the value in snap is
+        * even, we only had to wait for the next rcu_barrier() to complete,
+        * in other words, we need the value of snap_done to be only two
+        * greater than the value of snap.  The "(snap + 3) & ~0x1" computes
+        * this for us (thank you, Linus!).
+        */
+       if (ULONG_CMP_GE(snap_done, (snap + 3) & ~0x1)) {
                _rcu_barrier_trace(rsp, "EarlyExit", -1, snap_done);
                smp_mb(); /* caller's subsequent code after above check. */
                mutex_unlock(&rsp->barrier_mutex);
@@ -3020,6 +3046,25 @@ static int rcu_cpu_notify(struct notifier_block *self,
        return NOTIFY_OK;
 }
 
+static int rcu_pm_notify(struct notifier_block *self,
+                        unsigned long action, void *hcpu)
+{
+       switch (action) {
+       case PM_HIBERNATION_PREPARE:
+       case PM_SUSPEND_PREPARE:
+               if (nr_cpu_ids <= 256) /* Expediting bad for large systems. */
+                       rcu_expedited = 1;
+               break;
+       case PM_POST_HIBERNATION:
+       case PM_POST_SUSPEND:
+               rcu_expedited = 0;
+               break;
+       default:
+               break;
+       }
+       return NOTIFY_OK;
+}
+
 /*
  * Spawn the kthread that handles this RCU flavor's grace periods.
  */
@@ -3261,6 +3306,7 @@ void __init rcu_init(void)
         * or the scheduler are operational.
         */
        cpu_notifier(rcu_cpu_notify, 0);
+       pm_notifier(rcu_pm_notify, 0);
        for_each_online_cpu(cpu)
                rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
 }