rcu: Accelerate grace period if last non-dynticked CPU

[linux-2.6-block.git] / kernel / rcutree_plugin.h

diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index 37fbccdf41d58d4dda85cc1788d972b6602d79b2..a82566696b0b55ab2433507e824374a124139a30 100644 (file)
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -61,6 +61,15 @@ long rcu_batches_completed(void)
 }
 EXPORT_SYMBOL_GPL(rcu_batches_completed);
 
+/*
+ * Force a quiescent state for preemptible RCU.
+ */
+void rcu_force_quiescent_state(void)
+{
+       force_quiescent_state(&rcu_preempt_state, 0);
+}
+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
+
 /*
  * Record a preemptable-RCU quiescent state for the specified CPU.  Note
  * that this just means that the task currently running on the CPU is
@@ -295,6 +304,9 @@ void __rcu_read_unlock(void)
        if (--ACCESS_ONCE(t->rcu_read_lock_nesting) == 0 &&
            unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
                rcu_read_unlock_special(t);
+#ifdef CONFIG_PROVE_LOCKING
+       WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
+#endif /* #ifdef CONFIG_PROVE_LOCKING */
 }
 EXPORT_SYMBOL_GPL(__rcu_read_unlock);
 
@@ -712,6 +724,16 @@ long rcu_batches_completed(void)
 }
 EXPORT_SYMBOL_GPL(rcu_batches_completed);
 
+/*
+ * Force a quiescent state for RCU, which, because there is no preemptible
+ * RCU, becomes the same as rcu-sched.
+ */
+void rcu_force_quiescent_state(void)
+{
+       rcu_sched_force_quiescent_state();
+}
+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
+
 /*
  * Because preemptable RCU does not exist, we never have to check for
  * CPUs being in quiescent states.
@@ -884,3 +906,72 @@ static void __init __rcu_init_preempt(void)
 }
 
 #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
+
+#if !defined(CONFIG_RCU_FAST_NO_HZ)
+
+/*
+ * Check to see if any future RCU-related work will need to be done
+ * by the current CPU, even if none need be done immediately, returning
+ * 1 if so.  This function is part of the RCU implementation; it is -not-
+ * an exported member of the RCU API.
+ *
+ * Because we have preemptible RCU, just check whether this CPU needs
+ * any flavor of RCU.  Do not chew up lots of CPU cycles with preemption
+ * disabled in a most-likely vain attempt to cause RCU not to need this CPU.
+ */
+int rcu_needs_cpu(int cpu)
+{
+       return rcu_needs_cpu_quick_check(cpu);
+}
+
+#else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
+
+#define RCU_NEEDS_CPU_FLUSHES 5
+
+/*
+ * Check to see if any future RCU-related work will need to be done
+ * by the current CPU, even if none need be done immediately, returning
+ * 1 if so.  This function is part of the RCU implementation; it is -not-
+ * an exported member of the RCU API.
+ *
+ * Because we are not supporting preemptible RCU, attempt to accelerate
+ * any current grace periods so that RCU no longer needs this CPU, but
+ * only if all other CPUs are already in dynticks-idle mode.  This will
+ * allow the CPU cores to be powered down immediately, as opposed to after
+ * waiting many milliseconds for grace periods to elapse.
+ */
+int rcu_needs_cpu(int cpu)
+{
+       int c = 1;
+       int i;
+       int thatcpu;
+
+       /* Don't bother unless we are the last non-dyntick-idle CPU. */
+       for_each_cpu_not(thatcpu, nohz_cpu_mask)
+               if (thatcpu != cpu)
+                       return rcu_needs_cpu_quick_check(cpu);
+
+       /* Try to push remaining RCU-sched and RCU-bh callbacks through. */
+       for (i = 0; i < RCU_NEEDS_CPU_FLUSHES && c; i++) {
+               c = 0;
+               if (per_cpu(rcu_sched_data, cpu).nxtlist) {
+                       rcu_sched_qs(cpu);
+                       force_quiescent_state(&rcu_sched_state, 0);
+                       __rcu_process_callbacks(&rcu_sched_state,
+                                               &per_cpu(rcu_sched_data, cpu));
+                       c = !!per_cpu(rcu_sched_data, cpu).nxtlist;
+               }
+               if (per_cpu(rcu_bh_data, cpu).nxtlist) {
+                       rcu_bh_qs(cpu);
+                       force_quiescent_state(&rcu_bh_state, 0);
+                       __rcu_process_callbacks(&rcu_bh_state,
+                                               &per_cpu(rcu_bh_data, cpu));
+                       c = !!per_cpu(rcu_bh_data, cpu).nxtlist;
+               }
+       }
+
+       /* If RCU callbacks are still pending, RCU still needs this CPU. */
+       return c;
+}
+
+#endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */