rcu: Report expedited grace periods at context-switch time

This commit reduces the latency of expedited RCU grace periods by
reporting a quiescent state for the CPU at context-switch time.
In CONFIG_PREEMPT=y kernels, if the outgoing task is still within an
RCU read-side critical section (and thus still blocking some grace
period, perhaps including this expedited grace period), then that task
will already have been placed on one of the leaf rcu_node structures'
->blkd_tasks list.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index c686bf63bba53a81610e2dbedd1d3767f93c6619..0d7107fb3dec617929c1f5b6bc2c5eedaeede7e8 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -332,7 +332,7 @@ static void rcu_preempt_qs(void)
 static void rcu_preempt_note_context_switch(bool preempt)
 {
        struct task_struct *t = current;
-       struct rcu_data *rdp;
+       struct rcu_data *rdp = this_cpu_ptr(rcu_state_p->rda);
        struct rcu_node *rnp;
 
        lockdep_assert_irqs_disabled();
@@ -341,7 +341,6 @@ static void rcu_preempt_note_context_switch(bool preempt)
            !t->rcu_read_unlock_special.b.blocked) {
 
                /* Possibly blocking in an RCU read-side critical section. */
-               rdp = this_cpu_ptr(rcu_state_p->rda);
                rnp = rdp->mynode;
                raw_spin_lock_rcu_node(rnp);
                t->rcu_read_unlock_special.b.blocked = true;
@@ -383,6 +382,8 @@ static void rcu_preempt_note_context_switch(bool preempt)
         * means that we continue to block the current grace period.
         */
        rcu_preempt_qs();
+       if (rdp->deferred_qs)
+               rcu_report_exp_rdp(rcu_state_p, rdp, true);
 }
 
 /*