rcu-tasks: Abstract checking of callback lists

This commit adds a rcu_tasks_need_gpcb() function that returns an
indication of whether another grace period is required, and if no grace
period is required, whether there are callbacks that need to be invoked.
The function scans all per-CPU lists currently in use.

Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 608af87a50658e40c8494e836bad90c62e0eead7..e63987a5af56ee45d34b654e8e36fb3d36710d39 100644 (file)
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -229,11 +229,38 @@ static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp)
        wait_rcu_gp(rtp->call_func);
 }
 
+// Advance callbacks and indicate whether either a grace period or
+// callback invocation is needed.
+static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)
+{
+       int cpu;
+       unsigned long flags;
+       int needgpcb = 0;
+
+       for (cpu = 0; cpu < rtp->percpu_enqueue_lim; cpu++) {
+               struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
+
+               /* Advance and accelerate any new callbacks. */
+               if (rcu_segcblist_empty(&rtpcp->cblist))
+                       continue;
+               raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
+               rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
+               (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));
+               if (rcu_segcblist_pend_cbs(&rtpcp->cblist))
+                       needgpcb |= 0x3;
+               if (!rcu_segcblist_empty(&rtpcp->cblist))
+                       needgpcb |= 0x1;
+               raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
+       }
+       return needgpcb;
+}
+
 /* RCU-tasks kthread that detects grace periods and invokes callbacks. */
 static int __noreturn rcu_tasks_kthread(void *arg)
 {
        unsigned long flags;
        int len;
+       int needgpcb;
        struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
        struct rcu_head *rhp;
        struct rcu_tasks *rtp = arg;
@@ -249,37 +276,25 @@ static int __noreturn rcu_tasks_kthread(void *arg)
         * This loop is terminated by the system going down.  ;-)
         */
        for (;;) {
-               struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, 0);  // for_each...
+               struct rcu_tasks_percpu *rtpcp;
 
                set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
 
-               /* Pick up any new callbacks. */
-               raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
-               rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
-               (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));
-               raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
-
                /* If there were none, wait a bit and start over. */
-               if (!rcu_segcblist_pend_cbs(&rtpcp->cblist)) {
-                       wait_event_interruptible(rtp->cbs_wq,
-                                                rcu_segcblist_pend_cbs(&rtpcp->cblist));
-                       if (!rcu_segcblist_pend_cbs(&rtpcp->cblist)) {
-                               WARN_ON(signal_pending(current));
-                               set_tasks_gp_state(rtp, RTGS_WAIT_WAIT_CBS);
-                               schedule_timeout_idle(HZ/10);
-                       }
-                       continue;
+               wait_event_idle(rtp->cbs_wq, (needgpcb = rcu_tasks_need_gpcb(rtp)));
+
+               if (needgpcb & 0x2) {
+                       // Wait for one grace period.
+                       set_tasks_gp_state(rtp, RTGS_WAIT_GP);
+                       rtp->gp_start = jiffies;
+                       rcu_seq_start(&rtp->tasks_gp_seq);
+                       rtp->gp_func(rtp);
+                       rcu_seq_end(&rtp->tasks_gp_seq);
                }
 
-               // Wait for one grace period.
-               set_tasks_gp_state(rtp, RTGS_WAIT_GP);
-               rtp->gp_start = jiffies;
-               rcu_seq_start(&rtp->tasks_gp_seq);
-               rtp->gp_func(rtp);
-               rcu_seq_end(&rtp->tasks_gp_seq);
-
                /* Invoke the callbacks. */
                set_tasks_gp_state(rtp, RTGS_INVOKE_CBS);
+               rtpcp = per_cpu_ptr(rtp->rtpcpu, 0);
                raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
                rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
                rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl);