srcu: Make Tiny SRCU use full-sized grace-period counters

This commit makes Tiny SRCU use full-sized grace-period counters to
further avoid counter-wrap issues when using polled grace-period APIs.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 4fcec6f5af908764f3073006e394ba8a76f7009b..5aa5e0faf6a121a2770e6362cf368ce3efceefc7 100644 (file)
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -15,10 +15,10 @@
 
 struct srcu_struct {
        short srcu_lock_nesting[2];     /* srcu_read_lock() nesting depth. */
-       unsigned short srcu_idx;        /* Current reader array element in bit 0x2. */
-       unsigned short srcu_idx_max;    /* Furthest future srcu_idx request. */
        u8 srcu_gp_running;             /* GP workqueue running? */
        u8 srcu_gp_waiting;             /* GP waiting for readers? */
+       unsigned long srcu_idx;         /* Current reader array element in bit 0x2. */
+       unsigned long srcu_idx_max;     /* Furthest future srcu_idx request. */
        struct swait_queue_head srcu_wq;
                                        /* Last srcu_read_unlock() wakes GP. */
        struct rcu_head *srcu_cb_head;  /* Pending callbacks: Head. */
@@ -82,7 +82,7 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
        int idx;
 
        idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1;
-       pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %hu->%hu\n",
+       pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n",
                 tt, tf, idx,
                 data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])),
                 data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])),

diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index a2af24f2146761f4974319e93365875e9aea738e..33adafdad26138906557fa78ee024d4bef11965e 100644 (file)
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -117,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp)
        struct srcu_struct *ssp;
 
        ssp = container_of(wp, struct srcu_struct, srcu_work);
-       if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
+       if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
                return; /* Already running or nothing to do. */
 
        /* Remove recently arrived callbacks and wait for readers. */
@@ -150,17 +150,17 @@ void srcu_drive_gp(struct work_struct *wp)
         * straighten that out.
         */
        WRITE_ONCE(ssp->srcu_gp_running, false);
-       if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
+       if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
                schedule_work(&ssp->srcu_work);
 }
 EXPORT_SYMBOL_GPL(srcu_drive_gp);
 
 static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
 {
-       unsigned short cookie;
+       unsigned long cookie;
 
        cookie = get_state_synchronize_srcu(ssp);
-       if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
+       if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
                return;
        WRITE_ONCE(ssp->srcu_idx_max, cookie);
        if (!READ_ONCE(ssp->srcu_gp_running)) {
@@ -215,7 +215,7 @@ unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
        barrier();
        ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
        barrier();
-       return ret & USHRT_MAX;
+       return ret;
 }
 EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
 
@@ -240,10 +240,10 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
  */
 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
 {
-       unsigned short cur_s = READ_ONCE(ssp->srcu_idx);
+       unsigned long cur_s = READ_ONCE(ssp->srcu_idx);
 
        barrier();
-       return USHORT_CMP_GE(cur_s, cookie) || USHORT_CMP_LT(cur_s, cookie - 3);
+       return ULONG_CMP_GE(cur_s, cookie) || ULONG_CMP_LT(cur_s, cookie - 3);
 }
 EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);