rcu: Add full-sized polling for start_poll()

The start_poll_synchronize_rcu() API compresses the combined expedited and
normal grace-period states into a single unsigned long, which conserves
storage, but can miss grace periods in certain cases involving overlapping
normal and expedited grace periods.  Missing the occasional grace period
is usually not a problem, but there are use cases that care about each
and every grace period.

This commit therefore adds the next member of the full-state RCU
grace-period polling API, namely the start_poll_synchronize_rcu_full()
function.  This uses up to three times the storage (rcu_gp_oldstate
structure instead of unsigned long), but is guaranteed not to miss
grace periods.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 6e299955c4e9aca0c1b6b2940b7c29172a7c2a0a..6bc30e46a819c4bf011296f87359335e63cfa8ba 100644 (file)
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -26,6 +26,12 @@ static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
 }
 
 unsigned long start_poll_synchronize_rcu(void);
+
+static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+       rgosp->rgos_norm = start_poll_synchronize_rcu();
+}
+
 bool poll_state_synchronize_rcu(unsigned long oldstate);
 
 static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 7b769f1b417aa947532cd5b6410599457cf73188..8f2e0f0b26f635ccf118e95a0bb14d4a4c250ad4 100644 (file)
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -52,6 +52,7 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate);
 unsigned long get_state_synchronize_rcu(void);
 void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 unsigned long start_poll_synchronize_rcu(void);
+void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 bool poll_state_synchronize_rcu(unsigned long oldstate);
 bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 void cond_synchronize_rcu(unsigned long oldstate);

diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 3d85420108477f5dd4335e8a3f00703b5448c6ab..68387ccc7ddfcf01980a8e8046a1d424611110d2 100644 (file)
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -88,6 +88,7 @@ torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives");
 torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
 torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives");
+torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives");
 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
@@ -198,12 +199,14 @@ static int rcu_torture_writer_state;
 #define RTWS_COND_SYNC         7
 #define RTWS_COND_SYNC_EXP     8
 #define RTWS_POLL_GET          9
-#define RTWS_POLL_GET_EXP      10
-#define RTWS_POLL_WAIT         11
-#define RTWS_POLL_WAIT_EXP     12
-#define RTWS_SYNC              13
-#define RTWS_STUTTER           14
-#define RTWS_STOPPING          15
+#define RTWS_POLL_GET_FULL     10
+#define RTWS_POLL_GET_EXP      11
+#define RTWS_POLL_WAIT         12
+#define RTWS_POLL_WAIT_FULL    13
+#define RTWS_POLL_WAIT_EXP     14
+#define RTWS_SYNC              15
+#define RTWS_STUTTER           16
+#define RTWS_STOPPING          17
 static const char * const rcu_torture_writer_state_names[] = {
        "RTWS_FIXED_DELAY",
        "RTWS_DELAY",
@@ -215,8 +218,10 @@ static const char * const rcu_torture_writer_state_names[] = {
        "RTWS_COND_SYNC",
        "RTWS_COND_SYNC_EXP",
        "RTWS_POLL_GET",
+       "RTWS_POLL_GET_FULL",
        "RTWS_POLL_GET_EXP",
        "RTWS_POLL_WAIT",
+       "RTWS_POLL_WAIT_FULL",
        "RTWS_POLL_WAIT_EXP",
        "RTWS_SYNC",
        "RTWS_STUTTER",
@@ -339,6 +344,7 @@ struct rcu_torture_ops {
        unsigned long (*get_gp_completed)(void);
        void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp);
        unsigned long (*start_gp_poll)(void);
+       void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp);
        bool (*poll_gp_state)(unsigned long oldstate);
        bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp);
        bool (*poll_need_2gp)(bool poll, bool poll_full);
@@ -515,6 +521,7 @@ static struct rcu_torture_ops rcu_ops = {
        .get_gp_completed       = get_completed_synchronize_rcu,
        .get_gp_completed_full  = get_completed_synchronize_rcu_full,
        .start_gp_poll          = start_poll_synchronize_rcu,
+       .start_gp_poll_full     = start_poll_synchronize_rcu_full,
        .poll_gp_state          = poll_state_synchronize_rcu,
        .poll_gp_state_full     = poll_state_synchronize_rcu_full,
        .poll_need_2gp          = rcu_poll_need_2gp,
@@ -1163,13 +1170,13 @@ static void rcu_torture_write_types(void)
 {
        bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp;
        bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll;
-       bool gp_sync1 = gp_sync;
+       bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;
 
        /* Initialize synctype[] array.  If none set, take default. */
        if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp &&
-           !gp_normal1 && !gp_poll1 && !gp_sync1)
+           !gp_normal1 && !gp_poll1 && !gp_poll_full1 && !gp_sync1)
                gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 =
-                          gp_normal1 = gp_poll1 = gp_sync1 = true;
+                          gp_normal1 = gp_poll1 = gp_poll_full1 = gp_sync1 = true;
        if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
                synctype[nsynctypes++] = RTWS_COND_GET;
                pr_info("%s: Testing conditional GPs.\n", __func__);
@@ -1200,6 +1207,12 @@ static void rcu_torture_write_types(void)
        } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
                pr_alert("%s: gp_poll without primitives.\n", __func__);
        }
+       if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
+               synctype[nsynctypes++] = RTWS_POLL_GET_FULL;
+               pr_info("%s: Testing polling full-state GPs.\n", __func__);
+       } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {
+               pr_alert("%s: gp_poll_full without primitives.\n", __func__);
+       }
        if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {
                synctype[nsynctypes++] = RTWS_POLL_GET_EXP;
                pr_info("%s: Testing polling expedited GPs.\n", __func__);
@@ -1262,6 +1275,7 @@ rcu_torture_writer(void *arg)
        struct rcu_gp_oldstate cookie_full;
        int expediting = 0;
        unsigned long gp_snap;
+       struct rcu_gp_oldstate gp_snap_full;
        int i;
        int idx;
        int oldnice = task_nice(current);
@@ -1376,6 +1390,15 @@ rcu_torture_writer(void *arg)
                                                                  &rand);
                                rcu_torture_pipe_update(old_rp);
                                break;
+                       case RTWS_POLL_GET_FULL:
+                               rcu_torture_writer_state = RTWS_POLL_GET_FULL;
+                               cur_ops->start_gp_poll_full(&gp_snap_full);
+                               rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
+                               while (!cur_ops->poll_gp_state_full(&gp_snap_full))
+                                       torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+                                                                 &rand);
+                               rcu_torture_pipe_update(old_rp);
+                               break;
                        case RTWS_POLL_GET_EXP:
                                rcu_torture_writer_state = RTWS_POLL_GET_EXP;
                                gp_snap = cur_ops->start_gp_poll_exp();
@@ -1454,6 +1477,7 @@ static int
 rcu_torture_fakewriter(void *arg)
 {
        unsigned long gp_snap;
+       struct rcu_gp_oldstate gp_snap_full;
        DEFINE_TORTURE_RANDOM(rand);
 
        VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
@@ -1499,6 +1523,13 @@ rcu_torture_fakewriter(void *arg)
                                                                  &rand);
                                }
                                break;
+                       case RTWS_POLL_GET_FULL:
+                               cur_ops->start_gp_poll_full(&gp_snap_full);
+                               while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
+                                       torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+                                                                 &rand);
+                               }
+                               break;
                        case RTWS_POLL_GET_EXP:
                                gp_snap = cur_ops->start_gp_poll_exp();
                                while (!cur_ops->poll_gp_state_exp(gp_snap)) {

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 3fa79ee78b5b4f321dfe3b2ced57602dfd522da6..89572385fd1aa9ebe4b80848ebc1baec881b07ac 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3589,22 +3589,13 @@ void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
 }
 EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full);
 
-/**
- * start_poll_synchronize_rcu - Snapshot and start RCU grace period
- *
- * Returns a cookie that is used by a later call to cond_synchronize_rcu()
- * or poll_state_synchronize_rcu() to determine whether or not a full
- * grace period has elapsed in the meantime.  If the needed grace period
- * is not already slated to start, notifies RCU core of the need for that
- * grace period.
- *
- * Interrupts must be enabled for the case where it is necessary to awaken
- * the grace-period kthread.
+/*
+ * Helper function for start_poll_synchronize_rcu() and
+ * start_poll_synchronize_rcu_full().
  */
-unsigned long start_poll_synchronize_rcu(void)
+static void start_poll_synchronize_rcu_common(void)
 {
        unsigned long flags;
-       unsigned long gp_seq = get_state_synchronize_rcu();
        bool needwake;
        struct rcu_data *rdp;
        struct rcu_node *rnp;
@@ -3624,10 +3615,51 @@ unsigned long start_poll_synchronize_rcu(void)
        raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
        if (needwake)
                rcu_gp_kthread_wake();
+}
+
+/**
+ * start_poll_synchronize_rcu - Snapshot and start RCU grace period
+ *
+ * Returns a cookie that is used by a later call to cond_synchronize_rcu()
+ * or poll_state_synchronize_rcu() to determine whether or not a full
+ * grace period has elapsed in the meantime.  If the needed grace period
+ * is not already slated to start, notifies RCU core of the need for that
+ * grace period.
+ *
+ * Interrupts must be enabled for the case where it is necessary to awaken
+ * the grace-period kthread.
+ */
+unsigned long start_poll_synchronize_rcu(void)
+{
+       unsigned long gp_seq = get_state_synchronize_rcu();
+
+       start_poll_synchronize_rcu_common();
        return gp_seq;
 }
 EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu);
 
+/**
+ * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period
+ * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full()
+ *
+ * Places the normal and expedited grace-period states in *@rgos.  This
+ * state value can be passed to a later call to cond_synchronize_rcu_full()
+ * or poll_state_synchronize_rcu_full() to determine whether or not a
+ * grace period (whether normal or expedited) has elapsed in the meantime.
+ * If the needed grace period is not already slated to start, notifies
+ * RCU core of the need for that grace period.
+ *
+ * Interrupts must be enabled for the case where it is necessary to awaken
+ * the grace-period kthread.
+ */
+void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+       get_state_synchronize_rcu_full(rgosp);
+
+       start_poll_synchronize_rcu_common();
+}
+EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full);
+
 /**
  * poll_state_synchronize_rcu - Has the specified RCU grace period completed?
  *