sched/kcpustat: Introduce vtime-aware kcpustat accessor for CPUTIME_SYSTEM

Kcpustat is not correctly supported on nohz_full CPUs. The tick doesn't
fire and the cputime therefore doesn't move forward. The issue has shown
up after the vanishing of the remaining 1Hz which has made the stall
visible.

We are solving that with checking the task running on a CPU through RCU
and reading its vtime delta that we add to the raw kcpustat values.

We make sure that we fetch a coherent raw-kcpustat/vtime-delta couple
sequence while checking that the CPU referred by the target vtime is the
correct one, under the locked vtime seqcount.

Only CPUTIME_SYSTEM is handled here as a start because it's the trivial
case. User and guest time will require more preparation work to
correctly handle niceness.

Reported-by: Yauheni Kaliuta <yauheni.kaliuta@redhat.com>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@surriel.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Wanpeng Li <wanpengli@tencent.com>
Link: https://lkml.kernel.org/r/20191025020303.19342-1-frederic@kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index 7ee2bb43b251a890aae37c56d0ee8edde7012ac5..79781196eb2542057a7e3be7f4fe610b98a9e53b 100644 (file)
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -78,6 +78,17 @@ static inline unsigned int kstat_cpu_irqs_sum(unsigned int cpu)
        return kstat_cpu(cpu).irqs_sum;
 }
 
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
+extern u64 kcpustat_field(struct kernel_cpustat *kcpustat,
+                         enum cpu_usage_stat usage, int cpu);
+#else
+static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat,
+                                enum cpu_usage_stat usage, int cpu)
+{
+       return kcpustat->cpustat[usage];
+}
+#endif
+
 extern void account_user_time(struct task_struct *, u64);
 extern void account_guest_time(struct task_struct *, u64);
 extern void account_system_time(struct task_struct *, int, u64);

diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index b931a19df09326f61fbefc2773857215adcca8e5..e0cd20693ef56a6ebfac68088df6d65b9cd3b881 100644 (file)
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -911,4 +911,86 @@ void task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
                        *utime += vtime->utime + delta;
        } while (read_seqcount_retry(&vtime->seqcount, seq));
 }
+
+static int kcpustat_field_vtime(u64 *cpustat,
+                               struct vtime *vtime,
+                               enum cpu_usage_stat usage,
+                               int cpu, u64 *val)
+{
+       unsigned int seq;
+       int err;
+
+       do {
+               seq = read_seqcount_begin(&vtime->seqcount);
+
+               /*
+                * We raced against context switch, fetch the
+                * kcpustat task again.
+                */
+               if (vtime->cpu != cpu && vtime->cpu != -1)
+                       return -EAGAIN;
+
+               /*
+                * Two possible things here:
+                * 1) We are seeing the scheduling out task (prev) or any past one.
+                * 2) We are seeing the scheduling in task (next) but it hasn't
+                *    passed though vtime_task_switch() yet so the pending
+                *    cputime of the prev task may not be flushed yet.
+                *
+                * Case 1) is ok but 2) is not. So wait for a safe VTIME state.
+                */
+               if (vtime->state == VTIME_INACTIVE)
+                       return -EAGAIN;
+
+               err = 0;
+
+               *val = cpustat[usage];
+
+               if (vtime->state == VTIME_SYS)
+                       *val += vtime->stime + vtime_delta(vtime);
+
+       } while (read_seqcount_retry(&vtime->seqcount, seq));
+
+       return 0;
+}
+
+u64 kcpustat_field(struct kernel_cpustat *kcpustat,
+                  enum cpu_usage_stat usage, int cpu)
+{
+       u64 *cpustat = kcpustat->cpustat;
+       struct rq *rq;
+       u64 val;
+       int err;
+
+       if (!vtime_accounting_enabled_cpu(cpu))
+               return cpustat[usage];
+
+       /* Only support sys vtime for now */
+       if (usage != CPUTIME_SYSTEM)
+               return cpustat[usage];
+
+       rq = cpu_rq(cpu);
+
+       for (;;) {
+               struct task_struct *curr;
+               struct vtime *vtime;
+
+               rcu_read_lock();
+               curr = rcu_dereference(rq->curr);
+               if (WARN_ON_ONCE(!curr)) {
+                       rcu_read_unlock();
+                       return cpustat[usage];
+               }
+
+               vtime = &curr->vtime;
+               err = kcpustat_field_vtime(cpustat, vtime, usage, cpu, &val);
+               rcu_read_unlock();
+
+               if (!err)
+                       return val;
+
+               cpu_relax();
+       }
+}
+EXPORT_SYMBOL_GPL(kcpustat_field);
 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */