sched/rt: Add rt_rq utilization tracking

schedutil governor relies on cfs_rq's util_avg to choose the OPP when CFS
tasks are running. When the CPU is overloaded by CFS and RT tasks, CFS tasks
are preempted by RT tasks and in this case util_avg reflects the remaining
capacity but not what CFS want to use. In such case, schedutil can select a
lower OPP whereas the CPU is overloaded. In order to have a more accurate
view of the utilization of the CPU, we track the utilization of RT tasks.
Only util_avg is correctly tracked but not load_avg and runnable_load_avg
which are useless for rt_rq.

rt_rq uses rq_clock_task and cfs_rq uses cfs_rq_clock_task but they are
the same at the root group level, so the PELT windows of the util_sum are
aligned.

Signed-off-by: Vincent Guittot <vincent.guittot@linaro.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Morten.Rasmussen@arm.com
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: claudio@evidence.eu.com
Cc: daniel.lezcano@linaro.org
Cc: dietmar.eggemann@arm.com
Cc: joel@joelfernandes.org
Cc: juri.lelli@redhat.com
Cc: luca.abeni@santannapisa.it
Cc: patrick.bellasi@arm.com
Cc: quentin.perret@arm.com
Cc: rjw@rjwysocki.net
Cc: valentin.schneider@arm.com
Cc: viresh.kumar@linaro.org
Link: http://lkml.kernel.org/r/1530200714-4504-3-git-send-email-vincent.guittot@linaro.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 39ab46cea6c5008549f6a246d13b6ee14c891171..5b453213cd18b4f488ede2fea6433268c52e0de7 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7290,6 +7290,14 @@ static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq)
        return false;
 }
 
+static inline bool rt_rq_has_blocked(struct rq *rq)
+{
+       if (READ_ONCE(rq->avg_rt.util_avg))
+               return true;
+
+       return false;
+}
+
 #ifdef CONFIG_FAIR_GROUP_SCHED
 
 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
@@ -7349,6 +7357,10 @@ static void update_blocked_averages(int cpu)
                if (cfs_rq_has_blocked(cfs_rq))
                        done = false;
        }
+       update_rt_rq_load_avg(rq_clock_task(rq), rq, 0);
+       /* Don't need periodic decay once load/util_avg are null */
+       if (rt_rq_has_blocked(rq))
+               done = false;
 
 #ifdef CONFIG_NO_HZ_COMMON
        rq->last_blocked_load_update_tick = jiffies;
@@ -7414,9 +7426,10 @@ static inline void update_blocked_averages(int cpu)
        rq_lock_irqsave(rq, &rf);
        update_rq_clock(rq);
        update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq);
+       update_rt_rq_load_avg(rq_clock_task(rq), rq, 0);
 #ifdef CONFIG_NO_HZ_COMMON
        rq->last_blocked_load_update_tick = jiffies;
-       if (!cfs_rq_has_blocked(cfs_rq))
+       if (!cfs_rq_has_blocked(cfs_rq) && !rt_rq_has_blocked(rq))
                rq->has_blocked_load = 0;
 #endif
        rq_unlock_irqrestore(rq, &rf);

diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c
index e6ecbb2b8698ba1a749d4278ec7bebf073ef9554..a00b1ba3dd5b617ebb8578e720d6c6df99c2e35a 100644 (file)
--- a/kernel/sched/pelt.c
+++ b/kernel/sched/pelt.c
@@ -309,3 +309,28 @@ int __update_load_avg_cfs_rq(u64 now, int cpu, struct cfs_rq *cfs_rq)
 
        return 0;
 }
+
+/*
+ * rt_rq:
+ *
+ *   util_sum = \Sum se->avg.util_sum but se->avg.util_sum is not tracked
+ *   util_sum = cpu_scale * load_sum
+ *   runnable_load_sum = load_sum
+ *
+ *   load_avg and runnable_load_avg are not supported and meaningless.
+ *
+ */
+
+int update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
+{
+       if (___update_load_sum(now, rq->cpu, &rq->avg_rt,
+                               running,
+                               running,
+                               running)) {
+
+               ___update_load_avg(&rq->avg_rt, 1, 1);
+               return 1;
+       }
+
+       return 0;
+}

diff --git a/kernel/sched/pelt.h b/kernel/sched/pelt.h
index 9cac73efd64ae6de8979f28b10c3cdc6ec4e5b09..b2983b741d577c833166cceed5569238e36fbb07 100644 (file)
--- a/kernel/sched/pelt.h
+++ b/kernel/sched/pelt.h
@@ -3,6 +3,7 @@
 int __update_load_avg_blocked_se(u64 now, int cpu, struct sched_entity *se);
 int __update_load_avg_se(u64 now, int cpu, struct cfs_rq *cfs_rq, struct sched_entity *se);
 int __update_load_avg_cfs_rq(u64 now, int cpu, struct cfs_rq *cfs_rq);
+int update_rt_rq_load_avg(u64 now, struct rq *rq, int running);
 
 /*
  * When a task is dequeued, its estimated utilization should not be update if
@@ -38,6 +39,12 @@ update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
        return 0;
 }
 
+static inline int
+update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
+{
+       return 0;
+}
+
 #endif
 
 

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 572567078b60b59dd14e3a608c6e9207f1c7e024..0dc8ad1915e6927a1d5b18bb53a4a190ff7bc0fb 100644 (file)
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -5,6 +5,8 @@
  */
 #include "sched.h"
 
+#include "pelt.h"
+
 int sched_rr_timeslice = RR_TIMESLICE;
 int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;
 
@@ -1576,6 +1578,14 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 
        rt_queue_push_tasks(rq);
 
+       /*
+        * If prev task was rt, put_prev_task() has already updated the
+        * utilization. We only care of the case where we start to schedule a
+        * rt task
+        */
+       if (rq->curr->sched_class != &rt_sched_class)
+               update_rt_rq_load_avg(rq_clock_task(rq), rq, 0);
+
        return p;
 }
 
@@ -1583,6 +1593,8 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
 {
        update_curr_rt(rq);
 
+       update_rt_rq_load_avg(rq_clock_task(rq), rq, 1);
+
        /*
         * The previous task needs to be made eligible for pushing
         * if it is still active
@@ -2312,6 +2324,7 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
        struct sched_rt_entity *rt_se = &p->rt;
 
        update_curr_rt(rq);
+       update_rt_rq_load_avg(rq_clock_task(rq), rq, 1);
 
        watchdog(rq, p);
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 00d6f2594c4ed32c48e3c5aa43986286ec2ee522..405dd9ba6b3918aaf17a2c146d91461d2b8a9c84 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -594,6 +594,7 @@ struct rt_rq {
        unsigned long           rt_nr_total;
        int                     overloaded;
        struct plist_head       pushable_tasks;
+
 #endif /* CONFIG_SMP */
        int                     rt_queued;
 
@@ -854,6 +855,7 @@ struct rq {
 
        u64                     rt_avg;
        u64                     age_stamp;
+       struct sched_avg        avg_rt;
        u64                     idle_stamp;
        u64                     avg_idle;
 
@@ -2212,4 +2214,9 @@ static inline unsigned long cpu_util_cfs(struct rq *rq)
 
        return util;
 }
+
+static inline unsigned long cpu_util_rt(struct rq *rq)
+{
+       return rq->avg_rt.util_avg;
+}
 #endif