Merge tag 'pci-v6.16-fixes-3' of git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci
[linux-2.6-block.git] / kernel / sched / rt.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
bb44e5d1
IM
2/*
3 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
4 * policies)
5 */
371bf427 6
ce0dbbbb 7int sched_rr_timeslice = RR_TIMESLICE;
d505b8af
HC
8/* More than 4 hours if BW_SHIFT equals 20. */
9static const u64 max_rt_runtime = MAX_BW;
ce0dbbbb 10
d9ab0e63
ZN
11/*
12 * period over which we measure -rt task CPU usage in us.
13 * default: 1s
14 */
089768df 15int sysctl_sched_rt_period = 1000000;
d9ab0e63
ZN
16
17/*
18 * part of the period that we allow rt tasks to run in us.
19 * default: 0.95s
20 */
21int sysctl_sched_rt_runtime = 950000;
22
28f152cd 23#ifdef CONFIG_SYSCTL
c7fcb998 24static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ;
78eb4ea2 25static int sched_rt_handler(const struct ctl_table *table, int write, void *buffer,
d9ab0e63 26 size_t *lenp, loff_t *ppos);
78eb4ea2 27static int sched_rr_handler(const struct ctl_table *table, int write, void *buffer,
dafd7a9d 28 size_t *lenp, loff_t *ppos);
1751f872 29static const struct ctl_table sched_rt_sysctls[] = {
d9ab0e63
ZN
30 {
31 .procname = "sched_rt_period_us",
32 .data = &sysctl_sched_rt_period,
089768df 33 .maxlen = sizeof(int),
d9ab0e63
ZN
34 .mode = 0644,
35 .proc_handler = sched_rt_handler,
079be8fc
CH
36 .extra1 = SYSCTL_ONE,
37 .extra2 = SYSCTL_INT_MAX,
d9ab0e63
ZN
38 },
39 {
40 .procname = "sched_rt_runtime_us",
41 .data = &sysctl_sched_rt_runtime,
42 .maxlen = sizeof(int),
43 .mode = 0644,
44 .proc_handler = sched_rt_handler,
079be8fc 45 .extra1 = SYSCTL_NEG_ONE,
089768df 46 .extra2 = (void *)&sysctl_sched_rt_period,
d9ab0e63 47 },
dafd7a9d
ZN
48 {
49 .procname = "sched_rr_timeslice_ms",
50 .data = &sysctl_sched_rr_timeslice,
51 .maxlen = sizeof(int),
52 .mode = 0644,
53 .proc_handler = sched_rr_handler,
54 },
d9ab0e63
ZN
55};
56
57static int __init sched_rt_sysctl_init(void)
58{
59 register_sysctl_init("kernel", sched_rt_sysctls);
60 return 0;
61}
62late_initcall(sched_rt_sysctl_init);
63#endif
64
5f6bd380
PZ
65void init_rt_rq(struct rt_rq *rt_rq)
66{
67 struct rt_prio_array *array;
68 int i;
69
70 array = &rt_rq->active;
71 for (i = 0; i < MAX_RT_PRIO; i++) {
72 INIT_LIST_HEAD(array->queue + i);
73 __clear_bit(i, array->bitmap);
74 }
75 /* delimiter for bitsearch: */
76 __set_bit(MAX_RT_PRIO, array->bitmap);
77
78#if defined CONFIG_SMP
79 rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
80 rt_rq->highest_prio.next = MAX_RT_PRIO-1;
81 rt_rq->overloaded = 0;
82 plist_head_init(&rt_rq->pushable_tasks);
83#endif /* CONFIG_SMP */
84 /* We start is dequeued state, because no RT tasks are queued */
85 rt_rq->rt_queued = 0;
86
87#ifdef CONFIG_RT_GROUP_SCHED
88 rt_rq->rt_time = 0;
89 rt_rq->rt_throttled = 0;
90 rt_rq->rt_runtime = 0;
91 raw_spin_lock_init(&rt_rq->rt_runtime_lock);
a5a25b32 92 rt_rq->tg = &root_task_group;
5f6bd380
PZ
93#endif
94}
95
96#ifdef CONFIG_RT_GROUP_SCHED
97
98static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
99
029632fb
PZ
100static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
101{
102 struct rt_bandwidth *rt_b =
103 container_of(timer, struct rt_bandwidth, rt_period_timer);
029632fb 104 int idle = 0;
77a4d1a1 105 int overrun;
029632fb 106
77a4d1a1 107 raw_spin_lock(&rt_b->rt_runtime_lock);
029632fb 108 for (;;) {
77a4d1a1 109 overrun = hrtimer_forward_now(timer, rt_b->rt_period);
029632fb
PZ
110 if (!overrun)
111 break;
112
77a4d1a1 113 raw_spin_unlock(&rt_b->rt_runtime_lock);
029632fb 114 idle = do_sched_rt_period_timer(rt_b, overrun);
77a4d1a1 115 raw_spin_lock(&rt_b->rt_runtime_lock);
029632fb 116 }
4cfafd30
PZ
117 if (idle)
118 rt_b->rt_period_active = 0;
77a4d1a1 119 raw_spin_unlock(&rt_b->rt_runtime_lock);
029632fb
PZ
120
121 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
122}
123
124void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
125{
126 rt_b->rt_period = ns_to_ktime(period);
127 rt_b->rt_runtime = runtime;
128
129 raw_spin_lock_init(&rt_b->rt_runtime_lock);
130
ee13da87
NC
131 hrtimer_setup(&rt_b->rt_period_timer, sched_rt_period_timer, CLOCK_MONOTONIC,
132 HRTIMER_MODE_REL_HARD);
029632fb
PZ
133}
134
9b58e976 135static inline void do_start_rt_bandwidth(struct rt_bandwidth *rt_b)
029632fb 136{
029632fb 137 raw_spin_lock(&rt_b->rt_runtime_lock);
4cfafd30
PZ
138 if (!rt_b->rt_period_active) {
139 rt_b->rt_period_active = 1;
c3a990dc
SR
140 /*
141 * SCHED_DEADLINE updates the bandwidth, as a run away
142 * RT task with a DL task could hog a CPU. But DL does
143 * not reset the period. If a deadline task was running
144 * without an RT task running, it can cause RT tasks to
145 * throttle when they start up. Kick the timer right away
146 * to update the period.
147 */
148 hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
d5096aa6
SAS
149 hrtimer_start_expires(&rt_b->rt_period_timer,
150 HRTIMER_MODE_ABS_PINNED_HARD);
4cfafd30 151 }
029632fb
PZ
152 raw_spin_unlock(&rt_b->rt_runtime_lock);
153}
154
9b58e976
LH
155static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
156{
157 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
158 return;
159
160 do_start_rt_bandwidth(rt_b);
161}
162
029632fb
PZ
163static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
164{
165 hrtimer_cancel(&rt_b->rt_period_timer);
166}
8f48894f
PZ
167
168#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)
169
398a153b
GH
170static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
171{
8f48894f 172 WARN_ON_ONCE(!rt_entity_is_task(rt_se));
dd5bdaf2 173
398a153b
GH
174 return container_of(rt_se, struct task_struct, rt);
175}
176
398a153b
GH
177static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
178{
87f1fb77
MK
179 /* Cannot fold with non-CONFIG_RT_GROUP_SCHED version, layout */
180 WARN_ON(!rt_group_sched_enabled() && rt_rq->tg != &root_task_group);
398a153b
GH
181 return rt_rq->rq;
182}
183
184static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
185{
87f1fb77 186 WARN_ON(!rt_group_sched_enabled() && rt_se->rt_rq->tg != &root_task_group);
398a153b
GH
187 return rt_se->rt_rq;
188}
189
653d07a6
KT
190static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
191{
192 struct rt_rq *rt_rq = rt_se->rt_rq;
193
87f1fb77 194 WARN_ON(!rt_group_sched_enabled() && rt_rq->tg != &root_task_group);
653d07a6
KT
195 return rt_rq->rq;
196}
197
b027789e 198void unregister_rt_sched_group(struct task_group *tg)
029632fb 199{
d6809c2f
MK
200 if (!rt_group_sched_enabled())
201 return;
202
029632fb
PZ
203 if (tg->rt_se)
204 destroy_rt_bandwidth(&tg->rt_bandwidth);
b027789e
MK
205}
206
207void free_rt_sched_group(struct task_group *tg)
208{
209 int i;
210
d6809c2f
MK
211 if (!rt_group_sched_enabled())
212 return;
213
029632fb
PZ
214 for_each_possible_cpu(i) {
215 if (tg->rt_rq)
216 kfree(tg->rt_rq[i]);
217 if (tg->rt_se)
218 kfree(tg->rt_se[i]);
219 }
220
221 kfree(tg->rt_rq);
222 kfree(tg->rt_se);
223}
224
225void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
226 struct sched_rt_entity *rt_se, int cpu,
227 struct sched_rt_entity *parent)
228{
229 struct rq *rq = cpu_rq(cpu);
230
934fc331 231 rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
029632fb
PZ
232 rt_rq->rt_nr_boosted = 0;
233 rt_rq->rq = rq;
234 rt_rq->tg = tg;
235
236 tg->rt_rq[cpu] = rt_rq;
237 tg->rt_se[cpu] = rt_se;
238
239 if (!rt_se)
240 return;
241
242 if (!parent)
243 rt_se->rt_rq = &rq->rt;
244 else
245 rt_se->rt_rq = parent->my_q;
246
247 rt_se->my_q = rt_rq;
248 rt_se->parent = parent;
249 INIT_LIST_HEAD(&rt_se->run_list);
250}
251
252int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
253{
254 struct rt_rq *rt_rq;
255 struct sched_rt_entity *rt_se;
256 int i;
257
d6809c2f
MK
258 if (!rt_group_sched_enabled())
259 return 1;
260
6396bb22 261 tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL);
029632fb
PZ
262 if (!tg->rt_rq)
263 goto err;
6396bb22 264 tg->rt_se = kcalloc(nr_cpu_ids, sizeof(rt_se), GFP_KERNEL);
029632fb
PZ
265 if (!tg->rt_se)
266 goto err;
267
5f6bd380 268 init_rt_bandwidth(&tg->rt_bandwidth, ktime_to_ns(global_rt_period()), 0);
029632fb
PZ
269
270 for_each_possible_cpu(i) {
271 rt_rq = kzalloc_node(sizeof(struct rt_rq),
272 GFP_KERNEL, cpu_to_node(i));
273 if (!rt_rq)
274 goto err;
275
276 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
277 GFP_KERNEL, cpu_to_node(i));
278 if (!rt_se)
279 goto err_free_rq;
280
07c54f7a 281 init_rt_rq(rt_rq);
029632fb
PZ
282 rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
283 init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
284 }
285
286 return 1;
287
288err_free_rq:
289 kfree(rt_rq);
290err:
291 return 0;
292}
293
398a153b
GH
294#else /* CONFIG_RT_GROUP_SCHED */
295
a1ba4d8b
PZ
296#define rt_entity_is_task(rt_se) (1)
297
8f48894f
PZ
298static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
299{
300 return container_of(rt_se, struct task_struct, rt);
301}
302
398a153b
GH
303static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
304{
305 return container_of(rt_rq, struct rq, rt);
306}
307
653d07a6 308static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
398a153b
GH
309{
310 struct task_struct *p = rt_task_of(rt_se);
653d07a6
KT
311
312 return task_rq(p);
313}
314
315static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
316{
317 struct rq *rq = rq_of_rt_se(rt_se);
398a153b
GH
318
319 return &rq->rt;
320}
321
b027789e
MK
322void unregister_rt_sched_group(struct task_group *tg) { }
323
029632fb
PZ
324void free_rt_sched_group(struct task_group *tg) { }
325
326int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
327{
328 return 1;
329}
398a153b
GH
330#endif /* CONFIG_RT_GROUP_SCHED */
331
4fd29176 332#ifdef CONFIG_SMP
84de4274 333
dc877341
PZ
334static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
335{
336 /* Try to pull RT tasks here if we lower this rq's prio */
120455c5 337 return rq->online && rq->rt.highest_prio.curr > prev->prio;
dc877341
PZ
338}
339
637f5085 340static inline int rt_overloaded(struct rq *rq)
4fd29176 341{
637f5085 342 return atomic_read(&rq->rd->rto_count);
4fd29176 343}
84de4274 344
4fd29176
SR
345static inline void rt_set_overload(struct rq *rq)
346{
1f11eb6a
GH
347 if (!rq->online)
348 return;
349
c6c4927b 350 cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
4fd29176
SR
351 /*
352 * Make sure the mask is visible before we set
353 * the overload count. That is checked to determine
354 * if we should look at the mask. It would be a shame
355 * if we looked at the mask, but the mask was not
356 * updated yet.
7c3f2ab7
PZ
357 *
358 * Matched by the barrier in pull_rt_task().
4fd29176 359 */
7c3f2ab7 360 smp_wmb();
637f5085 361 atomic_inc(&rq->rd->rto_count);
4fd29176 362}
84de4274 363
4fd29176
SR
364static inline void rt_clear_overload(struct rq *rq)
365{
1f11eb6a
GH
366 if (!rq->online)
367 return;
368
4fd29176 369 /* the order here really doesn't matter */
637f5085 370 atomic_dec(&rq->rd->rto_count);
c6c4927b 371 cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
4fd29176 372}
73fe6aae 373
5181f4a4
SR
374static inline int has_pushable_tasks(struct rq *rq)
375{
376 return !plist_head_empty(&rq->rt.pushable_tasks);
377}
378
8e5bad7d
KC
379static DEFINE_PER_CPU(struct balance_callback, rt_push_head);
380static DEFINE_PER_CPU(struct balance_callback, rt_pull_head);
e3fca9e7
PZ
381
382static void push_rt_tasks(struct rq *);
fd7a4bed 383static void pull_rt_task(struct rq *);
e3fca9e7 384
02d8ec94 385static inline void rt_queue_push_tasks(struct rq *rq)
dc877341 386{
e3fca9e7
PZ
387 if (!has_pushable_tasks(rq))
388 return;
389
fd7a4bed
PZ
390 queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);
391}
392
02d8ec94 393static inline void rt_queue_pull_task(struct rq *rq)
fd7a4bed
PZ
394{
395 queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);
dc877341
PZ
396}
397
917b627d
GH
398static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
399{
400 plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
401 plist_node_init(&p->pushable_tasks, p->prio);
402 plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);
5181f4a4
SR
403
404 /* Update the highest prio pushable task */
405 if (p->prio < rq->rt.highest_prio.next)
406 rq->rt.highest_prio.next = p->prio;
612f769e
VS
407
408 if (!rq->rt.overloaded) {
409 rt_set_overload(rq);
410 rq->rt.overloaded = 1;
411 }
917b627d
GH
412}
413
414static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
415{
416 plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
917b627d 417
5181f4a4
SR
418 /* Update the new highest prio pushable task */
419 if (has_pushable_tasks(rq)) {
420 p = plist_first_entry(&rq->rt.pushable_tasks,
421 struct task_struct, pushable_tasks);
422 rq->rt.highest_prio.next = p->prio;
934fc331
PZ
423 } else {
424 rq->rt.highest_prio.next = MAX_RT_PRIO-1;
612f769e
VS
425
426 if (rq->rt.overloaded) {
427 rt_clear_overload(rq);
428 rq->rt.overloaded = 0;
429 }
934fc331 430 }
bcf08df3
IM
431}
432
917b627d
GH
433#else
434
ceacc2c1 435static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
fa85ae24 436{
6f505b16
PZ
437}
438
ceacc2c1
PZ
439static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
440{
441}
442
02d8ec94 443static inline void rt_queue_push_tasks(struct rq *rq)
dc877341
PZ
444{
445}
4fd29176
SR
446#endif /* CONFIG_SMP */
447
f4ebcbc0 448static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
5c66d1b9 449static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count);
f4ebcbc0 450
6f505b16
PZ
451static inline int on_rt_rq(struct sched_rt_entity *rt_se)
452{
ff77e468 453 return rt_se->on_rq;
6f505b16
PZ
454}
455
804d402f
QY
456#ifdef CONFIG_UCLAMP_TASK
457/*
458 * Verify the fitness of task @p to run on @cpu taking into account the uclamp
459 * settings.
460 *
461 * This check is only important for heterogeneous systems where uclamp_min value
462 * is higher than the capacity of a @cpu. For non-heterogeneous system this
463 * function will always return true.
464 *
465 * The function will return true if the capacity of the @cpu is >= the
466 * uclamp_min and false otherwise.
467 *
468 * Note that uclamp_min will be clamped to uclamp_max if uclamp_min
469 * > uclamp_max.
470 */
471static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
472{
473 unsigned int min_cap;
474 unsigned int max_cap;
475 unsigned int cpu_cap;
476
477 /* Only heterogeneous systems can benefit from this check */
740cf8a7 478 if (!sched_asym_cpucap_active())
804d402f
QY
479 return true;
480
481 min_cap = uclamp_eff_value(p, UCLAMP_MIN);
482 max_cap = uclamp_eff_value(p, UCLAMP_MAX);
483
7bc26384 484 cpu_cap = arch_scale_cpu_capacity(cpu);
804d402f
QY
485
486 return cpu_cap >= min(min_cap, max_cap);
487}
488#else
489static inline bool rt_task_fits_capacity(struct task_struct *p, int cpu)
490{
491 return true;
492}
493#endif
494
052f1dc7 495#ifdef CONFIG_RT_GROUP_SCHED
6f505b16 496
9f0c1e56 497static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
6f505b16 498{
ac086bc2
PZ
499 return rt_rq->rt_runtime;
500}
501
502static inline u64 sched_rt_period(struct rt_rq *rt_rq)
503{
504 return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
6f505b16
PZ
505}
506
ec514c48
CX
507typedef struct task_group *rt_rq_iter_t;
508
1c09ab0d
YZ
509static inline struct task_group *next_task_group(struct task_group *tg)
510{
87f1fb77
MK
511 if (!rt_group_sched_enabled()) {
512 WARN_ON(tg != &root_task_group);
61d3164f 513 return NULL;
87f1fb77 514 }
61d3164f 515
1c09ab0d
YZ
516 do {
517 tg = list_entry_rcu(tg->list.next,
518 typeof(struct task_group), list);
519 } while (&tg->list != &task_groups && task_group_is_autogroup(tg));
520
521 if (&tg->list == &task_groups)
522 tg = NULL;
523
524 return tg;
525}
526
527#define for_each_rt_rq(rt_rq, iter, rq) \
61d3164f
MK
528 for (iter = &root_task_group; \
529 iter && (rt_rq = iter->rt_rq[cpu_of(rq)]); \
530 iter = next_task_group(iter))
ec514c48 531
6f505b16
PZ
532#define for_each_sched_rt_entity(rt_se) \
533 for (; rt_se; rt_se = rt_se->parent)
534
535static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
536{
537 return rt_se->my_q;
538}
539
ff77e468
PZ
540static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
541static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
6f505b16 542
9f0c1e56 543static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
6f505b16 544{
af0c8b2b 545 struct task_struct *donor = rq_of_rt_rq(rt_rq)->donor;
8875125e 546 struct rq *rq = rq_of_rt_rq(rt_rq);
74b7eb58
YZ
547 struct sched_rt_entity *rt_se;
548
8875125e 549 int cpu = cpu_of(rq);
0c3b9168
BS
550
551 rt_se = rt_rq->tg->rt_se[cpu];
6f505b16 552
f6121f4f 553 if (rt_rq->rt_nr_running) {
f4ebcbc0
KT
554 if (!rt_se)
555 enqueue_top_rt_rq(rt_rq);
556 else if (!on_rt_rq(rt_se))
ff77e468 557 enqueue_rt_entity(rt_se, 0);
f4ebcbc0 558
af0c8b2b 559 if (rt_rq->highest_prio.curr < donor->prio)
8875125e 560 resched_curr(rq);
6f505b16
PZ
561 }
562}
563
9f0c1e56 564static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
6f505b16 565{
74b7eb58 566 struct sched_rt_entity *rt_se;
0c3b9168 567 int cpu = cpu_of(rq_of_rt_rq(rt_rq));
74b7eb58 568
0c3b9168 569 rt_se = rt_rq->tg->rt_se[cpu];
6f505b16 570
296b2ffe 571 if (!rt_se) {
5c66d1b9 572 dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
296b2ffe
VG
573 /* Kick cpufreq (see the comment in kernel/sched/sched.h). */
574 cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);
575 }
f4ebcbc0 576 else if (on_rt_rq(rt_se))
ff77e468 577 dequeue_rt_entity(rt_se, 0);
6f505b16
PZ
578}
579
46383648
KT
580static inline int rt_rq_throttled(struct rt_rq *rt_rq)
581{
582 return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
583}
584
23b0fdfc
PZ
585static int rt_se_boosted(struct sched_rt_entity *rt_se)
586{
587 struct rt_rq *rt_rq = group_rt_rq(rt_se);
588 struct task_struct *p;
589
590 if (rt_rq)
591 return !!rt_rq->rt_nr_boosted;
592
593 p = rt_task_of(rt_se);
594 return p->prio != p->normal_prio;
595}
596
d0b27fa7 597#ifdef CONFIG_SMP
c6c4927b 598static inline const struct cpumask *sched_rt_period_mask(void)
d0b27fa7 599{
424c93fe 600 return this_rq()->rd->span;
d0b27fa7 601}
6f505b16 602#else
c6c4927b 603static inline const struct cpumask *sched_rt_period_mask(void)
d0b27fa7 604{
c6c4927b 605 return cpu_online_mask;
d0b27fa7
PZ
606}
607#endif
6f505b16 608
d0b27fa7
PZ
609static inline
610struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
6f505b16 611{
d0b27fa7
PZ
612 return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
613}
9f0c1e56 614
ac086bc2
PZ
615static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
616{
617 return &rt_rq->tg->rt_bandwidth;
618}
619
faa59937
JL
620bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
621{
622 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
623
624 return (hrtimer_active(&rt_b->rt_period_timer) ||
625 rt_rq->rt_time < rt_b->rt_runtime);
626}
627
ac086bc2 628#ifdef CONFIG_SMP
78333cdd
PZ
629/*
630 * We ran out of runtime, see if we can borrow some from our neighbours.
631 */
269b26a5 632static void do_balance_runtime(struct rt_rq *rt_rq)
ac086bc2
PZ
633{
634 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
aa7f6730 635 struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
269b26a5 636 int i, weight;
ac086bc2
PZ
637 u64 rt_period;
638
c6c4927b 639 weight = cpumask_weight(rd->span);
ac086bc2 640
0986b11b 641 raw_spin_lock(&rt_b->rt_runtime_lock);
ac086bc2 642 rt_period = ktime_to_ns(rt_b->rt_period);
c6c4927b 643 for_each_cpu(i, rd->span) {
ac086bc2
PZ
644 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
645 s64 diff;
646
647 if (iter == rt_rq)
648 continue;
649
0986b11b 650 raw_spin_lock(&iter->rt_runtime_lock);
78333cdd
PZ
651 /*
652 * Either all rqs have inf runtime and there's nothing to steal
653 * or __disable_runtime() below sets a specific rq to inf to
3b03706f 654 * indicate its been disabled and disallow stealing.
78333cdd 655 */
7def2be1
PZ
656 if (iter->rt_runtime == RUNTIME_INF)
657 goto next;
658
78333cdd
PZ
659 /*
660 * From runqueues with spare time, take 1/n part of their
661 * spare time, but no more than our period.
662 */
ac086bc2
PZ
663 diff = iter->rt_runtime - iter->rt_time;
664 if (diff > 0) {
58838cf3 665 diff = div_u64((u64)diff, weight);
ac086bc2
PZ
666 if (rt_rq->rt_runtime + diff > rt_period)
667 diff = rt_period - rt_rq->rt_runtime;
668 iter->rt_runtime -= diff;
669 rt_rq->rt_runtime += diff;
ac086bc2 670 if (rt_rq->rt_runtime == rt_period) {
0986b11b 671 raw_spin_unlock(&iter->rt_runtime_lock);
ac086bc2
PZ
672 break;
673 }
674 }
7def2be1 675next:
0986b11b 676 raw_spin_unlock(&iter->rt_runtime_lock);
ac086bc2 677 }
0986b11b 678 raw_spin_unlock(&rt_b->rt_runtime_lock);
ac086bc2 679}
7def2be1 680
78333cdd
PZ
681/*
682 * Ensure this RQ takes back all the runtime it lend to its neighbours.
683 */
7def2be1
PZ
684static void __disable_runtime(struct rq *rq)
685{
686 struct root_domain *rd = rq->rd;
ec514c48 687 rt_rq_iter_t iter;
7def2be1
PZ
688 struct rt_rq *rt_rq;
689
690 if (unlikely(!scheduler_running))
691 return;
692
ec514c48 693 for_each_rt_rq(rt_rq, iter, rq) {
7def2be1
PZ
694 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
695 s64 want;
696 int i;
697
0986b11b
TG
698 raw_spin_lock(&rt_b->rt_runtime_lock);
699 raw_spin_lock(&rt_rq->rt_runtime_lock);
78333cdd
PZ
700 /*
701 * Either we're all inf and nobody needs to borrow, or we're
702 * already disabled and thus have nothing to do, or we have
703 * exactly the right amount of runtime to take out.
704 */
7def2be1
PZ
705 if (rt_rq->rt_runtime == RUNTIME_INF ||
706 rt_rq->rt_runtime == rt_b->rt_runtime)
707 goto balanced;
0986b11b 708 raw_spin_unlock(&rt_rq->rt_runtime_lock);
7def2be1 709
78333cdd
PZ
710 /*
711 * Calculate the difference between what we started out with
712 * and what we current have, that's the amount of runtime
713 * we lend and now have to reclaim.
714 */
7def2be1
PZ
715 want = rt_b->rt_runtime - rt_rq->rt_runtime;
716
78333cdd
PZ
717 /*
718 * Greedy reclaim, take back as much as we can.
719 */
c6c4927b 720 for_each_cpu(i, rd->span) {
7def2be1
PZ
721 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
722 s64 diff;
723
78333cdd
PZ
724 /*
725 * Can't reclaim from ourselves or disabled runqueues.
726 */
f1679d08 727 if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
7def2be1
PZ
728 continue;
729
0986b11b 730 raw_spin_lock(&iter->rt_runtime_lock);
7def2be1
PZ
731 if (want > 0) {
732 diff = min_t(s64, iter->rt_runtime, want);
733 iter->rt_runtime -= diff;
734 want -= diff;
735 } else {
736 iter->rt_runtime -= want;
737 want -= want;
738 }
0986b11b 739 raw_spin_unlock(&iter->rt_runtime_lock);
7def2be1
PZ
740
741 if (!want)
742 break;
743 }
744
0986b11b 745 raw_spin_lock(&rt_rq->rt_runtime_lock);
78333cdd
PZ
746 /*
747 * We cannot be left wanting - that would mean some runtime
748 * leaked out of the system.
749 */
09348d75 750 WARN_ON_ONCE(want);
7def2be1 751balanced:
78333cdd
PZ
752 /*
753 * Disable all the borrow logic by pretending we have inf
754 * runtime - in which case borrowing doesn't make sense.
755 */
7def2be1 756 rt_rq->rt_runtime = RUNTIME_INF;
a4c96ae3 757 rt_rq->rt_throttled = 0;
0986b11b
TG
758 raw_spin_unlock(&rt_rq->rt_runtime_lock);
759 raw_spin_unlock(&rt_b->rt_runtime_lock);
99b62567
KT
760
761 /* Make rt_rq available for pick_next_task() */
762 sched_rt_rq_enqueue(rt_rq);
7def2be1
PZ
763 }
764}
765
7def2be1
PZ
766static void __enable_runtime(struct rq *rq)
767{
ec514c48 768 rt_rq_iter_t iter;
7def2be1
PZ
769 struct rt_rq *rt_rq;
770
771 if (unlikely(!scheduler_running))
772 return;
773
78333cdd
PZ
774 /*
775 * Reset each runqueue's bandwidth settings
776 */
ec514c48 777 for_each_rt_rq(rt_rq, iter, rq) {
7def2be1
PZ
778 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
779
0986b11b
TG
780 raw_spin_lock(&rt_b->rt_runtime_lock);
781 raw_spin_lock(&rt_rq->rt_runtime_lock);
7def2be1
PZ
782 rt_rq->rt_runtime = rt_b->rt_runtime;
783 rt_rq->rt_time = 0;
baf25731 784 rt_rq->rt_throttled = 0;
0986b11b
TG
785 raw_spin_unlock(&rt_rq->rt_runtime_lock);
786 raw_spin_unlock(&rt_b->rt_runtime_lock);
7def2be1
PZ
787 }
788}
789
269b26a5 790static void balance_runtime(struct rt_rq *rt_rq)
eff6549b 791{
4a6184ce 792 if (!sched_feat(RT_RUNTIME_SHARE))
269b26a5 793 return;
4a6184ce 794
eff6549b 795 if (rt_rq->rt_time > rt_rq->rt_runtime) {
0986b11b 796 raw_spin_unlock(&rt_rq->rt_runtime_lock);
269b26a5 797 do_balance_runtime(rt_rq);
0986b11b 798 raw_spin_lock(&rt_rq->rt_runtime_lock);
eff6549b 799 }
eff6549b 800}
55e12e5e 801#else /* !CONFIG_SMP */
269b26a5 802static inline void balance_runtime(struct rt_rq *rt_rq) {}
55e12e5e 803#endif /* CONFIG_SMP */
ac086bc2 804
eff6549b
PZ
805static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
806{
42c62a58 807 int i, idle = 1, throttled = 0;
c6c4927b 808 const struct cpumask *span;
eff6549b 809
eff6549b 810 span = sched_rt_period_mask();
5f6bd380 811
e221d028
MG
812 /*
813 * FIXME: isolated CPUs should really leave the root task group,
814 * whether they are isolcpus or were isolated via cpusets, lest
815 * the timer run on a CPU which does not service all runqueues,
816 * potentially leaving other CPUs indefinitely throttled. If
817 * isolation is really required, the user will turn the throttle
818 * off to kill the perturbations it causes anyway. Meanwhile,
819 * this maintains functionality for boot and/or troubleshooting.
820 */
821 if (rt_b == &root_task_group.rt_bandwidth)
822 span = cpu_online_mask;
5f6bd380 823
c6c4927b 824 for_each_cpu(i, span) {
eff6549b
PZ
825 int enqueue = 0;
826 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
827 struct rq *rq = rq_of_rt_rq(rt_rq);
2679a837 828 struct rq_flags rf;
c249f255
DK
829 int skip;
830
831 /*
832 * When span == cpu_online_mask, taking each rq->lock
833 * can be time-consuming. Try to avoid it when possible.
834 */
835 raw_spin_lock(&rt_rq->rt_runtime_lock);
f3d133ee
HL
836 if (!sched_feat(RT_RUNTIME_SHARE) && rt_rq->rt_runtime != RUNTIME_INF)
837 rt_rq->rt_runtime = rt_b->rt_runtime;
c249f255
DK
838 skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;
839 raw_spin_unlock(&rt_rq->rt_runtime_lock);
840 if (skip)
841 continue;
eff6549b 842
2679a837 843 rq_lock(rq, &rf);
d29a2064
DB
844 update_rq_clock(rq);
845
eff6549b
PZ
846 if (rt_rq->rt_time) {
847 u64 runtime;
848
0986b11b 849 raw_spin_lock(&rt_rq->rt_runtime_lock);
eff6549b
PZ
850 if (rt_rq->rt_throttled)
851 balance_runtime(rt_rq);
852 runtime = rt_rq->rt_runtime;
853 rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
854 if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
855 rt_rq->rt_throttled = 0;
856 enqueue = 1;
61eadef6
MG
857
858 /*
9edfbfed 859 * When we're idle and a woken (rt) task is
e23edc86 860 * throttled wakeup_preempt() will set
9edfbfed
PZ
861 * skip_update and the time between the wakeup
862 * and this unthrottle will get accounted as
863 * 'runtime'.
61eadef6
MG
864 */
865 if (rt_rq->rt_nr_running && rq->curr == rq->idle)
adcc8da8 866 rq_clock_cancel_skipupdate(rq);
eff6549b
PZ
867 }
868 if (rt_rq->rt_time || rt_rq->rt_nr_running)
869 idle = 0;
0986b11b 870 raw_spin_unlock(&rt_rq->rt_runtime_lock);
0c3b9168 871 } else if (rt_rq->rt_nr_running) {
6c3df255 872 idle = 0;
0c3b9168
BS
873 if (!rt_rq_throttled(rt_rq))
874 enqueue = 1;
875 }
42c62a58
PZ
876 if (rt_rq->rt_throttled)
877 throttled = 1;
eff6549b
PZ
878
879 if (enqueue)
880 sched_rt_rq_enqueue(rt_rq);
2679a837 881 rq_unlock(rq, &rf);
eff6549b
PZ
882 }
883
42c62a58
PZ
884 if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
885 return 1;
886
eff6549b
PZ
887 return idle;
888}
ac086bc2 889
9f0c1e56 890static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
6f505b16 891{
9f0c1e56 892 u64 runtime = sched_rt_runtime(rt_rq);
fa85ae24 893
fa85ae24 894 if (rt_rq->rt_throttled)
23b0fdfc 895 return rt_rq_throttled(rt_rq);
fa85ae24 896
5b680fd6 897 if (runtime >= sched_rt_period(rt_rq))
ac086bc2
PZ
898 return 0;
899
b79f3833
PZ
900 balance_runtime(rt_rq);
901 runtime = sched_rt_runtime(rt_rq);
902 if (runtime == RUNTIME_INF)
903 return 0;
ac086bc2 904
9f0c1e56 905 if (rt_rq->rt_time > runtime) {
7abc63b1
PZ
906 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
907
908 /*
909 * Don't actually throttle groups that have no runtime assigned
910 * but accrue some time due to boosting.
911 */
912 if (likely(rt_b->rt_runtime)) {
913 rt_rq->rt_throttled = 1;
c224815d 914 printk_deferred_once("sched: RT throttling activated\n");
7abc63b1
PZ
915 } else {
916 /*
917 * In case we did anyway, make it go away,
918 * replenishment is a joke, since it will replenish us
919 * with exactly 0 ns.
920 */
921 rt_rq->rt_time = 0;
922 }
923
23b0fdfc 924 if (rt_rq_throttled(rt_rq)) {
9f0c1e56 925 sched_rt_rq_dequeue(rt_rq);
23b0fdfc
PZ
926 return 1;
927 }
fa85ae24
PZ
928 }
929
930 return 0;
931}
932
5f6bd380
PZ
933#else /* !CONFIG_RT_GROUP_SCHED */
934
935typedef struct rt_rq *rt_rq_iter_t;
936
937#define for_each_rt_rq(rt_rq, iter, rq) \
938 for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
939
940#define for_each_sched_rt_entity(rt_se) \
941 for (; rt_se; rt_se = NULL)
942
943static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
944{
945 return NULL;
946}
947
948static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
949{
950 struct rq *rq = rq_of_rt_rq(rt_rq);
951
952 if (!rt_rq->rt_nr_running)
953 return;
954
955 enqueue_top_rt_rq(rt_rq);
956 resched_curr(rq);
957}
958
959static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
960{
961 dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
962}
963
964static inline int rt_rq_throttled(struct rt_rq *rt_rq)
965{
966 return false;
967}
968
969static inline const struct cpumask *sched_rt_period_mask(void)
970{
971 return cpu_online_mask;
972}
973
974static inline
975struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
976{
977 return &cpu_rq(cpu)->rt;
978}
979
980#ifdef CONFIG_SMP
981static void __enable_runtime(struct rq *rq) { }
982static void __disable_runtime(struct rq *rq) { }
983#endif
984
985#endif /* CONFIG_RT_GROUP_SCHED */
986
987static inline int rt_se_prio(struct sched_rt_entity *rt_se)
988{
989#ifdef CONFIG_RT_GROUP_SCHED
990 struct rt_rq *rt_rq = group_rt_rq(rt_se);
991
992 if (rt_rq)
993 return rt_rq->highest_prio.curr;
994#endif
995
996 return rt_task_of(rt_se)->prio;
997}
998
bb44e5d1
IM
999/*
1000 * Update the current task's runtime statistics. Skip current tasks that
1001 * are not in our scheduling class.
1002 */
a9957449 1003static void update_curr_rt(struct rq *rq)
bb44e5d1 1004{
af0c8b2b 1005 struct task_struct *donor = rq->donor;
5d69eca5 1006 s64 delta_exec;
bb44e5d1 1007
af0c8b2b 1008 if (donor->sched_class != &rt_sched_class)
bb44e5d1
IM
1009 return;
1010
5d69eca5
PZ
1011 delta_exec = update_curr_common(rq);
1012 if (unlikely(delta_exec <= 0))
fc79e240 1013 return;
6cfb0d5d 1014
5f6bd380 1015#ifdef CONFIG_RT_GROUP_SCHED
af0c8b2b 1016 struct sched_rt_entity *rt_se = &donor->rt;
5f6bd380 1017
0b148fa0
PZ
1018 if (!rt_bandwidth_enabled())
1019 return;
1020
354d60c2 1021 for_each_sched_rt_entity(rt_se) {
0b07939c 1022 struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
9b58e976 1023 int exceeded;
354d60c2 1024
cc2991cf 1025 if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
0986b11b 1026 raw_spin_lock(&rt_rq->rt_runtime_lock);
cc2991cf 1027 rt_rq->rt_time += delta_exec;
9b58e976
LH
1028 exceeded = sched_rt_runtime_exceeded(rt_rq);
1029 if (exceeded)
8875125e 1030 resched_curr(rq);
0986b11b 1031 raw_spin_unlock(&rt_rq->rt_runtime_lock);
9b58e976
LH
1032 if (exceeded)
1033 do_start_rt_bandwidth(sched_rt_bandwidth(rt_rq));
cc2991cf 1034 }
354d60c2 1035 }
5f6bd380 1036#endif
bb44e5d1
IM
1037}
1038
f4ebcbc0 1039static void
5c66d1b9 1040dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count)
f4ebcbc0
KT
1041{
1042 struct rq *rq = rq_of_rt_rq(rt_rq);
1043
1044 BUG_ON(&rq->rt != rt_rq);
1045
1046 if (!rt_rq->rt_queued)
1047 return;
1048
1049 BUG_ON(!rq->nr_running);
1050
5c66d1b9 1051 sub_nr_running(rq, count);
f4ebcbc0 1052 rt_rq->rt_queued = 0;
8f111bc3 1053
f4ebcbc0
KT
1054}
1055
1056static void
1057enqueue_top_rt_rq(struct rt_rq *rt_rq)
1058{
1059 struct rq *rq = rq_of_rt_rq(rt_rq);
1060
1061 BUG_ON(&rq->rt != rt_rq);
1062
1063 if (rt_rq->rt_queued)
1064 return;
296b2ffe
VG
1065
1066 if (rt_rq_throttled(rt_rq))
f4ebcbc0
KT
1067 return;
1068
296b2ffe
VG
1069 if (rt_rq->rt_nr_running) {
1070 add_nr_running(rq, rt_rq->rt_nr_running);
1071 rt_rq->rt_queued = 1;
1072 }
8f111bc3
PZ
1073
1074 /* Kick cpufreq (see the comment in kernel/sched/sched.h). */
1075 cpufreq_update_util(rq, 0);
f4ebcbc0
KT
1076}
1077
398a153b 1078#if defined CONFIG_SMP
e864c499 1079
398a153b
GH
1080static void
1081inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
63489e45 1082{
4d984277 1083 struct rq *rq = rq_of_rt_rq(rt_rq);
1f11eb6a 1084
757dfcaa
KT
1085 /*
1086 * Change rq's cpupri only if rt_rq is the top queue.
1087 */
433bce5d 1088 if (IS_ENABLED(CONFIG_RT_GROUP_SCHED) && &rq->rt != rt_rq)
757dfcaa 1089 return;
433bce5d 1090
5181f4a4
SR
1091 if (rq->online && prio < prev_prio)
1092 cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
398a153b 1093}
73fe6aae 1094
398a153b
GH
1095static void
1096dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
1097{
1098 struct rq *rq = rq_of_rt_rq(rt_rq);
d0b27fa7 1099
757dfcaa
KT
1100 /*
1101 * Change rq's cpupri only if rt_rq is the top queue.
1102 */
433bce5d 1103 if (IS_ENABLED(CONFIG_RT_GROUP_SCHED) && &rq->rt != rt_rq)
757dfcaa 1104 return;
433bce5d 1105
398a153b
GH
1106 if (rq->online && rt_rq->highest_prio.curr != prev_prio)
1107 cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
63489e45
SR
1108}
1109
398a153b
GH
1110#else /* CONFIG_SMP */
1111
6f505b16 1112static inline
398a153b
GH
1113void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1114static inline
1115void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
1116
1117#endif /* CONFIG_SMP */
6e0534f2 1118
052f1dc7 1119#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
398a153b
GH
1120static void
1121inc_rt_prio(struct rt_rq *rt_rq, int prio)
1122{
1123 int prev_prio = rt_rq->highest_prio.curr;
1124
1125 if (prio < prev_prio)
1126 rt_rq->highest_prio.curr = prio;
1127
1128 inc_rt_prio_smp(rt_rq, prio, prev_prio);
1129}
1130
1131static void
1132dec_rt_prio(struct rt_rq *rt_rq, int prio)
1133{
1134 int prev_prio = rt_rq->highest_prio.curr;
1135
6f505b16 1136 if (rt_rq->rt_nr_running) {
764a9d6f 1137
398a153b 1138 WARN_ON(prio < prev_prio);
764a9d6f 1139
e864c499 1140 /*
398a153b 1141 * This may have been our highest task, and therefore
402de7fc 1142 * we may have some re-computation to do
e864c499 1143 */
398a153b 1144 if (prio == prev_prio) {
e864c499
GH
1145 struct rt_prio_array *array = &rt_rq->active;
1146
1147 rt_rq->highest_prio.curr =
764a9d6f 1148 sched_find_first_bit(array->bitmap);
e864c499
GH
1149 }
1150
934fc331
PZ
1151 } else {
1152 rt_rq->highest_prio.curr = MAX_RT_PRIO-1;
1153 }
73fe6aae 1154
398a153b
GH
1155 dec_rt_prio_smp(rt_rq, prio, prev_prio);
1156}
1f11eb6a 1157
398a153b
GH
1158#else
1159
1160static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}
1161static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}
1162
1163#endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */
6e0534f2 1164
052f1dc7 1165#ifdef CONFIG_RT_GROUP_SCHED
398a153b
GH
1166
1167static void
1168inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1169{
1170 if (rt_se_boosted(rt_se))
1171 rt_rq->rt_nr_boosted++;
1172
a5a25b32 1173 start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
398a153b
GH
1174}
1175
1176static void
1177dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1178{
23b0fdfc
PZ
1179 if (rt_se_boosted(rt_se))
1180 rt_rq->rt_nr_boosted--;
1181
1182 WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
398a153b
GH
1183}
1184
1185#else /* CONFIG_RT_GROUP_SCHED */
1186
1187static void
1188inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1189{
398a153b
GH
1190}
1191
1192static inline
1193void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}
1194
1195#endif /* CONFIG_RT_GROUP_SCHED */
1196
22abdef3
KT
1197static inline
1198unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
1199{
1200 struct rt_rq *group_rq = group_rt_rq(rt_se);
1201
1202 if (group_rq)
1203 return group_rq->rt_nr_running;
1204 else
1205 return 1;
1206}
1207
01d36d0a
FW
1208static inline
1209unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
1210{
1211 struct rt_rq *group_rq = group_rt_rq(rt_se);
1212 struct task_struct *tsk;
1213
1214 if (group_rq)
1215 return group_rq->rr_nr_running;
1216
1217 tsk = rt_task_of(rt_se);
1218
1219 return (tsk->policy == SCHED_RR) ? 1 : 0;
1220}
1221
398a153b
GH
1222static inline
1223void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1224{
1225 int prio = rt_se_prio(rt_se);
1226
1227 WARN_ON(!rt_prio(prio));
22abdef3 1228 rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
01d36d0a 1229 rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);
398a153b
GH
1230
1231 inc_rt_prio(rt_rq, prio);
398a153b
GH
1232 inc_rt_group(rt_se, rt_rq);
1233}
1234
1235static inline
1236void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
1237{
1238 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
1239 WARN_ON(!rt_rq->rt_nr_running);
22abdef3 1240 rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
01d36d0a 1241 rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);
398a153b
GH
1242
1243 dec_rt_prio(rt_rq, rt_se_prio(rt_se));
398a153b 1244 dec_rt_group(rt_se, rt_rq);
63489e45
SR
1245}
1246
ff77e468
PZ
1247/*
1248 * Change rt_se->run_list location unless SAVE && !MOVE
1249 *
1250 * assumes ENQUEUE/DEQUEUE flags match
1251 */
1252static inline bool move_entity(unsigned int flags)
1253{
1254 if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE)
1255 return false;
1256
1257 return true;
1258}
1259
1260static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)
1261{
1262 list_del_init(&rt_se->run_list);
1263
1264 if (list_empty(array->queue + rt_se_prio(rt_se)))
1265 __clear_bit(rt_se_prio(rt_se), array->bitmap);
1266
1267 rt_se->on_list = 0;
1268}
1269
57a5c2da
YS
1270static inline struct sched_statistics *
1271__schedstats_from_rt_se(struct sched_rt_entity *rt_se)
1272{
57a5c2da
YS
1273 /* schedstats is not supported for rt group. */
1274 if (!rt_entity_is_task(rt_se))
1275 return NULL;
57a5c2da
YS
1276
1277 return &rt_task_of(rt_se)->stats;
1278}
1279
1280static inline void
1281update_stats_wait_start_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1282{
1283 struct sched_statistics *stats;
1284 struct task_struct *p = NULL;
1285
1286 if (!schedstat_enabled())
1287 return;
1288
1289 if (rt_entity_is_task(rt_se))
1290 p = rt_task_of(rt_se);
1291
1292 stats = __schedstats_from_rt_se(rt_se);
1293 if (!stats)
1294 return;
1295
1296 __update_stats_wait_start(rq_of_rt_rq(rt_rq), p, stats);
1297}
1298
1299static inline void
1300update_stats_enqueue_sleeper_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1301{
1302 struct sched_statistics *stats;
1303 struct task_struct *p = NULL;
1304
1305 if (!schedstat_enabled())
1306 return;
1307
1308 if (rt_entity_is_task(rt_se))
1309 p = rt_task_of(rt_se);
1310
1311 stats = __schedstats_from_rt_se(rt_se);
1312 if (!stats)
1313 return;
1314
1315 __update_stats_enqueue_sleeper(rq_of_rt_rq(rt_rq), p, stats);
1316}
1317
1318static inline void
1319update_stats_enqueue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
1320 int flags)
1321{
1322 if (!schedstat_enabled())
1323 return;
1324
1325 if (flags & ENQUEUE_WAKEUP)
1326 update_stats_enqueue_sleeper_rt(rt_rq, rt_se);
1327}
1328
1329static inline void
1330update_stats_wait_end_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se)
1331{
1332 struct sched_statistics *stats;
1333 struct task_struct *p = NULL;
1334
1335 if (!schedstat_enabled())
1336 return;
1337
1338 if (rt_entity_is_task(rt_se))
1339 p = rt_task_of(rt_se);
1340
1341 stats = __schedstats_from_rt_se(rt_se);
1342 if (!stats)
1343 return;
1344
1345 __update_stats_wait_end(rq_of_rt_rq(rt_rq), p, stats);
1346}
1347
1348static inline void
1349update_stats_dequeue_rt(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
1350 int flags)
1351{
1352 struct task_struct *p = NULL;
1353
1354 if (!schedstat_enabled())
1355 return;
1356
1357 if (rt_entity_is_task(rt_se))
1358 p = rt_task_of(rt_se);
1359
1360 if ((flags & DEQUEUE_SLEEP) && p) {
1361 unsigned int state;
1362
1363 state = READ_ONCE(p->__state);
1364 if (state & TASK_INTERRUPTIBLE)
1365 __schedstat_set(p->stats.sleep_start,
1366 rq_clock(rq_of_rt_rq(rt_rq)));
1367
1368 if (state & TASK_UNINTERRUPTIBLE)
1369 __schedstat_set(p->stats.block_start,
1370 rq_clock(rq_of_rt_rq(rt_rq)));
1371 }
1372}
1373
ff77e468 1374static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
bb44e5d1 1375{
6f505b16
PZ
1376 struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1377 struct rt_prio_array *array = &rt_rq->active;
1378 struct rt_rq *group_rq = group_rt_rq(rt_se);
20b6331b 1379 struct list_head *queue = array->queue + rt_se_prio(rt_se);
bb44e5d1 1380
ad2a3f13
PZ
1381 /*
1382 * Don't enqueue the group if its throttled, or when empty.
1383 * The latter is a consequence of the former when a child group
1384 * get throttled and the current group doesn't have any other
1385 * active members.
1386 */
ff77e468
PZ
1387 if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) {
1388 if (rt_se->on_list)
1389 __delist_rt_entity(rt_se, array);
6f505b16 1390 return;
ff77e468 1391 }
63489e45 1392
ff77e468
PZ
1393 if (move_entity(flags)) {
1394 WARN_ON_ONCE(rt_se->on_list);
1395 if (flags & ENQUEUE_HEAD)
1396 list_add(&rt_se->run_list, queue);
1397 else
1398 list_add_tail(&rt_se->run_list, queue);
1399
1400 __set_bit(rt_se_prio(rt_se), array->bitmap);
1401 rt_se->on_list = 1;
1402 }
1403 rt_se->on_rq = 1;
78f2c7db 1404
6f505b16
PZ
1405 inc_rt_tasks(rt_se, rt_rq);
1406}
1407
ff77e468 1408static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
6f505b16
PZ
1409{
1410 struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
1411 struct rt_prio_array *array = &rt_rq->active;
1412
ff77e468
PZ
1413 if (move_entity(flags)) {
1414 WARN_ON_ONCE(!rt_se->on_list);
1415 __delist_rt_entity(rt_se, array);
1416 }
1417 rt_se->on_rq = 0;
6f505b16
PZ
1418
1419 dec_rt_tasks(rt_se, rt_rq);
1420}
1421
1422/*
1423 * Because the prio of an upper entry depends on the lower
1424 * entries, we must remove entries top - down.
6f505b16 1425 */
ff77e468 1426static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
6f505b16 1427{
ad2a3f13 1428 struct sched_rt_entity *back = NULL;
5c66d1b9 1429 unsigned int rt_nr_running;
6f505b16 1430
58d6c2d7
PZ
1431 for_each_sched_rt_entity(rt_se) {
1432 rt_se->back = back;
1433 back = rt_se;
1434 }
1435
5c66d1b9 1436 rt_nr_running = rt_rq_of_se(back)->rt_nr_running;
f4ebcbc0 1437
58d6c2d7
PZ
1438 for (rt_se = back; rt_se; rt_se = rt_se->back) {
1439 if (on_rt_rq(rt_se))
ff77e468 1440 __dequeue_rt_entity(rt_se, flags);
ad2a3f13 1441 }
5c66d1b9
NSJ
1442
1443 dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running);
ad2a3f13
PZ
1444}
1445
ff77e468 1446static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
ad2a3f13 1447{
f4ebcbc0
KT
1448 struct rq *rq = rq_of_rt_se(rt_se);
1449
57a5c2da
YS
1450 update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1451
ff77e468 1452 dequeue_rt_stack(rt_se, flags);
ad2a3f13 1453 for_each_sched_rt_entity(rt_se)
ff77e468 1454 __enqueue_rt_entity(rt_se, flags);
f4ebcbc0 1455 enqueue_top_rt_rq(&rq->rt);
ad2a3f13
PZ
1456}
1457
ff77e468 1458static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
ad2a3f13 1459{
f4ebcbc0
KT
1460 struct rq *rq = rq_of_rt_se(rt_se);
1461
57a5c2da
YS
1462 update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags);
1463
ff77e468 1464 dequeue_rt_stack(rt_se, flags);
ad2a3f13
PZ
1465
1466 for_each_sched_rt_entity(rt_se) {
1467 struct rt_rq *rt_rq = group_rt_rq(rt_se);
1468
1469 if (rt_rq && rt_rq->rt_nr_running)
ff77e468 1470 __enqueue_rt_entity(rt_se, flags);
58d6c2d7 1471 }
f4ebcbc0 1472 enqueue_top_rt_rq(&rq->rt);
bb44e5d1
IM
1473}
1474
1475/*
1476 * Adding/removing a task to/from a priority array:
1477 */
ea87bb78 1478static void
371fd7e7 1479enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
6f505b16
PZ
1480{
1481 struct sched_rt_entity *rt_se = &p->rt;
1482
371fd7e7 1483 if (flags & ENQUEUE_WAKEUP)
6f505b16
PZ
1484 rt_se->timeout = 0;
1485
57a5c2da
YS
1486 check_schedstat_required();
1487 update_stats_wait_start_rt(rt_rq_of_se(rt_se), rt_se);
1488
ff77e468 1489 enqueue_rt_entity(rt_se, flags);
c09595f6 1490
4b53a341 1491 if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
917b627d 1492 enqueue_pushable_task(rq, p);
6f505b16
PZ
1493}
1494
863ccdbb 1495static bool dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
bb44e5d1 1496{
6f505b16 1497 struct sched_rt_entity *rt_se = &p->rt;
bb44e5d1 1498
f1e14ef6 1499 update_curr_rt(rq);
ff77e468 1500 dequeue_rt_entity(rt_se, flags);
c09595f6 1501
917b627d 1502 dequeue_pushable_task(rq, p);
863ccdbb
PZ
1503
1504 return true;
bb44e5d1
IM
1505}
1506
1507/*
60686317
RW
1508 * Put task to the head or the end of the run list without the overhead of
1509 * dequeue followed by enqueue.
bb44e5d1 1510 */
7ebefa8c
DA
1511static void
1512requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
6f505b16 1513{
1cdad715 1514 if (on_rt_rq(rt_se)) {
7ebefa8c
DA
1515 struct rt_prio_array *array = &rt_rq->active;
1516 struct list_head *queue = array->queue + rt_se_prio(rt_se);
1517
1518 if (head)
1519 list_move(&rt_se->run_list, queue);
1520 else
1521 list_move_tail(&rt_se->run_list, queue);
1cdad715 1522 }
6f505b16
PZ
1523}
1524
7ebefa8c 1525static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
bb44e5d1 1526{
6f505b16
PZ
1527 struct sched_rt_entity *rt_se = &p->rt;
1528 struct rt_rq *rt_rq;
bb44e5d1 1529
6f505b16
PZ
1530 for_each_sched_rt_entity(rt_se) {
1531 rt_rq = rt_rq_of_se(rt_se);
7ebefa8c 1532 requeue_rt_entity(rt_rq, rt_se, head);
6f505b16 1533 }
bb44e5d1
IM
1534}
1535
6f505b16 1536static void yield_task_rt(struct rq *rq)
bb44e5d1 1537{
7ebefa8c 1538 requeue_task_rt(rq, rq->curr, 0);
bb44e5d1
IM
1539}
1540
e7693a36 1541#ifdef CONFIG_SMP
318e0893
GH
1542static int find_lowest_rq(struct task_struct *task);
1543
0017d735 1544static int
3aef1551 1545select_task_rq_rt(struct task_struct *p, int cpu, int flags)
e7693a36 1546{
af0c8b2b 1547 struct task_struct *curr, *donor;
7608dec2 1548 struct rq *rq;
804d402f 1549 bool test;
c37495fd
SR
1550
1551 /* For anything but wake ups, just return the task_cpu */
3aef1551 1552 if (!(flags & (WF_TTWU | WF_FORK)))
c37495fd
SR
1553 goto out;
1554
7608dec2
PZ
1555 rq = cpu_rq(cpu);
1556
1557 rcu_read_lock();
316c1608 1558 curr = READ_ONCE(rq->curr); /* unlocked access */
af0c8b2b 1559 donor = READ_ONCE(rq->donor);
7608dec2 1560
318e0893 1561 /*
7608dec2 1562 * If the current task on @p's runqueue is an RT task, then
e1f47d89
SR
1563 * try to see if we can wake this RT task up on another
1564 * runqueue. Otherwise simply start this RT task
1565 * on its current runqueue.
1566 *
43fa5460
SR
1567 * We want to avoid overloading runqueues. If the woken
1568 * task is a higher priority, then it will stay on this CPU
1569 * and the lower prio task should be moved to another CPU.
1570 * Even though this will probably make the lower prio task
1571 * lose its cache, we do not want to bounce a higher task
1572 * around just because it gave up its CPU, perhaps for a
1573 * lock?
1574 *
1575 * For equal prio tasks, we just let the scheduler sort it out.
7608dec2 1576 *
402de7fc 1577 * Otherwise, just let it ride on the affine RQ and the
7608dec2
PZ
1578 * post-schedule router will push the preempted task away
1579 *
1580 * This test is optimistic, if we get it wrong the load-balancer
1581 * will have to sort it out.
804d402f
QY
1582 *
1583 * We take into account the capacity of the CPU to ensure it fits the
1584 * requirement of the task - which is only important on heterogeneous
1585 * systems like big.LITTLE.
318e0893 1586 */
804d402f 1587 test = curr &&
af0c8b2b
PZ
1588 unlikely(rt_task(donor)) &&
1589 (curr->nr_cpus_allowed < 2 || donor->prio <= p->prio);
804d402f
QY
1590
1591 if (test || !rt_task_fits_capacity(p, cpu)) {
7608dec2 1592 int target = find_lowest_rq(p);
318e0893 1593
b28bc1e0
QY
1594 /*
1595 * Bail out if we were forcing a migration to find a better
1596 * fitting CPU but our search failed.
1597 */
1598 if (!test && target != -1 && !rt_task_fits_capacity(p, target))
1599 goto out_unlock;
1600
80e3d87b
TC
1601 /*
1602 * Don't bother moving it if the destination CPU is
1603 * not running a lower priority task.
1604 */
1605 if (target != -1 &&
1606 p->prio < cpu_rq(target)->rt.highest_prio.curr)
7608dec2 1607 cpu = target;
318e0893 1608 }
b28bc1e0
QY
1609
1610out_unlock:
7608dec2 1611 rcu_read_unlock();
318e0893 1612
c37495fd 1613out:
7608dec2 1614 return cpu;
e7693a36 1615}
7ebefa8c
DA
1616
1617static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
1618{
4b53a341 1619 if (rq->curr->nr_cpus_allowed == 1 ||
af0c8b2b 1620 !cpupri_find(&rq->rd->cpupri, rq->donor, NULL))
7ebefa8c
DA
1621 return;
1622
308a623a
WL
1623 /*
1624 * p is migratable, so let's not schedule it and
1625 * see if it is pushed or pulled somewhere else.
1626 */
804d402f 1627 if (p->nr_cpus_allowed != 1 &&
a1bd02e1 1628 cpupri_find(&rq->rd->cpupri, p, NULL))
13b8bd0a 1629 return;
24600ce8 1630
7ebefa8c 1631 /*
97fb7a0a
IM
1632 * There appear to be other CPUs that can accept
1633 * the current task but none can run 'p', so lets reschedule
1634 * to try and push the current task away:
7ebefa8c
DA
1635 */
1636 requeue_task_rt(rq, p, 1);
8875125e 1637 resched_curr(rq);
7ebefa8c
DA
1638}
1639
6e2df058
PZ
1640static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
1641{
1642 if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) {
1643 /*
1644 * This is OK, because current is on_cpu, which avoids it being
1645 * picked for load-balance and preemption/IRQs are still
1646 * disabled avoiding further scheduler activity on it and we've
1647 * not yet started the picking loop.
1648 */
1649 rq_unpin_lock(rq, rf);
1650 pull_rt_task(rq);
1651 rq_repin_lock(rq, rf);
1652 }
1653
1654 return sched_stop_runnable(rq) || sched_dl_runnable(rq) || sched_rt_runnable(rq);
1655}
e7693a36
GH
1656#endif /* CONFIG_SMP */
1657
bb44e5d1
IM
1658/*
1659 * Preempt the current task with a newly woken task if needed:
1660 */
e23edc86 1661static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags)
bb44e5d1 1662{
af0c8b2b
PZ
1663 struct task_struct *donor = rq->donor;
1664
1665 if (p->prio < donor->prio) {
8875125e 1666 resched_curr(rq);
45c01e82
GH
1667 return;
1668 }
1669
1670#ifdef CONFIG_SMP
1671 /*
1672 * If:
1673 *
1674 * - the newly woken task is of equal priority to the current task
1675 * - the newly woken task is non-migratable while current is migratable
1676 * - current will be preempted on the next reschedule
1677 *
1678 * we should check to see if current can readily move to a different
1679 * cpu. If so, we will reschedule to allow the push logic to try
1680 * to move current somewhere else, making room for our non-migratable
1681 * task.
1682 */
af0c8b2b 1683 if (p->prio == donor->prio && !test_tsk_need_resched(rq->curr))
7ebefa8c 1684 check_preempt_equal_prio(rq, p);
45c01e82 1685#endif
bb44e5d1
IM
1686}
1687
a0e813f2 1688static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
ff1cdc94 1689{
57a5c2da
YS
1690 struct sched_rt_entity *rt_se = &p->rt;
1691 struct rt_rq *rt_rq = &rq->rt;
1692
ff1cdc94 1693 p->se.exec_start = rq_clock_task(rq);
57a5c2da
YS
1694 if (on_rt_rq(&p->rt))
1695 update_stats_wait_end_rt(rt_rq, rt_se);
ff1cdc94
MS
1696
1697 /* The running task is never eligible for pushing */
1698 dequeue_pushable_task(rq, p);
f95d4eae 1699
a0e813f2
PZ
1700 if (!first)
1701 return;
1702
f95d4eae
PZ
1703 /*
1704 * If prev task was rt, put_prev_task() has already updated the
1705 * utilization. We only care of the case where we start to schedule a
1706 * rt task
1707 */
af0c8b2b 1708 if (rq->donor->sched_class != &rt_sched_class)
f95d4eae
PZ
1709 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
1710
1711 rt_queue_push_tasks(rq);
ff1cdc94
MS
1712}
1713
821aecd0 1714static struct sched_rt_entity *pick_next_rt_entity(struct rt_rq *rt_rq)
bb44e5d1 1715{
6f505b16
PZ
1716 struct rt_prio_array *array = &rt_rq->active;
1717 struct sched_rt_entity *next = NULL;
bb44e5d1
IM
1718 struct list_head *queue;
1719 int idx;
1720
1721 idx = sched_find_first_bit(array->bitmap);
6f505b16 1722 BUG_ON(idx >= MAX_RT_PRIO);
bb44e5d1
IM
1723
1724 queue = array->queue + idx;
f7d2728c 1725 if (WARN_ON_ONCE(list_empty(queue)))
7c4a5b89 1726 return NULL;
6f505b16 1727 next = list_entry(queue->next, struct sched_rt_entity, run_list);
326587b8 1728
6f505b16
PZ
1729 return next;
1730}
bb44e5d1 1731
917b627d 1732static struct task_struct *_pick_next_task_rt(struct rq *rq)
6f505b16
PZ
1733{
1734 struct sched_rt_entity *rt_se;
606dba2e 1735 struct rt_rq *rt_rq = &rq->rt;
6f505b16
PZ
1736
1737 do {
821aecd0 1738 rt_se = pick_next_rt_entity(rt_rq);
7c4a5b89
PB
1739 if (unlikely(!rt_se))
1740 return NULL;
6f505b16
PZ
1741 rt_rq = group_rt_rq(rt_se);
1742 } while (rt_rq);
1743
ff1cdc94 1744 return rt_task_of(rt_se);
917b627d
GH
1745}
1746
21f56ffe 1747static struct task_struct *pick_task_rt(struct rq *rq)
917b627d 1748{
606dba2e 1749 struct task_struct *p;
606dba2e 1750
6e2df058 1751 if (!sched_rt_runnable(rq))
606dba2e
PZ
1752 return NULL;
1753
606dba2e 1754 p = _pick_next_task_rt(rq);
21f56ffe
PZ
1755
1756 return p;
1757}
1758
b2d70222 1759static void put_prev_task_rt(struct rq *rq, struct task_struct *p, struct task_struct *next)
bb44e5d1 1760{
57a5c2da
YS
1761 struct sched_rt_entity *rt_se = &p->rt;
1762 struct rt_rq *rt_rq = &rq->rt;
1763
1764 if (on_rt_rq(&p->rt))
1765 update_stats_wait_start_rt(rt_rq, rt_se);
1766
f1e14ef6 1767 update_curr_rt(rq);
917b627d 1768
23127296 1769 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
371bf427 1770
917b627d
GH
1771 /*
1772 * The previous task needs to be made eligible for pushing
1773 * if it is still active
1774 */
4b53a341 1775 if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
917b627d 1776 enqueue_pushable_task(rq, p);
bb44e5d1
IM
1777}
1778
681f3e68 1779#ifdef CONFIG_SMP
6f505b16 1780
e8fa1362
SR
1781/* Only try algorithms three times */
1782#define RT_MAX_TRIES 3
1783
e23ee747
KT
1784/*
1785 * Return the highest pushable rq's task, which is suitable to be executed
97fb7a0a 1786 * on the CPU, NULL otherwise
e23ee747
KT
1787 */
1788static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
e8fa1362 1789{
e23ee747
KT
1790 struct plist_head *head = &rq->rt.pushable_tasks;
1791 struct task_struct *p;
3d07467b 1792
e23ee747
KT
1793 if (!has_pushable_tasks(rq))
1794 return NULL;
3d07467b 1795
e23ee747 1796 plist_for_each_entry(p, head, pushable_tasks) {
18adad1d 1797 if (task_is_pushable(rq, p, cpu))
e23ee747 1798 return p;
f65eda4f
SR
1799 }
1800
e23ee747 1801 return NULL;
e8fa1362
SR
1802}
1803
0e3900e6 1804static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
e8fa1362 1805
6e1254d2
GH
1806static int find_lowest_rq(struct task_struct *task)
1807{
1808 struct sched_domain *sd;
4ba29684 1809 struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
6e1254d2
GH
1810 int this_cpu = smp_processor_id();
1811 int cpu = task_cpu(task);
a1bd02e1 1812 int ret;
06f90dbd 1813
0da938c4
SR
1814 /* Make sure the mask is initialized first */
1815 if (unlikely(!lowest_mask))
1816 return -1;
1817
4b53a341 1818 if (task->nr_cpus_allowed == 1)
6e0534f2 1819 return -1; /* No other targets possible */
6e1254d2 1820
a1bd02e1
QY
1821 /*
1822 * If we're on asym system ensure we consider the different capacities
1823 * of the CPUs when searching for the lowest_mask.
1824 */
740cf8a7 1825 if (sched_asym_cpucap_active()) {
a1bd02e1
QY
1826
1827 ret = cpupri_find_fitness(&task_rq(task)->rd->cpupri,
1828 task, lowest_mask,
1829 rt_task_fits_capacity);
1830 } else {
1831
1832 ret = cpupri_find(&task_rq(task)->rd->cpupri,
1833 task, lowest_mask);
1834 }
1835
1836 if (!ret)
6e0534f2 1837 return -1; /* No targets found */
6e1254d2
GH
1838
1839 /*
97fb7a0a 1840 * At this point we have built a mask of CPUs representing the
6e1254d2
GH
1841 * lowest priority tasks in the system. Now we want to elect
1842 * the best one based on our affinity and topology.
1843 *
97fb7a0a 1844 * We prioritize the last CPU that the task executed on since
6e1254d2
GH
1845 * it is most likely cache-hot in that location.
1846 */
96f874e2 1847 if (cpumask_test_cpu(cpu, lowest_mask))
6e1254d2
GH
1848 return cpu;
1849
1850 /*
1851 * Otherwise, we consult the sched_domains span maps to figure
97fb7a0a 1852 * out which CPU is logically closest to our hot cache data.
6e1254d2 1853 */
e2c88063
RR
1854 if (!cpumask_test_cpu(this_cpu, lowest_mask))
1855 this_cpu = -1; /* Skip this_cpu opt if not among lowest */
6e1254d2 1856
cd4ae6ad 1857 rcu_read_lock();
e2c88063
RR
1858 for_each_domain(cpu, sd) {
1859 if (sd->flags & SD_WAKE_AFFINE) {
1860 int best_cpu;
6e1254d2 1861
e2c88063
RR
1862 /*
1863 * "this_cpu" is cheaper to preempt than a
1864 * remote processor.
1865 */
1866 if (this_cpu != -1 &&
cd4ae6ad
XF
1867 cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
1868 rcu_read_unlock();
e2c88063 1869 return this_cpu;
cd4ae6ad 1870 }
e2c88063 1871
14e292f8
PZ
1872 best_cpu = cpumask_any_and_distribute(lowest_mask,
1873 sched_domain_span(sd));
cd4ae6ad
XF
1874 if (best_cpu < nr_cpu_ids) {
1875 rcu_read_unlock();
e2c88063 1876 return best_cpu;
cd4ae6ad 1877 }
6e1254d2
GH
1878 }
1879 }
cd4ae6ad 1880 rcu_read_unlock();
6e1254d2
GH
1881
1882 /*
1883 * And finally, if there were no matches within the domains
1884 * just give the caller *something* to work with from the compatible
1885 * locations.
1886 */
e2c88063
RR
1887 if (this_cpu != -1)
1888 return this_cpu;
1889
14e292f8 1890 cpu = cpumask_any_distribute(lowest_mask);
e2c88063
RR
1891 if (cpu < nr_cpu_ids)
1892 return cpu;
97fb7a0a 1893
e2c88063 1894 return -1;
07b4032c
GH
1895}
1896
690e47d1
HA
1897static struct task_struct *pick_next_pushable_task(struct rq *rq)
1898{
1899 struct task_struct *p;
1900
1901 if (!has_pushable_tasks(rq))
1902 return NULL;
1903
1904 p = plist_first_entry(&rq->rt.pushable_tasks,
1905 struct task_struct, pushable_tasks);
1906
1907 BUG_ON(rq->cpu != task_cpu(p));
1908 BUG_ON(task_current(rq, p));
1909 BUG_ON(task_current_donor(rq, p));
1910 BUG_ON(p->nr_cpus_allowed <= 1);
1911
1912 BUG_ON(!task_on_rq_queued(p));
1913 BUG_ON(!rt_task(p));
1914
1915 return p;
1916}
1917
07b4032c 1918/* Will lock the rq it finds */
4df64c0b 1919static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
07b4032c
GH
1920{
1921 struct rq *lowest_rq = NULL;
07b4032c 1922 int tries;
4df64c0b 1923 int cpu;
e8fa1362 1924
07b4032c
GH
1925 for (tries = 0; tries < RT_MAX_TRIES; tries++) {
1926 cpu = find_lowest_rq(task);
1927
2de0b463 1928 if ((cpu == -1) || (cpu == rq->cpu))
e8fa1362
SR
1929 break;
1930
07b4032c
GH
1931 lowest_rq = cpu_rq(cpu);
1932
80e3d87b
TC
1933 if (lowest_rq->rt.highest_prio.curr <= task->prio) {
1934 /*
1935 * Target rq has tasks of equal or higher priority,
1936 * retrying does not release any lock and is unlikely
1937 * to yield a different result.
1938 */
1939 lowest_rq = NULL;
1940 break;
1941 }
1942
e8fa1362 1943 /* if the prio of this runqueue changed, try again */
07b4032c 1944 if (double_lock_balance(rq, lowest_rq)) {
e8fa1362
SR
1945 /*
1946 * We had to unlock the run queue. In
1947 * the mean time, task could have
690e47d1
HA
1948 * migrated already or had its affinity changed,
1949 * therefore check if the task is still at the
1950 * head of the pushable tasks list.
feffe5bb
SS
1951 * It is possible the task was scheduled, set
1952 * "migrate_disabled" and then got preempted, so we must
1953 * check the task migration disable flag here too.
e8fa1362 1954 */
690e47d1 1955 if (unlikely(is_migration_disabled(task) ||
95158a89 1956 !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_mask) ||
690e47d1 1957 task != pick_next_pushable_task(rq))) {
4df64c0b 1958
7f1b4393 1959 double_unlock_balance(rq, lowest_rq);
e8fa1362
SR
1960 lowest_rq = NULL;
1961 break;
1962 }
1963 }
1964
1965 /* If this rq is still suitable use it. */
e864c499 1966 if (lowest_rq->rt.highest_prio.curr > task->prio)
e8fa1362
SR
1967 break;
1968
1969 /* try again */
1b12bbc7 1970 double_unlock_balance(rq, lowest_rq);
e8fa1362
SR
1971 lowest_rq = NULL;
1972 }
1973
1974 return lowest_rq;
1975}
1976
1977/*
1978 * If the current CPU has more than one RT task, see if the non
1979 * running task can migrate over to a CPU that is running a task
1980 * of lesser priority.
1981 */
a7c81556 1982static int push_rt_task(struct rq *rq, bool pull)
e8fa1362
SR
1983{
1984 struct task_struct *next_task;
1985 struct rq *lowest_rq;
311e800e 1986 int ret = 0;
e8fa1362 1987
a22d7fc1
GH
1988 if (!rq->rt.overloaded)
1989 return 0;
1990
917b627d 1991 next_task = pick_next_pushable_task(rq);
e8fa1362
SR
1992 if (!next_task)
1993 return 0;
1994
49246274 1995retry:
49bef33e
VS
1996 /*
1997 * It's possible that the next_task slipped in of
1998 * higher priority than current. If that's the case
1999 * just reschedule current.
2000 */
af0c8b2b 2001 if (unlikely(next_task->prio < rq->donor->prio)) {
49bef33e
VS
2002 resched_curr(rq);
2003 return 0;
2004 }
2005
a7c81556
PZ
2006 if (is_migration_disabled(next_task)) {
2007 struct task_struct *push_task = NULL;
2008 int cpu;
2009
2010 if (!pull || rq->push_busy)
2011 return 0;
2012
49bef33e
VS
2013 /*
2014 * Invoking find_lowest_rq() on anything but an RT task doesn't
2015 * make sense. Per the above priority check, curr has to
2016 * be of higher priority than next_task, so no need to
2017 * reschedule when bailing out.
2018 *
2019 * Note that the stoppers are masqueraded as SCHED_FIFO
2020 * (cf. sched_set_stop_task()), so we can't rely on rt_task().
2021 */
af0c8b2b 2022 if (rq->donor->sched_class != &rt_sched_class)
49bef33e
VS
2023 return 0;
2024
a7c81556
PZ
2025 cpu = find_lowest_rq(rq->curr);
2026 if (cpu == -1 || cpu == rq->cpu)
2027 return 0;
2028
2029 /*
2030 * Given we found a CPU with lower priority than @next_task,
2031 * therefore it should be running. However we cannot migrate it
2032 * to this other CPU, instead attempt to push the current
2033 * running task on this CPU away.
2034 */
2035 push_task = get_push_task(rq);
2036 if (push_task) {
f0498d2a 2037 preempt_disable();
5cb9eaa3 2038 raw_spin_rq_unlock(rq);
a7c81556
PZ
2039 stop_one_cpu_nowait(rq->cpu, push_cpu_stop,
2040 push_task, &rq->push_work);
f0498d2a 2041 preempt_enable();
5cb9eaa3 2042 raw_spin_rq_lock(rq);
a7c81556
PZ
2043 }
2044
2045 return 0;
2046 }
2047
9ebc6053 2048 if (WARN_ON(next_task == rq->curr))
e8fa1362
SR
2049 return 0;
2050
697f0a48 2051 /* We might release rq lock */
e8fa1362
SR
2052 get_task_struct(next_task);
2053
2054 /* find_lock_lowest_rq locks the rq if found */
697f0a48 2055 lowest_rq = find_lock_lowest_rq(next_task, rq);
e8fa1362
SR
2056 if (!lowest_rq) {
2057 struct task_struct *task;
2058 /*
311e800e 2059 * find_lock_lowest_rq releases rq->lock
1563513d
GH
2060 * so it is possible that next_task has migrated.
2061 *
2062 * We need to make sure that the task is still on the same
2063 * run-queue and is also still the next task eligible for
2064 * pushing.
e8fa1362 2065 */
917b627d 2066 task = pick_next_pushable_task(rq);
de16b91e 2067 if (task == next_task) {
1563513d 2068 /*
311e800e
HD
2069 * The task hasn't migrated, and is still the next
2070 * eligible task, but we failed to find a run-queue
2071 * to push it to. Do not retry in this case, since
97fb7a0a 2072 * other CPUs will pull from us when ready.
1563513d 2073 */
1563513d 2074 goto out;
e8fa1362 2075 }
917b627d 2076
1563513d
GH
2077 if (!task)
2078 /* No more tasks, just exit */
2079 goto out;
2080
917b627d 2081 /*
1563513d 2082 * Something has shifted, try again.
917b627d 2083 */
1563513d
GH
2084 put_task_struct(next_task);
2085 next_task = task;
2086 goto retry;
e8fa1362
SR
2087 }
2088
2b05a0b4 2089 move_queued_task_locked(rq, lowest_rq, next_task);
8875125e 2090 resched_curr(lowest_rq);
a7c81556 2091 ret = 1;
e8fa1362 2092
1b12bbc7 2093 double_unlock_balance(rq, lowest_rq);
e8fa1362
SR
2094out:
2095 put_task_struct(next_task);
2096
311e800e 2097 return ret;
e8fa1362
SR
2098}
2099
e8fa1362
SR
2100static void push_rt_tasks(struct rq *rq)
2101{
2102 /* push_rt_task will return true if it moved an RT */
a7c81556 2103 while (push_rt_task(rq, false))
e8fa1362
SR
2104 ;
2105}
2106
b6366f04 2107#ifdef HAVE_RT_PUSH_IPI
4bdced5c 2108
b6366f04 2109/*
4bdced5c
SRRH
2110 * When a high priority task schedules out from a CPU and a lower priority
2111 * task is scheduled in, a check is made to see if there's any RT tasks
2112 * on other CPUs that are waiting to run because a higher priority RT task
2113 * is currently running on its CPU. In this case, the CPU with multiple RT
2114 * tasks queued on it (overloaded) needs to be notified that a CPU has opened
2115 * up that may be able to run one of its non-running queued RT tasks.
2116 *
2117 * All CPUs with overloaded RT tasks need to be notified as there is currently
2118 * no way to know which of these CPUs have the highest priority task waiting
2119 * to run. Instead of trying to take a spinlock on each of these CPUs,
2120 * which has shown to cause large latency when done on machines with many
2121 * CPUs, sending an IPI to the CPUs to have them push off the overloaded
2122 * RT tasks waiting to run.
2123 *
2124 * Just sending an IPI to each of the CPUs is also an issue, as on large
2125 * count CPU machines, this can cause an IPI storm on a CPU, especially
2126 * if its the only CPU with multiple RT tasks queued, and a large number
2127 * of CPUs scheduling a lower priority task at the same time.
2128 *
402de7fc 2129 * Each root domain has its own IRQ work function that can iterate over
4bdced5c 2130 * all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT
3b03706f 2131 * task must be checked if there's one or many CPUs that are lowering
402de7fc 2132 * their priority, there's a single IRQ work iterator that will try to
4bdced5c
SRRH
2133 * push off RT tasks that are waiting to run.
2134 *
2135 * When a CPU schedules a lower priority task, it will kick off the
402de7fc 2136 * IRQ work iterator that will jump to each CPU with overloaded RT tasks.
4bdced5c
SRRH
2137 * As it only takes the first CPU that schedules a lower priority task
2138 * to start the process, the rto_start variable is incremented and if
2139 * the atomic result is one, then that CPU will try to take the rto_lock.
2140 * This prevents high contention on the lock as the process handles all
2141 * CPUs scheduling lower priority tasks.
2142 *
2143 * All CPUs that are scheduling a lower priority task will increment the
402de7fc 2144 * rt_loop_next variable. This will make sure that the IRQ work iterator
4bdced5c
SRRH
2145 * checks all RT overloaded CPUs whenever a CPU schedules a new lower
2146 * priority task, even if the iterator is in the middle of a scan. Incrementing
2147 * the rt_loop_next will cause the iterator to perform another scan.
b6366f04 2148 *
b6366f04 2149 */
ad0f1d9d 2150static int rto_next_cpu(struct root_domain *rd)
b6366f04 2151{
4bdced5c 2152 int next;
b6366f04
SR
2153 int cpu;
2154
b6366f04 2155 /*
4bdced5c
SRRH
2156 * When starting the IPI RT pushing, the rto_cpu is set to -1,
2157 * rt_next_cpu() will simply return the first CPU found in
2158 * the rto_mask.
2159 *
97fb7a0a 2160 * If rto_next_cpu() is called with rto_cpu is a valid CPU, it
4bdced5c
SRRH
2161 * will return the next CPU found in the rto_mask.
2162 *
2163 * If there are no more CPUs left in the rto_mask, then a check is made
2164 * against rto_loop and rto_loop_next. rto_loop is only updated with
2165 * the rto_lock held, but any CPU may increment the rto_loop_next
2166 * without any locking.
b6366f04 2167 */
4bdced5c 2168 for (;;) {
b6366f04 2169
4bdced5c
SRRH
2170 /* When rto_cpu is -1 this acts like cpumask_first() */
2171 cpu = cpumask_next(rd->rto_cpu, rd->rto_mask);
b6366f04 2172
4bdced5c 2173 rd->rto_cpu = cpu;
b6366f04 2174
4bdced5c
SRRH
2175 if (cpu < nr_cpu_ids)
2176 return cpu;
b6366f04 2177
4bdced5c
SRRH
2178 rd->rto_cpu = -1;
2179
2180 /*
2181 * ACQUIRE ensures we see the @rto_mask changes
2182 * made prior to the @next value observed.
2183 *
2184 * Matches WMB in rt_set_overload().
2185 */
2186 next = atomic_read_acquire(&rd->rto_loop_next);
b6366f04 2187
4bdced5c 2188 if (rd->rto_loop == next)
b6366f04 2189 break;
4bdced5c
SRRH
2190
2191 rd->rto_loop = next;
b6366f04
SR
2192 }
2193
4bdced5c 2194 return -1;
b6366f04
SR
2195}
2196
4bdced5c
SRRH
2197static inline bool rto_start_trylock(atomic_t *v)
2198{
2199 return !atomic_cmpxchg_acquire(v, 0, 1);
2200}
b6366f04 2201
4bdced5c 2202static inline void rto_start_unlock(atomic_t *v)
b6366f04 2203{
4bdced5c
SRRH
2204 atomic_set_release(v, 0);
2205}
b6366f04 2206
4bdced5c
SRRH
2207static void tell_cpu_to_push(struct rq *rq)
2208{
2209 int cpu = -1;
b6366f04 2210
4bdced5c
SRRH
2211 /* Keep the loop going if the IPI is currently active */
2212 atomic_inc(&rq->rd->rto_loop_next);
b6366f04 2213
4bdced5c
SRRH
2214 /* Only one CPU can initiate a loop at a time */
2215 if (!rto_start_trylock(&rq->rd->rto_loop_start))
b6366f04
SR
2216 return;
2217
4bdced5c 2218 raw_spin_lock(&rq->rd->rto_lock);
b6366f04 2219
4bdced5c 2220 /*
97fb7a0a 2221 * The rto_cpu is updated under the lock, if it has a valid CPU
4bdced5c
SRRH
2222 * then the IPI is still running and will continue due to the
2223 * update to loop_next, and nothing needs to be done here.
402de7fc 2224 * Otherwise it is finishing up and an IPI needs to be sent.
4bdced5c
SRRH
2225 */
2226 if (rq->rd->rto_cpu < 0)
ad0f1d9d 2227 cpu = rto_next_cpu(rq->rd);
4bdced5c
SRRH
2228
2229 raw_spin_unlock(&rq->rd->rto_lock);
2230
2231 rto_start_unlock(&rq->rd->rto_loop_start);
2232
364f5665
SRV
2233 if (cpu >= 0) {
2234 /* Make sure the rd does not get freed while pushing */
2235 sched_get_rd(rq->rd);
4bdced5c 2236 irq_work_queue_on(&rq->rd->rto_push_work, cpu);
364f5665 2237 }
b6366f04
SR
2238}
2239
2240/* Called from hardirq context */
4bdced5c 2241void rto_push_irq_work_func(struct irq_work *work)
b6366f04 2242{
ad0f1d9d
SRV
2243 struct root_domain *rd =
2244 container_of(work, struct root_domain, rto_push_work);
4bdced5c 2245 struct rq *rq;
b6366f04
SR
2246 int cpu;
2247
4bdced5c 2248 rq = this_rq();
b6366f04 2249
4bdced5c
SRRH
2250 /*
2251 * We do not need to grab the lock to check for has_pushable_tasks.
2252 * When it gets updated, a check is made if a push is possible.
2253 */
b6366f04 2254 if (has_pushable_tasks(rq)) {
5cb9eaa3 2255 raw_spin_rq_lock(rq);
a7c81556
PZ
2256 while (push_rt_task(rq, true))
2257 ;
5cb9eaa3 2258 raw_spin_rq_unlock(rq);
b6366f04
SR
2259 }
2260
ad0f1d9d 2261 raw_spin_lock(&rd->rto_lock);
b6366f04 2262
4bdced5c 2263 /* Pass the IPI to the next rt overloaded queue */
ad0f1d9d 2264 cpu = rto_next_cpu(rd);
b6366f04 2265
ad0f1d9d 2266 raw_spin_unlock(&rd->rto_lock);
b6366f04 2267
364f5665
SRV
2268 if (cpu < 0) {
2269 sched_put_rd(rd);
b6366f04 2270 return;
364f5665 2271 }
b6366f04 2272
b6366f04 2273 /* Try the next RT overloaded CPU */
ad0f1d9d 2274 irq_work_queue_on(&rd->rto_push_work, cpu);
b6366f04
SR
2275}
2276#endif /* HAVE_RT_PUSH_IPI */
2277
8046d680 2278static void pull_rt_task(struct rq *this_rq)
f65eda4f 2279{
8046d680
PZ
2280 int this_cpu = this_rq->cpu, cpu;
2281 bool resched = false;
a7c81556 2282 struct task_struct *p, *push_task;
f65eda4f 2283 struct rq *src_rq;
f73c52a5 2284 int rt_overload_count = rt_overloaded(this_rq);
f65eda4f 2285
f73c52a5 2286 if (likely(!rt_overload_count))
8046d680 2287 return;
f65eda4f 2288
7c3f2ab7
PZ
2289 /*
2290 * Match the barrier from rt_set_overloaded; this guarantees that if we
2291 * see overloaded we must also see the rto_mask bit.
2292 */
2293 smp_rmb();
2294
f73c52a5
SR
2295 /* If we are the only overloaded CPU do nothing */
2296 if (rt_overload_count == 1 &&
2297 cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask))
2298 return;
2299
b6366f04
SR
2300#ifdef HAVE_RT_PUSH_IPI
2301 if (sched_feat(RT_PUSH_IPI)) {
2302 tell_cpu_to_push(this_rq);
8046d680 2303 return;
b6366f04
SR
2304 }
2305#endif
2306
c6c4927b 2307 for_each_cpu(cpu, this_rq->rd->rto_mask) {
f65eda4f
SR
2308 if (this_cpu == cpu)
2309 continue;
2310
2311 src_rq = cpu_rq(cpu);
74ab8e4f
GH
2312
2313 /*
2314 * Don't bother taking the src_rq->lock if the next highest
2315 * task is known to be lower-priority than our current task.
2316 * This may look racy, but if this value is about to go
2317 * logically higher, the src_rq will push this task away.
2318 * And if its going logically lower, we do not care
2319 */
2320 if (src_rq->rt.highest_prio.next >=
2321 this_rq->rt.highest_prio.curr)
2322 continue;
2323
f65eda4f
SR
2324 /*
2325 * We can potentially drop this_rq's lock in
2326 * double_lock_balance, and another CPU could
a8728944 2327 * alter this_rq
f65eda4f 2328 */
a7c81556 2329 push_task = NULL;
a8728944 2330 double_lock_balance(this_rq, src_rq);
f65eda4f
SR
2331
2332 /*
e23ee747
KT
2333 * We can pull only a task, which is pushable
2334 * on its rq, and no others.
f65eda4f 2335 */
e23ee747 2336 p = pick_highest_pushable_task(src_rq, this_cpu);
f65eda4f
SR
2337
2338 /*
2339 * Do we have an RT task that preempts
2340 * the to-be-scheduled task?
2341 */
a8728944 2342 if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
f65eda4f 2343 WARN_ON(p == src_rq->curr);
da0c1e65 2344 WARN_ON(!task_on_rq_queued(p));
f65eda4f
SR
2345
2346 /*
2347 * There's a chance that p is higher in priority
97fb7a0a 2348 * than what's currently running on its CPU.
3b03706f 2349 * This is just that p is waking up and hasn't
f65eda4f
SR
2350 * had a chance to schedule. We only pull
2351 * p if it is lower in priority than the
a8728944 2352 * current task on the run queue
f65eda4f 2353 */
af0c8b2b 2354 if (p->prio < src_rq->donor->prio)
614ee1f6 2355 goto skip;
f65eda4f 2356
a7c81556
PZ
2357 if (is_migration_disabled(p)) {
2358 push_task = get_push_task(src_rq);
2359 } else {
2b05a0b4 2360 move_queued_task_locked(src_rq, this_rq, p);
a7c81556
PZ
2361 resched = true;
2362 }
f65eda4f
SR
2363 /*
2364 * We continue with the search, just in
2365 * case there's an even higher prio task
25985edc 2366 * in another runqueue. (low likelihood
f65eda4f 2367 * but possible)
f65eda4f 2368 */
f65eda4f 2369 }
49246274 2370skip:
1b12bbc7 2371 double_unlock_balance(this_rq, src_rq);
a7c81556
PZ
2372
2373 if (push_task) {
f0498d2a 2374 preempt_disable();
5cb9eaa3 2375 raw_spin_rq_unlock(this_rq);
a7c81556
PZ
2376 stop_one_cpu_nowait(src_rq->cpu, push_cpu_stop,
2377 push_task, &src_rq->push_work);
f0498d2a 2378 preempt_enable();
5cb9eaa3 2379 raw_spin_rq_lock(this_rq);
a7c81556 2380 }
f65eda4f
SR
2381 }
2382
8046d680
PZ
2383 if (resched)
2384 resched_curr(this_rq);
f65eda4f
SR
2385}
2386
8ae121ac
GH
2387/*
2388 * If we are not running and we are not going to reschedule soon, we should
2389 * try to push tasks away now
2390 */
efbbd05a 2391static void task_woken_rt(struct rq *rq, struct task_struct *p)
4642dafd 2392{
0b9d46fc 2393 bool need_to_push = !task_on_cpu(rq, p) &&
804d402f
QY
2394 !test_tsk_need_resched(rq->curr) &&
2395 p->nr_cpus_allowed > 1 &&
af0c8b2b 2396 (dl_task(rq->donor) || rt_task(rq->donor)) &&
804d402f 2397 (rq->curr->nr_cpus_allowed < 2 ||
af0c8b2b 2398 rq->donor->prio <= p->prio);
804d402f 2399
d94a9df4 2400 if (need_to_push)
4642dafd
SR
2401 push_rt_tasks(rq);
2402}
2403
bdd7c81b 2404/* Assumes rq->lock is held */
1f11eb6a 2405static void rq_online_rt(struct rq *rq)
bdd7c81b
IM
2406{
2407 if (rq->rt.overloaded)
2408 rt_set_overload(rq);
6e0534f2 2409
7def2be1
PZ
2410 __enable_runtime(rq);
2411
e864c499 2412 cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
bdd7c81b
IM
2413}
2414
2415/* Assumes rq->lock is held */
1f11eb6a 2416static void rq_offline_rt(struct rq *rq)
bdd7c81b
IM
2417{
2418 if (rq->rt.overloaded)
2419 rt_clear_overload(rq);
6e0534f2 2420
7def2be1
PZ
2421 __disable_runtime(rq);
2422
6e0534f2 2423 cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
bdd7c81b 2424}
cb469845
SR
2425
2426/*
2427 * When switch from the rt queue, we bring ourselves to a position
2428 * that we might want to pull RT tasks from other runqueues.
2429 */
da7a735e 2430static void switched_from_rt(struct rq *rq, struct task_struct *p)
cb469845
SR
2431{
2432 /*
2433 * If there are other RT tasks then we will reschedule
2434 * and the scheduling of the other RT tasks will handle
2435 * the balancing. But if we are the last RT task
2436 * we may need to handle the pulling of RT tasks
2437 * now.
2438 */
da0c1e65 2439 if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)
1158ddb5
KT
2440 return;
2441
02d8ec94 2442 rt_queue_pull_task(rq);
cb469845 2443}
3d8cbdf8 2444
11c785b7 2445void __init init_sched_rt_class(void)
3d8cbdf8
RR
2446{
2447 unsigned int i;
2448
029632fb 2449 for_each_possible_cpu(i) {
eaa95840 2450 zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
6ca09dfc 2451 GFP_KERNEL, cpu_to_node(i));
029632fb 2452 }
3d8cbdf8 2453}
cb469845
SR
2454#endif /* CONFIG_SMP */
2455
2456/*
2457 * When switching a task to RT, we may overload the runqueue
2458 * with RT tasks. In this case we try to push them off to
2459 * other runqueues.
2460 */
da7a735e 2461static void switched_to_rt(struct rq *rq, struct task_struct *p)
cb469845 2462{
cb469845 2463 /*
fecfcbc2
VD
2464 * If we are running, update the avg_rt tracking, as the running time
2465 * will now on be accounted into the latter.
2466 */
2467 if (task_current(rq, p)) {
2468 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
2469 return;
2470 }
2471
2472 /*
2473 * If we are not running we may need to preempt the current
2474 * running task. If that current running task is also an RT task
cb469845
SR
2475 * then see if we can move to another run queue.
2476 */
fecfcbc2 2477 if (task_on_rq_queued(p)) {
cb469845 2478#ifdef CONFIG_SMP
d94a9df4 2479 if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
02d8ec94 2480 rt_queue_push_tasks(rq);
619bd4a7 2481#endif /* CONFIG_SMP */
af0c8b2b 2482 if (p->prio < rq->donor->prio && cpu_online(cpu_of(rq)))
8875125e 2483 resched_curr(rq);
cb469845
SR
2484 }
2485}
2486
2487/*
2488 * Priority of the task has changed. This may cause
2489 * us to initiate a push or pull.
2490 */
da7a735e
PZ
2491static void
2492prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
cb469845 2493{
da0c1e65 2494 if (!task_on_rq_queued(p))
da7a735e
PZ
2495 return;
2496
af0c8b2b 2497 if (task_current_donor(rq, p)) {
cb469845
SR
2498#ifdef CONFIG_SMP
2499 /*
2500 * If our priority decreases while running, we
2501 * may need to pull tasks to this runqueue.
2502 */
2503 if (oldprio < p->prio)
02d8ec94 2504 rt_queue_pull_task(rq);
fd7a4bed 2505
cb469845
SR
2506 /*
2507 * If there's a higher priority task waiting to run
fd7a4bed 2508 * then reschedule.
cb469845 2509 */
fd7a4bed 2510 if (p->prio > rq->rt.highest_prio.curr)
8875125e 2511 resched_curr(rq);
cb469845
SR
2512#else
2513 /* For UP simply resched on drop of prio */
2514 if (oldprio < p->prio)
8875125e 2515 resched_curr(rq);
e8fa1362 2516#endif /* CONFIG_SMP */
cb469845
SR
2517 } else {
2518 /*
2519 * This task is not running, but if it is
2520 * greater than the current running task
2521 * then reschedule.
2522 */
af0c8b2b 2523 if (p->prio < rq->donor->prio)
8875125e 2524 resched_curr(rq);
cb469845
SR
2525 }
2526}
2527
b18b6a9c 2528#ifdef CONFIG_POSIX_TIMERS
78f2c7db
PZ
2529static void watchdog(struct rq *rq, struct task_struct *p)
2530{
2531 unsigned long soft, hard;
2532
78d7d407
JS
2533 /* max may change after cur was read, this will be fixed next tick */
2534 soft = task_rlimit(p, RLIMIT_RTTIME);
2535 hard = task_rlimit_max(p, RLIMIT_RTTIME);
78f2c7db
PZ
2536
2537 if (soft != RLIM_INFINITY) {
2538 unsigned long next;
2539
57d2aa00
YX
2540 if (p->rt.watchdog_stamp != jiffies) {
2541 p->rt.timeout++;
2542 p->rt.watchdog_stamp = jiffies;
2543 }
2544
78f2c7db 2545 next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
3a245c0f
TG
2546 if (p->rt.timeout > next) {
2547 posix_cputimers_rt_watchdog(&p->posix_cputimers,
2548 p->se.sum_exec_runtime);
2549 }
78f2c7db
PZ
2550 }
2551}
b18b6a9c
NP
2552#else
2553static inline void watchdog(struct rq *rq, struct task_struct *p) { }
2554#endif
bb44e5d1 2555
d84b3131
FW
2556/*
2557 * scheduler tick hitting a task of our scheduling class.
2558 *
2559 * NOTE: This function can be called remotely by the tick offload that
2560 * goes along full dynticks. Therefore no local assumption can be made
2561 * and everything must be accessed through the @rq and @curr passed in
2562 * parameters.
2563 */
8f4d37ec 2564static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
bb44e5d1 2565{
454c7999
CC
2566 struct sched_rt_entity *rt_se = &p->rt;
2567
67e2be02 2568 update_curr_rt(rq);
23127296 2569 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
67e2be02 2570
78f2c7db
PZ
2571 watchdog(rq, p);
2572
bb44e5d1 2573 /*
402de7fc 2574 * RR tasks need a special form of time-slice management.
bb44e5d1
IM
2575 * FIFO tasks have no timeslices.
2576 */
2577 if (p->policy != SCHED_RR)
2578 return;
2579
fa717060 2580 if (--p->rt.time_slice)
bb44e5d1
IM
2581 return;
2582
ce0dbbbb 2583 p->rt.time_slice = sched_rr_timeslice;
bb44e5d1 2584
98fbc798 2585 /*
e9aa39bb
LB
2586 * Requeue to the end of queue if we (and all of our ancestors) are not
2587 * the only element on the queue
98fbc798 2588 */
454c7999
CC
2589 for_each_sched_rt_entity(rt_se) {
2590 if (rt_se->run_list.prev != rt_se->run_list.next) {
2591 requeue_task_rt(rq, p, 0);
8aa6f0eb 2592 resched_curr(rq);
454c7999
CC
2593 return;
2594 }
98fbc798 2595 }
bb44e5d1
IM
2596}
2597
6d686f45 2598static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
0d721cea
PW
2599{
2600 /*
2601 * Time slice is 0 for SCHED_FIFO tasks
2602 */
2603 if (task->policy == SCHED_RR)
ce0dbbbb 2604 return sched_rr_timeslice;
0d721cea
PW
2605 else
2606 return 0;
2607}
2608
530bfad1
HJ
2609#ifdef CONFIG_SCHED_CORE
2610static int task_is_throttled_rt(struct task_struct *p, int cpu)
2611{
2612 struct rt_rq *rt_rq;
2613
87f1fb77 2614#ifdef CONFIG_RT_GROUP_SCHED // XXX maybe add task_rt_rq(), see also sched_rt_period_rt_rq
530bfad1 2615 rt_rq = task_group(p)->rt_rq[cpu];
87f1fb77 2616 WARN_ON(!rt_group_sched_enabled() && rt_rq->tg != &root_task_group);
530bfad1
HJ
2617#else
2618 rt_rq = &cpu_rq(cpu)->rt;
2619#endif
2620
2621 return rt_rq_throttled(rt_rq);
2622}
2623#endif
2624
43c31ac0
PZ
2625DEFINE_SCHED_CLASS(rt) = {
2626
bb44e5d1
IM
2627 .enqueue_task = enqueue_task_rt,
2628 .dequeue_task = dequeue_task_rt,
2629 .yield_task = yield_task_rt,
2630
e23edc86 2631 .wakeup_preempt = wakeup_preempt_rt,
bb44e5d1 2632
fd03c5b8 2633 .pick_task = pick_task_rt,
bb44e5d1 2634 .put_prev_task = put_prev_task_rt,
03b7fad1 2635 .set_next_task = set_next_task_rt,
bb44e5d1 2636
681f3e68 2637#ifdef CONFIG_SMP
6e2df058 2638 .balance = balance_rt,
4ce72a2c 2639 .select_task_rq = select_task_rq_rt,
6c37067e 2640 .set_cpus_allowed = set_cpus_allowed_common,
1f11eb6a
GH
2641 .rq_online = rq_online_rt,
2642 .rq_offline = rq_offline_rt,
efbbd05a 2643 .task_woken = task_woken_rt,
cb469845 2644 .switched_from = switched_from_rt,
a7c81556 2645 .find_lock_rq = find_lock_lowest_rq,
681f3e68 2646#endif
bb44e5d1
IM
2647
2648 .task_tick = task_tick_rt,
cb469845 2649
0d721cea
PW
2650 .get_rr_interval = get_rr_interval_rt,
2651
cb469845
SR
2652 .prio_changed = prio_changed_rt,
2653 .switched_to = switched_to_rt,
6e998916
SG
2654
2655 .update_curr = update_curr_rt,
982d9cdc 2656
530bfad1
HJ
2657#ifdef CONFIG_SCHED_CORE
2658 .task_is_throttled = task_is_throttled_rt,
2659#endif
2660
982d9cdc
PB
2661#ifdef CONFIG_UCLAMP_TASK
2662 .uclamp_enabled = 1,
2663#endif
bb44e5d1 2664};
ada18de2 2665
8887cd99
NP
2666#ifdef CONFIG_RT_GROUP_SCHED
2667/*
2668 * Ensure that the real time constraints are schedulable.
2669 */
2670static DEFINE_MUTEX(rt_constraints_mutex);
2671
8887cd99
NP
2672static inline int tg_has_rt_tasks(struct task_group *tg)
2673{
b4fb015e
KK
2674 struct task_struct *task;
2675 struct css_task_iter it;
2676 int ret = 0;
8887cd99
NP
2677
2678 /*
2679 * Autogroups do not have RT tasks; see autogroup_create().
2680 */
2681 if (task_group_is_autogroup(tg))
2682 return 0;
2683
b4fb015e
KK
2684 css_task_iter_start(&tg->css, 0, &it);
2685 while (!ret && (task = css_task_iter_next(&it)))
2686 ret |= rt_task(task);
2687 css_task_iter_end(&it);
8887cd99 2688
b4fb015e 2689 return ret;
8887cd99
NP
2690}
2691
2692struct rt_schedulable_data {
2693 struct task_group *tg;
2694 u64 rt_period;
2695 u64 rt_runtime;
2696};
2697
2698static int tg_rt_schedulable(struct task_group *tg, void *data)
2699{
2700 struct rt_schedulable_data *d = data;
2701 struct task_group *child;
2702 unsigned long total, sum = 0;
2703 u64 period, runtime;
2704
2705 period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2706 runtime = tg->rt_bandwidth.rt_runtime;
2707
2708 if (tg == d->tg) {
2709 period = d->rt_period;
2710 runtime = d->rt_runtime;
2711 }
2712
2713 /*
2714 * Cannot have more runtime than the period.
2715 */
2716 if (runtime > period && runtime != RUNTIME_INF)
2717 return -EINVAL;
2718
2719 /*
b4fb015e 2720 * Ensure we don't starve existing RT tasks if runtime turns zero.
8887cd99 2721 */
b4fb015e
KK
2722 if (rt_bandwidth_enabled() && !runtime &&
2723 tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg))
8887cd99
NP
2724 return -EBUSY;
2725
87f1fb77
MK
2726 if (WARN_ON(!rt_group_sched_enabled() && tg != &root_task_group))
2727 return -EBUSY;
2728
8887cd99
NP
2729 total = to_ratio(period, runtime);
2730
2731 /*
2732 * Nobody can have more than the global setting allows.
2733 */
2734 if (total > to_ratio(global_rt_period(), global_rt_runtime()))
2735 return -EINVAL;
2736
2737 /*
2738 * The sum of our children's runtime should not exceed our own.
2739 */
2740 list_for_each_entry_rcu(child, &tg->children, siblings) {
2741 period = ktime_to_ns(child->rt_bandwidth.rt_period);
2742 runtime = child->rt_bandwidth.rt_runtime;
2743
2744 if (child == d->tg) {
2745 period = d->rt_period;
2746 runtime = d->rt_runtime;
2747 }
2748
2749 sum += to_ratio(period, runtime);
2750 }
2751
2752 if (sum > total)
2753 return -EINVAL;
2754
2755 return 0;
2756}
2757
2758static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
2759{
2760 int ret;
2761
2762 struct rt_schedulable_data data = {
2763 .tg = tg,
2764 .rt_period = period,
2765 .rt_runtime = runtime,
2766 };
2767
2768 rcu_read_lock();
2769 ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
2770 rcu_read_unlock();
2771
2772 return ret;
2773}
2774
2775static int tg_set_rt_bandwidth(struct task_group *tg,
2776 u64 rt_period, u64 rt_runtime)
2777{
2778 int i, err = 0;
2779
2780 /*
2781 * Disallowing the root group RT runtime is BAD, it would disallow the
2782 * kernel creating (and or operating) RT threads.
2783 */
2784 if (tg == &root_task_group && rt_runtime == 0)
2785 return -EINVAL;
2786
2787 /* No period doesn't make any sense. */
2788 if (rt_period == 0)
2789 return -EINVAL;
2790
d505b8af
HC
2791 /*
2792 * Bound quota to defend quota against overflow during bandwidth shift.
2793 */
2794 if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime)
2795 return -EINVAL;
2796
8887cd99 2797 mutex_lock(&rt_constraints_mutex);
8887cd99
NP
2798 err = __rt_schedulable(tg, rt_period, rt_runtime);
2799 if (err)
2800 goto unlock;
2801
2802 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2803 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
2804 tg->rt_bandwidth.rt_runtime = rt_runtime;
2805
2806 for_each_possible_cpu(i) {
2807 struct rt_rq *rt_rq = tg->rt_rq[i];
2808
2809 raw_spin_lock(&rt_rq->rt_runtime_lock);
2810 rt_rq->rt_runtime = rt_runtime;
2811 raw_spin_unlock(&rt_rq->rt_runtime_lock);
2812 }
2813 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
2814unlock:
8887cd99
NP
2815 mutex_unlock(&rt_constraints_mutex);
2816
2817 return err;
2818}
2819
2820int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
2821{
2822 u64 rt_runtime, rt_period;
2823
2824 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
2825 rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
2826 if (rt_runtime_us < 0)
2827 rt_runtime = RUNTIME_INF;
1a010e29
KK
2828 else if ((u64)rt_runtime_us > U64_MAX / NSEC_PER_USEC)
2829 return -EINVAL;
8887cd99
NP
2830
2831 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2832}
2833
2834long sched_group_rt_runtime(struct task_group *tg)
2835{
2836 u64 rt_runtime_us;
2837
2838 if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
2839 return -1;
2840
2841 rt_runtime_us = tg->rt_bandwidth.rt_runtime;
2842 do_div(rt_runtime_us, NSEC_PER_USEC);
2843 return rt_runtime_us;
2844}
2845
2846int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
2847{
2848 u64 rt_runtime, rt_period;
2849
1a010e29
KK
2850 if (rt_period_us > U64_MAX / NSEC_PER_USEC)
2851 return -EINVAL;
2852
8887cd99
NP
2853 rt_period = rt_period_us * NSEC_PER_USEC;
2854 rt_runtime = tg->rt_bandwidth.rt_runtime;
2855
2856 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
2857}
2858
2859long sched_group_rt_period(struct task_group *tg)
2860{
2861 u64 rt_period_us;
2862
2863 rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
2864 do_div(rt_period_us, NSEC_PER_USEC);
2865 return rt_period_us;
2866}
2867
28f152cd 2868#ifdef CONFIG_SYSCTL
8887cd99
NP
2869static int sched_rt_global_constraints(void)
2870{
2871 int ret = 0;
2872
2873 mutex_lock(&rt_constraints_mutex);
8887cd99 2874 ret = __rt_schedulable(NULL, 0, 0);
8887cd99
NP
2875 mutex_unlock(&rt_constraints_mutex);
2876
2877 return ret;
2878}
28f152cd 2879#endif /* CONFIG_SYSCTL */
8887cd99
NP
2880
2881int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
2882{
402de7fc 2883 /* Don't accept real-time tasks when there is no way for them to run */
277e0909 2884 if (rt_group_sched_enabled() && rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
8887cd99
NP
2885 return 0;
2886
2887 return 1;
2888}
2889
2890#else /* !CONFIG_RT_GROUP_SCHED */
28f152cd
BZ
2891
2892#ifdef CONFIG_SYSCTL
8887cd99
NP
2893static int sched_rt_global_constraints(void)
2894{
8887cd99
NP
2895 return 0;
2896}
28f152cd 2897#endif /* CONFIG_SYSCTL */
8887cd99
NP
2898#endif /* CONFIG_RT_GROUP_SCHED */
2899
28f152cd 2900#ifdef CONFIG_SYSCTL
8887cd99
NP
2901static int sched_rt_global_validate(void)
2902{
8887cd99 2903 if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
d505b8af
HC
2904 ((sysctl_sched_rt_runtime > sysctl_sched_rt_period) ||
2905 ((u64)sysctl_sched_rt_runtime *
2906 NSEC_PER_USEC > max_rt_runtime)))
8887cd99
NP
2907 return -EINVAL;
2908
2909 return 0;
2910}
2911
2912static void sched_rt_do_global(void)
2913{
8887cd99
NP
2914}
2915
78eb4ea2 2916static int sched_rt_handler(const struct ctl_table *table, int write, void *buffer,
32927393 2917 size_t *lenp, loff_t *ppos)
8887cd99
NP
2918{
2919 int old_period, old_runtime;
2920 static DEFINE_MUTEX(mutex);
2921 int ret;
2922
2923 mutex_lock(&mutex);
45007c6f 2924 sched_domains_mutex_lock();
8887cd99
NP
2925 old_period = sysctl_sched_rt_period;
2926 old_runtime = sysctl_sched_rt_runtime;
2927
079be8fc 2928 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
8887cd99
NP
2929
2930 if (!ret && write) {
2931 ret = sched_rt_global_validate();
2932 if (ret)
2933 goto undo;
2934
2935 ret = sched_dl_global_validate();
2936 if (ret)
2937 goto undo;
2938
2939 ret = sched_rt_global_constraints();
2940 if (ret)
2941 goto undo;
2942
2943 sched_rt_do_global();
2944 sched_dl_do_global();
2945 }
2946 if (0) {
2947undo:
2948 sysctl_sched_rt_period = old_period;
2949 sysctl_sched_rt_runtime = old_runtime;
2950 }
45007c6f 2951 sched_domains_mutex_unlock();
8887cd99
NP
2952 mutex_unlock(&mutex);
2953
2954 return ret;
2955}
2956
78eb4ea2 2957static int sched_rr_handler(const struct ctl_table *table, int write, void *buffer,
32927393 2958 size_t *lenp, loff_t *ppos)
8887cd99
NP
2959{
2960 int ret;
2961 static DEFINE_MUTEX(mutex);
2962
2963 mutex_lock(&mutex);
2964 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2965 /*
2966 * Make sure that internally we keep jiffies.
402de7fc 2967 * Also, writing zero resets the time-slice to default:
8887cd99
NP
2968 */
2969 if (!ret && write) {
2970 sched_rr_timeslice =
2971 sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE :
2972 msecs_to_jiffies(sysctl_sched_rr_timeslice);
c1fc6484
CH
2973
2974 if (sysctl_sched_rr_timeslice <= 0)
2975 sysctl_sched_rr_timeslice = jiffies_to_msecs(RR_TIMESLICE);
8887cd99
NP
2976 }
2977 mutex_unlock(&mutex);
97fb7a0a 2978
8887cd99
NP
2979 return ret;
2980}
28f152cd 2981#endif /* CONFIG_SYSCTL */
8887cd99 2982
029632fb 2983void print_rt_stats(struct seq_file *m, int cpu)
ada18de2 2984{
ec514c48 2985 rt_rq_iter_t iter;
ada18de2
PZ
2986 struct rt_rq *rt_rq;
2987
2988 rcu_read_lock();
ec514c48 2989 for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
ada18de2
PZ
2990 print_rt_rq(m, cpu, rt_rq);
2991 rcu_read_unlock();
2992}