ppc: Remove unused cpp symbols in kvm headers
[linux-2.6-block.git] / kernel / sched / core.c
CommitLineData
1da177e4 1/*
391e43da 2 * kernel/sched/core.c
1da177e4
LT
3 *
4 * Kernel scheduler and related syscalls
5 *
6 * Copyright (C) 1991-2002 Linus Torvalds
7 *
8 * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and
9 * make semaphores SMP safe
10 * 1998-11-19 Implemented schedule_timeout() and related stuff
11 * by Andrea Arcangeli
12 * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar:
13 * hybrid priority-list and round-robin design with
14 * an array-switch method of distributing timeslices
15 * and per-CPU runqueues. Cleanups and useful suggestions
16 * by Davide Libenzi, preemptible kernel bits by Robert Love.
17 * 2003-09-03 Interactivity tuning by Con Kolivas.
18 * 2004-04-02 Scheduler domains code by Nick Piggin
c31f2e8a
IM
19 * 2007-04-15 Work begun on replacing all interactivity tuning with a
20 * fair scheduling design by Con Kolivas.
21 * 2007-05-05 Load balancing (smp-nice) and other improvements
22 * by Peter Williams
23 * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith
24 * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri
b9131769
IM
25 * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins,
26 * Thomas Gleixner, Mike Kravetz
1da177e4
LT
27 */
28
29#include <linux/mm.h>
30#include <linux/module.h>
31#include <linux/nmi.h>
32#include <linux/init.h>
dff06c15 33#include <linux/uaccess.h>
1da177e4 34#include <linux/highmem.h>
1da177e4
LT
35#include <asm/mmu_context.h>
36#include <linux/interrupt.h>
c59ede7b 37#include <linux/capability.h>
1da177e4
LT
38#include <linux/completion.h>
39#include <linux/kernel_stat.h>
9a11b49a 40#include <linux/debug_locks.h>
cdd6c482 41#include <linux/perf_event.h>
1da177e4
LT
42#include <linux/security.h>
43#include <linux/notifier.h>
44#include <linux/profile.h>
7dfb7103 45#include <linux/freezer.h>
198e2f18 46#include <linux/vmalloc.h>
1da177e4
LT
47#include <linux/blkdev.h>
48#include <linux/delay.h>
b488893a 49#include <linux/pid_namespace.h>
1da177e4
LT
50#include <linux/smp.h>
51#include <linux/threads.h>
52#include <linux/timer.h>
53#include <linux/rcupdate.h>
54#include <linux/cpu.h>
55#include <linux/cpuset.h>
56#include <linux/percpu.h>
b5aadf7f 57#include <linux/proc_fs.h>
1da177e4 58#include <linux/seq_file.h>
e692ab53 59#include <linux/sysctl.h>
1da177e4
LT
60#include <linux/syscalls.h>
61#include <linux/times.h>
8f0ab514 62#include <linux/tsacct_kern.h>
c6fd91f0 63#include <linux/kprobes.h>
0ff92245 64#include <linux/delayacct.h>
dff06c15 65#include <linux/unistd.h>
f5ff8422 66#include <linux/pagemap.h>
8f4d37ec 67#include <linux/hrtimer.h>
30914a58 68#include <linux/tick.h>
f00b45c1
PZ
69#include <linux/debugfs.h>
70#include <linux/ctype.h>
6cd8a4bb 71#include <linux/ftrace.h>
5a0e3ad6 72#include <linux/slab.h>
f1c6f1a7 73#include <linux/init_task.h>
40401530 74#include <linux/binfmts.h>
91d1aa43 75#include <linux/context_tracking.h>
52f5684c 76#include <linux/compiler.h>
1da177e4 77
96f951ed 78#include <asm/switch_to.h>
5517d86b 79#include <asm/tlb.h>
838225b4 80#include <asm/irq_regs.h>
db7e527d 81#include <asm/mutex.h>
e6e6685a
GC
82#ifdef CONFIG_PARAVIRT
83#include <asm/paravirt.h>
84#endif
1da177e4 85
029632fb 86#include "sched.h"
ea138446 87#include "../workqueue_internal.h"
29d5e047 88#include "../smpboot.h"
6e0534f2 89
a8d154b0 90#define CREATE_TRACE_POINTS
ad8d75ff 91#include <trace/events/sched.h>
a8d154b0 92
029632fb 93void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period)
d0b27fa7 94{
58088ad0
PT
95 unsigned long delta;
96 ktime_t soft, hard, now;
d0b27fa7 97
58088ad0
PT
98 for (;;) {
99 if (hrtimer_active(period_timer))
100 break;
101
102 now = hrtimer_cb_get_time(period_timer);
103 hrtimer_forward(period_timer, now, period);
d0b27fa7 104
58088ad0
PT
105 soft = hrtimer_get_softexpires(period_timer);
106 hard = hrtimer_get_expires(period_timer);
107 delta = ktime_to_ns(ktime_sub(hard, soft));
108 __hrtimer_start_range_ns(period_timer, soft, delta,
109 HRTIMER_MODE_ABS_PINNED, 0);
110 }
111}
112
029632fb
PZ
113DEFINE_MUTEX(sched_domains_mutex);
114DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
dc61b1d6 115
fe44d621 116static void update_rq_clock_task(struct rq *rq, s64 delta);
305e6835 117
029632fb 118void update_rq_clock(struct rq *rq)
3e51f33f 119{
fe44d621 120 s64 delta;
305e6835 121
9edfbfed
PZ
122 lockdep_assert_held(&rq->lock);
123
124 if (rq->clock_skip_update & RQCF_ACT_SKIP)
f26f9aff 125 return;
aa483808 126
fe44d621 127 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
4036ac15
MG
128 if (delta < 0)
129 return;
fe44d621
PZ
130 rq->clock += delta;
131 update_rq_clock_task(rq, delta);
3e51f33f
PZ
132}
133
bf5c91ba
IM
134/*
135 * Debugging: various feature bits
136 */
f00b45c1 137
f00b45c1
PZ
138#define SCHED_FEAT(name, enabled) \
139 (1UL << __SCHED_FEAT_##name) * enabled |
140
bf5c91ba 141const_debug unsigned int sysctl_sched_features =
391e43da 142#include "features.h"
f00b45c1
PZ
143 0;
144
145#undef SCHED_FEAT
146
147#ifdef CONFIG_SCHED_DEBUG
148#define SCHED_FEAT(name, enabled) \
149 #name ,
150
1292531f 151static const char * const sched_feat_names[] = {
391e43da 152#include "features.h"
f00b45c1
PZ
153};
154
155#undef SCHED_FEAT
156
34f3a814 157static int sched_feat_show(struct seq_file *m, void *v)
f00b45c1 158{
f00b45c1
PZ
159 int i;
160
f8b6d1cc 161 for (i = 0; i < __SCHED_FEAT_NR; i++) {
34f3a814
LZ
162 if (!(sysctl_sched_features & (1UL << i)))
163 seq_puts(m, "NO_");
164 seq_printf(m, "%s ", sched_feat_names[i]);
f00b45c1 165 }
34f3a814 166 seq_puts(m, "\n");
f00b45c1 167
34f3a814 168 return 0;
f00b45c1
PZ
169}
170
f8b6d1cc
PZ
171#ifdef HAVE_JUMP_LABEL
172
c5905afb
IM
173#define jump_label_key__true STATIC_KEY_INIT_TRUE
174#define jump_label_key__false STATIC_KEY_INIT_FALSE
f8b6d1cc
PZ
175
176#define SCHED_FEAT(name, enabled) \
177 jump_label_key__##enabled ,
178
c5905afb 179struct static_key sched_feat_keys[__SCHED_FEAT_NR] = {
f8b6d1cc
PZ
180#include "features.h"
181};
182
183#undef SCHED_FEAT
184
185static void sched_feat_disable(int i)
186{
c5905afb
IM
187 if (static_key_enabled(&sched_feat_keys[i]))
188 static_key_slow_dec(&sched_feat_keys[i]);
f8b6d1cc
PZ
189}
190
191static void sched_feat_enable(int i)
192{
c5905afb
IM
193 if (!static_key_enabled(&sched_feat_keys[i]))
194 static_key_slow_inc(&sched_feat_keys[i]);
f8b6d1cc
PZ
195}
196#else
197static void sched_feat_disable(int i) { };
198static void sched_feat_enable(int i) { };
199#endif /* HAVE_JUMP_LABEL */
200
1a687c2e 201static int sched_feat_set(char *cmp)
f00b45c1 202{
f00b45c1 203 int i;
1a687c2e 204 int neg = 0;
f00b45c1 205
524429c3 206 if (strncmp(cmp, "NO_", 3) == 0) {
f00b45c1
PZ
207 neg = 1;
208 cmp += 3;
209 }
210
f8b6d1cc 211 for (i = 0; i < __SCHED_FEAT_NR; i++) {
7740191c 212 if (strcmp(cmp, sched_feat_names[i]) == 0) {
f8b6d1cc 213 if (neg) {
f00b45c1 214 sysctl_sched_features &= ~(1UL << i);
f8b6d1cc
PZ
215 sched_feat_disable(i);
216 } else {
f00b45c1 217 sysctl_sched_features |= (1UL << i);
f8b6d1cc
PZ
218 sched_feat_enable(i);
219 }
f00b45c1
PZ
220 break;
221 }
222 }
223
1a687c2e
MG
224 return i;
225}
226
227static ssize_t
228sched_feat_write(struct file *filp, const char __user *ubuf,
229 size_t cnt, loff_t *ppos)
230{
231 char buf[64];
232 char *cmp;
233 int i;
5cd08fbf 234 struct inode *inode;
1a687c2e
MG
235
236 if (cnt > 63)
237 cnt = 63;
238
239 if (copy_from_user(&buf, ubuf, cnt))
240 return -EFAULT;
241
242 buf[cnt] = 0;
243 cmp = strstrip(buf);
244
5cd08fbf
JB
245 /* Ensure the static_key remains in a consistent state */
246 inode = file_inode(filp);
247 mutex_lock(&inode->i_mutex);
1a687c2e 248 i = sched_feat_set(cmp);
5cd08fbf 249 mutex_unlock(&inode->i_mutex);
f8b6d1cc 250 if (i == __SCHED_FEAT_NR)
f00b45c1
PZ
251 return -EINVAL;
252
42994724 253 *ppos += cnt;
f00b45c1
PZ
254
255 return cnt;
256}
257
34f3a814
LZ
258static int sched_feat_open(struct inode *inode, struct file *filp)
259{
260 return single_open(filp, sched_feat_show, NULL);
261}
262
828c0950 263static const struct file_operations sched_feat_fops = {
34f3a814
LZ
264 .open = sched_feat_open,
265 .write = sched_feat_write,
266 .read = seq_read,
267 .llseek = seq_lseek,
268 .release = single_release,
f00b45c1
PZ
269};
270
271static __init int sched_init_debug(void)
272{
f00b45c1
PZ
273 debugfs_create_file("sched_features", 0644, NULL, NULL,
274 &sched_feat_fops);
275
276 return 0;
277}
278late_initcall(sched_init_debug);
f8b6d1cc 279#endif /* CONFIG_SCHED_DEBUG */
bf5c91ba 280
b82d9fdd
PZ
281/*
282 * Number of tasks to iterate in a single balance run.
283 * Limited because this is done with IRQs disabled.
284 */
285const_debug unsigned int sysctl_sched_nr_migrate = 32;
286
e9e9250b
PZ
287/*
288 * period over which we average the RT time consumption, measured
289 * in ms.
290 *
291 * default: 1s
292 */
293const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC;
294
fa85ae24 295/*
9f0c1e56 296 * period over which we measure -rt task cpu usage in us.
fa85ae24
PZ
297 * default: 1s
298 */
9f0c1e56 299unsigned int sysctl_sched_rt_period = 1000000;
fa85ae24 300
029632fb 301__read_mostly int scheduler_running;
6892b75e 302
9f0c1e56
PZ
303/*
304 * part of the period that we allow rt tasks to run in us.
305 * default: 0.95s
306 */
307int sysctl_sched_rt_runtime = 950000;
fa85ae24 308
1da177e4 309/*
cc2a73b5 310 * this_rq_lock - lock this runqueue and disable interrupts.
1da177e4 311 */
a9957449 312static struct rq *this_rq_lock(void)
1da177e4
LT
313 __acquires(rq->lock)
314{
70b97a7f 315 struct rq *rq;
1da177e4
LT
316
317 local_irq_disable();
318 rq = this_rq();
05fa785c 319 raw_spin_lock(&rq->lock);
1da177e4
LT
320
321 return rq;
322}
323
8f4d37ec
PZ
324#ifdef CONFIG_SCHED_HRTICK
325/*
326 * Use HR-timers to deliver accurate preemption points.
8f4d37ec 327 */
8f4d37ec 328
8f4d37ec
PZ
329static void hrtick_clear(struct rq *rq)
330{
331 if (hrtimer_active(&rq->hrtick_timer))
332 hrtimer_cancel(&rq->hrtick_timer);
333}
334
8f4d37ec
PZ
335/*
336 * High-resolution timer tick.
337 * Runs from hardirq context with interrupts disabled.
338 */
339static enum hrtimer_restart hrtick(struct hrtimer *timer)
340{
341 struct rq *rq = container_of(timer, struct rq, hrtick_timer);
342
343 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
344
05fa785c 345 raw_spin_lock(&rq->lock);
3e51f33f 346 update_rq_clock(rq);
8f4d37ec 347 rq->curr->sched_class->task_tick(rq, rq->curr, 1);
05fa785c 348 raw_spin_unlock(&rq->lock);
8f4d37ec
PZ
349
350 return HRTIMER_NORESTART;
351}
352
95e904c7 353#ifdef CONFIG_SMP
971ee28c
PZ
354
355static int __hrtick_restart(struct rq *rq)
356{
357 struct hrtimer *timer = &rq->hrtick_timer;
358 ktime_t time = hrtimer_get_softexpires(timer);
359
360 return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0);
361}
362
31656519
PZ
363/*
364 * called from hardirq (IPI) context
365 */
366static void __hrtick_start(void *arg)
b328ca18 367{
31656519 368 struct rq *rq = arg;
b328ca18 369
05fa785c 370 raw_spin_lock(&rq->lock);
971ee28c 371 __hrtick_restart(rq);
31656519 372 rq->hrtick_csd_pending = 0;
05fa785c 373 raw_spin_unlock(&rq->lock);
b328ca18
PZ
374}
375
31656519
PZ
376/*
377 * Called to set the hrtick timer state.
378 *
379 * called with rq->lock held and irqs disabled
380 */
029632fb 381void hrtick_start(struct rq *rq, u64 delay)
b328ca18 382{
31656519 383 struct hrtimer *timer = &rq->hrtick_timer;
177ef2a6 384 ktime_t time;
385 s64 delta;
386
387 /*
388 * Don't schedule slices shorter than 10000ns, that just
389 * doesn't make sense and can cause timer DoS.
390 */
391 delta = max_t(s64, delay, 10000LL);
392 time = ktime_add_ns(timer->base->get_time(), delta);
b328ca18 393
cc584b21 394 hrtimer_set_expires(timer, time);
31656519
PZ
395
396 if (rq == this_rq()) {
971ee28c 397 __hrtick_restart(rq);
31656519 398 } else if (!rq->hrtick_csd_pending) {
c46fff2a 399 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
31656519
PZ
400 rq->hrtick_csd_pending = 1;
401 }
b328ca18
PZ
402}
403
404static int
405hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
406{
407 int cpu = (int)(long)hcpu;
408
409 switch (action) {
410 case CPU_UP_CANCELED:
411 case CPU_UP_CANCELED_FROZEN:
412 case CPU_DOWN_PREPARE:
413 case CPU_DOWN_PREPARE_FROZEN:
414 case CPU_DEAD:
415 case CPU_DEAD_FROZEN:
31656519 416 hrtick_clear(cpu_rq(cpu));
b328ca18
PZ
417 return NOTIFY_OK;
418 }
419
420 return NOTIFY_DONE;
421}
422
fa748203 423static __init void init_hrtick(void)
b328ca18
PZ
424{
425 hotcpu_notifier(hotplug_hrtick, 0);
426}
31656519
PZ
427#else
428/*
429 * Called to set the hrtick timer state.
430 *
431 * called with rq->lock held and irqs disabled
432 */
029632fb 433void hrtick_start(struct rq *rq, u64 delay)
31656519 434{
86893335
WL
435 /*
436 * Don't schedule slices shorter than 10000ns, that just
437 * doesn't make sense. Rely on vruntime for fairness.
438 */
439 delay = max_t(u64, delay, 10000LL);
7f1e2ca9 440 __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0,
5c333864 441 HRTIMER_MODE_REL_PINNED, 0);
31656519 442}
b328ca18 443
006c75f1 444static inline void init_hrtick(void)
8f4d37ec 445{
8f4d37ec 446}
31656519 447#endif /* CONFIG_SMP */
8f4d37ec 448
31656519 449static void init_rq_hrtick(struct rq *rq)
8f4d37ec 450{
31656519
PZ
451#ifdef CONFIG_SMP
452 rq->hrtick_csd_pending = 0;
8f4d37ec 453
31656519
PZ
454 rq->hrtick_csd.flags = 0;
455 rq->hrtick_csd.func = __hrtick_start;
456 rq->hrtick_csd.info = rq;
457#endif
8f4d37ec 458
31656519
PZ
459 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
460 rq->hrtick_timer.function = hrtick;
8f4d37ec 461}
006c75f1 462#else /* CONFIG_SCHED_HRTICK */
8f4d37ec
PZ
463static inline void hrtick_clear(struct rq *rq)
464{
465}
466
8f4d37ec
PZ
467static inline void init_rq_hrtick(struct rq *rq)
468{
469}
470
b328ca18
PZ
471static inline void init_hrtick(void)
472{
473}
006c75f1 474#endif /* CONFIG_SCHED_HRTICK */
8f4d37ec 475
fd99f91a
PZ
476/*
477 * cmpxchg based fetch_or, macro so it works for different integer types
478 */
479#define fetch_or(ptr, val) \
480({ typeof(*(ptr)) __old, __val = *(ptr); \
481 for (;;) { \
482 __old = cmpxchg((ptr), __val, __val | (val)); \
483 if (__old == __val) \
484 break; \
485 __val = __old; \
486 } \
487 __old; \
488})
489
e3baac47 490#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
fd99f91a
PZ
491/*
492 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
493 * this avoids any races wrt polling state changes and thereby avoids
494 * spurious IPIs.
495 */
496static bool set_nr_and_not_polling(struct task_struct *p)
497{
498 struct thread_info *ti = task_thread_info(p);
499 return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
500}
e3baac47
PZ
501
502/*
503 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
504 *
505 * If this returns true, then the idle task promises to call
506 * sched_ttwu_pending() and reschedule soon.
507 */
508static bool set_nr_if_polling(struct task_struct *p)
509{
510 struct thread_info *ti = task_thread_info(p);
511 typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags);
512
513 for (;;) {
514 if (!(val & _TIF_POLLING_NRFLAG))
515 return false;
516 if (val & _TIF_NEED_RESCHED)
517 return true;
518 old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
519 if (old == val)
520 break;
521 val = old;
522 }
523 return true;
524}
525
fd99f91a
PZ
526#else
527static bool set_nr_and_not_polling(struct task_struct *p)
528{
529 set_tsk_need_resched(p);
530 return true;
531}
e3baac47
PZ
532
533#ifdef CONFIG_SMP
534static bool set_nr_if_polling(struct task_struct *p)
535{
536 return false;
537}
538#endif
fd99f91a
PZ
539#endif
540
c24d20db 541/*
8875125e 542 * resched_curr - mark rq's current task 'to be rescheduled now'.
c24d20db
IM
543 *
544 * On UP this means the setting of the need_resched flag, on SMP it
545 * might also involve a cross-CPU call to trigger the scheduler on
546 * the target CPU.
547 */
8875125e 548void resched_curr(struct rq *rq)
c24d20db 549{
8875125e 550 struct task_struct *curr = rq->curr;
c24d20db
IM
551 int cpu;
552
8875125e 553 lockdep_assert_held(&rq->lock);
c24d20db 554
8875125e 555 if (test_tsk_need_resched(curr))
c24d20db
IM
556 return;
557
8875125e 558 cpu = cpu_of(rq);
fd99f91a 559
f27dde8d 560 if (cpu == smp_processor_id()) {
8875125e 561 set_tsk_need_resched(curr);
f27dde8d 562 set_preempt_need_resched();
c24d20db 563 return;
f27dde8d 564 }
c24d20db 565
8875125e 566 if (set_nr_and_not_polling(curr))
c24d20db 567 smp_send_reschedule(cpu);
dfc68f29
AL
568 else
569 trace_sched_wake_idle_without_ipi(cpu);
c24d20db
IM
570}
571
029632fb 572void resched_cpu(int cpu)
c24d20db
IM
573{
574 struct rq *rq = cpu_rq(cpu);
575 unsigned long flags;
576
05fa785c 577 if (!raw_spin_trylock_irqsave(&rq->lock, flags))
c24d20db 578 return;
8875125e 579 resched_curr(rq);
05fa785c 580 raw_spin_unlock_irqrestore(&rq->lock, flags);
c24d20db 581}
06d8308c 582
b021fe3e 583#ifdef CONFIG_SMP
3451d024 584#ifdef CONFIG_NO_HZ_COMMON
83cd4fe2
VP
585/*
586 * In the semi idle case, use the nearest busy cpu for migrating timers
587 * from an idle cpu. This is good for power-savings.
588 *
589 * We don't do similar optimization for completely idle system, as
590 * selecting an idle cpu will add more delays to the timers than intended
591 * (as that cpu's timer base may not be uptodate wrt jiffies etc).
592 */
6201b4d6 593int get_nohz_timer_target(int pinned)
83cd4fe2
VP
594{
595 int cpu = smp_processor_id();
596 int i;
597 struct sched_domain *sd;
598
6201b4d6
VK
599 if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu))
600 return cpu;
601
057f3fad 602 rcu_read_lock();
83cd4fe2 603 for_each_domain(cpu, sd) {
057f3fad
PZ
604 for_each_cpu(i, sched_domain_span(sd)) {
605 if (!idle_cpu(i)) {
606 cpu = i;
607 goto unlock;
608 }
609 }
83cd4fe2 610 }
057f3fad
PZ
611unlock:
612 rcu_read_unlock();
83cd4fe2
VP
613 return cpu;
614}
06d8308c
TG
615/*
616 * When add_timer_on() enqueues a timer into the timer wheel of an
617 * idle CPU then this timer might expire before the next timer event
618 * which is scheduled to wake up that CPU. In case of a completely
619 * idle system the next event might even be infinite time into the
620 * future. wake_up_idle_cpu() ensures that the CPU is woken up and
621 * leaves the inner idle loop so the newly added timer is taken into
622 * account when the CPU goes back to idle and evaluates the timer
623 * wheel for the next timer event.
624 */
1c20091e 625static void wake_up_idle_cpu(int cpu)
06d8308c
TG
626{
627 struct rq *rq = cpu_rq(cpu);
628
629 if (cpu == smp_processor_id())
630 return;
631
67b9ca70 632 if (set_nr_and_not_polling(rq->idle))
06d8308c 633 smp_send_reschedule(cpu);
dfc68f29
AL
634 else
635 trace_sched_wake_idle_without_ipi(cpu);
45bf76df
IM
636}
637
c5bfece2 638static bool wake_up_full_nohz_cpu(int cpu)
1c20091e 639{
53c5fa16
FW
640 /*
641 * We just need the target to call irq_exit() and re-evaluate
642 * the next tick. The nohz full kick at least implies that.
643 * If needed we can still optimize that later with an
644 * empty IRQ.
645 */
c5bfece2 646 if (tick_nohz_full_cpu(cpu)) {
1c20091e
FW
647 if (cpu != smp_processor_id() ||
648 tick_nohz_tick_stopped())
53c5fa16 649 tick_nohz_full_kick_cpu(cpu);
1c20091e
FW
650 return true;
651 }
652
653 return false;
654}
655
656void wake_up_nohz_cpu(int cpu)
657{
c5bfece2 658 if (!wake_up_full_nohz_cpu(cpu))
1c20091e
FW
659 wake_up_idle_cpu(cpu);
660}
661
ca38062e 662static inline bool got_nohz_idle_kick(void)
45bf76df 663{
1c792db7 664 int cpu = smp_processor_id();
873b4c65
VG
665
666 if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)))
667 return false;
668
669 if (idle_cpu(cpu) && !need_resched())
670 return true;
671
672 /*
673 * We can't run Idle Load Balance on this CPU for this time so we
674 * cancel it and clear NOHZ_BALANCE_KICK
675 */
676 clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu));
677 return false;
45bf76df
IM
678}
679
3451d024 680#else /* CONFIG_NO_HZ_COMMON */
45bf76df 681
ca38062e 682static inline bool got_nohz_idle_kick(void)
2069dd75 683{
ca38062e 684 return false;
2069dd75
PZ
685}
686
3451d024 687#endif /* CONFIG_NO_HZ_COMMON */
d842de87 688
ce831b38
FW
689#ifdef CONFIG_NO_HZ_FULL
690bool sched_can_stop_tick(void)
691{
3882ec64
FW
692 /*
693 * More than one running task need preemption.
694 * nr_running update is assumed to be visible
695 * after IPI is sent from wakers.
696 */
541b8264
VK
697 if (this_rq()->nr_running > 1)
698 return false;
ce831b38 699
541b8264 700 return true;
ce831b38
FW
701}
702#endif /* CONFIG_NO_HZ_FULL */
d842de87 703
029632fb 704void sched_avg_update(struct rq *rq)
18d95a28 705{
e9e9250b
PZ
706 s64 period = sched_avg_period();
707
78becc27 708 while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
0d98bb26
WD
709 /*
710 * Inline assembly required to prevent the compiler
711 * optimising this loop into a divmod call.
712 * See __iter_div_u64_rem() for another example of this.
713 */
714 asm("" : "+rm" (rq->age_stamp));
e9e9250b
PZ
715 rq->age_stamp += period;
716 rq->rt_avg /= 2;
717 }
18d95a28
PZ
718}
719
6d6bc0ad 720#endif /* CONFIG_SMP */
18d95a28 721
a790de99
PT
722#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
723 (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
c09595f6 724/*
8277434e
PT
725 * Iterate task_group tree rooted at *from, calling @down when first entering a
726 * node and @up when leaving it for the final time.
727 *
728 * Caller must hold rcu_lock or sufficient equivalent.
c09595f6 729 */
029632fb 730int walk_tg_tree_from(struct task_group *from,
8277434e 731 tg_visitor down, tg_visitor up, void *data)
c09595f6
PZ
732{
733 struct task_group *parent, *child;
eb755805 734 int ret;
c09595f6 735
8277434e
PT
736 parent = from;
737
c09595f6 738down:
eb755805
PZ
739 ret = (*down)(parent, data);
740 if (ret)
8277434e 741 goto out;
c09595f6
PZ
742 list_for_each_entry_rcu(child, &parent->children, siblings) {
743 parent = child;
744 goto down;
745
746up:
747 continue;
748 }
eb755805 749 ret = (*up)(parent, data);
8277434e
PT
750 if (ret || parent == from)
751 goto out;
c09595f6
PZ
752
753 child = parent;
754 parent = parent->parent;
755 if (parent)
756 goto up;
8277434e 757out:
eb755805 758 return ret;
c09595f6
PZ
759}
760
029632fb 761int tg_nop(struct task_group *tg, void *data)
eb755805 762{
e2b245f8 763 return 0;
eb755805 764}
18d95a28
PZ
765#endif
766
45bf76df
IM
767static void set_load_weight(struct task_struct *p)
768{
f05998d4
NR
769 int prio = p->static_prio - MAX_RT_PRIO;
770 struct load_weight *load = &p->se.load;
771
dd41f596
IM
772 /*
773 * SCHED_IDLE tasks get minimal weight:
774 */
775 if (p->policy == SCHED_IDLE) {
c8b28116 776 load->weight = scale_load(WEIGHT_IDLEPRIO);
f05998d4 777 load->inv_weight = WMULT_IDLEPRIO;
dd41f596
IM
778 return;
779 }
71f8bd46 780
c8b28116 781 load->weight = scale_load(prio_to_weight[prio]);
f05998d4 782 load->inv_weight = prio_to_wmult[prio];
71f8bd46
IM
783}
784
371fd7e7 785static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
2087a1ad 786{
a64692a3 787 update_rq_clock(rq);
43148951 788 sched_info_queued(rq, p);
371fd7e7 789 p->sched_class->enqueue_task(rq, p, flags);
71f8bd46
IM
790}
791
371fd7e7 792static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
71f8bd46 793{
a64692a3 794 update_rq_clock(rq);
43148951 795 sched_info_dequeued(rq, p);
371fd7e7 796 p->sched_class->dequeue_task(rq, p, flags);
71f8bd46
IM
797}
798
029632fb 799void activate_task(struct rq *rq, struct task_struct *p, int flags)
1e3c88bd
PZ
800{
801 if (task_contributes_to_load(p))
802 rq->nr_uninterruptible--;
803
371fd7e7 804 enqueue_task(rq, p, flags);
1e3c88bd
PZ
805}
806
029632fb 807void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
1e3c88bd
PZ
808{
809 if (task_contributes_to_load(p))
810 rq->nr_uninterruptible++;
811
371fd7e7 812 dequeue_task(rq, p, flags);
1e3c88bd
PZ
813}
814
fe44d621 815static void update_rq_clock_task(struct rq *rq, s64 delta)
aa483808 816{
095c0aa8
GC
817/*
818 * In theory, the compile should just see 0 here, and optimize out the call
819 * to sched_rt_avg_update. But I don't trust it...
820 */
821#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
822 s64 steal = 0, irq_delta = 0;
823#endif
824#ifdef CONFIG_IRQ_TIME_ACCOUNTING
8e92c201 825 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
fe44d621
PZ
826
827 /*
828 * Since irq_time is only updated on {soft,}irq_exit, we might run into
829 * this case when a previous update_rq_clock() happened inside a
830 * {soft,}irq region.
831 *
832 * When this happens, we stop ->clock_task and only update the
833 * prev_irq_time stamp to account for the part that fit, so that a next
834 * update will consume the rest. This ensures ->clock_task is
835 * monotonic.
836 *
837 * It does however cause some slight miss-attribution of {soft,}irq
838 * time, a more accurate solution would be to update the irq_time using
839 * the current rq->clock timestamp, except that would require using
840 * atomic ops.
841 */
842 if (irq_delta > delta)
843 irq_delta = delta;
844
845 rq->prev_irq_time += irq_delta;
846 delta -= irq_delta;
095c0aa8
GC
847#endif
848#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
c5905afb 849 if (static_key_false((&paravirt_steal_rq_enabled))) {
095c0aa8
GC
850 steal = paravirt_steal_clock(cpu_of(rq));
851 steal -= rq->prev_steal_time_rq;
852
853 if (unlikely(steal > delta))
854 steal = delta;
855
095c0aa8 856 rq->prev_steal_time_rq += steal;
095c0aa8
GC
857 delta -= steal;
858 }
859#endif
860
fe44d621
PZ
861 rq->clock_task += delta;
862
095c0aa8 863#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
5d4dfddd 864 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
095c0aa8
GC
865 sched_rt_avg_update(rq, irq_delta + steal);
866#endif
aa483808
VP
867}
868
34f971f6
PZ
869void sched_set_stop_task(int cpu, struct task_struct *stop)
870{
871 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
872 struct task_struct *old_stop = cpu_rq(cpu)->stop;
873
874 if (stop) {
875 /*
876 * Make it appear like a SCHED_FIFO task, its something
877 * userspace knows about and won't get confused about.
878 *
879 * Also, it will make PI more or less work without too
880 * much confusion -- but then, stop work should not
881 * rely on PI working anyway.
882 */
883 sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
884
885 stop->sched_class = &stop_sched_class;
886 }
887
888 cpu_rq(cpu)->stop = stop;
889
890 if (old_stop) {
891 /*
892 * Reset it back to a normal scheduling class so that
893 * it can die in pieces.
894 */
895 old_stop->sched_class = &rt_sched_class;
896 }
897}
898
14531189 899/*
dd41f596 900 * __normal_prio - return the priority that is based on the static prio
14531189 901 */
14531189
IM
902static inline int __normal_prio(struct task_struct *p)
903{
dd41f596 904 return p->static_prio;
14531189
IM
905}
906
b29739f9
IM
907/*
908 * Calculate the expected normal priority: i.e. priority
909 * without taking RT-inheritance into account. Might be
910 * boosted by interactivity modifiers. Changes upon fork,
911 * setprio syscalls, and whenever the interactivity
912 * estimator recalculates.
913 */
36c8b586 914static inline int normal_prio(struct task_struct *p)
b29739f9
IM
915{
916 int prio;
917
aab03e05
DF
918 if (task_has_dl_policy(p))
919 prio = MAX_DL_PRIO-1;
920 else if (task_has_rt_policy(p))
b29739f9
IM
921 prio = MAX_RT_PRIO-1 - p->rt_priority;
922 else
923 prio = __normal_prio(p);
924 return prio;
925}
926
927/*
928 * Calculate the current priority, i.e. the priority
929 * taken into account by the scheduler. This value might
930 * be boosted by RT tasks, or might be boosted by
931 * interactivity modifiers. Will be RT if the task got
932 * RT-boosted. If not then it returns p->normal_prio.
933 */
36c8b586 934static int effective_prio(struct task_struct *p)
b29739f9
IM
935{
936 p->normal_prio = normal_prio(p);
937 /*
938 * If we are RT tasks or we were boosted to RT priority,
939 * keep the priority unchanged. Otherwise, update priority
940 * to the normal priority:
941 */
942 if (!rt_prio(p->prio))
943 return p->normal_prio;
944 return p->prio;
945}
946
1da177e4
LT
947/**
948 * task_curr - is this task currently executing on a CPU?
949 * @p: the task in question.
e69f6186
YB
950 *
951 * Return: 1 if the task is currently executing. 0 otherwise.
1da177e4 952 */
36c8b586 953inline int task_curr(const struct task_struct *p)
1da177e4
LT
954{
955 return cpu_curr(task_cpu(p)) == p;
956}
957
67dfa1b7
KT
958/*
959 * Can drop rq->lock because from sched_class::switched_from() methods drop it.
960 */
cb469845
SR
961static inline void check_class_changed(struct rq *rq, struct task_struct *p,
962 const struct sched_class *prev_class,
da7a735e 963 int oldprio)
cb469845
SR
964{
965 if (prev_class != p->sched_class) {
966 if (prev_class->switched_from)
da7a735e 967 prev_class->switched_from(rq, p);
67dfa1b7 968 /* Possble rq->lock 'hole'. */
da7a735e 969 p->sched_class->switched_to(rq, p);
2d3d891d 970 } else if (oldprio != p->prio || dl_task(p))
da7a735e 971 p->sched_class->prio_changed(rq, p, oldprio);
cb469845
SR
972}
973
029632fb 974void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
1e5a7405
PZ
975{
976 const struct sched_class *class;
977
978 if (p->sched_class == rq->curr->sched_class) {
979 rq->curr->sched_class->check_preempt_curr(rq, p, flags);
980 } else {
981 for_each_class(class) {
982 if (class == rq->curr->sched_class)
983 break;
984 if (class == p->sched_class) {
8875125e 985 resched_curr(rq);
1e5a7405
PZ
986 break;
987 }
988 }
989 }
990
991 /*
992 * A queue event has occurred, and we're going to schedule. In
993 * this case, we can save a useless back to back clock update.
994 */
da0c1e65 995 if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr))
9edfbfed 996 rq_clock_skip_update(rq, true);
1e5a7405
PZ
997}
998
1da177e4 999#ifdef CONFIG_SMP
dd41f596 1000void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
c65cc870 1001{
e2912009
PZ
1002#ifdef CONFIG_SCHED_DEBUG
1003 /*
1004 * We should never call set_task_cpu() on a blocked task,
1005 * ttwu() will sort out the placement.
1006 */
077614ee 1007 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
e2336f6e 1008 !p->on_rq);
0122ec5b
PZ
1009
1010#ifdef CONFIG_LOCKDEP
6c6c54e1
PZ
1011 /*
1012 * The caller should hold either p->pi_lock or rq->lock, when changing
1013 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
1014 *
1015 * sched_move_task() holds both and thus holding either pins the cgroup,
8323f26c 1016 * see task_group().
6c6c54e1
PZ
1017 *
1018 * Furthermore, all task_rq users should acquire both locks, see
1019 * task_rq_lock().
1020 */
0122ec5b
PZ
1021 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
1022 lockdep_is_held(&task_rq(p)->lock)));
1023#endif
e2912009
PZ
1024#endif
1025
de1d7286 1026 trace_sched_migrate_task(p, new_cpu);
cbc34ed1 1027
0c69774e 1028 if (task_cpu(p) != new_cpu) {
0a74bef8
PT
1029 if (p->sched_class->migrate_task_rq)
1030 p->sched_class->migrate_task_rq(p, new_cpu);
0c69774e 1031 p->se.nr_migrations++;
86038c5e 1032 perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0);
0c69774e 1033 }
dd41f596
IM
1034
1035 __set_task_cpu(p, new_cpu);
c65cc870
IM
1036}
1037
ac66f547
PZ
1038static void __migrate_swap_task(struct task_struct *p, int cpu)
1039{
da0c1e65 1040 if (task_on_rq_queued(p)) {
ac66f547
PZ
1041 struct rq *src_rq, *dst_rq;
1042
1043 src_rq = task_rq(p);
1044 dst_rq = cpu_rq(cpu);
1045
1046 deactivate_task(src_rq, p, 0);
1047 set_task_cpu(p, cpu);
1048 activate_task(dst_rq, p, 0);
1049 check_preempt_curr(dst_rq, p, 0);
1050 } else {
1051 /*
1052 * Task isn't running anymore; make it appear like we migrated
1053 * it before it went to sleep. This means on wakeup we make the
1054 * previous cpu our targer instead of where it really is.
1055 */
1056 p->wake_cpu = cpu;
1057 }
1058}
1059
1060struct migration_swap_arg {
1061 struct task_struct *src_task, *dst_task;
1062 int src_cpu, dst_cpu;
1063};
1064
1065static int migrate_swap_stop(void *data)
1066{
1067 struct migration_swap_arg *arg = data;
1068 struct rq *src_rq, *dst_rq;
1069 int ret = -EAGAIN;
1070
1071 src_rq = cpu_rq(arg->src_cpu);
1072 dst_rq = cpu_rq(arg->dst_cpu);
1073
74602315
PZ
1074 double_raw_lock(&arg->src_task->pi_lock,
1075 &arg->dst_task->pi_lock);
ac66f547
PZ
1076 double_rq_lock(src_rq, dst_rq);
1077 if (task_cpu(arg->dst_task) != arg->dst_cpu)
1078 goto unlock;
1079
1080 if (task_cpu(arg->src_task) != arg->src_cpu)
1081 goto unlock;
1082
1083 if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task)))
1084 goto unlock;
1085
1086 if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task)))
1087 goto unlock;
1088
1089 __migrate_swap_task(arg->src_task, arg->dst_cpu);
1090 __migrate_swap_task(arg->dst_task, arg->src_cpu);
1091
1092 ret = 0;
1093
1094unlock:
1095 double_rq_unlock(src_rq, dst_rq);
74602315
PZ
1096 raw_spin_unlock(&arg->dst_task->pi_lock);
1097 raw_spin_unlock(&arg->src_task->pi_lock);
ac66f547
PZ
1098
1099 return ret;
1100}
1101
1102/*
1103 * Cross migrate two tasks
1104 */
1105int migrate_swap(struct task_struct *cur, struct task_struct *p)
1106{
1107 struct migration_swap_arg arg;
1108 int ret = -EINVAL;
1109
ac66f547
PZ
1110 arg = (struct migration_swap_arg){
1111 .src_task = cur,
1112 .src_cpu = task_cpu(cur),
1113 .dst_task = p,
1114 .dst_cpu = task_cpu(p),
1115 };
1116
1117 if (arg.src_cpu == arg.dst_cpu)
1118 goto out;
1119
6acce3ef
PZ
1120 /*
1121 * These three tests are all lockless; this is OK since all of them
1122 * will be re-checked with proper locks held further down the line.
1123 */
ac66f547
PZ
1124 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu))
1125 goto out;
1126
1127 if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task)))
1128 goto out;
1129
1130 if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task)))
1131 goto out;
1132
286549dc 1133 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu);
ac66f547
PZ
1134 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg);
1135
1136out:
ac66f547
PZ
1137 return ret;
1138}
1139
969c7921 1140struct migration_arg {
36c8b586 1141 struct task_struct *task;
1da177e4 1142 int dest_cpu;
70b97a7f 1143};
1da177e4 1144
969c7921
TH
1145static int migration_cpu_stop(void *data);
1146
1da177e4
LT
1147/*
1148 * wait_task_inactive - wait for a thread to unschedule.
1149 *
85ba2d86
RM
1150 * If @match_state is nonzero, it's the @p->state value just checked and
1151 * not expected to change. If it changes, i.e. @p might have woken up,
1152 * then return zero. When we succeed in waiting for @p to be off its CPU,
1153 * we return a positive number (its total switch count). If a second call
1154 * a short while later returns the same number, the caller can be sure that
1155 * @p has remained unscheduled the whole time.
1156 *
1da177e4
LT
1157 * The caller must ensure that the task *will* unschedule sometime soon,
1158 * else this function might spin for a *long* time. This function can't
1159 * be called with interrupts off, or it may introduce deadlock with
1160 * smp_call_function() if an IPI is sent by the same process we are
1161 * waiting to become inactive.
1162 */
85ba2d86 1163unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1da177e4
LT
1164{
1165 unsigned long flags;
da0c1e65 1166 int running, queued;
85ba2d86 1167 unsigned long ncsw;
70b97a7f 1168 struct rq *rq;
1da177e4 1169
3a5c359a
AK
1170 for (;;) {
1171 /*
1172 * We do the initial early heuristics without holding
1173 * any task-queue locks at all. We'll only try to get
1174 * the runqueue lock when things look like they will
1175 * work out!
1176 */
1177 rq = task_rq(p);
fa490cfd 1178
3a5c359a
AK
1179 /*
1180 * If the task is actively running on another CPU
1181 * still, just relax and busy-wait without holding
1182 * any locks.
1183 *
1184 * NOTE! Since we don't hold any locks, it's not
1185 * even sure that "rq" stays as the right runqueue!
1186 * But we don't care, since "task_running()" will
1187 * return false if the runqueue has changed and p
1188 * is actually now running somewhere else!
1189 */
85ba2d86
RM
1190 while (task_running(rq, p)) {
1191 if (match_state && unlikely(p->state != match_state))
1192 return 0;
3a5c359a 1193 cpu_relax();
85ba2d86 1194 }
fa490cfd 1195
3a5c359a
AK
1196 /*
1197 * Ok, time to look more closely! We need the rq
1198 * lock now, to be *sure*. If we're wrong, we'll
1199 * just go back and repeat.
1200 */
1201 rq = task_rq_lock(p, &flags);
27a9da65 1202 trace_sched_wait_task(p);
3a5c359a 1203 running = task_running(rq, p);
da0c1e65 1204 queued = task_on_rq_queued(p);
85ba2d86 1205 ncsw = 0;
f31e11d8 1206 if (!match_state || p->state == match_state)
93dcf55f 1207 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
0122ec5b 1208 task_rq_unlock(rq, p, &flags);
fa490cfd 1209
85ba2d86
RM
1210 /*
1211 * If it changed from the expected state, bail out now.
1212 */
1213 if (unlikely(!ncsw))
1214 break;
1215
3a5c359a
AK
1216 /*
1217 * Was it really running after all now that we
1218 * checked with the proper locks actually held?
1219 *
1220 * Oops. Go back and try again..
1221 */
1222 if (unlikely(running)) {
1223 cpu_relax();
1224 continue;
1225 }
fa490cfd 1226
3a5c359a
AK
1227 /*
1228 * It's not enough that it's not actively running,
1229 * it must be off the runqueue _entirely_, and not
1230 * preempted!
1231 *
80dd99b3 1232 * So if it was still runnable (but just not actively
3a5c359a
AK
1233 * running right now), it's preempted, and we should
1234 * yield - it could be a while.
1235 */
da0c1e65 1236 if (unlikely(queued)) {
8eb90c30
TG
1237 ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
1238
1239 set_current_state(TASK_UNINTERRUPTIBLE);
1240 schedule_hrtimeout(&to, HRTIMER_MODE_REL);
3a5c359a
AK
1241 continue;
1242 }
fa490cfd 1243
3a5c359a
AK
1244 /*
1245 * Ahh, all good. It wasn't running, and it wasn't
1246 * runnable, which means that it will never become
1247 * running in the future either. We're all done!
1248 */
1249 break;
1250 }
85ba2d86
RM
1251
1252 return ncsw;
1da177e4
LT
1253}
1254
1255/***
1256 * kick_process - kick a running thread to enter/exit the kernel
1257 * @p: the to-be-kicked thread
1258 *
1259 * Cause a process which is running on another CPU to enter
1260 * kernel-mode, without any delay. (to get signals handled.)
1261 *
25985edc 1262 * NOTE: this function doesn't have to take the runqueue lock,
1da177e4
LT
1263 * because all it wants to ensure is that the remote task enters
1264 * the kernel. If the IPI races and the task has been migrated
1265 * to another CPU then no harm is done and the purpose has been
1266 * achieved as well.
1267 */
36c8b586 1268void kick_process(struct task_struct *p)
1da177e4
LT
1269{
1270 int cpu;
1271
1272 preempt_disable();
1273 cpu = task_cpu(p);
1274 if ((cpu != smp_processor_id()) && task_curr(p))
1275 smp_send_reschedule(cpu);
1276 preempt_enable();
1277}
b43e3521 1278EXPORT_SYMBOL_GPL(kick_process);
476d139c 1279#endif /* CONFIG_SMP */
1da177e4 1280
970b13ba 1281#ifdef CONFIG_SMP
30da688e 1282/*
013fdb80 1283 * ->cpus_allowed is protected by both rq->lock and p->pi_lock
30da688e 1284 */
5da9a0fb
PZ
1285static int select_fallback_rq(int cpu, struct task_struct *p)
1286{
aa00d89c
TC
1287 int nid = cpu_to_node(cpu);
1288 const struct cpumask *nodemask = NULL;
2baab4e9
PZ
1289 enum { cpuset, possible, fail } state = cpuset;
1290 int dest_cpu;
5da9a0fb 1291
aa00d89c
TC
1292 /*
1293 * If the node that the cpu is on has been offlined, cpu_to_node()
1294 * will return -1. There is no cpu on the node, and we should
1295 * select the cpu on the other node.
1296 */
1297 if (nid != -1) {
1298 nodemask = cpumask_of_node(nid);
1299
1300 /* Look for allowed, online CPU in same node. */
1301 for_each_cpu(dest_cpu, nodemask) {
1302 if (!cpu_online(dest_cpu))
1303 continue;
1304 if (!cpu_active(dest_cpu))
1305 continue;
1306 if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
1307 return dest_cpu;
1308 }
2baab4e9 1309 }
5da9a0fb 1310
2baab4e9
PZ
1311 for (;;) {
1312 /* Any allowed, online CPU? */
e3831edd 1313 for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
2baab4e9
PZ
1314 if (!cpu_online(dest_cpu))
1315 continue;
1316 if (!cpu_active(dest_cpu))
1317 continue;
1318 goto out;
1319 }
5da9a0fb 1320
2baab4e9
PZ
1321 switch (state) {
1322 case cpuset:
1323 /* No more Mr. Nice Guy. */
1324 cpuset_cpus_allowed_fallback(p);
1325 state = possible;
1326 break;
1327
1328 case possible:
1329 do_set_cpus_allowed(p, cpu_possible_mask);
1330 state = fail;
1331 break;
1332
1333 case fail:
1334 BUG();
1335 break;
1336 }
1337 }
1338
1339out:
1340 if (state != cpuset) {
1341 /*
1342 * Don't tell them about moving exiting tasks or
1343 * kernel threads (both mm NULL), since they never
1344 * leave kernel.
1345 */
1346 if (p->mm && printk_ratelimit()) {
aac74dc4 1347 printk_deferred("process %d (%s) no longer affine to cpu%d\n",
2baab4e9
PZ
1348 task_pid_nr(p), p->comm, cpu);
1349 }
5da9a0fb
PZ
1350 }
1351
1352 return dest_cpu;
1353}
1354
e2912009 1355/*
013fdb80 1356 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable.
e2912009 1357 */
970b13ba 1358static inline
ac66f547 1359int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
970b13ba 1360{
6c1d9410
WL
1361 if (p->nr_cpus_allowed > 1)
1362 cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags);
e2912009
PZ
1363
1364 /*
1365 * In order not to call set_task_cpu() on a blocking task we need
1366 * to rely on ttwu() to place the task on a valid ->cpus_allowed
1367 * cpu.
1368 *
1369 * Since this is common to all placement strategies, this lives here.
1370 *
1371 * [ this allows ->select_task() to simply return task_cpu(p) and
1372 * not worry about this generic constraint ]
1373 */
fa17b507 1374 if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
70f11205 1375 !cpu_online(cpu)))
5da9a0fb 1376 cpu = select_fallback_rq(task_cpu(p), p);
e2912009
PZ
1377
1378 return cpu;
970b13ba 1379}
09a40af5
MG
1380
1381static void update_avg(u64 *avg, u64 sample)
1382{
1383 s64 diff = sample - *avg;
1384 *avg += diff >> 3;
1385}
970b13ba
PZ
1386#endif
1387
d7c01d27 1388static void
b84cb5df 1389ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
9ed3811a 1390{
d7c01d27 1391#ifdef CONFIG_SCHEDSTATS
b84cb5df
PZ
1392 struct rq *rq = this_rq();
1393
d7c01d27
PZ
1394#ifdef CONFIG_SMP
1395 int this_cpu = smp_processor_id();
1396
1397 if (cpu == this_cpu) {
1398 schedstat_inc(rq, ttwu_local);
1399 schedstat_inc(p, se.statistics.nr_wakeups_local);
1400 } else {
1401 struct sched_domain *sd;
1402
1403 schedstat_inc(p, se.statistics.nr_wakeups_remote);
057f3fad 1404 rcu_read_lock();
d7c01d27
PZ
1405 for_each_domain(this_cpu, sd) {
1406 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
1407 schedstat_inc(sd, ttwu_wake_remote);
1408 break;
1409 }
1410 }
057f3fad 1411 rcu_read_unlock();
d7c01d27 1412 }
f339b9dc
PZ
1413
1414 if (wake_flags & WF_MIGRATED)
1415 schedstat_inc(p, se.statistics.nr_wakeups_migrate);
1416
d7c01d27
PZ
1417#endif /* CONFIG_SMP */
1418
1419 schedstat_inc(rq, ttwu_count);
9ed3811a 1420 schedstat_inc(p, se.statistics.nr_wakeups);
d7c01d27
PZ
1421
1422 if (wake_flags & WF_SYNC)
9ed3811a 1423 schedstat_inc(p, se.statistics.nr_wakeups_sync);
d7c01d27 1424
d7c01d27
PZ
1425#endif /* CONFIG_SCHEDSTATS */
1426}
1427
1428static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags)
1429{
9ed3811a 1430 activate_task(rq, p, en_flags);
da0c1e65 1431 p->on_rq = TASK_ON_RQ_QUEUED;
c2f7115e
PZ
1432
1433 /* if a worker is waking up, notify workqueue */
1434 if (p->flags & PF_WQ_WORKER)
1435 wq_worker_waking_up(p, cpu_of(rq));
9ed3811a
TH
1436}
1437
23f41eeb
PZ
1438/*
1439 * Mark the task runnable and perform wakeup-preemption.
1440 */
89363381 1441static void
23f41eeb 1442ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
9ed3811a 1443{
9ed3811a 1444 check_preempt_curr(rq, p, wake_flags);
a8d7ad52 1445 trace_sched_wakeup(p, true);
9ed3811a
TH
1446
1447 p->state = TASK_RUNNING;
1448#ifdef CONFIG_SMP
1449 if (p->sched_class->task_woken)
1450 p->sched_class->task_woken(rq, p);
1451
e69c6341 1452 if (rq->idle_stamp) {
78becc27 1453 u64 delta = rq_clock(rq) - rq->idle_stamp;
9bd721c5 1454 u64 max = 2*rq->max_idle_balance_cost;
9ed3811a 1455
abfafa54
JL
1456 update_avg(&rq->avg_idle, delta);
1457
1458 if (rq->avg_idle > max)
9ed3811a 1459 rq->avg_idle = max;
abfafa54 1460
9ed3811a
TH
1461 rq->idle_stamp = 0;
1462 }
1463#endif
1464}
1465
c05fbafb
PZ
1466static void
1467ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
1468{
1469#ifdef CONFIG_SMP
1470 if (p->sched_contributes_to_load)
1471 rq->nr_uninterruptible--;
1472#endif
1473
1474 ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING);
1475 ttwu_do_wakeup(rq, p, wake_flags);
1476}
1477
1478/*
1479 * Called in case the task @p isn't fully descheduled from its runqueue,
1480 * in this case we must do a remote wakeup. Its a 'light' wakeup though,
1481 * since all we need to do is flip p->state to TASK_RUNNING, since
1482 * the task is still ->on_rq.
1483 */
1484static int ttwu_remote(struct task_struct *p, int wake_flags)
1485{
1486 struct rq *rq;
1487 int ret = 0;
1488
1489 rq = __task_rq_lock(p);
da0c1e65 1490 if (task_on_rq_queued(p)) {
1ad4ec0d
FW
1491 /* check_preempt_curr() may use rq clock */
1492 update_rq_clock(rq);
c05fbafb
PZ
1493 ttwu_do_wakeup(rq, p, wake_flags);
1494 ret = 1;
1495 }
1496 __task_rq_unlock(rq);
1497
1498 return ret;
1499}
1500
317f3941 1501#ifdef CONFIG_SMP
e3baac47 1502void sched_ttwu_pending(void)
317f3941
PZ
1503{
1504 struct rq *rq = this_rq();
fa14ff4a
PZ
1505 struct llist_node *llist = llist_del_all(&rq->wake_list);
1506 struct task_struct *p;
e3baac47 1507 unsigned long flags;
317f3941 1508
e3baac47
PZ
1509 if (!llist)
1510 return;
1511
1512 raw_spin_lock_irqsave(&rq->lock, flags);
317f3941 1513
fa14ff4a
PZ
1514 while (llist) {
1515 p = llist_entry(llist, struct task_struct, wake_entry);
1516 llist = llist_next(llist);
317f3941
PZ
1517 ttwu_do_activate(rq, p, 0);
1518 }
1519
e3baac47 1520 raw_spin_unlock_irqrestore(&rq->lock, flags);
317f3941
PZ
1521}
1522
1523void scheduler_ipi(void)
1524{
f27dde8d
PZ
1525 /*
1526 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
1527 * TIF_NEED_RESCHED remotely (for the first time) will also send
1528 * this IPI.
1529 */
8cb75e0c 1530 preempt_fold_need_resched();
f27dde8d 1531
fd2ac4f4 1532 if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
c5d753a5
PZ
1533 return;
1534
1535 /*
1536 * Not all reschedule IPI handlers call irq_enter/irq_exit, since
1537 * traditionally all their work was done from the interrupt return
1538 * path. Now that we actually do some work, we need to make sure
1539 * we do call them.
1540 *
1541 * Some archs already do call them, luckily irq_enter/exit nest
1542 * properly.
1543 *
1544 * Arguably we should visit all archs and update all handlers,
1545 * however a fair share of IPIs are still resched only so this would
1546 * somewhat pessimize the simple resched case.
1547 */
1548 irq_enter();
fa14ff4a 1549 sched_ttwu_pending();
ca38062e
SS
1550
1551 /*
1552 * Check if someone kicked us for doing the nohz idle load balance.
1553 */
873b4c65 1554 if (unlikely(got_nohz_idle_kick())) {
6eb57e0d 1555 this_rq()->idle_balance = 1;
ca38062e 1556 raise_softirq_irqoff(SCHED_SOFTIRQ);
6eb57e0d 1557 }
c5d753a5 1558 irq_exit();
317f3941
PZ
1559}
1560
1561static void ttwu_queue_remote(struct task_struct *p, int cpu)
1562{
e3baac47
PZ
1563 struct rq *rq = cpu_rq(cpu);
1564
1565 if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
1566 if (!set_nr_if_polling(rq->idle))
1567 smp_send_reschedule(cpu);
1568 else
1569 trace_sched_wake_idle_without_ipi(cpu);
1570 }
317f3941 1571}
d6aa8f85 1572
f6be8af1
CL
1573void wake_up_if_idle(int cpu)
1574{
1575 struct rq *rq = cpu_rq(cpu);
1576 unsigned long flags;
1577
fd7de1e8
AL
1578 rcu_read_lock();
1579
1580 if (!is_idle_task(rcu_dereference(rq->curr)))
1581 goto out;
f6be8af1
CL
1582
1583 if (set_nr_if_polling(rq->idle)) {
1584 trace_sched_wake_idle_without_ipi(cpu);
1585 } else {
1586 raw_spin_lock_irqsave(&rq->lock, flags);
1587 if (is_idle_task(rq->curr))
1588 smp_send_reschedule(cpu);
1589 /* Else cpu is not in idle, do nothing here */
1590 raw_spin_unlock_irqrestore(&rq->lock, flags);
1591 }
fd7de1e8
AL
1592
1593out:
1594 rcu_read_unlock();
f6be8af1
CL
1595}
1596
39be3501 1597bool cpus_share_cache(int this_cpu, int that_cpu)
518cd623
PZ
1598{
1599 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
1600}
d6aa8f85 1601#endif /* CONFIG_SMP */
317f3941 1602
c05fbafb
PZ
1603static void ttwu_queue(struct task_struct *p, int cpu)
1604{
1605 struct rq *rq = cpu_rq(cpu);
1606
17d9f311 1607#if defined(CONFIG_SMP)
39be3501 1608 if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
f01114cb 1609 sched_clock_cpu(cpu); /* sync clocks x-cpu */
317f3941
PZ
1610 ttwu_queue_remote(p, cpu);
1611 return;
1612 }
1613#endif
1614
c05fbafb
PZ
1615 raw_spin_lock(&rq->lock);
1616 ttwu_do_activate(rq, p, 0);
1617 raw_spin_unlock(&rq->lock);
9ed3811a
TH
1618}
1619
1620/**
1da177e4 1621 * try_to_wake_up - wake up a thread
9ed3811a 1622 * @p: the thread to be awakened
1da177e4 1623 * @state: the mask of task states that can be woken
9ed3811a 1624 * @wake_flags: wake modifier flags (WF_*)
1da177e4
LT
1625 *
1626 * Put it on the run-queue if it's not already there. The "current"
1627 * thread is always on the run-queue (except when the actual
1628 * re-schedule is in progress), and as such you're allowed to do
1629 * the simpler "current->state = TASK_RUNNING" to mark yourself
1630 * runnable without the overhead of this.
1631 *
e69f6186 1632 * Return: %true if @p was woken up, %false if it was already running.
9ed3811a 1633 * or @state didn't match @p's state.
1da177e4 1634 */
e4a52bcb
PZ
1635static int
1636try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
1da177e4 1637{
1da177e4 1638 unsigned long flags;
c05fbafb 1639 int cpu, success = 0;
2398f2c6 1640
e0acd0a6
ON
1641 /*
1642 * If we are going to wake up a thread waiting for CONDITION we
1643 * need to ensure that CONDITION=1 done by the caller can not be
1644 * reordered with p->state check below. This pairs with mb() in
1645 * set_current_state() the waiting thread does.
1646 */
1647 smp_mb__before_spinlock();
013fdb80 1648 raw_spin_lock_irqsave(&p->pi_lock, flags);
e9c84311 1649 if (!(p->state & state))
1da177e4
LT
1650 goto out;
1651
c05fbafb 1652 success = 1; /* we're going to change ->state */
1da177e4 1653 cpu = task_cpu(p);
1da177e4 1654
c05fbafb
PZ
1655 if (p->on_rq && ttwu_remote(p, wake_flags))
1656 goto stat;
1da177e4 1657
1da177e4 1658#ifdef CONFIG_SMP
e9c84311 1659 /*
c05fbafb
PZ
1660 * If the owning (remote) cpu is still in the middle of schedule() with
1661 * this task as prev, wait until its done referencing the task.
e9c84311 1662 */
f3e94786 1663 while (p->on_cpu)
e4a52bcb 1664 cpu_relax();
0970d299 1665 /*
e4a52bcb 1666 * Pairs with the smp_wmb() in finish_lock_switch().
0970d299 1667 */
e4a52bcb 1668 smp_rmb();
1da177e4 1669
a8e4f2ea 1670 p->sched_contributes_to_load = !!task_contributes_to_load(p);
e9c84311 1671 p->state = TASK_WAKING;
e7693a36 1672
e4a52bcb 1673 if (p->sched_class->task_waking)
74f8e4b2 1674 p->sched_class->task_waking(p);
efbbd05a 1675
ac66f547 1676 cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
f339b9dc
PZ
1677 if (task_cpu(p) != cpu) {
1678 wake_flags |= WF_MIGRATED;
e4a52bcb 1679 set_task_cpu(p, cpu);
f339b9dc 1680 }
1da177e4 1681#endif /* CONFIG_SMP */
1da177e4 1682
c05fbafb
PZ
1683 ttwu_queue(p, cpu);
1684stat:
b84cb5df 1685 ttwu_stat(p, cpu, wake_flags);
1da177e4 1686out:
013fdb80 1687 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1da177e4
LT
1688
1689 return success;
1690}
1691
21aa9af0
TH
1692/**
1693 * try_to_wake_up_local - try to wake up a local task with rq lock held
1694 * @p: the thread to be awakened
1695 *
2acca55e 1696 * Put @p on the run-queue if it's not already there. The caller must
21aa9af0 1697 * ensure that this_rq() is locked, @p is bound to this_rq() and not
2acca55e 1698 * the current task.
21aa9af0
TH
1699 */
1700static void try_to_wake_up_local(struct task_struct *p)
1701{
1702 struct rq *rq = task_rq(p);
21aa9af0 1703
383efcd0
TH
1704 if (WARN_ON_ONCE(rq != this_rq()) ||
1705 WARN_ON_ONCE(p == current))
1706 return;
1707
21aa9af0
TH
1708 lockdep_assert_held(&rq->lock);
1709
2acca55e
PZ
1710 if (!raw_spin_trylock(&p->pi_lock)) {
1711 raw_spin_unlock(&rq->lock);
1712 raw_spin_lock(&p->pi_lock);
1713 raw_spin_lock(&rq->lock);
1714 }
1715
21aa9af0 1716 if (!(p->state & TASK_NORMAL))
2acca55e 1717 goto out;
21aa9af0 1718
da0c1e65 1719 if (!task_on_rq_queued(p))
d7c01d27
PZ
1720 ttwu_activate(rq, p, ENQUEUE_WAKEUP);
1721
23f41eeb 1722 ttwu_do_wakeup(rq, p, 0);
b84cb5df 1723 ttwu_stat(p, smp_processor_id(), 0);
2acca55e
PZ
1724out:
1725 raw_spin_unlock(&p->pi_lock);
21aa9af0
TH
1726}
1727
50fa610a
DH
1728/**
1729 * wake_up_process - Wake up a specific process
1730 * @p: The process to be woken up.
1731 *
1732 * Attempt to wake up the nominated process and move it to the set of runnable
e69f6186
YB
1733 * processes.
1734 *
1735 * Return: 1 if the process was woken up, 0 if it was already running.
50fa610a
DH
1736 *
1737 * It may be assumed that this function implies a write memory barrier before
1738 * changing the task state if and only if any tasks are woken up.
1739 */
7ad5b3a5 1740int wake_up_process(struct task_struct *p)
1da177e4 1741{
9067ac85
ON
1742 WARN_ON(task_is_stopped_or_traced(p));
1743 return try_to_wake_up(p, TASK_NORMAL, 0);
1da177e4 1744}
1da177e4
LT
1745EXPORT_SYMBOL(wake_up_process);
1746
7ad5b3a5 1747int wake_up_state(struct task_struct *p, unsigned int state)
1da177e4
LT
1748{
1749 return try_to_wake_up(p, state, 0);
1750}
1751
a5e7be3b
JL
1752/*
1753 * This function clears the sched_dl_entity static params.
1754 */
1755void __dl_clear_params(struct task_struct *p)
1756{
1757 struct sched_dl_entity *dl_se = &p->dl;
1758
1759 dl_se->dl_runtime = 0;
1760 dl_se->dl_deadline = 0;
1761 dl_se->dl_period = 0;
1762 dl_se->flags = 0;
1763 dl_se->dl_bw = 0;
40767b0d
PZ
1764
1765 dl_se->dl_throttled = 0;
1766 dl_se->dl_new = 1;
1767 dl_se->dl_yielded = 0;
a5e7be3b
JL
1768}
1769
1da177e4
LT
1770/*
1771 * Perform scheduler related setup for a newly forked process p.
1772 * p is forked by current.
dd41f596
IM
1773 *
1774 * __sched_fork() is basic setup used by init_idle() too:
1775 */
5e1576ed 1776static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
dd41f596 1777{
fd2f4419
PZ
1778 p->on_rq = 0;
1779
1780 p->se.on_rq = 0;
dd41f596
IM
1781 p->se.exec_start = 0;
1782 p->se.sum_exec_runtime = 0;
f6cf891c 1783 p->se.prev_sum_exec_runtime = 0;
6c594c21 1784 p->se.nr_migrations = 0;
da7a735e 1785 p->se.vruntime = 0;
bb04159d
KT
1786#ifdef CONFIG_SMP
1787 p->se.avg.decay_count = 0;
1788#endif
fd2f4419 1789 INIT_LIST_HEAD(&p->se.group_node);
6cfb0d5d
IM
1790
1791#ifdef CONFIG_SCHEDSTATS
41acab88 1792 memset(&p->se.statistics, 0, sizeof(p->se.statistics));
6cfb0d5d 1793#endif
476d139c 1794
aab03e05 1795 RB_CLEAR_NODE(&p->dl.rb_node);
40767b0d 1796 init_dl_task_timer(&p->dl);
a5e7be3b 1797 __dl_clear_params(p);
aab03e05 1798
fa717060 1799 INIT_LIST_HEAD(&p->rt.run_list);
476d139c 1800
e107be36
AK
1801#ifdef CONFIG_PREEMPT_NOTIFIERS
1802 INIT_HLIST_HEAD(&p->preempt_notifiers);
1803#endif
cbee9f88
PZ
1804
1805#ifdef CONFIG_NUMA_BALANCING
1806 if (p->mm && atomic_read(&p->mm->mm_users) == 1) {
7e8d16b6 1807 p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay);
cbee9f88
PZ
1808 p->mm->numa_scan_seq = 0;
1809 }
1810
5e1576ed
RR
1811 if (clone_flags & CLONE_VM)
1812 p->numa_preferred_nid = current->numa_preferred_nid;
1813 else
1814 p->numa_preferred_nid = -1;
1815
cbee9f88
PZ
1816 p->node_stamp = 0ULL;
1817 p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0;
4b96a29b 1818 p->numa_scan_period = sysctl_numa_balancing_scan_delay;
cbee9f88 1819 p->numa_work.next = &p->numa_work;
44dba3d5 1820 p->numa_faults = NULL;
7e2703e6
RR
1821 p->last_task_numa_placement = 0;
1822 p->last_sum_exec_runtime = 0;
8c8a743c 1823
8c8a743c 1824 p->numa_group = NULL;
cbee9f88 1825#endif /* CONFIG_NUMA_BALANCING */
dd41f596
IM
1826}
1827
1a687c2e 1828#ifdef CONFIG_NUMA_BALANCING
3105b86a 1829#ifdef CONFIG_SCHED_DEBUG
1a687c2e
MG
1830void set_numabalancing_state(bool enabled)
1831{
1832 if (enabled)
1833 sched_feat_set("NUMA");
1834 else
1835 sched_feat_set("NO_NUMA");
1836}
3105b86a
MG
1837#else
1838__read_mostly bool numabalancing_enabled;
1839
1840void set_numabalancing_state(bool enabled)
1841{
1842 numabalancing_enabled = enabled;
dd41f596 1843}
3105b86a 1844#endif /* CONFIG_SCHED_DEBUG */
54a43d54
AK
1845
1846#ifdef CONFIG_PROC_SYSCTL
1847int sysctl_numa_balancing(struct ctl_table *table, int write,
1848 void __user *buffer, size_t *lenp, loff_t *ppos)
1849{
1850 struct ctl_table t;
1851 int err;
1852 int state = numabalancing_enabled;
1853
1854 if (write && !capable(CAP_SYS_ADMIN))
1855 return -EPERM;
1856
1857 t = *table;
1858 t.data = &state;
1859 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
1860 if (err < 0)
1861 return err;
1862 if (write)
1863 set_numabalancing_state(state);
1864 return err;
1865}
1866#endif
1867#endif
dd41f596
IM
1868
1869/*
1870 * fork()/clone()-time setup:
1871 */
aab03e05 1872int sched_fork(unsigned long clone_flags, struct task_struct *p)
dd41f596 1873{
0122ec5b 1874 unsigned long flags;
dd41f596
IM
1875 int cpu = get_cpu();
1876
5e1576ed 1877 __sched_fork(clone_flags, p);
06b83b5f 1878 /*
0017d735 1879 * We mark the process as running here. This guarantees that
06b83b5f
PZ
1880 * nobody will actually run it, and a signal or other external
1881 * event cannot wake it up and insert it on the runqueue either.
1882 */
0017d735 1883 p->state = TASK_RUNNING;
dd41f596 1884
c350a04e
MG
1885 /*
1886 * Make sure we do not leak PI boosting priority to the child.
1887 */
1888 p->prio = current->normal_prio;
1889
b9dc29e7
MG
1890 /*
1891 * Revert to default priority/policy on fork if requested.
1892 */
1893 if (unlikely(p->sched_reset_on_fork)) {
aab03e05 1894 if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
b9dc29e7 1895 p->policy = SCHED_NORMAL;
6c697bdf 1896 p->static_prio = NICE_TO_PRIO(0);
c350a04e
MG
1897 p->rt_priority = 0;
1898 } else if (PRIO_TO_NICE(p->static_prio) < 0)
1899 p->static_prio = NICE_TO_PRIO(0);
1900
1901 p->prio = p->normal_prio = __normal_prio(p);
1902 set_load_weight(p);
6c697bdf 1903
b9dc29e7
MG
1904 /*
1905 * We don't need the reset flag anymore after the fork. It has
1906 * fulfilled its duty:
1907 */
1908 p->sched_reset_on_fork = 0;
1909 }
ca94c442 1910
aab03e05
DF
1911 if (dl_prio(p->prio)) {
1912 put_cpu();
1913 return -EAGAIN;
1914 } else if (rt_prio(p->prio)) {
1915 p->sched_class = &rt_sched_class;
1916 } else {
2ddbf952 1917 p->sched_class = &fair_sched_class;
aab03e05 1918 }
b29739f9 1919
cd29fe6f
PZ
1920 if (p->sched_class->task_fork)
1921 p->sched_class->task_fork(p);
1922
86951599
PZ
1923 /*
1924 * The child is not yet in the pid-hash so no cgroup attach races,
1925 * and the cgroup is pinned to this child due to cgroup_fork()
1926 * is ran before sched_fork().
1927 *
1928 * Silence PROVE_RCU.
1929 */
0122ec5b 1930 raw_spin_lock_irqsave(&p->pi_lock, flags);
5f3edc1b 1931 set_task_cpu(p, cpu);
0122ec5b 1932 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
5f3edc1b 1933
52f17b6c 1934#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
dd41f596 1935 if (likely(sched_info_on()))
52f17b6c 1936 memset(&p->sched_info, 0, sizeof(p->sched_info));
1da177e4 1937#endif
3ca7a440
PZ
1938#if defined(CONFIG_SMP)
1939 p->on_cpu = 0;
4866cde0 1940#endif
01028747 1941 init_task_preempt_count(p);
806c09a7 1942#ifdef CONFIG_SMP
917b627d 1943 plist_node_init(&p->pushable_tasks, MAX_PRIO);
1baca4ce 1944 RB_CLEAR_NODE(&p->pushable_dl_tasks);
806c09a7 1945#endif
917b627d 1946
476d139c 1947 put_cpu();
aab03e05 1948 return 0;
1da177e4
LT
1949}
1950
332ac17e
DF
1951unsigned long to_ratio(u64 period, u64 runtime)
1952{
1953 if (runtime == RUNTIME_INF)
1954 return 1ULL << 20;
1955
1956 /*
1957 * Doing this here saves a lot of checks in all
1958 * the calling paths, and returning zero seems
1959 * safe for them anyway.
1960 */
1961 if (period == 0)
1962 return 0;
1963
1964 return div64_u64(runtime << 20, period);
1965}
1966
1967#ifdef CONFIG_SMP
1968inline struct dl_bw *dl_bw_of(int i)
1969{
66339c31
KT
1970 rcu_lockdep_assert(rcu_read_lock_sched_held(),
1971 "sched RCU must be held");
332ac17e
DF
1972 return &cpu_rq(i)->rd->dl_bw;
1973}
1974
de212f18 1975static inline int dl_bw_cpus(int i)
332ac17e 1976{
de212f18
PZ
1977 struct root_domain *rd = cpu_rq(i)->rd;
1978 int cpus = 0;
1979
66339c31
KT
1980 rcu_lockdep_assert(rcu_read_lock_sched_held(),
1981 "sched RCU must be held");
de212f18
PZ
1982 for_each_cpu_and(i, rd->span, cpu_active_mask)
1983 cpus++;
1984
1985 return cpus;
332ac17e
DF
1986}
1987#else
1988inline struct dl_bw *dl_bw_of(int i)
1989{
1990 return &cpu_rq(i)->dl.dl_bw;
1991}
1992
de212f18 1993static inline int dl_bw_cpus(int i)
332ac17e
DF
1994{
1995 return 1;
1996}
1997#endif
1998
332ac17e
DF
1999/*
2000 * We must be sure that accepting a new task (or allowing changing the
2001 * parameters of an existing one) is consistent with the bandwidth
2002 * constraints. If yes, this function also accordingly updates the currently
2003 * allocated bandwidth to reflect the new situation.
2004 *
2005 * This function is called while holding p's rq->lock.
40767b0d
PZ
2006 *
2007 * XXX we should delay bw change until the task's 0-lag point, see
2008 * __setparam_dl().
332ac17e
DF
2009 */
2010static int dl_overflow(struct task_struct *p, int policy,
2011 const struct sched_attr *attr)
2012{
2013
2014 struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
4df1638c 2015 u64 period = attr->sched_period ?: attr->sched_deadline;
332ac17e
DF
2016 u64 runtime = attr->sched_runtime;
2017 u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
de212f18 2018 int cpus, err = -1;
332ac17e
DF
2019
2020 if (new_bw == p->dl.dl_bw)
2021 return 0;
2022
2023 /*
2024 * Either if a task, enters, leave, or stays -deadline but changes
2025 * its parameters, we may need to update accordingly the total
2026 * allocated bandwidth of the container.
2027 */
2028 raw_spin_lock(&dl_b->lock);
de212f18 2029 cpus = dl_bw_cpus(task_cpu(p));
332ac17e
DF
2030 if (dl_policy(policy) && !task_has_dl_policy(p) &&
2031 !__dl_overflow(dl_b, cpus, 0, new_bw)) {
2032 __dl_add(dl_b, new_bw);
2033 err = 0;
2034 } else if (dl_policy(policy) && task_has_dl_policy(p) &&
2035 !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
2036 __dl_clear(dl_b, p->dl.dl_bw);
2037 __dl_add(dl_b, new_bw);
2038 err = 0;
2039 } else if (!dl_policy(policy) && task_has_dl_policy(p)) {
2040 __dl_clear(dl_b, p->dl.dl_bw);
2041 err = 0;
2042 }
2043 raw_spin_unlock(&dl_b->lock);
2044
2045 return err;
2046}
2047
2048extern void init_dl_bw(struct dl_bw *dl_b);
2049
1da177e4
LT
2050/*
2051 * wake_up_new_task - wake up a newly created task for the first time.
2052 *
2053 * This function will do some initial scheduler statistics housekeeping
2054 * that must be done for every newly created context, then puts the task
2055 * on the runqueue and wakes it.
2056 */
3e51e3ed 2057void wake_up_new_task(struct task_struct *p)
1da177e4
LT
2058{
2059 unsigned long flags;
dd41f596 2060 struct rq *rq;
fabf318e 2061
ab2515c4 2062 raw_spin_lock_irqsave(&p->pi_lock, flags);
fabf318e
PZ
2063#ifdef CONFIG_SMP
2064 /*
2065 * Fork balancing, do it here and not earlier because:
2066 * - cpus_allowed can change in the fork path
2067 * - any previously selected cpu might disappear through hotplug
fabf318e 2068 */
ac66f547 2069 set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0));
0017d735
PZ
2070#endif
2071
a75cdaa9
AS
2072 /* Initialize new task's runnable average */
2073 init_task_runnable_average(p);
ab2515c4 2074 rq = __task_rq_lock(p);
cd29fe6f 2075 activate_task(rq, p, 0);
da0c1e65 2076 p->on_rq = TASK_ON_RQ_QUEUED;
89363381 2077 trace_sched_wakeup_new(p, true);
a7558e01 2078 check_preempt_curr(rq, p, WF_FORK);
9a897c5a 2079#ifdef CONFIG_SMP
efbbd05a
PZ
2080 if (p->sched_class->task_woken)
2081 p->sched_class->task_woken(rq, p);
9a897c5a 2082#endif
0122ec5b 2083 task_rq_unlock(rq, p, &flags);
1da177e4
LT
2084}
2085
e107be36
AK
2086#ifdef CONFIG_PREEMPT_NOTIFIERS
2087
2088/**
80dd99b3 2089 * preempt_notifier_register - tell me when current is being preempted & rescheduled
421cee29 2090 * @notifier: notifier struct to register
e107be36
AK
2091 */
2092void preempt_notifier_register(struct preempt_notifier *notifier)
2093{
2094 hlist_add_head(&notifier->link, &current->preempt_notifiers);
2095}
2096EXPORT_SYMBOL_GPL(preempt_notifier_register);
2097
2098/**
2099 * preempt_notifier_unregister - no longer interested in preemption notifications
421cee29 2100 * @notifier: notifier struct to unregister
e107be36
AK
2101 *
2102 * This is safe to call from within a preemption notifier.
2103 */
2104void preempt_notifier_unregister(struct preempt_notifier *notifier)
2105{
2106 hlist_del(&notifier->link);
2107}
2108EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
2109
2110static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2111{
2112 struct preempt_notifier *notifier;
e107be36 2113
b67bfe0d 2114 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
e107be36
AK
2115 notifier->ops->sched_in(notifier, raw_smp_processor_id());
2116}
2117
2118static void
2119fire_sched_out_preempt_notifiers(struct task_struct *curr,
2120 struct task_struct *next)
2121{
2122 struct preempt_notifier *notifier;
e107be36 2123
b67bfe0d 2124 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
e107be36
AK
2125 notifier->ops->sched_out(notifier, next);
2126}
2127
6d6bc0ad 2128#else /* !CONFIG_PREEMPT_NOTIFIERS */
e107be36
AK
2129
2130static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2131{
2132}
2133
2134static void
2135fire_sched_out_preempt_notifiers(struct task_struct *curr,
2136 struct task_struct *next)
2137{
2138}
2139
6d6bc0ad 2140#endif /* CONFIG_PREEMPT_NOTIFIERS */
e107be36 2141
4866cde0
NP
2142/**
2143 * prepare_task_switch - prepare to switch tasks
2144 * @rq: the runqueue preparing to switch
421cee29 2145 * @prev: the current task that is being switched out
4866cde0
NP
2146 * @next: the task we are going to switch to.
2147 *
2148 * This is called with the rq lock held and interrupts off. It must
2149 * be paired with a subsequent finish_task_switch after the context
2150 * switch.
2151 *
2152 * prepare_task_switch sets up locking and calls architecture specific
2153 * hooks.
2154 */
e107be36
AK
2155static inline void
2156prepare_task_switch(struct rq *rq, struct task_struct *prev,
2157 struct task_struct *next)
4866cde0 2158{
895dd92c 2159 trace_sched_switch(prev, next);
43148951 2160 sched_info_switch(rq, prev, next);
fe4b04fa 2161 perf_event_task_sched_out(prev, next);
e107be36 2162 fire_sched_out_preempt_notifiers(prev, next);
4866cde0
NP
2163 prepare_lock_switch(rq, next);
2164 prepare_arch_switch(next);
2165}
2166
1da177e4
LT
2167/**
2168 * finish_task_switch - clean up after a task-switch
2169 * @prev: the thread we just switched away from.
2170 *
4866cde0
NP
2171 * finish_task_switch must be called after the context switch, paired
2172 * with a prepare_task_switch call before the context switch.
2173 * finish_task_switch will reconcile locking set up by prepare_task_switch,
2174 * and do any other architecture-specific cleanup actions.
1da177e4
LT
2175 *
2176 * Note that we may have delayed dropping an mm in context_switch(). If
41a2d6cf 2177 * so, we finish that here outside of the runqueue lock. (Doing it
1da177e4
LT
2178 * with the lock held can cause deadlocks; see schedule() for
2179 * details.)
dfa50b60
ON
2180 *
2181 * The context switch have flipped the stack from under us and restored the
2182 * local variables which were saved when this task called schedule() in the
2183 * past. prev == current is still correct but we need to recalculate this_rq
2184 * because prev may have moved to another CPU.
1da177e4 2185 */
dfa50b60 2186static struct rq *finish_task_switch(struct task_struct *prev)
1da177e4
LT
2187 __releases(rq->lock)
2188{
dfa50b60 2189 struct rq *rq = this_rq();
1da177e4 2190 struct mm_struct *mm = rq->prev_mm;
55a101f8 2191 long prev_state;
1da177e4
LT
2192
2193 rq->prev_mm = NULL;
2194
2195 /*
2196 * A task struct has one reference for the use as "current".
c394cc9f 2197 * If a task dies, then it sets TASK_DEAD in tsk->state and calls
55a101f8
ON
2198 * schedule one last time. The schedule call will never return, and
2199 * the scheduled task must drop that reference.
c394cc9f 2200 * The test for TASK_DEAD must occur while the runqueue locks are
1da177e4
LT
2201 * still held, otherwise prev could be scheduled on another cpu, die
2202 * there before we look at prev->state, and then the reference would
2203 * be dropped twice.
2204 * Manfred Spraul <manfred@colorfullife.com>
2205 */
55a101f8 2206 prev_state = prev->state;
bf9fae9f 2207 vtime_task_switch(prev);
4866cde0 2208 finish_arch_switch(prev);
a8d757ef 2209 perf_event_task_sched_in(prev, current);
4866cde0 2210 finish_lock_switch(rq, prev);
01f23e16 2211 finish_arch_post_lock_switch();
e8fa1362 2212
e107be36 2213 fire_sched_in_preempt_notifiers(current);
1da177e4
LT
2214 if (mm)
2215 mmdrop(mm);
c394cc9f 2216 if (unlikely(prev_state == TASK_DEAD)) {
e6c390f2
DF
2217 if (prev->sched_class->task_dead)
2218 prev->sched_class->task_dead(prev);
2219
c6fd91f0 2220 /*
2221 * Remove function-return probe instances associated with this
2222 * task and put them back on the free list.
9761eea8 2223 */
c6fd91f0 2224 kprobe_flush_task(prev);
1da177e4 2225 put_task_struct(prev);
c6fd91f0 2226 }
99e5ada9
FW
2227
2228 tick_nohz_task_switch(current);
dfa50b60 2229 return rq;
1da177e4
LT
2230}
2231
3f029d3c
GH
2232#ifdef CONFIG_SMP
2233
3f029d3c
GH
2234/* rq->lock is NOT held, but preemption is disabled */
2235static inline void post_schedule(struct rq *rq)
2236{
2237 if (rq->post_schedule) {
2238 unsigned long flags;
2239
05fa785c 2240 raw_spin_lock_irqsave(&rq->lock, flags);
3f029d3c
GH
2241 if (rq->curr->sched_class->post_schedule)
2242 rq->curr->sched_class->post_schedule(rq);
05fa785c 2243 raw_spin_unlock_irqrestore(&rq->lock, flags);
3f029d3c
GH
2244
2245 rq->post_schedule = 0;
2246 }
2247}
2248
2249#else
da19ab51 2250
3f029d3c
GH
2251static inline void post_schedule(struct rq *rq)
2252{
1da177e4
LT
2253}
2254
3f029d3c
GH
2255#endif
2256
1da177e4
LT
2257/**
2258 * schedule_tail - first thing a freshly forked thread must call.
2259 * @prev: the thread we just switched away from.
2260 */
722a9f92 2261asmlinkage __visible void schedule_tail(struct task_struct *prev)
1da177e4
LT
2262 __releases(rq->lock)
2263{
1a43a14a 2264 struct rq *rq;
da19ab51 2265
1a43a14a
ON
2266 /* finish_task_switch() drops rq->lock and enables preemtion */
2267 preempt_disable();
dfa50b60 2268 rq = finish_task_switch(prev);
3f029d3c 2269 post_schedule(rq);
1a43a14a 2270 preempt_enable();
70b97a7f 2271
1da177e4 2272 if (current->set_child_tid)
b488893a 2273 put_user(task_pid_vnr(current), current->set_child_tid);
1da177e4
LT
2274}
2275
2276/*
dfa50b60 2277 * context_switch - switch to the new MM and the new thread's register state.
1da177e4 2278 */
dfa50b60 2279static inline struct rq *
70b97a7f 2280context_switch(struct rq *rq, struct task_struct *prev,
36c8b586 2281 struct task_struct *next)
1da177e4 2282{
dd41f596 2283 struct mm_struct *mm, *oldmm;
1da177e4 2284
e107be36 2285 prepare_task_switch(rq, prev, next);
fe4b04fa 2286
dd41f596
IM
2287 mm = next->mm;
2288 oldmm = prev->active_mm;
9226d125
ZA
2289 /*
2290 * For paravirt, this is coupled with an exit in switch_to to
2291 * combine the page table reload and the switch backend into
2292 * one hypercall.
2293 */
224101ed 2294 arch_start_context_switch(prev);
9226d125 2295
31915ab4 2296 if (!mm) {
1da177e4
LT
2297 next->active_mm = oldmm;
2298 atomic_inc(&oldmm->mm_count);
2299 enter_lazy_tlb(oldmm, next);
2300 } else
2301 switch_mm(oldmm, mm, next);
2302
31915ab4 2303 if (!prev->mm) {
1da177e4 2304 prev->active_mm = NULL;
1da177e4
LT
2305 rq->prev_mm = oldmm;
2306 }
3a5f5e48
IM
2307 /*
2308 * Since the runqueue lock will be released by the next
2309 * task (which is an invalid locking op but in the case
2310 * of the scheduler it's an obvious special-case), so we
2311 * do an early lockdep release here:
2312 */
8a25d5de 2313 spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
1da177e4 2314
91d1aa43 2315 context_tracking_task_switch(prev, next);
1da177e4
LT
2316 /* Here we just switch the register state and the stack. */
2317 switch_to(prev, next, prev);
dd41f596 2318 barrier();
dfa50b60
ON
2319
2320 return finish_task_switch(prev);
1da177e4
LT
2321}
2322
2323/*
1c3e8264 2324 * nr_running and nr_context_switches:
1da177e4
LT
2325 *
2326 * externally visible scheduler statistics: current number of runnable
1c3e8264 2327 * threads, total number of context switches performed since bootup.
1da177e4
LT
2328 */
2329unsigned long nr_running(void)
2330{
2331 unsigned long i, sum = 0;
2332
2333 for_each_online_cpu(i)
2334 sum += cpu_rq(i)->nr_running;
2335
2336 return sum;
f711f609 2337}
1da177e4 2338
2ee507c4
TC
2339/*
2340 * Check if only the current task is running on the cpu.
2341 */
2342bool single_task_running(void)
2343{
2344 if (cpu_rq(smp_processor_id())->nr_running == 1)
2345 return true;
2346 else
2347 return false;
2348}
2349EXPORT_SYMBOL(single_task_running);
2350
1da177e4 2351unsigned long long nr_context_switches(void)
46cb4b7c 2352{
cc94abfc
SR
2353 int i;
2354 unsigned long long sum = 0;
46cb4b7c 2355
0a945022 2356 for_each_possible_cpu(i)
1da177e4 2357 sum += cpu_rq(i)->nr_switches;
46cb4b7c 2358
1da177e4
LT
2359 return sum;
2360}
483b4ee6 2361
1da177e4
LT
2362unsigned long nr_iowait(void)
2363{
2364 unsigned long i, sum = 0;
483b4ee6 2365
0a945022 2366 for_each_possible_cpu(i)
1da177e4 2367 sum += atomic_read(&cpu_rq(i)->nr_iowait);
46cb4b7c 2368
1da177e4
LT
2369 return sum;
2370}
483b4ee6 2371
8c215bd3 2372unsigned long nr_iowait_cpu(int cpu)
69d25870 2373{
8c215bd3 2374 struct rq *this = cpu_rq(cpu);
69d25870
AV
2375 return atomic_read(&this->nr_iowait);
2376}
46cb4b7c 2377
372ba8cb
MG
2378void get_iowait_load(unsigned long *nr_waiters, unsigned long *load)
2379{
2380 struct rq *this = this_rq();
2381 *nr_waiters = atomic_read(&this->nr_iowait);
2382 *load = this->cpu_load[0];
2383}
2384
dd41f596 2385#ifdef CONFIG_SMP
8a0be9ef 2386
46cb4b7c 2387/*
38022906
PZ
2388 * sched_exec - execve() is a valuable balancing opportunity, because at
2389 * this point the task has the smallest effective memory and cache footprint.
46cb4b7c 2390 */
38022906 2391void sched_exec(void)
46cb4b7c 2392{
38022906 2393 struct task_struct *p = current;
1da177e4 2394 unsigned long flags;
0017d735 2395 int dest_cpu;
46cb4b7c 2396
8f42ced9 2397 raw_spin_lock_irqsave(&p->pi_lock, flags);
ac66f547 2398 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0);
0017d735
PZ
2399 if (dest_cpu == smp_processor_id())
2400 goto unlock;
38022906 2401
8f42ced9 2402 if (likely(cpu_active(dest_cpu))) {
969c7921 2403 struct migration_arg arg = { p, dest_cpu };
46cb4b7c 2404
8f42ced9
PZ
2405 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
2406 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
1da177e4
LT
2407 return;
2408 }
0017d735 2409unlock:
8f42ced9 2410 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1da177e4 2411}
dd41f596 2412
1da177e4
LT
2413#endif
2414
1da177e4 2415DEFINE_PER_CPU(struct kernel_stat, kstat);
3292beb3 2416DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
1da177e4
LT
2417
2418EXPORT_PER_CPU_SYMBOL(kstat);
3292beb3 2419EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
1da177e4 2420
c5f8d995
HS
2421/*
2422 * Return accounted runtime for the task.
2423 * In case the task is currently running, return the runtime plus current's
2424 * pending runtime that have not been accounted yet.
2425 */
2426unsigned long long task_sched_runtime(struct task_struct *p)
2427{
2428 unsigned long flags;
2429 struct rq *rq;
6e998916 2430 u64 ns;
c5f8d995 2431
911b2898
PZ
2432#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
2433 /*
2434 * 64-bit doesn't need locks to atomically read a 64bit value.
2435 * So we have a optimization chance when the task's delta_exec is 0.
2436 * Reading ->on_cpu is racy, but this is ok.
2437 *
2438 * If we race with it leaving cpu, we'll take a lock. So we're correct.
2439 * If we race with it entering cpu, unaccounted time is 0. This is
2440 * indistinguishable from the read occurring a few cycles earlier.
4036ac15
MG
2441 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
2442 * been accounted, so we're correct here as well.
911b2898 2443 */
da0c1e65 2444 if (!p->on_cpu || !task_on_rq_queued(p))
911b2898
PZ
2445 return p->se.sum_exec_runtime;
2446#endif
2447
c5f8d995 2448 rq = task_rq_lock(p, &flags);
6e998916
SG
2449 /*
2450 * Must be ->curr _and_ ->on_rq. If dequeued, we would
2451 * project cycles that may never be accounted to this
2452 * thread, breaking clock_gettime().
2453 */
2454 if (task_current(rq, p) && task_on_rq_queued(p)) {
2455 update_rq_clock(rq);
2456 p->sched_class->update_curr(rq);
2457 }
2458 ns = p->se.sum_exec_runtime;
0122ec5b 2459 task_rq_unlock(rq, p, &flags);
c5f8d995
HS
2460
2461 return ns;
2462}
48f24c4d 2463
7835b98b
CL
2464/*
2465 * This function gets called by the timer code, with HZ frequency.
2466 * We call it with interrupts disabled.
7835b98b
CL
2467 */
2468void scheduler_tick(void)
2469{
7835b98b
CL
2470 int cpu = smp_processor_id();
2471 struct rq *rq = cpu_rq(cpu);
dd41f596 2472 struct task_struct *curr = rq->curr;
3e51f33f
PZ
2473
2474 sched_clock_tick();
dd41f596 2475
05fa785c 2476 raw_spin_lock(&rq->lock);
3e51f33f 2477 update_rq_clock(rq);
fa85ae24 2478 curr->sched_class->task_tick(rq, curr, 0);
83dfd523 2479 update_cpu_load_active(rq);
05fa785c 2480 raw_spin_unlock(&rq->lock);
7835b98b 2481
e9d2b064 2482 perf_event_task_tick();
e220d2dc 2483
e418e1c2 2484#ifdef CONFIG_SMP
6eb57e0d 2485 rq->idle_balance = idle_cpu(cpu);
7caff66f 2486 trigger_load_balance(rq);
e418e1c2 2487#endif
265f22a9 2488 rq_last_tick_reset(rq);
1da177e4
LT
2489}
2490
265f22a9
FW
2491#ifdef CONFIG_NO_HZ_FULL
2492/**
2493 * scheduler_tick_max_deferment
2494 *
2495 * Keep at least one tick per second when a single
2496 * active task is running because the scheduler doesn't
2497 * yet completely support full dynticks environment.
2498 *
2499 * This makes sure that uptime, CFS vruntime, load
2500 * balancing, etc... continue to move forward, even
2501 * with a very low granularity.
e69f6186
YB
2502 *
2503 * Return: Maximum deferment in nanoseconds.
265f22a9
FW
2504 */
2505u64 scheduler_tick_max_deferment(void)
2506{
2507 struct rq *rq = this_rq();
2508 unsigned long next, now = ACCESS_ONCE(jiffies);
2509
2510 next = rq->last_sched_tick + HZ;
2511
2512 if (time_before_eq(next, now))
2513 return 0;
2514
8fe8ff09 2515 return jiffies_to_nsecs(next - now);
1da177e4 2516}
265f22a9 2517#endif
1da177e4 2518
132380a0 2519notrace unsigned long get_parent_ip(unsigned long addr)
6cd8a4bb
SR
2520{
2521 if (in_lock_functions(addr)) {
2522 addr = CALLER_ADDR2;
2523 if (in_lock_functions(addr))
2524 addr = CALLER_ADDR3;
2525 }
2526 return addr;
2527}
1da177e4 2528
7e49fcce
SR
2529#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
2530 defined(CONFIG_PREEMPT_TRACER))
2531
edafe3a5 2532void preempt_count_add(int val)
1da177e4 2533{
6cd8a4bb 2534#ifdef CONFIG_DEBUG_PREEMPT
1da177e4
LT
2535 /*
2536 * Underflow?
2537 */
9a11b49a
IM
2538 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
2539 return;
6cd8a4bb 2540#endif
bdb43806 2541 __preempt_count_add(val);
6cd8a4bb 2542#ifdef CONFIG_DEBUG_PREEMPT
1da177e4
LT
2543 /*
2544 * Spinlock count overflowing soon?
2545 */
33859f7f
MOS
2546 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
2547 PREEMPT_MASK - 10);
6cd8a4bb 2548#endif
8f47b187
TG
2549 if (preempt_count() == val) {
2550 unsigned long ip = get_parent_ip(CALLER_ADDR1);
2551#ifdef CONFIG_DEBUG_PREEMPT
2552 current->preempt_disable_ip = ip;
2553#endif
2554 trace_preempt_off(CALLER_ADDR0, ip);
2555 }
1da177e4 2556}
bdb43806 2557EXPORT_SYMBOL(preempt_count_add);
edafe3a5 2558NOKPROBE_SYMBOL(preempt_count_add);
1da177e4 2559
edafe3a5 2560void preempt_count_sub(int val)
1da177e4 2561{
6cd8a4bb 2562#ifdef CONFIG_DEBUG_PREEMPT
1da177e4
LT
2563 /*
2564 * Underflow?
2565 */
01e3eb82 2566 if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
9a11b49a 2567 return;
1da177e4
LT
2568 /*
2569 * Is the spinlock portion underflowing?
2570 */
9a11b49a
IM
2571 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
2572 !(preempt_count() & PREEMPT_MASK)))
2573 return;
6cd8a4bb 2574#endif
9a11b49a 2575
6cd8a4bb
SR
2576 if (preempt_count() == val)
2577 trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
bdb43806 2578 __preempt_count_sub(val);
1da177e4 2579}
bdb43806 2580EXPORT_SYMBOL(preempt_count_sub);
edafe3a5 2581NOKPROBE_SYMBOL(preempt_count_sub);
1da177e4
LT
2582
2583#endif
2584
2585/*
dd41f596 2586 * Print scheduling while atomic bug:
1da177e4 2587 */
dd41f596 2588static noinline void __schedule_bug(struct task_struct *prev)
1da177e4 2589{
664dfa65
DJ
2590 if (oops_in_progress)
2591 return;
2592
3df0fc5b
PZ
2593 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
2594 prev->comm, prev->pid, preempt_count());
838225b4 2595
dd41f596 2596 debug_show_held_locks(prev);
e21f5b15 2597 print_modules();
dd41f596
IM
2598 if (irqs_disabled())
2599 print_irqtrace_events(prev);
8f47b187
TG
2600#ifdef CONFIG_DEBUG_PREEMPT
2601 if (in_atomic_preempt_off()) {
2602 pr_err("Preemption disabled at:");
2603 print_ip_sym(current->preempt_disable_ip);
2604 pr_cont("\n");
2605 }
2606#endif
6135fc1e 2607 dump_stack();
373d4d09 2608 add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
dd41f596 2609}
1da177e4 2610
dd41f596
IM
2611/*
2612 * Various schedule()-time debugging checks and statistics:
2613 */
2614static inline void schedule_debug(struct task_struct *prev)
2615{
0d9e2632
AT
2616#ifdef CONFIG_SCHED_STACK_END_CHECK
2617 BUG_ON(unlikely(task_stack_end_corrupted(prev)));
2618#endif
1da177e4 2619 /*
41a2d6cf 2620 * Test if we are atomic. Since do_exit() needs to call into
192301e7
ON
2621 * schedule() atomically, we ignore that path. Otherwise whine
2622 * if we are scheduling when we should not.
1da177e4 2623 */
192301e7 2624 if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD))
dd41f596 2625 __schedule_bug(prev);
b3fbab05 2626 rcu_sleep_check();
dd41f596 2627
1da177e4
LT
2628 profile_hit(SCHED_PROFILING, __builtin_return_address(0));
2629
2d72376b 2630 schedstat_inc(this_rq(), sched_count);
dd41f596
IM
2631}
2632
2633/*
2634 * Pick up the highest-prio task:
2635 */
2636static inline struct task_struct *
606dba2e 2637pick_next_task(struct rq *rq, struct task_struct *prev)
dd41f596 2638{
37e117c0 2639 const struct sched_class *class = &fair_sched_class;
dd41f596 2640 struct task_struct *p;
1da177e4
LT
2641
2642 /*
dd41f596
IM
2643 * Optimization: we know that if all tasks are in
2644 * the fair class we can call that function directly:
1da177e4 2645 */
37e117c0 2646 if (likely(prev->sched_class == class &&
38033c37 2647 rq->nr_running == rq->cfs.h_nr_running)) {
606dba2e 2648 p = fair_sched_class.pick_next_task(rq, prev);
6ccdc84b
PZ
2649 if (unlikely(p == RETRY_TASK))
2650 goto again;
2651
2652 /* assumes fair_sched_class->next == idle_sched_class */
2653 if (unlikely(!p))
2654 p = idle_sched_class.pick_next_task(rq, prev);
2655
2656 return p;
1da177e4
LT
2657 }
2658
37e117c0 2659again:
34f971f6 2660 for_each_class(class) {
606dba2e 2661 p = class->pick_next_task(rq, prev);
37e117c0
PZ
2662 if (p) {
2663 if (unlikely(p == RETRY_TASK))
2664 goto again;
dd41f596 2665 return p;
37e117c0 2666 }
dd41f596 2667 }
34f971f6
PZ
2668
2669 BUG(); /* the idle class will always have a runnable task */
dd41f596 2670}
1da177e4 2671
dd41f596 2672/*
c259e01a 2673 * __schedule() is the main scheduler function.
edde96ea
PE
2674 *
2675 * The main means of driving the scheduler and thus entering this function are:
2676 *
2677 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc.
2678 *
2679 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
2680 * paths. For example, see arch/x86/entry_64.S.
2681 *
2682 * To drive preemption between tasks, the scheduler sets the flag in timer
2683 * interrupt handler scheduler_tick().
2684 *
2685 * 3. Wakeups don't really cause entry into schedule(). They add a
2686 * task to the run-queue and that's it.
2687 *
2688 * Now, if the new task added to the run-queue preempts the current
2689 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
2690 * called on the nearest possible occasion:
2691 *
2692 * - If the kernel is preemptible (CONFIG_PREEMPT=y):
2693 *
2694 * - in syscall or exception context, at the next outmost
2695 * preempt_enable(). (this might be as soon as the wake_up()'s
2696 * spin_unlock()!)
2697 *
2698 * - in IRQ context, return from interrupt-handler to
2699 * preemptible context
2700 *
2701 * - If the kernel is not preemptible (CONFIG_PREEMPT is not set)
2702 * then at the next:
2703 *
2704 * - cond_resched() call
2705 * - explicit schedule() call
2706 * - return from syscall or exception to user-space
2707 * - return from interrupt-handler to user-space
bfd9b2b5
FW
2708 *
2709 * WARNING: all callers must re-check need_resched() afterward and reschedule
2710 * accordingly in case an event triggered the need for rescheduling (such as
2711 * an interrupt waking up a task) while preemption was disabled in __schedule().
dd41f596 2712 */
c259e01a 2713static void __sched __schedule(void)
dd41f596
IM
2714{
2715 struct task_struct *prev, *next;
67ca7bde 2716 unsigned long *switch_count;
dd41f596 2717 struct rq *rq;
31656519 2718 int cpu;
dd41f596 2719
ff743345 2720 preempt_disable();
dd41f596
IM
2721 cpu = smp_processor_id();
2722 rq = cpu_rq(cpu);
38200cf2 2723 rcu_note_context_switch();
dd41f596 2724 prev = rq->curr;
dd41f596 2725
dd41f596 2726 schedule_debug(prev);
1da177e4 2727
31656519 2728 if (sched_feat(HRTICK))
f333fdc9 2729 hrtick_clear(rq);
8f4d37ec 2730
e0acd0a6
ON
2731 /*
2732 * Make sure that signal_pending_state()->signal_pending() below
2733 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
2734 * done by the caller to avoid the race with signal_wake_up().
2735 */
2736 smp_mb__before_spinlock();
05fa785c 2737 raw_spin_lock_irq(&rq->lock);
1da177e4 2738
9edfbfed
PZ
2739 rq->clock_skip_update <<= 1; /* promote REQ to ACT */
2740
246d86b5 2741 switch_count = &prev->nivcsw;
1da177e4 2742 if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) {
21aa9af0 2743 if (unlikely(signal_pending_state(prev->state, prev))) {
1da177e4 2744 prev->state = TASK_RUNNING;
21aa9af0 2745 } else {
2acca55e
PZ
2746 deactivate_task(rq, prev, DEQUEUE_SLEEP);
2747 prev->on_rq = 0;
2748
21aa9af0 2749 /*
2acca55e
PZ
2750 * If a worker went to sleep, notify and ask workqueue
2751 * whether it wants to wake up a task to maintain
2752 * concurrency.
21aa9af0
TH
2753 */
2754 if (prev->flags & PF_WQ_WORKER) {
2755 struct task_struct *to_wakeup;
2756
2757 to_wakeup = wq_worker_sleeping(prev, cpu);
2758 if (to_wakeup)
2759 try_to_wake_up_local(to_wakeup);
2760 }
21aa9af0 2761 }
dd41f596 2762 switch_count = &prev->nvcsw;
1da177e4
LT
2763 }
2764
9edfbfed 2765 if (task_on_rq_queued(prev))
606dba2e
PZ
2766 update_rq_clock(rq);
2767
2768 next = pick_next_task(rq, prev);
f26f9aff 2769 clear_tsk_need_resched(prev);
f27dde8d 2770 clear_preempt_need_resched();
9edfbfed 2771 rq->clock_skip_update = 0;
1da177e4 2772
1da177e4 2773 if (likely(prev != next)) {
1da177e4
LT
2774 rq->nr_switches++;
2775 rq->curr = next;
2776 ++*switch_count;
2777
dfa50b60
ON
2778 rq = context_switch(rq, prev, next); /* unlocks the rq */
2779 cpu = cpu_of(rq);
1da177e4 2780 } else
05fa785c 2781 raw_spin_unlock_irq(&rq->lock);
1da177e4 2782
3f029d3c 2783 post_schedule(rq);
1da177e4 2784
ba74c144 2785 sched_preempt_enable_no_resched();
1da177e4 2786}
c259e01a 2787
9c40cef2
TG
2788static inline void sched_submit_work(struct task_struct *tsk)
2789{
3c7d5184 2790 if (!tsk->state || tsk_is_pi_blocked(tsk))
9c40cef2
TG
2791 return;
2792 /*
2793 * If we are going to sleep and we have plugged IO queued,
2794 * make sure to submit it to avoid deadlocks.
2795 */
2796 if (blk_needs_flush_plug(tsk))
2797 blk_schedule_flush_plug(tsk);
2798}
2799
722a9f92 2800asmlinkage __visible void __sched schedule(void)
c259e01a 2801{
9c40cef2
TG
2802 struct task_struct *tsk = current;
2803
2804 sched_submit_work(tsk);
bfd9b2b5
FW
2805 do {
2806 __schedule();
2807 } while (need_resched());
c259e01a 2808}
1da177e4
LT
2809EXPORT_SYMBOL(schedule);
2810
91d1aa43 2811#ifdef CONFIG_CONTEXT_TRACKING
722a9f92 2812asmlinkage __visible void __sched schedule_user(void)
20ab65e3
FW
2813{
2814 /*
2815 * If we come here after a random call to set_need_resched(),
2816 * or we have been woken up remotely but the IPI has not yet arrived,
2817 * we haven't yet exited the RCU idle mode. Do it here manually until
2818 * we find a better solution.
7cc78f8f
AL
2819 *
2820 * NB: There are buggy callers of this function. Ideally we
2821 * should warn if prev_state != IN_USER, but that will trigger
2822 * too frequently to make sense yet.
20ab65e3 2823 */
7cc78f8f 2824 enum ctx_state prev_state = exception_enter();
20ab65e3 2825 schedule();
7cc78f8f 2826 exception_exit(prev_state);
20ab65e3
FW
2827}
2828#endif
2829
c5491ea7
TG
2830/**
2831 * schedule_preempt_disabled - called with preemption disabled
2832 *
2833 * Returns with preemption disabled. Note: preempt_count must be 1
2834 */
2835void __sched schedule_preempt_disabled(void)
2836{
ba74c144 2837 sched_preempt_enable_no_resched();
c5491ea7
TG
2838 schedule();
2839 preempt_disable();
2840}
2841
06b1f808 2842static void __sched notrace preempt_schedule_common(void)
a18b5d01
FW
2843{
2844 do {
2845 __preempt_count_add(PREEMPT_ACTIVE);
2846 __schedule();
2847 __preempt_count_sub(PREEMPT_ACTIVE);
2848
2849 /*
2850 * Check again in case we missed a preemption opportunity
2851 * between schedule and now.
2852 */
2853 barrier();
2854 } while (need_resched());
2855}
2856
1da177e4
LT
2857#ifdef CONFIG_PREEMPT
2858/*
2ed6e34f 2859 * this is the entry point to schedule() from in-kernel preemption
41a2d6cf 2860 * off of preempt_enable. Kernel preemptions off return from interrupt
1da177e4
LT
2861 * occur there and call schedule directly.
2862 */
722a9f92 2863asmlinkage __visible void __sched notrace preempt_schedule(void)
1da177e4 2864{
1da177e4
LT
2865 /*
2866 * If there is a non-zero preempt_count or interrupts are disabled,
41a2d6cf 2867 * we do not want to preempt the current task. Just return..
1da177e4 2868 */
fbb00b56 2869 if (likely(!preemptible()))
1da177e4
LT
2870 return;
2871
a18b5d01 2872 preempt_schedule_common();
1da177e4 2873}
376e2424 2874NOKPROBE_SYMBOL(preempt_schedule);
1da177e4 2875EXPORT_SYMBOL(preempt_schedule);
009f60e2
ON
2876
2877#ifdef CONFIG_CONTEXT_TRACKING
2878/**
2879 * preempt_schedule_context - preempt_schedule called by tracing
2880 *
2881 * The tracing infrastructure uses preempt_enable_notrace to prevent
2882 * recursion and tracing preempt enabling caused by the tracing
2883 * infrastructure itself. But as tracing can happen in areas coming
2884 * from userspace or just about to enter userspace, a preempt enable
2885 * can occur before user_exit() is called. This will cause the scheduler
2886 * to be called when the system is still in usermode.
2887 *
2888 * To prevent this, the preempt_enable_notrace will use this function
2889 * instead of preempt_schedule() to exit user context if needed before
2890 * calling the scheduler.
2891 */
2892asmlinkage __visible void __sched notrace preempt_schedule_context(void)
2893{
2894 enum ctx_state prev_ctx;
2895
2896 if (likely(!preemptible()))
2897 return;
2898
2899 do {
2900 __preempt_count_add(PREEMPT_ACTIVE);
2901 /*
2902 * Needs preempt disabled in case user_exit() is traced
2903 * and the tracer calls preempt_enable_notrace() causing
2904 * an infinite recursion.
2905 */
2906 prev_ctx = exception_enter();
2907 __schedule();
2908 exception_exit(prev_ctx);
2909
2910 __preempt_count_sub(PREEMPT_ACTIVE);
2911 barrier();
2912 } while (need_resched());
2913}
2914EXPORT_SYMBOL_GPL(preempt_schedule_context);
2915#endif /* CONFIG_CONTEXT_TRACKING */
2916
32e475d7 2917#endif /* CONFIG_PREEMPT */
1da177e4
LT
2918
2919/*
2ed6e34f 2920 * this is the entry point to schedule() from kernel preemption
1da177e4
LT
2921 * off of irq context.
2922 * Note, that this is called and return with irqs disabled. This will
2923 * protect us against recursive calling from irq.
2924 */
722a9f92 2925asmlinkage __visible void __sched preempt_schedule_irq(void)
1da177e4 2926{
b22366cd 2927 enum ctx_state prev_state;
6478d880 2928
2ed6e34f 2929 /* Catch callers which need to be fixed */
f27dde8d 2930 BUG_ON(preempt_count() || !irqs_disabled());
1da177e4 2931
b22366cd
FW
2932 prev_state = exception_enter();
2933
3a5c359a 2934 do {
bdb43806 2935 __preempt_count_add(PREEMPT_ACTIVE);
3a5c359a 2936 local_irq_enable();
c259e01a 2937 __schedule();
3a5c359a 2938 local_irq_disable();
bdb43806 2939 __preempt_count_sub(PREEMPT_ACTIVE);
1da177e4 2940
3a5c359a
AK
2941 /*
2942 * Check again in case we missed a preemption opportunity
2943 * between schedule and now.
2944 */
2945 barrier();
5ed0cec0 2946 } while (need_resched());
b22366cd
FW
2947
2948 exception_exit(prev_state);
1da177e4
LT
2949}
2950
63859d4f 2951int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
95cdf3b7 2952 void *key)
1da177e4 2953{
63859d4f 2954 return try_to_wake_up(curr->private, mode, wake_flags);
1da177e4 2955}
1da177e4
LT
2956EXPORT_SYMBOL(default_wake_function);
2957
b29739f9
IM
2958#ifdef CONFIG_RT_MUTEXES
2959
2960/*
2961 * rt_mutex_setprio - set the current priority of a task
2962 * @p: task
2963 * @prio: prio value (kernel-internal form)
2964 *
2965 * This function changes the 'effective' priority of a task. It does
2966 * not touch ->normal_prio like __setscheduler().
2967 *
c365c292
TG
2968 * Used by the rt_mutex code to implement priority inheritance
2969 * logic. Call site only calls if the priority of the task changed.
b29739f9 2970 */
36c8b586 2971void rt_mutex_setprio(struct task_struct *p, int prio)
b29739f9 2972{
da0c1e65 2973 int oldprio, queued, running, enqueue_flag = 0;
70b97a7f 2974 struct rq *rq;
83ab0aa0 2975 const struct sched_class *prev_class;
b29739f9 2976
aab03e05 2977 BUG_ON(prio > MAX_PRIO);
b29739f9 2978
0122ec5b 2979 rq = __task_rq_lock(p);
b29739f9 2980
1c4dd99b
TG
2981 /*
2982 * Idle task boosting is a nono in general. There is one
2983 * exception, when PREEMPT_RT and NOHZ is active:
2984 *
2985 * The idle task calls get_next_timer_interrupt() and holds
2986 * the timer wheel base->lock on the CPU and another CPU wants
2987 * to access the timer (probably to cancel it). We can safely
2988 * ignore the boosting request, as the idle CPU runs this code
2989 * with interrupts disabled and will complete the lock
2990 * protected section without being interrupted. So there is no
2991 * real need to boost.
2992 */
2993 if (unlikely(p == rq->idle)) {
2994 WARN_ON(p != rq->curr);
2995 WARN_ON(p->pi_blocked_on);
2996 goto out_unlock;
2997 }
2998
a8027073 2999 trace_sched_pi_setprio(p, prio);
d5f9f942 3000 oldprio = p->prio;
83ab0aa0 3001 prev_class = p->sched_class;
da0c1e65 3002 queued = task_on_rq_queued(p);
051a1d1a 3003 running = task_current(rq, p);
da0c1e65 3004 if (queued)
69be72c1 3005 dequeue_task(rq, p, 0);
0e1f3483 3006 if (running)
f3cd1c4e 3007 put_prev_task(rq, p);
dd41f596 3008
2d3d891d
DF
3009 /*
3010 * Boosting condition are:
3011 * 1. -rt task is running and holds mutex A
3012 * --> -dl task blocks on mutex A
3013 *
3014 * 2. -dl task is running and holds mutex A
3015 * --> -dl task blocks on mutex A and could preempt the
3016 * running task
3017 */
3018 if (dl_prio(prio)) {
466af29b
ON
3019 struct task_struct *pi_task = rt_mutex_get_top_task(p);
3020 if (!dl_prio(p->normal_prio) ||
3021 (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
2d3d891d
DF
3022 p->dl.dl_boosted = 1;
3023 p->dl.dl_throttled = 0;
3024 enqueue_flag = ENQUEUE_REPLENISH;
3025 } else
3026 p->dl.dl_boosted = 0;
aab03e05 3027 p->sched_class = &dl_sched_class;
2d3d891d
DF
3028 } else if (rt_prio(prio)) {
3029 if (dl_prio(oldprio))
3030 p->dl.dl_boosted = 0;
3031 if (oldprio < prio)
3032 enqueue_flag = ENQUEUE_HEAD;
dd41f596 3033 p->sched_class = &rt_sched_class;
2d3d891d
DF
3034 } else {
3035 if (dl_prio(oldprio))
3036 p->dl.dl_boosted = 0;
dd41f596 3037 p->sched_class = &fair_sched_class;
2d3d891d 3038 }
dd41f596 3039
b29739f9
IM
3040 p->prio = prio;
3041
0e1f3483
HS
3042 if (running)
3043 p->sched_class->set_curr_task(rq);
da0c1e65 3044 if (queued)
2d3d891d 3045 enqueue_task(rq, p, enqueue_flag);
cb469845 3046
da7a735e 3047 check_class_changed(rq, p, prev_class, oldprio);
1c4dd99b 3048out_unlock:
0122ec5b 3049 __task_rq_unlock(rq);
b29739f9 3050}
b29739f9 3051#endif
d50dde5a 3052
36c8b586 3053void set_user_nice(struct task_struct *p, long nice)
1da177e4 3054{
da0c1e65 3055 int old_prio, delta, queued;
1da177e4 3056 unsigned long flags;
70b97a7f 3057 struct rq *rq;
1da177e4 3058
75e45d51 3059 if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
1da177e4
LT
3060 return;
3061 /*
3062 * We have to be careful, if called from sys_setpriority(),
3063 * the task might be in the middle of scheduling on another CPU.
3064 */
3065 rq = task_rq_lock(p, &flags);
3066 /*
3067 * The RT priorities are set via sched_setscheduler(), but we still
3068 * allow the 'normal' nice value to be set - but as expected
3069 * it wont have any effect on scheduling until the task is
aab03e05 3070 * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR:
1da177e4 3071 */
aab03e05 3072 if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
1da177e4
LT
3073 p->static_prio = NICE_TO_PRIO(nice);
3074 goto out_unlock;
3075 }
da0c1e65
KT
3076 queued = task_on_rq_queued(p);
3077 if (queued)
69be72c1 3078 dequeue_task(rq, p, 0);
1da177e4 3079
1da177e4 3080 p->static_prio = NICE_TO_PRIO(nice);
2dd73a4f 3081 set_load_weight(p);
b29739f9
IM
3082 old_prio = p->prio;
3083 p->prio = effective_prio(p);
3084 delta = p->prio - old_prio;
1da177e4 3085
da0c1e65 3086 if (queued) {
371fd7e7 3087 enqueue_task(rq, p, 0);
1da177e4 3088 /*
d5f9f942
AM
3089 * If the task increased its priority or is running and
3090 * lowered its priority, then reschedule its CPU:
1da177e4 3091 */
d5f9f942 3092 if (delta < 0 || (delta > 0 && task_running(rq, p)))
8875125e 3093 resched_curr(rq);
1da177e4
LT
3094 }
3095out_unlock:
0122ec5b 3096 task_rq_unlock(rq, p, &flags);
1da177e4 3097}
1da177e4
LT
3098EXPORT_SYMBOL(set_user_nice);
3099
e43379f1
MM
3100/*
3101 * can_nice - check if a task can reduce its nice value
3102 * @p: task
3103 * @nice: nice value
3104 */
36c8b586 3105int can_nice(const struct task_struct *p, const int nice)
e43379f1 3106{
024f4747 3107 /* convert nice value [19,-20] to rlimit style value [1,40] */
7aa2c016 3108 int nice_rlim = nice_to_rlimit(nice);
48f24c4d 3109
78d7d407 3110 return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
e43379f1
MM
3111 capable(CAP_SYS_NICE));
3112}
3113
1da177e4
LT
3114#ifdef __ARCH_WANT_SYS_NICE
3115
3116/*
3117 * sys_nice - change the priority of the current process.
3118 * @increment: priority increment
3119 *
3120 * sys_setpriority is a more generic, but much slower function that
3121 * does similar things.
3122 */
5add95d4 3123SYSCALL_DEFINE1(nice, int, increment)
1da177e4 3124{
48f24c4d 3125 long nice, retval;
1da177e4
LT
3126
3127 /*
3128 * Setpriority might change our priority at the same moment.
3129 * We don't have to worry. Conceptually one call occurs first
3130 * and we have a single winner.
3131 */
a9467fa3 3132 increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
d0ea0268 3133 nice = task_nice(current) + increment;
1da177e4 3134
a9467fa3 3135 nice = clamp_val(nice, MIN_NICE, MAX_NICE);
e43379f1
MM
3136 if (increment < 0 && !can_nice(current, nice))
3137 return -EPERM;
3138
1da177e4
LT
3139 retval = security_task_setnice(current, nice);
3140 if (retval)
3141 return retval;
3142
3143 set_user_nice(current, nice);
3144 return 0;
3145}
3146
3147#endif
3148
3149/**
3150 * task_prio - return the priority value of a given task.
3151 * @p: the task in question.
3152 *
e69f6186 3153 * Return: The priority value as seen by users in /proc.
1da177e4
LT
3154 * RT tasks are offset by -200. Normal tasks are centered
3155 * around 0, value goes from -16 to +15.
3156 */
36c8b586 3157int task_prio(const struct task_struct *p)
1da177e4
LT
3158{
3159 return p->prio - MAX_RT_PRIO;
3160}
3161
1da177e4
LT
3162/**
3163 * idle_cpu - is a given cpu idle currently?
3164 * @cpu: the processor in question.
e69f6186
YB
3165 *
3166 * Return: 1 if the CPU is currently idle. 0 otherwise.
1da177e4
LT
3167 */
3168int idle_cpu(int cpu)
3169{
908a3283
TG
3170 struct rq *rq = cpu_rq(cpu);
3171
3172 if (rq->curr != rq->idle)
3173 return 0;
3174
3175 if (rq->nr_running)
3176 return 0;
3177
3178#ifdef CONFIG_SMP
3179 if (!llist_empty(&rq->wake_list))
3180 return 0;
3181#endif
3182
3183 return 1;
1da177e4
LT
3184}
3185
1da177e4
LT
3186/**
3187 * idle_task - return the idle task for a given cpu.
3188 * @cpu: the processor in question.
e69f6186
YB
3189 *
3190 * Return: The idle task for the cpu @cpu.
1da177e4 3191 */
36c8b586 3192struct task_struct *idle_task(int cpu)
1da177e4
LT
3193{
3194 return cpu_rq(cpu)->idle;
3195}
3196
3197/**
3198 * find_process_by_pid - find a process with a matching PID value.
3199 * @pid: the pid in question.
e69f6186
YB
3200 *
3201 * The task of @pid, if found. %NULL otherwise.
1da177e4 3202 */
a9957449 3203static struct task_struct *find_process_by_pid(pid_t pid)
1da177e4 3204{
228ebcbe 3205 return pid ? find_task_by_vpid(pid) : current;
1da177e4
LT
3206}
3207
aab03e05
DF
3208/*
3209 * This function initializes the sched_dl_entity of a newly becoming
3210 * SCHED_DEADLINE task.
3211 *
3212 * Only the static values are considered here, the actual runtime and the
3213 * absolute deadline will be properly calculated when the task is enqueued
3214 * for the first time with its new policy.
3215 */
3216static void
3217__setparam_dl(struct task_struct *p, const struct sched_attr *attr)
3218{
3219 struct sched_dl_entity *dl_se = &p->dl;
3220
aab03e05
DF
3221 dl_se->dl_runtime = attr->sched_runtime;
3222 dl_se->dl_deadline = attr->sched_deadline;
755378a4 3223 dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
aab03e05 3224 dl_se->flags = attr->sched_flags;
332ac17e 3225 dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
40767b0d
PZ
3226
3227 /*
3228 * Changing the parameters of a task is 'tricky' and we're not doing
3229 * the correct thing -- also see task_dead_dl() and switched_from_dl().
3230 *
3231 * What we SHOULD do is delay the bandwidth release until the 0-lag
3232 * point. This would include retaining the task_struct until that time
3233 * and change dl_overflow() to not immediately decrement the current
3234 * amount.
3235 *
3236 * Instead we retain the current runtime/deadline and let the new
3237 * parameters take effect after the current reservation period lapses.
3238 * This is safe (albeit pessimistic) because the 0-lag point is always
3239 * before the current scheduling deadline.
3240 *
3241 * We can still have temporary overloads because we do not delay the
3242 * change in bandwidth until that time; so admission control is
3243 * not on the safe side. It does however guarantee tasks will never
3244 * consume more than promised.
3245 */
aab03e05
DF
3246}
3247
c13db6b1
SR
3248/*
3249 * sched_setparam() passes in -1 for its policy, to let the functions
3250 * it calls know not to change it.
3251 */
3252#define SETPARAM_POLICY -1
3253
c365c292
TG
3254static void __setscheduler_params(struct task_struct *p,
3255 const struct sched_attr *attr)
1da177e4 3256{
d50dde5a
DF
3257 int policy = attr->sched_policy;
3258
c13db6b1 3259 if (policy == SETPARAM_POLICY)
39fd8fd2
PZ
3260 policy = p->policy;
3261
1da177e4 3262 p->policy = policy;
d50dde5a 3263
aab03e05
DF
3264 if (dl_policy(policy))
3265 __setparam_dl(p, attr);
39fd8fd2 3266 else if (fair_policy(policy))
d50dde5a
DF
3267 p->static_prio = NICE_TO_PRIO(attr->sched_nice);
3268
39fd8fd2
PZ
3269 /*
3270 * __sched_setscheduler() ensures attr->sched_priority == 0 when
3271 * !rt_policy. Always setting this ensures that things like
3272 * getparam()/getattr() don't report silly values for !rt tasks.
3273 */
3274 p->rt_priority = attr->sched_priority;
383afd09 3275 p->normal_prio = normal_prio(p);
c365c292
TG
3276 set_load_weight(p);
3277}
39fd8fd2 3278
c365c292
TG
3279/* Actually do priority change: must hold pi & rq lock. */
3280static void __setscheduler(struct rq *rq, struct task_struct *p,
3281 const struct sched_attr *attr)
3282{
3283 __setscheduler_params(p, attr);
d50dde5a 3284
383afd09
SR
3285 /*
3286 * If we get here, there was no pi waiters boosting the
3287 * task. It is safe to use the normal prio.
3288 */
3289 p->prio = normal_prio(p);
3290
aab03e05
DF
3291 if (dl_prio(p->prio))
3292 p->sched_class = &dl_sched_class;
3293 else if (rt_prio(p->prio))
ffd44db5
PZ
3294 p->sched_class = &rt_sched_class;
3295 else
3296 p->sched_class = &fair_sched_class;
1da177e4 3297}
aab03e05
DF
3298
3299static void
3300__getparam_dl(struct task_struct *p, struct sched_attr *attr)
3301{
3302 struct sched_dl_entity *dl_se = &p->dl;
3303
3304 attr->sched_priority = p->rt_priority;
3305 attr->sched_runtime = dl_se->dl_runtime;
3306 attr->sched_deadline = dl_se->dl_deadline;
755378a4 3307 attr->sched_period = dl_se->dl_period;
aab03e05
DF
3308 attr->sched_flags = dl_se->flags;
3309}
3310
3311/*
3312 * This function validates the new parameters of a -deadline task.
3313 * We ask for the deadline not being zero, and greater or equal
755378a4 3314 * than the runtime, as well as the period of being zero or
332ac17e 3315 * greater than deadline. Furthermore, we have to be sure that
b0827819
JL
3316 * user parameters are above the internal resolution of 1us (we
3317 * check sched_runtime only since it is always the smaller one) and
3318 * below 2^63 ns (we have to check both sched_deadline and
3319 * sched_period, as the latter can be zero).
aab03e05
DF
3320 */
3321static bool
3322__checkparam_dl(const struct sched_attr *attr)
3323{
b0827819
JL
3324 /* deadline != 0 */
3325 if (attr->sched_deadline == 0)
3326 return false;
3327
3328 /*
3329 * Since we truncate DL_SCALE bits, make sure we're at least
3330 * that big.
3331 */
3332 if (attr->sched_runtime < (1ULL << DL_SCALE))
3333 return false;
3334
3335 /*
3336 * Since we use the MSB for wrap-around and sign issues, make
3337 * sure it's not set (mind that period can be equal to zero).
3338 */
3339 if (attr->sched_deadline & (1ULL << 63) ||
3340 attr->sched_period & (1ULL << 63))
3341 return false;
3342
3343 /* runtime <= deadline <= period (if period != 0) */
3344 if ((attr->sched_period != 0 &&
3345 attr->sched_period < attr->sched_deadline) ||
3346 attr->sched_deadline < attr->sched_runtime)
3347 return false;
3348
3349 return true;
aab03e05
DF
3350}
3351
c69e8d9c
DH
3352/*
3353 * check the target process has a UID that matches the current process's
3354 */
3355static bool check_same_owner(struct task_struct *p)
3356{
3357 const struct cred *cred = current_cred(), *pcred;
3358 bool match;
3359
3360 rcu_read_lock();
3361 pcred = __task_cred(p);
9c806aa0
EB
3362 match = (uid_eq(cred->euid, pcred->euid) ||
3363 uid_eq(cred->euid, pcred->uid));
c69e8d9c
DH
3364 rcu_read_unlock();
3365 return match;
3366}
3367
75381608
WL
3368static bool dl_param_changed(struct task_struct *p,
3369 const struct sched_attr *attr)
3370{
3371 struct sched_dl_entity *dl_se = &p->dl;
3372
3373 if (dl_se->dl_runtime != attr->sched_runtime ||
3374 dl_se->dl_deadline != attr->sched_deadline ||
3375 dl_se->dl_period != attr->sched_period ||
3376 dl_se->flags != attr->sched_flags)
3377 return true;
3378
3379 return false;
3380}
3381
d50dde5a
DF
3382static int __sched_setscheduler(struct task_struct *p,
3383 const struct sched_attr *attr,
3384 bool user)
1da177e4 3385{
383afd09
SR
3386 int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
3387 MAX_RT_PRIO - 1 - attr->sched_priority;
da0c1e65 3388 int retval, oldprio, oldpolicy = -1, queued, running;
d50dde5a 3389 int policy = attr->sched_policy;
1da177e4 3390 unsigned long flags;
83ab0aa0 3391 const struct sched_class *prev_class;
70b97a7f 3392 struct rq *rq;
ca94c442 3393 int reset_on_fork;
1da177e4 3394
66e5393a
SR
3395 /* may grab non-irq protected spin_locks */
3396 BUG_ON(in_interrupt());
1da177e4
LT
3397recheck:
3398 /* double check policy once rq lock held */
ca94c442
LP
3399 if (policy < 0) {
3400 reset_on_fork = p->sched_reset_on_fork;
1da177e4 3401 policy = oldpolicy = p->policy;
ca94c442 3402 } else {
7479f3c9 3403 reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK);
ca94c442 3404
aab03e05
DF
3405 if (policy != SCHED_DEADLINE &&
3406 policy != SCHED_FIFO && policy != SCHED_RR &&
ca94c442
LP
3407 policy != SCHED_NORMAL && policy != SCHED_BATCH &&
3408 policy != SCHED_IDLE)
3409 return -EINVAL;
3410 }
3411
7479f3c9
PZ
3412 if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK))
3413 return -EINVAL;
3414
1da177e4
LT
3415 /*
3416 * Valid priorities for SCHED_FIFO and SCHED_RR are
dd41f596
IM
3417 * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
3418 * SCHED_BATCH and SCHED_IDLE is 0.
1da177e4 3419 */
0bb040a4 3420 if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
d50dde5a 3421 (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
1da177e4 3422 return -EINVAL;
aab03e05
DF
3423 if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
3424 (rt_policy(policy) != (attr->sched_priority != 0)))
1da177e4
LT
3425 return -EINVAL;
3426
37e4ab3f
OC
3427 /*
3428 * Allow unprivileged RT tasks to decrease priority:
3429 */
961ccddd 3430 if (user && !capable(CAP_SYS_NICE)) {
d50dde5a 3431 if (fair_policy(policy)) {
d0ea0268 3432 if (attr->sched_nice < task_nice(p) &&
eaad4513 3433 !can_nice(p, attr->sched_nice))
d50dde5a
DF
3434 return -EPERM;
3435 }
3436
e05606d3 3437 if (rt_policy(policy)) {
a44702e8
ON
3438 unsigned long rlim_rtprio =
3439 task_rlimit(p, RLIMIT_RTPRIO);
8dc3e909
ON
3440
3441 /* can't set/change the rt policy */
3442 if (policy != p->policy && !rlim_rtprio)
3443 return -EPERM;
3444
3445 /* can't increase priority */
d50dde5a
DF
3446 if (attr->sched_priority > p->rt_priority &&
3447 attr->sched_priority > rlim_rtprio)
8dc3e909
ON
3448 return -EPERM;
3449 }
c02aa73b 3450
d44753b8
JL
3451 /*
3452 * Can't set/change SCHED_DEADLINE policy at all for now
3453 * (safest behavior); in the future we would like to allow
3454 * unprivileged DL tasks to increase their relative deadline
3455 * or reduce their runtime (both ways reducing utilization)
3456 */
3457 if (dl_policy(policy))
3458 return -EPERM;
3459
dd41f596 3460 /*
c02aa73b
DH
3461 * Treat SCHED_IDLE as nice 20. Only allow a switch to
3462 * SCHED_NORMAL if the RLIMIT_NICE would normally permit it.
dd41f596 3463 */
c02aa73b 3464 if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) {
d0ea0268 3465 if (!can_nice(p, task_nice(p)))
c02aa73b
DH
3466 return -EPERM;
3467 }
5fe1d75f 3468
37e4ab3f 3469 /* can't change other user's priorities */
c69e8d9c 3470 if (!check_same_owner(p))
37e4ab3f 3471 return -EPERM;
ca94c442
LP
3472
3473 /* Normal users shall not reset the sched_reset_on_fork flag */
3474 if (p->sched_reset_on_fork && !reset_on_fork)
3475 return -EPERM;
37e4ab3f 3476 }
1da177e4 3477
725aad24 3478 if (user) {
b0ae1981 3479 retval = security_task_setscheduler(p);
725aad24
JF
3480 if (retval)
3481 return retval;
3482 }
3483
b29739f9
IM
3484 /*
3485 * make sure no PI-waiters arrive (or leave) while we are
3486 * changing the priority of the task:
0122ec5b 3487 *
25985edc 3488 * To be able to change p->policy safely, the appropriate
1da177e4
LT
3489 * runqueue lock must be held.
3490 */
0122ec5b 3491 rq = task_rq_lock(p, &flags);
dc61b1d6 3492
34f971f6
PZ
3493 /*
3494 * Changing the policy of the stop threads its a very bad idea
3495 */
3496 if (p == rq->stop) {
0122ec5b 3497 task_rq_unlock(rq, p, &flags);
34f971f6
PZ
3498 return -EINVAL;
3499 }
3500
a51e9198 3501 /*
d6b1e911
TG
3502 * If not changing anything there's no need to proceed further,
3503 * but store a possible modification of reset_on_fork.
a51e9198 3504 */
d50dde5a 3505 if (unlikely(policy == p->policy)) {
d0ea0268 3506 if (fair_policy(policy) && attr->sched_nice != task_nice(p))
d50dde5a
DF
3507 goto change;
3508 if (rt_policy(policy) && attr->sched_priority != p->rt_priority)
3509 goto change;
75381608 3510 if (dl_policy(policy) && dl_param_changed(p, attr))
aab03e05 3511 goto change;
d50dde5a 3512
d6b1e911 3513 p->sched_reset_on_fork = reset_on_fork;
45afb173 3514 task_rq_unlock(rq, p, &flags);
a51e9198
DF
3515 return 0;
3516 }
d50dde5a 3517change:
a51e9198 3518
dc61b1d6 3519 if (user) {
332ac17e 3520#ifdef CONFIG_RT_GROUP_SCHED
dc61b1d6
PZ
3521 /*
3522 * Do not allow realtime tasks into groups that have no runtime
3523 * assigned.
3524 */
3525 if (rt_bandwidth_enabled() && rt_policy(policy) &&
f4493771
MG
3526 task_group(p)->rt_bandwidth.rt_runtime == 0 &&
3527 !task_group_is_autogroup(task_group(p))) {
0122ec5b 3528 task_rq_unlock(rq, p, &flags);
dc61b1d6
PZ
3529 return -EPERM;
3530 }
dc61b1d6 3531#endif
332ac17e
DF
3532#ifdef CONFIG_SMP
3533 if (dl_bandwidth_enabled() && dl_policy(policy)) {
3534 cpumask_t *span = rq->rd->span;
332ac17e
DF
3535
3536 /*
3537 * Don't allow tasks with an affinity mask smaller than
3538 * the entire root_domain to become SCHED_DEADLINE. We
3539 * will also fail if there's no bandwidth available.
3540 */
e4099a5e
PZ
3541 if (!cpumask_subset(span, &p->cpus_allowed) ||
3542 rq->rd->dl_bw.bw == 0) {
332ac17e
DF
3543 task_rq_unlock(rq, p, &flags);
3544 return -EPERM;
3545 }
3546 }
3547#endif
3548 }
dc61b1d6 3549
1da177e4
LT
3550 /* recheck policy now with rq lock held */
3551 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
3552 policy = oldpolicy = -1;
0122ec5b 3553 task_rq_unlock(rq, p, &flags);
1da177e4
LT
3554 goto recheck;
3555 }
332ac17e
DF
3556
3557 /*
3558 * If setscheduling to SCHED_DEADLINE (or changing the parameters
3559 * of a SCHED_DEADLINE task) we need to check if enough bandwidth
3560 * is available.
3561 */
e4099a5e 3562 if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) {
332ac17e
DF
3563 task_rq_unlock(rq, p, &flags);
3564 return -EBUSY;
3565 }
3566
c365c292
TG
3567 p->sched_reset_on_fork = reset_on_fork;
3568 oldprio = p->prio;
3569
3570 /*
3571 * Special case for priority boosted tasks.
3572 *
3573 * If the new priority is lower or equal (user space view)
3574 * than the current (boosted) priority, we just store the new
3575 * normal parameters and do not touch the scheduler class and
3576 * the runqueue. This will be done when the task deboost
3577 * itself.
3578 */
3579 if (rt_mutex_check_prio(p, newprio)) {
3580 __setscheduler_params(p, attr);
3581 task_rq_unlock(rq, p, &flags);
3582 return 0;
3583 }
3584
da0c1e65 3585 queued = task_on_rq_queued(p);
051a1d1a 3586 running = task_current(rq, p);
da0c1e65 3587 if (queued)
4ca9b72b 3588 dequeue_task(rq, p, 0);
0e1f3483 3589 if (running)
f3cd1c4e 3590 put_prev_task(rq, p);
f6b53205 3591
83ab0aa0 3592 prev_class = p->sched_class;
d50dde5a 3593 __setscheduler(rq, p, attr);
f6b53205 3594
0e1f3483
HS
3595 if (running)
3596 p->sched_class->set_curr_task(rq);
da0c1e65 3597 if (queued) {
81a44c54
TG
3598 /*
3599 * We enqueue to tail when the priority of a task is
3600 * increased (user space view).
3601 */
3602 enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0);
3603 }
cb469845 3604
da7a735e 3605 check_class_changed(rq, p, prev_class, oldprio);
0122ec5b 3606 task_rq_unlock(rq, p, &flags);
b29739f9 3607
95e02ca9
TG
3608 rt_mutex_adjust_pi(p);
3609
1da177e4
LT
3610 return 0;
3611}
961ccddd 3612
7479f3c9
PZ
3613static int _sched_setscheduler(struct task_struct *p, int policy,
3614 const struct sched_param *param, bool check)
3615{
3616 struct sched_attr attr = {
3617 .sched_policy = policy,
3618 .sched_priority = param->sched_priority,
3619 .sched_nice = PRIO_TO_NICE(p->static_prio),
3620 };
3621
c13db6b1
SR
3622 /* Fixup the legacy SCHED_RESET_ON_FORK hack. */
3623 if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) {
7479f3c9
PZ
3624 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
3625 policy &= ~SCHED_RESET_ON_FORK;
3626 attr.sched_policy = policy;
3627 }
3628
3629 return __sched_setscheduler(p, &attr, check);
3630}
961ccddd
RR
3631/**
3632 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
3633 * @p: the task in question.
3634 * @policy: new policy.
3635 * @param: structure containing the new RT priority.
3636 *
e69f6186
YB
3637 * Return: 0 on success. An error code otherwise.
3638 *
961ccddd
RR
3639 * NOTE that the task may be already dead.
3640 */
3641int sched_setscheduler(struct task_struct *p, int policy,
fe7de49f 3642 const struct sched_param *param)
961ccddd 3643{
7479f3c9 3644 return _sched_setscheduler(p, policy, param, true);
961ccddd 3645}
1da177e4
LT
3646EXPORT_SYMBOL_GPL(sched_setscheduler);
3647
d50dde5a
DF
3648int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
3649{
3650 return __sched_setscheduler(p, attr, true);
3651}
3652EXPORT_SYMBOL_GPL(sched_setattr);
3653
961ccddd
RR
3654/**
3655 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
3656 * @p: the task in question.
3657 * @policy: new policy.
3658 * @param: structure containing the new RT priority.
3659 *
3660 * Just like sched_setscheduler, only don't bother checking if the
3661 * current context has permission. For example, this is needed in
3662 * stop_machine(): we create temporary high priority worker threads,
3663 * but our caller might not have that capability.
e69f6186
YB
3664 *
3665 * Return: 0 on success. An error code otherwise.
961ccddd
RR
3666 */
3667int sched_setscheduler_nocheck(struct task_struct *p, int policy,
fe7de49f 3668 const struct sched_param *param)
961ccddd 3669{
7479f3c9 3670 return _sched_setscheduler(p, policy, param, false);
961ccddd
RR
3671}
3672
95cdf3b7
IM
3673static int
3674do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
1da177e4 3675{
1da177e4
LT
3676 struct sched_param lparam;
3677 struct task_struct *p;
36c8b586 3678 int retval;
1da177e4
LT
3679
3680 if (!param || pid < 0)
3681 return -EINVAL;
3682 if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
3683 return -EFAULT;
5fe1d75f
ON
3684
3685 rcu_read_lock();
3686 retval = -ESRCH;
1da177e4 3687 p = find_process_by_pid(pid);
5fe1d75f
ON
3688 if (p != NULL)
3689 retval = sched_setscheduler(p, policy, &lparam);
3690 rcu_read_unlock();
36c8b586 3691
1da177e4
LT
3692 return retval;
3693}
3694
d50dde5a
DF
3695/*
3696 * Mimics kernel/events/core.c perf_copy_attr().
3697 */
3698static int sched_copy_attr(struct sched_attr __user *uattr,
3699 struct sched_attr *attr)
3700{
3701 u32 size;
3702 int ret;
3703
3704 if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0))
3705 return -EFAULT;
3706
3707 /*
3708 * zero the full structure, so that a short copy will be nice.
3709 */
3710 memset(attr, 0, sizeof(*attr));
3711
3712 ret = get_user(size, &uattr->size);
3713 if (ret)
3714 return ret;
3715
3716 if (size > PAGE_SIZE) /* silly large */
3717 goto err_size;
3718
3719 if (!size) /* abi compat */
3720 size = SCHED_ATTR_SIZE_VER0;
3721
3722 if (size < SCHED_ATTR_SIZE_VER0)
3723 goto err_size;
3724
3725 /*
3726 * If we're handed a bigger struct than we know of,
3727 * ensure all the unknown bits are 0 - i.e. new
3728 * user-space does not rely on any kernel feature
3729 * extensions we dont know about yet.
3730 */
3731 if (size > sizeof(*attr)) {
3732 unsigned char __user *addr;
3733 unsigned char __user *end;
3734 unsigned char val;
3735
3736 addr = (void __user *)uattr + sizeof(*attr);
3737 end = (void __user *)uattr + size;
3738
3739 for (; addr < end; addr++) {
3740 ret = get_user(val, addr);
3741 if (ret)
3742 return ret;
3743 if (val)
3744 goto err_size;
3745 }
3746 size = sizeof(*attr);
3747 }
3748
3749 ret = copy_from_user(attr, uattr, size);
3750 if (ret)
3751 return -EFAULT;
3752
3753 /*
3754 * XXX: do we want to be lenient like existing syscalls; or do we want
3755 * to be strict and return an error on out-of-bounds values?
3756 */
75e45d51 3757 attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE);
d50dde5a 3758
e78c7bca 3759 return 0;
d50dde5a
DF
3760
3761err_size:
3762 put_user(sizeof(*attr), &uattr->size);
e78c7bca 3763 return -E2BIG;
d50dde5a
DF
3764}
3765
1da177e4
LT
3766/**
3767 * sys_sched_setscheduler - set/change the scheduler policy and RT priority
3768 * @pid: the pid in question.
3769 * @policy: new policy.
3770 * @param: structure containing the new RT priority.
e69f6186
YB
3771 *
3772 * Return: 0 on success. An error code otherwise.
1da177e4 3773 */
5add95d4
HC
3774SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
3775 struct sched_param __user *, param)
1da177e4 3776{
c21761f1
JB
3777 /* negative values for policy are not valid */
3778 if (policy < 0)
3779 return -EINVAL;
3780
1da177e4
LT
3781 return do_sched_setscheduler(pid, policy, param);
3782}
3783
3784/**
3785 * sys_sched_setparam - set/change the RT priority of a thread
3786 * @pid: the pid in question.
3787 * @param: structure containing the new RT priority.
e69f6186
YB
3788 *
3789 * Return: 0 on success. An error code otherwise.
1da177e4 3790 */
5add95d4 3791SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
1da177e4 3792{
c13db6b1 3793 return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
1da177e4
LT
3794}
3795
d50dde5a
DF
3796/**
3797 * sys_sched_setattr - same as above, but with extended sched_attr
3798 * @pid: the pid in question.
5778fccf 3799 * @uattr: structure containing the extended parameters.
db66d756 3800 * @flags: for future extension.
d50dde5a 3801 */
6d35ab48
PZ
3802SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
3803 unsigned int, flags)
d50dde5a
DF
3804{
3805 struct sched_attr attr;
3806 struct task_struct *p;
3807 int retval;
3808
6d35ab48 3809 if (!uattr || pid < 0 || flags)
d50dde5a
DF
3810 return -EINVAL;
3811
143cf23d
MK
3812 retval = sched_copy_attr(uattr, &attr);
3813 if (retval)
3814 return retval;
d50dde5a 3815
b14ed2c2 3816 if ((int)attr.sched_policy < 0)
dbdb2275 3817 return -EINVAL;
d50dde5a
DF
3818
3819 rcu_read_lock();
3820 retval = -ESRCH;
3821 p = find_process_by_pid(pid);
3822 if (p != NULL)
3823 retval = sched_setattr(p, &attr);
3824 rcu_read_unlock();
3825
3826 return retval;
3827}
3828
1da177e4
LT
3829/**
3830 * sys_sched_getscheduler - get the policy (scheduling class) of a thread
3831 * @pid: the pid in question.
e69f6186
YB
3832 *
3833 * Return: On success, the policy of the thread. Otherwise, a negative error
3834 * code.
1da177e4 3835 */
5add95d4 3836SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
1da177e4 3837{
36c8b586 3838 struct task_struct *p;
3a5c359a 3839 int retval;
1da177e4
LT
3840
3841 if (pid < 0)
3a5c359a 3842 return -EINVAL;
1da177e4
LT
3843
3844 retval = -ESRCH;
5fe85be0 3845 rcu_read_lock();
1da177e4
LT
3846 p = find_process_by_pid(pid);
3847 if (p) {
3848 retval = security_task_getscheduler(p);
3849 if (!retval)
ca94c442
LP
3850 retval = p->policy
3851 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
1da177e4 3852 }
5fe85be0 3853 rcu_read_unlock();
1da177e4
LT
3854 return retval;
3855}
3856
3857/**
ca94c442 3858 * sys_sched_getparam - get the RT priority of a thread
1da177e4
LT
3859 * @pid: the pid in question.
3860 * @param: structure containing the RT priority.
e69f6186
YB
3861 *
3862 * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
3863 * code.
1da177e4 3864 */
5add95d4 3865SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
1da177e4 3866{
ce5f7f82 3867 struct sched_param lp = { .sched_priority = 0 };
36c8b586 3868 struct task_struct *p;
3a5c359a 3869 int retval;
1da177e4
LT
3870
3871 if (!param || pid < 0)
3a5c359a 3872 return -EINVAL;
1da177e4 3873
5fe85be0 3874 rcu_read_lock();
1da177e4
LT
3875 p = find_process_by_pid(pid);
3876 retval = -ESRCH;
3877 if (!p)
3878 goto out_unlock;
3879
3880 retval = security_task_getscheduler(p);
3881 if (retval)
3882 goto out_unlock;
3883
ce5f7f82
PZ
3884 if (task_has_rt_policy(p))
3885 lp.sched_priority = p->rt_priority;
5fe85be0 3886 rcu_read_unlock();
1da177e4
LT
3887
3888 /*
3889 * This one might sleep, we cannot do it with a spinlock held ...
3890 */
3891 retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
3892
1da177e4
LT
3893 return retval;
3894
3895out_unlock:
5fe85be0 3896 rcu_read_unlock();
1da177e4
LT
3897 return retval;
3898}
3899
d50dde5a
DF
3900static int sched_read_attr(struct sched_attr __user *uattr,
3901 struct sched_attr *attr,
3902 unsigned int usize)
3903{
3904 int ret;
3905
3906 if (!access_ok(VERIFY_WRITE, uattr, usize))
3907 return -EFAULT;
3908
3909 /*
3910 * If we're handed a smaller struct than we know of,
3911 * ensure all the unknown bits are 0 - i.e. old
3912 * user-space does not get uncomplete information.
3913 */
3914 if (usize < sizeof(*attr)) {
3915 unsigned char *addr;
3916 unsigned char *end;
3917
3918 addr = (void *)attr + usize;
3919 end = (void *)attr + sizeof(*attr);
3920
3921 for (; addr < end; addr++) {
3922 if (*addr)
22400674 3923 return -EFBIG;
d50dde5a
DF
3924 }
3925
3926 attr->size = usize;
3927 }
3928
4efbc454 3929 ret = copy_to_user(uattr, attr, attr->size);
d50dde5a
DF
3930 if (ret)
3931 return -EFAULT;
3932
22400674 3933 return 0;
d50dde5a
DF
3934}
3935
3936/**
aab03e05 3937 * sys_sched_getattr - similar to sched_getparam, but with sched_attr
d50dde5a 3938 * @pid: the pid in question.
5778fccf 3939 * @uattr: structure containing the extended parameters.
d50dde5a 3940 * @size: sizeof(attr) for fwd/bwd comp.
db66d756 3941 * @flags: for future extension.
d50dde5a 3942 */
6d35ab48
PZ
3943SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
3944 unsigned int, size, unsigned int, flags)
d50dde5a
DF
3945{
3946 struct sched_attr attr = {
3947 .size = sizeof(struct sched_attr),
3948 };
3949 struct task_struct *p;
3950 int retval;
3951
3952 if (!uattr || pid < 0 || size > PAGE_SIZE ||
6d35ab48 3953 size < SCHED_ATTR_SIZE_VER0 || flags)
d50dde5a
DF
3954 return -EINVAL;
3955
3956 rcu_read_lock();
3957 p = find_process_by_pid(pid);
3958 retval = -ESRCH;
3959 if (!p)
3960 goto out_unlock;
3961
3962 retval = security_task_getscheduler(p);
3963 if (retval)
3964 goto out_unlock;
3965
3966 attr.sched_policy = p->policy;
7479f3c9
PZ
3967 if (p->sched_reset_on_fork)
3968 attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK;
aab03e05
DF
3969 if (task_has_dl_policy(p))
3970 __getparam_dl(p, &attr);
3971 else if (task_has_rt_policy(p))
d50dde5a
DF
3972 attr.sched_priority = p->rt_priority;
3973 else
d0ea0268 3974 attr.sched_nice = task_nice(p);
d50dde5a
DF
3975
3976 rcu_read_unlock();
3977
3978 retval = sched_read_attr(uattr, &attr, size);
3979 return retval;
3980
3981out_unlock:
3982 rcu_read_unlock();
3983 return retval;
3984}
3985
96f874e2 3986long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
1da177e4 3987{
5a16f3d3 3988 cpumask_var_t cpus_allowed, new_mask;
36c8b586
IM
3989 struct task_struct *p;
3990 int retval;
1da177e4 3991
23f5d142 3992 rcu_read_lock();
1da177e4
LT
3993
3994 p = find_process_by_pid(pid);
3995 if (!p) {
23f5d142 3996 rcu_read_unlock();
1da177e4
LT
3997 return -ESRCH;
3998 }
3999
23f5d142 4000 /* Prevent p going away */
1da177e4 4001 get_task_struct(p);
23f5d142 4002 rcu_read_unlock();
1da177e4 4003
14a40ffc
TH
4004 if (p->flags & PF_NO_SETAFFINITY) {
4005 retval = -EINVAL;
4006 goto out_put_task;
4007 }
5a16f3d3
RR
4008 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
4009 retval = -ENOMEM;
4010 goto out_put_task;
4011 }
4012 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
4013 retval = -ENOMEM;
4014 goto out_free_cpus_allowed;
4015 }
1da177e4 4016 retval = -EPERM;
4c44aaaf
EB
4017 if (!check_same_owner(p)) {
4018 rcu_read_lock();
4019 if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
4020 rcu_read_unlock();
16303ab2 4021 goto out_free_new_mask;
4c44aaaf
EB
4022 }
4023 rcu_read_unlock();
4024 }
1da177e4 4025
b0ae1981 4026 retval = security_task_setscheduler(p);
e7834f8f 4027 if (retval)
16303ab2 4028 goto out_free_new_mask;
e7834f8f 4029
e4099a5e
PZ
4030
4031 cpuset_cpus_allowed(p, cpus_allowed);
4032 cpumask_and(new_mask, in_mask, cpus_allowed);
4033
332ac17e
DF
4034 /*
4035 * Since bandwidth control happens on root_domain basis,
4036 * if admission test is enabled, we only admit -deadline
4037 * tasks allowed to run on all the CPUs in the task's
4038 * root_domain.
4039 */
4040#ifdef CONFIG_SMP
f1e3a093
KT
4041 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
4042 rcu_read_lock();
4043 if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) {
332ac17e 4044 retval = -EBUSY;
f1e3a093 4045 rcu_read_unlock();
16303ab2 4046 goto out_free_new_mask;
332ac17e 4047 }
f1e3a093 4048 rcu_read_unlock();
332ac17e
DF
4049 }
4050#endif
49246274 4051again:
5a16f3d3 4052 retval = set_cpus_allowed_ptr(p, new_mask);
1da177e4 4053
8707d8b8 4054 if (!retval) {
5a16f3d3
RR
4055 cpuset_cpus_allowed(p, cpus_allowed);
4056 if (!cpumask_subset(new_mask, cpus_allowed)) {
8707d8b8
PM
4057 /*
4058 * We must have raced with a concurrent cpuset
4059 * update. Just reset the cpus_allowed to the
4060 * cpuset's cpus_allowed
4061 */
5a16f3d3 4062 cpumask_copy(new_mask, cpus_allowed);
8707d8b8
PM
4063 goto again;
4064 }
4065 }
16303ab2 4066out_free_new_mask:
5a16f3d3
RR
4067 free_cpumask_var(new_mask);
4068out_free_cpus_allowed:
4069 free_cpumask_var(cpus_allowed);
4070out_put_task:
1da177e4 4071 put_task_struct(p);
1da177e4
LT
4072 return retval;
4073}
4074
4075static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
96f874e2 4076 struct cpumask *new_mask)
1da177e4 4077{
96f874e2
RR
4078 if (len < cpumask_size())
4079 cpumask_clear(new_mask);
4080 else if (len > cpumask_size())
4081 len = cpumask_size();
4082
1da177e4
LT
4083 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
4084}
4085
4086/**
4087 * sys_sched_setaffinity - set the cpu affinity of a process
4088 * @pid: pid of the process
4089 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4090 * @user_mask_ptr: user-space pointer to the new cpu mask
e69f6186
YB
4091 *
4092 * Return: 0 on success. An error code otherwise.
1da177e4 4093 */
5add95d4
HC
4094SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
4095 unsigned long __user *, user_mask_ptr)
1da177e4 4096{
5a16f3d3 4097 cpumask_var_t new_mask;
1da177e4
LT
4098 int retval;
4099
5a16f3d3
RR
4100 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
4101 return -ENOMEM;
1da177e4 4102
5a16f3d3
RR
4103 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
4104 if (retval == 0)
4105 retval = sched_setaffinity(pid, new_mask);
4106 free_cpumask_var(new_mask);
4107 return retval;
1da177e4
LT
4108}
4109
96f874e2 4110long sched_getaffinity(pid_t pid, struct cpumask *mask)
1da177e4 4111{
36c8b586 4112 struct task_struct *p;
31605683 4113 unsigned long flags;
1da177e4 4114 int retval;
1da177e4 4115
23f5d142 4116 rcu_read_lock();
1da177e4
LT
4117
4118 retval = -ESRCH;
4119 p = find_process_by_pid(pid);
4120 if (!p)
4121 goto out_unlock;
4122
e7834f8f
DQ
4123 retval = security_task_getscheduler(p);
4124 if (retval)
4125 goto out_unlock;
4126
013fdb80 4127 raw_spin_lock_irqsave(&p->pi_lock, flags);
6acce3ef 4128 cpumask_and(mask, &p->cpus_allowed, cpu_active_mask);
013fdb80 4129 raw_spin_unlock_irqrestore(&p->pi_lock, flags);
1da177e4
LT
4130
4131out_unlock:
23f5d142 4132 rcu_read_unlock();
1da177e4 4133
9531b62f 4134 return retval;
1da177e4
LT
4135}
4136
4137/**
4138 * sys_sched_getaffinity - get the cpu affinity of a process
4139 * @pid: pid of the process
4140 * @len: length in bytes of the bitmask pointed to by user_mask_ptr
4141 * @user_mask_ptr: user-space pointer to hold the current cpu mask
e69f6186
YB
4142 *
4143 * Return: 0 on success. An error code otherwise.
1da177e4 4144 */
5add95d4
HC
4145SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
4146 unsigned long __user *, user_mask_ptr)
1da177e4
LT
4147{
4148 int ret;
f17c8607 4149 cpumask_var_t mask;
1da177e4 4150
84fba5ec 4151 if ((len * BITS_PER_BYTE) < nr_cpu_ids)
cd3d8031
KM
4152 return -EINVAL;
4153 if (len & (sizeof(unsigned long)-1))
1da177e4
LT
4154 return -EINVAL;
4155
f17c8607
RR
4156 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
4157 return -ENOMEM;
1da177e4 4158
f17c8607
RR
4159 ret = sched_getaffinity(pid, mask);
4160 if (ret == 0) {
8bc037fb 4161 size_t retlen = min_t(size_t, len, cpumask_size());
cd3d8031
KM
4162
4163 if (copy_to_user(user_mask_ptr, mask, retlen))
f17c8607
RR
4164 ret = -EFAULT;
4165 else
cd3d8031 4166 ret = retlen;
f17c8607
RR
4167 }
4168 free_cpumask_var(mask);
1da177e4 4169
f17c8607 4170 return ret;
1da177e4
LT
4171}
4172
4173/**
4174 * sys_sched_yield - yield the current processor to other threads.
4175 *
dd41f596
IM
4176 * This function yields the current CPU to other tasks. If there are no
4177 * other threads running on this CPU then this function will return.
e69f6186
YB
4178 *
4179 * Return: 0.
1da177e4 4180 */
5add95d4 4181SYSCALL_DEFINE0(sched_yield)
1da177e4 4182{
70b97a7f 4183 struct rq *rq = this_rq_lock();
1da177e4 4184
2d72376b 4185 schedstat_inc(rq, yld_count);
4530d7ab 4186 current->sched_class->yield_task(rq);
1da177e4
LT
4187
4188 /*
4189 * Since we are going to call schedule() anyway, there's
4190 * no need to preempt or enable interrupts:
4191 */
4192 __release(rq->lock);
8a25d5de 4193 spin_release(&rq->lock.dep_map, 1, _THIS_IP_);
9828ea9d 4194 do_raw_spin_unlock(&rq->lock);
ba74c144 4195 sched_preempt_enable_no_resched();
1da177e4
LT
4196
4197 schedule();
4198
4199 return 0;
4200}
4201
02b67cc3 4202int __sched _cond_resched(void)
1da177e4 4203{
d86ee480 4204 if (should_resched()) {
a18b5d01 4205 preempt_schedule_common();
1da177e4
LT
4206 return 1;
4207 }
4208 return 0;
4209}
02b67cc3 4210EXPORT_SYMBOL(_cond_resched);
1da177e4
LT
4211
4212/*
613afbf8 4213 * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
1da177e4
LT
4214 * call schedule, and on return reacquire the lock.
4215 *
41a2d6cf 4216 * This works OK both with and without CONFIG_PREEMPT. We do strange low-level
1da177e4
LT
4217 * operations here to prevent schedule() from being called twice (once via
4218 * spin_unlock(), once by hand).
4219 */
613afbf8 4220int __cond_resched_lock(spinlock_t *lock)
1da177e4 4221{
d86ee480 4222 int resched = should_resched();
6df3cecb
JK
4223 int ret = 0;
4224
f607c668
PZ
4225 lockdep_assert_held(lock);
4226
4a81e832 4227 if (spin_needbreak(lock) || resched) {
1da177e4 4228 spin_unlock(lock);
d86ee480 4229 if (resched)
a18b5d01 4230 preempt_schedule_common();
95c354fe
NP
4231 else
4232 cpu_relax();
6df3cecb 4233 ret = 1;
1da177e4 4234 spin_lock(lock);
1da177e4 4235 }
6df3cecb 4236 return ret;
1da177e4 4237}
613afbf8 4238EXPORT_SYMBOL(__cond_resched_lock);
1da177e4 4239
613afbf8 4240int __sched __cond_resched_softirq(void)
1da177e4
LT
4241{
4242 BUG_ON(!in_softirq());
4243
d86ee480 4244 if (should_resched()) {
98d82567 4245 local_bh_enable();
a18b5d01 4246 preempt_schedule_common();
1da177e4
LT
4247 local_bh_disable();
4248 return 1;
4249 }
4250 return 0;
4251}
613afbf8 4252EXPORT_SYMBOL(__cond_resched_softirq);
1da177e4 4253
1da177e4
LT
4254/**
4255 * yield - yield the current processor to other threads.
4256 *
8e3fabfd
PZ
4257 * Do not ever use this function, there's a 99% chance you're doing it wrong.
4258 *
4259 * The scheduler is at all times free to pick the calling task as the most
4260 * eligible task to run, if removing the yield() call from your code breaks
4261 * it, its already broken.
4262 *
4263 * Typical broken usage is:
4264 *
4265 * while (!event)
4266 * yield();
4267 *
4268 * where one assumes that yield() will let 'the other' process run that will
4269 * make event true. If the current task is a SCHED_FIFO task that will never
4270 * happen. Never use yield() as a progress guarantee!!
4271 *
4272 * If you want to use yield() to wait for something, use wait_event().
4273 * If you want to use yield() to be 'nice' for others, use cond_resched().
4274 * If you still want to use yield(), do not!
1da177e4
LT
4275 */
4276void __sched yield(void)
4277{
4278 set_current_state(TASK_RUNNING);
4279 sys_sched_yield();
4280}
1da177e4
LT
4281EXPORT_SYMBOL(yield);
4282
d95f4122
MG
4283/**
4284 * yield_to - yield the current processor to another thread in
4285 * your thread group, or accelerate that thread toward the
4286 * processor it's on.
16addf95
RD
4287 * @p: target task
4288 * @preempt: whether task preemption is allowed or not
d95f4122
MG
4289 *
4290 * It's the caller's job to ensure that the target task struct
4291 * can't go away on us before we can do any checks.
4292 *
e69f6186 4293 * Return:
7b270f60
PZ
4294 * true (>0) if we indeed boosted the target task.
4295 * false (0) if we failed to boost the target.
4296 * -ESRCH if there's no task to yield to.
d95f4122 4297 */
fa93384f 4298int __sched yield_to(struct task_struct *p, bool preempt)
d95f4122
MG
4299{
4300 struct task_struct *curr = current;
4301 struct rq *rq, *p_rq;
4302 unsigned long flags;
c3c18640 4303 int yielded = 0;
d95f4122
MG
4304
4305 local_irq_save(flags);
4306 rq = this_rq();
4307
4308again:
4309 p_rq = task_rq(p);
7b270f60
PZ
4310 /*
4311 * If we're the only runnable task on the rq and target rq also
4312 * has only one task, there's absolutely no point in yielding.
4313 */
4314 if (rq->nr_running == 1 && p_rq->nr_running == 1) {
4315 yielded = -ESRCH;
4316 goto out_irq;
4317 }
4318
d95f4122 4319 double_rq_lock(rq, p_rq);
39e24d8f 4320 if (task_rq(p) != p_rq) {
d95f4122
MG
4321 double_rq_unlock(rq, p_rq);
4322 goto again;
4323 }
4324
4325 if (!curr->sched_class->yield_to_task)
7b270f60 4326 goto out_unlock;
d95f4122
MG
4327
4328 if (curr->sched_class != p->sched_class)
7b270f60 4329 goto out_unlock;
d95f4122
MG
4330
4331 if (task_running(p_rq, p) || p->state)
7b270f60 4332 goto out_unlock;
d95f4122
MG
4333
4334 yielded = curr->sched_class->yield_to_task(rq, p, preempt);
6d1cafd8 4335 if (yielded) {
d95f4122 4336 schedstat_inc(rq, yld_count);
6d1cafd8
VP
4337 /*
4338 * Make p's CPU reschedule; pick_next_entity takes care of
4339 * fairness.
4340 */
4341 if (preempt && rq != p_rq)
8875125e 4342 resched_curr(p_rq);
6d1cafd8 4343 }
d95f4122 4344
7b270f60 4345out_unlock:
d95f4122 4346 double_rq_unlock(rq, p_rq);
7b270f60 4347out_irq:
d95f4122
MG
4348 local_irq_restore(flags);
4349
7b270f60 4350 if (yielded > 0)
d95f4122
MG
4351 schedule();
4352
4353 return yielded;
4354}
4355EXPORT_SYMBOL_GPL(yield_to);
4356
1da177e4 4357/*
41a2d6cf 4358 * This task is about to go to sleep on IO. Increment rq->nr_iowait so
1da177e4 4359 * that process accounting knows that this is a task in IO wait state.
1da177e4 4360 */
1da177e4
LT
4361long __sched io_schedule_timeout(long timeout)
4362{
9cff8ade
N
4363 int old_iowait = current->in_iowait;
4364 struct rq *rq;
1da177e4
LT
4365 long ret;
4366
9cff8ade
N
4367 current->in_iowait = 1;
4368 if (old_iowait)
4369 blk_schedule_flush_plug(current);
4370 else
4371 blk_flush_plug(current);
4372
0ff92245 4373 delayacct_blkio_start();
9cff8ade 4374 rq = raw_rq();
1da177e4
LT
4375 atomic_inc(&rq->nr_iowait);
4376 ret = schedule_timeout(timeout);
9cff8ade 4377 current->in_iowait = old_iowait;
1da177e4 4378 atomic_dec(&rq->nr_iowait);
0ff92245 4379 delayacct_blkio_end();
9cff8ade 4380
1da177e4
LT
4381 return ret;
4382}
9cff8ade 4383EXPORT_SYMBOL(io_schedule_timeout);
1da177e4
LT
4384
4385/**
4386 * sys_sched_get_priority_max - return maximum RT priority.
4387 * @policy: scheduling class.
4388 *
e69f6186
YB
4389 * Return: On success, this syscall returns the maximum
4390 * rt_priority that can be used by a given scheduling class.
4391 * On failure, a negative error code is returned.
1da177e4 4392 */
5add95d4 4393SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
1da177e4
LT
4394{
4395 int ret = -EINVAL;
4396
4397 switch (policy) {
4398 case SCHED_FIFO:
4399 case SCHED_RR:
4400 ret = MAX_USER_RT_PRIO-1;
4401 break;
aab03e05 4402 case SCHED_DEADLINE:
1da177e4 4403 case SCHED_NORMAL:
b0a9499c 4404 case SCHED_BATCH:
dd41f596 4405 case SCHED_IDLE:
1da177e4
LT
4406 ret = 0;
4407 break;
4408 }
4409 return ret;
4410}
4411
4412/**
4413 * sys_sched_get_priority_min - return minimum RT priority.
4414 * @policy: scheduling class.
4415 *
e69f6186
YB
4416 * Return: On success, this syscall returns the minimum
4417 * rt_priority that can be used by a given scheduling class.
4418 * On failure, a negative error code is returned.
1da177e4 4419 */
5add95d4 4420SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
1da177e4
LT
4421{
4422 int ret = -EINVAL;
4423
4424 switch (policy) {
4425 case SCHED_FIFO:
4426 case SCHED_RR:
4427 ret = 1;
4428 break;
aab03e05 4429 case SCHED_DEADLINE:
1da177e4 4430 case SCHED_NORMAL:
b0a9499c 4431 case SCHED_BATCH:
dd41f596 4432 case SCHED_IDLE:
1da177e4
LT
4433 ret = 0;
4434 }
4435 return ret;
4436}
4437
4438/**
4439 * sys_sched_rr_get_interval - return the default timeslice of a process.
4440 * @pid: pid of the process.
4441 * @interval: userspace pointer to the timeslice value.
4442 *
4443 * this syscall writes the default timeslice value of a given process
4444 * into the user-space timespec buffer. A value of '0' means infinity.
e69f6186
YB
4445 *
4446 * Return: On success, 0 and the timeslice is in @interval. Otherwise,
4447 * an error code.
1da177e4 4448 */
17da2bd9 4449SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
754fe8d2 4450 struct timespec __user *, interval)
1da177e4 4451{
36c8b586 4452 struct task_struct *p;
a4ec24b4 4453 unsigned int time_slice;
dba091b9
TG
4454 unsigned long flags;
4455 struct rq *rq;
3a5c359a 4456 int retval;
1da177e4 4457 struct timespec t;
1da177e4
LT
4458
4459 if (pid < 0)
3a5c359a 4460 return -EINVAL;
1da177e4
LT
4461
4462 retval = -ESRCH;
1a551ae7 4463 rcu_read_lock();
1da177e4
LT
4464 p = find_process_by_pid(pid);
4465 if (!p)
4466 goto out_unlock;
4467
4468 retval = security_task_getscheduler(p);
4469 if (retval)
4470 goto out_unlock;
4471
dba091b9 4472 rq = task_rq_lock(p, &flags);
a57beec5
PZ
4473 time_slice = 0;
4474 if (p->sched_class->get_rr_interval)
4475 time_slice = p->sched_class->get_rr_interval(rq, p);
0122ec5b 4476 task_rq_unlock(rq, p, &flags);
a4ec24b4 4477
1a551ae7 4478 rcu_read_unlock();
a4ec24b4 4479 jiffies_to_timespec(time_slice, &t);
1da177e4 4480 retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
1da177e4 4481 return retval;
3a5c359a 4482
1da177e4 4483out_unlock:
1a551ae7 4484 rcu_read_unlock();
1da177e4
LT
4485 return retval;
4486}
4487
7c731e0a 4488static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
36c8b586 4489
82a1fcb9 4490void sched_show_task(struct task_struct *p)
1da177e4 4491{
1da177e4 4492 unsigned long free = 0;
4e79752c 4493 int ppid;
1f8a7633 4494 unsigned long state = p->state;
1da177e4 4495
1f8a7633
TH
4496 if (state)
4497 state = __ffs(state) + 1;
28d0686c 4498 printk(KERN_INFO "%-15.15s %c", p->comm,
2ed6e34f 4499 state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?');
4bd77321 4500#if BITS_PER_LONG == 32
1da177e4 4501 if (state == TASK_RUNNING)
3df0fc5b 4502 printk(KERN_CONT " running ");
1da177e4 4503 else
3df0fc5b 4504 printk(KERN_CONT " %08lx ", thread_saved_pc(p));
1da177e4
LT
4505#else
4506 if (state == TASK_RUNNING)
3df0fc5b 4507 printk(KERN_CONT " running task ");
1da177e4 4508 else
3df0fc5b 4509 printk(KERN_CONT " %016lx ", thread_saved_pc(p));
1da177e4
LT
4510#endif
4511#ifdef CONFIG_DEBUG_STACK_USAGE
7c9f8861 4512 free = stack_not_used(p);
1da177e4 4513#endif
a90e984c 4514 ppid = 0;
4e79752c 4515 rcu_read_lock();
a90e984c
ON
4516 if (pid_alive(p))
4517 ppid = task_pid_nr(rcu_dereference(p->real_parent));
4e79752c 4518 rcu_read_unlock();
3df0fc5b 4519 printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
4e79752c 4520 task_pid_nr(p), ppid,
aa47b7e0 4521 (unsigned long)task_thread_info(p)->flags);
1da177e4 4522
3d1cb205 4523 print_worker_info(KERN_INFO, p);
5fb5e6de 4524 show_stack(p, NULL);
1da177e4
LT
4525}
4526
e59e2ae2 4527void show_state_filter(unsigned long state_filter)
1da177e4 4528{
36c8b586 4529 struct task_struct *g, *p;
1da177e4 4530
4bd77321 4531#if BITS_PER_LONG == 32
3df0fc5b
PZ
4532 printk(KERN_INFO
4533 " task PC stack pid father\n");
1da177e4 4534#else
3df0fc5b
PZ
4535 printk(KERN_INFO
4536 " task PC stack pid father\n");
1da177e4 4537#endif
510f5acc 4538 rcu_read_lock();
5d07f420 4539 for_each_process_thread(g, p) {
1da177e4
LT
4540 /*
4541 * reset the NMI-timeout, listing all files on a slow
25985edc 4542 * console might take a lot of time:
1da177e4
LT
4543 */
4544 touch_nmi_watchdog();
39bc89fd 4545 if (!state_filter || (p->state & state_filter))
82a1fcb9 4546 sched_show_task(p);
5d07f420 4547 }
1da177e4 4548
04c9167f
JF
4549 touch_all_softlockup_watchdogs();
4550
dd41f596
IM
4551#ifdef CONFIG_SCHED_DEBUG
4552 sysrq_sched_debug_show();
4553#endif
510f5acc 4554 rcu_read_unlock();
e59e2ae2
IM
4555 /*
4556 * Only show locks if all tasks are dumped:
4557 */
93335a21 4558 if (!state_filter)
e59e2ae2 4559 debug_show_all_locks();
1da177e4
LT
4560}
4561
0db0628d 4562void init_idle_bootup_task(struct task_struct *idle)
1df21055 4563{
dd41f596 4564 idle->sched_class = &idle_sched_class;
1df21055
IM
4565}
4566
f340c0d1
IM
4567/**
4568 * init_idle - set up an idle thread for a given CPU
4569 * @idle: task in question
4570 * @cpu: cpu the idle task belongs to
4571 *
4572 * NOTE: this function does not set the idle thread's NEED_RESCHED
4573 * flag, to make booting more robust.
4574 */
0db0628d 4575void init_idle(struct task_struct *idle, int cpu)
1da177e4 4576{
70b97a7f 4577 struct rq *rq = cpu_rq(cpu);
1da177e4
LT
4578 unsigned long flags;
4579
05fa785c 4580 raw_spin_lock_irqsave(&rq->lock, flags);
5cbd54ef 4581
5e1576ed 4582 __sched_fork(0, idle);
06b83b5f 4583 idle->state = TASK_RUNNING;
dd41f596
IM
4584 idle->se.exec_start = sched_clock();
4585
1e1b6c51 4586 do_set_cpus_allowed(idle, cpumask_of(cpu));
6506cf6c
PZ
4587 /*
4588 * We're having a chicken and egg problem, even though we are
4589 * holding rq->lock, the cpu isn't yet set to this cpu so the
4590 * lockdep check in task_group() will fail.
4591 *
4592 * Similar case to sched_fork(). / Alternatively we could
4593 * use task_rq_lock() here and obtain the other rq->lock.
4594 *
4595 * Silence PROVE_RCU
4596 */
4597 rcu_read_lock();
dd41f596 4598 __set_task_cpu(idle, cpu);
6506cf6c 4599 rcu_read_unlock();
1da177e4 4600
1da177e4 4601 rq->curr = rq->idle = idle;
da0c1e65 4602 idle->on_rq = TASK_ON_RQ_QUEUED;
3ca7a440
PZ
4603#if defined(CONFIG_SMP)
4604 idle->on_cpu = 1;
4866cde0 4605#endif
05fa785c 4606 raw_spin_unlock_irqrestore(&rq->lock, flags);
1da177e4
LT
4607
4608 /* Set the preempt count _outside_ the spinlocks! */
01028747 4609 init_idle_preempt_count(idle, cpu);
55cd5340 4610
dd41f596
IM
4611 /*
4612 * The idle tasks have their own, simple scheduling class:
4613 */
4614 idle->sched_class = &idle_sched_class;
868baf07 4615 ftrace_graph_init_idle_task(idle, cpu);
45eacc69 4616 vtime_init_idle(idle, cpu);
f1c6f1a7
CE
4617#if defined(CONFIG_SMP)
4618 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
4619#endif
19978ca6
IM
4620}
4621
f82f8042
JL
4622int cpuset_cpumask_can_shrink(const struct cpumask *cur,
4623 const struct cpumask *trial)
4624{
4625 int ret = 1, trial_cpus;
4626 struct dl_bw *cur_dl_b;
4627 unsigned long flags;
4628
bb2bc55a
MG
4629 if (!cpumask_weight(cur))
4630 return ret;
4631
75e23e49 4632 rcu_read_lock_sched();
f82f8042
JL
4633 cur_dl_b = dl_bw_of(cpumask_any(cur));
4634 trial_cpus = cpumask_weight(trial);
4635
4636 raw_spin_lock_irqsave(&cur_dl_b->lock, flags);
4637 if (cur_dl_b->bw != -1 &&
4638 cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw)
4639 ret = 0;
4640 raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags);
75e23e49 4641 rcu_read_unlock_sched();
f82f8042
JL
4642
4643 return ret;
4644}
4645
7f51412a
JL
4646int task_can_attach(struct task_struct *p,
4647 const struct cpumask *cs_cpus_allowed)
4648{
4649 int ret = 0;
4650
4651 /*
4652 * Kthreads which disallow setaffinity shouldn't be moved
4653 * to a new cpuset; we don't want to change their cpu
4654 * affinity and isolating such threads by their set of
4655 * allowed nodes is unnecessary. Thus, cpusets are not
4656 * applicable for such threads. This prevents checking for
4657 * success of set_cpus_allowed_ptr() on all attached tasks
4658 * before cpus_allowed may be changed.
4659 */
4660 if (p->flags & PF_NO_SETAFFINITY) {
4661 ret = -EINVAL;
4662 goto out;
4663 }
4664
4665#ifdef CONFIG_SMP
4666 if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
4667 cs_cpus_allowed)) {
4668 unsigned int dest_cpu = cpumask_any_and(cpu_active_mask,
4669 cs_cpus_allowed);
75e23e49 4670 struct dl_bw *dl_b;
7f51412a
JL
4671 bool overflow;
4672 int cpus;
4673 unsigned long flags;
4674
75e23e49
JL
4675 rcu_read_lock_sched();
4676 dl_b = dl_bw_of(dest_cpu);
7f51412a
JL
4677 raw_spin_lock_irqsave(&dl_b->lock, flags);
4678 cpus = dl_bw_cpus(dest_cpu);
4679 overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw);
4680 if (overflow)
4681 ret = -EBUSY;
4682 else {
4683 /*
4684 * We reserve space for this task in the destination
4685 * root_domain, as we can't fail after this point.
4686 * We will free resources in the source root_domain
4687 * later on (see set_cpus_allowed_dl()).
4688 */
4689 __dl_add(dl_b, p->dl.dl_bw);
4690 }
4691 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
75e23e49 4692 rcu_read_unlock_sched();
7f51412a
JL
4693
4694 }
4695#endif
4696out:
4697 return ret;
4698}
4699
1da177e4 4700#ifdef CONFIG_SMP
a15b12ac
KT
4701/*
4702 * move_queued_task - move a queued task to new rq.
4703 *
4704 * Returns (locked) new rq. Old rq's lock is released.
4705 */
4706static struct rq *move_queued_task(struct task_struct *p, int new_cpu)
4707{
4708 struct rq *rq = task_rq(p);
4709
4710 lockdep_assert_held(&rq->lock);
4711
4712 dequeue_task(rq, p, 0);
4713 p->on_rq = TASK_ON_RQ_MIGRATING;
4714 set_task_cpu(p, new_cpu);
4715 raw_spin_unlock(&rq->lock);
4716
4717 rq = cpu_rq(new_cpu);
4718
4719 raw_spin_lock(&rq->lock);
4720 BUG_ON(task_cpu(p) != new_cpu);
4721 p->on_rq = TASK_ON_RQ_QUEUED;
4722 enqueue_task(rq, p, 0);
4723 check_preempt_curr(rq, p, 0);
4724
4725 return rq;
4726}
4727
1e1b6c51
KM
4728void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
4729{
1b537c7d 4730 if (p->sched_class->set_cpus_allowed)
1e1b6c51 4731 p->sched_class->set_cpus_allowed(p, new_mask);
4939602a
PZ
4732
4733 cpumask_copy(&p->cpus_allowed, new_mask);
29baa747 4734 p->nr_cpus_allowed = cpumask_weight(new_mask);
1e1b6c51
KM
4735}
4736
1da177e4
LT
4737/*
4738 * This is how migration works:
4739 *
969c7921
TH
4740 * 1) we invoke migration_cpu_stop() on the target CPU using
4741 * stop_one_cpu().
4742 * 2) stopper starts to run (implicitly forcing the migrated thread
4743 * off the CPU)
4744 * 3) it checks whether the migrated task is still in the wrong runqueue.
4745 * 4) if it's in the wrong runqueue then the migration thread removes
1da177e4 4746 * it and puts it into the right queue.
969c7921
TH
4747 * 5) stopper completes and stop_one_cpu() returns and the migration
4748 * is done.
1da177e4
LT
4749 */
4750
4751/*
4752 * Change a given task's CPU affinity. Migrate the thread to a
4753 * proper CPU and schedule it away if the CPU it's executing on
4754 * is removed from the allowed bitmask.
4755 *
4756 * NOTE: the caller must have a valid reference to the task, the
41a2d6cf 4757 * task must not exit() & deallocate itself prematurely. The
1da177e4
LT
4758 * call is not atomic; no spinlocks may be held.
4759 */
96f874e2 4760int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
1da177e4
LT
4761{
4762 unsigned long flags;
70b97a7f 4763 struct rq *rq;
969c7921 4764 unsigned int dest_cpu;
48f24c4d 4765 int ret = 0;
1da177e4
LT
4766
4767 rq = task_rq_lock(p, &flags);
e2912009 4768
db44fc01
YZ
4769 if (cpumask_equal(&p->cpus_allowed, new_mask))
4770 goto out;
4771
6ad4c188 4772 if (!cpumask_intersects(new_mask, cpu_active_mask)) {
1da177e4
LT
4773 ret = -EINVAL;
4774 goto out;
4775 }
4776
1e1b6c51 4777 do_set_cpus_allowed(p, new_mask);
73fe6aae 4778
1da177e4 4779 /* Can the task run on the task's current CPU? If so, we're done */
96f874e2 4780 if (cpumask_test_cpu(task_cpu(p), new_mask))
1da177e4
LT
4781 goto out;
4782
969c7921 4783 dest_cpu = cpumask_any_and(cpu_active_mask, new_mask);
a15b12ac 4784 if (task_running(rq, p) || p->state == TASK_WAKING) {
969c7921 4785 struct migration_arg arg = { p, dest_cpu };
1da177e4 4786 /* Need help from migration thread: drop lock and wait. */
0122ec5b 4787 task_rq_unlock(rq, p, &flags);
969c7921 4788 stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
1da177e4
LT
4789 tlb_migrate_finish(p->mm);
4790 return 0;
a15b12ac
KT
4791 } else if (task_on_rq_queued(p))
4792 rq = move_queued_task(p, dest_cpu);
1da177e4 4793out:
0122ec5b 4794 task_rq_unlock(rq, p, &flags);
48f24c4d 4795
1da177e4
LT
4796 return ret;
4797}
cd8ba7cd 4798EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
1da177e4
LT
4799
4800/*
41a2d6cf 4801 * Move (not current) task off this cpu, onto dest cpu. We're doing
1da177e4
LT
4802 * this because either it can't run here any more (set_cpus_allowed()
4803 * away from this CPU, or CPU going down), or because we're
4804 * attempting to rebalance this task on exec (sched_exec).
4805 *
4806 * So we race with normal scheduler movements, but that's OK, as long
4807 * as the task is no longer on this CPU.
efc30814
KK
4808 *
4809 * Returns non-zero if task was successfully migrated.
1da177e4 4810 */
efc30814 4811static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
1da177e4 4812{
a1e01829 4813 struct rq *rq;
e2912009 4814 int ret = 0;
1da177e4 4815
e761b772 4816 if (unlikely(!cpu_active(dest_cpu)))
efc30814 4817 return ret;
1da177e4 4818
a1e01829 4819 rq = cpu_rq(src_cpu);
1da177e4 4820
0122ec5b 4821 raw_spin_lock(&p->pi_lock);
a1e01829 4822 raw_spin_lock(&rq->lock);
1da177e4
LT
4823 /* Already moved. */
4824 if (task_cpu(p) != src_cpu)
b1e38734 4825 goto done;
a1e01829 4826
1da177e4 4827 /* Affinity changed (again). */
fa17b507 4828 if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
b1e38734 4829 goto fail;
1da177e4 4830
e2912009
PZ
4831 /*
4832 * If we're not on a rq, the next wake-up will ensure we're
4833 * placed properly.
4834 */
a15b12ac
KT
4835 if (task_on_rq_queued(p))
4836 rq = move_queued_task(p, dest_cpu);
b1e38734 4837done:
efc30814 4838 ret = 1;
b1e38734 4839fail:
a1e01829 4840 raw_spin_unlock(&rq->lock);
0122ec5b 4841 raw_spin_unlock(&p->pi_lock);
efc30814 4842 return ret;
1da177e4
LT
4843}
4844
e6628d5b
MG
4845#ifdef CONFIG_NUMA_BALANCING
4846/* Migrate current task p to target_cpu */
4847int migrate_task_to(struct task_struct *p, int target_cpu)
4848{
4849 struct migration_arg arg = { p, target_cpu };
4850 int curr_cpu = task_cpu(p);
4851
4852 if (curr_cpu == target_cpu)
4853 return 0;
4854
4855 if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p)))
4856 return -EINVAL;
4857
4858 /* TODO: This is not properly updating schedstats */
4859
286549dc 4860 trace_sched_move_numa(p, curr_cpu, target_cpu);
e6628d5b
MG
4861 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg);
4862}
0ec8aa00
PZ
4863
4864/*
4865 * Requeue a task on a given node and accurately track the number of NUMA
4866 * tasks on the runqueues
4867 */
4868void sched_setnuma(struct task_struct *p, int nid)
4869{
4870 struct rq *rq;
4871 unsigned long flags;
da0c1e65 4872 bool queued, running;
0ec8aa00
PZ
4873
4874 rq = task_rq_lock(p, &flags);
da0c1e65 4875 queued = task_on_rq_queued(p);
0ec8aa00
PZ
4876 running = task_current(rq, p);
4877
da0c1e65 4878 if (queued)
0ec8aa00
PZ
4879 dequeue_task(rq, p, 0);
4880 if (running)
f3cd1c4e 4881 put_prev_task(rq, p);
0ec8aa00
PZ
4882
4883 p->numa_preferred_nid = nid;
0ec8aa00
PZ
4884
4885 if (running)
4886 p->sched_class->set_curr_task(rq);
da0c1e65 4887 if (queued)
0ec8aa00
PZ
4888 enqueue_task(rq, p, 0);
4889 task_rq_unlock(rq, p, &flags);
4890}
e6628d5b
MG
4891#endif
4892
1da177e4 4893/*
969c7921
TH
4894 * migration_cpu_stop - this will be executed by a highprio stopper thread
4895 * and performs thread migration by bumping thread off CPU then
4896 * 'pushing' onto another runqueue.
1da177e4 4897 */
969c7921 4898static int migration_cpu_stop(void *data)
1da177e4 4899{
969c7921 4900 struct migration_arg *arg = data;
f7b4cddc 4901
969c7921
TH
4902 /*
4903 * The original target cpu might have gone down and we might
4904 * be on another cpu but it doesn't matter.
4905 */
f7b4cddc 4906 local_irq_disable();
5cd038f5
LJ
4907 /*
4908 * We need to explicitly wake pending tasks before running
4909 * __migrate_task() such that we will not miss enforcing cpus_allowed
4910 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
4911 */
4912 sched_ttwu_pending();
969c7921 4913 __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu);
f7b4cddc 4914 local_irq_enable();
1da177e4 4915 return 0;
f7b4cddc
ON
4916}
4917
1da177e4 4918#ifdef CONFIG_HOTPLUG_CPU
48c5ccae 4919
054b9108 4920/*
48c5ccae
PZ
4921 * Ensures that the idle task is using init_mm right before its cpu goes
4922 * offline.
054b9108 4923 */
48c5ccae 4924void idle_task_exit(void)
1da177e4 4925{
48c5ccae 4926 struct mm_struct *mm = current->active_mm;
e76bd8d9 4927
48c5ccae 4928 BUG_ON(cpu_online(smp_processor_id()));
e76bd8d9 4929
a53efe5f 4930 if (mm != &init_mm) {
48c5ccae 4931 switch_mm(mm, &init_mm, current);
a53efe5f
MS
4932 finish_arch_post_lock_switch();
4933 }
48c5ccae 4934 mmdrop(mm);
1da177e4
LT
4935}
4936
4937/*
5d180232
PZ
4938 * Since this CPU is going 'away' for a while, fold any nr_active delta
4939 * we might have. Assumes we're called after migrate_tasks() so that the
4940 * nr_active count is stable.
4941 *
4942 * Also see the comment "Global load-average calculations".
1da177e4 4943 */
5d180232 4944static void calc_load_migrate(struct rq *rq)
1da177e4 4945{
5d180232
PZ
4946 long delta = calc_load_fold_active(rq);
4947 if (delta)
4948 atomic_long_add(delta, &calc_load_tasks);
1da177e4
LT
4949}
4950
3f1d2a31
PZ
4951static void put_prev_task_fake(struct rq *rq, struct task_struct *prev)
4952{
4953}
4954
4955static const struct sched_class fake_sched_class = {
4956 .put_prev_task = put_prev_task_fake,
4957};
4958
4959static struct task_struct fake_task = {
4960 /*
4961 * Avoid pull_{rt,dl}_task()
4962 */
4963 .prio = MAX_PRIO + 1,
4964 .sched_class = &fake_sched_class,
4965};
4966
48f24c4d 4967/*
48c5ccae
PZ
4968 * Migrate all tasks from the rq, sleeping tasks will be migrated by
4969 * try_to_wake_up()->select_task_rq().
4970 *
4971 * Called with rq->lock held even though we'er in stop_machine() and
4972 * there's no concurrency possible, we hold the required locks anyway
4973 * because of lock validation efforts.
1da177e4 4974 */
48c5ccae 4975static void migrate_tasks(unsigned int dead_cpu)
1da177e4 4976{
70b97a7f 4977 struct rq *rq = cpu_rq(dead_cpu);
48c5ccae
PZ
4978 struct task_struct *next, *stop = rq->stop;
4979 int dest_cpu;
1da177e4
LT
4980
4981 /*
48c5ccae
PZ
4982 * Fudge the rq selection such that the below task selection loop
4983 * doesn't get stuck on the currently eligible stop task.
4984 *
4985 * We're currently inside stop_machine() and the rq is either stuck
4986 * in the stop_machine_cpu_stop() loop, or we're executing this code,
4987 * either way we should never end up calling schedule() until we're
4988 * done here.
1da177e4 4989 */
48c5ccae 4990 rq->stop = NULL;
48f24c4d 4991
77bd3970
FW
4992 /*
4993 * put_prev_task() and pick_next_task() sched
4994 * class method both need to have an up-to-date
4995 * value of rq->clock[_task]
4996 */
4997 update_rq_clock(rq);
4998
dd41f596 4999 for ( ; ; ) {
48c5ccae
PZ
5000 /*
5001 * There's this thread running, bail when that's the only
5002 * remaining thread.
5003 */
5004 if (rq->nr_running == 1)
dd41f596 5005 break;
48c5ccae 5006
3f1d2a31 5007 next = pick_next_task(rq, &fake_task);
48c5ccae 5008 BUG_ON(!next);
79c53799 5009 next->sched_class->put_prev_task(rq, next);
e692ab53 5010
48c5ccae
PZ
5011 /* Find suitable destination for @next, with force if needed. */
5012 dest_cpu = select_fallback_rq(dead_cpu, next);
5013 raw_spin_unlock(&rq->lock);
5014
5015 __migrate_task(next, dead_cpu, dest_cpu);
5016
5017 raw_spin_lock(&rq->lock);
1da177e4 5018 }
dce48a84 5019
48c5ccae 5020 rq->stop = stop;
dce48a84 5021}
48c5ccae 5022
1da177e4
LT
5023#endif /* CONFIG_HOTPLUG_CPU */
5024
e692ab53
NP
5025#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
5026
5027static struct ctl_table sd_ctl_dir[] = {
e0361851
AD
5028 {
5029 .procname = "sched_domain",
c57baf1e 5030 .mode = 0555,
e0361851 5031 },
56992309 5032 {}
e692ab53
NP
5033};
5034
5035static struct ctl_table sd_ctl_root[] = {
e0361851
AD
5036 {
5037 .procname = "kernel",
c57baf1e 5038 .mode = 0555,
e0361851
AD
5039 .child = sd_ctl_dir,
5040 },
56992309 5041 {}
e692ab53
NP
5042};
5043
5044static struct ctl_table *sd_alloc_ctl_entry(int n)
5045{
5046 struct ctl_table *entry =
5cf9f062 5047 kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
e692ab53 5048
e692ab53
NP
5049 return entry;
5050}
5051
6382bc90
MM
5052static void sd_free_ctl_entry(struct ctl_table **tablep)
5053{
cd790076 5054 struct ctl_table *entry;
6382bc90 5055
cd790076
MM
5056 /*
5057 * In the intermediate directories, both the child directory and
5058 * procname are dynamically allocated and could fail but the mode
41a2d6cf 5059 * will always be set. In the lowest directory the names are
cd790076
MM
5060 * static strings and all have proc handlers.
5061 */
5062 for (entry = *tablep; entry->mode; entry++) {
6382bc90
MM
5063 if (entry->child)
5064 sd_free_ctl_entry(&entry->child);
cd790076
MM
5065 if (entry->proc_handler == NULL)
5066 kfree(entry->procname);
5067 }
6382bc90
MM
5068
5069 kfree(*tablep);
5070 *tablep = NULL;
5071}
5072
201c373e 5073static int min_load_idx = 0;
fd9b86d3 5074static int max_load_idx = CPU_LOAD_IDX_MAX-1;
201c373e 5075
e692ab53 5076static void
e0361851 5077set_table_entry(struct ctl_table *entry,
e692ab53 5078 const char *procname, void *data, int maxlen,
201c373e
NK
5079 umode_t mode, proc_handler *proc_handler,
5080 bool load_idx)
e692ab53 5081{
e692ab53
NP
5082 entry->procname = procname;
5083 entry->data = data;
5084 entry->maxlen = maxlen;
5085 entry->mode = mode;
5086 entry->proc_handler = proc_handler;
201c373e
NK
5087
5088 if (load_idx) {
5089 entry->extra1 = &min_load_idx;
5090 entry->extra2 = &max_load_idx;
5091 }
e692ab53
NP
5092}
5093
5094static struct ctl_table *
5095sd_alloc_ctl_domain_table(struct sched_domain *sd)
5096{
37e6bae8 5097 struct ctl_table *table = sd_alloc_ctl_entry(14);
e692ab53 5098
ad1cdc1d
MM
5099 if (table == NULL)
5100 return NULL;
5101
e0361851 5102 set_table_entry(&table[0], "min_interval", &sd->min_interval,
201c373e 5103 sizeof(long), 0644, proc_doulongvec_minmax, false);
e0361851 5104 set_table_entry(&table[1], "max_interval", &sd->max_interval,
201c373e 5105 sizeof(long), 0644, proc_doulongvec_minmax, false);
e0361851 5106 set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
201c373e 5107 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5108 set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
201c373e 5109 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5110 set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
201c373e 5111 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5112 set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
201c373e 5113 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5114 set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
201c373e 5115 sizeof(int), 0644, proc_dointvec_minmax, true);
e0361851 5116 set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
201c373e 5117 sizeof(int), 0644, proc_dointvec_minmax, false);
e0361851 5118 set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
201c373e 5119 sizeof(int), 0644, proc_dointvec_minmax, false);
ace8b3d6 5120 set_table_entry(&table[9], "cache_nice_tries",
e692ab53 5121 &sd->cache_nice_tries,
201c373e 5122 sizeof(int), 0644, proc_dointvec_minmax, false);
ace8b3d6 5123 set_table_entry(&table[10], "flags", &sd->flags,
201c373e 5124 sizeof(int), 0644, proc_dointvec_minmax, false);
37e6bae8
AS
5125 set_table_entry(&table[11], "max_newidle_lb_cost",
5126 &sd->max_newidle_lb_cost,
5127 sizeof(long), 0644, proc_doulongvec_minmax, false);
5128 set_table_entry(&table[12], "name", sd->name,
201c373e 5129 CORENAME_MAX_SIZE, 0444, proc_dostring, false);
37e6bae8 5130 /* &table[13] is terminator */
e692ab53
NP
5131
5132 return table;
5133}
5134
be7002e6 5135static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
e692ab53
NP
5136{
5137 struct ctl_table *entry, *table;
5138 struct sched_domain *sd;
5139 int domain_num = 0, i;
5140 char buf[32];
5141
5142 for_each_domain(cpu, sd)
5143 domain_num++;
5144 entry = table = sd_alloc_ctl_entry(domain_num + 1);
ad1cdc1d
MM
5145 if (table == NULL)
5146 return NULL;
e692ab53
NP
5147
5148 i = 0;
5149 for_each_domain(cpu, sd) {
5150 snprintf(buf, 32, "domain%d", i);
e692ab53 5151 entry->procname = kstrdup(buf, GFP_KERNEL);
c57baf1e 5152 entry->mode = 0555;
e692ab53
NP
5153 entry->child = sd_alloc_ctl_domain_table(sd);
5154 entry++;
5155 i++;
5156 }
5157 return table;
5158}
5159
5160static struct ctl_table_header *sd_sysctl_header;
6382bc90 5161static void register_sched_domain_sysctl(void)
e692ab53 5162{
6ad4c188 5163 int i, cpu_num = num_possible_cpus();
e692ab53
NP
5164 struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1);
5165 char buf[32];
5166
7378547f
MM
5167 WARN_ON(sd_ctl_dir[0].child);
5168 sd_ctl_dir[0].child = entry;
5169
ad1cdc1d
MM
5170 if (entry == NULL)
5171 return;
5172
6ad4c188 5173 for_each_possible_cpu(i) {
e692ab53 5174 snprintf(buf, 32, "cpu%d", i);
e692ab53 5175 entry->procname = kstrdup(buf, GFP_KERNEL);
c57baf1e 5176 entry->mode = 0555;
e692ab53 5177 entry->child = sd_alloc_ctl_cpu_table(i);
97b6ea7b 5178 entry++;
e692ab53 5179 }
7378547f
MM
5180
5181 WARN_ON(sd_sysctl_header);
e692ab53
NP
5182 sd_sysctl_header = register_sysctl_table(sd_ctl_root);
5183}
6382bc90 5184
7378547f 5185/* may be called multiple times per register */
6382bc90
MM
5186static void unregister_sched_domain_sysctl(void)
5187{
7378547f
MM
5188 if (sd_sysctl_header)
5189 unregister_sysctl_table(sd_sysctl_header);
6382bc90 5190 sd_sysctl_header = NULL;
7378547f
MM
5191 if (sd_ctl_dir[0].child)
5192 sd_free_ctl_entry(&sd_ctl_dir[0].child);
6382bc90 5193}
e692ab53 5194#else
6382bc90
MM
5195static void register_sched_domain_sysctl(void)
5196{
5197}
5198static void unregister_sched_domain_sysctl(void)
e692ab53
NP
5199{
5200}
5201#endif
5202
1f11eb6a
GH
5203static void set_rq_online(struct rq *rq)
5204{
5205 if (!rq->online) {
5206 const struct sched_class *class;
5207
c6c4927b 5208 cpumask_set_cpu(rq->cpu, rq->rd->online);
1f11eb6a
GH
5209 rq->online = 1;
5210
5211 for_each_class(class) {
5212 if (class->rq_online)
5213 class->rq_online(rq);
5214 }
5215 }
5216}
5217
5218static void set_rq_offline(struct rq *rq)
5219{
5220 if (rq->online) {
5221 const struct sched_class *class;
5222
5223 for_each_class(class) {
5224 if (class->rq_offline)
5225 class->rq_offline(rq);
5226 }
5227
c6c4927b 5228 cpumask_clear_cpu(rq->cpu, rq->rd->online);
1f11eb6a
GH
5229 rq->online = 0;
5230 }
5231}
5232
1da177e4
LT
5233/*
5234 * migration_call - callback that gets triggered when a CPU is added.
5235 * Here we can start up the necessary migration thread for the new CPU.
5236 */
0db0628d 5237static int
48f24c4d 5238migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
1da177e4 5239{
48f24c4d 5240 int cpu = (long)hcpu;
1da177e4 5241 unsigned long flags;
969c7921 5242 struct rq *rq = cpu_rq(cpu);
1da177e4 5243
48c5ccae 5244 switch (action & ~CPU_TASKS_FROZEN) {
5be9361c 5245
1da177e4 5246 case CPU_UP_PREPARE:
a468d389 5247 rq->calc_load_update = calc_load_update;
1da177e4 5248 break;
48f24c4d 5249
1da177e4 5250 case CPU_ONLINE:
1f94ef59 5251 /* Update our root-domain */
05fa785c 5252 raw_spin_lock_irqsave(&rq->lock, flags);
1f94ef59 5253 if (rq->rd) {
c6c4927b 5254 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
1f11eb6a
GH
5255
5256 set_rq_online(rq);
1f94ef59 5257 }
05fa785c 5258 raw_spin_unlock_irqrestore(&rq->lock, flags);
1da177e4 5259 break;
48f24c4d 5260
1da177e4 5261#ifdef CONFIG_HOTPLUG_CPU
08f503b0 5262 case CPU_DYING:
317f3941 5263 sched_ttwu_pending();
57d885fe 5264 /* Update our root-domain */
05fa785c 5265 raw_spin_lock_irqsave(&rq->lock, flags);
57d885fe 5266 if (rq->rd) {
c6c4927b 5267 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
1f11eb6a 5268 set_rq_offline(rq);
57d885fe 5269 }
48c5ccae
PZ
5270 migrate_tasks(cpu);
5271 BUG_ON(rq->nr_running != 1); /* the migration thread */
05fa785c 5272 raw_spin_unlock_irqrestore(&rq->lock, flags);
5d180232 5273 break;
48c5ccae 5274
5d180232 5275 case CPU_DEAD:
f319da0c 5276 calc_load_migrate(rq);
57d885fe 5277 break;
1da177e4
LT
5278#endif
5279 }
49c022e6
PZ
5280
5281 update_max_interval();
5282
1da177e4
LT
5283 return NOTIFY_OK;
5284}
5285
f38b0820
PM
5286/*
5287 * Register at high priority so that task migration (migrate_all_tasks)
5288 * happens before everything else. This has to be lower priority than
cdd6c482 5289 * the notifier in the perf_event subsystem, though.
1da177e4 5290 */
0db0628d 5291static struct notifier_block migration_notifier = {
1da177e4 5292 .notifier_call = migration_call,
50a323b7 5293 .priority = CPU_PRI_MIGRATION,
1da177e4
LT
5294};
5295
a803f026
CM
5296static void __cpuinit set_cpu_rq_start_time(void)
5297{
5298 int cpu = smp_processor_id();
5299 struct rq *rq = cpu_rq(cpu);
5300 rq->age_stamp = sched_clock_cpu(cpu);
5301}
5302
0db0628d 5303static int sched_cpu_active(struct notifier_block *nfb,
3a101d05
TH
5304 unsigned long action, void *hcpu)
5305{
5306 switch (action & ~CPU_TASKS_FROZEN) {
a803f026
CM
5307 case CPU_STARTING:
5308 set_cpu_rq_start_time();
5309 return NOTIFY_OK;
3a101d05
TH
5310 case CPU_DOWN_FAILED:
5311 set_cpu_active((long)hcpu, true);
5312 return NOTIFY_OK;
5313 default:
5314 return NOTIFY_DONE;
5315 }
5316}
5317
0db0628d 5318static int sched_cpu_inactive(struct notifier_block *nfb,
3a101d05
TH
5319 unsigned long action, void *hcpu)
5320{
de212f18
PZ
5321 unsigned long flags;
5322 long cpu = (long)hcpu;
f10e00f4 5323 struct dl_bw *dl_b;
de212f18 5324
3a101d05
TH
5325 switch (action & ~CPU_TASKS_FROZEN) {
5326 case CPU_DOWN_PREPARE:
de212f18
PZ
5327 set_cpu_active(cpu, false);
5328
5329 /* explicitly allow suspend */
5330 if (!(action & CPU_TASKS_FROZEN)) {
de212f18
PZ
5331 bool overflow;
5332 int cpus;
5333
f10e00f4
KT
5334 rcu_read_lock_sched();
5335 dl_b = dl_bw_of(cpu);
5336
de212f18
PZ
5337 raw_spin_lock_irqsave(&dl_b->lock, flags);
5338 cpus = dl_bw_cpus(cpu);
5339 overflow = __dl_overflow(dl_b, cpus, 0, 0);
5340 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
5341
f10e00f4
KT
5342 rcu_read_unlock_sched();
5343
de212f18
PZ
5344 if (overflow)
5345 return notifier_from_errno(-EBUSY);
5346 }
3a101d05 5347 return NOTIFY_OK;
3a101d05 5348 }
de212f18
PZ
5349
5350 return NOTIFY_DONE;
3a101d05
TH
5351}
5352
7babe8db 5353static int __init migration_init(void)
1da177e4
LT
5354{
5355 void *cpu = (void *)(long)smp_processor_id();
07dccf33 5356 int err;
48f24c4d 5357
3a101d05 5358 /* Initialize migration for the boot CPU */
07dccf33
AM
5359 err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu);
5360 BUG_ON(err == NOTIFY_BAD);
1da177e4
LT
5361 migration_call(&migration_notifier, CPU_ONLINE, cpu);
5362 register_cpu_notifier(&migration_notifier);
7babe8db 5363
3a101d05
TH
5364 /* Register cpu active notifiers */
5365 cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE);
5366 cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE);
5367
a004cd42 5368 return 0;
1da177e4 5369}
7babe8db 5370early_initcall(migration_init);
1da177e4
LT
5371#endif
5372
5373#ifdef CONFIG_SMP
476f3534 5374
4cb98839
PZ
5375static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */
5376
3e9830dc 5377#ifdef CONFIG_SCHED_DEBUG
4dcf6aff 5378
d039ac60 5379static __read_mostly int sched_debug_enabled;
f6630114 5380
d039ac60 5381static int __init sched_debug_setup(char *str)
f6630114 5382{
d039ac60 5383 sched_debug_enabled = 1;
f6630114
MT
5384
5385 return 0;
5386}
d039ac60
PZ
5387early_param("sched_debug", sched_debug_setup);
5388
5389static inline bool sched_debug(void)
5390{
5391 return sched_debug_enabled;
5392}
f6630114 5393
7c16ec58 5394static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
96f874e2 5395 struct cpumask *groupmask)
1da177e4 5396{
4dcf6aff 5397 struct sched_group *group = sd->groups;
1da177e4 5398
96f874e2 5399 cpumask_clear(groupmask);
4dcf6aff
IM
5400
5401 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
5402
5403 if (!(sd->flags & SD_LOAD_BALANCE)) {
3df0fc5b 5404 printk("does not load-balance\n");
4dcf6aff 5405 if (sd->parent)
3df0fc5b
PZ
5406 printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
5407 " has parent");
4dcf6aff 5408 return -1;
41c7ce9a
NP
5409 }
5410
333470ee
TH
5411 printk(KERN_CONT "span %*pbl level %s\n",
5412 cpumask_pr_args(sched_domain_span(sd)), sd->name);
4dcf6aff 5413
758b2cdc 5414 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
3df0fc5b
PZ
5415 printk(KERN_ERR "ERROR: domain->span does not contain "
5416 "CPU%d\n", cpu);
4dcf6aff 5417 }
758b2cdc 5418 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
3df0fc5b
PZ
5419 printk(KERN_ERR "ERROR: domain->groups does not contain"
5420 " CPU%d\n", cpu);
4dcf6aff 5421 }
1da177e4 5422
4dcf6aff 5423 printk(KERN_DEBUG "%*s groups:", level + 1, "");
1da177e4 5424 do {
4dcf6aff 5425 if (!group) {
3df0fc5b
PZ
5426 printk("\n");
5427 printk(KERN_ERR "ERROR: group is NULL\n");
1da177e4
LT
5428 break;
5429 }
5430
c3decf0d 5431 /*
63b2ca30
NP
5432 * Even though we initialize ->capacity to something semi-sane,
5433 * we leave capacity_orig unset. This allows us to detect if
c3decf0d
PZ
5434 * domain iteration is still funny without causing /0 traps.
5435 */
63b2ca30 5436 if (!group->sgc->capacity_orig) {
3df0fc5b 5437 printk(KERN_CONT "\n");
63b2ca30 5438 printk(KERN_ERR "ERROR: domain->cpu_capacity not set\n");
4dcf6aff
IM
5439 break;
5440 }
1da177e4 5441
758b2cdc 5442 if (!cpumask_weight(sched_group_cpus(group))) {
3df0fc5b
PZ
5443 printk(KERN_CONT "\n");
5444 printk(KERN_ERR "ERROR: empty group\n");
4dcf6aff
IM
5445 break;
5446 }
1da177e4 5447
cb83b629
PZ
5448 if (!(sd->flags & SD_OVERLAP) &&
5449 cpumask_intersects(groupmask, sched_group_cpus(group))) {
3df0fc5b
PZ
5450 printk(KERN_CONT "\n");
5451 printk(KERN_ERR "ERROR: repeated CPUs\n");
4dcf6aff
IM
5452 break;
5453 }
1da177e4 5454
758b2cdc 5455 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
1da177e4 5456
333470ee
TH
5457 printk(KERN_CONT " %*pbl",
5458 cpumask_pr_args(sched_group_cpus(group)));
ca8ce3d0 5459 if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
63b2ca30
NP
5460 printk(KERN_CONT " (cpu_capacity = %d)",
5461 group->sgc->capacity);
381512cf 5462 }
1da177e4 5463
4dcf6aff
IM
5464 group = group->next;
5465 } while (group != sd->groups);
3df0fc5b 5466 printk(KERN_CONT "\n");
1da177e4 5467
758b2cdc 5468 if (!cpumask_equal(sched_domain_span(sd), groupmask))
3df0fc5b 5469 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
1da177e4 5470
758b2cdc
RR
5471 if (sd->parent &&
5472 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
3df0fc5b
PZ
5473 printk(KERN_ERR "ERROR: parent span is not a superset "
5474 "of domain->span\n");
4dcf6aff
IM
5475 return 0;
5476}
1da177e4 5477
4dcf6aff
IM
5478static void sched_domain_debug(struct sched_domain *sd, int cpu)
5479{
5480 int level = 0;
1da177e4 5481
d039ac60 5482 if (!sched_debug_enabled)
f6630114
MT
5483 return;
5484
4dcf6aff
IM
5485 if (!sd) {
5486 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
5487 return;
5488 }
1da177e4 5489
4dcf6aff
IM
5490 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
5491
5492 for (;;) {
4cb98839 5493 if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
4dcf6aff 5494 break;
1da177e4
LT
5495 level++;
5496 sd = sd->parent;
33859f7f 5497 if (!sd)
4dcf6aff
IM
5498 break;
5499 }
1da177e4 5500}
6d6bc0ad 5501#else /* !CONFIG_SCHED_DEBUG */
48f24c4d 5502# define sched_domain_debug(sd, cpu) do { } while (0)
d039ac60
PZ
5503static inline bool sched_debug(void)
5504{
5505 return false;
5506}
6d6bc0ad 5507#endif /* CONFIG_SCHED_DEBUG */
1da177e4 5508
1a20ff27 5509static int sd_degenerate(struct sched_domain *sd)
245af2c7 5510{
758b2cdc 5511 if (cpumask_weight(sched_domain_span(sd)) == 1)
245af2c7
SS
5512 return 1;
5513
5514 /* Following flags need at least 2 groups */
5515 if (sd->flags & (SD_LOAD_BALANCE |
5516 SD_BALANCE_NEWIDLE |
5517 SD_BALANCE_FORK |
89c4710e 5518 SD_BALANCE_EXEC |
5d4dfddd 5519 SD_SHARE_CPUCAPACITY |
d77b3ed5
VG
5520 SD_SHARE_PKG_RESOURCES |
5521 SD_SHARE_POWERDOMAIN)) {
245af2c7
SS
5522 if (sd->groups != sd->groups->next)
5523 return 0;
5524 }
5525
5526 /* Following flags don't use groups */
c88d5910 5527 if (sd->flags & (SD_WAKE_AFFINE))
245af2c7
SS
5528 return 0;
5529
5530 return 1;
5531}
5532
48f24c4d
IM
5533static int
5534sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
245af2c7
SS
5535{
5536 unsigned long cflags = sd->flags, pflags = parent->flags;
5537
5538 if (sd_degenerate(parent))
5539 return 1;
5540
758b2cdc 5541 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
245af2c7
SS
5542 return 0;
5543
245af2c7
SS
5544 /* Flags needing groups don't count if only 1 group in parent */
5545 if (parent->groups == parent->groups->next) {
5546 pflags &= ~(SD_LOAD_BALANCE |
5547 SD_BALANCE_NEWIDLE |
5548 SD_BALANCE_FORK |
89c4710e 5549 SD_BALANCE_EXEC |
5d4dfddd 5550 SD_SHARE_CPUCAPACITY |
10866e62 5551 SD_SHARE_PKG_RESOURCES |
d77b3ed5
VG
5552 SD_PREFER_SIBLING |
5553 SD_SHARE_POWERDOMAIN);
5436499e
KC
5554 if (nr_node_ids == 1)
5555 pflags &= ~SD_SERIALIZE;
245af2c7
SS
5556 }
5557 if (~cflags & pflags)
5558 return 0;
5559
5560 return 1;
5561}
5562
dce840a0 5563static void free_rootdomain(struct rcu_head *rcu)
c6c4927b 5564{
dce840a0 5565 struct root_domain *rd = container_of(rcu, struct root_domain, rcu);
047106ad 5566
68e74568 5567 cpupri_cleanup(&rd->cpupri);
6bfd6d72 5568 cpudl_cleanup(&rd->cpudl);
1baca4ce 5569 free_cpumask_var(rd->dlo_mask);
c6c4927b
RR
5570 free_cpumask_var(rd->rto_mask);
5571 free_cpumask_var(rd->online);
5572 free_cpumask_var(rd->span);
5573 kfree(rd);
5574}
5575
57d885fe
GH
5576static void rq_attach_root(struct rq *rq, struct root_domain *rd)
5577{
a0490fa3 5578 struct root_domain *old_rd = NULL;
57d885fe 5579 unsigned long flags;
57d885fe 5580
05fa785c 5581 raw_spin_lock_irqsave(&rq->lock, flags);
57d885fe
GH
5582
5583 if (rq->rd) {
a0490fa3 5584 old_rd = rq->rd;
57d885fe 5585
c6c4927b 5586 if (cpumask_test_cpu(rq->cpu, old_rd->online))
1f11eb6a 5587 set_rq_offline(rq);
57d885fe 5588
c6c4927b 5589 cpumask_clear_cpu(rq->cpu, old_rd->span);
dc938520 5590
a0490fa3 5591 /*
0515973f 5592 * If we dont want to free the old_rd yet then
a0490fa3
IM
5593 * set old_rd to NULL to skip the freeing later
5594 * in this function:
5595 */
5596 if (!atomic_dec_and_test(&old_rd->refcount))
5597 old_rd = NULL;
57d885fe
GH
5598 }
5599
5600 atomic_inc(&rd->refcount);
5601 rq->rd = rd;
5602
c6c4927b 5603 cpumask_set_cpu(rq->cpu, rd->span);
00aec93d 5604 if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
1f11eb6a 5605 set_rq_online(rq);
57d885fe 5606
05fa785c 5607 raw_spin_unlock_irqrestore(&rq->lock, flags);
a0490fa3
IM
5608
5609 if (old_rd)
dce840a0 5610 call_rcu_sched(&old_rd->rcu, free_rootdomain);
57d885fe
GH
5611}
5612
68c38fc3 5613static int init_rootdomain(struct root_domain *rd)
57d885fe
GH
5614{
5615 memset(rd, 0, sizeof(*rd));
5616
68c38fc3 5617 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
0c910d28 5618 goto out;
68c38fc3 5619 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
c6c4927b 5620 goto free_span;
1baca4ce 5621 if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
c6c4927b 5622 goto free_online;
1baca4ce
JL
5623 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
5624 goto free_dlo_mask;
6e0534f2 5625
332ac17e 5626 init_dl_bw(&rd->dl_bw);
6bfd6d72
JL
5627 if (cpudl_init(&rd->cpudl) != 0)
5628 goto free_dlo_mask;
332ac17e 5629
68c38fc3 5630 if (cpupri_init(&rd->cpupri) != 0)
68e74568 5631 goto free_rto_mask;
c6c4927b 5632 return 0;
6e0534f2 5633
68e74568
RR
5634free_rto_mask:
5635 free_cpumask_var(rd->rto_mask);
1baca4ce
JL
5636free_dlo_mask:
5637 free_cpumask_var(rd->dlo_mask);
c6c4927b
RR
5638free_online:
5639 free_cpumask_var(rd->online);
5640free_span:
5641 free_cpumask_var(rd->span);
0c910d28 5642out:
c6c4927b 5643 return -ENOMEM;
57d885fe
GH
5644}
5645
029632fb
PZ
5646/*
5647 * By default the system creates a single root-domain with all cpus as
5648 * members (mimicking the global state we have today).
5649 */
5650struct root_domain def_root_domain;
5651
57d885fe
GH
5652static void init_defrootdomain(void)
5653{
68c38fc3 5654 init_rootdomain(&def_root_domain);
c6c4927b 5655
57d885fe
GH
5656 atomic_set(&def_root_domain.refcount, 1);
5657}
5658
dc938520 5659static struct root_domain *alloc_rootdomain(void)
57d885fe
GH
5660{
5661 struct root_domain *rd;
5662
5663 rd = kmalloc(sizeof(*rd), GFP_KERNEL);
5664 if (!rd)
5665 return NULL;
5666
68c38fc3 5667 if (init_rootdomain(rd) != 0) {
c6c4927b
RR
5668 kfree(rd);
5669 return NULL;
5670 }
57d885fe
GH
5671
5672 return rd;
5673}
5674
63b2ca30 5675static void free_sched_groups(struct sched_group *sg, int free_sgc)
e3589f6c
PZ
5676{
5677 struct sched_group *tmp, *first;
5678
5679 if (!sg)
5680 return;
5681
5682 first = sg;
5683 do {
5684 tmp = sg->next;
5685
63b2ca30
NP
5686 if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
5687 kfree(sg->sgc);
e3589f6c
PZ
5688
5689 kfree(sg);
5690 sg = tmp;
5691 } while (sg != first);
5692}
5693
dce840a0
PZ
5694static void free_sched_domain(struct rcu_head *rcu)
5695{
5696 struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);
e3589f6c
PZ
5697
5698 /*
5699 * If its an overlapping domain it has private groups, iterate and
5700 * nuke them all.
5701 */
5702 if (sd->flags & SD_OVERLAP) {
5703 free_sched_groups(sd->groups, 1);
5704 } else if (atomic_dec_and_test(&sd->groups->ref)) {
63b2ca30 5705 kfree(sd->groups->sgc);
dce840a0 5706 kfree(sd->groups);
9c3f75cb 5707 }
dce840a0
PZ
5708 kfree(sd);
5709}
5710
5711static void destroy_sched_domain(struct sched_domain *sd, int cpu)
5712{
5713 call_rcu(&sd->rcu, free_sched_domain);
5714}
5715
5716static void destroy_sched_domains(struct sched_domain *sd, int cpu)
5717{
5718 for (; sd; sd = sd->parent)
5719 destroy_sched_domain(sd, cpu);
5720}
5721
518cd623
PZ
5722/*
5723 * Keep a special pointer to the highest sched_domain that has
5724 * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this
5725 * allows us to avoid some pointer chasing select_idle_sibling().
5726 *
5727 * Also keep a unique ID per domain (we use the first cpu number in
5728 * the cpumask of the domain), this allows us to quickly tell if
39be3501 5729 * two cpus are in the same cache domain, see cpus_share_cache().
518cd623
PZ
5730 */
5731DEFINE_PER_CPU(struct sched_domain *, sd_llc);
7d9ffa89 5732DEFINE_PER_CPU(int, sd_llc_size);
518cd623 5733DEFINE_PER_CPU(int, sd_llc_id);
fb13c7ee 5734DEFINE_PER_CPU(struct sched_domain *, sd_numa);
37dc6b50
PM
5735DEFINE_PER_CPU(struct sched_domain *, sd_busy);
5736DEFINE_PER_CPU(struct sched_domain *, sd_asym);
518cd623
PZ
5737
5738static void update_top_cache_domain(int cpu)
5739{
5740 struct sched_domain *sd;
5d4cf996 5741 struct sched_domain *busy_sd = NULL;
518cd623 5742 int id = cpu;
7d9ffa89 5743 int size = 1;
518cd623
PZ
5744
5745 sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES);
7d9ffa89 5746 if (sd) {
518cd623 5747 id = cpumask_first(sched_domain_span(sd));
7d9ffa89 5748 size = cpumask_weight(sched_domain_span(sd));
5d4cf996 5749 busy_sd = sd->parent; /* sd_busy */
7d9ffa89 5750 }
5d4cf996 5751 rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd);
518cd623
PZ
5752
5753 rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
7d9ffa89 5754 per_cpu(sd_llc_size, cpu) = size;
518cd623 5755 per_cpu(sd_llc_id, cpu) = id;
fb13c7ee
MG
5756
5757 sd = lowest_flag_domain(cpu, SD_NUMA);
5758 rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
37dc6b50
PM
5759
5760 sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
5761 rcu_assign_pointer(per_cpu(sd_asym, cpu), sd);
518cd623
PZ
5762}
5763
1da177e4 5764/*
0eab9146 5765 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
1da177e4
LT
5766 * hold the hotplug lock.
5767 */
0eab9146
IM
5768static void
5769cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
1da177e4 5770{
70b97a7f 5771 struct rq *rq = cpu_rq(cpu);
245af2c7
SS
5772 struct sched_domain *tmp;
5773
5774 /* Remove the sched domains which do not contribute to scheduling. */
f29c9b1c 5775 for (tmp = sd; tmp; ) {
245af2c7
SS
5776 struct sched_domain *parent = tmp->parent;
5777 if (!parent)
5778 break;
f29c9b1c 5779
1a848870 5780 if (sd_parent_degenerate(tmp, parent)) {
245af2c7 5781 tmp->parent = parent->parent;
1a848870
SS
5782 if (parent->parent)
5783 parent->parent->child = tmp;
10866e62
PZ
5784 /*
5785 * Transfer SD_PREFER_SIBLING down in case of a
5786 * degenerate parent; the spans match for this
5787 * so the property transfers.
5788 */
5789 if (parent->flags & SD_PREFER_SIBLING)
5790 tmp->flags |= SD_PREFER_SIBLING;
dce840a0 5791 destroy_sched_domain(parent, cpu);
f29c9b1c
LZ
5792 } else
5793 tmp = tmp->parent;
245af2c7
SS
5794 }
5795
1a848870 5796 if (sd && sd_degenerate(sd)) {
dce840a0 5797 tmp = sd;
245af2c7 5798 sd = sd->parent;
dce840a0 5799 destroy_sched_domain(tmp, cpu);
1a848870
SS
5800 if (sd)
5801 sd->child = NULL;
5802 }
1da177e4 5803
4cb98839 5804 sched_domain_debug(sd, cpu);
1da177e4 5805
57d885fe 5806 rq_attach_root(rq, rd);
dce840a0 5807 tmp = rq->sd;
674311d5 5808 rcu_assign_pointer(rq->sd, sd);
dce840a0 5809 destroy_sched_domains(tmp, cpu);
518cd623
PZ
5810
5811 update_top_cache_domain(cpu);
1da177e4
LT
5812}
5813
5814/* cpus with isolated domains */
dcc30a35 5815static cpumask_var_t cpu_isolated_map;
1da177e4
LT
5816
5817/* Setup the mask of cpus configured for isolated domains */
5818static int __init isolated_cpu_setup(char *str)
5819{
bdddd296 5820 alloc_bootmem_cpumask_var(&cpu_isolated_map);
968ea6d8 5821 cpulist_parse(str, cpu_isolated_map);
1da177e4
LT
5822 return 1;
5823}
5824
8927f494 5825__setup("isolcpus=", isolated_cpu_setup);
1da177e4 5826
49a02c51 5827struct s_data {
21d42ccf 5828 struct sched_domain ** __percpu sd;
49a02c51
AH
5829 struct root_domain *rd;
5830};
5831
2109b99e 5832enum s_alloc {
2109b99e 5833 sa_rootdomain,
21d42ccf 5834 sa_sd,
dce840a0 5835 sa_sd_storage,
2109b99e
AH
5836 sa_none,
5837};
5838
c1174876
PZ
5839/*
5840 * Build an iteration mask that can exclude certain CPUs from the upwards
5841 * domain traversal.
5842 *
5843 * Asymmetric node setups can result in situations where the domain tree is of
5844 * unequal depth, make sure to skip domains that already cover the entire
5845 * range.
5846 *
5847 * In that case build_sched_domains() will have terminated the iteration early
5848 * and our sibling sd spans will be empty. Domains should always include the
5849 * cpu they're built on, so check that.
5850 *
5851 */
5852static void build_group_mask(struct sched_domain *sd, struct sched_group *sg)
5853{
5854 const struct cpumask *span = sched_domain_span(sd);
5855 struct sd_data *sdd = sd->private;
5856 struct sched_domain *sibling;
5857 int i;
5858
5859 for_each_cpu(i, span) {
5860 sibling = *per_cpu_ptr(sdd->sd, i);
5861 if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
5862 continue;
5863
5864 cpumask_set_cpu(i, sched_group_mask(sg));
5865 }
5866}
5867
5868/*
5869 * Return the canonical balance cpu for this group, this is the first cpu
5870 * of this group that's also in the iteration mask.
5871 */
5872int group_balance_cpu(struct sched_group *sg)
5873{
5874 return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg));
5875}
5876
e3589f6c
PZ
5877static int
5878build_overlap_sched_groups(struct sched_domain *sd, int cpu)
5879{
5880 struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg;
5881 const struct cpumask *span = sched_domain_span(sd);
5882 struct cpumask *covered = sched_domains_tmpmask;
5883 struct sd_data *sdd = sd->private;
aaecac4a 5884 struct sched_domain *sibling;
e3589f6c
PZ
5885 int i;
5886
5887 cpumask_clear(covered);
5888
5889 for_each_cpu(i, span) {
5890 struct cpumask *sg_span;
5891
5892 if (cpumask_test_cpu(i, covered))
5893 continue;
5894
aaecac4a 5895 sibling = *per_cpu_ptr(sdd->sd, i);
c1174876
PZ
5896
5897 /* See the comment near build_group_mask(). */
aaecac4a 5898 if (!cpumask_test_cpu(i, sched_domain_span(sibling)))
c1174876
PZ
5899 continue;
5900
e3589f6c 5901 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
4d78a223 5902 GFP_KERNEL, cpu_to_node(cpu));
e3589f6c
PZ
5903
5904 if (!sg)
5905 goto fail;
5906
5907 sg_span = sched_group_cpus(sg);
aaecac4a
ZZ
5908 if (sibling->child)
5909 cpumask_copy(sg_span, sched_domain_span(sibling->child));
5910 else
e3589f6c
PZ
5911 cpumask_set_cpu(i, sg_span);
5912
5913 cpumask_or(covered, covered, sg_span);
5914
63b2ca30
NP
5915 sg->sgc = *per_cpu_ptr(sdd->sgc, i);
5916 if (atomic_inc_return(&sg->sgc->ref) == 1)
c1174876
PZ
5917 build_group_mask(sd, sg);
5918
c3decf0d 5919 /*
63b2ca30 5920 * Initialize sgc->capacity such that even if we mess up the
c3decf0d
PZ
5921 * domains and no possible iteration will get us here, we won't
5922 * die on a /0 trap.
5923 */
ca8ce3d0 5924 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
63b2ca30 5925 sg->sgc->capacity_orig = sg->sgc->capacity;
e3589f6c 5926
c1174876
PZ
5927 /*
5928 * Make sure the first group of this domain contains the
5929 * canonical balance cpu. Otherwise the sched_domain iteration
5930 * breaks. See update_sg_lb_stats().
5931 */
74a5ce20 5932 if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
c1174876 5933 group_balance_cpu(sg) == cpu)
e3589f6c
PZ
5934 groups = sg;
5935
5936 if (!first)
5937 first = sg;
5938 if (last)
5939 last->next = sg;
5940 last = sg;
5941 last->next = first;
5942 }
5943 sd->groups = groups;
5944
5945 return 0;
5946
5947fail:
5948 free_sched_groups(first, 0);
5949
5950 return -ENOMEM;
5951}
5952
dce840a0 5953static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg)
1da177e4 5954{
dce840a0
PZ
5955 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
5956 struct sched_domain *child = sd->child;
1da177e4 5957
dce840a0
PZ
5958 if (child)
5959 cpu = cpumask_first(sched_domain_span(child));
1e9f28fa 5960
9c3f75cb 5961 if (sg) {
dce840a0 5962 *sg = *per_cpu_ptr(sdd->sg, cpu);
63b2ca30
NP
5963 (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu);
5964 atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */
9c3f75cb 5965 }
dce840a0
PZ
5966
5967 return cpu;
1e9f28fa 5968}
1e9f28fa 5969
01a08546 5970/*
dce840a0
PZ
5971 * build_sched_groups will build a circular linked list of the groups
5972 * covered by the given span, and will set each group's ->cpumask correctly,
ced549fa 5973 * and ->cpu_capacity to 0.
e3589f6c
PZ
5974 *
5975 * Assumes the sched_domain tree is fully constructed
01a08546 5976 */
e3589f6c
PZ
5977static int
5978build_sched_groups(struct sched_domain *sd, int cpu)
1da177e4 5979{
dce840a0
PZ
5980 struct sched_group *first = NULL, *last = NULL;
5981 struct sd_data *sdd = sd->private;
5982 const struct cpumask *span = sched_domain_span(sd);
f96225fd 5983 struct cpumask *covered;
dce840a0 5984 int i;
9c1cfda2 5985
e3589f6c
PZ
5986 get_group(cpu, sdd, &sd->groups);
5987 atomic_inc(&sd->groups->ref);
5988
0936629f 5989 if (cpu != cpumask_first(span))
e3589f6c
PZ
5990 return 0;
5991
f96225fd
PZ
5992 lockdep_assert_held(&sched_domains_mutex);
5993 covered = sched_domains_tmpmask;
5994
dce840a0 5995 cpumask_clear(covered);
6711cab4 5996
dce840a0
PZ
5997 for_each_cpu(i, span) {
5998 struct sched_group *sg;
cd08e923 5999 int group, j;
6711cab4 6000
dce840a0
PZ
6001 if (cpumask_test_cpu(i, covered))
6002 continue;
6711cab4 6003
cd08e923 6004 group = get_group(i, sdd, &sg);
c1174876 6005 cpumask_setall(sched_group_mask(sg));
0601a88d 6006
dce840a0
PZ
6007 for_each_cpu(j, span) {
6008 if (get_group(j, sdd, NULL) != group)
6009 continue;
0601a88d 6010
dce840a0
PZ
6011 cpumask_set_cpu(j, covered);
6012 cpumask_set_cpu(j, sched_group_cpus(sg));
6013 }
0601a88d 6014
dce840a0
PZ
6015 if (!first)
6016 first = sg;
6017 if (last)
6018 last->next = sg;
6019 last = sg;
6020 }
6021 last->next = first;
e3589f6c
PZ
6022
6023 return 0;
0601a88d 6024}
51888ca2 6025
89c4710e 6026/*
63b2ca30 6027 * Initialize sched groups cpu_capacity.
89c4710e 6028 *
63b2ca30 6029 * cpu_capacity indicates the capacity of sched group, which is used while
89c4710e 6030 * distributing the load between different sched groups in a sched domain.
63b2ca30
NP
6031 * Typically cpu_capacity for all the groups in a sched domain will be same
6032 * unless there are asymmetries in the topology. If there are asymmetries,
6033 * group having more cpu_capacity will pickup more load compared to the
6034 * group having less cpu_capacity.
89c4710e 6035 */
63b2ca30 6036static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
89c4710e 6037{
e3589f6c 6038 struct sched_group *sg = sd->groups;
89c4710e 6039
94c95ba6 6040 WARN_ON(!sg);
e3589f6c
PZ
6041
6042 do {
6043 sg->group_weight = cpumask_weight(sched_group_cpus(sg));
6044 sg = sg->next;
6045 } while (sg != sd->groups);
89c4710e 6046
c1174876 6047 if (cpu != group_balance_cpu(sg))
e3589f6c 6048 return;
aae6d3dd 6049
63b2ca30
NP
6050 update_group_capacity(sd, cpu);
6051 atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight);
89c4710e
SS
6052}
6053
7c16ec58
MT
6054/*
6055 * Initializers for schedule domains
6056 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
6057 */
6058
1d3504fc 6059static int default_relax_domain_level = -1;
60495e77 6060int sched_domain_level_max;
1d3504fc
HS
6061
6062static int __init setup_relax_domain_level(char *str)
6063{
a841f8ce
DS
6064 if (kstrtoint(str, 0, &default_relax_domain_level))
6065 pr_warn("Unable to set relax_domain_level\n");
30e0e178 6066
1d3504fc
HS
6067 return 1;
6068}
6069__setup("relax_domain_level=", setup_relax_domain_level);
6070
6071static void set_domain_attribute(struct sched_domain *sd,
6072 struct sched_domain_attr *attr)
6073{
6074 int request;
6075
6076 if (!attr || attr->relax_domain_level < 0) {
6077 if (default_relax_domain_level < 0)
6078 return;
6079 else
6080 request = default_relax_domain_level;
6081 } else
6082 request = attr->relax_domain_level;
6083 if (request < sd->level) {
6084 /* turn off idle balance on this domain */
c88d5910 6085 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
1d3504fc
HS
6086 } else {
6087 /* turn on idle balance on this domain */
c88d5910 6088 sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
1d3504fc
HS
6089 }
6090}
6091
54ab4ff4
PZ
6092static void __sdt_free(const struct cpumask *cpu_map);
6093static int __sdt_alloc(const struct cpumask *cpu_map);
6094
2109b99e
AH
6095static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
6096 const struct cpumask *cpu_map)
6097{
6098 switch (what) {
2109b99e 6099 case sa_rootdomain:
822ff793
PZ
6100 if (!atomic_read(&d->rd->refcount))
6101 free_rootdomain(&d->rd->rcu); /* fall through */
21d42ccf
PZ
6102 case sa_sd:
6103 free_percpu(d->sd); /* fall through */
dce840a0 6104 case sa_sd_storage:
54ab4ff4 6105 __sdt_free(cpu_map); /* fall through */
2109b99e
AH
6106 case sa_none:
6107 break;
6108 }
6109}
3404c8d9 6110
2109b99e
AH
6111static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
6112 const struct cpumask *cpu_map)
6113{
dce840a0
PZ
6114 memset(d, 0, sizeof(*d));
6115
54ab4ff4
PZ
6116 if (__sdt_alloc(cpu_map))
6117 return sa_sd_storage;
dce840a0
PZ
6118 d->sd = alloc_percpu(struct sched_domain *);
6119 if (!d->sd)
6120 return sa_sd_storage;
2109b99e 6121 d->rd = alloc_rootdomain();
dce840a0 6122 if (!d->rd)
21d42ccf 6123 return sa_sd;
2109b99e
AH
6124 return sa_rootdomain;
6125}
57d885fe 6126
dce840a0
PZ
6127/*
6128 * NULL the sd_data elements we've used to build the sched_domain and
6129 * sched_group structure so that the subsequent __free_domain_allocs()
6130 * will not free the data we're using.
6131 */
6132static void claim_allocations(int cpu, struct sched_domain *sd)
6133{
6134 struct sd_data *sdd = sd->private;
dce840a0
PZ
6135
6136 WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
6137 *per_cpu_ptr(sdd->sd, cpu) = NULL;
6138
e3589f6c 6139 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
dce840a0 6140 *per_cpu_ptr(sdd->sg, cpu) = NULL;
e3589f6c 6141
63b2ca30
NP
6142 if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
6143 *per_cpu_ptr(sdd->sgc, cpu) = NULL;
dce840a0
PZ
6144}
6145
cb83b629 6146#ifdef CONFIG_NUMA
cb83b629 6147static int sched_domains_numa_levels;
e3fe70b1 6148enum numa_topology_type sched_numa_topology_type;
cb83b629 6149static int *sched_domains_numa_distance;
9942f79b 6150int sched_max_numa_distance;
cb83b629
PZ
6151static struct cpumask ***sched_domains_numa_masks;
6152static int sched_domains_curr_level;
143e1e28 6153#endif
cb83b629 6154
143e1e28
VG
6155/*
6156 * SD_flags allowed in topology descriptions.
6157 *
5d4dfddd 6158 * SD_SHARE_CPUCAPACITY - describes SMT topologies
143e1e28
VG
6159 * SD_SHARE_PKG_RESOURCES - describes shared caches
6160 * SD_NUMA - describes NUMA topologies
d77b3ed5 6161 * SD_SHARE_POWERDOMAIN - describes shared power domain
143e1e28
VG
6162 *
6163 * Odd one out:
6164 * SD_ASYM_PACKING - describes SMT quirks
6165 */
6166#define TOPOLOGY_SD_FLAGS \
5d4dfddd 6167 (SD_SHARE_CPUCAPACITY | \
143e1e28
VG
6168 SD_SHARE_PKG_RESOURCES | \
6169 SD_NUMA | \
d77b3ed5
VG
6170 SD_ASYM_PACKING | \
6171 SD_SHARE_POWERDOMAIN)
cb83b629
PZ
6172
6173static struct sched_domain *
143e1e28 6174sd_init(struct sched_domain_topology_level *tl, int cpu)
cb83b629
PZ
6175{
6176 struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu);
143e1e28
VG
6177 int sd_weight, sd_flags = 0;
6178
6179#ifdef CONFIG_NUMA
6180 /*
6181 * Ugly hack to pass state to sd_numa_mask()...
6182 */
6183 sched_domains_curr_level = tl->numa_level;
6184#endif
6185
6186 sd_weight = cpumask_weight(tl->mask(cpu));
6187
6188 if (tl->sd_flags)
6189 sd_flags = (*tl->sd_flags)();
6190 if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
6191 "wrong sd_flags in topology description\n"))
6192 sd_flags &= ~TOPOLOGY_SD_FLAGS;
cb83b629
PZ
6193
6194 *sd = (struct sched_domain){
6195 .min_interval = sd_weight,
6196 .max_interval = 2*sd_weight,
6197 .busy_factor = 32,
870a0bb5 6198 .imbalance_pct = 125,
143e1e28
VG
6199
6200 .cache_nice_tries = 0,
6201 .busy_idx = 0,
6202 .idle_idx = 0,
cb83b629
PZ
6203 .newidle_idx = 0,
6204 .wake_idx = 0,
6205 .forkexec_idx = 0,
6206
6207 .flags = 1*SD_LOAD_BALANCE
6208 | 1*SD_BALANCE_NEWIDLE
143e1e28
VG
6209 | 1*SD_BALANCE_EXEC
6210 | 1*SD_BALANCE_FORK
cb83b629 6211 | 0*SD_BALANCE_WAKE
143e1e28 6212 | 1*SD_WAKE_AFFINE
5d4dfddd 6213 | 0*SD_SHARE_CPUCAPACITY
cb83b629 6214 | 0*SD_SHARE_PKG_RESOURCES
143e1e28 6215 | 0*SD_SERIALIZE
cb83b629 6216 | 0*SD_PREFER_SIBLING
143e1e28
VG
6217 | 0*SD_NUMA
6218 | sd_flags
cb83b629 6219 ,
143e1e28 6220
cb83b629
PZ
6221 .last_balance = jiffies,
6222 .balance_interval = sd_weight,
143e1e28 6223 .smt_gain = 0,
2b4cfe64
JL
6224 .max_newidle_lb_cost = 0,
6225 .next_decay_max_lb_cost = jiffies,
143e1e28
VG
6226#ifdef CONFIG_SCHED_DEBUG
6227 .name = tl->name,
6228#endif
cb83b629 6229 };
cb83b629
PZ
6230
6231 /*
143e1e28 6232 * Convert topological properties into behaviour.
cb83b629 6233 */
143e1e28 6234
5d4dfddd 6235 if (sd->flags & SD_SHARE_CPUCAPACITY) {
143e1e28
VG
6236 sd->imbalance_pct = 110;
6237 sd->smt_gain = 1178; /* ~15% */
143e1e28
VG
6238
6239 } else if (sd->flags & SD_SHARE_PKG_RESOURCES) {
6240 sd->imbalance_pct = 117;
6241 sd->cache_nice_tries = 1;
6242 sd->busy_idx = 2;
6243
6244#ifdef CONFIG_NUMA
6245 } else if (sd->flags & SD_NUMA) {
6246 sd->cache_nice_tries = 2;
6247 sd->busy_idx = 3;
6248 sd->idle_idx = 2;
6249
6250 sd->flags |= SD_SERIALIZE;
6251 if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
6252 sd->flags &= ~(SD_BALANCE_EXEC |
6253 SD_BALANCE_FORK |
6254 SD_WAKE_AFFINE);
6255 }
6256
6257#endif
6258 } else {
6259 sd->flags |= SD_PREFER_SIBLING;
6260 sd->cache_nice_tries = 1;
6261 sd->busy_idx = 2;
6262 sd->idle_idx = 1;
6263 }
6264
6265 sd->private = &tl->data;
cb83b629
PZ
6266
6267 return sd;
6268}
6269
143e1e28
VG
6270/*
6271 * Topology list, bottom-up.
6272 */
6273static struct sched_domain_topology_level default_topology[] = {
6274#ifdef CONFIG_SCHED_SMT
6275 { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
6276#endif
6277#ifdef CONFIG_SCHED_MC
6278 { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
143e1e28
VG
6279#endif
6280 { cpu_cpu_mask, SD_INIT_NAME(DIE) },
6281 { NULL, },
6282};
6283
6284struct sched_domain_topology_level *sched_domain_topology = default_topology;
6285
6286#define for_each_sd_topology(tl) \
6287 for (tl = sched_domain_topology; tl->mask; tl++)
6288
6289void set_sched_topology(struct sched_domain_topology_level *tl)
6290{
6291 sched_domain_topology = tl;
6292}
6293
6294#ifdef CONFIG_NUMA
6295
cb83b629
PZ
6296static const struct cpumask *sd_numa_mask(int cpu)
6297{
6298 return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
6299}
6300
d039ac60
PZ
6301static void sched_numa_warn(const char *str)
6302{
6303 static int done = false;
6304 int i,j;
6305
6306 if (done)
6307 return;
6308
6309 done = true;
6310
6311 printk(KERN_WARNING "ERROR: %s\n\n", str);
6312
6313 for (i = 0; i < nr_node_ids; i++) {
6314 printk(KERN_WARNING " ");
6315 for (j = 0; j < nr_node_ids; j++)
6316 printk(KERN_CONT "%02d ", node_distance(i,j));
6317 printk(KERN_CONT "\n");
6318 }
6319 printk(KERN_WARNING "\n");
6320}
6321
9942f79b 6322bool find_numa_distance(int distance)
d039ac60
PZ
6323{
6324 int i;
6325
6326 if (distance == node_distance(0, 0))
6327 return true;
6328
6329 for (i = 0; i < sched_domains_numa_levels; i++) {
6330 if (sched_domains_numa_distance[i] == distance)
6331 return true;
6332 }
6333
6334 return false;
6335}
6336
e3fe70b1
RR
6337/*
6338 * A system can have three types of NUMA topology:
6339 * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system
6340 * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes
6341 * NUMA_BACKPLANE: nodes can reach other nodes through a backplane
6342 *
6343 * The difference between a glueless mesh topology and a backplane
6344 * topology lies in whether communication between not directly
6345 * connected nodes goes through intermediary nodes (where programs
6346 * could run), or through backplane controllers. This affects
6347 * placement of programs.
6348 *
6349 * The type of topology can be discerned with the following tests:
6350 * - If the maximum distance between any nodes is 1 hop, the system
6351 * is directly connected.
6352 * - If for two nodes A and B, located N > 1 hops away from each other,
6353 * there is an intermediary node C, which is < N hops away from both
6354 * nodes A and B, the system is a glueless mesh.
6355 */
6356static void init_numa_topology_type(void)
6357{
6358 int a, b, c, n;
6359
6360 n = sched_max_numa_distance;
6361
6362 if (n <= 1)
6363 sched_numa_topology_type = NUMA_DIRECT;
6364
6365 for_each_online_node(a) {
6366 for_each_online_node(b) {
6367 /* Find two nodes furthest removed from each other. */
6368 if (node_distance(a, b) < n)
6369 continue;
6370
6371 /* Is there an intermediary node between a and b? */
6372 for_each_online_node(c) {
6373 if (node_distance(a, c) < n &&
6374 node_distance(b, c) < n) {
6375 sched_numa_topology_type =
6376 NUMA_GLUELESS_MESH;
6377 return;
6378 }
6379 }
6380
6381 sched_numa_topology_type = NUMA_BACKPLANE;
6382 return;
6383 }
6384 }
6385}
6386
cb83b629
PZ
6387static void sched_init_numa(void)
6388{
6389 int next_distance, curr_distance = node_distance(0, 0);
6390 struct sched_domain_topology_level *tl;
6391 int level = 0;
6392 int i, j, k;
6393
cb83b629
PZ
6394 sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL);
6395 if (!sched_domains_numa_distance)
6396 return;
6397
6398 /*
6399 * O(nr_nodes^2) deduplicating selection sort -- in order to find the
6400 * unique distances in the node_distance() table.
6401 *
6402 * Assumes node_distance(0,j) includes all distances in
6403 * node_distance(i,j) in order to avoid cubic time.
cb83b629
PZ
6404 */
6405 next_distance = curr_distance;
6406 for (i = 0; i < nr_node_ids; i++) {
6407 for (j = 0; j < nr_node_ids; j++) {
d039ac60
PZ
6408 for (k = 0; k < nr_node_ids; k++) {
6409 int distance = node_distance(i, k);
6410
6411 if (distance > curr_distance &&
6412 (distance < next_distance ||
6413 next_distance == curr_distance))
6414 next_distance = distance;
6415
6416 /*
6417 * While not a strong assumption it would be nice to know
6418 * about cases where if node A is connected to B, B is not
6419 * equally connected to A.
6420 */
6421 if (sched_debug() && node_distance(k, i) != distance)
6422 sched_numa_warn("Node-distance not symmetric");
6423
6424 if (sched_debug() && i && !find_numa_distance(distance))
6425 sched_numa_warn("Node-0 not representative");
6426 }
6427 if (next_distance != curr_distance) {
6428 sched_domains_numa_distance[level++] = next_distance;
6429 sched_domains_numa_levels = level;
6430 curr_distance = next_distance;
6431 } else break;
cb83b629 6432 }
d039ac60
PZ
6433
6434 /*
6435 * In case of sched_debug() we verify the above assumption.
6436 */
6437 if (!sched_debug())
6438 break;
cb83b629 6439 }
c123588b
AR
6440
6441 if (!level)
6442 return;
6443
cb83b629
PZ
6444 /*
6445 * 'level' contains the number of unique distances, excluding the
6446 * identity distance node_distance(i,i).
6447 *
28b4a521 6448 * The sched_domains_numa_distance[] array includes the actual distance
cb83b629
PZ
6449 * numbers.
6450 */
6451
5f7865f3
TC
6452 /*
6453 * Here, we should temporarily reset sched_domains_numa_levels to 0.
6454 * If it fails to allocate memory for array sched_domains_numa_masks[][],
6455 * the array will contain less then 'level' members. This could be
6456 * dangerous when we use it to iterate array sched_domains_numa_masks[][]
6457 * in other functions.
6458 *
6459 * We reset it to 'level' at the end of this function.
6460 */
6461 sched_domains_numa_levels = 0;
6462
cb83b629
PZ
6463 sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL);
6464 if (!sched_domains_numa_masks)
6465 return;
6466
6467 /*
6468 * Now for each level, construct a mask per node which contains all
6469 * cpus of nodes that are that many hops away from us.
6470 */
6471 for (i = 0; i < level; i++) {
6472 sched_domains_numa_masks[i] =
6473 kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL);
6474 if (!sched_domains_numa_masks[i])
6475 return;
6476
6477 for (j = 0; j < nr_node_ids; j++) {
2ea45800 6478 struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL);
cb83b629
PZ
6479 if (!mask)
6480 return;
6481
6482 sched_domains_numa_masks[i][j] = mask;
6483
6484 for (k = 0; k < nr_node_ids; k++) {
dd7d8634 6485 if (node_distance(j, k) > sched_domains_numa_distance[i])
cb83b629
PZ
6486 continue;
6487
6488 cpumask_or(mask, mask, cpumask_of_node(k));
6489 }
6490 }
6491 }
6492
143e1e28
VG
6493 /* Compute default topology size */
6494 for (i = 0; sched_domain_topology[i].mask; i++);
6495
c515db8c 6496 tl = kzalloc((i + level + 1) *
cb83b629
PZ
6497 sizeof(struct sched_domain_topology_level), GFP_KERNEL);
6498 if (!tl)
6499 return;
6500
6501 /*
6502 * Copy the default topology bits..
6503 */
143e1e28
VG
6504 for (i = 0; sched_domain_topology[i].mask; i++)
6505 tl[i] = sched_domain_topology[i];
cb83b629
PZ
6506
6507 /*
6508 * .. and append 'j' levels of NUMA goodness.
6509 */
6510 for (j = 0; j < level; i++, j++) {
6511 tl[i] = (struct sched_domain_topology_level){
cb83b629 6512 .mask = sd_numa_mask,
143e1e28 6513 .sd_flags = cpu_numa_flags,
cb83b629
PZ
6514 .flags = SDTL_OVERLAP,
6515 .numa_level = j,
143e1e28 6516 SD_INIT_NAME(NUMA)
cb83b629
PZ
6517 };
6518 }
6519
6520 sched_domain_topology = tl;
5f7865f3
TC
6521
6522 sched_domains_numa_levels = level;
9942f79b 6523 sched_max_numa_distance = sched_domains_numa_distance[level - 1];
e3fe70b1
RR
6524
6525 init_numa_topology_type();
cb83b629 6526}
301a5cba
TC
6527
6528static void sched_domains_numa_masks_set(int cpu)
6529{
6530 int i, j;
6531 int node = cpu_to_node(cpu);
6532
6533 for (i = 0; i < sched_domains_numa_levels; i++) {
6534 for (j = 0; j < nr_node_ids; j++) {
6535 if (node_distance(j, node) <= sched_domains_numa_distance[i])
6536 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
6537 }
6538 }
6539}
6540
6541static void sched_domains_numa_masks_clear(int cpu)
6542{
6543 int i, j;
6544 for (i = 0; i < sched_domains_numa_levels; i++) {
6545 for (j = 0; j < nr_node_ids; j++)
6546 cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
6547 }
6548}
6549
6550/*
6551 * Update sched_domains_numa_masks[level][node] array when new cpus
6552 * are onlined.
6553 */
6554static int sched_domains_numa_masks_update(struct notifier_block *nfb,
6555 unsigned long action,
6556 void *hcpu)
6557{
6558 int cpu = (long)hcpu;
6559
6560 switch (action & ~CPU_TASKS_FROZEN) {
6561 case CPU_ONLINE:
6562 sched_domains_numa_masks_set(cpu);
6563 break;
6564
6565 case CPU_DEAD:
6566 sched_domains_numa_masks_clear(cpu);
6567 break;
6568
6569 default:
6570 return NOTIFY_DONE;
6571 }
6572
6573 return NOTIFY_OK;
cb83b629
PZ
6574}
6575#else
6576static inline void sched_init_numa(void)
6577{
6578}
301a5cba
TC
6579
6580static int sched_domains_numa_masks_update(struct notifier_block *nfb,
6581 unsigned long action,
6582 void *hcpu)
6583{
6584 return 0;
6585}
cb83b629
PZ
6586#endif /* CONFIG_NUMA */
6587
54ab4ff4
PZ
6588static int __sdt_alloc(const struct cpumask *cpu_map)
6589{
6590 struct sched_domain_topology_level *tl;
6591 int j;
6592
27723a68 6593 for_each_sd_topology(tl) {
54ab4ff4
PZ
6594 struct sd_data *sdd = &tl->data;
6595
6596 sdd->sd = alloc_percpu(struct sched_domain *);
6597 if (!sdd->sd)
6598 return -ENOMEM;
6599
6600 sdd->sg = alloc_percpu(struct sched_group *);
6601 if (!sdd->sg)
6602 return -ENOMEM;
6603
63b2ca30
NP
6604 sdd->sgc = alloc_percpu(struct sched_group_capacity *);
6605 if (!sdd->sgc)
9c3f75cb
PZ
6606 return -ENOMEM;
6607
54ab4ff4
PZ
6608 for_each_cpu(j, cpu_map) {
6609 struct sched_domain *sd;
6610 struct sched_group *sg;
63b2ca30 6611 struct sched_group_capacity *sgc;
54ab4ff4
PZ
6612
6613 sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
6614 GFP_KERNEL, cpu_to_node(j));
6615 if (!sd)
6616 return -ENOMEM;
6617
6618 *per_cpu_ptr(sdd->sd, j) = sd;
6619
6620 sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
6621 GFP_KERNEL, cpu_to_node(j));
6622 if (!sg)
6623 return -ENOMEM;
6624
30b4e9eb
IM
6625 sg->next = sg;
6626
54ab4ff4 6627 *per_cpu_ptr(sdd->sg, j) = sg;
9c3f75cb 6628
63b2ca30 6629 sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
9c3f75cb 6630 GFP_KERNEL, cpu_to_node(j));
63b2ca30 6631 if (!sgc)
9c3f75cb
PZ
6632 return -ENOMEM;
6633
63b2ca30 6634 *per_cpu_ptr(sdd->sgc, j) = sgc;
54ab4ff4
PZ
6635 }
6636 }
6637
6638 return 0;
6639}
6640
6641static void __sdt_free(const struct cpumask *cpu_map)
6642{
6643 struct sched_domain_topology_level *tl;
6644 int j;
6645
27723a68 6646 for_each_sd_topology(tl) {
54ab4ff4
PZ
6647 struct sd_data *sdd = &tl->data;
6648
6649 for_each_cpu(j, cpu_map) {
fb2cf2c6 6650 struct sched_domain *sd;
6651
6652 if (sdd->sd) {
6653 sd = *per_cpu_ptr(sdd->sd, j);
6654 if (sd && (sd->flags & SD_OVERLAP))
6655 free_sched_groups(sd->groups, 0);
6656 kfree(*per_cpu_ptr(sdd->sd, j));
6657 }
6658
6659 if (sdd->sg)
6660 kfree(*per_cpu_ptr(sdd->sg, j));
63b2ca30
NP
6661 if (sdd->sgc)
6662 kfree(*per_cpu_ptr(sdd->sgc, j));
54ab4ff4
PZ
6663 }
6664 free_percpu(sdd->sd);
fb2cf2c6 6665 sdd->sd = NULL;
54ab4ff4 6666 free_percpu(sdd->sg);
fb2cf2c6 6667 sdd->sg = NULL;
63b2ca30
NP
6668 free_percpu(sdd->sgc);
6669 sdd->sgc = NULL;
54ab4ff4
PZ
6670 }
6671}
6672
2c402dc3 6673struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
4a850cbe
VK
6674 const struct cpumask *cpu_map, struct sched_domain_attr *attr,
6675 struct sched_domain *child, int cpu)
2c402dc3 6676{
143e1e28 6677 struct sched_domain *sd = sd_init(tl, cpu);
2c402dc3 6678 if (!sd)
d069b916 6679 return child;
2c402dc3 6680
2c402dc3 6681 cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu));
60495e77
PZ
6682 if (child) {
6683 sd->level = child->level + 1;
6684 sched_domain_level_max = max(sched_domain_level_max, sd->level);
d069b916 6685 child->parent = sd;
c75e0128 6686 sd->child = child;
6ae72dff
PZ
6687
6688 if (!cpumask_subset(sched_domain_span(child),
6689 sched_domain_span(sd))) {
6690 pr_err("BUG: arch topology borken\n");
6691#ifdef CONFIG_SCHED_DEBUG
6692 pr_err(" the %s domain not a subset of the %s domain\n",
6693 child->name, sd->name);
6694#endif
6695 /* Fixup, ensure @sd has at least @child cpus. */
6696 cpumask_or(sched_domain_span(sd),
6697 sched_domain_span(sd),
6698 sched_domain_span(child));
6699 }
6700
60495e77 6701 }
a841f8ce 6702 set_domain_attribute(sd, attr);
2c402dc3
PZ
6703
6704 return sd;
6705}
6706
2109b99e
AH
6707/*
6708 * Build sched domains for a given set of cpus and attach the sched domains
6709 * to the individual cpus
6710 */
dce840a0
PZ
6711static int build_sched_domains(const struct cpumask *cpu_map,
6712 struct sched_domain_attr *attr)
2109b99e 6713{
1c632169 6714 enum s_alloc alloc_state;
dce840a0 6715 struct sched_domain *sd;
2109b99e 6716 struct s_data d;
822ff793 6717 int i, ret = -ENOMEM;
9c1cfda2 6718
2109b99e
AH
6719 alloc_state = __visit_domain_allocation_hell(&d, cpu_map);
6720 if (alloc_state != sa_rootdomain)
6721 goto error;
9c1cfda2 6722
dce840a0 6723 /* Set up domains for cpus specified by the cpu_map. */
abcd083a 6724 for_each_cpu(i, cpu_map) {
eb7a74e6
PZ
6725 struct sched_domain_topology_level *tl;
6726
3bd65a80 6727 sd = NULL;
27723a68 6728 for_each_sd_topology(tl) {
4a850cbe 6729 sd = build_sched_domain(tl, cpu_map, attr, sd, i);
22da9569
VK
6730 if (tl == sched_domain_topology)
6731 *per_cpu_ptr(d.sd, i) = sd;
e3589f6c
PZ
6732 if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP))
6733 sd->flags |= SD_OVERLAP;
d110235d
PZ
6734 if (cpumask_equal(cpu_map, sched_domain_span(sd)))
6735 break;
e3589f6c 6736 }
dce840a0
PZ
6737 }
6738
6739 /* Build the groups for the domains */
6740 for_each_cpu(i, cpu_map) {
6741 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
6742 sd->span_weight = cpumask_weight(sched_domain_span(sd));
e3589f6c
PZ
6743 if (sd->flags & SD_OVERLAP) {
6744 if (build_overlap_sched_groups(sd, i))
6745 goto error;
6746 } else {
6747 if (build_sched_groups(sd, i))
6748 goto error;
6749 }
1cf51902 6750 }
a06dadbe 6751 }
9c1cfda2 6752
ced549fa 6753 /* Calculate CPU capacity for physical packages and nodes */
a9c9a9b6
PZ
6754 for (i = nr_cpumask_bits-1; i >= 0; i--) {
6755 if (!cpumask_test_cpu(i, cpu_map))
6756 continue;
9c1cfda2 6757
dce840a0
PZ
6758 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
6759 claim_allocations(i, sd);
63b2ca30 6760 init_sched_groups_capacity(i, sd);
dce840a0 6761 }
f712c0c7 6762 }
9c1cfda2 6763
1da177e4 6764 /* Attach the domains */
dce840a0 6765 rcu_read_lock();
abcd083a 6766 for_each_cpu(i, cpu_map) {
21d42ccf 6767 sd = *per_cpu_ptr(d.sd, i);
49a02c51 6768 cpu_attach_domain(sd, d.rd, i);
1da177e4 6769 }
dce840a0 6770 rcu_read_unlock();
51888ca2 6771
822ff793 6772 ret = 0;
51888ca2 6773error:
2109b99e 6774 __free_domain_allocs(&d, alloc_state, cpu_map);
822ff793 6775 return ret;
1da177e4 6776}
029190c5 6777
acc3f5d7 6778static cpumask_var_t *doms_cur; /* current sched domains */
029190c5 6779static int ndoms_cur; /* number of sched domains in 'doms_cur' */
4285f594
IM
6780static struct sched_domain_attr *dattr_cur;
6781 /* attribues of custom domains in 'doms_cur' */
029190c5
PJ
6782
6783/*
6784 * Special case: If a kmalloc of a doms_cur partition (array of
4212823f
RR
6785 * cpumask) fails, then fallback to a single sched domain,
6786 * as determined by the single cpumask fallback_doms.
029190c5 6787 */
4212823f 6788static cpumask_var_t fallback_doms;
029190c5 6789
ee79d1bd
HC
6790/*
6791 * arch_update_cpu_topology lets virtualized architectures update the
6792 * cpu core maps. It is supposed to return 1 if the topology changed
6793 * or 0 if it stayed the same.
6794 */
52f5684c 6795int __weak arch_update_cpu_topology(void)
22e52b07 6796{
ee79d1bd 6797 return 0;
22e52b07
HC
6798}
6799
acc3f5d7
RR
6800cpumask_var_t *alloc_sched_domains(unsigned int ndoms)
6801{
6802 int i;
6803 cpumask_var_t *doms;
6804
6805 doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL);
6806 if (!doms)
6807 return NULL;
6808 for (i = 0; i < ndoms; i++) {
6809 if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) {
6810 free_sched_domains(doms, i);
6811 return NULL;
6812 }
6813 }
6814 return doms;
6815}
6816
6817void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
6818{
6819 unsigned int i;
6820 for (i = 0; i < ndoms; i++)
6821 free_cpumask_var(doms[i]);
6822 kfree(doms);
6823}
6824
1a20ff27 6825/*
41a2d6cf 6826 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
029190c5
PJ
6827 * For now this just excludes isolated cpus, but could be used to
6828 * exclude other special cases in the future.
1a20ff27 6829 */
c4a8849a 6830static int init_sched_domains(const struct cpumask *cpu_map)
1a20ff27 6831{
7378547f
MM
6832 int err;
6833
22e52b07 6834 arch_update_cpu_topology();
029190c5 6835 ndoms_cur = 1;
acc3f5d7 6836 doms_cur = alloc_sched_domains(ndoms_cur);
029190c5 6837 if (!doms_cur)
acc3f5d7
RR
6838 doms_cur = &fallback_doms;
6839 cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map);
dce840a0 6840 err = build_sched_domains(doms_cur[0], NULL);
6382bc90 6841 register_sched_domain_sysctl();
7378547f
MM
6842
6843 return err;
1a20ff27
DG
6844}
6845
1a20ff27
DG
6846/*
6847 * Detach sched domains from a group of cpus specified in cpu_map
6848 * These cpus will now be attached to the NULL domain
6849 */
96f874e2 6850static void detach_destroy_domains(const struct cpumask *cpu_map)
1a20ff27
DG
6851{
6852 int i;
6853
dce840a0 6854 rcu_read_lock();
abcd083a 6855 for_each_cpu(i, cpu_map)
57d885fe 6856 cpu_attach_domain(NULL, &def_root_domain, i);
dce840a0 6857 rcu_read_unlock();
1a20ff27
DG
6858}
6859
1d3504fc
HS
6860/* handle null as "default" */
6861static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
6862 struct sched_domain_attr *new, int idx_new)
6863{
6864 struct sched_domain_attr tmp;
6865
6866 /* fast path */
6867 if (!new && !cur)
6868 return 1;
6869
6870 tmp = SD_ATTR_INIT;
6871 return !memcmp(cur ? (cur + idx_cur) : &tmp,
6872 new ? (new + idx_new) : &tmp,
6873 sizeof(struct sched_domain_attr));
6874}
6875
029190c5
PJ
6876/*
6877 * Partition sched domains as specified by the 'ndoms_new'
41a2d6cf 6878 * cpumasks in the array doms_new[] of cpumasks. This compares
029190c5
PJ
6879 * doms_new[] to the current sched domain partitioning, doms_cur[].
6880 * It destroys each deleted domain and builds each new domain.
6881 *
acc3f5d7 6882 * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'.
41a2d6cf
IM
6883 * The masks don't intersect (don't overlap.) We should setup one
6884 * sched domain for each mask. CPUs not in any of the cpumasks will
6885 * not be load balanced. If the same cpumask appears both in the
029190c5
PJ
6886 * current 'doms_cur' domains and in the new 'doms_new', we can leave
6887 * it as it is.
6888 *
acc3f5d7
RR
6889 * The passed in 'doms_new' should be allocated using
6890 * alloc_sched_domains. This routine takes ownership of it and will
6891 * free_sched_domains it when done with it. If the caller failed the
6892 * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1,
6893 * and partition_sched_domains() will fallback to the single partition
6894 * 'fallback_doms', it also forces the domains to be rebuilt.
029190c5 6895 *
96f874e2 6896 * If doms_new == NULL it will be replaced with cpu_online_mask.
700018e0
LZ
6897 * ndoms_new == 0 is a special case for destroying existing domains,
6898 * and it will not create the default domain.
dfb512ec 6899 *
029190c5
PJ
6900 * Call with hotplug lock held
6901 */
acc3f5d7 6902void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
1d3504fc 6903 struct sched_domain_attr *dattr_new)
029190c5 6904{
dfb512ec 6905 int i, j, n;
d65bd5ec 6906 int new_topology;
029190c5 6907
712555ee 6908 mutex_lock(&sched_domains_mutex);
a1835615 6909
7378547f
MM
6910 /* always unregister in case we don't destroy any domains */
6911 unregister_sched_domain_sysctl();
6912
d65bd5ec
HC
6913 /* Let architecture update cpu core mappings. */
6914 new_topology = arch_update_cpu_topology();
6915
dfb512ec 6916 n = doms_new ? ndoms_new : 0;
029190c5
PJ
6917
6918 /* Destroy deleted domains */
6919 for (i = 0; i < ndoms_cur; i++) {
d65bd5ec 6920 for (j = 0; j < n && !new_topology; j++) {
acc3f5d7 6921 if (cpumask_equal(doms_cur[i], doms_new[j])
1d3504fc 6922 && dattrs_equal(dattr_cur, i, dattr_new, j))
029190c5
PJ
6923 goto match1;
6924 }
6925 /* no match - a current sched domain not in new doms_new[] */
acc3f5d7 6926 detach_destroy_domains(doms_cur[i]);
029190c5
PJ
6927match1:
6928 ;
6929 }
6930
c8d2d47a 6931 n = ndoms_cur;
e761b772 6932 if (doms_new == NULL) {
c8d2d47a 6933 n = 0;
acc3f5d7 6934 doms_new = &fallback_doms;
6ad4c188 6935 cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map);
faa2f98f 6936 WARN_ON_ONCE(dattr_new);
e761b772
MK
6937 }
6938
029190c5
PJ
6939 /* Build new domains */
6940 for (i = 0; i < ndoms_new; i++) {
c8d2d47a 6941 for (j = 0; j < n && !new_topology; j++) {
acc3f5d7 6942 if (cpumask_equal(doms_new[i], doms_cur[j])
1d3504fc 6943 && dattrs_equal(dattr_new, i, dattr_cur, j))
029190c5
PJ
6944 goto match2;
6945 }
6946 /* no match - add a new doms_new */
dce840a0 6947 build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
029190c5
PJ
6948match2:
6949 ;
6950 }
6951
6952 /* Remember the new sched domains */
acc3f5d7
RR
6953 if (doms_cur != &fallback_doms)
6954 free_sched_domains(doms_cur, ndoms_cur);
1d3504fc 6955 kfree(dattr_cur); /* kfree(NULL) is safe */
029190c5 6956 doms_cur = doms_new;
1d3504fc 6957 dattr_cur = dattr_new;
029190c5 6958 ndoms_cur = ndoms_new;
7378547f
MM
6959
6960 register_sched_domain_sysctl();
a1835615 6961
712555ee 6962 mutex_unlock(&sched_domains_mutex);
029190c5
PJ
6963}
6964
d35be8ba
SB
6965static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */
6966
1da177e4 6967/*
3a101d05
TH
6968 * Update cpusets according to cpu_active mask. If cpusets are
6969 * disabled, cpuset_update_active_cpus() becomes a simple wrapper
6970 * around partition_sched_domains().
d35be8ba
SB
6971 *
6972 * If we come here as part of a suspend/resume, don't touch cpusets because we
6973 * want to restore it back to its original state upon resume anyway.
1da177e4 6974 */
0b2e918a
TH
6975static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action,
6976 void *hcpu)
e761b772 6977{
d35be8ba
SB
6978 switch (action) {
6979 case CPU_ONLINE_FROZEN:
6980 case CPU_DOWN_FAILED_FROZEN:
6981
6982 /*
6983 * num_cpus_frozen tracks how many CPUs are involved in suspend
6984 * resume sequence. As long as this is not the last online
6985 * operation in the resume sequence, just build a single sched
6986 * domain, ignoring cpusets.
6987 */
6988 num_cpus_frozen--;
6989 if (likely(num_cpus_frozen)) {
6990 partition_sched_domains(1, NULL, NULL);
6991 break;
6992 }
6993
6994 /*
6995 * This is the last CPU online operation. So fall through and
6996 * restore the original sched domains by considering the
6997 * cpuset configurations.
6998 */
6999
e761b772 7000 case CPU_ONLINE:
6ad4c188 7001 case CPU_DOWN_FAILED:
7ddf96b0 7002 cpuset_update_active_cpus(true);
d35be8ba 7003 break;
3a101d05
TH
7004 default:
7005 return NOTIFY_DONE;
7006 }
d35be8ba 7007 return NOTIFY_OK;
3a101d05 7008}
e761b772 7009
0b2e918a
TH
7010static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action,
7011 void *hcpu)
3a101d05 7012{
d35be8ba 7013 switch (action) {
3a101d05 7014 case CPU_DOWN_PREPARE:
7ddf96b0 7015 cpuset_update_active_cpus(false);
d35be8ba
SB
7016 break;
7017 case CPU_DOWN_PREPARE_FROZEN:
7018 num_cpus_frozen++;
7019 partition_sched_domains(1, NULL, NULL);
7020 break;
e761b772
MK
7021 default:
7022 return NOTIFY_DONE;
7023 }
d35be8ba 7024 return NOTIFY_OK;
e761b772 7025}
e761b772 7026
1da177e4
LT
7027void __init sched_init_smp(void)
7028{
dcc30a35
RR
7029 cpumask_var_t non_isolated_cpus;
7030
7031 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
cb5fd13f 7032 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
5c1e1767 7033
cb83b629
PZ
7034 sched_init_numa();
7035
6acce3ef
PZ
7036 /*
7037 * There's no userspace yet to cause hotplug operations; hence all the
7038 * cpu masks are stable and all blatant races in the below code cannot
7039 * happen.
7040 */
712555ee 7041 mutex_lock(&sched_domains_mutex);
c4a8849a 7042 init_sched_domains(cpu_active_mask);
dcc30a35
RR
7043 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
7044 if (cpumask_empty(non_isolated_cpus))
7045 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
712555ee 7046 mutex_unlock(&sched_domains_mutex);
e761b772 7047
301a5cba 7048 hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE);
3a101d05
TH
7049 hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE);
7050 hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE);
e761b772 7051
b328ca18 7052 init_hrtick();
5c1e1767
NP
7053
7054 /* Move init over to a non-isolated CPU */
dcc30a35 7055 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
5c1e1767 7056 BUG();
19978ca6 7057 sched_init_granularity();
dcc30a35 7058 free_cpumask_var(non_isolated_cpus);
4212823f 7059
0e3900e6 7060 init_sched_rt_class();
1baca4ce 7061 init_sched_dl_class();
1da177e4
LT
7062}
7063#else
7064void __init sched_init_smp(void)
7065{
19978ca6 7066 sched_init_granularity();
1da177e4
LT
7067}
7068#endif /* CONFIG_SMP */
7069
cd1bb94b
AB
7070const_debug unsigned int sysctl_timer_migration = 1;
7071
1da177e4
LT
7072int in_sched_functions(unsigned long addr)
7073{
1da177e4
LT
7074 return in_lock_functions(addr) ||
7075 (addr >= (unsigned long)__sched_text_start
7076 && addr < (unsigned long)__sched_text_end);
7077}
7078
029632fb 7079#ifdef CONFIG_CGROUP_SCHED
27b4b931
LZ
7080/*
7081 * Default task group.
7082 * Every task in system belongs to this group at bootup.
7083 */
029632fb 7084struct task_group root_task_group;
35cf4e50 7085LIST_HEAD(task_groups);
052f1dc7 7086#endif
6f505b16 7087
e6252c3e 7088DECLARE_PER_CPU(cpumask_var_t, load_balance_mask);
6f505b16 7089
1da177e4
LT
7090void __init sched_init(void)
7091{
dd41f596 7092 int i, j;
434d53b0
MT
7093 unsigned long alloc_size = 0, ptr;
7094
7095#ifdef CONFIG_FAIR_GROUP_SCHED
7096 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7097#endif
7098#ifdef CONFIG_RT_GROUP_SCHED
7099 alloc_size += 2 * nr_cpu_ids * sizeof(void **);
7100#endif
434d53b0 7101 if (alloc_size) {
36b7b6d4 7102 ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
434d53b0
MT
7103
7104#ifdef CONFIG_FAIR_GROUP_SCHED
07e06b01 7105 root_task_group.se = (struct sched_entity **)ptr;
434d53b0
MT
7106 ptr += nr_cpu_ids * sizeof(void **);
7107
07e06b01 7108 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
434d53b0 7109 ptr += nr_cpu_ids * sizeof(void **);
eff766a6 7110
6d6bc0ad 7111#endif /* CONFIG_FAIR_GROUP_SCHED */
434d53b0 7112#ifdef CONFIG_RT_GROUP_SCHED
07e06b01 7113 root_task_group.rt_se = (struct sched_rt_entity **)ptr;
434d53b0
MT
7114 ptr += nr_cpu_ids * sizeof(void **);
7115
07e06b01 7116 root_task_group.rt_rq = (struct rt_rq **)ptr;
eff766a6
PZ
7117 ptr += nr_cpu_ids * sizeof(void **);
7118
6d6bc0ad 7119#endif /* CONFIG_RT_GROUP_SCHED */
b74e6278 7120 }
df7c8e84 7121#ifdef CONFIG_CPUMASK_OFFSTACK
b74e6278
AT
7122 for_each_possible_cpu(i) {
7123 per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
7124 cpumask_size(), GFP_KERNEL, cpu_to_node(i));
434d53b0 7125 }
b74e6278 7126#endif /* CONFIG_CPUMASK_OFFSTACK */
dd41f596 7127
332ac17e
DF
7128 init_rt_bandwidth(&def_rt_bandwidth,
7129 global_rt_period(), global_rt_runtime());
7130 init_dl_bandwidth(&def_dl_bandwidth,
1724813d 7131 global_rt_period(), global_rt_runtime());
332ac17e 7132
57d885fe
GH
7133#ifdef CONFIG_SMP
7134 init_defrootdomain();
7135#endif
7136
d0b27fa7 7137#ifdef CONFIG_RT_GROUP_SCHED
07e06b01 7138 init_rt_bandwidth(&root_task_group.rt_bandwidth,
d0b27fa7 7139 global_rt_period(), global_rt_runtime());
6d6bc0ad 7140#endif /* CONFIG_RT_GROUP_SCHED */
d0b27fa7 7141
7c941438 7142#ifdef CONFIG_CGROUP_SCHED
07e06b01
YZ
7143 list_add(&root_task_group.list, &task_groups);
7144 INIT_LIST_HEAD(&root_task_group.children);
f4d6f6c2 7145 INIT_LIST_HEAD(&root_task_group.siblings);
5091faa4 7146 autogroup_init(&init_task);
54c707e9 7147
7c941438 7148#endif /* CONFIG_CGROUP_SCHED */
6f505b16 7149
0a945022 7150 for_each_possible_cpu(i) {
70b97a7f 7151 struct rq *rq;
1da177e4
LT
7152
7153 rq = cpu_rq(i);
05fa785c 7154 raw_spin_lock_init(&rq->lock);
7897986b 7155 rq->nr_running = 0;
dce48a84
TG
7156 rq->calc_load_active = 0;
7157 rq->calc_load_update = jiffies + LOAD_FREQ;
acb5a9ba 7158 init_cfs_rq(&rq->cfs);
6f505b16 7159 init_rt_rq(&rq->rt, rq);
aab03e05 7160 init_dl_rq(&rq->dl, rq);
dd41f596 7161#ifdef CONFIG_FAIR_GROUP_SCHED
029632fb 7162 root_task_group.shares = ROOT_TASK_GROUP_LOAD;
6f505b16 7163 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list);
354d60c2 7164 /*
07e06b01 7165 * How much cpu bandwidth does root_task_group get?
354d60c2
DG
7166 *
7167 * In case of task-groups formed thr' the cgroup filesystem, it
7168 * gets 100% of the cpu resources in the system. This overall
7169 * system cpu resource is divided among the tasks of
07e06b01 7170 * root_task_group and its child task-groups in a fair manner,
354d60c2
DG
7171 * based on each entity's (task or task-group's) weight
7172 * (se->load.weight).
7173 *
07e06b01 7174 * In other words, if root_task_group has 10 tasks of weight
354d60c2
DG
7175 * 1024) and two child groups A0 and A1 (of weight 1024 each),
7176 * then A0's share of the cpu resource is:
7177 *
0d905bca 7178 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
354d60c2 7179 *
07e06b01
YZ
7180 * We achieve this by letting root_task_group's tasks sit
7181 * directly in rq->cfs (i.e root_task_group->se[] = NULL).
354d60c2 7182 */
ab84d31e 7183 init_cfs_bandwidth(&root_task_group.cfs_bandwidth);
07e06b01 7184 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL);
354d60c2
DG
7185#endif /* CONFIG_FAIR_GROUP_SCHED */
7186
7187 rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime;
052f1dc7 7188#ifdef CONFIG_RT_GROUP_SCHED
07e06b01 7189 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL);
dd41f596 7190#endif
1da177e4 7191
dd41f596
IM
7192 for (j = 0; j < CPU_LOAD_IDX_MAX; j++)
7193 rq->cpu_load[j] = 0;
fdf3e95d
VP
7194
7195 rq->last_load_update_tick = jiffies;
7196
1da177e4 7197#ifdef CONFIG_SMP
41c7ce9a 7198 rq->sd = NULL;
57d885fe 7199 rq->rd = NULL;
ca8ce3d0 7200 rq->cpu_capacity = SCHED_CAPACITY_SCALE;
3f029d3c 7201 rq->post_schedule = 0;
1da177e4 7202 rq->active_balance = 0;
dd41f596 7203 rq->next_balance = jiffies;
1da177e4 7204 rq->push_cpu = 0;
0a2966b4 7205 rq->cpu = i;
1f11eb6a 7206 rq->online = 0;
eae0c9df
MG
7207 rq->idle_stamp = 0;
7208 rq->avg_idle = 2*sysctl_sched_migration_cost;
9bd721c5 7209 rq->max_idle_balance_cost = sysctl_sched_migration_cost;
367456c7
PZ
7210
7211 INIT_LIST_HEAD(&rq->cfs_tasks);
7212
dc938520 7213 rq_attach_root(rq, &def_root_domain);
3451d024 7214#ifdef CONFIG_NO_HZ_COMMON
1c792db7 7215 rq->nohz_flags = 0;
83cd4fe2 7216#endif
265f22a9
FW
7217#ifdef CONFIG_NO_HZ_FULL
7218 rq->last_sched_tick = 0;
7219#endif
1da177e4 7220#endif
8f4d37ec 7221 init_rq_hrtick(rq);
1da177e4 7222 atomic_set(&rq->nr_iowait, 0);
1da177e4
LT
7223 }
7224
2dd73a4f 7225 set_load_weight(&init_task);
b50f60ce 7226
e107be36
AK
7227#ifdef CONFIG_PREEMPT_NOTIFIERS
7228 INIT_HLIST_HEAD(&init_task.preempt_notifiers);
7229#endif
7230
1da177e4
LT
7231 /*
7232 * The boot idle thread does lazy MMU switching as well:
7233 */
7234 atomic_inc(&init_mm.mm_count);
7235 enter_lazy_tlb(&init_mm, current);
7236
1b537c7d
YD
7237 /*
7238 * During early bootup we pretend to be a normal task:
7239 */
7240 current->sched_class = &fair_sched_class;
7241
1da177e4
LT
7242 /*
7243 * Make us the idle thread. Technically, schedule() should not be
7244 * called from this thread, however somewhere below it might be,
7245 * but because we are the idle thread, we just pick up running again
7246 * when this runqueue becomes "idle".
7247 */
7248 init_idle(current, smp_processor_id());
dce48a84
TG
7249
7250 calc_load_update = jiffies + LOAD_FREQ;
7251
bf4d83f6 7252#ifdef CONFIG_SMP
4cb98839 7253 zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT);
bdddd296
RR
7254 /* May be allocated at isolcpus cmdline parse time */
7255 if (cpu_isolated_map == NULL)
7256 zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT);
29d5e047 7257 idle_thread_set_boot_cpu();
a803f026 7258 set_cpu_rq_start_time();
029632fb
PZ
7259#endif
7260 init_sched_fair_class();
6a7b3dc3 7261
6892b75e 7262 scheduler_running = 1;
1da177e4
LT
7263}
7264
d902db1e 7265#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
e4aafea2
FW
7266static inline int preempt_count_equals(int preempt_offset)
7267{
234da7bc 7268 int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
e4aafea2 7269
4ba8216c 7270 return (nested == preempt_offset);
e4aafea2
FW
7271}
7272
d894837f 7273void __might_sleep(const char *file, int line, int preempt_offset)
1da177e4 7274{
8eb23b9f
PZ
7275 /*
7276 * Blocking primitives will set (and therefore destroy) current->state,
7277 * since we will exit with TASK_RUNNING make sure we enter with it,
7278 * otherwise we will destroy state.
7279 */
00845eb9 7280 WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
8eb23b9f
PZ
7281 "do not call blocking ops when !TASK_RUNNING; "
7282 "state=%lx set at [<%p>] %pS\n",
7283 current->state,
7284 (void *)current->task_state_change,
00845eb9 7285 (void *)current->task_state_change);
8eb23b9f 7286
3427445a
PZ
7287 ___might_sleep(file, line, preempt_offset);
7288}
7289EXPORT_SYMBOL(__might_sleep);
7290
7291void ___might_sleep(const char *file, int line, int preempt_offset)
1da177e4 7292{
1da177e4
LT
7293 static unsigned long prev_jiffy; /* ratelimiting */
7294
b3fbab05 7295 rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
db273be2
TG
7296 if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
7297 !is_idle_task(current)) ||
e4aafea2 7298 system_state != SYSTEM_RUNNING || oops_in_progress)
aef745fc
IM
7299 return;
7300 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
7301 return;
7302 prev_jiffy = jiffies;
7303
3df0fc5b
PZ
7304 printk(KERN_ERR
7305 "BUG: sleeping function called from invalid context at %s:%d\n",
7306 file, line);
7307 printk(KERN_ERR
7308 "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
7309 in_atomic(), irqs_disabled(),
7310 current->pid, current->comm);
aef745fc 7311
a8b686b3
ES
7312 if (task_stack_end_corrupted(current))
7313 printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
7314
aef745fc
IM
7315 debug_show_held_locks(current);
7316 if (irqs_disabled())
7317 print_irqtrace_events(current);
8f47b187
TG
7318#ifdef CONFIG_DEBUG_PREEMPT
7319 if (!preempt_count_equals(preempt_offset)) {
7320 pr_err("Preemption disabled at:");
7321 print_ip_sym(current->preempt_disable_ip);
7322 pr_cont("\n");
7323 }
7324#endif
aef745fc 7325 dump_stack();
1da177e4 7326}
3427445a 7327EXPORT_SYMBOL(___might_sleep);
1da177e4
LT
7328#endif
7329
7330#ifdef CONFIG_MAGIC_SYSRQ
3a5e4dc1
AK
7331static void normalize_task(struct rq *rq, struct task_struct *p)
7332{
da7a735e 7333 const struct sched_class *prev_class = p->sched_class;
d50dde5a
DF
7334 struct sched_attr attr = {
7335 .sched_policy = SCHED_NORMAL,
7336 };
da7a735e 7337 int old_prio = p->prio;
da0c1e65 7338 int queued;
3e51f33f 7339
da0c1e65
KT
7340 queued = task_on_rq_queued(p);
7341 if (queued)
4ca9b72b 7342 dequeue_task(rq, p, 0);
d50dde5a 7343 __setscheduler(rq, p, &attr);
da0c1e65 7344 if (queued) {
4ca9b72b 7345 enqueue_task(rq, p, 0);
8875125e 7346 resched_curr(rq);
3a5e4dc1 7347 }
da7a735e
PZ
7348
7349 check_class_changed(rq, p, prev_class, old_prio);
3a5e4dc1
AK
7350}
7351
1da177e4
LT
7352void normalize_rt_tasks(void)
7353{
a0f98a1c 7354 struct task_struct *g, *p;
1da177e4 7355 unsigned long flags;
70b97a7f 7356 struct rq *rq;
1da177e4 7357
3472eaa1 7358 read_lock(&tasklist_lock);
5d07f420 7359 for_each_process_thread(g, p) {
178be793
IM
7360 /*
7361 * Only normalize user tasks:
7362 */
3472eaa1 7363 if (p->flags & PF_KTHREAD)
178be793
IM
7364 continue;
7365
6cfb0d5d 7366 p->se.exec_start = 0;
6cfb0d5d 7367#ifdef CONFIG_SCHEDSTATS
41acab88
LDM
7368 p->se.statistics.wait_start = 0;
7369 p->se.statistics.sleep_start = 0;
7370 p->se.statistics.block_start = 0;
6cfb0d5d 7371#endif
dd41f596 7372
aab03e05 7373 if (!dl_task(p) && !rt_task(p)) {
dd41f596
IM
7374 /*
7375 * Renice negative nice level userspace
7376 * tasks back to 0:
7377 */
3472eaa1 7378 if (task_nice(p) < 0)
dd41f596 7379 set_user_nice(p, 0);
1da177e4 7380 continue;
dd41f596 7381 }
1da177e4 7382
3472eaa1 7383 rq = task_rq_lock(p, &flags);
178be793 7384 normalize_task(rq, p);
3472eaa1 7385 task_rq_unlock(rq, p, &flags);
5d07f420 7386 }
3472eaa1 7387 read_unlock(&tasklist_lock);
1da177e4
LT
7388}
7389
7390#endif /* CONFIG_MAGIC_SYSRQ */
1df5c10a 7391
67fc4e0c 7392#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
1df5c10a 7393/*
67fc4e0c 7394 * These functions are only useful for the IA64 MCA handling, or kdb.
1df5c10a
LT
7395 *
7396 * They can only be called when the whole system has been
7397 * stopped - every CPU needs to be quiescent, and no scheduling
7398 * activity can take place. Using them for anything else would
7399 * be a serious bug, and as a result, they aren't even visible
7400 * under any other configuration.
7401 */
7402
7403/**
7404 * curr_task - return the current task for a given cpu.
7405 * @cpu: the processor in question.
7406 *
7407 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
e69f6186
YB
7408 *
7409 * Return: The current task for @cpu.
1df5c10a 7410 */
36c8b586 7411struct task_struct *curr_task(int cpu)
1df5c10a
LT
7412{
7413 return cpu_curr(cpu);
7414}
7415
67fc4e0c
JW
7416#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
7417
7418#ifdef CONFIG_IA64
1df5c10a
LT
7419/**
7420 * set_curr_task - set the current task for a given cpu.
7421 * @cpu: the processor in question.
7422 * @p: the task pointer to set.
7423 *
7424 * Description: This function must only be used when non-maskable interrupts
41a2d6cf
IM
7425 * are serviced on a separate stack. It allows the architecture to switch the
7426 * notion of the current task on a cpu in a non-blocking manner. This function
1df5c10a
LT
7427 * must be called with all CPU's synchronized, and interrupts disabled, the
7428 * and caller must save the original value of the current task (see
7429 * curr_task() above) and restore that value before reenabling interrupts and
7430 * re-starting the system.
7431 *
7432 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
7433 */
36c8b586 7434void set_curr_task(int cpu, struct task_struct *p)
1df5c10a
LT
7435{
7436 cpu_curr(cpu) = p;
7437}
7438
7439#endif
29f59db3 7440
7c941438 7441#ifdef CONFIG_CGROUP_SCHED
029632fb
PZ
7442/* task_group_lock serializes the addition/removal of task groups */
7443static DEFINE_SPINLOCK(task_group_lock);
7444
bccbe08a
PZ
7445static void free_sched_group(struct task_group *tg)
7446{
7447 free_fair_sched_group(tg);
7448 free_rt_sched_group(tg);
e9aa1dd1 7449 autogroup_free(tg);
bccbe08a
PZ
7450 kfree(tg);
7451}
7452
7453/* allocate runqueue etc for a new task group */
ec7dc8ac 7454struct task_group *sched_create_group(struct task_group *parent)
bccbe08a
PZ
7455{
7456 struct task_group *tg;
bccbe08a
PZ
7457
7458 tg = kzalloc(sizeof(*tg), GFP_KERNEL);
7459 if (!tg)
7460 return ERR_PTR(-ENOMEM);
7461
ec7dc8ac 7462 if (!alloc_fair_sched_group(tg, parent))
bccbe08a
PZ
7463 goto err;
7464
ec7dc8ac 7465 if (!alloc_rt_sched_group(tg, parent))
bccbe08a
PZ
7466 goto err;
7467
ace783b9
LZ
7468 return tg;
7469
7470err:
7471 free_sched_group(tg);
7472 return ERR_PTR(-ENOMEM);
7473}
7474
7475void sched_online_group(struct task_group *tg, struct task_group *parent)
7476{
7477 unsigned long flags;
7478
8ed36996 7479 spin_lock_irqsave(&task_group_lock, flags);
6f505b16 7480 list_add_rcu(&tg->list, &task_groups);
f473aa5e
PZ
7481
7482 WARN_ON(!parent); /* root should already exist */
7483
7484 tg->parent = parent;
f473aa5e 7485 INIT_LIST_HEAD(&tg->children);
09f2724a 7486 list_add_rcu(&tg->siblings, &parent->children);
8ed36996 7487 spin_unlock_irqrestore(&task_group_lock, flags);
29f59db3
SV
7488}
7489
9b5b7751 7490/* rcu callback to free various structures associated with a task group */
6f505b16 7491static void free_sched_group_rcu(struct rcu_head *rhp)
29f59db3 7492{
29f59db3 7493 /* now it should be safe to free those cfs_rqs */
6f505b16 7494 free_sched_group(container_of(rhp, struct task_group, rcu));
29f59db3
SV
7495}
7496
9b5b7751 7497/* Destroy runqueue etc associated with a task group */
4cf86d77 7498void sched_destroy_group(struct task_group *tg)
ace783b9
LZ
7499{
7500 /* wait for possible concurrent references to cfs_rqs complete */
7501 call_rcu(&tg->rcu, free_sched_group_rcu);
7502}
7503
7504void sched_offline_group(struct task_group *tg)
29f59db3 7505{
8ed36996 7506 unsigned long flags;
9b5b7751 7507 int i;
29f59db3 7508
3d4b47b4
PZ
7509 /* end participation in shares distribution */
7510 for_each_possible_cpu(i)
bccbe08a 7511 unregister_fair_sched_group(tg, i);
3d4b47b4
PZ
7512
7513 spin_lock_irqsave(&task_group_lock, flags);
6f505b16 7514 list_del_rcu(&tg->list);
f473aa5e 7515 list_del_rcu(&tg->siblings);
8ed36996 7516 spin_unlock_irqrestore(&task_group_lock, flags);
29f59db3
SV
7517}
7518
9b5b7751 7519/* change task's runqueue when it moves between groups.
3a252015
IM
7520 * The caller of this function should have put the task in its new group
7521 * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to
7522 * reflect its new group.
9b5b7751
SV
7523 */
7524void sched_move_task(struct task_struct *tsk)
29f59db3 7525{
8323f26c 7526 struct task_group *tg;
da0c1e65 7527 int queued, running;
29f59db3
SV
7528 unsigned long flags;
7529 struct rq *rq;
7530
7531 rq = task_rq_lock(tsk, &flags);
7532
051a1d1a 7533 running = task_current(rq, tsk);
da0c1e65 7534 queued = task_on_rq_queued(tsk);
29f59db3 7535
da0c1e65 7536 if (queued)
29f59db3 7537 dequeue_task(rq, tsk, 0);
0e1f3483 7538 if (unlikely(running))
f3cd1c4e 7539 put_prev_task(rq, tsk);
29f59db3 7540
f7b8a47d
KT
7541 /*
7542 * All callers are synchronized by task_rq_lock(); we do not use RCU
7543 * which is pointless here. Thus, we pass "true" to task_css_check()
7544 * to prevent lockdep warnings.
7545 */
7546 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
8323f26c
PZ
7547 struct task_group, css);
7548 tg = autogroup_task_group(tsk, tg);
7549 tsk->sched_task_group = tg;
7550
810b3817 7551#ifdef CONFIG_FAIR_GROUP_SCHED
b2b5ce02 7552 if (tsk->sched_class->task_move_group)
da0c1e65 7553 tsk->sched_class->task_move_group(tsk, queued);
b2b5ce02 7554 else
810b3817 7555#endif
b2b5ce02 7556 set_task_rq(tsk, task_cpu(tsk));
810b3817 7557
0e1f3483
HS
7558 if (unlikely(running))
7559 tsk->sched_class->set_curr_task(rq);
da0c1e65 7560 if (queued)
371fd7e7 7561 enqueue_task(rq, tsk, 0);
29f59db3 7562
0122ec5b 7563 task_rq_unlock(rq, tsk, &flags);
29f59db3 7564}
7c941438 7565#endif /* CONFIG_CGROUP_SCHED */
29f59db3 7566
a790de99
PT
7567#ifdef CONFIG_RT_GROUP_SCHED
7568/*
7569 * Ensure that the real time constraints are schedulable.
7570 */
7571static DEFINE_MUTEX(rt_constraints_mutex);
9f0c1e56 7572
9a7e0b18
PZ
7573/* Must be called with tasklist_lock held */
7574static inline int tg_has_rt_tasks(struct task_group *tg)
b40b2e8e 7575{
9a7e0b18 7576 struct task_struct *g, *p;
b40b2e8e 7577
1fe89e1b
PZ
7578 /*
7579 * Autogroups do not have RT tasks; see autogroup_create().
7580 */
7581 if (task_group_is_autogroup(tg))
7582 return 0;
7583
5d07f420 7584 for_each_process_thread(g, p) {
8651c658 7585 if (rt_task(p) && task_group(p) == tg)
9a7e0b18 7586 return 1;
5d07f420 7587 }
b40b2e8e 7588
9a7e0b18
PZ
7589 return 0;
7590}
b40b2e8e 7591
9a7e0b18
PZ
7592struct rt_schedulable_data {
7593 struct task_group *tg;
7594 u64 rt_period;
7595 u64 rt_runtime;
7596};
b40b2e8e 7597
a790de99 7598static int tg_rt_schedulable(struct task_group *tg, void *data)
9a7e0b18
PZ
7599{
7600 struct rt_schedulable_data *d = data;
7601 struct task_group *child;
7602 unsigned long total, sum = 0;
7603 u64 period, runtime;
b40b2e8e 7604
9a7e0b18
PZ
7605 period = ktime_to_ns(tg->rt_bandwidth.rt_period);
7606 runtime = tg->rt_bandwidth.rt_runtime;
b40b2e8e 7607
9a7e0b18
PZ
7608 if (tg == d->tg) {
7609 period = d->rt_period;
7610 runtime = d->rt_runtime;
b40b2e8e 7611 }
b40b2e8e 7612
4653f803
PZ
7613 /*
7614 * Cannot have more runtime than the period.
7615 */
7616 if (runtime > period && runtime != RUNTIME_INF)
7617 return -EINVAL;
6f505b16 7618
4653f803
PZ
7619 /*
7620 * Ensure we don't starve existing RT tasks.
7621 */
9a7e0b18
PZ
7622 if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
7623 return -EBUSY;
6f505b16 7624
9a7e0b18 7625 total = to_ratio(period, runtime);
6f505b16 7626
4653f803
PZ
7627 /*
7628 * Nobody can have more than the global setting allows.
7629 */
7630 if (total > to_ratio(global_rt_period(), global_rt_runtime()))
7631 return -EINVAL;
6f505b16 7632
4653f803
PZ
7633 /*
7634 * The sum of our children's runtime should not exceed our own.
7635 */
9a7e0b18
PZ
7636 list_for_each_entry_rcu(child, &tg->children, siblings) {
7637 period = ktime_to_ns(child->rt_bandwidth.rt_period);
7638 runtime = child->rt_bandwidth.rt_runtime;
6f505b16 7639
9a7e0b18
PZ
7640 if (child == d->tg) {
7641 period = d->rt_period;
7642 runtime = d->rt_runtime;
7643 }
6f505b16 7644
9a7e0b18 7645 sum += to_ratio(period, runtime);
9f0c1e56 7646 }
6f505b16 7647
9a7e0b18
PZ
7648 if (sum > total)
7649 return -EINVAL;
7650
7651 return 0;
6f505b16
PZ
7652}
7653
9a7e0b18 7654static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
521f1a24 7655{
8277434e
PT
7656 int ret;
7657
9a7e0b18
PZ
7658 struct rt_schedulable_data data = {
7659 .tg = tg,
7660 .rt_period = period,
7661 .rt_runtime = runtime,
7662 };
7663
8277434e
PT
7664 rcu_read_lock();
7665 ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
7666 rcu_read_unlock();
7667
7668 return ret;
521f1a24
DG
7669}
7670
ab84d31e 7671static int tg_set_rt_bandwidth(struct task_group *tg,
d0b27fa7 7672 u64 rt_period, u64 rt_runtime)
6f505b16 7673{
ac086bc2 7674 int i, err = 0;
9f0c1e56 7675
2636ed5f
PZ
7676 /*
7677 * Disallowing the root group RT runtime is BAD, it would disallow the
7678 * kernel creating (and or operating) RT threads.
7679 */
7680 if (tg == &root_task_group && rt_runtime == 0)
7681 return -EINVAL;
7682
7683 /* No period doesn't make any sense. */
7684 if (rt_period == 0)
7685 return -EINVAL;
7686
9f0c1e56 7687 mutex_lock(&rt_constraints_mutex);
521f1a24 7688 read_lock(&tasklist_lock);
9a7e0b18
PZ
7689 err = __rt_schedulable(tg, rt_period, rt_runtime);
7690 if (err)
9f0c1e56 7691 goto unlock;
ac086bc2 7692
0986b11b 7693 raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
d0b27fa7
PZ
7694 tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
7695 tg->rt_bandwidth.rt_runtime = rt_runtime;
ac086bc2
PZ
7696
7697 for_each_possible_cpu(i) {
7698 struct rt_rq *rt_rq = tg->rt_rq[i];
7699
0986b11b 7700 raw_spin_lock(&rt_rq->rt_runtime_lock);
ac086bc2 7701 rt_rq->rt_runtime = rt_runtime;
0986b11b 7702 raw_spin_unlock(&rt_rq->rt_runtime_lock);
ac086bc2 7703 }
0986b11b 7704 raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
49246274 7705unlock:
521f1a24 7706 read_unlock(&tasklist_lock);
9f0c1e56
PZ
7707 mutex_unlock(&rt_constraints_mutex);
7708
7709 return err;
6f505b16
PZ
7710}
7711
25cc7da7 7712static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
d0b27fa7
PZ
7713{
7714 u64 rt_runtime, rt_period;
7715
7716 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
7717 rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
7718 if (rt_runtime_us < 0)
7719 rt_runtime = RUNTIME_INF;
7720
ab84d31e 7721 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
d0b27fa7
PZ
7722}
7723
25cc7da7 7724static long sched_group_rt_runtime(struct task_group *tg)
9f0c1e56
PZ
7725{
7726 u64 rt_runtime_us;
7727
d0b27fa7 7728 if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
9f0c1e56
PZ
7729 return -1;
7730
d0b27fa7 7731 rt_runtime_us = tg->rt_bandwidth.rt_runtime;
9f0c1e56
PZ
7732 do_div(rt_runtime_us, NSEC_PER_USEC);
7733 return rt_runtime_us;
7734}
d0b27fa7 7735
25cc7da7 7736static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us)
d0b27fa7
PZ
7737{
7738 u64 rt_runtime, rt_period;
7739
7740 rt_period = (u64)rt_period_us * NSEC_PER_USEC;
7741 rt_runtime = tg->rt_bandwidth.rt_runtime;
7742
ab84d31e 7743 return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
d0b27fa7
PZ
7744}
7745
25cc7da7 7746static long sched_group_rt_period(struct task_group *tg)
d0b27fa7
PZ
7747{
7748 u64 rt_period_us;
7749
7750 rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
7751 do_div(rt_period_us, NSEC_PER_USEC);
7752 return rt_period_us;
7753}
332ac17e 7754#endif /* CONFIG_RT_GROUP_SCHED */
d0b27fa7 7755
332ac17e 7756#ifdef CONFIG_RT_GROUP_SCHED
d0b27fa7
PZ
7757static int sched_rt_global_constraints(void)
7758{
7759 int ret = 0;
7760
7761 mutex_lock(&rt_constraints_mutex);
9a7e0b18 7762 read_lock(&tasklist_lock);
4653f803 7763 ret = __rt_schedulable(NULL, 0, 0);
9a7e0b18 7764 read_unlock(&tasklist_lock);
d0b27fa7
PZ
7765 mutex_unlock(&rt_constraints_mutex);
7766
7767 return ret;
7768}
54e99124 7769
25cc7da7 7770static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
54e99124
DG
7771{
7772 /* Don't accept realtime tasks when there is no way for them to run */
7773 if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
7774 return 0;
7775
7776 return 1;
7777}
7778
6d6bc0ad 7779#else /* !CONFIG_RT_GROUP_SCHED */
d0b27fa7
PZ
7780static int sched_rt_global_constraints(void)
7781{
ac086bc2 7782 unsigned long flags;
332ac17e 7783 int i, ret = 0;
ec5d4989 7784
0986b11b 7785 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
ac086bc2
PZ
7786 for_each_possible_cpu(i) {
7787 struct rt_rq *rt_rq = &cpu_rq(i)->rt;
7788
0986b11b 7789 raw_spin_lock(&rt_rq->rt_runtime_lock);
ac086bc2 7790 rt_rq->rt_runtime = global_rt_runtime();
0986b11b 7791 raw_spin_unlock(&rt_rq->rt_runtime_lock);
ac086bc2 7792 }
0986b11b 7793 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
ac086bc2 7794
332ac17e 7795 return ret;
d0b27fa7 7796}
6d6bc0ad 7797#endif /* CONFIG_RT_GROUP_SCHED */
d0b27fa7 7798
332ac17e
DF
7799static int sched_dl_global_constraints(void)
7800{
1724813d
PZ
7801 u64 runtime = global_rt_runtime();
7802 u64 period = global_rt_period();
332ac17e 7803 u64 new_bw = to_ratio(period, runtime);
f10e00f4 7804 struct dl_bw *dl_b;
1724813d 7805 int cpu, ret = 0;
49516342 7806 unsigned long flags;
332ac17e
DF
7807
7808 /*
7809 * Here we want to check the bandwidth not being set to some
7810 * value smaller than the currently allocated bandwidth in
7811 * any of the root_domains.
7812 *
7813 * FIXME: Cycling on all the CPUs is overdoing, but simpler than
7814 * cycling on root_domains... Discussion on different/better
7815 * solutions is welcome!
7816 */
1724813d 7817 for_each_possible_cpu(cpu) {
f10e00f4
KT
7818 rcu_read_lock_sched();
7819 dl_b = dl_bw_of(cpu);
332ac17e 7820
49516342 7821 raw_spin_lock_irqsave(&dl_b->lock, flags);
1724813d
PZ
7822 if (new_bw < dl_b->total_bw)
7823 ret = -EBUSY;
49516342 7824 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
1724813d 7825
f10e00f4
KT
7826 rcu_read_unlock_sched();
7827
1724813d
PZ
7828 if (ret)
7829 break;
332ac17e
DF
7830 }
7831
1724813d 7832 return ret;
332ac17e
DF
7833}
7834
1724813d 7835static void sched_dl_do_global(void)
ce0dbbbb 7836{
1724813d 7837 u64 new_bw = -1;
f10e00f4 7838 struct dl_bw *dl_b;
1724813d 7839 int cpu;
49516342 7840 unsigned long flags;
ce0dbbbb 7841
1724813d
PZ
7842 def_dl_bandwidth.dl_period = global_rt_period();
7843 def_dl_bandwidth.dl_runtime = global_rt_runtime();
7844
7845 if (global_rt_runtime() != RUNTIME_INF)
7846 new_bw = to_ratio(global_rt_period(), global_rt_runtime());
7847
7848 /*
7849 * FIXME: As above...
7850 */
7851 for_each_possible_cpu(cpu) {
f10e00f4
KT
7852 rcu_read_lock_sched();
7853 dl_b = dl_bw_of(cpu);
1724813d 7854
49516342 7855 raw_spin_lock_irqsave(&dl_b->lock, flags);
1724813d 7856 dl_b->bw = new_bw;
49516342 7857 raw_spin_unlock_irqrestore(&dl_b->lock, flags);
f10e00f4
KT
7858
7859 rcu_read_unlock_sched();
ce0dbbbb 7860 }
1724813d
PZ
7861}
7862
7863static int sched_rt_global_validate(void)
7864{
7865 if (sysctl_sched_rt_period <= 0)
7866 return -EINVAL;
7867
e9e7cb38
JL
7868 if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
7869 (sysctl_sched_rt_runtime > sysctl_sched_rt_period))
1724813d
PZ
7870 return -EINVAL;
7871
7872 return 0;
7873}
7874
7875static void sched_rt_do_global(void)
7876{
7877 def_rt_bandwidth.rt_runtime = global_rt_runtime();
7878 def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
ce0dbbbb
CW
7879}
7880
d0b27fa7 7881int sched_rt_handler(struct ctl_table *table, int write,
8d65af78 7882 void __user *buffer, size_t *lenp,
d0b27fa7
PZ
7883 loff_t *ppos)
7884{
d0b27fa7
PZ
7885 int old_period, old_runtime;
7886 static DEFINE_MUTEX(mutex);
1724813d 7887 int ret;
d0b27fa7
PZ
7888
7889 mutex_lock(&mutex);
7890 old_period = sysctl_sched_rt_period;
7891 old_runtime = sysctl_sched_rt_runtime;
7892
8d65af78 7893 ret = proc_dointvec(table, write, buffer, lenp, ppos);
d0b27fa7
PZ
7894
7895 if (!ret && write) {
1724813d
PZ
7896 ret = sched_rt_global_validate();
7897 if (ret)
7898 goto undo;
7899
d0b27fa7 7900 ret = sched_rt_global_constraints();
1724813d
PZ
7901 if (ret)
7902 goto undo;
7903
7904 ret = sched_dl_global_constraints();
7905 if (ret)
7906 goto undo;
7907
7908 sched_rt_do_global();
7909 sched_dl_do_global();
7910 }
7911 if (0) {
7912undo:
7913 sysctl_sched_rt_period = old_period;
7914 sysctl_sched_rt_runtime = old_runtime;
d0b27fa7
PZ
7915 }
7916 mutex_unlock(&mutex);
7917
7918 return ret;
7919}
68318b8e 7920
1724813d 7921int sched_rr_handler(struct ctl_table *table, int write,
332ac17e
DF
7922 void __user *buffer, size_t *lenp,
7923 loff_t *ppos)
7924{
7925 int ret;
332ac17e 7926 static DEFINE_MUTEX(mutex);
332ac17e
DF
7927
7928 mutex_lock(&mutex);
332ac17e 7929 ret = proc_dointvec(table, write, buffer, lenp, ppos);
1724813d
PZ
7930 /* make sure that internally we keep jiffies */
7931 /* also, writing zero resets timeslice to default */
332ac17e 7932 if (!ret && write) {
1724813d
PZ
7933 sched_rr_timeslice = sched_rr_timeslice <= 0 ?
7934 RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice);
332ac17e
DF
7935 }
7936 mutex_unlock(&mutex);
332ac17e
DF
7937 return ret;
7938}
7939
052f1dc7 7940#ifdef CONFIG_CGROUP_SCHED
68318b8e 7941
a7c6d554 7942static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
68318b8e 7943{
a7c6d554 7944 return css ? container_of(css, struct task_group, css) : NULL;
68318b8e
SV
7945}
7946
eb95419b
TH
7947static struct cgroup_subsys_state *
7948cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
68318b8e 7949{
eb95419b
TH
7950 struct task_group *parent = css_tg(parent_css);
7951 struct task_group *tg;
68318b8e 7952
eb95419b 7953 if (!parent) {
68318b8e 7954 /* This is early initialization for the top cgroup */
07e06b01 7955 return &root_task_group.css;
68318b8e
SV
7956 }
7957
ec7dc8ac 7958 tg = sched_create_group(parent);
68318b8e
SV
7959 if (IS_ERR(tg))
7960 return ERR_PTR(-ENOMEM);
7961
68318b8e
SV
7962 return &tg->css;
7963}
7964
eb95419b 7965static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
ace783b9 7966{
eb95419b 7967 struct task_group *tg = css_tg(css);
5c9d535b 7968 struct task_group *parent = css_tg(css->parent);
ace783b9 7969
63876986
TH
7970 if (parent)
7971 sched_online_group(tg, parent);
ace783b9
LZ
7972 return 0;
7973}
7974
eb95419b 7975static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
68318b8e 7976{
eb95419b 7977 struct task_group *tg = css_tg(css);
68318b8e
SV
7978
7979 sched_destroy_group(tg);
7980}
7981
eb95419b 7982static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
ace783b9 7983{
eb95419b 7984 struct task_group *tg = css_tg(css);
ace783b9
LZ
7985
7986 sched_offline_group(tg);
7987}
7988
eeb61e53
KT
7989static void cpu_cgroup_fork(struct task_struct *task)
7990{
7991 sched_move_task(task);
7992}
7993
eb95419b 7994static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css,
bb9d97b6 7995 struct cgroup_taskset *tset)
68318b8e 7996{
bb9d97b6
TH
7997 struct task_struct *task;
7998
924f0d9a 7999 cgroup_taskset_for_each(task, tset) {
b68aa230 8000#ifdef CONFIG_RT_GROUP_SCHED
eb95419b 8001 if (!sched_rt_can_attach(css_tg(css), task))
bb9d97b6 8002 return -EINVAL;
b68aa230 8003#else
bb9d97b6
TH
8004 /* We don't support RT-tasks being in separate groups */
8005 if (task->sched_class != &fair_sched_class)
8006 return -EINVAL;
b68aa230 8007#endif
bb9d97b6 8008 }
be367d09
BB
8009 return 0;
8010}
68318b8e 8011
eb95419b 8012static void cpu_cgroup_attach(struct cgroup_subsys_state *css,
bb9d97b6 8013 struct cgroup_taskset *tset)
68318b8e 8014{
bb9d97b6
TH
8015 struct task_struct *task;
8016
924f0d9a 8017 cgroup_taskset_for_each(task, tset)
bb9d97b6 8018 sched_move_task(task);
68318b8e
SV
8019}
8020
eb95419b
TH
8021static void cpu_cgroup_exit(struct cgroup_subsys_state *css,
8022 struct cgroup_subsys_state *old_css,
8023 struct task_struct *task)
068c5cc5
PZ
8024{
8025 /*
8026 * cgroup_exit() is called in the copy_process() failure path.
8027 * Ignore this case since the task hasn't ran yet, this avoids
8028 * trying to poke a half freed task state from generic code.
8029 */
8030 if (!(task->flags & PF_EXITING))
8031 return;
8032
8033 sched_move_task(task);
8034}
8035
052f1dc7 8036#ifdef CONFIG_FAIR_GROUP_SCHED
182446d0
TH
8037static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
8038 struct cftype *cftype, u64 shareval)
68318b8e 8039{
182446d0 8040 return sched_group_set_shares(css_tg(css), scale_load(shareval));
68318b8e
SV
8041}
8042
182446d0
TH
8043static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
8044 struct cftype *cft)
68318b8e 8045{
182446d0 8046 struct task_group *tg = css_tg(css);
68318b8e 8047
c8b28116 8048 return (u64) scale_load_down(tg->shares);
68318b8e 8049}
ab84d31e
PT
8050
8051#ifdef CONFIG_CFS_BANDWIDTH
a790de99
PT
8052static DEFINE_MUTEX(cfs_constraints_mutex);
8053
ab84d31e
PT
8054const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */
8055const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */
8056
a790de99
PT
8057static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
8058
ab84d31e
PT
8059static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota)
8060{
56f570e5 8061 int i, ret = 0, runtime_enabled, runtime_was_enabled;
029632fb 8062 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
ab84d31e
PT
8063
8064 if (tg == &root_task_group)
8065 return -EINVAL;
8066
8067 /*
8068 * Ensure we have at some amount of bandwidth every period. This is
8069 * to prevent reaching a state of large arrears when throttled via
8070 * entity_tick() resulting in prolonged exit starvation.
8071 */
8072 if (quota < min_cfs_quota_period || period < min_cfs_quota_period)
8073 return -EINVAL;
8074
8075 /*
8076 * Likewise, bound things on the otherside by preventing insane quota
8077 * periods. This also allows us to normalize in computing quota
8078 * feasibility.
8079 */
8080 if (period > max_cfs_quota_period)
8081 return -EINVAL;
8082
0e59bdae
KT
8083 /*
8084 * Prevent race between setting of cfs_rq->runtime_enabled and
8085 * unthrottle_offline_cfs_rqs().
8086 */
8087 get_online_cpus();
a790de99
PT
8088 mutex_lock(&cfs_constraints_mutex);
8089 ret = __cfs_schedulable(tg, period, quota);
8090 if (ret)
8091 goto out_unlock;
8092
58088ad0 8093 runtime_enabled = quota != RUNTIME_INF;
56f570e5 8094 runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
1ee14e6c
BS
8095 /*
8096 * If we need to toggle cfs_bandwidth_used, off->on must occur
8097 * before making related changes, and on->off must occur afterwards
8098 */
8099 if (runtime_enabled && !runtime_was_enabled)
8100 cfs_bandwidth_usage_inc();
ab84d31e
PT
8101 raw_spin_lock_irq(&cfs_b->lock);
8102 cfs_b->period = ns_to_ktime(period);
8103 cfs_b->quota = quota;
58088ad0 8104
a9cf55b2 8105 __refill_cfs_bandwidth_runtime(cfs_b);
58088ad0
PT
8106 /* restart the period timer (if active) to handle new period expiry */
8107 if (runtime_enabled && cfs_b->timer_active) {
8108 /* force a reprogram */
09dc4ab0 8109 __start_cfs_bandwidth(cfs_b, true);
58088ad0 8110 }
ab84d31e
PT
8111 raw_spin_unlock_irq(&cfs_b->lock);
8112
0e59bdae 8113 for_each_online_cpu(i) {
ab84d31e 8114 struct cfs_rq *cfs_rq = tg->cfs_rq[i];
029632fb 8115 struct rq *rq = cfs_rq->rq;
ab84d31e
PT
8116
8117 raw_spin_lock_irq(&rq->lock);
58088ad0 8118 cfs_rq->runtime_enabled = runtime_enabled;
ab84d31e 8119 cfs_rq->runtime_remaining = 0;
671fd9da 8120
029632fb 8121 if (cfs_rq->throttled)
671fd9da 8122 unthrottle_cfs_rq(cfs_rq);
ab84d31e
PT
8123 raw_spin_unlock_irq(&rq->lock);
8124 }
1ee14e6c
BS
8125 if (runtime_was_enabled && !runtime_enabled)
8126 cfs_bandwidth_usage_dec();
a790de99
PT
8127out_unlock:
8128 mutex_unlock(&cfs_constraints_mutex);
0e59bdae 8129 put_online_cpus();
ab84d31e 8130
a790de99 8131 return ret;
ab84d31e
PT
8132}
8133
8134int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
8135{
8136 u64 quota, period;
8137
029632fb 8138 period = ktime_to_ns(tg->cfs_bandwidth.period);
ab84d31e
PT
8139 if (cfs_quota_us < 0)
8140 quota = RUNTIME_INF;
8141 else
8142 quota = (u64)cfs_quota_us * NSEC_PER_USEC;
8143
8144 return tg_set_cfs_bandwidth(tg, period, quota);
8145}
8146
8147long tg_get_cfs_quota(struct task_group *tg)
8148{
8149 u64 quota_us;
8150
029632fb 8151 if (tg->cfs_bandwidth.quota == RUNTIME_INF)
ab84d31e
PT
8152 return -1;
8153
029632fb 8154 quota_us = tg->cfs_bandwidth.quota;
ab84d31e
PT
8155 do_div(quota_us, NSEC_PER_USEC);
8156
8157 return quota_us;
8158}
8159
8160int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
8161{
8162 u64 quota, period;
8163
8164 period = (u64)cfs_period_us * NSEC_PER_USEC;
029632fb 8165 quota = tg->cfs_bandwidth.quota;
ab84d31e 8166
ab84d31e
PT
8167 return tg_set_cfs_bandwidth(tg, period, quota);
8168}
8169
8170long tg_get_cfs_period(struct task_group *tg)
8171{
8172 u64 cfs_period_us;
8173
029632fb 8174 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period);
ab84d31e
PT
8175 do_div(cfs_period_us, NSEC_PER_USEC);
8176
8177 return cfs_period_us;
8178}
8179
182446d0
TH
8180static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
8181 struct cftype *cft)
ab84d31e 8182{
182446d0 8183 return tg_get_cfs_quota(css_tg(css));
ab84d31e
PT
8184}
8185
182446d0
TH
8186static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
8187 struct cftype *cftype, s64 cfs_quota_us)
ab84d31e 8188{
182446d0 8189 return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
ab84d31e
PT
8190}
8191
182446d0
TH
8192static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
8193 struct cftype *cft)
ab84d31e 8194{
182446d0 8195 return tg_get_cfs_period(css_tg(css));
ab84d31e
PT
8196}
8197
182446d0
TH
8198static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
8199 struct cftype *cftype, u64 cfs_period_us)
ab84d31e 8200{
182446d0 8201 return tg_set_cfs_period(css_tg(css), cfs_period_us);
ab84d31e
PT
8202}
8203
a790de99
PT
8204struct cfs_schedulable_data {
8205 struct task_group *tg;
8206 u64 period, quota;
8207};
8208
8209/*
8210 * normalize group quota/period to be quota/max_period
8211 * note: units are usecs
8212 */
8213static u64 normalize_cfs_quota(struct task_group *tg,
8214 struct cfs_schedulable_data *d)
8215{
8216 u64 quota, period;
8217
8218 if (tg == d->tg) {
8219 period = d->period;
8220 quota = d->quota;
8221 } else {
8222 period = tg_get_cfs_period(tg);
8223 quota = tg_get_cfs_quota(tg);
8224 }
8225
8226 /* note: these should typically be equivalent */
8227 if (quota == RUNTIME_INF || quota == -1)
8228 return RUNTIME_INF;
8229
8230 return to_ratio(period, quota);
8231}
8232
8233static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
8234{
8235 struct cfs_schedulable_data *d = data;
029632fb 8236 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
a790de99
PT
8237 s64 quota = 0, parent_quota = -1;
8238
8239 if (!tg->parent) {
8240 quota = RUNTIME_INF;
8241 } else {
029632fb 8242 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
a790de99
PT
8243
8244 quota = normalize_cfs_quota(tg, d);
9c58c79a 8245 parent_quota = parent_b->hierarchical_quota;
a790de99
PT
8246
8247 /*
8248 * ensure max(child_quota) <= parent_quota, inherit when no
8249 * limit is set
8250 */
8251 if (quota == RUNTIME_INF)
8252 quota = parent_quota;
8253 else if (parent_quota != RUNTIME_INF && quota > parent_quota)
8254 return -EINVAL;
8255 }
9c58c79a 8256 cfs_b->hierarchical_quota = quota;
a790de99
PT
8257
8258 return 0;
8259}
8260
8261static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
8262{
8277434e 8263 int ret;
a790de99
PT
8264 struct cfs_schedulable_data data = {
8265 .tg = tg,
8266 .period = period,
8267 .quota = quota,
8268 };
8269
8270 if (quota != RUNTIME_INF) {
8271 do_div(data.period, NSEC_PER_USEC);
8272 do_div(data.quota, NSEC_PER_USEC);
8273 }
8274
8277434e
PT
8275 rcu_read_lock();
8276 ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data);
8277 rcu_read_unlock();
8278
8279 return ret;
a790de99 8280}
e8da1b18 8281
2da8ca82 8282static int cpu_stats_show(struct seq_file *sf, void *v)
e8da1b18 8283{
2da8ca82 8284 struct task_group *tg = css_tg(seq_css(sf));
029632fb 8285 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
e8da1b18 8286
44ffc75b
TH
8287 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods);
8288 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled);
8289 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time);
e8da1b18
NR
8290
8291 return 0;
8292}
ab84d31e 8293#endif /* CONFIG_CFS_BANDWIDTH */
6d6bc0ad 8294#endif /* CONFIG_FAIR_GROUP_SCHED */
68318b8e 8295
052f1dc7 8296#ifdef CONFIG_RT_GROUP_SCHED
182446d0
TH
8297static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
8298 struct cftype *cft, s64 val)
6f505b16 8299{
182446d0 8300 return sched_group_set_rt_runtime(css_tg(css), val);
6f505b16
PZ
8301}
8302
182446d0
TH
8303static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
8304 struct cftype *cft)
6f505b16 8305{
182446d0 8306 return sched_group_rt_runtime(css_tg(css));
6f505b16 8307}
d0b27fa7 8308
182446d0
TH
8309static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
8310 struct cftype *cftype, u64 rt_period_us)
d0b27fa7 8311{
182446d0 8312 return sched_group_set_rt_period(css_tg(css), rt_period_us);
d0b27fa7
PZ
8313}
8314
182446d0
TH
8315static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
8316 struct cftype *cft)
d0b27fa7 8317{
182446d0 8318 return sched_group_rt_period(css_tg(css));
d0b27fa7 8319}
6d6bc0ad 8320#endif /* CONFIG_RT_GROUP_SCHED */
6f505b16 8321
fe5c7cc2 8322static struct cftype cpu_files[] = {
052f1dc7 8323#ifdef CONFIG_FAIR_GROUP_SCHED
fe5c7cc2
PM
8324 {
8325 .name = "shares",
f4c753b7
PM
8326 .read_u64 = cpu_shares_read_u64,
8327 .write_u64 = cpu_shares_write_u64,
fe5c7cc2 8328 },
052f1dc7 8329#endif
ab84d31e
PT
8330#ifdef CONFIG_CFS_BANDWIDTH
8331 {
8332 .name = "cfs_quota_us",
8333 .read_s64 = cpu_cfs_quota_read_s64,
8334 .write_s64 = cpu_cfs_quota_write_s64,
8335 },
8336 {
8337 .name = "cfs_period_us",
8338 .read_u64 = cpu_cfs_period_read_u64,
8339 .write_u64 = cpu_cfs_period_write_u64,
8340 },
e8da1b18
NR
8341 {
8342 .name = "stat",
2da8ca82 8343 .seq_show = cpu_stats_show,
e8da1b18 8344 },
ab84d31e 8345#endif
052f1dc7 8346#ifdef CONFIG_RT_GROUP_SCHED
6f505b16 8347 {
9f0c1e56 8348 .name = "rt_runtime_us",
06ecb27c
PM
8349 .read_s64 = cpu_rt_runtime_read,
8350 .write_s64 = cpu_rt_runtime_write,
6f505b16 8351 },
d0b27fa7
PZ
8352 {
8353 .name = "rt_period_us",
f4c753b7
PM
8354 .read_u64 = cpu_rt_period_read_uint,
8355 .write_u64 = cpu_rt_period_write_uint,
d0b27fa7 8356 },
052f1dc7 8357#endif
4baf6e33 8358 { } /* terminate */
68318b8e
SV
8359};
8360
073219e9 8361struct cgroup_subsys cpu_cgrp_subsys = {
92fb9748
TH
8362 .css_alloc = cpu_cgroup_css_alloc,
8363 .css_free = cpu_cgroup_css_free,
ace783b9
LZ
8364 .css_online = cpu_cgroup_css_online,
8365 .css_offline = cpu_cgroup_css_offline,
eeb61e53 8366 .fork = cpu_cgroup_fork,
bb9d97b6
TH
8367 .can_attach = cpu_cgroup_can_attach,
8368 .attach = cpu_cgroup_attach,
068c5cc5 8369 .exit = cpu_cgroup_exit,
5577964e 8370 .legacy_cftypes = cpu_files,
68318b8e
SV
8371 .early_init = 1,
8372};
8373
052f1dc7 8374#endif /* CONFIG_CGROUP_SCHED */
d842de87 8375
b637a328
PM
8376void dump_cpu_task(int cpu)
8377{
8378 pr_info("Task dump for CPU %d:\n", cpu);
8379 sched_show_task(cpu_curr(cpu));
8380}