Commit | Line | Data |
---|---|---|
78634061 FW |
1 | /* |
2 | * Housekeeping management. Manage the targets for routine code that can run on | |
3 | * any CPU: unbound workqueues, timers, kthreads and any offloadable work. | |
4 | * | |
5 | * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker | |
1bda3f80 | 6 | * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker |
78634061 FW |
7 | * |
8 | */ | |
d84b3131 | 9 | #include "sched.h" |
78634061 | 10 | |
e179f5a0 FW |
11 | DEFINE_STATIC_KEY_FALSE(housekeeping_overriden); |
12 | EXPORT_SYMBOL_GPL(housekeeping_overriden); | |
7e56a1cf | 13 | static cpumask_var_t housekeeping_mask; |
de201559 | 14 | static unsigned int housekeeping_flags; |
7e56a1cf | 15 | |
de201559 | 16 | int housekeeping_any_cpu(enum hk_flags flags) |
7e56a1cf | 17 | { |
e179f5a0 | 18 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
19 | if (housekeeping_flags & flags) |
20 | return cpumask_any_and(housekeeping_mask, cpu_online_mask); | |
7e56a1cf FW |
21 | return smp_processor_id(); |
22 | } | |
23 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); | |
24 | ||
de201559 | 25 | const struct cpumask *housekeeping_cpumask(enum hk_flags flags) |
7e56a1cf | 26 | { |
e179f5a0 | 27 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
28 | if (housekeeping_flags & flags) |
29 | return housekeeping_mask; | |
7e56a1cf FW |
30 | return cpu_possible_mask; |
31 | } | |
32 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); | |
33 | ||
de201559 | 34 | void housekeeping_affine(struct task_struct *t, enum hk_flags flags) |
7e56a1cf | 35 | { |
e179f5a0 | 36 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
37 | if (housekeeping_flags & flags) |
38 | set_cpus_allowed_ptr(t, housekeeping_mask); | |
7e56a1cf FW |
39 | } |
40 | EXPORT_SYMBOL_GPL(housekeeping_affine); | |
41 | ||
de201559 | 42 | bool housekeeping_test_cpu(int cpu, enum hk_flags flags) |
7e56a1cf | 43 | { |
e179f5a0 | 44 | if (static_branch_unlikely(&housekeeping_overriden)) |
de201559 FW |
45 | if (housekeeping_flags & flags) |
46 | return cpumask_test_cpu(cpu, housekeeping_mask); | |
7e56a1cf FW |
47 | return true; |
48 | } | |
49 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); | |
78634061 FW |
50 | |
51 | void __init housekeeping_init(void) | |
52 | { | |
6f1982fe | 53 | if (!housekeeping_flags) |
78634061 FW |
54 | return; |
55 | ||
6f1982fe FW |
56 | static_branch_enable(&housekeeping_overriden); |
57 | ||
d84b3131 FW |
58 | if (housekeeping_flags & HK_FLAG_TICK) |
59 | sched_tick_offload_init(); | |
60 | ||
6f1982fe FW |
61 | /* We need at least one CPU to handle housekeeping work */ |
62 | WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); | |
63 | } | |
64 | ||
edb93821 | 65 | static int __init housekeeping_setup(char *str, enum hk_flags flags) |
6f1982fe FW |
66 | { |
67 | cpumask_var_t non_housekeeping_mask; | |
edb93821 | 68 | int err; |
6f1982fe FW |
69 | |
70 | alloc_bootmem_cpumask_var(&non_housekeeping_mask); | |
edb93821 FW |
71 | err = cpulist_parse(str, non_housekeeping_mask); |
72 | if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) { | |
73 | pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); | |
6f1982fe FW |
74 | free_bootmem_cpumask_var(non_housekeeping_mask); |
75 | return 0; | |
78634061 FW |
76 | } |
77 | ||
edb93821 FW |
78 | if (!housekeeping_flags) { |
79 | alloc_bootmem_cpumask_var(&housekeeping_mask); | |
80 | cpumask_andnot(housekeeping_mask, | |
81 | cpu_possible_mask, non_housekeeping_mask); | |
82 | if (cpumask_empty(housekeeping_mask)) | |
83 | cpumask_set_cpu(smp_processor_id(), housekeeping_mask); | |
84 | } else { | |
85 | cpumask_var_t tmp; | |
86 | ||
87 | alloc_bootmem_cpumask_var(&tmp); | |
88 | cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); | |
89 | if (!cpumask_equal(tmp, housekeeping_mask)) { | |
90 | pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); | |
91 | free_bootmem_cpumask_var(tmp); | |
92 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
93 | return 0; | |
94 | } | |
95 | free_bootmem_cpumask_var(tmp); | |
96 | } | |
de201559 | 97 | |
edb93821 FW |
98 | if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { |
99 | if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { | |
100 | tick_nohz_full_setup(non_housekeeping_mask); | |
101 | } else { | |
102 | pr_warn("Housekeeping: nohz unsupported." | |
103 | " Build with CONFIG_NO_HZ_FULL\n"); | |
104 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
105 | return 0; | |
106 | } | |
107 | } | |
e179f5a0 | 108 | |
edb93821 | 109 | housekeeping_flags |= flags; |
6f1982fe FW |
110 | |
111 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
112 | ||
113 | return 1; | |
78634061 | 114 | } |
edb93821 FW |
115 | |
116 | static int __init housekeeping_nohz_full_setup(char *str) | |
117 | { | |
118 | unsigned int flags; | |
119 | ||
1bda3f80 | 120 | flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | HK_FLAG_MISC; |
edb93821 FW |
121 | |
122 | return housekeeping_setup(str, flags); | |
123 | } | |
6f1982fe | 124 | __setup("nohz_full=", housekeeping_nohz_full_setup); |
edb93821 FW |
125 | |
126 | static int __init housekeeping_isolcpus_setup(char *str) | |
127 | { | |
150dfee9 FW |
128 | unsigned int flags = 0; |
129 | ||
130 | while (isalpha(*str)) { | |
131 | if (!strncmp(str, "nohz,", 5)) { | |
132 | str += 5; | |
133 | flags |= HK_FLAG_TICK; | |
134 | continue; | |
135 | } | |
136 | ||
137 | if (!strncmp(str, "domain,", 7)) { | |
138 | str += 7; | |
139 | flags |= HK_FLAG_DOMAIN; | |
140 | continue; | |
141 | } | |
142 | ||
143 | pr_warn("isolcpus: Error, unknown flag\n"); | |
144 | return 0; | |
145 | } | |
146 | ||
147 | /* Default behaviour for isolcpus without flags */ | |
148 | if (!flags) | |
149 | flags |= HK_FLAG_DOMAIN; | |
150 | ||
151 | return housekeeping_setup(str, flags); | |
edb93821 FW |
152 | } |
153 | __setup("isolcpus=", housekeeping_isolcpus_setup); |