Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
78634061 FW |
2 | /* |
3 | * Housekeeping management. Manage the targets for routine code that can run on | |
4 | * any CPU: unbound workqueues, timers, kthreads and any offloadable work. | |
5 | * | |
6 | * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker | |
1bda3f80 | 7 | * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker |
78634061 FW |
8 | * |
9 | */ | |
78634061 | 10 | |
04d4e665 FW |
11 | enum hk_flags { |
12 | HK_FLAG_TIMER = BIT(HK_TYPE_TIMER), | |
13 | HK_FLAG_RCU = BIT(HK_TYPE_RCU), | |
14 | HK_FLAG_MISC = BIT(HK_TYPE_MISC), | |
15 | HK_FLAG_SCHED = BIT(HK_TYPE_SCHED), | |
16 | HK_FLAG_TICK = BIT(HK_TYPE_TICK), | |
17 | HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), | |
18 | HK_FLAG_WQ = BIT(HK_TYPE_WQ), | |
19 | HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), | |
20 | HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD), | |
21 | }; | |
22 | ||
dfcb245e IM |
23 | DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); |
24 | EXPORT_SYMBOL_GPL(housekeeping_overridden); | |
ed3b362d FW |
25 | |
26 | struct housekeeping { | |
27 | cpumask_var_t cpumasks[HK_TYPE_MAX]; | |
28 | unsigned long flags; | |
29 | }; | |
30 | ||
31 | static struct housekeeping housekeeping; | |
7e56a1cf | 32 | |
04d4e665 | 33 | bool housekeeping_enabled(enum hk_type type) |
0c5f81da | 34 | { |
ed3b362d | 35 | return !!(housekeeping.flags & BIT(type)); |
0c5f81da WL |
36 | } |
37 | EXPORT_SYMBOL_GPL(housekeeping_enabled); | |
38 | ||
04d4e665 | 39 | int housekeeping_any_cpu(enum hk_type type) |
7e56a1cf | 40 | { |
e0e8d491 WL |
41 | int cpu; |
42 | ||
43 | if (static_branch_unlikely(&housekeeping_overridden)) { | |
ed3b362d FW |
44 | if (housekeeping.flags & BIT(type)) { |
45 | cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); | |
e0e8d491 WL |
46 | if (cpu < nr_cpu_ids) |
47 | return cpu; | |
48 | ||
5097cbcb ON |
49 | cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask); |
50 | if (likely(cpu < nr_cpu_ids)) | |
51 | return cpu; | |
52 | /* | |
53 | * Unless we have another problem this can only happen | |
54 | * at boot time before start_secondary() brings the 1st | |
55 | * housekeeping CPU up. | |
56 | */ | |
57 | WARN_ON_ONCE(system_state == SYSTEM_RUNNING || | |
58 | type != HK_TYPE_TIMER); | |
e0e8d491 WL |
59 | } |
60 | } | |
7e56a1cf FW |
61 | return smp_processor_id(); |
62 | } | |
63 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); | |
64 | ||
04d4e665 | 65 | const struct cpumask *housekeeping_cpumask(enum hk_type type) |
7e56a1cf | 66 | { |
dfcb245e | 67 | if (static_branch_unlikely(&housekeeping_overridden)) |
ed3b362d FW |
68 | if (housekeeping.flags & BIT(type)) |
69 | return housekeeping.cpumasks[type]; | |
7e56a1cf FW |
70 | return cpu_possible_mask; |
71 | } | |
72 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); | |
73 | ||
04d4e665 | 74 | void housekeeping_affine(struct task_struct *t, enum hk_type type) |
7e56a1cf | 75 | { |
dfcb245e | 76 | if (static_branch_unlikely(&housekeeping_overridden)) |
ed3b362d FW |
77 | if (housekeeping.flags & BIT(type)) |
78 | set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); | |
7e56a1cf FW |
79 | } |
80 | EXPORT_SYMBOL_GPL(housekeeping_affine); | |
81 | ||
04d4e665 | 82 | bool housekeeping_test_cpu(int cpu, enum hk_type type) |
7e56a1cf | 83 | { |
dfcb245e | 84 | if (static_branch_unlikely(&housekeeping_overridden)) |
ed3b362d FW |
85 | if (housekeeping.flags & BIT(type)) |
86 | return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); | |
7e56a1cf FW |
87 | return true; |
88 | } | |
89 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); | |
78634061 FW |
90 | |
91 | void __init housekeeping_init(void) | |
92 | { | |
ed3b362d FW |
93 | enum hk_type type; |
94 | ||
95 | if (!housekeeping.flags) | |
78634061 FW |
96 | return; |
97 | ||
dfcb245e | 98 | static_branch_enable(&housekeeping_overridden); |
6f1982fe | 99 | |
ed3b362d | 100 | if (housekeeping.flags & HK_FLAG_TICK) |
d84b3131 FW |
101 | sched_tick_offload_init(); |
102 | ||
ed3b362d FW |
103 | for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { |
104 | /* We need at least one CPU to handle housekeeping work */ | |
105 | WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); | |
106 | } | |
107 | } | |
108 | ||
109 | static void __init housekeeping_setup_type(enum hk_type type, | |
110 | cpumask_var_t housekeeping_staging) | |
111 | { | |
112 | ||
113 | alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); | |
114 | cpumask_copy(housekeeping.cpumasks[type], | |
115 | housekeeping_staging); | |
6f1982fe FW |
116 | } |
117 | ||
ed3b362d | 118 | static int __init housekeeping_setup(char *str, unsigned long flags) |
6f1982fe | 119 | { |
6367b600 | 120 | cpumask_var_t non_housekeeping_mask, housekeeping_staging; |
257bf89d | 121 | unsigned int first_cpu; |
0cd3e59d | 122 | int err = 0; |
6f1982fe | 123 | |
ed3b362d | 124 | if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) { |
65e53f86 FW |
125 | if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { |
126 | pr_warn("Housekeeping: nohz unsupported." | |
127 | " Build with CONFIG_NO_HZ_FULL\n"); | |
128 | return 0; | |
129 | } | |
130 | } | |
131 | ||
6f1982fe | 132 | alloc_bootmem_cpumask_var(&non_housekeeping_mask); |
915a2bc3 | 133 | if (cpulist_parse(str, non_housekeeping_mask) < 0) { |
edb93821 | 134 | pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); |
0cd3e59d | 135 | goto free_non_housekeeping_mask; |
78634061 FW |
136 | } |
137 | ||
6367b600 FW |
138 | alloc_bootmem_cpumask_var(&housekeeping_staging); |
139 | cpumask_andnot(housekeeping_staging, | |
140 | cpu_possible_mask, non_housekeeping_mask); | |
9219565a | 141 | |
257bf89d ON |
142 | first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging); |
143 | if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) { | |
6367b600 FW |
144 | __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); |
145 | __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); | |
ed3b362d | 146 | if (!housekeeping.flags) { |
9219565a NP |
147 | pr_warn("Housekeeping: must include one present CPU, " |
148 | "using boot CPU:%d\n", smp_processor_id()); | |
9219565a | 149 | } |
6367b600 FW |
150 | } |
151 | ||
257bf89d ON |
152 | if (cpumask_empty(non_housekeeping_mask)) |
153 | goto free_housekeeping_staging; | |
154 | ||
ed3b362d FW |
155 | if (!housekeeping.flags) { |
156 | /* First setup call ("nohz_full=" or "isolcpus=") */ | |
157 | enum hk_type type; | |
158 | ||
159 | for_each_set_bit(type, &flags, HK_TYPE_MAX) | |
160 | housekeeping_setup_type(type, housekeeping_staging); | |
edb93821 | 161 | } else { |
ed3b362d FW |
162 | /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ |
163 | enum hk_type type; | |
164 | unsigned long iter_flags = flags & housekeeping.flags; | |
165 | ||
166 | for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { | |
167 | if (!cpumask_equal(housekeeping_staging, | |
168 | housekeeping.cpumasks[type])) { | |
169 | pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); | |
170 | goto free_housekeeping_staging; | |
171 | } | |
edb93821 | 172 | } |
ed3b362d FW |
173 | |
174 | iter_flags = flags & ~housekeeping.flags; | |
175 | ||
176 | for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) | |
177 | housekeeping_setup_type(type, housekeeping_staging); | |
edb93821 | 178 | } |
6367b600 | 179 | |
ed3b362d | 180 | if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) |
65e53f86 | 181 | tick_nohz_full_setup(non_housekeeping_mask); |
e179f5a0 | 182 | |
ed3b362d | 183 | housekeeping.flags |= flags; |
0cd3e59d | 184 | err = 1; |
6f1982fe | 185 | |
0cd3e59d FW |
186 | free_housekeeping_staging: |
187 | free_bootmem_cpumask_var(housekeeping_staging); | |
188 | free_non_housekeeping_mask: | |
6f1982fe FW |
189 | free_bootmem_cpumask_var(non_housekeeping_mask); |
190 | ||
0cd3e59d | 191 | return err; |
78634061 | 192 | } |
edb93821 FW |
193 | |
194 | static int __init housekeeping_nohz_full_setup(char *str) | |
195 | { | |
ed3b362d | 196 | unsigned long flags; |
edb93821 | 197 | |
9cc5b865 MT |
198 | flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | |
199 | HK_FLAG_MISC | HK_FLAG_KTHREAD; | |
edb93821 FW |
200 | |
201 | return housekeeping_setup(str, flags); | |
202 | } | |
6f1982fe | 203 | __setup("nohz_full=", housekeeping_nohz_full_setup); |
edb93821 FW |
204 | |
205 | static int __init housekeeping_isolcpus_setup(char *str) | |
206 | { | |
ed3b362d | 207 | unsigned long flags = 0; |
3662daf0 PX |
208 | bool illegal = false; |
209 | char *par; | |
210 | int len; | |
150dfee9 FW |
211 | |
212 | while (isalpha(*str)) { | |
213 | if (!strncmp(str, "nohz,", 5)) { | |
214 | str += 5; | |
215 | flags |= HK_FLAG_TICK; | |
216 | continue; | |
217 | } | |
218 | ||
219 | if (!strncmp(str, "domain,", 7)) { | |
220 | str += 7; | |
221 | flags |= HK_FLAG_DOMAIN; | |
222 | continue; | |
223 | } | |
224 | ||
11ea68f5 ML |
225 | if (!strncmp(str, "managed_irq,", 12)) { |
226 | str += 12; | |
227 | flags |= HK_FLAG_MANAGED_IRQ; | |
228 | continue; | |
229 | } | |
230 | ||
3662daf0 PX |
231 | /* |
232 | * Skip unknown sub-parameter and validate that it is not | |
233 | * containing an invalid character. | |
234 | */ | |
235 | for (par = str, len = 0; *str && *str != ','; str++, len++) { | |
236 | if (!isalpha(*str) && *str != '_') | |
237 | illegal = true; | |
238 | } | |
239 | ||
240 | if (illegal) { | |
241 | pr_warn("isolcpus: Invalid flag %.*s\n", len, par); | |
242 | return 0; | |
243 | } | |
244 | ||
245 | pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); | |
246 | str++; | |
150dfee9 FW |
247 | } |
248 | ||
249 | /* Default behaviour for isolcpus without flags */ | |
250 | if (!flags) | |
251 | flags |= HK_FLAG_DOMAIN; | |
252 | ||
253 | return housekeeping_setup(str, flags); | |
edb93821 FW |
254 | } |
255 | __setup("isolcpus=", housekeeping_isolcpus_setup); |