Commit | Line | Data |
---|---|---|
457c8996 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
78634061 FW |
2 | /* |
3 | * Housekeeping management. Manage the targets for routine code that can run on | |
4 | * any CPU: unbound workqueues, timers, kthreads and any offloadable work. | |
5 | * | |
6 | * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker | |
1bda3f80 | 7 | * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker |
78634061 FW |
8 | * |
9 | */ | |
d84b3131 | 10 | #include "sched.h" |
78634061 | 11 | |
dfcb245e IM |
12 | DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); |
13 | EXPORT_SYMBOL_GPL(housekeeping_overridden); | |
7e56a1cf | 14 | static cpumask_var_t housekeeping_mask; |
de201559 | 15 | static unsigned int housekeeping_flags; |
7e56a1cf | 16 | |
0c5f81da WL |
17 | bool housekeeping_enabled(enum hk_flags flags) |
18 | { | |
19 | return !!(housekeeping_flags & flags); | |
20 | } | |
21 | EXPORT_SYMBOL_GPL(housekeeping_enabled); | |
22 | ||
de201559 | 23 | int housekeeping_any_cpu(enum hk_flags flags) |
7e56a1cf | 24 | { |
e0e8d491 WL |
25 | int cpu; |
26 | ||
27 | if (static_branch_unlikely(&housekeeping_overridden)) { | |
28 | if (housekeeping_flags & flags) { | |
29 | cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id()); | |
30 | if (cpu < nr_cpu_ids) | |
31 | return cpu; | |
32 | ||
de201559 | 33 | return cpumask_any_and(housekeeping_mask, cpu_online_mask); |
e0e8d491 WL |
34 | } |
35 | } | |
7e56a1cf FW |
36 | return smp_processor_id(); |
37 | } | |
38 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); | |
39 | ||
de201559 | 40 | const struct cpumask *housekeeping_cpumask(enum hk_flags flags) |
7e56a1cf | 41 | { |
dfcb245e | 42 | if (static_branch_unlikely(&housekeeping_overridden)) |
de201559 FW |
43 | if (housekeeping_flags & flags) |
44 | return housekeeping_mask; | |
7e56a1cf FW |
45 | return cpu_possible_mask; |
46 | } | |
47 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); | |
48 | ||
de201559 | 49 | void housekeeping_affine(struct task_struct *t, enum hk_flags flags) |
7e56a1cf | 50 | { |
dfcb245e | 51 | if (static_branch_unlikely(&housekeeping_overridden)) |
de201559 FW |
52 | if (housekeeping_flags & flags) |
53 | set_cpus_allowed_ptr(t, housekeeping_mask); | |
7e56a1cf FW |
54 | } |
55 | EXPORT_SYMBOL_GPL(housekeeping_affine); | |
56 | ||
de201559 | 57 | bool housekeeping_test_cpu(int cpu, enum hk_flags flags) |
7e56a1cf | 58 | { |
dfcb245e | 59 | if (static_branch_unlikely(&housekeeping_overridden)) |
de201559 FW |
60 | if (housekeeping_flags & flags) |
61 | return cpumask_test_cpu(cpu, housekeeping_mask); | |
7e56a1cf FW |
62 | return true; |
63 | } | |
64 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); | |
78634061 FW |
65 | |
66 | void __init housekeeping_init(void) | |
67 | { | |
6f1982fe | 68 | if (!housekeeping_flags) |
78634061 FW |
69 | return; |
70 | ||
dfcb245e | 71 | static_branch_enable(&housekeeping_overridden); |
6f1982fe | 72 | |
d84b3131 FW |
73 | if (housekeeping_flags & HK_FLAG_TICK) |
74 | sched_tick_offload_init(); | |
75 | ||
6f1982fe FW |
76 | /* We need at least one CPU to handle housekeeping work */ |
77 | WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); | |
78 | } | |
79 | ||
edb93821 | 80 | static int __init housekeeping_setup(char *str, enum hk_flags flags) |
6f1982fe FW |
81 | { |
82 | cpumask_var_t non_housekeeping_mask; | |
9219565a | 83 | cpumask_var_t tmp; |
6f1982fe FW |
84 | |
85 | alloc_bootmem_cpumask_var(&non_housekeeping_mask); | |
915a2bc3 | 86 | if (cpulist_parse(str, non_housekeeping_mask) < 0) { |
edb93821 | 87 | pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); |
6f1982fe FW |
88 | free_bootmem_cpumask_var(non_housekeeping_mask); |
89 | return 0; | |
78634061 FW |
90 | } |
91 | ||
9219565a | 92 | alloc_bootmem_cpumask_var(&tmp); |
edb93821 FW |
93 | if (!housekeeping_flags) { |
94 | alloc_bootmem_cpumask_var(&housekeeping_mask); | |
95 | cpumask_andnot(housekeeping_mask, | |
96 | cpu_possible_mask, non_housekeeping_mask); | |
9219565a NP |
97 | |
98 | cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); | |
99 | if (cpumask_empty(tmp)) { | |
100 | pr_warn("Housekeeping: must include one present CPU, " | |
101 | "using boot CPU:%d\n", smp_processor_id()); | |
c89d92ed | 102 | __cpumask_set_cpu(smp_processor_id(), housekeeping_mask); |
9219565a NP |
103 | __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); |
104 | } | |
edb93821 | 105 | } else { |
9219565a NP |
106 | cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask); |
107 | if (cpumask_empty(tmp)) | |
108 | __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); | |
edb93821 FW |
109 | cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask); |
110 | if (!cpumask_equal(tmp, housekeeping_mask)) { | |
111 | pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); | |
112 | free_bootmem_cpumask_var(tmp); | |
113 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
114 | return 0; | |
115 | } | |
edb93821 | 116 | } |
9219565a | 117 | free_bootmem_cpumask_var(tmp); |
de201559 | 118 | |
edb93821 FW |
119 | if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) { |
120 | if (IS_ENABLED(CONFIG_NO_HZ_FULL)) { | |
121 | tick_nohz_full_setup(non_housekeeping_mask); | |
122 | } else { | |
123 | pr_warn("Housekeeping: nohz unsupported." | |
124 | " Build with CONFIG_NO_HZ_FULL\n"); | |
125 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
126 | return 0; | |
127 | } | |
128 | } | |
e179f5a0 | 129 | |
edb93821 | 130 | housekeeping_flags |= flags; |
6f1982fe FW |
131 | |
132 | free_bootmem_cpumask_var(non_housekeeping_mask); | |
133 | ||
134 | return 1; | |
78634061 | 135 | } |
edb93821 FW |
136 | |
137 | static int __init housekeeping_nohz_full_setup(char *str) | |
138 | { | |
139 | unsigned int flags; | |
140 | ||
9cc5b865 MT |
141 | flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | |
142 | HK_FLAG_MISC | HK_FLAG_KTHREAD; | |
edb93821 FW |
143 | |
144 | return housekeeping_setup(str, flags); | |
145 | } | |
6f1982fe | 146 | __setup("nohz_full=", housekeeping_nohz_full_setup); |
edb93821 FW |
147 | |
148 | static int __init housekeeping_isolcpus_setup(char *str) | |
149 | { | |
150dfee9 | 150 | unsigned int flags = 0; |
3662daf0 PX |
151 | bool illegal = false; |
152 | char *par; | |
153 | int len; | |
150dfee9 FW |
154 | |
155 | while (isalpha(*str)) { | |
156 | if (!strncmp(str, "nohz,", 5)) { | |
157 | str += 5; | |
158 | flags |= HK_FLAG_TICK; | |
159 | continue; | |
160 | } | |
161 | ||
162 | if (!strncmp(str, "domain,", 7)) { | |
163 | str += 7; | |
164 | flags |= HK_FLAG_DOMAIN; | |
165 | continue; | |
166 | } | |
167 | ||
11ea68f5 ML |
168 | if (!strncmp(str, "managed_irq,", 12)) { |
169 | str += 12; | |
170 | flags |= HK_FLAG_MANAGED_IRQ; | |
171 | continue; | |
172 | } | |
173 | ||
3662daf0 PX |
174 | /* |
175 | * Skip unknown sub-parameter and validate that it is not | |
176 | * containing an invalid character. | |
177 | */ | |
178 | for (par = str, len = 0; *str && *str != ','; str++, len++) { | |
179 | if (!isalpha(*str) && *str != '_') | |
180 | illegal = true; | |
181 | } | |
182 | ||
183 | if (illegal) { | |
184 | pr_warn("isolcpus: Invalid flag %.*s\n", len, par); | |
185 | return 0; | |
186 | } | |
187 | ||
188 | pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); | |
189 | str++; | |
150dfee9 FW |
190 | } |
191 | ||
192 | /* Default behaviour for isolcpus without flags */ | |
193 | if (!flags) | |
194 | flags |= HK_FLAG_DOMAIN; | |
195 | ||
196 | return housekeeping_setup(str, flags); | |
edb93821 FW |
197 | } |
198 | __setup("isolcpus=", housekeeping_isolcpus_setup); |