Commit | Line | Data |
---|---|---|
f6e763b9 MB |
1 | /* |
2 | * arch/arm64/kernel/topology.c | |
3 | * | |
4 | * Copyright (C) 2011,2013,2014 Linaro Limited. | |
5 | * | |
6 | * Based on the arm32 version written by Vincent Guittot in turn based on | |
7 | * arch/sh/kernel/topology.c | |
8 | * | |
9 | * This file is subject to the terms and conditions of the GNU General Public | |
10 | * License. See the file "COPYING" in the main directory of this archive | |
11 | * for more details. | |
12 | */ | |
13 | ||
2f0a5d10 | 14 | #include <linux/acpi.h> |
615ffd63 | 15 | #include <linux/arch_topology.h> |
37c3ec2d | 16 | #include <linux/cacheinfo.h> |
f6e763b9 MB |
17 | #include <linux/cpu.h> |
18 | #include <linux/cpumask.h> | |
19 | #include <linux/init.h> | |
20 | #include <linux/percpu.h> | |
21 | #include <linux/node.h> | |
22 | #include <linux/nodemask.h> | |
ebdc9447 | 23 | #include <linux/of.h> |
f6e763b9 | 24 | #include <linux/sched.h> |
105ab3d8 | 25 | #include <linux/sched/topology.h> |
7202bde8 | 26 | #include <linux/slab.h> |
2f0a5d10 | 27 | #include <linux/smp.h> |
be8f185d | 28 | #include <linux/string.h> |
f6e763b9 | 29 | |
be8f185d | 30 | #include <asm/cpu.h> |
4e6f7084 | 31 | #include <asm/cputype.h> |
f6e763b9 MB |
32 | #include <asm/topology.h> |
33 | ||
ebdc9447 MB |
34 | static int __init get_cpu_for_node(struct device_node *node) |
35 | { | |
36 | struct device_node *cpu_node; | |
37 | int cpu; | |
38 | ||
39 | cpu_node = of_parse_phandle(node, "cpu", 0); | |
40 | if (!cpu_node) | |
41 | return -1; | |
42 | ||
52cac110 SP |
43 | cpu = of_cpu_node_to_id(cpu_node); |
44 | if (cpu >= 0) | |
45 | topology_parse_cpu_capacity(cpu_node, cpu); | |
46 | else | |
47 | pr_crit("Unable to find CPU node for %pOF\n", cpu_node); | |
ebdc9447 MB |
48 | |
49 | of_node_put(cpu_node); | |
52cac110 | 50 | return cpu; |
ebdc9447 MB |
51 | } |
52 | ||
868abc07 | 53 | static int __init parse_core(struct device_node *core, int package_id, |
ebdc9447 MB |
54 | int core_id) |
55 | { | |
56 | char name[10]; | |
57 | bool leaf = true; | |
58 | int i = 0; | |
59 | int cpu; | |
60 | struct device_node *t; | |
61 | ||
62 | do { | |
63 | snprintf(name, sizeof(name), "thread%d", i); | |
64 | t = of_get_child_by_name(core, name); | |
65 | if (t) { | |
66 | leaf = false; | |
67 | cpu = get_cpu_for_node(t); | |
68 | if (cpu >= 0) { | |
868abc07 | 69 | cpu_topology[cpu].package_id = package_id; |
ebdc9447 MB |
70 | cpu_topology[cpu].core_id = core_id; |
71 | cpu_topology[cpu].thread_id = i; | |
72 | } else { | |
a270f327 RH |
73 | pr_err("%pOF: Can't get CPU for thread\n", |
74 | t); | |
ebdc9447 MB |
75 | of_node_put(t); |
76 | return -EINVAL; | |
77 | } | |
78 | of_node_put(t); | |
79 | } | |
80 | i++; | |
81 | } while (t); | |
82 | ||
83 | cpu = get_cpu_for_node(core); | |
84 | if (cpu >= 0) { | |
85 | if (!leaf) { | |
a270f327 RH |
86 | pr_err("%pOF: Core has both threads and CPU\n", |
87 | core); | |
ebdc9447 MB |
88 | return -EINVAL; |
89 | } | |
90 | ||
868abc07 | 91 | cpu_topology[cpu].package_id = package_id; |
ebdc9447 MB |
92 | cpu_topology[cpu].core_id = core_id; |
93 | } else if (leaf) { | |
a270f327 | 94 | pr_err("%pOF: Can't get CPU for leaf core\n", core); |
ebdc9447 MB |
95 | return -EINVAL; |
96 | } | |
97 | ||
98 | return 0; | |
99 | } | |
100 | ||
101 | static int __init parse_cluster(struct device_node *cluster, int depth) | |
102 | { | |
103 | char name[10]; | |
104 | bool leaf = true; | |
105 | bool has_cores = false; | |
106 | struct device_node *c; | |
868abc07 | 107 | static int package_id __initdata; |
ebdc9447 MB |
108 | int core_id = 0; |
109 | int i, ret; | |
110 | ||
111 | /* | |
112 | * First check for child clusters; we currently ignore any | |
113 | * information about the nesting of clusters and present the | |
114 | * scheduler with a flat list of them. | |
115 | */ | |
116 | i = 0; | |
117 | do { | |
118 | snprintf(name, sizeof(name), "cluster%d", i); | |
119 | c = of_get_child_by_name(cluster, name); | |
120 | if (c) { | |
121 | leaf = false; | |
122 | ret = parse_cluster(c, depth + 1); | |
123 | of_node_put(c); | |
124 | if (ret != 0) | |
125 | return ret; | |
126 | } | |
127 | i++; | |
128 | } while (c); | |
129 | ||
130 | /* Now check for cores */ | |
131 | i = 0; | |
132 | do { | |
133 | snprintf(name, sizeof(name), "core%d", i); | |
134 | c = of_get_child_by_name(cluster, name); | |
135 | if (c) { | |
136 | has_cores = true; | |
137 | ||
138 | if (depth == 0) { | |
a270f327 RH |
139 | pr_err("%pOF: cpu-map children should be clusters\n", |
140 | c); | |
ebdc9447 MB |
141 | of_node_put(c); |
142 | return -EINVAL; | |
143 | } | |
144 | ||
145 | if (leaf) { | |
868abc07 | 146 | ret = parse_core(c, package_id, core_id++); |
ebdc9447 | 147 | } else { |
a270f327 RH |
148 | pr_err("%pOF: Non-leaf cluster with core %s\n", |
149 | cluster, name); | |
ebdc9447 MB |
150 | ret = -EINVAL; |
151 | } | |
152 | ||
153 | of_node_put(c); | |
154 | if (ret != 0) | |
155 | return ret; | |
156 | } | |
157 | i++; | |
158 | } while (c); | |
159 | ||
160 | if (leaf && !has_cores) | |
a270f327 | 161 | pr_warn("%pOF: empty cluster\n", cluster); |
ebdc9447 MB |
162 | |
163 | if (leaf) | |
868abc07 | 164 | package_id++; |
ebdc9447 MB |
165 | |
166 | return 0; | |
167 | } | |
168 | ||
169 | static int __init parse_dt_topology(void) | |
170 | { | |
171 | struct device_node *cn, *map; | |
172 | int ret = 0; | |
173 | int cpu; | |
174 | ||
175 | cn = of_find_node_by_path("/cpus"); | |
176 | if (!cn) { | |
177 | pr_err("No CPU information found in DT\n"); | |
178 | return 0; | |
179 | } | |
180 | ||
181 | /* | |
182 | * When topology is provided cpu-map is essentially a root | |
183 | * cluster with restricted subnodes. | |
184 | */ | |
185 | map = of_get_child_by_name(cn, "cpu-map"); | |
c105aa31 | 186 | if (!map) |
ebdc9447 MB |
187 | goto out; |
188 | ||
189 | ret = parse_cluster(map, 0); | |
190 | if (ret != 0) | |
191 | goto out_map; | |
192 | ||
4ca4f26a | 193 | topology_normalize_cpu_scale(); |
7202bde8 | 194 | |
ebdc9447 MB |
195 | /* |
196 | * Check that all cores are in the topology; the SMP code will | |
197 | * only mark cores described in the DT as possible. | |
198 | */ | |
4e6f7084 | 199 | for_each_possible_cpu(cpu) |
868abc07 | 200 | if (cpu_topology[cpu].package_id == -1) |
ebdc9447 | 201 | ret = -EINVAL; |
ebdc9447 MB |
202 | |
203 | out_map: | |
204 | of_node_put(map); | |
205 | out: | |
206 | of_node_put(cn); | |
207 | return ret; | |
208 | } | |
209 | ||
f6e763b9 MB |
210 | /* |
211 | * cpu topology table | |
212 | */ | |
213 | struct cpu_topology cpu_topology[NR_CPUS]; | |
214 | EXPORT_SYMBOL_GPL(cpu_topology); | |
215 | ||
216 | const struct cpumask *cpu_coregroup_mask(int cpu) | |
217 | { | |
e67ecf64 | 218 | const cpumask_t *core_mask = cpumask_of_node(cpu_to_node(cpu)); |
37c3ec2d | 219 | |
e67ecf64 SH |
220 | /* Find the smaller of NUMA, core or LLC siblings */ |
221 | if (cpumask_subset(&cpu_topology[cpu].core_sibling, core_mask)) { | |
222 | /* not numa in package, lets use the package siblings */ | |
223 | core_mask = &cpu_topology[cpu].core_sibling; | |
224 | } | |
37c3ec2d | 225 | if (cpu_topology[cpu].llc_id != -1) { |
f70ff127 SH |
226 | if (cpumask_subset(&cpu_topology[cpu].llc_sibling, core_mask)) |
227 | core_mask = &cpu_topology[cpu].llc_sibling; | |
37c3ec2d JL |
228 | } |
229 | ||
230 | return core_mask; | |
f6e763b9 MB |
231 | } |
232 | ||
233 | static void update_siblings_masks(unsigned int cpuid) | |
234 | { | |
235 | struct cpu_topology *cpu_topo, *cpuid_topo = &cpu_topology[cpuid]; | |
236 | int cpu; | |
237 | ||
f6e763b9 | 238 | /* update core and thread sibling masks */ |
5ec8b591 | 239 | for_each_online_cpu(cpu) { |
f6e763b9 MB |
240 | cpu_topo = &cpu_topology[cpu]; |
241 | ||
e156ab71 | 242 | if (cpuid_topo->llc_id == cpu_topo->llc_id) { |
f70ff127 SH |
243 | cpumask_set_cpu(cpu, &cpuid_topo->llc_sibling); |
244 | cpumask_set_cpu(cpuid, &cpu_topo->llc_sibling); | |
e156ab71 | 245 | } |
37c3ec2d | 246 | |
868abc07 | 247 | if (cpuid_topo->package_id != cpu_topo->package_id) |
f6e763b9 MB |
248 | continue; |
249 | ||
250 | cpumask_set_cpu(cpuid, &cpu_topo->core_sibling); | |
5ec8b591 | 251 | cpumask_set_cpu(cpu, &cpuid_topo->core_sibling); |
f6e763b9 MB |
252 | |
253 | if (cpuid_topo->core_id != cpu_topo->core_id) | |
254 | continue; | |
255 | ||
256 | cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling); | |
5ec8b591 | 257 | cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling); |
f6e763b9 MB |
258 | } |
259 | } | |
260 | ||
261 | void store_cpu_topology(unsigned int cpuid) | |
262 | { | |
4e6f7084 ZSL |
263 | struct cpu_topology *cpuid_topo = &cpu_topology[cpuid]; |
264 | u64 mpidr; | |
265 | ||
868abc07 | 266 | if (cpuid_topo->package_id != -1) |
4e6f7084 ZSL |
267 | goto topology_populated; |
268 | ||
269 | mpidr = read_cpuid_mpidr(); | |
270 | ||
271 | /* Uniprocessor systems can rely on default topology values */ | |
272 | if (mpidr & MPIDR_UP_BITMASK) | |
273 | return; | |
274 | ||
275 | /* Create cpu topology mapping based on MPIDR. */ | |
276 | if (mpidr & MPIDR_MT_BITMASK) { | |
277 | /* Multiprocessor system : Multi-threads per core */ | |
278 | cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
279 | cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); | |
868abc07 | 280 | cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) | |
1cefdaea | 281 | MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8; |
4e6f7084 ZSL |
282 | } else { |
283 | /* Multiprocessor system : Single-thread per core */ | |
284 | cpuid_topo->thread_id = -1; | |
285 | cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); | |
868abc07 | 286 | cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) | |
1cefdaea MB |
287 | MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 | |
288 | MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16; | |
4e6f7084 ZSL |
289 | } |
290 | ||
291 | pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n", | |
868abc07 | 292 | cpuid, cpuid_topo->package_id, cpuid_topo->core_id, |
4e6f7084 ZSL |
293 | cpuid_topo->thread_id, mpidr); |
294 | ||
295 | topology_populated: | |
f6e763b9 MB |
296 | update_siblings_masks(cpuid); |
297 | } | |
298 | ||
31b46035 SH |
299 | static void clear_cpu_topology(int cpu) |
300 | { | |
301 | struct cpu_topology *cpu_topo = &cpu_topology[cpu]; | |
302 | ||
f70ff127 SH |
303 | cpumask_clear(&cpu_topo->llc_sibling); |
304 | cpumask_set_cpu(cpu, &cpu_topo->llc_sibling); | |
31b46035 SH |
305 | |
306 | cpumask_clear(&cpu_topo->core_sibling); | |
307 | cpumask_set_cpu(cpu, &cpu_topo->core_sibling); | |
308 | cpumask_clear(&cpu_topo->thread_sibling); | |
309 | cpumask_set_cpu(cpu, &cpu_topo->thread_sibling); | |
310 | } | |
311 | ||
ebdc9447 | 312 | static void __init reset_cpu_topology(void) |
f6e763b9 MB |
313 | { |
314 | unsigned int cpu; | |
315 | ||
f6e763b9 MB |
316 | for_each_possible_cpu(cpu) { |
317 | struct cpu_topology *cpu_topo = &cpu_topology[cpu]; | |
318 | ||
319 | cpu_topo->thread_id = -1; | |
c31bf048 | 320 | cpu_topo->core_id = 0; |
868abc07 | 321 | cpu_topo->package_id = -1; |
37c3ec2d | 322 | cpu_topo->llc_id = -1; |
37c3ec2d | 323 | |
31b46035 | 324 | clear_cpu_topology(cpu); |
f6e763b9 MB |
325 | } |
326 | } | |
ebdc9447 | 327 | |
5bdd2b3f SH |
328 | void remove_cpu_topology(unsigned int cpu) |
329 | { | |
330 | int sibling; | |
331 | ||
332 | for_each_cpu(sibling, topology_core_cpumask(cpu)) | |
333 | cpumask_clear_cpu(cpu, topology_core_cpumask(sibling)); | |
334 | for_each_cpu(sibling, topology_sibling_cpumask(cpu)) | |
335 | cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling)); | |
336 | for_each_cpu(sibling, topology_llc_cpumask(cpu)) | |
337 | cpumask_clear_cpu(cpu, topology_llc_cpumask(sibling)); | |
338 | ||
339 | clear_cpu_topology(cpu); | |
340 | } | |
341 | ||
2f0a5d10 JL |
342 | #ifdef CONFIG_ACPI |
343 | /* | |
344 | * Propagate the topology information of the processor_topology_node tree to the | |
345 | * cpu_topology array. | |
346 | */ | |
347 | static int __init parse_acpi_topology(void) | |
348 | { | |
349 | bool is_threaded; | |
350 | int cpu, topology_id; | |
351 | ||
352 | is_threaded = read_cpuid_mpidr() & MPIDR_MT_BITMASK; | |
353 | ||
354 | for_each_possible_cpu(cpu) { | |
37c3ec2d JL |
355 | int i, cache_id; |
356 | ||
2f0a5d10 JL |
357 | topology_id = find_acpi_cpu_topology(cpu, 0); |
358 | if (topology_id < 0) | |
359 | return topology_id; | |
360 | ||
361 | if (is_threaded) { | |
362 | cpu_topology[cpu].thread_id = topology_id; | |
363 | topology_id = find_acpi_cpu_topology(cpu, 1); | |
364 | cpu_topology[cpu].core_id = topology_id; | |
365 | } else { | |
366 | cpu_topology[cpu].thread_id = -1; | |
367 | cpu_topology[cpu].core_id = topology_id; | |
368 | } | |
369 | topology_id = find_acpi_cpu_topology_package(cpu); | |
370 | cpu_topology[cpu].package_id = topology_id; | |
37c3ec2d JL |
371 | |
372 | i = acpi_find_last_cache_level(cpu); | |
373 | ||
374 | if (i > 0) { | |
375 | /* | |
376 | * this is the only part of cpu_topology that has | |
377 | * a direct relationship with the cache topology | |
378 | */ | |
379 | cache_id = find_acpi_cpu_cache_topology(cpu, i); | |
380 | if (cache_id > 0) | |
381 | cpu_topology[cpu].llc_id = cache_id; | |
382 | } | |
2f0a5d10 JL |
383 | } |
384 | ||
385 | return 0; | |
386 | } | |
387 | ||
388 | #else | |
389 | static inline int __init parse_acpi_topology(void) | |
390 | { | |
391 | return -EINVAL; | |
392 | } | |
393 | #endif | |
394 | ||
ebdc9447 MB |
395 | void __init init_cpu_topology(void) |
396 | { | |
397 | reset_cpu_topology(); | |
398 | ||
399 | /* | |
400 | * Discard anything that was parsed if we hit an error so we | |
401 | * don't use partial information. | |
402 | */ | |
2f0a5d10 JL |
403 | if (!acpi_disabled && parse_acpi_topology()) |
404 | reset_cpu_topology(); | |
405 | else if (of_have_populated_dt() && parse_dt_topology()) | |
ebdc9447 MB |
406 | reset_cpu_topology(); |
407 | } |