Commit | Line | Data |
---|---|---|
989d42e8 | 1 | // SPDX-License-Identifier: GPL-2.0 |
246246cb SH |
2 | /* |
3 | * cacheinfo support - processor cache information via sysfs | |
4 | * | |
5 | * Based on arch/x86/kernel/cpu/intel_cacheinfo.c | |
6 | * Author: Sudeep Holla <sudeep.holla@arm.com> | |
246246cb | 7 | */ |
8e1073b1 SH |
8 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
9 | ||
55877ef4 | 10 | #include <linux/acpi.h> |
246246cb SH |
11 | #include <linux/bitops.h> |
12 | #include <linux/cacheinfo.h> | |
13 | #include <linux/compiler.h> | |
14 | #include <linux/cpu.h> | |
15 | #include <linux/device.h> | |
16 | #include <linux/init.h> | |
b9581552 | 17 | #include <linux/of.h> |
246246cb SH |
18 | #include <linux/sched.h> |
19 | #include <linux/slab.h> | |
20 | #include <linux/smp.h> | |
21 | #include <linux/sysfs.h> | |
22 | ||
23 | /* pointer to per cpu cacheinfo */ | |
24 | static DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo); | |
25 | #define ci_cacheinfo(cpu) (&per_cpu(ci_cpu_cacheinfo, cpu)) | |
26 | #define cache_leaves(cpu) (ci_cacheinfo(cpu)->num_leaves) | |
27 | #define per_cpu_cacheinfo(cpu) (ci_cacheinfo(cpu)->info_list) | |
b14e8d21 SH |
28 | #define per_cpu_cacheinfo_idx(cpu, idx) \ |
29 | (per_cpu_cacheinfo(cpu) + (idx)) | |
246246cb | 30 | |
ef9f643a PG |
31 | /* Set if no cache information is found in DT/ACPI. */ |
32 | static bool use_arch_info; | |
33 | ||
246246cb SH |
34 | struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu) |
35 | { | |
36 | return ci_cacheinfo(cpu); | |
37 | } | |
38 | ||
246246cb SH |
39 | static inline bool cache_leaves_are_shared(struct cacheinfo *this_leaf, |
40 | struct cacheinfo *sib_leaf) | |
41 | { | |
9447eb0f SH |
42 | /* |
43 | * For non DT/ACPI systems, assume unique level 1 caches, | |
7a306e3e | 44 | * system-wide shared caches for all other levels. |
9447eb0f | 45 | */ |
ef9f643a PG |
46 | if (!(IS_ENABLED(CONFIG_OF) || IS_ENABLED(CONFIG_ACPI)) || |
47 | use_arch_info) | |
7a306e3e | 48 | return (this_leaf->level != 1) && (sib_leaf->level != 1); |
9447eb0f | 49 | |
f16d1bec SH |
50 | if ((sib_leaf->attributes & CACHE_ID) && |
51 | (this_leaf->attributes & CACHE_ID)) | |
52 | return sib_leaf->id == this_leaf->id; | |
53 | ||
9b97387c | 54 | return sib_leaf->fw_token == this_leaf->fw_token; |
246246cb | 55 | } |
dfea747d | 56 | |
cc1cfc47 SH |
57 | bool last_level_cache_is_valid(unsigned int cpu) |
58 | { | |
59 | struct cacheinfo *llc; | |
60 | ||
61 | if (!cache_leaves(cpu)) | |
62 | return false; | |
63 | ||
64 | llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); | |
65 | ||
f16d1bec SH |
66 | return (llc->attributes & CACHE_ID) || !!llc->fw_token; |
67 | ||
cc1cfc47 SH |
68 | } |
69 | ||
70 | bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y) | |
71 | { | |
72 | struct cacheinfo *llc_x, *llc_y; | |
73 | ||
74 | if (!last_level_cache_is_valid(cpu_x) || | |
75 | !last_level_cache_is_valid(cpu_y)) | |
76 | return false; | |
77 | ||
78 | llc_x = per_cpu_cacheinfo_idx(cpu_x, cache_leaves(cpu_x) - 1); | |
79 | llc_y = per_cpu_cacheinfo_idx(cpu_y, cache_leaves(cpu_y) - 1); | |
80 | ||
81 | return cache_leaves_are_shared(llc_x, llc_y); | |
82 | } | |
83 | ||
9447eb0f | 84 | #ifdef CONFIG_OF |
cde0fbff PG |
85 | |
86 | static bool of_check_cache_nodes(struct device_node *np); | |
87 | ||
dfea747d SH |
88 | /* OF properties to query for a given cache type */ |
89 | struct cache_type_info { | |
90 | const char *size_prop; | |
91 | const char *line_size_props[2]; | |
92 | const char *nr_sets_prop; | |
93 | }; | |
94 | ||
95 | static const struct cache_type_info cache_type_info[] = { | |
96 | { | |
97 | .size_prop = "cache-size", | |
98 | .line_size_props = { "cache-line-size", | |
99 | "cache-block-size", }, | |
100 | .nr_sets_prop = "cache-sets", | |
101 | }, { | |
102 | .size_prop = "i-cache-size", | |
103 | .line_size_props = { "i-cache-line-size", | |
104 | "i-cache-block-size", }, | |
105 | .nr_sets_prop = "i-cache-sets", | |
106 | }, { | |
107 | .size_prop = "d-cache-size", | |
108 | .line_size_props = { "d-cache-line-size", | |
109 | "d-cache-block-size", }, | |
110 | .nr_sets_prop = "d-cache-sets", | |
111 | }, | |
112 | }; | |
113 | ||
114 | static inline int get_cacheinfo_idx(enum cache_type type) | |
115 | { | |
116 | if (type == CACHE_TYPE_UNIFIED) | |
117 | return 0; | |
118 | return type; | |
119 | } | |
120 | ||
2ff075c7 | 121 | static void cache_size(struct cacheinfo *this_leaf, struct device_node *np) |
dfea747d SH |
122 | { |
123 | const char *propname; | |
dfea747d SH |
124 | int ct_idx; |
125 | ||
126 | ct_idx = get_cacheinfo_idx(this_leaf->type); | |
127 | propname = cache_type_info[ct_idx].size_prop; | |
128 | ||
3a34c986 | 129 | of_property_read_u32(np, propname, &this_leaf->size); |
dfea747d SH |
130 | } |
131 | ||
132 | /* not cache_line_size() because that's a macro in include/linux/cache.h */ | |
2ff075c7 JL |
133 | static void cache_get_line_size(struct cacheinfo *this_leaf, |
134 | struct device_node *np) | |
dfea747d | 135 | { |
dfea747d SH |
136 | int i, lim, ct_idx; |
137 | ||
138 | ct_idx = get_cacheinfo_idx(this_leaf->type); | |
139 | lim = ARRAY_SIZE(cache_type_info[ct_idx].line_size_props); | |
140 | ||
141 | for (i = 0; i < lim; i++) { | |
448a5a55 SH |
142 | int ret; |
143 | u32 line_size; | |
dfea747d SH |
144 | const char *propname; |
145 | ||
146 | propname = cache_type_info[ct_idx].line_size_props[i]; | |
448a5a55 SH |
147 | ret = of_property_read_u32(np, propname, &line_size); |
148 | if (!ret) { | |
149 | this_leaf->coherency_line_size = line_size; | |
dfea747d | 150 | break; |
448a5a55 | 151 | } |
dfea747d | 152 | } |
dfea747d SH |
153 | } |
154 | ||
2ff075c7 | 155 | static void cache_nr_sets(struct cacheinfo *this_leaf, struct device_node *np) |
dfea747d SH |
156 | { |
157 | const char *propname; | |
dfea747d SH |
158 | int ct_idx; |
159 | ||
160 | ct_idx = get_cacheinfo_idx(this_leaf->type); | |
161 | propname = cache_type_info[ct_idx].nr_sets_prop; | |
162 | ||
3a34c986 | 163 | of_property_read_u32(np, propname, &this_leaf->number_of_sets); |
dfea747d SH |
164 | } |
165 | ||
166 | static void cache_associativity(struct cacheinfo *this_leaf) | |
167 | { | |
168 | unsigned int line_size = this_leaf->coherency_line_size; | |
169 | unsigned int nr_sets = this_leaf->number_of_sets; | |
170 | unsigned int size = this_leaf->size; | |
171 | ||
172 | /* | |
173 | * If the cache is fully associative, there is no need to | |
174 | * check the other properties. | |
175 | */ | |
176 | if (!(nr_sets == 1) && (nr_sets > 0 && size > 0 && line_size > 0)) | |
177 | this_leaf->ways_of_associativity = (size / nr_sets) / line_size; | |
178 | } | |
179 | ||
2ff075c7 JL |
180 | static bool cache_node_is_unified(struct cacheinfo *this_leaf, |
181 | struct device_node *np) | |
f57ab9a0 | 182 | { |
2ff075c7 | 183 | return of_property_read_bool(np, "cache-unified"); |
f57ab9a0 SH |
184 | } |
185 | ||
2ff075c7 JL |
186 | static void cache_of_set_props(struct cacheinfo *this_leaf, |
187 | struct device_node *np) | |
dfea747d | 188 | { |
2ff075c7 JL |
189 | /* |
190 | * init_cache_level must setup the cache level correctly | |
191 | * overriding the architecturally specified levels, so | |
192 | * if type is NONE at this stage, it should be unified | |
193 | */ | |
194 | if (this_leaf->type == CACHE_TYPE_NOCACHE && | |
195 | cache_node_is_unified(this_leaf, np)) | |
196 | this_leaf->type = CACHE_TYPE_UNIFIED; | |
197 | cache_size(this_leaf, np); | |
198 | cache_get_line_size(this_leaf, np); | |
199 | cache_nr_sets(this_leaf, np); | |
200 | cache_associativity(this_leaf); | |
dfea747d | 201 | } |
d529a18a JL |
202 | |
203 | static int cache_setup_of_node(unsigned int cpu) | |
204 | { | |
3da72e18 | 205 | struct device_node *np, *prev; |
d529a18a | 206 | struct cacheinfo *this_leaf; |
d529a18a JL |
207 | unsigned int index = 0; |
208 | ||
d4ec840b | 209 | np = of_cpu_device_node_get(cpu); |
d529a18a JL |
210 | if (!np) { |
211 | pr_err("Failed to find cpu%d device node\n", cpu); | |
212 | return -ENOENT; | |
213 | } | |
214 | ||
cde0fbff PG |
215 | if (!of_check_cache_nodes(np)) { |
216 | of_node_put(np); | |
217 | return -ENOENT; | |
218 | } | |
219 | ||
3da72e18 PG |
220 | prev = np; |
221 | ||
d529a18a | 222 | while (index < cache_leaves(cpu)) { |
b14e8d21 | 223 | this_leaf = per_cpu_cacheinfo_idx(cpu, index); |
3da72e18 | 224 | if (this_leaf->level != 1) { |
d529a18a | 225 | np = of_find_next_cache_node(np); |
3da72e18 PG |
226 | of_node_put(prev); |
227 | prev = np; | |
228 | if (!np) | |
229 | break; | |
230 | } | |
2ff075c7 | 231 | cache_of_set_props(this_leaf, np); |
9b97387c | 232 | this_leaf->fw_token = np; |
d529a18a JL |
233 | index++; |
234 | } | |
235 | ||
3da72e18 PG |
236 | of_node_put(np); |
237 | ||
d529a18a JL |
238 | if (index != cache_leaves(cpu)) /* not all OF nodes populated */ |
239 | return -ENOENT; | |
240 | ||
241 | return 0; | |
242 | } | |
c3719bd9 | 243 | |
cde0fbff PG |
244 | static bool of_check_cache_nodes(struct device_node *np) |
245 | { | |
246 | struct device_node *next; | |
247 | ||
248 | if (of_property_present(np, "cache-size") || | |
249 | of_property_present(np, "i-cache-size") || | |
250 | of_property_present(np, "d-cache-size") || | |
251 | of_property_present(np, "cache-unified")) | |
252 | return true; | |
253 | ||
254 | next = of_find_next_cache_node(np); | |
255 | if (next) { | |
256 | of_node_put(next); | |
257 | return true; | |
258 | } | |
259 | ||
260 | return false; | |
261 | } | |
262 | ||
de0df442 | 263 | static int of_count_cache_leaves(struct device_node *np) |
c3719bd9 | 264 | { |
de0df442 | 265 | unsigned int leaves = 0; |
c3719bd9 PG |
266 | |
267 | if (of_property_read_bool(np, "cache-size")) | |
268 | ++leaves; | |
269 | if (of_property_read_bool(np, "i-cache-size")) | |
270 | ++leaves; | |
271 | if (of_property_read_bool(np, "d-cache-size")) | |
272 | ++leaves; | |
de0df442 PG |
273 | |
274 | if (!leaves) { | |
275 | /* The '[i-|d-|]cache-size' property is required, but | |
276 | * if absent, fallback on the 'cache-unified' property. | |
277 | */ | |
278 | if (of_property_read_bool(np, "cache-unified")) | |
279 | return 1; | |
280 | else | |
281 | return 2; | |
282 | } | |
283 | ||
284 | return leaves; | |
285 | } | |
286 | ||
287 | int init_of_cache_level(unsigned int cpu) | |
288 | { | |
289 | struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); | |
290 | struct device_node *np = of_cpu_device_node_get(cpu); | |
291 | struct device_node *prev = NULL; | |
292 | unsigned int levels = 0, leaves, level; | |
293 | ||
cde0fbff PG |
294 | if (!of_check_cache_nodes(np)) { |
295 | of_node_put(np); | |
296 | return -ENOENT; | |
297 | } | |
298 | ||
de0df442 | 299 | leaves = of_count_cache_leaves(np); |
c3719bd9 PG |
300 | if (leaves > 0) |
301 | levels = 1; | |
302 | ||
303 | prev = np; | |
304 | while ((np = of_find_next_cache_node(np))) { | |
305 | of_node_put(prev); | |
306 | prev = np; | |
307 | if (!of_device_is_compatible(np, "cache")) | |
8844c3df | 308 | goto err_out; |
c3719bd9 | 309 | if (of_property_read_u32(np, "cache-level", &level)) |
8844c3df | 310 | goto err_out; |
c3719bd9 | 311 | if (level <= levels) |
8844c3df | 312 | goto err_out; |
de0df442 PG |
313 | |
314 | leaves += of_count_cache_leaves(np); | |
c3719bd9 PG |
315 | levels = level; |
316 | } | |
317 | ||
318 | of_node_put(np); | |
319 | this_cpu_ci->num_levels = levels; | |
320 | this_cpu_ci->num_leaves = leaves; | |
321 | ||
322 | return 0; | |
8844c3df PG |
323 | |
324 | err_out: | |
325 | of_node_put(np); | |
326 | return -EINVAL; | |
c3719bd9 PG |
327 | } |
328 | ||
246246cb SH |
329 | #else |
330 | static inline int cache_setup_of_node(unsigned int cpu) { return 0; } | |
c3719bd9 | 331 | int init_of_cache_level(unsigned int cpu) { return 0; } |
246246cb SH |
332 | #endif |
333 | ||
582b468b JL |
334 | int __weak cache_setup_acpi(unsigned int cpu) |
335 | { | |
336 | return -ENOTSUPP; | |
337 | } | |
338 | ||
9a83c84c SZ |
339 | unsigned int coherency_max_size; |
340 | ||
36bbc5b4 SH |
341 | static int cache_setup_properties(unsigned int cpu) |
342 | { | |
343 | int ret = 0; | |
344 | ||
345 | if (of_have_populated_dt()) | |
346 | ret = cache_setup_of_node(cpu); | |
347 | else if (!acpi_disabled) | |
348 | ret = cache_setup_acpi(cpu); | |
349 | ||
ef9f643a PG |
350 | // Assume there is no cache information available in DT/ACPI from now. |
351 | if (ret && use_arch_cache_info()) | |
352 | use_arch_info = true; | |
353 | ||
36bbc5b4 SH |
354 | return ret; |
355 | } | |
356 | ||
246246cb SH |
357 | static int cache_shared_cpu_map_setup(unsigned int cpu) |
358 | { | |
359 | struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); | |
360 | struct cacheinfo *this_leaf, *sib_leaf; | |
198102c9 | 361 | unsigned int index, sib_index; |
55877ef4 | 362 | int ret = 0; |
246246cb | 363 | |
fac51482 SH |
364 | if (this_cpu_ci->cpu_map_populated) |
365 | return 0; | |
366 | ||
36bbc5b4 SH |
367 | /* |
368 | * skip setting up cache properties if LLC is valid, just need | |
369 | * to update the shared cpu_map if the cache attributes were | |
370 | * populated early before all the cpus are brought online | |
371 | */ | |
ef9f643a | 372 | if (!last_level_cache_is_valid(cpu) && !use_arch_info) { |
36bbc5b4 SH |
373 | ret = cache_setup_properties(cpu); |
374 | if (ret) | |
375 | return ret; | |
376 | } | |
246246cb SH |
377 | |
378 | for (index = 0; index < cache_leaves(cpu); index++) { | |
379 | unsigned int i; | |
380 | ||
b14e8d21 | 381 | this_leaf = per_cpu_cacheinfo_idx(cpu, index); |
246246cb SH |
382 | |
383 | cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); | |
384 | for_each_online_cpu(i) { | |
385 | struct cpu_cacheinfo *sib_cpu_ci = get_cpu_cacheinfo(i); | |
386 | ||
387 | if (i == cpu || !sib_cpu_ci->info_list) | |
388 | continue;/* skip if itself or no cacheinfo */ | |
198102c9 YXW |
389 | for (sib_index = 0; sib_index < cache_leaves(i); sib_index++) { |
390 | sib_leaf = per_cpu_cacheinfo_idx(i, sib_index); | |
126310c9 PN |
391 | |
392 | /* | |
393 | * Comparing cache IDs only makes sense if the leaves | |
394 | * belong to the same cache level of same type. Skip | |
395 | * the check if level and type do not match. | |
396 | */ | |
397 | if (sib_leaf->level != this_leaf->level || | |
398 | sib_leaf->type != this_leaf->type) | |
399 | continue; | |
400 | ||
198102c9 YXW |
401 | if (cache_leaves_are_shared(this_leaf, sib_leaf)) { |
402 | cpumask_set_cpu(cpu, &sib_leaf->shared_cpu_map); | |
403 | cpumask_set_cpu(i, &this_leaf->shared_cpu_map); | |
404 | break; | |
405 | } | |
246246cb SH |
406 | } |
407 | } | |
9a83c84c SZ |
408 | /* record the maximum cache line size */ |
409 | if (this_leaf->coherency_line_size > coherency_max_size) | |
410 | coherency_max_size = this_leaf->coherency_line_size; | |
246246cb SH |
411 | } |
412 | ||
c26fabe7 PN |
413 | /* shared_cpu_map is now populated for the cpu */ |
414 | this_cpu_ci->cpu_map_populated = true; | |
246246cb SH |
415 | return 0; |
416 | } | |
417 | ||
418 | static void cache_shared_cpu_map_remove(unsigned int cpu) | |
419 | { | |
c26fabe7 | 420 | struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); |
246246cb | 421 | struct cacheinfo *this_leaf, *sib_leaf; |
198102c9 | 422 | unsigned int sibling, index, sib_index; |
246246cb SH |
423 | |
424 | for (index = 0; index < cache_leaves(cpu); index++) { | |
b14e8d21 | 425 | this_leaf = per_cpu_cacheinfo_idx(cpu, index); |
246246cb | 426 | for_each_cpu(sibling, &this_leaf->shared_cpu_map) { |
52110313 SH |
427 | struct cpu_cacheinfo *sib_cpu_ci = |
428 | get_cpu_cacheinfo(sibling); | |
2110d70c | 429 | |
52110313 SH |
430 | if (sibling == cpu || !sib_cpu_ci->info_list) |
431 | continue;/* skip if itself or no cacheinfo */ | |
2110d70c | 432 | |
198102c9 YXW |
433 | for (sib_index = 0; sib_index < cache_leaves(sibling); sib_index++) { |
434 | sib_leaf = per_cpu_cacheinfo_idx(sibling, sib_index); | |
126310c9 PN |
435 | |
436 | /* | |
437 | * Comparing cache IDs only makes sense if the leaves | |
438 | * belong to the same cache level of same type. Skip | |
439 | * the check if level and type do not match. | |
440 | */ | |
441 | if (sib_leaf->level != this_leaf->level || | |
442 | sib_leaf->type != this_leaf->type) | |
443 | continue; | |
444 | ||
198102c9 YXW |
445 | if (cache_leaves_are_shared(this_leaf, sib_leaf)) { |
446 | cpumask_clear_cpu(cpu, &sib_leaf->shared_cpu_map); | |
447 | cpumask_clear_cpu(sibling, &this_leaf->shared_cpu_map); | |
448 | break; | |
449 | } | |
450 | } | |
246246cb | 451 | } |
246246cb | 452 | } |
c26fabe7 PN |
453 | |
454 | /* cpu is no longer populated in the shared map */ | |
455 | this_cpu_ci->cpu_map_populated = false; | |
246246cb SH |
456 | } |
457 | ||
458 | static void free_cache_attributes(unsigned int cpu) | |
459 | { | |
2110d70c BP |
460 | if (!per_cpu_cacheinfo(cpu)) |
461 | return; | |
462 | ||
246246cb | 463 | cache_shared_cpu_map_remove(cpu); |
246246cb SH |
464 | } |
465 | ||
6539cffa RR |
466 | int __weak early_cache_level(unsigned int cpu) |
467 | { | |
468 | return -ENOENT; | |
469 | } | |
470 | ||
246246cb SH |
471 | int __weak init_cache_level(unsigned int cpu) |
472 | { | |
473 | return -ENOENT; | |
474 | } | |
475 | ||
476 | int __weak populate_cache_leaves(unsigned int cpu) | |
477 | { | |
478 | return -ENOENT; | |
479 | } | |
480 | ||
5944ce09 PG |
481 | static inline |
482 | int allocate_cache_info(int cpu) | |
483 | { | |
484 | per_cpu_cacheinfo(cpu) = kcalloc(cache_leaves(cpu), | |
485 | sizeof(struct cacheinfo), GFP_ATOMIC); | |
486 | if (!per_cpu_cacheinfo(cpu)) { | |
487 | cache_leaves(cpu) = 0; | |
488 | return -ENOMEM; | |
489 | } | |
490 | ||
491 | return 0; | |
492 | } | |
493 | ||
494 | int fetch_cache_info(unsigned int cpu) | |
495 | { | |
6539cffa | 496 | struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); |
ecaef469 | 497 | unsigned int levels = 0, split_levels = 0; |
5944ce09 PG |
498 | int ret; |
499 | ||
500 | if (acpi_disabled) { | |
501 | ret = init_of_cache_level(cpu); | |
5944ce09 PG |
502 | } else { |
503 | ret = acpi_get_cache_info(cpu, &levels, &split_levels); | |
6539cffa RR |
504 | if (!ret) { |
505 | this_cpu_ci->num_levels = levels; | |
506 | /* | |
507 | * This assumes that: | |
508 | * - there cannot be any split caches (data/instruction) | |
509 | * above a unified cache | |
510 | * - data/instruction caches come by pair | |
511 | */ | |
512 | this_cpu_ci->num_leaves = levels + split_levels; | |
513 | } | |
514 | } | |
515 | ||
516 | if (ret || !cache_leaves(cpu)) { | |
517 | ret = early_cache_level(cpu); | |
518 | if (ret) | |
5944ce09 PG |
519 | return ret; |
520 | ||
6539cffa RR |
521 | if (!cache_leaves(cpu)) |
522 | return -ENOENT; | |
523 | ||
524 | this_cpu_ci->early_ci_levels = true; | |
5944ce09 | 525 | } |
5944ce09 PG |
526 | |
527 | return allocate_cache_info(cpu); | |
528 | } | |
529 | ||
6539cffa | 530 | static inline int init_level_allocate_ci(unsigned int cpu) |
246246cb | 531 | { |
6539cffa | 532 | unsigned int early_leaves = cache_leaves(cpu); |
246246cb | 533 | |
5944ce09 PG |
534 | /* Since early initialization/allocation of the cacheinfo is allowed |
535 | * via fetch_cache_info() and this also gets called as CPU hotplug | |
536 | * callbacks via cacheinfo_cpu_online, the init/alloc can be skipped | |
537 | * as it will happen only once (the cacheinfo memory is never freed). | |
6539cffa RR |
538 | * Just populate the cacheinfo. However, if the cacheinfo has been |
539 | * allocated early through the arch-specific early_cache_level() call, | |
540 | * there is a chance the info is wrong (this can happen on arm64). In | |
541 | * that case, call init_cache_level() anyway to give the arch-specific | |
542 | * code a chance to make things right. | |
36bbc5b4 | 543 | */ |
6539cffa RR |
544 | if (per_cpu_cacheinfo(cpu) && !ci_cacheinfo(cpu)->early_ci_levels) |
545 | return 0; | |
36bbc5b4 | 546 | |
3370e13a | 547 | if (init_cache_level(cpu) || !cache_leaves(cpu)) |
246246cb SH |
548 | return -ENOENT; |
549 | ||
6539cffa RR |
550 | /* |
551 | * Now that we have properly initialized the cache level info, make | |
552 | * sure we don't try to do that again the next time we are called | |
553 | * (e.g. as CPU hotplug callbacks). | |
554 | */ | |
555 | ci_cacheinfo(cpu)->early_ci_levels = false; | |
556 | ||
557 | if (cache_leaves(cpu) <= early_leaves) | |
558 | return 0; | |
559 | ||
560 | kfree(per_cpu_cacheinfo(cpu)); | |
561 | return allocate_cache_info(cpu); | |
562 | } | |
563 | ||
564 | int detect_cache_attributes(unsigned int cpu) | |
565 | { | |
566 | int ret; | |
567 | ||
568 | ret = init_level_allocate_ci(cpu); | |
5944ce09 PG |
569 | if (ret) |
570 | return ret; | |
246246cb | 571 | |
2ff075c7 | 572 | /* |
5c271238 YY |
573 | * If LLC is valid the cache leaves were already populated so just go to |
574 | * update the cpu map. | |
2ff075c7 | 575 | */ |
5c271238 YY |
576 | if (!last_level_cache_is_valid(cpu)) { |
577 | /* | |
578 | * populate_cache_leaves() may completely setup the cache leaves and | |
579 | * shared_cpu_map or it may leave it partially setup. | |
580 | */ | |
581 | ret = populate_cache_leaves(cpu); | |
582 | if (ret) | |
583 | goto free_ci; | |
584 | } | |
36bbc5b4 | 585 | |
246246cb | 586 | /* |
9b97387c JL |
587 | * For systems using DT for cache hierarchy, fw_token |
588 | * and shared_cpu_map will be set up here only if they are | |
589 | * not populated already | |
246246cb SH |
590 | */ |
591 | ret = cache_shared_cpu_map_setup(cpu); | |
8a7d95f9 | 592 | if (ret) { |
55877ef4 | 593 | pr_warn("Unable to detect cache hierarchy for CPU %d\n", cpu); |
246246cb | 594 | goto free_ci; |
8a7d95f9 | 595 | } |
dfea747d | 596 | |
246246cb SH |
597 | return 0; |
598 | ||
599 | free_ci: | |
600 | free_cache_attributes(cpu); | |
601 | return ret; | |
602 | } | |
603 | ||
604 | /* pointer to cpuX/cache device */ | |
605 | static DEFINE_PER_CPU(struct device *, ci_cache_dev); | |
606 | #define per_cpu_cache_dev(cpu) (per_cpu(ci_cache_dev, cpu)) | |
607 | ||
608 | static cpumask_t cache_dev_map; | |
609 | ||
610 | /* pointer to array of devices for cpuX/cache/indexY */ | |
611 | static DEFINE_PER_CPU(struct device **, ci_index_dev); | |
612 | #define per_cpu_index_dev(cpu) (per_cpu(ci_index_dev, cpu)) | |
613 | #define per_cache_index_dev(cpu, idx) ((per_cpu_index_dev(cpu))[idx]) | |
614 | ||
615 | #define show_one(file_name, object) \ | |
616 | static ssize_t file_name##_show(struct device *dev, \ | |
617 | struct device_attribute *attr, char *buf) \ | |
618 | { \ | |
619 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); \ | |
948b3edb | 620 | return sysfs_emit(buf, "%u\n", this_leaf->object); \ |
246246cb SH |
621 | } |
622 | ||
e9a2ea5a | 623 | show_one(id, id); |
246246cb SH |
624 | show_one(level, level); |
625 | show_one(coherency_line_size, coherency_line_size); | |
626 | show_one(number_of_sets, number_of_sets); | |
627 | show_one(physical_line_partition, physical_line_partition); | |
628 | show_one(ways_of_associativity, ways_of_associativity); | |
629 | ||
630 | static ssize_t size_show(struct device *dev, | |
631 | struct device_attribute *attr, char *buf) | |
632 | { | |
633 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); | |
634 | ||
aa838896 | 635 | return sysfs_emit(buf, "%uK\n", this_leaf->size >> 10); |
246246cb SH |
636 | } |
637 | ||
e015e036 JP |
638 | static ssize_t shared_cpu_map_show(struct device *dev, |
639 | struct device_attribute *attr, char *buf) | |
246246cb SH |
640 | { |
641 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); | |
642 | const struct cpumask *mask = &this_leaf->shared_cpu_map; | |
643 | ||
e015e036 | 644 | return sysfs_emit(buf, "%*pb\n", nr_cpu_ids, mask); |
246246cb SH |
645 | } |
646 | ||
647 | static ssize_t shared_cpu_list_show(struct device *dev, | |
648 | struct device_attribute *attr, char *buf) | |
649 | { | |
e015e036 JP |
650 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); |
651 | const struct cpumask *mask = &this_leaf->shared_cpu_map; | |
652 | ||
653 | return sysfs_emit(buf, "%*pbl\n", nr_cpu_ids, mask); | |
246246cb SH |
654 | } |
655 | ||
656 | static ssize_t type_show(struct device *dev, | |
657 | struct device_attribute *attr, char *buf) | |
658 | { | |
659 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); | |
973c3911 | 660 | const char *output; |
246246cb SH |
661 | |
662 | switch (this_leaf->type) { | |
663 | case CACHE_TYPE_DATA: | |
973c3911 JP |
664 | output = "Data"; |
665 | break; | |
246246cb | 666 | case CACHE_TYPE_INST: |
973c3911 JP |
667 | output = "Instruction"; |
668 | break; | |
246246cb | 669 | case CACHE_TYPE_UNIFIED: |
973c3911 JP |
670 | output = "Unified"; |
671 | break; | |
246246cb SH |
672 | default: |
673 | return -EINVAL; | |
674 | } | |
973c3911 JP |
675 | |
676 | return sysfs_emit(buf, "%s\n", output); | |
246246cb SH |
677 | } |
678 | ||
679 | static ssize_t allocation_policy_show(struct device *dev, | |
680 | struct device_attribute *attr, char *buf) | |
681 | { | |
682 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); | |
683 | unsigned int ci_attr = this_leaf->attributes; | |
973c3911 | 684 | const char *output; |
246246cb SH |
685 | |
686 | if ((ci_attr & CACHE_READ_ALLOCATE) && (ci_attr & CACHE_WRITE_ALLOCATE)) | |
973c3911 | 687 | output = "ReadWriteAllocate"; |
246246cb | 688 | else if (ci_attr & CACHE_READ_ALLOCATE) |
973c3911 | 689 | output = "ReadAllocate"; |
246246cb | 690 | else if (ci_attr & CACHE_WRITE_ALLOCATE) |
973c3911 JP |
691 | output = "WriteAllocate"; |
692 | else | |
693 | return 0; | |
694 | ||
695 | return sysfs_emit(buf, "%s\n", output); | |
246246cb SH |
696 | } |
697 | ||
698 | static ssize_t write_policy_show(struct device *dev, | |
699 | struct device_attribute *attr, char *buf) | |
700 | { | |
701 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); | |
702 | unsigned int ci_attr = this_leaf->attributes; | |
703 | int n = 0; | |
704 | ||
705 | if (ci_attr & CACHE_WRITE_THROUGH) | |
aa838896 | 706 | n = sysfs_emit(buf, "WriteThrough\n"); |
246246cb | 707 | else if (ci_attr & CACHE_WRITE_BACK) |
aa838896 | 708 | n = sysfs_emit(buf, "WriteBack\n"); |
246246cb SH |
709 | return n; |
710 | } | |
711 | ||
e9a2ea5a | 712 | static DEVICE_ATTR_RO(id); |
246246cb SH |
713 | static DEVICE_ATTR_RO(level); |
714 | static DEVICE_ATTR_RO(type); | |
715 | static DEVICE_ATTR_RO(coherency_line_size); | |
716 | static DEVICE_ATTR_RO(ways_of_associativity); | |
717 | static DEVICE_ATTR_RO(number_of_sets); | |
718 | static DEVICE_ATTR_RO(size); | |
719 | static DEVICE_ATTR_RO(allocation_policy); | |
720 | static DEVICE_ATTR_RO(write_policy); | |
721 | static DEVICE_ATTR_RO(shared_cpu_map); | |
722 | static DEVICE_ATTR_RO(shared_cpu_list); | |
723 | static DEVICE_ATTR_RO(physical_line_partition); | |
724 | ||
725 | static struct attribute *cache_default_attrs[] = { | |
e9a2ea5a | 726 | &dev_attr_id.attr, |
246246cb SH |
727 | &dev_attr_type.attr, |
728 | &dev_attr_level.attr, | |
729 | &dev_attr_shared_cpu_map.attr, | |
730 | &dev_attr_shared_cpu_list.attr, | |
731 | &dev_attr_coherency_line_size.attr, | |
732 | &dev_attr_ways_of_associativity.attr, | |
733 | &dev_attr_number_of_sets.attr, | |
734 | &dev_attr_size.attr, | |
735 | &dev_attr_allocation_policy.attr, | |
736 | &dev_attr_write_policy.attr, | |
737 | &dev_attr_physical_line_partition.attr, | |
738 | NULL | |
739 | }; | |
740 | ||
741 | static umode_t | |
742 | cache_default_attrs_is_visible(struct kobject *kobj, | |
743 | struct attribute *attr, int unused) | |
744 | { | |
745 | struct device *dev = kobj_to_dev(kobj); | |
746 | struct cacheinfo *this_leaf = dev_get_drvdata(dev); | |
747 | const struct cpumask *mask = &this_leaf->shared_cpu_map; | |
748 | umode_t mode = attr->mode; | |
749 | ||
e9a2ea5a FY |
750 | if ((attr == &dev_attr_id.attr) && (this_leaf->attributes & CACHE_ID)) |
751 | return mode; | |
246246cb SH |
752 | if ((attr == &dev_attr_type.attr) && this_leaf->type) |
753 | return mode; | |
754 | if ((attr == &dev_attr_level.attr) && this_leaf->level) | |
755 | return mode; | |
756 | if ((attr == &dev_attr_shared_cpu_map.attr) && !cpumask_empty(mask)) | |
757 | return mode; | |
758 | if ((attr == &dev_attr_shared_cpu_list.attr) && !cpumask_empty(mask)) | |
759 | return mode; | |
760 | if ((attr == &dev_attr_coherency_line_size.attr) && | |
761 | this_leaf->coherency_line_size) | |
762 | return mode; | |
763 | if ((attr == &dev_attr_ways_of_associativity.attr) && | |
764 | this_leaf->size) /* allow 0 = full associativity */ | |
765 | return mode; | |
766 | if ((attr == &dev_attr_number_of_sets.attr) && | |
767 | this_leaf->number_of_sets) | |
768 | return mode; | |
769 | if ((attr == &dev_attr_size.attr) && this_leaf->size) | |
770 | return mode; | |
771 | if ((attr == &dev_attr_write_policy.attr) && | |
772 | (this_leaf->attributes & CACHE_WRITE_POLICY_MASK)) | |
773 | return mode; | |
774 | if ((attr == &dev_attr_allocation_policy.attr) && | |
775 | (this_leaf->attributes & CACHE_ALLOCATE_POLICY_MASK)) | |
776 | return mode; | |
777 | if ((attr == &dev_attr_physical_line_partition.attr) && | |
778 | this_leaf->physical_line_partition) | |
779 | return mode; | |
780 | ||
781 | return 0; | |
782 | } | |
783 | ||
784 | static const struct attribute_group cache_default_group = { | |
785 | .attrs = cache_default_attrs, | |
786 | .is_visible = cache_default_attrs_is_visible, | |
787 | }; | |
788 | ||
789 | static const struct attribute_group *cache_default_groups[] = { | |
790 | &cache_default_group, | |
791 | NULL, | |
792 | }; | |
793 | ||
794 | static const struct attribute_group *cache_private_groups[] = { | |
795 | &cache_default_group, | |
796 | NULL, /* Place holder for private group */ | |
797 | NULL, | |
798 | }; | |
799 | ||
800 | const struct attribute_group * | |
801 | __weak cache_get_priv_group(struct cacheinfo *this_leaf) | |
802 | { | |
803 | return NULL; | |
804 | } | |
805 | ||
806 | static const struct attribute_group ** | |
807 | cache_get_attribute_groups(struct cacheinfo *this_leaf) | |
808 | { | |
809 | const struct attribute_group *priv_group = | |
810 | cache_get_priv_group(this_leaf); | |
811 | ||
812 | if (!priv_group) | |
813 | return cache_default_groups; | |
814 | ||
815 | if (!cache_private_groups[1]) | |
816 | cache_private_groups[1] = priv_group; | |
817 | ||
818 | return cache_private_groups; | |
819 | } | |
820 | ||
821 | /* Add/Remove cache interface for CPU device */ | |
822 | static void cpu_cache_sysfs_exit(unsigned int cpu) | |
823 | { | |
824 | int i; | |
825 | struct device *ci_dev; | |
826 | ||
827 | if (per_cpu_index_dev(cpu)) { | |
828 | for (i = 0; i < cache_leaves(cpu); i++) { | |
829 | ci_dev = per_cache_index_dev(cpu, i); | |
830 | if (!ci_dev) | |
831 | continue; | |
832 | device_unregister(ci_dev); | |
833 | } | |
834 | kfree(per_cpu_index_dev(cpu)); | |
835 | per_cpu_index_dev(cpu) = NULL; | |
836 | } | |
837 | device_unregister(per_cpu_cache_dev(cpu)); | |
838 | per_cpu_cache_dev(cpu) = NULL; | |
839 | } | |
840 | ||
841 | static int cpu_cache_sysfs_init(unsigned int cpu) | |
842 | { | |
843 | struct device *dev = get_cpu_device(cpu); | |
844 | ||
845 | if (per_cpu_cacheinfo(cpu) == NULL) | |
846 | return -ENOENT; | |
847 | ||
848 | per_cpu_cache_dev(cpu) = cpu_device_create(dev, NULL, NULL, "cache"); | |
849 | if (IS_ERR(per_cpu_cache_dev(cpu))) | |
850 | return PTR_ERR(per_cpu_cache_dev(cpu)); | |
851 | ||
852 | /* Allocate all required memory */ | |
853 | per_cpu_index_dev(cpu) = kcalloc(cache_leaves(cpu), | |
854 | sizeof(struct device *), GFP_KERNEL); | |
855 | if (unlikely(per_cpu_index_dev(cpu) == NULL)) | |
856 | goto err_out; | |
857 | ||
858 | return 0; | |
859 | ||
860 | err_out: | |
861 | cpu_cache_sysfs_exit(cpu); | |
862 | return -ENOMEM; | |
863 | } | |
864 | ||
865 | static int cache_add_dev(unsigned int cpu) | |
866 | { | |
867 | unsigned int i; | |
868 | int rc; | |
869 | struct device *ci_dev, *parent; | |
870 | struct cacheinfo *this_leaf; | |
246246cb SH |
871 | const struct attribute_group **cache_groups; |
872 | ||
873 | rc = cpu_cache_sysfs_init(cpu); | |
874 | if (unlikely(rc < 0)) | |
875 | return rc; | |
876 | ||
877 | parent = per_cpu_cache_dev(cpu); | |
878 | for (i = 0; i < cache_leaves(cpu); i++) { | |
b14e8d21 | 879 | this_leaf = per_cpu_cacheinfo_idx(cpu, i); |
246246cb SH |
880 | if (this_leaf->disable_sysfs) |
881 | continue; | |
ca388e43 JH |
882 | if (this_leaf->type == CACHE_TYPE_NOCACHE) |
883 | break; | |
246246cb SH |
884 | cache_groups = cache_get_attribute_groups(this_leaf); |
885 | ci_dev = cpu_device_create(parent, this_leaf, cache_groups, | |
886 | "index%1u", i); | |
887 | if (IS_ERR(ci_dev)) { | |
888 | rc = PTR_ERR(ci_dev); | |
889 | goto err; | |
890 | } | |
891 | per_cache_index_dev(cpu, i) = ci_dev; | |
892 | } | |
893 | cpumask_set_cpu(cpu, &cache_dev_map); | |
894 | ||
895 | return 0; | |
896 | err: | |
897 | cpu_cache_sysfs_exit(cpu); | |
898 | return rc; | |
899 | } | |
900 | ||
94a3bfe4 HY |
901 | /* |
902 | * Calculate the size of the per-CPU data cache slice. This can be | |
903 | * used to estimate the size of the data cache slice that can be used | |
904 | * by one CPU under ideal circumstances. UNIFIED caches are counted | |
905 | * in addition to DATA caches. So, please consider code cache usage | |
906 | * when use the result. | |
907 | * | |
908 | * Because the cache inclusive/non-inclusive information isn't | |
909 | * available, we just use the size of the per-CPU slice of LLC to make | |
910 | * the result more predictable across architectures. | |
911 | */ | |
912 | static void update_per_cpu_data_slice_size_cpu(unsigned int cpu) | |
913 | { | |
914 | struct cpu_cacheinfo *ci; | |
915 | struct cacheinfo *llc; | |
916 | unsigned int nr_shared; | |
917 | ||
918 | if (!last_level_cache_is_valid(cpu)) | |
919 | return; | |
920 | ||
921 | ci = ci_cacheinfo(cpu); | |
922 | llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1); | |
923 | ||
924 | if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED) | |
925 | return; | |
926 | ||
927 | nr_shared = cpumask_weight(&llc->shared_cpu_map); | |
928 | if (nr_shared) | |
929 | ci->per_cpu_data_slice_size = llc->size / nr_shared; | |
930 | } | |
931 | ||
932 | static void update_per_cpu_data_slice_size(bool cpu_online, unsigned int cpu) | |
933 | { | |
934 | unsigned int icpu; | |
935 | ||
936 | for_each_online_cpu(icpu) { | |
937 | if (!cpu_online && icpu == cpu) | |
938 | continue; | |
939 | update_per_cpu_data_slice_size_cpu(icpu); | |
940 | } | |
941 | } | |
942 | ||
7cc277b4 | 943 | static int cacheinfo_cpu_online(unsigned int cpu) |
246246cb | 944 | { |
7cc277b4 | 945 | int rc = detect_cache_attributes(cpu); |
246246cb | 946 | |
7cc277b4 SAS |
947 | if (rc) |
948 | return rc; | |
949 | rc = cache_add_dev(cpu); | |
950 | if (rc) | |
94a3bfe4 HY |
951 | goto err; |
952 | update_per_cpu_data_slice_size(true, cpu); | |
362d37a1 | 953 | setup_pcp_cacheinfo(); |
94a3bfe4 HY |
954 | return 0; |
955 | err: | |
956 | free_cache_attributes(cpu); | |
7cc277b4 | 957 | return rc; |
246246cb SH |
958 | } |
959 | ||
7cc277b4 | 960 | static int cacheinfo_cpu_pre_down(unsigned int cpu) |
246246cb | 961 | { |
7cc277b4 SAS |
962 | if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map)) |
963 | cpu_cache_sysfs_exit(cpu); | |
964 | ||
965 | free_cache_attributes(cpu); | |
94a3bfe4 | 966 | update_per_cpu_data_slice_size(false, cpu); |
362d37a1 | 967 | setup_pcp_cacheinfo(); |
7cc277b4 | 968 | return 0; |
246246cb SH |
969 | } |
970 | ||
971 | static int __init cacheinfo_sysfs_init(void) | |
972 | { | |
83b44fe3 JM |
973 | return cpuhp_setup_state(CPUHP_AP_BASE_CACHEINFO_ONLINE, |
974 | "base/cacheinfo:online", | |
7cc277b4 | 975 | cacheinfo_cpu_online, cacheinfo_cpu_pre_down); |
246246cb | 976 | } |
246246cb | 977 | device_initcall(cacheinfo_sysfs_init); |