Commit | Line | Data |
---|---|---|
b2441318 | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
113c6097 FY |
2 | #ifndef _ASM_X86_INTEL_RDT_H |
3 | #define _ASM_X86_INTEL_RDT_H | |
4 | ||
5b825c3a | 5 | #include <linux/sched.h> |
8ff42c02 | 6 | #include <linux/kernfs.h> |
5ff193fb FY |
7 | #include <linux/jump_label.h> |
8 | ||
9 | #define IA32_L3_QOS_CFG 0xc81 | |
99adde9b | 10 | #define IA32_L2_QOS_CFG 0xc82 |
113c6097 | 11 | #define IA32_L3_CBM_BASE 0xc90 |
c1c7c3f9 | 12 | #define IA32_L2_CBM_BASE 0xd10 |
05b93417 | 13 | #define IA32_MBA_THRTL_BASE 0xd50 |
113c6097 | 14 | |
5ff193fb FY |
15 | #define L3_QOS_CDP_ENABLE 0x01ULL |
16 | ||
99adde9b FY |
17 | #define L2_QOS_CDP_ENABLE 0x01ULL |
18 | ||
6a445edc VS |
19 | /* |
20 | * Event IDs are used to program IA32_QM_EVTSEL before reading event | |
21 | * counter from IA32_QM_CTR | |
22 | */ | |
23 | #define QOS_L3_OCCUP_EVENT_ID 0x01 | |
24 | #define QOS_L3_MBM_TOTAL_EVENT_ID 0x02 | |
25 | #define QOS_L3_MBM_LOCAL_EVENT_ID 0x03 | |
9f52425b | 26 | |
24247aee VS |
27 | #define CQM_LIMBOCHECK_INTERVAL 1000 |
28 | ||
9f52425b | 29 | #define MBM_CNTR_WIDTH 24 |
e3302683 | 30 | #define MBM_OVERFLOW_INTERVAL 1000 |
9f52425b | 31 | |
edf6fa1c VS |
32 | #define RMID_VAL_ERROR BIT_ULL(63) |
33 | #define RMID_VAL_UNAVAIL BIT_ULL(62) | |
6a445edc | 34 | |
e3302683 VS |
35 | DECLARE_STATIC_KEY_FALSE(rdt_enable_key); |
36 | ||
6a445edc VS |
37 | /** |
38 | * struct mon_evt - Entry in the event list of a resource | |
39 | * @evtid: event id | |
40 | * @name: name of the event | |
41 | */ | |
42 | struct mon_evt { | |
43 | u32 evtid; | |
44 | char *name; | |
45 | struct list_head list; | |
46 | }; | |
47 | ||
d89b7379 VS |
48 | /** |
49 | * struct mon_data_bits - Monitoring details for each event file | |
50 | * @rid: Resource id associated with the event file. | |
51 | * @evtid: Event id associated with the event file | |
52 | * @domid: The domain to which the event file belongs | |
53 | */ | |
54 | union mon_data_bits { | |
55 | void *priv; | |
56 | struct { | |
57 | unsigned int rid : 10; | |
58 | unsigned int evtid : 8; | |
59 | unsigned int domid : 14; | |
60 | } u; | |
61 | }; | |
62 | ||
63 | struct rmid_read { | |
64 | struct rdtgroup *rgrp; | |
9f52425b | 65 | struct rdt_domain *d; |
d89b7379 | 66 | int evtid; |
a4de1dfd | 67 | bool first; |
d89b7379 VS |
68 | u64 val; |
69 | }; | |
70 | ||
6a445edc VS |
71 | extern unsigned int intel_cqm_threshold; |
72 | extern bool rdt_alloc_capable; | |
73 | extern bool rdt_mon_capable; | |
74 | extern unsigned int rdt_mon_features; | |
c7d9aac6 VS |
75 | |
76 | enum rdt_group_type { | |
77 | RDTCTRL_GROUP = 0, | |
78 | RDTMON_GROUP, | |
79 | RDT_NUM_GROUP, | |
80 | }; | |
81 | ||
82 | /** | |
83 | * struct mongroup - store mon group's data in resctrl fs. | |
d89b7379 | 84 | * @mon_data_kn kernlfs node for the mon_data directory |
c7d9aac6 VS |
85 | * @parent: parent rdtgrp |
86 | * @crdtgrp_list: child rdtgroup node list | |
87 | * @rmid: rmid for this rdtgroup | |
88 | */ | |
89 | struct mongroup { | |
d89b7379 | 90 | struct kernfs_node *mon_data_kn; |
c7d9aac6 VS |
91 | struct rdtgroup *parent; |
92 | struct list_head crdtgrp_list; | |
93 | u32 rmid; | |
94 | }; | |
95 | ||
5ff193fb FY |
96 | /** |
97 | * struct rdtgroup - store rdtgroup's data in resctrl file system. | |
98 | * @kn: kernfs node | |
99 | * @rdtgroup_list: linked list for all rdtgroups | |
100 | * @closid: closid for this rdtgroup | |
12e0110c | 101 | * @cpu_mask: CPUs assigned to this rdtgroup |
60cf5e10 FY |
102 | * @flags: status bits |
103 | * @waitcount: how many cpus expect to find this | |
12e0110c | 104 | * group when they acquire rdtgroup_mutex |
c7d9aac6 VS |
105 | * @type: indicates type of this rdtgroup - either |
106 | * monitor only or ctrl_mon group | |
107 | * @mon: mongroup related data | |
5ff193fb FY |
108 | */ |
109 | struct rdtgroup { | |
110 | struct kernfs_node *kn; | |
111 | struct list_head rdtgroup_list; | |
0734ded1 | 112 | u32 closid; |
12e0110c | 113 | struct cpumask cpu_mask; |
60cf5e10 FY |
114 | int flags; |
115 | atomic_t waitcount; | |
c7d9aac6 VS |
116 | enum rdt_group_type type; |
117 | struct mongroup mon; | |
5ff193fb FY |
118 | }; |
119 | ||
60cf5e10 FY |
120 | /* rdtgroup.flags */ |
121 | #define RDT_DELETED 1 | |
122 | ||
4ffa3c97 JO |
123 | /* rftype.flags */ |
124 | #define RFTYPE_FLAGS_CPUS_LIST 1 | |
125 | ||
5dc1d5c6 T |
126 | /* |
127 | * Define the file type flags for base and info directories. | |
128 | */ | |
129 | #define RFTYPE_INFO BIT(0) | |
130 | #define RFTYPE_BASE BIT(1) | |
131 | #define RF_CTRLSHIFT 4 | |
d4ab3320 | 132 | #define RF_MONSHIFT 5 |
9b3a7fd0 | 133 | #define RF_TOPSHIFT 6 |
5dc1d5c6 | 134 | #define RFTYPE_CTRL BIT(RF_CTRLSHIFT) |
d4ab3320 | 135 | #define RFTYPE_MON BIT(RF_MONSHIFT) |
9b3a7fd0 | 136 | #define RFTYPE_TOP BIT(RF_TOPSHIFT) |
5dc1d5c6 T |
137 | #define RFTYPE_RES_CACHE BIT(8) |
138 | #define RFTYPE_RES_MB BIT(9) | |
139 | #define RF_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL) | |
d4ab3320 | 140 | #define RF_MON_INFO (RFTYPE_INFO | RFTYPE_MON) |
9b3a7fd0 | 141 | #define RF_TOP_INFO (RFTYPE_INFO | RFTYPE_TOP) |
5dc1d5c6 T |
142 | #define RF_CTRL_BASE (RFTYPE_BASE | RFTYPE_CTRL) |
143 | ||
5ff193fb FY |
144 | /* List of all resource groups */ |
145 | extern struct list_head rdt_all_groups; | |
146 | ||
de016df8 VS |
147 | extern int max_name_width, max_data_width; |
148 | ||
5ff193fb FY |
149 | int __init rdtgroup_init(void); |
150 | ||
4e978d06 FY |
151 | /** |
152 | * struct rftype - describe each file in the resctrl file system | |
17f8ba1d TG |
153 | * @name: File name |
154 | * @mode: Access mode | |
155 | * @kf_ops: File operations | |
4ffa3c97 | 156 | * @flags: File specific RFTYPE_FLAGS_* flags |
5dc1d5c6 | 157 | * @fflags: File specific RF_* or RFTYPE_* flags |
17f8ba1d TG |
158 | * @seq_show: Show content of the file |
159 | * @write: Write to the file | |
4e978d06 FY |
160 | */ |
161 | struct rftype { | |
162 | char *name; | |
163 | umode_t mode; | |
164 | struct kernfs_ops *kf_ops; | |
4ffa3c97 | 165 | unsigned long flags; |
5dc1d5c6 | 166 | unsigned long fflags; |
4e978d06 FY |
167 | |
168 | int (*seq_show)(struct kernfs_open_file *of, | |
169 | struct seq_file *sf, void *v); | |
170 | /* | |
171 | * write() is the generic write callback which maps directly to | |
172 | * kernfs write operation and overrides all other operations. | |
173 | * Maximum write size is determined by ->max_write_len. | |
174 | */ | |
175 | ssize_t (*write)(struct kernfs_open_file *of, | |
176 | char *buf, size_t nbytes, loff_t off); | |
177 | }; | |
178 | ||
9f52425b TL |
179 | /** |
180 | * struct mbm_state - status for each MBM counter in each domain | |
181 | * @chunks: Total data moved (multiply by rdt_group.mon_scale to get bytes) | |
182 | * @prev_msr Value of IA32_QM_CTR for this RMID last time we read it | |
183 | */ | |
184 | struct mbm_state { | |
185 | u64 chunks; | |
186 | u64 prev_msr; | |
187 | }; | |
188 | ||
0921c547 TG |
189 | /** |
190 | * struct rdt_domain - group of cpus sharing an RDT resource | |
191 | * @list: all instances of this resource | |
192 | * @id: unique id for this instance | |
193 | * @cpu_mask: which cpus share this resource | |
edf6fa1c VS |
194 | * @rmid_busy_llc: |
195 | * bitmap of which limbo RMIDs are above threshold | |
9f52425b TL |
196 | * @mbm_total: saved state for MBM total bandwidth |
197 | * @mbm_local: saved state for MBM local bandwidth | |
e3302683 | 198 | * @mbm_over: worker to periodically read MBM h/w counters |
24247aee | 199 | * @cqm_limbo: worker to periodically read CQM h/w counters |
e3302683 VS |
200 | * @mbm_work_cpu: |
201 | * worker cpu for MBM h/w counters | |
24247aee VS |
202 | * @cqm_work_cpu: |
203 | * worker cpu for CQM h/w counters | |
0921c547 TG |
204 | * @ctrl_val: array of cache or mem ctrl values (indexed by CLOSID) |
205 | * @new_ctrl: new ctrl value to be loaded | |
206 | * @have_new_ctrl: did user provide new_ctrl for this domain | |
207 | */ | |
208 | struct rdt_domain { | |
209 | struct list_head list; | |
210 | int id; | |
211 | struct cpumask cpu_mask; | |
edf6fa1c | 212 | unsigned long *rmid_busy_llc; |
9f52425b TL |
213 | struct mbm_state *mbm_total; |
214 | struct mbm_state *mbm_local; | |
e3302683 | 215 | struct delayed_work mbm_over; |
24247aee | 216 | struct delayed_work cqm_limbo; |
e3302683 | 217 | int mbm_work_cpu; |
24247aee | 218 | int cqm_work_cpu; |
0921c547 TG |
219 | u32 *ctrl_val; |
220 | u32 new_ctrl; | |
221 | bool have_new_ctrl; | |
222 | }; | |
223 | ||
224 | /** | |
225 | * struct msr_param - set a range of MSRs from a domain | |
226 | * @res: The resource to use | |
227 | * @low: Beginning index from base MSR | |
228 | * @high: End index | |
229 | */ | |
230 | struct msr_param { | |
231 | struct rdt_resource *res; | |
232 | int low; | |
233 | int high; | |
234 | }; | |
235 | ||
d3e11b4d TG |
236 | /** |
237 | * struct rdt_cache - Cache allocation related data | |
238 | * @cbm_len: Length of the cache bit mask | |
239 | * @min_cbm_bits: Minimum number of consecutive bits to be set | |
240 | * @cbm_idx_mult: Multiplier of CBM index | |
241 | * @cbm_idx_offset: Offset of CBM index. CBM index is computed by: | |
242 | * closid * cbm_idx_multi + cbm_idx_offset | |
243 | * in a cache bit mask | |
0dd2d749 FY |
244 | * @shareable_bits: Bitmask of shareable resource with other |
245 | * executing entities | |
d3e11b4d TG |
246 | */ |
247 | struct rdt_cache { | |
248 | unsigned int cbm_len; | |
249 | unsigned int min_cbm_bits; | |
250 | unsigned int cbm_idx_mult; | |
251 | unsigned int cbm_idx_offset; | |
0dd2d749 | 252 | unsigned int shareable_bits; |
d3e11b4d TG |
253 | }; |
254 | ||
05b93417 VS |
255 | /** |
256 | * struct rdt_membw - Memory bandwidth allocation related data | |
257 | * @max_delay: Max throttle delay. Delay is the hardware | |
258 | * representation for memory bandwidth. | |
259 | * @min_bw: Minimum memory bandwidth percentage user can request | |
260 | * @bw_gran: Granularity at which the memory bandwidth is allocated | |
261 | * @delay_linear: True if memory B/W delay is in linear scale | |
19c635ab | 262 | * @mba_sc: True if MBA software controller(mba_sc) is enabled |
05b93417 VS |
263 | * @mb_map: Mapping of memory B/W percentage to memory B/W delay |
264 | */ | |
265 | struct rdt_membw { | |
266 | u32 max_delay; | |
267 | u32 min_bw; | |
268 | u32 bw_gran; | |
269 | u32 delay_linear; | |
19c635ab | 270 | bool mba_sc; |
05b93417 VS |
271 | u32 *mb_map; |
272 | }; | |
273 | ||
6a445edc VS |
274 | static inline bool is_llc_occupancy_enabled(void) |
275 | { | |
276 | return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID)); | |
277 | } | |
278 | ||
9f52425b TL |
279 | static inline bool is_mbm_total_enabled(void) |
280 | { | |
281 | return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID)); | |
282 | } | |
283 | ||
284 | static inline bool is_mbm_local_enabled(void) | |
285 | { | |
286 | return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID)); | |
287 | } | |
288 | ||
289 | static inline bool is_mbm_enabled(void) | |
290 | { | |
291 | return (is_mbm_total_enabled() || is_mbm_local_enabled()); | |
292 | } | |
293 | ||
a4de1dfd VS |
294 | static inline bool is_mbm_event(int e) |
295 | { | |
296 | return (e >= QOS_L3_MBM_TOTAL_EVENT_ID && | |
297 | e <= QOS_L3_MBM_LOCAL_EVENT_ID); | |
298 | } | |
299 | ||
c1c7c3f9 FY |
300 | /** |
301 | * struct rdt_resource - attributes of an RDT resource | |
d89b7379 | 302 | * @rid: The index of the resource |
1b5c0b75 | 303 | * @alloc_enabled: Is allocation enabled on this machine |
6a445edc | 304 | * @mon_enabled: Is monitoring enabled for this feature |
1b5c0b75 | 305 | * @alloc_capable: Is allocation available on this machine |
6a445edc | 306 | * @mon_capable: Is monitor feature available on this machine |
d3e11b4d TG |
307 | * @name: Name to use in "schemata" file |
308 | * @num_closid: Number of CLOSIDs available | |
309 | * @cache_level: Which cache level defines scope of this resource | |
310 | * @default_ctrl: Specifies default cache cbm or memory B/W percent. | |
311 | * @msr_base: Base MSR address for CBMs | |
0921c547 | 312 | * @msr_update: Function pointer to update QOS MSRs |
d3e11b4d TG |
313 | * @data_width: Character width of data when displaying |
314 | * @domains: All domains for this resource | |
315 | * @cache: Cache allocation related data | |
c6ea67de VS |
316 | * @format_str: Per resource format string to show domain value |
317 | * @parse_ctrlval: Per resource function pointer to parse control values | |
6a445edc VS |
318 | * @evt_list: List of monitoring events |
319 | * @num_rmid: Number of RMIDs available | |
320 | * @mon_scale: cqm counter * mon_scale = occupancy in bytes | |
5dc1d5c6 | 321 | * @fflags: flags to choose base and info files |
c1c7c3f9 FY |
322 | */ |
323 | struct rdt_resource { | |
d89b7379 | 324 | int rid; |
1b5c0b75 | 325 | bool alloc_enabled; |
6a445edc | 326 | bool mon_enabled; |
1b5c0b75 | 327 | bool alloc_capable; |
6a445edc | 328 | bool mon_capable; |
c1c7c3f9 FY |
329 | char *name; |
330 | int num_closid; | |
d3e11b4d | 331 | int cache_level; |
2545e9f5 | 332 | u32 default_ctrl; |
d3e11b4d | 333 | unsigned int msr_base; |
0921c547 TG |
334 | void (*msr_update) (struct rdt_domain *d, struct msr_param *m, |
335 | struct rdt_resource *r); | |
de016df8 | 336 | int data_width; |
c1c7c3f9 | 337 | struct list_head domains; |
a83827d0 TG |
338 | struct rdt_cache cache; |
339 | struct rdt_membw membw; | |
c6ea67de VS |
340 | const char *format_str; |
341 | int (*parse_ctrlval) (char *buf, struct rdt_resource *r, | |
342 | struct rdt_domain *d); | |
6a445edc VS |
343 | struct list_head evt_list; |
344 | int num_rmid; | |
345 | unsigned int mon_scale; | |
5dc1d5c6 | 346 | unsigned long fflags; |
c1c7c3f9 FY |
347 | }; |
348 | ||
c6ea67de | 349 | int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d); |
64e8ed3d | 350 | int parse_bw(char *buf, struct rdt_resource *r, struct rdt_domain *d); |
6a507a6a | 351 | |
2264d9c7 TL |
352 | extern struct mutex rdtgroup_mutex; |
353 | ||
c1c7c3f9 | 354 | extern struct rdt_resource rdt_resources_all[]; |
5ff193fb | 355 | extern struct rdtgroup rdtgroup_default; |
1b5c0b75 | 356 | DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key); |
5ff193fb FY |
357 | |
358 | int __init rdtgroup_init(void); | |
c1c7c3f9 FY |
359 | |
360 | enum { | |
361 | RDT_RESOURCE_L3, | |
362 | RDT_RESOURCE_L3DATA, | |
363 | RDT_RESOURCE_L3CODE, | |
364 | RDT_RESOURCE_L2, | |
def10853 FY |
365 | RDT_RESOURCE_L2DATA, |
366 | RDT_RESOURCE_L2CODE, | |
05b93417 | 367 | RDT_RESOURCE_MBA, |
c1c7c3f9 FY |
368 | |
369 | /* Must be the last */ | |
370 | RDT_NUM_RESOURCES, | |
371 | }; | |
372 | ||
895c663e VS |
373 | #define for_each_capable_rdt_resource(r) \ |
374 | for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\ | |
375 | r++) \ | |
376 | if (r->alloc_capable || r->mon_capable) | |
377 | ||
1b5c0b75 | 378 | #define for_each_alloc_capable_rdt_resource(r) \ |
c1c7c3f9 | 379 | for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\ |
17f8ba1d | 380 | r++) \ |
1b5c0b75 | 381 | if (r->alloc_capable) |
c1c7c3f9 | 382 | |
6a445edc VS |
383 | #define for_each_mon_capable_rdt_resource(r) \ |
384 | for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\ | |
385 | r++) \ | |
386 | if (r->mon_capable) | |
387 | ||
1b5c0b75 | 388 | #define for_each_alloc_enabled_rdt_resource(r) \ |
2264d9c7 TL |
389 | for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\ |
390 | r++) \ | |
1b5c0b75 | 391 | if (r->alloc_enabled) |
2264d9c7 | 392 | |
d4ab3320 VS |
393 | #define for_each_mon_enabled_rdt_resource(r) \ |
394 | for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\ | |
395 | r++) \ | |
396 | if (r->mon_enabled) | |
397 | ||
c1c7c3f9 FY |
398 | /* CPUID.(EAX=10H, ECX=ResID=1).EAX */ |
399 | union cpuid_0x10_1_eax { | |
400 | struct { | |
401 | unsigned int cbm_len:5; | |
402 | } split; | |
403 | unsigned int full; | |
404 | }; | |
405 | ||
ab66a33b VS |
406 | /* CPUID.(EAX=10H, ECX=ResID=3).EAX */ |
407 | union cpuid_0x10_3_eax { | |
408 | struct { | |
409 | unsigned int max_delay:12; | |
410 | } split; | |
411 | unsigned int full; | |
412 | }; | |
413 | ||
2545e9f5 VS |
414 | /* CPUID.(EAX=10H, ECX=ResID).EDX */ |
415 | union cpuid_0x10_x_edx { | |
c1c7c3f9 FY |
416 | struct { |
417 | unsigned int cos_max:16; | |
418 | } split; | |
419 | unsigned int full; | |
420 | }; | |
2264d9c7 | 421 | |
9b3a7fd0 TL |
422 | void rdt_last_cmd_clear(void); |
423 | void rdt_last_cmd_puts(const char *s); | |
424 | void rdt_last_cmd_printf(const char *fmt, ...); | |
425 | ||
2545e9f5 | 426 | void rdt_ctrl_update(void *arg); |
60cf5e10 FY |
427 | struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn); |
428 | void rdtgroup_kn_unlock(struct kernfs_node *kn); | |
d89b7379 VS |
429 | struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id, |
430 | struct list_head **pos); | |
60ec2440 TL |
431 | ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, |
432 | char *buf, size_t nbytes, loff_t off); | |
433 | int rdtgroup_schemata_show(struct kernfs_open_file *of, | |
434 | struct seq_file *s, void *v); | |
edf6fa1c | 435 | struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r); |
c7d9aac6 VS |
436 | int alloc_rmid(void); |
437 | void free_rmid(u32 rmid); | |
6a445edc | 438 | int rdt_get_mon_l3_config(struct rdt_resource *r); |
d89b7379 VS |
439 | void mon_event_count(void *info); |
440 | int rdtgroup_mondata_show(struct seq_file *m, void *arg); | |
895c663e VS |
441 | void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, |
442 | unsigned int dom_id); | |
443 | void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, | |
444 | struct rdt_domain *d); | |
a4de1dfd VS |
445 | void mon_event_read(struct rmid_read *rr, struct rdt_domain *d, |
446 | struct rdtgroup *rdtgrp, int evtid, int first); | |
24247aee VS |
447 | void mbm_setup_overflow_handler(struct rdt_domain *dom, |
448 | unsigned long delay_ms); | |
e3302683 | 449 | void mbm_handle_overflow(struct work_struct *work); |
19c635ab | 450 | bool is_mba_sc(struct rdt_resource *r); |
24247aee VS |
451 | void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms); |
452 | void cqm_handle_limbo(struct work_struct *work); | |
453 | bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d); | |
454 | void __check_limbo(struct rdt_domain *d, bool force_free); | |
4f341a5e | 455 | |
113c6097 | 456 | #endif /* _ASM_X86_INTEL_RDT_H */ |