Commit | Line | Data |
---|---|---|
dd08ebf6 MB |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2022 Intel Corporation | |
4 | */ | |
5 | ||
dd08ebf6 | 6 | #include "xe_gt_mcr.h" |
ea9f879d | 7 | |
226bfec8 | 8 | #include "regs/xe_gt_regs.h" |
bde5d767 | 9 | #include "xe_assert.h" |
ea9f879d | 10 | #include "xe_gt.h" |
fc7c7498 | 11 | #include "xe_gt_printk.h" |
dd08ebf6 MB |
12 | #include "xe_gt_topology.h" |
13 | #include "xe_gt_types.h" | |
fc7c7498 | 14 | #include "xe_guc_hwconfig.h" |
dd08ebf6 | 15 | #include "xe_mmio.h" |
be46d7aa | 16 | #include "xe_sriov.h" |
dd08ebf6 | 17 | |
dd08ebf6 MB |
18 | /** |
19 | * DOC: GT Multicast/Replicated (MCR) Register Support | |
20 | * | |
21 | * Some GT registers are designed as "multicast" or "replicated" registers: | |
22 | * multiple instances of the same register share a single MMIO offset. MCR | |
23 | * registers are generally used when the hardware needs to potentially track | |
24 | * independent values of a register per hardware unit (e.g., per-subslice, | |
25 | * per-L3bank, etc.). The specific types of replication that exist vary | |
26 | * per-platform. | |
27 | * | |
28 | * MMIO accesses to MCR registers are controlled according to the settings | |
29 | * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR | |
62421b45 | 30 | * registers can be done in either multicast (a single write updates all |
dd08ebf6 | 31 | * instances of the register to the same value) or unicast (a write updates only |
62421b45 LDM |
32 | * one specific instance) form. Reads of MCR registers always operate in a |
33 | * unicast manner regardless of how the multicast/unicast bit is set in | |
34 | * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is | |
35 | * referred to as "steering." | |
dd08ebf6 MB |
36 | * |
37 | * If MCR register operations are steered toward a hardware unit that is | |
38 | * fused off or currently powered down due to power gating, the MMIO operation | |
39 | * is "terminated" by the hardware. Terminated read operations will return a | |
40 | * value of zero and terminated unicast write operations will be silently | |
6b980aa8 LDM |
41 | * ignored. During device initialization, the goal of the various |
42 | * ``init_steering_*()`` functions is to apply the platform-specific rules for | |
43 | * each MCR register type to identify a steering target that will select a | |
44 | * non-terminated instance. | |
be46d7aa MW |
45 | * |
46 | * MCR registers are not available on Virtual Function (VF). | |
dd08ebf6 MB |
47 | */ |
48 | ||
ce8bf5bd LDM |
49 | #define STEER_SEMAPHORE XE_REG(0xFD0) |
50 | ||
36e22be4 LDM |
51 | static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr) |
52 | { | |
53 | return reg_mcr.__reg; | |
54 | } | |
55 | ||
dd08ebf6 MB |
56 | enum { |
57 | MCR_OP_READ, | |
58 | MCR_OP_WRITE | |
59 | }; | |
60 | ||
61 | static const struct xe_mmio_range xelp_l3bank_steering_table[] = { | |
62 | { 0x00B100, 0x00B3FF }, | |
63 | {}, | |
64 | }; | |
65 | ||
e84535d8 LDM |
66 | static const struct xe_mmio_range xehp_l3bank_steering_table[] = { |
67 | { 0x008C80, 0x008CFF }, | |
68 | { 0x00B100, 0x00B3FF }, | |
69 | {}, | |
70 | }; | |
71 | ||
dd08ebf6 MB |
72 | /* |
73 | * Although the bspec lists more "MSLICE" ranges than shown here, some of those | |
74 | * are of a "GAM" subclass that has special rules and doesn't need to be | |
75 | * included here. | |
76 | */ | |
77 | static const struct xe_mmio_range xehp_mslice_steering_table[] = { | |
78 | { 0x00DD00, 0x00DDFF }, | |
79 | { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ | |
80 | {}, | |
81 | }; | |
82 | ||
83 | static const struct xe_mmio_range xehp_lncf_steering_table[] = { | |
84 | { 0x00B000, 0x00B0FF }, | |
85 | { 0x00D880, 0x00D8FF }, | |
86 | {}, | |
87 | }; | |
88 | ||
89 | /* | |
90 | * We have several types of MCR registers where steering to (0,0) will always | |
91 | * provide us with a non-terminated value. We'll stick them all in the same | |
92 | * table for simplicity. | |
93 | */ | |
94 | static const struct xe_mmio_range xehpc_instance0_steering_table[] = { | |
95 | { 0x004000, 0x004AFF }, /* HALF-BSLICE */ | |
96 | { 0x008800, 0x00887F }, /* CC */ | |
97 | { 0x008A80, 0x008AFF }, /* TILEPSMI */ | |
98 | { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ | |
99 | { 0x00B100, 0x00B3FF }, /* L3BANK */ | |
100 | { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ | |
101 | { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ | |
102 | { 0x00DD00, 0x00DDFF }, /* BSLICE */ | |
103 | { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ | |
104 | { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ | |
105 | { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ | |
106 | { 0x024180, 0x0241FF }, /* HALF-BSLICE */ | |
107 | {}, | |
108 | }; | |
109 | ||
110 | static const struct xe_mmio_range xelpg_instance0_steering_table[] = { | |
111 | { 0x000B00, 0x000BFF }, /* SQIDI */ | |
112 | { 0x001000, 0x001FFF }, /* SQIDI */ | |
113 | { 0x004000, 0x0048FF }, /* GAM */ | |
114 | { 0x008700, 0x0087FF }, /* SQIDI */ | |
115 | { 0x00B000, 0x00B0FF }, /* NODE */ | |
116 | { 0x00C800, 0x00CFFF }, /* GAM */ | |
117 | { 0x00D880, 0x00D8FF }, /* NODE */ | |
118 | { 0x00DD00, 0x00DDFF }, /* OAAL2 */ | |
119 | {}, | |
120 | }; | |
121 | ||
122 | static const struct xe_mmio_range xelpg_l3bank_steering_table[] = { | |
123 | { 0x00B100, 0x00B3FF }, | |
124 | {}, | |
125 | }; | |
126 | ||
127 | static const struct xe_mmio_range xelp_dss_steering_table[] = { | |
128 | { 0x008150, 0x00815F }, | |
129 | { 0x009520, 0x00955F }, | |
130 | { 0x00DE80, 0x00E8FF }, | |
131 | { 0x024A00, 0x024A7F }, | |
132 | {}, | |
133 | }; | |
134 | ||
135 | /* DSS steering is used for GSLICE ranges as well */ | |
136 | static const struct xe_mmio_range xehp_dss_steering_table[] = { | |
137 | { 0x005200, 0x0052FF }, /* GSLICE */ | |
138 | { 0x005400, 0x007FFF }, /* GSLICE */ | |
139 | { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ | |
140 | { 0x008D00, 0x008DFF }, /* DSS */ | |
141 | { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ | |
142 | { 0x009680, 0x0096FF }, /* DSS */ | |
143 | { 0x00D800, 0x00D87F }, /* GSLICE */ | |
144 | { 0x00DC00, 0x00DCFF }, /* GSLICE */ | |
145 | { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */ | |
146 | { 0x017000, 0x017FFF }, /* GSLICE */ | |
147 | { 0x024A00, 0x024A7F }, /* DSS */ | |
148 | {}, | |
149 | }; | |
150 | ||
151 | /* DSS steering is used for COMPUTE ranges as well */ | |
152 | static const struct xe_mmio_range xehpc_dss_steering_table[] = { | |
153 | { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */ | |
154 | { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */ | |
155 | { 0x009680, 0x0096FF }, /* DSS */ | |
156 | { 0x00DC00, 0x00DCFF }, /* COMPUTE */ | |
157 | { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */ | |
158 | {}, | |
159 | }; | |
160 | ||
161 | /* DSS steering is used for SLICE ranges as well */ | |
162 | static const struct xe_mmio_range xelpg_dss_steering_table[] = { | |
163 | { 0x005200, 0x0052FF }, /* SLICE */ | |
164 | { 0x005500, 0x007FFF }, /* SLICE */ | |
165 | { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ | |
166 | { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ | |
167 | { 0x009680, 0x0096FF }, /* DSS */ | |
168 | { 0x00D800, 0x00D87F }, /* SLICE */ | |
169 | { 0x00DC00, 0x00DCFF }, /* SLICE */ | |
170 | { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ | |
171 | {}, | |
172 | }; | |
173 | ||
174 | static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = { | |
175 | { 0x393200, 0x39323F }, | |
176 | { 0x393400, 0x3934FF }, | |
177 | {}, | |
178 | }; | |
179 | ||
564d64f8 LDM |
180 | static const struct xe_mmio_range dg2_implicit_steering_table[] = { |
181 | { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */ | |
182 | { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */ | |
183 | { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */ | |
184 | { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */ | |
185 | { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */ | |
186 | { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */ | |
dd08ebf6 MB |
187 | {}, |
188 | }; | |
189 | ||
5c82000f MR |
190 | static const struct xe_mmio_range xe2lpg_dss_steering_table[] = { |
191 | { 0x005200, 0x0052FF }, /* SLICE */ | |
192 | { 0x005500, 0x007FFF }, /* SLICE */ | |
193 | { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ | |
194 | { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ | |
195 | { 0x009680, 0x0096FF }, /* DSS */ | |
196 | { 0x00D800, 0x00D87F }, /* SLICE */ | |
197 | { 0x00DC00, 0x00DCFF }, /* SLICE */ | |
198 | { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ | |
199 | { 0x00E980, 0x00E9FF }, /* SLICE */ | |
200 | { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */ | |
201 | {}, | |
202 | }; | |
203 | ||
204 | static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = { | |
205 | { 0x000B00, 0x000BFF }, | |
206 | { 0x001000, 0x001FFF }, | |
207 | {}, | |
208 | }; | |
209 | ||
210 | static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = { | |
211 | { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */ | |
212 | { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */ | |
213 | { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */ | |
214 | { 0x00C800, 0x00CFFF }, /* GAM */ | |
215 | { 0x00D880, 0x00D8FF }, /* NODE */ | |
216 | { 0x00DD00, 0x00DDFF }, /* MEMPIPE */ | |
217 | { 0x00E900, 0x00E97F }, /* MEMPIPE */ | |
218 | { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */ | |
219 | { 0x013400, 0x0135FF }, /* MEMPIPE */ | |
220 | {}, | |
221 | }; | |
222 | ||
8e99b545 MR |
223 | static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = { |
224 | { 0x388160, 0x38817F }, | |
225 | { 0x389480, 0x3894CF }, | |
226 | {}, | |
227 | }; | |
228 | ||
229 | static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = { | |
230 | { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */ | |
231 | { 0x384900, 0x384AFF }, /* GAM */ | |
232 | { 0x389560, 0x3895FF }, /* MEDIAINF */ | |
233 | { 0x38B600, 0x38B8FF }, /* L3BANK */ | |
234 | { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */ | |
235 | { 0x38F000, 0x38F0FF }, /* GAM */ | |
236 | { 0x393C00, 0x393C7F }, /* MEDIAINF */ | |
237 | {}, | |
238 | }; | |
239 | ||
dd08ebf6 MB |
240 | static void init_steering_l3bank(struct xe_gt *gt) |
241 | { | |
242 | if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { | |
d9b79ad2 | 243 | u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, |
ce8bf5bd | 244 | xe_mmio_read32(gt, MIRROR_FUSE3)); |
dd08ebf6 | 245 | u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK, |
ce8bf5bd | 246 | xe_mmio_read32(gt, XEHP_FUSE4)); |
dd08ebf6 MB |
247 | |
248 | /* | |
249 | * Group selects mslice, instance selects bank within mslice. | |
250 | * Bank 0 is always valid _except_ when the bank mask is 010b. | |
251 | */ | |
252 | gt->steering[L3BANK].group_target = __ffs(mslice_mask); | |
253 | gt->steering[L3BANK].instance_target = | |
254 | bank_mask & BIT(0) ? 0 : 2; | |
e84535d8 | 255 | } else if (gt_to_xe(gt)->info.platform == XE_DG2) { |
d9b79ad2 | 256 | u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK, |
ce8bf5bd | 257 | xe_mmio_read32(gt, MIRROR_FUSE3)); |
e84535d8 LDM |
258 | u32 bank = __ffs(mslice_mask) * 8; |
259 | ||
260 | /* | |
261 | * Like mslice registers, look for a valid mslice and steer to | |
262 | * the first L3BANK of that quad. Access to the Nth L3 bank is | |
263 | * split between the first bits of group and instance | |
264 | */ | |
265 | gt->steering[L3BANK].group_target = (bank >> 2) & 0x7; | |
266 | gt->steering[L3BANK].instance_target = bank & 0x3; | |
dd08ebf6 | 267 | } else { |
d9b79ad2 | 268 | u32 fuse = REG_FIELD_GET(L3BANK_MASK, |
ce8bf5bd | 269 | ~xe_mmio_read32(gt, MIRROR_FUSE3)); |
dd08ebf6 MB |
270 | |
271 | gt->steering[L3BANK].group_target = 0; /* unused */ | |
272 | gt->steering[L3BANK].instance_target = __ffs(fuse); | |
273 | } | |
274 | } | |
275 | ||
276 | static void init_steering_mslice(struct xe_gt *gt) | |
277 | { | |
d9b79ad2 | 278 | u32 mask = REG_FIELD_GET(MEML3_EN_MASK, |
ce8bf5bd | 279 | xe_mmio_read32(gt, MIRROR_FUSE3)); |
dd08ebf6 MB |
280 | |
281 | /* | |
282 | * mslice registers are valid (not terminated) if either the meml3 | |
283 | * associated with the mslice is present, or at least one DSS associated | |
284 | * with the mslice is present. There will always be at least one meml3 | |
285 | * so we can just use that to find a non-terminated mslice and ignore | |
286 | * the DSS fusing. | |
287 | */ | |
288 | gt->steering[MSLICE].group_target = __ffs(mask); | |
289 | gt->steering[MSLICE].instance_target = 0; /* unused */ | |
290 | ||
291 | /* | |
292 | * LNCF termination is also based on mslice presence, so we'll set | |
293 | * it up here. Either LNCF within a non-terminated mslice will work, | |
294 | * so we just always pick LNCF 0 here. | |
295 | */ | |
296 | gt->steering[LNCF].group_target = __ffs(mask) << 1; | |
297 | gt->steering[LNCF].instance_target = 0; /* unused */ | |
298 | } | |
299 | ||
bde5d767 ZD |
300 | static unsigned int dss_per_group(struct xe_gt *gt) |
301 | { | |
fc7c7498 MR |
302 | struct xe_guc *guc = >->uc.guc; |
303 | u32 max_slices = 0, max_subslices = 0; | |
304 | int ret; | |
305 | ||
306 | /* | |
307 | * Try to query the GuC's hwconfig table for the maximum number of | |
308 | * slices and subslices. These don't reflect the platform's actual | |
309 | * slice/DSS counts, just the physical layout by which we should | |
310 | * determine the steering targets. On older platforms with older GuC | |
311 | * firmware releases it's possible that these attributes may not be | |
312 | * included in the table, so we can always fall back to the old | |
313 | * hardcoded layouts. | |
314 | */ | |
315 | #define HWCONFIG_ATTR_MAX_SLICES 1 | |
316 | #define HWCONFIG_ATTR_MAX_SUBSLICES 70 | |
317 | ||
318 | ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SLICES, | |
319 | &max_slices); | |
320 | if (ret < 0 || max_slices == 0) | |
321 | goto fallback; | |
322 | ||
323 | ret = xe_guc_hwconfig_lookup_u32(guc, HWCONFIG_ATTR_MAX_SUBSLICES, | |
324 | &max_subslices); | |
325 | if (ret < 0 || max_subslices == 0) | |
326 | goto fallback; | |
327 | ||
328 | return DIV_ROUND_UP(max_subslices, max_slices); | |
329 | ||
330 | fallback: | |
331 | xe_gt_dbg(gt, "GuC hwconfig cannot provide dss/slice; using typical fallback values\n"); | |
788d2ad6 AH |
332 | if (gt_to_xe(gt)->info.platform == XE_PVC) |
333 | return 8; | |
334 | else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1250) | |
335 | return 4; | |
336 | else | |
337 | return 6; | |
bde5d767 ZD |
338 | } |
339 | ||
340 | /** | |
341 | * xe_gt_mcr_get_dss_steering - Get the group/instance steering for a DSS | |
342 | * @gt: GT structure | |
343 | * @dss: DSS ID to obtain steering for | |
344 | * @group: pointer to storage for steering group ID | |
345 | * @instance: pointer to storage for steering instance ID | |
346 | */ | |
347 | void xe_gt_mcr_get_dss_steering(struct xe_gt *gt, unsigned int dss, u16 *group, u16 *instance) | |
dd08ebf6 | 348 | { |
bde5d767 ZD |
349 | xe_gt_assert(gt, dss < XE_MAX_DSS_FUSE_BITS); |
350 | ||
fc7c7498 MR |
351 | *group = dss / gt->steering_dss_per_grp; |
352 | *instance = dss % gt->steering_dss_per_grp; | |
bde5d767 ZD |
353 | } |
354 | ||
355 | static void init_steering_dss(struct xe_gt *gt) | |
356 | { | |
fc7c7498 MR |
357 | gt->steering_dss_per_grp = dss_per_group(gt); |
358 | ||
bde5d767 ZD |
359 | xe_gt_mcr_get_dss_steering(gt, |
360 | min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0), | |
361 | xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)), | |
362 | >->steering[DSS].group_target, | |
363 | >->steering[DSS].instance_target); | |
dd08ebf6 MB |
364 | } |
365 | ||
366 | static void init_steering_oaddrm(struct xe_gt *gt) | |
367 | { | |
368 | /* | |
369 | * First instance is only terminated if the entire first media slice | |
370 | * is absent (i.e., no VCS0 or VECS0). | |
371 | */ | |
372 | if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0)) | |
373 | gt->steering[OADDRM].group_target = 0; | |
374 | else | |
375 | gt->steering[OADDRM].group_target = 1; | |
376 | ||
4f82ac61 | 377 | gt->steering[OADDRM].instance_target = 0; /* unused */ |
dd08ebf6 MB |
378 | } |
379 | ||
5c82000f MR |
380 | static void init_steering_sqidi_psmi(struct xe_gt *gt) |
381 | { | |
382 | u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK, | |
383 | xe_mmio_read32(gt, MIRROR_FUSE3)); | |
384 | u32 select = __ffs(mask); | |
385 | ||
386 | gt->steering[SQIDI_PSMI].group_target = select >> 1; | |
387 | gt->steering[SQIDI_PSMI].instance_target = select & 0x1; | |
388 | } | |
389 | ||
dd08ebf6 MB |
390 | static void init_steering_inst0(struct xe_gt *gt) |
391 | { | |
4f82ac61 MR |
392 | gt->steering[INSTANCE0].group_target = 0; /* unused */ |
393 | gt->steering[INSTANCE0].instance_target = 0; /* unused */ | |
dd08ebf6 MB |
394 | } |
395 | ||
396 | static const struct { | |
397 | const char *name; | |
72e8d73b | 398 | void (*init)(struct xe_gt *gt); |
dd08ebf6 | 399 | } xe_steering_types[] = { |
3319b213 LDM |
400 | [L3BANK] = { "L3BANK", init_steering_l3bank }, |
401 | [MSLICE] = { "MSLICE", init_steering_mslice }, | |
402 | [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */ | |
403 | [DSS] = { "DSS", init_steering_dss }, | |
8e99b545 | 404 | [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm }, |
5c82000f | 405 | [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi }, |
3319b213 | 406 | [INSTANCE0] = { "INSTANCE 0", init_steering_inst0 }, |
564d64f8 | 407 | [IMPLICIT_STEERING] = { "IMPLICIT", NULL }, |
dd08ebf6 MB |
408 | }; |
409 | ||
2d3789e3 MW |
410 | /** |
411 | * xe_gt_mcr_init_early - Early initialization of the MCR support | |
412 | * @gt: GT structure | |
413 | * | |
414 | * Perform early software only initialization of the MCR lock to allow | |
415 | * the synchronization on accessing the STEER_SEMAPHORE register and | |
416 | * use the xe_gt_mcr_multicast_write() function. | |
417 | */ | |
418 | void xe_gt_mcr_init_early(struct xe_gt *gt) | |
dd08ebf6 | 419 | { |
564d64f8 | 420 | BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES); |
dd08ebf6 MB |
421 | BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES); |
422 | ||
2d3789e3 MW |
423 | spin_lock_init(>->mcr_lock); |
424 | } | |
425 | ||
426 | /** | |
427 | * xe_gt_mcr_init - Normal initialization of the MCR support | |
428 | * @gt: GT structure | |
429 | * | |
430 | * Perform normal initialization of the MCR for all usages. | |
431 | */ | |
432 | void xe_gt_mcr_init(struct xe_gt *gt) | |
433 | { | |
434 | struct xe_device *xe = gt_to_xe(gt); | |
435 | ||
be46d7aa MW |
436 | if (IS_SRIOV_VF(xe)) |
437 | return; | |
438 | ||
dd08ebf6 MB |
439 | if (gt->info.type == XE_GT_TYPE_MEDIA) { |
440 | drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13); | |
441 | ||
7929ffce | 442 | if (MEDIA_VERx100(xe) >= 1301) { |
8e99b545 MR |
443 | gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table; |
444 | gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table; | |
445 | } else { | |
446 | gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table; | |
447 | } | |
dd08ebf6 | 448 | } else { |
5c82000f MR |
449 | if (GRAPHICS_VER(xe) >= 20) { |
450 | gt->steering[DSS].ranges = xe2lpg_dss_steering_table; | |
451 | gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table; | |
452 | gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table; | |
453 | } else if (GRAPHICS_VERx100(xe) >= 1270) { | |
436dbd6b LDM |
454 | gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; |
455 | gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; | |
456 | gt->steering[DSS].ranges = xelpg_dss_steering_table; | |
457 | } else if (xe->info.platform == XE_PVC) { | |
458 | gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table; | |
459 | gt->steering[DSS].ranges = xehpc_dss_steering_table; | |
460 | } else if (xe->info.platform == XE_DG2) { | |
461 | gt->steering[L3BANK].ranges = xehp_l3bank_steering_table; | |
462 | gt->steering[MSLICE].ranges = xehp_mslice_steering_table; | |
463 | gt->steering[LNCF].ranges = xehp_lncf_steering_table; | |
464 | gt->steering[DSS].ranges = xehp_dss_steering_table; | |
465 | gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table; | |
466 | } else { | |
467 | gt->steering[L3BANK].ranges = xelp_l3bank_steering_table; | |
468 | gt->steering[DSS].ranges = xelp_dss_steering_table; | |
469 | } | |
dd08ebf6 MB |
470 | } |
471 | ||
472 | /* Select non-terminated steering target for each type */ | |
473 | for (int i = 0; i < NUM_STEERING_TYPES; i++) | |
474 | if (gt->steering[i].ranges && xe_steering_types[i].init) | |
475 | xe_steering_types[i].init(gt); | |
476 | } | |
477 | ||
564d64f8 LDM |
478 | /** |
479 | * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers | |
480 | * @gt: GT structure | |
481 | * | |
482 | * Some register ranges don't need to have their steering control registers | |
483 | * changed on each access - it's sufficient to set them once on initialization. | |
484 | * This function sets those registers for each platform * | |
485 | */ | |
486 | void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt) | |
487 | { | |
488 | struct xe_device *xe = gt_to_xe(gt); | |
489 | ||
be46d7aa MW |
490 | if (IS_SRIOV_VF(xe)) |
491 | return; | |
492 | ||
564d64f8 | 493 | if (xe->info.platform == XE_DG2) { |
d9b79ad2 LDM |
494 | u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) | |
495 | REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2); | |
564d64f8 | 496 | |
ce8bf5bd LDM |
497 | xe_mmio_write32(gt, MCFG_MCR_SELECTOR, steer_val); |
498 | xe_mmio_write32(gt, SF_MCR_SELECTOR, steer_val); | |
564d64f8 LDM |
499 | /* |
500 | * For GAM registers, all reads should be directed to instance 1 | |
501 | * (unicast reads against other instances are not allowed), | |
502 | * and instance 1 is already the hardware's default steering | |
503 | * target, which we never change | |
504 | */ | |
505 | } | |
506 | } | |
507 | ||
dd08ebf6 MB |
508 | /* |
509 | * xe_gt_mcr_get_nonterminated_steering - find group/instance values that | |
510 | * will steer a register to a non-terminated instance | |
511 | * @gt: GT structure | |
512 | * @reg: register for which the steering is required | |
513 | * @group: return variable for group steering | |
514 | * @instance: return variable for instance steering | |
515 | * | |
516 | * This function returns a group/instance pair that is guaranteed to work for | |
517 | * read steering of the given register. Note that a value will be returned even | |
518 | * if the register is not replicated and therefore does not actually require | |
519 | * steering. | |
520 | * | |
521 | * Returns true if the caller should steer to the @group/@instance values | |
564d64f8 | 522 | * returned. Returns false if the caller need not perform any steering |
dd08ebf6 MB |
523 | */ |
524 | static bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, | |
36e22be4 | 525 | struct xe_reg_mcr reg_mcr, |
dd08ebf6 MB |
526 | u8 *group, u8 *instance) |
527 | { | |
36e22be4 | 528 | const struct xe_reg reg = to_xe_reg(reg_mcr); |
564d64f8 LDM |
529 | const struct xe_mmio_range *implicit_ranges; |
530 | ||
531 | for (int type = 0; type < IMPLICIT_STEERING; type++) { | |
dd08ebf6 MB |
532 | if (!gt->steering[type].ranges) |
533 | continue; | |
534 | ||
535 | for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) { | |
9fa81f91 | 536 | if (xe_mmio_in_range(gt, >->steering[type].ranges[i], reg)) { |
dd08ebf6 MB |
537 | *group = gt->steering[type].group_target; |
538 | *instance = gt->steering[type].instance_target; | |
539 | return true; | |
540 | } | |
541 | } | |
542 | } | |
543 | ||
564d64f8 LDM |
544 | implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges; |
545 | if (implicit_ranges) | |
546 | for (int i = 0; implicit_ranges[i].end > 0; i++) | |
9fa81f91 | 547 | if (xe_mmio_in_range(gt, &implicit_ranges[i], reg)) |
564d64f8 | 548 | return false; |
dd08ebf6 MB |
549 | |
550 | /* | |
564d64f8 LDM |
551 | * Not found in a steering table and not a register with implicit |
552 | * steering. Just steer to 0/0 as a guess and raise a warning. | |
dd08ebf6 MB |
553 | */ |
554 | drm_WARN(>_to_xe(gt)->drm, true, | |
555 | "Did not find MCR register %#x in any MCR steering table\n", | |
ee21379a | 556 | reg.addr); |
dd08ebf6 MB |
557 | *group = 0; |
558 | *instance = 0; | |
559 | ||
560 | return true; | |
561 | } | |
562 | ||
dd08ebf6 MB |
563 | /* |
564 | * Obtain exclusive access to MCR steering. On MTL and beyond we also need | |
565 | * to synchronize with external clients (e.g., firmware), so a semaphore | |
566 | * register will also need to be taken. | |
567 | */ | |
97fd7a7e | 568 | static void mcr_lock(struct xe_gt *gt) __acquires(>->mcr_lock) |
dd08ebf6 MB |
569 | { |
570 | struct xe_device *xe = gt_to_xe(gt); | |
dc97898e | 571 | int ret = 0; |
dd08ebf6 MB |
572 | |
573 | spin_lock(>->mcr_lock); | |
574 | ||
575 | /* | |
576 | * Starting with MTL we also need to grab a semaphore register | |
577 | * to synchronize with external agents (e.g., firmware) that now | |
43b5d81e DCS |
578 | * shares the same steering control register. The semaphore is obtained |
579 | * when a read to the relevant register returns 1. | |
dd08ebf6 MB |
580 | */ |
581 | if (GRAPHICS_VERx100(xe) >= 1270) | |
43b5d81e | 582 | ret = xe_mmio_wait32(gt, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL, |
a56d8dab | 583 | true); |
dd08ebf6 MB |
584 | |
585 | drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT); | |
586 | } | |
587 | ||
97fd7a7e | 588 | static void mcr_unlock(struct xe_gt *gt) __releases(>->mcr_lock) |
fb1d55ef | 589 | { |
43b5d81e | 590 | /* Release hardware semaphore - this is done by writing 1 to the register */ |
dd08ebf6 MB |
591 | if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) |
592 | xe_mmio_write32(gt, STEER_SEMAPHORE, 0x1); | |
593 | ||
594 | spin_unlock(>->mcr_lock); | |
595 | } | |
596 | ||
597 | /* | |
598 | * Access a register with specific MCR steering | |
599 | * | |
600 | * Caller needs to make sure the relevant forcewake wells are up. | |
601 | */ | |
36e22be4 LDM |
602 | static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, |
603 | u8 rw_flag, int group, int instance, u32 value) | |
dd08ebf6 | 604 | { |
36e22be4 | 605 | const struct xe_reg reg = to_xe_reg(reg_mcr); |
ce8bf5bd LDM |
606 | struct xe_reg steer_reg; |
607 | u32 steer_val, val = 0; | |
dd08ebf6 MB |
608 | |
609 | lockdep_assert_held(>->mcr_lock); | |
610 | ||
611 | if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { | |
ce8bf5bd | 612 | steer_reg = MTL_MCR_SELECTOR; |
dd08ebf6 MB |
613 | steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) | |
614 | REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance); | |
615 | } else { | |
ce8bf5bd | 616 | steer_reg = MCR_SELECTOR; |
d9b79ad2 LDM |
617 | steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) | |
618 | REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance); | |
dd08ebf6 MB |
619 | } |
620 | ||
621 | /* | |
215bb2ce LDM |
622 | * Always leave the hardware in multicast mode when doing reads and only |
623 | * change it to unicast mode when doing writes of a specific instance. | |
624 | * | |
625 | * The setting of the multicast/unicast bit usually wouldn't matter for | |
626 | * read operations (which always return the value from a single register | |
627 | * instance regardless of how that bit is set), but some platforms may | |
628 | * have workarounds requiring us to remain in multicast mode for reads, | |
629 | * e.g. Wa_22013088509 on PVC. There's no real downside to this, so | |
630 | * we'll just go ahead and do so on all platforms; we'll only clear the | |
631 | * multicast bit from the mask when explicitly doing a write operation. | |
dd08ebf6 MB |
632 | * |
633 | * No need to save old steering reg value. | |
634 | */ | |
635 | if (rw_flag == MCR_OP_READ) | |
d9b79ad2 | 636 | steer_val |= MCR_MULTICAST; |
dd08ebf6 MB |
637 | |
638 | xe_mmio_write32(gt, steer_reg, steer_val); | |
639 | ||
640 | if (rw_flag == MCR_OP_READ) | |
ce8bf5bd | 641 | val = xe_mmio_read32(gt, reg); |
dd08ebf6 | 642 | else |
ce8bf5bd | 643 | xe_mmio_write32(gt, reg, value); |
dd08ebf6 MB |
644 | |
645 | /* | |
646 | * If we turned off the multicast bit (during a write) we're required | |
647 | * to turn it back on before finishing. The group and instance values | |
648 | * don't matter since they'll be re-programmed on the next MCR | |
649 | * operation. | |
650 | */ | |
651 | if (rw_flag == MCR_OP_WRITE) | |
d9b79ad2 | 652 | xe_mmio_write32(gt, steer_reg, MCR_MULTICAST); |
dd08ebf6 MB |
653 | |
654 | return val; | |
655 | } | |
656 | ||
657 | /** | |
658 | * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register | |
659 | * @gt: GT structure | |
36e22be4 | 660 | * @reg_mcr: register to read |
dd08ebf6 MB |
661 | * |
662 | * Reads a GT MCR register. The read will be steered to a non-terminated | |
663 | * instance (i.e., one that isn't fused off or powered down by power gating). | |
664 | * This function assumes the caller is already holding any necessary forcewake | |
665 | * domains. | |
666 | * | |
667 | * Returns the value from a non-terminated instance of @reg. | |
668 | */ | |
36e22be4 | 669 | u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr) |
dd08ebf6 | 670 | { |
36e22be4 | 671 | const struct xe_reg reg = to_xe_reg(reg_mcr); |
dd08ebf6 MB |
672 | u8 group, instance; |
673 | u32 val; | |
674 | bool steer; | |
675 | ||
be46d7aa MW |
676 | xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); |
677 | ||
36e22be4 LDM |
678 | steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr, |
679 | &group, &instance); | |
dd08ebf6 MB |
680 | |
681 | if (steer) { | |
682 | mcr_lock(gt); | |
36e22be4 | 683 | val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, |
dd08ebf6 MB |
684 | group, instance, 0); |
685 | mcr_unlock(gt); | |
686 | } else { | |
ce8bf5bd | 687 | val = xe_mmio_read32(gt, reg); |
dd08ebf6 MB |
688 | } |
689 | ||
690 | return val; | |
691 | } | |
692 | ||
693 | /** | |
694 | * xe_gt_mcr_unicast_read - read a specific instance of an MCR register | |
695 | * @gt: GT structure | |
36e22be4 | 696 | * @reg_mcr: the MCR register to read |
dd08ebf6 MB |
697 | * @group: the MCR group |
698 | * @instance: the MCR instance | |
699 | * | |
700 | * Returns the value read from an MCR register after steering toward a specific | |
701 | * group/instance. | |
702 | */ | |
703 | u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, | |
36e22be4 | 704 | struct xe_reg_mcr reg_mcr, |
dd08ebf6 MB |
705 | int group, int instance) |
706 | { | |
707 | u32 val; | |
708 | ||
be46d7aa MW |
709 | xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); |
710 | ||
dd08ebf6 | 711 | mcr_lock(gt); |
36e22be4 | 712 | val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0); |
dd08ebf6 MB |
713 | mcr_unlock(gt); |
714 | ||
715 | return val; | |
716 | } | |
717 | ||
718 | /** | |
719 | * xe_gt_mcr_unicast_write - write a specific instance of an MCR register | |
720 | * @gt: GT structure | |
36e22be4 | 721 | * @reg_mcr: the MCR register to write |
dd08ebf6 MB |
722 | * @value: value to write |
723 | * @group: the MCR group | |
724 | * @instance: the MCR instance | |
725 | * | |
726 | * Write an MCR register in unicast mode after steering toward a specific | |
727 | * group/instance. | |
728 | */ | |
36e22be4 LDM |
729 | void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, |
730 | u32 value, int group, int instance) | |
dd08ebf6 | 731 | { |
be46d7aa MW |
732 | xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); |
733 | ||
dd08ebf6 | 734 | mcr_lock(gt); |
36e22be4 | 735 | rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value); |
dd08ebf6 MB |
736 | mcr_unlock(gt); |
737 | } | |
738 | ||
739 | /** | |
740 | * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register | |
741 | * @gt: GT structure | |
36e22be4 | 742 | * @reg_mcr: the MCR register to write |
dd08ebf6 MB |
743 | * @value: value to write |
744 | * | |
745 | * Write an MCR register in multicast mode to update all instances. | |
746 | */ | |
36e22be4 LDM |
747 | void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr, |
748 | u32 value) | |
dd08ebf6 | 749 | { |
36e22be4 LDM |
750 | struct xe_reg reg = to_xe_reg(reg_mcr); |
751 | ||
be46d7aa MW |
752 | xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt))); |
753 | ||
dd08ebf6 MB |
754 | /* |
755 | * Synchronize with any unicast operations. Once we have exclusive | |
756 | * access, the MULTICAST bit should already be set, so there's no need | |
757 | * to touch the steering register. | |
758 | */ | |
759 | mcr_lock(gt); | |
ce8bf5bd | 760 | xe_mmio_write32(gt, reg, value); |
dd08ebf6 MB |
761 | mcr_unlock(gt); |
762 | } | |
763 | ||
764 | void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p) | |
765 | { | |
766 | for (int i = 0; i < NUM_STEERING_TYPES; i++) { | |
767 | if (gt->steering[i].ranges) { | |
768 | drm_printf(p, "%s steering: group=%#x, instance=%#x\n", | |
769 | xe_steering_types[i].name, | |
770 | gt->steering[i].group_target, | |
771 | gt->steering[i].instance_target); | |
772 | for (int j = 0; gt->steering[i].ranges[j].end; j++) | |
773 | drm_printf(p, "\t0x%06x - 0x%06x\n", | |
774 | gt->steering[i].ranges[j].start, | |
775 | gt->steering[i].ranges[j].end); | |
776 | } | |
777 | } | |
778 | } |