projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'rtc-6.9' of git://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux
[linux-block.git] / drivers / gpu / drm / xe / xe_gt_mcr.c
CommitLineData
dd08ebf6
MB		 1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
dd08ebf6 6#include "xe_gt_mcr.h"
ea9f879d 7
226bfec8 8#include "regs/xe_gt_regs.h"
ea9f879d 9#include "xe_gt.h"
dd08ebf6
MB		 10#include "xe_gt_topology.h"
11#include "xe_gt_types.h"
12#include "xe_mmio.h"
be46d7aa 13#include "xe_sriov.h"
dd08ebf6 14
dd08ebf6
MB		 15/**
16 * DOC: GT Multicast/Replicated (MCR) Register Support
17 *
18 * Some GT registers are designed as "multicast" or "replicated" registers:
19 * multiple instances of the same register share a single MMIO offset. MCR
20 * registers are generally used when the hardware needs to potentially track
21 * independent values of a register per hardware unit (e.g., per-subslice,
22 * per-L3bank, etc.). The specific types of replication that exist vary
23 * per-platform.
24 *
25 * MMIO accesses to MCR registers are controlled according to the settings
26 * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR
62421b45 27 * registers can be done in either multicast (a single write updates all
dd08ebf6 28 * instances of the register to the same value) or unicast (a write updates only
62421b45
LDM		 29 * one specific instance) form. Reads of MCR registers always operate in a
30 * unicast manner regardless of how the multicast/unicast bit is set in
31 * MCR_SELECTOR. Selection of a specific MCR instance for unicast operations is
32 * referred to as "steering."
dd08ebf6
MB		 33 *
34 * If MCR register operations are steered toward a hardware unit that is
35 * fused off or currently powered down due to power gating, the MMIO operation
36 * is "terminated" by the hardware. Terminated read operations will return a
37 * value of zero and terminated unicast write operations will be silently
6b980aa8
LDM		 38 * ignored. During device initialization, the goal of the various
39 * ``init_steering_*()`` functions is to apply the platform-specific rules for
40 * each MCR register type to identify a steering target that will select a
41 * non-terminated instance.
be46d7aa
MW		 42 *
43 * MCR registers are not available on Virtual Function (VF).
dd08ebf6
MB		 44 */
45
ce8bf5bd
LDM		 46#define STEER_SEMAPHORE XE_REG(0xFD0)
47
36e22be4
LDM		 48static inline struct xe_reg to_xe_reg(struct xe_reg_mcr reg_mcr)
49{
50 return reg_mcr.__reg;
51}
52
dd08ebf6
MB		 53enum {
54 MCR_OP_READ,
55 MCR_OP_WRITE
56};
57
58static const struct xe_mmio_range xelp_l3bank_steering_table[] = {
59 { 0x00B100, 0x00B3FF },
60 {},
61};
62
e84535d8
LDM		 63static const struct xe_mmio_range xehp_l3bank_steering_table[] = {
64 { 0x008C80, 0x008CFF },
65 { 0x00B100, 0x00B3FF },
66 {},
67};
68
dd08ebf6
MB		 69/*
70 * Although the bspec lists more "MSLICE" ranges than shown here, some of those
71 * are of a "GAM" subclass that has special rules and doesn't need to be
72 * included here.
73 */
74static const struct xe_mmio_range xehp_mslice_steering_table[] = {
75 { 0x00DD00, 0x00DDFF },
76 { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */
77 {},
78};
79
80static const struct xe_mmio_range xehp_lncf_steering_table[] = {
81 { 0x00B000, 0x00B0FF },
82 { 0x00D880, 0x00D8FF },
83 {},
84};
85
86/*
87 * We have several types of MCR registers where steering to (0,0) will always
88 * provide us with a non-terminated value. We'll stick them all in the same
89 * table for simplicity.
90 */
91static const struct xe_mmio_range xehpc_instance0_steering_table[] = {
92 { 0x004000, 0x004AFF }, /* HALF-BSLICE */
93 { 0x008800, 0x00887F }, /* CC */
94 { 0x008A80, 0x008AFF }, /* TILEPSMI */
95 { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */
96 { 0x00B100, 0x00B3FF }, /* L3BANK */
97 { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */
98 { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */
99 { 0x00DD00, 0x00DDFF }, /* BSLICE */
100 { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */
101 { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */
102 { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */
103 { 0x024180, 0x0241FF }, /* HALF-BSLICE */
104 {},
105};
106
107static const struct xe_mmio_range xelpg_instance0_steering_table[] = {
108 { 0x000B00, 0x000BFF }, /* SQIDI */
109 { 0x001000, 0x001FFF }, /* SQIDI */
110 { 0x004000, 0x0048FF }, /* GAM */
111 { 0x008700, 0x0087FF }, /* SQIDI */
112 { 0x00B000, 0x00B0FF }, /* NODE */
113 { 0x00C800, 0x00CFFF }, /* GAM */
114 { 0x00D880, 0x00D8FF }, /* NODE */
115 { 0x00DD00, 0x00DDFF }, /* OAAL2 */
116 {},
117};
118
119static const struct xe_mmio_range xelpg_l3bank_steering_table[] = {
120 { 0x00B100, 0x00B3FF },
121 {},
122};
123
124static const struct xe_mmio_range xelp_dss_steering_table[] = {
125 { 0x008150, 0x00815F },
126 { 0x009520, 0x00955F },
127 { 0x00DE80, 0x00E8FF },
128 { 0x024A00, 0x024A7F },
129 {},
130};
131
132/* DSS steering is used for GSLICE ranges as well */
133static const struct xe_mmio_range xehp_dss_steering_table[] = {
134 { 0x005200, 0x0052FF }, /* GSLICE */
135 { 0x005400, 0x007FFF }, /* GSLICE */
136 { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
137 { 0x008D00, 0x008DFF }, /* DSS */
138 { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
139 { 0x009680, 0x0096FF }, /* DSS */
140 { 0x00D800, 0x00D87F }, /* GSLICE */
141 { 0x00DC00, 0x00DCFF }, /* GSLICE */
142 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */
143 { 0x017000, 0x017FFF }, /* GSLICE */
144 { 0x024A00, 0x024A7F }, /* DSS */
145 {},
146};
147
148/* DSS steering is used for COMPUTE ranges as well */
149static const struct xe_mmio_range xehpc_dss_steering_table[] = {
150 { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */
151 { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */
152 { 0x009680, 0x0096FF }, /* DSS */
153 { 0x00DC00, 0x00DCFF }, /* COMPUTE */
154 { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */
155 {},
156};
157
158/* DSS steering is used for SLICE ranges as well */
159static const struct xe_mmio_range xelpg_dss_steering_table[] = {
160 { 0x005200, 0x0052FF }, /* SLICE */
161 { 0x005500, 0x007FFF }, /* SLICE */
162 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
163 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
164 { 0x009680, 0x0096FF }, /* DSS */
165 { 0x00D800, 0x00D87F }, /* SLICE */
166 { 0x00DC00, 0x00DCFF }, /* SLICE */
167 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
168 {},
169};
170
171static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = {
172 { 0x393200, 0x39323F },
173 { 0x393400, 0x3934FF },
174 {},
175};
176
564d64f8
LDM		 177static const struct xe_mmio_range dg2_implicit_steering_table[] = {
178 { 0x000B00, 0x000BFF }, /* SF (SQIDI replication) */
179 { 0x001000, 0x001FFF }, /* SF (SQIDI replication) */
180 { 0x004000, 0x004AFF }, /* GAM (MSLICE replication) */
181 { 0x008700, 0x0087FF }, /* MCFG (SQIDI replication) */
182 { 0x00C800, 0x00CFFF }, /* GAM (MSLICE replication) */
183 { 0x00F000, 0x00FFFF }, /* GAM (MSLICE replication) */
dd08ebf6
MB		 184 {},
185};
186
5c82000f
MR		 187static const struct xe_mmio_range xe2lpg_dss_steering_table[] = {
188 { 0x005200, 0x0052FF }, /* SLICE */
189 { 0x005500, 0x007FFF }, /* SLICE */
190 { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */
191 { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */
192 { 0x009680, 0x0096FF }, /* DSS */
193 { 0x00D800, 0x00D87F }, /* SLICE */
194 { 0x00DC00, 0x00DCFF }, /* SLICE */
195 { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */
196 { 0x00E980, 0x00E9FF }, /* SLICE */
197 { 0x013000, 0x0133FF }, /* DSS (0x13000-0x131FF), SLICE (0x13200-0x133FF) */
198 {},
199};
200
201static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = {
202 { 0x000B00, 0x000BFF },
203 { 0x001000, 0x001FFF },
204 {},
205};
206
207static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = {
208 { 0x004000, 0x004AFF }, /* GAM, rsvd, GAMWKR */
209 { 0x008700, 0x00887F }, /* SQIDI, MEMPIPE */
210 { 0x00B000, 0x00B3FF }, /* NODE, L3BANK */
211 { 0x00C800, 0x00CFFF }, /* GAM */
212 { 0x00D880, 0x00D8FF }, /* NODE */
213 { 0x00DD00, 0x00DDFF }, /* MEMPIPE */
214 { 0x00E900, 0x00E97F }, /* MEMPIPE */
215 { 0x00F000, 0x00FFFF }, /* GAM, GAMWKR */
216 { 0x013400, 0x0135FF }, /* MEMPIPE */
217 {},
218};
219
8e99b545
MR		 220static const struct xe_mmio_range xe2lpm_gpmxmt_steering_table[] = {
221 { 0x388160, 0x38817F },
222 { 0x389480, 0x3894CF },
223 {},
224};
225
226static const struct xe_mmio_range xe2lpm_instance0_steering_table[] = {
227 { 0x384000, 0x3847DF }, /* GAM, rsvd, GAM */
228 { 0x384900, 0x384AFF }, /* GAM */
229 { 0x389560, 0x3895FF }, /* MEDIAINF */
230 { 0x38B600, 0x38B8FF }, /* L3BANK */
231 { 0x38C800, 0x38D07F }, /* GAM, MEDIAINF */
232 { 0x38F000, 0x38F0FF }, /* GAM */
233 { 0x393C00, 0x393C7F }, /* MEDIAINF */
234 {},
235};
236
dd08ebf6
MB		 237static void init_steering_l3bank(struct xe_gt *gt)
238{
239 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
d9b79ad2 240 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
ce8bf5bd 241 xe_mmio_read32(gt, MIRROR_FUSE3));
dd08ebf6 242 u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK,
ce8bf5bd 243 xe_mmio_read32(gt, XEHP_FUSE4));
dd08ebf6
MB		 244
245 /*
246 * Group selects mslice, instance selects bank within mslice.
247 * Bank 0 is always valid _except_ when the bank mask is 010b.
248 */
249 gt->steering[L3BANK].group_target = __ffs(mslice_mask);
250 gt->steering[L3BANK].instance_target =
251 bank_mask & BIT(0) ? 0 : 2;
e84535d8 252 } else if (gt_to_xe(gt)->info.platform == XE_DG2) {
d9b79ad2 253 u32 mslice_mask = REG_FIELD_GET(MEML3_EN_MASK,
ce8bf5bd 254 xe_mmio_read32(gt, MIRROR_FUSE3));
e84535d8
LDM		 255 u32 bank = __ffs(mslice_mask) * 8;
256
257 /*
258 * Like mslice registers, look for a valid mslice and steer to
259 * the first L3BANK of that quad. Access to the Nth L3 bank is
260 * split between the first bits of group and instance
261 */
262 gt->steering[L3BANK].group_target = (bank >> 2) & 0x7;
263 gt->steering[L3BANK].instance_target = bank & 0x3;
dd08ebf6 264 } else {
d9b79ad2 265 u32 fuse = REG_FIELD_GET(L3BANK_MASK,
ce8bf5bd 266 ~xe_mmio_read32(gt, MIRROR_FUSE3));
dd08ebf6
MB		 267
268 gt->steering[L3BANK].group_target = 0; /* unused */
269 gt->steering[L3BANK].instance_target = __ffs(fuse);
270 }
271}
272
273static void init_steering_mslice(struct xe_gt *gt)
274{
d9b79ad2 275 u32 mask = REG_FIELD_GET(MEML3_EN_MASK,
ce8bf5bd 276 xe_mmio_read32(gt, MIRROR_FUSE3));
dd08ebf6
MB		 277
278 /*
279 * mslice registers are valid (not terminated) if either the meml3
280 * associated with the mslice is present, or at least one DSS associated
281 * with the mslice is present. There will always be at least one meml3
282 * so we can just use that to find a non-terminated mslice and ignore
283 * the DSS fusing.
284 */
285 gt->steering[MSLICE].group_target = __ffs(mask);
286 gt->steering[MSLICE].instance_target = 0; /* unused */
287
288 /*
289 * LNCF termination is also based on mslice presence, so we'll set
290 * it up here. Either LNCF within a non-terminated mslice will work,
291 * so we just always pick LNCF 0 here.
292 */
293 gt->steering[LNCF].group_target = __ffs(mask) << 1;
294 gt->steering[LNCF].instance_target = 0; /* unused */
295}
296
297static void init_steering_dss(struct xe_gt *gt)
298{
299 unsigned int dss = min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0),
300 xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0));
301 unsigned int dss_per_grp = gt_to_xe(gt)->info.platform == XE_PVC ? 8 : 4;
302
303 gt->steering[DSS].group_target = dss / dss_per_grp;
304 gt->steering[DSS].instance_target = dss % dss_per_grp;
305}
306
307static void init_steering_oaddrm(struct xe_gt *gt)
308{
309 /*
310 * First instance is only terminated if the entire first media slice
311 * is absent (i.e., no VCS0 or VECS0).
312 */
313 if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0))
314 gt->steering[OADDRM].group_target = 0;
315 else
316 gt->steering[OADDRM].group_target = 1;
317
318 gt->steering[DSS].instance_target = 0; /* unused */
319}
320
5c82000f
MR		 321static void init_steering_sqidi_psmi(struct xe_gt *gt)
322{
323 u32 mask = REG_FIELD_GET(XE2_NODE_ENABLE_MASK,
324 xe_mmio_read32(gt, MIRROR_FUSE3));
325 u32 select = __ffs(mask);
326
327 gt->steering[SQIDI_PSMI].group_target = select >> 1;
328 gt->steering[SQIDI_PSMI].instance_target = select & 0x1;
329}
330
dd08ebf6
MB		 331static void init_steering_inst0(struct xe_gt *gt)
332{
333 gt->steering[DSS].group_target = 0; /* unused */
334 gt->steering[DSS].instance_target = 0; /* unused */
335}
336
337static const struct {
338 const char *name;
72e8d73b 339 void (*init)(struct xe_gt *gt);
dd08ebf6 340} xe_steering_types[] = {
3319b213
LDM		 341 [L3BANK] = { "L3BANK", init_steering_l3bank },
342 [MSLICE] = { "MSLICE", init_steering_mslice },
343 [LNCF] = { "LNCF", NULL }, /* initialized by mslice init */
344 [DSS] = { "DSS", init_steering_dss },
8e99b545 345 [OADDRM] = { "OADDRM / GPMXMT", init_steering_oaddrm },
5c82000f 346 [SQIDI_PSMI] = { "SQIDI_PSMI", init_steering_sqidi_psmi },
3319b213 347 [INSTANCE0] = { "INSTANCE 0", init_steering_inst0 },
564d64f8 348 [IMPLICIT_STEERING] = { "IMPLICIT", NULL },
dd08ebf6
MB		 349};
350
351void xe_gt_mcr_init(struct xe_gt *gt)
352{
353 struct xe_device *xe = gt_to_xe(gt);
354
564d64f8 355 BUILD_BUG_ON(IMPLICIT_STEERING + 1 != NUM_STEERING_TYPES);
dd08ebf6
MB		 356 BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES);
357
be46d7aa
MW		 358 if (IS_SRIOV_VF(xe))
359 return;
360
dd08ebf6
MB		 361 spin_lock_init(&gt->mcr_lock);
362
363 if (gt->info.type == XE_GT_TYPE_MEDIA) {
364 drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13);
365
8e99b545
MR		 366 if (MEDIA_VER(xe) >= 20) {
367 gt->steering[OADDRM].ranges = xe2lpm_gpmxmt_steering_table;
368 gt->steering[INSTANCE0].ranges = xe2lpm_instance0_steering_table;
369 } else {
370 gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table;
371 }
dd08ebf6 372 } else {
5c82000f
MR		 373 if (GRAPHICS_VER(xe) >= 20) {
374 gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
375 gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table;
376 gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table;
377 } else if (GRAPHICS_VERx100(xe) >= 1270) {
436dbd6b
LDM		 378 gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table;
379 gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
380 gt->steering[DSS].ranges = xelpg_dss_steering_table;
381 } else if (xe->info.platform == XE_PVC) {
382 gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table;
383 gt->steering[DSS].ranges = xehpc_dss_steering_table;
384 } else if (xe->info.platform == XE_DG2) {
385 gt->steering[L3BANK].ranges = xehp_l3bank_steering_table;
386 gt->steering[MSLICE].ranges = xehp_mslice_steering_table;
387 gt->steering[LNCF].ranges = xehp_lncf_steering_table;
388 gt->steering[DSS].ranges = xehp_dss_steering_table;
389 gt->steering[IMPLICIT_STEERING].ranges = dg2_implicit_steering_table;
390 } else {
391 gt->steering[L3BANK].ranges = xelp_l3bank_steering_table;
392 gt->steering[DSS].ranges = xelp_dss_steering_table;
393 }
dd08ebf6
MB		 394 }
395
396 /* Select non-terminated steering target for each type */
397 for (int i = 0; i < NUM_STEERING_TYPES; i++)
398 if (gt->steering[i].ranges && xe_steering_types[i].init)
399 xe_steering_types[i].init(gt);
400}
401
564d64f8
LDM		 402/**
403 * xe_gt_mcr_set_implicit_defaults - Initialize steer control registers
404 * @gt: GT structure
405 *
406 * Some register ranges don't need to have their steering control registers
407 * changed on each access - it's sufficient to set them once on initialization.
408 * This function sets those registers for each platform *
409 */
410void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt)
411{
412 struct xe_device *xe = gt_to_xe(gt);
413
be46d7aa
MW		 414 if (IS_SRIOV_VF(xe))
415 return;
416
564d64f8 417 if (xe->info.platform == XE_DG2) {
d9b79ad2
LDM		 418 u32 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, 0) |
419 REG_FIELD_PREP(MCR_SUBSLICE_MASK, 2);
564d64f8 420
ce8bf5bd
LDM		 421 xe_mmio_write32(gt, MCFG_MCR_SELECTOR, steer_val);
422 xe_mmio_write32(gt, SF_MCR_SELECTOR, steer_val);
564d64f8
LDM		 423 /*
424 * For GAM registers, all reads should be directed to instance 1
425 * (unicast reads against other instances are not allowed),
426 * and instance 1 is already the hardware's default steering
427 * target, which we never change
428 */
429 }
430}
431
dd08ebf6
MB		 432/*
433 * xe_gt_mcr_get_nonterminated_steering - find group/instance values that
434 * will steer a register to a non-terminated instance
435 * @gt: GT structure
436 * @reg: register for which the steering is required
437 * @group: return variable for group steering
438 * @instance: return variable for instance steering
439 *
440 * This function returns a group/instance pair that is guaranteed to work for
441 * read steering of the given register. Note that a value will be returned even
442 * if the register is not replicated and therefore does not actually require
443 * steering.
444 *
445 * Returns true if the caller should steer to the @group/@instance values
564d64f8 446 * returned. Returns false if the caller need not perform any steering
dd08ebf6
MB		 447 */
448static bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
36e22be4 449 struct xe_reg_mcr reg_mcr,
dd08ebf6
MB		 450 u8 *group, u8 *instance)
451{
36e22be4 452 const struct xe_reg reg = to_xe_reg(reg_mcr);
564d64f8
LDM		 453 const struct xe_mmio_range *implicit_ranges;
454
455 for (int type = 0; type < IMPLICIT_STEERING; type++) {
dd08ebf6
MB		 456 if (!gt->steering[type].ranges)
457 continue;
458
459 for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) {
9fa81f91 460 if (xe_mmio_in_range(gt, &gt->steering[type].ranges[i], reg)) {
dd08ebf6
MB		 461 *group = gt->steering[type].group_target;
462 *instance = gt->steering[type].instance_target;
463 return true;
464 }
465 }
466 }
467
564d64f8
LDM		 468 implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges;
469 if (implicit_ranges)
470 for (int i = 0; implicit_ranges[i].end > 0; i++)
9fa81f91 471 if (xe_mmio_in_range(gt, &implicit_ranges[i], reg))
564d64f8 472 return false;
dd08ebf6
MB		 473
474 /*
564d64f8
LDM		 475 * Not found in a steering table and not a register with implicit
476 * steering. Just steer to 0/0 as a guess and raise a warning.
dd08ebf6
MB		 477 */
478 drm_WARN(&gt_to_xe(gt)->drm, true,
479 "Did not find MCR register %#x in any MCR steering table\n",
ee21379a 480 reg.addr);
dd08ebf6
MB		 481 *group = 0;
482 *instance = 0;
483
484 return true;
485}
486
dd08ebf6
MB		 487/*
488 * Obtain exclusive access to MCR steering. On MTL and beyond we also need
489 * to synchronize with external clients (e.g., firmware), so a semaphore
490 * register will also need to be taken.
491 */
97fd7a7e 492static void mcr_lock(struct xe_gt *gt) __acquires(&gt->mcr_lock)
dd08ebf6
MB		 493{
494 struct xe_device *xe = gt_to_xe(gt);
dc97898e 495 int ret = 0;
dd08ebf6
MB		 496
497 spin_lock(&gt->mcr_lock);
498
499 /*
500 * Starting with MTL we also need to grab a semaphore register
501 * to synchronize with external agents (e.g., firmware) that now
43b5d81e
DCS		 502 * shares the same steering control register. The semaphore is obtained
503 * when a read to the relevant register returns 1.
dd08ebf6
MB		 504 */
505 if (GRAPHICS_VERx100(xe) >= 1270)
43b5d81e 506 ret = xe_mmio_wait32(gt, STEER_SEMAPHORE, 0x1, 0x1, 10, NULL,
a56d8dab 507 true);
dd08ebf6
MB		 508
509 drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT);
510}
511
97fd7a7e 512static void mcr_unlock(struct xe_gt *gt) __releases(&gt->mcr_lock)
fb1d55ef 513{
43b5d81e 514 /* Release hardware semaphore - this is done by writing 1 to the register */
dd08ebf6
MB		 515 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270)
516 xe_mmio_write32(gt, STEER_SEMAPHORE, 0x1);
517
518 spin_unlock(&gt->mcr_lock);
519}
520
521/*
522 * Access a register with specific MCR steering
523 *
524 * Caller needs to make sure the relevant forcewake wells are up.
525 */
36e22be4
LDM		 526static u32 rw_with_mcr_steering(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
527 u8 rw_flag, int group, int instance, u32 value)
dd08ebf6 528{
36e22be4 529 const struct xe_reg reg = to_xe_reg(reg_mcr);
ce8bf5bd
LDM		 530 struct xe_reg steer_reg;
531 u32 steer_val, val = 0;
dd08ebf6
MB		 532
533 lockdep_assert_held(&gt->mcr_lock);
534
535 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
ce8bf5bd 536 steer_reg = MTL_MCR_SELECTOR;
dd08ebf6
MB		 537 steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) |
538 REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance);
539 } else {
ce8bf5bd 540 steer_reg = MCR_SELECTOR;
d9b79ad2
LDM		 541 steer_val = REG_FIELD_PREP(MCR_SLICE_MASK, group) |
542 REG_FIELD_PREP(MCR_SUBSLICE_MASK, instance);
dd08ebf6
MB		 543 }
544
545 /*
215bb2ce
LDM		 546 * Always leave the hardware in multicast mode when doing reads and only
547 * change it to unicast mode when doing writes of a specific instance.
548 *
549 * The setting of the multicast/unicast bit usually wouldn't matter for
550 * read operations (which always return the value from a single register
551 * instance regardless of how that bit is set), but some platforms may
552 * have workarounds requiring us to remain in multicast mode for reads,
553 * e.g. Wa_22013088509 on PVC. There's no real downside to this, so
554 * we'll just go ahead and do so on all platforms; we'll only clear the
555 * multicast bit from the mask when explicitly doing a write operation.
dd08ebf6
MB		 556 *
557 * No need to save old steering reg value.
558 */
559 if (rw_flag == MCR_OP_READ)
d9b79ad2 560 steer_val |= MCR_MULTICAST;
dd08ebf6
MB		 561
562 xe_mmio_write32(gt, steer_reg, steer_val);
563
564 if (rw_flag == MCR_OP_READ)
ce8bf5bd 565 val = xe_mmio_read32(gt, reg);
dd08ebf6 566 else
ce8bf5bd 567 xe_mmio_write32(gt, reg, value);
dd08ebf6
MB		 568
569 /*
570 * If we turned off the multicast bit (during a write) we're required
571 * to turn it back on before finishing. The group and instance values
572 * don't matter since they'll be re-programmed on the next MCR
573 * operation.
574 */
575 if (rw_flag == MCR_OP_WRITE)
d9b79ad2 576 xe_mmio_write32(gt, steer_reg, MCR_MULTICAST);
dd08ebf6
MB		 577
578 return val;
579}
580
581/**
582 * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register
583 * @gt: GT structure
36e22be4 584 * @reg_mcr: register to read
dd08ebf6
MB		 585 *
586 * Reads a GT MCR register. The read will be steered to a non-terminated
587 * instance (i.e., one that isn't fused off or powered down by power gating).
588 * This function assumes the caller is already holding any necessary forcewake
589 * domains.
590 *
591 * Returns the value from a non-terminated instance of @reg.
592 */
36e22be4 593u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, struct xe_reg_mcr reg_mcr)
dd08ebf6 594{
36e22be4 595 const struct xe_reg reg = to_xe_reg(reg_mcr);
dd08ebf6
MB		 596 u8 group, instance;
597 u32 val;
598 bool steer;
599
be46d7aa
MW		 600 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
601
36e22be4
LDM		 602 steer = xe_gt_mcr_get_nonterminated_steering(gt, reg_mcr,
603 &group, &instance);
dd08ebf6
MB		 604
605 if (steer) {
606 mcr_lock(gt);
36e22be4 607 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ,
dd08ebf6
MB		 608 group, instance, 0);
609 mcr_unlock(gt);
610 } else {
ce8bf5bd 611 val = xe_mmio_read32(gt, reg);
dd08ebf6
MB		 612 }
613
614 return val;
615}
616
617/**
618 * xe_gt_mcr_unicast_read - read a specific instance of an MCR register
619 * @gt: GT structure
36e22be4 620 * @reg_mcr: the MCR register to read
dd08ebf6
MB		 621 * @group: the MCR group
622 * @instance: the MCR instance
623 *
624 * Returns the value read from an MCR register after steering toward a specific
625 * group/instance.
626 */
627u32 xe_gt_mcr_unicast_read(struct xe_gt *gt,
36e22be4 628 struct xe_reg_mcr reg_mcr,
dd08ebf6
MB		 629 int group, int instance)
630{
631 u32 val;
632
be46d7aa
MW		 633 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
634
dd08ebf6 635 mcr_lock(gt);
36e22be4 636 val = rw_with_mcr_steering(gt, reg_mcr, MCR_OP_READ, group, instance, 0);
dd08ebf6
MB		 637 mcr_unlock(gt);
638
639 return val;
640}
641
642/**
643 * xe_gt_mcr_unicast_write - write a specific instance of an MCR register
644 * @gt: GT structure
36e22be4 645 * @reg_mcr: the MCR register to write
dd08ebf6
MB		 646 * @value: value to write
647 * @group: the MCR group
648 * @instance: the MCR instance
649 *
650 * Write an MCR register in unicast mode after steering toward a specific
651 * group/instance.
652 */
36e22be4
LDM		 653void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
654 u32 value, int group, int instance)
dd08ebf6 655{
be46d7aa
MW		 656 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
657
dd08ebf6 658 mcr_lock(gt);
36e22be4 659 rw_with_mcr_steering(gt, reg_mcr, MCR_OP_WRITE, group, instance, value);
dd08ebf6
MB		 660 mcr_unlock(gt);
661}
662
663/**
664 * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register
665 * @gt: GT structure
36e22be4 666 * @reg_mcr: the MCR register to write
dd08ebf6
MB		 667 * @value: value to write
668 *
669 * Write an MCR register in multicast mode to update all instances.
670 */
36e22be4
LDM		 671void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr reg_mcr,
672 u32 value)
dd08ebf6 673{
36e22be4
LDM		 674 struct xe_reg reg = to_xe_reg(reg_mcr);
675
be46d7aa
MW		 676 xe_gt_assert(gt, !IS_SRIOV_VF(gt_to_xe(gt)));
677
dd08ebf6
MB		 678 /*
679 * Synchronize with any unicast operations. Once we have exclusive
680 * access, the MULTICAST bit should already be set, so there's no need
681 * to touch the steering register.
682 */
683 mcr_lock(gt);
ce8bf5bd 684 xe_mmio_write32(gt, reg, value);
dd08ebf6
MB		 685 mcr_unlock(gt);
686}
687
688void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p)
689{
690 for (int i = 0; i < NUM_STEERING_TYPES; i++) {
691 if (gt->steering[i].ranges) {
692 drm_printf(p, "%s steering: group=%#x, instance=%#x\n",
693 xe_steering_types[i].name,
694 gt->steering[i].group_target,
695 gt->steering[i].instance_target);
696 for (int j = 0; gt->steering[i].ranges[j].end; j++)
697 drm_printf(p, "\t0x%06x - 0x%06x\n",
698 gt->steering[i].ranges[j].start,
699 gt->steering[i].ranges[j].end);
700 }
701 }
702}
Linux 6.x block layer and io_uring tree(s)