Commit | Line | Data |
---|---|---|
a6581ebe MW |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2023 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <drm/drm_managed.h> | |
7 | ||
8 | #include "regs/xe_gt_regs.h" | |
9 | #include "regs/xe_guc_regs.h" | |
10 | #include "regs/xe_regs.h" | |
11 | ||
12 | #include "xe_assert.h" | |
13 | #include "xe_bo.h" | |
14 | #include "xe_device.h" | |
15 | #include "xe_device_types.h" | |
16 | #include "xe_gt.h" | |
17 | #include "xe_gt_printk.h" | |
18 | #include "xe_guc.h" | |
19 | #include "xe_hw_engine.h" | |
20 | #include "xe_map.h" | |
21 | #include "xe_memirq.h" | |
22 | #include "xe_sriov.h" | |
23 | #include "xe_sriov_printk.h" | |
24 | ||
25 | #define memirq_assert(m, condition) xe_tile_assert(memirq_to_tile(m), condition) | |
26 | #define memirq_debug(m, msg...) xe_sriov_dbg_verbose(memirq_to_xe(m), "MEMIRQ: " msg) | |
27 | ||
28 | static struct xe_tile *memirq_to_tile(struct xe_memirq *memirq) | |
29 | { | |
30 | return container_of(memirq, struct xe_tile, sriov.vf.memirq); | |
31 | } | |
32 | ||
33 | static struct xe_device *memirq_to_xe(struct xe_memirq *memirq) | |
34 | { | |
35 | return tile_to_xe(memirq_to_tile(memirq)); | |
36 | } | |
37 | ||
38 | static const char *guc_name(struct xe_guc *guc) | |
39 | { | |
40 | return xe_gt_is_media_type(guc_to_gt(guc)) ? "media GuC" : "GuC"; | |
41 | } | |
42 | ||
43 | /** | |
44 | * DOC: Memory Based Interrupts | |
45 | * | |
46 | * MMIO register based interrupts infrastructure used for non-virtualized mode | |
47 | * or SRIOV-8 (which supports 8 Virtual Functions) does not scale efficiently | |
48 | * to allow delivering interrupts to a large number of Virtual machines or | |
49 | * containers. Memory based interrupt status reporting provides an efficient | |
50 | * and scalable infrastructure. | |
51 | * | |
52 | * For memory based interrupt status reporting hardware sequence is: | |
53 | * * Engine writes the interrupt event to memory | |
54 | * (Pointer to memory location is provided by SW. This memory surface must | |
55 | * be mapped to system memory and must be marked as un-cacheable (UC) on | |
56 | * Graphics IP Caches) | |
57 | * * Engine triggers an interrupt to host. | |
58 | */ | |
59 | ||
60 | /** | |
61 | * DOC: Memory Based Interrupts Page Layout | |
62 | * | |
63 | * `Memory Based Interrupts`_ requires three different objects, which are | |
64 | * called "page" in the specs, even if they aren't page-sized or aligned. | |
65 | * | |
66 | * To simplify the code we allocate a single page size object and then use | |
67 | * offsets to embedded "pages". The address of those "pages" are then | |
68 | * programmed in the HW via LRI and LRM in the context image. | |
69 | * | |
70 | * - _`Interrupt Status Report Page`: this page contains the interrupt | |
71 | * status vectors for each unit. Each bit in the interrupt vectors is | |
72 | * converted to a byte, with the byte being set to 0xFF when an | |
73 | * interrupt is triggered; interrupt vectors are 16b big so each unit | |
74 | * gets 16B. One space is reserved for each bit in one of the | |
75 | * GT_INTR_DWx registers, so this object needs a total of 1024B. | |
76 | * This object needs to be 4KiB aligned. | |
77 | * | |
78 | * - _`Interrupt Source Report Page`: this is the equivalent of the | |
79 | * GEN11_GT_INTR_DWx registers, with each bit in those registers being | |
80 | * mapped to a byte here. The offsets are the same, just bytes instead | |
81 | * of bits. This object needs to be cacheline aligned. | |
82 | * | |
83 | * - Interrupt Mask: the HW needs a location to fetch the interrupt | |
84 | * mask vector to be used by the LRM in the context, so we just use | |
85 | * the next available space in the interrupt page. | |
86 | * | |
87 | * :: | |
88 | * | |
89 | * 0x0000 +===========+ <== Interrupt Status Report Page | |
90 | * | | | |
91 | * | | ____ +----+----------------+ | |
92 | * | | / | 0 | USER INTERRUPT | | |
93 | * +-----------+ __/ | 1 | | | |
94 | * | HWE(n) | __ | | CTX SWITCH | | |
95 | * +-----------+ \ | | WAIT SEMAPHORE | | |
96 | * | | \____ | 15 | | | |
97 | * | | +----+----------------+ | |
98 | * | | | |
99 | * 0x0400 +===========+ <== Interrupt Source Report Page | |
100 | * | HWE(0) | | |
101 | * | HWE(1) | | |
102 | * | | | |
103 | * | HWE(x) | | |
104 | * 0x0440 +===========+ <== Interrupt Enable Mask | |
105 | * | | | |
106 | * | | | |
107 | * +-----------+ | |
108 | */ | |
109 | ||
110 | static void __release_xe_bo(struct drm_device *drm, void *arg) | |
111 | { | |
112 | struct xe_bo *bo = arg; | |
113 | ||
114 | xe_bo_unpin_map_no_vm(bo); | |
115 | } | |
116 | ||
117 | static int memirq_alloc_pages(struct xe_memirq *memirq) | |
118 | { | |
119 | struct xe_device *xe = memirq_to_xe(memirq); | |
120 | struct xe_tile *tile = memirq_to_tile(memirq); | |
121 | struct xe_bo *bo; | |
122 | int err; | |
123 | ||
124 | BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET, SZ_64)); | |
125 | BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET, SZ_4K)); | |
126 | ||
127 | /* XXX: convert to managed bo */ | |
128 | bo = xe_bo_create_pin_map(xe, tile, NULL, SZ_4K, | |
129 | ttm_bo_type_kernel, | |
130 | XE_BO_CREATE_SYSTEM_BIT | | |
131 | XE_BO_CREATE_GGTT_BIT | | |
132 | XE_BO_NEEDS_UC | | |
133 | XE_BO_NEEDS_CPU_ACCESS); | |
134 | if (IS_ERR(bo)) { | |
135 | err = PTR_ERR(bo); | |
136 | goto out; | |
137 | } | |
138 | ||
139 | memirq_assert(memirq, !xe_bo_is_vram(bo)); | |
140 | memirq_assert(memirq, !memirq->bo); | |
141 | ||
142 | iosys_map_memset(&bo->vmap, 0, 0, SZ_4K); | |
143 | ||
144 | memirq->bo = bo; | |
145 | memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET); | |
146 | memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET); | |
147 | memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET); | |
148 | ||
149 | memirq_assert(memirq, !memirq->source.is_iomem); | |
150 | memirq_assert(memirq, !memirq->status.is_iomem); | |
151 | memirq_assert(memirq, !memirq->mask.is_iomem); | |
152 | ||
153 | memirq_debug(memirq, "page offsets: source %#x status %#x\n", | |
154 | xe_memirq_source_ptr(memirq), xe_memirq_status_ptr(memirq)); | |
155 | ||
156 | return drmm_add_action_or_reset(&xe->drm, __release_xe_bo, memirq->bo); | |
157 | ||
158 | out: | |
159 | xe_sriov_err(memirq_to_xe(memirq), | |
160 | "Failed to allocate memirq page (%pe)\n", ERR_PTR(err)); | |
161 | return err; | |
162 | } | |
163 | ||
164 | static void memirq_set_enable(struct xe_memirq *memirq, bool enable) | |
165 | { | |
166 | iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0); | |
167 | ||
168 | memirq->enabled = enable; | |
169 | } | |
170 | ||
171 | /** | |
172 | * xe_memirq_init - Initialize data used by `Memory Based Interrupts`_. | |
173 | * @memirq: the &xe_memirq to initialize | |
174 | * | |
175 | * Allocate `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_ | |
176 | * used by `Memory Based Interrupts`_. | |
177 | * | |
178 | * These allocations are managed and will be implicitly released on unload. | |
179 | * | |
180 | * Note: This function shall be called only by the VF driver. | |
181 | * | |
182 | * If this function fails then VF driver won't be able to operate correctly. | |
183 | * If `Memory Based Interrupts`_ are not used this function will return 0. | |
184 | * | |
185 | * Return: 0 on success or a negative error code on failure. | |
186 | */ | |
187 | int xe_memirq_init(struct xe_memirq *memirq) | |
188 | { | |
189 | struct xe_device *xe = memirq_to_xe(memirq); | |
190 | int err; | |
191 | ||
192 | memirq_assert(memirq, IS_SRIOV_VF(xe)); | |
193 | ||
194 | if (!xe_device_has_memirq(xe)) | |
195 | return 0; | |
196 | ||
197 | err = memirq_alloc_pages(memirq); | |
198 | if (unlikely(err)) | |
199 | return err; | |
200 | ||
201 | /* we need to start with all irqs enabled */ | |
202 | memirq_set_enable(memirq, true); | |
203 | ||
204 | return 0; | |
205 | } | |
206 | ||
207 | /** | |
208 | * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_. | |
209 | * @memirq: the &xe_memirq to query | |
210 | * | |
211 | * Shall be called only on VF driver when `Memory Based Interrupts`_ are used | |
212 | * and xe_memirq_init() didn't fail. | |
213 | * | |
214 | * Return: GGTT's offset of the `Interrupt Source Report Page`_. | |
215 | */ | |
216 | u32 xe_memirq_source_ptr(struct xe_memirq *memirq) | |
217 | { | |
218 | memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); | |
219 | memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); | |
220 | memirq_assert(memirq, memirq->bo); | |
221 | ||
222 | return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET; | |
223 | } | |
224 | ||
225 | /** | |
226 | * xe_memirq_status_ptr - Get GGTT's offset of the `Interrupt Status Report Page`_. | |
227 | * @memirq: the &xe_memirq to query | |
228 | * | |
229 | * Shall be called only on VF driver when `Memory Based Interrupts`_ are used | |
230 | * and xe_memirq_init() didn't fail. | |
231 | * | |
232 | * Return: GGTT's offset of the `Interrupt Status Report Page`_. | |
233 | */ | |
234 | u32 xe_memirq_status_ptr(struct xe_memirq *memirq) | |
235 | { | |
236 | memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); | |
237 | memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); | |
238 | memirq_assert(memirq, memirq->bo); | |
239 | ||
240 | return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET; | |
241 | } | |
242 | ||
243 | /** | |
244 | * xe_memirq_enable_ptr - Get GGTT's offset of the Interrupt Enable Mask. | |
245 | * @memirq: the &xe_memirq to query | |
246 | * | |
247 | * Shall be called only on VF driver when `Memory Based Interrupts`_ are used | |
248 | * and xe_memirq_init() didn't fail. | |
249 | * | |
250 | * Return: GGTT's offset of the Interrupt Enable Mask. | |
251 | */ | |
252 | u32 xe_memirq_enable_ptr(struct xe_memirq *memirq) | |
253 | { | |
254 | memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); | |
255 | memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); | |
256 | memirq_assert(memirq, memirq->bo); | |
257 | ||
258 | return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_ENABLE_OFFSET; | |
259 | } | |
260 | ||
261 | /** | |
262 | * xe_memirq_init_guc - Prepare GuC for `Memory Based Interrupts`_. | |
263 | * @memirq: the &xe_memirq | |
264 | * @guc: the &xe_guc to setup | |
265 | * | |
266 | * Register `Interrupt Source Report Page`_ and `Interrupt Status Report Page`_ | |
267 | * to be used by the GuC when `Memory Based Interrupts`_ are required. | |
268 | * | |
269 | * Shall be called only on VF driver when `Memory Based Interrupts`_ are used | |
270 | * and xe_memirq_init() didn't fail. | |
271 | * | |
272 | * Return: 0 on success or a negative error code on failure. | |
273 | */ | |
274 | int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc) | |
275 | { | |
276 | bool is_media = xe_gt_is_media_type(guc_to_gt(guc)); | |
277 | u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC); | |
278 | u32 source, status; | |
279 | int err; | |
280 | ||
281 | memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); | |
282 | memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); | |
283 | memirq_assert(memirq, memirq->bo); | |
284 | ||
285 | source = xe_memirq_source_ptr(memirq) + offset; | |
286 | status = xe_memirq_status_ptr(memirq) + offset * SZ_16; | |
287 | ||
288 | err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY, | |
289 | source); | |
290 | if (unlikely(err)) | |
291 | goto failed; | |
292 | ||
293 | err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY, | |
294 | status); | |
295 | if (unlikely(err)) | |
296 | goto failed; | |
297 | ||
298 | return 0; | |
299 | ||
300 | failed: | |
301 | xe_sriov_err(memirq_to_xe(memirq), | |
302 | "Failed to setup report pages in %s (%pe)\n", | |
303 | guc_name(guc), ERR_PTR(err)); | |
304 | return err; | |
305 | } | |
306 | ||
307 | /** | |
308 | * xe_memirq_reset - Disable processing of `Memory Based Interrupts`_. | |
309 | * @memirq: struct xe_memirq | |
310 | * | |
311 | * This is part of the driver IRQ setup flow. | |
312 | * | |
313 | * This function shall only be used by the VF driver on platforms that use | |
314 | * `Memory Based Interrupts`_. | |
315 | */ | |
316 | void xe_memirq_reset(struct xe_memirq *memirq) | |
317 | { | |
318 | memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); | |
319 | memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); | |
320 | ||
321 | if (memirq->bo) | |
322 | memirq_set_enable(memirq, false); | |
323 | } | |
324 | ||
325 | /** | |
326 | * xe_memirq_postinstall - Enable processing of `Memory Based Interrupts`_. | |
327 | * @memirq: the &xe_memirq | |
328 | * | |
329 | * This is part of the driver IRQ setup flow. | |
330 | * | |
331 | * This function shall only be used by the VF driver on platforms that use | |
332 | * `Memory Based Interrupts`_. | |
333 | */ | |
334 | void xe_memirq_postinstall(struct xe_memirq *memirq) | |
335 | { | |
336 | memirq_assert(memirq, IS_SRIOV_VF(memirq_to_xe(memirq))); | |
337 | memirq_assert(memirq, xe_device_has_memirq(memirq_to_xe(memirq))); | |
338 | ||
339 | if (memirq->bo) | |
340 | memirq_set_enable(memirq, true); | |
341 | } | |
342 | ||
343 | static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector, | |
344 | u16 offset, const char *name) | |
345 | { | |
346 | u8 value; | |
347 | ||
348 | value = iosys_map_rd(vector, offset, u8); | |
349 | if (value) { | |
350 | if (value != 0xff) | |
351 | xe_sriov_err_ratelimited(memirq_to_xe(memirq), | |
352 | "Unexpected memirq value %#x from %s at %u\n", | |
353 | value, name, offset); | |
354 | iosys_map_wr(vector, offset, u8, 0x00); | |
355 | } | |
356 | ||
357 | return value; | |
358 | } | |
359 | ||
360 | static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status, | |
361 | struct xe_hw_engine *hwe) | |
362 | { | |
363 | memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr); | |
364 | ||
365 | if (memirq_received(memirq, status, ilog2(GT_RENDER_USER_INTERRUPT), hwe->name)) | |
366 | xe_hw_engine_handle_irq(hwe, GT_RENDER_USER_INTERRUPT); | |
367 | } | |
368 | ||
369 | static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status, | |
370 | struct xe_guc *guc) | |
371 | { | |
372 | const char *name = guc_name(guc); | |
373 | ||
374 | memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr); | |
375 | ||
376 | if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name)) | |
377 | xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST); | |
378 | } | |
379 | ||
380 | /** | |
381 | * xe_memirq_handler - The `Memory Based Interrupts`_ Handler. | |
382 | * @memirq: the &xe_memirq | |
383 | * | |
384 | * This function reads and dispatches `Memory Based Interrupts`. | |
385 | */ | |
386 | void xe_memirq_handler(struct xe_memirq *memirq) | |
387 | { | |
388 | struct xe_device *xe = memirq_to_xe(memirq); | |
389 | struct xe_tile *tile = memirq_to_tile(memirq); | |
390 | struct xe_hw_engine *hwe; | |
391 | enum xe_hw_engine_id id; | |
392 | struct iosys_map map; | |
393 | unsigned int gtid; | |
394 | struct xe_gt *gt; | |
395 | ||
396 | if (!memirq->bo) | |
397 | return; | |
398 | ||
399 | memirq_assert(memirq, !memirq->source.is_iomem); | |
400 | memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr); | |
401 | memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32); | |
402 | ||
403 | for_each_gt(gt, xe, gtid) { | |
404 | if (gt->tile != tile) | |
405 | continue; | |
406 | ||
407 | for_each_hw_engine(hwe, gt, id) { | |
408 | if (memirq_received(memirq, &memirq->source, hwe->irq_offset, "SRC")) { | |
409 | map = IOSYS_MAP_INIT_OFFSET(&memirq->status, | |
410 | hwe->irq_offset * SZ_16); | |
411 | memirq_dispatch_engine(memirq, &map, hwe); | |
412 | } | |
413 | } | |
414 | } | |
415 | ||
416 | /* GuC and media GuC (if present) must be checked separately */ | |
417 | ||
418 | if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) { | |
419 | map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16); | |
420 | memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc); | |
421 | } | |
422 | ||
423 | if (!tile->media_gt) | |
424 | return; | |
425 | ||
426 | if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) { | |
427 | map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16); | |
428 | memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc); | |
429 | } | |
430 | } |