projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/xe: Reinstate render / compute cache invalidation in ring ops
[linux-block.git] / drivers / gpu / drm / xe / xe_gt.c
CommitLineData
dd08ebf6
MB		 1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
ea9f879d
LDM		 6#include "xe_gt.h"
7
dd08ebf6
MB		 8#include <linux/minmax.h>
9
10#include <drm/drm_managed.h>
11
226bfec8 12#include "regs/xe_gt_regs.h"
dd08ebf6
MB		 13#include "xe_bb.h"
14#include "xe_bo.h"
15#include "xe_device.h"
16#include "xe_engine.h"
17#include "xe_execlist.h"
18#include "xe_force_wake.h"
19#include "xe_ggtt.h"
dd08ebf6
MB		 20#include "xe_gt_clock.h"
21#include "xe_gt_mcr.h"
22#include "xe_gt_pagefault.h"
23#include "xe_gt_sysfs.h"
a9351846 24#include "xe_gt_tlb_invalidation.h"
dd08ebf6
MB		 25#include "xe_gt_topology.h"
26#include "xe_hw_fence.h"
27#include "xe_irq.h"
28#include "xe_lrc.h"
29#include "xe_map.h"
30#include "xe_migrate.h"
31#include "xe_mmio.h"
32#include "xe_mocs.h"
33#include "xe_reg_sr.h"
34#include "xe_ring_ops.h"
35#include "xe_sa.h"
36#include "xe_sched_job.h"
37#include "xe_ttm_gtt_mgr.h"
38#include "xe_ttm_vram_mgr.h"
39#include "xe_tuning.h"
40#include "xe_uc.h"
41#include "xe_vm.h"
42#include "xe_wa.h"
43#include "xe_wopcm.h"
44
dd08ebf6
MB		 45struct xe_gt *xe_find_full_gt(struct xe_gt *gt)
46{
47 struct xe_gt *search;
48 u8 id;
49
50 XE_BUG_ON(!xe_gt_is_media_type(gt));
51
52 for_each_gt(search, gt_to_xe(gt), id) {
53 if (search->info.vram_id == gt->info.vram_id)
54 return search;
55 }
56
57 XE_BUG_ON("NOT POSSIBLE");
58 return NULL;
59}
60
61int xe_gt_alloc(struct xe_device *xe, struct xe_gt *gt)
62{
63 struct drm_device *drm = &xe->drm;
64
65 XE_BUG_ON(gt->info.type == XE_GT_TYPE_UNINITIALIZED);
66
67 if (!xe_gt_is_media_type(gt)) {
68 gt->mem.ggtt = drmm_kzalloc(drm, sizeof(*gt->mem.ggtt),
69 GFP_KERNEL);
70 if (!gt->mem.ggtt)
71 return -ENOMEM;
72
73 gt->mem.vram_mgr = drmm_kzalloc(drm, sizeof(*gt->mem.vram_mgr),
74 GFP_KERNEL);
75 if (!gt->mem.vram_mgr)
76 return -ENOMEM;
77
78 gt->mem.gtt_mgr = drmm_kzalloc(drm, sizeof(*gt->mem.gtt_mgr),
79 GFP_KERNEL);
80 if (!gt->mem.gtt_mgr)
81 return -ENOMEM;
82 } else {
83 struct xe_gt *full_gt = xe_find_full_gt(gt);
84
85 gt->mem.ggtt = full_gt->mem.ggtt;
86 gt->mem.vram_mgr = full_gt->mem.vram_mgr;
87 gt->mem.gtt_mgr = full_gt->mem.gtt_mgr;
88 }
89
90 gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0);
91
92 return 0;
93}
94
95/* FIXME: These should be in a common file */
96#define CHV_PPAT_SNOOP REG_BIT(6)
97#define GEN8_PPAT_AGE(x) ((x)<<4)
98#define GEN8_PPAT_LLCeLLC (3<<2)
99#define GEN8_PPAT_LLCELLC (2<<2)
100#define GEN8_PPAT_LLC (1<<2)
101#define GEN8_PPAT_WB (3<<0)
102#define GEN8_PPAT_WT (2<<0)
103#define GEN8_PPAT_WC (1<<0)
104#define GEN8_PPAT_UC (0<<0)
105#define GEN8_PPAT_ELLC_OVERRIDE (0<<2)
106#define GEN8_PPAT(i, x) ((u64)(x) << ((i) * 8))
107#define GEN12_PPAT_CLOS(x) ((x)<<2)
108
109static void tgl_setup_private_ppat(struct xe_gt *gt)
110{
111 /* TGL doesn't support LLC or AGE settings */
112 xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, GEN8_PPAT_WB);
113 xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, GEN8_PPAT_WC);
114 xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, GEN8_PPAT_WT);
115 xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, GEN8_PPAT_UC);
116 xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg, GEN8_PPAT_WB);
117 xe_mmio_write32(gt, GEN12_PAT_INDEX(5).reg, GEN8_PPAT_WB);
118 xe_mmio_write32(gt, GEN12_PAT_INDEX(6).reg, GEN8_PPAT_WB);
119 xe_mmio_write32(gt, GEN12_PAT_INDEX(7).reg, GEN8_PPAT_WB);
120}
121
122static void pvc_setup_private_ppat(struct xe_gt *gt)
123{
124 xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, GEN8_PPAT_UC);
125 xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, GEN8_PPAT_WC);
126 xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, GEN8_PPAT_WT);
127 xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, GEN8_PPAT_WB);
128 xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg,
129 GEN12_PPAT_CLOS(1) | GEN8_PPAT_WT);
130 xe_mmio_write32(gt, GEN12_PAT_INDEX(5).reg,
131 GEN12_PPAT_CLOS(1) | GEN8_PPAT_WB);
132 xe_mmio_write32(gt, GEN12_PAT_INDEX(6).reg,
133 GEN12_PPAT_CLOS(2) | GEN8_PPAT_WT);
134 xe_mmio_write32(gt, GEN12_PAT_INDEX(7).reg,
135 GEN12_PPAT_CLOS(2) | GEN8_PPAT_WB);
136}
137
138#define MTL_PPAT_L4_CACHE_POLICY_MASK REG_GENMASK(3, 2)
139#define MTL_PAT_INDEX_COH_MODE_MASK REG_GENMASK(1, 0)
140#define MTL_PPAT_3_UC REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 3)
141#define MTL_PPAT_1_WT REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 1)
142#define MTL_PPAT_0_WB REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 0)
143#define MTL_3_COH_2W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 3)
144#define MTL_2_COH_1W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 2)
145#define MTL_0_COH_NON REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 0)
146
147static void mtl_setup_private_ppat(struct xe_gt *gt)
148{
149 xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, MTL_PPAT_0_WB);
150 xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg,
151 MTL_PPAT_1_WT | MTL_2_COH_1W);
152 xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg,
153 MTL_PPAT_3_UC | MTL_2_COH_1W);
154 xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg,
155 MTL_PPAT_0_WB | MTL_2_COH_1W);
156 xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg,
157 MTL_PPAT_0_WB | MTL_3_COH_2W);
158}
159
160static void setup_private_ppat(struct xe_gt *gt)
161{
162 struct xe_device *xe = gt_to_xe(gt);
163
164 if (xe->info.platform == XE_METEORLAKE)
165 mtl_setup_private_ppat(gt);
166 else if (xe->info.platform == XE_PVC)
167 pvc_setup_private_ppat(gt);
168 else
169 tgl_setup_private_ppat(gt);
170}
171
172static int gt_ttm_mgr_init(struct xe_gt *gt)
173{
174 struct xe_device *xe = gt_to_xe(gt);
175 int err;
176 struct sysinfo si;
177 u64 gtt_size;
178
179 si_meminfo(&si);
180 gtt_size = (u64)si.totalram * si.mem_unit * 3/4;
181
182 if (gt->mem.vram.size) {
183 err = xe_ttm_vram_mgr_init(gt, gt->mem.vram_mgr);
184 if (err)
185 return err;
186 gtt_size = min(max((XE_DEFAULT_GTT_SIZE_MB << 20),
2c387882 187 (u64)gt->mem.vram.size),
dd08ebf6
MB		 188 gtt_size);
189 xe->info.mem_region_mask |= BIT(gt->info.vram_id) << 1;
190 }
191
192 err = xe_ttm_gtt_mgr_init(gt, gt->mem.gtt_mgr, gtt_size);
193 if (err)
194 return err;
195
196 return 0;
197}
198
da3799c9
MB		 199void xe_gt_sanitize(struct xe_gt *gt)
200{
201 /*
202 * FIXME: if xe_uc_sanitize is called here, on TGL driver will not
203 * reload
204 */
205 gt->uc.guc.submission_state.enabled = false;
206}
207
dd08ebf6
MB		 208static void gt_fini(struct drm_device *drm, void *arg)
209{
210 struct xe_gt *gt = arg;
211 int i;
212
213 destroy_workqueue(gt->ordered_wq);
214
215 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
216 xe_hw_fence_irq_finish(&gt->fence_irq[i]);
217}
218
219static void gt_reset_worker(struct work_struct *w);
220
671ca05d 221static int emit_nop_job(struct xe_gt *gt, struct xe_engine *e)
dd08ebf6
MB		 222{
223 struct xe_sched_job *job;
224 struct xe_bb *bb;
225 struct dma_fence *fence;
226 u64 batch_ofs;
227 long timeout;
228
229 bb = xe_bb_new(gt, 4, false);
230 if (IS_ERR(bb))
231 return PTR_ERR(bb);
232
233 batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool.bo);
234 job = xe_bb_create_wa_job(e, bb, batch_ofs);
235 if (IS_ERR(job)) {
236 xe_bb_free(bb, NULL);
237 return PTR_ERR(bb);
238 }
239
240 xe_sched_job_arm(job);
241 fence = dma_fence_get(&job->drm.s_fence->finished);
242 xe_sched_job_push(job);
243
244 timeout = dma_fence_wait_timeout(fence, false, HZ);
245 dma_fence_put(fence);
246 xe_bb_free(bb, NULL);
247 if (timeout < 0)
248 return timeout;
249 else if (!timeout)
250 return -ETIME;
251
252 return 0;
253}
254
671ca05d 255static int emit_wa_job(struct xe_gt *gt, struct xe_engine *e)
dd08ebf6
MB		 256{
257 struct xe_reg_sr *sr = &e->hwe->reg_lrc;
258 struct xe_reg_sr_entry *entry;
259 unsigned long reg;
260 struct xe_sched_job *job;
261 struct xe_bb *bb;
262 struct dma_fence *fence;
263 u64 batch_ofs;
264 long timeout;
265 int count = 0;
266
267 bb = xe_bb_new(gt, SZ_4K, false); /* Just pick a large BB size */
268 if (IS_ERR(bb))
269 return PTR_ERR(bb);
270
271 xa_for_each(&sr->xa, reg, entry)
272 ++count;
273
274 if (count) {
275 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM(count);
276 xa_for_each(&sr->xa, reg, entry) {
277 bb->cs[bb->len++] = reg;
278 bb->cs[bb->len++] = entry->set_bits;
279 }
280 }
281 bb->cs[bb->len++] = MI_NOOP;
282 bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
283
284 batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool.bo);
285 job = xe_bb_create_wa_job(e, bb, batch_ofs);
286 if (IS_ERR(job)) {
287 xe_bb_free(bb, NULL);
288 return PTR_ERR(bb);
289 }
290
291 xe_sched_job_arm(job);
292 fence = dma_fence_get(&job->drm.s_fence->finished);
293 xe_sched_job_push(job);
294
295 timeout = dma_fence_wait_timeout(fence, false, HZ);
296 dma_fence_put(fence);
297 xe_bb_free(bb, NULL);
298 if (timeout < 0)
299 return timeout;
300 else if (!timeout)
301 return -ETIME;
302
303 return 0;
304}
305
306int xe_gt_record_default_lrcs(struct xe_gt *gt)
307{
308 struct xe_device *xe = gt_to_xe(gt);
309 struct xe_hw_engine *hwe;
310 enum xe_hw_engine_id id;
311 int err = 0;
312
313 for_each_hw_engine(hwe, gt, id) {
314 struct xe_engine *e, *nop_e;
315 struct xe_vm *vm;
316 void *default_lrc;
317
318 if (gt->default_lrc[hwe->class])
319 continue;
320
766849c4 321 xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
dd08ebf6 322 xe_wa_process_lrc(hwe);
3dbec470 323 xe_tuning_process_lrc(hwe);
dd08ebf6
MB		 324
325 default_lrc = drmm_kzalloc(&xe->drm,
326 xe_lrc_size(xe, hwe->class),
327 GFP_KERNEL);
328 if (!default_lrc)
329 return -ENOMEM;
330
331 vm = xe_migrate_get_vm(gt->migrate);
332 e = xe_engine_create(xe, vm, BIT(hwe->logical_instance), 1,
333 hwe, ENGINE_FLAG_WA);
334 if (IS_ERR(e)) {
335 err = PTR_ERR(e);
336 goto put_vm;
337 }
338
339 /* Prime golden LRC with known good state */
340 err = emit_wa_job(gt, e);
341 if (err)
342 goto put_engine;
343
344 nop_e = xe_engine_create(xe, vm, BIT(hwe->logical_instance),
345 1, hwe, ENGINE_FLAG_WA);
346 if (IS_ERR(nop_e)) {
347 err = PTR_ERR(nop_e);
348 goto put_engine;
349 }
350
351 /* Switch to different LRC */
352 err = emit_nop_job(gt, nop_e);
353 if (err)
354 goto put_nop_e;
355
356 /* Reload golden LRC to record the effect of any indirect W/A */
357 err = emit_nop_job(gt, e);
358 if (err)
359 goto put_nop_e;
360
361 xe_map_memcpy_from(xe, default_lrc,
362 &e->lrc[0].bo->vmap,
363 xe_lrc_pphwsp_offset(&e->lrc[0]),
364 xe_lrc_size(xe, hwe->class));
365
366 gt->default_lrc[hwe->class] = default_lrc;
367put_nop_e:
368 xe_engine_put(nop_e);
369put_engine:
370 xe_engine_put(e);
371put_vm:
372 xe_vm_put(vm);
373 if (err)
374 break;
375 }
376
377 return err;
378}
379
380int xe_gt_init_early(struct xe_gt *gt)
381{
382 int err;
383
384 xe_force_wake_init_gt(gt, gt_to_fw(gt));
385
386 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
387 if (err)
388 return err;
389
390 xe_gt_topology_init(gt);
391 xe_gt_mcr_init(gt);
392
393 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
394 if (err)
395 return err;
396
397 xe_reg_sr_init(&gt->reg_sr, "GT", gt_to_xe(gt));
398 xe_wa_process_gt(gt);
399 xe_tuning_process_gt(gt);
400
401 return 0;
402}
403
404/**
405 * xe_gt_init_noalloc - Init GT up to the point where allocations can happen.
406 * @gt: The GT to initialize.
407 *
408 * This function prepares the GT to allow memory allocations to VRAM, but is not
409 * allowed to allocate memory itself. This state is useful for display readout,
410 * because the inherited display framebuffer will otherwise be overwritten as it
411 * is usually put at the start of VRAM.
412 *
413 * Returns: 0 on success, negative error code on error.
414 */
415int xe_gt_init_noalloc(struct xe_gt *gt)
416{
417 int err, err2;
418
419 if (xe_gt_is_media_type(gt))
420 return 0;
421
422 xe_device_mem_access_get(gt_to_xe(gt));
423 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
424 if (err)
425 goto err;
426
427 err = gt_ttm_mgr_init(gt);
428 if (err)
429 goto err_force_wake;
430
431 err = xe_ggtt_init_noalloc(gt, gt->mem.ggtt);
432
433err_force_wake:
434 err2 = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
435 XE_WARN_ON(err2);
436 xe_device_mem_access_put(gt_to_xe(gt));
437err:
438 return err;
439}
440
441static int gt_fw_domain_init(struct xe_gt *gt)
442{
443 int err, i;
444
445 xe_device_mem_access_get(gt_to_xe(gt));
446 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
447 if (err)
448 goto err_hw_fence_irq;
449
6c8c1e74
PL		 450 setup_private_ppat(gt);
451
dd08ebf6
MB		 452 if (!xe_gt_is_media_type(gt)) {
453 err = xe_ggtt_init(gt, gt->mem.ggtt);
454 if (err)
455 goto err_force_wake;
456 }
457
458 /* Allow driver to load if uC init fails (likely missing firmware) */
459 err = xe_uc_init(&gt->uc);
460 XE_WARN_ON(err);
461
462 err = xe_uc_init_hwconfig(&gt->uc);
463 if (err)
464 goto err_force_wake;
465
da34c2cf
MB		 466 /* XXX: Fake that we pull the engine mask from hwconfig blob */
467 gt->info.engine_mask = gt->info.__engine_mask;
468
dd08ebf6
MB		 469 /* Enables per hw engine IRQs */
470 xe_gt_irq_postinstall(gt);
471
472 /* Rerun MCR init as we now have hw engine list */
473 xe_gt_mcr_init(gt);
474
475 err = xe_hw_engines_init_early(gt);
476 if (err)
477 goto err_force_wake;
478
479 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
480 XE_WARN_ON(err);
481 xe_device_mem_access_put(gt_to_xe(gt));
482
483 return 0;
484
485err_force_wake:
486 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
487err_hw_fence_irq:
488 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
489 xe_hw_fence_irq_finish(&gt->fence_irq[i]);
490 xe_device_mem_access_put(gt_to_xe(gt));
491
492 return err;
493}
494
495static int all_fw_domain_init(struct xe_gt *gt)
496{
497 int err, i;
498
499 xe_device_mem_access_get(gt_to_xe(gt));
500 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
501 if (err)
502 goto err_hw_fence_irq;
503
564d64f8 504 xe_gt_mcr_set_implicit_defaults(gt);
dd08ebf6
MB		 505 xe_reg_sr_apply_mmio(&gt->reg_sr, gt);
506
507 err = xe_gt_clock_init(gt);
508 if (err)
509 goto err_force_wake;
510
511 xe_mocs_init(gt);
512 err = xe_execlist_init(gt);
513 if (err)
514 goto err_force_wake;
515
516 err = xe_hw_engines_init(gt);
517 if (err)
518 goto err_force_wake;
519
520 err = xe_uc_init_post_hwconfig(&gt->uc);
521 if (err)
522 goto err_force_wake;
523
524 /*
525 * FIXME: This should be ok as SA should only be used by gt->migrate and
526 * vm->gt->migrate and both should be pointing to a non-media GT. But to
527 * realy safe, convert gt->kernel_bb_pool to a pointer and point a media
528 * GT to the kernel_bb_pool on a real tile.
529 */
530 if (!xe_gt_is_media_type(gt)) {
531 err = xe_sa_bo_manager_init(gt, &gt->kernel_bb_pool, SZ_1M, 16);
532 if (err)
533 goto err_force_wake;
534
535 /*
536 * USM has its only SA pool to non-block behind user operations
537 */
538 if (gt_to_xe(gt)->info.supports_usm) {
539 err = xe_sa_bo_manager_init(gt, &gt->usm.bb_pool,
540 SZ_1M, 16);
541 if (err)
542 goto err_force_wake;
543 }
544 }
545
546 if (!xe_gt_is_media_type(gt)) {
547 gt->migrate = xe_migrate_init(gt);
548 if (IS_ERR(gt->migrate))
549 goto err_force_wake;
550 } else {
551 gt->migrate = xe_find_full_gt(gt)->migrate;
552 }
553
554 err = xe_uc_init_hw(&gt->uc);
555 if (err)
556 goto err_force_wake;
557
558 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
559 XE_WARN_ON(err);
560 xe_device_mem_access_put(gt_to_xe(gt));
561
562 return 0;
563
564err_force_wake:
565 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
566err_hw_fence_irq:
567 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
568 xe_hw_fence_irq_finish(&gt->fence_irq[i]);
569 xe_device_mem_access_put(gt_to_xe(gt));
570
571 return err;
572}
573
574int xe_gt_init(struct xe_gt *gt)
575{
576 int err;
577 int i;
578
579 INIT_WORK(&gt->reset.worker, gt_reset_worker);
580
581 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) {
582 gt->ring_ops[i] = xe_ring_ops_get(gt, i);
583 xe_hw_fence_irq_init(&gt->fence_irq[i]);
584 }
585
a9351846
MB		 586 err = xe_gt_tlb_invalidation_init(gt);
587 if (err)
588 return err;
589
dd08ebf6
MB		 590 err = xe_gt_pagefault_init(gt);
591 if (err)
592 return err;
593
594 xe_gt_sysfs_init(gt);
595
596 err = gt_fw_domain_init(gt);
597 if (err)
598 return err;
599
600 xe_force_wake_init_engines(gt, gt_to_fw(gt));
601
602 err = all_fw_domain_init(gt);
603 if (err)
604 return err;
605
dd08ebf6
MB		 606 err = drmm_add_action_or_reset(&gt_to_xe(gt)->drm, gt_fini, gt);
607 if (err)
608 return err;
609
610 return 0;
611}
612
671ca05d 613static int do_gt_reset(struct xe_gt *gt)
dd08ebf6
MB		 614{
615 struct xe_device *xe = gt_to_xe(gt);
616 int err;
617
618 xe_mmio_write32(gt, GEN6_GDRST.reg, GEN11_GRDOM_FULL);
81593af6 619 err = xe_mmio_wait32(gt, GEN6_GDRST.reg, 0, GEN11_GRDOM_FULL, 5000,
7dc9b92d 620 NULL, false);
dd08ebf6
MB		 621 if (err)
622 drm_err(&xe->drm,
623 "GT reset failed to clear GEN11_GRDOM_FULL\n");
624
625 return err;
626}
627
628static int do_gt_restart(struct xe_gt *gt)
629{
630 struct xe_hw_engine *hwe;
631 enum xe_hw_engine_id id;
632 int err;
633
634 setup_private_ppat(gt);
635
564d64f8 636 xe_gt_mcr_set_implicit_defaults(gt);
dd08ebf6
MB		 637 xe_reg_sr_apply_mmio(&gt->reg_sr, gt);
638
639 err = xe_wopcm_init(&gt->uc.wopcm);
640 if (err)
641 return err;
642
643 for_each_hw_engine(hwe, gt, id)
644 xe_hw_engine_enable_ring(hwe);
645
646 err = xe_uc_init_hw(&gt->uc);
647 if (err)
648 return err;
649
650 xe_mocs_init(gt);
651 err = xe_uc_start(&gt->uc);
652 if (err)
653 return err;
654
655 for_each_hw_engine(hwe, gt, id) {
656 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);
657 xe_reg_sr_apply_whitelist(&hwe->reg_whitelist,
658 hwe->mmio_base, gt);
659 }
660
661 return 0;
662}
663
664static int gt_reset(struct xe_gt *gt)
665{
666 struct xe_device *xe = gt_to_xe(gt);
667 int err;
668
669 /* We only support GT resets with GuC submission */
670 if (!xe_device_guc_submission_enabled(gt_to_xe(gt)))
671 return -ENODEV;
672
673 drm_info(&xe->drm, "GT reset started\n");
674
da3799c9
MB		 675 xe_gt_sanitize(gt);
676
dd08ebf6
MB		 677 xe_device_mem_access_get(gt_to_xe(gt));
678 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
679 if (err)
680 goto err_msg;
681
682 xe_uc_stop_prepare(&gt->uc);
683 xe_gt_pagefault_reset(gt);
fc108a8b 684 xe_gt_tlb_invalidation_reset(gt);
dd08ebf6
MB		 685
686 err = xe_uc_stop(&gt->uc);
687 if (err)
688 goto err_out;
689
690 err = do_gt_reset(gt);
691 if (err)
692 goto err_out;
693
694 err = do_gt_restart(gt);
695 if (err)
696 goto err_out;
697
698 xe_device_mem_access_put(gt_to_xe(gt));
699 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
700 XE_WARN_ON(err);
701
702 drm_info(&xe->drm, "GT reset done\n");
703
704 return 0;
705
706err_out:
707 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
708err_msg:
709 XE_WARN_ON(xe_uc_start(&gt->uc));
710 xe_device_mem_access_put(gt_to_xe(gt));
711 drm_err(&xe->drm, "GT reset failed, err=%d\n", err);
712
713 return err;
714}
715
716static void gt_reset_worker(struct work_struct *w)
717{
718 struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker);
719
720 gt_reset(gt);
721}
722
723void xe_gt_reset_async(struct xe_gt *gt)
724{
725 struct xe_device *xe = gt_to_xe(gt);
726
727 drm_info(&xe->drm, "Try GT reset\n");
728
729 /* Don't do a reset while one is already in flight */
730 if (xe_uc_reset_prepare(&gt->uc))
731 return;
732
733 drm_info(&xe->drm, "Doing GT reset\n");
734 queue_work(gt->ordered_wq, &gt->reset.worker);
735}
736
737void xe_gt_suspend_prepare(struct xe_gt *gt)
738{
739 xe_device_mem_access_get(gt_to_xe(gt));
740 XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
741
742 xe_uc_stop_prepare(&gt->uc);
743
744 xe_device_mem_access_put(gt_to_xe(gt));
745 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
746}
747
748int xe_gt_suspend(struct xe_gt *gt)
749{
750 struct xe_device *xe = gt_to_xe(gt);
751 int err;
752
753 /* For now suspend/resume is only allowed with GuC */
754 if (!xe_device_guc_submission_enabled(gt_to_xe(gt)))
755 return -ENODEV;
756
da3799c9
MB		 757 xe_gt_sanitize(gt);
758
dd08ebf6
MB		 759 xe_device_mem_access_get(gt_to_xe(gt));
760 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
761 if (err)
762 goto err_msg;
763
764 err = xe_uc_suspend(&gt->uc);
765 if (err)
766 goto err_force_wake;
767
768 xe_device_mem_access_put(gt_to_xe(gt));
769 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
770 drm_info(&xe->drm, "GT suspended\n");
771
772 return 0;
773
774err_force_wake:
775 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
776err_msg:
777 xe_device_mem_access_put(gt_to_xe(gt));
778 drm_err(&xe->drm, "GT suspend failed: %d\n", err);
779
780 return err;
781}
782
783int xe_gt_resume(struct xe_gt *gt)
784{
785 struct xe_device *xe = gt_to_xe(gt);
786 int err;
787
788 xe_device_mem_access_get(gt_to_xe(gt));
789 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
790 if (err)
791 goto err_msg;
792
793 err = do_gt_restart(gt);
794 if (err)
795 goto err_force_wake;
796
797 xe_device_mem_access_put(gt_to_xe(gt));
798 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
799 drm_info(&xe->drm, "GT resumed\n");
800
801 return 0;
802
803err_force_wake:
804 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
805err_msg:
806 xe_device_mem_access_put(gt_to_xe(gt));
807 drm_err(&xe->drm, "GT resume failed: %d\n", err);
808
809 return err;
810}
811
812void xe_gt_migrate_wait(struct xe_gt *gt)
813{
814 xe_migrate_wait(gt->migrate);
815}
816
817struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,
818 enum xe_engine_class class,
819 u16 instance, bool logical)
820{
821 struct xe_hw_engine *hwe;
822 enum xe_hw_engine_id id;
823
824 for_each_hw_engine(hwe, gt, id)
825 if (hwe->class == class &&
826 ((!logical && hwe->instance == instance) ||
827 (logical && hwe->logical_instance == instance)))
828 return hwe;
829
830 return NULL;
831}
832
833struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt,
834 enum xe_engine_class class)
835{
836 struct xe_hw_engine *hwe;
837 enum xe_hw_engine_id id;
838
839 for_each_hw_engine(hwe, gt, id) {
840 switch (class) {
841 case XE_ENGINE_CLASS_RENDER:
842 case XE_ENGINE_CLASS_COMPUTE:
843 if (hwe->class == XE_ENGINE_CLASS_RENDER ||
844 hwe->class == XE_ENGINE_CLASS_COMPUTE)
845 return hwe;
846 break;
847 default:
848 if (hwe->class == class)
849 return hwe;
850 }
851 }
852
853 return NULL;
854}
Linux 6.x block layer and io_uring tree(s)