Commit | Line | Data |
---|---|---|
dd08ebf6 MB |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2022 Intel Corporation | |
4 | */ | |
5 | ||
ea9f879d LDM |
6 | #include "xe_gt.h" |
7 | ||
dd08ebf6 MB |
8 | #include <linux/minmax.h> |
9 | ||
10 | #include <drm/drm_managed.h> | |
4f027e30 | 11 | #include <drm/xe_drm.h> |
dd08ebf6 | 12 | |
72ac3047 | 13 | #include "instructions/xe_gfxpipe_commands.h" |
0134f130 | 14 | #include "instructions/xe_mi_commands.h" |
226bfec8 | 15 | #include "regs/xe_gt_regs.h" |
c73acc1e | 16 | #include "xe_assert.h" |
dd08ebf6 MB |
17 | #include "xe_bb.h" |
18 | #include "xe_bo.h" | |
19 | #include "xe_device.h" | |
c22a4ed0 | 20 | #include "xe_exec_queue.h" |
dd08ebf6 MB |
21 | #include "xe_execlist.h" |
22 | #include "xe_force_wake.h" | |
23 | #include "xe_ggtt.h" | |
aae84bf1 | 24 | #include "xe_gsc.h" |
0d97ecce | 25 | #include "xe_gt_ccs_mode.h" |
dd08ebf6 | 26 | #include "xe_gt_clock.h" |
bef52b5c | 27 | #include "xe_gt_freq.h" |
c550f64f | 28 | #include "xe_gt_idle.h" |
dd08ebf6 MB |
29 | #include "xe_gt_mcr.h" |
30 | #include "xe_gt_pagefault.h" | |
3e535bd5 | 31 | #include "xe_gt_printk.h" |
dd08ebf6 | 32 | #include "xe_gt_sysfs.h" |
a9351846 | 33 | #include "xe_gt_tlb_invalidation.h" |
dd08ebf6 | 34 | #include "xe_gt_topology.h" |
9b9529ce | 35 | #include "xe_guc_exec_queue_types.h" |
43efd3ba | 36 | #include "xe_guc_pc.h" |
dd08ebf6 | 37 | #include "xe_hw_fence.h" |
038ff941 | 38 | #include "xe_hw_engine_class_sysfs.h" |
dd08ebf6 | 39 | #include "xe_irq.h" |
b1d20405 | 40 | #include "xe_lmtt.h" |
dd08ebf6 MB |
41 | #include "xe_lrc.h" |
42 | #include "xe_map.h" | |
43 | #include "xe_migrate.h" | |
44 | #include "xe_mmio.h" | |
576c6380 | 45 | #include "xe_pat.h" |
dd08ebf6 MB |
46 | #include "xe_mocs.h" |
47 | #include "xe_reg_sr.h" | |
48 | #include "xe_ring_ops.h" | |
49 | #include "xe_sa.h" | |
50 | #include "xe_sched_job.h" | |
b1d20405 | 51 | #include "xe_sriov.h" |
dd08ebf6 MB |
52 | #include "xe_tuning.h" |
53 | #include "xe_uc.h" | |
54 | #include "xe_vm.h" | |
55 | #include "xe_wa.h" | |
56 | #include "xe_wopcm.h" | |
57 | ||
f6929e80 | 58 | struct xe_gt *xe_gt_alloc(struct xe_tile *tile) |
dd08ebf6 | 59 | { |
f6929e80 | 60 | struct xe_gt *gt; |
dd08ebf6 | 61 | |
f6929e80 MR |
62 | gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL); |
63 | if (!gt) | |
64 | return ERR_PTR(-ENOMEM); | |
65 | ||
66 | gt->tile = tile; | |
dd08ebf6 MB |
67 | gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0); |
68 | ||
f6929e80 | 69 | return gt; |
dd08ebf6 MB |
70 | } |
71 | ||
da3799c9 MB |
72 | void xe_gt_sanitize(struct xe_gt *gt) |
73 | { | |
74 | /* | |
75 | * FIXME: if xe_uc_sanitize is called here, on TGL driver will not | |
76 | * reload | |
77 | */ | |
78 | gt->uc.guc.submission_state.enabled = false; | |
79 | } | |
80 | ||
997a55ca DCS |
81 | /** |
82 | * xe_gt_remove() - Clean up the GT structures before driver removal | |
83 | * @gt: the GT object | |
84 | * | |
85 | * This function should only act on objects/structures that must be cleaned | |
86 | * before the driver removal callback is complete and therefore can't be | |
87 | * deferred to a drmm action. | |
88 | */ | |
89 | void xe_gt_remove(struct xe_gt *gt) | |
90 | { | |
91 | xe_uc_remove(>->uc); | |
92 | } | |
93 | ||
dd08ebf6 MB |
94 | static void gt_fini(struct drm_device *drm, void *arg) |
95 | { | |
96 | struct xe_gt *gt = arg; | |
97 | int i; | |
98 | ||
99 | destroy_workqueue(gt->ordered_wq); | |
100 | ||
101 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) | |
102 | xe_hw_fence_irq_finish(>->fence_irq[i]); | |
103 | } | |
104 | ||
105 | static void gt_reset_worker(struct work_struct *w); | |
106 | ||
9b9529ce | 107 | static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) |
dd08ebf6 MB |
108 | { |
109 | struct xe_sched_job *job; | |
110 | struct xe_bb *bb; | |
111 | struct dma_fence *fence; | |
dd08ebf6 MB |
112 | long timeout; |
113 | ||
114 | bb = xe_bb_new(gt, 4, false); | |
115 | if (IS_ERR(bb)) | |
116 | return PTR_ERR(bb); | |
117 | ||
9e952635 | 118 | job = xe_bb_create_job(q, bb); |
dd08ebf6 MB |
119 | if (IS_ERR(job)) { |
120 | xe_bb_free(bb, NULL); | |
99c5952f | 121 | return PTR_ERR(job); |
dd08ebf6 MB |
122 | } |
123 | ||
124 | xe_sched_job_arm(job); | |
125 | fence = dma_fence_get(&job->drm.s_fence->finished); | |
126 | xe_sched_job_push(job); | |
127 | ||
128 | timeout = dma_fence_wait_timeout(fence, false, HZ); | |
129 | dma_fence_put(fence); | |
130 | xe_bb_free(bb, NULL); | |
131 | if (timeout < 0) | |
132 | return timeout; | |
133 | else if (!timeout) | |
134 | return -ETIME; | |
135 | ||
136 | return 0; | |
137 | } | |
138 | ||
278c3582 LDM |
139 | /* |
140 | * Convert back from encoded value to type-safe, only to be used when reg.mcr | |
141 | * is true | |
142 | */ | |
143 | static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) | |
144 | { | |
145 | return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; | |
146 | } | |
147 | ||
9b9529ce | 148 | static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) |
dd08ebf6 | 149 | { |
9b9529ce | 150 | struct xe_reg_sr *sr = &q->hwe->reg_lrc; |
dd08ebf6 | 151 | struct xe_reg_sr_entry *entry; |
278c3582 | 152 | unsigned long idx; |
dd08ebf6 MB |
153 | struct xe_sched_job *job; |
154 | struct xe_bb *bb; | |
155 | struct dma_fence *fence; | |
dd08ebf6 MB |
156 | long timeout; |
157 | int count = 0; | |
158 | ||
b1543a49 MR |
159 | if (q->hwe->class == XE_ENGINE_CLASS_RENDER) |
160 | /* Big enough to emit all of the context's 3DSTATE */ | |
161 | bb = xe_bb_new(gt, xe_lrc_size(gt_to_xe(gt), q->hwe->class), false); | |
162 | else | |
163 | /* Just pick a large BB size */ | |
164 | bb = xe_bb_new(gt, SZ_4K, false); | |
165 | ||
dd08ebf6 MB |
166 | if (IS_ERR(bb)) |
167 | return PTR_ERR(bb); | |
168 | ||
278c3582 | 169 | xa_for_each(&sr->xa, idx, entry) |
dd08ebf6 MB |
170 | ++count; |
171 | ||
172 | if (count) { | |
12a66a47 LDM |
173 | xe_gt_dbg(gt, "LRC WA %s save-restore batch\n", sr->name); |
174 | ||
e12a6488 | 175 | bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count); |
278c3582 LDM |
176 | |
177 | xa_for_each(&sr->xa, idx, entry) { | |
178 | struct xe_reg reg = entry->reg; | |
179 | struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); | |
180 | u32 val; | |
181 | ||
182 | /* | |
183 | * Skip reading the register if it's not really needed | |
184 | */ | |
185 | if (reg.masked) | |
186 | val = entry->clr_bits << 16; | |
187 | else if (entry->clr_bits + 1) | |
188 | val = (reg.mcr ? | |
189 | xe_gt_mcr_unicast_read_any(gt, reg_mcr) : | |
190 | xe_mmio_read32(gt, reg)) & (~entry->clr_bits); | |
191 | else | |
192 | val = 0; | |
193 | ||
194 | val |= entry->set_bits; | |
195 | ||
196 | bb->cs[bb->len++] = reg.addr; | |
197 | bb->cs[bb->len++] = val; | |
198 | xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); | |
dd08ebf6 MB |
199 | } |
200 | } | |
dd08ebf6 | 201 | |
b1543a49 MR |
202 | xe_lrc_emit_hwe_state_instructions(q, bb); |
203 | ||
9e952635 | 204 | job = xe_bb_create_job(q, bb); |
dd08ebf6 MB |
205 | if (IS_ERR(job)) { |
206 | xe_bb_free(bb, NULL); | |
99c5952f | 207 | return PTR_ERR(job); |
dd08ebf6 MB |
208 | } |
209 | ||
210 | xe_sched_job_arm(job); | |
211 | fence = dma_fence_get(&job->drm.s_fence->finished); | |
212 | xe_sched_job_push(job); | |
213 | ||
214 | timeout = dma_fence_wait_timeout(fence, false, HZ); | |
215 | dma_fence_put(fence); | |
216 | xe_bb_free(bb, NULL); | |
217 | if (timeout < 0) | |
218 | return timeout; | |
219 | else if (!timeout) | |
220 | return -ETIME; | |
221 | ||
222 | return 0; | |
223 | } | |
224 | ||
225 | int xe_gt_record_default_lrcs(struct xe_gt *gt) | |
226 | { | |
227 | struct xe_device *xe = gt_to_xe(gt); | |
228 | struct xe_hw_engine *hwe; | |
229 | enum xe_hw_engine_id id; | |
230 | int err = 0; | |
231 | ||
232 | for_each_hw_engine(hwe, gt, id) { | |
9b9529ce | 233 | struct xe_exec_queue *q, *nop_q; |
dd08ebf6 MB |
234 | void *default_lrc; |
235 | ||
236 | if (gt->default_lrc[hwe->class]) | |
237 | continue; | |
238 | ||
766849c4 | 239 | xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); |
dd08ebf6 | 240 | xe_wa_process_lrc(hwe); |
bb95a4f9 | 241 | xe_hw_engine_setup_default_lrc_state(hwe); |
3dbec470 | 242 | xe_tuning_process_lrc(hwe); |
dd08ebf6 MB |
243 | |
244 | default_lrc = drmm_kzalloc(&xe->drm, | |
245 | xe_lrc_size(xe, hwe->class), | |
246 | GFP_KERNEL); | |
247 | if (!default_lrc) | |
248 | return -ENOMEM; | |
249 | ||
9e952635 | 250 | q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1, |
25ce7c50 | 251 | hwe, EXEC_QUEUE_FLAG_KERNEL, 0); |
9b9529ce FD |
252 | if (IS_ERR(q)) { |
253 | err = PTR_ERR(q); | |
254 | xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n", | |
255 | hwe->name, q); | |
9e952635 | 256 | return err; |
dd08ebf6 MB |
257 | } |
258 | ||
259 | /* Prime golden LRC with known good state */ | |
9b9529ce | 260 | err = emit_wa_job(gt, q); |
3d4451d3 | 261 | if (err) { |
3e535bd5 | 262 | xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n", |
9b9529ce FD |
263 | hwe->name, ERR_PTR(err), q->guc->id); |
264 | goto put_exec_queue; | |
3d4451d3 | 265 | } |
dd08ebf6 | 266 | |
9e952635 | 267 | nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), |
25ce7c50 | 268 | 1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0); |
9b9529ce FD |
269 | if (IS_ERR(nop_q)) { |
270 | err = PTR_ERR(nop_q); | |
271 | xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n", | |
272 | hwe->name, nop_q); | |
273 | goto put_exec_queue; | |
dd08ebf6 MB |
274 | } |
275 | ||
276 | /* Switch to different LRC */ | |
9b9529ce | 277 | err = emit_nop_job(gt, nop_q); |
3d4451d3 | 278 | if (err) { |
3e535bd5 | 279 | xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n", |
9b9529ce FD |
280 | hwe->name, ERR_PTR(err), nop_q->guc->id); |
281 | goto put_nop_q; | |
3d4451d3 | 282 | } |
dd08ebf6 MB |
283 | |
284 | /* Reload golden LRC to record the effect of any indirect W/A */ | |
9b9529ce | 285 | err = emit_nop_job(gt, q); |
3d4451d3 | 286 | if (err) { |
3e535bd5 | 287 | xe_gt_err(gt, "hwe %s: emit_nop_job failed (%pe) guc_id=%u\n", |
9b9529ce FD |
288 | hwe->name, ERR_PTR(err), q->guc->id); |
289 | goto put_nop_q; | |
3d4451d3 | 290 | } |
dd08ebf6 MB |
291 | |
292 | xe_map_memcpy_from(xe, default_lrc, | |
9b9529ce FD |
293 | &q->lrc[0].bo->vmap, |
294 | xe_lrc_pphwsp_offset(&q->lrc[0]), | |
dd08ebf6 MB |
295 | xe_lrc_size(xe, hwe->class)); |
296 | ||
297 | gt->default_lrc[hwe->class] = default_lrc; | |
9b9529ce FD |
298 | put_nop_q: |
299 | xe_exec_queue_put(nop_q); | |
300 | put_exec_queue: | |
301 | xe_exec_queue_put(q); | |
dd08ebf6 MB |
302 | if (err) |
303 | break; | |
304 | } | |
305 | ||
306 | return err; | |
307 | } | |
308 | ||
309 | int xe_gt_init_early(struct xe_gt *gt) | |
310 | { | |
311 | int err; | |
312 | ||
dd08ebf6 MB |
313 | err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); |
314 | if (err) | |
315 | return err; | |
316 | ||
dd08ebf6 MB |
317 | err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); |
318 | if (err) | |
319 | return err; | |
320 | ||
321 | xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt)); | |
49d329a0 LDM |
322 | |
323 | err = xe_wa_init(gt); | |
324 | if (err) | |
325 | return err; | |
326 | ||
dd08ebf6 | 327 | xe_wa_process_gt(gt); |
9616e74b | 328 | xe_wa_process_oob(gt); |
dd08ebf6 MB |
329 | xe_tuning_process_gt(gt); |
330 | ||
331 | return 0; | |
332 | } | |
333 | ||
d2300987 LDM |
334 | static void dump_pat_on_error(struct xe_gt *gt) |
335 | { | |
336 | struct drm_printer p; | |
337 | char prefix[32]; | |
338 | ||
339 | snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id); | |
e7835e02 | 340 | p = drm_dbg_printer(>_to_xe(gt)->drm, DRM_UT_DRIVER, prefix); |
d2300987 LDM |
341 | |
342 | xe_pat_dump(gt, &p); | |
343 | } | |
344 | ||
dd08ebf6 MB |
345 | static int gt_fw_domain_init(struct xe_gt *gt) |
346 | { | |
347 | int err, i; | |
348 | ||
349 | xe_device_mem_access_get(gt_to_xe(gt)); | |
350 | err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
351 | if (err) | |
352 | goto err_hw_fence_irq; | |
353 | ||
354 | if (!xe_gt_is_media_type(gt)) { | |
ad703e06 | 355 | err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt); |
dd08ebf6 MB |
356 | if (err) |
357 | goto err_force_wake; | |
b1d20405 MW |
358 | if (IS_SRIOV_PF(gt_to_xe(gt))) |
359 | xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt); | |
dd08ebf6 MB |
360 | } |
361 | ||
1c2097bb RT |
362 | xe_gt_idle_sysfs_init(>->gtidle); |
363 | ||
3e29c149 MR |
364 | /* Enable per hw engine IRQs */ |
365 | xe_irq_enable_hwe(gt); | |
dd08ebf6 MB |
366 | |
367 | /* Rerun MCR init as we now have hw engine list */ | |
368 | xe_gt_mcr_init(gt); | |
369 | ||
370 | err = xe_hw_engines_init_early(gt); | |
371 | if (err) | |
372 | goto err_force_wake; | |
373 | ||
038ff941 TU |
374 | err = xe_hw_engine_class_sysfs_init(gt); |
375 | if (err) | |
376 | drm_warn(>_to_xe(gt)->drm, | |
377 | "failed to register engines sysfs directory, err: %d\n", | |
378 | err); | |
379 | ||
f3bc5bb4 | 380 | /* Initialize CCS mode sysfs after early initialization of HW engines */ |
d6dab901 HPG |
381 | err = xe_gt_ccs_mode_sysfs_init(gt); |
382 | if (err) | |
383 | goto err_force_wake; | |
f3bc5bb4 | 384 | |
facd3887 MR |
385 | /* |
386 | * Stash hardware-reported version. Since this register does not exist | |
387 | * on pre-MTL platforms, reading it there will (correctly) return 0. | |
388 | */ | |
389 | gt->info.gmdid = xe_mmio_read32(gt, GMD_ID); | |
390 | ||
dd08ebf6 MB |
391 | err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); |
392 | XE_WARN_ON(err); | |
393 | xe_device_mem_access_put(gt_to_xe(gt)); | |
394 | ||
395 | return 0; | |
396 | ||
397 | err_force_wake: | |
d2300987 | 398 | dump_pat_on_error(gt); |
dd08ebf6 MB |
399 | xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); |
400 | err_hw_fence_irq: | |
401 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) | |
402 | xe_hw_fence_irq_finish(>->fence_irq[i]); | |
403 | xe_device_mem_access_put(gt_to_xe(gt)); | |
404 | ||
405 | return err; | |
406 | } | |
407 | ||
408 | static int all_fw_domain_init(struct xe_gt *gt) | |
409 | { | |
410 | int err, i; | |
411 | ||
412 | xe_device_mem_access_get(gt_to_xe(gt)); | |
413 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
414 | if (err) | |
415 | goto err_hw_fence_irq; | |
416 | ||
564d64f8 | 417 | xe_gt_mcr_set_implicit_defaults(gt); |
dd08ebf6 MB |
418 | xe_reg_sr_apply_mmio(>->reg_sr, gt); |
419 | ||
420 | err = xe_gt_clock_init(gt); | |
421 | if (err) | |
422 | goto err_force_wake; | |
423 | ||
424 | xe_mocs_init(gt); | |
425 | err = xe_execlist_init(gt); | |
426 | if (err) | |
427 | goto err_force_wake; | |
428 | ||
429 | err = xe_hw_engines_init(gt); | |
430 | if (err) | |
431 | goto err_force_wake; | |
432 | ||
dd08ebf6 | 433 | if (!xe_gt_is_media_type(gt)) { |
dd08ebf6 MB |
434 | /* |
435 | * USM has its only SA pool to non-block behind user operations | |
436 | */ | |
5a92da34 | 437 | if (gt_to_xe(gt)->info.has_usm) { |
72f86ed3 MB |
438 | struct xe_device *xe = gt_to_xe(gt); |
439 | ||
440 | gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt), | |
441 | IS_DGFX(xe) ? SZ_1M : SZ_512K, 16); | |
0a12a612 MR |
442 | if (IS_ERR(gt->usm.bb_pool)) { |
443 | err = PTR_ERR(gt->usm.bb_pool); | |
dd08ebf6 | 444 | goto err_force_wake; |
0a12a612 | 445 | } |
dd08ebf6 MB |
446 | } |
447 | } | |
448 | ||
449 | if (!xe_gt_is_media_type(gt)) { | |
08dea767 MR |
450 | struct xe_tile *tile = gt_to_tile(gt); |
451 | ||
452 | tile->migrate = xe_migrate_init(tile); | |
453 | if (IS_ERR(tile->migrate)) { | |
454 | err = PTR_ERR(tile->migrate); | |
dd08ebf6 | 455 | goto err_force_wake; |
99c5952f | 456 | } |
dd08ebf6 MB |
457 | } |
458 | ||
bf8ec3c3 MW |
459 | err = xe_uc_init_post_hwconfig(>->uc); |
460 | if (err) | |
461 | goto err_force_wake; | |
462 | ||
dd08ebf6 MB |
463 | err = xe_uc_init_hw(>->uc); |
464 | if (err) | |
465 | goto err_force_wake; | |
466 | ||
0d97ecce NV |
467 | /* Configure default CCS mode of 1 engine with all resources */ |
468 | if (xe_gt_ccs_mode_enabled(gt)) { | |
469 | gt->ccs_mode = 1; | |
470 | xe_gt_apply_ccs_mode(gt); | |
471 | } | |
472 | ||
b1d20405 MW |
473 | if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt)) |
474 | xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); | |
475 | ||
dd08ebf6 MB |
476 | err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); |
477 | XE_WARN_ON(err); | |
478 | xe_device_mem_access_put(gt_to_xe(gt)); | |
479 | ||
480 | return 0; | |
481 | ||
482 | err_force_wake: | |
483 | xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
484 | err_hw_fence_irq: | |
485 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) | |
486 | xe_hw_fence_irq_finish(>->fence_irq[i]); | |
487 | xe_device_mem_access_put(gt_to_xe(gt)); | |
488 | ||
489 | return err; | |
490 | } | |
491 | ||
bf8ec3c3 MW |
492 | /* |
493 | * Initialize enough GT to be able to load GuC in order to obtain hwconfig and | |
494 | * enable CTB communication. | |
495 | */ | |
496 | int xe_gt_init_hwconfig(struct xe_gt *gt) | |
497 | { | |
498 | int err; | |
499 | ||
500 | xe_device_mem_access_get(gt_to_xe(gt)); | |
501 | err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
502 | if (err) | |
503 | goto out; | |
504 | ||
9eeeed8d | 505 | xe_gt_topology_init(gt); |
bf8ec3c3 MW |
506 | xe_gt_mcr_init(gt); |
507 | xe_pat_init(gt); | |
508 | ||
509 | err = xe_uc_init(>->uc); | |
510 | if (err) | |
511 | goto out_fw; | |
512 | ||
513 | err = xe_uc_init_hwconfig(>->uc); | |
514 | if (err) | |
515 | goto out_fw; | |
516 | ||
517 | /* XXX: Fake that we pull the engine mask from hwconfig blob */ | |
518 | gt->info.engine_mask = gt->info.__engine_mask; | |
519 | ||
520 | out_fw: | |
521 | xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); | |
522 | out: | |
523 | xe_device_mem_access_put(gt_to_xe(gt)); | |
524 | ||
525 | return err; | |
526 | } | |
527 | ||
dd08ebf6 MB |
528 | int xe_gt_init(struct xe_gt *gt) |
529 | { | |
530 | int err; | |
531 | int i; | |
532 | ||
533 | INIT_WORK(>->reset.worker, gt_reset_worker); | |
534 | ||
535 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) { | |
536 | gt->ring_ops[i] = xe_ring_ops_get(gt, i); | |
537 | xe_hw_fence_irq_init(>->fence_irq[i]); | |
538 | } | |
539 | ||
a9351846 MB |
540 | err = xe_gt_tlb_invalidation_init(gt); |
541 | if (err) | |
542 | return err; | |
543 | ||
dd08ebf6 MB |
544 | err = xe_gt_pagefault_init(gt); |
545 | if (err) | |
546 | return err; | |
547 | ||
17a6726c MR |
548 | xe_mocs_init_early(gt); |
549 | ||
dd08ebf6 MB |
550 | xe_gt_sysfs_init(gt); |
551 | ||
552 | err = gt_fw_domain_init(gt); | |
553 | if (err) | |
554 | return err; | |
555 | ||
bef52b5c RV |
556 | xe_gt_freq_init(gt); |
557 | ||
dd08ebf6 MB |
558 | xe_force_wake_init_engines(gt, gt_to_fw(gt)); |
559 | ||
560 | err = all_fw_domain_init(gt); | |
561 | if (err) | |
562 | return err; | |
563 | ||
dd08ebf6 MB |
564 | err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt); |
565 | if (err) | |
566 | return err; | |
567 | ||
568 | return 0; | |
569 | } | |
570 | ||
671ca05d | 571 | static int do_gt_reset(struct xe_gt *gt) |
dd08ebf6 | 572 | { |
dd08ebf6 MB |
573 | int err; |
574 | ||
aae84bf1 DCS |
575 | xe_gsc_wa_14015076503(gt, true); |
576 | ||
ce8bf5bd | 577 | xe_mmio_write32(gt, GDRST, GRDOM_FULL); |
063e09af | 578 | err = xe_mmio_wait32(gt, GDRST, GRDOM_FULL, 0, 5000, NULL, false); |
dd08ebf6 | 579 | if (err) |
0bc519d2 | 580 | xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n", |
3e535bd5 | 581 | ERR_PTR(err)); |
dd08ebf6 | 582 | |
aae84bf1 DCS |
583 | xe_gsc_wa_14015076503(gt, false); |
584 | ||
dd08ebf6 MB |
585 | return err; |
586 | } | |
587 | ||
588 | static int do_gt_restart(struct xe_gt *gt) | |
589 | { | |
590 | struct xe_hw_engine *hwe; | |
591 | enum xe_hw_engine_id id; | |
592 | int err; | |
593 | ||
576c6380 | 594 | xe_pat_init(gt); |
dd08ebf6 | 595 | |
564d64f8 | 596 | xe_gt_mcr_set_implicit_defaults(gt); |
dd08ebf6 MB |
597 | xe_reg_sr_apply_mmio(>->reg_sr, gt); |
598 | ||
599 | err = xe_wopcm_init(>->uc.wopcm); | |
600 | if (err) | |
601 | return err; | |
602 | ||
603 | for_each_hw_engine(hwe, gt, id) | |
604 | xe_hw_engine_enable_ring(hwe); | |
605 | ||
7704f32c MW |
606 | err = xe_uc_sanitize_reset(>->uc); |
607 | if (err) | |
608 | return err; | |
609 | ||
dd08ebf6 MB |
610 | err = xe_uc_init_hw(>->uc); |
611 | if (err) | |
612 | return err; | |
613 | ||
b1d20405 MW |
614 | if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt)) |
615 | xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); | |
616 | ||
dd08ebf6 MB |
617 | xe_mocs_init(gt); |
618 | err = xe_uc_start(>->uc); | |
619 | if (err) | |
620 | return err; | |
621 | ||
622 | for_each_hw_engine(hwe, gt, id) { | |
623 | xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); | |
1011812c | 624 | xe_reg_sr_apply_whitelist(hwe); |
dd08ebf6 MB |
625 | } |
626 | ||
0d97ecce NV |
627 | /* Get CCS mode in sync between sw/hw */ |
628 | xe_gt_apply_ccs_mode(gt); | |
629 | ||
dd08ebf6 MB |
630 | return 0; |
631 | } | |
632 | ||
633 | static int gt_reset(struct xe_gt *gt) | |
634 | { | |
dd08ebf6 MB |
635 | int err; |
636 | ||
637 | /* We only support GT resets with GuC submission */ | |
c4991ee0 | 638 | if (!xe_device_uc_enabled(gt_to_xe(gt))) |
dd08ebf6 MB |
639 | return -ENODEV; |
640 | ||
3e535bd5 | 641 | xe_gt_info(gt, "reset started\n"); |
dd08ebf6 | 642 | |
8f3013e0 HPG |
643 | if (xe_fault_inject_gt_reset()) { |
644 | err = -ECANCELED; | |
645 | goto err_fail; | |
646 | } | |
647 | ||
da3799c9 MB |
648 | xe_gt_sanitize(gt); |
649 | ||
dd08ebf6 MB |
650 | xe_device_mem_access_get(gt_to_xe(gt)); |
651 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
652 | if (err) | |
653 | goto err_msg; | |
654 | ||
1737785a | 655 | xe_uc_gucrc_disable(>->uc); |
dd08ebf6 MB |
656 | xe_uc_stop_prepare(>->uc); |
657 | xe_gt_pagefault_reset(gt); | |
658 | ||
659 | err = xe_uc_stop(>->uc); | |
660 | if (err) | |
661 | goto err_out; | |
662 | ||
83a7173b MB |
663 | xe_gt_tlb_invalidation_reset(gt); |
664 | ||
dd08ebf6 MB |
665 | err = do_gt_reset(gt); |
666 | if (err) | |
667 | goto err_out; | |
668 | ||
669 | err = do_gt_restart(gt); | |
670 | if (err) | |
671 | goto err_out; | |
672 | ||
dd08ebf6 | 673 | err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); |
03af26c9 | 674 | xe_device_mem_access_put(gt_to_xe(gt)); |
dd08ebf6 MB |
675 | XE_WARN_ON(err); |
676 | ||
3e535bd5 | 677 | xe_gt_info(gt, "reset done\n"); |
dd08ebf6 MB |
678 | |
679 | return 0; | |
680 | ||
681 | err_out: | |
682 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
683 | err_msg: | |
684 | XE_WARN_ON(xe_uc_start(>->uc)); | |
685 | xe_device_mem_access_put(gt_to_xe(gt)); | |
8f3013e0 | 686 | err_fail: |
3e535bd5 | 687 | xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err)); |
dd08ebf6 | 688 | |
57162274 AH |
689 | gt_to_xe(gt)->needs_flr_on_fini = true; |
690 | ||
dd08ebf6 MB |
691 | return err; |
692 | } | |
693 | ||
694 | static void gt_reset_worker(struct work_struct *w) | |
695 | { | |
696 | struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker); | |
697 | ||
698 | gt_reset(gt); | |
699 | } | |
700 | ||
701 | void xe_gt_reset_async(struct xe_gt *gt) | |
702 | { | |
3e535bd5 | 703 | xe_gt_info(gt, "trying reset\n"); |
dd08ebf6 MB |
704 | |
705 | /* Don't do a reset while one is already in flight */ | |
8f3013e0 | 706 | if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(>->uc)) |
dd08ebf6 MB |
707 | return; |
708 | ||
3e535bd5 | 709 | xe_gt_info(gt, "reset queued\n"); |
dd08ebf6 MB |
710 | queue_work(gt->ordered_wq, >->reset.worker); |
711 | } | |
712 | ||
713 | void xe_gt_suspend_prepare(struct xe_gt *gt) | |
714 | { | |
715 | xe_device_mem_access_get(gt_to_xe(gt)); | |
716 | XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
717 | ||
718 | xe_uc_stop_prepare(>->uc); | |
719 | ||
dd08ebf6 | 720 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); |
03af26c9 | 721 | xe_device_mem_access_put(gt_to_xe(gt)); |
dd08ebf6 MB |
722 | } |
723 | ||
724 | int xe_gt_suspend(struct xe_gt *gt) | |
725 | { | |
dd08ebf6 MB |
726 | int err; |
727 | ||
da3799c9 MB |
728 | xe_gt_sanitize(gt); |
729 | ||
dd08ebf6 MB |
730 | xe_device_mem_access_get(gt_to_xe(gt)); |
731 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
732 | if (err) | |
733 | goto err_msg; | |
734 | ||
735 | err = xe_uc_suspend(>->uc); | |
736 | if (err) | |
737 | goto err_force_wake; | |
738 | ||
dd08ebf6 | 739 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); |
03af26c9 | 740 | xe_device_mem_access_put(gt_to_xe(gt)); |
3e535bd5 | 741 | xe_gt_info(gt, "suspended\n"); |
dd08ebf6 MB |
742 | |
743 | return 0; | |
744 | ||
745 | err_force_wake: | |
746 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
747 | err_msg: | |
748 | xe_device_mem_access_put(gt_to_xe(gt)); | |
3e535bd5 | 749 | xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err)); |
dd08ebf6 MB |
750 | |
751 | return err; | |
752 | } | |
753 | ||
754 | int xe_gt_resume(struct xe_gt *gt) | |
755 | { | |
dd08ebf6 MB |
756 | int err; |
757 | ||
758 | xe_device_mem_access_get(gt_to_xe(gt)); | |
759 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
760 | if (err) | |
761 | goto err_msg; | |
762 | ||
763 | err = do_gt_restart(gt); | |
764 | if (err) | |
765 | goto err_force_wake; | |
766 | ||
dd08ebf6 | 767 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); |
03af26c9 | 768 | xe_device_mem_access_put(gt_to_xe(gt)); |
3e535bd5 | 769 | xe_gt_info(gt, "resumed\n"); |
dd08ebf6 MB |
770 | |
771 | return 0; | |
772 | ||
773 | err_force_wake: | |
774 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
775 | err_msg: | |
776 | xe_device_mem_access_put(gt_to_xe(gt)); | |
3e535bd5 | 777 | xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err)); |
dd08ebf6 MB |
778 | |
779 | return err; | |
780 | } | |
781 | ||
dd08ebf6 MB |
782 | struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, |
783 | enum xe_engine_class class, | |
784 | u16 instance, bool logical) | |
785 | { | |
786 | struct xe_hw_engine *hwe; | |
787 | enum xe_hw_engine_id id; | |
788 | ||
789 | for_each_hw_engine(hwe, gt, id) | |
790 | if (hwe->class == class && | |
791 | ((!logical && hwe->instance == instance) || | |
792 | (logical && hwe->logical_instance == instance))) | |
793 | return hwe; | |
794 | ||
795 | return NULL; | |
796 | } | |
797 | ||
798 | struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, | |
799 | enum xe_engine_class class) | |
800 | { | |
801 | struct xe_hw_engine *hwe; | |
802 | enum xe_hw_engine_id id; | |
803 | ||
804 | for_each_hw_engine(hwe, gt, id) { | |
805 | switch (class) { | |
806 | case XE_ENGINE_CLASS_RENDER: | |
807 | case XE_ENGINE_CLASS_COMPUTE: | |
808 | if (hwe->class == XE_ENGINE_CLASS_RENDER || | |
809 | hwe->class == XE_ENGINE_CLASS_COMPUTE) | |
810 | return hwe; | |
811 | break; | |
812 | default: | |
813 | if (hwe->class == class) | |
814 | return hwe; | |
815 | } | |
816 | } | |
817 | ||
818 | return NULL; | |
819 | } |