Commit | Line | Data |
---|---|---|
dd08ebf6 MB |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2022 Intel Corporation | |
4 | */ | |
5 | ||
ea9f879d LDM |
6 | #include "xe_gt.h" |
7 | ||
dd08ebf6 MB |
8 | #include <linux/minmax.h> |
9 | ||
10 | #include <drm/drm_managed.h> | |
4f027e30 | 11 | #include <drm/xe_drm.h> |
dd08ebf6 | 12 | |
72ac3047 | 13 | #include "instructions/xe_gfxpipe_commands.h" |
0134f130 | 14 | #include "instructions/xe_mi_commands.h" |
226bfec8 | 15 | #include "regs/xe_gt_regs.h" |
c73acc1e | 16 | #include "xe_assert.h" |
dd08ebf6 MB |
17 | #include "xe_bb.h" |
18 | #include "xe_bo.h" | |
19 | #include "xe_device.h" | |
c22a4ed0 | 20 | #include "xe_exec_queue.h" |
dd08ebf6 MB |
21 | #include "xe_execlist.h" |
22 | #include "xe_force_wake.h" | |
23 | #include "xe_ggtt.h" | |
aae84bf1 | 24 | #include "xe_gsc.h" |
0d97ecce | 25 | #include "xe_gt_ccs_mode.h" |
dd08ebf6 | 26 | #include "xe_gt_clock.h" |
bef52b5c | 27 | #include "xe_gt_freq.h" |
c550f64f | 28 | #include "xe_gt_idle.h" |
dd08ebf6 MB |
29 | #include "xe_gt_mcr.h" |
30 | #include "xe_gt_pagefault.h" | |
3e535bd5 | 31 | #include "xe_gt_printk.h" |
dd08ebf6 | 32 | #include "xe_gt_sysfs.h" |
a9351846 | 33 | #include "xe_gt_tlb_invalidation.h" |
dd08ebf6 | 34 | #include "xe_gt_topology.h" |
9b9529ce | 35 | #include "xe_guc_exec_queue_types.h" |
43efd3ba | 36 | #include "xe_guc_pc.h" |
dd08ebf6 | 37 | #include "xe_hw_fence.h" |
038ff941 | 38 | #include "xe_hw_engine_class_sysfs.h" |
dd08ebf6 | 39 | #include "xe_irq.h" |
b1d20405 | 40 | #include "xe_lmtt.h" |
dd08ebf6 MB |
41 | #include "xe_lrc.h" |
42 | #include "xe_map.h" | |
43 | #include "xe_migrate.h" | |
44 | #include "xe_mmio.h" | |
576c6380 | 45 | #include "xe_pat.h" |
dd08ebf6 MB |
46 | #include "xe_mocs.h" |
47 | #include "xe_reg_sr.h" | |
48 | #include "xe_ring_ops.h" | |
49 | #include "xe_sa.h" | |
50 | #include "xe_sched_job.h" | |
b1d20405 | 51 | #include "xe_sriov.h" |
dd08ebf6 MB |
52 | #include "xe_tuning.h" |
53 | #include "xe_uc.h" | |
54 | #include "xe_vm.h" | |
55 | #include "xe_wa.h" | |
56 | #include "xe_wopcm.h" | |
57 | ||
f6929e80 | 58 | struct xe_gt *xe_gt_alloc(struct xe_tile *tile) |
dd08ebf6 | 59 | { |
f6929e80 | 60 | struct xe_gt *gt; |
dd08ebf6 | 61 | |
f6929e80 MR |
62 | gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL); |
63 | if (!gt) | |
64 | return ERR_PTR(-ENOMEM); | |
65 | ||
66 | gt->tile = tile; | |
dd08ebf6 MB |
67 | gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0); |
68 | ||
f6929e80 | 69 | return gt; |
dd08ebf6 MB |
70 | } |
71 | ||
da3799c9 MB |
72 | void xe_gt_sanitize(struct xe_gt *gt) |
73 | { | |
74 | /* | |
75 | * FIXME: if xe_uc_sanitize is called here, on TGL driver will not | |
76 | * reload | |
77 | */ | |
78 | gt->uc.guc.submission_state.enabled = false; | |
79 | } | |
80 | ||
997a55ca DCS |
81 | /** |
82 | * xe_gt_remove() - Clean up the GT structures before driver removal | |
83 | * @gt: the GT object | |
84 | * | |
85 | * This function should only act on objects/structures that must be cleaned | |
86 | * before the driver removal callback is complete and therefore can't be | |
87 | * deferred to a drmm action. | |
88 | */ | |
89 | void xe_gt_remove(struct xe_gt *gt) | |
90 | { | |
91 | xe_uc_remove(>->uc); | |
92 | } | |
93 | ||
dd08ebf6 MB |
94 | static void gt_fini(struct drm_device *drm, void *arg) |
95 | { | |
96 | struct xe_gt *gt = arg; | |
97 | int i; | |
98 | ||
99 | destroy_workqueue(gt->ordered_wq); | |
100 | ||
101 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) | |
102 | xe_hw_fence_irq_finish(>->fence_irq[i]); | |
103 | } | |
104 | ||
105 | static void gt_reset_worker(struct work_struct *w); | |
106 | ||
9b9529ce | 107 | static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) |
dd08ebf6 MB |
108 | { |
109 | struct xe_sched_job *job; | |
110 | struct xe_bb *bb; | |
111 | struct dma_fence *fence; | |
dd08ebf6 MB |
112 | long timeout; |
113 | ||
114 | bb = xe_bb_new(gt, 4, false); | |
115 | if (IS_ERR(bb)) | |
116 | return PTR_ERR(bb); | |
117 | ||
9e952635 | 118 | job = xe_bb_create_job(q, bb); |
dd08ebf6 MB |
119 | if (IS_ERR(job)) { |
120 | xe_bb_free(bb, NULL); | |
99c5952f | 121 | return PTR_ERR(job); |
dd08ebf6 MB |
122 | } |
123 | ||
124 | xe_sched_job_arm(job); | |
125 | fence = dma_fence_get(&job->drm.s_fence->finished); | |
126 | xe_sched_job_push(job); | |
127 | ||
128 | timeout = dma_fence_wait_timeout(fence, false, HZ); | |
129 | dma_fence_put(fence); | |
130 | xe_bb_free(bb, NULL); | |
131 | if (timeout < 0) | |
132 | return timeout; | |
133 | else if (!timeout) | |
134 | return -ETIME; | |
135 | ||
136 | return 0; | |
137 | } | |
138 | ||
278c3582 LDM |
139 | /* |
140 | * Convert back from encoded value to type-safe, only to be used when reg.mcr | |
141 | * is true | |
142 | */ | |
143 | static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg) | |
144 | { | |
145 | return (const struct xe_reg_mcr){.__reg.raw = reg.raw }; | |
146 | } | |
147 | ||
9b9529ce | 148 | static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) |
dd08ebf6 | 149 | { |
9b9529ce | 150 | struct xe_reg_sr *sr = &q->hwe->reg_lrc; |
dd08ebf6 | 151 | struct xe_reg_sr_entry *entry; |
278c3582 | 152 | unsigned long idx; |
dd08ebf6 MB |
153 | struct xe_sched_job *job; |
154 | struct xe_bb *bb; | |
155 | struct dma_fence *fence; | |
dd08ebf6 MB |
156 | long timeout; |
157 | int count = 0; | |
158 | ||
b1543a49 MR |
159 | if (q->hwe->class == XE_ENGINE_CLASS_RENDER) |
160 | /* Big enough to emit all of the context's 3DSTATE */ | |
161 | bb = xe_bb_new(gt, xe_lrc_size(gt_to_xe(gt), q->hwe->class), false); | |
162 | else | |
163 | /* Just pick a large BB size */ | |
164 | bb = xe_bb_new(gt, SZ_4K, false); | |
165 | ||
dd08ebf6 MB |
166 | if (IS_ERR(bb)) |
167 | return PTR_ERR(bb); | |
168 | ||
278c3582 | 169 | xa_for_each(&sr->xa, idx, entry) |
dd08ebf6 MB |
170 | ++count; |
171 | ||
172 | if (count) { | |
12a66a47 LDM |
173 | xe_gt_dbg(gt, "LRC WA %s save-restore batch\n", sr->name); |
174 | ||
e12a6488 | 175 | bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count); |
278c3582 LDM |
176 | |
177 | xa_for_each(&sr->xa, idx, entry) { | |
178 | struct xe_reg reg = entry->reg; | |
179 | struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg); | |
180 | u32 val; | |
181 | ||
182 | /* | |
183 | * Skip reading the register if it's not really needed | |
184 | */ | |
185 | if (reg.masked) | |
186 | val = entry->clr_bits << 16; | |
187 | else if (entry->clr_bits + 1) | |
188 | val = (reg.mcr ? | |
189 | xe_gt_mcr_unicast_read_any(gt, reg_mcr) : | |
190 | xe_mmio_read32(gt, reg)) & (~entry->clr_bits); | |
191 | else | |
192 | val = 0; | |
193 | ||
194 | val |= entry->set_bits; | |
195 | ||
196 | bb->cs[bb->len++] = reg.addr; | |
197 | bb->cs[bb->len++] = val; | |
198 | xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val); | |
dd08ebf6 MB |
199 | } |
200 | } | |
dd08ebf6 | 201 | |
b1543a49 MR |
202 | xe_lrc_emit_hwe_state_instructions(q, bb); |
203 | ||
9e952635 | 204 | job = xe_bb_create_job(q, bb); |
dd08ebf6 MB |
205 | if (IS_ERR(job)) { |
206 | xe_bb_free(bb, NULL); | |
99c5952f | 207 | return PTR_ERR(job); |
dd08ebf6 MB |
208 | } |
209 | ||
210 | xe_sched_job_arm(job); | |
211 | fence = dma_fence_get(&job->drm.s_fence->finished); | |
212 | xe_sched_job_push(job); | |
213 | ||
214 | timeout = dma_fence_wait_timeout(fence, false, HZ); | |
215 | dma_fence_put(fence); | |
216 | xe_bb_free(bb, NULL); | |
217 | if (timeout < 0) | |
218 | return timeout; | |
219 | else if (!timeout) | |
220 | return -ETIME; | |
221 | ||
222 | return 0; | |
223 | } | |
224 | ||
225 | int xe_gt_record_default_lrcs(struct xe_gt *gt) | |
226 | { | |
227 | struct xe_device *xe = gt_to_xe(gt); | |
228 | struct xe_hw_engine *hwe; | |
229 | enum xe_hw_engine_id id; | |
230 | int err = 0; | |
231 | ||
232 | for_each_hw_engine(hwe, gt, id) { | |
9b9529ce | 233 | struct xe_exec_queue *q, *nop_q; |
dd08ebf6 MB |
234 | void *default_lrc; |
235 | ||
236 | if (gt->default_lrc[hwe->class]) | |
237 | continue; | |
238 | ||
766849c4 | 239 | xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); |
dd08ebf6 | 240 | xe_wa_process_lrc(hwe); |
bb95a4f9 | 241 | xe_hw_engine_setup_default_lrc_state(hwe); |
3dbec470 | 242 | xe_tuning_process_lrc(hwe); |
dd08ebf6 MB |
243 | |
244 | default_lrc = drmm_kzalloc(&xe->drm, | |
245 | xe_lrc_size(xe, hwe->class), | |
246 | GFP_KERNEL); | |
247 | if (!default_lrc) | |
248 | return -ENOMEM; | |
249 | ||
9e952635 | 250 | q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1, |
25ce7c50 | 251 | hwe, EXEC_QUEUE_FLAG_KERNEL, 0); |
9b9529ce FD |
252 | if (IS_ERR(q)) { |
253 | err = PTR_ERR(q); | |
254 | xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n", | |
255 | hwe->name, q); | |
9e952635 | 256 | return err; |
dd08ebf6 MB |
257 | } |
258 | ||
259 | /* Prime golden LRC with known good state */ | |
9b9529ce | 260 | err = emit_wa_job(gt, q); |
3d4451d3 | 261 | if (err) { |
3e535bd5 | 262 | xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n", |
9b9529ce FD |
263 | hwe->name, ERR_PTR(err), q->guc->id); |
264 | goto put_exec_queue; | |
3d4451d3 | 265 | } |
dd08ebf6 | 266 | |
9e952635 | 267 | nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), |
25ce7c50 | 268 | 1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0); |
9b9529ce FD |
269 | if (IS_ERR(nop_q)) { |
270 | err = PTR_ERR(nop_q); | |
271 | xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n", | |
272 | hwe->name, nop_q); | |
273 | goto put_exec_queue; | |
dd08ebf6 MB |
274 | } |
275 | ||
276 | /* Switch to different LRC */ | |
9b9529ce | 277 | err = emit_nop_job(gt, nop_q); |
3d4451d3 | 278 | if (err) { |
3e535bd5 | 279 | xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n", |
9b9529ce FD |
280 | hwe->name, ERR_PTR(err), nop_q->guc->id); |
281 | goto put_nop_q; | |
3d4451d3 | 282 | } |
dd08ebf6 MB |
283 | |
284 | /* Reload golden LRC to record the effect of any indirect W/A */ | |
9b9529ce | 285 | err = emit_nop_job(gt, q); |
3d4451d3 | 286 | if (err) { |
3e535bd5 | 287 | xe_gt_err(gt, "hwe %s: emit_nop_job failed (%pe) guc_id=%u\n", |
9b9529ce FD |
288 | hwe->name, ERR_PTR(err), q->guc->id); |
289 | goto put_nop_q; | |
3d4451d3 | 290 | } |
dd08ebf6 MB |
291 | |
292 | xe_map_memcpy_from(xe, default_lrc, | |
9b9529ce FD |
293 | &q->lrc[0].bo->vmap, |
294 | xe_lrc_pphwsp_offset(&q->lrc[0]), | |
dd08ebf6 MB |
295 | xe_lrc_size(xe, hwe->class)); |
296 | ||
297 | gt->default_lrc[hwe->class] = default_lrc; | |
9b9529ce FD |
298 | put_nop_q: |
299 | xe_exec_queue_put(nop_q); | |
300 | put_exec_queue: | |
301 | xe_exec_queue_put(q); | |
dd08ebf6 MB |
302 | if (err) |
303 | break; | |
304 | } | |
305 | ||
306 | return err; | |
307 | } | |
308 | ||
309 | int xe_gt_init_early(struct xe_gt *gt) | |
310 | { | |
311 | int err; | |
312 | ||
dd08ebf6 MB |
313 | err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); |
314 | if (err) | |
315 | return err; | |
316 | ||
317 | xe_gt_topology_init(gt); | |
318 | xe_gt_mcr_init(gt); | |
319 | ||
320 | err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); | |
321 | if (err) | |
322 | return err; | |
323 | ||
324 | xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt)); | |
49d329a0 LDM |
325 | |
326 | err = xe_wa_init(gt); | |
327 | if (err) | |
328 | return err; | |
329 | ||
dd08ebf6 | 330 | xe_wa_process_gt(gt); |
9616e74b | 331 | xe_wa_process_oob(gt); |
dd08ebf6 MB |
332 | xe_tuning_process_gt(gt); |
333 | ||
334 | return 0; | |
335 | } | |
336 | ||
d2300987 LDM |
337 | static void dump_pat_on_error(struct xe_gt *gt) |
338 | { | |
339 | struct drm_printer p; | |
340 | char prefix[32]; | |
341 | ||
342 | snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id); | |
343 | p = drm_debug_printer(prefix); | |
344 | ||
345 | xe_pat_dump(gt, &p); | |
346 | } | |
347 | ||
dd08ebf6 MB |
348 | static int gt_fw_domain_init(struct xe_gt *gt) |
349 | { | |
350 | int err, i; | |
351 | ||
352 | xe_device_mem_access_get(gt_to_xe(gt)); | |
353 | err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); | |
354 | if (err) | |
355 | goto err_hw_fence_irq; | |
356 | ||
576c6380 | 357 | xe_pat_init(gt); |
6c8c1e74 | 358 | |
dd08ebf6 | 359 | if (!xe_gt_is_media_type(gt)) { |
ad703e06 | 360 | err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt); |
dd08ebf6 MB |
361 | if (err) |
362 | goto err_force_wake; | |
b1d20405 MW |
363 | if (IS_SRIOV_PF(gt_to_xe(gt))) |
364 | xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt); | |
dd08ebf6 MB |
365 | } |
366 | ||
dd08ebf6 | 367 | err = xe_uc_init(>->uc); |
33de290b CB |
368 | if (err) |
369 | goto err_force_wake; | |
dd08ebf6 | 370 | |
43efd3ba VB |
371 | /* Raise GT freq to speed up HuC/GuC load */ |
372 | xe_guc_pc_init_early(>->uc.guc.pc); | |
373 | ||
dd08ebf6 MB |
374 | err = xe_uc_init_hwconfig(>->uc); |
375 | if (err) | |
376 | goto err_force_wake; | |
377 | ||
1c2097bb RT |
378 | xe_gt_idle_sysfs_init(>->gtidle); |
379 | ||
da34c2cf MB |
380 | /* XXX: Fake that we pull the engine mask from hwconfig blob */ |
381 | gt->info.engine_mask = gt->info.__engine_mask; | |
382 | ||
3e29c149 MR |
383 | /* Enable per hw engine IRQs */ |
384 | xe_irq_enable_hwe(gt); | |
dd08ebf6 MB |
385 | |
386 | /* Rerun MCR init as we now have hw engine list */ | |
387 | xe_gt_mcr_init(gt); | |
388 | ||
389 | err = xe_hw_engines_init_early(gt); | |
390 | if (err) | |
391 | goto err_force_wake; | |
392 | ||
038ff941 TU |
393 | err = xe_hw_engine_class_sysfs_init(gt); |
394 | if (err) | |
395 | drm_warn(>_to_xe(gt)->drm, | |
396 | "failed to register engines sysfs directory, err: %d\n", | |
397 | err); | |
398 | ||
f3bc5bb4 NV |
399 | /* Initialize CCS mode sysfs after early initialization of HW engines */ |
400 | xe_gt_ccs_mode_sysfs_init(gt); | |
401 | ||
dd08ebf6 MB |
402 | err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); |
403 | XE_WARN_ON(err); | |
404 | xe_device_mem_access_put(gt_to_xe(gt)); | |
405 | ||
406 | return 0; | |
407 | ||
408 | err_force_wake: | |
d2300987 | 409 | dump_pat_on_error(gt); |
dd08ebf6 MB |
410 | xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); |
411 | err_hw_fence_irq: | |
412 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) | |
413 | xe_hw_fence_irq_finish(>->fence_irq[i]); | |
414 | xe_device_mem_access_put(gt_to_xe(gt)); | |
415 | ||
416 | return err; | |
417 | } | |
418 | ||
419 | static int all_fw_domain_init(struct xe_gt *gt) | |
420 | { | |
421 | int err, i; | |
422 | ||
423 | xe_device_mem_access_get(gt_to_xe(gt)); | |
424 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
425 | if (err) | |
426 | goto err_hw_fence_irq; | |
427 | ||
564d64f8 | 428 | xe_gt_mcr_set_implicit_defaults(gt); |
dd08ebf6 MB |
429 | xe_reg_sr_apply_mmio(>->reg_sr, gt); |
430 | ||
431 | err = xe_gt_clock_init(gt); | |
432 | if (err) | |
433 | goto err_force_wake; | |
434 | ||
435 | xe_mocs_init(gt); | |
436 | err = xe_execlist_init(gt); | |
437 | if (err) | |
438 | goto err_force_wake; | |
439 | ||
440 | err = xe_hw_engines_init(gt); | |
441 | if (err) | |
442 | goto err_force_wake; | |
443 | ||
444 | err = xe_uc_init_post_hwconfig(>->uc); | |
445 | if (err) | |
446 | goto err_force_wake; | |
447 | ||
dd08ebf6 | 448 | if (!xe_gt_is_media_type(gt)) { |
dd08ebf6 MB |
449 | /* |
450 | * USM has its only SA pool to non-block behind user operations | |
451 | */ | |
5a92da34 | 452 | if (gt_to_xe(gt)->info.has_usm) { |
876611c2 | 453 | gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt), SZ_1M, 16); |
0a12a612 MR |
454 | if (IS_ERR(gt->usm.bb_pool)) { |
455 | err = PTR_ERR(gt->usm.bb_pool); | |
dd08ebf6 | 456 | goto err_force_wake; |
0a12a612 | 457 | } |
dd08ebf6 MB |
458 | } |
459 | } | |
460 | ||
461 | if (!xe_gt_is_media_type(gt)) { | |
08dea767 MR |
462 | struct xe_tile *tile = gt_to_tile(gt); |
463 | ||
464 | tile->migrate = xe_migrate_init(tile); | |
465 | if (IS_ERR(tile->migrate)) { | |
466 | err = PTR_ERR(tile->migrate); | |
dd08ebf6 | 467 | goto err_force_wake; |
99c5952f | 468 | } |
dd08ebf6 MB |
469 | } |
470 | ||
471 | err = xe_uc_init_hw(>->uc); | |
472 | if (err) | |
473 | goto err_force_wake; | |
474 | ||
0d97ecce NV |
475 | /* Configure default CCS mode of 1 engine with all resources */ |
476 | if (xe_gt_ccs_mode_enabled(gt)) { | |
477 | gt->ccs_mode = 1; | |
478 | xe_gt_apply_ccs_mode(gt); | |
479 | } | |
480 | ||
b1d20405 MW |
481 | if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt)) |
482 | xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); | |
483 | ||
dd08ebf6 MB |
484 | err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); |
485 | XE_WARN_ON(err); | |
486 | xe_device_mem_access_put(gt_to_xe(gt)); | |
487 | ||
488 | return 0; | |
489 | ||
490 | err_force_wake: | |
491 | xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
492 | err_hw_fence_irq: | |
493 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) | |
494 | xe_hw_fence_irq_finish(>->fence_irq[i]); | |
495 | xe_device_mem_access_put(gt_to_xe(gt)); | |
496 | ||
497 | return err; | |
498 | } | |
499 | ||
500 | int xe_gt_init(struct xe_gt *gt) | |
501 | { | |
502 | int err; | |
503 | int i; | |
504 | ||
505 | INIT_WORK(>->reset.worker, gt_reset_worker); | |
506 | ||
507 | for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) { | |
508 | gt->ring_ops[i] = xe_ring_ops_get(gt, i); | |
509 | xe_hw_fence_irq_init(>->fence_irq[i]); | |
510 | } | |
511 | ||
a9351846 MB |
512 | err = xe_gt_tlb_invalidation_init(gt); |
513 | if (err) | |
514 | return err; | |
515 | ||
dd08ebf6 MB |
516 | err = xe_gt_pagefault_init(gt); |
517 | if (err) | |
518 | return err; | |
519 | ||
17a6726c MR |
520 | xe_mocs_init_early(gt); |
521 | ||
dd08ebf6 MB |
522 | xe_gt_sysfs_init(gt); |
523 | ||
524 | err = gt_fw_domain_init(gt); | |
525 | if (err) | |
526 | return err; | |
527 | ||
bef52b5c RV |
528 | xe_gt_freq_init(gt); |
529 | ||
dd08ebf6 MB |
530 | xe_force_wake_init_engines(gt, gt_to_fw(gt)); |
531 | ||
532 | err = all_fw_domain_init(gt); | |
533 | if (err) | |
534 | return err; | |
535 | ||
dd08ebf6 MB |
536 | err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt); |
537 | if (err) | |
538 | return err; | |
539 | ||
540 | return 0; | |
541 | } | |
542 | ||
671ca05d | 543 | static int do_gt_reset(struct xe_gt *gt) |
dd08ebf6 | 544 | { |
dd08ebf6 MB |
545 | int err; |
546 | ||
aae84bf1 DCS |
547 | xe_gsc_wa_14015076503(gt, true); |
548 | ||
ce8bf5bd | 549 | xe_mmio_write32(gt, GDRST, GRDOM_FULL); |
063e09af | 550 | err = xe_mmio_wait32(gt, GDRST, GRDOM_FULL, 0, 5000, NULL, false); |
dd08ebf6 | 551 | if (err) |
0bc519d2 | 552 | xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n", |
3e535bd5 | 553 | ERR_PTR(err)); |
dd08ebf6 | 554 | |
aae84bf1 DCS |
555 | xe_gsc_wa_14015076503(gt, false); |
556 | ||
dd08ebf6 MB |
557 | return err; |
558 | } | |
559 | ||
560 | static int do_gt_restart(struct xe_gt *gt) | |
561 | { | |
562 | struct xe_hw_engine *hwe; | |
563 | enum xe_hw_engine_id id; | |
564 | int err; | |
565 | ||
576c6380 | 566 | xe_pat_init(gt); |
dd08ebf6 | 567 | |
564d64f8 | 568 | xe_gt_mcr_set_implicit_defaults(gt); |
dd08ebf6 MB |
569 | xe_reg_sr_apply_mmio(>->reg_sr, gt); |
570 | ||
571 | err = xe_wopcm_init(>->uc.wopcm); | |
572 | if (err) | |
573 | return err; | |
574 | ||
575 | for_each_hw_engine(hwe, gt, id) | |
576 | xe_hw_engine_enable_ring(hwe); | |
577 | ||
7704f32c MW |
578 | err = xe_uc_sanitize_reset(>->uc); |
579 | if (err) | |
580 | return err; | |
581 | ||
dd08ebf6 MB |
582 | err = xe_uc_init_hw(>->uc); |
583 | if (err) | |
584 | return err; | |
585 | ||
b1d20405 MW |
586 | if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt)) |
587 | xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt); | |
588 | ||
dd08ebf6 MB |
589 | xe_mocs_init(gt); |
590 | err = xe_uc_start(>->uc); | |
591 | if (err) | |
592 | return err; | |
593 | ||
594 | for_each_hw_engine(hwe, gt, id) { | |
595 | xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); | |
1011812c | 596 | xe_reg_sr_apply_whitelist(hwe); |
dd08ebf6 MB |
597 | } |
598 | ||
0d97ecce NV |
599 | /* Get CCS mode in sync between sw/hw */ |
600 | xe_gt_apply_ccs_mode(gt); | |
601 | ||
dd08ebf6 MB |
602 | return 0; |
603 | } | |
604 | ||
605 | static int gt_reset(struct xe_gt *gt) | |
606 | { | |
dd08ebf6 MB |
607 | int err; |
608 | ||
609 | /* We only support GT resets with GuC submission */ | |
c4991ee0 | 610 | if (!xe_device_uc_enabled(gt_to_xe(gt))) |
dd08ebf6 MB |
611 | return -ENODEV; |
612 | ||
3e535bd5 | 613 | xe_gt_info(gt, "reset started\n"); |
dd08ebf6 | 614 | |
8f3013e0 HPG |
615 | if (xe_fault_inject_gt_reset()) { |
616 | err = -ECANCELED; | |
617 | goto err_fail; | |
618 | } | |
619 | ||
da3799c9 MB |
620 | xe_gt_sanitize(gt); |
621 | ||
dd08ebf6 MB |
622 | xe_device_mem_access_get(gt_to_xe(gt)); |
623 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
624 | if (err) | |
625 | goto err_msg; | |
626 | ||
1737785a | 627 | xe_uc_gucrc_disable(>->uc); |
dd08ebf6 MB |
628 | xe_uc_stop_prepare(>->uc); |
629 | xe_gt_pagefault_reset(gt); | |
630 | ||
631 | err = xe_uc_stop(>->uc); | |
632 | if (err) | |
633 | goto err_out; | |
634 | ||
635 | err = do_gt_reset(gt); | |
636 | if (err) | |
637 | goto err_out; | |
638 | ||
7b24cc3e MA |
639 | xe_gt_tlb_invalidation_reset(gt); |
640 | ||
dd08ebf6 MB |
641 | err = do_gt_restart(gt); |
642 | if (err) | |
643 | goto err_out; | |
644 | ||
dd08ebf6 | 645 | err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); |
03af26c9 | 646 | xe_device_mem_access_put(gt_to_xe(gt)); |
dd08ebf6 MB |
647 | XE_WARN_ON(err); |
648 | ||
3e535bd5 | 649 | xe_gt_info(gt, "reset done\n"); |
dd08ebf6 MB |
650 | |
651 | return 0; | |
652 | ||
653 | err_out: | |
654 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
655 | err_msg: | |
656 | XE_WARN_ON(xe_uc_start(>->uc)); | |
657 | xe_device_mem_access_put(gt_to_xe(gt)); | |
8f3013e0 | 658 | err_fail: |
3e535bd5 | 659 | xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err)); |
dd08ebf6 | 660 | |
57162274 AH |
661 | gt_to_xe(gt)->needs_flr_on_fini = true; |
662 | ||
dd08ebf6 MB |
663 | return err; |
664 | } | |
665 | ||
666 | static void gt_reset_worker(struct work_struct *w) | |
667 | { | |
668 | struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker); | |
669 | ||
670 | gt_reset(gt); | |
671 | } | |
672 | ||
673 | void xe_gt_reset_async(struct xe_gt *gt) | |
674 | { | |
3e535bd5 | 675 | xe_gt_info(gt, "trying reset\n"); |
dd08ebf6 MB |
676 | |
677 | /* Don't do a reset while one is already in flight */ | |
8f3013e0 | 678 | if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(>->uc)) |
dd08ebf6 MB |
679 | return; |
680 | ||
3e535bd5 | 681 | xe_gt_info(gt, "reset queued\n"); |
dd08ebf6 MB |
682 | queue_work(gt->ordered_wq, >->reset.worker); |
683 | } | |
684 | ||
685 | void xe_gt_suspend_prepare(struct xe_gt *gt) | |
686 | { | |
687 | xe_device_mem_access_get(gt_to_xe(gt)); | |
688 | XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
689 | ||
690 | xe_uc_stop_prepare(>->uc); | |
691 | ||
dd08ebf6 | 692 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); |
03af26c9 | 693 | xe_device_mem_access_put(gt_to_xe(gt)); |
dd08ebf6 MB |
694 | } |
695 | ||
696 | int xe_gt_suspend(struct xe_gt *gt) | |
697 | { | |
dd08ebf6 MB |
698 | int err; |
699 | ||
da3799c9 MB |
700 | xe_gt_sanitize(gt); |
701 | ||
dd08ebf6 MB |
702 | xe_device_mem_access_get(gt_to_xe(gt)); |
703 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
704 | if (err) | |
705 | goto err_msg; | |
706 | ||
707 | err = xe_uc_suspend(>->uc); | |
708 | if (err) | |
709 | goto err_force_wake; | |
710 | ||
dd08ebf6 | 711 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); |
03af26c9 | 712 | xe_device_mem_access_put(gt_to_xe(gt)); |
3e535bd5 | 713 | xe_gt_info(gt, "suspended\n"); |
dd08ebf6 MB |
714 | |
715 | return 0; | |
716 | ||
717 | err_force_wake: | |
718 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
719 | err_msg: | |
720 | xe_device_mem_access_put(gt_to_xe(gt)); | |
3e535bd5 | 721 | xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err)); |
dd08ebf6 MB |
722 | |
723 | return err; | |
724 | } | |
725 | ||
726 | int xe_gt_resume(struct xe_gt *gt) | |
727 | { | |
dd08ebf6 MB |
728 | int err; |
729 | ||
730 | xe_device_mem_access_get(gt_to_xe(gt)); | |
731 | err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); | |
732 | if (err) | |
733 | goto err_msg; | |
734 | ||
735 | err = do_gt_restart(gt); | |
736 | if (err) | |
737 | goto err_force_wake; | |
738 | ||
dd08ebf6 | 739 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); |
03af26c9 | 740 | xe_device_mem_access_put(gt_to_xe(gt)); |
3e535bd5 | 741 | xe_gt_info(gt, "resumed\n"); |
dd08ebf6 MB |
742 | |
743 | return 0; | |
744 | ||
745 | err_force_wake: | |
746 | XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); | |
747 | err_msg: | |
748 | xe_device_mem_access_put(gt_to_xe(gt)); | |
3e535bd5 | 749 | xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err)); |
dd08ebf6 MB |
750 | |
751 | return err; | |
752 | } | |
753 | ||
dd08ebf6 MB |
754 | struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, |
755 | enum xe_engine_class class, | |
756 | u16 instance, bool logical) | |
757 | { | |
758 | struct xe_hw_engine *hwe; | |
759 | enum xe_hw_engine_id id; | |
760 | ||
761 | for_each_hw_engine(hwe, gt, id) | |
762 | if (hwe->class == class && | |
763 | ((!logical && hwe->instance == instance) || | |
764 | (logical && hwe->logical_instance == instance))) | |
765 | return hwe; | |
766 | ||
767 | return NULL; | |
768 | } | |
769 | ||
770 | struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, | |
771 | enum xe_engine_class class) | |
772 | { | |
773 | struct xe_hw_engine *hwe; | |
774 | enum xe_hw_engine_id id; | |
775 | ||
776 | for_each_hw_engine(hwe, gt, id) { | |
777 | switch (class) { | |
778 | case XE_ENGINE_CLASS_RENDER: | |
779 | case XE_ENGINE_CLASS_COMPUTE: | |
780 | if (hwe->class == XE_ENGINE_CLASS_RENDER || | |
781 | hwe->class == XE_ENGINE_CLASS_COMPUTE) | |
782 | return hwe; | |
783 | break; | |
784 | default: | |
785 | if (hwe->class == class) | |
786 | return hwe; | |
787 | } | |
788 | } | |
789 | ||
790 | return NULL; | |
791 | } |