Commit | Line | Data |
---|---|---|
24635c51 TU |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2019 Intel Corporation | |
4 | */ | |
5 | ||
9dd4b065 | 6 | #include "debugfs_gt.h" |
724e9564 | 7 | #include "i915_drv.h" |
e26b6d43 | 8 | #include "intel_context.h" |
24635c51 | 9 | #include "intel_gt.h" |
16e87459 | 10 | #include "intel_gt_buffer_pool.h" |
9c878557 | 11 | #include "intel_gt_clock_utils.h" |
99f2eb96 | 12 | #include "intel_gt_pm.h" |
66101975 | 13 | #include "intel_gt_requests.h" |
61fa60ff | 14 | #include "intel_mocs.h" |
c1132367 | 15 | #include "intel_rc6.h" |
e26b6d43 | 16 | #include "intel_renderstate.h" |
3e7abf81 | 17 | #include "intel_rps.h" |
eaf522f6 | 18 | #include "intel_uncore.h" |
42014f69 | 19 | #include "intel_pm.h" |
be1cb55a | 20 | #include "shmem_utils.h" |
24635c51 | 21 | |
724e9564 | 22 | void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) |
24635c51 | 23 | { |
724e9564 TU |
24 | gt->i915 = i915; |
25 | gt->uncore = &i915->uncore; | |
26 | ||
d762043f AS |
27 | spin_lock_init(>->irq_lock); |
28 | ||
d762043f | 29 | INIT_LIST_HEAD(>->closed_vma); |
24635c51 | 30 | spin_lock_init(>->closed_lock); |
99f2eb96 | 31 | |
16e87459 | 32 | intel_gt_init_buffer_pool(gt); |
cb823ed9 | 33 | intel_gt_init_reset(gt); |
66101975 | 34 | intel_gt_init_requests(gt); |
4605bb73 | 35 | intel_gt_init_timelines(gt); |
99f2eb96 | 36 | intel_gt_pm_init_early(gt); |
a06375a9 CW |
37 | |
38 | intel_rps_init_early(>->rps); | |
6f76098f | 39 | intel_uc_init_early(>->uc); |
24635c51 | 40 | } |
eaf522f6 | 41 | |
797a6153 | 42 | void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt) |
d8a44248 | 43 | { |
797a6153 | 44 | gt->ggtt = ggtt; |
d8a44248 TU |
45 | } |
46 | ||
d0eb6866 DCS |
47 | int intel_gt_init_mmio(struct intel_gt *gt) |
48 | { | |
f170523a CW |
49 | intel_gt_init_clock_frequency(gt); |
50 | ||
d0eb6866 | 51 | intel_uc_init_mmio(>->uc); |
9b413f01 | 52 | intel_sseu_info_init(gt); |
d0eb6866 DCS |
53 | |
54 | return intel_engines_init_mmio(gt); | |
55 | } | |
56 | ||
61fa60ff TU |
57 | static void init_unused_ring(struct intel_gt *gt, u32 base) |
58 | { | |
59 | struct intel_uncore *uncore = gt->uncore; | |
60 | ||
61 | intel_uncore_write(uncore, RING_CTL(base), 0); | |
62 | intel_uncore_write(uncore, RING_HEAD(base), 0); | |
63 | intel_uncore_write(uncore, RING_TAIL(base), 0); | |
64 | intel_uncore_write(uncore, RING_START(base), 0); | |
65 | } | |
66 | ||
67 | static void init_unused_rings(struct intel_gt *gt) | |
68 | { | |
69 | struct drm_i915_private *i915 = gt->i915; | |
70 | ||
71 | if (IS_I830(i915)) { | |
72 | init_unused_ring(gt, PRB1_BASE); | |
73 | init_unused_ring(gt, SRB0_BASE); | |
74 | init_unused_ring(gt, SRB1_BASE); | |
75 | init_unused_ring(gt, SRB2_BASE); | |
76 | init_unused_ring(gt, SRB3_BASE); | |
77 | } else if (IS_GEN(i915, 2)) { | |
78 | init_unused_ring(gt, SRB0_BASE); | |
79 | init_unused_ring(gt, SRB1_BASE); | |
80 | } else if (IS_GEN(i915, 3)) { | |
81 | init_unused_ring(gt, PRB1_BASE); | |
82 | init_unused_ring(gt, PRB2_BASE); | |
83 | } | |
84 | } | |
85 | ||
86 | int intel_gt_init_hw(struct intel_gt *gt) | |
87 | { | |
88 | struct drm_i915_private *i915 = gt->i915; | |
89 | struct intel_uncore *uncore = gt->uncore; | |
90 | int ret; | |
91 | ||
61fa60ff TU |
92 | gt->last_init_time = ktime_get(); |
93 | ||
94 | /* Double layer security blanket, see i915_gem_init() */ | |
95 | intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); | |
96 | ||
97 | if (HAS_EDRAM(i915) && INTEL_GEN(i915) < 9) | |
98 | intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf)); | |
99 | ||
100 | if (IS_HASWELL(i915)) | |
101 | intel_uncore_write(uncore, | |
102 | MI_PREDICATE_RESULT_2, | |
103 | IS_HSW_GT3(i915) ? | |
104 | LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); | |
105 | ||
106 | /* Apply the GT workarounds... */ | |
107 | intel_gt_apply_workarounds(gt); | |
108 | /* ...and determine whether they are sticking. */ | |
109 | intel_gt_verify_workarounds(gt, "init"); | |
110 | ||
111 | intel_gt_init_swizzling(gt); | |
112 | ||
113 | /* | |
114 | * At least 830 can leave some of the unused rings | |
115 | * "active" (ie. head != tail) after resume which | |
116 | * will prevent c3 entry. Makes sure all unused rings | |
117 | * are totally idle. | |
118 | */ | |
119 | init_unused_rings(gt); | |
120 | ||
121 | ret = i915_ppgtt_init_hw(gt); | |
122 | if (ret) { | |
123 | DRM_ERROR("Enabling PPGTT failed (%d)\n", ret); | |
124 | goto out; | |
125 | } | |
126 | ||
127 | /* We can't enable contexts until all firmware is loaded */ | |
128 | ret = intel_uc_init_hw(>->uc); | |
129 | if (ret) { | |
130 | i915_probe_error(i915, "Enabling uc failed (%d)\n", ret); | |
131 | goto out; | |
132 | } | |
133 | ||
134 | intel_mocs_init(gt); | |
135 | ||
136 | out: | |
137 | intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); | |
138 | return ret; | |
139 | } | |
140 | ||
eaf522f6 TU |
141 | static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set) |
142 | { | |
143 | intel_uncore_rmw(uncore, reg, 0, set); | |
144 | } | |
145 | ||
146 | static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr) | |
147 | { | |
148 | intel_uncore_rmw(uncore, reg, clr, 0); | |
149 | } | |
150 | ||
151 | static void clear_register(struct intel_uncore *uncore, i915_reg_t reg) | |
152 | { | |
153 | intel_uncore_rmw(uncore, reg, 0, 0); | |
154 | } | |
155 | ||
156 | static void gen8_clear_engine_error_register(struct intel_engine_cs *engine) | |
157 | { | |
158 | GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0); | |
159 | GEN6_RING_FAULT_REG_POSTING_READ(engine); | |
160 | } | |
161 | ||
162 | void | |
163 | intel_gt_clear_error_registers(struct intel_gt *gt, | |
164 | intel_engine_mask_t engine_mask) | |
165 | { | |
166 | struct drm_i915_private *i915 = gt->i915; | |
167 | struct intel_uncore *uncore = gt->uncore; | |
168 | u32 eir; | |
169 | ||
170 | if (!IS_GEN(i915, 2)) | |
171 | clear_register(uncore, PGTBL_ER); | |
172 | ||
173 | if (INTEL_GEN(i915) < 4) | |
174 | clear_register(uncore, IPEIR(RENDER_RING_BASE)); | |
175 | else | |
176 | clear_register(uncore, IPEIR_I965); | |
177 | ||
178 | clear_register(uncore, EIR); | |
179 | eir = intel_uncore_read(uncore, EIR); | |
180 | if (eir) { | |
181 | /* | |
182 | * some errors might have become stuck, | |
183 | * mask them. | |
184 | */ | |
185 | DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir); | |
186 | rmw_set(uncore, EMR, eir); | |
187 | intel_uncore_write(uncore, GEN2_IIR, | |
188 | I915_MASTER_ERROR_INTERRUPT); | |
189 | } | |
190 | ||
91b59cd9 LDM |
191 | if (INTEL_GEN(i915) >= 12) { |
192 | rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID); | |
193 | intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG); | |
194 | } else if (INTEL_GEN(i915) >= 8) { | |
eaf522f6 TU |
195 | rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID); |
196 | intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG); | |
197 | } else if (INTEL_GEN(i915) >= 6) { | |
198 | struct intel_engine_cs *engine; | |
199 | enum intel_engine_id id; | |
200 | ||
a50134b1 | 201 | for_each_engine_masked(engine, gt, engine_mask, id) |
eaf522f6 TU |
202 | gen8_clear_engine_error_register(engine); |
203 | } | |
204 | } | |
205 | ||
206 | static void gen6_check_faults(struct intel_gt *gt) | |
207 | { | |
208 | struct intel_engine_cs *engine; | |
209 | enum intel_engine_id id; | |
210 | u32 fault; | |
211 | ||
5d904e3c | 212 | for_each_engine(engine, gt, id) { |
eaf522f6 TU |
213 | fault = GEN6_RING_FAULT_REG_READ(engine); |
214 | if (fault & RING_FAULT_VALID) { | |
1a6c83ef WK |
215 | drm_dbg(&engine->i915->drm, "Unexpected fault\n" |
216 | "\tAddr: 0x%08lx\n" | |
217 | "\tAddress space: %s\n" | |
218 | "\tSource ID: %d\n" | |
219 | "\tType: %d\n", | |
220 | fault & PAGE_MASK, | |
221 | fault & RING_FAULT_GTTSEL_MASK ? | |
222 | "GGTT" : "PPGTT", | |
223 | RING_FAULT_SRCID(fault), | |
224 | RING_FAULT_FAULT_TYPE(fault)); | |
eaf522f6 TU |
225 | } |
226 | } | |
227 | } | |
228 | ||
229 | static void gen8_check_faults(struct intel_gt *gt) | |
230 | { | |
231 | struct intel_uncore *uncore = gt->uncore; | |
91b59cd9 LDM |
232 | i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg; |
233 | u32 fault; | |
234 | ||
235 | if (INTEL_GEN(gt->i915) >= 12) { | |
236 | fault_reg = GEN12_RING_FAULT_REG; | |
237 | fault_data0_reg = GEN12_FAULT_TLB_DATA0; | |
238 | fault_data1_reg = GEN12_FAULT_TLB_DATA1; | |
239 | } else { | |
240 | fault_reg = GEN8_RING_FAULT_REG; | |
241 | fault_data0_reg = GEN8_FAULT_TLB_DATA0; | |
242 | fault_data1_reg = GEN8_FAULT_TLB_DATA1; | |
243 | } | |
eaf522f6 | 244 | |
91b59cd9 | 245 | fault = intel_uncore_read(uncore, fault_reg); |
eaf522f6 TU |
246 | if (fault & RING_FAULT_VALID) { |
247 | u32 fault_data0, fault_data1; | |
248 | u64 fault_addr; | |
249 | ||
91b59cd9 LDM |
250 | fault_data0 = intel_uncore_read(uncore, fault_data0_reg); |
251 | fault_data1 = intel_uncore_read(uncore, fault_data1_reg); | |
252 | ||
eaf522f6 TU |
253 | fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) | |
254 | ((u64)fault_data0 << 12); | |
255 | ||
1a6c83ef WK |
256 | drm_dbg(&uncore->i915->drm, "Unexpected fault\n" |
257 | "\tAddr: 0x%08x_%08x\n" | |
258 | "\tAddress space: %s\n" | |
259 | "\tEngine ID: %d\n" | |
260 | "\tSource ID: %d\n" | |
261 | "\tType: %d\n", | |
262 | upper_32_bits(fault_addr), lower_32_bits(fault_addr), | |
263 | fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT", | |
264 | GEN8_RING_FAULT_ENGINE_ID(fault), | |
265 | RING_FAULT_SRCID(fault), | |
266 | RING_FAULT_FAULT_TYPE(fault)); | |
eaf522f6 TU |
267 | } |
268 | } | |
269 | ||
270 | void intel_gt_check_and_clear_faults(struct intel_gt *gt) | |
271 | { | |
272 | struct drm_i915_private *i915 = gt->i915; | |
273 | ||
274 | /* From GEN8 onwards we only have one 'All Engine Fault Register' */ | |
275 | if (INTEL_GEN(i915) >= 8) | |
276 | gen8_check_faults(gt); | |
277 | else if (INTEL_GEN(i915) >= 6) | |
278 | gen6_check_faults(gt); | |
279 | else | |
280 | return; | |
281 | ||
282 | intel_gt_clear_error_registers(gt, ALL_ENGINES); | |
283 | } | |
a1c8a09e TU |
284 | |
285 | void intel_gt_flush_ggtt_writes(struct intel_gt *gt) | |
286 | { | |
cd6a8513 | 287 | struct intel_uncore *uncore = gt->uncore; |
a1c8a09e TU |
288 | intel_wakeref_t wakeref; |
289 | ||
290 | /* | |
291 | * No actual flushing is required for the GTT write domain for reads | |
292 | * from the GTT domain. Writes to it "immediately" go to main memory | |
293 | * as far as we know, so there's no chipset flush. It also doesn't | |
294 | * land in the GPU render cache. | |
295 | * | |
296 | * However, we do have to enforce the order so that all writes through | |
297 | * the GTT land before any writes to the device, such as updates to | |
298 | * the GATT itself. | |
299 | * | |
300 | * We also have to wait a bit for the writes to land from the GTT. | |
301 | * An uncached read (i.e. mmio) seems to be ideal for the round-trip | |
302 | * timing. This issue has only been observed when switching quickly | |
303 | * between GTT writes and CPU reads from inside the kernel on recent hw, | |
304 | * and it appears to only affect discrete GTT blocks (i.e. on LLC | |
305 | * system agents we cannot reproduce this behaviour, until Cannonlake | |
306 | * that was!). | |
307 | */ | |
308 | ||
309 | wmb(); | |
310 | ||
cd6a8513 | 311 | if (INTEL_INFO(gt->i915)->has_coherent_ggtt) |
a1c8a09e TU |
312 | return; |
313 | ||
baea429d | 314 | intel_gt_chipset_flush(gt); |
a1c8a09e | 315 | |
b6422694 | 316 | with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) { |
2850748e | 317 | unsigned long flags; |
a1c8a09e | 318 | |
2850748e | 319 | spin_lock_irqsave(&uncore->lock, flags); |
a1c8a09e TU |
320 | intel_uncore_posting_read_fw(uncore, |
321 | RING_HEAD(RENDER_RING_BASE)); | |
2850748e | 322 | spin_unlock_irqrestore(&uncore->lock, flags); |
a1c8a09e TU |
323 | } |
324 | } | |
baea429d TU |
325 | |
326 | void intel_gt_chipset_flush(struct intel_gt *gt) | |
327 | { | |
328 | wmb(); | |
329 | if (INTEL_GEN(gt->i915) < 6) | |
330 | intel_gtt_chipset_flush(); | |
331 | } | |
db56f974 | 332 | |
42014f69 AS |
333 | void intel_gt_driver_register(struct intel_gt *gt) |
334 | { | |
3e7abf81 | 335 | intel_rps_driver_register(>->rps); |
9dd4b065 AS |
336 | |
337 | debugfs_gt_register(gt); | |
42014f69 AS |
338 | } |
339 | ||
340 | static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) | |
db56f974 TU |
341 | { |
342 | struct drm_i915_private *i915 = gt->i915; | |
343 | struct drm_i915_gem_object *obj; | |
344 | struct i915_vma *vma; | |
345 | int ret; | |
346 | ||
347 | obj = i915_gem_object_create_stolen(i915, size); | |
0e5493ca | 348 | if (IS_ERR(obj)) |
db56f974 TU |
349 | obj = i915_gem_object_create_internal(i915, size); |
350 | if (IS_ERR(obj)) { | |
351 | DRM_ERROR("Failed to allocate scratch page\n"); | |
352 | return PTR_ERR(obj); | |
353 | } | |
354 | ||
355 | vma = i915_vma_instance(obj, >->ggtt->vm, NULL); | |
356 | if (IS_ERR(vma)) { | |
357 | ret = PTR_ERR(vma); | |
358 | goto err_unref; | |
359 | } | |
360 | ||
47b08693 | 361 | ret = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH); |
db56f974 TU |
362 | if (ret) |
363 | goto err_unref; | |
364 | ||
1aff1903 CW |
365 | gt->scratch = i915_vma_make_unshrinkable(vma); |
366 | ||
db56f974 TU |
367 | return 0; |
368 | ||
369 | err_unref: | |
370 | i915_gem_object_put(obj); | |
371 | return ret; | |
372 | } | |
373 | ||
42014f69 | 374 | static void intel_gt_fini_scratch(struct intel_gt *gt) |
db56f974 TU |
375 | { |
376 | i915_vma_unpin_and_release(>->scratch, 0); | |
377 | } | |
cb823ed9 | 378 | |
e6ba7648 CW |
379 | static struct i915_address_space *kernel_vm(struct intel_gt *gt) |
380 | { | |
381 | if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING) | |
2c86e55d | 382 | return &i915_ppgtt_create(gt)->vm; |
e6ba7648 CW |
383 | else |
384 | return i915_vm_get(>->ggtt->vm); | |
385 | } | |
386 | ||
e26b6d43 CW |
387 | static int __engines_record_defaults(struct intel_gt *gt) |
388 | { | |
389 | struct i915_request *requests[I915_NUM_ENGINES] = {}; | |
390 | struct intel_engine_cs *engine; | |
391 | enum intel_engine_id id; | |
392 | int err = 0; | |
393 | ||
394 | /* | |
395 | * As we reset the gpu during very early sanitisation, the current | |
396 | * register state on the GPU should reflect its defaults values. | |
397 | * We load a context onto the hw (with restore-inhibit), then switch | |
398 | * over to a second context to save that default register state. We | |
399 | * can then prime every new context with that state so they all start | |
400 | * from the same default HW values. | |
401 | */ | |
402 | ||
403 | for_each_engine(engine, gt, id) { | |
404 | struct intel_renderstate so; | |
405 | struct intel_context *ce; | |
406 | struct i915_request *rq; | |
407 | ||
c2d78a9b CW |
408 | /* We must be able to switch to something! */ |
409 | GEM_BUG_ON(!engine->kernel_context); | |
410 | ||
e26b6d43 CW |
411 | ce = intel_context_create(engine); |
412 | if (IS_ERR(ce)) { | |
413 | err = PTR_ERR(ce); | |
414 | goto out; | |
415 | } | |
416 | ||
bfdf8b1d ML |
417 | err = intel_renderstate_init(&so, ce); |
418 | if (err) | |
419 | goto err; | |
420 | ||
421 | rq = i915_request_create(ce); | |
e26b6d43 CW |
422 | if (IS_ERR(rq)) { |
423 | err = PTR_ERR(rq); | |
bfdf8b1d | 424 | goto err_fini; |
e26b6d43 CW |
425 | } |
426 | ||
427 | err = intel_engine_emit_ctx_wa(rq); | |
428 | if (err) | |
429 | goto err_rq; | |
430 | ||
431 | err = intel_renderstate_emit(&so, rq); | |
432 | if (err) | |
433 | goto err_rq; | |
434 | ||
435 | err_rq: | |
436 | requests[id] = i915_request_get(rq); | |
437 | i915_request_add(rq); | |
bfdf8b1d ML |
438 | err_fini: |
439 | intel_renderstate_fini(&so, ce); | |
440 | err: | |
441 | if (err) { | |
442 | intel_context_put(ce); | |
e26b6d43 | 443 | goto out; |
bfdf8b1d | 444 | } |
e26b6d43 CW |
445 | } |
446 | ||
447 | /* Flush the default context image to memory, and enable powersaving. */ | |
448 | if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { | |
449 | err = -EIO; | |
450 | goto out; | |
451 | } | |
452 | ||
453 | for (id = 0; id < ARRAY_SIZE(requests); id++) { | |
454 | struct i915_request *rq; | |
be1cb55a | 455 | struct file *state; |
e26b6d43 CW |
456 | |
457 | rq = requests[id]; | |
458 | if (!rq) | |
459 | continue; | |
460 | ||
70a76a9b CW |
461 | if (rq->fence.error) { |
462 | err = -EIO; | |
463 | goto out; | |
464 | } | |
465 | ||
e26b6d43 | 466 | GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags)); |
be1cb55a | 467 | if (!rq->context->state) |
e26b6d43 CW |
468 | continue; |
469 | ||
be1cb55a CW |
470 | /* Keep a copy of the state's backing pages; free the obj */ |
471 | state = shmem_create_from_object(rq->context->state->obj); | |
472 | if (IS_ERR(state)) { | |
473 | err = PTR_ERR(state); | |
e26b6d43 CW |
474 | goto out; |
475 | } | |
be1cb55a | 476 | rq->engine->default_state = state; |
e26b6d43 CW |
477 | } |
478 | ||
479 | out: | |
480 | /* | |
481 | * If we have to abandon now, we expect the engines to be idle | |
482 | * and ready to be torn-down. The quickest way we can accomplish | |
483 | * this is by declaring ourselves wedged. | |
484 | */ | |
485 | if (err) | |
486 | intel_gt_set_wedged(gt); | |
487 | ||
488 | for (id = 0; id < ARRAY_SIZE(requests); id++) { | |
489 | struct intel_context *ce; | |
490 | struct i915_request *rq; | |
491 | ||
492 | rq = requests[id]; | |
493 | if (!rq) | |
494 | continue; | |
495 | ||
496 | ce = rq->context; | |
497 | i915_request_put(rq); | |
498 | intel_context_put(ce); | |
499 | } | |
500 | return err; | |
501 | } | |
502 | ||
503 | static int __engines_verify_workarounds(struct intel_gt *gt) | |
504 | { | |
505 | struct intel_engine_cs *engine; | |
506 | enum intel_engine_id id; | |
507 | int err = 0; | |
508 | ||
509 | if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) | |
510 | return 0; | |
511 | ||
512 | for_each_engine(engine, gt, id) { | |
513 | if (intel_engine_verify_workarounds(engine, "load")) | |
514 | err = -EIO; | |
515 | } | |
516 | ||
9c652711 CW |
517 | /* Flush and restore the kernel context for safety */ |
518 | if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) | |
519 | err = -EIO; | |
520 | ||
e26b6d43 CW |
521 | return err; |
522 | } | |
523 | ||
524 | static void __intel_gt_disable(struct intel_gt *gt) | |
525 | { | |
3f04bdce | 526 | intel_gt_set_wedged_on_fini(gt); |
e26b6d43 CW |
527 | |
528 | intel_gt_suspend_prepare(gt); | |
529 | intel_gt_suspend_late(gt); | |
530 | ||
531 | GEM_BUG_ON(intel_gt_pm_is_awake(gt)); | |
532 | } | |
533 | ||
42014f69 AS |
534 | int intel_gt_init(struct intel_gt *gt) |
535 | { | |
536 | int err; | |
537 | ||
e26b6d43 | 538 | err = i915_inject_probe_error(gt->i915, -ENODEV); |
42014f69 AS |
539 | if (err) |
540 | return err; | |
541 | ||
e26b6d43 CW |
542 | /* |
543 | * This is just a security blanket to placate dragons. | |
544 | * On some systems, we very sporadically observe that the first TLBs | |
545 | * used by the CS may be stale, despite us poking the TLB reset. If | |
546 | * we hold the forcewake during initialisation these problems | |
547 | * just magically go away. | |
548 | */ | |
549 | intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); | |
550 | ||
551 | err = intel_gt_init_scratch(gt, IS_GEN(gt->i915, 2) ? SZ_256K : SZ_4K); | |
552 | if (err) | |
553 | goto out_fw; | |
554 | ||
c1132367 AS |
555 | intel_gt_pm_init(gt); |
556 | ||
e6ba7648 CW |
557 | gt->vm = kernel_vm(gt); |
558 | if (!gt->vm) { | |
559 | err = -ENOMEM; | |
e26b6d43 | 560 | goto err_pm; |
e6ba7648 CW |
561 | } |
562 | ||
e26b6d43 CW |
563 | err = intel_engines_init(gt); |
564 | if (err) | |
565 | goto err_engines; | |
566 | ||
3acffa8c DCS |
567 | err = intel_uc_init(>->uc); |
568 | if (err) | |
569 | goto err_engines; | |
e26b6d43 | 570 | |
e26b6d43 CW |
571 | err = intel_gt_resume(gt); |
572 | if (err) | |
cfe6b30f | 573 | goto err_uc_init; |
e26b6d43 CW |
574 | |
575 | err = __engines_record_defaults(gt); | |
576 | if (err) | |
577 | goto err_gt; | |
578 | ||
579 | err = __engines_verify_workarounds(gt); | |
580 | if (err) | |
581 | goto err_gt; | |
582 | ||
583 | err = i915_inject_probe_error(gt->i915, -EIO); | |
584 | if (err) | |
585 | goto err_gt; | |
586 | ||
587 | goto out_fw; | |
588 | err_gt: | |
589 | __intel_gt_disable(gt); | |
e26b6d43 CW |
590 | intel_uc_fini_hw(>->uc); |
591 | err_uc_init: | |
592 | intel_uc_fini(>->uc); | |
593 | err_engines: | |
594 | intel_engines_release(gt); | |
e26b6d43 CW |
595 | i915_vm_put(fetch_and_zero(>->vm)); |
596 | err_pm: | |
597 | intel_gt_pm_fini(gt); | |
e6ba7648 | 598 | intel_gt_fini_scratch(gt); |
e26b6d43 CW |
599 | out_fw: |
600 | if (err) | |
601 | intel_gt_set_wedged_on_init(gt); | |
602 | intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); | |
e6ba7648 | 603 | return err; |
42014f69 AS |
604 | } |
605 | ||
606 | void intel_gt_driver_remove(struct intel_gt *gt) | |
607 | { | |
e26b6d43 CW |
608 | __intel_gt_disable(gt); |
609 | ||
a9410a62 | 610 | intel_uc_driver_remove(>->uc); |
e26b6d43 CW |
611 | |
612 | intel_engines_release(gt); | |
42014f69 AS |
613 | } |
614 | ||
615 | void intel_gt_driver_unregister(struct intel_gt *gt) | |
616 | { | |
25dc89d5 CW |
617 | intel_wakeref_t wakeref; |
618 | ||
3e7abf81 | 619 | intel_rps_driver_unregister(>->rps); |
6065682f CW |
620 | |
621 | /* | |
622 | * Upon unregistering the device to prevent any new users, cancel | |
623 | * all in-flight requests so that we can quickly unbind the active | |
624 | * resources. | |
625 | */ | |
626 | intel_gt_set_wedged(gt); | |
25dc89d5 CW |
627 | |
628 | /* Scrub all HW state upon release */ | |
629 | with_intel_runtime_pm(gt->uncore->rpm, wakeref) | |
630 | __intel_gt_reset(gt, ALL_ENGINES); | |
42014f69 AS |
631 | } |
632 | ||
633 | void intel_gt_driver_release(struct intel_gt *gt) | |
634 | { | |
e6ba7648 CW |
635 | struct i915_address_space *vm; |
636 | ||
637 | vm = fetch_and_zero(>->vm); | |
638 | if (vm) /* FIXME being called twice on error paths :( */ | |
639 | i915_vm_put(vm); | |
640 | ||
c1132367 | 641 | intel_gt_pm_fini(gt); |
42014f69 | 642 | intel_gt_fini_scratch(gt); |
16e87459 | 643 | intel_gt_fini_buffer_pool(gt); |
42014f69 AS |
644 | } |
645 | ||
6cf72db6 | 646 | void intel_gt_driver_late_release(struct intel_gt *gt) |
cb823ed9 | 647 | { |
22ca8a45 CW |
648 | /* We need to wait for inflight RCU frees to release their grip */ |
649 | rcu_barrier(); | |
650 | ||
6f76098f | 651 | intel_uc_driver_late_release(>->uc); |
dea397e8 | 652 | intel_gt_fini_requests(gt); |
cb823ed9 | 653 | intel_gt_fini_reset(gt); |
4605bb73 | 654 | intel_gt_fini_timelines(gt); |
e26b6d43 | 655 | intel_engines_free(gt); |
cb823ed9 | 656 | } |
792592e7 DCS |
657 | |
658 | void intel_gt_info_print(const struct intel_gt_info *info, | |
659 | struct drm_printer *p) | |
660 | { | |
661 | drm_printf(p, "available engines: %x\n", info->engine_mask); | |
0b6613c6 VSD |
662 | |
663 | intel_sseu_dump(&info->sseu, p); | |
792592e7 | 664 | } |