Commit | Line | Data |
---|---|---|
24f90d66 | 1 | // SPDX-License-Identifier: MIT |
79ffac85 | 2 | /* |
79ffac85 CW |
3 | * Copyright © 2019 Intel Corporation |
4 | */ | |
5 | ||
6 | #include "i915_drv.h" | |
7 | ||
b3786b29 | 8 | #include "intel_breadcrumbs.h" |
e6ba7648 | 9 | #include "intel_context.h" |
79ffac85 | 10 | #include "intel_engine.h" |
058179e7 | 11 | #include "intel_engine_heartbeat.h" |
79ffac85 | 12 | #include "intel_engine_pm.h" |
cb823ed9 | 13 | #include "intel_gt.h" |
79ffac85 | 14 | #include "intel_gt_pm.h" |
c1132367 | 15 | #include "intel_rc6.h" |
2871ea85 | 16 | #include "intel_ring.h" |
be1cb55a | 17 | #include "shmem_utils.h" |
79ffac85 | 18 | |
89db9537 CW |
19 | static void dbg_poison_ce(struct intel_context *ce) |
20 | { | |
21 | if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) | |
22 | return; | |
23 | ||
24 | if (ce->state) { | |
25 | struct drm_i915_gem_object *obj = ce->state->obj; | |
fa85bfd1 | 26 | int type = i915_coherent_map_type(ce->engine->i915, obj, true); |
89db9537 CW |
27 | void *map; |
28 | ||
d8be1357 | 29 | if (!i915_gem_object_trylock(obj, NULL)) |
b51ed60e ML |
30 | return; |
31 | ||
89db9537 CW |
32 | map = i915_gem_object_pin_map(obj, type); |
33 | if (!IS_ERR(map)) { | |
34 | memset(map, CONTEXT_REDZONE, obj->base.size); | |
35 | i915_gem_object_flush_map(obj); | |
36 | i915_gem_object_unpin_map(obj); | |
37 | } | |
b51ed60e | 38 | i915_gem_object_unlock(obj); |
89db9537 CW |
39 | } |
40 | } | |
41 | ||
c34c5bca | 42 | static int __engine_unpark(struct intel_wakeref *wf) |
79ffac85 CW |
43 | { |
44 | struct intel_engine_cs *engine = | |
45 | container_of(wf, typeof(*engine), wakeref); | |
fb218f20 | 46 | struct intel_context *ce; |
79ffac85 | 47 | |
639f2f24 | 48 | ENGINE_TRACE(engine, "\n"); |
79ffac85 | 49 | |
0c91621c | 50 | intel_gt_pm_get(engine->gt); |
79ffac85 | 51 | |
fb218f20 CW |
52 | /* Discard stale context state from across idling */ |
53 | ce = engine->kernel_context; | |
1d0e2c93 CW |
54 | if (ce) { |
55 | GEM_BUG_ON(test_bit(CONTEXT_VALID_BIT, &ce->flags)); | |
56 | ||
89db9537 CW |
57 | /* Flush all pending HW writes before we touch the context */ |
58 | while (unlikely(intel_context_inflight(ce))) | |
59 | intel_engine_flush_submission(engine); | |
60 | ||
1d0e2c93 | 61 | /* First poison the image to verify we never fully trust it */ |
89db9537 | 62 | dbg_poison_ce(ce); |
1d0e2c93 | 63 | |
89db9537 | 64 | /* Scrub the context image after our loss of control */ |
fb218f20 | 65 | ce->ops->reset(ce); |
b436a5f8 CW |
66 | |
67 | CE_TRACE(ce, "reset { seqno:%x, *hwsp:%x, ring:%x }\n", | |
68 | ce->timeline->seqno, | |
69 | READ_ONCE(*ce->timeline->hwsp_seqno), | |
70 | ce->ring->emit); | |
71 | GEM_BUG_ON(ce->timeline->seqno != | |
72 | READ_ONCE(*ce->timeline->hwsp_seqno)); | |
1d0e2c93 | 73 | } |
fb218f20 | 74 | |
79ffac85 CW |
75 | if (engine->unpark) |
76 | engine->unpark(engine); | |
77 | ||
e3ed90b8 | 78 | intel_breadcrumbs_unpark(engine->breadcrumbs); |
058179e7 | 79 | intel_engine_unpark_heartbeat(engine); |
79ffac85 CW |
80 | return 0; |
81 | } | |
82 | ||
b81e4d9b CW |
83 | static void duration(struct dma_fence *fence, struct dma_fence_cb *cb) |
84 | { | |
85 | struct i915_request *rq = to_request(fence); | |
86 | ||
87 | ewma__engine_latency_add(&rq->engine->latency, | |
88 | ktime_us_delta(rq->fence.timestamp, | |
89 | rq->duration.emitted)); | |
90 | } | |
91 | ||
5cba2884 | 92 | static void |
88a4655e CW |
93 | __queue_and_release_pm(struct i915_request *rq, |
94 | struct intel_timeline *tl, | |
95 | struct intel_engine_cs *engine) | |
5cba2884 CW |
96 | { |
97 | struct intel_gt_timelines *timelines = &engine->gt->timelines; | |
98 | ||
bf9cc891 | 99 | ENGINE_TRACE(engine, "parking\n"); |
88a4655e CW |
100 | |
101 | /* | |
102 | * We have to serialise all potential retirement paths with our | |
103 | * submission, as we don't want to underflow either the | |
104 | * engine->wakeref.counter or our timeline->active_count. | |
105 | * | |
106 | * Equally, we cannot allow a new submission to start until | |
107 | * after we finish queueing, nor could we allow that submitter | |
108 | * to retire us before we are ready! | |
109 | */ | |
5cba2884 CW |
110 | spin_lock(&timelines->lock); |
111 | ||
88a4655e | 112 | /* Let intel_gt_retire_requests() retire us (acquired under lock) */ |
5cba2884 CW |
113 | if (!atomic_fetch_inc(&tl->active_count)) |
114 | list_add_tail(&tl->link, &timelines->active_list); | |
115 | ||
88a4655e | 116 | /* Hand the request over to HW and so engine_retire() */ |
16f2941a | 117 | __i915_request_queue_bh(rq); |
88a4655e CW |
118 | |
119 | /* Let new submissions commence (and maybe retire this timeline) */ | |
5cba2884 CW |
120 | __intel_wakeref_defer_park(&engine->wakeref); |
121 | ||
122 | spin_unlock(&timelines->lock); | |
123 | } | |
124 | ||
79ffac85 CW |
125 | static bool switch_to_kernel_context(struct intel_engine_cs *engine) |
126 | { | |
5cba2884 | 127 | struct intel_context *ce = engine->kernel_context; |
79ffac85 | 128 | struct i915_request *rq; |
6dcb85a0 | 129 | bool result = true; |
79ffac85 | 130 | |
36332429 MB |
131 | /* |
132 | * This is execlist specific behaviour intended to ensure the GPU is | |
133 | * idle by switching to a known 'safe' context. With GuC submission, the | |
134 | * same idle guarantee is achieved by other means (disabling | |
135 | * scheduling). Further, switching to a 'safe' context has no effect | |
136 | * with GuC submission as the scheduler can just switch back again. | |
137 | * | |
138 | * FIXME: Move this backend scheduler specific behaviour into the | |
139 | * scheduler backend. | |
140 | */ | |
141 | if (intel_engine_uses_guc(engine)) | |
142 | return true; | |
143 | ||
45b152f7 CW |
144 | /* GPU is pointing to the void, as good as in the kernel context. */ |
145 | if (intel_gt_is_wedged(engine->gt)) | |
146 | return true; | |
147 | ||
e6ba7648 | 148 | GEM_BUG_ON(!intel_context_is_barrier(ce)); |
2a19abb0 | 149 | GEM_BUG_ON(ce->timeline->hwsp_ggtt != engine->status_page.vma); |
e6ba7648 | 150 | |
79ffac85 CW |
151 | /* Already inside the kernel context, safe to power down. */ |
152 | if (engine->wakeref_serial == engine->serial) | |
153 | return true; | |
154 | ||
79ffac85 CW |
155 | /* |
156 | * Note, we do this without taking the timeline->mutex. We cannot | |
157 | * as we may be called while retiring the kernel context and so | |
158 | * already underneath the timeline->mutex. Instead we rely on the | |
c34c5bca | 159 | * exclusive property of the __engine_park that prevents anyone |
79ffac85 CW |
160 | * else from creating a request on this engine. This also requires |
161 | * that the ring is empty and we avoid any waits while constructing | |
162 | * the context, as they assume protection by the timeline->mutex. | |
163 | * This should hold true as we can only park the engine after | |
164 | * retiring the last request, thus all rings should be empty and | |
165 | * all timelines idle. | |
5cba2884 CW |
166 | * |
167 | * For unlocking, there are 2 other parties and the GPU who have a | |
168 | * stake here. | |
169 | * | |
170 | * A new gpu user will be waiting on the engine-pm to start their | |
171 | * engine_unpark. New waiters are predicated on engine->wakeref.count | |
172 | * and so intel_wakeref_defer_park() acts like a mutex_unlock of the | |
173 | * engine->wakeref. | |
174 | * | |
175 | * The other party is intel_gt_retire_requests(), which is walking the | |
176 | * list of active timelines looking for completions. Meanwhile as soon | |
177 | * as we call __i915_request_queue(), the GPU may complete our request. | |
178 | * Ergo, if we put ourselves on the timelines.active_list | |
179 | * (se intel_timeline_enter()) before we increment the | |
180 | * engine->wakeref.count, we may see the request completion and retire | |
0d961c46 | 181 | * it causing an underflow of the engine->wakeref. |
79ffac85 | 182 | */ |
bce45c26 | 183 | set_bit(CONTEXT_IS_PARKING, &ce->flags); |
5cba2884 | 184 | GEM_BUG_ON(atomic_read(&ce->timeline->active_count) < 0); |
6c69a454 | 185 | |
5cba2884 | 186 | rq = __i915_request_create(ce, GFP_NOWAIT); |
79ffac85 CW |
187 | if (IS_ERR(rq)) |
188 | /* Context switch failed, hope for the best! Maybe reset? */ | |
6dcb85a0 | 189 | goto out_unlock; |
79ffac85 CW |
190 | |
191 | /* Check again on the next retirement. */ | |
192 | engine->wakeref_serial = engine->serial + 1; | |
d8af05ff | 193 | i915_request_add_active_barriers(rq); |
a79ca656 CW |
194 | |
195 | /* Install ourselves as a preemption barrier */ | |
b5e8e954 | 196 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; |
b81e4d9b CW |
197 | if (likely(!__i915_request_commit(rq))) { /* engine should be idle! */ |
198 | /* | |
199 | * Use an interrupt for precise measurement of duration, | |
200 | * otherwise we rely on someone else retiring all the requests | |
201 | * which may delay the signaling (i.e. we will likely wait | |
202 | * until the background request retirement running every | |
203 | * second or two). | |
204 | */ | |
205 | BUILD_BUG_ON(sizeof(rq->duration) > sizeof(rq->submitq)); | |
206 | dma_fence_add_callback(&rq->fence, &rq->duration.cb, duration); | |
207 | rq->duration.emitted = ktime_get(); | |
208 | } | |
79ffac85 | 209 | |
88a4655e CW |
210 | /* Expose ourselves to the world */ |
211 | __queue_and_release_pm(rq, ce->timeline, engine); | |
5cba2884 | 212 | |
6dcb85a0 CW |
213 | result = false; |
214 | out_unlock: | |
bce45c26 | 215 | clear_bit(CONTEXT_IS_PARKING, &ce->flags); |
6dcb85a0 | 216 | return result; |
79ffac85 CW |
217 | } |
218 | ||
b7234840 CW |
219 | static void call_idle_barriers(struct intel_engine_cs *engine) |
220 | { | |
221 | struct llist_node *node, *next; | |
222 | ||
223 | llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) { | |
224 | struct dma_fence_cb *cb = | |
225 | container_of((struct list_head *)node, | |
226 | typeof(*cb), node); | |
227 | ||
df9f85d8 | 228 | cb->func(ERR_PTR(-EAGAIN), cb); |
b7234840 CW |
229 | } |
230 | } | |
231 | ||
c34c5bca | 232 | static int __engine_park(struct intel_wakeref *wf) |
79ffac85 CW |
233 | { |
234 | struct intel_engine_cs *engine = | |
235 | container_of(wf, typeof(*engine), wakeref); | |
236 | ||
44d89409 CW |
237 | engine->saturated = 0; |
238 | ||
79ffac85 CW |
239 | /* |
240 | * If one and only one request is completed between pm events, | |
241 | * we know that we are inside the kernel context and it is | |
242 | * safe to power down. (We are paranoid in case that runtime | |
243 | * suspend causes corruption to the active context image, and | |
244 | * want to avoid that impacting userspace.) | |
245 | */ | |
246 | if (!switch_to_kernel_context(engine)) | |
247 | return -EBUSY; | |
248 | ||
bf9cc891 | 249 | ENGINE_TRACE(engine, "parked\n"); |
79ffac85 | 250 | |
b7234840 CW |
251 | call_idle_barriers(engine); /* cleanup after wedging */ |
252 | ||
058179e7 | 253 | intel_engine_park_heartbeat(engine); |
b3786b29 | 254 | intel_breadcrumbs_park(engine->breadcrumbs); |
79ffac85 CW |
255 | |
256 | /* Must be reset upon idling, or we may miss the busy wakeup. */ | |
3e28d371 | 257 | GEM_BUG_ON(engine->sched_engine->queue_priority_hint != INT_MIN); |
79ffac85 CW |
258 | |
259 | if (engine->park) | |
260 | engine->park(engine); | |
261 | ||
07779a76 CW |
262 | /* While gt calls i915_vma_parked(), we have to break the lock cycle */ |
263 | intel_gt_pm_put_async(engine->gt); | |
79ffac85 CW |
264 | return 0; |
265 | } | |
266 | ||
c7302f20 CW |
267 | static const struct intel_wakeref_ops wf_ops = { |
268 | .get = __engine_unpark, | |
269 | .put = __engine_park, | |
270 | }; | |
79ffac85 CW |
271 | |
272 | void intel_engine_init__pm(struct intel_engine_cs *engine) | |
273 | { | |
cd6a8513 | 274 | struct intel_runtime_pm *rpm = engine->uncore->rpm; |
c7302f20 CW |
275 | |
276 | intel_wakeref_init(&engine->wakeref, rpm, &wf_ops); | |
058179e7 | 277 | intel_engine_init_heartbeat(engine); |
79ffac85 | 278 | } |
c7302f20 | 279 | |
3e42cc61 TH |
280 | /** |
281 | * intel_engine_reset_pinned_contexts - Reset the pinned contexts of | |
282 | * an engine. | |
283 | * @engine: The engine whose pinned contexts we want to reset. | |
284 | * | |
285 | * Typically the pinned context LMEM images lose or get their content | |
286 | * corrupted on suspend. This function resets their images. | |
287 | */ | |
288 | void intel_engine_reset_pinned_contexts(struct intel_engine_cs *engine) | |
289 | { | |
290 | struct intel_context *ce; | |
291 | ||
292 | list_for_each_entry(ce, &engine->pinned_contexts_list, | |
293 | pinned_contexts_link) { | |
294 | /* kernel context gets reset at __engine_unpark() */ | |
295 | if (ce == engine->kernel_context) | |
296 | continue; | |
297 | ||
298 | dbg_poison_ce(ce); | |
299 | ce->ops->reset(ce); | |
300 | } | |
301 | } | |
302 | ||
c7302f20 CW |
303 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) |
304 | #include "selftest_engine_pm.c" | |
305 | #endif |