Commit | Line | Data |
---|---|---|
79ffac85 CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2019 Intel Corporation | |
5 | */ | |
6 | ||
7 | #include "i915_drv.h" | |
8 | ||
b3786b29 | 9 | #include "intel_breadcrumbs.h" |
e6ba7648 | 10 | #include "intel_context.h" |
79ffac85 | 11 | #include "intel_engine.h" |
058179e7 | 12 | #include "intel_engine_heartbeat.h" |
79ffac85 | 13 | #include "intel_engine_pm.h" |
cb823ed9 | 14 | #include "intel_gt.h" |
79ffac85 | 15 | #include "intel_gt_pm.h" |
c1132367 | 16 | #include "intel_rc6.h" |
2871ea85 | 17 | #include "intel_ring.h" |
be1cb55a | 18 | #include "shmem_utils.h" |
79ffac85 | 19 | |
89db9537 CW |
20 | static void dbg_poison_ce(struct intel_context *ce) |
21 | { | |
22 | if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) | |
23 | return; | |
24 | ||
25 | if (ce->state) { | |
26 | struct drm_i915_gem_object *obj = ce->state->obj; | |
27 | int type = i915_coherent_map_type(ce->engine->i915); | |
28 | void *map; | |
29 | ||
30 | map = i915_gem_object_pin_map(obj, type); | |
31 | if (!IS_ERR(map)) { | |
32 | memset(map, CONTEXT_REDZONE, obj->base.size); | |
33 | i915_gem_object_flush_map(obj); | |
34 | i915_gem_object_unpin_map(obj); | |
35 | } | |
36 | } | |
37 | } | |
38 | ||
c34c5bca | 39 | static int __engine_unpark(struct intel_wakeref *wf) |
79ffac85 CW |
40 | { |
41 | struct intel_engine_cs *engine = | |
42 | container_of(wf, typeof(*engine), wakeref); | |
fb218f20 | 43 | struct intel_context *ce; |
79ffac85 | 44 | |
639f2f24 | 45 | ENGINE_TRACE(engine, "\n"); |
79ffac85 | 46 | |
0c91621c | 47 | intel_gt_pm_get(engine->gt); |
79ffac85 | 48 | |
fb218f20 CW |
49 | /* Discard stale context state from across idling */ |
50 | ce = engine->kernel_context; | |
1d0e2c93 CW |
51 | if (ce) { |
52 | GEM_BUG_ON(test_bit(CONTEXT_VALID_BIT, &ce->flags)); | |
53 | ||
89db9537 CW |
54 | /* Flush all pending HW writes before we touch the context */ |
55 | while (unlikely(intel_context_inflight(ce))) | |
56 | intel_engine_flush_submission(engine); | |
57 | ||
1d0e2c93 | 58 | /* First poison the image to verify we never fully trust it */ |
89db9537 | 59 | dbg_poison_ce(ce); |
1d0e2c93 | 60 | |
89db9537 | 61 | /* Scrub the context image after our loss of control */ |
fb218f20 | 62 | ce->ops->reset(ce); |
b436a5f8 CW |
63 | |
64 | CE_TRACE(ce, "reset { seqno:%x, *hwsp:%x, ring:%x }\n", | |
65 | ce->timeline->seqno, | |
66 | READ_ONCE(*ce->timeline->hwsp_seqno), | |
67 | ce->ring->emit); | |
68 | GEM_BUG_ON(ce->timeline->seqno != | |
69 | READ_ONCE(*ce->timeline->hwsp_seqno)); | |
1d0e2c93 | 70 | } |
fb218f20 | 71 | |
79ffac85 CW |
72 | if (engine->unpark) |
73 | engine->unpark(engine); | |
74 | ||
e3ed90b8 | 75 | intel_breadcrumbs_unpark(engine->breadcrumbs); |
058179e7 | 76 | intel_engine_unpark_heartbeat(engine); |
79ffac85 CW |
77 | return 0; |
78 | } | |
79 | ||
d6773926 CW |
80 | #if IS_ENABLED(CONFIG_LOCKDEP) |
81 | ||
9834dfef | 82 | static unsigned long __timeline_mark_lock(struct intel_context *ce) |
6c69a454 | 83 | { |
d6773926 CW |
84 | unsigned long flags; |
85 | ||
86 | local_irq_save(flags); | |
6c69a454 | 87 | mutex_acquire(&ce->timeline->mutex.dep_map, 2, 0, _THIS_IP_); |
6dcb85a0 CW |
88 | |
89 | return flags; | |
6c69a454 CW |
90 | } |
91 | ||
9834dfef CW |
92 | static void __timeline_mark_unlock(struct intel_context *ce, |
93 | unsigned long flags) | |
6c69a454 | 94 | { |
5facae4f | 95 | mutex_release(&ce->timeline->mutex.dep_map, _THIS_IP_); |
6dcb85a0 | 96 | local_irq_restore(flags); |
6c69a454 | 97 | } |
d6773926 CW |
98 | |
99 | #else | |
100 | ||
9834dfef | 101 | static unsigned long __timeline_mark_lock(struct intel_context *ce) |
d6773926 | 102 | { |
6dcb85a0 | 103 | return 0; |
d6773926 CW |
104 | } |
105 | ||
9834dfef CW |
106 | static void __timeline_mark_unlock(struct intel_context *ce, |
107 | unsigned long flags) | |
d6773926 CW |
108 | { |
109 | } | |
110 | ||
111 | #endif /* !IS_ENABLED(CONFIG_LOCKDEP) */ | |
6c69a454 | 112 | |
b81e4d9b CW |
113 | static void duration(struct dma_fence *fence, struct dma_fence_cb *cb) |
114 | { | |
115 | struct i915_request *rq = to_request(fence); | |
116 | ||
117 | ewma__engine_latency_add(&rq->engine->latency, | |
118 | ktime_us_delta(rq->fence.timestamp, | |
119 | rq->duration.emitted)); | |
120 | } | |
121 | ||
5cba2884 | 122 | static void |
88a4655e CW |
123 | __queue_and_release_pm(struct i915_request *rq, |
124 | struct intel_timeline *tl, | |
125 | struct intel_engine_cs *engine) | |
5cba2884 CW |
126 | { |
127 | struct intel_gt_timelines *timelines = &engine->gt->timelines; | |
128 | ||
bf9cc891 | 129 | ENGINE_TRACE(engine, "parking\n"); |
88a4655e CW |
130 | |
131 | /* | |
132 | * We have to serialise all potential retirement paths with our | |
133 | * submission, as we don't want to underflow either the | |
134 | * engine->wakeref.counter or our timeline->active_count. | |
135 | * | |
136 | * Equally, we cannot allow a new submission to start until | |
137 | * after we finish queueing, nor could we allow that submitter | |
138 | * to retire us before we are ready! | |
139 | */ | |
5cba2884 CW |
140 | spin_lock(&timelines->lock); |
141 | ||
88a4655e | 142 | /* Let intel_gt_retire_requests() retire us (acquired under lock) */ |
5cba2884 CW |
143 | if (!atomic_fetch_inc(&tl->active_count)) |
144 | list_add_tail(&tl->link, &timelines->active_list); | |
145 | ||
88a4655e | 146 | /* Hand the request over to HW and so engine_retire() */ |
16f2941a | 147 | __i915_request_queue_bh(rq); |
88a4655e CW |
148 | |
149 | /* Let new submissions commence (and maybe retire this timeline) */ | |
5cba2884 CW |
150 | __intel_wakeref_defer_park(&engine->wakeref); |
151 | ||
152 | spin_unlock(&timelines->lock); | |
153 | } | |
154 | ||
79ffac85 CW |
155 | static bool switch_to_kernel_context(struct intel_engine_cs *engine) |
156 | { | |
5cba2884 | 157 | struct intel_context *ce = engine->kernel_context; |
79ffac85 | 158 | struct i915_request *rq; |
6dcb85a0 CW |
159 | unsigned long flags; |
160 | bool result = true; | |
79ffac85 | 161 | |
45b152f7 CW |
162 | /* GPU is pointing to the void, as good as in the kernel context. */ |
163 | if (intel_gt_is_wedged(engine->gt)) | |
164 | return true; | |
165 | ||
e6ba7648 | 166 | GEM_BUG_ON(!intel_context_is_barrier(ce)); |
2a19abb0 | 167 | GEM_BUG_ON(ce->timeline->hwsp_ggtt != engine->status_page.vma); |
e6ba7648 | 168 | |
79ffac85 CW |
169 | /* Already inside the kernel context, safe to power down. */ |
170 | if (engine->wakeref_serial == engine->serial) | |
171 | return true; | |
172 | ||
79ffac85 CW |
173 | /* |
174 | * Note, we do this without taking the timeline->mutex. We cannot | |
175 | * as we may be called while retiring the kernel context and so | |
176 | * already underneath the timeline->mutex. Instead we rely on the | |
c34c5bca | 177 | * exclusive property of the __engine_park that prevents anyone |
79ffac85 CW |
178 | * else from creating a request on this engine. This also requires |
179 | * that the ring is empty and we avoid any waits while constructing | |
180 | * the context, as they assume protection by the timeline->mutex. | |
181 | * This should hold true as we can only park the engine after | |
182 | * retiring the last request, thus all rings should be empty and | |
183 | * all timelines idle. | |
5cba2884 CW |
184 | * |
185 | * For unlocking, there are 2 other parties and the GPU who have a | |
186 | * stake here. | |
187 | * | |
188 | * A new gpu user will be waiting on the engine-pm to start their | |
189 | * engine_unpark. New waiters are predicated on engine->wakeref.count | |
190 | * and so intel_wakeref_defer_park() acts like a mutex_unlock of the | |
191 | * engine->wakeref. | |
192 | * | |
193 | * The other party is intel_gt_retire_requests(), which is walking the | |
194 | * list of active timelines looking for completions. Meanwhile as soon | |
195 | * as we call __i915_request_queue(), the GPU may complete our request. | |
196 | * Ergo, if we put ourselves on the timelines.active_list | |
197 | * (se intel_timeline_enter()) before we increment the | |
198 | * engine->wakeref.count, we may see the request completion and retire | |
0d961c46 | 199 | * it causing an underflow of the engine->wakeref. |
79ffac85 | 200 | */ |
5cba2884 CW |
201 | flags = __timeline_mark_lock(ce); |
202 | GEM_BUG_ON(atomic_read(&ce->timeline->active_count) < 0); | |
6c69a454 | 203 | |
5cba2884 | 204 | rq = __i915_request_create(ce, GFP_NOWAIT); |
79ffac85 CW |
205 | if (IS_ERR(rq)) |
206 | /* Context switch failed, hope for the best! Maybe reset? */ | |
6dcb85a0 | 207 | goto out_unlock; |
79ffac85 CW |
208 | |
209 | /* Check again on the next retirement. */ | |
210 | engine->wakeref_serial = engine->serial + 1; | |
d8af05ff | 211 | i915_request_add_active_barriers(rq); |
a79ca656 CW |
212 | |
213 | /* Install ourselves as a preemption barrier */ | |
b5e8e954 | 214 | rq->sched.attr.priority = I915_PRIORITY_BARRIER; |
b81e4d9b CW |
215 | if (likely(!__i915_request_commit(rq))) { /* engine should be idle! */ |
216 | /* | |
217 | * Use an interrupt for precise measurement of duration, | |
218 | * otherwise we rely on someone else retiring all the requests | |
219 | * which may delay the signaling (i.e. we will likely wait | |
220 | * until the background request retirement running every | |
221 | * second or two). | |
222 | */ | |
223 | BUILD_BUG_ON(sizeof(rq->duration) > sizeof(rq->submitq)); | |
224 | dma_fence_add_callback(&rq->fence, &rq->duration.cb, duration); | |
225 | rq->duration.emitted = ktime_get(); | |
226 | } | |
79ffac85 | 227 | |
88a4655e CW |
228 | /* Expose ourselves to the world */ |
229 | __queue_and_release_pm(rq, ce->timeline, engine); | |
5cba2884 | 230 | |
6dcb85a0 CW |
231 | result = false; |
232 | out_unlock: | |
5cba2884 | 233 | __timeline_mark_unlock(ce, flags); |
6dcb85a0 | 234 | return result; |
79ffac85 CW |
235 | } |
236 | ||
b7234840 CW |
237 | static void call_idle_barriers(struct intel_engine_cs *engine) |
238 | { | |
239 | struct llist_node *node, *next; | |
240 | ||
241 | llist_for_each_safe(node, next, llist_del_all(&engine->barrier_tasks)) { | |
242 | struct dma_fence_cb *cb = | |
243 | container_of((struct list_head *)node, | |
244 | typeof(*cb), node); | |
245 | ||
df9f85d8 | 246 | cb->func(ERR_PTR(-EAGAIN), cb); |
b7234840 CW |
247 | } |
248 | } | |
249 | ||
c34c5bca | 250 | static int __engine_park(struct intel_wakeref *wf) |
79ffac85 CW |
251 | { |
252 | struct intel_engine_cs *engine = | |
253 | container_of(wf, typeof(*engine), wakeref); | |
254 | ||
44d89409 CW |
255 | engine->saturated = 0; |
256 | ||
79ffac85 CW |
257 | /* |
258 | * If one and only one request is completed between pm events, | |
259 | * we know that we are inside the kernel context and it is | |
260 | * safe to power down. (We are paranoid in case that runtime | |
261 | * suspend causes corruption to the active context image, and | |
262 | * want to avoid that impacting userspace.) | |
263 | */ | |
264 | if (!switch_to_kernel_context(engine)) | |
265 | return -EBUSY; | |
266 | ||
bf9cc891 | 267 | ENGINE_TRACE(engine, "parked\n"); |
79ffac85 | 268 | |
b7234840 CW |
269 | call_idle_barriers(engine); /* cleanup after wedging */ |
270 | ||
058179e7 | 271 | intel_engine_park_heartbeat(engine); |
b3786b29 | 272 | intel_breadcrumbs_park(engine->breadcrumbs); |
79ffac85 CW |
273 | |
274 | /* Must be reset upon idling, or we may miss the busy wakeup. */ | |
275 | GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN); | |
276 | ||
277 | if (engine->park) | |
278 | engine->park(engine); | |
279 | ||
79ffac85 CW |
280 | engine->execlists.no_priolist = false; |
281 | ||
07779a76 CW |
282 | /* While gt calls i915_vma_parked(), we have to break the lock cycle */ |
283 | intel_gt_pm_put_async(engine->gt); | |
79ffac85 CW |
284 | return 0; |
285 | } | |
286 | ||
c7302f20 CW |
287 | static const struct intel_wakeref_ops wf_ops = { |
288 | .get = __engine_unpark, | |
289 | .put = __engine_park, | |
290 | }; | |
79ffac85 CW |
291 | |
292 | void intel_engine_init__pm(struct intel_engine_cs *engine) | |
293 | { | |
cd6a8513 | 294 | struct intel_runtime_pm *rpm = engine->uncore->rpm; |
c7302f20 CW |
295 | |
296 | intel_wakeref_init(&engine->wakeref, rpm, &wf_ops); | |
058179e7 | 297 | intel_engine_init_heartbeat(engine); |
79ffac85 | 298 | } |
c7302f20 CW |
299 | |
300 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
301 | #include "selftest_engine_pm.c" | |
302 | #endif |