Commit | Line | Data |
---|---|---|
64d6c500 CW |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2019 Intel Corporation | |
5 | */ | |
6 | ||
7 | #ifndef _I915_ACTIVE_H_ | |
8 | #define _I915_ACTIVE_H_ | |
9 | ||
21950ee7 CW |
10 | #include <linux/lockdep.h> |
11 | ||
64d6c500 | 12 | #include "i915_active_types.h" |
21950ee7 CW |
13 | #include "i915_request.h" |
14 | ||
b1e3177b CW |
15 | struct i915_request; |
16 | struct intel_engine_cs; | |
17 | struct intel_timeline; | |
18 | ||
21950ee7 CW |
19 | /* |
20 | * We treat requests as fences. This is not be to confused with our | |
21 | * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync. | |
22 | * We use the fences to synchronize access from the CPU with activity on the | |
23 | * GPU, for example, we should not rewrite an object's PTE whilst the GPU | |
24 | * is reading them. We also track fences at a higher level to provide | |
25 | * implicit synchronisation around GEM objects, e.g. set-domain will wait | |
26 | * for outstanding GPU rendering before marking the object ready for CPU | |
27 | * access, or a pageflip will wait until the GPU is complete before showing | |
28 | * the frame on the scanout. | |
29 | * | |
30 | * In order to use a fence, the object must track the fence it needs to | |
31 | * serialise with. For example, GEM objects want to track both read and | |
32 | * write access so that we can perform concurrent read operations between | |
33 | * the CPU and GPU engines, as well as waiting for all rendering to | |
34 | * complete, or waiting for the last GPU user of a "fence register". The | |
b1e3177b | 35 | * object then embeds a #i915_active_fence to track the most recent (in |
21950ee7 | 36 | * retirement order) request relevant for the desired mode of access. |
b1e3177b | 37 | * The #i915_active_fence is updated with i915_active_fence_set() to |
21950ee7 CW |
38 | * track the most recent fence request, typically this is done as part of |
39 | * i915_vma_move_to_active(). | |
40 | * | |
b1e3177b | 41 | * When the #i915_active_fence completes (is retired), it will |
21950ee7 | 42 | * signal its completion to the owner through a callback as well as mark |
b1e3177b | 43 | * itself as idle (i915_active_fence.request == NULL). The owner |
21950ee7 CW |
44 | * can then perform any action, such as delayed freeing of an active |
45 | * resource including itself. | |
46 | */ | |
47 | ||
b1e3177b | 48 | void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb); |
21950ee7 CW |
49 | |
50 | /** | |
b1e3177b | 51 | * __i915_active_fence_init - prepares the activity tracker for use |
21950ee7 | 52 | * @active - the active tracker |
b1e3177b | 53 | * @fence - initial fence to track, can be NULL |
21950ee7 CW |
54 | * @func - a callback when then the tracker is retired (becomes idle), |
55 | * can be NULL | |
56 | * | |
b1e3177b CW |
57 | * i915_active_fence_init() prepares the embedded @active struct for use as |
58 | * an activity tracker, that is for tracking the last known active fence | |
59 | * associated with it. When the last fence becomes idle, when it is retired | |
21950ee7 CW |
60 | * after completion, the optional callback @func is invoked. |
61 | */ | |
62 | static inline void | |
b1e3177b | 63 | __i915_active_fence_init(struct i915_active_fence *active, |
b1e3177b CW |
64 | void *fence, |
65 | dma_fence_func_t fn) | |
21950ee7 | 66 | { |
b1e3177b CW |
67 | RCU_INIT_POINTER(active->fence, fence); |
68 | active->cb.func = fn ?: i915_active_noop; | |
21950ee7 CW |
69 | } |
70 | ||
df9f85d8 CW |
71 | #define INIT_ACTIVE_FENCE(A) \ |
72 | __i915_active_fence_init((A), NULL, NULL) | |
21950ee7 | 73 | |
b1e3177b CW |
74 | struct dma_fence * |
75 | __i915_active_fence_set(struct i915_active_fence *active, | |
76 | struct dma_fence *fence); | |
21950ee7 | 77 | |
21950ee7 | 78 | /** |
b1e3177b | 79 | * i915_active_fence_set - updates the tracker to watch the current fence |
21950ee7 | 80 | * @active - the active tracker |
b1e3177b | 81 | * @rq - the request to watch |
21950ee7 | 82 | * |
b1e3177b CW |
83 | * i915_active_fence_set() watches the given @rq for completion. While |
84 | * that @rq is busy, the @active reports busy. When that @rq is signaled | |
85 | * (or else retired) the @active tracker is updated to report idle. | |
21950ee7 | 86 | */ |
b1e3177b CW |
87 | int __must_check |
88 | i915_active_fence_set(struct i915_active_fence *active, | |
89 | struct i915_request *rq); | |
21950ee7 | 90 | /** |
b1e3177b | 91 | * i915_active_fence_get - return a reference to the active fence |
21950ee7 CW |
92 | * @active - the active tracker |
93 | * | |
b1e3177b | 94 | * i915_active_fence_get() returns a reference to the active fence, |
21950ee7 CW |
95 | * or NULL if the active tracker is idle. The reference is obtained under RCU, |
96 | * so no locking is required by the caller. | |
97 | * | |
b1e3177b | 98 | * The reference should be freed with dma_fence_put(). |
21950ee7 | 99 | */ |
b1e3177b CW |
100 | static inline struct dma_fence * |
101 | i915_active_fence_get(struct i915_active_fence *active) | |
21950ee7 | 102 | { |
b1e3177b | 103 | struct dma_fence *fence; |
21950ee7 CW |
104 | |
105 | rcu_read_lock(); | |
b1e3177b | 106 | fence = dma_fence_get_rcu_safe(&active->fence); |
21950ee7 CW |
107 | rcu_read_unlock(); |
108 | ||
b1e3177b | 109 | return fence; |
21950ee7 CW |
110 | } |
111 | ||
112 | /** | |
b1e3177b | 113 | * i915_active_fence_isset - report whether the active tracker is assigned |
21950ee7 CW |
114 | * @active - the active tracker |
115 | * | |
b1e3177b CW |
116 | * i915_active_fence_isset() returns true if the active tracker is currently |
117 | * assigned to a fence. Due to the lazy retiring, that fence may be idle | |
21950ee7 CW |
118 | * and this may report stale information. |
119 | */ | |
120 | static inline bool | |
b1e3177b | 121 | i915_active_fence_isset(const struct i915_active_fence *active) |
21950ee7 | 122 | { |
b1e3177b | 123 | return rcu_access_pointer(active->fence); |
21950ee7 CW |
124 | } |
125 | ||
64d6c500 CW |
126 | /* |
127 | * GPU activity tracking | |
128 | * | |
129 | * Each set of commands submitted to the GPU compromises a single request that | |
130 | * signals a fence upon completion. struct i915_request combines the | |
131 | * command submission, scheduling and fence signaling roles. If we want to see | |
132 | * if a particular task is complete, we need to grab the fence (struct | |
133 | * i915_request) for that task and check or wait for it to be signaled. More | |
134 | * often though we want to track the status of a bunch of tasks, for example | |
135 | * to wait for the GPU to finish accessing some memory across a variety of | |
136 | * different command pipelines from different clients. We could choose to | |
137 | * track every single request associated with the task, but knowing that | |
138 | * each request belongs to an ordered timeline (later requests within a | |
139 | * timeline must wait for earlier requests), we need only track the | |
140 | * latest request in each timeline to determine the overall status of the | |
141 | * task. | |
142 | * | |
143 | * struct i915_active provides this tracking across timelines. It builds a | |
144 | * composite shared-fence, and is updated as new work is submitted to the task, | |
145 | * forming a snapshot of the current status. It should be embedded into the | |
146 | * different resources that need to track their associated GPU activity to | |
147 | * provide a callback when that GPU activity has ceased, or otherwise to | |
148 | * provide a serialisation point either for request submission or for CPU | |
149 | * synchronisation. | |
150 | */ | |
151 | ||
b1e3177b | 152 | void __i915_active_init(struct i915_active *ref, |
12c255b5 CW |
153 | int (*active)(struct i915_active *ref), |
154 | void (*retire)(struct i915_active *ref), | |
ae303004 CW |
155 | struct lock_class_key *mkey, |
156 | struct lock_class_key *wkey); | |
157 | ||
158 | /* Specialise each class of i915_active to avoid impossible lockdep cycles. */ | |
b1e3177b | 159 | #define i915_active_init(ref, active, retire) do { \ |
ae303004 CW |
160 | static struct lock_class_key __mkey; \ |
161 | static struct lock_class_key __wkey; \ | |
12c255b5 | 162 | \ |
ae303004 | 163 | __i915_active_init(ref, active, retire, &__mkey, &__wkey); \ |
12c255b5 | 164 | } while (0) |
64d6c500 | 165 | |
5d934137 CW |
166 | struct dma_fence * |
167 | __i915_active_ref(struct i915_active *ref, u64 idx, struct dma_fence *fence); | |
168 | int i915_active_ref(struct i915_active *ref, u64 idx, struct dma_fence *fence); | |
64d6c500 | 169 | |
d19d71fc CW |
170 | static inline int |
171 | i915_active_add_request(struct i915_active *ref, struct i915_request *rq) | |
172 | { | |
5d934137 CW |
173 | return i915_active_ref(ref, |
174 | i915_request_timeline(rq)->fence_context, | |
175 | &rq->fence); | |
d19d71fc CW |
176 | } |
177 | ||
e3793468 CW |
178 | struct dma_fence * |
179 | i915_active_set_exclusive(struct i915_active *ref, struct dma_fence *f); | |
2850748e CW |
180 | |
181 | static inline bool i915_active_has_exclusive(struct i915_active *ref) | |
182 | { | |
b1e3177b | 183 | return rcu_access_pointer(ref->excl.fence); |
2850748e CW |
184 | } |
185 | ||
d75a92a8 CW |
186 | int __i915_active_wait(struct i915_active *ref, int state); |
187 | static inline int i915_active_wait(struct i915_active *ref) | |
188 | { | |
189 | return __i915_active_wait(ref, TASK_INTERRUPTIBLE); | |
190 | } | |
64d6c500 | 191 | |
29e6ecf3 CW |
192 | int i915_sw_fence_await_active(struct i915_sw_fence *fence, |
193 | struct i915_active *ref, | |
194 | unsigned int flags); | |
195 | int i915_request_await_active(struct i915_request *rq, | |
196 | struct i915_active *ref, | |
197 | unsigned int flags); | |
442dbc5c CW |
198 | #define I915_ACTIVE_AWAIT_EXCL BIT(0) |
199 | #define I915_ACTIVE_AWAIT_ACTIVE BIT(1) | |
3b0a0579 | 200 | #define I915_ACTIVE_AWAIT_BARRIER BIT(2) |
64d6c500 | 201 | |
12c255b5 | 202 | int i915_active_acquire(struct i915_active *ref); |
5d934137 | 203 | int i915_active_acquire_for_context(struct i915_active *ref, u64 idx); |
b1e3177b | 204 | bool i915_active_acquire_if_busy(struct i915_active *ref); |
5d934137 | 205 | |
64d6c500 | 206 | void i915_active_release(struct i915_active *ref); |
79c7a28e | 207 | |
e5429340 CW |
208 | static inline void __i915_active_acquire(struct i915_active *ref) |
209 | { | |
210 | GEM_BUG_ON(!atomic_read(&ref->count)); | |
211 | atomic_inc(&ref->count); | |
212 | } | |
213 | ||
64d6c500 CW |
214 | static inline bool |
215 | i915_active_is_idle(const struct i915_active *ref) | |
216 | { | |
12c255b5 | 217 | return !atomic_read(&ref->count); |
64d6c500 CW |
218 | } |
219 | ||
220 | void i915_active_fini(struct i915_active *ref); | |
221 | ||
ce476c80 CW |
222 | int i915_active_acquire_preallocate_barrier(struct i915_active *ref, |
223 | struct intel_engine_cs *engine); | |
224 | void i915_active_acquire_barrier(struct i915_active *ref); | |
d8af05ff | 225 | void i915_request_add_active_barriers(struct i915_request *rq); |
ce476c80 | 226 | |
164a4128 | 227 | void i915_active_print(struct i915_active *ref, struct drm_printer *m); |
38813767 | 228 | void i915_active_unlock_wait(struct i915_active *ref); |
164a4128 | 229 | |
229007e0 CW |
230 | struct i915_active *i915_active_create(void); |
231 | struct i915_active *i915_active_get(struct i915_active *ref); | |
232 | void i915_active_put(struct i915_active *ref); | |
233 | ||
af5c6fcf CW |
234 | static inline int __i915_request_await_exclusive(struct i915_request *rq, |
235 | struct i915_active *active) | |
236 | { | |
237 | struct dma_fence *fence; | |
238 | int err = 0; | |
239 | ||
240 | fence = i915_active_fence_get(&active->excl); | |
241 | if (fence) { | |
242 | err = i915_request_await_dma_fence(rq, fence); | |
243 | dma_fence_put(fence); | |
244 | } | |
245 | ||
246 | return err; | |
247 | } | |
248 | ||
64d6c500 | 249 | #endif /* _I915_ACTIVE_H_ */ |