Commit | Line | Data |
---|---|---|
09593216 CK |
1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | ||
3 | #include <drm/drm_exec.h> | |
4 | #include <drm/drm_gem.h> | |
5 | #include <linux/dma-resv.h> | |
6 | ||
7 | /** | |
8 | * DOC: Overview | |
9 | * | |
10 | * This component mainly abstracts the retry loop necessary for locking | |
11 | * multiple GEM objects while preparing hardware operations (e.g. command | |
12 | * submissions, page table updates etc..). | |
13 | * | |
14 | * If a contention is detected while locking a GEM object the cleanup procedure | |
15 | * unlocks all previously locked GEM objects and locks the contended one first | |
16 | * before locking any further objects. | |
17 | * | |
18 | * After an object is locked fences slots can optionally be reserved on the | |
19 | * dma_resv object inside the GEM object. | |
20 | * | |
21 | * A typical usage pattern should look like this:: | |
22 | * | |
23 | * struct drm_gem_object *obj; | |
24 | * struct drm_exec exec; | |
25 | * unsigned long index; | |
26 | * int ret; | |
27 | * | |
28 | * drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT); | |
29 | * drm_exec_until_all_locked(&exec) { | |
30 | * ret = drm_exec_prepare_obj(&exec, boA, 1); | |
31 | * drm_exec_retry_on_contention(&exec); | |
32 | * if (ret) | |
33 | * goto error; | |
34 | * | |
35 | * ret = drm_exec_prepare_obj(&exec, boB, 1); | |
36 | * drm_exec_retry_on_contention(&exec); | |
37 | * if (ret) | |
38 | * goto error; | |
39 | * } | |
40 | * | |
41 | * drm_exec_for_each_locked_object(&exec, index, obj) { | |
42 | * dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ); | |
43 | * ... | |
44 | * } | |
45 | * drm_exec_fini(&exec); | |
46 | * | |
47 | * See struct dma_exec for more details. | |
48 | */ | |
49 | ||
50 | /* Dummy value used to initially enter the retry loop */ | |
51 | #define DRM_EXEC_DUMMY ((void *)~0) | |
52 | ||
53 | /* Unlock all objects and drop references */ | |
54 | static void drm_exec_unlock_all(struct drm_exec *exec) | |
55 | { | |
56 | struct drm_gem_object *obj; | |
57 | unsigned long index; | |
58 | ||
d20b484c | 59 | drm_exec_for_each_locked_object_reverse(exec, index, obj) { |
09593216 CK |
60 | dma_resv_unlock(obj->resv); |
61 | drm_gem_object_put(obj); | |
62 | } | |
63 | ||
64 | drm_gem_object_put(exec->prelocked); | |
65 | exec->prelocked = NULL; | |
66 | } | |
67 | ||
68 | /** | |
69 | * drm_exec_init - initialize a drm_exec object | |
70 | * @exec: the drm_exec object to initialize | |
71 | * @flags: controls locking behavior, see DRM_EXEC_* defines | |
05d24935 | 72 | * @nr: the initial # of objects |
09593216 CK |
73 | * |
74 | * Initialize the object and make sure that we can track locked objects. | |
05d24935 RC |
75 | * |
76 | * If nr is non-zero then it is used as the initial objects table size. | |
77 | * In either case, the table will grow (be re-allocated) on demand. | |
09593216 | 78 | */ |
4db102dc | 79 | void drm_exec_init(struct drm_exec *exec, u32 flags, unsigned nr) |
09593216 | 80 | { |
05d24935 RC |
81 | if (!nr) |
82 | nr = PAGE_SIZE / sizeof(void *); | |
83 | ||
09593216 | 84 | exec->flags = flags; |
05d24935 | 85 | exec->objects = kvmalloc_array(nr, sizeof(void *), GFP_KERNEL); |
09593216 CK |
86 | |
87 | /* If allocation here fails, just delay that till the first use */ | |
05d24935 | 88 | exec->max_objects = exec->objects ? nr : 0; |
09593216 CK |
89 | exec->num_objects = 0; |
90 | exec->contended = DRM_EXEC_DUMMY; | |
91 | exec->prelocked = NULL; | |
92 | } | |
93 | EXPORT_SYMBOL(drm_exec_init); | |
94 | ||
95 | /** | |
96 | * drm_exec_fini - finalize a drm_exec object | |
97 | * @exec: the drm_exec object to finalize | |
98 | * | |
99 | * Unlock all locked objects, drop the references to objects and free all memory | |
100 | * used for tracking the state. | |
101 | */ | |
102 | void drm_exec_fini(struct drm_exec *exec) | |
103 | { | |
104 | drm_exec_unlock_all(exec); | |
105 | kvfree(exec->objects); | |
106 | if (exec->contended != DRM_EXEC_DUMMY) { | |
107 | drm_gem_object_put(exec->contended); | |
108 | ww_acquire_fini(&exec->ticket); | |
109 | } | |
110 | } | |
111 | EXPORT_SYMBOL(drm_exec_fini); | |
112 | ||
113 | /** | |
114 | * drm_exec_cleanup - cleanup when contention is detected | |
115 | * @exec: the drm_exec object to cleanup | |
116 | * | |
117 | * Cleanup the current state and return true if we should stay inside the retry | |
118 | * loop, false if there wasn't any contention detected and we can keep the | |
119 | * objects locked. | |
120 | */ | |
121 | bool drm_exec_cleanup(struct drm_exec *exec) | |
122 | { | |
123 | if (likely(!exec->contended)) { | |
124 | ww_acquire_done(&exec->ticket); | |
125 | return false; | |
126 | } | |
127 | ||
128 | if (likely(exec->contended == DRM_EXEC_DUMMY)) { | |
129 | exec->contended = NULL; | |
130 | ww_acquire_init(&exec->ticket, &reservation_ww_class); | |
131 | return true; | |
132 | } | |
133 | ||
134 | drm_exec_unlock_all(exec); | |
135 | exec->num_objects = 0; | |
136 | return true; | |
137 | } | |
138 | EXPORT_SYMBOL(drm_exec_cleanup); | |
139 | ||
140 | /* Track the locked object in the array */ | |
141 | static int drm_exec_obj_locked(struct drm_exec *exec, | |
142 | struct drm_gem_object *obj) | |
143 | { | |
144 | if (unlikely(exec->num_objects == exec->max_objects)) { | |
145 | size_t size = exec->max_objects * sizeof(void *); | |
146 | void *tmp; | |
147 | ||
148 | tmp = kvrealloc(exec->objects, size, size + PAGE_SIZE, | |
149 | GFP_KERNEL); | |
150 | if (!tmp) | |
151 | return -ENOMEM; | |
152 | ||
153 | exec->objects = tmp; | |
154 | exec->max_objects += PAGE_SIZE / sizeof(void *); | |
155 | } | |
156 | drm_gem_object_get(obj); | |
157 | exec->objects[exec->num_objects++] = obj; | |
158 | ||
159 | return 0; | |
160 | } | |
161 | ||
162 | /* Make sure the contended object is locked first */ | |
163 | static int drm_exec_lock_contended(struct drm_exec *exec) | |
164 | { | |
165 | struct drm_gem_object *obj = exec->contended; | |
166 | int ret; | |
167 | ||
168 | if (likely(!obj)) | |
169 | return 0; | |
170 | ||
171 | /* Always cleanup the contention so that error handling can kick in */ | |
172 | exec->contended = NULL; | |
173 | if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) { | |
174 | ret = dma_resv_lock_slow_interruptible(obj->resv, | |
175 | &exec->ticket); | |
176 | if (unlikely(ret)) | |
177 | goto error_dropref; | |
178 | } else { | |
179 | dma_resv_lock_slow(obj->resv, &exec->ticket); | |
180 | } | |
181 | ||
182 | ret = drm_exec_obj_locked(exec, obj); | |
183 | if (unlikely(ret)) | |
184 | goto error_unlock; | |
185 | ||
186 | exec->prelocked = obj; | |
187 | return 0; | |
188 | ||
189 | error_unlock: | |
190 | dma_resv_unlock(obj->resv); | |
191 | ||
192 | error_dropref: | |
193 | drm_gem_object_put(obj); | |
194 | return ret; | |
195 | } | |
196 | ||
197 | /** | |
198 | * drm_exec_lock_obj - lock a GEM object for use | |
199 | * @exec: the drm_exec object with the state | |
200 | * @obj: the GEM object to lock | |
201 | * | |
202 | * Lock a GEM object for use and grab a reference to it. | |
203 | * | |
204 | * Returns: -EDEADLK if a contention is detected, -EALREADY when object is | |
205 | * already locked (can be suppressed by setting the DRM_EXEC_IGNORE_DUPLICATES | |
206 | * flag), -ENOMEM when memory allocation failed and zero for success. | |
207 | */ | |
208 | int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj) | |
209 | { | |
210 | int ret; | |
211 | ||
212 | ret = drm_exec_lock_contended(exec); | |
213 | if (unlikely(ret)) | |
214 | return ret; | |
215 | ||
216 | if (exec->prelocked == obj) { | |
217 | drm_gem_object_put(exec->prelocked); | |
218 | exec->prelocked = NULL; | |
219 | return 0; | |
220 | } | |
221 | ||
222 | if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) | |
223 | ret = dma_resv_lock_interruptible(obj->resv, &exec->ticket); | |
224 | else | |
225 | ret = dma_resv_lock(obj->resv, &exec->ticket); | |
226 | ||
227 | if (unlikely(ret == -EDEADLK)) { | |
228 | drm_gem_object_get(obj); | |
229 | exec->contended = obj; | |
230 | return -EDEADLK; | |
231 | } | |
232 | ||
233 | if (unlikely(ret == -EALREADY) && | |
234 | exec->flags & DRM_EXEC_IGNORE_DUPLICATES) | |
235 | return 0; | |
236 | ||
237 | if (unlikely(ret)) | |
238 | return ret; | |
239 | ||
240 | ret = drm_exec_obj_locked(exec, obj); | |
241 | if (ret) | |
242 | goto error_unlock; | |
243 | ||
244 | return 0; | |
245 | ||
246 | error_unlock: | |
247 | dma_resv_unlock(obj->resv); | |
248 | return ret; | |
249 | } | |
250 | EXPORT_SYMBOL(drm_exec_lock_obj); | |
251 | ||
252 | /** | |
253 | * drm_exec_unlock_obj - unlock a GEM object in this exec context | |
254 | * @exec: the drm_exec object with the state | |
255 | * @obj: the GEM object to unlock | |
256 | * | |
257 | * Unlock the GEM object and remove it from the collection of locked objects. | |
258 | * Should only be used to unlock the most recently locked objects. It's not time | |
259 | * efficient to unlock objects locked long ago. | |
260 | */ | |
261 | void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj) | |
262 | { | |
263 | unsigned int i; | |
264 | ||
265 | for (i = exec->num_objects; i--;) { | |
266 | if (exec->objects[i] == obj) { | |
267 | dma_resv_unlock(obj->resv); | |
268 | for (++i; i < exec->num_objects; ++i) | |
269 | exec->objects[i - 1] = exec->objects[i]; | |
270 | --exec->num_objects; | |
271 | drm_gem_object_put(obj); | |
272 | return; | |
273 | } | |
274 | ||
275 | } | |
276 | } | |
277 | EXPORT_SYMBOL(drm_exec_unlock_obj); | |
278 | ||
279 | /** | |
280 | * drm_exec_prepare_obj - prepare a GEM object for use | |
281 | * @exec: the drm_exec object with the state | |
282 | * @obj: the GEM object to prepare | |
283 | * @num_fences: how many fences to reserve | |
284 | * | |
285 | * Prepare a GEM object for use by locking it and reserving fence slots. | |
286 | * | |
287 | * Returns: -EDEADLK if a contention is detected, -EALREADY when object is | |
288 | * already locked, -ENOMEM when memory allocation failed and zero for success. | |
289 | */ | |
290 | int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj, | |
291 | unsigned int num_fences) | |
292 | { | |
293 | int ret; | |
294 | ||
295 | ret = drm_exec_lock_obj(exec, obj); | |
296 | if (ret) | |
297 | return ret; | |
298 | ||
299 | ret = dma_resv_reserve_fences(obj->resv, num_fences); | |
300 | if (ret) { | |
301 | drm_exec_unlock_obj(exec, obj); | |
302 | return ret; | |
303 | } | |
304 | ||
305 | return 0; | |
306 | } | |
307 | EXPORT_SYMBOL(drm_exec_prepare_obj); | |
308 | ||
309 | /** | |
310 | * drm_exec_prepare_array - helper to prepare an array of objects | |
311 | * @exec: the drm_exec object with the state | |
312 | * @objects: array of GEM object to prepare | |
313 | * @num_objects: number of GEM objects in the array | |
314 | * @num_fences: number of fences to reserve on each GEM object | |
315 | * | |
316 | * Prepares all GEM objects in an array, aborts on first error. | |
317 | * Reserves @num_fences on each GEM object after locking it. | |
318 | * | |
319 | * Returns: -EDEADLOCK on contention, -EALREADY when object is already locked, | |
320 | * -ENOMEM when memory allocation failed and zero for success. | |
321 | */ | |
322 | int drm_exec_prepare_array(struct drm_exec *exec, | |
323 | struct drm_gem_object **objects, | |
324 | unsigned int num_objects, | |
325 | unsigned int num_fences) | |
326 | { | |
327 | int ret; | |
328 | ||
329 | for (unsigned int i = 0; i < num_objects; ++i) { | |
330 | ret = drm_exec_prepare_obj(exec, objects[i], num_fences); | |
331 | if (unlikely(ret)) | |
332 | return ret; | |
333 | } | |
334 | ||
335 | return 0; | |
336 | } | |
337 | EXPORT_SYMBOL(drm_exec_prepare_array); | |
338 | ||
339 | MODULE_DESCRIPTION("DRM execution context"); | |
340 | MODULE_LICENSE("Dual MIT/GPL"); |