2 * Copyright 2017 Red Hat
3 * Parts ported from amdgpu (fence wait code).
4 * Copyright 2016 Advanced Micro Devices, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
32 * DRM synchronisation objects (syncobj) are a persistent objects,
33 * that contain an optional fence. The fence can be updated with a new
36 * syncobj's can be waited upon, where it will wait for the underlying
39 * syncobj's can be export to fd's and back, these fd's are opaque and
40 * have no other use case, except passing the syncobj between processes.
42 * Their primary use-case is to implement Vulkan fences and semaphores.
44 * syncobj have a kref reference count, but also have an optional file.
45 * The file is only created once the syncobj is exported.
46 * The file takes a reference on the kref.
50 #include <linux/file.h>
52 #include <linux/anon_inodes.h>
53 #include <linux/sync_file.h>
54 #include <linux/sched/signal.h>
56 #include "drm_internal.h"
57 #include <drm/drm_syncobj.h>
60 * drm_syncobj_find - lookup and reference a sync object.
61 * @file_private: drm file private pointer
62 * @handle: sync object handle to lookup.
64 * Returns a reference to the syncobj pointed to by handle or NULL.
66 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
69 struct drm_syncobj *syncobj;
71 spin_lock(&file_private->syncobj_table_lock);
73 /* Check if we currently have a reference on the object */
74 syncobj = idr_find(&file_private->syncobj_idr, handle);
76 drm_syncobj_get(syncobj);
78 spin_unlock(&file_private->syncobj_table_lock);
82 EXPORT_SYMBOL(drm_syncobj_find);
84 static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
85 struct drm_syncobj_cb *cb,
86 drm_syncobj_func_t func)
89 list_add_tail(&cb->node, &syncobj->cb_list);
92 static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
93 struct dma_fence **fence,
94 struct drm_syncobj_cb *cb,
95 drm_syncobj_func_t func)
99 *fence = drm_syncobj_fence_get(syncobj);
103 spin_lock(&syncobj->lock);
104 /* We've already tried once to get a fence and failed. Now that we
105 * have the lock, try one more time just to be sure we don't add a
106 * callback when a fence has already been set.
108 if (syncobj->fence) {
109 *fence = dma_fence_get(syncobj->fence);
113 drm_syncobj_add_callback_locked(syncobj, cb, func);
116 spin_unlock(&syncobj->lock);
122 * drm_syncobj_add_callback - adds a callback to syncobj::cb_list
123 * @syncobj: Sync object to which to add the callback
124 * @cb: Callback to add
125 * @func: Func to use when initializing the drm_syncobj_cb struct
127 * This adds a callback to be called next time the fence is replaced
129 void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
130 struct drm_syncobj_cb *cb,
131 drm_syncobj_func_t func)
133 spin_lock(&syncobj->lock);
134 drm_syncobj_add_callback_locked(syncobj, cb, func);
135 spin_unlock(&syncobj->lock);
137 EXPORT_SYMBOL(drm_syncobj_add_callback);
140 * drm_syncobj_add_callback - removes a callback to syncobj::cb_list
141 * @syncobj: Sync object from which to remove the callback
142 * @cb: Callback to remove
144 void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
145 struct drm_syncobj_cb *cb)
147 spin_lock(&syncobj->lock);
148 list_del_init(&cb->node);
149 spin_unlock(&syncobj->lock);
151 EXPORT_SYMBOL(drm_syncobj_remove_callback);
154 * drm_syncobj_replace_fence - replace fence in a sync object.
155 * @syncobj: Sync object to replace fence in
156 * @fence: fence to install in sync file.
158 * This replaces the fence on a sync object.
160 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
161 struct dma_fence *fence)
163 struct dma_fence *old_fence;
164 struct drm_syncobj_cb *cur, *tmp;
167 dma_fence_get(fence);
169 spin_lock(&syncobj->lock);
171 old_fence = syncobj->fence;
172 syncobj->fence = fence;
174 if (fence != old_fence) {
175 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
176 list_del_init(&cur->node);
177 cur->func(syncobj, cur);
181 spin_unlock(&syncobj->lock);
183 dma_fence_put(old_fence);
185 EXPORT_SYMBOL(drm_syncobj_replace_fence);
187 struct drm_syncobj_null_fence {
188 struct dma_fence base;
192 static const char *drm_syncobj_null_fence_get_name(struct dma_fence *fence)
194 return "syncobjnull";
197 static bool drm_syncobj_null_fence_enable_signaling(struct dma_fence *fence)
199 dma_fence_enable_sw_signaling(fence);
200 return !dma_fence_is_signaled(fence);
203 static const struct dma_fence_ops drm_syncobj_null_fence_ops = {
204 .get_driver_name = drm_syncobj_null_fence_get_name,
205 .get_timeline_name = drm_syncobj_null_fence_get_name,
206 .enable_signaling = drm_syncobj_null_fence_enable_signaling,
207 .wait = dma_fence_default_wait,
211 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
213 struct drm_syncobj_null_fence *fence;
214 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
218 spin_lock_init(&fence->lock);
219 dma_fence_init(&fence->base, &drm_syncobj_null_fence_ops,
221 dma_fence_signal(&fence->base);
223 drm_syncobj_replace_fence(syncobj, &fence->base);
225 dma_fence_put(&fence->base);
230 int drm_syncobj_find_fence(struct drm_file *file_private,
232 struct dma_fence **fence)
234 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
240 *fence = drm_syncobj_fence_get(syncobj);
244 drm_syncobj_put(syncobj);
247 EXPORT_SYMBOL(drm_syncobj_find_fence);
250 * drm_syncobj_free - free a sync object.
251 * @kref: kref to free.
253 * Only to be called from kref_put in drm_syncobj_put.
255 void drm_syncobj_free(struct kref *kref)
257 struct drm_syncobj *syncobj = container_of(kref,
260 drm_syncobj_replace_fence(syncobj, NULL);
263 EXPORT_SYMBOL(drm_syncobj_free);
266 * drm_syncobj_create - create a new syncobj
267 * @out_syncobj: returned syncobj
268 * @flags: DRM_SYNCOBJ_* flags
269 * @fence: if non-NULL, the syncobj will represent this fence
271 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
272 struct dma_fence *fence)
275 struct drm_syncobj *syncobj;
277 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
281 kref_init(&syncobj->refcount);
282 INIT_LIST_HEAD(&syncobj->cb_list);
283 spin_lock_init(&syncobj->lock);
285 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
286 ret = drm_syncobj_assign_null_handle(syncobj);
288 drm_syncobj_put(syncobj);
294 drm_syncobj_replace_fence(syncobj, fence);
296 *out_syncobj = syncobj;
299 EXPORT_SYMBOL(drm_syncobj_create);
302 * drm_syncobj_get_handle - get a handle from a syncobj
304 int drm_syncobj_get_handle(struct drm_file *file_private,
305 struct drm_syncobj *syncobj, u32 *handle)
309 /* take a reference to put in the idr */
310 drm_syncobj_get(syncobj);
312 idr_preload(GFP_KERNEL);
313 spin_lock(&file_private->syncobj_table_lock);
314 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
315 spin_unlock(&file_private->syncobj_table_lock);
320 drm_syncobj_put(syncobj);
327 EXPORT_SYMBOL(drm_syncobj_get_handle);
329 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
330 u32 *handle, uint32_t flags)
333 struct drm_syncobj *syncobj;
335 ret = drm_syncobj_create(&syncobj, flags, NULL);
339 ret = drm_syncobj_get_handle(file_private, syncobj, handle);
340 drm_syncobj_put(syncobj);
344 static int drm_syncobj_destroy(struct drm_file *file_private,
347 struct drm_syncobj *syncobj;
349 spin_lock(&file_private->syncobj_table_lock);
350 syncobj = idr_remove(&file_private->syncobj_idr, handle);
351 spin_unlock(&file_private->syncobj_table_lock);
356 drm_syncobj_put(syncobj);
360 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
362 struct drm_syncobj *syncobj = file->private_data;
364 drm_syncobj_put(syncobj);
368 static const struct file_operations drm_syncobj_file_fops = {
369 .release = drm_syncobj_file_release,
372 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
377 fd = get_unused_fd_flags(O_CLOEXEC);
381 file = anon_inode_getfile("syncobj_file",
382 &drm_syncobj_file_fops,
386 return PTR_ERR(file);
389 drm_syncobj_get(syncobj);
390 fd_install(fd, file);
395 EXPORT_SYMBOL(drm_syncobj_get_fd);
397 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
398 u32 handle, int *p_fd)
400 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
406 ret = drm_syncobj_get_fd(syncobj, p_fd);
407 drm_syncobj_put(syncobj);
411 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
414 struct drm_syncobj *syncobj;
422 if (file->f_op != &drm_syncobj_file_fops) {
427 /* take a reference to put in the idr */
428 syncobj = file->private_data;
429 drm_syncobj_get(syncobj);
431 idr_preload(GFP_KERNEL);
432 spin_lock(&file_private->syncobj_table_lock);
433 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
434 spin_unlock(&file_private->syncobj_table_lock);
441 drm_syncobj_put(syncobj);
447 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
450 struct dma_fence *fence = sync_file_get_fence(fd);
451 struct drm_syncobj *syncobj;
456 syncobj = drm_syncobj_find(file_private, handle);
458 dma_fence_put(fence);
462 drm_syncobj_replace_fence(syncobj, fence);
463 dma_fence_put(fence);
464 drm_syncobj_put(syncobj);
468 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
469 int handle, int *p_fd)
472 struct dma_fence *fence;
473 struct sync_file *sync_file;
474 int fd = get_unused_fd_flags(O_CLOEXEC);
479 ret = drm_syncobj_find_fence(file_private, handle, &fence);
483 sync_file = sync_file_create(fence);
485 dma_fence_put(fence);
492 fd_install(fd, sync_file->file);
501 * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
502 * @file_private: drm file-private structure to set up
504 * Called at device open time, sets up the structure for handling refcounting
508 drm_syncobj_open(struct drm_file *file_private)
510 idr_init(&file_private->syncobj_idr);
511 spin_lock_init(&file_private->syncobj_table_lock);
515 drm_syncobj_release_handle(int id, void *ptr, void *data)
517 struct drm_syncobj *syncobj = ptr;
519 drm_syncobj_put(syncobj);
524 * drm_syncobj_release - release file-private sync object resources
525 * @file_private: drm file-private structure to clean up
527 * Called at close time when the filp is going away.
529 * Releases any remaining references on objects by this filp.
532 drm_syncobj_release(struct drm_file *file_private)
534 idr_for_each(&file_private->syncobj_idr,
535 &drm_syncobj_release_handle, file_private);
536 idr_destroy(&file_private->syncobj_idr);
540 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
541 struct drm_file *file_private)
543 struct drm_syncobj_create *args = data;
545 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
548 /* no valid flags yet */
549 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
552 return drm_syncobj_create_as_handle(file_private,
553 &args->handle, args->flags);
557 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
558 struct drm_file *file_private)
560 struct drm_syncobj_destroy *args = data;
562 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
565 /* make sure padding is empty */
568 return drm_syncobj_destroy(file_private, args->handle);
572 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
573 struct drm_file *file_private)
575 struct drm_syncobj_handle *args = data;
577 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
583 if (args->flags != 0 &&
584 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
587 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
588 return drm_syncobj_export_sync_file(file_private, args->handle,
591 return drm_syncobj_handle_to_fd(file_private, args->handle,
596 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
597 struct drm_file *file_private)
599 struct drm_syncobj_handle *args = data;
601 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
607 if (args->flags != 0 &&
608 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
611 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
612 return drm_syncobj_import_sync_file_fence(file_private,
616 return drm_syncobj_fd_to_handle(file_private, args->fd,
620 struct syncobj_wait_entry {
621 struct task_struct *task;
622 struct dma_fence *fence;
623 struct dma_fence_cb fence_cb;
624 struct drm_syncobj_cb syncobj_cb;
627 static void syncobj_wait_fence_func(struct dma_fence *fence,
628 struct dma_fence_cb *cb)
630 struct syncobj_wait_entry *wait =
631 container_of(cb, struct syncobj_wait_entry, fence_cb);
633 wake_up_process(wait->task);
636 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
637 struct drm_syncobj_cb *cb)
639 struct syncobj_wait_entry *wait =
640 container_of(cb, struct syncobj_wait_entry, syncobj_cb);
642 /* This happens inside the syncobj lock */
643 wait->fence = dma_fence_get(syncobj->fence);
644 wake_up_process(wait->task);
647 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
653 struct syncobj_wait_entry *entries;
654 struct dma_fence *fence;
656 uint32_t signaled_count, i;
658 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
662 /* Walk the list of sync objects and initialize entries. We do
663 * this up-front so that we can properly return -EINVAL if there is
664 * a syncobj with a missing fence and then never have the chance of
665 * returning -EINVAL again.
668 for (i = 0; i < count; ++i) {
669 entries[i].task = current;
670 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
671 if (!entries[i].fence) {
672 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
676 goto cleanup_entries;
680 if (dma_fence_is_signaled(entries[i].fence)) {
681 if (signaled_count == 0 && idx)
687 /* Initialize ret to the max of timeout and 1. That way, the
688 * default return value indicates a successful wait and not a
691 ret = max_t(signed long, timeout, 1);
693 if (signaled_count == count ||
694 (signaled_count > 0 &&
695 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
696 goto cleanup_entries;
698 /* There's a very annoying laxness in the dma_fence API here, in
699 * that backends are not required to automatically report when a
700 * fence is signaled prior to fence->ops->enable_signaling() being
701 * called. So here if we fail to match signaled_count, we need to
702 * fallthough and try a 0 timeout wait!
705 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
706 for (i = 0; i < count; ++i) {
707 drm_syncobj_fence_get_or_add_callback(syncobjs[i],
709 &entries[i].syncobj_cb,
710 syncobj_wait_syncobj_func);
715 set_current_state(TASK_INTERRUPTIBLE);
718 for (i = 0; i < count; ++i) {
719 fence = entries[i].fence;
723 if (dma_fence_is_signaled(fence) ||
724 (!entries[i].fence_cb.func &&
725 dma_fence_add_callback(fence,
726 &entries[i].fence_cb,
727 syncobj_wait_fence_func))) {
728 /* The fence has been signaled */
729 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
739 if (signaled_count == count)
743 /* If we are doing a 0 timeout wait and we got
744 * here, then we just timed out.
750 ret = schedule_timeout(ret);
752 if (ret > 0 && signal_pending(current))
757 __set_current_state(TASK_RUNNING);
760 for (i = 0; i < count; ++i) {
761 if (entries[i].syncobj_cb.func)
762 drm_syncobj_remove_callback(syncobjs[i],
763 &entries[i].syncobj_cb);
764 if (entries[i].fence_cb.func)
765 dma_fence_remove_callback(entries[i].fence,
766 &entries[i].fence_cb);
767 dma_fence_put(entries[i].fence);
775 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
777 * @timeout_nsec: timeout nsec component in ns, 0 for poll
779 * Calculate the timeout in jiffies from an absolute time in sec/nsec.
781 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
783 ktime_t abs_timeout, now;
784 u64 timeout_ns, timeout_jiffies64;
786 /* make 0 timeout means poll - absolute 0 doesn't seem valid */
787 if (timeout_nsec == 0)
790 abs_timeout = ns_to_ktime(timeout_nsec);
793 if (!ktime_after(abs_timeout, now))
796 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
798 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
799 /* clamp timeout to avoid infinite timeout */
800 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
801 return MAX_SCHEDULE_TIMEOUT - 1;
803 return timeout_jiffies64 + 1;
806 static int drm_syncobj_array_wait(struct drm_device *dev,
807 struct drm_file *file_private,
808 struct drm_syncobj_wait *wait,
809 struct drm_syncobj **syncobjs)
811 signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
815 ret = drm_syncobj_array_wait_timeout(syncobjs,
822 wait->first_signaled = first;
828 static int drm_syncobj_array_find(struct drm_file *file_private,
829 void __user *user_handles,
830 uint32_t count_handles,
831 struct drm_syncobj ***syncobjs_out)
833 uint32_t i, *handles;
834 struct drm_syncobj **syncobjs;
837 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
841 if (copy_from_user(handles, user_handles,
842 sizeof(uint32_t) * count_handles)) {
844 goto err_free_handles;
847 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
848 if (syncobjs == NULL) {
850 goto err_free_handles;
853 for (i = 0; i < count_handles; i++) {
854 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
857 goto err_put_syncobjs;
862 *syncobjs_out = syncobjs;
867 drm_syncobj_put(syncobjs[i]);
875 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
879 for (i = 0; i < count; i++)
880 drm_syncobj_put(syncobjs[i]);
885 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
886 struct drm_file *file_private)
888 struct drm_syncobj_wait *args = data;
889 struct drm_syncobj **syncobjs;
892 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
895 if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
896 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
899 if (args->count_handles == 0)
902 ret = drm_syncobj_array_find(file_private,
903 u64_to_user_ptr(args->handles),
909 ret = drm_syncobj_array_wait(dev, file_private,
912 drm_syncobj_array_free(syncobjs, args->count_handles);
918 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
919 struct drm_file *file_private)
921 struct drm_syncobj_array *args = data;
922 struct drm_syncobj **syncobjs;
926 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
932 if (args->count_handles == 0)
935 ret = drm_syncobj_array_find(file_private,
936 u64_to_user_ptr(args->handles),
942 for (i = 0; i < args->count_handles; i++)
943 drm_syncobj_replace_fence(syncobjs[i], NULL);
945 drm_syncobj_array_free(syncobjs, args->count_handles);
951 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
952 struct drm_file *file_private)
954 struct drm_syncobj_array *args = data;
955 struct drm_syncobj **syncobjs;
959 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
965 if (args->count_handles == 0)
968 ret = drm_syncobj_array_find(file_private,
969 u64_to_user_ptr(args->handles),
975 for (i = 0; i < args->count_handles; i++) {
976 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
981 drm_syncobj_array_free(syncobjs, args->count_handles);