drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
  33  * container for a synchronization primitive which can be used by userspace
  34  * to explicitly synchronize GPU commands, can be shared between userspace
  35  * processes, and can be shared between different DRM drivers.
  36  * Their primary use-case is to implement Vulkan fences and semaphores.
  37  * The syncobj userspace API provides ioctls for several operations:
  38  *
  39  *  - Creation and destruction of syncobjs
  40  *  - Import and export of syncobjs to/from a syncobj file descriptor
  41  *  - Import and export a syncobj's underlying fence to/from a sync file
  42  *  - Reset a syncobj (set its fence to NULL)
  43  *  - Signal a syncobj (set a trivially signaled fence)
  44  *  - Wait for a syncobj's fence to appear and be signaled
  45  *
  46  * The syncobj userspace API also provides operations to manipulate a syncobj
  47  * in terms of a timeline of struct &dma_fence_chain rather than a single
  48  * struct &dma_fence, through the following operations:
  49  *
  50  *   - Signal a given point on the timeline
  51  *   - Wait for a given point to appear and/or be signaled
  52  *   - Import and export from/to a given point of a timeline
  53  *
  54  * At it's core, a syncobj is simply a wrapper around a pointer to a struct
  55  * &dma_fence which may be NULL.
  56  * When a syncobj is first created, its pointer is either NULL or a pointer
  57  * to an already signaled fence depending on whether the
  58  * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
  59  * &DRM_IOCTL_SYNCOBJ_CREATE.
  60  *
  61  * If the syncobj is considered as a binary (its state is either signaled or
  62  * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
  63  * the syncobj, the syncobj's fence is replaced with a fence which will be
  64  * signaled by the completion of that work.
  65  * If the syncobj is considered as a timeline primitive, when GPU work is
  66  * enqueued in a DRM driver to signal the a given point of the syncobj, a new
  67  * struct &dma_fence_chain pointing to the DRM driver's fence and also
  68  * pointing to the previous fence that was in the syncobj. The new struct
  69  * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
  70  * completion of the DRM driver's work and also any work associated with the
  71  * fence previously in the syncobj.
  72  *
  73  * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
  74  * time the work is enqueued, it waits on the syncobj's fence before
  75  * submitting the work to hardware. That fence is either :
  76  *
  77  *    - The syncobj's current fence if the syncobj is considered as a binary
  78  *      primitive.
  79  *    - The struct &dma_fence associated with a given point if the syncobj is
  80  *      considered as a timeline primitive.
  81  *
  82  * If the syncobj's fence is NULL or not present in the syncobj's timeline,
  83  * the enqueue operation is expected to fail.
  84  *
  85  * With binary syncobj, all manipulation of the syncobjs's fence happens in
  86  * terms of the current fence at the time the ioctl is called by userspace
  87  * regardless of whether that operation is an immediate host-side operation
  88  * (signal or reset) or or an operation which is enqueued in some driver
  89  * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
  90  * to manipulate a syncobj from the host by resetting its pointer to NULL or
  91  * setting its pointer to a fence which is already signaled.
  92  *
  93  * With a timeline syncobj, all manipulation of the synobj's fence happens in
  94  * terms of a u64 value referring to point in the timeline. See
  95  * dma_fence_chain_find_seqno() to see how a given point is found in the
  96  * timeline.
  97  *
  98  * Note that applications should be careful to always use timeline set of
  99  * ioctl() when dealing with syncobj considered as timeline. Using a binary
 100  * set of ioctl() with a syncobj considered as timeline could result incorrect
 101  * synchronization. The use of binary syncobj is supported through the
 102  * timeline set of ioctl() by using a point value of 0, this will reproduce
 103  * the behavior of the binary set of ioctl() (for example replace the
 104  * syncobj's fence when signaling).
 105  *
 106  *
 107  * Host-side wait on syncobjs
 108  * --------------------------
 109  *
 110  * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
 111  * host-side wait on all of the syncobj fences simultaneously.
 112  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
 113  * all of the syncobj fences to be signaled before it returns.
 114  * Otherwise, it returns once at least one syncobj fence has been signaled
 115  * and the index of a signaled fence is written back to the client.
 116  *
 117  * Unlike the enqueued GPU work dependencies which fail if they see a NULL
 118  * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
 119  * the host-side wait will first wait for the syncobj to receive a non-NULL
 120  * fence and then wait on that fence.
 121  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
 122  * syncobjs in the array has a NULL fence, -EINVAL will be returned.
 123  * Assuming the syncobj starts off with a NULL fence, this allows a client
 124  * to do a host wait in one thread (or process) which waits on GPU work
 125  * submitted in another thread (or process) without having to manually
 126  * synchronize between the two.
 127  * This requirement is inherited from the Vulkan fence API.
 128  *
 129  * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
 130  * handles as well as an array of u64 points and does a host-side wait on all
 131  * of syncobj fences at the given points simultaneously.
 132  *
 133  * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
 134  * fence to materialize on the timeline without waiting for the fence to be
 135  * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
 136  * requirement is inherited from the wait-before-signal behavior required by
 137  * the Vulkan timeline semaphore API.
 138  *
 139  * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
 140  * blocking: an eventfd will be signaled when the syncobj is. This is useful to
 141  * integrate the wait in an event loop.
 142  *
 143  *
 144  * Import/export of syncobjs
 145  * -------------------------
 146  *
 147  * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
 148  * provide two mechanisms for import/export of syncobjs.
 149  *
 150  * The first lets the client import or export an entire syncobj to a file
 151  * descriptor.
 152  * These fd's are opaque and have no other use case, except passing the
 153  * syncobj between processes.
 154  * All exported file descriptors and any syncobj handles created as a
 155  * result of importing those file descriptors own a reference to the
 156  * same underlying struct &drm_syncobj and the syncobj can be used
 157  * persistently across all the processes with which it is shared.
 158  * The syncobj is freed only once the last reference is dropped.
 159  * Unlike dma-buf, importing a syncobj creates a new handle (with its own
 160  * reference) for every import instead of de-duplicating.
 161  * The primary use-case of this persistent import/export is for shared
 162  * Vulkan fences and semaphores.
 163  *
 164  * The second import/export mechanism, which is indicated by
 165  * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
 166  * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
 167  * import/export the syncobj's current fence from/to a &sync_file.
 168  * When a syncobj is exported to a sync file, that sync file wraps the
 169  * sycnobj's fence at the time of export and any later signal or reset
 170  * operations on the syncobj will not affect the exported sync file.
 171  * When a sync file is imported into a syncobj, the syncobj's fence is set
 172  * to the fence wrapped by that sync file.
 173  * Because sync files are immutable, resetting or signaling the syncobj
 174  * will not affect any sync files whose fences have been imported into the
 175  * syncobj.
 176  *
 177  *
 178  * Import/export of timeline points in timeline syncobjs
 179  * -----------------------------------------------------
 180  *
 181  * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
 182  * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
 183  * into another syncobj.
 184  *
 185  * Note that if you want to transfer a struct &dma_fence_chain from a given
 186  * point on a timeline syncobj from/into a binary syncobj, you can use the
 187  * point 0 to mean take/replace the fence in the syncobj.
 188  */
 189
 190 #include <linux/anon_inodes.h>
 191 #include <linux/dma-fence-unwrap.h>
 192 #include <linux/eventfd.h>
 193 #include <linux/file.h>
 194 #include <linux/fs.h>
 195 #include <linux/sched/signal.h>
 196 #include <linux/sync_file.h>
 197 #include <linux/uaccess.h>
 198
 199 #include <drm/drm.h>
 200 #include <drm/drm_drv.h>
 201 #include <drm/drm_file.h>
 202 #include <drm/drm_gem.h>
 203 #include <drm/drm_print.h>
 204 #include <drm/drm_syncobj.h>
 205 #include <drm/drm_utils.h>
 206
 207 #include "drm_internal.h"
 208
 209 struct syncobj_wait_entry {
 210         struct list_head node;
 211         struct task_struct *task;
 212         struct dma_fence *fence;
 213         struct dma_fence_cb fence_cb;
 214         u64    point;
 215 };
 216
 217 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 218                                       struct syncobj_wait_entry *wait);
 219
 220 struct syncobj_eventfd_entry {
 221         struct list_head node;
 222         struct dma_fence *fence;
 223         struct dma_fence_cb fence_cb;
 224         struct drm_syncobj *syncobj;
 225         struct eventfd_ctx *ev_fd_ctx;
 226         u64 point;
 227         u32 flags;
 228 };
 229
 230 static void
 231 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
 232                            struct syncobj_eventfd_entry *entry);
 233
 234 /**
 235  * drm_syncobj_find - lookup and reference a sync object.
 236  * @file_private: drm file private pointer
 237  * @handle: sync object handle to lookup.
 238  *
 239  * Returns a reference to the syncobj pointed to by handle or NULL. The
 240  * reference must be released by calling drm_syncobj_put().
 241  */
 242 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
 243                                      u32 handle)
 244 {
 245         struct drm_syncobj *syncobj;
 246
 247         spin_lock(&file_private->syncobj_table_lock);
 248
 249         /* Check if we currently have a reference on the object */
 250         syncobj = idr_find(&file_private->syncobj_idr, handle);
 251         if (syncobj)
 252                 drm_syncobj_get(syncobj);
 253
 254         spin_unlock(&file_private->syncobj_table_lock);
 255
 256         return syncobj;
 257 }
 258 EXPORT_SYMBOL(drm_syncobj_find);
 259
 260 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
 261                                        struct syncobj_wait_entry *wait)
 262 {
 263         struct dma_fence *fence;
 264
 265         if (wait->fence)
 266                 return;
 267
 268         spin_lock(&syncobj->lock);
 269         /* We've already tried once to get a fence and failed.  Now that we
 270          * have the lock, try one more time just to be sure we don't add a
 271          * callback when a fence has already been set.
 272          */
 273         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
 274         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
 275                 dma_fence_put(fence);
 276                 list_add_tail(&wait->node, &syncobj->cb_list);
 277         } else if (!fence) {
 278                 wait->fence = dma_fence_get_stub();
 279         } else {
 280                 wait->fence = fence;
 281         }
 282         spin_unlock(&syncobj->lock);
 283 }
 284
 285 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
 286                                     struct syncobj_wait_entry *wait)
 287 {
 288         if (!wait->node.next)
 289                 return;
 290
 291         spin_lock(&syncobj->lock);
 292         list_del_init(&wait->node);
 293         spin_unlock(&syncobj->lock);
 294 }
 295
 296 static void
 297 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
 298 {
 299         eventfd_ctx_put(entry->ev_fd_ctx);
 300         dma_fence_put(entry->fence);
 301         /* This happens either inside the syncobj lock, or after the node has
 302          * already been removed from the list.
 303          */
 304         list_del(&entry->node);
 305         kfree(entry);
 306 }
 307
 308 static void
 309 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
 310                         struct syncobj_eventfd_entry *entry)
 311 {
 312         spin_lock(&syncobj->lock);
 313         list_add_tail(&entry->node, &syncobj->ev_fd_list);
 314         syncobj_eventfd_entry_func(syncobj, entry);
 315         spin_unlock(&syncobj->lock);
 316 }
 317
 318 /**
 319  * drm_syncobj_add_point - add new timeline point to the syncobj
 320  * @syncobj: sync object to add timeline point do
 321  * @chain: chain node to use to add the point
 322  * @fence: fence to encapsulate in the chain node
 323  * @point: sequence number to use for the point
 324  *
 325  * Add the chain node as new timeline point to the syncobj.
 326  */
 327 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
 328                            struct dma_fence_chain *chain,
 329                            struct dma_fence *fence,
 330                            uint64_t point)
 331 {
 332         struct syncobj_wait_entry *wait_cur, *wait_tmp;
 333         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 334         struct dma_fence *prev;
 335
 336         dma_fence_get(fence);
 337
 338         spin_lock(&syncobj->lock);
 339
 340         prev = drm_syncobj_fence_get(syncobj);
 341         /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
 342         if (prev && prev->seqno >= point)
 343                 DRM_DEBUG("You are adding an unorder point to timeline!\n");
 344         dma_fence_chain_init(chain, prev, fence, point);
 345         rcu_assign_pointer(syncobj->fence, &chain->base);
 346
 347         list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
 348                 syncobj_wait_syncobj_func(syncobj, wait_cur);
 349         list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 350                 syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
 351         spin_unlock(&syncobj->lock);
 352
 353         /* Walk the chain once to trigger garbage collection */
 354         dma_fence_chain_for_each(fence, prev);
 355         dma_fence_put(prev);
 356 }
 357 EXPORT_SYMBOL(drm_syncobj_add_point);
 358
 359 /**
 360  * drm_syncobj_replace_fence - replace fence in a sync object.
 361  * @syncobj: Sync object to replace fence in
 362  * @fence: fence to install in sync file.
 363  *
 364  * This replaces the fence on a sync object.
 365  */
 366 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 367                                struct dma_fence *fence)
 368 {
 369         struct dma_fence *old_fence;
 370         struct syncobj_wait_entry *wait_cur, *wait_tmp;
 371         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 372
 373         if (fence)
 374                 dma_fence_get(fence);
 375
 376         spin_lock(&syncobj->lock);
 377
 378         old_fence = rcu_dereference_protected(syncobj->fence,
 379                                               lockdep_is_held(&syncobj->lock));
 380         rcu_assign_pointer(syncobj->fence, fence);
 381
 382         if (fence != old_fence) {
 383                 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
 384                         syncobj_wait_syncobj_func(syncobj, wait_cur);
 385                 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 386                         syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
 387         }
 388
 389         spin_unlock(&syncobj->lock);
 390
 391         dma_fence_put(old_fence);
 392 }
 393 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 394
 395 /**
 396  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
 397  * @syncobj: sync object to assign the fence on
 398  *
 399  * Assign a already signaled stub fence to the sync object.
 400  */
 401 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 402 {
 403         struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());
 404
 405         if (!fence)
 406                 return -ENOMEM;
 407
 408         drm_syncobj_replace_fence(syncobj, fence);
 409         dma_fence_put(fence);
 410         return 0;
 411 }
 412
 413 /* 5s default for wait submission */
 414 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
 415 /**
 416  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 417  * @file_private: drm file private pointer
 418  * @handle: sync object handle to lookup.
 419  * @point: timeline point
 420  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
 421  * @fence: out parameter for the fence
 422  *
 423  * This is just a convenience function that combines drm_syncobj_find() and
 424  * drm_syncobj_fence_get().
 425  *
 426  * Returns 0 on success or a negative error value on failure. On success @fence
 427  * contains a reference to the fence, which must be released by calling
 428  * dma_fence_put().
 429  */
 430 int drm_syncobj_find_fence(struct drm_file *file_private,
 431                            u32 handle, u64 point, u64 flags,
 432                            struct dma_fence **fence)
 433 {
 434         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 435         struct syncobj_wait_entry wait;
 436         u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
 437         int ret;
 438
 439         if (!syncobj)
 440                 return -ENOENT;
 441
 442         /* Waiting for userspace with locks help is illegal cause that can
 443          * trivial deadlock with page faults for example. Make lockdep complain
 444          * about it early on.
 445          */
 446         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 447                 might_sleep();
 448                 lockdep_assert_none_held_once();
 449         }
 450
 451         *fence = drm_syncobj_fence_get(syncobj);
 452
 453         if (*fence) {
 454                 ret = dma_fence_chain_find_seqno(fence, point);
 455                 if (!ret) {
 456                         /* If the requested seqno is already signaled
 457                          * drm_syncobj_find_fence may return a NULL
 458                          * fence. To make sure the recipient gets
 459                          * signalled, use a new fence instead.
 460                          */
 461                         if (!*fence)
 462                                 *fence = dma_fence_get_stub();
 463
 464                         goto out;
 465                 }
 466                 dma_fence_put(*fence);
 467         } else {
 468                 ret = -EINVAL;
 469         }
 470
 471         if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 472                 goto out;
 473
 474         memset(&wait, 0, sizeof(wait));
 475         wait.task = current;
 476         wait.point = point;
 477         drm_syncobj_fence_add_wait(syncobj, &wait);
 478
 479         do {
 480                 set_current_state(TASK_INTERRUPTIBLE);
 481                 if (wait.fence) {
 482                         ret = 0;
 483                         break;
 484                 }
 485                 if (timeout == 0) {
 486                         ret = -ETIME;
 487                         break;
 488                 }
 489
 490                 if (signal_pending(current)) {
 491                         ret = -ERESTARTSYS;
 492                         break;
 493                 }
 494
 495                 timeout = schedule_timeout(timeout);
 496         } while (1);
 497
 498         __set_current_state(TASK_RUNNING);
 499         *fence = wait.fence;
 500
 501         if (wait.node.next)
 502                 drm_syncobj_remove_wait(syncobj, &wait);
 503
 504 out:
 505         drm_syncobj_put(syncobj);
 506
 507         return ret;
 508 }
 509 EXPORT_SYMBOL(drm_syncobj_find_fence);
 510
 511 /**
 512  * drm_syncobj_free - free a sync object.
 513  * @kref: kref to free.
 514  *
 515  * Only to be called from kref_put in drm_syncobj_put.
 516  */
 517 void drm_syncobj_free(struct kref *kref)
 518 {
 519         struct drm_syncobj *syncobj = container_of(kref,
 520                                                    struct drm_syncobj,
 521                                                    refcount);
 522         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
 523
 524         drm_syncobj_replace_fence(syncobj, NULL);
 525
 526         list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
 527                 syncobj_eventfd_entry_free(ev_fd_cur);
 528
 529         kfree(syncobj);
 530 }
 531 EXPORT_SYMBOL(drm_syncobj_free);
 532
 533 /**
 534  * drm_syncobj_create - create a new syncobj
 535  * @out_syncobj: returned syncobj
 536  * @flags: DRM_SYNCOBJ_* flags
 537  * @fence: if non-NULL, the syncobj will represent this fence
 538  *
 539  * This is the first function to create a sync object. After creating, drivers
 540  * probably want to make it available to userspace, either through
 541  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 542  *
 543  * Returns 0 on success or a negative error value on failure.
 544  */
 545 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 546                        struct dma_fence *fence)
 547 {
 548         int ret;
 549         struct drm_syncobj *syncobj;
 550
 551         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 552         if (!syncobj)
 553                 return -ENOMEM;
 554
 555         kref_init(&syncobj->refcount);
 556         INIT_LIST_HEAD(&syncobj->cb_list);
 557         INIT_LIST_HEAD(&syncobj->ev_fd_list);
 558         spin_lock_init(&syncobj->lock);
 559
 560         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 561                 ret = drm_syncobj_assign_null_handle(syncobj);
 562                 if (ret < 0) {
 563                         drm_syncobj_put(syncobj);
 564                         return ret;
 565                 }
 566         }
 567
 568         if (fence)
 569                 drm_syncobj_replace_fence(syncobj, fence);
 570
 571         *out_syncobj = syncobj;
 572         return 0;
 573 }
 574 EXPORT_SYMBOL(drm_syncobj_create);
 575
 576 /**
 577  * drm_syncobj_get_handle - get a handle from a syncobj
 578  * @file_private: drm file private pointer
 579  * @syncobj: Sync object to export
 580  * @handle: out parameter with the new handle
 581  *
 582  * Exports a sync object created with drm_syncobj_create() as a handle on
 583  * @file_private to userspace.
 584  *
 585  * Returns 0 on success or a negative error value on failure.
 586  */
 587 int drm_syncobj_get_handle(struct drm_file *file_private,
 588                            struct drm_syncobj *syncobj, u32 *handle)
 589 {
 590         int ret;
 591
 592         /* take a reference to put in the idr */
 593         drm_syncobj_get(syncobj);
 594
 595         idr_preload(GFP_KERNEL);
 596         spin_lock(&file_private->syncobj_table_lock);
 597         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 598         spin_unlock(&file_private->syncobj_table_lock);
 599
 600         idr_preload_end();
 601
 602         if (ret < 0) {
 603                 drm_syncobj_put(syncobj);
 604                 return ret;
 605         }
 606
 607         *handle = ret;
 608         return 0;
 609 }
 610 EXPORT_SYMBOL(drm_syncobj_get_handle);
 611
 612 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 613                                         u32 *handle, uint32_t flags)
 614 {
 615         int ret;
 616         struct drm_syncobj *syncobj;
 617
 618         ret = drm_syncobj_create(&syncobj, flags, NULL);
 619         if (ret)
 620                 return ret;
 621
 622         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 623         drm_syncobj_put(syncobj);
 624         return ret;
 625 }
 626
 627 static int drm_syncobj_destroy(struct drm_file *file_private,
 628                                u32 handle)
 629 {
 630         struct drm_syncobj *syncobj;
 631
 632         spin_lock(&file_private->syncobj_table_lock);
 633         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 634         spin_unlock(&file_private->syncobj_table_lock);
 635
 636         if (!syncobj)
 637                 return -EINVAL;
 638
 639         drm_syncobj_put(syncobj);
 640         return 0;
 641 }
 642
 643 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 644 {
 645         struct drm_syncobj *syncobj = file->private_data;
 646
 647         drm_syncobj_put(syncobj);
 648         return 0;
 649 }
 650
 651 static const struct file_operations drm_syncobj_file_fops = {
 652         .release = drm_syncobj_file_release,
 653 };
 654
 655 /**
 656  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 657  * @syncobj: Sync object to export
 658  * @p_fd: out parameter with the new file descriptor
 659  *
 660  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 661  *
 662  * Returns 0 on success or a negative error value on failure.
 663  */
 664 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 665 {
 666         struct file *file;
 667         int fd;
 668
 669         fd = get_unused_fd_flags(O_CLOEXEC);
 670         if (fd < 0)
 671                 return fd;
 672
 673         file = anon_inode_getfile("syncobj_file",
 674                                   &drm_syncobj_file_fops,
 675                                   syncobj, 0);
 676         if (IS_ERR(file)) {
 677                 put_unused_fd(fd);
 678                 return PTR_ERR(file);
 679         }
 680
 681         drm_syncobj_get(syncobj);
 682         fd_install(fd, file);
 683
 684         *p_fd = fd;
 685         return 0;
 686 }
 687 EXPORT_SYMBOL(drm_syncobj_get_fd);
 688
 689 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 690                                     u32 handle, int *p_fd)
 691 {
 692         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 693         int ret;
 694
 695         if (!syncobj)
 696                 return -EINVAL;
 697
 698         ret = drm_syncobj_get_fd(syncobj, p_fd);
 699         drm_syncobj_put(syncobj);
 700         return ret;
 701 }
 702
 703 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 704                                     int fd, u32 *handle)
 705 {
 706         struct drm_syncobj *syncobj;
 707         struct fd f = fdget(fd);
 708         int ret;
 709
 710         if (!f.file)
 711                 return -EINVAL;
 712
 713         if (f.file->f_op != &drm_syncobj_file_fops) {
 714                 fdput(f);
 715                 return -EINVAL;
 716         }
 717
 718         /* take a reference to put in the idr */
 719         syncobj = f.file->private_data;
 720         drm_syncobj_get(syncobj);
 721
 722         idr_preload(GFP_KERNEL);
 723         spin_lock(&file_private->syncobj_table_lock);
 724         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 725         spin_unlock(&file_private->syncobj_table_lock);
 726         idr_preload_end();
 727
 728         if (ret > 0) {
 729                 *handle = ret;
 730                 ret = 0;
 731         } else
 732                 drm_syncobj_put(syncobj);
 733
 734         fdput(f);
 735         return ret;
 736 }
 737
 738 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 739                                               int fd, int handle)
 740 {
 741         struct dma_fence *fence = sync_file_get_fence(fd);
 742         struct drm_syncobj *syncobj;
 743
 744         if (!fence)
 745                 return -EINVAL;
 746
 747         syncobj = drm_syncobj_find(file_private, handle);
 748         if (!syncobj) {
 749                 dma_fence_put(fence);
 750                 return -ENOENT;
 751         }
 752
 753         drm_syncobj_replace_fence(syncobj, fence);
 754         dma_fence_put(fence);
 755         drm_syncobj_put(syncobj);
 756         return 0;
 757 }
 758
 759 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 760                                         int handle, int *p_fd)
 761 {
 762         int ret;
 763         struct dma_fence *fence;
 764         struct sync_file *sync_file;
 765         int fd = get_unused_fd_flags(O_CLOEXEC);
 766
 767         if (fd < 0)
 768                 return fd;
 769
 770         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
 771         if (ret)
 772                 goto err_put_fd;
 773
 774         sync_file = sync_file_create(fence);
 775
 776         dma_fence_put(fence);
 777
 778         if (!sync_file) {
 779                 ret = -EINVAL;
 780                 goto err_put_fd;
 781         }
 782
 783         fd_install(fd, sync_file->file);
 784
 785         *p_fd = fd;
 786         return 0;
 787 err_put_fd:
 788         put_unused_fd(fd);
 789         return ret;
 790 }
 791 /**
 792  * drm_syncobj_open - initializes syncobj file-private structures at devnode open time
 793  * @file_private: drm file-private structure to set up
 794  *
 795  * Called at device open time, sets up the structure for handling refcounting
 796  * of sync objects.
 797  */
 798 void
 799 drm_syncobj_open(struct drm_file *file_private)
 800 {
 801         idr_init_base(&file_private->syncobj_idr, 1);
 802         spin_lock_init(&file_private->syncobj_table_lock);
 803 }
 804
 805 static int
 806 drm_syncobj_release_handle(int id, void *ptr, void *data)
 807 {
 808         struct drm_syncobj *syncobj = ptr;
 809
 810         drm_syncobj_put(syncobj);
 811         return 0;
 812 }
 813
 814 /**
 815  * drm_syncobj_release - release file-private sync object resources
 816  * @file_private: drm file-private structure to clean up
 817  *
 818  * Called at close time when the filp is going away.
 819  *
 820  * Releases any remaining references on objects by this filp.
 821  */
 822 void
 823 drm_syncobj_release(struct drm_file *file_private)
 824 {
 825         idr_for_each(&file_private->syncobj_idr,
 826                      &drm_syncobj_release_handle, file_private);
 827         idr_destroy(&file_private->syncobj_idr);
 828 }
 829
 830 int
 831 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 832                          struct drm_file *file_private)
 833 {
 834         struct drm_syncobj_create *args = data;
 835
 836         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 837                 return -EOPNOTSUPP;
 838
 839         /* no valid flags yet */
 840         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 841                 return -EINVAL;
 842
 843         return drm_syncobj_create_as_handle(file_private,
 844                                             &args->handle, args->flags);
 845 }
 846
 847 int
 848 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 849                           struct drm_file *file_private)
 850 {
 851         struct drm_syncobj_destroy *args = data;
 852
 853         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 854                 return -EOPNOTSUPP;
 855
 856         /* make sure padding is empty */
 857         if (args->pad)
 858                 return -EINVAL;
 859         return drm_syncobj_destroy(file_private, args->handle);
 860 }
 861
 862 int
 863 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 864                                    struct drm_file *file_private)
 865 {
 866         struct drm_syncobj_handle *args = data;
 867
 868         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 869                 return -EOPNOTSUPP;
 870
 871         if (args->pad)
 872                 return -EINVAL;
 873
 874         if (args->flags != 0 &&
 875             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 876                 return -EINVAL;
 877
 878         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 879                 return drm_syncobj_export_sync_file(file_private, args->handle,
 880                                                     &args->fd);
 881
 882         return drm_syncobj_handle_to_fd(file_private, args->handle,
 883                                         &args->fd);
 884 }
 885
 886 int
 887 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 888                                    struct drm_file *file_private)
 889 {
 890         struct drm_syncobj_handle *args = data;
 891
 892         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 893                 return -EOPNOTSUPP;
 894
 895         if (args->pad)
 896                 return -EINVAL;
 897
 898         if (args->flags != 0 &&
 899             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 900                 return -EINVAL;
 901
 902         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 903                 return drm_syncobj_import_sync_file_fence(file_private,
 904                                                           args->fd,
 905                                                           args->handle);
 906
 907         return drm_syncobj_fd_to_handle(file_private, args->fd,
 908                                         &args->handle);
 909 }
 910
 911 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
 912                                             struct drm_syncobj_transfer *args)
 913 {
 914         struct drm_syncobj *timeline_syncobj = NULL;
 915         struct dma_fence *fence, *tmp;
 916         struct dma_fence_chain *chain;
 917         int ret;
 918
 919         timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 920         if (!timeline_syncobj) {
 921                 return -ENOENT;
 922         }
 923         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 924                                      args->src_point, args->flags,
 925                                      &tmp);
 926         if (ret)
 927                 goto err_put_timeline;
 928
 929         fence = dma_fence_unwrap_merge(tmp);
 930         dma_fence_put(tmp);
 931         if (!fence) {
 932                 ret = -ENOMEM;
 933                 goto err_put_timeline;
 934         }
 935
 936         chain = dma_fence_chain_alloc();
 937         if (!chain) {
 938                 ret = -ENOMEM;
 939                 goto err_free_fence;
 940         }
 941
 942         drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
 943 err_free_fence:
 944         dma_fence_put(fence);
 945 err_put_timeline:
 946         drm_syncobj_put(timeline_syncobj);
 947
 948         return ret;
 949 }
 950
 951 static int
 952 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
 953                                struct drm_syncobj_transfer *args)
 954 {
 955         struct drm_syncobj *binary_syncobj = NULL;
 956         struct dma_fence *fence;
 957         int ret;
 958
 959         binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
 960         if (!binary_syncobj)
 961                 return -ENOENT;
 962         ret = drm_syncobj_find_fence(file_private, args->src_handle,
 963                                      args->src_point, args->flags, &fence);
 964         if (ret)
 965                 goto err;
 966         drm_syncobj_replace_fence(binary_syncobj, fence);
 967         dma_fence_put(fence);
 968 err:
 969         drm_syncobj_put(binary_syncobj);
 970
 971         return ret;
 972 }
 973 int
 974 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
 975                            struct drm_file *file_private)
 976 {
 977         struct drm_syncobj_transfer *args = data;
 978         int ret;
 979
 980         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
 981                 return -EOPNOTSUPP;
 982
 983         if (args->pad)
 984                 return -EINVAL;
 985
 986         if (args->dst_point)
 987                 ret = drm_syncobj_transfer_to_timeline(file_private, args);
 988         else
 989                 ret = drm_syncobj_transfer_to_binary(file_private, args);
 990
 991         return ret;
 992 }
 993
 994 static void syncobj_wait_fence_func(struct dma_fence *fence,
 995                                     struct dma_fence_cb *cb)
 996 {
 997         struct syncobj_wait_entry *wait =
 998                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 999
1000         wake_up_process(wait->task);
1001 }
1002
1003 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1004                                       struct syncobj_wait_entry *wait)
1005 {
1006         struct dma_fence *fence;
1007
1008         /* This happens inside the syncobj lock */
1009         fence = rcu_dereference_protected(syncobj->fence,
1010                                           lockdep_is_held(&syncobj->lock));
1011         dma_fence_get(fence);
1012         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
1013                 dma_fence_put(fence);
1014                 return;
1015         } else if (!fence) {
1016                 wait->fence = dma_fence_get_stub();
1017         } else {
1018                 wait->fence = fence;
1019         }
1020
1021         wake_up_process(wait->task);
1022         list_del_init(&wait->node);
1023 }
1024
1025 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1026                                                   void __user *user_points,
1027                                                   uint32_t count,
1028                                                   uint32_t flags,
1029                                                   signed long timeout,
1030                                                   uint32_t *idx)
1031 {
1032         struct syncobj_wait_entry *entries;
1033         struct dma_fence *fence;
1034         uint64_t *points;
1035         uint32_t signaled_count, i;
1036
1037         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
1038                 lockdep_assert_none_held_once();
1039
1040         points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
1041         if (points == NULL)
1042                 return -ENOMEM;
1043
1044         if (!user_points) {
1045                 memset(points, 0, count * sizeof(uint64_t));
1046
1047         } else if (copy_from_user(points, user_points,
1048                                   sizeof(uint64_t) * count)) {
1049                 timeout = -EFAULT;
1050                 goto err_free_points;
1051         }
1052
1053         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
1054         if (!entries) {
1055                 timeout = -ENOMEM;
1056                 goto err_free_points;
1057         }
1058         /* Walk the list of sync objects and initialize entries.  We do
1059          * this up-front so that we can properly return -EINVAL if there is
1060          * a syncobj with a missing fence and then never have the chance of
1061          * returning -EINVAL again.
1062          */
1063         signaled_count = 0;
1064         for (i = 0; i < count; ++i) {
1065                 struct dma_fence *fence;
1066
1067                 entries[i].task = current;
1068                 entries[i].point = points[i];
1069                 fence = drm_syncobj_fence_get(syncobjs[i]);
1070                 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
1071                         dma_fence_put(fence);
1072                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1073                                 continue;
1074                         } else {
1075                                 timeout = -EINVAL;
1076                                 goto cleanup_entries;
1077                         }
1078                 }
1079
1080                 if (fence)
1081                         entries[i].fence = fence;
1082                 else
1083                         entries[i].fence = dma_fence_get_stub();
1084
1085                 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1086                     dma_fence_is_signaled(entries[i].fence)) {
1087                         if (signaled_count == 0 && idx)
1088                                 *idx = i;
1089                         signaled_count++;
1090                 }
1091         }
1092
1093         if (signaled_count == count ||
1094             (signaled_count > 0 &&
1095              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1096                 goto cleanup_entries;
1097
1098         /* There's a very annoying laxness in the dma_fence API here, in
1099          * that backends are not required to automatically report when a
1100          * fence is signaled prior to fence->ops->enable_signaling() being
1101          * called.  So here if we fail to match signaled_count, we need to
1102          * fallthough and try a 0 timeout wait!
1103          */
1104
1105         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1106                 for (i = 0; i < count; ++i)
1107                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1108         }
1109
1110         do {
1111                 set_current_state(TASK_INTERRUPTIBLE);
1112
1113                 signaled_count = 0;
1114                 for (i = 0; i < count; ++i) {
1115                         fence = entries[i].fence;
1116                         if (!fence)
1117                                 continue;
1118
1119                         if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1120                             dma_fence_is_signaled(fence) ||
1121                             (!entries[i].fence_cb.func &&
1122                              dma_fence_add_callback(fence,
1123                                                     &entries[i].fence_cb,
1124                                                     syncobj_wait_fence_func))) {
1125                                 /* The fence has been signaled */
1126                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1127                                         signaled_count++;
1128                                 } else {
1129                                         if (idx)
1130                                                 *idx = i;
1131                                         goto done_waiting;
1132                                 }
1133                         }
1134                 }
1135
1136                 if (signaled_count == count)
1137                         goto done_waiting;
1138
1139                 if (timeout == 0) {
1140                         timeout = -ETIME;
1141                         goto done_waiting;
1142                 }
1143
1144                 if (signal_pending(current)) {
1145                         timeout = -ERESTARTSYS;
1146                         goto done_waiting;
1147                 }
1148
1149                 timeout = schedule_timeout(timeout);
1150         } while (1);
1151
1152 done_waiting:
1153         __set_current_state(TASK_RUNNING);
1154
1155 cleanup_entries:
1156         for (i = 0; i < count; ++i) {
1157                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1158                 if (entries[i].fence_cb.func)
1159                         dma_fence_remove_callback(entries[i].fence,
1160                                                   &entries[i].fence_cb);
1161                 dma_fence_put(entries[i].fence);
1162         }
1163         kfree(entries);
1164
1165 err_free_points:
1166         kfree(points);
1167
1168         return timeout;
1169 }
1170
1171 /**
1172  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1173  *
1174  * @timeout_nsec: timeout nsec component in ns, 0 for poll
1175  *
1176  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1177  */
1178 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1179 {
1180         ktime_t abs_timeout, now;
1181         u64 timeout_ns, timeout_jiffies64;
1182
1183         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
1184         if (timeout_nsec == 0)
1185                 return 0;
1186
1187         abs_timeout = ns_to_ktime(timeout_nsec);
1188         now = ktime_get();
1189
1190         if (!ktime_after(abs_timeout, now))
1191                 return 0;
1192
1193         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1194
1195         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1196         /*  clamp timeout to avoid infinite timeout */
1197         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1198                 return MAX_SCHEDULE_TIMEOUT - 1;
1199
1200         return timeout_jiffies64 + 1;
1201 }
1202 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1203
1204 static int drm_syncobj_array_wait(struct drm_device *dev,
1205                                   struct drm_file *file_private,
1206                                   struct drm_syncobj_wait *wait,
1207                                   struct drm_syncobj_timeline_wait *timeline_wait,
1208                                   struct drm_syncobj **syncobjs, bool timeline)
1209 {
1210         signed long timeout = 0;
1211         uint32_t first = ~0;
1212
1213         if (!timeline) {
1214                 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1215                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1216                                                          NULL,
1217                                                          wait->count_handles,
1218                                                          wait->flags,
1219                                                          timeout, &first);
1220                 if (timeout < 0)
1221                         return timeout;
1222                 wait->first_signaled = first;
1223         } else {
1224                 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1225                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1226                                                          u64_to_user_ptr(timeline_wait->points),
1227                                                          timeline_wait->count_handles,
1228                                                          timeline_wait->flags,
1229                                                          timeout, &first);
1230                 if (timeout < 0)
1231                         return timeout;
1232                 timeline_wait->first_signaled = first;
1233         }
1234         return 0;
1235 }
1236
1237 static int drm_syncobj_array_find(struct drm_file *file_private,
1238                                   void __user *user_handles,
1239                                   uint32_t count_handles,
1240                                   struct drm_syncobj ***syncobjs_out)
1241 {
1242         uint32_t i, *handles;
1243         struct drm_syncobj **syncobjs;
1244         int ret;
1245
1246         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1247         if (handles == NULL)
1248                 return -ENOMEM;
1249
1250         if (copy_from_user(handles, user_handles,
1251                            sizeof(uint32_t) * count_handles)) {
1252                 ret = -EFAULT;
1253                 goto err_free_handles;
1254         }
1255
1256         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1257         if (syncobjs == NULL) {
1258                 ret = -ENOMEM;
1259                 goto err_free_handles;
1260         }
1261
1262         for (i = 0; i < count_handles; i++) {
1263                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1264                 if (!syncobjs[i]) {
1265                         ret = -ENOENT;
1266                         goto err_put_syncobjs;
1267                 }
1268         }
1269
1270         kfree(handles);
1271         *syncobjs_out = syncobjs;
1272         return 0;
1273
1274 err_put_syncobjs:
1275         while (i-- > 0)
1276                 drm_syncobj_put(syncobjs[i]);
1277         kfree(syncobjs);
1278 err_free_handles:
1279         kfree(handles);
1280
1281         return ret;
1282 }
1283
1284 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1285                                    uint32_t count)
1286 {
1287         uint32_t i;
1288
1289         for (i = 0; i < count; i++)
1290                 drm_syncobj_put(syncobjs[i]);
1291         kfree(syncobjs);
1292 }
1293
1294 int
1295 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1296                        struct drm_file *file_private)
1297 {
1298         struct drm_syncobj_wait *args = data;
1299         struct drm_syncobj **syncobjs;
1300         int ret = 0;
1301
1302         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1303                 return -EOPNOTSUPP;
1304
1305         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1306                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1307                 return -EINVAL;
1308
1309         if (args->count_handles == 0)
1310                 return -EINVAL;
1311
1312         ret = drm_syncobj_array_find(file_private,
1313                                      u64_to_user_ptr(args->handles),
1314                                      args->count_handles,
1315                                      &syncobjs);
1316         if (ret < 0)
1317                 return ret;
1318
1319         ret = drm_syncobj_array_wait(dev, file_private,
1320                                      args, NULL, syncobjs, false);
1321
1322         drm_syncobj_array_free(syncobjs, args->count_handles);
1323
1324         return ret;
1325 }
1326
1327 int
1328 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1329                                 struct drm_file *file_private)
1330 {
1331         struct drm_syncobj_timeline_wait *args = data;
1332         struct drm_syncobj **syncobjs;
1333         int ret = 0;
1334
1335         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1336                 return -EOPNOTSUPP;
1337
1338         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1339                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1340                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1341                 return -EINVAL;
1342
1343         if (args->count_handles == 0)
1344                 return -EINVAL;
1345
1346         ret = drm_syncobj_array_find(file_private,
1347                                      u64_to_user_ptr(args->handles),
1348                                      args->count_handles,
1349                                      &syncobjs);
1350         if (ret < 0)
1351                 return ret;
1352
1353         ret = drm_syncobj_array_wait(dev, file_private,
1354                                      NULL, args, syncobjs, true);
1355
1356         drm_syncobj_array_free(syncobjs, args->count_handles);
1357
1358         return ret;
1359 }
1360
1361 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1362                                              struct dma_fence_cb *cb)
1363 {
1364         struct syncobj_eventfd_entry *entry =
1365                 container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1366
1367         eventfd_signal(entry->ev_fd_ctx, 1);
1368         syncobj_eventfd_entry_free(entry);
1369 }
1370
1371 static void
1372 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1373                            struct syncobj_eventfd_entry *entry)
1374 {
1375         int ret;
1376         struct dma_fence *fence;
1377
1378         /* This happens inside the syncobj lock */
1379         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1380         ret = dma_fence_chain_find_seqno(&fence, entry->point);
1381         if (ret != 0 || !fence) {
1382                 dma_fence_put(fence);
1383                 return;
1384         }
1385
1386         list_del_init(&entry->node);
1387         entry->fence = fence;
1388
1389         if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1390                 eventfd_signal(entry->ev_fd_ctx, 1);
1391                 syncobj_eventfd_entry_free(entry);
1392         } else {
1393                 ret = dma_fence_add_callback(fence, &entry->fence_cb,
1394                                              syncobj_eventfd_entry_fence_func);
1395                 if (ret == -ENOENT) {
1396                         eventfd_signal(entry->ev_fd_ctx, 1);
1397                         syncobj_eventfd_entry_free(entry);
1398                 }
1399         }
1400 }
1401
1402 int
1403 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1404                           struct drm_file *file_private)
1405 {
1406         struct drm_syncobj_eventfd *args = data;
1407         struct drm_syncobj *syncobj;
1408         struct eventfd_ctx *ev_fd_ctx;
1409         struct syncobj_eventfd_entry *entry;
1410
1411         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1412                 return -EOPNOTSUPP;
1413
1414         if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1415                 return -EINVAL;
1416
1417         if (args->pad)
1418                 return -EINVAL;
1419
1420         syncobj = drm_syncobj_find(file_private, args->handle);
1421         if (!syncobj)
1422                 return -ENOENT;
1423
1424         ev_fd_ctx = eventfd_ctx_fdget(args->fd);
1425         if (IS_ERR(ev_fd_ctx))
1426                 return PTR_ERR(ev_fd_ctx);
1427
1428         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1429         if (!entry) {
1430                 eventfd_ctx_put(ev_fd_ctx);
1431                 return -ENOMEM;
1432         }
1433         entry->syncobj = syncobj;
1434         entry->ev_fd_ctx = ev_fd_ctx;
1435         entry->point = args->point;
1436         entry->flags = args->flags;
1437
1438         drm_syncobj_add_eventfd(syncobj, entry);
1439         drm_syncobj_put(syncobj);
1440
1441         return 0;
1442 }
1443
1444 int
1445 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1446                         struct drm_file *file_private)
1447 {
1448         struct drm_syncobj_array *args = data;
1449         struct drm_syncobj **syncobjs;
1450         uint32_t i;
1451         int ret;
1452
1453         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1454                 return -EOPNOTSUPP;
1455
1456         if (args->pad != 0)
1457                 return -EINVAL;
1458
1459         if (args->count_handles == 0)
1460                 return -EINVAL;
1461
1462         ret = drm_syncobj_array_find(file_private,
1463                                      u64_to_user_ptr(args->handles),
1464                                      args->count_handles,
1465                                      &syncobjs);
1466         if (ret < 0)
1467                 return ret;
1468
1469         for (i = 0; i < args->count_handles; i++)
1470                 drm_syncobj_replace_fence(syncobjs[i], NULL);
1471
1472         drm_syncobj_array_free(syncobjs, args->count_handles);
1473
1474         return 0;
1475 }
1476
1477 int
1478 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1479                          struct drm_file *file_private)
1480 {
1481         struct drm_syncobj_array *args = data;
1482         struct drm_syncobj **syncobjs;
1483         uint32_t i;
1484         int ret;
1485
1486         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1487                 return -EOPNOTSUPP;
1488
1489         if (args->pad != 0)
1490                 return -EINVAL;
1491
1492         if (args->count_handles == 0)
1493                 return -EINVAL;
1494
1495         ret = drm_syncobj_array_find(file_private,
1496                                      u64_to_user_ptr(args->handles),
1497                                      args->count_handles,
1498                                      &syncobjs);
1499         if (ret < 0)
1500                 return ret;
1501
1502         for (i = 0; i < args->count_handles; i++) {
1503                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1504                 if (ret < 0)
1505                         break;
1506         }
1507
1508         drm_syncobj_array_free(syncobjs, args->count_handles);
1509
1510         return ret;
1511 }
1512
1513 int
1514 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1515                                   struct drm_file *file_private)
1516 {
1517         struct drm_syncobj_timeline_array *args = data;
1518         struct drm_syncobj **syncobjs;
1519         struct dma_fence_chain **chains;
1520         uint64_t *points;
1521         uint32_t i, j;
1522         int ret;
1523
1524         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1525                 return -EOPNOTSUPP;
1526
1527         if (args->flags != 0)
1528                 return -EINVAL;
1529
1530         if (args->count_handles == 0)
1531                 return -EINVAL;
1532
1533         ret = drm_syncobj_array_find(file_private,
1534                                      u64_to_user_ptr(args->handles),
1535                                      args->count_handles,
1536                                      &syncobjs);
1537         if (ret < 0)
1538                 return ret;
1539
1540         points = kmalloc_array(args->count_handles, sizeof(*points),
1541                                GFP_KERNEL);
1542         if (!points) {
1543                 ret = -ENOMEM;
1544                 goto out;
1545         }
1546         if (!u64_to_user_ptr(args->points)) {
1547                 memset(points, 0, args->count_handles * sizeof(uint64_t));
1548         } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1549                                   sizeof(uint64_t) * args->count_handles)) {
1550                 ret = -EFAULT;
1551                 goto err_points;
1552         }
1553
1554         chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1555         if (!chains) {
1556                 ret = -ENOMEM;
1557                 goto err_points;
1558         }
1559         for (i = 0; i < args->count_handles; i++) {
1560                 chains[i] = dma_fence_chain_alloc();
1561                 if (!chains[i]) {
1562                         for (j = 0; j < i; j++)
1563                                 dma_fence_chain_free(chains[j]);
1564                         ret = -ENOMEM;
1565                         goto err_chains;
1566                 }
1567         }
1568
1569         for (i = 0; i < args->count_handles; i++) {
1570                 struct dma_fence *fence = dma_fence_get_stub();
1571
1572                 drm_syncobj_add_point(syncobjs[i], chains[i],
1573                                       fence, points[i]);
1574                 dma_fence_put(fence);
1575         }
1576 err_chains:
1577         kfree(chains);
1578 err_points:
1579         kfree(points);
1580 out:
1581         drm_syncobj_array_free(syncobjs, args->count_handles);
1582
1583         return ret;
1584 }
1585
1586 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1587                             struct drm_file *file_private)
1588 {
1589         struct drm_syncobj_timeline_array *args = data;
1590         struct drm_syncobj **syncobjs;
1591         uint64_t __user *points = u64_to_user_ptr(args->points);
1592         uint32_t i;
1593         int ret;
1594
1595         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1596                 return -EOPNOTSUPP;
1597
1598         if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1599                 return -EINVAL;
1600
1601         if (args->count_handles == 0)
1602                 return -EINVAL;
1603
1604         ret = drm_syncobj_array_find(file_private,
1605                                      u64_to_user_ptr(args->handles),
1606                                      args->count_handles,
1607                                      &syncobjs);
1608         if (ret < 0)
1609                 return ret;
1610
1611         for (i = 0; i < args->count_handles; i++) {
1612                 struct dma_fence_chain *chain;
1613                 struct dma_fence *fence;
1614                 uint64_t point;
1615
1616                 fence = drm_syncobj_fence_get(syncobjs[i]);
1617                 chain = to_dma_fence_chain(fence);
1618                 if (chain) {
1619                         struct dma_fence *iter, *last_signaled =
1620                                 dma_fence_get(fence);
1621
1622                         if (args->flags &
1623                             DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1624                                 point = fence->seqno;
1625                         } else {
1626                                 dma_fence_chain_for_each(iter, fence) {
1627                                         if (iter->context != fence->context) {
1628                                                 dma_fence_put(iter);
1629                                                 /* It is most likely that timeline has
1630                                                 * unorder points. */
1631                                                 break;
1632                                         }
1633                                         dma_fence_put(last_signaled);
1634                                         last_signaled = dma_fence_get(iter);
1635                                 }
1636                                 point = dma_fence_is_signaled(last_signaled) ?
1637                                         last_signaled->seqno :
1638                                         to_dma_fence_chain(last_signaled)->prev_seqno;
1639                         }
1640                         dma_fence_put(last_signaled);
1641                 } else {
1642                         point = 0;
1643                 }
1644                 dma_fence_put(fence);
1645                 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1646                 ret = ret ? -EFAULT : 0;
1647                 if (ret)
1648                         break;
1649         }
1650         drm_syncobj_array_free(syncobjs, args->count_handles);
1651
1652         return ret;
1653 }