drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj) are a persistent objects,
  33  * that contain an optional fence. The fence can be updated with a new
  34  * fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 /**
  60  * drm_syncobj_find - lookup and reference a sync object.
  61  * @file_private: drm file private pointer
  62  * @handle: sync object handle to lookup.
  63  *
  64  * Returns a reference to the syncobj pointed to by handle or NULL.
  65  */
  66 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  67                                      u32 handle)
  68 {
  69         struct drm_syncobj *syncobj;
  70
  71         spin_lock(&file_private->syncobj_table_lock);
  72
  73         /* Check if we currently have a reference on the object */
  74         syncobj = idr_find(&file_private->syncobj_idr, handle);
  75         if (syncobj)
  76                 drm_syncobj_get(syncobj);
  77
  78         spin_unlock(&file_private->syncobj_table_lock);
  79
  80         return syncobj;
  81 }
  82 EXPORT_SYMBOL(drm_syncobj_find);
  83
  84 static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
  85                                             struct drm_syncobj_cb *cb,
  86                                             drm_syncobj_func_t func)
  87 {
  88         cb->func = func;
  89         list_add_tail(&cb->node, &syncobj->cb_list);
  90 }
  91
  92 static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
  93                                                  struct dma_fence **fence,
  94                                                  struct drm_syncobj_cb *cb,
  95                                                  drm_syncobj_func_t func)
  96 {
  97         int ret;
  98
  99         *fence = drm_syncobj_fence_get(syncobj);
 100         if (*fence)
 101                 return 1;
 102
 103         spin_lock(&syncobj->lock);
 104         /* We've already tried once to get a fence and failed.  Now that we
 105          * have the lock, try one more time just to be sure we don't add a
 106          * callback when a fence has already been set.
 107          */
 108         if (syncobj->fence) {
 109                 *fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 110                                                                  lockdep_is_held(&syncobj->lock)));
 111                 ret = 1;
 112         } else {
 113                 *fence = NULL;
 114                 drm_syncobj_add_callback_locked(syncobj, cb, func);
 115                 ret = 0;
 116         }
 117         spin_unlock(&syncobj->lock);
 118
 119         return ret;
 120 }
 121
 122 /**
 123  * drm_syncobj_add_callback - adds a callback to syncobj::cb_list
 124  * @syncobj: Sync object to which to add the callback
 125  * @cb: Callback to add
 126  * @func: Func to use when initializing the drm_syncobj_cb struct
 127  *
 128  * This adds a callback to be called next time the fence is replaced
 129  */
 130 void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
 131                               struct drm_syncobj_cb *cb,
 132                               drm_syncobj_func_t func)
 133 {
 134         spin_lock(&syncobj->lock);
 135         drm_syncobj_add_callback_locked(syncobj, cb, func);
 136         spin_unlock(&syncobj->lock);
 137 }
 138 EXPORT_SYMBOL(drm_syncobj_add_callback);
 139
 140 /**
 141  * drm_syncobj_add_callback - removes a callback to syncobj::cb_list
 142  * @syncobj: Sync object from which to remove the callback
 143  * @cb: Callback to remove
 144  */
 145 void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
 146                                  struct drm_syncobj_cb *cb)
 147 {
 148         spin_lock(&syncobj->lock);
 149         list_del_init(&cb->node);
 150         spin_unlock(&syncobj->lock);
 151 }
 152 EXPORT_SYMBOL(drm_syncobj_remove_callback);
 153
 154 /**
 155  * drm_syncobj_replace_fence - replace fence in a sync object.
 156  * @syncobj: Sync object to replace fence in
 157  * @fence: fence to install in sync file.
 158  *
 159  * This replaces the fence on a sync object.
 160  */
 161 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 162                                struct dma_fence *fence)
 163 {
 164         struct dma_fence *old_fence;
 165         struct drm_syncobj_cb *cur, *tmp;
 166
 167         if (fence)
 168                 dma_fence_get(fence);
 169
 170         spin_lock(&syncobj->lock);
 171
 172         old_fence = rcu_dereference_protected(syncobj->fence,
 173                                               lockdep_is_held(&syncobj->lock));
 174         rcu_assign_pointer(syncobj->fence, fence);
 175
 176         if (fence != old_fence) {
 177                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
 178                         list_del_init(&cur->node);
 179                         cur->func(syncobj, cur);
 180                 }
 181         }
 182
 183         spin_unlock(&syncobj->lock);
 184
 185         dma_fence_put(old_fence);
 186 }
 187 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 188
 189 struct drm_syncobj_null_fence {
 190         struct dma_fence base;
 191         spinlock_t lock;
 192 };
 193
 194 static const char *drm_syncobj_null_fence_get_name(struct dma_fence *fence)
 195 {
 196         return "syncobjnull";
 197 }
 198
 199 static bool drm_syncobj_null_fence_enable_signaling(struct dma_fence *fence)
 200 {
 201     dma_fence_enable_sw_signaling(fence);
 202     return !dma_fence_is_signaled(fence);
 203 }
 204
 205 static const struct dma_fence_ops drm_syncobj_null_fence_ops = {
 206         .get_driver_name = drm_syncobj_null_fence_get_name,
 207         .get_timeline_name = drm_syncobj_null_fence_get_name,
 208         .enable_signaling = drm_syncobj_null_fence_enable_signaling,
 209         .wait = dma_fence_default_wait,
 210         .release = NULL,
 211 };
 212
 213 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 214 {
 215         struct drm_syncobj_null_fence *fence;
 216         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 217         if (fence == NULL)
 218                 return -ENOMEM;
 219
 220         spin_lock_init(&fence->lock);
 221         dma_fence_init(&fence->base, &drm_syncobj_null_fence_ops,
 222                        &fence->lock, 0, 0);
 223         dma_fence_signal(&fence->base);
 224
 225         drm_syncobj_replace_fence(syncobj, &fence->base);
 226
 227         dma_fence_put(&fence->base);
 228
 229         return 0;
 230 }
 231
 232 int drm_syncobj_find_fence(struct drm_file *file_private,
 233                            u32 handle,
 234                            struct dma_fence **fence)
 235 {
 236         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 237         int ret = 0;
 238
 239         if (!syncobj)
 240                 return -ENOENT;
 241
 242         *fence = drm_syncobj_fence_get(syncobj);
 243         if (!*fence) {
 244                 ret = -EINVAL;
 245         }
 246         drm_syncobj_put(syncobj);
 247         return ret;
 248 }
 249 EXPORT_SYMBOL(drm_syncobj_find_fence);
 250
 251 /**
 252  * drm_syncobj_free - free a sync object.
 253  * @kref: kref to free.
 254  *
 255  * Only to be called from kref_put in drm_syncobj_put.
 256  */
 257 void drm_syncobj_free(struct kref *kref)
 258 {
 259         struct drm_syncobj *syncobj = container_of(kref,
 260                                                    struct drm_syncobj,
 261                                                    refcount);
 262         drm_syncobj_replace_fence(syncobj, NULL);
 263         kfree(syncobj);
 264 }
 265 EXPORT_SYMBOL(drm_syncobj_free);
 266
 267 static int drm_syncobj_create(struct drm_file *file_private,
 268                               u32 *handle, uint32_t flags)
 269 {
 270         int ret;
 271         struct drm_syncobj *syncobj;
 272
 273         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 274         if (!syncobj)
 275                 return -ENOMEM;
 276
 277         kref_init(&syncobj->refcount);
 278         INIT_LIST_HEAD(&syncobj->cb_list);
 279         spin_lock_init(&syncobj->lock);
 280
 281         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 282                 ret = drm_syncobj_assign_null_handle(syncobj);
 283                 if (ret < 0) {
 284                         drm_syncobj_put(syncobj);
 285                         return ret;
 286                 }
 287         }
 288
 289         idr_preload(GFP_KERNEL);
 290         spin_lock(&file_private->syncobj_table_lock);
 291         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 292         spin_unlock(&file_private->syncobj_table_lock);
 293
 294         idr_preload_end();
 295
 296         if (ret < 0) {
 297                 drm_syncobj_put(syncobj);
 298                 return ret;
 299         }
 300
 301         *handle = ret;
 302         return 0;
 303 }
 304
 305 static int drm_syncobj_destroy(struct drm_file *file_private,
 306                                u32 handle)
 307 {
 308         struct drm_syncobj *syncobj;
 309
 310         spin_lock(&file_private->syncobj_table_lock);
 311         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 312         spin_unlock(&file_private->syncobj_table_lock);
 313
 314         if (!syncobj)
 315                 return -EINVAL;
 316
 317         drm_syncobj_put(syncobj);
 318         return 0;
 319 }
 320
 321 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 322 {
 323         struct drm_syncobj *syncobj = file->private_data;
 324
 325         drm_syncobj_put(syncobj);
 326         return 0;
 327 }
 328
 329 static const struct file_operations drm_syncobj_file_fops = {
 330         .release = drm_syncobj_file_release,
 331 };
 332
 333 static int drm_syncobj_alloc_file(struct drm_syncobj *syncobj)
 334 {
 335         struct file *file = anon_inode_getfile("syncobj_file",
 336                                                &drm_syncobj_file_fops,
 337                                                syncobj, 0);
 338         if (IS_ERR(file))
 339                 return PTR_ERR(file);
 340
 341         drm_syncobj_get(syncobj);
 342         if (cmpxchg(&syncobj->file, NULL, file)) {
 343                 /* lost the race */
 344                 fput(file);
 345         }
 346
 347         return 0;
 348 }
 349
 350 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 351                                     u32 handle, int *p_fd)
 352 {
 353         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 354         int ret;
 355         int fd;
 356
 357         if (!syncobj)
 358                 return -EINVAL;
 359
 360         fd = get_unused_fd_flags(O_CLOEXEC);
 361         if (fd < 0) {
 362                 drm_syncobj_put(syncobj);
 363                 return fd;
 364         }
 365
 366         if (!syncobj->file) {
 367                 ret = drm_syncobj_alloc_file(syncobj);
 368                 if (ret)
 369                         goto out_put_fd;
 370         }
 371         fd_install(fd, syncobj->file);
 372         drm_syncobj_put(syncobj);
 373         *p_fd = fd;
 374         return 0;
 375 out_put_fd:
 376         put_unused_fd(fd);
 377         drm_syncobj_put(syncobj);
 378         return ret;
 379 }
 380
 381 static struct drm_syncobj *drm_syncobj_fdget(int fd)
 382 {
 383         struct file *file = fget(fd);
 384
 385         if (!file)
 386                 return NULL;
 387         if (file->f_op != &drm_syncobj_file_fops)
 388                 goto err;
 389
 390         return file->private_data;
 391 err:
 392         fput(file);
 393         return NULL;
 394 };
 395
 396 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 397                                     int fd, u32 *handle)
 398 {
 399         struct drm_syncobj *syncobj = drm_syncobj_fdget(fd);
 400         int ret;
 401
 402         if (!syncobj)
 403                 return -EINVAL;
 404
 405         /* take a reference to put in the idr */
 406         drm_syncobj_get(syncobj);
 407
 408         idr_preload(GFP_KERNEL);
 409         spin_lock(&file_private->syncobj_table_lock);
 410         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 411         spin_unlock(&file_private->syncobj_table_lock);
 412         idr_preload_end();
 413
 414         if (ret < 0) {
 415                 fput(syncobj->file);
 416                 return ret;
 417         }
 418         *handle = ret;
 419         return 0;
 420 }
 421
 422 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 423                                               int fd, int handle)
 424 {
 425         struct dma_fence *fence = sync_file_get_fence(fd);
 426         struct drm_syncobj *syncobj;
 427
 428         if (!fence)
 429                 return -EINVAL;
 430
 431         syncobj = drm_syncobj_find(file_private, handle);
 432         if (!syncobj) {
 433                 dma_fence_put(fence);
 434                 return -ENOENT;
 435         }
 436
 437         drm_syncobj_replace_fence(syncobj, fence);
 438         dma_fence_put(fence);
 439         drm_syncobj_put(syncobj);
 440         return 0;
 441 }
 442
 443 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 444                                         int handle, int *p_fd)
 445 {
 446         int ret;
 447         struct dma_fence *fence;
 448         struct sync_file *sync_file;
 449         int fd = get_unused_fd_flags(O_CLOEXEC);
 450
 451         if (fd < 0)
 452                 return fd;
 453
 454         ret = drm_syncobj_find_fence(file_private, handle, &fence);
 455         if (ret)
 456                 goto err_put_fd;
 457
 458         sync_file = sync_file_create(fence);
 459
 460         dma_fence_put(fence);
 461
 462         if (!sync_file) {
 463                 ret = -EINVAL;
 464                 goto err_put_fd;
 465         }
 466
 467         fd_install(fd, sync_file->file);
 468
 469         *p_fd = fd;
 470         return 0;
 471 err_put_fd:
 472         put_unused_fd(fd);
 473         return ret;
 474 }
 475 /**
 476  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 477  * @file_private: drm file-private structure to set up
 478  *
 479  * Called at device open time, sets up the structure for handling refcounting
 480  * of sync objects.
 481  */
 482 void
 483 drm_syncobj_open(struct drm_file *file_private)
 484 {
 485         idr_init(&file_private->syncobj_idr);
 486         spin_lock_init(&file_private->syncobj_table_lock);
 487 }
 488
 489 static int
 490 drm_syncobj_release_handle(int id, void *ptr, void *data)
 491 {
 492         struct drm_syncobj *syncobj = ptr;
 493
 494         drm_syncobj_put(syncobj);
 495         return 0;
 496 }
 497
 498 /**
 499  * drm_syncobj_release - release file-private sync object resources
 500  * @file_private: drm file-private structure to clean up
 501  *
 502  * Called at close time when the filp is going away.
 503  *
 504  * Releases any remaining references on objects by this filp.
 505  */
 506 void
 507 drm_syncobj_release(struct drm_file *file_private)
 508 {
 509         idr_for_each(&file_private->syncobj_idr,
 510                      &drm_syncobj_release_handle, file_private);
 511         idr_destroy(&file_private->syncobj_idr);
 512 }
 513
 514 int
 515 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 516                          struct drm_file *file_private)
 517 {
 518         struct drm_syncobj_create *args = data;
 519
 520         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 521                 return -ENODEV;
 522
 523         /* no valid flags yet */
 524         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 525                 return -EINVAL;
 526
 527         return drm_syncobj_create(file_private,
 528                                   &args->handle, args->flags);
 529 }
 530
 531 int
 532 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 533                           struct drm_file *file_private)
 534 {
 535         struct drm_syncobj_destroy *args = data;
 536
 537         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 538                 return -ENODEV;
 539
 540         /* make sure padding is empty */
 541         if (args->pad)
 542                 return -EINVAL;
 543         return drm_syncobj_destroy(file_private, args->handle);
 544 }
 545
 546 int
 547 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 548                                    struct drm_file *file_private)
 549 {
 550         struct drm_syncobj_handle *args = data;
 551
 552         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 553                 return -ENODEV;
 554
 555         if (args->pad)
 556                 return -EINVAL;
 557
 558         if (args->flags != 0 &&
 559             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 560                 return -EINVAL;
 561
 562         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 563                 return drm_syncobj_export_sync_file(file_private, args->handle,
 564                                                     &args->fd);
 565
 566         return drm_syncobj_handle_to_fd(file_private, args->handle,
 567                                         &args->fd);
 568 }
 569
 570 int
 571 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 572                                    struct drm_file *file_private)
 573 {
 574         struct drm_syncobj_handle *args = data;
 575
 576         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 577                 return -ENODEV;
 578
 579         if (args->pad)
 580                 return -EINVAL;
 581
 582         if (args->flags != 0 &&
 583             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 584                 return -EINVAL;
 585
 586         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 587                 return drm_syncobj_import_sync_file_fence(file_private,
 588                                                           args->fd,
 589                                                           args->handle);
 590
 591         return drm_syncobj_fd_to_handle(file_private, args->fd,
 592                                         &args->handle);
 593 }
 594
 595 struct syncobj_wait_entry {
 596         struct task_struct *task;
 597         struct dma_fence *fence;
 598         struct dma_fence_cb fence_cb;
 599         struct drm_syncobj_cb syncobj_cb;
 600 };
 601
 602 static void syncobj_wait_fence_func(struct dma_fence *fence,
 603                                     struct dma_fence_cb *cb)
 604 {
 605         struct syncobj_wait_entry *wait =
 606                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 607
 608         wake_up_process(wait->task);
 609 }
 610
 611 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 612                                       struct drm_syncobj_cb *cb)
 613 {
 614         struct syncobj_wait_entry *wait =
 615                 container_of(cb, struct syncobj_wait_entry, syncobj_cb);
 616
 617         /* This happens inside the syncobj lock */
 618         wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 619                                                               lockdep_is_held(&syncobj->lock)));
 620         wake_up_process(wait->task);
 621 }
 622
 623 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 624                                                   uint32_t count,
 625                                                   uint32_t flags,
 626                                                   signed long timeout,
 627                                                   uint32_t *idx)
 628 {
 629         struct syncobj_wait_entry *entries;
 630         struct dma_fence *fence;
 631         signed long ret;
 632         uint32_t signaled_count, i;
 633
 634         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 635         if (!entries)
 636                 return -ENOMEM;
 637
 638         /* Walk the list of sync objects and initialize entries.  We do
 639          * this up-front so that we can properly return -EINVAL if there is
 640          * a syncobj with a missing fence and then never have the chance of
 641          * returning -EINVAL again.
 642          */
 643         signaled_count = 0;
 644         for (i = 0; i < count; ++i) {
 645                 entries[i].task = current;
 646                 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
 647                 if (!entries[i].fence) {
 648                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 649                                 continue;
 650                         } else {
 651                                 ret = -EINVAL;
 652                                 goto cleanup_entries;
 653                         }
 654                 }
 655
 656                 if (dma_fence_is_signaled(entries[i].fence)) {
 657                         if (signaled_count == 0 && idx)
 658                                 *idx = i;
 659                         signaled_count++;
 660                 }
 661         }
 662
 663         /* Initialize ret to the max of timeout and 1.  That way, the
 664          * default return value indicates a successful wait and not a
 665          * timeout.
 666          */
 667         ret = max_t(signed long, timeout, 1);
 668
 669         if (signaled_count == count ||
 670             (signaled_count > 0 &&
 671              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 672                 goto cleanup_entries;
 673
 674         /* There's a very annoying laxness in the dma_fence API here, in
 675          * that backends are not required to automatically report when a
 676          * fence is signaled prior to fence->ops->enable_signaling() being
 677          * called.  So here if we fail to match signaled_count, we need to
 678          * fallthough and try a 0 timeout wait!
 679          */
 680
 681         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 682                 for (i = 0; i < count; ++i) {
 683                         drm_syncobj_fence_get_or_add_callback(syncobjs[i],
 684                                                               &entries[i].fence,
 685                                                               &entries[i].syncobj_cb,
 686                                                               syncobj_wait_syncobj_func);
 687                 }
 688         }
 689
 690         do {
 691                 set_current_state(TASK_INTERRUPTIBLE);
 692
 693                 signaled_count = 0;
 694                 for (i = 0; i < count; ++i) {
 695                         fence = entries[i].fence;
 696                         if (!fence)
 697                                 continue;
 698
 699                         if (dma_fence_is_signaled(fence) ||
 700                             (!entries[i].fence_cb.func &&
 701                              dma_fence_add_callback(fence,
 702                                                     &entries[i].fence_cb,
 703                                                     syncobj_wait_fence_func))) {
 704                                 /* The fence has been signaled */
 705                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 706                                         signaled_count++;
 707                                 } else {
 708                                         if (idx)
 709                                                 *idx = i;
 710                                         goto done_waiting;
 711                                 }
 712                         }
 713                 }
 714
 715                 if (signaled_count == count)
 716                         goto done_waiting;
 717
 718                 if (timeout == 0) {
 719                         /* If we are doing a 0 timeout wait and we got
 720                          * here, then we just timed out.
 721                          */
 722                         ret = 0;
 723                         goto done_waiting;
 724                 }
 725
 726                 ret = schedule_timeout(ret);
 727
 728                 if (ret > 0 && signal_pending(current))
 729                         ret = -ERESTARTSYS;
 730         } while (ret > 0);
 731
 732 done_waiting:
 733         __set_current_state(TASK_RUNNING);
 734
 735 cleanup_entries:
 736         for (i = 0; i < count; ++i) {
 737                 if (entries[i].syncobj_cb.func)
 738                         drm_syncobj_remove_callback(syncobjs[i],
 739                                                     &entries[i].syncobj_cb);
 740                 if (entries[i].fence_cb.func)
 741                         dma_fence_remove_callback(entries[i].fence,
 742                                                   &entries[i].fence_cb);
 743                 dma_fence_put(entries[i].fence);
 744         }
 745         kfree(entries);
 746
 747         return ret;
 748 }
 749
 750 /**
 751  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 752  *
 753  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 754  *
 755  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 756  */
 757 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 758 {
 759         ktime_t abs_timeout, now;
 760         u64 timeout_ns, timeout_jiffies64;
 761
 762         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 763         if (timeout_nsec == 0)
 764                 return 0;
 765
 766         abs_timeout = ns_to_ktime(timeout_nsec);
 767         now = ktime_get();
 768
 769         if (!ktime_after(abs_timeout, now))
 770                 return 0;
 771
 772         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 773
 774         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 775         /*  clamp timeout to avoid infinite timeout */
 776         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 777                 return MAX_SCHEDULE_TIMEOUT - 1;
 778
 779         return timeout_jiffies64 + 1;
 780 }
 781
 782 static int drm_syncobj_array_wait(struct drm_device *dev,
 783                                   struct drm_file *file_private,
 784                                   struct drm_syncobj_wait *wait,
 785                                   struct drm_syncobj **syncobjs)
 786 {
 787         signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 788         signed long ret = 0;
 789         uint32_t first = ~0;
 790
 791         ret = drm_syncobj_array_wait_timeout(syncobjs,
 792                                              wait->count_handles,
 793                                              wait->flags,
 794                                              timeout, &first);
 795         if (ret < 0)
 796                 return ret;
 797
 798         wait->first_signaled = first;
 799         if (ret == 0)
 800                 return -ETIME;
 801         return 0;
 802 }
 803
 804 static int drm_syncobj_array_find(struct drm_file *file_private,
 805                                   void __user *user_handles,
 806                                   uint32_t count_handles,
 807                                   struct drm_syncobj ***syncobjs_out)
 808 {
 809         uint32_t i, *handles;
 810         struct drm_syncobj **syncobjs;
 811         int ret;
 812
 813         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
 814         if (handles == NULL)
 815                 return -ENOMEM;
 816
 817         if (copy_from_user(handles, user_handles,
 818                            sizeof(uint32_t) * count_handles)) {
 819                 ret = -EFAULT;
 820                 goto err_free_handles;
 821         }
 822
 823         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
 824         if (syncobjs == NULL) {
 825                 ret = -ENOMEM;
 826                 goto err_free_handles;
 827         }
 828
 829         for (i = 0; i < count_handles; i++) {
 830                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
 831                 if (!syncobjs[i]) {
 832                         ret = -ENOENT;
 833                         goto err_put_syncobjs;
 834                 }
 835         }
 836
 837         kfree(handles);
 838         *syncobjs_out = syncobjs;
 839         return 0;
 840
 841 err_put_syncobjs:
 842         while (i-- > 0)
 843                 drm_syncobj_put(syncobjs[i]);
 844         kfree(syncobjs);
 845 err_free_handles:
 846         kfree(handles);
 847
 848         return ret;
 849 }
 850
 851 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
 852                                    uint32_t count)
 853 {
 854         uint32_t i;
 855         for (i = 0; i < count; i++)
 856                 drm_syncobj_put(syncobjs[i]);
 857         kfree(syncobjs);
 858 }
 859
 860 int
 861 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 862                        struct drm_file *file_private)
 863 {
 864         struct drm_syncobj_wait *args = data;
 865         struct drm_syncobj **syncobjs;
 866         int ret = 0;
 867
 868         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 869                 return -ENODEV;
 870
 871         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 872                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 873                 return -EINVAL;
 874
 875         if (args->count_handles == 0)
 876                 return -EINVAL;
 877
 878         ret = drm_syncobj_array_find(file_private,
 879                                      u64_to_user_ptr(args->handles),
 880                                      args->count_handles,
 881                                      &syncobjs);
 882         if (ret < 0)
 883                 return ret;
 884
 885         ret = drm_syncobj_array_wait(dev, file_private,
 886                                      args, syncobjs);
 887
 888         drm_syncobj_array_free(syncobjs, args->count_handles);
 889
 890         return ret;
 891 }
 892
 893 int
 894 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 895                         struct drm_file *file_private)
 896 {
 897         struct drm_syncobj_array *args = data;
 898         struct drm_syncobj **syncobjs;
 899         uint32_t i;
 900         int ret;
 901
 902         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 903                 return -ENODEV;
 904
 905         if (args->pad != 0)
 906                 return -EINVAL;
 907
 908         if (args->count_handles == 0)
 909                 return -EINVAL;
 910
 911         ret = drm_syncobj_array_find(file_private,
 912                                      u64_to_user_ptr(args->handles),
 913                                      args->count_handles,
 914                                      &syncobjs);
 915         if (ret < 0)
 916                 return ret;
 917
 918         for (i = 0; i < args->count_handles; i++)
 919                 drm_syncobj_replace_fence(syncobjs[i], NULL);
 920
 921         drm_syncobj_array_free(syncobjs, args->count_handles);
 922
 923         return 0;
 924 }
 925
 926 int
 927 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 928                          struct drm_file *file_private)
 929 {
 930         struct drm_syncobj_array *args = data;
 931         struct drm_syncobj **syncobjs;
 932         uint32_t i;
 933         int ret;
 934
 935         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 936                 return -ENODEV;
 937
 938         if (args->pad != 0)
 939                 return -EINVAL;
 940
 941         if (args->count_handles == 0)
 942                 return -EINVAL;
 943
 944         ret = drm_syncobj_array_find(file_private,
 945                                      u64_to_user_ptr(args->handles),
 946                                      args->count_handles,
 947                                      &syncobjs);
 948         if (ret < 0)
 949                 return ret;
 950
 951         for (i = 0; i < args->count_handles; i++) {
 952                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
 953                 if (ret < 0)
 954                         break;
 955         }
 956
 957         drm_syncobj_array_free(syncobjs, args->count_handles);
 958
 959         return ret;
 960 }