drivers/dma-buf/dma-fence-unwrap.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * dma-fence-util: misc functions for dma_fence objects
   4  *
   5  * Copyright (C) 2022 Advanced Micro Devices, Inc.
   6  * Authors:
   7  *      Christian König <christian.koenig@amd.com>
   8  */
   9
  10 #include <linux/dma-fence.h>
  11 #include <linux/dma-fence-array.h>
  12 #include <linux/dma-fence-chain.h>
  13 #include <linux/dma-fence-unwrap.h>
  14 #include <linux/slab.h>
  15
  16 /* Internal helper to start new array iteration, don't use directly */
  17 static struct dma_fence *
  18 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
  19 {
  20         cursor->array = dma_fence_chain_contained(cursor->chain);
  21         cursor->index = 0;
  22         return dma_fence_array_first(cursor->array);
  23 }
  24
  25 /**
  26  * dma_fence_unwrap_first - return the first fence from fence containers
  27  * @head: the entrypoint into the containers
  28  * @cursor: current position inside the containers
  29  *
  30  * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
  31  * first fence.
  32  */
  33 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
  34                                          struct dma_fence_unwrap *cursor)
  35 {
  36         cursor->chain = dma_fence_get(head);
  37         return __dma_fence_unwrap_array(cursor);
  38 }
  39 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
  40
  41 /**
  42  * dma_fence_unwrap_next - return the next fence from a fence containers
  43  * @cursor: current position inside the containers
  44  *
  45  * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
  46  * the next fence from them.
  47  */
  48 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
  49 {
  50         struct dma_fence *tmp;
  51
  52         ++cursor->index;
  53         tmp = dma_fence_array_next(cursor->array, cursor->index);
  54         if (tmp)
  55                 return tmp;
  56
  57         cursor->chain = dma_fence_chain_walk(cursor->chain);
  58         return __dma_fence_unwrap_array(cursor);
  59 }
  60 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
  61
  62 /* Implementation for the dma_fence_merge() marco, don't use directly */
  63 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
  64                                            struct dma_fence **fences,
  65                                            struct dma_fence_unwrap *iter)
  66 {
  67         struct dma_fence_array *result;
  68         struct dma_fence *tmp, **array;
  69         ktime_t timestamp;
  70         unsigned int i;
  71         size_t count;
  72
  73         count = 0;
  74         timestamp = ns_to_ktime(0);
  75         for (i = 0; i < num_fences; ++i) {
  76                 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
  77                         if (!dma_fence_is_signaled(tmp)) {
  78                                 ++count;
  79                         } else if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT,
  80                                             &tmp->flags)) {
  81                                 if (ktime_after(tmp->timestamp, timestamp))
  82                                         timestamp = tmp->timestamp;
  83                         } else {
  84                                 /*
  85                                  * Use the current time if the fence is
  86                                  * currently signaling.
  87                                  */
  88                                 timestamp = ktime_get();
  89                         }
  90                 }
  91         }
  92
  93         /*
  94          * If we couldn't find a pending fence just return a private signaled
  95          * fence with the timestamp of the last signaled one.
  96          */
  97         if (count == 0)
  98                 return dma_fence_allocate_private_stub(timestamp);
  99
 100         array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
 101         if (!array)
 102                 return NULL;
 103
 104         /*
 105          * This trashes the input fence array and uses it as position for the
 106          * following merge loop. This works because the dma_fence_merge()
 107          * wrapper macro is creating this temporary array on the stack together
 108          * with the iterators.
 109          */
 110         for (i = 0; i < num_fences; ++i)
 111                 fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]);
 112
 113         count = 0;
 114         do {
 115                 unsigned int sel;
 116
 117 restart:
 118                 tmp = NULL;
 119                 for (i = 0; i < num_fences; ++i) {
 120                         struct dma_fence *next;
 121
 122                         while (fences[i] && dma_fence_is_signaled(fences[i]))
 123                                 fences[i] = dma_fence_unwrap_next(&iter[i]);
 124
 125                         next = fences[i];
 126                         if (!next)
 127                                 continue;
 128
 129                         /*
 130                          * We can't guarantee that inpute fences are ordered by
 131                          * context, but it is still quite likely when this
 132                          * function is used multiple times. So attempt to order
 133                          * the fences by context as we pass over them and merge
 134                          * fences with the same context.
 135                          */
 136                         if (!tmp || tmp->context > next->context) {
 137                                 tmp = next;
 138                                 sel = i;
 139
 140                         } else if (tmp->context < next->context) {
 141                                 continue;
 142
 143                         } else if (dma_fence_is_later(tmp, next)) {
 144                                 fences[i] = dma_fence_unwrap_next(&iter[i]);
 145                                 goto restart;
 146                         } else {
 147                                 fences[sel] = dma_fence_unwrap_next(&iter[sel]);
 148                                 goto restart;
 149                         }
 150                 }
 151
 152                 if (tmp) {
 153                         array[count++] = dma_fence_get(tmp);
 154                         fences[sel] = dma_fence_unwrap_next(&iter[sel]);
 155                 }
 156         } while (tmp);
 157
 158         if (count == 0) {
 159                 tmp = dma_fence_allocate_private_stub(ktime_get());
 160                 goto return_tmp;
 161         }
 162
 163         if (count == 1) {
 164                 tmp = array[0];
 165                 goto return_tmp;
 166         }
 167
 168         result = dma_fence_array_create(count, array,
 169                                         dma_fence_context_alloc(1),
 170                                         1, false);
 171         if (!result) {
 172                 tmp = NULL;
 173                 goto return_tmp;
 174         }
 175         return &result->base;
 176
 177 return_tmp:
 178         kfree(array);
 179         return tmp;
 180 }
 181 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);