drivers/gpu/drm/xe/xe_hw_fence.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2021 Intel Corporation
   4  */
   5
   6 #include "xe_hw_fence.h"
   7
   8 #include <linux/device.h>
   9 #include <linux/slab.h>
  10
  11 #include "xe_bo.h"
  12 #include "xe_device.h"
  13 #include "xe_gt.h"
  14 #include "xe_hw_engine.h"
  15 #include "xe_macros.h"
  16 #include "xe_map.h"
  17 #include "xe_trace.h"
  18
  19 static struct kmem_cache *xe_hw_fence_slab;
  20
  21 int __init xe_hw_fence_module_init(void)
  22 {
  23         xe_hw_fence_slab = kmem_cache_create("xe_hw_fence",
  24                                              sizeof(struct xe_hw_fence), 0,
  25                                              SLAB_HWCACHE_ALIGN, NULL);
  26         if (!xe_hw_fence_slab)
  27                 return -ENOMEM;
  28
  29         return 0;
  30 }
  31
  32 void xe_hw_fence_module_exit(void)
  33 {
  34         rcu_barrier();
  35         kmem_cache_destroy(xe_hw_fence_slab);
  36 }
  37
  38 static struct xe_hw_fence *fence_alloc(void)
  39 {
  40         return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL);
  41 }
  42
  43 static void fence_free(struct rcu_head *rcu)
  44 {
  45         struct xe_hw_fence *fence =
  46                 container_of(rcu, struct xe_hw_fence, dma.rcu);
  47
  48         if (!WARN_ON_ONCE(!fence))
  49                 kmem_cache_free(xe_hw_fence_slab, fence);
  50 }
  51
  52 static void hw_fence_irq_run_cb(struct irq_work *work)
  53 {
  54         struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work);
  55         struct xe_hw_fence *fence, *next;
  56         bool tmp;
  57
  58         tmp = dma_fence_begin_signalling();
  59         spin_lock(&irq->lock);
  60         if (irq->enabled) {
  61                 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
  62                         struct dma_fence *dma_fence = &fence->dma;
  63
  64                         trace_xe_hw_fence_try_signal(fence);
  65                         if (dma_fence_is_signaled_locked(dma_fence)) {
  66                                 trace_xe_hw_fence_signal(fence);
  67                                 list_del_init(&fence->irq_link);
  68                                 dma_fence_put(dma_fence);
  69                         }
  70                 }
  71         }
  72         spin_unlock(&irq->lock);
  73         dma_fence_end_signalling(tmp);
  74 }
  75
  76 void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq)
  77 {
  78         spin_lock_init(&irq->lock);
  79         init_irq_work(&irq->work, hw_fence_irq_run_cb);
  80         INIT_LIST_HEAD(&irq->pending);
  81         irq->enabled = true;
  82 }
  83
  84 void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq)
  85 {
  86         struct xe_hw_fence *fence, *next;
  87         unsigned long flags;
  88         int err;
  89         bool tmp;
  90
  91         if (XE_WARN_ON(!list_empty(&irq->pending))) {
  92                 tmp = dma_fence_begin_signalling();
  93                 spin_lock_irqsave(&irq->lock, flags);
  94                 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
  95                         list_del_init(&fence->irq_link);
  96                         err = dma_fence_signal_locked(&fence->dma);
  97                         dma_fence_put(&fence->dma);
  98                         XE_WARN_ON(err);
  99                 }
 100                 spin_unlock_irqrestore(&irq->lock, flags);
 101                 dma_fence_end_signalling(tmp);
 102         }
 103 }
 104
 105 void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq)
 106 {
 107         irq_work_queue(&irq->work);
 108 }
 109
 110 void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq)
 111 {
 112         spin_lock_irq(&irq->lock);
 113         irq->enabled = false;
 114         spin_unlock_irq(&irq->lock);
 115 }
 116
 117 void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq)
 118 {
 119         spin_lock_irq(&irq->lock);
 120         irq->enabled = true;
 121         spin_unlock_irq(&irq->lock);
 122
 123         irq_work_queue(&irq->work);
 124 }
 125
 126 void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt,
 127                           struct xe_hw_fence_irq *irq, const char *name)
 128 {
 129         ctx->gt = gt;
 130         ctx->irq = irq;
 131         ctx->dma_fence_ctx = dma_fence_context_alloc(1);
 132         ctx->next_seqno = XE_FENCE_INITIAL_SEQNO;
 133         sprintf(ctx->name, "%s", name);
 134 }
 135
 136 void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx)
 137 {
 138 }
 139
 140 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence);
 141
 142 static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence)
 143 {
 144         return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock);
 145 }
 146
 147 static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence)
 148 {
 149         struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
 150
 151         return dev_name(gt_to_xe(fence->ctx->gt)->drm.dev);
 152 }
 153
 154 static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence)
 155 {
 156         struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
 157
 158         return fence->ctx->name;
 159 }
 160
 161 static bool xe_hw_fence_signaled(struct dma_fence *dma_fence)
 162 {
 163         struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
 164         struct xe_device *xe = gt_to_xe(fence->ctx->gt);
 165         u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32);
 166
 167         return dma_fence->error ||
 168                 !__dma_fence_is_later(dma_fence->seqno, seqno, dma_fence->ops);
 169 }
 170
 171 static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence)
 172 {
 173         struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
 174         struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence);
 175
 176         dma_fence_get(dma_fence);
 177         list_add_tail(&fence->irq_link, &irq->pending);
 178
 179         /* SW completed (no HW IRQ) so kick handler to signal fence */
 180         if (xe_hw_fence_signaled(dma_fence))
 181                 xe_hw_fence_irq_run(irq);
 182
 183         return true;
 184 }
 185
 186 static void xe_hw_fence_release(struct dma_fence *dma_fence)
 187 {
 188         struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
 189
 190         trace_xe_hw_fence_free(fence);
 191         XE_WARN_ON(!list_empty(&fence->irq_link));
 192         call_rcu(&dma_fence->rcu, fence_free);
 193 }
 194
 195 static const struct dma_fence_ops xe_hw_fence_ops = {
 196         .get_driver_name = xe_hw_fence_get_driver_name,
 197         .get_timeline_name = xe_hw_fence_get_timeline_name,
 198         .enable_signaling = xe_hw_fence_enable_signaling,
 199         .signaled = xe_hw_fence_signaled,
 200         .release = xe_hw_fence_release,
 201 };
 202
 203 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence)
 204 {
 205         if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops))
 206                 return NULL;
 207
 208         return container_of(fence, struct xe_hw_fence, dma);
 209 }
 210
 211 struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx,
 212                                        struct iosys_map seqno_map)
 213 {
 214         struct xe_hw_fence *fence;
 215
 216         fence = fence_alloc();
 217         if (!fence)
 218                 return ERR_PTR(-ENOMEM);
 219
 220         fence->ctx = ctx;
 221         fence->seqno_map = seqno_map;
 222         INIT_LIST_HEAD(&fence->irq_link);
 223
 224         dma_fence_init(&fence->dma, &xe_hw_fence_ops, &ctx->irq->lock,
 225                        ctx->dma_fence_ctx, ctx->next_seqno++);
 226
 227         trace_xe_hw_fence_create(fence);
 228
 229         return fence;
 230 }