Commit | Line | Data |
---|---|---|
dd08ebf6 MB |
1 | // SPDX-License-Identifier: MIT |
2 | /* | |
3 | * Copyright © 2021 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include "xe_hw_fence.h" | |
7 | ||
8 | #include <linux/device.h> | |
9 | #include <linux/slab.h> | |
10 | ||
11 | #include "xe_bo.h" | |
12 | #include "xe_device.h" | |
13 | #include "xe_gt.h" | |
14 | #include "xe_hw_engine.h" | |
15 | #include "xe_macros.h" | |
16 | #include "xe_map.h" | |
17 | #include "xe_trace.h" | |
18 | ||
19 | static struct kmem_cache *xe_hw_fence_slab; | |
20 | ||
21 | int __init xe_hw_fence_module_init(void) | |
22 | { | |
23 | xe_hw_fence_slab = kmem_cache_create("xe_hw_fence", | |
24 | sizeof(struct xe_hw_fence), 0, | |
25 | SLAB_HWCACHE_ALIGN, NULL); | |
26 | if (!xe_hw_fence_slab) | |
27 | return -ENOMEM; | |
28 | ||
29 | return 0; | |
30 | } | |
31 | ||
32 | void xe_hw_fence_module_exit(void) | |
33 | { | |
34 | rcu_barrier(); | |
35 | kmem_cache_destroy(xe_hw_fence_slab); | |
36 | } | |
37 | ||
38 | static struct xe_hw_fence *fence_alloc(void) | |
39 | { | |
40 | return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL); | |
41 | } | |
42 | ||
43 | static void fence_free(struct rcu_head *rcu) | |
44 | { | |
45 | struct xe_hw_fence *fence = | |
46 | container_of(rcu, struct xe_hw_fence, dma.rcu); | |
47 | ||
48 | if (!WARN_ON_ONCE(!fence)) | |
49 | kmem_cache_free(xe_hw_fence_slab, fence); | |
50 | } | |
51 | ||
52 | static void hw_fence_irq_run_cb(struct irq_work *work) | |
53 | { | |
54 | struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work); | |
55 | struct xe_hw_fence *fence, *next; | |
56 | bool tmp; | |
57 | ||
58 | tmp = dma_fence_begin_signalling(); | |
59 | spin_lock(&irq->lock); | |
60 | if (irq->enabled) { | |
61 | list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { | |
62 | struct dma_fence *dma_fence = &fence->dma; | |
63 | ||
64 | trace_xe_hw_fence_try_signal(fence); | |
65 | if (dma_fence_is_signaled_locked(dma_fence)) { | |
66 | trace_xe_hw_fence_signal(fence); | |
67 | list_del_init(&fence->irq_link); | |
68 | dma_fence_put(dma_fence); | |
69 | } | |
70 | } | |
71 | } | |
72 | spin_unlock(&irq->lock); | |
73 | dma_fence_end_signalling(tmp); | |
74 | } | |
75 | ||
76 | void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq) | |
77 | { | |
78 | spin_lock_init(&irq->lock); | |
79 | init_irq_work(&irq->work, hw_fence_irq_run_cb); | |
80 | INIT_LIST_HEAD(&irq->pending); | |
81 | irq->enabled = true; | |
82 | } | |
83 | ||
84 | void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq) | |
85 | { | |
86 | struct xe_hw_fence *fence, *next; | |
87 | unsigned long flags; | |
88 | int err; | |
89 | bool tmp; | |
90 | ||
91 | if (XE_WARN_ON(!list_empty(&irq->pending))) { | |
92 | tmp = dma_fence_begin_signalling(); | |
93 | spin_lock_irqsave(&irq->lock, flags); | |
94 | list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { | |
95 | list_del_init(&fence->irq_link); | |
96 | err = dma_fence_signal_locked(&fence->dma); | |
97 | dma_fence_put(&fence->dma); | |
98 | XE_WARN_ON(err); | |
99 | } | |
100 | spin_unlock_irqrestore(&irq->lock, flags); | |
101 | dma_fence_end_signalling(tmp); | |
102 | } | |
103 | } | |
104 | ||
105 | void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq) | |
106 | { | |
107 | irq_work_queue(&irq->work); | |
108 | } | |
109 | ||
110 | void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq) | |
111 | { | |
112 | spin_lock_irq(&irq->lock); | |
113 | irq->enabled = false; | |
114 | spin_unlock_irq(&irq->lock); | |
115 | } | |
116 | ||
117 | void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq) | |
118 | { | |
119 | spin_lock_irq(&irq->lock); | |
120 | irq->enabled = true; | |
121 | spin_unlock_irq(&irq->lock); | |
122 | ||
123 | irq_work_queue(&irq->work); | |
124 | } | |
125 | ||
126 | void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, | |
127 | struct xe_hw_fence_irq *irq, const char *name) | |
128 | { | |
129 | ctx->gt = gt; | |
130 | ctx->irq = irq; | |
131 | ctx->dma_fence_ctx = dma_fence_context_alloc(1); | |
7c51050b | 132 | ctx->next_seqno = XE_FENCE_INITIAL_SEQNO; |
dd08ebf6 MB |
133 | sprintf(ctx->name, "%s", name); |
134 | } | |
135 | ||
136 | void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx) | |
137 | { | |
138 | } | |
139 | ||
140 | static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence); | |
141 | ||
142 | static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence) | |
143 | { | |
144 | return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock); | |
145 | } | |
146 | ||
147 | static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence) | |
148 | { | |
149 | struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); | |
150 | ||
151 | return dev_name(gt_to_xe(fence->ctx->gt)->drm.dev); | |
152 | } | |
153 | ||
154 | static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence) | |
155 | { | |
156 | struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); | |
157 | ||
158 | return fence->ctx->name; | |
159 | } | |
160 | ||
161 | static bool xe_hw_fence_signaled(struct dma_fence *dma_fence) | |
162 | { | |
163 | struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); | |
164 | struct xe_device *xe = gt_to_xe(fence->ctx->gt); | |
165 | u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32); | |
166 | ||
167 | return dma_fence->error || | |
7c51050b | 168 | !__dma_fence_is_later(dma_fence->seqno, seqno, dma_fence->ops); |
dd08ebf6 MB |
169 | } |
170 | ||
171 | static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence) | |
172 | { | |
173 | struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); | |
174 | struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence); | |
175 | ||
176 | dma_fence_get(dma_fence); | |
177 | list_add_tail(&fence->irq_link, &irq->pending); | |
178 | ||
179 | /* SW completed (no HW IRQ) so kick handler to signal fence */ | |
180 | if (xe_hw_fence_signaled(dma_fence)) | |
181 | xe_hw_fence_irq_run(irq); | |
182 | ||
183 | return true; | |
184 | } | |
185 | ||
186 | static void xe_hw_fence_release(struct dma_fence *dma_fence) | |
187 | { | |
188 | struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); | |
189 | ||
190 | trace_xe_hw_fence_free(fence); | |
99fea682 | 191 | XE_WARN_ON(!list_empty(&fence->irq_link)); |
dd08ebf6 MB |
192 | call_rcu(&dma_fence->rcu, fence_free); |
193 | } | |
194 | ||
195 | static const struct dma_fence_ops xe_hw_fence_ops = { | |
196 | .get_driver_name = xe_hw_fence_get_driver_name, | |
197 | .get_timeline_name = xe_hw_fence_get_timeline_name, | |
198 | .enable_signaling = xe_hw_fence_enable_signaling, | |
199 | .signaled = xe_hw_fence_signaled, | |
200 | .release = xe_hw_fence_release, | |
201 | }; | |
202 | ||
203 | static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence) | |
204 | { | |
205 | if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops)) | |
206 | return NULL; | |
207 | ||
208 | return container_of(fence, struct xe_hw_fence, dma); | |
209 | } | |
210 | ||
211 | struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx, | |
212 | struct iosys_map seqno_map) | |
213 | { | |
214 | struct xe_hw_fence *fence; | |
215 | ||
216 | fence = fence_alloc(); | |
217 | if (!fence) | |
218 | return ERR_PTR(-ENOMEM); | |
219 | ||
dd08ebf6 MB |
220 | fence->ctx = ctx; |
221 | fence->seqno_map = seqno_map; | |
222 | INIT_LIST_HEAD(&fence->irq_link); | |
223 | ||
f9c15a67 JRS |
224 | dma_fence_init(&fence->dma, &xe_hw_fence_ops, &ctx->irq->lock, |
225 | ctx->dma_fence_ctx, ctx->next_seqno++); | |
226 | ||
dd08ebf6 MB |
227 | trace_xe_hw_fence_create(fence); |
228 | ||
229 | return fence; | |
230 | } |