Commit | Line | Data |
---|---|---|
6ee73861 BS |
1 | /* |
2 | * Copyright (C) 2007 Ben Skeggs. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining | |
6 | * a copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sublicense, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * The above copyright notice and this permission notice (including the | |
14 | * next paragraph) shall be included in all copies or substantial | |
15 | * portions of the Software. | |
16 | * | |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. | |
20 | * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE | |
21 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION | |
22 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION | |
23 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | |
24 | * | |
25 | */ | |
26 | ||
760285e7 | 27 | #include <drm/drmP.h> |
6ee73861 | 28 | |
bd35fe5a MS |
29 | #include <linux/ktime.h> |
30 | #include <linux/hrtimer.h> | |
29ba89b2 | 31 | #include <trace/events/fence.h> |
bd35fe5a | 32 | |
8ed1730c | 33 | #include <nvif/cl826e.h> |
867920f8 BS |
34 | #include <nvif/notify.h> |
35 | #include <nvif/event.h> | |
36 | ||
4dc28134 | 37 | #include "nouveau_drv.h" |
6ee73861 | 38 | #include "nouveau_dma.h" |
ebb945a9 | 39 | #include "nouveau_fence.h" |
6ee73861 | 40 | |
29ba89b2 ML |
41 | static const struct fence_ops nouveau_fence_ops_uevent; |
42 | static const struct fence_ops nouveau_fence_ops_legacy; | |
43 | ||
44 | static inline struct nouveau_fence * | |
45 | from_fence(struct fence *fence) | |
46 | { | |
47 | return container_of(fence, struct nouveau_fence, base); | |
48 | } | |
49 | ||
50 | static inline struct nouveau_fence_chan * | |
51 | nouveau_fctx(struct nouveau_fence *fence) | |
52 | { | |
53 | return container_of(fence->base.lock, struct nouveau_fence_chan, lock); | |
54 | } | |
c4c7044f | 55 | |
0ec5f02f | 56 | static int |
c4c7044f BS |
57 | nouveau_fence_signal(struct nouveau_fence *fence) |
58 | { | |
0ec5f02f ML |
59 | int drop = 0; |
60 | ||
29ba89b2 ML |
61 | fence_signal_locked(&fence->base); |
62 | list_del(&fence->head); | |
0ec5f02f | 63 | rcu_assign_pointer(fence->channel, NULL); |
29ba89b2 ML |
64 | |
65 | if (test_bit(FENCE_FLAG_USER_BITS, &fence->base.flags)) { | |
66 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); | |
c4c7044f | 67 | |
29ba89b2 | 68 | if (!--fctx->notify_ref) |
0ec5f02f | 69 | drop = 1; |
c4c7044f BS |
70 | } |
71 | ||
29ba89b2 | 72 | fence_put(&fence->base); |
0ec5f02f | 73 | return drop; |
29ba89b2 ML |
74 | } |
75 | ||
76 | static struct nouveau_fence * | |
77 | nouveau_local_fence(struct fence *fence, struct nouveau_drm *drm) { | |
78 | struct nouveau_fence_priv *priv = (void*)drm->fence; | |
79 | ||
80 | if (fence->ops != &nouveau_fence_ops_legacy && | |
81 | fence->ops != &nouveau_fence_ops_uevent) | |
82 | return NULL; | |
83 | ||
84 | if (fence->context < priv->context_base || | |
85 | fence->context >= priv->context_base + priv->contexts) | |
86 | return NULL; | |
87 | ||
88 | return from_fence(fence); | |
c4c7044f BS |
89 | } |
90 | ||
5e120f6e BS |
91 | void |
92 | nouveau_fence_context_del(struct nouveau_fence_chan *fctx) | |
93 | { | |
29ba89b2 ML |
94 | struct nouveau_fence *fence; |
95 | ||
29ba89b2 ML |
96 | spin_lock_irq(&fctx->lock); |
97 | while (!list_empty(&fctx->pending)) { | |
98 | fence = list_entry(fctx->pending.next, typeof(*fence), head); | |
99 | ||
0ec5f02f ML |
100 | if (nouveau_fence_signal(fence)) |
101 | nvif_notify_put(&fctx->notify); | |
5e120f6e | 102 | } |
29ba89b2 | 103 | spin_unlock_irq(&fctx->lock); |
0ec5f02f ML |
104 | |
105 | nvif_notify_fini(&fctx->notify); | |
106 | fctx->dead = 1; | |
107 | ||
108 | /* | |
109 | * Ensure that all accesses to fence->channel complete before freeing | |
110 | * the channel. | |
111 | */ | |
112 | synchronize_rcu(); | |
29ba89b2 ML |
113 | } |
114 | ||
15a996bb ML |
115 | static void |
116 | nouveau_fence_context_put(struct kref *fence_ref) | |
117 | { | |
118 | kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref)); | |
119 | } | |
120 | ||
121 | void | |
122 | nouveau_fence_context_free(struct nouveau_fence_chan *fctx) | |
123 | { | |
124 | kref_put(&fctx->fence_ref, nouveau_fence_context_put); | |
125 | } | |
126 | ||
0ec5f02f | 127 | static int |
29ba89b2 ML |
128 | nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) |
129 | { | |
130 | struct nouveau_fence *fence; | |
0ec5f02f | 131 | int drop = 0; |
29ba89b2 ML |
132 | u32 seq = fctx->read(chan); |
133 | ||
134 | while (!list_empty(&fctx->pending)) { | |
135 | fence = list_entry(fctx->pending.next, typeof(*fence), head); | |
136 | ||
137 | if ((int)(seq - fence->base.seqno) < 0) | |
0ec5f02f | 138 | break; |
29ba89b2 | 139 | |
0ec5f02f | 140 | drop |= nouveau_fence_signal(fence); |
29ba89b2 | 141 | } |
0ec5f02f ML |
142 | |
143 | return drop; | |
29ba89b2 ML |
144 | } |
145 | ||
146 | static int | |
147 | nouveau_fence_wait_uevent_handler(struct nvif_notify *notify) | |
148 | { | |
149 | struct nouveau_fence_chan *fctx = | |
150 | container_of(notify, typeof(*fctx), notify); | |
151 | unsigned long flags; | |
0ec5f02f | 152 | int ret = NVIF_NOTIFY_KEEP; |
29ba89b2 ML |
153 | |
154 | spin_lock_irqsave(&fctx->lock, flags); | |
155 | if (!list_empty(&fctx->pending)) { | |
156 | struct nouveau_fence *fence; | |
0ec5f02f | 157 | struct nouveau_channel *chan; |
29ba89b2 ML |
158 | |
159 | fence = list_entry(fctx->pending.next, typeof(*fence), head); | |
0ec5f02f ML |
160 | chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); |
161 | if (nouveau_fence_update(fence->channel, fctx)) | |
162 | ret = NVIF_NOTIFY_DROP; | |
29ba89b2 ML |
163 | } |
164 | spin_unlock_irqrestore(&fctx->lock, flags); | |
165 | ||
0ec5f02f | 166 | return ret; |
5e120f6e BS |
167 | } |
168 | ||
169 | void | |
29ba89b2 | 170 | nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx) |
5e120f6e | 171 | { |
29ba89b2 | 172 | struct nouveau_fence_priv *priv = (void*)chan->drm->fence; |
a01ca78c | 173 | struct nouveau_cli *cli = (void *)chan->user.client; |
29ba89b2 ML |
174 | int ret; |
175 | ||
f589be88 | 176 | INIT_LIST_HEAD(&fctx->flip); |
5e120f6e BS |
177 | INIT_LIST_HEAD(&fctx->pending); |
178 | spin_lock_init(&fctx->lock); | |
29ba89b2 ML |
179 | fctx->context = priv->context_base + chan->chid; |
180 | ||
15a996bb ML |
181 | if (chan == chan->drm->cechan) |
182 | strcpy(fctx->name, "copy engine channel"); | |
183 | else if (chan == chan->drm->channel) | |
184 | strcpy(fctx->name, "generic kernel channel"); | |
185 | else | |
989aa5b7 | 186 | strcpy(fctx->name, nvxx_client(&cli->base)->name); |
15a996bb ML |
187 | |
188 | kref_init(&fctx->fence_ref); | |
29ba89b2 ML |
189 | if (!priv->uevent) |
190 | return; | |
191 | ||
a01ca78c BS |
192 | ret = nvif_notify_init(&chan->user, nouveau_fence_wait_uevent_handler, |
193 | false, G82_CHANNEL_DMA_V0_NTFY_UEVENT, | |
194 | &(struct nvif_notify_uevent_req) { }, | |
195 | sizeof(struct nvif_notify_uevent_req), | |
196 | sizeof(struct nvif_notify_uevent_rep), | |
197 | &fctx->notify); | |
29ba89b2 ML |
198 | |
199 | WARN_ON(ret); | |
5e120f6e | 200 | } |
6ee73861 | 201 | |
29ba89b2 ML |
202 | struct nouveau_fence_work { |
203 | struct work_struct work; | |
204 | struct fence_cb cb; | |
205 | void (*func)(void *); | |
206 | void *data; | |
207 | }; | |
208 | ||
c4c7044f BS |
209 | static void |
210 | nouveau_fence_work_handler(struct work_struct *kwork) | |
211 | { | |
29ba89b2 | 212 | struct nouveau_fence_work *work = container_of(kwork, typeof(*work), work); |
c4c7044f BS |
213 | work->func(work->data); |
214 | kfree(work); | |
215 | } | |
216 | ||
29ba89b2 ML |
217 | static void nouveau_fence_work_cb(struct fence *fence, struct fence_cb *cb) |
218 | { | |
219 | struct nouveau_fence_work *work = container_of(cb, typeof(*work), cb); | |
220 | ||
221 | schedule_work(&work->work); | |
222 | } | |
223 | ||
c4c7044f | 224 | void |
f2c24b83 | 225 | nouveau_fence_work(struct fence *fence, |
c4c7044f BS |
226 | void (*func)(void *), void *data) |
227 | { | |
29ba89b2 | 228 | struct nouveau_fence_work *work; |
c4c7044f | 229 | |
f2c24b83 | 230 | if (fence_is_signaled(fence)) |
29ba89b2 | 231 | goto err; |
c4c7044f | 232 | |
c4c7044f BS |
233 | work = kmalloc(sizeof(*work), GFP_KERNEL); |
234 | if (!work) { | |
bb6178b0 ML |
235 | /* |
236 | * this might not be a nouveau fence any more, | |
237 | * so force a lazy wait here | |
238 | */ | |
f2c24b83 | 239 | WARN_ON(nouveau_fence_wait((struct nouveau_fence *)fence, |
bb6178b0 | 240 | true, false)); |
29ba89b2 | 241 | goto err; |
c4c7044f BS |
242 | } |
243 | ||
29ba89b2 | 244 | INIT_WORK(&work->work, nouveau_fence_work_handler); |
c4c7044f BS |
245 | work->func = func; |
246 | work->data = data; | |
6ee73861 | 247 | |
f2c24b83 | 248 | if (fence_add_callback(fence, &work->cb, nouveau_fence_work_cb) < 0) |
29ba89b2 ML |
249 | goto err_free; |
250 | return; | |
b08abd4e | 251 | |
29ba89b2 ML |
252 | err_free: |
253 | kfree(work); | |
254 | err: | |
255 | func(data); | |
6ee73861 BS |
256 | } |
257 | ||
258 | int | |
d375e7d5 | 259 | nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan) |
6ee73861 | 260 | { |
e193b1d4 | 261 | struct nouveau_fence_chan *fctx = chan->fence; |
29ba89b2 | 262 | struct nouveau_fence_priv *priv = (void*)chan->drm->fence; |
6ee73861 BS |
263 | int ret; |
264 | ||
5e120f6e | 265 | fence->channel = chan; |
bfd8303a | 266 | fence->timeout = jiffies + (15 * HZ); |
6ee73861 | 267 | |
29ba89b2 ML |
268 | if (priv->uevent) |
269 | fence_init(&fence->base, &nouveau_fence_ops_uevent, | |
e3be4c23 | 270 | &fctx->lock, fctx->context, ++fctx->sequence); |
29ba89b2 ML |
271 | else |
272 | fence_init(&fence->base, &nouveau_fence_ops_legacy, | |
e3be4c23 | 273 | &fctx->lock, fctx->context, ++fctx->sequence); |
15a996bb | 274 | kref_get(&fctx->fence_ref); |
29ba89b2 ML |
275 | |
276 | trace_fence_emit(&fence->base); | |
827520ce | 277 | ret = fctx->emit(fence); |
5e120f6e | 278 | if (!ret) { |
29ba89b2 ML |
279 | fence_get(&fence->base); |
280 | spin_lock_irq(&fctx->lock); | |
0ec5f02f ML |
281 | |
282 | if (nouveau_fence_update(chan, fctx)) | |
283 | nvif_notify_put(&fctx->notify); | |
284 | ||
5e120f6e | 285 | list_add_tail(&fence->head, &fctx->pending); |
29ba89b2 | 286 | spin_unlock_irq(&fctx->lock); |
529c4959 | 287 | } |
6ee73861 | 288 | |
5e120f6e | 289 | return ret; |
6ee73861 BS |
290 | } |
291 | ||
6ee73861 | 292 | bool |
d375e7d5 | 293 | nouveau_fence_done(struct nouveau_fence *fence) |
6ee73861 | 294 | { |
29ba89b2 ML |
295 | if (fence->base.ops == &nouveau_fence_ops_legacy || |
296 | fence->base.ops == &nouveau_fence_ops_uevent) { | |
297 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); | |
0ec5f02f | 298 | struct nouveau_channel *chan; |
29ba89b2 | 299 | unsigned long flags; |
6ee73861 | 300 | |
29ba89b2 ML |
301 | if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags)) |
302 | return true; | |
79ca2770 | 303 | |
29ba89b2 | 304 | spin_lock_irqsave(&fctx->lock, flags); |
0ec5f02f ML |
305 | chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); |
306 | if (chan && nouveau_fence_update(chan, fctx)) | |
307 | nvif_notify_put(&fctx->notify); | |
29ba89b2 ML |
308 | spin_unlock_irqrestore(&fctx->lock, flags); |
309 | } | |
310 | return fence_is_signaled(&fence->base); | |
e18c080f BS |
311 | } |
312 | ||
29ba89b2 ML |
313 | static long |
314 | nouveau_fence_wait_legacy(struct fence *f, bool intr, long wait) | |
e18c080f | 315 | { |
29ba89b2 ML |
316 | struct nouveau_fence *fence = from_fence(f); |
317 | unsigned long sleep_time = NSEC_PER_MSEC / 1000; | |
318 | unsigned long t = jiffies, timeout = t + wait; | |
e18c080f | 319 | |
29ba89b2 ML |
320 | while (!nouveau_fence_done(fence)) { |
321 | ktime_t kt; | |
51cb4b39 | 322 | |
29ba89b2 | 323 | t = jiffies; |
e18c080f | 324 | |
29ba89b2 ML |
325 | if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) { |
326 | __set_current_state(TASK_RUNNING); | |
327 | return 0; | |
e18c080f | 328 | } |
29ba89b2 ML |
329 | |
330 | __set_current_state(intr ? TASK_INTERRUPTIBLE : | |
331 | TASK_UNINTERRUPTIBLE); | |
332 | ||
333 | kt = ktime_set(0, sleep_time); | |
334 | schedule_hrtimeout(&kt, HRTIMER_MODE_REL); | |
335 | sleep_time *= 2; | |
336 | if (sleep_time > NSEC_PER_MSEC) | |
337 | sleep_time = NSEC_PER_MSEC; | |
338 | ||
339 | if (intr && signal_pending(current)) | |
340 | return -ERESTARTSYS; | |
e18c080f BS |
341 | } |
342 | ||
29ba89b2 | 343 | __set_current_state(TASK_RUNNING); |
e18c080f | 344 | |
29ba89b2 | 345 | return timeout - t; |
e18c080f BS |
346 | } |
347 | ||
29ba89b2 ML |
348 | static int |
349 | nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr) | |
6ee73861 | 350 | { |
6ee73861 BS |
351 | int ret = 0; |
352 | ||
d375e7d5 | 353 | while (!nouveau_fence_done(fence)) { |
29ba89b2 | 354 | if (time_after_eq(jiffies, fence->timeout)) { |
6ee73861 BS |
355 | ret = -EBUSY; |
356 | break; | |
357 | } | |
358 | ||
29ba89b2 ML |
359 | __set_current_state(intr ? |
360 | TASK_INTERRUPTIBLE : | |
361 | TASK_UNINTERRUPTIBLE); | |
6ee73861 BS |
362 | |
363 | if (intr && signal_pending(current)) { | |
9ddc8c52 | 364 | ret = -ERESTARTSYS; |
6ee73861 BS |
365 | break; |
366 | } | |
367 | } | |
368 | ||
369 | __set_current_state(TASK_RUNNING); | |
d375e7d5 BS |
370 | return ret; |
371 | } | |
372 | ||
5e120f6e | 373 | int |
29ba89b2 ML |
374 | nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr) |
375 | { | |
376 | long ret; | |
377 | ||
378 | if (!lazy) | |
379 | return nouveau_fence_wait_busy(fence, intr); | |
380 | ||
381 | ret = fence_wait_timeout(&fence->base, intr, 15 * HZ); | |
382 | if (ret < 0) | |
383 | return ret; | |
384 | else if (!ret) | |
385 | return -EBUSY; | |
386 | else | |
387 | return 0; | |
388 | } | |
389 | ||
390 | int | |
e3be4c23 | 391 | nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan, bool exclusive, bool intr) |
5e120f6e | 392 | { |
827520ce | 393 | struct nouveau_fence_chan *fctx = chan->fence; |
809e9447 | 394 | struct fence *fence; |
29ba89b2 ML |
395 | struct reservation_object *resv = nvbo->bo.resv; |
396 | struct reservation_object_list *fobj; | |
809e9447 | 397 | struct nouveau_fence *f; |
29ba89b2 ML |
398 | int ret = 0, i; |
399 | ||
809e9447 ML |
400 | if (!exclusive) { |
401 | ret = reservation_object_reserve_shared(resv); | |
402 | ||
403 | if (ret) | |
404 | return ret; | |
405 | } | |
406 | ||
407 | fobj = reservation_object_get_list(resv); | |
f2c24b83 | 408 | fence = reservation_object_get_excl(resv); |
29ba89b2 | 409 | |
809e9447 ML |
410 | if (fence && (!exclusive || !fobj || !fobj->shared_count)) { |
411 | struct nouveau_channel *prev = NULL; | |
0ec5f02f | 412 | bool must_wait = true; |
5e120f6e | 413 | |
809e9447 | 414 | f = nouveau_local_fence(fence, chan->drm); |
0ec5f02f ML |
415 | if (f) { |
416 | rcu_read_lock(); | |
417 | prev = rcu_dereference(f->channel); | |
418 | if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0)) | |
419 | must_wait = false; | |
420 | rcu_read_unlock(); | |
421 | } | |
809e9447 | 422 | |
0ec5f02f | 423 | if (must_wait) |
e3be4c23 | 424 | ret = fence_wait(fence, intr); |
5e120f6e | 425 | |
29ba89b2 | 426 | return ret; |
809e9447 | 427 | } |
5e120f6e | 428 | |
809e9447 | 429 | if (!exclusive || !fobj) |
29ba89b2 ML |
430 | return ret; |
431 | ||
432 | for (i = 0; i < fobj->shared_count && !ret; ++i) { | |
809e9447 | 433 | struct nouveau_channel *prev = NULL; |
0ec5f02f | 434 | bool must_wait = true; |
809e9447 | 435 | |
29ba89b2 ML |
436 | fence = rcu_dereference_protected(fobj->shared[i], |
437 | reservation_object_held(resv)); | |
438 | ||
809e9447 | 439 | f = nouveau_local_fence(fence, chan->drm); |
0ec5f02f ML |
440 | if (f) { |
441 | rcu_read_lock(); | |
442 | prev = rcu_dereference(f->channel); | |
443 | if (prev && (prev == chan || fctx->sync(f, prev, chan) == 0)) | |
444 | must_wait = false; | |
445 | rcu_read_unlock(); | |
446 | } | |
809e9447 | 447 | |
0ec5f02f | 448 | if (must_wait) |
e3be4c23 | 449 | ret = fence_wait(fence, intr); |
29ba89b2 ML |
450 | } |
451 | ||
452 | return ret; | |
d375e7d5 BS |
453 | } |
454 | ||
455 | void | |
456 | nouveau_fence_unref(struct nouveau_fence **pfence) | |
457 | { | |
458 | if (*pfence) | |
29ba89b2 | 459 | fence_put(&(*pfence)->base); |
d375e7d5 BS |
460 | *pfence = NULL; |
461 | } | |
462 | ||
d375e7d5 | 463 | int |
264ce192 BS |
464 | nouveau_fence_new(struct nouveau_channel *chan, bool sysmem, |
465 | struct nouveau_fence **pfence) | |
d375e7d5 BS |
466 | { |
467 | struct nouveau_fence *fence; | |
468 | int ret = 0; | |
6ee73861 | 469 | |
e193b1d4 | 470 | if (unlikely(!chan->fence)) |
5e120f6e BS |
471 | return -ENODEV; |
472 | ||
d375e7d5 BS |
473 | fence = kzalloc(sizeof(*fence), GFP_KERNEL); |
474 | if (!fence) | |
475 | return -ENOMEM; | |
264ce192 BS |
476 | |
477 | fence->sysmem = sysmem; | |
d375e7d5 | 478 | |
b5d8f052 CD |
479 | ret = nouveau_fence_emit(fence, chan); |
480 | if (ret) | |
481 | nouveau_fence_unref(&fence); | |
d375e7d5 BS |
482 | |
483 | *pfence = fence; | |
6ee73861 BS |
484 | return ret; |
485 | } | |
29ba89b2 ML |
486 | |
487 | static const char *nouveau_fence_get_get_driver_name(struct fence *fence) | |
488 | { | |
489 | return "nouveau"; | |
490 | } | |
491 | ||
492 | static const char *nouveau_fence_get_timeline_name(struct fence *f) | |
493 | { | |
494 | struct nouveau_fence *fence = from_fence(f); | |
495 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); | |
496 | ||
0ec5f02f | 497 | return !fctx->dead ? fctx->name : "dead channel"; |
29ba89b2 ML |
498 | } |
499 | ||
500 | /* | |
501 | * In an ideal world, read would not assume the channel context is still alive. | |
502 | * This function may be called from another device, running into free memory as a | |
503 | * result. The drm node should still be there, so we can derive the index from | |
504 | * the fence context. | |
505 | */ | |
506 | static bool nouveau_fence_is_signaled(struct fence *f) | |
507 | { | |
508 | struct nouveau_fence *fence = from_fence(f); | |
509 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); | |
0ec5f02f ML |
510 | struct nouveau_channel *chan; |
511 | bool ret = false; | |
512 | ||
513 | rcu_read_lock(); | |
514 | chan = rcu_dereference(fence->channel); | |
515 | if (chan) | |
516 | ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0; | |
517 | rcu_read_unlock(); | |
29ba89b2 | 518 | |
0ec5f02f | 519 | return ret; |
29ba89b2 ML |
520 | } |
521 | ||
522 | static bool nouveau_fence_no_signaling(struct fence *f) | |
523 | { | |
524 | struct nouveau_fence *fence = from_fence(f); | |
525 | ||
526 | /* | |
527 | * caller should have a reference on the fence, | |
528 | * else fence could get freed here | |
529 | */ | |
530 | WARN_ON(atomic_read(&fence->base.refcount.refcount) <= 1); | |
531 | ||
532 | /* | |
533 | * This needs uevents to work correctly, but fence_add_callback relies on | |
534 | * being able to enable signaling. It will still get signaled eventually, | |
535 | * just not right away. | |
536 | */ | |
537 | if (nouveau_fence_is_signaled(f)) { | |
538 | list_del(&fence->head); | |
539 | ||
540 | fence_put(&fence->base); | |
541 | return false; | |
542 | } | |
543 | ||
544 | return true; | |
545 | } | |
546 | ||
15a996bb ML |
547 | static void nouveau_fence_release(struct fence *f) |
548 | { | |
549 | struct nouveau_fence *fence = from_fence(f); | |
550 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); | |
551 | ||
552 | kref_put(&fctx->fence_ref, nouveau_fence_context_put); | |
553 | fence_free(&fence->base); | |
554 | } | |
555 | ||
29ba89b2 ML |
556 | static const struct fence_ops nouveau_fence_ops_legacy = { |
557 | .get_driver_name = nouveau_fence_get_get_driver_name, | |
558 | .get_timeline_name = nouveau_fence_get_timeline_name, | |
559 | .enable_signaling = nouveau_fence_no_signaling, | |
560 | .signaled = nouveau_fence_is_signaled, | |
561 | .wait = nouveau_fence_wait_legacy, | |
15a996bb | 562 | .release = nouveau_fence_release |
29ba89b2 ML |
563 | }; |
564 | ||
565 | static bool nouveau_fence_enable_signaling(struct fence *f) | |
566 | { | |
567 | struct nouveau_fence *fence = from_fence(f); | |
568 | struct nouveau_fence_chan *fctx = nouveau_fctx(fence); | |
569 | bool ret; | |
570 | ||
571 | if (!fctx->notify_ref++) | |
572 | nvif_notify_get(&fctx->notify); | |
573 | ||
574 | ret = nouveau_fence_no_signaling(f); | |
575 | if (ret) | |
576 | set_bit(FENCE_FLAG_USER_BITS, &fence->base.flags); | |
577 | else if (!--fctx->notify_ref) | |
578 | nvif_notify_put(&fctx->notify); | |
579 | ||
580 | return ret; | |
581 | } | |
582 | ||
583 | static const struct fence_ops nouveau_fence_ops_uevent = { | |
584 | .get_driver_name = nouveau_fence_get_get_driver_name, | |
585 | .get_timeline_name = nouveau_fence_get_timeline_name, | |
586 | .enable_signaling = nouveau_fence_enable_signaling, | |
587 | .signaled = nouveau_fence_is_signaled, | |
588 | .wait = fence_default_wait, | |
589 | .release = NULL | |
590 | }; |