Commit | Line | Data |
---|---|---|
9952f691 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
6579324a TB |
2 | /* |
3 | * Tegra host1x Command DMA | |
4 | * | |
5 | * Copyright (c) 2010-2013, NVIDIA Corporation. | |
6579324a TB |
6 | */ |
7 | ||
8 | ||
9 | #include <asm/cacheflush.h> | |
10 | #include <linux/device.h> | |
11 | #include <linux/dma-mapping.h> | |
35d747a8 | 12 | #include <linux/host1x.h> |
6579324a TB |
13 | #include <linux/interrupt.h> |
14 | #include <linux/kernel.h> | |
15 | #include <linux/kfifo.h> | |
16 | #include <linux/slab.h> | |
17 | #include <trace/events/host1x.h> | |
18 | ||
19 | #include "cdma.h" | |
20 | #include "channel.h" | |
21 | #include "dev.h" | |
22 | #include "debug.h" | |
6579324a TB |
23 | #include "job.h" |
24 | ||
25 | /* | |
26 | * push_buffer | |
27 | * | |
28 | * The push buffer is a circular array of words to be fetched by command DMA. | |
29 | * Note that it works slightly differently to the sync queue; fence == pos | |
30 | * means that the push buffer is full, not empty. | |
31 | */ | |
32 | ||
e1f338c0 TR |
33 | /* |
34 | * Typically the commands written into the push buffer are a pair of words. We | |
35 | * use slots to represent each of these pairs and to simplify things. Note the | |
36 | * strange number of slots allocated here. 512 slots will fit exactly within a | |
37 | * single memory page. We also need one additional word at the end of the push | |
38 | * buffer for the RESTART opcode that will instruct the CDMA to jump back to | |
39 | * the beginning of the push buffer. With 512 slots, this means that we'll use | |
40 | * 2 memory pages and waste 4092 bytes of the second page that will never be | |
41 | * used. | |
42 | */ | |
43 | #define HOST1X_PUSHBUFFER_SLOTS 511 | |
6579324a TB |
44 | |
45 | /* | |
46 | * Clean up push buffer resources | |
47 | */ | |
48 | static void host1x_pushbuffer_destroy(struct push_buffer *pb) | |
49 | { | |
50 | struct host1x_cdma *cdma = pb_to_cdma(pb); | |
51 | struct host1x *host1x = cdma_to_host1x(cdma); | |
52 | ||
2f8a6da8 | 53 | if (!pb->mapped) |
404bfb78 MP |
54 | return; |
55 | ||
56 | if (host1x->domain) { | |
57 | iommu_unmap(host1x->domain, pb->dma, pb->alloc_size); | |
58 | free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma)); | |
59 | } | |
60 | ||
61 | dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys); | |
6579324a TB |
62 | |
63 | pb->mapped = NULL; | |
64 | pb->phys = 0; | |
65 | } | |
66 | ||
67 | /* | |
68 | * Init push buffer resources | |
69 | */ | |
70 | static int host1x_pushbuffer_init(struct push_buffer *pb) | |
71 | { | |
72 | struct host1x_cdma *cdma = pb_to_cdma(pb); | |
73 | struct host1x *host1x = cdma_to_host1x(cdma); | |
404bfb78 MP |
74 | struct iova *alloc; |
75 | u32 size; | |
76 | int err; | |
6579324a TB |
77 | |
78 | pb->mapped = NULL; | |
79 | pb->phys = 0; | |
404bfb78 MP |
80 | pb->size = HOST1X_PUSHBUFFER_SLOTS * 8; |
81 | ||
82 | size = pb->size + 4; | |
6579324a TB |
83 | |
84 | /* initialize buffer pointers */ | |
404bfb78 | 85 | pb->fence = pb->size - 8; |
6579324a TB |
86 | pb->pos = 0; |
87 | ||
404bfb78 MP |
88 | if (host1x->domain) { |
89 | unsigned long shift; | |
90 | ||
91 | size = iova_align(&host1x->iova, size); | |
92 | ||
93 | pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, | |
94 | GFP_KERNEL); | |
95 | if (!pb->mapped) | |
96 | return -ENOMEM; | |
97 | ||
98 | shift = iova_shift(&host1x->iova); | |
99 | alloc = alloc_iova(&host1x->iova, size >> shift, | |
100 | host1x->iova_end >> shift, true); | |
101 | if (!alloc) { | |
102 | err = -ENOMEM; | |
103 | goto iommu_free_mem; | |
104 | } | |
105 | ||
106 | pb->dma = iova_dma_addr(&host1x->iova, alloc); | |
107 | err = iommu_map(host1x->domain, pb->dma, pb->phys, size, | |
108 | IOMMU_READ); | |
109 | if (err) | |
110 | goto iommu_free_iova; | |
111 | } else { | |
112 | pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys, | |
113 | GFP_KERNEL); | |
114 | if (!pb->mapped) | |
115 | return -ENOMEM; | |
116 | ||
117 | pb->dma = pb->phys; | |
118 | } | |
119 | ||
120 | pb->alloc_size = size; | |
6579324a TB |
121 | |
122 | host1x_hw_pushbuffer_init(host1x, pb); | |
123 | ||
124 | return 0; | |
125 | ||
404bfb78 MP |
126 | iommu_free_iova: |
127 | __free_iova(&host1x->iova, alloc); | |
128 | iommu_free_mem: | |
27db6a00 | 129 | dma_free_wc(host1x->dev, size, pb->mapped, pb->phys); |
404bfb78 MP |
130 | |
131 | return err; | |
6579324a TB |
132 | } |
133 | ||
134 | /* | |
135 | * Push two words to the push buffer | |
136 | * Caller must ensure push buffer is not full | |
137 | */ | |
138 | static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2) | |
139 | { | |
ebb2475c TR |
140 | u32 *p = (u32 *)((void *)pb->mapped + pb->pos); |
141 | ||
142 | WARN_ON(pb->pos == pb->fence); | |
6579324a TB |
143 | *(p++) = op1; |
144 | *(p++) = op2; | |
e1f338c0 TR |
145 | pb->pos += 8; |
146 | ||
147 | if (pb->pos >= pb->size) | |
148 | pb->pos -= pb->size; | |
6579324a TB |
149 | } |
150 | ||
151 | /* | |
152 | * Pop a number of two word slots from the push buffer | |
153 | * Caller must ensure push buffer is not empty | |
154 | */ | |
155 | static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots) | |
156 | { | |
157 | /* Advance the next write position */ | |
e1f338c0 TR |
158 | pb->fence += slots * 8; |
159 | ||
160 | if (pb->fence >= pb->size) | |
161 | pb->fence -= pb->size; | |
6579324a TB |
162 | } |
163 | ||
164 | /* | |
165 | * Return the number of two word slots free in the push buffer | |
166 | */ | |
167 | static u32 host1x_pushbuffer_space(struct push_buffer *pb) | |
168 | { | |
e1f338c0 TR |
169 | unsigned int fence = pb->fence; |
170 | ||
171 | if (pb->fence < pb->pos) | |
172 | fence += pb->size; | |
173 | ||
174 | return (fence - pb->pos) / 8; | |
6579324a TB |
175 | } |
176 | ||
177 | /* | |
178 | * Sleep (if necessary) until the requested event happens | |
179 | * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty. | |
180 | * - Returns 1 | |
181 | * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer | |
182 | * - Return the amount of space (> 0) | |
183 | * Must be called with the cdma lock held. | |
184 | */ | |
185 | unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma, | |
186 | enum cdma_event event) | |
187 | { | |
188 | for (;;) { | |
0b8070d1 | 189 | struct push_buffer *pb = &cdma->push_buffer; |
6579324a TB |
190 | unsigned int space; |
191 | ||
0b8070d1 TR |
192 | switch (event) { |
193 | case CDMA_EVENT_SYNC_QUEUE_EMPTY: | |
6579324a | 194 | space = list_empty(&cdma->sync_queue) ? 1 : 0; |
0b8070d1 TR |
195 | break; |
196 | ||
197 | case CDMA_EVENT_PUSH_BUFFER_SPACE: | |
6579324a | 198 | space = host1x_pushbuffer_space(pb); |
0b8070d1 TR |
199 | break; |
200 | ||
201 | default: | |
6579324a TB |
202 | WARN_ON(1); |
203 | return -EINVAL; | |
204 | } | |
205 | ||
206 | if (space) | |
207 | return space; | |
208 | ||
209 | trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), | |
210 | event); | |
211 | ||
212 | /* If somebody has managed to already start waiting, yield */ | |
213 | if (cdma->event != CDMA_EVENT_NONE) { | |
214 | mutex_unlock(&cdma->lock); | |
215 | schedule(); | |
216 | mutex_lock(&cdma->lock); | |
217 | continue; | |
218 | } | |
0b8070d1 | 219 | |
6579324a TB |
220 | cdma->event = event; |
221 | ||
222 | mutex_unlock(&cdma->lock); | |
0747a672 | 223 | wait_for_completion(&cdma->complete); |
6579324a TB |
224 | mutex_lock(&cdma->lock); |
225 | } | |
0b8070d1 | 226 | |
6579324a TB |
227 | return 0; |
228 | } | |
229 | ||
5a5fccbd TR |
230 | /* |
231 | * Sleep (if necessary) until the push buffer has enough free space. | |
232 | * | |
233 | * Must be called with the cdma lock held. | |
234 | */ | |
235 | int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x, | |
236 | struct host1x_cdma *cdma, | |
237 | unsigned int needed) | |
238 | { | |
239 | while (true) { | |
240 | struct push_buffer *pb = &cdma->push_buffer; | |
241 | unsigned int space; | |
242 | ||
243 | space = host1x_pushbuffer_space(pb); | |
244 | if (space >= needed) | |
245 | break; | |
246 | ||
247 | trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev), | |
248 | CDMA_EVENT_PUSH_BUFFER_SPACE); | |
249 | ||
250 | host1x_hw_cdma_flush(host1x, cdma); | |
251 | ||
252 | /* If somebody has managed to already start waiting, yield */ | |
253 | if (cdma->event != CDMA_EVENT_NONE) { | |
254 | mutex_unlock(&cdma->lock); | |
255 | schedule(); | |
256 | mutex_lock(&cdma->lock); | |
257 | continue; | |
258 | } | |
259 | ||
260 | cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE; | |
261 | ||
262 | mutex_unlock(&cdma->lock); | |
263 | wait_for_completion(&cdma->complete); | |
264 | mutex_lock(&cdma->lock); | |
265 | } | |
266 | ||
267 | return 0; | |
268 | } | |
6579324a TB |
269 | /* |
270 | * Start timer that tracks the time spent by the job. | |
271 | * Must be called with the cdma lock held. | |
272 | */ | |
273 | static void cdma_start_timer_locked(struct host1x_cdma *cdma, | |
274 | struct host1x_job *job) | |
275 | { | |
276 | struct host1x *host = cdma_to_host1x(cdma); | |
277 | ||
278 | if (cdma->timeout.client) { | |
279 | /* timer already started */ | |
280 | return; | |
281 | } | |
282 | ||
283 | cdma->timeout.client = job->client; | |
284 | cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id); | |
285 | cdma->timeout.syncpt_val = job->syncpt_end; | |
286 | cdma->timeout.start_ktime = ktime_get(); | |
287 | ||
288 | schedule_delayed_work(&cdma->timeout.wq, | |
289 | msecs_to_jiffies(job->timeout)); | |
290 | } | |
291 | ||
292 | /* | |
293 | * Stop timer when a buffer submission completes. | |
294 | * Must be called with the cdma lock held. | |
295 | */ | |
296 | static void stop_cdma_timer_locked(struct host1x_cdma *cdma) | |
297 | { | |
298 | cancel_delayed_work(&cdma->timeout.wq); | |
bf3d41cc | 299 | cdma->timeout.client = NULL; |
6579324a TB |
300 | } |
301 | ||
302 | /* | |
303 | * For all sync queue entries that have already finished according to the | |
304 | * current sync point registers: | |
305 | * - unpin & unref their mems | |
306 | * - pop their push buffer slots | |
307 | * - remove them from the sync queue | |
308 | * This is normally called from the host code's worker thread, but can be | |
309 | * called manually if necessary. | |
310 | * Must be called with the cdma lock held. | |
311 | */ | |
312 | static void update_cdma_locked(struct host1x_cdma *cdma) | |
313 | { | |
314 | bool signal = false; | |
315 | struct host1x *host1x = cdma_to_host1x(cdma); | |
316 | struct host1x_job *job, *n; | |
317 | ||
318 | /* If CDMA is stopped, queue is cleared and we can return */ | |
319 | if (!cdma->running) | |
320 | return; | |
321 | ||
322 | /* | |
323 | * Walk the sync queue, reading the sync point registers as necessary, | |
324 | * to consume as many sync queue entries as possible without blocking | |
325 | */ | |
326 | list_for_each_entry_safe(job, n, &cdma->sync_queue, list) { | |
327 | struct host1x_syncpt *sp = | |
328 | host1x_syncpt_get(host1x, job->syncpt_id); | |
329 | ||
330 | /* Check whether this syncpt has completed, and bail if not */ | |
331 | if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) { | |
332 | /* Start timer on next pending syncpt */ | |
333 | if (job->timeout) | |
334 | cdma_start_timer_locked(cdma, job); | |
0b8070d1 | 335 | |
6579324a TB |
336 | break; |
337 | } | |
338 | ||
339 | /* Cancel timeout, when a buffer completes */ | |
340 | if (cdma->timeout.client) | |
341 | stop_cdma_timer_locked(cdma); | |
342 | ||
343 | /* Unpin the memory */ | |
344 | host1x_job_unpin(job); | |
345 | ||
346 | /* Pop push buffer slots */ | |
347 | if (job->num_slots) { | |
348 | struct push_buffer *pb = &cdma->push_buffer; | |
6df633d0 | 349 | |
6579324a | 350 | host1x_pushbuffer_pop(pb, job->num_slots); |
0b8070d1 | 351 | |
6579324a TB |
352 | if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE) |
353 | signal = true; | |
354 | } | |
355 | ||
356 | list_del(&job->list); | |
357 | host1x_job_put(job); | |
358 | } | |
359 | ||
360 | if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY && | |
361 | list_empty(&cdma->sync_queue)) | |
362 | signal = true; | |
363 | ||
364 | if (signal) { | |
365 | cdma->event = CDMA_EVENT_NONE; | |
0747a672 | 366 | complete(&cdma->complete); |
6579324a TB |
367 | } |
368 | } | |
369 | ||
370 | void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma, | |
371 | struct device *dev) | |
372 | { | |
6579324a | 373 | struct host1x *host1x = cdma_to_host1x(cdma); |
0b8070d1 | 374 | u32 restart_addr, syncpt_incrs, syncpt_val; |
79930baf | 375 | struct host1x_job *job, *next_job = NULL; |
6579324a TB |
376 | |
377 | syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt); | |
378 | ||
379 | dev_dbg(dev, "%s: starting cleanup (thresh %d)\n", | |
380 | __func__, syncpt_val); | |
381 | ||
382 | /* | |
383 | * Move the sync_queue read pointer to the first entry that hasn't | |
384 | * completed based on the current HW syncpt value. It's likely there | |
385 | * won't be any (i.e. we're still at the head), but covers the case | |
386 | * where a syncpt incr happens just prior/during the teardown. | |
387 | */ | |
388 | ||
389 | dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n", | |
390 | __func__); | |
391 | ||
392 | list_for_each_entry(job, &cdma->sync_queue, list) { | |
79930baf DO |
393 | if (syncpt_val < job->syncpt_end) { |
394 | ||
395 | if (!list_is_last(&job->list, &cdma->sync_queue)) | |
396 | next_job = list_next_entry(job, list); | |
397 | ||
5d6f0436 | 398 | goto syncpt_incr; |
79930baf | 399 | } |
6579324a TB |
400 | |
401 | host1x_job_dump(dev, job); | |
402 | } | |
403 | ||
5d6f0436 DO |
404 | /* all jobs have been completed */ |
405 | job = NULL; | |
406 | ||
407 | syncpt_incr: | |
408 | ||
6579324a | 409 | /* |
e8bad659 | 410 | * Increment with CPU the remaining syncpts of a partially executed job. |
6579324a | 411 | * |
79930baf DO |
412 | * CDMA will continue execution starting with the next job or will get |
413 | * into idle state. | |
6579324a | 414 | */ |
79930baf DO |
415 | if (next_job) |
416 | restart_addr = next_job->first_get; | |
6579324a TB |
417 | else |
418 | restart_addr = cdma->last_pos; | |
419 | ||
e8bad659 DO |
420 | /* do CPU increments for the remaining syncpts */ |
421 | if (job) { | |
5d6f0436 DO |
422 | dev_dbg(dev, "%s: perform CPU incr on pending buffers\n", |
423 | __func__); | |
424 | ||
6579324a TB |
425 | /* won't need a timeout when replayed */ |
426 | job->timeout = 0; | |
427 | ||
428 | syncpt_incrs = job->syncpt_end - syncpt_val; | |
429 | dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs); | |
430 | ||
431 | host1x_job_dump(dev, job); | |
432 | ||
433 | /* safe to use CPU to incr syncpts */ | |
434 | host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get, | |
435 | syncpt_incrs, job->syncpt_end, | |
436 | job->num_slots); | |
6579324a | 437 | |
5d6f0436 DO |
438 | dev_dbg(dev, "%s: finished sync_queue modification\n", |
439 | __func__); | |
440 | } | |
6579324a TB |
441 | |
442 | /* roll back DMAGET and start up channel again */ | |
443 | host1x_hw_cdma_resume(host1x, cdma, restart_addr); | |
444 | } | |
445 | ||
446 | /* | |
447 | * Create a cdma | |
448 | */ | |
449 | int host1x_cdma_init(struct host1x_cdma *cdma) | |
450 | { | |
451 | int err; | |
452 | ||
453 | mutex_init(&cdma->lock); | |
0747a672 | 454 | init_completion(&cdma->complete); |
6579324a TB |
455 | |
456 | INIT_LIST_HEAD(&cdma->sync_queue); | |
457 | ||
458 | cdma->event = CDMA_EVENT_NONE; | |
459 | cdma->running = false; | |
460 | cdma->torndown = false; | |
461 | ||
462 | err = host1x_pushbuffer_init(&cdma->push_buffer); | |
463 | if (err) | |
464 | return err; | |
0b8070d1 | 465 | |
6579324a TB |
466 | return 0; |
467 | } | |
468 | ||
469 | /* | |
470 | * Destroy a cdma | |
471 | */ | |
472 | int host1x_cdma_deinit(struct host1x_cdma *cdma) | |
473 | { | |
474 | struct push_buffer *pb = &cdma->push_buffer; | |
475 | struct host1x *host1x = cdma_to_host1x(cdma); | |
476 | ||
477 | if (cdma->running) { | |
478 | pr_warn("%s: CDMA still running\n", __func__); | |
479 | return -EBUSY; | |
480 | } | |
481 | ||
482 | host1x_pushbuffer_destroy(pb); | |
483 | host1x_hw_cdma_timeout_destroy(host1x, cdma); | |
484 | ||
485 | return 0; | |
486 | } | |
487 | ||
488 | /* | |
489 | * Begin a cdma submit | |
490 | */ | |
491 | int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job) | |
492 | { | |
493 | struct host1x *host1x = cdma_to_host1x(cdma); | |
494 | ||
495 | mutex_lock(&cdma->lock); | |
496 | ||
497 | if (job->timeout) { | |
498 | /* init state on first submit with timeout value */ | |
499 | if (!cdma->timeout.initialized) { | |
500 | int err; | |
6df633d0 | 501 | |
6579324a TB |
502 | err = host1x_hw_cdma_timeout_init(host1x, cdma, |
503 | job->syncpt_id); | |
504 | if (err) { | |
505 | mutex_unlock(&cdma->lock); | |
506 | return err; | |
507 | } | |
508 | } | |
509 | } | |
0b8070d1 | 510 | |
6579324a TB |
511 | if (!cdma->running) |
512 | host1x_hw_cdma_start(host1x, cdma); | |
513 | ||
514 | cdma->slots_free = 0; | |
515 | cdma->slots_used = 0; | |
516 | cdma->first_get = cdma->push_buffer.pos; | |
517 | ||
518 | trace_host1x_cdma_begin(dev_name(job->channel->dev)); | |
519 | return 0; | |
520 | } | |
521 | ||
522 | /* | |
523 | * Push two words into a push buffer slot | |
524 | * Blocks as necessary if the push buffer is full. | |
525 | */ | |
526 | void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2) | |
527 | { | |
528 | struct host1x *host1x = cdma_to_host1x(cdma); | |
529 | struct push_buffer *pb = &cdma->push_buffer; | |
530 | u32 slots_free = cdma->slots_free; | |
531 | ||
6236451d TB |
532 | if (host1x_debug_trace_cmdbuf) |
533 | trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev), | |
534 | op1, op2); | |
535 | ||
6579324a TB |
536 | if (slots_free == 0) { |
537 | host1x_hw_cdma_flush(host1x, cdma); | |
538 | slots_free = host1x_cdma_wait_locked(cdma, | |
539 | CDMA_EVENT_PUSH_BUFFER_SPACE); | |
540 | } | |
0b8070d1 | 541 | |
6579324a TB |
542 | cdma->slots_free = slots_free - 1; |
543 | cdma->slots_used++; | |
544 | host1x_pushbuffer_push(pb, op1, op2); | |
545 | } | |
546 | ||
5a5fccbd TR |
547 | /* |
548 | * Push four words into two consecutive push buffer slots. Note that extra | |
549 | * care needs to be taken not to split the two slots across the end of the | |
550 | * push buffer. Otherwise the RESTART opcode at the end of the push buffer | |
551 | * that ensures processing will restart at the beginning will break up the | |
552 | * four words. | |
553 | * | |
554 | * Blocks as necessary if the push buffer is full. | |
555 | */ | |
556 | void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2, | |
557 | u32 op3, u32 op4) | |
558 | { | |
559 | struct host1x_channel *channel = cdma_to_channel(cdma); | |
560 | struct host1x *host1x = cdma_to_host1x(cdma); | |
561 | struct push_buffer *pb = &cdma->push_buffer; | |
562 | unsigned int needed = 2, extra = 0, i; | |
563 | unsigned int space = cdma->slots_free; | |
564 | ||
565 | if (host1x_debug_trace_cmdbuf) | |
566 | trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2, | |
567 | op3, op4); | |
568 | ||
569 | /* compute number of extra slots needed for padding */ | |
570 | if (pb->pos + 16 > pb->size) { | |
571 | extra = (pb->size - pb->pos) / 8; | |
572 | needed += extra; | |
573 | } | |
574 | ||
575 | host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed); | |
576 | space = host1x_pushbuffer_space(pb); | |
577 | ||
578 | cdma->slots_free = space - needed; | |
579 | cdma->slots_used += needed; | |
580 | ||
581 | /* | |
582 | * Note that we rely on the fact that this is only used to submit wide | |
583 | * gather opcodes, which consist of 3 words, and they are padded with | |
584 | * a NOP to avoid having to deal with fractional slots (a slot always | |
585 | * represents 2 words). The fourth opcode passed to this function will | |
586 | * therefore always be a NOP. | |
587 | * | |
588 | * This works around a slight ambiguity when it comes to opcodes. For | |
589 | * all current host1x incarnations the NOP opcode uses the exact same | |
590 | * encoding (0x20000000), so we could hard-code the value here, but a | |
591 | * new incarnation may change it and break that assumption. | |
592 | */ | |
593 | for (i = 0; i < extra; i++) | |
594 | host1x_pushbuffer_push(pb, op4, op4); | |
595 | ||
596 | host1x_pushbuffer_push(pb, op1, op2); | |
597 | host1x_pushbuffer_push(pb, op3, op4); | |
598 | } | |
599 | ||
6579324a TB |
600 | /* |
601 | * End a cdma submit | |
602 | * Kick off DMA, add job to the sync queue, and a number of slots to be freed | |
603 | * from the pushbuffer. The handles for a submit must all be pinned at the same | |
604 | * time, but they can be unpinned in smaller chunks. | |
605 | */ | |
606 | void host1x_cdma_end(struct host1x_cdma *cdma, | |
607 | struct host1x_job *job) | |
608 | { | |
609 | struct host1x *host1x = cdma_to_host1x(cdma); | |
610 | bool idle = list_empty(&cdma->sync_queue); | |
611 | ||
612 | host1x_hw_cdma_flush(host1x, cdma); | |
613 | ||
614 | job->first_get = cdma->first_get; | |
615 | job->num_slots = cdma->slots_used; | |
616 | host1x_job_get(job); | |
617 | list_add_tail(&job->list, &cdma->sync_queue); | |
618 | ||
619 | /* start timer on idle -> active transitions */ | |
620 | if (job->timeout && idle) | |
621 | cdma_start_timer_locked(cdma, job); | |
622 | ||
623 | trace_host1x_cdma_end(dev_name(job->channel->dev)); | |
624 | mutex_unlock(&cdma->lock); | |
625 | } | |
626 | ||
627 | /* | |
628 | * Update cdma state according to current sync point values | |
629 | */ | |
630 | void host1x_cdma_update(struct host1x_cdma *cdma) | |
631 | { | |
632 | mutex_lock(&cdma->lock); | |
633 | update_cdma_locked(cdma); | |
634 | mutex_unlock(&cdma->lock); | |
635 | } |