1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
4 * Copyright 2015 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include <drm/ttm/ttm_bo_api.h>
30 #include "vmwgfx_drv.h"
33 * Size of inline command buffers. Try to make sure that a page size is a
34 * multiple of the DMA pool allocation size.
36 #define VMW_CMDBUF_INLINE_ALIGN 64
37 #define VMW_CMDBUF_INLINE_SIZE \
38 (1024 - ALIGN(sizeof(SVGACBHeader), VMW_CMDBUF_INLINE_ALIGN))
41 * struct vmw_cmdbuf_context - Command buffer context queues
43 * @submitted: List of command buffers that have been submitted to the
44 * manager but not yet submitted to hardware.
45 * @hw_submitted: List of command buffers submitted to hardware.
46 * @preempted: List of preempted command buffers.
47 * @num_hw_submitted: Number of buffers currently being processed by hardware
49 struct vmw_cmdbuf_context {
50 struct list_head submitted;
51 struct list_head hw_submitted;
52 struct list_head preempted;
53 unsigned num_hw_submitted;
54 bool block_submission;
58 * struct vmw_cmdbuf_man: - Command buffer manager
60 * @cur_mutex: Mutex protecting the command buffer used for incremental small
61 * kernel command submissions, @cur.
62 * @space_mutex: Mutex to protect against starvation when we allocate
63 * main pool buffer space.
64 * @error_mutex: Mutex to serialize the work queue error handling.
65 * Note this is not needed if the same workqueue handler
66 * can't race with itself...
67 * @work: A struct work_struct implementeing command buffer error handling.
69 * @dev_priv: Pointer to the device private struct. Immutable.
70 * @ctx: Array of command buffer context queues. The queues and the context
71 * data is protected by @lock.
72 * @error: List of command buffers that have caused device errors.
74 * @mm: Range manager for the command buffer space. Manager allocations and
75 * frees are protected by @lock.
76 * @cmd_space: Buffer object for the command buffer space, unless we were
77 * able to make a contigous coherent DMA memory allocation, @handle. Immutable.
78 * @map_obj: Mapping state for @cmd_space. Immutable.
79 * @map: Pointer to command buffer space. May be a mapped buffer object or
80 * a contigous coherent DMA memory allocation. Immutable.
81 * @cur: Command buffer for small kernel command submissions. Protected by
83 * @cur_pos: Space already used in @cur. Protected by @cur_mutex.
84 * @default_size: Default size for the @cur command buffer. Immutable.
85 * @max_hw_submitted: Max number of in-flight command buffers the device can
87 * @lock: Spinlock protecting command submission queues.
88 * @header: Pool of DMA memory for device command buffer headers.
89 * Internal protection.
90 * @dheaders: Pool of DMA memory for device command buffer headers with trailing
91 * space for inline data. Internal protection.
92 * @alloc_queue: Wait queue for processes waiting to allocate command buffer
94 * @idle_queue: Wait queue for processes waiting for command buffer idle.
95 * @irq_on: Whether the process function has requested irq to be turned on.
97 * @using_mob: Whether the command buffer space is a MOB or a contigous DMA
98 * allocation. Immutable.
99 * @has_pool: Has a large pool of DMA memory which allows larger allocations.
100 * Typically this is false only during bootstrap.
101 * @handle: DMA address handle for the command buffer space if @using_mob is
103 * @size: The size of the command buffer space. Immutable.
104 * @num_contexts: Number of contexts actually enabled.
106 struct vmw_cmdbuf_man {
107 struct mutex cur_mutex;
108 struct mutex space_mutex;
109 struct mutex error_mutex;
110 struct work_struct work;
111 struct vmw_private *dev_priv;
112 struct vmw_cmdbuf_context ctx[SVGA_CB_CONTEXT_MAX];
113 struct list_head error;
115 struct ttm_buffer_object *cmd_space;
116 struct ttm_bo_kmap_obj map_obj;
118 struct vmw_cmdbuf_header *cur;
121 unsigned max_hw_submitted;
123 struct dma_pool *headers;
124 struct dma_pool *dheaders;
125 wait_queue_head_t alloc_queue;
126 wait_queue_head_t idle_queue;
136 * struct vmw_cmdbuf_header - Command buffer metadata
138 * @man: The command buffer manager.
139 * @cb_header: Device command buffer header, allocated from a DMA pool.
140 * @cb_context: The device command buffer context.
141 * @list: List head for attaching to the manager lists.
142 * @node: The range manager node.
143 * @handle. The DMA address of @cb_header. Handed to the device on command
145 * @cmd: Pointer to the command buffer space of this buffer.
146 * @size: Size of the command buffer space of this buffer.
147 * @reserved: Reserved space of this buffer.
148 * @inline_space: Whether inline command buffer space is used.
150 struct vmw_cmdbuf_header {
151 struct vmw_cmdbuf_man *man;
152 SVGACBHeader *cb_header;
153 SVGACBContext cb_context;
154 struct list_head list;
155 struct drm_mm_node node;
164 * struct vmw_cmdbuf_dheader - Device command buffer header with inline
165 * command buffer space.
167 * @cb_header: Device command buffer header.
168 * @cmd: Inline command buffer space.
170 struct vmw_cmdbuf_dheader {
171 SVGACBHeader cb_header;
172 u8 cmd[VMW_CMDBUF_INLINE_SIZE] __aligned(VMW_CMDBUF_INLINE_ALIGN);
176 * struct vmw_cmdbuf_alloc_info - Command buffer space allocation metadata
178 * @page_size: Size of requested command buffer space in pages.
179 * @node: Pointer to the range manager node.
180 * @done: True if this allocation has succeeded.
182 struct vmw_cmdbuf_alloc_info {
184 struct drm_mm_node *node;
188 /* Loop over each context in the command buffer manager. */
189 #define for_each_cmdbuf_ctx(_man, _i, _ctx) \
190 for (_i = 0, _ctx = &(_man)->ctx[0]; (_i) < (_man)->num_contexts; \
193 static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man, u32 context,
195 static int vmw_cmdbuf_preempt(struct vmw_cmdbuf_man *man, u32 context);
198 * vmw_cmdbuf_cur_lock - Helper to lock the cur_mutex.
200 * @man: The range manager.
201 * @interruptible: Whether to wait interruptible when locking.
203 static int vmw_cmdbuf_cur_lock(struct vmw_cmdbuf_man *man, bool interruptible)
206 if (mutex_lock_interruptible(&man->cur_mutex))
209 mutex_lock(&man->cur_mutex);
216 * vmw_cmdbuf_cur_unlock - Helper to unlock the cur_mutex.
218 * @man: The range manager.
220 static void vmw_cmdbuf_cur_unlock(struct vmw_cmdbuf_man *man)
222 mutex_unlock(&man->cur_mutex);
226 * vmw_cmdbuf_header_inline_free - Free a struct vmw_cmdbuf_header that has
227 * been used for the device context with inline command buffers.
228 * Need not be called locked.
230 * @header: Pointer to the header to free.
232 static void vmw_cmdbuf_header_inline_free(struct vmw_cmdbuf_header *header)
234 struct vmw_cmdbuf_dheader *dheader;
236 if (WARN_ON_ONCE(!header->inline_space))
239 dheader = container_of(header->cb_header, struct vmw_cmdbuf_dheader,
241 dma_pool_free(header->man->dheaders, dheader, header->handle);
246 * __vmw_cmdbuf_header_free - Free a struct vmw_cmdbuf_header and its
247 * associated structures.
249 * header: Pointer to the header to free.
251 * For internal use. Must be called with man::lock held.
253 static void __vmw_cmdbuf_header_free(struct vmw_cmdbuf_header *header)
255 struct vmw_cmdbuf_man *man = header->man;
257 lockdep_assert_held_once(&man->lock);
259 if (header->inline_space) {
260 vmw_cmdbuf_header_inline_free(header);
264 drm_mm_remove_node(&header->node);
265 wake_up_all(&man->alloc_queue);
266 if (header->cb_header)
267 dma_pool_free(man->headers, header->cb_header,
273 * vmw_cmdbuf_header_free - Free a struct vmw_cmdbuf_header and its
274 * associated structures.
276 * @header: Pointer to the header to free.
278 void vmw_cmdbuf_header_free(struct vmw_cmdbuf_header *header)
280 struct vmw_cmdbuf_man *man = header->man;
282 /* Avoid locking if inline_space */
283 if (header->inline_space) {
284 vmw_cmdbuf_header_inline_free(header);
287 spin_lock(&man->lock);
288 __vmw_cmdbuf_header_free(header);
289 spin_unlock(&man->lock);
294 * vmw_cmbuf_header_submit: Submit a command buffer to hardware.
296 * @header: The header of the buffer to submit.
298 static int vmw_cmdbuf_header_submit(struct vmw_cmdbuf_header *header)
300 struct vmw_cmdbuf_man *man = header->man;
303 val = upper_32_bits(header->handle);
304 vmw_write(man->dev_priv, SVGA_REG_COMMAND_HIGH, val);
306 val = lower_32_bits(header->handle);
307 val |= header->cb_context & SVGA_CB_CONTEXT_MASK;
308 vmw_write(man->dev_priv, SVGA_REG_COMMAND_LOW, val);
310 return header->cb_header->status;
314 * vmw_cmdbuf_ctx_init: Initialize a command buffer context.
316 * @ctx: The command buffer context to initialize
318 static void vmw_cmdbuf_ctx_init(struct vmw_cmdbuf_context *ctx)
320 INIT_LIST_HEAD(&ctx->hw_submitted);
321 INIT_LIST_HEAD(&ctx->submitted);
322 INIT_LIST_HEAD(&ctx->preempted);
323 ctx->num_hw_submitted = 0;
327 * vmw_cmdbuf_ctx_submit: Submit command buffers from a command buffer
330 * @man: The command buffer manager.
331 * @ctx: The command buffer context.
333 * Submits command buffers to hardware until there are no more command
334 * buffers to submit or the hardware can't handle more command buffers.
336 static void vmw_cmdbuf_ctx_submit(struct vmw_cmdbuf_man *man,
337 struct vmw_cmdbuf_context *ctx)
339 while (ctx->num_hw_submitted < man->max_hw_submitted &&
340 !list_empty(&ctx->submitted) &&
341 !ctx->block_submission) {
342 struct vmw_cmdbuf_header *entry;
345 entry = list_first_entry(&ctx->submitted,
346 struct vmw_cmdbuf_header,
349 status = vmw_cmdbuf_header_submit(entry);
351 /* This should never happen */
352 if (WARN_ON_ONCE(status == SVGA_CB_STATUS_QUEUE_FULL)) {
353 entry->cb_header->status = SVGA_CB_STATUS_NONE;
357 list_del(&entry->list);
358 list_add_tail(&entry->list, &ctx->hw_submitted);
359 ctx->num_hw_submitted++;
365 * vmw_cmdbuf_ctx_submit: Process a command buffer context.
367 * @man: The command buffer manager.
368 * @ctx: The command buffer context.
370 * Submit command buffers to hardware if possible, and process finished
371 * buffers. Typically freeing them, but on preemption or error take
372 * appropriate action. Wake up waiters if appropriate.
374 static void vmw_cmdbuf_ctx_process(struct vmw_cmdbuf_man *man,
375 struct vmw_cmdbuf_context *ctx,
378 struct vmw_cmdbuf_header *entry, *next;
380 vmw_cmdbuf_ctx_submit(man, ctx);
382 list_for_each_entry_safe(entry, next, &ctx->hw_submitted, list) {
383 SVGACBStatus status = entry->cb_header->status;
385 if (status == SVGA_CB_STATUS_NONE)
388 list_del(&entry->list);
389 wake_up_all(&man->idle_queue);
390 ctx->num_hw_submitted--;
392 case SVGA_CB_STATUS_COMPLETED:
393 __vmw_cmdbuf_header_free(entry);
395 case SVGA_CB_STATUS_COMMAND_ERROR:
396 WARN_ONCE(true, "Command buffer error.\n");
397 entry->cb_header->status = SVGA_CB_STATUS_NONE;
398 list_add_tail(&entry->list, &man->error);
399 schedule_work(&man->work);
401 case SVGA_CB_STATUS_PREEMPTED:
402 entry->cb_header->status = SVGA_CB_STATUS_NONE;
403 list_add_tail(&entry->list, &ctx->preempted);
405 case SVGA_CB_STATUS_CB_HEADER_ERROR:
406 WARN_ONCE(true, "Command buffer header error.\n");
407 __vmw_cmdbuf_header_free(entry);
410 WARN_ONCE(true, "Undefined command buffer status.\n");
411 __vmw_cmdbuf_header_free(entry);
416 vmw_cmdbuf_ctx_submit(man, ctx);
417 if (!list_empty(&ctx->submitted))
422 * vmw_cmdbuf_man_process - Process all command buffer contexts and
423 * switch on and off irqs as appropriate.
425 * @man: The command buffer manager.
427 * Calls vmw_cmdbuf_ctx_process() on all contexts. If any context has
428 * command buffers left that are not submitted to hardware, Make sure
429 * IRQ handling is turned on. Otherwise, make sure it's turned off.
431 static void vmw_cmdbuf_man_process(struct vmw_cmdbuf_man *man)
434 struct vmw_cmdbuf_context *ctx;
439 for_each_cmdbuf_ctx(man, i, ctx)
440 vmw_cmdbuf_ctx_process(man, ctx, ¬empty);
442 if (man->irq_on && !notempty) {
443 vmw_generic_waiter_remove(man->dev_priv,
444 SVGA_IRQFLAG_COMMAND_BUFFER,
445 &man->dev_priv->cmdbuf_waiters);
447 } else if (!man->irq_on && notempty) {
448 vmw_generic_waiter_add(man->dev_priv,
449 SVGA_IRQFLAG_COMMAND_BUFFER,
450 &man->dev_priv->cmdbuf_waiters);
453 /* Rerun in case we just missed an irq. */
459 * vmw_cmdbuf_ctx_add - Schedule a command buffer for submission on a
460 * command buffer context
462 * @man: The command buffer manager.
463 * @header: The header of the buffer to submit.
464 * @cb_context: The command buffer context to use.
466 * This function adds @header to the "submitted" queue of the command
467 * buffer context identified by @cb_context. It then calls the command buffer
468 * manager processing to potentially submit the buffer to hardware.
469 * @man->lock needs to be held when calling this function.
471 static void vmw_cmdbuf_ctx_add(struct vmw_cmdbuf_man *man,
472 struct vmw_cmdbuf_header *header,
473 SVGACBContext cb_context)
475 if (!(header->cb_header->flags & SVGA_CB_FLAG_DX_CONTEXT))
476 header->cb_header->dxContext = 0;
477 header->cb_context = cb_context;
478 list_add_tail(&header->list, &man->ctx[cb_context].submitted);
480 vmw_cmdbuf_man_process(man);
484 * vmw_cmdbuf_irqthread - The main part of the command buffer interrupt
485 * handler implemented as a threaded irq task.
487 * @man: Pointer to the command buffer manager.
489 * The bottom half of the interrupt handler simply calls into the
490 * command buffer processor to free finished buffers and submit any
491 * queued buffers to hardware.
493 void vmw_cmdbuf_irqthread(struct vmw_cmdbuf_man *man)
495 spin_lock(&man->lock);
496 vmw_cmdbuf_man_process(man);
497 spin_unlock(&man->lock);
501 * vmw_cmdbuf_work_func - The deferred work function that handles
502 * command buffer errors.
504 * @work: The work func closure argument.
506 * Restarting the command buffer context after an error requires process
507 * context, so it is deferred to this work function.
509 static void vmw_cmdbuf_work_func(struct work_struct *work)
511 struct vmw_cmdbuf_man *man =
512 container_of(work, struct vmw_cmdbuf_man, work);
513 struct vmw_cmdbuf_header *entry, *next;
515 bool send_fence = false;
516 struct list_head restart_head[SVGA_CB_CONTEXT_MAX];
518 struct vmw_cmdbuf_context *ctx;
519 bool global_block = false;
521 for_each_cmdbuf_ctx(man, i, ctx)
522 INIT_LIST_HEAD(&restart_head[i]);
524 mutex_lock(&man->error_mutex);
525 spin_lock(&man->lock);
526 list_for_each_entry_safe(entry, next, &man->error, list) {
527 SVGACBHeader *cb_hdr = entry->cb_header;
528 SVGA3dCmdHeader *header = (SVGA3dCmdHeader *)
529 (entry->cmd + cb_hdr->errorOffset);
530 u32 error_cmd_size, new_start_offset;
531 const char *cmd_name;
533 list_del_init(&entry->list);
536 if (!vmw_cmd_describe(header, &error_cmd_size, &cmd_name)) {
537 VMW_DEBUG_USER("Unknown command causing device error.\n");
538 VMW_DEBUG_USER("Command buffer offset is %lu\n",
539 (unsigned long) cb_hdr->errorOffset);
540 __vmw_cmdbuf_header_free(entry);
545 VMW_DEBUG_USER("Command \"%s\" causing device error.\n",
547 VMW_DEBUG_USER("Command buffer offset is %lu\n",
548 (unsigned long) cb_hdr->errorOffset);
549 VMW_DEBUG_USER("Command size is %lu\n",
550 (unsigned long) error_cmd_size);
552 new_start_offset = cb_hdr->errorOffset + error_cmd_size;
554 if (new_start_offset >= cb_hdr->length) {
555 __vmw_cmdbuf_header_free(entry);
561 cb_hdr->ptr.mob.mobOffset += new_start_offset;
563 cb_hdr->ptr.pa += (u64) new_start_offset;
565 entry->cmd += new_start_offset;
566 cb_hdr->length -= new_start_offset;
567 cb_hdr->errorOffset = 0;
570 list_add_tail(&entry->list, &restart_head[entry->cb_context]);
573 for_each_cmdbuf_ctx(man, i, ctx)
574 man->ctx[i].block_submission = true;
576 spin_unlock(&man->lock);
578 /* Preempt all contexts */
579 if (global_block && vmw_cmdbuf_preempt(man, 0))
580 DRM_ERROR("Failed preempting command buffer contexts\n");
582 spin_lock(&man->lock);
583 for_each_cmdbuf_ctx(man, i, ctx) {
584 /* Move preempted command buffers to the preempted queue. */
585 vmw_cmdbuf_ctx_process(man, ctx, &dummy);
588 * Add the preempted queue after the command buffer
589 * that caused an error.
591 list_splice_init(&ctx->preempted, restart_head[i].prev);
594 * Finally add all command buffers first in the submitted
595 * queue, to rerun them.
598 ctx->block_submission = false;
599 list_splice_init(&restart_head[i], &ctx->submitted);
602 vmw_cmdbuf_man_process(man);
603 spin_unlock(&man->lock);
605 if (global_block && vmw_cmdbuf_startstop(man, 0, true))
606 DRM_ERROR("Failed restarting command buffer contexts\n");
608 /* Send a new fence in case one was removed */
610 vmw_fifo_send_fence(man->dev_priv, &dummy);
611 wake_up_all(&man->idle_queue);
614 mutex_unlock(&man->error_mutex);
618 * vmw_cmdbuf_man idle - Check whether the command buffer manager is idle.
620 * @man: The command buffer manager.
621 * @check_preempted: Check also the preempted queue for pending command buffers.
624 static bool vmw_cmdbuf_man_idle(struct vmw_cmdbuf_man *man,
625 bool check_preempted)
627 struct vmw_cmdbuf_context *ctx;
631 spin_lock(&man->lock);
632 vmw_cmdbuf_man_process(man);
633 for_each_cmdbuf_ctx(man, i, ctx) {
634 if (!list_empty(&ctx->submitted) ||
635 !list_empty(&ctx->hw_submitted) ||
636 (check_preempted && !list_empty(&ctx->preempted)))
640 idle = list_empty(&man->error);
643 spin_unlock(&man->lock);
649 * __vmw_cmdbuf_cur_flush - Flush the current command buffer for small kernel
650 * command submissions
652 * @man: The command buffer manager.
654 * Flushes the current command buffer without allocating a new one. A new one
655 * is automatically allocated when needed. Call with @man->cur_mutex held.
657 static void __vmw_cmdbuf_cur_flush(struct vmw_cmdbuf_man *man)
659 struct vmw_cmdbuf_header *cur = man->cur;
661 lockdep_assert_held_once(&man->cur_mutex);
666 spin_lock(&man->lock);
667 if (man->cur_pos == 0) {
668 __vmw_cmdbuf_header_free(cur);
672 man->cur->cb_header->length = man->cur_pos;
673 vmw_cmdbuf_ctx_add(man, man->cur, SVGA_CB_CONTEXT_0);
675 spin_unlock(&man->lock);
681 * vmw_cmdbuf_cur_flush - Flush the current command buffer for small kernel
682 * command submissions
684 * @man: The command buffer manager.
685 * @interruptible: Whether to sleep interruptible when sleeping.
687 * Flushes the current command buffer without allocating a new one. A new one
688 * is automatically allocated when needed.
690 int vmw_cmdbuf_cur_flush(struct vmw_cmdbuf_man *man,
693 int ret = vmw_cmdbuf_cur_lock(man, interruptible);
698 __vmw_cmdbuf_cur_flush(man);
699 vmw_cmdbuf_cur_unlock(man);
705 * vmw_cmdbuf_idle - Wait for command buffer manager idle.
707 * @man: The command buffer manager.
708 * @interruptible: Sleep interruptible while waiting.
709 * @timeout: Time out after this many ticks.
711 * Wait until the command buffer manager has processed all command buffers,
712 * or until a timeout occurs. If a timeout occurs, the function will return
715 int vmw_cmdbuf_idle(struct vmw_cmdbuf_man *man, bool interruptible,
716 unsigned long timeout)
720 ret = vmw_cmdbuf_cur_flush(man, interruptible);
721 vmw_generic_waiter_add(man->dev_priv,
722 SVGA_IRQFLAG_COMMAND_BUFFER,
723 &man->dev_priv->cmdbuf_waiters);
726 ret = wait_event_interruptible_timeout
727 (man->idle_queue, vmw_cmdbuf_man_idle(man, true),
730 ret = wait_event_timeout
731 (man->idle_queue, vmw_cmdbuf_man_idle(man, true),
734 vmw_generic_waiter_remove(man->dev_priv,
735 SVGA_IRQFLAG_COMMAND_BUFFER,
736 &man->dev_priv->cmdbuf_waiters);
738 if (!vmw_cmdbuf_man_idle(man, true))
750 * vmw_cmdbuf_try_alloc - Try to allocate buffer space from the main pool.
752 * @man: The command buffer manager.
753 * @info: Allocation info. Will hold the size on entry and allocated mm node
754 * on successful return.
756 * Try to allocate buffer space from the main pool. Returns true if succeeded.
757 * If a fatal error was hit, the error code is returned in @info->ret.
759 static bool vmw_cmdbuf_try_alloc(struct vmw_cmdbuf_man *man,
760 struct vmw_cmdbuf_alloc_info *info)
767 memset(info->node, 0, sizeof(*info->node));
768 spin_lock(&man->lock);
769 ret = drm_mm_insert_node(&man->mm, info->node, info->page_size);
771 vmw_cmdbuf_man_process(man);
772 ret = drm_mm_insert_node(&man->mm, info->node, info->page_size);
775 spin_unlock(&man->lock);
782 * vmw_cmdbuf_alloc_space - Allocate buffer space from the main pool.
784 * @man: The command buffer manager.
785 * @node: Pointer to pre-allocated range-manager node.
786 * @size: The size of the allocation.
787 * @interruptible: Whether to sleep interruptible while waiting for space.
789 * This function allocates buffer space from the main pool, and if there is
790 * no space available ATM, it turns on IRQ handling and sleeps waiting for it to
793 static int vmw_cmdbuf_alloc_space(struct vmw_cmdbuf_man *man,
794 struct drm_mm_node *node,
798 struct vmw_cmdbuf_alloc_info info;
800 info.page_size = PAGE_ALIGN(size) >> PAGE_SHIFT;
805 * To prevent starvation of large requests, only one allocating call
806 * at a time waiting for space.
809 if (mutex_lock_interruptible(&man->space_mutex))
812 mutex_lock(&man->space_mutex);
815 /* Try to allocate space without waiting. */
816 if (vmw_cmdbuf_try_alloc(man, &info))
819 vmw_generic_waiter_add(man->dev_priv,
820 SVGA_IRQFLAG_COMMAND_BUFFER,
821 &man->dev_priv->cmdbuf_waiters);
826 ret = wait_event_interruptible
827 (man->alloc_queue, vmw_cmdbuf_try_alloc(man, &info));
829 vmw_generic_waiter_remove
830 (man->dev_priv, SVGA_IRQFLAG_COMMAND_BUFFER,
831 &man->dev_priv->cmdbuf_waiters);
832 mutex_unlock(&man->space_mutex);
836 wait_event(man->alloc_queue, vmw_cmdbuf_try_alloc(man, &info));
838 vmw_generic_waiter_remove(man->dev_priv,
839 SVGA_IRQFLAG_COMMAND_BUFFER,
840 &man->dev_priv->cmdbuf_waiters);
843 mutex_unlock(&man->space_mutex);
849 * vmw_cmdbuf_space_pool - Set up a command buffer header with command buffer
850 * space from the main pool.
852 * @man: The command buffer manager.
853 * @header: Pointer to the header to set up.
854 * @size: The requested size of the buffer space.
855 * @interruptible: Whether to sleep interruptible while waiting for space.
857 static int vmw_cmdbuf_space_pool(struct vmw_cmdbuf_man *man,
858 struct vmw_cmdbuf_header *header,
862 SVGACBHeader *cb_hdr;
869 ret = vmw_cmdbuf_alloc_space(man, &header->node, size, interruptible);
874 header->cb_header = dma_pool_zalloc(man->headers, GFP_KERNEL,
876 if (!header->cb_header) {
878 goto out_no_cb_header;
881 header->size = header->node.size << PAGE_SHIFT;
882 cb_hdr = header->cb_header;
883 offset = header->node.start << PAGE_SHIFT;
884 header->cmd = man->map + offset;
885 if (man->using_mob) {
886 cb_hdr->flags = SVGA_CB_FLAG_MOB;
887 cb_hdr->ptr.mob.mobid = man->cmd_space->mem.start;
888 cb_hdr->ptr.mob.mobOffset = offset;
890 cb_hdr->ptr.pa = (u64)man->handle + (u64)offset;
896 spin_lock(&man->lock);
897 drm_mm_remove_node(&header->node);
898 spin_unlock(&man->lock);
904 * vmw_cmdbuf_space_inline - Set up a command buffer header with
905 * inline command buffer space.
907 * @man: The command buffer manager.
908 * @header: Pointer to the header to set up.
909 * @size: The requested size of the buffer space.
911 static int vmw_cmdbuf_space_inline(struct vmw_cmdbuf_man *man,
912 struct vmw_cmdbuf_header *header,
915 struct vmw_cmdbuf_dheader *dheader;
916 SVGACBHeader *cb_hdr;
918 if (WARN_ON_ONCE(size > VMW_CMDBUF_INLINE_SIZE))
921 dheader = dma_pool_zalloc(man->dheaders, GFP_KERNEL,
926 header->inline_space = true;
927 header->size = VMW_CMDBUF_INLINE_SIZE;
928 cb_hdr = &dheader->cb_header;
929 header->cb_header = cb_hdr;
930 header->cmd = dheader->cmd;
931 cb_hdr->status = SVGA_CB_STATUS_NONE;
932 cb_hdr->flags = SVGA_CB_FLAG_NONE;
933 cb_hdr->ptr.pa = (u64)header->handle +
934 (u64)offsetof(struct vmw_cmdbuf_dheader, cmd);
940 * vmw_cmdbuf_alloc - Allocate a command buffer header complete with
941 * command buffer space.
943 * @man: The command buffer manager.
944 * @size: The requested size of the buffer space.
945 * @interruptible: Whether to sleep interruptible while waiting for space.
946 * @p_header: points to a header pointer to populate on successful return.
948 * Returns a pointer to command buffer space if successful. Otherwise
949 * returns an error pointer. The header pointer returned in @p_header should
950 * be used for upcoming calls to vmw_cmdbuf_reserve() and vmw_cmdbuf_commit().
952 void *vmw_cmdbuf_alloc(struct vmw_cmdbuf_man *man,
953 size_t size, bool interruptible,
954 struct vmw_cmdbuf_header **p_header)
956 struct vmw_cmdbuf_header *header;
961 header = kzalloc(sizeof(*header), GFP_KERNEL);
963 return ERR_PTR(-ENOMEM);
965 if (size <= VMW_CMDBUF_INLINE_SIZE)
966 ret = vmw_cmdbuf_space_inline(man, header, size);
968 ret = vmw_cmdbuf_space_pool(man, header, size, interruptible);
976 INIT_LIST_HEAD(&header->list);
977 header->cb_header->status = SVGA_CB_STATUS_NONE;
984 * vmw_cmdbuf_reserve_cur - Reserve space for commands in the current
987 * @man: The command buffer manager.
988 * @size: The requested size of the commands.
989 * @ctx_id: The context id if any. Otherwise set to SVGA3D_REG_INVALID.
990 * @interruptible: Whether to sleep interruptible while waiting for space.
992 * Returns a pointer to command buffer space if successful. Otherwise
993 * returns an error pointer.
995 static void *vmw_cmdbuf_reserve_cur(struct vmw_cmdbuf_man *man,
1000 struct vmw_cmdbuf_header *cur;
1003 if (vmw_cmdbuf_cur_lock(man, interruptible))
1004 return ERR_PTR(-ERESTARTSYS);
1007 if (cur && (size + man->cur_pos > cur->size ||
1008 ((cur->cb_header->flags & SVGA_CB_FLAG_DX_CONTEXT) &&
1009 ctx_id != cur->cb_header->dxContext)))
1010 __vmw_cmdbuf_cur_flush(man);
1013 ret = vmw_cmdbuf_alloc(man,
1014 max_t(size_t, size, man->default_size),
1015 interruptible, &man->cur);
1017 vmw_cmdbuf_cur_unlock(man);
1024 if (ctx_id != SVGA3D_INVALID_ID) {
1025 cur->cb_header->flags |= SVGA_CB_FLAG_DX_CONTEXT;
1026 cur->cb_header->dxContext = ctx_id;
1029 cur->reserved = size;
1031 return (void *) (man->cur->cmd + man->cur_pos);
1035 * vmw_cmdbuf_commit_cur - Commit commands in the current command buffer.
1037 * @man: The command buffer manager.
1038 * @size: The size of the commands actually written.
1039 * @flush: Whether to flush the command buffer immediately.
1041 static void vmw_cmdbuf_commit_cur(struct vmw_cmdbuf_man *man,
1042 size_t size, bool flush)
1044 struct vmw_cmdbuf_header *cur = man->cur;
1046 lockdep_assert_held_once(&man->cur_mutex);
1048 WARN_ON(size > cur->reserved);
1049 man->cur_pos += size;
1051 cur->cb_header->flags &= ~SVGA_CB_FLAG_DX_CONTEXT;
1053 __vmw_cmdbuf_cur_flush(man);
1054 vmw_cmdbuf_cur_unlock(man);
1058 * vmw_cmdbuf_reserve - Reserve space for commands in a command buffer.
1060 * @man: The command buffer manager.
1061 * @size: The requested size of the commands.
1062 * @ctx_id: The context id if any. Otherwise set to SVGA3D_REG_INVALID.
1063 * @interruptible: Whether to sleep interruptible while waiting for space.
1064 * @header: Header of the command buffer. NULL if the current command buffer
1067 * Returns a pointer to command buffer space if successful. Otherwise
1068 * returns an error pointer.
1070 void *vmw_cmdbuf_reserve(struct vmw_cmdbuf_man *man, size_t size,
1071 int ctx_id, bool interruptible,
1072 struct vmw_cmdbuf_header *header)
1075 return vmw_cmdbuf_reserve_cur(man, size, ctx_id, interruptible);
1077 if (size > header->size)
1078 return ERR_PTR(-EINVAL);
1080 if (ctx_id != SVGA3D_INVALID_ID) {
1081 header->cb_header->flags |= SVGA_CB_FLAG_DX_CONTEXT;
1082 header->cb_header->dxContext = ctx_id;
1085 header->reserved = size;
1090 * vmw_cmdbuf_commit - Commit commands in a command buffer.
1092 * @man: The command buffer manager.
1093 * @size: The size of the commands actually written.
1094 * @header: Header of the command buffer. NULL if the current command buffer
1096 * @flush: Whether to flush the command buffer immediately.
1098 void vmw_cmdbuf_commit(struct vmw_cmdbuf_man *man, size_t size,
1099 struct vmw_cmdbuf_header *header, bool flush)
1102 vmw_cmdbuf_commit_cur(man, size, flush);
1106 (void) vmw_cmdbuf_cur_lock(man, false);
1107 __vmw_cmdbuf_cur_flush(man);
1108 WARN_ON(size > header->reserved);
1110 man->cur_pos = size;
1112 header->cb_header->flags &= ~SVGA_CB_FLAG_DX_CONTEXT;
1114 __vmw_cmdbuf_cur_flush(man);
1115 vmw_cmdbuf_cur_unlock(man);
1120 * vmw_cmdbuf_send_device_command - Send a command through the device context.
1122 * @man: The command buffer manager.
1123 * @command: Pointer to the command to send.
1124 * @size: Size of the command.
1126 * Synchronously sends a device context command.
1128 static int vmw_cmdbuf_send_device_command(struct vmw_cmdbuf_man *man,
1129 const void *command,
1132 struct vmw_cmdbuf_header *header;
1134 void *cmd = vmw_cmdbuf_alloc(man, size, false, &header);
1137 return PTR_ERR(cmd);
1139 memcpy(cmd, command, size);
1140 header->cb_header->length = size;
1141 header->cb_context = SVGA_CB_CONTEXT_DEVICE;
1142 spin_lock(&man->lock);
1143 status = vmw_cmdbuf_header_submit(header);
1144 spin_unlock(&man->lock);
1145 vmw_cmdbuf_header_free(header);
1147 if (status != SVGA_CB_STATUS_COMPLETED) {
1148 DRM_ERROR("Device context command failed with status %d\n",
1157 * vmw_cmdbuf_preempt - Send a preempt command through the device
1160 * @man: The command buffer manager.
1162 * Synchronously sends a preempt command.
1164 static int vmw_cmdbuf_preempt(struct vmw_cmdbuf_man *man, u32 context)
1168 SVGADCCmdPreempt body;
1171 cmd.id = SVGA_DC_CMD_PREEMPT;
1172 cmd.body.context = SVGA_CB_CONTEXT_0 + context;
1173 cmd.body.ignoreIDZero = 0;
1175 return vmw_cmdbuf_send_device_command(man, &cmd, sizeof(cmd));
1180 * vmw_cmdbuf_startstop - Send a start / stop command through the device
1183 * @man: The command buffer manager.
1184 * @enable: Whether to enable or disable the context.
1186 * Synchronously sends a device start / stop context command.
1188 static int vmw_cmdbuf_startstop(struct vmw_cmdbuf_man *man, u32 context,
1193 SVGADCCmdStartStop body;
1196 cmd.id = SVGA_DC_CMD_START_STOP_CONTEXT;
1197 cmd.body.enable = (enable) ? 1 : 0;
1198 cmd.body.context = SVGA_CB_CONTEXT_0 + context;
1200 return vmw_cmdbuf_send_device_command(man, &cmd, sizeof(cmd));
1204 * vmw_cmdbuf_set_pool_size - Set command buffer manager sizes
1206 * @man: The command buffer manager.
1207 * @size: The size of the main space pool.
1208 * @default_size: The default size of the command buffer for small kernel
1211 * Set the size and allocate the main command buffer space pool,
1212 * as well as the default size of the command buffer for
1213 * small kernel submissions. If successful, this enables large command
1214 * submissions. Note that this function requires that rudimentary command
1215 * submission is already available and that the MOB memory manager is alive.
1216 * Returns 0 on success. Negative error code on failure.
1218 int vmw_cmdbuf_set_pool_size(struct vmw_cmdbuf_man *man,
1219 size_t size, size_t default_size)
1221 struct vmw_private *dev_priv = man->dev_priv;
1228 /* First, try to allocate a huge chunk of DMA memory */
1229 size = PAGE_ALIGN(size);
1230 man->map = dma_alloc_coherent(&dev_priv->dev->pdev->dev, size,
1231 &man->handle, GFP_KERNEL);
1233 man->using_mob = false;
1236 * DMA memory failed. If we can have command buffers in a
1237 * MOB, try to use that instead. Note that this will
1238 * actually call into the already enabled manager, when
1241 if (!(dev_priv->capabilities & SVGA_CAP_DX))
1244 ret = ttm_bo_create(&dev_priv->bdev, size, ttm_bo_type_device,
1245 &vmw_mob_ne_placement, 0, false,
1250 man->using_mob = true;
1251 ret = ttm_bo_kmap(man->cmd_space, 0, size >> PAGE_SHIFT,
1256 man->map = ttm_kmap_obj_virtual(&man->map_obj, &dummy);
1260 drm_mm_init(&man->mm, 0, size >> PAGE_SHIFT);
1262 man->has_pool = true;
1265 * For now, set the default size to VMW_CMDBUF_INLINE_SIZE to
1266 * prevent deadlocks from happening when vmw_cmdbuf_space_pool()
1267 * needs to wait for space and we block on further command
1268 * submissions to be able to free up space.
1270 man->default_size = VMW_CMDBUF_INLINE_SIZE;
1271 DRM_INFO("Using command buffers with %s pool.\n",
1272 (man->using_mob) ? "MOB" : "DMA");
1277 if (man->using_mob) {
1278 ttm_bo_put(man->cmd_space);
1279 man->cmd_space = NULL;
1286 * vmw_cmdbuf_man_create: Create a command buffer manager and enable it for
1287 * inline command buffer submissions only.
1289 * @dev_priv: Pointer to device private structure.
1291 * Returns a pointer to a cummand buffer manager to success or error pointer
1292 * on failure. The command buffer manager will be enabled for submissions of
1293 * size VMW_CMDBUF_INLINE_SIZE only.
1295 struct vmw_cmdbuf_man *vmw_cmdbuf_man_create(struct vmw_private *dev_priv)
1297 struct vmw_cmdbuf_man *man;
1298 struct vmw_cmdbuf_context *ctx;
1302 if (!(dev_priv->capabilities & SVGA_CAP_COMMAND_BUFFERS))
1303 return ERR_PTR(-ENOSYS);
1305 man = kzalloc(sizeof(*man), GFP_KERNEL);
1307 return ERR_PTR(-ENOMEM);
1309 man->num_contexts = (dev_priv->capabilities & SVGA_CAP_HP_CMD_QUEUE) ?
1311 man->headers = dma_pool_create("vmwgfx cmdbuf",
1312 &dev_priv->dev->pdev->dev,
1313 sizeof(SVGACBHeader),
1315 if (!man->headers) {
1320 man->dheaders = dma_pool_create("vmwgfx inline cmdbuf",
1321 &dev_priv->dev->pdev->dev,
1322 sizeof(struct vmw_cmdbuf_dheader),
1324 if (!man->dheaders) {
1329 for_each_cmdbuf_ctx(man, i, ctx)
1330 vmw_cmdbuf_ctx_init(ctx);
1332 INIT_LIST_HEAD(&man->error);
1333 spin_lock_init(&man->lock);
1334 mutex_init(&man->cur_mutex);
1335 mutex_init(&man->space_mutex);
1336 mutex_init(&man->error_mutex);
1337 man->default_size = VMW_CMDBUF_INLINE_SIZE;
1338 init_waitqueue_head(&man->alloc_queue);
1339 init_waitqueue_head(&man->idle_queue);
1340 man->dev_priv = dev_priv;
1341 man->max_hw_submitted = SVGA_CB_MAX_QUEUED_PER_CONTEXT - 1;
1342 INIT_WORK(&man->work, &vmw_cmdbuf_work_func);
1343 vmw_generic_waiter_add(dev_priv, SVGA_IRQFLAG_ERROR,
1344 &dev_priv->error_waiters);
1345 ret = vmw_cmdbuf_startstop(man, 0, true);
1347 DRM_ERROR("Failed starting command buffer contexts\n");
1348 vmw_cmdbuf_man_destroy(man);
1349 return ERR_PTR(ret);
1355 dma_pool_destroy(man->headers);
1359 return ERR_PTR(ret);
1363 * vmw_cmdbuf_remove_pool - Take down the main buffer space pool.
1365 * @man: Pointer to a command buffer manager.
1367 * This function removes the main buffer space pool, and should be called
1368 * before MOB memory management is removed. When this function has been called,
1369 * only small command buffer submissions of size VMW_CMDBUF_INLINE_SIZE or
1370 * less are allowed, and the default size of the command buffer for small kernel
1371 * submissions is also set to this size.
1373 void vmw_cmdbuf_remove_pool(struct vmw_cmdbuf_man *man)
1378 man->has_pool = false;
1379 man->default_size = VMW_CMDBUF_INLINE_SIZE;
1380 (void) vmw_cmdbuf_idle(man, false, 10*HZ);
1381 if (man->using_mob) {
1382 (void) ttm_bo_kunmap(&man->map_obj);
1383 ttm_bo_put(man->cmd_space);
1384 man->cmd_space = NULL;
1386 dma_free_coherent(&man->dev_priv->dev->pdev->dev,
1387 man->size, man->map, man->handle);
1392 * vmw_cmdbuf_man_destroy - Take down a command buffer manager.
1394 * @man: Pointer to a command buffer manager.
1396 * This function idles and then destroys a command buffer manager.
1398 void vmw_cmdbuf_man_destroy(struct vmw_cmdbuf_man *man)
1400 WARN_ON_ONCE(man->has_pool);
1401 (void) vmw_cmdbuf_idle(man, false, 10*HZ);
1403 if (vmw_cmdbuf_startstop(man, 0, false))
1404 DRM_ERROR("Failed stopping command buffer contexts.\n");
1406 vmw_generic_waiter_remove(man->dev_priv, SVGA_IRQFLAG_ERROR,
1407 &man->dev_priv->error_waiters);
1408 (void) cancel_work_sync(&man->work);
1409 dma_pool_destroy(man->dheaders);
1410 dma_pool_destroy(man->headers);
1411 mutex_destroy(&man->cur_mutex);
1412 mutex_destroy(&man->space_mutex);
1413 mutex_destroy(&man->error_mutex);