2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
24 * Authors: Dave Airlie
28 #include <linux/fence-array.h>
30 #include <drm/amdgpu_drm.h>
32 #include "amdgpu_trace.h"
36 * GPUVM is similar to the legacy gart on older asics, however
37 * rather than there being a single global gart table
38 * for the entire GPU, there are multiple VM page tables active
39 * at any given time. The VM page tables can contain a mix
40 * vram pages and system memory pages and system memory pages
41 * can be mapped as snooped (cached system pages) or unsnooped
42 * (uncached system pages).
43 * Each VM has an ID associated with it and there is a page table
44 * associated with each VMID. When execting a command buffer,
45 * the kernel tells the the ring what VMID to use for that command
46 * buffer. VMIDs are allocated dynamically as commands are submitted.
47 * The userspace drivers maintain their own address space and the kernel
48 * sets up their pages tables accordingly when they submit their
49 * command buffers and a VMID is assigned.
50 * Cayman/Trinity support up to 8 active VMs at any given time;
54 /* Local structure. Encapsulate some VM table update parameters to reduce
55 * the number of function parameters
57 struct amdgpu_pte_update_params {
58 /* amdgpu device we do this update for */
59 struct amdgpu_device *adev;
60 /* address where to copy page table entries from */
62 /* indirect buffer to fill with commands */
64 /* Function which actually does the update */
65 void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
66 uint64_t addr, unsigned count, uint32_t incr,
68 /* indicate update pt or its shadow */
73 * amdgpu_vm_num_pde - return the number of page directory entries
75 * @adev: amdgpu_device pointer
77 * Calculate the number of page directory entries.
79 static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
81 return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
85 * amdgpu_vm_directory_size - returns the size of the page directory in bytes
87 * @adev: amdgpu_device pointer
89 * Calculate the size of the page directory in bytes.
91 static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
93 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
97 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
99 * @vm: vm providing the BOs
100 * @validated: head of validation list
101 * @entry: entry to add
103 * Add the page directory to the list of BOs to
104 * validate for command submission.
106 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
107 struct list_head *validated,
108 struct amdgpu_bo_list_entry *entry)
110 entry->robj = vm->page_directory;
112 entry->tv.bo = &vm->page_directory->tbo;
113 entry->tv.shared = true;
114 entry->user_pages = NULL;
115 list_add(&entry->tv.head, validated);
119 * amdgpu_vm_validate_pt_bos - validate the page table BOs
121 * @adev: amdgpu device pointer
122 * @vm: vm providing the BOs
123 * @validate: callback to do the validation
124 * @param: parameter for the validation callback
126 * Validate the page table BOs on command submission if neccessary.
128 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
129 int (*validate)(void *p, struct amdgpu_bo *bo),
132 uint64_t num_evictions;
136 /* We only need to validate the page tables
137 * if they aren't already valid.
139 num_evictions = atomic64_read(&adev->num_evictions);
140 if (num_evictions == vm->last_eviction_counter)
143 /* add the vm page table to the list */
144 for (i = 0; i <= vm->max_pde_used; ++i) {
145 struct amdgpu_bo *bo = vm->page_tables[i].bo;
150 r = validate(param, bo);
159 * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
161 * @adev: amdgpu device instance
162 * @vm: vm providing the BOs
164 * Move the PT BOs to the tail of the LRU.
166 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
167 struct amdgpu_vm *vm)
169 struct ttm_bo_global *glob = adev->mman.bdev.glob;
172 spin_lock(&glob->lru_lock);
173 for (i = 0; i <= vm->max_pde_used; ++i) {
174 struct amdgpu_bo *bo = vm->page_tables[i].bo;
179 ttm_bo_move_to_lru_tail(&bo->tbo);
181 spin_unlock(&glob->lru_lock);
184 static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev,
185 struct amdgpu_vm_id *id)
187 return id->current_gpu_reset_count !=
188 atomic_read(&adev->gpu_reset_counter) ? true : false;
192 * amdgpu_vm_grab_id - allocate the next free VMID
194 * @vm: vm to allocate id for
195 * @ring: ring we want to submit job to
196 * @sync: sync object where we add dependencies
197 * @fence: fence protecting ID from reuse
199 * Allocate an id for the vm, adding fences to the sync obj as necessary.
201 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
202 struct amdgpu_sync *sync, struct fence *fence,
203 struct amdgpu_job *job)
205 struct amdgpu_device *adev = ring->adev;
206 uint64_t fence_context = adev->fence_context + ring->idx;
207 struct fence *updates = sync->last_vm_update;
208 struct amdgpu_vm_id *id, *idle;
209 struct fence **fences;
213 fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids,
218 mutex_lock(&adev->vm_manager.lock);
220 /* Check if we have an idle VMID */
222 list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) {
223 fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
229 /* If we can't find a idle VMID to use, wait till one becomes available */
230 if (&idle->list == &adev->vm_manager.ids_lru) {
231 u64 fence_context = adev->vm_manager.fence_context + ring->idx;
232 unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
233 struct fence_array *array;
236 for (j = 0; j < i; ++j)
237 fence_get(fences[j]);
239 array = fence_array_create(i, fences, fence_context,
242 for (j = 0; j < i; ++j)
243 fence_put(fences[j]);
250 r = amdgpu_sync_fence(ring->adev, sync, &array->base);
251 fence_put(&array->base);
255 mutex_unlock(&adev->vm_manager.lock);
261 job->vm_needs_flush = true;
262 /* Check if we can use a VMID already assigned to this VM */
265 struct fence *flushed;
268 if (i == AMDGPU_MAX_RINGS)
271 /* Check all the prerequisites to using this VMID */
274 if (amdgpu_vm_is_gpu_reset(adev, id))
277 if (atomic64_read(&id->owner) != vm->client_id)
280 if (job->vm_pd_addr != id->pd_gpu_addr)
286 if (id->last_flush->context != fence_context &&
287 !fence_is_signaled(id->last_flush))
290 flushed = id->flushed_updates;
292 (!flushed || fence_is_later(updates, flushed)))
295 /* Good we can use this VMID. Remember this submission as
298 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
302 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
303 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
304 vm->ids[ring->idx] = id;
306 job->vm_id = id - adev->vm_manager.ids;
307 job->vm_needs_flush = false;
308 trace_amdgpu_vm_grab_id(vm, ring->idx, job);
310 mutex_unlock(&adev->vm_manager.lock);
313 } while (i != ring->idx);
315 /* Still no ID to use? Then use the idle one found earlier */
318 /* Remember this submission as user of the VMID */
319 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
323 fence_put(id->first);
324 id->first = fence_get(fence);
326 fence_put(id->last_flush);
327 id->last_flush = NULL;
329 fence_put(id->flushed_updates);
330 id->flushed_updates = fence_get(updates);
332 id->pd_gpu_addr = job->vm_pd_addr;
333 id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
334 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
335 atomic64_set(&id->owner, vm->client_id);
336 vm->ids[ring->idx] = id;
338 job->vm_id = id - adev->vm_manager.ids;
339 trace_amdgpu_vm_grab_id(vm, ring->idx, job);
342 mutex_unlock(&adev->vm_manager.lock);
346 static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
348 struct amdgpu_device *adev = ring->adev;
349 const struct amdgpu_ip_block_version *ip_block;
351 if (ring->type != AMDGPU_RING_TYPE_COMPUTE)
352 /* only compute rings */
355 ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
359 if (ip_block->major <= 7) {
360 /* gfx7 has no workaround */
362 } else if (ip_block->major == 8) {
363 if (adev->gfx.mec_fw_version >= 673)
364 /* gfx8 is fixed in MEC firmware 673 */
373 * amdgpu_vm_flush - hardware flush the vm
375 * @ring: ring to use for flush
376 * @vm_id: vmid number to use
377 * @pd_addr: address of the page directory
379 * Emit a VM flush when it is necessary.
381 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
383 struct amdgpu_device *adev = ring->adev;
384 struct amdgpu_vm_id *id = &adev->vm_manager.ids[job->vm_id];
385 bool gds_switch_needed = ring->funcs->emit_gds_switch && (
386 id->gds_base != job->gds_base ||
387 id->gds_size != job->gds_size ||
388 id->gws_base != job->gws_base ||
389 id->gws_size != job->gws_size ||
390 id->oa_base != job->oa_base ||
391 id->oa_size != job->oa_size);
394 if (ring->funcs->emit_pipeline_sync && (
395 job->vm_needs_flush || gds_switch_needed ||
396 amdgpu_vm_ring_has_compute_vm_bug(ring)))
397 amdgpu_ring_emit_pipeline_sync(ring);
399 if (ring->funcs->emit_vm_flush && (job->vm_needs_flush ||
400 amdgpu_vm_is_gpu_reset(adev, id))) {
403 trace_amdgpu_vm_flush(job->vm_pd_addr, ring->idx, job->vm_id);
404 amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
406 r = amdgpu_fence_emit(ring, &fence);
410 mutex_lock(&adev->vm_manager.lock);
411 fence_put(id->last_flush);
412 id->last_flush = fence;
413 mutex_unlock(&adev->vm_manager.lock);
416 if (gds_switch_needed) {
417 id->gds_base = job->gds_base;
418 id->gds_size = job->gds_size;
419 id->gws_base = job->gws_base;
420 id->gws_size = job->gws_size;
421 id->oa_base = job->oa_base;
422 id->oa_size = job->oa_size;
423 amdgpu_ring_emit_gds_switch(ring, job->vm_id,
424 job->gds_base, job->gds_size,
425 job->gws_base, job->gws_size,
426 job->oa_base, job->oa_size);
433 * amdgpu_vm_reset_id - reset VMID to zero
435 * @adev: amdgpu device structure
436 * @vm_id: vmid number to use
438 * Reset saved GDW, GWS and OA to force switch on next flush.
440 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
442 struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
453 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
456 * @bo: requested buffer object
458 * Find @bo inside the requested vm.
459 * Search inside the @bos vm list for the requested vm
460 * Returns the found bo_va or NULL if none is found
462 * Object has to be reserved!
464 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
465 struct amdgpu_bo *bo)
467 struct amdgpu_bo_va *bo_va;
469 list_for_each_entry(bo_va, &bo->va, bo_list) {
470 if (bo_va->vm == vm) {
478 * amdgpu_vm_do_set_ptes - helper to call the right asic function
480 * @params: see amdgpu_pte_update_params definition
481 * @pe: addr of the page entry
482 * @addr: dst addr to write into pe
483 * @count: number of page entries to update
484 * @incr: increase next addr by incr bytes
485 * @flags: hw access flags
487 * Traces the parameters and calls the right asic functions
488 * to setup the page table using the DMA.
490 static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
491 uint64_t pe, uint64_t addr,
492 unsigned count, uint32_t incr,
495 trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
498 amdgpu_vm_write_pte(params->adev, params->ib, pe,
499 addr | flags, count, incr);
502 amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
508 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
510 * @params: see amdgpu_pte_update_params definition
511 * @pe: addr of the page entry
512 * @addr: dst addr to write into pe
513 * @count: number of page entries to update
514 * @incr: increase next addr by incr bytes
515 * @flags: hw access flags
517 * Traces the parameters and calls the DMA function to copy the PTEs.
519 static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
520 uint64_t pe, uint64_t addr,
521 unsigned count, uint32_t incr,
524 uint64_t src = (params->src + (addr >> 12) * 8);
527 trace_amdgpu_vm_copy_ptes(pe, src, count);
529 amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
533 * amdgpu_vm_clear_bo - initially clear the page dir/table
535 * @adev: amdgpu_device pointer
538 * need to reserve bo first before calling it.
540 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
541 struct amdgpu_vm *vm,
542 struct amdgpu_bo *bo)
544 struct amdgpu_ring *ring;
545 struct fence *fence = NULL;
546 struct amdgpu_job *job;
547 struct amdgpu_pte_update_params params;
552 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
554 r = reservation_object_reserve_shared(bo->tbo.resv);
558 r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
562 r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
566 addr = amdgpu_bo_gpu_offset(bo);
567 entries = amdgpu_bo_size(bo) / 8;
569 r = amdgpu_job_alloc_with_ib(adev, 64, &job);
573 memset(¶ms, 0, sizeof(params));
575 params.ib = &job->ibs[0];
576 amdgpu_vm_do_set_ptes(¶ms, addr, 0, entries, 0, 0);
577 amdgpu_ring_pad_ib(ring, &job->ibs[0]);
579 WARN_ON(job->ibs[0].length_dw > 64);
580 r = amdgpu_job_submit(job, ring, &vm->entity,
581 AMDGPU_FENCE_OWNER_VM, &fence);
585 amdgpu_bo_fence(bo, fence, true);
590 amdgpu_job_free(job);
597 * amdgpu_vm_map_gart - Resolve gart mapping of addr
599 * @pages_addr: optional DMA address to use for lookup
600 * @addr: the unmapped addr
602 * Look up the physical address of the page that the pte resolves
603 * to and return the pointer for the page table entry.
605 static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
609 /* page table offset */
610 result = pages_addr[addr >> PAGE_SHIFT];
612 /* in case cpu page size != gpu page size*/
613 result |= addr & (~PAGE_MASK);
615 result &= 0xFFFFFFFFFFFFF000ULL;
621 * amdgpu_vm_update_pdes - make sure that page directory is valid
623 * @adev: amdgpu_device pointer
625 * @start: start of GPU address range
626 * @end: end of GPU address range
628 * Allocates new page tables if necessary
629 * and updates the page directory.
630 * Returns 0 for success, error for failure.
632 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
633 struct amdgpu_vm *vm)
635 struct amdgpu_bo *shadow;
636 struct amdgpu_ring *ring;
637 uint64_t pd_addr, shadow_addr;
638 uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
639 uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
640 unsigned count = 0, pt_idx, ndw;
641 struct amdgpu_job *job;
642 struct amdgpu_pte_update_params params;
643 struct fence *fence = NULL;
647 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
648 shadow = vm->page_directory->shadow;
653 /* assume the worst case */
654 ndw += vm->max_pde_used * 6;
656 pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
658 r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
661 shadow_addr = amdgpu_bo_gpu_offset(shadow);
667 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
671 memset(¶ms, 0, sizeof(params));
673 params.ib = &job->ibs[0];
675 /* walk over the address space and update the page directory */
676 for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
677 struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo;
684 struct amdgpu_bo *pt_shadow = bo->shadow;
686 r = amdgpu_ttm_bind(&pt_shadow->tbo,
687 &pt_shadow->tbo.mem);
692 pt = amdgpu_bo_gpu_offset(bo);
693 if (vm->page_tables[pt_idx].addr == pt)
696 vm->page_tables[pt_idx].addr = pt;
698 pde = pd_addr + pt_idx * 8;
699 if (((last_pde + 8 * count) != pde) ||
700 ((last_pt + incr * count) != pt) ||
701 (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
705 amdgpu_vm_do_set_ptes(¶ms,
711 amdgpu_vm_do_set_ptes(¶ms, last_pde,
712 last_pt, count, incr,
718 last_shadow = shadow_addr + pt_idx * 8;
726 if (vm->page_directory->shadow)
727 amdgpu_vm_do_set_ptes(¶ms, last_shadow, last_pt,
728 count, incr, AMDGPU_PTE_VALID);
730 amdgpu_vm_do_set_ptes(¶ms, last_pde, last_pt,
731 count, incr, AMDGPU_PTE_VALID);
734 if (params.ib->length_dw == 0) {
735 amdgpu_job_free(job);
739 amdgpu_ring_pad_ib(ring, params.ib);
740 amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
741 AMDGPU_FENCE_OWNER_VM);
743 amdgpu_sync_resv(adev, &job->sync, shadow->tbo.resv,
744 AMDGPU_FENCE_OWNER_VM);
746 WARN_ON(params.ib->length_dw > ndw);
747 r = amdgpu_job_submit(job, ring, &vm->entity,
748 AMDGPU_FENCE_OWNER_VM, &fence);
752 amdgpu_bo_fence(vm->page_directory, fence, true);
753 fence_put(vm->page_directory_fence);
754 vm->page_directory_fence = fence_get(fence);
760 amdgpu_job_free(job);
765 * amdgpu_vm_update_ptes - make sure that page tables are valid
767 * @params: see amdgpu_pte_update_params definition
769 * @start: start of GPU address range
770 * @end: end of GPU address range
771 * @dst: destination address to map to, the next dst inside the function
772 * @flags: mapping flags
774 * Update the page tables in the range @start - @end.
776 static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
777 struct amdgpu_vm *vm,
778 uint64_t start, uint64_t end,
779 uint64_t dst, uint32_t flags)
781 const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
783 uint64_t cur_pe_start, cur_nptes, cur_dst;
784 uint64_t addr; /* next GPU address to be updated */
786 struct amdgpu_bo *pt;
787 unsigned nptes; /* next number of ptes to be updated */
788 uint64_t next_pe_start;
790 /* initialize the variables */
792 pt_idx = addr >> amdgpu_vm_block_size;
793 pt = vm->page_tables[pt_idx].bo;
794 if (params->shadow) {
799 if ((addr & ~mask) == (end & ~mask))
802 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
804 cur_pe_start = amdgpu_bo_gpu_offset(pt);
805 cur_pe_start += (addr & mask) * 8;
811 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
813 /* walk over the address space and update the page tables */
815 pt_idx = addr >> amdgpu_vm_block_size;
816 pt = vm->page_tables[pt_idx].bo;
817 if (params->shadow) {
823 if ((addr & ~mask) == (end & ~mask))
826 nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
828 next_pe_start = amdgpu_bo_gpu_offset(pt);
829 next_pe_start += (addr & mask) * 8;
831 if ((cur_pe_start + 8 * cur_nptes) == next_pe_start &&
832 ((cur_nptes + nptes) <= AMDGPU_VM_MAX_UPDATE_SIZE)) {
833 /* The next ptb is consecutive to current ptb.
834 * Don't call the update function now.
835 * Will update two ptbs together in future.
839 params->func(params, cur_pe_start, cur_dst, cur_nptes,
840 AMDGPU_GPU_PAGE_SIZE, flags);
842 cur_pe_start = next_pe_start;
849 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
852 params->func(params, cur_pe_start, cur_dst, cur_nptes,
853 AMDGPU_GPU_PAGE_SIZE, flags);
857 * amdgpu_vm_frag_ptes - add fragment information to PTEs
859 * @params: see amdgpu_pte_update_params definition
861 * @start: first PTE to handle
862 * @end: last PTE to handle
863 * @dst: addr those PTEs should point to
864 * @flags: hw mapping flags
866 static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params *params,
867 struct amdgpu_vm *vm,
868 uint64_t start, uint64_t end,
869 uint64_t dst, uint32_t flags)
872 * The MC L1 TLB supports variable sized pages, based on a fragment
873 * field in the PTE. When this field is set to a non-zero value, page
874 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
875 * flags are considered valid for all PTEs within the fragment range
876 * and corresponding mappings are assumed to be physically contiguous.
878 * The L1 TLB can store a single PTE for the whole fragment,
879 * significantly increasing the space available for translation
880 * caching. This leads to large improvements in throughput when the
881 * TLB is under pressure.
883 * The L2 TLB distributes small and large fragments into two
884 * asymmetric partitions. The large fragment cache is significantly
885 * larger. Thus, we try to use large fragments wherever possible.
886 * Userspace can support this by aligning virtual base address and
887 * allocation size to the fragment size.
890 /* SI and newer are optimized for 64KB */
891 uint64_t frag_flags = AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG);
892 uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
894 uint64_t frag_start = ALIGN(start, frag_align);
895 uint64_t frag_end = end & ~(frag_align - 1);
897 /* system pages are non continuously */
898 if (params->src || !(flags & AMDGPU_PTE_VALID) ||
899 (frag_start >= frag_end)) {
901 amdgpu_vm_update_ptes(params, vm, start, end, dst, flags);
905 /* handle the 4K area at the beginning */
906 if (start != frag_start) {
907 amdgpu_vm_update_ptes(params, vm, start, frag_start,
909 dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
912 /* handle the area in the middle */
913 amdgpu_vm_update_ptes(params, vm, frag_start, frag_end, dst,
916 /* handle the 4K area at the end */
917 if (frag_end != end) {
918 dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
919 amdgpu_vm_update_ptes(params, vm, frag_end, end, dst, flags);
924 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
926 * @adev: amdgpu_device pointer
927 * @exclusive: fence we need to sync to
928 * @src: address where to copy page table entries from
929 * @pages_addr: DMA addresses to use for mapping
931 * @start: start of mapped range
932 * @last: last mapped entry
933 * @flags: flags for the entries
934 * @addr: addr to set the area to
935 * @fence: optional resulting fence
937 * Fill in the page table entries between @start and @last.
938 * Returns 0 for success, -EINVAL for failure.
940 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
941 struct fence *exclusive,
943 dma_addr_t *pages_addr,
944 struct amdgpu_vm *vm,
945 uint64_t start, uint64_t last,
946 uint32_t flags, uint64_t addr,
947 struct fence **fence)
949 struct amdgpu_ring *ring;
950 void *owner = AMDGPU_FENCE_OWNER_VM;
951 unsigned nptes, ncmds, ndw;
952 struct amdgpu_job *job;
953 struct amdgpu_pte_update_params params;
954 struct fence *f = NULL;
957 memset(¶ms, 0, sizeof(params));
961 ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
963 memset(¶ms, 0, sizeof(params));
967 /* sync to everything on unmapping */
968 if (!(flags & AMDGPU_PTE_VALID))
969 owner = AMDGPU_FENCE_OWNER_UNDEFINED;
971 nptes = last - start + 1;
974 * reserve space for one command every (1 << BLOCK_SIZE)
975 * entries or 2k dwords (whatever is smaller)
977 ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;
983 /* only copy commands needed */
986 params.func = amdgpu_vm_do_copy_ptes;
988 } else if (pages_addr) {
989 /* copy commands needed */
995 params.func = amdgpu_vm_do_copy_ptes;
998 /* set page commands needed */
1001 /* two extra commands for begin/end of fragment */
1004 params.func = amdgpu_vm_do_set_ptes;
1007 r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
1011 params.ib = &job->ibs[0];
1013 if (!src && pages_addr) {
1017 /* Put the PTEs at the end of the IB. */
1018 i = ndw - nptes * 2;
1019 pte= (uint64_t *)&(job->ibs->ptr[i]);
1020 params.src = job->ibs->gpu_addr + i * 4;
1022 for (i = 0; i < nptes; ++i) {
1023 pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
1024 AMDGPU_GPU_PAGE_SIZE);
1030 r = amdgpu_sync_fence(adev, &job->sync, exclusive);
1034 r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
1039 r = reservation_object_reserve_shared(vm->page_directory->tbo.resv);
1043 params.shadow = true;
1044 amdgpu_vm_frag_ptes(¶ms, vm, start, last + 1, addr, flags);
1045 params.shadow = false;
1046 amdgpu_vm_frag_ptes(¶ms, vm, start, last + 1, addr, flags);
1048 amdgpu_ring_pad_ib(ring, params.ib);
1049 WARN_ON(params.ib->length_dw > ndw);
1050 r = amdgpu_job_submit(job, ring, &vm->entity,
1051 AMDGPU_FENCE_OWNER_VM, &f);
1055 amdgpu_bo_fence(vm->page_directory, f, true);
1058 *fence = fence_get(f);
1064 amdgpu_job_free(job);
1069 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1071 * @adev: amdgpu_device pointer
1072 * @exclusive: fence we need to sync to
1073 * @gtt_flags: flags as they are used for GTT
1074 * @pages_addr: DMA addresses to use for mapping
1076 * @mapping: mapped range and flags to use for the update
1077 * @flags: HW flags for the mapping
1078 * @nodes: array of drm_mm_nodes with the MC addresses
1079 * @fence: optional resulting fence
1081 * Split the mapping into smaller chunks so that each update fits
1083 * Returns 0 for success, -EINVAL for failure.
1085 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1086 struct fence *exclusive,
1088 dma_addr_t *pages_addr,
1089 struct amdgpu_vm *vm,
1090 struct amdgpu_bo_va_mapping *mapping,
1092 struct drm_mm_node *nodes,
1093 struct fence **fence)
1095 uint64_t pfn, src = 0, start = mapping->it.start;
1098 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1099 * but in case of something, we filter the flags in first place
1101 if (!(mapping->flags & AMDGPU_PTE_READABLE))
1102 flags &= ~AMDGPU_PTE_READABLE;
1103 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1104 flags &= ~AMDGPU_PTE_WRITEABLE;
1106 trace_amdgpu_vm_bo_update(mapping);
1108 pfn = mapping->offset >> PAGE_SHIFT;
1110 while (pfn >= nodes->size) {
1117 uint64_t max_entries;
1118 uint64_t addr, last;
1121 addr = nodes->start << PAGE_SHIFT;
1122 max_entries = (nodes->size - pfn) *
1123 (PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
1126 max_entries = S64_MAX;
1130 if (flags == gtt_flags)
1131 src = adev->gart.table_addr +
1132 (addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
1134 max_entries = min(max_entries, 16ull * 1024ull);
1136 } else if (flags & AMDGPU_PTE_VALID) {
1137 addr += adev->vm_manager.vram_base_offset;
1139 addr += pfn << PAGE_SHIFT;
1141 last = min((uint64_t)mapping->it.last, start + max_entries - 1);
1142 r = amdgpu_vm_bo_update_mapping(adev, exclusive,
1143 src, pages_addr, vm,
1144 start, last, flags, addr,
1149 pfn += last - start + 1;
1150 if (nodes && nodes->size == pfn) {
1156 } while (unlikely(start != mapping->it.last + 1));
1162 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1164 * @adev: amdgpu_device pointer
1165 * @bo_va: requested BO and VM object
1166 * @clear: if true clear the entries
1168 * Fill in the page table entries for @bo_va.
1169 * Returns 0 for success, -EINVAL for failure.
1171 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1172 struct amdgpu_bo_va *bo_va,
1175 struct amdgpu_vm *vm = bo_va->vm;
1176 struct amdgpu_bo_va_mapping *mapping;
1177 dma_addr_t *pages_addr = NULL;
1178 uint32_t gtt_flags, flags;
1179 struct ttm_mem_reg *mem;
1180 struct drm_mm_node *nodes;
1181 struct fence *exclusive;
1189 struct ttm_dma_tt *ttm;
1191 mem = &bo_va->bo->tbo.mem;
1192 nodes = mem->mm_node;
1193 if (mem->mem_type == TTM_PL_TT) {
1194 ttm = container_of(bo_va->bo->tbo.ttm, struct
1196 pages_addr = ttm->dma_address;
1198 exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
1201 flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
1202 gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
1203 adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ? flags : 0;
1205 spin_lock(&vm->status_lock);
1206 if (!list_empty(&bo_va->vm_status))
1207 list_splice_init(&bo_va->valids, &bo_va->invalids);
1208 spin_unlock(&vm->status_lock);
1210 list_for_each_entry(mapping, &bo_va->invalids, list) {
1211 r = amdgpu_vm_bo_split_mapping(adev, exclusive,
1212 gtt_flags, pages_addr, vm,
1213 mapping, flags, nodes,
1214 &bo_va->last_pt_update);
1219 if (trace_amdgpu_vm_bo_mapping_enabled()) {
1220 list_for_each_entry(mapping, &bo_va->valids, list)
1221 trace_amdgpu_vm_bo_mapping(mapping);
1223 list_for_each_entry(mapping, &bo_va->invalids, list)
1224 trace_amdgpu_vm_bo_mapping(mapping);
1227 spin_lock(&vm->status_lock);
1228 list_splice_init(&bo_va->invalids, &bo_va->valids);
1229 list_del_init(&bo_va->vm_status);
1231 list_add(&bo_va->vm_status, &vm->cleared);
1232 spin_unlock(&vm->status_lock);
1238 * amdgpu_vm_clear_freed - clear freed BOs in the PT
1240 * @adev: amdgpu_device pointer
1243 * Make sure all freed BOs are cleared in the PT.
1244 * Returns 0 for success.
1246 * PTs have to be reserved and mutex must be locked!
1248 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1249 struct amdgpu_vm *vm)
1251 struct amdgpu_bo_va_mapping *mapping;
1254 while (!list_empty(&vm->freed)) {
1255 mapping = list_first_entry(&vm->freed,
1256 struct amdgpu_bo_va_mapping, list);
1257 list_del(&mapping->list);
1259 r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping,
1271 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
1273 * @adev: amdgpu_device pointer
1276 * Make sure all invalidated BOs are cleared in the PT.
1277 * Returns 0 for success.
1279 * PTs have to be reserved and mutex must be locked!
1281 int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
1282 struct amdgpu_vm *vm, struct amdgpu_sync *sync)
1284 struct amdgpu_bo_va *bo_va = NULL;
1287 spin_lock(&vm->status_lock);
1288 while (!list_empty(&vm->invalidated)) {
1289 bo_va = list_first_entry(&vm->invalidated,
1290 struct amdgpu_bo_va, vm_status);
1291 spin_unlock(&vm->status_lock);
1293 r = amdgpu_vm_bo_update(adev, bo_va, true);
1297 spin_lock(&vm->status_lock);
1299 spin_unlock(&vm->status_lock);
1302 r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
1308 * amdgpu_vm_bo_add - add a bo to a specific vm
1310 * @adev: amdgpu_device pointer
1312 * @bo: amdgpu buffer object
1314 * Add @bo into the requested vm.
1315 * Add @bo to the list of bos associated with the vm
1316 * Returns newly added bo_va or NULL for failure
1318 * Object has to be reserved!
1320 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
1321 struct amdgpu_vm *vm,
1322 struct amdgpu_bo *bo)
1324 struct amdgpu_bo_va *bo_va;
1326 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
1327 if (bo_va == NULL) {
1332 bo_va->ref_count = 1;
1333 INIT_LIST_HEAD(&bo_va->bo_list);
1334 INIT_LIST_HEAD(&bo_va->valids);
1335 INIT_LIST_HEAD(&bo_va->invalids);
1336 INIT_LIST_HEAD(&bo_va->vm_status);
1338 list_add_tail(&bo_va->bo_list, &bo->va);
1344 * amdgpu_vm_bo_map - map bo inside a vm
1346 * @adev: amdgpu_device pointer
1347 * @bo_va: bo_va to store the address
1348 * @saddr: where to map the BO
1349 * @offset: requested offset in the BO
1350 * @flags: attributes of pages (read/write/valid/etc.)
1352 * Add a mapping of the BO at the specefied addr into the VM.
1353 * Returns 0 for success, error for failure.
1355 * Object has to be reserved and unreserved outside!
1357 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
1358 struct amdgpu_bo_va *bo_va,
1359 uint64_t saddr, uint64_t offset,
1360 uint64_t size, uint32_t flags)
1362 struct amdgpu_bo_va_mapping *mapping;
1363 struct amdgpu_vm *vm = bo_va->vm;
1364 struct interval_tree_node *it;
1365 unsigned last_pfn, pt_idx;
1369 /* validate the parameters */
1370 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1371 size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1374 /* make sure object fit at this offset */
1375 eaddr = saddr + size - 1;
1376 if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo)))
1379 last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
1380 if (last_pfn >= adev->vm_manager.max_pfn) {
1381 dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
1382 last_pfn, adev->vm_manager.max_pfn);
1386 saddr /= AMDGPU_GPU_PAGE_SIZE;
1387 eaddr /= AMDGPU_GPU_PAGE_SIZE;
1389 it = interval_tree_iter_first(&vm->va, saddr, eaddr);
1391 struct amdgpu_bo_va_mapping *tmp;
1392 tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
1393 /* bo and tmp overlap, invalid addr */
1394 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1395 "0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
1396 tmp->it.start, tmp->it.last + 1);
1401 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1407 INIT_LIST_HEAD(&mapping->list);
1408 mapping->it.start = saddr;
1409 mapping->it.last = eaddr;
1410 mapping->offset = offset;
1411 mapping->flags = flags;
1413 list_add(&mapping->list, &bo_va->invalids);
1414 interval_tree_insert(&mapping->it, &vm->va);
1416 /* Make sure the page tables are allocated */
1417 saddr >>= amdgpu_vm_block_size;
1418 eaddr >>= amdgpu_vm_block_size;
1420 BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));
1422 if (eaddr > vm->max_pde_used)
1423 vm->max_pde_used = eaddr;
1425 /* walk over the address space and allocate the page tables */
1426 for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
1427 struct reservation_object *resv = vm->page_directory->tbo.resv;
1428 struct amdgpu_bo *pt;
1430 if (vm->page_tables[pt_idx].bo)
1433 r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
1434 AMDGPU_GPU_PAGE_SIZE, true,
1435 AMDGPU_GEM_DOMAIN_VRAM,
1436 AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
1437 AMDGPU_GEM_CREATE_SHADOW |
1438 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
1443 /* Keep a reference to the page table to avoid freeing
1444 * them up in the wrong order.
1446 pt->parent = amdgpu_bo_ref(vm->page_directory);
1448 r = amdgpu_vm_clear_bo(adev, vm, pt);
1450 amdgpu_bo_unref(&pt->shadow);
1451 amdgpu_bo_unref(&pt);
1456 r = amdgpu_vm_clear_bo(adev, vm, pt->shadow);
1458 amdgpu_bo_unref(&pt->shadow);
1459 amdgpu_bo_unref(&pt);
1464 vm->page_tables[pt_idx].bo = pt;
1465 vm->page_tables[pt_idx].addr = 0;
1471 list_del(&mapping->list);
1472 interval_tree_remove(&mapping->it, &vm->va);
1473 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1481 * amdgpu_vm_bo_unmap - remove bo mapping from vm
1483 * @adev: amdgpu_device pointer
1484 * @bo_va: bo_va to remove the address from
1485 * @saddr: where to the BO is mapped
1487 * Remove a mapping of the BO at the specefied addr from the VM.
1488 * Returns 0 for success, error for failure.
1490 * Object has to be reserved and unreserved outside!
1492 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
1493 struct amdgpu_bo_va *bo_va,
1496 struct amdgpu_bo_va_mapping *mapping;
1497 struct amdgpu_vm *vm = bo_va->vm;
1500 saddr /= AMDGPU_GPU_PAGE_SIZE;
1502 list_for_each_entry(mapping, &bo_va->valids, list) {
1503 if (mapping->it.start == saddr)
1507 if (&mapping->list == &bo_va->valids) {
1510 list_for_each_entry(mapping, &bo_va->invalids, list) {
1511 if (mapping->it.start == saddr)
1515 if (&mapping->list == &bo_va->invalids)
1519 list_del(&mapping->list);
1520 interval_tree_remove(&mapping->it, &vm->va);
1521 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1524 list_add(&mapping->list, &vm->freed);
1532 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
1534 * @adev: amdgpu_device pointer
1535 * @bo_va: requested bo_va
1537 * Remove @bo_va->bo from the requested vm.
1539 * Object have to be reserved!
1541 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
1542 struct amdgpu_bo_va *bo_va)
1544 struct amdgpu_bo_va_mapping *mapping, *next;
1545 struct amdgpu_vm *vm = bo_va->vm;
1547 list_del(&bo_va->bo_list);
1549 spin_lock(&vm->status_lock);
1550 list_del(&bo_va->vm_status);
1551 spin_unlock(&vm->status_lock);
1553 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
1554 list_del(&mapping->list);
1555 interval_tree_remove(&mapping->it, &vm->va);
1556 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1557 list_add(&mapping->list, &vm->freed);
1559 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
1560 list_del(&mapping->list);
1561 interval_tree_remove(&mapping->it, &vm->va);
1565 fence_put(bo_va->last_pt_update);
1570 * amdgpu_vm_bo_invalidate - mark the bo as invalid
1572 * @adev: amdgpu_device pointer
1574 * @bo: amdgpu buffer object
1576 * Mark @bo as invalid.
1578 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
1579 struct amdgpu_bo *bo)
1581 struct amdgpu_bo_va *bo_va;
1583 list_for_each_entry(bo_va, &bo->va, bo_list) {
1584 spin_lock(&bo_va->vm->status_lock);
1585 if (list_empty(&bo_va->vm_status))
1586 list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
1587 spin_unlock(&bo_va->vm->status_lock);
1592 * amdgpu_vm_init - initialize a vm instance
1594 * @adev: amdgpu_device pointer
1599 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1601 const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
1602 AMDGPU_VM_PTE_COUNT * 8);
1603 unsigned pd_size, pd_entries;
1604 unsigned ring_instance;
1605 struct amdgpu_ring *ring;
1606 struct amd_sched_rq *rq;
1609 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1612 vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
1613 spin_lock_init(&vm->status_lock);
1614 INIT_LIST_HEAD(&vm->invalidated);
1615 INIT_LIST_HEAD(&vm->cleared);
1616 INIT_LIST_HEAD(&vm->freed);
1618 pd_size = amdgpu_vm_directory_size(adev);
1619 pd_entries = amdgpu_vm_num_pdes(adev);
1621 /* allocate page table array */
1622 vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
1623 if (vm->page_tables == NULL) {
1624 DRM_ERROR("Cannot allocate memory for page table array\n");
1628 /* create scheduler entity for page table updates */
1630 ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
1631 ring_instance %= adev->vm_manager.vm_pte_num_rings;
1632 ring = adev->vm_manager.vm_pte_rings[ring_instance];
1633 rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
1634 r = amd_sched_entity_init(&ring->sched, &vm->entity,
1635 rq, amdgpu_sched_jobs);
1639 vm->page_directory_fence = NULL;
1641 r = amdgpu_bo_create(adev, pd_size, align, true,
1642 AMDGPU_GEM_DOMAIN_VRAM,
1643 AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
1644 AMDGPU_GEM_CREATE_SHADOW |
1645 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
1646 NULL, NULL, &vm->page_directory);
1648 goto error_free_sched_entity;
1650 r = amdgpu_bo_reserve(vm->page_directory, false);
1652 goto error_free_page_directory;
1654 r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory);
1656 goto error_unreserve;
1658 if (vm->page_directory->shadow) {
1659 r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory->shadow);
1661 goto error_unreserve;
1664 vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
1665 amdgpu_bo_unreserve(vm->page_directory);
1670 amdgpu_bo_unreserve(vm->page_directory);
1672 error_free_page_directory:
1673 amdgpu_bo_unref(&vm->page_directory->shadow);
1674 amdgpu_bo_unref(&vm->page_directory);
1675 vm->page_directory = NULL;
1677 error_free_sched_entity:
1678 amd_sched_entity_fini(&ring->sched, &vm->entity);
1681 drm_free_large(vm->page_tables);
1687 * amdgpu_vm_fini - tear down a vm instance
1689 * @adev: amdgpu_device pointer
1693 * Unbind the VM and remove all bos from the vm bo list
1695 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1697 struct amdgpu_bo_va_mapping *mapping, *tmp;
1700 amd_sched_entity_fini(vm->entity.sched, &vm->entity);
1702 if (!RB_EMPTY_ROOT(&vm->va)) {
1703 dev_err(adev->dev, "still active bo inside vm\n");
1705 rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
1706 list_del(&mapping->list);
1707 interval_tree_remove(&mapping->it, &vm->va);
1710 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
1711 list_del(&mapping->list);
1715 for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) {
1716 struct amdgpu_bo *pt = vm->page_tables[i].bo;
1721 amdgpu_bo_unref(&pt->shadow);
1722 amdgpu_bo_unref(&pt);
1724 drm_free_large(vm->page_tables);
1726 amdgpu_bo_unref(&vm->page_directory->shadow);
1727 amdgpu_bo_unref(&vm->page_directory);
1728 fence_put(vm->page_directory_fence);
1732 * amdgpu_vm_manager_init - init the VM manager
1734 * @adev: amdgpu_device pointer
1736 * Initialize the VM manager structures
1738 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
1742 INIT_LIST_HEAD(&adev->vm_manager.ids_lru);
1744 /* skip over VMID 0, since it is the system VM */
1745 for (i = 1; i < adev->vm_manager.num_ids; ++i) {
1746 amdgpu_vm_reset_id(adev, i);
1747 amdgpu_sync_create(&adev->vm_manager.ids[i].active);
1748 list_add_tail(&adev->vm_manager.ids[i].list,
1749 &adev->vm_manager.ids_lru);
1752 adev->vm_manager.fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
1753 for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
1754 adev->vm_manager.seqno[i] = 0;
1756 atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
1757 atomic64_set(&adev->vm_manager.client_counter, 0);
1761 * amdgpu_vm_manager_fini - cleanup VM manager
1763 * @adev: amdgpu_device pointer
1765 * Cleanup the VM manager and free resources.
1767 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
1771 for (i = 0; i < AMDGPU_NUM_VM; ++i) {
1772 struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];
1774 fence_put(adev->vm_manager.ids[i].first);
1775 amdgpu_sync_free(&adev->vm_manager.ids[i].active);
1776 fence_put(id->flushed_updates);