Commit | Line | Data |
---|---|---|
771fe6b9 JG |
1 | /* |
2 | * Copyright 2008 Advanced Micro Devices, Inc. | |
3 | * Copyright 2008 Red Hat Inc. | |
4 | * Copyright 2009 Jerome Glisse. | |
5 | * | |
6 | * Permission is hereby granted, free of charge, to any person obtaining a | |
7 | * copy of this software and associated documentation files (the "Software"), | |
8 | * to deal in the Software without restriction, including without limitation | |
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
10 | * and/or sell copies of the Software, and to permit persons to whom the | |
11 | * Software is furnished to do so, subject to the following conditions: | |
12 | * | |
13 | * The above copyright notice and this permission notice shall be included in | |
14 | * all copies or substantial portions of the Software. | |
15 | * | |
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
19 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
20 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
21 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
22 | * OTHER DEALINGS IN THE SOFTWARE. | |
23 | * | |
24 | * Authors: Dave Airlie | |
25 | * Alex Deucher | |
26 | * Jerome Glisse | |
27 | */ | |
28 | #include "drmP.h" | |
29 | #include "radeon_drm.h" | |
30 | #include "radeon.h" | |
31 | #include "radeon_reg.h" | |
32 | ||
03eec93b AD |
33 | /* |
34 | * GART | |
35 | * The GART (Graphics Aperture Remapping Table) is an aperture | |
36 | * in the GPU's address space. System pages can be mapped into | |
37 | * the aperture and look like contiguous pages from the GPU's | |
38 | * perspective. A page table maps the pages in the aperture | |
39 | * to the actual backing pages in system memory. | |
40 | * | |
41 | * Radeon GPUs support both an internal GART, as described above, | |
42 | * and AGP. AGP works similarly, but the GART table is configured | |
43 | * and maintained by the northbridge rather than the driver. | |
44 | * Radeon hw has a separate AGP aperture that is programmed to | |
45 | * point to the AGP aperture provided by the northbridge and the | |
46 | * requests are passed through to the northbridge aperture. | |
47 | * Both AGP and internal GART can be used at the same time, however | |
48 | * that is not currently supported by the driver. | |
49 | * | |
50 | * This file handles the common internal GART management. | |
51 | */ | |
52 | ||
771fe6b9 JG |
53 | /* |
54 | * Common GART table functions. | |
55 | */ | |
03eec93b AD |
56 | /** |
57 | * radeon_gart_table_ram_alloc - allocate system ram for gart page table | |
58 | * | |
59 | * @rdev: radeon_device pointer | |
60 | * | |
61 | * Allocate system memory for GART page table | |
62 | * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the | |
63 | * gart table to be in system memory. | |
64 | * Returns 0 for success, -ENOMEM for failure. | |
65 | */ | |
771fe6b9 JG |
66 | int radeon_gart_table_ram_alloc(struct radeon_device *rdev) |
67 | { | |
68 | void *ptr; | |
69 | ||
70 | ptr = pci_alloc_consistent(rdev->pdev, rdev->gart.table_size, | |
71 | &rdev->gart.table_addr); | |
72 | if (ptr == NULL) { | |
73 | return -ENOMEM; | |
74 | } | |
75 | #ifdef CONFIG_X86 | |
76 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |
77 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |
78 | set_memory_uc((unsigned long)ptr, | |
79 | rdev->gart.table_size >> PAGE_SHIFT); | |
80 | } | |
81 | #endif | |
c9a1be96 JG |
82 | rdev->gart.ptr = ptr; |
83 | memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size); | |
771fe6b9 JG |
84 | return 0; |
85 | } | |
86 | ||
03eec93b AD |
87 | /** |
88 | * radeon_gart_table_ram_free - free system ram for gart page table | |
89 | * | |
90 | * @rdev: radeon_device pointer | |
91 | * | |
92 | * Free system memory for GART page table | |
93 | * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the | |
94 | * gart table to be in system memory. | |
95 | */ | |
771fe6b9 JG |
96 | void radeon_gart_table_ram_free(struct radeon_device *rdev) |
97 | { | |
c9a1be96 | 98 | if (rdev->gart.ptr == NULL) { |
771fe6b9 JG |
99 | return; |
100 | } | |
101 | #ifdef CONFIG_X86 | |
102 | if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 || | |
103 | rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) { | |
c9a1be96 | 104 | set_memory_wb((unsigned long)rdev->gart.ptr, |
771fe6b9 JG |
105 | rdev->gart.table_size >> PAGE_SHIFT); |
106 | } | |
107 | #endif | |
108 | pci_free_consistent(rdev->pdev, rdev->gart.table_size, | |
c9a1be96 | 109 | (void *)rdev->gart.ptr, |
771fe6b9 | 110 | rdev->gart.table_addr); |
c9a1be96 | 111 | rdev->gart.ptr = NULL; |
771fe6b9 JG |
112 | rdev->gart.table_addr = 0; |
113 | } | |
114 | ||
03eec93b AD |
115 | /** |
116 | * radeon_gart_table_vram_alloc - allocate vram for gart page table | |
117 | * | |
118 | * @rdev: radeon_device pointer | |
119 | * | |
120 | * Allocate video memory for GART page table | |
121 | * (pcie r4xx, r5xx+). These asics require the | |
122 | * gart table to be in video memory. | |
123 | * Returns 0 for success, error for failure. | |
124 | */ | |
771fe6b9 JG |
125 | int radeon_gart_table_vram_alloc(struct radeon_device *rdev) |
126 | { | |
771fe6b9 JG |
127 | int r; |
128 | ||
c9a1be96 | 129 | if (rdev->gart.robj == NULL) { |
441921d5 | 130 | r = radeon_bo_create(rdev, rdev->gart.table_size, |
268b2510 | 131 | PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM, |
40f5cf99 | 132 | NULL, &rdev->gart.robj); |
771fe6b9 JG |
133 | if (r) { |
134 | return r; | |
135 | } | |
136 | } | |
4aac0473 JG |
137 | return 0; |
138 | } | |
139 | ||
03eec93b AD |
140 | /** |
141 | * radeon_gart_table_vram_pin - pin gart page table in vram | |
142 | * | |
143 | * @rdev: radeon_device pointer | |
144 | * | |
145 | * Pin the GART page table in vram so it will not be moved | |
146 | * by the memory manager (pcie r4xx, r5xx+). These asics require the | |
147 | * gart table to be in video memory. | |
148 | * Returns 0 for success, error for failure. | |
149 | */ | |
4aac0473 JG |
150 | int radeon_gart_table_vram_pin(struct radeon_device *rdev) |
151 | { | |
152 | uint64_t gpu_addr; | |
153 | int r; | |
154 | ||
c9a1be96 | 155 | r = radeon_bo_reserve(rdev->gart.robj, false); |
4c788679 | 156 | if (unlikely(r != 0)) |
771fe6b9 | 157 | return r; |
c9a1be96 | 158 | r = radeon_bo_pin(rdev->gart.robj, |
4c788679 | 159 | RADEON_GEM_DOMAIN_VRAM, &gpu_addr); |
771fe6b9 | 160 | if (r) { |
c9a1be96 | 161 | radeon_bo_unreserve(rdev->gart.robj); |
771fe6b9 JG |
162 | return r; |
163 | } | |
c9a1be96 | 164 | r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr); |
4c788679 | 165 | if (r) |
c9a1be96 JG |
166 | radeon_bo_unpin(rdev->gart.robj); |
167 | radeon_bo_unreserve(rdev->gart.robj); | |
771fe6b9 | 168 | rdev->gart.table_addr = gpu_addr; |
4c788679 | 169 | return r; |
771fe6b9 JG |
170 | } |
171 | ||
03eec93b AD |
172 | /** |
173 | * radeon_gart_table_vram_unpin - unpin gart page table in vram | |
174 | * | |
175 | * @rdev: radeon_device pointer | |
176 | * | |
177 | * Unpin the GART page table in vram (pcie r4xx, r5xx+). | |
178 | * These asics require the gart table to be in video memory. | |
179 | */ | |
c9a1be96 | 180 | void radeon_gart_table_vram_unpin(struct radeon_device *rdev) |
771fe6b9 | 181 | { |
4c788679 JG |
182 | int r; |
183 | ||
c9a1be96 | 184 | if (rdev->gart.robj == NULL) { |
771fe6b9 JG |
185 | return; |
186 | } | |
c9a1be96 | 187 | r = radeon_bo_reserve(rdev->gart.robj, false); |
4c788679 | 188 | if (likely(r == 0)) { |
c9a1be96 JG |
189 | radeon_bo_kunmap(rdev->gart.robj); |
190 | radeon_bo_unpin(rdev->gart.robj); | |
191 | radeon_bo_unreserve(rdev->gart.robj); | |
192 | rdev->gart.ptr = NULL; | |
193 | } | |
194 | } | |
195 | ||
03eec93b AD |
196 | /** |
197 | * radeon_gart_table_vram_free - free gart page table vram | |
198 | * | |
199 | * @rdev: radeon_device pointer | |
200 | * | |
201 | * Free the video memory used for the GART page table | |
202 | * (pcie r4xx, r5xx+). These asics require the gart table to | |
203 | * be in video memory. | |
204 | */ | |
c9a1be96 JG |
205 | void radeon_gart_table_vram_free(struct radeon_device *rdev) |
206 | { | |
207 | if (rdev->gart.robj == NULL) { | |
208 | return; | |
4c788679 | 209 | } |
c9a1be96 JG |
210 | radeon_gart_table_vram_unpin(rdev); |
211 | radeon_bo_unref(&rdev->gart.robj); | |
771fe6b9 JG |
212 | } |
213 | ||
771fe6b9 JG |
214 | /* |
215 | * Common gart functions. | |
216 | */ | |
03eec93b AD |
217 | /** |
218 | * radeon_gart_unbind - unbind pages from the gart page table | |
219 | * | |
220 | * @rdev: radeon_device pointer | |
221 | * @offset: offset into the GPU's gart aperture | |
222 | * @pages: number of pages to unbind | |
223 | * | |
224 | * Unbinds the requested pages from the gart page table and | |
225 | * replaces them with the dummy page (all asics). | |
226 | */ | |
771fe6b9 JG |
227 | void radeon_gart_unbind(struct radeon_device *rdev, unsigned offset, |
228 | int pages) | |
229 | { | |
230 | unsigned t; | |
231 | unsigned p; | |
232 | int i, j; | |
82568565 | 233 | u64 page_base; |
771fe6b9 JG |
234 | |
235 | if (!rdev->gart.ready) { | |
fcf4de5a | 236 | WARN(1, "trying to unbind memory from uninitialized GART !\n"); |
771fe6b9 JG |
237 | return; |
238 | } | |
a77f1718 MT |
239 | t = offset / RADEON_GPU_PAGE_SIZE; |
240 | p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); | |
771fe6b9 JG |
241 | for (i = 0; i < pages; i++, p++) { |
242 | if (rdev->gart.pages[p]) { | |
771fe6b9 | 243 | rdev->gart.pages[p] = NULL; |
82568565 DA |
244 | rdev->gart.pages_addr[p] = rdev->dummy_page.addr; |
245 | page_base = rdev->gart.pages_addr[p]; | |
a77f1718 | 246 | for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { |
c9a1be96 JG |
247 | if (rdev->gart.ptr) { |
248 | radeon_gart_set_page(rdev, t, page_base); | |
249 | } | |
82568565 | 250 | page_base += RADEON_GPU_PAGE_SIZE; |
771fe6b9 JG |
251 | } |
252 | } | |
253 | } | |
254 | mb(); | |
255 | radeon_gart_tlb_flush(rdev); | |
256 | } | |
257 | ||
03eec93b AD |
258 | /** |
259 | * radeon_gart_bind - bind pages into the gart page table | |
260 | * | |
261 | * @rdev: radeon_device pointer | |
262 | * @offset: offset into the GPU's gart aperture | |
263 | * @pages: number of pages to bind | |
264 | * @pagelist: pages to bind | |
265 | * @dma_addr: DMA addresses of pages | |
266 | * | |
267 | * Binds the requested pages to the gart page table | |
268 | * (all asics). | |
269 | * Returns 0 for success, -EINVAL for failure. | |
270 | */ | |
771fe6b9 | 271 | int radeon_gart_bind(struct radeon_device *rdev, unsigned offset, |
c39d3516 | 272 | int pages, struct page **pagelist, dma_addr_t *dma_addr) |
771fe6b9 JG |
273 | { |
274 | unsigned t; | |
275 | unsigned p; | |
276 | uint64_t page_base; | |
277 | int i, j; | |
278 | ||
279 | if (!rdev->gart.ready) { | |
fcf4de5a | 280 | WARN(1, "trying to bind memory to uninitialized GART !\n"); |
771fe6b9 JG |
281 | return -EINVAL; |
282 | } | |
a77f1718 MT |
283 | t = offset / RADEON_GPU_PAGE_SIZE; |
284 | p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); | |
771fe6b9 JG |
285 | |
286 | for (i = 0; i < pages; i++, p++) { | |
c52494f6 | 287 | rdev->gart.pages_addr[p] = dma_addr[i]; |
771fe6b9 | 288 | rdev->gart.pages[p] = pagelist[i]; |
c9a1be96 JG |
289 | if (rdev->gart.ptr) { |
290 | page_base = rdev->gart.pages_addr[p]; | |
291 | for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { | |
292 | radeon_gart_set_page(rdev, t, page_base); | |
293 | page_base += RADEON_GPU_PAGE_SIZE; | |
294 | } | |
771fe6b9 JG |
295 | } |
296 | } | |
297 | mb(); | |
298 | radeon_gart_tlb_flush(rdev); | |
299 | return 0; | |
300 | } | |
301 | ||
03eec93b AD |
302 | /** |
303 | * radeon_gart_restore - bind all pages in the gart page table | |
304 | * | |
305 | * @rdev: radeon_device pointer | |
306 | * | |
307 | * Binds all pages in the gart page table (all asics). | |
308 | * Used to rebuild the gart table on device startup or resume. | |
309 | */ | |
82568565 DA |
310 | void radeon_gart_restore(struct radeon_device *rdev) |
311 | { | |
312 | int i, j, t; | |
313 | u64 page_base; | |
314 | ||
c9a1be96 JG |
315 | if (!rdev->gart.ptr) { |
316 | return; | |
317 | } | |
82568565 DA |
318 | for (i = 0, t = 0; i < rdev->gart.num_cpu_pages; i++) { |
319 | page_base = rdev->gart.pages_addr[i]; | |
320 | for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) { | |
321 | radeon_gart_set_page(rdev, t, page_base); | |
322 | page_base += RADEON_GPU_PAGE_SIZE; | |
323 | } | |
324 | } | |
325 | mb(); | |
326 | radeon_gart_tlb_flush(rdev); | |
327 | } | |
328 | ||
03eec93b AD |
329 | /** |
330 | * radeon_gart_init - init the driver info for managing the gart | |
331 | * | |
332 | * @rdev: radeon_device pointer | |
333 | * | |
334 | * Allocate the dummy page and init the gart driver info (all asics). | |
335 | * Returns 0 for success, error for failure. | |
336 | */ | |
771fe6b9 JG |
337 | int radeon_gart_init(struct radeon_device *rdev) |
338 | { | |
82568565 DA |
339 | int r, i; |
340 | ||
771fe6b9 JG |
341 | if (rdev->gart.pages) { |
342 | return 0; | |
343 | } | |
a77f1718 MT |
344 | /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */ |
345 | if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) { | |
771fe6b9 JG |
346 | DRM_ERROR("Page size is smaller than GPU page size!\n"); |
347 | return -EINVAL; | |
348 | } | |
82568565 DA |
349 | r = radeon_dummy_page_init(rdev); |
350 | if (r) | |
351 | return r; | |
771fe6b9 JG |
352 | /* Compute table size */ |
353 | rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE; | |
a77f1718 | 354 | rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE; |
771fe6b9 JG |
355 | DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", |
356 | rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages); | |
357 | /* Allocate pages table */ | |
358 | rdev->gart.pages = kzalloc(sizeof(void *) * rdev->gart.num_cpu_pages, | |
359 | GFP_KERNEL); | |
360 | if (rdev->gart.pages == NULL) { | |
361 | radeon_gart_fini(rdev); | |
362 | return -ENOMEM; | |
363 | } | |
364 | rdev->gart.pages_addr = kzalloc(sizeof(dma_addr_t) * | |
365 | rdev->gart.num_cpu_pages, GFP_KERNEL); | |
366 | if (rdev->gart.pages_addr == NULL) { | |
367 | radeon_gart_fini(rdev); | |
368 | return -ENOMEM; | |
369 | } | |
82568565 DA |
370 | /* set GART entry to point to the dummy page by default */ |
371 | for (i = 0; i < rdev->gart.num_cpu_pages; i++) { | |
372 | rdev->gart.pages_addr[i] = rdev->dummy_page.addr; | |
373 | } | |
771fe6b9 JG |
374 | return 0; |
375 | } | |
376 | ||
03eec93b AD |
377 | /** |
378 | * radeon_gart_fini - tear down the driver info for managing the gart | |
379 | * | |
380 | * @rdev: radeon_device pointer | |
381 | * | |
382 | * Tear down the gart driver info and free the dummy page (all asics). | |
383 | */ | |
771fe6b9 JG |
384 | void radeon_gart_fini(struct radeon_device *rdev) |
385 | { | |
386 | if (rdev->gart.pages && rdev->gart.pages_addr && rdev->gart.ready) { | |
387 | /* unbind pages */ | |
388 | radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages); | |
389 | } | |
390 | rdev->gart.ready = false; | |
391 | kfree(rdev->gart.pages); | |
392 | kfree(rdev->gart.pages_addr); | |
393 | rdev->gart.pages = NULL; | |
394 | rdev->gart.pages_addr = NULL; | |
92656d70 AD |
395 | |
396 | radeon_dummy_page_fini(rdev); | |
771fe6b9 | 397 | } |
721604a1 | 398 | |
09db8644 AD |
399 | /* |
400 | * GPUVM | |
401 | * GPUVM is similar to the legacy gart on older asics, however | |
402 | * rather than there being a single global gart table | |
403 | * for the entire GPU, there are multiple VM page tables active | |
404 | * at any given time. The VM page tables can contain a mix | |
405 | * vram pages and system memory pages and system memory pages | |
406 | * can be mapped as snooped (cached system pages) or unsnooped | |
407 | * (uncached system pages). | |
408 | * Each VM has an ID associated with it and there is a page table | |
409 | * associated with each VMID. When execting a command buffer, | |
410 | * the kernel tells the the ring what VMID to use for that command | |
411 | * buffer. VMIDs are allocated dynamically as commands are submitted. | |
412 | * The userspace drivers maintain their own address space and the kernel | |
413 | * sets up their pages tables accordingly when they submit their | |
414 | * command buffers and a VMID is assigned. | |
415 | * Cayman/Trinity support up to 8 active VMs at any given time; | |
416 | * SI supports 16. | |
417 | */ | |
418 | ||
721604a1 JG |
419 | /* |
420 | * vm helpers | |
421 | * | |
422 | * TODO bind a default page at vm initialization for default address | |
423 | */ | |
c6105f24 | 424 | |
09db8644 AD |
425 | /** |
426 | * radeon_vm_manager_init - init the vm manager | |
427 | * | |
428 | * @rdev: radeon_device pointer | |
429 | * | |
430 | * Init the vm manager (cayman+). | |
431 | * Returns 0 for success, error for failure. | |
432 | */ | |
721604a1 JG |
433 | int radeon_vm_manager_init(struct radeon_device *rdev) |
434 | { | |
c6105f24 CK |
435 | struct radeon_vm *vm; |
436 | struct radeon_bo_va *bo_va; | |
721604a1 JG |
437 | int r; |
438 | ||
c6105f24 | 439 | if (!rdev->vm_manager.enabled) { |
e6b0b6a8 | 440 | /* allocate enough for 2 full VM pts */ |
c6105f24 | 441 | r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager, |
e6b0b6a8 | 442 | rdev->vm_manager.max_pfn * 8 * 2, |
c6105f24 CK |
443 | RADEON_GEM_DOMAIN_VRAM); |
444 | if (r) { | |
445 | dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n", | |
446 | (rdev->vm_manager.max_pfn * 8) >> 10); | |
447 | return r; | |
448 | } | |
67e915e4 | 449 | |
05b07147 | 450 | r = radeon_asic_vm_init(rdev); |
c6105f24 CK |
451 | if (r) |
452 | return r; | |
453 | ||
454 | rdev->vm_manager.enabled = true; | |
455 | ||
456 | r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager); | |
457 | if (r) | |
458 | return r; | |
721604a1 | 459 | } |
67e915e4 | 460 | |
c6105f24 CK |
461 | /* restore page table */ |
462 | list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) { | |
ee60e29f | 463 | if (vm->sa_bo == NULL) |
c6105f24 | 464 | continue; |
67e915e4 | 465 | |
c6105f24 CK |
466 | list_for_each_entry(bo_va, &vm->va, vm_list) { |
467 | struct ttm_mem_reg *mem = NULL; | |
468 | if (bo_va->valid) | |
469 | mem = &bo_va->bo->tbo.mem; | |
470 | ||
471 | bo_va->valid = false; | |
472 | r = radeon_vm_bo_update_pte(rdev, vm, bo_va->bo, mem); | |
473 | if (r) { | |
474 | DRM_ERROR("Failed to update pte for vm %d!\n", vm->id); | |
475 | } | |
476 | } | |
c6105f24 CK |
477 | } |
478 | return 0; | |
721604a1 JG |
479 | } |
480 | ||
36ff39c4 | 481 | /* global mutex must be lock */ |
09db8644 AD |
482 | /** |
483 | * radeon_vm_unbind_locked - unbind a specific vm | |
484 | * | |
485 | * @rdev: radeon_device pointer | |
486 | * @vm: vm to unbind | |
487 | * | |
488 | * Unbind the requested vm (cayman+). | |
489 | * Wait for use of the VM to finish, then unbind the page table, | |
490 | * and free the page table memory. | |
491 | */ | |
721604a1 JG |
492 | static void radeon_vm_unbind_locked(struct radeon_device *rdev, |
493 | struct radeon_vm *vm) | |
494 | { | |
495 | struct radeon_bo_va *bo_va; | |
496 | ||
721604a1 | 497 | /* wait for vm use to end */ |
35e56bd0 CK |
498 | while (vm->fence) { |
499 | int r; | |
500 | r = radeon_fence_wait(vm->fence, false); | |
501 | if (r) | |
502 | DRM_ERROR("error while waiting for fence: %d\n", r); | |
503 | if (r == -EDEADLK) { | |
504 | mutex_unlock(&rdev->vm_manager.lock); | |
505 | r = radeon_gpu_reset(rdev); | |
506 | mutex_lock(&rdev->vm_manager.lock); | |
507 | if (!r) | |
508 | continue; | |
509 | } | |
510 | break; | |
721604a1 | 511 | } |
35e56bd0 | 512 | radeon_fence_unref(&vm->fence); |
9b40e5d8 | 513 | radeon_fence_unref(&vm->last_flush); |
721604a1 JG |
514 | |
515 | /* hw unbind */ | |
721604a1 | 516 | list_del_init(&vm->list); |
557017a0 | 517 | radeon_sa_bo_free(rdev, &vm->sa_bo, NULL); |
721604a1 JG |
518 | vm->pt = NULL; |
519 | ||
520 | list_for_each_entry(bo_va, &vm->va, vm_list) { | |
521 | bo_va->valid = false; | |
522 | } | |
523 | } | |
524 | ||
09db8644 AD |
525 | /** |
526 | * radeon_vm_manager_fini - tear down the vm manager | |
527 | * | |
528 | * @rdev: radeon_device pointer | |
529 | * | |
530 | * Tear down the VM manager (cayman+). | |
531 | */ | |
721604a1 | 532 | void radeon_vm_manager_fini(struct radeon_device *rdev) |
721604a1 JG |
533 | { |
534 | struct radeon_vm *vm, *tmp; | |
ee60e29f | 535 | int i; |
721604a1 | 536 | |
c6105f24 CK |
537 | if (!rdev->vm_manager.enabled) |
538 | return; | |
539 | ||
36ff39c4 | 540 | mutex_lock(&rdev->vm_manager.lock); |
721604a1 JG |
541 | /* unbind all active vm */ |
542 | list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) { | |
543 | radeon_vm_unbind_locked(rdev, vm); | |
544 | } | |
ee60e29f CK |
545 | for (i = 0; i < RADEON_NUM_VM; ++i) { |
546 | radeon_fence_unref(&rdev->vm_manager.active[i]); | |
547 | } | |
05b07147 | 548 | radeon_asic_vm_fini(rdev); |
36ff39c4 | 549 | mutex_unlock(&rdev->vm_manager.lock); |
c6105f24 CK |
550 | |
551 | radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager); | |
552 | radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager); | |
553 | rdev->vm_manager.enabled = false; | |
721604a1 JG |
554 | } |
555 | ||
36ff39c4 | 556 | /* global mutex must be locked */ |
09db8644 AD |
557 | /** |
558 | * radeon_vm_unbind - locked version of unbind | |
559 | * | |
560 | * @rdev: radeon_device pointer | |
561 | * @vm: vm to unbind | |
562 | * | |
563 | * Locked version that wraps radeon_vm_unbind_locked (cayman+). | |
564 | */ | |
721604a1 JG |
565 | void radeon_vm_unbind(struct radeon_device *rdev, struct radeon_vm *vm) |
566 | { | |
567 | mutex_lock(&vm->mutex); | |
568 | radeon_vm_unbind_locked(rdev, vm); | |
569 | mutex_unlock(&vm->mutex); | |
570 | } | |
571 | ||
36ff39c4 | 572 | /* global and local mutex must be locked */ |
09db8644 AD |
573 | /** |
574 | * radeon_vm_bind - bind a page table to a VMID | |
575 | * | |
576 | * @rdev: radeon_device pointer | |
577 | * @vm: vm to bind | |
578 | * | |
579 | * Bind the requested vm (cayman+). | |
580 | * Suballocate memory for the page table, allocate a VMID | |
581 | * and bind the page table to it, and finally start to populate | |
582 | * the page table. | |
583 | * Returns 0 for success, error for failure. | |
584 | */ | |
721604a1 JG |
585 | int radeon_vm_bind(struct radeon_device *rdev, struct radeon_vm *vm) |
586 | { | |
587 | struct radeon_vm *vm_evict; | |
ee60e29f | 588 | int r; |
721604a1 JG |
589 | |
590 | if (vm == NULL) { | |
591 | return -EINVAL; | |
592 | } | |
593 | ||
ee60e29f | 594 | if (vm->sa_bo != NULL) { |
721604a1 JG |
595 | /* update lru */ |
596 | list_del_init(&vm->list); | |
597 | list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); | |
598 | return 0; | |
599 | } | |
600 | ||
601 | retry: | |
602 | r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager, &vm->sa_bo, | |
603 | RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8), | |
557017a0 | 604 | RADEON_GPU_PAGE_SIZE, false); |
721604a1 JG |
605 | if (r) { |
606 | if (list_empty(&rdev->vm_manager.lru_vm)) { | |
607 | return r; | |
608 | } | |
609 | vm_evict = list_first_entry(&rdev->vm_manager.lru_vm, struct radeon_vm, list); | |
610 | radeon_vm_unbind(rdev, vm_evict); | |
611 | goto retry; | |
612 | } | |
2e0d9910 CK |
613 | vm->pt = radeon_sa_bo_cpu_addr(vm->sa_bo); |
614 | vm->pt_gpu_addr = radeon_sa_bo_gpu_addr(vm->sa_bo); | |
721604a1 JG |
615 | memset(vm->pt, 0, RADEON_GPU_PAGE_ALIGN(vm->last_pfn * 8)); |
616 | ||
ee60e29f CK |
617 | list_add_tail(&vm->list, &rdev->vm_manager.lru_vm); |
618 | return radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo, | |
619 | &rdev->ring_tmp_bo.bo->tbo.mem); | |
620 | } | |
621 | ||
622 | /** | |
623 | * radeon_vm_grab_id - allocate the next free VMID | |
624 | * | |
625 | * @rdev: radeon_device pointer | |
626 | * @vm: vm to allocate id for | |
627 | * @ring: ring we want to submit job to | |
628 | * | |
629 | * Allocate an id for the vm (cayman+). | |
630 | * Returns the fence we need to sync to (if any). | |
631 | * | |
632 | * Global and local mutex must be locked! | |
633 | */ | |
634 | struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev, | |
635 | struct radeon_vm *vm, int ring) | |
636 | { | |
637 | struct radeon_fence *best[RADEON_NUM_RINGS] = {}; | |
638 | unsigned choices[2] = {}; | |
639 | unsigned i; | |
640 | ||
641 | /* check if the id is still valid */ | |
642 | if (vm->fence && vm->fence == rdev->vm_manager.active[vm->id]) | |
643 | return NULL; | |
644 | ||
645 | /* we definately need to flush */ | |
646 | radeon_fence_unref(&vm->last_flush); | |
647 | ||
648 | /* skip over VMID 0, since it is the system VM */ | |
649 | for (i = 1; i < rdev->vm_manager.nvm; ++i) { | |
650 | struct radeon_fence *fence = rdev->vm_manager.active[i]; | |
651 | ||
652 | if (fence == NULL) { | |
653 | /* found a free one */ | |
654 | vm->id = i; | |
655 | return NULL; | |
656 | } | |
657 | ||
658 | if (radeon_fence_is_earlier(fence, best[fence->ring])) { | |
659 | best[fence->ring] = fence; | |
660 | choices[fence->ring == ring ? 0 : 1] = i; | |
721604a1 | 661 | } |
721604a1 JG |
662 | } |
663 | ||
ee60e29f CK |
664 | for (i = 0; i < 2; ++i) { |
665 | if (choices[i]) { | |
666 | vm->id = choices[i]; | |
667 | return rdev->vm_manager.active[choices[i]]; | |
668 | } | |
721604a1 | 669 | } |
ee60e29f CK |
670 | |
671 | /* should never happen */ | |
672 | BUG(); | |
673 | return NULL; | |
674 | } | |
675 | ||
676 | /** | |
677 | * radeon_vm_fence - remember fence for vm | |
678 | * | |
679 | * @rdev: radeon_device pointer | |
680 | * @vm: vm we want to fence | |
681 | * @fence: fence to remember | |
682 | * | |
683 | * Fence the vm (cayman+). | |
684 | * Set the fence used to protect page table and id. | |
685 | * | |
686 | * Global and local mutex must be locked! | |
687 | */ | |
688 | void radeon_vm_fence(struct radeon_device *rdev, | |
689 | struct radeon_vm *vm, | |
690 | struct radeon_fence *fence) | |
691 | { | |
692 | radeon_fence_unref(&rdev->vm_manager.active[vm->id]); | |
693 | rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence); | |
694 | ||
695 | radeon_fence_unref(&vm->fence); | |
696 | vm->fence = radeon_fence_ref(fence); | |
721604a1 JG |
697 | } |
698 | ||
699 | /* object have to be reserved */ | |
09db8644 AD |
700 | /** |
701 | * radeon_vm_bo_add - add a bo to a specific vm | |
702 | * | |
703 | * @rdev: radeon_device pointer | |
704 | * @vm: requested vm | |
705 | * @bo: radeon buffer object | |
706 | * @offset: requested offset of the buffer in the VM address space | |
707 | * @flags: attributes of pages (read/write/valid/etc.) | |
708 | * | |
709 | * Add @bo into the requested vm (cayman+). | |
710 | * Add @bo to the list of bos associated with the vm and validate | |
711 | * the offset requested within the vm address space. | |
712 | * Returns 0 for success, error for failure. | |
713 | */ | |
721604a1 JG |
714 | int radeon_vm_bo_add(struct radeon_device *rdev, |
715 | struct radeon_vm *vm, | |
716 | struct radeon_bo *bo, | |
717 | uint64_t offset, | |
718 | uint32_t flags) | |
719 | { | |
720 | struct radeon_bo_va *bo_va, *tmp; | |
721 | struct list_head *head; | |
722 | uint64_t size = radeon_bo_size(bo), last_offset = 0; | |
723 | unsigned last_pfn; | |
724 | ||
725 | bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL); | |
726 | if (bo_va == NULL) { | |
727 | return -ENOMEM; | |
728 | } | |
729 | bo_va->vm = vm; | |
730 | bo_va->bo = bo; | |
731 | bo_va->soffset = offset; | |
732 | bo_va->eoffset = offset + size; | |
733 | bo_va->flags = flags; | |
734 | bo_va->valid = false; | |
735 | INIT_LIST_HEAD(&bo_va->bo_list); | |
736 | INIT_LIST_HEAD(&bo_va->vm_list); | |
737 | /* make sure object fit at this offset */ | |
738 | if (bo_va->soffset >= bo_va->eoffset) { | |
739 | kfree(bo_va); | |
740 | return -EINVAL; | |
741 | } | |
742 | ||
743 | last_pfn = bo_va->eoffset / RADEON_GPU_PAGE_SIZE; | |
744 | if (last_pfn > rdev->vm_manager.max_pfn) { | |
745 | kfree(bo_va); | |
746 | dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n", | |
747 | last_pfn, rdev->vm_manager.max_pfn); | |
748 | return -EINVAL; | |
749 | } | |
750 | ||
751 | mutex_lock(&vm->mutex); | |
752 | if (last_pfn > vm->last_pfn) { | |
bb409155 CK |
753 | /* release mutex and lock in right order */ |
754 | mutex_unlock(&vm->mutex); | |
36ff39c4 | 755 | mutex_lock(&rdev->vm_manager.lock); |
bb409155 CK |
756 | mutex_lock(&vm->mutex); |
757 | /* and check again */ | |
758 | if (last_pfn > vm->last_pfn) { | |
759 | /* grow va space 32M by 32M */ | |
760 | unsigned align = ((32 << 20) >> 12) - 1; | |
761 | radeon_vm_unbind_locked(rdev, vm); | |
762 | vm->last_pfn = (last_pfn + align) & ~align; | |
763 | } | |
36ff39c4 | 764 | mutex_unlock(&rdev->vm_manager.lock); |
721604a1 JG |
765 | } |
766 | head = &vm->va; | |
767 | last_offset = 0; | |
768 | list_for_each_entry(tmp, &vm->va, vm_list) { | |
769 | if (bo_va->soffset >= last_offset && bo_va->eoffset < tmp->soffset) { | |
770 | /* bo can be added before this one */ | |
771 | break; | |
772 | } | |
773 | if (bo_va->soffset >= tmp->soffset && bo_va->soffset < tmp->eoffset) { | |
774 | /* bo and tmp overlap, invalid offset */ | |
721604a1 JG |
775 | dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n", |
776 | bo, (unsigned)bo_va->soffset, tmp->bo, | |
777 | (unsigned)tmp->soffset, (unsigned)tmp->eoffset); | |
55ba70c4 | 778 | kfree(bo_va); |
721604a1 JG |
779 | mutex_unlock(&vm->mutex); |
780 | return -EINVAL; | |
781 | } | |
782 | last_offset = tmp->eoffset; | |
783 | head = &tmp->vm_list; | |
784 | } | |
785 | list_add(&bo_va->vm_list, head); | |
786 | list_add_tail(&bo_va->bo_list, &bo->va); | |
787 | mutex_unlock(&vm->mutex); | |
788 | return 0; | |
789 | } | |
790 | ||
09db8644 AD |
791 | /** |
792 | * radeon_vm_get_addr - get the physical address of the page | |
793 | * | |
794 | * @rdev: radeon_device pointer | |
795 | * @mem: ttm mem | |
796 | * @pfn: pfn | |
797 | * | |
798 | * Look up the physical address of the page that the pte resolves | |
799 | * to (cayman+). | |
800 | * Returns the physical address of the page. | |
801 | */ | |
721604a1 JG |
802 | static u64 radeon_vm_get_addr(struct radeon_device *rdev, |
803 | struct ttm_mem_reg *mem, | |
804 | unsigned pfn) | |
805 | { | |
806 | u64 addr = 0; | |
807 | ||
808 | switch (mem->mem_type) { | |
809 | case TTM_PL_VRAM: | |
810 | addr = (mem->start << PAGE_SHIFT); | |
811 | addr += pfn * RADEON_GPU_PAGE_SIZE; | |
812 | addr += rdev->vm_manager.vram_base_offset; | |
813 | break; | |
814 | case TTM_PL_TT: | |
815 | /* offset inside page table */ | |
816 | addr = mem->start << PAGE_SHIFT; | |
817 | addr += pfn * RADEON_GPU_PAGE_SIZE; | |
818 | addr = addr >> PAGE_SHIFT; | |
819 | /* page table offset */ | |
820 | addr = rdev->gart.pages_addr[addr]; | |
821 | /* in case cpu page size != gpu page size*/ | |
822 | addr += (pfn * RADEON_GPU_PAGE_SIZE) & (~PAGE_MASK); | |
823 | break; | |
824 | default: | |
825 | break; | |
826 | } | |
827 | return addr; | |
828 | } | |
829 | ||
36ff39c4 | 830 | /* object have to be reserved & global and local mutex must be locked */ |
09db8644 AD |
831 | /** |
832 | * radeon_vm_bo_update_pte - map a bo into the vm page table | |
833 | * | |
834 | * @rdev: radeon_device pointer | |
835 | * @vm: requested vm | |
836 | * @bo: radeon buffer object | |
837 | * @mem: ttm mem | |
838 | * | |
839 | * Fill in the page table entries for @bo (cayman+). | |
840 | * Returns 0 for success, -EINVAL for failure. | |
841 | */ | |
721604a1 JG |
842 | int radeon_vm_bo_update_pte(struct radeon_device *rdev, |
843 | struct radeon_vm *vm, | |
844 | struct radeon_bo *bo, | |
845 | struct ttm_mem_reg *mem) | |
846 | { | |
847 | struct radeon_bo_va *bo_va; | |
848 | unsigned ngpu_pages, i; | |
849 | uint64_t addr = 0, pfn; | |
850 | uint32_t flags; | |
851 | ||
852 | /* nothing to do if vm isn't bound */ | |
ee60e29f | 853 | if (vm->sa_bo == NULL) |
04bd27ae | 854 | return 0; |
721604a1 JG |
855 | |
856 | bo_va = radeon_bo_va(bo, vm); | |
857 | if (bo_va == NULL) { | |
858 | dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); | |
859 | return -EINVAL; | |
860 | } | |
861 | ||
e43b5ec0 | 862 | if (bo_va->valid && mem) |
721604a1 JG |
863 | return 0; |
864 | ||
865 | ngpu_pages = radeon_bo_ngpu_pages(bo); | |
866 | bo_va->flags &= ~RADEON_VM_PAGE_VALID; | |
867 | bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM; | |
868 | if (mem) { | |
869 | if (mem->mem_type != TTM_PL_SYSTEM) { | |
870 | bo_va->flags |= RADEON_VM_PAGE_VALID; | |
871 | bo_va->valid = true; | |
872 | } | |
873 | if (mem->mem_type == TTM_PL_TT) { | |
874 | bo_va->flags |= RADEON_VM_PAGE_SYSTEM; | |
875 | } | |
876 | } | |
877 | pfn = bo_va->soffset / RADEON_GPU_PAGE_SIZE; | |
05b07147 | 878 | flags = radeon_asic_vm_page_flags(rdev, bo_va->vm, bo_va->flags); |
721604a1 JG |
879 | for (i = 0, addr = 0; i < ngpu_pages; i++) { |
880 | if (mem && bo_va->valid) { | |
881 | addr = radeon_vm_get_addr(rdev, mem, i); | |
882 | } | |
05b07147 | 883 | radeon_asic_vm_set_page(rdev, bo_va->vm, i + pfn, addr, flags); |
721604a1 | 884 | } |
9b40e5d8 | 885 | radeon_fence_unref(&vm->last_flush); |
721604a1 JG |
886 | return 0; |
887 | } | |
888 | ||
889 | /* object have to be reserved */ | |
09db8644 AD |
890 | /** |
891 | * radeon_vm_bo_rmv - remove a bo to a specific vm | |
892 | * | |
893 | * @rdev: radeon_device pointer | |
894 | * @vm: requested vm | |
895 | * @bo: radeon buffer object | |
896 | * | |
897 | * Remove @bo from the requested vm (cayman+). | |
898 | * Remove @bo from the list of bos associated with the vm and | |
899 | * remove the ptes for @bo in the page table. | |
900 | * Returns 0 for success. | |
901 | */ | |
721604a1 JG |
902 | int radeon_vm_bo_rmv(struct radeon_device *rdev, |
903 | struct radeon_vm *vm, | |
904 | struct radeon_bo *bo) | |
905 | { | |
906 | struct radeon_bo_va *bo_va; | |
e43b5ec0 | 907 | int r; |
721604a1 JG |
908 | |
909 | bo_va = radeon_bo_va(bo, vm); | |
910 | if (bo_va == NULL) | |
911 | return 0; | |
912 | ||
e43b5ec0 JG |
913 | /* wait for va use to end */ |
914 | while (bo_va->fence) { | |
915 | r = radeon_fence_wait(bo_va->fence, false); | |
916 | if (r) { | |
917 | DRM_ERROR("error while waiting for fence: %d\n", r); | |
918 | } | |
919 | if (r == -EDEADLK) { | |
920 | r = radeon_gpu_reset(rdev); | |
921 | if (!r) | |
922 | continue; | |
923 | } | |
924 | break; | |
925 | } | |
926 | radeon_fence_unref(&bo_va->fence); | |
927 | ||
36ff39c4 | 928 | mutex_lock(&rdev->vm_manager.lock); |
bb409155 | 929 | mutex_lock(&vm->mutex); |
721604a1 | 930 | radeon_vm_bo_update_pte(rdev, vm, bo, NULL); |
36ff39c4 | 931 | mutex_unlock(&rdev->vm_manager.lock); |
721604a1 | 932 | list_del(&bo_va->vm_list); |
a7eef882 | 933 | mutex_unlock(&vm->mutex); |
108b0d34 | 934 | list_del(&bo_va->bo_list); |
721604a1 JG |
935 | |
936 | kfree(bo_va); | |
937 | return 0; | |
938 | } | |
939 | ||
09db8644 AD |
940 | /** |
941 | * radeon_vm_bo_invalidate - mark the bo as invalid | |
942 | * | |
943 | * @rdev: radeon_device pointer | |
944 | * @vm: requested vm | |
945 | * @bo: radeon buffer object | |
946 | * | |
947 | * Mark @bo as invalid (cayman+). | |
948 | */ | |
721604a1 JG |
949 | void radeon_vm_bo_invalidate(struct radeon_device *rdev, |
950 | struct radeon_bo *bo) | |
951 | { | |
952 | struct radeon_bo_va *bo_va; | |
953 | ||
954 | BUG_ON(!atomic_read(&bo->tbo.reserved)); | |
955 | list_for_each_entry(bo_va, &bo->va, bo_list) { | |
956 | bo_va->valid = false; | |
957 | } | |
958 | } | |
959 | ||
09db8644 AD |
960 | /** |
961 | * radeon_vm_init - initialize a vm instance | |
962 | * | |
963 | * @rdev: radeon_device pointer | |
964 | * @vm: requested vm | |
965 | * | |
966 | * Init @vm (cayman+). | |
967 | * Map the IB pool and any other shared objects into the VM | |
968 | * by default as it's used by all VMs. | |
969 | * Returns 0 for success, error for failure. | |
970 | */ | |
721604a1 JG |
971 | int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm) |
972 | { | |
973 | int r; | |
974 | ||
ee60e29f | 975 | vm->id = 0; |
721604a1 JG |
976 | vm->fence = NULL; |
977 | mutex_init(&vm->mutex); | |
978 | INIT_LIST_HEAD(&vm->list); | |
979 | INIT_LIST_HEAD(&vm->va); | |
c21b328e AD |
980 | /* SI requires equal sized PTs for all VMs, so always set |
981 | * last_pfn to max_pfn. cayman allows variable sized | |
982 | * pts so we can grow then as needed. Once we switch | |
983 | * to two level pts we can unify this again. | |
984 | */ | |
985 | if (rdev->family >= CHIP_TAHITI) | |
986 | vm->last_pfn = rdev->vm_manager.max_pfn; | |
987 | else | |
988 | vm->last_pfn = 0; | |
721604a1 JG |
989 | /* map the ib pool buffer at 0 in virtual address space, set |
990 | * read only | |
991 | */ | |
c507f7ef | 992 | r = radeon_vm_bo_add(rdev, vm, rdev->ring_tmp_bo.bo, 0, |
721604a1 JG |
993 | RADEON_VM_PAGE_READABLE | RADEON_VM_PAGE_SNOOPED); |
994 | return r; | |
995 | } | |
996 | ||
09db8644 | 997 | /** |
f59abbf2 | 998 | * radeon_vm_fini - tear down a vm instance |
09db8644 AD |
999 | * |
1000 | * @rdev: radeon_device pointer | |
1001 | * @vm: requested vm | |
1002 | * | |
1003 | * Tear down @vm (cayman+). | |
1004 | * Unbind the VM and remove all bos from the vm bo list | |
1005 | */ | |
721604a1 JG |
1006 | void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm) |
1007 | { | |
1008 | struct radeon_bo_va *bo_va, *tmp; | |
1009 | int r; | |
1010 | ||
36ff39c4 | 1011 | mutex_lock(&rdev->vm_manager.lock); |
bb409155 | 1012 | mutex_lock(&vm->mutex); |
721604a1 | 1013 | radeon_vm_unbind_locked(rdev, vm); |
36ff39c4 | 1014 | mutex_unlock(&rdev->vm_manager.lock); |
721604a1 | 1015 | |
e43b5ec0 JG |
1016 | /* remove all bo at this point non are busy any more because unbind |
1017 | * waited for the last vm fence to signal | |
1018 | */ | |
c507f7ef | 1019 | r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false); |
721604a1 | 1020 | if (!r) { |
c507f7ef | 1021 | bo_va = radeon_bo_va(rdev->ring_tmp_bo.bo, vm); |
721604a1 JG |
1022 | list_del_init(&bo_va->bo_list); |
1023 | list_del_init(&bo_va->vm_list); | |
e43b5ec0 | 1024 | radeon_fence_unref(&bo_va->fence); |
c507f7ef | 1025 | radeon_bo_unreserve(rdev->ring_tmp_bo.bo); |
721604a1 JG |
1026 | kfree(bo_va); |
1027 | } | |
1028 | if (!list_empty(&vm->va)) { | |
1029 | dev_err(rdev->dev, "still active bo inside vm\n"); | |
1030 | } | |
1031 | list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) { | |
1032 | list_del_init(&bo_va->vm_list); | |
1033 | r = radeon_bo_reserve(bo_va->bo, false); | |
1034 | if (!r) { | |
1035 | list_del_init(&bo_va->bo_list); | |
e43b5ec0 | 1036 | radeon_fence_unref(&bo_va->fence); |
721604a1 JG |
1037 | radeon_bo_unreserve(bo_va->bo); |
1038 | kfree(bo_va); | |
1039 | } | |
1040 | } | |
1041 | mutex_unlock(&vm->mutex); | |
1042 | } |