Commit | Line | Data |
---|---|---|
d38ceaf9 AD |
1 | /* |
2 | * Copyright 2009 Jerome Glisse. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sub license, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
20 | * | |
21 | * The above copyright notice and this permission notice (including the | |
22 | * next paragraph) shall be included in all copies or substantial portions | |
23 | * of the Software. | |
24 | * | |
25 | */ | |
26 | /* | |
27 | * Authors: | |
28 | * Jerome Glisse <glisse@freedesktop.org> | |
29 | * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> | |
30 | * Dave Airlie | |
31 | */ | |
32 | #include <linux/list.h> | |
33 | #include <linux/slab.h> | |
34 | #include <drm/drmP.h> | |
35 | #include <drm/amdgpu_drm.h> | |
a187f17f | 36 | #include <drm/drm_cache.h> |
d38ceaf9 AD |
37 | #include "amdgpu.h" |
38 | #include "amdgpu_trace.h" | |
39 | ||
d38ceaf9 AD |
40 | static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo) |
41 | { | |
a7d64de6 | 42 | struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); |
b82485fd | 43 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); |
d38ceaf9 | 44 | |
6375bbb4 | 45 | amdgpu_bo_kunmap(bo); |
d38ceaf9 | 46 | |
d38ceaf9 | 47 | drm_gem_object_release(&bo->gem_base); |
82b9c55b | 48 | amdgpu_bo_unref(&bo->parent); |
0c4e7fa5 | 49 | if (!list_empty(&bo->shadow_list)) { |
a7d64de6 | 50 | mutex_lock(&adev->shadow_list_lock); |
0c4e7fa5 | 51 | list_del_init(&bo->shadow_list); |
a7d64de6 | 52 | mutex_unlock(&adev->shadow_list_lock); |
0c4e7fa5 | 53 | } |
d38ceaf9 AD |
54 | kfree(bo->metadata); |
55 | kfree(bo); | |
56 | } | |
57 | ||
58 | bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) | |
59 | { | |
60 | if (bo->destroy == &amdgpu_ttm_bo_destroy) | |
61 | return true; | |
62 | return false; | |
63 | } | |
64 | ||
c09312a6 | 65 | void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain) |
d38ceaf9 | 66 | { |
c09312a6 CK |
67 | struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); |
68 | struct ttm_placement *placement = &abo->placement; | |
69 | struct ttm_place *places = abo->placements; | |
70 | u64 flags = abo->flags; | |
6369f6f1 | 71 | u32 c = 0; |
7e5a547f | 72 | |
d38ceaf9 | 73 | if (domain & AMDGPU_GEM_DOMAIN_VRAM) { |
faceaf6a CK |
74 | unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT; |
75 | ||
faceaf6a | 76 | places[c].fpfn = 0; |
89bb5752 | 77 | places[c].lpfn = 0; |
faceaf6a | 78 | places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | |
7e5a547f | 79 | TTM_PL_FLAG_VRAM; |
89bb5752 | 80 | |
faceaf6a CK |
81 | if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) |
82 | places[c].lpfn = visible_pfn; | |
83 | else | |
84 | places[c].flags |= TTM_PL_FLAG_TOPDOWN; | |
89bb5752 CK |
85 | |
86 | if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) | |
87 | places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; | |
faceaf6a | 88 | c++; |
d38ceaf9 AD |
89 | } |
90 | ||
91 | if (domain & AMDGPU_GEM_DOMAIN_GTT) { | |
faceaf6a | 92 | places[c].fpfn = 0; |
cf273a59 CK |
93 | if (flags & AMDGPU_GEM_CREATE_SHADOW) |
94 | places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT; | |
95 | else | |
96 | places[c].lpfn = 0; | |
faceaf6a CK |
97 | places[c].flags = TTM_PL_FLAG_TT; |
98 | if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
99 | places[c].flags |= TTM_PL_FLAG_WC | | |
100 | TTM_PL_FLAG_UNCACHED; | |
101 | else | |
102 | places[c].flags |= TTM_PL_FLAG_CACHED; | |
103 | c++; | |
d38ceaf9 AD |
104 | } |
105 | ||
106 | if (domain & AMDGPU_GEM_DOMAIN_CPU) { | |
faceaf6a CK |
107 | places[c].fpfn = 0; |
108 | places[c].lpfn = 0; | |
109 | places[c].flags = TTM_PL_FLAG_SYSTEM; | |
110 | if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
111 | places[c].flags |= TTM_PL_FLAG_WC | | |
112 | TTM_PL_FLAG_UNCACHED; | |
113 | else | |
114 | places[c].flags |= TTM_PL_FLAG_CACHED; | |
115 | c++; | |
d38ceaf9 AD |
116 | } |
117 | ||
118 | if (domain & AMDGPU_GEM_DOMAIN_GDS) { | |
faceaf6a CK |
119 | places[c].fpfn = 0; |
120 | places[c].lpfn = 0; | |
121 | places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS; | |
122 | c++; | |
d38ceaf9 | 123 | } |
faceaf6a | 124 | |
d38ceaf9 | 125 | if (domain & AMDGPU_GEM_DOMAIN_GWS) { |
faceaf6a CK |
126 | places[c].fpfn = 0; |
127 | places[c].lpfn = 0; | |
128 | places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS; | |
129 | c++; | |
d38ceaf9 | 130 | } |
faceaf6a | 131 | |
d38ceaf9 | 132 | if (domain & AMDGPU_GEM_DOMAIN_OA) { |
faceaf6a CK |
133 | places[c].fpfn = 0; |
134 | places[c].lpfn = 0; | |
135 | places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA; | |
136 | c++; | |
d38ceaf9 AD |
137 | } |
138 | ||
139 | if (!c) { | |
faceaf6a CK |
140 | places[c].fpfn = 0; |
141 | places[c].lpfn = 0; | |
142 | places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; | |
143 | c++; | |
d38ceaf9 | 144 | } |
faceaf6a | 145 | |
7e5a547f | 146 | placement->num_placement = c; |
faceaf6a | 147 | placement->placement = places; |
d38ceaf9 | 148 | |
faceaf6a CK |
149 | placement->num_busy_placement = c; |
150 | placement->busy_placement = places; | |
d38ceaf9 AD |
151 | } |
152 | ||
7c204889 | 153 | /** |
9d903cbd | 154 | * amdgpu_bo_create_reserved - create reserved BO for kernel use |
7c204889 CK |
155 | * |
156 | * @adev: amdgpu device object | |
157 | * @size: size for the new BO | |
158 | * @align: alignment for the new BO | |
159 | * @domain: where to place it | |
160 | * @bo_ptr: resulting BO | |
161 | * @gpu_addr: GPU addr of the pinned BO | |
162 | * @cpu_addr: optional CPU address mapping | |
163 | * | |
9d903cbd CK |
164 | * Allocates and pins a BO for kernel internal use, and returns it still |
165 | * reserved. | |
7c204889 CK |
166 | * |
167 | * Returns 0 on success, negative error code otherwise. | |
168 | */ | |
9d903cbd CK |
169 | int amdgpu_bo_create_reserved(struct amdgpu_device *adev, |
170 | unsigned long size, int align, | |
171 | u32 domain, struct amdgpu_bo **bo_ptr, | |
172 | u64 *gpu_addr, void **cpu_addr) | |
7c204889 | 173 | { |
53766e5a | 174 | bool free = false; |
7c204889 CK |
175 | int r; |
176 | ||
53766e5a CK |
177 | if (!*bo_ptr) { |
178 | r = amdgpu_bo_create(adev, size, align, true, domain, | |
179 | AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | | |
180 | AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, | |
2046d46d | 181 | NULL, NULL, 0, bo_ptr); |
53766e5a CK |
182 | if (r) { |
183 | dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", | |
184 | r); | |
185 | return r; | |
186 | } | |
187 | free = true; | |
7c204889 CK |
188 | } |
189 | ||
190 | r = amdgpu_bo_reserve(*bo_ptr, false); | |
191 | if (r) { | |
192 | dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); | |
193 | goto error_free; | |
194 | } | |
195 | ||
196 | r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr); | |
197 | if (r) { | |
198 | dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); | |
199 | goto error_unreserve; | |
200 | } | |
201 | ||
202 | if (cpu_addr) { | |
203 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
204 | if (r) { | |
205 | dev_err(adev->dev, "(%d) kernel bo map failed\n", r); | |
206 | goto error_unreserve; | |
207 | } | |
208 | } | |
209 | ||
7c204889 CK |
210 | return 0; |
211 | ||
212 | error_unreserve: | |
213 | amdgpu_bo_unreserve(*bo_ptr); | |
214 | ||
215 | error_free: | |
53766e5a CK |
216 | if (free) |
217 | amdgpu_bo_unref(bo_ptr); | |
7c204889 CK |
218 | |
219 | return r; | |
220 | } | |
221 | ||
9d903cbd CK |
222 | /** |
223 | * amdgpu_bo_create_kernel - create BO for kernel use | |
224 | * | |
225 | * @adev: amdgpu device object | |
226 | * @size: size for the new BO | |
227 | * @align: alignment for the new BO | |
228 | * @domain: where to place it | |
229 | * @bo_ptr: resulting BO | |
230 | * @gpu_addr: GPU addr of the pinned BO | |
231 | * @cpu_addr: optional CPU address mapping | |
232 | * | |
233 | * Allocates and pins a BO for kernel internal use. | |
234 | * | |
235 | * Returns 0 on success, negative error code otherwise. | |
236 | */ | |
237 | int amdgpu_bo_create_kernel(struct amdgpu_device *adev, | |
238 | unsigned long size, int align, | |
239 | u32 domain, struct amdgpu_bo **bo_ptr, | |
240 | u64 *gpu_addr, void **cpu_addr) | |
241 | { | |
242 | int r; | |
243 | ||
244 | r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, | |
245 | gpu_addr, cpu_addr); | |
246 | ||
247 | if (r) | |
248 | return r; | |
249 | ||
250 | amdgpu_bo_unreserve(*bo_ptr); | |
251 | ||
252 | return 0; | |
253 | } | |
254 | ||
aa1d562e JZ |
255 | /** |
256 | * amdgpu_bo_free_kernel - free BO for kernel use | |
257 | * | |
258 | * @bo: amdgpu BO to free | |
259 | * | |
260 | * unmaps and unpin a BO for kernel internal use. | |
261 | */ | |
262 | void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, | |
263 | void **cpu_addr) | |
264 | { | |
265 | if (*bo == NULL) | |
266 | return; | |
267 | ||
f3aa745e | 268 | if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { |
aa1d562e JZ |
269 | if (cpu_addr) |
270 | amdgpu_bo_kunmap(*bo); | |
271 | ||
272 | amdgpu_bo_unpin(*bo); | |
273 | amdgpu_bo_unreserve(*bo); | |
274 | } | |
275 | amdgpu_bo_unref(bo); | |
276 | ||
277 | if (gpu_addr) | |
278 | *gpu_addr = 0; | |
279 | ||
280 | if (cpu_addr) | |
281 | *cpu_addr = NULL; | |
282 | } | |
283 | ||
c09312a6 CK |
284 | static int amdgpu_bo_do_create(struct amdgpu_device *adev, |
285 | unsigned long size, int byte_align, | |
286 | bool kernel, u32 domain, u64 flags, | |
287 | struct sg_table *sg, | |
288 | struct reservation_object *resv, | |
289 | uint64_t init_value, | |
290 | struct amdgpu_bo **bo_ptr) | |
d38ceaf9 AD |
291 | { |
292 | struct amdgpu_bo *bo; | |
293 | enum ttm_bo_type type; | |
294 | unsigned long page_align; | |
00f06b24 | 295 | u64 initial_bytes_moved, bytes_moved; |
d38ceaf9 AD |
296 | size_t acc_size; |
297 | int r; | |
298 | ||
d38ceaf9 AD |
299 | page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; |
300 | size = ALIGN(size, PAGE_SIZE); | |
301 | ||
302 | if (kernel) { | |
303 | type = ttm_bo_type_kernel; | |
304 | } else if (sg) { | |
305 | type = ttm_bo_type_sg; | |
306 | } else { | |
307 | type = ttm_bo_type_device; | |
308 | } | |
309 | *bo_ptr = NULL; | |
310 | ||
311 | acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, | |
312 | sizeof(struct amdgpu_bo)); | |
313 | ||
314 | bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); | |
315 | if (bo == NULL) | |
316 | return -ENOMEM; | |
317 | r = drm_gem_object_init(adev->ddev, &bo->gem_base, size); | |
318 | if (unlikely(r)) { | |
319 | kfree(bo); | |
320 | return r; | |
321 | } | |
0c4e7fa5 | 322 | INIT_LIST_HEAD(&bo->shadow_list); |
d38ceaf9 | 323 | INIT_LIST_HEAD(&bo->va); |
6d7d9c5a | 324 | bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM | |
1ea863fd CK |
325 | AMDGPU_GEM_DOMAIN_GTT | |
326 | AMDGPU_GEM_DOMAIN_CPU | | |
327 | AMDGPU_GEM_DOMAIN_GDS | | |
328 | AMDGPU_GEM_DOMAIN_GWS | | |
329 | AMDGPU_GEM_DOMAIN_OA); | |
6d7d9c5a | 330 | bo->allowed_domains = bo->preferred_domains; |
1ea863fd CK |
331 | if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) |
332 | bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; | |
d38ceaf9 AD |
333 | |
334 | bo->flags = flags; | |
a187f17f | 335 | |
a2e2f299 NH |
336 | #ifdef CONFIG_X86_32 |
337 | /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit | |
338 | * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 | |
339 | */ | |
340 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; | |
341 | #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) | |
342 | /* Don't try to enable write-combining when it can't work, or things | |
343 | * may be slow | |
344 | * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 | |
345 | */ | |
346 | ||
31bb90f1 | 347 | #ifndef CONFIG_COMPILE_TEST |
a2e2f299 NH |
348 | #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ |
349 | thanks to write-combining | |
31bb90f1 | 350 | #endif |
a2e2f299 NH |
351 | |
352 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
353 | DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " | |
354 | "better performance thanks to write-combining\n"); | |
355 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; | |
356 | #else | |
a187f17f OG |
357 | /* For architectures that don't support WC memory, |
358 | * mask out the WC flag from the BO | |
359 | */ | |
360 | if (!drm_arch_can_wc_memory()) | |
361 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; | |
a2e2f299 | 362 | #endif |
a187f17f | 363 | |
c09312a6 CK |
364 | bo->tbo.bdev = &adev->mman.bdev; |
365 | amdgpu_ttm_placement_from_domain(bo, domain); | |
f45dc74c | 366 | |
fad06127 | 367 | initial_bytes_moved = atomic64_read(&adev->num_bytes_moved); |
c09312a6 | 368 | /* Kernel allocation are uninterruptible */ |
59c66c91 NH |
369 | r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, |
370 | &bo->placement, page_align, !kernel, NULL, | |
371 | acc_size, sg, resv, &amdgpu_ttm_bo_destroy); | |
00f06b24 JB |
372 | bytes_moved = atomic64_read(&adev->num_bytes_moved) - |
373 | initial_bytes_moved; | |
374 | if (adev->mc.visible_vram_size < adev->mc.real_vram_size && | |
375 | bo->tbo.mem.mem_type == TTM_PL_VRAM && | |
376 | bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT) | |
377 | amdgpu_cs_report_moved_bytes(adev, bytes_moved, bytes_moved); | |
378 | else | |
379 | amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0); | |
fad06127 | 380 | |
b9d022c6 | 381 | if (unlikely(r != 0)) |
d38ceaf9 | 382 | return r; |
4fea83ff | 383 | |
373308a5 | 384 | if (kernel) |
c309cd03 | 385 | bo->tbo.priority = 1; |
e1f055b3 | 386 | |
4fea83ff FC |
387 | if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && |
388 | bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { | |
f54d1867 | 389 | struct dma_fence *fence; |
4fea83ff | 390 | |
2046d46d | 391 | r = amdgpu_fill_buffer(bo, init_value, bo->tbo.resv, &fence); |
c3af1258 CK |
392 | if (unlikely(r)) |
393 | goto fail_unreserve; | |
394 | ||
4fea83ff | 395 | amdgpu_bo_fence(bo, fence, false); |
f54d1867 CW |
396 | dma_fence_put(bo->tbo.moving); |
397 | bo->tbo.moving = dma_fence_get(fence); | |
398 | dma_fence_put(fence); | |
4fea83ff | 399 | } |
f45dc74c | 400 | if (!resv) |
59c66c91 | 401 | amdgpu_bo_unreserve(bo); |
d38ceaf9 AD |
402 | *bo_ptr = bo; |
403 | ||
404 | trace_amdgpu_bo_create(bo); | |
405 | ||
96cf8271 JB |
406 | /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ |
407 | if (type == ttm_bo_type_device) | |
408 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
409 | ||
d38ceaf9 | 410 | return 0; |
4fea83ff FC |
411 | |
412 | fail_unreserve: | |
f1543f58 NH |
413 | if (!resv) |
414 | ww_mutex_unlock(&bo->tbo.resv->lock); | |
4fea83ff FC |
415 | amdgpu_bo_unref(&bo); |
416 | return r; | |
d38ceaf9 AD |
417 | } |
418 | ||
e7893c4b CZ |
419 | static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, |
420 | unsigned long size, int byte_align, | |
421 | struct amdgpu_bo *bo) | |
422 | { | |
e7893c4b CZ |
423 | int r; |
424 | ||
425 | if (bo->shadow) | |
426 | return 0; | |
427 | ||
c09312a6 CK |
428 | r = amdgpu_bo_do_create(adev, size, byte_align, true, |
429 | AMDGPU_GEM_DOMAIN_GTT, | |
430 | AMDGPU_GEM_CREATE_CPU_GTT_USWC | | |
431 | AMDGPU_GEM_CREATE_SHADOW, | |
432 | NULL, bo->tbo.resv, 0, | |
433 | &bo->shadow); | |
0c4e7fa5 | 434 | if (!r) { |
e7893c4b | 435 | bo->shadow->parent = amdgpu_bo_ref(bo); |
0c4e7fa5 CZ |
436 | mutex_lock(&adev->shadow_list_lock); |
437 | list_add_tail(&bo->shadow_list, &adev->shadow_list); | |
438 | mutex_unlock(&adev->shadow_list_lock); | |
439 | } | |
e7893c4b CZ |
440 | |
441 | return r; | |
442 | } | |
443 | ||
2046d46d YZ |
444 | /* init_value will only take effect when flags contains |
445 | * AMDGPU_GEM_CREATE_VRAM_CLEARED. | |
446 | */ | |
7e5a547f CZ |
447 | int amdgpu_bo_create(struct amdgpu_device *adev, |
448 | unsigned long size, int byte_align, | |
449 | bool kernel, u32 domain, u64 flags, | |
72d7668b CK |
450 | struct sg_table *sg, |
451 | struct reservation_object *resv, | |
2046d46d | 452 | uint64_t init_value, |
72d7668b | 453 | struct amdgpu_bo **bo_ptr) |
7e5a547f | 454 | { |
cf273a59 | 455 | uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW; |
e7893c4b | 456 | int r; |
7e5a547f | 457 | |
c09312a6 CK |
458 | r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain, |
459 | parent_flags, sg, resv, init_value, bo_ptr); | |
e7893c4b CZ |
460 | if (r) |
461 | return r; | |
462 | ||
cf273a59 CK |
463 | if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) { |
464 | if (!resv) | |
465 | WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv, | |
466 | NULL)); | |
36ea83d1 | 467 | |
e7893c4b | 468 | r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr)); |
36ea83d1 NH |
469 | |
470 | if (!resv) | |
cf273a59 | 471 | reservation_object_unlock((*bo_ptr)->tbo.resv); |
36ea83d1 | 472 | |
e7893c4b CZ |
473 | if (r) |
474 | amdgpu_bo_unref(bo_ptr); | |
475 | } | |
476 | ||
477 | return r; | |
7e5a547f CZ |
478 | } |
479 | ||
20f4eff1 CZ |
480 | int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, |
481 | struct amdgpu_ring *ring, | |
482 | struct amdgpu_bo *bo, | |
483 | struct reservation_object *resv, | |
f54d1867 | 484 | struct dma_fence **fence, |
20f4eff1 CZ |
485 | bool direct) |
486 | ||
487 | { | |
488 | struct amdgpu_bo *shadow = bo->shadow; | |
489 | uint64_t bo_addr, shadow_addr; | |
490 | int r; | |
491 | ||
492 | if (!shadow) | |
493 | return -EINVAL; | |
494 | ||
495 | bo_addr = amdgpu_bo_gpu_offset(bo); | |
496 | shadow_addr = amdgpu_bo_gpu_offset(bo->shadow); | |
497 | ||
498 | r = reservation_object_reserve_shared(bo->tbo.resv); | |
499 | if (r) | |
500 | goto err; | |
501 | ||
502 | r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr, | |
503 | amdgpu_bo_size(bo), resv, fence, | |
fc9c8f54 | 504 | direct, false); |
20f4eff1 CZ |
505 | if (!r) |
506 | amdgpu_bo_fence(bo, *fence, true); | |
507 | ||
508 | err: | |
509 | return r; | |
510 | } | |
511 | ||
82521316 RH |
512 | int amdgpu_bo_validate(struct amdgpu_bo *bo) |
513 | { | |
514 | uint32_t domain; | |
515 | int r; | |
516 | ||
517 | if (bo->pin_count) | |
518 | return 0; | |
519 | ||
6d7d9c5a | 520 | domain = bo->preferred_domains; |
82521316 RH |
521 | |
522 | retry: | |
523 | amdgpu_ttm_placement_from_domain(bo, domain); | |
524 | r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); | |
525 | if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { | |
526 | domain = bo->allowed_domains; | |
527 | goto retry; | |
528 | } | |
529 | ||
530 | return r; | |
531 | } | |
532 | ||
20f4eff1 CZ |
533 | int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev, |
534 | struct amdgpu_ring *ring, | |
535 | struct amdgpu_bo *bo, | |
536 | struct reservation_object *resv, | |
f54d1867 | 537 | struct dma_fence **fence, |
20f4eff1 CZ |
538 | bool direct) |
539 | ||
540 | { | |
541 | struct amdgpu_bo *shadow = bo->shadow; | |
542 | uint64_t bo_addr, shadow_addr; | |
543 | int r; | |
544 | ||
545 | if (!shadow) | |
546 | return -EINVAL; | |
547 | ||
548 | bo_addr = amdgpu_bo_gpu_offset(bo); | |
549 | shadow_addr = amdgpu_bo_gpu_offset(bo->shadow); | |
550 | ||
551 | r = reservation_object_reserve_shared(bo->tbo.resv); | |
552 | if (r) | |
553 | goto err; | |
554 | ||
555 | r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr, | |
556 | amdgpu_bo_size(bo), resv, fence, | |
fc9c8f54 | 557 | direct, false); |
20f4eff1 CZ |
558 | if (!r) |
559 | amdgpu_bo_fence(bo, *fence, true); | |
560 | ||
561 | err: | |
562 | return r; | |
563 | } | |
564 | ||
d38ceaf9 AD |
565 | int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) |
566 | { | |
f5e1c740 | 567 | void *kptr; |
587f3c70 | 568 | long r; |
d38ceaf9 | 569 | |
271c8125 CK |
570 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) |
571 | return -EPERM; | |
572 | ||
f5e1c740 CK |
573 | kptr = amdgpu_bo_kptr(bo); |
574 | if (kptr) { | |
575 | if (ptr) | |
576 | *ptr = kptr; | |
d38ceaf9 AD |
577 | return 0; |
578 | } | |
587f3c70 CK |
579 | |
580 | r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false, | |
581 | MAX_SCHEDULE_TIMEOUT); | |
582 | if (r < 0) | |
583 | return r; | |
584 | ||
d38ceaf9 | 585 | r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); |
587f3c70 | 586 | if (r) |
d38ceaf9 | 587 | return r; |
587f3c70 | 588 | |
587f3c70 | 589 | if (ptr) |
f5e1c740 | 590 | *ptr = amdgpu_bo_kptr(bo); |
587f3c70 | 591 | |
d38ceaf9 AD |
592 | return 0; |
593 | } | |
594 | ||
f5e1c740 CK |
595 | void *amdgpu_bo_kptr(struct amdgpu_bo *bo) |
596 | { | |
597 | bool is_iomem; | |
598 | ||
599 | return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); | |
600 | } | |
601 | ||
d38ceaf9 AD |
602 | void amdgpu_bo_kunmap(struct amdgpu_bo *bo) |
603 | { | |
f5e1c740 CK |
604 | if (bo->kmap.bo) |
605 | ttm_bo_kunmap(&bo->kmap); | |
d38ceaf9 AD |
606 | } |
607 | ||
608 | struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) | |
609 | { | |
610 | if (bo == NULL) | |
611 | return NULL; | |
612 | ||
613 | ttm_bo_reference(&bo->tbo); | |
614 | return bo; | |
615 | } | |
616 | ||
617 | void amdgpu_bo_unref(struct amdgpu_bo **bo) | |
618 | { | |
619 | struct ttm_buffer_object *tbo; | |
620 | ||
621 | if ((*bo) == NULL) | |
622 | return; | |
623 | ||
624 | tbo = &((*bo)->tbo); | |
625 | ttm_bo_unref(&tbo); | |
626 | if (tbo == NULL) | |
627 | *bo = NULL; | |
628 | } | |
629 | ||
7e5a547f CZ |
630 | int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, |
631 | u64 min_offset, u64 max_offset, | |
d38ceaf9 AD |
632 | u64 *gpu_addr) |
633 | { | |
a7d64de6 | 634 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
d38ceaf9 AD |
635 | int r, i; |
636 | ||
cc325d19 | 637 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) |
d38ceaf9 AD |
638 | return -EPERM; |
639 | ||
7e5a547f CZ |
640 | if (WARN_ON_ONCE(min_offset > max_offset)) |
641 | return -EINVAL; | |
642 | ||
803d89ad CJHR |
643 | /* A shared bo cannot be migrated to VRAM */ |
644 | if (bo->prime_shared_count && (domain == AMDGPU_GEM_DOMAIN_VRAM)) | |
645 | return -EINVAL; | |
646 | ||
d38ceaf9 | 647 | if (bo->pin_count) { |
408778e8 FC |
648 | uint32_t mem_type = bo->tbo.mem.mem_type; |
649 | ||
650 | if (domain != amdgpu_mem_type_to_domain(mem_type)) | |
651 | return -EINVAL; | |
652 | ||
d38ceaf9 AD |
653 | bo->pin_count++; |
654 | if (gpu_addr) | |
655 | *gpu_addr = amdgpu_bo_gpu_offset(bo); | |
656 | ||
657 | if (max_offset != 0) { | |
27798e07 | 658 | u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset; |
d38ceaf9 AD |
659 | WARN_ON_ONCE(max_offset < |
660 | (amdgpu_bo_gpu_offset(bo) - domain_start)); | |
661 | } | |
662 | ||
663 | return 0; | |
664 | } | |
03f48dd5 CK |
665 | |
666 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; | |
e9c7577c CK |
667 | /* force to pin into visible video ram */ |
668 | if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) | |
669 | bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
d38ceaf9 AD |
670 | amdgpu_ttm_placement_from_domain(bo, domain); |
671 | for (i = 0; i < bo->placement.num_placement; i++) { | |
e9c7577c CK |
672 | unsigned fpfn, lpfn; |
673 | ||
674 | fpfn = min_offset >> PAGE_SHIFT; | |
675 | lpfn = max_offset >> PAGE_SHIFT; | |
676 | ||
7e5a547f CZ |
677 | if (fpfn > bo->placements[i].fpfn) |
678 | bo->placements[i].fpfn = fpfn; | |
78d0e182 CK |
679 | if (!bo->placements[i].lpfn || |
680 | (lpfn && lpfn < bo->placements[i].lpfn)) | |
7e5a547f | 681 | bo->placements[i].lpfn = lpfn; |
d38ceaf9 AD |
682 | bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; |
683 | } | |
684 | ||
685 | r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); | |
6681c5eb | 686 | if (unlikely(r)) { |
a7d64de6 | 687 | dev_err(adev->dev, "%p pin failed\n", bo); |
6681c5eb CK |
688 | goto error; |
689 | } | |
690 | ||
691 | bo->pin_count = 1; | |
07306b4f CZ |
692 | if (gpu_addr != NULL) { |
693 | r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem); | |
694 | if (unlikely(r)) { | |
695 | dev_err(adev->dev, "%p bind failed\n", bo); | |
696 | goto error; | |
697 | } | |
6681c5eb | 698 | *gpu_addr = amdgpu_bo_gpu_offset(bo); |
07306b4f | 699 | } |
6681c5eb | 700 | if (domain == AMDGPU_GEM_DOMAIN_VRAM) { |
a7d64de6 | 701 | adev->vram_pin_size += amdgpu_bo_size(bo); |
6681c5eb | 702 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) |
a7d64de6 | 703 | adev->invisible_pin_size += amdgpu_bo_size(bo); |
32ab75f0 | 704 | } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { |
a7d64de6 | 705 | adev->gart_pin_size += amdgpu_bo_size(bo); |
d38ceaf9 | 706 | } |
6681c5eb CK |
707 | |
708 | error: | |
d38ceaf9 AD |
709 | return r; |
710 | } | |
711 | ||
712 | int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr) | |
713 | { | |
7e5a547f | 714 | return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr); |
d38ceaf9 AD |
715 | } |
716 | ||
717 | int amdgpu_bo_unpin(struct amdgpu_bo *bo) | |
718 | { | |
a7d64de6 | 719 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
d38ceaf9 AD |
720 | int r, i; |
721 | ||
722 | if (!bo->pin_count) { | |
a7d64de6 | 723 | dev_warn(adev->dev, "%p unpin not necessary\n", bo); |
d38ceaf9 AD |
724 | return 0; |
725 | } | |
726 | bo->pin_count--; | |
727 | if (bo->pin_count) | |
728 | return 0; | |
729 | for (i = 0; i < bo->placement.num_placement; i++) { | |
730 | bo->placements[i].lpfn = 0; | |
731 | bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; | |
732 | } | |
733 | r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false); | |
6681c5eb | 734 | if (unlikely(r)) { |
a7d64de6 | 735 | dev_err(adev->dev, "%p validate failed for unpin\n", bo); |
6681c5eb | 736 | goto error; |
d38ceaf9 | 737 | } |
6681c5eb CK |
738 | |
739 | if (bo->tbo.mem.mem_type == TTM_PL_VRAM) { | |
a7d64de6 | 740 | adev->vram_pin_size -= amdgpu_bo_size(bo); |
6681c5eb | 741 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) |
a7d64de6 | 742 | adev->invisible_pin_size -= amdgpu_bo_size(bo); |
441f90ec | 743 | } else if (bo->tbo.mem.mem_type == TTM_PL_TT) { |
a7d64de6 | 744 | adev->gart_pin_size -= amdgpu_bo_size(bo); |
6681c5eb CK |
745 | } |
746 | ||
747 | error: | |
d38ceaf9 AD |
748 | return r; |
749 | } | |
750 | ||
751 | int amdgpu_bo_evict_vram(struct amdgpu_device *adev) | |
752 | { | |
753 | /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ | |
2f7d10b3 | 754 | if (0 && (adev->flags & AMD_IS_APU)) { |
d38ceaf9 AD |
755 | /* Useless to evict on IGP chips */ |
756 | return 0; | |
757 | } | |
758 | return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM); | |
759 | } | |
760 | ||
1f8628c7 AD |
761 | static const char *amdgpu_vram_names[] = { |
762 | "UNKNOWN", | |
763 | "GDDR1", | |
764 | "DDR2", | |
765 | "GDDR3", | |
766 | "GDDR4", | |
767 | "GDDR5", | |
768 | "HBM", | |
769 | "DDR3" | |
770 | }; | |
771 | ||
d38ceaf9 AD |
772 | int amdgpu_bo_init(struct amdgpu_device *adev) |
773 | { | |
7cf321d1 DA |
774 | /* reserve PAT memory space to WC for VRAM */ |
775 | arch_io_reserve_memtype_wc(adev->mc.aper_base, | |
776 | adev->mc.aper_size); | |
777 | ||
d38ceaf9 AD |
778 | /* Add an MTRR for the VRAM */ |
779 | adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base, | |
780 | adev->mc.aper_size); | |
781 | DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", | |
782 | adev->mc.mc_vram_size >> 20, | |
783 | (unsigned long long)adev->mc.aper_size >> 20); | |
1f8628c7 AD |
784 | DRM_INFO("RAM width %dbits %s\n", |
785 | adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]); | |
d38ceaf9 AD |
786 | return amdgpu_ttm_init(adev); |
787 | } | |
788 | ||
789 | void amdgpu_bo_fini(struct amdgpu_device *adev) | |
790 | { | |
791 | amdgpu_ttm_fini(adev); | |
792 | arch_phys_wc_del(adev->mc.vram_mtrr); | |
7cf321d1 | 793 | arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size); |
d38ceaf9 AD |
794 | } |
795 | ||
796 | int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, | |
797 | struct vm_area_struct *vma) | |
798 | { | |
799 | return ttm_fbdev_mmap(vma, &bo->tbo); | |
800 | } | |
801 | ||
802 | int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) | |
803 | { | |
9079ac76 MO |
804 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
805 | ||
806 | if (adev->family <= AMDGPU_FAMILY_CZ && | |
807 | AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) | |
d38ceaf9 | 808 | return -EINVAL; |
d38ceaf9 AD |
809 | |
810 | bo->tiling_flags = tiling_flags; | |
811 | return 0; | |
812 | } | |
813 | ||
814 | void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) | |
815 | { | |
816 | lockdep_assert_held(&bo->tbo.resv->lock.base); | |
817 | ||
818 | if (tiling_flags) | |
819 | *tiling_flags = bo->tiling_flags; | |
820 | } | |
821 | ||
822 | int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, | |
823 | uint32_t metadata_size, uint64_t flags) | |
824 | { | |
825 | void *buffer; | |
826 | ||
827 | if (!metadata_size) { | |
828 | if (bo->metadata_size) { | |
829 | kfree(bo->metadata); | |
0092d3ed | 830 | bo->metadata = NULL; |
d38ceaf9 AD |
831 | bo->metadata_size = 0; |
832 | } | |
833 | return 0; | |
834 | } | |
835 | ||
836 | if (metadata == NULL) | |
837 | return -EINVAL; | |
838 | ||
71affda5 | 839 | buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); |
d38ceaf9 AD |
840 | if (buffer == NULL) |
841 | return -ENOMEM; | |
842 | ||
d38ceaf9 AD |
843 | kfree(bo->metadata); |
844 | bo->metadata_flags = flags; | |
845 | bo->metadata = buffer; | |
846 | bo->metadata_size = metadata_size; | |
847 | ||
848 | return 0; | |
849 | } | |
850 | ||
851 | int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, | |
852 | size_t buffer_size, uint32_t *metadata_size, | |
853 | uint64_t *flags) | |
854 | { | |
855 | if (!buffer && !metadata_size) | |
856 | return -EINVAL; | |
857 | ||
858 | if (buffer) { | |
859 | if (buffer_size < bo->metadata_size) | |
860 | return -EINVAL; | |
861 | ||
862 | if (bo->metadata_size) | |
863 | memcpy(buffer, bo->metadata, bo->metadata_size); | |
864 | } | |
865 | ||
866 | if (metadata_size) | |
867 | *metadata_size = bo->metadata_size; | |
868 | if (flags) | |
869 | *flags = bo->metadata_flags; | |
870 | ||
871 | return 0; | |
872 | } | |
873 | ||
874 | void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, | |
66257db7 | 875 | bool evict, |
d38ceaf9 AD |
876 | struct ttm_mem_reg *new_mem) |
877 | { | |
a7d64de6 | 878 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
765e7fbf | 879 | struct amdgpu_bo *abo; |
15da301d | 880 | struct ttm_mem_reg *old_mem = &bo->mem; |
d38ceaf9 AD |
881 | |
882 | if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) | |
883 | return; | |
884 | ||
b82485fd | 885 | abo = ttm_to_amdgpu_bo(bo); |
3f3333f8 | 886 | amdgpu_vm_bo_invalidate(adev, abo, evict); |
d38ceaf9 | 887 | |
6375bbb4 CK |
888 | amdgpu_bo_kunmap(abo); |
889 | ||
661a7606 NH |
890 | /* remember the eviction */ |
891 | if (evict) | |
892 | atomic64_inc(&adev->num_evictions); | |
893 | ||
d38ceaf9 AD |
894 | /* update statistics */ |
895 | if (!new_mem) | |
896 | return; | |
897 | ||
898 | /* move_notify is called before move happens */ | |
765e7fbf | 899 | trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type); |
d38ceaf9 AD |
900 | } |
901 | ||
902 | int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) | |
903 | { | |
a7d64de6 | 904 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
5fb1941d | 905 | struct amdgpu_bo *abo; |
96cf8271 JB |
906 | unsigned long offset, size; |
907 | int r; | |
d38ceaf9 AD |
908 | |
909 | if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) | |
910 | return 0; | |
5fb1941d | 911 | |
b82485fd | 912 | abo = ttm_to_amdgpu_bo(bo); |
96cf8271 JB |
913 | |
914 | /* Remember that this BO was accessed by the CPU */ | |
915 | abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
916 | ||
5fb1941d CK |
917 | if (bo->mem.mem_type != TTM_PL_VRAM) |
918 | return 0; | |
919 | ||
920 | size = bo->mem.num_pages << PAGE_SHIFT; | |
921 | offset = bo->mem.start << PAGE_SHIFT; | |
9bbdcc0f | 922 | if ((offset + size) <= adev->mc.visible_vram_size) |
5fb1941d CK |
923 | return 0; |
924 | ||
104ece97 MD |
925 | /* Can't move a pinned BO to visible VRAM */ |
926 | if (abo->pin_count > 0) | |
927 | return -EINVAL; | |
928 | ||
5fb1941d | 929 | /* hurrah the memory is not visible ! */ |
68e2c5ff | 930 | atomic64_inc(&adev->num_vram_cpu_page_faults); |
41d9a6a7 JB |
931 | amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | |
932 | AMDGPU_GEM_DOMAIN_GTT); | |
933 | ||
934 | /* Avoid costly evictions; only set GTT as a busy placement */ | |
935 | abo->placement.num_busy_placement = 1; | |
936 | abo->placement.busy_placement = &abo->placements[1]; | |
937 | ||
5fb1941d | 938 | r = ttm_bo_validate(bo, &abo->placement, false, false); |
41d9a6a7 | 939 | if (unlikely(r != 0)) |
5fb1941d | 940 | return r; |
5fb1941d CK |
941 | |
942 | offset = bo->mem.start << PAGE_SHIFT; | |
943 | /* this should never happen */ | |
41d9a6a7 JB |
944 | if (bo->mem.mem_type == TTM_PL_VRAM && |
945 | (offset + size) > adev->mc.visible_vram_size) | |
5fb1941d CK |
946 | return -EINVAL; |
947 | ||
d38ceaf9 AD |
948 | return 0; |
949 | } | |
950 | ||
951 | /** | |
952 | * amdgpu_bo_fence - add fence to buffer object | |
953 | * | |
954 | * @bo: buffer object in question | |
955 | * @fence: fence to add | |
956 | * @shared: true if fence should be added shared | |
957 | * | |
958 | */ | |
f54d1867 | 959 | void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, |
d38ceaf9 AD |
960 | bool shared) |
961 | { | |
962 | struct reservation_object *resv = bo->tbo.resv; | |
963 | ||
964 | if (shared) | |
e40a3115 | 965 | reservation_object_add_shared_fence(resv, fence); |
d38ceaf9 | 966 | else |
e40a3115 | 967 | reservation_object_add_excl_fence(resv, fence); |
d38ceaf9 | 968 | } |
cdb7e8f2 CK |
969 | |
970 | /** | |
971 | * amdgpu_bo_gpu_offset - return GPU offset of bo | |
972 | * @bo: amdgpu object for which we query the offset | |
973 | * | |
974 | * Returns current GPU offset of the object. | |
975 | * | |
976 | * Note: object should either be pinned or reserved when calling this | |
977 | * function, it might be useful to add check for this for debugging. | |
978 | */ | |
979 | u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) | |
980 | { | |
981 | WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM); | |
c855e250 CK |
982 | WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT && |
983 | !amdgpu_ttm_is_bound(bo->tbo.ttm)); | |
cdb7e8f2 CK |
984 | WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) && |
985 | !bo->pin_count); | |
9702d40d | 986 | WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET); |
03f48dd5 CK |
987 | WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM && |
988 | !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); | |
cdb7e8f2 CK |
989 | |
990 | return bo->tbo.offset; | |
991 | } |