Commit | Line | Data |
---|---|---|
d38ceaf9 AD |
1 | /* |
2 | * Copyright 2009 Jerome Glisse. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sub license, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
20 | * | |
21 | * The above copyright notice and this permission notice (including the | |
22 | * next paragraph) shall be included in all copies or substantial portions | |
23 | * of the Software. | |
24 | * | |
25 | */ | |
26 | /* | |
27 | * Authors: | |
28 | * Jerome Glisse <glisse@freedesktop.org> | |
29 | * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> | |
30 | * Dave Airlie | |
31 | */ | |
32 | #include <linux/list.h> | |
33 | #include <linux/slab.h> | |
fdf2f6c5 | 34 | |
d38ceaf9 | 35 | #include <drm/amdgpu_drm.h> |
a187f17f | 36 | #include <drm/drm_cache.h> |
d38ceaf9 AD |
37 | #include "amdgpu.h" |
38 | #include "amdgpu_trace.h" | |
a46a2cd1 | 39 | #include "amdgpu_amdkfd.h" |
d38ceaf9 | 40 | |
6f4e8d6e SL |
41 | /** |
42 | * DOC: amdgpu_object | |
43 | * | |
44 | * This defines the interfaces to operate on an &amdgpu_bo buffer object which | |
45 | * represents memory used by driver (VRAM, system memory, etc.). The driver | |
46 | * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces | |
47 | * to create/destroy/set buffer object which are then managed by the kernel TTM | |
48 | * memory manager. | |
49 | * The interfaces are also used internally by kernel clients, including gfx, | |
50 | * uvd, etc. for kernel managed allocations used by the GPU. | |
51 | * | |
52 | */ | |
53 | ||
15e6b768 MD |
54 | /** |
55 | * amdgpu_bo_subtract_pin_size - Remove BO from pin_size accounting | |
56 | * | |
57 | * @bo: &amdgpu_bo buffer object | |
58 | * | |
59 | * This function is called when a BO stops being pinned, and updates the | |
60 | * &amdgpu_device pin_size values accordingly. | |
61 | */ | |
62 | static void amdgpu_bo_subtract_pin_size(struct amdgpu_bo *bo) | |
63 | { | |
64 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
65 | ||
66 | if (bo->tbo.mem.mem_type == TTM_PL_VRAM) { | |
67 | atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); | |
68 | atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), | |
69 | &adev->visible_pin_size); | |
70 | } else if (bo->tbo.mem.mem_type == TTM_PL_TT) { | |
71 | atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); | |
72 | } | |
73 | } | |
74 | ||
c704ab18 | 75 | static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) |
d38ceaf9 | 76 | { |
a7d64de6 | 77 | struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); |
b82485fd | 78 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); |
d38ceaf9 | 79 | |
456607d8 | 80 | if (bo->pin_count > 0) |
15e6b768 MD |
81 | amdgpu_bo_subtract_pin_size(bo); |
82 | ||
a46a2cd1 | 83 | if (bo->kfd_bo) |
611736d8 | 84 | amdgpu_amdkfd_unreserve_memory_limit(bo); |
a46a2cd1 | 85 | |
6375bbb4 | 86 | amdgpu_bo_kunmap(bo); |
d38ceaf9 | 87 | |
7f8fb919 CK |
88 | if (bo->gem_base.import_attach) |
89 | drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg); | |
d38ceaf9 | 90 | drm_gem_object_release(&bo->gem_base); |
36e499b2 | 91 | /* in case amdgpu_device_recover_vram got NULL of bo->parent */ |
0c4e7fa5 | 92 | if (!list_empty(&bo->shadow_list)) { |
a7d64de6 | 93 | mutex_lock(&adev->shadow_list_lock); |
0c4e7fa5 | 94 | list_del_init(&bo->shadow_list); |
a7d64de6 | 95 | mutex_unlock(&adev->shadow_list_lock); |
0c4e7fa5 | 96 | } |
36e499b2 | 97 | amdgpu_bo_unref(&bo->parent); |
98 | ||
d38ceaf9 AD |
99 | kfree(bo->metadata); |
100 | kfree(bo); | |
101 | } | |
102 | ||
6f4e8d6e | 103 | /** |
c704ab18 | 104 | * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo |
6f4e8d6e SL |
105 | * @bo: buffer object to be checked |
106 | * | |
107 | * Uses destroy function associated with the object to determine if this is | |
108 | * an &amdgpu_bo. | |
109 | * | |
2472e11b MD |
110 | * Returns: |
111 | * true if the object belongs to &amdgpu_bo, false if not. | |
6f4e8d6e | 112 | */ |
c704ab18 | 113 | bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) |
d38ceaf9 | 114 | { |
c704ab18 | 115 | if (bo->destroy == &amdgpu_bo_destroy) |
d38ceaf9 AD |
116 | return true; |
117 | return false; | |
118 | } | |
119 | ||
6f4e8d6e | 120 | /** |
c704ab18 | 121 | * amdgpu_bo_placement_from_domain - set buffer's placement |
6f4e8d6e SL |
122 | * @abo: &amdgpu_bo buffer object whose placement is to be set |
123 | * @domain: requested domain | |
124 | * | |
125 | * Sets buffer's placement according to requested domain and the buffer's | |
126 | * flags. | |
127 | */ | |
c704ab18 | 128 | void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) |
d38ceaf9 | 129 | { |
c09312a6 CK |
130 | struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); |
131 | struct ttm_placement *placement = &abo->placement; | |
132 | struct ttm_place *places = abo->placements; | |
133 | u64 flags = abo->flags; | |
6369f6f1 | 134 | u32 c = 0; |
7e5a547f | 135 | |
d38ceaf9 | 136 | if (domain & AMDGPU_GEM_DOMAIN_VRAM) { |
770d13b1 | 137 | unsigned visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; |
faceaf6a | 138 | |
faceaf6a | 139 | places[c].fpfn = 0; |
89bb5752 | 140 | places[c].lpfn = 0; |
faceaf6a | 141 | places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | |
7e5a547f | 142 | TTM_PL_FLAG_VRAM; |
89bb5752 | 143 | |
faceaf6a CK |
144 | if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) |
145 | places[c].lpfn = visible_pfn; | |
146 | else | |
147 | places[c].flags |= TTM_PL_FLAG_TOPDOWN; | |
89bb5752 CK |
148 | |
149 | if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) | |
150 | places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; | |
faceaf6a | 151 | c++; |
d38ceaf9 AD |
152 | } |
153 | ||
154 | if (domain & AMDGPU_GEM_DOMAIN_GTT) { | |
faceaf6a | 155 | places[c].fpfn = 0; |
3d5fe658 | 156 | places[c].lpfn = 0; |
faceaf6a CK |
157 | places[c].flags = TTM_PL_FLAG_TT; |
158 | if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
159 | places[c].flags |= TTM_PL_FLAG_WC | | |
160 | TTM_PL_FLAG_UNCACHED; | |
161 | else | |
162 | places[c].flags |= TTM_PL_FLAG_CACHED; | |
163 | c++; | |
d38ceaf9 AD |
164 | } |
165 | ||
166 | if (domain & AMDGPU_GEM_DOMAIN_CPU) { | |
faceaf6a CK |
167 | places[c].fpfn = 0; |
168 | places[c].lpfn = 0; | |
169 | places[c].flags = TTM_PL_FLAG_SYSTEM; | |
170 | if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
171 | places[c].flags |= TTM_PL_FLAG_WC | | |
172 | TTM_PL_FLAG_UNCACHED; | |
173 | else | |
174 | places[c].flags |= TTM_PL_FLAG_CACHED; | |
175 | c++; | |
d38ceaf9 AD |
176 | } |
177 | ||
178 | if (domain & AMDGPU_GEM_DOMAIN_GDS) { | |
faceaf6a CK |
179 | places[c].fpfn = 0; |
180 | places[c].lpfn = 0; | |
181 | places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS; | |
182 | c++; | |
d38ceaf9 | 183 | } |
faceaf6a | 184 | |
d38ceaf9 | 185 | if (domain & AMDGPU_GEM_DOMAIN_GWS) { |
faceaf6a CK |
186 | places[c].fpfn = 0; |
187 | places[c].lpfn = 0; | |
188 | places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS; | |
189 | c++; | |
d38ceaf9 | 190 | } |
faceaf6a | 191 | |
d38ceaf9 | 192 | if (domain & AMDGPU_GEM_DOMAIN_OA) { |
faceaf6a CK |
193 | places[c].fpfn = 0; |
194 | places[c].lpfn = 0; | |
195 | places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA; | |
196 | c++; | |
d38ceaf9 AD |
197 | } |
198 | ||
199 | if (!c) { | |
faceaf6a CK |
200 | places[c].fpfn = 0; |
201 | places[c].lpfn = 0; | |
202 | places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; | |
203 | c++; | |
d38ceaf9 | 204 | } |
faceaf6a | 205 | |
bf314ca3 CK |
206 | BUG_ON(c >= AMDGPU_BO_MAX_PLACEMENTS); |
207 | ||
7e5a547f | 208 | placement->num_placement = c; |
faceaf6a | 209 | placement->placement = places; |
d38ceaf9 | 210 | |
faceaf6a CK |
211 | placement->num_busy_placement = c; |
212 | placement->busy_placement = places; | |
d38ceaf9 AD |
213 | } |
214 | ||
7c204889 | 215 | /** |
9d903cbd | 216 | * amdgpu_bo_create_reserved - create reserved BO for kernel use |
7c204889 CK |
217 | * |
218 | * @adev: amdgpu device object | |
219 | * @size: size for the new BO | |
220 | * @align: alignment for the new BO | |
221 | * @domain: where to place it | |
64350f1b | 222 | * @bo_ptr: used to initialize BOs in structures |
7c204889 CK |
223 | * @gpu_addr: GPU addr of the pinned BO |
224 | * @cpu_addr: optional CPU address mapping | |
225 | * | |
9d903cbd CK |
226 | * Allocates and pins a BO for kernel internal use, and returns it still |
227 | * reserved. | |
7c204889 | 228 | * |
64350f1b AG |
229 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
230 | * | |
2472e11b MD |
231 | * Returns: |
232 | * 0 on success, negative error code otherwise. | |
7c204889 | 233 | */ |
9d903cbd CK |
234 | int amdgpu_bo_create_reserved(struct amdgpu_device *adev, |
235 | unsigned long size, int align, | |
236 | u32 domain, struct amdgpu_bo **bo_ptr, | |
237 | u64 *gpu_addr, void **cpu_addr) | |
7c204889 | 238 | { |
3216c6b7 | 239 | struct amdgpu_bo_param bp; |
53766e5a | 240 | bool free = false; |
7c204889 CK |
241 | int r; |
242 | ||
21a7e77f CK |
243 | if (!size) { |
244 | amdgpu_bo_unref(bo_ptr); | |
245 | return 0; | |
246 | } | |
247 | ||
3216c6b7 CZ |
248 | memset(&bp, 0, sizeof(bp)); |
249 | bp.size = size; | |
250 | bp.byte_align = align; | |
251 | bp.domain = domain; | |
252 | bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | | |
253 | AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; | |
254 | bp.type = ttm_bo_type_kernel; | |
255 | bp.resv = NULL; | |
256 | ||
53766e5a | 257 | if (!*bo_ptr) { |
3216c6b7 | 258 | r = amdgpu_bo_create(adev, &bp, bo_ptr); |
53766e5a CK |
259 | if (r) { |
260 | dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", | |
261 | r); | |
262 | return r; | |
263 | } | |
264 | free = true; | |
7c204889 CK |
265 | } |
266 | ||
267 | r = amdgpu_bo_reserve(*bo_ptr, false); | |
268 | if (r) { | |
269 | dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); | |
270 | goto error_free; | |
271 | } | |
272 | ||
7b7c6c81 | 273 | r = amdgpu_bo_pin(*bo_ptr, domain); |
7c204889 CK |
274 | if (r) { |
275 | dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); | |
276 | goto error_unreserve; | |
277 | } | |
bb812f1e JZ |
278 | |
279 | r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); | |
280 | if (r) { | |
281 | dev_err(adev->dev, "%p bind failed\n", *bo_ptr); | |
282 | goto error_unpin; | |
283 | } | |
284 | ||
7b7c6c81 JZ |
285 | if (gpu_addr) |
286 | *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); | |
7c204889 CK |
287 | |
288 | if (cpu_addr) { | |
289 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
290 | if (r) { | |
291 | dev_err(adev->dev, "(%d) kernel bo map failed\n", r); | |
dc407ee0 | 292 | goto error_unpin; |
7c204889 CK |
293 | } |
294 | } | |
295 | ||
7c204889 CK |
296 | return 0; |
297 | ||
bb812f1e JZ |
298 | error_unpin: |
299 | amdgpu_bo_unpin(*bo_ptr); | |
7c204889 CK |
300 | error_unreserve: |
301 | amdgpu_bo_unreserve(*bo_ptr); | |
302 | ||
303 | error_free: | |
53766e5a CK |
304 | if (free) |
305 | amdgpu_bo_unref(bo_ptr); | |
7c204889 CK |
306 | |
307 | return r; | |
308 | } | |
309 | ||
9d903cbd CK |
310 | /** |
311 | * amdgpu_bo_create_kernel - create BO for kernel use | |
312 | * | |
313 | * @adev: amdgpu device object | |
314 | * @size: size for the new BO | |
315 | * @align: alignment for the new BO | |
316 | * @domain: where to place it | |
64350f1b | 317 | * @bo_ptr: used to initialize BOs in structures |
9d903cbd CK |
318 | * @gpu_addr: GPU addr of the pinned BO |
319 | * @cpu_addr: optional CPU address mapping | |
320 | * | |
321 | * Allocates and pins a BO for kernel internal use. | |
322 | * | |
64350f1b AG |
323 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
324 | * | |
2472e11b MD |
325 | * Returns: |
326 | * 0 on success, negative error code otherwise. | |
9d903cbd CK |
327 | */ |
328 | int amdgpu_bo_create_kernel(struct amdgpu_device *adev, | |
329 | unsigned long size, int align, | |
330 | u32 domain, struct amdgpu_bo **bo_ptr, | |
331 | u64 *gpu_addr, void **cpu_addr) | |
332 | { | |
333 | int r; | |
334 | ||
335 | r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, | |
336 | gpu_addr, cpu_addr); | |
337 | ||
338 | if (r) | |
339 | return r; | |
340 | ||
ddaf5013 TSD |
341 | if (*bo_ptr) |
342 | amdgpu_bo_unreserve(*bo_ptr); | |
9d903cbd CK |
343 | |
344 | return 0; | |
345 | } | |
346 | ||
aa1d562e JZ |
347 | /** |
348 | * amdgpu_bo_free_kernel - free BO for kernel use | |
349 | * | |
350 | * @bo: amdgpu BO to free | |
2472e11b MD |
351 | * @gpu_addr: pointer to where the BO's GPU memory space address was stored |
352 | * @cpu_addr: pointer to where the BO's CPU memory space address was stored | |
aa1d562e JZ |
353 | * |
354 | * unmaps and unpin a BO for kernel internal use. | |
355 | */ | |
356 | void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, | |
357 | void **cpu_addr) | |
358 | { | |
359 | if (*bo == NULL) | |
360 | return; | |
361 | ||
f3aa745e | 362 | if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { |
aa1d562e JZ |
363 | if (cpu_addr) |
364 | amdgpu_bo_kunmap(*bo); | |
365 | ||
366 | amdgpu_bo_unpin(*bo); | |
367 | amdgpu_bo_unreserve(*bo); | |
368 | } | |
369 | amdgpu_bo_unref(bo); | |
370 | ||
371 | if (gpu_addr) | |
372 | *gpu_addr = 0; | |
373 | ||
374 | if (cpu_addr) | |
375 | *cpu_addr = NULL; | |
376 | } | |
377 | ||
79c63123 AG |
378 | /* Validate bo size is bit bigger then the request domain */ |
379 | static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, | |
380 | unsigned long size, u32 domain) | |
381 | { | |
382 | struct ttm_mem_type_manager *man = NULL; | |
383 | ||
384 | /* | |
385 | * If GTT is part of requested domains the check must succeed to | |
386 | * allow fall back to GTT | |
387 | */ | |
388 | if (domain & AMDGPU_GEM_DOMAIN_GTT) { | |
389 | man = &adev->mman.bdev.man[TTM_PL_TT]; | |
390 | ||
391 | if (size < (man->size << PAGE_SHIFT)) | |
392 | return true; | |
393 | else | |
394 | goto fail; | |
395 | } | |
396 | ||
397 | if (domain & AMDGPU_GEM_DOMAIN_VRAM) { | |
398 | man = &adev->mman.bdev.man[TTM_PL_VRAM]; | |
399 | ||
400 | if (size < (man->size << PAGE_SHIFT)) | |
401 | return true; | |
402 | else | |
403 | goto fail; | |
404 | } | |
405 | ||
406 | ||
407 | /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */ | |
408 | return true; | |
409 | ||
410 | fail: | |
299c776c MD |
411 | DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, |
412 | man->size << PAGE_SHIFT); | |
79c63123 AG |
413 | return false; |
414 | } | |
415 | ||
a906dbb1 CZ |
416 | static int amdgpu_bo_do_create(struct amdgpu_device *adev, |
417 | struct amdgpu_bo_param *bp, | |
c09312a6 | 418 | struct amdgpu_bo **bo_ptr) |
d38ceaf9 | 419 | { |
9251859a | 420 | struct ttm_operation_ctx ctx = { |
a906dbb1 | 421 | .interruptible = (bp->type != ttm_bo_type_kernel), |
9251859a | 422 | .no_wait_gpu = false, |
a906dbb1 | 423 | .resv = bp->resv, |
d330fca1 | 424 | .flags = TTM_OPT_FLAG_ALLOW_RES_EVICT |
9251859a | 425 | }; |
d38ceaf9 | 426 | struct amdgpu_bo *bo; |
a906dbb1 | 427 | unsigned long page_align, size = bp->size; |
d38ceaf9 AD |
428 | size_t acc_size; |
429 | int r; | |
430 | ||
fe57085a MO |
431 | /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ |
432 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { | |
433 | /* GWS and OA don't need any alignment. */ | |
434 | page_align = bp->byte_align; | |
77a2faa5 | 435 | size <<= PAGE_SHIFT; |
fe57085a MO |
436 | } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { |
437 | /* Both size and alignment must be a multiple of 4. */ | |
438 | page_align = ALIGN(bp->byte_align, 4); | |
439 | size = ALIGN(size, 4) << PAGE_SHIFT; | |
440 | } else { | |
441 | /* Memory should be aligned at least to a page size. */ | |
442 | page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; | |
77a2faa5 | 443 | size = ALIGN(size, PAGE_SIZE); |
fe57085a | 444 | } |
d38ceaf9 | 445 | |
a906dbb1 | 446 | if (!amdgpu_bo_validate_size(adev, size, bp->domain)) |
79c63123 AG |
447 | return -ENOMEM; |
448 | ||
d38ceaf9 AD |
449 | *bo_ptr = NULL; |
450 | ||
451 | acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, | |
452 | sizeof(struct amdgpu_bo)); | |
453 | ||
454 | bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); | |
455 | if (bo == NULL) | |
456 | return -ENOMEM; | |
c06cc6f7 | 457 | drm_gem_private_object_init(adev->ddev, &bo->gem_base, size); |
0c4e7fa5 | 458 | INIT_LIST_HEAD(&bo->shadow_list); |
646b9025 | 459 | bo->vm_bo = NULL; |
3f188453 | 460 | bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : |
aa2b2e28 | 461 | bp->domain; |
08082104 | 462 | bo->allowed_domains = bo->preferred_domains; |
a906dbb1 | 463 | if (bp->type != ttm_bo_type_kernel && |
08082104 CK |
464 | bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) |
465 | bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; | |
d38ceaf9 | 466 | |
a906dbb1 | 467 | bo->flags = bp->flags; |
a187f17f | 468 | |
a2e2f299 NH |
469 | #ifdef CONFIG_X86_32 |
470 | /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit | |
471 | * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 | |
472 | */ | |
473 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; | |
474 | #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) | |
475 | /* Don't try to enable write-combining when it can't work, or things | |
476 | * may be slow | |
477 | * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 | |
478 | */ | |
479 | ||
31bb90f1 | 480 | #ifndef CONFIG_COMPILE_TEST |
a2e2f299 NH |
481 | #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ |
482 | thanks to write-combining | |
31bb90f1 | 483 | #endif |
a2e2f299 NH |
484 | |
485 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
486 | DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " | |
487 | "better performance thanks to write-combining\n"); | |
488 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; | |
489 | #else | |
a187f17f OG |
490 | /* For architectures that don't support WC memory, |
491 | * mask out the WC flag from the BO | |
492 | */ | |
493 | if (!drm_arch_can_wc_memory()) | |
494 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; | |
a2e2f299 | 495 | #endif |
a187f17f | 496 | |
c09312a6 | 497 | bo->tbo.bdev = &adev->mman.bdev; |
94de7349 CK |
498 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | |
499 | AMDGPU_GEM_DOMAIN_GDS)) | |
500 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
501 | else | |
502 | amdgpu_bo_placement_from_domain(bo, bp->domain); | |
a50cb948 JZ |
503 | if (bp->type == ttm_bo_type_kernel) |
504 | bo->tbo.priority = 1; | |
08082104 | 505 | |
a906dbb1 | 506 | r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type, |
724daa4f | 507 | &bo->placement, page_align, &ctx, acc_size, |
c704ab18 | 508 | NULL, bp->resv, &amdgpu_bo_destroy); |
08082104 | 509 | if (unlikely(r != 0)) |
a695e437 CK |
510 | return r; |
511 | ||
c8c5e569 | 512 | if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && |
00f06b24 | 513 | bo->tbo.mem.mem_type == TTM_PL_VRAM && |
770d13b1 | 514 | bo->tbo.mem.start < adev->gmc.visible_vram_size >> PAGE_SHIFT) |
6af046d2 CK |
515 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, |
516 | ctx.bytes_moved); | |
00f06b24 | 517 | else |
6af046d2 | 518 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); |
fad06127 | 519 | |
a906dbb1 | 520 | if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && |
4fea83ff | 521 | bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { |
f54d1867 | 522 | struct dma_fence *fence; |
4fea83ff | 523 | |
8febe617 | 524 | r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence); |
c3af1258 CK |
525 | if (unlikely(r)) |
526 | goto fail_unreserve; | |
527 | ||
4fea83ff | 528 | amdgpu_bo_fence(bo, fence, false); |
f54d1867 CW |
529 | dma_fence_put(bo->tbo.moving); |
530 | bo->tbo.moving = dma_fence_get(fence); | |
531 | dma_fence_put(fence); | |
4fea83ff | 532 | } |
a906dbb1 | 533 | if (!bp->resv) |
59c66c91 | 534 | amdgpu_bo_unreserve(bo); |
d38ceaf9 AD |
535 | *bo_ptr = bo; |
536 | ||
537 | trace_amdgpu_bo_create(bo); | |
538 | ||
96cf8271 | 539 | /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ |
a906dbb1 | 540 | if (bp->type == ttm_bo_type_device) |
96cf8271 JB |
541 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; |
542 | ||
d38ceaf9 | 543 | return 0; |
4fea83ff FC |
544 | |
545 | fail_unreserve: | |
a906dbb1 | 546 | if (!bp->resv) |
f1543f58 | 547 | ww_mutex_unlock(&bo->tbo.resv->lock); |
4fea83ff FC |
548 | amdgpu_bo_unref(&bo); |
549 | return r; | |
d38ceaf9 AD |
550 | } |
551 | ||
e7893c4b | 552 | static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, |
7fcb0657 | 553 | unsigned long size, |
e7893c4b CZ |
554 | struct amdgpu_bo *bo) |
555 | { | |
3216c6b7 | 556 | struct amdgpu_bo_param bp; |
e7893c4b CZ |
557 | int r; |
558 | ||
559 | if (bo->shadow) | |
560 | return 0; | |
561 | ||
3216c6b7 CZ |
562 | memset(&bp, 0, sizeof(bp)); |
563 | bp.size = size; | |
3216c6b7 CZ |
564 | bp.domain = AMDGPU_GEM_DOMAIN_GTT; |
565 | bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC | | |
566 | AMDGPU_GEM_CREATE_SHADOW; | |
567 | bp.type = ttm_bo_type_kernel; | |
568 | bp.resv = bo->tbo.resv; | |
569 | ||
a906dbb1 | 570 | r = amdgpu_bo_do_create(adev, &bp, &bo->shadow); |
0c4e7fa5 | 571 | if (!r) { |
e7893c4b | 572 | bo->shadow->parent = amdgpu_bo_ref(bo); |
0c4e7fa5 | 573 | mutex_lock(&adev->shadow_list_lock); |
403009bf | 574 | list_add_tail(&bo->shadow->shadow_list, &adev->shadow_list); |
0c4e7fa5 CZ |
575 | mutex_unlock(&adev->shadow_list_lock); |
576 | } | |
e7893c4b CZ |
577 | |
578 | return r; | |
579 | } | |
580 | ||
6f4e8d6e SL |
581 | /** |
582 | * amdgpu_bo_create - create an &amdgpu_bo buffer object | |
583 | * @adev: amdgpu device object | |
584 | * @bp: parameters to be used for the buffer object | |
585 | * @bo_ptr: pointer to the buffer object pointer | |
586 | * | |
587 | * Creates an &amdgpu_bo buffer object; and if requested, also creates a | |
588 | * shadow object. | |
589 | * Shadow object is used to backup the original buffer object, and is always | |
590 | * in GTT. | |
591 | * | |
2472e11b MD |
592 | * Returns: |
593 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 594 | */ |
3216c6b7 CZ |
595 | int amdgpu_bo_create(struct amdgpu_device *adev, |
596 | struct amdgpu_bo_param *bp, | |
72d7668b | 597 | struct amdgpu_bo **bo_ptr) |
7e5a547f | 598 | { |
3216c6b7 | 599 | u64 flags = bp->flags; |
e7893c4b | 600 | int r; |
7e5a547f | 601 | |
3216c6b7 CZ |
602 | bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW; |
603 | r = amdgpu_bo_do_create(adev, bp, bo_ptr); | |
e7893c4b CZ |
604 | if (r) |
605 | return r; | |
606 | ||
16688677 | 607 | if ((flags & AMDGPU_GEM_CREATE_SHADOW) && !(adev->flags & AMD_IS_APU)) { |
3216c6b7 | 608 | if (!bp->resv) |
cf273a59 CK |
609 | WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv, |
610 | NULL)); | |
36ea83d1 | 611 | |
7fcb0657 | 612 | r = amdgpu_bo_create_shadow(adev, bp->size, *bo_ptr); |
36ea83d1 | 613 | |
3216c6b7 | 614 | if (!bp->resv) |
cf273a59 | 615 | reservation_object_unlock((*bo_ptr)->tbo.resv); |
36ea83d1 | 616 | |
e7893c4b CZ |
617 | if (r) |
618 | amdgpu_bo_unref(bo_ptr); | |
619 | } | |
620 | ||
621 | return r; | |
7e5a547f CZ |
622 | } |
623 | ||
6f4e8d6e SL |
624 | /** |
625 | * amdgpu_bo_validate - validate an &amdgpu_bo buffer object | |
626 | * @bo: pointer to the buffer object | |
627 | * | |
628 | * Sets placement according to domain; and changes placement and caching | |
629 | * policy of the buffer object according to the placement. | |
630 | * This is used for validating shadow bos. It calls ttm_bo_validate() to | |
631 | * make sure the buffer is resident where it needs to be. | |
632 | * | |
2472e11b MD |
633 | * Returns: |
634 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 635 | */ |
82521316 RH |
636 | int amdgpu_bo_validate(struct amdgpu_bo *bo) |
637 | { | |
19be5570 | 638 | struct ttm_operation_ctx ctx = { false, false }; |
82521316 RH |
639 | uint32_t domain; |
640 | int r; | |
641 | ||
642 | if (bo->pin_count) | |
643 | return 0; | |
644 | ||
6d7d9c5a | 645 | domain = bo->preferred_domains; |
82521316 RH |
646 | |
647 | retry: | |
c704ab18 | 648 | amdgpu_bo_placement_from_domain(bo, domain); |
19be5570 | 649 | r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
82521316 RH |
650 | if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { |
651 | domain = bo->allowed_domains; | |
652 | goto retry; | |
653 | } | |
654 | ||
655 | return r; | |
656 | } | |
657 | ||
6f4e8d6e | 658 | /** |
403009bf CK |
659 | * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow |
660 | * | |
661 | * @shadow: &amdgpu_bo shadow to be restored | |
6f4e8d6e | 662 | * @fence: dma_fence associated with the operation |
6f4e8d6e SL |
663 | * |
664 | * Copies a buffer object's shadow content back to the object. | |
665 | * This is used for recovering a buffer from its shadow in case of a gpu | |
666 | * reset where vram context may be lost. | |
667 | * | |
2472e11b MD |
668 | * Returns: |
669 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 670 | */ |
403009bf | 671 | int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence) |
20f4eff1 CZ |
672 | |
673 | { | |
403009bf CK |
674 | struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev); |
675 | struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; | |
676 | uint64_t shadow_addr, parent_addr; | |
20f4eff1 | 677 | |
403009bf CK |
678 | shadow_addr = amdgpu_bo_gpu_offset(shadow); |
679 | parent_addr = amdgpu_bo_gpu_offset(shadow->parent); | |
20f4eff1 | 680 | |
403009bf CK |
681 | return amdgpu_copy_buffer(ring, shadow_addr, parent_addr, |
682 | amdgpu_bo_size(shadow), NULL, fence, | |
683 | true, false); | |
20f4eff1 CZ |
684 | } |
685 | ||
6f4e8d6e SL |
686 | /** |
687 | * amdgpu_bo_kmap - map an &amdgpu_bo buffer object | |
688 | * @bo: &amdgpu_bo buffer object to be mapped | |
689 | * @ptr: kernel virtual address to be returned | |
690 | * | |
691 | * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls | |
692 | * amdgpu_bo_kptr() to get the kernel virtual address. | |
693 | * | |
2472e11b MD |
694 | * Returns: |
695 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 696 | */ |
d38ceaf9 AD |
697 | int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) |
698 | { | |
f5e1c740 | 699 | void *kptr; |
587f3c70 | 700 | long r; |
d38ceaf9 | 701 | |
271c8125 CK |
702 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) |
703 | return -EPERM; | |
704 | ||
f5e1c740 CK |
705 | kptr = amdgpu_bo_kptr(bo); |
706 | if (kptr) { | |
707 | if (ptr) | |
708 | *ptr = kptr; | |
d38ceaf9 AD |
709 | return 0; |
710 | } | |
587f3c70 CK |
711 | |
712 | r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false, | |
713 | MAX_SCHEDULE_TIMEOUT); | |
714 | if (r < 0) | |
715 | return r; | |
716 | ||
d38ceaf9 | 717 | r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); |
587f3c70 | 718 | if (r) |
d38ceaf9 | 719 | return r; |
587f3c70 | 720 | |
587f3c70 | 721 | if (ptr) |
f5e1c740 | 722 | *ptr = amdgpu_bo_kptr(bo); |
587f3c70 | 723 | |
d38ceaf9 AD |
724 | return 0; |
725 | } | |
726 | ||
6f4e8d6e SL |
727 | /** |
728 | * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object | |
729 | * @bo: &amdgpu_bo buffer object | |
730 | * | |
731 | * Calls ttm_kmap_obj_virtual() to get the kernel virtual address | |
732 | * | |
2472e11b MD |
733 | * Returns: |
734 | * the virtual address of a buffer object area. | |
6f4e8d6e | 735 | */ |
f5e1c740 CK |
736 | void *amdgpu_bo_kptr(struct amdgpu_bo *bo) |
737 | { | |
738 | bool is_iomem; | |
739 | ||
740 | return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); | |
741 | } | |
742 | ||
6f4e8d6e SL |
743 | /** |
744 | * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object | |
745 | * @bo: &amdgpu_bo buffer object to be unmapped | |
746 | * | |
747 | * Unmaps a kernel map set up by amdgpu_bo_kmap(). | |
748 | */ | |
d38ceaf9 AD |
749 | void amdgpu_bo_kunmap(struct amdgpu_bo *bo) |
750 | { | |
f5e1c740 CK |
751 | if (bo->kmap.bo) |
752 | ttm_bo_kunmap(&bo->kmap); | |
d38ceaf9 AD |
753 | } |
754 | ||
6f4e8d6e SL |
755 | /** |
756 | * amdgpu_bo_ref - reference an &amdgpu_bo buffer object | |
757 | * @bo: &amdgpu_bo buffer object | |
758 | * | |
759 | * References the contained &ttm_buffer_object. | |
760 | * | |
2472e11b MD |
761 | * Returns: |
762 | * a refcounted pointer to the &amdgpu_bo buffer object. | |
6f4e8d6e | 763 | */ |
d38ceaf9 AD |
764 | struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) |
765 | { | |
766 | if (bo == NULL) | |
767 | return NULL; | |
768 | ||
71d5ef11 | 769 | ttm_bo_get(&bo->tbo); |
d38ceaf9 AD |
770 | return bo; |
771 | } | |
772 | ||
6f4e8d6e SL |
773 | /** |
774 | * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object | |
775 | * @bo: &amdgpu_bo buffer object | |
776 | * | |
777 | * Unreferences the contained &ttm_buffer_object and clear the pointer | |
778 | */ | |
d38ceaf9 AD |
779 | void amdgpu_bo_unref(struct amdgpu_bo **bo) |
780 | { | |
781 | struct ttm_buffer_object *tbo; | |
782 | ||
783 | if ((*bo) == NULL) | |
784 | return; | |
785 | ||
786 | tbo = &((*bo)->tbo); | |
fea872b2 TZ |
787 | ttm_bo_put(tbo); |
788 | *bo = NULL; | |
d38ceaf9 AD |
789 | } |
790 | ||
6f4e8d6e SL |
791 | /** |
792 | * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object | |
793 | * @bo: &amdgpu_bo buffer object to be pinned | |
794 | * @domain: domain to be pinned to | |
795 | * @min_offset: the start of requested address range | |
796 | * @max_offset: the end of requested address range | |
6f4e8d6e SL |
797 | * |
798 | * Pins the buffer object according to requested domain and address range. If | |
799 | * the memory is unbound gart memory, binds the pages into gart table. Adjusts | |
800 | * pin_count and pin_size accordingly. | |
801 | * | |
802 | * Pinning means to lock pages in memory along with keeping them at a fixed | |
803 | * offset. It is required when a buffer can not be moved, for example, when | |
804 | * a display buffer is being scanned out. | |
805 | * | |
806 | * Compared with amdgpu_bo_pin(), this function gives more flexibility on | |
807 | * where to pin a buffer if there are specific restrictions on where a buffer | |
808 | * must be located. | |
809 | * | |
2472e11b MD |
810 | * Returns: |
811 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 812 | */ |
7e5a547f | 813 | int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, |
7b7c6c81 | 814 | u64 min_offset, u64 max_offset) |
d38ceaf9 | 815 | { |
a7d64de6 | 816 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
19be5570 | 817 | struct ttm_operation_ctx ctx = { false, false }; |
d38ceaf9 AD |
818 | int r, i; |
819 | ||
cc325d19 | 820 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) |
d38ceaf9 AD |
821 | return -EPERM; |
822 | ||
7e5a547f CZ |
823 | if (WARN_ON_ONCE(min_offset > max_offset)) |
824 | return -EINVAL; | |
825 | ||
803d89ad | 826 | /* A shared bo cannot be migrated to VRAM */ |
9b3f217f SL |
827 | if (bo->prime_shared_count) { |
828 | if (domain & AMDGPU_GEM_DOMAIN_GTT) | |
829 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
830 | else | |
831 | return -EINVAL; | |
832 | } | |
803d89ad | 833 | |
6c8d74ca SL |
834 | /* This assumes only APU display buffers are pinned with (VRAM|GTT). |
835 | * See function amdgpu_display_supported_domains() | |
836 | */ | |
84b74608 | 837 | domain = amdgpu_bo_get_preferred_pin_domain(adev, domain); |
6c8d74ca | 838 | |
d38ceaf9 | 839 | if (bo->pin_count) { |
408778e8 FC |
840 | uint32_t mem_type = bo->tbo.mem.mem_type; |
841 | ||
f5318959 | 842 | if (!(domain & amdgpu_mem_type_to_domain(mem_type))) |
408778e8 FC |
843 | return -EINVAL; |
844 | ||
d38ceaf9 | 845 | bo->pin_count++; |
d38ceaf9 AD |
846 | |
847 | if (max_offset != 0) { | |
27798e07 | 848 | u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset; |
d38ceaf9 AD |
849 | WARN_ON_ONCE(max_offset < |
850 | (amdgpu_bo_gpu_offset(bo) - domain_start)); | |
851 | } | |
852 | ||
853 | return 0; | |
854 | } | |
03f48dd5 CK |
855 | |
856 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; | |
e9c7577c CK |
857 | /* force to pin into visible video ram */ |
858 | if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) | |
859 | bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
c704ab18 | 860 | amdgpu_bo_placement_from_domain(bo, domain); |
d38ceaf9 | 861 | for (i = 0; i < bo->placement.num_placement; i++) { |
e9c7577c CK |
862 | unsigned fpfn, lpfn; |
863 | ||
864 | fpfn = min_offset >> PAGE_SHIFT; | |
865 | lpfn = max_offset >> PAGE_SHIFT; | |
866 | ||
7e5a547f CZ |
867 | if (fpfn > bo->placements[i].fpfn) |
868 | bo->placements[i].fpfn = fpfn; | |
78d0e182 CK |
869 | if (!bo->placements[i].lpfn || |
870 | (lpfn && lpfn < bo->placements[i].lpfn)) | |
7e5a547f | 871 | bo->placements[i].lpfn = lpfn; |
d38ceaf9 AD |
872 | bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; |
873 | } | |
874 | ||
19be5570 | 875 | r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
6681c5eb | 876 | if (unlikely(r)) { |
a7d64de6 | 877 | dev_err(adev->dev, "%p pin failed\n", bo); |
6681c5eb CK |
878 | goto error; |
879 | } | |
880 | ||
881 | bo->pin_count = 1; | |
5e91fb57 CK |
882 | |
883 | domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); | |
6681c5eb | 884 | if (domain == AMDGPU_GEM_DOMAIN_VRAM) { |
a5ccfe5c MD |
885 | atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); |
886 | atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), | |
887 | &adev->visible_pin_size); | |
32ab75f0 | 888 | } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { |
a5ccfe5c | 889 | atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); |
d38ceaf9 | 890 | } |
6681c5eb CK |
891 | |
892 | error: | |
d38ceaf9 AD |
893 | return r; |
894 | } | |
895 | ||
6f4e8d6e SL |
896 | /** |
897 | * amdgpu_bo_pin - pin an &amdgpu_bo buffer object | |
898 | * @bo: &amdgpu_bo buffer object to be pinned | |
899 | * @domain: domain to be pinned to | |
6f4e8d6e SL |
900 | * |
901 | * A simple wrapper to amdgpu_bo_pin_restricted(). | |
902 | * Provides a simpler API for buffers that do not have any strict restrictions | |
903 | * on where a buffer must be located. | |
904 | * | |
2472e11b MD |
905 | * Returns: |
906 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 907 | */ |
7b7c6c81 | 908 | int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) |
d38ceaf9 | 909 | { |
7b7c6c81 | 910 | return amdgpu_bo_pin_restricted(bo, domain, 0, 0); |
d38ceaf9 AD |
911 | } |
912 | ||
6f4e8d6e SL |
913 | /** |
914 | * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object | |
915 | * @bo: &amdgpu_bo buffer object to be unpinned | |
916 | * | |
917 | * Decreases the pin_count, and clears the flags if pin_count reaches 0. | |
918 | * Changes placement and pin size accordingly. | |
919 | * | |
2472e11b MD |
920 | * Returns: |
921 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 922 | */ |
d38ceaf9 AD |
923 | int amdgpu_bo_unpin(struct amdgpu_bo *bo) |
924 | { | |
a7d64de6 | 925 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
19be5570 | 926 | struct ttm_operation_ctx ctx = { false, false }; |
d38ceaf9 AD |
927 | int r, i; |
928 | ||
a3a0ebd1 | 929 | if (WARN_ON_ONCE(!bo->pin_count)) { |
a7d64de6 | 930 | dev_warn(adev->dev, "%p unpin not necessary\n", bo); |
d38ceaf9 AD |
931 | return 0; |
932 | } | |
933 | bo->pin_count--; | |
934 | if (bo->pin_count) | |
935 | return 0; | |
6681c5eb | 936 | |
15e6b768 | 937 | amdgpu_bo_subtract_pin_size(bo); |
6681c5eb | 938 | |
34d6d599 MD |
939 | for (i = 0; i < bo->placement.num_placement; i++) { |
940 | bo->placements[i].lpfn = 0; | |
941 | bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; | |
942 | } | |
943 | r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
944 | if (unlikely(r)) | |
945 | dev_err(adev->dev, "%p validate failed for unpin\n", bo); | |
946 | ||
d38ceaf9 AD |
947 | return r; |
948 | } | |
949 | ||
6f4e8d6e SL |
950 | /** |
951 | * amdgpu_bo_evict_vram - evict VRAM buffers | |
952 | * @adev: amdgpu device object | |
953 | * | |
954 | * Evicts all VRAM buffers on the lru list of the memory type. | |
955 | * Mainly used for evicting vram at suspend time. | |
956 | * | |
2472e11b MD |
957 | * Returns: |
958 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 959 | */ |
d38ceaf9 AD |
960 | int amdgpu_bo_evict_vram(struct amdgpu_device *adev) |
961 | { | |
962 | /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ | |
2c1e9bca PM |
963 | #ifndef CONFIG_HIBERNATION |
964 | if (adev->flags & AMD_IS_APU) { | |
d38ceaf9 AD |
965 | /* Useless to evict on IGP chips */ |
966 | return 0; | |
967 | } | |
2c1e9bca | 968 | #endif |
d38ceaf9 AD |
969 | return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM); |
970 | } | |
971 | ||
1f8628c7 AD |
972 | static const char *amdgpu_vram_names[] = { |
973 | "UNKNOWN", | |
974 | "GDDR1", | |
975 | "DDR2", | |
976 | "GDDR3", | |
977 | "GDDR4", | |
978 | "GDDR5", | |
979 | "HBM", | |
bc227cfa TSD |
980 | "DDR3", |
981 | "DDR4", | |
5228fe30 | 982 | "GDDR6", |
1f8628c7 AD |
983 | }; |
984 | ||
6f4e8d6e SL |
985 | /** |
986 | * amdgpu_bo_init - initialize memory manager | |
987 | * @adev: amdgpu device object | |
988 | * | |
989 | * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. | |
990 | * | |
2472e11b MD |
991 | * Returns: |
992 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 993 | */ |
d38ceaf9 AD |
994 | int amdgpu_bo_init(struct amdgpu_device *adev) |
995 | { | |
7cf321d1 | 996 | /* reserve PAT memory space to WC for VRAM */ |
770d13b1 CK |
997 | arch_io_reserve_memtype_wc(adev->gmc.aper_base, |
998 | adev->gmc.aper_size); | |
7cf321d1 | 999 | |
d38ceaf9 | 1000 | /* Add an MTRR for the VRAM */ |
770d13b1 CK |
1001 | adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, |
1002 | adev->gmc.aper_size); | |
d38ceaf9 | 1003 | DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", |
770d13b1 CK |
1004 | adev->gmc.mc_vram_size >> 20, |
1005 | (unsigned long long)adev->gmc.aper_size >> 20); | |
1f8628c7 | 1006 | DRM_INFO("RAM width %dbits %s\n", |
770d13b1 | 1007 | adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); |
d38ceaf9 AD |
1008 | return amdgpu_ttm_init(adev); |
1009 | } | |
1010 | ||
6f4e8d6e SL |
1011 | /** |
1012 | * amdgpu_bo_late_init - late init | |
1013 | * @adev: amdgpu device object | |
1014 | * | |
1015 | * Calls amdgpu_ttm_late_init() to free resources used earlier during | |
1016 | * initialization. | |
1017 | * | |
2472e11b MD |
1018 | * Returns: |
1019 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1020 | */ |
6f752ec2 AG |
1021 | int amdgpu_bo_late_init(struct amdgpu_device *adev) |
1022 | { | |
1023 | amdgpu_ttm_late_init(adev); | |
1024 | ||
1025 | return 0; | |
1026 | } | |
1027 | ||
6f4e8d6e SL |
1028 | /** |
1029 | * amdgpu_bo_fini - tear down memory manager | |
1030 | * @adev: amdgpu device object | |
1031 | * | |
1032 | * Reverses amdgpu_bo_init() to tear down memory manager. | |
1033 | */ | |
d38ceaf9 AD |
1034 | void amdgpu_bo_fini(struct amdgpu_device *adev) |
1035 | { | |
1036 | amdgpu_ttm_fini(adev); | |
770d13b1 CK |
1037 | arch_phys_wc_del(adev->gmc.vram_mtrr); |
1038 | arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); | |
d38ceaf9 AD |
1039 | } |
1040 | ||
6f4e8d6e SL |
1041 | /** |
1042 | * amdgpu_bo_fbdev_mmap - mmap fbdev memory | |
1043 | * @bo: &amdgpu_bo buffer object | |
1044 | * @vma: vma as input from the fbdev mmap method | |
1045 | * | |
1046 | * Calls ttm_fbdev_mmap() to mmap fbdev memory if it is backed by a bo. | |
1047 | * | |
2472e11b MD |
1048 | * Returns: |
1049 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1050 | */ |
d38ceaf9 AD |
1051 | int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, |
1052 | struct vm_area_struct *vma) | |
1053 | { | |
1054 | return ttm_fbdev_mmap(vma, &bo->tbo); | |
1055 | } | |
1056 | ||
6f4e8d6e SL |
1057 | /** |
1058 | * amdgpu_bo_set_tiling_flags - set tiling flags | |
1059 | * @bo: &amdgpu_bo buffer object | |
1060 | * @tiling_flags: new flags | |
1061 | * | |
1062 | * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or | |
1063 | * kernel driver to set the tiling flags on a buffer. | |
1064 | * | |
2472e11b MD |
1065 | * Returns: |
1066 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1067 | */ |
d38ceaf9 AD |
1068 | int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) |
1069 | { | |
9079ac76 MO |
1070 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
1071 | ||
1072 | if (adev->family <= AMDGPU_FAMILY_CZ && | |
1073 | AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) | |
d38ceaf9 | 1074 | return -EINVAL; |
d38ceaf9 AD |
1075 | |
1076 | bo->tiling_flags = tiling_flags; | |
1077 | return 0; | |
1078 | } | |
1079 | ||
6f4e8d6e SL |
1080 | /** |
1081 | * amdgpu_bo_get_tiling_flags - get tiling flags | |
1082 | * @bo: &amdgpu_bo buffer object | |
1083 | * @tiling_flags: returned flags | |
1084 | * | |
1085 | * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to | |
1086 | * set the tiling flags on a buffer. | |
1087 | */ | |
d38ceaf9 AD |
1088 | void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) |
1089 | { | |
1090 | lockdep_assert_held(&bo->tbo.resv->lock.base); | |
1091 | ||
1092 | if (tiling_flags) | |
1093 | *tiling_flags = bo->tiling_flags; | |
1094 | } | |
1095 | ||
6f4e8d6e SL |
1096 | /** |
1097 | * amdgpu_bo_set_metadata - set metadata | |
1098 | * @bo: &amdgpu_bo buffer object | |
1099 | * @metadata: new metadata | |
1100 | * @metadata_size: size of the new metadata | |
1101 | * @flags: flags of the new metadata | |
1102 | * | |
1103 | * Sets buffer object's metadata, its size and flags. | |
1104 | * Used via GEM ioctl. | |
1105 | * | |
2472e11b MD |
1106 | * Returns: |
1107 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1108 | */ |
d38ceaf9 AD |
1109 | int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, |
1110 | uint32_t metadata_size, uint64_t flags) | |
1111 | { | |
1112 | void *buffer; | |
1113 | ||
1114 | if (!metadata_size) { | |
1115 | if (bo->metadata_size) { | |
1116 | kfree(bo->metadata); | |
0092d3ed | 1117 | bo->metadata = NULL; |
d38ceaf9 AD |
1118 | bo->metadata_size = 0; |
1119 | } | |
1120 | return 0; | |
1121 | } | |
1122 | ||
1123 | if (metadata == NULL) | |
1124 | return -EINVAL; | |
1125 | ||
71affda5 | 1126 | buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); |
d38ceaf9 AD |
1127 | if (buffer == NULL) |
1128 | return -ENOMEM; | |
1129 | ||
d38ceaf9 AD |
1130 | kfree(bo->metadata); |
1131 | bo->metadata_flags = flags; | |
1132 | bo->metadata = buffer; | |
1133 | bo->metadata_size = metadata_size; | |
1134 | ||
1135 | return 0; | |
1136 | } | |
1137 | ||
6f4e8d6e SL |
1138 | /** |
1139 | * amdgpu_bo_get_metadata - get metadata | |
1140 | * @bo: &amdgpu_bo buffer object | |
1141 | * @buffer: returned metadata | |
1142 | * @buffer_size: size of the buffer | |
1143 | * @metadata_size: size of the returned metadata | |
1144 | * @flags: flags of the returned metadata | |
1145 | * | |
1146 | * Gets buffer object's metadata, its size and flags. buffer_size shall not be | |
1147 | * less than metadata_size. | |
1148 | * Used via GEM ioctl. | |
1149 | * | |
2472e11b MD |
1150 | * Returns: |
1151 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1152 | */ |
d38ceaf9 AD |
1153 | int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, |
1154 | size_t buffer_size, uint32_t *metadata_size, | |
1155 | uint64_t *flags) | |
1156 | { | |
1157 | if (!buffer && !metadata_size) | |
1158 | return -EINVAL; | |
1159 | ||
1160 | if (buffer) { | |
1161 | if (buffer_size < bo->metadata_size) | |
1162 | return -EINVAL; | |
1163 | ||
1164 | if (bo->metadata_size) | |
1165 | memcpy(buffer, bo->metadata, bo->metadata_size); | |
1166 | } | |
1167 | ||
1168 | if (metadata_size) | |
1169 | *metadata_size = bo->metadata_size; | |
1170 | if (flags) | |
1171 | *flags = bo->metadata_flags; | |
1172 | ||
1173 | return 0; | |
1174 | } | |
1175 | ||
6f4e8d6e SL |
1176 | /** |
1177 | * amdgpu_bo_move_notify - notification about a memory move | |
1178 | * @bo: pointer to a buffer object | |
1179 | * @evict: if this move is evicting the buffer from the graphics address space | |
1180 | * @new_mem: new information of the bufer object | |
1181 | * | |
1182 | * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs | |
1183 | * bookkeeping. | |
1184 | * TTM driver callback which is called when ttm moves a buffer. | |
1185 | */ | |
d38ceaf9 | 1186 | void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, |
66257db7 | 1187 | bool evict, |
d38ceaf9 AD |
1188 | struct ttm_mem_reg *new_mem) |
1189 | { | |
a7d64de6 | 1190 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
765e7fbf | 1191 | struct amdgpu_bo *abo; |
15da301d | 1192 | struct ttm_mem_reg *old_mem = &bo->mem; |
d38ceaf9 | 1193 | |
c704ab18 | 1194 | if (!amdgpu_bo_is_amdgpu_bo(bo)) |
d38ceaf9 AD |
1195 | return; |
1196 | ||
b82485fd | 1197 | abo = ttm_to_amdgpu_bo(bo); |
3f3333f8 | 1198 | amdgpu_vm_bo_invalidate(adev, abo, evict); |
d38ceaf9 | 1199 | |
6375bbb4 CK |
1200 | amdgpu_bo_kunmap(abo); |
1201 | ||
661a7606 NH |
1202 | /* remember the eviction */ |
1203 | if (evict) | |
1204 | atomic64_inc(&adev->num_evictions); | |
1205 | ||
d38ceaf9 AD |
1206 | /* update statistics */ |
1207 | if (!new_mem) | |
1208 | return; | |
1209 | ||
1210 | /* move_notify is called before move happens */ | |
c704ab18 | 1211 | trace_amdgpu_bo_move(abo, new_mem->mem_type, old_mem->mem_type); |
d38ceaf9 AD |
1212 | } |
1213 | ||
6f4e8d6e SL |
1214 | /** |
1215 | * amdgpu_bo_fault_reserve_notify - notification about a memory fault | |
1216 | * @bo: pointer to a buffer object | |
1217 | * | |
1218 | * Notifies the driver we are taking a fault on this BO and have reserved it, | |
1219 | * also performs bookkeeping. | |
1220 | * TTM driver callback for dealing with vm faults. | |
1221 | * | |
2472e11b MD |
1222 | * Returns: |
1223 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1224 | */ |
d38ceaf9 AD |
1225 | int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) |
1226 | { | |
a7d64de6 | 1227 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
19be5570 | 1228 | struct ttm_operation_ctx ctx = { false, false }; |
5fb1941d | 1229 | struct amdgpu_bo *abo; |
96cf8271 JB |
1230 | unsigned long offset, size; |
1231 | int r; | |
d38ceaf9 | 1232 | |
c704ab18 | 1233 | if (!amdgpu_bo_is_amdgpu_bo(bo)) |
d38ceaf9 | 1234 | return 0; |
5fb1941d | 1235 | |
b82485fd | 1236 | abo = ttm_to_amdgpu_bo(bo); |
96cf8271 JB |
1237 | |
1238 | /* Remember that this BO was accessed by the CPU */ | |
1239 | abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
1240 | ||
5fb1941d CK |
1241 | if (bo->mem.mem_type != TTM_PL_VRAM) |
1242 | return 0; | |
1243 | ||
1244 | size = bo->mem.num_pages << PAGE_SHIFT; | |
1245 | offset = bo->mem.start << PAGE_SHIFT; | |
770d13b1 | 1246 | if ((offset + size) <= adev->gmc.visible_vram_size) |
5fb1941d CK |
1247 | return 0; |
1248 | ||
104ece97 MD |
1249 | /* Can't move a pinned BO to visible VRAM */ |
1250 | if (abo->pin_count > 0) | |
1251 | return -EINVAL; | |
1252 | ||
5fb1941d | 1253 | /* hurrah the memory is not visible ! */ |
68e2c5ff | 1254 | atomic64_inc(&adev->num_vram_cpu_page_faults); |
c704ab18 CK |
1255 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | |
1256 | AMDGPU_GEM_DOMAIN_GTT); | |
41d9a6a7 JB |
1257 | |
1258 | /* Avoid costly evictions; only set GTT as a busy placement */ | |
1259 | abo->placement.num_busy_placement = 1; | |
1260 | abo->placement.busy_placement = &abo->placements[1]; | |
1261 | ||
19be5570 | 1262 | r = ttm_bo_validate(bo, &abo->placement, &ctx); |
41d9a6a7 | 1263 | if (unlikely(r != 0)) |
5fb1941d | 1264 | return r; |
5fb1941d CK |
1265 | |
1266 | offset = bo->mem.start << PAGE_SHIFT; | |
1267 | /* this should never happen */ | |
41d9a6a7 | 1268 | if (bo->mem.mem_type == TTM_PL_VRAM && |
770d13b1 | 1269 | (offset + size) > adev->gmc.visible_vram_size) |
5fb1941d CK |
1270 | return -EINVAL; |
1271 | ||
d38ceaf9 AD |
1272 | return 0; |
1273 | } | |
1274 | ||
1275 | /** | |
1276 | * amdgpu_bo_fence - add fence to buffer object | |
1277 | * | |
1278 | * @bo: buffer object in question | |
1279 | * @fence: fence to add | |
1280 | * @shared: true if fence should be added shared | |
1281 | * | |
1282 | */ | |
f54d1867 | 1283 | void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, |
d38ceaf9 AD |
1284 | bool shared) |
1285 | { | |
1286 | struct reservation_object *resv = bo->tbo.resv; | |
1287 | ||
1288 | if (shared) | |
e40a3115 | 1289 | reservation_object_add_shared_fence(resv, fence); |
d38ceaf9 | 1290 | else |
e40a3115 | 1291 | reservation_object_add_excl_fence(resv, fence); |
d38ceaf9 | 1292 | } |
cdb7e8f2 | 1293 | |
e8e32426 FK |
1294 | /** |
1295 | * amdgpu_sync_wait_resv - Wait for BO reservation fences | |
1296 | * | |
1297 | * @bo: buffer object | |
1298 | * @owner: fence owner | |
1299 | * @intr: Whether the wait is interruptible | |
1300 | * | |
1301 | * Returns: | |
1302 | * 0 on success, errno otherwise. | |
1303 | */ | |
1304 | int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) | |
1305 | { | |
1306 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
1307 | struct amdgpu_sync sync; | |
1308 | int r; | |
1309 | ||
1310 | amdgpu_sync_create(&sync); | |
1311 | amdgpu_sync_resv(adev, &sync, bo->tbo.resv, owner, false); | |
1312 | r = amdgpu_sync_wait(&sync, intr); | |
1313 | amdgpu_sync_free(&sync); | |
1314 | ||
1315 | return r; | |
1316 | } | |
1317 | ||
cdb7e8f2 CK |
1318 | /** |
1319 | * amdgpu_bo_gpu_offset - return GPU offset of bo | |
1320 | * @bo: amdgpu object for which we query the offset | |
1321 | * | |
cdb7e8f2 CK |
1322 | * Note: object should either be pinned or reserved when calling this |
1323 | * function, it might be useful to add check for this for debugging. | |
2472e11b MD |
1324 | * |
1325 | * Returns: | |
1326 | * current GPU offset of the object. | |
cdb7e8f2 CK |
1327 | */ |
1328 | u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) | |
1329 | { | |
1330 | WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM); | |
1331 | WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) && | |
4947b2f2 | 1332 | !bo->pin_count && bo->tbo.type != ttm_bo_type_kernel); |
9702d40d | 1333 | WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET); |
03f48dd5 CK |
1334 | WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM && |
1335 | !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); | |
cdb7e8f2 | 1336 | |
ad9a5b78 | 1337 | return amdgpu_gmc_sign_extend(bo->tbo.offset); |
cdb7e8f2 | 1338 | } |
84b74608 | 1339 | |
2472e11b MD |
1340 | /** |
1341 | * amdgpu_bo_get_preferred_pin_domain - get preferred domain for scanout | |
1342 | * @adev: amdgpu device object | |
1343 | * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` | |
1344 | * | |
1345 | * Returns: | |
1346 | * Which of the allowed domains is preferred for pinning the BO for scanout. | |
1347 | */ | |
84b74608 DS |
1348 | uint32_t amdgpu_bo_get_preferred_pin_domain(struct amdgpu_device *adev, |
1349 | uint32_t domain) | |
1350 | { | |
1351 | if (domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) { | |
1352 | domain = AMDGPU_GEM_DOMAIN_VRAM; | |
1353 | if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) | |
1354 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
1355 | } | |
1356 | return domain; | |
1357 | } |