Commit | Line | Data |
---|---|---|
d38ceaf9 AD |
1 | /* |
2 | * Copyright 2009 Jerome Glisse. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sub license, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
20 | * | |
21 | * The above copyright notice and this permission notice (including the | |
22 | * next paragraph) shall be included in all copies or substantial portions | |
23 | * of the Software. | |
24 | * | |
25 | */ | |
26 | /* | |
27 | * Authors: | |
28 | * Jerome Glisse <glisse@freedesktop.org> | |
29 | * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> | |
30 | * Dave Airlie | |
31 | */ | |
32 | #include <linux/list.h> | |
33 | #include <linux/slab.h> | |
2d4dad27 | 34 | #include <linux/dma-buf.h> |
fdf2f6c5 | 35 | |
62d5f9f7 | 36 | #include <drm/drm_drv.h> |
d38ceaf9 | 37 | #include <drm/amdgpu_drm.h> |
a187f17f | 38 | #include <drm/drm_cache.h> |
d38ceaf9 AD |
39 | #include "amdgpu.h" |
40 | #include "amdgpu_trace.h" | |
a46a2cd1 | 41 | #include "amdgpu_amdkfd.h" |
d38ceaf9 | 42 | |
6f4e8d6e SL |
43 | /** |
44 | * DOC: amdgpu_object | |
45 | * | |
46 | * This defines the interfaces to operate on an &amdgpu_bo buffer object which | |
47 | * represents memory used by driver (VRAM, system memory, etc.). The driver | |
48 | * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces | |
49 | * to create/destroy/set buffer object which are then managed by the kernel TTM | |
50 | * memory manager. | |
51 | * The interfaces are also used internally by kernel clients, including gfx, | |
52 | * uvd, etc. for kernel managed allocations used by the GPU. | |
53 | * | |
54 | */ | |
55 | ||
c704ab18 | 56 | static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) |
d38ceaf9 | 57 | { |
b82485fd | 58 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); |
d38ceaf9 | 59 | |
6375bbb4 | 60 | amdgpu_bo_kunmap(bo); |
d38ceaf9 | 61 | |
c105de28 GH |
62 | if (bo->tbo.base.import_attach) |
63 | drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); | |
64 | drm_gem_object_release(&bo->tbo.base); | |
23e24fbb ND |
65 | amdgpu_bo_unref(&bo->parent); |
66 | kvfree(bo); | |
67 | } | |
68 | ||
69 | static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) | |
70 | { | |
71 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); | |
72 | struct amdgpu_bo_user *ubo; | |
73 | ||
74 | ubo = to_amdgpu_bo_user(bo); | |
75 | kfree(ubo->metadata); | |
76 | amdgpu_bo_destroy(tbo); | |
77 | } | |
78 | ||
79 | static void amdgpu_bo_vm_destroy(struct ttm_buffer_object *tbo) | |
80 | { | |
81 | struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); | |
cbb63ecc | 82 | struct amdgpu_bo *shadow_bo = ttm_to_amdgpu_bo(tbo), *bo; |
e18aaea7 | 83 | struct amdgpu_bo_vm *vmbo; |
23e24fbb | 84 | |
cbb63ecc | 85 | bo = shadow_bo->parent; |
e18aaea7 | 86 | vmbo = to_amdgpu_bo_vm(bo); |
36e499b2 | 87 | /* in case amdgpu_device_recover_vram got NULL of bo->parent */ |
e18aaea7 | 88 | if (!list_empty(&vmbo->shadow_list)) { |
a7d64de6 | 89 | mutex_lock(&adev->shadow_list_lock); |
e18aaea7 | 90 | list_del_init(&vmbo->shadow_list); |
a7d64de6 | 91 | mutex_unlock(&adev->shadow_list_lock); |
0c4e7fa5 | 92 | } |
36e499b2 | 93 | |
23e24fbb | 94 | amdgpu_bo_destroy(tbo); |
d38ceaf9 AD |
95 | } |
96 | ||
6f4e8d6e | 97 | /** |
c704ab18 | 98 | * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo |
6f4e8d6e SL |
99 | * @bo: buffer object to be checked |
100 | * | |
101 | * Uses destroy function associated with the object to determine if this is | |
102 | * an &amdgpu_bo. | |
103 | * | |
2472e11b MD |
104 | * Returns: |
105 | * true if the object belongs to &amdgpu_bo, false if not. | |
6f4e8d6e | 106 | */ |
c704ab18 | 107 | bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) |
d38ceaf9 | 108 | { |
23e24fbb ND |
109 | if (bo->destroy == &amdgpu_bo_destroy || |
110 | bo->destroy == &amdgpu_bo_user_destroy || | |
111 | bo->destroy == &amdgpu_bo_vm_destroy) | |
d38ceaf9 | 112 | return true; |
23e24fbb | 113 | |
d38ceaf9 AD |
114 | return false; |
115 | } | |
116 | ||
6f4e8d6e | 117 | /** |
c704ab18 | 118 | * amdgpu_bo_placement_from_domain - set buffer's placement |
6f4e8d6e SL |
119 | * @abo: &amdgpu_bo buffer object whose placement is to be set |
120 | * @domain: requested domain | |
121 | * | |
122 | * Sets buffer's placement according to requested domain and the buffer's | |
123 | * flags. | |
124 | */ | |
c704ab18 | 125 | void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) |
d38ceaf9 | 126 | { |
c09312a6 CK |
127 | struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); |
128 | struct ttm_placement *placement = &abo->placement; | |
129 | struct ttm_place *places = abo->placements; | |
130 | u64 flags = abo->flags; | |
6369f6f1 | 131 | u32 c = 0; |
7e5a547f | 132 | |
d38ceaf9 | 133 | if (domain & AMDGPU_GEM_DOMAIN_VRAM) { |
1d6ecab1 | 134 | unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; |
3ebfd221 | 135 | int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); |
faceaf6a | 136 | |
3ebfd221 PY |
137 | if (adev->gmc.mem_partitions && mem_id >= 0) { |
138 | places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; | |
6cfba94a PY |
139 | /* |
140 | * memory partition range lpfn is inclusive start + size - 1 | |
141 | * TTM place lpfn is exclusive start + size | |
142 | */ | |
3ebfd221 | 143 | places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; |
7f6db894 PY |
144 | } else { |
145 | places[c].fpfn = 0; | |
146 | places[c].lpfn = 0; | |
147 | } | |
48e07c23 | 148 | places[c].mem_type = TTM_PL_VRAM; |
ce65b874 | 149 | places[c].flags = 0; |
89bb5752 | 150 | |
faceaf6a | 151 | if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) |
7f6db894 | 152 | places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); |
59eddd4e | 153 | else |
faceaf6a | 154 | places[c].flags |= TTM_PL_FLAG_TOPDOWN; |
89bb5752 CK |
155 | |
156 | if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) | |
157 | places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; | |
faceaf6a | 158 | c++; |
d38ceaf9 AD |
159 | } |
160 | ||
dc3499c7 AD |
161 | if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { |
162 | places[c].fpfn = 0; | |
163 | places[c].lpfn = 0; | |
164 | places[c].mem_type = AMDGPU_PL_DOORBELL; | |
165 | places[c].flags = 0; | |
166 | c++; | |
167 | } | |
168 | ||
d38ceaf9 | 169 | if (domain & AMDGPU_GEM_DOMAIN_GTT) { |
faceaf6a | 170 | places[c].fpfn = 0; |
3d5fe658 | 171 | places[c].lpfn = 0; |
b453e42a FK |
172 | places[c].mem_type = |
173 | abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? | |
174 | AMDGPU_PL_PREEMPT : TTM_PL_TT; | |
48e07c23 | 175 | places[c].flags = 0; |
faceaf6a | 176 | c++; |
d38ceaf9 AD |
177 | } |
178 | ||
179 | if (domain & AMDGPU_GEM_DOMAIN_CPU) { | |
faceaf6a CK |
180 | places[c].fpfn = 0; |
181 | places[c].lpfn = 0; | |
48e07c23 CK |
182 | places[c].mem_type = TTM_PL_SYSTEM; |
183 | places[c].flags = 0; | |
faceaf6a | 184 | c++; |
d38ceaf9 AD |
185 | } |
186 | ||
187 | if (domain & AMDGPU_GEM_DOMAIN_GDS) { | |
faceaf6a CK |
188 | places[c].fpfn = 0; |
189 | places[c].lpfn = 0; | |
48e07c23 | 190 | places[c].mem_type = AMDGPU_PL_GDS; |
ce65b874 | 191 | places[c].flags = 0; |
faceaf6a | 192 | c++; |
d38ceaf9 | 193 | } |
faceaf6a | 194 | |
d38ceaf9 | 195 | if (domain & AMDGPU_GEM_DOMAIN_GWS) { |
faceaf6a CK |
196 | places[c].fpfn = 0; |
197 | places[c].lpfn = 0; | |
48e07c23 | 198 | places[c].mem_type = AMDGPU_PL_GWS; |
ce65b874 | 199 | places[c].flags = 0; |
faceaf6a | 200 | c++; |
d38ceaf9 | 201 | } |
faceaf6a | 202 | |
d38ceaf9 | 203 | if (domain & AMDGPU_GEM_DOMAIN_OA) { |
faceaf6a CK |
204 | places[c].fpfn = 0; |
205 | places[c].lpfn = 0; | |
48e07c23 | 206 | places[c].mem_type = AMDGPU_PL_OA; |
ce65b874 | 207 | places[c].flags = 0; |
faceaf6a | 208 | c++; |
d38ceaf9 AD |
209 | } |
210 | ||
211 | if (!c) { | |
faceaf6a CK |
212 | places[c].fpfn = 0; |
213 | places[c].lpfn = 0; | |
48e07c23 | 214 | places[c].mem_type = TTM_PL_SYSTEM; |
ce65b874 | 215 | places[c].flags = 0; |
faceaf6a | 216 | c++; |
d38ceaf9 | 217 | } |
faceaf6a | 218 | |
ea7acd7c | 219 | BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); |
bf314ca3 | 220 | |
7e5a547f | 221 | placement->num_placement = c; |
faceaf6a | 222 | placement->placement = places; |
d38ceaf9 AD |
223 | } |
224 | ||
7c204889 | 225 | /** |
9d903cbd | 226 | * amdgpu_bo_create_reserved - create reserved BO for kernel use |
7c204889 CK |
227 | * |
228 | * @adev: amdgpu device object | |
229 | * @size: size for the new BO | |
230 | * @align: alignment for the new BO | |
231 | * @domain: where to place it | |
64350f1b | 232 | * @bo_ptr: used to initialize BOs in structures |
7c204889 CK |
233 | * @gpu_addr: GPU addr of the pinned BO |
234 | * @cpu_addr: optional CPU address mapping | |
235 | * | |
9d903cbd CK |
236 | * Allocates and pins a BO for kernel internal use, and returns it still |
237 | * reserved. | |
7c204889 | 238 | * |
64350f1b AG |
239 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
240 | * | |
2472e11b MD |
241 | * Returns: |
242 | * 0 on success, negative error code otherwise. | |
7c204889 | 243 | */ |
9d903cbd CK |
244 | int amdgpu_bo_create_reserved(struct amdgpu_device *adev, |
245 | unsigned long size, int align, | |
246 | u32 domain, struct amdgpu_bo **bo_ptr, | |
247 | u64 *gpu_addr, void **cpu_addr) | |
7c204889 | 248 | { |
3216c6b7 | 249 | struct amdgpu_bo_param bp; |
53766e5a | 250 | bool free = false; |
7c204889 CK |
251 | int r; |
252 | ||
21a7e77f CK |
253 | if (!size) { |
254 | amdgpu_bo_unref(bo_ptr); | |
255 | return 0; | |
256 | } | |
257 | ||
3216c6b7 CZ |
258 | memset(&bp, 0, sizeof(bp)); |
259 | bp.size = size; | |
260 | bp.byte_align = align; | |
261 | bp.domain = domain; | |
828d6fde TY |
262 | bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
263 | : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; | |
264 | bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; | |
3216c6b7 CZ |
265 | bp.type = ttm_bo_type_kernel; |
266 | bp.resv = NULL; | |
9fd5543e | 267 | bp.bo_ptr_size = sizeof(struct amdgpu_bo); |
3216c6b7 | 268 | |
53766e5a | 269 | if (!*bo_ptr) { |
3216c6b7 | 270 | r = amdgpu_bo_create(adev, &bp, bo_ptr); |
53766e5a CK |
271 | if (r) { |
272 | dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", | |
273 | r); | |
274 | return r; | |
275 | } | |
276 | free = true; | |
7c204889 CK |
277 | } |
278 | ||
279 | r = amdgpu_bo_reserve(*bo_ptr, false); | |
280 | if (r) { | |
281 | dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); | |
282 | goto error_free; | |
283 | } | |
284 | ||
7b7c6c81 | 285 | r = amdgpu_bo_pin(*bo_ptr, domain); |
7c204889 CK |
286 | if (r) { |
287 | dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); | |
288 | goto error_unreserve; | |
289 | } | |
bb812f1e JZ |
290 | |
291 | r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); | |
292 | if (r) { | |
293 | dev_err(adev->dev, "%p bind failed\n", *bo_ptr); | |
294 | goto error_unpin; | |
295 | } | |
296 | ||
7b7c6c81 JZ |
297 | if (gpu_addr) |
298 | *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); | |
7c204889 CK |
299 | |
300 | if (cpu_addr) { | |
301 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
302 | if (r) { | |
303 | dev_err(adev->dev, "(%d) kernel bo map failed\n", r); | |
dc407ee0 | 304 | goto error_unpin; |
7c204889 CK |
305 | } |
306 | } | |
307 | ||
7c204889 CK |
308 | return 0; |
309 | ||
bb812f1e JZ |
310 | error_unpin: |
311 | amdgpu_bo_unpin(*bo_ptr); | |
7c204889 CK |
312 | error_unreserve: |
313 | amdgpu_bo_unreserve(*bo_ptr); | |
314 | ||
315 | error_free: | |
53766e5a CK |
316 | if (free) |
317 | amdgpu_bo_unref(bo_ptr); | |
7c204889 CK |
318 | |
319 | return r; | |
320 | } | |
321 | ||
9d903cbd CK |
322 | /** |
323 | * amdgpu_bo_create_kernel - create BO for kernel use | |
324 | * | |
325 | * @adev: amdgpu device object | |
326 | * @size: size for the new BO | |
327 | * @align: alignment for the new BO | |
328 | * @domain: where to place it | |
64350f1b | 329 | * @bo_ptr: used to initialize BOs in structures |
9d903cbd CK |
330 | * @gpu_addr: GPU addr of the pinned BO |
331 | * @cpu_addr: optional CPU address mapping | |
332 | * | |
333 | * Allocates and pins a BO for kernel internal use. | |
334 | * | |
64350f1b AG |
335 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
336 | * | |
2472e11b MD |
337 | * Returns: |
338 | * 0 on success, negative error code otherwise. | |
9d903cbd CK |
339 | */ |
340 | int amdgpu_bo_create_kernel(struct amdgpu_device *adev, | |
341 | unsigned long size, int align, | |
342 | u32 domain, struct amdgpu_bo **bo_ptr, | |
343 | u64 *gpu_addr, void **cpu_addr) | |
344 | { | |
345 | int r; | |
346 | ||
347 | r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, | |
348 | gpu_addr, cpu_addr); | |
349 | ||
350 | if (r) | |
351 | return r; | |
352 | ||
ddaf5013 TSD |
353 | if (*bo_ptr) |
354 | amdgpu_bo_unreserve(*bo_ptr); | |
9d903cbd CK |
355 | |
356 | return 0; | |
357 | } | |
358 | ||
de7b45ba CK |
359 | /** |
360 | * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location | |
361 | * | |
362 | * @adev: amdgpu device object | |
363 | * @offset: offset of the BO | |
364 | * @size: size of the BO | |
de7b45ba CK |
365 | * @bo_ptr: used to initialize BOs in structures |
366 | * @cpu_addr: optional CPU address mapping | |
367 | * | |
3273f116 | 368 | * Creates a kernel BO at a specific offset in VRAM. |
de7b45ba CK |
369 | * |
370 | * Returns: | |
371 | * 0 on success, negative error code otherwise. | |
372 | */ | |
373 | int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, | |
3273f116 | 374 | uint64_t offset, uint64_t size, |
de7b45ba CK |
375 | struct amdgpu_bo **bo_ptr, void **cpu_addr) |
376 | { | |
377 | struct ttm_operation_ctx ctx = { false, false }; | |
378 | unsigned int i; | |
379 | int r; | |
380 | ||
381 | offset &= PAGE_MASK; | |
382 | size = ALIGN(size, PAGE_SIZE); | |
383 | ||
3273f116 LT |
384 | r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, |
385 | AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, | |
386 | cpu_addr); | |
de7b45ba CK |
387 | if (r) |
388 | return r; | |
389 | ||
37912e96 AD |
390 | if ((*bo_ptr) == NULL) |
391 | return 0; | |
392 | ||
de7b45ba CK |
393 | /* |
394 | * Remove the original mem node and create a new one at the request | |
395 | * position. | |
396 | */ | |
4a246528 CK |
397 | if (cpu_addr) |
398 | amdgpu_bo_kunmap(*bo_ptr); | |
399 | ||
bfa3357e | 400 | ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); |
4a246528 | 401 | |
de7b45ba CK |
402 | for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { |
403 | (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; | |
404 | (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; | |
405 | } | |
de7b45ba | 406 | r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, |
bfa3357e | 407 | &(*bo_ptr)->tbo.resource, &ctx); |
de7b45ba CK |
408 | if (r) |
409 | goto error; | |
410 | ||
411 | if (cpu_addr) { | |
412 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
413 | if (r) | |
414 | goto error; | |
415 | } | |
416 | ||
417 | amdgpu_bo_unreserve(*bo_ptr); | |
418 | return 0; | |
419 | ||
420 | error: | |
421 | amdgpu_bo_unreserve(*bo_ptr); | |
422 | amdgpu_bo_unref(bo_ptr); | |
423 | return r; | |
424 | } | |
425 | ||
aa1d562e JZ |
426 | /** |
427 | * amdgpu_bo_free_kernel - free BO for kernel use | |
428 | * | |
429 | * @bo: amdgpu BO to free | |
2472e11b MD |
430 | * @gpu_addr: pointer to where the BO's GPU memory space address was stored |
431 | * @cpu_addr: pointer to where the BO's CPU memory space address was stored | |
aa1d562e JZ |
432 | * |
433 | * unmaps and unpin a BO for kernel internal use. | |
434 | */ | |
435 | void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, | |
436 | void **cpu_addr) | |
437 | { | |
438 | if (*bo == NULL) | |
439 | return; | |
440 | ||
4d2ccd96 CK |
441 | WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); |
442 | ||
f3aa745e | 443 | if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { |
aa1d562e JZ |
444 | if (cpu_addr) |
445 | amdgpu_bo_kunmap(*bo); | |
446 | ||
447 | amdgpu_bo_unpin(*bo); | |
448 | amdgpu_bo_unreserve(*bo); | |
449 | } | |
450 | amdgpu_bo_unref(bo); | |
451 | ||
452 | if (gpu_addr) | |
453 | *gpu_addr = 0; | |
454 | ||
455 | if (cpu_addr) | |
456 | *cpu_addr = NULL; | |
457 | } | |
458 | ||
8f9a9a09 | 459 | /* Validate bo size is bit bigger than the request domain */ |
79c63123 AG |
460 | static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, |
461 | unsigned long size, u32 domain) | |
462 | { | |
9de59bc2 | 463 | struct ttm_resource_manager *man = NULL; |
79c63123 AG |
464 | |
465 | /* | |
466 | * If GTT is part of requested domains the check must succeed to | |
7554886d | 467 | * allow fall back to GTT. |
79c63123 | 468 | */ |
8f9a9a09 | 469 | if (domain & AMDGPU_GEM_DOMAIN_GTT) |
6c28aed6 | 470 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); |
8f9a9a09 | 471 | else if (domain & AMDGPU_GEM_DOMAIN_VRAM) |
6c28aed6 | 472 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); |
8f9a9a09 MJ |
473 | else |
474 | return true; | |
79c63123 | 475 | |
8f9a9a09 MJ |
476 | if (!man) { |
477 | if (domain & AMDGPU_GEM_DOMAIN_GTT) | |
478 | WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); | |
479 | return false; | |
79c63123 AG |
480 | } |
481 | ||
dc3499c7 | 482 | /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ |
8f9a9a09 MJ |
483 | if (size < man->size) |
484 | return true; | |
79c63123 | 485 | |
8f9a9a09 | 486 | DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); |
79c63123 AG |
487 | return false; |
488 | } | |
489 | ||
3d1b8ec7 AG |
490 | bool amdgpu_bo_support_uswc(u64 bo_flags) |
491 | { | |
492 | ||
493 | #ifdef CONFIG_X86_32 | |
494 | /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit | |
495 | * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 | |
496 | */ | |
497 | return false; | |
498 | #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) | |
499 | /* Don't try to enable write-combining when it can't work, or things | |
500 | * may be slow | |
501 | * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 | |
502 | */ | |
503 | ||
504 | #ifndef CONFIG_COMPILE_TEST | |
505 | #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ | |
506 | thanks to write-combining | |
507 | #endif | |
508 | ||
509 | if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
510 | DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " | |
511 | "better performance thanks to write-combining\n"); | |
512 | return false; | |
513 | #else | |
514 | /* For architectures that don't support WC memory, | |
515 | * mask out the WC flag from the BO | |
516 | */ | |
517 | if (!drm_arch_can_wc_memory()) | |
518 | return false; | |
519 | ||
520 | return true; | |
521 | #endif | |
522 | } | |
523 | ||
cd2454d6 ND |
524 | /** |
525 | * amdgpu_bo_create - create an &amdgpu_bo buffer object | |
526 | * @adev: amdgpu device object | |
527 | * @bp: parameters to be used for the buffer object | |
528 | * @bo_ptr: pointer to the buffer object pointer | |
529 | * | |
530 | * Creates an &amdgpu_bo buffer object. | |
531 | * | |
532 | * Returns: | |
533 | * 0 for success or a negative error code on failure. | |
534 | */ | |
535 | int amdgpu_bo_create(struct amdgpu_device *adev, | |
a906dbb1 | 536 | struct amdgpu_bo_param *bp, |
c09312a6 | 537 | struct amdgpu_bo **bo_ptr) |
d38ceaf9 | 538 | { |
9251859a | 539 | struct ttm_operation_ctx ctx = { |
a906dbb1 | 540 | .interruptible = (bp->type != ttm_bo_type_kernel), |
061468c4 | 541 | .no_wait_gpu = bp->no_wait_gpu, |
586052b0 CK |
542 | /* We opt to avoid OOM on system pages allocations */ |
543 | .gfp_retry_mayfail = true, | |
c44dfe4d CK |
544 | .allow_res_evict = bp->type != ttm_bo_type_kernel, |
545 | .resv = bp->resv | |
9251859a | 546 | }; |
d38ceaf9 | 547 | struct amdgpu_bo *bo; |
a906dbb1 | 548 | unsigned long page_align, size = bp->size; |
d38ceaf9 AD |
549 | int r; |
550 | ||
fe57085a MO |
551 | /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ |
552 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { | |
553 | /* GWS and OA don't need any alignment. */ | |
554 | page_align = bp->byte_align; | |
77a2faa5 | 555 | size <<= PAGE_SHIFT; |
e3c92eb4 | 556 | |
fe57085a MO |
557 | } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { |
558 | /* Both size and alignment must be a multiple of 4. */ | |
559 | page_align = ALIGN(bp->byte_align, 4); | |
560 | size = ALIGN(size, 4) << PAGE_SHIFT; | |
561 | } else { | |
562 | /* Memory should be aligned at least to a page size. */ | |
563 | page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; | |
77a2faa5 | 564 | size = ALIGN(size, PAGE_SIZE); |
fe57085a | 565 | } |
d38ceaf9 | 566 | |
a906dbb1 | 567 | if (!amdgpu_bo_validate_size(adev, size, bp->domain)) |
79c63123 AG |
568 | return -ENOMEM; |
569 | ||
9fd5543e | 570 | BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); |
d38ceaf9 | 571 | |
9fd5543e | 572 | *bo_ptr = NULL; |
31c759bb | 573 | bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); |
d38ceaf9 AD |
574 | if (bo == NULL) |
575 | return -ENOMEM; | |
4a580877 | 576 | drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); |
646b9025 | 577 | bo->vm_bo = NULL; |
3f188453 | 578 | bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : |
aa2b2e28 | 579 | bp->domain; |
08082104 | 580 | bo->allowed_domains = bo->preferred_domains; |
a906dbb1 | 581 | if (bp->type != ttm_bo_type_kernel && |
fab2cc83 | 582 | !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && |
08082104 CK |
583 | bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) |
584 | bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; | |
d38ceaf9 | 585 | |
a906dbb1 | 586 | bo->flags = bp->flags; |
a187f17f | 587 | |
3ebfd221 PY |
588 | if (adev->gmc.mem_partitions) |
589 | /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ | |
590 | bo->xcp_id = bp->xcp_id_plus1 - 1; | |
591 | else | |
592 | /* For GPUs without spatial partitioning */ | |
593 | bo->xcp_id = 0; | |
f24e924b | 594 | |
3d1b8ec7 | 595 | if (!amdgpu_bo_support_uswc(bo->flags)) |
a187f17f OG |
596 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; |
597 | ||
fc6ea4be FK |
598 | if (adev->ras_enabled) |
599 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; | |
600 | ||
c09312a6 | 601 | bo->tbo.bdev = &adev->mman.bdev; |
47722220 CK |
602 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | |
603 | AMDGPU_GEM_DOMAIN_GDS)) | |
604 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
605 | else | |
606 | amdgpu_bo_placement_from_domain(bo, bp->domain); | |
a50cb948 | 607 | if (bp->type == ttm_bo_type_kernel) |
b0b13d53 FK |
608 | bo->tbo.priority = 2; |
609 | else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) | |
a50cb948 | 610 | bo->tbo.priority = 1; |
08082104 | 611 | |
23e24fbb ND |
612 | if (!bp->destroy) |
613 | bp->destroy = &amdgpu_bo_destroy; | |
614 | ||
347987a2 | 615 | r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, |
f07069da | 616 | &bo->placement, page_align, &ctx, NULL, |
23e24fbb | 617 | bp->resv, bp->destroy); |
08082104 | 618 | if (unlikely(r != 0)) |
a695e437 CK |
619 | return r; |
620 | ||
c8c5e569 | 621 | if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && |
a6ff969f | 622 | amdgpu_res_cpu_visible(adev, bo->tbo.resource)) |
6af046d2 CK |
623 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, |
624 | ctx.bytes_moved); | |
00f06b24 | 625 | else |
6af046d2 | 626 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); |
fad06127 | 627 | |
a906dbb1 | 628 | if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && |
d3116756 | 629 | bo->tbo.resource->mem_type == TTM_PL_VRAM) { |
f54d1867 | 630 | struct dma_fence *fence; |
4fea83ff | 631 | |
c3aaca43 | 632 | r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence, true); |
c3af1258 CK |
633 | if (unlikely(r)) |
634 | goto fail_unreserve; | |
635 | ||
8bb31587 CK |
636 | dma_resv_add_fence(bo->tbo.base.resv, fence, |
637 | DMA_RESV_USAGE_KERNEL); | |
f54d1867 | 638 | dma_fence_put(fence); |
4fea83ff | 639 | } |
a906dbb1 | 640 | if (!bp->resv) |
59c66c91 | 641 | amdgpu_bo_unreserve(bo); |
d38ceaf9 AD |
642 | *bo_ptr = bo; |
643 | ||
644 | trace_amdgpu_bo_create(bo); | |
645 | ||
96cf8271 | 646 | /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ |
a906dbb1 | 647 | if (bp->type == ttm_bo_type_device) |
96cf8271 JB |
648 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; |
649 | ||
d38ceaf9 | 650 | return 0; |
4fea83ff FC |
651 | |
652 | fail_unreserve: | |
a906dbb1 | 653 | if (!bp->resv) |
52791eee | 654 | dma_resv_unlock(bo->tbo.base.resv); |
4fea83ff FC |
655 | amdgpu_bo_unref(&bo); |
656 | return r; | |
d38ceaf9 AD |
657 | } |
658 | ||
9ad0d033 ND |
659 | /** |
660 | * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object | |
661 | * @adev: amdgpu device object | |
662 | * @bp: parameters to be used for the buffer object | |
663 | * @ubo_ptr: pointer to the buffer object pointer | |
664 | * | |
665 | * Create a BO to be used by user application; | |
666 | * | |
667 | * Returns: | |
668 | * 0 for success or a negative error code on failure. | |
669 | */ | |
670 | ||
671 | int amdgpu_bo_create_user(struct amdgpu_device *adev, | |
672 | struct amdgpu_bo_param *bp, | |
673 | struct amdgpu_bo_user **ubo_ptr) | |
674 | { | |
675 | struct amdgpu_bo *bo_ptr; | |
676 | int r; | |
677 | ||
9ad0d033 | 678 | bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); |
23e24fbb | 679 | bp->destroy = &amdgpu_bo_user_destroy; |
cd2454d6 | 680 | r = amdgpu_bo_create(adev, bp, &bo_ptr); |
9ad0d033 ND |
681 | if (r) |
682 | return r; | |
683 | ||
684 | *ubo_ptr = to_amdgpu_bo_user(bo_ptr); | |
685 | return r; | |
686 | } | |
6fdd6f4a ND |
687 | |
688 | /** | |
689 | * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object | |
690 | * @adev: amdgpu device object | |
691 | * @bp: parameters to be used for the buffer object | |
692 | * @vmbo_ptr: pointer to the buffer object pointer | |
693 | * | |
694 | * Create a BO to be for GPUVM. | |
695 | * | |
696 | * Returns: | |
697 | * 0 for success or a negative error code on failure. | |
698 | */ | |
699 | ||
700 | int amdgpu_bo_create_vm(struct amdgpu_device *adev, | |
701 | struct amdgpu_bo_param *bp, | |
702 | struct amdgpu_bo_vm **vmbo_ptr) | |
703 | { | |
704 | struct amdgpu_bo *bo_ptr; | |
705 | int r; | |
706 | ||
707 | /* bo_ptr_size will be determined by the caller and it depends on | |
708 | * num of amdgpu_vm_pt entries. | |
709 | */ | |
710 | BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); | |
711 | r = amdgpu_bo_create(adev, bp, &bo_ptr); | |
712 | if (r) | |
713 | return r; | |
714 | ||
715 | *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); | |
716 | return r; | |
717 | } | |
718 | ||
1fdc79f6 ND |
719 | /** |
720 | * amdgpu_bo_add_to_shadow_list - add a BO to the shadow list | |
721 | * | |
d8c33180 | 722 | * @vmbo: BO that will be inserted into the shadow list |
1fdc79f6 ND |
723 | * |
724 | * Insert a BO to the shadow list. | |
725 | */ | |
e18aaea7 | 726 | void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo) |
1fdc79f6 | 727 | { |
e18aaea7 | 728 | struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev); |
1fdc79f6 ND |
729 | |
730 | mutex_lock(&adev->shadow_list_lock); | |
e18aaea7 | 731 | list_add_tail(&vmbo->shadow_list, &adev->shadow_list); |
cbb63ecc HZ |
732 | vmbo->shadow->parent = amdgpu_bo_ref(&vmbo->bo); |
733 | vmbo->shadow->tbo.destroy = &amdgpu_bo_vm_destroy; | |
1fdc79f6 ND |
734 | mutex_unlock(&adev->shadow_list_lock); |
735 | } | |
736 | ||
6f4e8d6e | 737 | /** |
403009bf CK |
738 | * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow |
739 | * | |
740 | * @shadow: &amdgpu_bo shadow to be restored | |
6f4e8d6e | 741 | * @fence: dma_fence associated with the operation |
6f4e8d6e SL |
742 | * |
743 | * Copies a buffer object's shadow content back to the object. | |
744 | * This is used for recovering a buffer from its shadow in case of a gpu | |
745 | * reset where vram context may be lost. | |
746 | * | |
2472e11b MD |
747 | * Returns: |
748 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 749 | */ |
403009bf | 750 | int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence) |
20f4eff1 CZ |
751 | |
752 | { | |
403009bf CK |
753 | struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev); |
754 | struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; | |
755 | uint64_t shadow_addr, parent_addr; | |
20f4eff1 | 756 | |
403009bf CK |
757 | shadow_addr = amdgpu_bo_gpu_offset(shadow); |
758 | parent_addr = amdgpu_bo_gpu_offset(shadow->parent); | |
20f4eff1 | 759 | |
403009bf CK |
760 | return amdgpu_copy_buffer(ring, shadow_addr, parent_addr, |
761 | amdgpu_bo_size(shadow), NULL, fence, | |
c9dc9cfe | 762 | true, false, false); |
20f4eff1 CZ |
763 | } |
764 | ||
6f4e8d6e SL |
765 | /** |
766 | * amdgpu_bo_kmap - map an &amdgpu_bo buffer object | |
767 | * @bo: &amdgpu_bo buffer object to be mapped | |
768 | * @ptr: kernel virtual address to be returned | |
769 | * | |
770 | * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls | |
771 | * amdgpu_bo_kptr() to get the kernel virtual address. | |
772 | * | |
2472e11b MD |
773 | * Returns: |
774 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 775 | */ |
d38ceaf9 AD |
776 | int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) |
777 | { | |
f5e1c740 | 778 | void *kptr; |
587f3c70 | 779 | long r; |
d38ceaf9 | 780 | |
271c8125 CK |
781 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) |
782 | return -EPERM; | |
783 | ||
c35fcfa3 CK |
784 | r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, |
785 | false, MAX_SCHEDULE_TIMEOUT); | |
786 | if (r < 0) | |
787 | return r; | |
788 | ||
f5e1c740 CK |
789 | kptr = amdgpu_bo_kptr(bo); |
790 | if (kptr) { | |
791 | if (ptr) | |
792 | *ptr = kptr; | |
d38ceaf9 AD |
793 | return 0; |
794 | } | |
587f3c70 | 795 | |
e3c92eb4 | 796 | r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); |
587f3c70 | 797 | if (r) |
d38ceaf9 | 798 | return r; |
587f3c70 | 799 | |
587f3c70 | 800 | if (ptr) |
f5e1c740 | 801 | *ptr = amdgpu_bo_kptr(bo); |
587f3c70 | 802 | |
d38ceaf9 AD |
803 | return 0; |
804 | } | |
805 | ||
6f4e8d6e SL |
806 | /** |
807 | * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object | |
808 | * @bo: &amdgpu_bo buffer object | |
809 | * | |
810 | * Calls ttm_kmap_obj_virtual() to get the kernel virtual address | |
811 | * | |
2472e11b MD |
812 | * Returns: |
813 | * the virtual address of a buffer object area. | |
6f4e8d6e | 814 | */ |
f5e1c740 CK |
815 | void *amdgpu_bo_kptr(struct amdgpu_bo *bo) |
816 | { | |
817 | bool is_iomem; | |
818 | ||
819 | return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); | |
820 | } | |
821 | ||
6f4e8d6e SL |
822 | /** |
823 | * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object | |
824 | * @bo: &amdgpu_bo buffer object to be unmapped | |
825 | * | |
826 | * Unmaps a kernel map set up by amdgpu_bo_kmap(). | |
827 | */ | |
d38ceaf9 AD |
828 | void amdgpu_bo_kunmap(struct amdgpu_bo *bo) |
829 | { | |
f5e1c740 CK |
830 | if (bo->kmap.bo) |
831 | ttm_bo_kunmap(&bo->kmap); | |
d38ceaf9 AD |
832 | } |
833 | ||
6f4e8d6e SL |
834 | /** |
835 | * amdgpu_bo_ref - reference an &amdgpu_bo buffer object | |
836 | * @bo: &amdgpu_bo buffer object | |
837 | * | |
838 | * References the contained &ttm_buffer_object. | |
839 | * | |
2472e11b MD |
840 | * Returns: |
841 | * a refcounted pointer to the &amdgpu_bo buffer object. | |
6f4e8d6e | 842 | */ |
d38ceaf9 AD |
843 | struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) |
844 | { | |
845 | if (bo == NULL) | |
846 | return NULL; | |
847 | ||
71d5ef11 | 848 | ttm_bo_get(&bo->tbo); |
d38ceaf9 AD |
849 | return bo; |
850 | } | |
851 | ||
6f4e8d6e SL |
852 | /** |
853 | * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object | |
854 | * @bo: &amdgpu_bo buffer object | |
855 | * | |
856 | * Unreferences the contained &ttm_buffer_object and clear the pointer | |
857 | */ | |
d38ceaf9 AD |
858 | void amdgpu_bo_unref(struct amdgpu_bo **bo) |
859 | { | |
860 | struct ttm_buffer_object *tbo; | |
861 | ||
862 | if ((*bo) == NULL) | |
863 | return; | |
864 | ||
865 | tbo = &((*bo)->tbo); | |
fea872b2 TZ |
866 | ttm_bo_put(tbo); |
867 | *bo = NULL; | |
d38ceaf9 AD |
868 | } |
869 | ||
6f4e8d6e SL |
870 | /** |
871 | * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object | |
872 | * @bo: &amdgpu_bo buffer object to be pinned | |
873 | * @domain: domain to be pinned to | |
874 | * @min_offset: the start of requested address range | |
875 | * @max_offset: the end of requested address range | |
6f4e8d6e SL |
876 | * |
877 | * Pins the buffer object according to requested domain and address range. If | |
878 | * the memory is unbound gart memory, binds the pages into gart table. Adjusts | |
879 | * pin_count and pin_size accordingly. | |
880 | * | |
881 | * Pinning means to lock pages in memory along with keeping them at a fixed | |
882 | * offset. It is required when a buffer can not be moved, for example, when | |
883 | * a display buffer is being scanned out. | |
884 | * | |
885 | * Compared with amdgpu_bo_pin(), this function gives more flexibility on | |
886 | * where to pin a buffer if there are specific restrictions on where a buffer | |
887 | * must be located. | |
888 | * | |
2472e11b MD |
889 | * Returns: |
890 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 891 | */ |
7e5a547f | 892 | int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, |
7b7c6c81 | 893 | u64 min_offset, u64 max_offset) |
d38ceaf9 | 894 | { |
a7d64de6 | 895 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
19be5570 | 896 | struct ttm_operation_ctx ctx = { false, false }; |
d38ceaf9 AD |
897 | int r, i; |
898 | ||
cc325d19 | 899 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) |
d38ceaf9 AD |
900 | return -EPERM; |
901 | ||
7e5a547f CZ |
902 | if (WARN_ON_ONCE(min_offset > max_offset)) |
903 | return -EINVAL; | |
904 | ||
f5ba1404 LL |
905 | /* Check domain to be pinned to against preferred domains */ |
906 | if (bo->preferred_domains & domain) | |
907 | domain = bo->preferred_domains & domain; | |
908 | ||
803d89ad | 909 | /* A shared bo cannot be migrated to VRAM */ |
8c505bdc | 910 | if (bo->tbo.base.import_attach) { |
9b3f217f SL |
911 | if (domain & AMDGPU_GEM_DOMAIN_GTT) |
912 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
913 | else | |
914 | return -EINVAL; | |
915 | } | |
803d89ad | 916 | |
4671078e | 917 | if (bo->tbo.pin_count) { |
d3116756 CK |
918 | uint32_t mem_type = bo->tbo.resource->mem_type; |
919 | uint32_t mem_flags = bo->tbo.resource->placement; | |
408778e8 | 920 | |
f5318959 | 921 | if (!(domain & amdgpu_mem_type_to_domain(mem_type))) |
408778e8 FC |
922 | return -EINVAL; |
923 | ||
e1a4b67a | 924 | if ((mem_type == TTM_PL_VRAM) && |
925 | (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && | |
eda1068d FK |
926 | !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) |
927 | return -EINVAL; | |
928 | ||
4671078e | 929 | ttm_bo_pin(&bo->tbo); |
d38ceaf9 AD |
930 | |
931 | if (max_offset != 0) { | |
b1a8ef95 ND |
932 | u64 domain_start = amdgpu_ttm_domain_start(adev, |
933 | mem_type); | |
d38ceaf9 AD |
934 | WARN_ON_ONCE(max_offset < |
935 | (amdgpu_bo_gpu_offset(bo) - domain_start)); | |
936 | } | |
937 | ||
938 | return 0; | |
939 | } | |
03f48dd5 | 940 | |
9deb0b3d CK |
941 | /* This assumes only APU display buffers are pinned with (VRAM|GTT). |
942 | * See function amdgpu_display_supported_domains() | |
943 | */ | |
d035f84d | 944 | domain = amdgpu_bo_get_preferred_domain(adev, domain); |
9deb0b3d | 945 | |
a448cb00 CK |
946 | if (bo->tbo.base.import_attach) |
947 | dma_buf_pin(bo->tbo.base.import_attach); | |
948 | ||
e9c7577c CK |
949 | /* force to pin into visible video ram */ |
950 | if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) | |
951 | bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
c704ab18 | 952 | amdgpu_bo_placement_from_domain(bo, domain); |
d38ceaf9 | 953 | for (i = 0; i < bo->placement.num_placement; i++) { |
1d6ecab1 | 954 | unsigned int fpfn, lpfn; |
e9c7577c CK |
955 | |
956 | fpfn = min_offset >> PAGE_SHIFT; | |
957 | lpfn = max_offset >> PAGE_SHIFT; | |
958 | ||
7e5a547f CZ |
959 | if (fpfn > bo->placements[i].fpfn) |
960 | bo->placements[i].fpfn = fpfn; | |
78d0e182 CK |
961 | if (!bo->placements[i].lpfn || |
962 | (lpfn && lpfn < bo->placements[i].lpfn)) | |
7e5a547f | 963 | bo->placements[i].lpfn = lpfn; |
d38ceaf9 AD |
964 | } |
965 | ||
19be5570 | 966 | r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
6681c5eb | 967 | if (unlikely(r)) { |
a7d64de6 | 968 | dev_err(adev->dev, "%p pin failed\n", bo); |
6681c5eb CK |
969 | goto error; |
970 | } | |
971 | ||
4671078e | 972 | ttm_bo_pin(&bo->tbo); |
5e91fb57 | 973 | |
d3116756 | 974 | domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); |
6681c5eb | 975 | if (domain == AMDGPU_GEM_DOMAIN_VRAM) { |
a5ccfe5c MD |
976 | atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); |
977 | atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), | |
978 | &adev->visible_pin_size); | |
32ab75f0 | 979 | } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { |
a5ccfe5c | 980 | atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); |
d38ceaf9 | 981 | } |
6681c5eb CK |
982 | |
983 | error: | |
d38ceaf9 AD |
984 | return r; |
985 | } | |
986 | ||
6f4e8d6e SL |
987 | /** |
988 | * amdgpu_bo_pin - pin an &amdgpu_bo buffer object | |
989 | * @bo: &amdgpu_bo buffer object to be pinned | |
990 | * @domain: domain to be pinned to | |
6f4e8d6e SL |
991 | * |
992 | * A simple wrapper to amdgpu_bo_pin_restricted(). | |
993 | * Provides a simpler API for buffers that do not have any strict restrictions | |
994 | * on where a buffer must be located. | |
995 | * | |
2472e11b MD |
996 | * Returns: |
997 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 998 | */ |
7b7c6c81 | 999 | int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) |
d38ceaf9 | 1000 | { |
eda1068d | 1001 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; |
7b7c6c81 | 1002 | return amdgpu_bo_pin_restricted(bo, domain, 0, 0); |
d38ceaf9 AD |
1003 | } |
1004 | ||
6f4e8d6e SL |
1005 | /** |
1006 | * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object | |
1007 | * @bo: &amdgpu_bo buffer object to be unpinned | |
1008 | * | |
1009 | * Decreases the pin_count, and clears the flags if pin_count reaches 0. | |
1010 | * Changes placement and pin size accordingly. | |
1011 | * | |
2472e11b MD |
1012 | * Returns: |
1013 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1014 | */ |
4671078e | 1015 | void amdgpu_bo_unpin(struct amdgpu_bo *bo) |
d38ceaf9 | 1016 | { |
e2ac8531 CK |
1017 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
1018 | ||
4671078e CK |
1019 | ttm_bo_unpin(&bo->tbo); |
1020 | if (bo->tbo.pin_count) | |
1021 | return; | |
6681c5eb | 1022 | |
a448cb00 CK |
1023 | if (bo->tbo.base.import_attach) |
1024 | dma_buf_unpin(bo->tbo.base.import_attach); | |
e2ac8531 | 1025 | |
d3116756 | 1026 | if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { |
e2ac8531 CK |
1027 | atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); |
1028 | atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), | |
1029 | &adev->visible_pin_size); | |
d3116756 | 1030 | } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { |
e2ac8531 CK |
1031 | atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); |
1032 | } | |
dc3499c7 | 1033 | |
d38ceaf9 AD |
1034 | } |
1035 | ||
1d6ecab1 | 1036 | static const char * const amdgpu_vram_names[] = { |
1f8628c7 AD |
1037 | "UNKNOWN", |
1038 | "GDDR1", | |
1039 | "DDR2", | |
1040 | "GDDR3", | |
1041 | "GDDR4", | |
1042 | "GDDR5", | |
1043 | "HBM", | |
bc227cfa TSD |
1044 | "DDR3", |
1045 | "DDR4", | |
5228fe30 | 1046 | "GDDR6", |
d534ca71 AD |
1047 | "DDR5", |
1048 | "LPDDR4", | |
1049 | "LPDDR5" | |
1f8628c7 AD |
1050 | }; |
1051 | ||
6f4e8d6e SL |
1052 | /** |
1053 | * amdgpu_bo_init - initialize memory manager | |
1054 | * @adev: amdgpu device object | |
1055 | * | |
1056 | * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. | |
1057 | * | |
2472e11b MD |
1058 | * Returns: |
1059 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1060 | */ |
d38ceaf9 AD |
1061 | int amdgpu_bo_init(struct amdgpu_device *adev) |
1062 | { | |
35d5f224 | 1063 | /* On A+A platform, VRAM can be mapped as WB */ |
228ce176 | 1064 | if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { |
35d5f224 | 1065 | /* reserve PAT memory space to WC for VRAM */ |
26db557e | 1066 | int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, |
35d5f224 OZ |
1067 | adev->gmc.aper_size); |
1068 | ||
26db557e ND |
1069 | if (r) { |
1070 | DRM_ERROR("Unable to set WC memtype for the aperture base\n"); | |
1071 | return r; | |
1072 | } | |
1073 | ||
35d5f224 OZ |
1074 | /* Add an MTRR for the VRAM */ |
1075 | adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, | |
1076 | adev->gmc.aper_size); | |
1077 | } | |
7cf321d1 | 1078 | |
d38ceaf9 | 1079 | DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", |
770d13b1 CK |
1080 | adev->gmc.mc_vram_size >> 20, |
1081 | (unsigned long long)adev->gmc.aper_size >> 20); | |
1f8628c7 | 1082 | DRM_INFO("RAM width %dbits %s\n", |
770d13b1 | 1083 | adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); |
d38ceaf9 AD |
1084 | return amdgpu_ttm_init(adev); |
1085 | } | |
1086 | ||
6f4e8d6e SL |
1087 | /** |
1088 | * amdgpu_bo_fini - tear down memory manager | |
1089 | * @adev: amdgpu device object | |
1090 | * | |
1091 | * Reverses amdgpu_bo_init() to tear down memory manager. | |
1092 | */ | |
d38ceaf9 AD |
1093 | void amdgpu_bo_fini(struct amdgpu_device *adev) |
1094 | { | |
62d5f9f7 LS |
1095 | int idx; |
1096 | ||
d38ceaf9 | 1097 | amdgpu_ttm_fini(adev); |
62d5f9f7 LS |
1098 | |
1099 | if (drm_dev_enter(adev_to_drm(adev), &idx)) { | |
a0ba1279 | 1100 | if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { |
62d5f9f7 LS |
1101 | arch_phys_wc_del(adev->gmc.vram_mtrr); |
1102 | arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); | |
1103 | } | |
1104 | drm_dev_exit(idx); | |
1105 | } | |
d38ceaf9 AD |
1106 | } |
1107 | ||
6f4e8d6e SL |
1108 | /** |
1109 | * amdgpu_bo_set_tiling_flags - set tiling flags | |
1110 | * @bo: &amdgpu_bo buffer object | |
1111 | * @tiling_flags: new flags | |
1112 | * | |
1113 | * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or | |
1114 | * kernel driver to set the tiling flags on a buffer. | |
1115 | * | |
2472e11b MD |
1116 | * Returns: |
1117 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1118 | */ |
d38ceaf9 AD |
1119 | int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) |
1120 | { | |
9079ac76 | 1121 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
cc1bcf85 | 1122 | struct amdgpu_bo_user *ubo; |
9079ac76 | 1123 | |
030bb4ad | 1124 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
9079ac76 MO |
1125 | if (adev->family <= AMDGPU_FAMILY_CZ && |
1126 | AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) | |
d38ceaf9 | 1127 | return -EINVAL; |
d38ceaf9 | 1128 | |
cc1bcf85 ND |
1129 | ubo = to_amdgpu_bo_user(bo); |
1130 | ubo->tiling_flags = tiling_flags; | |
d38ceaf9 AD |
1131 | return 0; |
1132 | } | |
1133 | ||
6f4e8d6e SL |
1134 | /** |
1135 | * amdgpu_bo_get_tiling_flags - get tiling flags | |
1136 | * @bo: &amdgpu_bo buffer object | |
1137 | * @tiling_flags: returned flags | |
1138 | * | |
1139 | * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to | |
1140 | * set the tiling flags on a buffer. | |
1141 | */ | |
d38ceaf9 AD |
1142 | void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) |
1143 | { | |
cc1bcf85 ND |
1144 | struct amdgpu_bo_user *ubo; |
1145 | ||
030bb4ad | 1146 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
52791eee | 1147 | dma_resv_assert_held(bo->tbo.base.resv); |
cc1bcf85 | 1148 | ubo = to_amdgpu_bo_user(bo); |
d38ceaf9 AD |
1149 | |
1150 | if (tiling_flags) | |
cc1bcf85 | 1151 | *tiling_flags = ubo->tiling_flags; |
d38ceaf9 AD |
1152 | } |
1153 | ||
6f4e8d6e SL |
1154 | /** |
1155 | * amdgpu_bo_set_metadata - set metadata | |
1156 | * @bo: &amdgpu_bo buffer object | |
1157 | * @metadata: new metadata | |
1158 | * @metadata_size: size of the new metadata | |
1159 | * @flags: flags of the new metadata | |
1160 | * | |
1161 | * Sets buffer object's metadata, its size and flags. | |
1162 | * Used via GEM ioctl. | |
1163 | * | |
2472e11b MD |
1164 | * Returns: |
1165 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1166 | */ |
1d6ecab1 SS |
1167 | int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, |
1168 | u32 metadata_size, uint64_t flags) | |
d38ceaf9 | 1169 | { |
cc1bcf85 | 1170 | struct amdgpu_bo_user *ubo; |
d38ceaf9 AD |
1171 | void *buffer; |
1172 | ||
030bb4ad | 1173 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
cc1bcf85 | 1174 | ubo = to_amdgpu_bo_user(bo); |
d38ceaf9 | 1175 | if (!metadata_size) { |
cc1bcf85 ND |
1176 | if (ubo->metadata_size) { |
1177 | kfree(ubo->metadata); | |
1178 | ubo->metadata = NULL; | |
1179 | ubo->metadata_size = 0; | |
d38ceaf9 AD |
1180 | } |
1181 | return 0; | |
1182 | } | |
1183 | ||
1184 | if (metadata == NULL) | |
1185 | return -EINVAL; | |
1186 | ||
71affda5 | 1187 | buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); |
d38ceaf9 AD |
1188 | if (buffer == NULL) |
1189 | return -ENOMEM; | |
1190 | ||
cc1bcf85 ND |
1191 | kfree(ubo->metadata); |
1192 | ubo->metadata_flags = flags; | |
1193 | ubo->metadata = buffer; | |
1194 | ubo->metadata_size = metadata_size; | |
d38ceaf9 AD |
1195 | |
1196 | return 0; | |
1197 | } | |
1198 | ||
6f4e8d6e SL |
1199 | /** |
1200 | * amdgpu_bo_get_metadata - get metadata | |
1201 | * @bo: &amdgpu_bo buffer object | |
1202 | * @buffer: returned metadata | |
1203 | * @buffer_size: size of the buffer | |
1204 | * @metadata_size: size of the returned metadata | |
1205 | * @flags: flags of the returned metadata | |
1206 | * | |
1207 | * Gets buffer object's metadata, its size and flags. buffer_size shall not be | |
1208 | * less than metadata_size. | |
1209 | * Used via GEM ioctl. | |
1210 | * | |
2472e11b MD |
1211 | * Returns: |
1212 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1213 | */ |
d38ceaf9 AD |
1214 | int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, |
1215 | size_t buffer_size, uint32_t *metadata_size, | |
1216 | uint64_t *flags) | |
1217 | { | |
cc1bcf85 ND |
1218 | struct amdgpu_bo_user *ubo; |
1219 | ||
d38ceaf9 AD |
1220 | if (!buffer && !metadata_size) |
1221 | return -EINVAL; | |
1222 | ||
030bb4ad | 1223 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
cc1bcf85 | 1224 | ubo = to_amdgpu_bo_user(bo); |
eba98523 SZ |
1225 | if (metadata_size) |
1226 | *metadata_size = ubo->metadata_size; | |
1227 | ||
d38ceaf9 | 1228 | if (buffer) { |
cc1bcf85 | 1229 | if (buffer_size < ubo->metadata_size) |
d38ceaf9 AD |
1230 | return -EINVAL; |
1231 | ||
cc1bcf85 ND |
1232 | if (ubo->metadata_size) |
1233 | memcpy(buffer, ubo->metadata, ubo->metadata_size); | |
d38ceaf9 AD |
1234 | } |
1235 | ||
d38ceaf9 | 1236 | if (flags) |
cc1bcf85 | 1237 | *flags = ubo->metadata_flags; |
d38ceaf9 AD |
1238 | |
1239 | return 0; | |
1240 | } | |
1241 | ||
6f4e8d6e SL |
1242 | /** |
1243 | * amdgpu_bo_move_notify - notification about a memory move | |
1244 | * @bo: pointer to a buffer object | |
1245 | * @evict: if this move is evicting the buffer from the graphics address space | |
d3a9331a | 1246 | * @new_mem: new resource for backing the BO |
6f4e8d6e SL |
1247 | * |
1248 | * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs | |
1249 | * bookkeeping. | |
1250 | * TTM driver callback which is called when ttm moves a buffer. | |
1251 | */ | |
d3a9331a CK |
1252 | void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, |
1253 | bool evict, | |
1254 | struct ttm_resource *new_mem) | |
d38ceaf9 | 1255 | { |
a7d64de6 | 1256 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
d3a9331a | 1257 | struct ttm_resource *old_mem = bo->resource; |
765e7fbf | 1258 | struct amdgpu_bo *abo; |
d38ceaf9 | 1259 | |
c704ab18 | 1260 | if (!amdgpu_bo_is_amdgpu_bo(bo)) |
d38ceaf9 AD |
1261 | return; |
1262 | ||
b82485fd | 1263 | abo = ttm_to_amdgpu_bo(bo); |
3f3333f8 | 1264 | amdgpu_vm_bo_invalidate(adev, abo, evict); |
d38ceaf9 | 1265 | |
6375bbb4 CK |
1266 | amdgpu_bo_kunmap(abo); |
1267 | ||
2d4dad27 | 1268 | if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach && |
d3a9331a | 1269 | old_mem && old_mem->mem_type != TTM_PL_SYSTEM) |
2d4dad27 CK |
1270 | dma_buf_move_notify(abo->tbo.base.dma_buf); |
1271 | ||
d3a9331a CK |
1272 | /* move_notify is called before move happens */ |
1273 | trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1, | |
1274 | old_mem ? old_mem->mem_type : -1); | |
d38ceaf9 AD |
1275 | } |
1276 | ||
d6530c33 MO |
1277 | void amdgpu_bo_get_memory(struct amdgpu_bo *bo, |
1278 | struct amdgpu_mem_stats *stats) | |
87444254 | 1279 | { |
a6ff969f CK |
1280 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
1281 | struct ttm_resource *res = bo->tbo.resource; | |
d6530c33 | 1282 | uint64_t size = amdgpu_bo_size(bo); |
ba1a58d5 | 1283 | struct drm_gem_object *obj; |
ca0b954a | 1284 | unsigned int domain; |
ba1a58d5 | 1285 | bool shared; |
ca0b954a CK |
1286 | |
1287 | /* Abort if the BO doesn't currently have a backing store */ | |
a6ff969f | 1288 | if (!res) |
ca0b954a | 1289 | return; |
87444254 | 1290 | |
ba1a58d5 AD |
1291 | obj = &bo->tbo.base; |
1292 | shared = drm_gem_object_is_shared_for_memory_stats(obj); | |
1293 | ||
a6ff969f | 1294 | domain = amdgpu_mem_type_to_domain(res->mem_type); |
87444254 RS |
1295 | switch (domain) { |
1296 | case AMDGPU_GEM_DOMAIN_VRAM: | |
d6530c33 | 1297 | stats->vram += size; |
a6ff969f | 1298 | if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) |
d6530c33 | 1299 | stats->visible_vram += size; |
ba1a58d5 AD |
1300 | if (shared) |
1301 | stats->vram_shared += size; | |
87444254 RS |
1302 | break; |
1303 | case AMDGPU_GEM_DOMAIN_GTT: | |
d6530c33 | 1304 | stats->gtt += size; |
ba1a58d5 AD |
1305 | if (shared) |
1306 | stats->gtt_shared += size; | |
87444254 RS |
1307 | break; |
1308 | case AMDGPU_GEM_DOMAIN_CPU: | |
1309 | default: | |
d6530c33 | 1310 | stats->cpu += size; |
ba1a58d5 AD |
1311 | if (shared) |
1312 | stats->cpu_shared += size; | |
87444254 RS |
1313 | break; |
1314 | } | |
d6530c33 MO |
1315 | |
1316 | if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) { | |
1317 | stats->requested_vram += size; | |
1318 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) | |
1319 | stats->requested_visible_vram += size; | |
1320 | ||
1321 | if (domain != AMDGPU_GEM_DOMAIN_VRAM) { | |
1322 | stats->evicted_vram += size; | |
1323 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) | |
1324 | stats->evicted_visible_vram += size; | |
1325 | } | |
1326 | } else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT) { | |
1327 | stats->requested_gtt += size; | |
1328 | } | |
87444254 RS |
1329 | } |
1330 | ||
ab2f7a5c | 1331 | /** |
736b1729 | 1332 | * amdgpu_bo_release_notify - notification about a BO being released |
ab2f7a5c FK |
1333 | * @bo: pointer to a buffer object |
1334 | * | |
1335 | * Wipes VRAM buffers whose contents should not be leaked before the | |
1336 | * memory is released. | |
1337 | */ | |
1338 | void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) | |
1339 | { | |
32f90e65 | 1340 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
ab2f7a5c FK |
1341 | struct dma_fence *fence = NULL; |
1342 | struct amdgpu_bo *abo; | |
1343 | int r; | |
1344 | ||
1345 | if (!amdgpu_bo_is_amdgpu_bo(bo)) | |
1346 | return; | |
1347 | ||
1348 | abo = ttm_to_amdgpu_bo(bo); | |
1349 | ||
65d2765d CK |
1350 | WARN_ON(abo->vm_bo); |
1351 | ||
ab2f7a5c | 1352 | if (abo->kfd_bo) |
5702d052 | 1353 | amdgpu_amdkfd_release_notify(abo); |
ab2f7a5c | 1354 | |
f4a3c42b | 1355 | /* We only remove the fence if the resv has individualized. */ |
9fe58d0b | 1356 | WARN_ON_ONCE(bo->type == ttm_bo_type_kernel |
1357 | && bo->base.resv != &bo->base._resv); | |
f4a3c42b | 1358 | if (bo->base.resv == &bo->base._resv) |
1359 | amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo); | |
1360 | ||
63af82cf | 1361 | if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || |
32f90e65 | 1362 | !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || |
93bb18d2 | 1363 | adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) |
ab2f7a5c FK |
1364 | return; |
1365 | ||
447c7997 RB |
1366 | if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv))) |
1367 | return; | |
ab2f7a5c | 1368 | |
c3aaca43 | 1369 | r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence, true); |
ab2f7a5c FK |
1370 | if (!WARN_ON(r)) { |
1371 | amdgpu_bo_fence(abo, fence, false); | |
1372 | dma_fence_put(fence); | |
1373 | } | |
1374 | ||
5f680625 | 1375 | dma_resv_unlock(bo->base.resv); |
ab2f7a5c FK |
1376 | } |
1377 | ||
6f4e8d6e SL |
1378 | /** |
1379 | * amdgpu_bo_fault_reserve_notify - notification about a memory fault | |
1380 | * @bo: pointer to a buffer object | |
1381 | * | |
1382 | * Notifies the driver we are taking a fault on this BO and have reserved it, | |
1383 | * also performs bookkeeping. | |
1384 | * TTM driver callback for dealing with vm faults. | |
1385 | * | |
2472e11b MD |
1386 | * Returns: |
1387 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1388 | */ |
d3ef581a | 1389 | vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) |
d38ceaf9 | 1390 | { |
a7d64de6 | 1391 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
19be5570 | 1392 | struct ttm_operation_ctx ctx = { false, false }; |
d3ef581a | 1393 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); |
96cf8271 | 1394 | int r; |
d38ceaf9 | 1395 | |
96cf8271 JB |
1396 | /* Remember that this BO was accessed by the CPU */ |
1397 | abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
1398 | ||
a6ff969f | 1399 | if (amdgpu_res_cpu_visible(adev, bo->resource)) |
5fb1941d CK |
1400 | return 0; |
1401 | ||
104ece97 | 1402 | /* Can't move a pinned BO to visible VRAM */ |
4671078e | 1403 | if (abo->tbo.pin_count > 0) |
d3ef581a | 1404 | return VM_FAULT_SIGBUS; |
104ece97 | 1405 | |
5fb1941d | 1406 | /* hurrah the memory is not visible ! */ |
68e2c5ff | 1407 | atomic64_inc(&adev->num_vram_cpu_page_faults); |
c704ab18 CK |
1408 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | |
1409 | AMDGPU_GEM_DOMAIN_GTT); | |
41d9a6a7 JB |
1410 | |
1411 | /* Avoid costly evictions; only set GTT as a busy placement */ | |
a78a8da5 | 1412 | abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; |
41d9a6a7 | 1413 | |
19be5570 | 1414 | r = ttm_bo_validate(bo, &abo->placement, &ctx); |
d3ef581a CK |
1415 | if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) |
1416 | return VM_FAULT_NOPAGE; | |
1417 | else if (unlikely(r)) | |
1418 | return VM_FAULT_SIGBUS; | |
5fb1941d | 1419 | |
5fb1941d | 1420 | /* this should never happen */ |
d3116756 | 1421 | if (bo->resource->mem_type == TTM_PL_VRAM && |
a6ff969f | 1422 | !amdgpu_res_cpu_visible(adev, bo->resource)) |
d3ef581a | 1423 | return VM_FAULT_SIGBUS; |
5fb1941d | 1424 | |
d3ef581a | 1425 | ttm_bo_move_to_lru_tail_unlocked(bo); |
d38ceaf9 AD |
1426 | return 0; |
1427 | } | |
1428 | ||
1429 | /** | |
1430 | * amdgpu_bo_fence - add fence to buffer object | |
1431 | * | |
1432 | * @bo: buffer object in question | |
1433 | * @fence: fence to add | |
1434 | * @shared: true if fence should be added shared | |
1435 | * | |
1436 | */ | |
f54d1867 | 1437 | void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, |
d38ceaf9 AD |
1438 | bool shared) |
1439 | { | |
52791eee | 1440 | struct dma_resv *resv = bo->tbo.base.resv; |
c8d4c18b CK |
1441 | int r; |
1442 | ||
1443 | r = dma_resv_reserve_fences(resv, 1); | |
1444 | if (r) { | |
1445 | /* As last resort on OOM we block for the fence */ | |
1446 | dma_fence_wait(fence, false); | |
1447 | return; | |
1448 | } | |
d38ceaf9 | 1449 | |
73511edf CK |
1450 | dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : |
1451 | DMA_RESV_USAGE_WRITE); | |
d38ceaf9 | 1452 | } |
cdb7e8f2 | 1453 | |
e8e32426 | 1454 | /** |
9f3cc18d | 1455 | * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences |
e8e32426 | 1456 | * |
9f3cc18d CK |
1457 | * @adev: amdgpu device pointer |
1458 | * @resv: reservation object to sync to | |
1459 | * @sync_mode: synchronization mode | |
e8e32426 FK |
1460 | * @owner: fence owner |
1461 | * @intr: Whether the wait is interruptible | |
1462 | * | |
9f3cc18d CK |
1463 | * Extract the fences from the reservation object and waits for them to finish. |
1464 | * | |
e8e32426 FK |
1465 | * Returns: |
1466 | * 0 on success, errno otherwise. | |
1467 | */ | |
9f3cc18d CK |
1468 | int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, |
1469 | enum amdgpu_sync_mode sync_mode, void *owner, | |
1470 | bool intr) | |
e8e32426 | 1471 | { |
e8e32426 FK |
1472 | struct amdgpu_sync sync; |
1473 | int r; | |
1474 | ||
1475 | amdgpu_sync_create(&sync); | |
9f3cc18d | 1476 | amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); |
e8e32426 FK |
1477 | r = amdgpu_sync_wait(&sync, intr); |
1478 | amdgpu_sync_free(&sync); | |
e8e32426 FK |
1479 | return r; |
1480 | } | |
1481 | ||
9f3cc18d CK |
1482 | /** |
1483 | * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv | |
1484 | * @bo: buffer object to wait for | |
1485 | * @owner: fence owner | |
1486 | * @intr: Whether the wait is interruptible | |
1487 | * | |
1488 | * Wrapper to wait for fences in a BO. | |
1489 | * Returns: | |
1490 | * 0 on success, errno otherwise. | |
1491 | */ | |
1492 | int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) | |
1493 | { | |
1494 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
1495 | ||
1496 | return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, | |
1497 | AMDGPU_SYNC_NE_OWNER, owner, intr); | |
1498 | } | |
1499 | ||
cdb7e8f2 CK |
1500 | /** |
1501 | * amdgpu_bo_gpu_offset - return GPU offset of bo | |
1502 | * @bo: amdgpu object for which we query the offset | |
1503 | * | |
cdb7e8f2 CK |
1504 | * Note: object should either be pinned or reserved when calling this |
1505 | * function, it might be useful to add check for this for debugging. | |
2472e11b MD |
1506 | * |
1507 | * Returns: | |
1508 | * current GPU offset of the object. | |
cdb7e8f2 CK |
1509 | */ |
1510 | u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) | |
1511 | { | |
d3116756 | 1512 | WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); |
52791eee | 1513 | WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && |
4671078e | 1514 | !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); |
d3116756 CK |
1515 | WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); |
1516 | WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && | |
03f48dd5 | 1517 | !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); |
cdb7e8f2 | 1518 | |
b1a8ef95 ND |
1519 | return amdgpu_bo_gpu_offset_no_check(bo); |
1520 | } | |
1521 | ||
1522 | /** | |
1523 | * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo | |
1524 | * @bo: amdgpu object for which we query the offset | |
1525 | * | |
1526 | * Returns: | |
1527 | * current GPU offset of the object without raising warnings. | |
1528 | */ | |
1529 | u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) | |
1530 | { | |
1531 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
ca0b0069 | 1532 | uint64_t offset = AMDGPU_BO_INVALID_OFFSET; |
b1a8ef95 | 1533 | |
ca0b0069 AD |
1534 | if (bo->tbo.resource->mem_type == TTM_PL_TT) |
1535 | offset = amdgpu_gmc_agp_addr(&bo->tbo); | |
b1a8ef95 | 1536 | |
ca0b0069 AD |
1537 | if (offset == AMDGPU_BO_INVALID_OFFSET) |
1538 | offset = (bo->tbo.resource->start << PAGE_SHIFT) + | |
1539 | amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); | |
b1a8ef95 ND |
1540 | |
1541 | return amdgpu_gmc_sign_extend(offset); | |
cdb7e8f2 | 1542 | } |
84b74608 | 1543 | |
2472e11b | 1544 | /** |
d035f84d | 1545 | * amdgpu_bo_get_preferred_domain - get preferred domain |
2472e11b MD |
1546 | * @adev: amdgpu device object |
1547 | * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` | |
1548 | * | |
1549 | * Returns: | |
d035f84d | 1550 | * Which of the allowed domains is preferred for allocating the BO. |
2472e11b | 1551 | */ |
d035f84d | 1552 | uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, |
84b74608 DS |
1553 | uint32_t domain) |
1554 | { | |
81d0bcf9 AD |
1555 | if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && |
1556 | ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { | |
84b74608 DS |
1557 | domain = AMDGPU_GEM_DOMAIN_VRAM; |
1558 | if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) | |
1559 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
1560 | } | |
1561 | return domain; | |
1562 | } | |
ff72bc40 MBP |
1563 | |
1564 | #if defined(CONFIG_DEBUG_FS) | |
1565 | #define amdgpu_bo_print_flag(m, bo, flag) \ | |
1566 | do { \ | |
1567 | if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ | |
1568 | seq_printf((m), " " #flag); \ | |
1569 | } \ | |
1570 | } while (0) | |
1571 | ||
1572 | /** | |
25dd7a44 | 1573 | * amdgpu_bo_print_info - print BO info in debugfs file |
ff72bc40 MBP |
1574 | * |
1575 | * @id: Index or Id of the BO | |
1576 | * @bo: Requested BO for printing info | |
1577 | * @m: debugfs file | |
1578 | * | |
1579 | * Print BO information in debugfs file | |
1580 | * | |
1581 | * Returns: | |
1582 | * Size of the BO in bytes. | |
1583 | */ | |
1584 | u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) | |
1585 | { | |
a6ff969f | 1586 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
ff72bc40 MBP |
1587 | struct dma_buf_attachment *attachment; |
1588 | struct dma_buf *dma_buf; | |
ff72bc40 MBP |
1589 | const char *placement; |
1590 | unsigned int pin_count; | |
1591 | u64 size; | |
1592 | ||
818c158f PEPP |
1593 | if (dma_resv_trylock(bo->tbo.base.resv)) { |
1594 | unsigned int domain; | |
a6ff969f | 1595 | |
818c158f PEPP |
1596 | domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); |
1597 | switch (domain) { | |
1598 | case AMDGPU_GEM_DOMAIN_VRAM: | |
a6ff969f | 1599 | if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) |
818c158f PEPP |
1600 | placement = "VRAM VISIBLE"; |
1601 | else | |
1602 | placement = "VRAM"; | |
1603 | break; | |
1604 | case AMDGPU_GEM_DOMAIN_GTT: | |
1605 | placement = "GTT"; | |
1606 | break; | |
1607 | case AMDGPU_GEM_DOMAIN_CPU: | |
1608 | default: | |
1609 | placement = "CPU"; | |
1610 | break; | |
1611 | } | |
1612 | dma_resv_unlock(bo->tbo.base.resv); | |
1613 | } else { | |
1614 | placement = "UNKNOWN"; | |
ff72bc40 MBP |
1615 | } |
1616 | ||
1617 | size = amdgpu_bo_size(bo); | |
1618 | seq_printf(m, "\t\t0x%08x: %12lld byte %s", | |
1619 | id, size, placement); | |
1620 | ||
5b8c5969 | 1621 | pin_count = READ_ONCE(bo->tbo.pin_count); |
ff72bc40 MBP |
1622 | if (pin_count) |
1623 | seq_printf(m, " pin count %d", pin_count); | |
1624 | ||
1625 | dma_buf = READ_ONCE(bo->tbo.base.dma_buf); | |
1626 | attachment = READ_ONCE(bo->tbo.base.import_attach); | |
1627 | ||
1628 | if (attachment) | |
26fd808b | 1629 | seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino); |
ff72bc40 | 1630 | else if (dma_buf) |
26fd808b | 1631 | seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino); |
ff72bc40 MBP |
1632 | |
1633 | amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); | |
1634 | amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); | |
1635 | amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); | |
1636 | amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); | |
ff72bc40 MBP |
1637 | amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); |
1638 | amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); | |
1639 | amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); | |
1640 | ||
1641 | seq_puts(m, "\n"); | |
1642 | ||
1643 | return size; | |
1644 | } | |
1645 | #endif |