Commit | Line | Data |
---|---|---|
d38ceaf9 AD |
1 | /* |
2 | * Copyright 2009 Jerome Glisse. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sub license, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
20 | * | |
21 | * The above copyright notice and this permission notice (including the | |
22 | * next paragraph) shall be included in all copies or substantial portions | |
23 | * of the Software. | |
24 | * | |
25 | */ | |
26 | /* | |
27 | * Authors: | |
28 | * Jerome Glisse <glisse@freedesktop.org> | |
29 | * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> | |
30 | * Dave Airlie | |
31 | */ | |
32 | #include <linux/list.h> | |
33 | #include <linux/slab.h> | |
2d4dad27 | 34 | #include <linux/dma-buf.h> |
fdf2f6c5 | 35 | |
62d5f9f7 | 36 | #include <drm/drm_drv.h> |
d38ceaf9 | 37 | #include <drm/amdgpu_drm.h> |
a187f17f | 38 | #include <drm/drm_cache.h> |
d38ceaf9 AD |
39 | #include "amdgpu.h" |
40 | #include "amdgpu_trace.h" | |
a46a2cd1 | 41 | #include "amdgpu_amdkfd.h" |
d38ceaf9 | 42 | |
6f4e8d6e SL |
43 | /** |
44 | * DOC: amdgpu_object | |
45 | * | |
46 | * This defines the interfaces to operate on an &amdgpu_bo buffer object which | |
47 | * represents memory used by driver (VRAM, system memory, etc.). The driver | |
48 | * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces | |
49 | * to create/destroy/set buffer object which are then managed by the kernel TTM | |
50 | * memory manager. | |
51 | * The interfaces are also used internally by kernel clients, including gfx, | |
52 | * uvd, etc. for kernel managed allocations used by the GPU. | |
53 | * | |
54 | */ | |
55 | ||
c704ab18 | 56 | static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) |
d38ceaf9 | 57 | { |
b82485fd | 58 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); |
d38ceaf9 | 59 | |
6375bbb4 | 60 | amdgpu_bo_kunmap(bo); |
d38ceaf9 | 61 | |
c105de28 GH |
62 | if (bo->tbo.base.import_attach) |
63 | drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); | |
64 | drm_gem_object_release(&bo->tbo.base); | |
23e24fbb ND |
65 | amdgpu_bo_unref(&bo->parent); |
66 | kvfree(bo); | |
67 | } | |
68 | ||
69 | static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) | |
70 | { | |
71 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); | |
72 | struct amdgpu_bo_user *ubo; | |
73 | ||
74 | ubo = to_amdgpu_bo_user(bo); | |
75 | kfree(ubo->metadata); | |
76 | amdgpu_bo_destroy(tbo); | |
77 | } | |
78 | ||
79 | static void amdgpu_bo_vm_destroy(struct ttm_buffer_object *tbo) | |
80 | { | |
81 | struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); | |
cbb63ecc | 82 | struct amdgpu_bo *shadow_bo = ttm_to_amdgpu_bo(tbo), *bo; |
e18aaea7 | 83 | struct amdgpu_bo_vm *vmbo; |
23e24fbb | 84 | |
cbb63ecc | 85 | bo = shadow_bo->parent; |
e18aaea7 | 86 | vmbo = to_amdgpu_bo_vm(bo); |
36e499b2 | 87 | /* in case amdgpu_device_recover_vram got NULL of bo->parent */ |
e18aaea7 | 88 | if (!list_empty(&vmbo->shadow_list)) { |
a7d64de6 | 89 | mutex_lock(&adev->shadow_list_lock); |
e18aaea7 | 90 | list_del_init(&vmbo->shadow_list); |
a7d64de6 | 91 | mutex_unlock(&adev->shadow_list_lock); |
0c4e7fa5 | 92 | } |
36e499b2 | 93 | |
23e24fbb | 94 | amdgpu_bo_destroy(tbo); |
d38ceaf9 AD |
95 | } |
96 | ||
6f4e8d6e | 97 | /** |
c704ab18 | 98 | * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo |
6f4e8d6e SL |
99 | * @bo: buffer object to be checked |
100 | * | |
101 | * Uses destroy function associated with the object to determine if this is | |
102 | * an &amdgpu_bo. | |
103 | * | |
2472e11b MD |
104 | * Returns: |
105 | * true if the object belongs to &amdgpu_bo, false if not. | |
6f4e8d6e | 106 | */ |
c704ab18 | 107 | bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) |
d38ceaf9 | 108 | { |
23e24fbb ND |
109 | if (bo->destroy == &amdgpu_bo_destroy || |
110 | bo->destroy == &amdgpu_bo_user_destroy || | |
111 | bo->destroy == &amdgpu_bo_vm_destroy) | |
d38ceaf9 | 112 | return true; |
23e24fbb | 113 | |
d38ceaf9 AD |
114 | return false; |
115 | } | |
116 | ||
6f4e8d6e | 117 | /** |
c704ab18 | 118 | * amdgpu_bo_placement_from_domain - set buffer's placement |
6f4e8d6e SL |
119 | * @abo: &amdgpu_bo buffer object whose placement is to be set |
120 | * @domain: requested domain | |
121 | * | |
122 | * Sets buffer's placement according to requested domain and the buffer's | |
123 | * flags. | |
124 | */ | |
c704ab18 | 125 | void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) |
d38ceaf9 | 126 | { |
c09312a6 CK |
127 | struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); |
128 | struct ttm_placement *placement = &abo->placement; | |
129 | struct ttm_place *places = abo->placements; | |
130 | u64 flags = abo->flags; | |
6369f6f1 | 131 | u32 c = 0; |
7e5a547f | 132 | |
d38ceaf9 | 133 | if (domain & AMDGPU_GEM_DOMAIN_VRAM) { |
1d6ecab1 | 134 | unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; |
3ebfd221 | 135 | int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); |
faceaf6a | 136 | |
3ebfd221 PY |
137 | if (adev->gmc.mem_partitions && mem_id >= 0) { |
138 | places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; | |
6cfba94a PY |
139 | /* |
140 | * memory partition range lpfn is inclusive start + size - 1 | |
141 | * TTM place lpfn is exclusive start + size | |
142 | */ | |
3ebfd221 | 143 | places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; |
7f6db894 PY |
144 | } else { |
145 | places[c].fpfn = 0; | |
146 | places[c].lpfn = 0; | |
147 | } | |
48e07c23 | 148 | places[c].mem_type = TTM_PL_VRAM; |
ce65b874 | 149 | places[c].flags = 0; |
89bb5752 | 150 | |
faceaf6a | 151 | if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) |
7f6db894 | 152 | places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); |
59eddd4e | 153 | else |
faceaf6a | 154 | places[c].flags |= TTM_PL_FLAG_TOPDOWN; |
89bb5752 CK |
155 | |
156 | if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) | |
157 | places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; | |
faceaf6a | 158 | c++; |
d38ceaf9 AD |
159 | } |
160 | ||
dc3499c7 AD |
161 | if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { |
162 | places[c].fpfn = 0; | |
163 | places[c].lpfn = 0; | |
164 | places[c].mem_type = AMDGPU_PL_DOORBELL; | |
165 | places[c].flags = 0; | |
166 | c++; | |
167 | } | |
168 | ||
d38ceaf9 | 169 | if (domain & AMDGPU_GEM_DOMAIN_GTT) { |
faceaf6a | 170 | places[c].fpfn = 0; |
3d5fe658 | 171 | places[c].lpfn = 0; |
b453e42a FK |
172 | places[c].mem_type = |
173 | abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? | |
174 | AMDGPU_PL_PREEMPT : TTM_PL_TT; | |
48e07c23 | 175 | places[c].flags = 0; |
216c1282 CK |
176 | /* |
177 | * When GTT is just an alternative to VRAM make sure that we | |
178 | * only use it as fallback and still try to fill up VRAM first. | |
179 | */ | |
180 | if (domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) | |
181 | places[c].flags |= TTM_PL_FLAG_FALLBACK; | |
faceaf6a | 182 | c++; |
d38ceaf9 AD |
183 | } |
184 | ||
185 | if (domain & AMDGPU_GEM_DOMAIN_CPU) { | |
faceaf6a CK |
186 | places[c].fpfn = 0; |
187 | places[c].lpfn = 0; | |
48e07c23 CK |
188 | places[c].mem_type = TTM_PL_SYSTEM; |
189 | places[c].flags = 0; | |
faceaf6a | 190 | c++; |
d38ceaf9 AD |
191 | } |
192 | ||
193 | if (domain & AMDGPU_GEM_DOMAIN_GDS) { | |
faceaf6a CK |
194 | places[c].fpfn = 0; |
195 | places[c].lpfn = 0; | |
48e07c23 | 196 | places[c].mem_type = AMDGPU_PL_GDS; |
ce65b874 | 197 | places[c].flags = 0; |
faceaf6a | 198 | c++; |
d38ceaf9 | 199 | } |
faceaf6a | 200 | |
d38ceaf9 | 201 | if (domain & AMDGPU_GEM_DOMAIN_GWS) { |
faceaf6a CK |
202 | places[c].fpfn = 0; |
203 | places[c].lpfn = 0; | |
48e07c23 | 204 | places[c].mem_type = AMDGPU_PL_GWS; |
ce65b874 | 205 | places[c].flags = 0; |
faceaf6a | 206 | c++; |
d38ceaf9 | 207 | } |
faceaf6a | 208 | |
d38ceaf9 | 209 | if (domain & AMDGPU_GEM_DOMAIN_OA) { |
faceaf6a CK |
210 | places[c].fpfn = 0; |
211 | places[c].lpfn = 0; | |
48e07c23 | 212 | places[c].mem_type = AMDGPU_PL_OA; |
ce65b874 | 213 | places[c].flags = 0; |
faceaf6a | 214 | c++; |
d38ceaf9 AD |
215 | } |
216 | ||
217 | if (!c) { | |
faceaf6a CK |
218 | places[c].fpfn = 0; |
219 | places[c].lpfn = 0; | |
48e07c23 | 220 | places[c].mem_type = TTM_PL_SYSTEM; |
ce65b874 | 221 | places[c].flags = 0; |
faceaf6a | 222 | c++; |
d38ceaf9 | 223 | } |
faceaf6a | 224 | |
ea7acd7c | 225 | BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); |
bf314ca3 | 226 | |
7e5a547f | 227 | placement->num_placement = c; |
faceaf6a | 228 | placement->placement = places; |
d38ceaf9 AD |
229 | } |
230 | ||
7c204889 | 231 | /** |
9d903cbd | 232 | * amdgpu_bo_create_reserved - create reserved BO for kernel use |
7c204889 CK |
233 | * |
234 | * @adev: amdgpu device object | |
235 | * @size: size for the new BO | |
236 | * @align: alignment for the new BO | |
237 | * @domain: where to place it | |
64350f1b | 238 | * @bo_ptr: used to initialize BOs in structures |
7c204889 CK |
239 | * @gpu_addr: GPU addr of the pinned BO |
240 | * @cpu_addr: optional CPU address mapping | |
241 | * | |
9d903cbd CK |
242 | * Allocates and pins a BO for kernel internal use, and returns it still |
243 | * reserved. | |
7c204889 | 244 | * |
64350f1b AG |
245 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
246 | * | |
2472e11b MD |
247 | * Returns: |
248 | * 0 on success, negative error code otherwise. | |
7c204889 | 249 | */ |
9d903cbd CK |
250 | int amdgpu_bo_create_reserved(struct amdgpu_device *adev, |
251 | unsigned long size, int align, | |
252 | u32 domain, struct amdgpu_bo **bo_ptr, | |
253 | u64 *gpu_addr, void **cpu_addr) | |
7c204889 | 254 | { |
3216c6b7 | 255 | struct amdgpu_bo_param bp; |
53766e5a | 256 | bool free = false; |
7c204889 CK |
257 | int r; |
258 | ||
21a7e77f CK |
259 | if (!size) { |
260 | amdgpu_bo_unref(bo_ptr); | |
261 | return 0; | |
262 | } | |
263 | ||
3216c6b7 CZ |
264 | memset(&bp, 0, sizeof(bp)); |
265 | bp.size = size; | |
266 | bp.byte_align = align; | |
267 | bp.domain = domain; | |
828d6fde TY |
268 | bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
269 | : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; | |
270 | bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; | |
3216c6b7 CZ |
271 | bp.type = ttm_bo_type_kernel; |
272 | bp.resv = NULL; | |
9fd5543e | 273 | bp.bo_ptr_size = sizeof(struct amdgpu_bo); |
3216c6b7 | 274 | |
53766e5a | 275 | if (!*bo_ptr) { |
3216c6b7 | 276 | r = amdgpu_bo_create(adev, &bp, bo_ptr); |
53766e5a CK |
277 | if (r) { |
278 | dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", | |
279 | r); | |
280 | return r; | |
281 | } | |
282 | free = true; | |
7c204889 CK |
283 | } |
284 | ||
285 | r = amdgpu_bo_reserve(*bo_ptr, false); | |
286 | if (r) { | |
287 | dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); | |
288 | goto error_free; | |
289 | } | |
290 | ||
7b7c6c81 | 291 | r = amdgpu_bo_pin(*bo_ptr, domain); |
7c204889 CK |
292 | if (r) { |
293 | dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); | |
294 | goto error_unreserve; | |
295 | } | |
bb812f1e JZ |
296 | |
297 | r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); | |
298 | if (r) { | |
299 | dev_err(adev->dev, "%p bind failed\n", *bo_ptr); | |
300 | goto error_unpin; | |
301 | } | |
302 | ||
7b7c6c81 JZ |
303 | if (gpu_addr) |
304 | *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); | |
7c204889 CK |
305 | |
306 | if (cpu_addr) { | |
307 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
308 | if (r) { | |
309 | dev_err(adev->dev, "(%d) kernel bo map failed\n", r); | |
dc407ee0 | 310 | goto error_unpin; |
7c204889 CK |
311 | } |
312 | } | |
313 | ||
7c204889 CK |
314 | return 0; |
315 | ||
bb812f1e JZ |
316 | error_unpin: |
317 | amdgpu_bo_unpin(*bo_ptr); | |
7c204889 CK |
318 | error_unreserve: |
319 | amdgpu_bo_unreserve(*bo_ptr); | |
320 | ||
321 | error_free: | |
53766e5a CK |
322 | if (free) |
323 | amdgpu_bo_unref(bo_ptr); | |
7c204889 CK |
324 | |
325 | return r; | |
326 | } | |
327 | ||
9d903cbd CK |
328 | /** |
329 | * amdgpu_bo_create_kernel - create BO for kernel use | |
330 | * | |
331 | * @adev: amdgpu device object | |
332 | * @size: size for the new BO | |
333 | * @align: alignment for the new BO | |
334 | * @domain: where to place it | |
64350f1b | 335 | * @bo_ptr: used to initialize BOs in structures |
9d903cbd CK |
336 | * @gpu_addr: GPU addr of the pinned BO |
337 | * @cpu_addr: optional CPU address mapping | |
338 | * | |
339 | * Allocates and pins a BO for kernel internal use. | |
340 | * | |
64350f1b AG |
341 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
342 | * | |
2472e11b MD |
343 | * Returns: |
344 | * 0 on success, negative error code otherwise. | |
9d903cbd CK |
345 | */ |
346 | int amdgpu_bo_create_kernel(struct amdgpu_device *adev, | |
347 | unsigned long size, int align, | |
348 | u32 domain, struct amdgpu_bo **bo_ptr, | |
349 | u64 *gpu_addr, void **cpu_addr) | |
350 | { | |
351 | int r; | |
352 | ||
353 | r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, | |
354 | gpu_addr, cpu_addr); | |
355 | ||
356 | if (r) | |
357 | return r; | |
358 | ||
ddaf5013 TSD |
359 | if (*bo_ptr) |
360 | amdgpu_bo_unreserve(*bo_ptr); | |
9d903cbd CK |
361 | |
362 | return 0; | |
363 | } | |
364 | ||
de7b45ba CK |
365 | /** |
366 | * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location | |
367 | * | |
368 | * @adev: amdgpu device object | |
369 | * @offset: offset of the BO | |
370 | * @size: size of the BO | |
de7b45ba CK |
371 | * @bo_ptr: used to initialize BOs in structures |
372 | * @cpu_addr: optional CPU address mapping | |
373 | * | |
3273f116 | 374 | * Creates a kernel BO at a specific offset in VRAM. |
de7b45ba CK |
375 | * |
376 | * Returns: | |
377 | * 0 on success, negative error code otherwise. | |
378 | */ | |
379 | int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, | |
3273f116 | 380 | uint64_t offset, uint64_t size, |
de7b45ba CK |
381 | struct amdgpu_bo **bo_ptr, void **cpu_addr) |
382 | { | |
383 | struct ttm_operation_ctx ctx = { false, false }; | |
384 | unsigned int i; | |
385 | int r; | |
386 | ||
387 | offset &= PAGE_MASK; | |
388 | size = ALIGN(size, PAGE_SIZE); | |
389 | ||
3273f116 LT |
390 | r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, |
391 | AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, | |
392 | cpu_addr); | |
de7b45ba CK |
393 | if (r) |
394 | return r; | |
395 | ||
37912e96 AD |
396 | if ((*bo_ptr) == NULL) |
397 | return 0; | |
398 | ||
de7b45ba CK |
399 | /* |
400 | * Remove the original mem node and create a new one at the request | |
401 | * position. | |
402 | */ | |
4a246528 CK |
403 | if (cpu_addr) |
404 | amdgpu_bo_kunmap(*bo_ptr); | |
405 | ||
bfa3357e | 406 | ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); |
4a246528 | 407 | |
de7b45ba CK |
408 | for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { |
409 | (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; | |
410 | (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; | |
411 | } | |
de7b45ba | 412 | r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, |
bfa3357e | 413 | &(*bo_ptr)->tbo.resource, &ctx); |
de7b45ba CK |
414 | if (r) |
415 | goto error; | |
416 | ||
417 | if (cpu_addr) { | |
418 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
419 | if (r) | |
420 | goto error; | |
421 | } | |
422 | ||
423 | amdgpu_bo_unreserve(*bo_ptr); | |
424 | return 0; | |
425 | ||
426 | error: | |
427 | amdgpu_bo_unreserve(*bo_ptr); | |
428 | amdgpu_bo_unref(bo_ptr); | |
429 | return r; | |
430 | } | |
431 | ||
aa1d562e JZ |
432 | /** |
433 | * amdgpu_bo_free_kernel - free BO for kernel use | |
434 | * | |
435 | * @bo: amdgpu BO to free | |
2472e11b MD |
436 | * @gpu_addr: pointer to where the BO's GPU memory space address was stored |
437 | * @cpu_addr: pointer to where the BO's CPU memory space address was stored | |
aa1d562e JZ |
438 | * |
439 | * unmaps and unpin a BO for kernel internal use. | |
440 | */ | |
441 | void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, | |
442 | void **cpu_addr) | |
443 | { | |
444 | if (*bo == NULL) | |
445 | return; | |
446 | ||
4d2ccd96 CK |
447 | WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); |
448 | ||
f3aa745e | 449 | if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { |
aa1d562e JZ |
450 | if (cpu_addr) |
451 | amdgpu_bo_kunmap(*bo); | |
452 | ||
453 | amdgpu_bo_unpin(*bo); | |
454 | amdgpu_bo_unreserve(*bo); | |
455 | } | |
456 | amdgpu_bo_unref(bo); | |
457 | ||
458 | if (gpu_addr) | |
459 | *gpu_addr = 0; | |
460 | ||
461 | if (cpu_addr) | |
462 | *cpu_addr = NULL; | |
463 | } | |
464 | ||
8f9a9a09 | 465 | /* Validate bo size is bit bigger than the request domain */ |
79c63123 AG |
466 | static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, |
467 | unsigned long size, u32 domain) | |
468 | { | |
9de59bc2 | 469 | struct ttm_resource_manager *man = NULL; |
79c63123 AG |
470 | |
471 | /* | |
472 | * If GTT is part of requested domains the check must succeed to | |
7554886d | 473 | * allow fall back to GTT. |
79c63123 | 474 | */ |
8f9a9a09 | 475 | if (domain & AMDGPU_GEM_DOMAIN_GTT) |
6c28aed6 | 476 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); |
8f9a9a09 | 477 | else if (domain & AMDGPU_GEM_DOMAIN_VRAM) |
6c28aed6 | 478 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); |
8f9a9a09 MJ |
479 | else |
480 | return true; | |
79c63123 | 481 | |
8f9a9a09 MJ |
482 | if (!man) { |
483 | if (domain & AMDGPU_GEM_DOMAIN_GTT) | |
484 | WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); | |
485 | return false; | |
79c63123 AG |
486 | } |
487 | ||
dc3499c7 | 488 | /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ |
8f9a9a09 MJ |
489 | if (size < man->size) |
490 | return true; | |
79c63123 | 491 | |
8f9a9a09 | 492 | DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); |
79c63123 AG |
493 | return false; |
494 | } | |
495 | ||
3d1b8ec7 AG |
496 | bool amdgpu_bo_support_uswc(u64 bo_flags) |
497 | { | |
498 | ||
499 | #ifdef CONFIG_X86_32 | |
500 | /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit | |
501 | * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 | |
502 | */ | |
503 | return false; | |
504 | #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) | |
505 | /* Don't try to enable write-combining when it can't work, or things | |
506 | * may be slow | |
507 | * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 | |
508 | */ | |
509 | ||
510 | #ifndef CONFIG_COMPILE_TEST | |
511 | #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ | |
512 | thanks to write-combining | |
513 | #endif | |
514 | ||
515 | if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
516 | DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " | |
517 | "better performance thanks to write-combining\n"); | |
518 | return false; | |
519 | #else | |
520 | /* For architectures that don't support WC memory, | |
521 | * mask out the WC flag from the BO | |
522 | */ | |
523 | if (!drm_arch_can_wc_memory()) | |
524 | return false; | |
525 | ||
526 | return true; | |
527 | #endif | |
528 | } | |
529 | ||
cd2454d6 ND |
530 | /** |
531 | * amdgpu_bo_create - create an &amdgpu_bo buffer object | |
532 | * @adev: amdgpu device object | |
533 | * @bp: parameters to be used for the buffer object | |
534 | * @bo_ptr: pointer to the buffer object pointer | |
535 | * | |
536 | * Creates an &amdgpu_bo buffer object. | |
537 | * | |
538 | * Returns: | |
539 | * 0 for success or a negative error code on failure. | |
540 | */ | |
541 | int amdgpu_bo_create(struct amdgpu_device *adev, | |
a906dbb1 | 542 | struct amdgpu_bo_param *bp, |
c09312a6 | 543 | struct amdgpu_bo **bo_ptr) |
d38ceaf9 | 544 | { |
9251859a | 545 | struct ttm_operation_ctx ctx = { |
a906dbb1 | 546 | .interruptible = (bp->type != ttm_bo_type_kernel), |
061468c4 | 547 | .no_wait_gpu = bp->no_wait_gpu, |
586052b0 CK |
548 | /* We opt to avoid OOM on system pages allocations */ |
549 | .gfp_retry_mayfail = true, | |
c44dfe4d CK |
550 | .allow_res_evict = bp->type != ttm_bo_type_kernel, |
551 | .resv = bp->resv | |
9251859a | 552 | }; |
d38ceaf9 | 553 | struct amdgpu_bo *bo; |
a906dbb1 | 554 | unsigned long page_align, size = bp->size; |
d38ceaf9 AD |
555 | int r; |
556 | ||
fe57085a MO |
557 | /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ |
558 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { | |
559 | /* GWS and OA don't need any alignment. */ | |
560 | page_align = bp->byte_align; | |
77a2faa5 | 561 | size <<= PAGE_SHIFT; |
e3c92eb4 | 562 | |
fe57085a MO |
563 | } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { |
564 | /* Both size and alignment must be a multiple of 4. */ | |
565 | page_align = ALIGN(bp->byte_align, 4); | |
566 | size = ALIGN(size, 4) << PAGE_SHIFT; | |
567 | } else { | |
568 | /* Memory should be aligned at least to a page size. */ | |
569 | page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; | |
77a2faa5 | 570 | size = ALIGN(size, PAGE_SIZE); |
fe57085a | 571 | } |
d38ceaf9 | 572 | |
a906dbb1 | 573 | if (!amdgpu_bo_validate_size(adev, size, bp->domain)) |
79c63123 AG |
574 | return -ENOMEM; |
575 | ||
9fd5543e | 576 | BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); |
d38ceaf9 | 577 | |
9fd5543e | 578 | *bo_ptr = NULL; |
31c759bb | 579 | bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); |
d38ceaf9 AD |
580 | if (bo == NULL) |
581 | return -ENOMEM; | |
4a580877 | 582 | drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); |
646b9025 | 583 | bo->vm_bo = NULL; |
3f188453 | 584 | bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : |
aa2b2e28 | 585 | bp->domain; |
08082104 | 586 | bo->allowed_domains = bo->preferred_domains; |
a906dbb1 | 587 | if (bp->type != ttm_bo_type_kernel && |
fab2cc83 | 588 | !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && |
08082104 CK |
589 | bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) |
590 | bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; | |
d38ceaf9 | 591 | |
a906dbb1 | 592 | bo->flags = bp->flags; |
a187f17f | 593 | |
3ebfd221 PY |
594 | if (adev->gmc.mem_partitions) |
595 | /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ | |
596 | bo->xcp_id = bp->xcp_id_plus1 - 1; | |
597 | else | |
598 | /* For GPUs without spatial partitioning */ | |
599 | bo->xcp_id = 0; | |
f24e924b | 600 | |
3d1b8ec7 | 601 | if (!amdgpu_bo_support_uswc(bo->flags)) |
a187f17f OG |
602 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; |
603 | ||
fc6ea4be FK |
604 | if (adev->ras_enabled) |
605 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; | |
606 | ||
c09312a6 | 607 | bo->tbo.bdev = &adev->mman.bdev; |
47722220 CK |
608 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | |
609 | AMDGPU_GEM_DOMAIN_GDS)) | |
610 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
611 | else | |
612 | amdgpu_bo_placement_from_domain(bo, bp->domain); | |
a50cb948 JZ |
613 | if (bp->type == ttm_bo_type_kernel) |
614 | bo->tbo.priority = 1; | |
08082104 | 615 | |
23e24fbb ND |
616 | if (!bp->destroy) |
617 | bp->destroy = &amdgpu_bo_destroy; | |
618 | ||
347987a2 | 619 | r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, |
f07069da | 620 | &bo->placement, page_align, &ctx, NULL, |
23e24fbb | 621 | bp->resv, bp->destroy); |
08082104 | 622 | if (unlikely(r != 0)) |
a695e437 CK |
623 | return r; |
624 | ||
c8c5e569 | 625 | if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && |
d3116756 | 626 | bo->tbo.resource->mem_type == TTM_PL_VRAM && |
aed01a68 | 627 | amdgpu_bo_in_cpu_visible_vram(bo)) |
6af046d2 CK |
628 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, |
629 | ctx.bytes_moved); | |
00f06b24 | 630 | else |
6af046d2 | 631 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); |
fad06127 | 632 | |
a906dbb1 | 633 | if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && |
d3116756 | 634 | bo->tbo.resource->mem_type == TTM_PL_VRAM) { |
f54d1867 | 635 | struct dma_fence *fence; |
4fea83ff | 636 | |
c3aaca43 | 637 | r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence, true); |
c3af1258 CK |
638 | if (unlikely(r)) |
639 | goto fail_unreserve; | |
640 | ||
8bb31587 CK |
641 | dma_resv_add_fence(bo->tbo.base.resv, fence, |
642 | DMA_RESV_USAGE_KERNEL); | |
f54d1867 | 643 | dma_fence_put(fence); |
4fea83ff | 644 | } |
a906dbb1 | 645 | if (!bp->resv) |
59c66c91 | 646 | amdgpu_bo_unreserve(bo); |
d38ceaf9 AD |
647 | *bo_ptr = bo; |
648 | ||
649 | trace_amdgpu_bo_create(bo); | |
650 | ||
96cf8271 | 651 | /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ |
a906dbb1 | 652 | if (bp->type == ttm_bo_type_device) |
96cf8271 JB |
653 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; |
654 | ||
d38ceaf9 | 655 | return 0; |
4fea83ff FC |
656 | |
657 | fail_unreserve: | |
a906dbb1 | 658 | if (!bp->resv) |
52791eee | 659 | dma_resv_unlock(bo->tbo.base.resv); |
4fea83ff FC |
660 | amdgpu_bo_unref(&bo); |
661 | return r; | |
d38ceaf9 AD |
662 | } |
663 | ||
9ad0d033 ND |
664 | /** |
665 | * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object | |
666 | * @adev: amdgpu device object | |
667 | * @bp: parameters to be used for the buffer object | |
668 | * @ubo_ptr: pointer to the buffer object pointer | |
669 | * | |
670 | * Create a BO to be used by user application; | |
671 | * | |
672 | * Returns: | |
673 | * 0 for success or a negative error code on failure. | |
674 | */ | |
675 | ||
676 | int amdgpu_bo_create_user(struct amdgpu_device *adev, | |
677 | struct amdgpu_bo_param *bp, | |
678 | struct amdgpu_bo_user **ubo_ptr) | |
679 | { | |
680 | struct amdgpu_bo *bo_ptr; | |
681 | int r; | |
682 | ||
9ad0d033 | 683 | bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); |
23e24fbb | 684 | bp->destroy = &amdgpu_bo_user_destroy; |
cd2454d6 | 685 | r = amdgpu_bo_create(adev, bp, &bo_ptr); |
9ad0d033 ND |
686 | if (r) |
687 | return r; | |
688 | ||
689 | *ubo_ptr = to_amdgpu_bo_user(bo_ptr); | |
690 | return r; | |
691 | } | |
6fdd6f4a ND |
692 | |
693 | /** | |
694 | * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object | |
695 | * @adev: amdgpu device object | |
696 | * @bp: parameters to be used for the buffer object | |
697 | * @vmbo_ptr: pointer to the buffer object pointer | |
698 | * | |
699 | * Create a BO to be for GPUVM. | |
700 | * | |
701 | * Returns: | |
702 | * 0 for success or a negative error code on failure. | |
703 | */ | |
704 | ||
705 | int amdgpu_bo_create_vm(struct amdgpu_device *adev, | |
706 | struct amdgpu_bo_param *bp, | |
707 | struct amdgpu_bo_vm **vmbo_ptr) | |
708 | { | |
709 | struct amdgpu_bo *bo_ptr; | |
710 | int r; | |
711 | ||
712 | /* bo_ptr_size will be determined by the caller and it depends on | |
713 | * num of amdgpu_vm_pt entries. | |
714 | */ | |
715 | BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); | |
716 | r = amdgpu_bo_create(adev, bp, &bo_ptr); | |
717 | if (r) | |
718 | return r; | |
719 | ||
720 | *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); | |
721 | return r; | |
722 | } | |
723 | ||
1fdc79f6 ND |
724 | /** |
725 | * amdgpu_bo_add_to_shadow_list - add a BO to the shadow list | |
726 | * | |
d8c33180 | 727 | * @vmbo: BO that will be inserted into the shadow list |
1fdc79f6 ND |
728 | * |
729 | * Insert a BO to the shadow list. | |
730 | */ | |
e18aaea7 | 731 | void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo) |
1fdc79f6 | 732 | { |
e18aaea7 | 733 | struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev); |
1fdc79f6 ND |
734 | |
735 | mutex_lock(&adev->shadow_list_lock); | |
e18aaea7 | 736 | list_add_tail(&vmbo->shadow_list, &adev->shadow_list); |
cbb63ecc HZ |
737 | vmbo->shadow->parent = amdgpu_bo_ref(&vmbo->bo); |
738 | vmbo->shadow->tbo.destroy = &amdgpu_bo_vm_destroy; | |
1fdc79f6 ND |
739 | mutex_unlock(&adev->shadow_list_lock); |
740 | } | |
741 | ||
6f4e8d6e | 742 | /** |
403009bf CK |
743 | * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow |
744 | * | |
745 | * @shadow: &amdgpu_bo shadow to be restored | |
6f4e8d6e | 746 | * @fence: dma_fence associated with the operation |
6f4e8d6e SL |
747 | * |
748 | * Copies a buffer object's shadow content back to the object. | |
749 | * This is used for recovering a buffer from its shadow in case of a gpu | |
750 | * reset where vram context may be lost. | |
751 | * | |
2472e11b MD |
752 | * Returns: |
753 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 754 | */ |
403009bf | 755 | int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence) |
20f4eff1 CZ |
756 | |
757 | { | |
403009bf CK |
758 | struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev); |
759 | struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; | |
760 | uint64_t shadow_addr, parent_addr; | |
20f4eff1 | 761 | |
403009bf CK |
762 | shadow_addr = amdgpu_bo_gpu_offset(shadow); |
763 | parent_addr = amdgpu_bo_gpu_offset(shadow->parent); | |
20f4eff1 | 764 | |
403009bf CK |
765 | return amdgpu_copy_buffer(ring, shadow_addr, parent_addr, |
766 | amdgpu_bo_size(shadow), NULL, fence, | |
c9dc9cfe | 767 | true, false, false); |
20f4eff1 CZ |
768 | } |
769 | ||
6f4e8d6e SL |
770 | /** |
771 | * amdgpu_bo_kmap - map an &amdgpu_bo buffer object | |
772 | * @bo: &amdgpu_bo buffer object to be mapped | |
773 | * @ptr: kernel virtual address to be returned | |
774 | * | |
775 | * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls | |
776 | * amdgpu_bo_kptr() to get the kernel virtual address. | |
777 | * | |
2472e11b MD |
778 | * Returns: |
779 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 780 | */ |
d38ceaf9 AD |
781 | int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) |
782 | { | |
f5e1c740 | 783 | void *kptr; |
587f3c70 | 784 | long r; |
d38ceaf9 | 785 | |
271c8125 CK |
786 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) |
787 | return -EPERM; | |
788 | ||
c35fcfa3 CK |
789 | r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, |
790 | false, MAX_SCHEDULE_TIMEOUT); | |
791 | if (r < 0) | |
792 | return r; | |
793 | ||
f5e1c740 CK |
794 | kptr = amdgpu_bo_kptr(bo); |
795 | if (kptr) { | |
796 | if (ptr) | |
797 | *ptr = kptr; | |
d38ceaf9 AD |
798 | return 0; |
799 | } | |
587f3c70 | 800 | |
e3c92eb4 | 801 | r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); |
587f3c70 | 802 | if (r) |
d38ceaf9 | 803 | return r; |
587f3c70 | 804 | |
587f3c70 | 805 | if (ptr) |
f5e1c740 | 806 | *ptr = amdgpu_bo_kptr(bo); |
587f3c70 | 807 | |
d38ceaf9 AD |
808 | return 0; |
809 | } | |
810 | ||
6f4e8d6e SL |
811 | /** |
812 | * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object | |
813 | * @bo: &amdgpu_bo buffer object | |
814 | * | |
815 | * Calls ttm_kmap_obj_virtual() to get the kernel virtual address | |
816 | * | |
2472e11b MD |
817 | * Returns: |
818 | * the virtual address of a buffer object area. | |
6f4e8d6e | 819 | */ |
f5e1c740 CK |
820 | void *amdgpu_bo_kptr(struct amdgpu_bo *bo) |
821 | { | |
822 | bool is_iomem; | |
823 | ||
824 | return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); | |
825 | } | |
826 | ||
6f4e8d6e SL |
827 | /** |
828 | * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object | |
829 | * @bo: &amdgpu_bo buffer object to be unmapped | |
830 | * | |
831 | * Unmaps a kernel map set up by amdgpu_bo_kmap(). | |
832 | */ | |
d38ceaf9 AD |
833 | void amdgpu_bo_kunmap(struct amdgpu_bo *bo) |
834 | { | |
f5e1c740 CK |
835 | if (bo->kmap.bo) |
836 | ttm_bo_kunmap(&bo->kmap); | |
d38ceaf9 AD |
837 | } |
838 | ||
6f4e8d6e SL |
839 | /** |
840 | * amdgpu_bo_ref - reference an &amdgpu_bo buffer object | |
841 | * @bo: &amdgpu_bo buffer object | |
842 | * | |
843 | * References the contained &ttm_buffer_object. | |
844 | * | |
2472e11b MD |
845 | * Returns: |
846 | * a refcounted pointer to the &amdgpu_bo buffer object. | |
6f4e8d6e | 847 | */ |
d38ceaf9 AD |
848 | struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) |
849 | { | |
850 | if (bo == NULL) | |
851 | return NULL; | |
852 | ||
71d5ef11 | 853 | ttm_bo_get(&bo->tbo); |
d38ceaf9 AD |
854 | return bo; |
855 | } | |
856 | ||
6f4e8d6e SL |
857 | /** |
858 | * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object | |
859 | * @bo: &amdgpu_bo buffer object | |
860 | * | |
861 | * Unreferences the contained &ttm_buffer_object and clear the pointer | |
862 | */ | |
d38ceaf9 AD |
863 | void amdgpu_bo_unref(struct amdgpu_bo **bo) |
864 | { | |
865 | struct ttm_buffer_object *tbo; | |
866 | ||
867 | if ((*bo) == NULL) | |
868 | return; | |
869 | ||
870 | tbo = &((*bo)->tbo); | |
fea872b2 TZ |
871 | ttm_bo_put(tbo); |
872 | *bo = NULL; | |
d38ceaf9 AD |
873 | } |
874 | ||
6f4e8d6e SL |
875 | /** |
876 | * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object | |
877 | * @bo: &amdgpu_bo buffer object to be pinned | |
878 | * @domain: domain to be pinned to | |
879 | * @min_offset: the start of requested address range | |
880 | * @max_offset: the end of requested address range | |
6f4e8d6e SL |
881 | * |
882 | * Pins the buffer object according to requested domain and address range. If | |
883 | * the memory is unbound gart memory, binds the pages into gart table. Adjusts | |
884 | * pin_count and pin_size accordingly. | |
885 | * | |
886 | * Pinning means to lock pages in memory along with keeping them at a fixed | |
887 | * offset. It is required when a buffer can not be moved, for example, when | |
888 | * a display buffer is being scanned out. | |
889 | * | |
890 | * Compared with amdgpu_bo_pin(), this function gives more flexibility on | |
891 | * where to pin a buffer if there are specific restrictions on where a buffer | |
892 | * must be located. | |
893 | * | |
2472e11b MD |
894 | * Returns: |
895 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 896 | */ |
7e5a547f | 897 | int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, |
7b7c6c81 | 898 | u64 min_offset, u64 max_offset) |
d38ceaf9 | 899 | { |
a7d64de6 | 900 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
19be5570 | 901 | struct ttm_operation_ctx ctx = { false, false }; |
d38ceaf9 AD |
902 | int r, i; |
903 | ||
cc325d19 | 904 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) |
d38ceaf9 AD |
905 | return -EPERM; |
906 | ||
7e5a547f CZ |
907 | if (WARN_ON_ONCE(min_offset > max_offset)) |
908 | return -EINVAL; | |
909 | ||
f5ba1404 LL |
910 | /* Check domain to be pinned to against preferred domains */ |
911 | if (bo->preferred_domains & domain) | |
912 | domain = bo->preferred_domains & domain; | |
913 | ||
803d89ad | 914 | /* A shared bo cannot be migrated to VRAM */ |
8c505bdc | 915 | if (bo->tbo.base.import_attach) { |
9b3f217f SL |
916 | if (domain & AMDGPU_GEM_DOMAIN_GTT) |
917 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
918 | else | |
919 | return -EINVAL; | |
920 | } | |
803d89ad | 921 | |
4671078e | 922 | if (bo->tbo.pin_count) { |
d3116756 CK |
923 | uint32_t mem_type = bo->tbo.resource->mem_type; |
924 | uint32_t mem_flags = bo->tbo.resource->placement; | |
408778e8 | 925 | |
f5318959 | 926 | if (!(domain & amdgpu_mem_type_to_domain(mem_type))) |
408778e8 FC |
927 | return -EINVAL; |
928 | ||
e1a4b67a | 929 | if ((mem_type == TTM_PL_VRAM) && |
930 | (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && | |
eda1068d FK |
931 | !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) |
932 | return -EINVAL; | |
933 | ||
4671078e | 934 | ttm_bo_pin(&bo->tbo); |
d38ceaf9 AD |
935 | |
936 | if (max_offset != 0) { | |
b1a8ef95 ND |
937 | u64 domain_start = amdgpu_ttm_domain_start(adev, |
938 | mem_type); | |
d38ceaf9 AD |
939 | WARN_ON_ONCE(max_offset < |
940 | (amdgpu_bo_gpu_offset(bo) - domain_start)); | |
941 | } | |
942 | ||
943 | return 0; | |
944 | } | |
03f48dd5 | 945 | |
9deb0b3d CK |
946 | /* This assumes only APU display buffers are pinned with (VRAM|GTT). |
947 | * See function amdgpu_display_supported_domains() | |
948 | */ | |
d035f84d | 949 | domain = amdgpu_bo_get_preferred_domain(adev, domain); |
9deb0b3d | 950 | |
a448cb00 CK |
951 | if (bo->tbo.base.import_attach) |
952 | dma_buf_pin(bo->tbo.base.import_attach); | |
953 | ||
e9c7577c CK |
954 | /* force to pin into visible video ram */ |
955 | if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) | |
956 | bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
c704ab18 | 957 | amdgpu_bo_placement_from_domain(bo, domain); |
d38ceaf9 | 958 | for (i = 0; i < bo->placement.num_placement; i++) { |
1d6ecab1 | 959 | unsigned int fpfn, lpfn; |
e9c7577c CK |
960 | |
961 | fpfn = min_offset >> PAGE_SHIFT; | |
962 | lpfn = max_offset >> PAGE_SHIFT; | |
963 | ||
7e5a547f CZ |
964 | if (fpfn > bo->placements[i].fpfn) |
965 | bo->placements[i].fpfn = fpfn; | |
78d0e182 CK |
966 | if (!bo->placements[i].lpfn || |
967 | (lpfn && lpfn < bo->placements[i].lpfn)) | |
7e5a547f | 968 | bo->placements[i].lpfn = lpfn; |
d38ceaf9 AD |
969 | } |
970 | ||
19be5570 | 971 | r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
6681c5eb | 972 | if (unlikely(r)) { |
a7d64de6 | 973 | dev_err(adev->dev, "%p pin failed\n", bo); |
6681c5eb CK |
974 | goto error; |
975 | } | |
976 | ||
4671078e | 977 | ttm_bo_pin(&bo->tbo); |
5e91fb57 | 978 | |
d3116756 | 979 | domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); |
6681c5eb | 980 | if (domain == AMDGPU_GEM_DOMAIN_VRAM) { |
a5ccfe5c MD |
981 | atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); |
982 | atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), | |
983 | &adev->visible_pin_size); | |
32ab75f0 | 984 | } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { |
a5ccfe5c | 985 | atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); |
d38ceaf9 | 986 | } |
6681c5eb CK |
987 | |
988 | error: | |
d38ceaf9 AD |
989 | return r; |
990 | } | |
991 | ||
6f4e8d6e SL |
992 | /** |
993 | * amdgpu_bo_pin - pin an &amdgpu_bo buffer object | |
994 | * @bo: &amdgpu_bo buffer object to be pinned | |
995 | * @domain: domain to be pinned to | |
6f4e8d6e SL |
996 | * |
997 | * A simple wrapper to amdgpu_bo_pin_restricted(). | |
998 | * Provides a simpler API for buffers that do not have any strict restrictions | |
999 | * on where a buffer must be located. | |
1000 | * | |
2472e11b MD |
1001 | * Returns: |
1002 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1003 | */ |
7b7c6c81 | 1004 | int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) |
d38ceaf9 | 1005 | { |
eda1068d | 1006 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; |
7b7c6c81 | 1007 | return amdgpu_bo_pin_restricted(bo, domain, 0, 0); |
d38ceaf9 AD |
1008 | } |
1009 | ||
6f4e8d6e SL |
1010 | /** |
1011 | * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object | |
1012 | * @bo: &amdgpu_bo buffer object to be unpinned | |
1013 | * | |
1014 | * Decreases the pin_count, and clears the flags if pin_count reaches 0. | |
1015 | * Changes placement and pin size accordingly. | |
1016 | * | |
2472e11b MD |
1017 | * Returns: |
1018 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1019 | */ |
4671078e | 1020 | void amdgpu_bo_unpin(struct amdgpu_bo *bo) |
d38ceaf9 | 1021 | { |
e2ac8531 CK |
1022 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
1023 | ||
4671078e CK |
1024 | ttm_bo_unpin(&bo->tbo); |
1025 | if (bo->tbo.pin_count) | |
1026 | return; | |
6681c5eb | 1027 | |
a448cb00 CK |
1028 | if (bo->tbo.base.import_attach) |
1029 | dma_buf_unpin(bo->tbo.base.import_attach); | |
e2ac8531 | 1030 | |
d3116756 | 1031 | if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { |
e2ac8531 CK |
1032 | atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); |
1033 | atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), | |
1034 | &adev->visible_pin_size); | |
d3116756 | 1035 | } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { |
e2ac8531 CK |
1036 | atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); |
1037 | } | |
dc3499c7 | 1038 | |
d38ceaf9 AD |
1039 | } |
1040 | ||
1d6ecab1 | 1041 | static const char * const amdgpu_vram_names[] = { |
1f8628c7 AD |
1042 | "UNKNOWN", |
1043 | "GDDR1", | |
1044 | "DDR2", | |
1045 | "GDDR3", | |
1046 | "GDDR4", | |
1047 | "GDDR5", | |
1048 | "HBM", | |
bc227cfa TSD |
1049 | "DDR3", |
1050 | "DDR4", | |
5228fe30 | 1051 | "GDDR6", |
d534ca71 AD |
1052 | "DDR5", |
1053 | "LPDDR4", | |
1054 | "LPDDR5" | |
1f8628c7 AD |
1055 | }; |
1056 | ||
6f4e8d6e SL |
1057 | /** |
1058 | * amdgpu_bo_init - initialize memory manager | |
1059 | * @adev: amdgpu device object | |
1060 | * | |
1061 | * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. | |
1062 | * | |
2472e11b MD |
1063 | * Returns: |
1064 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1065 | */ |
d38ceaf9 AD |
1066 | int amdgpu_bo_init(struct amdgpu_device *adev) |
1067 | { | |
35d5f224 | 1068 | /* On A+A platform, VRAM can be mapped as WB */ |
228ce176 | 1069 | if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { |
35d5f224 | 1070 | /* reserve PAT memory space to WC for VRAM */ |
26db557e | 1071 | int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, |
35d5f224 OZ |
1072 | adev->gmc.aper_size); |
1073 | ||
26db557e ND |
1074 | if (r) { |
1075 | DRM_ERROR("Unable to set WC memtype for the aperture base\n"); | |
1076 | return r; | |
1077 | } | |
1078 | ||
35d5f224 OZ |
1079 | /* Add an MTRR for the VRAM */ |
1080 | adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, | |
1081 | adev->gmc.aper_size); | |
1082 | } | |
7cf321d1 | 1083 | |
d38ceaf9 | 1084 | DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", |
770d13b1 CK |
1085 | adev->gmc.mc_vram_size >> 20, |
1086 | (unsigned long long)adev->gmc.aper_size >> 20); | |
1f8628c7 | 1087 | DRM_INFO("RAM width %dbits %s\n", |
770d13b1 | 1088 | adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); |
d38ceaf9 AD |
1089 | return amdgpu_ttm_init(adev); |
1090 | } | |
1091 | ||
6f4e8d6e SL |
1092 | /** |
1093 | * amdgpu_bo_fini - tear down memory manager | |
1094 | * @adev: amdgpu device object | |
1095 | * | |
1096 | * Reverses amdgpu_bo_init() to tear down memory manager. | |
1097 | */ | |
d38ceaf9 AD |
1098 | void amdgpu_bo_fini(struct amdgpu_device *adev) |
1099 | { | |
62d5f9f7 LS |
1100 | int idx; |
1101 | ||
d38ceaf9 | 1102 | amdgpu_ttm_fini(adev); |
62d5f9f7 LS |
1103 | |
1104 | if (drm_dev_enter(adev_to_drm(adev), &idx)) { | |
a0ba1279 | 1105 | if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { |
62d5f9f7 LS |
1106 | arch_phys_wc_del(adev->gmc.vram_mtrr); |
1107 | arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); | |
1108 | } | |
1109 | drm_dev_exit(idx); | |
1110 | } | |
d38ceaf9 AD |
1111 | } |
1112 | ||
6f4e8d6e SL |
1113 | /** |
1114 | * amdgpu_bo_set_tiling_flags - set tiling flags | |
1115 | * @bo: &amdgpu_bo buffer object | |
1116 | * @tiling_flags: new flags | |
1117 | * | |
1118 | * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or | |
1119 | * kernel driver to set the tiling flags on a buffer. | |
1120 | * | |
2472e11b MD |
1121 | * Returns: |
1122 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1123 | */ |
d38ceaf9 AD |
1124 | int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) |
1125 | { | |
9079ac76 | 1126 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
cc1bcf85 | 1127 | struct amdgpu_bo_user *ubo; |
9079ac76 | 1128 | |
030bb4ad | 1129 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
9079ac76 MO |
1130 | if (adev->family <= AMDGPU_FAMILY_CZ && |
1131 | AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) | |
d38ceaf9 | 1132 | return -EINVAL; |
d38ceaf9 | 1133 | |
cc1bcf85 ND |
1134 | ubo = to_amdgpu_bo_user(bo); |
1135 | ubo->tiling_flags = tiling_flags; | |
d38ceaf9 AD |
1136 | return 0; |
1137 | } | |
1138 | ||
6f4e8d6e SL |
1139 | /** |
1140 | * amdgpu_bo_get_tiling_flags - get tiling flags | |
1141 | * @bo: &amdgpu_bo buffer object | |
1142 | * @tiling_flags: returned flags | |
1143 | * | |
1144 | * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to | |
1145 | * set the tiling flags on a buffer. | |
1146 | */ | |
d38ceaf9 AD |
1147 | void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) |
1148 | { | |
cc1bcf85 ND |
1149 | struct amdgpu_bo_user *ubo; |
1150 | ||
030bb4ad | 1151 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
52791eee | 1152 | dma_resv_assert_held(bo->tbo.base.resv); |
cc1bcf85 | 1153 | ubo = to_amdgpu_bo_user(bo); |
d38ceaf9 AD |
1154 | |
1155 | if (tiling_flags) | |
cc1bcf85 | 1156 | *tiling_flags = ubo->tiling_flags; |
d38ceaf9 AD |
1157 | } |
1158 | ||
6f4e8d6e SL |
1159 | /** |
1160 | * amdgpu_bo_set_metadata - set metadata | |
1161 | * @bo: &amdgpu_bo buffer object | |
1162 | * @metadata: new metadata | |
1163 | * @metadata_size: size of the new metadata | |
1164 | * @flags: flags of the new metadata | |
1165 | * | |
1166 | * Sets buffer object's metadata, its size and flags. | |
1167 | * Used via GEM ioctl. | |
1168 | * | |
2472e11b MD |
1169 | * Returns: |
1170 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1171 | */ |
1d6ecab1 SS |
1172 | int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, |
1173 | u32 metadata_size, uint64_t flags) | |
d38ceaf9 | 1174 | { |
cc1bcf85 | 1175 | struct amdgpu_bo_user *ubo; |
d38ceaf9 AD |
1176 | void *buffer; |
1177 | ||
030bb4ad | 1178 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
cc1bcf85 | 1179 | ubo = to_amdgpu_bo_user(bo); |
d38ceaf9 | 1180 | if (!metadata_size) { |
cc1bcf85 ND |
1181 | if (ubo->metadata_size) { |
1182 | kfree(ubo->metadata); | |
1183 | ubo->metadata = NULL; | |
1184 | ubo->metadata_size = 0; | |
d38ceaf9 AD |
1185 | } |
1186 | return 0; | |
1187 | } | |
1188 | ||
1189 | if (metadata == NULL) | |
1190 | return -EINVAL; | |
1191 | ||
71affda5 | 1192 | buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); |
d38ceaf9 AD |
1193 | if (buffer == NULL) |
1194 | return -ENOMEM; | |
1195 | ||
cc1bcf85 ND |
1196 | kfree(ubo->metadata); |
1197 | ubo->metadata_flags = flags; | |
1198 | ubo->metadata = buffer; | |
1199 | ubo->metadata_size = metadata_size; | |
d38ceaf9 AD |
1200 | |
1201 | return 0; | |
1202 | } | |
1203 | ||
6f4e8d6e SL |
1204 | /** |
1205 | * amdgpu_bo_get_metadata - get metadata | |
1206 | * @bo: &amdgpu_bo buffer object | |
1207 | * @buffer: returned metadata | |
1208 | * @buffer_size: size of the buffer | |
1209 | * @metadata_size: size of the returned metadata | |
1210 | * @flags: flags of the returned metadata | |
1211 | * | |
1212 | * Gets buffer object's metadata, its size and flags. buffer_size shall not be | |
1213 | * less than metadata_size. | |
1214 | * Used via GEM ioctl. | |
1215 | * | |
2472e11b MD |
1216 | * Returns: |
1217 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1218 | */ |
d38ceaf9 AD |
1219 | int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, |
1220 | size_t buffer_size, uint32_t *metadata_size, | |
1221 | uint64_t *flags) | |
1222 | { | |
cc1bcf85 ND |
1223 | struct amdgpu_bo_user *ubo; |
1224 | ||
d38ceaf9 AD |
1225 | if (!buffer && !metadata_size) |
1226 | return -EINVAL; | |
1227 | ||
030bb4ad | 1228 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
cc1bcf85 | 1229 | ubo = to_amdgpu_bo_user(bo); |
eba98523 SZ |
1230 | if (metadata_size) |
1231 | *metadata_size = ubo->metadata_size; | |
1232 | ||
d38ceaf9 | 1233 | if (buffer) { |
cc1bcf85 | 1234 | if (buffer_size < ubo->metadata_size) |
d38ceaf9 AD |
1235 | return -EINVAL; |
1236 | ||
cc1bcf85 ND |
1237 | if (ubo->metadata_size) |
1238 | memcpy(buffer, ubo->metadata, ubo->metadata_size); | |
d38ceaf9 AD |
1239 | } |
1240 | ||
d38ceaf9 | 1241 | if (flags) |
cc1bcf85 | 1242 | *flags = ubo->metadata_flags; |
d38ceaf9 AD |
1243 | |
1244 | return 0; | |
1245 | } | |
1246 | ||
6f4e8d6e SL |
1247 | /** |
1248 | * amdgpu_bo_move_notify - notification about a memory move | |
1249 | * @bo: pointer to a buffer object | |
1250 | * @evict: if this move is evicting the buffer from the graphics address space | |
6f4e8d6e SL |
1251 | * |
1252 | * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs | |
1253 | * bookkeeping. | |
1254 | * TTM driver callback which is called when ttm moves a buffer. | |
1255 | */ | |
94aeb411 | 1256 | void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, bool evict) |
d38ceaf9 | 1257 | { |
a7d64de6 | 1258 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
765e7fbf | 1259 | struct amdgpu_bo *abo; |
d38ceaf9 | 1260 | |
c704ab18 | 1261 | if (!amdgpu_bo_is_amdgpu_bo(bo)) |
d38ceaf9 AD |
1262 | return; |
1263 | ||
b82485fd | 1264 | abo = ttm_to_amdgpu_bo(bo); |
3f3333f8 | 1265 | amdgpu_vm_bo_invalidate(adev, abo, evict); |
d38ceaf9 | 1266 | |
6375bbb4 CK |
1267 | amdgpu_bo_kunmap(abo); |
1268 | ||
2d4dad27 | 1269 | if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach && |
d3116756 | 1270 | bo->resource->mem_type != TTM_PL_SYSTEM) |
2d4dad27 CK |
1271 | dma_buf_move_notify(abo->tbo.base.dma_buf); |
1272 | ||
661a7606 NH |
1273 | /* remember the eviction */ |
1274 | if (evict) | |
1275 | atomic64_inc(&adev->num_evictions); | |
d38ceaf9 AD |
1276 | } |
1277 | ||
d6530c33 MO |
1278 | void amdgpu_bo_get_memory(struct amdgpu_bo *bo, |
1279 | struct amdgpu_mem_stats *stats) | |
87444254 | 1280 | { |
d6530c33 | 1281 | uint64_t size = amdgpu_bo_size(bo); |
ba1a58d5 | 1282 | struct drm_gem_object *obj; |
ca0b954a | 1283 | unsigned int domain; |
ba1a58d5 | 1284 | bool shared; |
ca0b954a CK |
1285 | |
1286 | /* Abort if the BO doesn't currently have a backing store */ | |
1287 | if (!bo->tbo.resource) | |
1288 | return; | |
87444254 | 1289 | |
ba1a58d5 AD |
1290 | obj = &bo->tbo.base; |
1291 | shared = drm_gem_object_is_shared_for_memory_stats(obj); | |
1292 | ||
d3116756 | 1293 | domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); |
87444254 RS |
1294 | switch (domain) { |
1295 | case AMDGPU_GEM_DOMAIN_VRAM: | |
d6530c33 MO |
1296 | stats->vram += size; |
1297 | if (amdgpu_bo_in_cpu_visible_vram(bo)) | |
1298 | stats->visible_vram += size; | |
ba1a58d5 AD |
1299 | if (shared) |
1300 | stats->vram_shared += size; | |
87444254 RS |
1301 | break; |
1302 | case AMDGPU_GEM_DOMAIN_GTT: | |
d6530c33 | 1303 | stats->gtt += size; |
ba1a58d5 AD |
1304 | if (shared) |
1305 | stats->gtt_shared += size; | |
87444254 RS |
1306 | break; |
1307 | case AMDGPU_GEM_DOMAIN_CPU: | |
1308 | default: | |
d6530c33 | 1309 | stats->cpu += size; |
ba1a58d5 AD |
1310 | if (shared) |
1311 | stats->cpu_shared += size; | |
87444254 RS |
1312 | break; |
1313 | } | |
d6530c33 MO |
1314 | |
1315 | if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) { | |
1316 | stats->requested_vram += size; | |
1317 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) | |
1318 | stats->requested_visible_vram += size; | |
1319 | ||
1320 | if (domain != AMDGPU_GEM_DOMAIN_VRAM) { | |
1321 | stats->evicted_vram += size; | |
1322 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) | |
1323 | stats->evicted_visible_vram += size; | |
1324 | } | |
1325 | } else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT) { | |
1326 | stats->requested_gtt += size; | |
1327 | } | |
87444254 RS |
1328 | } |
1329 | ||
ab2f7a5c | 1330 | /** |
736b1729 | 1331 | * amdgpu_bo_release_notify - notification about a BO being released |
ab2f7a5c FK |
1332 | * @bo: pointer to a buffer object |
1333 | * | |
1334 | * Wipes VRAM buffers whose contents should not be leaked before the | |
1335 | * memory is released. | |
1336 | */ | |
1337 | void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) | |
1338 | { | |
32f90e65 | 1339 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
ab2f7a5c FK |
1340 | struct dma_fence *fence = NULL; |
1341 | struct amdgpu_bo *abo; | |
1342 | int r; | |
1343 | ||
1344 | if (!amdgpu_bo_is_amdgpu_bo(bo)) | |
1345 | return; | |
1346 | ||
1347 | abo = ttm_to_amdgpu_bo(bo); | |
1348 | ||
65d2765d CK |
1349 | WARN_ON(abo->vm_bo); |
1350 | ||
ab2f7a5c | 1351 | if (abo->kfd_bo) |
5702d052 | 1352 | amdgpu_amdkfd_release_notify(abo); |
ab2f7a5c | 1353 | |
f4a3c42b | 1354 | /* We only remove the fence if the resv has individualized. */ |
9fe58d0b | 1355 | WARN_ON_ONCE(bo->type == ttm_bo_type_kernel |
1356 | && bo->base.resv != &bo->base._resv); | |
f4a3c42b | 1357 | if (bo->base.resv == &bo->base._resv) |
1358 | amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo); | |
1359 | ||
63af82cf | 1360 | if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || |
32f90e65 | 1361 | !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || |
93bb18d2 | 1362 | adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) |
ab2f7a5c FK |
1363 | return; |
1364 | ||
447c7997 RB |
1365 | if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv))) |
1366 | return; | |
ab2f7a5c | 1367 | |
c3aaca43 | 1368 | r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence, true); |
ab2f7a5c FK |
1369 | if (!WARN_ON(r)) { |
1370 | amdgpu_bo_fence(abo, fence, false); | |
1371 | dma_fence_put(fence); | |
1372 | } | |
1373 | ||
5f680625 | 1374 | dma_resv_unlock(bo->base.resv); |
ab2f7a5c FK |
1375 | } |
1376 | ||
6f4e8d6e SL |
1377 | /** |
1378 | * amdgpu_bo_fault_reserve_notify - notification about a memory fault | |
1379 | * @bo: pointer to a buffer object | |
1380 | * | |
1381 | * Notifies the driver we are taking a fault on this BO and have reserved it, | |
1382 | * also performs bookkeeping. | |
1383 | * TTM driver callback for dealing with vm faults. | |
1384 | * | |
2472e11b MD |
1385 | * Returns: |
1386 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1387 | */ |
d3ef581a | 1388 | vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) |
d38ceaf9 | 1389 | { |
a7d64de6 | 1390 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
19be5570 | 1391 | struct ttm_operation_ctx ctx = { false, false }; |
d3ef581a | 1392 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); |
96cf8271 | 1393 | int r; |
d38ceaf9 | 1394 | |
96cf8271 JB |
1395 | /* Remember that this BO was accessed by the CPU */ |
1396 | abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
1397 | ||
d3116756 | 1398 | if (bo->resource->mem_type != TTM_PL_VRAM) |
5fb1941d CK |
1399 | return 0; |
1400 | ||
aed01a68 | 1401 | if (amdgpu_bo_in_cpu_visible_vram(abo)) |
5fb1941d CK |
1402 | return 0; |
1403 | ||
104ece97 | 1404 | /* Can't move a pinned BO to visible VRAM */ |
4671078e | 1405 | if (abo->tbo.pin_count > 0) |
d3ef581a | 1406 | return VM_FAULT_SIGBUS; |
104ece97 | 1407 | |
5fb1941d | 1408 | /* hurrah the memory is not visible ! */ |
68e2c5ff | 1409 | atomic64_inc(&adev->num_vram_cpu_page_faults); |
c704ab18 CK |
1410 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | |
1411 | AMDGPU_GEM_DOMAIN_GTT); | |
41d9a6a7 JB |
1412 | |
1413 | /* Avoid costly evictions; only set GTT as a busy placement */ | |
a78a8da5 | 1414 | abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; |
41d9a6a7 | 1415 | |
19be5570 | 1416 | r = ttm_bo_validate(bo, &abo->placement, &ctx); |
d3ef581a CK |
1417 | if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) |
1418 | return VM_FAULT_NOPAGE; | |
1419 | else if (unlikely(r)) | |
1420 | return VM_FAULT_SIGBUS; | |
5fb1941d | 1421 | |
5fb1941d | 1422 | /* this should never happen */ |
d3116756 | 1423 | if (bo->resource->mem_type == TTM_PL_VRAM && |
aed01a68 | 1424 | !amdgpu_bo_in_cpu_visible_vram(abo)) |
d3ef581a | 1425 | return VM_FAULT_SIGBUS; |
5fb1941d | 1426 | |
d3ef581a | 1427 | ttm_bo_move_to_lru_tail_unlocked(bo); |
d38ceaf9 AD |
1428 | return 0; |
1429 | } | |
1430 | ||
1431 | /** | |
1432 | * amdgpu_bo_fence - add fence to buffer object | |
1433 | * | |
1434 | * @bo: buffer object in question | |
1435 | * @fence: fence to add | |
1436 | * @shared: true if fence should be added shared | |
1437 | * | |
1438 | */ | |
f54d1867 | 1439 | void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, |
d38ceaf9 AD |
1440 | bool shared) |
1441 | { | |
52791eee | 1442 | struct dma_resv *resv = bo->tbo.base.resv; |
c8d4c18b CK |
1443 | int r; |
1444 | ||
1445 | r = dma_resv_reserve_fences(resv, 1); | |
1446 | if (r) { | |
1447 | /* As last resort on OOM we block for the fence */ | |
1448 | dma_fence_wait(fence, false); | |
1449 | return; | |
1450 | } | |
d38ceaf9 | 1451 | |
73511edf CK |
1452 | dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : |
1453 | DMA_RESV_USAGE_WRITE); | |
d38ceaf9 | 1454 | } |
cdb7e8f2 | 1455 | |
e8e32426 | 1456 | /** |
9f3cc18d | 1457 | * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences |
e8e32426 | 1458 | * |
9f3cc18d CK |
1459 | * @adev: amdgpu device pointer |
1460 | * @resv: reservation object to sync to | |
1461 | * @sync_mode: synchronization mode | |
e8e32426 FK |
1462 | * @owner: fence owner |
1463 | * @intr: Whether the wait is interruptible | |
1464 | * | |
9f3cc18d CK |
1465 | * Extract the fences from the reservation object and waits for them to finish. |
1466 | * | |
e8e32426 FK |
1467 | * Returns: |
1468 | * 0 on success, errno otherwise. | |
1469 | */ | |
9f3cc18d CK |
1470 | int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, |
1471 | enum amdgpu_sync_mode sync_mode, void *owner, | |
1472 | bool intr) | |
e8e32426 | 1473 | { |
e8e32426 FK |
1474 | struct amdgpu_sync sync; |
1475 | int r; | |
1476 | ||
1477 | amdgpu_sync_create(&sync); | |
9f3cc18d | 1478 | amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); |
e8e32426 FK |
1479 | r = amdgpu_sync_wait(&sync, intr); |
1480 | amdgpu_sync_free(&sync); | |
e8e32426 FK |
1481 | return r; |
1482 | } | |
1483 | ||
9f3cc18d CK |
1484 | /** |
1485 | * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv | |
1486 | * @bo: buffer object to wait for | |
1487 | * @owner: fence owner | |
1488 | * @intr: Whether the wait is interruptible | |
1489 | * | |
1490 | * Wrapper to wait for fences in a BO. | |
1491 | * Returns: | |
1492 | * 0 on success, errno otherwise. | |
1493 | */ | |
1494 | int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) | |
1495 | { | |
1496 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
1497 | ||
1498 | return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, | |
1499 | AMDGPU_SYNC_NE_OWNER, owner, intr); | |
1500 | } | |
1501 | ||
cdb7e8f2 CK |
1502 | /** |
1503 | * amdgpu_bo_gpu_offset - return GPU offset of bo | |
1504 | * @bo: amdgpu object for which we query the offset | |
1505 | * | |
cdb7e8f2 CK |
1506 | * Note: object should either be pinned or reserved when calling this |
1507 | * function, it might be useful to add check for this for debugging. | |
2472e11b MD |
1508 | * |
1509 | * Returns: | |
1510 | * current GPU offset of the object. | |
cdb7e8f2 CK |
1511 | */ |
1512 | u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) | |
1513 | { | |
d3116756 | 1514 | WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); |
52791eee | 1515 | WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && |
4671078e | 1516 | !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); |
d3116756 CK |
1517 | WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); |
1518 | WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && | |
03f48dd5 | 1519 | !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); |
cdb7e8f2 | 1520 | |
b1a8ef95 ND |
1521 | return amdgpu_bo_gpu_offset_no_check(bo); |
1522 | } | |
1523 | ||
1524 | /** | |
1525 | * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo | |
1526 | * @bo: amdgpu object for which we query the offset | |
1527 | * | |
1528 | * Returns: | |
1529 | * current GPU offset of the object without raising warnings. | |
1530 | */ | |
1531 | u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) | |
1532 | { | |
1533 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
ca0b0069 | 1534 | uint64_t offset = AMDGPU_BO_INVALID_OFFSET; |
b1a8ef95 | 1535 | |
ca0b0069 AD |
1536 | if (bo->tbo.resource->mem_type == TTM_PL_TT) |
1537 | offset = amdgpu_gmc_agp_addr(&bo->tbo); | |
b1a8ef95 | 1538 | |
ca0b0069 AD |
1539 | if (offset == AMDGPU_BO_INVALID_OFFSET) |
1540 | offset = (bo->tbo.resource->start << PAGE_SHIFT) + | |
1541 | amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); | |
b1a8ef95 ND |
1542 | |
1543 | return amdgpu_gmc_sign_extend(offset); | |
cdb7e8f2 | 1544 | } |
84b74608 | 1545 | |
2472e11b | 1546 | /** |
d035f84d | 1547 | * amdgpu_bo_get_preferred_domain - get preferred domain |
2472e11b MD |
1548 | * @adev: amdgpu device object |
1549 | * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` | |
1550 | * | |
1551 | * Returns: | |
d035f84d | 1552 | * Which of the allowed domains is preferred for allocating the BO. |
2472e11b | 1553 | */ |
d035f84d | 1554 | uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, |
84b74608 DS |
1555 | uint32_t domain) |
1556 | { | |
81d0bcf9 AD |
1557 | if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && |
1558 | ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { | |
84b74608 DS |
1559 | domain = AMDGPU_GEM_DOMAIN_VRAM; |
1560 | if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) | |
1561 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
1562 | } | |
1563 | return domain; | |
1564 | } | |
ff72bc40 MBP |
1565 | |
1566 | #if defined(CONFIG_DEBUG_FS) | |
1567 | #define amdgpu_bo_print_flag(m, bo, flag) \ | |
1568 | do { \ | |
1569 | if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ | |
1570 | seq_printf((m), " " #flag); \ | |
1571 | } \ | |
1572 | } while (0) | |
1573 | ||
1574 | /** | |
25dd7a44 | 1575 | * amdgpu_bo_print_info - print BO info in debugfs file |
ff72bc40 MBP |
1576 | * |
1577 | * @id: Index or Id of the BO | |
1578 | * @bo: Requested BO for printing info | |
1579 | * @m: debugfs file | |
1580 | * | |
1581 | * Print BO information in debugfs file | |
1582 | * | |
1583 | * Returns: | |
1584 | * Size of the BO in bytes. | |
1585 | */ | |
1586 | u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) | |
1587 | { | |
1588 | struct dma_buf_attachment *attachment; | |
1589 | struct dma_buf *dma_buf; | |
ff72bc40 MBP |
1590 | const char *placement; |
1591 | unsigned int pin_count; | |
1592 | u64 size; | |
1593 | ||
818c158f PEPP |
1594 | if (dma_resv_trylock(bo->tbo.base.resv)) { |
1595 | unsigned int domain; | |
1596 | domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); | |
1597 | switch (domain) { | |
1598 | case AMDGPU_GEM_DOMAIN_VRAM: | |
1599 | if (amdgpu_bo_in_cpu_visible_vram(bo)) | |
1600 | placement = "VRAM VISIBLE"; | |
1601 | else | |
1602 | placement = "VRAM"; | |
1603 | break; | |
1604 | case AMDGPU_GEM_DOMAIN_GTT: | |
1605 | placement = "GTT"; | |
1606 | break; | |
1607 | case AMDGPU_GEM_DOMAIN_CPU: | |
1608 | default: | |
1609 | placement = "CPU"; | |
1610 | break; | |
1611 | } | |
1612 | dma_resv_unlock(bo->tbo.base.resv); | |
1613 | } else { | |
1614 | placement = "UNKNOWN"; | |
ff72bc40 MBP |
1615 | } |
1616 | ||
1617 | size = amdgpu_bo_size(bo); | |
1618 | seq_printf(m, "\t\t0x%08x: %12lld byte %s", | |
1619 | id, size, placement); | |
1620 | ||
5b8c5969 | 1621 | pin_count = READ_ONCE(bo->tbo.pin_count); |
ff72bc40 MBP |
1622 | if (pin_count) |
1623 | seq_printf(m, " pin count %d", pin_count); | |
1624 | ||
1625 | dma_buf = READ_ONCE(bo->tbo.base.dma_buf); | |
1626 | attachment = READ_ONCE(bo->tbo.base.import_attach); | |
1627 | ||
1628 | if (attachment) | |
26fd808b | 1629 | seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino); |
ff72bc40 | 1630 | else if (dma_buf) |
26fd808b | 1631 | seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino); |
ff72bc40 MBP |
1632 | |
1633 | amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); | |
1634 | amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); | |
1635 | amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); | |
1636 | amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); | |
ff72bc40 MBP |
1637 | amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); |
1638 | amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); | |
1639 | amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); | |
1640 | ||
1641 | seq_puts(m, "\n"); | |
1642 | ||
1643 | return size; | |
1644 | } | |
1645 | #endif |