Commit | Line | Data |
---|---|---|
d38ceaf9 AD |
1 | /* |
2 | * Copyright 2009 Jerome Glisse. | |
3 | * All Rights Reserved. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the | |
7 | * "Software"), to deal in the Software without restriction, including | |
8 | * without limitation the rights to use, copy, modify, merge, publish, | |
9 | * distribute, sub license, and/or sell copies of the Software, and to | |
10 | * permit persons to whom the Software is furnished to do so, subject to | |
11 | * the following conditions: | |
12 | * | |
13 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
14 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
15 | * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL | |
16 | * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, | |
17 | * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR | |
18 | * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE | |
19 | * USE OR OTHER DEALINGS IN THE SOFTWARE. | |
20 | * | |
21 | * The above copyright notice and this permission notice (including the | |
22 | * next paragraph) shall be included in all copies or substantial portions | |
23 | * of the Software. | |
24 | * | |
25 | */ | |
26 | /* | |
27 | * Authors: | |
28 | * Jerome Glisse <glisse@freedesktop.org> | |
29 | * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> | |
30 | * Dave Airlie | |
31 | */ | |
32 | #include <linux/list.h> | |
33 | #include <linux/slab.h> | |
2d4dad27 | 34 | #include <linux/dma-buf.h> |
fdf2f6c5 | 35 | |
62d5f9f7 | 36 | #include <drm/drm_drv.h> |
d38ceaf9 | 37 | #include <drm/amdgpu_drm.h> |
a187f17f | 38 | #include <drm/drm_cache.h> |
d38ceaf9 AD |
39 | #include "amdgpu.h" |
40 | #include "amdgpu_trace.h" | |
a46a2cd1 | 41 | #include "amdgpu_amdkfd.h" |
d38ceaf9 | 42 | |
6f4e8d6e SL |
43 | /** |
44 | * DOC: amdgpu_object | |
45 | * | |
46 | * This defines the interfaces to operate on an &amdgpu_bo buffer object which | |
47 | * represents memory used by driver (VRAM, system memory, etc.). The driver | |
48 | * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces | |
49 | * to create/destroy/set buffer object which are then managed by the kernel TTM | |
50 | * memory manager. | |
51 | * The interfaces are also used internally by kernel clients, including gfx, | |
52 | * uvd, etc. for kernel managed allocations used by the GPU. | |
53 | * | |
54 | */ | |
55 | ||
c704ab18 | 56 | static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) |
d38ceaf9 | 57 | { |
b82485fd | 58 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); |
d38ceaf9 | 59 | |
6375bbb4 | 60 | amdgpu_bo_kunmap(bo); |
d38ceaf9 | 61 | |
c105de28 GH |
62 | if (bo->tbo.base.import_attach) |
63 | drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); | |
64 | drm_gem_object_release(&bo->tbo.base); | |
23e24fbb ND |
65 | amdgpu_bo_unref(&bo->parent); |
66 | kvfree(bo); | |
67 | } | |
68 | ||
69 | static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) | |
70 | { | |
71 | struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); | |
72 | struct amdgpu_bo_user *ubo; | |
73 | ||
74 | ubo = to_amdgpu_bo_user(bo); | |
75 | kfree(ubo->metadata); | |
76 | amdgpu_bo_destroy(tbo); | |
77 | } | |
78 | ||
79 | static void amdgpu_bo_vm_destroy(struct ttm_buffer_object *tbo) | |
80 | { | |
81 | struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); | |
cbb63ecc | 82 | struct amdgpu_bo *shadow_bo = ttm_to_amdgpu_bo(tbo), *bo; |
e18aaea7 | 83 | struct amdgpu_bo_vm *vmbo; |
23e24fbb | 84 | |
cbb63ecc | 85 | bo = shadow_bo->parent; |
e18aaea7 | 86 | vmbo = to_amdgpu_bo_vm(bo); |
36e499b2 | 87 | /* in case amdgpu_device_recover_vram got NULL of bo->parent */ |
e18aaea7 | 88 | if (!list_empty(&vmbo->shadow_list)) { |
a7d64de6 | 89 | mutex_lock(&adev->shadow_list_lock); |
e18aaea7 | 90 | list_del_init(&vmbo->shadow_list); |
a7d64de6 | 91 | mutex_unlock(&adev->shadow_list_lock); |
0c4e7fa5 | 92 | } |
36e499b2 | 93 | |
23e24fbb | 94 | amdgpu_bo_destroy(tbo); |
d38ceaf9 AD |
95 | } |
96 | ||
6f4e8d6e | 97 | /** |
c704ab18 | 98 | * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo |
6f4e8d6e SL |
99 | * @bo: buffer object to be checked |
100 | * | |
101 | * Uses destroy function associated with the object to determine if this is | |
102 | * an &amdgpu_bo. | |
103 | * | |
2472e11b MD |
104 | * Returns: |
105 | * true if the object belongs to &amdgpu_bo, false if not. | |
6f4e8d6e | 106 | */ |
c704ab18 | 107 | bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) |
d38ceaf9 | 108 | { |
23e24fbb ND |
109 | if (bo->destroy == &amdgpu_bo_destroy || |
110 | bo->destroy == &amdgpu_bo_user_destroy || | |
111 | bo->destroy == &amdgpu_bo_vm_destroy) | |
d38ceaf9 | 112 | return true; |
23e24fbb | 113 | |
d38ceaf9 AD |
114 | return false; |
115 | } | |
116 | ||
6f4e8d6e | 117 | /** |
c704ab18 | 118 | * amdgpu_bo_placement_from_domain - set buffer's placement |
6f4e8d6e SL |
119 | * @abo: &amdgpu_bo buffer object whose placement is to be set |
120 | * @domain: requested domain | |
121 | * | |
122 | * Sets buffer's placement according to requested domain and the buffer's | |
123 | * flags. | |
124 | */ | |
c704ab18 | 125 | void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) |
d38ceaf9 | 126 | { |
c09312a6 CK |
127 | struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); |
128 | struct ttm_placement *placement = &abo->placement; | |
129 | struct ttm_place *places = abo->placements; | |
130 | u64 flags = abo->flags; | |
6369f6f1 | 131 | u32 c = 0; |
7e5a547f | 132 | |
d38ceaf9 | 133 | if (domain & AMDGPU_GEM_DOMAIN_VRAM) { |
1d6ecab1 | 134 | unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; |
3ebfd221 | 135 | int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); |
faceaf6a | 136 | |
3ebfd221 PY |
137 | if (adev->gmc.mem_partitions && mem_id >= 0) { |
138 | places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; | |
6cfba94a PY |
139 | /* |
140 | * memory partition range lpfn is inclusive start + size - 1 | |
141 | * TTM place lpfn is exclusive start + size | |
142 | */ | |
3ebfd221 | 143 | places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; |
7f6db894 PY |
144 | } else { |
145 | places[c].fpfn = 0; | |
146 | places[c].lpfn = 0; | |
147 | } | |
48e07c23 | 148 | places[c].mem_type = TTM_PL_VRAM; |
ce65b874 | 149 | places[c].flags = 0; |
89bb5752 | 150 | |
faceaf6a | 151 | if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) |
7f6db894 | 152 | places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); |
59eddd4e | 153 | else |
faceaf6a | 154 | places[c].flags |= TTM_PL_FLAG_TOPDOWN; |
89bb5752 CK |
155 | |
156 | if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) | |
157 | places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; | |
faceaf6a | 158 | c++; |
d38ceaf9 AD |
159 | } |
160 | ||
dc3499c7 AD |
161 | if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { |
162 | places[c].fpfn = 0; | |
163 | places[c].lpfn = 0; | |
164 | places[c].mem_type = AMDGPU_PL_DOORBELL; | |
165 | places[c].flags = 0; | |
166 | c++; | |
167 | } | |
168 | ||
d38ceaf9 | 169 | if (domain & AMDGPU_GEM_DOMAIN_GTT) { |
faceaf6a | 170 | places[c].fpfn = 0; |
3d5fe658 | 171 | places[c].lpfn = 0; |
b453e42a FK |
172 | places[c].mem_type = |
173 | abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? | |
174 | AMDGPU_PL_PREEMPT : TTM_PL_TT; | |
48e07c23 | 175 | places[c].flags = 0; |
216c1282 CK |
176 | /* |
177 | * When GTT is just an alternative to VRAM make sure that we | |
178 | * only use it as fallback and still try to fill up VRAM first. | |
179 | */ | |
180 | if (domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) | |
181 | places[c].flags |= TTM_PL_FLAG_FALLBACK; | |
faceaf6a | 182 | c++; |
d38ceaf9 AD |
183 | } |
184 | ||
185 | if (domain & AMDGPU_GEM_DOMAIN_CPU) { | |
faceaf6a CK |
186 | places[c].fpfn = 0; |
187 | places[c].lpfn = 0; | |
48e07c23 CK |
188 | places[c].mem_type = TTM_PL_SYSTEM; |
189 | places[c].flags = 0; | |
faceaf6a | 190 | c++; |
d38ceaf9 AD |
191 | } |
192 | ||
193 | if (domain & AMDGPU_GEM_DOMAIN_GDS) { | |
faceaf6a CK |
194 | places[c].fpfn = 0; |
195 | places[c].lpfn = 0; | |
48e07c23 | 196 | places[c].mem_type = AMDGPU_PL_GDS; |
ce65b874 | 197 | places[c].flags = 0; |
faceaf6a | 198 | c++; |
d38ceaf9 | 199 | } |
faceaf6a | 200 | |
d38ceaf9 | 201 | if (domain & AMDGPU_GEM_DOMAIN_GWS) { |
faceaf6a CK |
202 | places[c].fpfn = 0; |
203 | places[c].lpfn = 0; | |
48e07c23 | 204 | places[c].mem_type = AMDGPU_PL_GWS; |
ce65b874 | 205 | places[c].flags = 0; |
faceaf6a | 206 | c++; |
d38ceaf9 | 207 | } |
faceaf6a | 208 | |
d38ceaf9 | 209 | if (domain & AMDGPU_GEM_DOMAIN_OA) { |
faceaf6a CK |
210 | places[c].fpfn = 0; |
211 | places[c].lpfn = 0; | |
48e07c23 | 212 | places[c].mem_type = AMDGPU_PL_OA; |
ce65b874 | 213 | places[c].flags = 0; |
faceaf6a | 214 | c++; |
d38ceaf9 AD |
215 | } |
216 | ||
217 | if (!c) { | |
faceaf6a CK |
218 | places[c].fpfn = 0; |
219 | places[c].lpfn = 0; | |
48e07c23 | 220 | places[c].mem_type = TTM_PL_SYSTEM; |
ce65b874 | 221 | places[c].flags = 0; |
faceaf6a | 222 | c++; |
d38ceaf9 | 223 | } |
faceaf6a | 224 | |
ea7acd7c | 225 | BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); |
bf314ca3 | 226 | |
7e5a547f | 227 | placement->num_placement = c; |
faceaf6a | 228 | placement->placement = places; |
d38ceaf9 AD |
229 | } |
230 | ||
7c204889 | 231 | /** |
9d903cbd | 232 | * amdgpu_bo_create_reserved - create reserved BO for kernel use |
7c204889 CK |
233 | * |
234 | * @adev: amdgpu device object | |
235 | * @size: size for the new BO | |
236 | * @align: alignment for the new BO | |
237 | * @domain: where to place it | |
64350f1b | 238 | * @bo_ptr: used to initialize BOs in structures |
7c204889 CK |
239 | * @gpu_addr: GPU addr of the pinned BO |
240 | * @cpu_addr: optional CPU address mapping | |
241 | * | |
9d903cbd CK |
242 | * Allocates and pins a BO for kernel internal use, and returns it still |
243 | * reserved. | |
7c204889 | 244 | * |
64350f1b AG |
245 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
246 | * | |
2472e11b MD |
247 | * Returns: |
248 | * 0 on success, negative error code otherwise. | |
7c204889 | 249 | */ |
9d903cbd CK |
250 | int amdgpu_bo_create_reserved(struct amdgpu_device *adev, |
251 | unsigned long size, int align, | |
252 | u32 domain, struct amdgpu_bo **bo_ptr, | |
253 | u64 *gpu_addr, void **cpu_addr) | |
7c204889 | 254 | { |
3216c6b7 | 255 | struct amdgpu_bo_param bp; |
53766e5a | 256 | bool free = false; |
7c204889 CK |
257 | int r; |
258 | ||
21a7e77f CK |
259 | if (!size) { |
260 | amdgpu_bo_unref(bo_ptr); | |
261 | return 0; | |
262 | } | |
263 | ||
3216c6b7 CZ |
264 | memset(&bp, 0, sizeof(bp)); |
265 | bp.size = size; | |
266 | bp.byte_align = align; | |
267 | bp.domain = domain; | |
828d6fde TY |
268 | bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
269 | : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; | |
270 | bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; | |
3216c6b7 CZ |
271 | bp.type = ttm_bo_type_kernel; |
272 | bp.resv = NULL; | |
9fd5543e | 273 | bp.bo_ptr_size = sizeof(struct amdgpu_bo); |
3216c6b7 | 274 | |
53766e5a | 275 | if (!*bo_ptr) { |
3216c6b7 | 276 | r = amdgpu_bo_create(adev, &bp, bo_ptr); |
53766e5a CK |
277 | if (r) { |
278 | dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", | |
279 | r); | |
280 | return r; | |
281 | } | |
282 | free = true; | |
7c204889 CK |
283 | } |
284 | ||
285 | r = amdgpu_bo_reserve(*bo_ptr, false); | |
286 | if (r) { | |
287 | dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); | |
288 | goto error_free; | |
289 | } | |
290 | ||
7b7c6c81 | 291 | r = amdgpu_bo_pin(*bo_ptr, domain); |
7c204889 CK |
292 | if (r) { |
293 | dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); | |
294 | goto error_unreserve; | |
295 | } | |
bb812f1e JZ |
296 | |
297 | r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); | |
298 | if (r) { | |
299 | dev_err(adev->dev, "%p bind failed\n", *bo_ptr); | |
300 | goto error_unpin; | |
301 | } | |
302 | ||
7b7c6c81 JZ |
303 | if (gpu_addr) |
304 | *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); | |
7c204889 CK |
305 | |
306 | if (cpu_addr) { | |
307 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
308 | if (r) { | |
309 | dev_err(adev->dev, "(%d) kernel bo map failed\n", r); | |
dc407ee0 | 310 | goto error_unpin; |
7c204889 CK |
311 | } |
312 | } | |
313 | ||
7c204889 CK |
314 | return 0; |
315 | ||
bb812f1e JZ |
316 | error_unpin: |
317 | amdgpu_bo_unpin(*bo_ptr); | |
7c204889 CK |
318 | error_unreserve: |
319 | amdgpu_bo_unreserve(*bo_ptr); | |
320 | ||
321 | error_free: | |
53766e5a CK |
322 | if (free) |
323 | amdgpu_bo_unref(bo_ptr); | |
7c204889 CK |
324 | |
325 | return r; | |
326 | } | |
327 | ||
9d903cbd CK |
328 | /** |
329 | * amdgpu_bo_create_kernel - create BO for kernel use | |
330 | * | |
331 | * @adev: amdgpu device object | |
332 | * @size: size for the new BO | |
333 | * @align: alignment for the new BO | |
334 | * @domain: where to place it | |
64350f1b | 335 | * @bo_ptr: used to initialize BOs in structures |
9d903cbd CK |
336 | * @gpu_addr: GPU addr of the pinned BO |
337 | * @cpu_addr: optional CPU address mapping | |
338 | * | |
339 | * Allocates and pins a BO for kernel internal use. | |
340 | * | |
64350f1b AG |
341 | * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. |
342 | * | |
2472e11b MD |
343 | * Returns: |
344 | * 0 on success, negative error code otherwise. | |
9d903cbd CK |
345 | */ |
346 | int amdgpu_bo_create_kernel(struct amdgpu_device *adev, | |
347 | unsigned long size, int align, | |
348 | u32 domain, struct amdgpu_bo **bo_ptr, | |
349 | u64 *gpu_addr, void **cpu_addr) | |
350 | { | |
351 | int r; | |
352 | ||
353 | r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, | |
354 | gpu_addr, cpu_addr); | |
355 | ||
356 | if (r) | |
357 | return r; | |
358 | ||
ddaf5013 TSD |
359 | if (*bo_ptr) |
360 | amdgpu_bo_unreserve(*bo_ptr); | |
9d903cbd CK |
361 | |
362 | return 0; | |
363 | } | |
364 | ||
de7b45ba CK |
365 | /** |
366 | * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location | |
367 | * | |
368 | * @adev: amdgpu device object | |
369 | * @offset: offset of the BO | |
370 | * @size: size of the BO | |
de7b45ba CK |
371 | * @bo_ptr: used to initialize BOs in structures |
372 | * @cpu_addr: optional CPU address mapping | |
373 | * | |
3273f116 | 374 | * Creates a kernel BO at a specific offset in VRAM. |
de7b45ba CK |
375 | * |
376 | * Returns: | |
377 | * 0 on success, negative error code otherwise. | |
378 | */ | |
379 | int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, | |
3273f116 | 380 | uint64_t offset, uint64_t size, |
de7b45ba CK |
381 | struct amdgpu_bo **bo_ptr, void **cpu_addr) |
382 | { | |
383 | struct ttm_operation_ctx ctx = { false, false }; | |
384 | unsigned int i; | |
385 | int r; | |
386 | ||
387 | offset &= PAGE_MASK; | |
388 | size = ALIGN(size, PAGE_SIZE); | |
389 | ||
3273f116 LT |
390 | r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, |
391 | AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, | |
392 | cpu_addr); | |
de7b45ba CK |
393 | if (r) |
394 | return r; | |
395 | ||
37912e96 AD |
396 | if ((*bo_ptr) == NULL) |
397 | return 0; | |
398 | ||
de7b45ba CK |
399 | /* |
400 | * Remove the original mem node and create a new one at the request | |
401 | * position. | |
402 | */ | |
4a246528 CK |
403 | if (cpu_addr) |
404 | amdgpu_bo_kunmap(*bo_ptr); | |
405 | ||
bfa3357e | 406 | ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); |
4a246528 | 407 | |
de7b45ba CK |
408 | for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { |
409 | (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; | |
410 | (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; | |
411 | } | |
de7b45ba | 412 | r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, |
bfa3357e | 413 | &(*bo_ptr)->tbo.resource, &ctx); |
de7b45ba CK |
414 | if (r) |
415 | goto error; | |
416 | ||
417 | if (cpu_addr) { | |
418 | r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); | |
419 | if (r) | |
420 | goto error; | |
421 | } | |
422 | ||
423 | amdgpu_bo_unreserve(*bo_ptr); | |
424 | return 0; | |
425 | ||
426 | error: | |
427 | amdgpu_bo_unreserve(*bo_ptr); | |
428 | amdgpu_bo_unref(bo_ptr); | |
429 | return r; | |
430 | } | |
431 | ||
aa1d562e JZ |
432 | /** |
433 | * amdgpu_bo_free_kernel - free BO for kernel use | |
434 | * | |
435 | * @bo: amdgpu BO to free | |
2472e11b MD |
436 | * @gpu_addr: pointer to where the BO's GPU memory space address was stored |
437 | * @cpu_addr: pointer to where the BO's CPU memory space address was stored | |
aa1d562e JZ |
438 | * |
439 | * unmaps and unpin a BO for kernel internal use. | |
440 | */ | |
441 | void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, | |
442 | void **cpu_addr) | |
443 | { | |
444 | if (*bo == NULL) | |
445 | return; | |
446 | ||
4d2ccd96 CK |
447 | WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); |
448 | ||
f3aa745e | 449 | if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { |
aa1d562e JZ |
450 | if (cpu_addr) |
451 | amdgpu_bo_kunmap(*bo); | |
452 | ||
453 | amdgpu_bo_unpin(*bo); | |
454 | amdgpu_bo_unreserve(*bo); | |
455 | } | |
456 | amdgpu_bo_unref(bo); | |
457 | ||
458 | if (gpu_addr) | |
459 | *gpu_addr = 0; | |
460 | ||
461 | if (cpu_addr) | |
462 | *cpu_addr = NULL; | |
463 | } | |
464 | ||
8f9a9a09 | 465 | /* Validate bo size is bit bigger than the request domain */ |
79c63123 AG |
466 | static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, |
467 | unsigned long size, u32 domain) | |
468 | { | |
9de59bc2 | 469 | struct ttm_resource_manager *man = NULL; |
79c63123 AG |
470 | |
471 | /* | |
472 | * If GTT is part of requested domains the check must succeed to | |
7554886d | 473 | * allow fall back to GTT. |
79c63123 | 474 | */ |
8f9a9a09 | 475 | if (domain & AMDGPU_GEM_DOMAIN_GTT) |
6c28aed6 | 476 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); |
8f9a9a09 | 477 | else if (domain & AMDGPU_GEM_DOMAIN_VRAM) |
6c28aed6 | 478 | man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); |
8f9a9a09 MJ |
479 | else |
480 | return true; | |
79c63123 | 481 | |
8f9a9a09 MJ |
482 | if (!man) { |
483 | if (domain & AMDGPU_GEM_DOMAIN_GTT) | |
484 | WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); | |
485 | return false; | |
79c63123 AG |
486 | } |
487 | ||
dc3499c7 | 488 | /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ |
8f9a9a09 MJ |
489 | if (size < man->size) |
490 | return true; | |
79c63123 | 491 | |
8f9a9a09 | 492 | DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); |
79c63123 AG |
493 | return false; |
494 | } | |
495 | ||
3d1b8ec7 AG |
496 | bool amdgpu_bo_support_uswc(u64 bo_flags) |
497 | { | |
498 | ||
499 | #ifdef CONFIG_X86_32 | |
500 | /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit | |
501 | * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 | |
502 | */ | |
503 | return false; | |
504 | #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) | |
505 | /* Don't try to enable write-combining when it can't work, or things | |
506 | * may be slow | |
507 | * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 | |
508 | */ | |
509 | ||
510 | #ifndef CONFIG_COMPILE_TEST | |
511 | #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ | |
512 | thanks to write-combining | |
513 | #endif | |
514 | ||
515 | if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) | |
516 | DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " | |
517 | "better performance thanks to write-combining\n"); | |
518 | return false; | |
519 | #else | |
520 | /* For architectures that don't support WC memory, | |
521 | * mask out the WC flag from the BO | |
522 | */ | |
523 | if (!drm_arch_can_wc_memory()) | |
524 | return false; | |
525 | ||
526 | return true; | |
527 | #endif | |
528 | } | |
529 | ||
cd2454d6 ND |
530 | /** |
531 | * amdgpu_bo_create - create an &amdgpu_bo buffer object | |
532 | * @adev: amdgpu device object | |
533 | * @bp: parameters to be used for the buffer object | |
534 | * @bo_ptr: pointer to the buffer object pointer | |
535 | * | |
536 | * Creates an &amdgpu_bo buffer object. | |
537 | * | |
538 | * Returns: | |
539 | * 0 for success or a negative error code on failure. | |
540 | */ | |
541 | int amdgpu_bo_create(struct amdgpu_device *adev, | |
a906dbb1 | 542 | struct amdgpu_bo_param *bp, |
c09312a6 | 543 | struct amdgpu_bo **bo_ptr) |
d38ceaf9 | 544 | { |
9251859a | 545 | struct ttm_operation_ctx ctx = { |
a906dbb1 | 546 | .interruptible = (bp->type != ttm_bo_type_kernel), |
061468c4 | 547 | .no_wait_gpu = bp->no_wait_gpu, |
586052b0 CK |
548 | /* We opt to avoid OOM on system pages allocations */ |
549 | .gfp_retry_mayfail = true, | |
c44dfe4d CK |
550 | .allow_res_evict = bp->type != ttm_bo_type_kernel, |
551 | .resv = bp->resv | |
9251859a | 552 | }; |
d38ceaf9 | 553 | struct amdgpu_bo *bo; |
a906dbb1 | 554 | unsigned long page_align, size = bp->size; |
d38ceaf9 AD |
555 | int r; |
556 | ||
fe57085a MO |
557 | /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ |
558 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { | |
559 | /* GWS and OA don't need any alignment. */ | |
560 | page_align = bp->byte_align; | |
77a2faa5 | 561 | size <<= PAGE_SHIFT; |
e3c92eb4 | 562 | |
fe57085a MO |
563 | } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { |
564 | /* Both size and alignment must be a multiple of 4. */ | |
565 | page_align = ALIGN(bp->byte_align, 4); | |
566 | size = ALIGN(size, 4) << PAGE_SHIFT; | |
567 | } else { | |
568 | /* Memory should be aligned at least to a page size. */ | |
569 | page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; | |
77a2faa5 | 570 | size = ALIGN(size, PAGE_SIZE); |
fe57085a | 571 | } |
d38ceaf9 | 572 | |
a906dbb1 | 573 | if (!amdgpu_bo_validate_size(adev, size, bp->domain)) |
79c63123 AG |
574 | return -ENOMEM; |
575 | ||
9fd5543e | 576 | BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); |
d38ceaf9 | 577 | |
9fd5543e | 578 | *bo_ptr = NULL; |
31c759bb | 579 | bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); |
d38ceaf9 AD |
580 | if (bo == NULL) |
581 | return -ENOMEM; | |
4a580877 | 582 | drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); |
646b9025 | 583 | bo->vm_bo = NULL; |
3f188453 | 584 | bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : |
aa2b2e28 | 585 | bp->domain; |
08082104 | 586 | bo->allowed_domains = bo->preferred_domains; |
a906dbb1 | 587 | if (bp->type != ttm_bo_type_kernel && |
fab2cc83 | 588 | !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && |
08082104 CK |
589 | bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) |
590 | bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; | |
d38ceaf9 | 591 | |
a906dbb1 | 592 | bo->flags = bp->flags; |
a187f17f | 593 | |
3ebfd221 PY |
594 | if (adev->gmc.mem_partitions) |
595 | /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ | |
596 | bo->xcp_id = bp->xcp_id_plus1 - 1; | |
597 | else | |
598 | /* For GPUs without spatial partitioning */ | |
599 | bo->xcp_id = 0; | |
f24e924b | 600 | |
3d1b8ec7 | 601 | if (!amdgpu_bo_support_uswc(bo->flags)) |
a187f17f OG |
602 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; |
603 | ||
fc6ea4be FK |
604 | if (adev->ras_enabled) |
605 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; | |
606 | ||
c09312a6 | 607 | bo->tbo.bdev = &adev->mman.bdev; |
47722220 CK |
608 | if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | |
609 | AMDGPU_GEM_DOMAIN_GDS)) | |
610 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
611 | else | |
612 | amdgpu_bo_placement_from_domain(bo, bp->domain); | |
a50cb948 | 613 | if (bp->type == ttm_bo_type_kernel) |
e76691f4 FK |
614 | bo->tbo.priority = 2; |
615 | else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) | |
a50cb948 | 616 | bo->tbo.priority = 1; |
08082104 | 617 | |
23e24fbb ND |
618 | if (!bp->destroy) |
619 | bp->destroy = &amdgpu_bo_destroy; | |
620 | ||
347987a2 | 621 | r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, |
f07069da | 622 | &bo->placement, page_align, &ctx, NULL, |
23e24fbb | 623 | bp->resv, bp->destroy); |
08082104 | 624 | if (unlikely(r != 0)) |
a695e437 CK |
625 | return r; |
626 | ||
c8c5e569 | 627 | if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && |
394ae060 | 628 | amdgpu_res_cpu_visible(adev, bo->tbo.resource)) |
6af046d2 CK |
629 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, |
630 | ctx.bytes_moved); | |
00f06b24 | 631 | else |
6af046d2 | 632 | amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); |
fad06127 | 633 | |
a906dbb1 | 634 | if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && |
d3116756 | 635 | bo->tbo.resource->mem_type == TTM_PL_VRAM) { |
f54d1867 | 636 | struct dma_fence *fence; |
4fea83ff | 637 | |
c3aaca43 | 638 | r = amdgpu_fill_buffer(bo, 0, bo->tbo.base.resv, &fence, true); |
c3af1258 CK |
639 | if (unlikely(r)) |
640 | goto fail_unreserve; | |
641 | ||
8bb31587 CK |
642 | dma_resv_add_fence(bo->tbo.base.resv, fence, |
643 | DMA_RESV_USAGE_KERNEL); | |
f54d1867 | 644 | dma_fence_put(fence); |
4fea83ff | 645 | } |
a906dbb1 | 646 | if (!bp->resv) |
59c66c91 | 647 | amdgpu_bo_unreserve(bo); |
d38ceaf9 AD |
648 | *bo_ptr = bo; |
649 | ||
650 | trace_amdgpu_bo_create(bo); | |
651 | ||
96cf8271 | 652 | /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ |
a906dbb1 | 653 | if (bp->type == ttm_bo_type_device) |
96cf8271 JB |
654 | bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; |
655 | ||
d38ceaf9 | 656 | return 0; |
4fea83ff FC |
657 | |
658 | fail_unreserve: | |
a906dbb1 | 659 | if (!bp->resv) |
52791eee | 660 | dma_resv_unlock(bo->tbo.base.resv); |
4fea83ff FC |
661 | amdgpu_bo_unref(&bo); |
662 | return r; | |
d38ceaf9 AD |
663 | } |
664 | ||
9ad0d033 ND |
665 | /** |
666 | * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object | |
667 | * @adev: amdgpu device object | |
668 | * @bp: parameters to be used for the buffer object | |
669 | * @ubo_ptr: pointer to the buffer object pointer | |
670 | * | |
671 | * Create a BO to be used by user application; | |
672 | * | |
673 | * Returns: | |
674 | * 0 for success or a negative error code on failure. | |
675 | */ | |
676 | ||
677 | int amdgpu_bo_create_user(struct amdgpu_device *adev, | |
678 | struct amdgpu_bo_param *bp, | |
679 | struct amdgpu_bo_user **ubo_ptr) | |
680 | { | |
681 | struct amdgpu_bo *bo_ptr; | |
682 | int r; | |
683 | ||
9ad0d033 | 684 | bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); |
23e24fbb | 685 | bp->destroy = &amdgpu_bo_user_destroy; |
cd2454d6 | 686 | r = amdgpu_bo_create(adev, bp, &bo_ptr); |
9ad0d033 ND |
687 | if (r) |
688 | return r; | |
689 | ||
690 | *ubo_ptr = to_amdgpu_bo_user(bo_ptr); | |
691 | return r; | |
692 | } | |
6fdd6f4a ND |
693 | |
694 | /** | |
695 | * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object | |
696 | * @adev: amdgpu device object | |
697 | * @bp: parameters to be used for the buffer object | |
698 | * @vmbo_ptr: pointer to the buffer object pointer | |
699 | * | |
700 | * Create a BO to be for GPUVM. | |
701 | * | |
702 | * Returns: | |
703 | * 0 for success or a negative error code on failure. | |
704 | */ | |
705 | ||
706 | int amdgpu_bo_create_vm(struct amdgpu_device *adev, | |
707 | struct amdgpu_bo_param *bp, | |
708 | struct amdgpu_bo_vm **vmbo_ptr) | |
709 | { | |
710 | struct amdgpu_bo *bo_ptr; | |
711 | int r; | |
712 | ||
713 | /* bo_ptr_size will be determined by the caller and it depends on | |
714 | * num of amdgpu_vm_pt entries. | |
715 | */ | |
716 | BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); | |
717 | r = amdgpu_bo_create(adev, bp, &bo_ptr); | |
718 | if (r) | |
719 | return r; | |
720 | ||
721 | *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); | |
722 | return r; | |
723 | } | |
724 | ||
1fdc79f6 ND |
725 | /** |
726 | * amdgpu_bo_add_to_shadow_list - add a BO to the shadow list | |
727 | * | |
d8c33180 | 728 | * @vmbo: BO that will be inserted into the shadow list |
1fdc79f6 ND |
729 | * |
730 | * Insert a BO to the shadow list. | |
731 | */ | |
e18aaea7 | 732 | void amdgpu_bo_add_to_shadow_list(struct amdgpu_bo_vm *vmbo) |
1fdc79f6 | 733 | { |
e18aaea7 | 734 | struct amdgpu_device *adev = amdgpu_ttm_adev(vmbo->bo.tbo.bdev); |
1fdc79f6 ND |
735 | |
736 | mutex_lock(&adev->shadow_list_lock); | |
e18aaea7 | 737 | list_add_tail(&vmbo->shadow_list, &adev->shadow_list); |
cbb63ecc HZ |
738 | vmbo->shadow->parent = amdgpu_bo_ref(&vmbo->bo); |
739 | vmbo->shadow->tbo.destroy = &amdgpu_bo_vm_destroy; | |
1fdc79f6 ND |
740 | mutex_unlock(&adev->shadow_list_lock); |
741 | } | |
742 | ||
6f4e8d6e | 743 | /** |
403009bf CK |
744 | * amdgpu_bo_restore_shadow - restore an &amdgpu_bo shadow |
745 | * | |
746 | * @shadow: &amdgpu_bo shadow to be restored | |
6f4e8d6e | 747 | * @fence: dma_fence associated with the operation |
6f4e8d6e SL |
748 | * |
749 | * Copies a buffer object's shadow content back to the object. | |
750 | * This is used for recovering a buffer from its shadow in case of a gpu | |
751 | * reset where vram context may be lost. | |
752 | * | |
2472e11b MD |
753 | * Returns: |
754 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 755 | */ |
403009bf | 756 | int amdgpu_bo_restore_shadow(struct amdgpu_bo *shadow, struct dma_fence **fence) |
20f4eff1 CZ |
757 | |
758 | { | |
403009bf CK |
759 | struct amdgpu_device *adev = amdgpu_ttm_adev(shadow->tbo.bdev); |
760 | struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; | |
761 | uint64_t shadow_addr, parent_addr; | |
20f4eff1 | 762 | |
403009bf CK |
763 | shadow_addr = amdgpu_bo_gpu_offset(shadow); |
764 | parent_addr = amdgpu_bo_gpu_offset(shadow->parent); | |
20f4eff1 | 765 | |
403009bf CK |
766 | return amdgpu_copy_buffer(ring, shadow_addr, parent_addr, |
767 | amdgpu_bo_size(shadow), NULL, fence, | |
c9dc9cfe | 768 | true, false, false); |
20f4eff1 CZ |
769 | } |
770 | ||
6f4e8d6e SL |
771 | /** |
772 | * amdgpu_bo_kmap - map an &amdgpu_bo buffer object | |
773 | * @bo: &amdgpu_bo buffer object to be mapped | |
774 | * @ptr: kernel virtual address to be returned | |
775 | * | |
776 | * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls | |
777 | * amdgpu_bo_kptr() to get the kernel virtual address. | |
778 | * | |
2472e11b MD |
779 | * Returns: |
780 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 781 | */ |
d38ceaf9 AD |
782 | int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) |
783 | { | |
f5e1c740 | 784 | void *kptr; |
587f3c70 | 785 | long r; |
d38ceaf9 | 786 | |
271c8125 CK |
787 | if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) |
788 | return -EPERM; | |
789 | ||
c35fcfa3 CK |
790 | r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, |
791 | false, MAX_SCHEDULE_TIMEOUT); | |
792 | if (r < 0) | |
793 | return r; | |
794 | ||
f5e1c740 CK |
795 | kptr = amdgpu_bo_kptr(bo); |
796 | if (kptr) { | |
797 | if (ptr) | |
798 | *ptr = kptr; | |
d38ceaf9 AD |
799 | return 0; |
800 | } | |
587f3c70 | 801 | |
e3c92eb4 | 802 | r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); |
587f3c70 | 803 | if (r) |
d38ceaf9 | 804 | return r; |
587f3c70 | 805 | |
587f3c70 | 806 | if (ptr) |
f5e1c740 | 807 | *ptr = amdgpu_bo_kptr(bo); |
587f3c70 | 808 | |
d38ceaf9 AD |
809 | return 0; |
810 | } | |
811 | ||
6f4e8d6e SL |
812 | /** |
813 | * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object | |
814 | * @bo: &amdgpu_bo buffer object | |
815 | * | |
816 | * Calls ttm_kmap_obj_virtual() to get the kernel virtual address | |
817 | * | |
2472e11b MD |
818 | * Returns: |
819 | * the virtual address of a buffer object area. | |
6f4e8d6e | 820 | */ |
f5e1c740 CK |
821 | void *amdgpu_bo_kptr(struct amdgpu_bo *bo) |
822 | { | |
823 | bool is_iomem; | |
824 | ||
825 | return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); | |
826 | } | |
827 | ||
6f4e8d6e SL |
828 | /** |
829 | * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object | |
830 | * @bo: &amdgpu_bo buffer object to be unmapped | |
831 | * | |
832 | * Unmaps a kernel map set up by amdgpu_bo_kmap(). | |
833 | */ | |
d38ceaf9 AD |
834 | void amdgpu_bo_kunmap(struct amdgpu_bo *bo) |
835 | { | |
f5e1c740 CK |
836 | if (bo->kmap.bo) |
837 | ttm_bo_kunmap(&bo->kmap); | |
d38ceaf9 AD |
838 | } |
839 | ||
6f4e8d6e SL |
840 | /** |
841 | * amdgpu_bo_ref - reference an &amdgpu_bo buffer object | |
842 | * @bo: &amdgpu_bo buffer object | |
843 | * | |
844 | * References the contained &ttm_buffer_object. | |
845 | * | |
2472e11b MD |
846 | * Returns: |
847 | * a refcounted pointer to the &amdgpu_bo buffer object. | |
6f4e8d6e | 848 | */ |
d38ceaf9 AD |
849 | struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) |
850 | { | |
851 | if (bo == NULL) | |
852 | return NULL; | |
853 | ||
71d5ef11 | 854 | ttm_bo_get(&bo->tbo); |
d38ceaf9 AD |
855 | return bo; |
856 | } | |
857 | ||
6f4e8d6e SL |
858 | /** |
859 | * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object | |
860 | * @bo: &amdgpu_bo buffer object | |
861 | * | |
862 | * Unreferences the contained &ttm_buffer_object and clear the pointer | |
863 | */ | |
d38ceaf9 AD |
864 | void amdgpu_bo_unref(struct amdgpu_bo **bo) |
865 | { | |
866 | struct ttm_buffer_object *tbo; | |
867 | ||
868 | if ((*bo) == NULL) | |
869 | return; | |
870 | ||
871 | tbo = &((*bo)->tbo); | |
fea872b2 TZ |
872 | ttm_bo_put(tbo); |
873 | *bo = NULL; | |
d38ceaf9 AD |
874 | } |
875 | ||
6f4e8d6e SL |
876 | /** |
877 | * amdgpu_bo_pin_restricted - pin an &amdgpu_bo buffer object | |
878 | * @bo: &amdgpu_bo buffer object to be pinned | |
879 | * @domain: domain to be pinned to | |
880 | * @min_offset: the start of requested address range | |
881 | * @max_offset: the end of requested address range | |
6f4e8d6e SL |
882 | * |
883 | * Pins the buffer object according to requested domain and address range. If | |
884 | * the memory is unbound gart memory, binds the pages into gart table. Adjusts | |
885 | * pin_count and pin_size accordingly. | |
886 | * | |
887 | * Pinning means to lock pages in memory along with keeping them at a fixed | |
888 | * offset. It is required when a buffer can not be moved, for example, when | |
889 | * a display buffer is being scanned out. | |
890 | * | |
891 | * Compared with amdgpu_bo_pin(), this function gives more flexibility on | |
892 | * where to pin a buffer if there are specific restrictions on where a buffer | |
893 | * must be located. | |
894 | * | |
2472e11b MD |
895 | * Returns: |
896 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 897 | */ |
7e5a547f | 898 | int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, |
7b7c6c81 | 899 | u64 min_offset, u64 max_offset) |
d38ceaf9 | 900 | { |
a7d64de6 | 901 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
19be5570 | 902 | struct ttm_operation_ctx ctx = { false, false }; |
d38ceaf9 AD |
903 | int r, i; |
904 | ||
cc325d19 | 905 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) |
d38ceaf9 AD |
906 | return -EPERM; |
907 | ||
7e5a547f CZ |
908 | if (WARN_ON_ONCE(min_offset > max_offset)) |
909 | return -EINVAL; | |
910 | ||
f5ba1404 LL |
911 | /* Check domain to be pinned to against preferred domains */ |
912 | if (bo->preferred_domains & domain) | |
913 | domain = bo->preferred_domains & domain; | |
914 | ||
803d89ad | 915 | /* A shared bo cannot be migrated to VRAM */ |
8c505bdc | 916 | if (bo->tbo.base.import_attach) { |
9b3f217f SL |
917 | if (domain & AMDGPU_GEM_DOMAIN_GTT) |
918 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
919 | else | |
920 | return -EINVAL; | |
921 | } | |
803d89ad | 922 | |
4671078e | 923 | if (bo->tbo.pin_count) { |
d3116756 CK |
924 | uint32_t mem_type = bo->tbo.resource->mem_type; |
925 | uint32_t mem_flags = bo->tbo.resource->placement; | |
408778e8 | 926 | |
f5318959 | 927 | if (!(domain & amdgpu_mem_type_to_domain(mem_type))) |
408778e8 FC |
928 | return -EINVAL; |
929 | ||
e1a4b67a | 930 | if ((mem_type == TTM_PL_VRAM) && |
931 | (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && | |
eda1068d FK |
932 | !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) |
933 | return -EINVAL; | |
934 | ||
4671078e | 935 | ttm_bo_pin(&bo->tbo); |
d38ceaf9 AD |
936 | |
937 | if (max_offset != 0) { | |
b1a8ef95 ND |
938 | u64 domain_start = amdgpu_ttm_domain_start(adev, |
939 | mem_type); | |
d38ceaf9 AD |
940 | WARN_ON_ONCE(max_offset < |
941 | (amdgpu_bo_gpu_offset(bo) - domain_start)); | |
942 | } | |
943 | ||
944 | return 0; | |
945 | } | |
03f48dd5 | 946 | |
9deb0b3d CK |
947 | /* This assumes only APU display buffers are pinned with (VRAM|GTT). |
948 | * See function amdgpu_display_supported_domains() | |
949 | */ | |
d035f84d | 950 | domain = amdgpu_bo_get_preferred_domain(adev, domain); |
9deb0b3d | 951 | |
a448cb00 CK |
952 | if (bo->tbo.base.import_attach) |
953 | dma_buf_pin(bo->tbo.base.import_attach); | |
954 | ||
e9c7577c CK |
955 | /* force to pin into visible video ram */ |
956 | if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) | |
957 | bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
c704ab18 | 958 | amdgpu_bo_placement_from_domain(bo, domain); |
d38ceaf9 | 959 | for (i = 0; i < bo->placement.num_placement; i++) { |
1d6ecab1 | 960 | unsigned int fpfn, lpfn; |
e9c7577c CK |
961 | |
962 | fpfn = min_offset >> PAGE_SHIFT; | |
963 | lpfn = max_offset >> PAGE_SHIFT; | |
964 | ||
7e5a547f CZ |
965 | if (fpfn > bo->placements[i].fpfn) |
966 | bo->placements[i].fpfn = fpfn; | |
78d0e182 CK |
967 | if (!bo->placements[i].lpfn || |
968 | (lpfn && lpfn < bo->placements[i].lpfn)) | |
7e5a547f | 969 | bo->placements[i].lpfn = lpfn; |
d38ceaf9 AD |
970 | } |
971 | ||
19be5570 | 972 | r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
6681c5eb | 973 | if (unlikely(r)) { |
a7d64de6 | 974 | dev_err(adev->dev, "%p pin failed\n", bo); |
6681c5eb CK |
975 | goto error; |
976 | } | |
977 | ||
4671078e | 978 | ttm_bo_pin(&bo->tbo); |
5e91fb57 | 979 | |
d3116756 | 980 | domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); |
6681c5eb | 981 | if (domain == AMDGPU_GEM_DOMAIN_VRAM) { |
a5ccfe5c MD |
982 | atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); |
983 | atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), | |
984 | &adev->visible_pin_size); | |
32ab75f0 | 985 | } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { |
a5ccfe5c | 986 | atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); |
d38ceaf9 | 987 | } |
6681c5eb CK |
988 | |
989 | error: | |
d38ceaf9 AD |
990 | return r; |
991 | } | |
992 | ||
6f4e8d6e SL |
993 | /** |
994 | * amdgpu_bo_pin - pin an &amdgpu_bo buffer object | |
995 | * @bo: &amdgpu_bo buffer object to be pinned | |
996 | * @domain: domain to be pinned to | |
6f4e8d6e SL |
997 | * |
998 | * A simple wrapper to amdgpu_bo_pin_restricted(). | |
999 | * Provides a simpler API for buffers that do not have any strict restrictions | |
1000 | * on where a buffer must be located. | |
1001 | * | |
2472e11b MD |
1002 | * Returns: |
1003 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1004 | */ |
7b7c6c81 | 1005 | int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) |
d38ceaf9 | 1006 | { |
eda1068d | 1007 | bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; |
7b7c6c81 | 1008 | return amdgpu_bo_pin_restricted(bo, domain, 0, 0); |
d38ceaf9 AD |
1009 | } |
1010 | ||
6f4e8d6e SL |
1011 | /** |
1012 | * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object | |
1013 | * @bo: &amdgpu_bo buffer object to be unpinned | |
1014 | * | |
1015 | * Decreases the pin_count, and clears the flags if pin_count reaches 0. | |
1016 | * Changes placement and pin size accordingly. | |
1017 | * | |
2472e11b MD |
1018 | * Returns: |
1019 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1020 | */ |
4671078e | 1021 | void amdgpu_bo_unpin(struct amdgpu_bo *bo) |
d38ceaf9 | 1022 | { |
e2ac8531 CK |
1023 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
1024 | ||
4671078e CK |
1025 | ttm_bo_unpin(&bo->tbo); |
1026 | if (bo->tbo.pin_count) | |
1027 | return; | |
6681c5eb | 1028 | |
a448cb00 CK |
1029 | if (bo->tbo.base.import_attach) |
1030 | dma_buf_unpin(bo->tbo.base.import_attach); | |
e2ac8531 | 1031 | |
d3116756 | 1032 | if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { |
e2ac8531 CK |
1033 | atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); |
1034 | atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), | |
1035 | &adev->visible_pin_size); | |
d3116756 | 1036 | } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { |
e2ac8531 CK |
1037 | atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); |
1038 | } | |
dc3499c7 | 1039 | |
d38ceaf9 AD |
1040 | } |
1041 | ||
1d6ecab1 | 1042 | static const char * const amdgpu_vram_names[] = { |
1f8628c7 AD |
1043 | "UNKNOWN", |
1044 | "GDDR1", | |
1045 | "DDR2", | |
1046 | "GDDR3", | |
1047 | "GDDR4", | |
1048 | "GDDR5", | |
1049 | "HBM", | |
bc227cfa TSD |
1050 | "DDR3", |
1051 | "DDR4", | |
5228fe30 | 1052 | "GDDR6", |
d534ca71 AD |
1053 | "DDR5", |
1054 | "LPDDR4", | |
1055 | "LPDDR5" | |
1f8628c7 AD |
1056 | }; |
1057 | ||
6f4e8d6e SL |
1058 | /** |
1059 | * amdgpu_bo_init - initialize memory manager | |
1060 | * @adev: amdgpu device object | |
1061 | * | |
1062 | * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. | |
1063 | * | |
2472e11b MD |
1064 | * Returns: |
1065 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1066 | */ |
d38ceaf9 AD |
1067 | int amdgpu_bo_init(struct amdgpu_device *adev) |
1068 | { | |
35d5f224 | 1069 | /* On A+A platform, VRAM can be mapped as WB */ |
228ce176 | 1070 | if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { |
35d5f224 | 1071 | /* reserve PAT memory space to WC for VRAM */ |
26db557e | 1072 | int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, |
35d5f224 OZ |
1073 | adev->gmc.aper_size); |
1074 | ||
26db557e ND |
1075 | if (r) { |
1076 | DRM_ERROR("Unable to set WC memtype for the aperture base\n"); | |
1077 | return r; | |
1078 | } | |
1079 | ||
35d5f224 OZ |
1080 | /* Add an MTRR for the VRAM */ |
1081 | adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, | |
1082 | adev->gmc.aper_size); | |
1083 | } | |
7cf321d1 | 1084 | |
d38ceaf9 | 1085 | DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", |
770d13b1 CK |
1086 | adev->gmc.mc_vram_size >> 20, |
1087 | (unsigned long long)adev->gmc.aper_size >> 20); | |
1f8628c7 | 1088 | DRM_INFO("RAM width %dbits %s\n", |
770d13b1 | 1089 | adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); |
d38ceaf9 AD |
1090 | return amdgpu_ttm_init(adev); |
1091 | } | |
1092 | ||
6f4e8d6e SL |
1093 | /** |
1094 | * amdgpu_bo_fini - tear down memory manager | |
1095 | * @adev: amdgpu device object | |
1096 | * | |
1097 | * Reverses amdgpu_bo_init() to tear down memory manager. | |
1098 | */ | |
d38ceaf9 AD |
1099 | void amdgpu_bo_fini(struct amdgpu_device *adev) |
1100 | { | |
62d5f9f7 LS |
1101 | int idx; |
1102 | ||
d38ceaf9 | 1103 | amdgpu_ttm_fini(adev); |
62d5f9f7 LS |
1104 | |
1105 | if (drm_dev_enter(adev_to_drm(adev), &idx)) { | |
a0ba1279 | 1106 | if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { |
62d5f9f7 LS |
1107 | arch_phys_wc_del(adev->gmc.vram_mtrr); |
1108 | arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); | |
1109 | } | |
1110 | drm_dev_exit(idx); | |
1111 | } | |
d38ceaf9 AD |
1112 | } |
1113 | ||
6f4e8d6e SL |
1114 | /** |
1115 | * amdgpu_bo_set_tiling_flags - set tiling flags | |
1116 | * @bo: &amdgpu_bo buffer object | |
1117 | * @tiling_flags: new flags | |
1118 | * | |
1119 | * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or | |
1120 | * kernel driver to set the tiling flags on a buffer. | |
1121 | * | |
2472e11b MD |
1122 | * Returns: |
1123 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1124 | */ |
d38ceaf9 AD |
1125 | int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) |
1126 | { | |
9079ac76 | 1127 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
cc1bcf85 | 1128 | struct amdgpu_bo_user *ubo; |
9079ac76 | 1129 | |
030bb4ad | 1130 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
9079ac76 MO |
1131 | if (adev->family <= AMDGPU_FAMILY_CZ && |
1132 | AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) | |
d38ceaf9 | 1133 | return -EINVAL; |
d38ceaf9 | 1134 | |
cc1bcf85 ND |
1135 | ubo = to_amdgpu_bo_user(bo); |
1136 | ubo->tiling_flags = tiling_flags; | |
d38ceaf9 AD |
1137 | return 0; |
1138 | } | |
1139 | ||
6f4e8d6e SL |
1140 | /** |
1141 | * amdgpu_bo_get_tiling_flags - get tiling flags | |
1142 | * @bo: &amdgpu_bo buffer object | |
1143 | * @tiling_flags: returned flags | |
1144 | * | |
1145 | * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to | |
1146 | * set the tiling flags on a buffer. | |
1147 | */ | |
d38ceaf9 AD |
1148 | void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) |
1149 | { | |
cc1bcf85 ND |
1150 | struct amdgpu_bo_user *ubo; |
1151 | ||
030bb4ad | 1152 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
52791eee | 1153 | dma_resv_assert_held(bo->tbo.base.resv); |
cc1bcf85 | 1154 | ubo = to_amdgpu_bo_user(bo); |
d38ceaf9 AD |
1155 | |
1156 | if (tiling_flags) | |
cc1bcf85 | 1157 | *tiling_flags = ubo->tiling_flags; |
d38ceaf9 AD |
1158 | } |
1159 | ||
6f4e8d6e SL |
1160 | /** |
1161 | * amdgpu_bo_set_metadata - set metadata | |
1162 | * @bo: &amdgpu_bo buffer object | |
1163 | * @metadata: new metadata | |
1164 | * @metadata_size: size of the new metadata | |
1165 | * @flags: flags of the new metadata | |
1166 | * | |
1167 | * Sets buffer object's metadata, its size and flags. | |
1168 | * Used via GEM ioctl. | |
1169 | * | |
2472e11b MD |
1170 | * Returns: |
1171 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1172 | */ |
1d6ecab1 SS |
1173 | int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, |
1174 | u32 metadata_size, uint64_t flags) | |
d38ceaf9 | 1175 | { |
cc1bcf85 | 1176 | struct amdgpu_bo_user *ubo; |
d38ceaf9 AD |
1177 | void *buffer; |
1178 | ||
030bb4ad | 1179 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
cc1bcf85 | 1180 | ubo = to_amdgpu_bo_user(bo); |
d38ceaf9 | 1181 | if (!metadata_size) { |
cc1bcf85 ND |
1182 | if (ubo->metadata_size) { |
1183 | kfree(ubo->metadata); | |
1184 | ubo->metadata = NULL; | |
1185 | ubo->metadata_size = 0; | |
d38ceaf9 AD |
1186 | } |
1187 | return 0; | |
1188 | } | |
1189 | ||
1190 | if (metadata == NULL) | |
1191 | return -EINVAL; | |
1192 | ||
71affda5 | 1193 | buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); |
d38ceaf9 AD |
1194 | if (buffer == NULL) |
1195 | return -ENOMEM; | |
1196 | ||
cc1bcf85 ND |
1197 | kfree(ubo->metadata); |
1198 | ubo->metadata_flags = flags; | |
1199 | ubo->metadata = buffer; | |
1200 | ubo->metadata_size = metadata_size; | |
d38ceaf9 AD |
1201 | |
1202 | return 0; | |
1203 | } | |
1204 | ||
6f4e8d6e SL |
1205 | /** |
1206 | * amdgpu_bo_get_metadata - get metadata | |
1207 | * @bo: &amdgpu_bo buffer object | |
1208 | * @buffer: returned metadata | |
1209 | * @buffer_size: size of the buffer | |
1210 | * @metadata_size: size of the returned metadata | |
1211 | * @flags: flags of the returned metadata | |
1212 | * | |
1213 | * Gets buffer object's metadata, its size and flags. buffer_size shall not be | |
1214 | * less than metadata_size. | |
1215 | * Used via GEM ioctl. | |
1216 | * | |
2472e11b MD |
1217 | * Returns: |
1218 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1219 | */ |
d38ceaf9 AD |
1220 | int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, |
1221 | size_t buffer_size, uint32_t *metadata_size, | |
1222 | uint64_t *flags) | |
1223 | { | |
cc1bcf85 ND |
1224 | struct amdgpu_bo_user *ubo; |
1225 | ||
d38ceaf9 AD |
1226 | if (!buffer && !metadata_size) |
1227 | return -EINVAL; | |
1228 | ||
030bb4ad | 1229 | BUG_ON(bo->tbo.type == ttm_bo_type_kernel); |
cc1bcf85 | 1230 | ubo = to_amdgpu_bo_user(bo); |
eba98523 SZ |
1231 | if (metadata_size) |
1232 | *metadata_size = ubo->metadata_size; | |
1233 | ||
d38ceaf9 | 1234 | if (buffer) { |
cc1bcf85 | 1235 | if (buffer_size < ubo->metadata_size) |
d38ceaf9 AD |
1236 | return -EINVAL; |
1237 | ||
cc1bcf85 ND |
1238 | if (ubo->metadata_size) |
1239 | memcpy(buffer, ubo->metadata, ubo->metadata_size); | |
d38ceaf9 AD |
1240 | } |
1241 | ||
d38ceaf9 | 1242 | if (flags) |
cc1bcf85 | 1243 | *flags = ubo->metadata_flags; |
d38ceaf9 AD |
1244 | |
1245 | return 0; | |
1246 | } | |
1247 | ||
6f4e8d6e SL |
1248 | /** |
1249 | * amdgpu_bo_move_notify - notification about a memory move | |
1250 | * @bo: pointer to a buffer object | |
1251 | * @evict: if this move is evicting the buffer from the graphics address space | |
6f4e8d6e SL |
1252 | * |
1253 | * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs | |
1254 | * bookkeeping. | |
1255 | * TTM driver callback which is called when ttm moves a buffer. | |
1256 | */ | |
94aeb411 | 1257 | void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, bool evict) |
d38ceaf9 | 1258 | { |
a7d64de6 | 1259 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
765e7fbf | 1260 | struct amdgpu_bo *abo; |
d38ceaf9 | 1261 | |
c704ab18 | 1262 | if (!amdgpu_bo_is_amdgpu_bo(bo)) |
d38ceaf9 AD |
1263 | return; |
1264 | ||
b82485fd | 1265 | abo = ttm_to_amdgpu_bo(bo); |
3f3333f8 | 1266 | amdgpu_vm_bo_invalidate(adev, abo, evict); |
d38ceaf9 | 1267 | |
6375bbb4 CK |
1268 | amdgpu_bo_kunmap(abo); |
1269 | ||
2d4dad27 | 1270 | if (abo->tbo.base.dma_buf && !abo->tbo.base.import_attach && |
d3116756 | 1271 | bo->resource->mem_type != TTM_PL_SYSTEM) |
2d4dad27 CK |
1272 | dma_buf_move_notify(abo->tbo.base.dma_buf); |
1273 | ||
661a7606 NH |
1274 | /* remember the eviction */ |
1275 | if (evict) | |
1276 | atomic64_inc(&adev->num_evictions); | |
d38ceaf9 AD |
1277 | } |
1278 | ||
d6530c33 MO |
1279 | void amdgpu_bo_get_memory(struct amdgpu_bo *bo, |
1280 | struct amdgpu_mem_stats *stats) | |
87444254 | 1281 | { |
394ae060 CK |
1282 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
1283 | struct ttm_resource *res = bo->tbo.resource; | |
d6530c33 | 1284 | uint64_t size = amdgpu_bo_size(bo); |
ba1a58d5 | 1285 | struct drm_gem_object *obj; |
ca0b954a | 1286 | unsigned int domain; |
ba1a58d5 | 1287 | bool shared; |
ca0b954a CK |
1288 | |
1289 | /* Abort if the BO doesn't currently have a backing store */ | |
394ae060 | 1290 | if (!res) |
ca0b954a | 1291 | return; |
87444254 | 1292 | |
ba1a58d5 AD |
1293 | obj = &bo->tbo.base; |
1294 | shared = drm_gem_object_is_shared_for_memory_stats(obj); | |
1295 | ||
394ae060 | 1296 | domain = amdgpu_mem_type_to_domain(res->mem_type); |
87444254 RS |
1297 | switch (domain) { |
1298 | case AMDGPU_GEM_DOMAIN_VRAM: | |
d6530c33 | 1299 | stats->vram += size; |
394ae060 | 1300 | if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) |
d6530c33 | 1301 | stats->visible_vram += size; |
ba1a58d5 AD |
1302 | if (shared) |
1303 | stats->vram_shared += size; | |
87444254 RS |
1304 | break; |
1305 | case AMDGPU_GEM_DOMAIN_GTT: | |
d6530c33 | 1306 | stats->gtt += size; |
ba1a58d5 AD |
1307 | if (shared) |
1308 | stats->gtt_shared += size; | |
87444254 RS |
1309 | break; |
1310 | case AMDGPU_GEM_DOMAIN_CPU: | |
1311 | default: | |
d6530c33 | 1312 | stats->cpu += size; |
ba1a58d5 AD |
1313 | if (shared) |
1314 | stats->cpu_shared += size; | |
87444254 RS |
1315 | break; |
1316 | } | |
d6530c33 MO |
1317 | |
1318 | if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) { | |
1319 | stats->requested_vram += size; | |
1320 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) | |
1321 | stats->requested_visible_vram += size; | |
1322 | ||
1323 | if (domain != AMDGPU_GEM_DOMAIN_VRAM) { | |
1324 | stats->evicted_vram += size; | |
1325 | if (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) | |
1326 | stats->evicted_visible_vram += size; | |
1327 | } | |
1328 | } else if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_GTT) { | |
1329 | stats->requested_gtt += size; | |
1330 | } | |
87444254 RS |
1331 | } |
1332 | ||
ab2f7a5c | 1333 | /** |
736b1729 | 1334 | * amdgpu_bo_release_notify - notification about a BO being released |
ab2f7a5c FK |
1335 | * @bo: pointer to a buffer object |
1336 | * | |
1337 | * Wipes VRAM buffers whose contents should not be leaked before the | |
1338 | * memory is released. | |
1339 | */ | |
1340 | void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) | |
1341 | { | |
32f90e65 | 1342 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
ab2f7a5c FK |
1343 | struct dma_fence *fence = NULL; |
1344 | struct amdgpu_bo *abo; | |
1345 | int r; | |
1346 | ||
1347 | if (!amdgpu_bo_is_amdgpu_bo(bo)) | |
1348 | return; | |
1349 | ||
1350 | abo = ttm_to_amdgpu_bo(bo); | |
1351 | ||
65d2765d CK |
1352 | WARN_ON(abo->vm_bo); |
1353 | ||
ab2f7a5c | 1354 | if (abo->kfd_bo) |
5702d052 | 1355 | amdgpu_amdkfd_release_notify(abo); |
ab2f7a5c | 1356 | |
f4a3c42b | 1357 | /* We only remove the fence if the resv has individualized. */ |
9fe58d0b | 1358 | WARN_ON_ONCE(bo->type == ttm_bo_type_kernel |
1359 | && bo->base.resv != &bo->base._resv); | |
f4a3c42b | 1360 | if (bo->base.resv == &bo->base._resv) |
1361 | amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo); | |
1362 | ||
63af82cf | 1363 | if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || |
32f90e65 | 1364 | !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || |
93bb18d2 | 1365 | adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) |
ab2f7a5c FK |
1366 | return; |
1367 | ||
447c7997 RB |
1368 | if (WARN_ON_ONCE(!dma_resv_trylock(bo->base.resv))) |
1369 | return; | |
ab2f7a5c | 1370 | |
c3aaca43 | 1371 | r = amdgpu_fill_buffer(abo, AMDGPU_POISON, bo->base.resv, &fence, true); |
ab2f7a5c FK |
1372 | if (!WARN_ON(r)) { |
1373 | amdgpu_bo_fence(abo, fence, false); | |
1374 | dma_fence_put(fence); | |
1375 | } | |
1376 | ||
5f680625 | 1377 | dma_resv_unlock(bo->base.resv); |
ab2f7a5c FK |
1378 | } |
1379 | ||
6f4e8d6e SL |
1380 | /** |
1381 | * amdgpu_bo_fault_reserve_notify - notification about a memory fault | |
1382 | * @bo: pointer to a buffer object | |
1383 | * | |
1384 | * Notifies the driver we are taking a fault on this BO and have reserved it, | |
1385 | * also performs bookkeeping. | |
1386 | * TTM driver callback for dealing with vm faults. | |
1387 | * | |
2472e11b MD |
1388 | * Returns: |
1389 | * 0 for success or a negative error code on failure. | |
6f4e8d6e | 1390 | */ |
d3ef581a | 1391 | vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) |
d38ceaf9 | 1392 | { |
a7d64de6 | 1393 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); |
19be5570 | 1394 | struct ttm_operation_ctx ctx = { false, false }; |
d3ef581a | 1395 | struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); |
96cf8271 | 1396 | int r; |
d38ceaf9 | 1397 | |
96cf8271 JB |
1398 | /* Remember that this BO was accessed by the CPU */ |
1399 | abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; | |
1400 | ||
394ae060 | 1401 | if (amdgpu_res_cpu_visible(adev, bo->resource)) |
5fb1941d CK |
1402 | return 0; |
1403 | ||
104ece97 | 1404 | /* Can't move a pinned BO to visible VRAM */ |
4671078e | 1405 | if (abo->tbo.pin_count > 0) |
d3ef581a | 1406 | return VM_FAULT_SIGBUS; |
104ece97 | 1407 | |
5fb1941d | 1408 | /* hurrah the memory is not visible ! */ |
68e2c5ff | 1409 | atomic64_inc(&adev->num_vram_cpu_page_faults); |
c704ab18 CK |
1410 | amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | |
1411 | AMDGPU_GEM_DOMAIN_GTT); | |
41d9a6a7 JB |
1412 | |
1413 | /* Avoid costly evictions; only set GTT as a busy placement */ | |
a78a8da5 | 1414 | abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; |
41d9a6a7 | 1415 | |
19be5570 | 1416 | r = ttm_bo_validate(bo, &abo->placement, &ctx); |
d3ef581a CK |
1417 | if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) |
1418 | return VM_FAULT_NOPAGE; | |
1419 | else if (unlikely(r)) | |
1420 | return VM_FAULT_SIGBUS; | |
5fb1941d | 1421 | |
5fb1941d | 1422 | /* this should never happen */ |
d3116756 | 1423 | if (bo->resource->mem_type == TTM_PL_VRAM && |
394ae060 | 1424 | !amdgpu_res_cpu_visible(adev, bo->resource)) |
d3ef581a | 1425 | return VM_FAULT_SIGBUS; |
5fb1941d | 1426 | |
d3ef581a | 1427 | ttm_bo_move_to_lru_tail_unlocked(bo); |
d38ceaf9 AD |
1428 | return 0; |
1429 | } | |
1430 | ||
1431 | /** | |
1432 | * amdgpu_bo_fence - add fence to buffer object | |
1433 | * | |
1434 | * @bo: buffer object in question | |
1435 | * @fence: fence to add | |
1436 | * @shared: true if fence should be added shared | |
1437 | * | |
1438 | */ | |
f54d1867 | 1439 | void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, |
d38ceaf9 AD |
1440 | bool shared) |
1441 | { | |
52791eee | 1442 | struct dma_resv *resv = bo->tbo.base.resv; |
c8d4c18b CK |
1443 | int r; |
1444 | ||
1445 | r = dma_resv_reserve_fences(resv, 1); | |
1446 | if (r) { | |
1447 | /* As last resort on OOM we block for the fence */ | |
1448 | dma_fence_wait(fence, false); | |
1449 | return; | |
1450 | } | |
d38ceaf9 | 1451 | |
73511edf CK |
1452 | dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : |
1453 | DMA_RESV_USAGE_WRITE); | |
d38ceaf9 | 1454 | } |
cdb7e8f2 | 1455 | |
e8e32426 | 1456 | /** |
9f3cc18d | 1457 | * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences |
e8e32426 | 1458 | * |
9f3cc18d CK |
1459 | * @adev: amdgpu device pointer |
1460 | * @resv: reservation object to sync to | |
1461 | * @sync_mode: synchronization mode | |
e8e32426 FK |
1462 | * @owner: fence owner |
1463 | * @intr: Whether the wait is interruptible | |
1464 | * | |
9f3cc18d CK |
1465 | * Extract the fences from the reservation object and waits for them to finish. |
1466 | * | |
e8e32426 FK |
1467 | * Returns: |
1468 | * 0 on success, errno otherwise. | |
1469 | */ | |
9f3cc18d CK |
1470 | int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, |
1471 | enum amdgpu_sync_mode sync_mode, void *owner, | |
1472 | bool intr) | |
e8e32426 | 1473 | { |
e8e32426 FK |
1474 | struct amdgpu_sync sync; |
1475 | int r; | |
1476 | ||
1477 | amdgpu_sync_create(&sync); | |
9f3cc18d | 1478 | amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); |
e8e32426 FK |
1479 | r = amdgpu_sync_wait(&sync, intr); |
1480 | amdgpu_sync_free(&sync); | |
e8e32426 FK |
1481 | return r; |
1482 | } | |
1483 | ||
9f3cc18d CK |
1484 | /** |
1485 | * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv | |
1486 | * @bo: buffer object to wait for | |
1487 | * @owner: fence owner | |
1488 | * @intr: Whether the wait is interruptible | |
1489 | * | |
1490 | * Wrapper to wait for fences in a BO. | |
1491 | * Returns: | |
1492 | * 0 on success, errno otherwise. | |
1493 | */ | |
1494 | int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) | |
1495 | { | |
1496 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
1497 | ||
1498 | return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, | |
1499 | AMDGPU_SYNC_NE_OWNER, owner, intr); | |
1500 | } | |
1501 | ||
cdb7e8f2 CK |
1502 | /** |
1503 | * amdgpu_bo_gpu_offset - return GPU offset of bo | |
1504 | * @bo: amdgpu object for which we query the offset | |
1505 | * | |
cdb7e8f2 CK |
1506 | * Note: object should either be pinned or reserved when calling this |
1507 | * function, it might be useful to add check for this for debugging. | |
2472e11b MD |
1508 | * |
1509 | * Returns: | |
1510 | * current GPU offset of the object. | |
cdb7e8f2 CK |
1511 | */ |
1512 | u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) | |
1513 | { | |
d3116756 | 1514 | WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); |
52791eee | 1515 | WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && |
4671078e | 1516 | !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); |
d3116756 CK |
1517 | WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); |
1518 | WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && | |
03f48dd5 | 1519 | !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); |
cdb7e8f2 | 1520 | |
b1a8ef95 ND |
1521 | return amdgpu_bo_gpu_offset_no_check(bo); |
1522 | } | |
1523 | ||
1524 | /** | |
1525 | * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo | |
1526 | * @bo: amdgpu object for which we query the offset | |
1527 | * | |
1528 | * Returns: | |
1529 | * current GPU offset of the object without raising warnings. | |
1530 | */ | |
1531 | u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) | |
1532 | { | |
1533 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); | |
ca0b0069 | 1534 | uint64_t offset = AMDGPU_BO_INVALID_OFFSET; |
b1a8ef95 | 1535 | |
ca0b0069 AD |
1536 | if (bo->tbo.resource->mem_type == TTM_PL_TT) |
1537 | offset = amdgpu_gmc_agp_addr(&bo->tbo); | |
b1a8ef95 | 1538 | |
ca0b0069 AD |
1539 | if (offset == AMDGPU_BO_INVALID_OFFSET) |
1540 | offset = (bo->tbo.resource->start << PAGE_SHIFT) + | |
1541 | amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); | |
b1a8ef95 ND |
1542 | |
1543 | return amdgpu_gmc_sign_extend(offset); | |
cdb7e8f2 | 1544 | } |
84b74608 | 1545 | |
2472e11b | 1546 | /** |
d035f84d | 1547 | * amdgpu_bo_get_preferred_domain - get preferred domain |
2472e11b MD |
1548 | * @adev: amdgpu device object |
1549 | * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` | |
1550 | * | |
1551 | * Returns: | |
d035f84d | 1552 | * Which of the allowed domains is preferred for allocating the BO. |
2472e11b | 1553 | */ |
d035f84d | 1554 | uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, |
84b74608 DS |
1555 | uint32_t domain) |
1556 | { | |
81d0bcf9 AD |
1557 | if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && |
1558 | ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { | |
84b74608 DS |
1559 | domain = AMDGPU_GEM_DOMAIN_VRAM; |
1560 | if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) | |
1561 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
1562 | } | |
1563 | return domain; | |
1564 | } | |
ff72bc40 MBP |
1565 | |
1566 | #if defined(CONFIG_DEBUG_FS) | |
1567 | #define amdgpu_bo_print_flag(m, bo, flag) \ | |
1568 | do { \ | |
1569 | if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ | |
1570 | seq_printf((m), " " #flag); \ | |
1571 | } \ | |
1572 | } while (0) | |
1573 | ||
1574 | /** | |
25dd7a44 | 1575 | * amdgpu_bo_print_info - print BO info in debugfs file |
ff72bc40 MBP |
1576 | * |
1577 | * @id: Index or Id of the BO | |
1578 | * @bo: Requested BO for printing info | |
1579 | * @m: debugfs file | |
1580 | * | |
1581 | * Print BO information in debugfs file | |
1582 | * | |
1583 | * Returns: | |
1584 | * Size of the BO in bytes. | |
1585 | */ | |
1586 | u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) | |
1587 | { | |
394ae060 | 1588 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
ff72bc40 MBP |
1589 | struct dma_buf_attachment *attachment; |
1590 | struct dma_buf *dma_buf; | |
ff72bc40 MBP |
1591 | const char *placement; |
1592 | unsigned int pin_count; | |
1593 | u64 size; | |
1594 | ||
818c158f PEPP |
1595 | if (dma_resv_trylock(bo->tbo.base.resv)) { |
1596 | unsigned int domain; | |
394ae060 | 1597 | |
818c158f PEPP |
1598 | domain = amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type); |
1599 | switch (domain) { | |
1600 | case AMDGPU_GEM_DOMAIN_VRAM: | |
394ae060 | 1601 | if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) |
818c158f PEPP |
1602 | placement = "VRAM VISIBLE"; |
1603 | else | |
1604 | placement = "VRAM"; | |
1605 | break; | |
1606 | case AMDGPU_GEM_DOMAIN_GTT: | |
1607 | placement = "GTT"; | |
1608 | break; | |
1609 | case AMDGPU_GEM_DOMAIN_CPU: | |
1610 | default: | |
1611 | placement = "CPU"; | |
1612 | break; | |
1613 | } | |
1614 | dma_resv_unlock(bo->tbo.base.resv); | |
1615 | } else { | |
1616 | placement = "UNKNOWN"; | |
ff72bc40 MBP |
1617 | } |
1618 | ||
1619 | size = amdgpu_bo_size(bo); | |
1620 | seq_printf(m, "\t\t0x%08x: %12lld byte %s", | |
1621 | id, size, placement); | |
1622 | ||
5b8c5969 | 1623 | pin_count = READ_ONCE(bo->tbo.pin_count); |
ff72bc40 MBP |
1624 | if (pin_count) |
1625 | seq_printf(m, " pin count %d", pin_count); | |
1626 | ||
1627 | dma_buf = READ_ONCE(bo->tbo.base.dma_buf); | |
1628 | attachment = READ_ONCE(bo->tbo.base.import_attach); | |
1629 | ||
1630 | if (attachment) | |
26fd808b | 1631 | seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino); |
ff72bc40 | 1632 | else if (dma_buf) |
26fd808b | 1633 | seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino); |
ff72bc40 MBP |
1634 | |
1635 | amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); | |
1636 | amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); | |
1637 | amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); | |
1638 | amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); | |
ff72bc40 MBP |
1639 | amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); |
1640 | amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); | |
1641 | amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); | |
1642 | ||
1643 | seq_puts(m, "\n"); | |
1644 | ||
1645 | return size; | |
1646 | } | |
1647 | #endif |