Commit | Line | Data |
---|---|---|
e7c94bfb | 1 | // SPDX-License-Identifier: MIT |
a46a2cd1 FK |
2 | /* |
3 | * Copyright 2014-2018 Advanced Micro Devices, Inc. | |
4 | * | |
5 | * Permission is hereby granted, free of charge, to any person obtaining a | |
6 | * copy of this software and associated documentation files (the "Software"), | |
7 | * to deal in the Software without restriction, including without limitation | |
8 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
9 | * and/or sell copies of the Software, and to permit persons to whom the | |
10 | * Software is furnished to do so, subject to the following conditions: | |
11 | * | |
12 | * The above copyright notice and this permission notice shall be included in | |
13 | * all copies or substantial portions of the Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
19 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
20 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
21 | * OTHER DEALINGS IN THE SOFTWARE. | |
22 | */ | |
c366be54 | 23 | #include <linux/dma-buf.h> |
a46a2cd1 | 24 | #include <linux/list.h> |
548da31d | 25 | #include <linux/pagemap.h> |
5ae0283e | 26 | #include <linux/sched/mm.h> |
c366be54 | 27 | #include <linux/sched/task.h> |
18192001 | 28 | #include <linux/fdtable.h> |
a3185f91 | 29 | #include <drm/ttm/ttm_tt.h> |
c366be54 | 30 | |
8abc1eb2 CK |
31 | #include <drm/drm_exec.h> |
32 | ||
a46a2cd1 | 33 | #include "amdgpu_object.h" |
875440fd | 34 | #include "amdgpu_gem.h" |
a46a2cd1 | 35 | #include "amdgpu_vm.h" |
d9483ecd | 36 | #include "amdgpu_hmm.h" |
a46a2cd1 | 37 | #include "amdgpu_amdkfd.h" |
2fbd6f94 | 38 | #include "amdgpu_dma_buf.h" |
1d251d90 | 39 | #include <uapi/linux/kfd_ioctl.h> |
72b4db0f | 40 | #include "amdgpu_xgmi.h" |
8dc1db31 | 41 | #include "kfd_priv.h" |
c7f21978 | 42 | #include "kfd_smi_events.h" |
a46a2cd1 | 43 | |
5ae0283e FK |
44 | /* Userptr restore delay, just long enough to allow consecutive VM |
45 | * changes to accumulate | |
46 | */ | |
47 | #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 | |
9b37d45d | 48 | #define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29) |
5ae0283e | 49 | |
9731dd4c | 50 | /* |
aec208ee | 51 | * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB |
9731dd4c DP |
52 | * BO chunk |
53 | */ | |
aec208ee | 54 | #define VRAM_AVAILABLITY_ALIGN (1 << 21) |
9731dd4c | 55 | |
a46a2cd1 FK |
56 | /* Impose limit on how much memory KFD can use */ |
57 | static struct { | |
58 | uint64_t max_system_mem_limit; | |
5d240da9 | 59 | uint64_t max_ttm_mem_limit; |
a46a2cd1 | 60 | int64_t system_mem_used; |
5d240da9 | 61 | int64_t ttm_mem_used; |
a46a2cd1 FK |
62 | spinlock_t mem_limit_lock; |
63 | } kfd_mem_limit; | |
64 | ||
a46a2cd1 FK |
65 | static const char * const domain_bit_to_string[] = { |
66 | "CPU", | |
67 | "GTT", | |
68 | "VRAM", | |
69 | "GDS", | |
70 | "GWS", | |
71 | "OA" | |
72 | }; | |
73 | ||
74 | #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] | |
75 | ||
5ae0283e | 76 | static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); |
a46a2cd1 | 77 | |
c780b2ee | 78 | static bool kfd_mem_is_attached(struct amdgpu_vm *avm, |
a46a2cd1 FK |
79 | struct kgd_mem *mem) |
80 | { | |
c780b2ee | 81 | struct kfd_mem_attachment *entry; |
a46a2cd1 | 82 | |
c780b2ee | 83 | list_for_each_entry(entry, &mem->attachments, list) |
a46a2cd1 | 84 | if (entry->bo_va->base.vm == avm) |
c780b2ee | 85 | return true; |
a46a2cd1 | 86 | |
c780b2ee | 87 | return false; |
a46a2cd1 FK |
88 | } |
89 | ||
207bbfb6 SX |
90 | /** |
91 | * reuse_dmamap() - Check whether adev can share the original | |
92 | * userptr BO | |
93 | * | |
94 | * If both adev and bo_adev are in direct mapping or | |
95 | * in the same iommu group, they can share the original BO. | |
96 | * | |
97 | * @adev: Device to which can or cannot share the original BO | |
98 | * @bo_adev: Device to which allocated BO belongs to | |
99 | * | |
100 | * Return: returns true if adev can share original userptr BO, | |
101 | * false otherwise. | |
102 | */ | |
103 | static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev) | |
104 | { | |
105 | return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) || | |
106 | (adev->dev->iommu_group == bo_adev->dev->iommu_group); | |
107 | } | |
108 | ||
a46a2cd1 | 109 | /* Set memory usage limits. Current, limits are |
b72ff190 | 110 | * System (TTM + userptr) memory - 15/16th System RAM |
5d240da9 | 111 | * TTM memory - 3/8th System RAM |
a46a2cd1 FK |
112 | */ |
113 | void amdgpu_amdkfd_gpuvm_init_mem_limits(void) | |
114 | { | |
115 | struct sysinfo si; | |
116 | uint64_t mem; | |
117 | ||
27fb73a0 MJ |
118 | if (kfd_mem_limit.max_system_mem_limit) |
119 | return; | |
120 | ||
a46a2cd1 | 121 | si_meminfo(&si); |
9b37d45d | 122 | mem = si.totalram - si.totalhigh; |
a46a2cd1 FK |
123 | mem *= si.mem_unit; |
124 | ||
125 | spin_lock_init(&kfd_mem_limit.mem_limit_lock); | |
9b37d45d RB |
126 | kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6); |
127 | if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT) | |
128 | kfd_mem_limit.max_system_mem_limit >>= 1; | |
129 | else | |
130 | kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT; | |
131 | ||
27fb73a0 | 132 | kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT; |
5d240da9 | 133 | pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", |
5ae0283e | 134 | (kfd_mem_limit.max_system_mem_limit >> 20), |
5d240da9 | 135 | (kfd_mem_limit.max_ttm_mem_limit >> 20)); |
a46a2cd1 FK |
136 | } |
137 | ||
c46ebb6a PY |
138 | void amdgpu_amdkfd_reserve_system_mem(uint64_t size) |
139 | { | |
140 | kfd_mem_limit.system_mem_used += size; | |
141 | } | |
142 | ||
29a39c90 FK |
143 | /* Estimate page table size needed to represent a given memory size |
144 | * | |
145 | * With 4KB pages, we need one 8 byte PTE for each 4KB of memory | |
146 | * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB | |
147 | * of memory (factor 256K, >> 18). ROCm user mode tries to optimize | |
148 | * for 2MB pages for TLB efficiency. However, small allocations and | |
149 | * fragmented system memory still need some 4KB pages. We choose a | |
150 | * compromise that should work in most cases without reserving too | |
151 | * much memory for page tables unnecessarily (factor 16K, >> 14). | |
152 | */ | |
f07069da | 153 | |
86bd6706 | 154 | #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) |
f07069da | 155 | |
f441dd33 | 156 | /** |
77608faa | 157 | * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size |
86bd6706 | 158 | * of buffer. |
f441dd33 RE |
159 | * |
160 | * @adev: Device to which allocated BO belongs to | |
161 | * @size: Size of buffer, in bytes, encapsulated by B0. This should be | |
162 | * equivalent to amdgpu_bo_size(BO) | |
163 | * @alloc_flag: Flag used in allocating a BO as noted above | |
932fc494 SS |
164 | * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is |
165 | * managed as one compute node in driver for app | |
f441dd33 | 166 | * |
932fc494 SS |
167 | * Return: |
168 | * returns -ENOMEM in case of error, ZERO otherwise | |
f441dd33 | 169 | */ |
f9af3c16 | 170 | int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, |
1c77527a | 171 | uint64_t size, u32 alloc_flag, int8_t xcp_id) |
a46a2cd1 | 172 | { |
29a39c90 FK |
173 | uint64_t reserved_for_pt = |
174 | ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); | |
86bd6706 | 175 | size_t system_mem_needed, ttm_mem_needed, vram_needed; |
a46a2cd1 | 176 | int ret = 0; |
1c77527a | 177 | uint64_t vram_size = 0; |
a46a2cd1 | 178 | |
86bd6706 AS |
179 | system_mem_needed = 0; |
180 | ttm_mem_needed = 0; | |
611736d8 | 181 | vram_needed = 0; |
f441dd33 | 182 | if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { |
86bd6706 AS |
183 | system_mem_needed = size; |
184 | ttm_mem_needed = size; | |
f441dd33 | 185 | } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { |
9731dd4c DP |
186 | /* |
187 | * Conservatively round up the allocation requirement to 2 MB | |
188 | * to avoid fragmentation caused by 4K allocations in the tail | |
189 | * 2M BO chunk. | |
190 | */ | |
aec208ee | 191 | vram_needed = size; |
1c77527a MJ |
192 | /* |
193 | * For GFX 9.4.3, get the VRAM size from XCP structs | |
194 | */ | |
195 | if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) | |
196 | return -EINVAL; | |
197 | ||
198 | vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id); | |
199 | if (adev->gmc.is_app_apu) { | |
200 | system_mem_needed = size; | |
201 | ttm_mem_needed = size; | |
202 | } | |
f441dd33 | 203 | } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { |
86bd6706 AS |
204 | system_mem_needed = size; |
205 | } else if (!(alloc_flag & | |
206 | (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | | |
207 | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { | |
f441dd33 RE |
208 | pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); |
209 | return -ENOMEM; | |
5d240da9 EH |
210 | } |
211 | ||
611736d8 FK |
212 | spin_lock(&kfd_mem_limit.mem_limit_lock); |
213 | ||
b80f050f PY |
214 | if (kfd_mem_limit.system_mem_used + system_mem_needed > |
215 | kfd_mem_limit.max_system_mem_limit) | |
216 | pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); | |
217 | ||
5d240da9 | 218 | if ((kfd_mem_limit.system_mem_used + system_mem_needed > |
b80f050f | 219 | kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || |
611736d8 FK |
220 | (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > |
221 | kfd_mem_limit.max_ttm_mem_limit) || | |
1c77527a | 222 | (adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] + vram_needed > |
f06446ef | 223 | vram_size - reserved_for_pt - atomic64_read(&adev->vram_pin_size))) { |
5d240da9 | 224 | ret = -ENOMEM; |
f441dd33 | 225 | goto release; |
a46a2cd1 | 226 | } |
5d240da9 | 227 | |
f441dd33 RE |
228 | /* Update memory accounting by decreasing available system |
229 | * memory, TTM memory and GPU memory as computed above | |
230 | */ | |
f9af3c16 AS |
231 | WARN_ONCE(vram_needed && !adev, |
232 | "adev reference can't be null when vram is used"); | |
1c77527a MJ |
233 | if (adev && xcp_id >= 0) { |
234 | adev->kfd.vram_used[xcp_id] += vram_needed; | |
235 | adev->kfd.vram_used_aligned[xcp_id] += adev->gmc.is_app_apu ? | |
236 | vram_needed : | |
237 | ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); | |
1ac354be | 238 | } |
f441dd33 RE |
239 | kfd_mem_limit.system_mem_used += system_mem_needed; |
240 | kfd_mem_limit.ttm_mem_used += ttm_mem_needed; | |
241 | ||
242 | release: | |
a46a2cd1 FK |
243 | spin_unlock(&kfd_mem_limit.mem_limit_lock); |
244 | return ret; | |
245 | } | |
246 | ||
f9af3c16 | 247 | void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, |
1c77527a | 248 | uint64_t size, u32 alloc_flag, int8_t xcp_id) |
a46a2cd1 | 249 | { |
a46a2cd1 | 250 | spin_lock(&kfd_mem_limit.mem_limit_lock); |
f441dd33 RE |
251 | |
252 | if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { | |
86bd6706 AS |
253 | kfd_mem_limit.system_mem_used -= size; |
254 | kfd_mem_limit.ttm_mem_used -= size; | |
f441dd33 | 255 | } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { |
f9af3c16 AS |
256 | WARN_ONCE(!adev, |
257 | "adev reference can't be null when alloc mem flags vram is set"); | |
1c77527a MJ |
258 | if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) |
259 | goto release; | |
260 | ||
1ac354be | 261 | if (adev) { |
1c77527a MJ |
262 | adev->kfd.vram_used[xcp_id] -= size; |
263 | if (adev->gmc.is_app_apu) { | |
264 | adev->kfd.vram_used_aligned[xcp_id] -= size; | |
265 | kfd_mem_limit.system_mem_used -= size; | |
266 | kfd_mem_limit.ttm_mem_used -= size; | |
267 | } else { | |
268 | adev->kfd.vram_used_aligned[xcp_id] -= | |
269 | ALIGN(size, VRAM_AVAILABLITY_ALIGN); | |
270 | } | |
1ac354be | 271 | } |
f441dd33 | 272 | } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { |
86bd6706 AS |
273 | kfd_mem_limit.system_mem_used -= size; |
274 | } else if (!(alloc_flag & | |
275 | (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | | |
276 | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { | |
f441dd33 RE |
277 | pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); |
278 | goto release; | |
5ae0283e | 279 | } |
1c77527a MJ |
280 | WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0, |
281 | "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id); | |
5d240da9 | 282 | WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, |
f441dd33 RE |
283 | "KFD TTM memory accounting unbalanced"); |
284 | WARN_ONCE(kfd_mem_limit.system_mem_used < 0, | |
285 | "KFD system memory accounting unbalanced"); | |
a46a2cd1 | 286 | |
f441dd33 | 287 | release: |
a46a2cd1 FK |
288 | spin_unlock(&kfd_mem_limit.mem_limit_lock); |
289 | } | |
290 | ||
5702d052 | 291 | void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) |
a46a2cd1 | 292 | { |
611736d8 | 293 | struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); |
f441dd33 RE |
294 | u32 alloc_flags = bo->kfd_bo->alloc_flags; |
295 | u64 size = amdgpu_bo_size(bo); | |
a46a2cd1 | 296 | |
1c77527a MJ |
297 | amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags, |
298 | bo->xcp_id); | |
5702d052 FK |
299 | |
300 | kfree(bo->kfd_bo); | |
a46a2cd1 FK |
301 | } |
302 | ||
08a2fd23 | 303 | /** |
837d4e07 | 304 | * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information |
08a2fd23 | 305 | * about USERPTR or DOOREBELL or MMIO BO. |
837d4e07 | 306 | * |
08a2fd23 RE |
307 | * @adev: Device for which dmamap BO is being created |
308 | * @mem: BO of peer device that is being DMA mapped. Provides parameters | |
309 | * in building the dmamap BO | |
310 | * @bo_out: Output parameter updated with handle of dmamap BO | |
311 | */ | |
312 | static int | |
313 | create_dmamap_sg_bo(struct amdgpu_device *adev, | |
314 | struct kgd_mem *mem, struct amdgpu_bo **bo_out) | |
315 | { | |
316 | struct drm_gem_object *gem_obj; | |
af152c21 SX |
317 | int ret; |
318 | uint64_t flags = 0; | |
08a2fd23 RE |
319 | |
320 | ret = amdgpu_bo_reserve(mem->bo, false); | |
321 | if (ret) | |
322 | return ret; | |
323 | ||
af152c21 SX |
324 | if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) |
325 | flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT | | |
326 | AMDGPU_GEM_CREATE_UNCACHED); | |
327 | ||
328 | ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1, | |
329 | AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags, | |
f24e924b | 330 | ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0); |
08a2fd23 RE |
331 | |
332 | amdgpu_bo_unreserve(mem->bo); | |
333 | ||
334 | if (ret) { | |
335 | pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); | |
336 | return -EINVAL; | |
337 | } | |
338 | ||
339 | *bo_out = gem_to_amdgpu_bo(gem_obj); | |
340 | (*bo_out)->parent = amdgpu_bo_ref(mem->bo); | |
341 | return ret; | |
342 | } | |
343 | ||
2d086fde | 344 | /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's |
a46a2cd1 FK |
345 | * reservation object. |
346 | * | |
347 | * @bo: [IN] Remove eviction fence(s) from this BO | |
2d086fde | 348 | * @ef: [IN] This eviction fence is removed if it |
a46a2cd1 | 349 | * is present in the shared list. |
a46a2cd1 | 350 | * |
a46a2cd1 FK |
351 | * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. |
352 | */ | |
353 | static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, | |
2d086fde | 354 | struct amdgpu_amdkfd_fence *ef) |
a46a2cd1 | 355 | { |
548e7432 | 356 | struct dma_fence *replacement; |
a46a2cd1 | 357 | |
2d086fde | 358 | if (!ef) |
a46a2cd1 FK |
359 | return -EINVAL; |
360 | ||
548e7432 CK |
361 | /* TODO: Instead of block before we should use the fence of the page |
362 | * table update and TLB flush here directly. | |
a46a2cd1 | 363 | */ |
548e7432 CK |
364 | replacement = dma_fence_get_stub(); |
365 | dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, | |
42470840 | 366 | replacement, DMA_RESV_USAGE_BOOKKEEP); |
548e7432 | 367 | dma_fence_put(replacement); |
a46a2cd1 FK |
368 | return 0; |
369 | } | |
370 | ||
f4a3c42b | 371 | int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) |
372 | { | |
373 | struct amdgpu_bo *root = bo; | |
374 | struct amdgpu_vm_bo_base *vm_bo; | |
375 | struct amdgpu_vm *vm; | |
376 | struct amdkfd_process_info *info; | |
377 | struct amdgpu_amdkfd_fence *ef; | |
378 | int ret; | |
379 | ||
380 | /* we can always get vm_bo from root PD bo.*/ | |
381 | while (root->parent) | |
382 | root = root->parent; | |
383 | ||
384 | vm_bo = root->vm_bo; | |
385 | if (!vm_bo) | |
386 | return 0; | |
387 | ||
388 | vm = vm_bo->vm; | |
389 | if (!vm) | |
390 | return 0; | |
391 | ||
392 | info = vm->process_info; | |
393 | if (!info || !info->eviction_fence) | |
394 | return 0; | |
395 | ||
396 | ef = container_of(dma_fence_get(&info->eviction_fence->base), | |
397 | struct amdgpu_amdkfd_fence, base); | |
398 | ||
399 | BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); | |
400 | ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); | |
401 | dma_resv_unlock(bo->tbo.base.resv); | |
402 | ||
403 | dma_fence_put(&ef->base); | |
404 | return ret; | |
405 | } | |
406 | ||
a46a2cd1 FK |
407 | static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, |
408 | bool wait) | |
409 | { | |
410 | struct ttm_operation_ctx ctx = { false, false }; | |
411 | int ret; | |
412 | ||
413 | if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), | |
414 | "Called with userptr BO")) | |
415 | return -EINVAL; | |
416 | ||
c704ab18 | 417 | amdgpu_bo_placement_from_domain(bo, domain); |
a46a2cd1 FK |
418 | |
419 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
420 | if (ret) | |
421 | goto validate_fail; | |
2d086fde | 422 | if (wait) |
c60cd590 | 423 | amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); |
a46a2cd1 FK |
424 | |
425 | validate_fail: | |
426 | return ret; | |
427 | } | |
428 | ||
50661eb1 FK |
429 | int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, |
430 | uint32_t domain, | |
431 | struct dma_fence *fence) | |
0e2e7c5b FK |
432 | { |
433 | int ret = amdgpu_bo_reserve(bo, false); | |
434 | ||
435 | if (ret) | |
436 | return ret; | |
437 | ||
438 | ret = amdgpu_amdkfd_bo_validate(bo, domain, true); | |
439 | if (ret) | |
440 | goto unreserve_out; | |
441 | ||
442 | ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1); | |
443 | if (ret) | |
444 | goto unreserve_out; | |
445 | ||
446 | dma_resv_add_fence(bo->tbo.base.resv, fence, | |
447 | DMA_RESV_USAGE_BOOKKEEP); | |
448 | ||
449 | unreserve_out: | |
450 | amdgpu_bo_unreserve(bo); | |
451 | ||
452 | return ret; | |
453 | } | |
454 | ||
bc05716d | 455 | static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) |
a46a2cd1 | 456 | { |
bc05716d | 457 | return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); |
a46a2cd1 FK |
458 | } |
459 | ||
460 | /* vm_validate_pt_pd_bos - Validate page table and directory BOs | |
461 | * | |
462 | * Page directories are not updated here because huge page handling | |
463 | * during page table updates can invalidate page directory entries | |
464 | * again. Page directories are only updated after updating page | |
465 | * tables. | |
466 | */ | |
50661eb1 FK |
467 | static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm, |
468 | struct ww_acquire_ctx *ticket) | |
a46a2cd1 | 469 | { |
391629bd | 470 | struct amdgpu_bo *pd = vm->root.bo; |
a46a2cd1 | 471 | struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); |
a46a2cd1 FK |
472 | int ret; |
473 | ||
50661eb1 FK |
474 | ret = amdgpu_vm_validate(adev, vm, ticket, |
475 | amdgpu_amdkfd_validate_vm_bo, NULL); | |
a46a2cd1 | 476 | if (ret) { |
dd4fa6c1 | 477 | pr_err("failed to validate PT BOs\n"); |
a46a2cd1 FK |
478 | return ret; |
479 | } | |
480 | ||
391629bd | 481 | vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); |
a46a2cd1 | 482 | |
a46a2cd1 FK |
483 | return 0; |
484 | } | |
485 | ||
a46a2cd1 FK |
486 | static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) |
487 | { | |
391629bd | 488 | struct amdgpu_bo *pd = vm->root.bo; |
a46a2cd1 FK |
489 | struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); |
490 | int ret; | |
491 | ||
807e2994 | 492 | ret = amdgpu_vm_update_pdes(adev, vm, false); |
a46a2cd1 FK |
493 | if (ret) |
494 | return ret; | |
495 | ||
174b328b | 496 | return amdgpu_sync_fence(sync, vm->last_update); |
a46a2cd1 FK |
497 | } |
498 | ||
d0ba51b1 FK |
499 | static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) |
500 | { | |
d1a372af FK |
501 | uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE | |
502 | AMDGPU_VM_MTYPE_DEFAULT; | |
d0ba51b1 | 503 | |
1d251d90 | 504 | if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) |
d0ba51b1 | 505 | mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; |
1d251d90 | 506 | if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) |
d0ba51b1 FK |
507 | mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; |
508 | ||
d1a372af | 509 | return amdgpu_gem_va_map_flags(adev, mapping_flags); |
d0ba51b1 FK |
510 | } |
511 | ||
08a2fd23 RE |
512 | /** |
513 | * create_sg_table() - Create an sg_table for a contiguous DMA addr range | |
514 | * @addr: The starting address to point to | |
515 | * @size: Size of memory area in bytes being pointed to | |
516 | * | |
517 | * Allocates an instance of sg_table and initializes it to point to memory | |
518 | * area specified by input parameters. The address used to build is assumed | |
519 | * to be DMA mapped, if needed. | |
520 | * | |
521 | * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table | |
522 | * because they are physically contiguous. | |
523 | * | |
524 | * Return: Initialized instance of SG Table or NULL | |
525 | */ | |
526 | static struct sg_table *create_sg_table(uint64_t addr, uint32_t size) | |
527 | { | |
528 | struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL); | |
529 | ||
530 | if (!sg) | |
531 | return NULL; | |
532 | if (sg_alloc_table(sg, 1, GFP_KERNEL)) { | |
533 | kfree(sg); | |
534 | return NULL; | |
535 | } | |
536 | sg_dma_address(sg->sgl) = addr; | |
537 | sg->sgl->length = size; | |
538 | #ifdef CONFIG_NEED_SG_DMA_LENGTH | |
539 | sg->sgl->dma_length = size; | |
540 | #endif | |
541 | return sg; | |
542 | } | |
543 | ||
264fb4d3 FK |
544 | static int |
545 | kfd_mem_dmamap_userptr(struct kgd_mem *mem, | |
546 | struct kfd_mem_attachment *attachment) | |
547 | { | |
548 | enum dma_data_direction direction = | |
549 | mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? | |
550 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE; | |
551 | struct ttm_operation_ctx ctx = {.interruptible = true}; | |
552 | struct amdgpu_bo *bo = attachment->bo_va->base.bo; | |
553 | struct amdgpu_device *adev = attachment->adev; | |
554 | struct ttm_tt *src_ttm = mem->bo->tbo.ttm; | |
555 | struct ttm_tt *ttm = bo->tbo.ttm; | |
556 | int ret; | |
557 | ||
7b5a4d7b RM |
558 | if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) |
559 | return -EINVAL; | |
560 | ||
264fb4d3 FK |
561 | ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL); |
562 | if (unlikely(!ttm->sg)) | |
563 | return -ENOMEM; | |
564 | ||
264fb4d3 FK |
565 | /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ |
566 | ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, | |
567 | ttm->num_pages, 0, | |
568 | (u64)ttm->num_pages << PAGE_SHIFT, | |
569 | GFP_KERNEL); | |
570 | if (unlikely(ret)) | |
571 | goto free_sg; | |
572 | ||
573 | ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); | |
574 | if (unlikely(ret)) | |
575 | goto release_sg; | |
576 | ||
264fb4d3 FK |
577 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); |
578 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
579 | if (ret) | |
580 | goto unmap_sg; | |
581 | ||
582 | return 0; | |
583 | ||
584 | unmap_sg: | |
585 | dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); | |
586 | release_sg: | |
587 | pr_err("DMA map userptr failed: %d\n", ret); | |
588 | sg_free_table(ttm->sg); | |
589 | free_sg: | |
590 | kfree(ttm->sg); | |
591 | ttm->sg = NULL; | |
592 | return ret; | |
593 | } | |
594 | ||
5ac3c3e4 FK |
595 | static int |
596 | kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment) | |
597 | { | |
598 | struct ttm_operation_ctx ctx = {.interruptible = true}; | |
599 | struct amdgpu_bo *bo = attachment->bo_va->base.bo; | |
b9274387 FK |
600 | int ret; |
601 | ||
602 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
603 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
604 | if (ret) | |
605 | return ret; | |
5ac3c3e4 FK |
606 | |
607 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); | |
608 | return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
609 | } | |
610 | ||
08a2fd23 RE |
611 | /** |
612 | * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO | |
613 | * @mem: SG BO of the DOORBELL or MMIO resource on the owning device | |
614 | * @attachment: Virtual address attachment of the BO on accessing device | |
615 | * | |
616 | * An access request from the device that owns DOORBELL does not require DMA mapping. | |
617 | * This is because the request doesn't go through PCIe root complex i.e. it instead | |
618 | * loops back. The need to DMA map arises only when accessing peer device's DOORBELL | |
619 | * | |
620 | * In contrast, all access requests for MMIO need to be DMA mapped without regard to | |
621 | * device ownership. This is because access requests for MMIO go through PCIe root | |
622 | * complex. | |
623 | * | |
624 | * This is accomplished in two steps: | |
625 | * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used | |
626 | * in updating requesting device's page table | |
627 | * - Signal TTM to mark memory pointed to by requesting device's BO as GPU | |
628 | * accessible. This allows an update of requesting device's page table | |
629 | * with entries associated with DOOREBELL or MMIO memory | |
630 | * | |
631 | * This method is invoked in the following contexts: | |
632 | * - Mapping of DOORBELL or MMIO BO of same or peer device | |
633 | * - Validating an evicted DOOREBELL or MMIO BO on device seeking access | |
634 | * | |
635 | * Return: ZERO if successful, NON-ZERO otherwise | |
636 | */ | |
637 | static int | |
638 | kfd_mem_dmamap_sg_bo(struct kgd_mem *mem, | |
639 | struct kfd_mem_attachment *attachment) | |
640 | { | |
641 | struct ttm_operation_ctx ctx = {.interruptible = true}; | |
642 | struct amdgpu_bo *bo = attachment->bo_va->base.bo; | |
643 | struct amdgpu_device *adev = attachment->adev; | |
644 | struct ttm_tt *ttm = bo->tbo.ttm; | |
645 | enum dma_data_direction dir; | |
646 | dma_addr_t dma_addr; | |
647 | bool mmio; | |
648 | int ret; | |
649 | ||
650 | /* Expect SG Table of dmapmap BO to be NULL */ | |
651 | mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP); | |
652 | if (unlikely(ttm->sg)) { | |
653 | pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio); | |
654 | return -EINVAL; | |
655 | } | |
656 | ||
657 | dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? | |
658 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE; | |
659 | dma_addr = mem->bo->tbo.sg->sgl->dma_address; | |
660 | pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length); | |
661 | pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr); | |
662 | dma_addr = dma_map_resource(adev->dev, dma_addr, | |
663 | mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); | |
664 | ret = dma_mapping_error(adev->dev, dma_addr); | |
665 | if (unlikely(ret)) | |
666 | return ret; | |
667 | pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr); | |
668 | ||
669 | ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length); | |
670 | if (unlikely(!ttm->sg)) { | |
671 | ret = -ENOMEM; | |
672 | goto unmap_sg; | |
673 | } | |
674 | ||
675 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); | |
676 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
677 | if (unlikely(ret)) | |
678 | goto free_sg; | |
679 | ||
680 | return ret; | |
681 | ||
682 | free_sg: | |
683 | sg_free_table(ttm->sg); | |
684 | kfree(ttm->sg); | |
685 | ttm->sg = NULL; | |
686 | unmap_sg: | |
687 | dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length, | |
688 | dir, DMA_ATTR_SKIP_CPU_SYNC); | |
689 | return ret; | |
690 | } | |
691 | ||
264fb4d3 FK |
692 | static int |
693 | kfd_mem_dmamap_attachment(struct kgd_mem *mem, | |
694 | struct kfd_mem_attachment *attachment) | |
695 | { | |
696 | switch (attachment->type) { | |
697 | case KFD_MEM_ATT_SHARED: | |
698 | return 0; | |
699 | case KFD_MEM_ATT_USERPTR: | |
700 | return kfd_mem_dmamap_userptr(mem, attachment); | |
5ac3c3e4 FK |
701 | case KFD_MEM_ATT_DMABUF: |
702 | return kfd_mem_dmamap_dmabuf(attachment); | |
08a2fd23 RE |
703 | case KFD_MEM_ATT_SG: |
704 | return kfd_mem_dmamap_sg_bo(mem, attachment); | |
264fb4d3 FK |
705 | default: |
706 | WARN_ON_ONCE(1); | |
707 | } | |
708 | return -EINVAL; | |
709 | } | |
710 | ||
711 | static void | |
712 | kfd_mem_dmaunmap_userptr(struct kgd_mem *mem, | |
713 | struct kfd_mem_attachment *attachment) | |
714 | { | |
715 | enum dma_data_direction direction = | |
716 | mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? | |
717 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE; | |
718 | struct ttm_operation_ctx ctx = {.interruptible = false}; | |
719 | struct amdgpu_bo *bo = attachment->bo_va->base.bo; | |
720 | struct amdgpu_device *adev = attachment->adev; | |
721 | struct ttm_tt *ttm = bo->tbo.ttm; | |
722 | ||
723 | if (unlikely(!ttm->sg)) | |
724 | return; | |
725 | ||
726 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
727 | ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
728 | ||
729 | dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); | |
730 | sg_free_table(ttm->sg); | |
546dc20f | 731 | kfree(ttm->sg); |
264fb4d3 FK |
732 | ttm->sg = NULL; |
733 | } | |
734 | ||
5ac3c3e4 FK |
735 | static void |
736 | kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment) | |
737 | { | |
b9274387 FK |
738 | /* This is a no-op. We don't want to trigger eviction fences when |
739 | * unmapping DMABufs. Therefore the invalidation (moving to system | |
740 | * domain) is done in kfd_mem_dmamap_dmabuf. | |
741 | */ | |
5ac3c3e4 FK |
742 | } |
743 | ||
08a2fd23 RE |
744 | /** |
745 | * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO | |
746 | * @mem: SG BO of the DOORBELL or MMIO resource on the owning device | |
747 | * @attachment: Virtual address attachment of the BO on accessing device | |
748 | * | |
749 | * The method performs following steps: | |
750 | * - Signal TTM to mark memory pointed to by BO as GPU inaccessible | |
751 | * - Free SG Table that is used to encapsulate DMA mapped memory of | |
752 | * peer device's DOORBELL or MMIO memory | |
753 | * | |
754 | * This method is invoked in the following contexts: | |
755 | * UNMapping of DOORBELL or MMIO BO on a device having access to its memory | |
756 | * Eviction of DOOREBELL or MMIO BO on device having access to its memory | |
757 | * | |
758 | * Return: void | |
759 | */ | |
760 | static void | |
761 | kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem, | |
762 | struct kfd_mem_attachment *attachment) | |
763 | { | |
764 | struct ttm_operation_ctx ctx = {.interruptible = true}; | |
765 | struct amdgpu_bo *bo = attachment->bo_va->base.bo; | |
766 | struct amdgpu_device *adev = attachment->adev; | |
767 | struct ttm_tt *ttm = bo->tbo.ttm; | |
768 | enum dma_data_direction dir; | |
769 | ||
770 | if (unlikely(!ttm->sg)) { | |
101b8104 | 771 | pr_debug("SG Table of BO is NULL"); |
08a2fd23 RE |
772 | return; |
773 | } | |
774 | ||
775 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
776 | ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
777 | ||
778 | dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? | |
779 | DMA_BIDIRECTIONAL : DMA_TO_DEVICE; | |
780 | dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address, | |
781 | ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); | |
782 | sg_free_table(ttm->sg); | |
783 | kfree(ttm->sg); | |
784 | ttm->sg = NULL; | |
785 | bo->tbo.sg = NULL; | |
786 | } | |
787 | ||
264fb4d3 FK |
788 | static void |
789 | kfd_mem_dmaunmap_attachment(struct kgd_mem *mem, | |
790 | struct kfd_mem_attachment *attachment) | |
791 | { | |
792 | switch (attachment->type) { | |
793 | case KFD_MEM_ATT_SHARED: | |
794 | break; | |
795 | case KFD_MEM_ATT_USERPTR: | |
796 | kfd_mem_dmaunmap_userptr(mem, attachment); | |
797 | break; | |
5ac3c3e4 FK |
798 | case KFD_MEM_ATT_DMABUF: |
799 | kfd_mem_dmaunmap_dmabuf(attachment); | |
800 | break; | |
08a2fd23 RE |
801 | case KFD_MEM_ATT_SG: |
802 | kfd_mem_dmaunmap_sg_bo(mem, attachment); | |
803 | break; | |
264fb4d3 FK |
804 | default: |
805 | WARN_ON_ONCE(1); | |
806 | } | |
807 | } | |
808 | ||
fd234e75 FK |
809 | static int kfd_mem_export_dmabuf(struct kgd_mem *mem) |
810 | { | |
811 | if (!mem->dmabuf) { | |
18192001 FK |
812 | struct amdgpu_device *bo_adev; |
813 | struct dma_buf *dmabuf; | |
814 | int r, fd; | |
815 | ||
816 | bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); | |
817 | r = drm_gem_prime_handle_to_fd(&bo_adev->ddev, bo_adev->kfd.client.file, | |
818 | mem->gem_handle, | |
fd234e75 | 819 | mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? |
18192001 FK |
820 | DRM_RDWR : 0, &fd); |
821 | if (r) | |
822 | return r; | |
823 | dmabuf = dma_buf_get(fd); | |
824 | close_fd(fd); | |
825 | if (WARN_ON_ONCE(IS_ERR(dmabuf))) | |
826 | return PTR_ERR(dmabuf); | |
827 | mem->dmabuf = dmabuf; | |
fd234e75 FK |
828 | } |
829 | ||
830 | return 0; | |
831 | } | |
832 | ||
5ac3c3e4 FK |
833 | static int |
834 | kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem, | |
835 | struct amdgpu_bo **bo) | |
836 | { | |
837 | struct drm_gem_object *gobj; | |
71330557 | 838 | int ret; |
5ac3c3e4 | 839 | |
fd234e75 FK |
840 | ret = kfd_mem_export_dmabuf(mem); |
841 | if (ret) | |
842 | return ret; | |
5ac3c3e4 | 843 | |
a3fbb0d8 | 844 | gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf); |
5ac3c3e4 FK |
845 | if (IS_ERR(gobj)) |
846 | return PTR_ERR(gobj); | |
847 | ||
5ac3c3e4 | 848 | *bo = gem_to_amdgpu_bo(gobj); |
5bb19893 | 849 | (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE; |
5ac3c3e4 FK |
850 | |
851 | return 0; | |
852 | } | |
853 | ||
c780b2ee | 854 | /* kfd_mem_attach - Add a BO to a VM |
a46a2cd1 FK |
855 | * |
856 | * Everything that needs to bo done only once when a BO is first added | |
857 | * to a VM. It can later be mapped and unmapped many times without | |
858 | * repeating these steps. | |
859 | * | |
264fb4d3 | 860 | * 0. Create BO for DMA mapping, if needed |
a46a2cd1 FK |
861 | * 1. Allocate and initialize BO VA entry data structure |
862 | * 2. Add BO to the VM | |
863 | * 3. Determine ASIC-specific PTE flags | |
864 | * 4. Alloc page tables and directories if needed | |
865 | * 4a. Validate new page tables and directories | |
866 | */ | |
c780b2ee | 867 | static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem, |
7141394e | 868 | struct amdgpu_vm *vm, bool is_aql) |
a46a2cd1 | 869 | { |
264fb4d3 | 870 | struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); |
4e94272f | 871 | unsigned long bo_size = mem->bo->tbo.base.size; |
a46a2cd1 | 872 | uint64_t va = mem->va; |
7141394e FK |
873 | struct kfd_mem_attachment *attachment[2] = {NULL, NULL}; |
874 | struct amdgpu_bo *bo[2] = {NULL, NULL}; | |
ffa88b00 | 875 | struct amdgpu_bo_va *bo_va; |
08a2fd23 | 876 | bool same_hive = false; |
7141394e | 877 | int i, ret; |
a46a2cd1 FK |
878 | |
879 | if (!va) { | |
880 | pr_err("Invalid VA when adding BO to VM\n"); | |
881 | return -EINVAL; | |
882 | } | |
883 | ||
08a2fd23 RE |
884 | /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices |
885 | * | |
886 | * The access path of MMIO and DOORBELL BOs of is always over PCIe. | |
887 | * In contrast the access path of VRAM BOs depens upon the type of | |
888 | * link that connects the peer device. Access over PCIe is allowed | |
889 | * if peer device has large BAR. In contrast, access over xGMI is | |
890 | * allowed for both small and large BAR configurations of peer device | |
891 | */ | |
e181be58 | 892 | if ((adev != bo_adev && !adev->gmc.is_app_apu) && |
08a2fd23 RE |
893 | ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) || |
894 | (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) || | |
895 | (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { | |
896 | if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM) | |
897 | same_hive = amdgpu_xgmi_same_hive(adev, bo_adev); | |
898 | if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev)) | |
899 | return -EINVAL; | |
900 | } | |
901 | ||
7141394e FK |
902 | for (i = 0; i <= is_aql; i++) { |
903 | attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL); | |
904 | if (unlikely(!attachment[i])) { | |
905 | ret = -ENOMEM; | |
906 | goto unwind; | |
907 | } | |
a46a2cd1 | 908 | |
7141394e FK |
909 | pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, |
910 | va + bo_size, vm); | |
a46a2cd1 | 911 | |
08a2fd23 | 912 | if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) || |
207bbfb6 | 913 | (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) || |
5d44a766 PY |
914 | (mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) || |
915 | same_hive) { | |
28fe4164 | 916 | /* Mappings on the local GPU, or VRAM mappings in the |
5d44a766 | 917 | * local hive, or userptr, or GTT mapping can reuse dma map |
207bbfb6 | 918 | * address space share the original BO |
264fb4d3 FK |
919 | */ |
920 | attachment[i]->type = KFD_MEM_ATT_SHARED; | |
921 | bo[i] = mem->bo; | |
922 | drm_gem_object_get(&bo[i]->tbo.base); | |
923 | } else if (i > 0) { | |
924 | /* Multiple mappings on the same GPU share the BO */ | |
925 | attachment[i]->type = KFD_MEM_ATT_SHARED; | |
926 | bo[i] = bo[0]; | |
927 | drm_gem_object_get(&bo[i]->tbo.base); | |
928 | } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { | |
929 | /* Create an SG BO to DMA-map userptrs on other GPUs */ | |
930 | attachment[i]->type = KFD_MEM_ATT_USERPTR; | |
08a2fd23 | 931 | ret = create_dmamap_sg_bo(adev, mem, &bo[i]); |
264fb4d3 FK |
932 | if (ret) |
933 | goto unwind; | |
08a2fd23 RE |
934 | /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */ |
935 | } else if (mem->bo->tbo.type == ttm_bo_type_sg) { | |
936 | WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL || | |
937 | mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP), | |
938 | "Handing invalid SG BO in ATTACH request"); | |
939 | attachment[i]->type = KFD_MEM_ATT_SG; | |
940 | ret = create_dmamap_sg_bo(adev, mem, &bo[i]); | |
941 | if (ret) | |
942 | goto unwind; | |
943 | /* Enable acces to GTT and VRAM BOs of peer devices */ | |
944 | } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT || | |
945 | mem->domain == AMDGPU_GEM_DOMAIN_VRAM) { | |
5ac3c3e4 FK |
946 | attachment[i]->type = KFD_MEM_ATT_DMABUF; |
947 | ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]); | |
948 | if (ret) | |
949 | goto unwind; | |
5027605a | 950 | pr_debug("Employ DMABUF mechanism to enable peer GPU access\n"); |
264fb4d3 | 951 | } else { |
08a2fd23 RE |
952 | WARN_ONCE(true, "Handling invalid ATTACH request"); |
953 | ret = -EINVAL; | |
954 | goto unwind; | |
264fb4d3 | 955 | } |
a46a2cd1 | 956 | |
7141394e | 957 | /* Add BO to VM internal data structures */ |
7ef6b7f8 KR |
958 | ret = amdgpu_bo_reserve(bo[i], false); |
959 | if (ret) { | |
960 | pr_debug("Unable to reserve BO during memory attach"); | |
961 | goto unwind; | |
962 | } | |
ffa88b00 XC |
963 | bo_va = amdgpu_vm_bo_find(vm, bo[i]); |
964 | if (!bo_va) | |
965 | bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]); | |
966 | else | |
967 | ++bo_va->ref_count; | |
968 | attachment[i]->bo_va = bo_va; | |
7ef6b7f8 | 969 | amdgpu_bo_unreserve(bo[i]); |
7141394e FK |
970 | if (unlikely(!attachment[i]->bo_va)) { |
971 | ret = -ENOMEM; | |
972 | pr_err("Failed to add BO object to VM. ret == %d\n", | |
973 | ret); | |
974 | goto unwind; | |
975 | } | |
7141394e FK |
976 | attachment[i]->va = va; |
977 | attachment[i]->pte_flags = get_pte_flags(adev, mem); | |
978 | attachment[i]->adev = adev; | |
979 | list_add(&attachment[i]->list, &mem->attachments); | |
a46a2cd1 | 980 | |
7141394e FK |
981 | va += bo_size; |
982 | } | |
a46a2cd1 | 983 | |
a46a2cd1 FK |
984 | return 0; |
985 | ||
7141394e FK |
986 | unwind: |
987 | for (; i >= 0; i--) { | |
988 | if (!attachment[i]) | |
989 | continue; | |
990 | if (attachment[i]->bo_va) { | |
7ef6b7f8 | 991 | amdgpu_bo_reserve(bo[i], true); |
ffa88b00 XC |
992 | if (--attachment[i]->bo_va->ref_count == 0) |
993 | amdgpu_vm_bo_del(adev, attachment[i]->bo_va); | |
7ef6b7f8 | 994 | amdgpu_bo_unreserve(bo[i]); |
7141394e FK |
995 | list_del(&attachment[i]->list); |
996 | } | |
997 | if (bo[i]) | |
998 | drm_gem_object_put(&bo[i]->tbo.base); | |
999 | kfree(attachment[i]); | |
1000 | } | |
a46a2cd1 FK |
1001 | return ret; |
1002 | } | |
1003 | ||
c780b2ee | 1004 | static void kfd_mem_detach(struct kfd_mem_attachment *attachment) |
a46a2cd1 | 1005 | { |
4e94272f FK |
1006 | struct amdgpu_bo *bo = attachment->bo_va->base.bo; |
1007 | ||
c780b2ee FK |
1008 | pr_debug("\t remove VA 0x%llx in entry %p\n", |
1009 | attachment->va, attachment); | |
ffa88b00 XC |
1010 | if (--attachment->bo_va->ref_count == 0) |
1011 | amdgpu_vm_bo_del(attachment->adev, attachment->bo_va); | |
4e94272f | 1012 | drm_gem_object_put(&bo->tbo.base); |
c780b2ee FK |
1013 | list_del(&attachment->list); |
1014 | kfree(attachment); | |
a46a2cd1 FK |
1015 | } |
1016 | ||
1017 | static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, | |
5ae0283e FK |
1018 | struct amdkfd_process_info *process_info, |
1019 | bool userptr) | |
a46a2cd1 | 1020 | { |
a46a2cd1 | 1021 | mutex_lock(&process_info->lock); |
5ae0283e | 1022 | if (userptr) |
8abc1eb2 CK |
1023 | list_add_tail(&mem->validate_list, |
1024 | &process_info->userptr_valid_list); | |
5ae0283e | 1025 | else |
8abc1eb2 | 1026 | list_add_tail(&mem->validate_list, &process_info->kfd_bo_list); |
a46a2cd1 FK |
1027 | mutex_unlock(&process_info->lock); |
1028 | } | |
1029 | ||
71efab6a OZ |
1030 | static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem, |
1031 | struct amdkfd_process_info *process_info) | |
1032 | { | |
71efab6a | 1033 | mutex_lock(&process_info->lock); |
8abc1eb2 | 1034 | list_del(&mem->validate_list); |
71efab6a OZ |
1035 | mutex_unlock(&process_info->lock); |
1036 | } | |
1037 | ||
5ae0283e FK |
1038 | /* Initializes user pages. It registers the MMU notifier and validates |
1039 | * the userptr BO in the GTT domain. | |
1040 | * | |
1041 | * The BO must already be on the userptr_valid_list. Otherwise an | |
1042 | * eviction and restore may happen that leaves the new BO unmapped | |
1043 | * with the user mode queues running. | |
1044 | * | |
1045 | * Takes the process_info->lock to protect against concurrent restore | |
1046 | * workers. | |
1047 | * | |
1048 | * Returns 0 for success, negative errno for errors. | |
1049 | */ | |
011bbb03 RB |
1050 | static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr, |
1051 | bool criu_resume) | |
5ae0283e FK |
1052 | { |
1053 | struct amdkfd_process_info *process_info = mem->process_info; | |
1054 | struct amdgpu_bo *bo = mem->bo; | |
1055 | struct ttm_operation_ctx ctx = { true, false }; | |
fec8fdb5 | 1056 | struct hmm_range *range; |
5ae0283e FK |
1057 | int ret = 0; |
1058 | ||
1059 | mutex_lock(&process_info->lock); | |
1060 | ||
77f47d23 | 1061 | ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0); |
5ae0283e FK |
1062 | if (ret) { |
1063 | pr_err("%s: Failed to set userptr: %d\n", __func__, ret); | |
1064 | goto out; | |
1065 | } | |
1066 | ||
d9483ecd | 1067 | ret = amdgpu_hmm_register(bo, user_addr); |
5ae0283e FK |
1068 | if (ret) { |
1069 | pr_err("%s: Failed to register MMU notifier: %d\n", | |
1070 | __func__, ret); | |
1071 | goto out; | |
1072 | } | |
1073 | ||
011bbb03 RB |
1074 | if (criu_resume) { |
1075 | /* | |
1076 | * During a CRIU restore operation, the userptr buffer objects | |
1077 | * will be validated in the restore_userptr_work worker at a | |
1078 | * later stage when it is scheduled by another ioctl called by | |
1079 | * CRIU master process for the target pid for restore. | |
1080 | */ | |
f95f51a4 FK |
1081 | mutex_lock(&process_info->notifier_lock); |
1082 | mem->invalid++; | |
1083 | mutex_unlock(&process_info->notifier_lock); | |
011bbb03 RB |
1084 | mutex_unlock(&process_info->lock); |
1085 | return 0; | |
1086 | } | |
1087 | ||
fec8fdb5 | 1088 | ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range); |
5ae0283e FK |
1089 | if (ret) { |
1090 | pr_err("%s: Failed to get user pages: %d\n", __func__, ret); | |
899fbde1 | 1091 | goto unregister_out; |
5ae0283e FK |
1092 | } |
1093 | ||
5ae0283e FK |
1094 | ret = amdgpu_bo_reserve(bo, true); |
1095 | if (ret) { | |
1096 | pr_err("%s: Failed to reserve BO\n", __func__); | |
1097 | goto release_out; | |
1098 | } | |
c704ab18 | 1099 | amdgpu_bo_placement_from_domain(bo, mem->domain); |
5ae0283e FK |
1100 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
1101 | if (ret) | |
1102 | pr_err("%s: failed to validate BO\n", __func__); | |
1103 | amdgpu_bo_unreserve(bo); | |
1104 | ||
1105 | release_out: | |
fec8fdb5 | 1106 | amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range); |
5ae0283e FK |
1107 | unregister_out: |
1108 | if (ret) | |
d9483ecd | 1109 | amdgpu_hmm_unregister(bo); |
5ae0283e FK |
1110 | out: |
1111 | mutex_unlock(&process_info->lock); | |
1112 | return ret; | |
1113 | } | |
1114 | ||
a46a2cd1 FK |
1115 | /* Reserving a BO and its page table BOs must happen atomically to |
1116 | * avoid deadlocks. Some operations update multiple VMs at once. Track | |
1117 | * all the reservation info in a context structure. Optionally a sync | |
1118 | * object can track VM updates. | |
1119 | */ | |
1120 | struct bo_vm_reservation_context { | |
8abc1eb2 CK |
1121 | /* DRM execution context for the reservation */ |
1122 | struct drm_exec exec; | |
1123 | /* Number of VMs reserved */ | |
1124 | unsigned int n_vms; | |
1125 | /* Pointer to sync object */ | |
1126 | struct amdgpu_sync *sync; | |
a46a2cd1 FK |
1127 | }; |
1128 | ||
1129 | enum bo_vm_match { | |
1130 | BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */ | |
1131 | BO_VM_MAPPED, /* Match VMs where a BO is mapped */ | |
1132 | BO_VM_ALL, /* Match all VMs a BO was added to */ | |
1133 | }; | |
1134 | ||
1135 | /** | |
1136 | * reserve_bo_and_vm - reserve a BO and a VM unconditionally. | |
1137 | * @mem: KFD BO structure. | |
1138 | * @vm: the VM to reserve. | |
1139 | * @ctx: the struct that will be used in unreserve_bo_and_vms(). | |
1140 | */ | |
1141 | static int reserve_bo_and_vm(struct kgd_mem *mem, | |
1142 | struct amdgpu_vm *vm, | |
1143 | struct bo_vm_reservation_context *ctx) | |
1144 | { | |
1145 | struct amdgpu_bo *bo = mem->bo; | |
1146 | int ret; | |
1147 | ||
1148 | WARN_ON(!vm); | |
1149 | ||
a46a2cd1 FK |
1150 | ctx->n_vms = 1; |
1151 | ctx->sync = &mem->sync; | |
05d24935 | 1152 | drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); |
8abc1eb2 CK |
1153 | drm_exec_until_all_locked(&ctx->exec) { |
1154 | ret = amdgpu_vm_lock_pd(vm, &ctx->exec, 2); | |
1155 | drm_exec_retry_on_contention(&ctx->exec); | |
1156 | if (unlikely(ret)) | |
1157 | goto error; | |
1158 | ||
4984fc57 | 1159 | ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); |
8abc1eb2 CK |
1160 | drm_exec_retry_on_contention(&ctx->exec); |
1161 | if (unlikely(ret)) | |
1162 | goto error; | |
a46a2cd1 | 1163 | } |
10f39758 | 1164 | return 0; |
8abc1eb2 CK |
1165 | |
1166 | error: | |
1167 | pr_err("Failed to reserve buffers in ttm.\n"); | |
1168 | drm_exec_fini(&ctx->exec); | |
1169 | return ret; | |
a46a2cd1 FK |
1170 | } |
1171 | ||
1172 | /** | |
1173 | * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally | |
1174 | * @mem: KFD BO structure. | |
1175 | * @vm: the VM to reserve. If NULL, then all VMs associated with the BO | |
1176 | * is used. Otherwise, a single VM associated with the BO. | |
1177 | * @map_type: the mapping status that will be used to filter the VMs. | |
1178 | * @ctx: the struct that will be used in unreserve_bo_and_vms(). | |
1179 | * | |
1180 | * Returns 0 for success, negative for failure. | |
1181 | */ | |
1182 | static int reserve_bo_and_cond_vms(struct kgd_mem *mem, | |
1183 | struct amdgpu_vm *vm, enum bo_vm_match map_type, | |
1184 | struct bo_vm_reservation_context *ctx) | |
1185 | { | |
c780b2ee | 1186 | struct kfd_mem_attachment *entry; |
8abc1eb2 | 1187 | struct amdgpu_bo *bo = mem->bo; |
a46a2cd1 FK |
1188 | int ret; |
1189 | ||
a46a2cd1 | 1190 | ctx->sync = &mem->sync; |
05d24935 | 1191 | drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); |
8abc1eb2 CK |
1192 | drm_exec_until_all_locked(&ctx->exec) { |
1193 | ctx->n_vms = 0; | |
1194 | list_for_each_entry(entry, &mem->attachments, list) { | |
1195 | if ((vm && vm != entry->bo_va->base.vm) || | |
1196 | (entry->is_mapped != map_type | |
1197 | && map_type != BO_VM_ALL)) | |
1198 | continue; | |
a46a2cd1 | 1199 | |
8abc1eb2 CK |
1200 | ret = amdgpu_vm_lock_pd(entry->bo_va->base.vm, |
1201 | &ctx->exec, 2); | |
1202 | drm_exec_retry_on_contention(&ctx->exec); | |
1203 | if (unlikely(ret)) | |
1204 | goto error; | |
1205 | ++ctx->n_vms; | |
1206 | } | |
a46a2cd1 | 1207 | |
8abc1eb2 CK |
1208 | ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); |
1209 | drm_exec_retry_on_contention(&ctx->exec); | |
1210 | if (unlikely(ret)) | |
1211 | goto error; | |
a46a2cd1 | 1212 | } |
10f39758 | 1213 | return 0; |
8abc1eb2 CK |
1214 | |
1215 | error: | |
1216 | pr_err("Failed to reserve buffers in ttm.\n"); | |
1217 | drm_exec_fini(&ctx->exec); | |
1218 | return ret; | |
a46a2cd1 FK |
1219 | } |
1220 | ||
1221 | /** | |
1222 | * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context | |
1223 | * @ctx: Reservation context to unreserve | |
1224 | * @wait: Optionally wait for a sync object representing pending VM updates | |
1225 | * @intr: Whether the wait is interruptible | |
1226 | * | |
1227 | * Also frees any resources allocated in | |
1228 | * reserve_bo_and_(cond_)vm(s). Returns the status from | |
1229 | * amdgpu_sync_wait. | |
1230 | */ | |
1231 | static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx, | |
1232 | bool wait, bool intr) | |
1233 | { | |
1234 | int ret = 0; | |
1235 | ||
1236 | if (wait) | |
1237 | ret = amdgpu_sync_wait(ctx->sync, intr); | |
1238 | ||
8abc1eb2 | 1239 | drm_exec_fini(&ctx->exec); |
a46a2cd1 | 1240 | ctx->sync = NULL; |
a46a2cd1 FK |
1241 | return ret; |
1242 | } | |
1243 | ||
b72ed8a2 | 1244 | static void unmap_bo_from_gpuvm(struct kgd_mem *mem, |
c780b2ee | 1245 | struct kfd_mem_attachment *entry, |
a46a2cd1 FK |
1246 | struct amdgpu_sync *sync) |
1247 | { | |
1248 | struct amdgpu_bo_va *bo_va = entry->bo_va; | |
b72ed8a2 | 1249 | struct amdgpu_device *adev = entry->adev; |
a46a2cd1 | 1250 | struct amdgpu_vm *vm = bo_va->base.vm; |
a46a2cd1 | 1251 | |
a46a2cd1 FK |
1252 | amdgpu_vm_bo_unmap(adev, bo_va, entry->va); |
1253 | ||
1254 | amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); | |
1255 | ||
174b328b | 1256 | amdgpu_sync_fence(sync, bo_va->last_pt_update); |
a46a2cd1 FK |
1257 | } |
1258 | ||
b72ed8a2 FK |
1259 | static int update_gpuvm_pte(struct kgd_mem *mem, |
1260 | struct kfd_mem_attachment *entry, | |
4d30a83c | 1261 | struct amdgpu_sync *sync) |
a46a2cd1 | 1262 | { |
1e608013 | 1263 | struct amdgpu_bo_va *bo_va = entry->bo_va; |
b72ed8a2 FK |
1264 | struct amdgpu_device *adev = entry->adev; |
1265 | int ret; | |
1266 | ||
1267 | ret = kfd_mem_dmamap_attachment(mem, entry); | |
1268 | if (ret) | |
1269 | return ret; | |
a46a2cd1 FK |
1270 | |
1271 | /* Update the page tables */ | |
8f8cc3fb | 1272 | ret = amdgpu_vm_bo_update(adev, bo_va, false); |
a46a2cd1 FK |
1273 | if (ret) { |
1274 | pr_err("amdgpu_vm_bo_update failed\n"); | |
1275 | return ret; | |
1276 | } | |
1277 | ||
174b328b | 1278 | return amdgpu_sync_fence(sync, bo_va->last_pt_update); |
a46a2cd1 FK |
1279 | } |
1280 | ||
b72ed8a2 FK |
1281 | static int map_bo_to_gpuvm(struct kgd_mem *mem, |
1282 | struct kfd_mem_attachment *entry, | |
1283 | struct amdgpu_sync *sync, | |
4d30a83c | 1284 | bool no_update_pte) |
a46a2cd1 FK |
1285 | { |
1286 | int ret; | |
1287 | ||
1288 | /* Set virtual address for the allocation */ | |
b72ed8a2 | 1289 | ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0, |
a46a2cd1 FK |
1290 | amdgpu_bo_size(entry->bo_va->base.bo), |
1291 | entry->pte_flags); | |
1292 | if (ret) { | |
1293 | pr_err("Failed to map VA 0x%llx in vm. ret %d\n", | |
1294 | entry->va, ret); | |
1295 | return ret; | |
1296 | } | |
1297 | ||
5ae0283e FK |
1298 | if (no_update_pte) |
1299 | return 0; | |
1300 | ||
4d30a83c | 1301 | ret = update_gpuvm_pte(mem, entry, sync); |
a46a2cd1 FK |
1302 | if (ret) { |
1303 | pr_err("update_gpuvm_pte() failed\n"); | |
1304 | goto update_gpuvm_pte_failed; | |
1305 | } | |
1306 | ||
1307 | return 0; | |
1308 | ||
1309 | update_gpuvm_pte_failed: | |
b72ed8a2 | 1310 | unmap_bo_from_gpuvm(mem, entry, sync); |
101b8104 | 1311 | kfd_mem_dmaunmap_attachment(mem, entry); |
a46a2cd1 FK |
1312 | return ret; |
1313 | } | |
1314 | ||
50661eb1 FK |
1315 | static int process_validate_vms(struct amdkfd_process_info *process_info, |
1316 | struct ww_acquire_ctx *ticket) | |
a46a2cd1 | 1317 | { |
5b21d3e5 | 1318 | struct amdgpu_vm *peer_vm; |
a46a2cd1 FK |
1319 | int ret; |
1320 | ||
1321 | list_for_each_entry(peer_vm, &process_info->vm_list_head, | |
1322 | vm_list_node) { | |
50661eb1 | 1323 | ret = vm_validate_pt_pd_bos(peer_vm, ticket); |
a46a2cd1 FK |
1324 | if (ret) |
1325 | return ret; | |
1326 | } | |
1327 | ||
1328 | return 0; | |
1329 | } | |
1330 | ||
9130cc01 HK |
1331 | static int process_sync_pds_resv(struct amdkfd_process_info *process_info, |
1332 | struct amdgpu_sync *sync) | |
1333 | { | |
1334 | struct amdgpu_vm *peer_vm; | |
1335 | int ret; | |
1336 | ||
1337 | list_for_each_entry(peer_vm, &process_info->vm_list_head, | |
1338 | vm_list_node) { | |
391629bd | 1339 | struct amdgpu_bo *pd = peer_vm->root.bo; |
9130cc01 | 1340 | |
5d319660 CK |
1341 | ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv, |
1342 | AMDGPU_SYNC_NE_OWNER, | |
1343 | AMDGPU_FENCE_OWNER_KFD); | |
9130cc01 HK |
1344 | if (ret) |
1345 | return ret; | |
1346 | } | |
1347 | ||
1348 | return 0; | |
1349 | } | |
1350 | ||
a46a2cd1 FK |
1351 | static int process_update_pds(struct amdkfd_process_info *process_info, |
1352 | struct amdgpu_sync *sync) | |
1353 | { | |
5b21d3e5 | 1354 | struct amdgpu_vm *peer_vm; |
a46a2cd1 FK |
1355 | int ret; |
1356 | ||
1357 | list_for_each_entry(peer_vm, &process_info->vm_list_head, | |
1358 | vm_list_node) { | |
5b21d3e5 | 1359 | ret = vm_update_pds(peer_vm, sync); |
a46a2cd1 FK |
1360 | if (ret) |
1361 | return ret; | |
1362 | } | |
1363 | ||
1364 | return 0; | |
1365 | } | |
1366 | ||
ede0dd86 FK |
1367 | static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, |
1368 | struct dma_fence **ef) | |
a46a2cd1 | 1369 | { |
3486625b | 1370 | struct amdkfd_process_info *info = NULL; |
ede0dd86 | 1371 | int ret; |
a46a2cd1 FK |
1372 | |
1373 | if (!*process_info) { | |
1374 | info = kzalloc(sizeof(*info), GFP_KERNEL); | |
ede0dd86 FK |
1375 | if (!info) |
1376 | return -ENOMEM; | |
a46a2cd1 FK |
1377 | |
1378 | mutex_init(&info->lock); | |
f95f51a4 | 1379 | mutex_init(&info->notifier_lock); |
a46a2cd1 FK |
1380 | INIT_LIST_HEAD(&info->vm_list_head); |
1381 | INIT_LIST_HEAD(&info->kfd_bo_list); | |
5ae0283e FK |
1382 | INIT_LIST_HEAD(&info->userptr_valid_list); |
1383 | INIT_LIST_HEAD(&info->userptr_inval_list); | |
a46a2cd1 FK |
1384 | |
1385 | info->eviction_fence = | |
1386 | amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), | |
eb2cec55 AS |
1387 | current->mm, |
1388 | NULL); | |
a46a2cd1 FK |
1389 | if (!info->eviction_fence) { |
1390 | pr_err("Failed to create eviction fence\n"); | |
ede0dd86 | 1391 | ret = -ENOMEM; |
a46a2cd1 FK |
1392 | goto create_evict_fence_fail; |
1393 | } | |
1394 | ||
5ae0283e | 1395 | info->pid = get_task_pid(current->group_leader, PIDTYPE_PID); |
5ae0283e FK |
1396 | INIT_DELAYED_WORK(&info->restore_userptr_work, |
1397 | amdgpu_amdkfd_restore_userptr_worker); | |
1398 | ||
a46a2cd1 | 1399 | *process_info = info; |
a46a2cd1 FK |
1400 | } |
1401 | ||
ede0dd86 | 1402 | vm->process_info = *process_info; |
a46a2cd1 | 1403 | |
3486625b | 1404 | /* Validate page directory and attach eviction fence */ |
391629bd | 1405 | ret = amdgpu_bo_reserve(vm->root.bo, true); |
3486625b FK |
1406 | if (ret) |
1407 | goto reserve_pd_fail; | |
50661eb1 | 1408 | ret = vm_validate_pt_pd_bos(vm, NULL); |
3486625b FK |
1409 | if (ret) { |
1410 | pr_err("validate_pt_pd_bos() failed\n"); | |
1411 | goto validate_pd_fail; | |
1412 | } | |
391629bd | 1413 | ret = amdgpu_bo_sync_wait(vm->root.bo, |
d38ca8f0 | 1414 | AMDGPU_FENCE_OWNER_KFD, false); |
3486625b FK |
1415 | if (ret) |
1416 | goto wait_pd_fail; | |
c8d4c18b | 1417 | ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); |
dd68722c FK |
1418 | if (ret) |
1419 | goto reserve_shared_fail; | |
42470840 CK |
1420 | dma_resv_add_fence(vm->root.bo->tbo.base.resv, |
1421 | &vm->process_info->eviction_fence->base, | |
1422 | DMA_RESV_USAGE_BOOKKEEP); | |
391629bd | 1423 | amdgpu_bo_unreserve(vm->root.bo); |
3486625b FK |
1424 | |
1425 | /* Update process info */ | |
ede0dd86 FK |
1426 | mutex_lock(&vm->process_info->lock); |
1427 | list_add_tail(&vm->vm_list_node, | |
1428 | &(vm->process_info->vm_list_head)); | |
1429 | vm->process_info->n_vms++; | |
9a1c1339 FK |
1430 | |
1431 | *ef = dma_fence_get(&vm->process_info->eviction_fence->base); | |
ede0dd86 | 1432 | mutex_unlock(&vm->process_info->lock); |
a46a2cd1 | 1433 | |
ede0dd86 | 1434 | return 0; |
a46a2cd1 | 1435 | |
dd68722c | 1436 | reserve_shared_fail: |
3486625b FK |
1437 | wait_pd_fail: |
1438 | validate_pd_fail: | |
391629bd | 1439 | amdgpu_bo_unreserve(vm->root.bo); |
3486625b | 1440 | reserve_pd_fail: |
ede0dd86 FK |
1441 | vm->process_info = NULL; |
1442 | if (info) { | |
ede0dd86 | 1443 | dma_fence_put(&info->eviction_fence->base); |
ede0dd86 | 1444 | *process_info = NULL; |
5ae0283e | 1445 | put_pid(info->pid); |
a46a2cd1 | 1446 | create_evict_fence_fail: |
ede0dd86 | 1447 | mutex_destroy(&info->lock); |
f95f51a4 | 1448 | mutex_destroy(&info->notifier_lock); |
ede0dd86 FK |
1449 | kfree(info); |
1450 | } | |
1451 | return ret; | |
1452 | } | |
1453 | ||
d25e35bc RE |
1454 | /** |
1455 | * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria | |
1456 | * @bo: Handle of buffer object being pinned | |
1457 | * @domain: Domain into which BO should be pinned | |
1458 | * | |
1459 | * - USERPTR BOs are UNPINNABLE and will return error | |
1460 | * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their | |
1461 | * PIN count incremented. It is valid to PIN a BO multiple times | |
1462 | * | |
1463 | * Return: ZERO if successful in pinning, Non-Zero in case of error. | |
1464 | */ | |
1465 | static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) | |
1466 | { | |
1467 | int ret = 0; | |
1468 | ||
1469 | ret = amdgpu_bo_reserve(bo, false); | |
1470 | if (unlikely(ret)) | |
1471 | return ret; | |
1472 | ||
1473 | ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0); | |
1474 | if (ret) | |
1475 | pr_err("Error in Pinning BO to domain: %d\n", domain); | |
1476 | ||
1477 | amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); | |
1478 | amdgpu_bo_unreserve(bo); | |
1479 | ||
1480 | return ret; | |
1481 | } | |
1482 | ||
1483 | /** | |
1484 | * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria | |
1485 | * @bo: Handle of buffer object being unpinned | |
1486 | * | |
1487 | * - Is a illegal request for USERPTR BOs and is ignored | |
1488 | * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their | |
1489 | * PIN count decremented. Calls to UNPIN must balance calls to PIN | |
1490 | */ | |
37ba5bbc | 1491 | static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo) |
d25e35bc RE |
1492 | { |
1493 | int ret = 0; | |
1494 | ||
1495 | ret = amdgpu_bo_reserve(bo, false); | |
1496 | if (unlikely(ret)) | |
1497 | return; | |
1498 | ||
1499 | amdgpu_bo_unpin(bo); | |
1500 | amdgpu_bo_unreserve(bo); | |
1501 | } | |
1502 | ||
1a799c4c | 1503 | int amdgpu_amdkfd_gpuvm_set_vm_pasid(struct amdgpu_device *adev, |
23b02b0e | 1504 | struct amdgpu_vm *avm, u32 pasid) |
1a799c4c | 1505 | |
a46a2cd1 | 1506 | { |
ede0dd86 | 1507 | int ret; |
a46a2cd1 | 1508 | |
88f7f881 ND |
1509 | /* Free the original amdgpu allocated pasid, |
1510 | * will be replaced with kfd allocated pasid. | |
1511 | */ | |
1512 | if (avm->pasid) { | |
1513 | amdgpu_pasid_free(avm->pasid); | |
1514 | amdgpu_vm_set_pasid(adev, avm, 0); | |
1515 | } | |
1516 | ||
1a799c4c | 1517 | ret = amdgpu_vm_set_pasid(adev, avm, pasid); |
ede0dd86 FK |
1518 | if (ret) |
1519 | return ret; | |
1520 | ||
1a799c4c PY |
1521 | return 0; |
1522 | } | |
1523 | ||
1524 | int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev, | |
23b02b0e | 1525 | struct amdgpu_vm *avm, |
1a799c4c PY |
1526 | void **process_info, |
1527 | struct dma_fence **ef) | |
1528 | { | |
1a799c4c PY |
1529 | int ret; |
1530 | ||
1a799c4c PY |
1531 | /* Already a compute VM? */ |
1532 | if (avm->process_info) | |
1533 | return -EINVAL; | |
1534 | ||
1535 | /* Convert VM into a compute VM */ | |
1536 | ret = amdgpu_vm_make_compute(adev, avm); | |
1537 | if (ret) | |
1538 | return ret; | |
1539 | ||
ede0dd86 FK |
1540 | /* Initialize KFD part of the VM and process info */ |
1541 | ret = init_kfd_vm(avm, process_info, ef); | |
1542 | if (ret) | |
1543 | return ret; | |
1544 | ||
b40a6ab2 | 1545 | amdgpu_vm_set_task_info(avm); |
ede0dd86 FK |
1546 | |
1547 | return 0; | |
1548 | } | |
1549 | ||
1550 | void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, | |
1551 | struct amdgpu_vm *vm) | |
1552 | { | |
1553 | struct amdkfd_process_info *process_info = vm->process_info; | |
ede0dd86 FK |
1554 | |
1555 | if (!process_info) | |
a46a2cd1 FK |
1556 | return; |
1557 | ||
ede0dd86 | 1558 | /* Update process info */ |
a46a2cd1 FK |
1559 | mutex_lock(&process_info->lock); |
1560 | process_info->n_vms--; | |
ede0dd86 | 1561 | list_del(&vm->vm_list_node); |
a46a2cd1 FK |
1562 | mutex_unlock(&process_info->lock); |
1563 | ||
f4a3c42b | 1564 | vm->process_info = NULL; |
1565 | ||
ede0dd86 | 1566 | /* Release per-process resources when last compute VM is destroyed */ |
a46a2cd1 FK |
1567 | if (!process_info->n_vms) { |
1568 | WARN_ON(!list_empty(&process_info->kfd_bo_list)); | |
5ae0283e FK |
1569 | WARN_ON(!list_empty(&process_info->userptr_valid_list)); |
1570 | WARN_ON(!list_empty(&process_info->userptr_inval_list)); | |
a46a2cd1 FK |
1571 | |
1572 | dma_fence_put(&process_info->eviction_fence->base); | |
5ae0283e FK |
1573 | cancel_delayed_work_sync(&process_info->restore_userptr_work); |
1574 | put_pid(process_info->pid); | |
a46a2cd1 | 1575 | mutex_destroy(&process_info->lock); |
f95f51a4 | 1576 | mutex_destroy(&process_info->notifier_lock); |
a46a2cd1 FK |
1577 | kfree(process_info); |
1578 | } | |
ede0dd86 FK |
1579 | } |
1580 | ||
dff63da9 GS |
1581 | void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev, |
1582 | void *drm_priv) | |
bf47afba | 1583 | { |
b40a6ab2 | 1584 | struct amdgpu_vm *avm; |
bf47afba | 1585 | |
dff63da9 | 1586 | if (WARN_ON(!adev || !drm_priv)) |
f3729f7b | 1587 | return; |
bf47afba | 1588 | |
b40a6ab2 FK |
1589 | avm = drm_priv_to_vm(drm_priv); |
1590 | ||
1591 | pr_debug("Releasing process vm %p\n", avm); | |
bf47afba | 1592 | |
f3729f7b DV |
1593 | /* The original pasid of amdgpu vm has already been |
1594 | * released during making a amdgpu vm to a compute vm | |
1595 | * The current pasid is managed by kfd and will be | |
1596 | * released on kfd process destroy. Set amdgpu pasid | |
1597 | * to 0 to avoid duplicate release. | |
1598 | */ | |
bf47afba OZ |
1599 | amdgpu_vm_release_compute(adev, avm); |
1600 | } | |
1601 | ||
b40a6ab2 | 1602 | uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv) |
a46a2cd1 | 1603 | { |
b40a6ab2 | 1604 | struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); |
391629bd | 1605 | struct amdgpu_bo *pd = avm->root.bo; |
e715c6d0 | 1606 | struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); |
a46a2cd1 | 1607 | |
e715c6d0 SL |
1608 | if (adev->asic_type < CHIP_VEGA10) |
1609 | return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT; | |
1610 | return avm->pd_phys_addr; | |
a46a2cd1 FK |
1611 | } |
1612 | ||
011bbb03 RB |
1613 | void amdgpu_amdkfd_block_mmu_notifications(void *p) |
1614 | { | |
1615 | struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; | |
1616 | ||
1617 | mutex_lock(&pinfo->lock); | |
1618 | WRITE_ONCE(pinfo->block_mmu_notifications, true); | |
1619 | mutex_unlock(&pinfo->lock); | |
1620 | } | |
1621 | ||
1622 | int amdgpu_amdkfd_criu_resume(void *p) | |
1623 | { | |
1624 | int ret = 0; | |
1625 | struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; | |
1626 | ||
1627 | mutex_lock(&pinfo->lock); | |
1628 | pr_debug("scheduling work\n"); | |
f95f51a4 FK |
1629 | mutex_lock(&pinfo->notifier_lock); |
1630 | pinfo->evicted_bos++; | |
1631 | mutex_unlock(&pinfo->notifier_lock); | |
011bbb03 RB |
1632 | if (!READ_ONCE(pinfo->block_mmu_notifications)) { |
1633 | ret = -EINVAL; | |
1634 | goto out_unlock; | |
1635 | } | |
1636 | WRITE_ONCE(pinfo->block_mmu_notifications, false); | |
9a1c1339 FK |
1637 | queue_delayed_work(system_freezable_wq, |
1638 | &pinfo->restore_userptr_work, 0); | |
011bbb03 RB |
1639 | |
1640 | out_unlock: | |
1641 | mutex_unlock(&pinfo->lock); | |
1642 | return ret; | |
1643 | } | |
1644 | ||
1c77527a MJ |
1645 | size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev, |
1646 | uint8_t xcp_id) | |
9731dd4c DP |
1647 | { |
1648 | uint64_t reserved_for_pt = | |
1649 | ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); | |
7cb3cfc0 | 1650 | ssize_t available; |
1c77527a | 1651 | uint64_t vram_available, system_mem_available, ttm_mem_available; |
e7c94bfb | 1652 | |
9731dd4c | 1653 | spin_lock(&kfd_mem_limit.mem_limit_lock); |
1c77527a MJ |
1654 | vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id) |
1655 | - adev->kfd.vram_used_aligned[xcp_id] | |
9731dd4c DP |
1656 | - atomic64_read(&adev->vram_pin_size) |
1657 | - reserved_for_pt; | |
1c77527a MJ |
1658 | |
1659 | if (adev->gmc.is_app_apu) { | |
1660 | system_mem_available = no_system_mem_limit ? | |
1661 | kfd_mem_limit.max_system_mem_limit : | |
1662 | kfd_mem_limit.max_system_mem_limit - | |
1663 | kfd_mem_limit.system_mem_used; | |
1664 | ||
1665 | ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit - | |
1666 | kfd_mem_limit.ttm_mem_used; | |
1667 | ||
1668 | available = min3(system_mem_available, ttm_mem_available, | |
1669 | vram_available); | |
1670 | available = ALIGN_DOWN(available, PAGE_SIZE); | |
1671 | } else { | |
1672 | available = ALIGN_DOWN(vram_available, VRAM_AVAILABLITY_ALIGN); | |
1673 | } | |
1674 | ||
9731dd4c DP |
1675 | spin_unlock(&kfd_mem_limit.mem_limit_lock); |
1676 | ||
7cb3cfc0 DP |
1677 | if (available < 0) |
1678 | available = 0; | |
1679 | ||
1c77527a | 1680 | return available; |
9731dd4c DP |
1681 | } |
1682 | ||
a46a2cd1 | 1683 | int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( |
dff63da9 | 1684 | struct amdgpu_device *adev, uint64_t va, uint64_t size, |
b40a6ab2 | 1685 | void *drm_priv, struct kgd_mem **mem, |
011bbb03 | 1686 | uint64_t *offset, uint32_t flags, bool criu_resume) |
a46a2cd1 | 1687 | { |
b40a6ab2 | 1688 | struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); |
3ebfd221 | 1689 | struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm); |
b408a548 FK |
1690 | enum ttm_bo_type bo_type = ttm_bo_type_device; |
1691 | struct sg_table *sg = NULL; | |
5ae0283e | 1692 | uint64_t user_addr = 0; |
a46a2cd1 | 1693 | struct amdgpu_bo *bo; |
a872c152 | 1694 | struct drm_gem_object *gobj = NULL; |
5ae0283e | 1695 | u32 domain, alloc_domain; |
0c2dece8 | 1696 | uint64_t aligned_size; |
3ebfd221 | 1697 | int8_t xcp_id = -1; |
a46a2cd1 | 1698 | u64 alloc_flags; |
a46a2cd1 FK |
1699 | int ret; |
1700 | ||
1701 | /* | |
1702 | * Check on which domain to allocate BO | |
1703 | */ | |
1d251d90 | 1704 | if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { |
5ae0283e | 1705 | domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM; |
970c1646 RB |
1706 | |
1707 | if (adev->gmc.is_app_apu) { | |
1708 | domain = AMDGPU_GEM_DOMAIN_GTT; | |
53c5692e | 1709 | alloc_domain = AMDGPU_GEM_DOMAIN_GTT; |
970c1646 RB |
1710 | alloc_flags = 0; |
1711 | } else { | |
1712 | alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; | |
1713 | alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ? | |
99e7d65c | 1714 | AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0; |
970c1646 | 1715 | } |
18cf073f GC |
1716 | xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? |
1717 | 0 : fpriv->xcp_id; | |
1d251d90 | 1718 | } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { |
5ae0283e FK |
1719 | domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT; |
1720 | alloc_flags = 0; | |
7226f40a | 1721 | } else { |
5ae0283e FK |
1722 | domain = AMDGPU_GEM_DOMAIN_GTT; |
1723 | alloc_domain = AMDGPU_GEM_DOMAIN_CPU; | |
5bb19893 | 1724 | alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE; |
7226f40a LY |
1725 | |
1726 | if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { | |
1727 | if (!offset || !*offset) | |
1728 | return -EINVAL; | |
1729 | user_addr = untagged_addr(*offset); | |
1730 | } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | | |
1731 | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { | |
1732 | bo_type = ttm_bo_type_sg; | |
1733 | if (size > UINT_MAX) | |
1734 | return -EINVAL; | |
08a2fd23 | 1735 | sg = create_sg_table(*offset, size); |
7226f40a LY |
1736 | if (!sg) |
1737 | return -ENOMEM; | |
1738 | } else { | |
b408a548 | 1739 | return -EINVAL; |
7226f40a | 1740 | } |
a46a2cd1 FK |
1741 | } |
1742 | ||
d1a372af FK |
1743 | if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT) |
1744 | alloc_flags |= AMDGPU_GEM_CREATE_COHERENT; | |
5f248462 DF |
1745 | if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT) |
1746 | alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT; | |
d1a372af FK |
1747 | if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED) |
1748 | alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED; | |
1749 | ||
a46a2cd1 | 1750 | *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); |
b408a548 FK |
1751 | if (!*mem) { |
1752 | ret = -ENOMEM; | |
1753 | goto err; | |
1754 | } | |
c780b2ee | 1755 | INIT_LIST_HEAD(&(*mem)->attachments); |
a46a2cd1 | 1756 | mutex_init(&(*mem)->lock); |
1d251d90 | 1757 | (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM); |
a46a2cd1 FK |
1758 | |
1759 | /* Workaround for AQL queue wraparound bug. Map the same | |
1760 | * memory twice. That means we only actually allocate half | |
1761 | * the memory. | |
1762 | */ | |
1763 | if ((*mem)->aql_queue) | |
0c2dece8 PY |
1764 | size >>= 1; |
1765 | aligned_size = PAGE_ALIGN(size); | |
a46a2cd1 | 1766 | |
d0ba51b1 | 1767 | (*mem)->alloc_flags = flags; |
a46a2cd1 FK |
1768 | |
1769 | amdgpu_sync_create(&(*mem)->sync); | |
1770 | ||
1c77527a MJ |
1771 | ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags, |
1772 | xcp_id); | |
a46a2cd1 | 1773 | if (ret) { |
325f4b59 | 1774 | pr_debug("Insufficient memory\n"); |
5d240da9 | 1775 | goto err_reserve_limit; |
a46a2cd1 FK |
1776 | } |
1777 | ||
3ebfd221 | 1778 | pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n", |
53c5692e | 1779 | va, (*mem)->aql_queue ? size << 1 : size, |
3ebfd221 | 1780 | domain_string(alloc_domain), xcp_id); |
a46a2cd1 | 1781 | |
0c2dece8 | 1782 | ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags, |
3ebfd221 | 1783 | bo_type, NULL, &gobj, xcp_id + 1); |
a46a2cd1 FK |
1784 | if (ret) { |
1785 | pr_debug("Failed to create BO on domain %s. ret %d\n", | |
875440fd | 1786 | domain_string(alloc_domain), ret); |
a46a2cd1 FK |
1787 | goto err_bo_create; |
1788 | } | |
d4ec4bdc FK |
1789 | ret = drm_vma_node_allow(&gobj->vma_node, drm_priv); |
1790 | if (ret) { | |
1791 | pr_debug("Failed to allow vma node access. ret %d\n", ret); | |
1792 | goto err_node_allow; | |
1793 | } | |
18192001 FK |
1794 | ret = drm_gem_handle_create(adev->kfd.client.file, gobj, &(*mem)->gem_handle); |
1795 | if (ret) | |
1796 | goto err_gem_handle_create; | |
875440fd | 1797 | bo = gem_to_amdgpu_bo(gobj); |
b408a548 FK |
1798 | if (bo_type == ttm_bo_type_sg) { |
1799 | bo->tbo.sg = sg; | |
1800 | bo->tbo.ttm->sg = sg; | |
1801 | } | |
a46a2cd1 FK |
1802 | bo->kfd_bo = *mem; |
1803 | (*mem)->bo = bo; | |
5ae0283e | 1804 | if (user_addr) |
f04c79cf | 1805 | bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO; |
a46a2cd1 FK |
1806 | |
1807 | (*mem)->va = va; | |
5ae0283e | 1808 | (*mem)->domain = domain; |
a46a2cd1 | 1809 | (*mem)->mapped_to_gpu_memory = 0; |
5b21d3e5 | 1810 | (*mem)->process_info = avm->process_info; |
970c1646 | 1811 | |
5ae0283e FK |
1812 | add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr); |
1813 | ||
1814 | if (user_addr) { | |
fb0a0625 | 1815 | pr_debug("creating userptr BO for user_addr = %llx\n", user_addr); |
011bbb03 | 1816 | ret = init_user_pages(*mem, user_addr, criu_resume); |
71efab6a | 1817 | if (ret) |
5ae0283e | 1818 | goto allocate_init_user_pages_failed; |
a899fe8b PY |
1819 | } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | |
1820 | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { | |
d25e35bc RE |
1821 | ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT); |
1822 | if (ret) { | |
1823 | pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n"); | |
1824 | goto err_pin_bo; | |
1825 | } | |
1826 | bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; | |
1827 | bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; | |
0e2e7c5b FK |
1828 | } else { |
1829 | mutex_lock(&avm->process_info->lock); | |
1830 | if (avm->process_info->eviction_fence && | |
1831 | !dma_fence_is_signaled(&avm->process_info->eviction_fence->base)) | |
1832 | ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain, | |
1833 | &avm->process_info->eviction_fence->base); | |
1834 | mutex_unlock(&avm->process_info->lock); | |
1835 | if (ret) | |
1836 | goto err_validate_bo; | |
d25e35bc RE |
1837 | } |
1838 | ||
a899fe8b PY |
1839 | if (offset) |
1840 | *offset = amdgpu_bo_mmap_offset(bo); | |
1841 | ||
a46a2cd1 FK |
1842 | return 0; |
1843 | ||
5ae0283e | 1844 | allocate_init_user_pages_failed: |
d25e35bc | 1845 | err_pin_bo: |
0e2e7c5b | 1846 | err_validate_bo: |
a899fe8b | 1847 | remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); |
18192001 FK |
1848 | drm_gem_handle_delete(adev->kfd.client.file, (*mem)->gem_handle); |
1849 | err_gem_handle_create: | |
d4ec4bdc FK |
1850 | drm_vma_node_revoke(&gobj->vma_node, drm_priv); |
1851 | err_node_allow: | |
5ae0283e | 1852 | /* Don't unreserve system mem limit twice */ |
5d240da9 | 1853 | goto err_reserve_limit; |
a46a2cd1 | 1854 | err_bo_create: |
1c77527a | 1855 | amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id); |
5d240da9 | 1856 | err_reserve_limit: |
25e9227c | 1857 | amdgpu_sync_free(&(*mem)->sync); |
a46a2cd1 | 1858 | mutex_destroy(&(*mem)->lock); |
a872c152 PY |
1859 | if (gobj) |
1860 | drm_gem_object_put(gobj); | |
1861 | else | |
1862 | kfree(*mem); | |
b408a548 FK |
1863 | err: |
1864 | if (sg) { | |
1865 | sg_free_table(sg); | |
1866 | kfree(sg); | |
1867 | } | |
a46a2cd1 FK |
1868 | return ret; |
1869 | } | |
1870 | ||
1871 | int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( | |
dff63da9 | 1872 | struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv, |
d4ec4bdc | 1873 | uint64_t *size) |
a46a2cd1 FK |
1874 | { |
1875 | struct amdkfd_process_info *process_info = mem->process_info; | |
8c392cd5 | 1876 | unsigned long bo_size = mem->bo->tbo.base.size; |
0dc204bc | 1877 | bool use_release_notifier = (mem->bo->kfd_bo == mem); |
c780b2ee | 1878 | struct kfd_mem_attachment *entry, *tmp; |
a46a2cd1 | 1879 | struct bo_vm_reservation_context ctx; |
fe158997 | 1880 | unsigned int mapped_to_gpu_memory; |
a46a2cd1 | 1881 | int ret; |
44ea03e1 | 1882 | bool is_imported = false; |
a46a2cd1 FK |
1883 | |
1884 | mutex_lock(&mem->lock); | |
d25e35bc | 1885 | |
6bd8d4b7 | 1886 | /* Unpin MMIO/DOORBELL BO's that were pinned during allocation */ |
d25e35bc RE |
1887 | if (mem->alloc_flags & |
1888 | (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | | |
1889 | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { | |
1890 | amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); | |
1891 | } | |
1892 | ||
fe158997 | 1893 | mapped_to_gpu_memory = mem->mapped_to_gpu_memory; |
d4566dee | 1894 | is_imported = mem->is_imported; |
fe158997 BZ |
1895 | mutex_unlock(&mem->lock); |
1896 | /* lock is not needed after this, since mem is unused and will | |
1897 | * be freed anyway | |
1898 | */ | |
a46a2cd1 | 1899 | |
fe158997 | 1900 | if (mapped_to_gpu_memory > 0) { |
a46a2cd1 FK |
1901 | pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n", |
1902 | mem->va, bo_size); | |
a46a2cd1 FK |
1903 | return -EBUSY; |
1904 | } | |
1905 | ||
a46a2cd1 | 1906 | /* Make sure restore workers don't access the BO any more */ |
a46a2cd1 | 1907 | mutex_lock(&process_info->lock); |
8abc1eb2 | 1908 | list_del(&mem->validate_list); |
a46a2cd1 FK |
1909 | mutex_unlock(&process_info->lock); |
1910 | ||
f95f51a4 FK |
1911 | /* Cleanup user pages and MMU notifiers */ |
1912 | if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { | |
1913 | amdgpu_hmm_unregister(mem->bo); | |
1914 | mutex_lock(&process_info->notifier_lock); | |
1915 | amdgpu_ttm_tt_discard_user_pages(mem->bo->tbo.ttm, mem->range); | |
1916 | mutex_unlock(&process_info->notifier_lock); | |
1917 | } | |
f7646585 | 1918 | |
a46a2cd1 FK |
1919 | ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); |
1920 | if (unlikely(ret)) | |
1921 | return ret; | |
1922 | ||
a46a2cd1 | 1923 | amdgpu_amdkfd_remove_eviction_fence(mem->bo, |
2d086fde | 1924 | process_info->eviction_fence); |
a46a2cd1 FK |
1925 | pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, |
1926 | mem->va + bo_size * (1 + mem->aql_queue)); | |
1927 | ||
1928 | /* Remove from VM internal data structures */ | |
101b8104 PY |
1929 | list_for_each_entry_safe(entry, tmp, &mem->attachments, list) { |
1930 | kfd_mem_dmaunmap_attachment(mem, entry); | |
c780b2ee | 1931 | kfd_mem_detach(entry); |
101b8104 | 1932 | } |
a46a2cd1 | 1933 | |
7ef6b7f8 KR |
1934 | ret = unreserve_bo_and_vms(&ctx, false, false); |
1935 | ||
a46a2cd1 FK |
1936 | /* Free the sync object */ |
1937 | amdgpu_sync_free(&mem->sync); | |
1938 | ||
d8e408a8 OZ |
1939 | /* If the SG is not NULL, it's one we created for a doorbell or mmio |
1940 | * remap BO. We need to free it. | |
b408a548 FK |
1941 | */ |
1942 | if (mem->bo->tbo.sg) { | |
1943 | sg_free_table(mem->bo->tbo.sg); | |
1944 | kfree(mem->bo->tbo.sg); | |
1945 | } | |
1946 | ||
d4566dee | 1947 | /* Update the size of the BO being freed if it was allocated from |
f915f3af HK |
1948 | * VRAM and is not imported. For APP APU VRAM allocations are done |
1949 | * in GTT domain | |
d4566dee MJ |
1950 | */ |
1951 | if (size) { | |
f915f3af HK |
1952 | if (!is_imported && |
1953 | (mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM || | |
1954 | (adev->gmc.is_app_apu && | |
1955 | mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_GTT))) | |
d4566dee MJ |
1956 | *size = bo_size; |
1957 | else | |
1958 | *size = 0; | |
1959 | } | |
1960 | ||
a46a2cd1 | 1961 | /* Free the BO*/ |
d4ec4bdc | 1962 | drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv); |
0188006d | 1963 | drm_gem_handle_delete(adev->kfd.client.file, mem->gem_handle); |
18192001 | 1964 | if (mem->dmabuf) { |
5ac3c3e4 | 1965 | dma_buf_put(mem->dmabuf); |
18192001 FK |
1966 | mem->dmabuf = NULL; |
1967 | } | |
a46a2cd1 | 1968 | mutex_destroy(&mem->lock); |
5702d052 FK |
1969 | |
1970 | /* If this releases the last reference, it will end up calling | |
1971 | * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why | |
1972 | * this needs to be the last call here. | |
1973 | */ | |
1974 | drm_gem_object_put(&mem->bo->tbo.base); | |
a46a2cd1 | 1975 | |
0dc204bc LY |
1976 | /* |
1977 | * For kgd_mem allocated in amdgpu_amdkfd_gpuvm_import_dmabuf(), | |
1978 | * explicitly free it here. | |
1979 | */ | |
1980 | if (!use_release_notifier) | |
1981 | kfree(mem); | |
1982 | ||
a46a2cd1 FK |
1983 | return ret; |
1984 | } | |
1985 | ||
1986 | int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( | |
dff63da9 | 1987 | struct amdgpu_device *adev, struct kgd_mem *mem, |
4d30a83c | 1988 | void *drm_priv) |
a46a2cd1 | 1989 | { |
b40a6ab2 | 1990 | struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); |
a46a2cd1 FK |
1991 | int ret; |
1992 | struct amdgpu_bo *bo; | |
1993 | uint32_t domain; | |
c780b2ee | 1994 | struct kfd_mem_attachment *entry; |
a46a2cd1 | 1995 | struct bo_vm_reservation_context ctx; |
a46a2cd1 | 1996 | unsigned long bo_size; |
5ae0283e | 1997 | bool is_invalid_userptr = false; |
a46a2cd1 FK |
1998 | |
1999 | bo = mem->bo; | |
a46a2cd1 FK |
2000 | if (!bo) { |
2001 | pr_err("Invalid BO when mapping memory to GPU\n"); | |
5ae0283e | 2002 | return -EINVAL; |
a46a2cd1 FK |
2003 | } |
2004 | ||
5ae0283e FK |
2005 | /* Make sure restore is not running concurrently. Since we |
2006 | * don't map invalid userptr BOs, we rely on the next restore | |
2007 | * worker to do the mapping | |
2008 | */ | |
2009 | mutex_lock(&mem->process_info->lock); | |
2010 | ||
f95f51a4 FK |
2011 | /* Lock notifier lock. If we find an invalid userptr BO, we can be |
2012 | * sure that the MMU notifier is no longer running | |
2013 | * concurrently and the queues are actually stopped | |
2014 | */ | |
2015 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { | |
2016 | mutex_lock(&mem->process_info->notifier_lock); | |
2017 | is_invalid_userptr = !!mem->invalid; | |
2018 | mutex_unlock(&mem->process_info->notifier_lock); | |
2019 | } | |
2020 | ||
5ae0283e FK |
2021 | mutex_lock(&mem->lock); |
2022 | ||
a46a2cd1 | 2023 | domain = mem->domain; |
8c392cd5 | 2024 | bo_size = bo->tbo.base.size; |
a46a2cd1 FK |
2025 | |
2026 | pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", | |
2027 | mem->va, | |
2028 | mem->va + bo_size * (1 + mem->aql_queue), | |
b40a6ab2 | 2029 | avm, domain_string(domain)); |
a46a2cd1 | 2030 | |
9e5d2753 FK |
2031 | if (!kfd_mem_is_attached(avm, mem)) { |
2032 | ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue); | |
2033 | if (ret) | |
2034 | goto out; | |
2035 | } | |
2036 | ||
b40a6ab2 | 2037 | ret = reserve_bo_and_vm(mem, avm, &ctx); |
a46a2cd1 FK |
2038 | if (unlikely(ret)) |
2039 | goto out; | |
2040 | ||
5ae0283e FK |
2041 | /* Userptr can be marked as "not invalid", but not actually be |
2042 | * validated yet (still in the system domain). In that case | |
2043 | * the queues are still stopped and we can leave mapping for | |
2044 | * the next restore worker | |
2045 | */ | |
0f04e538 | 2046 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && |
d3116756 | 2047 | bo->tbo.resource->mem_type == TTM_PL_SYSTEM) |
5ae0283e FK |
2048 | is_invalid_userptr = true; |
2049 | ||
50661eb1 | 2050 | ret = vm_validate_pt_pd_bos(avm, NULL); |
9e5d2753 FK |
2051 | if (unlikely(ret)) |
2052 | goto out_unreserve; | |
a46a2cd1 | 2053 | |
c780b2ee FK |
2054 | list_for_each_entry(entry, &mem->attachments, list) { |
2055 | if (entry->bo_va->base.vm != avm || entry->is_mapped) | |
2056 | continue; | |
a46a2cd1 | 2057 | |
c780b2ee FK |
2058 | pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n", |
2059 | entry->va, entry->va + bo_size, entry); | |
a46a2cd1 | 2060 | |
b72ed8a2 | 2061 | ret = map_bo_to_gpuvm(mem, entry, ctx.sync, |
4d30a83c | 2062 | is_invalid_userptr); |
c780b2ee FK |
2063 | if (ret) { |
2064 | pr_err("Failed to map bo to gpuvm\n"); | |
9e5d2753 | 2065 | goto out_unreserve; |
c780b2ee | 2066 | } |
a46a2cd1 | 2067 | |
c780b2ee FK |
2068 | ret = vm_update_pds(avm, ctx.sync); |
2069 | if (ret) { | |
2070 | pr_err("Failed to update page directories\n"); | |
9e5d2753 | 2071 | goto out_unreserve; |
a46a2cd1 | 2072 | } |
c780b2ee FK |
2073 | |
2074 | entry->is_mapped = true; | |
2075 | mem->mapped_to_gpu_memory++; | |
2076 | pr_debug("\t INC mapping count %d\n", | |
2077 | mem->mapped_to_gpu_memory); | |
a46a2cd1 FK |
2078 | } |
2079 | ||
a46a2cd1 FK |
2080 | ret = unreserve_bo_and_vms(&ctx, false, false); |
2081 | ||
2082 | goto out; | |
2083 | ||
9e5d2753 | 2084 | out_unreserve: |
a46a2cd1 FK |
2085 | unreserve_bo_and_vms(&ctx, false, false); |
2086 | out: | |
2087 | mutex_unlock(&mem->process_info->lock); | |
2088 | mutex_unlock(&mem->lock); | |
2089 | return ret; | |
2090 | } | |
2091 | ||
0c93bd49 | 2092 | int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv) |
101b8104 PY |
2093 | { |
2094 | struct kfd_mem_attachment *entry; | |
2095 | struct amdgpu_vm *vm; | |
0c93bd49 | 2096 | int ret; |
101b8104 PY |
2097 | |
2098 | vm = drm_priv_to_vm(drm_priv); | |
2099 | ||
2100 | mutex_lock(&mem->lock); | |
2101 | ||
0c93bd49 LY |
2102 | ret = amdgpu_bo_reserve(mem->bo, true); |
2103 | if (ret) | |
2104 | goto out; | |
2105 | ||
101b8104 | 2106 | list_for_each_entry(entry, &mem->attachments, list) { |
0c93bd49 LY |
2107 | if (entry->bo_va->base.vm != vm) |
2108 | continue; | |
2109 | if (entry->bo_va->base.bo->tbo.ttm && | |
2110 | !entry->bo_va->base.bo->tbo.ttm->sg) | |
2111 | continue; | |
2112 | ||
2113 | kfd_mem_dmaunmap_attachment(mem, entry); | |
101b8104 PY |
2114 | } |
2115 | ||
0c93bd49 LY |
2116 | amdgpu_bo_unreserve(mem->bo); |
2117 | out: | |
101b8104 | 2118 | mutex_unlock(&mem->lock); |
0c93bd49 LY |
2119 | |
2120 | return ret; | |
101b8104 PY |
2121 | } |
2122 | ||
a46a2cd1 | 2123 | int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( |
dff63da9 | 2124 | struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv) |
a46a2cd1 | 2125 | { |
b40a6ab2 | 2126 | struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); |
8c392cd5 | 2127 | unsigned long bo_size = mem->bo->tbo.base.size; |
c780b2ee | 2128 | struct kfd_mem_attachment *entry; |
a46a2cd1 FK |
2129 | struct bo_vm_reservation_context ctx; |
2130 | int ret; | |
2131 | ||
2132 | mutex_lock(&mem->lock); | |
2133 | ||
b40a6ab2 | 2134 | ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx); |
a46a2cd1 FK |
2135 | if (unlikely(ret)) |
2136 | goto out; | |
2137 | /* If no VMs were reserved, it means the BO wasn't actually mapped */ | |
2138 | if (ctx.n_vms == 0) { | |
2139 | ret = -EINVAL; | |
2140 | goto unreserve_out; | |
2141 | } | |
2142 | ||
50661eb1 | 2143 | ret = vm_validate_pt_pd_bos(avm, NULL); |
a46a2cd1 FK |
2144 | if (unlikely(ret)) |
2145 | goto unreserve_out; | |
2146 | ||
2147 | pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n", | |
2148 | mem->va, | |
2149 | mem->va + bo_size * (1 + mem->aql_queue), | |
b40a6ab2 | 2150 | avm); |
a46a2cd1 | 2151 | |
c780b2ee FK |
2152 | list_for_each_entry(entry, &mem->attachments, list) { |
2153 | if (entry->bo_va->base.vm != avm || !entry->is_mapped) | |
2154 | continue; | |
a46a2cd1 | 2155 | |
c780b2ee FK |
2156 | pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n", |
2157 | entry->va, entry->va + bo_size, entry); | |
a46a2cd1 | 2158 | |
b72ed8a2 FK |
2159 | unmap_bo_from_gpuvm(mem, entry, ctx.sync); |
2160 | entry->is_mapped = false; | |
c780b2ee FK |
2161 | |
2162 | mem->mapped_to_gpu_memory--; | |
2163 | pr_debug("\t DEC mapping count %d\n", | |
2164 | mem->mapped_to_gpu_memory); | |
a46a2cd1 FK |
2165 | } |
2166 | ||
a46a2cd1 FK |
2167 | unreserve_out: |
2168 | unreserve_bo_and_vms(&ctx, false, false); | |
2169 | out: | |
2170 | mutex_unlock(&mem->lock); | |
2171 | return ret; | |
2172 | } | |
2173 | ||
2174 | int amdgpu_amdkfd_gpuvm_sync_memory( | |
dff63da9 | 2175 | struct amdgpu_device *adev, struct kgd_mem *mem, bool intr) |
a46a2cd1 FK |
2176 | { |
2177 | struct amdgpu_sync sync; | |
2178 | int ret; | |
2179 | ||
2180 | amdgpu_sync_create(&sync); | |
2181 | ||
2182 | mutex_lock(&mem->lock); | |
2183 | amdgpu_sync_clone(&mem->sync, &sync); | |
2184 | mutex_unlock(&mem->lock); | |
2185 | ||
2186 | ret = amdgpu_sync_wait(&sync, intr); | |
2187 | amdgpu_sync_free(&sync); | |
2188 | return ret; | |
2189 | } | |
2190 | ||
e77a541f GS |
2191 | /** |
2192 | * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count | |
e77a541f GS |
2193 | * @bo: Buffer object to be mapped |
2194 | * | |
2195 | * Before return, bo reference count is incremented. To release the reference and unpin/ | |
2196 | * unmap the BO, call amdgpu_amdkfd_free_gtt_mem. | |
2197 | */ | |
1761d9a6 | 2198 | int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo) |
e77a541f GS |
2199 | { |
2200 | int ret; | |
2201 | ||
2202 | ret = amdgpu_bo_reserve(bo, true); | |
2203 | if (ret) { | |
2204 | pr_err("Failed to reserve bo. ret %d\n", ret); | |
2205 | goto err_reserve_bo_failed; | |
2206 | } | |
2207 | ||
2208 | ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); | |
2209 | if (ret) { | |
2210 | pr_err("Failed to pin bo. ret %d\n", ret); | |
2211 | goto err_pin_bo_failed; | |
2212 | } | |
2213 | ||
2214 | ret = amdgpu_ttm_alloc_gart(&bo->tbo); | |
2215 | if (ret) { | |
2216 | pr_err("Failed to bind bo to GART. ret %d\n", ret); | |
2217 | goto err_map_bo_gart_failed; | |
2218 | } | |
2219 | ||
2220 | amdgpu_amdkfd_remove_eviction_fence( | |
7f347e3f | 2221 | bo, bo->vm_bo->vm->process_info->eviction_fence); |
e77a541f GS |
2222 | |
2223 | amdgpu_bo_unreserve(bo); | |
2224 | ||
2225 | bo = amdgpu_bo_ref(bo); | |
2226 | ||
2227 | return 0; | |
2228 | ||
2229 | err_map_bo_gart_failed: | |
2230 | amdgpu_bo_unpin(bo); | |
2231 | err_pin_bo_failed: | |
2232 | amdgpu_bo_unreserve(bo); | |
2233 | err_reserve_bo_failed: | |
2234 | ||
2235 | return ret; | |
2236 | } | |
2237 | ||
4e2d1044 FK |
2238 | /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access |
2239 | * | |
2240 | * @mem: Buffer object to be mapped for CPU access | |
2241 | * @kptr[out]: pointer in kernel CPU address space | |
2242 | * @size[out]: size of the buffer | |
2243 | * | |
2244 | * Pins the BO and maps it for kernel CPU access. The eviction fence is removed | |
2245 | * from the BO, since pinned BOs cannot be evicted. The bo must remain on the | |
2246 | * validate_list, so the GPU mapping can be restored after a page table was | |
2247 | * evicted. | |
2248 | * | |
2249 | * Return: 0 on success, error code on failure | |
2250 | */ | |
2251 | int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem, | |
2252 | void **kptr, uint64_t *size) | |
a46a2cd1 FK |
2253 | { |
2254 | int ret; | |
2255 | struct amdgpu_bo *bo = mem->bo; | |
2256 | ||
2257 | if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { | |
2258 | pr_err("userptr can't be mapped to kernel\n"); | |
2259 | return -EINVAL; | |
2260 | } | |
2261 | ||
a46a2cd1 FK |
2262 | mutex_lock(&mem->process_info->lock); |
2263 | ||
2264 | ret = amdgpu_bo_reserve(bo, true); | |
2265 | if (ret) { | |
2266 | pr_err("Failed to reserve bo. ret %d\n", ret); | |
2267 | goto bo_reserve_failed; | |
2268 | } | |
2269 | ||
7b7c6c81 | 2270 | ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); |
a46a2cd1 FK |
2271 | if (ret) { |
2272 | pr_err("Failed to pin bo. ret %d\n", ret); | |
2273 | goto pin_failed; | |
2274 | } | |
2275 | ||
2276 | ret = amdgpu_bo_kmap(bo, kptr); | |
2277 | if (ret) { | |
2278 | pr_err("Failed to map bo to kernel. ret %d\n", ret); | |
2279 | goto kmap_failed; | |
2280 | } | |
2281 | ||
2282 | amdgpu_amdkfd_remove_eviction_fence( | |
2d086fde | 2283 | bo, mem->process_info->eviction_fence); |
a46a2cd1 FK |
2284 | |
2285 | if (size) | |
2286 | *size = amdgpu_bo_size(bo); | |
2287 | ||
2288 | amdgpu_bo_unreserve(bo); | |
2289 | ||
2290 | mutex_unlock(&mem->process_info->lock); | |
2291 | return 0; | |
2292 | ||
2293 | kmap_failed: | |
2294 | amdgpu_bo_unpin(bo); | |
2295 | pin_failed: | |
2296 | amdgpu_bo_unreserve(bo); | |
2297 | bo_reserve_failed: | |
2298 | mutex_unlock(&mem->process_info->lock); | |
2299 | ||
2300 | return ret; | |
2301 | } | |
2302 | ||
4e2d1044 FK |
2303 | /** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access |
2304 | * | |
2305 | * @mem: Buffer object to be unmapped for CPU access | |
2306 | * | |
2307 | * Removes the kernel CPU mapping and unpins the BO. It does not restore the | |
2308 | * eviction fence, so this function should only be used for cleanup before the | |
2309 | * BO is destroyed. | |
2310 | */ | |
2311 | void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem) | |
68df0f19 LY |
2312 | { |
2313 | struct amdgpu_bo *bo = mem->bo; | |
2314 | ||
2315 | amdgpu_bo_reserve(bo, true); | |
2316 | amdgpu_bo_kunmap(bo); | |
2317 | amdgpu_bo_unpin(bo); | |
2318 | amdgpu_bo_unreserve(bo); | |
2319 | } | |
2320 | ||
dff63da9 GS |
2321 | int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev, |
2322 | struct kfd_vm_fault_info *mem) | |
b97dfa27 | 2323 | { |
b97dfa27 | 2324 | if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) { |
2325 | *mem = *adev->gmc.vm_fault_info; | |
e7c94bfb | 2326 | mb(); /* make sure read happened */ |
b97dfa27 | 2327 | atomic_set(&adev->gmc.vm_fault_info_updated, 0); |
2328 | } | |
2329 | return 0; | |
2330 | } | |
2331 | ||
0188006d FK |
2332 | static int import_obj_create(struct amdgpu_device *adev, |
2333 | struct dma_buf *dma_buf, | |
2334 | struct drm_gem_object *obj, | |
2335 | uint64_t va, void *drm_priv, | |
2336 | struct kgd_mem **mem, uint64_t *size, | |
2337 | uint64_t *mmap_offset) | |
1dde0ea9 | 2338 | { |
b40a6ab2 | 2339 | struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); |
1dde0ea9 | 2340 | struct amdgpu_bo *bo; |
d4ec4bdc | 2341 | int ret; |
1dde0ea9 | 2342 | |
1dde0ea9 FK |
2343 | bo = gem_to_amdgpu_bo(obj); |
2344 | if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | | |
0188006d | 2345 | AMDGPU_GEM_DOMAIN_GTT))) |
1dde0ea9 | 2346 | /* Only VRAM and GTT BOs are supported */ |
0188006d | 2347 | return -EINVAL; |
1dde0ea9 FK |
2348 | |
2349 | *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); | |
0188006d FK |
2350 | if (!*mem) |
2351 | return -ENOMEM; | |
1dde0ea9 | 2352 | |
d4ec4bdc | 2353 | ret = drm_vma_node_allow(&obj->vma_node, drm_priv); |
2d81c4cd FK |
2354 | if (ret) |
2355 | goto err_free_mem; | |
d4ec4bdc | 2356 | |
1dde0ea9 FK |
2357 | if (size) |
2358 | *size = amdgpu_bo_size(bo); | |
2359 | ||
2360 | if (mmap_offset) | |
2361 | *mmap_offset = amdgpu_bo_mmap_offset(bo); | |
2362 | ||
c780b2ee | 2363 | INIT_LIST_HEAD(&(*mem)->attachments); |
1dde0ea9 | 2364 | mutex_init(&(*mem)->lock); |
f3729f7b | 2365 | |
d0ba51b1 FK |
2366 | (*mem)->alloc_flags = |
2367 | ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? | |
1d251d90 YZ |
2368 | KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT) |
2369 | | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | |
2370 | | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; | |
1dde0ea9 | 2371 | |
2d81c4cd FK |
2372 | get_dma_buf(dma_buf); |
2373 | (*mem)->dmabuf = dma_buf; | |
fd9a9f88 | 2374 | (*mem)->bo = bo; |
1dde0ea9 | 2375 | (*mem)->va = va; |
228ce176 | 2376 | (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && !adev->gmc.is_app_apu ? |
1dde0ea9 | 2377 | AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT; |
228ce176 | 2378 | |
1dde0ea9 FK |
2379 | (*mem)->mapped_to_gpu_memory = 0; |
2380 | (*mem)->process_info = avm->process_info; | |
2381 | add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false); | |
2382 | amdgpu_sync_create(&(*mem)->sync); | |
d4566dee | 2383 | (*mem)->is_imported = true; |
1dde0ea9 | 2384 | |
0e2e7c5b FK |
2385 | mutex_lock(&avm->process_info->lock); |
2386 | if (avm->process_info->eviction_fence && | |
2387 | !dma_fence_is_signaled(&avm->process_info->eviction_fence->base)) | |
2388 | ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain, | |
2389 | &avm->process_info->eviction_fence->base); | |
2390 | mutex_unlock(&avm->process_info->lock); | |
2391 | if (ret) | |
2392 | goto err_remove_mem; | |
2393 | ||
1dde0ea9 | 2394 | return 0; |
2d81c4cd | 2395 | |
0e2e7c5b FK |
2396 | err_remove_mem: |
2397 | remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); | |
2398 | drm_vma_node_revoke(&obj->vma_node, drm_priv); | |
2d81c4cd FK |
2399 | err_free_mem: |
2400 | kfree(*mem); | |
0188006d FK |
2401 | return ret; |
2402 | } | |
2403 | ||
2404 | int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd, | |
2405 | uint64_t va, void *drm_priv, | |
2406 | struct kgd_mem **mem, uint64_t *size, | |
2407 | uint64_t *mmap_offset) | |
2408 | { | |
2409 | struct drm_gem_object *obj; | |
2410 | uint32_t handle; | |
2411 | int ret; | |
2412 | ||
2413 | ret = drm_gem_prime_fd_to_handle(&adev->ddev, adev->kfd.client.file, fd, | |
2414 | &handle); | |
2415 | if (ret) | |
2416 | return ret; | |
2417 | obj = drm_gem_object_lookup(adev->kfd.client.file, handle); | |
2418 | if (!obj) { | |
2419 | ret = -EINVAL; | |
2420 | goto err_release_handle; | |
2421 | } | |
2422 | ||
2423 | ret = import_obj_create(adev, obj->dma_buf, obj, va, drm_priv, mem, size, | |
2424 | mmap_offset); | |
2425 | if (ret) | |
2426 | goto err_put_obj; | |
2427 | ||
2428 | (*mem)->gem_handle = handle; | |
2429 | ||
2430 | return 0; | |
2431 | ||
2d81c4cd FK |
2432 | err_put_obj: |
2433 | drm_gem_object_put(obj); | |
0188006d FK |
2434 | err_release_handle: |
2435 | drm_gem_handle_delete(adev->kfd.client.file, handle); | |
2d81c4cd | 2436 | return ret; |
1dde0ea9 FK |
2437 | } |
2438 | ||
fd234e75 FK |
2439 | int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem, |
2440 | struct dma_buf **dma_buf) | |
2441 | { | |
2442 | int ret; | |
2443 | ||
2444 | mutex_lock(&mem->lock); | |
2445 | ret = kfd_mem_export_dmabuf(mem); | |
2446 | if (ret) | |
2447 | goto out; | |
2448 | ||
2449 | get_dma_buf(mem->dmabuf); | |
2450 | *dma_buf = mem->dmabuf; | |
2451 | out: | |
2452 | mutex_unlock(&mem->lock); | |
2453 | return ret; | |
2454 | } | |
2455 | ||
5ae0283e FK |
2456 | /* Evict a userptr BO by stopping the queues if necessary |
2457 | * | |
2458 | * Runs in MMU notifier, may be in RECLAIM_FS context. This means it | |
2459 | * cannot do any memory allocations, and cannot take any locks that | |
f95f51a4 | 2460 | * are held elsewhere while allocating memory. |
5ae0283e FK |
2461 | * |
2462 | * It doesn't do anything to the BO itself. The real work happens in | |
2463 | * restore, where we get updated page addresses. This function only | |
2464 | * ensures that GPU access to the BO is stopped. | |
2465 | */ | |
f95f51a4 FK |
2466 | int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni, |
2467 | unsigned long cur_seq, struct kgd_mem *mem) | |
e52482de | 2468 | { |
5ae0283e | 2469 | struct amdkfd_process_info *process_info = mem->process_info; |
5ae0283e FK |
2470 | int r = 0; |
2471 | ||
f95f51a4 FK |
2472 | /* Do not process MMU notifications during CRIU restore until |
2473 | * KFD_CRIU_OP_RESUME IOCTL is received | |
2474 | */ | |
011bbb03 RB |
2475 | if (READ_ONCE(process_info->block_mmu_notifications)) |
2476 | return 0; | |
2477 | ||
f95f51a4 FK |
2478 | mutex_lock(&process_info->notifier_lock); |
2479 | mmu_interval_set_seq(mni, cur_seq); | |
2480 | ||
2481 | mem->invalid++; | |
2482 | if (++process_info->evicted_bos == 1) { | |
5ae0283e | 2483 | /* First eviction, stop the queues */ |
f95f51a4 FK |
2484 | r = kgd2kfd_quiesce_mm(mni->mm, |
2485 | KFD_QUEUE_EVICTION_TRIGGER_USERPTR); | |
5ae0283e FK |
2486 | if (r) |
2487 | pr_err("Failed to quiesce KFD\n"); | |
9a1c1339 FK |
2488 | queue_delayed_work(system_freezable_wq, |
2489 | &process_info->restore_userptr_work, | |
5ae0283e FK |
2490 | msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); |
2491 | } | |
f95f51a4 | 2492 | mutex_unlock(&process_info->notifier_lock); |
5ae0283e FK |
2493 | |
2494 | return r; | |
2495 | } | |
2496 | ||
2497 | /* Update invalid userptr BOs | |
2498 | * | |
2499 | * Moves invalidated (evicted) userptr BOs from userptr_valid_list to | |
2500 | * userptr_inval_list and updates user pages for all BOs that have | |
2501 | * been invalidated since their last update. | |
2502 | */ | |
2503 | static int update_invalid_user_pages(struct amdkfd_process_info *process_info, | |
2504 | struct mm_struct *mm) | |
2505 | { | |
2506 | struct kgd_mem *mem, *tmp_mem; | |
2507 | struct amdgpu_bo *bo; | |
2508 | struct ttm_operation_ctx ctx = { false, false }; | |
f95f51a4 FK |
2509 | uint32_t invalid; |
2510 | int ret = 0; | |
5ae0283e | 2511 | |
f95f51a4 FK |
2512 | mutex_lock(&process_info->notifier_lock); |
2513 | ||
2514 | /* Move all invalidated BOs to the userptr_inval_list */ | |
5ae0283e FK |
2515 | list_for_each_entry_safe(mem, tmp_mem, |
2516 | &process_info->userptr_valid_list, | |
8abc1eb2 | 2517 | validate_list) |
f95f51a4 | 2518 | if (mem->invalid) |
8abc1eb2 | 2519 | list_move_tail(&mem->validate_list, |
f95f51a4 | 2520 | &process_info->userptr_inval_list); |
5ae0283e FK |
2521 | |
2522 | /* Go through userptr_inval_list and update any invalid user_pages */ | |
2523 | list_for_each_entry(mem, &process_info->userptr_inval_list, | |
8abc1eb2 | 2524 | validate_list) { |
f95f51a4 | 2525 | invalid = mem->invalid; |
5ae0283e FK |
2526 | if (!invalid) |
2527 | /* BO hasn't been invalidated since the last | |
f95f51a4 | 2528 | * revalidation attempt. Keep its page list. |
5ae0283e FK |
2529 | */ |
2530 | continue; | |
2531 | ||
2532 | bo = mem->bo; | |
2533 | ||
f95f51a4 FK |
2534 | amdgpu_ttm_tt_discard_user_pages(bo->tbo.ttm, mem->range); |
2535 | mem->range = NULL; | |
2536 | ||
2537 | /* BO reservations and getting user pages (hmm_range_fault) | |
2538 | * must happen outside the notifier lock | |
2539 | */ | |
2540 | mutex_unlock(&process_info->notifier_lock); | |
2541 | ||
2542 | /* Move the BO to system (CPU) domain if necessary to unmap | |
2543 | * and free the SG table | |
2544 | */ | |
2545 | if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) { | |
2546 | if (amdgpu_bo_reserve(bo, true)) | |
2547 | return -EAGAIN; | |
2548 | amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); | |
2549 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); | |
2550 | amdgpu_bo_unreserve(bo); | |
2551 | if (ret) { | |
2552 | pr_err("%s: Failed to invalidate userptr BO\n", | |
2553 | __func__); | |
2554 | return -EAGAIN; | |
2555 | } | |
2556 | } | |
2557 | ||
5ae0283e | 2558 | /* Get updated user pages */ |
fec8fdb5 | 2559 | ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, |
f95f51a4 | 2560 | &mem->range); |
5ae0283e | 2561 | if (ret) { |
3b8a23ae PY |
2562 | pr_debug("Failed %d to get user pages\n", ret); |
2563 | ||
2564 | /* Return -EFAULT bad address error as success. It will | |
2565 | * fail later with a VM fault if the GPU tries to access | |
2566 | * it. Better than hanging indefinitely with stalled | |
2567 | * user mode queues. | |
2568 | * | |
2569 | * Return other error -EBUSY or -ENOMEM to retry restore | |
2570 | */ | |
2571 | if (ret != -EFAULT) | |
2572 | return ret; | |
e52482de | 2573 | |
f95f51a4 | 2574 | ret = 0; |
e82fdb16 | 2575 | } |
6c55d6e9 | 2576 | |
f95f51a4 FK |
2577 | mutex_lock(&process_info->notifier_lock); |
2578 | ||
f4fd28b6 FK |
2579 | /* Mark the BO as valid unless it was invalidated |
2580 | * again concurrently. | |
2581 | */ | |
f95f51a4 FK |
2582 | if (mem->invalid != invalid) { |
2583 | ret = -EAGAIN; | |
2584 | goto unlock_out; | |
2585 | } | |
3af470cb XC |
2586 | /* set mem valid if mem has hmm range associated */ |
2587 | if (mem->range) | |
2588 | mem->invalid = 0; | |
6c55d6e9 | 2589 | } |
e82fdb16 | 2590 | |
f95f51a4 FK |
2591 | unlock_out: |
2592 | mutex_unlock(&process_info->notifier_lock); | |
2593 | ||
2594 | return ret; | |
6c55d6e9 PY |
2595 | } |
2596 | ||
5ae0283e FK |
2597 | /* Validate invalid userptr BOs |
2598 | * | |
f95f51a4 FK |
2599 | * Validates BOs on the userptr_inval_list. Also updates GPUVM page tables |
2600 | * with new page addresses and waits for the page table updates to complete. | |
5ae0283e FK |
2601 | */ |
2602 | static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) | |
2603 | { | |
8abc1eb2 | 2604 | struct ttm_operation_ctx ctx = { false, false }; |
5ae0283e | 2605 | struct amdgpu_sync sync; |
8abc1eb2 | 2606 | struct drm_exec exec; |
5ae0283e FK |
2607 | |
2608 | struct amdgpu_vm *peer_vm; | |
2609 | struct kgd_mem *mem, *tmp_mem; | |
2610 | struct amdgpu_bo *bo; | |
8abc1eb2 | 2611 | int ret; |
5ae0283e | 2612 | |
8abc1eb2 | 2613 | amdgpu_sync_create(&sync); |
5ae0283e | 2614 | |
05d24935 | 2615 | drm_exec_init(&exec, 0, 0); |
5ae0283e | 2616 | /* Reserve all BOs and page tables for validation */ |
8abc1eb2 CK |
2617 | drm_exec_until_all_locked(&exec) { |
2618 | /* Reserve all the page directories */ | |
2619 | list_for_each_entry(peer_vm, &process_info->vm_list_head, | |
2620 | vm_list_node) { | |
2621 | ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2); | |
2622 | drm_exec_retry_on_contention(&exec); | |
2623 | if (unlikely(ret)) | |
2624 | goto unreserve_out; | |
2625 | } | |
5ae0283e | 2626 | |
8abc1eb2 CK |
2627 | /* Reserve the userptr_inval_list entries to resv_list */ |
2628 | list_for_each_entry(mem, &process_info->userptr_inval_list, | |
2629 | validate_list) { | |
2630 | struct drm_gem_object *gobj; | |
2631 | ||
2632 | gobj = &mem->bo->tbo.base; | |
2633 | ret = drm_exec_prepare_obj(&exec, gobj, 1); | |
2634 | drm_exec_retry_on_contention(&exec); | |
2635 | if (unlikely(ret)) | |
2636 | goto unreserve_out; | |
2637 | } | |
2638 | } | |
5ae0283e | 2639 | |
50661eb1 | 2640 | ret = process_validate_vms(process_info, NULL); |
5ae0283e FK |
2641 | if (ret) |
2642 | goto unreserve_out; | |
2643 | ||
2644 | /* Validate BOs and update GPUVM page tables */ | |
2645 | list_for_each_entry_safe(mem, tmp_mem, | |
2646 | &process_info->userptr_inval_list, | |
8abc1eb2 | 2647 | validate_list) { |
c780b2ee | 2648 | struct kfd_mem_attachment *attachment; |
5ae0283e FK |
2649 | |
2650 | bo = mem->bo; | |
2651 | ||
899fbde1 PY |
2652 | /* Validate the BO if we got user pages */ |
2653 | if (bo->tbo.ttm->pages[0]) { | |
c704ab18 | 2654 | amdgpu_bo_placement_from_domain(bo, mem->domain); |
5ae0283e FK |
2655 | ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); |
2656 | if (ret) { | |
2657 | pr_err("%s: failed to validate BO\n", __func__); | |
2658 | goto unreserve_out; | |
2659 | } | |
2660 | } | |
2661 | ||
5ae0283e FK |
2662 | /* Update mapping. If the BO was not validated |
2663 | * (because we couldn't get user pages), this will | |
2664 | * clear the page table entries, which will result in | |
2665 | * VM faults if the GPU tries to access the invalid | |
2666 | * memory. | |
2667 | */ | |
c780b2ee FK |
2668 | list_for_each_entry(attachment, &mem->attachments, list) { |
2669 | if (!attachment->is_mapped) | |
5ae0283e FK |
2670 | continue; |
2671 | ||
b72ed8a2 | 2672 | kfd_mem_dmaunmap_attachment(mem, attachment); |
4d30a83c | 2673 | ret = update_gpuvm_pte(mem, attachment, &sync); |
5ae0283e FK |
2674 | if (ret) { |
2675 | pr_err("%s: update PTE failed\n", __func__); | |
2676 | /* make sure this gets validated again */ | |
f95f51a4 FK |
2677 | mutex_lock(&process_info->notifier_lock); |
2678 | mem->invalid++; | |
2679 | mutex_unlock(&process_info->notifier_lock); | |
5ae0283e FK |
2680 | goto unreserve_out; |
2681 | } | |
2682 | } | |
2683 | } | |
2684 | ||
2685 | /* Update page directories */ | |
2686 | ret = process_update_pds(process_info, &sync); | |
2687 | ||
2688 | unreserve_out: | |
8abc1eb2 | 2689 | drm_exec_fini(&exec); |
5ae0283e FK |
2690 | amdgpu_sync_wait(&sync, false); |
2691 | amdgpu_sync_free(&sync); | |
5ae0283e FK |
2692 | |
2693 | return ret; | |
2694 | } | |
2695 | ||
f95f51a4 FK |
2696 | /* Confirm that all user pages are valid while holding the notifier lock |
2697 | * | |
2698 | * Moves valid BOs from the userptr_inval_list back to userptr_val_list. | |
2699 | */ | |
2700 | static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info) | |
2701 | { | |
2702 | struct kgd_mem *mem, *tmp_mem; | |
2703 | int ret = 0; | |
2704 | ||
2705 | list_for_each_entry_safe(mem, tmp_mem, | |
2706 | &process_info->userptr_inval_list, | |
8abc1eb2 | 2707 | validate_list) { |
3af470cb XC |
2708 | bool valid; |
2709 | ||
2710 | /* keep mem without hmm range at userptr_inval_list */ | |
2711 | if (!mem->range) | |
2712 | continue; | |
2713 | ||
2714 | /* Only check mem with hmm range associated */ | |
2715 | valid = amdgpu_ttm_tt_get_user_pages_done( | |
2716 | mem->bo->tbo.ttm, mem->range); | |
f95f51a4 FK |
2717 | |
2718 | mem->range = NULL; | |
2719 | if (!valid) { | |
2720 | WARN(!mem->invalid, "Invalid BO not marked invalid"); | |
2721 | ret = -EAGAIN; | |
2722 | continue; | |
2723 | } | |
3af470cb XC |
2724 | |
2725 | if (mem->invalid) { | |
2726 | WARN(1, "Valid BO is marked invalid"); | |
2727 | ret = -EAGAIN; | |
2728 | continue; | |
2729 | } | |
f95f51a4 | 2730 | |
8abc1eb2 | 2731 | list_move_tail(&mem->validate_list, |
f95f51a4 FK |
2732 | &process_info->userptr_valid_list); |
2733 | } | |
2734 | ||
2735 | return ret; | |
2736 | } | |
2737 | ||
5ae0283e FK |
2738 | /* Worker callback to restore evicted userptr BOs |
2739 | * | |
2740 | * Tries to update and validate all userptr BOs. If successful and no | |
2741 | * concurrent evictions happened, the queues are restarted. Otherwise, | |
2742 | * reschedule for another attempt later. | |
2743 | */ | |
2744 | static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) | |
2745 | { | |
2746 | struct delayed_work *dwork = to_delayed_work(work); | |
2747 | struct amdkfd_process_info *process_info = | |
2748 | container_of(dwork, struct amdkfd_process_info, | |
2749 | restore_userptr_work); | |
2750 | struct task_struct *usertask; | |
2751 | struct mm_struct *mm; | |
f95f51a4 | 2752 | uint32_t evicted_bos; |
5ae0283e | 2753 | |
f95f51a4 FK |
2754 | mutex_lock(&process_info->notifier_lock); |
2755 | evicted_bos = process_info->evicted_bos; | |
2756 | mutex_unlock(&process_info->notifier_lock); | |
5ae0283e FK |
2757 | if (!evicted_bos) |
2758 | return; | |
2759 | ||
2760 | /* Reference task and mm in case of concurrent process termination */ | |
2761 | usertask = get_pid_task(process_info->pid, PIDTYPE_PID); | |
2762 | if (!usertask) | |
2763 | return; | |
2764 | mm = get_task_mm(usertask); | |
2765 | if (!mm) { | |
2766 | put_task_struct(usertask); | |
2767 | return; | |
2768 | } | |
2769 | ||
2770 | mutex_lock(&process_info->lock); | |
2771 | ||
2772 | if (update_invalid_user_pages(process_info, mm)) | |
2773 | goto unlock_out; | |
2774 | /* userptr_inval_list can be empty if all evicted userptr BOs | |
2775 | * have been freed. In that case there is nothing to validate | |
2776 | * and we can just restart the queues. | |
2777 | */ | |
2778 | if (!list_empty(&process_info->userptr_inval_list)) { | |
5ae0283e FK |
2779 | if (validate_invalid_user_pages(process_info)) |
2780 | goto unlock_out; | |
2781 | } | |
2782 | /* Final check for concurrent evicton and atomic update. If | |
2783 | * another eviction happens after successful update, it will | |
2784 | * be a first eviction that calls quiesce_mm. The eviction | |
2785 | * reference counting inside KFD will handle this case. | |
2786 | */ | |
f95f51a4 FK |
2787 | mutex_lock(&process_info->notifier_lock); |
2788 | if (process_info->evicted_bos != evicted_bos) | |
2789 | goto unlock_notifier_out; | |
2790 | ||
2791 | if (confirm_valid_user_pages_locked(process_info)) { | |
2792 | WARN(1, "User pages unexpectedly invalid"); | |
2793 | goto unlock_notifier_out; | |
2794 | } | |
2795 | ||
2796 | process_info->evicted_bos = evicted_bos = 0; | |
2797 | ||
8e07e267 | 2798 | if (kgd2kfd_resume_mm(mm)) { |
5ae0283e FK |
2799 | pr_err("%s: Failed to resume KFD\n", __func__); |
2800 | /* No recovery from this failure. Probably the CP is | |
2801 | * hanging. No point trying again. | |
2802 | */ | |
2803 | } | |
6c55d6e9 | 2804 | |
f95f51a4 FK |
2805 | unlock_notifier_out: |
2806 | mutex_unlock(&process_info->notifier_lock); | |
5ae0283e FK |
2807 | unlock_out: |
2808 | mutex_unlock(&process_info->lock); | |
5ae0283e FK |
2809 | |
2810 | /* If validation failed, reschedule another attempt */ | |
c7f21978 | 2811 | if (evicted_bos) { |
9a1c1339 FK |
2812 | queue_delayed_work(system_freezable_wq, |
2813 | &process_info->restore_userptr_work, | |
5ae0283e | 2814 | msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); |
c7f21978 PY |
2815 | |
2816 | kfd_smi_event_queue_restore_rescheduled(mm); | |
2817 | } | |
2818 | mmput(mm); | |
2819 | put_task_struct(usertask); | |
5ae0283e FK |
2820 | } |
2821 | ||
c147ddc6 | 2822 | static void replace_eviction_fence(struct dma_fence __rcu **ef, |
9a1c1339 FK |
2823 | struct dma_fence *new_ef) |
2824 | { | |
2825 | struct dma_fence *old_ef = rcu_replace_pointer(*ef, new_ef, true | |
2826 | /* protected by process_info->lock */); | |
2827 | ||
2828 | /* If we're replacing an unsignaled eviction fence, that fence will | |
2829 | * never be signaled, and if anyone is still waiting on that fence, | |
2830 | * they will hang forever. This should never happen. We should only | |
2831 | * replace the fence in restore_work that only gets scheduled after | |
2832 | * eviction work signaled the fence. | |
2833 | */ | |
2834 | WARN_ONCE(!dma_fence_is_signaled(old_ef), | |
2835 | "Replacing unsignaled eviction fence"); | |
2836 | dma_fence_put(old_ef); | |
2837 | } | |
2838 | ||
a46a2cd1 FK |
2839 | /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given |
2840 | * KFD process identified by process_info | |
2841 | * | |
2842 | * @process_info: amdkfd_process_info of the KFD process | |
2843 | * | |
2844 | * After memory eviction, restore thread calls this function. The function | |
2845 | * should be called when the Process is still valid. BO restore involves - | |
2846 | * | |
2847 | * 1. Release old eviction fence and create new one | |
2848 | * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list. | |
2849 | * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of | |
2850 | * BOs that need to be reserved. | |
2851 | * 4. Reserve all the BOs | |
2852 | * 5. Validate of PD and PT BOs. | |
2853 | * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence | |
2854 | * 7. Add fence to all PD and PT BOs. | |
2855 | * 8. Unreserve all BOs | |
2856 | */ | |
c147ddc6 | 2857 | int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu **ef) |
a46a2cd1 | 2858 | { |
a46a2cd1 | 2859 | struct amdkfd_process_info *process_info = info; |
5b21d3e5 | 2860 | struct amdgpu_vm *peer_vm; |
a46a2cd1 | 2861 | struct kgd_mem *mem; |
a46a2cd1 FK |
2862 | struct list_head duplicate_save; |
2863 | struct amdgpu_sync sync_obj; | |
15024daf PY |
2864 | unsigned long failed_size = 0; |
2865 | unsigned long total_size = 0; | |
8abc1eb2 CK |
2866 | struct drm_exec exec; |
2867 | int ret; | |
a46a2cd1 FK |
2868 | |
2869 | INIT_LIST_HEAD(&duplicate_save); | |
a46a2cd1 | 2870 | |
a46a2cd1 | 2871 | mutex_lock(&process_info->lock); |
a46a2cd1 | 2872 | |
71b9d192 | 2873 | drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); |
8abc1eb2 CK |
2874 | drm_exec_until_all_locked(&exec) { |
2875 | list_for_each_entry(peer_vm, &process_info->vm_list_head, | |
2876 | vm_list_node) { | |
2877 | ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2); | |
2878 | drm_exec_retry_on_contention(&exec); | |
71b9d192 MJ |
2879 | if (unlikely(ret)) { |
2880 | pr_err("Locking VM PD failed, ret: %d\n", ret); | |
8abc1eb2 | 2881 | goto ttm_reserve_fail; |
71b9d192 | 2882 | } |
8abc1eb2 | 2883 | } |
a46a2cd1 | 2884 | |
8abc1eb2 CK |
2885 | /* Reserve all BOs and page tables/directory. Add all BOs from |
2886 | * kfd_bo_list to ctx.list | |
2887 | */ | |
2888 | list_for_each_entry(mem, &process_info->kfd_bo_list, | |
2889 | validate_list) { | |
2890 | struct drm_gem_object *gobj; | |
2891 | ||
2892 | gobj = &mem->bo->tbo.base; | |
2893 | ret = drm_exec_prepare_obj(&exec, gobj, 1); | |
2894 | drm_exec_retry_on_contention(&exec); | |
71b9d192 MJ |
2895 | if (unlikely(ret)) { |
2896 | pr_err("drm_exec_prepare_obj failed, ret: %d\n", ret); | |
8abc1eb2 | 2897 | goto ttm_reserve_fail; |
71b9d192 | 2898 | } |
8abc1eb2 | 2899 | } |
a46a2cd1 FK |
2900 | } |
2901 | ||
2902 | amdgpu_sync_create(&sync_obj); | |
2903 | ||
9c783a11 | 2904 | /* Validate BOs managed by KFD */ |
a46a2cd1 | 2905 | list_for_each_entry(mem, &process_info->kfd_bo_list, |
8abc1eb2 | 2906 | validate_list) { |
a46a2cd1 FK |
2907 | |
2908 | struct amdgpu_bo *bo = mem->bo; | |
2909 | uint32_t domain = mem->domain; | |
8bb31587 CK |
2910 | struct dma_resv_iter cursor; |
2911 | struct dma_fence *fence; | |
a46a2cd1 | 2912 | |
15024daf PY |
2913 | total_size += amdgpu_bo_size(bo); |
2914 | ||
a46a2cd1 FK |
2915 | ret = amdgpu_amdkfd_bo_validate(bo, domain, false); |
2916 | if (ret) { | |
15024daf PY |
2917 | pr_debug("Memory eviction: Validate BOs failed\n"); |
2918 | failed_size += amdgpu_bo_size(bo); | |
2919 | ret = amdgpu_amdkfd_bo_validate(bo, | |
2920 | AMDGPU_GEM_DOMAIN_GTT, false); | |
2921 | if (ret) { | |
2922 | pr_debug("Memory eviction: Try again\n"); | |
2923 | goto validate_map_fail; | |
2924 | } | |
a46a2cd1 | 2925 | } |
8bb31587 CK |
2926 | dma_resv_for_each_fence(&cursor, bo->tbo.base.resv, |
2927 | DMA_RESV_USAGE_KERNEL, fence) { | |
2928 | ret = amdgpu_sync_fence(&sync_obj, fence); | |
2929 | if (ret) { | |
2930 | pr_debug("Memory eviction: Sync BO fence failed. Try again\n"); | |
2931 | goto validate_map_fail; | |
2932 | } | |
3d97da44 | 2933 | } |
9c783a11 LY |
2934 | } |
2935 | ||
2936 | if (failed_size) | |
2937 | pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size); | |
2938 | ||
2939 | /* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO | |
2940 | * validations above would invalidate DMABuf imports again. | |
2941 | */ | |
2942 | ret = process_validate_vms(process_info, &exec.ticket); | |
2943 | if (ret) { | |
2944 | pr_debug("Validating VMs failed, ret: %d\n", ret); | |
2945 | goto validate_map_fail; | |
2946 | } | |
2947 | ||
2948 | /* Update mappings managed by KFD. */ | |
2949 | list_for_each_entry(mem, &process_info->kfd_bo_list, | |
2950 | validate_list) { | |
2951 | struct kfd_mem_attachment *attachment; | |
2952 | ||
c780b2ee | 2953 | list_for_each_entry(attachment, &mem->attachments, list) { |
b72ed8a2 FK |
2954 | if (!attachment->is_mapped) |
2955 | continue; | |
2956 | ||
8a774fe9 | 2957 | if (attachment->bo_va->base.bo->tbo.pin_count) |
2958 | continue; | |
2959 | ||
b72ed8a2 | 2960 | kfd_mem_dmaunmap_attachment(mem, attachment); |
4d30a83c | 2961 | ret = update_gpuvm_pte(mem, attachment, &sync_obj); |
a46a2cd1 FK |
2962 | if (ret) { |
2963 | pr_debug("Memory eviction: update PTE failed. Try again\n"); | |
2964 | goto validate_map_fail; | |
2965 | } | |
2966 | } | |
2967 | } | |
2968 | ||
e6ed364e FK |
2969 | /* Update mappings not managed by KFD */ |
2970 | list_for_each_entry(peer_vm, &process_info->vm_list_head, | |
2971 | vm_list_node) { | |
2972 | struct amdgpu_device *adev = amdgpu_ttm_adev( | |
2973 | peer_vm->root.bo->tbo.bdev); | |
2974 | ||
2975 | ret = amdgpu_vm_handle_moved(adev, peer_vm, &exec.ticket); | |
2976 | if (ret) { | |
2977 | pr_debug("Memory eviction: handle moved failed. Try again\n"); | |
2978 | goto validate_map_fail; | |
2979 | } | |
2980 | } | |
2981 | ||
a46a2cd1 FK |
2982 | /* Update page directories */ |
2983 | ret = process_update_pds(process_info, &sync_obj); | |
2984 | if (ret) { | |
2985 | pr_debug("Memory eviction: update PDs failed. Try again\n"); | |
2986 | goto validate_map_fail; | |
2987 | } | |
2988 | ||
e6ed364e FK |
2989 | /* Sync with fences on all the page tables. They implicitly depend on any |
2990 | * move fences from amdgpu_vm_handle_moved above. | |
2991 | */ | |
2992 | ret = process_sync_pds_resv(process_info, &sync_obj); | |
2993 | if (ret) { | |
2994 | pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n"); | |
2995 | goto validate_map_fail; | |
2996 | } | |
2997 | ||
3d97da44 | 2998 | /* Wait for validate and PT updates to finish */ |
a46a2cd1 FK |
2999 | amdgpu_sync_wait(&sync_obj, false); |
3000 | ||
9a1c1339 FK |
3001 | /* The old eviction fence may be unsignaled if restore happens |
3002 | * after a GPU reset or suspend/resume. Keep the old fence in that | |
3003 | * case. Otherwise release the old eviction fence and create new | |
3004 | * one, because fence only goes from unsignaled to signaled once | |
3005 | * and cannot be reused. Use context and mm from the old fence. | |
3006 | * | |
3007 | * If an old eviction fence signals after this check, that's OK. | |
3008 | * Anyone signaling an eviction fence must stop the queues first | |
3009 | * and schedule another restore worker. | |
a46a2cd1 | 3010 | */ |
9a1c1339 FK |
3011 | if (dma_fence_is_signaled(&process_info->eviction_fence->base)) { |
3012 | struct amdgpu_amdkfd_fence *new_fence = | |
3013 | amdgpu_amdkfd_fence_create( | |
a46a2cd1 | 3014 | process_info->eviction_fence->base.context, |
eb2cec55 AS |
3015 | process_info->eviction_fence->mm, |
3016 | NULL); | |
9a1c1339 FK |
3017 | |
3018 | if (!new_fence) { | |
3019 | pr_err("Failed to create eviction fence\n"); | |
3020 | ret = -ENOMEM; | |
3021 | goto validate_map_fail; | |
3022 | } | |
3023 | dma_fence_put(&process_info->eviction_fence->base); | |
3024 | process_info->eviction_fence = new_fence; | |
3025 | replace_eviction_fence(ef, dma_fence_get(&new_fence->base)); | |
3026 | } else { | |
3027 | WARN_ONCE(*ef != &process_info->eviction_fence->base, | |
3028 | "KFD eviction fence doesn't match KGD process_info"); | |
a46a2cd1 | 3029 | } |
a46a2cd1 | 3030 | |
4fac4fcf | 3031 | /* Attach new eviction fence to all BOs except pinned ones */ |
8abc1eb2 | 3032 | list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) { |
4fac4fcf LY |
3033 | if (mem->bo->tbo.pin_count) |
3034 | continue; | |
3035 | ||
42470840 CK |
3036 | dma_resv_add_fence(mem->bo->tbo.base.resv, |
3037 | &process_info->eviction_fence->base, | |
3038 | DMA_RESV_USAGE_BOOKKEEP); | |
4fac4fcf | 3039 | } |
50661eb1 | 3040 | /* Attach eviction fence to PD / PT BOs and DMABuf imports */ |
a46a2cd1 FK |
3041 | list_for_each_entry(peer_vm, &process_info->vm_list_head, |
3042 | vm_list_node) { | |
391629bd | 3043 | struct amdgpu_bo *bo = peer_vm->root.bo; |
a46a2cd1 | 3044 | |
42470840 CK |
3045 | dma_resv_add_fence(bo->tbo.base.resv, |
3046 | &process_info->eviction_fence->base, | |
3047 | DMA_RESV_USAGE_BOOKKEEP); | |
a46a2cd1 FK |
3048 | } |
3049 | ||
3050 | validate_map_fail: | |
a46a2cd1 FK |
3051 | amdgpu_sync_free(&sync_obj); |
3052 | ttm_reserve_fail: | |
8abc1eb2 | 3053 | drm_exec_fini(&exec); |
a46a2cd1 | 3054 | mutex_unlock(&process_info->lock); |
a46a2cd1 FK |
3055 | return ret; |
3056 | } | |
71efab6a OZ |
3057 | |
3058 | int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem) | |
3059 | { | |
3060 | struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; | |
3061 | struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws; | |
3062 | int ret; | |
3063 | ||
3064 | if (!info || !gws) | |
3065 | return -EINVAL; | |
3066 | ||
3067 | *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); | |
3068 | if (!*mem) | |
443e902e | 3069 | return -ENOMEM; |
71efab6a OZ |
3070 | |
3071 | mutex_init(&(*mem)->lock); | |
c780b2ee | 3072 | INIT_LIST_HEAD(&(*mem)->attachments); |
71efab6a OZ |
3073 | (*mem)->bo = amdgpu_bo_ref(gws_bo); |
3074 | (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; | |
3075 | (*mem)->process_info = process_info; | |
3076 | add_kgd_mem_to_kfd_bo_list(*mem, process_info, false); | |
3077 | amdgpu_sync_create(&(*mem)->sync); | |
3078 | ||
3079 | ||
3080 | /* Validate gws bo the first time it is added to process */ | |
3081 | mutex_lock(&(*mem)->process_info->lock); | |
3082 | ret = amdgpu_bo_reserve(gws_bo, false); | |
3083 | if (unlikely(ret)) { | |
3084 | pr_err("Reserve gws bo failed %d\n", ret); | |
3085 | goto bo_reservation_failure; | |
3086 | } | |
3087 | ||
3088 | ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true); | |
3089 | if (ret) { | |
3090 | pr_err("GWS BO validate failed %d\n", ret); | |
3091 | goto bo_validation_failure; | |
3092 | } | |
3093 | /* GWS resource is shared b/t amdgpu and amdkfd | |
3094 | * Add process eviction fence to bo so they can | |
3095 | * evict each other. | |
3096 | */ | |
c8d4c18b | 3097 | ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1); |
96cf624b OZ |
3098 | if (ret) |
3099 | goto reserve_shared_fail; | |
42470840 CK |
3100 | dma_resv_add_fence(gws_bo->tbo.base.resv, |
3101 | &process_info->eviction_fence->base, | |
3102 | DMA_RESV_USAGE_BOOKKEEP); | |
71efab6a OZ |
3103 | amdgpu_bo_unreserve(gws_bo); |
3104 | mutex_unlock(&(*mem)->process_info->lock); | |
3105 | ||
3106 | return ret; | |
3107 | ||
96cf624b | 3108 | reserve_shared_fail: |
71efab6a OZ |
3109 | bo_validation_failure: |
3110 | amdgpu_bo_unreserve(gws_bo); | |
3111 | bo_reservation_failure: | |
3112 | mutex_unlock(&(*mem)->process_info->lock); | |
3113 | amdgpu_sync_free(&(*mem)->sync); | |
3114 | remove_kgd_mem_from_kfd_bo_list(*mem, process_info); | |
3115 | amdgpu_bo_unref(&gws_bo); | |
3116 | mutex_destroy(&(*mem)->lock); | |
3117 | kfree(*mem); | |
3118 | *mem = NULL; | |
3119 | return ret; | |
3120 | } | |
3121 | ||
3122 | int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem) | |
3123 | { | |
3124 | int ret; | |
3125 | struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; | |
3126 | struct kgd_mem *kgd_mem = (struct kgd_mem *)mem; | |
3127 | struct amdgpu_bo *gws_bo = kgd_mem->bo; | |
3128 | ||
3129 | /* Remove BO from process's validate list so restore worker won't touch | |
3130 | * it anymore | |
3131 | */ | |
3132 | remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info); | |
3133 | ||
3134 | ret = amdgpu_bo_reserve(gws_bo, false); | |
3135 | if (unlikely(ret)) { | |
3136 | pr_err("Reserve gws bo failed %d\n", ret); | |
3137 | //TODO add BO back to validate_list? | |
3138 | return ret; | |
3139 | } | |
3140 | amdgpu_amdkfd_remove_eviction_fence(gws_bo, | |
3141 | process_info->eviction_fence); | |
3142 | amdgpu_bo_unreserve(gws_bo); | |
3143 | amdgpu_sync_free(&kgd_mem->sync); | |
3144 | amdgpu_bo_unref(&gws_bo); | |
3145 | mutex_destroy(&kgd_mem->lock); | |
3146 | kfree(mem); | |
3147 | return 0; | |
3148 | } | |
fd7d08ba YZ |
3149 | |
3150 | /* Returns GPU-specific tiling mode information */ | |
dff63da9 | 3151 | int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev, |
fd7d08ba YZ |
3152 | struct tile_config *config) |
3153 | { | |
fd7d08ba YZ |
3154 | config->gb_addr_config = adev->gfx.config.gb_addr_config; |
3155 | config->tile_config_ptr = adev->gfx.config.tile_mode_array; | |
3156 | config->num_tile_configs = | |
3157 | ARRAY_SIZE(adev->gfx.config.tile_mode_array); | |
3158 | config->macro_tile_config_ptr = | |
3159 | adev->gfx.config.macrotile_mode_array; | |
3160 | config->num_macro_tile_configs = | |
3161 | ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); | |
3162 | ||
3163 | /* Those values are not set from GFX9 onwards */ | |
3164 | config->num_banks = adev->gfx.config.num_banks; | |
3165 | config->num_ranks = adev->gfx.config.num_ranks; | |
3166 | ||
3167 | return 0; | |
3168 | } | |
5ccbb057 RB |
3169 | |
3170 | bool amdgpu_amdkfd_bo_mapped_to_dev(struct amdgpu_device *adev, struct kgd_mem *mem) | |
3171 | { | |
3172 | struct kfd_mem_attachment *entry; | |
3173 | ||
3174 | list_for_each_entry(entry, &mem->attachments, list) { | |
3175 | if (entry->is_mapped && entry->adev == adev) | |
3176 | return true; | |
3177 | } | |
3178 | return false; | |
3179 | } | |
3d2af401 AS |
3180 | |
3181 | #if defined(CONFIG_DEBUG_FS) | |
3182 | ||
3183 | int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data) | |
3184 | { | |
3185 | ||
3186 | spin_lock(&kfd_mem_limit.mem_limit_lock); | |
3187 | seq_printf(m, "System mem used %lldM out of %lluM\n", | |
3188 | (kfd_mem_limit.system_mem_used >> 20), | |
3189 | (kfd_mem_limit.max_system_mem_limit >> 20)); | |
3190 | seq_printf(m, "TTM mem used %lldM out of %lluM\n", | |
3191 | (kfd_mem_limit.ttm_mem_used >> 20), | |
3192 | (kfd_mem_limit.max_ttm_mem_limit >> 20)); | |
3193 | spin_unlock(&kfd_mem_limit.mem_limit_lock); | |
3194 | ||
3195 | return 0; | |
3196 | } | |
3197 | ||
3198 | #endif |