Commit | Line | Data |
---|---|---|
647371a6 JL |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Copyright (C) 2020-2023 Intel Corporation | |
4 | */ | |
5 | ||
6 | #include <linux/dma-buf.h> | |
7 | #include <linux/highmem.h> | |
8 | #include <linux/module.h> | |
9 | #include <linux/set_memory.h> | |
10 | #include <linux/xarray.h> | |
11 | ||
12 | #include <drm/drm_cache.h> | |
13 | #include <drm/drm_debugfs.h> | |
14 | #include <drm/drm_file.h> | |
15 | #include <drm/drm_utils.h> | |
16 | ||
17 | #include "ivpu_drv.h" | |
18 | #include "ivpu_gem.h" | |
19 | #include "ivpu_hw.h" | |
20 | #include "ivpu_mmu.h" | |
21 | #include "ivpu_mmu_context.h" | |
22 | ||
647371a6 JL |
23 | static const struct drm_gem_object_funcs ivpu_gem_funcs; |
24 | ||
48aea7f2 JL |
25 | static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) |
26 | { | |
37dee2a2 JL |
27 | ivpu_dbg(vdev, BO, |
28 | "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n", | |
29 | action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0, | |
30 | (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, | |
31 | (bool)bo->base.base.import_attach); | |
647371a6 JL |
32 | } |
33 | ||
34 | /* | |
35 | * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. | |
36 | * | |
37 | * This function pins physical memory pages, then maps the physical pages | |
38 | * to IOMMU address space and finally updates the VPU MMU page tables | |
39 | * to allow the VPU to translate VPU address to IOMMU address. | |
40 | */ | |
41 | int __must_check ivpu_bo_pin(struct ivpu_bo *bo) | |
42 | { | |
43 | struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); | |
44 | int ret = 0; | |
45 | ||
46 | mutex_lock(&bo->lock); | |
47 | ||
8d88e4cd | 48 | ivpu_dbg_bo(vdev, bo, "pin"); |
37dee2a2 | 49 | drm_WARN_ON(&vdev->drm, !bo->ctx); |
647371a6 | 50 | |
8d88e4cd JL |
51 | if (!bo->mmu_mapped) { |
52 | struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base); | |
53 | ||
54 | if (IS_ERR(sgt)) { | |
55 | ret = PTR_ERR(sgt); | |
56 | ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); | |
647371a6 | 57 | goto unlock; |
8d88e4cd | 58 | } |
647371a6 | 59 | |
8d88e4cd | 60 | ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, |
647371a6 JL |
61 | ivpu_bo_is_snooped(bo)); |
62 | if (ret) { | |
63 | ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); | |
64 | goto unlock; | |
65 | } | |
66 | bo->mmu_mapped = true; | |
67 | } | |
68 | ||
69 | unlock: | |
70 | mutex_unlock(&bo->lock); | |
71 | ||
72 | return ret; | |
73 | } | |
74 | ||
75 | static int | |
76 | ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, | |
77 | const struct ivpu_addr_range *range) | |
78 | { | |
79 | struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); | |
f1cc6ace JL |
80 | int idx, ret; |
81 | ||
82 | if (!drm_dev_enter(&vdev->drm, &idx)) | |
83 | return -ENODEV; | |
647371a6 | 84 | |
48aea7f2 JL |
85 | mutex_lock(&bo->lock); |
86 | ||
8d88e4cd | 87 | ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); |
647371a6 JL |
88 | if (!ret) { |
89 | bo->ctx = ctx; | |
90 | bo->vpu_addr = bo->mm_node.start; | |
48aea7f2 JL |
91 | } else { |
92 | ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret); | |
647371a6 | 93 | } |
8d88e4cd | 94 | |
48aea7f2 | 95 | ivpu_dbg_bo(vdev, bo, "alloc"); |
647371a6 | 96 | |
48aea7f2 | 97 | mutex_unlock(&bo->lock); |
b0352241 | 98 | |
f1cc6ace JL |
99 | drm_dev_exit(idx); |
100 | ||
647371a6 JL |
101 | return ret; |
102 | } | |
103 | ||
48aea7f2 | 104 | static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) |
647371a6 JL |
105 | { |
106 | struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); | |
647371a6 | 107 | |
37dee2a2 | 108 | lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount)); |
647371a6 JL |
109 | |
110 | if (bo->mmu_mapped) { | |
48aea7f2 JL |
111 | drm_WARN_ON(&vdev->drm, !bo->ctx); |
112 | drm_WARN_ON(&vdev->drm, !bo->vpu_addr); | |
8d88e4cd JL |
113 | drm_WARN_ON(&vdev->drm, !bo->base.sgt); |
114 | ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); | |
647371a6 JL |
115 | bo->mmu_mapped = false; |
116 | } | |
117 | ||
48aea7f2 JL |
118 | if (bo->ctx) { |
119 | ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); | |
48aea7f2 JL |
120 | bo->ctx = NULL; |
121 | } | |
a8c099d5 JL |
122 | |
123 | if (bo->base.base.import_attach) | |
124 | return; | |
125 | ||
126 | dma_resv_lock(bo->base.base.resv, NULL); | |
127 | if (bo->base.sgt) { | |
128 | dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); | |
129 | sg_free_table(bo->base.sgt); | |
130 | kfree(bo->base.sgt); | |
131 | bo->base.sgt = NULL; | |
132 | } | |
133 | dma_resv_unlock(bo->base.base.resv); | |
48aea7f2 | 134 | } |
647371a6 | 135 | |
f1cc6ace | 136 | void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) |
647371a6 | 137 | { |
48aea7f2 JL |
138 | struct ivpu_bo *bo; |
139 | ||
140 | if (drm_WARN_ON(&vdev->drm, !ctx)) | |
141 | return; | |
647371a6 | 142 | |
48aea7f2 JL |
143 | mutex_lock(&vdev->bo_list_lock); |
144 | list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { | |
145 | mutex_lock(&bo->lock); | |
37dee2a2 JL |
146 | if (bo->ctx == ctx) { |
147 | ivpu_dbg_bo(vdev, bo, "unbind"); | |
48aea7f2 | 148 | ivpu_bo_unbind_locked(bo); |
37dee2a2 | 149 | } |
48aea7f2 JL |
150 | mutex_unlock(&bo->lock); |
151 | } | |
152 | mutex_unlock(&vdev->bo_list_lock); | |
647371a6 JL |
153 | } |
154 | ||
8d88e4cd | 155 | struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) |
647371a6 JL |
156 | { |
157 | struct ivpu_bo *bo; | |
647371a6 | 158 | |
8d88e4cd | 159 | if (size == 0 || !PAGE_ALIGNED(size)) |
647371a6 JL |
160 | return ERR_PTR(-EINVAL); |
161 | ||
8d88e4cd JL |
162 | bo = kzalloc(sizeof(*bo), GFP_KERNEL); |
163 | if (!bo) | |
164 | return ERR_PTR(-ENOMEM); | |
165 | ||
166 | bo->base.base.funcs = &ivpu_gem_funcs; | |
167 | bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ | |
168 | ||
169 | INIT_LIST_HEAD(&bo->bo_list_node); | |
170 | mutex_init(&bo->lock); | |
171 | ||
172 | return &bo->base.base; | |
173 | } | |
174 | ||
42328003 | 175 | static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags) |
8d88e4cd JL |
176 | { |
177 | struct drm_gem_shmem_object *shmem; | |
178 | struct ivpu_bo *bo; | |
179 | ||
647371a6 JL |
180 | switch (flags & DRM_IVPU_BO_CACHE_MASK) { |
181 | case DRM_IVPU_BO_CACHED: | |
647371a6 JL |
182 | case DRM_IVPU_BO_WC: |
183 | break; | |
184 | default: | |
185 | return ERR_PTR(-EINVAL); | |
186 | } | |
187 | ||
8d88e4cd JL |
188 | shmem = drm_gem_shmem_create(&vdev->drm, size); |
189 | if (IS_ERR(shmem)) | |
190 | return ERR_CAST(shmem); | |
647371a6 | 191 | |
8d88e4cd JL |
192 | bo = to_ivpu_bo(&shmem->base); |
193 | bo->base.map_wc = flags & DRM_IVPU_BO_WC; | |
647371a6 | 194 | bo->flags = flags; |
647371a6 | 195 | |
48aea7f2 JL |
196 | mutex_lock(&vdev->bo_list_lock); |
197 | list_add_tail(&bo->bo_list_node, &vdev->bo_list); | |
198 | mutex_unlock(&vdev->bo_list_lock); | |
199 | ||
8d88e4cd | 200 | return bo; |
647371a6 JL |
201 | } |
202 | ||
42328003 | 203 | static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) |
b0352241 JL |
204 | { |
205 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
206 | struct ivpu_device *vdev = file_priv->vdev; | |
207 | struct ivpu_bo *bo = to_ivpu_bo(obj); | |
208 | struct ivpu_addr_range *range; | |
209 | ||
b7a0e756 JL |
210 | if (bo->ctx) { |
211 | ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n", | |
212 | file_priv->ctx.id, bo->ctx->id); | |
213 | return -EALREADY; | |
214 | } | |
215 | ||
b0352241 JL |
216 | if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) |
217 | range = &vdev->hw->ranges.shave; | |
218 | else if (bo->flags & DRM_IVPU_BO_DMA_MEM) | |
219 | range = &vdev->hw->ranges.dma; | |
220 | else | |
221 | range = &vdev->hw->ranges.user; | |
222 | ||
223 | return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); | |
224 | } | |
225 | ||
42328003 | 226 | static void ivpu_gem_bo_free(struct drm_gem_object *obj) |
647371a6 | 227 | { |
8d88e4cd | 228 | struct ivpu_device *vdev = to_ivpu_device(obj->dev); |
647371a6 | 229 | struct ivpu_bo *bo = to_ivpu_bo(obj); |
647371a6 | 230 | |
37dee2a2 JL |
231 | ivpu_dbg_bo(vdev, bo, "free"); |
232 | ||
48aea7f2 JL |
233 | mutex_lock(&vdev->bo_list_lock); |
234 | list_del(&bo->bo_list_node); | |
235 | mutex_unlock(&vdev->bo_list_lock); | |
647371a6 JL |
236 | |
237 | drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); | |
238 | ||
f1cc6ace | 239 | ivpu_bo_unbind_locked(bo); |
647371a6 | 240 | mutex_destroy(&bo->lock); |
647371a6 | 241 | |
8d88e4cd JL |
242 | drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1); |
243 | drm_gem_shmem_free(&bo->base); | |
647371a6 JL |
244 | } |
245 | ||
647371a6 | 246 | static const struct drm_gem_object_funcs ivpu_gem_funcs = { |
42328003 WK |
247 | .free = ivpu_gem_bo_free, |
248 | .open = ivpu_gem_bo_open, | |
8d88e4cd JL |
249 | .print_info = drm_gem_shmem_object_print_info, |
250 | .pin = drm_gem_shmem_object_pin, | |
251 | .unpin = drm_gem_shmem_object_unpin, | |
252 | .get_sg_table = drm_gem_shmem_object_get_sg_table, | |
253 | .vmap = drm_gem_shmem_object_vmap, | |
254 | .vunmap = drm_gem_shmem_object_vunmap, | |
255 | .mmap = drm_gem_shmem_object_mmap, | |
256 | .vm_ops = &drm_gem_shmem_vm_ops, | |
647371a6 JL |
257 | }; |
258 | ||
8d88e4cd | 259 | int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
647371a6 JL |
260 | { |
261 | struct ivpu_file_priv *file_priv = file->driver_priv; | |
262 | struct ivpu_device *vdev = file_priv->vdev; | |
263 | struct drm_ivpu_bo_create *args = data; | |
264 | u64 size = PAGE_ALIGN(args->size); | |
265 | struct ivpu_bo *bo; | |
266 | int ret; | |
267 | ||
268 | if (args->flags & ~DRM_IVPU_BO_FLAGS) | |
269 | return -EINVAL; | |
270 | ||
271 | if (size == 0) | |
272 | return -EINVAL; | |
273 | ||
42328003 | 274 | bo = ivpu_bo_alloc(vdev, size, args->flags); |
647371a6 | 275 | if (IS_ERR(bo)) { |
42328003 | 276 | ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)", |
647371a6 JL |
277 | bo, file_priv->ctx.id, args->size, args->flags); |
278 | return PTR_ERR(bo); | |
279 | } | |
280 | ||
37dee2a2 JL |
281 | ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); |
282 | if (!ret) | |
647371a6 | 283 | args->vpu_addr = bo->vpu_addr; |
647371a6 | 284 | |
8d88e4cd | 285 | drm_gem_object_put(&bo->base.base); |
647371a6 | 286 | |
647371a6 JL |
287 | return ret; |
288 | } | |
289 | ||
290 | struct ivpu_bo * | |
42328003 WK |
291 | ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, |
292 | struct ivpu_addr_range *range, u64 size, u32 flags) | |
647371a6 | 293 | { |
8d88e4cd | 294 | struct iosys_map map; |
647371a6 | 295 | struct ivpu_bo *bo; |
647371a6 JL |
296 | int ret; |
297 | ||
42328003 WK |
298 | if (drm_WARN_ON(&vdev->drm, !range)) |
299 | return NULL; | |
647371a6 | 300 | |
42328003 WK |
301 | drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); |
302 | drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); | |
303 | drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); | |
647371a6 | 304 | |
42328003 | 305 | bo = ivpu_bo_alloc(vdev, size, flags); |
647371a6 | 306 | if (IS_ERR(bo)) { |
42328003 WK |
307 | ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)", |
308 | bo, range->start, size, flags); | |
647371a6 JL |
309 | return NULL; |
310 | } | |
311 | ||
42328003 | 312 | ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); |
b0352241 JL |
313 | if (ret) |
314 | goto err_put; | |
315 | ||
647371a6 JL |
316 | ret = ivpu_bo_pin(bo); |
317 | if (ret) | |
318 | goto err_put; | |
319 | ||
42328003 WK |
320 | if (flags & DRM_IVPU_BO_MAPPABLE) { |
321 | dma_resv_lock(bo->base.base.resv, NULL); | |
322 | ret = drm_gem_shmem_vmap(&bo->base, &map); | |
323 | dma_resv_unlock(bo->base.base.resv); | |
324 | ||
325 | if (ret) | |
326 | goto err_put; | |
327 | } | |
647371a6 | 328 | |
647371a6 JL |
329 | return bo; |
330 | ||
331 | err_put: | |
8d88e4cd | 332 | drm_gem_object_put(&bo->base.base); |
647371a6 JL |
333 | return NULL; |
334 | } | |
335 | ||
42328003 WK |
336 | struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) |
337 | { | |
338 | return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags); | |
339 | } | |
340 | ||
341 | void ivpu_bo_free(struct ivpu_bo *bo) | |
647371a6 | 342 | { |
8d88e4cd | 343 | struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); |
647371a6 | 344 | |
42328003 WK |
345 | if (bo->flags & DRM_IVPU_BO_MAPPABLE) { |
346 | dma_resv_lock(bo->base.base.resv, NULL); | |
347 | drm_gem_shmem_vunmap(&bo->base, &map); | |
348 | dma_resv_unlock(bo->base.base.resv); | |
349 | } | |
7f663199 | 350 | |
8d88e4cd | 351 | drm_gem_object_put(&bo->base.base); |
647371a6 JL |
352 | } |
353 | ||
354 | int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) | |
355 | { | |
647371a6 JL |
356 | struct drm_ivpu_bo_info *args = data; |
357 | struct drm_gem_object *obj; | |
358 | struct ivpu_bo *bo; | |
359 | int ret = 0; | |
360 | ||
361 | obj = drm_gem_object_lookup(file, args->handle); | |
362 | if (!obj) | |
363 | return -ENOENT; | |
364 | ||
365 | bo = to_ivpu_bo(obj); | |
366 | ||
367 | mutex_lock(&bo->lock); | |
647371a6 JL |
368 | args->flags = bo->flags; |
369 | args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); | |
370 | args->vpu_addr = bo->vpu_addr; | |
371 | args->size = obj->size; | |
647371a6 | 372 | mutex_unlock(&bo->lock); |
b0352241 | 373 | |
647371a6 JL |
374 | drm_gem_object_put(obj); |
375 | return ret; | |
376 | } | |
377 | ||
cd727221 JL |
378 | int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) |
379 | { | |
380 | struct drm_ivpu_bo_wait *args = data; | |
381 | struct drm_gem_object *obj; | |
382 | unsigned long timeout; | |
383 | long ret; | |
384 | ||
385 | timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); | |
386 | ||
387 | obj = drm_gem_object_lookup(file, args->handle); | |
388 | if (!obj) | |
389 | return -EINVAL; | |
390 | ||
391 | ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); | |
392 | if (ret == 0) { | |
393 | ret = -ETIMEDOUT; | |
394 | } else if (ret > 0) { | |
395 | ret = 0; | |
396 | args->job_status = to_ivpu_bo(obj)->job_status; | |
397 | } | |
398 | ||
399 | drm_gem_object_put(obj); | |
400 | ||
401 | return ret; | |
402 | } | |
403 | ||
647371a6 JL |
404 | static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) |
405 | { | |
48aea7f2 JL |
406 | mutex_lock(&bo->lock); |
407 | ||
37dee2a2 JL |
408 | drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", |
409 | bo, bo->ctx->id, bo->vpu_addr, bo->base.base.size, | |
410 | bo->flags, kref_read(&bo->base.base.refcount)); | |
8d88e4cd JL |
411 | |
412 | if (bo->base.pages) | |
413 | drm_printf(p, " has_pages"); | |
414 | ||
415 | if (bo->mmu_mapped) | |
416 | drm_printf(p, " mmu_mapped"); | |
647371a6 | 417 | |
37dee2a2 JL |
418 | if (bo->base.base.import_attach) |
419 | drm_printf(p, " imported"); | |
420 | ||
8d88e4cd | 421 | drm_printf(p, "\n"); |
48aea7f2 JL |
422 | |
423 | mutex_unlock(&bo->lock); | |
647371a6 JL |
424 | } |
425 | ||
426 | void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) | |
427 | { | |
428 | struct ivpu_device *vdev = to_ivpu_device(dev); | |
647371a6 JL |
429 | struct ivpu_bo *bo; |
430 | ||
37dee2a2 JL |
431 | drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", |
432 | "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); | |
647371a6 | 433 | |
48aea7f2 JL |
434 | mutex_lock(&vdev->bo_list_lock); |
435 | list_for_each_entry(bo, &vdev->bo_list, bo_list_node) | |
647371a6 | 436 | ivpu_bo_print_info(bo, p); |
48aea7f2 | 437 | mutex_unlock(&vdev->bo_list_lock); |
647371a6 JL |
438 | } |
439 | ||
440 | void ivpu_bo_list_print(struct drm_device *dev) | |
441 | { | |
442 | struct drm_printer p = drm_info_printer(dev->dev); | |
443 | ||
444 | ivpu_bo_list(dev, &p); | |
445 | } |