[linux-block.git] / drivers / accel / ivpu / ivpu_gem.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2020-2023 Intel Corporation
 */

#include <linux/dma-buf.h>
#include <linux/highmem.h>
#include <linux/module.h>
#include <linux/set_memory.h>
#include <linux/xarray.h>

#include <drm/drm_cache.h>
#include <drm/drm_debugfs.h>
#include <drm/drm_file.h>
#include <drm/drm_utils.h>

#include "ivpu_drv.h"
#include "ivpu_gem.h"
#include "ivpu_hw.h"
#include "ivpu_mmu.h"
#include "ivpu_mmu_context.h"

MODULE_IMPORT_NS("DMA_BUF");

static const struct drm_gem_object_funcs ivpu_gem_funcs;

static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
{
	ivpu_dbg(vdev, BO,
		 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",
		 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx_id,
		 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
		 (bool)drm_gem_is_imported(&bo->base.base));
}

static inline int ivpu_bo_lock(struct ivpu_bo *bo)
{
	return dma_resv_lock(bo->base.base.resv, NULL);
}

static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
{
	dma_resv_unlock(bo->base.base.resv);
}

/*
 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
 *
 * This function pins physical memory pages, then maps the physical pages
 * to IOMMU address space and finally updates the VPU MMU page tables
 * to allow the VPU to translate VPU address to IOMMU address.
 */
int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
{
	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
	struct sg_table *sgt;
	int ret = 0;

	ivpu_dbg_bo(vdev, bo, "pin");

	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
	if (IS_ERR(sgt)) {
		ret = PTR_ERR(sgt);
		ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
		return ret;
	}

	ivpu_bo_lock(bo);

	if (!bo->mmu_mapped) {
		drm_WARN_ON(&vdev->drm, !bo->ctx);
		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
					       ivpu_bo_is_snooped(bo));
		if (ret) {
			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
			goto unlock;
		}
		bo->mmu_mapped = true;
	}

unlock:
	ivpu_bo_unlock(bo);

	return ret;
}

static int
ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
		       const struct ivpu_addr_range *range)
{
	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
	int idx, ret;

	if (!drm_dev_enter(&vdev->drm, &idx))
		return -ENODEV;

	ivpu_bo_lock(bo);

	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
	if (!ret) {
		bo->ctx = ctx;
		bo->vpu_addr = bo->mm_node.start;
	} else {
		ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
	}

	ivpu_bo_unlock(bo);

	drm_dev_exit(idx);

	return ret;
}

static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
{
	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);

	lockdep_assert(dma_resv_held(bo->base.base.resv) || !kref_read(&bo->base.base.refcount));

	if (bo->mmu_mapped) {
		drm_WARN_ON(&vdev->drm, !bo->ctx);
		drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
		drm_WARN_ON(&vdev->drm, !bo->base.sgt);
		ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
		bo->mmu_mapped = false;
	}

	if (bo->ctx) {
		ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
		bo->ctx = NULL;
	}

	if (drm_gem_is_imported(&bo->base.base))
		return;

	if (bo->base.sgt) {
		dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
		sg_free_table(bo->base.sgt);
		kfree(bo->base.sgt);
		bo->base.sgt = NULL;
	}
}

void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
{
	struct ivpu_bo *bo;

	if (drm_WARN_ON(&vdev->drm, !ctx))
		return;

	mutex_lock(&vdev->bo_list_lock);
	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
		ivpu_bo_lock(bo);
		if (bo->ctx == ctx) {
			ivpu_dbg_bo(vdev, bo, "unbind");
			ivpu_bo_unbind_locked(bo);
		}
		ivpu_bo_unlock(bo);
	}
	mutex_unlock(&vdev->bo_list_lock);
}

struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
{
	struct ivpu_bo *bo;

	if (size == 0 || !PAGE_ALIGNED(size))
		return ERR_PTR(-EINVAL);

	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
	if (!bo)
		return ERR_PTR(-ENOMEM);

	bo->base.base.funcs = &ivpu_gem_funcs;
	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */

	INIT_LIST_HEAD(&bo->bo_list_node);

	return &bo->base.base;
}

struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev,
					     struct dma_buf *dma_buf)
{
	struct device *attach_dev = dev->dev;
	struct dma_buf_attachment *attach;
	struct sg_table *sgt;
	struct drm_gem_object *obj;
	int ret;

	attach = dma_buf_attach(dma_buf, attach_dev);
	if (IS_ERR(attach))
		return ERR_CAST(attach);

	get_dma_buf(dma_buf);

	sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
	if (IS_ERR(sgt)) {
		ret = PTR_ERR(sgt);
		goto fail_detach;
	}

	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
	if (IS_ERR(obj)) {
		ret = PTR_ERR(obj);
		goto fail_unmap;
	}

	obj->import_attach = attach;
	obj->resv = dma_buf->resv;

	return obj;

fail_unmap:
	dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
fail_detach:
	dma_buf_detach(dma_buf, attach);
	dma_buf_put(dma_buf);

	return ERR_PTR(ret);
}

static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags, u32 ctx_id)
{
	struct drm_gem_shmem_object *shmem;
	struct ivpu_bo *bo;

	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
	case DRM_IVPU_BO_CACHED:
	case DRM_IVPU_BO_WC:
		break;
	default:
		return ERR_PTR(-EINVAL);
	}

	shmem = drm_gem_shmem_create(&vdev->drm, size);
	if (IS_ERR(shmem))
		return ERR_CAST(shmem);

	bo = to_ivpu_bo(&shmem->base);
	bo->ctx_id = ctx_id;
	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
	bo->flags = flags;

	mutex_lock(&vdev->bo_list_lock);
	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
	mutex_unlock(&vdev->bo_list_lock);

	ivpu_dbg_bo(vdev, bo, "alloc");

	return bo;
}

static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
{
	struct ivpu_file_priv *file_priv = file->driver_priv;
	struct ivpu_device *vdev = file_priv->vdev;
	struct ivpu_bo *bo = to_ivpu_bo(obj);
	struct ivpu_addr_range *range;

	if (bo->ctx) {
		ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
			  file_priv->ctx.id, bo->ctx->id);
		return -EALREADY;
	}

	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
		range = &vdev->hw->ranges.shave;
	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
		range = &vdev->hw->ranges.dma;
	else
		range = &vdev->hw->ranges.user;

	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
}

static void ivpu_gem_bo_free(struct drm_gem_object *obj)
{
	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
	struct ivpu_bo *bo = to_ivpu_bo(obj);

	ivpu_dbg_bo(vdev, bo, "free");

	mutex_lock(&vdev->bo_list_lock);
	list_del(&bo->bo_list_node);
	mutex_unlock(&vdev->bo_list_lock);

	drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
		    !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
	drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
	drm_WARN_ON(&vdev->drm, bo->base.vaddr);

	ivpu_bo_unbind_locked(bo);
	drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
	drm_WARN_ON(&vdev->drm, bo->ctx);

	drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1);
	drm_gem_shmem_free(&bo->base);
}

static const struct drm_gem_object_funcs ivpu_gem_funcs = {
	.free = ivpu_gem_bo_free,
	.open = ivpu_gem_bo_open,
	.print_info = drm_gem_shmem_object_print_info,
	.pin = drm_gem_shmem_object_pin,
	.unpin = drm_gem_shmem_object_unpin,
	.get_sg_table = drm_gem_shmem_object_get_sg_table,
	.vmap = drm_gem_shmem_object_vmap,
	.vunmap = drm_gem_shmem_object_vunmap,
	.mmap = drm_gem_shmem_object_mmap,
	.vm_ops = &drm_gem_shmem_vm_ops,
};

int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
	struct ivpu_file_priv *file_priv = file->driver_priv;
	struct ivpu_device *vdev = file_priv->vdev;
	struct drm_ivpu_bo_create *args = data;
	u64 size = PAGE_ALIGN(args->size);
	struct ivpu_bo *bo;
	int ret;

	if (args->flags & ~DRM_IVPU_BO_FLAGS)
		return -EINVAL;

	if (size == 0)
		return -EINVAL;

	bo = ivpu_bo_alloc(vdev, size, args->flags, file_priv->ctx.id);
	if (IS_ERR(bo)) {
		ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
			 bo, file_priv->ctx.id, args->size, args->flags);
		return PTR_ERR(bo);
	}

	ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
	if (ret)
		ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)",
			 bo, file_priv->ctx.id, args->size, args->flags);
	else
		args->vpu_addr = bo->vpu_addr;

	drm_gem_object_put(&bo->base.base);

	return ret;
}

struct ivpu_bo *
ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
	       struct ivpu_addr_range *range, u64 size, u32 flags)
{
	struct iosys_map map;
	struct ivpu_bo *bo;
	int ret;

	if (drm_WARN_ON(&vdev->drm, !range))
		return NULL;

	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));

	bo = ivpu_bo_alloc(vdev, size, flags, IVPU_GLOBAL_CONTEXT_MMU_SSID);
	if (IS_ERR(bo)) {
		ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
			 bo, range->start, size, flags);
		return NULL;
	}

	ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
	if (ret)
		goto err_put;

	ret = ivpu_bo_pin(bo);
	if (ret)
		goto err_put;

	if (flags & DRM_IVPU_BO_MAPPABLE) {
		ivpu_bo_lock(bo);
		ret = drm_gem_shmem_vmap_locked(&bo->base, &map);
		ivpu_bo_unlock(bo);

		if (ret)
			goto err_put;
	}

	return bo;

err_put:
	drm_gem_object_put(&bo->base.base);
	return NULL;
}

struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
{
	return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
}

void ivpu_bo_free(struct ivpu_bo *bo)
{
	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);

	if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
		ivpu_bo_lock(bo);
		drm_gem_shmem_vunmap_locked(&bo->base, &map);
		ivpu_bo_unlock(bo);
	}

	drm_gem_object_put(&bo->base.base);
}

int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
	struct drm_ivpu_bo_info *args = data;
	struct drm_gem_object *obj;
	struct ivpu_bo *bo;
	int ret = 0;

	obj = drm_gem_object_lookup(file, args->handle);
	if (!obj)
		return -ENOENT;

	bo = to_ivpu_bo(obj);

	ivpu_bo_lock(bo);
	args->flags = bo->flags;
	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
	args->vpu_addr = bo->vpu_addr;
	args->size = obj->size;
	ivpu_bo_unlock(bo);

	drm_gem_object_put(obj);
	return ret;
}

int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
	struct drm_ivpu_bo_wait *args = data;
	struct drm_gem_object *obj;
	unsigned long timeout;
	long ret;

	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);

	/* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
	timeout += 1;

	obj = drm_gem_object_lookup(file, args->handle);
	if (!obj)
		return -EINVAL;

	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
	if (ret == 0) {
		ret = -ETIMEDOUT;
	} else if (ret > 0) {
		ret = 0;
		args->job_status = to_ivpu_bo(obj)->job_status;
	}

	drm_gem_object_put(obj);

	return ret;
}

static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
{
	ivpu_bo_lock(bo);

	drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
		   bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
		   bo->flags, kref_read(&bo->base.base.refcount));

	if (bo->base.pages)
		drm_printf(p, " has_pages");

	if (bo->mmu_mapped)
		drm_printf(p, " mmu_mapped");

	if (drm_gem_is_imported(&bo->base.base))
		drm_printf(p, " imported");

	drm_printf(p, "\n");

	ivpu_bo_unlock(bo);
}

void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
{
	struct ivpu_device *vdev = to_ivpu_device(dev);
	struct ivpu_bo *bo;

	drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
		   "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");

	mutex_lock(&vdev->bo_list_lock);
	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
		ivpu_bo_print_info(bo, p);
	mutex_unlock(&vdev->bo_list_lock);
}

void ivpu_bo_list_print(struct drm_device *dev)
{
	struct drm_printer p = drm_info_printer(dev->dev);

	ivpu_bo_list(dev, &p);
}
Commit	Line	Data
647371a6 JL	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2020-2023 Intel Corporation
	4	*/
	5
	6	#include <linux/dma-buf.h>
	7	#include <linux/highmem.h>
	8	#include <linux/module.h>
	9	#include <linux/set_memory.h>
	10	#include <linux/xarray.h>
	11
	12	#include <drm/drm_cache.h>
	13	#include <drm/drm_debugfs.h>
	14	#include <drm/drm_file.h>
	15	#include <drm/drm_utils.h>
	16
	17	#include "ivpu_drv.h"
	18	#include "ivpu_gem.h"
	19	#include "ivpu_hw.h"
	20	#include "ivpu_mmu.h"
	21	#include "ivpu_mmu_context.h"
	22
67725f5e TR	23	MODULE_IMPORT_NS("DMA_BUF");
67725f5e TR	24
647371a6 JL	25	static const struct drm_gem_object_funcs ivpu_gem_funcs;
647371a6 JL	26
48aea7f2 JL	27	static inline void ivpu_dbg_bo(struct ivpu_device vdev, struct ivpu_bo bo, const char *action)
48aea7f2 JL	28	{
37dee2a2 JL	29	ivpu_dbg(vdev, BO,
37dee2a2 JL	30	"%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",
a01e93ee	31	action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx_id,
37dee2a2	32	(bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
dca4ee84	33	(bool)drm_gem_is_imported(&bo->base.base));
647371a6 JL	34	}
647371a6 JL	35
98d3f772 JL	36	static inline int ivpu_bo_lock(struct ivpu_bo *bo)
	37	{
	38	return dma_resv_lock(bo->base.base.resv, NULL);
	39	}
	40
	41	static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
	42	{
	43	dma_resv_unlock(bo->base.base.resv);
	44	}
	45
647371a6 JL	46	/*
	47	* ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
	48	*
	49	* This function pins physical memory pages, then maps the physical pages
	50	* to IOMMU address space and finally updates the VPU MMU page tables
	51	* to allow the VPU to translate VPU address to IOMMU address.
	52	*/
	53	int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
	54	{
	55	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
98d3f772	56	struct sg_table *sgt;
647371a6 JL	57	int ret = 0;
647371a6 JL	58
8d88e4cd	59	ivpu_dbg_bo(vdev, bo, "pin");
647371a6	60
98d3f772 JL	61	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
	62	if (IS_ERR(sgt)) {
	63	ret = PTR_ERR(sgt);
	64	ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
	65	return ret;
	66	}
8d88e4cd	67
98d3f772	68	ivpu_bo_lock(bo);
647371a6	69
98d3f772 JL	70	if (!bo->mmu_mapped) {
98d3f772 JL	71	drm_WARN_ON(&vdev->drm, !bo->ctx);
8d88e4cd	72	ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
647371a6 JL	73	ivpu_bo_is_snooped(bo));
	74	if (ret) {
	75	ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
	76	goto unlock;
	77	}
	78	bo->mmu_mapped = true;
	79	}
	80
	81	unlock:
98d3f772	82	ivpu_bo_unlock(bo);
647371a6 JL	83
	84	return ret;
	85	}
	86
	87	static int
	88	ivpu_bo_alloc_vpu_addr(struct ivpu_bo bo, struct ivpu_mmu_context ctx,
	89	const struct ivpu_addr_range *range)
	90	{
	91	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
f1cc6ace JL	92	int idx, ret;
	93
	94	if (!drm_dev_enter(&vdev->drm, &idx))
	95	return -ENODEV;
647371a6	96
98d3f772	97	ivpu_bo_lock(bo);
48aea7f2	98
8d88e4cd	99	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
647371a6 JL	100	if (!ret) {
	101	bo->ctx = ctx;
	102	bo->vpu_addr = bo->mm_node.start;
48aea7f2 JL	103	} else {
48aea7f2 JL	104	ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
647371a6	105	}
8d88e4cd	106
98d3f772	107	ivpu_bo_unlock(bo);
b0352241	108
f1cc6ace JL	109	drm_dev_exit(idx);
f1cc6ace JL	110
647371a6 JL	111	return ret;
	112	}
	113
48aea7f2	114	static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
647371a6 JL	115	{
647371a6 JL	116	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
647371a6	117
98d3f772	118	lockdep_assert(dma_resv_held(bo->base.base.resv) \|\| !kref_read(&bo->base.base.refcount));
647371a6 JL	119
647371a6 JL	120	if (bo->mmu_mapped) {
48aea7f2 JL	121	drm_WARN_ON(&vdev->drm, !bo->ctx);
48aea7f2 JL	122	drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
8d88e4cd JL	123	drm_WARN_ON(&vdev->drm, !bo->base.sgt);
8d88e4cd JL	124	ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
647371a6 JL	125	bo->mmu_mapped = false;
	126	}
	127
48aea7f2 JL	128	if (bo->ctx) {
48aea7f2 JL	129	ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
48aea7f2 JL	130	bo->ctx = NULL;
48aea7f2 JL	131	}
a8c099d5	132
dca4ee84	133	if (drm_gem_is_imported(&bo->base.base))
a8c099d5 JL	134	return;
a8c099d5 JL	135
a8c099d5 JL	136	if (bo->base.sgt) {
	137	dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
	138	sg_free_table(bo->base.sgt);
	139	kfree(bo->base.sgt);
	140	bo->base.sgt = NULL;
	141	}
48aea7f2	142	}
647371a6	143
f1cc6ace	144	void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device vdev, struct ivpu_mmu_context ctx)
647371a6	145	{
48aea7f2 JL	146	struct ivpu_bo *bo;
	147
	148	if (drm_WARN_ON(&vdev->drm, !ctx))
	149	return;
647371a6	150
48aea7f2 JL	151	mutex_lock(&vdev->bo_list_lock);
48aea7f2 JL	152	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
98d3f772	153	ivpu_bo_lock(bo);
37dee2a2 JL	154	if (bo->ctx == ctx) {
37dee2a2 JL	155	ivpu_dbg_bo(vdev, bo, "unbind");
48aea7f2	156	ivpu_bo_unbind_locked(bo);
37dee2a2	157	}
98d3f772	158	ivpu_bo_unlock(bo);
48aea7f2 JL	159	}
48aea7f2 JL	160	mutex_unlock(&vdev->bo_list_lock);
647371a6 JL	161	}
647371a6 JL	162
8d88e4cd	163	struct drm_gem_object ivpu_gem_create_object(struct drm_device dev, size_t size)
647371a6 JL	164	{
647371a6 JL	165	struct ivpu_bo *bo;
647371a6	166
8d88e4cd	167	if (size == 0 \|\| !PAGE_ALIGNED(size))
647371a6 JL	168	return ERR_PTR(-EINVAL);
647371a6 JL	169
8d88e4cd JL	170	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
	171	if (!bo)
	172	return ERR_PTR(-ENOMEM);
	173
	174	bo->base.base.funcs = &ivpu_gem_funcs;
	175	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
	176
	177	INIT_LIST_HEAD(&bo->bo_list_node);
8d88e4cd JL	178
	179	return &bo->base.base;
	180	}
	181
67725f5e TR	182	struct drm_gem_object ivpu_gem_prime_import(struct drm_device dev,
	183	struct dma_buf *dma_buf)
	184	{
	185	struct device *attach_dev = dev->dev;
	186	struct dma_buf_attachment *attach;
	187	struct sg_table *sgt;
	188	struct drm_gem_object *obj;
	189	int ret;
	190
	191	attach = dma_buf_attach(dma_buf, attach_dev);
	192	if (IS_ERR(attach))
	193	return ERR_CAST(attach);
	194
	195	get_dma_buf(dma_buf);
	196
	197	sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
	198	if (IS_ERR(sgt)) {
	199	ret = PTR_ERR(sgt);
	200	goto fail_detach;
	201	}
	202
	203	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
	204	if (IS_ERR(obj)) {
	205	ret = PTR_ERR(obj);
	206	goto fail_unmap;
	207	}
	208
	209	obj->import_attach = attach;
	210	obj->resv = dma_buf->resv;
	211
	212	return obj;
	213
	214	fail_unmap:
	215	dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
	216	fail_detach:
	217	dma_buf_detach(dma_buf, attach);
	218	dma_buf_put(dma_buf);
	219
	220	return ERR_PTR(ret);
	221	}
	222
a01e93ee	223	static struct ivpu_bo ivpu_bo_alloc(struct ivpu_device vdev, u64 size, u32 flags, u32 ctx_id)
8d88e4cd JL	224	{
	225	struct drm_gem_shmem_object *shmem;
	226	struct ivpu_bo *bo;
	227
647371a6 JL	228	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
647371a6 JL	229	case DRM_IVPU_BO_CACHED:
647371a6 JL	230	case DRM_IVPU_BO_WC:
	231	break;
	232	default:
	233	return ERR_PTR(-EINVAL);
	234	}
	235
8d88e4cd JL	236	shmem = drm_gem_shmem_create(&vdev->drm, size);
	237	if (IS_ERR(shmem))
	238	return ERR_CAST(shmem);
647371a6	239
8d88e4cd	240	bo = to_ivpu_bo(&shmem->base);
a01e93ee	241	bo->ctx_id = ctx_id;
8d88e4cd	242	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
647371a6	243	bo->flags = flags;
647371a6	244
48aea7f2 JL	245	mutex_lock(&vdev->bo_list_lock);
	246	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
	247	mutex_unlock(&vdev->bo_list_lock);
	248
a01e93ee JL	249	ivpu_dbg_bo(vdev, bo, "alloc");
a01e93ee JL	250
8d88e4cd	251	return bo;
647371a6 JL	252	}
647371a6 JL	253
42328003	254	static int ivpu_gem_bo_open(struct drm_gem_object obj, struct drm_file file)
b0352241 JL	255	{
	256	struct ivpu_file_priv *file_priv = file->driver_priv;
	257	struct ivpu_device *vdev = file_priv->vdev;
	258	struct ivpu_bo *bo = to_ivpu_bo(obj);
	259	struct ivpu_addr_range *range;
	260
b7a0e756 JL	261	if (bo->ctx) {
	262	ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
	263	file_priv->ctx.id, bo->ctx->id);
	264	return -EALREADY;
	265	}
	266
b0352241 JL	267	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
	268	range = &vdev->hw->ranges.shave;
	269	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
	270	range = &vdev->hw->ranges.dma;
	271	else
	272	range = &vdev->hw->ranges.user;
	273
	274	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
	275	}
	276
42328003	277	static void ivpu_gem_bo_free(struct drm_gem_object *obj)
647371a6	278	{
8d88e4cd	279	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
647371a6	280	struct ivpu_bo *bo = to_ivpu_bo(obj);
647371a6	281
37dee2a2 JL	282	ivpu_dbg_bo(vdev, bo, "free");
37dee2a2 JL	283
48aea7f2 JL	284	mutex_lock(&vdev->bo_list_lock);
	285	list_del(&bo->bo_list_node);
	286	mutex_unlock(&vdev->bo_list_lock);
647371a6	287
91274fd4 JL	288	drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
91274fd4 JL	289	!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
a01e93ee JL	290	drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
a01e93ee JL	291	drm_WARN_ON(&vdev->drm, bo->base.vaddr);
647371a6	292
f1cc6ace	293	ivpu_bo_unbind_locked(bo);
a01e93ee JL	294	drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
a01e93ee JL	295	drm_WARN_ON(&vdev->drm, bo->ctx);
647371a6	296
d5d0daff	297	drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1);
8d88e4cd	298	drm_gem_shmem_free(&bo->base);
647371a6 JL	299	}
647371a6 JL	300
647371a6	301	static const struct drm_gem_object_funcs ivpu_gem_funcs = {
42328003 WK	302	.free = ivpu_gem_bo_free,
42328003 WK	303	.open = ivpu_gem_bo_open,
8d88e4cd JL	304	.print_info = drm_gem_shmem_object_print_info,
	305	.pin = drm_gem_shmem_object_pin,
	306	.unpin = drm_gem_shmem_object_unpin,
	307	.get_sg_table = drm_gem_shmem_object_get_sg_table,
	308	.vmap = drm_gem_shmem_object_vmap,
	309	.vunmap = drm_gem_shmem_object_vunmap,
	310	.mmap = drm_gem_shmem_object_mmap,
	311	.vm_ops = &drm_gem_shmem_vm_ops,
647371a6 JL	312	};
647371a6 JL	313
8d88e4cd	314	int ivpu_bo_create_ioctl(struct drm_device dev, void data, struct drm_file *file)
647371a6 JL	315	{
	316	struct ivpu_file_priv *file_priv = file->driver_priv;
	317	struct ivpu_device *vdev = file_priv->vdev;
	318	struct drm_ivpu_bo_create *args = data;
	319	u64 size = PAGE_ALIGN(args->size);
	320	struct ivpu_bo *bo;
	321	int ret;
	322
	323	if (args->flags & ~DRM_IVPU_BO_FLAGS)
	324	return -EINVAL;
	325
	326	if (size == 0)
	327	return -EINVAL;
	328
a01e93ee	329	bo = ivpu_bo_alloc(vdev, size, args->flags, file_priv->ctx.id);
647371a6	330	if (IS_ERR(bo)) {
42328003	331	ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
647371a6 JL	332	bo, file_priv->ctx.id, args->size, args->flags);
	333	return PTR_ERR(bo);
	334	}
	335
37dee2a2	336	ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
a01e93ee JL	337	if (ret)
	338	ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)",
	339	bo, file_priv->ctx.id, args->size, args->flags);
	340	else
647371a6	341	args->vpu_addr = bo->vpu_addr;
647371a6	342
8d88e4cd	343	drm_gem_object_put(&bo->base.base);
647371a6	344
647371a6 JL	345	return ret;
	346	}
	347
	348	struct ivpu_bo *
42328003 WK	349	ivpu_bo_create(struct ivpu_device vdev, struct ivpu_mmu_context ctx,
42328003 WK	350	struct ivpu_addr_range *range, u64 size, u32 flags)
647371a6	351	{
8d88e4cd	352	struct iosys_map map;
647371a6	353	struct ivpu_bo *bo;
647371a6 JL	354	int ret;
647371a6 JL	355
42328003 WK	356	if (drm_WARN_ON(&vdev->drm, !range))
42328003 WK	357	return NULL;
647371a6	358
42328003 WK	359	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
	360	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
	361	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
647371a6	362
a01e93ee	363	bo = ivpu_bo_alloc(vdev, size, flags, IVPU_GLOBAL_CONTEXT_MMU_SSID);
647371a6	364	if (IS_ERR(bo)) {
42328003 WK	365	ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
42328003 WK	366	bo, range->start, size, flags);
647371a6 JL	367	return NULL;
	368	}
	369
42328003	370	ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
b0352241 JL	371	if (ret)
	372	goto err_put;
	373
647371a6 JL	374	ret = ivpu_bo_pin(bo);
	375	if (ret)
	376	goto err_put;
	377
42328003	378	if (flags & DRM_IVPU_BO_MAPPABLE) {
98d3f772	379	ivpu_bo_lock(bo);
835b14ce	380	ret = drm_gem_shmem_vmap_locked(&bo->base, &map);
98d3f772	381	ivpu_bo_unlock(bo);
42328003 WK	382
	383	if (ret)
	384	goto err_put;
	385	}
647371a6	386
647371a6 JL	387	return bo;
	388
	389	err_put:
8d88e4cd	390	drm_gem_object_put(&bo->base.base);
647371a6 JL	391	return NULL;
	392	}
	393
42328003 WK	394	struct ivpu_bo ivpu_bo_create_global(struct ivpu_device vdev, u64 size, u32 flags)
	395	{
	396	return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
	397	}
	398
	399	void ivpu_bo_free(struct ivpu_bo *bo)
647371a6	400	{
8d88e4cd	401	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
647371a6	402
42328003	403	if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
98d3f772	404	ivpu_bo_lock(bo);
835b14ce	405	drm_gem_shmem_vunmap_locked(&bo->base, &map);
98d3f772	406	ivpu_bo_unlock(bo);
42328003	407	}
7f663199	408
8d88e4cd	409	drm_gem_object_put(&bo->base.base);
647371a6 JL	410	}
	411
	412	int ivpu_bo_info_ioctl(struct drm_device dev, void data, struct drm_file *file)
	413	{
647371a6 JL	414	struct drm_ivpu_bo_info *args = data;
	415	struct drm_gem_object *obj;
	416	struct ivpu_bo *bo;
	417	int ret = 0;
	418
	419	obj = drm_gem_object_lookup(file, args->handle);
	420	if (!obj)
	421	return -ENOENT;
	422
	423	bo = to_ivpu_bo(obj);
	424
98d3f772	425	ivpu_bo_lock(bo);
647371a6 JL	426	args->flags = bo->flags;
	427	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
	428	args->vpu_addr = bo->vpu_addr;
	429	args->size = obj->size;
98d3f772	430	ivpu_bo_unlock(bo);
b0352241	431
647371a6 JL	432	drm_gem_object_put(obj);
	433	return ret;
	434	}
	435
cd727221 JL	436	int ivpu_bo_wait_ioctl(struct drm_device dev, void data, struct drm_file *file)
	437	{
	438	struct drm_ivpu_bo_wait *args = data;
	439	struct drm_gem_object *obj;
	440	unsigned long timeout;
	441	long ret;
	442
	443	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
	444
707542dd KW	445	/* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
	446	timeout += 1;
	447
cd727221 JL	448	obj = drm_gem_object_lookup(file, args->handle);
	449	if (!obj)
	450	return -EINVAL;
	451
	452	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
	453	if (ret == 0) {
	454	ret = -ETIMEDOUT;
	455	} else if (ret > 0) {
	456	ret = 0;
	457	args->job_status = to_ivpu_bo(obj)->job_status;
	458	}
	459
	460	drm_gem_object_put(obj);
	461
	462	return ret;
	463	}
	464
647371a6 JL	465	static void ivpu_bo_print_info(struct ivpu_bo bo, struct drm_printer p)
647371a6 JL	466	{
98d3f772	467	ivpu_bo_lock(bo);
48aea7f2	468
37dee2a2	469	drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
a01e93ee	470	bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
37dee2a2	471	bo->flags, kref_read(&bo->base.base.refcount));
8d88e4cd JL	472
	473	if (bo->base.pages)
	474	drm_printf(p, " has_pages");
	475
	476	if (bo->mmu_mapped)
	477	drm_printf(p, " mmu_mapped");
647371a6	478
dca4ee84	479	if (drm_gem_is_imported(&bo->base.base))
37dee2a2 JL	480	drm_printf(p, " imported");
37dee2a2 JL	481
8d88e4cd	482	drm_printf(p, "\n");
48aea7f2	483
98d3f772	484	ivpu_bo_unlock(bo);
647371a6 JL	485	}
	486
	487	void ivpu_bo_list(struct drm_device dev, struct drm_printer p)
	488	{
	489	struct ivpu_device *vdev = to_ivpu_device(dev);
647371a6 JL	490	struct ivpu_bo *bo;
647371a6 JL	491
37dee2a2 JL	492	drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
37dee2a2 JL	493	"bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
647371a6	494
48aea7f2 JL	495	mutex_lock(&vdev->bo_list_lock);
48aea7f2 JL	496	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
647371a6	497	ivpu_bo_print_info(bo, p);
48aea7f2	498	mutex_unlock(&vdev->bo_list_lock);
647371a6 JL	499	}
	500
	501	void ivpu_bo_list_print(struct drm_device *dev)
	502	{
	503	struct drm_printer p = drm_info_printer(dev->dev);
	504
	505	ivpu_bo_list(dev, &p);
	506	}