[linux-2.6-block.git] / drivers / gpu / drm / rockchip / rockchip_drm_gem.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
 * Author:Mark Yao <mark.yao@rock-chips.com>
 */

#include <linux/dma-buf.h>
#include <linux/iommu.h>
#include <linux/vmalloc.h>

#include <drm/drm.h>
#include <drm/drm_fb_helper.h>
#include <drm/drm_gem.h>
#include <drm/drm_gem_dma_helper.h>
#include <drm/drm_prime.h>
#include <drm/drm_vma_manager.h>

#include "rockchip_drm_drv.h"
#include "rockchip_drm_gem.h"

static int rockchip_gem_iommu_map(struct rockchip_gem_object *rk_obj)
{
	struct drm_device *drm = rk_obj->base.dev;
	struct rockchip_drm_private *private = drm->dev_private;
	int prot = IOMMU_READ | IOMMU_WRITE;
	ssize_t ret;

	mutex_lock(&private->mm_lock);
	ret = drm_mm_insert_node_generic(&private->mm, &rk_obj->mm,
					 rk_obj->base.size, PAGE_SIZE,
					 0, 0);
	mutex_unlock(&private->mm_lock);

	if (ret < 0) {
		DRM_ERROR("out of I/O virtual memory: %zd\n", ret);
		return ret;
	}

	rk_obj->dma_addr = rk_obj->mm.start;

	ret = iommu_map_sgtable(private->domain, rk_obj->dma_addr, rk_obj->sgt,
				prot);
	if (ret < (ssize_t)rk_obj->base.size) {
		DRM_ERROR("failed to map buffer: size=%zd request_size=%zd\n",
			  ret, rk_obj->base.size);
		ret = -ENOMEM;
		goto err_remove_node;
	}

	rk_obj->size = ret;

	return 0;

err_remove_node:
	mutex_lock(&private->mm_lock);
	drm_mm_remove_node(&rk_obj->mm);
	mutex_unlock(&private->mm_lock);

	return ret;
}

static int rockchip_gem_iommu_unmap(struct rockchip_gem_object *rk_obj)
{
	struct drm_device *drm = rk_obj->base.dev;
	struct rockchip_drm_private *private = drm->dev_private;

	iommu_unmap(private->domain, rk_obj->dma_addr, rk_obj->size);

	mutex_lock(&private->mm_lock);

	drm_mm_remove_node(&rk_obj->mm);

	mutex_unlock(&private->mm_lock);

	return 0;
}

static int rockchip_gem_get_pages(struct rockchip_gem_object *rk_obj)
{
	struct drm_device *drm = rk_obj->base.dev;
	int ret, i;
	struct scatterlist *s;

	rk_obj->pages = drm_gem_get_pages(&rk_obj->base);
	if (IS_ERR(rk_obj->pages))
		return PTR_ERR(rk_obj->pages);

	rk_obj->num_pages = rk_obj->base.size >> PAGE_SHIFT;

	rk_obj->sgt = drm_prime_pages_to_sg(rk_obj->base.dev,
					    rk_obj->pages, rk_obj->num_pages);
	if (IS_ERR(rk_obj->sgt)) {
		ret = PTR_ERR(rk_obj->sgt);
		goto err_put_pages;
	}

	/*
	 * Fake up the SG table so that dma_sync_sg_for_device() can be used
	 * to flush the pages associated with it.
	 *
	 * TODO: Replace this by drm_clflush_sg() once it can be implemented
	 * without relying on symbols that are not exported.
	 */
	for_each_sgtable_sg(rk_obj->sgt, s, i)
		sg_dma_address(s) = sg_phys(s);

	dma_sync_sgtable_for_device(drm->dev, rk_obj->sgt, DMA_TO_DEVICE);

	return 0;

err_put_pages:
	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, false, false);
	return ret;
}

static void rockchip_gem_put_pages(struct rockchip_gem_object *rk_obj)
{
	sg_free_table(rk_obj->sgt);
	kfree(rk_obj->sgt);
	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, true, true);
}

static int rockchip_gem_alloc_iommu(struct rockchip_gem_object *rk_obj,
				    bool alloc_kmap)
{
	int ret;

	ret = rockchip_gem_get_pages(rk_obj);
	if (ret < 0)
		return ret;

	ret = rockchip_gem_iommu_map(rk_obj);
	if (ret < 0)
		goto err_free;

	if (alloc_kmap) {
		rk_obj->kvaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
				      pgprot_writecombine(PAGE_KERNEL));
		if (!rk_obj->kvaddr) {
			DRM_ERROR("failed to vmap() buffer\n");
			ret = -ENOMEM;
			goto err_unmap;
		}
	}

	return 0;

err_unmap:
	rockchip_gem_iommu_unmap(rk_obj);
err_free:
	rockchip_gem_put_pages(rk_obj);

	return ret;
}

static int rockchip_gem_alloc_dma(struct rockchip_gem_object *rk_obj,
				  bool alloc_kmap)
{
	struct drm_gem_object *obj = &rk_obj->base;
	struct drm_device *drm = obj->dev;

	rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;

	if (!alloc_kmap)
		rk_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;

	rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size,
					 &rk_obj->dma_addr, GFP_KERNEL,
					 rk_obj->dma_attrs);
	if (!rk_obj->kvaddr) {
		DRM_ERROR("failed to allocate %zu byte dma buffer", obj->size);
		return -ENOMEM;
	}

	return 0;
}

static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj,
				  bool alloc_kmap)
{
	struct drm_gem_object *obj = &rk_obj->base;
	struct drm_device *drm = obj->dev;
	struct rockchip_drm_private *private = drm->dev_private;

	if (private->domain)
		return rockchip_gem_alloc_iommu(rk_obj, alloc_kmap);
	else
		return rockchip_gem_alloc_dma(rk_obj, alloc_kmap);
}

static void rockchip_gem_free_iommu(struct rockchip_gem_object *rk_obj)
{
	vunmap(rk_obj->kvaddr);
	rockchip_gem_iommu_unmap(rk_obj);
	rockchip_gem_put_pages(rk_obj);
}

static void rockchip_gem_free_dma(struct rockchip_gem_object *rk_obj)
{
	struct drm_gem_object *obj = &rk_obj->base;
	struct drm_device *drm = obj->dev;

	dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr,
		       rk_obj->dma_attrs);
}

static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj)
{
	if (rk_obj->pages)
		rockchip_gem_free_iommu(rk_obj);
	else
		rockchip_gem_free_dma(rk_obj);
}

static int rockchip_drm_gem_object_mmap_iommu(struct drm_gem_object *obj,
					      struct vm_area_struct *vma)
{
	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
	unsigned int count = obj->size >> PAGE_SHIFT;
	unsigned long user_count = vma_pages(vma);

	if (user_count == 0)
		return -ENXIO;

	return vm_map_pages(vma, rk_obj->pages, count);
}

static int rockchip_drm_gem_object_mmap_dma(struct drm_gem_object *obj,
					    struct vm_area_struct *vma)
{
	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
	struct drm_device *drm = obj->dev;

	return dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr,
			      obj->size, rk_obj->dma_attrs);
}

static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj,
					struct vm_area_struct *vma)
{
	int ret;
	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);

	/*
	 * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the
	 * whole buffer from the start.
	 */
	vma->vm_pgoff = 0;

	/*
	 * We allocated a struct page table for rk_obj, so clear
	 * VM_PFNMAP flag that was set by drm_gem_mmap_obj()/drm_gem_mmap().
	 */
	vm_flags_mod(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP, VM_PFNMAP);

	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);

	if (rk_obj->pages)
		ret = rockchip_drm_gem_object_mmap_iommu(obj, vma);
	else
		ret = rockchip_drm_gem_object_mmap_dma(obj, vma);

	return ret;
}

static void rockchip_gem_release_object(struct rockchip_gem_object *rk_obj)
{
	drm_gem_object_release(&rk_obj->base);
	kfree(rk_obj);
}

static const struct drm_gem_object_funcs rockchip_gem_object_funcs = {
	.free = rockchip_gem_free_object,
	.get_sg_table = rockchip_gem_prime_get_sg_table,
	.vmap = rockchip_gem_prime_vmap,
	.vunmap	= rockchip_gem_prime_vunmap,
	.mmap = rockchip_drm_gem_object_mmap,
	.vm_ops = &drm_gem_dma_vm_ops,
};

static struct rockchip_gem_object *
	rockchip_gem_alloc_object(struct drm_device *drm, unsigned int size)
{
	struct rockchip_gem_object *rk_obj;
	struct drm_gem_object *obj;

	size = round_up(size, PAGE_SIZE);

	rk_obj = kzalloc(sizeof(*rk_obj), GFP_KERNEL);
	if (!rk_obj)
		return ERR_PTR(-ENOMEM);

	obj = &rk_obj->base;

	obj->funcs = &rockchip_gem_object_funcs;

	drm_gem_object_init(drm, obj, size);

	return rk_obj;
}

struct rockchip_gem_object *
rockchip_gem_create_object(struct drm_device *drm, unsigned int size,
			   bool alloc_kmap)
{
	struct rockchip_gem_object *rk_obj;
	int ret;

	rk_obj = rockchip_gem_alloc_object(drm, size);
	if (IS_ERR(rk_obj))
		return rk_obj;

	ret = rockchip_gem_alloc_buf(rk_obj, alloc_kmap);
	if (ret)
		goto err_free_rk_obj;

	return rk_obj;

err_free_rk_obj:
	rockchip_gem_release_object(rk_obj);
	return ERR_PTR(ret);
}

/*
 * rockchip_gem_free_object - (struct drm_gem_object_funcs)->free
 * callback function
 */
void rockchip_gem_free_object(struct drm_gem_object *obj)
{
	struct drm_device *drm = obj->dev;
	struct rockchip_drm_private *private = drm->dev_private;
	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);

	if (obj->import_attach) {
		if (private->domain) {
			rockchip_gem_iommu_unmap(rk_obj);
		} else {
			dma_unmap_sgtable(drm->dev, rk_obj->sgt,
					  DMA_BIDIRECTIONAL, 0);
		}
		drm_prime_gem_destroy(obj, rk_obj->sgt);
	} else {
		rockchip_gem_free_buf(rk_obj);
	}

	rockchip_gem_release_object(rk_obj);
}

/*
 * rockchip_gem_create_with_handle - allocate an object with the given
 * size and create a gem handle on it
 *
 * returns a struct rockchip_gem_object* on success or ERR_PTR values
 * on failure.
 */
static struct rockchip_gem_object *
rockchip_gem_create_with_handle(struct drm_file *file_priv,
				struct drm_device *drm, unsigned int size,
				unsigned int *handle)
{
	struct rockchip_gem_object *rk_obj;
	struct drm_gem_object *obj;
	bool is_framebuffer;
	int ret;

	is_framebuffer = drm->fb_helper && file_priv == drm->fb_helper->client.file;

	rk_obj = rockchip_gem_create_object(drm, size, is_framebuffer);
	if (IS_ERR(rk_obj))
		return ERR_CAST(rk_obj);

	obj = &rk_obj->base;

	/*
	 * allocate a id of idr table where the obj is registered
	 * and handle has the id what user can see.
	 */
	ret = drm_gem_handle_create(file_priv, obj, handle);
	if (ret)
		goto err_handle_create;

	/* drop reference from allocate - handle holds it now. */
	drm_gem_object_put(obj);

	return rk_obj;

err_handle_create:
	rockchip_gem_free_object(obj);

	return ERR_PTR(ret);
}

/*
 * rockchip_gem_dumb_create - (struct drm_driver)->dumb_create callback
 * function
 *
 * This aligns the pitch and size arguments to the minimum required. wrap
 * this into your own function if you need bigger alignment.
 */
int rockchip_gem_dumb_create(struct drm_file *file_priv,
			     struct drm_device *dev,
			     struct drm_mode_create_dumb *args)
{
	struct rockchip_gem_object *rk_obj;
	int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);

	/*
	 * align to 64 bytes since Mali requires it.
	 */
	args->pitch = ALIGN(min_pitch, 64);
	args->size = args->pitch * args->height;

	rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size,
						 &args->handle);

	return PTR_ERR_OR_ZERO(rk_obj);
}

/*
 * Allocate a sg_table for this GEM object.
 * Note: Both the table's contents, and the sg_table itself must be freed by
 *       the caller.
 * Returns a pointer to the newly allocated sg_table, or an ERR_PTR() error.
 */
struct sg_table *rockchip_gem_prime_get_sg_table(struct drm_gem_object *obj)
{
	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
	struct drm_device *drm = obj->dev;
	struct sg_table *sgt;
	int ret;

	if (rk_obj->pages)
		return drm_prime_pages_to_sg(obj->dev, rk_obj->pages, rk_obj->num_pages);

	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
	if (!sgt)
		return ERR_PTR(-ENOMEM);

	ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr,
				    rk_obj->dma_addr, obj->size,
				    rk_obj->dma_attrs);
	if (ret) {
		DRM_ERROR("failed to allocate sgt, %d\n", ret);
		kfree(sgt);
		return ERR_PTR(ret);
	}

	return sgt;
}

static int
rockchip_gem_iommu_map_sg(struct drm_device *drm,
			  struct dma_buf_attachment *attach,
			  struct sg_table *sg,
			  struct rockchip_gem_object *rk_obj)
{
	rk_obj->sgt = sg;
	return rockchip_gem_iommu_map(rk_obj);
}

static int
rockchip_gem_dma_map_sg(struct drm_device *drm,
			struct dma_buf_attachment *attach,
			struct sg_table *sg,
			struct rockchip_gem_object *rk_obj)
{
	int err = dma_map_sgtable(drm->dev, sg, DMA_BIDIRECTIONAL, 0);
	if (err)
		return err;

	if (drm_prime_get_contiguous_size(sg) < attach->dmabuf->size) {
		DRM_ERROR("failed to map sg_table to contiguous linear address.\n");
		dma_unmap_sgtable(drm->dev, sg, DMA_BIDIRECTIONAL, 0);
		return -EINVAL;
	}

	rk_obj->dma_addr = sg_dma_address(sg->sgl);
	rk_obj->sgt = sg;
	return 0;
}

struct drm_gem_object *
rockchip_gem_prime_import_sg_table(struct drm_device *drm,
				   struct dma_buf_attachment *attach,
				   struct sg_table *sg)
{
	struct rockchip_drm_private *private = drm->dev_private;
	struct rockchip_gem_object *rk_obj;
	int ret;

	rk_obj = rockchip_gem_alloc_object(drm, attach->dmabuf->size);
	if (IS_ERR(rk_obj))
		return ERR_CAST(rk_obj);

	if (private->domain)
		ret = rockchip_gem_iommu_map_sg(drm, attach, sg, rk_obj);
	else
		ret = rockchip_gem_dma_map_sg(drm, attach, sg, rk_obj);

	if (ret < 0) {
		DRM_ERROR("failed to import sg table: %d\n", ret);
		goto err_free_rk_obj;
	}

	return &rk_obj->base;

err_free_rk_obj:
	rockchip_gem_release_object(rk_obj);
	return ERR_PTR(ret);
}

int rockchip_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map)
{
	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);

	if (rk_obj->pages) {
		void *vaddr;

		if (rk_obj->kvaddr)
			vaddr = rk_obj->kvaddr;
		else
			vaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
				     pgprot_writecombine(PAGE_KERNEL));

		if (!vaddr)
			return -ENOMEM;
		iosys_map_set_vaddr(map, vaddr);
		return 0;
	}

	if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING)
		return -ENOMEM;
	iosys_map_set_vaddr(map, rk_obj->kvaddr);

	return 0;
}

void rockchip_gem_prime_vunmap(struct drm_gem_object *obj,
			       struct iosys_map *map)
{
	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);

	if (rk_obj->pages) {
		if (map->vaddr != rk_obj->kvaddr)
			vunmap(map->vaddr);
		return;
	}

	/* Nothing to do if allocated by DMA mapping API. */
}
Commit	Line	Data
9c92ab61	1	// SPDX-License-Identifier: GPL-2.0-only
2048e328 MY	2	/*
	3	* Copyright (C) Fuzhou Rockchip Electronics Co.Ltd
	4	* Author:Mark Yao <mark.yao@rock-chips.com>
2048e328 MY	5	*/
2048e328 MY	6
c2156ccd SR	7	#include <linux/dma-buf.h>
c2156ccd SR	8	#include <linux/iommu.h>
9590a99c	9	#include <linux/vmalloc.h>
c2156ccd	10
2048e328	11	#include <drm/drm.h>
45b64fd9	12	#include <drm/drm_fb_helper.h>
2048e328	13	#include <drm/drm_gem.h>
4a83c26a	14	#include <drm/drm_gem_dma_helper.h>
c2156ccd	15	#include <drm/drm_prime.h>
2048e328	16	#include <drm/drm_vma_manager.h>
6fd0bfe2	17
2048e328 MY	18	#include "rockchip_drm_drv.h"
	19	#include "rockchip_drm_gem.h"
	20
38f993b7 TF	21	static int rockchip_gem_iommu_map(struct rockchip_gem_object *rk_obj)
	22	{
	23	struct drm_device *drm = rk_obj->base.dev;
	24	struct rockchip_drm_private *private = drm->dev_private;
	25	int prot = IOMMU_READ \| IOMMU_WRITE;
	26	ssize_t ret;
	27
	28	mutex_lock(&private->mm_lock);
38f993b7 TF	29	ret = drm_mm_insert_node_generic(&private->mm, &rk_obj->mm,
	30	rk_obj->base.size, PAGE_SIZE,
	31	0, 0);
38f993b7	32	mutex_unlock(&private->mm_lock);
2d7b5637	33
38f993b7 TF	34	if (ret < 0) {
	35	DRM_ERROR("out of I/O virtual memory: %zd\n", ret);
	36	return ret;
	37	}
	38
	39	rk_obj->dma_addr = rk_obj->mm.start;
	40
82c245b5 MS	41	ret = iommu_map_sgtable(private->domain, rk_obj->dma_addr, rk_obj->sgt,
82c245b5 MS	42	prot);
6471da5e	43	if (ret < (ssize_t)rk_obj->base.size) {
38f993b7 TF	44	DRM_ERROR("failed to map buffer: size=%zd request_size=%zd\n",
	45	ret, rk_obj->base.size);
	46	ret = -ENOMEM;
	47	goto err_remove_node;
	48	}
	49
	50	rk_obj->size = ret;
	51
	52	return 0;
	53
	54	err_remove_node:
2d7b5637	55	mutex_lock(&private->mm_lock);
38f993b7	56	drm_mm_remove_node(&rk_obj->mm);
2d7b5637	57	mutex_unlock(&private->mm_lock);
38f993b7 TF	58
	59	return ret;
	60	}
	61
	62	static int rockchip_gem_iommu_unmap(struct rockchip_gem_object *rk_obj)
	63	{
	64	struct drm_device *drm = rk_obj->base.dev;
	65	struct rockchip_drm_private *private = drm->dev_private;
	66
	67	iommu_unmap(private->domain, rk_obj->dma_addr, rk_obj->size);
	68
	69	mutex_lock(&private->mm_lock);
	70
	71	drm_mm_remove_node(&rk_obj->mm);
	72
	73	mutex_unlock(&private->mm_lock);
	74
	75	return 0;
	76	}
	77
	78	static int rockchip_gem_get_pages(struct rockchip_gem_object *rk_obj)
	79	{
	80	struct drm_device *drm = rk_obj->base.dev;
	81	int ret, i;
	82	struct scatterlist *s;
	83
	84	rk_obj->pages = drm_gem_get_pages(&rk_obj->base);
	85	if (IS_ERR(rk_obj->pages))
	86	return PTR_ERR(rk_obj->pages);
	87
	88	rk_obj->num_pages = rk_obj->base.size >> PAGE_SHIFT;
	89
707d561f GH	90	rk_obj->sgt = drm_prime_pages_to_sg(rk_obj->base.dev,
707d561f GH	91	rk_obj->pages, rk_obj->num_pages);
38f993b7 TF	92	if (IS_ERR(rk_obj->sgt)) {
	93	ret = PTR_ERR(rk_obj->sgt);
	94	goto err_put_pages;
	95	}
	96
	97	/*
	98	* Fake up the SG table so that dma_sync_sg_for_device() can be used
	99	* to flush the pages associated with it.
	100	*
	101	* TODO: Replace this by drm_clflush_sg() once it can be implemented
	102	* without relying on symbols that are not exported.
	103	*/
82c245b5	104	for_each_sgtable_sg(rk_obj->sgt, s, i)
38f993b7 TF	105	sg_dma_address(s) = sg_phys(s);
38f993b7 TF	106
82c245b5	107	dma_sync_sgtable_for_device(drm->dev, rk_obj->sgt, DMA_TO_DEVICE);
38f993b7 TF	108
	109	return 0;
	110
	111	err_put_pages:
	112	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, false, false);
	113	return ret;
	114	}
	115
	116	static void rockchip_gem_put_pages(struct rockchip_gem_object *rk_obj)
	117	{
	118	sg_free_table(rk_obj->sgt);
	119	kfree(rk_obj->sgt);
	120	drm_gem_put_pages(&rk_obj->base, rk_obj->pages, true, true);
	121	}
	122
	123	static int rockchip_gem_alloc_iommu(struct rockchip_gem_object *rk_obj,
	124	bool alloc_kmap)
	125	{
	126	int ret;
	127
	128	ret = rockchip_gem_get_pages(rk_obj);
	129	if (ret < 0)
	130	return ret;
	131
	132	ret = rockchip_gem_iommu_map(rk_obj);
	133	if (ret < 0)
	134	goto err_free;
	135
	136	if (alloc_kmap) {
	137	rk_obj->kvaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
	138	pgprot_writecombine(PAGE_KERNEL));
	139	if (!rk_obj->kvaddr) {
	140	DRM_ERROR("failed to vmap() buffer\n");
	141	ret = -ENOMEM;
	142	goto err_unmap;
	143	}
	144	}
	145
	146	return 0;
	147
	148	err_unmap:
	149	rockchip_gem_iommu_unmap(rk_obj);
	150	err_free:
	151	rockchip_gem_put_pages(rk_obj);
	152
	153	return ret;
	154	}
	155
	156	static int rockchip_gem_alloc_dma(struct rockchip_gem_object *rk_obj,
f76c83b5	157	bool alloc_kmap)
2048e328 MY	158	{
	159	struct drm_gem_object *obj = &rk_obj->base;
	160	struct drm_device *drm = obj->dev;
	161
00085f1e	162	rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE;
2048e328	163
f76c83b5	164	if (!alloc_kmap)
00085f1e	165	rk_obj->dma_attrs \|= DMA_ATTR_NO_KERNEL_MAPPING;
f76c83b5	166
2048e328 MY	167	rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size,
2048e328 MY	168	&rk_obj->dma_addr, GFP_KERNEL,
00085f1e	169	rk_obj->dma_attrs);
4b9a90c0	170	if (!rk_obj->kvaddr) {
913bb40a	171	DRM_ERROR("failed to allocate %zu byte dma buffer", obj->size);
4b9a90c0	172	return -ENOMEM;
2048e328 MY	173	}
	174
	175	return 0;
	176	}
	177
38f993b7 TF	178	static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj,
	179	bool alloc_kmap)
	180	{
	181	struct drm_gem_object *obj = &rk_obj->base;
	182	struct drm_device *drm = obj->dev;
	183	struct rockchip_drm_private *private = drm->dev_private;
	184
	185	if (private->domain)
	186	return rockchip_gem_alloc_iommu(rk_obj, alloc_kmap);
	187	else
	188	return rockchip_gem_alloc_dma(rk_obj, alloc_kmap);
	189	}
	190
	191	static void rockchip_gem_free_iommu(struct rockchip_gem_object *rk_obj)
	192	{
	193	vunmap(rk_obj->kvaddr);
	194	rockchip_gem_iommu_unmap(rk_obj);
	195	rockchip_gem_put_pages(rk_obj);
	196	}
	197
	198	static void rockchip_gem_free_dma(struct rockchip_gem_object *rk_obj)
2048e328 MY	199	{
	200	struct drm_gem_object *obj = &rk_obj->base;
	201	struct drm_device *drm = obj->dev;
	202
	203	dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr,
00085f1e	204	rk_obj->dma_attrs);
2048e328 MY	205	}
2048e328 MY	206
38f993b7 TF	207	static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj)
	208	{
	209	if (rk_obj->pages)
	210	rockchip_gem_free_iommu(rk_obj);
	211	else
	212	rockchip_gem_free_dma(rk_obj);
	213	}
41315b79	214
38f993b7 TF	215	static int rockchip_drm_gem_object_mmap_iommu(struct drm_gem_object *obj,
38f993b7 TF	216	struct vm_area_struct *vma)
2048e328	217	{
38f993b7	218	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
2f69b3c8	219	unsigned int count = obj->size >> PAGE_SHIFT;
f2a44dd0	220	unsigned long user_count = vma_pages(vma);
38f993b7 TF	221
	222	if (user_count == 0)
	223	return -ENXIO;
38f993b7	224
2f69b3c8	225	return vm_map_pages(vma, rk_obj->pages, count);
38f993b7 TF	226	}
	227
	228	static int rockchip_drm_gem_object_mmap_dma(struct drm_gem_object *obj,
	229	struct vm_area_struct *vma)
	230	{
2048e328 MY	231	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
2048e328 MY	232	struct drm_device *drm = obj->dev;
2048e328	233
38f993b7 TF	234	return dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr,
	235	obj->size, rk_obj->dma_attrs);
	236	}
	237
	238	static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj,
	239	struct vm_area_struct *vma)
	240	{
	241	int ret;
	242	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
	243
f8b53070 TZ	244	/*
	245	* Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the
	246	* whole buffer from the start.
	247	*/
	248	vma->vm_pgoff = 0;
	249
41315b79	250	/*
38f993b7	251	* We allocated a struct page table for rk_obj, so clear
41315b79 DK	252	* VM_PFNMAP flag that was set by drm_gem_mmap_obj()/drm_gem_mmap().
41315b79 DK	253	*/
1c71222e	254	vm_flags_mod(vma, VM_IO \| VM_DONTEXPAND \| VM_DONTDUMP, VM_PFNMAP);
2048e328	255
f8b53070 TZ	256	vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
	257	vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
	258
38f993b7 TF	259	if (rk_obj->pages)
	260	ret = rockchip_drm_gem_object_mmap_iommu(obj, vma);
	261	else
	262	ret = rockchip_drm_gem_object_mmap_dma(obj, vma);
	263
41315b79 DK	264	return ret;
	265	}
	266
38f993b7 TF	267	static void rockchip_gem_release_object(struct rockchip_gem_object *rk_obj)
	268	{
	269	drm_gem_object_release(&rk_obj->base);
	270	kfree(rk_obj);
	271	}
	272
0d590af3 TZ	273	static const struct drm_gem_object_funcs rockchip_gem_object_funcs = {
	274	.free = rockchip_gem_free_object,
	275	.get_sg_table = rockchip_gem_prime_get_sg_table,
	276	.vmap = rockchip_gem_prime_vmap,
	277	.vunmap = rockchip_gem_prime_vunmap,
f8b53070	278	.mmap = rockchip_drm_gem_object_mmap,
4a83c26a	279	.vm_ops = &drm_gem_dma_vm_ops,
0d590af3 TZ	280	};
0d590af3 TZ	281
c7337670	282	static struct rockchip_gem_object *
6fd0bfe2	283	rockchip_gem_alloc_object(struct drm_device *drm, unsigned int size)
2048e328 MY	284	{
	285	struct rockchip_gem_object *rk_obj;
	286	struct drm_gem_object *obj;
2048e328 MY	287
	288	size = round_up(size, PAGE_SIZE);
	289
	290	rk_obj = kzalloc(sizeof(*rk_obj), GFP_KERNEL);
	291	if (!rk_obj)
	292	return ERR_PTR(-ENOMEM);
	293
	294	obj = &rk_obj->base;
	295
0d590af3 TZ	296	obj->funcs = &rockchip_gem_object_funcs;
0d590af3 TZ	297
38f993b7	298	drm_gem_object_init(drm, obj, size);
2048e328	299
6fd0bfe2 HS	300	return rk_obj;
	301	}
	302
	303	struct rockchip_gem_object *
	304	rockchip_gem_create_object(struct drm_device *drm, unsigned int size,
	305	bool alloc_kmap)
	306	{
	307	struct rockchip_gem_object *rk_obj;
	308	int ret;
	309
	310	rk_obj = rockchip_gem_alloc_object(drm, size);
	311	if (IS_ERR(rk_obj))
	312	return rk_obj;
	313
f76c83b5	314	ret = rockchip_gem_alloc_buf(rk_obj, alloc_kmap);
2048e328 MY	315	if (ret)
	316	goto err_free_rk_obj;
	317
	318	return rk_obj;
	319
	320	err_free_rk_obj:
38f993b7	321	rockchip_gem_release_object(rk_obj);
2048e328 MY	322	return ERR_PTR(ret);
	323	}
	324
	325	/*
0d590af3	326	* rockchip_gem_free_object - (struct drm_gem_object_funcs)->free
fcb1e57f	327	* callback function
2048e328 MY	328	*/
	329	void rockchip_gem_free_object(struct drm_gem_object *obj)
	330	{
6fd0bfe2 HS	331	struct drm_device *drm = obj->dev;
	332	struct rockchip_drm_private *private = drm->dev_private;
	333	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
2048e328	334
6fd0bfe2 HS	335	if (obj->import_attach) {
	336	if (private->domain) {
	337	rockchip_gem_iommu_unmap(rk_obj);
	338	} else {
82c245b5 MS	339	dma_unmap_sgtable(drm->dev, rk_obj->sgt,
82c245b5 MS	340	DMA_BIDIRECTIONAL, 0);
6fd0bfe2 HS	341	}
	342	drm_prime_gem_destroy(obj, rk_obj->sgt);
	343	} else {
	344	rockchip_gem_free_buf(rk_obj);
	345	}
2048e328	346
38f993b7	347	rockchip_gem_release_object(rk_obj);
2048e328 MY	348	}
	349
	350	/*
	351	* rockchip_gem_create_with_handle - allocate an object with the given
	352	* size and create a gem handle on it
	353	*
	354	* returns a struct rockchip_gem_object* on success or ERR_PTR values
	355	* on failure.
	356	*/
	357	static struct rockchip_gem_object *
	358	rockchip_gem_create_with_handle(struct drm_file *file_priv,
	359	struct drm_device *drm, unsigned int size,
	360	unsigned int *handle)
	361	{
	362	struct rockchip_gem_object *rk_obj;
	363	struct drm_gem_object *obj;
ab78c74c	364	bool is_framebuffer;
2048e328 MY	365	int ret;
2048e328 MY	366
ab78c74c JK	367	is_framebuffer = drm->fb_helper && file_priv == drm->fb_helper->client.file;
	368
	369	rk_obj = rockchip_gem_create_object(drm, size, is_framebuffer);
2048e328 MY	370	if (IS_ERR(rk_obj))
	371	return ERR_CAST(rk_obj);
	372
	373	obj = &rk_obj->base;
	374
	375	/*
	376	* allocate a id of idr table where the obj is registered
	377	* and handle has the id what user can see.
	378	*/
	379	ret = drm_gem_handle_create(file_priv, obj, handle);
	380	if (ret)
	381	goto err_handle_create;
	382
	383	/* drop reference from allocate - handle holds it now. */
b430fe08	384	drm_gem_object_put(obj);
2048e328 MY	385
	386	return rk_obj;
	387
	388	err_handle_create:
	389	rockchip_gem_free_object(obj);
	390
	391	return ERR_PTR(ret);
	392	}
	393
2048e328 MY	394	/*
	395	* rockchip_gem_dumb_create - (struct drm_driver)->dumb_create callback
	396	* function
	397	*
	398	* This aligns the pitch and size arguments to the minimum required. wrap
	399	* this into your own function if you need bigger alignment.
	400	*/
	401	int rockchip_gem_dumb_create(struct drm_file *file_priv,
	402	struct drm_device *dev,
	403	struct drm_mode_create_dumb *args)
	404	{
	405	struct rockchip_gem_object *rk_obj;
	406	int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
	407
	408	/*
	409	* align to 64 bytes since Mali requires it.
	410	*/
e3c4abdb MY	411	args->pitch = ALIGN(min_pitch, 64);
e3c4abdb MY	412	args->size = args->pitch * args->height;
2048e328 MY	413
	414	rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size,
	415	&args->handle);
	416
	417	return PTR_ERR_OR_ZERO(rk_obj);
	418	}
	419
	420	/*
	421	* Allocate a sg_table for this GEM object.
	422	* Note: Both the table's contents, and the sg_table itself must be freed by
	423	* the caller.
	424	* Returns a pointer to the newly allocated sg_table, or an ERR_PTR() error.
	425	*/
	426	struct sg_table rockchip_gem_prime_get_sg_table(struct drm_gem_object obj)
	427	{
	428	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
	429	struct drm_device *drm = obj->dev;
	430	struct sg_table *sgt;
	431	int ret;
	432
38f993b7	433	if (rk_obj->pages)
707d561f	434	return drm_prime_pages_to_sg(obj->dev, rk_obj->pages, rk_obj->num_pages);
38f993b7	435
2048e328 MY	436	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
	437	if (!sgt)
	438	return ERR_PTR(-ENOMEM);
	439
	440	ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr,
	441	rk_obj->dma_addr, obj->size,
00085f1e	442	rk_obj->dma_attrs);
2048e328 MY	443	if (ret) {
	444	DRM_ERROR("failed to allocate sgt, %d\n", ret);
	445	kfree(sgt);
	446	return ERR_PTR(ret);
	447	}
	448
	449	return sgt;
	450	}
	451
6fd0bfe2 HS	452	static int
	453	rockchip_gem_iommu_map_sg(struct drm_device *drm,
	454	struct dma_buf_attachment *attach,
	455	struct sg_table *sg,
	456	struct rockchip_gem_object *rk_obj)
	457	{
	458	rk_obj->sgt = sg;
	459	return rockchip_gem_iommu_map(rk_obj);
	460	}
	461
	462	static int
	463	rockchip_gem_dma_map_sg(struct drm_device *drm,
	464	struct dma_buf_attachment *attach,
	465	struct sg_table *sg,
	466	struct rockchip_gem_object *rk_obj)
	467	{
82c245b5 MS	468	int err = dma_map_sgtable(drm->dev, sg, DMA_BIDIRECTIONAL, 0);
	469	if (err)
	470	return err;
6fd0bfe2	471
6a8f15c6	472	if (drm_prime_get_contiguous_size(sg) < attach->dmabuf->size) {
6fd0bfe2	473	DRM_ERROR("failed to map sg_table to contiguous linear address.\n");
82c245b5	474	dma_unmap_sgtable(drm->dev, sg, DMA_BIDIRECTIONAL, 0);
6fd0bfe2 HS	475	return -EINVAL;
	476	}
	477
	478	rk_obj->dma_addr = sg_dma_address(sg->sgl);
	479	rk_obj->sgt = sg;
	480	return 0;
	481	}
	482
	483	struct drm_gem_object *
	484	rockchip_gem_prime_import_sg_table(struct drm_device *drm,
	485	struct dma_buf_attachment *attach,
	486	struct sg_table *sg)
	487	{
	488	struct rockchip_drm_private *private = drm->dev_private;
	489	struct rockchip_gem_object *rk_obj;
	490	int ret;
	491
	492	rk_obj = rockchip_gem_alloc_object(drm, attach->dmabuf->size);
	493	if (IS_ERR(rk_obj))
	494	return ERR_CAST(rk_obj);
	495
	496	if (private->domain)
	497	ret = rockchip_gem_iommu_map_sg(drm, attach, sg, rk_obj);
	498	else
	499	ret = rockchip_gem_dma_map_sg(drm, attach, sg, rk_obj);
	500
	501	if (ret < 0) {
	502	DRM_ERROR("failed to import sg table: %d\n", ret);
	503	goto err_free_rk_obj;
	504	}
	505
	506	return &rk_obj->base;
	507
	508	err_free_rk_obj:
	509	rockchip_gem_release_object(rk_obj);
	510	return ERR_PTR(ret);
	511	}
	512
7938f421	513	int rockchip_gem_prime_vmap(struct drm_gem_object obj, struct iosys_map map)
2048e328 MY	514	{
	515	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
	516
49a3f51d	517	if (rk_obj->pages) {
0020d4cf JK	518	void *vaddr;
	519
	520	if (rk_obj->kvaddr)
	521	vaddr = rk_obj->kvaddr;
	522	else
	523	vaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP,
	524	pgprot_writecombine(PAGE_KERNEL));
	525
49a3f51d TZ	526	if (!vaddr)
49a3f51d TZ	527	return -ENOMEM;
7938f421	528	iosys_map_set_vaddr(map, vaddr);
49a3f51d TZ	529	return 0;
49a3f51d TZ	530	}
38f993b7	531
00085f1e	532	if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING)
49a3f51d	533	return -ENOMEM;
7938f421	534	iosys_map_set_vaddr(map, rk_obj->kvaddr);
f76c83b5	535
49a3f51d	536	return 0;
2048e328 MY	537	}
2048e328 MY	538
7938f421 LDM	539	void rockchip_gem_prime_vunmap(struct drm_gem_object *obj,
7938f421 LDM	540	struct iosys_map *map)
2048e328	541	{
38f993b7 TF	542	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
	543
	544	if (rk_obj->pages) {
0020d4cf JK	545	if (map->vaddr != rk_obj->kvaddr)
0020d4cf JK	546	vunmap(map->vaddr);
38f993b7 TF	547	return;
	548	}
	549
	550	/* Nothing to do if allocated by DMA mapping API. */
2048e328	551	}