[linux-2.6-block.git] / drivers / tee / tee_shm.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2015-2016, Linaro Limited
 */
#include <linux/device.h>
#include <linux/dma-buf.h>
#include <linux/fdtable.h>
#include <linux/idr.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/tee_drv.h>
#include "tee_private.h"

static void tee_shm_release(struct tee_shm *shm)
{
	struct tee_device *teedev = shm->teedev;

	mutex_lock(&teedev->mutex);
	idr_remove(&teedev->idr, shm->id);
	if (shm->ctx)
		list_del(&shm->link);
	mutex_unlock(&teedev->mutex);

	if (shm->flags & TEE_SHM_POOL) {
		struct tee_shm_pool_mgr *poolm;

		if (shm->flags & TEE_SHM_DMA_BUF)
			poolm = teedev->pool->dma_buf_mgr;
		else
			poolm = teedev->pool->private_mgr;

		poolm->ops->free(poolm, shm);
	} else if (shm->flags & TEE_SHM_REGISTER) {
		size_t n;
		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);

		if (rc)
			dev_err(teedev->dev.parent,
				"unregister shm %p failed: %d", shm, rc);

		for (n = 0; n < shm->num_pages; n++)
			put_page(shm->pages[n]);

		kfree(shm->pages);
	}

	if (shm->ctx)
		teedev_ctx_put(shm->ctx);

	kfree(shm);

	tee_device_put(teedev);
}

static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment
			*attach, enum dma_data_direction dir)
{
	return NULL;
}

static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach,
				     struct sg_table *table,
				     enum dma_data_direction dir)
{
}

static void tee_shm_op_release(struct dma_buf *dmabuf)
{
	struct tee_shm *shm = dmabuf->priv;

	tee_shm_release(shm);
}

static void *tee_shm_op_map(struct dma_buf *dmabuf, unsigned long pgnum)
{
	return NULL;
}

static int tee_shm_op_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
	struct tee_shm *shm = dmabuf->priv;
	size_t size = vma->vm_end - vma->vm_start;

	/* Refuse sharing shared memory provided by application */
	if (shm->flags & TEE_SHM_REGISTER)
		return -EINVAL;

	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
			       size, vma->vm_page_prot);
}

static const struct dma_buf_ops tee_shm_dma_buf_ops = {
	.map_dma_buf = tee_shm_op_map_dma_buf,
	.unmap_dma_buf = tee_shm_op_unmap_dma_buf,
	.release = tee_shm_op_release,
	.map = tee_shm_op_map,
	.mmap = tee_shm_op_mmap,
};

static struct tee_shm *__tee_shm_alloc(struct tee_context *ctx,
				       struct tee_device *teedev,
				       size_t size, u32 flags)
{
	struct tee_shm_pool_mgr *poolm = NULL;
	struct tee_shm *shm;
	void *ret;
	int rc;

	if (ctx && ctx->teedev != teedev) {
		dev_err(teedev->dev.parent, "ctx and teedev mismatch\n");
		return ERR_PTR(-EINVAL);
	}

	if (!(flags & TEE_SHM_MAPPED)) {
		dev_err(teedev->dev.parent,
			"only mapped allocations supported\n");
		return ERR_PTR(-EINVAL);
	}

	if ((flags & ~(TEE_SHM_MAPPED | TEE_SHM_DMA_BUF))) {
		dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);
		return ERR_PTR(-EINVAL);
	}

	if (!tee_device_get(teedev))
		return ERR_PTR(-EINVAL);

	if (!teedev->pool) {
		/* teedev has been detached from driver */
		ret = ERR_PTR(-EINVAL);
		goto err_dev_put;
	}

	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	if (!shm) {
		ret = ERR_PTR(-ENOMEM);
		goto err_dev_put;
	}

	shm->flags = flags | TEE_SHM_POOL;
	shm->teedev = teedev;
	shm->ctx = ctx;
	if (flags & TEE_SHM_DMA_BUF)
		poolm = teedev->pool->dma_buf_mgr;
	else
		poolm = teedev->pool->private_mgr;

	rc = poolm->ops->alloc(poolm, shm, size);
	if (rc) {
		ret = ERR_PTR(rc);
		goto err_kfree;
	}

	mutex_lock(&teedev->mutex);
	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	mutex_unlock(&teedev->mutex);
	if (shm->id < 0) {
		ret = ERR_PTR(shm->id);
		goto err_pool_free;
	}

	if (flags & TEE_SHM_DMA_BUF) {
		DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

		exp_info.ops = &tee_shm_dma_buf_ops;
		exp_info.size = shm->size;
		exp_info.flags = O_RDWR;
		exp_info.priv = shm;

		shm->dmabuf = dma_buf_export(&exp_info);
		if (IS_ERR(shm->dmabuf)) {
			ret = ERR_CAST(shm->dmabuf);
			goto err_rem;
		}
	}

	if (ctx) {
		teedev_ctx_get(ctx);
		mutex_lock(&teedev->mutex);
		list_add_tail(&shm->link, &ctx->list_shm);
		mutex_unlock(&teedev->mutex);
	}

	return shm;
err_rem:
	mutex_lock(&teedev->mutex);
	idr_remove(&teedev->idr, shm->id);
	mutex_unlock(&teedev->mutex);
err_pool_free:
	poolm->ops->free(poolm, shm);
err_kfree:
	kfree(shm);
err_dev_put:
	tee_device_put(teedev);
	return ret;
}

/**
 * tee_shm_alloc() - Allocate shared memory
 * @ctx:	Context that allocates the shared memory
 * @size:	Requested size of shared memory
 * @flags:	Flags setting properties for the requested shared memory.
 *
 * Memory allocated as global shared memory is automatically freed when the
 * TEE file pointer is closed. The @flags field uses the bits defined by
 * TEE_SHM_* in <linux/tee_drv.h>. TEE_SHM_MAPPED must currently always be
 * set. If TEE_SHM_DMA_BUF global shared memory will be allocated and
 * associated with a dma-buf handle, else driver private memory.
 */
struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
{
	return __tee_shm_alloc(ctx, ctx->teedev, size, flags);
}
EXPORT_SYMBOL_GPL(tee_shm_alloc);

struct tee_shm *tee_shm_priv_alloc(struct tee_device *teedev, size_t size)
{
	return __tee_shm_alloc(NULL, teedev, size, TEE_SHM_MAPPED);
}
EXPORT_SYMBOL_GPL(tee_shm_priv_alloc);

struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
				 size_t length, u32 flags)
{
	struct tee_device *teedev = ctx->teedev;
	const u32 req_flags = TEE_SHM_DMA_BUF | TEE_SHM_USER_MAPPED;
	struct tee_shm *shm;
	void *ret;
	int rc;
	int num_pages;
	unsigned long start;

	if (flags != req_flags)
		return ERR_PTR(-ENOTSUPP);

	if (!tee_device_get(teedev))
		return ERR_PTR(-EINVAL);

	if (!teedev->desc->ops->shm_register ||
	    !teedev->desc->ops->shm_unregister) {
		tee_device_put(teedev);
		return ERR_PTR(-ENOTSUPP);
	}

	teedev_ctx_get(ctx);

	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	if (!shm) {
		ret = ERR_PTR(-ENOMEM);
		goto err;
	}

	shm->flags = flags | TEE_SHM_REGISTER;
	shm->teedev = teedev;
	shm->ctx = ctx;
	shm->id = -1;
	start = rounddown(addr, PAGE_SIZE);
	shm->offset = addr - start;
	shm->size = length;
	num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
	if (!shm->pages) {
		ret = ERR_PTR(-ENOMEM);
		goto err;
	}

	rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);
	if (rc > 0)
		shm->num_pages = rc;
	if (rc != num_pages) {
		if (rc >= 0)
			rc = -ENOMEM;
		ret = ERR_PTR(rc);
		goto err;
	}

	mutex_lock(&teedev->mutex);
	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	mutex_unlock(&teedev->mutex);

	if (shm->id < 0) {
		ret = ERR_PTR(shm->id);
		goto err;
	}

	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
					     shm->num_pages, start);
	if (rc) {
		ret = ERR_PTR(rc);
		goto err;
	}

	if (flags & TEE_SHM_DMA_BUF) {
		DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

		exp_info.ops = &tee_shm_dma_buf_ops;
		exp_info.size = shm->size;
		exp_info.flags = O_RDWR;
		exp_info.priv = shm;

		shm->dmabuf = dma_buf_export(&exp_info);
		if (IS_ERR(shm->dmabuf)) {
			ret = ERR_CAST(shm->dmabuf);
			teedev->desc->ops->shm_unregister(ctx, shm);
			goto err;
		}
	}

	mutex_lock(&teedev->mutex);
	list_add_tail(&shm->link, &ctx->list_shm);
	mutex_unlock(&teedev->mutex);

	return shm;
err:
	if (shm) {
		size_t n;

		if (shm->id >= 0) {
			mutex_lock(&teedev->mutex);
			idr_remove(&teedev->idr, shm->id);
			mutex_unlock(&teedev->mutex);
		}
		if (shm->pages) {
			for (n = 0; n < shm->num_pages; n++)
				put_page(shm->pages[n]);
			kfree(shm->pages);
		}
	}
	kfree(shm);
	teedev_ctx_put(ctx);
	tee_device_put(teedev);
	return ret;
}
EXPORT_SYMBOL_GPL(tee_shm_register);

/**
 * tee_shm_get_fd() - Increase reference count and return file descriptor
 * @shm:	Shared memory handle
 * @returns user space file descriptor to shared memory
 */
int tee_shm_get_fd(struct tee_shm *shm)
{
	int fd;

	if (!(shm->flags & TEE_SHM_DMA_BUF))
		return -EINVAL;

	get_dma_buf(shm->dmabuf);
	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);
	if (fd < 0)
		dma_buf_put(shm->dmabuf);
	return fd;
}

/**
 * tee_shm_free() - Free shared memory
 * @shm:	Handle to shared memory to free
 */
void tee_shm_free(struct tee_shm *shm)
{
	/*
	 * dma_buf_put() decreases the dmabuf reference counter and will
	 * call tee_shm_release() when the last reference is gone.
	 *
	 * In the case of driver private memory we call tee_shm_release
	 * directly instead as it doesn't have a reference counter.
	 */
	if (shm->flags & TEE_SHM_DMA_BUF)
		dma_buf_put(shm->dmabuf);
	else
		tee_shm_release(shm);
}
EXPORT_SYMBOL_GPL(tee_shm_free);

/**
 * tee_shm_va2pa() - Get physical address of a virtual address
 * @shm:	Shared memory handle
 * @va:		Virtual address to tranlsate
 * @pa:		Returned physical address
 * @returns 0 on success and < 0 on failure
 */
int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return -EINVAL;
	/* Check that we're in the range of the shm */
	if ((char *)va < (char *)shm->kaddr)
		return -EINVAL;
	if ((char *)va >= ((char *)shm->kaddr + shm->size))
		return -EINVAL;

	return tee_shm_get_pa(
			shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);
}
EXPORT_SYMBOL_GPL(tee_shm_va2pa);

/**
 * tee_shm_pa2va() - Get virtual address of a physical address
 * @shm:	Shared memory handle
 * @pa:		Physical address to tranlsate
 * @va:		Returned virtual address
 * @returns 0 on success and < 0 on failure
 */
int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return -EINVAL;
	/* Check that we're in the range of the shm */
	if (pa < shm->paddr)
		return -EINVAL;
	if (pa >= (shm->paddr + shm->size))
		return -EINVAL;

	if (va) {
		void *v = tee_shm_get_va(shm, pa - shm->paddr);

		if (IS_ERR(v))
			return PTR_ERR(v);
		*va = v;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(tee_shm_pa2va);

/**
 * tee_shm_get_va() - Get virtual address of a shared memory plus an offset
 * @shm:	Shared memory handle
 * @offs:	Offset from start of this shared memory
 * @returns virtual address of the shared memory + offs if offs is within
 *	the bounds of this shared memory, else an ERR_PTR
 */
void *tee_shm_get_va(struct tee_shm *shm, size_t offs)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return ERR_PTR(-EINVAL);
	if (offs >= shm->size)
		return ERR_PTR(-EINVAL);
	return (char *)shm->kaddr + offs;
}
EXPORT_SYMBOL_GPL(tee_shm_get_va);

/**
 * tee_shm_get_pa() - Get physical address of a shared memory plus an offset
 * @shm:	Shared memory handle
 * @offs:	Offset from start of this shared memory
 * @pa:		Physical address to return
 * @returns 0 if offs is within the bounds of this shared memory, else an
 *	error code.
 */
int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)
{
	if (offs >= shm->size)
		return -EINVAL;
	if (pa)
		*pa = shm->paddr + offs;
	return 0;
}
EXPORT_SYMBOL_GPL(tee_shm_get_pa);

/**
 * tee_shm_get_from_id() - Find shared memory object and increase reference
 * count
 * @ctx:	Context owning the shared memory
 * @id:		Id of shared memory object
 * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
 */
struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id)
{
	struct tee_device *teedev;
	struct tee_shm *shm;

	if (!ctx)
		return ERR_PTR(-EINVAL);

	teedev = ctx->teedev;
	mutex_lock(&teedev->mutex);
	shm = idr_find(&teedev->idr, id);
	if (!shm || shm->ctx != ctx)
		shm = ERR_PTR(-EINVAL);
	else if (shm->flags & TEE_SHM_DMA_BUF)
		get_dma_buf(shm->dmabuf);
	mutex_unlock(&teedev->mutex);
	return shm;
}
EXPORT_SYMBOL_GPL(tee_shm_get_from_id);

/**
 * tee_shm_put() - Decrease reference count on a shared memory handle
 * @shm:	Shared memory handle
 */
void tee_shm_put(struct tee_shm *shm)
{
	if (shm->flags & TEE_SHM_DMA_BUF)
		dma_buf_put(shm->dmabuf);
}
EXPORT_SYMBOL_GPL(tee_shm_put);
Commit	Line	Data
9c92ab61	1	// SPDX-License-Identifier: GPL-2.0-only
967c9cca JW	2	/*
967c9cca JW	3	* Copyright (c) 2015-2016, Linaro Limited
967c9cca JW	4	*/
	5	#include <linux/device.h>
	6	#include <linux/dma-buf.h>
	7	#include <linux/fdtable.h>
	8	#include <linux/idr.h>
	9	#include <linux/sched.h>
	10	#include <linux/slab.h>
	11	#include <linux/tee_drv.h>
	12	#include "tee_private.h"
	13
	14	static void tee_shm_release(struct tee_shm *shm)
	15	{
	16	struct tee_device *teedev = shm->teedev;
967c9cca JW	17
	18	mutex_lock(&teedev->mutex);
	19	idr_remove(&teedev->idr, shm->id);
	20	if (shm->ctx)
	21	list_del(&shm->link);
	22	mutex_unlock(&teedev->mutex);
	23
033ddf12 JW	24	if (shm->flags & TEE_SHM_POOL) {
	25	struct tee_shm_pool_mgr *poolm;
	26
	27	if (shm->flags & TEE_SHM_DMA_BUF)
	28	poolm = teedev->pool->dma_buf_mgr;
	29	else
	30	poolm = teedev->pool->private_mgr;
	31
	32	poolm->ops->free(poolm, shm);
	33	} else if (shm->flags & TEE_SHM_REGISTER) {
	34	size_t n;
	35	int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);
	36
	37	if (rc)
	38	dev_err(teedev->dev.parent,
	39	"unregister shm %p failed: %d", shm, rc);
	40
	41	for (n = 0; n < shm->num_pages; n++)
	42	put_page(shm->pages[n]);
	43
	44	kfree(shm->pages);
	45	}
967c9cca	46
217e0250 VB	47	if (shm->ctx)
	48	teedev_ctx_put(shm->ctx);
	49
967c9cca JW	50	kfree(shm);
	51
	52	tee_device_put(teedev);
	53	}
	54
	55	static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment
	56	*attach, enum dma_data_direction dir)
	57	{
	58	return NULL;
	59	}
	60
	61	static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach,
	62	struct sg_table *table,
	63	enum dma_data_direction dir)
	64	{
	65	}
	66
	67	static void tee_shm_op_release(struct dma_buf *dmabuf)
	68	{
	69	struct tee_shm *shm = dmabuf->priv;
	70
	71	tee_shm_release(shm);
	72	}
	73
a2d9214c	74	static void tee_shm_op_map(struct dma_buf dmabuf, unsigned long pgnum)
967c9cca JW	75	{
	76	return NULL;
	77	}
	78
	79	static int tee_shm_op_mmap(struct dma_buf dmabuf, struct vm_area_struct vma)
	80	{
	81	struct tee_shm *shm = dmabuf->priv;
	82	size_t size = vma->vm_end - vma->vm_start;
	83
033ddf12 JW	84	/* Refuse sharing shared memory provided by application */
	85	if (shm->flags & TEE_SHM_REGISTER)
	86	return -EINVAL;
	87
967c9cca JW	88	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
	89	size, vma->vm_page_prot);
	90	}
	91
53e3ca5c	92	static const struct dma_buf_ops tee_shm_dma_buf_ops = {
967c9cca JW	93	.map_dma_buf = tee_shm_op_map_dma_buf,
	94	.unmap_dma_buf = tee_shm_op_unmap_dma_buf,
	95	.release = tee_shm_op_release,
a2d9214c	96	.map = tee_shm_op_map,
967c9cca JW	97	.mmap = tee_shm_op_mmap,
	98	};
	99
80ec6f5d CIK	100	static struct tee_shm __tee_shm_alloc(struct tee_context ctx,
	101	struct tee_device *teedev,
	102	size_t size, u32 flags)
967c9cca	103	{
967c9cca JW	104	struct tee_shm_pool_mgr *poolm = NULL;
	105	struct tee_shm *shm;
	106	void *ret;
	107	int rc;
	108
033ddf12 JW	109	if (ctx && ctx->teedev != teedev) {
	110	dev_err(teedev->dev.parent, "ctx and teedev mismatch\n");
	111	return ERR_PTR(-EINVAL);
	112	}
	113
967c9cca JW	114	if (!(flags & TEE_SHM_MAPPED)) {
	115	dev_err(teedev->dev.parent,
	116	"only mapped allocations supported\n");
	117	return ERR_PTR(-EINVAL);
	118	}
	119
	120	if ((flags & ~(TEE_SHM_MAPPED \| TEE_SHM_DMA_BUF))) {
	121	dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);
	122	return ERR_PTR(-EINVAL);
	123	}
	124
	125	if (!tee_device_get(teedev))
	126	return ERR_PTR(-EINVAL);
	127
	128	if (!teedev->pool) {
	129	/* teedev has been detached from driver */
	130	ret = ERR_PTR(-EINVAL);
	131	goto err_dev_put;
	132	}
	133
	134	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	135	if (!shm) {
	136	ret = ERR_PTR(-ENOMEM);
	137	goto err_dev_put;
	138	}
	139
033ddf12	140	shm->flags = flags \| TEE_SHM_POOL;
967c9cca JW	141	shm->teedev = teedev;
	142	shm->ctx = ctx;
	143	if (flags & TEE_SHM_DMA_BUF)
e2aca5d8	144	poolm = teedev->pool->dma_buf_mgr;
967c9cca	145	else
e2aca5d8	146	poolm = teedev->pool->private_mgr;
967c9cca JW	147
	148	rc = poolm->ops->alloc(poolm, shm, size);
	149	if (rc) {
	150	ret = ERR_PTR(rc);
	151	goto err_kfree;
	152	}
	153
	154	mutex_lock(&teedev->mutex);
	155	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	156	mutex_unlock(&teedev->mutex);
	157	if (shm->id < 0) {
	158	ret = ERR_PTR(shm->id);
	159	goto err_pool_free;
	160	}
	161
	162	if (flags & TEE_SHM_DMA_BUF) {
	163	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	164
	165	exp_info.ops = &tee_shm_dma_buf_ops;
	166	exp_info.size = shm->size;
	167	exp_info.flags = O_RDWR;
	168	exp_info.priv = shm;
	169
	170	shm->dmabuf = dma_buf_export(&exp_info);
	171	if (IS_ERR(shm->dmabuf)) {
	172	ret = ERR_CAST(shm->dmabuf);
	173	goto err_rem;
	174	}
	175	}
033ddf12 JW	176
033ddf12 JW	177	if (ctx) {
217e0250	178	teedev_ctx_get(ctx);
033ddf12 JW	179	mutex_lock(&teedev->mutex);
	180	list_add_tail(&shm->link, &ctx->list_shm);
	181	mutex_unlock(&teedev->mutex);
	182	}
967c9cca JW	183
	184	return shm;
	185	err_rem:
	186	mutex_lock(&teedev->mutex);
	187	idr_remove(&teedev->idr, shm->id);
	188	mutex_unlock(&teedev->mutex);
	189	err_pool_free:
	190	poolm->ops->free(poolm, shm);
	191	err_kfree:
	192	kfree(shm);
	193	err_dev_put:
	194	tee_device_put(teedev);
	195	return ret;
	196	}
033ddf12 JW	197
	198	/**
	199	* tee_shm_alloc() - Allocate shared memory
	200	* @ctx: Context that allocates the shared memory
	201	* @size: Requested size of shared memory
	202	* @flags: Flags setting properties for the requested shared memory.
	203	*
	204	* Memory allocated as global shared memory is automatically freed when the
	205	* TEE file pointer is closed. The @flags field uses the bits defined by
	206	* TEE_SHM_* in <linux/tee_drv.h>. TEE_SHM_MAPPED must currently always be
	207	* set. If TEE_SHM_DMA_BUF global shared memory will be allocated and
	208	* associated with a dma-buf handle, else driver private memory.
	209	*/
	210	struct tee_shm tee_shm_alloc(struct tee_context ctx, size_t size, u32 flags)
	211	{
	212	return __tee_shm_alloc(ctx, ctx->teedev, size, flags);
	213	}
967c9cca JW	214	EXPORT_SYMBOL_GPL(tee_shm_alloc);
967c9cca JW	215
033ddf12 JW	216	struct tee_shm tee_shm_priv_alloc(struct tee_device teedev, size_t size)
	217	{
	218	return __tee_shm_alloc(NULL, teedev, size, TEE_SHM_MAPPED);
	219	}
	220	EXPORT_SYMBOL_GPL(tee_shm_priv_alloc);
	221
	222	struct tee_shm tee_shm_register(struct tee_context ctx, unsigned long addr,
	223	size_t length, u32 flags)
	224	{
	225	struct tee_device *teedev = ctx->teedev;
	226	const u32 req_flags = TEE_SHM_DMA_BUF \| TEE_SHM_USER_MAPPED;
	227	struct tee_shm *shm;
	228	void *ret;
	229	int rc;
	230	int num_pages;
	231	unsigned long start;
	232
	233	if (flags != req_flags)
	234	return ERR_PTR(-ENOTSUPP);
	235
	236	if (!tee_device_get(teedev))
	237	return ERR_PTR(-EINVAL);
	238
	239	if (!teedev->desc->ops->shm_register \|\|
	240	!teedev->desc->ops->shm_unregister) {
	241	tee_device_put(teedev);
	242	return ERR_PTR(-ENOTSUPP);
	243	}
	244
217e0250 VB	245	teedev_ctx_get(ctx);
217e0250 VB	246
033ddf12 JW	247	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	248	if (!shm) {
	249	ret = ERR_PTR(-ENOMEM);
	250	goto err;
	251	}
	252
	253	shm->flags = flags \| TEE_SHM_REGISTER;
	254	shm->teedev = teedev;
	255	shm->ctx = ctx;
	256	shm->id = -1;
	257	start = rounddown(addr, PAGE_SIZE);
	258	shm->offset = addr - start;
	259	shm->size = length;
	260	num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
	261	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
	262	if (!shm->pages) {
	263	ret = ERR_PTR(-ENOMEM);
	264	goto err;
	265	}
	266
73b0140b	267	rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);
033ddf12 JW	268	if (rc > 0)
	269	shm->num_pages = rc;
	270	if (rc != num_pages) {
2490cdf6	271	if (rc >= 0)
033ddf12 JW	272	rc = -ENOMEM;
	273	ret = ERR_PTR(rc);
	274	goto err;
	275	}
	276
	277	mutex_lock(&teedev->mutex);
	278	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	279	mutex_unlock(&teedev->mutex);
	280
	281	if (shm->id < 0) {
	282	ret = ERR_PTR(shm->id);
	283	goto err;
	284	}
	285
	286	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
95ffe4ca	287	shm->num_pages, start);
033ddf12 JW	288	if (rc) {
	289	ret = ERR_PTR(rc);
	290	goto err;
	291	}
	292
	293	if (flags & TEE_SHM_DMA_BUF) {
	294	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	295
	296	exp_info.ops = &tee_shm_dma_buf_ops;
	297	exp_info.size = shm->size;
	298	exp_info.flags = O_RDWR;
	299	exp_info.priv = shm;
	300
	301	shm->dmabuf = dma_buf_export(&exp_info);
	302	if (IS_ERR(shm->dmabuf)) {
	303	ret = ERR_CAST(shm->dmabuf);
	304	teedev->desc->ops->shm_unregister(ctx, shm);
	305	goto err;
	306	}
	307	}
	308
	309	mutex_lock(&teedev->mutex);
	310	list_add_tail(&shm->link, &ctx->list_shm);
	311	mutex_unlock(&teedev->mutex);
	312
	313	return shm;
	314	err:
	315	if (shm) {
	316	size_t n;
	317
	318	if (shm->id >= 0) {
	319	mutex_lock(&teedev->mutex);
	320	idr_remove(&teedev->idr, shm->id);
	321	mutex_unlock(&teedev->mutex);
	322	}
c94f31b5 CIK	323	if (shm->pages) {
	324	for (n = 0; n < shm->num_pages; n++)
	325	put_page(shm->pages[n]);
	326	kfree(shm->pages);
	327	}
033ddf12 JW	328	}
033ddf12 JW	329	kfree(shm);
217e0250	330	teedev_ctx_put(ctx);
033ddf12 JW	331	tee_device_put(teedev);
	332	return ret;
	333	}
	334	EXPORT_SYMBOL_GPL(tee_shm_register);
	335
967c9cca JW	336	/**
	337	* tee_shm_get_fd() - Increase reference count and return file descriptor
	338	* @shm: Shared memory handle
	339	* @returns user space file descriptor to shared memory
	340	*/
	341	int tee_shm_get_fd(struct tee_shm *shm)
	342	{
967c9cca JW	343	int fd;
967c9cca JW	344
033ddf12	345	if (!(shm->flags & TEE_SHM_DMA_BUF))
967c9cca JW	346	return -EINVAL;
967c9cca JW	347
bb765d1c	348	get_dma_buf(shm->dmabuf);
967c9cca	349	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);
bb765d1c JH	350	if (fd < 0)
bb765d1c JH	351	dma_buf_put(shm->dmabuf);
967c9cca JW	352	return fd;
	353	}
	354
	355	/**
	356	* tee_shm_free() - Free shared memory
	357	* @shm: Handle to shared memory to free
	358	*/
	359	void tee_shm_free(struct tee_shm *shm)
	360	{
	361	/*
	362	* dma_buf_put() decreases the dmabuf reference counter and will
	363	* call tee_shm_release() when the last reference is gone.
	364	*
	365	* In the case of driver private memory we call tee_shm_release
	366	* directly instead as it doesn't have a reference counter.
	367	*/
	368	if (shm->flags & TEE_SHM_DMA_BUF)
	369	dma_buf_put(shm->dmabuf);
	370	else
	371	tee_shm_release(shm);
	372	}
	373	EXPORT_SYMBOL_GPL(tee_shm_free);
	374
	375	/**
	376	* tee_shm_va2pa() - Get physical address of a virtual address
	377	* @shm: Shared memory handle
	378	* @va: Virtual address to tranlsate
	379	* @pa: Returned physical address
	380	* @returns 0 on success and < 0 on failure
	381	*/
	382	int tee_shm_va2pa(struct tee_shm shm, void va, phys_addr_t *pa)
	383	{
033ddf12 JW	384	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	385	return -EINVAL;
967c9cca JW	386	/* Check that we're in the range of the shm */
	387	if ((char )va < (char )shm->kaddr)
	388	return -EINVAL;
	389	if ((char )va >= ((char )shm->kaddr + shm->size))
	390	return -EINVAL;
	391
	392	return tee_shm_get_pa(
	393	shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);
	394	}
	395	EXPORT_SYMBOL_GPL(tee_shm_va2pa);
	396
	397	/**
	398	* tee_shm_pa2va() - Get virtual address of a physical address
	399	* @shm: Shared memory handle
	400	* @pa: Physical address to tranlsate
	401	* @va: Returned virtual address
	402	* @returns 0 on success and < 0 on failure
	403	*/
	404	int tee_shm_pa2va(struct tee_shm shm, phys_addr_t pa, void *va)
	405	{
033ddf12 JW	406	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	407	return -EINVAL;
967c9cca JW	408	/* Check that we're in the range of the shm */
	409	if (pa < shm->paddr)
	410	return -EINVAL;
	411	if (pa >= (shm->paddr + shm->size))
	412	return -EINVAL;
	413
	414	if (va) {
	415	void *v = tee_shm_get_va(shm, pa - shm->paddr);
	416
	417	if (IS_ERR(v))
	418	return PTR_ERR(v);
	419	*va = v;
	420	}
	421	return 0;
	422	}
	423	EXPORT_SYMBOL_GPL(tee_shm_pa2va);
	424
	425	/**
	426	* tee_shm_get_va() - Get virtual address of a shared memory plus an offset
	427	* @shm: Shared memory handle
	428	* @offs: Offset from start of this shared memory
	429	* @returns virtual address of the shared memory + offs if offs is within
	430	* the bounds of this shared memory, else an ERR_PTR
	431	*/
	432	void tee_shm_get_va(struct tee_shm shm, size_t offs)
	433	{
033ddf12 JW	434	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	435	return ERR_PTR(-EINVAL);
967c9cca JW	436	if (offs >= shm->size)
	437	return ERR_PTR(-EINVAL);
	438	return (char *)shm->kaddr + offs;
	439	}
	440	EXPORT_SYMBOL_GPL(tee_shm_get_va);
	441
	442	/**
	443	* tee_shm_get_pa() - Get physical address of a shared memory plus an offset
	444	* @shm: Shared memory handle
	445	* @offs: Offset from start of this shared memory
	446	* @pa: Physical address to return
	447	* @returns 0 if offs is within the bounds of this shared memory, else an
	448	* error code.
	449	*/
	450	int tee_shm_get_pa(struct tee_shm shm, size_t offs, phys_addr_t pa)
	451	{
	452	if (offs >= shm->size)
	453	return -EINVAL;
	454	if (pa)
	455	*pa = shm->paddr + offs;
	456	return 0;
	457	}
	458	EXPORT_SYMBOL_GPL(tee_shm_get_pa);
	459
	460	/**
	461	* tee_shm_get_from_id() - Find shared memory object and increase reference
	462	* count
	463	* @ctx: Context owning the shared memory
	464	* @id: Id of shared memory object
	465	* @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
	466	*/
	467	struct tee_shm tee_shm_get_from_id(struct tee_context ctx, int id)
	468	{
	469	struct tee_device *teedev;
	470	struct tee_shm *shm;
	471
	472	if (!ctx)
	473	return ERR_PTR(-EINVAL);
	474
	475	teedev = ctx->teedev;
	476	mutex_lock(&teedev->mutex);
	477	shm = idr_find(&teedev->idr, id);
	478	if (!shm \|\| shm->ctx != ctx)
	479	shm = ERR_PTR(-EINVAL);
	480	else if (shm->flags & TEE_SHM_DMA_BUF)
	481	get_dma_buf(shm->dmabuf);
	482	mutex_unlock(&teedev->mutex);
	483	return shm;
	484	}
	485	EXPORT_SYMBOL_GPL(tee_shm_get_from_id);
	486
967c9cca JW	487	/**
	488	* tee_shm_put() - Decrease reference count on a shared memory handle
	489	* @shm: Shared memory handle
	490	*/
	491	void tee_shm_put(struct tee_shm *shm)
	492	{
	493	if (shm->flags & TEE_SHM_DMA_BUF)
	494	dma_buf_put(shm->dmabuf);
	495	}
	496	EXPORT_SYMBOL_GPL(tee_shm_put);