[linux-block.git] / drivers / tee / tee_shm.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2015-2016, Linaro Limited
 */
#include <linux/device.h>
#include <linux/dma-buf.h>
#include <linux/fdtable.h>
#include <linux/idr.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/tee_drv.h>
#include "tee_private.h"

static void tee_shm_release(struct tee_shm *shm)
{
	struct tee_device *teedev = shm->teedev;

	mutex_lock(&teedev->mutex);
	idr_remove(&teedev->idr, shm->id);
	mutex_unlock(&teedev->mutex);

	if (shm->flags & TEE_SHM_POOL) {
		struct tee_shm_pool_mgr *poolm;

		if (shm->flags & TEE_SHM_DMA_BUF)
			poolm = teedev->pool->dma_buf_mgr;
		else
			poolm = teedev->pool->private_mgr;

		poolm->ops->free(poolm, shm);
	} else if (shm->flags & TEE_SHM_REGISTER) {
		size_t n;
		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);

		if (rc)
			dev_err(teedev->dev.parent,
				"unregister shm %p failed: %d", shm, rc);

		for (n = 0; n < shm->num_pages; n++)
			put_page(shm->pages[n]);

		kfree(shm->pages);
	}

	if (shm->ctx)
		teedev_ctx_put(shm->ctx);

	kfree(shm);

	tee_device_put(teedev);
}

static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment
			*attach, enum dma_data_direction dir)
{
	return NULL;
}

static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach,
				     struct sg_table *table,
				     enum dma_data_direction dir)
{
}

static void tee_shm_op_release(struct dma_buf *dmabuf)
{
	struct tee_shm *shm = dmabuf->priv;

	tee_shm_release(shm);
}

static int tee_shm_op_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
	struct tee_shm *shm = dmabuf->priv;
	size_t size = vma->vm_end - vma->vm_start;

	/* Refuse sharing shared memory provided by application */
	if (shm->flags & TEE_SHM_REGISTER)
		return -EINVAL;

	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
			       size, vma->vm_page_prot);
}

static const struct dma_buf_ops tee_shm_dma_buf_ops = {
	.map_dma_buf = tee_shm_op_map_dma_buf,
	.unmap_dma_buf = tee_shm_op_unmap_dma_buf,
	.release = tee_shm_op_release,
	.mmap = tee_shm_op_mmap,
};

static struct tee_shm *__tee_shm_alloc(struct tee_context *ctx,
				       struct tee_device *teedev,
				       size_t size, u32 flags)
{
	struct tee_shm_pool_mgr *poolm = NULL;
	struct tee_shm *shm;
	void *ret;
	int rc;

	if (ctx && ctx->teedev != teedev) {
		dev_err(teedev->dev.parent, "ctx and teedev mismatch\n");
		return ERR_PTR(-EINVAL);
	}

	if (!(flags & TEE_SHM_MAPPED)) {
		dev_err(teedev->dev.parent,
			"only mapped allocations supported\n");
		return ERR_PTR(-EINVAL);
	}

	if ((flags & ~(TEE_SHM_MAPPED | TEE_SHM_DMA_BUF))) {
		dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);
		return ERR_PTR(-EINVAL);
	}

	if (!tee_device_get(teedev))
		return ERR_PTR(-EINVAL);

	if (!teedev->pool) {
		/* teedev has been detached from driver */
		ret = ERR_PTR(-EINVAL);
		goto err_dev_put;
	}

	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	if (!shm) {
		ret = ERR_PTR(-ENOMEM);
		goto err_dev_put;
	}

	shm->flags = flags | TEE_SHM_POOL;
	shm->teedev = teedev;
	shm->ctx = ctx;
	if (flags & TEE_SHM_DMA_BUF)
		poolm = teedev->pool->dma_buf_mgr;
	else
		poolm = teedev->pool->private_mgr;

	rc = poolm->ops->alloc(poolm, shm, size);
	if (rc) {
		ret = ERR_PTR(rc);
		goto err_kfree;
	}

	mutex_lock(&teedev->mutex);
	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	mutex_unlock(&teedev->mutex);
	if (shm->id < 0) {
		ret = ERR_PTR(shm->id);
		goto err_pool_free;
	}

	if (flags & TEE_SHM_DMA_BUF) {
		DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

		exp_info.ops = &tee_shm_dma_buf_ops;
		exp_info.size = shm->size;
		exp_info.flags = O_RDWR;
		exp_info.priv = shm;

		shm->dmabuf = dma_buf_export(&exp_info);
		if (IS_ERR(shm->dmabuf)) {
			ret = ERR_CAST(shm->dmabuf);
			goto err_rem;
		}
	}

	if (ctx)
		teedev_ctx_get(ctx);

	return shm;
err_rem:
	mutex_lock(&teedev->mutex);
	idr_remove(&teedev->idr, shm->id);
	mutex_unlock(&teedev->mutex);
err_pool_free:
	poolm->ops->free(poolm, shm);
err_kfree:
	kfree(shm);
err_dev_put:
	tee_device_put(teedev);
	return ret;
}

/**
 * tee_shm_alloc() - Allocate shared memory
 * @ctx:	Context that allocates the shared memory
 * @size:	Requested size of shared memory
 * @flags:	Flags setting properties for the requested shared memory.
 *
 * Memory allocated as global shared memory is automatically freed when the
 * TEE file pointer is closed. The @flags field uses the bits defined by
 * TEE_SHM_* in <linux/tee_drv.h>. TEE_SHM_MAPPED must currently always be
 * set. If TEE_SHM_DMA_BUF global shared memory will be allocated and
 * associated with a dma-buf handle, else driver private memory.
 */
struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
{
	return __tee_shm_alloc(ctx, ctx->teedev, size, flags);
}
EXPORT_SYMBOL_GPL(tee_shm_alloc);

struct tee_shm *tee_shm_priv_alloc(struct tee_device *teedev, size_t size)
{
	return __tee_shm_alloc(NULL, teedev, size, TEE_SHM_MAPPED);
}
EXPORT_SYMBOL_GPL(tee_shm_priv_alloc);

struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
				 size_t length, u32 flags)
{
	struct tee_device *teedev = ctx->teedev;
	const u32 req_flags = TEE_SHM_DMA_BUF | TEE_SHM_USER_MAPPED;
	struct tee_shm *shm;
	void *ret;
	int rc;
	int num_pages;
	unsigned long start;

	if (flags != req_flags)
		return ERR_PTR(-ENOTSUPP);

	if (!tee_device_get(teedev))
		return ERR_PTR(-EINVAL);

	if (!teedev->desc->ops->shm_register ||
	    !teedev->desc->ops->shm_unregister) {
		tee_device_put(teedev);
		return ERR_PTR(-ENOTSUPP);
	}

	teedev_ctx_get(ctx);

	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	if (!shm) {
		ret = ERR_PTR(-ENOMEM);
		goto err;
	}

	shm->flags = flags | TEE_SHM_REGISTER;
	shm->teedev = teedev;
	shm->ctx = ctx;
	shm->id = -1;
	addr = untagged_addr(addr);
	start = rounddown(addr, PAGE_SIZE);
	shm->offset = addr - start;
	shm->size = length;
	num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
	if (!shm->pages) {
		ret = ERR_PTR(-ENOMEM);
		goto err;
	}

	rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);
	if (rc > 0)
		shm->num_pages = rc;
	if (rc != num_pages) {
		if (rc >= 0)
			rc = -ENOMEM;
		ret = ERR_PTR(rc);
		goto err;
	}

	mutex_lock(&teedev->mutex);
	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	mutex_unlock(&teedev->mutex);

	if (shm->id < 0) {
		ret = ERR_PTR(shm->id);
		goto err;
	}

	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
					     shm->num_pages, start);
	if (rc) {
		ret = ERR_PTR(rc);
		goto err;
	}

	if (flags & TEE_SHM_DMA_BUF) {
		DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

		exp_info.ops = &tee_shm_dma_buf_ops;
		exp_info.size = shm->size;
		exp_info.flags = O_RDWR;
		exp_info.priv = shm;

		shm->dmabuf = dma_buf_export(&exp_info);
		if (IS_ERR(shm->dmabuf)) {
			ret = ERR_CAST(shm->dmabuf);
			teedev->desc->ops->shm_unregister(ctx, shm);
			goto err;
		}
	}

	return shm;
err:
	if (shm) {
		size_t n;

		if (shm->id >= 0) {
			mutex_lock(&teedev->mutex);
			idr_remove(&teedev->idr, shm->id);
			mutex_unlock(&teedev->mutex);
		}
		if (shm->pages) {
			for (n = 0; n < shm->num_pages; n++)
				put_page(shm->pages[n]);
			kfree(shm->pages);
		}
	}
	kfree(shm);
	teedev_ctx_put(ctx);
	tee_device_put(teedev);
	return ret;
}
EXPORT_SYMBOL_GPL(tee_shm_register);

/**
 * tee_shm_get_fd() - Increase reference count and return file descriptor
 * @shm:	Shared memory handle
 * @returns user space file descriptor to shared memory
 */
int tee_shm_get_fd(struct tee_shm *shm)
{
	int fd;

	if (!(shm->flags & TEE_SHM_DMA_BUF))
		return -EINVAL;

	get_dma_buf(shm->dmabuf);
	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);
	if (fd < 0)
		dma_buf_put(shm->dmabuf);
	return fd;
}

/**
 * tee_shm_free() - Free shared memory
 * @shm:	Handle to shared memory to free
 */
void tee_shm_free(struct tee_shm *shm)
{
	/*
	 * dma_buf_put() decreases the dmabuf reference counter and will
	 * call tee_shm_release() when the last reference is gone.
	 *
	 * In the case of driver private memory we call tee_shm_release
	 * directly instead as it doesn't have a reference counter.
	 */
	if (shm->flags & TEE_SHM_DMA_BUF)
		dma_buf_put(shm->dmabuf);
	else
		tee_shm_release(shm);
}
EXPORT_SYMBOL_GPL(tee_shm_free);

/**
 * tee_shm_va2pa() - Get physical address of a virtual address
 * @shm:	Shared memory handle
 * @va:		Virtual address to tranlsate
 * @pa:		Returned physical address
 * @returns 0 on success and < 0 on failure
 */
int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return -EINVAL;
	/* Check that we're in the range of the shm */
	if ((char *)va < (char *)shm->kaddr)
		return -EINVAL;
	if ((char *)va >= ((char *)shm->kaddr + shm->size))
		return -EINVAL;

	return tee_shm_get_pa(
			shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);
}
EXPORT_SYMBOL_GPL(tee_shm_va2pa);

/**
 * tee_shm_pa2va() - Get virtual address of a physical address
 * @shm:	Shared memory handle
 * @pa:		Physical address to tranlsate
 * @va:		Returned virtual address
 * @returns 0 on success and < 0 on failure
 */
int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return -EINVAL;
	/* Check that we're in the range of the shm */
	if (pa < shm->paddr)
		return -EINVAL;
	if (pa >= (shm->paddr + shm->size))
		return -EINVAL;

	if (va) {
		void *v = tee_shm_get_va(shm, pa - shm->paddr);

		if (IS_ERR(v))
			return PTR_ERR(v);
		*va = v;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(tee_shm_pa2va);

/**
 * tee_shm_get_va() - Get virtual address of a shared memory plus an offset
 * @shm:	Shared memory handle
 * @offs:	Offset from start of this shared memory
 * @returns virtual address of the shared memory + offs if offs is within
 *	the bounds of this shared memory, else an ERR_PTR
 */
void *tee_shm_get_va(struct tee_shm *shm, size_t offs)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return ERR_PTR(-EINVAL);
	if (offs >= shm->size)
		return ERR_PTR(-EINVAL);
	return (char *)shm->kaddr + offs;
}
EXPORT_SYMBOL_GPL(tee_shm_get_va);

/**
 * tee_shm_get_pa() - Get physical address of a shared memory plus an offset
 * @shm:	Shared memory handle
 * @offs:	Offset from start of this shared memory
 * @pa:		Physical address to return
 * @returns 0 if offs is within the bounds of this shared memory, else an
 *	error code.
 */
int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)
{
	if (offs >= shm->size)
		return -EINVAL;
	if (pa)
		*pa = shm->paddr + offs;
	return 0;
}
EXPORT_SYMBOL_GPL(tee_shm_get_pa);

/**
 * tee_shm_get_from_id() - Find shared memory object and increase reference
 * count
 * @ctx:	Context owning the shared memory
 * @id:		Id of shared memory object
 * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
 */
struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id)
{
	struct tee_device *teedev;
	struct tee_shm *shm;

	if (!ctx)
		return ERR_PTR(-EINVAL);

	teedev = ctx->teedev;
	mutex_lock(&teedev->mutex);
	shm = idr_find(&teedev->idr, id);
	if (!shm || shm->ctx != ctx)
		shm = ERR_PTR(-EINVAL);
	else if (shm->flags & TEE_SHM_DMA_BUF)
		get_dma_buf(shm->dmabuf);
	mutex_unlock(&teedev->mutex);
	return shm;
}
EXPORT_SYMBOL_GPL(tee_shm_get_from_id);

/**
 * tee_shm_put() - Decrease reference count on a shared memory handle
 * @shm:	Shared memory handle
 */
void tee_shm_put(struct tee_shm *shm)
{
	if (shm->flags & TEE_SHM_DMA_BUF)
		dma_buf_put(shm->dmabuf);
}
EXPORT_SYMBOL_GPL(tee_shm_put);
Commit	Line	Data
9c92ab61	1	// SPDX-License-Identifier: GPL-2.0-only
967c9cca JW	2	/*
967c9cca JW	3	* Copyright (c) 2015-2016, Linaro Limited
967c9cca JW	4	*/
	5	#include <linux/device.h>
	6	#include <linux/dma-buf.h>
	7	#include <linux/fdtable.h>
	8	#include <linux/idr.h>
	9	#include <linux/sched.h>
	10	#include <linux/slab.h>
	11	#include <linux/tee_drv.h>
	12	#include "tee_private.h"
	13
	14	static void tee_shm_release(struct tee_shm *shm)
	15	{
	16	struct tee_device *teedev = shm->teedev;
967c9cca JW	17
	18	mutex_lock(&teedev->mutex);
	19	idr_remove(&teedev->idr, shm->id);
967c9cca JW	20	mutex_unlock(&teedev->mutex);
967c9cca JW	21
033ddf12 JW	22	if (shm->flags & TEE_SHM_POOL) {
	23	struct tee_shm_pool_mgr *poolm;
	24
	25	if (shm->flags & TEE_SHM_DMA_BUF)
	26	poolm = teedev->pool->dma_buf_mgr;
	27	else
	28	poolm = teedev->pool->private_mgr;
	29
	30	poolm->ops->free(poolm, shm);
	31	} else if (shm->flags & TEE_SHM_REGISTER) {
	32	size_t n;
	33	int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);
	34
	35	if (rc)
	36	dev_err(teedev->dev.parent,
	37	"unregister shm %p failed: %d", shm, rc);
	38
	39	for (n = 0; n < shm->num_pages; n++)
	40	put_page(shm->pages[n]);
	41
	42	kfree(shm->pages);
	43	}
967c9cca	44
217e0250 VB	45	if (shm->ctx)
	46	teedev_ctx_put(shm->ctx);
	47
967c9cca JW	48	kfree(shm);
	49
	50	tee_device_put(teedev);
	51	}
	52
	53	static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment
	54	*attach, enum dma_data_direction dir)
	55	{
	56	return NULL;
	57	}
	58
	59	static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach,
	60	struct sg_table *table,
	61	enum dma_data_direction dir)
	62	{
	63	}
	64
	65	static void tee_shm_op_release(struct dma_buf *dmabuf)
	66	{
	67	struct tee_shm *shm = dmabuf->priv;
	68
	69	tee_shm_release(shm);
	70	}
	71
967c9cca JW	72	static int tee_shm_op_mmap(struct dma_buf dmabuf, struct vm_area_struct vma)
	73	{
	74	struct tee_shm *shm = dmabuf->priv;
	75	size_t size = vma->vm_end - vma->vm_start;
	76
033ddf12 JW	77	/* Refuse sharing shared memory provided by application */
	78	if (shm->flags & TEE_SHM_REGISTER)
	79	return -EINVAL;
	80
967c9cca JW	81	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
	82	size, vma->vm_page_prot);
	83	}
	84
53e3ca5c	85	static const struct dma_buf_ops tee_shm_dma_buf_ops = {
967c9cca JW	86	.map_dma_buf = tee_shm_op_map_dma_buf,
	87	.unmap_dma_buf = tee_shm_op_unmap_dma_buf,
	88	.release = tee_shm_op_release,
967c9cca JW	89	.mmap = tee_shm_op_mmap,
	90	};
	91
80ec6f5d CIK	92	static struct tee_shm __tee_shm_alloc(struct tee_context ctx,
	93	struct tee_device *teedev,
	94	size_t size, u32 flags)
967c9cca	95	{
967c9cca JW	96	struct tee_shm_pool_mgr *poolm = NULL;
	97	struct tee_shm *shm;
	98	void *ret;
	99	int rc;
	100
033ddf12 JW	101	if (ctx && ctx->teedev != teedev) {
	102	dev_err(teedev->dev.parent, "ctx and teedev mismatch\n");
	103	return ERR_PTR(-EINVAL);
	104	}
	105
967c9cca JW	106	if (!(flags & TEE_SHM_MAPPED)) {
	107	dev_err(teedev->dev.parent,
	108	"only mapped allocations supported\n");
	109	return ERR_PTR(-EINVAL);
	110	}
	111
	112	if ((flags & ~(TEE_SHM_MAPPED \| TEE_SHM_DMA_BUF))) {
	113	dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);
	114	return ERR_PTR(-EINVAL);
	115	}
	116
	117	if (!tee_device_get(teedev))
	118	return ERR_PTR(-EINVAL);
	119
	120	if (!teedev->pool) {
	121	/* teedev has been detached from driver */
	122	ret = ERR_PTR(-EINVAL);
	123	goto err_dev_put;
	124	}
	125
	126	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	127	if (!shm) {
	128	ret = ERR_PTR(-ENOMEM);
	129	goto err_dev_put;
	130	}
	131
033ddf12	132	shm->flags = flags \| TEE_SHM_POOL;
967c9cca JW	133	shm->teedev = teedev;
	134	shm->ctx = ctx;
	135	if (flags & TEE_SHM_DMA_BUF)
e2aca5d8	136	poolm = teedev->pool->dma_buf_mgr;
967c9cca	137	else
e2aca5d8	138	poolm = teedev->pool->private_mgr;
967c9cca JW	139
	140	rc = poolm->ops->alloc(poolm, shm, size);
	141	if (rc) {
	142	ret = ERR_PTR(rc);
	143	goto err_kfree;
	144	}
	145
	146	mutex_lock(&teedev->mutex);
	147	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	148	mutex_unlock(&teedev->mutex);
	149	if (shm->id < 0) {
	150	ret = ERR_PTR(shm->id);
	151	goto err_pool_free;
	152	}
	153
	154	if (flags & TEE_SHM_DMA_BUF) {
	155	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	156
	157	exp_info.ops = &tee_shm_dma_buf_ops;
	158	exp_info.size = shm->size;
	159	exp_info.flags = O_RDWR;
	160	exp_info.priv = shm;
	161
	162	shm->dmabuf = dma_buf_export(&exp_info);
	163	if (IS_ERR(shm->dmabuf)) {
	164	ret = ERR_CAST(shm->dmabuf);
	165	goto err_rem;
	166	}
	167	}
033ddf12	168
59a135f6	169	if (ctx)
217e0250	170	teedev_ctx_get(ctx);
967c9cca JW	171
	172	return shm;
	173	err_rem:
	174	mutex_lock(&teedev->mutex);
	175	idr_remove(&teedev->idr, shm->id);
	176	mutex_unlock(&teedev->mutex);
	177	err_pool_free:
	178	poolm->ops->free(poolm, shm);
	179	err_kfree:
	180	kfree(shm);
	181	err_dev_put:
	182	tee_device_put(teedev);
	183	return ret;
	184	}
033ddf12 JW	185
	186	/**
	187	* tee_shm_alloc() - Allocate shared memory
	188	* @ctx: Context that allocates the shared memory
	189	* @size: Requested size of shared memory
	190	* @flags: Flags setting properties for the requested shared memory.
	191	*
	192	* Memory allocated as global shared memory is automatically freed when the
	193	* TEE file pointer is closed. The @flags field uses the bits defined by
	194	* TEE_SHM_* in <linux/tee_drv.h>. TEE_SHM_MAPPED must currently always be
	195	* set. If TEE_SHM_DMA_BUF global shared memory will be allocated and
	196	* associated with a dma-buf handle, else driver private memory.
	197	*/
	198	struct tee_shm tee_shm_alloc(struct tee_context ctx, size_t size, u32 flags)
	199	{
	200	return __tee_shm_alloc(ctx, ctx->teedev, size, flags);
	201	}
967c9cca JW	202	EXPORT_SYMBOL_GPL(tee_shm_alloc);
967c9cca JW	203
033ddf12 JW	204	struct tee_shm tee_shm_priv_alloc(struct tee_device teedev, size_t size)
	205	{
	206	return __tee_shm_alloc(NULL, teedev, size, TEE_SHM_MAPPED);
	207	}
	208	EXPORT_SYMBOL_GPL(tee_shm_priv_alloc);
	209
	210	struct tee_shm tee_shm_register(struct tee_context ctx, unsigned long addr,
	211	size_t length, u32 flags)
	212	{
	213	struct tee_device *teedev = ctx->teedev;
	214	const u32 req_flags = TEE_SHM_DMA_BUF \| TEE_SHM_USER_MAPPED;
	215	struct tee_shm *shm;
	216	void *ret;
	217	int rc;
	218	int num_pages;
	219	unsigned long start;
	220
	221	if (flags != req_flags)
	222	return ERR_PTR(-ENOTSUPP);
	223
	224	if (!tee_device_get(teedev))
	225	return ERR_PTR(-EINVAL);
	226
	227	if (!teedev->desc->ops->shm_register \|\|
	228	!teedev->desc->ops->shm_unregister) {
	229	tee_device_put(teedev);
	230	return ERR_PTR(-ENOTSUPP);
	231	}
	232
217e0250 VB	233	teedev_ctx_get(ctx);
217e0250 VB	234
033ddf12 JW	235	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	236	if (!shm) {
	237	ret = ERR_PTR(-ENOMEM);
	238	goto err;
	239	}
	240
	241	shm->flags = flags \| TEE_SHM_REGISTER;
	242	shm->teedev = teedev;
	243	shm->ctx = ctx;
	244	shm->id = -1;
78063a9d	245	addr = untagged_addr(addr);
033ddf12 JW	246	start = rounddown(addr, PAGE_SIZE);
	247	shm->offset = addr - start;
	248	shm->size = length;
	249	num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
	250	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
	251	if (!shm->pages) {
	252	ret = ERR_PTR(-ENOMEM);
	253	goto err;
	254	}
	255
73b0140b	256	rc = get_user_pages_fast(start, num_pages, FOLL_WRITE, shm->pages);
033ddf12 JW	257	if (rc > 0)
	258	shm->num_pages = rc;
	259	if (rc != num_pages) {
2490cdf6	260	if (rc >= 0)
033ddf12 JW	261	rc = -ENOMEM;
	262	ret = ERR_PTR(rc);
	263	goto err;
	264	}
	265
	266	mutex_lock(&teedev->mutex);
	267	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	268	mutex_unlock(&teedev->mutex);
	269
	270	if (shm->id < 0) {
	271	ret = ERR_PTR(shm->id);
	272	goto err;
	273	}
	274
	275	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
95ffe4ca	276	shm->num_pages, start);
033ddf12 JW	277	if (rc) {
	278	ret = ERR_PTR(rc);
	279	goto err;
	280	}
	281
	282	if (flags & TEE_SHM_DMA_BUF) {
	283	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	284
	285	exp_info.ops = &tee_shm_dma_buf_ops;
	286	exp_info.size = shm->size;
	287	exp_info.flags = O_RDWR;
	288	exp_info.priv = shm;
	289
	290	shm->dmabuf = dma_buf_export(&exp_info);
	291	if (IS_ERR(shm->dmabuf)) {
	292	ret = ERR_CAST(shm->dmabuf);
	293	teedev->desc->ops->shm_unregister(ctx, shm);
	294	goto err;
	295	}
	296	}
	297
033ddf12 JW	298	return shm;
	299	err:
	300	if (shm) {
	301	size_t n;
	302
	303	if (shm->id >= 0) {
	304	mutex_lock(&teedev->mutex);
	305	idr_remove(&teedev->idr, shm->id);
	306	mutex_unlock(&teedev->mutex);
	307	}
c94f31b5 CIK	308	if (shm->pages) {
	309	for (n = 0; n < shm->num_pages; n++)
	310	put_page(shm->pages[n]);
	311	kfree(shm->pages);
	312	}
033ddf12 JW	313	}
033ddf12 JW	314	kfree(shm);
217e0250	315	teedev_ctx_put(ctx);
033ddf12 JW	316	tee_device_put(teedev);
	317	return ret;
	318	}
	319	EXPORT_SYMBOL_GPL(tee_shm_register);
	320
967c9cca JW	321	/**
	322	* tee_shm_get_fd() - Increase reference count and return file descriptor
	323	* @shm: Shared memory handle
	324	* @returns user space file descriptor to shared memory
	325	*/
	326	int tee_shm_get_fd(struct tee_shm *shm)
	327	{
967c9cca JW	328	int fd;
967c9cca JW	329
033ddf12	330	if (!(shm->flags & TEE_SHM_DMA_BUF))
967c9cca JW	331	return -EINVAL;
967c9cca JW	332
bb765d1c	333	get_dma_buf(shm->dmabuf);
967c9cca	334	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);
bb765d1c JH	335	if (fd < 0)
bb765d1c JH	336	dma_buf_put(shm->dmabuf);
967c9cca JW	337	return fd;
	338	}
	339
	340	/**
	341	* tee_shm_free() - Free shared memory
	342	* @shm: Handle to shared memory to free
	343	*/
	344	void tee_shm_free(struct tee_shm *shm)
	345	{
	346	/*
	347	* dma_buf_put() decreases the dmabuf reference counter and will
	348	* call tee_shm_release() when the last reference is gone.
	349	*
	350	* In the case of driver private memory we call tee_shm_release
	351	* directly instead as it doesn't have a reference counter.
	352	*/
	353	if (shm->flags & TEE_SHM_DMA_BUF)
	354	dma_buf_put(shm->dmabuf);
	355	else
	356	tee_shm_release(shm);
	357	}
	358	EXPORT_SYMBOL_GPL(tee_shm_free);
	359
	360	/**
	361	* tee_shm_va2pa() - Get physical address of a virtual address
	362	* @shm: Shared memory handle
	363	* @va: Virtual address to tranlsate
	364	* @pa: Returned physical address
	365	* @returns 0 on success and < 0 on failure
	366	*/
	367	int tee_shm_va2pa(struct tee_shm shm, void va, phys_addr_t *pa)
	368	{
033ddf12 JW	369	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	370	return -EINVAL;
967c9cca JW	371	/* Check that we're in the range of the shm */
	372	if ((char )va < (char )shm->kaddr)
	373	return -EINVAL;
	374	if ((char )va >= ((char )shm->kaddr + shm->size))
	375	return -EINVAL;
	376
	377	return tee_shm_get_pa(
	378	shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);
	379	}
	380	EXPORT_SYMBOL_GPL(tee_shm_va2pa);
	381
	382	/**
	383	* tee_shm_pa2va() - Get virtual address of a physical address
	384	* @shm: Shared memory handle
	385	* @pa: Physical address to tranlsate
	386	* @va: Returned virtual address
	387	* @returns 0 on success and < 0 on failure
	388	*/
	389	int tee_shm_pa2va(struct tee_shm shm, phys_addr_t pa, void *va)
	390	{
033ddf12 JW	391	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	392	return -EINVAL;
967c9cca JW	393	/* Check that we're in the range of the shm */
	394	if (pa < shm->paddr)
	395	return -EINVAL;
	396	if (pa >= (shm->paddr + shm->size))
	397	return -EINVAL;
	398
	399	if (va) {
	400	void *v = tee_shm_get_va(shm, pa - shm->paddr);
	401
	402	if (IS_ERR(v))
	403	return PTR_ERR(v);
	404	*va = v;
	405	}
	406	return 0;
	407	}
	408	EXPORT_SYMBOL_GPL(tee_shm_pa2va);
	409
	410	/**
	411	* tee_shm_get_va() - Get virtual address of a shared memory plus an offset
	412	* @shm: Shared memory handle
	413	* @offs: Offset from start of this shared memory
	414	* @returns virtual address of the shared memory + offs if offs is within
	415	* the bounds of this shared memory, else an ERR_PTR
	416	*/
	417	void tee_shm_get_va(struct tee_shm shm, size_t offs)
	418	{
033ddf12 JW	419	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	420	return ERR_PTR(-EINVAL);
967c9cca JW	421	if (offs >= shm->size)
	422	return ERR_PTR(-EINVAL);
	423	return (char *)shm->kaddr + offs;
	424	}
	425	EXPORT_SYMBOL_GPL(tee_shm_get_va);
	426
	427	/**
	428	* tee_shm_get_pa() - Get physical address of a shared memory plus an offset
	429	* @shm: Shared memory handle
	430	* @offs: Offset from start of this shared memory
	431	* @pa: Physical address to return
	432	* @returns 0 if offs is within the bounds of this shared memory, else an
	433	* error code.
	434	*/
	435	int tee_shm_get_pa(struct tee_shm shm, size_t offs, phys_addr_t pa)
	436	{
	437	if (offs >= shm->size)
	438	return -EINVAL;
	439	if (pa)
	440	*pa = shm->paddr + offs;
	441	return 0;
	442	}
	443	EXPORT_SYMBOL_GPL(tee_shm_get_pa);
	444
	445	/**
	446	* tee_shm_get_from_id() - Find shared memory object and increase reference
	447	* count
	448	* @ctx: Context owning the shared memory
	449	* @id: Id of shared memory object
	450	* @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
	451	*/
	452	struct tee_shm tee_shm_get_from_id(struct tee_context ctx, int id)
	453	{
	454	struct tee_device *teedev;
	455	struct tee_shm *shm;
	456
	457	if (!ctx)
	458	return ERR_PTR(-EINVAL);
	459
	460	teedev = ctx->teedev;
	461	mutex_lock(&teedev->mutex);
	462	shm = idr_find(&teedev->idr, id);
	463	if (!shm \|\| shm->ctx != ctx)
	464	shm = ERR_PTR(-EINVAL);
	465	else if (shm->flags & TEE_SHM_DMA_BUF)
	466	get_dma_buf(shm->dmabuf);
	467	mutex_unlock(&teedev->mutex);
	468	return shm;
	469	}
	470	EXPORT_SYMBOL_GPL(tee_shm_get_from_id);
	471
967c9cca JW	472	/**
	473	* tee_shm_put() - Decrease reference count on a shared memory handle
	474	* @shm: Shared memory handle
	475	*/
	476	void tee_shm_put(struct tee_shm *shm)
	477	{
	478	if (shm->flags & TEE_SHM_DMA_BUF)
	479	dma_buf_put(shm->dmabuf);
	480	}
	481	EXPORT_SYMBOL_GPL(tee_shm_put);