[linux-2.6-block.git] / drivers / tee / tee_shm.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2015-2016, Linaro Limited
 */
#include <linux/device.h>
#include <linux/dma-buf.h>
#include <linux/fdtable.h>
#include <linux/idr.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/tee_drv.h>
#include <linux/uio.h>
#include "tee_private.h"

static void release_registered_pages(struct tee_shm *shm)
{
	if (shm->pages) {
		if (shm->flags & TEE_SHM_USER_MAPPED) {
			unpin_user_pages(shm->pages, shm->num_pages);
		} else {
			size_t n;

			for (n = 0; n < shm->num_pages; n++)
				put_page(shm->pages[n]);
		}

		kfree(shm->pages);
	}
}

static void tee_shm_release(struct tee_shm *shm)
{
	struct tee_device *teedev = shm->ctx->teedev;

	if (shm->flags & TEE_SHM_DMA_BUF) {
		mutex_lock(&teedev->mutex);
		idr_remove(&teedev->idr, shm->id);
		mutex_unlock(&teedev->mutex);
	}

	if (shm->flags & TEE_SHM_POOL) {
		struct tee_shm_pool_mgr *poolm;

		if (shm->flags & TEE_SHM_DMA_BUF)
			poolm = teedev->pool->dma_buf_mgr;
		else
			poolm = teedev->pool->private_mgr;

		poolm->ops->free(poolm, shm);
	} else if (shm->flags & TEE_SHM_REGISTER) {
		int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);

		if (rc)
			dev_err(teedev->dev.parent,
				"unregister shm %p failed: %d", shm, rc);

		release_registered_pages(shm);
	}

	teedev_ctx_put(shm->ctx);

	kfree(shm);

	tee_device_put(teedev);
}

static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment
			*attach, enum dma_data_direction dir)
{
	return NULL;
}

static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach,
				     struct sg_table *table,
				     enum dma_data_direction dir)
{
}

static void tee_shm_op_release(struct dma_buf *dmabuf)
{
	struct tee_shm *shm = dmabuf->priv;

	tee_shm_release(shm);
}

static int tee_shm_op_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
	struct tee_shm *shm = dmabuf->priv;
	size_t size = vma->vm_end - vma->vm_start;

	/* Refuse sharing shared memory provided by application */
	if (shm->flags & TEE_SHM_USER_MAPPED)
		return -EINVAL;

	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
			       size, vma->vm_page_prot);
}

static const struct dma_buf_ops tee_shm_dma_buf_ops = {
	.map_dma_buf = tee_shm_op_map_dma_buf,
	.unmap_dma_buf = tee_shm_op_unmap_dma_buf,
	.release = tee_shm_op_release,
	.mmap = tee_shm_op_mmap,
};

struct tee_shm *tee_shm_alloc(struct tee_context *ctx, size_t size, u32 flags)
{
	struct tee_device *teedev = ctx->teedev;
	struct tee_shm_pool_mgr *poolm = NULL;
	struct tee_shm *shm;
	void *ret;
	int rc;

	if (!(flags & TEE_SHM_MAPPED)) {
		dev_err(teedev->dev.parent,
			"only mapped allocations supported\n");
		return ERR_PTR(-EINVAL);
	}

	if ((flags & ~(TEE_SHM_MAPPED | TEE_SHM_DMA_BUF))) {
		dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);
		return ERR_PTR(-EINVAL);
	}

	if (!tee_device_get(teedev))
		return ERR_PTR(-EINVAL);

	if (!teedev->pool) {
		/* teedev has been detached from driver */
		ret = ERR_PTR(-EINVAL);
		goto err_dev_put;
	}

	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	if (!shm) {
		ret = ERR_PTR(-ENOMEM);
		goto err_dev_put;
	}

	shm->flags = flags | TEE_SHM_POOL;
	shm->ctx = ctx;
	if (flags & TEE_SHM_DMA_BUF)
		poolm = teedev->pool->dma_buf_mgr;
	else
		poolm = teedev->pool->private_mgr;

	rc = poolm->ops->alloc(poolm, shm, size);
	if (rc) {
		ret = ERR_PTR(rc);
		goto err_kfree;
	}


	if (flags & TEE_SHM_DMA_BUF) {
		DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

		mutex_lock(&teedev->mutex);
		shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
		mutex_unlock(&teedev->mutex);
		if (shm->id < 0) {
			ret = ERR_PTR(shm->id);
			goto err_pool_free;
		}

		exp_info.ops = &tee_shm_dma_buf_ops;
		exp_info.size = shm->size;
		exp_info.flags = O_RDWR;
		exp_info.priv = shm;

		shm->dmabuf = dma_buf_export(&exp_info);
		if (IS_ERR(shm->dmabuf)) {
			ret = ERR_CAST(shm->dmabuf);
			goto err_rem;
		}
	}

	teedev_ctx_get(ctx);

	return shm;
err_rem:
	if (flags & TEE_SHM_DMA_BUF) {
		mutex_lock(&teedev->mutex);
		idr_remove(&teedev->idr, shm->id);
		mutex_unlock(&teedev->mutex);
	}
err_pool_free:
	poolm->ops->free(poolm, shm);
err_kfree:
	kfree(shm);
err_dev_put:
	tee_device_put(teedev);
	return ret;
}
EXPORT_SYMBOL_GPL(tee_shm_alloc);

struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
				 size_t length, u32 flags)
{
	struct tee_device *teedev = ctx->teedev;
	const u32 req_user_flags = TEE_SHM_DMA_BUF | TEE_SHM_USER_MAPPED;
	const u32 req_kernel_flags = TEE_SHM_DMA_BUF | TEE_SHM_KERNEL_MAPPED;
	struct tee_shm *shm;
	void *ret;
	int rc;
	int num_pages;
	unsigned long start;

	if (flags != req_user_flags && flags != req_kernel_flags)
		return ERR_PTR(-ENOTSUPP);

	if (!tee_device_get(teedev))
		return ERR_PTR(-EINVAL);

	if (!teedev->desc->ops->shm_register ||
	    !teedev->desc->ops->shm_unregister) {
		tee_device_put(teedev);
		return ERR_PTR(-ENOTSUPP);
	}

	teedev_ctx_get(ctx);

	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	if (!shm) {
		ret = ERR_PTR(-ENOMEM);
		goto err;
	}

	shm->flags = flags | TEE_SHM_REGISTER;
	shm->ctx = ctx;
	shm->id = -1;
	addr = untagged_addr(addr);
	start = rounddown(addr, PAGE_SIZE);
	shm->offset = addr - start;
	shm->size = length;
	num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
	if (!shm->pages) {
		ret = ERR_PTR(-ENOMEM);
		goto err;
	}

	if (flags & TEE_SHM_USER_MAPPED) {
		rc = pin_user_pages_fast(start, num_pages, FOLL_WRITE,
					 shm->pages);
	} else {
		struct kvec *kiov;
		int i;

		kiov = kcalloc(num_pages, sizeof(*kiov), GFP_KERNEL);
		if (!kiov) {
			ret = ERR_PTR(-ENOMEM);
			goto err;
		}

		for (i = 0; i < num_pages; i++) {
			kiov[i].iov_base = (void *)(start + i * PAGE_SIZE);
			kiov[i].iov_len = PAGE_SIZE;
		}

		rc = get_kernel_pages(kiov, num_pages, 0, shm->pages);
		kfree(kiov);
	}
	if (rc > 0)
		shm->num_pages = rc;
	if (rc != num_pages) {
		if (rc >= 0)
			rc = -ENOMEM;
		ret = ERR_PTR(rc);
		goto err;
	}

	mutex_lock(&teedev->mutex);
	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	mutex_unlock(&teedev->mutex);

	if (shm->id < 0) {
		ret = ERR_PTR(shm->id);
		goto err;
	}

	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
					     shm->num_pages, start);
	if (rc) {
		ret = ERR_PTR(rc);
		goto err;
	}

	if (flags & TEE_SHM_DMA_BUF) {
		DEFINE_DMA_BUF_EXPORT_INFO(exp_info);

		exp_info.ops = &tee_shm_dma_buf_ops;
		exp_info.size = shm->size;
		exp_info.flags = O_RDWR;
		exp_info.priv = shm;

		shm->dmabuf = dma_buf_export(&exp_info);
		if (IS_ERR(shm->dmabuf)) {
			ret = ERR_CAST(shm->dmabuf);
			teedev->desc->ops->shm_unregister(ctx, shm);
			goto err;
		}
	}

	return shm;
err:
	if (shm) {
		if (shm->id >= 0) {
			mutex_lock(&teedev->mutex);
			idr_remove(&teedev->idr, shm->id);
			mutex_unlock(&teedev->mutex);
		}
		release_registered_pages(shm);
	}
	kfree(shm);
	teedev_ctx_put(ctx);
	tee_device_put(teedev);
	return ret;
}
EXPORT_SYMBOL_GPL(tee_shm_register);

/**
 * tee_shm_get_fd() - Increase reference count and return file descriptor
 * @shm:	Shared memory handle
 * @returns user space file descriptor to shared memory
 */
int tee_shm_get_fd(struct tee_shm *shm)
{
	int fd;

	if (!(shm->flags & TEE_SHM_DMA_BUF))
		return -EINVAL;

	get_dma_buf(shm->dmabuf);
	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);
	if (fd < 0)
		dma_buf_put(shm->dmabuf);
	return fd;
}

/**
 * tee_shm_free() - Free shared memory
 * @shm:	Handle to shared memory to free
 */
void tee_shm_free(struct tee_shm *shm)
{
	/*
	 * dma_buf_put() decreases the dmabuf reference counter and will
	 * call tee_shm_release() when the last reference is gone.
	 *
	 * In the case of driver private memory we call tee_shm_release
	 * directly instead as it doesn't have a reference counter.
	 */
	if (shm->flags & TEE_SHM_DMA_BUF)
		dma_buf_put(shm->dmabuf);
	else
		tee_shm_release(shm);
}
EXPORT_SYMBOL_GPL(tee_shm_free);

/**
 * tee_shm_va2pa() - Get physical address of a virtual address
 * @shm:	Shared memory handle
 * @va:		Virtual address to tranlsate
 * @pa:		Returned physical address
 * @returns 0 on success and < 0 on failure
 */
int tee_shm_va2pa(struct tee_shm *shm, void *va, phys_addr_t *pa)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return -EINVAL;
	/* Check that we're in the range of the shm */
	if ((char *)va < (char *)shm->kaddr)
		return -EINVAL;
	if ((char *)va >= ((char *)shm->kaddr + shm->size))
		return -EINVAL;

	return tee_shm_get_pa(
			shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);
}
EXPORT_SYMBOL_GPL(tee_shm_va2pa);

/**
 * tee_shm_pa2va() - Get virtual address of a physical address
 * @shm:	Shared memory handle
 * @pa:		Physical address to tranlsate
 * @va:		Returned virtual address
 * @returns 0 on success and < 0 on failure
 */
int tee_shm_pa2va(struct tee_shm *shm, phys_addr_t pa, void **va)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return -EINVAL;
	/* Check that we're in the range of the shm */
	if (pa < shm->paddr)
		return -EINVAL;
	if (pa >= (shm->paddr + shm->size))
		return -EINVAL;

	if (va) {
		void *v = tee_shm_get_va(shm, pa - shm->paddr);

		if (IS_ERR(v))
			return PTR_ERR(v);
		*va = v;
	}
	return 0;
}
EXPORT_SYMBOL_GPL(tee_shm_pa2va);

/**
 * tee_shm_get_va() - Get virtual address of a shared memory plus an offset
 * @shm:	Shared memory handle
 * @offs:	Offset from start of this shared memory
 * @returns virtual address of the shared memory + offs if offs is within
 *	the bounds of this shared memory, else an ERR_PTR
 */
void *tee_shm_get_va(struct tee_shm *shm, size_t offs)
{
	if (!(shm->flags & TEE_SHM_MAPPED))
		return ERR_PTR(-EINVAL);
	if (offs >= shm->size)
		return ERR_PTR(-EINVAL);
	return (char *)shm->kaddr + offs;
}
EXPORT_SYMBOL_GPL(tee_shm_get_va);

/**
 * tee_shm_get_pa() - Get physical address of a shared memory plus an offset
 * @shm:	Shared memory handle
 * @offs:	Offset from start of this shared memory
 * @pa:		Physical address to return
 * @returns 0 if offs is within the bounds of this shared memory, else an
 *	error code.
 */
int tee_shm_get_pa(struct tee_shm *shm, size_t offs, phys_addr_t *pa)
{
	if (offs >= shm->size)
		return -EINVAL;
	if (pa)
		*pa = shm->paddr + offs;
	return 0;
}
EXPORT_SYMBOL_GPL(tee_shm_get_pa);

/**
 * tee_shm_get_from_id() - Find shared memory object and increase reference
 * count
 * @ctx:	Context owning the shared memory
 * @id:		Id of shared memory object
 * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
 */
struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id)
{
	struct tee_device *teedev;
	struct tee_shm *shm;

	if (!ctx)
		return ERR_PTR(-EINVAL);

	teedev = ctx->teedev;
	mutex_lock(&teedev->mutex);
	shm = idr_find(&teedev->idr, id);
	if (!shm || shm->ctx != ctx)
		shm = ERR_PTR(-EINVAL);
	else if (shm->flags & TEE_SHM_DMA_BUF)
		get_dma_buf(shm->dmabuf);
	mutex_unlock(&teedev->mutex);
	return shm;
}
EXPORT_SYMBOL_GPL(tee_shm_get_from_id);

/**
 * tee_shm_put() - Decrease reference count on a shared memory handle
 * @shm:	Shared memory handle
 */
void tee_shm_put(struct tee_shm *shm)
{
	if (shm->flags & TEE_SHM_DMA_BUF)
		dma_buf_put(shm->dmabuf);
}
EXPORT_SYMBOL_GPL(tee_shm_put);
Commit	Line	Data
9c92ab61	1	// SPDX-License-Identifier: GPL-2.0-only
967c9cca JW	2	/*
967c9cca JW	3	* Copyright (c) 2015-2016, Linaro Limited
967c9cca JW	4	*/
	5	#include <linux/device.h>
	6	#include <linux/dma-buf.h>
	7	#include <linux/fdtable.h>
	8	#include <linux/idr.h>
	9	#include <linux/sched.h>
	10	#include <linux/slab.h>
	11	#include <linux/tee_drv.h>
2a6ba3f7	12	#include <linux/uio.h>
967c9cca JW	13	#include "tee_private.h"
967c9cca JW	14
4300cd63 JH	15	static void release_registered_pages(struct tee_shm *shm)
	16	{
	17	if (shm->pages) {
	18	if (shm->flags & TEE_SHM_USER_MAPPED) {
	19	unpin_user_pages(shm->pages, shm->num_pages);
	20	} else {
	21	size_t n;
	22
	23	for (n = 0; n < shm->num_pages; n++)
	24	put_page(shm->pages[n]);
	25	}
	26
	27	kfree(shm->pages);
	28	}
	29	}
	30
967c9cca JW	31	static void tee_shm_release(struct tee_shm *shm)
967c9cca JW	32	{
5271b201	33	struct tee_device *teedev = shm->ctx->teedev;
967c9cca	34
f1bbaced JW	35	if (shm->flags & TEE_SHM_DMA_BUF) {
	36	mutex_lock(&teedev->mutex);
	37	idr_remove(&teedev->idr, shm->id);
	38	mutex_unlock(&teedev->mutex);
	39	}
967c9cca	40
033ddf12 JW	41	if (shm->flags & TEE_SHM_POOL) {
	42	struct tee_shm_pool_mgr *poolm;
	43
	44	if (shm->flags & TEE_SHM_DMA_BUF)
	45	poolm = teedev->pool->dma_buf_mgr;
	46	else
	47	poolm = teedev->pool->private_mgr;
	48
	49	poolm->ops->free(poolm, shm);
	50	} else if (shm->flags & TEE_SHM_REGISTER) {
033ddf12 JW	51	int rc = teedev->desc->ops->shm_unregister(shm->ctx, shm);
	52
	53	if (rc)
	54	dev_err(teedev->dev.parent,
	55	"unregister shm %p failed: %d", shm, rc);
	56
4300cd63	57	release_registered_pages(shm);
033ddf12	58	}
967c9cca	59
5271b201	60	teedev_ctx_put(shm->ctx);
217e0250	61
967c9cca JW	62	kfree(shm);
	63
	64	tee_device_put(teedev);
	65	}
	66
	67	static struct sg_table *tee_shm_op_map_dma_buf(struct dma_buf_attachment
	68	*attach, enum dma_data_direction dir)
	69	{
	70	return NULL;
	71	}
	72
	73	static void tee_shm_op_unmap_dma_buf(struct dma_buf_attachment *attach,
	74	struct sg_table *table,
	75	enum dma_data_direction dir)
	76	{
	77	}
	78
	79	static void tee_shm_op_release(struct dma_buf *dmabuf)
	80	{
	81	struct tee_shm *shm = dmabuf->priv;
	82
	83	tee_shm_release(shm);
	84	}
	85
967c9cca JW	86	static int tee_shm_op_mmap(struct dma_buf dmabuf, struct vm_area_struct vma)
	87	{
	88	struct tee_shm *shm = dmabuf->priv;
	89	size_t size = vma->vm_end - vma->vm_start;
	90
033ddf12	91	/* Refuse sharing shared memory provided by application */
758ecf13	92	if (shm->flags & TEE_SHM_USER_MAPPED)
033ddf12 JW	93	return -EINVAL;
033ddf12 JW	94
967c9cca JW	95	return remap_pfn_range(vma, vma->vm_start, shm->paddr >> PAGE_SHIFT,
	96	size, vma->vm_page_prot);
	97	}
	98
53e3ca5c	99	static const struct dma_buf_ops tee_shm_dma_buf_ops = {
967c9cca JW	100	.map_dma_buf = tee_shm_op_map_dma_buf,
	101	.unmap_dma_buf = tee_shm_op_unmap_dma_buf,
	102	.release = tee_shm_op_release,
967c9cca JW	103	.mmap = tee_shm_op_mmap,
	104	};
	105
c180f9bb	106	struct tee_shm tee_shm_alloc(struct tee_context ctx, size_t size, u32 flags)
967c9cca	107	{
c180f9bb	108	struct tee_device *teedev = ctx->teedev;
967c9cca JW	109	struct tee_shm_pool_mgr *poolm = NULL;
	110	struct tee_shm *shm;
	111	void *ret;
	112	int rc;
	113
	114	if (!(flags & TEE_SHM_MAPPED)) {
	115	dev_err(teedev->dev.parent,
	116	"only mapped allocations supported\n");
	117	return ERR_PTR(-EINVAL);
	118	}
	119
	120	if ((flags & ~(TEE_SHM_MAPPED \| TEE_SHM_DMA_BUF))) {
	121	dev_err(teedev->dev.parent, "invalid shm flags 0x%x", flags);
	122	return ERR_PTR(-EINVAL);
	123	}
	124
	125	if (!tee_device_get(teedev))
	126	return ERR_PTR(-EINVAL);
	127
	128	if (!teedev->pool) {
	129	/* teedev has been detached from driver */
	130	ret = ERR_PTR(-EINVAL);
	131	goto err_dev_put;
	132	}
	133
	134	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	135	if (!shm) {
	136	ret = ERR_PTR(-ENOMEM);
	137	goto err_dev_put;
	138	}
	139
033ddf12	140	shm->flags = flags \| TEE_SHM_POOL;
967c9cca JW	141	shm->ctx = ctx;
967c9cca JW	142	if (flags & TEE_SHM_DMA_BUF)
e2aca5d8	143	poolm = teedev->pool->dma_buf_mgr;
967c9cca	144	else
e2aca5d8	145	poolm = teedev->pool->private_mgr;
967c9cca JW	146
	147	rc = poolm->ops->alloc(poolm, shm, size);
	148	if (rc) {
	149	ret = ERR_PTR(rc);
	150	goto err_kfree;
	151	}
	152
967c9cca JW	153
	154	if (flags & TEE_SHM_DMA_BUF) {
	155	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	156
f1bbaced JW	157	mutex_lock(&teedev->mutex);
	158	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	159	mutex_unlock(&teedev->mutex);
	160	if (shm->id < 0) {
	161	ret = ERR_PTR(shm->id);
	162	goto err_pool_free;
	163	}
	164
967c9cca JW	165	exp_info.ops = &tee_shm_dma_buf_ops;
	166	exp_info.size = shm->size;
	167	exp_info.flags = O_RDWR;
	168	exp_info.priv = shm;
	169
	170	shm->dmabuf = dma_buf_export(&exp_info);
	171	if (IS_ERR(shm->dmabuf)) {
	172	ret = ERR_CAST(shm->dmabuf);
	173	goto err_rem;
	174	}
	175	}
033ddf12	176
1115899e	177	teedev_ctx_get(ctx);
967c9cca JW	178
	179	return shm;
	180	err_rem:
f1bbaced JW	181	if (flags & TEE_SHM_DMA_BUF) {
	182	mutex_lock(&teedev->mutex);
	183	idr_remove(&teedev->idr, shm->id);
	184	mutex_unlock(&teedev->mutex);
	185	}
967c9cca JW	186	err_pool_free:
	187	poolm->ops->free(poolm, shm);
	188	err_kfree:
	189	kfree(shm);
	190	err_dev_put:
	191	tee_device_put(teedev);
	192	return ret;
	193	}
	194	EXPORT_SYMBOL_GPL(tee_shm_alloc);
	195
033ddf12 JW	196	struct tee_shm tee_shm_register(struct tee_context ctx, unsigned long addr,
	197	size_t length, u32 flags)
	198	{
	199	struct tee_device *teedev = ctx->teedev;
2a6ba3f7 SG	200	const u32 req_user_flags = TEE_SHM_DMA_BUF \| TEE_SHM_USER_MAPPED;
2a6ba3f7 SG	201	const u32 req_kernel_flags = TEE_SHM_DMA_BUF \| TEE_SHM_KERNEL_MAPPED;
033ddf12 JW	202	struct tee_shm *shm;
	203	void *ret;
	204	int rc;
	205	int num_pages;
	206	unsigned long start;
	207
2a6ba3f7	208	if (flags != req_user_flags && flags != req_kernel_flags)
033ddf12 JW	209	return ERR_PTR(-ENOTSUPP);
	210
	211	if (!tee_device_get(teedev))
	212	return ERR_PTR(-EINVAL);
	213
	214	if (!teedev->desc->ops->shm_register \|\|
	215	!teedev->desc->ops->shm_unregister) {
	216	tee_device_put(teedev);
	217	return ERR_PTR(-ENOTSUPP);
	218	}
	219
217e0250 VB	220	teedev_ctx_get(ctx);
217e0250 VB	221
033ddf12 JW	222	shm = kzalloc(sizeof(*shm), GFP_KERNEL);
	223	if (!shm) {
	224	ret = ERR_PTR(-ENOMEM);
	225	goto err;
	226	}
	227
	228	shm->flags = flags \| TEE_SHM_REGISTER;
033ddf12 JW	229	shm->ctx = ctx;
033ddf12 JW	230	shm->id = -1;
78063a9d	231	addr = untagged_addr(addr);
033ddf12 JW	232	start = rounddown(addr, PAGE_SIZE);
	233	shm->offset = addr - start;
	234	shm->size = length;
	235	num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
	236	shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
	237	if (!shm->pages) {
	238	ret = ERR_PTR(-ENOMEM);
	239	goto err;
	240	}
	241
2a6ba3f7	242	if (flags & TEE_SHM_USER_MAPPED) {
4300cd63	243	rc = pin_user_pages_fast(start, num_pages, FOLL_WRITE,
2a6ba3f7 SG	244	shm->pages);
	245	} else {
	246	struct kvec *kiov;
	247	int i;
	248
	249	kiov = kcalloc(num_pages, sizeof(*kiov), GFP_KERNEL);
	250	if (!kiov) {
	251	ret = ERR_PTR(-ENOMEM);
	252	goto err;
	253	}
	254
	255	for (i = 0; i < num_pages; i++) {
	256	kiov[i].iov_base = (void )(start + i PAGE_SIZE);
	257	kiov[i].iov_len = PAGE_SIZE;
	258	}
	259
	260	rc = get_kernel_pages(kiov, num_pages, 0, shm->pages);
	261	kfree(kiov);
	262	}
033ddf12 JW	263	if (rc > 0)
	264	shm->num_pages = rc;
	265	if (rc != num_pages) {
2490cdf6	266	if (rc >= 0)
033ddf12 JW	267	rc = -ENOMEM;
	268	ret = ERR_PTR(rc);
	269	goto err;
	270	}
	271
	272	mutex_lock(&teedev->mutex);
	273	shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
	274	mutex_unlock(&teedev->mutex);
	275
	276	if (shm->id < 0) {
	277	ret = ERR_PTR(shm->id);
	278	goto err;
	279	}
	280
	281	rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
95ffe4ca	282	shm->num_pages, start);
033ddf12 JW	283	if (rc) {
	284	ret = ERR_PTR(rc);
	285	goto err;
	286	}
	287
	288	if (flags & TEE_SHM_DMA_BUF) {
	289	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
	290
	291	exp_info.ops = &tee_shm_dma_buf_ops;
	292	exp_info.size = shm->size;
	293	exp_info.flags = O_RDWR;
	294	exp_info.priv = shm;
	295
	296	shm->dmabuf = dma_buf_export(&exp_info);
	297	if (IS_ERR(shm->dmabuf)) {
	298	ret = ERR_CAST(shm->dmabuf);
	299	teedev->desc->ops->shm_unregister(ctx, shm);
	300	goto err;
	301	}
	302	}
	303
033ddf12 JW	304	return shm;
	305	err:
	306	if (shm) {
033ddf12 JW	307	if (shm->id >= 0) {
	308	mutex_lock(&teedev->mutex);
	309	idr_remove(&teedev->idr, shm->id);
	310	mutex_unlock(&teedev->mutex);
	311	}
4300cd63	312	release_registered_pages(shm);
033ddf12 JW	313	}
033ddf12 JW	314	kfree(shm);
217e0250	315	teedev_ctx_put(ctx);
033ddf12 JW	316	tee_device_put(teedev);
	317	return ret;
	318	}
	319	EXPORT_SYMBOL_GPL(tee_shm_register);
	320
967c9cca JW	321	/**
	322	* tee_shm_get_fd() - Increase reference count and return file descriptor
	323	* @shm: Shared memory handle
	324	* @returns user space file descriptor to shared memory
	325	*/
	326	int tee_shm_get_fd(struct tee_shm *shm)
	327	{
967c9cca JW	328	int fd;
967c9cca JW	329
033ddf12	330	if (!(shm->flags & TEE_SHM_DMA_BUF))
967c9cca JW	331	return -EINVAL;
967c9cca JW	332
bb765d1c	333	get_dma_buf(shm->dmabuf);
967c9cca	334	fd = dma_buf_fd(shm->dmabuf, O_CLOEXEC);
bb765d1c JH	335	if (fd < 0)
bb765d1c JH	336	dma_buf_put(shm->dmabuf);
967c9cca JW	337	return fd;
	338	}
	339
	340	/**
	341	* tee_shm_free() - Free shared memory
	342	* @shm: Handle to shared memory to free
	343	*/
	344	void tee_shm_free(struct tee_shm *shm)
	345	{
	346	/*
	347	* dma_buf_put() decreases the dmabuf reference counter and will
	348	* call tee_shm_release() when the last reference is gone.
	349	*
	350	* In the case of driver private memory we call tee_shm_release
	351	* directly instead as it doesn't have a reference counter.
	352	*/
	353	if (shm->flags & TEE_SHM_DMA_BUF)
	354	dma_buf_put(shm->dmabuf);
	355	else
	356	tee_shm_release(shm);
	357	}
	358	EXPORT_SYMBOL_GPL(tee_shm_free);
	359
	360	/**
	361	* tee_shm_va2pa() - Get physical address of a virtual address
	362	* @shm: Shared memory handle
	363	* @va: Virtual address to tranlsate
	364	* @pa: Returned physical address
	365	* @returns 0 on success and < 0 on failure
	366	*/
	367	int tee_shm_va2pa(struct tee_shm shm, void va, phys_addr_t *pa)
	368	{
033ddf12 JW	369	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	370	return -EINVAL;
967c9cca JW	371	/* Check that we're in the range of the shm */
	372	if ((char )va < (char )shm->kaddr)
	373	return -EINVAL;
	374	if ((char )va >= ((char )shm->kaddr + shm->size))
	375	return -EINVAL;
	376
	377	return tee_shm_get_pa(
	378	shm, (unsigned long)va - (unsigned long)shm->kaddr, pa);
	379	}
	380	EXPORT_SYMBOL_GPL(tee_shm_va2pa);
	381
	382	/**
	383	* tee_shm_pa2va() - Get virtual address of a physical address
	384	* @shm: Shared memory handle
	385	* @pa: Physical address to tranlsate
	386	* @va: Returned virtual address
	387	* @returns 0 on success and < 0 on failure
	388	*/
	389	int tee_shm_pa2va(struct tee_shm shm, phys_addr_t pa, void *va)
	390	{
033ddf12 JW	391	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	392	return -EINVAL;
967c9cca JW	393	/* Check that we're in the range of the shm */
	394	if (pa < shm->paddr)
	395	return -EINVAL;
	396	if (pa >= (shm->paddr + shm->size))
	397	return -EINVAL;
	398
	399	if (va) {
	400	void *v = tee_shm_get_va(shm, pa - shm->paddr);
	401
	402	if (IS_ERR(v))
	403	return PTR_ERR(v);
	404	*va = v;
	405	}
	406	return 0;
	407	}
	408	EXPORT_SYMBOL_GPL(tee_shm_pa2va);
	409
	410	/**
	411	* tee_shm_get_va() - Get virtual address of a shared memory plus an offset
	412	* @shm: Shared memory handle
	413	* @offs: Offset from start of this shared memory
	414	* @returns virtual address of the shared memory + offs if offs is within
	415	* the bounds of this shared memory, else an ERR_PTR
	416	*/
	417	void tee_shm_get_va(struct tee_shm shm, size_t offs)
	418	{
033ddf12 JW	419	if (!(shm->flags & TEE_SHM_MAPPED))
033ddf12 JW	420	return ERR_PTR(-EINVAL);
967c9cca JW	421	if (offs >= shm->size)
	422	return ERR_PTR(-EINVAL);
	423	return (char *)shm->kaddr + offs;
	424	}
	425	EXPORT_SYMBOL_GPL(tee_shm_get_va);
	426
	427	/**
	428	* tee_shm_get_pa() - Get physical address of a shared memory plus an offset
	429	* @shm: Shared memory handle
	430	* @offs: Offset from start of this shared memory
	431	* @pa: Physical address to return
	432	* @returns 0 if offs is within the bounds of this shared memory, else an
	433	* error code.
	434	*/
	435	int tee_shm_get_pa(struct tee_shm shm, size_t offs, phys_addr_t pa)
	436	{
	437	if (offs >= shm->size)
	438	return -EINVAL;
	439	if (pa)
	440	*pa = shm->paddr + offs;
	441	return 0;
	442	}
	443	EXPORT_SYMBOL_GPL(tee_shm_get_pa);
	444
	445	/**
	446	* tee_shm_get_from_id() - Find shared memory object and increase reference
	447	* count
	448	* @ctx: Context owning the shared memory
	449	* @id: Id of shared memory object
	450	* @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure
	451	*/
	452	struct tee_shm tee_shm_get_from_id(struct tee_context ctx, int id)
	453	{
	454	struct tee_device *teedev;
	455	struct tee_shm *shm;
	456
	457	if (!ctx)
	458	return ERR_PTR(-EINVAL);
	459
	460	teedev = ctx->teedev;
	461	mutex_lock(&teedev->mutex);
	462	shm = idr_find(&teedev->idr, id);
	463	if (!shm \|\| shm->ctx != ctx)
	464	shm = ERR_PTR(-EINVAL);
	465	else if (shm->flags & TEE_SHM_DMA_BUF)
	466	get_dma_buf(shm->dmabuf);
	467	mutex_unlock(&teedev->mutex);
	468	return shm;
	469	}
	470	EXPORT_SYMBOL_GPL(tee_shm_get_from_id);
	471
967c9cca JW	472	/**
	473	* tee_shm_put() - Decrease reference count on a shared memory handle
	474	* @shm: Shared memory handle
	475	*/
	476	void tee_shm_put(struct tee_shm *shm)
	477	{
	478	if (shm->flags & TEE_SHM_DMA_BUF)
	479	dma_buf_put(shm->dmabuf);
	480	}
	481	EXPORT_SYMBOL_GPL(tee_shm_put);