[linux-2.6-block.git] / drivers / iommu / tegra-gart.c

/*
 * IOMMU API for GART in Tegra20
 *
 * Copyright (c) 2010-2012, NVIDIA CORPORATION.  All rights reserved.
 *
 * Author: Hiroshi DOYU <hdoyu@nvidia.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */

#define dev_fmt(fmt)	"gart: " fmt

#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/list.h>
#include <linux/moduleparam.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/vmalloc.h>

#include <soc/tegra/mc.h>

/* bitmap of the page sizes currently supported */
#define GART_IOMMU_PGSIZES	(SZ_4K)

#define GART_REG_BASE		0x24
#define GART_CONFIG		(0x24 - GART_REG_BASE)
#define GART_ENTRY_ADDR		(0x28 - GART_REG_BASE)
#define GART_ENTRY_DATA		(0x2c - GART_REG_BASE)
#define GART_ENTRY_PHYS_ADDR_VALID	(1 << 31)

#define GART_PAGE_SHIFT		12
#define GART_PAGE_SIZE		(1 << GART_PAGE_SHIFT)
#define GART_PAGE_MASK						\
	(~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID)

struct gart_client {
	struct device		*dev;
	struct list_head	list;
};

struct gart_device {
	void __iomem		*regs;
	u32			*savedata;
	u32			page_count;	/* total remappable size */
	dma_addr_t		iovmm_base;	/* offset to vmm_area */
	spinlock_t		pte_lock;	/* for pagetable */
	struct list_head	client;
	spinlock_t		client_lock;	/* for client list */
	struct iommu_domain	*active_domain;	/* current active domain */
	struct device		*dev;

	struct iommu_device	iommu;		/* IOMMU Core handle */
};

struct gart_domain {
	struct iommu_domain domain;		/* generic domain handle */
	struct gart_device *gart;		/* link to gart device   */
};

static struct gart_device *gart_handle; /* unique for a system */

static bool gart_debug;

#define GART_PTE(_pfn)						\
	(GART_ENTRY_PHYS_ADDR_VALID | ((_pfn) << PAGE_SHIFT))

static struct gart_domain *to_gart_domain(struct iommu_domain *dom)
{
	return container_of(dom, struct gart_domain, domain);
}

/*
 * Any interaction between any block on PPSB and a block on APB or AHB
 * must have these read-back to ensure the APB/AHB bus transaction is
 * complete before initiating activity on the PPSB block.
 */
#define FLUSH_GART_REGS(gart)	((void)readl((gart)->regs + GART_CONFIG))

#define for_each_gart_pte(gart, iova)					\
	for (iova = gart->iovmm_base;					\
	     iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
	     iova += GART_PAGE_SIZE)

static inline void gart_set_pte(struct gart_device *gart,
				unsigned long offs, u32 pte)
{
	writel(offs, gart->regs + GART_ENTRY_ADDR);
	writel(pte, gart->regs + GART_ENTRY_DATA);

	dev_dbg(gart->dev, "%s %08lx:%08x\n",
		 pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK);
}

static inline unsigned long gart_read_pte(struct gart_device *gart,
					  unsigned long offs)
{
	unsigned long pte;

	writel(offs, gart->regs + GART_ENTRY_ADDR);
	pte = readl(gart->regs + GART_ENTRY_DATA);

	return pte;
}

static void do_gart_setup(struct gart_device *gart, const u32 *data)
{
	unsigned long iova;

	for_each_gart_pte(gart, iova)
		gart_set_pte(gart, iova, data ? *(data++) : 0);

	writel(1, gart->regs + GART_CONFIG);
	FLUSH_GART_REGS(gart);
}

#ifdef DEBUG
static void gart_dump_table(struct gart_device *gart)
{
	unsigned long iova;
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	for_each_gart_pte(gart, iova) {
		unsigned long pte;

		pte = gart_read_pte(gart, iova);

		dev_dbg(gart->dev, "%s %08lx:%08lx\n",
			(GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ",
			iova, pte & GART_PAGE_MASK);
	}
	spin_unlock_irqrestore(&gart->pte_lock, flags);
}
#else
static inline void gart_dump_table(struct gart_device *gart)
{
}
#endif

static inline bool gart_iova_range_valid(struct gart_device *gart,
					 unsigned long iova, size_t bytes)
{
	unsigned long iova_start, iova_end, gart_start, gart_end;

	iova_start = iova;
	iova_end = iova_start + bytes - 1;
	gart_start = gart->iovmm_base;
	gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;

	if (iova_start < gart_start)
		return false;
	if (iova_end > gart_end)
		return false;
	return true;
}

static int gart_iommu_attach_dev(struct iommu_domain *domain,
				 struct device *dev)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;
	struct gart_client *client, *c;
	int err = 0;

	client = kzalloc(sizeof(*c), GFP_KERNEL);
	if (!client)
		return -ENOMEM;
	client->dev = dev;

	spin_lock(&gart->client_lock);
	list_for_each_entry(c, &gart->client, list) {
		if (c->dev == dev) {
			dev_err(gart->dev,
				"%s is already attached\n", dev_name(dev));
			err = -EINVAL;
			goto fail;
		}
	}
	if (gart->active_domain && gart->active_domain != domain) {
		dev_err(gart->dev, "Only one domain can be active at a time\n");
		err = -EINVAL;
		goto fail;
	}
	gart->active_domain = domain;
	list_add(&client->list, &gart->client);
	spin_unlock(&gart->client_lock);
	dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));
	return 0;

fail:
	kfree(client);
	spin_unlock(&gart->client_lock);
	return err;
}

static void __gart_iommu_detach_dev(struct iommu_domain *domain,
				    struct device *dev)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;
	struct gart_client *c;

	list_for_each_entry(c, &gart->client, list) {
		if (c->dev == dev) {
			list_del(&c->list);
			kfree(c);
			if (list_empty(&gart->client))
				gart->active_domain = NULL;
			dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));
			return;
		}
	}

	dev_err(gart->dev, "Couldn't find %s to detach\n", dev_name(dev));
}

static void gart_iommu_detach_dev(struct iommu_domain *domain,
				  struct device *dev)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;

	spin_lock(&gart->client_lock);
	__gart_iommu_detach_dev(domain, dev);
	spin_unlock(&gart->client_lock);
}

static struct iommu_domain *gart_iommu_domain_alloc(unsigned type)
{
	struct gart_domain *gart_domain;
	struct gart_device *gart;

	if (type != IOMMU_DOMAIN_UNMANAGED)
		return NULL;

	gart = gart_handle;
	if (!gart)
		return NULL;

	gart_domain = kzalloc(sizeof(*gart_domain), GFP_KERNEL);
	if (!gart_domain)
		return NULL;

	gart_domain->gart = gart;
	gart_domain->domain.geometry.aperture_start = gart->iovmm_base;
	gart_domain->domain.geometry.aperture_end = gart->iovmm_base +
					gart->page_count * GART_PAGE_SIZE - 1;
	gart_domain->domain.geometry.force_aperture = true;

	return &gart_domain->domain;
}

static void gart_iommu_domain_free(struct iommu_domain *domain)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;

	if (gart) {
		spin_lock(&gart->client_lock);
		if (!list_empty(&gart->client)) {
			struct gart_client *c, *tmp;

			list_for_each_entry_safe(c, tmp, &gart->client, list)
				__gart_iommu_detach_dev(domain, c->dev);
		}
		spin_unlock(&gart->client_lock);
	}

	kfree(gart_domain);
}

static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
			  phys_addr_t pa, size_t bytes, int prot)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;
	unsigned long flags;
	unsigned long pfn;
	unsigned long pte;

	if (!gart_iova_range_valid(gart, iova, bytes))
		return -EINVAL;

	spin_lock_irqsave(&gart->pte_lock, flags);
	pfn = __phys_to_pfn(pa);
	if (!pfn_valid(pfn)) {
		dev_err(gart->dev, "Invalid page: %pa\n", &pa);
		spin_unlock_irqrestore(&gart->pte_lock, flags);
		return -EINVAL;
	}
	if (gart_debug) {
		pte = gart_read_pte(gart, iova);
		if (pte & GART_ENTRY_PHYS_ADDR_VALID) {
			spin_unlock_irqrestore(&gart->pte_lock, flags);
			dev_err(gart->dev, "Page entry is in-use\n");
			return -EBUSY;
		}
	}
	gart_set_pte(gart, iova, GART_PTE(pfn));
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
			       size_t bytes)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;
	unsigned long flags;

	if (!gart_iova_range_valid(gart, iova, bytes))
		return 0;

	spin_lock_irqsave(&gart->pte_lock, flags);
	gart_set_pte(gart, iova, 0);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return bytes;
}

static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
					   dma_addr_t iova)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;
	unsigned long pte;
	phys_addr_t pa;
	unsigned long flags;

	if (!gart_iova_range_valid(gart, iova, 0))
		return -EINVAL;

	spin_lock_irqsave(&gart->pte_lock, flags);
	pte = gart_read_pte(gart, iova);
	spin_unlock_irqrestore(&gart->pte_lock, flags);

	pa = (pte & GART_PAGE_MASK);
	if (!pfn_valid(__phys_to_pfn(pa))) {
		dev_err(gart->dev, "No entry for %08llx:%pa\n",
			 (unsigned long long)iova, &pa);
		gart_dump_table(gart);
		return -EINVAL;
	}
	return pa;
}

static bool gart_iommu_capable(enum iommu_cap cap)
{
	return false;
}

static int gart_iommu_add_device(struct device *dev)
{
	struct iommu_group *group;

	if (!dev->iommu_fwspec)
		return -ENODEV;

	group = iommu_group_get_for_dev(dev);
	if (IS_ERR(group))
		return PTR_ERR(group);

	iommu_group_put(group);

	iommu_device_link(&gart_handle->iommu, dev);

	return 0;
}

static void gart_iommu_remove_device(struct device *dev)
{
	iommu_group_remove_device(dev);
	iommu_device_unlink(&gart_handle->iommu, dev);
}

static int gart_iommu_of_xlate(struct device *dev,
			       struct of_phandle_args *args)
{
	return 0;
}

static void gart_iommu_sync(struct iommu_domain *domain)
{
	struct gart_domain *gart_domain = to_gart_domain(domain);
	struct gart_device *gart = gart_domain->gart;

	FLUSH_GART_REGS(gart);
}

static const struct iommu_ops gart_iommu_ops = {
	.capable	= gart_iommu_capable,
	.domain_alloc	= gart_iommu_domain_alloc,
	.domain_free	= gart_iommu_domain_free,
	.attach_dev	= gart_iommu_attach_dev,
	.detach_dev	= gart_iommu_detach_dev,
	.add_device	= gart_iommu_add_device,
	.remove_device	= gart_iommu_remove_device,
	.device_group	= generic_device_group,
	.map		= gart_iommu_map,
	.unmap		= gart_iommu_unmap,
	.iova_to_phys	= gart_iommu_iova_to_phys,
	.pgsize_bitmap	= GART_IOMMU_PGSIZES,
	.of_xlate	= gart_iommu_of_xlate,
	.iotlb_sync_map	= gart_iommu_sync,
	.iotlb_sync	= gart_iommu_sync,
};

int tegra_gart_suspend(struct gart_device *gart)
{
	unsigned long iova;
	u32 *data = gart->savedata;
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	for_each_gart_pte(gart, iova)
		*(data++) = gart_read_pte(gart, iova);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

int tegra_gart_resume(struct gart_device *gart)
{
	unsigned long flags;

	spin_lock_irqsave(&gart->pte_lock, flags);
	do_gart_setup(gart, gart->savedata);
	spin_unlock_irqrestore(&gart->pte_lock, flags);
	return 0;
}

struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc)
{
	struct gart_device *gart;
	struct resource *res_remap;
	void __iomem *gart_regs;
	int ret;

	BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);

	/* the GART memory aperture is required */
	res_remap = platform_get_resource(to_platform_device(dev),
					  IORESOURCE_MEM, 1);
	if (!res_remap) {
		dev_err(dev, "GART memory aperture expected\n");
		return ERR_PTR(-ENXIO);
	}

	gart = kzalloc(sizeof(*gart), GFP_KERNEL);
	if (!gart) {
		dev_err(dev, "failed to allocate gart_device\n");
		return ERR_PTR(-ENOMEM);
	}

	ret = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart");
	if (ret) {
		dev_err(dev, "Failed to register IOMMU in sysfs\n");
		goto free_gart;
	}

	iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
	iommu_device_set_fwnode(&gart->iommu, dev->fwnode);

	ret = iommu_device_register(&gart->iommu);
	if (ret) {
		dev_err(dev, "Failed to register IOMMU\n");
		goto remove_sysfs;
	}

	gart->dev = dev;
	gart_regs = mc->regs + GART_REG_BASE;
	spin_lock_init(&gart->pte_lock);
	spin_lock_init(&gart->client_lock);
	INIT_LIST_HEAD(&gart->client);
	gart->regs = gart_regs;
	gart->iovmm_base = (dma_addr_t)res_remap->start;
	gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);

	gart->savedata = vmalloc(array_size(sizeof(u32), gart->page_count));
	if (!gart->savedata) {
		dev_err(dev, "failed to allocate context save area\n");
		ret = -ENOMEM;
		goto unregister_iommu;
	}

	do_gart_setup(gart, NULL);

	gart_handle = gart;

	return gart;

unregister_iommu:
	iommu_device_unregister(&gart->iommu);
remove_sysfs:
	iommu_device_sysfs_remove(&gart->iommu);
free_gart:
	kfree(gart);

	return ERR_PTR(ret);
}

module_param(gart_debug, bool, 0644);
MODULE_PARM_DESC(gart_debug, "Enable GART debugging");
Commit	Line	Data
d53e54b4 HD	1	/*
	2	* IOMMU API for GART in Tegra20
	3	*
	4	* Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved.
	5	*
39fcbbcc PG	6	* Author: Hiroshi DOYU <hdoyu@nvidia.com>
39fcbbcc PG	7	*
d53e54b4 HD	8	* This program is free software; you can redistribute it and/or modify it
	9	* under the terms and conditions of the GNU General Public License,
	10	* version 2, as published by the Free Software Foundation.
	11	*
	12	* This program is distributed in the hope it will be useful, but WITHOUT
	13	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	14	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	15	* more details.
	16	*
	17	* You should have received a copy of the GNU General Public License along with
	18	* this program; if not, write to the Free Software Foundation, Inc.,
	19	* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
	20	*/
	21
5dd82cdb DO	22	#define dev_fmt(fmt) "gart: " fmt
5dd82cdb DO	23
4f821c10 DO	24	#include <linux/io.h>
	25	#include <linux/iommu.h>
	26	#include <linux/list.h>
39fcbbcc	27	#include <linux/moduleparam.h>
ce2785a7	28	#include <linux/platform_device.h>
d53e54b4	29	#include <linux/slab.h>
4f821c10	30	#include <linux/spinlock.h>
d53e54b4	31	#include <linux/vmalloc.h>
d53e54b4	32
ce2785a7 DO	33	#include <soc/tegra/mc.h>
ce2785a7 DO	34
d53e54b4 HD	35	/* bitmap of the page sizes currently supported */
	36	#define GART_IOMMU_PGSIZES (SZ_4K)
	37
774dfc9b HD	38	#define GART_REG_BASE 0x24
	39	#define GART_CONFIG (0x24 - GART_REG_BASE)
	40	#define GART_ENTRY_ADDR (0x28 - GART_REG_BASE)
	41	#define GART_ENTRY_DATA (0x2c - GART_REG_BASE)
d53e54b4 HD	42	#define GART_ENTRY_PHYS_ADDR_VALID (1 << 31)
	43
	44	#define GART_PAGE_SHIFT 12
	45	#define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT)
	46	#define GART_PAGE_MASK \
	47	(~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID)
	48
	49	struct gart_client {
	50	struct device *dev;
	51	struct list_head list;
	52	};
	53
	54	struct gart_device {
	55	void __iomem *regs;
	56	u32 *savedata;
	57	u32 page_count; /* total remappable size */
	58	dma_addr_t iovmm_base; /* offset to vmm_area */
	59	spinlock_t pte_lock; /* for pagetable */
	60	struct list_head client;
	61	spinlock_t client_lock; /* for client list */
7d849b7b	62	struct iommu_domain active_domain; / current active domain */
d53e54b4	63	struct device *dev;
c184ae83 JR	64
c184ae83 JR	65	struct iommu_device iommu; /* IOMMU Core handle */
d53e54b4 HD	66	};
d53e54b4 HD	67
b5cbb386 JR	68	struct gart_domain {
	69	struct iommu_domain domain; /* generic domain handle */
	70	struct gart_device gart; / link to gart device */
	71	};
	72
d53e54b4 HD	73	static struct gart_device gart_handle; / unique for a system */
d53e54b4 HD	74
40c9b882 DO	75	static bool gart_debug;
40c9b882 DO	76
d53e54b4 HD	77	#define GART_PTE(_pfn) \
	78	(GART_ENTRY_PHYS_ADDR_VALID \| ((_pfn) << PAGE_SHIFT))
	79
b5cbb386 JR	80	static struct gart_domain to_gart_domain(struct iommu_domain dom)
	81	{
	82	return container_of(dom, struct gart_domain, domain);
	83	}
	84
d53e54b4 HD	85	/*
	86	* Any interaction between any block on PPSB and a block on APB or AHB
	87	* must have these read-back to ensure the APB/AHB bus transaction is
	88	* complete before initiating activity on the PPSB block.
	89	*/
	90	#define FLUSH_GART_REGS(gart) ((void)readl((gart)->regs + GART_CONFIG))
	91
	92	#define for_each_gart_pte(gart, iova) \
	93	for (iova = gart->iovmm_base; \
	94	iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
	95	iova += GART_PAGE_SIZE)
	96
	97	static inline void gart_set_pte(struct gart_device *gart,
	98	unsigned long offs, u32 pte)
	99	{
	100	writel(offs, gart->regs + GART_ENTRY_ADDR);
	101	writel(pte, gart->regs + GART_ENTRY_DATA);
	102
	103	dev_dbg(gart->dev, "%s %08lx:%08x\n",
	104	pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK);
	105	}
	106
	107	static inline unsigned long gart_read_pte(struct gart_device *gart,
	108	unsigned long offs)
	109	{
	110	unsigned long pte;
	111
	112	writel(offs, gart->regs + GART_ENTRY_ADDR);
	113	pte = readl(gart->regs + GART_ENTRY_DATA);
	114
	115	return pte;
	116	}
	117
	118	static void do_gart_setup(struct gart_device gart, const u32 data)
	119	{
	120	unsigned long iova;
	121
	122	for_each_gart_pte(gart, iova)
	123	gart_set_pte(gart, iova, data ? *(data++) : 0);
	124
	125	writel(1, gart->regs + GART_CONFIG);
	126	FLUSH_GART_REGS(gart);
	127	}
	128
	129	#ifdef DEBUG
	130	static void gart_dump_table(struct gart_device *gart)
	131	{
	132	unsigned long iova;
	133	unsigned long flags;
	134
	135	spin_lock_irqsave(&gart->pte_lock, flags);
	136	for_each_gart_pte(gart, iova) {
	137	unsigned long pte;
	138
	139	pte = gart_read_pte(gart, iova);
	140
	141	dev_dbg(gart->dev, "%s %08lx:%08lx\n",
	142	(GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ",
	143	iova, pte & GART_PAGE_MASK);
	144	}
	145	spin_unlock_irqrestore(&gart->pte_lock, flags);
	146	}
	147	#else
	148	static inline void gart_dump_table(struct gart_device *gart)
149	{
150	}
151	#endif
152
153	static inline bool gart_iova_range_valid(struct gart_device *gart,
154	unsigned long iova, size_t bytes)
155	{
156	unsigned long iova_start, iova_end, gart_start, gart_end;
157
158	iova_start = iova;
159	iova_end = iova_start + bytes - 1;
160	gart_start = gart->iovmm_base;
161	gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;
162
163	if (iova_start < gart_start)
164	return false;
165	if (iova_end > gart_end)
166	return false;
167	return true;
168	}
169
170	static int gart_iommu_attach_dev(struct iommu_domain *domain,
171	struct device *dev)
172	{
b5cbb386	173	struct gart_domain *gart_domain = to_gart_domain(domain);
7f65ef01	174	struct gart_device *gart = gart_domain->gart;
d53e54b4 HD	175	struct gart_client client, c;
	176	int err = 0;
	177
167d67d5	178	client = kzalloc(sizeof(*c), GFP_KERNEL);
d53e54b4 HD	179	if (!client)
	180	return -ENOMEM;
	181	client->dev = dev;
	182
	183	spin_lock(&gart->client_lock);
	184	list_for_each_entry(c, &gart->client, list) {
	185	if (c->dev == dev) {
	186	dev_err(gart->dev,
	187	"%s is already attached\n", dev_name(dev));
	188	err = -EINVAL;
	189	goto fail;
	190	}
	191	}
7d849b7b DO	192	if (gart->active_domain && gart->active_domain != domain) {
	193	dev_err(gart->dev, "Only one domain can be active at a time\n");
	194	err = -EINVAL;
	195	goto fail;
	196	}
	197	gart->active_domain = domain;
d53e54b4 HD	198	list_add(&client->list, &gart->client);
	199	spin_unlock(&gart->client_lock);
	200	dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));
	201	return 0;
	202
	203	fail:
167d67d5	204	kfree(client);
d53e54b4 HD	205	spin_unlock(&gart->client_lock);
	206	return err;
	207	}
	208
c3086fad DO	209	static void __gart_iommu_detach_dev(struct iommu_domain *domain,
c3086fad DO	210	struct device *dev)
d53e54b4	211	{
b5cbb386 JR	212	struct gart_domain *gart_domain = to_gart_domain(domain);
b5cbb386 JR	213	struct gart_device *gart = gart_domain->gart;
d53e54b4 HD	214	struct gart_client *c;
d53e54b4 HD	215
d53e54b4 HD	216	list_for_each_entry(c, &gart->client, list) {
	217	if (c->dev == dev) {
	218	list_del(&c->list);
167d67d5	219	kfree(c);
7d849b7b DO	220	if (list_empty(&gart->client))
7d849b7b DO	221	gart->active_domain = NULL;
d53e54b4	222	dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));
c3086fad	223	return;
d53e54b4 HD	224	}
d53e54b4 HD	225	}
c3086fad DO	226
	227	dev_err(gart->dev, "Couldn't find %s to detach\n", dev_name(dev));
	228	}
	229
	230	static void gart_iommu_detach_dev(struct iommu_domain *domain,
	231	struct device *dev)
	232	{
	233	struct gart_domain *gart_domain = to_gart_domain(domain);
	234	struct gart_device *gart = gart_domain->gart;
	235
	236	spin_lock(&gart->client_lock);
	237	__gart_iommu_detach_dev(domain, dev);
d53e54b4 HD	238	spin_unlock(&gart->client_lock);
	239	}
	240
b5cbb386	241	static struct iommu_domain *gart_iommu_domain_alloc(unsigned type)
d53e54b4	242	{
b5cbb386	243	struct gart_domain *gart_domain;
836a8ac9	244	struct gart_device *gart;
d53e54b4	245
b5cbb386 JR	246	if (type != IOMMU_DOMAIN_UNMANAGED)
b5cbb386 JR	247	return NULL;
d53e54b4	248
836a8ac9	249	gart = gart_handle;
d53e54b4	250	if (!gart)
7f65ef01	251	return NULL;
d53e54b4	252
b5cbb386 JR	253	gart_domain = kzalloc(sizeof(*gart_domain), GFP_KERNEL);
	254	if (!gart_domain)
	255	return NULL;
836a8ac9	256
7f65ef01 JR	257	gart_domain->gart = gart;
	258	gart_domain->domain.geometry.aperture_start = gart->iovmm_base;
	259	gart_domain->domain.geometry.aperture_end = gart->iovmm_base +
836a8ac9	260	gart->page_count * GART_PAGE_SIZE - 1;
7f65ef01	261	gart_domain->domain.geometry.force_aperture = true;
836a8ac9	262
b5cbb386	263	return &gart_domain->domain;
d53e54b4 HD	264	}
d53e54b4 HD	265
b5cbb386	266	static void gart_iommu_domain_free(struct iommu_domain *domain)
d53e54b4	267	{
b5cbb386 JR	268	struct gart_domain *gart_domain = to_gart_domain(domain);
b5cbb386 JR	269	struct gart_device *gart = gart_domain->gart;
d53e54b4	270
b5cbb386 JR	271	if (gart) {
	272	spin_lock(&gart->client_lock);
	273	if (!list_empty(&gart->client)) {
8e924910	274	struct gart_client c, tmp;
d53e54b4	275
8e924910	276	list_for_each_entry_safe(c, tmp, &gart->client, list)
c3086fad	277	__gart_iommu_detach_dev(domain, c->dev);
b5cbb386 JR	278	}
b5cbb386 JR	279	spin_unlock(&gart->client_lock);
d53e54b4	280	}
b5cbb386 JR	281
b5cbb386 JR	282	kfree(gart_domain);
d53e54b4 HD	283	}
	284
	285	static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
	286	phys_addr_t pa, size_t bytes, int prot)
	287	{
b5cbb386 JR	288	struct gart_domain *gart_domain = to_gart_domain(domain);
b5cbb386 JR	289	struct gart_device *gart = gart_domain->gart;
d53e54b4 HD	290	unsigned long flags;
d53e54b4 HD	291	unsigned long pfn;
40c9b882	292	unsigned long pte;
d53e54b4 HD	293
	294	if (!gart_iova_range_valid(gart, iova, bytes))
	295	return -EINVAL;
	296
	297	spin_lock_irqsave(&gart->pte_lock, flags);
	298	pfn = __phys_to_pfn(pa);
	299	if (!pfn_valid(pfn)) {
e56b3dab	300	dev_err(gart->dev, "Invalid page: %pa\n", &pa);
09c32533	301	spin_unlock_irqrestore(&gart->pte_lock, flags);
d53e54b4 HD	302	return -EINVAL;
d53e54b4 HD	303	}
40c9b882 DO	304	if (gart_debug) {
	305	pte = gart_read_pte(gart, iova);
	306	if (pte & GART_ENTRY_PHYS_ADDR_VALID) {
	307	spin_unlock_irqrestore(&gart->pte_lock, flags);
	308	dev_err(gart->dev, "Page entry is in-use\n");
	309	return -EBUSY;
	310	}
	311	}
d53e54b4	312	gart_set_pte(gart, iova, GART_PTE(pfn));
d53e54b4 HD	313	spin_unlock_irqrestore(&gart->pte_lock, flags);
	314	return 0;
	315	}
	316
	317	static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
	318	size_t bytes)
	319	{
b5cbb386 JR	320	struct gart_domain *gart_domain = to_gart_domain(domain);
b5cbb386 JR	321	struct gart_device *gart = gart_domain->gart;
d53e54b4 HD	322	unsigned long flags;
	323
	324	if (!gart_iova_range_valid(gart, iova, bytes))
	325	return 0;
	326
	327	spin_lock_irqsave(&gart->pte_lock, flags);
	328	gart_set_pte(gart, iova, 0);
d53e54b4	329	spin_unlock_irqrestore(&gart->pte_lock, flags);
130a2fdf	330	return bytes;
d53e54b4 HD	331	}
	332
	333	static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac	334	dma_addr_t iova)
d53e54b4	335	{
b5cbb386 JR	336	struct gart_domain *gart_domain = to_gart_domain(domain);
b5cbb386 JR	337	struct gart_device *gart = gart_domain->gart;
d53e54b4 HD	338	unsigned long pte;
	339	phys_addr_t pa;
	340	unsigned long flags;
	341
	342	if (!gart_iova_range_valid(gart, iova, 0))
	343	return -EINVAL;
	344
	345	spin_lock_irqsave(&gart->pte_lock, flags);
	346	pte = gart_read_pte(gart, iova);
	347	spin_unlock_irqrestore(&gart->pte_lock, flags);
	348
	349	pa = (pte & GART_PAGE_MASK);
	350	if (!pfn_valid(__phys_to_pfn(pa))) {
e56b3dab TR	351	dev_err(gart->dev, "No entry for %08llx:%pa\n",
e56b3dab TR	352	(unsigned long long)iova, &pa);
d53e54b4 HD	353	gart_dump_table(gart);
	354	return -EINVAL;
	355	}
	356	return pa;
	357	}
	358
7c2aa644	359	static bool gart_iommu_capable(enum iommu_cap cap)
d53e54b4	360	{
7c2aa644	361	return false;
d53e54b4 HD	362	}
d53e54b4 HD	363
15f9a310 RM	364	static int gart_iommu_add_device(struct device *dev)
15f9a310 RM	365	{
4b6f0ea3	366	struct iommu_group *group;
15f9a310	367
4b6f0ea3 DO	368	if (!dev->iommu_fwspec)
	369	return -ENODEV;
	370
	371	group = iommu_group_get_for_dev(dev);
15f9a310 RM	372	if (IS_ERR(group))
	373	return PTR_ERR(group);
	374
	375	iommu_group_put(group);
c184ae83 JR	376
	377	iommu_device_link(&gart_handle->iommu, dev);
	378
15f9a310 RM	379	return 0;
	380	}
	381
	382	static void gart_iommu_remove_device(struct device *dev)
	383	{
	384	iommu_group_remove_device(dev);
c184ae83	385	iommu_device_unlink(&gart_handle->iommu, dev);
15f9a310 RM	386	}
15f9a310 RM	387
4b6f0ea3 DO	388	static int gart_iommu_of_xlate(struct device *dev,
	389	struct of_phandle_args *args)
	390	{
	391	return 0;
	392	}
	393
2fc0ac18 DO	394	static void gart_iommu_sync(struct iommu_domain *domain)
	395	{
	396	struct gart_domain *gart_domain = to_gart_domain(domain);
	397	struct gart_device *gart = gart_domain->gart;
	398
	399	FLUSH_GART_REGS(gart);
	400	}
	401
b22f6434	402	static const struct iommu_ops gart_iommu_ops = {
7c2aa644	403	.capable = gart_iommu_capable,
b5cbb386 JR	404	.domain_alloc = gart_iommu_domain_alloc,
b5cbb386 JR	405	.domain_free = gart_iommu_domain_free,
d53e54b4 HD	406	.attach_dev = gart_iommu_attach_dev,
d53e54b4 HD	407	.detach_dev = gart_iommu_detach_dev,
15f9a310 RM	408	.add_device = gart_iommu_add_device,
	409	.remove_device = gart_iommu_remove_device,
	410	.device_group = generic_device_group,
d53e54b4 HD	411	.map = gart_iommu_map,
	412	.unmap = gart_iommu_unmap,
	413	.iova_to_phys = gart_iommu_iova_to_phys,
d53e54b4	414	.pgsize_bitmap = GART_IOMMU_PGSIZES,
4b6f0ea3	415	.of_xlate = gart_iommu_of_xlate,
2fc0ac18 DO	416	.iotlb_sync_map = gart_iommu_sync,
2fc0ac18 DO	417	.iotlb_sync = gart_iommu_sync,
d53e54b4 HD	418	};
d53e54b4 HD	419
ce2785a7	420	int tegra_gart_suspend(struct gart_device *gart)
d53e54b4	421	{
d53e54b4 HD	422	unsigned long iova;
	423	u32 *data = gart->savedata;
	424	unsigned long flags;
	425
	426	spin_lock_irqsave(&gart->pte_lock, flags);
	427	for_each_gart_pte(gart, iova)
	428	*(data++) = gart_read_pte(gart, iova);
	429	spin_unlock_irqrestore(&gart->pte_lock, flags);
	430	return 0;
	431	}
	432
ce2785a7	433	int tegra_gart_resume(struct gart_device *gart)
d53e54b4	434	{
d53e54b4 HD	435	unsigned long flags;
	436
	437	spin_lock_irqsave(&gart->pte_lock, flags);
	438	do_gart_setup(gart, gart->savedata);
	439	spin_unlock_irqrestore(&gart->pte_lock, flags);
	440	return 0;
	441	}
	442
ce2785a7	443	struct gart_device tegra_gart_probe(struct device dev, struct tegra_mc *mc)
d53e54b4 HD	444	{
d53e54b4 HD	445	struct gart_device *gart;
ce2785a7	446	struct resource *res_remap;
d53e54b4	447	void __iomem *gart_regs;
c184ae83	448	int ret;
d53e54b4	449
d53e54b4 HD	450	BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
	451
	452	/* the GART memory aperture is required */
ce2785a7 DO	453	res_remap = platform_get_resource(to_platform_device(dev),
	454	IORESOURCE_MEM, 1);
	455	if (!res_remap) {
d53e54b4	456	dev_err(dev, "GART memory aperture expected\n");
ce2785a7	457	return ERR_PTR(-ENXIO);
d53e54b4 HD	458	}
d53e54b4 HD	459
167d67d5	460	gart = kzalloc(sizeof(*gart), GFP_KERNEL);
d53e54b4 HD	461	if (!gart) {
d53e54b4 HD	462	dev_err(dev, "failed to allocate gart_device\n");
ce2785a7	463	return ERR_PTR(-ENOMEM);
d53e54b4 HD	464	}
d53e54b4 HD	465
ce2785a7	466	ret = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart");
c184ae83 JR	467	if (ret) {
c184ae83 JR	468	dev_err(dev, "Failed to register IOMMU in sysfs\n");
167d67d5	469	goto free_gart;
c184ae83 JR	470	}
	471
	472	iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
4b6f0ea3	473	iommu_device_set_fwnode(&gart->iommu, dev->fwnode);
c184ae83 JR	474
	475	ret = iommu_device_register(&gart->iommu);
	476	if (ret) {
	477	dev_err(dev, "Failed to register IOMMU\n");
ae95c46d	478	goto remove_sysfs;
c184ae83 JR	479	}
c184ae83 JR	480
ce2785a7 DO	481	gart->dev = dev;
ce2785a7 DO	482	gart_regs = mc->regs + GART_REG_BASE;
d53e54b4 HD	483	spin_lock_init(&gart->pte_lock);
	484	spin_lock_init(&gart->client_lock);
	485	INIT_LIST_HEAD(&gart->client);
	486	gart->regs = gart_regs;
	487	gart->iovmm_base = (dma_addr_t)res_remap->start;
	488	gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);
	489
42bc47b3	490	gart->savedata = vmalloc(array_size(sizeof(u32), gart->page_count));
d53e54b4 HD	491	if (!gart->savedata) {
d53e54b4 HD	492	dev_err(dev, "failed to allocate context save area\n");
ae95c46d DO	493	ret = -ENOMEM;
ae95c46d DO	494	goto unregister_iommu;
d53e54b4 HD	495	}
d53e54b4 HD	496
d53e54b4 HD	497	do_gart_setup(gart, NULL);
	498
	499	gart_handle = gart;
c7e3ca51	500
ce2785a7	501	return gart;
ae95c46d DO	502
	503	unregister_iommu:
	504	iommu_device_unregister(&gart->iommu);
	505	remove_sysfs:
	506	iommu_device_sysfs_remove(&gart->iommu);
167d67d5 DO	507	free_gart:
167d67d5 DO	508	kfree(gart);
ae95c46d	509
ce2785a7	510	return ERR_PTR(ret);
d53e54b4	511	}
d53e54b4	512
39fcbbcc	513	module_param(gart_debug, bool, 0644);
40c9b882	514	MODULE_PARM_DESC(gart_debug, "Enable GART debugging");