[linux-block.git] / arch / sparc / mm / iommu.c

// SPDX-License-Identifier: GPL-2.0
/*
 * iommu.c:  IOMMU specific routines for memory management.
 *
 * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
 * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
 * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
 * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
 */
 
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/dma-map-ops.h>
#include <linux/of.h>
#include <linux/of_device.h>

#include <asm/io.h>
#include <asm/mxcc.h>
#include <asm/mbus.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/bitext.h>
#include <asm/iommu.h>
#include <asm/dma.h>

#include "mm_32.h"

/*
 * This can be sized dynamically, but we will do this
 * only when we have a guidance about actual I/O pressures.
 */
#define IOMMU_RNGE	IOMMU_RNGE_256MB
#define IOMMU_START	0xF0000000
#define IOMMU_WINSIZE	(256*1024*1024U)
#define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 256KB */
#define IOMMU_ORDER	6				/* 4096 * (1<<6) */

static int viking_flush;
/* viking.S */
extern void viking_flush_page(unsigned long page);
extern void viking_mxcc_flush_page(unsigned long page);

/*
 * Values precomputed according to CPU type.
 */
static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
static pgprot_t dvma_prot;		/* Consistent mapping pte flags */

#define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)

static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
static const struct dma_map_ops sbus_iommu_dma_pflush_ops;

static void __init sbus_iommu_init(struct platform_device *op)
{
	struct iommu_struct *iommu;
	unsigned int impl, vers;
	unsigned long *bitmap;
	unsigned long control;
	unsigned long base;
	unsigned long tmp;

	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
	if (!iommu) {
		prom_printf("Unable to allocate iommu structure\n");
		prom_halt();
	}

	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
				 "iommu_regs");
	if (!iommu->regs) {
		prom_printf("Cannot map IOMMU registers\n");
		prom_halt();
	}

	control = sbus_readl(&iommu->regs->control);
	impl = (control & IOMMU_CTRL_IMPL) >> 28;
	vers = (control & IOMMU_CTRL_VERS) >> 24;
	control &= ~(IOMMU_CTRL_RNGE);
	control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
	sbus_writel(control, &iommu->regs->control);

	iommu_invalidate(iommu->regs);
	iommu->start = IOMMU_START;
	iommu->end = 0xffffffff;

	/* Allocate IOMMU page table */
	/* Stupid alignment constraints give me a headache. 
	   We need 256K or 512K or 1M or 2M area aligned to
           its size and current gfp will fortunately give
           it to us. */
        tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
	if (!tmp) {
		prom_printf("Unable to allocate iommu table [0x%lx]\n",
			    IOMMU_NPTES * sizeof(iopte_t));
		prom_halt();
	}
	iommu->page_table = (iopte_t *)tmp;

	/* Initialize new table. */
	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
	flush_cache_all();
	flush_tlb_all();

	base = __pa((unsigned long)iommu->page_table) >> 4;
	sbus_writel(base, &iommu->regs->base);
	iommu_invalidate(iommu->regs);

	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
	if (!bitmap) {
		prom_printf("Unable to allocate iommu bitmap [%d]\n",
			    (int)(IOMMU_NPTES>>3));
		prom_halt();
	}
	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
	/* To be coherent on HyperSparc, the page color of DVMA
	 * and physical addresses must match.
	 */
	if (srmmu_modtype == HyperSparc)
		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
	else
		iommu->usemap.num_colors = 1;

	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
	       impl, vers, iommu->page_table,
	       (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);

	op->dev.archdata.iommu = iommu;

	if (flush_page_for_dma_global)
		op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
	 else
		op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
}

static int __init iommu_init(void)
{
	struct device_node *dp;

	for_each_node_by_name(dp, "iommu") {
		struct platform_device *op = of_find_device_by_node(dp);

		sbus_iommu_init(op);
		of_propagate_archdata(op);
	}

	return 0;
}

subsys_initcall(iommu_init);

/* Flush the iotlb entries to ram. */
/* This could be better if we didn't have to flush whole pages. */
static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
{
	unsigned long start;
	unsigned long end;

	start = (unsigned long)iopte;
	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
	start &= PAGE_MASK;
	if (viking_mxcc_present) {
		while(start < end) {
			viking_mxcc_flush_page(start);
			start += PAGE_SIZE;
		}
	} else if (viking_flush) {
		while(start < end) {
			viking_flush_page(start);
			start += PAGE_SIZE;
		}
	} else {
		while(start < end) {
			__flush_page_to_ram(start);
			start += PAGE_SIZE;
		}
	}
}

static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page,
		unsigned long offset, size_t len, bool per_page_flush)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	phys_addr_t paddr = page_to_phys(page) + offset;
	unsigned long off = paddr & ~PAGE_MASK;
	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
	unsigned long pfn = __phys_to_pfn(paddr);
	unsigned int busa, busa0;
	iopte_t *iopte, *iopte0;
	int ioptex, i;

	/* XXX So what is maxphys for us and how do drivers know it? */
	if (!len || len > 256 * 1024)
		return DMA_MAPPING_ERROR;

	/*
	 * We expect unmapped highmem pages to be not in the cache.
	 * XXX Is this a good assumption?
	 * XXX What if someone else unmaps it here and races us?
	 */
	if (per_page_flush && !PageHighMem(page)) {
		unsigned long vaddr, p;

		vaddr = (unsigned long)page_address(page) + offset;
		for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
			flush_page_for_dma(p);
	}

	/* page color = pfn of page */
	ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
	if (ioptex < 0)
		panic("iommu out");
	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
	iopte0 = &iommu->page_table[ioptex];

	busa = busa0;
	iopte = iopte0;
	for (i = 0; i < npages; i++) {
		iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
		iommu_invalidate_page(iommu->regs, busa);
		busa += PAGE_SIZE;
		iopte++;
		pfn++;
	}

	iommu_flush_iotlb(iopte0, npages);
	return busa0 + off;
}

static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
		struct page *page, unsigned long offset, size_t len,
		enum dma_data_direction dir, unsigned long attrs)
{
	flush_page_for_dma(0);
	return __sbus_iommu_map_page(dev, page, offset, len, false);
}

static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
		struct page *page, unsigned long offset, size_t len,
		enum dma_data_direction dir, unsigned long attrs)
{
	return __sbus_iommu_map_page(dev, page, offset, len, true);
}

static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl,
		int nents, enum dma_data_direction dir, unsigned long attrs,
		bool per_page_flush)
{
	struct scatterlist *sg;
	int j;

	for_each_sg(sgl, sg, nents, j) {
		sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
				sg->offset, sg->length, per_page_flush);
		if (sg->dma_address == DMA_MAPPING_ERROR)
			return -EIO;
		sg->dma_length = sg->length;
	}

	return nents;
}

static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl,
		int nents, enum dma_data_direction dir, unsigned long attrs)
{
	flush_page_for_dma(0);
	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
}

static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl,
		int nents, enum dma_data_direction dir, unsigned long attrs)
{
	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
}

static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
		size_t len, enum dma_data_direction dir, unsigned long attrs)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	unsigned int busa = dma_addr & PAGE_MASK;
	unsigned long off = dma_addr & ~PAGE_MASK;
	unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	unsigned int i;

	BUG_ON(busa < iommu->start);
	for (i = 0; i < npages; i++) {
		iopte_val(iommu->page_table[ioptex + i]) = 0;
		iommu_invalidate_page(iommu->regs, busa);
		busa += PAGE_SIZE;
	}
	bit_map_clear(&iommu->usemap, ioptex, npages);
}

static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl,
		int nents, enum dma_data_direction dir, unsigned long attrs)
{
	struct scatterlist *sg;
	int i;

	for_each_sg(sgl, sg, nents, i) {
		sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
				attrs);
		sg->dma_address = 0x21212121;
	}
}

#ifdef CONFIG_SBUS
static void *sbus_iommu_alloc(struct device *dev, size_t len,
		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	unsigned long va, addr, page, end, ret;
	iopte_t *iopte = iommu->page_table;
	iopte_t *first;
	int ioptex;

	/* XXX So what is maxphys for us and how do drivers know it? */
	if (!len || len > 256 * 1024)
		return NULL;

	len = PAGE_ALIGN(len);
	va = __get_free_pages(gfp | __GFP_ZERO, get_order(len));
	if (va == 0)
		return NULL;

	addr = ret = sparc_dma_alloc_resource(dev, len);
	if (!addr)
		goto out_free_pages;

	BUG_ON((va & ~PAGE_MASK) != 0);
	BUG_ON((addr & ~PAGE_MASK) != 0);
	BUG_ON((len & ~PAGE_MASK) != 0);

	/* page color = physical address */
	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
		addr >> PAGE_SHIFT);
	if (ioptex < 0)
		panic("iommu out");

	iopte += ioptex;
	first = iopte;
	end = addr + len;
	while(addr < end) {
		page = va;
		{
			pmd_t *pmdp;
			pte_t *ptep;

			if (viking_mxcc_present)
				viking_mxcc_flush_page(page);
			else if (viking_flush)
				viking_flush_page(page);
			else
				__flush_page_to_ram(page);

			pmdp = pmd_off_k(addr);
			ptep = pte_offset_map(pmdp, addr);

			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
		}
		iopte_val(*iopte++) =
		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
		addr += PAGE_SIZE;
		va += PAGE_SIZE;
	}
	/* P3: why do we need this?
	 *
	 * DAVEM: Because there are several aspects, none of which
	 *        are handled by a single interface.  Some cpus are
	 *        completely not I/O DMA coherent, and some have
	 *        virtually indexed caches.  The driver DMA flushing
	 *        methods handle the former case, but here during
	 *        IOMMU page table modifications, and usage of non-cacheable
	 *        cpu mappings of pages potentially in the cpu caches, we have
	 *        to handle the latter case as well.
	 */
	flush_cache_all();
	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
	flush_tlb_all();
	iommu_invalidate(iommu->regs);

	*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
	return (void *)ret;

out_free_pages:
	free_pages(va, get_order(len));
	return NULL;
}

static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr,
			       dma_addr_t busa, unsigned long attrs)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	iopte_t *iopte = iommu->page_table;
	struct page *page = virt_to_page(cpu_addr);
	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	unsigned long end;

	if (!sparc_dma_free_resource(cpu_addr, len))
		return;

	BUG_ON((busa & ~PAGE_MASK) != 0);
	BUG_ON((len & ~PAGE_MASK) != 0);

	iopte += ioptex;
	end = busa + len;
	while (busa < end) {
		iopte_val(*iopte++) = 0;
		busa += PAGE_SIZE;
	}
	flush_tlb_all();
	iommu_invalidate(iommu->regs);
	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);

	__free_pages(page, get_order(len));
}
#endif

static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
#ifdef CONFIG_SBUS
	.alloc			= sbus_iommu_alloc,
	.free			= sbus_iommu_free,
#endif
	.map_page		= sbus_iommu_map_page_gflush,
	.unmap_page		= sbus_iommu_unmap_page,
	.map_sg			= sbus_iommu_map_sg_gflush,
	.unmap_sg		= sbus_iommu_unmap_sg,
};

static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
#ifdef CONFIG_SBUS
	.alloc			= sbus_iommu_alloc,
	.free			= sbus_iommu_free,
#endif
	.map_page		= sbus_iommu_map_page_pflush,
	.unmap_page		= sbus_iommu_unmap_page,
	.map_sg			= sbus_iommu_map_sg_pflush,
	.unmap_sg		= sbus_iommu_unmap_sg,
};

void __init ld_mmu_iommu(void)
{
	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
	} else {
		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
	}
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* iommu.c: IOMMU specific routines for memory management.
	4	*
	5	* Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
	6	* Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
	7	* Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
	8	* Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
	9	*/
	10
1da177e4 LT	11	#include <linux/kernel.h>
	12	#include <linux/init.h>
	13	#include <linux/mm.h>
	14	#include <linux/slab.h>
0a0f0d8b	15	#include <linux/dma-map-ops.h>
9dc69230 DM	16	#include <linux/of.h>
9dc69230 DM	17	#include <linux/of_device.h>
1da177e4	18
1da177e4 LT	19	#include <asm/io.h>
	20	#include <asm/mxcc.h>
	21	#include <asm/mbus.h>
	22	#include <asm/cacheflush.h>
	23	#include <asm/tlbflush.h>
	24	#include <asm/bitext.h>
	25	#include <asm/iommu.h>
	26	#include <asm/dma.h>
	27
e8c29c83 SR	28	#include "mm_32.h"
e8c29c83 SR	29
1da177e4 LT	30	/*
	31	* This can be sized dynamically, but we will do this
	32	* only when we have a guidance about actual I/O pressures.
	33	*/
	34	#define IOMMU_RNGE IOMMU_RNGE_256MB
	35	#define IOMMU_START 0xF0000000
	36	#define IOMMU_WINSIZE (25610241024U)
9a0ac1b6	37	#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */
1da177e4 LT	38	#define IOMMU_ORDER 6 /* 4096 * (1<<6) */
1da177e4 LT	39
1da177e4 LT	40	static int viking_flush;
	41	/* viking.S */
	42	extern void viking_flush_page(unsigned long page);
	43	extern void viking_mxcc_flush_page(unsigned long page);
	44
	45	/*
	46	* Values precomputed according to CPU type.
	47	*/
	48	static unsigned int ioperm_noc; /* Consistent mapping iopte flags */
	49	static pgprot_t dvma_prot; /* Consistent mapping pte flags */
	50
	51	#define IOPERM (IOPTE_CACHE \| IOPTE_WRITE \| IOPTE_VALID)
	52	#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) \| (perm)) & ~IOPTE_WAZ)
	53
255a69a9 CH	54	static const struct dma_map_ops sbus_iommu_dma_gflush_ops;
	55	static const struct dma_map_ops sbus_iommu_dma_pflush_ops;
	56
cd4cd730	57	static void __init sbus_iommu_init(struct platform_device *op)
1da177e4	58	{
1da177e4	59	struct iommu_struct *iommu;
e0039348	60	unsigned int impl, vers;
1da177e4	61	unsigned long *bitmap;
f977ea49 SR	62	unsigned long control;
f977ea49 SR	63	unsigned long base;
e0039348 DM	64	unsigned long tmp;
e0039348 DM	65
71cd03b0	66	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
1da177e4 LT	67	if (!iommu) {
	68	prom_printf("Unable to allocate iommu structure\n");
	69	prom_halt();
	70	}
e0039348	71
046e26a8	72	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
e0039348	73	"iommu_regs");
1da177e4 LT	74	if (!iommu->regs) {
	75	prom_printf("Cannot map IOMMU registers\n");
	76	prom_halt();
	77	}
f977ea49 SR	78
	79	control = sbus_readl(&iommu->regs->control);
	80	impl = (control & IOMMU_CTRL_IMPL) >> 28;
	81	vers = (control & IOMMU_CTRL_VERS) >> 24;
	82	control &= ~(IOMMU_CTRL_RNGE);
	83	control \|= (IOMMU_RNGE_256MB \| IOMMU_CTRL_ENAB);
	84	sbus_writel(control, &iommu->regs->control);
	85
1da177e4 LT	86	iommu_invalidate(iommu->regs);
	87	iommu->start = IOMMU_START;
	88	iommu->end = 0xffffffff;
	89
	90	/* Allocate IOMMU page table */
	91	/* Stupid alignment constraints give me a headache.
	92	We need 256K or 512K or 1M or 2M area aligned to
	93	its size and current gfp will fortunately give
	94	it to us. */
	95	tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
	96	if (!tmp) {
5da444aa AM	97	prom_printf("Unable to allocate iommu table [0x%lx]\n",
5da444aa AM	98	IOMMU_NPTES * sizeof(iopte_t));
1da177e4 LT	99	prom_halt();
	100	}
	101	iommu->page_table = (iopte_t *)tmp;
	102
	103	/* Initialize new table. */
	104	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
	105	flush_cache_all();
	106	flush_tlb_all();
f977ea49 SR	107
	108	base = __pa((unsigned long)iommu->page_table) >> 4;
	109	sbus_writel(base, &iommu->regs->base);
1da177e4 LT	110	iommu_invalidate(iommu->regs);
	111
	112	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
	113	if (!bitmap) {
	114	prom_printf("Unable to allocate iommu bitmap [%d]\n",
	115	(int)(IOMMU_NPTES>>3));
	116	prom_halt();
	117	}
	118	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
	119	/* To be coherent on HyperSparc, the page color of DVMA
	120	* and physical addresses must match.
	121	*/
	122	if (srmmu_modtype == HyperSparc)
	123	iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
	124	else
	125	iommu->usemap.num_colors = 1;
	126
046e26a8 DM	127	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
	128	impl, vers, iommu->page_table,
	129	(int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
1da177e4	130
e0039348	131	op->dev.archdata.iommu = iommu;
255a69a9 CH	132
	133	if (flush_page_for_dma_global)
	134	op->dev.dma_ops = &sbus_iommu_dma_gflush_ops;
	135	else
	136	op->dev.dma_ops = &sbus_iommu_dma_pflush_ops;
1da177e4 LT	137	}
1da177e4 LT	138
046e26a8 DM	139	static int __init iommu_init(void)
	140	{
	141	struct device_node *dp;
	142
	143	for_each_node_by_name(dp, "iommu") {
cd4cd730	144	struct platform_device *op = of_find_device_by_node(dp);
046e26a8 DM	145
	146	sbus_iommu_init(op);
	147	of_propagate_archdata(op);
	148	}
	149
	150	return 0;
	151	}
	152
	153	subsys_initcall(iommu_init);
	154
1da177e4 LT	155	/* Flush the iotlb entries to ram. */
	156	/* This could be better if we didn't have to flush whole pages. */
	157	static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
	158	{
	159	unsigned long start;
	160	unsigned long end;
	161
3185d4d2	162	start = (unsigned long)iopte;
1da177e4	163	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
3185d4d2	164	start &= PAGE_MASK;
1da177e4 LT	165	if (viking_mxcc_present) {
	166	while(start < end) {
	167	viking_mxcc_flush_page(start);
	168	start += PAGE_SIZE;
	169	}
	170	} else if (viking_flush) {
	171	while(start < end) {
	172	viking_flush_page(start);
	173	start += PAGE_SIZE;
	174	}
	175	} else {
	176	while(start < end) {
	177	__flush_page_to_ram(start);
	178	start += PAGE_SIZE;
	179	}
	180	}
	181	}
	182
ce65d36f	183	static dma_addr_t __sbus_iommu_map_page(struct device dev, struct page page,
8668b38c	184	unsigned long offset, size_t len, bool per_page_flush)
1da177e4	185	{
376b1371	186	struct iommu_struct *iommu = dev->archdata.iommu;
7e996890 CH	187	phys_addr_t paddr = page_to_phys(page) + offset;
7e996890 CH	188	unsigned long off = paddr & ~PAGE_MASK;
ce65d36f	189	unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
376b1371 CH	190	unsigned long pfn = __phys_to_pfn(paddr);
	191	unsigned int busa, busa0;
	192	iopte_t iopte, iopte0;
	193	int ioptex, i;
8668b38c	194
ce65d36f CH	195	/* XXX So what is maxphys for us and how do drivers know it? */
	196	if (!len \|\| len > 256 * 1024)
	197	return DMA_MAPPING_ERROR;
8668b38c	198
edb1f072 CH	199	/*
	200	* We expect unmapped highmem pages to be not in the cache.
	201	* XXX Is this a good assumption?
	202	* XXX What if someone else unmaps it here and races us?
	203	*/
7e996890 CH	204	if (per_page_flush && !PageHighMem(page)) {
7e996890 CH	205	unsigned long vaddr, p;
8668b38c	206
7e996890 CH	207	vaddr = (unsigned long)page_address(page) + offset;
7e996890 CH	208	for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE)
8668b38c	209	flush_page_for_dma(p);
8668b38c CH	210	}
8668b38c CH	211
376b1371 CH	212	/* page color = pfn of page */
	213	ioptex = bit_map_string_get(&iommu->usemap, npages, pfn);
	214	if (ioptex < 0)
	215	panic("iommu out");
	216	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
	217	iopte0 = &iommu->page_table[ioptex];
	218
	219	busa = busa0;
	220	iopte = iopte0;
	221	for (i = 0; i < npages; i++) {
	222	iopte_val(*iopte) = MKIOPTE(pfn, IOPERM);
	223	iommu_invalidate_page(iommu->regs, busa);
	224	busa += PAGE_SIZE;
	225	iopte++;
	226	pfn++;
	227	}
	228
	229	iommu_flush_iotlb(iopte0, npages);
	230	return busa0 + off;
1da177e4 LT	231	}
1da177e4 LT	232
ce65d36f CH	233	static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev,
	234	struct page *page, unsigned long offset, size_t len,
	235	enum dma_data_direction dir, unsigned long attrs)
1da177e4 LT	236	{
1da177e4 LT	237	flush_page_for_dma(0);
8668b38c	238	return __sbus_iommu_map_page(dev, page, offset, len, false);
1da177e4 LT	239	}
1da177e4 LT	240
ce65d36f CH	241	static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev,
	242	struct page *page, unsigned long offset, size_t len,
	243	enum dma_data_direction dir, unsigned long attrs)
1da177e4	244	{
8668b38c	245	return __sbus_iommu_map_page(dev, page, offset, len, true);
1da177e4 LT	246	}
1da177e4 LT	247
ff5cbec0 CH	248	static int __sbus_iommu_map_sg(struct device dev, struct scatterlist sgl,
	249	int nents, enum dma_data_direction dir, unsigned long attrs,
	250	bool per_page_flush)
1da177e4	251	{
6c503d0d	252	struct scatterlist *sg;
edb1f072	253	int j;
1da177e4	254
6c503d0d	255	for_each_sg(sgl, sg, nents, j) {
edb1f072 CH	256	sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg),
	257	sg->offset, sg->length, per_page_flush);
	258	if (sg->dma_address == DMA_MAPPING_ERROR)
e02373fd	259	return -EIO;
aa83a26a	260	sg->dma_length = sg->length;
1da177e4	261	}
ce65d36f	262
6c503d0d	263	return nents;
1da177e4 LT	264	}
1da177e4 LT	265
ff5cbec0 CH	266	static int sbus_iommu_map_sg_gflush(struct device dev, struct scatterlist sgl,
	267	int nents, enum dma_data_direction dir, unsigned long attrs)
	268	{
	269	flush_page_for_dma(0);
	270	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false);
	271	}
	272
	273	static int sbus_iommu_map_sg_pflush(struct device dev, struct scatterlist sgl,
	274	int nents, enum dma_data_direction dir, unsigned long attrs)
	275	{
	276	return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true);
	277	}
	278
f25b23bc CH	279	static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr,
f25b23bc CH	280	size_t len, enum dma_data_direction dir, unsigned long attrs)
1da177e4	281	{
260489fa	282	struct iommu_struct *iommu = dev->archdata.iommu;
f25b23bc CH	283	unsigned int busa = dma_addr & PAGE_MASK;
	284	unsigned long off = dma_addr & ~PAGE_MASK;
	285	unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	286	unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	287	unsigned int i;
1da177e4	288
1ae61388	289	BUG_ON(busa < iommu->start);
1da177e4 LT	290	for (i = 0; i < npages; i++) {
	291	iopte_val(iommu->page_table[ioptex + i]) = 0;
	292	iommu_invalidate_page(iommu->regs, busa);
	293	busa += PAGE_SIZE;
	294	}
	295	bit_map_clear(&iommu->usemap, ioptex, npages);
	296	}
	297
6c503d0d CH	298	static void sbus_iommu_unmap_sg(struct device dev, struct scatterlist sgl,
6c503d0d CH	299	int nents, enum dma_data_direction dir, unsigned long attrs)
1da177e4	300	{
6c503d0d	301	struct scatterlist *sg;
a7fce1f7	302	int i;
1da177e4	303
6c503d0d	304	for_each_sg(sgl, sg, nents, i) {
a7fce1f7 CH	305	sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir,
a7fce1f7 CH	306	attrs);
aa83a26a	307	sg->dma_address = 0x21212121;
1da177e4 LT	308	}
	309	}
	310
	311	#ifdef CONFIG_SBUS
ce65d36f CH	312	static void sbus_iommu_alloc(struct device dev, size_t len,
ce65d36f CH	313	dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
1da177e4	314	{
4b1c5df2	315	struct iommu_struct *iommu = dev->archdata.iommu;
ce65d36f	316	unsigned long va, addr, page, end, ret;
1da177e4 LT	317	iopte_t *iopte = iommu->page_table;
	318	iopte_t *first;
	319	int ioptex;
	320
ce65d36f CH	321	/* XXX So what is maxphys for us and how do drivers know it? */
	322	if (!len \|\| len > 256 * 1024)
	323	return NULL;
	324
	325	len = PAGE_ALIGN(len);
518a2f19	326	va = __get_free_pages(gfp \| __GFP_ZERO, get_order(len));
ce65d36f CH	327	if (va == 0)
	328	return NULL;
	329
	330	addr = ret = sparc_dma_alloc_resource(dev, len);
	331	if (!addr)
	332	goto out_free_pages;
	333
1ae61388 ES	334	BUG_ON((va & ~PAGE_MASK) != 0);
	335	BUG_ON((addr & ~PAGE_MASK) != 0);
	336	BUG_ON((len & ~PAGE_MASK) != 0);
1da177e4 LT	337
	338	/* page color = physical address */
	339	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
	340	addr >> PAGE_SHIFT);
	341	if (ioptex < 0)
	342	panic("iommu out");
	343
	344	iopte += ioptex;
	345	first = iopte;
	346	end = addr + len;
	347	while(addr < end) {
	348	page = va;
	349	{
1da177e4 LT	350	pmd_t *pmdp;
	351	pte_t *ptep;
	352
	353	if (viking_mxcc_present)
	354	viking_mxcc_flush_page(page);
	355	else if (viking_flush)
	356	viking_flush_page(page);
	357	else
	358	__flush_page_to_ram(page);
	359
e05c7b1f	360	pmdp = pmd_off_k(addr);
1da177e4 LT	361	ptep = pte_offset_map(pmdp, addr);
	362
	363	set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
	364	}
	365	iopte_val(*iopte++) =
	366	MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
	367	addr += PAGE_SIZE;
	368	va += PAGE_SIZE;
	369	}
	370	/* P3: why do we need this?
	371	*
	372	* DAVEM: Because there are several aspects, none of which
	373	* are handled by a single interface. Some cpus are
	374	* completely not I/O DMA coherent, and some have
	375	* virtually indexed caches. The driver DMA flushing
	376	* methods handle the former case, but here during
	377	* IOMMU page table modifications, and usage of non-cacheable
	378	* cpu mappings of pages potentially in the cpu caches, we have
	379	* to handle the latter case as well.
	380	*/
	381	flush_cache_all();
	382	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
	383	flush_tlb_all();
	384	iommu_invalidate(iommu->regs);
	385
ce65d36f CH	386	*dma_handle = iommu->start + (ioptex << PAGE_SHIFT);
	387	return (void *)ret;
	388
	389	out_free_pages:
	390	free_pages(va, get_order(len));
	391	return NULL;
1da177e4 LT	392	}
1da177e4 LT	393
ce65d36f CH	394	static void sbus_iommu_free(struct device dev, size_t len, void cpu_addr,
ce65d36f CH	395	dma_addr_t busa, unsigned long attrs)
1da177e4	396	{
4b1c5df2	397	struct iommu_struct *iommu = dev->archdata.iommu;
1da177e4	398	iopte_t *iopte = iommu->page_table;
ce65d36f	399	struct page *page = virt_to_page(cpu_addr);
1da177e4	400	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
ce65d36f CH	401	unsigned long end;
	402
	403	if (!sparc_dma_free_resource(cpu_addr, len))
	404	return;
1da177e4	405
1ae61388 ES	406	BUG_ON((busa & ~PAGE_MASK) != 0);
1ae61388 ES	407	BUG_ON((len & ~PAGE_MASK) != 0);
1da177e4 LT	408
	409	iopte += ioptex;
	410	end = busa + len;
	411	while (busa < end) {
	412	iopte_val(*iopte++) = 0;
	413	busa += PAGE_SIZE;
	414	}
	415	flush_tlb_all();
	416	iommu_invalidate(iommu->regs);
	417	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
ce65d36f CH	418
ce65d36f CH	419	__free_pages(page, get_order(len));
1da177e4	420	}
1da177e4 LT	421	#endif
1da177e4 LT	422
ce65d36f	423	static const struct dma_map_ops sbus_iommu_dma_gflush_ops = {
d894d964	424	#ifdef CONFIG_SBUS
ce65d36f CH	425	.alloc = sbus_iommu_alloc,
ce65d36f CH	426	.free = sbus_iommu_free,
d894d964	427	#endif
ce65d36f CH	428	.map_page = sbus_iommu_map_page_gflush,
	429	.unmap_page = sbus_iommu_unmap_page,
	430	.map_sg = sbus_iommu_map_sg_gflush,
	431	.unmap_sg = sbus_iommu_unmap_sg,
d894d964 DM	432	};
d894d964 DM	433
ce65d36f	434	static const struct dma_map_ops sbus_iommu_dma_pflush_ops = {
d894d964	435	#ifdef CONFIG_SBUS
ce65d36f CH	436	.alloc = sbus_iommu_alloc,
ce65d36f CH	437	.free = sbus_iommu_free,
d894d964	438	#endif
ce65d36f CH	439	.map_page = sbus_iommu_map_page_pflush,
	440	.unmap_page = sbus_iommu_unmap_page,
	441	.map_sg = sbus_iommu_map_sg_pflush,
	442	.unmap_sg = sbus_iommu_unmap_sg,
d894d964 DM	443	};
d894d964 DM	444
1da177e4 LT	445	void __init ld_mmu_iommu(void)
1da177e4 LT	446	{
1da177e4 LT	447	if (viking_mxcc_present \|\| srmmu_modtype == HyperSparc) {
	448	dvma_prot = __pgprot(SRMMU_CACHE \| SRMMU_ET_PTE \| SRMMU_PRIV);
	449	ioperm_noc = IOPTE_CACHE \| IOPTE_WRITE \| IOPTE_VALID;
	450	} else {
	451	dvma_prot = __pgprot(SRMMU_ET_PTE \| SRMMU_PRIV);
	452	ioperm_noc = IOPTE_WRITE \| IOPTE_VALID;
	453	}
	454	}