[linux-2.6-block.git] / arch / powerpc / platforms / cell / iommu.c

/*
 * IOMMU implementation for Cell Broadband Processor Architecture
 * We just establish a linear mapping at boot by setting all the
 * IOPT cache entries in the CPU.
 * The mapping functions should be identical to pci_direct_iommu, 
 * except for the handling of the high order bit that is required
 * by the Spider bridge. These should be split into a separate
 * file at the point where we get a different bridge chip.
 *
 * Copyright (C) 2005 IBM Deutschland Entwicklung GmbH,
 *			 Arnd Bergmann <arndb@de.ibm.com>
 *
 * Based on linear mapping
 * Copyright (C) 2003 Benjamin Herrenschmidt (benh@kernel.crashing.org)
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#undef DEBUG

#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#include <linux/kernel.h>
#include <linux/compiler.h>

#include <asm/sections.h>
#include <asm/iommu.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
#include <asm/pmac_feature.h>
#include <asm/abs_addr.h>
#include <asm/system.h>
#include <asm/ppc-pci.h>
#include <asm/udbg.h>

#include "iommu.h"

static inline unsigned long 
get_iopt_entry(unsigned long real_address, unsigned long ioid,
			 unsigned long prot)
{
	return (prot & IOPT_PROT_MASK)
	     | (IOPT_COHERENT)
	     | (IOPT_ORDER_VC)
	     | (real_address & IOPT_RPN_MASK)
	     | (ioid & IOPT_IOID_MASK);
}

typedef struct {
	unsigned long val;
} ioste;

static inline ioste
mk_ioste(unsigned long val)
{
	ioste ioste = { .val = val, };
	return ioste;
}

static inline ioste
get_iost_entry(unsigned long iopt_base, unsigned long io_address, unsigned page_size)
{
	unsigned long ps;
	unsigned long iostep;
	unsigned long nnpt;
	unsigned long shift;

	switch (page_size) {
	case 0x1000000:
		ps = IOST_PS_16M;
		nnpt = 0;  /* one page per segment */
		shift = 5; /* segment has 16 iopt entries */
		break;

	case 0x100000:
		ps = IOST_PS_1M;
		nnpt = 0;  /* one page per segment */
		shift = 1; /* segment has 256 iopt entries */
		break;

	case 0x10000:
		ps = IOST_PS_64K;
		nnpt = 0x07; /* 8 pages per io page table */
		shift = 0;   /* all entries are used */
		break;

	case 0x1000:
		ps = IOST_PS_4K;
		nnpt = 0x7f; /* 128 pages per io page table */
		shift = 0;   /* all entries are used */
		break;

	default: /* not a known compile time constant */
		{
			/* BUILD_BUG_ON() is not usable here */
			extern void __get_iost_entry_bad_page_size(void);
			__get_iost_entry_bad_page_size();
		}
		break;
	}

	iostep = iopt_base +
			 /* need 8 bytes per iopte */
			(((io_address / page_size * 8)
			 /* align io page tables on 4k page boundaries */
				 << shift) 
			 /* nnpt+1 pages go into each iopt */
				 & ~(nnpt << 12));

	nnpt++; /* this seems to work, but the documentation is not clear
		   about wether we put nnpt or nnpt-1 into the ioste bits.
		   In theory, this can't work for 4k pages. */
	return mk_ioste(IOST_VALID_MASK
			| (iostep & IOST_PT_BASE_MASK)
			| ((nnpt << 5) & IOST_NNPT_MASK)
			| (ps & IOST_PS_MASK));
}

/* compute the address of an io pte */
static inline unsigned long
get_ioptep(ioste iost_entry, unsigned long io_address)
{
	unsigned long iopt_base;
	unsigned long page_size;
	unsigned long page_number;
	unsigned long iopt_offset;

	iopt_base = iost_entry.val & IOST_PT_BASE_MASK;
	page_size = iost_entry.val & IOST_PS_MASK;

	/* decode page size to compute page number */
	page_number = (io_address & 0x0fffffff) >> (10 + 2 * page_size);
	/* page number is an offset into the io page table */
	iopt_offset = (page_number << 3) & 0x7fff8ul;
	return iopt_base + iopt_offset;
}

/* compute the tag field of the iopt cache entry */
static inline unsigned long
get_ioc_tag(ioste iost_entry, unsigned long io_address)
{
	unsigned long iopte = get_ioptep(iost_entry, io_address);

	return IOPT_VALID_MASK
	     | ((iopte & 0x00000000000000ff8ul) >> 3)
	     | ((iopte & 0x0000003fffffc0000ul) >> 9);
}

/* compute the hashed 6 bit index for the 4-way associative pte cache */
static inline unsigned long
get_ioc_hash(ioste iost_entry, unsigned long io_address)
{
	unsigned long iopte = get_ioptep(iost_entry, io_address);

	return ((iopte & 0x000000000000001f8ul) >> 3)
	     ^ ((iopte & 0x00000000000020000ul) >> 17)
	     ^ ((iopte & 0x00000000000010000ul) >> 15)
	     ^ ((iopte & 0x00000000000008000ul) >> 13)
	     ^ ((iopte & 0x00000000000004000ul) >> 11)
	     ^ ((iopte & 0x00000000000002000ul) >> 9)
	     ^ ((iopte & 0x00000000000001000ul) >> 7);
}

/* same as above, but pretend that we have a simpler 1-way associative
   pte cache with an 8 bit index */
static inline unsigned long
get_ioc_hash_1way(ioste iost_entry, unsigned long io_address)
{
	unsigned long iopte = get_ioptep(iost_entry, io_address);

	return ((iopte & 0x000000000000001f8ul) >> 3)
	     ^ ((iopte & 0x00000000000020000ul) >> 17)
	     ^ ((iopte & 0x00000000000010000ul) >> 15)
	     ^ ((iopte & 0x00000000000008000ul) >> 13)
	     ^ ((iopte & 0x00000000000004000ul) >> 11)
	     ^ ((iopte & 0x00000000000002000ul) >> 9)
	     ^ ((iopte & 0x00000000000001000ul) >> 7)
	     ^ ((iopte & 0x0000000000000c000ul) >> 8);
}

static inline ioste
get_iost_cache(void __iomem *base, unsigned long index)
{
	unsigned long __iomem *p = (base + IOC_ST_CACHE_DIR);
	return mk_ioste(in_be64(&p[index]));
}

static inline void
set_iost_cache(void __iomem *base, unsigned long index, ioste ste)
{
	unsigned long __iomem *p = (base + IOC_ST_CACHE_DIR);
	pr_debug("ioste %02lx was %016lx, store %016lx", index,
			get_iost_cache(base, index).val, ste.val);
	out_be64(&p[index], ste.val);
	pr_debug(" now %016lx\n", get_iost_cache(base, index).val);
}

static inline unsigned long
get_iopt_cache(void __iomem *base, unsigned long index, unsigned long *tag)
{
	unsigned long __iomem *tags = (void *)(base + IOC_PT_CACHE_DIR);
	unsigned long __iomem *p = (void *)(base + IOC_PT_CACHE_REG);	

	*tag = tags[index];
	rmb();
	return *p;
}

static inline void
set_iopt_cache(void __iomem *base, unsigned long index,
		 unsigned long tag, unsigned long val)
{
	unsigned long __iomem *tags = base + IOC_PT_CACHE_DIR;
	unsigned long __iomem *p = base + IOC_PT_CACHE_REG;

	out_be64(p, val);
	out_be64(&tags[index], tag);
}

static inline void
set_iost_origin(void __iomem *base)
{
	unsigned long __iomem *p = base + IOC_ST_ORIGIN;
	unsigned long origin = IOSTO_ENABLE | IOSTO_SW;

	pr_debug("iost_origin %016lx, now %016lx\n", in_be64(p), origin);
	out_be64(p, origin);
}

static inline void
set_iocmd_config(void __iomem *base)
{
	unsigned long __iomem *p = base + 0xc00;
	unsigned long conf;

	conf = in_be64(p);
	pr_debug("iost_conf %016lx, now %016lx\n", conf, conf | IOCMD_CONF_TE);
	out_be64(p, conf | IOCMD_CONF_TE);
}

static void enable_mapping(void __iomem *base, void __iomem *mmio_base)
{
	set_iocmd_config(base);
	set_iost_origin(mmio_base);
}

struct cell_iommu {
	unsigned long base;
	unsigned long mmio_base;
	void __iomem *mapped_base;
	void __iomem *mapped_mmio_base;
};

static struct cell_iommu cell_iommus[NR_CPUS];

/* initialize the iommu to support a simple linear mapping
 * for each DMA window used by any device. For now, we
 * happen to know that there is only one DMA window in use,
 * starting at iopt_phys_offset. */
static void cell_do_map_iommu(struct cell_iommu *iommu,
			      unsigned int ioid,
			      unsigned long map_start,
			      unsigned long map_size)
{
	unsigned long io_address, real_address;
	void __iomem *ioc_base, *ioc_mmio_base;
	ioste ioste;
	unsigned long index;

	/* we pretend the io page table was at a very high address */
	const unsigned long fake_iopt = 0x10000000000ul;
	const unsigned long io_page_size = 0x1000000; /* use 16M pages */
	const unsigned long io_segment_size = 0x10000000; /* 256M */

	ioc_base = iommu->mapped_base;
	ioc_mmio_base = iommu->mapped_mmio_base;

	for (real_address = 0, io_address = map_start;
	     io_address <= map_start + map_size;
	     real_address += io_page_size, io_address += io_page_size) {
		ioste = get_iost_entry(fake_iopt, io_address, io_page_size);
		if ((real_address % io_segment_size) == 0) /* segment start */
			set_iost_cache(ioc_mmio_base,
				       io_address >> 28, ioste);
		index = get_ioc_hash_1way(ioste, io_address);
		pr_debug("addr %08lx, index %02lx, ioste %016lx\n",
					 io_address, index, ioste.val);
		set_iopt_cache(ioc_mmio_base,
			get_ioc_hash_1way(ioste, io_address),
			get_ioc_tag(ioste, io_address),
			get_iopt_entry(real_address, ioid, IOPT_PROT_RW));
	}
}

static void pci_dma_cell_bus_setup(struct pci_bus *b)
{
	const unsigned int *ioid;
	unsigned long map_start, map_size, token;
	const unsigned long *dma_window;
	struct cell_iommu *iommu;
	struct device_node *d;

	d = pci_bus_to_OF_node(b);

	ioid = get_property(d, "ioid", NULL);
	if (!ioid)
		pr_debug("No ioid entry found !\n");

	dma_window = get_property(d, "ibm,dma-window", NULL);
	if (!dma_window)
		pr_debug("No ibm,dma-window entry found !\n");

	map_start = dma_window[1];
	map_size = dma_window[2];
	token = dma_window[0] >> 32;

	iommu = &cell_iommus[token];

	cell_do_map_iommu(iommu, *ioid, map_start, map_size);
}


static int cell_map_iommu_hardcoded(int num_nodes)
{
	struct cell_iommu *iommu = NULL;

	pr_debug("%s(%d): Using hardcoded defaults\n", __FUNCTION__, __LINE__);

	/* node 0 */
	iommu = &cell_iommus[0];
	iommu->mapped_base = ioremap(0x20000511000ul, 0x1000);
	iommu->mapped_mmio_base = ioremap(0x20000510000ul, 0x1000);

	enable_mapping(iommu->mapped_base, iommu->mapped_mmio_base);

	cell_do_map_iommu(iommu, 0x048a,
			  0x20000000ul,0x20000000ul);

	if (num_nodes < 2)
		return 0;

	/* node 1 */
	iommu = &cell_iommus[1];
	iommu->mapped_base = ioremap(0x30000511000ul, 0x1000);
	iommu->mapped_mmio_base = ioremap(0x30000510000ul, 0x1000);

	enable_mapping(iommu->mapped_base, iommu->mapped_mmio_base);

	cell_do_map_iommu(iommu, 0x048a,
			  0x20000000,0x20000000ul);

	return 0;
}


static int cell_map_iommu(void)
{
	unsigned int num_nodes = 0;
	const unsigned int *node_id;
	const unsigned long *base, *mmio_base;
	struct device_node *dn;
	struct cell_iommu *iommu = NULL;

	/* determine number of nodes (=iommus) */
	pr_debug("%s(%d): determining number of nodes...", __FUNCTION__, __LINE__);
	for(dn = of_find_node_by_type(NULL, "cpu");
	    dn;
	    dn = of_find_node_by_type(dn, "cpu")) {
		node_id = get_property(dn, "node-id", NULL);

		if (num_nodes < *node_id)
			num_nodes = *node_id;
		}

	num_nodes++;
	pr_debug("%i found.\n", num_nodes);

	/* map the iommu registers for each node */
	pr_debug("%s(%d): Looping through nodes\n", __FUNCTION__, __LINE__);
	for(dn = of_find_node_by_type(NULL, "cpu");
	    dn;
	    dn = of_find_node_by_type(dn, "cpu")) {

		node_id = get_property(dn, "node-id", NULL);
		base = get_property(dn, "ioc-cache", NULL);
		mmio_base = get_property(dn, "ioc-translation", NULL);

		if (!base || !mmio_base || !node_id)
			return cell_map_iommu_hardcoded(num_nodes);

		iommu = &cell_iommus[*node_id];
		iommu->base = *base;
		iommu->mmio_base = *mmio_base;

		iommu->mapped_base = ioremap(*base, 0x1000);
		iommu->mapped_mmio_base = ioremap(*mmio_base, 0x1000);

		enable_mapping(iommu->mapped_base,
			       iommu->mapped_mmio_base);

		/* everything else will be done in iommu_bus_setup */
	}

	return 1;
}

static void *cell_alloc_coherent(struct device *hwdev, size_t size,
			   dma_addr_t *dma_handle, gfp_t flag)
{
	void *ret;

	ret = (void *)__get_free_pages(flag, get_order(size));
	if (ret != NULL) {
		memset(ret, 0, size);
		*dma_handle = virt_to_abs(ret) | CELL_DMA_VALID;
	}
	return ret;
}

static void cell_free_coherent(struct device *hwdev, size_t size,
				 void *vaddr, dma_addr_t dma_handle)
{
	free_pages((unsigned long)vaddr, get_order(size));
}

static dma_addr_t cell_map_single(struct device *hwdev, void *ptr,
		size_t size, enum dma_data_direction direction)
{
	return virt_to_abs(ptr) | CELL_DMA_VALID;
}

static void cell_unmap_single(struct device *hwdev, dma_addr_t dma_addr,
		size_t size, enum dma_data_direction direction)
{
}

static int cell_map_sg(struct device *hwdev, struct scatterlist *sg,
		int nents, enum dma_data_direction direction)
{
	int i;

	for (i = 0; i < nents; i++, sg++) {
		sg->dma_address = (page_to_phys(sg->page) + sg->offset)
					| CELL_DMA_VALID;
		sg->dma_length = sg->length;
	}

	return nents;
}

static void cell_unmap_sg(struct device *hwdev, struct scatterlist *sg,
		int nents, enum dma_data_direction direction)
{
}

static int cell_dma_supported(struct device *dev, u64 mask)
{
	return mask < 0x100000000ull;
}

static struct dma_mapping_ops cell_iommu_ops = {
	.alloc_coherent = cell_alloc_coherent,
	.free_coherent = cell_free_coherent,
	.map_single = cell_map_single,
	.unmap_single = cell_unmap_single,
	.map_sg = cell_map_sg,
	.unmap_sg = cell_unmap_sg,
	.dma_supported = cell_dma_supported,
};

void cell_init_iommu(void)
{
	int setup_bus = 0;

	if (of_find_node_by_path("/mambo")) {
		pr_info("Not using iommu on systemsim\n");
	} else {

		if (!(of_chosen &&
		      get_property(of_chosen, "linux,iommu-off", NULL)))
			setup_bus = cell_map_iommu();

		if (setup_bus) {
			pr_debug("%s: IOMMU mapping activated\n", __FUNCTION__);
			ppc_md.pci_dma_bus_setup = pci_dma_cell_bus_setup;
		} else {
			pr_debug("%s: IOMMU mapping activated, "
				 "no device action necessary\n", __FUNCTION__);
			/* Direct I/O, IOMMU off */
		}
	}

	pci_dma_ops = &cell_iommu_ops;
}
Commit	Line	Data
ae209cf1	1	/*
f3f66f59	2	* IOMMU implementation for Cell Broadband Processor Architecture
ae209cf1 AB	3	* We just establish a linear mapping at boot by setting all the
	4	* IOPT cache entries in the CPU.
	5	* The mapping functions should be identical to pci_direct_iommu,
	6	* except for the handling of the high order bit that is required
	7	* by the Spider bridge. These should be split into a separate
	8	* file at the point where we get a different bridge chip.
	9	*
	10	* Copyright (C) 2005 IBM Deutschland Entwicklung GmbH,
	11	* Arnd Bergmann <arndb@de.ibm.com>
	12	*
	13	* Based on linear mapping
	14	* Copyright (C) 2003 Benjamin Herrenschmidt (benh@kernel.crashing.org)
	15	*
	16	* This program is free software; you can redistribute it and/or
	17	* modify it under the terms of the GNU General Public License
	18	* as published by the Free Software Foundation; either version
	19	* 2 of the License, or (at your option) any later version.
	20	*/
	21
	22	#undef DEBUG
	23
	24	#include <linux/kernel.h>
	25	#include <linux/pci.h>
	26	#include <linux/delay.h>
	27	#include <linux/string.h>
	28	#include <linux/init.h>
	29	#include <linux/bootmem.h>
	30	#include <linux/mm.h>
	31	#include <linux/dma-mapping.h>
49d65b3a JO	32	#include <linux/kernel.h>
49d65b3a JO	33	#include <linux/compiler.h>
ae209cf1 AB	34
	35	#include <asm/sections.h>
	36	#include <asm/iommu.h>
	37	#include <asm/io.h>
	38	#include <asm/prom.h>
	39	#include <asm/pci-bridge.h>
	40	#include <asm/machdep.h>
	41	#include <asm/pmac_feature.h>
	42	#include <asm/abs_addr.h>
	43	#include <asm/system.h>
d387899f	44	#include <asm/ppc-pci.h>
49d65b3a	45	#include <asm/udbg.h>
ae209cf1	46
f3f66f59	47	#include "iommu.h"
ae209cf1 AB	48
	49	static inline unsigned long
	50	get_iopt_entry(unsigned long real_address, unsigned long ioid,
	51	unsigned long prot)
	52	{
	53	return (prot & IOPT_PROT_MASK)
	54	\| (IOPT_COHERENT)
	55	\| (IOPT_ORDER_VC)
	56	\| (real_address & IOPT_RPN_MASK)
	57	\| (ioid & IOPT_IOID_MASK);
	58	}
	59
	60	typedef struct {
	61	unsigned long val;
	62	} ioste;
	63
	64	static inline ioste
	65	mk_ioste(unsigned long val)
	66	{
	67	ioste ioste = { .val = val, };
	68	return ioste;
	69	}
	70
	71	static inline ioste
	72	get_iost_entry(unsigned long iopt_base, unsigned long io_address, unsigned page_size)
	73	{
	74	unsigned long ps;
	75	unsigned long iostep;
	76	unsigned long nnpt;
	77	unsigned long shift;
	78
	79	switch (page_size) {
	80	case 0x1000000:
	81	ps = IOST_PS_16M;
	82	nnpt = 0; /* one page per segment */
	83	shift = 5; /* segment has 16 iopt entries */
	84	break;
	85
	86	case 0x100000:
	87	ps = IOST_PS_1M;
	88	nnpt = 0; /* one page per segment */
	89	shift = 1; /* segment has 256 iopt entries */
	90	break;
	91
	92	case 0x10000:
	93	ps = IOST_PS_64K;
	94	nnpt = 0x07; /* 8 pages per io page table */
	95	shift = 0; /* all entries are used */
	96	break;
	97
	98	case 0x1000:
	99	ps = IOST_PS_4K;
	100	nnpt = 0x7f; /* 128 pages per io page table */
	101	shift = 0; /* all entries are used */
	102	break;
	103
	104	default: /* not a known compile time constant */
c215a16a AV	105	{
	106	/* BUILD_BUG_ON() is not usable here */
	107	extern void __get_iost_entry_bad_page_size(void);
	108	__get_iost_entry_bad_page_size();
	109	}
ae209cf1 AB	110	break;
	111	}
	112
	113	iostep = iopt_base +
	114	/* need 8 bytes per iopte */
	115	(((io_address / page_size * 8)
	116	/* align io page tables on 4k page boundaries */
	117	<< shift)
	118	/* nnpt+1 pages go into each iopt */
	119	& ~(nnpt << 12));
	120
	121	nnpt++; /* this seems to work, but the documentation is not clear
	122	about wether we put nnpt or nnpt-1 into the ioste bits.
	123	In theory, this can't work for 4k pages. */
	124	return mk_ioste(IOST_VALID_MASK
	125	\| (iostep & IOST_PT_BASE_MASK)
	126	\| ((nnpt << 5) & IOST_NNPT_MASK)
	127	\| (ps & IOST_PS_MASK));
	128	}
	129
	130	/* compute the address of an io pte */
	131	static inline unsigned long
	132	get_ioptep(ioste iost_entry, unsigned long io_address)
	133	{
	134	unsigned long iopt_base;
	135	unsigned long page_size;
	136	unsigned long page_number;
	137	unsigned long iopt_offset;
	138
	139	iopt_base = iost_entry.val & IOST_PT_BASE_MASK;
	140	page_size = iost_entry.val & IOST_PS_MASK;
	141
	142	/* decode page size to compute page number */
	143	page_number = (io_address & 0x0fffffff) >> (10 + 2 * page_size);
	144	/* page number is an offset into the io page table */
	145	iopt_offset = (page_number << 3) & 0x7fff8ul;
	146	return iopt_base + iopt_offset;
	147	}
	148
	149	/* compute the tag field of the iopt cache entry */
	150	static inline unsigned long
	151	get_ioc_tag(ioste iost_entry, unsigned long io_address)
	152	{
	153	unsigned long iopte = get_ioptep(iost_entry, io_address);
	154
	155	return IOPT_VALID_MASK
	156	\| ((iopte & 0x00000000000000ff8ul) >> 3)
	157	\| ((iopte & 0x0000003fffffc0000ul) >> 9);
	158	}
	159
	160	/* compute the hashed 6 bit index for the 4-way associative pte cache */
	161	static inline unsigned long
	162	get_ioc_hash(ioste iost_entry, unsigned long io_address)
	163	{
	164	unsigned long iopte = get_ioptep(iost_entry, io_address);
	165
	166	return ((iopte & 0x000000000000001f8ul) >> 3)
	167	^ ((iopte & 0x00000000000020000ul) >> 17)
	168	^ ((iopte & 0x00000000000010000ul) >> 15)
	169	^ ((iopte & 0x00000000000008000ul) >> 13)
	170	^ ((iopte & 0x00000000000004000ul) >> 11)
	171	^ ((iopte & 0x00000000000002000ul) >> 9)
	172	^ ((iopte & 0x00000000000001000ul) >> 7);
	173	}
174
175	/* same as above, but pretend that we have a simpler 1-way associative
176	pte cache with an 8 bit index */
177	static inline unsigned long
178	get_ioc_hash_1way(ioste iost_entry, unsigned long io_address)
179	{
180	unsigned long iopte = get_ioptep(iost_entry, io_address);
181
182	return ((iopte & 0x000000000000001f8ul) >> 3)
183	^ ((iopte & 0x00000000000020000ul) >> 17)
184	^ ((iopte & 0x00000000000010000ul) >> 15)
185	^ ((iopte & 0x00000000000008000ul) >> 13)
186	^ ((iopte & 0x00000000000004000ul) >> 11)
187	^ ((iopte & 0x00000000000002000ul) >> 9)
188	^ ((iopte & 0x00000000000001000ul) >> 7)
189	^ ((iopte & 0x0000000000000c000ul) >> 8);
190	}
191
192	static inline ioste
193	get_iost_cache(void __iomem *base, unsigned long index)
194	{
195	unsigned long __iomem *p = (base + IOC_ST_CACHE_DIR);
196	return mk_ioste(in_be64(&p[index]));
197	}
198
199	static inline void
200	set_iost_cache(void __iomem *base, unsigned long index, ioste ste)
201	{
202	unsigned long __iomem *p = (base + IOC_ST_CACHE_DIR);
203	pr_debug("ioste %02lx was %016lx, store %016lx", index,
204	get_iost_cache(base, index).val, ste.val);
205	out_be64(&p[index], ste.val);
206	pr_debug(" now %016lx\n", get_iost_cache(base, index).val);
207	}
208
209	static inline unsigned long
210	get_iopt_cache(void __iomem base, unsigned long index, unsigned long tag)
211	{
212	unsigned long __iomem tags = (void )(base + IOC_PT_CACHE_DIR);
213	unsigned long __iomem p = (void )(base + IOC_PT_CACHE_REG);
214
215	*tag = tags[index];
216	rmb();
217	return *p;
218	}
219
220	static inline void
221	set_iopt_cache(void __iomem *base, unsigned long index,
222	unsigned long tag, unsigned long val)
223	{
224	unsigned long __iomem *tags = base + IOC_PT_CACHE_DIR;
225	unsigned long __iomem *p = base + IOC_PT_CACHE_REG;
ae209cf1 AB	226
	227	out_be64(p, val);
	228	out_be64(&tags[index], tag);
	229	}
	230
	231	static inline void
	232	set_iost_origin(void __iomem *base)
	233	{
	234	unsigned long __iomem *p = base + IOC_ST_ORIGIN;
	235	unsigned long origin = IOSTO_ENABLE \| IOSTO_SW;
	236
	237	pr_debug("iost_origin %016lx, now %016lx\n", in_be64(p), origin);
	238	out_be64(p, origin);
	239	}
	240
	241	static inline void
	242	set_iocmd_config(void __iomem *base)
	243	{
	244	unsigned long __iomem *p = base + 0xc00;
	245	unsigned long conf;
	246
	247	conf = in_be64(p);
	248	pr_debug("iost_conf %016lx, now %016lx\n", conf, conf \| IOCMD_CONF_TE);
	249	out_be64(p, conf \| IOCMD_CONF_TE);
	250	}
	251
49d65b3a	252	static void enable_mapping(void __iomem base, void __iomem mmio_base)
ae209cf1	253	{
49d65b3a JO	254	set_iocmd_config(base);
49d65b3a JO	255	set_iost_origin(mmio_base);
ae209cf1 AB	256	}
ae209cf1 AB	257
49d65b3a JO	258	struct cell_iommu {
	259	unsigned long base;
	260	unsigned long mmio_base;
	261	void __iomem *mapped_base;
	262	void __iomem *mapped_mmio_base;
	263	};
	264
	265	static struct cell_iommu cell_iommus[NR_CPUS];
ae209cf1 AB	266
	267	/* initialize the iommu to support a simple linear mapping
	268	* for each DMA window used by any device. For now, we
	269	* happen to know that there is only one DMA window in use,
	270	* starting at iopt_phys_offset. */
49d65b3a JO	271	static void cell_do_map_iommu(struct cell_iommu *iommu,
	272	unsigned int ioid,
	273	unsigned long map_start,
	274	unsigned long map_size)
ae209cf1	275	{
49d65b3a JO	276	unsigned long io_address, real_address;
49d65b3a JO	277	void __iomem ioc_base, ioc_mmio_base;
ae209cf1 AB	278	ioste ioste;
	279	unsigned long index;
	280
49d65b3a JO	281	/* we pretend the io page table was at a very high address */
	282	const unsigned long fake_iopt = 0x10000000000ul;
	283	const unsigned long io_page_size = 0x1000000; /* use 16M pages */
	284	const unsigned long io_segment_size = 0x10000000; /* 256M */
	285
	286	ioc_base = iommu->mapped_base;
	287	ioc_mmio_base = iommu->mapped_mmio_base;
	288
5fa500bf	289	for (real_address = 0, io_address = map_start;
49d65b3a JO	290	io_address <= map_start + map_size;
	291	real_address += io_page_size, io_address += io_page_size) {
	292	ioste = get_iost_entry(fake_iopt, io_address, io_page_size);
	293	if ((real_address % io_segment_size) == 0) /* segment start */
	294	set_iost_cache(ioc_mmio_base,
	295	io_address >> 28, ioste);
	296	index = get_ioc_hash_1way(ioste, io_address);
	297	pr_debug("addr %08lx, index %02lx, ioste %016lx\n",
	298	io_address, index, ioste.val);
	299	set_iopt_cache(ioc_mmio_base,
	300	get_ioc_hash_1way(ioste, io_address),
	301	get_ioc_tag(ioste, io_address),
5fa500bf	302	get_iopt_entry(real_address, ioid, IOPT_PROT_RW));
49d65b3a JO	303	}
49d65b3a JO	304	}
ae209cf1	305
12d04eef	306	static void pci_dma_cell_bus_setup(struct pci_bus *b)
49d65b3a	307	{
c61c27d5 JK	308	const unsigned int *ioid;
	309	unsigned long map_start, map_size, token;
	310	const unsigned long *dma_window;
49d65b3a	311	struct cell_iommu *iommu;
12d04eef BH	312	struct device_node *d;
	313
	314	d = pci_bus_to_OF_node(b);
ae209cf1	315
c61c27d5	316	ioid = get_property(d, "ioid", NULL);
49d65b3a JO	317	if (!ioid)
49d65b3a JO	318	pr_debug("No ioid entry found !\n");
ae209cf1	319
c61c27d5	320	dma_window = get_property(d, "ibm,dma-window", NULL);
49d65b3a JO	321	if (!dma_window)
	322	pr_debug("No ibm,dma-window entry found !\n");
	323
	324	map_start = dma_window[1];
	325	map_size = dma_window[2];
	326	token = dma_window[0] >> 32;
	327
	328	iommu = &cell_iommus[token];
	329
	330	cell_do_map_iommu(iommu, *ioid, map_start, map_size);
	331	}
	332
49d65b3a JO	333
	334	static int cell_map_iommu_hardcoded(int num_nodes)
	335	{
	336	struct cell_iommu *iommu = NULL;
	337
	338	pr_debug("%s(%d): Using hardcoded defaults\n", __FUNCTION__, __LINE__);
	339
	340	/* node 0 */
	341	iommu = &cell_iommus[0];
43b4f406 AB	342	iommu->mapped_base = ioremap(0x20000511000ul, 0x1000);
43b4f406 AB	343	iommu->mapped_mmio_base = ioremap(0x20000510000ul, 0x1000);
49d65b3a JO	344
	345	enable_mapping(iommu->mapped_base, iommu->mapped_mmio_base);
	346
	347	cell_do_map_iommu(iommu, 0x048a,
	348	0x20000000ul,0x20000000ul);
	349
	350	if (num_nodes < 2)
	351	return 0;
	352
	353	/* node 1 */
	354	iommu = &cell_iommus[1];
43b4f406 AB	355	iommu->mapped_base = ioremap(0x30000511000ul, 0x1000);
43b4f406 AB	356	iommu->mapped_mmio_base = ioremap(0x30000510000ul, 0x1000);
49d65b3a JO	357
	358	enable_mapping(iommu->mapped_base, iommu->mapped_mmio_base);
	359
	360	cell_do_map_iommu(iommu, 0x048a,
	361	0x20000000,0x20000000ul);
	362
	363	return 0;
ae209cf1 AB	364	}
	365
	366
49d65b3a JO	367	static int cell_map_iommu(void)
49d65b3a JO	368	{
c61c27d5 JK	369	unsigned int num_nodes = 0;
	370	const unsigned int *node_id;
	371	const unsigned long base, mmio_base;
49d65b3a JO	372	struct device_node *dn;
	373	struct cell_iommu *iommu = NULL;
	374
	375	/* determine number of nodes (=iommus) */
	376	pr_debug("%s(%d): determining number of nodes...", __FUNCTION__, __LINE__);
	377	for(dn = of_find_node_by_type(NULL, "cpu");
	378	dn;
	379	dn = of_find_node_by_type(dn, "cpu")) {
c61c27d5	380	node_id = get_property(dn, "node-id", NULL);
49d65b3a JO	381
	382	if (num_nodes < *node_id)
	383	num_nodes = *node_id;
	384	}
	385
	386	num_nodes++;
	387	pr_debug("%i found.\n", num_nodes);
	388
	389	/* map the iommu registers for each node */
	390	pr_debug("%s(%d): Looping through nodes\n", __FUNCTION__, __LINE__);
	391	for(dn = of_find_node_by_type(NULL, "cpu");
	392	dn;
	393	dn = of_find_node_by_type(dn, "cpu")) {
	394
c61c27d5 JK	395	node_id = get_property(dn, "node-id", NULL);
	396	base = get_property(dn, "ioc-cache", NULL);
	397	mmio_base = get_property(dn, "ioc-translation", NULL);
49d65b3a JO	398
	399	if (!base \|\| !mmio_base \|\| !node_id)
	400	return cell_map_iommu_hardcoded(num_nodes);
	401
	402	iommu = &cell_iommus[*node_id];
	403	iommu->base = *base;
	404	iommu->mmio_base = *mmio_base;
	405
47952d5e AB	406	iommu->mapped_base = ioremap(*base, 0x1000);
47952d5e AB	407	iommu->mapped_mmio_base = ioremap(*mmio_base, 0x1000);
49d65b3a JO	408
	409	enable_mapping(iommu->mapped_base,
	410	iommu->mapped_mmio_base);
	411
	412	/* everything else will be done in iommu_bus_setup */
	413	}
	414
	415	return 1;
	416	}
	417
f3f66f59	418	static void cell_alloc_coherent(struct device hwdev, size_t size,
dd0fc66f	419	dma_addr_t *dma_handle, gfp_t flag)
ae209cf1 AB	420	{
	421	void *ret;
	422
	423	ret = (void *)__get_free_pages(flag, get_order(size));
	424	if (ret != NULL) {
	425	memset(ret, 0, size);
f3f66f59	426	*dma_handle = virt_to_abs(ret) \| CELL_DMA_VALID;
ae209cf1 AB	427	}
	428	return ret;
	429	}
	430
f3f66f59	431	static void cell_free_coherent(struct device *hwdev, size_t size,
ae209cf1 AB	432	void *vaddr, dma_addr_t dma_handle)
	433	{
	434	free_pages((unsigned long)vaddr, get_order(size));
	435	}
	436
f3f66f59	437	static dma_addr_t cell_map_single(struct device hwdev, void ptr,
ae209cf1 AB	438	size_t size, enum dma_data_direction direction)
ae209cf1 AB	439	{
f3f66f59	440	return virt_to_abs(ptr) \| CELL_DMA_VALID;
ae209cf1 AB	441	}
ae209cf1 AB	442
f3f66f59	443	static void cell_unmap_single(struct device *hwdev, dma_addr_t dma_addr,
ae209cf1 AB	444	size_t size, enum dma_data_direction direction)
	445	{
	446	}
	447
f3f66f59	448	static int cell_map_sg(struct device hwdev, struct scatterlist sg,
ae209cf1 AB	449	int nents, enum dma_data_direction direction)
	450	{
	451	int i;
	452
	453	for (i = 0; i < nents; i++, sg++) {
	454	sg->dma_address = (page_to_phys(sg->page) + sg->offset)
f3f66f59	455	\| CELL_DMA_VALID;
ae209cf1 AB	456	sg->dma_length = sg->length;
	457	}
	458
	459	return nents;
	460	}
	461
f3f66f59	462	static void cell_unmap_sg(struct device hwdev, struct scatterlist sg,
ae209cf1 AB	463	int nents, enum dma_data_direction direction)
	464	{
	465	}
	466
f3f66f59	467	static int cell_dma_supported(struct device *dev, u64 mask)
ae209cf1 AB	468	{
	469	return mask < 0x100000000ull;
	470	}
	471
df310656 AB	472	static struct dma_mapping_ops cell_iommu_ops = {
	473	.alloc_coherent = cell_alloc_coherent,
	474	.free_coherent = cell_free_coherent,
	475	.map_single = cell_map_single,
	476	.unmap_single = cell_unmap_single,
	477	.map_sg = cell_map_sg,
	478	.unmap_sg = cell_unmap_sg,
	479	.dma_supported = cell_dma_supported,
	480	};
	481
f3f66f59	482	void cell_init_iommu(void)
ae209cf1	483	{
49d65b3a JO	484	int setup_bus = 0;
	485
	486	if (of_find_node_by_path("/mambo")) {
	487	pr_info("Not using iommu on systemsim\n");
	488	} else {
	489
	490	if (!(of_chosen &&
	491	get_property(of_chosen, "linux,iommu-off", NULL)))
	492	setup_bus = cell_map_iommu();
	493
	494	if (setup_bus) {
	495	pr_debug("%s: IOMMU mapping activated\n", __FUNCTION__);
12d04eef	496	ppc_md.pci_dma_bus_setup = pci_dma_cell_bus_setup;
49d65b3a JO	497	} else {
	498	pr_debug("%s: IOMMU mapping activated, "
	499	"no device action necessary\n", __FUNCTION__);
	500	/* Direct I/O, IOMMU off */
49d65b3a JO	501	}
49d65b3a JO	502	}
ae209cf1	503
12d04eef	504	pci_dma_ops = &cell_iommu_ops;
ae209cf1	505	}