[linux-2.6-block.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dmar.h>
#include <linux/bootmem.h>
#include <linux/pci.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/iommu.h>
#include <asm/calgary.h>
#include <asm/amd_iommu.h>

static int forbid_dac __read_mostly;

const struct dma_mapping_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/* This tells the BIO block layer to assume merging. Default to off
   because we cannot guarantee merging later. */
int iommu_bio_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_bio_merge);

dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);

/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to older i386. */
struct device fallback_dev = {
	.bus_id = "fallback device",
	.coherent_dma_mask = DMA_32BIT_MASK,
	.dma_mask = &fallback_dev.coherent_dma_mask,
};

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

#ifdef CONFIG_X86_64
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
	if (max_pfn <= MAX_DMA32_PFN)
		return;

	/*
	 * check aperture_64.c allocate_aperture() for reason about
	 * using 512M as goal
	 */
	align = 64ULL<<20;
	size = round_up(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
				 512ULL<<20);
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{

	if (max_pfn <= MAX_DMA32_PFN)
		return;

	if (!dma32_bootmem_ptr)
		return;

	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}

void __init pci_iommu_alloc(void)
{
	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();
	/*
	 * The order of these functions is important for
	 * fall-back/fail-over reasons
	 */
	gart_iommu_hole_init();

	detect_calgary();

	detect_intel_iommu();

	amd_iommu_detect();

	pci_swiotlb_init();
}
#endif

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_bio_merge = 4096;
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = -1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

#ifdef CONFIG_X86_32
int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
				dma_addr_t device_addr, size_t size, int flags)
{
	void __iomem *mem_base = NULL;
	int pages = size >> PAGE_SHIFT;
	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);

	if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
		goto out;
	if (!size)
		goto out;
	if (dev->dma_mem)
		goto out;

	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */

	mem_base = ioremap(bus_addr, size);
	if (!mem_base)
		goto out;

	dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	if (!dev->dma_mem)
		goto out;
	dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	if (!dev->dma_mem->bitmap)
		goto free1_out;

	dev->dma_mem->virt_base = mem_base;
	dev->dma_mem->device_base = device_addr;
	dev->dma_mem->size = pages;
	dev->dma_mem->flags = flags;

	if (flags & DMA_MEMORY_MAP)
		return DMA_MEMORY_MAP;

	return DMA_MEMORY_IO;

 free1_out:
	kfree(dev->dma_mem);
 out:
	if (mem_base)
		iounmap(mem_base);
	return 0;
}
EXPORT_SYMBOL(dma_declare_coherent_memory);

void dma_release_declared_memory(struct device *dev)
{
	struct dma_coherent_mem *mem = dev->dma_mem;

	if (!mem)
		return;
	dev->dma_mem = NULL;
	iounmap(mem->virt_base);
	kfree(mem->bitmap);
	kfree(mem);
}
EXPORT_SYMBOL(dma_release_declared_memory);

void *dma_mark_declared_memory_occupied(struct device *dev,
					dma_addr_t device_addr, size_t size)
{
	struct dma_coherent_mem *mem = dev->dma_mem;
	int pos, err;
	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);

	pages >>= PAGE_SHIFT;

	if (!mem)
		return ERR_PTR(-EINVAL);

	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
	if (err != 0)
		return ERR_PTR(err);
	return mem->virt_base + (pos << PAGE_SHIFT);
}
EXPORT_SYMBOL(dma_mark_declared_memory_occupied);

static int dma_alloc_from_coherent_mem(struct device *dev, ssize_t size,
				       dma_addr_t *dma_handle, void **ret)
{
	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	int order = get_order(size);

	if (mem) {
		int page = bitmap_find_free_region(mem->bitmap, mem->size,
						     order);
		if (page >= 0) {
			*dma_handle = mem->device_base + (page << PAGE_SHIFT);
			*ret = mem->virt_base + (page << PAGE_SHIFT);
			memset(*ret, 0, size);
		}
		if (mem->flags & DMA_MEMORY_EXCLUSIVE)
			*ret = NULL;
	}
	return (mem != NULL);
}

static int dma_release_coherent(struct device *dev, int order, void *vaddr)
{
	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;

	if (mem && vaddr >= mem->virt_base && vaddr <
		   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;

		bitmap_release_region(mem->bitmap, page, order);
		return 1;
	}
	return 0;
}
#else
#define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
#define dma_release_coherent(dev, order, vaddr) (0)
#endif /* CONFIG_X86_32 */

int dma_supported(struct device *dev, u64 mask)
{
#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		dev_info(dev, "PCI: Disallowing DAC for device\n");
		return 0;
	}
#endif

	if (dma_ops->dma_supported)
		return dma_ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_24BIT_MASK)
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

/* Allocate DMA memory on node near device */
static noinline struct page *
dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
{
	int node;

	node = dev_to_node(dev);

	return alloc_pages_node(node, gfp, order);
}

/*
 * Allocate memory for a coherent mapping.
 */
void *
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
		   gfp_t gfp)
{
	void *memory = NULL;
	struct page *page;
	unsigned long dma_mask = 0;
	dma_addr_t bus;
	int noretry = 0;

	/* ignore region specifiers */
	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);

	if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
		return memory;

	if (!dev) {
		dev = &fallback_dev;
		gfp |= GFP_DMA;
	}
	dma_mask = dev->coherent_dma_mask;
	if (dma_mask == 0)
		dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;

	/* Device not DMA able */
	if (dev->dma_mask == NULL)
		return NULL;

	/* Don't invoke OOM killer or retry in lower 16MB DMA zone */
	if (gfp & __GFP_DMA)
		noretry = 1;

#ifdef CONFIG_X86_64
	/* Why <=? Even when the mask is smaller than 4GB it is often
	   larger than 16MB and in this case we have a chance of
	   finding fitting memory in the next higher zone first. If
	   not retry with true GFP_DMA. -AK */
	if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
		gfp |= GFP_DMA32;
		if (dma_mask < DMA_32BIT_MASK)
			noretry = 1;
	}
#endif

 again:
	page = dma_alloc_pages(dev,
		noretry ? gfp | __GFP_NORETRY : gfp, get_order(size));
	if (page == NULL)
		return NULL;

	{
		int high, mmu;
		bus = page_to_phys(page);
		memory = page_address(page);
		high = (bus + size) >= dma_mask;
		mmu = high;
		if (force_iommu && !(gfp & GFP_DMA))
			mmu = 1;
		else if (high) {
			free_pages((unsigned long)memory,
				   get_order(size));

			/* Don't use the 16MB ZONE_DMA unless absolutely
			   needed. It's better to use remapping first. */
			if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
				gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
				goto again;
			}

			/* Let low level make its own zone decisions */
			gfp &= ~(GFP_DMA32|GFP_DMA);

			if (dma_ops->alloc_coherent)
				return dma_ops->alloc_coherent(dev, size,
							   dma_handle, gfp);
			return NULL;
		}

		memset(memory, 0, size);
		if (!mmu) {
			*dma_handle = bus;
			return memory;
		}
	}

	if (dma_ops->alloc_coherent) {
		free_pages((unsigned long)memory, get_order(size));
		gfp &= ~(GFP_DMA|GFP_DMA32);
		return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
	}

	if (dma_ops->map_simple) {
		*dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
					      size,
					      PCI_DMA_BIDIRECTIONAL);
		if (*dma_handle != bad_dma_address)
			return memory;
	}

	if (panic_on_overflow)
		panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
		      (unsigned long)size);
	free_pages((unsigned long)memory, get_order(size));
	return NULL;
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Unmap coherent memory.
 * The caller must ensure that the device has finished accessing the mapping.
 */
void dma_free_coherent(struct device *dev, size_t size,
			 void *vaddr, dma_addr_t bus)
{
	int order = get_order(size);
	WARN_ON(irqs_disabled());	/* for portability */
	if (dma_release_coherent(dev, order, vaddr))
		return;
	if (dma_ops->unmap_single)
		dma_ops->unmap_single(dev, bus, size, 0);
	free_pages((unsigned long)vaddr, order);
}
EXPORT_SYMBOL(dma_free_coherent);

static int __init pci_iommu_init(void)
{
	calgary_iommu_init();

	intel_iommu_init();

	amd_iommu_init();

	gart_iommu_init();

	no_iommu_init();
	return 0;
}

void pci_iommu_shutdown(void)
{
	gart_iommu_shutdown();
}
/* Must execute after PCI subsystem */
fs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		printk(KERN_INFO "PCI: VIA PCI bridge detected."
				 "Disabling DAC.\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
cb5867a5	2	#include <linux/dmar.h>
116890d5	3	#include <linux/bootmem.h>
bca5c096	4	#include <linux/pci.h>
cb5867a5	5
116890d5 GC	6	#include <asm/proto.h>
116890d5 GC	7	#include <asm/dma.h>
46a7fa27	8	#include <asm/iommu.h>
cb5867a5	9	#include <asm/calgary.h>
a69ca340	10	#include <asm/amd_iommu.h>
459121c9	11
08e1a13e	12	static int forbid_dac __read_mostly;
bca5c096	13
85c246ee GC	14	const struct dma_mapping_ops *dma_ops;
	15	EXPORT_SYMBOL(dma_ops);
	16
b4cdc430	17	static int iommu_sac_force __read_mostly;
8e0c3797	18
f9c258de GC	19	#ifdef CONFIG_IOMMU_DEBUG
	20	int panic_on_overflow __read_mostly = 1;
	21	int force_iommu __read_mostly = 1;
	22	#else
	23	int panic_on_overflow __read_mostly = 0;
	24	int force_iommu __read_mostly = 0;
	25	#endif
	26
fae9a0d8 GC	27	int iommu_merge __read_mostly = 0;
	28
	29	int no_iommu __read_mostly;
	30	/* Set this to 1 if there is a HW IOMMU in the system */
	31	int iommu_detected __read_mostly = 0;
	32
	33	/* This tells the BIO block layer to assume merging. Default to off
	34	because we cannot guarantee merging later. */
	35	int iommu_bio_merge __read_mostly = 0;
	36	EXPORT_SYMBOL(iommu_bio_merge);
	37
cac67877 GC	38	dma_addr_t bad_dma_address __read_mostly = 0;
cac67877 GC	39	EXPORT_SYMBOL(bad_dma_address);
fae9a0d8	40
098cb7f2 GC	41	/* Dummy device used for NULL arguments (normally ISA). Better would
	42	be probably a smaller DMA mask, but this is bug-to-bug compatible
	43	to older i386. */
	44	struct device fallback_dev = {
	45	.bus_id = "fallback device",
	46	.coherent_dma_mask = DMA_32BIT_MASK,
	47	.dma_mask = &fallback_dev.coherent_dma_mask,
	48	};
	49
459121c9 GC	50	int dma_set_mask(struct device *dev, u64 mask)
	51	{
	52	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	53	return -EIO;
	54
	55	*dev->dma_mask = mask;
	56
	57	return 0;
	58	}
	59	EXPORT_SYMBOL(dma_set_mask);
	60
116890d5 GC	61	#ifdef CONFIG_X86_64
	62	static __initdata void *dma32_bootmem_ptr;
	63	static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
	64
	65	static int __init parse_dma32_size_opt(char *p)
	66	{
	67	if (!p)
	68	return -EINVAL;
	69	dma32_bootmem_size = memparse(p, &p);
	70	return 0;
	71	}
	72	early_param("dma32_size", parse_dma32_size_opt);
	73
	74	void __init dma32_reserve_bootmem(void)
	75	{
	76	unsigned long size, align;
c987d12f	77	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	78	return;
116890d5 GC	79
7677b2ef YL	80	/*
	81	* check aperture_64.c allocate_aperture() for reason about
	82	* using 512M as goal
	83	*/
116890d5 GC	84	align = 64ULL<<20;
	85	size = round_up(dma32_bootmem_size, align);
	86	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
7677b2ef	87	512ULL<<20);
116890d5 GC	88	if (dma32_bootmem_ptr)
	89	dma32_bootmem_size = size;
	90	else
	91	dma32_bootmem_size = 0;
	92	}
	93	static void __init dma32_free_bootmem(void)
	94	{
116890d5	95
c987d12f	96	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	97	return;
	98
	99	if (!dma32_bootmem_ptr)
	100	return;
	101
330fce23	102	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
116890d5 GC	103
	104	dma32_bootmem_ptr = NULL;
	105	dma32_bootmem_size = 0;
	106	}
	107
	108	void __init pci_iommu_alloc(void)
	109	{
	110	/* free the range so iommu could get some range less than 4G */
	111	dma32_free_bootmem();
	112	/*
	113	* The order of these functions is important for
	114	* fall-back/fail-over reasons
	115	*/
116890d5	116	gart_iommu_hole_init();
116890d5	117
116890d5	118	detect_calgary();
116890d5 GC	119
	120	detect_intel_iommu();
	121
a69ca340 JR	122	amd_iommu_detect();
a69ca340 JR	123
116890d5	124	pci_swiotlb_init();
116890d5 GC	125	}
	126	#endif
	127
fae9a0d8 GC	128	/*
	129	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	130	* documentation.
	131	*/
	132	static __init int iommu_setup(char *p)
	133	{
	134	iommu_merge = 1;
	135
	136	if (!p)
	137	return -EINVAL;
	138
	139	while (*p) {
	140	if (!strncmp(p, "off", 3))
	141	no_iommu = 1;
	142	/* gart_parse_options has more force support */
	143	if (!strncmp(p, "force", 5))
	144	force_iommu = 1;
	145	if (!strncmp(p, "noforce", 7)) {
	146	iommu_merge = 0;
	147	force_iommu = 0;
	148	}
	149
	150	if (!strncmp(p, "biomerge", 8)) {
	151	iommu_bio_merge = 4096;
	152	iommu_merge = 1;
	153	force_iommu = 1;
	154	}
	155	if (!strncmp(p, "panic", 5))
	156	panic_on_overflow = 1;
	157	if (!strncmp(p, "nopanic", 7))
	158	panic_on_overflow = 0;
	159	if (!strncmp(p, "merge", 5)) {
	160	iommu_merge = 1;
	161	force_iommu = 1;
	162	}
	163	if (!strncmp(p, "nomerge", 7))
	164	iommu_merge = 0;
	165	if (!strncmp(p, "forcesac", 8))
	166	iommu_sac_force = 1;
	167	if (!strncmp(p, "allowdac", 8))
	168	forbid_dac = 0;
	169	if (!strncmp(p, "nodac", 5))
	170	forbid_dac = -1;
	171	if (!strncmp(p, "usedac", 6)) {
	172	forbid_dac = -1;
	173	return 1;
	174	}
	175	#ifdef CONFIG_SWIOTLB
	176	if (!strncmp(p, "soft", 4))
	177	swiotlb = 1;
	178	#endif
	179
fae9a0d8	180	gart_parse_options(p);
fae9a0d8 GC	181
	182	#ifdef CONFIG_CALGARY_IOMMU
	183	if (!strncmp(p, "calgary", 7))
	184	use_calgary = 1;
	185	#endif /* CONFIG_CALGARY_IOMMU */
	186
	187	p += strcspn(p, ",");
	188	if (*p == ',')
	189	++p;
	190	}
	191	return 0;
	192	}
	193	early_param("iommu", iommu_setup);
	194
8e8edc64 GC	195	#ifdef CONFIG_X86_32
	196	int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
	197	dma_addr_t device_addr, size_t size, int flags)
	198	{
	199	void __iomem *mem_base = NULL;
	200	int pages = size >> PAGE_SHIFT;
	201	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
	202
	203	if ((flags & (DMA_MEMORY_MAP \| DMA_MEMORY_IO)) == 0)
	204	goto out;
	205	if (!size)
	206	goto out;
	207	if (dev->dma_mem)
	208	goto out;
	209
	210	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
	211
	212	mem_base = ioremap(bus_addr, size);
	213	if (!mem_base)
	214	goto out;
	215
	216	dev->dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	217	if (!dev->dma_mem)
	218	goto out;
	219	dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	220	if (!dev->dma_mem->bitmap)
	221	goto free1_out;
	222
	223	dev->dma_mem->virt_base = mem_base;
	224	dev->dma_mem->device_base = device_addr;
	225	dev->dma_mem->size = pages;
	226	dev->dma_mem->flags = flags;
	227
	228	if (flags & DMA_MEMORY_MAP)
	229	return DMA_MEMORY_MAP;
	230
	231	return DMA_MEMORY_IO;
	232
	233	free1_out:
	234	kfree(dev->dma_mem);
	235	out:
	236	if (mem_base)
	237	iounmap(mem_base);
	238	return 0;
	239	}
	240	EXPORT_SYMBOL(dma_declare_coherent_memory);
	241
	242	void dma_release_declared_memory(struct device *dev)
	243	{
	244	struct dma_coherent_mem *mem = dev->dma_mem;
	245
	246	if (!mem)
	247	return;
	248	dev->dma_mem = NULL;
	249	iounmap(mem->virt_base);
	250	kfree(mem->bitmap);
	251	kfree(mem);
	252	}
	253	EXPORT_SYMBOL(dma_release_declared_memory);
	254
	255	void dma_mark_declared_memory_occupied(struct device dev,
	256	dma_addr_t device_addr, size_t size)
	257	{
	258	struct dma_coherent_mem *mem = dev->dma_mem;
259	int pos, err;
260	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1);
261
262	pages >>= PAGE_SHIFT;
263
264	if (!mem)
265	return ERR_PTR(-EINVAL);
266
267	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
268	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
269	if (err != 0)
270	return ERR_PTR(err);
271	return mem->virt_base + (pos << PAGE_SHIFT);
272	}
273	EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
098cb7f2 GC	274
	275	static int dma_alloc_from_coherent_mem(struct device *dev, ssize_t size,
	276	dma_addr_t dma_handle, void *ret)
	277	{
	278	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	279	int order = get_order(size);
	280
	281	if (mem) {
	282	int page = bitmap_find_free_region(mem->bitmap, mem->size,
	283	order);
	284	if (page >= 0) {
	285	*dma_handle = mem->device_base + (page << PAGE_SHIFT);
	286	*ret = mem->virt_base + (page << PAGE_SHIFT);
	287	memset(*ret, 0, size);
	288	}
	289	if (mem->flags & DMA_MEMORY_EXCLUSIVE)
	290	*ret = NULL;
	291	}
	292	return (mem != NULL);
	293	}
	294
	295	static int dma_release_coherent(struct device dev, int order, void vaddr)
	296	{
	297	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
	298
	299	if (mem && vaddr >= mem->virt_base && vaddr <
	300	(mem->virt_base + (mem->size << PAGE_SHIFT))) {
	301	int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
	302
	303	bitmap_release_region(mem->bitmap, page, order);
	304	return 1;
	305	}
	306	return 0;
	307	}
	308	#else
	309	#define dma_alloc_from_coherent_mem(dev, size, handle, ret) (0)
	310	#define dma_release_coherent(dev, order, vaddr) (0)
8e8edc64 GC	311	#endif /* CONFIG_X86_32 */
8e8edc64 GC	312
8e0c3797 GC	313	int dma_supported(struct device *dev, u64 mask)
	314	{
	315	#ifdef CONFIG_PCI
	316	if (mask > 0xffffffff && forbid_dac > 0) {
fc3a8828	317	dev_info(dev, "PCI: Disallowing DAC for device\n");
8e0c3797 GC	318	return 0;
	319	}
	320	#endif
	321
	322	if (dma_ops->dma_supported)
	323	return dma_ops->dma_supported(dev, mask);
	324
	325	/* Copied from i386. Doesn't make much sense, because it will
	326	only work for pci_alloc_coherent.
	327	The caller just has to use GFP_DMA in this case. */
	328	if (mask < DMA_24BIT_MASK)
	329	return 0;
	330
	331	/* Tell the device to use SAC when IOMMU force is on. This
	332	allows the driver to use cheaper accesses in some cases.
	333
	334	Problem with this is that if we overflow the IOMMU area and
	335	return DAC as fallback address the device may not handle it
	336	correctly.
	337
	338	As a special case some controllers have a 39bit address
	339	mode that is as efficient as 32bit (aic79xx). Don't force
	340	SAC for these. Assume all masks <= 40 bits are of this
	341	type. Normally this doesn't make any difference, but gives
	342	more gentle handling of IOMMU overflow. */
	343	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
fc3a8828	344	dev_info(dev, "Force SAC with mask %Lx\n", mask);
8e0c3797 GC	345	return 0;
	346	}
	347
	348	return 1;
	349	}
	350	EXPORT_SYMBOL(dma_supported);
	351
098cb7f2	352	/* Allocate DMA memory on node near device */
311f8349	353	static noinline struct page *
098cb7f2 GC	354	dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
	355	{
	356	int node;
	357
	358	node = dev_to_node(dev);
	359
	360	return alloc_pages_node(node, gfp, order);
	361	}
	362
	363	/*
	364	* Allocate memory for a coherent mapping.
	365	*/
	366	void *
	367	dma_alloc_coherent(struct device dev, size_t size, dma_addr_t dma_handle,
	368	gfp_t gfp)
	369	{
	370	void *memory = NULL;
	371	struct page *page;
	372	unsigned long dma_mask = 0;
	373	dma_addr_t bus;
b7f09ae5	374	int noretry = 0;
098cb7f2 GC	375
	376	/* ignore region specifiers */
	377	gfp &= ~(__GFP_DMA \| __GFP_HIGHMEM \| __GFP_DMA32);
	378
	379	if (dma_alloc_from_coherent_mem(dev, size, dma_handle, &memory))
	380	return memory;
	381
4a367f3a	382	if (!dev) {
098cb7f2	383	dev = &fallback_dev;
4a367f3a TI	384	gfp \|= GFP_DMA;
4a367f3a TI	385	}
098cb7f2 GC	386	dma_mask = dev->coherent_dma_mask;
098cb7f2 GC	387	if (dma_mask == 0)
4a367f3a	388	dma_mask = (gfp & GFP_DMA) ? DMA_24BIT_MASK : DMA_32BIT_MASK;
098cb7f2 GC	389
	390	/* Device not DMA able */
	391	if (dev->dma_mask == NULL)
	392	return NULL;
	393
b7f09ae5 MS	394	/* Don't invoke OOM killer or retry in lower 16MB DMA zone */
	395	if (gfp & __GFP_DMA)
	396	noretry = 1;
098cb7f2 GC	397
	398	#ifdef CONFIG_X86_64
	399	/* Why <=? Even when the mask is smaller than 4GB it is often
	400	larger than 16MB and in this case we have a chance of
	401	finding fitting memory in the next higher zone first. If
	402	not retry with true GFP_DMA. -AK */
b7f09ae5	403	if (dma_mask <= DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
098cb7f2	404	gfp \|= GFP_DMA32;
b7f09ae5 MS	405	if (dma_mask < DMA_32BIT_MASK)
	406	noretry = 1;
	407	}
098cb7f2 GC	408	#endif
	409
	410	again:
db9f600b	411	page = dma_alloc_pages(dev,
b7f09ae5	412	noretry ? gfp \| __GFP_NORETRY : gfp, get_order(size));
098cb7f2 GC	413	if (page == NULL)
	414	return NULL;
	415
	416	{
	417	int high, mmu;
	418	bus = page_to_phys(page);
	419	memory = page_address(page);
	420	high = (bus + size) >= dma_mask;
	421	mmu = high;
	422	if (force_iommu && !(gfp & GFP_DMA))
	423	mmu = 1;
	424	else if (high) {
	425	free_pages((unsigned long)memory,
	426	get_order(size));
	427
	428	/* Don't use the 16MB ZONE_DMA unless absolutely
	429	needed. It's better to use remapping first. */
	430	if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
	431	gfp = (gfp & ~GFP_DMA32) \| GFP_DMA;
	432	goto again;
	433	}
	434
	435	/* Let low level make its own zone decisions */
	436	gfp &= ~(GFP_DMA32\|GFP_DMA);
	437
	438	if (dma_ops->alloc_coherent)
	439	return dma_ops->alloc_coherent(dev, size,
	440	dma_handle, gfp);
	441	return NULL;
	442	}
	443
	444	memset(memory, 0, size);
	445	if (!mmu) {
	446	*dma_handle = bus;
	447	return memory;
	448	}
	449	}
	450
	451	if (dma_ops->alloc_coherent) {
	452	free_pages((unsigned long)memory, get_order(size));
	453	gfp &= ~(GFP_DMA\|GFP_DMA32);
	454	return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
	455	}
	456
	457	if (dma_ops->map_simple) {
	458	*dma_handle = dma_ops->map_simple(dev, virt_to_phys(memory),
	459	size,
	460	PCI_DMA_BIDIRECTIONAL);
	461	if (*dma_handle != bad_dma_address)
	462	return memory;
	463	}
	464
	465	if (panic_on_overflow)
	466	panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",
	467	(unsigned long)size);
	468	free_pages((unsigned long)memory, get_order(size));
	469	return NULL;
	470	}
	471	EXPORT_SYMBOL(dma_alloc_coherent);
	472
	473	/*
	474	* Unmap coherent memory.
	475	* The caller must ensure that the device has finished accessing the mapping.
	476	*/
477	void dma_free_coherent(struct device *dev, size_t size,
478	void *vaddr, dma_addr_t bus)
479	{
480	int order = get_order(size);
481	WARN_ON(irqs_disabled()); /* for portability */
482	if (dma_release_coherent(dev, order, vaddr))
483	return;
484	if (dma_ops->unmap_single)
485	dma_ops->unmap_single(dev, bus, size, 0);
486	free_pages((unsigned long)vaddr, order);
487	}
488	EXPORT_SYMBOL(dma_free_coherent);
8e0c3797	489
cb5867a5 GC	490	static int __init pci_iommu_init(void)
cb5867a5 GC	491	{
cb5867a5	492	calgary_iommu_init();
cb5867a5 GC	493
	494	intel_iommu_init();
	495
a69ca340 JR	496	amd_iommu_init();
a69ca340 JR	497
cb5867a5	498	gart_iommu_init();
459121c9	499
cb5867a5 GC	500	no_iommu_init();
	501	return 0;
	502	}
	503
	504	void pci_iommu_shutdown(void)
	505	{
	506	gart_iommu_shutdown();
	507	}
	508	/* Must execute after PCI subsystem */
	509	fs_initcall(pci_iommu_init);
bca5c096 GC	510
	511	#ifdef CONFIG_PCI
	512	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	513
	514	static __devinit void via_no_dac(struct pci_dev *dev)
	515	{
	516	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
	517	printk(KERN_INFO "PCI: VIA PCI bridge detected."
	518	"Disabling DAC.\n");
	519	forbid_dac = 1;
	520	}
	521	}
	522	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	523	#endif