[linux-block.git] / arch / x86 / kernel / pci-dma_64.c

/*
 * Dynamic DMA mapping support.
 */

#include <linux/types.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/pci.h>
#include <linux/module.h>
#include <linux/dmar.h>
#include <asm/io.h>
#include <asm/gart.h>
#include <asm/calgary.h>

int iommu_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_merge);

dma_addr_t bad_dma_address __read_mostly;
EXPORT_SYMBOL(bad_dma_address);

/* This tells the BIO block layer to assume merging. Default to off
   because we cannot guarantee merging later. */
int iommu_bio_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_bio_merge);

static int iommu_sac_force __read_mostly = 0;

int no_iommu __read_mostly;
#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly= 0;
#endif

/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to i386. */
struct device fallback_dev = {
	.bus_id = "fallback device",
	.coherent_dma_mask = DMA_32BIT_MASK,
	.dma_mask = &fallback_dev.coherent_dma_mask,
};

/* Allocate DMA memory on node near device */
noinline static void *
dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
{
	struct page *page;
	int node;

	node = dev_to_node(dev);
	if (node == -1)
		node = numa_node_id();

	if (node < first_node(node_online_map))
		node = first_node(node_online_map);

	page = alloc_pages_node(node, gfp, order);
	return page ? page_address(page) : NULL;
}

/*
 * Allocate memory for a coherent mapping.
 */
void *
dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
		   gfp_t gfp)
{
	void *memory;
	unsigned long dma_mask = 0;
	u64 bus;

	if (!dev)
		dev = &fallback_dev;
	dma_mask = dev->coherent_dma_mask;
	if (dma_mask == 0)
		dma_mask = DMA_32BIT_MASK;

	/* Device not DMA able */
	if (dev->dma_mask == NULL)
		return NULL;

	/* Don't invoke OOM killer */
	gfp |= __GFP_NORETRY;

	/* Kludge to make it bug-to-bug compatible with i386. i386
	   uses the normal dma_mask for alloc_coherent. */
	dma_mask &= *dev->dma_mask;

	/* Why <=? Even when the mask is smaller than 4GB it is often
	   larger than 16MB and in this case we have a chance of
	   finding fitting memory in the next higher zone first. If
	   not retry with true GFP_DMA. -AK */
	if (dma_mask <= DMA_32BIT_MASK)
		gfp |= GFP_DMA32;

 again:
	memory = dma_alloc_pages(dev, gfp, get_order(size));
	if (memory == NULL)
		return NULL;

	{
		int high, mmu;
		bus = virt_to_bus(memory);
	        high = (bus + size) >= dma_mask;
		mmu = high;
		if (force_iommu && !(gfp & GFP_DMA))
			mmu = 1;
		else if (high) {
			free_pages((unsigned long)memory,
				   get_order(size));

			/* Don't use the 16MB ZONE_DMA unless absolutely
			   needed. It's better to use remapping first. */
			if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
				gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
				goto again;
			}

			/* Let low level make its own zone decisions */
			gfp &= ~(GFP_DMA32|GFP_DMA);

			if (dma_ops->alloc_coherent)
				return dma_ops->alloc_coherent(dev, size,
							   dma_handle, gfp);
			return NULL;
		}

		memset(memory, 0, size);
		if (!mmu) {
			*dma_handle = virt_to_bus(memory);
			return memory;
		}
	}

	if (dma_ops->alloc_coherent) {
		free_pages((unsigned long)memory, get_order(size));
		gfp &= ~(GFP_DMA|GFP_DMA32);
		return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
	}

	if (dma_ops->map_simple) {
		*dma_handle = dma_ops->map_simple(dev, memory,
					      size,
					      PCI_DMA_BIDIRECTIONAL);
		if (*dma_handle != bad_dma_address)
			return memory;
	}

	if (panic_on_overflow)
		panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
	free_pages((unsigned long)memory, get_order(size));
	return NULL;
}
EXPORT_SYMBOL(dma_alloc_coherent);

/*
 * Unmap coherent memory.
 * The caller must ensure that the device has finished accessing the mapping.
 */
void dma_free_coherent(struct device *dev, size_t size,
			 void *vaddr, dma_addr_t bus)
{
	WARN_ON(irqs_disabled());	/* for portability */
	if (dma_ops->unmap_single)
		dma_ops->unmap_single(dev, bus, size, 0);
	free_pages((unsigned long)vaddr, get_order(size));
}
EXPORT_SYMBOL(dma_free_coherent);

static int forbid_dac __read_mostly;

int dma_supported(struct device *dev, u64 mask)
{
#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {


		printk(KERN_INFO "PCI: Disallowing DAC for device %s\n", dev->bus_id);
		return 0;
	}
#endif

	if (dma_ops->dma_supported)
		return dma_ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
        if (mask < DMA_24BIT_MASK)
                return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
		printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;
	*dev->dma_mask = mask;
	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
__init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p,"off",3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p,"force",5))
			force_iommu = 1;
		if (!strncmp(p,"noforce",7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge",8)) {
			iommu_bio_merge = 4096;
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic",5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic",7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge",5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge",7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac",8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = -1;

#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft",4))
			swiotlb = 1;
#endif

#ifdef CONFIG_GART_IOMMU
		gart_parse_options(p);
#endif

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

void __init pci_iommu_alloc(void)
{
	/*
	 * The order of these functions is important for
	 * fall-back/fail-over reasons
	 */
#ifdef CONFIG_GART_IOMMU
	gart_iommu_hole_init();
#endif

#ifdef CONFIG_CALGARY_IOMMU
	detect_calgary();
#endif

	detect_intel_iommu();

#ifdef CONFIG_SWIOTLB
	pci_swiotlb_init();
#endif
}

static int __init pci_iommu_init(void)
{
#ifdef CONFIG_CALGARY_IOMMU
	calgary_iommu_init();
#endif

	intel_iommu_init();

#ifdef CONFIG_GART_IOMMU
	gart_iommu_init();
#endif

	no_iommu_init();
	return 0;
}

void pci_iommu_shutdown(void)
{
	gart_iommu_shutdown();
}

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		printk(KERN_INFO "PCI: VIA PCI bridge detected. Disabling DAC.\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
/* Must execute after PCI subsystem */
fs_initcall(pci_iommu_init);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* Dynamic DMA mapping support.
	3	*/
	4
	5	#include <linux/types.h>
	6	#include <linux/mm.h>
	7	#include <linux/string.h>
	8	#include <linux/pci.h>
	9	#include <linux/module.h>
ba395927	10	#include <linux/dmar.h>
1da177e4	11	#include <asm/io.h>
395624fc	12	#include <asm/gart.h>
e465058d	13	#include <asm/calgary.h>
1da177e4	14
bc84cf17	15	int iommu_merge __read_mostly = 0;
17a941d8 MBY	16	EXPORT_SYMBOL(iommu_merge);
	17
	18	dma_addr_t bad_dma_address __read_mostly;
	19	EXPORT_SYMBOL(bad_dma_address);
	20
	21	/* This tells the BIO block layer to assume merging. Default to off
	22	because we cannot guarantee merging later. */
	23	int iommu_bio_merge __read_mostly = 0;
	24	EXPORT_SYMBOL(iommu_bio_merge);
	25
caa51716	26	static int iommu_sac_force __read_mostly = 0;
17a941d8 MBY	27
	28	int no_iommu __read_mostly;
	29	#ifdef CONFIG_IOMMU_DEBUG
	30	int panic_on_overflow __read_mostly = 1;
	31	int force_iommu __read_mostly = 1;
	32	#else
	33	int panic_on_overflow __read_mostly = 0;
	34	int force_iommu __read_mostly= 0;
	35	#endif
	36
8d4f6b93 JM	37	/* Set this to 1 if there is a HW IOMMU in the system */
	38	int iommu_detected __read_mostly = 0;
	39
17a941d8 MBY	40	/* Dummy device used for NULL arguments (normally ISA). Better would
	41	be probably a smaller DMA mask, but this is bug-to-bug compatible
	42	to i386. */
	43	struct device fallback_dev = {
	44	.bus_id = "fallback device",
9f2036f3	45	.coherent_dma_mask = DMA_32BIT_MASK,
17a941d8 MBY	46	.dma_mask = &fallback_dev.coherent_dma_mask,
	47	};
	48
	49	/* Allocate DMA memory on node near device */
	50	noinline static void *
	51	dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
	52	{
	53	struct page *page;
	54	int node;
f6855f7f YL	55
	56	node = dev_to_node(dev);
	57	if (node == -1)
17a941d8	58	node = numa_node_id();
0d015324 DY	59
	60	if (node < first_node(node_online_map))
	61	node = first_node(node_online_map);
	62
17a941d8 MBY	63	page = alloc_pages_node(node, gfp, order);
	64	return page ? page_address(page) : NULL;
	65	}
	66
	67	/*
	68	* Allocate memory for a coherent mapping.
1da177e4	69	*/
17a941d8 MBY	70	void *
	71	dma_alloc_coherent(struct device dev, size_t size, dma_addr_t dma_handle,
	72	gfp_t gfp)
1da177e4	73	{
17a941d8 MBY	74	void *memory;
	75	unsigned long dma_mask = 0;
	76	u64 bus;
	77
	78	if (!dev)
	79	dev = &fallback_dev;
	80	dma_mask = dev->coherent_dma_mask;
	81	if (dma_mask == 0)
9f2036f3	82	dma_mask = DMA_32BIT_MASK;
17a941d8	83
8154549c AK	84	/* Device not DMA able */
	85	if (dev->dma_mask == NULL)
	86	return NULL;
	87
3056d6be AK	88	/* Don't invoke OOM killer */
	89	gfp \|= __GFP_NORETRY;
	90
17a941d8 MBY	91	/* Kludge to make it bug-to-bug compatible with i386. i386
	92	uses the normal dma_mask for alloc_coherent. */
	93	dma_mask &= *dev->dma_mask;
	94
	95	/* Why <=? Even when the mask is smaller than 4GB it is often
	96	larger than 16MB and in this case we have a chance of
	97	finding fitting memory in the next higher zone first. If
	98	not retry with true GFP_DMA. -AK */
9f2036f3	99	if (dma_mask <= DMA_32BIT_MASK)
17a941d8 MBY	100	gfp \|= GFP_DMA32;
	101
	102	again:
	103	memory = dma_alloc_pages(dev, gfp, get_order(size));
	104	if (memory == NULL)
	105	return NULL;
	106
	107	{
	108	int high, mmu;
	109	bus = virt_to_bus(memory);
	110	high = (bus + size) >= dma_mask;
	111	mmu = high;
	112	if (force_iommu && !(gfp & GFP_DMA))
	113	mmu = 1;
	114	else if (high) {
	115	free_pages((unsigned long)memory,
	116	get_order(size));
	117
	118	/* Don't use the 16MB ZONE_DMA unless absolutely
	119	needed. It's better to use remapping first. */
9f2036f3	120	if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
17a941d8 MBY	121	gfp = (gfp & ~GFP_DMA32) \| GFP_DMA;
	122	goto again;
	123	}
	124
6bca52b5 AK	125	/* Let low level make its own zone decisions */
	126	gfp &= ~(GFP_DMA32\|GFP_DMA);
	127
17a941d8 MBY	128	if (dma_ops->alloc_coherent)
	129	return dma_ops->alloc_coherent(dev, size,
	130	dma_handle, gfp);
	131	return NULL;
	132	}
	133
	134	memset(memory, 0, size);
	135	if (!mmu) {
	136	*dma_handle = virt_to_bus(memory);
	137	return memory;
	138	}
	139	}
	140
	141	if (dma_ops->alloc_coherent) {
	142	free_pages((unsigned long)memory, get_order(size));
	143	gfp &= ~(GFP_DMA\|GFP_DMA32);
	144	return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
	145	}
	146
	147	if (dma_ops->map_simple) {
	148	*dma_handle = dma_ops->map_simple(dev, memory,
	149	size,
	150	PCI_DMA_BIDIRECTIONAL);
	151	if (*dma_handle != bad_dma_address)
	152	return memory;
1da177e4	153	}
1da177e4	154
17a941d8 MBY	155	if (panic_on_overflow)
	156	panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
	157	free_pages((unsigned long)memory, get_order(size));
	158	return NULL;
	159	}
	160	EXPORT_SYMBOL(dma_alloc_coherent);
1da177e4	161
17a941d8 MBY	162	/*
	163	* Unmap coherent memory.
	164	* The caller must ensure that the device has finished accessing the mapping.
1da177e4	165	*/
17a941d8 MBY	166	void dma_free_coherent(struct device *dev, size_t size,
	167	void *vaddr, dma_addr_t bus)
	168	{
aa24886e	169	WARN_ON(irqs_disabled()); /* for portability */
17a941d8 MBY	170	if (dma_ops->unmap_single)
	171	dma_ops->unmap_single(dev, bus, size, 0);
	172	free_pages((unsigned long)vaddr, get_order(size));
	173	}
	174	EXPORT_SYMBOL(dma_free_coherent);
	175
ece66840 AK	176	static int forbid_dac __read_mostly;
ece66840 AK	177
17a941d8 MBY	178	int dma_supported(struct device *dev, u64 mask)
17a941d8 MBY	179	{
ece66840 AK	180	#ifdef CONFIG_PCI
	181	if (mask > 0xffffffff && forbid_dac > 0) {
	182
	183
	184
	185	printk(KERN_INFO "PCI: Disallowing DAC for device %s\n", dev->bus_id);
	186	return 0;
	187	}
	188	#endif
	189
17a941d8 MBY	190	if (dma_ops->dma_supported)
	191	return dma_ops->dma_supported(dev, mask);
	192
	193	/* Copied from i386. Doesn't make much sense, because it will
	194	only work for pci_alloc_coherent.
	195	The caller just has to use GFP_DMA in this case. */
9f2036f3	196	if (mask < DMA_24BIT_MASK)
17a941d8 MBY	197	return 0;
	198
	199	/* Tell the device to use SAC when IOMMU force is on. This
	200	allows the driver to use cheaper accesses in some cases.
	201
	202	Problem with this is that if we overflow the IOMMU area and
	203	return DAC as fallback address the device may not handle it
	204	correctly.
	205
	206	As a special case some controllers have a 39bit address
	207	mode that is as efficient as 32bit (aic79xx). Don't force
	208	SAC for these. Assume all masks <= 40 bits are of this
	209	type. Normally this doesn't make any difference, but gives
	210	more gentle handling of IOMMU overflow. */
9f2036f3	211	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
17a941d8 MBY	212	printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
	213	return 0;
	214	}
	215
	216	return 1;
	217	}
	218	EXPORT_SYMBOL(dma_supported);
	219
	220	int dma_set_mask(struct device *dev, u64 mask)
1da177e4	221	{
17a941d8 MBY	222	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	223	return -EIO;
	224	*dev->dma_mask = mask;
	225	return 0;
1da177e4	226	}
17a941d8 MBY	227	EXPORT_SYMBOL(dma_set_mask);
17a941d8 MBY	228
5558870b KW	229	/*
	230	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	231	* documentation.
	232	*/
17a941d8 MBY	233	__init int iommu_setup(char *p)
17a941d8 MBY	234	{
ded318ec	235	iommu_merge = 1;
1da177e4	236
2c8c0e6b AK	237	if (!p)
	238	return -EINVAL;
	239
ded318ec AK	240	while (*p) {
	241	if (!strncmp(p,"off",3))
	242	no_iommu = 1;
	243	/* gart_parse_options has more force support */
	244	if (!strncmp(p,"force",5))
	245	force_iommu = 1;
	246	if (!strncmp(p,"noforce",7)) {
	247	iommu_merge = 0;
	248	force_iommu = 0;
	249	}
	250
	251	if (!strncmp(p, "biomerge",8)) {
	252	iommu_bio_merge = 4096;
	253	iommu_merge = 1;
	254	force_iommu = 1;
	255	}
	256	if (!strncmp(p, "panic",5))
	257	panic_on_overflow = 1;
	258	if (!strncmp(p, "nopanic",7))
	259	panic_on_overflow = 0;
	260	if (!strncmp(p, "merge",5)) {
	261	iommu_merge = 1;
	262	force_iommu = 1;
	263	}
	264	if (!strncmp(p, "nomerge",7))
	265	iommu_merge = 0;
	266	if (!strncmp(p, "forcesac",8))
	267	iommu_sac_force = 1;
	268	if (!strncmp(p, "allowdac", 8))
	269	forbid_dac = 0;
	270	if (!strncmp(p, "nodac", 5))
	271	forbid_dac = -1;
17a941d8 MBY	272
17a941d8 MBY	273	#ifdef CONFIG_SWIOTLB
ded318ec AK	274	if (!strncmp(p, "soft",4))
ded318ec AK	275	swiotlb = 1;
17a941d8 MBY	276	#endif
17a941d8 MBY	277
966396d3	278	#ifdef CONFIG_GART_IOMMU
ded318ec	279	gart_parse_options(p);
17a941d8 MBY	280	#endif
17a941d8 MBY	281
bff6547b MBY	282	#ifdef CONFIG_CALGARY_IOMMU
	283	if (!strncmp(p, "calgary", 7))
	284	use_calgary = 1;
	285	#endif /* CONFIG_CALGARY_IOMMU */
	286
ded318ec AK	287	p += strcspn(p, ",");
	288	if (*p == ',')
	289	++p;
	290	}
	291	return 0;
17a941d8	292	}
2c8c0e6b	293	early_param("iommu", iommu_setup);
0dc243ae JM	294
	295	void __init pci_iommu_alloc(void)
	296	{
	297	/*
	298	* The order of these functions is important for
	299	* fall-back/fail-over reasons
	300	*/
966396d3	301	#ifdef CONFIG_GART_IOMMU
0440d4c0	302	gart_iommu_hole_init();
0dc243ae JM	303	#endif
0dc243ae JM	304
e465058d JM	305	#ifdef CONFIG_CALGARY_IOMMU
	306	detect_calgary();
	307	#endif
	308
ba395927 KA	309	detect_intel_iommu();
ba395927 KA	310
0dc243ae JM	311	#ifdef CONFIG_SWIOTLB
	312	pci_swiotlb_init();
	313	#endif
	314	}
	315
	316	static int __init pci_iommu_init(void)
	317	{
e465058d JM	318	#ifdef CONFIG_CALGARY_IOMMU
	319	calgary_iommu_init();
	320	#endif
	321
ba395927 KA	322	intel_iommu_init();
ba395927 KA	323
966396d3	324	#ifdef CONFIG_GART_IOMMU
0dc243ae JM	325	gart_iommu_init();
	326	#endif
	327
	328	no_iommu_init();
	329	return 0;
	330	}
	331
bc2cea6a YL	332	void pci_iommu_shutdown(void)
	333	{
	334	gart_iommu_shutdown();
	335	}
	336
388c19e1 AK	337	#ifdef CONFIG_PCI
	338	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	339
	340	static __devinit void via_no_dac(struct pci_dev *dev)
	341	{
	342	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
	343	printk(KERN_INFO "PCI: VIA PCI bridge detected. Disabling DAC.\n");
	344	forbid_dac = 1;
	345	}
	346	}
	347	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	348	#endif
0dc243ae JM	349	/* Must execute after PCI subsystem */
0dc243ae JM	350	fs_initcall(pci_iommu_init);