[linux-2.6-block.git] / arch / mips / mm / dma-default.c

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 2000  Ani Joshi <ajoshi@unixbox.com>
 * Copyright (C) 2000, 2001, 06	 Ralf Baechle <ralf@linux-mips.org>
 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
 */

#include <linux/types.h>
#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/string.h>
#include <linux/gfp.h>
#include <linux/highmem.h>

#include <asm/cache.h>
#include <asm/io.h>

#include <dma-coherence.h>

int coherentio = 0;	/* User defined DMA coherency from command line. */
EXPORT_SYMBOL_GPL(coherentio);
int hw_coherentio = 0;	/* Actual hardware supported DMA coherency setting. */

static int __init setcoherentio(char *str)
{
	coherentio = 1;
	pr_info("Hardware DMA cache coherency (command line)\n");
	return 0;
}
early_param("coherentio", setcoherentio);

static int __init setnocoherentio(char *str)
{
	coherentio = 0;
	pr_info("Software DMA cache coherency (command line)\n");
	return 0;
}
early_param("nocoherentio", setnocoherentio);

static inline struct page *dma_addr_to_page(struct device *dev,
	dma_addr_t dma_addr)
{
	return pfn_to_page(
		plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
}

/*
 * Warning on the terminology - Linux calls an uncached area coherent;
 * MIPS terminology calls memory areas with hardware maintained coherency
 * coherent.
 */

static inline int cpu_is_noncoherent_r10000(struct device *dev)
{
	return !plat_device_is_coherent(dev) &&
	       (current_cpu_type() == CPU_R10000 ||
	       current_cpu_type() == CPU_R12000);
}

static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
{
	gfp_t dma_flag;

	/* ignore region specifiers */
	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);

#ifdef CONFIG_ISA
	if (dev == NULL)
		dma_flag = __GFP_DMA;
	else
#endif
#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
			dma_flag = __GFP_DMA;
	else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
			dma_flag = __GFP_DMA32;
	else
#endif
#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
		dma_flag = __GFP_DMA32;
	else
#endif
#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
		dma_flag = __GFP_DMA;
	else
#endif
		dma_flag = 0;

	/* Don't invoke OOM killer */
	gfp |= __GFP_NORETRY;

	return gfp | dma_flag;
}

void *dma_alloc_noncoherent(struct device *dev, size_t size,
	dma_addr_t * dma_handle, gfp_t gfp)
{
	void *ret;

	gfp = massage_gfp_flags(dev, gfp);

	ret = (void *) __get_free_pages(gfp, get_order(size));

	if (ret != NULL) {
		memset(ret, 0, size);
		*dma_handle = plat_map_dma_mem(dev, ret, size);
	}

	return ret;
}
EXPORT_SYMBOL(dma_alloc_noncoherent);

static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
	dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
{
	void *ret;

	if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
		return ret;

	gfp = massage_gfp_flags(dev, gfp);

	ret = (void *) __get_free_pages(gfp, get_order(size));

	if (ret) {
		memset(ret, 0, size);
		*dma_handle = plat_map_dma_mem(dev, ret, size);

		if (!plat_device_is_coherent(dev)) {
			dma_cache_wback_inv((unsigned long) ret, size);
			if (!hw_coherentio)
				ret = UNCAC_ADDR(ret);
		}
	}

	return ret;
}


void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
	dma_addr_t dma_handle)
{
	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
	free_pages((unsigned long) vaddr, get_order(size));
}
EXPORT_SYMBOL(dma_free_noncoherent);

static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
	dma_addr_t dma_handle, struct dma_attrs *attrs)
{
	unsigned long addr = (unsigned long) vaddr;
	int order = get_order(size);

	if (dma_release_from_coherent(dev, order, vaddr))
		return;

	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);

	if (!plat_device_is_coherent(dev) && !hw_coherentio)
		addr = CAC_ADDR(addr);

	free_pages(addr, get_order(size));
}

static inline void __dma_sync_virtual(void *addr, size_t size,
	enum dma_data_direction direction)
{
	switch (direction) {
	case DMA_TO_DEVICE:
		dma_cache_wback((unsigned long)addr, size);
		break;

	case DMA_FROM_DEVICE:
		dma_cache_inv((unsigned long)addr, size);
		break;

	case DMA_BIDIRECTIONAL:
		dma_cache_wback_inv((unsigned long)addr, size);
		break;

	default:
		BUG();
	}
}

/*
 * A single sg entry may refer to multiple physically contiguous
 * pages. But we still need to process highmem pages individually.
 * If highmem is not configured then the bulk of this loop gets
 * optimized out.
 */
static inline void __dma_sync(struct page *page,
	unsigned long offset, size_t size, enum dma_data_direction direction)
{
	size_t left = size;

	do {
		size_t len = left;

		if (PageHighMem(page)) {
			void *addr;

			if (offset + len > PAGE_SIZE) {
				if (offset >= PAGE_SIZE) {
					page += offset >> PAGE_SHIFT;
					offset &= ~PAGE_MASK;
				}
				len = PAGE_SIZE - offset;
			}

			addr = kmap_atomic(page);
			__dma_sync_virtual(addr + offset, len, direction);
			kunmap_atomic(addr);
		} else
			__dma_sync_virtual(page_address(page) + offset,
					   size, direction);
		offset = 0;
		page++;
		left -= len;
	} while (left);
}

static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
	size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
{
	if (cpu_is_noncoherent_r10000(dev))
		__dma_sync(dma_addr_to_page(dev, dma_addr),
			   dma_addr & ~PAGE_MASK, size, direction);

	plat_unmap_dma_mem(dev, dma_addr, size, direction);
}

static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
	int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
{
	int i;

	for (i = 0; i < nents; i++, sg++) {
		if (!plat_device_is_coherent(dev))
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		sg->dma_length = sg->length;
#endif
		sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
				  sg->offset;
	}

	return nents;
}

static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
	unsigned long offset, size_t size, enum dma_data_direction direction,
	struct dma_attrs *attrs)
{
	if (!plat_device_is_coherent(dev))
		__dma_sync(page, offset, size, direction);

	return plat_map_dma_mem_page(dev, page) + offset;
}

static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
	int nhwentries, enum dma_data_direction direction,
	struct dma_attrs *attrs)
{
	int i;

	for (i = 0; i < nhwentries; i++, sg++) {
		if (!plat_device_is_coherent(dev) &&
		    direction != DMA_TO_DEVICE)
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
		plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
	}
}

static void mips_dma_sync_single_for_cpu(struct device *dev,
	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
	if (cpu_is_noncoherent_r10000(dev))
		__dma_sync(dma_addr_to_page(dev, dma_handle),
			   dma_handle & ~PAGE_MASK, size, direction);
}

static void mips_dma_sync_single_for_device(struct device *dev,
	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
	plat_extra_sync_for_device(dev);
	if (!plat_device_is_coherent(dev))
		__dma_sync(dma_addr_to_page(dev, dma_handle),
			   dma_handle & ~PAGE_MASK, size, direction);
}

static void mips_dma_sync_sg_for_cpu(struct device *dev,
	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
{
	int i;

	/* Make sure that gcc doesn't leave the empty loop body.  */
	for (i = 0; i < nelems; i++, sg++) {
		if (cpu_is_noncoherent_r10000(dev))
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
	}
}

static void mips_dma_sync_sg_for_device(struct device *dev,
	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
{
	int i;

	/* Make sure that gcc doesn't leave the empty loop body.  */
	for (i = 0; i < nelems; i++, sg++) {
		if (!plat_device_is_coherent(dev))
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
	}
}

int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
	return plat_dma_mapping_error(dev, dma_addr);
}

int mips_dma_supported(struct device *dev, u64 mask)
{
	return plat_dma_supported(dev, mask);
}

void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
			 enum dma_data_direction direction)
{
	BUG_ON(direction == DMA_NONE);

	plat_extra_sync_for_device(dev);
	if (!plat_device_is_coherent(dev))
		__dma_sync_virtual(vaddr, size, direction);
}

EXPORT_SYMBOL(dma_cache_sync);

static struct dma_map_ops mips_default_dma_map_ops = {
	.alloc = mips_dma_alloc_coherent,
	.free = mips_dma_free_coherent,
	.map_page = mips_dma_map_page,
	.unmap_page = mips_dma_unmap_page,
	.map_sg = mips_dma_map_sg,
	.unmap_sg = mips_dma_unmap_sg,
	.sync_single_for_cpu = mips_dma_sync_single_for_cpu,
	.sync_single_for_device = mips_dma_sync_single_for_device,
	.sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
	.sync_sg_for_device = mips_dma_sync_sg_for_device,
	.mapping_error = mips_dma_mapping_error,
	.dma_supported = mips_dma_supported
};

struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
EXPORT_SYMBOL(mips_dma_map_ops);

#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)

static int __init mips_dma_init(void)
{
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

	return 0;
}
fs_initcall(mips_dma_init);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
70342287	7	* Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
1da177e4 LT	8	* swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
1da177e4 LT	9	*/
9a88cbb5	10
1da177e4	11	#include <linux/types.h>
9a88cbb5	12	#include <linux/dma-mapping.h>
1da177e4 LT	13	#include <linux/mm.h>
1da177e4 LT	14	#include <linux/module.h>
4fcc47a0	15	#include <linux/scatterlist.h>
6e86b0bf	16	#include <linux/string.h>
5a0e3ad6	17	#include <linux/gfp.h>
e36863a5	18	#include <linux/highmem.h>
1da177e4 LT	19
	20	#include <asm/cache.h>
	21	#include <asm/io.h>
	22
9a88cbb5 RB	23	#include <dma-coherence.h>
9a88cbb5 RB	24
b6d92b4a SH	25	int coherentio = 0; /* User defined DMA coherency from command line. */
	26	EXPORT_SYMBOL_GPL(coherentio);
	27	int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
	28
	29	static int __init setcoherentio(char *str)
	30	{
	31	coherentio = 1;
	32	pr_info("Hardware DMA cache coherency (command line)\n");
	33	return 0;
	34	}
	35	early_param("coherentio", setcoherentio);
	36
	37	static int __init setnocoherentio(char *str)
	38	{
	39	coherentio = 0;
	40	pr_info("Software DMA cache coherency (command line)\n");
	41	return 0;
	42	}
	43	early_param("nocoherentio", setnocoherentio);
	44
e36863a5	45	static inline struct page dma_addr_to_page(struct device dev,
3807ef3f	46	dma_addr_t dma_addr)
c9d06962	47	{
e36863a5 DD	48	return pfn_to_page(
e36863a5 DD	49	plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
c9d06962 FBH	50	}
c9d06962 FBH	51
1da177e4 LT	52	/*
	53	* Warning on the terminology - Linux calls an uncached area coherent;
	54	* MIPS terminology calls memory areas with hardware maintained coherency
	55	* coherent.
	56	*/
	57
9a88cbb5 RB	58	static inline int cpu_is_noncoherent_r10000(struct device *dev)
	59	{
	60	return !plat_device_is_coherent(dev) &&
10cc3529 RB	61	(current_cpu_type() == CPU_R10000 \|\|
10cc3529 RB	62	current_cpu_type() == CPU_R12000);
9a88cbb5 RB	63	}
9a88cbb5 RB	64
cce335ae RB	65	static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
cce335ae RB	66	{
a2e715a8 RB	67	gfp_t dma_flag;
a2e715a8 RB	68
cce335ae RB	69	/* ignore region specifiers */
	70	gfp &= ~(__GFP_DMA \| __GFP_DMA32 \| __GFP_HIGHMEM);
	71
a2e715a8	72	#ifdef CONFIG_ISA
cce335ae	73	if (dev == NULL)
a2e715a8	74	dma_flag = __GFP_DMA;
cce335ae RB	75	else
cce335ae RB	76	#endif
a2e715a8	77	#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
cce335ae	78	if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
a2e715a8 RB	79	dma_flag = __GFP_DMA;
	80	else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	81	dma_flag = __GFP_DMA32;
	82	else
	83	#endif
	84	#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
	85	if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	86	dma_flag = __GFP_DMA32;
	87	else
	88	#endif
	89	#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
	90	if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	91	dma_flag = __GFP_DMA;
cce335ae RB	92	else
cce335ae RB	93	#endif
a2e715a8	94	dma_flag = 0;
cce335ae RB	95
	96	/* Don't invoke OOM killer */
	97	gfp \|= __GFP_NORETRY;
	98
a2e715a8	99	return gfp \| dma_flag;
cce335ae RB	100	}
cce335ae RB	101
1da177e4	102	void dma_alloc_noncoherent(struct device dev, size_t size,
185a8ff5	103	dma_addr_t * dma_handle, gfp_t gfp)
1da177e4 LT	104	{
1da177e4 LT	105	void *ret;
9a88cbb5	106
cce335ae	107	gfp = massage_gfp_flags(dev, gfp);
1da177e4	108
1da177e4 LT	109	ret = (void *) __get_free_pages(gfp, get_order(size));
	110
	111	if (ret != NULL) {
	112	memset(ret, 0, size);
9a88cbb5	113	*dma_handle = plat_map_dma_mem(dev, ret, size);
1da177e4 LT	114	}
	115
	116	return ret;
	117	}
1da177e4 LT	118	EXPORT_SYMBOL(dma_alloc_noncoherent);
1da177e4 LT	119
48e1fd5a	120	static void mips_dma_alloc_coherent(struct device dev, size_t size,
e8d51e54	121	dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
1da177e4 LT	122	{
	123	void *ret;
	124
f8ac0425 YY	125	if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
	126	return ret;
	127
cce335ae	128	gfp = massage_gfp_flags(dev, gfp);
9a88cbb5	129
9a88cbb5 RB	130	ret = (void *) __get_free_pages(gfp, get_order(size));
9a88cbb5 RB	131
1da177e4	132	if (ret) {
9a88cbb5 RB	133	memset(ret, 0, size);
	134	*dma_handle = plat_map_dma_mem(dev, ret, size);
	135
	136	if (!plat_device_is_coherent(dev)) {
	137	dma_cache_wback_inv((unsigned long) ret, size);
b6d92b4a SH	138	if (!hw_coherentio)
b6d92b4a SH	139	ret = UNCAC_ADDR(ret);
9a88cbb5	140	}
1da177e4 LT	141	}
	142
	143	return ret;
	144	}
	145
1da177e4 LT	146
	147	void dma_free_noncoherent(struct device dev, size_t size, void vaddr,
	148	dma_addr_t dma_handle)
	149	{
d3f634b9	150	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
1da177e4 LT	151	free_pages((unsigned long) vaddr, get_order(size));
1da177e4 LT	152	}
1da177e4 LT	153	EXPORT_SYMBOL(dma_free_noncoherent);
1da177e4 LT	154
48e1fd5a	155	static void mips_dma_free_coherent(struct device dev, size_t size, void vaddr,
e8d51e54	156	dma_addr_t dma_handle, struct dma_attrs *attrs)
1da177e4 LT	157	{
1da177e4 LT	158	unsigned long addr = (unsigned long) vaddr;
f8ac0425 YY	159	int order = get_order(size);
	160
	161	if (dma_release_from_coherent(dev, order, vaddr))
	162	return;
1da177e4	163
d3f634b9	164	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
11531ac2	165
b6d92b4a	166	if (!plat_device_is_coherent(dev) && !hw_coherentio)
9a88cbb5 RB	167	addr = CAC_ADDR(addr);
9a88cbb5 RB	168
1da177e4 LT	169	free_pages(addr, get_order(size));
	170	}
	171
e36863a5	172	static inline void __dma_sync_virtual(void *addr, size_t size,
1da177e4 LT	173	enum dma_data_direction direction)
	174	{
	175	switch (direction) {
	176	case DMA_TO_DEVICE:
e36863a5	177	dma_cache_wback((unsigned long)addr, size);
1da177e4 LT	178	break;
	179
	180	case DMA_FROM_DEVICE:
e36863a5	181	dma_cache_inv((unsigned long)addr, size);
1da177e4 LT	182	break;
	183
	184	case DMA_BIDIRECTIONAL:
e36863a5	185	dma_cache_wback_inv((unsigned long)addr, size);
1da177e4 LT	186	break;
	187
	188	default:
	189	BUG();
	190	}
	191	}
	192
e36863a5 DD	193	/*
	194	* A single sg entry may refer to multiple physically contiguous
	195	* pages. But we still need to process highmem pages individually.
	196	* If highmem is not configured then the bulk of this loop gets
	197	* optimized out.
	198	*/
	199	static inline void __dma_sync(struct page *page,
	200	unsigned long offset, size_t size, enum dma_data_direction direction)
	201	{
	202	size_t left = size;
	203
	204	do {
	205	size_t len = left;
	206
	207	if (PageHighMem(page)) {
	208	void *addr;
	209
	210	if (offset + len > PAGE_SIZE) {
	211	if (offset >= PAGE_SIZE) {
	212	page += offset >> PAGE_SHIFT;
	213	offset &= ~PAGE_MASK;
	214	}
	215	len = PAGE_SIZE - offset;
	216	}
	217
	218	addr = kmap_atomic(page);
	219	__dma_sync_virtual(addr + offset, len, direction);
	220	kunmap_atomic(addr);
	221	} else
	222	__dma_sync_virtual(page_address(page) + offset,
	223	size, direction);
	224	offset = 0;
	225	page++;
	226	left -= len;
	227	} while (left);
	228	}
	229
48e1fd5a DD	230	static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
48e1fd5a DD	231	size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
1da177e4	232	{
9a88cbb5	233	if (cpu_is_noncoherent_r10000(dev))
e36863a5 DD	234	__dma_sync(dma_addr_to_page(dev, dma_addr),
e36863a5 DD	235	dma_addr & ~PAGE_MASK, size, direction);
1da177e4	236
d3f634b9	237	plat_unmap_dma_mem(dev, dma_addr, size, direction);
1da177e4 LT	238	}
1da177e4 LT	239
48e1fd5a DD	240	static int mips_dma_map_sg(struct device dev, struct scatterlist sg,
48e1fd5a DD	241	int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
1da177e4 LT	242	{
	243	int i;
	244
1da177e4	245	for (i = 0; i < nents; i++, sg++) {
e36863a5 DD	246	if (!plat_device_is_coherent(dev))
	247	__dma_sync(sg_page(sg), sg->offset, sg->length,
	248	direction);
4954a9a2 J	249	#ifdef CONFIG_NEED_SG_DMA_LENGTH
	250	sg->dma_length = sg->length;
	251	#endif
e36863a5 DD	252	sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
e36863a5 DD	253	sg->offset;
1da177e4 LT	254	}
	255
	256	return nents;
	257	}
	258
48e1fd5a DD	259	static dma_addr_t mips_dma_map_page(struct device dev, struct page page,
	260	unsigned long offset, size_t size, enum dma_data_direction direction,
	261	struct dma_attrs *attrs)
1da177e4	262	{
48e1fd5a	263	if (!plat_device_is_coherent(dev))
e36863a5	264	__dma_sync(page, offset, size, direction);
1da177e4	265
e36863a5	266	return plat_map_dma_mem_page(dev, page) + offset;
1da177e4 LT	267	}
1da177e4 LT	268
48e1fd5a DD	269	static void mips_dma_unmap_sg(struct device dev, struct scatterlist sg,
	270	int nhwentries, enum dma_data_direction direction,
	271	struct dma_attrs *attrs)
1da177e4	272	{
1da177e4 LT	273	int i;
1da177e4 LT	274
1da177e4	275	for (i = 0; i < nhwentries; i++, sg++) {
9a88cbb5	276	if (!plat_device_is_coherent(dev) &&
e36863a5 DD	277	direction != DMA_TO_DEVICE)
	278	__dma_sync(sg_page(sg), sg->offset, sg->length,
	279	direction);
d3f634b9	280	plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
1da177e4 LT	281	}
	282	}
	283
48e1fd5a DD	284	static void mips_dma_sync_single_for_cpu(struct device *dev,
48e1fd5a DD	285	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
1da177e4	286	{
e36863a5 DD	287	if (cpu_is_noncoherent_r10000(dev))
	288	__dma_sync(dma_addr_to_page(dev, dma_handle),
	289	dma_handle & ~PAGE_MASK, size, direction);
1da177e4 LT	290	}
1da177e4 LT	291
48e1fd5a DD	292	static void mips_dma_sync_single_for_device(struct device *dev,
48e1fd5a DD	293	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
1da177e4	294	{
843aef49	295	plat_extra_sync_for_device(dev);
e36863a5 DD	296	if (!plat_device_is_coherent(dev))
	297	__dma_sync(dma_addr_to_page(dev, dma_handle),
	298	dma_handle & ~PAGE_MASK, size, direction);
1da177e4 LT	299	}
1da177e4 LT	300
48e1fd5a DD	301	static void mips_dma_sync_sg_for_cpu(struct device *dev,
48e1fd5a DD	302	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
1da177e4 LT	303	{
1da177e4 LT	304	int i;
42a3b4f2	305
1da177e4	306	/* Make sure that gcc doesn't leave the empty loop body. */
9a88cbb5	307	for (i = 0; i < nelems; i++, sg++) {
5b648a98	308	if (cpu_is_noncoherent_r10000(dev))
e36863a5 DD	309	__dma_sync(sg_page(sg), sg->offset, sg->length,
e36863a5 DD	310	direction);
9a88cbb5	311	}
1da177e4 LT	312	}
1da177e4 LT	313
48e1fd5a DD	314	static void mips_dma_sync_sg_for_device(struct device *dev,
48e1fd5a DD	315	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
1da177e4 LT	316	{
	317	int i;
	318
1da177e4	319	/* Make sure that gcc doesn't leave the empty loop body. */
9a88cbb5 RB	320	for (i = 0; i < nelems; i++, sg++) {
9a88cbb5 RB	321	if (!plat_device_is_coherent(dev))
e36863a5 DD	322	__dma_sync(sg_page(sg), sg->offset, sg->length,
e36863a5 DD	323	direction);
9a88cbb5	324	}
1da177e4 LT	325	}
1da177e4 LT	326
48e1fd5a	327	int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1da177e4	328	{
843aef49	329	return plat_dma_mapping_error(dev, dma_addr);
1da177e4 LT	330	}
1da177e4 LT	331
48e1fd5a	332	int mips_dma_supported(struct device *dev, u64 mask)
1da177e4	333	{
843aef49	334	return plat_dma_supported(dev, mask);
1da177e4 LT	335	}
1da177e4 LT	336
a3aad4aa	337	void dma_cache_sync(struct device dev, void vaddr, size_t size,
48e1fd5a	338	enum dma_data_direction direction)
1da177e4	339	{
9a88cbb5	340	BUG_ON(direction == DMA_NONE);
1da177e4	341
843aef49	342	plat_extra_sync_for_device(dev);
9a88cbb5	343	if (!plat_device_is_coherent(dev))
e36863a5	344	__dma_sync_virtual(vaddr, size, direction);
1da177e4 LT	345	}
1da177e4 LT	346
a3aad4aa RB	347	EXPORT_SYMBOL(dma_cache_sync);
a3aad4aa RB	348
48e1fd5a	349	static struct dma_map_ops mips_default_dma_map_ops = {
e8d51e54 AP	350	.alloc = mips_dma_alloc_coherent,
e8d51e54 AP	351	.free = mips_dma_free_coherent,
48e1fd5a DD	352	.map_page = mips_dma_map_page,
	353	.unmap_page = mips_dma_unmap_page,
	354	.map_sg = mips_dma_map_sg,
	355	.unmap_sg = mips_dma_unmap_sg,
	356	.sync_single_for_cpu = mips_dma_sync_single_for_cpu,
	357	.sync_single_for_device = mips_dma_sync_single_for_device,
	358	.sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
	359	.sync_sg_for_device = mips_dma_sync_sg_for_device,
	360	.mapping_error = mips_dma_mapping_error,
	361	.dma_supported = mips_dma_supported
	362	};
	363
	364	struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
	365	EXPORT_SYMBOL(mips_dma_map_ops);
	366
	367	#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
	368
	369	static int __init mips_dma_init(void)
	370	{
	371	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	372
	373	return 0;
	374	}
	375	fs_initcall(mips_dma_init);