[linux-2.6-block.git] / arch / mips / mm / dma-default.c

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 2000  Ani Joshi <ajoshi@unixbox.com>
 * Copyright (C) 2000, 2001, 06	 Ralf Baechle <ralf@linux-mips.org>
 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
 */

#include <linux/types.h>
#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/string.h>
#include <linux/gfp.h>
#include <linux/highmem.h>
#include <linux/dma-contiguous.h>

#include <asm/cache.h>
#include <asm/cpu-type.h>
#include <asm/io.h>

#include <dma-coherence.h>

#ifdef CONFIG_DMA_MAYBE_COHERENT
int coherentio = 0;	/* User defined DMA coherency from command line. */
EXPORT_SYMBOL_GPL(coherentio);
int hw_coherentio = 0;	/* Actual hardware supported DMA coherency setting. */

static int __init setcoherentio(char *str)
{
	coherentio = 1;
	pr_info("Hardware DMA cache coherency (command line)\n");
	return 0;
}
early_param("coherentio", setcoherentio);

static int __init setnocoherentio(char *str)
{
	coherentio = 0;
	pr_info("Software DMA cache coherency (command line)\n");
	return 0;
}
early_param("nocoherentio", setnocoherentio);
#endif

static inline struct page *dma_addr_to_page(struct device *dev,
	dma_addr_t dma_addr)
{
	return pfn_to_page(
		plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
}

/*
 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
 * speculatively fill random cachelines with stale data at any time,
 * requiring an extra flush post-DMA.
 *
 * Warning on the terminology - Linux calls an uncached area coherent;
 * MIPS terminology calls memory areas with hardware maintained coherency
 * coherent.
 *
 * Note that the R14000 and R16000 should also be checked for in this
 * condition.  However this function is only called on non-I/O-coherent
 * systems and only the R10000 and R12000 are used in such systems, the
 * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
 */
static inline int cpu_needs_post_dma_flush(struct device *dev)
{
	return !plat_device_is_coherent(dev) &&
	       (boot_cpu_type() == CPU_R10000 ||
		boot_cpu_type() == CPU_R12000 ||
		boot_cpu_type() == CPU_BMIPS5000);
}

static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
{
	gfp_t dma_flag;

	/* ignore region specifiers */
	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);

#ifdef CONFIG_ISA
	if (dev == NULL)
		dma_flag = __GFP_DMA;
	else
#endif
#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
			dma_flag = __GFP_DMA;
	else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
			dma_flag = __GFP_DMA32;
	else
#endif
#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
		dma_flag = __GFP_DMA32;
	else
#endif
#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
		dma_flag = __GFP_DMA;
	else
#endif
		dma_flag = 0;

	/* Don't invoke OOM killer */
	gfp |= __GFP_NORETRY;

	return gfp | dma_flag;
}

void *dma_alloc_noncoherent(struct device *dev, size_t size,
	dma_addr_t * dma_handle, gfp_t gfp)
{
	void *ret;

	gfp = massage_gfp_flags(dev, gfp);

	ret = (void *) __get_free_pages(gfp, get_order(size));

	if (ret != NULL) {
		memset(ret, 0, size);
		*dma_handle = plat_map_dma_mem(dev, ret, size);
	}

	return ret;
}
EXPORT_SYMBOL(dma_alloc_noncoherent);

static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
	dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
{
	void *ret;
	struct page *page = NULL;
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

	if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
		return ret;

	gfp = massage_gfp_flags(dev, gfp);

	if (IS_ENABLED(CONFIG_DMA_CMA) && !(gfp & GFP_ATOMIC))
		page = dma_alloc_from_contiguous(dev,
					count, get_order(size));
	if (!page)
		page = alloc_pages(gfp, get_order(size));

	if (!page)
		return NULL;

	ret = page_address(page);
	memset(ret, 0, size);
	*dma_handle = plat_map_dma_mem(dev, ret, size);
	if (!plat_device_is_coherent(dev)) {
		dma_cache_wback_inv((unsigned long) ret, size);
		if (!hw_coherentio)
			ret = UNCAC_ADDR(ret);
	}

	return ret;
}


void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
	dma_addr_t dma_handle)
{
	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
	free_pages((unsigned long) vaddr, get_order(size));
}
EXPORT_SYMBOL(dma_free_noncoherent);

static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
	dma_addr_t dma_handle, struct dma_attrs *attrs)
{
	unsigned long addr = (unsigned long) vaddr;
	int order = get_order(size);
	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
	struct page *page = NULL;

	if (dma_release_from_coherent(dev, order, vaddr))
		return;

	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);

	if (!plat_device_is_coherent(dev) && !hw_coherentio)
		addr = CAC_ADDR(addr);

	page = virt_to_page((void *) addr);

	if (!dma_release_from_contiguous(dev, page, count))
		__free_pages(page, get_order(size));
}

static inline void __dma_sync_virtual(void *addr, size_t size,
	enum dma_data_direction direction)
{
	switch (direction) {
	case DMA_TO_DEVICE:
		dma_cache_wback((unsigned long)addr, size);
		break;

	case DMA_FROM_DEVICE:
		dma_cache_inv((unsigned long)addr, size);
		break;

	case DMA_BIDIRECTIONAL:
		dma_cache_wback_inv((unsigned long)addr, size);
		break;

	default:
		BUG();
	}
}

/*
 * A single sg entry may refer to multiple physically contiguous
 * pages. But we still need to process highmem pages individually.
 * If highmem is not configured then the bulk of this loop gets
 * optimized out.
 */
static inline void __dma_sync(struct page *page,
	unsigned long offset, size_t size, enum dma_data_direction direction)
{
	size_t left = size;

	do {
		size_t len = left;

		if (PageHighMem(page)) {
			void *addr;

			if (offset + len > PAGE_SIZE) {
				if (offset >= PAGE_SIZE) {
					page += offset >> PAGE_SHIFT;
					offset &= ~PAGE_MASK;
				}
				len = PAGE_SIZE - offset;
			}

			addr = kmap_atomic(page);
			__dma_sync_virtual(addr + offset, len, direction);
			kunmap_atomic(addr);
		} else
			__dma_sync_virtual(page_address(page) + offset,
					   size, direction);
		offset = 0;
		page++;
		left -= len;
	} while (left);
}

static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
	size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
{
	if (cpu_needs_post_dma_flush(dev))
		__dma_sync(dma_addr_to_page(dev, dma_addr),
			   dma_addr & ~PAGE_MASK, size, direction);
	plat_post_dma_flush(dev);
	plat_unmap_dma_mem(dev, dma_addr, size, direction);
}

static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
	int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
{
	int i;

	for (i = 0; i < nents; i++, sg++) {
		if (!plat_device_is_coherent(dev))
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		sg->dma_length = sg->length;
#endif
		sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
				  sg->offset;
	}

	return nents;
}

static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
	unsigned long offset, size_t size, enum dma_data_direction direction,
	struct dma_attrs *attrs)
{
	if (!plat_device_is_coherent(dev))
		__dma_sync(page, offset, size, direction);

	return plat_map_dma_mem_page(dev, page) + offset;
}

static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
	int nhwentries, enum dma_data_direction direction,
	struct dma_attrs *attrs)
{
	int i;

	for (i = 0; i < nhwentries; i++, sg++) {
		if (!plat_device_is_coherent(dev) &&
		    direction != DMA_TO_DEVICE)
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
		plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
	}
}

static void mips_dma_sync_single_for_cpu(struct device *dev,
	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
	if (cpu_needs_post_dma_flush(dev))
		__dma_sync(dma_addr_to_page(dev, dma_handle),
			   dma_handle & ~PAGE_MASK, size, direction);
	plat_post_dma_flush(dev);
}

static void mips_dma_sync_single_for_device(struct device *dev,
	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
	if (!plat_device_is_coherent(dev))
		__dma_sync(dma_addr_to_page(dev, dma_handle),
			   dma_handle & ~PAGE_MASK, size, direction);
}

static void mips_dma_sync_sg_for_cpu(struct device *dev,
	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
{
	int i;

	if (cpu_needs_post_dma_flush(dev))
		for (i = 0; i < nelems; i++, sg++)
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
	plat_post_dma_flush(dev);
}

static void mips_dma_sync_sg_for_device(struct device *dev,
	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
{
	int i;

	if (!plat_device_is_coherent(dev))
		for (i = 0; i < nelems; i++, sg++)
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
}

int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
	return 0;
}

int mips_dma_supported(struct device *dev, u64 mask)
{
	return plat_dma_supported(dev, mask);
}

void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
			 enum dma_data_direction direction)
{
	BUG_ON(direction == DMA_NONE);

	if (!plat_device_is_coherent(dev))
		__dma_sync_virtual(vaddr, size, direction);
}

EXPORT_SYMBOL(dma_cache_sync);

static struct dma_map_ops mips_default_dma_map_ops = {
	.alloc = mips_dma_alloc_coherent,
	.free = mips_dma_free_coherent,
	.map_page = mips_dma_map_page,
	.unmap_page = mips_dma_unmap_page,
	.map_sg = mips_dma_map_sg,
	.unmap_sg = mips_dma_unmap_sg,
	.sync_single_for_cpu = mips_dma_sync_single_for_cpu,
	.sync_single_for_device = mips_dma_sync_single_for_device,
	.sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
	.sync_sg_for_device = mips_dma_sync_sg_for_device,
	.mapping_error = mips_dma_mapping_error,
	.dma_supported = mips_dma_supported
};

struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
EXPORT_SYMBOL(mips_dma_map_ops);

#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)

static int __init mips_dma_init(void)
{
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

	return 0;
}
fs_initcall(mips_dma_init);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
70342287	7	* Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
1da177e4 LT	8	* swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
1da177e4 LT	9	*/
9a88cbb5	10
1da177e4	11	#include <linux/types.h>
9a88cbb5	12	#include <linux/dma-mapping.h>
1da177e4 LT	13	#include <linux/mm.h>
1da177e4 LT	14	#include <linux/module.h>
4fcc47a0	15	#include <linux/scatterlist.h>
6e86b0bf	16	#include <linux/string.h>
5a0e3ad6	17	#include <linux/gfp.h>
e36863a5	18	#include <linux/highmem.h>
f4649382	19	#include <linux/dma-contiguous.h>
1da177e4 LT	20
1da177e4 LT	21	#include <asm/cache.h>
69f24d17	22	#include <asm/cpu-type.h>
1da177e4 LT	23	#include <asm/io.h>
1da177e4 LT	24
9a88cbb5 RB	25	#include <dma-coherence.h>
9a88cbb5 RB	26
885014bc	27	#ifdef CONFIG_DMA_MAYBE_COHERENT
b6d92b4a SH	28	int coherentio = 0; /* User defined DMA coherency from command line. */
	29	EXPORT_SYMBOL_GPL(coherentio);
	30	int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
	31
	32	static int __init setcoherentio(char *str)
	33	{
	34	coherentio = 1;
	35	pr_info("Hardware DMA cache coherency (command line)\n");
	36	return 0;
	37	}
	38	early_param("coherentio", setcoherentio);
	39
	40	static int __init setnocoherentio(char *str)
	41	{
	42	coherentio = 0;
	43	pr_info("Software DMA cache coherency (command line)\n");
	44	return 0;
	45	}
	46	early_param("nocoherentio", setnocoherentio);
885014bc	47	#endif
b6d92b4a	48
e36863a5	49	static inline struct page dma_addr_to_page(struct device dev,
3807ef3f	50	dma_addr_t dma_addr)
c9d06962	51	{
e36863a5 DD	52	return pfn_to_page(
e36863a5 DD	53	plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
c9d06962 FBH	54	}
c9d06962 FBH	55
1da177e4	56	/*
f86f55d3 JQ	57	* The affected CPUs below in 'cpu_needs_post_dma_flush()' can
	58	* speculatively fill random cachelines with stale data at any time,
	59	* requiring an extra flush post-DMA.
	60	*
1da177e4 LT	61	* Warning on the terminology - Linux calls an uncached area coherent;
	62	* MIPS terminology calls memory areas with hardware maintained coherency
	63	* coherent.
0dc294c0 RB	64	*
	65	* Note that the R14000 and R16000 should also be checked for in this
	66	* condition. However this function is only called on non-I/O-coherent
	67	* systems and only the R10000 and R12000 are used in such systems, the
	68	* SGI IP28 Indigo² rsp. SGI IP32 aka O2.
1da177e4	69	*/
f86f55d3	70	static inline int cpu_needs_post_dma_flush(struct device *dev)
9a88cbb5 RB	71	{
9a88cbb5 RB	72	return !plat_device_is_coherent(dev) &&
d451e734	73	(boot_cpu_type() == CPU_R10000 \|\|
eb37e6dd RB	74	boot_cpu_type() == CPU_R12000 \|\|
eb37e6dd RB	75	boot_cpu_type() == CPU_BMIPS5000);
9a88cbb5 RB	76	}
9a88cbb5 RB	77
cce335ae RB	78	static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
cce335ae RB	79	{
a2e715a8 RB	80	gfp_t dma_flag;
a2e715a8 RB	81
cce335ae RB	82	/* ignore region specifiers */
	83	gfp &= ~(__GFP_DMA \| __GFP_DMA32 \| __GFP_HIGHMEM);
	84
a2e715a8	85	#ifdef CONFIG_ISA
cce335ae	86	if (dev == NULL)
a2e715a8	87	dma_flag = __GFP_DMA;
cce335ae RB	88	else
cce335ae RB	89	#endif
a2e715a8	90	#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
cce335ae	91	if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
a2e715a8 RB	92	dma_flag = __GFP_DMA;
	93	else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	94	dma_flag = __GFP_DMA32;
	95	else
	96	#endif
	97	#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
	98	if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	99	dma_flag = __GFP_DMA32;
	100	else
	101	#endif
	102	#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
	103	if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	104	dma_flag = __GFP_DMA;
cce335ae RB	105	else
cce335ae RB	106	#endif
a2e715a8	107	dma_flag = 0;
cce335ae RB	108
	109	/* Don't invoke OOM killer */
	110	gfp \|= __GFP_NORETRY;
	111
a2e715a8	112	return gfp \| dma_flag;
cce335ae RB	113	}
cce335ae RB	114
1da177e4	115	void dma_alloc_noncoherent(struct device dev, size_t size,
185a8ff5	116	dma_addr_t * dma_handle, gfp_t gfp)
1da177e4 LT	117	{
1da177e4 LT	118	void *ret;
9a88cbb5	119
cce335ae	120	gfp = massage_gfp_flags(dev, gfp);
1da177e4	121
1da177e4 LT	122	ret = (void *) __get_free_pages(gfp, get_order(size));
	123
	124	if (ret != NULL) {
	125	memset(ret, 0, size);
9a88cbb5	126	*dma_handle = plat_map_dma_mem(dev, ret, size);
1da177e4 LT	127	}
	128
	129	return ret;
	130	}
1da177e4 LT	131	EXPORT_SYMBOL(dma_alloc_noncoherent);
1da177e4 LT	132
48e1fd5a	133	static void mips_dma_alloc_coherent(struct device dev, size_t size,
e8d51e54	134	dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
1da177e4 LT	135	{
1da177e4 LT	136	void *ret;
f4649382 ZLK	137	struct page *page = NULL;
f4649382 ZLK	138	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1da177e4	139
f8ac0425 YY	140	if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
	141	return ret;
	142
cce335ae	143	gfp = massage_gfp_flags(dev, gfp);
9a88cbb5	144
f4649382 ZLK	145	if (IS_ENABLED(CONFIG_DMA_CMA) && !(gfp & GFP_ATOMIC))
	146	page = dma_alloc_from_contiguous(dev,
	147	count, get_order(size));
	148	if (!page)
	149	page = alloc_pages(gfp, get_order(size));
	150
	151	if (!page)
	152	return NULL;
	153
	154	ret = page_address(page);
	155	memset(ret, 0, size);
	156	*dma_handle = plat_map_dma_mem(dev, ret, size);
	157	if (!plat_device_is_coherent(dev)) {
	158	dma_cache_wback_inv((unsigned long) ret, size);
	159	if (!hw_coherentio)
	160	ret = UNCAC_ADDR(ret);
1da177e4 LT	161	}
	162
	163	return ret;
	164	}
	165
1da177e4 LT	166
	167	void dma_free_noncoherent(struct device dev, size_t size, void vaddr,
	168	dma_addr_t dma_handle)
	169	{
d3f634b9	170	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
1da177e4 LT	171	free_pages((unsigned long) vaddr, get_order(size));
1da177e4 LT	172	}
1da177e4 LT	173	EXPORT_SYMBOL(dma_free_noncoherent);
1da177e4 LT	174
48e1fd5a	175	static void mips_dma_free_coherent(struct device dev, size_t size, void vaddr,
e8d51e54	176	dma_addr_t dma_handle, struct dma_attrs *attrs)
1da177e4 LT	177	{
1da177e4 LT	178	unsigned long addr = (unsigned long) vaddr;
f8ac0425	179	int order = get_order(size);
f4649382 ZLK	180	unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
f4649382 ZLK	181	struct page *page = NULL;
f8ac0425 YY	182
	183	if (dma_release_from_coherent(dev, order, vaddr))
	184	return;
1da177e4	185
d3f634b9	186	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
11531ac2	187
b6d92b4a	188	if (!plat_device_is_coherent(dev) && !hw_coherentio)
9a88cbb5 RB	189	addr = CAC_ADDR(addr);
9a88cbb5 RB	190
f4649382 ZLK	191	page = virt_to_page((void *) addr);
	192
	193	if (!dma_release_from_contiguous(dev, page, count))
	194	__free_pages(page, get_order(size));
1da177e4 LT	195	}
1da177e4 LT	196
e36863a5	197	static inline void __dma_sync_virtual(void *addr, size_t size,
1da177e4 LT	198	enum dma_data_direction direction)
	199	{
	200	switch (direction) {
	201	case DMA_TO_DEVICE:
e36863a5	202	dma_cache_wback((unsigned long)addr, size);
1da177e4 LT	203	break;
	204
	205	case DMA_FROM_DEVICE:
e36863a5	206	dma_cache_inv((unsigned long)addr, size);
1da177e4 LT	207	break;
	208
	209	case DMA_BIDIRECTIONAL:
e36863a5	210	dma_cache_wback_inv((unsigned long)addr, size);
1da177e4 LT	211	break;
	212
	213	default:
	214	BUG();
	215	}
	216	}
	217
e36863a5 DD	218	/*
	219	* A single sg entry may refer to multiple physically contiguous
	220	* pages. But we still need to process highmem pages individually.
	221	* If highmem is not configured then the bulk of this loop gets
	222	* optimized out.
	223	*/
	224	static inline void __dma_sync(struct page *page,
	225	unsigned long offset, size_t size, enum dma_data_direction direction)
	226	{
	227	size_t left = size;
	228
	229	do {
	230	size_t len = left;
	231
	232	if (PageHighMem(page)) {
	233	void *addr;
	234
	235	if (offset + len > PAGE_SIZE) {
	236	if (offset >= PAGE_SIZE) {
	237	page += offset >> PAGE_SHIFT;
	238	offset &= ~PAGE_MASK;
	239	}
	240	len = PAGE_SIZE - offset;
	241	}
	242
	243	addr = kmap_atomic(page);
	244	__dma_sync_virtual(addr + offset, len, direction);
	245	kunmap_atomic(addr);
	246	} else
	247	__dma_sync_virtual(page_address(page) + offset,
	248	size, direction);
	249	offset = 0;
	250	page++;
	251	left -= len;
	252	} while (left);
	253	}
	254
48e1fd5a DD	255	static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
48e1fd5a DD	256	size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
1da177e4	257	{
f86f55d3	258	if (cpu_needs_post_dma_flush(dev))
e36863a5 DD	259	__dma_sync(dma_addr_to_page(dev, dma_addr),
e36863a5 DD	260	dma_addr & ~PAGE_MASK, size, direction);
0acbfc66	261	plat_post_dma_flush(dev);
d3f634b9	262	plat_unmap_dma_mem(dev, dma_addr, size, direction);
1da177e4 LT	263	}
1da177e4 LT	264
48e1fd5a DD	265	static int mips_dma_map_sg(struct device dev, struct scatterlist sg,
48e1fd5a DD	266	int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
1da177e4 LT	267	{
	268	int i;
	269
1da177e4	270	for (i = 0; i < nents; i++, sg++) {
e36863a5 DD	271	if (!plat_device_is_coherent(dev))
	272	__dma_sync(sg_page(sg), sg->offset, sg->length,
	273	direction);
4954a9a2 J	274	#ifdef CONFIG_NEED_SG_DMA_LENGTH
	275	sg->dma_length = sg->length;
	276	#endif
e36863a5 DD	277	sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
e36863a5 DD	278	sg->offset;
1da177e4 LT	279	}
	280
	281	return nents;
	282	}
	283
48e1fd5a DD	284	static dma_addr_t mips_dma_map_page(struct device dev, struct page page,
	285	unsigned long offset, size_t size, enum dma_data_direction direction,
	286	struct dma_attrs *attrs)
1da177e4	287	{
48e1fd5a	288	if (!plat_device_is_coherent(dev))
e36863a5	289	__dma_sync(page, offset, size, direction);
1da177e4	290
e36863a5	291	return plat_map_dma_mem_page(dev, page) + offset;
1da177e4 LT	292	}
1da177e4 LT	293
48e1fd5a DD	294	static void mips_dma_unmap_sg(struct device dev, struct scatterlist sg,
	295	int nhwentries, enum dma_data_direction direction,
	296	struct dma_attrs *attrs)
1da177e4	297	{
1da177e4 LT	298	int i;
1da177e4 LT	299
1da177e4	300	for (i = 0; i < nhwentries; i++, sg++) {
9a88cbb5	301	if (!plat_device_is_coherent(dev) &&
e36863a5 DD	302	direction != DMA_TO_DEVICE)
	303	__dma_sync(sg_page(sg), sg->offset, sg->length,
	304	direction);
d3f634b9	305	plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
1da177e4 LT	306	}
	307	}
	308
48e1fd5a DD	309	static void mips_dma_sync_single_for_cpu(struct device *dev,
48e1fd5a DD	310	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
1da177e4	311	{
f86f55d3	312	if (cpu_needs_post_dma_flush(dev))
e36863a5 DD	313	__dma_sync(dma_addr_to_page(dev, dma_handle),
e36863a5 DD	314	dma_handle & ~PAGE_MASK, size, direction);
0acbfc66	315	plat_post_dma_flush(dev);
1da177e4 LT	316	}
1da177e4 LT	317
48e1fd5a DD	318	static void mips_dma_sync_single_for_device(struct device *dev,
48e1fd5a DD	319	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
1da177e4	320	{
e36863a5 DD	321	if (!plat_device_is_coherent(dev))
	322	__dma_sync(dma_addr_to_page(dev, dma_handle),
	323	dma_handle & ~PAGE_MASK, size, direction);
1da177e4 LT	324	}
1da177e4 LT	325
48e1fd5a DD	326	static void mips_dma_sync_sg_for_cpu(struct device *dev,
48e1fd5a DD	327	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
1da177e4 LT	328	{
1da177e4 LT	329	int i;
42a3b4f2	330
55c25c2f J	331	if (cpu_needs_post_dma_flush(dev))
55c25c2f J	332	for (i = 0; i < nelems; i++, sg++)
e36863a5 DD	333	__dma_sync(sg_page(sg), sg->offset, sg->length,
e36863a5 DD	334	direction);
0acbfc66	335	plat_post_dma_flush(dev);
1da177e4 LT	336	}
1da177e4 LT	337
48e1fd5a DD	338	static void mips_dma_sync_sg_for_device(struct device *dev,
48e1fd5a DD	339	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
1da177e4 LT	340	{
	341	int i;
	342
55c25c2f J	343	if (!plat_device_is_coherent(dev))
55c25c2f J	344	for (i = 0; i < nelems; i++, sg++)
e36863a5 DD	345	__dma_sync(sg_page(sg), sg->offset, sg->length,
e36863a5 DD	346	direction);
1da177e4 LT	347	}
1da177e4 LT	348
48e1fd5a	349	int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1da177e4	350	{
4e7f7266	351	return 0;
1da177e4 LT	352	}
1da177e4 LT	353
48e1fd5a	354	int mips_dma_supported(struct device *dev, u64 mask)
1da177e4	355	{
843aef49	356	return plat_dma_supported(dev, mask);
1da177e4 LT	357	}
1da177e4 LT	358
a3aad4aa	359	void dma_cache_sync(struct device dev, void vaddr, size_t size,
48e1fd5a	360	enum dma_data_direction direction)
1da177e4	361	{
9a88cbb5	362	BUG_ON(direction == DMA_NONE);
1da177e4	363
9a88cbb5	364	if (!plat_device_is_coherent(dev))
e36863a5	365	__dma_sync_virtual(vaddr, size, direction);
1da177e4 LT	366	}
1da177e4 LT	367
a3aad4aa RB	368	EXPORT_SYMBOL(dma_cache_sync);
a3aad4aa RB	369
48e1fd5a	370	static struct dma_map_ops mips_default_dma_map_ops = {
e8d51e54 AP	371	.alloc = mips_dma_alloc_coherent,
e8d51e54 AP	372	.free = mips_dma_free_coherent,
48e1fd5a DD	373	.map_page = mips_dma_map_page,
	374	.unmap_page = mips_dma_unmap_page,
	375	.map_sg = mips_dma_map_sg,
	376	.unmap_sg = mips_dma_unmap_sg,
	377	.sync_single_for_cpu = mips_dma_sync_single_for_cpu,
	378	.sync_single_for_device = mips_dma_sync_single_for_device,
	379	.sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
	380	.sync_sg_for_device = mips_dma_sync_sg_for_device,
	381	.mapping_error = mips_dma_mapping_error,
	382	.dma_supported = mips_dma_supported
	383	};
	384
	385	struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
	386	EXPORT_SYMBOL(mips_dma_map_ops);
	387
	388	#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
	389
	390	static int __init mips_dma_init(void)
	391	{
	392	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	393
	394	return 0;
	395	}
	396	fs_initcall(mips_dma_init);