[linux-2.6-block.git] / kernel / dma / remap.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2012 ARM Ltd.
 * Copyright (c) 2014 The Linux Foundation
 */
#include <linux/dma-direct.h>
#include <linux/dma-noncoherent.h>
#include <linux/dma-contiguous.h>
#include <linux/init.h>
#include <linux/genalloc.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

struct page **dma_common_find_pages(void *cpu_addr)
{
	struct vm_struct *area = find_vm_area(cpu_addr);

	if (!area || area->flags != VM_DMA_COHERENT)
		return NULL;
	return area->pages;
}

static struct vm_struct *__dma_common_pages_remap(struct page **pages,
			size_t size, pgprot_t prot, const void *caller)
{
	struct vm_struct *area;

	area = get_vm_area_caller(size, VM_DMA_COHERENT, caller);
	if (!area)
		return NULL;

	if (map_vm_area(area, prot, pages)) {
		vunmap(area->addr);
		return NULL;
	}

	return area;
}

/*
 * Remaps an array of PAGE_SIZE pages into another vm_area.
 * Cannot be used in non-sleeping contexts
 */
void *dma_common_pages_remap(struct page **pages, size_t size,
			 pgprot_t prot, const void *caller)
{
	struct vm_struct *area;

	area = __dma_common_pages_remap(pages, size, prot, caller);
	if (!area)
		return NULL;

	area->pages = pages;

	return area->addr;
}

/*
 * Remaps an allocated contiguous region into another vm_area.
 * Cannot be used in non-sleeping contexts
 */
void *dma_common_contiguous_remap(struct page *page, size_t size,
			pgprot_t prot, const void *caller)
{
	int i;
	struct page **pages;
	struct vm_struct *area;

	pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
	if (!pages)
		return NULL;

	for (i = 0; i < (size >> PAGE_SHIFT); i++)
		pages[i] = nth_page(page, i);

	area = __dma_common_pages_remap(pages, size, prot, caller);

	kfree(pages);

	if (!area)
		return NULL;
	return area->addr;
}

/*
 * Unmaps a range previously mapped by dma_common_*_remap
 */
void dma_common_free_remap(void *cpu_addr, size_t size)
{
	struct vm_struct *area = find_vm_area(cpu_addr);

	if (!area || area->flags != VM_DMA_COHERENT) {
		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
		return;
	}

	unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
	vunmap(cpu_addr);
}

#ifdef CONFIG_DMA_DIRECT_REMAP
static struct gen_pool *atomic_pool __ro_after_init;

#define DEFAULT_DMA_COHERENT_POOL_SIZE  SZ_256K
static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;

static int __init early_coherent_pool(char *p)
{
	atomic_pool_size = memparse(p, &p);
	return 0;
}
early_param("coherent_pool", early_coherent_pool);

static gfp_t dma_atomic_pool_gfp(void)
{
	if (IS_ENABLED(CONFIG_ZONE_DMA))
		return GFP_DMA;
	if (IS_ENABLED(CONFIG_ZONE_DMA32))
		return GFP_DMA32;
	return GFP_KERNEL;
}

static int __init dma_atomic_pool_init(void)
{
	unsigned int pool_size_order = get_order(atomic_pool_size);
	unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
	struct page *page;
	void *addr;
	int ret;

	if (dev_get_cma_area(NULL))
		page = dma_alloc_from_contiguous(NULL, nr_pages,
						 pool_size_order, false);
	else
		page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);
	if (!page)
		goto out;

	arch_dma_prep_coherent(page, atomic_pool_size);

	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
	if (!atomic_pool)
		goto free_page;

	addr = dma_common_contiguous_remap(page, atomic_pool_size,
					   pgprot_dmacoherent(PAGE_KERNEL),
					   __builtin_return_address(0));
	if (!addr)
		goto destroy_genpool;

	ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
				page_to_phys(page), atomic_pool_size, -1);
	if (ret)
		goto remove_mapping;
	gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);

	pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
		atomic_pool_size / 1024);
	return 0;

remove_mapping:
	dma_common_free_remap(addr, atomic_pool_size);
destroy_genpool:
	gen_pool_destroy(atomic_pool);
	atomic_pool = NULL;
free_page:
	if (!dma_release_from_contiguous(NULL, page, nr_pages))
		__free_pages(page, pool_size_order);
out:
	pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
		atomic_pool_size / 1024);
	return -ENOMEM;
}
postcore_initcall(dma_atomic_pool_init);

bool dma_in_atomic_pool(void *start, size_t size)
{
	if (unlikely(!atomic_pool))
		return false;

	return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
}

void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
{
	unsigned long val;
	void *ptr = NULL;

	if (!atomic_pool) {
		WARN(1, "coherent pool not initialised!\n");
		return NULL;
	}

	val = gen_pool_alloc(atomic_pool, size);
	if (val) {
		phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);

		*ret_page = pfn_to_page(__phys_to_pfn(phys));
		ptr = (void *)val;
		memset(ptr, 0, size);
	}

	return ptr;
}

bool dma_free_from_pool(void *start, size_t size)
{
	if (!dma_in_atomic_pool(start, size))
		return false;
	gen_pool_free(atomic_pool, (unsigned long)start, size);
	return true;
}

void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
		gfp_t flags, unsigned long attrs)
{
	struct page *page = NULL;
	void *ret;

	size = PAGE_ALIGN(size);

	if (!gfpflags_allow_blocking(flags)) {
		ret = dma_alloc_from_pool(size, &page, flags);
		if (!ret)
			return NULL;
		goto done;
	}

	page = __dma_direct_alloc_pages(dev, size, flags, attrs);
	if (!page)
		return NULL;

	/* remove any dirty cache lines on the kernel alias */
	arch_dma_prep_coherent(page, size);

	/* create a coherent mapping */
	ret = dma_common_contiguous_remap(page, size,
			dma_pgprot(dev, PAGE_KERNEL, attrs),
			__builtin_return_address(0));
	if (!ret) {
		dma_free_contiguous(dev, page, size);
		return ret;
	}

	memset(ret, 0, size);
done:
	*dma_handle = phys_to_dma(dev, page_to_phys(page));
	return ret;
}

void arch_dma_free(struct device *dev, size_t size, void *vaddr,
		dma_addr_t dma_handle, unsigned long attrs)
{
	if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) {
		phys_addr_t phys = dma_to_phys(dev, dma_handle);
		struct page *page = pfn_to_page(__phys_to_pfn(phys));

		vunmap(vaddr);
		dma_free_contiguous(dev, page, size);
	}
}
#endif /* CONFIG_DMA_DIRECT_REMAP */
Commit	Line	Data
f0edfea8 CH	1	// SPDX-License-Identifier: GPL-2.0
f0edfea8 CH	2	/*
0c3b3171	3	* Copyright (C) 2012 ARM Ltd.
f0edfea8 CH	4	* Copyright (c) 2014 The Linux Foundation
f0edfea8 CH	5	*/
0c3b3171 CH	6	#include <linux/dma-direct.h>
	7	#include <linux/dma-noncoherent.h>
	8	#include <linux/dma-contiguous.h>
	9	#include <linux/init.h>
	10	#include <linux/genalloc.h>
f0edfea8 CH	11	#include <linux/slab.h>
	12	#include <linux/vmalloc.h>
	13
5cf45379 CH	14	struct page *dma_common_find_pages(void cpu_addr)
	15	{
	16	struct vm_struct *area = find_vm_area(cpu_addr);
	17
	18	if (!area \|\| area->flags != VM_DMA_COHERENT)
	19	return NULL;
	20	return area->pages;
	21	}
	22
f0edfea8	23	static struct vm_struct __dma_common_pages_remap(struct page *pages,
51231740	24	size_t size, pgprot_t prot, const void *caller)
f0edfea8 CH	25	{
	26	struct vm_struct *area;
	27
51231740	28	area = get_vm_area_caller(size, VM_DMA_COHERENT, caller);
f0edfea8 CH	29	if (!area)
	30	return NULL;
	31
	32	if (map_vm_area(area, prot, pages)) {
	33	vunmap(area->addr);
	34	return NULL;
	35	}
	36
	37	return area;
	38	}
	39
	40	/*
	41	* Remaps an array of PAGE_SIZE pages into another vm_area.
	42	* Cannot be used in non-sleeping contexts
	43	*/
	44	void dma_common_pages_remap(struct page *pages, size_t size,
51231740	45	pgprot_t prot, const void *caller)
f0edfea8 CH	46	{
	47	struct vm_struct *area;
	48
51231740	49	area = __dma_common_pages_remap(pages, size, prot, caller);
f0edfea8 CH	50	if (!area)
	51	return NULL;
	52
	53	area->pages = pages;
	54
	55	return area->addr;
	56	}
	57
	58	/*
	59	* Remaps an allocated contiguous region into another vm_area.
	60	* Cannot be used in non-sleeping contexts
	61	*/
	62	void dma_common_contiguous_remap(struct page page, size_t size,
f0edfea8 CH	63	pgprot_t prot, const void *caller)
	64	{
	65	int i;
	66	struct page **pages;
	67	struct vm_struct *area;
	68
	69	pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
	70	if (!pages)
	71	return NULL;
	72
	73	for (i = 0; i < (size >> PAGE_SHIFT); i++)
	74	pages[i] = nth_page(page, i);
	75
51231740	76	area = __dma_common_pages_remap(pages, size, prot, caller);
f0edfea8 CH	77
	78	kfree(pages);
	79
	80	if (!area)
	81	return NULL;
	82	return area->addr;
	83	}
	84
	85	/*
	86	* Unmaps a range previously mapped by dma_common_*_remap
	87	*/
51231740	88	void dma_common_free_remap(void *cpu_addr, size_t size)
f0edfea8	89	{
2cf2aa6a	90	struct vm_struct *area = find_vm_area(cpu_addr);
f0edfea8	91
2cf2aa6a	92	if (!area \|\| area->flags != VM_DMA_COHERENT) {
f0edfea8 CH	93	WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
	94	return;
	95	}
	96
	97	unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
	98	vunmap(cpu_addr);
	99	}
0c3b3171 CH	100
	101	#ifdef CONFIG_DMA_DIRECT_REMAP
	102	static struct gen_pool *atomic_pool __ro_after_init;
	103
	104	#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
	105	static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
	106
	107	static int __init early_coherent_pool(char *p)
	108	{
	109	atomic_pool_size = memparse(p, &p);
	110	return 0;
	111	}
	112	early_param("coherent_pool", early_coherent_pool);
	113
8e3a68fb CH	114	static gfp_t dma_atomic_pool_gfp(void)
	115	{
	116	if (IS_ENABLED(CONFIG_ZONE_DMA))
	117	return GFP_DMA;
	118	if (IS_ENABLED(CONFIG_ZONE_DMA32))
	119	return GFP_DMA32;
	120	return GFP_KERNEL;
	121	}
	122
	123	static int __init dma_atomic_pool_init(void)
0c3b3171 CH	124	{
	125	unsigned int pool_size_order = get_order(atomic_pool_size);
	126	unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
	127	struct page *page;
	128	void *addr;
	129	int ret;
	130
	131	if (dev_get_cma_area(NULL))
	132	page = dma_alloc_from_contiguous(NULL, nr_pages,
	133	pool_size_order, false);
	134	else
8e3a68fb	135	page = alloc_pages(dma_atomic_pool_gfp(), pool_size_order);
0c3b3171 CH	136	if (!page)
	137	goto out;
	138
0c3b3171 CH	139	arch_dma_prep_coherent(page, atomic_pool_size);
	140
	141	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
	142	if (!atomic_pool)
	143	goto free_page;
	144
51231740	145	addr = dma_common_contiguous_remap(page, atomic_pool_size,
8e3a68fb CH	146	pgprot_dmacoherent(PAGE_KERNEL),
8e3a68fb CH	147	__builtin_return_address(0));
0c3b3171 CH	148	if (!addr)
	149	goto destroy_genpool;
	150
	151	ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
	152	page_to_phys(page), atomic_pool_size, -1);
	153	if (ret)
	154	goto remove_mapping;
	155	gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
	156
	157	pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
	158	atomic_pool_size / 1024);
	159	return 0;
	160
	161	remove_mapping:
51231740	162	dma_common_free_remap(addr, atomic_pool_size);
0c3b3171 CH	163	destroy_genpool:
	164	gen_pool_destroy(atomic_pool);
	165	atomic_pool = NULL;
	166	free_page:
	167	if (!dma_release_from_contiguous(NULL, page, nr_pages))
	168	__free_pages(page, pool_size_order);
	169	out:
	170	pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
	171	atomic_pool_size / 1024);
	172	return -ENOMEM;
	173	}
8e3a68fb	174	postcore_initcall(dma_atomic_pool_init);
0c3b3171 CH	175
	176	bool dma_in_atomic_pool(void *start, size_t size)
	177	{
4b4b077c FF	178	if (unlikely(!atomic_pool))
	179	return false;
	180
0c3b3171 CH	181	return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
	182	}
	183
	184	void dma_alloc_from_pool(size_t size, struct page *ret_page, gfp_t flags)
	185	{
	186	unsigned long val;
	187	void *ptr = NULL;
	188
	189	if (!atomic_pool) {
	190	WARN(1, "coherent pool not initialised!\n");
	191	return NULL;
	192	}
	193
	194	val = gen_pool_alloc(atomic_pool, size);
	195	if (val) {
	196	phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
	197
	198	*ret_page = pfn_to_page(__phys_to_pfn(phys));
	199	ptr = (void *)val;
	200	memset(ptr, 0, size);
	201	}
	202
	203	return ptr;
	204	}
	205
	206	bool dma_free_from_pool(void *start, size_t size)
	207	{
	208	if (!dma_in_atomic_pool(start, size))
	209	return false;
	210	gen_pool_free(atomic_pool, (unsigned long)start, size);
	211	return true;
	212	}
	213
	214	void arch_dma_alloc(struct device dev, size_t size, dma_addr_t *dma_handle,
	215	gfp_t flags, unsigned long attrs)
	216	{
	217	struct page *page = NULL;
bfd56cd6	218	void *ret;
0c3b3171 CH	219
	220	size = PAGE_ALIGN(size);
	221
d98849af	222	if (!gfpflags_allow_blocking(flags)) {
0c3b3171 CH	223	ret = dma_alloc_from_pool(size, &page, flags);
	224	if (!ret)
	225	return NULL;
8270f3a1	226	goto done;
0c3b3171 CH	227	}
0c3b3171 CH	228
4e1003aa	229	page = __dma_direct_alloc_pages(dev, size, flags, attrs);
bfd56cd6	230	if (!page)
0c3b3171	231	return NULL;
0c3b3171 CH	232
	233	/* remove any dirty cache lines on the kernel alias */
	234	arch_dma_prep_coherent(page, size);
	235
	236	/* create a coherent mapping */
51231740	237	ret = dma_common_contiguous_remap(page, size,
33dcb37c	238	dma_pgprot(dev, PAGE_KERNEL, attrs),
0c3b3171	239	__builtin_return_address(0));
a1da439c	240	if (!ret) {
acaade1a	241	dma_free_contiguous(dev, page, size);
a1da439c MS	242	return ret;
	243	}
	244
a1da439c	245	memset(ret, 0, size);
8270f3a1 CH	246	done:
8270f3a1 CH	247	*dma_handle = phys_to_dma(dev, page_to_phys(page));
0c3b3171 CH	248	return ret;
	249	}
	250
	251	void arch_dma_free(struct device dev, size_t size, void vaddr,
	252	dma_addr_t dma_handle, unsigned long attrs)
	253	{
d98849af	254	if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) {
bfd56cd6 CH	255	phys_addr_t phys = dma_to_phys(dev, dma_handle);
bfd56cd6 CH	256	struct page *page = pfn_to_page(__phys_to_pfn(phys));
0c3b3171 CH	257
0c3b3171 CH	258	vunmap(vaddr);
acaade1a	259	dma_free_contiguous(dev, page, size);
0c3b3171 CH	260	}
0c3b3171 CH	261	}
0c3b3171	262	#endif /* CONFIG_DMA_DIRECT_REMAP */