[linux-2.6-block.git] / kernel / dma / remap.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2012 ARM Ltd.
 * Copyright (c) 2014 The Linux Foundation
 */
#include <linux/dma-direct.h>
#include <linux/dma-noncoherent.h>
#include <linux/dma-contiguous.h>
#include <linux/init.h>
#include <linux/genalloc.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>

static struct vm_struct *__dma_common_pages_remap(struct page **pages,
			size_t size, unsigned long vm_flags, pgprot_t prot,
			const void *caller)
{
	struct vm_struct *area;

	area = get_vm_area_caller(size, vm_flags, caller);
	if (!area)
		return NULL;

	if (map_vm_area(area, prot, pages)) {
		vunmap(area->addr);
		return NULL;
	}

	return area;
}

/*
 * Remaps an array of PAGE_SIZE pages into another vm_area.
 * Cannot be used in non-sleeping contexts
 */
void *dma_common_pages_remap(struct page **pages, size_t size,
			unsigned long vm_flags, pgprot_t prot,
			const void *caller)
{
	struct vm_struct *area;

	area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller);
	if (!area)
		return NULL;

	area->pages = pages;

	return area->addr;
}

/*
 * Remaps an allocated contiguous region into another vm_area.
 * Cannot be used in non-sleeping contexts
 */
void *dma_common_contiguous_remap(struct page *page, size_t size,
			unsigned long vm_flags,
			pgprot_t prot, const void *caller)
{
	int i;
	struct page **pages;
	struct vm_struct *area;

	pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
	if (!pages)
		return NULL;

	for (i = 0; i < (size >> PAGE_SHIFT); i++)
		pages[i] = nth_page(page, i);

	area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller);

	kfree(pages);

	if (!area)
		return NULL;
	return area->addr;
}

/*
 * Unmaps a range previously mapped by dma_common_*_remap
 */
void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
{
	struct vm_struct *area = find_vm_area(cpu_addr);

	if (!area || (area->flags & vm_flags) != vm_flags) {
		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
		return;
	}

	unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
	vunmap(cpu_addr);
}

#ifdef CONFIG_DMA_DIRECT_REMAP
static struct gen_pool *atomic_pool __ro_after_init;

#define DEFAULT_DMA_COHERENT_POOL_SIZE  SZ_256K
static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;

static int __init early_coherent_pool(char *p)
{
	atomic_pool_size = memparse(p, &p);
	return 0;
}
early_param("coherent_pool", early_coherent_pool);

int __init dma_atomic_pool_init(gfp_t gfp, pgprot_t prot)
{
	unsigned int pool_size_order = get_order(atomic_pool_size);
	unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
	struct page *page;
	void *addr;
	int ret;

	if (dev_get_cma_area(NULL))
		page = dma_alloc_from_contiguous(NULL, nr_pages,
						 pool_size_order, false);
	else
		page = alloc_pages(gfp, pool_size_order);
	if (!page)
		goto out;

	arch_dma_prep_coherent(page, atomic_pool_size);

	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
	if (!atomic_pool)
		goto free_page;

	addr = dma_common_contiguous_remap(page, atomic_pool_size, VM_USERMAP,
					   prot, __builtin_return_address(0));
	if (!addr)
		goto destroy_genpool;

	ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
				page_to_phys(page), atomic_pool_size, -1);
	if (ret)
		goto remove_mapping;
	gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);

	pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
		atomic_pool_size / 1024);
	return 0;

remove_mapping:
	dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP);
destroy_genpool:
	gen_pool_destroy(atomic_pool);
	atomic_pool = NULL;
free_page:
	if (!dma_release_from_contiguous(NULL, page, nr_pages))
		__free_pages(page, pool_size_order);
out:
	pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
		atomic_pool_size / 1024);
	return -ENOMEM;
}

bool dma_in_atomic_pool(void *start, size_t size)
{
	if (unlikely(!atomic_pool))
		return false;

	return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
}

void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags)
{
	unsigned long val;
	void *ptr = NULL;

	if (!atomic_pool) {
		WARN(1, "coherent pool not initialised!\n");
		return NULL;
	}

	val = gen_pool_alloc(atomic_pool, size);
	if (val) {
		phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);

		*ret_page = pfn_to_page(__phys_to_pfn(phys));
		ptr = (void *)val;
		memset(ptr, 0, size);
	}

	return ptr;
}

bool dma_free_from_pool(void *start, size_t size)
{
	if (!dma_in_atomic_pool(start, size))
		return false;
	gen_pool_free(atomic_pool, (unsigned long)start, size);
	return true;
}

void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
		gfp_t flags, unsigned long attrs)
{
	struct page *page = NULL;
	void *ret;

	size = PAGE_ALIGN(size);

	if (!gfpflags_allow_blocking(flags)) {
		ret = dma_alloc_from_pool(size, &page, flags);
		if (!ret)
			return NULL;
		goto done;
	}

	page = __dma_direct_alloc_pages(dev, size, dma_handle, flags, attrs);
	if (!page)
		return NULL;

	/* remove any dirty cache lines on the kernel alias */
	arch_dma_prep_coherent(page, size);

	/* create a coherent mapping */
	ret = dma_common_contiguous_remap(page, size, VM_USERMAP,
			arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs),
			__builtin_return_address(0));
	if (!ret) {
		__dma_direct_free_pages(dev, size, page);
		return ret;
	}

	memset(ret, 0, size);
done:
	*dma_handle = phys_to_dma(dev, page_to_phys(page));
	return ret;
}

void arch_dma_free(struct device *dev, size_t size, void *vaddr,
		dma_addr_t dma_handle, unsigned long attrs)
{
	if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) {
		phys_addr_t phys = dma_to_phys(dev, dma_handle);
		struct page *page = pfn_to_page(__phys_to_pfn(phys));

		vunmap(vaddr);
		__dma_direct_free_pages(dev, size, page);
	}
}

long arch_dma_coherent_to_pfn(struct device *dev, void *cpu_addr,
		dma_addr_t dma_addr)
{
	return __phys_to_pfn(dma_to_phys(dev, dma_addr));
}
#endif /* CONFIG_DMA_DIRECT_REMAP */
Commit	Line	Data
f0edfea8 CH	1	// SPDX-License-Identifier: GPL-2.0
f0edfea8 CH	2	/*
0c3b3171	3	* Copyright (C) 2012 ARM Ltd.
f0edfea8 CH	4	* Copyright (c) 2014 The Linux Foundation
f0edfea8 CH	5	*/
0c3b3171 CH	6	#include <linux/dma-direct.h>
	7	#include <linux/dma-noncoherent.h>
	8	#include <linux/dma-contiguous.h>
	9	#include <linux/init.h>
	10	#include <linux/genalloc.h>
f0edfea8 CH	11	#include <linux/slab.h>
	12	#include <linux/vmalloc.h>
	13
	14	static struct vm_struct __dma_common_pages_remap(struct page *pages,
	15	size_t size, unsigned long vm_flags, pgprot_t prot,
	16	const void *caller)
	17	{
	18	struct vm_struct *area;
	19
	20	area = get_vm_area_caller(size, vm_flags, caller);
	21	if (!area)
	22	return NULL;
	23
	24	if (map_vm_area(area, prot, pages)) {
	25	vunmap(area->addr);
	26	return NULL;
	27	}
	28
	29	return area;
	30	}
	31
	32	/*
	33	* Remaps an array of PAGE_SIZE pages into another vm_area.
	34	* Cannot be used in non-sleeping contexts
	35	*/
	36	void dma_common_pages_remap(struct page *pages, size_t size,
	37	unsigned long vm_flags, pgprot_t prot,
	38	const void *caller)
	39	{
	40	struct vm_struct *area;
	41
	42	area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller);
	43	if (!area)
	44	return NULL;
	45
	46	area->pages = pages;
	47
	48	return area->addr;
	49	}
	50
	51	/*
	52	* Remaps an allocated contiguous region into another vm_area.
	53	* Cannot be used in non-sleeping contexts
	54	*/
	55	void dma_common_contiguous_remap(struct page page, size_t size,
	56	unsigned long vm_flags,
	57	pgprot_t prot, const void *caller)
	58	{
	59	int i;
	60	struct page **pages;
	61	struct vm_struct *area;
	62
	63	pages = kmalloc(sizeof(struct page *) << get_order(size), GFP_KERNEL);
	64	if (!pages)
	65	return NULL;
	66
	67	for (i = 0; i < (size >> PAGE_SHIFT); i++)
	68	pages[i] = nth_page(page, i);
	69
	70	area = __dma_common_pages_remap(pages, size, vm_flags, prot, caller);
	71
	72	kfree(pages);
	73
	74	if (!area)
75	return NULL;
76	return area->addr;
77	}
78
79	/*
80	* Unmaps a range previously mapped by dma_common_*_remap
81	*/
82	void dma_common_free_remap(void *cpu_addr, size_t size, unsigned long vm_flags)
83	{
84	struct vm_struct *area = find_vm_area(cpu_addr);
85
86	if (!area \|\| (area->flags & vm_flags) != vm_flags) {
87	WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
88	return;
89	}
90
91	unmap_kernel_range((unsigned long)cpu_addr, PAGE_ALIGN(size));
92	vunmap(cpu_addr);
93	}
0c3b3171 CH	94
	95	#ifdef CONFIG_DMA_DIRECT_REMAP
	96	static struct gen_pool *atomic_pool __ro_after_init;
	97
	98	#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
	99	static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE;
	100
	101	static int __init early_coherent_pool(char *p)
	102	{
	103	atomic_pool_size = memparse(p, &p);
	104	return 0;
	105	}
	106	early_param("coherent_pool", early_coherent_pool);
	107
	108	int __init dma_atomic_pool_init(gfp_t gfp, pgprot_t prot)
	109	{
	110	unsigned int pool_size_order = get_order(atomic_pool_size);
	111	unsigned long nr_pages = atomic_pool_size >> PAGE_SHIFT;
	112	struct page *page;
	113	void *addr;
	114	int ret;
	115
	116	if (dev_get_cma_area(NULL))
	117	page = dma_alloc_from_contiguous(NULL, nr_pages,
	118	pool_size_order, false);
	119	else
	120	page = alloc_pages(gfp, pool_size_order);
	121	if (!page)
	122	goto out;
	123
0c3b3171 CH	124	arch_dma_prep_coherent(page, atomic_pool_size);
	125
	126	atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
	127	if (!atomic_pool)
	128	goto free_page;
	129
	130	addr = dma_common_contiguous_remap(page, atomic_pool_size, VM_USERMAP,
	131	prot, __builtin_return_address(0));
	132	if (!addr)
	133	goto destroy_genpool;
	134
	135	ret = gen_pool_add_virt(atomic_pool, (unsigned long)addr,
	136	page_to_phys(page), atomic_pool_size, -1);
	137	if (ret)
	138	goto remove_mapping;
	139	gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, NULL);
	140
	141	pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
	142	atomic_pool_size / 1024);
	143	return 0;
	144
	145	remove_mapping:
	146	dma_common_free_remap(addr, atomic_pool_size, VM_USERMAP);
	147	destroy_genpool:
	148	gen_pool_destroy(atomic_pool);
	149	atomic_pool = NULL;
	150	free_page:
	151	if (!dma_release_from_contiguous(NULL, page, nr_pages))
	152	__free_pages(page, pool_size_order);
	153	out:
	154	pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
	155	atomic_pool_size / 1024);
	156	return -ENOMEM;
	157	}
	158
	159	bool dma_in_atomic_pool(void *start, size_t size)
	160	{
4b4b077c FF	161	if (unlikely(!atomic_pool))
	162	return false;
	163
0c3b3171 CH	164	return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
	165	}
	166
	167	void dma_alloc_from_pool(size_t size, struct page *ret_page, gfp_t flags)
	168	{
	169	unsigned long val;
	170	void *ptr = NULL;
	171
	172	if (!atomic_pool) {
	173	WARN(1, "coherent pool not initialised!\n");
	174	return NULL;
	175	}
	176
	177	val = gen_pool_alloc(atomic_pool, size);
	178	if (val) {
	179	phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
	180
	181	*ret_page = pfn_to_page(__phys_to_pfn(phys));
	182	ptr = (void *)val;
	183	memset(ptr, 0, size);
	184	}
	185
	186	return ptr;
	187	}
	188
	189	bool dma_free_from_pool(void *start, size_t size)
	190	{
	191	if (!dma_in_atomic_pool(start, size))
	192	return false;
	193	gen_pool_free(atomic_pool, (unsigned long)start, size);
	194	return true;
	195	}
	196
	197	void arch_dma_alloc(struct device dev, size_t size, dma_addr_t *dma_handle,
	198	gfp_t flags, unsigned long attrs)
	199	{
	200	struct page *page = NULL;
bfd56cd6	201	void *ret;
0c3b3171 CH	202
	203	size = PAGE_ALIGN(size);
	204
d98849af	205	if (!gfpflags_allow_blocking(flags)) {
0c3b3171 CH	206	ret = dma_alloc_from_pool(size, &page, flags);
	207	if (!ret)
	208	return NULL;
8270f3a1	209	goto done;
0c3b3171 CH	210	}
0c3b3171 CH	211
bfd56cd6 CH	212	page = __dma_direct_alloc_pages(dev, size, dma_handle, flags, attrs);
bfd56cd6 CH	213	if (!page)
0c3b3171	214	return NULL;
0c3b3171 CH	215
	216	/* remove any dirty cache lines on the kernel alias */
	217	arch_dma_prep_coherent(page, size);
	218
	219	/* create a coherent mapping */
	220	ret = dma_common_contiguous_remap(page, size, VM_USERMAP,
	221	arch_dma_mmap_pgprot(dev, PAGE_KERNEL, attrs),
	222	__builtin_return_address(0));
a1da439c	223	if (!ret) {
bfd56cd6	224	__dma_direct_free_pages(dev, size, page);
a1da439c MS	225	return ret;
	226	}
	227
a1da439c	228	memset(ret, 0, size);
8270f3a1 CH	229	done:
8270f3a1 CH	230	*dma_handle = phys_to_dma(dev, page_to_phys(page));
0c3b3171 CH	231	return ret;
	232	}
	233
	234	void arch_dma_free(struct device dev, size_t size, void vaddr,
	235	dma_addr_t dma_handle, unsigned long attrs)
	236	{
d98849af	237	if (!dma_free_from_pool(vaddr, PAGE_ALIGN(size))) {
bfd56cd6 CH	238	phys_addr_t phys = dma_to_phys(dev, dma_handle);
bfd56cd6 CH	239	struct page *page = pfn_to_page(__phys_to_pfn(phys));
0c3b3171 CH	240
0c3b3171 CH	241	vunmap(vaddr);
bfd56cd6	242	__dma_direct_free_pages(dev, size, page);
0c3b3171 CH	243	}
	244	}
	245
	246	long arch_dma_coherent_to_pfn(struct device dev, void cpu_addr,
	247	dma_addr_t dma_addr)
	248	{
	249	return __phys_to_pfn(dma_to_phys(dev, dma_addr));
	250	}
	251	#endif /* CONFIG_DMA_DIRECT_REMAP */