[linux-2.6-block.git] / drivers / dax / device.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */
#include <linux/memremap.h>
#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/device.h>
#include <linux/pfn_t.h>
#include <linux/cdev.h>
#include <linux/slab.h>
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include "dax-private.h"
#include "bus.h"

static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
		const char *func)
{
	struct dax_region *dax_region = dev_dax->region;
	struct device *dev = &dev_dax->dev;
	unsigned long mask;

	if (!dax_alive(dev_dax->dax_dev))
		return -ENXIO;

	/* prevent private mappings from being established */
	if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, attempted private mapping\n",
				current->comm, func);
		return -EINVAL;
	}

	mask = dax_region->align - 1;
	if (vma->vm_start & mask || vma->vm_end & mask) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
				current->comm, func, vma->vm_start, vma->vm_end,
				mask);
		return -EINVAL;
	}

	if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV
			&& (vma->vm_flags & VM_DONTCOPY) == 0) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, dax range requires MADV_DONTFORK\n",
				current->comm, func);
		return -EINVAL;
	}

	if (!vma_is_dax(vma)) {
		dev_info_ratelimited(dev,
				"%s: %s: fail, vma is not DAX capable\n",
				current->comm, func);
		return -EINVAL;
	}

	return 0;
}

/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
		unsigned long size)
{
	struct resource *res = &dev_dax->region->res;
	phys_addr_t phys;

	phys = pgoff * PAGE_SIZE + res->start;
	if (phys >= res->start && phys <= res->end) {
		if (phys + size - 1 <= res->end)
			return phys;
	}

	return -1;
}

static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	struct device *dev = &dev_dax->dev;
	struct dax_region *dax_region;
	phys_addr_t phys;
	unsigned int fault_size = PAGE_SIZE;

	if (check_vma(dev_dax, vmf->vma, __func__))
		return VM_FAULT_SIGBUS;

	dax_region = dev_dax->region;
	if (dax_region->align > PAGE_SIZE) {
		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
			dax_region->align, fault_size);
		return VM_FAULT_SIGBUS;
	}

	if (fault_size != dax_region->align)
		return VM_FAULT_SIGBUS;

	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
	if (phys == -1) {
		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);
		return VM_FAULT_SIGBUS;
	}

	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);

	return vmf_insert_mixed(vmf->vma, vmf->address, *pfn);
}

static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	unsigned long pmd_addr = vmf->address & PMD_MASK;
	struct device *dev = &dev_dax->dev;
	struct dax_region *dax_region;
	phys_addr_t phys;
	pgoff_t pgoff;
	unsigned int fault_size = PMD_SIZE;

	if (check_vma(dev_dax, vmf->vma, __func__))
		return VM_FAULT_SIGBUS;

	dax_region = dev_dax->region;
	if (dax_region->align > PMD_SIZE) {
		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
			dax_region->align, fault_size);
		return VM_FAULT_SIGBUS;
	}

	/* dax pmd mappings require pfn_t_devmap() */
	if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
		dev_dbg(dev, "region lacks devmap flags\n");
		return VM_FAULT_SIGBUS;
	}

	if (fault_size < dax_region->align)
		return VM_FAULT_SIGBUS;
	else if (fault_size > dax_region->align)
		return VM_FAULT_FALLBACK;

	/* if we are outside of the VMA */
	if (pmd_addr < vmf->vma->vm_start ||
			(pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
		return VM_FAULT_SIGBUS;

	pgoff = linear_page_index(vmf->vma, pmd_addr);
	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
	if (phys == -1) {
		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
		return VM_FAULT_SIGBUS;
	}

	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);

	return vmf_insert_pfn_pmd(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	unsigned long pud_addr = vmf->address & PUD_MASK;
	struct device *dev = &dev_dax->dev;
	struct dax_region *dax_region;
	phys_addr_t phys;
	pgoff_t pgoff;
	unsigned int fault_size = PUD_SIZE;


	if (check_vma(dev_dax, vmf->vma, __func__))
		return VM_FAULT_SIGBUS;

	dax_region = dev_dax->region;
	if (dax_region->align > PUD_SIZE) {
		dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
			dax_region->align, fault_size);
		return VM_FAULT_SIGBUS;
	}

	/* dax pud mappings require pfn_t_devmap() */
	if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
		dev_dbg(dev, "region lacks devmap flags\n");
		return VM_FAULT_SIGBUS;
	}

	if (fault_size < dax_region->align)
		return VM_FAULT_SIGBUS;
	else if (fault_size > dax_region->align)
		return VM_FAULT_FALLBACK;

	/* if we are outside of the VMA */
	if (pud_addr < vmf->vma->vm_start ||
			(pud_addr + PUD_SIZE) > vmf->vma->vm_end)
		return VM_FAULT_SIGBUS;

	pgoff = linear_page_index(vmf->vma, pud_addr);
	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
	if (phys == -1) {
		dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
		return VM_FAULT_SIGBUS;
	}

	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);

	return vmf_insert_pfn_pud(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
}
#else
static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
				struct vm_fault *vmf, pfn_t *pfn)
{
	return VM_FAULT_FALLBACK;
}
#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */

static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
		enum page_entry_size pe_size)
{
	struct file *filp = vmf->vma->vm_file;
	unsigned long fault_size;
	vm_fault_t rc = VM_FAULT_SIGBUS;
	int id;
	pfn_t pfn;
	struct dev_dax *dev_dax = filp->private_data;

	dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
			(vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
			vmf->vma->vm_start, vmf->vma->vm_end, pe_size);

	id = dax_read_lock();
	switch (pe_size) {
	case PE_SIZE_PTE:
		fault_size = PAGE_SIZE;
		rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);
		break;
	case PE_SIZE_PMD:
		fault_size = PMD_SIZE;
		rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);
		break;
	case PE_SIZE_PUD:
		fault_size = PUD_SIZE;
		rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);
		break;
	default:
		rc = VM_FAULT_SIGBUS;
	}

	if (rc == VM_FAULT_NOPAGE) {
		unsigned long i;
		pgoff_t pgoff;

		/*
		 * In the device-dax case the only possibility for a
		 * VM_FAULT_NOPAGE result is when device-dax capacity is
		 * mapped. No need to consider the zero page, or racing
		 * conflicting mappings.
		 */
		pgoff = linear_page_index(vmf->vma, vmf->address
				& ~(fault_size - 1));
		for (i = 0; i < fault_size / PAGE_SIZE; i++) {
			struct page *page;

			page = pfn_to_page(pfn_t_to_pfn(pfn) + i);
			if (page->mapping)
				continue;
			page->mapping = filp->f_mapping;
			page->index = pgoff + i;
		}
	}
	dax_read_unlock(id);

	return rc;
}

static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
{
	return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
}

static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
{
	struct file *filp = vma->vm_file;
	struct dev_dax *dev_dax = filp->private_data;
	struct dax_region *dax_region = dev_dax->region;

	if (!IS_ALIGNED(addr, dax_region->align))
		return -EINVAL;
	return 0;
}

static unsigned long dev_dax_pagesize(struct vm_area_struct *vma)
{
	struct file *filp = vma->vm_file;
	struct dev_dax *dev_dax = filp->private_data;
	struct dax_region *dax_region = dev_dax->region;

	return dax_region->align;
}

static const struct vm_operations_struct dax_vm_ops = {
	.fault = dev_dax_fault,
	.huge_fault = dev_dax_huge_fault,
	.split = dev_dax_split,
	.pagesize = dev_dax_pagesize,
};

static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct dev_dax *dev_dax = filp->private_data;
	int rc, id;

	dev_dbg(&dev_dax->dev, "trace\n");

	/*
	 * We lock to check dax_dev liveness and will re-check at
	 * fault time.
	 */
	id = dax_read_lock();
	rc = check_vma(dev_dax, vma, __func__);
	dax_read_unlock(id);
	if (rc)
		return rc;

	vma->vm_ops = &dax_vm_ops;
	vma->vm_flags |= VM_HUGEPAGE;
	return 0;
}

/* return an unmapped area aligned to the dax region specified alignment */
static unsigned long dax_get_unmapped_area(struct file *filp,
		unsigned long addr, unsigned long len, unsigned long pgoff,
		unsigned long flags)
{
	unsigned long off, off_end, off_align, len_align, addr_align, align;
	struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
	struct dax_region *dax_region;

	if (!dev_dax || addr)
		goto out;

	dax_region = dev_dax->region;
	align = dax_region->align;
	off = pgoff << PAGE_SHIFT;
	off_end = off + len;
	off_align = round_up(off, align);

	if ((off_end <= off_align) || ((off_end - off_align) < align))
		goto out;

	len_align = len + align;
	if ((off + len_align) < off)
		goto out;

	addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
			pgoff, flags);
	if (!IS_ERR_VALUE(addr_align)) {
		addr_align += (off - addr_align) & (align - 1);
		return addr_align;
	}
 out:
	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
}

static const struct address_space_operations dev_dax_aops = {
	.set_page_dirty		= noop_set_page_dirty,
	.invalidatepage		= noop_invalidatepage,
};

static int dax_open(struct inode *inode, struct file *filp)
{
	struct dax_device *dax_dev = inode_dax(inode);
	struct inode *__dax_inode = dax_inode(dax_dev);
	struct dev_dax *dev_dax = dax_get_private(dax_dev);

	dev_dbg(&dev_dax->dev, "trace\n");
	inode->i_mapping = __dax_inode->i_mapping;
	inode->i_mapping->host = __dax_inode;
	inode->i_mapping->a_ops = &dev_dax_aops;
	filp->f_mapping = inode->i_mapping;
	filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
	filp->private_data = dev_dax;
	inode->i_flags = S_DAX;

	return 0;
}

static int dax_release(struct inode *inode, struct file *filp)
{
	struct dev_dax *dev_dax = filp->private_data;

	dev_dbg(&dev_dax->dev, "trace\n");
	return 0;
}

static const struct file_operations dax_fops = {
	.llseek = noop_llseek,
	.owner = THIS_MODULE,
	.open = dax_open,
	.release = dax_release,
	.get_unmapped_area = dax_get_unmapped_area,
	.mmap = dax_mmap,
	.mmap_supported_flags = MAP_SYNC,
};

static void dev_dax_cdev_del(void *cdev)
{
	cdev_del(cdev);
}

static void dev_dax_kill(void *dev_dax)
{
	kill_dev_dax(dev_dax);
}

int dev_dax_probe(struct device *dev)
{
	struct dev_dax *dev_dax = to_dev_dax(dev);
	struct dax_device *dax_dev = dev_dax->dax_dev;
	struct resource *res = &dev_dax->region->res;
	struct inode *inode;
	struct cdev *cdev;
	void *addr;
	int rc;

	/* 1:1 map region resource range to device-dax instance range */
	if (!devm_request_mem_region(dev, res->start, resource_size(res),
				dev_name(dev))) {
		dev_warn(dev, "could not reserve region %pR\n", res);
		return -EBUSY;
	}

	dev_dax->pgmap.type = MEMORY_DEVICE_DEVDAX;
	addr = devm_memremap_pages(dev, &dev_dax->pgmap);
	if (IS_ERR(addr))
		return PTR_ERR(addr);

	inode = dax_inode(dax_dev);
	cdev = inode->i_cdev;
	cdev_init(cdev, &dax_fops);
	if (dev->class) {
		/* for the CONFIG_DEV_DAX_PMEM_COMPAT case */
		cdev->owner = dev->parent->driver->owner;
	} else
		cdev->owner = dev->driver->owner;
	cdev_set_parent(cdev, &dev->kobj);
	rc = cdev_add(cdev, dev->devt, 1);
	if (rc)
		return rc;

	rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);
	if (rc)
		return rc;

	run_dax(dax_dev);
	return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
}
EXPORT_SYMBOL_GPL(dev_dax_probe);

static int dev_dax_remove(struct device *dev)
{
	/* all probe actions are unwound by devm */
	return 0;
}

static struct dax_device_driver device_dax_driver = {
	.drv = {
		.probe = dev_dax_probe,
		.remove = dev_dax_remove,
	},
	.match_always = 1,
};

static int __init dax_init(void)
{
	return dax_driver_register(&device_dax_driver);
}

static void __exit dax_exit(void)
{
	dax_driver_unregister(&device_dax_driver);
}

MODULE_AUTHOR("Intel Corporation");
MODULE_LICENSE("GPL v2");
module_init(dax_init);
module_exit(dax_exit);
MODULE_ALIAS_DAX_DEVICE(0);
Commit	Line	Data
51cf784c DW	1	// SPDX-License-Identifier: GPL-2.0
51cf784c DW	2	/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */
89ec9f2c	3	#include <linux/memremap.h>
ab68f262 DW	4	#include <linux/pagemap.h>
	5	#include <linux/module.h>
	6	#include <linux/device.h>
	7	#include <linux/pfn_t.h>
ba09c01d	8	#include <linux/cdev.h>
ab68f262 DW	9	#include <linux/slab.h>
	10	#include <linux/dax.h>
	11	#include <linux/fs.h>
	12	#include <linux/mm.h>
ef842302	13	#include <linux/mman.h>
efebc711	14	#include "dax-private.h"
51cf784c	15	#include "bus.h"
ab68f262	16
5f0694b3	17	static int check_vma(struct dev_dax dev_dax, struct vm_area_struct vma,
dee41079 DW	18	const char *func)
dee41079 DW	19	{
5f0694b3 DW	20	struct dax_region *dax_region = dev_dax->region;
5f0694b3 DW	21	struct device *dev = &dev_dax->dev;
dee41079 DW	22	unsigned long mask;
dee41079 DW	23
7b6be844	24	if (!dax_alive(dev_dax->dax_dev))
dee41079 DW	25	return -ENXIO;
dee41079 DW	26
4cb19355	27	/* prevent private mappings from being established */
325896ff	28	if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
5a14e91d JM	29	dev_info_ratelimited(dev,
5a14e91d JM	30	"%s: %s: fail, attempted private mapping\n",
dee41079 DW	31	current->comm, func);
	32	return -EINVAL;
	33	}
	34
	35	mask = dax_region->align - 1;
	36	if (vma->vm_start & mask \|\| vma->vm_end & mask) {
5a14e91d JM	37	dev_info_ratelimited(dev,
5a14e91d JM	38	"%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
dee41079 DW	39	current->comm, func, vma->vm_start, vma->vm_end,
	40	mask);
	41	return -EINVAL;
	42	}
	43
	44	if ((dax_region->pfn_flags & (PFN_DEV\|PFN_MAP)) == PFN_DEV
	45	&& (vma->vm_flags & VM_DONTCOPY) == 0) {
5a14e91d JM	46	dev_info_ratelimited(dev,
5a14e91d JM	47	"%s: %s: fail, dax range requires MADV_DONTFORK\n",
dee41079 DW	48	current->comm, func);
	49	return -EINVAL;
	50	}
	51
	52	if (!vma_is_dax(vma)) {
5a14e91d JM	53	dev_info_ratelimited(dev,
5a14e91d JM	54	"%s: %s: fail, vma is not DAX capable\n",
dee41079 DW	55	current->comm, func);
	56	return -EINVAL;
	57	}
	58
	59	return 0;
	60	}
	61
efebc711	62	/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
73616367	63	__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
dee41079 DW	64	unsigned long size)
dee41079 DW	65	{
753a0850 DW	66	struct resource *res = &dev_dax->region->res;
753a0850 DW	67	phys_addr_t phys;
dee41079	68
753a0850 DW	69	phys = pgoff * PAGE_SIZE + res->start;
753a0850 DW	70	if (phys >= res->start && phys <= res->end) {
dee41079 DW	71	if (phys + size - 1 <= res->end)
	72	return phys;
	73	}
	74
	75	return -1;
	76	}
	77
226ab561	78	static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
2232c638	79	struct vm_fault vmf, pfn_t pfn)
dee41079	80	{
5f0694b3	81	struct device *dev = &dev_dax->dev;
dee41079	82	struct dax_region *dax_region;
dee41079	83	phys_addr_t phys;
0134ed4f	84	unsigned int fault_size = PAGE_SIZE;
dee41079	85
5f0694b3	86	if (check_vma(dev_dax, vmf->vma, __func__))
dee41079 DW	87	return VM_FAULT_SIGBUS;
dee41079 DW	88
5f0694b3	89	dax_region = dev_dax->region;
dee41079	90	if (dax_region->align > PAGE_SIZE) {
6daaca52 DW	91	dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
6daaca52 DW	92	dax_region->align, fault_size);
dee41079 DW	93	return VM_FAULT_SIGBUS;
	94	}
	95
0134ed4f DJ	96	if (fault_size != dax_region->align)
	97	return VM_FAULT_SIGBUS;
	98
73616367	99	phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
dee41079	100	if (phys == -1) {
6daaca52	101	dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);
dee41079 DW	102	return VM_FAULT_SIGBUS;
	103	}
	104
2232c638	105	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
dee41079	106
2232c638	107	return vmf_insert_mixed(vmf->vma, vmf->address, *pfn);
dee41079 DW	108	}
dee41079 DW	109
226ab561	110	static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
2232c638	111	struct vm_fault vmf, pfn_t pfn)
dee41079	112	{
d8a849e1	113	unsigned long pmd_addr = vmf->address & PMD_MASK;
5f0694b3	114	struct device *dev = &dev_dax->dev;
dee41079 DW	115	struct dax_region *dax_region;
	116	phys_addr_t phys;
	117	pgoff_t pgoff;
0134ed4f	118	unsigned int fault_size = PMD_SIZE;
dee41079	119
5f0694b3	120	if (check_vma(dev_dax, vmf->vma, __func__))
dee41079 DW	121	return VM_FAULT_SIGBUS;
dee41079 DW	122
5f0694b3	123	dax_region = dev_dax->region;
dee41079	124	if (dax_region->align > PMD_SIZE) {
6daaca52 DW	125	dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
6daaca52 DW	126	dax_region->align, fault_size);
dee41079 DW	127	return VM_FAULT_SIGBUS;
	128	}
	129
	130	/* dax pmd mappings require pfn_t_devmap() */
	131	if ((dax_region->pfn_flags & (PFN_DEV\|PFN_MAP)) != (PFN_DEV\|PFN_MAP)) {
6daaca52	132	dev_dbg(dev, "region lacks devmap flags\n");
dee41079 DW	133	return VM_FAULT_SIGBUS;
	134	}
	135
0134ed4f DJ	136	if (fault_size < dax_region->align)
	137	return VM_FAULT_SIGBUS;
	138	else if (fault_size > dax_region->align)
	139	return VM_FAULT_FALLBACK;
	140
	141	/* if we are outside of the VMA */
	142	if (pmd_addr < vmf->vma->vm_start \|\|
	143	(pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
	144	return VM_FAULT_SIGBUS;
	145
f4200391	146	pgoff = linear_page_index(vmf->vma, pmd_addr);
73616367	147	phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
dee41079	148	if (phys == -1) {
6daaca52	149	dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
dee41079 DW	150	return VM_FAULT_SIGBUS;
	151	}
	152
2232c638	153	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
dee41079	154
fce86ff5	155	return vmf_insert_pfn_pmd(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
dee41079 DW	156	}
dee41079 DW	157
9557feee	158	#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
226ab561	159	static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
2232c638	160	struct vm_fault vmf, pfn_t pfn)
9557feee DJ	161	{
9557feee DJ	162	unsigned long pud_addr = vmf->address & PUD_MASK;
5f0694b3	163	struct device *dev = &dev_dax->dev;
9557feee DJ	164	struct dax_region *dax_region;
	165	phys_addr_t phys;
	166	pgoff_t pgoff;
70b085b0 DJ	167	unsigned int fault_size = PUD_SIZE;
70b085b0 DJ	168
9557feee	169
5f0694b3	170	if (check_vma(dev_dax, vmf->vma, __func__))
9557feee DJ	171	return VM_FAULT_SIGBUS;
9557feee DJ	172
5f0694b3	173	dax_region = dev_dax->region;
9557feee	174	if (dax_region->align > PUD_SIZE) {
6daaca52 DW	175	dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
6daaca52 DW	176	dax_region->align, fault_size);
9557feee DJ	177	return VM_FAULT_SIGBUS;
	178	}
	179
	180	/* dax pud mappings require pfn_t_devmap() */
	181	if ((dax_region->pfn_flags & (PFN_DEV\|PFN_MAP)) != (PFN_DEV\|PFN_MAP)) {
6daaca52	182	dev_dbg(dev, "region lacks devmap flags\n");
9557feee DJ	183	return VM_FAULT_SIGBUS;
	184	}
	185
70b085b0 DJ	186	if (fault_size < dax_region->align)
	187	return VM_FAULT_SIGBUS;
	188	else if (fault_size > dax_region->align)
	189	return VM_FAULT_FALLBACK;
	190
	191	/* if we are outside of the VMA */
	192	if (pud_addr < vmf->vma->vm_start \|\|
	193	(pud_addr + PUD_SIZE) > vmf->vma->vm_end)
	194	return VM_FAULT_SIGBUS;
	195
9557feee	196	pgoff = linear_page_index(vmf->vma, pud_addr);
73616367	197	phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
9557feee	198	if (phys == -1) {
6daaca52	199	dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
9557feee DJ	200	return VM_FAULT_SIGBUS;
	201	}
	202
2232c638	203	*pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
9557feee	204
fce86ff5	205	return vmf_insert_pfn_pud(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
9557feee DJ	206	}
9557feee DJ	207	#else
226ab561	208	static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
2232c638	209	struct vm_fault vmf, pfn_t pfn)
9557feee DJ	210	{
	211	return VM_FAULT_FALLBACK;
	212	}
	213	#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
	214
226ab561	215	static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
c791ace1	216	enum page_entry_size pe_size)
dee41079	217	{
f4200391	218	struct file *filp = vmf->vma->vm_file;
2232c638	219	unsigned long fault_size;
36bdac1e SJ	220	vm_fault_t rc = VM_FAULT_SIGBUS;
36bdac1e SJ	221	int id;
2232c638	222	pfn_t pfn;
5f0694b3	223	struct dev_dax *dev_dax = filp->private_data;
dee41079	224
6daaca52 DW	225	dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
6daaca52 DW	226	(vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
76202620	227	vmf->vma->vm_start, vmf->vma->vm_end, pe_size);
dee41079	228
7b6be844	229	id = dax_read_lock();
c791ace1 DJ	230	switch (pe_size) {
c791ace1 DJ	231	case PE_SIZE_PTE:
2232c638 DW	232	fault_size = PAGE_SIZE;
2232c638 DW	233	rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);
a2d58167	234	break;
c791ace1	235	case PE_SIZE_PMD:
2232c638 DW	236	fault_size = PMD_SIZE;
2232c638 DW	237	rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);
9557feee	238	break;
c791ace1	239	case PE_SIZE_PUD:
2232c638 DW	240	fault_size = PUD_SIZE;
2232c638 DW	241	rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);
a2d58167 DJ	242	break;
a2d58167 DJ	243	default:
54eafcc9	244	rc = VM_FAULT_SIGBUS;
a2d58167	245	}
2232c638 DW	246
	247	if (rc == VM_FAULT_NOPAGE) {
	248	unsigned long i;
35de2995	249	pgoff_t pgoff;
2232c638 DW	250
	251	/*
	252	* In the device-dax case the only possibility for a
	253	* VM_FAULT_NOPAGE result is when device-dax capacity is
	254	* mapped. No need to consider the zero page, or racing
	255	* conflicting mappings.
	256	*/
35de2995 DW	257	pgoff = linear_page_index(vmf->vma, vmf->address
35de2995 DW	258	& ~(fault_size - 1));
2232c638 DW	259	for (i = 0; i < fault_size / PAGE_SIZE; i++) {
	260	struct page *page;
	261
	262	page = pfn_to_page(pfn_t_to_pfn(pfn) + i);
	263	if (page->mapping)
	264	continue;
	265	page->mapping = filp->f_mapping;
35de2995	266	page->index = pgoff + i;
2232c638 DW	267	}
2232c638 DW	268	}
7b6be844	269	dax_read_unlock(id);
dee41079 DW	270
	271	return rc;
	272	}
	273
226ab561	274	static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
c791ace1	275	{
5f0694b3	276	return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
c791ace1 DJ	277	}
c791ace1 DJ	278
9702cffd DW	279	static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
	280	{
	281	struct file *filp = vma->vm_file;
	282	struct dev_dax *dev_dax = filp->private_data;
	283	struct dax_region *dax_region = dev_dax->region;
	284
	285	if (!IS_ALIGNED(addr, dax_region->align))
	286	return -EINVAL;
	287	return 0;
	288	}
	289
c1d53b92 DW	290	static unsigned long dev_dax_pagesize(struct vm_area_struct *vma)
	291	{
	292	struct file *filp = vma->vm_file;
	293	struct dev_dax *dev_dax = filp->private_data;
	294	struct dax_region *dax_region = dev_dax->region;
	295
	296	return dax_region->align;
	297	}
	298
5f0694b3 DW	299	static const struct vm_operations_struct dax_vm_ops = {
	300	.fault = dev_dax_fault,
	301	.huge_fault = dev_dax_huge_fault,
9702cffd	302	.split = dev_dax_split,
c1d53b92	303	.pagesize = dev_dax_pagesize,
dee41079 DW	304	};
dee41079 DW	305
af69f51e	306	static int dax_mmap(struct file filp, struct vm_area_struct vma)
dee41079	307	{
5f0694b3	308	struct dev_dax *dev_dax = filp->private_data;
7b6be844	309	int rc, id;
dee41079	310
6daaca52	311	dev_dbg(&dev_dax->dev, "trace\n");
dee41079	312
7b6be844 DW	313	/*
	314	* We lock to check dax_dev liveness and will re-check at
	315	* fault time.
	316	*/
	317	id = dax_read_lock();
5f0694b3	318	rc = check_vma(dev_dax, vma, __func__);
7b6be844	319	dax_read_unlock(id);
dee41079 DW	320	if (rc)
	321	return rc;
	322
5f0694b3	323	vma->vm_ops = &dax_vm_ops;
e1fb4a08	324	vma->vm_flags \|= VM_HUGEPAGE;
dee41079	325	return 0;
043a9255 DW	326	}
	327
	328	/* return an unmapped area aligned to the dax region specified alignment */
af69f51e	329	static unsigned long dax_get_unmapped_area(struct file *filp,
043a9255 DW	330	unsigned long addr, unsigned long len, unsigned long pgoff,
	331	unsigned long flags)
	332	{
	333	unsigned long off, off_end, off_align, len_align, addr_align, align;
5f0694b3	334	struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
043a9255 DW	335	struct dax_region *dax_region;
043a9255 DW	336
5f0694b3	337	if (!dev_dax \|\| addr)
043a9255 DW	338	goto out;
043a9255 DW	339
5f0694b3	340	dax_region = dev_dax->region;
043a9255 DW	341	align = dax_region->align;
	342	off = pgoff << PAGE_SHIFT;
	343	off_end = off + len;
	344	off_align = round_up(off, align);
	345
	346	if ((off_end <= off_align) \|\| ((off_end - off_align) < align))
	347	goto out;
	348
	349	len_align = len + align;
	350	if ((off + len_align) < off)
	351	goto out;
dee41079	352
043a9255 DW	353	addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
	354	pgoff, flags);
	355	if (!IS_ERR_VALUE(addr_align)) {
	356	addr_align += (off - addr_align) & (align - 1);
	357	return addr_align;
	358	}
	359	out:
	360	return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
	361	}
	362
41c9b1be DJ	363	static const struct address_space_operations dev_dax_aops = {
	364	.set_page_dirty = noop_set_page_dirty,
	365	.invalidatepage = noop_invalidatepage,
	366	};
	367
af69f51e	368	static int dax_open(struct inode inode, struct file filp)
043a9255	369	{
7b6be844 DW	370	struct dax_device *dax_dev = inode_dax(inode);
	371	struct inode *__dax_inode = dax_inode(dax_dev);
	372	struct dev_dax *dev_dax = dax_get_private(dax_dev);
043a9255	373
6daaca52	374	dev_dbg(&dev_dax->dev, "trace\n");
7b6be844 DW	375	inode->i_mapping = __dax_inode->i_mapping;
7b6be844 DW	376	inode->i_mapping->host = __dax_inode;
41c9b1be	377	inode->i_mapping->a_ops = &dev_dax_aops;
3bc52c45	378	filp->f_mapping = inode->i_mapping;
5660e13d	379	filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
5f0694b3	380	filp->private_data = dev_dax;
ebd84d72	381	inode->i_flags = S_DAX;
043a9255	382
043a9255 DW	383	return 0;
043a9255 DW	384	}
dee41079	385
af69f51e	386	static int dax_release(struct inode inode, struct file filp)
043a9255	387	{
5f0694b3	388	struct dev_dax *dev_dax = filp->private_data;
043a9255	389
6daaca52	390	dev_dbg(&dev_dax->dev, "trace\n");
043a9255	391	return 0;
dee41079 DW	392	}
dee41079 DW	393
ab68f262 DW	394	static const struct file_operations dax_fops = {
	395	.llseek = noop_llseek,
	396	.owner = THIS_MODULE,
af69f51e DW	397	.open = dax_open,
	398	.release = dax_release,
	399	.get_unmapped_area = dax_get_unmapped_area,
	400	.mmap = dax_mmap,
ef842302	401	.mmap_supported_flags = MAP_SYNC,
ab68f262 DW	402	};
ab68f262 DW	403
9567da0b	404	static void dev_dax_cdev_del(void *cdev)
043a9255	405	{
9567da0b DW	406	cdev_del(cdev);
9567da0b DW	407	}
043a9255	408
9567da0b DW	409	static void dev_dax_kill(void *dev_dax)
	410	{
	411	kill_dev_dax(dev_dax);
ebd84d72 DW	412	}
ebd84d72 DW	413
730926c3	414	int dev_dax_probe(struct device *dev)
043a9255	415	{
9567da0b DW	416	struct dev_dax *dev_dax = to_dev_dax(dev);
9567da0b DW	417	struct dax_device *dax_dev = dev_dax->dax_dev;
89ec9f2c	418	struct resource *res = &dev_dax->region->res;
7b6be844	419	struct inode *inode;
ba09c01d	420	struct cdev *cdev;
89ec9f2c	421	void *addr;
753a0850	422	int rc;
43fe51e1	423
89ec9f2c DW	424	/* 1:1 map region resource range to device-dax instance range */
	425	if (!devm_request_mem_region(dev, res->start, resource_size(res),
	426	dev_name(dev))) {
	427	dev_warn(dev, "could not reserve region %pR\n", res);
	428	return -EBUSY;
	429	}
	430
3ed2dcdf	431	dev_dax->pgmap.type = MEMORY_DEVICE_DEVDAX;
89ec9f2c	432	addr = devm_memremap_pages(dev, &dev_dax->pgmap);
50f44ee7	433	if (IS_ERR(addr))
89ec9f2c	434	return PTR_ERR(addr);
89ec9f2c	435
7b6be844 DW	436	inode = dax_inode(dax_dev);
7b6be844 DW	437	cdev = inode->i_cdev;
ba09c01d	438	cdev_init(cdev, &dax_fops);
730926c3 DW	439	if (dev->class) {
	440	/* for the CONFIG_DEV_DAX_PMEM_COMPAT case */
	441	cdev->owner = dev->parent->driver->owner;
	442	} else
	443	cdev->owner = dev->driver->owner;
9567da0b DW	444	cdev_set_parent(cdev, &dev->kobj);
9567da0b DW	445	rc = cdev_add(cdev, dev->devt, 1);
d76911ee	446	if (rc)
9567da0b	447	return rc;
d76911ee	448
9567da0b DW	449	rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);
	450	if (rc)
	451	return rc;
043a9255	452
9567da0b DW	453	run_dax(dax_dev);
	454	return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
	455	}
730926c3	456	EXPORT_SYMBOL_GPL(dev_dax_probe);
043a9255	457
9567da0b DW	458	static int dev_dax_remove(struct device *dev)
	459	{
	460	/* all probe actions are unwound by devm */
	461	return 0;
043a9255	462	}
9567da0b	463
d200781e DW	464	static struct dax_device_driver device_dax_driver = {
	465	.drv = {
	466	.probe = dev_dax_probe,
	467	.remove = dev_dax_remove,
	468	},
	469	.match_always = 1,
9567da0b	470	};
043a9255	471
ab68f262 DW	472	static int __init dax_init(void)
ab68f262 DW	473	{
9567da0b	474	return dax_driver_register(&device_dax_driver);
ab68f262 DW	475	}
	476
	477	static void __exit dax_exit(void)
	478	{
d200781e	479	dax_driver_unregister(&device_dax_driver);
ab68f262 DW	480	}
	481
	482	MODULE_AUTHOR("Intel Corporation");
	483	MODULE_LICENSE("GPL v2");
9567da0b	484	module_init(dax_init);
ab68f262	485	module_exit(dax_exit);
9567da0b	486	MODULE_ALIAS_DAX_DEVICE(0);