[linux-block.git] / include / linux / dax.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_DAX_H
#define _LINUX_DAX_H

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/radix-tree.h>

typedef unsigned long dax_entry_t;

struct dax_device;
struct gendisk;
struct iomap_ops;
struct iomap_iter;
struct iomap;

enum dax_access_mode {
	DAX_ACCESS,
	DAX_RECOVERY_WRITE,
};

struct dax_operations {
	/*
	 * direct_access: translate a device-relative
	 * logical-page-offset into an absolute physical pfn. Return the
	 * number of pages available for DAX at that pfn.
	 */
	long (*direct_access)(struct dax_device *, pgoff_t, long,
			enum dax_access_mode, void **, pfn_t *);
	/*
	 * Validate whether this device is usable as an fsdax backing
	 * device.
	 */
	bool (*dax_supported)(struct dax_device *, struct block_device *, int,
			sector_t, sector_t);
	/* zero_page_range: required operation. Zero page range   */
	int (*zero_page_range)(struct dax_device *, pgoff_t, size_t);
	/*
	 * recovery_write: recover a poisoned range by DAX device driver
	 * capable of clearing poison.
	 */
	size_t (*recovery_write)(struct dax_device *dax_dev, pgoff_t pgoff,
			void *addr, size_t bytes, struct iov_iter *iter);
};

struct dax_holder_operations {
	/*
	 * notify_failure - notify memory failure into inner holder device
	 * @dax_dev: the dax device which contains the holder
	 * @offset: offset on this dax device where memory failure occurs
	 * @len: length of this memory failure event
	 * @flags: action flags for memory failure handler
	 */
	int (*notify_failure)(struct dax_device *dax_dev, u64 offset,
			u64 len, int mf_flags);
};

#if IS_ENABLED(CONFIG_DAX)
struct dax_device *alloc_dax(void *private, const struct dax_operations *ops);
void *dax_holder(struct dax_device *dax_dev);
void put_dax(struct dax_device *dax_dev);
void kill_dax(struct dax_device *dax_dev);
void dax_write_cache(struct dax_device *dax_dev, bool wc);
bool dax_write_cache_enabled(struct dax_device *dax_dev);
bool dax_synchronous(struct dax_device *dax_dev);
void set_dax_synchronous(struct dax_device *dax_dev);
size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
		void *addr, size_t bytes, struct iov_iter *i);
/*
 * Check if given mapping is supported by the file / underlying device.
 */
static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
					     struct dax_device *dax_dev)
{
	if (!(vma->vm_flags & VM_SYNC))
		return true;
	if (!IS_DAX(file_inode(vma->vm_file)))
		return false;
	return dax_synchronous(dax_dev);
}
#else
static inline void *dax_holder(struct dax_device *dax_dev)
{
	return NULL;
}
static inline struct dax_device *alloc_dax(void *private,
		const struct dax_operations *ops)
{
	/*
	 * Callers should check IS_ENABLED(CONFIG_DAX) to know if this
	 * NULL is an error or expected.
	 */
	return NULL;
}
static inline void put_dax(struct dax_device *dax_dev)
{
}
static inline void kill_dax(struct dax_device *dax_dev)
{
}
static inline void dax_write_cache(struct dax_device *dax_dev, bool wc)
{
}
static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
{
	return false;
}
static inline bool dax_synchronous(struct dax_device *dax_dev)
{
	return true;
}
static inline void set_dax_synchronous(struct dax_device *dax_dev)
{
}
static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
				struct dax_device *dax_dev)
{
	return !(vma->vm_flags & VM_SYNC);
}
static inline size_t dax_recovery_write(struct dax_device *dax_dev,
		pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
{
	return 0;
}
#endif

void set_dax_nocache(struct dax_device *dax_dev);
void set_dax_nomc(struct dax_device *dax_dev);

struct writeback_control;
#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk);
void dax_remove_host(struct gendisk *disk);
struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
		void *holder, const struct dax_holder_operations *ops);
void fs_put_dax(struct dax_device *dax_dev, void *holder);
#else
static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
{
	return 0;
}
static inline void dax_remove_host(struct gendisk *disk)
{
}
static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev,
		u64 *start_off, void *holder,
		const struct dax_holder_operations *ops)
{
	return NULL;
}
static inline void fs_put_dax(struct dax_device *dax_dev, void *holder)
{
}
#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */

#if IS_ENABLED(CONFIG_FS_DAX)
int dax_writeback_mapping_range(struct address_space *mapping,
		struct dax_device *dax_dev, struct writeback_control *wbc);

struct page *dax_layout_busy_page(struct address_space *mapping);
struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
dax_entry_t dax_lock_page(struct page *page);
void dax_unlock_page(struct page *page, dax_entry_t cookie);
dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
		unsigned long index, struct page **page);
void dax_unlock_mapping_entry(struct address_space *mapping,
		unsigned long index, dax_entry_t cookie);
#else
static inline struct page *dax_layout_busy_page(struct address_space *mapping)
{
	return NULL;
}

static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages)
{
	return NULL;
}

static inline int dax_writeback_mapping_range(struct address_space *mapping,
		struct dax_device *dax_dev, struct writeback_control *wbc)
{
	return -EOPNOTSUPP;
}

static inline dax_entry_t dax_lock_page(struct page *page)
{
	if (IS_DAX(page->mapping->host))
		return ~0UL;
	return 0;
}

static inline void dax_unlock_page(struct page *page, dax_entry_t cookie)
{
}

static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
		unsigned long index, struct page **page)
{
	return 0;
}

static inline void dax_unlock_mapping_entry(struct address_space *mapping,
		unsigned long index, dax_entry_t cookie)
{
}
#endif

int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
		const struct iomap_ops *ops);
int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
		const struct iomap_ops *ops);
int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
		const struct iomap_ops *ops);

#if IS_ENABLED(CONFIG_DAX)
int dax_read_lock(void);
void dax_read_unlock(int id);
#else
static inline int dax_read_lock(void)
{
	return 0;
}

static inline void dax_read_unlock(int id)
{
}
#endif /* CONFIG_DAX */
bool dax_alive(struct dax_device *dax_dev);
void *dax_get_private(struct dax_device *dax_dev);
long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
		enum dax_access_mode mode, void **kaddr, pfn_t *pfn);
size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
		size_t bytes, struct iov_iter *i);
size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
		size_t bytes, struct iov_iter *i);
int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
			size_t nr_pages);
int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len,
		int mf_flags);
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size);

ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
		const struct iomap_ops *ops);
vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
		    pfn_t *pfnp, int *errp, const struct iomap_ops *ops);
vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
		enum page_entry_size pe_size, pfn_t pfn);
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
				      pgoff_t index);
int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
				  struct inode *dest, loff_t destoff,
				  loff_t len, bool *is_same,
				  const struct iomap_ops *ops);
int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
			      struct file *file_out, loff_t pos_out,
			      loff_t *len, unsigned int remap_flags,
			      const struct iomap_ops *ops);
static inline bool dax_mapping(struct address_space *mapping)
{
	return mapping->host && IS_DAX(mapping->host);
}

#ifdef CONFIG_DEV_DAX_HMEM_DEVICES
void hmem_register_resource(int target_nid, struct resource *r);
#else
static inline void hmem_register_resource(int target_nid, struct resource *r)
{
}
#endif

typedef int (*walk_hmem_fn)(struct device *dev, int target_nid,
			    const struct resource *res);
int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
#endif
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
c94c2acf MW	2	#ifndef _LINUX_DAX_H
	3	#define _LINUX_DAX_H
	4
	5	#include <linux/fs.h>
	6	#include <linux/mm.h>
4f622938	7	#include <linux/radix-tree.h>
c94c2acf	8
27359fd6 MW	9	typedef unsigned long dax_entry_t;
27359fd6 MW	10
fb08a190 CH	11	struct dax_device;
fb08a190 CH	12	struct gendisk;
a254e568	13	struct iomap_ops;
c6f40468	14	struct iomap_iter;
4f3b4f16	15	struct iomap;
fb08a190	16
e511c4a3 JC	17	enum dax_access_mode {
	18	DAX_ACCESS,
	19	DAX_RECOVERY_WRITE,
	20	};
	21
6568b08b DW	22	struct dax_operations {
	23	/*
	24	* direct_access: translate a device-relative
	25	* logical-page-offset into an absolute physical pfn. Return the
	26	* number of pages available for DAX at that pfn.
	27	*/
	28	long (direct_access)(struct dax_device , pgoff_t, long,
e511c4a3	29	enum dax_access_mode, void *, pfn_t );
7bf7eac8 DW	30	/*
	31	* Validate whether this device is usable as an fsdax backing
	32	* device.
	33	*/
	34	bool (dax_supported)(struct dax_device , struct block_device *, int,
	35	sector_t, sector_t);
f605a263 VG	36	/* zero_page_range: required operation. Zero page range */
f605a263 VG	37	int (zero_page_range)(struct dax_device , pgoff_t, size_t);
047218ec JC	38	/*
	39	* recovery_write: recover a poisoned range by DAX device driver
	40	* capable of clearing poison.
	41	*/
	42	size_t (recovery_write)(struct dax_device dax_dev, pgoff_t pgoff,
	43	void addr, size_t bytes, struct iov_iter iter);
6568b08b	44	};
a254e568	45
8012b866 SR	46	struct dax_holder_operations {
	47	/*
	48	* notify_failure - notify memory failure into inner holder device
	49	* @dax_dev: the dax device which contains the holder
	50	* @offset: offset on this dax device where memory failure occurs
	51	* @len: length of this memory failure event
	52	* @flags: action flags for memory failure handler
	53	*/
	54	int (notify_failure)(struct dax_device dax_dev, u64 offset,
	55	u64 len, int mf_flags);
	56	};
	57
f5705aa8	58	#if IS_ENABLED(CONFIG_DAX)
30c6828a	59	struct dax_device alloc_dax(void private, const struct dax_operations *ops);
8012b866	60	void dax_holder(struct dax_device dax_dev);
f5705aa8	61	void put_dax(struct dax_device *dax_dev);
976431b0 DW	62	void kill_dax(struct dax_device *dax_dev);
	63	void dax_write_cache(struct dax_device *dax_dev, bool wc);
	64	bool dax_write_cache_enabled(struct dax_device *dax_dev);
fd1d00ec CH	65	bool dax_synchronous(struct dax_device *dax_dev);
fd1d00ec CH	66	void set_dax_synchronous(struct dax_device *dax_dev);
047218ec JC	67	size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
047218ec JC	68	void addr, size_t bytes, struct iov_iter i);
32de1484 PG	69	/*
	70	* Check if given mapping is supported by the file / underlying device.
	71	*/
	72	static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
	73	struct dax_device *dax_dev)
	74	{
	75	if (!(vma->vm_flags & VM_SYNC))
	76	return true;
	77	if (!IS_DAX(file_inode(vma->vm_file)))
	78	return false;
	79	return dax_synchronous(dax_dev);
	80	}
f5705aa8	81	#else
8012b866 SR	82	static inline void dax_holder(struct dax_device dax_dev)
	83	{
	84	return NULL;
	85	}
fb08a190	86	static inline struct dax_device alloc_dax(void private,
30c6828a	87	const struct dax_operations *ops)
976431b0 DW	88	{
	89	/*
	90	* Callers should check IS_ENABLED(CONFIG_DAX) to know if this
	91	* NULL is an error or expected.
	92	*/
	93	return NULL;
	94	}
f5705aa8 DW	95	static inline void put_dax(struct dax_device *dax_dev)
	96	{
	97	}
976431b0 DW	98	static inline void kill_dax(struct dax_device *dax_dev)
	99	{
	100	}
	101	static inline void dax_write_cache(struct dax_device *dax_dev, bool wc)
	102	{
	103	}
	104	static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
	105	{
	106	return false;
	107	}
fefc1d97 PG	108	static inline bool dax_synchronous(struct dax_device *dax_dev)
	109	{
	110	return true;
	111	}
	112	static inline void set_dax_synchronous(struct dax_device *dax_dev)
	113	{
	114	}
32de1484 PG	115	static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
	116	struct dax_device *dax_dev)
	117	{
	118	return !(vma->vm_flags & VM_SYNC);
	119	}
047218ec JC	120	static inline size_t dax_recovery_write(struct dax_device *dax_dev,
	121	pgoff_t pgoff, void addr, size_t bytes, struct iov_iter i)
	122	{
	123	return 0;
	124	}
f5705aa8 DW	125	#endif
f5705aa8 DW	126
7ac5360c CH	127	void set_dax_nocache(struct dax_device *dax_dev);
	128	void set_dax_nomc(struct dax_device *dax_dev);
	129
f44c7763	130	struct writeback_control;
2ede8923	131	#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
fb08a190 CH	132	int dax_add_host(struct dax_device dax_dev, struct gendisk disk);
fb08a190 CH	133	void dax_remove_host(struct gendisk *disk);
8012b866 SR	134	struct dax_device fs_dax_get_by_bdev(struct block_device bdev, u64 *start_off,
	135	void holder, const struct dax_holder_operations ops);
	136	void fs_put_dax(struct dax_device dax_dev, void holder);
ef510424	137	#else
fb08a190 CH	138	static inline int dax_add_host(struct dax_device dax_dev, struct gendisk disk)
	139	{
	140	return 0;
	141	}
	142	static inline void dax_remove_host(struct gendisk *disk)
	143	{
	144	}
cd913c76	145	static inline struct dax_device fs_dax_get_by_bdev(struct block_device bdev,
8012b866 SR	146	u64 start_off, void holder,
8012b866 SR	147	const struct dax_holder_operations *ops)
78f35473 DW	148	{
	149	return NULL;
	150	}
8012b866	151	static inline void fs_put_dax(struct dax_device dax_dev, void holder)
2ede8923 CH	152	{
	153	}
	154	#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
f44c7763	155
2ede8923 CH	156	#if IS_ENABLED(CONFIG_FS_DAX)
	157	int dax_writeback_mapping_range(struct address_space *mapping,
	158	struct dax_device dax_dev, struct writeback_control wbc);
	159
	160	struct page dax_layout_busy_page(struct address_space mapping);
	161	struct page dax_layout_busy_page_range(struct address_space mapping, loff_t start, loff_t end);
	162	dax_entry_t dax_lock_page(struct page *page);
	163	void dax_unlock_page(struct page *page, dax_entry_t cookie);
2f437eff SR	164	dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
	165	unsigned long index, struct page **page);
	166	void dax_unlock_mapping_entry(struct address_space *mapping,
	167	unsigned long index, dax_entry_t cookie);
2ede8923	168	#else
5fac7408 DW	169	static inline struct page dax_layout_busy_page(struct address_space mapping)
	170	{
	171	return NULL;
	172	}
	173
6bbdd563 VG	174	static inline struct page dax_layout_busy_page_range(struct address_space mapping, pgoff_t start, pgoff_t nr_pages)
	175	{
	176	return NULL;
	177	}
	178
f44c7763	179	static inline int dax_writeback_mapping_range(struct address_space *mapping,
3f666c56	180	struct dax_device dax_dev, struct writeback_control wbc)
f44c7763 DW	181	{
	182	return -EOPNOTSUPP;
	183	}
c2a7d2a1	184
27359fd6	185	static inline dax_entry_t dax_lock_page(struct page *page)
c2a7d2a1 DW	186	{
c2a7d2a1 DW	187	if (IS_DAX(page->mapping->host))
27359fd6 MW	188	return ~0UL;
27359fd6 MW	189	return 0;
c2a7d2a1 DW	190	}
c2a7d2a1 DW	191
27359fd6	192	static inline void dax_unlock_page(struct page *page, dax_entry_t cookie)
c2a7d2a1 DW	193	{
c2a7d2a1 DW	194	}
2f437eff SR	195
	196	static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
	197	unsigned long index, struct page **page)
	198	{
	199	return 0;
	200	}
	201
	202	static inline void dax_unlock_mapping_entry(struct address_space *mapping,
	203	unsigned long index, dax_entry_t cookie)
	204	{
	205	}
ef510424 DW	206	#endif
ef510424 DW	207
d984648e SR	208	int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
d984648e SR	209	const struct iomap_ops *ops);
c6f40468 CH	210	int dax_zero_range(struct inode inode, loff_t pos, loff_t len, bool did_zero,
	211	const struct iomap_ops *ops);
	212	int dax_truncate_page(struct inode inode, loff_t pos, bool did_zero,
	213	const struct iomap_ops *ops);
	214
e2ec5128	215	#if IS_ENABLED(CONFIG_DAX)
7b6be844 DW	216	int dax_read_lock(void);
7b6be844 DW	217	void dax_read_unlock(int id);
e2ec5128 JK	218	#else
	219	static inline int dax_read_lock(void)
	220	{
	221	return 0;
	222	}
	223
	224	static inline void dax_read_unlock(int id)
	225	{
	226	}
	227	#endif /* CONFIG_DAX */
c1d6e828	228	bool dax_alive(struct dax_device *dax_dev);
c1d6e828	229	void dax_get_private(struct dax_device dax_dev);
b0686260	230	long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
e511c4a3	231	enum dax_access_mode mode, void *kaddr, pfn_t pfn);
7e026c8c DW	232	size_t dax_copy_from_iter(struct dax_device dax_dev, pgoff_t pgoff, void addr,
7e026c8c DW	233	size_t bytes, struct iov_iter *i);
b3a9a0c3 DW	234	size_t dax_copy_to_iter(struct dax_device dax_dev, pgoff_t pgoff, void addr,
b3a9a0c3 DW	235	size_t bytes, struct iov_iter *i);
f605a263 VG	236	int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
f605a263 VG	237	size_t nr_pages);
8012b866 SR	238	int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len,
8012b866 SR	239	int mf_flags);
c3ca015f	240	void dax_flush(struct dax_device dax_dev, void addr, size_t size);
7b6be844	241
11c59c92	242	ssize_t dax_iomap_rw(struct kiocb iocb, struct iov_iter iter,
8ff6daa1	243	const struct iomap_ops *ops);
f77bc3a8	244	vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
c0b24625	245	pfn_t pfnp, int errp, const struct iomap_ops *ops);
ab77dab4 SJ	246	vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
ab77dab4 SJ	247	enum page_entry_size pe_size, pfn_t pfn);
ac401cc7	248	int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
c6dcf52c JK	249	int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
c6dcf52c JK	250	pgoff_t index);
6f7db389 SR	251	int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
	252	struct inode *dest, loff_t destoff,
	253	loff_t len, bool *is_same,
	254	const struct iomap_ops *ops);
	255	int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
	256	struct file *file_out, loff_t pos_out,
	257	loff_t *len, unsigned int remap_flags,
	258	const struct iomap_ops *ops);
f9fe48be RZ	259	static inline bool dax_mapping(struct address_space *mapping)
	260	{
	261	return mapping->host && IS_DAX(mapping->host);
	262	}
7f6d5b52	263
c01044cc	264	#ifdef CONFIG_DEV_DAX_HMEM_DEVICES
7dab174e	265	void hmem_register_resource(int target_nid, struct resource *r);
c01044cc	266	#else
7dab174e	267	static inline void hmem_register_resource(int target_nid, struct resource *r)
c01044cc DW	268	{
	269	}
	270	#endif
	271
7dab174e DW	272	typedef int (walk_hmem_fn)(struct device dev, int target_nid,
	273	const struct resource *res);
	274	int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
c94c2acf	275	#endif