[linux-block.git] / include / linux / dax.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_DAX_H
#define _LINUX_DAX_H

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/radix-tree.h>

typedef unsigned long dax_entry_t;

struct dax_device;
struct gendisk;
struct iomap_ops;
struct iomap_iter;
struct iomap;

enum dax_access_mode {
	DAX_ACCESS,
	DAX_RECOVERY_WRITE,
};

struct dax_operations {
	/*
	 * direct_access: translate a device-relative
	 * logical-page-offset into an absolute physical pfn. Return the
	 * number of pages available for DAX at that pfn.
	 */
	long (*direct_access)(struct dax_device *, pgoff_t, long,
			enum dax_access_mode, void **, pfn_t *);
	/*
	 * Validate whether this device is usable as an fsdax backing
	 * device.
	 */
	bool (*dax_supported)(struct dax_device *, struct block_device *, int,
			sector_t, sector_t);
	/* zero_page_range: required operation. Zero page range   */
	int (*zero_page_range)(struct dax_device *, pgoff_t, size_t);
	/*
	 * recovery_write: recover a poisoned range by DAX device driver
	 * capable of clearing poison.
	 */
	size_t (*recovery_write)(struct dax_device *dax_dev, pgoff_t pgoff,
			void *addr, size_t bytes, struct iov_iter *iter);
};

struct dax_holder_operations {
	/*
	 * notify_failure - notify memory failure into inner holder device
	 * @dax_dev: the dax device which contains the holder
	 * @offset: offset on this dax device where memory failure occurs
	 * @len: length of this memory failure event
	 * @flags: action flags for memory failure handler
	 */
	int (*notify_failure)(struct dax_device *dax_dev, u64 offset,
			u64 len, int mf_flags);
};

#if IS_ENABLED(CONFIG_DAX)
struct dax_device *alloc_dax(void *private, const struct dax_operations *ops);
void *dax_holder(struct dax_device *dax_dev);
void put_dax(struct dax_device *dax_dev);
void kill_dax(struct dax_device *dax_dev);
void dax_write_cache(struct dax_device *dax_dev, bool wc);
bool dax_write_cache_enabled(struct dax_device *dax_dev);
bool dax_synchronous(struct dax_device *dax_dev);
void set_dax_nocache(struct dax_device *dax_dev);
void set_dax_nomc(struct dax_device *dax_dev);
void set_dax_synchronous(struct dax_device *dax_dev);
size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
		void *addr, size_t bytes, struct iov_iter *i);
/*
 * Check if given mapping is supported by the file / underlying device.
 */
static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
					     struct dax_device *dax_dev)
{
	if (!(vma->vm_flags & VM_SYNC))
		return true;
	if (!IS_DAX(file_inode(vma->vm_file)))
		return false;
	return dax_synchronous(dax_dev);
}
#else
static inline void *dax_holder(struct dax_device *dax_dev)
{
	return NULL;
}
static inline struct dax_device *alloc_dax(void *private,
		const struct dax_operations *ops)
{
	return ERR_PTR(-EOPNOTSUPP);
}
static inline void put_dax(struct dax_device *dax_dev)
{
}
static inline void kill_dax(struct dax_device *dax_dev)
{
}
static inline void dax_write_cache(struct dax_device *dax_dev, bool wc)
{
}
static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
{
	return false;
}
static inline bool dax_synchronous(struct dax_device *dax_dev)
{
	return true;
}
static inline void set_dax_nocache(struct dax_device *dax_dev)
{
}
static inline void set_dax_nomc(struct dax_device *dax_dev)
{
}
static inline void set_dax_synchronous(struct dax_device *dax_dev)
{
}
static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
				struct dax_device *dax_dev)
{
	return !(vma->vm_flags & VM_SYNC);
}
static inline size_t dax_recovery_write(struct dax_device *dax_dev,
		pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
{
	return 0;
}
#endif

struct writeback_control;
#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk);
void dax_remove_host(struct gendisk *disk);
struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
		void *holder, const struct dax_holder_operations *ops);
void fs_put_dax(struct dax_device *dax_dev, void *holder);
#else
static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
{
	return 0;
}
static inline void dax_remove_host(struct gendisk *disk)
{
}
static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev,
		u64 *start_off, void *holder,
		const struct dax_holder_operations *ops)
{
	return NULL;
}
static inline void fs_put_dax(struct dax_device *dax_dev, void *holder)
{
}
#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */

#if IS_ENABLED(CONFIG_FS_DAX)
int dax_writeback_mapping_range(struct address_space *mapping,
		struct dax_device *dax_dev, struct writeback_control *wbc);

struct page *dax_layout_busy_page(struct address_space *mapping);
struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
dax_entry_t dax_lock_folio(struct folio *folio);
void dax_unlock_folio(struct folio *folio, dax_entry_t cookie);
dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
		unsigned long index, struct page **page);
void dax_unlock_mapping_entry(struct address_space *mapping,
		unsigned long index, dax_entry_t cookie);
#else
static inline struct page *dax_layout_busy_page(struct address_space *mapping)
{
	return NULL;
}

static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages)
{
	return NULL;
}

static inline int dax_writeback_mapping_range(struct address_space *mapping,
		struct dax_device *dax_dev, struct writeback_control *wbc)
{
	return -EOPNOTSUPP;
}

static inline dax_entry_t dax_lock_folio(struct folio *folio)
{
	if (IS_DAX(folio->mapping->host))
		return ~0UL;
	return 0;
}

static inline void dax_unlock_folio(struct folio *folio, dax_entry_t cookie)
{
}

static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
		unsigned long index, struct page **page)
{
	return 0;
}

static inline void dax_unlock_mapping_entry(struct address_space *mapping,
		unsigned long index, dax_entry_t cookie)
{
}
#endif

int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
		const struct iomap_ops *ops);
int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
		const struct iomap_ops *ops);
int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
		const struct iomap_ops *ops);

#if IS_ENABLED(CONFIG_DAX)
int dax_read_lock(void);
void dax_read_unlock(int id);
#else
static inline int dax_read_lock(void)
{
	return 0;
}

static inline void dax_read_unlock(int id)
{
}
#endif /* CONFIG_DAX */
bool dax_alive(struct dax_device *dax_dev);
void *dax_get_private(struct dax_device *dax_dev);
long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
		enum dax_access_mode mode, void **kaddr, pfn_t *pfn);
size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
		size_t bytes, struct iov_iter *i);
size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
		size_t bytes, struct iov_iter *i);
int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
			size_t nr_pages);
int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len,
		int mf_flags);
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size);

ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
		const struct iomap_ops *ops);
vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,
		    pfn_t *pfnp, int *errp, const struct iomap_ops *ops);
vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
		unsigned int order, pfn_t pfn);
int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
				      pgoff_t index);
int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
				  struct inode *dest, loff_t destoff,
				  loff_t len, bool *is_same,
				  const struct iomap_ops *ops);
int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
			      struct file *file_out, loff_t pos_out,
			      loff_t *len, unsigned int remap_flags,
			      const struct iomap_ops *ops);
static inline bool dax_mapping(struct address_space *mapping)
{
	return mapping->host && IS_DAX(mapping->host);
}

/*
 * Due to dax's memory and block duo personalities, hwpoison reporting
 * takes into consideration which personality is presently visible.
 * When dax acts like a block device, such as in block IO, an encounter of
 * dax hwpoison is reported as -EIO.
 * When dax acts like memory, such as in page fault, a detection of hwpoison
 * is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON.
 */
static inline int dax_mem2blk_err(int err)
{
	return (err == -EHWPOISON) ? -EIO : err;
}

#ifdef CONFIG_DEV_DAX_HMEM_DEVICES
void hmem_register_resource(int target_nid, struct resource *r);
#else
static inline void hmem_register_resource(int target_nid, struct resource *r)
{
}
#endif

typedef int (*walk_hmem_fn)(struct device *dev, int target_nid,
			    const struct resource *res);
int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
#endif
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
c94c2acf MW	2	#ifndef _LINUX_DAX_H
	3	#define _LINUX_DAX_H
	4
	5	#include <linux/fs.h>
	6	#include <linux/mm.h>
4f622938	7	#include <linux/radix-tree.h>
c94c2acf	8
27359fd6 MW	9	typedef unsigned long dax_entry_t;
27359fd6 MW	10
fb08a190 CH	11	struct dax_device;
fb08a190 CH	12	struct gendisk;
a254e568	13	struct iomap_ops;
c6f40468	14	struct iomap_iter;
4f3b4f16	15	struct iomap;
fb08a190	16
e511c4a3 JC	17	enum dax_access_mode {
	18	DAX_ACCESS,
	19	DAX_RECOVERY_WRITE,
	20	};
	21
6568b08b DW	22	struct dax_operations {
	23	/*
	24	* direct_access: translate a device-relative
	25	* logical-page-offset into an absolute physical pfn. Return the
	26	* number of pages available for DAX at that pfn.
	27	*/
	28	long (direct_access)(struct dax_device , pgoff_t, long,
e511c4a3	29	enum dax_access_mode, void *, pfn_t );
7bf7eac8 DW	30	/*
	31	* Validate whether this device is usable as an fsdax backing
	32	* device.
	33	*/
	34	bool (dax_supported)(struct dax_device , struct block_device *, int,
	35	sector_t, sector_t);
f605a263 VG	36	/* zero_page_range: required operation. Zero page range */
f605a263 VG	37	int (zero_page_range)(struct dax_device , pgoff_t, size_t);
047218ec JC	38	/*
	39	* recovery_write: recover a poisoned range by DAX device driver
	40	* capable of clearing poison.
	41	*/
	42	size_t (recovery_write)(struct dax_device dax_dev, pgoff_t pgoff,
	43	void addr, size_t bytes, struct iov_iter iter);
6568b08b	44	};
a254e568	45
8012b866 SR	46	struct dax_holder_operations {
	47	/*
	48	* notify_failure - notify memory failure into inner holder device
	49	* @dax_dev: the dax device which contains the holder
	50	* @offset: offset on this dax device where memory failure occurs
	51	* @len: length of this memory failure event
	52	* @flags: action flags for memory failure handler
	53	*/
	54	int (notify_failure)(struct dax_device dax_dev, u64 offset,
	55	u64 len, int mf_flags);
	56	};
	57
f5705aa8	58	#if IS_ENABLED(CONFIG_DAX)
30c6828a	59	struct dax_device alloc_dax(void private, const struct dax_operations *ops);
8012b866	60	void dax_holder(struct dax_device dax_dev);
f5705aa8	61	void put_dax(struct dax_device *dax_dev);
976431b0 DW	62	void kill_dax(struct dax_device *dax_dev);
	63	void dax_write_cache(struct dax_device *dax_dev, bool wc);
	64	bool dax_write_cache_enabled(struct dax_device *dax_dev);
fd1d00ec	65	bool dax_synchronous(struct dax_device *dax_dev);
2807c54b MD	66	void set_dax_nocache(struct dax_device *dax_dev);
2807c54b MD	67	void set_dax_nomc(struct dax_device *dax_dev);
fd1d00ec	68	void set_dax_synchronous(struct dax_device *dax_dev);
047218ec JC	69	size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
047218ec JC	70	void addr, size_t bytes, struct iov_iter i);
32de1484 PG	71	/*
	72	* Check if given mapping is supported by the file / underlying device.
	73	*/
	74	static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
	75	struct dax_device *dax_dev)
	76	{
	77	if (!(vma->vm_flags & VM_SYNC))
	78	return true;
	79	if (!IS_DAX(file_inode(vma->vm_file)))
	80	return false;
	81	return dax_synchronous(dax_dev);
	82	}
f5705aa8	83	#else
8012b866 SR	84	static inline void dax_holder(struct dax_device dax_dev)
	85	{
	86	return NULL;
	87	}
fb08a190	88	static inline struct dax_device alloc_dax(void private,
30c6828a	89	const struct dax_operations *ops)
976431b0	90	{
6d439c18	91	return ERR_PTR(-EOPNOTSUPP);
976431b0	92	}
f5705aa8 DW	93	static inline void put_dax(struct dax_device *dax_dev)
	94	{
	95	}
976431b0 DW	96	static inline void kill_dax(struct dax_device *dax_dev)
	97	{
	98	}
	99	static inline void dax_write_cache(struct dax_device *dax_dev, bool wc)
	100	{
	101	}
	102	static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
	103	{
	104	return false;
	105	}
fefc1d97 PG	106	static inline bool dax_synchronous(struct dax_device *dax_dev)
	107	{
	108	return true;
	109	}
2807c54b MD	110	static inline void set_dax_nocache(struct dax_device *dax_dev)
	111	{
	112	}
	113	static inline void set_dax_nomc(struct dax_device *dax_dev)
	114	{
	115	}
fefc1d97 PG	116	static inline void set_dax_synchronous(struct dax_device *dax_dev)
	117	{
	118	}
32de1484 PG	119	static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
	120	struct dax_device *dax_dev)
	121	{
	122	return !(vma->vm_flags & VM_SYNC);
	123	}
047218ec JC	124	static inline size_t dax_recovery_write(struct dax_device *dax_dev,
	125	pgoff_t pgoff, void addr, size_t bytes, struct iov_iter i)
	126	{
	127	return 0;
	128	}
f5705aa8 DW	129	#endif
f5705aa8 DW	130
f44c7763	131	struct writeback_control;
2ede8923	132	#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
fb08a190 CH	133	int dax_add_host(struct dax_device dax_dev, struct gendisk disk);
fb08a190 CH	134	void dax_remove_host(struct gendisk *disk);
8012b866 SR	135	struct dax_device fs_dax_get_by_bdev(struct block_device bdev, u64 *start_off,
	136	void holder, const struct dax_holder_operations ops);
	137	void fs_put_dax(struct dax_device dax_dev, void holder);
ef510424	138	#else
fb08a190 CH	139	static inline int dax_add_host(struct dax_device dax_dev, struct gendisk disk)
	140	{
	141	return 0;
	142	}
	143	static inline void dax_remove_host(struct gendisk *disk)
	144	{
	145	}
cd913c76	146	static inline struct dax_device fs_dax_get_by_bdev(struct block_device bdev,
8012b866 SR	147	u64 start_off, void holder,
8012b866 SR	148	const struct dax_holder_operations *ops)
78f35473 DW	149	{
	150	return NULL;
	151	}
8012b866	152	static inline void fs_put_dax(struct dax_device dax_dev, void holder)
2ede8923 CH	153	{
	154	}
	155	#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
f44c7763	156
2ede8923 CH	157	#if IS_ENABLED(CONFIG_FS_DAX)
	158	int dax_writeback_mapping_range(struct address_space *mapping,
	159	struct dax_device dax_dev, struct writeback_control wbc);
	160
	161	struct page dax_layout_busy_page(struct address_space mapping);
	162	struct page dax_layout_busy_page_range(struct address_space mapping, loff_t start, loff_t end);
91e79d22 MWO	163	dax_entry_t dax_lock_folio(struct folio *folio);
91e79d22 MWO	164	void dax_unlock_folio(struct folio *folio, dax_entry_t cookie);
2f437eff SR	165	dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
	166	unsigned long index, struct page **page);
	167	void dax_unlock_mapping_entry(struct address_space *mapping,
	168	unsigned long index, dax_entry_t cookie);
2ede8923	169	#else
5fac7408 DW	170	static inline struct page dax_layout_busy_page(struct address_space mapping)
	171	{
	172	return NULL;
	173	}
	174
6bbdd563 VG	175	static inline struct page dax_layout_busy_page_range(struct address_space mapping, pgoff_t start, pgoff_t nr_pages)
	176	{
	177	return NULL;
	178	}
	179
f44c7763	180	static inline int dax_writeback_mapping_range(struct address_space *mapping,
3f666c56	181	struct dax_device dax_dev, struct writeback_control wbc)
f44c7763 DW	182	{
	183	return -EOPNOTSUPP;
	184	}
c2a7d2a1	185
91e79d22	186	static inline dax_entry_t dax_lock_folio(struct folio *folio)
c2a7d2a1	187	{
91e79d22	188	if (IS_DAX(folio->mapping->host))
27359fd6 MW	189	return ~0UL;
27359fd6 MW	190	return 0;
c2a7d2a1 DW	191	}
c2a7d2a1 DW	192
91e79d22	193	static inline void dax_unlock_folio(struct folio *folio, dax_entry_t cookie)
c2a7d2a1 DW	194	{
c2a7d2a1 DW	195	}
2f437eff SR	196
	197	static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
	198	unsigned long index, struct page **page)
	199	{
	200	return 0;
	201	}
	202
	203	static inline void dax_unlock_mapping_entry(struct address_space *mapping,
	204	unsigned long index, dax_entry_t cookie)
	205	{
	206	}
ef510424 DW	207	#endif
ef510424 DW	208
d984648e SR	209	int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
d984648e SR	210	const struct iomap_ops *ops);
c6f40468 CH	211	int dax_zero_range(struct inode inode, loff_t pos, loff_t len, bool did_zero,
	212	const struct iomap_ops *ops);
	213	int dax_truncate_page(struct inode inode, loff_t pos, bool did_zero,
	214	const struct iomap_ops *ops);
	215
e2ec5128	216	#if IS_ENABLED(CONFIG_DAX)
7b6be844 DW	217	int dax_read_lock(void);
7b6be844 DW	218	void dax_read_unlock(int id);
e2ec5128 JK	219	#else
	220	static inline int dax_read_lock(void)
	221	{
	222	return 0;
	223	}
	224
	225	static inline void dax_read_unlock(int id)
	226	{
	227	}
	228	#endif /* CONFIG_DAX */
c1d6e828	229	bool dax_alive(struct dax_device *dax_dev);
c1d6e828	230	void dax_get_private(struct dax_device dax_dev);
b0686260	231	long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
e511c4a3	232	enum dax_access_mode mode, void *kaddr, pfn_t pfn);
7e026c8c DW	233	size_t dax_copy_from_iter(struct dax_device dax_dev, pgoff_t pgoff, void addr,
7e026c8c DW	234	size_t bytes, struct iov_iter *i);
b3a9a0c3 DW	235	size_t dax_copy_to_iter(struct dax_device dax_dev, pgoff_t pgoff, void addr,
b3a9a0c3 DW	236	size_t bytes, struct iov_iter *i);
f605a263 VG	237	int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
f605a263 VG	238	size_t nr_pages);
8012b866 SR	239	int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len,
8012b866 SR	240	int mf_flags);
c3ca015f	241	void dax_flush(struct dax_device dax_dev, void addr, size_t size);
7b6be844	242
11c59c92	243	ssize_t dax_iomap_rw(struct kiocb iocb, struct iov_iter iter,
8ff6daa1	244	const struct iomap_ops *ops);
1d024e7a	245	vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,
c0b24625	246	pfn_t pfnp, int errp, const struct iomap_ops *ops);
ab77dab4	247	vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
1d024e7a	248	unsigned int order, pfn_t pfn);
ac401cc7	249	int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
c6dcf52c JK	250	int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
c6dcf52c JK	251	pgoff_t index);
6f7db389 SR	252	int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
	253	struct inode *dest, loff_t destoff,
	254	loff_t len, bool *is_same,
	255	const struct iomap_ops *ops);
	256	int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
	257	struct file *file_out, loff_t pos_out,
	258	loff_t *len, unsigned int remap_flags,
	259	const struct iomap_ops *ops);
f9fe48be RZ	260	static inline bool dax_mapping(struct address_space *mapping)
	261	{
	262	return mapping->host && IS_DAX(mapping->host);
	263	}
7f6d5b52	264
1ea7ca1b JC	265	/*
	266	* Due to dax's memory and block duo personalities, hwpoison reporting
	267	* takes into consideration which personality is presently visible.
	268	* When dax acts like a block device, such as in block IO, an encounter of
	269	* dax hwpoison is reported as -EIO.
	270	* When dax acts like memory, such as in page fault, a detection of hwpoison
	271	* is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON.
	272	*/
	273	static inline int dax_mem2blk_err(int err)
	274	{
	275	return (err == -EHWPOISON) ? -EIO : err;
	276	}
	277
c01044cc	278	#ifdef CONFIG_DEV_DAX_HMEM_DEVICES
7dab174e	279	void hmem_register_resource(int target_nid, struct resource *r);
c01044cc	280	#else
7dab174e	281	static inline void hmem_register_resource(int target_nid, struct resource *r)
c01044cc DW	282	{
	283	}
	284	#endif
	285
7dab174e DW	286	typedef int (walk_hmem_fn)(struct device dev, int target_nid,
	287	const struct resource *res);
	288	int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
c94c2acf	289	#endif