[linux-block.git] / mm / swap.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _MM_SWAP_H
#define _MM_SWAP_H

struct mempolicy;
extern int page_cluster;

#ifdef CONFIG_SWAP
#include <linux/swapops.h> /* for swp_offset */
#include <linux/blk_types.h> /* for bio_end_io_t */

/* linux/mm/page_io.c */
int sio_pool_init(void);
struct swap_iocb;
void swap_read_folio(struct folio *folio, struct swap_iocb **plug);
void __swap_read_unplug(struct swap_iocb *plug);
static inline void swap_read_unplug(struct swap_iocb *plug)
{
	if (unlikely(plug))
		__swap_read_unplug(plug);
}
void swap_write_unplug(struct swap_iocb *sio);
int swap_writeout(struct folio *folio, struct writeback_control *wbc);
void __swap_writepage(struct folio *folio, struct writeback_control *wbc);

/* linux/mm/swap_state.c */
/* One swap address space for each 64M swap space */
#define SWAP_ADDRESS_SPACE_SHIFT	14
#define SWAP_ADDRESS_SPACE_PAGES	(1 << SWAP_ADDRESS_SPACE_SHIFT)
#define SWAP_ADDRESS_SPACE_MASK		(SWAP_ADDRESS_SPACE_PAGES - 1)
extern struct address_space *swapper_spaces[];
#define swap_address_space(entry)			    \
	(&swapper_spaces[swp_type(entry)][swp_offset(entry) \
		>> SWAP_ADDRESS_SPACE_SHIFT])

/*
 * Return the swap device position of the swap entry.
 */
static inline loff_t swap_dev_pos(swp_entry_t entry)
{
	return ((loff_t)swp_offset(entry)) << PAGE_SHIFT;
}

/*
 * Return the swap cache index of the swap entry.
 */
static inline pgoff_t swap_cache_index(swp_entry_t entry)
{
	BUILD_BUG_ON((SWP_OFFSET_MASK | SWAP_ADDRESS_SPACE_MASK) != SWP_OFFSET_MASK);
	return swp_offset(entry) & SWAP_ADDRESS_SPACE_MASK;
}

void show_swap_cache_info(void);
void *get_shadow_from_swap_cache(swp_entry_t entry);
int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
		      gfp_t gfp, void **shadowp);
void __delete_from_swap_cache(struct folio *folio,
			      swp_entry_t entry, void *shadow);
void delete_from_swap_cache(struct folio *folio);
void clear_shadow_from_swap_cache(int type, unsigned long begin,
				  unsigned long end);
void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
struct folio *swap_cache_get_folio(swp_entry_t entry,
		struct vm_area_struct *vma, unsigned long addr);
struct folio *filemap_get_incore_folio(struct address_space *mapping,
		pgoff_t index);

struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
		struct vm_area_struct *vma, unsigned long addr,
		struct swap_iocb **plug);
struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_flags,
		struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated,
		bool skip_if_exists);
struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
		struct mempolicy *mpol, pgoff_t ilx);
struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
		struct vm_fault *vmf);

static inline unsigned int folio_swap_flags(struct folio *folio)
{
	return swp_swap_info(folio->swap)->flags;
}

/*
 * Return the count of contiguous swap entries that share the same
 * zeromap status as the starting entry. If is_zeromap is not NULL,
 * it will return the zeromap status of the starting entry.
 */
static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
		bool *is_zeromap)
{
	struct swap_info_struct *sis = swp_swap_info(entry);
	unsigned long start = swp_offset(entry);
	unsigned long end = start + max_nr;
	bool first_bit;

	first_bit = test_bit(start, sis->zeromap);
	if (is_zeromap)
		*is_zeromap = first_bit;

	if (max_nr <= 1)
		return max_nr;
	if (first_bit)
		return find_next_zero_bit(sis->zeromap, end, start) - start;
	else
		return find_next_bit(sis->zeromap, end, start) - start;
}

static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
{
	struct swap_info_struct *si = swp_swap_info(entry);
	pgoff_t offset = swp_offset(entry);
	int i;

	/*
	 * While allocating a large folio and doing mTHP swapin, we need to
	 * ensure all entries are not cached, otherwise, the mTHP folio will
	 * be in conflict with the folio in swap cache.
	 */
	for (i = 0; i < max_nr; i++) {
		if ((si->swap_map[offset + i] & SWAP_HAS_CACHE))
			return i;
	}

	return i;
}

#else /* CONFIG_SWAP */
struct swap_iocb;
static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
{
}
static inline void swap_write_unplug(struct swap_iocb *sio)
{
}

static inline struct address_space *swap_address_space(swp_entry_t entry)
{
	return NULL;
}

static inline pgoff_t swap_cache_index(swp_entry_t entry)
{
	return 0;
}

static inline void show_swap_cache_info(void)
{
}

static inline struct folio *swap_cluster_readahead(swp_entry_t entry,
			gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx)
{
	return NULL;
}

static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask,
			struct vm_fault *vmf)
{
	return NULL;
}

static inline int swap_writeout(struct folio *f, struct writeback_control *wbc)
{
	return 0;
}

static inline void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr)
{
}

static inline struct folio *swap_cache_get_folio(swp_entry_t entry,
		struct vm_area_struct *vma, unsigned long addr)
{
	return NULL;
}

static inline
struct folio *filemap_get_incore_folio(struct address_space *mapping,
		pgoff_t index)
{
	return filemap_get_folio(mapping, index);
}

static inline void *get_shadow_from_swap_cache(swp_entry_t entry)
{
	return NULL;
}

static inline int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
					gfp_t gfp_mask, void **shadowp)
{
	return -1;
}

static inline void __delete_from_swap_cache(struct folio *folio,
					swp_entry_t entry, void *shadow)
{
}

static inline void delete_from_swap_cache(struct folio *folio)
{
}

static inline void clear_shadow_from_swap_cache(int type, unsigned long begin,
				unsigned long end)
{
}

static inline unsigned int folio_swap_flags(struct folio *folio)
{
	return 0;
}

static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
		bool *has_zeromap)
{
	return 0;
}

static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
{
	return 0;
}
#endif /* CONFIG_SWAP */

/**
 * folio_index - File index of a folio.
 * @folio: The folio.
 *
 * For a folio which is either in the page cache or the swap cache,
 * return its index within the address_space it belongs to.  If you know
 * the folio is definitely in the page cache, you can look at the folio's
 * index directly.
 *
 * Return: The index (offset in units of pages) of a folio in its file.
 */
static inline pgoff_t folio_index(struct folio *folio)
{
	if (unlikely(folio_test_swapcache(folio)))
		return swap_cache_index(folio->swap);
	return folio->index;
}

#endif /* _MM_SWAP_H */
Commit	Line	Data
014bb1de N	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef _MM_SWAP_H
	3	#define _MM_SWAP_H
	4
ddc1a5cb	5	struct mempolicy;
7e05627e	6	extern int page_cluster;
ddc1a5cb	7
014bb1de	8	#ifdef CONFIG_SWAP
545ebe71	9	#include <linux/swapops.h> /* for swp_offset */
014bb1de N	10	#include <linux/blk_types.h> /* for bio_end_io_t */
	11
	12	/* linux/mm/page_io.c */
e1209d3a	13	int sio_pool_init(void);
5169b844	14	struct swap_iocb;
b2d1f38b	15	void swap_read_folio(struct folio folio, struct swap_iocb *plug);
5169b844 N	16	void __swap_read_unplug(struct swap_iocb *plug);
	17	static inline void swap_read_unplug(struct swap_iocb *plug)
	18	{
	19	if (unlikely(plug))
	20	__swap_read_unplug(plug);
	21	}
2282679f	22	void swap_write_unplug(struct swap_iocb *sio);
84798514	23	int swap_writeout(struct folio folio, struct writeback_control wbc);
b99b4e0d	24	void __swap_writepage(struct folio folio, struct writeback_control wbc);
014bb1de N	25
	26	/* linux/mm/swap_state.c */
	27	/* One swap address space for each 64M swap space */
	28	#define SWAP_ADDRESS_SPACE_SHIFT 14
	29	#define SWAP_ADDRESS_SPACE_PAGES (1 << SWAP_ADDRESS_SPACE_SHIFT)
7aad25b4	30	#define SWAP_ADDRESS_SPACE_MASK (SWAP_ADDRESS_SPACE_PAGES - 1)
014bb1de N	31	extern struct address_space *swapper_spaces[];
	32	#define swap_address_space(entry) \
	33	(&swapper_spaces[swp_type(entry)][swp_offset(entry) \
	34	>> SWAP_ADDRESS_SPACE_SHIFT])
	35
545ebe71 KS	36	/*
	37	* Return the swap device position of the swap entry.
	38	*/
	39	static inline loff_t swap_dev_pos(swp_entry_t entry)
	40	{
	41	return ((loff_t)swp_offset(entry)) << PAGE_SHIFT;
	42	}
	43
7aad25b4 KS	44	/*
	45	* Return the swap cache index of the swap entry.
	46	*/
	47	static inline pgoff_t swap_cache_index(swp_entry_t entry)
	48	{
	49	BUILD_BUG_ON((SWP_OFFSET_MASK \| SWAP_ADDRESS_SPACE_MASK) != SWP_OFFSET_MASK);
	50	return swp_offset(entry) & SWAP_ADDRESS_SPACE_MASK;
	51	}
	52
014bb1de	53	void show_swap_cache_info(void);
014bb1de	54	void *get_shadow_from_swap_cache(swp_entry_t entry);
a4c366f0	55	int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
014bb1de	56	gfp_t gfp, void **shadowp);
ceff9d33	57	void __delete_from_swap_cache(struct folio *folio,
014bb1de	58	swp_entry_t entry, void *shadow);
75fa68a5	59	void delete_from_swap_cache(struct folio *folio);
014bb1de N	60	void clear_shadow_from_swap_cache(int type, unsigned long begin,
014bb1de N	61	unsigned long end);
9f101bef	62	void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr);
c9edc242 MWO	63	struct folio *swap_cache_get_folio(swp_entry_t entry,
c9edc242 MWO	64	struct vm_area_struct *vma, unsigned long addr);
524984ff MWO	65	struct folio filemap_get_incore_folio(struct address_space mapping,
524984ff MWO	66	pgoff_t index);
014bb1de	67
6e03492e MWO	68	struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
	69	struct vm_area_struct *vma, unsigned long addr,
	70	struct swap_iocb **plug);
96c7b0b4 MWO	71	struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_flags,
	72	struct mempolicy mpol, pgoff_t ilx, bool new_page_allocated,
	73	bool skip_if_exists);
a4575c41 MWO	74	struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag,
a4575c41 MWO	75	struct mempolicy *mpol, pgoff_t ilx);
94dc8bff MWO	76	struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag,
94dc8bff MWO	77	struct vm_fault *vmf);
014bb1de	78
b98c359f	79	static inline unsigned int folio_swap_flags(struct folio *folio)
d791ea67	80	{
69fe7d67	81	return swp_swap_info(folio->swap)->flags;
d791ea67	82	}
9d57090e BS	83
	84	/*
	85	* Return the count of contiguous swap entries that share the same
	86	* zeromap status as the starting entry. If is_zeromap is not NULL,
	87	* it will return the zeromap status of the starting entry.
	88	*/
	89	static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
	90	bool *is_zeromap)
	91	{
	92	struct swap_info_struct *sis = swp_swap_info(entry);
	93	unsigned long start = swp_offset(entry);
	94	unsigned long end = start + max_nr;
	95	bool first_bit;
	96
	97	first_bit = test_bit(start, sis->zeromap);
	98	if (is_zeromap)
	99	*is_zeromap = first_bit;
	100
	101	if (max_nr <= 1)
	102	return max_nr;
	103	if (first_bit)
	104	return find_next_zero_bit(sis->zeromap, end, start) - start;
	105	else
	106	return find_next_bit(sis->zeromap, end, start) - start;
	107	}
	108
a05dd8ae KS	109	static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
	110	{
	111	struct swap_info_struct *si = swp_swap_info(entry);
	112	pgoff_t offset = swp_offset(entry);
	113	int i;
	114
	115	/*
	116	* While allocating a large folio and doing mTHP swapin, we need to
	117	* ensure all entries are not cached, otherwise, the mTHP folio will
	118	* be in conflict with the folio in swap cache.
	119	*/
	120	for (i = 0; i < max_nr; i++) {
	121	if ((si->swap_map[offset + i] & SWAP_HAS_CACHE))
	122	return i;
	123	}
	124
	125	return i;
	126	}
	127
014bb1de	128	#else /* CONFIG_SWAP */
5169b844	129	struct swap_iocb;
b2d1f38b	130	static inline void swap_read_folio(struct folio folio, struct swap_iocb *plug)
014bb1de	131	{
014bb1de	132	}
2282679f N	133	static inline void swap_write_unplug(struct swap_iocb *sio)
	134	{
	135	}
014bb1de N	136
	137	static inline struct address_space *swap_address_space(swp_entry_t entry)
	138	{
	139	return NULL;
	140	}
	141
7aad25b4 KS	142	static inline pgoff_t swap_cache_index(swp_entry_t entry)
	143	{
	144	return 0;
	145	}
	146
014bb1de N	147	static inline void show_swap_cache_info(void)
	148	{
	149	}
	150
a4575c41	151	static inline struct folio *swap_cluster_readahead(swp_entry_t entry,
ddc1a5cb	152	gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx)
014bb1de N	153	{
	154	return NULL;
	155	}
	156
94dc8bff	157	static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask,
014bb1de N	158	struct vm_fault *vmf)
	159	{
	160	return NULL;
	161	}
	162
84798514	163	static inline int swap_writeout(struct folio f, struct writeback_control wbc)
014bb1de N	164	{
	165	return 0;
	166	}
	167
9f101bef	168	static inline void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr)
13ddaf26 KS	169	{
	170	}
	171
c9edc242 MWO	172	static inline struct folio *swap_cache_get_folio(swp_entry_t entry,
	173	struct vm_area_struct *vma, unsigned long addr)
	174	{
	175	return NULL;
	176	}
	177
014bb1de	178	static inline
524984ff MWO	179	struct folio filemap_get_incore_folio(struct address_space mapping,
524984ff MWO	180	pgoff_t index)
014bb1de	181	{
524984ff	182	return filemap_get_folio(mapping, index);
014bb1de N	183	}
014bb1de N	184
014bb1de N	185	static inline void *get_shadow_from_swap_cache(swp_entry_t entry)
	186	{
	187	return NULL;
	188	}
	189
a4c366f0	190	static inline int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
014bb1de N	191	gfp_t gfp_mask, void **shadowp)
	192	{
	193	return -1;
	194	}
	195
ceff9d33	196	static inline void __delete_from_swap_cache(struct folio *folio,
014bb1de N	197	swp_entry_t entry, void *shadow)
	198	{
	199	}
	200
75fa68a5	201	static inline void delete_from_swap_cache(struct folio *folio)
014bb1de N	202	{
	203	}
	204
	205	static inline void clear_shadow_from_swap_cache(int type, unsigned long begin,
	206	unsigned long end)
	207	{
	208	}
	209
b98c359f	210	static inline unsigned int folio_swap_flags(struct folio *folio)
d791ea67 N	211	{
	212	return 0;
	213	}
9d57090e BS	214
	215	static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr,
	216	bool *has_zeromap)
	217	{
	218	return 0;
	219	}
	220
a05dd8ae KS	221	static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
	222	{
	223	return 0;
	224	}
014bb1de	225	#endif /* CONFIG_SWAP */
9f101bef	226
7d0f0f06 KS	227	/**
	228	* folio_index - File index of a folio.
	229	* @folio: The folio.
	230	*
	231	* For a folio which is either in the page cache or the swap cache,
	232	* return its index within the address_space it belongs to. If you know
	233	* the folio is definitely in the page cache, you can look at the folio's
	234	* index directly.
	235	*
	236	* Return: The index (offset in units of pages) of a folio in its file.
	237	*/
	238	static inline pgoff_t folio_index(struct folio *folio)
	239	{
	240	if (unlikely(folio_test_swapcache(folio)))
	241	return swap_cache_index(folio->swap);
	242	return folio->index;
	243	}
	244
014bb1de	245	#endif /* _MM_SWAP_H */