[linux-block.git] / fs / btrfs / compression.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2008 Oracle.  All rights reserved.
 */

#ifndef BTRFS_COMPRESSION_H
#define BTRFS_COMPRESSION_H

#include <linux/sizes.h>
#include <linux/mm.h>
#include <linux/list.h>
#include <linux/workqueue.h>
#include <linux/wait.h>
#include "bio.h"

struct address_space;
struct page;
struct inode;
struct btrfs_inode;
struct btrfs_ordered_extent;
struct btrfs_bio;

/*
 * We want to make sure that amount of RAM required to uncompress an extent is
 * reasonable, so we limit the total size in ram of a compressed extent to
 * 128k.  This is a crucial number because it also controls how easily we can
 * spread reads across cpus for decompression.
 *
 * We also want to make sure the amount of IO required to do a random read is
 * reasonably small, so we limit the size of a compressed extent to 128k.
 */

/* Maximum length of compressed data stored on disk */
#define BTRFS_MAX_COMPRESSED		(SZ_128K)
#define BTRFS_MAX_COMPRESSED_PAGES	(BTRFS_MAX_COMPRESSED / PAGE_SIZE)
static_assert((BTRFS_MAX_COMPRESSED % PAGE_SIZE) == 0);

/* Maximum size of data before compression */
#define BTRFS_MAX_UNCOMPRESSED		(SZ_128K)

#define	BTRFS_ZLIB_DEFAULT_LEVEL		3

struct compressed_bio {
	/* Number of compressed folios in the array. */
	unsigned int nr_folios;

	/* The folios with the compressed data on them. */
	struct folio **compressed_folios;

	/* starting offset in the inode for our pages */
	u64 start;

	/* Number of bytes in the inode we're working on */
	unsigned int len;

	/* Number of bytes on disk */
	unsigned int compressed_len;

	/* The compression algorithm for this bio */
	u8 compress_type;

	/* Whether this is a write for writeback. */
	bool writeback;

	union {
		/* For reads, this is the bio we are copying the data into */
		struct btrfs_bio *orig_bbio;
		struct work_struct write_end_work;
	};

	/* Must be last. */
	struct btrfs_bio bbio;
};

static inline unsigned int btrfs_compress_type(unsigned int type_level)
{
	return (type_level & 0xF);
}

static inline unsigned int btrfs_compress_level(unsigned int type_level)
{
	return ((type_level & 0xF0) >> 4);
}

/* @range_end must be exclusive. */
static inline u32 btrfs_calc_input_length(u64 range_end, u64 cur)
{
	u64 page_end = round_down(cur, PAGE_SIZE) + PAGE_SIZE;

	return min(range_end, page_end) - cur;
}

int __init btrfs_init_compress(void);
void __cold btrfs_exit_compress(void);

int btrfs_compress_folios(unsigned int type_level, struct address_space *mapping,
			  u64 start, struct folio **folios, unsigned long *out_folios,
			 unsigned long *total_in, unsigned long *total_out);
int btrfs_decompress(int type, const u8 *data_in, struct folio *dest_folio,
		     unsigned long start_byte, size_t srclen, size_t destlen);
int btrfs_decompress_buf2page(const char *buf, u32 buf_len,
			      struct compressed_bio *cb, u32 decompressed);

void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered,
				   struct folio **compressed_folios,
				   unsigned int nr_folios, blk_opf_t write_flags,
				   bool writeback);
void btrfs_submit_compressed_read(struct btrfs_bio *bbio);

unsigned int btrfs_compress_str2level(unsigned int type, const char *str);

struct folio *btrfs_alloc_compr_folio(void);
void btrfs_free_compr_folio(struct folio *folio);

enum btrfs_compression_type {
	BTRFS_COMPRESS_NONE  = 0,
	BTRFS_COMPRESS_ZLIB  = 1,
	BTRFS_COMPRESS_LZO   = 2,
	BTRFS_COMPRESS_ZSTD  = 3,
	BTRFS_NR_COMPRESS_TYPES = 4,
};

struct workspace_manager {
	struct list_head idle_ws;
	spinlock_t ws_lock;
	/* Number of free workspaces */
	int free_ws;
	/* Total number of allocated workspaces */
	atomic_t total_ws;
	/* Waiters for a free workspace */
	wait_queue_head_t ws_wait;
};

struct list_head *btrfs_get_workspace(int type, unsigned int level);
void btrfs_put_workspace(int type, struct list_head *ws);

struct btrfs_compress_op {
	struct workspace_manager *workspace_manager;
	/* Maximum level supported by the compression algorithm */
	unsigned int max_level;
	unsigned int default_level;
};

/* The heuristic workspaces are managed via the 0th workspace manager */
#define BTRFS_NR_WORKSPACE_MANAGERS	BTRFS_NR_COMPRESS_TYPES

extern const struct btrfs_compress_op btrfs_heuristic_compress;
extern const struct btrfs_compress_op btrfs_zlib_compress;
extern const struct btrfs_compress_op btrfs_lzo_compress;
extern const struct btrfs_compress_op btrfs_zstd_compress;

const char* btrfs_compress_type2str(enum btrfs_compression_type type);
bool btrfs_compress_is_valid_type(const char *str, size_t len);

int btrfs_compress_heuristic(struct btrfs_inode *inode, u64 start, u64 end);

int btrfs_compress_filemap_get_folio(struct address_space *mapping, u64 start,
				     struct folio **in_folio_ret);

int zlib_compress_folios(struct list_head *ws, struct address_space *mapping,
			 u64 start, struct folio **folios, unsigned long *out_folios,
		unsigned long *total_in, unsigned long *total_out);
int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
int zlib_decompress(struct list_head *ws, const u8 *data_in,
		struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
		size_t destlen);
struct list_head *zlib_alloc_workspace(unsigned int level);
void zlib_free_workspace(struct list_head *ws);
struct list_head *zlib_get_workspace(unsigned int level);

int lzo_compress_folios(struct list_head *ws, struct address_space *mapping,
			u64 start, struct folio **folios, unsigned long *out_folios,
		unsigned long *total_in, unsigned long *total_out);
int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
int lzo_decompress(struct list_head *ws, const u8 *data_in,
		struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
		size_t destlen);
struct list_head *lzo_alloc_workspace(unsigned int level);
void lzo_free_workspace(struct list_head *ws);

int zstd_compress_folios(struct list_head *ws, struct address_space *mapping,
			 u64 start, struct folio **folios, unsigned long *out_folios,
		unsigned long *total_in, unsigned long *total_out);
int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb);
int zstd_decompress(struct list_head *ws, const u8 *data_in,
		struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
		size_t destlen);
void zstd_init_workspace_manager(void);
void zstd_cleanup_workspace_manager(void);
struct list_head *zstd_alloc_workspace(unsigned int level);
void zstd_free_workspace(struct list_head *ws);
struct list_head *zstd_get_workspace(unsigned int level);
void zstd_put_workspace(struct list_head *ws);

#endif
Commit	Line	Data
9888c340	1	/* SPDX-License-Identifier: GPL-2.0 */
c8b97818 CM	2	/*
c8b97818 CM	3	* Copyright (C) 2008 Oracle. All rights reserved.
c8b97818 CM	4	*/
c8b97818 CM	5
9888c340 DS	6	#ifndef BTRFS_COMPRESSION_H
9888c340 DS	7	#define BTRFS_COMPRESSION_H
c8b97818	8
d5c1d68f	9	#include <linux/sizes.h>
602035d7 DS	10	#include <linux/mm.h>
	11	#include <linux/list.h>
	12	#include <linux/workqueue.h>
	13	#include <linux/wait.h>
544fe4a9	14	#include "bio.h"
d5c1d68f	15
602035d7 DS	16	struct address_space;
	17	struct page;
	18	struct inode;
c7ee1819	19	struct btrfs_inode;
d611935b	20	struct btrfs_ordered_extent;
602035d7	21	struct btrfs_bio;
c7ee1819	22
ff763866 DS	23	/*
	24	* We want to make sure that amount of RAM required to uncompress an extent is
	25	* reasonable, so we limit the total size in ram of a compressed extent to
	26	* 128k. This is a crucial number because it also controls how easily we can
	27	* spread reads across cpus for decompression.
	28	*
	29	* We also want to make sure the amount of IO required to do a random read is
	30	* reasonably small, so we limit the size of a compressed extent to 128k.
	31	*/
	32
	33	/* Maximum length of compressed data stored on disk */
	34	#define BTRFS_MAX_COMPRESSED (SZ_128K)
544fe4a9	35	#define BTRFS_MAX_COMPRESSED_PAGES (BTRFS_MAX_COMPRESSED / PAGE_SIZE)
a55e65b8 DS	36	static_assert((BTRFS_MAX_COMPRESSED % PAGE_SIZE) == 0);
a55e65b8 DS	37
ff763866 DS	38	/* Maximum size of data before compression */
	39	#define BTRFS_MAX_UNCOMPRESSED (SZ_128K)
	40
eae8d825 QW	41	#define BTRFS_ZLIB_DEFAULT_LEVEL 3
eae8d825 QW	42
e1ddce71	43	struct compressed_bio {
400b172b QW	44	/* Number of compressed folios in the array. */
400b172b QW	45	unsigned int nr_folios;
282ab3ff	46
400b172b QW	47	/* The folios with the compressed data on them. */
400b172b QW	48	struct folio **compressed_folios;
e1ddce71	49
e1ddce71 AJ	50	/* starting offset in the inode for our pages */
	51	u64 start;
	52
282ab3ff DS	53	/* Number of bytes in the inode we're working on */
282ab3ff DS	54	unsigned int len;
e1ddce71	55
282ab3ff DS	56	/* Number of bytes on disk */
282ab3ff DS	57	unsigned int compressed_len;
e1ddce71	58
282ab3ff DS	59	/* The compression algorithm for this bio */
282ab3ff DS	60	u8 compress_type;
e1ddce71	61
7c0c7269 OS	62	/* Whether this is a write for writeback. */
	63	bool writeback;
	64
fed8a72d CH	65	union {
fed8a72d CH	66	/* For reads, this is the bio we are copying the data into */
b7d463a1	67	struct btrfs_bio *orig_bbio;
fed8a72d CH	68	struct work_struct write_end_work;
fed8a72d CH	69	};
544fe4a9 CH	70
	71	/* Must be last. */
	72	struct btrfs_bio bbio;
e1ddce71 AJ	73	};
e1ddce71 AJ	74
1972708a DZ	75	static inline unsigned int btrfs_compress_type(unsigned int type_level)
	76	{
	77	return (type_level & 0xF);
	78	}
	79
	80	static inline unsigned int btrfs_compress_level(unsigned int type_level)
	81	{
	82	return ((type_level & 0xF0) >> 4);
	83	}
	84
fd1e75d0 QW	85	/* @range_end must be exclusive. */
	86	static inline u32 btrfs_calc_input_length(u64 range_end, u64 cur)
	87	{
	88	u64 page_end = round_down(cur, PAGE_SIZE) + PAGE_SIZE;
	89
	90	return min(range_end, page_end) - cur;
	91	}
	92
5565b8e0	93	int __init btrfs_init_compress(void);
e67c718b	94	void __cold btrfs_exit_compress(void);
261507a0	95
400b172b QW	96	int btrfs_compress_folios(unsigned int type_level, struct address_space *mapping,
	97	u64 start, struct folio *folios, unsigned long out_folios,
	98	unsigned long total_in, unsigned long total_out);
aeb6d881	99	int btrfs_decompress(int type, const u8 data_in, struct folio dest_folio,
261507a0	100	unsigned long start_byte, size_t srclen, size_t destlen);
1c3dc173 QW	101	int btrfs_decompress_buf2page(const char *buf, u32 buf_len,
1c3dc173 QW	102	struct compressed_bio *cb, u32 decompressed);
261507a0	103
d611935b	104	void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered,
400b172b QW	105	struct folio **compressed_folios,
	106	unsigned int nr_folios, blk_opf_t write_flags,
	107	bool writeback);
e1949310	108	void btrfs_submit_compressed_read(struct btrfs_bio *bbio);
ebb8765b	109
d0ab62ce	110	unsigned int btrfs_compress_str2level(unsigned int type, const char *str);
f51d2b59	111
98fe01af QW	112	struct folio *btrfs_alloc_compr_folio(void);
98fe01af QW	113	void btrfs_free_compr_folio(struct folio *folio);
9ba965dc	114
ebb8765b AJ	115	enum btrfs_compression_type {
	116	BTRFS_COMPRESS_NONE = 0,
	117	BTRFS_COMPRESS_ZLIB = 1,
	118	BTRFS_COMPRESS_LZO = 2,
5c1aab1d	119	BTRFS_COMPRESS_ZSTD = 3,
ce96b7ff	120	BTRFS_NR_COMPRESS_TYPES = 4,
ebb8765b AJ	121	};
ebb8765b AJ	122
92ee5530	123	struct workspace_manager {
92ee5530 DZ	124	struct list_head idle_ws;
	125	spinlock_t ws_lock;
	126	/* Number of free workspaces */
	127	int free_ws;
	128	/* Total number of allocated workspaces */
	129	atomic_t total_ws;
	130	/* Waiters for a free workspace */
	131	wait_queue_head_t ws_wait;
	132	};
	133
5907a9bb	134	struct list_head *btrfs_get_workspace(int type, unsigned int level);
a3bbd2a9	135	void btrfs_put_workspace(int type, struct list_head *ws);
92ee5530	136
261507a0	137	struct btrfs_compress_op {
be951045	138	struct workspace_manager *workspace_manager;
e18333a7 DS	139	/* Maximum level supported by the compression algorithm */
	140	unsigned int max_level;
	141	unsigned int default_level;
261507a0 LZ	142	};
261507a0 LZ	143
ca4ac360	144	/* The heuristic workspaces are managed via the 0th workspace manager */
ce96b7ff	145	#define BTRFS_NR_WORKSPACE_MANAGERS BTRFS_NR_COMPRESS_TYPES
ca4ac360 DZ	146
ca4ac360 DZ	147	extern const struct btrfs_compress_op btrfs_heuristic_compress;
e8c9f186 DS	148	extern const struct btrfs_compress_op btrfs_zlib_compress;
e8c9f186 DS	149	extern const struct btrfs_compress_op btrfs_lzo_compress;
5c1aab1d	150	extern const struct btrfs_compress_op btrfs_zstd_compress;
261507a0	151
e128f9c3	152	const char* btrfs_compress_type2str(enum btrfs_compression_type type);
aa53e3bf	153	bool btrfs_compress_is_valid_type(const char *str, size_t len);
e128f9c3	154
e2877c2a	155	int btrfs_compress_heuristic(struct btrfs_inode *inode, u64 start, u64 end);
c2fcdcdf	156
400b172b QW	157	int btrfs_compress_filemap_get_folio(struct address_space *mapping, u64 start,
400b172b QW	158	struct folio **in_folio_ret);
6de35954	159
400b172b QW	160	int zlib_compress_folios(struct list_head ws, struct address_space mapping,
400b172b QW	161	u64 start, struct folio *folios, unsigned long out_folios,
cb4c9198 DS	162	unsigned long total_in, unsigned long total_out);
cb4c9198 DS	163	int zlib_decompress_bio(struct list_head ws, struct compressed_bio cb);
3e09b5b2	164	int zlib_decompress(struct list_head ws, const u8 data_in,
54c78d49	165	struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
cb4c9198 DS	166	size_t destlen);
	167	struct list_head *zlib_alloc_workspace(unsigned int level);
	168	void zlib_free_workspace(struct list_head *ws);
	169	struct list_head *zlib_get_workspace(unsigned int level);
	170
400b172b QW	171	int lzo_compress_folios(struct list_head ws, struct address_space mapping,
400b172b QW	172	u64 start, struct folio *folios, unsigned long out_folios,
cb4c9198 DS	173	unsigned long total_in, unsigned long total_out);
cb4c9198 DS	174	int lzo_decompress_bio(struct list_head ws, struct compressed_bio cb);
3e09b5b2	175	int lzo_decompress(struct list_head ws, const u8 data_in,
9f9a4e43	176	struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
cb4c9198 DS	177	size_t destlen);
	178	struct list_head *lzo_alloc_workspace(unsigned int level);
	179	void lzo_free_workspace(struct list_head *ws);
	180
400b172b QW	181	int zstd_compress_folios(struct list_head ws, struct address_space mapping,
400b172b QW	182	u64 start, struct folio *folios, unsigned long out_folios,
cb4c9198 DS	183	unsigned long total_in, unsigned long total_out);
cb4c9198 DS	184	int zstd_decompress_bio(struct list_head ws, struct compressed_bio cb);
3e09b5b2	185	int zstd_decompress(struct list_head ws, const u8 data_in,
b70f3a45	186	struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
cb4c9198 DS	187	size_t destlen);
	188	void zstd_init_workspace_manager(void);
	189	void zstd_cleanup_workspace_manager(void);
	190	struct list_head *zstd_alloc_workspace(unsigned int level);
	191	void zstd_free_workspace(struct list_head *ws);
	192	struct list_head *zstd_get_workspace(unsigned int level);
	193	void zstd_put_workspace(struct list_head *ws);
	194
c8b97818	195	#endif