[linux-block.git] / fs / erofs / decompressor_lzma.c

// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/xz.h>
#include <linux/module.h>
#include "compress.h"

struct z_erofs_lzma {
	struct z_erofs_lzma *next;
	struct xz_dec_microlzma *state;
	struct xz_buf buf;
	u8 bounce[PAGE_SIZE];
};

/* considering the LZMA performance, no need to use a lockless list for now */
static DEFINE_SPINLOCK(z_erofs_lzma_lock);
static unsigned int z_erofs_lzma_max_dictsize;
static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
static struct z_erofs_lzma *z_erofs_lzma_head;
static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);

module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);

void z_erofs_lzma_exit(void)
{
	/* there should be no running fs instance */
	while (z_erofs_lzma_avail_strms) {
		struct z_erofs_lzma *strm;

		spin_lock(&z_erofs_lzma_lock);
		strm = z_erofs_lzma_head;
		if (!strm) {
			spin_unlock(&z_erofs_lzma_lock);
			DBG_BUGON(1);
			return;
		}
		z_erofs_lzma_head = NULL;
		spin_unlock(&z_erofs_lzma_lock);

		while (strm) {
			struct z_erofs_lzma *n = strm->next;

			if (strm->state)
				xz_dec_microlzma_end(strm->state);
			kfree(strm);
			--z_erofs_lzma_avail_strms;
			strm = n;
		}
	}
}

int z_erofs_lzma_init(void)
{
	unsigned int i;

	/* by default, use # of possible CPUs instead */
	if (!z_erofs_lzma_nstrms)
		z_erofs_lzma_nstrms = num_possible_cpus();

	for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
		struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);

		if (!strm) {
			z_erofs_lzma_exit();
			return -ENOMEM;
		}
		spin_lock(&z_erofs_lzma_lock);
		strm->next = z_erofs_lzma_head;
		z_erofs_lzma_head = strm;
		spin_unlock(&z_erofs_lzma_lock);
		++z_erofs_lzma_avail_strms;
	}
	return 0;
}

int z_erofs_load_lzma_config(struct super_block *sb,
			     struct erofs_super_block *dsb,
			     struct z_erofs_lzma_cfgs *lzma, int size)
{
	static DEFINE_MUTEX(lzma_resize_mutex);
	unsigned int dict_size, i;
	struct z_erofs_lzma *strm, *head = NULL;
	int err;

	if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
		erofs_err(sb, "invalid lzma cfgs, size=%u", size);
		return -EINVAL;
	}
	if (lzma->format) {
		erofs_err(sb, "unidentified lzma format %x, please check kernel version",
			  le16_to_cpu(lzma->format));
		return -EINVAL;
	}
	dict_size = le32_to_cpu(lzma->dict_size);
	if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
		erofs_err(sb, "unsupported lzma dictionary size %u",
			  dict_size);
		return -EINVAL;
	}

	erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");

	/* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
	mutex_lock(&lzma_resize_mutex);

	if (z_erofs_lzma_max_dictsize >= dict_size) {
		mutex_unlock(&lzma_resize_mutex);
		return 0;
	}

	/* 1. collect/isolate all streams for the following check */
	for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
		struct z_erofs_lzma *last;

again:
		spin_lock(&z_erofs_lzma_lock);
		strm = z_erofs_lzma_head;
		if (!strm) {
			spin_unlock(&z_erofs_lzma_lock);
			wait_event(z_erofs_lzma_wq,
				   READ_ONCE(z_erofs_lzma_head));
			goto again;
		}
		z_erofs_lzma_head = NULL;
		spin_unlock(&z_erofs_lzma_lock);

		for (last = strm; last->next; last = last->next)
			++i;
		last->next = head;
		head = strm;
	}

	err = 0;
	/* 2. walk each isolated stream and grow max dict_size if needed */
	for (strm = head; strm; strm = strm->next) {
		if (strm->state)
			xz_dec_microlzma_end(strm->state);
		strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
		if (!strm->state)
			err = -ENOMEM;
	}

	/* 3. push back all to the global list and update max dict_size */
	spin_lock(&z_erofs_lzma_lock);
	DBG_BUGON(z_erofs_lzma_head);
	z_erofs_lzma_head = head;
	spin_unlock(&z_erofs_lzma_lock);
	wake_up_all(&z_erofs_lzma_wq);

	z_erofs_lzma_max_dictsize = dict_size;
	mutex_unlock(&lzma_resize_mutex);
	return err;
}

int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
			    struct page **pagepool)
{
	const unsigned int nrpages_out =
		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	const unsigned int nrpages_in =
		PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
	unsigned int inlen, outlen, pageofs;
	struct z_erofs_lzma *strm;
	u8 *kin;
	bool bounced = false;
	int no, ni, j, err = 0;

	/* 1. get the exact LZMA compressed size */
	kin = kmap(*rq->in);
	err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
				   min_t(unsigned int, rq->inputsize,
					 EROFS_BLKSIZ - rq->pageofs_in));
	if (err) {
		kunmap(*rq->in);
		return err;
	}

	/* 2. get an available lzma context */
again:
	spin_lock(&z_erofs_lzma_lock);
	strm = z_erofs_lzma_head;
	if (!strm) {
		spin_unlock(&z_erofs_lzma_lock);
		wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
		goto again;
	}
	z_erofs_lzma_head = strm->next;
	spin_unlock(&z_erofs_lzma_lock);

	/* 3. multi-call decompress */
	inlen = rq->inputsize;
	outlen = rq->outputsize;
	xz_dec_microlzma_reset(strm->state, inlen, outlen,
			       !rq->partial_decoding);
	pageofs = rq->pageofs_out;
	strm->buf.in = kin + rq->pageofs_in;
	strm->buf.in_pos = 0;
	strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
	inlen -= strm->buf.in_size;
	strm->buf.out = NULL;
	strm->buf.out_pos = 0;
	strm->buf.out_size = 0;

	for (ni = 0, no = -1;;) {
		enum xz_ret xz_err;

		if (strm->buf.out_pos == strm->buf.out_size) {
			if (strm->buf.out) {
				kunmap(rq->out[no]);
				strm->buf.out = NULL;
			}

			if (++no >= nrpages_out || !outlen) {
				erofs_err(rq->sb, "decompressed buf out of bound");
				err = -EFSCORRUPTED;
				break;
			}
			strm->buf.out_pos = 0;
			strm->buf.out_size = min_t(u32, outlen,
						   PAGE_SIZE - pageofs);
			outlen -= strm->buf.out_size;
			if (rq->out[no])
				strm->buf.out = kmap(rq->out[no]) + pageofs;
			pageofs = 0;
		} else if (strm->buf.in_pos == strm->buf.in_size) {
			kunmap(rq->in[ni]);

			if (++ni >= nrpages_in || !inlen) {
				erofs_err(rq->sb, "compressed buf out of bound");
				err = -EFSCORRUPTED;
				break;
			}
			strm->buf.in_pos = 0;
			strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
			inlen -= strm->buf.in_size;
			kin = kmap(rq->in[ni]);
			strm->buf.in = kin;
			bounced = false;
		}

		/*
		 * Handle overlapping: Use bounced buffer if the compressed
		 * data is under processing; Otherwise, Use short-lived pages
		 * from the on-stack pagepool where pages share with the same
		 * request.
		 */
		if (!bounced && rq->out[no] == rq->in[ni]) {
			memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
			strm->buf.in = strm->bounce;
			bounced = true;
		}
		for (j = ni + 1; j < nrpages_in; ++j) {
			struct page *tmppage;

			if (rq->out[no] != rq->in[j])
				continue;

			DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
							rq->in[j]));
			tmppage = erofs_allocpage(pagepool,
						  GFP_KERNEL | __GFP_NOFAIL);
			set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
			copy_highpage(tmppage, rq->in[j]);
			rq->in[j] = tmppage;
		}
		xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
		DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
		DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);

		if (xz_err != XZ_OK) {
			if (xz_err == XZ_STREAM_END && !outlen)
				break;
			erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
				  xz_err, rq->inputsize, rq->outputsize);
			err = -EFSCORRUPTED;
			break;
		}
	}
	if (no < nrpages_out && strm->buf.out)
		kunmap(rq->in[no]);
	if (ni < nrpages_in)
		kunmap(rq->in[ni]);
	/* 4. push back LZMA stream context to the global list */
	spin_lock(&z_erofs_lzma_lock);
	strm->next = z_erofs_lzma_head;
	z_erofs_lzma_head = strm;
	spin_unlock(&z_erofs_lzma_lock);
	wake_up(&z_erofs_lzma_wq);
	return err;
}
Commit	Line	Data
622ceadd GX	1	// SPDX-License-Identifier: GPL-2.0-or-later
	2	#include <linux/xz.h>
	3	#include <linux/module.h>
	4	#include "compress.h"
	5
	6	struct z_erofs_lzma {
	7	struct z_erofs_lzma *next;
	8	struct xz_dec_microlzma *state;
	9	struct xz_buf buf;
	10	u8 bounce[PAGE_SIZE];
	11	};
	12
	13	/* considering the LZMA performance, no need to use a lockless list for now */
	14	static DEFINE_SPINLOCK(z_erofs_lzma_lock);
	15	static unsigned int z_erofs_lzma_max_dictsize;
	16	static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
	17	static struct z_erofs_lzma *z_erofs_lzma_head;
	18	static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
	19
	20	module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
	21
	22	void z_erofs_lzma_exit(void)
	23	{
	24	/* there should be no running fs instance */
	25	while (z_erofs_lzma_avail_strms) {
	26	struct z_erofs_lzma *strm;
	27
	28	spin_lock(&z_erofs_lzma_lock);
	29	strm = z_erofs_lzma_head;
	30	if (!strm) {
	31	spin_unlock(&z_erofs_lzma_lock);
	32	DBG_BUGON(1);
	33	return;
	34	}
	35	z_erofs_lzma_head = NULL;
	36	spin_unlock(&z_erofs_lzma_lock);
	37
	38	while (strm) {
	39	struct z_erofs_lzma *n = strm->next;
	40
	41	if (strm->state)
	42	xz_dec_microlzma_end(strm->state);
	43	kfree(strm);
	44	--z_erofs_lzma_avail_strms;
	45	strm = n;
	46	}
	47	}
	48	}
	49
	50	int z_erofs_lzma_init(void)
	51	{
	52	unsigned int i;
	53
	54	/* by default, use # of possible CPUs instead */
	55	if (!z_erofs_lzma_nstrms)
	56	z_erofs_lzma_nstrms = num_possible_cpus();
	57
	58	for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
	59	struct z_erofs_lzma strm = kzalloc(sizeof(strm), GFP_KERNEL);
	60
	61	if (!strm) {
	62	z_erofs_lzma_exit();
	63	return -ENOMEM;
	64	}
65	spin_lock(&z_erofs_lzma_lock);
66	strm->next = z_erofs_lzma_head;
67	z_erofs_lzma_head = strm;
68	spin_unlock(&z_erofs_lzma_lock);
69	++z_erofs_lzma_avail_strms;
70	}
71	return 0;
72	}
73
74	int z_erofs_load_lzma_config(struct super_block *sb,
75	struct erofs_super_block *dsb,
76	struct z_erofs_lzma_cfgs *lzma, int size)
77	{
78	static DEFINE_MUTEX(lzma_resize_mutex);
79	unsigned int dict_size, i;
80	struct z_erofs_lzma strm, head = NULL;
81	int err;
82
83	if (!lzma \|\| size < sizeof(struct z_erofs_lzma_cfgs)) {
84	erofs_err(sb, "invalid lzma cfgs, size=%u", size);
85	return -EINVAL;
86	}
87	if (lzma->format) {
88	erofs_err(sb, "unidentified lzma format %x, please check kernel version",
89	le16_to_cpu(lzma->format));
90	return -EINVAL;
91	}
92	dict_size = le32_to_cpu(lzma->dict_size);
93	if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE \|\| dict_size < 4096) {
94	erofs_err(sb, "unsupported lzma dictionary size %u",
95	dict_size);
96	return -EINVAL;
97	}
98
99	erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");
100
101	/* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
102	mutex_lock(&lzma_resize_mutex);
103
104	if (z_erofs_lzma_max_dictsize >= dict_size) {
105	mutex_unlock(&lzma_resize_mutex);
106	return 0;
107	}
108
109	/* 1. collect/isolate all streams for the following check */
110	for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
111	struct z_erofs_lzma *last;
112
113	again:
114	spin_lock(&z_erofs_lzma_lock);
115	strm = z_erofs_lzma_head;
116	if (!strm) {
117	spin_unlock(&z_erofs_lzma_lock);
118	wait_event(z_erofs_lzma_wq,
119	READ_ONCE(z_erofs_lzma_head));
120	goto again;
121	}
122	z_erofs_lzma_head = NULL;
123	spin_unlock(&z_erofs_lzma_lock);
124
125	for (last = strm; last->next; last = last->next)
126	++i;
127	last->next = head;
128	head = strm;
129	}
130
131	err = 0;
132	/* 2. walk each isolated stream and grow max dict_size if needed */
133	for (strm = head; strm; strm = strm->next) {
134	if (strm->state)
135	xz_dec_microlzma_end(strm->state);
136	strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
137	if (!strm->state)
138	err = -ENOMEM;
139	}
140
141	/* 3. push back all to the global list and update max dict_size */
142	spin_lock(&z_erofs_lzma_lock);
143	DBG_BUGON(z_erofs_lzma_head);
144	z_erofs_lzma_head = head;
145	spin_unlock(&z_erofs_lzma_lock);
2df7c4bd	146	wake_up_all(&z_erofs_lzma_wq);
622ceadd GX	147
	148	z_erofs_lzma_max_dictsize = dict_size;
	149	mutex_unlock(&lzma_resize_mutex);
	150	return err;
	151	}
	152
	153	int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
eaa9172a	154	struct page **pagepool)
622ceadd GX	155	{
	156	const unsigned int nrpages_out =
	157	PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	158	const unsigned int nrpages_in =
	159	PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
10e5f6e4	160	unsigned int inlen, outlen, pageofs;
622ceadd GX	161	struct z_erofs_lzma *strm;
	162	u8 *kin;
	163	bool bounced = false;
	164	int no, ni, j, err = 0;
	165
	166	/* 1. get the exact LZMA compressed size */
	167	kin = kmap(*rq->in);
10e5f6e4 GX	168	err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
	169	min_t(unsigned int, rq->inputsize,
	170	EROFS_BLKSIZ - rq->pageofs_in));
	171	if (err) {
622ceadd	172	kunmap(*rq->in);
10e5f6e4	173	return err;
622ceadd	174	}
622ceadd GX	175
	176	/* 2. get an available lzma context */
	177	again:
	178	spin_lock(&z_erofs_lzma_lock);
	179	strm = z_erofs_lzma_head;
	180	if (!strm) {
	181	spin_unlock(&z_erofs_lzma_lock);
	182	wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
	183	goto again;
	184	}
	185	z_erofs_lzma_head = strm->next;
	186	spin_unlock(&z_erofs_lzma_lock);
	187
	188	/* 3. multi-call decompress */
	189	inlen = rq->inputsize;
	190	outlen = rq->outputsize;
	191	xz_dec_microlzma_reset(strm->state, inlen, outlen,
	192	!rq->partial_decoding);
	193	pageofs = rq->pageofs_out;
10e5f6e4	194	strm->buf.in = kin + rq->pageofs_in;
622ceadd	195	strm->buf.in_pos = 0;
10e5f6e4	196	strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
622ceadd GX	197	inlen -= strm->buf.in_size;
	198	strm->buf.out = NULL;
	199	strm->buf.out_pos = 0;
	200	strm->buf.out_size = 0;
	201
	202	for (ni = 0, no = -1;;) {
	203	enum xz_ret xz_err;
	204
	205	if (strm->buf.out_pos == strm->buf.out_size) {
	206	if (strm->buf.out) {
	207	kunmap(rq->out[no]);
	208	strm->buf.out = NULL;
	209	}
	210
	211	if (++no >= nrpages_out \|\| !outlen) {
	212	erofs_err(rq->sb, "decompressed buf out of bound");
	213	err = -EFSCORRUPTED;
	214	break;
	215	}
	216	strm->buf.out_pos = 0;
	217	strm->buf.out_size = min_t(u32, outlen,
	218	PAGE_SIZE - pageofs);
	219	outlen -= strm->buf.out_size;
	220	if (rq->out[no])
	221	strm->buf.out = kmap(rq->out[no]) + pageofs;
	222	pageofs = 0;
	223	} else if (strm->buf.in_pos == strm->buf.in_size) {
	224	kunmap(rq->in[ni]);
	225
	226	if (++ni >= nrpages_in \|\| !inlen) {
	227	erofs_err(rq->sb, "compressed buf out of bound");
	228	err = -EFSCORRUPTED;
	229	break;
	230	}
	231	strm->buf.in_pos = 0;
	232	strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
	233	inlen -= strm->buf.in_size;
	234	kin = kmap(rq->in[ni]);
	235	strm->buf.in = kin;
	236	bounced = false;
	237	}
	238
	239	/*
	240	* Handle overlapping: Use bounced buffer if the compressed
	241	* data is under processing; Otherwise, Use short-lived pages
	242	* from the on-stack pagepool where pages share with the same
	243	* request.
	244	*/
	245	if (!bounced && rq->out[no] == rq->in[ni]) {
	246	memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
	247	strm->buf.in = strm->bounce;
	248	bounced = true;
	249	}
	250	for (j = ni + 1; j < nrpages_in; ++j) {
	251	struct page *tmppage;
	252
	253	if (rq->out[no] != rq->in[j])
	254	continue;
	255
	256	DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
	257	rq->in[j]));
	258	tmppage = erofs_allocpage(pagepool,
	259	GFP_KERNEL \| __GFP_NOFAIL);
	260	set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
261	copy_highpage(tmppage, rq->in[j]);
262	rq->in[j] = tmppage;
263	}
264	xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
265	DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
266	DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
267
268	if (xz_err != XZ_OK) {
269	if (xz_err == XZ_STREAM_END && !outlen)
270	break;
271	erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
272	xz_err, rq->inputsize, rq->outputsize);
273	err = -EFSCORRUPTED;
274	break;
275	}
276	}
277	if (no < nrpages_out && strm->buf.out)
278	kunmap(rq->in[no]);
279	if (ni < nrpages_in)
280	kunmap(rq->in[ni]);
281	/* 4. push back LZMA stream context to the global list */
282	spin_lock(&z_erofs_lzma_lock);
283	strm->next = z_erofs_lzma_head;
284	z_erofs_lzma_head = strm;
285	spin_unlock(&z_erofs_lzma_lock);
286	wake_up(&z_erofs_lzma_wq);
287	return err;
288	}