[linux-2.6-block.git] / fs / erofs / decompressor.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2019 HUAWEI, Inc.
 *             https://www.huawei.com/
 */
#include "compress.h"
#include <linux/lz4.h>

#ifndef LZ4_DISTANCE_MAX	/* history window size */
#define LZ4_DISTANCE_MAX 65535	/* set to maximum value by default */
#endif

#define LZ4_MAX_DISTANCE_PAGES	(DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1)
#ifndef LZ4_DECOMPRESS_INPLACE_MARGIN
#define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize)  (((srcsize) >> 8) + 32)
#endif

struct z_erofs_lz4_decompress_ctx {
	struct z_erofs_decompress_req *rq;
	/* # of encoded, decoded pages */
	unsigned int inpages, outpages;
	/* decoded block total length (used for in-place decompression) */
	unsigned int oend;
};

static int z_erofs_load_lz4_config(struct super_block *sb,
			    struct erofs_super_block *dsb, void *data, int size)
{
	struct erofs_sb_info *sbi = EROFS_SB(sb);
	struct z_erofs_lz4_cfgs *lz4 = data;
	u16 distance;

	if (lz4) {
		if (size < sizeof(struct z_erofs_lz4_cfgs)) {
			erofs_err(sb, "invalid lz4 cfgs, size=%u", size);
			return -EINVAL;
		}
		distance = le16_to_cpu(lz4->max_distance);

		sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks);
		if (!sbi->lz4.max_pclusterblks) {
			sbi->lz4.max_pclusterblks = 1;	/* reserved case */
		} else if (sbi->lz4.max_pclusterblks >
			   erofs_blknr(sb, Z_EROFS_PCLUSTER_MAX_SIZE)) {
			erofs_err(sb, "too large lz4 pclusterblks %u",
				  sbi->lz4.max_pclusterblks);
			return -EINVAL;
		}
	} else {
		distance = le16_to_cpu(dsb->u1.lz4_max_distance);
		sbi->lz4.max_pclusterblks = 1;
	}

	sbi->lz4.max_distance_pages = distance ?
					DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
					LZ4_MAX_DISTANCE_PAGES;
	return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks);
}

/*
 * Fill all gaps with bounce pages if it's a sparse page list. Also check if
 * all physical pages are consecutive, which can be seen for moderate CR.
 */
static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
					struct page **pagepool)
{
	struct z_erofs_decompress_req *rq = ctx->rq;
	struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
	unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
					   BITS_PER_LONG)] = { 0 };
	unsigned int lz4_max_distance_pages =
				EROFS_SB(rq->sb)->lz4.max_distance_pages;
	void *kaddr = NULL;
	unsigned int i, j, top;

	top = 0;
	for (i = j = 0; i < ctx->outpages; ++i, ++j) {
		struct page *const page = rq->out[i];
		struct page *victim;

		if (j >= lz4_max_distance_pages)
			j = 0;

		/* 'valid' bounced can only be tested after a complete round */
		if (!rq->fillgaps && test_bit(j, bounced)) {
			DBG_BUGON(i < lz4_max_distance_pages);
			DBG_BUGON(top >= lz4_max_distance_pages);
			availables[top++] = rq->out[i - lz4_max_distance_pages];
		}

		if (page) {
			__clear_bit(j, bounced);
			if (!PageHighMem(page)) {
				if (!i) {
					kaddr = page_address(page);
					continue;
				}
				if (kaddr &&
				    kaddr + PAGE_SIZE == page_address(page)) {
					kaddr += PAGE_SIZE;
					continue;
				}
			}
			kaddr = NULL;
			continue;
		}
		kaddr = NULL;
		__set_bit(j, bounced);

		if (top) {
			victim = availables[--top];
			get_page(victim);
		} else {
			victim = erofs_allocpage(pagepool, rq->gfp);
			if (!victim)
				return -ENOMEM;
			set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
		}
		rq->out[i] = victim;
	}
	return kaddr ? 1 : 0;
}

static void *z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx *ctx,
			void *inpage, void *out, unsigned int *inputmargin,
			int *maptype, bool may_inplace)
{
	struct z_erofs_decompress_req *rq = ctx->rq;
	unsigned int omargin, total, i;
	struct page **in;
	void *src, *tmp;

	if (rq->inplace_io) {
		omargin = PAGE_ALIGN(ctx->oend) - ctx->oend;
		if (rq->partial_decoding || !may_inplace ||
		    omargin < LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize))
			goto docopy;

		for (i = 0; i < ctx->inpages; ++i)
			if (rq->out[ctx->outpages - ctx->inpages + i] !=
			    rq->in[i])
				goto docopy;
		kunmap_local(inpage);
		*maptype = 3;
		return out + ((ctx->outpages - ctx->inpages) << PAGE_SHIFT);
	}

	if (ctx->inpages <= 1) {
		*maptype = 0;
		return inpage;
	}
	kunmap_local(inpage);
	src = erofs_vm_map_ram(rq->in, ctx->inpages);
	if (!src)
		return ERR_PTR(-ENOMEM);
	*maptype = 1;
	return src;

docopy:
	/* Or copy compressed data which can be overlapped to per-CPU buffer */
	in = rq->in;
	src = erofs_get_pcpubuf(ctx->inpages);
	if (!src) {
		DBG_BUGON(1);
		kunmap_local(inpage);
		return ERR_PTR(-EFAULT);
	}

	tmp = src;
	total = rq->inputsize;
	while (total) {
		unsigned int page_copycnt =
			min_t(unsigned int, total, PAGE_SIZE - *inputmargin);

		if (!inpage)
			inpage = kmap_local_page(*in);
		memcpy(tmp, inpage + *inputmargin, page_copycnt);
		kunmap_local(inpage);
		inpage = NULL;
		tmp += page_copycnt;
		total -= page_copycnt;
		++in;
		*inputmargin = 0;
	}
	*maptype = 2;
	return src;
}

/*
 * Get the exact inputsize with zero_padding feature.
 *  - For LZ4, it should work if zero_padding feature is on (5.3+);
 *  - For MicroLZMA, it'd be enabled all the time.
 */
int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf,
			 unsigned int padbufsize)
{
	const char *padend;

	padend = memchr_inv(padbuf, 0, padbufsize);
	if (!padend)
		return -EFSCORRUPTED;
	rq->inputsize -= padend - padbuf;
	rq->pageofs_in += padend - padbuf;
	return 0;
}

static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
				      u8 *dst)
{
	struct z_erofs_decompress_req *rq = ctx->rq;
	bool support_0padding = false, may_inplace = false;
	unsigned int inputmargin;
	u8 *out, *headpage, *src;
	int ret, maptype;

	DBG_BUGON(*rq->in == NULL);
	headpage = kmap_local_page(*rq->in);

	/* LZ4 decompression inplace is only safe if zero_padding is enabled */
	if (erofs_sb_has_zero_padding(EROFS_SB(rq->sb))) {
		support_0padding = true;
		ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
				min_t(unsigned int, rq->inputsize,
				      rq->sb->s_blocksize - rq->pageofs_in));
		if (ret) {
			kunmap_local(headpage);
			return ret;
		}
		may_inplace = !((rq->pageofs_in + rq->inputsize) &
				(rq->sb->s_blocksize - 1));
	}

	inputmargin = rq->pageofs_in;
	src = z_erofs_lz4_handle_overlap(ctx, headpage, dst, &inputmargin,
					 &maptype, may_inplace);
	if (IS_ERR(src))
		return PTR_ERR(src);

	out = dst + rq->pageofs_out;
	/* legacy format could compress extra data in a pcluster. */
	if (rq->partial_decoding || !support_0padding)
		ret = LZ4_decompress_safe_partial(src + inputmargin, out,
				rq->inputsize, rq->outputsize, rq->outputsize);
	else
		ret = LZ4_decompress_safe(src + inputmargin, out,
					  rq->inputsize, rq->outputsize);

	if (ret != rq->outputsize) {
		erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
			  ret, rq->inputsize, inputmargin, rq->outputsize);
		if (ret >= 0)
			memset(out + ret, 0, rq->outputsize - ret);
		ret = -EFSCORRUPTED;
	} else {
		ret = 0;
	}

	if (maptype == 0) {
		kunmap_local(headpage);
	} else if (maptype == 1) {
		vm_unmap_ram(src, ctx->inpages);
	} else if (maptype == 2) {
		erofs_put_pcpubuf(src);
	} else if (maptype != 3) {
		DBG_BUGON(1);
		return -EFAULT;
	}
	return ret;
}

static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
				  struct page **pagepool)
{
	struct z_erofs_lz4_decompress_ctx ctx;
	unsigned int dst_maptype;
	void *dst;
	int ret;

	ctx.rq = rq;
	ctx.oend = rq->pageofs_out + rq->outputsize;
	ctx.outpages = PAGE_ALIGN(ctx.oend) >> PAGE_SHIFT;
	ctx.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;

	/* one optimized fast path only for non bigpcluster cases yet */
	if (ctx.inpages == 1 && ctx.outpages == 1 && !rq->inplace_io) {
		DBG_BUGON(!*rq->out);
		dst = kmap_local_page(*rq->out);
		dst_maptype = 0;
		goto dstmap_out;
	}

	/* general decoding path which can be used for all cases */
	ret = z_erofs_lz4_prepare_dstpages(&ctx, pagepool);
	if (ret < 0) {
		return ret;
	} else if (ret > 0) {
		dst = page_address(*rq->out);
		dst_maptype = 1;
	} else {
		dst = erofs_vm_map_ram(rq->out, ctx.outpages);
		if (!dst)
			return -ENOMEM;
		dst_maptype = 2;
	}

dstmap_out:
	ret = z_erofs_lz4_decompress_mem(&ctx, dst);
	if (!dst_maptype)
		kunmap_local(dst);
	else if (dst_maptype == 2)
		vm_unmap_ram(dst, ctx.outpages);
	return ret;
}

static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq,
				   struct page **pagepool)
{
	const unsigned int nrpages_in =
		PAGE_ALIGN(rq->pageofs_in + rq->inputsize) >> PAGE_SHIFT;
	const unsigned int nrpages_out =
		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	const unsigned int bs = rq->sb->s_blocksize;
	unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt;
	u8 *kin;

	DBG_BUGON(rq->outputsize > rq->inputsize);
	if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) {
		cur = bs - (rq->pageofs_out & (bs - 1));
		pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK;
		cur = min(cur, rq->outputsize);
		if (cur && rq->out[0]) {
			kin = kmap_local_page(rq->in[nrpages_in - 1]);
			if (rq->out[0] == rq->in[nrpages_in - 1]) {
				memmove(kin + rq->pageofs_out, kin + pi, cur);
				flush_dcache_page(rq->out[0]);
			} else {
				memcpy_to_page(rq->out[0], rq->pageofs_out,
					       kin + pi, cur);
			}
			kunmap_local(kin);
		}
		rq->outputsize -= cur;
	}

	for (; rq->outputsize; rq->pageofs_in = 0, cur += PAGE_SIZE, ni++) {
		insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize);
		rq->outputsize -= insz;
		if (!rq->in[ni])
			continue;
		kin = kmap_local_page(rq->in[ni]);
		pi = 0;
		do {
			no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT;
			po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK;
			DBG_BUGON(no >= nrpages_out);
			cnt = min(insz - pi, PAGE_SIZE - po);
			if (rq->out[no] == rq->in[ni]) {
				memmove(kin + po,
					kin + rq->pageofs_in + pi, cnt);
				flush_dcache_page(rq->out[no]);
			} else if (rq->out[no]) {
				memcpy_to_page(rq->out[no], po,
					       kin + rq->pageofs_in + pi, cnt);
			}
			pi += cnt;
		} while (pi < insz);
		kunmap_local(kin);
	}
	DBG_BUGON(ni > nrpages_in);
	return 0;
}

const struct z_erofs_decompressor erofs_decompressors[] = {
	[Z_EROFS_COMPRESSION_SHIFTED] = {
		.decompress = z_erofs_transform_plain,
		.name = "shifted"
	},
	[Z_EROFS_COMPRESSION_INTERLACED] = {
		.decompress = z_erofs_transform_plain,
		.name = "interlaced"
	},
	[Z_EROFS_COMPRESSION_LZ4] = {
		.config = z_erofs_load_lz4_config,
		.decompress = z_erofs_lz4_decompress,
		.name = "lz4"
	},
#ifdef CONFIG_EROFS_FS_ZIP_LZMA
	[Z_EROFS_COMPRESSION_LZMA] = {
		.config = z_erofs_load_lzma_config,
		.decompress = z_erofs_lzma_decompress,
		.name = "lzma"
	},
#endif
#ifdef CONFIG_EROFS_FS_ZIP_DEFLATE
	[Z_EROFS_COMPRESSION_DEFLATE] = {
		.config = z_erofs_load_deflate_config,
		.decompress = z_erofs_deflate_decompress,
		.name = "deflate"
	},
#endif
};

int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb)
{
	struct erofs_sb_info *sbi = EROFS_SB(sb);
	struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
	unsigned int algs, alg;
	erofs_off_t offset;
	int size, ret = 0;

	if (!erofs_sb_has_compr_cfgs(sbi)) {
		sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4;
		return z_erofs_load_lz4_config(sb, dsb, NULL, 0);
	}

	sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
	if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) {
		erofs_err(sb, "unidentified algorithms %x, please upgrade kernel",
			  sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS);
		return -EOPNOTSUPP;
	}

	erofs_init_metabuf(&buf, sb);
	offset = EROFS_SUPER_OFFSET + sbi->sb_size;
	alg = 0;
	for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) {
		void *data;

		if (!(algs & 1))
			continue;

		data = erofs_read_metadata(sb, &buf, &offset, &size);
		if (IS_ERR(data)) {
			ret = PTR_ERR(data);
			break;
		}

		if (alg >= ARRAY_SIZE(erofs_decompressors) ||
		    !erofs_decompressors[alg].config) {
			erofs_err(sb, "algorithm %d isn't enabled on this kernel",
				  alg);
			ret = -EOPNOTSUPP;
		} else {
			ret = erofs_decompressors[alg].config(sb,
					dsb, data, size);
		}

		kfree(data);
		if (ret)
			break;
	}
	erofs_put_metabuf(&buf);
	return ret;
}
Commit	Line	Data
29b24f6c	1	// SPDX-License-Identifier: GPL-2.0-only
7fc45dbc	2	/*
7fc45dbc	3	* Copyright (C) 2019 HUAWEI, Inc.
592e7cd0	4	* https://www.huawei.com/
7fc45dbc GX	5	*/
	6	#include "compress.h"
	7	#include <linux/lz4.h>
	8
	9	#ifndef LZ4_DISTANCE_MAX /* history window size */
	10	#define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
	11	#endif
	12
af89bcef	13	#define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1)
0ffd71bc GX	14	#ifndef LZ4_DECOMPRESS_INPLACE_MARGIN
	15	#define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize) (((srcsize) >> 8) + 32)
	16	#endif
7fc45dbc	17
d67aee76 GX	18	struct z_erofs_lz4_decompress_ctx {
	19	struct z_erofs_decompress_req *rq;
	20	/* # of encoded, decoded pages */
	21	unsigned int inpages, outpages;
	22	/* decoded block total length (used for in-place decompression) */
	23	unsigned int oend;
	24	};
	25
efb4fb02 GX	26	static int z_erofs_load_lz4_config(struct super_block *sb,
efb4fb02 GX	27	struct erofs_super_block dsb, void data, int size)
5d50538f	28	{
4fea63f7	29	struct erofs_sb_info *sbi = EROFS_SB(sb);
efb4fb02	30	struct z_erofs_lz4_cfgs *lz4 = data;
46249cde GX	31	u16 distance;
	32
	33	if (lz4) {
	34	if (size < sizeof(struct z_erofs_lz4_cfgs)) {
	35	erofs_err(sb, "invalid lz4 cfgs, size=%u", size);
	36	return -EINVAL;
	37	}
	38	distance = le16_to_cpu(lz4->max_distance);
4fea63f7 GX	39
	40	sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks);
	41	if (!sbi->lz4.max_pclusterblks) {
	42	sbi->lz4.max_pclusterblks = 1; /* reserved case */
	43	} else if (sbi->lz4.max_pclusterblks >
3acea5fc	44	erofs_blknr(sb, Z_EROFS_PCLUSTER_MAX_SIZE)) {
4fea63f7 GX	45	erofs_err(sb, "too large lz4 pclusterblks %u",
	46	sbi->lz4.max_pclusterblks);
	47	return -EINVAL;
4fea63f7	48	}
46249cde	49	} else {
14373711	50	distance = le16_to_cpu(dsb->u1.lz4_max_distance);
4fea63f7	51	sbi->lz4.max_pclusterblks = 1;
46249cde	52	}
5d50538f	53
4fea63f7	54	sbi->lz4.max_distance_pages = distance ?
5d50538f HJ	55	DIV_ROUND_UP(distance, PAGE_SIZE) + 1 :
5d50538f HJ	56	LZ4_MAX_DISTANCE_PAGES;
4fea63f7	57	return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks);
5d50538f HJ	58	}
5d50538f HJ	59
966edfb0 GX	60	/*
	61	* Fill all gaps with bounce pages if it's a sparse page list. Also check if
	62	* all physical pages are consecutive, which can be seen for moderate CR.
	63	*/
d67aee76	64	static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx,
eaa9172a	65	struct page **pagepool)
7fc45dbc	66	{
d67aee76	67	struct z_erofs_decompress_req *rq = ctx->rq;
7fc45dbc	68	struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
af89bcef GX	69	unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
af89bcef GX	70	BITS_PER_LONG)] = { 0 };
5d50538f HJ	71	unsigned int lz4_max_distance_pages =
5d50538f HJ	72	EROFS_SB(rq->sb)->lz4.max_distance_pages;
7fc45dbc	73	void *kaddr = NULL;
af89bcef	74	unsigned int i, j, top;
7fc45dbc	75
af89bcef	76	top = 0;
d67aee76	77	for (i = j = 0; i < ctx->outpages; ++i, ++j) {
7fc45dbc	78	struct page *const page = rq->out[i];
af89bcef	79	struct page *victim;
7fc45dbc	80
5d50538f	81	if (j >= lz4_max_distance_pages)
af89bcef GX	82	j = 0;
	83
	84	/* 'valid' bounced can only be tested after a complete round */
267f2492	85	if (!rq->fillgaps && test_bit(j, bounced)) {
5d50538f HJ	86	DBG_BUGON(i < lz4_max_distance_pages);
	87	DBG_BUGON(top >= lz4_max_distance_pages);
	88	availables[top++] = rq->out[i - lz4_max_distance_pages];
af89bcef	89	}
7fc45dbc GX	90
7fc45dbc GX	91	if (page) {
af89bcef	92	__clear_bit(j, bounced);
448b5a15 GX	93	if (!PageHighMem(page)) {
	94	if (!i) {
	95	kaddr = page_address(page);
	96	continue;
	97	}
	98	if (kaddr &&
	99	kaddr + PAGE_SIZE == page_address(page)) {
7fc45dbc	100	kaddr += PAGE_SIZE;
448b5a15 GX	101	continue;
448b5a15 GX	102	}
7fc45dbc	103	}
448b5a15	104	kaddr = NULL;
7fc45dbc GX	105	continue;
	106	}
	107	kaddr = NULL;
af89bcef	108	__set_bit(j, bounced);
7fc45dbc	109
af89bcef GX	110	if (top) {
	111	victim = availables[--top];
	112	get_page(victim);
7fc45dbc	113	} else {
d9281660 CG	114	victim = erofs_allocpage(pagepool, rq->gfp);
	115	if (!victim)
	116	return -ENOMEM;
6aaa7b06	117	set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE);
7fc45dbc	118	}
af89bcef	119	rq->out[i] = victim;
7fc45dbc GX	120	}
	121	return kaddr ? 1 : 0;
	122	}
	123
d67aee76	124	static void z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx ctx,
3c12466b GX	125	void inpage, void out, unsigned int *inputmargin,
3c12466b GX	126	int *maptype, bool may_inplace)
7fc45dbc	127	{
d67aee76	128	struct z_erofs_decompress_req *rq = ctx->rq;
3c12466b	129	unsigned int omargin, total, i;
598162d0 GX	130	struct page **in;
	131	void src, tmp;
	132
598162d0	133	if (rq->inplace_io) {
d67aee76	134	omargin = PAGE_ALIGN(ctx->oend) - ctx->oend;
ab749bad	135	if (rq->partial_decoding \|\| !may_inplace \|\|
d67aee76	136	omargin < LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize))
598162d0 GX	137	goto docopy;
598162d0 GX	138
3c12466b GX	139	for (i = 0; i < ctx->inpages; ++i)
	140	if (rq->out[ctx->outpages - ctx->inpages + i] !=
	141	rq->in[i])
	142	goto docopy;
	143	kunmap_local(inpage);
	144	*maptype = 3;
	145	return out + ((ctx->outpages - ctx->inpages) << PAGE_SHIFT);
598162d0 GX	146	}
598162d0 GX	147
d67aee76	148	if (ctx->inpages <= 1) {
598162d0 GX	149	*maptype = 0;
	150	return inpage;
	151	}
123ec246	152	kunmap_local(inpage);
d67aee76	153	src = erofs_vm_map_ram(rq->in, ctx->inpages);
598162d0 GX	154	if (!src)
	155	return ERR_PTR(-ENOMEM);
	156	*maptype = 1;
	157	return src;
	158
	159	docopy:
	160	/* Or copy compressed data which can be overlapped to per-CPU buffer */
	161	in = rq->in;
d67aee76	162	src = erofs_get_pcpubuf(ctx->inpages);
598162d0 GX	163	if (!src) {
598162d0 GX	164	DBG_BUGON(1);
123ec246	165	kunmap_local(inpage);
598162d0 GX	166	return ERR_PTR(-EFAULT);
	167	}
	168
	169	tmp = src;
	170	total = rq->inputsize;
	171	while (total) {
	172	unsigned int page_copycnt =
	173	min_t(unsigned int, total, PAGE_SIZE - *inputmargin);
	174
	175	if (!inpage)
123ec246	176	inpage = kmap_local_page(*in);
598162d0	177	memcpy(tmp, inpage + *inputmargin, page_copycnt);
123ec246	178	kunmap_local(inpage);
598162d0 GX	179	inpage = NULL;
	180	tmp += page_copycnt;
	181	total -= page_copycnt;
7fc45dbc	182	++in;
598162d0	183	*inputmargin = 0;
7fc45dbc	184	}
598162d0 GX	185	*maptype = 2;
598162d0 GX	186	return src;
7fc45dbc GX	187	}
7fc45dbc GX	188
10e5f6e4 GX	189	/*
	190	* Get the exact inputsize with zero_padding feature.
	191	* - For LZ4, it should work if zero_padding feature is on (5.3+);
	192	* - For MicroLZMA, it'd be enabled all the time.
	193	*/
	194	int z_erofs_fixup_insize(struct z_erofs_decompress_req rq, const char padbuf,
	195	unsigned int padbufsize)
	196	{
	197	const char *padend;
	198
	199	padend = memchr_inv(padbuf, 0, padbufsize);
	200	if (!padend)
	201	return -EFSCORRUPTED;
	202	rq->inputsize -= padend - padbuf;
	203	rq->pageofs_in += padend - padbuf;
	204	return 0;
	205	}
	206
d67aee76	207	static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx,
3c12466b	208	u8 *dst)
7fc45dbc	209	{
d67aee76	210	struct z_erofs_decompress_req *rq = ctx->rq;
ab749bad	211	bool support_0padding = false, may_inplace = false;
598162d0	212	unsigned int inputmargin;
3c12466b	213	u8 out, headpage, *src;
598162d0	214	int ret, maptype;
7fc45dbc	215
598162d0	216	DBG_BUGON(*rq->in == NULL);
123ec246	217	headpage = kmap_local_page(*rq->in);
0ffd71bc	218
10e5f6e4	219	/* LZ4 decompression inplace is only safe if zero_padding is enabled */
7e508f2c	220	if (erofs_sb_has_zero_padding(EROFS_SB(rq->sb))) {
0ffd71bc	221	support_0padding = true;
10e5f6e4 GX	222	ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in,
10e5f6e4 GX	223	min_t(unsigned int, rq->inputsize,
3acea5fc	224	rq->sb->s_blocksize - rq->pageofs_in));
10e5f6e4	225	if (ret) {
123ec246	226	kunmap_local(headpage);
10e5f6e4	227	return ret;
0ffd71bc	228	}
ab749bad	229	may_inplace = !((rq->pageofs_in + rq->inputsize) &
3acea5fc	230	(rq->sb->s_blocksize - 1));
0ffd71bc	231	}
7fc45dbc	232
10e5f6e4	233	inputmargin = rq->pageofs_in;
3c12466b	234	src = z_erofs_lz4_handle_overlap(ctx, headpage, dst, &inputmargin,
ab749bad	235	&maptype, may_inplace);
598162d0 GX	236	if (IS_ERR(src))
598162d0 GX	237	return PTR_ERR(src);
7fc45dbc	238
3c12466b	239	out = dst + rq->pageofs_out;
af1038ab GX	240	/* legacy format could compress extra data in a pcluster. */
	241	if (rq->partial_decoding \|\| !support_0padding)
	242	ret = LZ4_decompress_safe_partial(src + inputmargin, out,
598162d0	243	rq->inputsize, rq->outputsize, rq->outputsize);
af1038ab GX	244	else
af1038ab GX	245	ret = LZ4_decompress_safe(src + inputmargin, out,
598162d0	246	rq->inputsize, rq->outputsize);
af1038ab	247
aa99a76b GX	248	if (ret != rq->outputsize) {
aa99a76b GX	249	erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",
598162d0	250	ret, rq->inputsize, inputmargin, rq->outputsize);
aa99a76b GX	251	if (ret >= 0)
aa99a76b GX	252	memset(out + ret, 0, rq->outputsize - ret);
496530c7	253	ret = -EFSCORRUPTED;
5b6e7e12 YH	254	} else {
5b6e7e12 YH	255	ret = 0;
7fc45dbc GX	256	}
7fc45dbc GX	257
598162d0	258	if (maptype == 0) {
123ec246	259	kunmap_local(headpage);
598162d0	260	} else if (maptype == 1) {
d67aee76	261	vm_unmap_ram(src, ctx->inpages);
598162d0 GX	262	} else if (maptype == 2) {
598162d0 GX	263	erofs_put_pcpubuf(src);
3c12466b	264	} else if (maptype != 3) {
598162d0 GX	265	DBG_BUGON(1);
	266	return -EFAULT;
	267	}
7fc45dbc GX	268	return ret;
	269	}
	270
966edfb0	271	static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq,
eaa9172a	272	struct page **pagepool)
7fc45dbc	273	{
d67aee76	274	struct z_erofs_lz4_decompress_ctx ctx;
7fc45dbc GX	275	unsigned int dst_maptype;
7fc45dbc GX	276	void *dst;
598162d0	277	int ret;
7fc45dbc	278
d67aee76 GX	279	ctx.rq = rq;
	280	ctx.oend = rq->pageofs_out + rq->outputsize;
	281	ctx.outpages = PAGE_ALIGN(ctx.oend) >> PAGE_SHIFT;
	282	ctx.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
	283
5b6e7e12	284	/* one optimized fast path only for non bigpcluster cases yet */
d67aee76	285	if (ctx.inpages == 1 && ctx.outpages == 1 && !rq->inplace_io) {
5b6e7e12	286	DBG_BUGON(!*rq->out);
123ec246	287	dst = kmap_local_page(*rq->out);
5b6e7e12 YH	288	dst_maptype = 0;
5b6e7e12 YH	289	goto dstmap_out;
7fc45dbc GX	290	}
7fc45dbc GX	291
598162d0	292	/* general decoding path which can be used for all cases */
d67aee76 GX	293	ret = z_erofs_lz4_prepare_dstpages(&ctx, pagepool);
d67aee76 GX	294	if (ret < 0) {
7fc45dbc	295	return ret;
d67aee76	296	} else if (ret > 0) {
7fc45dbc GX	297	dst = page_address(*rq->out);
7fc45dbc GX	298	dst_maptype = 1;
d67aee76 GX	299	} else {
	300	dst = erofs_vm_map_ram(rq->out, ctx.outpages);
	301	if (!dst)
	302	return -ENOMEM;
	303	dst_maptype = 2;
7fc45dbc GX	304	}
7fc45dbc GX	305
7fc45dbc	306	dstmap_out:
3c12466b	307	ret = z_erofs_lz4_decompress_mem(&ctx, dst);
7fc45dbc	308	if (!dst_maptype)
123ec246	309	kunmap_local(dst);
7fc45dbc	310	else if (dst_maptype == 2)
d67aee76	311	vm_unmap_ram(dst, ctx.outpages);
7fc45dbc GX	312	return ret;
	313	}
	314
fdffc091 YH	315	static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq,
fdffc091 YH	316	struct page **pagepool)
7fc45dbc	317	{
1ca01520 GX	318	const unsigned int nrpages_in =
	319	PAGE_ALIGN(rq->pageofs_in + rq->inputsize) >> PAGE_SHIFT;
	320	const unsigned int nrpages_out =
7fc45dbc	321	PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
1ca01520 GX	322	const unsigned int bs = rq->sb->s_blocksize;
	323	unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt;
	324	u8 *kin;
	325
	326	DBG_BUGON(rq->outputsize > rq->inputsize);
	327	if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) {
	328	cur = bs - (rq->pageofs_out & (bs - 1));
	329	pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK;
	330	cur = min(cur, rq->outputsize);
	331	if (cur && rq->out[0]) {
	332	kin = kmap_local_page(rq->in[nrpages_in - 1]);
	333	if (rq->out[0] == rq->in[nrpages_in - 1]) {
	334	memmove(kin + rq->pageofs_out, kin + pi, cur);
	335	flush_dcache_page(rq->out[0]);
	336	} else {
	337	memcpy_to_page(rq->out[0], rq->pageofs_out,
	338	kin + pi, cur);
	339	}
	340	kunmap_local(kin);
	341	}
	342	rq->outputsize -= cur;
7fc45dbc GX	343	}
7fc45dbc GX	344
1ca01520 GX	345	for (; rq->outputsize; rq->pageofs_in = 0, cur += PAGE_SIZE, ni++) {
	346	insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize);
	347	rq->outputsize -= insz;
	348	if (!rq->in[ni])
	349	continue;
	350	kin = kmap_local_page(rq->in[ni]);
	351	pi = 0;
	352	do {
	353	no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT;
	354	po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK;
	355	DBG_BUGON(no >= nrpages_out);
	356	cnt = min(insz - pi, PAGE_SIZE - po);
	357	if (rq->out[no] == rq->in[ni]) {
	358	memmove(kin + po,
	359	kin + rq->pageofs_in + pi, cnt);
	360	flush_dcache_page(rq->out[no]);
	361	} else if (rq->out[no]) {
	362	memcpy_to_page(rq->out[no], po,
	363	kin + rq->pageofs_in + pi, cnt);
	364	}
	365	pi += cnt;
	366	} while (pi < insz);
	367	kunmap_local(kin);
7fc45dbc	368	}
1ca01520	369	DBG_BUGON(ni > nrpages_in);
7fc45dbc GX	370	return 0;
	371	}
	372
597e2953	373	const struct z_erofs_decompressor erofs_decompressors[] = {
966edfb0	374	[Z_EROFS_COMPRESSION_SHIFTED] = {
fdffc091	375	.decompress = z_erofs_transform_plain,
966edfb0 GX	376	.name = "shifted"
966edfb0 GX	377	},
fdffc091 YH	378	[Z_EROFS_COMPRESSION_INTERLACED] = {
	379	.decompress = z_erofs_transform_plain,
	380	.name = "interlaced"
	381	},
966edfb0	382	[Z_EROFS_COMPRESSION_LZ4] = {
efb4fb02	383	.config = z_erofs_load_lz4_config,
966edfb0 GX	384	.decompress = z_erofs_lz4_decompress,
	385	.name = "lz4"
	386	},
622ceadd GX	387	#ifdef CONFIG_EROFS_FS_ZIP_LZMA
622ceadd GX	388	[Z_EROFS_COMPRESSION_LZMA] = {
efb4fb02	389	.config = z_erofs_load_lzma_config,
622ceadd GX	390	.decompress = z_erofs_lzma_decompress,
	391	.name = "lzma"
	392	},
	393	#endif
ffa09b3b GX	394	#ifdef CONFIG_EROFS_FS_ZIP_DEFLATE
ffa09b3b GX	395	[Z_EROFS_COMPRESSION_DEFLATE] = {
efb4fb02	396	.config = z_erofs_load_deflate_config,
ffa09b3b GX	397	.decompress = z_erofs_deflate_decompress,
	398	.name = "deflate"
	399	},
	400	#endif
966edfb0	401	};
efb4fb02 GX	402
	403	int z_erofs_parse_cfgs(struct super_block sb, struct erofs_super_block dsb)
	404	{
	405	struct erofs_sb_info *sbi = EROFS_SB(sb);
	406	struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
	407	unsigned int algs, alg;
	408	erofs_off_t offset;
	409	int size, ret = 0;
	410
	411	if (!erofs_sb_has_compr_cfgs(sbi)) {
118a8cf5	412	sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4;
efb4fb02 GX	413	return z_erofs_load_lz4_config(sb, dsb, NULL, 0);
	414	}
	415
	416	sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs);
	417	if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) {
	418	erofs_err(sb, "unidentified algorithms %x, please upgrade kernel",
	419	sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS);
	420	return -EOPNOTSUPP;
	421	}
	422
	423	erofs_init_metabuf(&buf, sb);
	424	offset = EROFS_SUPER_OFFSET + sbi->sb_size;
	425	alg = 0;
	426	for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) {
	427	void *data;
	428
	429	if (!(algs & 1))
	430	continue;
	431
	432	data = erofs_read_metadata(sb, &buf, &offset, &size);
	433	if (IS_ERR(data)) {
	434	ret = PTR_ERR(data);
	435	break;
	436	}
	437
	438	if (alg >= ARRAY_SIZE(erofs_decompressors) \|\|
	439	!erofs_decompressors[alg].config) {
	440	erofs_err(sb, "algorithm %d isn't enabled on this kernel",
	441	alg);
	442	ret = -EOPNOTSUPP;
	443	} else {
	444	ret = erofs_decompressors[alg].config(sb,
	445	dsb, data, size);
	446	}
	447
	448	kfree(data);
	449	if (ret)
	450	break;
	451	}
	452	erofs_put_metabuf(&buf);
	453	return ret;
	454	}