[linux-block.git] / fs / erofs / decompressor.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2019 HUAWEI, Inc.
 *             http://www.huawei.com/
 * Created by Gao Xiang <gaoxiang25@huawei.com>
 */
#include "compress.h"
#include <linux/module.h>
#include <linux/lz4.h>

#ifndef LZ4_DISTANCE_MAX	/* history window size */
#define LZ4_DISTANCE_MAX 65535	/* set to maximum value by default */
#endif

#define LZ4_MAX_DISTANCE_PAGES	(DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1)
#ifndef LZ4_DECOMPRESS_INPLACE_MARGIN
#define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize)  (((srcsize) >> 8) + 32)
#endif

struct z_erofs_decompressor {
	/*
	 * if destpages have sparsed pages, fill them with bounce pages.
	 * it also check whether destpages indicate continuous physical memory.
	 */
	int (*prepare_destpages)(struct z_erofs_decompress_req *rq,
				 struct list_head *pagepool);
	int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out);
	char *name;
};

static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
					 struct list_head *pagepool)
{
	const unsigned int nr =
		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
	unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
					   BITS_PER_LONG)] = { 0 };
	void *kaddr = NULL;
	unsigned int i, j, top;

	top = 0;
	for (i = j = 0; i < nr; ++i, ++j) {
		struct page *const page = rq->out[i];
		struct page *victim;

		if (j >= LZ4_MAX_DISTANCE_PAGES)
			j = 0;

		/* 'valid' bounced can only be tested after a complete round */
		if (test_bit(j, bounced)) {
			DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES);
			DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES);
			availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];
		}

		if (page) {
			__clear_bit(j, bounced);
			if (kaddr) {
				if (kaddr + PAGE_SIZE == page_address(page))
					kaddr += PAGE_SIZE;
				else
					kaddr = NULL;
			} else if (!i) {
				kaddr = page_address(page);
			}
			continue;
		}
		kaddr = NULL;
		__set_bit(j, bounced);

		if (top) {
			victim = availables[--top];
			get_page(victim);
		} else {
			victim = erofs_allocpage(pagepool, GFP_KERNEL);
			if (!victim)
				return -ENOMEM;
			victim->mapping = Z_EROFS_MAPPING_STAGING;
		}
		rq->out[i] = victim;
	}
	return kaddr ? 1 : 0;
}

static void *generic_copy_inplace_data(struct z_erofs_decompress_req *rq,
				       u8 *src, unsigned int pageofs_in)
{
	/*
	 * if in-place decompression is ongoing, those decompressed
	 * pages should be copied in order to avoid being overlapped.
	 */
	struct page **in = rq->in;
	u8 *const tmp = erofs_get_pcpubuf(0);
	u8 *tmpp = tmp;
	unsigned int inlen = rq->inputsize - pageofs_in;
	unsigned int count = min_t(uint, inlen, PAGE_SIZE - pageofs_in);

	while (tmpp < tmp + inlen) {
		if (!src)
			src = kmap_atomic(*in);
		memcpy(tmpp, src + pageofs_in, count);
		kunmap_atomic(src);
		src = NULL;
		tmpp += count;
		pageofs_in = 0;
		count = PAGE_SIZE;
		++in;
	}
	return tmp;
}

static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)
{
	unsigned int inputmargin, inlen;
	u8 *src;
	bool copied, support_0padding;
	int ret;

	if (rq->inputsize > PAGE_SIZE)
		return -EOPNOTSUPP;

	src = kmap_atomic(*rq->in);
	inputmargin = 0;
	support_0padding = false;

	/* decompression inplace is only safe when 0padding is enabled */
	if (EROFS_SB(rq->sb)->feature_incompat &
	    EROFS_FEATURE_INCOMPAT_LZ4_0PADDING) {
		support_0padding = true;

		while (!src[inputmargin & ~PAGE_MASK])
			if (!(++inputmargin & ~PAGE_MASK))
				break;

		if (inputmargin >= rq->inputsize) {
			kunmap_atomic(src);
			return -EIO;
		}
	}

	copied = false;
	inlen = rq->inputsize - inputmargin;
	if (rq->inplace_io) {
		const uint oend = (rq->pageofs_out +
				   rq->outputsize) & ~PAGE_MASK;
		const uint nr = PAGE_ALIGN(rq->pageofs_out +
					   rq->outputsize) >> PAGE_SHIFT;

		if (rq->partial_decoding || !support_0padding ||
		    rq->out[nr - 1] != rq->in[0] ||
		    rq->inputsize - oend <
		      LZ4_DECOMPRESS_INPLACE_MARGIN(inlen)) {
			src = generic_copy_inplace_data(rq, src, inputmargin);
			inputmargin = 0;
			copied = true;
		}
	}

	ret = LZ4_decompress_safe_partial(src + inputmargin, out,
					  inlen, rq->outputsize,
					  rq->outputsize);
	if (ret < 0) {
		erofs_err(rq->sb, "failed to decompress, in[%u, %u] out[%u]",
			  inlen, inputmargin, rq->outputsize);
		WARN_ON(1);
		print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET,
			       16, 1, src + inputmargin, inlen, true);
		print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET,
			       16, 1, out, rq->outputsize, true);
		ret = -EIO;
	}

	if (copied)
		erofs_put_pcpubuf(src);
	else
		kunmap_atomic(src);
	return ret;
}

static struct z_erofs_decompressor decompressors[] = {
	[Z_EROFS_COMPRESSION_SHIFTED] = {
		.name = "shifted"
	},
	[Z_EROFS_COMPRESSION_LZ4] = {
		.prepare_destpages = z_erofs_lz4_prepare_destpages,
		.decompress = z_erofs_lz4_decompress,
		.name = "lz4"
	},
};

static void copy_from_pcpubuf(struct page **out, const char *dst,
			      unsigned short pageofs_out,
			      unsigned int outputsize)
{
	const char *end = dst + outputsize;
	const unsigned int righthalf = PAGE_SIZE - pageofs_out;
	const char *cur = dst - pageofs_out;

	while (cur < end) {
		struct page *const page = *out++;

		if (page) {
			char *buf = kmap_atomic(page);

			if (cur >= dst) {
				memcpy(buf, cur, min_t(uint, PAGE_SIZE,
						       end - cur));
			} else {
				memcpy(buf + pageofs_out, cur + pageofs_out,
				       min_t(uint, righthalf, end - cur));
			}
			kunmap_atomic(buf);
		}
		cur += PAGE_SIZE;
	}
}

static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq,
				      struct list_head *pagepool)
{
	const unsigned int nrpages_out =
		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	const struct z_erofs_decompressor *alg = decompressors + rq->alg;
	unsigned int dst_maptype;
	void *dst;
	int ret, i;

	if (nrpages_out == 1 && !rq->inplace_io) {
		DBG_BUGON(!*rq->out);
		dst = kmap_atomic(*rq->out);
		dst_maptype = 0;
		goto dstmap_out;
	}

	/*
	 * For the case of small output size (especially much less
	 * than PAGE_SIZE), memcpy the decompressed data rather than
	 * compressed data is preferred.
	 */
	if (rq->outputsize <= PAGE_SIZE * 7 / 8) {
		dst = erofs_get_pcpubuf(0);
		if (IS_ERR(dst))
			return PTR_ERR(dst);

		rq->inplace_io = false;
		ret = alg->decompress(rq, dst);
		if (!ret)
			copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,
					  rq->outputsize);

		erofs_put_pcpubuf(dst);
		return ret;
	}

	ret = alg->prepare_destpages(rq, pagepool);
	if (ret < 0) {
		return ret;
	} else if (ret) {
		dst = page_address(*rq->out);
		dst_maptype = 1;
		goto dstmap_out;
	}

	i = 0;
	while (1) {
		dst = vm_map_ram(rq->out, nrpages_out, -1, PAGE_KERNEL);

		/* retry two more times (totally 3 times) */
		if (dst || ++i >= 3)
			break;
		vm_unmap_aliases();
	}

	if (!dst)
		return -ENOMEM;

	dst_maptype = 2;

dstmap_out:
	ret = alg->decompress(rq, dst + rq->pageofs_out);

	if (!dst_maptype)
		kunmap_atomic(dst);
	else if (dst_maptype == 2)
		vm_unmap_ram(dst, nrpages_out);
	return ret;
}

static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq,
				     struct list_head *pagepool)
{
	const unsigned int nrpages_out =
		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out;
	unsigned char *src, *dst;

	if (nrpages_out > 2) {
		DBG_BUGON(1);
		return -EIO;
	}

	if (rq->out[0] == *rq->in) {
		DBG_BUGON(nrpages_out != 1);
		return 0;
	}

	src = kmap_atomic(*rq->in);
	if (rq->out[0]) {
		dst = kmap_atomic(rq->out[0]);
		memcpy(dst + rq->pageofs_out, src, righthalf);
		kunmap_atomic(dst);
	}

	if (nrpages_out == 2) {
		DBG_BUGON(!rq->out[1]);
		if (rq->out[1] == *rq->in) {
			memmove(src, src + righthalf, rq->pageofs_out);
		} else {
			dst = kmap_atomic(rq->out[1]);
			memcpy(dst, src + righthalf, rq->pageofs_out);
			kunmap_atomic(dst);
		}
	}
	kunmap_atomic(src);
	return 0;
}

int z_erofs_decompress(struct z_erofs_decompress_req *rq,
		       struct list_head *pagepool)
{
	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED)
		return z_erofs_shifted_transform(rq, pagepool);
	return z_erofs_decompress_generic(rq, pagepool);
}
Commit	Line	Data
29b24f6c	1	// SPDX-License-Identifier: GPL-2.0-only
7fc45dbc	2	/*
7fc45dbc GX	3	* Copyright (C) 2019 HUAWEI, Inc.
	4	* http://www.huawei.com/
	5	* Created by Gao Xiang <gaoxiang25@huawei.com>
	6	*/
	7	#include "compress.h"
46c2d149	8	#include <linux/module.h>
7fc45dbc GX	9	#include <linux/lz4.h>
	10
	11	#ifndef LZ4_DISTANCE_MAX /* history window size */
	12	#define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */
	13	#endif
	14
af89bcef	15	#define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1)
0ffd71bc GX	16	#ifndef LZ4_DECOMPRESS_INPLACE_MARGIN
	17	#define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize) (((srcsize) >> 8) + 32)
	18	#endif
7fc45dbc GX	19
	20	struct z_erofs_decompressor {
	21	/*
	22	* if destpages have sparsed pages, fill them with bounce pages.
	23	* it also check whether destpages indicate continuous physical memory.
	24	*/
	25	int (prepare_destpages)(struct z_erofs_decompress_req rq,
	26	struct list_head *pagepool);
	27	int (decompress)(struct z_erofs_decompress_req rq, u8 *out);
	28	char *name;
	29	};
	30
99634bf3 GX	31	static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,
99634bf3 GX	32	struct list_head *pagepool)
7fc45dbc GX	33	{
	34	const unsigned int nr =
	35	PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	36	struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL };
af89bcef GX	37	unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES,
af89bcef GX	38	BITS_PER_LONG)] = { 0 };
7fc45dbc	39	void *kaddr = NULL;
af89bcef	40	unsigned int i, j, top;
7fc45dbc	41
af89bcef GX	42	top = 0;
af89bcef GX	43	for (i = j = 0; i < nr; ++i, ++j) {
7fc45dbc	44	struct page *const page = rq->out[i];
af89bcef	45	struct page *victim;
7fc45dbc	46
af89bcef GX	47	if (j >= LZ4_MAX_DISTANCE_PAGES)
	48	j = 0;
	49
	50	/* 'valid' bounced can only be tested after a complete round */
	51	if (test_bit(j, bounced)) {
	52	DBG_BUGON(i < LZ4_MAX_DISTANCE_PAGES);
	53	DBG_BUGON(top >= LZ4_MAX_DISTANCE_PAGES);
	54	availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];
	55	}
7fc45dbc GX	56
7fc45dbc GX	57	if (page) {
af89bcef	58	__clear_bit(j, bounced);
7fc45dbc GX	59	if (kaddr) {
	60	if (kaddr + PAGE_SIZE == page_address(page))
	61	kaddr += PAGE_SIZE;
	62	else
	63	kaddr = NULL;
	64	} else if (!i) {
	65	kaddr = page_address(page);
	66	}
	67	continue;
	68	}
	69	kaddr = NULL;
af89bcef	70	__set_bit(j, bounced);
7fc45dbc	71
af89bcef GX	72	if (top) {
	73	victim = availables[--top];
	74	get_page(victim);
7fc45dbc	75	} else {
5ddcee1f	76	victim = erofs_allocpage(pagepool, GFP_KERNEL);
8d8a09b0	77	if (!victim)
b25a1519	78	return -ENOMEM;
af89bcef	79	victim->mapping = Z_EROFS_MAPPING_STAGING;
7fc45dbc	80	}
af89bcef	81	rq->out[i] = victim;
7fc45dbc GX	82	}
	83	return kaddr ? 1 : 0;
	84	}
	85
	86	static void generic_copy_inplace_data(struct z_erofs_decompress_req rq,
	87	u8 *src, unsigned int pageofs_in)
	88	{
	89	/*
	90	* if in-place decompression is ongoing, those decompressed
	91	* pages should be copied in order to avoid being overlapped.
	92	*/
	93	struct page **in = rq->in;
	94	u8 *const tmp = erofs_get_pcpubuf(0);
	95	u8 *tmpp = tmp;
	96	unsigned int inlen = rq->inputsize - pageofs_in;
	97	unsigned int count = min_t(uint, inlen, PAGE_SIZE - pageofs_in);
	98
	99	while (tmpp < tmp + inlen) {
	100	if (!src)
	101	src = kmap_atomic(*in);
	102	memcpy(tmpp, src + pageofs_in, count);
	103	kunmap_atomic(src);
	104	src = NULL;
	105	tmpp += count;
	106	pageofs_in = 0;
	107	count = PAGE_SIZE;
	108	++in;
	109	}
	110	return tmp;
	111	}
	112
99634bf3	113	static int z_erofs_lz4_decompress(struct z_erofs_decompress_req rq, u8 out)
7fc45dbc GX	114	{
	115	unsigned int inputmargin, inlen;
	116	u8 *src;
0ffd71bc	117	bool copied, support_0padding;
7fc45dbc GX	118	int ret;
	119
	120	if (rq->inputsize > PAGE_SIZE)
ff784a78	121	return -EOPNOTSUPP;
7fc45dbc GX	122
	123	src = kmap_atomic(*rq->in);
	124	inputmargin = 0;
0ffd71bc GX	125	support_0padding = false;
	126
	127	/* decompression inplace is only safe when 0padding is enabled */
426a9308 GX	128	if (EROFS_SB(rq->sb)->feature_incompat &
426a9308 GX	129	EROFS_FEATURE_INCOMPAT_LZ4_0PADDING) {
0ffd71bc GX	130	support_0padding = true;
	131
	132	while (!src[inputmargin & ~PAGE_MASK])
	133	if (!(++inputmargin & ~PAGE_MASK))
	134	break;
	135
	136	if (inputmargin >= rq->inputsize) {
	137	kunmap_atomic(src);
	138	return -EIO;
	139	}
	140	}
7fc45dbc GX	141
	142	copied = false;
	143	inlen = rq->inputsize - inputmargin;
	144	if (rq->inplace_io) {
0ffd71bc GX	145	const uint oend = (rq->pageofs_out +
	146	rq->outputsize) & ~PAGE_MASK;
	147	const uint nr = PAGE_ALIGN(rq->pageofs_out +
	148	rq->outputsize) >> PAGE_SHIFT;
	149
	150	if (rq->partial_decoding \|\| !support_0padding \|\|
	151	rq->out[nr - 1] != rq->in[0] \|\|
	152	rq->inputsize - oend <
	153	LZ4_DECOMPRESS_INPLACE_MARGIN(inlen)) {
	154	src = generic_copy_inplace_data(rq, src, inputmargin);
	155	inputmargin = 0;
	156	copied = true;
	157	}
7fc45dbc GX	158	}
	159
	160	ret = LZ4_decompress_safe_partial(src + inputmargin, out,
	161	inlen, rq->outputsize,
	162	rq->outputsize);
	163	if (ret < 0) {
4f761fa2 GX	164	erofs_err(rq->sb, "failed to decompress, in[%u, %u] out[%u]",
4f761fa2 GX	165	inlen, inputmargin, rq->outputsize);
7fc45dbc GX	166	WARN_ON(1);
	167	print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET,
	168	16, 1, src + inputmargin, inlen, true);
	169	print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET,
	170	16, 1, out, rq->outputsize, true);
	171	ret = -EIO;
	172	}
	173
	174	if (copied)
	175	erofs_put_pcpubuf(src);
	176	else
	177	kunmap_atomic(src);
	178	return ret;
	179	}
	180
	181	static struct z_erofs_decompressor decompressors[] = {
	182	[Z_EROFS_COMPRESSION_SHIFTED] = {
	183	.name = "shifted"
	184	},
	185	[Z_EROFS_COMPRESSION_LZ4] = {
99634bf3 GX	186	.prepare_destpages = z_erofs_lz4_prepare_destpages,
99634bf3 GX	187	.decompress = z_erofs_lz4_decompress,
7fc45dbc GX	188	.name = "lz4"
	189	},
	190	};
	191
	192	static void copy_from_pcpubuf(struct page *out, const char dst,
	193	unsigned short pageofs_out,
	194	unsigned int outputsize)
	195	{
	196	const char *end = dst + outputsize;
	197	const unsigned int righthalf = PAGE_SIZE - pageofs_out;
	198	const char *cur = dst - pageofs_out;
	199
	200	while (cur < end) {
	201	struct page const page = out++;
	202
	203	if (page) {
	204	char *buf = kmap_atomic(page);
	205
	206	if (cur >= dst) {
	207	memcpy(buf, cur, min_t(uint, PAGE_SIZE,
	208	end - cur));
	209	} else {
	210	memcpy(buf + pageofs_out, cur + pageofs_out,
	211	min_t(uint, righthalf, end - cur));
	212	}
	213	kunmap_atomic(buf);
	214	}
	215	cur += PAGE_SIZE;
	216	}
	217	}
	218
99634bf3 GX	219	static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq,
99634bf3 GX	220	struct list_head *pagepool)
7fc45dbc GX	221	{
	222	const unsigned int nrpages_out =
	223	PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	224	const struct z_erofs_decompressor *alg = decompressors + rq->alg;
	225	unsigned int dst_maptype;
	226	void *dst;
73d03931	227	int ret, i;
7fc45dbc GX	228
	229	if (nrpages_out == 1 && !rq->inplace_io) {
	230	DBG_BUGON(!*rq->out);
	231	dst = kmap_atomic(*rq->out);
	232	dst_maptype = 0;
	233	goto dstmap_out;
	234	}
	235
	236	/*
	237	* For the case of small output size (especially much less
	238	* than PAGE_SIZE), memcpy the decompressed data rather than
	239	* compressed data is preferred.
	240	*/
	241	if (rq->outputsize <= PAGE_SIZE * 7 / 8) {
	242	dst = erofs_get_pcpubuf(0);
	243	if (IS_ERR(dst))
	244	return PTR_ERR(dst);
	245
	246	rq->inplace_io = false;
	247	ret = alg->decompress(rq, dst);
	248	if (!ret)
	249	copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,
	250	rq->outputsize);
	251
	252	erofs_put_pcpubuf(dst);
	253	return ret;
	254	}
	255
	256	ret = alg->prepare_destpages(rq, pagepool);
	257	if (ret < 0) {
	258	return ret;
	259	} else if (ret) {
	260	dst = page_address(*rq->out);
	261	dst_maptype = 1;
	262	goto dstmap_out;
	263	}
	264
73d03931 GX	265	i = 0;
	266	while (1) {
	267	dst = vm_map_ram(rq->out, nrpages_out, -1, PAGE_KERNEL);
	268
	269	/* retry two more times (totally 3 times) */
	270	if (dst \|\| ++i >= 3)
	271	break;
	272	vm_unmap_aliases();
	273	}
	274
7fc45dbc GX	275	if (!dst)
7fc45dbc GX	276	return -ENOMEM;
73d03931	277
7fc45dbc GX	278	dst_maptype = 2;
	279
	280	dstmap_out:
	281	ret = alg->decompress(rq, dst + rq->pageofs_out);
	282
	283	if (!dst_maptype)
	284	kunmap_atomic(dst);
	285	else if (dst_maptype == 2)
73d03931	286	vm_unmap_ram(dst, nrpages_out);
7fc45dbc GX	287	return ret;
	288	}
	289
99634bf3 GX	290	static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq,
99634bf3 GX	291	struct list_head *pagepool)
7fc45dbc GX	292	{
	293	const unsigned int nrpages_out =
	294	PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
	295	const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out;
	296	unsigned char src, dst;
	297
	298	if (nrpages_out > 2) {
	299	DBG_BUGON(1);
	300	return -EIO;
	301	}
	302
	303	if (rq->out[0] == *rq->in) {
	304	DBG_BUGON(nrpages_out != 1);
	305	return 0;
	306	}
	307
	308	src = kmap_atomic(*rq->in);
4d202437	309	if (rq->out[0]) {
7fc45dbc GX	310	dst = kmap_atomic(rq->out[0]);
7fc45dbc GX	311	memcpy(dst + rq->pageofs_out, src, righthalf);
4d202437	312	kunmap_atomic(dst);
7fc45dbc GX	313	}
7fc45dbc GX	314
4d202437	315	if (nrpages_out == 2) {
7fc45dbc	316	DBG_BUGON(!rq->out[1]);
4d202437 GX	317	if (rq->out[1] == *rq->in) {
	318	memmove(src, src + righthalf, rq->pageofs_out);
	319	} else {
	320	dst = kmap_atomic(rq->out[1]);
	321	memcpy(dst, src + righthalf, rq->pageofs_out);
	322	kunmap_atomic(dst);
	323	}
7fc45dbc	324	}
7fc45dbc GX	325	kunmap_atomic(src);
	326	return 0;
	327	}
	328
	329	int z_erofs_decompress(struct z_erofs_decompress_req *rq,
	330	struct list_head *pagepool)
	331	{
	332	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED)
99634bf3 GX	333	return z_erofs_shifted_transform(rq, pagepool);
99634bf3 GX	334	return z_erofs_decompress_generic(rq, pagepool);
7fc45dbc GX	335	}
7fc45dbc GX	336