Commit | Line | Data |
---|---|---|
622ceadd GX |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | #include <linux/xz.h> | |
3 | #include <linux/module.h> | |
4 | #include "compress.h" | |
5 | ||
6 | struct z_erofs_lzma { | |
7 | struct z_erofs_lzma *next; | |
8 | struct xz_dec_microlzma *state; | |
9 | struct xz_buf buf; | |
10 | u8 bounce[PAGE_SIZE]; | |
11 | }; | |
12 | ||
13 | /* considering the LZMA performance, no need to use a lockless list for now */ | |
14 | static DEFINE_SPINLOCK(z_erofs_lzma_lock); | |
15 | static unsigned int z_erofs_lzma_max_dictsize; | |
16 | static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; | |
17 | static struct z_erofs_lzma *z_erofs_lzma_head; | |
18 | static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); | |
19 | ||
20 | module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); | |
21 | ||
22 | void z_erofs_lzma_exit(void) | |
23 | { | |
24 | /* there should be no running fs instance */ | |
25 | while (z_erofs_lzma_avail_strms) { | |
26 | struct z_erofs_lzma *strm; | |
27 | ||
28 | spin_lock(&z_erofs_lzma_lock); | |
29 | strm = z_erofs_lzma_head; | |
30 | if (!strm) { | |
31 | spin_unlock(&z_erofs_lzma_lock); | |
32 | DBG_BUGON(1); | |
33 | return; | |
34 | } | |
35 | z_erofs_lzma_head = NULL; | |
36 | spin_unlock(&z_erofs_lzma_lock); | |
37 | ||
38 | while (strm) { | |
39 | struct z_erofs_lzma *n = strm->next; | |
40 | ||
41 | if (strm->state) | |
42 | xz_dec_microlzma_end(strm->state); | |
43 | kfree(strm); | |
44 | --z_erofs_lzma_avail_strms; | |
45 | strm = n; | |
46 | } | |
47 | } | |
48 | } | |
49 | ||
a279aded | 50 | int __init z_erofs_lzma_init(void) |
622ceadd GX |
51 | { |
52 | unsigned int i; | |
53 | ||
54 | /* by default, use # of possible CPUs instead */ | |
55 | if (!z_erofs_lzma_nstrms) | |
56 | z_erofs_lzma_nstrms = num_possible_cpus(); | |
57 | ||
58 | for (i = 0; i < z_erofs_lzma_nstrms; ++i) { | |
59 | struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); | |
60 | ||
61 | if (!strm) { | |
62 | z_erofs_lzma_exit(); | |
63 | return -ENOMEM; | |
64 | } | |
65 | spin_lock(&z_erofs_lzma_lock); | |
66 | strm->next = z_erofs_lzma_head; | |
67 | z_erofs_lzma_head = strm; | |
68 | spin_unlock(&z_erofs_lzma_lock); | |
69 | ++z_erofs_lzma_avail_strms; | |
70 | } | |
71 | return 0; | |
72 | } | |
73 | ||
74 | int z_erofs_load_lzma_config(struct super_block *sb, | |
75 | struct erofs_super_block *dsb, | |
76 | struct z_erofs_lzma_cfgs *lzma, int size) | |
77 | { | |
78 | static DEFINE_MUTEX(lzma_resize_mutex); | |
79 | unsigned int dict_size, i; | |
80 | struct z_erofs_lzma *strm, *head = NULL; | |
81 | int err; | |
82 | ||
83 | if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { | |
84 | erofs_err(sb, "invalid lzma cfgs, size=%u", size); | |
85 | return -EINVAL; | |
86 | } | |
87 | if (lzma->format) { | |
88 | erofs_err(sb, "unidentified lzma format %x, please check kernel version", | |
89 | le16_to_cpu(lzma->format)); | |
90 | return -EINVAL; | |
91 | } | |
92 | dict_size = le32_to_cpu(lzma->dict_size); | |
93 | if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { | |
94 | erofs_err(sb, "unsupported lzma dictionary size %u", | |
95 | dict_size); | |
96 | return -EINVAL; | |
97 | } | |
98 | ||
99 | erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!"); | |
100 | ||
101 | /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ | |
102 | mutex_lock(&lzma_resize_mutex); | |
103 | ||
104 | if (z_erofs_lzma_max_dictsize >= dict_size) { | |
105 | mutex_unlock(&lzma_resize_mutex); | |
106 | return 0; | |
107 | } | |
108 | ||
109 | /* 1. collect/isolate all streams for the following check */ | |
110 | for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { | |
111 | struct z_erofs_lzma *last; | |
112 | ||
113 | again: | |
114 | spin_lock(&z_erofs_lzma_lock); | |
115 | strm = z_erofs_lzma_head; | |
116 | if (!strm) { | |
117 | spin_unlock(&z_erofs_lzma_lock); | |
118 | wait_event(z_erofs_lzma_wq, | |
119 | READ_ONCE(z_erofs_lzma_head)); | |
120 | goto again; | |
121 | } | |
122 | z_erofs_lzma_head = NULL; | |
123 | spin_unlock(&z_erofs_lzma_lock); | |
124 | ||
125 | for (last = strm; last->next; last = last->next) | |
126 | ++i; | |
127 | last->next = head; | |
128 | head = strm; | |
129 | } | |
130 | ||
131 | err = 0; | |
132 | /* 2. walk each isolated stream and grow max dict_size if needed */ | |
133 | for (strm = head; strm; strm = strm->next) { | |
134 | if (strm->state) | |
135 | xz_dec_microlzma_end(strm->state); | |
136 | strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); | |
137 | if (!strm->state) | |
138 | err = -ENOMEM; | |
139 | } | |
140 | ||
141 | /* 3. push back all to the global list and update max dict_size */ | |
142 | spin_lock(&z_erofs_lzma_lock); | |
143 | DBG_BUGON(z_erofs_lzma_head); | |
144 | z_erofs_lzma_head = head; | |
145 | spin_unlock(&z_erofs_lzma_lock); | |
2df7c4bd | 146 | wake_up_all(&z_erofs_lzma_wq); |
622ceadd GX |
147 | |
148 | z_erofs_lzma_max_dictsize = dict_size; | |
149 | mutex_unlock(&lzma_resize_mutex); | |
150 | return err; | |
151 | } | |
152 | ||
153 | int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, | |
eaa9172a | 154 | struct page **pagepool) |
622ceadd GX |
155 | { |
156 | const unsigned int nrpages_out = | |
157 | PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; | |
158 | const unsigned int nrpages_in = | |
159 | PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; | |
10e5f6e4 | 160 | unsigned int inlen, outlen, pageofs; |
622ceadd GX |
161 | struct z_erofs_lzma *strm; |
162 | u8 *kin; | |
163 | bool bounced = false; | |
164 | int no, ni, j, err = 0; | |
165 | ||
166 | /* 1. get the exact LZMA compressed size */ | |
167 | kin = kmap(*rq->in); | |
10e5f6e4 GX |
168 | err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, |
169 | min_t(unsigned int, rq->inputsize, | |
170 | EROFS_BLKSIZ - rq->pageofs_in)); | |
171 | if (err) { | |
622ceadd | 172 | kunmap(*rq->in); |
10e5f6e4 | 173 | return err; |
622ceadd | 174 | } |
622ceadd GX |
175 | |
176 | /* 2. get an available lzma context */ | |
177 | again: | |
178 | spin_lock(&z_erofs_lzma_lock); | |
179 | strm = z_erofs_lzma_head; | |
180 | if (!strm) { | |
181 | spin_unlock(&z_erofs_lzma_lock); | |
182 | wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); | |
183 | goto again; | |
184 | } | |
185 | z_erofs_lzma_head = strm->next; | |
186 | spin_unlock(&z_erofs_lzma_lock); | |
187 | ||
188 | /* 3. multi-call decompress */ | |
189 | inlen = rq->inputsize; | |
190 | outlen = rq->outputsize; | |
191 | xz_dec_microlzma_reset(strm->state, inlen, outlen, | |
192 | !rq->partial_decoding); | |
193 | pageofs = rq->pageofs_out; | |
10e5f6e4 | 194 | strm->buf.in = kin + rq->pageofs_in; |
622ceadd | 195 | strm->buf.in_pos = 0; |
10e5f6e4 | 196 | strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in); |
622ceadd GX |
197 | inlen -= strm->buf.in_size; |
198 | strm->buf.out = NULL; | |
199 | strm->buf.out_pos = 0; | |
200 | strm->buf.out_size = 0; | |
201 | ||
202 | for (ni = 0, no = -1;;) { | |
203 | enum xz_ret xz_err; | |
204 | ||
205 | if (strm->buf.out_pos == strm->buf.out_size) { | |
206 | if (strm->buf.out) { | |
207 | kunmap(rq->out[no]); | |
208 | strm->buf.out = NULL; | |
209 | } | |
210 | ||
211 | if (++no >= nrpages_out || !outlen) { | |
212 | erofs_err(rq->sb, "decompressed buf out of bound"); | |
213 | err = -EFSCORRUPTED; | |
214 | break; | |
215 | } | |
216 | strm->buf.out_pos = 0; | |
217 | strm->buf.out_size = min_t(u32, outlen, | |
218 | PAGE_SIZE - pageofs); | |
219 | outlen -= strm->buf.out_size; | |
5c2a6425 GX |
220 | if (!rq->out[no] && rq->fillgaps) /* deduped */ |
221 | rq->out[no] = erofs_allocpage(pagepool, | |
222 | GFP_KERNEL | __GFP_NOFAIL); | |
622ceadd GX |
223 | if (rq->out[no]) |
224 | strm->buf.out = kmap(rq->out[no]) + pageofs; | |
225 | pageofs = 0; | |
226 | } else if (strm->buf.in_pos == strm->buf.in_size) { | |
227 | kunmap(rq->in[ni]); | |
228 | ||
229 | if (++ni >= nrpages_in || !inlen) { | |
230 | erofs_err(rq->sb, "compressed buf out of bound"); | |
231 | err = -EFSCORRUPTED; | |
232 | break; | |
233 | } | |
234 | strm->buf.in_pos = 0; | |
235 | strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE); | |
236 | inlen -= strm->buf.in_size; | |
237 | kin = kmap(rq->in[ni]); | |
238 | strm->buf.in = kin; | |
239 | bounced = false; | |
240 | } | |
241 | ||
242 | /* | |
243 | * Handle overlapping: Use bounced buffer if the compressed | |
244 | * data is under processing; Otherwise, Use short-lived pages | |
245 | * from the on-stack pagepool where pages share with the same | |
246 | * request. | |
247 | */ | |
248 | if (!bounced && rq->out[no] == rq->in[ni]) { | |
249 | memcpy(strm->bounce, strm->buf.in, strm->buf.in_size); | |
250 | strm->buf.in = strm->bounce; | |
251 | bounced = true; | |
252 | } | |
253 | for (j = ni + 1; j < nrpages_in; ++j) { | |
254 | struct page *tmppage; | |
255 | ||
256 | if (rq->out[no] != rq->in[j]) | |
257 | continue; | |
258 | ||
259 | DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb), | |
260 | rq->in[j])); | |
261 | tmppage = erofs_allocpage(pagepool, | |
262 | GFP_KERNEL | __GFP_NOFAIL); | |
263 | set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); | |
264 | copy_highpage(tmppage, rq->in[j]); | |
265 | rq->in[j] = tmppage; | |
266 | } | |
267 | xz_err = xz_dec_microlzma_run(strm->state, &strm->buf); | |
268 | DBG_BUGON(strm->buf.out_pos > strm->buf.out_size); | |
269 | DBG_BUGON(strm->buf.in_pos > strm->buf.in_size); | |
270 | ||
271 | if (xz_err != XZ_OK) { | |
272 | if (xz_err == XZ_STREAM_END && !outlen) | |
273 | break; | |
274 | erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]", | |
275 | xz_err, rq->inputsize, rq->outputsize); | |
276 | err = -EFSCORRUPTED; | |
277 | break; | |
278 | } | |
279 | } | |
280 | if (no < nrpages_out && strm->buf.out) | |
8f121dfb | 281 | kunmap(rq->out[no]); |
622ceadd GX |
282 | if (ni < nrpages_in) |
283 | kunmap(rq->in[ni]); | |
284 | /* 4. push back LZMA stream context to the global list */ | |
285 | spin_lock(&z_erofs_lzma_lock); | |
286 | strm->next = z_erofs_lzma_head; | |
287 | z_erofs_lzma_head = strm; | |
288 | spin_unlock(&z_erofs_lzma_lock); | |
289 | wake_up(&z_erofs_lzma_wq); | |
290 | return err; | |
291 | } |