Commit | Line | Data |
---|---|---|
622ceadd GX |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | #include <linux/xz.h> | |
622ceadd GX |
3 | #include "compress.h" |
4 | ||
5 | struct z_erofs_lzma { | |
6 | struct z_erofs_lzma *next; | |
7 | struct xz_dec_microlzma *state; | |
8 | struct xz_buf buf; | |
9 | u8 bounce[PAGE_SIZE]; | |
10 | }; | |
11 | ||
12 | /* considering the LZMA performance, no need to use a lockless list for now */ | |
13 | static DEFINE_SPINLOCK(z_erofs_lzma_lock); | |
14 | static unsigned int z_erofs_lzma_max_dictsize; | |
15 | static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; | |
16 | static struct z_erofs_lzma *z_erofs_lzma_head; | |
17 | static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); | |
18 | ||
19 | module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); | |
20 | ||
21 | void z_erofs_lzma_exit(void) | |
22 | { | |
23 | /* there should be no running fs instance */ | |
24 | while (z_erofs_lzma_avail_strms) { | |
25 | struct z_erofs_lzma *strm; | |
26 | ||
27 | spin_lock(&z_erofs_lzma_lock); | |
28 | strm = z_erofs_lzma_head; | |
29 | if (!strm) { | |
30 | spin_unlock(&z_erofs_lzma_lock); | |
31 | DBG_BUGON(1); | |
32 | return; | |
33 | } | |
34 | z_erofs_lzma_head = NULL; | |
35 | spin_unlock(&z_erofs_lzma_lock); | |
36 | ||
37 | while (strm) { | |
38 | struct z_erofs_lzma *n = strm->next; | |
39 | ||
40 | if (strm->state) | |
41 | xz_dec_microlzma_end(strm->state); | |
42 | kfree(strm); | |
43 | --z_erofs_lzma_avail_strms; | |
44 | strm = n; | |
45 | } | |
46 | } | |
47 | } | |
48 | ||
a279aded | 49 | int __init z_erofs_lzma_init(void) |
622ceadd GX |
50 | { |
51 | unsigned int i; | |
52 | ||
53 | /* by default, use # of possible CPUs instead */ | |
54 | if (!z_erofs_lzma_nstrms) | |
55 | z_erofs_lzma_nstrms = num_possible_cpus(); | |
56 | ||
57 | for (i = 0; i < z_erofs_lzma_nstrms; ++i) { | |
58 | struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); | |
59 | ||
60 | if (!strm) { | |
61 | z_erofs_lzma_exit(); | |
62 | return -ENOMEM; | |
63 | } | |
64 | spin_lock(&z_erofs_lzma_lock); | |
65 | strm->next = z_erofs_lzma_head; | |
66 | z_erofs_lzma_head = strm; | |
67 | spin_unlock(&z_erofs_lzma_lock); | |
68 | ++z_erofs_lzma_avail_strms; | |
69 | } | |
70 | return 0; | |
71 | } | |
72 | ||
73 | int z_erofs_load_lzma_config(struct super_block *sb, | |
efb4fb02 | 74 | struct erofs_super_block *dsb, void *data, int size) |
622ceadd GX |
75 | { |
76 | static DEFINE_MUTEX(lzma_resize_mutex); | |
efb4fb02 | 77 | struct z_erofs_lzma_cfgs *lzma = data; |
622ceadd GX |
78 | unsigned int dict_size, i; |
79 | struct z_erofs_lzma *strm, *head = NULL; | |
80 | int err; | |
81 | ||
82 | if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { | |
83 | erofs_err(sb, "invalid lzma cfgs, size=%u", size); | |
84 | return -EINVAL; | |
85 | } | |
86 | if (lzma->format) { | |
87 | erofs_err(sb, "unidentified lzma format %x, please check kernel version", | |
88 | le16_to_cpu(lzma->format)); | |
89 | return -EINVAL; | |
90 | } | |
91 | dict_size = le32_to_cpu(lzma->dict_size); | |
92 | if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { | |
93 | erofs_err(sb, "unsupported lzma dictionary size %u", | |
94 | dict_size); | |
95 | return -EINVAL; | |
96 | } | |
97 | ||
622ceadd GX |
98 | /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ |
99 | mutex_lock(&lzma_resize_mutex); | |
100 | ||
101 | if (z_erofs_lzma_max_dictsize >= dict_size) { | |
102 | mutex_unlock(&lzma_resize_mutex); | |
103 | return 0; | |
104 | } | |
105 | ||
106 | /* 1. collect/isolate all streams for the following check */ | |
107 | for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { | |
108 | struct z_erofs_lzma *last; | |
109 | ||
110 | again: | |
111 | spin_lock(&z_erofs_lzma_lock); | |
112 | strm = z_erofs_lzma_head; | |
113 | if (!strm) { | |
114 | spin_unlock(&z_erofs_lzma_lock); | |
115 | wait_event(z_erofs_lzma_wq, | |
116 | READ_ONCE(z_erofs_lzma_head)); | |
117 | goto again; | |
118 | } | |
119 | z_erofs_lzma_head = NULL; | |
120 | spin_unlock(&z_erofs_lzma_lock); | |
121 | ||
122 | for (last = strm; last->next; last = last->next) | |
123 | ++i; | |
124 | last->next = head; | |
125 | head = strm; | |
126 | } | |
127 | ||
128 | err = 0; | |
129 | /* 2. walk each isolated stream and grow max dict_size if needed */ | |
130 | for (strm = head; strm; strm = strm->next) { | |
131 | if (strm->state) | |
132 | xz_dec_microlzma_end(strm->state); | |
133 | strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); | |
134 | if (!strm->state) | |
135 | err = -ENOMEM; | |
136 | } | |
137 | ||
138 | /* 3. push back all to the global list and update max dict_size */ | |
139 | spin_lock(&z_erofs_lzma_lock); | |
140 | DBG_BUGON(z_erofs_lzma_head); | |
141 | z_erofs_lzma_head = head; | |
142 | spin_unlock(&z_erofs_lzma_lock); | |
2df7c4bd | 143 | wake_up_all(&z_erofs_lzma_wq); |
622ceadd GX |
144 | |
145 | z_erofs_lzma_max_dictsize = dict_size; | |
146 | mutex_unlock(&lzma_resize_mutex); | |
147 | return err; | |
148 | } | |
149 | ||
150 | int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, | |
d9281660 | 151 | struct page **pgpl) |
622ceadd GX |
152 | { |
153 | const unsigned int nrpages_out = | |
154 | PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; | |
155 | const unsigned int nrpages_in = | |
156 | PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; | |
10e5f6e4 | 157 | unsigned int inlen, outlen, pageofs; |
622ceadd GX |
158 | struct z_erofs_lzma *strm; |
159 | u8 *kin; | |
160 | bool bounced = false; | |
161 | int no, ni, j, err = 0; | |
162 | ||
163 | /* 1. get the exact LZMA compressed size */ | |
164 | kin = kmap(*rq->in); | |
10e5f6e4 | 165 | err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, |
3acea5fc JX |
166 | min_t(unsigned int, rq->inputsize, |
167 | rq->sb->s_blocksize - rq->pageofs_in)); | |
10e5f6e4 | 168 | if (err) { |
622ceadd | 169 | kunmap(*rq->in); |
10e5f6e4 | 170 | return err; |
622ceadd | 171 | } |
622ceadd GX |
172 | |
173 | /* 2. get an available lzma context */ | |
174 | again: | |
175 | spin_lock(&z_erofs_lzma_lock); | |
176 | strm = z_erofs_lzma_head; | |
177 | if (!strm) { | |
178 | spin_unlock(&z_erofs_lzma_lock); | |
179 | wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); | |
180 | goto again; | |
181 | } | |
182 | z_erofs_lzma_head = strm->next; | |
183 | spin_unlock(&z_erofs_lzma_lock); | |
184 | ||
185 | /* 3. multi-call decompress */ | |
186 | inlen = rq->inputsize; | |
187 | outlen = rq->outputsize; | |
188 | xz_dec_microlzma_reset(strm->state, inlen, outlen, | |
189 | !rq->partial_decoding); | |
190 | pageofs = rq->pageofs_out; | |
10e5f6e4 | 191 | strm->buf.in = kin + rq->pageofs_in; |
622ceadd | 192 | strm->buf.in_pos = 0; |
10e5f6e4 | 193 | strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in); |
622ceadd GX |
194 | inlen -= strm->buf.in_size; |
195 | strm->buf.out = NULL; | |
196 | strm->buf.out_pos = 0; | |
197 | strm->buf.out_size = 0; | |
198 | ||
199 | for (ni = 0, no = -1;;) { | |
200 | enum xz_ret xz_err; | |
201 | ||
202 | if (strm->buf.out_pos == strm->buf.out_size) { | |
203 | if (strm->buf.out) { | |
204 | kunmap(rq->out[no]); | |
205 | strm->buf.out = NULL; | |
206 | } | |
207 | ||
208 | if (++no >= nrpages_out || !outlen) { | |
209 | erofs_err(rq->sb, "decompressed buf out of bound"); | |
210 | err = -EFSCORRUPTED; | |
211 | break; | |
212 | } | |
213 | strm->buf.out_pos = 0; | |
214 | strm->buf.out_size = min_t(u32, outlen, | |
215 | PAGE_SIZE - pageofs); | |
216 | outlen -= strm->buf.out_size; | |
75a52216 | 217 | if (!rq->out[no] && rq->fillgaps) { /* deduped */ |
d9281660 CG |
218 | rq->out[no] = erofs_allocpage(pgpl, rq->gfp); |
219 | if (!rq->out[no]) { | |
220 | err = -ENOMEM; | |
221 | break; | |
222 | } | |
75a52216 GX |
223 | set_page_private(rq->out[no], |
224 | Z_EROFS_SHORTLIVED_PAGE); | |
225 | } | |
622ceadd GX |
226 | if (rq->out[no]) |
227 | strm->buf.out = kmap(rq->out[no]) + pageofs; | |
228 | pageofs = 0; | |
229 | } else if (strm->buf.in_pos == strm->buf.in_size) { | |
230 | kunmap(rq->in[ni]); | |
231 | ||
232 | if (++ni >= nrpages_in || !inlen) { | |
233 | erofs_err(rq->sb, "compressed buf out of bound"); | |
234 | err = -EFSCORRUPTED; | |
235 | break; | |
236 | } | |
237 | strm->buf.in_pos = 0; | |
238 | strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE); | |
239 | inlen -= strm->buf.in_size; | |
240 | kin = kmap(rq->in[ni]); | |
241 | strm->buf.in = kin; | |
242 | bounced = false; | |
243 | } | |
244 | ||
245 | /* | |
246 | * Handle overlapping: Use bounced buffer if the compressed | |
247 | * data is under processing; Otherwise, Use short-lived pages | |
248 | * from the on-stack pagepool where pages share with the same | |
249 | * request. | |
250 | */ | |
251 | if (!bounced && rq->out[no] == rq->in[ni]) { | |
252 | memcpy(strm->bounce, strm->buf.in, strm->buf.in_size); | |
253 | strm->buf.in = strm->bounce; | |
254 | bounced = true; | |
255 | } | |
256 | for (j = ni + 1; j < nrpages_in; ++j) { | |
257 | struct page *tmppage; | |
258 | ||
259 | if (rq->out[no] != rq->in[j]) | |
260 | continue; | |
261 | ||
262 | DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb), | |
263 | rq->in[j])); | |
d9281660 CG |
264 | tmppage = erofs_allocpage(pgpl, rq->gfp); |
265 | if (!tmppage) { | |
266 | err = -ENOMEM; | |
267 | goto failed; | |
268 | } | |
622ceadd GX |
269 | set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); |
270 | copy_highpage(tmppage, rq->in[j]); | |
271 | rq->in[j] = tmppage; | |
272 | } | |
273 | xz_err = xz_dec_microlzma_run(strm->state, &strm->buf); | |
274 | DBG_BUGON(strm->buf.out_pos > strm->buf.out_size); | |
275 | DBG_BUGON(strm->buf.in_pos > strm->buf.in_size); | |
276 | ||
277 | if (xz_err != XZ_OK) { | |
278 | if (xz_err == XZ_STREAM_END && !outlen) | |
279 | break; | |
280 | erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]", | |
281 | xz_err, rq->inputsize, rq->outputsize); | |
282 | err = -EFSCORRUPTED; | |
283 | break; | |
284 | } | |
285 | } | |
d9281660 | 286 | failed: |
622ceadd | 287 | if (no < nrpages_out && strm->buf.out) |
8f121dfb | 288 | kunmap(rq->out[no]); |
622ceadd GX |
289 | if (ni < nrpages_in) |
290 | kunmap(rq->in[ni]); | |
291 | /* 4. push back LZMA stream context to the global list */ | |
292 | spin_lock(&z_erofs_lzma_lock); | |
293 | strm->next = z_erofs_lzma_head; | |
294 | z_erofs_lzma_head = strm; | |
295 | spin_unlock(&z_erofs_lzma_lock); | |
296 | wake_up(&z_erofs_lzma_wq); | |
297 | return err; | |
298 | } |