Commit | Line | Data |
---|---|---|
c1d7c514 | 1 | // SPDX-License-Identifier: GPL-2.0 |
c8b97818 CM |
2 | /* |
3 | * Copyright (C) 2008 Oracle. All rights reserved. | |
4 | * | |
c8b97818 CM |
5 | * Based on jffs2 zlib code: |
6 | * Copyright © 2001-2007 Red Hat, Inc. | |
7 | * Created by David Woodhouse <dwmw2@infradead.org> | |
8 | */ | |
9 | ||
10 | #include <linux/kernel.h> | |
11 | #include <linux/slab.h> | |
12 | #include <linux/zlib.h> | |
13 | #include <linux/zutil.h> | |
6acafd1e | 14 | #include <linux/mm.h> |
c8b97818 CM |
15 | #include <linux/init.h> |
16 | #include <linux/err.h> | |
17 | #include <linux/sched.h> | |
18 | #include <linux/pagemap.h> | |
19 | #include <linux/bio.h> | |
e1ddce71 | 20 | #include <linux/refcount.h> |
b2950863 | 21 | #include "compression.h" |
c8b97818 | 22 | |
3fd396af MZ |
23 | /* workspace buffer size for s390 zlib hardware support */ |
24 | #define ZLIB_DFLTCC_BUF_SIZE (4 * PAGE_SIZE) | |
25 | ||
c8b97818 | 26 | struct workspace { |
78809913 | 27 | z_stream strm; |
c8b97818 | 28 | char *buf; |
3fd396af | 29 | unsigned int buf_size; |
c8b97818 | 30 | struct list_head list; |
f51d2b59 | 31 | int level; |
c8b97818 CM |
32 | }; |
33 | ||
92ee5530 DZ |
34 | static struct workspace_manager wsm; |
35 | ||
d20f395f | 36 | struct list_head *zlib_get_workspace(unsigned int level) |
92ee5530 | 37 | { |
5907a9bb | 38 | struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level); |
d0ab62ce DZ |
39 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
40 | ||
41 | workspace->level = level; | |
42 | ||
43 | return ws; | |
92ee5530 DZ |
44 | } |
45 | ||
d20f395f | 46 | void zlib_free_workspace(struct list_head *ws) |
261507a0 LZ |
47 | { |
48 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
c8b97818 | 49 | |
6acafd1e | 50 | kvfree(workspace->strm.workspace); |
261507a0 LZ |
51 | kfree(workspace->buf); |
52 | kfree(workspace); | |
53 | } | |
54 | ||
d20f395f | 55 | struct list_head *zlib_alloc_workspace(unsigned int level) |
c8b97818 CM |
56 | { |
57 | struct workspace *workspace; | |
78809913 | 58 | int workspacesize; |
8844355d | 59 | |
389a6cfc | 60 | workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); |
261507a0 LZ |
61 | if (!workspace) |
62 | return ERR_PTR(-ENOMEM); | |
c8b97818 | 63 | |
78809913 SS |
64 | workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL), |
65 | zlib_inflate_workspacesize()); | |
6acafd1e | 66 | workspace->strm.workspace = kvmalloc(workspacesize, GFP_KERNEL); |
7bf49943 | 67 | workspace->level = level; |
3fd396af MZ |
68 | workspace->buf = NULL; |
69 | /* | |
70 | * In case of s390 zlib hardware support, allocate lager workspace | |
71 | * buffer. If allocator fails, fall back to a single page buffer. | |
72 | */ | |
73 | if (zlib_deflate_dfltcc_enabled()) { | |
74 | workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE, | |
75 | __GFP_NOMEMALLOC | __GFP_NORETRY | | |
76 | __GFP_NOWARN | GFP_NOIO); | |
77 | workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE; | |
78 | } | |
79 | if (!workspace->buf) { | |
80 | workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); | |
81 | workspace->buf_size = PAGE_SIZE; | |
82 | } | |
78809913 | 83 | if (!workspace->strm.workspace || !workspace->buf) |
261507a0 | 84 | goto fail; |
c8b97818 | 85 | |
261507a0 | 86 | INIT_LIST_HEAD(&workspace->list); |
c8b97818 | 87 | |
261507a0 LZ |
88 | return &workspace->list; |
89 | fail: | |
90 | zlib_free_workspace(&workspace->list); | |
91 | return ERR_PTR(-ENOMEM); | |
c8b97818 CM |
92 | } |
93 | ||
c4bf665a DS |
94 | int zlib_compress_pages(struct list_head *ws, struct address_space *mapping, |
95 | u64 start, struct page **pages, unsigned long *out_pages, | |
96 | unsigned long *total_in, unsigned long *total_out) | |
c8b97818 | 97 | { |
261507a0 | 98 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
c8b97818 | 99 | int ret; |
c8b97818 CM |
100 | char *data_in; |
101 | char *cpage_out; | |
102 | int nr_pages = 0; | |
103 | struct page *in_page = NULL; | |
104 | struct page *out_page = NULL; | |
c8b97818 | 105 | unsigned long bytes_left; |
3fd396af | 106 | unsigned int in_buf_pages; |
38c31464 | 107 | unsigned long len = *total_out; |
4d3a800e | 108 | unsigned long nr_dest_pages = *out_pages; |
e5d74902 | 109 | const unsigned long max_out = nr_dest_pages * PAGE_SIZE; |
c8b97818 CM |
110 | |
111 | *out_pages = 0; | |
112 | *total_out = 0; | |
113 | *total_in = 0; | |
114 | ||
f51d2b59 | 115 | if (Z_OK != zlib_deflateInit(&workspace->strm, workspace->level)) { |
62e85577 | 116 | pr_warn("BTRFS: deflateInit failed\n"); |
60e1975a | 117 | ret = -EIO; |
c8b97818 CM |
118 | goto out; |
119 | } | |
120 | ||
78809913 SS |
121 | workspace->strm.total_in = 0; |
122 | workspace->strm.total_out = 0; | |
c8b97818 | 123 | |
b0ee5e1e | 124 | out_page = alloc_page(GFP_NOFS); |
4b72029d | 125 | if (out_page == NULL) { |
60e1975a | 126 | ret = -ENOMEM; |
4b72029d LZ |
127 | goto out; |
128 | } | |
696ab562 | 129 | cpage_out = page_address(out_page); |
c8b97818 CM |
130 | pages[0] = out_page; |
131 | nr_pages = 1; | |
132 | ||
3fd396af MZ |
133 | workspace->strm.next_in = workspace->buf; |
134 | workspace->strm.avail_in = 0; | |
78809913 | 135 | workspace->strm.next_out = cpage_out; |
09cbfeaf | 136 | workspace->strm.avail_out = PAGE_SIZE; |
c8b97818 | 137 | |
78809913 | 138 | while (workspace->strm.total_in < len) { |
3fd396af MZ |
139 | /* |
140 | * Get next input pages and copy the contents to | |
141 | * the workspace buffer if required. | |
142 | */ | |
143 | if (workspace->strm.avail_in == 0) { | |
144 | bytes_left = len - workspace->strm.total_in; | |
145 | in_buf_pages = min(DIV_ROUND_UP(bytes_left, PAGE_SIZE), | |
146 | workspace->buf_size / PAGE_SIZE); | |
147 | if (in_buf_pages > 1) { | |
148 | int i; | |
149 | ||
150 | for (i = 0; i < in_buf_pages; i++) { | |
696ab562 | 151 | if (in_page) |
3fd396af | 152 | put_page(in_page); |
3fd396af MZ |
153 | in_page = find_get_page(mapping, |
154 | start >> PAGE_SHIFT); | |
696ab562 | 155 | data_in = page_address(in_page); |
3fd396af MZ |
156 | memcpy(workspace->buf + i * PAGE_SIZE, |
157 | data_in, PAGE_SIZE); | |
158 | start += PAGE_SIZE; | |
159 | } | |
160 | workspace->strm.next_in = workspace->buf; | |
161 | } else { | |
696ab562 | 162 | if (in_page) |
3fd396af | 163 | put_page(in_page); |
3fd396af MZ |
164 | in_page = find_get_page(mapping, |
165 | start >> PAGE_SHIFT); | |
696ab562 | 166 | data_in = page_address(in_page); |
3fd396af MZ |
167 | start += PAGE_SIZE; |
168 | workspace->strm.next_in = data_in; | |
169 | } | |
170 | workspace->strm.avail_in = min(bytes_left, | |
171 | (unsigned long) workspace->buf_size); | |
172 | } | |
173 | ||
78809913 | 174 | ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH); |
c8b97818 | 175 | if (ret != Z_OK) { |
62e85577 | 176 | pr_debug("BTRFS: deflate in loop returned %d\n", |
c8b97818 | 177 | ret); |
78809913 | 178 | zlib_deflateEnd(&workspace->strm); |
60e1975a | 179 | ret = -EIO; |
c8b97818 CM |
180 | goto out; |
181 | } | |
182 | ||
183 | /* we're making it bigger, give up */ | |
78809913 SS |
184 | if (workspace->strm.total_in > 8192 && |
185 | workspace->strm.total_in < | |
186 | workspace->strm.total_out) { | |
130d5b41 | 187 | ret = -E2BIG; |
c8b97818 CM |
188 | goto out; |
189 | } | |
190 | /* we need another page for writing out. Test this | |
191 | * before the total_in so we will pull in a new page for | |
192 | * the stream end if required | |
193 | */ | |
78809913 | 194 | if (workspace->strm.avail_out == 0) { |
c8b97818 CM |
195 | if (nr_pages == nr_dest_pages) { |
196 | out_page = NULL; | |
60e1975a | 197 | ret = -E2BIG; |
c8b97818 CM |
198 | goto out; |
199 | } | |
b0ee5e1e | 200 | out_page = alloc_page(GFP_NOFS); |
4b72029d | 201 | if (out_page == NULL) { |
60e1975a | 202 | ret = -ENOMEM; |
4b72029d LZ |
203 | goto out; |
204 | } | |
696ab562 | 205 | cpage_out = page_address(out_page); |
c8b97818 CM |
206 | pages[nr_pages] = out_page; |
207 | nr_pages++; | |
09cbfeaf | 208 | workspace->strm.avail_out = PAGE_SIZE; |
78809913 | 209 | workspace->strm.next_out = cpage_out; |
c8b97818 CM |
210 | } |
211 | /* we're all done */ | |
78809913 | 212 | if (workspace->strm.total_in >= len) |
c8b97818 | 213 | break; |
3fd396af MZ |
214 | if (workspace->strm.total_out > max_out) |
215 | break; | |
c8b97818 | 216 | } |
78809913 | 217 | workspace->strm.avail_in = 0; |
3fd396af MZ |
218 | /* |
219 | * Call deflate with Z_FINISH flush parameter providing more output | |
220 | * space but no more input data, until it returns with Z_STREAM_END. | |
221 | */ | |
222 | while (ret != Z_STREAM_END) { | |
223 | ret = zlib_deflate(&workspace->strm, Z_FINISH); | |
224 | if (ret == Z_STREAM_END) | |
225 | break; | |
226 | if (ret != Z_OK && ret != Z_BUF_ERROR) { | |
227 | zlib_deflateEnd(&workspace->strm); | |
228 | ret = -EIO; | |
229 | goto out; | |
230 | } else if (workspace->strm.avail_out == 0) { | |
231 | /* get another page for the stream end */ | |
3fd396af MZ |
232 | if (nr_pages == nr_dest_pages) { |
233 | out_page = NULL; | |
234 | ret = -E2BIG; | |
235 | goto out; | |
236 | } | |
b0ee5e1e | 237 | out_page = alloc_page(GFP_NOFS); |
3fd396af MZ |
238 | if (out_page == NULL) { |
239 | ret = -ENOMEM; | |
240 | goto out; | |
241 | } | |
696ab562 | 242 | cpage_out = page_address(out_page); |
3fd396af MZ |
243 | pages[nr_pages] = out_page; |
244 | nr_pages++; | |
245 | workspace->strm.avail_out = PAGE_SIZE; | |
246 | workspace->strm.next_out = cpage_out; | |
247 | } | |
c8b97818 | 248 | } |
3fd396af | 249 | zlib_deflateEnd(&workspace->strm); |
c8b97818 | 250 | |
78809913 | 251 | if (workspace->strm.total_out >= workspace->strm.total_in) { |
60e1975a | 252 | ret = -E2BIG; |
c8b97818 CM |
253 | goto out; |
254 | } | |
255 | ||
256 | ret = 0; | |
78809913 SS |
257 | *total_out = workspace->strm.total_out; |
258 | *total_in = workspace->strm.total_in; | |
c8b97818 CM |
259 | out: |
260 | *out_pages = nr_pages; | |
696ab562 | 261 | if (in_page) |
09cbfeaf | 262 | put_page(in_page); |
c8b97818 CM |
263 | return ret; |
264 | } | |
265 | ||
c4bf665a | 266 | int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb) |
c8b97818 | 267 | { |
261507a0 | 268 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
3a39c18d | 269 | int ret = 0, ret2; |
c8b97818 | 270 | int wbits = MAX_WBITS; |
c8b97818 CM |
271 | char *data_in; |
272 | size_t total_out = 0; | |
c8b97818 | 273 | unsigned long page_in_index = 0; |
e1ddce71 | 274 | size_t srclen = cb->compressed_len; |
09cbfeaf | 275 | unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); |
c8b97818 | 276 | unsigned long buf_start; |
e1ddce71 AJ |
277 | struct page **pages_in = cb->compressed_pages; |
278 | u64 disk_start = cb->start; | |
279 | struct bio *orig_bio = cb->orig_bio; | |
c8b97818 | 280 | |
696ab562 | 281 | data_in = page_address(pages_in[page_in_index]); |
78809913 | 282 | workspace->strm.next_in = data_in; |
09cbfeaf | 283 | workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE); |
78809913 | 284 | workspace->strm.total_in = 0; |
c8b97818 | 285 | |
78809913 SS |
286 | workspace->strm.total_out = 0; |
287 | workspace->strm.next_out = workspace->buf; | |
3fd396af | 288 | workspace->strm.avail_out = workspace->buf_size; |
c8b97818 CM |
289 | |
290 | /* If it's deflate, and it's got no preset dictionary, then | |
291 | we can tell zlib to skip the adler32 check. */ | |
292 | if (srclen > 2 && !(data_in[1] & PRESET_DICT) && | |
293 | ((data_in[0] & 0x0f) == Z_DEFLATED) && | |
294 | !(((data_in[0]<<8) + data_in[1]) % 31)) { | |
295 | ||
296 | wbits = -((data_in[0] >> 4) + 8); | |
78809913 SS |
297 | workspace->strm.next_in += 2; |
298 | workspace->strm.avail_in -= 2; | |
c8b97818 CM |
299 | } |
300 | ||
78809913 | 301 | if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) { |
62e85577 | 302 | pr_warn("BTRFS: inflateInit failed\n"); |
60e1975a | 303 | return -EIO; |
c8b97818 | 304 | } |
78809913 SS |
305 | while (workspace->strm.total_in < srclen) { |
306 | ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH); | |
d397712b | 307 | if (ret != Z_OK && ret != Z_STREAM_END) |
c8b97818 | 308 | break; |
c8b97818 | 309 | |
3a39c18d | 310 | buf_start = total_out; |
78809913 | 311 | total_out = workspace->strm.total_out; |
c8b97818 | 312 | |
3a39c18d LZ |
313 | /* we didn't make progress in this inflate call, we're done */ |
314 | if (buf_start == total_out) | |
c8b97818 | 315 | break; |
c8b97818 | 316 | |
3a39c18d LZ |
317 | ret2 = btrfs_decompress_buf2page(workspace->buf, buf_start, |
318 | total_out, disk_start, | |
974b1adc | 319 | orig_bio); |
3a39c18d LZ |
320 | if (ret2 == 0) { |
321 | ret = 0; | |
322 | goto done; | |
c8b97818 | 323 | } |
3a39c18d | 324 | |
78809913 | 325 | workspace->strm.next_out = workspace->buf; |
3fd396af | 326 | workspace->strm.avail_out = workspace->buf_size; |
c8b97818 | 327 | |
78809913 | 328 | if (workspace->strm.avail_in == 0) { |
c8b97818 | 329 | unsigned long tmp; |
696ab562 | 330 | |
c8b97818 CM |
331 | page_in_index++; |
332 | if (page_in_index >= total_pages_in) { | |
333 | data_in = NULL; | |
334 | break; | |
335 | } | |
696ab562 | 336 | data_in = page_address(pages_in[page_in_index]); |
78809913 SS |
337 | workspace->strm.next_in = data_in; |
338 | tmp = srclen - workspace->strm.total_in; | |
339 | workspace->strm.avail_in = min(tmp, | |
09cbfeaf | 340 | PAGE_SIZE); |
c8b97818 CM |
341 | } |
342 | } | |
d397712b | 343 | if (ret != Z_STREAM_END) |
60e1975a | 344 | ret = -EIO; |
d397712b | 345 | else |
c8b97818 | 346 | ret = 0; |
c8b97818 | 347 | done: |
78809913 | 348 | zlib_inflateEnd(&workspace->strm); |
2f19cad9 | 349 | if (!ret) |
974b1adc | 350 | zero_fill_bio(orig_bio); |
c8b97818 CM |
351 | return ret; |
352 | } | |
353 | ||
c4bf665a DS |
354 | int zlib_decompress(struct list_head *ws, unsigned char *data_in, |
355 | struct page *dest_page, unsigned long start_byte, size_t srclen, | |
356 | size_t destlen) | |
c8b97818 | 357 | { |
261507a0 | 358 | struct workspace *workspace = list_entry(ws, struct workspace, list); |
c8b97818 CM |
359 | int ret = 0; |
360 | int wbits = MAX_WBITS; | |
2f19cad9 | 361 | unsigned long bytes_left; |
c8b97818 | 362 | unsigned long total_out = 0; |
2f19cad9 | 363 | unsigned long pg_offset = 0; |
c8b97818 | 364 | |
2f19cad9 CM |
365 | destlen = min_t(unsigned long, destlen, PAGE_SIZE); |
366 | bytes_left = destlen; | |
367 | ||
78809913 SS |
368 | workspace->strm.next_in = data_in; |
369 | workspace->strm.avail_in = srclen; | |
370 | workspace->strm.total_in = 0; | |
c8b97818 | 371 | |
78809913 | 372 | workspace->strm.next_out = workspace->buf; |
3fd396af | 373 | workspace->strm.avail_out = workspace->buf_size; |
78809913 | 374 | workspace->strm.total_out = 0; |
c8b97818 CM |
375 | /* If it's deflate, and it's got no preset dictionary, then |
376 | we can tell zlib to skip the adler32 check. */ | |
377 | if (srclen > 2 && !(data_in[1] & PRESET_DICT) && | |
378 | ((data_in[0] & 0x0f) == Z_DEFLATED) && | |
379 | !(((data_in[0]<<8) + data_in[1]) % 31)) { | |
380 | ||
381 | wbits = -((data_in[0] >> 4) + 8); | |
78809913 SS |
382 | workspace->strm.next_in += 2; |
383 | workspace->strm.avail_in -= 2; | |
c8b97818 CM |
384 | } |
385 | ||
78809913 | 386 | if (Z_OK != zlib_inflateInit2(&workspace->strm, wbits)) { |
62e85577 | 387 | pr_warn("BTRFS: inflateInit failed\n"); |
60e1975a | 388 | return -EIO; |
c8b97818 CM |
389 | } |
390 | ||
d397712b | 391 | while (bytes_left > 0) { |
c8b97818 CM |
392 | unsigned long buf_start; |
393 | unsigned long buf_offset; | |
394 | unsigned long bytes; | |
c8b97818 | 395 | |
78809913 | 396 | ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH); |
d397712b | 397 | if (ret != Z_OK && ret != Z_STREAM_END) |
c8b97818 | 398 | break; |
c8b97818 CM |
399 | |
400 | buf_start = total_out; | |
78809913 | 401 | total_out = workspace->strm.total_out; |
c8b97818 CM |
402 | |
403 | if (total_out == buf_start) { | |
60e1975a | 404 | ret = -EIO; |
c8b97818 CM |
405 | break; |
406 | } | |
407 | ||
d397712b | 408 | if (total_out <= start_byte) |
c8b97818 | 409 | goto next; |
c8b97818 | 410 | |
d397712b | 411 | if (total_out > start_byte && buf_start < start_byte) |
c8b97818 | 412 | buf_offset = start_byte - buf_start; |
d397712b | 413 | else |
c8b97818 | 414 | buf_offset = 0; |
c8b97818 | 415 | |
09cbfeaf | 416 | bytes = min(PAGE_SIZE - pg_offset, |
3fd396af | 417 | PAGE_SIZE - (buf_offset % PAGE_SIZE)); |
c8b97818 CM |
418 | bytes = min(bytes, bytes_left); |
419 | ||
3590ec58 IW |
420 | memcpy_to_page(dest_page, pg_offset, |
421 | workspace->buf + buf_offset, bytes); | |
c8b97818 CM |
422 | |
423 | pg_offset += bytes; | |
424 | bytes_left -= bytes; | |
425 | next: | |
78809913 | 426 | workspace->strm.next_out = workspace->buf; |
3fd396af | 427 | workspace->strm.avail_out = workspace->buf_size; |
c8b97818 | 428 | } |
d397712b CM |
429 | |
430 | if (ret != Z_STREAM_END && bytes_left != 0) | |
60e1975a | 431 | ret = -EIO; |
d397712b | 432 | else |
c8b97818 | 433 | ret = 0; |
d397712b | 434 | |
78809913 | 435 | zlib_inflateEnd(&workspace->strm); |
2f19cad9 CM |
436 | |
437 | /* | |
438 | * this should only happen if zlib returned fewer bytes than we | |
439 | * expected. btrfs_get_block is responsible for zeroing from the | |
440 | * end of the inline extent (destlen) to the end of the page | |
441 | */ | |
442 | if (pg_offset < destlen) { | |
d048b9c2 | 443 | memzero_page(dest_page, pg_offset, destlen - pg_offset); |
2f19cad9 | 444 | } |
c8b97818 CM |
445 | return ret; |
446 | } | |
447 | ||
e8c9f186 | 448 | const struct btrfs_compress_op btrfs_zlib_compress = { |
be951045 | 449 | .workspace_manager = &wsm, |
e18333a7 DS |
450 | .max_level = 9, |
451 | .default_level = BTRFS_ZLIB_DEFAULT_LEVEL, | |
261507a0 | 452 | }; |