Commit | Line | Data |
---|---|---|
a6fa6fae LZ |
1 | /* |
2 | * Copyright (C) 2008 Oracle. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public | |
6 | * License v2 as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public | |
14 | * License along with this program; if not, write to the | |
15 | * Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
16 | * Boston, MA 021110-1307, USA. | |
17 | */ | |
18 | ||
19 | #include <linux/kernel.h> | |
20 | #include <linux/slab.h> | |
6acafd1e | 21 | #include <linux/mm.h> |
a6fa6fae LZ |
22 | #include <linux/init.h> |
23 | #include <linux/err.h> | |
24 | #include <linux/sched.h> | |
25 | #include <linux/pagemap.h> | |
26 | #include <linux/bio.h> | |
27 | #include <linux/lzo.h> | |
e1ddce71 | 28 | #include <linux/refcount.h> |
a6fa6fae LZ |
29 | #include "compression.h" |
30 | ||
31 | #define LZO_LEN 4 | |
32 | ||
33 | struct workspace { | |
34 | void *mem; | |
3fb40375 JL |
35 | void *buf; /* where decompressed data goes */ |
36 | void *cbuf; /* where compressed data goes */ | |
a6fa6fae LZ |
37 | struct list_head list; |
38 | }; | |
39 | ||
40 | static void lzo_free_workspace(struct list_head *ws) | |
41 | { | |
42 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
43 | ||
6acafd1e DS |
44 | kvfree(workspace->buf); |
45 | kvfree(workspace->cbuf); | |
46 | kvfree(workspace->mem); | |
a6fa6fae LZ |
47 | kfree(workspace); |
48 | } | |
49 | ||
50 | static struct list_head *lzo_alloc_workspace(void) | |
51 | { | |
52 | struct workspace *workspace; | |
53 | ||
389a6cfc | 54 | workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); |
a6fa6fae LZ |
55 | if (!workspace) |
56 | return ERR_PTR(-ENOMEM); | |
57 | ||
6acafd1e DS |
58 | workspace->mem = kvmalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL); |
59 | workspace->buf = kvmalloc(lzo1x_worst_compress(PAGE_SIZE), GFP_KERNEL); | |
60 | workspace->cbuf = kvmalloc(lzo1x_worst_compress(PAGE_SIZE), GFP_KERNEL); | |
a6fa6fae LZ |
61 | if (!workspace->mem || !workspace->buf || !workspace->cbuf) |
62 | goto fail; | |
63 | ||
64 | INIT_LIST_HEAD(&workspace->list); | |
65 | ||
66 | return &workspace->list; | |
67 | fail: | |
68 | lzo_free_workspace(&workspace->list); | |
69 | return ERR_PTR(-ENOMEM); | |
70 | } | |
71 | ||
72 | static inline void write_compress_length(char *buf, size_t len) | |
73 | { | |
74 | __le32 dlen; | |
75 | ||
76 | dlen = cpu_to_le32(len); | |
77 | memcpy(buf, &dlen, LZO_LEN); | |
78 | } | |
79 | ||
14a3357b | 80 | static inline size_t read_compress_length(const char *buf) |
a6fa6fae LZ |
81 | { |
82 | __le32 dlen; | |
83 | ||
84 | memcpy(&dlen, buf, LZO_LEN); | |
85 | return le32_to_cpu(dlen); | |
86 | } | |
87 | ||
88 | static int lzo_compress_pages(struct list_head *ws, | |
89 | struct address_space *mapping, | |
38c31464 | 90 | u64 start, |
a6fa6fae | 91 | struct page **pages, |
a6fa6fae LZ |
92 | unsigned long *out_pages, |
93 | unsigned long *total_in, | |
e5d74902 | 94 | unsigned long *total_out) |
a6fa6fae LZ |
95 | { |
96 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
97 | int ret = 0; | |
98 | char *data_in; | |
99 | char *cpage_out; | |
100 | int nr_pages = 0; | |
101 | struct page *in_page = NULL; | |
102 | struct page *out_page = NULL; | |
103 | unsigned long bytes_left; | |
38c31464 | 104 | unsigned long len = *total_out; |
4d3a800e | 105 | unsigned long nr_dest_pages = *out_pages; |
e5d74902 | 106 | const unsigned long max_out = nr_dest_pages * PAGE_SIZE; |
a6fa6fae LZ |
107 | size_t in_len; |
108 | size_t out_len; | |
109 | char *buf; | |
110 | unsigned long tot_in = 0; | |
111 | unsigned long tot_out = 0; | |
112 | unsigned long pg_bytes_left; | |
113 | unsigned long out_offset; | |
114 | unsigned long bytes; | |
115 | ||
116 | *out_pages = 0; | |
117 | *total_out = 0; | |
118 | *total_in = 0; | |
119 | ||
09cbfeaf | 120 | in_page = find_get_page(mapping, start >> PAGE_SHIFT); |
a6fa6fae LZ |
121 | data_in = kmap(in_page); |
122 | ||
123 | /* | |
124 | * store the size of all chunks of compressed data in | |
125 | * the first 4 bytes | |
126 | */ | |
127 | out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); | |
128 | if (out_page == NULL) { | |
129 | ret = -ENOMEM; | |
130 | goto out; | |
131 | } | |
132 | cpage_out = kmap(out_page); | |
133 | out_offset = LZO_LEN; | |
134 | tot_out = LZO_LEN; | |
135 | pages[0] = out_page; | |
136 | nr_pages = 1; | |
09cbfeaf | 137 | pg_bytes_left = PAGE_SIZE - LZO_LEN; |
a6fa6fae LZ |
138 | |
139 | /* compress at most one page of data each time */ | |
09cbfeaf | 140 | in_len = min(len, PAGE_SIZE); |
a6fa6fae LZ |
141 | while (tot_in < len) { |
142 | ret = lzo1x_1_compress(data_in, in_len, workspace->cbuf, | |
143 | &out_len, workspace->mem); | |
144 | if (ret != LZO_E_OK) { | |
036b0217 | 145 | pr_debug("BTRFS: lzo in loop returned %d\n", |
a6fa6fae | 146 | ret); |
60e1975a | 147 | ret = -EIO; |
a6fa6fae LZ |
148 | goto out; |
149 | } | |
150 | ||
151 | /* store the size of this chunk of compressed data */ | |
152 | write_compress_length(cpage_out + out_offset, out_len); | |
153 | tot_out += LZO_LEN; | |
154 | out_offset += LZO_LEN; | |
155 | pg_bytes_left -= LZO_LEN; | |
156 | ||
157 | tot_in += in_len; | |
158 | tot_out += out_len; | |
159 | ||
160 | /* copy bytes from the working buffer into the pages */ | |
161 | buf = workspace->cbuf; | |
162 | while (out_len) { | |
163 | bytes = min_t(unsigned long, pg_bytes_left, out_len); | |
164 | ||
165 | memcpy(cpage_out + out_offset, buf, bytes); | |
166 | ||
167 | out_len -= bytes; | |
168 | pg_bytes_left -= bytes; | |
169 | buf += bytes; | |
170 | out_offset += bytes; | |
171 | ||
172 | /* | |
173 | * we need another page for writing out. | |
174 | * | |
175 | * Note if there's less than 4 bytes left, we just | |
176 | * skip to a new page. | |
177 | */ | |
178 | if ((out_len == 0 && pg_bytes_left < LZO_LEN) || | |
179 | pg_bytes_left == 0) { | |
180 | if (pg_bytes_left) { | |
181 | memset(cpage_out + out_offset, 0, | |
182 | pg_bytes_left); | |
183 | tot_out += pg_bytes_left; | |
184 | } | |
185 | ||
186 | /* we're done, don't allocate new page */ | |
187 | if (out_len == 0 && tot_in >= len) | |
188 | break; | |
189 | ||
190 | kunmap(out_page); | |
191 | if (nr_pages == nr_dest_pages) { | |
192 | out_page = NULL; | |
60e1975a | 193 | ret = -E2BIG; |
a6fa6fae LZ |
194 | goto out; |
195 | } | |
196 | ||
197 | out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); | |
198 | if (out_page == NULL) { | |
199 | ret = -ENOMEM; | |
200 | goto out; | |
201 | } | |
202 | cpage_out = kmap(out_page); | |
203 | pages[nr_pages++] = out_page; | |
204 | ||
09cbfeaf | 205 | pg_bytes_left = PAGE_SIZE; |
a6fa6fae LZ |
206 | out_offset = 0; |
207 | } | |
208 | } | |
209 | ||
210 | /* we're making it bigger, give up */ | |
59516f60 | 211 | if (tot_in > 8192 && tot_in < tot_out) { |
60e1975a | 212 | ret = -E2BIG; |
a6fa6fae | 213 | goto out; |
59516f60 | 214 | } |
a6fa6fae LZ |
215 | |
216 | /* we're all done */ | |
217 | if (tot_in >= len) | |
218 | break; | |
219 | ||
220 | if (tot_out > max_out) | |
221 | break; | |
222 | ||
223 | bytes_left = len - tot_in; | |
224 | kunmap(in_page); | |
09cbfeaf | 225 | put_page(in_page); |
a6fa6fae | 226 | |
09cbfeaf KS |
227 | start += PAGE_SIZE; |
228 | in_page = find_get_page(mapping, start >> PAGE_SHIFT); | |
a6fa6fae | 229 | data_in = kmap(in_page); |
09cbfeaf | 230 | in_len = min(bytes_left, PAGE_SIZE); |
a6fa6fae LZ |
231 | } |
232 | ||
1e9d7291 TT |
233 | if (tot_out >= tot_in) { |
234 | ret = -E2BIG; | |
a6fa6fae | 235 | goto out; |
1e9d7291 | 236 | } |
a6fa6fae LZ |
237 | |
238 | /* store the size of all chunks of compressed data */ | |
239 | cpage_out = kmap(pages[0]); | |
240 | write_compress_length(cpage_out, tot_out); | |
241 | ||
242 | kunmap(pages[0]); | |
243 | ||
244 | ret = 0; | |
245 | *total_out = tot_out; | |
246 | *total_in = tot_in; | |
247 | out: | |
248 | *out_pages = nr_pages; | |
249 | if (out_page) | |
250 | kunmap(out_page); | |
251 | ||
252 | if (in_page) { | |
253 | kunmap(in_page); | |
09cbfeaf | 254 | put_page(in_page); |
a6fa6fae LZ |
255 | } |
256 | ||
257 | return ret; | |
258 | } | |
259 | ||
e1ddce71 | 260 | static int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb) |
a6fa6fae LZ |
261 | { |
262 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
3a39c18d | 263 | int ret = 0, ret2; |
a6fa6fae | 264 | char *data_in; |
a6fa6fae | 265 | unsigned long page_in_index = 0; |
e1ddce71 | 266 | size_t srclen = cb->compressed_len; |
09cbfeaf | 267 | unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); |
a6fa6fae LZ |
268 | unsigned long buf_start; |
269 | unsigned long buf_offset = 0; | |
270 | unsigned long bytes; | |
271 | unsigned long working_bytes; | |
a6fa6fae LZ |
272 | size_t in_len; |
273 | size_t out_len; | |
274 | unsigned long in_offset; | |
275 | unsigned long in_page_bytes_left; | |
276 | unsigned long tot_in; | |
277 | unsigned long tot_out; | |
278 | unsigned long tot_len; | |
279 | char *buf; | |
ca9b688c | 280 | bool may_late_unmap, need_unmap; |
e1ddce71 AJ |
281 | struct page **pages_in = cb->compressed_pages; |
282 | u64 disk_start = cb->start; | |
283 | struct bio *orig_bio = cb->orig_bio; | |
a6fa6fae LZ |
284 | |
285 | data_in = kmap(pages_in[0]); | |
286 | tot_len = read_compress_length(data_in); | |
287 | ||
288 | tot_in = LZO_LEN; | |
289 | in_offset = LZO_LEN; | |
290 | tot_len = min_t(size_t, srclen, tot_len); | |
09cbfeaf | 291 | in_page_bytes_left = PAGE_SIZE - LZO_LEN; |
a6fa6fae LZ |
292 | |
293 | tot_out = 0; | |
a6fa6fae LZ |
294 | |
295 | while (tot_in < tot_len) { | |
296 | in_len = read_compress_length(data_in + in_offset); | |
297 | in_page_bytes_left -= LZO_LEN; | |
298 | in_offset += LZO_LEN; | |
299 | tot_in += LZO_LEN; | |
300 | ||
301 | tot_in += in_len; | |
302 | working_bytes = in_len; | |
ca9b688c | 303 | may_late_unmap = need_unmap = false; |
a6fa6fae LZ |
304 | |
305 | /* fast path: avoid using the working buffer */ | |
306 | if (in_page_bytes_left >= in_len) { | |
307 | buf = data_in + in_offset; | |
308 | bytes = in_len; | |
ca9b688c | 309 | may_late_unmap = true; |
a6fa6fae LZ |
310 | goto cont; |
311 | } | |
312 | ||
313 | /* copy bytes from the pages into the working buffer */ | |
314 | buf = workspace->cbuf; | |
315 | buf_offset = 0; | |
316 | while (working_bytes) { | |
317 | bytes = min(working_bytes, in_page_bytes_left); | |
318 | ||
319 | memcpy(buf + buf_offset, data_in + in_offset, bytes); | |
320 | buf_offset += bytes; | |
321 | cont: | |
322 | working_bytes -= bytes; | |
323 | in_page_bytes_left -= bytes; | |
324 | in_offset += bytes; | |
325 | ||
326 | /* check if we need to pick another page */ | |
327 | if ((working_bytes == 0 && in_page_bytes_left < LZO_LEN) | |
328 | || in_page_bytes_left == 0) { | |
329 | tot_in += in_page_bytes_left; | |
330 | ||
331 | if (working_bytes == 0 && tot_in >= tot_len) | |
332 | break; | |
333 | ||
ca9b688c | 334 | if (page_in_index + 1 >= total_pages_in) { |
60e1975a | 335 | ret = -EIO; |
a6fa6fae LZ |
336 | goto done; |
337 | } | |
ca9b688c LZ |
338 | |
339 | if (may_late_unmap) | |
340 | need_unmap = true; | |
341 | else | |
342 | kunmap(pages_in[page_in_index]); | |
343 | ||
344 | data_in = kmap(pages_in[++page_in_index]); | |
a6fa6fae | 345 | |
09cbfeaf | 346 | in_page_bytes_left = PAGE_SIZE; |
a6fa6fae LZ |
347 | in_offset = 0; |
348 | } | |
349 | } | |
350 | ||
09cbfeaf | 351 | out_len = lzo1x_worst_compress(PAGE_SIZE); |
a6fa6fae LZ |
352 | ret = lzo1x_decompress_safe(buf, in_len, workspace->buf, |
353 | &out_len); | |
ca9b688c LZ |
354 | if (need_unmap) |
355 | kunmap(pages_in[page_in_index - 1]); | |
a6fa6fae | 356 | if (ret != LZO_E_OK) { |
62e85577 | 357 | pr_warn("BTRFS: decompress failed\n"); |
60e1975a | 358 | ret = -EIO; |
a6fa6fae LZ |
359 | break; |
360 | } | |
361 | ||
a6fa6fae | 362 | buf_start = tot_out; |
a6fa6fae LZ |
363 | tot_out += out_len; |
364 | ||
3a39c18d | 365 | ret2 = btrfs_decompress_buf2page(workspace->buf, buf_start, |
974b1adc | 366 | tot_out, disk_start, orig_bio); |
3a39c18d | 367 | if (ret2 == 0) |
a6fa6fae | 368 | break; |
a6fa6fae LZ |
369 | } |
370 | done: | |
ca9b688c | 371 | kunmap(pages_in[page_in_index]); |
2f19cad9 | 372 | if (!ret) |
974b1adc | 373 | zero_fill_bio(orig_bio); |
a6fa6fae LZ |
374 | return ret; |
375 | } | |
376 | ||
377 | static int lzo_decompress(struct list_head *ws, unsigned char *data_in, | |
378 | struct page *dest_page, | |
379 | unsigned long start_byte, | |
380 | size_t srclen, size_t destlen) | |
381 | { | |
382 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
383 | size_t in_len; | |
384 | size_t out_len; | |
385 | size_t tot_len; | |
386 | int ret = 0; | |
387 | char *kaddr; | |
388 | unsigned long bytes; | |
389 | ||
390 | BUG_ON(srclen < LZO_LEN); | |
391 | ||
392 | tot_len = read_compress_length(data_in); | |
393 | data_in += LZO_LEN; | |
394 | ||
395 | in_len = read_compress_length(data_in); | |
396 | data_in += LZO_LEN; | |
397 | ||
09cbfeaf | 398 | out_len = PAGE_SIZE; |
a6fa6fae LZ |
399 | ret = lzo1x_decompress_safe(data_in, in_len, workspace->buf, &out_len); |
400 | if (ret != LZO_E_OK) { | |
62e85577 | 401 | pr_warn("BTRFS: decompress failed!\n"); |
60e1975a | 402 | ret = -EIO; |
a6fa6fae LZ |
403 | goto out; |
404 | } | |
405 | ||
406 | if (out_len < start_byte) { | |
60e1975a | 407 | ret = -EIO; |
a6fa6fae LZ |
408 | goto out; |
409 | } | |
410 | ||
2f19cad9 CM |
411 | /* |
412 | * the caller is already checking against PAGE_SIZE, but lets | |
413 | * move this check closer to the memcpy/memset | |
414 | */ | |
415 | destlen = min_t(unsigned long, destlen, PAGE_SIZE); | |
a6fa6fae LZ |
416 | bytes = min_t(unsigned long, destlen, out_len - start_byte); |
417 | ||
7ac687d9 | 418 | kaddr = kmap_atomic(dest_page); |
a6fa6fae | 419 | memcpy(kaddr, workspace->buf + start_byte, bytes); |
2f19cad9 CM |
420 | |
421 | /* | |
422 | * btrfs_getblock is doing a zero on the tail of the page too, | |
423 | * but this will cover anything missing from the decompressed | |
424 | * data. | |
425 | */ | |
426 | if (bytes < destlen) | |
427 | memset(kaddr+bytes, 0, destlen-bytes); | |
7ac687d9 | 428 | kunmap_atomic(kaddr); |
a6fa6fae LZ |
429 | out: |
430 | return ret; | |
431 | } | |
432 | ||
e8c9f186 | 433 | const struct btrfs_compress_op btrfs_lzo_compress = { |
a6fa6fae LZ |
434 | .alloc_workspace = lzo_alloc_workspace, |
435 | .free_workspace = lzo_free_workspace, | |
436 | .compress_pages = lzo_compress_pages, | |
974b1adc | 437 | .decompress_bio = lzo_decompress_bio, |
a6fa6fae LZ |
438 | .decompress = lzo_decompress, |
439 | }; |