projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'sound-5.7-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai...
[linux-2.6-block.git] / fs / f2fs / compress.c
CommitLineData
4c8ff709
CY		 1// SPDX-License-Identifier: GPL-2.0
2/*
3 * f2fs compress support
4 *
5 * Copyright (c) 2019 Chao Yu <chao@kernel.org>
6 */
7
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/writeback.h>
11#include <linux/backing-dev.h>
12#include <linux/lzo.h>
13#include <linux/lz4.h>
50cfa66f 14#include <linux/zstd.h>
4c8ff709
CY		 15
16#include "f2fs.h"
17#include "node.h"
18#include <trace/events/f2fs.h>
19
20struct f2fs_compress_ops {
21 int (*init_compress_ctx)(struct compress_ctx *cc);
22 void (*destroy_compress_ctx)(struct compress_ctx *cc);
23 int (*compress_pages)(struct compress_ctx *cc);
23b1faaa
CY		 24 int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
25 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
4c8ff709
CY		 26 int (*decompress_pages)(struct decompress_io_ctx *dic);
27};
28
29static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index)
30{
31 return index & (cc->cluster_size - 1);
32}
33
34static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index)
35{
36 return index >> cc->log_cluster_size;
37}
38
39static pgoff_t start_idx_of_cluster(struct compress_ctx *cc)
40{
41 return cc->cluster_idx << cc->log_cluster_size;
42}
43
44bool f2fs_is_compressed_page(struct page *page)
45{
46 if (!PagePrivate(page))
47 return false;
48 if (!page_private(page))
49 return false;
50 if (IS_ATOMIC_WRITTEN_PAGE(page) || IS_DUMMY_WRITTEN_PAGE(page))
51 return false;
52 f2fs_bug_on(F2FS_M_SB(page->mapping),
53 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC);
54 return true;
55}
56
57static void f2fs_set_compressed_page(struct page *page,
887347a0 58 struct inode *inode, pgoff_t index, void *data)
4c8ff709
CY		 59{
60 SetPagePrivate(page);
61 set_page_private(page, (unsigned long)data);
62
63 /* i_crypto_info and iv index */
64 page->index = index;
65 page->mapping = inode->i_mapping;
4c8ff709
CY		 66}
67
68static void f2fs_put_compressed_page(struct page *page)
69{
70 set_page_private(page, (unsigned long)NULL);
71 ClearPagePrivate(page);
72 page->mapping = NULL;
73 unlock_page(page);
74 put_page(page);
75}
76
77static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock)
78{
79 int i;
80
81 for (i = 0; i < len; i++) {
82 if (!cc->rpages[i])
83 continue;
84 if (unlock)
85 unlock_page(cc->rpages[i]);
86 else
87 put_page(cc->rpages[i]);
88 }
89}
90
91static void f2fs_put_rpages(struct compress_ctx *cc)
92{
93 f2fs_drop_rpages(cc, cc->cluster_size, false);
94}
95
96static void f2fs_unlock_rpages(struct compress_ctx *cc, int len)
97{
98 f2fs_drop_rpages(cc, len, true);
99}
100
101static void f2fs_put_rpages_mapping(struct compress_ctx *cc,
102 struct address_space *mapping,
103 pgoff_t start, int len)
104{
105 int i;
106
107 for (i = 0; i < len; i++) {
108 struct page *page = find_get_page(mapping, start + i);
109
110 put_page(page);
111 put_page(page);
112 }
113}
114
115static void f2fs_put_rpages_wbc(struct compress_ctx *cc,
116 struct writeback_control *wbc, bool redirty, int unlock)
117{
118 unsigned int i;
119
120 for (i = 0; i < cc->cluster_size; i++) {
121 if (!cc->rpages[i])
122 continue;
123 if (redirty)
124 redirty_page_for_writepage(wbc, cc->rpages[i]);
125 f2fs_put_page(cc->rpages[i], unlock);
126 }
127}
128
129struct page *f2fs_compress_control_page(struct page *page)
130{
131 return ((struct compress_io_ctx *)page_private(page))->rpages[0];
132}
133
134int f2fs_init_compress_ctx(struct compress_ctx *cc)
135{
136 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
137
138 if (cc->nr_rpages)
139 return 0;
140
141 cc->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
142 cc->log_cluster_size, GFP_NOFS);
143 return cc->rpages ? 0 : -ENOMEM;
144}
145
146void f2fs_destroy_compress_ctx(struct compress_ctx *cc)
147{
148 kfree(cc->rpages);
149 cc->rpages = NULL;
150 cc->nr_rpages = 0;
151 cc->nr_cpages = 0;
152 cc->cluster_idx = NULL_CLUSTER;
153}
154
155void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page)
156{
157 unsigned int cluster_ofs;
158
159 if (!f2fs_cluster_can_merge_page(cc, page->index))
160 f2fs_bug_on(F2FS_I_SB(cc->inode), 1);
161
162 cluster_ofs = offset_in_cluster(cc, page->index);
163 cc->rpages[cluster_ofs] = page;
164 cc->nr_rpages++;
165 cc->cluster_idx = cluster_idx(cc, page->index);
166}
167
168#ifdef CONFIG_F2FS_FS_LZO
169static int lzo_init_compress_ctx(struct compress_ctx *cc)
170{
171 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
172 LZO1X_MEM_COMPRESS, GFP_NOFS);
173 if (!cc->private)
174 return -ENOMEM;
175
176 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size);
177 return 0;
178}
179
180static void lzo_destroy_compress_ctx(struct compress_ctx *cc)
181{
182 kvfree(cc->private);
183 cc->private = NULL;
184}
185
186static int lzo_compress_pages(struct compress_ctx *cc)
187{
188 int ret;
189
190 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
191 &cc->clen, cc->private);
192 if (ret != LZO_E_OK) {
193 printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n",
194 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret);
195 return -EIO;
196 }
197 return 0;
198}
199
200static int lzo_decompress_pages(struct decompress_io_ctx *dic)
201{
202 int ret;
203
204 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,
205 dic->rbuf, &dic->rlen);
206 if (ret != LZO_E_OK) {
207 printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n",
208 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
209 return -EIO;
210 }
211
212 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {
213 printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, "
214 "expected:%lu\n", KERN_ERR,
215 F2FS_I_SB(dic->inode)->sb->s_id,
216 dic->rlen,
217 PAGE_SIZE << dic->log_cluster_size);
218 return -EIO;
219 }
220 return 0;
221}
222
223static const struct f2fs_compress_ops f2fs_lzo_ops = {
224 .init_compress_ctx = lzo_init_compress_ctx,
225 .destroy_compress_ctx = lzo_destroy_compress_ctx,
226 .compress_pages = lzo_compress_pages,
227 .decompress_pages = lzo_decompress_pages,
228};
229#endif
230
231#ifdef CONFIG_F2FS_FS_LZ4
232static int lz4_init_compress_ctx(struct compress_ctx *cc)
233{
234 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
235 LZ4_MEM_COMPRESS, GFP_NOFS);
236 if (!cc->private)
237 return -ENOMEM;
238
239 cc->clen = LZ4_compressBound(PAGE_SIZE << cc->log_cluster_size);
240 return 0;
241}
242
243static void lz4_destroy_compress_ctx(struct compress_ctx *cc)
244{
245 kvfree(cc->private);
246 cc->private = NULL;
247}
248
249static int lz4_compress_pages(struct compress_ctx *cc)
250{
251 int len;
252
253 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen,
254 cc->clen, cc->private);
255 if (!len) {
256 printk_ratelimited("%sF2FS-fs (%s): lz4 compress failed\n",
257 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id);
258 return -EIO;
259 }
260 cc->clen = len;
261 return 0;
262}
263
264static int lz4_decompress_pages(struct decompress_io_ctx *dic)
265{
266 int ret;
267
268 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,
269 dic->clen, dic->rlen);
270 if (ret < 0) {
271 printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n",
272 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
273 return -EIO;
274 }
275
276 if (ret != PAGE_SIZE << dic->log_cluster_size) {
277 printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, "
278 "expected:%lu\n", KERN_ERR,
279 F2FS_I_SB(dic->inode)->sb->s_id,
280 dic->rlen,
281 PAGE_SIZE << dic->log_cluster_size);
282 return -EIO;
283 }
284 return 0;
285}
286
287static const struct f2fs_compress_ops f2fs_lz4_ops = {
288 .init_compress_ctx = lz4_init_compress_ctx,
289 .destroy_compress_ctx = lz4_destroy_compress_ctx,
290 .compress_pages = lz4_compress_pages,
291 .decompress_pages = lz4_decompress_pages,
292};
293#endif
294
50cfa66f
CY		 295#ifdef CONFIG_F2FS_FS_ZSTD
296#define F2FS_ZSTD_DEFAULT_CLEVEL 1
297
298static int zstd_init_compress_ctx(struct compress_ctx *cc)
299{
300 ZSTD_parameters params;
301 ZSTD_CStream *stream;
302 void *workspace;
303 unsigned int workspace_size;
304
305 params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0);
306 workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams);
307
308 workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
309 workspace_size, GFP_NOFS);
310 if (!workspace)
311 return -ENOMEM;
312
313 stream = ZSTD_initCStream(params, 0, workspace, workspace_size);
314 if (!stream) {
315 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initCStream failed\n",
316 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
317 __func__);
318 kvfree(workspace);
319 return -EIO;
320 }
321
322 cc->private = workspace;
323 cc->private2 = stream;
324
325 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
326 return 0;
327}
328
329static void zstd_destroy_compress_ctx(struct compress_ctx *cc)
330{
331 kvfree(cc->private);
332 cc->private = NULL;
333 cc->private2 = NULL;
334}
335
336static int zstd_compress_pages(struct compress_ctx *cc)
337{
338 ZSTD_CStream *stream = cc->private2;
339 ZSTD_inBuffer inbuf;
340 ZSTD_outBuffer outbuf;
341 int src_size = cc->rlen;
342 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE;
343 int ret;
344
345 inbuf.pos = 0;
346 inbuf.src = cc->rbuf;
347 inbuf.size = src_size;
348
349 outbuf.pos = 0;
350 outbuf.dst = cc->cbuf->cdata;
351 outbuf.size = dst_size;
352
353 ret = ZSTD_compressStream(stream, &outbuf, &inbuf);
354 if (ZSTD_isError(ret)) {
355 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
356 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
357 __func__, ZSTD_getErrorCode(ret));
358 return -EIO;
359 }
360
361 ret = ZSTD_endStream(stream, &outbuf);
362 if (ZSTD_isError(ret)) {
363 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_endStream returned %d\n",
364 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
365 __func__, ZSTD_getErrorCode(ret));
366 return -EIO;
367 }
368
369 cc->clen = outbuf.pos;
370 return 0;
371}
372
373static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)
374{
375 ZSTD_DStream *stream;
376 void *workspace;
377 unsigned int workspace_size;
378
379 workspace_size = ZSTD_DStreamWorkspaceBound(MAX_COMPRESS_WINDOW_SIZE);
380
381 workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode),
382 workspace_size, GFP_NOFS);
383 if (!workspace)
384 return -ENOMEM;
385
386 stream = ZSTD_initDStream(MAX_COMPRESS_WINDOW_SIZE,
387 workspace, workspace_size);
388 if (!stream) {
389 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initDStream failed\n",
390 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
391 __func__);
392 kvfree(workspace);
393 return -EIO;
394 }
395
396 dic->private = workspace;
397 dic->private2 = stream;
398
399 return 0;
400}
401
402static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)
403{
404 kvfree(dic->private);
405 dic->private = NULL;
406 dic->private2 = NULL;
407}
408
409static int zstd_decompress_pages(struct decompress_io_ctx *dic)
410{
411 ZSTD_DStream *stream = dic->private2;
412 ZSTD_inBuffer inbuf;
413 ZSTD_outBuffer outbuf;
414 int ret;
415
416 inbuf.pos = 0;
417 inbuf.src = dic->cbuf->cdata;
418 inbuf.size = dic->clen;
419
420 outbuf.pos = 0;
421 outbuf.dst = dic->rbuf;
422 outbuf.size = dic->rlen;
423
424 ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
425 if (ZSTD_isError(ret)) {
426 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
427 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
428 __func__, ZSTD_getErrorCode(ret));
429 return -EIO;
430 }
431
432 if (dic->rlen != outbuf.pos) {
433 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, "
434 "expected:%lu\n", KERN_ERR,
435 F2FS_I_SB(dic->inode)->sb->s_id,
436 __func__, dic->rlen,
437 PAGE_SIZE << dic->log_cluster_size);
438 return -EIO;
439 }
440
441 return 0;
442}
443
444static const struct f2fs_compress_ops f2fs_zstd_ops = {
445 .init_compress_ctx = zstd_init_compress_ctx,
446 .destroy_compress_ctx = zstd_destroy_compress_ctx,
447 .compress_pages = zstd_compress_pages,
448 .init_decompress_ctx = zstd_init_decompress_ctx,
449 .destroy_decompress_ctx = zstd_destroy_decompress_ctx,
450 .decompress_pages = zstd_decompress_pages,
451};
452#endif
453
4c8ff709
CY		 454static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {
455#ifdef CONFIG_F2FS_FS_LZO
456 &f2fs_lzo_ops,
457#else
458 NULL,
459#endif
460#ifdef CONFIG_F2FS_FS_LZ4
461 &f2fs_lz4_ops,
462#else
463 NULL,
464#endif
50cfa66f
CY		 465#ifdef CONFIG_F2FS_FS_ZSTD
466 &f2fs_zstd_ops,
467#else
468 NULL,
469#endif
4c8ff709
CY		 470};
471
472bool f2fs_is_compress_backend_ready(struct inode *inode)
473{
474 if (!f2fs_compressed_file(inode))
475 return true;
476 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm];
477}
478
479static struct page *f2fs_grab_page(void)
480{
481 struct page *page;
482
483 page = alloc_page(GFP_NOFS);
484 if (!page)
485 return NULL;
486 lock_page(page);
487 return page;
488}
489
490static int f2fs_compress_pages(struct compress_ctx *cc)
491{
492 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
493 struct f2fs_inode_info *fi = F2FS_I(cc->inode);
494 const struct f2fs_compress_ops *cops =
495 f2fs_cops[fi->i_compress_algorithm];
496 unsigned int max_len, nr_cpages;
497 int i, ret;
498
499 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx,
500 cc->cluster_size, fi->i_compress_algorithm);
501
23b1faaa
CY		 502 if (cops->init_compress_ctx) {
503 ret = cops->init_compress_ctx(cc);
504 if (ret)
505 goto out;
506 }
4c8ff709
CY		 507
508 max_len = COMPRESS_HEADER_SIZE + cc->clen;
509 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE);
510
511 cc->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
512 cc->nr_cpages, GFP_NOFS);
513 if (!cc->cpages) {
514 ret = -ENOMEM;
515 goto destroy_compress_ctx;
516 }
517
518 for (i = 0; i < cc->nr_cpages; i++) {
519 cc->cpages[i] = f2fs_grab_page();
520 if (!cc->cpages[i]) {
521 ret = -ENOMEM;
522 goto out_free_cpages;
523 }
524 }
525
526 cc->rbuf = vmap(cc->rpages, cc->cluster_size, VM_MAP, PAGE_KERNEL_RO);
527 if (!cc->rbuf) {
528 ret = -ENOMEM;
529 goto out_free_cpages;
530 }
531
532 cc->cbuf = vmap(cc->cpages, cc->nr_cpages, VM_MAP, PAGE_KERNEL);
533 if (!cc->cbuf) {
534 ret = -ENOMEM;
535 goto out_vunmap_rbuf;
536 }
537
538 ret = cops->compress_pages(cc);
539 if (ret)
540 goto out_vunmap_cbuf;
541
542 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE;
543
544 if (cc->clen > max_len) {
545 ret = -EAGAIN;
546 goto out_vunmap_cbuf;
547 }
548
549 cc->cbuf->clen = cpu_to_le32(cc->clen);
4c8ff709
CY		 550
551 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++)
552 cc->cbuf->reserved[i] = cpu_to_le32(0);
553
7fa6d598
EB		 554 nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE);
555
556 /* zero out any unused part of the last page */
557 memset(&cc->cbuf->cdata[cc->clen], 0,
558 (nr_cpages * PAGE_SIZE) - (cc->clen + COMPRESS_HEADER_SIZE));
559
4c8ff709
CY		 560 vunmap(cc->cbuf);
561 vunmap(cc->rbuf);
562
4c8ff709
CY		 563 for (i = nr_cpages; i < cc->nr_cpages; i++) {
564 f2fs_put_compressed_page(cc->cpages[i]);
565 cc->cpages[i] = NULL;
566 }
567
23b1faaa
CY		 568 if (cops->destroy_compress_ctx)
569 cops->destroy_compress_ctx(cc);
09ff4801 570
4c8ff709
CY		 571 cc->nr_cpages = nr_cpages;
572
573 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
574 cc->clen, ret);
575 return 0;
576
577out_vunmap_cbuf:
578 vunmap(cc->cbuf);
579out_vunmap_rbuf:
580 vunmap(cc->rbuf);
581out_free_cpages:
582 for (i = 0; i < cc->nr_cpages; i++) {
583 if (cc->cpages[i])
584 f2fs_put_compressed_page(cc->cpages[i]);
585 }
586 kfree(cc->cpages);
587 cc->cpages = NULL;
588destroy_compress_ctx:
23b1faaa
CY		 589 if (cops->destroy_compress_ctx)
590 cops->destroy_compress_ctx(cc);
4c8ff709
CY		 591out:
592 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
593 cc->clen, ret);
594 return ret;
595}
596
597void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity)
598{
599 struct decompress_io_ctx *dic =
600 (struct decompress_io_ctx *)page_private(page);
601 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);
602 struct f2fs_inode_info *fi= F2FS_I(dic->inode);
603 const struct f2fs_compress_ops *cops =
604 f2fs_cops[fi->i_compress_algorithm];
605 int ret;
606
607 dec_page_count(sbi, F2FS_RD_DATA);
608
609 if (bio->bi_status || PageError(page))
610 dic->failed = true;
611
612 if (refcount_dec_not_one(&dic->ref))
613 return;
614
615 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,
616 dic->cluster_size, fi->i_compress_algorithm);
617
618 /* submit partial compressed pages */
619 if (dic->failed) {
620 ret = -EIO;
621 goto out_free_dic;
622 }
623
23b1faaa
CY		 624 if (cops->init_decompress_ctx) {
625 ret = cops->init_decompress_ctx(dic);
626 if (ret)
627 goto out_free_dic;
628 }
629
4c8ff709
CY		 630 dic->rbuf = vmap(dic->tpages, dic->cluster_size, VM_MAP, PAGE_KERNEL);
631 if (!dic->rbuf) {
632 ret = -ENOMEM;
23b1faaa 633 goto destroy_decompress_ctx;
4c8ff709
CY		 634 }
635
636 dic->cbuf = vmap(dic->cpages, dic->nr_cpages, VM_MAP, PAGE_KERNEL_RO);
637 if (!dic->cbuf) {
638 ret = -ENOMEM;
639 goto out_vunmap_rbuf;
640 }
641
642 dic->clen = le32_to_cpu(dic->cbuf->clen);
643 dic->rlen = PAGE_SIZE << dic->log_cluster_size;
644
645 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {
646 ret = -EFSCORRUPTED;
647 goto out_vunmap_cbuf;
648 }
649
650 ret = cops->decompress_pages(dic);
651
652out_vunmap_cbuf:
653 vunmap(dic->cbuf);
654out_vunmap_rbuf:
655 vunmap(dic->rbuf);
23b1faaa
CY		 656destroy_decompress_ctx:
657 if (cops->destroy_decompress_ctx)
658 cops->destroy_decompress_ctx(dic);
4c8ff709 659out_free_dic:
79bbefb1 660 if (verity)
887347a0 661 refcount_set(&dic->ref, dic->nr_cpages);
4c8ff709
CY		 662 if (!verity)
663 f2fs_decompress_end_io(dic->rpages, dic->cluster_size,
664 ret, false);
665
666 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,
667 dic->clen, ret);
668 if (!verity)
669 f2fs_free_dic(dic);
670}
671
672static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index)
673{
674 if (cc->cluster_idx == NULL_CLUSTER)
675 return true;
676 return cc->cluster_idx == cluster_idx(cc, index);
677}
678
679bool f2fs_cluster_is_empty(struct compress_ctx *cc)
680{
681 return cc->nr_rpages == 0;
682}
683
684static bool f2fs_cluster_is_full(struct compress_ctx *cc)
685{
686 return cc->cluster_size == cc->nr_rpages;
687}
688
689bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index)
690{
691 if (f2fs_cluster_is_empty(cc))
692 return true;
693 return is_page_in_cluster(cc, index);
694}
695
696static bool __cluster_may_compress(struct compress_ctx *cc)
697{
698 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
699 loff_t i_size = i_size_read(cc->inode);
700 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE);
701 int i;
702
703 for (i = 0; i < cc->cluster_size; i++) {
704 struct page *page = cc->rpages[i];
705
706 f2fs_bug_on(sbi, !page);
707
708 if (unlikely(f2fs_cp_error(sbi)))
709 return false;
710 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
711 return false;
712
713 /* beyond EOF */
714 if (page->index >= nr_pages)
715 return false;
716 }
717 return true;
718}
719
1a67cbe1 720static int __f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
4c8ff709
CY		 721{
722 struct dnode_of_data dn;
723 int ret;
724
725 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
726 ret = f2fs_get_dnode_of_data(&dn, start_idx_of_cluster(cc),
727 LOOKUP_NODE);
728 if (ret) {
729 if (ret == -ENOENT)
730 ret = 0;
731 goto fail;
732 }
733
734 if (dn.data_blkaddr == COMPRESS_ADDR) {
735 int i;
736
737 ret = 1;
738 for (i = 1; i < cc->cluster_size; i++) {
739 block_t blkaddr;
740
a2ced1ce 741 blkaddr = data_blkaddr(dn.inode,
4c8ff709 742 dn.node_page, dn.ofs_in_node + i);
1a67cbe1
CY		 743 if (compr) {
744 if (__is_valid_data_blkaddr(blkaddr))
745 ret++;
746 } else {
747 if (blkaddr != NULL_ADDR)
748 ret++;
749 }
4c8ff709
CY		 750 }
751 }
752fail:
753 f2fs_put_dnode(&dn);
754 return ret;
755}
756
1a67cbe1
CY		 757/* return # of compressed blocks in compressed cluster */
758static int f2fs_compressed_blocks(struct compress_ctx *cc)
759{
760 return __f2fs_cluster_blocks(cc, true);
761}
762
763/* return # of valid blocks in compressed cluster */
764static int f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
765{
766 return __f2fs_cluster_blocks(cc, false);
767}
768
4c8ff709
CY		 769int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index)
770{
771 struct compress_ctx cc = {
772 .inode = inode,
773 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
774 .cluster_size = F2FS_I(inode)->i_cluster_size,
775 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
776 };
777
1a67cbe1 778 return f2fs_cluster_blocks(&cc, false);
4c8ff709
CY		 779}
780
781static bool cluster_may_compress(struct compress_ctx *cc)
782{
783 if (!f2fs_compressed_file(cc->inode))
784 return false;
785 if (f2fs_is_atomic_file(cc->inode))
786 return false;
787 if (f2fs_is_mmap_file(cc->inode))
788 return false;
789 if (!f2fs_cluster_is_full(cc))
790 return false;
791 return __cluster_may_compress(cc);
792}
793
794static void set_cluster_writeback(struct compress_ctx *cc)
795{
796 int i;
797
798 for (i = 0; i < cc->cluster_size; i++) {
799 if (cc->rpages[i])
800 set_page_writeback(cc->rpages[i]);
801 }
802}
803
804static void set_cluster_dirty(struct compress_ctx *cc)
805{
806 int i;
807
808 for (i = 0; i < cc->cluster_size; i++)
809 if (cc->rpages[i])
810 set_page_dirty(cc->rpages[i]);
811}
812
813static int prepare_compress_overwrite(struct compress_ctx *cc,
814 struct page **pagep, pgoff_t index, void **fsdata)
815{
816 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
817 struct address_space *mapping = cc->inode->i_mapping;
818 struct page *page;
819 struct dnode_of_data dn;
820 sector_t last_block_in_bio;
821 unsigned fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT;
822 pgoff_t start_idx = start_idx_of_cluster(cc);
823 int i, ret;
824 bool prealloc;
825
826retry:
1a67cbe1 827 ret = f2fs_cluster_blocks(cc, false);
4c8ff709
CY		 828 if (ret <= 0)
829 return ret;
830
831 /* compressed case */
832 prealloc = (ret < cc->cluster_size);
833
834 ret = f2fs_init_compress_ctx(cc);
835 if (ret)
836 return ret;
837
838 /* keep page reference to avoid page reclaim */
839 for (i = 0; i < cc->cluster_size; i++) {
840 page = f2fs_pagecache_get_page(mapping, start_idx + i,
841 fgp_flag, GFP_NOFS);
842 if (!page) {
843 ret = -ENOMEM;
844 goto unlock_pages;
845 }
846
847 if (PageUptodate(page))
848 unlock_page(page);
849 else
850 f2fs_compress_ctx_add_page(cc, page);
851 }
852
853 if (!f2fs_cluster_is_empty(cc)) {
854 struct bio *bio = NULL;
855
856 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size,
0683728a 857 &last_block_in_bio, false, true);
4c8ff709
CY		 858 f2fs_destroy_compress_ctx(cc);
859 if (ret)
860 goto release_pages;
861 if (bio)
862 f2fs_submit_bio(sbi, bio, DATA);
863
864 ret = f2fs_init_compress_ctx(cc);
865 if (ret)
866 goto release_pages;
867 }
868
869 for (i = 0; i < cc->cluster_size; i++) {
870 f2fs_bug_on(sbi, cc->rpages[i]);
871
872 page = find_lock_page(mapping, start_idx + i);
873 f2fs_bug_on(sbi, !page);
874
875 f2fs_wait_on_page_writeback(page, DATA, true, true);
876
877 f2fs_compress_ctx_add_page(cc, page);
878 f2fs_put_page(page, 0);
879
880 if (!PageUptodate(page)) {
881 f2fs_unlock_rpages(cc, i + 1);
882 f2fs_put_rpages_mapping(cc, mapping, start_idx,
883 cc->cluster_size);
884 f2fs_destroy_compress_ctx(cc);
885 goto retry;
886 }
887 }
888
889 if (prealloc) {
890 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
891
892 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
893
894 for (i = cc->cluster_size - 1; i > 0; i--) {
895 ret = f2fs_get_block(&dn, start_idx + i);
896 if (ret) {
897 i = cc->cluster_size;
898 break;
899 }
900
901 if (dn.data_blkaddr != NEW_ADDR)
902 break;
903 }
904
905 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
906 }
907
908 if (likely(!ret)) {
909 *fsdata = cc->rpages;
910 *pagep = cc->rpages[offset_in_cluster(cc, index)];
911 return cc->cluster_size;
912 }
913
914unlock_pages:
915 f2fs_unlock_rpages(cc, i);
916release_pages:
917 f2fs_put_rpages_mapping(cc, mapping, start_idx, i);
918 f2fs_destroy_compress_ctx(cc);
919 return ret;
920}
921
922int f2fs_prepare_compress_overwrite(struct inode *inode,
923 struct page **pagep, pgoff_t index, void **fsdata)
924{
925 struct compress_ctx cc = {
926 .inode = inode,
927 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
928 .cluster_size = F2FS_I(inode)->i_cluster_size,
929 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
930 .rpages = NULL,
931 .nr_rpages = 0,
932 };
933
934 return prepare_compress_overwrite(&cc, pagep, index, fsdata);
935}
936
937bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
938 pgoff_t index, unsigned copied)
939
940{
941 struct compress_ctx cc = {
942 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
943 .cluster_size = F2FS_I(inode)->i_cluster_size,
944 .rpages = fsdata,
945 };
946 bool first_index = (index == cc.rpages[0]->index);
947
948 if (copied)
949 set_cluster_dirty(&cc);
950
951 f2fs_put_rpages_wbc(&cc, NULL, false, 1);
952 f2fs_destroy_compress_ctx(&cc);
953
954 return first_index;
955}
956
957static int f2fs_write_compressed_pages(struct compress_ctx *cc,
958 int *submitted,
959 struct writeback_control *wbc,
960 enum iostat_type io_type)
961{
962 struct inode *inode = cc->inode;
963 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
964 struct f2fs_inode_info *fi = F2FS_I(inode);
965 struct f2fs_io_info fio = {
966 .sbi = sbi,
967 .ino = cc->inode->i_ino,
968 .type = DATA,
969 .op = REQ_OP_WRITE,
970 .op_flags = wbc_to_write_flags(wbc),
971 .old_blkaddr = NEW_ADDR,
972 .page = NULL,
973 .encrypted_page = NULL,
974 .compressed_page = NULL,
975 .submitted = false,
4c8ff709
CY		 976 .io_type = io_type,
977 .io_wbc = wbc,
978 .encrypted = f2fs_encrypted_file(cc->inode),
979 };
980 struct dnode_of_data dn;
981 struct node_info ni;
982 struct compress_io_ctx *cic;
983 pgoff_t start_idx = start_idx_of_cluster(cc);
984 unsigned int last_index = cc->cluster_size - 1;
985 loff_t psize;
986 int i, err;
987
df77fbd8
CY		 988 if (!f2fs_trylock_op(sbi))
989 return -EAGAIN;
4c8ff709 990
df77fbd8 991 set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
4c8ff709
CY		 992
993 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
994 if (err)
995 goto out_unlock_op;
996
997 for (i = 0; i < cc->cluster_size; i++) {
a2ced1ce 998 if (data_blkaddr(dn.inode, dn.node_page,
4c8ff709
CY		 999 dn.ofs_in_node + i) == NULL_ADDR)
1000 goto out_put_dnode;
1001 }
1002
1003 psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT;
1004
1005 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni);
1006 if (err)
1007 goto out_put_dnode;
1008
1009 fio.version = ni.version;
1010
1011 cic = f2fs_kzalloc(sbi, sizeof(struct compress_io_ctx), GFP_NOFS);
1012 if (!cic)
1013 goto out_put_dnode;
1014
1015 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1016 cic->inode = inode;
887347a0 1017 refcount_set(&cic->ref, cc->nr_cpages);
4c8ff709
CY		 1018 cic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1019 cc->log_cluster_size, GFP_NOFS);
1020 if (!cic->rpages)
1021 goto out_put_cic;
1022
1023 cic->nr_rpages = cc->cluster_size;
1024
1025 for (i = 0; i < cc->nr_cpages; i++) {
1026 f2fs_set_compressed_page(cc->cpages[i], inode,
887347a0 1027 cc->rpages[i + 1]->index, cic);
4c8ff709
CY		 1028 fio.compressed_page = cc->cpages[i];
1029 if (fio.encrypted) {
1030 fio.page = cc->rpages[i + 1];
1031 err = f2fs_encrypt_one_page(&fio);
1032 if (err)
1033 goto out_destroy_crypt;
1034 cc->cpages[i] = fio.encrypted_page;
1035 }
1036 }
1037
1038 set_cluster_writeback(cc);
1039
1040 for (i = 0; i < cc->cluster_size; i++)
1041 cic->rpages[i] = cc->rpages[i];
1042
1043 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) {
1044 block_t blkaddr;
1045
a2ced1ce 1046 blkaddr = f2fs_data_blkaddr(&dn);
95978caa 1047 fio.page = cc->rpages[i];
4c8ff709
CY		 1048 fio.old_blkaddr = blkaddr;
1049
1050 /* cluster header */
1051 if (i == 0) {
1052 if (blkaddr == COMPRESS_ADDR)
1053 fio.compr_blocks++;
1054 if (__is_valid_data_blkaddr(blkaddr))
1055 f2fs_invalidate_blocks(sbi, blkaddr);
1056 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR);
1057 goto unlock_continue;
1058 }
1059
1060 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr))
1061 fio.compr_blocks++;
1062
1063 if (i > cc->nr_cpages) {
1064 if (__is_valid_data_blkaddr(blkaddr)) {
1065 f2fs_invalidate_blocks(sbi, blkaddr);
1066 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
1067 }
1068 goto unlock_continue;
1069 }
1070
1071 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR);
1072
1073 if (fio.encrypted)
1074 fio.encrypted_page = cc->cpages[i - 1];
1075 else
1076 fio.compressed_page = cc->cpages[i - 1];
1077
1078 cc->cpages[i - 1] = NULL;
1079 f2fs_outplace_write_data(&dn, &fio);
1080 (*submitted)++;
1081unlock_continue:
1082 inode_dec_dirty_pages(cc->inode);
1083 unlock_page(fio.page);
1084 }
1085
1086 if (fio.compr_blocks)
1087 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);
1088 f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true);
1089
1090 set_inode_flag(cc->inode, FI_APPEND_WRITE);
1091 if (cc->cluster_idx == 0)
1092 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1093
1094 f2fs_put_dnode(&dn);
1095 f2fs_unlock_op(sbi);
1096
c10c9820 1097 spin_lock(&fi->i_size_lock);
4c8ff709
CY		 1098 if (fi->last_disk_size < psize)
1099 fi->last_disk_size = psize;
c10c9820 1100 spin_unlock(&fi->i_size_lock);
4c8ff709
CY		 1101
1102 f2fs_put_rpages(cc);
1103 f2fs_destroy_compress_ctx(cc);
1104 return 0;
1105
1106out_destroy_crypt:
1107 kfree(cic->rpages);
1108
1109 for (--i; i >= 0; i--)
1110 fscrypt_finalize_bounce_page(&cc->cpages[i]);
1111 for (i = 0; i < cc->nr_cpages; i++) {
1112 if (!cc->cpages[i])
1113 continue;
1114 f2fs_put_page(cc->cpages[i], 1);
1115 }
1116out_put_cic:
1117 kfree(cic);
1118out_put_dnode:
1119 f2fs_put_dnode(&dn);
1120out_unlock_op:
1121 f2fs_unlock_op(sbi);
1122 return -EAGAIN;
1123}
1124
1125void f2fs_compress_write_end_io(struct bio *bio, struct page *page)
1126{
1127 struct f2fs_sb_info *sbi = bio->bi_private;
1128 struct compress_io_ctx *cic =
1129 (struct compress_io_ctx *)page_private(page);
1130 int i;
1131
1132 if (unlikely(bio->bi_status))
1133 mapping_set_error(cic->inode->i_mapping, -EIO);
1134
1135 f2fs_put_compressed_page(page);
1136
1137 dec_page_count(sbi, F2FS_WB_DATA);
1138
1139 if (refcount_dec_not_one(&cic->ref))
1140 return;
1141
1142 for (i = 0; i < cic->nr_rpages; i++) {
1143 WARN_ON(!cic->rpages[i]);
1144 clear_cold_data(cic->rpages[i]);
1145 end_page_writeback(cic->rpages[i]);
1146 }
1147
1148 kfree(cic->rpages);
1149 kfree(cic);
1150}
1151
1152static int f2fs_write_raw_pages(struct compress_ctx *cc,
1153 int *submitted,
1154 struct writeback_control *wbc,
1155 enum iostat_type io_type)
1156{
1157 struct address_space *mapping = cc->inode->i_mapping;
1158 int _submitted, compr_blocks, ret;
1159 int i = -1, err = 0;
1160
1161 compr_blocks = f2fs_compressed_blocks(cc);
1162 if (compr_blocks < 0) {
1163 err = compr_blocks;
1164 goto out_err;
1165 }
1166
1167 for (i = 0; i < cc->cluster_size; i++) {
1168 if (!cc->rpages[i])
1169 continue;
1170retry_write:
1171 if (cc->rpages[i]->mapping != mapping) {
1172 unlock_page(cc->rpages[i]);
1173 continue;
1174 }
1175
1176 BUG_ON(!PageLocked(cc->rpages[i]));
1177
1178 ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted,
1179 NULL, NULL, wbc, io_type,
1180 compr_blocks);
1181 if (ret) {
1182 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1183 unlock_page(cc->rpages[i]);
1184 ret = 0;
1185 } else if (ret == -EAGAIN) {
466357dc
CY		 1186 /*
1187 * for quota file, just redirty left pages to
1188 * avoid deadlock caused by cluster update race
1189 * from foreground operation.
1190 */
1191 if (IS_NOQUOTA(cc->inode)) {
1192 err = 0;
1193 goto out_err;
1194 }
4c8ff709
CY		 1195 ret = 0;
1196 cond_resched();
5df7731f
CY		 1197 congestion_wait(BLK_RW_ASYNC,
1198 DEFAULT_IO_TIMEOUT);
4c8ff709
CY		 1199 lock_page(cc->rpages[i]);
1200 clear_page_dirty_for_io(cc->rpages[i]);
1201 goto retry_write;
1202 }
1203 err = ret;
466357dc 1204 goto out_err;
4c8ff709
CY		 1205 }
1206
1207 *submitted += _submitted;
1208 }
1209 return 0;
4c8ff709
CY		 1210out_err:
1211 for (++i; i < cc->cluster_size; i++) {
1212 if (!cc->rpages[i])
1213 continue;
1214 redirty_page_for_writepage(wbc, cc->rpages[i]);
1215 unlock_page(cc->rpages[i]);
1216 }
1217 return err;
1218}
1219
1220int f2fs_write_multi_pages(struct compress_ctx *cc,
1221 int *submitted,
1222 struct writeback_control *wbc,
1223 enum iostat_type io_type)
1224{
1225 struct f2fs_inode_info *fi = F2FS_I(cc->inode);
1226 const struct f2fs_compress_ops *cops =
1227 f2fs_cops[fi->i_compress_algorithm];
1228 int err;
1229
1230 *submitted = 0;
1231 if (cluster_may_compress(cc)) {
1232 err = f2fs_compress_pages(cc);
1233 if (err == -EAGAIN) {
1234 goto write;
1235 } else if (err) {
1236 f2fs_put_rpages_wbc(cc, wbc, true, 1);
1237 goto destroy_out;
1238 }
1239
1240 err = f2fs_write_compressed_pages(cc, submitted,
1241 wbc, io_type);
1242 cops->destroy_compress_ctx(cc);
1243 if (!err)
1244 return 0;
1245 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN);
1246 }
1247write:
1248 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted);
1249
1250 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type);
1251 f2fs_put_rpages_wbc(cc, wbc, false, 0);
1252destroy_out:
1253 f2fs_destroy_compress_ctx(cc);
1254 return err;
1255}
1256
1257struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)
1258{
1259 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1260 struct decompress_io_ctx *dic;
1261 pgoff_t start_idx = start_idx_of_cluster(cc);
1262 int i;
1263
1264 dic = f2fs_kzalloc(sbi, sizeof(struct decompress_io_ctx), GFP_NOFS);
1265 if (!dic)
1266 return ERR_PTR(-ENOMEM);
1267
1268 dic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1269 cc->log_cluster_size, GFP_NOFS);
1270 if (!dic->rpages) {
1271 kfree(dic);
1272 return ERR_PTR(-ENOMEM);
1273 }
1274
1275 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1276 dic->inode = cc->inode;
887347a0 1277 refcount_set(&dic->ref, cc->nr_cpages);
4c8ff709
CY		 1278 dic->cluster_idx = cc->cluster_idx;
1279 dic->cluster_size = cc->cluster_size;
1280 dic->log_cluster_size = cc->log_cluster_size;
1281 dic->nr_cpages = cc->nr_cpages;
1282 dic->failed = false;
1283
1284 for (i = 0; i < dic->cluster_size; i++)
1285 dic->rpages[i] = cc->rpages[i];
1286 dic->nr_rpages = cc->cluster_size;
1287
1288 dic->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1289 dic->nr_cpages, GFP_NOFS);
1290 if (!dic->cpages)
1291 goto out_free;
1292
1293 for (i = 0; i < dic->nr_cpages; i++) {
1294 struct page *page;
1295
1296 page = f2fs_grab_page();
1297 if (!page)
1298 goto out_free;
1299
1300 f2fs_set_compressed_page(page, cc->inode,
887347a0 1301 start_idx + i + 1, dic);
4c8ff709
CY		 1302 dic->cpages[i] = page;
1303 }
1304
1305 dic->tpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1306 dic->cluster_size, GFP_NOFS);
1307 if (!dic->tpages)
1308 goto out_free;
1309
1310 for (i = 0; i < dic->cluster_size; i++) {
80d0d45a
CY		 1311 if (cc->rpages[i]) {
1312 dic->tpages[i] = cc->rpages[i];
4c8ff709 1313 continue;
80d0d45a 1314 }
4c8ff709
CY		 1315
1316 dic->tpages[i] = f2fs_grab_page();
1317 if (!dic->tpages[i])
1318 goto out_free;
1319 }
1320
4c8ff709
CY		 1321 return dic;
1322
1323out_free:
1324 f2fs_free_dic(dic);
1325 return ERR_PTR(-ENOMEM);
1326}
1327
1328void f2fs_free_dic(struct decompress_io_ctx *dic)
1329{
1330 int i;
1331
1332 if (dic->tpages) {
1333 for (i = 0; i < dic->cluster_size; i++) {
1334 if (dic->rpages[i])
1335 continue;
8908e753
CY		 1336 if (!dic->tpages[i])
1337 continue;
61fbae2b
CY		 1338 unlock_page(dic->tpages[i]);
1339 put_page(dic->tpages[i]);
4c8ff709
CY		 1340 }
1341 kfree(dic->tpages);
1342 }
1343
1344 if (dic->cpages) {
1345 for (i = 0; i < dic->nr_cpages; i++) {
1346 if (!dic->cpages[i])
1347 continue;
1348 f2fs_put_compressed_page(dic->cpages[i]);
1349 }
1350 kfree(dic->cpages);
1351 }
1352
1353 kfree(dic->rpages);
1354 kfree(dic);
1355}
1356
1357void f2fs_decompress_end_io(struct page **rpages,
1358 unsigned int cluster_size, bool err, bool verity)
1359{
1360 int i;
1361
1362 for (i = 0; i < cluster_size; i++) {
1363 struct page *rpage = rpages[i];
1364
1365 if (!rpage)
1366 continue;
1367
23c51bed
CY		 1368 if (err || PageError(rpage))
1369 goto clear_uptodate;
1370
1371 if (!verity || fsverity_verify_page(rpage)) {
1372 SetPageUptodate(rpage);
1373 goto unlock;
4c8ff709 1374 }
23c51bed
CY		 1375clear_uptodate:
1376 ClearPageUptodate(rpage);
1377 ClearPageError(rpage);
1378unlock:
4c8ff709
CY		 1379 unlock_page(rpage);
1380 }
1381}
Linux 6.x block layer and io_uring tree(s)