Commit | Line | Data |
---|---|---|
29b24f6c GX |
1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* | |
0d40d6e3 | 3 | * Copyright (C) 2018 HUAWEI, Inc. |
592e7cd0 | 4 | * https://www.huawei.com/ |
0d40d6e3 | 5 | */ |
57b78c9f GX |
6 | #ifndef __EROFS_FS_ZDATA_H |
7 | #define __EROFS_FS_ZDATA_H | |
0d40d6e3 GX |
8 | |
9 | #include "internal.h" | |
387bab87 | 10 | #include "tagptr.h" |
3883a79a | 11 | |
9f6cc76e | 12 | #define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE) |
387bab87 | 13 | #define Z_EROFS_INLINE_BVECS 2 |
88aaf5a7 | 14 | |
87ca34a7 GX |
15 | /* |
16 | * let's leave a type here in case of introducing | |
17 | * another tagged pointer later. | |
18 | */ | |
19 | typedef void *z_erofs_next_pcluster_t; | |
20 | ||
06a304cd GX |
21 | struct z_erofs_bvec { |
22 | struct page *page; | |
23 | int offset; | |
24 | unsigned int end; | |
25 | }; | |
26 | ||
27 | #define __Z_EROFS_BVSET(name, total) \ | |
28 | struct name { \ | |
29 | /* point to the next page which contains the following bvecs */ \ | |
30 | struct page *nextpage; \ | |
31 | struct z_erofs_bvec bvec[total]; \ | |
32 | } | |
33 | __Z_EROFS_BVSET(z_erofs_bvset,); | |
387bab87 | 34 | __Z_EROFS_BVSET(z_erofs_bvset_inline, Z_EROFS_INLINE_BVECS); |
06a304cd | 35 | |
3883a79a GX |
36 | /* |
37 | * Structure fields follow one of the following exclusion rules. | |
38 | * | |
39 | * I: Modifiable by initialization/destruction paths and read-only | |
97e86a85 | 40 | * for everyone else; |
3883a79a | 41 | * |
87ca34a7 | 42 | * L: Field should be protected by the pcluster lock; |
97e86a85 GX |
43 | * |
44 | * A: Field should be accessed / updated in atomic for parallelized code. | |
3883a79a | 45 | */ |
87ca34a7 GX |
46 | struct z_erofs_pcluster { |
47 | struct erofs_workgroup obj; | |
3883a79a GX |
48 | struct mutex lock; |
49 | ||
87ca34a7 GX |
50 | /* A: point to next chained pcluster or TAILs */ |
51 | z_erofs_next_pcluster_t next; | |
52 | ||
2bfab9c0 | 53 | /* L: the maximum decompression size of this round */ |
87ca34a7 GX |
54 | unsigned int length; |
55 | ||
06a304cd GX |
56 | /* L: total number of bvecs */ |
57 | unsigned int vcnt; | |
58 | ||
97e86a85 | 59 | /* I: page offset of start position of decompression */ |
87ca34a7 GX |
60 | unsigned short pageofs_out; |
61 | ||
62 | /* I: page offset of inline compressed data */ | |
63 | unsigned short pageofs_in; | |
97e86a85 | 64 | |
3883a79a | 65 | union { |
06a304cd GX |
66 | /* L: inline a certain number of bvec for bootstrap */ |
67 | struct z_erofs_bvset_inline bvset; | |
68 | ||
97e86a85 | 69 | /* I: can be used to free the pcluster by RCU. */ |
3883a79a GX |
70 | struct rcu_head rcu; |
71 | }; | |
cecf864d YH |
72 | |
73 | union { | |
74 | /* I: physical cluster size in pages */ | |
75 | unsigned short pclusterpages; | |
76 | ||
77 | /* I: tailpacking inline compressed size */ | |
78 | unsigned short tailpacking_size; | |
79 | }; | |
9f6cc76e | 80 | |
97e86a85 GX |
81 | /* I: compression algorithm format */ |
82 | unsigned char algorithmformat; | |
9f6cc76e | 83 | |
2bfab9c0 GX |
84 | /* L: whether partial decompression or not */ |
85 | bool partial; | |
86 | ||
ed722fbc GX |
87 | /* A: compressed bvecs (can be cached or inplaced pages) */ |
88 | struct z_erofs_bvec compressed_bvecs[]; | |
3883a79a GX |
89 | }; |
90 | ||
91 | /* let's avoid the valid 32-bit kernel addresses */ | |
92 | ||
93 | /* the chained workgroup has't submitted io (still open) */ | |
97e86a85 | 94 | #define Z_EROFS_PCLUSTER_TAIL ((void *)0x5F0ECAFE) |
3883a79a | 95 | /* the chained workgroup has already submitted io */ |
97e86a85 | 96 | #define Z_EROFS_PCLUSTER_TAIL_CLOSED ((void *)0x5F0EDEAD) |
3883a79a | 97 | |
97e86a85 | 98 | #define Z_EROFS_PCLUSTER_NIL (NULL) |
3883a79a | 99 | |
a4b1fab1 GX |
100 | struct z_erofs_decompressqueue { |
101 | struct super_block *sb; | |
3883a79a | 102 | atomic_t pending_bios; |
97e86a85 | 103 | z_erofs_next_pcluster_t head; |
3883a79a GX |
104 | |
105 | union { | |
60b30050 | 106 | struct completion done; |
3883a79a GX |
107 | struct work_struct work; |
108 | } u; | |
67148551 GX |
109 | |
110 | bool eio; | |
3883a79a GX |
111 | }; |
112 | ||
cecf864d YH |
113 | static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl) |
114 | { | |
115 | return !pcl->obj.index; | |
116 | } | |
117 | ||
118 | static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl) | |
119 | { | |
120 | if (z_erofs_is_inline_pcluster(pcl)) | |
121 | return 1; | |
122 | return pcl->pclusterpages; | |
123 | } | |
124 | ||
3883a79a | 125 | /* |
67148551 GX |
126 | * bit 31: I/O error occurred on this page |
127 | * bit 0 - 30: remaining parts to complete this page | |
3883a79a | 128 | */ |
67148551 | 129 | #define Z_EROFS_PAGE_EIO (1 << 31) |
3883a79a GX |
130 | |
131 | static inline void z_erofs_onlinepage_init(struct page *page) | |
132 | { | |
133 | union { | |
67148551 | 134 | atomic_t o; |
3883a79a | 135 | unsigned long v; |
3883a79a GX |
136 | } u = { .o = ATOMIC_INIT(1) }; |
137 | ||
138 | set_page_private(page, u.v); | |
139 | smp_wmb(); | |
140 | SetPagePrivate(page); | |
141 | } | |
142 | ||
67148551 | 143 | static inline void z_erofs_onlinepage_split(struct page *page) |
3883a79a | 144 | { |
67148551 GX |
145 | atomic_inc((atomic_t *)&page->private); |
146 | } | |
3883a79a | 147 | |
67148551 GX |
148 | static inline void z_erofs_page_mark_eio(struct page *page) |
149 | { | |
150 | int orig; | |
3883a79a | 151 | |
67148551 GX |
152 | do { |
153 | orig = atomic_read((atomic_t *)&page->private); | |
154 | } while (atomic_cmpxchg((atomic_t *)&page->private, orig, | |
155 | orig | Z_EROFS_PAGE_EIO) != orig); | |
3883a79a GX |
156 | } |
157 | ||
158 | static inline void z_erofs_onlinepage_endio(struct page *page) | |
159 | { | |
e82a9a17 | 160 | unsigned int v; |
3883a79a | 161 | |
800c16c8 | 162 | DBG_BUGON(!PagePrivate(page)); |
67148551 GX |
163 | v = atomic_dec_return((atomic_t *)&page->private); |
164 | if (!(v & ~Z_EROFS_PAGE_EIO)) { | |
6aaa7b06 | 165 | set_page_private(page, 0); |
3883a79a | 166 | ClearPagePrivate(page); |
67148551 | 167 | if (!(v & Z_EROFS_PAGE_EIO)) |
3883a79a GX |
168 | SetPageUptodate(page); |
169 | unlock_page(page); | |
170 | } | |
3883a79a GX |
171 | } |
172 | ||
4f05687f | 173 | #define Z_EROFS_ONSTACK_PAGES 32 |
3883a79a | 174 | |
0d40d6e3 | 175 | #endif |