Commit | Line | Data |
---|---|---|
29b24f6c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
152a333a | 2 | /* |
152a333a | 3 | * Copyright (C) 2018-2019 HUAWEI, Inc. |
592e7cd0 | 4 | * https://www.huawei.com/ |
152a333a GX |
5 | */ |
6 | #include "internal.h" | |
7 | #include <asm/unaligned.h> | |
8 | #include <trace/events/erofs.h> | |
9 | ||
152a333a GX |
10 | struct z_erofs_maprecorder { |
11 | struct inode *inode; | |
12 | struct erofs_map_blocks *map; | |
13 | void *kaddr; | |
14 | ||
15 | unsigned long lcn; | |
16 | /* compression extent information gathered */ | |
8f899262 | 17 | u8 type, headtype; |
152a333a GX |
18 | u16 clusterofs; |
19 | u16 delta[2]; | |
ea0b7b0d | 20 | erofs_blk_t pblk, compressedblks; |
ab92184f | 21 | erofs_off_t nextpackoff; |
5c2a6425 | 22 | bool partialref; |
152a333a GX |
23 | }; |
24 | ||
8241fdd3 GX |
25 | static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, |
26 | unsigned long lcn) | |
152a333a GX |
27 | { |
28 | struct inode *const inode = m->inode; | |
a5876e24 | 29 | struct erofs_inode *const vi = EROFS_I(inode); |
1c7f49a7 GX |
30 | const erofs_off_t pos = Z_EROFS_FULL_INDEX_ALIGN(erofs_iloc(inode) + |
31 | vi->inode_isize + vi->xattr_isize) + | |
32 | lcn * sizeof(struct z_erofs_lcluster_index); | |
33 | struct z_erofs_lcluster_index *di; | |
d6918942 | 34 | unsigned int advise; |
152a333a | 35 | |
31da107f | 36 | m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, |
e0981544 | 37 | erofs_pos(inode->i_sb, erofs_blknr(inode->i_sb, pos)), EROFS_KMAP); |
31da107f YH |
38 | if (IS_ERR(m->kaddr)) |
39 | return PTR_ERR(m->kaddr); | |
152a333a | 40 | |
1c7f49a7 | 41 | m->nextpackoff = pos + sizeof(struct z_erofs_lcluster_index); |
152a333a | 42 | m->lcn = lcn; |
3acea5fc | 43 | di = m->kaddr + erofs_blkoff(inode->i_sb, pos); |
152a333a GX |
44 | |
45 | advise = le16_to_cpu(di->di_advise); | |
d6918942 GX |
46 | m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; |
47 | if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { | |
152a333a GX |
48 | m->clusterofs = 1 << vi->z_logical_clusterbits; |
49 | m->delta[0] = le16_to_cpu(di->di_u.delta[0]); | |
1c7f49a7 | 50 | if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) { |
72bb5262 GX |
51 | if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | |
52 | Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { | |
cec6e93b GX |
53 | DBG_BUGON(1); |
54 | return -EFSCORRUPTED; | |
55 | } | |
ea0b7b0d | 56 | m->compressedblks = m->delta[0] & |
1c7f49a7 | 57 | ~Z_EROFS_LI_D0_CBLKCNT; |
cec6e93b GX |
58 | m->delta[0] = 1; |
59 | } | |
152a333a | 60 | m->delta[1] = le16_to_cpu(di->di_u.delta[1]); |
d6918942 GX |
61 | } else { |
62 | m->partialref = !!(advise & Z_EROFS_LI_PARTIAL_REF); | |
152a333a | 63 | m->clusterofs = le16_to_cpu(di->di_clusterofs); |
cc4efd3d GX |
64 | if (m->clusterofs >= 1 << vi->z_logical_clusterbits) { |
65 | DBG_BUGON(1); | |
66 | return -EFSCORRUPTED; | |
67 | } | |
152a333a | 68 | m->pblk = le32_to_cpu(di->di_u.blkaddr); |
152a333a | 69 | } |
152a333a GX |
70 | return 0; |
71 | } | |
72 | ||
73 | static unsigned int decode_compactedbits(unsigned int lobits, | |
152a333a GX |
74 | u8 *in, unsigned int pos, u8 *type) |
75 | { | |
76 | const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); | |
8d2517aa | 77 | const unsigned int lo = v & ((1 << lobits) - 1); |
152a333a GX |
78 | |
79 | *type = (v >> lobits) & 3; | |
80 | return lo; | |
81 | } | |
82 | ||
8d2517aa | 83 | static int get_compacted_la_distance(unsigned int lobits, |
d95ae5e2 GX |
84 | unsigned int encodebits, |
85 | unsigned int vcnt, u8 *in, int i) | |
86 | { | |
d95ae5e2 GX |
87 | unsigned int lo, d1 = 0; |
88 | u8 type; | |
89 | ||
90 | DBG_BUGON(i >= vcnt); | |
91 | ||
92 | do { | |
8d2517aa | 93 | lo = decode_compactedbits(lobits, in, encodebits * i, &type); |
d95ae5e2 | 94 | |
1c7f49a7 | 95 | if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) |
d95ae5e2 GX |
96 | return d1; |
97 | ++d1; | |
98 | } while (++i < vcnt); | |
99 | ||
1c7f49a7 GX |
100 | /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */ |
101 | if (!(lo & Z_EROFS_LI_D0_CBLKCNT)) | |
d95ae5e2 GX |
102 | d1 += lo - 1; |
103 | return d1; | |
104 | } | |
105 | ||
152a333a GX |
106 | static int unpack_compacted_index(struct z_erofs_maprecorder *m, |
107 | unsigned int amortizedshift, | |
ab92184f | 108 | erofs_off_t pos, bool lookahead) |
152a333a | 109 | { |
a5876e24 | 110 | struct erofs_inode *const vi = EROFS_I(m->inode); |
152a333a | 111 | const unsigned int lclusterbits = vi->z_logical_clusterbits; |
8d2517aa | 112 | unsigned int vcnt, base, lo, lobits, encodebits, nblk, eofs; |
152a333a GX |
113 | int i; |
114 | u8 *in, type; | |
b86269f4 | 115 | bool big_pcluster; |
152a333a | 116 | |
001b8ccd | 117 | if (1 << amortizedshift == 4 && lclusterbits <= 14) |
152a333a | 118 | vcnt = 2; |
8d2517aa | 119 | else if (1 << amortizedshift == 2 && lclusterbits <= 12) |
152a333a GX |
120 | vcnt = 16; |
121 | else | |
ff784a78 | 122 | return -EOPNOTSUPP; |
152a333a | 123 | |
ab92184f YH |
124 | /* it doesn't equal to round_up(..) */ |
125 | m->nextpackoff = round_down(pos, vcnt << amortizedshift) + | |
126 | (vcnt << amortizedshift); | |
b86269f4 | 127 | big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; |
8d2517aa | 128 | lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1U); |
152a333a | 129 | encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; |
3acea5fc | 130 | eofs = erofs_blkoff(m->inode->i_sb, pos); |
152a333a GX |
131 | base = round_down(eofs, vcnt << amortizedshift); |
132 | in = m->kaddr + base; | |
133 | ||
134 | i = (eofs - base) >> amortizedshift; | |
135 | ||
8d2517aa | 136 | lo = decode_compactedbits(lobits, in, encodebits * i, &type); |
152a333a | 137 | m->type = type; |
1c7f49a7 | 138 | if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { |
152a333a | 139 | m->clusterofs = 1 << lclusterbits; |
d95ae5e2 GX |
140 | |
141 | /* figure out lookahead_distance: delta[1] if needed */ | |
142 | if (lookahead) | |
8d2517aa | 143 | m->delta[1] = get_compacted_la_distance(lobits, |
d95ae5e2 | 144 | encodebits, vcnt, in, i); |
1c7f49a7 | 145 | if (lo & Z_EROFS_LI_D0_CBLKCNT) { |
b86269f4 GX |
146 | if (!big_pcluster) { |
147 | DBG_BUGON(1); | |
148 | return -EFSCORRUPTED; | |
149 | } | |
1c7f49a7 | 150 | m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; |
b86269f4 GX |
151 | m->delta[0] = 1; |
152 | return 0; | |
153 | } else if (i + 1 != (int)vcnt) { | |
152a333a GX |
154 | m->delta[0] = lo; |
155 | return 0; | |
156 | } | |
157 | /* | |
158 | * since the last lcluster in the pack is special, | |
159 | * of which lo saves delta[1] rather than delta[0]. | |
160 | * Hence, get delta[0] by the previous lcluster indirectly. | |
161 | */ | |
8d2517aa GX |
162 | lo = decode_compactedbits(lobits, in, |
163 | encodebits * (i - 1), &type); | |
1c7f49a7 | 164 | if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) |
152a333a | 165 | lo = 0; |
1c7f49a7 | 166 | else if (lo & Z_EROFS_LI_D0_CBLKCNT) |
b86269f4 | 167 | lo = 1; |
152a333a GX |
168 | m->delta[0] = lo + 1; |
169 | return 0; | |
170 | } | |
171 | m->clusterofs = lo; | |
172 | m->delta[0] = 0; | |
173 | /* figout out blkaddr (pblk) for HEAD lclusters */ | |
b86269f4 GX |
174 | if (!big_pcluster) { |
175 | nblk = 1; | |
176 | while (i > 0) { | |
177 | --i; | |
8d2517aa GX |
178 | lo = decode_compactedbits(lobits, in, |
179 | encodebits * i, &type); | |
1c7f49a7 | 180 | if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) |
b86269f4 GX |
181 | i -= lo; |
182 | ||
183 | if (i >= 0) | |
184 | ++nblk; | |
185 | } | |
186 | } else { | |
187 | nblk = 0; | |
188 | while (i > 0) { | |
189 | --i; | |
8d2517aa GX |
190 | lo = decode_compactedbits(lobits, in, |
191 | encodebits * i, &type); | |
1c7f49a7 GX |
192 | if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { |
193 | if (lo & Z_EROFS_LI_D0_CBLKCNT) { | |
b86269f4 | 194 | --i; |
1c7f49a7 | 195 | nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; |
b86269f4 GX |
196 | continue; |
197 | } | |
198 | /* bigpcluster shouldn't have plain d0 == 1 */ | |
199 | if (lo <= 1) { | |
200 | DBG_BUGON(1); | |
201 | return -EFSCORRUPTED; | |
202 | } | |
203 | i -= lo - 2; | |
204 | continue; | |
205 | } | |
152a333a | 206 | ++nblk; |
b86269f4 | 207 | } |
152a333a GX |
208 | } |
209 | in += (vcnt << amortizedshift) - sizeof(__le32); | |
210 | m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; | |
211 | return 0; | |
212 | } | |
213 | ||
8241fdd3 GX |
214 | static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, |
215 | unsigned long lcn, bool lookahead) | |
152a333a GX |
216 | { |
217 | struct inode *const inode = m->inode; | |
a5876e24 | 218 | struct erofs_inode *const vi = EROFS_I(inode); |
b780d3fc GX |
219 | const erofs_off_t ebase = sizeof(struct z_erofs_map_header) + |
220 | ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); | |
3acea5fc | 221 | unsigned int totalidx = erofs_iblks(inode); |
152a333a GX |
222 | unsigned int compacted_4b_initial, compacted_2b; |
223 | unsigned int amortizedshift; | |
224 | erofs_off_t pos; | |
152a333a | 225 | |
152a333a GX |
226 | if (lcn >= totalidx) |
227 | return -EINVAL; | |
228 | ||
229 | m->lcn = lcn; | |
230 | /* used to align to 32-byte (compacted_2b) alignment */ | |
231 | compacted_4b_initial = (32 - ebase % 32) / 4; | |
232 | if (compacted_4b_initial == 32 / 4) | |
233 | compacted_4b_initial = 0; | |
234 | ||
c40dd3ca YH |
235 | if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && |
236 | compacted_4b_initial < totalidx) | |
152a333a GX |
237 | compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); |
238 | else | |
239 | compacted_2b = 0; | |
240 | ||
241 | pos = ebase; | |
242 | if (lcn < compacted_4b_initial) { | |
243 | amortizedshift = 2; | |
244 | goto out; | |
245 | } | |
246 | pos += compacted_4b_initial * 4; | |
247 | lcn -= compacted_4b_initial; | |
248 | ||
249 | if (lcn < compacted_2b) { | |
250 | amortizedshift = 1; | |
251 | goto out; | |
252 | } | |
253 | pos += compacted_2b * 2; | |
254 | lcn -= compacted_2b; | |
255 | amortizedshift = 2; | |
256 | out: | |
257 | pos += lcn * (1 << amortizedshift); | |
31da107f | 258 | m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, |
e0981544 | 259 | erofs_pos(inode->i_sb, erofs_blknr(inode->i_sb, pos)), EROFS_KMAP); |
31da107f YH |
260 | if (IS_ERR(m->kaddr)) |
261 | return PTR_ERR(m->kaddr); | |
ab92184f | 262 | return unpack_compacted_index(m, amortizedshift, pos, lookahead); |
152a333a GX |
263 | } |
264 | ||
8241fdd3 GX |
265 | static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, |
266 | unsigned int lcn, bool lookahead) | |
152a333a | 267 | { |
8241fdd3 GX |
268 | switch (EROFS_I(m->inode)->datalayout) { |
269 | case EROFS_INODE_COMPRESSED_FULL: | |
270 | return z_erofs_load_full_lcluster(m, lcn); | |
271 | case EROFS_INODE_COMPRESSED_COMPACT: | |
272 | return z_erofs_load_compact_lcluster(m, lcn, lookahead); | |
273 | default: | |
274 | return -EINVAL; | |
275 | } | |
152a333a GX |
276 | } |
277 | ||
0c638f70 GX |
278 | static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, |
279 | unsigned int lookback_distance) | |
152a333a | 280 | { |
8241fdd3 | 281 | struct super_block *sb = m->inode->i_sb; |
a5876e24 | 282 | struct erofs_inode *const vi = EROFS_I(m->inode); |
152a333a | 283 | const unsigned int lclusterbits = vi->z_logical_clusterbits; |
152a333a | 284 | |
ab474fcc GX |
285 | while (m->lcn >= lookback_distance) { |
286 | unsigned long lcn = m->lcn - lookback_distance; | |
287 | int err; | |
288 | ||
8241fdd3 | 289 | err = z_erofs_load_lcluster_from_disk(m, lcn, false); |
ab474fcc GX |
290 | if (err) |
291 | return err; | |
292 | ||
293 | switch (m->type) { | |
1c7f49a7 | 294 | case Z_EROFS_LCLUSTER_TYPE_NONHEAD: |
ab474fcc | 295 | lookback_distance = m->delta[0]; |
8241fdd3 GX |
296 | if (!lookback_distance) |
297 | goto err_bogus; | |
ab474fcc | 298 | continue; |
1c7f49a7 GX |
299 | case Z_EROFS_LCLUSTER_TYPE_PLAIN: |
300 | case Z_EROFS_LCLUSTER_TYPE_HEAD1: | |
301 | case Z_EROFS_LCLUSTER_TYPE_HEAD2: | |
ab474fcc GX |
302 | m->headtype = m->type; |
303 | m->map->m_la = (lcn << lclusterbits) | m->clusterofs; | |
304 | return 0; | |
305 | default: | |
8241fdd3 | 306 | erofs_err(sb, "unknown type %u @ lcn %lu of nid %llu", |
ab474fcc | 307 | m->type, lcn, vi->nid); |
598bb891 | 308 | DBG_BUGON(1); |
ab474fcc | 309 | return -EOPNOTSUPP; |
598bb891 | 310 | } |
152a333a | 311 | } |
8241fdd3 GX |
312 | err_bogus: |
313 | erofs_err(sb, "bogus lookback distance %u @ lcn %lu of nid %llu", | |
314 | lookback_distance, m->lcn, vi->nid); | |
ab474fcc GX |
315 | DBG_BUGON(1); |
316 | return -EFSCORRUPTED; | |
152a333a GX |
317 | } |
318 | ||
cec6e93b GX |
319 | static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, |
320 | unsigned int initial_lcn) | |
321 | { | |
3acea5fc | 322 | struct super_block *sb = m->inode->i_sb; |
cec6e93b GX |
323 | struct erofs_inode *const vi = EROFS_I(m->inode); |
324 | struct erofs_map_blocks *const map = m->map; | |
325 | const unsigned int lclusterbits = vi->z_logical_clusterbits; | |
326 | unsigned long lcn; | |
327 | int err; | |
328 | ||
1c7f49a7 GX |
329 | DBG_BUGON(m->type != Z_EROFS_LCLUSTER_TYPE_PLAIN && |
330 | m->type != Z_EROFS_LCLUSTER_TYPE_HEAD1 && | |
331 | m->type != Z_EROFS_LCLUSTER_TYPE_HEAD2); | |
72bb5262 GX |
332 | DBG_BUGON(m->type != m->headtype); |
333 | ||
1c7f49a7 GX |
334 | if (m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || |
335 | ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1) && | |
72bb5262 | 336 | !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) || |
1c7f49a7 | 337 | ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && |
72bb5262 | 338 | !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { |
d467e980 | 339 | map->m_plen = 1ULL << lclusterbits; |
cec6e93b GX |
340 | return 0; |
341 | } | |
cec6e93b | 342 | lcn = m->lcn + 1; |
ea0b7b0d | 343 | if (m->compressedblks) |
cec6e93b | 344 | goto out; |
cec6e93b | 345 | |
8241fdd3 | 346 | err = z_erofs_load_lcluster_from_disk(m, lcn, false); |
cec6e93b GX |
347 | if (err) |
348 | return err; | |
349 | ||
0852b6ca GX |
350 | /* |
351 | * If the 1st NONHEAD lcluster has already been handled initially w/o | |
ea0b7b0d | 352 | * valid compressedblks, which means at least it mustn't be CBLKCNT, or |
0852b6ca GX |
353 | * an internal implemenatation error is detected. |
354 | * | |
355 | * The following code can also handle it properly anyway, but let's | |
356 | * BUG_ON in the debugging mode only for developers to notice that. | |
357 | */ | |
358 | DBG_BUGON(lcn == initial_lcn && | |
1c7f49a7 | 359 | m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); |
0852b6ca | 360 | |
cec6e93b | 361 | switch (m->type) { |
1c7f49a7 GX |
362 | case Z_EROFS_LCLUSTER_TYPE_PLAIN: |
363 | case Z_EROFS_LCLUSTER_TYPE_HEAD1: | |
364 | case Z_EROFS_LCLUSTER_TYPE_HEAD2: | |
0852b6ca GX |
365 | /* |
366 | * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type | |
367 | * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. | |
368 | */ | |
3acea5fc | 369 | m->compressedblks = 1 << (lclusterbits - sb->s_blocksize_bits); |
0852b6ca | 370 | break; |
1c7f49a7 | 371 | case Z_EROFS_LCLUSTER_TYPE_NONHEAD: |
cec6e93b GX |
372 | if (m->delta[0] != 1) |
373 | goto err_bonus_cblkcnt; | |
ea0b7b0d | 374 | if (m->compressedblks) |
cec6e93b GX |
375 | break; |
376 | fallthrough; | |
377 | default: | |
8241fdd3 GX |
378 | erofs_err(sb, "cannot found CBLKCNT @ lcn %lu of nid %llu", lcn, |
379 | vi->nid); | |
cec6e93b GX |
380 | DBG_BUGON(1); |
381 | return -EFSCORRUPTED; | |
382 | } | |
383 | out: | |
3acea5fc | 384 | map->m_plen = erofs_pos(sb, m->compressedblks); |
cec6e93b GX |
385 | return 0; |
386 | err_bonus_cblkcnt: | |
8241fdd3 | 387 | erofs_err(sb, "bogus CBLKCNT @ lcn %lu of nid %llu", lcn, vi->nid); |
cec6e93b GX |
388 | DBG_BUGON(1); |
389 | return -EFSCORRUPTED; | |
390 | } | |
391 | ||
d95ae5e2 GX |
392 | static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) |
393 | { | |
394 | struct inode *inode = m->inode; | |
395 | struct erofs_inode *vi = EROFS_I(inode); | |
396 | struct erofs_map_blocks *map = m->map; | |
397 | unsigned int lclusterbits = vi->z_logical_clusterbits; | |
398 | u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; | |
399 | int err; | |
400 | ||
401 | do { | |
402 | /* handle the last EOF pcluster (no next HEAD lcluster) */ | |
403 | if ((lcn << lclusterbits) >= inode->i_size) { | |
404 | map->m_llen = inode->i_size - map->m_la; | |
405 | return 0; | |
406 | } | |
407 | ||
8241fdd3 | 408 | err = z_erofs_load_lcluster_from_disk(m, lcn, true); |
d95ae5e2 GX |
409 | if (err) |
410 | return err; | |
411 | ||
1c7f49a7 | 412 | if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { |
d95ae5e2 GX |
413 | DBG_BUGON(!m->delta[1] && |
414 | m->clusterofs != 1 << lclusterbits); | |
1c7f49a7 GX |
415 | } else if (m->type == Z_EROFS_LCLUSTER_TYPE_PLAIN || |
416 | m->type == Z_EROFS_LCLUSTER_TYPE_HEAD1 || | |
417 | m->type == Z_EROFS_LCLUSTER_TYPE_HEAD2) { | |
d95ae5e2 GX |
418 | /* go on until the next HEAD lcluster */ |
419 | if (lcn != headlcn) | |
420 | break; | |
421 | m->delta[1] = 1; | |
422 | } else { | |
423 | erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", | |
424 | m->type, lcn, vi->nid); | |
425 | DBG_BUGON(1); | |
426 | return -EOPNOTSUPP; | |
427 | } | |
428 | lcn += m->delta[1]; | |
429 | } while (m->delta[1]); | |
430 | ||
431 | map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; | |
432 | return 0; | |
433 | } | |
434 | ||
ab92184f | 435 | static int z_erofs_do_map_blocks(struct inode *inode, |
1c7f49a7 | 436 | struct erofs_map_blocks *map, int flags) |
152a333a | 437 | { |
a5876e24 | 438 | struct erofs_inode *const vi = EROFS_I(inode); |
ab92184f | 439 | bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; |
b15b2e30 | 440 | bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; |
152a333a GX |
441 | struct z_erofs_maprecorder m = { |
442 | .inode = inode, | |
443 | .map = map, | |
444 | }; | |
445 | int err = 0; | |
118a8cf5 | 446 | unsigned int lclusterbits, endoff, afmt; |
cec6e93b | 447 | unsigned long initial_lcn; |
152a333a GX |
448 | unsigned long long ofs, end; |
449 | ||
152a333a | 450 | lclusterbits = vi->z_logical_clusterbits; |
ab92184f | 451 | ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; |
cec6e93b | 452 | initial_lcn = ofs >> lclusterbits; |
152a333a GX |
453 | endoff = ofs & ((1 << lclusterbits) - 1); |
454 | ||
8241fdd3 | 455 | err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); |
152a333a GX |
456 | if (err) |
457 | goto unmap_out; | |
458 | ||
ab92184f YH |
459 | if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) |
460 | vi->z_idataoff = m.nextpackoff; | |
461 | ||
8f899262 | 462 | map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; |
152a333a GX |
463 | end = (m.lcn + 1ULL) << lclusterbits; |
464 | ||
465 | switch (m.type) { | |
1c7f49a7 GX |
466 | case Z_EROFS_LCLUSTER_TYPE_PLAIN: |
467 | case Z_EROFS_LCLUSTER_TYPE_HEAD1: | |
468 | case Z_EROFS_LCLUSTER_TYPE_HEAD2: | |
152a333a | 469 | if (endoff >= m.clusterofs) { |
8f899262 | 470 | m.headtype = m.type; |
152a333a | 471 | map->m_la = (m.lcn << lclusterbits) | m.clusterofs; |
24331050 GX |
472 | /* |
473 | * For ztailpacking files, in order to inline data more | |
474 | * effectively, special EOF lclusters are now supported | |
475 | * which can have three parts at most. | |
476 | */ | |
477 | if (ztailpacking && end > inode->i_size) | |
478 | end = inode->i_size; | |
152a333a GX |
479 | break; |
480 | } | |
481 | /* m.lcn should be >= 1 if endoff < m.clusterofs */ | |
8d8a09b0 | 482 | if (!m.lcn) { |
4f761fa2 GX |
483 | erofs_err(inode->i_sb, |
484 | "invalid logical cluster 0 at nid %llu", | |
485 | vi->nid); | |
a6b9b1d5 | 486 | err = -EFSCORRUPTED; |
152a333a GX |
487 | goto unmap_out; |
488 | } | |
489 | end = (m.lcn << lclusterbits) | m.clusterofs; | |
b6a76183 | 490 | map->m_flags |= EROFS_MAP_FULL_MAPPED; |
152a333a | 491 | m.delta[0] = 1; |
df561f66 | 492 | fallthrough; |
1c7f49a7 | 493 | case Z_EROFS_LCLUSTER_TYPE_NONHEAD: |
fe6adcce | 494 | /* get the corresponding first chunk */ |
0c638f70 | 495 | err = z_erofs_extent_lookback(&m, m.delta[0]); |
8d8a09b0 | 496 | if (err) |
152a333a GX |
497 | goto unmap_out; |
498 | break; | |
499 | default: | |
4f761fa2 GX |
500 | erofs_err(inode->i_sb, |
501 | "unknown type %u @ offset %llu of nid %llu", | |
502 | m.type, ofs, vi->nid); | |
382329a9 | 503 | err = -EOPNOTSUPP; |
152a333a GX |
504 | goto unmap_out; |
505 | } | |
5c2a6425 GX |
506 | if (m.partialref) |
507 | map->m_flags |= EROFS_MAP_PARTIAL_REF; | |
152a333a | 508 | map->m_llen = end - map->m_la; |
152a333a | 509 | |
b15b2e30 | 510 | if (flags & EROFS_GET_BLOCKS_FINDTAIL) { |
ab92184f | 511 | vi->z_tailextent_headlcn = m.lcn; |
b15b2e30 | 512 | /* for non-compact indexes, fragmentoff is 64 bits */ |
8241fdd3 | 513 | if (fragment && vi->datalayout == EROFS_INODE_COMPRESSED_FULL) |
b15b2e30 YH |
514 | vi->z_fragmentoff |= (u64)m.pblk << 32; |
515 | } | |
ab92184f YH |
516 | if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { |
517 | map->m_flags |= EROFS_MAP_META; | |
518 | map->m_pa = vi->z_idataoff; | |
519 | map->m_plen = vi->z_idata_size; | |
b15b2e30 YH |
520 | } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { |
521 | map->m_flags |= EROFS_MAP_FRAGMENT; | |
ab92184f | 522 | } else { |
3acea5fc | 523 | map->m_pa = erofs_pos(inode->i_sb, m.pblk); |
ab92184f YH |
524 | err = z_erofs_get_extent_compressedlen(&m, initial_lcn); |
525 | if (err) | |
d5d188b8 | 526 | goto unmap_out; |
ab92184f | 527 | } |
d95ae5e2 | 528 | |
1c7f49a7 | 529 | if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { |
c505feba GX |
530 | if (map->m_llen > map->m_plen) { |
531 | DBG_BUGON(1); | |
532 | err = -EFSCORRUPTED; | |
533 | goto unmap_out; | |
534 | } | |
118a8cf5 GX |
535 | afmt = vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER ? |
536 | Z_EROFS_COMPRESSION_INTERLACED : | |
537 | Z_EROFS_COMPRESSION_SHIFTED; | |
fdffc091 | 538 | } else { |
118a8cf5 GX |
539 | afmt = m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2 ? |
540 | vi->z_algorithmtype[1] : vi->z_algorithmtype[0]; | |
541 | if (!(EROFS_I_SB(inode)->available_compr_algs & (1 << afmt))) { | |
542 | erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", | |
543 | afmt, vi->nid); | |
544 | err = -EFSCORRUPTED; | |
545 | goto unmap_out; | |
546 | } | |
fdffc091 | 547 | } |
118a8cf5 | 548 | map->m_algorithmformat = afmt; |
8f899262 | 549 | |
622ceadd GX |
550 | if ((flags & EROFS_GET_BLOCKS_FIEMAP) || |
551 | ((flags & EROFS_GET_BLOCKS_READMORE) && | |
ffa09b3b | 552 | (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA || |
7c35de4d GX |
553 | map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE || |
554 | map->m_algorithmformat == Z_EROFS_COMPRESSION_ZSTD) && | |
ffa09b3b | 555 | map->m_llen >= i_blocksize(inode))) { |
d95ae5e2 GX |
556 | err = z_erofs_get_extent_decompressedlen(&m); |
557 | if (!err) | |
558 | map->m_flags |= EROFS_MAP_FULL_MAPPED; | |
559 | } | |
d5d188b8 | 560 | |
152a333a | 561 | unmap_out: |
09c54379 | 562 | erofs_unmap_metabuf(&m.map->buf); |
ab92184f YH |
563 | return err; |
564 | } | |
565 | ||
999f2f9a GX |
566 | static int z_erofs_fill_inode_lazy(struct inode *inode) |
567 | { | |
568 | struct erofs_inode *const vi = EROFS_I(inode); | |
569 | struct super_block *const sb = inode->i_sb; | |
570 | int err, headnr; | |
571 | erofs_off_t pos; | |
572 | struct erofs_buf buf = __EROFS_BUF_INITIALIZER; | |
573 | void *kaddr; | |
574 | struct z_erofs_map_header *h; | |
575 | ||
576 | if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { | |
577 | /* | |
578 | * paired with smp_mb() at the end of the function to ensure | |
579 | * fields will only be observed after the bit is set. | |
580 | */ | |
581 | smp_mb(); | |
582 | return 0; | |
583 | } | |
584 | ||
585 | if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) | |
586 | return -ERESTARTSYS; | |
587 | ||
588 | err = 0; | |
589 | if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) | |
590 | goto out_unlock; | |
591 | ||
592 | pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); | |
e0981544 | 593 | kaddr = erofs_read_metabuf(&buf, sb, erofs_pos(sb, erofs_blknr(sb, pos)), EROFS_KMAP); |
999f2f9a GX |
594 | if (IS_ERR(kaddr)) { |
595 | err = PTR_ERR(kaddr); | |
596 | goto out_unlock; | |
597 | } | |
598 | ||
3acea5fc | 599 | h = kaddr + erofs_blkoff(sb, pos); |
999f2f9a GX |
600 | /* |
601 | * if the highest bit of the 8-byte map header is set, the whole file | |
602 | * is stored in the packed inode. The rest bits keeps z_fragmentoff. | |
603 | */ | |
604 | if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { | |
605 | vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; | |
606 | vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); | |
607 | vi->z_tailextent_headlcn = 0; | |
608 | goto done; | |
609 | } | |
610 | vi->z_advise = le16_to_cpu(h->h_advise); | |
611 | vi->z_algorithmtype[0] = h->h_algorithmtype & 15; | |
612 | vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; | |
613 | ||
614 | headnr = 0; | |
615 | if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || | |
616 | vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) { | |
617 | erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", | |
618 | headnr + 1, vi->z_algorithmtype[headnr], vi->nid); | |
619 | err = -EOPNOTSUPP; | |
620 | goto out_put_metabuf; | |
621 | } | |
622 | ||
3acea5fc | 623 | vi->z_logical_clusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 7); |
999f2f9a GX |
624 | if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && |
625 | vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | | |
626 | Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { | |
627 | erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", | |
628 | vi->nid); | |
629 | err = -EFSCORRUPTED; | |
630 | goto out_put_metabuf; | |
631 | } | |
1c7f49a7 | 632 | if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && |
999f2f9a GX |
633 | !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ |
634 | !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { | |
635 | erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", | |
636 | vi->nid); | |
637 | err = -EFSCORRUPTED; | |
638 | goto out_put_metabuf; | |
639 | } | |
640 | ||
641 | if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { | |
642 | struct erofs_map_blocks map = { | |
643 | .buf = __EROFS_BUF_INITIALIZER | |
644 | }; | |
645 | ||
646 | vi->z_idata_size = le16_to_cpu(h->h_idata_size); | |
647 | err = z_erofs_do_map_blocks(inode, &map, | |
648 | EROFS_GET_BLOCKS_FINDTAIL); | |
649 | erofs_put_metabuf(&map.buf); | |
650 | ||
651 | if (!map.m_plen || | |
3acea5fc | 652 | erofs_blkoff(sb, map.m_pa) + map.m_plen > sb->s_blocksize) { |
999f2f9a GX |
653 | erofs_err(sb, "invalid tail-packing pclustersize %llu", |
654 | map.m_plen); | |
655 | err = -EFSCORRUPTED; | |
656 | } | |
657 | if (err < 0) | |
658 | goto out_put_metabuf; | |
659 | } | |
660 | ||
661 | if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER && | |
662 | !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) { | |
663 | struct erofs_map_blocks map = { | |
664 | .buf = __EROFS_BUF_INITIALIZER | |
665 | }; | |
666 | ||
667 | vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); | |
668 | err = z_erofs_do_map_blocks(inode, &map, | |
669 | EROFS_GET_BLOCKS_FINDTAIL); | |
670 | erofs_put_metabuf(&map.buf); | |
671 | if (err < 0) | |
672 | goto out_put_metabuf; | |
673 | } | |
674 | done: | |
675 | /* paired with smp_mb() at the beginning of the function */ | |
676 | smp_mb(); | |
677 | set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); | |
678 | out_put_metabuf: | |
679 | erofs_put_metabuf(&buf); | |
680 | out_unlock: | |
681 | clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); | |
682 | return err; | |
683 | } | |
684 | ||
53a7f996 | 685 | int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, |
ab92184f YH |
686 | int flags) |
687 | { | |
b15b2e30 | 688 | struct erofs_inode *const vi = EROFS_I(inode); |
ab92184f YH |
689 | int err = 0; |
690 | ||
691 | trace_z_erofs_map_blocks_iter_enter(inode, map, flags); | |
692 | ||
693 | /* when trying to read beyond EOF, leave it unmapped */ | |
694 | if (map->m_la >= inode->i_size) { | |
695 | map->m_llen = map->m_la + 1 - inode->i_size; | |
696 | map->m_la = inode->i_size; | |
697 | map->m_flags = 0; | |
698 | goto out; | |
699 | } | |
700 | ||
701 | err = z_erofs_fill_inode_lazy(inode); | |
702 | if (err) | |
703 | goto out; | |
704 | ||
b15b2e30 YH |
705 | if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) && |
706 | !vi->z_tailextent_headlcn) { | |
707 | map->m_la = 0; | |
708 | map->m_llen = inode->i_size; | |
709 | map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | | |
710 | EROFS_MAP_FRAGMENT; | |
711 | goto out; | |
712 | } | |
713 | ||
ab92184f YH |
714 | err = z_erofs_do_map_blocks(inode, map, flags); |
715 | out: | |
152a333a | 716 | trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); |
152a333a GX |
717 | return err; |
718 | } | |
eadcd6b5 GX |
719 | |
720 | static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, | |
721 | loff_t length, unsigned int flags, | |
722 | struct iomap *iomap, struct iomap *srcmap) | |
723 | { | |
724 | int ret; | |
725 | struct erofs_map_blocks map = { .m_la = offset }; | |
726 | ||
727 | ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); | |
09c54379 | 728 | erofs_put_metabuf(&map.buf); |
eadcd6b5 GX |
729 | if (ret < 0) |
730 | return ret; | |
731 | ||
732 | iomap->bdev = inode->i_sb->s_bdev; | |
733 | iomap->offset = map.m_la; | |
734 | iomap->length = map.m_llen; | |
735 | if (map.m_flags & EROFS_MAP_MAPPED) { | |
736 | iomap->type = IOMAP_MAPPED; | |
b15b2e30 YH |
737 | iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ? |
738 | IOMAP_NULL_ADDR : map.m_pa; | |
eadcd6b5 GX |
739 | } else { |
740 | iomap->type = IOMAP_HOLE; | |
741 | iomap->addr = IOMAP_NULL_ADDR; | |
742 | /* | |
6acd87d5 SRP |
743 | * No strict rule on how to describe extents for post EOF, yet |
744 | * we need to do like below. Otherwise, iomap itself will get | |
eadcd6b5 | 745 | * into an endless loop on post EOF. |
6acd87d5 SRP |
746 | * |
747 | * Calculate the effective offset by subtracting extent start | |
748 | * (map.m_la) from the requested offset, and add it to length. | |
749 | * (NB: offset >= map.m_la always) | |
eadcd6b5 GX |
750 | */ |
751 | if (iomap->offset >= inode->i_size) | |
6acd87d5 | 752 | iomap->length = length + offset - map.m_la; |
eadcd6b5 GX |
753 | } |
754 | iomap->flags = 0; | |
755 | return 0; | |
756 | } | |
757 | ||
758 | const struct iomap_ops z_erofs_iomap_report_ops = { | |
759 | .iomap_begin = z_erofs_iomap_begin_report, | |
760 | }; |