Commit | Line | Data |
---|---|---|
29b24f6c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
152a333a | 2 | /* |
152a333a GX |
3 | * Copyright (C) 2018-2019 HUAWEI, Inc. |
4 | * http://www.huawei.com/ | |
5 | * Created by Gao Xiang <gaoxiang25@huawei.com> | |
6 | */ | |
7 | #include "internal.h" | |
8 | #include <asm/unaligned.h> | |
9 | #include <trace/events/erofs.h> | |
10 | ||
11 | int z_erofs_fill_inode(struct inode *inode) | |
12 | { | |
a5876e24 | 13 | struct erofs_inode *const vi = EROFS_I(inode); |
152a333a | 14 | |
8a765682 | 15 | if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { |
152a333a GX |
16 | vi->z_advise = 0; |
17 | vi->z_algorithmtype[0] = 0; | |
18 | vi->z_algorithmtype[1] = 0; | |
eace994a | 19 | vi->z_logical_clusterbits = LOG_BLOCK_SIZE; |
152a333a GX |
20 | vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits; |
21 | vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits; | |
a5876e24 | 22 | set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); |
152a333a GX |
23 | } |
24 | ||
0c638f70 | 25 | inode->i_mapping->a_ops = &z_erofs_aops; |
152a333a GX |
26 | return 0; |
27 | } | |
28 | ||
0c638f70 | 29 | static int z_erofs_fill_inode_lazy(struct inode *inode) |
152a333a | 30 | { |
a5876e24 | 31 | struct erofs_inode *const vi = EROFS_I(inode); |
152a333a GX |
32 | struct super_block *const sb = inode->i_sb; |
33 | int err; | |
34 | erofs_off_t pos; | |
35 | struct page *page; | |
36 | void *kaddr; | |
37 | struct z_erofs_map_header *h; | |
38 | ||
a5876e24 | 39 | if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) |
152a333a GX |
40 | return 0; |
41 | ||
a5876e24 | 42 | if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) |
152a333a GX |
43 | return -ERESTARTSYS; |
44 | ||
45 | err = 0; | |
a5876e24 | 46 | if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) |
152a333a GX |
47 | goto out_unlock; |
48 | ||
8a765682 | 49 | DBG_BUGON(vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); |
152a333a GX |
50 | |
51 | pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize + | |
52 | vi->xattr_isize, 8); | |
e655b5b3 | 53 | page = erofs_get_meta_page(sb, erofs_blknr(pos)); |
152a333a GX |
54 | if (IS_ERR(page)) { |
55 | err = PTR_ERR(page); | |
56 | goto out_unlock; | |
57 | } | |
58 | ||
59 | kaddr = kmap_atomic(page); | |
60 | ||
61 | h = kaddr + erofs_blkoff(pos); | |
62 | vi->z_advise = le16_to_cpu(h->h_advise); | |
63 | vi->z_algorithmtype[0] = h->h_algorithmtype & 15; | |
64 | vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; | |
65 | ||
66 | if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX) { | |
4f761fa2 GX |
67 | erofs_err(sb, "unknown compression format %u for nid %llu, please upgrade kernel", |
68 | vi->z_algorithmtype[0], vi->nid); | |
ff784a78 | 69 | err = -EOPNOTSUPP; |
152a333a GX |
70 | goto unmap_done; |
71 | } | |
72 | ||
73 | vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7); | |
74 | vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits + | |
75 | ((h->h_clusterbits >> 3) & 3); | |
76 | ||
77 | if (vi->z_physical_clusterbits[0] != LOG_BLOCK_SIZE) { | |
4f761fa2 GX |
78 | erofs_err(sb, "unsupported physical clusterbits %u for nid %llu, please upgrade kernel", |
79 | vi->z_physical_clusterbits[0], vi->nid); | |
ff784a78 | 80 | err = -EOPNOTSUPP; |
152a333a GX |
81 | goto unmap_done; |
82 | } | |
83 | ||
84 | vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits + | |
85 | ((h->h_clusterbits >> 5) & 7); | |
a5876e24 | 86 | set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); |
152a333a GX |
87 | unmap_done: |
88 | kunmap_atomic(kaddr); | |
89 | unlock_page(page); | |
90 | put_page(page); | |
152a333a | 91 | out_unlock: |
a5876e24 | 92 | clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); |
152a333a GX |
93 | return err; |
94 | } | |
95 | ||
96 | struct z_erofs_maprecorder { | |
97 | struct inode *inode; | |
98 | struct erofs_map_blocks *map; | |
99 | void *kaddr; | |
100 | ||
101 | unsigned long lcn; | |
102 | /* compression extent information gathered */ | |
103 | u8 type; | |
104 | u16 clusterofs; | |
105 | u16 delta[2]; | |
106 | erofs_blk_t pblk; | |
107 | }; | |
108 | ||
109 | static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, | |
110 | erofs_blk_t eblk) | |
111 | { | |
112 | struct super_block *const sb = m->inode->i_sb; | |
113 | struct erofs_map_blocks *const map = m->map; | |
114 | struct page *mpage = map->mpage; | |
115 | ||
116 | if (mpage) { | |
117 | if (mpage->index == eblk) { | |
118 | if (!m->kaddr) | |
119 | m->kaddr = kmap_atomic(mpage); | |
120 | return 0; | |
121 | } | |
122 | ||
123 | if (m->kaddr) { | |
124 | kunmap_atomic(m->kaddr); | |
125 | m->kaddr = NULL; | |
126 | } | |
127 | put_page(mpage); | |
128 | } | |
129 | ||
e655b5b3 | 130 | mpage = erofs_get_meta_page(sb, eblk); |
152a333a GX |
131 | if (IS_ERR(mpage)) { |
132 | map->mpage = NULL; | |
133 | return PTR_ERR(mpage); | |
134 | } | |
135 | m->kaddr = kmap_atomic(mpage); | |
136 | unlock_page(mpage); | |
137 | map->mpage = mpage; | |
138 | return 0; | |
139 | } | |
140 | ||
0c638f70 GX |
141 | static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, |
142 | unsigned long lcn) | |
152a333a GX |
143 | { |
144 | struct inode *const inode = m->inode; | |
a5876e24 | 145 | struct erofs_inode *const vi = EROFS_I(inode); |
152a333a GX |
146 | const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid); |
147 | const erofs_off_t pos = | |
148 | Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize + | |
149 | vi->xattr_isize) + | |
150 | lcn * sizeof(struct z_erofs_vle_decompressed_index); | |
151 | struct z_erofs_vle_decompressed_index *di; | |
152 | unsigned int advise, type; | |
153 | int err; | |
154 | ||
155 | err = z_erofs_reload_indexes(m, erofs_blknr(pos)); | |
156 | if (err) | |
157 | return err; | |
158 | ||
159 | m->lcn = lcn; | |
160 | di = m->kaddr + erofs_blkoff(pos); | |
161 | ||
162 | advise = le16_to_cpu(di->di_advise); | |
163 | type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) & | |
164 | ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1); | |
165 | switch (type) { | |
166 | case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: | |
167 | m->clusterofs = 1 << vi->z_logical_clusterbits; | |
168 | m->delta[0] = le16_to_cpu(di->di_u.delta[0]); | |
169 | m->delta[1] = le16_to_cpu(di->di_u.delta[1]); | |
170 | break; | |
171 | case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: | |
172 | case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: | |
173 | m->clusterofs = le16_to_cpu(di->di_clusterofs); | |
174 | m->pblk = le32_to_cpu(di->di_u.blkaddr); | |
175 | break; | |
176 | default: | |
177 | DBG_BUGON(1); | |
382329a9 | 178 | return -EOPNOTSUPP; |
152a333a GX |
179 | } |
180 | m->type = type; | |
181 | return 0; | |
182 | } | |
183 | ||
184 | static unsigned int decode_compactedbits(unsigned int lobits, | |
185 | unsigned int lomask, | |
186 | u8 *in, unsigned int pos, u8 *type) | |
187 | { | |
188 | const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); | |
189 | const unsigned int lo = v & lomask; | |
190 | ||
191 | *type = (v >> lobits) & 3; | |
192 | return lo; | |
193 | } | |
194 | ||
195 | static int unpack_compacted_index(struct z_erofs_maprecorder *m, | |
196 | unsigned int amortizedshift, | |
197 | unsigned int eofs) | |
198 | { | |
a5876e24 | 199 | struct erofs_inode *const vi = EROFS_I(m->inode); |
152a333a GX |
200 | const unsigned int lclusterbits = vi->z_logical_clusterbits; |
201 | const unsigned int lomask = (1 << lclusterbits) - 1; | |
202 | unsigned int vcnt, base, lo, encodebits, nblk; | |
203 | int i; | |
204 | u8 *in, type; | |
205 | ||
206 | if (1 << amortizedshift == 4) | |
207 | vcnt = 2; | |
208 | else if (1 << amortizedshift == 2 && lclusterbits == 12) | |
209 | vcnt = 16; | |
210 | else | |
ff784a78 | 211 | return -EOPNOTSUPP; |
152a333a GX |
212 | |
213 | encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; | |
214 | base = round_down(eofs, vcnt << amortizedshift); | |
215 | in = m->kaddr + base; | |
216 | ||
217 | i = (eofs - base) >> amortizedshift; | |
218 | ||
219 | lo = decode_compactedbits(lclusterbits, lomask, | |
220 | in, encodebits * i, &type); | |
221 | m->type = type; | |
222 | if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { | |
223 | m->clusterofs = 1 << lclusterbits; | |
224 | if (i + 1 != vcnt) { | |
225 | m->delta[0] = lo; | |
226 | return 0; | |
227 | } | |
228 | /* | |
229 | * since the last lcluster in the pack is special, | |
230 | * of which lo saves delta[1] rather than delta[0]. | |
231 | * Hence, get delta[0] by the previous lcluster indirectly. | |
232 | */ | |
233 | lo = decode_compactedbits(lclusterbits, lomask, | |
234 | in, encodebits * (i - 1), &type); | |
235 | if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) | |
236 | lo = 0; | |
237 | m->delta[0] = lo + 1; | |
238 | return 0; | |
239 | } | |
240 | m->clusterofs = lo; | |
241 | m->delta[0] = 0; | |
242 | /* figout out blkaddr (pblk) for HEAD lclusters */ | |
243 | nblk = 1; | |
244 | while (i > 0) { | |
245 | --i; | |
246 | lo = decode_compactedbits(lclusterbits, lomask, | |
247 | in, encodebits * i, &type); | |
248 | if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) | |
249 | i -= lo; | |
250 | ||
251 | if (i >= 0) | |
252 | ++nblk; | |
253 | } | |
254 | in += (vcnt << amortizedshift) - sizeof(__le32); | |
255 | m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; | |
256 | return 0; | |
257 | } | |
258 | ||
259 | static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, | |
260 | unsigned long lcn) | |
261 | { | |
262 | struct inode *const inode = m->inode; | |
a5876e24 | 263 | struct erofs_inode *const vi = EROFS_I(inode); |
152a333a GX |
264 | const unsigned int lclusterbits = vi->z_logical_clusterbits; |
265 | const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) + | |
266 | vi->inode_isize + vi->xattr_isize, 8) + | |
267 | sizeof(struct z_erofs_map_header); | |
268 | const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); | |
269 | unsigned int compacted_4b_initial, compacted_2b; | |
270 | unsigned int amortizedshift; | |
271 | erofs_off_t pos; | |
272 | int err; | |
273 | ||
274 | if (lclusterbits != 12) | |
ff784a78 | 275 | return -EOPNOTSUPP; |
152a333a GX |
276 | |
277 | if (lcn >= totalidx) | |
278 | return -EINVAL; | |
279 | ||
280 | m->lcn = lcn; | |
281 | /* used to align to 32-byte (compacted_2b) alignment */ | |
282 | compacted_4b_initial = (32 - ebase % 32) / 4; | |
283 | if (compacted_4b_initial == 32 / 4) | |
284 | compacted_4b_initial = 0; | |
285 | ||
286 | if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) | |
287 | compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); | |
288 | else | |
289 | compacted_2b = 0; | |
290 | ||
291 | pos = ebase; | |
292 | if (lcn < compacted_4b_initial) { | |
293 | amortizedshift = 2; | |
294 | goto out; | |
295 | } | |
296 | pos += compacted_4b_initial * 4; | |
297 | lcn -= compacted_4b_initial; | |
298 | ||
299 | if (lcn < compacted_2b) { | |
300 | amortizedshift = 1; | |
301 | goto out; | |
302 | } | |
303 | pos += compacted_2b * 2; | |
304 | lcn -= compacted_2b; | |
305 | amortizedshift = 2; | |
306 | out: | |
307 | pos += lcn * (1 << amortizedshift); | |
308 | err = z_erofs_reload_indexes(m, erofs_blknr(pos)); | |
309 | if (err) | |
310 | return err; | |
311 | return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos)); | |
312 | } | |
313 | ||
0c638f70 GX |
314 | static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, |
315 | unsigned int lcn) | |
152a333a | 316 | { |
a5876e24 | 317 | const unsigned int datamode = EROFS_I(m->inode)->datalayout; |
152a333a GX |
318 | |
319 | if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) | |
0c638f70 | 320 | return legacy_load_cluster_from_disk(m, lcn); |
152a333a GX |
321 | |
322 | if (datamode == EROFS_INODE_FLAT_COMPRESSION) | |
323 | return compacted_load_cluster_from_disk(m, lcn); | |
324 | ||
325 | return -EINVAL; | |
326 | } | |
327 | ||
0c638f70 GX |
328 | static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, |
329 | unsigned int lookback_distance) | |
152a333a | 330 | { |
a5876e24 | 331 | struct erofs_inode *const vi = EROFS_I(m->inode); |
152a333a GX |
332 | struct erofs_map_blocks *const map = m->map; |
333 | const unsigned int lclusterbits = vi->z_logical_clusterbits; | |
334 | unsigned long lcn = m->lcn; | |
335 | int err; | |
336 | ||
337 | if (lcn < lookback_distance) { | |
4f761fa2 GX |
338 | erofs_err(m->inode->i_sb, |
339 | "bogus lookback distance @ nid %llu", vi->nid); | |
152a333a | 340 | DBG_BUGON(1); |
a6b9b1d5 | 341 | return -EFSCORRUPTED; |
152a333a GX |
342 | } |
343 | ||
344 | /* load extent head logical cluster if needed */ | |
345 | lcn -= lookback_distance; | |
0c638f70 | 346 | err = z_erofs_load_cluster_from_disk(m, lcn); |
152a333a GX |
347 | if (err) |
348 | return err; | |
349 | ||
350 | switch (m->type) { | |
351 | case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: | |
8d8a09b0 | 352 | if (!m->delta[0]) { |
4f761fa2 GX |
353 | erofs_err(m->inode->i_sb, |
354 | "invalid lookback distance 0 @ nid %llu", | |
355 | vi->nid); | |
598bb891 GX |
356 | DBG_BUGON(1); |
357 | return -EFSCORRUPTED; | |
358 | } | |
0c638f70 | 359 | return z_erofs_extent_lookback(m, m->delta[0]); |
152a333a GX |
360 | case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: |
361 | map->m_flags &= ~EROFS_MAP_ZIPPED; | |
362 | /* fallthrough */ | |
363 | case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: | |
364 | map->m_la = (lcn << lclusterbits) | m->clusterofs; | |
365 | break; | |
366 | default: | |
4f761fa2 GX |
367 | erofs_err(m->inode->i_sb, |
368 | "unknown type %u @ lcn %lu of nid %llu", | |
369 | m->type, lcn, vi->nid); | |
152a333a | 370 | DBG_BUGON(1); |
382329a9 | 371 | return -EOPNOTSUPP; |
152a333a GX |
372 | } |
373 | return 0; | |
374 | } | |
375 | ||
376 | int z_erofs_map_blocks_iter(struct inode *inode, | |
377 | struct erofs_map_blocks *map, | |
378 | int flags) | |
379 | { | |
a5876e24 | 380 | struct erofs_inode *const vi = EROFS_I(inode); |
152a333a GX |
381 | struct z_erofs_maprecorder m = { |
382 | .inode = inode, | |
383 | .map = map, | |
384 | }; | |
385 | int err = 0; | |
386 | unsigned int lclusterbits, endoff; | |
387 | unsigned long long ofs, end; | |
388 | ||
389 | trace_z_erofs_map_blocks_iter_enter(inode, map, flags); | |
390 | ||
391 | /* when trying to read beyond EOF, leave it unmapped */ | |
8d8a09b0 | 392 | if (map->m_la >= inode->i_size) { |
152a333a GX |
393 | map->m_llen = map->m_la + 1 - inode->i_size; |
394 | map->m_la = inode->i_size; | |
395 | map->m_flags = 0; | |
396 | goto out; | |
397 | } | |
398 | ||
0c638f70 | 399 | err = z_erofs_fill_inode_lazy(inode); |
152a333a GX |
400 | if (err) |
401 | goto out; | |
402 | ||
403 | lclusterbits = vi->z_logical_clusterbits; | |
404 | ofs = map->m_la; | |
405 | m.lcn = ofs >> lclusterbits; | |
406 | endoff = ofs & ((1 << lclusterbits) - 1); | |
407 | ||
0c638f70 | 408 | err = z_erofs_load_cluster_from_disk(&m, m.lcn); |
152a333a GX |
409 | if (err) |
410 | goto unmap_out; | |
411 | ||
412 | map->m_flags = EROFS_MAP_ZIPPED; /* by default, compressed */ | |
413 | end = (m.lcn + 1ULL) << lclusterbits; | |
414 | ||
415 | switch (m.type) { | |
416 | case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: | |
417 | if (endoff >= m.clusterofs) | |
418 | map->m_flags &= ~EROFS_MAP_ZIPPED; | |
419 | /* fallthrough */ | |
420 | case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: | |
421 | if (endoff >= m.clusterofs) { | |
422 | map->m_la = (m.lcn << lclusterbits) | m.clusterofs; | |
423 | break; | |
424 | } | |
425 | /* m.lcn should be >= 1 if endoff < m.clusterofs */ | |
8d8a09b0 | 426 | if (!m.lcn) { |
4f761fa2 GX |
427 | erofs_err(inode->i_sb, |
428 | "invalid logical cluster 0 at nid %llu", | |
429 | vi->nid); | |
a6b9b1d5 | 430 | err = -EFSCORRUPTED; |
152a333a GX |
431 | goto unmap_out; |
432 | } | |
433 | end = (m.lcn << lclusterbits) | m.clusterofs; | |
b6a76183 | 434 | map->m_flags |= EROFS_MAP_FULL_MAPPED; |
152a333a GX |
435 | m.delta[0] = 1; |
436 | /* fallthrough */ | |
437 | case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: | |
438 | /* get the correspoinding first chunk */ | |
0c638f70 | 439 | err = z_erofs_extent_lookback(&m, m.delta[0]); |
8d8a09b0 | 440 | if (err) |
152a333a GX |
441 | goto unmap_out; |
442 | break; | |
443 | default: | |
4f761fa2 GX |
444 | erofs_err(inode->i_sb, |
445 | "unknown type %u @ offset %llu of nid %llu", | |
446 | m.type, ofs, vi->nid); | |
382329a9 | 447 | err = -EOPNOTSUPP; |
152a333a GX |
448 | goto unmap_out; |
449 | } | |
450 | ||
451 | map->m_llen = end - map->m_la; | |
452 | map->m_plen = 1 << lclusterbits; | |
453 | map->m_pa = blknr_to_addr(m.pblk); | |
454 | map->m_flags |= EROFS_MAP_MAPPED; | |
455 | ||
456 | unmap_out: | |
457 | if (m.kaddr) | |
458 | kunmap_atomic(m.kaddr); | |
459 | ||
460 | out: | |
4f761fa2 GX |
461 | erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", |
462 | __func__, map->m_la, map->m_pa, | |
463 | map->m_llen, map->m_plen, map->m_flags); | |
152a333a GX |
464 | |
465 | trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); | |
466 | ||
467 | /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ | |
468 | DBG_BUGON(err < 0 && err != -ENOMEM); | |
469 | return err; | |
470 | } | |
471 |